From noreply at buildbot.pypy.org  Sat Oct  1 00:00:38 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Sat,  1 Oct 2011 00:00:38 +0200 (CEST)
Subject: [pypy-commit] pypy lightweight-finalizers: fix translation. I don't
 think this case can happen in practice though, no
Message-ID: <20110930220038.8411482A01@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: lightweight-finalizers
Changeset: r47735:e1e51f303ca9
Date: 2011-09-30 19:00 -0300
http://bitbucket.org/pypy/pypy/changeset/e1e51f303ca9/

Log:	fix translation. I don't think this case can happen in practice
	though, no test.

diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -1687,7 +1687,7 @@
                 arena -= extra_words * WORD
                 allocsize += extra_words * WORD
             #
-            if self.has_raw_mem_ptr(obj):
+            if self.has_raw_mem_ptr(self.get_type_id(obj)):
                 self._free_raw_mem_from(obj)
             llarena.arena_free(arena)
             self.rawmalloced_total_size -= allocsize

From noreply at buildbot.pypy.org  Sat Oct  1 00:05:30 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Sat,  1 Oct 2011 00:05:30 +0200 (CEST)
Subject: [pypy-commit] pypy lightweight-finalizers: use the interface for
	tests
Message-ID: <20110930220530.6FFD682A01@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: lightweight-finalizers
Changeset: r47736:155952cafe97
Date: 2011-09-30 19:04 -0300
http://bitbucket.org/pypy/pypy/changeset/155952cafe97/

Log:	use the interface for tests

diff --git a/pypy/rlib/rsocket.py b/pypy/rlib/rsocket.py
--- a/pypy/rlib/rsocket.py
+++ b/pypy/rlib/rsocket.py
@@ -15,6 +15,7 @@
 # app-level code for PyPy.
 
 from pypy.rlib.objectmodel import instantiate, keepalive_until_here
+from pypy.rlib.rgc import owns_raw_memory
 from pypy.rlib import _rsocket_rffi as _c
 from pypy.rlib.rarithmetic import intmask
 from pypy.rpython.lltypesystem import lltype, rffi
@@ -55,6 +56,8 @@
 
 
 _FAMILIES = {}
+
+ at owns_raw_memory('addr_p')
 class Address(object):
     """The base class for RPython-level objects representing addresses.
     Fields:  addr    - a _c.sockaddr_ptr (memory owned by the Address instance)
@@ -75,11 +78,6 @@
         self.addr_p = addr
         self.addrlen = addrlen
 
-    def __del__(self):
-        addr = self.addr_p
-        if addr:
-            lltype.free(addr, flavor='raw')
-
     def setdata(self, addr, addrlen):
         # initialize self.addr and self.addrlen.  'addr' can be a different
         # pointer type than exactly sockaddr_ptr, and we cast it for you.

From notifications-noreply at bitbucket.org  Sat Oct  1 01:39:45 2011
From: notifications-noreply at bitbucket.org (Bitbucket)
Date: Fri, 30 Sep 2011 23:39:45 -0000
Subject: [pypy-commit] [COMMENT] Pull request #8 for pypy/pypy: added
 numpy.sort, tests, and docstring for numpy.array.sort
Message-ID: <20110930233945.2648.41995@bitbucket02.managed.contegix.com>

New comment on pull request:

https://bitbucket.org/pypy/pypy/pull-request/8/added-numpysort-tests-and-docstring-for#comment-250

yasirs said:

I abandoned it assuming these features should be added after merging the dtypes branch after reading Alex's comments. I'd still like to work on numpy development though.

--
This is a pull request comment notification from bitbucket.org.
You are receiving this either because you are participating
in a pull request, or you are following it.

From noreply at buildbot.pypy.org  Sat Oct  1 10:13:26 2011
From: noreply at buildbot.pypy.org (hakanardo)
Date: Sat,  1 Oct 2011 10:13:26 +0200 (CEST)
Subject: [pypy-commit] pypy jit-optimizeopt-cleanups: hg merge default
Message-ID: <20111001081326.DAE4A82A01@wyvern.cs.uni-duesseldorf.de>

Author: Hakan Ardo <hakan at debian.org>
Branch: jit-optimizeopt-cleanups
Changeset: r47737:2386583ea696
Date: 2011-10-01 09:41 +0200
http://bitbucket.org/pypy/pypy/changeset/2386583ea696/

Log:	hg merge default

diff --git a/dotviewer/graphparse.py b/dotviewer/graphparse.py
--- a/dotviewer/graphparse.py
+++ b/dotviewer/graphparse.py
@@ -36,48 +36,45 @@
     print >> sys.stderr, "Warning: could not guess file type, using 'dot'"
     return 'unknown'
 
-def dot2plain(content, contenttype, use_codespeak=False):
-    if contenttype == 'plain':
-        # already a .plain file
-        return content
+def dot2plain_graphviz(content, contenttype, use_codespeak=False):
+    if contenttype != 'neato':
+        cmdline = 'dot -Tplain'
+    else:
+        cmdline = 'neato -Tplain'
+    #print >> sys.stderr, '* running:', cmdline
+    close_fds = sys.platform != 'win32'
+    p = subprocess.Popen(cmdline, shell=True, close_fds=close_fds,
+                         stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+    (child_in, child_out) = (p.stdin, p.stdout)
+    try:
+        import thread
+    except ImportError:
+        bkgndwrite(child_in, content)
+    else:
+        thread.start_new_thread(bkgndwrite, (child_in, content))
+    plaincontent = child_out.read()
+    child_out.close()
+    if not plaincontent:    # 'dot' is likely not installed
+        raise PlainParseError("no result from running 'dot'")
+    return plaincontent
 
-    if not use_codespeak:
-        if contenttype != 'neato':
-            cmdline = 'dot -Tplain'
-        else:
-            cmdline = 'neato -Tplain'
-        #print >> sys.stderr, '* running:', cmdline
-        close_fds = sys.platform != 'win32'
-        p = subprocess.Popen(cmdline, shell=True, close_fds=close_fds,
-                             stdin=subprocess.PIPE, stdout=subprocess.PIPE)
-        (child_in, child_out) = (p.stdin, p.stdout)
-        try:
-            import thread
-        except ImportError:
-            bkgndwrite(child_in, content)
-        else:
-            thread.start_new_thread(bkgndwrite, (child_in, content))
-        plaincontent = child_out.read()
-        child_out.close()
-        if not plaincontent:    # 'dot' is likely not installed
-            raise PlainParseError("no result from running 'dot'")
-    else:
-        import urllib
-        request = urllib.urlencode({'dot': content})
-        url = 'http://codespeak.net/pypy/convertdot.cgi'
-        print >> sys.stderr, '* posting:', url
-        g = urllib.urlopen(url, data=request)
-        result = []
-        while True:
-            data = g.read(16384)
-            if not data:
-                break
-            result.append(data)
-        g.close()
-        plaincontent = ''.join(result)
-        # very simple-minded way to give a somewhat better error message
-        if plaincontent.startswith('<body'):
-            raise Exception("the dot on codespeak has very likely crashed")
+def dot2plain_codespeak(content, contenttype):
+    import urllib
+    request = urllib.urlencode({'dot': content})
+    url = 'http://codespeak.net/pypy/convertdot.cgi'
+    print >> sys.stderr, '* posting:', url
+    g = urllib.urlopen(url, data=request)
+    result = []
+    while True:
+        data = g.read(16384)
+        if not data:
+            break
+        result.append(data)
+    g.close()
+    plaincontent = ''.join(result)
+    # very simple-minded way to give a somewhat better error message
+    if plaincontent.startswith('<body'):
+        raise Exception("the dot on codespeak has very likely crashed")
     return plaincontent
 
 def bkgndwrite(f, data):
@@ -148,10 +145,13 @@
 
 def parse_dot(graph_id, content, links={}, fixedfont=False):
     contenttype = guess_type(content)
-    try:
-        plaincontent = dot2plain(content, contenttype, use_codespeak=False)
-        return list(parse_plain(graph_id, plaincontent, links, fixedfont))
-    except PlainParseError:
-        # failed, retry via codespeak
-        plaincontent = dot2plain(content, contenttype, use_codespeak=True)
-        return list(parse_plain(graph_id, plaincontent, links, fixedfont))
+    if contenttype == 'plain':
+        plaincontent = content
+    else:
+        try:
+            plaincontent = dot2plain_graphviz(content, contenttype)
+        except PlainParseError, e:
+            print e
+            # failed, retry via codespeak
+            plaincontent = dot2plain_codespeak(content, contenttype)
+    return list(parse_plain(graph_id, plaincontent, links, fixedfont))
diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -317,7 +317,7 @@
     RegrTest('test_multibytecodec.py', usemodules='_multibytecodec'),
     RegrTest('test_multibytecodec_support.py', skip="not a test"),
     RegrTest('test_multifile.py'),
-    RegrTest('test_multiprocessing.py', skip='FIXME leaves subprocesses'),
+    RegrTest('test_multiprocessing.py', skip="FIXME leaves subprocesses"),
     RegrTest('test_mutants.py', core="possibly"),
     RegrTest('test_mutex.py'),
     RegrTest('test_netrc.py'),
diff --git a/lib-python/modified-2.7/gzip.py b/lib-python/modified-2.7/gzip.py
deleted file mode 100644
--- a/lib-python/modified-2.7/gzip.py
+++ /dev/null
@@ -1,514 +0,0 @@
-"""Functions that read and write gzipped files.
-
-The user of the file doesn't have to worry about the compression,
-but random access is not allowed."""
-
-# based on Andrew Kuchling's minigzip.py distributed with the zlib module
-
-import struct, sys, time, os
-import zlib
-import io
-import __builtin__
-
-__all__ = ["GzipFile","open"]
-
-FTEXT, FHCRC, FEXTRA, FNAME, FCOMMENT = 1, 2, 4, 8, 16
-
-READ, WRITE = 1, 2
-
-def write32u(output, value):
-    # The L format writes the bit pattern correctly whether signed
-    # or unsigned.
-    output.write(struct.pack("<L", value))
-
-def read32(input):
-    return struct.unpack("<I", input.read(4))[0]
-
-def open(filename, mode="rb", compresslevel=9):
-    """Shorthand for GzipFile(filename, mode, compresslevel).
-
-    The filename argument is required; mode defaults to 'rb'
-    and compresslevel defaults to 9.
-
-    """
-    return GzipFile(filename, mode, compresslevel)
-
-class GzipFile(io.BufferedIOBase):
-    """The GzipFile class simulates most of the methods of a file object with
-    the exception of the readinto() and truncate() methods.
-
-    """
-
-    myfileobj = None
-    max_read_chunk = 10 * 1024 * 1024   # 10Mb
-
-    def __init__(self, filename=None, mode=None,
-                 compresslevel=9, fileobj=None, mtime=None):
-        """Constructor for the GzipFile class.
-
-        At least one of fileobj and filename must be given a
-        non-trivial value.
-
-        The new class instance is based on fileobj, which can be a regular
-        file, a StringIO object, or any other object which simulates a file.
-        It defaults to None, in which case filename is opened to provide
-        a file object.
-
-        When fileobj is not None, the filename argument is only used to be
-        included in the gzip file header, which may includes the original
-        filename of the uncompressed file.  It defaults to the filename of
-        fileobj, if discernible; otherwise, it defaults to the empty string,
-        and in this case the original filename is not included in the header.
-
-        The mode argument can be any of 'r', 'rb', 'a', 'ab', 'w', or 'wb',
-        depending on whether the file will be read or written.  The default
-        is the mode of fileobj if discernible; otherwise, the default is 'rb'.
-        Be aware that only the 'rb', 'ab', and 'wb' values should be used
-        for cross-platform portability.
-
-        The compresslevel argument is an integer from 1 to 9 controlling the
-        level of compression; 1 is fastest and produces the least compression,
-        and 9 is slowest and produces the most compression.  The default is 9.
-
-        The mtime argument is an optional numeric timestamp to be written
-        to the stream when compressing.  All gzip compressed streams
-        are required to contain a timestamp.  If omitted or None, the
-        current time is used.  This module ignores the timestamp when
-        decompressing; however, some programs, such as gunzip, make use
-        of it.  The format of the timestamp is the same as that of the
-        return value of time.time() and of the st_mtime member of the
-        object returned by os.stat().
-
-        """
-
-        # guarantee the file is opened in binary mode on platforms
-        # that care about that sort of thing
-        if mode and 'b' not in mode:
-            mode += 'b'
-        if fileobj is None:
-            fileobj = self.myfileobj = __builtin__.open(filename, mode or 'rb')
-        if filename is None:
-            if hasattr(fileobj, 'name'): filename = fileobj.name
-            else: filename = ''
-        if mode is None:
-            if hasattr(fileobj, 'mode'): mode = fileobj.mode
-            else: mode = 'rb'
-
-        if mode[0:1] == 'r':
-            self.mode = READ
-            # Set flag indicating start of a new member
-            self._new_member = True
-            # Buffer data read from gzip file. extrastart is offset in
-            # stream where buffer starts. extrasize is number of
-            # bytes remaining in buffer from current stream position.
-            self.extrabuf = ""
-            self.extrasize = 0
-            self.extrastart = 0
-            self.name = filename
-            # Starts small, scales exponentially
-            self.min_readsize = 100
-
-        elif mode[0:1] == 'w' or mode[0:1] == 'a':
-            self.mode = WRITE
-            self._init_write(filename)
-            self.compress = zlib.compressobj(compresslevel,
-                                             zlib.DEFLATED,
-                                             -zlib.MAX_WBITS,
-                                             zlib.DEF_MEM_LEVEL,
-                                             0)
-        else:
-            raise IOError, "Mode " + mode + " not supported"
-
-        self.fileobj = fileobj
-        self.offset = 0
-        self.mtime = mtime
-
-        if self.mode == WRITE:
-            self._write_gzip_header()
-
-    @property
-    def filename(self):
-        import warnings
-        warnings.warn("use the name attribute", DeprecationWarning, 2)
-        if self.mode == WRITE and self.name[-3:] != ".gz":
-            return self.name + ".gz"
-        return self.name
-
-    def __repr__(self):
-        s = repr(self.fileobj)
-        return '<gzip ' + s[1:-1] + ' ' + hex(id(self)) + '>'
-
-    def _check_closed(self):
-        """Raises a ValueError if the underlying file object has been closed.
-
-        """
-        if self.closed:
-            raise ValueError('I/O operation on closed file.')
-
-    def _init_write(self, filename):
-        self.name = filename
-        self.crc = zlib.crc32("") & 0xffffffffL
-        self.size = 0
-        self.writebuf = []
-        self.bufsize = 0
-
-    def _write_gzip_header(self):
-        self.fileobj.write('\037\213')             # magic header
-        self.fileobj.write('\010')                 # compression method
-        fname = os.path.basename(self.name)
-        if fname.endswith(".gz"):
-            fname = fname[:-3]
-        flags = 0
-        if fname:
-            flags = FNAME
-        self.fileobj.write(chr(flags))
-        mtime = self.mtime
-        if mtime is None:
-            mtime = time.time()
-        write32u(self.fileobj, long(mtime))
-        self.fileobj.write('\002')
-        self.fileobj.write('\377')
-        if fname:
-            self.fileobj.write(fname + '\000')
-
-    def _init_read(self):
-        self.crc = zlib.crc32("") & 0xffffffffL
-        self.size = 0
-
-    def _read_gzip_header(self):
-        magic = self.fileobj.read(2)
-        if magic != '\037\213':
-            raise IOError, 'Not a gzipped file'
-        method = ord( self.fileobj.read(1) )
-        if method != 8:
-            raise IOError, 'Unknown compression method'
-        flag = ord( self.fileobj.read(1) )
-        self.mtime = read32(self.fileobj)
-        # extraflag = self.fileobj.read(1)
-        # os = self.fileobj.read(1)
-        self.fileobj.read(2)
-
-        if flag & FEXTRA:
-            # Read & discard the extra field, if present
-            xlen = ord(self.fileobj.read(1))
-            xlen = xlen + 256*ord(self.fileobj.read(1))
-            self.fileobj.read(xlen)
-        if flag & FNAME:
-            # Read and discard a null-terminated string containing the filename
-            while True:
-                s = self.fileobj.read(1)
-                if not s or s=='\000':
-                    break
-        if flag & FCOMMENT:
-            # Read and discard a null-terminated string containing a comment
-            while True:
-                s = self.fileobj.read(1)
-                if not s or s=='\000':
-                    break
-        if flag & FHCRC:
-            self.fileobj.read(2)     # Read & discard the 16-bit header CRC
-
-    def write(self,data):
-        self._check_closed()
-        if self.mode != WRITE:
-            import errno
-            raise IOError(errno.EBADF, "write() on read-only GzipFile object")
-
-        if self.fileobj is None:
-            raise ValueError, "write() on closed GzipFile object"
-
-        # Convert data type if called by io.BufferedWriter.
-        if isinstance(data, memoryview):
-            data = data.tobytes()
-
-        if len(data) > 0:
-            self.size = self.size + len(data)
-            self.crc = zlib.crc32(data, self.crc) & 0xffffffffL
-            self.fileobj.write( self.compress.compress(data) )
-            self.offset += len(data)
-
-        return len(data)
-
-    def read(self, size=-1):
-        self._check_closed()
-        if self.mode != READ:
-            import errno
-            raise IOError(errno.EBADF, "read() on write-only GzipFile object")
-
-        if self.extrasize <= 0 and self.fileobj is None:
-            return ''
-
-        readsize = 1024
-        if size < 0:        # get the whole thing
-            try:
-                while True:
-                    self._read(readsize)
-                    readsize = min(self.max_read_chunk, readsize * 2)
-            except EOFError:
-                size = self.extrasize
-        elif size == 0:
-            return ""
-        else:               # just get some more of it
-            try:
-                while size > self.extrasize:
-                    self._read(readsize)
-                    readsize = min(self.max_read_chunk, readsize * 2)
-            except EOFError:
-                if size > self.extrasize:
-                    size = self.extrasize
-
-        offset = self.offset - self.extrastart
-        chunk = self.extrabuf[offset: offset + size]
-        self.extrasize = self.extrasize - size
-
-        self.offset += size
-        return chunk
-
-    def _unread(self, buf):
-        self.extrasize = len(buf) + self.extrasize
-        self.offset -= len(buf)
-
-    def _read(self, size=1024):
-        if self.fileobj is None:
-            raise EOFError, "Reached EOF"
-
-        if self._new_member:
-            # If the _new_member flag is set, we have to
-            # jump to the next member, if there is one.
-            #
-            # First, check if we're at the end of the file;
-            # if so, it's time to stop; no more members to read.
-            pos = self.fileobj.tell()   # Save current position
-            self.fileobj.seek(0, 2)     # Seek to end of file
-            if pos == self.fileobj.tell():
-                raise EOFError, "Reached EOF"
-            else:
-                self.fileobj.seek( pos ) # Return to original position
-
-            self._init_read()
-            self._read_gzip_header()
-            self.decompress = zlib.decompressobj(-zlib.MAX_WBITS)
-            self._new_member = False
-
-        # Read a chunk of data from the file
-        buf = self.fileobj.read(size)
-
-        # If the EOF has been reached, flush the decompression object
-        # and mark this object as finished.
-
-        if buf == "":
-            uncompress = self.decompress.flush()
-            self._read_eof()
-            self._add_read_data( uncompress )
-            raise EOFError, 'Reached EOF'
-
-        uncompress = self.decompress.decompress(buf)
-        self._add_read_data( uncompress )
-
-        if self.decompress.unused_data != "":
-            # Ending case: we've come to the end of a member in the file,
-            # so seek back to the start of the unused data, finish up
-            # this member, and read a new gzip header.
-            # (The number of bytes to seek back is the length of the unused
-            # data, minus 8 because _read_eof() will rewind a further 8 bytes)
-            self.fileobj.seek( -len(self.decompress.unused_data)+8, 1)
-
-            # Check the CRC and file size, and set the flag so we read
-            # a new member on the next call
-            self._read_eof()
-            self._new_member = True
-
-    def _add_read_data(self, data):
-        self.crc = zlib.crc32(data, self.crc) & 0xffffffffL
-        offset = self.offset - self.extrastart
-        self.extrabuf = self.extrabuf[offset:] + data
-        self.extrasize = self.extrasize + len(data)
-        self.extrastart = self.offset
-        self.size = self.size + len(data)
-
-    def _read_eof(self):
-        # We've read to the end of the file, so we have to rewind in order
-        # to reread the 8 bytes containing the CRC and the file size.
-        # We check the that the computed CRC and size of the
-        # uncompressed data matches the stored values.  Note that the size
-        # stored is the true file size mod 2**32.
-        self.fileobj.seek(-8, 1)
-        crc32 = read32(self.fileobj)
-        isize = read32(self.fileobj)  # may exceed 2GB
-        if crc32 != self.crc:
-            raise IOError("CRC check failed %s != %s" % (hex(crc32),
-                                                         hex(self.crc)))
-        elif isize != (self.size & 0xffffffffL):
-            raise IOError, "Incorrect length of data produced"
-
-        # Gzip files can be padded with zeroes and still have archives.
-        # Consume all zero bytes and set the file position to the first
-        # non-zero byte. See http://www.gzip.org/#faq8
-        c = "\x00"
-        while c == "\x00":
-            c = self.fileobj.read(1)
-        if c:
-            self.fileobj.seek(-1, 1)
-
-    @property
-    def closed(self):
-        return self.fileobj is None
-
-    def close(self):
-        if self.fileobj is None:
-            return
-        if self.mode == WRITE:
-            self.fileobj.write(self.compress.flush())
-            write32u(self.fileobj, self.crc)
-            # self.size may exceed 2GB, or even 4GB
-            write32u(self.fileobj, self.size & 0xffffffffL)
-            self.fileobj = None
-        elif self.mode == READ:
-            self.fileobj = None
-        if self.myfileobj:
-            self.myfileobj.close()
-            self.myfileobj = None
-
-    def flush(self,zlib_mode=zlib.Z_SYNC_FLUSH):
-        self._check_closed()
-        if self.mode == WRITE:
-            # Ensure the compressor's buffer is flushed
-            self.fileobj.write(self.compress.flush(zlib_mode))
-            self.fileobj.flush()
-
-    def fileno(self):
-        """Invoke the underlying file object's fileno() method.
-
-        This will raise AttributeError if the underlying file object
-        doesn't support fileno().
-        """
-        return self.fileobj.fileno()
-
-    def rewind(self):
-        '''Return the uncompressed stream file position indicator to the
-        beginning of the file'''
-        if self.mode != READ:
-            raise IOError("Can't rewind in write mode")
-        self.fileobj.seek(0)
-        self._new_member = True
-        self.extrabuf = ""
-        self.extrasize = 0
-        self.extrastart = 0
-        self.offset = 0
-
-    def readable(self):
-        return self.mode == READ
-
-    def writable(self):
-        return self.mode == WRITE
-
-    def seekable(self):
-        return True
-
-    def seek(self, offset, whence=0):
-        if whence:
-            if whence == 1:
-                offset = self.offset + offset
-            else:
-                raise ValueError('Seek from end not supported')
-        if self.mode == WRITE:
-            if offset < self.offset:
-                raise IOError('Negative seek in write mode')
-            count = offset - self.offset
-            for i in range(count // 1024):
-                self.write(1024 * '\0')
-            self.write((count % 1024) * '\0')
-        elif self.mode == READ:
-            if offset == self.offset:
-                self.read(0) # to make sure that this file is open
-                return self.offset
-            if offset < self.offset:
-                # for negative seek, rewind and do positive seek
-                self.rewind()
-            count = offset - self.offset
-            for i in range(count // 1024):
-                self.read(1024)
-            self.read(count % 1024)
-
-        return self.offset
-
-    def readline(self, size=-1):
-        if size < 0:
-            # Shortcut common case - newline found in buffer.
-            offset = self.offset - self.extrastart
-            i = self.extrabuf.find('\n', offset) + 1
-            if i > 0:
-                self.extrasize -= i - offset
-                self.offset += i - offset
-                return self.extrabuf[offset: i]
-
-            size = sys.maxint
-            readsize = self.min_readsize
-        else:
-            readsize = size
-        bufs = []
-        while size != 0:
-            c = self.read(readsize)
-            i = c.find('\n')
-
-            # We set i=size to break out of the loop under two
-            # conditions: 1) there's no newline, and the chunk is
-            # larger than size, or 2) there is a newline, but the
-            # resulting line would be longer than 'size'.
-            if (size <= i) or (i == -1 and len(c) > size):
-                i = size - 1
-
-            if i >= 0 or c == '':
-                bufs.append(c[:i + 1])    # Add portion of last chunk
-                self._unread(c[i + 1:])   # Push back rest of chunk
-                break
-
-            # Append chunk to list, decrease 'size',
-            bufs.append(c)
-            size = size - len(c)
-            readsize = min(size, readsize * 2)
-        if readsize > self.min_readsize:
-            self.min_readsize = min(readsize, self.min_readsize * 2, 512)
-        return ''.join(bufs) # Return resulting line
-
-
-def _test():
-    # Act like gzip; with -d, act like gunzip.
-    # The input file is not deleted, however, nor are any other gzip
-    # options or features supported.
-    args = sys.argv[1:]
-    decompress = args and args[0] == "-d"
-    if decompress:
-        args = args[1:]
-    if not args:
-        args = ["-"]
-    for arg in args:
-        if decompress:
-            if arg == "-":
-                f = GzipFile(filename="", mode="rb", fileobj=sys.stdin)
-                g = sys.stdout
-            else:
-                if arg[-3:] != ".gz":
-                    print "filename doesn't end in .gz:", repr(arg)
-                    continue
-                f = open(arg, "rb")
-                g = __builtin__.open(arg[:-3], "wb")
-        else:
-            if arg == "-":
-                f = sys.stdin
-                g = GzipFile(filename="", mode="wb", fileobj=sys.stdout)
-            else:
-                f = __builtin__.open(arg, "rb")
-                g = open(arg + ".gz", "wb")
-        while True:
-            chunk = f.read(1024)
-            if not chunk:
-                break
-            g.write(chunk)
-        if g is not sys.stdout:
-            g.close()
-        if f is not sys.stdin:
-            f.close()
-
-if __name__ == '__main__':
-    _test()
diff --git a/lib-python/modified-2.7/tarfile.py b/lib-python/modified-2.7/tarfile.py
--- a/lib-python/modified-2.7/tarfile.py
+++ b/lib-python/modified-2.7/tarfile.py
@@ -252,8 +252,8 @@
        the high bit set. So we calculate two checksums, unsigned and
        signed.
     """
-    unsigned_chksum = 256 + sum(struct.unpack("148B8x356B", buf[:512]))
-    signed_chksum = 256 + sum(struct.unpack("148b8x356b", buf[:512]))
+    unsigned_chksum = 256 + sum(struct.unpack("148B", buf[:148]) + struct.unpack("356B", buf[156:512]))
+    signed_chksum = 256 + sum(struct.unpack("148b", buf[:148]) + struct.unpack("356b", buf[156:512]))
     return unsigned_chksum, signed_chksum
 
 def copyfileobj(src, dst, length=None):
@@ -265,6 +265,7 @@
     if length is None:
         shutil.copyfileobj(src, dst)
         return
+
     BUFSIZE = 16 * 1024
     blocks, remainder = divmod(length, BUFSIZE)
     for b in xrange(blocks):
@@ -801,19 +802,19 @@
         if self.closed:
             raise ValueError("I/O operation on closed file")
 
+        buf = ""
         if self.buffer:
             if size is None:
-                buf = self.buffer + self.fileobj.read()
+                buf = self.buffer
                 self.buffer = ""
             else:
                 buf = self.buffer[:size]
                 self.buffer = self.buffer[size:]
-                buf += self.fileobj.read(size - len(buf))
+
+        if size is None:
+            buf += self.fileobj.read()
         else:
-            if size is None:
-                buf = self.fileobj.read()
-            else:
-                buf = self.fileobj.read(size)
+            buf += self.fileobj.read(size - len(buf))
 
         self.position += len(buf)
         return buf
diff --git a/lib-python/modified-2.7/test/test_multiprocessing.py b/lib-python/modified-2.7/test/test_multiprocessing.py
--- a/lib-python/modified-2.7/test/test_multiprocessing.py
+++ b/lib-python/modified-2.7/test/test_multiprocessing.py
@@ -510,7 +510,6 @@
 
         p.join()
 
-    @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
     def test_qsize(self):
         q = self.Queue()
         try:
@@ -532,7 +531,6 @@
             time.sleep(DELTA)
             q.task_done()
 
-    @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
     def test_task_done(self):
         queue = self.JoinableQueue()
 
@@ -1091,7 +1089,6 @@
 class _TestPoolWorkerLifetime(BaseTestCase):
 
     ALLOWED_TYPES = ('processes', )
-    @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
     def test_pool_worker_lifetime(self):
         p = multiprocessing.Pool(3, maxtasksperchild=10)
         self.assertEqual(3, len(p._pool))
@@ -1280,7 +1277,6 @@
         queue = manager.get_queue()
         queue.put('hello world')
 
-    @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
     def test_rapid_restart(self):
         authkey = os.urandom(32)
         manager = QueueManager(
@@ -1297,6 +1293,7 @@
         queue = manager.get_queue()
         self.assertEqual(queue.get(), 'hello world')
         del queue
+        test_support.gc_collect()
         manager.shutdown()
         manager = QueueManager(
             address=addr, authkey=authkey, serializer=SERIALIZER)
@@ -1573,7 +1570,6 @@
 
     ALLOWED_TYPES = ('processes',)
 
-    @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
     def test_heap(self):
         iterations = 5000
         maxblocks = 50
diff --git a/lib_pypy/_functools.py b/lib_pypy/_functools.py
--- a/lib_pypy/_functools.py
+++ b/lib_pypy/_functools.py
@@ -18,5 +18,5 @@
 
     def __call__(self, *fargs, **fkeywords):
         if self.keywords is not None:
-            fkeywords.update(self.keywords)
+            fkeywords = dict(self.keywords, **fkeywords)
         return self.func(*(self.args + fargs), **fkeywords)
diff --git a/lib_pypy/greenlet.py b/lib_pypy/greenlet.py
--- a/lib_pypy/greenlet.py
+++ b/lib_pypy/greenlet.py
@@ -96,7 +96,16 @@
 
     @property
     def gr_frame(self):
-        raise NotImplementedError("attribute 'gr_frame' of greenlet objects")
+        # xxx this doesn't work when called on either the current or
+        # the main greenlet of another thread
+        if self is getcurrent():
+            return None
+        if self.__main:
+            self = getcurrent()
+        f = _continulet.__reduce__(self)[2][0]
+        if not f:
+            return None
+        return f.f_back.f_back.f_back   # go past start(), __switch(), switch()
 
 # ____________________________________________________________
 # Internal stuff
diff --git a/lib_pypy/pypy_test/test_stackless_pickling.py b/lib_pypy/pypy_test/test_stackless_pickling.py
--- a/lib_pypy/pypy_test/test_stackless_pickling.py
+++ b/lib_pypy/pypy_test/test_stackless_pickling.py
@@ -1,7 +1,3 @@
-"""
-this test should probably not run from CPython or py.py.
-I'm not entirely sure, how to do that.
-"""
 from __future__ import absolute_import
 from py.test import skip
 try:
@@ -16,11 +12,15 @@
 
 class Test_StacklessPickling:
 
+    def test_pickle_main_coroutine(self):
+        import stackless, pickle
+        s = pickle.dumps(stackless.coroutine.getcurrent())
+        print s
+        c = pickle.loads(s)
+        assert c is stackless.coroutine.getcurrent()
+
     def test_basic_tasklet_pickling(self):
-        try:
-            import stackless
-        except ImportError:
-            skip("can't load stackless and don't know why!!!")
+        import stackless
         from stackless import run, schedule, tasklet
         import pickle
 
diff --git a/lib_pypy/pyrepl/completing_reader.py b/lib_pypy/pyrepl/completing_reader.py
--- a/lib_pypy/pyrepl/completing_reader.py
+++ b/lib_pypy/pyrepl/completing_reader.py
@@ -229,7 +229,8 @@
 
     def after_command(self, cmd):
         super(CompletingReader, self).after_command(cmd)
-        if not isinstance(cmd, complete) and not isinstance(cmd, self_insert):
+        if not isinstance(cmd, self.commands['complete']) \
+           and not isinstance(cmd, self.commands['self_insert']):
             self.cmpltn_reset()
 
     def calc_screen(self):
diff --git a/lib_pypy/stackless.py b/lib_pypy/stackless.py
--- a/lib_pypy/stackless.py
+++ b/lib_pypy/stackless.py
@@ -5,7 +5,6 @@
 """
 
 
-import traceback
 import _continuation
 
 class TaskletExit(Exception):
@@ -14,33 +13,46 @@
 CoroutineExit = TaskletExit
 
 
+def _coroutine_getcurrent():
+    "Returns the current coroutine (i.e. the one which called this function)."
+    try:
+        return _tls.current_coroutine
+    except AttributeError:
+        # first call in this thread: current == main
+        return _coroutine_getmain()
+
+def _coroutine_getmain():
+    try:
+        return _tls.main_coroutine
+    except AttributeError:
+        # create the main coroutine for this thread
+        continulet = _continuation.continulet
+        main = coroutine()
+        main._frame = continulet.__new__(continulet)
+        main._is_started = -1
+        _tls.current_coroutine = _tls.main_coroutine = main
+        return _tls.main_coroutine
+
+
 class coroutine(object):
-    "we can't have continulet as a base, because continulets can't be rebound"
+    _is_started = 0      # 0=no, 1=yes, -1=main
 
     def __init__(self):
         self._frame = None
-        self.is_zombie = False
-
-    def __getattr__(self, attr):
-        return getattr(self._frame, attr)
-
-    def __del__(self):
-        self.is_zombie = True
-        del self._frame
-        self._frame = None
 
     def bind(self, func, *argl, **argd):
         """coro.bind(f, *argl, **argd) -> None.
            binds function f to coro. f will be called with
            arguments *argl, **argd
         """
-        if self._frame is None or not self._frame.is_pending():
-            def run(c):
-                _tls.current_coroutine = self
-                return func(*argl, **argd)
-            self._frame = frame = _continuation.continulet(run)
-        else:
+        if self.is_alive:
             raise ValueError("cannot bind a bound coroutine")
+        def run(c):
+            _tls.current_coroutine = self
+            self._is_started = 1
+            return func(*argl, **argd)
+        self._is_started = 0
+        self._frame = _continuation.continulet(run)
 
     def switch(self):
         """coro.switch() -> returnvalue
@@ -48,7 +60,7 @@
            f finishes, the returnvalue is that of f, otherwise
            None is returned
         """
-        current = _getcurrent()
+        current = _coroutine_getcurrent()
         try:
             current._frame.switch(to=self._frame)
         finally:
@@ -56,37 +68,30 @@
 
     def kill(self):
         """coro.kill() : kill coroutine coro"""
-        current = _getcurrent()
+        current = _coroutine_getcurrent()
         try:
             current._frame.throw(CoroutineExit, to=self._frame)
         finally:
             _tls.current_coroutine = current
 
-    def _is_alive(self):
-        if self._frame is None:
-            return False
-        return not self._frame.is_pending()
-    is_alive = property(_is_alive)
-    del _is_alive
+    @property
+    def is_alive(self):
+        return self._is_started < 0 or (
+            self._frame is not None and self._frame.is_pending())
 
-    def getcurrent():
-        """coroutine.getcurrent() -> the currently running coroutine"""
-        return _getcurrent()
-    getcurrent = staticmethod(getcurrent)
+    @property
+    def is_zombie(self):
+        return self._is_started > 0 and not self._frame.is_pending()
+
+    getcurrent = staticmethod(_coroutine_getcurrent)
 
     def __reduce__(self):
-        raise TypeError, 'pickling is not possible based upon continulets'
+        if self._is_started < 0:
+            return _coroutine_getmain, ()
+        else:
+            return type(self), (), self.__dict__
 
 
-def _getcurrent():
-    "Returns the current coroutine (i.e. the one which called this function)."
-    try:
-        return _tls.current_coroutine
-    except AttributeError:
-        # first call in this thread: current == main
-        _coroutine_create_main()
-        return _tls.current_coroutine
-
 try:
     from thread import _local
 except ImportError:
@@ -95,14 +100,8 @@
 
 _tls = _local()
 
-def _coroutine_create_main():
-    # create the main coroutine for this thread
-    _tls.current_coroutine = None
-    main_coroutine = coroutine()
-    typ = _continuation.continulet
-    main_coroutine._frame = typ.__new__(typ)
-    _tls.main_coroutine = main_coroutine
-    _tls.current_coroutine = main_coroutine
+
+# ____________________________________________________________
 
 
 from collections import deque
@@ -148,10 +147,7 @@
     _last_task = next
     assert not next.blocked
     if next is not current:
-        #try:
-            next.switch()
-        #except CoroutineExit:  --- they are the same anyway
-        #    raise TaskletExit
+        next.switch()
     return current
 
 def set_schedule_callback(callback):
@@ -175,34 +171,6 @@
         raise self.type, self.value, self.traceback
 
 #
-# helpers for pickling
-#
-
-_stackless_primitive_registry = {}
-
-def register_stackless_primitive(thang, retval_expr='None'):
-    import types
-    func = thang
-    if isinstance(thang, types.MethodType):
-        func = thang.im_func
-    code = func.func_code
-    _stackless_primitive_registry[code] = retval_expr
-    # It is not too nice to attach info via the code object, but
-    # I can't think of a better solution without a real transform.
-
-def rewrite_stackless_primitive(coro_state, alive, tempval):
-    flags, frame, thunk, parent = coro_state
-    while frame is not None:
-        retval_expr = _stackless_primitive_registry.get(frame.f_code)
-        if retval_expr:
-            # this tasklet needs to stop pickling here and return its value.
-            tempval = eval(retval_expr, globals(), frame.f_locals)
-            coro_state = flags, frame, thunk, parent
-            break
-        frame = frame.f_back
-    return coro_state, alive, tempval
-
-#
 #
 
 class channel(object):
@@ -354,8 +322,6 @@
         """
         return self._channel_action(None, -1)
 
-    register_stackless_primitive(receive, retval_expr='receiver.tempval')
-
     def send_exception(self, exp_type, msg):
         self.send(bomb(exp_type, exp_type(msg)))
 
@@ -372,9 +338,8 @@
         the runnables list.
         """
         return self._channel_action(msg, 1)
-            
-    register_stackless_primitive(send)
-            
+
+
 class tasklet(coroutine):
     """
     A tasklet object represents a tiny task in a Python thread.
@@ -456,7 +421,7 @@
 
         self.func = None
         coroutine.bind(self, _func)
-        back = _getcurrent()
+        back = _coroutine_getcurrent()
         coroutine.switch(self)
         self.alive = True
         _scheduler_append(self)
@@ -480,39 +445,6 @@
             raise RuntimeError, "The current tasklet cannot be removed."
             # not sure if I will revive this  " Use t=tasklet().capture()"
         _scheduler_remove(self)
-        
-    def __reduce__(self):
-        one, two, coro_state = coroutine.__reduce__(self)
-        assert one is coroutine
-        assert two == ()
-        # we want to get rid of the parent thing.
-        # for now, we just drop it
-        a, frame, c, d = coro_state
-
-        # Removing all frames related to stackless.py.
-        # They point to stuff we don't want to be pickled.
-
-        pickleframe = frame
-        while frame is not None:
-            if frame.f_code == schedule.func_code:
-                # Removing everything including and after the
-                # call to stackless.schedule()
-                pickleframe = frame.f_back
-                break
-            frame = frame.f_back
-        if d:
-            assert isinstance(d, coroutine)
-        coro_state = a, pickleframe, c, None
-        coro_state, alive, tempval = rewrite_stackless_primitive(coro_state, self.alive, self.tempval)
-        inst_dict = self.__dict__.copy()
-        inst_dict.pop('tempval', None)
-        return self.__class__, (), (coro_state, alive, tempval, inst_dict)
-
-    def __setstate__(self, (coro_state, alive, tempval, inst_dict)):
-        coroutine.__setstate__(self, coro_state)
-        self.__dict__.update(inst_dict)
-        self.alive = alive
-        self.tempval = tempval
 
 def getmain():
     """
@@ -601,30 +533,7 @@
     global _last_task
     _global_task_id = 0
     _main_tasklet = coroutine.getcurrent()
-    try:
-        _main_tasklet.__class__ = tasklet
-    except TypeError: # we are running pypy-c
-        class TaskletProxy(object):
-            """TaskletProxy is needed to give the _main_coroutine tasklet behaviour"""
-            def __init__(self, coro):
-                self._coro = coro
-
-            def __getattr__(self,attr):
-                return getattr(self._coro,attr)
-
-            def __str__(self):
-                return '<tasklet %s a:%s>' % (self._task_id, self.is_alive)
-
-            def __reduce__(self):
-                return getmain, ()
-
-            __repr__ = __str__
-
-
-        global _main_coroutine
-        _main_coroutine = _main_tasklet
-        _main_tasklet = TaskletProxy(_main_tasklet)
-        assert _main_tasklet.is_alive and not _main_tasklet.is_zombie
+    _main_tasklet.__class__ = tasklet         # XXX HAAAAAAAAAAAAAAAAAAAAACK
     _last_task = _main_tasklet
     tasklet._init.im_func(_main_tasklet, label='main')
     _squeue = deque()
diff --git a/py/_code/source.py b/py/_code/source.py
--- a/py/_code/source.py
+++ b/py/_code/source.py
@@ -139,7 +139,7 @@
                 trysource = self[start:end]
                 if trysource.isparseable():
                     return start, end
-        return start, end
+        return start, len(self)
 
     def getblockend(self, lineno):
         # XXX
diff --git a/pypy/annotation/policy.py b/pypy/annotation/policy.py
--- a/pypy/annotation/policy.py
+++ b/pypy/annotation/policy.py
@@ -1,6 +1,6 @@
 # base annotation policy for specialization
 from pypy.annotation.specialize import default_specialize as default
-from pypy.annotation.specialize import specialize_argvalue, specialize_argtype, specialize_arglistitemtype
+from pypy.annotation.specialize import specialize_argvalue, specialize_argtype, specialize_arglistitemtype, specialize_arg_or_var
 from pypy.annotation.specialize import memo, specialize_call_location
 # for some reason, model must be imported first,
 # or we create a cycle.
@@ -73,6 +73,7 @@
     default_specialize = staticmethod(default)
     specialize__memo = staticmethod(memo)
     specialize__arg = staticmethod(specialize_argvalue) # specialize:arg(N)
+    specialize__arg_or_var = staticmethod(specialize_arg_or_var)
     specialize__argtype = staticmethod(specialize_argtype) # specialize:argtype(N)
     specialize__arglistitemtype = staticmethod(specialize_arglistitemtype)
     specialize__call_location = staticmethod(specialize_call_location)
diff --git a/pypy/annotation/specialize.py b/pypy/annotation/specialize.py
--- a/pypy/annotation/specialize.py
+++ b/pypy/annotation/specialize.py
@@ -353,6 +353,16 @@
     key = tuple(key)
     return maybe_star_args(funcdesc, key, args_s)
 
+def specialize_arg_or_var(funcdesc, args_s, *argindices):
+    for argno in argindices:
+        if not args_s[argno].is_constant():
+            break
+    else:
+        # all constant
+        return specialize_argvalue(funcdesc, args_s, *argindices)
+    # some not constant
+    return maybe_star_args(funcdesc, None, args_s)
+
 def specialize_argtype(funcdesc, args_s, *argindices):
     key = tuple([args_s[i].knowntype for i in argindices])
     for cls in key:
diff --git a/pypy/annotation/test/test_annrpython.py b/pypy/annotation/test/test_annrpython.py
--- a/pypy/annotation/test/test_annrpython.py
+++ b/pypy/annotation/test/test_annrpython.py
@@ -1194,6 +1194,20 @@
         assert len(executedesc._cache[(0, 'star', 2)].startblock.inputargs) == 4
         assert len(executedesc._cache[(1, 'star', 3)].startblock.inputargs) == 5
 
+    def test_specialize_arg_or_var(self):
+        def f(a):
+            return 1
+        f._annspecialcase_ = 'specialize:arg_or_var(0)'
+
+        def fn(a):
+            return f(3) + f(a)
+
+        a = self.RPythonAnnotator()
+        a.build_types(fn, [int])
+        executedesc = a.bookkeeper.getdesc(f)
+        assert sorted(executedesc._cache.keys()) == [None, (3,)]
+        # we got two different special
+
     def test_specialize_call_location(self):
         def g(a):
             return a
@@ -3190,6 +3204,8 @@
         s = a.build_types(f, [])
         assert isinstance(s, annmodel.SomeList)
         assert not s.listdef.listitem.resized
+        assert not s.listdef.listitem.immutable
+        assert s.listdef.listitem.mutated
 
     def test_delslice(self):
         def f():
diff --git a/pypy/annotation/unaryop.py b/pypy/annotation/unaryop.py
--- a/pypy/annotation/unaryop.py
+++ b/pypy/annotation/unaryop.py
@@ -352,6 +352,7 @@
         check_negative_slice(s_start, s_stop)
         if not isinstance(s_iterable, SomeList):
             raise Exception("list[start:stop] = x: x must be a list")
+        lst.listdef.mutate()
         lst.listdef.agree(s_iterable.listdef)
         # note that setslice is not allowed to resize a list in RPython
 
diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -53,6 +53,18 @@
 this is an ideal task to get started, because it does not require any deep
 knowledge of the internals.
 
+Optimized Unicode Representation
+--------------------------------
+
+CPython 3.3 will use an `optimized unicode representation`_ which switches between
+different ways to represent a unicode string, depending on whether the string
+fits into ASCII, has only two-byte characters or needs four-byte characters.
+
+The actual details would be rather differen in PyPy, but we would like to have
+the same optimization implemented.
+
+.. _`optimized unicode representation`: http://www.python.org/dev/peps/pep-0393/
+
 Translation Toolchain
 ---------------------
 
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -3,12 +3,12 @@
 from pypy.interpreter.executioncontext import ExecutionContext, ActionFlag
 from pypy.interpreter.executioncontext import UserDelAction, FrameTraceAction
 from pypy.interpreter.error import OperationError, operationerrfmt
-from pypy.interpreter.error import new_exception_class
+from pypy.interpreter.error import new_exception_class, typed_unwrap_error_msg
 from pypy.interpreter.argument import Arguments
 from pypy.interpreter.miscutils import ThreadLocals
 from pypy.tool.cache import Cache
 from pypy.tool.uid import HUGEVAL_BYTES
-from pypy.rlib.objectmodel import we_are_translated, newlist
+from pypy.rlib.objectmodel import we_are_translated, newlist, compute_unique_id
 from pypy.rlib.debug import make_sure_not_resized
 from pypy.rlib.timer import DummyTimer, Timer
 from pypy.rlib.rarithmetic import r_uint
@@ -186,6 +186,28 @@
     def _set_mapdict_storage_and_map(self, storage, map):
         raise NotImplementedError
 
+    # -------------------------------------------------------------------
+
+    def str_w(self, space):
+        w_msg = typed_unwrap_error_msg(space, "string", self)
+        raise OperationError(space.w_TypeError, w_msg)
+
+    def unicode_w(self, space):
+        raise OperationError(space.w_TypeError,
+                             typed_unwrap_error_msg(space, "unicode", self))
+
+    def int_w(self, space):
+        raise OperationError(space.w_TypeError,
+                             typed_unwrap_error_msg(space, "integer", self))
+    
+    def uint_w(self, space):
+        raise OperationError(space.w_TypeError,
+                             typed_unwrap_error_msg(space, "integer", self))
+    
+    def bigint_w(self, space):
+        raise OperationError(space.w_TypeError,
+                             typed_unwrap_error_msg(space, "integer", self))
+
 
 class Wrappable(W_Root):
     """A subclass of Wrappable is an internal, interpreter-level class
@@ -901,7 +923,7 @@
         ec.c_call_trace(frame, w_func, args)
         try:
             w_res = self.call_args(w_func, args)
-        except OperationError, e:
+        except OperationError:
             ec.c_exception_trace(frame, w_func)
             raise
         ec.c_return_trace(frame, w_func, args)
@@ -947,6 +969,9 @@
     def isinstance_w(self, w_obj, w_type):
         return self.is_true(self.isinstance(w_obj, w_type))
 
+    def id(self, w_obj):
+        return self.wrap(compute_unique_id(w_obj))
+
     # The code below only works
     # for the simple case (new-style instance).
     # These methods are patched with the full logic by the __builtin__
@@ -999,8 +1024,6 @@
 
     def eval(self, expression, w_globals, w_locals, hidden_applevel=False):
         "NOT_RPYTHON: For internal debugging."
-        import types
-        from pypy.interpreter.pycode import PyCode
         if isinstance(expression, str):
             compiler = self.createcompiler()
             expression = compiler.compile(expression, '?', 'eval', 0,
@@ -1012,7 +1035,6 @@
     def exec_(self, statement, w_globals, w_locals, hidden_applevel=False,
               filename=None):
         "NOT_RPYTHON: For internal debugging."
-        import types
         if filename is None:
             filename = '?'
         from pypy.interpreter.pycode import PyCode
@@ -1210,6 +1232,18 @@
             return None
         return self.str_w(w_obj)
 
+    def str_w(self, w_obj):
+        return w_obj.str_w(self)
+
+    def int_w(self, w_obj):
+        return w_obj.int_w(self)
+
+    def uint_w(self, w_obj):
+        return w_obj.uint_w(self)
+
+    def bigint_w(self, w_obj):
+        return w_obj.bigint_w(self)
+
     def realstr_w(self, w_obj):
         # Like str_w, but only works if w_obj is really of type 'str'.
         if not self.is_true(self.isinstance(w_obj, self.w_str)):
@@ -1217,6 +1251,9 @@
                                  self.wrap('argument must be a string'))
         return self.str_w(w_obj)
 
+    def unicode_w(self, w_obj):
+        return w_obj.unicode_w(self)
+
     def realunicode_w(self, w_obj):
         # Like unicode_w, but only works if w_obj is really of type
         # 'unicode'.
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -458,3 +458,7 @@
     if module:
         space.setattr(w_exc, space.wrap("__module__"), space.wrap(module))
     return w_exc
+
+def typed_unwrap_error_msg(space, expected, w_obj):
+    type_name = space.type(w_obj).getname(space)
+    return space.wrap("expected %s, got %s object" % (expected, type_name))
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -66,7 +66,7 @@
         make_sure_not_resized(self.locals_stack_w)
         check_nonneg(self.nlocals)
         #
-        if space.config.objspace.honor__builtins__ and w_globals is not None:
+        if space.config.objspace.honor__builtins__:
             self.builtin = space.builtin.pick_builtin(w_globals)
         # regular functions always have CO_OPTIMIZED and CO_NEWLOCALS.
         # class bodies only have CO_NEWLOCALS.
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -7252,6 +7252,16 @@
         self.optimize_loop(ops, expected)
 
 
+    def test_setarrayitem_followed_by_arraycopy(self):
+        ops = """
+        [p1, p2]
+        setarrayitem_gc(p1, 2, 10, descr=arraydescr)
+        setarrayitem_gc(p2, 3, 13, descr=arraydescr)        
+        call(0, p1, p2, 0, 0, 10, descr=arraycopydescr)
+        jump(p1, p2)
+        """
+        self.optimize_loop(ops, ops)
+
 class TestLLtype(OptimizeOptTest, LLtypeMixin):
     pass
 
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -3408,6 +3408,75 @@
         assert res == main(1, 10)
         self.check_loops(call=0)
 
+    def test_setarrayitem_followed_by_arraycopy(self):
+        myjitdriver = JitDriver(greens = [], reds = ['n', 'sa', 'x', 'y'])
+        def f(n):
+            sa = 0
+            x = [1,2,n]
+            y = [1,2,3]
+            while n > 0:
+                myjitdriver.jit_merge_point(sa=sa, n=n, x=x, y=y)
+                y[0] = n
+                x[0:3] = y
+                sa += x[0]
+                n -= 1
+            return sa
+        res = self.meta_interp(f, [16])
+        assert res == f(16)
+        
+
 
 class TestLLtype(BaseLLtypeTests, LLJitMixin):
-    pass
+    def test_tagged(self):
+        py.test.skip("implement me")
+        from pypy.rlib.objectmodel import UnboxedValue
+        class Base(object):
+            __slots__ = ()
+
+        class Int(UnboxedValue, Base):
+            __slots__ = ["a"]
+
+            def is_pos(self):
+                return self.a > 0
+
+            def dec(self):
+                return Int(self.a - 1)
+
+
+        class Float(Base):
+            def __init__(self, a):
+                self.a = a
+
+            def is_pos(self):
+                return self.a > 0
+
+            def dec(self):
+                return Float(self.a - 1)
+
+        driver = JitDriver(greens=['pc', 's'], reds=['o'])
+
+        def main(fl, n, s):
+            if s:
+                s = "--j"
+            else:
+                s = "---j"
+            if fl:
+                o = Float(float(n))
+            else:
+                o = Int(n)
+            pc = 0
+            while True:
+                driver.jit_merge_point(s=s, pc=pc, o=o)
+                c = s[pc]
+                if c == "j":
+                    driver.can_enter_jit(s=s, pc=pc, o=o)
+                    if o.is_pos():
+                        pc = 0
+                        continue
+                    else:
+                        break
+                elif c == "-":
+                    o = o.dec()
+                pc += 1
+            return pc
+        res = self.meta_interp(main, [False, 100, True], taggedpointers=True)
diff --git a/pypy/jit/metainterp/warmspot.py b/pypy/jit/metainterp/warmspot.py
--- a/pypy/jit/metainterp/warmspot.py
+++ b/pypy/jit/metainterp/warmspot.py
@@ -53,6 +53,8 @@
         extraconfigopts = {'translation.list_comprehension_operations': True}
     else:
         extraconfigopts = {}
+    if kwds.pop("taggedpointers", False):
+        extraconfigopts["translation.taggedpointers"] = True
     interp, graph = get_interpreter(function, args,
                                     backendopt=False,  # will be done below
                                     type_system=type_system,
diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py
--- a/pypy/module/__builtin__/functional.py
+++ b/pypy/module/__builtin__/functional.py
@@ -3,13 +3,13 @@
 
 """
 
+from pypy.interpreter.baseobjspace import Wrappable
 from pypy.interpreter.error import OperationError
-from pypy.interpreter.gateway import NoneNotWrapped
-from pypy.interpreter.gateway import interp2app, unwrap_spec
+from pypy.interpreter.gateway import NoneNotWrapped, interp2app, unwrap_spec
 from pypy.interpreter.typedef import TypeDef
-from pypy.interpreter.baseobjspace import Wrappable
+from pypy.rlib import jit
+from pypy.rlib.objectmodel import specialize
 from pypy.rlib.rarithmetic import r_uint, intmask
-from pypy.rlib.objectmodel import specialize
 from pypy.rlib.rbigint import rbigint
 
 
@@ -134,29 +134,15 @@
 
 
 @specialize.arg(2)
+ at jit.look_inside_iff(lambda space, args, implementation_of:
+    jit.isconstant(len(args.arguments_w)) and
+    len(args.arguments_w) == 2
+)
 def min_max(space, args, implementation_of):
     if implementation_of == "max":
         compare = space.gt
     else:
         compare = space.lt
-
-    args_w = args.arguments_w
-    if len(args_w) == 2 and not args.keywords:
-        # simple case, suitable for the JIT
-        w_arg0, w_arg1 = args_w
-        if space.is_true(compare(w_arg0, w_arg1)):
-            return w_arg0
-        else:
-            return w_arg1
-    else:
-        return min_max_loop(space, args, implementation_of)
-
- at specialize.arg(2)
-def min_max_loop(space, args, implementation_of):
-    if implementation_of == "max":
-        compare = space.gt
-    else:
-        compare = space.lt
     args_w = args.arguments_w
     if len(args_w) > 1:
         w_sequence = space.newtuple(args_w)
diff --git a/pypy/module/_continuation/__init__.py b/pypy/module/_continuation/__init__.py
--- a/pypy/module/_continuation/__init__.py
+++ b/pypy/module/_continuation/__init__.py
@@ -37,4 +37,5 @@
     interpleveldefs = {
         'continulet': 'interp_continuation.W_Continulet',
         'permute': 'interp_continuation.permute',
+        '_p': 'interp_continuation.unpickle',      # pickle support
     }
diff --git a/pypy/module/_continuation/interp_continuation.py b/pypy/module/_continuation/interp_continuation.py
--- a/pypy/module/_continuation/interp_continuation.py
+++ b/pypy/module/_continuation/interp_continuation.py
@@ -6,6 +6,7 @@
 from pypy.interpreter.typedef import TypeDef
 from pypy.interpreter.gateway import interp2app
 from pypy.interpreter.pycode import PyCode
+from pypy.interpreter.pyframe import PyFrame
 
 
 class W_Continulet(Wrappable):
@@ -23,24 +24,27 @@
         if ec.stacklet_thread is not self.sthread:
             global_state.clear()
             raise geterror(self.space, "inter-thread support is missing")
-        return ec
 
     def descr_init(self, w_callable, __args__):
         if self.sthread is not None:
             raise geterror(self.space, "continulet already __init__ialized")
+        #
+        # hackish: build the frame "by hand", passing it the correct arguments
+        space = self.space
+        w_args, w_kwds = __args__.topacked()
+        bottomframe = space.createframe(get_entrypoint_pycode(space),
+                                        get_w_module_dict(space), None)
+        bottomframe.locals_stack_w[0] = space.wrap(self)
+        bottomframe.locals_stack_w[1] = w_callable
+        bottomframe.locals_stack_w[2] = w_args
+        bottomframe.locals_stack_w[3] = w_kwds
+        self.bottomframe = bottomframe
+        #
         global_state.origin = self
-        global_state.w_callable = w_callable
-        global_state.args = __args__
-        self.bottomframe = make_fresh_frame(self.space)
-        self.sthread = build_sthread(self.space)
-        try:
-            self.h = self.sthread.new(new_stacklet_callback)
-            if self.sthread.is_empty_handle(self.h):    # early return
-                raise MemoryError
-        except MemoryError:
-            self.sthread = None
-            global_state.clear()
-            raise getmemoryerror(self.space)
+        sthread = build_sthread(self.space)
+        self.sthread = sthread
+        h = sthread.new(new_stacklet_callback)
+        post_switch(sthread, h)
 
     def switch(self, w_to):
         sthread = self.sthread
@@ -66,7 +70,7 @@
             if sthread.is_empty_handle(to.h):
                 global_state.clear()
                 raise geterror(self.space, "continulet already finished")
-        ec = self.check_sthread()
+        self.check_sthread()
         #
         global_state.origin = self
         if to is None:
@@ -76,13 +80,8 @@
             # double switch: the final destination is to.h
             global_state.destination = to
         #
-        try:
-            do_switch(sthread, global_state.destination.h)
-        except MemoryError:
-            global_state.clear()
-            raise getmemoryerror(self.space)
-        #
-        return get_result()
+        h = sthread.switch(global_state.destination.h)
+        return post_switch(sthread, h)
 
     def descr_switch(self, w_value=None, w_to=None):
         global_state.w_value = w_value
@@ -109,12 +108,26 @@
                  and not self.sthread.is_empty_handle(self.h))
         return self.space.newbool(valid)
 
+    def descr__reduce__(self):
+        from pypy.module._continuation import interp_pickle
+        return interp_pickle.reduce(self)
+
+    def descr__setstate__(self, w_args):
+        from pypy.module._continuation import interp_pickle
+        interp_pickle.setstate(self, w_args)
+
 
 def W_Continulet___new__(space, w_subtype, __args__):
     r = space.allocate_instance(W_Continulet, w_subtype)
     r.__init__(space)
     return space.wrap(r)
 
+def unpickle(space, w_subtype):
+    """Pickle support."""
+    r = space.allocate_instance(W_Continulet, w_subtype)
+    r.__init__(space)
+    return space.wrap(r)
+
 
 W_Continulet.typedef = TypeDef(
     'continulet',
@@ -124,9 +137,10 @@
     switch      = interp2app(W_Continulet.descr_switch),
     throw       = interp2app(W_Continulet.descr_throw),
     is_pending  = interp2app(W_Continulet.descr_is_pending),
+    __reduce__  = interp2app(W_Continulet.descr__reduce__),
+    __setstate__= interp2app(W_Continulet.descr__setstate__),
     )
 
-
 # ____________________________________________________________
 
 # Continulet objects maintain a dummy frame object in order to ensure
@@ -135,27 +149,40 @@
 
 class State:
     def __init__(self, space):
-        from pypy.interpreter.astcompiler.consts import CO_OPTIMIZED
-        self.space = space 
+        self.space = space
         w_module = space.getbuiltinmodule('_continuation')
         self.w_error = space.getattr(w_module, space.wrap('error'))
-        self.w_memoryerror = OperationError(space.w_MemoryError, space.w_None)
-        self.dummy_pycode = PyCode(space, 0, 0, 0, CO_OPTIMIZED,
-                                   '', [], [], [], '',
-                                   '<bottom of continulet>', 0, '', [], [],
-                                   hidden_applevel=True)
+        # the following function switches away immediately, so that
+        # continulet.__init__() doesn't immediately run func(), but it
+        # also has the hidden purpose of making sure we have a single
+        # bottomframe for the whole duration of the continulet's run.
+        # Hackish: only the func_code is used, and used in the context
+        # of w_globals == this module, so we can access the name
+        # 'continulet' directly.
+        w_code = space.appexec([], '''():
+            def start(c, func, args, kwds):
+                if continulet.switch(c) is not None:
+                    raise TypeError(
+                     "can\'t send non-None value to a just-started continulet")
+                return func(c, *args, **kwds)
+            return start.func_code
+        ''')
+        self.entrypoint_pycode = space.interp_w(PyCode, w_code)
+        self.entrypoint_pycode.hidden_applevel = True
+        self.w_unpickle = w_module.get('_p')
+        self.w_module_dict = w_module.getdict(space)
 
 def geterror(space, message):
     cs = space.fromcache(State)
     return OperationError(cs.w_error, space.wrap(message))
 
-def getmemoryerror(space):
+def get_entrypoint_pycode(space):
     cs = space.fromcache(State)
-    return cs.w_memoryerror
+    return cs.entrypoint_pycode
 
-def make_fresh_frame(space):
+def get_w_module_dict(space):
     cs = space.fromcache(State)
-    return space.FrameClass(space, cs.dummy_pycode, None, None)
+    return cs.w_module_dict
 
 # ____________________________________________________________
 
@@ -166,6 +193,9 @@
         StackletThread.__init__(self, space.config)
         self.space = space
         self.ec = ec
+        # for unpickling
+        from pypy.rlib.rweakref import RWeakKeyDictionary
+        self.frame2continulet = RWeakKeyDictionary(PyFrame, W_Continulet)
 
 ExecutionContext.stacklet_thread = None
 
@@ -176,8 +206,6 @@
     def clear(self):
         self.origin = None
         self.destination = None
-        self.w_callable = None
-        self.args = None
         self.w_value = None
         self.propagate_exception = None
 global_state = GlobalState()
@@ -185,27 +213,13 @@
 
 
 def new_stacklet_callback(h, arg):
-    self       = global_state.origin
-    w_callable = global_state.w_callable
-    args       = global_state.args
+    self = global_state.origin
+    self.h = h
     global_state.clear()
-    try:
-        do_switch(self.sthread, h)
-    except MemoryError:
-        return h       # oups!  do an early return in this case
-    #
     space = self.space
     try:
-        assert self.sthread.ec.topframeref() is None
-        self.sthread.ec.topframeref = jit.non_virtual_ref(self.bottomframe)
-        if global_state.propagate_exception is not None:
-            raise global_state.propagate_exception  # just propagate it further
-        if global_state.w_value is not space.w_None:
-            raise OperationError(space.w_TypeError, space.wrap(
-                "can't send non-None value to a just-started continulet"))
-
-        args = args.prepend(self.space.wrap(self))
-        w_result = space.call_args(w_callable, args)
+        frame = self.bottomframe
+        w_result = frame.execute_frame()
     except Exception, e:
         global_state.propagate_exception = e
     else:
@@ -215,9 +229,7 @@
     global_state.destination = self
     return self.h
 
-
-def do_switch(sthread, h):
-    h = sthread.switch(h)
+def post_switch(sthread, h):
     origin = global_state.origin
     self = global_state.destination
     global_state.origin = None
@@ -228,6 +240,8 @@
     sthread.ec.topframeref = self.bottomframe.f_backref
     self.bottomframe.f_backref = origin.bottomframe.f_backref
     origin.bottomframe.f_backref = current
+    #
+    return get_result()
 
 def get_result():
     if global_state.propagate_exception:
diff --git a/pypy/module/_continuation/interp_pickle.py b/pypy/module/_continuation/interp_pickle.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_continuation/interp_pickle.py
@@ -0,0 +1,128 @@
+from pypy.tool import stdlib_opcode as pythonopcode
+from pypy.rlib import jit
+from pypy.interpreter.error import OperationError
+from pypy.interpreter.pyframe import PyFrame
+from pypy.module._continuation.interp_continuation import State, global_state
+from pypy.module._continuation.interp_continuation import build_sthread
+from pypy.module._continuation.interp_continuation import post_switch
+from pypy.module._continuation.interp_continuation import get_result, geterror
+
+
+def getunpickle(space):
+    cs = space.fromcache(State)
+    return cs.w_unpickle
+
+
+def reduce(self):
+    # xxx this is known to be not completely correct with respect
+    # to subclasses, e.g. no __slots__ support, no looking for a
+    # __getnewargs__ or __getstate__ defined in the subclass, etc.
+    # Doing the right thing looks involved, though...
+    space = self.space
+    if self.sthread is None:
+        w_frame = space.w_False
+    elif self.sthread.is_empty_handle(self.h):
+        w_frame = space.w_None
+    else:
+        w_frame = space.wrap(self.bottomframe)
+    w_continulet_type = space.type(space.wrap(self))
+    w_dict = self.getdict(space) or space.w_None
+    args = [getunpickle(space),
+            space.newtuple([w_continulet_type]),
+            space.newtuple([w_frame, w_dict]),
+            ]
+    return space.newtuple(args)
+
+def setstate(self, w_args):
+    space = self.space
+    if self.sthread is not None:
+        raise geterror(space, "continulet.__setstate__() on an already-"
+                              "initialized continulet")
+    w_frame, w_dict = space.fixedview(w_args, expected_length=2)
+    if not space.is_w(w_dict, space.w_None):
+        self.setdict(space, w_dict)
+    if space.is_w(w_frame, space.w_False):
+        return    # not initialized
+    sthread = build_sthread(self.space)
+    self.sthread = sthread
+    self.bottomframe = space.interp_w(PyFrame, w_frame, can_be_None=True)
+    #
+    global_state.origin = self
+    if self.bottomframe is not None:
+        sthread.frame2continulet.set(self.bottomframe, self)
+    self.h = sthread.new(resume_trampoline_callback)
+    get_result()    # propagate the eventual MemoryError
+
+# ____________________________________________________________
+
+def resume_trampoline_callback(h, arg):
+    self = global_state.origin
+    self.h = h
+    space = self.space
+    sthread = self.sthread
+    try:
+        global_state.clear()
+        if self.bottomframe is None:
+            w_result = space.w_None
+        else:
+            h = sthread.switch(self.h)
+            try:
+                w_result = post_switch(sthread, h)
+                operr = None
+            except OperationError, e:
+                w_result = None
+                operr = e
+            #
+            while True:
+                ec = sthread.ec
+                frame = ec.topframeref()
+                assert frame is not None     # XXX better error message
+                exit_continulet = sthread.frame2continulet.get(frame)
+                #
+                continue_after_call(frame)
+                #
+                # small hack: unlink frame out of the execution context,
+                # because execute_frame will add it there again
+                ec.topframeref = frame.f_backref
+                #
+                try:
+                    w_result = frame.execute_frame(w_result, operr)
+                    operr = None
+                except OperationError, e:
+                    w_result = None
+                    operr = e
+                if exit_continulet is not None:
+                    self = exit_continulet
+                    break
+            sthread.ec.topframeref = jit.vref_None
+            if operr:
+                raise operr
+    except Exception, e:
+        global_state.propagate_exception = e
+    else:
+        global_state.w_value = w_result
+    global_state.origin = self
+    global_state.destination = self
+    return self.h
+
+def continue_after_call(frame):
+    code = frame.pycode.co_code
+    instr = frame.last_instr
+    opcode = ord(code[instr])
+    map = pythonopcode.opmap
+    call_ops = [map['CALL_FUNCTION'], map['CALL_FUNCTION_KW'],
+                map['CALL_FUNCTION_VAR'], map['CALL_FUNCTION_VAR_KW'],
+                map['CALL_METHOD']]
+    assert opcode in call_ops   # XXX check better, and complain better
+    instr += 1
+    oparg = ord(code[instr]) | ord(code[instr + 1]) << 8
+    nargs = oparg & 0xff
+    nkwds = (oparg >> 8) & 0xff
+    if nkwds == 0:     # only positional arguments
+        # fast paths leaves things on the stack, pop them
+        if (frame.space.config.objspace.opcodes.CALL_METHOD and
+            opcode == map['CALL_METHOD']):
+            frame.dropvalues(nargs + 2)
+        elif opcode == map['CALL_FUNCTION']:
+            frame.dropvalues(nargs + 1)
+    frame.last_instr = instr + 1    # continue after the call
diff --git a/pypy/module/_continuation/test/test_stacklet.py b/pypy/module/_continuation/test/test_stacklet.py
--- a/pypy/module/_continuation/test/test_stacklet.py
+++ b/pypy/module/_continuation/test/test_stacklet.py
@@ -13,7 +13,7 @@
         from _continuation import continulet
         #
         def empty_callback(c):
-            pass
+            never_called
         #
         c = continulet(empty_callback)
         assert type(c) is continulet
@@ -36,7 +36,7 @@
         from _continuation import continulet, error
         #
         def empty_callback(c1):
-            pass
+            never_called
         #
         c = continulet(empty_callback)
         raises(error, c.__init__, empty_callback)
diff --git a/pypy/module/_continuation/test/test_zpickle.py b/pypy/module/_continuation/test/test_zpickle.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_continuation/test/test_zpickle.py
@@ -0,0 +1,262 @@
+from pypy.conftest import gettestobjspace
+
+
+class AppTestCopy:
+    def setup_class(cls):
+        cls.space = gettestobjspace(usemodules=('_continuation',),
+                                    CALL_METHOD=True)
+        cls.space.config.translation.continuation = True
+
+    def test_basic_setup(self):
+        from _continuation import continulet
+        lst = [4]
+        co = continulet(lst.append)
+        assert lst == [4]
+        res = co.switch()
+        assert res is None
+        assert lst == [4, co]
+
+    def test_copy_continulet_not_started(self):
+        from _continuation import continulet, error
+        import copy
+        lst = []
+        co = continulet(lst.append)
+        co2, lst2 = copy.deepcopy((co, lst))
+        #
+        assert lst == []
+        co.switch()
+        assert lst == [co]
+        #
+        assert lst2 == []
+        co2.switch()
+        assert lst2 == [co2]
+
+    def test_copy_continulet_not_started_multiple(self):
+        from _continuation import continulet, error
+        import copy
+        lst = []
+        co = continulet(lst.append)
+        co2, lst2 = copy.deepcopy((co, lst))
+        co3, lst3 = copy.deepcopy((co, lst))
+        co4, lst4 = copy.deepcopy((co, lst))
+        #
+        assert lst == []
+        co.switch()
+        assert lst == [co]
+        #
+        assert lst2 == []
+        co2.switch()
+        assert lst2 == [co2]
+        #
+        assert lst3 == []
+        co3.switch()
+        assert lst3 == [co3]
+        #
+        assert lst4 == []
+        co4.switch()
+        assert lst4 == [co4]
+
+    def test_copy_continulet_real(self):
+        import new, sys
+        mod = new.module('test_copy_continulet_real')
+        sys.modules['test_copy_continulet_real'] = mod
+        exec '''if 1:
+            from _continuation import continulet
+            import copy
+            def f(co, x):
+                co.switch(x + 1)
+                co.switch(x + 2)
+                return x + 3
+            co = continulet(f, 40)
+            res = co.switch()
+            assert res == 41
+            co2 = copy.deepcopy(co)
+            #
+            res = co2.switch()
+            assert res == 42
+            assert co2.is_pending()
+            res = co2.switch()
+            assert res == 43
+            assert not co2.is_pending()
+            #
+            res = co.switch()
+            assert res == 42
+            assert co.is_pending()
+            res = co.switch()
+            assert res == 43
+            assert not co.is_pending()
+        ''' in mod.__dict__
+
+    def test_copy_continulet_already_finished(self):
+        from _continuation import continulet, error
+        import copy
+        lst = []
+        co = continulet(lst.append)
+        co.switch()
+        co2 = copy.deepcopy(co)
+        assert not co.is_pending()
+        assert not co2.is_pending()
+        raises(error, co.__init__, lst.append)
+        raises(error, co2.__init__, lst.append)
+        raises(error, co.switch)
+        raises(error, co2.switch)
+
+
+class AppTestPickle:
+    version = 0
+
+    def setup_class(cls):
+        cls.space = gettestobjspace(usemodules=('_continuation',),
+                                    CALL_METHOD=True)
+        cls.space.appexec([], """():
+            global continulet, A, __name__
+
+            import sys
+            __name__ = 'test_pickle_continulet'
+            thismodule = type(sys)(__name__)
+            sys.modules[__name__] = thismodule
+
+            from _continuation import continulet
+            class A(continulet):
+                pass
+
+            thismodule.__dict__.update(globals())
+        """)
+        cls.w_version = cls.space.wrap(cls.version)
+
+    def test_pickle_continulet_empty(self):
+        from _continuation import continulet
+        lst = [4]
+        co = continulet.__new__(continulet)
+        import pickle
+        pckl = pickle.dumps(co, self.version)
+        print repr(pckl)
+        co2 = pickle.loads(pckl)
+        assert co2 is not co
+        assert not co.is_pending()
+        assert not co2.is_pending()
+        # the empty unpickled coroutine can still be used:
+        result = [5]
+        co2.__init__(result.append)
+        res = co2.switch()
+        assert res is None
+        assert result == [5, co2]
+
+    def test_pickle_continulet_empty_subclass(self):
+        from test_pickle_continulet import continulet, A
+        lst = [4]
+        co = continulet.__new__(A)
+        co.foo = 'bar'
+        co.bar = 'baz'
+        import pickle
+        pckl = pickle.dumps(co, self.version)
+        print repr(pckl)
+        co2 = pickle.loads(pckl)
+        assert co2 is not co
+        assert not co.is_pending()
+        assert not co2.is_pending()
+        assert type(co) is type(co2) is A
+        assert co.foo == co2.foo == 'bar'
+        assert co.bar == co2.bar == 'baz'
+        # the empty unpickled coroutine can still be used:
+        result = [5]
+        co2.__init__(result.append)
+        res = co2.switch()
+        assert res is None
+        assert result == [5, co2]
+
+    def test_pickle_continulet_not_started(self):
+        from _continuation import continulet, error
+        import pickle
+        lst = []
+        co = continulet(lst.append)
+        pckl = pickle.dumps((co, lst))
+        print pckl
+        del co, lst
+        for i in range(2):
+            print 'resume...'
+            co2, lst2 = pickle.loads(pckl)
+            assert lst2 == []
+            co2.switch()
+            assert lst2 == [co2]
+
+    def test_pickle_continulet_real(self):
+        import new, sys
+        mod = new.module('test_pickle_continulet_real')
+        sys.modules['test_pickle_continulet_real'] = mod
+        mod.version = self.version
+        exec '''if 1:
+            from _continuation import continulet
+            import pickle
+            def f(co, x):
+                co.switch(x + 1)
+                co.switch(x + 2)
+                return x + 3
+            co = continulet(f, 40)
+            res = co.switch()
+            assert res == 41
+            pckl = pickle.dumps(co, version)
+            print repr(pckl)
+            co2 = pickle.loads(pckl)
+            #
+            res = co2.switch()
+            assert res == 42
+            assert co2.is_pending()
+            res = co2.switch()
+            assert res == 43
+            assert not co2.is_pending()
+            #
+            res = co.switch()
+            assert res == 42
+            assert co.is_pending()
+            res = co.switch()
+            assert res == 43
+            assert not co.is_pending()
+        ''' in mod.__dict__
+
+    def test_pickle_continulet_real_subclass(self):
+        import new, sys
+        mod = new.module('test_pickle_continulet_real_subclass')
+        sys.modules['test_pickle_continulet_real_subclass'] = mod
+        mod.version = self.version
+        exec '''if 1:
+            from _continuation import continulet
+            import pickle
+            class A(continulet):
+                def __init__(self):
+                    crash
+            def f(co):
+                co.switch(co.x + 1)
+                co.switch(co.x + 2)
+                return co.x + 3
+            co = A.__new__(A)
+            continulet.__init__(co, f)
+            co.x = 40
+            res = co.switch()
+            assert res == 41
+            pckl = pickle.dumps(co, version)
+            print repr(pckl)
+            co2 = pickle.loads(pckl)
+            #
+            assert type(co2) is A
+            res = co2.switch()
+            assert res == 42
+            assert co2.is_pending()
+            res = co2.switch()
+            assert res == 43
+            assert not co2.is_pending()
+            #
+            res = co.switch()
+            assert res == 42
+            assert co.is_pending()
+            res = co.switch()
+            assert res == 43
+            assert not co.is_pending()
+        ''' in mod.__dict__
+
+
+class AppTestPickle_v1(AppTestPickle):
+    version = 1
+
+class AppTestPickle_v2(AppTestPickle):
+    version = 2
diff --git a/pypy/module/_multiprocessing/interp_connection.py b/pypy/module/_multiprocessing/interp_connection.py
--- a/pypy/module/_multiprocessing/interp_connection.py
+++ b/pypy/module/_multiprocessing/interp_connection.py
@@ -225,7 +225,9 @@
             except OSError:
                 pass
 
-    def __init__(self, fd, flags):
+    def __init__(self, space, fd, flags):
+        if fd == self.INVALID_HANDLE_VALUE or fd < 0:
+            raise OperationError(space.w_IOError, space.wrap("invalid handle %d" % fd))
         W_BaseConnection.__init__(self, flags)
         self.fd = fd
 
@@ -234,7 +236,7 @@
         flags = (readable and READABLE) | (writable and WRITABLE)
 
         self = space.allocate_instance(W_FileConnection, w_subtype)
-        W_FileConnection.__init__(self, fd, flags)
+        W_FileConnection.__init__(self, space, fd, flags)
         return space.wrap(self)
 
     def fileno(self, space):
diff --git a/pypy/module/_multiprocessing/interp_semaphore.py b/pypy/module/_multiprocessing/interp_semaphore.py
--- a/pypy/module/_multiprocessing/interp_semaphore.py
+++ b/pypy/module/_multiprocessing/interp_semaphore.py
@@ -468,6 +468,9 @@
 
         self.count -= 1
 
+    def after_fork(self):
+        self.count = 0
+
     @unwrap_spec(kind=int, maxvalue=int)
     def rebuild(space, w_cls, w_handle, kind, maxvalue):
         self = space.allocate_instance(W_SemLock, w_cls)
@@ -512,6 +515,7 @@
     acquire = interp2app(W_SemLock.acquire),
     release = interp2app(W_SemLock.release),
     _rebuild = interp2app(W_SemLock.rebuild.im_func, as_classmethod=True),
+    _after_fork = interp2app(W_SemLock.after_fork),
     __enter__=interp2app(W_SemLock.enter),
     __exit__=interp2app(W_SemLock.exit),
     SEM_VALUE_MAX=SEM_VALUE_MAX,
diff --git a/pypy/module/_multiprocessing/test/test_connection.py b/pypy/module/_multiprocessing/test/test_connection.py
--- a/pypy/module/_multiprocessing/test/test_connection.py
+++ b/pypy/module/_multiprocessing/test/test_connection.py
@@ -145,3 +145,9 @@
             else:
                 c.close()
         space.delslice(w_connections, space.wrap(0), space.wrap(100))
+
+    def test_bad_fd(self):
+        import _multiprocessing
+
+        raises(IOError, _multiprocessing.Connection, -1)
+        raises(IOError, _multiprocessing.Connection, -15)
\ No newline at end of file
diff --git a/pypy/module/_multiprocessing/test/test_semaphore.py b/pypy/module/_multiprocessing/test/test_semaphore.py
--- a/pypy/module/_multiprocessing/test/test_semaphore.py
+++ b/pypy/module/_multiprocessing/test/test_semaphore.py
@@ -39,6 +39,10 @@
         sem.release()
         assert sem._count() == 0
 
+        sem.acquire()
+        sem._after_fork()
+        assert sem._count() == 0
+
     def test_recursive(self):
         from _multiprocessing import SemLock
         kind = self.RECURSIVE
diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py
--- a/pypy/module/_sre/interp_sre.py
+++ b/pypy/module/_sre/interp_sre.py
@@ -99,6 +99,7 @@
 # SRE_Pattern class
 
 class W_SRE_Pattern(Wrappable):
+    _immutable_fields_ = ["code", "flags"]
 
     def cannot_copy_w(self):
         space = self.space
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -57,6 +57,9 @@
         compile_extra=['-DPy_BUILD_CORE'],
         )
 
+class CConfig2:
+    _compilation_info_ = CConfig._compilation_info_
+
 class CConfig_constants:
     _compilation_info_ = CConfig._compilation_info_
 
@@ -300,9 +303,13 @@
         return unwrapper_raise # used in 'normal' RPython code.
     return decorate
 
-def cpython_struct(name, fields, forward=None):
+def cpython_struct(name, fields, forward=None, level=1):
     configname = name.replace(' ', '__')
-    setattr(CConfig, configname, rffi_platform.Struct(name, fields))
+    if level == 1:
+        config = CConfig
+    else:
+        config = CConfig2
+    setattr(config, configname, rffi_platform.Struct(name, fields))
     if forward is None:
         forward = lltype.ForwardReference()
     TYPES[configname] = forward
@@ -445,9 +452,10 @@
 #              'int*': rffi.INTP}
 
 def configure_types():
-    for name, TYPE in rffi_platform.configure(CConfig).iteritems():
-        if name in TYPES:
-            TYPES[name].become(TYPE)
+    for config in (CConfig, CConfig2):
+        for name, TYPE in rffi_platform.configure(config).iteritems():
+            if name in TYPES:
+                TYPES[name].become(TYPE)
 
 def build_type_checkers(type_name, cls=None):
     """
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -321,6 +321,15 @@
 
 } PyTypeObject;
 
+typedef struct {
+    PyTypeObject ht_type;
+    PyNumberMethods as_number;
+    PyMappingMethods as_mapping;
+    PySequenceMethods as_sequence;
+    PyBufferProcs as_buffer;
+    PyObject *ht_name, *ht_slots;
+} PyHeapTypeObject;
+
 /* Flag bits for printing: */
 #define Py_PRINT_RAW	1	/* No string quotes etc. */
 
diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py
--- a/pypy/module/cpyext/pyobject.py
+++ b/pypy/module/cpyext/pyobject.py
@@ -19,13 +19,42 @@
     basestruct = PyObject.TO
 
     def get_dealloc(self, space):
-        raise NotImplementedError
+        from pypy.module.cpyext.typeobject import subtype_dealloc
+        return llhelper(
+            subtype_dealloc.api_func.functype,
+            subtype_dealloc.api_func.get_wrapper(space))
+
     def allocate(self, space, w_type, itemcount=0):
-        raise NotImplementedError
+        # similar to PyType_GenericAlloc?
+        # except that it's not related to any pypy object.
+
+        pytype = rffi.cast(PyTypeObjectPtr, make_ref(space, w_type))
+        # Don't increase refcount for non-heaptypes
+        if pytype:
+            flags = rffi.cast(lltype.Signed, pytype.c_tp_flags)
+            if not flags & Py_TPFLAGS_HEAPTYPE:
+                Py_DecRef(space, w_type)
+
+        if pytype:
+            size = pytype.c_tp_basicsize
+        else:
+            size = rffi.sizeof(self.basestruct)
+        if itemcount:
+            size += itemcount * pytype.c_tp_itemsize
+        buf = lltype.malloc(rffi.VOIDP.TO, size,
+                            flavor='raw', zero=True)
+        pyobj = rffi.cast(PyObject, buf)
+        pyobj.c_ob_refcnt = 1
+        pyobj.c_ob_type = pytype
+        return pyobj
+
     def attach(self, space, pyobj, w_obj):
-        raise NotImplementedError
+        pass
+
     def realize(self, space, ref):
-        raise NotImplementedError
+        # For most types, a reference cannot exist without
+        # a real interpreter object
+        raise InvalidPointerException(str(ref))
 
 typedescr_cache = {}
 
@@ -40,6 +69,7 @@
     """
 
     tp_basestruct = kw.pop('basestruct', PyObject.TO)
+    tp_alloc      = kw.pop('alloc', None)
     tp_attach     = kw.pop('attach', None)
     tp_realize    = kw.pop('realize', None)
     tp_dealloc    = kw.pop('dealloc', None)
@@ -49,58 +79,24 @@
 
     class CpyTypedescr(BaseCpyTypedescr):
         basestruct = tp_basestruct
-        realize = tp_realize
 
-        def get_dealloc(self, space):
-            if tp_dealloc:
+        if tp_alloc:
+            def allocate(self, space, w_type, itemcount=0):
+                return tp_alloc(space, w_type)
+
+        if tp_dealloc:
+            def get_dealloc(self, space):
                 return llhelper(
                     tp_dealloc.api_func.functype,
                     tp_dealloc.api_func.get_wrapper(space))
-            else:
-                from pypy.module.cpyext.typeobject import subtype_dealloc
-                return llhelper(
-                    subtype_dealloc.api_func.functype,
-                    subtype_dealloc.api_func.get_wrapper(space))
-
-        def allocate(self, space, w_type, itemcount=0):
-            # similar to PyType_GenericAlloc?
-            # except that it's not related to any pypy object.
-
-            pytype = rffi.cast(PyTypeObjectPtr, make_ref(space, w_type))
-            # Don't increase refcount for non-heaptypes
-            if pytype:
-                flags = rffi.cast(lltype.Signed, pytype.c_tp_flags)
-                if not flags & Py_TPFLAGS_HEAPTYPE:
-                    Py_DecRef(space, w_type)
-
-            if pytype:
-                size = pytype.c_tp_basicsize
-            else:
-                size = rffi.sizeof(tp_basestruct)
-            if itemcount:
-                size += itemcount * pytype.c_tp_itemsize
-            buf = lltype.malloc(rffi.VOIDP.TO, size,
-                                flavor='raw', zero=True)
-            pyobj = rffi.cast(PyObject, buf)
-            pyobj.c_ob_refcnt = 1
-            pyobj.c_ob_type = pytype
-            return pyobj
 
         if tp_attach:
             def attach(self, space, pyobj, w_obj):
                 tp_attach(space, pyobj, w_obj)
-        else:
-            def attach(self, space, pyobj, w_obj):
-                pass
 
         if tp_realize:
             def realize(self, space, ref):
                 return tp_realize(space, ref)
-        else:
-            def realize(self, space, ref):
-                # For most types, a reference cannot exist without
-                # a real interpreter object
-                raise InvalidPointerException(str(ref))
     if typedef:
         CpyTypedescr.__name__ = "CpyTypedescr_%s" % (typedef.name,)
 
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -268,6 +268,21 @@
         assert type(obj) is foo.Custom
         assert type(foo.Custom) is foo.MetaType
 
+    def test_heaptype(self):
+        module = self.import_extension('foo', [
+           ("name_by_heaptype", "METH_O",
+            '''
+                 PyHeapTypeObject *heaptype = (PyHeapTypeObject *)args;
+                 Py_INCREF(heaptype->ht_name);
+                 return heaptype->ht_name;
+             '''
+             )
+            ])
+        class C(object):
+            pass
+        assert module.name_by_heaptype(C) == "C"
+        
+
 class TestTypes(BaseApiTest):
     def test_type_attributes(self, space, api):
         w_class = space.appexec([], """():
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -11,7 +11,7 @@
     generic_cpy_call, Py_TPFLAGS_READY, Py_TPFLAGS_READYING,
     Py_TPFLAGS_HEAPTYPE, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL,
     Py_TPFLAGS_HAVE_GETCHARBUFFER,
-    build_type_checkers)
+    build_type_checkers, PyObjectFields)
 from pypy.module.cpyext.pyobject import (
     PyObject, make_ref, create_ref, from_ref, get_typedescr, make_typedescr,
     track_reference, RefcountState, borrow_from)
@@ -25,7 +25,7 @@
 from pypy.module.cpyext.structmember import PyMember_GetOne, PyMember_SetOne
 from pypy.module.cpyext.typeobjectdefs import (
     PyTypeObjectPtr, PyTypeObject, PyGetSetDef, PyMemberDef, newfunc,
-    PyNumberMethods, PySequenceMethods, PyBufferProcs)
+    PyNumberMethods, PyMappingMethods, PySequenceMethods, PyBufferProcs)
 from pypy.module.cpyext.slotdefs import (
     slotdefs_for_tp_slots, slotdefs_for_wrappers, get_slot_tp_function)
 from pypy.interpreter.error import OperationError
@@ -39,6 +39,19 @@
 
 PyType_Check, PyType_CheckExact = build_type_checkers("Type", "w_type")
 
+PyHeapTypeObjectStruct = lltype.ForwardReference()
+PyHeapTypeObject = lltype.Ptr(PyHeapTypeObjectStruct)
+PyHeapTypeObjectFields = (
+    ("ht_type", PyTypeObject),
+    ("ht_name", PyObject),
+    ("as_number", PyNumberMethods),
+    ("as_mapping", PyMappingMethods),
+    ("as_sequence", PySequenceMethods),
+    ("as_buffer", PyBufferProcs),
+    )
+cpython_struct("PyHeapTypeObject", PyHeapTypeObjectFields, PyHeapTypeObjectStruct,
+               level=2)
+
 class W_GetSetPropertyEx(GetSetProperty):
     def __init__(self, getset, w_type):
         self.getset = getset
@@ -136,6 +149,8 @@
             assert len(slot_names) == 2
             struct = getattr(pto, slot_names[0])
             if not struct:
+                assert not space.config.translating
+                assert not pto.c_tp_flags & Py_TPFLAGS_HEAPTYPE
                 if slot_names[0] == 'c_tp_as_number':
                     STRUCT_TYPE = PyNumberMethods
                 elif slot_names[0] == 'c_tp_as_sequence':
@@ -301,6 +316,7 @@
 
     make_typedescr(space.w_type.instancetypedef,
                    basestruct=PyTypeObject,
+                   alloc=type_alloc,
                    attach=type_attach,
                    realize=type_realize,
                    dealloc=type_dealloc)
@@ -319,11 +335,13 @@
     track_reference(space, lltype.nullptr(PyObject.TO), space.w_type)
     track_reference(space, lltype.nullptr(PyObject.TO), space.w_object)
     track_reference(space, lltype.nullptr(PyObject.TO), space.w_tuple)
+    track_reference(space, lltype.nullptr(PyObject.TO), space.w_str)
 
     # create the objects
     py_type = create_ref(space, space.w_type)
     py_object = create_ref(space, space.w_object)
     py_tuple = create_ref(space, space.w_tuple)
+    py_str = create_ref(space, space.w_str)
 
     # form cycles
     pto_type = rffi.cast(PyTypeObjectPtr, py_type)
@@ -340,10 +358,15 @@
     pto_object.c_tp_bases.c_ob_type = pto_tuple
     pto_tuple.c_tp_bases.c_ob_type = pto_tuple
 
+    for typ in (py_type, py_object, py_tuple, py_str):
+        heaptype = rffi.cast(PyHeapTypeObject, typ)
+        heaptype.c_ht_name.c_ob_type = pto_type
+
     # Restore the mapping
     track_reference(space, py_type, space.w_type, replace=True)
     track_reference(space, py_object, space.w_object, replace=True)
     track_reference(space, py_tuple, space.w_tuple, replace=True)
+    track_reference(space, py_str, space.w_str, replace=True)
 
 
 @cpython_api([PyObject], lltype.Void, external=False)
@@ -416,17 +439,34 @@
     Py_DecRef(space, obj_pto.c_tp_cache) # let's do it like cpython
     Py_DecRef(space, obj_pto.c_tp_dict)
     if obj_pto.c_tp_flags & Py_TPFLAGS_HEAPTYPE:
-        if obj_pto.c_tp_as_buffer:
-            lltype.free(obj_pto.c_tp_as_buffer, flavor='raw')
-        if obj_pto.c_tp_as_number:
-            lltype.free(obj_pto.c_tp_as_number, flavor='raw')
-        if obj_pto.c_tp_as_sequence:
-            lltype.free(obj_pto.c_tp_as_sequence, flavor='raw')
+        heaptype = rffi.cast(PyHeapTypeObject, obj)
+        Py_DecRef(space, heaptype.c_ht_name)
         Py_DecRef(space, base_pyo)
-        rffi.free_charp(obj_pto.c_tp_name)
         PyObject_dealloc(space, obj)
 
 
+def type_alloc(space, w_metatype):
+    size = rffi.sizeof(PyHeapTypeObject)
+    metatype = rffi.cast(PyTypeObjectPtr, make_ref(space, w_metatype))
+    # Don't increase refcount for non-heaptypes
+    if metatype:
+        flags = rffi.cast(lltype.Signed, metatype.c_tp_flags)
+        if not flags & Py_TPFLAGS_HEAPTYPE:
+            Py_DecRef(space, w_metatype)
+
+    heaptype = lltype.malloc(PyHeapTypeObject.TO,
+                             flavor='raw', zero=True)
+    pto = heaptype.c_ht_type
+    pto.c_ob_refcnt = 1
+    pto.c_ob_type = metatype
+    pto.c_tp_flags |= Py_TPFLAGS_HEAPTYPE
+    pto.c_tp_as_number = heaptype.c_as_number
+    pto.c_tp_as_sequence = heaptype.c_as_sequence
+    pto.c_tp_as_mapping = heaptype.c_as_mapping
+    pto.c_tp_as_buffer = heaptype.c_as_buffer
+    
+    return rffi.cast(PyObject, heaptype)
+
 def type_attach(space, py_obj, w_type):
     """
     Fills a newly allocated PyTypeObject from an existing type.
@@ -445,12 +485,18 @@
     if space.is_w(w_type, space.w_str):
         setup_string_buffer_procs(space, pto)
 
-    pto.c_tp_flags |= Py_TPFLAGS_HEAPTYPE
     pto.c_tp_free = llhelper(PyObject_Del.api_func.functype,
             PyObject_Del.api_func.get_wrapper(space))
     pto.c_tp_alloc = llhelper(PyType_GenericAlloc.api_func.functype,
             PyType_GenericAlloc.api_func.get_wrapper(space))
-    pto.c_tp_name = rffi.str2charp(w_type.getname(space))
+    if pto.c_tp_flags & Py_TPFLAGS_HEAPTYPE:
+        w_typename = space.getattr(w_type, space.wrap('__name__'))
+        heaptype = rffi.cast(PyHeapTypeObject, pto)
+        heaptype.c_ht_name = make_ref(space, w_typename)
+        from pypy.module.cpyext.stringobject import PyString_AsString
+        pto.c_tp_name = PyString_AsString(space, heaptype.c_ht_name)
+    else:
+        pto.c_tp_name = rffi.str2charp(w_type.getname(space))
     pto.c_tp_basicsize = -1 # hopefully this makes malloc bail out
     pto.c_tp_itemsize = 0
     # uninitialized fields:
diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py
--- a/pypy/module/pypyjit/interp_jit.py
+++ b/pypy/module/pypyjit/interp_jit.py
@@ -13,7 +13,6 @@
 from pypy.interpreter.pyframe import PyFrame
 from pypy.interpreter.pyopcode import ExitFrame
 from pypy.interpreter.gateway import unwrap_spec
-from pypy.interpreter.baseobjspace import ObjSpace, W_Root
 from opcode import opmap
 from pypy.rlib.nonconst import NonConstant
 from pypy.jit.metainterp.resoperation import rop
@@ -221,7 +220,6 @@
     def __init__(self, space):
         self.w_compile_hook = space.w_None
 
- at unwrap_spec(ObjSpace, W_Root)
 def set_compile_hook(space, w_hook):
     """ set_compile_hook(hook)
 
diff --git a/pypy/module/pypyjit/test/test_policy.py b/pypy/module/pypyjit/test/test_policy.py
--- a/pypy/module/pypyjit/test/test_policy.py
+++ b/pypy/module/pypyjit/test/test_policy.py
@@ -3,8 +3,8 @@
 pypypolicy = policy.PyPyJitPolicy()
 
 def test_id_any():
-    from pypy.objspace.std.default import id__ANY
-    assert pypypolicy.look_inside_function(id__ANY)
+    from pypy.objspace.std.intobject import add__Int_Int
+    assert pypypolicy.look_inside_function(add__Int_Int)
 
 def test_bigint():
     from pypy.rlib.rbigint import rbigint
diff --git a/pypy/module/pypyjit/test_pypy_c/test_containers.py b/pypy/module/pypyjit/test_pypy_c/test_containers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_containers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_containers.py
@@ -49,3 +49,24 @@
             p33 = call(ConstClass(ll_get_value__dicttablePtr_Signed), p18, i28, descr=...)
             ...
         """)
+
+    def test_list(self):
+        def main(n):
+            i = 0
+            while i < n:
+                z = list(())
+                z.append(1)
+                i += z[-1] / len(z)
+            return i
+
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loop, = log.loops_by_filename(self.filepath)
+        assert loop.match("""
+            i7 = int_lt(i5, i6)
+            guard_true(i7, descr=...)
+            guard_not_invalidated(descr=...)
+            i9 = int_add(i5, 1)
+            --TICK--
+            jump(..., descr=...)
+        """)
\ No newline at end of file
diff --git a/pypy/module/pypyjit/test_pypy_c/test_min_max.py b/pypy/module/pypyjit/test_pypy_c/test_min_max.py
--- a/pypy/module/pypyjit/test_pypy_c/test_min_max.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_min_max.py
@@ -42,7 +42,7 @@
         assert len(guards) < 20
         assert loop.match_by_id('max',"""
             ...
-            p76 = call_may_force(ConstClass(min_max_loop__max), _, _, descr=...)
+            p76 = call_may_force(ConstClass(min_max_trampoline), _, _, descr=...)
             ...
         """)
 
@@ -63,6 +63,6 @@
         assert len(guards) < 20
         assert loop.match_by_id('max',"""
             ...
-            p76 = call_may_force(ConstClass(min_max_loop__max), _, _, descr=...)
+            p76 = call_may_force(ConstClass(min_max_trampoline), _, _, descr=...)
             ...
         """)
diff --git a/pypy/module/test_lib_pypy/test_greenlet.py b/pypy/module/test_lib_pypy/test_greenlet.py
--- a/pypy/module/test_lib_pypy/test_greenlet.py
+++ b/pypy/module/test_lib_pypy/test_greenlet.py
@@ -258,3 +258,25 @@
             assert sys.exc_info() == (None, None, None)
 
         greenlet(f).switch()
+
+    def test_gr_frame(self):
+        from greenlet import greenlet
+        import sys
+        def f2():
+            assert g.gr_frame is None
+            gmain.switch()
+            assert g.gr_frame is None
+        def f1():
+            assert gmain.gr_frame is gmain_frame
+            assert g.gr_frame is None
+            f2()
+            assert g.gr_frame is None
+        gmain = greenlet.getcurrent()
+        assert gmain.gr_frame is None
+        gmain_frame = sys._getframe()
+        g = greenlet(f1)
+        assert g.gr_frame is None
+        g.switch()
+        assert g.gr_frame.f_code.co_name == 'f2'
+        g.switch()
+        assert g.gr_frame is None
diff --git a/pypy/module/test_lib_pypy/test_stackless_pickle.py b/pypy/module/test_lib_pypy/test_stackless_pickle.py
--- a/pypy/module/test_lib_pypy/test_stackless_pickle.py
+++ b/pypy/module/test_lib_pypy/test_stackless_pickle.py
@@ -1,25 +1,27 @@
-import py; py.test.skip("XXX port me")
+import py
+py.test.skip("in-progress, maybe")
 from pypy.conftest import gettestobjspace, option
 
 class AppTest_Stackless:
 
     def setup_class(cls):
-        import py.test
-        py.test.importorskip('greenlet')
-        space = gettestobjspace(usemodules=('_stackless', '_socket'))
+        space = gettestobjspace(usemodules=('_continuation', '_socket'))
         cls.space = space
-        # cannot test the unpickle part on top of py.py
+        if option.runappdirect:
+            cls.w_lev = space.wrap(14)
+        else:
+            cls.w_lev = space.wrap(2)
 
     def test_pickle(self):
         import new, sys
 
         mod = new.module('mod')
         sys.modules['mod'] = mod
+        mod.lev = self.lev
         try:
             exec '''
 import pickle, sys
 import stackless
-lev = 14
 
 ch = stackless.channel()
 seen = []
diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -6,6 +6,7 @@
 from pypy.interpreter.typedef import default_identity_hash
 from pypy.tool.sourcetools import compile2, func_with_new_name
 from pypy.module.__builtin__.interp_classobj import W_InstanceObject
+from pypy.rlib.objectmodel import specialize
 
 def object_getattribute(space):
     "Utility that returns the app-level descriptor object.__getattribute__."
@@ -507,6 +508,7 @@
     def issubtype(space, w_sub, w_type):
         return space._type_issubtype(w_sub, w_type)
 
+    @specialize.arg_or_var(2)
     def isinstance(space, w_inst, w_type):
         return space.wrap(space._type_isinstance(w_inst, w_type))
 
diff --git a/pypy/objspace/std/boolobject.py b/pypy/objspace/std/boolobject.py
--- a/pypy/objspace/std/boolobject.py
+++ b/pypy/objspace/std/boolobject.py
@@ -1,8 +1,10 @@
+from pypy.rlib.rbigint import rbigint
+from pypy.rlib.rarithmetic import r_uint
+from pypy.interpreter.error import OperationError
 from pypy.objspace.std.model import registerimplementation, W_Object
 from pypy.objspace.std.register_all import register_all
 from pypy.objspace.std.intobject import W_IntObject
 
-
 class W_BoolObject(W_Object):
     from pypy.objspace.std.booltype import bool_typedef as typedef
     _immutable_fields_ = ['boolval']
@@ -20,6 +22,21 @@
     def unwrap(w_self, space):
         return w_self.boolval
 
+    def int_w(w_self, space):
+        return int(w_self.boolval)
+
+    def uint_w(w_self, space):
+        intval = int(w_self.boolval)
+        if intval < 0:
+            raise OperationError(space.w_ValueError,
+                                 space.wrap("cannot convert negative integer to unsigned"))
+        else:
+            return r_uint(intval)
+
+    def bigint_w(w_self, space):
+        return rbigint.fromint(int(w_self.boolval))
+
+
 registerimplementation(W_BoolObject)
 
 W_BoolObject.w_False = W_BoolObject(False)
diff --git a/pypy/objspace/std/default.py b/pypy/objspace/std/default.py
--- a/pypy/objspace/std/default.py
+++ b/pypy/objspace/std/default.py
@@ -1,48 +1,16 @@
 """Default implementation for some operation."""
 
-from pypy.interpreter.error import OperationError
+from pypy.interpreter.error import OperationError, typed_unwrap_error_msg
 from pypy.objspace.std.register_all import register_all
-from pypy.rlib import objectmodel
 
 
-# The following default implementations are used before delegation is tried.
-# 'id' is normally the address of the wrapper.
-
-def id__ANY(space, w_obj):
-    #print 'id:', w_obj
-    return space.wrap(objectmodel.compute_unique_id(w_obj))
-
 # __init__ should succeed if called internally as a multimethod
 
 def init__ANY(space, w_obj, __args__):
     pass
 
-def typed_unwrap_error_msg(space, expected, w_obj):
-    type_name = space.type(w_obj).getname(space)
-    return space.wrap("expected %s, got %s object" % (expected, type_name))
-
-def int_w__ANY(space,w_obj):
-    raise OperationError(space.w_TypeError,
-                         typed_unwrap_error_msg(space, "integer", w_obj))
-
-def str_w__ANY(space,w_obj):
-    raise OperationError(space.w_TypeError,
-                         typed_unwrap_error_msg(space, "string", w_obj))
-
 def float_w__ANY(space,w_obj):
     raise OperationError(space.w_TypeError,
                          typed_unwrap_error_msg(space, "float", w_obj))
 
-def uint_w__ANY(space,w_obj):
-    raise OperationError(space.w_TypeError,
-                         typed_unwrap_error_msg(space, "integer", w_obj))
-
-def unicode_w__ANY(space,w_obj):
-    raise OperationError(space.w_TypeError,
-                         typed_unwrap_error_msg(space, "unicode", w_obj))
-
-def bigint_w__ANY(space,w_obj):
-    raise OperationError(space.w_TypeError,
-                         typed_unwrap_error_msg(space, "integer", w_obj))
-
 register_all(vars())
diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py
--- a/pypy/objspace/std/intobject.py
+++ b/pypy/objspace/std/intobject.py
@@ -1,12 +1,13 @@
 from pypy.interpreter.error import OperationError
 from pypy.objspace.std import newformat
+from pypy.objspace.std.inttype import wrapint
 from pypy.objspace.std.model import registerimplementation, W_Object
-from pypy.objspace.std.register_all import register_all
 from pypy.objspace.std.multimethod import FailedToImplementArgs
 from pypy.objspace.std.noneobject import W_NoneObject
+from pypy.objspace.std.register_all import register_all
+from pypy.rlib import jit
 from pypy.rlib.rarithmetic import ovfcheck, ovfcheck_lshift, LONG_BIT, r_uint
 from pypy.rlib.rbigint import rbigint
-from pypy.objspace.std.inttype import wrapint
 
 """
 In order to have the same behavior running
@@ -20,7 +21,7 @@
     _immutable_fields_ = ['intval']
 
     from pypy.objspace.std.inttype import int_typedef as typedef
-    
+
     def __init__(w_self, intval):
         w_self.intval = intval
 
@@ -30,7 +31,18 @@
 
     def unwrap(w_self, space):
         return int(w_self.intval)
+    int_w = unwrap
 
+    def uint_w(w_self, space):
+        intval = w_self.intval
+        if intval < 0:
+            raise OperationError(space.w_ValueError,
+                                 space.wrap("cannot convert negative integer to unsigned"))
+        else:
+            return r_uint(intval)
+
+    def bigint_w(w_self, space):
+        return rbigint.fromint(w_self.intval)
 
 registerimplementation(W_IntObject)
 
@@ -39,20 +51,6 @@
 # alias and then teach copy_multimethods in smallintobject.py to override
 # it. See int__Int for example.
 
-def int_w__Int(space, w_int1):
-    return int(w_int1.intval)
-
-def uint_w__Int(space, w_int1):
-    intval = w_int1.intval
-    if intval < 0:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("cannot convert negative integer to unsigned"))
-    else:
-        return r_uint(intval)
-
-def bigint_w__Int(space, w_int1):
-    return rbigint.fromint(w_int1.intval)
-
 def repr__Int(space, w_int1):
     a = w_int1.intval
     res = str(a)
@@ -138,7 +136,7 @@
     x = float(w_int1.intval)
     y = float(w_int2.intval)
     if y == 0.0:
-        raise FailedToImplementArgs(space.w_ZeroDivisionError, space.wrap("float division"))    
+        raise FailedToImplementArgs(space.w_ZeroDivisionError, space.wrap("float division"))
     return space.wrap(x / y)
 
 def mod__Int_Int(space, w_int1, w_int2):
@@ -172,7 +170,8 @@
 
 
 # helper for pow()
-def _impl_int_int_pow(space, iv, iw, iz=0):
+ at jit.look_inside_iff(lambda space, iv, iw, iz: jit.isconstant(iw) and jit.isconstant(iz))
+def _impl_int_int_pow(space, iv, iw, iz):
     if iw < 0:
         if iz != 0:
             raise OperationError(space.w_TypeError,
@@ -200,7 +199,7 @@
     except OverflowError:
         raise FailedToImplementArgs(space.w_OverflowError,
                                 space.wrap("integer exponentiation"))
-    return wrapint(space, ix)
+    return ix
 
 def pow__Int_Int_Int(space, w_int1, w_int2, w_int3):
     x = w_int1.intval
@@ -209,12 +208,12 @@
     if z == 0:
         raise OperationError(space.w_ValueError,
                              space.wrap("pow() 3rd argument cannot be 0"))
-    return _impl_int_int_pow(space, x, y, z)
+    return space.wrap(_impl_int_int_pow(space, x, y, z))
 
 def pow__Int_Int_None(space, w_int1, w_int2, w_int3):
     x = w_int1.intval
     y = w_int2.intval
-    return _impl_int_int_pow(space, x, y)
+    return space.wrap(_impl_int_int_pow(space, x, y, 0))
 
 def neg__Int(space, w_int1):
     a = w_int1.intval
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -386,7 +386,11 @@
     if len(items)== 0:
         raise OperationError(space.w_IndexError,
                              space.wrap("pop from empty list"))
-    idx = space.int_w(w_idx)
+    if space.isinstance_w(w_idx, space.w_float):
+        raise OperationError(space.w_TypeError,
+            space.wrap("integer argument expected, got float")
+        )
+    idx = space.int_w(space.int(w_idx))
     try:
         return items.pop(idx)
     except IndexError:
diff --git a/pypy/objspace/std/longobject.py b/pypy/objspace/std/longobject.py
--- a/pypy/objspace/std/longobject.py
+++ b/pypy/objspace/std/longobject.py
@@ -45,6 +45,26 @@
     fromrarith_int._annspecialcase_ = "specialize:argtype(0)"
     fromrarith_int = staticmethod(fromrarith_int)
 
+    def int_w(w_self, space):
+        try:
+            return w_self.num.toint()
+        except OverflowError:
+            raise OperationError(space.w_OverflowError, space.wrap(
+                "long int too large to convert to int"))
+
+    def uint_w(w_self, space):
+        try:
+            return w_self.num.touint()
+        except ValueError:
+            raise OperationError(space.w_ValueError, space.wrap(
+                "cannot convert negative integer to unsigned int"))
+        except OverflowError:
+            raise OperationError(space.w_OverflowError, space.wrap(
+                "long int too large to convert to unsigned int"))
+
+    def bigint_w(w_self, space):
+        return w_self.num
+
     def __repr__(self):
         return '<W_LongObject(%d)>' % self.num.tolong()
 
@@ -104,27 +124,6 @@
         raise OperationError(space.w_OverflowError,
                              space.wrap("long int too large to convert to float"))
 
-def int_w__Long(space, w_value):
-    try:
-        return w_value.num.toint()
-    except OverflowError:
-        raise OperationError(space.w_OverflowError, space.wrap(
-            "long int too large to convert to int"))
-
-
-def uint_w__Long(space, w_value):
-    try:
-        return w_value.num.touint()
-    except ValueError:
-        raise OperationError(space.w_ValueError, space.wrap(
-            "cannot convert negative integer to unsigned int"))
-    except OverflowError:
-        raise OperationError(space.w_OverflowError, space.wrap(
-            "long int too large to convert to unsigned int"))
-
-def bigint_w__Long(space, w_value):
-    return w_value.num
-
 def repr__Long(space, w_long):
     return space.wrap(w_long.num.repr())
 
diff --git a/pypy/objspace/std/model.py b/pypy/objspace/std/model.py
--- a/pypy/objspace/std/model.py
+++ b/pypy/objspace/std/model.py
@@ -442,6 +442,13 @@
         mm.dispatch_tree = merge(self.dispatch_tree, other.dispatch_tree)
         return mm
 
+NOT_MULTIMETHODS = dict.fromkeys(
+    ['delattr', 'delete', 'get', 'id', 'inplace_div', 'inplace_floordiv',
+     'inplace_lshift', 'inplace_mod', 'inplace_pow', 'inplace_rshift',
+     'inplace_truediv', 'is_', 'set', 'setattr', 'type', 'userdel',
+     'isinstance', 'issubtype'])
+# XXX should we just remove those from the method table or we're happy
+#     with just not having multimethods?
 
 class MM:
     """StdObjSpace multimethods"""
@@ -451,22 +458,17 @@
     init    = StdObjSpaceMultiMethod('__init__', 1, general__args__=True)
     getnewargs = StdObjSpaceMultiMethod('__getnewargs__', 1)
     # special visible multimethods
-    int_w   = StdObjSpaceMultiMethod('int_w', 1, [])     # returns an unwrapped int
-    str_w   = StdObjSpaceMultiMethod('str_w', 1, [])     # returns an unwrapped string
     float_w = StdObjSpaceMultiMethod('float_w', 1, [])   # returns an unwrapped float
-    uint_w  = StdObjSpaceMultiMethod('uint_w', 1, [])    # returns an unwrapped unsigned int (r_uint)
-    unicode_w = StdObjSpaceMultiMethod('unicode_w', 1, [])    # returns an unwrapped list of unicode characters
-    bigint_w = StdObjSpaceMultiMethod('bigint_w', 1, []) # returns an unwrapped rbigint
     # NOTE: when adding more sometype_w() methods, you need to write a
     # stub in default.py to raise a space.w_TypeError
     marshal_w = StdObjSpaceMultiMethod('marshal_w', 1, [], extra_args=['marshaller'])
-    log     = StdObjSpaceMultiMethod('log', 1, [], extra_args=['base'])
 
     # add all regular multimethods here
     for _name, _symbol, _arity, _specialnames in ObjSpace.MethodTable:
-        if _name not in locals():
+        if _name not in locals() or _name in NOT_MULTIMETHODS:
             mm = StdObjSpaceMultiMethod(_symbol, _arity, _specialnames)
             locals()[_name] = mm
             del mm
 
     pow.extras['defaults'] = (None,)
+
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -1,19 +1,17 @@
 import __builtin__
 import types
-from pypy.interpreter import pyframe, function, special
+from pypy.interpreter import special
 from pypy.interpreter.baseobjspace import ObjSpace, Wrappable
 from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.interpreter.typedef import get_unique_interplevel_subclass
 from pypy.objspace.std import (builtinshortcut, stdtypedef, frame, model,
                                transparent, callmethod, proxyobject)
 from pypy.objspace.descroperation import DescrOperation, raiseattrerror
-from pypy.rlib.objectmodel import instantiate, r_dict, specialize
+from pypy.rlib.objectmodel import instantiate, r_dict, specialize, is_annotation_constant
 from pypy.rlib.debug import make_sure_not_resized
 from pypy.rlib.rarithmetic import base_int, widen
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.rlib import jit
-from pypy.rlib.rbigint import rbigint
-from pypy.tool.sourcetools import func_with_new_name
 
 # Object imports
 from pypy.objspace.std.boolobject import W_BoolObject
@@ -85,6 +83,12 @@
         if self.config.objspace.std.withtproxy:
             transparent.setup(self)
 
+        for type, classes in self.model.typeorder.iteritems():
+            if len(classes) >= 3:
+                # W_Root, AnyXxx and actual object
+                self.gettypefor(type).interplevel_cls = classes[0][0]
+
+
     def get_builtin_types(self):
         return self.builtin_types
 
@@ -569,10 +573,19 @@
             return self.wrap(w_sub.issubtype(w_type))
         raise OperationError(self.w_TypeError, self.wrap("need type objects"))
 
+    @specialize.arg_or_var(2)
     def _type_isinstance(self, w_inst, w_type):
-        if isinstance(w_type, W_TypeObject):
-            return self.type(w_inst).issubtype(w_type)
-        raise OperationError(self.w_TypeError, self.wrap("need type object"))
+        if not isinstance(w_type, W_TypeObject):
+            raise OperationError(self.w_TypeError,
+                                 self.wrap("need type object"))
+        if is_annotation_constant(w_type):
+            cls = w_type.interplevel_cls
+            if cls is not None:
+                assert w_inst is not None
+                if isinstance(w_inst, cls):
+                    return True
+        return self.type(w_inst).issubtype(w_type)
 
+    @specialize.arg_or_var(2)
     def isinstance_w(space, w_inst, w_type):
         return space._type_isinstance(w_inst, w_type)
diff --git a/pypy/objspace/std/ropeobject.py b/pypy/objspace/std/ropeobject.py
--- a/pypy/objspace/std/ropeobject.py
+++ b/pypy/objspace/std/ropeobject.py
@@ -34,12 +34,18 @@
 
     def unwrap(w_self, space):
         return w_self._node.flatten_string()
+    str_w = unwrap
 
     def create_if_subclassed(w_self):
         if type(w_self) is W_RopeObject:
             return w_self
         return W_RopeObject(w_self._node)
 
+    def unicode_w(w_self, space):
+        # XXX should this use the default encoding?
+        from pypy.objspace.std.unicodetype import plain_str2unicode
+        return plain_str2unicode(space, w_self._node.flatten_string())
+
 W_RopeObject.EMPTY = W_RopeObject(rope.LiteralStringNode.EMPTY)
 W_RopeObject.PREBUILT = [W_RopeObject(rope.LiteralStringNode.PREBUILT[i])
                              for i in range(256)]
@@ -663,9 +669,6 @@
         return W_RopeObject(rope.concatenate(
             rope.multiply(zero, middle), node))
 
-def str_w__Rope(space, w_str):
-    return w_str._node.flatten_string()
-
 def hash__Rope(space, w_str):
     return wrapint(space, rope.hash_rope(w_str._node))
 
diff --git a/pypy/objspace/std/ropeunicodeobject.py b/pypy/objspace/std/ropeunicodeobject.py
--- a/pypy/objspace/std/ropeunicodeobject.py
+++ b/pypy/objspace/std/ropeunicodeobject.py
@@ -91,11 +91,17 @@
         # for testing
         return w_self._node.flatten_unicode()
 
+    def str_w(w_self, space):
+        return space.str_w(space.str(w_self))
+
     def create_if_subclassed(w_self):
         if type(w_self) is W_RopeUnicodeObject:
             return w_self
         return W_RopeUnicodeObject(w_self._node)
 
+    def unicode_w(self, space):
+        return self._node.flatten_unicode()
+
 W_RopeUnicodeObject.EMPTY = W_RopeUnicodeObject(rope.LiteralStringNode.EMPTY)
 
 registerimplementation(W_RopeUnicodeObject)
@@ -157,12 +163,6 @@
     assert isinstance(w_uni, W_RopeUnicodeObject) # help the annotator!
     return w_uni
 
-def str_w__RopeUnicode(space, w_uni):
-    return space.str_w(space.str(w_uni))
-
-def unicode_w__RopeUnicode(space, w_uni):
-    return w_uni._node.flatten_unicode()
-
 def str__RopeUnicode(space, w_uni):
     return space.call_method(w_uni, 'encode')
 
@@ -185,7 +185,7 @@
 
 def eq__RopeUnicode_Rope(space, w_runi, w_rope):
     from pypy.objspace.std.unicodeobject import _unicode_string_comparison
-    return _unicode_string_comparison(space, w_runi, w_rope, 
+    return _unicode_string_comparison(space, w_runi, w_rope,
                     False,  unicode_from_string)
 
 def ne__RopeUnicode_RopeUnicode(space, w_str1, w_str2):
@@ -193,7 +193,7 @@
 
 def ne__RopeUnicode_Rope(space, w_runi, w_rope):
     from pypy.objspace.std.unicodeobject import _unicode_string_comparison
-    return _unicode_string_comparison(space, w_runi, w_rope, 
+    return _unicode_string_comparison(space, w_runi, w_rope,
                     True, unicode_from_string)
 
 def gt__RopeUnicode_RopeUnicode(space, w_str1, w_str2):
@@ -247,7 +247,7 @@
     if (len(l_w) == 1 and
         space.is_w(space.type(l_w[0]), space.w_unicode)):
         return l_w[0]
-    
+
     values_list = []
     for i in range(len(l_w)):
         w_item = l_w[i]
@@ -320,7 +320,7 @@
 
 
 def make_generic(funcname):
-    def func(space, w_self): 
+    def func(space, w_self):
         node = w_self._node
         if node.length() == 0:
             return space.w_False
@@ -578,7 +578,7 @@
         return w_self.create_if_subclassed()
     resultnode = rope.concatenate(rope.multiply(fillchar, padding), self)
     return W_RopeUnicodeObject(resultnode)
-    
+
 def unicode_zfill__RopeUnicode_ANY(space, w_self, w_width):
     self = w_self._node
     length = self.length()
@@ -744,7 +744,7 @@
     except OverflowError:
         raise OperationError(space.w_OverflowError,
                              space.wrap("string too long"))
-    
+
 
 def unicode_encode__RopeUnicode_ANY_ANY(space, w_unistr,
                                         w_encoding=None,
@@ -821,7 +821,7 @@
         try:
             w_newval = space.getitem(w_table, space.wrap(char))
         except OperationError, e:
-            if e.match(space, space.w_KeyError):
+            if e.match(space, space.w_LookupError):
                 result.append(crope)
             else:
                 raise
@@ -848,7 +848,7 @@
     hexdigits = "0123456789abcdef"
     node = w_unicode._node
     size = node.length()
-    
+
     singlequote = doublequote = False
     iter = rope.ItemIterator(node)
     for i in range(size):
@@ -900,7 +900,7 @@
                                   ])
                     j += 2
                     continue
-                
+
         if code >= 0x100:
             result.extend(['\\', "u",
                            hexdigits[(code >> 12) & 0xf],
@@ -932,7 +932,7 @@
             continue
         if code < ord(' ') or code >= 0x7f:
             result.extend(['\\', "x",
-                           hexdigits[(code >> 4) & 0xf], 
+                           hexdigits[(code >> 4) & 0xf],
                            hexdigits[(code >> 0) & 0xf],
                           ])
             j += 1
@@ -964,15 +964,15 @@
 
 def next__RopeUnicodeIter(space, w_ropeiter):
     if w_ropeiter.node is None:
-        raise OperationError(space.w_StopIteration, space.w_None) 
+        raise OperationError(space.w_StopIteration, space.w_None)
     try:
         unichar = w_ropeiter.item_iter.nextunichar()
         w_item = space.wrap(unichar)
     except StopIteration:
         w_ropeiter.node = None
         w_ropeiter.char_iter = None
-        raise OperationError(space.w_StopIteration, space.w_None) 
-    w_ropeiter.index += 1 
+        raise OperationError(space.w_StopIteration, space.w_None)
+    w_ropeiter.index += 1
     return w_item
 
 # XXX __length_hint__()
diff --git a/pypy/objspace/std/smallintobject.py b/pypy/objspace/std/smallintobject.py
--- a/pypy/objspace/std/smallintobject.py
+++ b/pypy/objspace/std/smallintobject.py
@@ -7,16 +7,30 @@
 from pypy.objspace.std.register_all import register_all
 from pypy.objspace.std.noneobject import W_NoneObject
 from pypy.objspace.std.intobject import W_IntObject
+from pypy.interpreter.error import OperationError
 from pypy.rlib.objectmodel import UnboxedValue
+from pypy.rlib.rbigint import rbigint
+from pypy.rlib.rarithmetic import r_uint
 from pypy.tool.sourcetools import func_with_new_name
 
-
 class W_SmallIntObject(W_Object, UnboxedValue):
     __slots__ = 'intval'
     from pypy.objspace.std.inttype import int_typedef as typedef
 
     def unwrap(w_self, space):
         return int(w_self.intval)
+    int_w = unwrap
+
+    def uint_w(w_self, space):
+        intval = w_self.intval
+        if intval < 0:
+            raise OperationError(space.w_ValueError,
+                                 space.wrap("cannot convert negative integer to unsigned"))
+        else:
+            return r_uint(intval)
+
+    def bigint_w(w_self, space):
+        return rbigint.fromint(w_self.intval)
 
 
 registerimplementation(W_SmallIntObject)
diff --git a/pypy/objspace/std/smalllongobject.py b/pypy/objspace/std/smalllongobject.py
--- a/pypy/objspace/std/smalllongobject.py
+++ b/pypy/objspace/std/smalllongobject.py
@@ -39,6 +39,30 @@
     def __repr__(w_self):
         return '<W_SmallLongObject(%d)>' % w_self.longlong
 
+    def int_w(w_self, space):
+        a = w_self.longlong
+        b = intmask(a)
+        if b == a:
+            return b
+        else:
+            raise OperationError(space.w_OverflowError, space.wrap(
+                "long int too large to convert to int"))
+
+    def uint_w(w_self, space):
+        a = w_self.longlong
+        if a < 0:
+            raise OperationError(space.w_ValueError, space.wrap(
+                "cannot convert negative integer to unsigned int"))
+        b = r_uint(a)
+        if r_longlong(b) == a:
+            return b
+        else:
+            raise OperationError(space.w_OverflowError, space.wrap(
+                "long int too large to convert to unsigned int"))
+
+    def bigint_w(w_self, space):
+        return w_self.asbigint()
+
 registerimplementation(W_SmallLongObject)
 
 # ____________________________________________________________
@@ -102,30 +126,6 @@
 def float__SmallLong(space, w_value):
     return space.newfloat(float(w_value.longlong))
 
-def int_w__SmallLong(space, w_value):
-    a = w_value.longlong
-    b = intmask(a)
-    if b == a:
-        return b
-    else:
-        raise OperationError(space.w_OverflowError, space.wrap(
-            "long int too large to convert to int"))
-
-def uint_w__SmallLong(space, w_value):
-    a = w_value.longlong
-    if a < 0:
-        raise OperationError(space.w_ValueError, space.wrap(
-            "cannot convert negative integer to unsigned int"))
-    b = r_uint(a)
-    if r_longlong(b) == a:
-        return b
-    else:
-        raise OperationError(space.w_OverflowError, space.wrap(
-            "long int too large to convert to unsigned int"))
-
-def bigint_w__SmallLong(space, w_value):
-    return w_value.asbigint()
-
 def lt__SmallLong_SmallLong(space, w_small1, w_small2):
     return space.newbool(w_small1.longlong <  w_small2.longlong)
 def le__SmallLong_SmallLong(space, w_small1, w_small2):
diff --git a/pypy/objspace/std/strbufobject.py b/pypy/objspace/std/strbufobject.py
--- a/pypy/objspace/std/strbufobject.py
+++ b/pypy/objspace/std/strbufobject.py
@@ -32,6 +32,9 @@
     def unwrap(self, space):
         return self.force()
 
+    def str_w(self, space):
+        return self.force()
+
 registerimplementation(W_StringBufferObject)
 
 # ____________________________________________________________
@@ -55,9 +58,6 @@
 def len__StringBuffer(space, w_self):
     return space.wrap(w_self.length)
 
-def str_w__StringBuffer(space, w_strbuf):
-    return w_strbuf.force()
-
 def add__StringBuffer_String(space, w_self, w_other):
     if w_self.builder.getlength() != w_self.length:
         builder = StringBuilder()
diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py
--- a/pypy/objspace/std/stringobject.py
+++ b/pypy/objspace/std/stringobject.py
@@ -33,17 +33,20 @@
     def unwrap(w_self, space):
         return w_self._value
 
+    def str_w(w_self, space):
+        return w_self._value
+
+    def unicode_w(w_self, space):
+        # XXX should this use the default encoding?
+        from pypy.objspace.std.unicodetype import plain_str2unicode
+        return plain_str2unicode(space, w_self._value)
+
 registerimplementation(W_StringObject)
 
 W_StringObject.EMPTY = W_StringObject('')
 W_StringObject.PREBUILT = [W_StringObject(chr(i)) for i in range(256)]
 del i
 
-def unicode_w__String(space, w_self):
-    # XXX should this use the default encoding?
-    from pypy.objspace.std.unicodetype import plain_str2unicode
-    return plain_str2unicode(space, w_self._value)
-
 def _is_generic(space, w_self, fun):
     v = w_self._value
     if len(v) == 0:
@@ -773,8 +776,6 @@
 
     return space.wrap("".join(buf))
 
-def str_w__String(space, w_str):
-    return w_str._value
 
 def hash__String(space, w_str):
     s = w_str._value
diff --git a/pypy/objspace/std/strjoinobject.py b/pypy/objspace/std/strjoinobject.py
--- a/pypy/objspace/std/strjoinobject.py
+++ b/pypy/objspace/std/strjoinobject.py
@@ -29,6 +29,7 @@
 
     def unwrap(w_self, space):
         return w_self.force()
+    str_w = unwrap
 
 registerimplementation(W_StringJoinObject)
 
@@ -45,9 +46,6 @@
         result += len(w_self.joined_strs[i])
     return space.wrap(result)
 
-def str_w__StringJoin(space, w_str):
-    return w_str.force()
-
 def add__StringJoin_StringJoin(space, w_self, w_other):
     if len(w_self.joined_strs) > w_self.until:
         w_self.force(True)
diff --git a/pypy/objspace/std/strsliceobject.py b/pypy/objspace/std/strsliceobject.py
--- a/pypy/objspace/std/strsliceobject.py
+++ b/pypy/objspace/std/strsliceobject.py
@@ -31,6 +31,9 @@
         w_self.stop = len(str)
         return str
 
+    def str_w(w_self, space):
+        return w_self.force()
+
     def __repr__(w_self):
         """ representation for debugging purposes """
         return "%s(%r[%d:%d])" % (w_self.__class__.__name__,
@@ -165,11 +168,6 @@
             return space.w_True
     return space.w_False
 
-
-def str_w__StringSlice(space, w_str):
-    return w_str.force()
-
-
 def getitem__StringSlice_ANY(space, w_str, w_index):
     ival = space.getindex_w(w_index, space.w_IndexError, "string index")
     slen = w_str.stop - w_str.start
diff --git a/pypy/objspace/std/test/test_boolobject.py b/pypy/objspace/std/test/test_boolobject.py
--- a/pypy/objspace/std/test/test_boolobject.py
+++ b/pypy/objspace/std/test/test_boolobject.py
@@ -17,6 +17,12 @@
         
     def test_false(self):
         assert not self.space.is_true(self.false)
+
+    def test_uint_w(self):
+        assert self.space.uint_w(self.true) == 1
+
+    def test_rbigint_w(self):
+        assert self.space.bigint_w(self.true)._digits == [1]
         
 class AppTestAppBoolTest:
     def test_bool_callable(self):
diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py
--- a/pypy/objspace/std/test/test_listobject.py
+++ b/pypy/objspace/std/test/test_listobject.py
@@ -705,6 +705,20 @@
         l.pop()
         assert l == range(9)
 
+    def test_pop_custom_int(self):
+        class A(object):
+            def __init__(self, x):
+                self.x = x
+
+            def __int__(self):
+                return self.x
+
+        l = range(10)
+        x = l.pop(A(-1))
+        assert x == 9
+        assert l == range(9)
+        raises(TypeError, range(10).pop, 1.0)
+
     def test_remove(self):
         c = list('hello world')
         c.remove('l')
diff --git a/pypy/objspace/std/test/test_stdobjspace.py b/pypy/objspace/std/test/test_stdobjspace.py
--- a/pypy/objspace/std/test/test_stdobjspace.py
+++ b/pypy/objspace/std/test/test_stdobjspace.py
@@ -46,3 +46,17 @@
         assert space.sliceindices(w_slice, w(3)) == (1,2,1)
         assert space.sliceindices(w_obj, w(3)) == (1,2,3)
 
+    def test_fastpath_isinstance(self):
+        from pypy.objspace.std.stringobject import W_StringObject
+        from pypy.objspace.std.intobject import W_IntObject
+        
+        space = self.space
+        assert space.w_str.interplevel_cls is W_StringObject
+        assert space.w_int.interplevel_cls is W_IntObject
+        class X(W_StringObject):
+            def __init__(self):
+                pass
+            
+            typedef = None
+
+        assert space.isinstance_w(X(), space.w_str)
diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -126,6 +126,12 @@
         raises(TypeError, type, 'test', 42, {})
         raises(TypeError, type, 'test', (object,), 42)
 
+    def test_call_type_subclass(self):
+        class A(type):
+            pass
+
+        assert A("hello") is str
+
     def test_bases(self):
         assert int.__bases__ == (object,)
         class X:
diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py
--- a/pypy/objspace/std/test/test_unicodeobject.py
+++ b/pypy/objspace/std/test/test_unicodeobject.py
@@ -443,6 +443,8 @@
         assert u'<i><i><i>c' == u'abababc'.translate({ord('a'):None, ord('b'):u'<i>'})
         assert u'c' == u'abababc'.translate({ord('a'):None, ord('b'):u''})
         assert u'xyyx' == u'xzx'.translate({ord('z'):u'yy'})
+        assert u'abcd' == u'ab\0d'.translate(u'c')
+        assert u'abcd' == u'abcd'.translate(u'')
 
         raises(TypeError, u'hello'.translate)
         raises(TypeError, u'abababc'.translate, {ord('a'):''})
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -115,6 +115,9 @@
     # of the __new__ is an instance of the type
     w_bltin_new = None
 
+    interplevel_cls = None # not None for prebuilt instances of
+                           # interpreter-level types
+
     @dont_look_inside
     def __init__(w_self, space, name, bases_w, dict_w,
                  overridetypedef=None):
@@ -819,14 +822,6 @@
 
 def call__Type(space, w_type, __args__):
     promote(w_type)
-    # special case for type(x)
-    if space.is_w(w_type, space.w_type):
-        try:
-            w_obj, = __args__.fixedunpack(1)
-        except ValueError:
-            pass
-        else:
-            return space.type(w_obj)
     # invoke the __new__ of the type
     if not we_are_jitted():
         # note that the annotator will figure out that w_type.w_bltin_new can
diff --git a/pypy/objspace/std/typetype.py b/pypy/objspace/std/typetype.py
--- a/pypy/objspace/std/typetype.py
+++ b/pypy/objspace/std/typetype.py
@@ -1,17 +1,28 @@
-from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.interpreter import gateway
 from pypy.interpreter.argument import Arguments
+from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.interpreter.typedef import (GetSetProperty, descr_get_dict,
                                       weakref_descr)
 from pypy.objspace.std.stdtypedef import StdTypeDef
 
-def descr__new__(space, w_typetype, w_name, w_bases, w_dict):
+
+def descr__new__(space, w_typetype, w_name, w_bases=gateway.NoneNotWrapped,
+    w_dict=gateway.NoneNotWrapped):
+
     "This is used to create user-defined classes only."
     from pypy.objspace.std.typeobject import W_TypeObject
     # XXX check types
 
     w_typetype = _precheck_for_new(space, w_typetype)
 
+    # special case for type(x)
+    if (space.is_w(space.type(w_typetype), space.w_type) and w_bases is None and
+        w_dict is None):
+        return space.type(w_name)
+    elif w_bases is None or w_dict is None:
+        raise OperationError(space.w_TypeError, space.wrap("type() takes 1 or 3 arguments"))
+
+
     bases_w = space.fixedview(w_bases)
 
     w_winner = w_typetype
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -40,6 +40,12 @@
             return w_self
         return W_UnicodeObject(w_self._value)
 
+    def str_w(self, space):
+        return space.str_w(space.str(self))
+
+    def unicode_w(self, space):
+        return self._value
+
 W_UnicodeObject.EMPTY = W_UnicodeObject(u'')
 
 registerimplementation(W_UnicodeObject)
@@ -99,12 +105,6 @@
         return space.not_(result)
     return result
 
-def str_w__Unicode(space, w_uni):
-    return space.str_w(str__Unicode(space, w_uni))
-
-def unicode_w__Unicode(space, w_uni):
-    return w_uni._value
-
 def str__Unicode(space, w_uni):
     from pypy.objspace.std.unicodetype import encode_object
     return encode_object(space, w_uni, None, None)
@@ -893,7 +893,7 @@
         try:
             w_newval = space.getitem(w_table, space.wrap(ord(unichar)))
         except OperationError, e:
-            if e.match(space, space.w_KeyError):
+            if e.match(space, space.w_LookupError):
                 result.append(unichar)
             else:
                 raise
diff --git a/pypy/rlib/objectmodel.py b/pypy/rlib/objectmodel.py
--- a/pypy/rlib/objectmodel.py
+++ b/pypy/rlib/objectmodel.py
@@ -46,6 +46,17 @@
 
         return decorated_func
 
+    def arg_or_var(self, *args):
+        """ Same as arg, but additionally allow for a 'variable' annotation,
+        that would simply be a situation where designated arg is not
+        a constant
+        """
+        def decorated_func(func):
+            func._annspecialcase_ = 'specialize:arg_or_var' + self._wrap(args)
+            return func
+
+        return decorated_func
+
     def argtype(self, *args):
         """ Specialize function based on types of arguments on given positions.
 
@@ -165,6 +176,24 @@
 def keepalive_until_here(*values):
     pass
 
+def is_annotation_constant(thing):
+    """ Returns whether the annotator can prove that the argument is constant.
+    For advanced usage only."""
+    return True
+
+class Entry(ExtRegistryEntry):
+    _about_ = is_annotation_constant
+
+    def compute_result_annotation(self, s_arg):
+        from pypy.annotation import model
+        r = model.SomeBool()
+        r.const = s_arg.is_constant()
+        return r
+
+    def specialize_call(self, hop):
+        from pypy.rpython.lltypesystem import lltype
+        return hop.inputconst(lltype.Bool, hop.s_result.const)
+
 # ____________________________________________________________
 
 class FREED_OBJECT(object):
diff --git a/pypy/rlib/rgc.py b/pypy/rlib/rgc.py
--- a/pypy/rlib/rgc.py
+++ b/pypy/rlib/rgc.py
@@ -143,6 +143,10 @@
     from pypy.rpython.lltypesystem.lloperation import llop
     from pypy.rlib.objectmodel import keepalive_until_here
 
+    # XXX: Hack to ensure that we get a proper effectinfo.write_descrs_arrays
+    if length > 0:
+        dest[dest_start] = source[source_start]
+
     # supports non-overlapping copies only
     if not we_are_translated():
         if source == dest:
diff --git a/pypy/rlib/ropenssl.py b/pypy/rlib/ropenssl.py
--- a/pypy/rlib/ropenssl.py
+++ b/pypy/rlib/ropenssl.py
@@ -62,8 +62,7 @@
         "OPENSSL_VERSION_NUMBER")
     SSLEAY_VERSION = rffi_platform.DefinedConstantString(
         "SSLEAY_VERSION", "SSLeay_version(SSLEAY_VERSION)")
-    OPENSSL_NO_SSL2 = rffi_platform.DefinedConstantInteger(
-        "OPENSSL_NO_SSL2")
+    OPENSSL_NO_SSL2 = rffi_platform.Defined("OPENSSL_NO_SSL2")
     SSL_FILETYPE_PEM = rffi_platform.ConstantInteger("SSL_FILETYPE_PEM")
     SSL_OP_ALL = rffi_platform.ConstantInteger("SSL_OP_ALL")
     SSL_VERIFY_NONE = rffi_platform.ConstantInteger("SSL_VERIFY_NONE")
diff --git a/pypy/rlib/rstacklet.py b/pypy/rlib/rstacklet.py
--- a/pypy/rlib/rstacklet.py
+++ b/pypy/rlib/rstacklet.py
@@ -99,12 +99,20 @@
         return False
     def add(self, h):
         if not self.sthread.is_empty_handle(h):
+            if h == self.sthread.get_null_handle():
+                raise StackletDebugError("unexpected null handle")
             self.active.append(h)
     def remove(self, h):
         try:
             i = self.active.index(h)
         except ValueError:
-            raise StackletDebugError
+            if self.sthread.is_empty_handle(h):
+                msg = "empty stacklet handle"
+            elif h == self.sthread.get_null_handle():
+                msg = "unexpected null handle"
+            else:
+                msg = "double usage of handle %r" % (h,)
+            raise StackletDebugError(msg)
         del self.active[i]
 debug = Debug()
 
diff --git a/pypy/rlib/test/test_objectmodel.py b/pypy/rlib/test/test_objectmodel.py
--- a/pypy/rlib/test/test_objectmodel.py
+++ b/pypy/rlib/test/test_objectmodel.py
@@ -339,6 +339,19 @@
         res = self.interpret(f, [42])
         assert res == 84
 
+    def test_isconstant(self):
+        from pypy.rlib.objectmodel import is_annotation_constant, specialize
+
+        @specialize.arg_or_var(0)
+        def f(arg):
+            if is_annotation_constant(arg):
+                return 1
+            return 10
+
+        def fn(arg):
+            return f(arg) + f(3)
+
+        assert self.interpret(fn, [15]) == 11
 
 class TestLLtype(BaseTestObjectModel, LLRtypeMixin):
 
@@ -451,5 +464,4 @@
         if llop.opname == 'malloc_varsize':
             break
     assert llop.args[2] is graph.startblock.inputargs[0]
-
     
diff --git a/pypy/rlib/test/test_rstacklet.py b/pypy/rlib/test/test_rstacklet.py
--- a/pypy/rlib/test/test_rstacklet.py
+++ b/pypy/rlib/test/test_rstacklet.py
@@ -264,6 +264,10 @@
     gcrootfinder = 'shadowstack'
 
 
+def test_dont_keep_debug_to_true():
+    assert not rstacklet.DEBUG
+
+
 def target(*args):
     return entry_point, None
 
diff --git a/pypy/rpython/lltypesystem/ll2ctypes.py b/pypy/rpython/lltypesystem/ll2ctypes.py
--- a/pypy/rpython/lltypesystem/ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/ll2ctypes.py
@@ -140,7 +140,8 @@
                 if isinstance(FIELDTYPE, lltype.Ptr):
                     cls = get_ctypes_type(FIELDTYPE, delayed_builders)
                 else:
-                    cls = get_ctypes_type(FIELDTYPE)
+                    cls = get_ctypes_type(FIELDTYPE, delayed_builders,
+                                          cannot_delay=True)
             fields.append((fieldname, cls))
         CStruct._fields_ = fields
 
@@ -169,7 +170,7 @@
         CStruct._normalized_ctype = get_ctypes_type(S)
         builder()    # no need to be lazy here
     else:
-        delayed_builders.append(builder)
+        delayed_builders.append((S, builder))
     return CStruct
 
 def build_ctypes_array(A, delayed_builders, max_n=0):
@@ -252,11 +253,19 @@
     else:
         return get_ctypes_type(FIELDTYPE)
 
-def get_ctypes_type(T, delayed_builders=None):
+def get_ctypes_type(T, delayed_builders=None, cannot_delay=False):
+    # Check delayed builders
+    if cannot_delay and delayed_builders:
+        for T2, builder in delayed_builders:
+            if T2 is T:
+                builder()
+                delayed_builders.remove((T2, builder))
+                return _ctypes_cache[T]
+
     try:
         return _ctypes_cache[T]
     except KeyError:
-        toplevel = delayed_builders is None
+        toplevel = cannot_delay or delayed_builders is None
         if toplevel:
             delayed_builders = []
         cls = build_new_ctypes_type(T, delayed_builders)
@@ -306,9 +315,11 @@
 
 def complete_builders(delayed_builders):
     while delayed_builders:
-        delayed_builders.pop()()
+        T, builder = delayed_builders[0]
+        builder()
+        delayed_builders.pop(0)
 
-def convert_struct(container, cstruct=None):
+def convert_struct(container, cstruct=None, delayed_converters=None):
     STRUCT = container._TYPE
     if cstruct is None:
         # if 'container' is an inlined substructure, convert the whole
@@ -325,23 +336,38 @@
             n = None
         cstruct = cls._malloc(n)
     add_storage(container, _struct_mixin, ctypes.pointer(cstruct))
+
+    if delayed_converters is None:
+        delayed_converters_was_None = True
+        delayed_converters = []
+    else:
+        delayed_converters_was_None = False
     for field_name in STRUCT._names:
         FIELDTYPE = getattr(STRUCT, field_name)
         field_value = getattr(container, field_name)
         if not isinstance(FIELDTYPE, lltype.ContainerType):
             # regular field
             if FIELDTYPE != lltype.Void:
-                setattr(cstruct, field_name, lltype2ctypes(field_value))
+                def convert(field_name=field_name, field_value=field_value):
+                    setattr(cstruct, field_name, lltype2ctypes(field_value))
+                if isinstance(FIELDTYPE, lltype.Ptr):
+                    delayed_converters.append(convert)
+                else:
+                    convert()
         else:
             # inlined substructure/subarray
             if isinstance(FIELDTYPE, lltype.Struct):
                 csubstruct = getattr(cstruct, field_name)
-                convert_struct(field_value, csubstruct)
+                convert_struct(field_value, csubstruct,
+                               delayed_converters=delayed_converters)
             elif field_name == STRUCT._arrayfld:    # inlined var-sized part
                 csubarray = getattr(cstruct, field_name)
                 convert_array(field_value, csubarray)
             else:
                 raise NotImplementedError('inlined field', FIELDTYPE)
+    if delayed_converters_was_None:
+        for converter in delayed_converters:
+            converter()
     remove_regular_struct_content(container)
 
 def remove_regular_struct_content(container):
@@ -358,7 +384,8 @@
         # bigger structure at once
         parent, parentindex = lltype.parentlink(container)
         if parent is not None:
-            convert_struct(parent)
+            if not isinstance(parent, _parentable_mixin):
+                convert_struct(parent)
             return
         # regular case: allocate a new ctypes array of the proper type
         cls = get_ctypes_type(ARRAY)
diff --git a/pypy/rpython/lltypesystem/test/test_ll2ctypes.py b/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
--- a/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
@@ -82,7 +82,6 @@
         assert not ALLOCATED     # detects memory leaks in the test
 
     def test_get_pointer(self):
-        py.test.skip("FIXME")
         # Equivalent of the C code::
         #     struct S1 { struct S2 *ptr; struct S2 buf; };
         #     struct S1 s1;
diff --git a/pypy/rpython/rlist.py b/pypy/rpython/rlist.py
--- a/pypy/rpython/rlist.py
+++ b/pypy/rpython/rlist.py
@@ -11,7 +11,7 @@
 from pypy.rlib.debug import ll_assert
 from pypy.rlib.rarithmetic import ovfcheck, widen, r_uint, intmask
 from pypy.rpython.annlowlevel import ADTInterface
-from pypy.rlib import rgc
+from pypy.rlib import rgc, jit
 
 ADTIFixedList = ADTInterface(None, {
     'll_newlist':      (['SELF', Signed        ], 'self'),
@@ -912,6 +912,8 @@
     return l
 # no oopspec -- the function is inlined by the JIT
 
+ at jit.look_inside_iff(lambda l, start: jit.isconstant(start) and jit.isvirtual(l))
+ at jit.oopspec('list.delslice_startonly(l, start)')
 def ll_listdelslice_startonly(l, start):
     ll_assert(start >= 0, "del l[start:] with unexpectedly negative start")
     ll_assert(start <= l.ll_length(), "del l[start:] with start > len(l)")
@@ -923,7 +925,6 @@
             l.ll_setitem_fast(j, null)
             j -= 1
     l._ll_resize_le(newlength)
-ll_listdelslice_startonly.oopspec = 'list.delslice_startonly(l, start)'
 
 def ll_listdelslice_startstop(l, start, stop):
     length = l.ll_length()
diff --git a/pypy/tool/pytest/appsupport.py b/pypy/tool/pytest/appsupport.py
--- a/pypy/tool/pytest/appsupport.py
+++ b/pypy/tool/pytest/appsupport.py
@@ -72,7 +72,9 @@
         space = self.space
         retval = []
         for arg in self.code.getargs():
-            w_val = space.getitem(self.w_locals, space.wrap(arg))
+            w_val = space.finditem(self.w_locals, space.wrap(arg))
+            if w_val is None:
+                w_val = space.wrap('<no value found>')
             retval.append((arg, w_val))
         return retval
 
diff --git a/pypy/translator/c/src/allocator.h b/pypy/translator/c/src/allocator.h
--- a/pypy/translator/c/src/allocator.h
+++ b/pypy/translator/c/src/allocator.h
@@ -6,11 +6,6 @@
 
 
 #ifndef PYPY_NOT_MAIN_FILE
-#ifdef AVR
-   #ifndef NO_OBMALLOC
-     #define NO_OBMALLOC
-   #endif
-#endif
 
 #if defined(TRIVIAL_MALLOC_DEBUG)
   void *PyObject_Malloc(size_t n) { return malloc(n); }
diff --git a/pypy/translator/c/src/g_include.h b/pypy/translator/c/src/g_include.h
--- a/pypy/translator/c/src/g_include.h
+++ b/pypy/translator/c/src/g_include.h
@@ -31,9 +31,7 @@
 #include "src/char.h"
 #include "src/float.h"
 #include "src/address.h"
-#ifndef AVR
 #include "src/unichar.h"
-#endif
 #include "src/llgroup.h"
 
 #include "src/instrument.h"
@@ -48,11 +46,9 @@
 #  include "src/rtyper.h"
 #  include "src/debug_traceback.h"
 #  include "src/debug_alloc.h"
-#ifndef AVR
 #  include "src/ll_os.h"
 #  include "src/ll_strtod.h"
 #endif
-#endif
 
 #ifdef PYPY_STANDALONE
 #  include "src/allocator.h"
diff --git a/pypy/translator/c/src/g_prerequisite.h b/pypy/translator/c/src/g_prerequisite.h
--- a/pypy/translator/c/src/g_prerequisite.h
+++ b/pypy/translator/c/src/g_prerequisite.h
@@ -13,10 +13,8 @@
 #  include <io.h>   /* needed, otherwise _lseeki64 truncates to 32-bits (??) */
 #endif
 
-#ifndef AVR
 #include "thread.h"   /* needs to be included early to define the
                          struct RPyOpaque_ThreadLock */
-#endif
 
 #include <stddef.h>
 
diff --git a/pypy/translator/c/src/main.h b/pypy/translator/c/src/main.h
--- a/pypy/translator/c/src/main.h
+++ b/pypy/translator/c/src/main.h
@@ -75,9 +75,7 @@
  memory_out:
     errmsg = "out of memory";
  error:
-#ifndef AVR
     fprintf(stderr, "Fatal error during initialization: %s\n", errmsg);
-#endif
     abort();
     return 1;
 }

From noreply at buildbot.pypy.org  Sat Oct  1 10:13:28 2011
From: noreply at buildbot.pypy.org (hakanardo)
Date: Sat,  1 Oct 2011 10:13:28 +0200 (CEST)
Subject: [pypy-commit] pypy jit-optimizeopt-cleanups: fix test
Message-ID: <20111001081328.137E582A01@wyvern.cs.uni-duesseldorf.de>

Author: Hakan Ardo <hakan at debian.org>
Branch: jit-optimizeopt-cleanups
Changeset: r47738:17466bccb8ea
Date: 2011-10-01 09:45 +0200
http://bitbucket.org/pypy/pypy/changeset/17466bccb8ea/

Log:	fix test

diff --git a/pypy/jit/metainterp/optimizeopt/test/test_util.py b/pypy/jit/metainterp/optimizeopt/test/test_util.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_util.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_util.py
@@ -182,7 +182,7 @@
                             EffectInfo.EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE,
                             can_invalidate=True))
     arraycopydescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT,
-             EffectInfo([], [], [], [], oopspecindex=EffectInfo.OS_ARRAYCOPY))
+             EffectInfo([], [], [], [arraydescr], oopspecindex=EffectInfo.OS_ARRAYCOPY))
 
     for _name, _os in [
         ('strconcatdescr',               'OS_STR_CONCAT'),

From noreply at buildbot.pypy.org  Sat Oct  1 10:13:29 2011
From: noreply at buildbot.pypy.org (hakanardo)
Date: Sat,  1 Oct 2011 10:13:29 +0200 (CEST)
Subject: [pypy-commit] pypy jit-optimizeopt-cleanups: added skipped test of
 still unsuported case
Message-ID: <20111001081329.4642582A01@wyvern.cs.uni-duesseldorf.de>

Author: Hakan Ardo <hakan at debian.org>
Branch: jit-optimizeopt-cleanups
Changeset: r47739:fdec24b8018a
Date: 2011-10-01 10:12 +0200
http://bitbucket.org/pypy/pypy/changeset/fdec24b8018a/

Log:	added skipped test of still unsuported case

diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -7262,6 +7262,27 @@
         """
         self.optimize_loop(ops, ops)
 
+    def test_heap_cache_virtuals_forced_by_delayed_setfield(self):
+        py.test.skip('not yet supoprted')
+        ops = """
+        [i1, p0]
+        p1 = new(descr=ssize)
+        setfield_gc(p1, i1, descr=valuedescr)
+        setfield_gc(p0, p1, descr=adescr)
+        call(p0, descr=writeadescr)
+        i2 = getfield_gc(p1, descr=valuedescr)
+        jump(i2, p0)
+        """
+        expected = """
+        [i1, p0]
+        p1 = new(descr=ssize)
+        setfield_gc(p1, i1, descr=valuedescr)
+        setfield_gc(p0, p1, descr=adescr)
+        call(p0, descr=writeadescr)
+        jump(i1, p0)
+        """
+        self.optimize_loop(ops, expected)
+
 class TestLLtype(OptimizeOptTest, LLtypeMixin):
     pass
 

From noreply at buildbot.pypy.org  Sat Oct  1 10:36:48 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  1 Oct 2011 10:36:48 +0200 (CEST)
Subject: [pypy-commit] pypy gil-improvement: Force the GIL to be released in
 this app-test; time.sleep()
Message-ID: <20111001083648.AB97F82A01@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: gil-improvement
Changeset: r47740:3e1481231b00
Date: 2011-10-01 10:36 +0200
http://bitbucket.org/pypy/pypy/changeset/3e1481231b00/

Log:	Force the GIL to be released in this app-test; time.sleep() doesn't
	do it when non-translated... It likely worked because
	sys.checkinterval used to be only 100.

diff --git a/pypy/module/thread/test/test_thread.py b/pypy/module/thread/test/test_thread.py
--- a/pypy/module/thread/test/test_thread.py
+++ b/pypy/module/thread/test/test_thread.py
@@ -225,7 +225,8 @@
 
         def busy_wait():
             for x in range(1000):
-                time.sleep(0.01)
+                print 'tick...', x  # <-force the GIL to be released, as
+                time.sleep(0.01)    #   time.sleep doesn't do non-translated
 
         # This is normally called by app_main.py
         signal.signal(signal.SIGINT, signal.default_int_handler)

From noreply at buildbot.pypy.org  Sat Oct  1 12:38:43 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  1 Oct 2011 12:38:43 +0200 (CEST)
Subject: [pypy-commit] pypy default: For now,
 always use "python" to run trackgcroot.py.  Running pypy is slower, 
Message-ID: <20111001103843.C466882A01@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r47741:a3808c44744d
Date: 2011-10-01 11:21 +0200
http://bitbucket.org/pypy/pypy/changeset/a3808c44744d/

Log:	For now, always use "python" to run trackgcroot.py. Running pypy is
	slower, particularly now that each invocation is typically 0.2-0.5
	seconds.

diff --git a/pypy/translator/c/genc.py b/pypy/translator/c/genc.py
--- a/pypy/translator/c/genc.py
+++ b/pypy/translator/c/genc.py
@@ -563,7 +563,10 @@
             else:
                 mk.definition('PYPY_MAIN_FUNCTION', "main")
 
-            if sys.platform == 'win32':
+            if (py.path.local.sysfind('python') or
+                py.path.local.sysfind('python.exe')):
+                python = 'python '
+            elif sys.platform == 'win32':
                 python = sys.executable.replace('\\', '/') + ' '
             else:
                 python = sys.executable + ' '

From noreply at buildbot.pypy.org  Sat Oct  1 12:38:44 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  1 Oct 2011 12:38:44 +0200 (CEST)
Subject: [pypy-commit] pypy gil-improvement: Fixes.
Message-ID: <20111001103844.F18AD82A01@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: gil-improvement
Changeset: r47742:e29e5f547365
Date: 2011-10-01 11:52 +0200
http://bitbucket.org/pypy/pypy/changeset/e29e5f547365/

Log:	Fixes.

diff --git a/pypy/translator/c/gcc/trackgcroot.py b/pypy/translator/c/gcc/trackgcroot.py
--- a/pypy/translator/c/gcc/trackgcroot.py
+++ b/pypy/translator/c/gcc/trackgcroot.py
@@ -486,6 +486,8 @@
         'paddq', 'pinsr',
         # zero-extending moves should not produce GC pointers
         'movz', 
+        # locked operations should not move GC pointers, at least so far
+        'lock',
         ])
 
     # a partial list is hopefully good enough for now; it's all to support
diff --git a/pypy/translator/c/src/thread_pthread.h b/pypy/translator/c/src/thread_pthread.h
--- a/pypy/translator/c/src/thread_pthread.h
+++ b/pypy/translator/c/src/thread_pthread.h
@@ -496,9 +496,11 @@
 
 static void _debug_print(const char *msg)
 {
+#if 0
     int col = (int)pthread_self();
     col = 31 + ((col / 8) % 8);
     fprintf(stderr, "\033[%dm%s\033[0m", col, msg);
+#endif
 }
 
 static volatile long pending_acquires = -1;

From noreply at buildbot.pypy.org  Sat Oct  1 12:38:46 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  1 Oct 2011 12:38:46 +0200 (CEST)
Subject: [pypy-commit] pypy gil-improvement: Bah.
Message-ID: <20111001103846.2786F82A01@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: gil-improvement
Changeset: r47743:577499446564
Date: 2011-10-01 11:57 +0200
http://bitbucket.org/pypy/pypy/changeset/577499446564/

Log:	Bah.

diff --git a/pypy/module/thread/gil.py b/pypy/module/thread/gil.py
--- a/pypy/module/thread/gil.py
+++ b/pypy/module/thread/gil.py
@@ -116,6 +116,7 @@
         spacestate.after_thread_switch()
 do_yield_thread._gctransformer_hint_close_stack_ = True
 do_yield_thread._dont_reach_me_in_del_ = True
+do_yield_thread._dont_inline_ = True
 
 # do_yield_thread() needs a different hint: _gctransformer_hint_close_stack_.
 # The *_external_call() functions are themselves called only from the rffi

From noreply at buildbot.pypy.org  Sat Oct  1 12:38:47 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  1 Oct 2011 12:38:47 +0200 (CEST)
Subject: [pypy-commit] pypy default: A failing test that shows a real (but
 rare) case in translation.
Message-ID: <20111001103847.4E92682A01@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r47744:6b1ea446d303
Date: 2011-10-01 12:36 +0200
http://bitbucket.org/pypy/pypy/changeset/6b1ea446d303/

Log:	A failing test that shows a real (but rare) case in translation.

diff --git a/pypy/rlib/test/test_jit.py b/pypy/rlib/test/test_jit.py
--- a/pypy/rlib/test/test_jit.py
+++ b/pypy/rlib/test/test_jit.py
@@ -1,7 +1,7 @@
 import py
 from pypy.conftest import option
 from pypy.rlib.jit import hint, we_are_jitted, JitDriver, elidable_promote
-from pypy.rlib.jit import JitHintError, oopspec
+from pypy.rlib.jit import JitHintError, oopspec, isconstant
 from pypy.translator.translator import TranslationContext, graphof
 from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin
 from pypy.rpython.lltypesystem import lltype
@@ -137,6 +137,16 @@
             t.view()
         # assert did not raise
 
+    def test_isconstant(self):
+        def f(n):
+            assert n >= 0
+            assert isconstant(n) is False
+            l = []
+            l.append(n)
+            return len(l)
+        res = self.interpret(f, [234])
+        assert res == 1
+
 
 class TestJITLLtype(BaseTestJIT, LLRtypeMixin):
     pass

From noreply at buildbot.pypy.org  Sat Oct  1 12:38:48 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  1 Oct 2011 12:38:48 +0200 (CEST)
Subject: [pypy-commit] pypy default: This simplification seems to fix the
	test.
Message-ID: <20111001103848.7804882A01@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r47745:f6cf1b602c40
Date: 2011-10-01 12:38 +0200
http://bitbucket.org/pypy/pypy/changeset/f6cf1b602c40/

Log:	This simplification seems to fix the test.

diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -167,10 +167,7 @@
 
     This is for advanced usage only.
     """
-    # I hate the annotator so much.
-    if NonConstant(False):
-        return True
-    return False
+    return NonConstant(False)
 
 @oopspec("jit.isvirtual(value)")
 @specialize.ll()

From noreply at buildbot.pypy.org  Sat Oct  1 12:44:54 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  1 Oct 2011 12:44:54 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix
 test_optimizeopt.test_setarrayitem_followed_by_arraycopy()
Message-ID: <20111001104454.5D00D82A01@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r47746:d021f859d101
Date: 2011-10-01 12:44 +0200
http://bitbucket.org/pypy/pypy/changeset/d021f859d101/

Log:	Fix test_optimizeopt.test_setarrayitem_followed_by_arraycopy() by
	putting manually the correct readonly_descrs_arrays and
	write_descrs_arrays lists.

diff --git a/pypy/jit/metainterp/optimizeopt/test/test_util.py b/pypy/jit/metainterp/optimizeopt/test/test_util.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_util.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_util.py
@@ -182,7 +182,8 @@
                             EffectInfo.EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE,
                             can_invalidate=True))
     arraycopydescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT,
-             EffectInfo([], [], [], [], oopspecindex=EffectInfo.OS_ARRAYCOPY))
+             EffectInfo([], [arraydescr], [], [arraydescr],
+                        oopspecindex=EffectInfo.OS_ARRAYCOPY))
 
     for _name, _os in [
         ('strconcatdescr',               'OS_STR_CONCAT'),

From noreply at buildbot.pypy.org  Sat Oct  1 12:45:40 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  1 Oct 2011 12:45:40 +0200 (CEST)
Subject: [pypy-commit] pypy default: Tweak the hack: it should be equivalent,
 but have no remaining effect
Message-ID: <20111001104540.564EE82A01@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r47747:78fddfb51114
Date: 2011-10-01 12:45 +0200
http://bitbucket.org/pypy/pypy/changeset/78fddfb51114/

Log:	Tweak the hack: it should be equivalent, but have no remaining
	effect at run-time.

diff --git a/pypy/rlib/rgc.py b/pypy/rlib/rgc.py
--- a/pypy/rlib/rgc.py
+++ b/pypy/rlib/rgc.py
@@ -3,6 +3,7 @@
 
 from pypy.rlib import jit
 from pypy.rlib.objectmodel import we_are_translated, enforceargs, specialize
+from pypy.rlib.nonconst import NonConstant
 from pypy.rpython.extregistry import ExtRegistryEntry
 from pypy.rpython.lltypesystem import lltype, llmemory
 
@@ -144,7 +145,7 @@
     from pypy.rlib.objectmodel import keepalive_until_here
 
     # XXX: Hack to ensure that we get a proper effectinfo.write_descrs_arrays
-    if length > 0:
+    if NonConstant(False):
         dest[dest_start] = source[source_start]
 
     # supports non-overlapping copies only

From noreply at buildbot.pypy.org  Sat Oct  1 12:57:50 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  1 Oct 2011 12:57:50 +0200 (CEST)
Subject: [pypy-commit] pypy gil-improvement: hg merge default
Message-ID: <20111001105750.5E27682A01@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: gil-improvement
Changeset: r47748:e6c3478fe62c
Date: 2011-10-01 12:56 +0200
http://bitbucket.org/pypy/pypy/changeset/e6c3478fe62c/

Log:	hg merge default

diff --git a/pypy/annotation/test/test_annrpython.py b/pypy/annotation/test/test_annrpython.py
--- a/pypy/annotation/test/test_annrpython.py
+++ b/pypy/annotation/test/test_annrpython.py
@@ -3204,6 +3204,8 @@
         s = a.build_types(f, [])
         assert isinstance(s, annmodel.SomeList)
         assert not s.listdef.listitem.resized
+        assert not s.listdef.listitem.immutable
+        assert s.listdef.listitem.mutated
 
     def test_delslice(self):
         def f():
diff --git a/pypy/annotation/unaryop.py b/pypy/annotation/unaryop.py
--- a/pypy/annotation/unaryop.py
+++ b/pypy/annotation/unaryop.py
@@ -352,6 +352,7 @@
         check_negative_slice(s_start, s_stop)
         if not isinstance(s_iterable, SomeList):
             raise Exception("list[start:stop] = x: x must be a list")
+        lst.listdef.mutate()
         lst.listdef.agree(s_iterable.listdef)
         # note that setslice is not allowed to resize a list in RPython
 
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_util.py b/pypy/jit/metainterp/optimizeopt/test/test_util.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_util.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_util.py
@@ -182,7 +182,8 @@
                             EffectInfo.EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE,
                             can_invalidate=True))
     arraycopydescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT,
-             EffectInfo([], [], [], [], oopspecindex=EffectInfo.OS_ARRAYCOPY))
+             EffectInfo([], [arraydescr], [], [arraydescr],
+                        oopspecindex=EffectInfo.OS_ARRAYCOPY))
 
     for _name, _os in [
         ('strconcatdescr',               'OS_STR_CONCAT'),
diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py
--- a/pypy/objspace/std/intobject.py
+++ b/pypy/objspace/std/intobject.py
@@ -1,12 +1,13 @@
 from pypy.interpreter.error import OperationError
 from pypy.objspace.std import newformat
+from pypy.objspace.std.inttype import wrapint
 from pypy.objspace.std.model import registerimplementation, W_Object
-from pypy.objspace.std.register_all import register_all
 from pypy.objspace.std.multimethod import FailedToImplementArgs
 from pypy.objspace.std.noneobject import W_NoneObject
+from pypy.objspace.std.register_all import register_all
+from pypy.rlib import jit
 from pypy.rlib.rarithmetic import ovfcheck, ovfcheck_lshift, LONG_BIT, r_uint
 from pypy.rlib.rbigint import rbigint
-from pypy.objspace.std.inttype import wrapint
 
 """
 In order to have the same behavior running
@@ -169,7 +170,8 @@
 
 
 # helper for pow()
-def _impl_int_int_pow(space, iv, iw, iz=0):
+ at jit.look_inside_iff(lambda space, iv, iw, iz: jit.isconstant(iw) and jit.isconstant(iz))
+def _impl_int_int_pow(space, iv, iw, iz):
     if iw < 0:
         if iz != 0:
             raise OperationError(space.w_TypeError,
@@ -211,7 +213,7 @@
 def pow__Int_Int_None(space, w_int1, w_int2, w_int3):
     x = w_int1.intval
     y = w_int2.intval
-    return space.wrap(_impl_int_int_pow(space, x, y))
+    return space.wrap(_impl_int_int_pow(space, x, y, 0))
 
 def neg__Int(space, w_int1):
     a = w_int1.intval
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -386,7 +386,11 @@
     if len(items)== 0:
         raise OperationError(space.w_IndexError,
                              space.wrap("pop from empty list"))
-    idx = space.int_w(w_idx)
+    if space.isinstance_w(w_idx, space.w_float):
+        raise OperationError(space.w_TypeError,
+            space.wrap("integer argument expected, got float")
+        )
+    idx = space.int_w(space.int(w_idx))
     try:
         return items.pop(idx)
     except IndexError:
diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py
--- a/pypy/objspace/std/test/test_listobject.py
+++ b/pypy/objspace/std/test/test_listobject.py
@@ -705,6 +705,20 @@
         l.pop()
         assert l == range(9)
 
+    def test_pop_custom_int(self):
+        class A(object):
+            def __init__(self, x):
+                self.x = x
+
+            def __int__(self):
+                return self.x
+
+        l = range(10)
+        x = l.pop(A(-1))
+        assert x == 9
+        assert l == range(9)
+        raises(TypeError, range(10).pop, 1.0)
+
     def test_remove(self):
         c = list('hello world')
         c.remove('l')
diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -126,6 +126,12 @@
         raises(TypeError, type, 'test', 42, {})
         raises(TypeError, type, 'test', (object,), 42)
 
+    def test_call_type_subclass(self):
+        class A(type):
+            pass
+
+        assert A("hello") is str
+
     def test_bases(self):
         assert int.__bases__ == (object,)
         class X:
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -822,14 +822,6 @@
 
 def call__Type(space, w_type, __args__):
     promote(w_type)
-    # special case for type(x)
-    if space.is_w(w_type, space.w_type):
-        try:
-            w_obj, = __args__.fixedunpack(1)
-        except ValueError:
-            pass
-        else:
-            return space.type(w_obj)
     # invoke the __new__ of the type
     if not we_are_jitted():
         # note that the annotator will figure out that w_type.w_bltin_new can
diff --git a/pypy/objspace/std/typetype.py b/pypy/objspace/std/typetype.py
--- a/pypy/objspace/std/typetype.py
+++ b/pypy/objspace/std/typetype.py
@@ -1,17 +1,28 @@
-from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.interpreter import gateway
 from pypy.interpreter.argument import Arguments
+from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.interpreter.typedef import (GetSetProperty, descr_get_dict,
                                       weakref_descr)
 from pypy.objspace.std.stdtypedef import StdTypeDef
 
-def descr__new__(space, w_typetype, w_name, w_bases, w_dict):
+
+def descr__new__(space, w_typetype, w_name, w_bases=gateway.NoneNotWrapped,
+    w_dict=gateway.NoneNotWrapped):
+
     "This is used to create user-defined classes only."
     from pypy.objspace.std.typeobject import W_TypeObject
     # XXX check types
 
     w_typetype = _precheck_for_new(space, w_typetype)
 
+    # special case for type(x)
+    if (space.is_w(space.type(w_typetype), space.w_type) and w_bases is None and
+        w_dict is None):
+        return space.type(w_name)
+    elif w_bases is None or w_dict is None:
+        raise OperationError(space.w_TypeError, space.wrap("type() takes 1 or 3 arguments"))
+
+
     bases_w = space.fixedview(w_bases)
 
     w_winner = w_typetype
diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -167,10 +167,7 @@
 
     This is for advanced usage only.
     """
-    # I hate the annotator so much.
-    if NonConstant(False):
-        return True
-    return False
+    return NonConstant(False)
 
 @oopspec("jit.isvirtual(value)")
 @specialize.ll()
diff --git a/pypy/rlib/rgc.py b/pypy/rlib/rgc.py
--- a/pypy/rlib/rgc.py
+++ b/pypy/rlib/rgc.py
@@ -3,6 +3,7 @@
 
 from pypy.rlib import jit
 from pypy.rlib.objectmodel import we_are_translated, enforceargs, specialize
+from pypy.rlib.nonconst import NonConstant
 from pypy.rpython.extregistry import ExtRegistryEntry
 from pypy.rpython.lltypesystem import lltype, llmemory
 
@@ -143,6 +144,10 @@
     from pypy.rpython.lltypesystem.lloperation import llop
     from pypy.rlib.objectmodel import keepalive_until_here
 
+    # XXX: Hack to ensure that we get a proper effectinfo.write_descrs_arrays
+    if NonConstant(False):
+        dest[dest_start] = source[source_start]
+
     # supports non-overlapping copies only
     if not we_are_translated():
         if source == dest:
diff --git a/pypy/rlib/test/test_jit.py b/pypy/rlib/test/test_jit.py
--- a/pypy/rlib/test/test_jit.py
+++ b/pypy/rlib/test/test_jit.py
@@ -1,7 +1,7 @@
 import py
 from pypy.conftest import option
 from pypy.rlib.jit import hint, we_are_jitted, JitDriver, elidable_promote
-from pypy.rlib.jit import JitHintError, oopspec
+from pypy.rlib.jit import JitHintError, oopspec, isconstant
 from pypy.translator.translator import TranslationContext, graphof
 from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin
 from pypy.rpython.lltypesystem import lltype
@@ -137,6 +137,16 @@
             t.view()
         # assert did not raise
 
+    def test_isconstant(self):
+        def f(n):
+            assert n >= 0
+            assert isconstant(n) is False
+            l = []
+            l.append(n)
+            return len(l)
+        res = self.interpret(f, [234])
+        assert res == 1
+
 
 class TestJITLLtype(BaseTestJIT, LLRtypeMixin):
     pass
diff --git a/pypy/translator/c/genc.py b/pypy/translator/c/genc.py
--- a/pypy/translator/c/genc.py
+++ b/pypy/translator/c/genc.py
@@ -563,7 +563,10 @@
             else:
                 mk.definition('PYPY_MAIN_FUNCTION', "main")
 
-            if sys.platform == 'win32':
+            if (py.path.local.sysfind('python') or
+                py.path.local.sysfind('python.exe')):
+                python = 'python '
+            elif sys.platform == 'win32':
                 python = sys.executable.replace('\\', '/') + ' '
             else:
                 python = sys.executable + ' '

From noreply at buildbot.pypy.org  Sat Oct  1 18:05:17 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  1 Oct 2011 18:05:17 +0200 (CEST)
Subject: [pypy-commit] pypy unsigned-dtypes: Test and fix.
Message-ID: <20111001160517.5762D82A01@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: unsigned-dtypes
Changeset: r47749:5f924fccbcc8
Date: 2011-10-01 18:05 +0200
http://bitbucket.org/pypy/pypy/changeset/5f924fccbcc8/

Log:	Test and fix.

diff --git a/pypy/rpython/lltypesystem/opimpl.py b/pypy/rpython/lltypesystem/opimpl.py
--- a/pypy/rpython/lltypesystem/opimpl.py
+++ b/pypy/rpython/lltypesystem/opimpl.py
@@ -357,7 +357,7 @@
 
 def op_cast_float_to_uint(f):
     assert type(f) is float
-    return r_uint(int(f))
+    return r_uint(long(f))
 
 def op_cast_float_to_longlong(f):
     assert type(f) is float
@@ -369,7 +369,7 @@
 
 def op_cast_float_to_ulonglong(f):
     assert type(f) is float
-    return r_ulonglong(r_longlong(f))
+    return r_ulonglong(long(f))
 
 def op_cast_char_to_int(b):
     assert type(b) is str and len(b) == 1
diff --git a/pypy/rpython/lltypesystem/test/test_lloperation.py b/pypy/rpython/lltypesystem/test/test_lloperation.py
--- a/pypy/rpython/lltypesystem/test/test_lloperation.py
+++ b/pypy/rpython/lltypesystem/test/test_lloperation.py
@@ -5,6 +5,7 @@
 from pypy.rpython.llinterp import LLFrame
 from pypy.rpython.test.test_llinterp import interpret
 from pypy.rpython import rclass
+from pypy.rlib.rarithmetic import LONGLONG_MASK, r_longlong, r_ulonglong
 
 LL_INTERP_OPERATIONS = [name[3:] for name in LLFrame.__dict__.keys()
                                  if name.startswith('op_')]
@@ -133,6 +134,14 @@
         py.test.raises(TypeError, llop.getinteriorfield,
                        lltype.Signed, s3, 'y')
 
+def test_cast_float_to_ulonglong():
+    f = 12350000000000000000.0
+    py.test.raises(OverflowError, r_longlong, f)
+    r_longlong(f / 2)   # does not raise OverflowError
+    #
+    x = llop.cast_float_to_ulonglong(lltype.UnsignedLongLong, f)
+    assert x == r_ulonglong(f)
+
 # ___________________________________________________________________________
 # This tests that the LLInterpreter and the LL_OPERATIONS tables are in sync.
 

From noreply at buildbot.pypy.org  Sat Oct  1 19:07:25 2011
From: noreply at buildbot.pypy.org (hakanardo)
Date: Sat,  1 Oct 2011 19:07:25 +0200 (CEST)
Subject: [pypy-commit] pypy jit-optimizeopt-cleanups: move pure_operations
 dict and optimize_CALL_PURE to OptPure
Message-ID: <20111001170725.6346482A01@wyvern.cs.uni-duesseldorf.de>

Author: Hakan Ardo <hakan at debian.org>
Branch: jit-optimizeopt-cleanups
Changeset: r47750:0a138d720760
Date: 2011-10-01 19:07 +0200
http://bitbucket.org/pypy/pypy/changeset/0a138d720760/

Log:	move pure_operations dict and optimize_CALL_PURE to OptPure

diff --git a/pypy/jit/metainterp/optimizeopt/optimizer.py b/pypy/jit/metainterp/optimizeopt/optimizer.py
--- a/pypy/jit/metainterp/optimizeopt/optimizer.py
+++ b/pypy/jit/metainterp/optimizeopt/optimizer.py
@@ -275,19 +275,19 @@
         return self.optimizer.new_const_item(arraydescr)
 
     def pure(self, opnum, args, result):
-        op = ResOperation(opnum, args, result)
-        key = self.optimizer.make_args_key(op)
-        if key not in self.optimizer.pure_operations:
-            self.optimizer.pure_operations[key] = op
+        if self.optimizer.optpure:
+            self.optimizer.optpure.pure(opnum, args, result)
 
     def has_pure_result(self, opnum, args, descr):
-        op = ResOperation(opnum, args, None, descr)
-        key = self.optimizer.make_args_key(op)
-        op = self.optimizer.pure_operations.get(key, None)
-        if op is None:
-            return False
-        return op.getdescr() is descr
+        if self.optimizer.optpure:
+            return self.optimizer.optpure.has_pure_result(opnum, args, descr)
+        return False
 
+    def get_pure_result(self, key):    
+        if self.optimizer.optpure:
+            return self.optimizer.optpure.get_pure_result(key)
+        return None
+    
     def setup(self):
         pass
 
@@ -322,7 +322,6 @@
         self.interned_refs = self.cpu.ts.new_ref_dict()
         self.resumedata_memo = resume.ResumeDataLoopMemo(metainterp_sd)
         self.bool_boxes = {}
-        self.pure_operations = args_dict()
         self.producer = {}
         self.pendingfields = []
         self.posponedop = None
diff --git a/pypy/jit/metainterp/optimizeopt/pure.py b/pypy/jit/metainterp/optimizeopt/pure.py
--- a/pypy/jit/metainterp/optimizeopt/pure.py
+++ b/pypy/jit/metainterp/optimizeopt/pure.py
@@ -1,11 +1,17 @@
 from pypy.jit.metainterp.optimizeopt.optimizer import Optimization
 from pypy.jit.metainterp.resoperation import rop, ResOperation
+from pypy.jit.metainterp.optimizeopt.util import (make_dispatcher_method,
+    args_dict)
 
 class OptPure(Optimization):
     def __init__(self):
         self.posponedop = None
+        self.pure_operations = args_dict()
 
     def propagate_forward(self, op):
+        dispatch_opt(self, op)
+
+    def optimize_default(self, op):
         canfold = op.is_always_pure()
         if op.is_ovf():
             self.posponedop = op
@@ -32,14 +38,14 @@
 
             # did we do the exact same operation already?
             args = self.optimizer.make_args_key(op)
-            oldop = self.optimizer.pure_operations.get(args, None)
+            oldop = self.pure_operations.get(args, None)
             if oldop is not None and oldop.getdescr() is op.getdescr():
                 assert oldop.getopnum() == op.getopnum()
                 self.optimizer.make_equal_to(op.result, self.getvalue(oldop.result),
                                    True)
                 return
             else:
-                self.optimizer.pure_operations[args] = op
+                self.pure_operations[args] = op
                 self.optimizer.remember_emitting_pure(op)
 
         # otherwise, the operation remains
@@ -49,6 +55,39 @@
         if nextop:
             self.emit_operation(nextop)
 
+    def optimize_CALL_PURE(self, op):
+        arg_consts = []
+        for i in range(op.numargs()):
+            arg = op.getarg(i)
+            const = self.get_constant_box(arg)
+            if const is None:
+                break
+            arg_consts.append(const)
+        else:
+            # all constant arguments: check if we already know the result
+            try:
+                result = self.optimizer.call_pure_results[arg_consts]
+            except KeyError:
+                pass
+            else:
+                self.make_constant(op.result, result)
+                return
+
+        args = self.optimizer.make_args_key(op)
+        oldop = self.pure_operations.get(args, None)
+        if oldop is not None and oldop.getdescr() is op.getdescr():
+            assert oldop.getopnum() == op.getopnum()
+            self.make_equal_to(op.result, self.getvalue(oldop.result))
+            return
+        else:
+            self.pure_operations[args] = op
+            self.optimizer.remember_emitting_pure(op)
+
+        # replace CALL_PURE with just CALL
+        args = op.getarglist()
+        self.emit_operation(ResOperation(rop.CALL, args, op.result,
+                                         op.getdescr()))
+
     def flush(self):
         assert self.posponedop is None
 
@@ -58,3 +97,23 @@
 
     def setup(self):
         self.optimizer.optpure = self
+
+    def pure(self, opnum, args, result):
+        op = ResOperation(opnum, args, result)
+        key = self.optimizer.make_args_key(op)
+        if key not in self.pure_operations:
+            self.pure_operations[key] = op
+
+    def has_pure_result(self, opnum, args, descr):
+        op = ResOperation(opnum, args, None, descr)
+        key = self.optimizer.make_args_key(op)
+        op = self.pure_operations.get(key, None)
+        if op is None:
+            return False
+        return op.getdescr() is descr
+
+    def get_pure_result(self, key):
+        return self.pure_operations.get(key, None)
+
+dispatch_opt = make_dispatcher_method(OptPure, 'optimize_',
+                                      default=OptPure.optimize_default)
diff --git a/pypy/jit/metainterp/optimizeopt/rewrite.py b/pypy/jit/metainterp/optimizeopt/rewrite.py
--- a/pypy/jit/metainterp/optimizeopt/rewrite.py
+++ b/pypy/jit/metainterp/optimizeopt/rewrite.py
@@ -32,7 +32,7 @@
         dispatch_opt(self, op)
 
     def try_boolinvers(self, op, targs):
-        oldop = self.optimizer.pure_operations.get(targs, None)
+        oldop = self.get_pure_result(targs)
         if oldop is not None and oldop.getdescr() is op.getdescr():
             value = self.getvalue(oldop.result)
             if value.is_constant():
@@ -60,7 +60,7 @@
             oldopnum = opboolreflex[op.getopnum()] # FIXME: add INT_ADD, INT_MUL
             targs = self.optimizer.make_args_key(ResOperation(oldopnum, [args[1], args[0]],
                                                               None))
-            oldop = self.optimizer.pure_operations.get(targs, None)
+            oldop = self.get_pure_result(targs)
             if oldop is not None and oldop.getdescr() is op.getdescr():
                 self.make_equal_to(op.result, self.getvalue(oldop.result))
                 return True
@@ -201,39 +201,6 @@
         self.emit_operation(op)
         self.pure(rop.FLOAT_NEG, [op.result], v1)
 
-    def optimize_CALL_PURE(self, op):
-        arg_consts = []
-        for i in range(op.numargs()):
-            arg = op.getarg(i)
-            const = self.get_constant_box(arg)
-            if const is None:
-                break
-            arg_consts.append(const)
-        else:
-            # all constant arguments: check if we already know the result
-            try:
-                result = self.optimizer.call_pure_results[arg_consts]
-            except KeyError:
-                pass
-            else:
-                self.make_constant(op.result, result)
-                return
-
-        args = self.optimizer.make_args_key(op)
-        oldop = self.optimizer.pure_operations.get(args, None)
-        if oldop is not None and oldop.getdescr() is op.getdescr():
-            assert oldop.getopnum() == op.getopnum()
-            self.make_equal_to(op.result, self.getvalue(oldop.result))
-            return
-        else:
-            self.optimizer.pure_operations[args] = op
-            self.optimizer.remember_emitting_pure(op)
-
-        # replace CALL_PURE with just CALL
-        args = op.getarglist()
-        self.emit_operation(ResOperation(rop.CALL, args, op.result,
-                                         op.getdescr()))
-
     def optimize_guard(self, op, constbox, emit_operation=True):
         value = self.getvalue(op.getarg(0))
         if value.is_constant():
diff --git a/pypy/jit/metainterp/optimizeopt/vstring.py b/pypy/jit/metainterp/optimizeopt/vstring.py
--- a/pypy/jit/metainterp/optimizeopt/vstring.py
+++ b/pypy/jit/metainterp/optimizeopt/vstring.py
@@ -508,6 +508,8 @@
                     return
         self.emit_operation(op)
 
+    optimize_CALL_PURE = optimize_CALL
+
     def opt_call_str_STR2UNICODE(self, op):
         # Constant-fold unicode("constant string").
         # More generally, supporting non-constant but virtual cases is

From noreply at buildbot.pypy.org  Sat Oct  1 19:19:47 2011
From: noreply at buildbot.pypy.org (hakanardo)
Date: Sat,  1 Oct 2011 19:19:47 +0200 (CEST)
Subject: [pypy-commit] pypy jit-optimizeopt-cleanups: move
 emitted_pure_operations dict to OptPure
Message-ID: <20111001171947.650BF82A01@wyvern.cs.uni-duesseldorf.de>

Author: Hakan Ardo <hakan at debian.org>
Branch: jit-optimizeopt-cleanups
Changeset: r47751:273253d0c8f7
Date: 2011-10-01 19:14 +0200
http://bitbucket.org/pypy/pypy/changeset/273253d0c8f7/

Log:	move emitted_pure_operations dict to OptPure

diff --git a/pypy/jit/metainterp/optimizeopt/optimizer.py b/pypy/jit/metainterp/optimizeopt/optimizer.py
--- a/pypy/jit/metainterp/optimizeopt/optimizer.py
+++ b/pypy/jit/metainterp/optimizeopt/optimizer.py
@@ -375,7 +375,8 @@
         return new
 
     def produce_potential_short_preamble_ops(self, sb):
-        raise NotImplementedError('This is implemented in unroll.UnrollableOptimizer')
+        for opt in self.optimizations:
+            opt.produce_potential_short_preamble_ops(sb)
 
     def turned_constant(self, value):
         for o in self.optimizations:
@@ -577,9 +578,6 @@
     def optimize_default(self, op):
         self.emit_operation(op)
 
-    def remember_emitting_pure(self, op):
-        pass
-
     def constant_fold(self, op):
         argboxes = [self.get_constant_box(op.getarg(i))
                     for i in range(op.numargs())]
diff --git a/pypy/jit/metainterp/optimizeopt/pure.py b/pypy/jit/metainterp/optimizeopt/pure.py
--- a/pypy/jit/metainterp/optimizeopt/pure.py
+++ b/pypy/jit/metainterp/optimizeopt/pure.py
@@ -7,6 +7,7 @@
     def __init__(self):
         self.posponedop = None
         self.pure_operations = args_dict()
+        self.emitted_pure_operations = {}
 
     def propagate_forward(self, op):
         dispatch_opt(self, op)
@@ -46,7 +47,7 @@
                 return
             else:
                 self.pure_operations[args] = op
-                self.optimizer.remember_emitting_pure(op)
+                self.remember_emitting_pure(op)
 
         # otherwise, the operation remains
         self.emit_operation(op)
@@ -81,7 +82,7 @@
             return
         else:
             self.pure_operations[args] = op
-            self.optimizer.remember_emitting_pure(op)
+            self.remember_emitting_pure(op)
 
         # replace CALL_PURE with just CALL
         args = op.getarglist()
@@ -115,5 +116,17 @@
     def get_pure_result(self, key):
         return self.pure_operations.get(key, None)
 
+    def remember_emitting_pure(self, op):
+        self.emitted_pure_operations[op] = True
+
+    def produce_potential_short_preamble_ops(self, sb):
+        for op in self.emitted_pure_operations:
+            if op.getopnum() == rop.GETARRAYITEM_GC_PURE or \
+               op.getopnum() == rop.STRGETITEM or \
+               op.getopnum() == rop.UNICODEGETITEM:
+                if not self.getvalue(op.getarg(1)).is_constant():
+                    continue
+            sb.add_potential(op)
+
 dispatch_opt = make_dispatcher_method(OptPure, 'optimize_',
                                       default=OptPure.optimize_default)
diff --git a/pypy/jit/metainterp/optimizeopt/unroll.py b/pypy/jit/metainterp/optimizeopt/unroll.py
--- a/pypy/jit/metainterp/optimizeopt/unroll.py
+++ b/pypy/jit/metainterp/optimizeopt/unroll.py
@@ -75,7 +75,6 @@
         self.importable_values = {}
         self.emitting_dissabled = False
         self.emitted_guards = 0
-        self.emitted_pure_operations = {}
 
     def ensure_imported(self, value):
         if not self.emitting_dissabled and value in self.importable_values:
@@ -96,21 +95,6 @@
         new = UnrollableOptimizer(self.metainterp_sd, self.loop)
         return self._new(new)
 
-    def remember_emitting_pure(self, op):
-        self.emitted_pure_operations[op] = True
-
-    def produce_potential_short_preamble_ops(self, sb):
-        for op in self.emitted_pure_operations:
-            if op.getopnum() == rop.GETARRAYITEM_GC_PURE or \
-               op.getopnum() == rop.STRGETITEM or \
-               op.getopnum() == rop.UNICODEGETITEM:
-                if not self.getvalue(op.getarg(1)).is_constant():
-                    continue
-            sb.add_potential(op)
-        for opt in self.optimizations:
-            opt.produce_potential_short_preamble_ops(sb)
-
-
 
 class UnrollOptimizer(Optimization):
     """Unroll the loop into two iterations. The first one will

From noreply at buildbot.pypy.org  Sat Oct  1 19:19:48 2011
From: noreply at buildbot.pypy.org (hakanardo)
Date: Sat,  1 Oct 2011 19:19:48 +0200 (CEST)
Subject: [pypy-commit] pypy jit-optimizeopt-cleanups: kill
	Optimizer.posponedop
Message-ID: <20111001171948.8E93F82A01@wyvern.cs.uni-duesseldorf.de>

Author: Hakan Ardo <hakan at debian.org>
Branch: jit-optimizeopt-cleanups
Changeset: r47752:e807e1a6d75a
Date: 2011-10-01 19:17 +0200
http://bitbucket.org/pypy/pypy/changeset/e807e1a6d75a/

Log:	kill Optimizer.posponedop

diff --git a/pypy/jit/metainterp/optimizeopt/optimizer.py b/pypy/jit/metainterp/optimizeopt/optimizer.py
--- a/pypy/jit/metainterp/optimizeopt/optimizer.py
+++ b/pypy/jit/metainterp/optimizeopt/optimizer.py
@@ -324,7 +324,6 @@
         self.bool_boxes = {}
         self.producer = {}
         self.pendingfields = []
-        self.posponedop = None
         self.exception_might_have_happened = False
         self.quasi_immutable_deps = None
         self.opaque_pointers = {}
@@ -361,14 +360,12 @@
     def flush(self):
         for o in self.optimizations:
             o.flush()
-        assert self.posponedop is None
 
     def new(self):
         new = Optimizer(self.metainterp_sd, self.loop)
         return self._new(new)
 
     def _new(self, new):
-        assert self.posponedop is None
         optimizations = [o.new() for o in self.optimizations]
         new.set_optimizations(optimizations)
         new.quasi_immutable_deps = self.quasi_immutable_deps

From noreply at buildbot.pypy.org  Sat Oct  1 19:19:49 2011
From: noreply at buildbot.pypy.org (hakanardo)
Date: Sat,  1 Oct 2011 19:19:49 +0200 (CEST)
Subject: [pypy-commit] pypy jit-optimizeopt-cleanups: kill OptString.enabled
Message-ID: <20111001171949.BFFE882A01@wyvern.cs.uni-duesseldorf.de>

Author: Hakan Ardo <hakan at debian.org>
Branch: jit-optimizeopt-cleanups
Changeset: r47753:4720f517fa9b
Date: 2011-10-01 19:19 +0200
http://bitbucket.org/pypy/pypy/changeset/4720f517fa9b/

Log:	kill OptString.enabled

diff --git a/pypy/jit/metainterp/optimizeopt/vstring.py b/pypy/jit/metainterp/optimizeopt/vstring.py
--- a/pypy/jit/metainterp/optimizeopt/vstring.py
+++ b/pypy/jit/metainterp/optimizeopt/vstring.py
@@ -369,8 +369,6 @@
 
 class OptString(optimizer.Optimization):
     "Handling of strings and unicodes."
-    enabled = True
-
     def new(self):
         return OptString()
 
@@ -680,9 +678,6 @@
         self.emit_operation(op)
 
     def propagate_forward(self, op):
-        if not self.enabled:
-            self.emit_operation(op)
-            return
         dispatch_opt(self, op)
 
 dispatch_opt = make_dispatcher_method(OptString, 'optimize_',

From noreply at buildbot.pypy.org  Sat Oct  1 19:32:47 2011
From: noreply at buildbot.pypy.org (hakanardo)
Date: Sat,  1 Oct 2011 19:32:47 +0200 (CEST)
Subject: [pypy-commit] pypy jit-optimizeopt-cleanups: hg merge default
Message-ID: <20111001173247.E8F9E82A01@wyvern.cs.uni-duesseldorf.de>

Author: Hakan Ardo <hakan at debian.org>
Branch: jit-optimizeopt-cleanups
Changeset: r47754:eb3b24bec9e8
Date: 2011-10-01 19:32 +0200
http://bitbucket.org/pypy/pypy/changeset/eb3b24bec9e8/

Log:	hg merge default

diff --git a/pypy/jit/metainterp/optimizeopt/test/test_util.py b/pypy/jit/metainterp/optimizeopt/test/test_util.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_util.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_util.py
@@ -182,7 +182,8 @@
                             EffectInfo.EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE,
                             can_invalidate=True))
     arraycopydescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT,
-             EffectInfo([], [], [], [arraydescr], oopspecindex=EffectInfo.OS_ARRAYCOPY))
+             EffectInfo([], [arraydescr], [], [arraydescr],
+                        oopspecindex=EffectInfo.OS_ARRAYCOPY))
 
     for _name, _os in [
         ('strconcatdescr',               'OS_STR_CONCAT'),
diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -167,10 +167,7 @@
 
     This is for advanced usage only.
     """
-    # I hate the annotator so much.
-    if NonConstant(False):
-        return True
-    return False
+    return NonConstant(False)
 
 @oopspec("jit.isvirtual(value)")
 @specialize.ll()
diff --git a/pypy/rlib/rgc.py b/pypy/rlib/rgc.py
--- a/pypy/rlib/rgc.py
+++ b/pypy/rlib/rgc.py
@@ -3,6 +3,7 @@
 
 from pypy.rlib import jit
 from pypy.rlib.objectmodel import we_are_translated, enforceargs, specialize
+from pypy.rlib.nonconst import NonConstant
 from pypy.rpython.extregistry import ExtRegistryEntry
 from pypy.rpython.lltypesystem import lltype, llmemory
 
@@ -144,7 +145,7 @@
     from pypy.rlib.objectmodel import keepalive_until_here
 
     # XXX: Hack to ensure that we get a proper effectinfo.write_descrs_arrays
-    if length > 0:
+    if NonConstant(False):
         dest[dest_start] = source[source_start]
 
     # supports non-overlapping copies only
diff --git a/pypy/rlib/test/test_jit.py b/pypy/rlib/test/test_jit.py
--- a/pypy/rlib/test/test_jit.py
+++ b/pypy/rlib/test/test_jit.py
@@ -1,7 +1,7 @@
 import py
 from pypy.conftest import option
 from pypy.rlib.jit import hint, we_are_jitted, JitDriver, elidable_promote
-from pypy.rlib.jit import JitHintError, oopspec
+from pypy.rlib.jit import JitHintError, oopspec, isconstant
 from pypy.translator.translator import TranslationContext, graphof
 from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin
 from pypy.rpython.lltypesystem import lltype
@@ -137,6 +137,16 @@
             t.view()
         # assert did not raise
 
+    def test_isconstant(self):
+        def f(n):
+            assert n >= 0
+            assert isconstant(n) is False
+            l = []
+            l.append(n)
+            return len(l)
+        res = self.interpret(f, [234])
+        assert res == 1
+
 
 class TestJITLLtype(BaseTestJIT, LLRtypeMixin):
     pass
diff --git a/pypy/translator/c/genc.py b/pypy/translator/c/genc.py
--- a/pypy/translator/c/genc.py
+++ b/pypy/translator/c/genc.py
@@ -563,7 +563,10 @@
             else:
                 mk.definition('PYPY_MAIN_FUNCTION', "main")
 
-            if sys.platform == 'win32':
+            if (py.path.local.sysfind('python') or
+                py.path.local.sysfind('python.exe')):
+                python = 'python '
+            elif sys.platform == 'win32':
                 python = sys.executable.replace('\\', '/') + ' '
             else:
                 python = sys.executable + ' '

From noreply at buildbot.pypy.org  Sat Oct  1 19:45:52 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sat,  1 Oct 2011 19:45:52 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix list.pop with rangelists.
Message-ID: <20111001174552.BE8ED82A01@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r47755:117dc445ce1f
Date: 2011-10-01 13:45 -0400
http://bitbucket.org/pypy/pypy/changeset/117dc445ce1f/

Log:	fix list.pop with rangelists.

diff --git a/pypy/objspace/std/rangeobject.py b/pypy/objspace/std/rangeobject.py
--- a/pypy/objspace/std/rangeobject.py
+++ b/pypy/objspace/std/rangeobject.py
@@ -23,7 +23,7 @@
 
 class W_RangeListObject(W_Object):
     typedef = listtype.list_typedef
-    
+
     def __init__(w_self, start, step, length):
         assert step != 0
         w_self.start = start
@@ -40,7 +40,7 @@
         if not length:
             w_self.w_list = space.newlist([])
             return w_self.w_list
-        
+
         arr = [None] * length  # this is to avoid using append.
 
         i = start
@@ -146,7 +146,11 @@
     if length == 0:
         raise OperationError(space.w_IndexError,
                              space.wrap("pop from empty list"))
-    idx = space.int_w(w_idx)
+    if space.isinstance_w(w_idx, space.w_float):
+        raise OperationError(space.w_TypeError,
+            space.wrap("integer argument expected, got float")
+        )
+    idx = space.int_w(space.int(w_idx))
     if idx == 0:
         result = w_rangelist.start
         w_rangelist.start += w_rangelist.step
diff --git a/pypy/rpython/lltypesystem/rlist.py b/pypy/rpython/lltypesystem/rlist.py
--- a/pypy/rpython/lltypesystem/rlist.py
+++ b/pypy/rpython/lltypesystem/rlist.py
@@ -1,15 +1,15 @@
+from pypy.rlib import rgc, jit
+from pypy.rlib.debug import ll_assert
+from pypy.rlib.objectmodel import enforceargs
+from pypy.rpython.lltypesystem import rstr
+from pypy.rpython.lltypesystem.lltype import (GcForwardReference, Ptr, GcArray,
+     GcStruct, Void, Signed, malloc, typeOf, nullptr, typeMethod)
+from pypy.rpython.rlist import (AbstractBaseListRepr, AbstractListRepr,
+    AbstractFixedSizeListRepr, AbstractListIteratorRepr, ll_setitem_nonneg,
+    ADTIList, ADTIFixedList, dum_nocheck)
+from pypy.rpython.rmodel import Repr, inputconst, externalvsinternal
 from pypy.tool.pairtype import pairtype, pair
-from pypy.rpython.rmodel import Repr, inputconst
-from pypy.rpython.rmodel import externalvsinternal
-from pypy.rpython.rlist import AbstractBaseListRepr, AbstractListRepr, \
-        AbstractFixedSizeListRepr, AbstractListIteratorRepr, \
-        ll_setitem_nonneg, ADTIList, ADTIFixedList
-from pypy.rpython.rlist import dum_nocheck
-from pypy.rpython.lltypesystem.lltype import GcForwardReference, Ptr, GcArray,\
-     GcStruct, Void, Signed, malloc, typeOf, nullptr, typeMethod
-from pypy.rpython.lltypesystem import rstr
-from pypy.rlib.debug import ll_assert
-from pypy.rlib import rgc, jit
+
 
 # ____________________________________________________________
 #
@@ -171,6 +171,7 @@
 
 # adapted C code
 
+ at enforceargs(None, int)
 def _ll_list_resize_really(l, newsize):
     """
     Ensure l.items has room for at least newsize elements, and set
@@ -210,7 +211,6 @@
         rgc.ll_arraycopy(items, newitems, 0, 0, p)
     l.length = newsize
     l.items = newitems
-_ll_list_resize_really._annenforceargs_ = (None, int)
 
 # this common case was factored out of _ll_list_resize
 # to see if inlining it gives some speed-up.

From noreply at buildbot.pypy.org  Sat Oct  1 19:59:13 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sat,  1 Oct 2011 19:59:13 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix for test_descr.
Message-ID: <20111001175913.615F382A01@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r47756:ad26a4fa9d67
Date: 2011-10-01 13:58 -0400
http://bitbucket.org/pypy/pypy/changeset/ad26a4fa9d67/

Log:	Fix for test_descr.

diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -132,6 +132,19 @@
 
         assert A("hello") is str
 
+        # Make sure type(x) doesn't call x.__class__.__init__
+        class T(type):
+            counter = 0
+            def __init__(self, *args):
+                T.counter += 1
+        class C:
+            __metaclass__ = T
+        assert T.counter == 1
+        a = C()
+        assert T.counter == 1
+        assert type(a) is C
+        assert T.counter == 1
+
     def test_bases(self):
         assert int.__bases__ == (object,)
         class X:
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -848,7 +848,13 @@
         call_init = space.isinstance_w(w_newobject, w_type)
 
     # maybe invoke the __init__ of the type
-    if call_init:
+    try:
+        __args__.fixedunpack(1)
+    except ValueError:
+        single_arg = False
+    else:
+        single_arg = True
+    if call_init and not (space.is_w(w_type, space.w_type) and single_arg):
         w_descr = space.lookup(w_newobject, '__init__')
         w_result = space.get_and_call_args(w_descr, w_newobject, __args__)
         if not space.is_w(w_result, space.w_None):

From noreply at buildbot.pypy.org  Sun Oct  2 00:42:28 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sun,  2 Oct 2011 00:42:28 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix returning an object which defines
 an __int__ from an __len__ method.
Message-ID: <20111001224228.2D1E282A01@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r47757:6e2b325c768e
Date: 2011-10-01 18:42 -0400
http://bitbucket.org/pypy/pypy/changeset/6e2b325c768e/

Log:	fix returning an object which defines an __int__ from an __len__
	method.

diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -258,15 +258,15 @@
             msg = "'%s' has no length" % (name,)
             raise OperationError(space.w_TypeError, space.wrap(msg))
         w_res = space.get_and_call_function(w_descr, w_obj)
-        space._check_len_result(w_res)
-        return w_res
+        return space.wrap(space._check_len_result(w_res))
 
     def _check_len_result(space, w_obj):
         # Will complain if result is too big.
-        result = space.int_w(w_obj)
+        result = space.int_w(space.int(w_obj))
         if result < 0:
             raise OperationError(space.w_ValueError,
                                  space.wrap("__len__() should return >= 0"))
+        return result
 
     def iter(space, w_obj):
         w_descr = space.lookup(w_obj, '__iter__')
diff --git a/pypy/objspace/test/test_descroperation.py b/pypy/objspace/test/test_descroperation.py
--- a/pypy/objspace/test/test_descroperation.py
+++ b/pypy/objspace/test/test_descroperation.py
@@ -667,5 +667,19 @@
                 return -1L
         raises(ValueError, len, Y())
 
+    def test_len_custom__int__(self):
+        class X(object):
+            def __init__(self, x):
+                self.x = x
+            def __len__(self):
+                return self.x
+            def __int__(self):
+                return self.x
+
+        l = len(X(3.0))
+        assert l == 3 and type(l) is int
+        l = len(X(X(2)))
+        assert l == 2 and type(l) is int
+
 class AppTestWithBuiltinShortcut(AppTest_Descroperation):
     OPTIONS = {'objspace.std.builtinshortcut': True}

From noreply at buildbot.pypy.org  Sun Oct  2 01:16:28 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Sun,  2 Oct 2011 01:16:28 +0200 (CEST)
Subject: [pypy-commit] pypy lightweight-finalizers: support the case of also
	having a del
Message-ID: <20111001231628.5C35C82A01@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: lightweight-finalizers
Changeset: r47758:444aae7ad4e7
Date: 2011-10-01 14:26 -0300
http://bitbucket.org/pypy/pypy/changeset/444aae7ad4e7/

Log:	support the case of also having a del

diff --git a/pypy/rlib/rgc.py b/pypy/rlib/rgc.py
--- a/pypy/rlib/rgc.py
+++ b/pypy/rlib/rgc.py
@@ -452,7 +452,10 @@
         def remove_raw_mem_attr(self):
             if getattr(self, name):
                 lltype.free(getattr(self, name), flavor='raw')
-        
+            if orig_del is not None:
+                orig_del(self)
+
+        orig_del = getattr(cls, '__del__', None)
         cls._raw_mem_ptr_name = name
         cls.__del__ = remove_raw_mem_attr
         return cls
diff --git a/pypy/rlib/test/test_rgc.py b/pypy/rlib/test/test_rgc.py
--- a/pypy/rlib/test/test_rgc.py
+++ b/pypy/rlib/test/test_rgc.py
@@ -191,3 +191,25 @@
     gc.collect()
     assert ptr._was_freed()
     assert not ptr2
+
+def test_raw_memory_owner_with_del():
+    T = lltype.Struct('X', ('x', lltype.Signed))
+    collected = []
+
+    @rgc.owns_raw_memory('p')
+    class X(object):
+        p = lltype.nullptr(T)
+        
+        def __init__(self, arg):
+            if arg:
+                self.p = lltype.malloc(T, flavor='raw')
+
+        def __del__(self):
+            collected.append(None)
+    
+    a = X(3)
+    ptr = a.p
+    del a
+    gc.collect()
+    assert ptr._was_freed()
+    assert collected

From noreply at buildbot.pypy.org  Sun Oct  2 01:16:29 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Sun,  2 Oct 2011 01:16:29 +0200 (CEST)
Subject: [pypy-commit] pypy lightweight-finalizers: a test and a fix
Message-ID: <20111001231629.9667182A01@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: lightweight-finalizers
Changeset: r47759:171d990aedfd
Date: 2011-10-01 15:15 -0300
http://bitbucket.org/pypy/pypy/changeset/171d990aedfd/

Log:	a test and a fix

diff --git a/pypy/rpython/memory/gctypelayout.py b/pypy/rpython/memory/gctypelayout.py
--- a/pypy/rpython/memory/gctypelayout.py
+++ b/pypy/rpython/memory/gctypelayout.py
@@ -250,7 +250,17 @@
     if builder.get_raw_mem_attr_name(TYPE):
         name = builder.get_raw_mem_attr_name(TYPE)
         infobits |= T_HAS_RAW_MEM_PTR
-        info.ofstorawptr = llmemory.offsetof(TYPE, 'inst_' + name)
+        # can be in a superclass
+        T = TYPE
+        l = []
+        while not hasattr(T, 'inst_' + name):
+            l.append(llmemory.offsetof(T, 'super'))
+            T = T.super
+        l.append(llmemory.offsetof(T, 'inst_' + name))
+        ofs = l[0]
+        for i in range(1, len(l)):
+            ofs += l[i]
+        info.ofstorawptr = ofs
     info.infobits = infobits | T_KEY_VALUE
 
 # ____________________________________________________________
diff --git a/pypy/rpython/memory/test/test_gc.py b/pypy/rpython/memory/test/test_gc.py
--- a/pypy/rpython/memory/test/test_gc.py
+++ b/pypy/rpython/memory/test/test_gc.py
@@ -141,10 +141,16 @@
                 if arg:
                     self.p = lltype.malloc(T, flavor='raw')
 
+        class B(AClass):
+            pass
+
         def f():
             a = AClass(0)
             for i in range(30):
-                a = AClass(3)
+                if i % 2:
+                    a = B(3)
+                else:
+                    a = AClass(3)
                 AClass(0)
             llop.gc__collect(lltype.Void)
             assert a.p
diff --git a/pypy/rpython/test/test_rclass.py b/pypy/rpython/test/test_rclass.py
--- a/pypy/rpython/test/test_rclass.py
+++ b/pypy/rpython/test/test_rclass.py
@@ -998,7 +998,7 @@
         RTTI = getRuntimeTypeInfo(TYPE)
         RTTI._obj.query_funcptr # should not raise
         assert RTTI._obj.raw_mem_attr_name == 'p'
-    
+
     def test_del_inheritance(self):
         from pypy.rlib import rgc
         class State:

From noreply at buildbot.pypy.org  Sun Oct  2 01:16:30 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Sun,  2 Oct 2011 01:16:30 +0200 (CEST)
Subject: [pypy-commit] pypy rmmap-nohint: no noticable improvements
Message-ID: <20111001231630.D0DE282A01@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: rmmap-nohint
Changeset: r47760:08713d1ad7d8
Date: 2011-10-01 15:34 -0300
http://bitbucket.org/pypy/pypy/changeset/08713d1ad7d8/

Log:	no noticable improvements


From noreply at buildbot.pypy.org  Sun Oct  2 01:16:32 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Sun,  2 Oct 2011 01:16:32 +0200 (CEST)
Subject: [pypy-commit] pypy lightweight-finalizers: Write a test that should
 remind that when we implement raw malloc in jit, 
Message-ID: <20111001231632.121FF82A01@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: lightweight-finalizers
Changeset: r47761:44e2476c82c3
Date: 2011-10-01 15:36 -0300
http://bitbucket.org/pypy/pypy/changeset/44e2476c82c3/

Log:	Write a test that should remind that when we implement raw malloc in
	jit, we should deal with lightweight finalizers in the JIT

diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -3408,6 +3408,30 @@
         assert res == main(1, 10)
         self.check_loops(call=0)
 
+    def test_virtual_lightweight_finalizer(self):
+        py.test.skip("raw mallocs unsupported, otherwise this would be a problem")
+        from pypy.rlib import rgc
+        import gc
+
+        S = lltype.Struct('S', ('x', lltype.Signed))
+        
+        @rgc.owns_raw_memory('p')
+        class A(object):
+            def __init__(self):
+                self.p = lltype.malloc(S, flavor='raw')
+
+        driver = JitDriver(greens = [], reds = ['i', 'a'])
+
+        def f(i):
+            a = None
+            while i > 0:
+                driver.jit_merge_point(i=i, a=a)
+                a = A()
+                i -= 1
+            gc.collect()
+
+        self.meta_interp(f, [10])
+
 
 class TestLLtype(BaseLLtypeTests, LLJitMixin):
     pass

From noreply at buildbot.pypy.org  Sun Oct  2 01:16:36 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Sun,  2 Oct 2011 01:16:36 +0200 (CEST)
Subject: [pypy-commit] pypy inline-dict-ops: merge default
Message-ID: <20111001231636.A107C82A01@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: inline-dict-ops
Changeset: r47762:2a6c7e0fe479
Date: 2011-10-01 15:45 -0300
http://bitbucket.org/pypy/pypy/changeset/2a6c7e0fe479/

Log:	merge default

diff too long, truncating to 10000 out of 16893 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -1,2 +1,3 @@
 b590cf6de4190623aad9aa698694c22e614d67b9 release-1.5
 b48df0bf4e75b81d98f19ce89d4a7dc3e1dab5e5 benchmarked
+d8ac7d23d3ec5f9a0fa1264972f74a010dbfd07f release-1.6
diff --git a/dotviewer/graphparse.py b/dotviewer/graphparse.py
--- a/dotviewer/graphparse.py
+++ b/dotviewer/graphparse.py
@@ -36,48 +36,45 @@
     print >> sys.stderr, "Warning: could not guess file type, using 'dot'"
     return 'unknown'
 
-def dot2plain(content, contenttype, use_codespeak=False):
-    if contenttype == 'plain':
-        # already a .plain file
-        return content
+def dot2plain_graphviz(content, contenttype, use_codespeak=False):
+    if contenttype != 'neato':
+        cmdline = 'dot -Tplain'
+    else:
+        cmdline = 'neato -Tplain'
+    #print >> sys.stderr, '* running:', cmdline
+    close_fds = sys.platform != 'win32'
+    p = subprocess.Popen(cmdline, shell=True, close_fds=close_fds,
+                         stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+    (child_in, child_out) = (p.stdin, p.stdout)
+    try:
+        import thread
+    except ImportError:
+        bkgndwrite(child_in, content)
+    else:
+        thread.start_new_thread(bkgndwrite, (child_in, content))
+    plaincontent = child_out.read()
+    child_out.close()
+    if not plaincontent:    # 'dot' is likely not installed
+        raise PlainParseError("no result from running 'dot'")
+    return plaincontent
 
-    if not use_codespeak:
-        if contenttype != 'neato':
-            cmdline = 'dot -Tplain'
-        else:
-            cmdline = 'neato -Tplain'
-        #print >> sys.stderr, '* running:', cmdline
-        close_fds = sys.platform != 'win32'
-        p = subprocess.Popen(cmdline, shell=True, close_fds=close_fds,
-                             stdin=subprocess.PIPE, stdout=subprocess.PIPE)
-        (child_in, child_out) = (p.stdin, p.stdout)
-        try:
-            import thread
-        except ImportError:
-            bkgndwrite(child_in, content)
-        else:
-            thread.start_new_thread(bkgndwrite, (child_in, content))
-        plaincontent = child_out.read()
-        child_out.close()
-        if not plaincontent:    # 'dot' is likely not installed
-            raise PlainParseError("no result from running 'dot'")
-    else:
-        import urllib
-        request = urllib.urlencode({'dot': content})
-        url = 'http://codespeak.net/pypy/convertdot.cgi'
-        print >> sys.stderr, '* posting:', url
-        g = urllib.urlopen(url, data=request)
-        result = []
-        while True:
-            data = g.read(16384)
-            if not data:
-                break
-            result.append(data)
-        g.close()
-        plaincontent = ''.join(result)
-        # very simple-minded way to give a somewhat better error message
-        if plaincontent.startswith('<body'):
-            raise Exception("the dot on codespeak has very likely crashed")
+def dot2plain_codespeak(content, contenttype):
+    import urllib
+    request = urllib.urlencode({'dot': content})
+    url = 'http://codespeak.net/pypy/convertdot.cgi'
+    print >> sys.stderr, '* posting:', url
+    g = urllib.urlopen(url, data=request)
+    result = []
+    while True:
+        data = g.read(16384)
+        if not data:
+            break
+        result.append(data)
+    g.close()
+    plaincontent = ''.join(result)
+    # very simple-minded way to give a somewhat better error message
+    if plaincontent.startswith('<body'):
+        raise Exception("the dot on codespeak has very likely crashed")
     return plaincontent
 
 def bkgndwrite(f, data):
@@ -148,10 +145,13 @@
 
 def parse_dot(graph_id, content, links={}, fixedfont=False):
     contenttype = guess_type(content)
-    try:
-        plaincontent = dot2plain(content, contenttype, use_codespeak=False)
-        return list(parse_plain(graph_id, plaincontent, links, fixedfont))
-    except PlainParseError:
-        # failed, retry via codespeak
-        plaincontent = dot2plain(content, contenttype, use_codespeak=True)
-        return list(parse_plain(graph_id, plaincontent, links, fixedfont))
+    if contenttype == 'plain':
+        plaincontent = content
+    else:
+        try:
+            plaincontent = dot2plain_graphviz(content, contenttype)
+        except PlainParseError, e:
+            print e
+            # failed, retry via codespeak
+            plaincontent = dot2plain_codespeak(content, contenttype)
+    return list(parse_plain(graph_id, plaincontent, links, fixedfont))
diff --git a/lib-python/2.7/ssl.py b/lib-python/2.7/ssl.py
--- a/lib-python/2.7/ssl.py
+++ b/lib-python/2.7/ssl.py
@@ -62,7 +62,6 @@
 from _ssl import OPENSSL_VERSION_NUMBER, OPENSSL_VERSION_INFO, OPENSSL_VERSION
 from _ssl import SSLError
 from _ssl import CERT_NONE, CERT_OPTIONAL, CERT_REQUIRED
-from _ssl import PROTOCOL_SSLv2, PROTOCOL_SSLv3, PROTOCOL_SSLv23, PROTOCOL_TLSv1
 from _ssl import RAND_status, RAND_egd, RAND_add
 from _ssl import \
      SSL_ERROR_ZERO_RETURN, \
@@ -74,6 +73,18 @@
      SSL_ERROR_WANT_CONNECT, \
      SSL_ERROR_EOF, \
      SSL_ERROR_INVALID_ERROR_CODE
+from _ssl import PROTOCOL_SSLv3, PROTOCOL_SSLv23, PROTOCOL_TLSv1
+_PROTOCOL_NAMES = {
+    PROTOCOL_TLSv1: "TLSv1",
+    PROTOCOL_SSLv23: "SSLv23",
+    PROTOCOL_SSLv3: "SSLv3",
+}
+try:
+    from _ssl import PROTOCOL_SSLv2
+except ImportError:
+    pass
+else:
+    _PROTOCOL_NAMES[PROTOCOL_SSLv2] = "SSLv2"
 
 from socket import socket, _fileobject, _delegate_methods, error as socket_error
 from socket import getnameinfo as _getnameinfo
@@ -408,16 +419,7 @@
     return DER_cert_to_PEM_cert(dercert)
 
 def get_protocol_name(protocol_code):
-    if protocol_code == PROTOCOL_TLSv1:
-        return "TLSv1"
-    elif protocol_code == PROTOCOL_SSLv23:
-        return "SSLv23"
-    elif protocol_code == PROTOCOL_SSLv2:
-        return "SSLv2"
-    elif protocol_code == PROTOCOL_SSLv3:
-        return "SSLv3"
-    else:
-        return "<unknown>"
+    return _PROTOCOL_NAMES.get(protocol_code, '<unknown>')
 
 
 # a replacement for the old socket.ssl function
diff --git a/lib-python/2.7/test/test_ssl.py b/lib-python/2.7/test/test_ssl.py
--- a/lib-python/2.7/test/test_ssl.py
+++ b/lib-python/2.7/test/test_ssl.py
@@ -58,32 +58,35 @@
 
 # Issue #9415: Ubuntu hijacks their OpenSSL and forcefully disables SSLv2
 def skip_if_broken_ubuntu_ssl(func):
-    # We need to access the lower-level wrapper in order to create an
-    # implicit SSL context without trying to connect or listen.
-    try:
-        import _ssl
-    except ImportError:
-        # The returned function won't get executed, just ignore the error
-        pass
-    @functools.wraps(func)
-    def f(*args, **kwargs):
+    if hasattr(ssl, 'PROTOCOL_SSLv2'):
+        # We need to access the lower-level wrapper in order to create an
+        # implicit SSL context without trying to connect or listen.
         try:
-            s = socket.socket(socket.AF_INET)
-            _ssl.sslwrap(s._sock, 0, None, None,
-                         ssl.CERT_NONE, ssl.PROTOCOL_SSLv2, None, None)
-        except ssl.SSLError as e:
-            if (ssl.OPENSSL_VERSION_INFO == (0, 9, 8, 15, 15) and
-                platform.linux_distribution() == ('debian', 'squeeze/sid', '')
-                and 'Invalid SSL protocol variant specified' in str(e)):
-                raise unittest.SkipTest("Patched Ubuntu OpenSSL breaks behaviour")
-        return func(*args, **kwargs)
-    return f
+            import _ssl
+        except ImportError:
+            # The returned function won't get executed, just ignore the error
+            pass
+        @functools.wraps(func)
+        def f(*args, **kwargs):
+            try:
+                s = socket.socket(socket.AF_INET)
+                _ssl.sslwrap(s._sock, 0, None, None,
+                             ssl.CERT_NONE, ssl.PROTOCOL_SSLv2, None, None)
+            except ssl.SSLError as e:
+                if (ssl.OPENSSL_VERSION_INFO == (0, 9, 8, 15, 15) and
+                    platform.linux_distribution() == ('debian', 'squeeze/sid', '')
+                    and 'Invalid SSL protocol variant specified' in str(e)):
+                    raise unittest.SkipTest("Patched Ubuntu OpenSSL breaks behaviour")
+            return func(*args, **kwargs)
+        return f
+    else:
+        return func
 
 
 class BasicSocketTests(unittest.TestCase):
 
     def test_constants(self):
-        ssl.PROTOCOL_SSLv2
+        #ssl.PROTOCOL_SSLv2
         ssl.PROTOCOL_SSLv23
         ssl.PROTOCOL_SSLv3
         ssl.PROTOCOL_TLSv1
@@ -964,7 +967,8 @@
             try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, True)
             try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, True, ssl.CERT_OPTIONAL)
             try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, True, ssl.CERT_REQUIRED)
-            try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv2, False)
+            if hasattr(ssl, 'PROTOCOL_SSLv2'):
+                try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv2, False)
             try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv23, False)
             try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_TLSv1, False)
 
@@ -976,7 +980,8 @@
             try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, True)
             try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, True, ssl.CERT_OPTIONAL)
             try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, True, ssl.CERT_REQUIRED)
-            try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv2, False)
+            if hasattr(ssl, 'PROTOCOL_SSLv2'):
+                try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv2, False)
             try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv3, False)
             try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv23, False)
 
diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -317,7 +317,7 @@
     RegrTest('test_multibytecodec.py', usemodules='_multibytecodec'),
     RegrTest('test_multibytecodec_support.py', skip="not a test"),
     RegrTest('test_multifile.py'),
-    RegrTest('test_multiprocessing.py', skip='FIXME leaves subprocesses'),
+    RegrTest('test_multiprocessing.py', skip="FIXME leaves subprocesses"),
     RegrTest('test_mutants.py', core="possibly"),
     RegrTest('test_mutex.py'),
     RegrTest('test_netrc.py'),
@@ -359,7 +359,7 @@
     RegrTest('test_property.py', core=True),
     RegrTest('test_pstats.py'),
     RegrTest('test_pty.py', skip="unsupported extension module"),
-    RegrTest('test_pwd.py', skip=skip_win32),
+    RegrTest('test_pwd.py', usemodules="pwd", skip=skip_win32),
     RegrTest('test_py3kwarn.py'),
     RegrTest('test_pyclbr.py'),
     RegrTest('test_pydoc.py'),
diff --git a/lib-python/modified-2.7/gzip.py b/lib-python/modified-2.7/gzip.py
deleted file mode 100644
--- a/lib-python/modified-2.7/gzip.py
+++ /dev/null
@@ -1,514 +0,0 @@
-"""Functions that read and write gzipped files.
-
-The user of the file doesn't have to worry about the compression,
-but random access is not allowed."""
-
-# based on Andrew Kuchling's minigzip.py distributed with the zlib module
-
-import struct, sys, time, os
-import zlib
-import io
-import __builtin__
-
-__all__ = ["GzipFile","open"]
-
-FTEXT, FHCRC, FEXTRA, FNAME, FCOMMENT = 1, 2, 4, 8, 16
-
-READ, WRITE = 1, 2
-
-def write32u(output, value):
-    # The L format writes the bit pattern correctly whether signed
-    # or unsigned.
-    output.write(struct.pack("<L", value))
-
-def read32(input):
-    return struct.unpack("<I", input.read(4))[0]
-
-def open(filename, mode="rb", compresslevel=9):
-    """Shorthand for GzipFile(filename, mode, compresslevel).
-
-    The filename argument is required; mode defaults to 'rb'
-    and compresslevel defaults to 9.
-
-    """
-    return GzipFile(filename, mode, compresslevel)
-
-class GzipFile(io.BufferedIOBase):
-    """The GzipFile class simulates most of the methods of a file object with
-    the exception of the readinto() and truncate() methods.
-
-    """
-
-    myfileobj = None
-    max_read_chunk = 10 * 1024 * 1024   # 10Mb
-
-    def __init__(self, filename=None, mode=None,
-                 compresslevel=9, fileobj=None, mtime=None):
-        """Constructor for the GzipFile class.
-
-        At least one of fileobj and filename must be given a
-        non-trivial value.
-
-        The new class instance is based on fileobj, which can be a regular
-        file, a StringIO object, or any other object which simulates a file.
-        It defaults to None, in which case filename is opened to provide
-        a file object.
-
-        When fileobj is not None, the filename argument is only used to be
-        included in the gzip file header, which may includes the original
-        filename of the uncompressed file.  It defaults to the filename of
-        fileobj, if discernible; otherwise, it defaults to the empty string,
-        and in this case the original filename is not included in the header.
-
-        The mode argument can be any of 'r', 'rb', 'a', 'ab', 'w', or 'wb',
-        depending on whether the file will be read or written.  The default
-        is the mode of fileobj if discernible; otherwise, the default is 'rb'.
-        Be aware that only the 'rb', 'ab', and 'wb' values should be used
-        for cross-platform portability.
-
-        The compresslevel argument is an integer from 1 to 9 controlling the
-        level of compression; 1 is fastest and produces the least compression,
-        and 9 is slowest and produces the most compression.  The default is 9.
-
-        The mtime argument is an optional numeric timestamp to be written
-        to the stream when compressing.  All gzip compressed streams
-        are required to contain a timestamp.  If omitted or None, the
-        current time is used.  This module ignores the timestamp when
-        decompressing; however, some programs, such as gunzip, make use
-        of it.  The format of the timestamp is the same as that of the
-        return value of time.time() and of the st_mtime member of the
-        object returned by os.stat().
-
-        """
-
-        # guarantee the file is opened in binary mode on platforms
-        # that care about that sort of thing
-        if mode and 'b' not in mode:
-            mode += 'b'
-        if fileobj is None:
-            fileobj = self.myfileobj = __builtin__.open(filename, mode or 'rb')
-        if filename is None:
-            if hasattr(fileobj, 'name'): filename = fileobj.name
-            else: filename = ''
-        if mode is None:
-            if hasattr(fileobj, 'mode'): mode = fileobj.mode
-            else: mode = 'rb'
-
-        if mode[0:1] == 'r':
-            self.mode = READ
-            # Set flag indicating start of a new member
-            self._new_member = True
-            # Buffer data read from gzip file. extrastart is offset in
-            # stream where buffer starts. extrasize is number of
-            # bytes remaining in buffer from current stream position.
-            self.extrabuf = ""
-            self.extrasize = 0
-            self.extrastart = 0
-            self.name = filename
-            # Starts small, scales exponentially
-            self.min_readsize = 100
-
-        elif mode[0:1] == 'w' or mode[0:1] == 'a':
-            self.mode = WRITE
-            self._init_write(filename)
-            self.compress = zlib.compressobj(compresslevel,
-                                             zlib.DEFLATED,
-                                             -zlib.MAX_WBITS,
-                                             zlib.DEF_MEM_LEVEL,
-                                             0)
-        else:
-            raise IOError, "Mode " + mode + " not supported"
-
-        self.fileobj = fileobj
-        self.offset = 0
-        self.mtime = mtime
-
-        if self.mode == WRITE:
-            self._write_gzip_header()
-
-    @property
-    def filename(self):
-        import warnings
-        warnings.warn("use the name attribute", DeprecationWarning, 2)
-        if self.mode == WRITE and self.name[-3:] != ".gz":
-            return self.name + ".gz"
-        return self.name
-
-    def __repr__(self):
-        s = repr(self.fileobj)
-        return '<gzip ' + s[1:-1] + ' ' + hex(id(self)) + '>'
-
-    def _check_closed(self):
-        """Raises a ValueError if the underlying file object has been closed.
-
-        """
-        if self.closed:
-            raise ValueError('I/O operation on closed file.')
-
-    def _init_write(self, filename):
-        self.name = filename
-        self.crc = zlib.crc32("") & 0xffffffffL
-        self.size = 0
-        self.writebuf = []
-        self.bufsize = 0
-
-    def _write_gzip_header(self):
-        self.fileobj.write('\037\213')             # magic header
-        self.fileobj.write('\010')                 # compression method
-        fname = os.path.basename(self.name)
-        if fname.endswith(".gz"):
-            fname = fname[:-3]
-        flags = 0
-        if fname:
-            flags = FNAME
-        self.fileobj.write(chr(flags))
-        mtime = self.mtime
-        if mtime is None:
-            mtime = time.time()
-        write32u(self.fileobj, long(mtime))
-        self.fileobj.write('\002')
-        self.fileobj.write('\377')
-        if fname:
-            self.fileobj.write(fname + '\000')
-
-    def _init_read(self):
-        self.crc = zlib.crc32("") & 0xffffffffL
-        self.size = 0
-
-    def _read_gzip_header(self):
-        magic = self.fileobj.read(2)
-        if magic != '\037\213':
-            raise IOError, 'Not a gzipped file'
-        method = ord( self.fileobj.read(1) )
-        if method != 8:
-            raise IOError, 'Unknown compression method'
-        flag = ord( self.fileobj.read(1) )
-        self.mtime = read32(self.fileobj)
-        # extraflag = self.fileobj.read(1)
-        # os = self.fileobj.read(1)
-        self.fileobj.read(2)
-
-        if flag & FEXTRA:
-            # Read & discard the extra field, if present
-            xlen = ord(self.fileobj.read(1))
-            xlen = xlen + 256*ord(self.fileobj.read(1))
-            self.fileobj.read(xlen)
-        if flag & FNAME:
-            # Read and discard a null-terminated string containing the filename
-            while True:
-                s = self.fileobj.read(1)
-                if not s or s=='\000':
-                    break
-        if flag & FCOMMENT:
-            # Read and discard a null-terminated string containing a comment
-            while True:
-                s = self.fileobj.read(1)
-                if not s or s=='\000':
-                    break
-        if flag & FHCRC:
-            self.fileobj.read(2)     # Read & discard the 16-bit header CRC
-
-    def write(self,data):
-        self._check_closed()
-        if self.mode != WRITE:
-            import errno
-            raise IOError(errno.EBADF, "write() on read-only GzipFile object")
-
-        if self.fileobj is None:
-            raise ValueError, "write() on closed GzipFile object"
-
-        # Convert data type if called by io.BufferedWriter.
-        if isinstance(data, memoryview):
-            data = data.tobytes()
-
-        if len(data) > 0:
-            self.size = self.size + len(data)
-            self.crc = zlib.crc32(data, self.crc) & 0xffffffffL
-            self.fileobj.write( self.compress.compress(data) )
-            self.offset += len(data)
-
-        return len(data)
-
-    def read(self, size=-1):
-        self._check_closed()
-        if self.mode != READ:
-            import errno
-            raise IOError(errno.EBADF, "read() on write-only GzipFile object")
-
-        if self.extrasize <= 0 and self.fileobj is None:
-            return ''
-
-        readsize = 1024
-        if size < 0:        # get the whole thing
-            try:
-                while True:
-                    self._read(readsize)
-                    readsize = min(self.max_read_chunk, readsize * 2)
-            except EOFError:
-                size = self.extrasize
-        elif size == 0:
-            return ""
-        else:               # just get some more of it
-            try:
-                while size > self.extrasize:
-                    self._read(readsize)
-                    readsize = min(self.max_read_chunk, readsize * 2)
-            except EOFError:
-                if size > self.extrasize:
-                    size = self.extrasize
-
-        offset = self.offset - self.extrastart
-        chunk = self.extrabuf[offset: offset + size]
-        self.extrasize = self.extrasize - size
-
-        self.offset += size
-        return chunk
-
-    def _unread(self, buf):
-        self.extrasize = len(buf) + self.extrasize
-        self.offset -= len(buf)
-
-    def _read(self, size=1024):
-        if self.fileobj is None:
-            raise EOFError, "Reached EOF"
-
-        if self._new_member:
-            # If the _new_member flag is set, we have to
-            # jump to the next member, if there is one.
-            #
-            # First, check if we're at the end of the file;
-            # if so, it's time to stop; no more members to read.
-            pos = self.fileobj.tell()   # Save current position
-            self.fileobj.seek(0, 2)     # Seek to end of file
-            if pos == self.fileobj.tell():
-                raise EOFError, "Reached EOF"
-            else:
-                self.fileobj.seek( pos ) # Return to original position
-
-            self._init_read()
-            self._read_gzip_header()
-            self.decompress = zlib.decompressobj(-zlib.MAX_WBITS)
-            self._new_member = False
-
-        # Read a chunk of data from the file
-        buf = self.fileobj.read(size)
-
-        # If the EOF has been reached, flush the decompression object
-        # and mark this object as finished.
-
-        if buf == "":
-            uncompress = self.decompress.flush()
-            self._read_eof()
-            self._add_read_data( uncompress )
-            raise EOFError, 'Reached EOF'
-
-        uncompress = self.decompress.decompress(buf)
-        self._add_read_data( uncompress )
-
-        if self.decompress.unused_data != "":
-            # Ending case: we've come to the end of a member in the file,
-            # so seek back to the start of the unused data, finish up
-            # this member, and read a new gzip header.
-            # (The number of bytes to seek back is the length of the unused
-            # data, minus 8 because _read_eof() will rewind a further 8 bytes)
-            self.fileobj.seek( -len(self.decompress.unused_data)+8, 1)
-
-            # Check the CRC and file size, and set the flag so we read
-            # a new member on the next call
-            self._read_eof()
-            self._new_member = True
-
-    def _add_read_data(self, data):
-        self.crc = zlib.crc32(data, self.crc) & 0xffffffffL
-        offset = self.offset - self.extrastart
-        self.extrabuf = self.extrabuf[offset:] + data
-        self.extrasize = self.extrasize + len(data)
-        self.extrastart = self.offset
-        self.size = self.size + len(data)
-
-    def _read_eof(self):
-        # We've read to the end of the file, so we have to rewind in order
-        # to reread the 8 bytes containing the CRC and the file size.
-        # We check the that the computed CRC and size of the
-        # uncompressed data matches the stored values.  Note that the size
-        # stored is the true file size mod 2**32.
-        self.fileobj.seek(-8, 1)
-        crc32 = read32(self.fileobj)
-        isize = read32(self.fileobj)  # may exceed 2GB
-        if crc32 != self.crc:
-            raise IOError("CRC check failed %s != %s" % (hex(crc32),
-                                                         hex(self.crc)))
-        elif isize != (self.size & 0xffffffffL):
-            raise IOError, "Incorrect length of data produced"
-
-        # Gzip files can be padded with zeroes and still have archives.
-        # Consume all zero bytes and set the file position to the first
-        # non-zero byte. See http://www.gzip.org/#faq8
-        c = "\x00"
-        while c == "\x00":
-            c = self.fileobj.read(1)
-        if c:
-            self.fileobj.seek(-1, 1)
-
-    @property
-    def closed(self):
-        return self.fileobj is None
-
-    def close(self):
-        if self.fileobj is None:
-            return
-        if self.mode == WRITE:
-            self.fileobj.write(self.compress.flush())
-            write32u(self.fileobj, self.crc)
-            # self.size may exceed 2GB, or even 4GB
-            write32u(self.fileobj, self.size & 0xffffffffL)
-            self.fileobj = None
-        elif self.mode == READ:
-            self.fileobj = None
-        if self.myfileobj:
-            self.myfileobj.close()
-            self.myfileobj = None
-
-    def flush(self,zlib_mode=zlib.Z_SYNC_FLUSH):
-        self._check_closed()
-        if self.mode == WRITE:
-            # Ensure the compressor's buffer is flushed
-            self.fileobj.write(self.compress.flush(zlib_mode))
-            self.fileobj.flush()
-
-    def fileno(self):
-        """Invoke the underlying file object's fileno() method.
-
-        This will raise AttributeError if the underlying file object
-        doesn't support fileno().
-        """
-        return self.fileobj.fileno()
-
-    def rewind(self):
-        '''Return the uncompressed stream file position indicator to the
-        beginning of the file'''
-        if self.mode != READ:
-            raise IOError("Can't rewind in write mode")
-        self.fileobj.seek(0)
-        self._new_member = True
-        self.extrabuf = ""
-        self.extrasize = 0
-        self.extrastart = 0
-        self.offset = 0
-
-    def readable(self):
-        return self.mode == READ
-
-    def writable(self):
-        return self.mode == WRITE
-
-    def seekable(self):
-        return True
-
-    def seek(self, offset, whence=0):
-        if whence:
-            if whence == 1:
-                offset = self.offset + offset
-            else:
-                raise ValueError('Seek from end not supported')
-        if self.mode == WRITE:
-            if offset < self.offset:
-                raise IOError('Negative seek in write mode')
-            count = offset - self.offset
-            for i in range(count // 1024):
-                self.write(1024 * '\0')
-            self.write((count % 1024) * '\0')
-        elif self.mode == READ:
-            if offset == self.offset:
-                self.read(0) # to make sure that this file is open
-                return self.offset
-            if offset < self.offset:
-                # for negative seek, rewind and do positive seek
-                self.rewind()
-            count = offset - self.offset
-            for i in range(count // 1024):
-                self.read(1024)
-            self.read(count % 1024)
-
-        return self.offset
-
-    def readline(self, size=-1):
-        if size < 0:
-            # Shortcut common case - newline found in buffer.
-            offset = self.offset - self.extrastart
-            i = self.extrabuf.find('\n', offset) + 1
-            if i > 0:
-                self.extrasize -= i - offset
-                self.offset += i - offset
-                return self.extrabuf[offset: i]
-
-            size = sys.maxint
-            readsize = self.min_readsize
-        else:
-            readsize = size
-        bufs = []
-        while size != 0:
-            c = self.read(readsize)
-            i = c.find('\n')
-
-            # We set i=size to break out of the loop under two
-            # conditions: 1) there's no newline, and the chunk is
-            # larger than size, or 2) there is a newline, but the
-            # resulting line would be longer than 'size'.
-            if (size <= i) or (i == -1 and len(c) > size):
-                i = size - 1
-
-            if i >= 0 or c == '':
-                bufs.append(c[:i + 1])    # Add portion of last chunk
-                self._unread(c[i + 1:])   # Push back rest of chunk
-                break
-
-            # Append chunk to list, decrease 'size',
-            bufs.append(c)
-            size = size - len(c)
-            readsize = min(size, readsize * 2)
-        if readsize > self.min_readsize:
-            self.min_readsize = min(readsize, self.min_readsize * 2, 512)
-        return ''.join(bufs) # Return resulting line
-
-
-def _test():
-    # Act like gzip; with -d, act like gunzip.
-    # The input file is not deleted, however, nor are any other gzip
-    # options or features supported.
-    args = sys.argv[1:]
-    decompress = args and args[0] == "-d"
-    if decompress:
-        args = args[1:]
-    if not args:
-        args = ["-"]
-    for arg in args:
-        if decompress:
-            if arg == "-":
-                f = GzipFile(filename="", mode="rb", fileobj=sys.stdin)
-                g = sys.stdout
-            else:
-                if arg[-3:] != ".gz":
-                    print "filename doesn't end in .gz:", repr(arg)
-                    continue
-                f = open(arg, "rb")
-                g = __builtin__.open(arg[:-3], "wb")
-        else:
-            if arg == "-":
-                f = sys.stdin
-                g = GzipFile(filename="", mode="wb", fileobj=sys.stdout)
-            else:
-                f = __builtin__.open(arg, "rb")
-                g = open(arg + ".gz", "wb")
-        while True:
-            chunk = f.read(1024)
-            if not chunk:
-                break
-            g.write(chunk)
-        if g is not sys.stdout:
-            g.close()
-        if f is not sys.stdin:
-            f.close()
-
-if __name__ == '__main__':
-    _test()
diff --git a/lib-python/modified-2.7/ssl.py b/lib-python/modified-2.7/ssl.py
--- a/lib-python/modified-2.7/ssl.py
+++ b/lib-python/modified-2.7/ssl.py
@@ -62,7 +62,6 @@
 from _ssl import OPENSSL_VERSION_NUMBER, OPENSSL_VERSION_INFO, OPENSSL_VERSION
 from _ssl import SSLError
 from _ssl import CERT_NONE, CERT_OPTIONAL, CERT_REQUIRED
-from _ssl import PROTOCOL_SSLv2, PROTOCOL_SSLv3, PROTOCOL_SSLv23, PROTOCOL_TLSv1
 from _ssl import RAND_status, RAND_egd, RAND_add
 from _ssl import \
      SSL_ERROR_ZERO_RETURN, \
@@ -74,6 +73,18 @@
      SSL_ERROR_WANT_CONNECT, \
      SSL_ERROR_EOF, \
      SSL_ERROR_INVALID_ERROR_CODE
+from _ssl import PROTOCOL_SSLv3, PROTOCOL_SSLv23, PROTOCOL_TLSv1
+_PROTOCOL_NAMES = {
+    PROTOCOL_TLSv1: "TLSv1",
+    PROTOCOL_SSLv23: "SSLv23",
+    PROTOCOL_SSLv3: "SSLv3",
+}
+try:
+    from _ssl import PROTOCOL_SSLv2
+except ImportError:
+    pass
+else:
+    _PROTOCOL_NAMES[PROTOCOL_SSLv2] = "SSLv2"
 
 from socket import socket, _fileobject, error as socket_error
 from socket import getnameinfo as _getnameinfo
@@ -400,16 +411,7 @@
     return DER_cert_to_PEM_cert(dercert)
 
 def get_protocol_name(protocol_code):
-    if protocol_code == PROTOCOL_TLSv1:
-        return "TLSv1"
-    elif protocol_code == PROTOCOL_SSLv23:
-        return "SSLv23"
-    elif protocol_code == PROTOCOL_SSLv2:
-        return "SSLv2"
-    elif protocol_code == PROTOCOL_SSLv3:
-        return "SSLv3"
-    else:
-        return "<unknown>"
+    return _PROTOCOL_NAMES.get(protocol_code, '<unknown>')
 
 
 # a replacement for the old socket.ssl function
diff --git a/lib-python/modified-2.7/tarfile.py b/lib-python/modified-2.7/tarfile.py
--- a/lib-python/modified-2.7/tarfile.py
+++ b/lib-python/modified-2.7/tarfile.py
@@ -252,8 +252,8 @@
        the high bit set. So we calculate two checksums, unsigned and
        signed.
     """
-    unsigned_chksum = 256 + sum(struct.unpack("148B8x356B", buf[:512]))
-    signed_chksum = 256 + sum(struct.unpack("148b8x356b", buf[:512]))
+    unsigned_chksum = 256 + sum(struct.unpack("148B", buf[:148]) + struct.unpack("356B", buf[156:512]))
+    signed_chksum = 256 + sum(struct.unpack("148b", buf[:148]) + struct.unpack("356b", buf[156:512]))
     return unsigned_chksum, signed_chksum
 
 def copyfileobj(src, dst, length=None):
@@ -265,6 +265,7 @@
     if length is None:
         shutil.copyfileobj(src, dst)
         return
+
     BUFSIZE = 16 * 1024
     blocks, remainder = divmod(length, BUFSIZE)
     for b in xrange(blocks):
@@ -801,19 +802,19 @@
         if self.closed:
             raise ValueError("I/O operation on closed file")
 
+        buf = ""
         if self.buffer:
             if size is None:
-                buf = self.buffer + self.fileobj.read()
+                buf = self.buffer
                 self.buffer = ""
             else:
                 buf = self.buffer[:size]
                 self.buffer = self.buffer[size:]
-                buf += self.fileobj.read(size - len(buf))
+
+        if size is None:
+            buf += self.fileobj.read()
         else:
-            if size is None:
-                buf = self.fileobj.read()
-            else:
-                buf = self.fileobj.read(size)
+            buf += self.fileobj.read(size - len(buf))
 
         self.position += len(buf)
         return buf
diff --git a/lib-python/modified-2.7/test/test_multiprocessing.py b/lib-python/modified-2.7/test/test_multiprocessing.py
--- a/lib-python/modified-2.7/test/test_multiprocessing.py
+++ b/lib-python/modified-2.7/test/test_multiprocessing.py
@@ -510,7 +510,6 @@
 
         p.join()
 
-    @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
     def test_qsize(self):
         q = self.Queue()
         try:
@@ -532,7 +531,6 @@
             time.sleep(DELTA)
             q.task_done()
 
-    @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
     def test_task_done(self):
         queue = self.JoinableQueue()
 
@@ -1091,7 +1089,6 @@
 class _TestPoolWorkerLifetime(BaseTestCase):
 
     ALLOWED_TYPES = ('processes', )
-    @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
     def test_pool_worker_lifetime(self):
         p = multiprocessing.Pool(3, maxtasksperchild=10)
         self.assertEqual(3, len(p._pool))
@@ -1280,7 +1277,6 @@
         queue = manager.get_queue()
         queue.put('hello world')
 
-    @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
     def test_rapid_restart(self):
         authkey = os.urandom(32)
         manager = QueueManager(
@@ -1297,6 +1293,7 @@
         queue = manager.get_queue()
         self.assertEqual(queue.get(), 'hello world')
         del queue
+        test_support.gc_collect()
         manager.shutdown()
         manager = QueueManager(
             address=addr, authkey=authkey, serializer=SERIALIZER)
@@ -1573,7 +1570,6 @@
 
     ALLOWED_TYPES = ('processes',)
 
-    @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
     def test_heap(self):
         iterations = 5000
         maxblocks = 50
diff --git a/lib-python/modified-2.7/test/test_ssl.py b/lib-python/modified-2.7/test/test_ssl.py
--- a/lib-python/modified-2.7/test/test_ssl.py
+++ b/lib-python/modified-2.7/test/test_ssl.py
@@ -58,32 +58,35 @@
 
 # Issue #9415: Ubuntu hijacks their OpenSSL and forcefully disables SSLv2
 def skip_if_broken_ubuntu_ssl(func):
-    # We need to access the lower-level wrapper in order to create an
-    # implicit SSL context without trying to connect or listen.
-    try:
-        import _ssl
-    except ImportError:
-        # The returned function won't get executed, just ignore the error
-        pass
-    @functools.wraps(func)
-    def f(*args, **kwargs):
+    if hasattr(ssl, 'PROTOCOL_SSLv2'):
+        # We need to access the lower-level wrapper in order to create an
+        # implicit SSL context without trying to connect or listen.
         try:
-            s = socket.socket(socket.AF_INET)
-            _ssl.sslwrap(s._sock, 0, None, None,
-                         ssl.CERT_NONE, ssl.PROTOCOL_SSLv2, None, None)
-        except ssl.SSLError as e:
-            if (ssl.OPENSSL_VERSION_INFO == (0, 9, 8, 15, 15) and
-                platform.linux_distribution() == ('debian', 'squeeze/sid', '')
-                and 'Invalid SSL protocol variant specified' in str(e)):
-                raise unittest.SkipTest("Patched Ubuntu OpenSSL breaks behaviour")
-        return func(*args, **kwargs)
-    return f
+            import _ssl
+        except ImportError:
+            # The returned function won't get executed, just ignore the error
+            pass
+        @functools.wraps(func)
+        def f(*args, **kwargs):
+            try:
+                s = socket.socket(socket.AF_INET)
+                _ssl.sslwrap(s._sock, 0, None, None,
+                             ssl.CERT_NONE, ssl.PROTOCOL_SSLv2, None, None)
+            except ssl.SSLError as e:
+                if (ssl.OPENSSL_VERSION_INFO == (0, 9, 8, 15, 15) and
+                    platform.linux_distribution() == ('debian', 'squeeze/sid', '')
+                    and 'Invalid SSL protocol variant specified' in str(e)):
+                    raise unittest.SkipTest("Patched Ubuntu OpenSSL breaks behaviour")
+            return func(*args, **kwargs)
+        return f
+    else:
+        return func
 
 
 class BasicSocketTests(unittest.TestCase):
 
     def test_constants(self):
-        ssl.PROTOCOL_SSLv2
+        #ssl.PROTOCOL_SSLv2
         ssl.PROTOCOL_SSLv23
         ssl.PROTOCOL_SSLv3
         ssl.PROTOCOL_TLSv1
@@ -966,7 +969,8 @@
             try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, True)
             try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, True, ssl.CERT_OPTIONAL)
             try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, True, ssl.CERT_REQUIRED)
-            try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv2, False)
+            if hasattr(ssl, 'PROTOCOL_SSLv2'):
+                try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv2, False)
             try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv23, False)
             try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_TLSv1, False)
 
@@ -978,7 +982,8 @@
             try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, True)
             try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, True, ssl.CERT_OPTIONAL)
             try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, True, ssl.CERT_REQUIRED)
-            try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv2, False)
+            if hasattr(ssl, 'PROTOCOL_SSLv2'):
+                try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv2, False)
             try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv3, False)
             try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv23, False)
 
diff --git a/lib_pypy/_elementtree.py b/lib_pypy/_elementtree.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_elementtree.py
@@ -0,0 +1,6 @@
+# Just use ElementTree.
+
+from xml.etree import ElementTree
+
+globals().update(ElementTree.__dict__)
+del __all__
diff --git a/lib_pypy/_functools.py b/lib_pypy/_functools.py
--- a/lib_pypy/_functools.py
+++ b/lib_pypy/_functools.py
@@ -14,10 +14,9 @@
             raise TypeError("the first argument must be callable")
         self.func = func
         self.args = args
-        self.keywords = keywords
+        self.keywords = keywords or None
 
     def __call__(self, *fargs, **fkeywords):
-        newkeywords = self.keywords.copy()
-        newkeywords.update(fkeywords)
-        return self.func(*(self.args + fargs), **newkeywords)
-
+        if self.keywords is not None:
+            fkeywords = dict(self.keywords, **fkeywords)
+        return self.func(*(self.args + fargs), **fkeywords)
diff --git a/lib_pypy/_pypy_interact.py b/lib_pypy/_pypy_interact.py
--- a/lib_pypy/_pypy_interact.py
+++ b/lib_pypy/_pypy_interact.py
@@ -56,6 +56,10 @@
                 prompt = getattr(sys, 'ps1', '>>> ')
             try:
                 line = raw_input(prompt)
+                # Can be None if sys.stdin was redefined
+                encoding = getattr(sys.stdin, 'encoding', None)
+                if encoding and not isinstance(line, unicode):
+                    line = line.decode(encoding)
             except EOFError:
                 console.write("\n")
                 break
diff --git a/lib_pypy/greenlet.py b/lib_pypy/greenlet.py
--- a/lib_pypy/greenlet.py
+++ b/lib_pypy/greenlet.py
@@ -48,23 +48,23 @@
     def switch(self, *args):
         "Switch execution to this greenlet, optionally passing the values "
         "given as argument(s).  Returns the value passed when switching back."
-        return self.__switch(_continulet.switch, args)
+        return self.__switch('switch', args)
 
     def throw(self, typ=GreenletExit, val=None, tb=None):
         "raise exception in greenlet, return value passed when switching back"
-        return self.__switch(_continulet.throw, typ, val, tb)
+        return self.__switch('throw', typ, val, tb)
 
-    def __switch(target, unbound_method, *args):
+    def __switch(target, methodname, *args):
         current = getcurrent()
         #
         while not target:
             if not target.__started:
-                if unbound_method != _continulet.throw:
+                if methodname == 'switch':
                     greenlet_func = _greenlet_start
                 else:
                     greenlet_func = _greenlet_throw
                 _continulet.__init__(target, greenlet_func, *args)
-                unbound_method = _continulet.switch
+                methodname = 'switch'
                 args = ()
                 target.__started = True
                 break
@@ -75,22 +75,8 @@
             target = target.parent
         #
         try:
-            if current.__main:
-                if target.__main:
-                    # switch from main to main
-                    if unbound_method == _continulet.throw:
-                        raise args[0], args[1], args[2]
-                    (args,) = args
-                else:
-                    # enter from main to target
-                    args = unbound_method(target, *args)
-            else:
-                if target.__main:
-                    # leave to go to target=main
-                    args = unbound_method(current, *args)
-                else:
-                    # switch from non-main to non-main
-                    args = unbound_method(current, *args, to=target)
+            unbound_method = getattr(_continulet, methodname)
+            args = unbound_method(current, *args, to=target)
         except GreenletExit, e:
             args = (e,)
         finally:
@@ -110,7 +96,16 @@
 
     @property
     def gr_frame(self):
-        raise NotImplementedError("attribute 'gr_frame' of greenlet objects")
+        # xxx this doesn't work when called on either the current or
+        # the main greenlet of another thread
+        if self is getcurrent():
+            return None
+        if self.__main:
+            self = getcurrent()
+        f = _continulet.__reduce__(self)[2][0]
+        if not f:
+            return None
+        return f.f_back.f_back.f_back   # go past start(), __switch(), switch()
 
 # ____________________________________________________________
 # Internal stuff
@@ -138,8 +133,7 @@
     try:
         res = greenlet.run(*args)
     finally:
-        if greenlet.parent is not _tls.main:
-            _continuation.permute(greenlet, greenlet.parent)
+        _continuation.permute(greenlet, greenlet.parent)
     return (res,)
 
 def _greenlet_throw(greenlet, exc, value, tb):
@@ -147,5 +141,4 @@
     try:
         raise exc, value, tb
     finally:
-        if greenlet.parent is not _tls.main:
-            _continuation.permute(greenlet, greenlet.parent)
+        _continuation.permute(greenlet, greenlet.parent)
diff --git a/lib_pypy/pypy_test/test_stackless_pickling.py b/lib_pypy/pypy_test/test_stackless_pickling.py
--- a/lib_pypy/pypy_test/test_stackless_pickling.py
+++ b/lib_pypy/pypy_test/test_stackless_pickling.py
@@ -1,7 +1,3 @@
-"""
-this test should probably not run from CPython or py.py.
-I'm not entirely sure, how to do that.
-"""
 from __future__ import absolute_import
 from py.test import skip
 try:
@@ -16,11 +12,15 @@
 
 class Test_StacklessPickling:
 
+    def test_pickle_main_coroutine(self):
+        import stackless, pickle
+        s = pickle.dumps(stackless.coroutine.getcurrent())
+        print s
+        c = pickle.loads(s)
+        assert c is stackless.coroutine.getcurrent()
+
     def test_basic_tasklet_pickling(self):
-        try:
-            import stackless
-        except ImportError:
-            skip("can't load stackless and don't know why!!!")
+        import stackless
         from stackless import run, schedule, tasklet
         import pickle
 
diff --git a/lib_pypy/pyrepl/completing_reader.py b/lib_pypy/pyrepl/completing_reader.py
--- a/lib_pypy/pyrepl/completing_reader.py
+++ b/lib_pypy/pyrepl/completing_reader.py
@@ -229,7 +229,8 @@
 
     def after_command(self, cmd):
         super(CompletingReader, self).after_command(cmd)
-        if not isinstance(cmd, complete) and not isinstance(cmd, self_insert):
+        if not isinstance(cmd, self.commands['complete']) \
+           and not isinstance(cmd, self.commands['self_insert']):
             self.cmpltn_reset()
 
     def calc_screen(self):
diff --git a/lib_pypy/pyrepl/reader.py b/lib_pypy/pyrepl/reader.py
--- a/lib_pypy/pyrepl/reader.py
+++ b/lib_pypy/pyrepl/reader.py
@@ -576,7 +576,7 @@
         self.console.push_char(char)
         self.handle1(0)
     
-    def readline(self):
+    def readline(self, returns_unicode=False):
         """Read a line.  The implementation of this method also shows
         how to drive Reader if you want more control over the event
         loop."""
@@ -585,6 +585,8 @@
             self.refresh()
             while not self.finished:
                 self.handle1()
+            if returns_unicode:
+                return self.get_unicode()
             return self.get_buffer()
         finally:
             self.restore()
diff --git a/lib_pypy/pyrepl/readline.py b/lib_pypy/pyrepl/readline.py
--- a/lib_pypy/pyrepl/readline.py
+++ b/lib_pypy/pyrepl/readline.py
@@ -198,7 +198,7 @@
         reader.ps1 = prompt
         return reader.readline()
 
-    def multiline_input(self, more_lines, ps1, ps2):
+    def multiline_input(self, more_lines, ps1, ps2, returns_unicode=False):
         """Read an input on possibly multiple lines, asking for more
         lines as long as 'more_lines(unicodetext)' returns an object whose
         boolean value is true.
@@ -209,7 +209,7 @@
             reader.more_lines = more_lines
             reader.ps1 = reader.ps2 = ps1
             reader.ps3 = reader.ps4 = ps2
-            return reader.readline()
+            return reader.readline(returns_unicode=returns_unicode)
         finally:
             reader.more_lines = saved
 
diff --git a/lib_pypy/pyrepl/simple_interact.py b/lib_pypy/pyrepl/simple_interact.py
--- a/lib_pypy/pyrepl/simple_interact.py
+++ b/lib_pypy/pyrepl/simple_interact.py
@@ -54,7 +54,8 @@
             ps1 = getattr(sys, 'ps1', '>>> ')
             ps2 = getattr(sys, 'ps2', '... ')
             try:
-                statement = multiline_input(more_lines, ps1, ps2)
+                statement = multiline_input(more_lines, ps1, ps2,
+                                            returns_unicode=True)
             except EOFError:
                 break
             more = console.push(statement)
diff --git a/lib_pypy/stackless.py b/lib_pypy/stackless.py
--- a/lib_pypy/stackless.py
+++ b/lib_pypy/stackless.py
@@ -4,121 +4,110 @@
 Please refer to their documentation.
 """
 
-DEBUG = True
 
-def dprint(*args):
-    for arg in args:
-        print arg,
-    print
+import _continuation
 
-import traceback
-import sys
+class TaskletExit(Exception):
+    pass
+
+CoroutineExit = TaskletExit
+
+
+def _coroutine_getcurrent():
+    "Returns the current coroutine (i.e. the one which called this function)."
+    try:
+        return _tls.current_coroutine
+    except AttributeError:
+        # first call in this thread: current == main
+        return _coroutine_getmain()
+
+def _coroutine_getmain():
+    try:
+        return _tls.main_coroutine
+    except AttributeError:
+        # create the main coroutine for this thread
+        continulet = _continuation.continulet
+        main = coroutine()
+        main._frame = continulet.__new__(continulet)
+        main._is_started = -1
+        _tls.current_coroutine = _tls.main_coroutine = main
+        return _tls.main_coroutine
+
+
+class coroutine(object):
+    _is_started = 0      # 0=no, 1=yes, -1=main
+
+    def __init__(self):
+        self._frame = None
+
+    def bind(self, func, *argl, **argd):
+        """coro.bind(f, *argl, **argd) -> None.
+           binds function f to coro. f will be called with
+           arguments *argl, **argd
+        """
+        if self.is_alive:
+            raise ValueError("cannot bind a bound coroutine")
+        def run(c):
+            _tls.current_coroutine = self
+            self._is_started = 1
+            return func(*argl, **argd)
+        self._is_started = 0
+        self._frame = _continuation.continulet(run)
+
+    def switch(self):
+        """coro.switch() -> returnvalue
+           switches to coroutine coro. If the bound function
+           f finishes, the returnvalue is that of f, otherwise
+           None is returned
+        """
+        current = _coroutine_getcurrent()
+        try:
+            current._frame.switch(to=self._frame)
+        finally:
+            _tls.current_coroutine = current
+
+    def kill(self):
+        """coro.kill() : kill coroutine coro"""
+        current = _coroutine_getcurrent()
+        try:
+            current._frame.throw(CoroutineExit, to=self._frame)
+        finally:
+            _tls.current_coroutine = current
+
+    @property
+    def is_alive(self):
+        return self._is_started < 0 or (
+            self._frame is not None and self._frame.is_pending())
+
+    @property
+    def is_zombie(self):
+        return self._is_started > 0 and not self._frame.is_pending()
+
+    getcurrent = staticmethod(_coroutine_getcurrent)
+
+    def __reduce__(self):
+        if self._is_started < 0:
+            return _coroutine_getmain, ()
+        else:
+            return type(self), (), self.__dict__
+
+
 try:
-    # If _stackless can be imported then TaskletExit and CoroutineExit are 
-    # automatically added to the builtins.
-    from _stackless import coroutine, greenlet
-except ImportError: # we are running from CPython
-    from greenlet import greenlet, GreenletExit
-    TaskletExit = CoroutineExit = GreenletExit
-    del GreenletExit
-    try:
-        from functools import partial
-    except ImportError: # we are not running python 2.5
-        class partial(object):
-            # just enough of 'partial' to be usefull
-            def __init__(self, func, *argl, **argd):
-                self.func = func
-                self.argl = argl
-                self.argd = argd
+    from thread import _local
+except ImportError:
+    class _local(object):    # assume no threads
+        pass
 
-            def __call__(self):
-                return self.func(*self.argl, **self.argd)
+_tls = _local()
 
-    class GWrap(greenlet):
-        """This is just a wrapper around greenlets to allow
-           to stick additional attributes to a greenlet.
-           To be more concrete, we need a backreference to
-           the coroutine object"""
 
-    class MWrap(object):
-        def __init__(self,something):
-            self.something = something
+# ____________________________________________________________
 
-        def __getattr__(self, attr):
-            return getattr(self.something, attr)
-
-    class coroutine(object):
-        "we can't have greenlet as a base, because greenlets can't be rebound"
-
-        def __init__(self):
-            self._frame = None
-            self.is_zombie = False
-
-        def __getattr__(self, attr):
-            return getattr(self._frame, attr)
-
-        def __del__(self):
-            self.is_zombie = True
-            del self._frame
-            self._frame = None
-
-        def bind(self, func, *argl, **argd):
-            """coro.bind(f, *argl, **argd) -> None.
-               binds function f to coro. f will be called with
-               arguments *argl, **argd
-            """
-            if self._frame is None or self._frame.dead:
-                self._frame = frame = GWrap()
-                frame.coro = self
-            if hasattr(self._frame, 'run') and self._frame.run:
-                raise ValueError("cannot bind a bound coroutine")
-            self._frame.run = partial(func, *argl, **argd)
-
-        def switch(self):
-            """coro.switch() -> returnvalue
-               switches to coroutine coro. If the bound function
-               f finishes, the returnvalue is that of f, otherwise
-               None is returned
-            """
-            try:
-                return greenlet.switch(self._frame)
-            except TypeError, exp: # self._frame is the main coroutine
-                return greenlet.switch(self._frame.something)
-
-        def kill(self):
-            """coro.kill() : kill coroutine coro"""
-            self._frame.throw()
-
-        def _is_alive(self):
-            if self._frame is None:
-                return False
-            return not self._frame.dead
-        is_alive = property(_is_alive)
-        del _is_alive
-
-        def getcurrent():
-            """coroutine.getcurrent() -> the currently running coroutine"""
-            try:
-                return greenlet.getcurrent().coro
-            except AttributeError:
-                return _maincoro
-        getcurrent = staticmethod(getcurrent)
-
-        def __reduce__(self):
-            raise TypeError, 'pickling is not possible based upon greenlets'
-
-    _maincoro = coroutine()
-    maingreenlet = greenlet.getcurrent()
-    _maincoro._frame = frame = MWrap(maingreenlet)
-    frame.coro = _maincoro
-    del frame
-    del maingreenlet
 
 from collections import deque
 
 import operator
-__all__ = 'run getcurrent getmain schedule tasklet channel coroutine \
-                greenlet'.split()
+__all__ = 'run getcurrent getmain schedule tasklet channel coroutine'.split()
 
 _global_task_id = 0
 _squeue = None
@@ -131,7 +120,8 @@
 def _scheduler_remove(value):
     try:
         del _squeue[operator.indexOf(_squeue, value)]
-    except ValueError:pass
+    except ValueError:
+        pass
 
 def _scheduler_append(value, normal=True):
     if normal:
@@ -157,10 +147,7 @@
     _last_task = next
     assert not next.blocked
     if next is not current:
-        try:
-            next.switch()
-        except CoroutineExit:
-            raise TaskletExit
+        next.switch()
     return current
 
 def set_schedule_callback(callback):
@@ -184,34 +171,6 @@
         raise self.type, self.value, self.traceback
 
 #
-# helpers for pickling
-#
-
-_stackless_primitive_registry = {}
-
-def register_stackless_primitive(thang, retval_expr='None'):
-    import types
-    func = thang
-    if isinstance(thang, types.MethodType):
-        func = thang.im_func
-    code = func.func_code
-    _stackless_primitive_registry[code] = retval_expr
-    # It is not too nice to attach info via the code object, but
-    # I can't think of a better solution without a real transform.
-
-def rewrite_stackless_primitive(coro_state, alive, tempval):
-    flags, frame, thunk, parent = coro_state
-    while frame is not None:
-        retval_expr = _stackless_primitive_registry.get(frame.f_code)
-        if retval_expr:
-            # this tasklet needs to stop pickling here and return its value.
-            tempval = eval(retval_expr, globals(), frame.f_locals)
-            coro_state = flags, frame, thunk, parent
-            break
-        frame = frame.f_back
-    return coro_state, alive, tempval
-
-#
 #
 
 class channel(object):
@@ -363,8 +322,6 @@
         """
         return self._channel_action(None, -1)
 
-    register_stackless_primitive(receive, retval_expr='receiver.tempval')
-
     def send_exception(self, exp_type, msg):
         self.send(bomb(exp_type, exp_type(msg)))
 
@@ -381,9 +338,8 @@
         the runnables list.
         """
         return self._channel_action(msg, 1)
-            
-    register_stackless_primitive(send)
-            
+
+
 class tasklet(coroutine):
     """
     A tasklet object represents a tiny task in a Python thread.
@@ -455,6 +411,7 @@
         def _func():
             try:
                 try:
+                    coroutine.switch(back)
                     func(*argl, **argd)
                 except TaskletExit:
                     pass
@@ -464,6 +421,8 @@
 
         self.func = None
         coroutine.bind(self, _func)
+        back = _coroutine_getcurrent()
+        coroutine.switch(self)
         self.alive = True
         _scheduler_append(self)
         return self
@@ -486,39 +445,6 @@
             raise RuntimeError, "The current tasklet cannot be removed."
             # not sure if I will revive this  " Use t=tasklet().capture()"
         _scheduler_remove(self)
-        
-    def __reduce__(self):
-        one, two, coro_state = coroutine.__reduce__(self)
-        assert one is coroutine
-        assert two == ()
-        # we want to get rid of the parent thing.
-        # for now, we just drop it
-        a, frame, c, d = coro_state
-
-        # Removing all frames related to stackless.py.
-        # They point to stuff we don't want to be pickled.
-
-        pickleframe = frame
-        while frame is not None:
-            if frame.f_code == schedule.func_code:
-                # Removing everything including and after the
-                # call to stackless.schedule()
-                pickleframe = frame.f_back
-                break
-            frame = frame.f_back
-        if d:
-            assert isinstance(d, coroutine)
-        coro_state = a, pickleframe, c, None
-        coro_state, alive, tempval = rewrite_stackless_primitive(coro_state, self.alive, self.tempval)
-        inst_dict = self.__dict__.copy()
-        inst_dict.pop('tempval', None)
-        return self.__class__, (), (coro_state, alive, tempval, inst_dict)
-
-    def __setstate__(self, (coro_state, alive, tempval, inst_dict)):
-        coroutine.__setstate__(self, coro_state)
-        self.__dict__.update(inst_dict)
-        self.alive = alive
-        self.tempval = tempval
 
 def getmain():
     """
@@ -607,30 +533,7 @@
     global _last_task
     _global_task_id = 0
     _main_tasklet = coroutine.getcurrent()
-    try:
-        _main_tasklet.__class__ = tasklet
-    except TypeError: # we are running pypy-c
-        class TaskletProxy(object):
-            """TaskletProxy is needed to give the _main_coroutine tasklet behaviour"""
-            def __init__(self, coro):
-                self._coro = coro
-
-            def __getattr__(self,attr):
-                return getattr(self._coro,attr)
-
-            def __str__(self):
-                return '<tasklet %s a:%s>' % (self._task_id, self.is_alive)
-
-            def __reduce__(self):
-                return getmain, ()
-
-            __repr__ = __str__
-
-
-        global _main_coroutine
-        _main_coroutine = _main_tasklet
-        _main_tasklet = TaskletProxy(_main_tasklet)
-        assert _main_tasklet.is_alive and not _main_tasklet.is_zombie
+    _main_tasklet.__class__ = tasklet         # XXX HAAAAAAAAAAAAAAAAAAAAACK
     _last_task = _main_tasklet
     tasklet._init.im_func(_main_tasklet, label='main')
     _squeue = deque()
diff --git a/py/_code/source.py b/py/_code/source.py
--- a/py/_code/source.py
+++ b/py/_code/source.py
@@ -139,7 +139,7 @@
                 trysource = self[start:end]
                 if trysource.isparseable():
                     return start, end
-        return start, end
+        return start, len(self)
 
     def getblockend(self, lineno):
         # XXX
diff --git a/pypy/annotation/annrpython.py b/pypy/annotation/annrpython.py
--- a/pypy/annotation/annrpython.py
+++ b/pypy/annotation/annrpython.py
@@ -149,7 +149,7 @@
         desc = olddesc.bind_self(classdef)
         args = self.bookkeeper.build_args("simple_call", args_s[:])
         desc.consider_call_site(self.bookkeeper, desc.getcallfamily(), [desc],
-            args, annmodel.s_ImpossibleValue)
+            args, annmodel.s_ImpossibleValue, None)
         result = []
         def schedule(graph, inputcells):
             result.append((graph, inputcells))
diff --git a/pypy/annotation/bookkeeper.py b/pypy/annotation/bookkeeper.py
--- a/pypy/annotation/bookkeeper.py
+++ b/pypy/annotation/bookkeeper.py
@@ -209,8 +209,8 @@
                 self.consider_call_site(call_op)
 
             for pbc, args_s in self.emulated_pbc_calls.itervalues():
-                self.consider_call_site_for_pbc(pbc, 'simple_call', 
-                                                args_s, s_ImpossibleValue)
+                self.consider_call_site_for_pbc(pbc, 'simple_call',
+                                                args_s, s_ImpossibleValue, None)
             self.emulated_pbc_calls = {}
         finally:
             self.leave()
@@ -257,18 +257,18 @@
             args_s = [lltype_to_annotation(adtmeth.ll_ptrtype)] + args_s
         if isinstance(s_callable, SomePBC):
             s_result = binding(call_op.result, s_ImpossibleValue)
-            self.consider_call_site_for_pbc(s_callable,
-                                            call_op.opname,
-                                            args_s, s_result)
+            self.consider_call_site_for_pbc(s_callable, call_op.opname, args_s,
+                                            s_result, call_op)
 
-    def consider_call_site_for_pbc(self, s_callable, opname, args_s, s_result):
+    def consider_call_site_for_pbc(self, s_callable, opname, args_s, s_result,
+                                   call_op):
         descs = list(s_callable.descriptions)
         if not descs:
             return
         family = descs[0].getcallfamily()
         args = self.build_args(opname, args_s)
         s_callable.getKind().consider_call_site(self, family, descs, args,
-                                                s_result)
+                                                s_result, call_op)
 
     def getuniqueclassdef(self, cls):
         """Get the ClassDef associated with the given user cls.
@@ -656,6 +656,7 @@
                 whence = None
             else:
                 whence = emulated # callback case
+            op = None
             s_previous_result = s_ImpossibleValue
 
         def schedule(graph, inputcells):
@@ -663,7 +664,7 @@
 
         results = []
         for desc in descs:
-            results.append(desc.pycall(schedule, args, s_previous_result))
+            results.append(desc.pycall(schedule, args, s_previous_result, op))
         s_result = unionof(*results)
         return s_result
 
diff --git a/pypy/annotation/description.py b/pypy/annotation/description.py
--- a/pypy/annotation/description.py
+++ b/pypy/annotation/description.py
@@ -255,7 +255,11 @@
             raise TypeError, "signature mismatch: %s" % e.getmsg(self.name)
         return inputcells
 
-    def specialize(self, inputcells):
+    def specialize(self, inputcells, op=None):
+        if (op is None and
+            getattr(self.bookkeeper, "position_key", None) is not None):
+            _, block, i = self.bookkeeper.position_key
+            op = block.operations[i]
         if self.specializer is None:
             # get the specializer based on the tag of the 'pyobj'
             # (if any), according to the current policy
@@ -269,11 +273,14 @@
                 enforceargs = Sig(*enforceargs)
                 self.pyobj._annenforceargs_ = enforceargs
             enforceargs(self, inputcells) # can modify inputcells in-place
-        return self.specializer(self, inputcells)
+        if getattr(self.pyobj, '_annspecialcase_', '').endswith("call_location"):
+            return self.specializer(self, inputcells, op)
+        else:
+            return self.specializer(self, inputcells)
 
-    def pycall(self, schedule, args, s_previous_result):
+    def pycall(self, schedule, args, s_previous_result, op=None):
         inputcells = self.parse_arguments(args)
-        result = self.specialize(inputcells)
+        result = self.specialize(inputcells, op)
         if isinstance(result, FunctionGraph):
             graph = result         # common case
             # if that graph has a different signature, we need to re-parse
@@ -296,17 +303,17 @@
                                              None,       # selfclassdef
                                              name)
 
-    def consider_call_site(bookkeeper, family, descs, args, s_result):
+    def consider_call_site(bookkeeper, family, descs, args, s_result, op):
         shape = rawshape(args)
-        row = FunctionDesc.row_to_consider(descs, args)
+        row = FunctionDesc.row_to_consider(descs, args, op)
         family.calltable_add_row(shape, row)
     consider_call_site = staticmethod(consider_call_site)
 
-    def variant_for_call_site(bookkeeper, family, descs, args):
+    def variant_for_call_site(bookkeeper, family, descs, args, op):
         shape = rawshape(args)
         bookkeeper.enter(None)
         try:
-            row = FunctionDesc.row_to_consider(descs, args)
+            row = FunctionDesc.row_to_consider(descs, args, op)
         finally:
             bookkeeper.leave()
         index = family.calltable_lookup_row(shape, row)
@@ -316,7 +323,7 @@
     def rowkey(self):
         return self
 
-    def row_to_consider(descs, args):
+    def row_to_consider(descs, args, op):
         # see comments in CallFamily
         from pypy.annotation.model import s_ImpossibleValue
         row = {}
@@ -324,7 +331,7 @@
             def enlist(graph, ignore):
                 row[desc.rowkey()] = graph
                 return s_ImpossibleValue   # meaningless
-            desc.pycall(enlist, args, s_ImpossibleValue)
+            desc.pycall(enlist, args, s_ImpossibleValue, op)
         return row
     row_to_consider = staticmethod(row_to_consider)
 
@@ -521,7 +528,7 @@
                             "specialization" % (self.name,))
         return self.getclassdef(None)
 
-    def pycall(self, schedule, args, s_previous_result):
+    def pycall(self, schedule, args, s_previous_result, op=None):
         from pypy.annotation.model import SomeInstance, SomeImpossibleValue
         if self.specialize:
             if self.specialize == 'specialize:ctr_location':
@@ -664,7 +671,7 @@
             cdesc = cdesc.basedesc
         return s_result     # common case
 
-    def consider_call_site(bookkeeper, family, descs, args, s_result):
+    def consider_call_site(bookkeeper, family, descs, args, s_result, op):
         from pypy.annotation.model import SomeInstance, SomePBC, s_None
         if len(descs) == 1:
             # call to a single class, look at the result annotation
@@ -709,7 +716,7 @@
             initdescs[0].mergecallfamilies(*initdescs[1:])
             initfamily = initdescs[0].getcallfamily()
             MethodDesc.consider_call_site(bookkeeper, initfamily, initdescs,
-                                          args, s_None)
+                                          args, s_None, op)
     consider_call_site = staticmethod(consider_call_site)
 
     def getallbases(self):
@@ -782,13 +789,13 @@
     def getuniquegraph(self):
         return self.funcdesc.getuniquegraph()
 
-    def pycall(self, schedule, args, s_previous_result):
+    def pycall(self, schedule, args, s_previous_result, op=None):
         from pypy.annotation.model import SomeInstance
         if self.selfclassdef is None:
             raise Exception("calling %r" % (self,))
         s_instance = SomeInstance(self.selfclassdef, flags = self.flags)
         args = args.prepend(s_instance)
-        return self.funcdesc.pycall(schedule, args, s_previous_result)
+        return self.funcdesc.pycall(schedule, args, s_previous_result, op)
 
     def bind_under(self, classdef, name):
         self.bookkeeper.warning("rebinding an already bound %r" % (self,))
@@ -801,10 +808,10 @@
                                              self.name,
                                              flags)
 
-    def consider_call_site(bookkeeper, family, descs, args, s_result):
+    def consider_call_site(bookkeeper, family, descs, args, s_result, op):
         shape = rawshape(args, nextra=1)     # account for the extra 'self'
         funcdescs = [methoddesc.funcdesc for methoddesc in descs]
-        row = FunctionDesc.row_to_consider(descs, args)
+        row = FunctionDesc.row_to_consider(descs, args, op)
         family.calltable_add_row(shape, row)
     consider_call_site = staticmethod(consider_call_site)
 
@@ -956,16 +963,16 @@
         return '<MethodOfFrozenDesc %r of %r>' % (self.funcdesc,
                                                   self.frozendesc)
 
-    def pycall(self, schedule, args, s_previous_result):
+    def pycall(self, schedule, args, s_previous_result, op=None):
         from pypy.annotation.model import SomePBC
         s_self = SomePBC([self.frozendesc])
         args = args.prepend(s_self)
-        return self.funcdesc.pycall(schedule, args, s_previous_result)
+        return self.funcdesc.pycall(schedule, args, s_previous_result, op)
 
-    def consider_call_site(bookkeeper, family, descs, args, s_result):
+    def consider_call_site(bookkeeper, family, descs, args, s_result, op):
         shape = rawshape(args, nextra=1)    # account for the extra 'self'
         funcdescs = [mofdesc.funcdesc for mofdesc in descs]
-        row = FunctionDesc.row_to_consider(descs, args)
+        row = FunctionDesc.row_to_consider(descs, args, op)
         family.calltable_add_row(shape, row)
     consider_call_site = staticmethod(consider_call_site)
 
diff --git a/pypy/annotation/policy.py b/pypy/annotation/policy.py
--- a/pypy/annotation/policy.py
+++ b/pypy/annotation/policy.py
@@ -1,7 +1,7 @@
 # base annotation policy for specialization
 from pypy.annotation.specialize import default_specialize as default
-from pypy.annotation.specialize import specialize_argvalue, specialize_argtype, specialize_arglistitemtype
-from pypy.annotation.specialize import memo
+from pypy.annotation.specialize import specialize_argvalue, specialize_argtype, specialize_arglistitemtype, specialize_arg_or_var
+from pypy.annotation.specialize import memo, specialize_call_location
 # for some reason, model must be imported first,
 # or we create a cycle.
 from pypy.annotation import model as annmodel
@@ -73,8 +73,10 @@
     default_specialize = staticmethod(default)
     specialize__memo = staticmethod(memo)
     specialize__arg = staticmethod(specialize_argvalue) # specialize:arg(N)
+    specialize__arg_or_var = staticmethod(specialize_arg_or_var)
     specialize__argtype = staticmethod(specialize_argtype) # specialize:argtype(N)
     specialize__arglistitemtype = staticmethod(specialize_arglistitemtype)
+    specialize__call_location = staticmethod(specialize_call_location)
 
     def specialize__ll(pol, *args):
         from pypy.rpython.annlowlevel import LowLevelAnnotatorPolicy
diff --git a/pypy/annotation/specialize.py b/pypy/annotation/specialize.py
--- a/pypy/annotation/specialize.py
+++ b/pypy/annotation/specialize.py
@@ -353,6 +353,16 @@
     key = tuple(key)
     return maybe_star_args(funcdesc, key, args_s)
 
+def specialize_arg_or_var(funcdesc, args_s, *argindices):
+    for argno in argindices:
+        if not args_s[argno].is_constant():
+            break
+    else:
+        # all constant
+        return specialize_argvalue(funcdesc, args_s, *argindices)
+    # some not constant
+    return maybe_star_args(funcdesc, None, args_s)
+
 def specialize_argtype(funcdesc, args_s, *argindices):
     key = tuple([args_s[i].knowntype for i in argindices])
     for cls in key:
@@ -370,3 +380,7 @@
     else:
         key = s.listdef.listitem.s_value.knowntype
     return maybe_star_args(funcdesc, key, args_s)
+
+def specialize_call_location(funcdesc, args_s, op):
+    assert op is not None
+    return maybe_star_args(funcdesc, op, args_s)
diff --git a/pypy/annotation/test/test_annrpython.py b/pypy/annotation/test/test_annrpython.py
--- a/pypy/annotation/test/test_annrpython.py
+++ b/pypy/annotation/test/test_annrpython.py
@@ -1099,8 +1099,8 @@
         allocdesc = a.bookkeeper.getdesc(alloc)
         s_C1 = a.bookkeeper.immutablevalue(C1)
         s_C2 = a.bookkeeper.immutablevalue(C2)
-        graph1 = allocdesc.specialize([s_C1])
-        graph2 = allocdesc.specialize([s_C2])
+        graph1 = allocdesc.specialize([s_C1], None)
+        graph2 = allocdesc.specialize([s_C2], None)
         assert a.binding(graph1.getreturnvar()).classdef == C1df
         assert a.binding(graph2.getreturnvar()).classdef == C2df
         assert graph1 in a.translator.graphs
@@ -1135,8 +1135,8 @@
         allocdesc = a.bookkeeper.getdesc(alloc)
         s_C1 = a.bookkeeper.immutablevalue(C1)
         s_C2 = a.bookkeeper.immutablevalue(C2)
-        graph1 = allocdesc.specialize([s_C1, s_C2])
-        graph2 = allocdesc.specialize([s_C2, s_C2])
+        graph1 = allocdesc.specialize([s_C1, s_C2], None)
+        graph2 = allocdesc.specialize([s_C2, s_C2], None)
         assert a.binding(graph1.getreturnvar()).classdef == C1df
         assert a.binding(graph2.getreturnvar()).classdef == C2df
         assert graph1 in a.translator.graphs
@@ -1194,6 +1194,33 @@
         assert len(executedesc._cache[(0, 'star', 2)].startblock.inputargs) == 4
         assert len(executedesc._cache[(1, 'star', 3)].startblock.inputargs) == 5
 
+    def test_specialize_arg_or_var(self):
+        def f(a):
+            return 1
+        f._annspecialcase_ = 'specialize:arg_or_var(0)'
+
+        def fn(a):
+            return f(3) + f(a)
+
+        a = self.RPythonAnnotator()
+        a.build_types(fn, [int])
+        executedesc = a.bookkeeper.getdesc(f)
+        assert sorted(executedesc._cache.keys()) == [None, (3,)]
+        # we got two different special
+
+    def test_specialize_call_location(self):
+        def g(a):
+            return a
+        g._annspecialcase_ = "specialize:call_location"
+        def f(x):
+            return g(x)
+        f._annspecialcase_ = "specialize:argtype(0)"
+        def h(y):
+            w = f(y)
+            return int(f(str(y))) + w
+        a = self.RPythonAnnotator()
+        assert a.build_types(h, [int]) == annmodel.SomeInteger()
+
     def test_assert_list_doesnt_lose_info(self):
         class T(object):
             pass
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -27,7 +27,7 @@
 # --allworkingmodules
 working_modules = default_modules.copy()
 working_modules.update(dict.fromkeys(
-    ["_socket", "unicodedata", "mmap", "fcntl", "_locale",
+    ["_socket", "unicodedata", "mmap", "fcntl", "_locale", "pwd",
      "rctime" , "select", "zipimport", "_lsprof",
      "crypt", "signal", "_rawffi", "termios", "zlib", "bz2",
      "struct", "_hashlib", "_md5", "_sha", "_minimal_curses", "cStringIO",
@@ -58,6 +58,7 @@
     # unix only modules
     del working_modules["crypt"]
     del working_modules["fcntl"]
+    del working_modules["pwd"]
     del working_modules["termios"]
     del working_modules["_minimal_curses"]
 
diff --git a/pypy/doc/config/objspace.usemodules.pwd.txt b/pypy/doc/config/objspace.usemodules.pwd.txt
new file mode 100644
--- /dev/null
+++ b/pypy/doc/config/objspace.usemodules.pwd.txt
@@ -0,0 +1,2 @@
+Use the 'pwd' module. 
+This module is expected to be fully working.
diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst
--- a/pypy/doc/index.rst
+++ b/pypy/doc/index.rst
@@ -21,8 +21,6 @@
 
 * `Papers`_: Academic papers, talks, and related projects
 
-* `Videos`_: Videos of PyPy talks and presentations
-
 * `speed.pypy.org`_: Daily benchmarks of how fast PyPy is
 
 * `potential project ideas`_: In case you want to get your feet wet...
diff --git a/pypy/doc/stackless.rst b/pypy/doc/stackless.rst
--- a/pypy/doc/stackless.rst
+++ b/pypy/doc/stackless.rst
@@ -66,7 +66,7 @@
 In practice, in PyPy, you cannot change the ``f_back`` of an abitrary
 frame, but only of frames stored in ``continulets``.
 
-Continulets are internally implemented using stacklets.  Stacklets are a
+Continulets are internally implemented using stacklets_.  Stacklets are a
 bit more primitive (they are really one-shot continuations), but that
 idea only works in C, not in Python.  The basic idea of continulets is
 to have at any point in time a complete valid stack; this is important
@@ -215,11 +215,6 @@
 
 * Support for other CPUs than x86 and x86-64
 
-* The app-level ``f_back`` field of frames crossing continulet boundaries
-  is None for now, unlike what I explain in the theoretical overview
-  above.  It mostly means that in a ``pdb.set_trace()`` you cannot go
-  ``up`` past countinulet boundaries.  This could be fixed.
-
 .. __: `recursion depth limit`_
 
 (*) Pickling, as well as changing threads, could be implemented by using
@@ -285,6 +280,24 @@
 to use other interfaces like genlets and greenlets.)
 
 
+Stacklets
++++++++++
+
+Continulets are internally implemented using stacklets, which is the
+generic RPython-level building block for "one-shot continuations".  For
+more information about them please see the documentation in the C source
+at `pypy/translator/c/src/stacklet/stacklet.h`_.
+
+The module ``pypy.rlib.rstacklet`` is a thin wrapper around the above
+functions.  The key point is that new() and switch() always return a
+fresh stacklet handle (or an empty one), and switch() additionally
+consumes one.  It makes no sense to have code in which the returned
+handle is ignored, or used more than once.  Note that ``stacklet.c`` is
+written assuming that the user knows that, and so no additional checking
+occurs; this can easily lead to obscure crashes if you don't use a
+wrapper like PyPy's '_continuation' module.
+
+
 Theory of composability
 +++++++++++++++++++++++
 
diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -125,6 +125,7 @@
 
     ###  Manipulation  ###
 
+    @jit.look_inside_iff(lambda self: not self._dont_jit)
     def unpack(self): # slowish
         "Return a ([w1,w2...], {'kw':w3...}) pair."
         kwds_w = {}
@@ -245,6 +246,8 @@
 
     ###  Parsing for function calls  ###
 
+    # XXX: this should be @jit.look_inside_iff, but we need key word arguments,
+    # and it doesn't support them for now.
     def _match_signature(self, w_firstarg, scope_w, signature, defaults_w=None,
                          blindargs=0):
         """Parse args and kwargs according to the signature of a code object,
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -3,18 +3,18 @@
 from pypy.interpreter.executioncontext import ExecutionContext, ActionFlag
 from pypy.interpreter.executioncontext import UserDelAction, FrameTraceAction
 from pypy.interpreter.error import OperationError, operationerrfmt
-from pypy.interpreter.error import new_exception_class
+from pypy.interpreter.error import new_exception_class, typed_unwrap_error_msg
 from pypy.interpreter.argument import Arguments
 from pypy.interpreter.miscutils import ThreadLocals
 from pypy.tool.cache import Cache
 from pypy.tool.uid import HUGEVAL_BYTES
-from pypy.rlib.objectmodel import we_are_translated
+from pypy.rlib.objectmodel import we_are_translated, newlist, compute_unique_id
 from pypy.rlib.debug import make_sure_not_resized
 from pypy.rlib.timer import DummyTimer, Timer
 from pypy.rlib.rarithmetic import r_uint
 from pypy.rlib import jit
 from pypy.tool.sourcetools import func_with_new_name
-import os, sys, py
+import os, sys
 
 __all__ = ['ObjSpace', 'OperationError', 'Wrappable', 'W_Root']
 
@@ -186,6 +186,28 @@
     def _set_mapdict_storage_and_map(self, storage, map):
         raise NotImplementedError
 
+    # -------------------------------------------------------------------
+
+    def str_w(self, space):
+        w_msg = typed_unwrap_error_msg(space, "string", self)
+        raise OperationError(space.w_TypeError, w_msg)
+
+    def unicode_w(self, space):
+        raise OperationError(space.w_TypeError,
+                             typed_unwrap_error_msg(space, "unicode", self))
+
+    def int_w(self, space):
+        raise OperationError(space.w_TypeError,
+                             typed_unwrap_error_msg(space, "integer", self))
+    
+    def uint_w(self, space):
+        raise OperationError(space.w_TypeError,
+                             typed_unwrap_error_msg(space, "integer", self))
+    
+    def bigint_w(self, space):
+        raise OperationError(space.w_TypeError,
+                             typed_unwrap_error_msg(space, "integer", self))
+
 
 class Wrappable(W_Root):
     """A subclass of Wrappable is an internal, interpreter-level class
@@ -757,7 +779,18 @@
         w_iterator = self.iter(w_iterable)
         # If we know the expected length we can preallocate.
         if expected_length == -1:
-            items = []
+            try:
+                lgt_estimate = self.len_w(w_iterable)
+            except OperationError, o:
+                if (not o.match(self, self.w_AttributeError) and
+                    not o.match(self, self.w_TypeError)):
+                    raise
+                items = []
+            else:
+                try:
+                    items = newlist(lgt_estimate)
+                except MemoryError:
+                    items = [] # it might have lied
         else:
             items = [None] * expected_length
         idx = 0
@@ -890,7 +923,7 @@
         ec.c_call_trace(frame, w_func, args)
         try:
             w_res = self.call_args(w_func, args)
-        except OperationError, e:
+        except OperationError:
             ec.c_exception_trace(frame, w_func)
             raise
         ec.c_return_trace(frame, w_func, args)
@@ -936,6 +969,9 @@
     def isinstance_w(self, w_obj, w_type):
         return self.is_true(self.isinstance(w_obj, w_type))
 
+    def id(self, w_obj):
+        return self.wrap(compute_unique_id(w_obj))
+
     # The code below only works
     # for the simple case (new-style instance).
     # These methods are patched with the full logic by the __builtin__
@@ -988,8 +1024,6 @@
 
     def eval(self, expression, w_globals, w_locals, hidden_applevel=False):
         "NOT_RPYTHON: For internal debugging."
-        import types
-        from pypy.interpreter.pycode import PyCode
         if isinstance(expression, str):
             compiler = self.createcompiler()
             expression = compiler.compile(expression, '?', 'eval', 0,
@@ -1001,7 +1035,6 @@
     def exec_(self, statement, w_globals, w_locals, hidden_applevel=False,
               filename=None):
         "NOT_RPYTHON: For internal debugging."
-        import types
         if filename is None:
             filename = '?'
         from pypy.interpreter.pycode import PyCode
@@ -1199,6 +1232,18 @@
             return None
         return self.str_w(w_obj)
 
+    def str_w(self, w_obj):
+        return w_obj.str_w(self)
+
+    def int_w(self, w_obj):
+        return w_obj.int_w(self)
+
+    def uint_w(self, w_obj):
+        return w_obj.uint_w(self)
+
+    def bigint_w(self, w_obj):
+        return w_obj.bigint_w(self)
+
     def realstr_w(self, w_obj):
         # Like str_w, but only works if w_obj is really of type 'str'.
         if not self.is_true(self.isinstance(w_obj, self.w_str)):
@@ -1206,6 +1251,9 @@
                                  self.wrap('argument must be a string'))
         return self.str_w(w_obj)
 
+    def unicode_w(self, w_obj):
+        return w_obj.unicode_w(self)
+
     def realunicode_w(self, w_obj):
         # Like unicode_w, but only works if w_obj is really of type
         # 'unicode'.
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -458,3 +458,7 @@
     if module:
         space.setattr(w_exc, space.wrap("__module__"), space.wrap(module))
     return w_exc
+
+def typed_unwrap_error_msg(space, expected, w_obj):
+    type_name = space.type(w_obj).getname(space)
+    return space.wrap("expected %s, got %s object" % (expected, type_name))
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -1,5 +1,4 @@
 import sys
-from pypy.interpreter.miscutils import Stack
 from pypy.interpreter.error import OperationError
 from pypy.rlib.rarithmetic import LONG_BIT
 from pypy.rlib.unroll import unrolling_iterable
@@ -48,6 +47,7 @@
         return frame
 
     @staticmethod
+    @jit.unroll_safe  # should usually loop 0 times, very rarely more than once
     def getnextframe_nohidden(frame):
         frame = frame.f_backref()
         while frame and frame.hide():
@@ -81,58 +81,6 @@
 
     # ________________________________________________________________
 
-
-    class Subcontext(object):
-        # coroutine: subcontext support
-
-        def __init__(self):
-            self.topframe = None
-            self.w_tracefunc = None
-            self.profilefunc = None
-            self.w_profilefuncarg = None
-            self.is_tracing = 0
-
-        def enter(self, ec):
-            ec.topframeref = jit.non_virtual_ref(self.topframe)
-            ec.w_tracefunc = self.w_tracefunc
-            ec.profilefunc = self.profilefunc
-            ec.w_profilefuncarg = self.w_profilefuncarg
-            ec.is_tracing = self.is_tracing
-            ec.space.frame_trace_action.fire()
-
-        def leave(self, ec):
-            self.topframe = ec.gettopframe()
-            self.w_tracefunc = ec.w_tracefunc
-            self.profilefunc = ec.profilefunc
-            self.w_profilefuncarg = ec.w_profilefuncarg
-            self.is_tracing = ec.is_tracing
-
-        def clear_framestack(self):
-            self.topframe = None
-
-        # the following interface is for pickling and unpickling
-        def getstate(self, space):
-            if self.topframe is None:
-                return space.w_None
-            return self.topframe
-
-        def setstate(self, space, w_state):
-            from pypy.interpreter.pyframe import PyFrame
-            if space.is_w(w_state, space.w_None):
-                self.topframe = None
-            else:
-                self.topframe = space.interp_w(PyFrame, w_state)
-
-        def getframestack(self):
-            lst = []
-            f = self.topframe
-            while f is not None:
-                lst.append(f)
-                f = f.f_backref()
-            lst.reverse()
-            return lst
-        # coroutine: I think this is all, folks!
-
     def c_call_trace(self, frame, w_func, args=None):
         "Profile the call of a builtin function"
         self._c_call_return_trace(frame, w_func, args, 'c_call')
diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -242,8 +242,10 @@
             # we have been seen by other means so rtyping should not choke
             # on us
             identifier = self.code.identifier
-            assert Function._all.get(identifier, self) is self, ("duplicate "
-                                                                 "function ids")
+            previous = Function._all.get(identifier, self)
+            assert previous is self, (
+                "duplicate function ids with identifier=%r: %r and %r" % (
+                identifier, previous, self))
             self.add_to_table()
         return False
 
diff --git a/pypy/interpreter/miscutils.py b/pypy/interpreter/miscutils.py
--- a/pypy/interpreter/miscutils.py
+++ b/pypy/interpreter/miscutils.py
@@ -2,154 +2,6 @@
 Miscellaneous utilities.
 """
 
-import types
-
-from pypy.rlib.rarithmetic import r_uint
-
-class RootStack:
-    pass
-
-class Stack(RootStack):
-    """Utility class implementing a stack."""
-
-    _annspecialcase_ = "specialize:ctr_location" # polymorphic
-
-    def __init__(self):
-        self.items = []
-
-    def clone(self):
-        s = self.__class__()
-        for item in self.items:
-            try:
-                item = item.clone()
-            except AttributeError:
-                pass
-            s.push(item)
-        return s
-
-    def push(self, item):
-        self.items.append(item)
-
-    def pop(self):
-        return self.items.pop()
-
-    def drop(self, n):
-        if n > 0:
-            del self.items[-n:]
-
-    def top(self, position=0):
-        """'position' is 0 for the top of the stack, 1 for the item below,
-        and so on.  It must not be negative."""
-        if position < 0:
-            raise ValueError, 'negative stack position'
-        if position >= len(self.items):
-            raise IndexError, 'not enough entries in stack'
-        return self.items[~position]
-
-    def set_top(self, value, position=0):
-        """'position' is 0 for the top of the stack, 1 for the item below,
-        and so on.  It must not be negative."""
-        if position < 0:
-            raise ValueError, 'negative stack position'
-        if position >= len(self.items):
-            raise IndexError, 'not enough entries in stack'
-        self.items[~position] = value
-
-    def depth(self):
-        return len(self.items)
-
-    def empty(self):
-        return len(self.items) == 0
-
-
-class FixedStack(RootStack):
-    _annspecialcase_ = "specialize:ctr_location" # polymorphic
-
-    # unfortunately, we have to re-do everything
-    def __init__(self):
-        pass
-
-    def setup(self, stacksize):
-        self.ptr = r_uint(0) # we point after the last element
-        self.items = [None] * stacksize
-
-    def clone(self):
-        # this is only needed if we support flow space
-        s = self.__class__()
-        s.setup(len(self.items))
-        for item in self.items[:self.ptr]:
-            try:
-                item = item.clone()
-            except AttributeError:
-                pass
-            s.push(item)
-        return s
-
-    def push(self, item):
-        ptr = self.ptr
-        self.items[ptr] = item
-        self.ptr = ptr + 1
-
-    def pop(self):
-        ptr = self.ptr - 1
-        ret = self.items[ptr]   # you get OverflowError if the stack is empty
-        self.items[ptr] = None
-        self.ptr = ptr
-        return ret
-
-    def drop(self, n):
-        while n > 0:
-            n -= 1
-            self.ptr -= 1
-            self.items[self.ptr] = None
-
-    def top(self, position=0):
-        # for a fixed stack, we assume correct indices
-        return self.items[self.ptr + ~position]
-
-    def set_top(self, value, position=0):
-        # for a fixed stack, we assume correct indices
-        self.items[self.ptr + ~position] = value
-
-    def depth(self):
-        return self.ptr
-
-    def empty(self):
-        return not self.ptr
-
-
-class InitializedClass(type):
-    """NOT_RPYTHON.  A meta-class that allows a class to initialize itself (or
-    its subclasses) by calling __initclass__() as a class method."""
-    def __init__(self, name, bases, dict):
-        super(InitializedClass, self).__init__(name, bases, dict)
-        for basecls in self.__mro__:
-            raw = basecls.__dict__.get('__initclass__')
-            if isinstance(raw, types.FunctionType):
-                raw(self)   # call it as a class method
-
-
-class RwDictProxy(object):
-    """NOT_RPYTHON.  A dict-like class standing for 'cls.__dict__', to work
-    around the fact that the latter is a read-only proxy for new-style
-    classes."""
-    
-    def __init__(self, cls):
-        self.cls = cls
-
-    def __getitem__(self, attr):
-        return self.cls.__dict__[attr]
-
-    def __setitem__(self, attr, value):
-        setattr(self.cls, attr, value)
-
-    def __contains__(self, value):
-        return value in self.cls.__dict__
-
-    def items(self):
-        return self.cls.__dict__.items()
-
-
 class ThreadLocals:
     """Pseudo thread-local storage, for 'space.threadlocals'.
     This is not really thread-local at all; the intention is that the PyPy
@@ -167,3 +19,7 @@
 
     def getmainthreadvalue(self):
         return self._value
+
+    def getallvalues(self):
+        return {0: self._value}
+
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -10,7 +10,7 @@
 from pypy.interpreter.argument import Signature
 from pypy.interpreter.error import OperationError
 from pypy.interpreter.gateway import NoneNotWrapped, unwrap_spec
-from pypy.interpreter.astcompiler.consts import (CO_OPTIMIZED,
+from pypy.interpreter.astcompiler.consts import (
     CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, CO_VARKEYWORDS, CO_NESTED,
     CO_GENERATOR, CO_CONTAINSGLOBALS)
 from pypy.rlib.rarithmetic import intmask
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -614,7 +614,8 @@
         return self.get_builtin().getdict(space)
 
     def fget_f_back(self, space):
-        return self.space.wrap(self.f_backref())
+        f_back = ExecutionContext.getnextframe_nohidden(self)
+        return self.space.wrap(f_back)
 
     def fget_f_lasti(self, space):
         return self.space.wrap(self.last_instr)
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1523,10 +1523,8 @@
 
         if not isinstance(prog, codetype):
             filename = '<string>'
-            if not isinstance(prog, str):
-                if isinstance(prog, basestring):
-                    prog = str(prog)
-                elif isinstance(prog, file):
+            if not isinstance(prog, basestring):
+                if isinstance(prog, file):
                     filename = prog.name
                     prog = prog.read()
                 else:
diff --git a/pypy/interpreter/pyparser/future.py b/pypy/interpreter/pyparser/future.py
--- a/pypy/interpreter/pyparser/future.py
+++ b/pypy/interpreter/pyparser/future.py
@@ -225,14 +225,16 @@
             raise DoneException
         self.consume_whitespace()
 
-    def consume_whitespace(self):
+    def consume_whitespace(self, newline_ok=False):
         while 1:
             c = self.getc()
             if c in whitespace:
                 self.pos += 1
                 continue
-            elif c == '\\':
-                self.pos += 1
+            elif c == '\\' or newline_ok:
+                slash = c == '\\'
+                if slash:
+                    self.pos += 1
                 c = self.getc()
                 if c == '\n':
                     self.pos += 1
@@ -243,8 +245,10 @@
                     if self.getc() == '\n':
                         self.pos += 1
                         self.atbol()
+                elif slash:
+                    raise DoneException
                 else:
-                    raise DoneException
+                    return
             else:
                 return
 
@@ -281,7 +285,7 @@
             return
         else:
             self.pos += 1
-            self.consume_whitespace()
+            self.consume_whitespace(paren_list)
             if paren_list and self.getc() == ')':
                 self.pos += 1
                 return # Handles trailing comma inside parenthesis
diff --git a/pypy/interpreter/pyparser/test/test_futureautomaton.py b/pypy/interpreter/pyparser/test/test_futureautomaton.py
--- a/pypy/interpreter/pyparser/test/test_futureautomaton.py
+++ b/pypy/interpreter/pyparser/test/test_futureautomaton.py
@@ -3,7 +3,7 @@
 from pypy.tool import stdlib___future__ as fut
 
 def run(s):
-    f = future.FutureAutomaton(future.futureFlags_2_5, s)
+    f = future.FutureAutomaton(future.futureFlags_2_7, s)
     try:
         f.start()
     except future.DoneException:
@@ -113,6 +113,14 @@
     assert f.lineno == 1
     assert f.col_offset == 0
 
+def test_paren_with_newline():
+    s = 'from __future__ import (division,\nabsolute_import)\n'
+    f = run(s)
+    assert f.pos == len(s)
+    assert f.flags == (fut.CO_FUTURE_DIVISION | fut.CO_FUTURE_ABSOLUTE_IMPORT)
+    assert f.lineno == 1
+    assert f.col_offset == 0
+
 def test_multiline():
     s = '"abc" #def\n  #ghi\nfrom  __future__ import (division as b, generators,)\nfrom __future__ import with_statement\n'
     f = run(s)
diff --git a/pypy/interpreter/test/test_exec.py b/pypy/interpreter/test/test_exec.py
--- a/pypy/interpreter/test/test_exec.py
+++ b/pypy/interpreter/test/test_exec.py
@@ -219,3 +219,30 @@
             raise e
 
         assert res == 1
+
+    def test_exec_unicode(self):
+        # 's' is a string
+        s = "x = u'\xd0\xb9\xd1\x86\xd1\x83\xd0\xba\xd0\xb5\xd0\xbd'"
+        # 'u' is a unicode
+        u = s.decode('utf-8')
+        exec u
+        assert len(x) == 6
+        assert ord(x[0]) == 0x0439
+        assert ord(x[1]) == 0x0446
+        assert ord(x[2]) == 0x0443
+        assert ord(x[3]) == 0x043a
+        assert ord(x[4]) == 0x0435
+        assert ord(x[5]) == 0x043d
+
+    def test_eval_unicode(self):
+        u = "u'%s'" % unichr(0x1234)
+        v = eval(u)
+        assert v == unichr(0x1234)
+
+    def test_compile_unicode(self):
+        s = "x = u'\xd0\xb9\xd1\x86\xd1\x83\xd0\xba\xd0\xb5\xd0\xbd'"
+        u = s.decode('utf-8')
+        c = compile(u, '<input>', 'exec')
+        exec c
+        assert len(x) == 6
+        assert ord(x[0]) == 0x0439
diff --git a/pypy/interpreter/test/test_objspace.py b/pypy/interpreter/test/test_objspace.py
--- a/pypy/interpreter/test/test_objspace.py
+++ b/pypy/interpreter/test/test_objspace.py
@@ -71,6 +71,23 @@
         assert err.value.match(space, space.w_ValueError)
         err = raises(OperationError, space.unpackiterable, w_l, 5)
         assert err.value.match(space, space.w_ValueError)
+        w_a = space.appexec((), """():
+        class A(object):
+            def __iter__(self):
+                return self
+            def next(self):
+                raise StopIteration
+            def __len__(self):
+                1/0
+        return A()
+        """)
+        try:
+            space.unpackiterable(w_a)
+        except OperationError, o:
+            if not o.match(space, space.w_ZeroDivisionError):
+                raise Exception("DID NOT RAISE")
+        else:
+            raise Exception("DID NOT RAISE")
 
     def test_fixedview(self):
         space = self.space
diff --git a/pypy/interpreter/test/test_pyframe.py b/pypy/interpreter/test/test_pyframe.py
--- a/pypy/interpreter/test/test_pyframe.py
+++ b/pypy/interpreter/test/test_pyframe.py
@@ -1,4 +1,5 @@
 from pypy.tool import udir
+from pypy.conftest import option
 
 
 class AppTestPyFrame:
@@ -6,6 +7,15 @@
     def setup_class(cls):
         cls.w_udir = cls.space.wrap(str(udir.udir))
         cls.w_tempfile1 = cls.space.wrap(str(udir.udir.join('tempfile1')))
+        if not option.runappdirect:
+            w_call_further = cls.space.appexec([], """():
+                def call_further(f):
+                    return f()
+                return call_further
+            """)
+            assert not w_call_further.code.hidden_applevel
+            w_call_further.code.hidden_applevel = True       # hack
+            cls.w_call_further = w_call_further
 
     # test for the presence of the attributes, not functionality
 
@@ -107,6 +117,22 @@
         frame = f()
         assert frame.f_back.f_code.co_name == 'f'
 
+    def test_f_back_hidden(self):
+        if not hasattr(self, 'call_further'):
+            skip("not for runappdirect testing")
+        import sys
+        def f():
+            return (sys._getframe(0),
+                    sys._getframe(1),
+                    sys._getframe(0).f_back)
+        def main():
+            return self.call_further(f)
+        f0, f1, f1bis = main()
+        assert f0.f_code.co_name == 'f'
+        assert f1.f_code.co_name == 'main'
+        assert f1bis is f1
+        assert f0.f_back is f1
+
     def test_f_exc_xxx(self):
         import sys
 
diff --git a/pypy/jit/backend/llsupport/llmodel.py b/pypy/jit/backend/llsupport/llmodel.py
--- a/pypy/jit/backend/llsupport/llmodel.py
+++ b/pypy/jit/backend/llsupport/llmodel.py
@@ -583,6 +583,16 @@
         u = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), string)
         u.chars[index] = unichr(newvalue)
 
+    def bh_copystrcontent(self, src, dst, srcstart, dststart, length):
+        src = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), src)
+        dst = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), dst)
+        rstr.copy_string_contents(src, dst, srcstart, dststart, length)
+
+    def bh_copyunicodecontent(self, src, dst, srcstart, dststart, length):
+        src = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), src)
+        dst = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), dst)
+        rstr.copy_unicode_contents(src, dst, srcstart, dststart, length)
+
     def bh_call_i(self, func, calldescr, args_i, args_r, args_f):
         assert isinstance(calldescr, BaseIntCallDescr)
         if not we_are_translated():
diff --git a/pypy/jit/backend/llsupport/regalloc.py b/pypy/jit/backend/llsupport/regalloc.py
--- a/pypy/jit/backend/llsupport/regalloc.py
+++ b/pypy/jit/backend/llsupport/regalloc.py
@@ -57,11 +57,13 @@
     all_regs              = []
     no_lower_byte_regs    = []
     save_around_call_regs = []
-    
+    frame_reg             = None
+
     def __init__(self, longevity, frame_manager=None, assembler=None):
         self.free_regs = self.all_regs[:]
         self.longevity = longevity
         self.reg_bindings = {}
+        self.bindings_to_frame_reg = {}
         self.position = -1
         self.frame_manager = frame_manager
         self.assembler = assembler
@@ -218,6 +220,10 @@
         self.reg_bindings[v] = loc
         return loc
 
+    def force_allocate_frame_reg(self, v):
+        """ Allocate the new variable v in the frame register."""
+        self.bindings_to_frame_reg[v] = None
+
     def force_spill_var(self, var):
         self._sync_var(var)
         try:
@@ -236,6 +242,8 @@
         try:
             return self.reg_bindings[box]
         except KeyError:
+            if box in self.bindings_to_frame_reg:
+                return self.frame_reg
             return self.frame_manager.loc(box)
 
     def return_constant(self, v, forbidden_vars=[], selected_reg=None):
@@ -264,8 +272,9 @@
         self._check_type(v)
         if isinstance(v, Const):
             return self.return_constant(v, forbidden_vars, selected_reg)
-        
         prev_loc = self.loc(v)
+        if prev_loc is self.frame_reg and selected_reg is None:
+            return prev_loc
         loc = self.force_allocate_reg(v, forbidden_vars, selected_reg,
                                       need_lower_byte=need_lower_byte)
         if prev_loc is not loc:
diff --git a/pypy/jit/backend/model.py b/pypy/jit/backend/model.py
--- a/pypy/jit/backend/model.py
+++ b/pypy/jit/backend/model.py
@@ -78,7 +78,7 @@
         Optionally, return a ``ops_offset`` dictionary.  See the docstring of
         ``compiled_loop`` for more informations about it.
         """
-        raise NotImplementedError    
+        raise NotImplementedError
 
     def dump_loop_token(self, looptoken):
         """Print a disassembled version of looptoken to stdout"""
@@ -302,6 +302,10 @@
         raise NotImplementedError
     def bh_unicodesetitem(self, string, index, newvalue):
         raise NotImplementedError
+    def bh_copystrcontent(self, src, dst, srcstart, dststart, length):
+        raise NotImplementedError
+    def bh_copyunicodecontent(self, src, dst, srcstart, dststart, length):
+        raise NotImplementedError
 
     def force(self, force_token):
         raise NotImplementedError
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -956,6 +956,7 @@
         if (isinstance(from_loc, RegLoc) and from_loc.is_xmm) or (isinstance(to_loc, RegLoc) and to_loc.is_xmm):
             self.mc.MOVSD(to_loc, from_loc)
         else:
+            assert to_loc is not ebp
             self.mc.MOV(to_loc, from_loc)
 
     regalloc_mov = mov # legacy interface
@@ -2522,11 +2523,6 @@
 
     genop_discard_cond_call_gc_wb_array = genop_discard_cond_call_gc_wb
 
-    def genop_force_token(self, op, arglocs, resloc):
-        # RegAlloc.consider_force_token ensures this:
-        assert isinstance(resloc, RegLoc)
-        self.mc.LEA_rb(resloc.value, FORCE_INDEX_OFS)
-
     def not_implemented_op_discard(self, op, arglocs):
         not_implemented("not implemented operation: %s" % op.getopname())
 
diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py
--- a/pypy/jit/backend/x86/regalloc.py
+++ b/pypy/jit/backend/x86/regalloc.py
@@ -30,6 +30,7 @@
     all_regs = [eax, ecx, edx, ebx, esi, edi]
     no_lower_byte_regs = [esi, edi]
     save_around_call_regs = [eax, edx, ecx]
+    frame_reg = ebp
 
     REGLOC_TO_GCROOTMAP_REG_INDEX = {
         ebx: 1,
@@ -313,8 +314,11 @@
                     self.fm.frame_bindings[arg] = loc
             else:
                 if isinstance(loc, RegLoc):
-                    self.rm.reg_bindings[arg] = loc
-                    used[loc] = None
+                    if loc is ebp:
+                        self.rm.bindings_to_frame_reg[arg] = None
+                    else:
+                        self.rm.reg_bindings[arg] = loc
+                        used[loc] = None
                 else:
                     self.fm.frame_bindings[arg] = loc
         self.rm.free_regs = []
@@ -1401,8 +1405,8 @@
                                             self.assembler.datablockwrapper)
 
     def consider_force_token(self, op):
-        loc = self.rm.force_allocate_reg(op.result)
-        self.Perform(op, [], loc)
+        # the FORCE_TOKEN operation returns directly 'ebp'
+        self.rm.force_allocate_frame_reg(op.result)
 
     def not_implemented_op(self, op):
         not_implemented("not implemented operation: %s" % op.getopname())
diff --git a/pypy/jit/backend/x86/runner.py b/pypy/jit/backend/x86/runner.py
--- a/pypy/jit/backend/x86/runner.py
+++ b/pypy/jit/backend/x86/runner.py
@@ -119,7 +119,8 @@
             setitem(index, null)
 
     def get_latest_force_token(self):
-        return self.assembler.fail_ebp + FORCE_INDEX_OFS
+        # the FORCE_TOKEN operation and this helper both return 'ebp'.
+        return self.assembler.fail_ebp
 
     def execute_token(self, executable_token):
         addr = executable_token._x86_bootstrap_code
@@ -153,8 +154,9 @@
                                        flavor='raw', zero=True,
                                        immortal=True)
 
-    def force(self, addr_of_force_index):
+    def force(self, addr_of_force_token):
         TP = rffi.CArrayPtr(lltype.Signed)
+        addr_of_force_index = addr_of_force_token + FORCE_INDEX_OFS
         fail_index = rffi.cast(TP, addr_of_force_index)[0]
         assert fail_index >= 0, "already forced!"
         faildescr = self.get_fail_descr_from_number(fail_index)
@@ -164,7 +166,7 @@
         # start of "no gc operation!" block
         fail_index_2 = self.assembler.grab_frame_values(
             bytecode,
-            addr_of_force_index - FORCE_INDEX_OFS,
+            addr_of_force_token,
             self.all_null_registers)
         self.assembler.leave_jitted_hook()
         # end of "no gc operation!" block
diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -1186,6 +1186,12 @@
             return SpaceOperation('%s_assert_green' % kind, args, None)
         elif oopspec_name == 'jit.current_trace_length':
             return SpaceOperation('current_trace_length', [], op.result)
+        elif oopspec_name == 'jit.isconstant':
+            kind = getkind(args[0].concretetype)
+            return SpaceOperation('%s_isconstant' % kind, args, op.result)
+        elif oopspec_name == 'jit.isvirtual':
+            kind = getkind(args[0].concretetype)
+            return SpaceOperation('%s_isvirtual' % kind, args, op.result)
         else:
             raise AssertionError("missing support for %r" % oopspec_name)
 
@@ -1443,6 +1449,14 @@
         else:
             assert 0, "args[0].concretetype must be STR or UNICODE"
         #
+        if oopspec_name == 'stroruni.copy_contents':
+            if SoU.TO == rstr.STR:
+                new_op = 'copystrcontent'
+            elif SoU.TO == rstr.UNICODE:
+                new_op = 'copyunicodecontent'
+            else:
+                assert 0
+            return SpaceOperation(new_op, args, op.result)
         if oopspec_name == "stroruni.equal":
             for otherindex, othername, argtypes, resulttype in [
                 (EffectInfo.OS_STREQ_SLICE_CHECKNULL,
diff --git a/pypy/jit/metainterp/blackhole.py b/pypy/jit/metainterp/blackhole.py
--- a/pypy/jit/metainterp/blackhole.py
+++ b/pypy/jit/metainterp/blackhole.py
@@ -834,6 +834,18 @@
     def bhimpl_current_trace_length():
         return -1
 
+    @arguments("i", returns="i")
+    def bhimpl_int_isconstant(x):
+        return False
+
+    @arguments("r", returns="i")
+    def bhimpl_ref_isconstant(x):
+        return False
+
+    @arguments("r", returns="i")
+    def bhimpl_ref_isvirtual(x):
+        return False
+
     # ----------
     # the main hints and recursive calls
 
@@ -1243,6 +1255,9 @@
     @arguments("cpu", "r", "i", "i")
     def bhimpl_strsetitem(cpu, string, index, newchr):
         cpu.bh_strsetitem(string, index, newchr)
+    @arguments("cpu", "r", "r", "i", "i", "i")
+    def bhimpl_copystrcontent(cpu, src, dst, srcstart, dststart, length):
+        cpu.bh_copystrcontent(src, dst, srcstart, dststart, length)
 
     @arguments("cpu", "i", returns="r")
     def bhimpl_newunicode(cpu, length):
@@ -1256,6 +1271,9 @@
     @arguments("cpu", "r", "i", "i")
     def bhimpl_unicodesetitem(cpu, unicode, index, newchr):
         cpu.bh_unicodesetitem(unicode, index, newchr)
+    @arguments("cpu", "r", "r", "i", "i", "i")
+    def bhimpl_copyunicodecontent(cpu, src, dst, srcstart, dststart, length):
+        cpu.bh_copyunicodecontent(src, dst, srcstart, dststart, length)
 
     @arguments(returns=(longlong.is_64_bit and "i" or "f"))
     def bhimpl_ll_read_timestamp():
@@ -1460,7 +1478,7 @@
 def resume_in_blackhole(metainterp_sd, jitdriver_sd, resumedescr,
                         all_virtuals=None):
     from pypy.jit.metainterp.resume import blackhole_from_resumedata
-    debug_start('jit-blackhole')
+    #debug_start('jit-blackhole')
     metainterp_sd.profiler.start_blackhole()
     blackholeinterp = blackhole_from_resumedata(
         metainterp_sd.blackholeinterpbuilder,
@@ -1479,12 +1497,12 @@
         _run_forever(blackholeinterp, current_exc)
     finally:
         metainterp_sd.profiler.end_blackhole()
-        debug_stop('jit-blackhole')
+        #debug_stop('jit-blackhole')
 
 def convert_and_run_from_pyjitpl(metainterp, raising_exception=False):
     # Get a chain of blackhole interpreters and fill them by copying
     # 'metainterp.framestack'.
-    debug_start('jit-blackhole')
+    #debug_start('jit-blackhole')
     metainterp_sd = metainterp.staticdata
     metainterp_sd.profiler.start_blackhole()
     nextbh = None
@@ -1507,4 +1525,4 @@
         _run_forever(firstbh, current_exc)
     finally:
         metainterp_sd.profiler.end_blackhole()
-        debug_stop('jit-blackhole')
+        #debug_stop('jit-blackhole')
diff --git a/pypy/jit/metainterp/heapcache.py b/pypy/jit/metainterp/heapcache.py
new file mode 100644
--- /dev/null
+++ b/pypy/jit/metainterp/heapcache.py
@@ -0,0 +1,210 @@
+from pypy.jit.metainterp.history import ConstInt
+from pypy.jit.metainterp.resoperation import rop
+
+
+class HeapCache(object):
+    def __init__(self):
+        self.reset()
+
+    def reset(self):
+        # contains boxes where the class is already known
+        self.known_class_boxes = {}
+        # store the boxes that contain newly allocated objects, this maps the
+        # boxes to a bool, the bool indicates whether or not the object has
+        # escaped the trace or not (True means the box never escaped, False
+        # means it did escape), its presences in the mapping shows that it was
+        # allocated inside the trace
+        self.new_boxes = {}
+        # Tracks which boxes should be marked as escaped when the key box
+        # escapes.
+        self.dependencies = {}
+        # contains frame boxes that are not virtualizables
+        self.nonstandard_virtualizables = {}
+        # heap cache
+        # maps descrs to {from_box, to_box} dicts
+        self.heap_cache = {}
+        # heap array cache
+        # maps descrs to {index: {from_box: to_box}} dicts
+        self.heap_array_cache = {}
+        # cache the length of arrays
+        self.length_cache = {}
+
+    def invalidate_caches(self, opnum, descr, argboxes):
+        self.mark_escaped(opnum, argboxes)
+        self.clear_caches(opnum, descr, argboxes)
+
+    def mark_escaped(self, opnum, argboxes):
+        idx = 0
+        if opnum == rop.SETFIELD_GC:
+            assert len(argboxes) == 2
+            box, valuebox = argboxes
+            if self.is_unescaped(box) and self.is_unescaped(valuebox):
+                self.dependencies.setdefault(box, []).append(valuebox)
+            else:
+                self._escape(valuebox)
+        # GETFIELD_GC doesn't escape it's argument
+        elif opnum != rop.GETFIELD_GC:
+            for box in argboxes:
+                # setarrayitem_gc don't escape its first argument
+                if not (idx == 0 and opnum in [rop.SETARRAYITEM_GC]):
+                    self._escape(box)
+                idx += 1
+
+    def _escape(self, box):
+        if box in self.new_boxes:
+            self.new_boxes[box] = False
+        if box in self.dependencies:
+            for dep in self.dependencies[box]:
+                self._escape(dep)
+            del self.dependencies[box]
+
+    def clear_caches(self, opnum, descr, argboxes):
+        if opnum == rop.SETFIELD_GC:
+            return
+        if opnum == rop.SETARRAYITEM_GC:
+            return
+        if opnum == rop.SETFIELD_RAW:
+            return
+        if opnum == rop.SETARRAYITEM_RAW:
+            return
+        if rop._OVF_FIRST <= opnum <= rop._OVF_LAST:
+            return
+        if rop._NOSIDEEFFECT_FIRST <= opnum <= rop._NOSIDEEFFECT_LAST:
+            return
+        if opnum == rop.CALL or opnum == rop.CALL_LOOPINVARIANT:
+            effectinfo = descr.get_extra_info()
+            ef = effectinfo.extraeffect
+            if ef == effectinfo.EF_LOOPINVARIANT or \
+               ef == effectinfo.EF_ELIDABLE_CANNOT_RAISE or \
+               ef == effectinfo.EF_ELIDABLE_CAN_RAISE:
+                return
+            # A special case for ll_arraycopy, because it is so common, and its
+            # effects are so well defined.
+            elif effectinfo.oopspecindex == effectinfo.OS_ARRAYCOPY:
+                # The destination box
+                if argboxes[2] in self.new_boxes:
+                    # XXX: no descr here so we invalidate any of them, not just
+                    # of the correct type
+                    # XXX: in theory the indices of the copy could be looked at
+                    # as well
+                    for descr, cache in self.heap_array_cache.iteritems():
+                        for idx, cache in cache.iteritems():
+                            for frombox in cache.keys():
+                                if frombox not in self.new_boxes:
+                                    del cache[frombox]
+                    return
+
+        self.heap_cache.clear()
+        self.heap_array_cache.clear()
+
+    def is_class_known(self, box):
+        return box in self.known_class_boxes
+
+    def class_now_known(self, box):
+        self.known_class_boxes[box] = None
+
+    def is_nonstandard_virtualizable(self, box):
+        return box in self.nonstandard_virtualizables
+
+    def nonstandard_virtualizables_now_known(self, box):
+        self.nonstandard_virtualizables[box] = None
+
+    def is_unescaped(self, box):
+        return self.new_boxes.get(box, False)
+
+    def new(self, box):
+        self.new_boxes[box] = True
+
+    def new_array(self, box, lengthbox):
+        self.new(box)
+        self.arraylen_now_known(box, lengthbox)
+
+    def getfield(self, box, descr):
+        d = self.heap_cache.get(descr, None)
+        if d:
+            tobox = d.get(box, None)
+            if tobox:
+                return tobox
+        return None
+
+    def getfield_now_known(self, box, descr, fieldbox):
+        self.heap_cache.setdefault(descr, {})[box] = fieldbox
+
+    def setfield(self, box, descr, fieldbox):
+        d = self.heap_cache.get(descr, None)
+        new_d = self._do_write_with_aliasing(d, box, fieldbox)
+        self.heap_cache[descr] = new_d
+
+    def _do_write_with_aliasing(self, d, box, fieldbox):
+        # slightly subtle logic here
+        # a write to an arbitrary box, all other boxes can alias this one
+        if not d or box not in self.new_boxes:
+            # therefore we throw away the cache
+            return {box: fieldbox}
+        # the object we are writing to is freshly allocated
+        # only remove some boxes from the cache
+        new_d = {}
+        for frombox, tobox in d.iteritems():
+            # the other box is *also* freshly allocated
+            # therefore frombox and box *must* contain different objects
+            # thus we can keep it in the cache
+            if frombox in self.new_boxes:
+                new_d[frombox] = tobox
+        new_d[box] = fieldbox
+        return new_d
+
+    def getarrayitem(self, box, descr, indexbox):
+        if not isinstance(indexbox, ConstInt):
+            return
+        index = indexbox.getint()
+        cache = self.heap_array_cache.get(descr, None)
+        if cache:
+            indexcache = cache.get(index, None)
+            if indexcache is not None:
+                return indexcache.get(box, None)
+
+    def getarrayitem_now_known(self, box, descr, indexbox, valuebox):
+        if not isinstance(indexbox, ConstInt):
+            return
+        index = indexbox.getint()
+        cache = self.heap_array_cache.setdefault(descr, {})
+        indexcache = cache.get(index, None)
+        if indexcache is not None:
+            indexcache[box] = valuebox
+        else:
+            cache[index] = {box: valuebox}
+
+    def setarrayitem(self, box, descr, indexbox, valuebox):
+        if not isinstance(indexbox, ConstInt):
+            cache = self.heap_array_cache.get(descr, None)
+            if cache is not None:
+                cache.clear()
+            return
+        index = indexbox.getint()
+        cache = self.heap_array_cache.setdefault(descr, {})
+        indexcache = cache.get(index, None)
+        cache[index] = self._do_write_with_aliasing(indexcache, box, valuebox)
+
+    def arraylen(self, box):
+        return self.length_cache.get(box, None)
+
+    def arraylen_now_known(self, box, lengthbox):
+        self.length_cache[box] = lengthbox
+
+    def _replace_box(self, d, oldbox, newbox):
+        new_d = {}
+        for frombox, tobox in d.iteritems():
+            if frombox is oldbox:
+                frombox = newbox
+            if tobox is oldbox:
+                tobox = newbox
+            new_d[frombox] = tobox
+        return new_d
+
+    def replace_box(self, oldbox, newbox):
+        for descr, d in self.heap_cache.iteritems():
+            self.heap_cache[descr] = self._replace_box(d, oldbox, newbox)
+        for descr, d in self.heap_array_cache.iteritems():
+            for index, cache in d.iteritems():
+                d[index] = self._replace_box(cache, oldbox, newbox)
+        self.length_cache = self._replace_box(self.length_cache, oldbox, newbox)
diff --git a/pypy/jit/metainterp/optimizeopt/heap.py b/pypy/jit/metainterp/optimizeopt/heap.py
--- a/pypy/jit/metainterp/optimizeopt/heap.py
+++ b/pypy/jit/metainterp/optimizeopt/heap.py
@@ -25,7 +25,7 @@
         #      'cached_fields'.
         #
         self._cached_fields = {}
-        self._cached_fields_getfield_op = {}        
+        self._cached_fields_getfield_op = {}
         self._lazy_setfield = None
         self._lazy_setfield_registered = False
 
@@ -37,6 +37,12 @@
             self.force_lazy_setfield(optheap)
             assert not self.possible_aliasing(optheap, structvalue)
         cached_fieldvalue = self._cached_fields.get(structvalue, None)
+
+        # Hack to ensure constants are imported from the preamble
+        if cached_fieldvalue and fieldvalue.is_constant(): 
+            optheap.optimizer.ensure_imported(cached_fieldvalue)
+            cached_fieldvalue = self._cached_fields.get(structvalue, None)
+
         if cached_fieldvalue is not fieldvalue:
             # common case: store the 'op' as lazy_setfield, and register
             # myself in the optheap's _lazy_setfields_and_arrayitems list
@@ -75,7 +81,7 @@
     def remember_field_value(self, structvalue, fieldvalue, getfield_op=None):
         assert self._lazy_setfield is None
         self._cached_fields[structvalue] = fieldvalue
-        self._cached_fields_getfield_op[structvalue] = getfield_op        
+        self._cached_fields_getfield_op[structvalue] = getfield_op
 
     def force_lazy_setfield(self, optheap, can_cache=True):
         op = self._lazy_setfield
@@ -132,9 +138,7 @@
                         result = newresult
                     getop = ResOperation(rop.GETFIELD_GC, [op.getarg(0)],
                                          result, op.getdescr())
-                    getop = shortboxes.add_potential(getop)
-                    self._cached_fields_getfield_op[structvalue] = getop
-                    self._cached_fields[structvalue] = optimizer.getvalue(result)
+                    shortboxes.add_potential(getop, synthetic=True)
                 elif op.result is not None:
                     shortboxes.add_potential(op)
 
@@ -163,7 +167,7 @@
 
     def new(self):
         return OptHeap()
-        
+
     def produce_potential_short_preamble_ops(self, sb):
         descrkeys = self.cached_fields.keys()
         if not we_are_translated():
diff --git a/pypy/jit/metainterp/optimizeopt/optimizer.py b/pypy/jit/metainterp/optimizeopt/optimizer.py
--- a/pypy/jit/metainterp/optimizeopt/optimizer.py
+++ b/pypy/jit/metainterp/optimizeopt/optimizer.py
@@ -10,6 +10,7 @@
 from pypy.jit.metainterp.typesystem import llhelper, oohelper
 from pypy.tool.pairtype import extendabletype
 from pypy.rlib.debug import debug_start, debug_stop, debug_print
+from pypy.rlib.objectmodel import specialize
 
 LEVEL_UNKNOWN    = '\x00'
 LEVEL_NONNULL    = '\x01'
@@ -25,6 +26,9 @@
         self.descr = descr
         self.bound = bound
 
+    def clone(self):
+        return LenBound(self.mode, self.descr, self.bound.clone())
+
 class OptValue(object):
     __metaclass__ = extendabletype
     _attrs_ = ('box', 'known_class', 'last_guard_index', 'level', 'intbound', 'lenbound')
@@ -67,7 +71,7 @@
             guards.append(op)
         elif self.level == LEVEL_KNOWNCLASS:
             op = ResOperation(rop.GUARD_NONNULL, [box], None)
-            guards.append(op)            
+            guards.append(op)
             op = ResOperation(rop.GUARD_CLASS, [box, self.known_class], None)
             guards.append(op)
         else:
@@ -88,8 +92,27 @@
                     assert False
                 guards.append(op)
                 self.lenbound.bound.make_guards(lenbox, guards)
+        return guards
 
-        return guards
+    def import_from(self, other, optimizer):
+        assert self.level <= LEVEL_NONNULL
+        if other.level == LEVEL_CONSTANT:
+            self.make_constant(other.get_key_box())
+            optimizer.turned_constant(self)
+        elif other.level == LEVEL_KNOWNCLASS:
+            self.make_constant_class(other.known_class, -1)
+        else:
+            if other.level == LEVEL_NONNULL:
+                self.ensure_nonnull()
+            self.intbound.intersect(other.intbound)
+            if other.lenbound:
+                if self.lenbound:
+                    assert other.lenbound.mode == self.lenbound.mode
+                    assert other.lenbound.descr == self.lenbound.descr
+                    self.lenbound.bound.intersect(other.lenbound.bound)
+                else:
+                    self.lenbound = other.lenbound.clone()
+
 
     def force_box(self):
         return self.box
@@ -123,7 +146,7 @@
         assert isinstance(constbox, Const)
         self.box = constbox
         self.level = LEVEL_CONSTANT
-        
+
         if isinstance(constbox, ConstInt):
             val = constbox.getint()
             self.intbound = IntBound(val, val)
@@ -200,6 +223,9 @@
     def __init__(self, box):
         self.make_constant(box)
 
+    def __repr__(self):
+        return 'Constant(%r)' % (self.box,)
+
 CONST_0      = ConstInt(0)
 CONST_1      = ConstInt(1)
 CVAL_ZERO    = ConstantValue(CONST_0)
@@ -308,7 +334,6 @@
         self.resumedata_memo = resume.ResumeDataLoopMemo(metainterp_sd)
         self.bool_boxes = {}
         self.pure_operations = args_dict()
-        self.emitted_pure_operations = {}
         self.producer = {}
         self.pendingfields = []
         self.posponedop = None
@@ -316,12 +341,11 @@
         self.quasi_immutable_deps = None
         self.opaque_pointers = {}
         self.newoperations = []
-        self.emitting_dissabled = False
-        self.emitted_guards = 0        
         if loop is not None:
             self.call_pure_results = loop.call_pure_results
 
         self.set_optimizations(optimizations)
+        self.setup()
 
     def set_optimizations(self, optimizations):
         if optimizations:
@@ -348,23 +372,18 @@
         assert self.posponedop is None
 
     def new(self):
+        new = Optimizer(self.metainterp_sd, self.loop)
+        return self._new(new)
+
+    def _new(self, new):
         assert self.posponedop is None
-        new = Optimizer(self.metainterp_sd, self.loop)
         optimizations = [o.new() for o in self.optimizations]
         new.set_optimizations(optimizations)
         new.quasi_immutable_deps = self.quasi_immutable_deps
         return new
-        
+
     def produce_potential_short_preamble_ops(self, sb):
-        for op in self.emitted_pure_operations:
-            if op.getopnum() == rop.GETARRAYITEM_GC_PURE or \
-               op.getopnum() == rop.STRGETITEM or \
-               op.getopnum() == rop.UNICODEGETITEM:
-                if not self.getvalue(op.getarg(1)).is_constant():
-                    continue
-            sb.add_potential(op)
-        for opt in self.optimizations:
-            opt.produce_potential_short_preamble_ops(sb)
+        raise NotImplementedError('This is implemented in unroll.UnrollableOptimizer')
 
     def turned_constant(self, value):
         for o in self.optimizations:
@@ -386,19 +405,26 @@
         else:
             return box
 
+    @specialize.argtype(0)
     def getvalue(self, box):
         box = self.getinterned(box)
         try:
             value = self.values[box]
         except KeyError:
             value = self.values[box] = OptValue(box)
+        self.ensure_imported(value)
         return value
 
+    def ensure_imported(self, value):
+        pass
+
+    @specialize.argtype(0)
     def get_constant_box(self, box):
         if isinstance(box, Const):
             return box
         try:
             value = self.values[box]
+            self.ensure_imported(value)
         except KeyError:
             return None
         if value.is_constant():
@@ -481,18 +507,22 @@
     def emit_operation(self, op):
         if op.returns_bool_result():
             self.bool_boxes[self.getvalue(op.result)] = None
-        if self.emitting_dissabled:
-            return
-        
+        self._emit_operation(op)
+
+    @specialize.argtype(0)
+    def _emit_operation(self, op):
         for i in range(op.numargs()):
             arg = op.getarg(i)
-            if arg in self.values:
-                box = self.values[arg].force_box()
-                op.setarg(i, box)
+            try:
+                value = self.values[arg]
+            except KeyError:
+                pass
+            else:
+                self.ensure_imported(value)
+                op.setarg(i, value.force_box())
         self.metainterp_sd.profiler.count(jitprof.OPT_OPS)
         if op.is_guard():
             self.metainterp_sd.profiler.count(jitprof.OPT_GUARDS)
-            self.emitted_guards += 1 # FIXME: can we reuse above counter?
             op = self.store_final_boxes_in_guard(op)
         elif op.can_raise():
             self.exception_might_have_happened = True
@@ -541,9 +571,10 @@
                 arg = value.get_key_box()
             args[i] = arg
         args[n] = ConstInt(op.getopnum())
-        args[n+1] = op.getdescr()
+        args[n + 1] = op.getdescr()
         return args
 
+    @specialize.argtype(0)
     def optimize_default(self, op):
         canfold = op.is_always_pure()
         if op.is_ovf():
@@ -579,13 +610,16 @@
                 return
             else:
                 self.pure_operations[args] = op
-                self.emitted_pure_operations[op] = True
+                self.remember_emitting_pure(op)
 
         # otherwise, the operation remains
         self.emit_operation(op)
         if nextop:
             self.emit_operation(nextop)
 
+    def remember_emitting_pure(self, op):
+        pass
+
     def constant_fold(self, op):
         argboxes = [self.get_constant_box(op.getarg(i))
                     for i in range(op.numargs())]
@@ -627,9 +661,9 @@
             arrayvalue = self.getvalue(op.getarg(0))
             arrayvalue.make_len_gt(MODE_UNICODE, op.getdescr(), indexvalue.box.getint())
         self.optimize_default(op)
-        
 
-    
+
+
 
 dispatch_opt = make_dispatcher_method(Optimizer, 'optimize_',
         default=Optimizer.optimize_default)
diff --git a/pypy/jit/metainterp/optimizeopt/rewrite.py b/pypy/jit/metainterp/optimizeopt/rewrite.py
--- a/pypy/jit/metainterp/optimizeopt/rewrite.py
+++ b/pypy/jit/metainterp/optimizeopt/rewrite.py
@@ -19,7 +19,7 @@
 
     def new(self):
         return OptRewrite()
-        
+
     def produce_potential_short_preamble_ops(self, sb):
         for op in self.loop_invariant_producer.values():
             sb.add_potential(op)
@@ -231,6 +231,17 @@
             else:
                 self.make_constant(op.result, result)
                 return
+
+        args = self.optimizer.make_args_key(op)
+        oldop = self.optimizer.pure_operations.get(args, None)
+        if oldop is not None and oldop.getdescr() is op.getdescr():
+            assert oldop.getopnum() == op.getopnum()
+            self.make_equal_to(op.result, self.getvalue(oldop.result))
+            return
+        else:
+            self.optimizer.pure_operations[args] = op
+            self.optimizer.remember_emitting_pure(op)
+
         # replace CALL_PURE with just CALL
         args = op.getarglist()
         self.emit_operation(ResOperation(rop.CALL, args, op.result,
@@ -351,7 +362,7 @@
         # expects a compile-time constant
         assert isinstance(arg, Const)
         key = make_hashable_int(arg.getint())
-        
+
         resvalue = self.loop_invariant_results.get(key, None)
         if resvalue is not None:
             self.make_equal_to(op.result, resvalue)
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -4711,6 +4711,83 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_empty_copystrunicontent(self):
+        ops = """
+        [p0, p1, i0, i2, i3]
+        i4 = int_eq(i3, 0)
+        guard_true(i4) []
+        copystrcontent(p0, p1, i0, i2, i3)
+        jump(p0, p1, i0, i2, i3)
+        """
+        expected = """
+        [p0, p1, i0, i2, i3]
+        i4 = int_eq(i3, 0)
+        guard_true(i4) []
+        jump(p0, p1, i0, i2, 0)
+        """
+        self.optimize_strunicode_loop(ops, expected)
+
+    def test_empty_copystrunicontent_virtual(self):
+        ops = """
+        [p0]
+        p1 = newstr(23)
+        copystrcontent(p0, p1, 0, 0, 0)
+        jump(p0)
+        """
+        expected = """
+        [p0]
+        jump(p0)
+        """
+        self.optimize_strunicode_loop(ops, expected)
+
+    def test_forced_virtuals_aliasing(self):
+        ops = """
+        [i0, i1]
+        p0 = new(descr=ssize)
+        p1 = new(descr=ssize)
+        escape(p0)
+        escape(p1)
+        setfield_gc(p0, i0, descr=adescr)
+        setfield_gc(p1, i1, descr=adescr)
+        i2 = getfield_gc(p0, descr=adescr)
+        jump(i2, i2)
+        """
+        expected = """
+        [i0, i1]
+        p0 = new(descr=ssize)
+        escape(p0)
+        p1 = new(descr=ssize)
+        escape(p1)
+        setfield_gc(p0, i0, descr=adescr)
+        setfield_gc(p1, i1, descr=adescr)
+        jump(i0, i0)
+        """
+        py.test.skip("not implemented")
+        # setfields on things that used to be virtual still can't alias each
+        # other
+        self.optimize_loop(ops, expected)
+
+    def test_plain_virtual_string_copy_content(self):
+        ops = """
+        []
+        p0 = newstr(6)
+        copystrcontent(s"hello!", p0, 0, 0, 6)
+        p1 = call(0, p0, s"abc123", descr=strconcatdescr)
+        i0 = strgetitem(p1, 0)
+        finish(i0)
+        """
+        expected = """
+        []
+        p0 = newstr(6)
+        copystrcontent(s"hello!", p0, 0, 0, 6)
+        p1 = newstr(12)
+        copystrcontent(p0, p1, 0, 0, 6)
+        copystrcontent(s"abc123", p1, 0, 6, 6)
+        i0 = strgetitem(p1, 0)
+        finish(i0)
+        """
+        self.optimize_strunicode_loop(ops, expected)
+
 
 class TestLLtype(BaseTestOptimizeBasic, LLtypeMixin):
     pass
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -102,9 +102,9 @@
             print "Short Preamble:"
             short = loop.preamble.token.short_preamble[0]
             print short.inputargs
-            print '\n'.join([str(o) for o in short.operations])        
+            print '\n'.join([str(o) for o in short.operations])
             print
-        
+
         assert expected != "crash!", "should have raised an exception"
         self.assert_equal(loop, expected)
         if expected_preamble:
@@ -113,7 +113,7 @@
         if expected_short:
             self.assert_equal(short, expected_short,
                               text_right='expected short preamble')
-            
+
         return loop
 
 class OptimizeOptTest(BaseTestWithUnroll):
@@ -472,7 +472,13 @@
         [i0]
         jump(i0)
         """
-        self.optimize_loop(ops, expected, preamble)
+        short = """
+        [i0]
+        i1 = int_is_true(i0)
+        guard_value(i1, 1) []
+        jump(i0)
+        """
+        self.optimize_loop(ops, expected, preamble, expected_short=short)
 
     def test_bound_int_is_true(self):
         ops = """
@@ -860,10 +866,10 @@
         setfield_gc(p3sub, i1, descr=valuedescr)
         setfield_gc(p1, p3sub, descr=nextdescr)
         # XXX: We get two extra operations here because the setfield
-        #      above is the result of forcing p1 and thus not 
+        #      above is the result of forcing p1 and thus not
         #      registered with the heap optimizer. I've makred tests
         #      below with VIRTUALHEAP if they suffer from this issue
-        p3sub2 = getfield_gc(p1, descr=nextdescr) 
+        p3sub2 = getfield_gc(p1, descr=nextdescr)
         guard_nonnull_class(p3sub2, ConstClass(node_vtable2)) []
         jump(i1, p1, p3sub2)
         """
@@ -1405,7 +1411,7 @@
         guard_isnull(p18) [p0, p8]
         p31 = new(descr=ssize)
         p35 = new_with_vtable(ConstClass(node_vtable))
-        setfield_gc(p35, p31, descr=valuedescr)        
+        setfield_gc(p35, p31, descr=valuedescr)
         jump(p0, p35)
         """
         expected = """
@@ -1420,7 +1426,7 @@
         guard_isnull(p18) [p0, p8]
         p31 = new(descr=ssize)
         p35 = new_with_vtable(ConstClass(node_vtable))
-        setfield_gc(p35, p31, descr=valuedescr)        
+        setfield_gc(p35, p31, descr=valuedescr)
         jump(p0, p35, p19, p18)
         """
         expected = """
@@ -1429,7 +1435,7 @@
         jump(p0, NULL)
         """
         self.optimize_loop(ops, expected)
-        
+
     def test_varray_1(self):
         ops = """
         [i1]
@@ -2175,7 +2181,7 @@
         jump(p1)
         """
         self.optimize_loop(ops, expected)
-        
+
     def test_duplicate_getarrayitem_2(self):
         ops = """
         [p1, i0]
@@ -2193,7 +2199,7 @@
         jump(p1, i7, i6)
         """
         self.optimize_loop(ops, expected)
-        
+
     def test_duplicate_getarrayitem_after_setarrayitem_1(self):
         ops = """
         [p1, p2]
@@ -2806,14 +2812,14 @@
         guard_no_overflow() []
         i3b = int_is_true(i3)
         guard_true(i3b) []
-        setfield_gc(p1, i1, descr=valuedescr)        
+        setfield_gc(p1, i1, descr=valuedescr)
         escape(i3)
         escape(i3)
         jump(i1, p1, i3)
         """
         expected = """
         [i1, p1, i3]
-        setfield_gc(p1, i1, descr=valuedescr)        
+        setfield_gc(p1, i1, descr=valuedescr)
         escape(i3)
         escape(i3)
         jump(i1, p1, i3)
@@ -2824,7 +2830,7 @@
         ops = """
         [p8, p11, i24]
         p26 = new_with_vtable(ConstClass(node_vtable))
-        setfield_gc(p26, i24, descr=adescr)        
+        setfield_gc(p26, i24, descr=adescr)
         i34 = getfield_gc_pure(p11, descr=valuedescr)
         i35 = getfield_gc_pure(p26, descr=adescr)
         i36 = int_add_ovf(i34, i35)
@@ -2833,10 +2839,10 @@
         """
         expected = """
         [p8, p11, i26]
-        jump(p8, p11, i26)        
-        """
-        self.optimize_loop(ops, expected)
-        
+        jump(p8, p11, i26)
+        """
+        self.optimize_loop(ops, expected)
+
     def test_ovf_guard_in_short_preamble2(self):
         ops = """
         [p8, p11, p12]
@@ -3185,13 +3191,18 @@
         jump(p1, i4, i3)
         '''
         expected = '''
+        [p1, i4, i3, i5]
+        setfield_gc(p1, i5, descr=valuedescr)
+        jump(p1, i3, i5, i5)
+        '''
+        preamble = '''
         [p1, i1, i4]
         setfield_gc(p1, i1, descr=valuedescr)
         i3 = call(p1, descr=plaincalldescr)
         setfield_gc(p1, i3, descr=valuedescr)
-        jump(p1, i4, i3)
+        jump(p1, i4, i3, i3)
         '''
-        self.optimize_loop(ops, expected, expected)
+        self.optimize_loop(ops, expected, preamble)
 
     def test_call_pure_invalidates_heap_knowledge(self):
         # CALL_PURE should still force the setfield_gc() to occur before it
@@ -3203,21 +3214,20 @@
         jump(p1, i4, i3)
         '''
         expected = '''
+        [p1, i4, i3, i5]
+        setfield_gc(p1, i4, descr=valuedescr)
+        jump(p1, i3, i5, i5)
+        '''
+        preamble = '''
         [p1, i1, i4]
         setfield_gc(p1, i1, descr=valuedescr)
         i3 = call(p1, descr=plaincalldescr)
         setfield_gc(p1, i1, descr=valuedescr)
-        jump(p1, i4, i3)
+        jump(p1, i4, i3, i3)
         '''
-        self.optimize_loop(ops, expected, expected)
+        self.optimize_loop(ops, expected, preamble)
 
     def test_call_pure_constant_folding(self):
-        # CALL_PURE is not marked as is_always_pure(), because it is wrong
-        # to call the function arbitrary many times at arbitrary points in
-        # time.  Check that it is either constant-folded (and replaced by
-        # the result of the call, recorded as the first arg), or turned into
-        # a regular CALL.
-        # XXX can this test be improved with unrolling?
         arg_consts = [ConstInt(i) for i in (123456, 4, 5, 6)]
         call_pure_results = {tuple(arg_consts): ConstInt(42)}
         ops = '''
@@ -3233,14 +3243,13 @@
         escape(i1)
         escape(i2)
         i4 = call(123456, 4, i0, 6, descr=plaincalldescr)
-        jump(i0, i4)
+        jump(i0, i4, i4)
         '''
         expected = '''
-        [i0, i2]
+        [i0, i4, i5]
         escape(42)
-        escape(i2)
-        i4 = call(123456, 4, i0, 6, descr=plaincalldescr)
-        jump(i0, i4)
+        escape(i4)
+        jump(i0, i5, i5)
         '''
         self.optimize_loop(ops, expected, preamble, call_pure_results)
 
@@ -3264,18 +3273,43 @@
         escape(i2)
         i4 = call(123456, 4, i0, 6, descr=plaincalldescr)
         guard_no_exception() []
-        jump(i0, i4)
+        jump(i0, i4, i4)
         '''
         expected = '''
-        [i0, i2]
+        [i0, i2, i3]
         escape(42)
         escape(i2)
-        i4 = call(123456, 4, i0, 6, descr=plaincalldescr)
-        guard_no_exception() []
-        jump(i0, i4)
+        jump(i0, i3, i3)
         '''
         self.optimize_loop(ops, expected, preamble, call_pure_results)
 
+    def test_call_pure_returning_virtual(self):
+        # XXX: This kind of loop invaraint call_pure will be forced
+        #      both in the preamble and in the peeled loop
+        ops = '''
+        [p1, i1, i2]
+        p2 = call_pure(0, p1, i1, i2, descr=strslicedescr)
+        escape(p2)
+        jump(p1, i1, i2)
+        '''
+        preamble = '''
+        [p1, i1, i2]
+        i6 = int_sub(i2, i1)
+        p2 = newstr(i6)
+        copystrcontent(p1, p2, i1, 0, i6)
+        escape(p2)
+        jump(p1, i1, i2, i6)
+        '''
+        expected = '''
+        [p1, i1, i2, i6]
+        p2 = newstr(i6)
+        copystrcontent(p1, p2, i1, 0, i6)
+        escape(p2)
+        jump(p1, i1, i2, i6)
+        '''
+        self.optimize_loop(ops, expected, preamble)
+        
+
     # ----------
 
     def test_vref_nonvirtual_nonescape(self):
@@ -5144,14 +5178,14 @@
         [i0, i1, i10, i11, i2, i3, i4]
         escape(i2)
         escape(i3)
-        escape(i4)        
+        escape(i4)
         i24 = int_mul_ovf(i10, i11)
         guard_no_overflow() []
         i23 = int_sub_ovf(i10, i11)
         guard_no_overflow() []
         i22 = int_add_ovf(i10, i11)
         guard_no_overflow() []
-        jump(i0, i1, i10, i11, i2, i3, i4) 
+        jump(i0, i1, i10, i11, i2, i3, i4)
         """
         self.optimize_loop(ops, expected)
 
@@ -5360,6 +5394,8 @@
         """
         self.optimize_strunicode_loop(ops, expected, expected)
 
+    # XXX Should some of the call's below now be call_pure?
+
     def test_str_concat_1(self):
         ops = """
         [p1, p2]
@@ -5693,14 +5729,14 @@
         ops = """
         [p0, i0]
         i1 = unicodegetitem(p0, i0)
-        i10 = unicodegetitem(p0, i0)        
+        i10 = unicodegetitem(p0, i0)
         i2 = int_lt(i1, 0)
         guard_false(i2) []
         jump(p0, i0)
         """
         expected = """
         [p0, i0]
-        i1 = unicodegetitem(p0, i0)        
+        i1 = unicodegetitem(p0, i0)
         jump(p0, i0)
         """
         self.optimize_loop(ops, expected)
@@ -5859,7 +5895,7 @@
         """
         preamble = """
         [p1, i1, i2, p3]
-        guard_nonnull(p3) []        
+        guard_nonnull(p3) []
         i4 = int_sub(i2, i1)
         i0 = call(0, p1, i1, i4, p3, descr=streq_slice_nonnull_descr)
         escape(i0)
@@ -6468,7 +6504,7 @@
         setfield_gc(p3, i1, descr=adescr)
         setfield_gc(p3, i2, descr=bdescr)
         i5 = int_gt(ii, 42)
-        guard_true(i5) []        
+        guard_true(i5) []
         jump(p0, p1, p3, ii2, ii, i1, i2)
         """
         self.optimize_loop(ops, expected)
@@ -6494,7 +6530,7 @@
         p1 = getfield_gc(p0, descr=nextdescr)
         guard_nonnull_class(p1, ConstClass(node_vtable)) []
         p2 = getfield_gc(p1, descr=nextdescr)
-        guard_nonnull_class(p2, ConstClass(node_vtable)) []        
+        guard_nonnull_class(p2, ConstClass(node_vtable)) []
         jump(p0)
         """
         expected = """
@@ -6508,11 +6544,11 @@
         guard_class(p1, ConstClass(node_vtable)) []
         p2 = getfield_gc(p1, descr=nextdescr)
         guard_nonnull(p2) []
-        guard_class(p2, ConstClass(node_vtable)) []        
+        guard_class(p2, ConstClass(node_vtable)) []
         jump(p0)
         """
         self.optimize_loop(ops, expected, expected_short=short)
-        
+
     def test_forced_virtual_pure_getfield(self):
         ops = """
         [p0]
@@ -6576,7 +6612,7 @@
         jump(p1, i2)
         """
         self.optimize_loop(ops, expected)
-        
+
     def test_loopinvariant_strlen(self):
         ops = """
         [p9]
@@ -6709,7 +6745,7 @@
         [p0, p1]
         p2 = new_with_vtable(ConstClass(node_vtable))
         p3 = new_with_vtable(ConstClass(node_vtable))
-        setfield_gc(p2, p3, descr=nextdescr) 
+        setfield_gc(p2, p3, descr=nextdescr)
         jump(p2, p3)
         """
         expected = """
@@ -6728,7 +6764,7 @@
         jump(p2, i2)
         """
         expected = """
-        [p1]        
+        [p1]
         p2 = getarrayitem_gc(p1, 7, descr=<GcPtrArrayDescr>)
         i1 = arraylen_gc(p1)
         jump(p2)
@@ -6769,8 +6805,8 @@
         jump(p0, p2, p1)
         """
         self.optimize_loop(ops, expected, expected_short=short)
-        
-        
+
+
     def test_loopinvariant_constant_strgetitem(self):
         ops = """
         [p0]
@@ -6824,11 +6860,11 @@
         expected = """
         [p0, i22, p1]
         call(i22, descr=nonwritedescr)
-        i3 = unicodelen(p1) # Should be killed by backend        
+        i3 = unicodelen(p1) # Should be killed by backend
         jump(p0, i22, p1)
         """
         self.optimize_loop(ops, expected, expected_short=short)
-        
+
     def test_propagate_virtual_arryalen(self):
         ops = """
         [p0]
@@ -6897,7 +6933,7 @@
         [p0, p1, p10, p11]
         i1 = arraylen_gc(p10, descr=arraydescr)
         getarrayitem_gc(p11, 1, descr=arraydescr)
-        call(i1, descr=nonwritedescr)        
+        call(i1, descr=nonwritedescr)
         jump(p1, p0, p11, p10)
         """
         self.optimize_loop(ops, expected)
@@ -6906,20 +6942,20 @@
         ops = """
         [p5]
         i10 = getfield_gc(p5, descr=valuedescr)
-        call(i10, descr=nonwritedescr) 
+        call(i10, descr=nonwritedescr)
         setfield_gc(p5, 1, descr=valuedescr)
         jump(p5)
         """
         preamble = """
         [p5]
         i10 = getfield_gc(p5, descr=valuedescr)
-        call(i10, descr=nonwritedescr) 
+        call(i10, descr=nonwritedescr)
         setfield_gc(p5, 1, descr=valuedescr)
         jump(p5)
         """
         expected = """
         [p5]
-        call(1, descr=nonwritedescr) 
+        call(1, descr=nonwritedescr)
         jump(p5)
         """
         self.optimize_loop(ops, expected, preamble)
@@ -6957,7 +6993,7 @@
         [p9]
         call_assembler(0, descr=asmdescr)
         i18 = getfield_gc(p9, descr=valuedescr)
-        guard_value(i18, 0) []        
+        guard_value(i18, 0) []
         jump(p9)
         """
         self.optimize_loop(ops, expected)
@@ -6986,17 +7022,37 @@
         i10 = getfield_gc(p5, descr=valuedescr)
         i11 = getfield_gc(p6, descr=nextdescr)
         call(i10, i11, descr=nonwritedescr)
-        setfield_gc(p6, i10, descr=nextdescr)        
+        setfield_gc(p6, i10, descr=nextdescr)
         jump(p5, p6)
         """
         expected = """
         [p5, p6, i10, i11]
         call(i10, i11, descr=nonwritedescr)
-        setfield_gc(p6, i10, descr=nextdescr)        
+        setfield_gc(p6, i10, descr=nextdescr)
         jump(p5, p6, i10, i10)
         """
         self.optimize_loop(ops, expected)
-        
+
+    def test_cached_pure_func_of_equal_fields(self):
+        ops = """
+        [p5, p6]
+        i10 = getfield_gc(p5, descr=valuedescr)
+        i11 = getfield_gc(p6, descr=nextdescr)
+        i12 = int_add(i10, 7)
+        i13 = int_add(i11, 7)
+        call(i12, i13, descr=nonwritedescr)
+        setfield_gc(p6, i10, descr=nextdescr)
+        jump(p5, p6)
+        """
+        expected = """
+        [p5, p6, i14, i12, i10]
+        i13 = int_add(i14, 7)
+        call(i12, i13, descr=nonwritedescr)
+        setfield_gc(p6, i10, descr=nextdescr)
+        jump(p5, p6, i10, i12, i10)
+        """
+        self.optimize_loop(ops, expected)
+
     def test_forced_counter(self):
         # XXX: VIRTUALHEAP (see above)
         py.test.skip("would be fixed by make heap optimizer aware of virtual setfields")
@@ -7086,8 +7142,84 @@
         """
         self.optimize_loop(ops, expected)
 
-        
+    def test_import_constants_when_folding_pure_operations(self):
+        ops = """
+        [p0]
+        f1 = getfield_gc(p0, descr=valuedescr)
+        f2 = float_abs(f1)
+        call(7.0, descr=nonwritedescr)
+        setfield_gc(p0, -7.0, descr=valuedescr)
+        jump(p0)
+        """
+        expected = """
+        [p0]
+        call(7.0, descr=nonwritedescr)
+        jump(p0)
+        """
+        self.optimize_loop(ops, expected)
+
+    def test_exploding_duplicatipon(self):
+        ops = """
+        [i1, i2]
+        i3 = int_add(i1, i1)
+        i4 = int_add(i3, i3)
+        i5 = int_add(i4, i4)
+        i6 = int_add(i5, i5)
+        call(i6, descr=nonwritedescr)
+        jump(i1, i3)
+        """
+        expected = """
+        [i1, i2, i6, i3]
+        call(i6, descr=nonwritedescr)
+        jump(i1, i3, i6, i3)
+        """
+        short = """
+        [i1, i2]
+        i3 = int_add(i1, i1)
+        i4 = int_add(i3, i3)
+        i5 = int_add(i4, i4)
+        i6 = int_add(i5, i5)
+        jump(i1, i2, i6, i3)
+        """
+        self.optimize_loop(ops, expected, expected_short=short)
+
+    def test_prioritize_getfield1(self):
+        ops = """
+        [p1, p2]
+        i1 = getfield_gc(p1, descr=valuedescr)
+        setfield_gc(p2, i1, descr=nextdescr)
+        i2 = int_neg(i1)
+        call(i2, descr=nonwritedescr)
+        jump(p1, p2)
+        """
+        expected = """
+        [p1, p2, i2, i1]
+        call(i2, descr=nonwritedescr)
+        setfield_gc(p2, i1, descr=nextdescr)
+        jump(p1, p2, i2, i1)
+        """
+        self.optimize_loop(ops, expected)
+
+    def test_prioritize_getfield2(self):
+        # Same as previous, but with descrs intercahnged which means
+        # that the getfield is discovered first when looking for
+        # potential short boxes during tests
+        ops = """
+        [p1, p2]
+        i1 = getfield_gc(p1, descr=nextdescr)
+        setfield_gc(p2, i1, descr=valuedescr)
+        i2 = int_neg(i1)
+        call(i2, descr=nonwritedescr)
+        jump(p1, p2)
+        """
+        expected = """
+        [p1, p2, i2, i1]
+        call(i2, descr=nonwritedescr)
+        setfield_gc(p2, i1, descr=valuedescr)
+        jump(p1, p2, i2, i1)
+        """
+        self.optimize_loop(ops, expected)
 
 class TestLLtype(OptimizeOptTest, LLtypeMixin):
     pass
-        
+
diff --git a/pypy/jit/metainterp/optimizeopt/unroll.py b/pypy/jit/metainterp/optimizeopt/unroll.py
--- a/pypy/jit/metainterp/optimizeopt/unroll.py
+++ b/pypy/jit/metainterp/optimizeopt/unroll.py
@@ -70,6 +70,47 @@
         self.snapshot_map[snapshot] = new_snapshot
         return new_snapshot
 
+class UnrollableOptimizer(Optimizer):
+    def setup(self):
+        self.importable_values = {}
+        self.emitting_dissabled = False
+        self.emitted_guards = 0
+        self.emitted_pure_operations = {}
+
+    def ensure_imported(self, value):
+        if not self.emitting_dissabled and value in self.importable_values:
+            imp = self.importable_values[value]
+            del self.importable_values[value]
+            imp.import_value(value)
+
+    def emit_operation(self, op):
+        if op.returns_bool_result():
+            self.bool_boxes[self.getvalue(op.result)] = None
+        if self.emitting_dissabled:
+            return
+        if op.is_guard():
+            self.emitted_guards += 1 # FIXME: can we use counter in self._emit_operation?
+        self._emit_operation(op)
+
+    def new(self):
+        new = UnrollableOptimizer(self.metainterp_sd, self.loop)
+        return self._new(new)
+
+    def remember_emitting_pure(self, op):
+        self.emitted_pure_operations[op] = True
+
+    def produce_potential_short_preamble_ops(self, sb):
+        for op in self.emitted_pure_operations:
+            if op.getopnum() == rop.GETARRAYITEM_GC_PURE or \
+               op.getopnum() == rop.STRGETITEM or \
+               op.getopnum() == rop.UNICODEGETITEM:
+                if not self.getvalue(op.getarg(1)).is_constant():
+                    continue
+            sb.add_potential(op)
+        for opt in self.optimizations:
+            opt.produce_potential_short_preamble_ops(sb)
+
+
 
 class UnrollOptimizer(Optimization):
     """Unroll the loop into two iterations. The first one will
@@ -77,7 +118,7 @@
     distinction anymore)"""
 
     def __init__(self, metainterp_sd, loop, optimizations):
-        self.optimizer = Optimizer(metainterp_sd, loop, optimizations)
+        self.optimizer = UnrollableOptimizer(metainterp_sd, loop, optimizations)
         self.cloned_operations = []
         for op in self.optimizer.loop.operations:
             newop = op.clone()
@@ -150,6 +191,7 @@
                 args = ", ".join([logops.repr_of_arg(arg) for arg in short_inputargs])
                 debug_print('short inputargs: ' + args)
                 self.short_boxes.debug_print(logops)
+                
 
             # Force virtuals amoung the jump_args of the preamble to get the
             # operations needed to setup the proper state of those virtuals
@@ -161,8 +203,9 @@
                 if box in seen:
                     continue
                 seen[box] = True
-                value = preamble_optimizer.getvalue(box)
-                inputarg_setup_ops.extend(value.make_guards(box))
+                preamble_value = preamble_optimizer.getvalue(box)
+                value = self.optimizer.getvalue(box)
+                value.import_from(preamble_value, self.optimizer)
             for box in short_inputargs:
                 if box in seen:
                     continue
@@ -181,23 +224,17 @@
             for op in self.short_boxes.operations():
                 self.ensure_short_op_emitted(op, self.optimizer, seen)
                 if op and op.result:
-                    # The order of these guards is not important as 
-                    # self.optimizer.emitting_dissabled is False
-                    value = preamble_optimizer.getvalue(op.result)
-                    for guard in value.make_guards(op.result):
-                        self.optimizer.send_extra_operation(guard)
+                    preamble_value = preamble_optimizer.getvalue(op.result)
+                    value = self.optimizer.getvalue(op.result)
+                    if not value.is_virtual():
+                        imp = ValueImporter(self, preamble_value, op)
+                        self.optimizer.importable_values[value] = imp
                     newresult = self.optimizer.getvalue(op.result).get_key_box()
                     if newresult is not op.result:
                         self.short_boxes.alias(newresult, op.result)
             self.optimizer.flush()
             self.optimizer.emitting_dissabled = False
 
-            # XXX Hack to prevent the arraylen/strlen/unicodelen ops generated
-            #     by value.make_guards() from ending up in pure_operations
-            for key, op in self.optimizer.pure_operations.items():
-                if not self.short_boxes.has_producer(op.result):
-                    del self.optimizer.pure_operations[key]
-
             initial_inputargs_len = len(inputargs)
             self.inliner = Inliner(loop.inputargs, jump_args)
 
@@ -276,16 +313,11 @@
 
         short_jumpargs = inputargs[:]
 
-        short = []
-        short_seen = {}
+        short = self.short = []
+        short_seen = self.short_seen = {}
         for box, const in self.constant_inputargs.items():
             short_seen[box] = True
 
-        for op in self.short_boxes.operations():
-            if op is not None:
-                if len(self.getvalue(op.result).make_guards(op.result)) > 0:
-                    self.add_op_to_short(op, short, short_seen, False, True)
-
         # This loop is equivalent to the main optimization loop in
         # Optimizer.propagate_all_forward
         jumpop = None
@@ -380,7 +412,7 @@
         if op.is_ovf():
             guard = ResOperation(rop.GUARD_NO_OVERFLOW, [], None)
             optimizer.send_extra_operation(guard)
-        
+
     def add_op_to_short(self, op, short, short_seen, emit=True, guards_needed=False):
         if op is None:
             return None
@@ -536,6 +568,13 @@
                         loop_token.failed_states.append(virtual_state)
         self.emit_operation(op)
 
+class ValueImporter(object):
+    def __init__(self, unroll, value, op):
+        self.unroll = unroll
+        self.preamble_value = value
+        self.op = op
 
-
-
+    def import_value(self, value):
+        value.import_from(self.preamble_value, self.unroll.optimizer)
+        self.unroll.add_op_to_short(self.op, self.unroll.short, self.unroll.short_seen, False, True)        
+        
diff --git a/pypy/jit/metainterp/optimizeopt/virtualize.py b/pypy/jit/metainterp/optimizeopt/virtualize.py
--- a/pypy/jit/metainterp/optimizeopt/virtualize.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualize.py
@@ -58,6 +58,9 @@
     def _really_force(self):
         raise NotImplementedError("abstract base")
 
+    def import_from(self, other, optimizer):
+        raise NotImplementedError("should not be called at this level")
+    
 def get_fielddescrlist_cache(cpu):
     if not hasattr(cpu, '_optimizeopt_fielddescrlist_cache'):
         result = descrlist_dict()
diff --git a/pypy/jit/metainterp/optimizeopt/virtualstate.py b/pypy/jit/metainterp/optimizeopt/virtualstate.py
--- a/pypy/jit/metainterp/optimizeopt/virtualstate.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualstate.py
@@ -12,6 +12,7 @@
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.rlib.debug import debug_start, debug_stop, debug_print
 from pypy.rlib.objectmodel import we_are_translated
+import os
 
 class AbstractVirtualStateInfo(resume.AbstractVirtualInfo):
     position = -1
@@ -461,8 +462,10 @@
 class ShortBoxes(object):
     def __init__(self, optimizer, surviving_boxes):
         self.potential_ops = {}
-        self.duplicates = {}
+        self.alternatives = {}
+        self.synthetic = {}
         self.aliases = {}
+        self.rename = {}
         self.optimizer = optimizer
         for box in surviving_boxes:
             self.potential_ops[box] = None
@@ -476,33 +479,81 @@
             except BoxNotProducable:
                 pass
 
+    def prioritized_alternatives(self, box):
+        if box not in self.alternatives:
+            return [self.potential_ops[box]]
+        alts = self.alternatives[box]
+        hi, lo = 0, len(alts) - 1
+        while hi < lo:
+            if alts[lo] is None: # Inputarg, lowest priority
+                alts[lo], alts[-1] = alts[-1], alts[lo]
+                lo -= 1
+            elif alts[lo] not in self.synthetic: # Hi priority
+                alts[hi], alts[lo] = alts[lo], alts[hi]
+                hi += 1
+            else: # Low priority
+                lo -= 1
+        return alts
+            
+    def renamed(self, box):
+        if box in self.rename:
+            return self.rename[box]
+        return box
+    
+    def add_to_short(self, box, op):
+        if op:
+            op = op.clone()
+            for i in range(op.numargs()):
+                op.setarg(i, self.renamed(op.getarg(i)))
+        if box in self.short_boxes:
+            if op is None:
+                oldop = self.short_boxes[box].clone()
+                oldres = oldop.result
+                newbox = oldop.result = oldres.clonebox()
+                self.rename[box] = newbox
+                self.short_boxes[box] = None
+                self.short_boxes[newbox] = oldop
+            else:
+                newop = op.clone()
+                newbox = newop.result = op.result.clonebox()
+                self.short_boxes[newop.result] = newop
+            value = self.optimizer.getvalue(box)
+            self.optimizer.make_equal_to(newbox, value)
+        else:
+            self.short_boxes[box] = op
+        
     def produce_short_preamble_box(self, box):
         if box in self.short_boxes:
             return 
         if isinstance(box, Const):
             return 
         if box in self.potential_ops:
-            op = self.potential_ops[box]
-            if op:
-                for arg in op.getarglist():
-                    self.produce_short_preamble_box(arg)
-            self.short_boxes[box] = op
+            ops = self.prioritized_alternatives(box)
+            produced_one = False
+            for op in ops:
+                try:
+                    if op:
+                        for arg in op.getarglist():
+                            self.produce_short_preamble_box(arg)
+                except BoxNotProducable:
+                    pass
+                else:
+                    produced_one = True
+                    self.add_to_short(box, op)
+            if not produced_one:
+                raise BoxNotProducable
         else:
             raise BoxNotProducable
 
-    def add_potential(self, op):
+    def add_potential(self, op, synthetic=False):
         if op.result not in self.potential_ops:
             self.potential_ops[op.result] = op
-            return op
-        newop = op.clone()
-        newop.result = op.result.clonebox()
-        self.potential_ops[newop.result] = newop
-        if op.result in self.duplicates:
-            self.duplicates[op.result].append(newop.result)
         else:
-            self.duplicates[op.result] = [newop.result]
-        self.optimizer.make_equal_to(newop.result, self.optimizer.getvalue(op.result))
-        return newop
+            if op.result not in self.alternatives:
+                self.alternatives[op.result] = [self.potential_ops[op.result]]
+            self.alternatives[op.result].append(op)
+        if synthetic:
+            self.synthetic[op] = True
 
     def debug_print(self, logops):
         debug_start('jit-short-boxes')
diff --git a/pypy/jit/metainterp/optimizeopt/vstring.py b/pypy/jit/metainterp/optimizeopt/vstring.py
--- a/pypy/jit/metainterp/optimizeopt/vstring.py
+++ b/pypy/jit/metainterp/optimizeopt/vstring.py
@@ -141,6 +141,11 @@
                                    for c in self._chars])
 
     def string_copy_parts(self, optimizer, targetbox, offsetbox, mode):
+        if not self.is_virtual() and targetbox is not self.box:
+            lengthbox = self.getstrlen(optimizer, mode)
+            srcbox = self.force_box()
+            return copy_str_content(optimizer, srcbox, targetbox,
+                                CONST_0, offsetbox, lengthbox, mode)
         for i in range(len(self._chars)):
             charbox = self._chars[i].force_box()
             if not (isinstance(charbox, Const) and charbox.same_constant(CONST_0)):
@@ -296,7 +301,7 @@
 
 
 def copy_str_content(optimizer, srcbox, targetbox,
-                     srcoffsetbox, offsetbox, lengthbox, mode):
+                     srcoffsetbox, offsetbox, lengthbox, mode, need_next_offset=True):
     if isinstance(srcbox, ConstPtr) and isinstance(srcoffsetbox, Const):
         M = 5
     else:
@@ -313,7 +318,10 @@
                                               None))
             offsetbox = _int_add(optimizer, offsetbox, CONST_1)
     else:
-        nextoffsetbox = _int_add(optimizer, offsetbox, lengthbox)
+        if need_next_offset:
+            nextoffsetbox = _int_add(optimizer, offsetbox, lengthbox)
+        else:
+            nextoffsetbox = None
         op = ResOperation(mode.COPYSTRCONTENT, [srcbox, targetbox,
                                                 srcoffsetbox, offsetbox,
                                                 lengthbox], None)
@@ -365,7 +373,7 @@
 
     def new(self):
         return OptString()
-    
+
     def make_vstring_plain(self, box, source_op, mode):
         vvalue = VStringPlainValue(self.optimizer, box, source_op, mode)
         self.make_equal_to(box, vvalue)
@@ -435,7 +443,11 @@
         #
         if isinstance(value, VStringPlainValue):  # even if no longer virtual
             if vindex.is_constant():
-                return value.getitem(vindex.box.getint())
+                res = value.getitem(vindex.box.getint())
+                # If it is uninitialized we can't return it, it was set by a
+                # COPYSTRCONTENT, not a STRSETITEM
+                if res is not optimizer.CVAL_UNINITIALIZED_ZERO:
+                    return res
         #
         resbox = _strgetitem(self.optimizer, value.force_box(), vindex.force_box(), mode)
         return self.getvalue(resbox)
@@ -450,6 +462,30 @@
         lengthbox = value.getstrlen(self.optimizer, mode)
         self.make_equal_to(op.result, self.getvalue(lengthbox))
 
+    def optimize_COPYSTRCONTENT(self, op):
+        self._optimize_COPYSTRCONTENT(op, mode_string)
+    def optimize_COPYUNICODECONTENT(self, op):
+        self._optimize_COPYSTRCONTENT(op, mode_unicode)
+
+    def _optimize_COPYSTRCONTENT(self, op, mode):
+        # args: src dst srcstart dststart length
+        src = self.getvalue(op.getarg(0))
+        dst = self.getvalue(op.getarg(1))
+        srcstart = self.getvalue(op.getarg(2))
+        dststart = self.getvalue(op.getarg(3))
+        length = self.getvalue(op.getarg(4))
+
+        if length.is_constant() and length.box.getint() == 0:
+            return
+        copy_str_content(self.optimizer,
+            src.force_box(),
+            dst.force_box(),
+            srcstart.force_box(),
+            dststart.force_box(),
+            length.force_box(),
+            mode, need_next_offset=False
+        )
+
     def optimize_CALL(self, op):
         # dispatch based on 'oopspecindex' to a method that handles
         # specifically the given oopspec call.  For non-oopspec calls,
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -17,6 +17,7 @@
 from pypy.jit.metainterp.jitprof import ABORT_TOO_LONG, ABORT_BRIDGE, \
                                         ABORT_FORCE_QUASIIMMUT, ABORT_BAD_LOOP
 from pypy.jit.metainterp.jitexc import JitException, get_llexception
+from pypy.jit.metainterp.heapcache import HeapCache
 from pypy.rlib.objectmodel import specialize
 from pypy.jit.codewriter.jitcode import JitCode, SwitchDictDescr
 from pypy.jit.codewriter import heaptracker
@@ -210,7 +211,8 @@
                 self.metainterp.clear_exception()
                 resbox = self.execute(rop.%s, b1, b2)
                 self.make_result_of_lastop(resbox)  # same as execute_varargs()
-                self.metainterp.handle_possible_overflow_error()
+                if not isinstance(resbox, Const):
+                    self.metainterp.handle_possible_overflow_error()
                 return resbox
         ''' % (_opimpl, _opimpl.upper())).compile()
 
@@ -322,7 +324,7 @@
     def _establish_nullity(self, box, orgpc):
         value = box.nonnull()
         if value:
-            if box not in self.metainterp.known_class_boxes:
+            if not self.metainterp.heapcache.is_class_known(box):
                 self.generate_guard(rop.GUARD_NONNULL, box, resumepc=orgpc)
         else:
             if not isinstance(box, Const):
@@ -367,14 +369,17 @@
 
     @arguments("descr")
     def opimpl_new(self, sizedescr):
-        return self.execute_with_descr(rop.NEW, sizedescr)
+        resbox = self.execute_with_descr(rop.NEW, sizedescr)
+        self.metainterp.heapcache.new(resbox)
+        return resbox
 
     @arguments("descr")
     def opimpl_new_with_vtable(self, sizedescr):
         cpu = self.metainterp.cpu
         cls = heaptracker.descr2vtable(cpu, sizedescr)
         resbox = self.execute(rop.NEW_WITH_VTABLE, ConstInt(cls))
-        self.metainterp.known_class_boxes[resbox] = None
+        self.metainterp.heapcache.new(resbox)
+        self.metainterp.heapcache.class_now_known(resbox)
         return resbox
 
 ##    @FixME  #arguments("box")
@@ -393,26 +398,30 @@
 ##        self.execute(rop.SUBCLASSOF, box1, box2)
 
     @arguments("descr", "box")
-    def opimpl_new_array(self, itemsizedescr, countbox):
-        return self.execute_with_descr(rop.NEW_ARRAY, itemsizedescr, countbox)
+    def opimpl_new_array(self, itemsizedescr, lengthbox):
+        resbox = self.execute_with_descr(rop.NEW_ARRAY, itemsizedescr, lengthbox)
+        self.metainterp.heapcache.new_array(resbox, lengthbox)
+        return resbox
+
+    @specialize.arg(1)
+    def _do_getarrayitem_gc_any(self, op, arraybox, arraydescr, indexbox):
+        tobox = self.metainterp.heapcache.getarrayitem(
+                arraybox, arraydescr, indexbox)
+        if tobox:
+            # sanity check: see whether the current array value
+            # corresponds to what the cache thinks the value is
+            resbox = executor.execute(self.metainterp.cpu, self.metainterp, op,
+                                      arraydescr, arraybox, indexbox)
+            assert resbox.constbox().same_constant(tobox.constbox())
+            return tobox
+        resbox = self.execute_with_descr(op, arraydescr, arraybox, indexbox)
+        self.metainterp.heapcache.getarrayitem_now_known(
+                arraybox, arraydescr, indexbox, resbox)
+        return resbox
 
     @arguments("box", "descr", "box")
     def _opimpl_getarrayitem_gc_any(self, arraybox, arraydescr, indexbox):
-        cache = self.metainterp.heap_array_cache.get(arraydescr, None)
-        if cache and isinstance(indexbox, ConstInt):
-            index = indexbox.getint()
-            frombox, tobox = cache.get(index, (None, None))
-            if frombox is arraybox:
-                return tobox
-        resbox = self.execute_with_descr(rop.GETARRAYITEM_GC,
-                                         arraydescr, arraybox, indexbox)
-        if isinstance(indexbox, ConstInt):
-            if not cache:
-                cache = self.metainterp.heap_array_cache[arraydescr] = {}
-            index = indexbox.getint()
-            cache[index] = arraybox, resbox
-        return resbox
-
+        return self._do_getarrayitem_gc_any(rop.GETARRAYITEM_GC, arraybox, arraydescr, indexbox)
 
     opimpl_getarrayitem_gc_i = _opimpl_getarrayitem_gc_any
     opimpl_getarrayitem_gc_r = _opimpl_getarrayitem_gc_any
@@ -428,8 +437,7 @@
 
     @arguments("box", "descr", "box")
     def _opimpl_getarrayitem_gc_pure_any(self, arraybox, arraydescr, indexbox):
-        return self.execute_with_descr(rop.GETARRAYITEM_GC_PURE,
-                                       arraydescr, arraybox, indexbox)
+        return self._do_getarrayitem_gc_any(rop.GETARRAYITEM_GC_PURE, arraybox, arraydescr, indexbox)
 
     opimpl_getarrayitem_gc_pure_i = _opimpl_getarrayitem_gc_pure_any
     opimpl_getarrayitem_gc_pure_r = _opimpl_getarrayitem_gc_pure_any
@@ -440,13 +448,8 @@
                                     indexbox, itembox):
         self.execute_with_descr(rop.SETARRAYITEM_GC, arraydescr, arraybox,
                                 indexbox, itembox)
-        if isinstance(indexbox, ConstInt):
-            cache = self.metainterp.heap_array_cache.setdefault(arraydescr, {})
-            cache[indexbox.getint()] = arraybox, itembox
-        else:
-            cache = self.metainterp.heap_array_cache.get(arraydescr, None)
-            if cache:
-                cache.clear()
+        self.metainterp.heapcache.setarrayitem(
+                arraybox, arraydescr, indexbox, itembox)
 
     opimpl_setarrayitem_gc_i = _opimpl_setarrayitem_gc_any
     opimpl_setarrayitem_gc_r = _opimpl_setarrayitem_gc_any
@@ -463,7 +466,12 @@
 
     @arguments("box", "descr")
     def opimpl_arraylen_gc(self, arraybox, arraydescr):
-        return self.execute_with_descr(rop.ARRAYLEN_GC, arraydescr, arraybox)
+        lengthbox = self.metainterp.heapcache.arraylen(arraybox)
+        if lengthbox is None:
+            lengthbox = self.execute_with_descr(
+                    rop.ARRAYLEN_GC, arraydescr, arraybox)
+            self.metainterp.heapcache.arraylen_now_known(arraybox, lengthbox)
+        return lengthbox
 
     @arguments("orgpc", "box", "descr", "box")
     def opimpl_check_neg_index(self, orgpc, arraybox, arraydescr, indexbox):
@@ -472,19 +480,17 @@
         negbox = self.implement_guard_value(orgpc, negbox)
         if negbox.getint():
             # the index is < 0; add the array length to it
-            lenbox = self.metainterp.execute_and_record(
-                rop.ARRAYLEN_GC, arraydescr, arraybox)
+            lengthbox = self.opimpl_arraylen_gc(arraybox, arraydescr)
             indexbox = self.metainterp.execute_and_record(
-                rop.INT_ADD, None, indexbox, lenbox)
+                rop.INT_ADD, None, indexbox, lengthbox)
         return indexbox
 
     @arguments("descr", "descr", "descr", "descr", "box")
     def opimpl_newlist(self, structdescr, lengthdescr, itemsdescr, arraydescr,
                        sizebox):
-        sbox = self.metainterp.execute_and_record(rop.NEW, structdescr)
+        sbox = self.opimpl_new(structdescr)
         self._opimpl_setfield_gc_any(sbox, lengthdescr, sizebox)
-        abox = self.metainterp.execute_and_record(rop.NEW_ARRAY, arraydescr,
-                                                  sizebox)
+        abox = self.opimpl_new_array(arraydescr, sizebox)
         self._opimpl_setfield_gc_any(sbox, itemsdescr, abox)
         return sbox
 
@@ -549,11 +555,15 @@
 
     @specialize.arg(1)
     def _opimpl_getfield_gc_any_pureornot(self, opnum, box, fielddescr):
-        frombox, tobox = self.metainterp.heap_cache.get(fielddescr, (None, None))
-        if frombox is box:
+        tobox = self.metainterp.heapcache.getfield(box, fielddescr)
+        if tobox is not None:
+            # sanity check: see whether the current struct value
+            # corresponds to what the cache thinks the value is
+            resbox = executor.execute(self.metainterp.cpu, self.metainterp,
+                                      rop.GETFIELD_GC, fielddescr, box)
             return tobox
         resbox = self.execute_with_descr(opnum, fielddescr, box)
-        self.metainterp.heap_cache[fielddescr] = (box, resbox)
+        self.metainterp.heapcache.getfield_now_known(box, fielddescr, resbox)
         return resbox
 
     @arguments("orgpc", "box", "descr")
@@ -574,11 +584,11 @@
 
     @arguments("box", "descr", "box")
     def _opimpl_setfield_gc_any(self, box, fielddescr, valuebox):
-        frombox, tobox = self.metainterp.heap_cache.get(fielddescr, (None, None))
-        if frombox is box and tobox is valuebox:
+        tobox = self.metainterp.heapcache.getfield(box, fielddescr)
+        if tobox is valuebox:
             return
         self.execute_with_descr(rop.SETFIELD_GC, fielddescr, box, valuebox)
-        self.metainterp.heap_cache[fielddescr] = (box, valuebox)
+        self.metainterp.heapcache.setfield(box, fielddescr, valuebox)
     opimpl_setfield_gc_i = _opimpl_setfield_gc_any
     opimpl_setfield_gc_r = _opimpl_setfield_gc_any
     opimpl_setfield_gc_f = _opimpl_setfield_gc_any
@@ -651,7 +661,7 @@
         standard_box = self.metainterp.virtualizable_boxes[-1]
         if standard_box is box:
             return False
-        if box in self.metainterp.nonstandard_virtualizables:
+        if self.metainterp.heapcache.is_nonstandard_virtualizable(box):
             return True
         eqbox = self.metainterp.execute_and_record(rop.PTR_EQ, None,
                                                    box, standard_box)
@@ -660,7 +670,7 @@
         if isstandard:
             self.metainterp.replace_box(box, standard_box)
         else:
-            self.metainterp.nonstandard_virtualizables[box] = None
+            self.metainterp.heapcache.nonstandard_virtualizables_now_known(box)
         return not isstandard
 
     def _get_virtualizable_field_index(self, fielddescr):
@@ -745,7 +755,7 @@
     def opimpl_arraylen_vable(self, pc, box, fdescr, adescr):
         if self._nonstandard_virtualizable(pc, box):
             arraybox = self._opimpl_getfield_gc_any(box, fdescr)
-            return self.execute_with_descr(rop.ARRAYLEN_GC, adescr, arraybox)
+            return self.opimpl_arraylen_gc(arraybox, adescr)
         vinfo = self.metainterp.jitdriver_sd.virtualizable_info
         virtualizable_box = self.metainterp.virtualizable_boxes[-1]
         virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box)
@@ -876,6 +886,14 @@
     def opimpl_newunicode(self, lengthbox):
         return self.execute(rop.NEWUNICODE, lengthbox)
 
+    @arguments("box", "box", "box", "box", "box")
+    def opimpl_copystrcontent(self, srcbox, dstbox, srcstartbox, dststartbox, lengthbox):
+        return self.execute(rop.COPYSTRCONTENT, srcbox, dstbox, srcstartbox, dststartbox, lengthbox)
+
+    @arguments("box", "box", "box", "box", "box")
+    def opimpl_copyunicodecontent(self, srcbox, dstbox, srcstartbox, dststartbox, lengthbox):
+        return self.execute(rop.COPYUNICODECONTENT, srcbox, dstbox, srcstartbox, dststartbox, lengthbox)
+
 ##    @FixME  #arguments("descr", "varargs")
 ##    def opimpl_residual_oosend_canraise(self, methdescr, varargs):
 ##        return self.execute_varargs(rop.OOSEND, varargs, descr=methdescr,
@@ -902,9 +920,9 @@
     @arguments("orgpc", "box")
     def opimpl_guard_class(self, orgpc, box):
         clsbox = self.cls_of_box(box)
-        if box not in self.metainterp.known_class_boxes:
+        if not self.metainterp.heapcache.is_class_known(box):
             self.generate_guard(rop.GUARD_CLASS, box, [clsbox], resumepc=orgpc)
-            self.metainterp.known_class_boxes[box] = None
+            self.metainterp.heapcache.class_now_known(box)
         return clsbox
 
     @arguments("int", "orgpc")
@@ -1070,6 +1088,18 @@
         return ConstInt(trace_length)
 
     @arguments("box")
+    def _opimpl_isconstant(self, box):
+        return ConstInt(isinstance(box, Const))
+
+    opimpl_int_isconstant = opimpl_ref_isconstant = _opimpl_isconstant
+
+    @arguments("box")
+    def _opimpl_isvirtual(self, box):
+        return ConstInt(self.metainterp.heapcache.is_unescaped(box))
+
+    opimpl_ref_isvirtual = _opimpl_isvirtual
+
+    @arguments("box")
     def opimpl_virtual_ref(self, box):
         # Details on the content of metainterp.virtualref_boxes:
         #
@@ -1510,16 +1540,7 @@
         self.last_exc_value_box = None
         self.retracing_loop_from = None
         self.call_pure_results = args_dict_box()
-        # contains boxes where the class is already known
-        self.known_class_boxes = {}
-        # contains frame boxes that are not virtualizables
-        self.nonstandard_virtualizables = {}
-        # heap cache
-        # maps descrs to (from_box, to_box) tuples
-        self.heap_cache = {}
-        # heap array cache
-        # maps descrs to {index: (from_box, to_box)} dicts
-        self.heap_array_cache = {}
+        self.heapcache = HeapCache()
 
     def perform_call(self, jitcode, boxes, greenkey=None):
         # causes the metainterp to enter the given subfunction
@@ -1692,32 +1713,18 @@
 
     def _record_helper_nonpure_varargs(self, opnum, resbox, descr, argboxes):
         assert resbox is None or isinstance(resbox, Box)
+        if (rop._OVF_FIRST <= opnum <= rop._OVF_LAST and
+            self.last_exc_value_box is None and
+            self._all_constants_varargs(argboxes)):
+            return resbox.constbox()
         # record the operation
         profiler = self.staticdata.profiler
         profiler.count_ops(opnum, RECORDED_OPS)
-        self._invalidate_caches(opnum, descr)
+        self.heapcache.invalidate_caches(opnum, descr, argboxes)
         op = self.history.record(opnum, argboxes, resbox, descr)
         self.attach_debug_info(op)
         return resbox
 
-    def _invalidate_caches(self, opnum, descr):
-        if opnum == rop.SETFIELD_GC:
-            return
-        if opnum == rop.SETARRAYITEM_GC:
-            return
-        if rop._NOSIDEEFFECT_FIRST <= opnum <= rop._NOSIDEEFFECT_LAST:
-            return
-        if opnum == rop.CALL:
-            effectinfo = descr.get_extra_info()
-            ef = effectinfo.extraeffect
-            if ef == effectinfo.EF_LOOPINVARIANT or \
-               ef == effectinfo.EF_ELIDABLE_CANNOT_RAISE or \
-               ef == effectinfo.EF_ELIDABLE_CAN_RAISE:
-                return
-        if self.heap_cache:
-            self.heap_cache.clear()
-        if self.heap_array_cache:
-            self.heap_array_cache.clear()
 
     def attach_debug_info(self, op):
         if (not we_are_translated() and op is not None
@@ -1880,10 +1887,7 @@
                 duplicates[box] = None
 
     def reached_loop_header(self, greenboxes, redboxes, resumedescr):
-        self.known_class_boxes = {}
-        self.nonstandard_virtualizables = {} # XXX maybe not needed?
-        self.heap_cache = {}
-        self.heap_array_cache = {}
+        self.heapcache.reset()
 
         duplicates = {}
         self.remove_consts_and_duplicates(redboxes, len(redboxes),
@@ -2391,17 +2395,7 @@
             for i in range(len(boxes)):
                 if boxes[i] is oldbox:
                     boxes[i] = newbox
-        for descr, (frombox, tobox) in self.heap_cache.iteritems():
-            change = False
-            if frombox is oldbox:
-                change = True
-                frombox = newbox
-            if tobox is oldbox:
-                change = True
-                tobox = newbox
-            if change:
-                self.heap_cache[descr] = frombox, tobox
-        # XXX what about self.heap_array_cache?
+        self.heapcache.replace_box(oldbox, newbox)
 
     def find_biggest_function(self):
         start_stack = []
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -1,23 +1,25 @@
+import sys
+
 import py
-import sys
-from pypy.rlib.jit import JitDriver, we_are_jitted, hint, dont_look_inside
-from pypy.rlib.jit import loop_invariant, elidable, promote
-from pypy.rlib.jit import jit_debug, assert_green, AssertGreenFailed
-from pypy.rlib.jit import unroll_safe, current_trace_length
+
+from pypy import conftest
+from pypy.jit.codewriter.policy import JitPolicy, StopAtXPolicy
 from pypy.jit.metainterp import pyjitpl, history
+from pypy.jit.metainterp.optimizeopt import ALL_OPTS_DICT
+from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin, noConst
+from pypy.jit.metainterp.typesystem import LLTypeHelper, OOTypeHelper
+from pypy.jit.metainterp.warmspot import get_stats
 from pypy.jit.metainterp.warmstate import set_future_value
-from pypy.jit.metainterp.warmspot import get_stats
-from pypy.jit.codewriter.policy import JitPolicy, StopAtXPolicy
-from pypy import conftest
+from pypy.rlib.jit import (JitDriver, we_are_jitted, hint, dont_look_inside,
+    loop_invariant, elidable, promote, jit_debug, assert_green,
+    AssertGreenFailed, unroll_safe, current_trace_length, look_inside_iff,
+    isconstant, isvirtual)
 from pypy.rlib.rarithmetic import ovfcheck
-from pypy.jit.metainterp.typesystem import LLTypeHelper, OOTypeHelper
 from pypy.rpython.lltypesystem import lltype, llmemory, rffi
 from pypy.rpython.ootypesystem import ootype
-from pypy.jit.metainterp.optimizeopt import ALL_OPTS_DICT
-from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin, noConst
+
 
 class BasicTests:
-
     def test_basic(self):
         def f(x, y):
             return x + y
@@ -99,14 +101,14 @@
                 myjitdriver.jit_merge_point(x=x, y=y, res=res)
                 res += x * x
                 x += 1
-                res += x * x                
+                res += x * x
                 y -= 1
             return res
         res = self.meta_interp(f, [6, 7])
         assert res == 1323
         self.check_loop_count(1)
         self.check_loops(int_mul=1)
-        
+
     def test_loop_variant_mul_ovf(self):
         myjitdriver = JitDriver(greens = [], reds = ['y', 'res', 'x'])
         def f(x, y):
@@ -1372,7 +1374,7 @@
             return x
         res = self.meta_interp(f, [299], listops=True)
         assert res == f(299)
-        self.check_loops(guard_class=0, guard_value=3)        
+        self.check_loops(guard_class=0, guard_value=3)
         self.check_loops(guard_class=0, guard_value=6, everywhere=True)
 
     def test_merge_guardnonnull_guardclass(self):
@@ -2118,7 +2120,7 @@
             return sa
         res = self.meta_interp(f, [32, 7])
         assert res == f(32, 7)
-        
+
     def test_caching_setarrayitem_fixed(self):
         myjitdriver = JitDriver(greens = [], reds = ['sa', 'i', 'n', 'a', 'node'])
         def f(n, a):
@@ -2138,7 +2140,7 @@
             return sa
         res = self.meta_interp(f, [32, 7])
         assert res == f(32, 7)
-        
+
     def test_caching_setarrayitem_var(self):
         myjitdriver = JitDriver(greens = [], reds = ['sa', 'i', 'n', 'a', 'b', 'node'])
         def f(n, a, b):
@@ -2668,7 +2670,7 @@
             myjitdriver.set_param('threshold', 3)
             myjitdriver.set_param('trace_eagerness', 1)
             myjitdriver.set_param('retrace_limit', 5)
-            myjitdriver.set_param('function_threshold', -1)            
+            myjitdriver.set_param('function_threshold', -1)
             pc = sa = i = 0
             while pc < len(bytecode):
                 myjitdriver.jit_merge_point(pc=pc, n=n, sa=sa, i=i)
@@ -2693,12 +2695,12 @@
         def g(n1, n2):
             for i in range(10):
                 f(n1)
-            for i in range(10):                
+            for i in range(10):
                 f(n2)
 
         nn = [10, 3]
         assert self.meta_interp(g, nn) == g(*nn)
-        
+
         # The attempts of retracing first loop will end up retracing the
         # second and thus fail 5 times, saturating the retrace_count. Instead a
         # bridge back to the preamble of the first loop is produced. A guard in
@@ -2709,7 +2711,7 @@
         self.check_tree_loop_count(2 + 3)
 
         # FIXME: Add a gloabl retrace counter and test that we are not trying more than 5 times.
-        
+
         def g(n):
             for i in range(n):
                 for j in range(10):
@@ -2945,15 +2947,15 @@
             a = [0, 1, 2, 3, 4]
             while i < n:
                 myjitdriver.jit_merge_point(sa=sa, n=n, a=a, i=i)
-                if i < n/2:
+                if i < n / 2:
                     sa += a[4]
-                elif i == n/2:
+                elif i == n / 2:
                     a.pop()
                 i += 1
         res = self.meta_interp(f, [32])
         assert res == f(32)
         self.check_loops(arraylen_gc=2)
-        
+
 class TestOOtype(BasicTests, OOJitMixin):
 
     def test_oohash(self):
@@ -3173,7 +3175,7 @@
         res = self.meta_interp(f, [32])
         assert res == f(32)
         self.check_tree_loop_count(3)
-        
+
     def test_two_loopinvariant_arrays3(self):
         from pypy.rpython.lltypesystem import lltype, llmemory, rffi
         myjitdriver = JitDriver(greens = [], reds = ['sa', 'n', 'i', 'a'])
@@ -3197,7 +3199,7 @@
         res = self.meta_interp(f, [32])
         assert res == f(32)
         self.check_tree_loop_count(2)
-        
+
     def test_two_loopinvariant_arrays_boxed(self):
         class A(object):
             def __init__(self, a):
@@ -3222,7 +3224,7 @@
         res = self.meta_interp(f, [32])
         assert res == f(32)
         self.check_loops(arraylen_gc=2, everywhere=True)
-        
+
     def test_release_gil_flush_heap_cache(self):
         if sys.platform == "win32":
             py.test.skip("needs 'time'")
@@ -3276,7 +3278,136 @@
             return n
 
         self.meta_interp(f, [10], repeat=3)
-        
+
+    def test_jit_merge_point_with_pbc(self):
+        driver = JitDriver(greens = [], reds = ['x'])
+
+        class A(object):
+            def __init__(self, x):
+                self.x = x
+            def _freeze_(self):
+                return True
+        pbc = A(1)
+
+        def main(x):
+            return f(x, pbc)
+
+        def f(x, pbc):
+            while x > 0:
+                driver.jit_merge_point(x = x)
+                x -= pbc.x
+            return x
+
+        self.meta_interp(main, [10])
+
+    def test_look_inside_iff_const(self):
+        @look_inside_iff(lambda arg: isconstant(arg))
+        def f(arg):
+            s = 0
+            while arg > 0:
+                s += arg
+                arg -= 1
+            return s
+
+        driver = JitDriver(greens = ['code'], reds = ['n', 'arg', 's'])
+
+        def main(code, n, arg):
+            s = 0
+            while n > 0:
+                driver.jit_merge_point(code=code, n=n, arg=arg, s=s)
+                if code == 0:
+                    s += f(arg)
+                else:
+                    s += f(1)
+                n -= 1
+            return s
+
+        res = self.meta_interp(main, [0, 10, 2], enable_opts='')
+        assert res == main(0, 10, 2)
+        self.check_loops(call=1)
+        res = self.meta_interp(main, [1, 10, 2], enable_opts='')
+        assert res == main(1, 10, 2)
+        self.check_loops(call=0)
+
+    def test_look_inside_iff_virtual(self):
+        # There's no good reason for this to be look_inside_iff, but it's a test!
+        @look_inside_iff(lambda arg, n: isvirtual(arg))
+        def f(arg, n):
+            if n == 100:
+                for i in xrange(n):
+                    n += i
+            return arg.x
+        class A(object):
+            def __init__(self, x):
+                self.x = x
+        driver = JitDriver(greens=['n'], reds=['i', 'a'])
+        def main(n):
+            i = 0
+            a = A(3)
+            while i < 20:
+                driver.jit_merge_point(i=i, n=n, a=a)
+                if n == 0:
+                    i += f(a, n)
+                else:
+                    i += f(A(2), n)
+        res = self.meta_interp(main, [0], enable_opts='')
+        assert res == main(0)
+        self.check_loops(call=1, getfield_gc=0)
+        res = self.meta_interp(main, [1], enable_opts='')
+        assert res == main(1)
+        self.check_loops(call=0, getfield_gc=0)
+
+    def test_reuse_elidable_result(self):
+        driver = JitDriver(reds=['n', 's'], greens = [])
+        def main(n):
+            s = 0
+            while n > 0:
+                driver.jit_merge_point(s=s, n=n)
+                s += len(str(n)) + len(str(n))
+                n -= 1
+            return s
+        res = self.meta_interp(main, [10])
+        assert res == main(10)
+        self.check_loops({
+            'call': 1, 'guard_no_exception': 1, 'guard_true': 1, 'int_add': 2,
+            'int_gt': 1, 'int_sub': 1, 'strlen': 1, 'jump': 1,
+        })
+
+    def test_look_inside_iff_const_getarrayitem_gc_pure(self):
+        driver = JitDriver(greens=['unroll'], reds=['s', 'n'])
+
+        class A(object):
+            _immutable_fields_ = ["x[*]"]
+            def __init__(self, x):
+                self.x = [x]
+
+        @look_inside_iff(lambda x: isconstant(x))
+        def f(x):
+            i = 0
+            for c in x:
+                i += 1
+            return i
+
+        def main(unroll, n):
+            s = 0
+            while n > 0:
+                driver.jit_merge_point(s=s, n=n, unroll=unroll)
+                if unroll:
+                    x = A("xx")
+                else:
+                    x = A("x" * n)
+                s += f(x.x[0])
+                n -= 1
+            return s
+
+        res = self.meta_interp(main, [0, 10])
+        assert res == main(0, 10)
+        # 2 calls, one for f() and one for char_mul
+        self.check_loops(call=2)
+        res = self.meta_interp(main, [1, 10])
+        assert res == main(1, 10)
+        self.check_loops(call=0)
+
 
 class TestLLtype(BaseLLtypeTests, LLJitMixin):
     pass
diff --git a/pypy/jit/metainterp/test/test_dict.py b/pypy/jit/metainterp/test/test_dict.py
--- a/pypy/jit/metainterp/test/test_dict.py
+++ b/pypy/jit/metainterp/test/test_dict.py
@@ -153,7 +153,10 @@
 
         res = self.meta_interp(f, [100], listops=True)
         assert res == f(50)
-        self.check_loops(call=4)
+        self.check_loops({"call": 7, "guard_false": 1, "guard_no_exception": 6,
+                          "guard_true": 1, "int_and": 1, "int_gt": 1,
+                          "int_is_true": 1, "int_sub": 1, "jump": 1,
+                          "new_with_vtable": 1, "setfield_gc": 1})
 
 
 class TestOOtype(DictTests, OOJitMixin):
diff --git a/pypy/jit/metainterp/test/test_heapcache.py b/pypy/jit/metainterp/test/test_heapcache.py
new file mode 100644
--- /dev/null
+++ b/pypy/jit/metainterp/test/test_heapcache.py
@@ -0,0 +1,365 @@
+from pypy.jit.metainterp.heapcache import HeapCache
+from pypy.jit.metainterp.resoperation import rop
+from pypy.jit.metainterp.history import ConstInt
+
+box1 = object()
+box2 = object()
+box3 = object()
+box4 = object()
+lengthbox1 = object()
+lengthbox2 = object()
+descr1 = object()
+descr2 = object()
+descr3 = object()
+
+index1 = ConstInt(0)
+index2 = ConstInt(1)
+
+
+class FakeEffektinfo(object):
+    EF_ELIDABLE_CANNOT_RAISE           = 0 #elidable function (and cannot raise)
+    EF_LOOPINVARIANT                   = 1 #special: call it only once per loop
+    EF_CANNOT_RAISE                    = 2 #a function which cannot raise
+    EF_ELIDABLE_CAN_RAISE              = 3 #elidable function (but can raise)
+    EF_CAN_RAISE                       = 4 #normal function (can raise)
+    EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE = 5 #can raise and force virtualizables
+    EF_RANDOM_EFFECTS                  = 6 #can do whatever
+
+    OS_ARRAYCOPY = 0
+
+    def __init__(self, extraeffect, oopspecindex):
+        self.extraeffect = extraeffect
+        self.oopspecindex = oopspecindex
+
+class FakeCallDescr(object):
+    def __init__(self, extraeffect, oopspecindex=None):
+        self.extraeffect = extraeffect
+        self.oopspecindex = oopspecindex
+
+    def get_extra_info(self):
+        return FakeEffektinfo(self.extraeffect, self.oopspecindex)
+
+class TestHeapCache(object):
+    def test_known_class_box(self):
+        h = HeapCache()
+        assert not h.is_class_known(1)
+        assert not h.is_class_known(2)
+        h.class_now_known(1)
+        assert h.is_class_known(1)
+        assert not h.is_class_known(2)
+
+        h.reset()
+        assert not h.is_class_known(1)
+        assert not h.is_class_known(2)
+
+    def test_nonstandard_virtualizable(self):
+        h = HeapCache()
+        assert not h.is_nonstandard_virtualizable(1)
+        assert not h.is_nonstandard_virtualizable(2)
+        h.nonstandard_virtualizables_now_known(1)
+        assert h.is_nonstandard_virtualizable(1)
+        assert not h.is_nonstandard_virtualizable(2)
+
+        h.reset()
+        assert not h.is_nonstandard_virtualizable(1)
+        assert not h.is_nonstandard_virtualizable(2)
+
+
+    def test_heapcache_fields(self):
+        h = HeapCache()
+        assert h.getfield(box1, descr1) is None
+        assert h.getfield(box1, descr2) is None
+        h.setfield(box1, descr1, box2)
+        assert h.getfield(box1, descr1) is box2
+        assert h.getfield(box1, descr2) is None
+        h.setfield(box1, descr2, box3)
+        assert h.getfield(box1, descr1) is box2
+        assert h.getfield(box1, descr2) is box3
+        h.setfield(box1, descr1, box3)
+        assert h.getfield(box1, descr1) is box3
+        assert h.getfield(box1, descr2) is box3
+        h.setfield(box3, descr1, box1)
+        assert h.getfield(box3, descr1) is box1
+        assert h.getfield(box1, descr1) is None
+        assert h.getfield(box1, descr2) is box3
+
+        h.reset()
+        assert h.getfield(box1, descr1) is None
+        assert h.getfield(box1, descr2) is None
+        assert h.getfield(box3, descr1) is None
+
+    def test_heapcache_read_fields_multiple(self):
+        h = HeapCache()
+        h.getfield_now_known(box1, descr1, box2)
+        h.getfield_now_known(box3, descr1, box4)
+        assert h.getfield(box1, descr1) is box2
+        assert h.getfield(box1, descr2) is None
+        assert h.getfield(box3, descr1) is box4
+        assert h.getfield(box3, descr2) is None
+
+        h.reset()
+        assert h.getfield(box1, descr1) is None
+        assert h.getfield(box1, descr2) is None
+        assert h.getfield(box3, descr1) is None
+        assert h.getfield(box3, descr2) is None
+
+    def test_heapcache_write_fields_multiple(self):
+        h = HeapCache()
+        h.setfield(box1, descr1, box2)
+        assert h.getfield(box1, descr1) is box2
+        h.setfield(box3, descr1, box4)
+        assert h.getfield(box3, descr1) is box4
+        assert h.getfield(box1, descr1) is None # box1 and box3 can alias
+
+        h = HeapCache()
+        h.new(box1)
+        h.setfield(box1, descr1, box2)
+        assert h.getfield(box1, descr1) is box2
+        h.setfield(box3, descr1, box4)
+        assert h.getfield(box3, descr1) is box4
+        assert h.getfield(box1, descr1) is None # box1 and box3 can alias
+
+        h = HeapCache()
+        h.new(box1)
+        h.new(box3)
+        h.setfield(box1, descr1, box2)
+        assert h.getfield(box1, descr1) is box2
+        h.setfield(box3, descr1, box4)
+        assert h.getfield(box3, descr1) is box4
+        assert h.getfield(box1, descr1) is box2 # box1 and box3 cannot alias
+        h.setfield(box1, descr1, box3)
+        assert h.getfield(box1, descr1) is box3
+
+
+    def test_heapcache_arrays(self):
+        h = HeapCache()
+        assert h.getarrayitem(box1, descr1, index1) is None
+        assert h.getarrayitem(box1, descr2, index1) is None
+        assert h.getarrayitem(box1, descr1, index2) is None
+        assert h.getarrayitem(box1, descr2, index2) is None
+
+        h.setarrayitem(box1, descr1, index1, box2)
+        assert h.getarrayitem(box1, descr1, index1) is box2
+        assert h.getarrayitem(box1, descr2, index1) is None
+        assert h.getarrayitem(box1, descr1, index2) is None
+        assert h.getarrayitem(box1, descr2, index2) is None
+        h.setarrayitem(box1, descr1, index2, box4)
+        assert h.getarrayitem(box1, descr1, index1) is box2
+        assert h.getarrayitem(box1, descr2, index1) is None
+        assert h.getarrayitem(box1, descr1, index2) is box4
+        assert h.getarrayitem(box1, descr2, index2) is None
+
+        h.setarrayitem(box1, descr2, index1, box3)
+        assert h.getarrayitem(box1, descr1, index1) is box2
+        assert h.getarrayitem(box1, descr2, index1) is box3
+        assert h.getarrayitem(box1, descr1, index2) is box4
+        assert h.getarrayitem(box1, descr2, index2) is None
+
+        h.setarrayitem(box1, descr1, index1, box3)
+        assert h.getarrayitem(box1, descr1, index1) is box3
+        assert h.getarrayitem(box1, descr2, index1) is box3
+        assert h.getarrayitem(box1, descr1, index2) is box4
+        assert h.getarrayitem(box1, descr2, index2) is None
+
+        h.setarrayitem(box3, descr1, index1, box1)
+        assert h.getarrayitem(box3, descr1, index1) is box1
+        assert h.getarrayitem(box1, descr1, index1) is None
+        assert h.getarrayitem(box1, descr2, index1) is box3
+        assert h.getarrayitem(box1, descr1, index2) is box4
+        assert h.getarrayitem(box1, descr2, index2) is None
+
+        h.reset()
+        assert h.getarrayitem(box1, descr1, index1) is None
+        assert h.getarrayitem(box1, descr2, index1) is None
+        assert h.getarrayitem(box3, descr1, index1) is None
+
+    def test_heapcache_array_nonconst_index(self):
+        h = HeapCache()
+        h.setarrayitem(box1, descr1, index1, box2)
+        h.setarrayitem(box1, descr1, index2, box4)
+        assert h.getarrayitem(box1, descr1, index1) is box2
+        assert h.getarrayitem(box1, descr1, index2) is box4
+        h.setarrayitem(box1, descr1, box2, box3)
+        assert h.getarrayitem(box1, descr1, index1) is None
+        assert h.getarrayitem(box1, descr1, index2) is None
+
+    def test_heapcache_read_fields_multiple_array(self):
+        h = HeapCache()
+        h.getarrayitem_now_known(box1, descr1, index1, box2)
+        h.getarrayitem_now_known(box3, descr1, index1, box4)
+        assert h.getarrayitem(box1, descr1, index1) is box2
+        assert h.getarrayitem(box1, descr2, index1) is None
+        assert h.getarrayitem(box3, descr1, index1) is box4
+        assert h.getarrayitem(box3, descr2, index1) is None
+
+        h.reset()
+        assert h.getarrayitem(box1, descr1, index1) is None
+        assert h.getarrayitem(box1, descr2, index1) is None
+        assert h.getarrayitem(box3, descr1, index1) is None
+        assert h.getarrayitem(box3, descr2, index1) is None
+
+    def test_heapcache_write_fields_multiple_array(self):
+        h = HeapCache()
+        h.setarrayitem(box1, descr1, index1, box2)
+        assert h.getarrayitem(box1, descr1, index1) is box2
+        h.setarrayitem(box3, descr1, index1, box4)
+        assert h.getarrayitem(box3, descr1, index1) is box4
+        assert h.getarrayitem(box1, descr1, index1) is None # box1 and box3 can alias
+
+        h = HeapCache()
+        h.new(box1)
+        h.setarrayitem(box1, descr1, index1, box2)
+        assert h.getarrayitem(box1, descr1, index1) is box2
+        h.setarrayitem(box3, descr1, index1, box4)
+        assert h.getarrayitem(box3, descr1, index1) is box4
+        assert h.getarrayitem(box1, descr1, index1) is None # box1 and box3 can alias
+
+        h = HeapCache()
+        h.new(box1)
+        h.new(box3)
+        h.setarrayitem(box1, descr1, index1, box2)
+        assert h.getarrayitem(box1, descr1, index1) is box2
+        h.setarrayitem(box3, descr1, index1, box4)
+        assert h.getarrayitem(box3, descr1, index1) is box4
+        assert h.getarrayitem(box1, descr1, index1) is box2 # box1 and box3 cannot alias
+        h.setarrayitem(box1, descr1, index1, box3)
+        assert h.getarrayitem(box3, descr1, index1) is box4
+        assert h.getarrayitem(box1, descr1, index1) is box3 # box1 and box3 cannot alias
+
+    def test_length_cache(self):
+        h = HeapCache()
+        h.new_array(box1, lengthbox1)
+        assert h.arraylen(box1) is lengthbox1
+
+        assert h.arraylen(box2) is None
+        h.arraylen_now_known(box2, lengthbox2)
+        assert h.arraylen(box2) is lengthbox2
+
+
+    def test_invalidate_cache(self):
+        h = HeapCache()
+        h.setfield(box1, descr1, box2)
+        h.setarrayitem(box1, descr1, index1, box2)
+        h.setarrayitem(box1, descr1, index2, box4)
+        h.invalidate_caches(rop.INT_ADD, None, [])
+        h.invalidate_caches(rop.INT_ADD_OVF, None, [])
+        h.invalidate_caches(rop.SETFIELD_RAW, None, [])
+        h.invalidate_caches(rop.SETARRAYITEM_RAW, None, [])
+        assert h.getfield(box1, descr1) is box2
+        assert h.getarrayitem(box1, descr1, index1) is box2
+        assert h.getarrayitem(box1, descr1, index2) is box4
+
+        h.invalidate_caches(
+            rop.CALL, FakeCallDescr(FakeEffektinfo.EF_ELIDABLE_CANNOT_RAISE), [])
+        assert h.getfield(box1, descr1) is box2
+        assert h.getarrayitem(box1, descr1, index1) is box2
+        assert h.getarrayitem(box1, descr1, index2) is box4
+
+        h.invalidate_caches(
+            rop.CALL_LOOPINVARIANT, FakeCallDescr(FakeEffektinfo.EF_LOOPINVARIANT), [])
+
+        h.invalidate_caches(
+            rop.CALL, FakeCallDescr(FakeEffektinfo.EF_RANDOM_EFFECTS), [])
+        assert h.getfield(box1, descr1) is None
+        assert h.getarrayitem(box1, descr1, index1) is None
+        assert h.getarrayitem(box1, descr1, index2) is None
+
+
+    def test_replace_box(self):
+        h = HeapCache()
+        h.setfield(box1, descr1, box2)
+        h.setfield(box1, descr2, box3)
+        h.setfield(box2, descr3, box3)
+        h.replace_box(box1, box4)
+        assert h.getfield(box1, descr1) is None
+        assert h.getfield(box1, descr2) is None
+        assert h.getfield(box4, descr1) is box2
+        assert h.getfield(box4, descr2) is box3
+        assert h.getfield(box2, descr3) is box3
+
+    def test_replace_box_array(self):
+        h = HeapCache()
+        h.setarrayitem(box1, descr1, index1, box2)
+        h.setarrayitem(box1, descr2, index1, box3)
+        h.arraylen_now_known(box1, lengthbox1)
+        h.setarrayitem(box2, descr1, index2, box1)
+        h.setarrayitem(box3, descr2, index2, box1)
+        h.setarrayitem(box2, descr3, index2, box3)
+        h.replace_box(box1, box4)
+        assert h.getarrayitem(box1, descr1, index1) is None
+        assert h.getarrayitem(box1, descr2, index1) is None
+        assert h.arraylen(box1) is None
+        assert h.arraylen(box4) is lengthbox1
+        assert h.getarrayitem(box4, descr1, index1) is box2
+        assert h.getarrayitem(box4, descr2, index1) is box3
+        assert h.getarrayitem(box2, descr1, index2) is box4
+        assert h.getarrayitem(box3, descr2, index2) is box4
+        assert h.getarrayitem(box2, descr3, index2) is box3
+
+        h.replace_box(lengthbox1, lengthbox2)
+        assert h.arraylen(box4) is lengthbox2
+
+    def test_ll_arraycopy(self):
+        h = HeapCache()
+        h.new_array(box1, lengthbox1)
+        h.setarrayitem(box1, descr1, index1, box2)
+        h.new_array(box2, lengthbox1)
+        # Just need the destination box for this call
+        h.invalidate_caches(
+            rop.CALL,
+            FakeCallDescr(FakeEffektinfo.EF_CANNOT_RAISE, FakeEffektinfo.OS_ARRAYCOPY),
+            [None, None, box2, None, None]
+        )
+        assert h.getarrayitem(box1, descr1, index1) is box2
+        h.invalidate_caches(
+            rop.CALL,
+            FakeCallDescr(FakeEffektinfo.EF_CANNOT_RAISE, FakeEffektinfo.OS_ARRAYCOPY),
+            [None, None, box3, None, None]
+        )
+        assert h.getarrayitem(box1, descr1, index1) is None
+
+        h.setarrayitem(box4, descr1, index1, box2)
+        assert h.getarrayitem(box4, descr1, index1) is box2
+        h.invalidate_caches(
+            rop.CALL,
+            FakeCallDescr(FakeEffektinfo.EF_CANNOT_RAISE, FakeEffektinfo.OS_ARRAYCOPY),
+            [None, None, box2, None, None]
+        )
+        assert h.getarrayitem(box4, descr1, index1) is None
+
+    def test_unescaped(self):
+        h = HeapCache()
+        assert not h.is_unescaped(box1)
+        h.new(box2)
+        assert h.is_unescaped(box2)
+        h.invalidate_caches(rop.SETFIELD_GC, None, [box2, box1])
+        assert h.is_unescaped(box2)
+        h.invalidate_caches(rop.SETFIELD_GC, None, [box1, box2])
+        assert not h.is_unescaped(box2)
+
+    def test_unescaped_testing(self):
+        h = HeapCache()
+        h.new(box1)
+        h.new(box2)
+        assert h.is_unescaped(box1)
+        assert h.is_unescaped(box2)
+        # Putting a virtual inside of another virtual doesn't escape it.
+        h.invalidate_caches(rop.SETFIELD_GC, None, [box1, box2])
+        assert h.is_unescaped(box2)
+        # Reading a field from a virtual doesn't escape it.
+        h.invalidate_caches(rop.GETFIELD_GC, None, [box1])
+        assert h.is_unescaped(box1)
+        # Escaping a virtual transitively escapes anything inside of it.
+        assert not h.is_unescaped(box3)
+        h.invalidate_caches(rop.SETFIELD_GC, None, [box3, box1])
+        assert not h.is_unescaped(box1)
+        assert not h.is_unescaped(box2)
+
+    def test_unescaped_array(self):
+        h = HeapCache()
+        h.new_array(box1, lengthbox1)
+        assert h.is_unescaped(box1)
+        h.invalidate_caches(rop.SETARRAYITEM_GC, None, [box1, index1, box2])
+        assert h.is_unescaped(box1)
+        h.invalidate_caches(rop.SETARRAYITEM_GC, None, [box2, index1, box1])
+        assert not h.is_unescaped(box1)
\ No newline at end of file
diff --git a/pypy/jit/metainterp/test/test_list.py b/pypy/jit/metainterp/test/test_list.py
--- a/pypy/jit/metainterp/test/test_list.py
+++ b/pypy/jit/metainterp/test/test_list.py
@@ -34,7 +34,7 @@
                 l = [x + 1]
                 n -= 1
             return l[0]
-        
+
         res = self.meta_interp(f, [10], listops=True)
         assert res == f(10)
         self.check_all_virtualized()
@@ -60,7 +60,7 @@
 
     def test_ll_fixed_setitem_fast(self):
         jitdriver = JitDriver(greens = [], reds = ['n', 'l'])
-        
+
         def f(n):
             l = [1, 2, 3]
 
@@ -116,7 +116,7 @@
         assert res == f(10)
         py.test.skip("'[non-null] * n' gives a residual call so far")
         self.check_loops(setarrayitem_gc=0, getarrayitem_gc=0, call=0)
-    
+
     def test_arraycopy_simpleoptimize(self):
         def f():
             l = [1, 2, 3, 4]
@@ -208,6 +208,26 @@
         assert res == f(15)
         self.check_loops(guard_exception=0)
 
+    def test_virtual_resize(self):
+        jitdriver = JitDriver(greens = [], reds = ['n', 's'])
+        def f(n):
+            s = 0
+            while n > 0:
+                jitdriver.jit_merge_point(n=n, s=s)
+                lst = []
+                lst += [1]
+                n -= len(lst)
+                s += lst[0]
+                lst.pop()
+                lst.append(1)
+                s /= lst.pop()
+            return s
+        res = self.meta_interp(f, [15], listops=True)
+        assert res == f(15)
+        self.check_loops({"int_add": 1, "int_sub": 1, "int_gt": 1,
+                          "guard_true": 1, "jump": 1})
+
+
 class TestOOtype(ListTests, OOJitMixin):
     pass
 
@@ -236,8 +256,6 @@
             return a * b
         res = self.meta_interp(f, [37])
         assert res == f(37)
-        # There is the one actual field on a, plus 2 getfield's from the list
-        # itself, 1 to get the length (which is then incremented and passed to
-        # the resize func), and then a read of the items field to actually
-        # perform the setarrayitem on
-        self.check_loops(getfield_gc=5, everywhere=True)
+        # There is the one actual field on a, plus several fields on the list
+        # itself
+        self.check_loops(getfield_gc=10, everywhere=True)
diff --git a/pypy/jit/metainterp/test/test_slist.py b/pypy/jit/metainterp/test/test_slist.py
--- a/pypy/jit/metainterp/test/test_slist.py
+++ b/pypy/jit/metainterp/test/test_slist.py
@@ -5,7 +5,6 @@
 class ListTests(object):
 
     def test_basic_list(self):
-        py.test.skip("not yet")
         myjitdriver = JitDriver(greens = [], reds = ['n', 'lst'])
         def f(n):
             lst = []
@@ -34,7 +33,7 @@
             return m
         res = self.interp_operations(f, [11], listops=True)
         assert res == 49
-        self.check_operations_history(call=5)
+        self.check_operations_history(call=3)
 
     def test_list_of_voids(self):
         myjitdriver = JitDriver(greens = [], reds = ['n', 'lst'])
@@ -93,7 +92,7 @@
             return x
         res = self.meta_interp(f, [-2], listops=True)
         assert res == 41
-        self.check_loops(call=1, guard_value=0)
+        self.check_loops(call=0, guard_value=0)
 
 # we don't support resizable lists on ootype
 #class TestOOtype(ListTests, OOJitMixin):
diff --git a/pypy/jit/metainterp/test/test_string.py b/pypy/jit/metainterp/test/test_string.py
--- a/pypy/jit/metainterp/test/test_string.py
+++ b/pypy/jit/metainterp/test/test_string.py
@@ -1,9 +1,11 @@
 import py
+
+from pypy.jit.codewriter.policy import StopAtXPolicy
+from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin
+from pypy.rlib.debug import debug_print
 from pypy.rlib.jit import JitDriver, dont_look_inside, we_are_jitted
-from pypy.rlib.debug import debug_print
-from pypy.jit.codewriter.policy import StopAtXPolicy
+from pypy.rlib.rstring import StringBuilder
 from pypy.rpython.ootypesystem import ootype
-from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin
 
 
 class StringTests:
@@ -27,7 +29,7 @@
             return i
         res = self.meta_interp(f, [10, True, _str('h')], listops=True)
         assert res == 5
-        self.check_loops(**{self.CALL: 1, self.CALL_PURE: 0})
+        self.check_loops(**{self.CALL: 1, self.CALL_PURE: 0, 'everywhere': True})
 
     def test_eq_folded(self):
         _str = self._str
@@ -327,7 +329,7 @@
     def test_str_slice_len_surviving(self):
         _str = self._str
         longstring = _str("Unrolling Trouble")
-        mydriver = JitDriver(reds = ['i', 'a', 'sa'], greens = []) 
+        mydriver = JitDriver(reds = ['i', 'a', 'sa'], greens = [])
         def f(a):
             i = sa = a
             while i < len(longstring):
@@ -343,7 +345,7 @@
         fillers = _str("abcdefghijklmnopqrstuvwxyz")
         data = _str("ABCDEFGHIJKLMNOPQRSTUVWXYZ")
 
-        mydriver = JitDriver(reds = ['line', 'noise', 'res'], greens = []) 
+        mydriver = JitDriver(reds = ['line', 'noise', 'res'], greens = [])
         def f():
             line = data
             noise = fillers
@@ -370,7 +372,7 @@
             def __init__(self, value):
                 self.value = value
         mydriver = JitDriver(reds = ['ratio', 'line', 'noise', 'res'],
-                             greens = []) 
+                             greens = [])
         def f():
             line = Str(data)
             noise = Str(fillers)
@@ -408,7 +410,7 @@
             return len(sa)
         assert self.meta_interp(f, [16]) == f(16)
 
-    def test_loop_invariant_string_slize(self):
+    def test_loop_invariant_string_slice(self):
         _str = self._str
         mydriver = JitDriver(reds = ['i', 'n', 'sa', 's', 's1'], greens = [])
         def f(n, c):
@@ -425,7 +427,7 @@
             return sa
         assert self.meta_interp(f, [16, 'a']) == f(16, 'a')
 
-    def test_loop_invariant_string_slize_boxed(self):
+    def test_loop_invariant_string_slice_boxed(self):
         class Str(object):
             def __init__(self, value):
                 self.value = value
@@ -445,7 +447,7 @@
             return sa
         assert self.meta_interp(f, [16, 'a']) == f(16, 'a')
 
-    def test_loop_invariant_string_slize_in_array(self):
+    def test_loop_invariant_string_slice_in_array(self):
         _str = self._str
         mydriver = JitDriver(reds = ['i', 'n', 'sa', 's', 's1'], greens = [])
         def f(n, c):
@@ -513,7 +515,7 @@
                 m -= 1
             return 42
         self.meta_interp(f, [6, 7])
-        self.check_loops(call=3,    # str(), _str(), escape()
+        self.check_loops(call=1,    # escape()
                          newunicode=1, unicodegetitem=0,
                          unicodesetitem=1, copyunicodecontent=1)
 
@@ -536,3 +538,55 @@
         self.check_loops(call_pure=0, call=1,
                          newunicode=0, unicodegetitem=0,
                          unicodesetitem=0, copyunicodecontent=0)
+
+    def test_join_chars(self):
+        jitdriver = JitDriver(reds=['a', 'b', 'c', 'i'], greens=[])
+        def f(a, b, c):
+            i = 0
+            while i < 10:
+                jitdriver.jit_merge_point(a=a, b=b, c=c, i=i)
+                x = []
+                if a:
+                    x.append("a")
+                if b:
+                    x.append("b")
+                if c:
+                    x.append("c")
+                i += len("".join(x))
+            return i
+        res = self.meta_interp(f, [1, 1, 1])
+        assert res == f(True, True, True)
+        # The "".join should be unrolled, since the length of x is known since
+        # it is virtual, ensure there are no calls to ll_join_chars, or
+        # allocations.
+        self.check_loops({
+            "guard_true": 5, "int_is_true": 3, "int_lt": 2, "int_add": 2, "jump": 2,
+        }, everywhere=True)
+
+    def test_virtual_copystringcontent(self):
+        jitdriver = JitDriver(reds=['n', 'result'], greens=[])
+        def main(n):
+            result = 0
+            while n >= 0:
+                jitdriver.jit_merge_point(n=n, result=result)
+                b = StringBuilder(6)
+                b.append("Hello!")
+                result += ord(b.build()[0])
+                n -= 1
+            return result
+        res = self.meta_interp(main, [9])
+        assert res == main(9)
+
+    def test_virtual_copystringcontent2(self):
+        jitdriver = JitDriver(reds=['n', 'result'], greens=[])
+        def main(n):
+            result = 0
+            while n >= 0:
+                jitdriver.jit_merge_point(n=n, result=result)
+                b = StringBuilder(6)
+                b.append("Hello!")
+                result += ord((b.build() + "xyz")[0])
+                n -= 1
+            return result
+        res = self.meta_interp(main, [9])
+        assert res == main(9)
diff --git a/pypy/jit/metainterp/test/test_tracingopts.py b/pypy/jit/metainterp/test/test_tracingopts.py
--- a/pypy/jit/metainterp/test/test_tracingopts.py
+++ b/pypy/jit/metainterp/test/test_tracingopts.py
@@ -1,7 +1,10 @@
+import sys
+
+from pypy.jit.metainterp.test.support import LLJitMixin
+from pypy.rlib import jit
+from pypy.rlib.rarithmetic import ovfcheck
+
 import py
-import sys
-from pypy.rlib import jit
-from pypy.jit.metainterp.test.support import LLJitMixin
 
 
 class TestLLtype(LLJitMixin):
@@ -257,6 +260,28 @@
         self.check_operations_history(setarrayitem_gc=2, setfield_gc=2,
                                       getarrayitem_gc=0, getfield_gc=2)
 
+    def test_promote_changes_array_cache(self):
+        a1 = [0, 0]
+        a2 = [0, 0]
+        def fn(n):
+            if n > 0:
+                a = a1
+            else:
+                a = a2
+            a[0] = n
+            jit.hint(n, promote=True)
+            x1 = a[0]
+            jit.hint(x1, promote=True)
+            a[n - n] = n + 1
+            return a[0] + x1
+        res = self.interp_operations(fn, [7])
+        assert res == 7 + 7 + 1
+        self.check_operations_history(getarrayitem_gc=0, guard_value=1)
+        res = self.interp_operations(fn, [-7])
+        assert res == -7 - 7 + 1
+        self.check_operations_history(getarrayitem_gc=0, guard_value=1)
+
+
     def test_list_caching(self):
         a1 = [0, 0]
         a2 = [0, 0]
@@ -357,7 +382,7 @@
         assert res == f(10, 1, 1)
         self.check_history(getarrayitem_gc=0, getfield_gc=0)
 
-    def test_heap_caching_pure(self):
+    def test_heap_caching_array_pure(self):
         class A(object):
             pass
         p1 = A()
@@ -405,3 +430,164 @@
         assert res == -7 + 7
         self.check_operations_history(getfield_gc=0)
         return
+
+    def test_heap_caching_multiple_objects(self):
+        class Gbl(object):
+            pass
+        g = Gbl()
+        class A(object):
+            pass
+        a1 = A()
+        g.a1 = a1
+        a1.x = 7
+        a2 = A()
+        g.a2 = a2
+        a2.x = 7
+        def gn(a1, a2):
+            return a1.x + a2.x
+        def fn(n):
+            if n < 0:
+                a1 = A()
+                g.a1 = a1
+                a1.x = n
+                a2 = A()
+                g.a2 = a2
+                a2.x = n - 1
+            else:
+                a1 = g.a1
+                a2 = g.a2
+            return a1.x + a2.x + gn(a1, a2)
+        res = self.interp_operations(fn, [-7])
+        assert res == 2 * -7 + 2 * -8
+        self.check_operations_history(setfield_gc=4, getfield_gc=0)
+        res = self.interp_operations(fn, [7])
+        assert res == 4 * 7
+        self.check_operations_history(getfield_gc=4)
+
+    def test_heap_caching_multiple_tuples(self):
+        class Gbl(object):
+            pass
+        g = Gbl()
+        def gn(a1, a2):
+            return a1[0] + a2[0]
+        def fn(n):
+            a1 = (n, )
+            g.a = a1
+            a2 = (n - 1, )
+            g.a = a2
+            jit.promote(n)
+            return a1[0] + a2[0] + gn(a1, a2)
+        res = self.interp_operations(fn, [7])
+        assert res == 2 * 7 + 2 * 6
+        self.check_operations_history(getfield_gc_pure=0)
+        res = self.interp_operations(fn, [-7])
+        assert res == 2 * -7 + 2 * -8
+        self.check_operations_history(getfield_gc_pure=0)
+
+    def test_heap_caching_multiple_arrays(self):
+        class Gbl(object):
+            pass
+        g = Gbl()
+        def fn(n):
+            a1 = [n, n, n]
+            g.a = a1
+            a1[0] = n
+            a2 = [n, n, n]
+            g.a = a2
+            a2[0] = n - 1
+            return a1[0] + a2[0] + a1[0] + a2[0]
+        res = self.interp_operations(fn, [7])
+        assert res == 2 * 7 + 2 * 6
+        self.check_operations_history(getarrayitem_gc=0)
+        res = self.interp_operations(fn, [-7])
+        assert res == 2 * -7 + 2 * -8
+        self.check_operations_history(getarrayitem_gc=0)
+
+    def test_heap_caching_multiple_arrays_getarrayitem(self):
+        class Gbl(object):
+            pass
+        g = Gbl()
+        g.a1 = [7, 8, 9]
+        g.a2 = [8, 9, 10, 11]
+
+        def fn(i):
+            if i < 0:
+                g.a1 = [7, 8, 9]
+                g.a2 = [7, 8, 9, 10]
+            jit.promote(i)
+            a1 = g.a1
+            a1[i + 1] = 15 # make lists mutable
+            a2 = g.a2
+            a2[i + 1] = 19
+            return a1[i] + a2[i] + a1[i] + a2[i]
+        res = self.interp_operations(fn, [0])
+        assert res == 2 * 7 + 2 * 8
+        self.check_operations_history(getarrayitem_gc=2)
+
+
+    def test_heap_caching_multiple_lists(self):
+        class Gbl(object):
+            pass
+        g = Gbl()
+        g.l = []
+        def fn(n):
+            if n < -100:
+                g.l.append(1)
+            a1 = [n, n, n]
+            g.l = a1
+            a1[0] = n
+            a2 = [n, n, n]
+            g.l = a2
+            a2[0] = n - 1
+            return a1[0] + a2[0] + a1[0] + a2[0]
+        res = self.interp_operations(fn, [7])
+        assert res == 2 * 7 + 2 * 6
+        self.check_operations_history(getarrayitem_gc=0, getfield_gc=0)
+        res = self.interp_operations(fn, [-7])
+        assert res == 2 * -7 + 2 * -8
+        self.check_operations_history(getarrayitem_gc=0, getfield_gc=0)
+
+    def test_length_caching(self):
+        class Gbl(object):
+            pass
+        g = Gbl()
+        g.a = [0] * 7
+        def fn(n):
+            a = g.a
+            res = len(a) + len(a)
+            a1 = [0] * n
+            g.a = a1
+            return len(a1) + res
+        res = self.interp_operations(fn, [7])
+        assert res == 7 * 3
+        self.check_operations_history(arraylen_gc=1)
+
+    def test_arraycopy(self):
+        class Gbl(object):
+            pass
+        g = Gbl()
+        g.a = [0] * 7
+        def fn(n):
+            assert n >= 0
+            a = g.a
+            x = [0] * n
+            x[2] = 21
+            return len(a[:n]) + x[2]
+        res = self.interp_operations(fn, [3])
+        assert res == 24
+        self.check_operations_history(getarrayitem_gc=0)
+
+    def test_fold_int_add_ovf(self):
+        def fn(n):
+            jit.promote(n)
+            try:
+                n = ovfcheck(n + 1)
+            except OverflowError:
+                return 12
+            else:
+                return n
+        res = self.interp_operations(fn, [3])
+        assert res == 4
+        self.check_operations_history(int_add_ovf=0)
+        res = self.interp_operations(fn, [sys.maxint])
+        assert res == 12
\ No newline at end of file
diff --git a/pypy/jit/metainterp/test/test_virtualstate.py b/pypy/jit/metainterp/test/test_virtualstate.py
--- a/pypy/jit/metainterp/test/test_virtualstate.py
+++ b/pypy/jit/metainterp/test/test_virtualstate.py
@@ -2,7 +2,7 @@
 import py
 from pypy.jit.metainterp.optimize import InvalidLoop
 from pypy.jit.metainterp.optimizeopt.virtualstate import VirtualStateInfo, VStructStateInfo, \
-     VArrayStateInfo, NotVirtualStateInfo, VirtualState
+     VArrayStateInfo, NotVirtualStateInfo, VirtualState, ShortBoxes
 from pypy.jit.metainterp.optimizeopt.optimizer import OptValue
 from pypy.jit.metainterp.history import BoxInt, BoxFloat, BoxPtr, ConstInt, ConstPtr
 from pypy.rpython.lltypesystem import lltype
@@ -11,6 +11,7 @@
 from pypy.jit.metainterp.history import TreeLoop, LoopToken
 from pypy.jit.metainterp.optimizeopt.test.test_optimizeopt import FakeDescr, FakeMetaInterpStaticData
 from pypy.jit.metainterp.optimize import RetraceLoop
+from pypy.jit.metainterp.resoperation import ResOperation, rop
 
 class TestBasic:
     someptr1 = LLtypeMixin.myptr
@@ -129,6 +130,7 @@
             info.fieldstate = [info]
             assert info.generalization_of(info, {}, {})
 
+
 class BaseTestGenerateGuards(BaseTest):
     def guards(self, info1, info2, box, expected):
         info1.position = info2.position = 0
@@ -910,3 +912,111 @@
 class TestLLtypeBridges(BaseTestBridges, LLtypeMixin):
     pass
 
+class FakeOptimizer:
+    def make_equal_to(*args):
+        pass
+    def getvalue(*args):
+        pass
+
+class TestShortBoxes:
+    p1 = BoxPtr()
+    p2 = BoxPtr()
+    p3 = BoxPtr()
+    p4 = BoxPtr()
+    i1 = BoxInt()
+    i2 = BoxInt()
+    i3 = BoxInt()
+    i4 = BoxInt()
+    
+    def test_short_box_duplication_direct(self):
+        class Optimizer(FakeOptimizer):
+            def produce_potential_short_preamble_ops(_self, sb):
+                sb.add_potential(ResOperation(rop.GETFIELD_GC, [self.p1], self.i1))
+                sb.add_potential(ResOperation(rop.GETFIELD_GC, [self.p2], self.i1))
+        sb = ShortBoxes(Optimizer(), [self.p1, self.p2])
+        assert len(sb.short_boxes) == 4
+        assert self.i1 in sb.short_boxes
+        assert sum([op.result is self.i1 for op in sb.short_boxes.values() if op]) == 1
+
+    def test_dont_duplicate_potential_boxes(self):
+        class Optimizer(FakeOptimizer):
+            def produce_potential_short_preamble_ops(_self, sb):
+                sb.add_potential(ResOperation(rop.GETFIELD_GC, [self.p1], self.i1))
+                sb.add_potential(ResOperation(rop.GETFIELD_GC, [BoxPtr()], self.i1))
+                sb.add_potential(ResOperation(rop.INT_NEG, [self.i1], self.i2))
+                sb.add_potential(ResOperation(rop.INT_ADD, [ConstInt(7), self.i2],
+                                              self.i3))
+        sb = ShortBoxes(Optimizer(), [self.p1, self.p2])
+        assert len(sb.short_boxes) == 5
+
+    def test_prioritize1(self):
+        class Optimizer(FakeOptimizer):
+            def produce_potential_short_preamble_ops(_self, sb):
+                sb.add_potential(ResOperation(rop.GETFIELD_GC, [self.p1], self.i1))
+                sb.add_potential(ResOperation(rop.GETFIELD_GC, [self.p2], self.i1))
+                sb.add_potential(ResOperation(rop.INT_NEG, [self.i1], self.i2))
+        sb = ShortBoxes(Optimizer(), [self.p1, self.p2])
+        assert len(sb.short_boxes.values()) == 5
+        int_neg = [op for op in sb.short_boxes.values()
+                   if op and op.getopnum() == rop.INT_NEG]
+        assert len(int_neg) == 1
+        int_neg = int_neg[0]
+        getfield = [op for op in sb.short_boxes.values()
+                    if op and op.result == int_neg.getarg(0)]
+        assert len(getfield) == 1
+        assert getfield[0].getarg(0) in [self.p1, self.p2]
+
+    def test_prioritize1bis(self):
+        class Optimizer(FakeOptimizer):
+            def produce_potential_short_preamble_ops(_self, sb):
+                sb.add_potential(ResOperation(rop.GETFIELD_GC, [self.p1], self.i1),
+                                 synthetic=True)
+                sb.add_potential(ResOperation(rop.GETFIELD_GC, [self.p2], self.i1),
+                                 synthetic=True)
+                sb.add_potential(ResOperation(rop.INT_NEG, [self.i1], self.i2))
+        sb = ShortBoxes(Optimizer(), [self.p1, self.p2])
+        assert len(sb.short_boxes.values()) == 5
+        int_neg = [op for op in sb.short_boxes.values()
+                   if op and op.getopnum() == rop.INT_NEG]
+        assert len(int_neg) == 1
+        int_neg = int_neg[0]
+        getfield = [op for op in sb.short_boxes.values()
+                    if op and op.result == int_neg.getarg(0)]
+        assert len(getfield) == 1
+        assert getfield[0].getarg(0) in [self.p1, self.p2]
+        
+    def test_prioritize2(self):
+        class Optimizer(FakeOptimizer):
+            def produce_potential_short_preamble_ops(_self, sb):
+                sb.add_potential(ResOperation(rop.GETFIELD_GC, [self.p1], self.i1),
+                                 synthetic=True)
+                sb.add_potential(ResOperation(rop.GETFIELD_GC, [self.p2], self.i1))
+                sb.add_potential(ResOperation(rop.INT_NEG, [self.i1], self.i2))
+        sb = ShortBoxes(Optimizer(), [self.p1, self.p2])
+        assert len(sb.short_boxes.values()) == 5
+        int_neg = [op for op in sb.short_boxes.values()
+                   if op and op.getopnum() == rop.INT_NEG]
+        assert len(int_neg) == 1
+        int_neg = int_neg[0]
+        getfield = [op for op in sb.short_boxes.values()
+                    if op and op.result == int_neg.getarg(0)]
+        assert len(getfield) == 1
+        assert getfield[0].getarg(0) == self.p2
+        
+    def test_prioritize3(self):
+        class Optimizer(FakeOptimizer):
+            def produce_potential_short_preamble_ops(_self, sb):
+                sb.add_potential(ResOperation(rop.GETFIELD_GC, [self.p1], self.i1))
+                sb.add_potential(ResOperation(rop.GETFIELD_GC, [self.p2], self.i1),
+                                 synthetic=True)
+                sb.add_potential(ResOperation(rop.INT_NEG, [self.i1], self.i2))
+        sb = ShortBoxes(Optimizer(), [self.p1, self.p2])
+        assert len(sb.short_boxes.values()) == 5
+        int_neg = [op for op in sb.short_boxes.values()
+                   if op and op.getopnum() == rop.INT_NEG]
+        assert len(int_neg) == 1
+        int_neg = int_neg[0]
+        getfield = [op for op in sb.short_boxes.values()
+                    if op and op.result == int_neg.getarg(0)]
+        assert len(getfield) == 1
+        assert getfield[0].getarg(0) == self.p1
diff --git a/pypy/jit/metainterp/warmstate.py b/pypy/jit/metainterp/warmstate.py
--- a/pypy/jit/metainterp/warmstate.py
+++ b/pypy/jit/metainterp/warmstate.py
@@ -367,9 +367,9 @@
             # ---------- execute assembler ----------
             while True:     # until interrupted by an exception
                 metainterp_sd.profiler.start_running()
-                debug_start("jit-running")
+                #debug_start("jit-running")
                 fail_descr = warmrunnerdesc.execute_token(loop_token)
-                debug_stop("jit-running")
+                #debug_stop("jit-running")
                 metainterp_sd.profiler.end_running()
                 loop_token = None     # for test_memmgr
                 if vinfo is not None:
diff --git a/pypy/jit/tl/pypyjit.py b/pypy/jit/tl/pypyjit.py
--- a/pypy/jit/tl/pypyjit.py
+++ b/pypy/jit/tl/pypyjit.py
@@ -40,7 +40,7 @@
 config.objspace.usemodules.array = False
 config.objspace.usemodules._weakref = True
 config.objspace.usemodules._sre = False
-config.objspace.usemodules._lsprof = True
+config.objspace.usemodules._lsprof = False
 #
 config.objspace.usemodules._ffi = True
 config.objspace.usemodules.micronumpy = False
@@ -77,7 +77,7 @@
 
 def read_code():
     from pypy.module.marshal.interp_marshal import dumps
-    
+
     filename = 'pypyjit_demo.py'
     source = readfile(filename)
     ec = space.getexecutioncontext()
diff --git a/pypy/jit/tl/pypyjit_demo.py b/pypy/jit/tl/pypyjit_demo.py
--- a/pypy/jit/tl/pypyjit_demo.py
+++ b/pypy/jit/tl/pypyjit_demo.py
@@ -2,22 +2,16 @@
 pypyjit.set_param(threshold=200)
 
 
-def main(a, b):
-    i = sa = 0
-    while i < 300:
-        if a > 0: # Specialises the loop
-            pass
-        if b < 2 and b > 0:
-            pass
-        if (a >> b) >= 0:
-            sa += 1
-        if (a << b) > 2:
-            sa += 10000
-        i += 1
-    return sa
+def f(n):
+    pairs = [(0.0, 1.0), (2.0, 3.0)] * n
+    mag = 0
+    for (x1, x2) in pairs:
+        dx = x1 - x2
+        mag += ((dx * dx ) ** (-1.5))            
+    return n
 
 try:
-    print main(2, 1)
+    print f(301)
 
 except Exception, e:
     print "Exception: ", type(e)
diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py
--- a/pypy/module/__builtin__/functional.py
+++ b/pypy/module/__builtin__/functional.py
@@ -3,13 +3,13 @@
 
 """
 
+from pypy.interpreter.baseobjspace import Wrappable
 from pypy.interpreter.error import OperationError
-from pypy.interpreter.gateway import NoneNotWrapped
-from pypy.interpreter.gateway import interp2app, unwrap_spec
+from pypy.interpreter.gateway import NoneNotWrapped, interp2app, unwrap_spec
 from pypy.interpreter.typedef import TypeDef
-from pypy.interpreter.baseobjspace import Wrappable
+from pypy.rlib import jit
+from pypy.rlib.objectmodel import specialize
 from pypy.rlib.rarithmetic import r_uint, intmask
-from pypy.rlib.objectmodel import specialize
 from pypy.rlib.rbigint import rbigint
 
 
@@ -134,29 +134,15 @@
 
 
 @specialize.arg(2)
+ at jit.look_inside_iff(lambda space, args, implementation_of:
+    jit.isconstant(len(args.arguments_w)) and
+    len(args.arguments_w) == 2
+)
 def min_max(space, args, implementation_of):
     if implementation_of == "max":
         compare = space.gt
     else:
         compare = space.lt
-
-    args_w = args.arguments_w
-    if len(args_w) == 2 and not args.keywords:
-        # simple case, suitable for the JIT
-        w_arg0, w_arg1 = args_w
-        if space.is_true(compare(w_arg0, w_arg1)):
-            return w_arg0
-        else:
-            return w_arg1
-    else:
-        return min_max_loop(space, args, implementation_of)
-
- at specialize.arg(2)
-def min_max_loop(space, args, implementation_of):
-    if implementation_of == "max":
-        compare = space.gt
-    else:
-        compare = space.lt
     args_w = args.arguments_w
     if len(args_w) > 1:
         w_sequence = space.newtuple(args_w)
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -8,6 +8,7 @@
     appleveldefs = {}
 
     interpleveldefs = {
+        "StringBuilder": "interp_builders.W_StringBuilder",
         "UnicodeBuilder": "interp_builders.W_UnicodeBuilder",
     }
 
diff --git a/pypy/module/__pypy__/interp_builders.py b/pypy/module/__pypy__/interp_builders.py
--- a/pypy/module/__pypy__/interp_builders.py
+++ b/pypy/module/__pypy__/interp_builders.py
@@ -2,49 +2,55 @@
 from pypy.interpreter.error import OperationError
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.typedef import TypeDef
-from pypy.rlib.rstring import UnicodeBuilder
+from pypy.rlib.rstring import UnicodeBuilder, StringBuilder
+from pypy.tool.sourcetools import func_with_new_name
 
 
-class W_UnicodeBuilder(Wrappable):
-    def __init__(self, space, size):
-        if size < 0:
-            self.builder = UnicodeBuilder()
-        else:
-            self.builder = UnicodeBuilder(size)
-        self.done = False
+def create_builder(name, strtype, builder_cls):
+    class W_Builder(Wrappable):
+        def __init__(self, space, size):
+            if size < 0:
+                self.builder = builder_cls()
+            else:
+                self.builder = builder_cls(size)
 
-    def _check_done(self, space):
-        if self.done:
-            raise OperationError(space.w_ValueError, space.wrap("Can't operate on a done builder"))
+        def _check_done(self, space):
+            if self.builder is None:
+                raise OperationError(space.w_ValueError, space.wrap("Can't operate on a done builder"))
 
-    @unwrap_spec(size=int)
-    def descr__new__(space, w_subtype, size=-1):
-        return W_UnicodeBuilder(space, size)
+        @unwrap_spec(size=int)
+        def descr__new__(space, w_subtype, size=-1):
+            return W_Builder(space, size)
 
-    @unwrap_spec(s=unicode)
-    def descr_append(self, space, s):
-        self._check_done(space)
-        self.builder.append(s)
+        @unwrap_spec(s=strtype)
+        def descr_append(self, space, s):
+            self._check_done(space)
+            self.builder.append(s)
 
-    @unwrap_spec(s=unicode, start=int, end=int)
-    def descr_append_slice(self, space, s, start, end):
-        self._check_done(space)
-        if not 0 <= start <= end <= len(s):
-            raise OperationError(space.w_ValueError, space.wrap("bad start/stop"))
-        self.builder.append_slice(s, start, end)
+        @unwrap_spec(s=strtype, start=int, end=int)
+        def descr_append_slice(self, space, s, start, end):
+            self._check_done(space)
+            if not 0 <= start <= end <= len(s):
+                raise OperationError(space.w_ValueError, space.wrap("bad start/stop"))
+            self.builder.append_slice(s, start, end)
 
-    def descr_build(self, space):
-        self._check_done(space)
-        w_s = space.wrap(self.builder.build())
-        self.done = True
-        return w_s
+        def descr_build(self, space):
+            self._check_done(space)
+            w_s = space.wrap(self.builder.build())
+            self.builder = None
+            return w_s
 
+    W_Builder.__name__ = "W_%s" % name
+    W_Builder.typedef = TypeDef(name,
+        __new__ = interp2app(func_with_new_name(
+                                    W_Builder.descr__new__.im_func,
+                                    '%s_new' % (name,))),
+        append = interp2app(W_Builder.descr_append),
+        append_slice = interp2app(W_Builder.descr_append_slice),
+        build = interp2app(W_Builder.descr_build),
+    )
+    W_Builder.typedef.acceptable_as_base_class = False
+    return W_Builder
 
-W_UnicodeBuilder.typedef = TypeDef("UnicodeBuilder",
-    __new__ = interp2app(W_UnicodeBuilder.descr__new__.im_func),
-
-    append = interp2app(W_UnicodeBuilder.descr_append),
-    append_slice = interp2app(W_UnicodeBuilder.descr_append_slice),
-    build = interp2app(W_UnicodeBuilder.descr_build),
-)
-W_UnicodeBuilder.typedef.acceptable_as_base_class = False
+W_StringBuilder = create_builder("StringBuilder", str, StringBuilder)
+W_UnicodeBuilder = create_builder("UnicodeBuilder", unicode, UnicodeBuilder)
diff --git a/pypy/module/__pypy__/test/test_builders.py b/pypy/module/__pypy__/test/test_builders.py
--- a/pypy/module/__pypy__/test/test_builders.py
+++ b/pypy/module/__pypy__/test/test_builders.py
@@ -31,4 +31,14 @@
         raises(ValueError, b.append_slice, u"1", 2, 1)
         s = b.build()
         assert s == "cde"
-        raises(ValueError, b.append_slice, u"abc", 1, 2)
\ No newline at end of file
+        raises(ValueError, b.append_slice, u"abc", 1, 2)
+
+    def test_stringbuilder(self):
+        from __pypy__.builders import StringBuilder
+        b = StringBuilder()
+        b.append("abc")
+        b.append("123")
+        b.append("you and me")
+        s = b.build()
+        assert s == "abc123you and me"
+        raises(ValueError, b.build)
\ No newline at end of file
diff --git a/pypy/module/_continuation/__init__.py b/pypy/module/_continuation/__init__.py
--- a/pypy/module/_continuation/__init__.py
+++ b/pypy/module/_continuation/__init__.py
@@ -37,4 +37,5 @@
     interpleveldefs = {
         'continulet': 'interp_continuation.W_Continulet',
         'permute': 'interp_continuation.permute',
+        '_p': 'interp_continuation.unpickle',      # pickle support
     }
diff --git a/pypy/module/_continuation/interp_continuation.py b/pypy/module/_continuation/interp_continuation.py
--- a/pypy/module/_continuation/interp_continuation.py
+++ b/pypy/module/_continuation/interp_continuation.py
@@ -5,6 +5,8 @@
 from pypy.interpreter.baseobjspace import Wrappable
 from pypy.interpreter.typedef import TypeDef
 from pypy.interpreter.gateway import interp2app
+from pypy.interpreter.pycode import PyCode
+from pypy.interpreter.pyframe import PyFrame
 
 
 class W_Continulet(Wrappable):
@@ -20,66 +22,69 @@
     def check_sthread(self):
         ec = self.space.getexecutioncontext()
         if ec.stacklet_thread is not self.sthread:
-            start_state.clear()
+            global_state.clear()
             raise geterror(self.space, "inter-thread support is missing")
-        return ec
 
     def descr_init(self, w_callable, __args__):
         if self.sthread is not None:
             raise geterror(self.space, "continulet already __init__ialized")
-        start_state.origin = self
-        start_state.w_callable = w_callable
-        start_state.args = __args__
-        self.sthread = build_sthread(self.space)
-        try:
-            self.h = self.sthread.new(new_stacklet_callback)
-            if self.sthread.is_empty_handle(self.h):    # early return
-                raise MemoryError
-        except MemoryError:
-            self.sthread = None
-            start_state.clear()
-            raise getmemoryerror(self.space)
+        #
+        # hackish: build the frame "by hand", passing it the correct arguments
+        space = self.space
+        w_args, w_kwds = __args__.topacked()
+        bottomframe = space.createframe(get_entrypoint_pycode(space),
+                                        get_w_module_dict(space), None)
+        bottomframe.locals_stack_w[0] = space.wrap(self)
+        bottomframe.locals_stack_w[1] = w_callable
+        bottomframe.locals_stack_w[2] = w_args
+        bottomframe.locals_stack_w[3] = w_kwds
+        self.bottomframe = bottomframe
+        #
+        global_state.origin = self
+        sthread = build_sthread(self.space)
+        self.sthread = sthread
+        h = sthread.new(new_stacklet_callback)
+        post_switch(sthread, h)
 
     def switch(self, w_to):
+        sthread = self.sthread
+        if sthread is not None and sthread.is_empty_handle(self.h):
+            global_state.clear()
+            raise geterror(self.space, "continulet already finished")
         to = self.space.interp_w(W_Continulet, w_to, can_be_None=True)
+        if to is not None and to.sthread is None:
+            to = None
+        if sthread is None:      # if self is non-initialized:
+            if to is not None:   #     if we are given a 'to'
+                self = to        #         then just use it and ignore 'self'
+                sthread = self.sthread
+                to = None
+            else:
+                return get_result()  # else: no-op
         if to is not None:
-            if to.sthread is None:
-                start_state.clear()
-                raise geterror(self.space, "continulet not initialized yet")
+            if to.sthread is not sthread:
+                global_state.clear()
+                raise geterror(self.space, "cross-thread double switch")
             if self is to:    # double-switch to myself: no-op
                 return get_result()
-        if self.sthread is None:
-            start_state.clear()
-            raise geterror(self.space, "continulet not initialized yet")
-        ec = self.check_sthread()
-        saved_topframeref = ec.topframeref
+            if sthread.is_empty_handle(to.h):
+                global_state.clear()
+                raise geterror(self.space, "continulet already finished")
+        self.check_sthread()
         #
-        start_state.origin = self
+        global_state.origin = self
         if to is None:
             # simple switch: going to self.h
-            start_state.destination = self
+            global_state.destination = self
         else:
             # double switch: the final destination is to.h
-            start_state.destination = to
+            global_state.destination = to
         #
-        h = start_state.destination.h
-        sthread = self.sthread
-        if sthread.is_empty_handle(h):
-            start_state.clear()
-            raise geterror(self.space, "continulet already finished")
-        #
-        try:
-            do_switch(sthread, h)
-        except MemoryError:
-            start_state.clear()
-            raise getmemoryerror(self.space)
-        #
-        ec = sthread.ec
-        ec.topframeref = saved_topframeref
-        return get_result()
+        h = sthread.switch(global_state.destination.h)
+        return post_switch(sthread, h)
 
     def descr_switch(self, w_value=None, w_to=None):
-        start_state.w_value = w_value
+        global_state.w_value = w_value
         return self.switch(w_to)
 
     def descr_throw(self, w_type, w_val=None, w_tb=None, w_to=None):
@@ -94,8 +99,8 @@
         #
         operr = OperationError(w_type, w_val, tb)
         operr.normalize_exception(space)
-        start_state.w_value = None
-        start_state.propagate_exception = operr
+        global_state.w_value = None
+        global_state.propagate_exception = operr
         return self.switch(w_to)
 
     def descr_is_pending(self):
@@ -103,12 +108,26 @@
                  and not self.sthread.is_empty_handle(self.h))
         return self.space.newbool(valid)
 
+    def descr__reduce__(self):
+        from pypy.module._continuation import interp_pickle
+        return interp_pickle.reduce(self)
+
+    def descr__setstate__(self, w_args):
+        from pypy.module._continuation import interp_pickle
+        interp_pickle.setstate(self, w_args)
+
 
 def W_Continulet___new__(space, w_subtype, __args__):
     r = space.allocate_instance(W_Continulet, w_subtype)
     r.__init__(space)
     return space.wrap(r)
 
+def unpickle(space, w_subtype):
+    """Pickle support."""
+    r = space.allocate_instance(W_Continulet, w_subtype)
+    r.__init__(space)
+    return space.wrap(r)
+
 
 W_Continulet.typedef = TypeDef(
     'continulet',
@@ -118,26 +137,52 @@
     switch      = interp2app(W_Continulet.descr_switch),
     throw       = interp2app(W_Continulet.descr_throw),
     is_pending  = interp2app(W_Continulet.descr_is_pending),
+    __reduce__  = interp2app(W_Continulet.descr__reduce__),
+    __setstate__= interp2app(W_Continulet.descr__setstate__),
     )
 
-
 # ____________________________________________________________
 
+# Continulet objects maintain a dummy frame object in order to ensure
+# that the 'f_back' chain is consistent.  We hide this dummy frame
+# object by giving it a dummy code object with hidden_applevel=True.
 
 class State:
     def __init__(self, space):
-        self.space = space 
+        self.space = space
         w_module = space.getbuiltinmodule('_continuation')
         self.w_error = space.getattr(w_module, space.wrap('error'))
-        self.w_memoryerror = OperationError(space.w_MemoryError, space.w_None)
+        # the following function switches away immediately, so that
+        # continulet.__init__() doesn't immediately run func(), but it
+        # also has the hidden purpose of making sure we have a single
+        # bottomframe for the whole duration of the continulet's run.
+        # Hackish: only the func_code is used, and used in the context
+        # of w_globals == this module, so we can access the name
+        # 'continulet' directly.
+        w_code = space.appexec([], '''():
+            def start(c, func, args, kwds):
+                if continulet.switch(c) is not None:
+                    raise TypeError(
+                     "can\'t send non-None value to a just-started continulet")
+                return func(c, *args, **kwds)
+            return start.func_code
+        ''')
+        self.entrypoint_pycode = space.interp_w(PyCode, w_code)
+        self.entrypoint_pycode.hidden_applevel = True
+        self.w_unpickle = w_module.get('_p')
+        self.w_module_dict = w_module.getdict(space)
 
 def geterror(space, message):
     cs = space.fromcache(State)
     return OperationError(cs.w_error, space.wrap(message))
 
-def getmemoryerror(space):
+def get_entrypoint_pycode(space):
     cs = space.fromcache(State)
-    return cs.w_memoryerror
+    return cs.entrypoint_pycode
+
+def get_w_module_dict(space):
+    cs = space.fromcache(State)
+    return cs.w_module_dict
 
 # ____________________________________________________________
 
@@ -148,71 +193,63 @@
         StackletThread.__init__(self, space.config)
         self.space = space
         self.ec = ec
+        # for unpickling
+        from pypy.rlib.rweakref import RWeakKeyDictionary
+        self.frame2continulet = RWeakKeyDictionary(PyFrame, W_Continulet)
 
 ExecutionContext.stacklet_thread = None
 
 # ____________________________________________________________
 
 
-class StartState:   # xxx a single global to pass around the function to start
+class GlobalState:
     def clear(self):
         self.origin = None
         self.destination = None
-        self.w_callable = None
-        self.args = None
         self.w_value = None
         self.propagate_exception = None
-start_state = StartState()
-start_state.clear()
+global_state = GlobalState()
+global_state.clear()
 
 
 def new_stacklet_callback(h, arg):
-    self       = start_state.origin
-    w_callable = start_state.w_callable
-    args       = start_state.args
-    start_state.clear()
-    try:
-        do_switch(self.sthread, h)
-    except MemoryError:
-        return h       # oups!  do an early return in this case
-    #
+    self = global_state.origin
+    self.h = h
+    global_state.clear()
     space = self.space
     try:
-        ec = self.sthread.ec
-        ec.topframeref = jit.vref_None
-
-        if start_state.propagate_exception is not None:
-            raise start_state.propagate_exception   # just propagate it further
-        if start_state.w_value is not space.w_None:
-            raise OperationError(space.w_TypeError, space.wrap(
-                "can't send non-None value to a just-started continulet"))
-
-        args = args.prepend(self.space.wrap(self))
-        w_result = space.call_args(w_callable, args)
+        frame = self.bottomframe
+        w_result = frame.execute_frame()
     except Exception, e:
-        start_state.propagate_exception = e
+        global_state.propagate_exception = e
     else:
-        start_state.w_value = w_result
-    start_state.origin = self
-    start_state.destination = self
+        global_state.w_value = w_result
+    self.sthread.ec.topframeref = jit.vref_None
+    global_state.origin = self
+    global_state.destination = self
     return self.h
 
-
-def do_switch(sthread, h):
-    h = sthread.switch(h)
-    origin = start_state.origin
-    self = start_state.destination
-    start_state.origin = None
-    start_state.destination = None
+def post_switch(sthread, h):
+    origin = global_state.origin
+    self = global_state.destination
+    global_state.origin = None
+    global_state.destination = None
     self.h, origin.h = origin.h, h
+    #
+    current = sthread.ec.topframeref
+    sthread.ec.topframeref = self.bottomframe.f_backref
+    self.bottomframe.f_backref = origin.bottomframe.f_backref
+    origin.bottomframe.f_backref = current
+    #
+    return get_result()
 
 def get_result():
-    if start_state.propagate_exception:
-        e = start_state.propagate_exception
-        start_state.propagate_exception = None
+    if global_state.propagate_exception:
+        e = global_state.propagate_exception
+        global_state.propagate_exception = None
         raise e
-    w_value = start_state.w_value
-    start_state.w_value = None
+    w_value = global_state.w_value
+    global_state.w_value = None
     return w_value
 
 def build_sthread(space):
@@ -232,7 +269,7 @@
         cont = space.interp_w(W_Continulet, w_cont)
         if cont.sthread is not sthread:
             if cont.sthread is None:
-                raise geterror(space, "got a non-initialized continulet")
+                continue   # ignore non-initialized continulets
             else:
                 raise geterror(space, "inter-thread support is missing")
         elif sthread.is_empty_handle(cont.h):
@@ -240,6 +277,9 @@
         contlist.append(cont)
     #
     if len(contlist) > 1:
-        other = contlist[-1].h
+        otherh = contlist[-1].h
+        otherb = contlist[-1].bottomframe.f_backref
         for cont in contlist:
-            other, cont.h = cont.h, other
+            otherh, cont.h = cont.h, otherh
+            b = cont.bottomframe
+            otherb, b.f_backref = b.f_backref, otherb
diff --git a/pypy/module/_continuation/interp_pickle.py b/pypy/module/_continuation/interp_pickle.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_continuation/interp_pickle.py
@@ -0,0 +1,128 @@
+from pypy.tool import stdlib_opcode as pythonopcode
+from pypy.rlib import jit
+from pypy.interpreter.error import OperationError
+from pypy.interpreter.pyframe import PyFrame
+from pypy.module._continuation.interp_continuation import State, global_state
+from pypy.module._continuation.interp_continuation import build_sthread
+from pypy.module._continuation.interp_continuation import post_switch
+from pypy.module._continuation.interp_continuation import get_result, geterror
+
+
+def getunpickle(space):
+    cs = space.fromcache(State)
+    return cs.w_unpickle
+
+
+def reduce(self):
+    # xxx this is known to be not completely correct with respect
+    # to subclasses, e.g. no __slots__ support, no looking for a
+    # __getnewargs__ or __getstate__ defined in the subclass, etc.
+    # Doing the right thing looks involved, though...
+    space = self.space
+    if self.sthread is None:
+        w_frame = space.w_False
+    elif self.sthread.is_empty_handle(self.h):
+        w_frame = space.w_None
+    else:
+        w_frame = space.wrap(self.bottomframe)
+    w_continulet_type = space.type(space.wrap(self))
+    w_dict = self.getdict(space) or space.w_None
+    args = [getunpickle(space),
+            space.newtuple([w_continulet_type]),
+            space.newtuple([w_frame, w_dict]),
+            ]
+    return space.newtuple(args)
+
+def setstate(self, w_args):
+    space = self.space
+    if self.sthread is not None:
+        raise geterror(space, "continulet.__setstate__() on an already-"
+                              "initialized continulet")
+    w_frame, w_dict = space.fixedview(w_args, expected_length=2)
+    if not space.is_w(w_dict, space.w_None):
+        self.setdict(space, w_dict)
+    if space.is_w(w_frame, space.w_False):
+        return    # not initialized
+    sthread = build_sthread(self.space)
+    self.sthread = sthread
+    self.bottomframe = space.interp_w(PyFrame, w_frame, can_be_None=True)
+    #
+    global_state.origin = self
+    if self.bottomframe is not None:
+        sthread.frame2continulet.set(self.bottomframe, self)
+    self.h = sthread.new(resume_trampoline_callback)
+    get_result()    # propagate the eventual MemoryError
+
+# ____________________________________________________________
+
+def resume_trampoline_callback(h, arg):
+    self = global_state.origin
+    self.h = h
+    space = self.space
+    sthread = self.sthread
+    try:
+        global_state.clear()
+        if self.bottomframe is None:
+            w_result = space.w_None
+        else:
+            h = sthread.switch(self.h)
+            try:
+                w_result = post_switch(sthread, h)
+                operr = None
+            except OperationError, e:
+                w_result = None
+                operr = e
+            #
+            while True:
+                ec = sthread.ec
+                frame = ec.topframeref()
+                assert frame is not None     # XXX better error message
+                exit_continulet = sthread.frame2continulet.get(frame)
+                #
+                continue_after_call(frame)
+                #
+                # small hack: unlink frame out of the execution context,
+                # because execute_frame will add it there again
+                ec.topframeref = frame.f_backref
+                #
+                try:
+                    w_result = frame.execute_frame(w_result, operr)
+                    operr = None
+                except OperationError, e:
+                    w_result = None
+                    operr = e
+                if exit_continulet is not None:
+                    self = exit_continulet
+                    break
+            sthread.ec.topframeref = jit.vref_None
+            if operr:
+                raise operr
+    except Exception, e:
+        global_state.propagate_exception = e
+    else:
+        global_state.w_value = w_result
+    global_state.origin = self
+    global_state.destination = self
+    return self.h
+
+def continue_after_call(frame):
+    code = frame.pycode.co_code
+    instr = frame.last_instr
+    opcode = ord(code[instr])
+    map = pythonopcode.opmap
+    call_ops = [map['CALL_FUNCTION'], map['CALL_FUNCTION_KW'],
+                map['CALL_FUNCTION_VAR'], map['CALL_FUNCTION_VAR_KW'],
+                map['CALL_METHOD']]
+    assert opcode in call_ops   # XXX check better, and complain better
+    instr += 1
+    oparg = ord(code[instr]) | ord(code[instr + 1]) << 8
+    nargs = oparg & 0xff
+    nkwds = (oparg >> 8) & 0xff
+    if nkwds == 0:     # only positional arguments
+        # fast paths leaves things on the stack, pop them
+        if (frame.space.config.objspace.opcodes.CALL_METHOD and
+            opcode == map['CALL_METHOD']):
+            frame.dropvalues(nargs + 2)
+        elif opcode == map['CALL_FUNCTION']:
+            frame.dropvalues(nargs + 1)
+    frame.last_instr = instr + 1    # continue after the call
diff --git a/pypy/module/_continuation/test/support.py b/pypy/module/_continuation/test/support.py
--- a/pypy/module/_continuation/test/support.py
+++ b/pypy/module/_continuation/test/support.py
@@ -9,4 +9,4 @@
             import pypy.rlib.rstacklet
         except CompilationError, e:
             py.test.skip("cannot import rstacklet: %s" % e)
-        cls.space = gettestobjspace(usemodules=['_continuation'])
+        cls.space = gettestobjspace(usemodules=['_continuation'], continuation=True)
diff --git a/pypy/module/_continuation/test/test_stacklet.py b/pypy/module/_continuation/test/test_stacklet.py
--- a/pypy/module/_continuation/test/test_stacklet.py
+++ b/pypy/module/_continuation/test/test_stacklet.py
@@ -13,7 +13,7 @@
         from _continuation import continulet
         #
         def empty_callback(c):
-            pass
+            never_called
         #
         c = continulet(empty_callback)
         assert type(c) is continulet
@@ -36,7 +36,7 @@
         from _continuation import continulet, error
         #
         def empty_callback(c1):
-            pass
+            never_called
         #
         c = continulet(empty_callback)
         raises(error, c.__init__, empty_callback)
@@ -135,12 +135,6 @@
         e = raises(error, c.switch)
         assert str(e.value) == "continulet already finished"
 
-    def test_not_initialized_yet(self):
-        from _continuation import continulet, error
-        c = continulet.__new__(continulet)
-        e = raises(error, c.switch)
-        assert str(e.value) == "continulet not initialized yet"
-
     def test_go_depth2(self):
         from _continuation import continulet
         #
@@ -254,6 +248,15 @@
         res = c_upper.switch('D')
         assert res == 'E'
 
+    def test_switch_not_initialized(self):
+        from _continuation import continulet
+        c0 = continulet.__new__(continulet)
+        res = c0.switch()
+        assert res is None
+        res = c0.switch(123)
+        assert res == 123
+        raises(ValueError, c0.throw, ValueError)
+
     def test_exception_with_switch_depth2(self):
         from _continuation import continulet
         #
@@ -312,7 +315,7 @@
         res = f()
         assert res == 2002
 
-    def test_f_back_is_None_for_now(self):
+    def test_f_back(self):
         import sys
         from _continuation import continulet
         #
@@ -321,6 +324,7 @@
             c.switch(sys._getframe(0).f_back)
             c.switch(sys._getframe(1))
             c.switch(sys._getframe(1).f_back)
+            assert sys._getframe(2) is f3.f_back
             c.switch(sys._getframe(2))
         def f(c):
             g(c)
@@ -331,10 +335,21 @@
         f2 = c.switch()
         assert f2.f_code.co_name == 'f'
         f3 = c.switch()
-        assert f3.f_code.co_name == 'f'
-        f4 = c.switch()
-        assert f4 is None
-        raises(ValueError, c.switch)    # "call stack is not deep enough"
+        assert f3 is f2
+        assert f1.f_back is f3
+        def main():
+            f4 = c.switch()
+            assert f4.f_code.co_name == 'main', repr(f4.f_code.co_name)
+            assert f3.f_back is f1    # not running, so a loop
+        def main2():
+            f5 = c.switch()
+            assert f5.f_code.co_name == 'main2', repr(f5.f_code.co_name)
+            assert f3.f_back is f1    # not running, so a loop
+        main()
+        main2()
+        res = c.switch()
+        assert res is None
+        assert f3.f_back is None
 
     def test_traceback_is_complete(self):
         import sys
@@ -487,16 +502,31 @@
         assert res == 'z'
         raises(TypeError, c1.switch, to=c2)  # "can't send non-None value"
 
-    def test_switch2_not_initialized_yet(self):
-        from _continuation import continulet, error
+    def test_switch2_not_initialized(self):
+        from _continuation import continulet
+        c0 = continulet.__new__(continulet)
+        c0bis = continulet.__new__(continulet)
+        res = c0.switch(123, to=c0)
+        assert res == 123
+        res = c0.switch(123, to=c0bis)
+        assert res == 123
+        raises(ValueError, c0.throw, ValueError, to=c0)
+        raises(ValueError, c0.throw, ValueError, to=c0bis)
         #
         def f1(c1):
-            not_reachable
-        #
+            c1.switch('a')
+            raises(ValueError, c1.switch, 'b')
+            raises(KeyError, c1.switch, 'c')
+            return 'd'
         c1 = continulet(f1)
-        c2 = continulet.__new__(continulet)
-        e = raises(error, c1.switch, to=c2)
-        assert str(e.value) == "continulet not initialized yet"
+        res = c0.switch(to=c1)
+        assert res == 'a'
+        res = c1.switch(to=c0)
+        assert res == 'b'
+        res = c1.throw(ValueError, to=c0)
+        assert res == 'c'
+        res = c0.throw(KeyError, to=c1)
+        assert res == 'd'
 
     def test_switch2_already_finished(self):
         from _continuation import continulet, error
@@ -609,6 +639,7 @@
         assert res == "ok"
 
     def test_permute(self):
+        import sys
         from _continuation import continulet, permute
         #
         def f1(c1):
@@ -617,14 +648,34 @@
             return "done"
         #
         def f2(c2):
+            assert sys._getframe(1).f_code.co_name == 'main'
             permute(c1, c2)
+            assert sys._getframe(1).f_code.co_name == 'f1'
             return "ok"
         #
         c1 = continulet(f1)
         c2 = continulet(f2)
+        def main():
+            c1.switch()
+            res = c2.switch()
+            assert res == "done"
+        main()
+
+    def test_permute_noninitialized(self):
+        from _continuation import continulet, permute
+        permute(continulet.__new__(continulet))    # ignored
+        permute(continulet.__new__(continulet),    # ignored
+                continulet.__new__(continulet))
+
+    def test_bug_finish_with_already_finished_stacklet(self):
+        from _continuation import continulet, error
+        # make an already-finished continulet
+        c1 = continulet(lambda x: x)
         c1.switch()
-        res = c2.switch()
-        assert res == "done"
+        # make another continulet
+        c2 = continulet(lambda x: x)
+        # this switch is forbidden, because it causes a crash when c2 finishes
+        raises(error, c1.switch, to=c2)
 
     def test_various_depths(self):
         skip("may fail on top of CPython")
diff --git a/pypy/module/_continuation/test/test_zpickle.py b/pypy/module/_continuation/test/test_zpickle.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_continuation/test/test_zpickle.py
@@ -0,0 +1,262 @@
+from pypy.conftest import gettestobjspace
+
+
+class AppTestCopy:
+    def setup_class(cls):
+        cls.space = gettestobjspace(usemodules=('_continuation',),
+                                    CALL_METHOD=True)
+        cls.space.config.translation.continuation = True
+
+    def test_basic_setup(self):
+        from _continuation import continulet
+        lst = [4]
+        co = continulet(lst.append)
+        assert lst == [4]
+        res = co.switch()
+        assert res is None
+        assert lst == [4, co]
+
+    def test_copy_continulet_not_started(self):
+        from _continuation import continulet, error
+        import copy
+        lst = []
+        co = continulet(lst.append)
+        co2, lst2 = copy.deepcopy((co, lst))
+        #
+        assert lst == []
+        co.switch()
+        assert lst == [co]
+        #
+        assert lst2 == []
+        co2.switch()
+        assert lst2 == [co2]
+
+    def test_copy_continulet_not_started_multiple(self):
+        from _continuation import continulet, error
+        import copy
+        lst = []
+        co = continulet(lst.append)
+        co2, lst2 = copy.deepcopy((co, lst))
+        co3, lst3 = copy.deepcopy((co, lst))
+        co4, lst4 = copy.deepcopy((co, lst))
+        #
+        assert lst == []
+        co.switch()
+        assert lst == [co]
+        #
+        assert lst2 == []
+        co2.switch()
+        assert lst2 == [co2]
+        #
+        assert lst3 == []
+        co3.switch()
+        assert lst3 == [co3]
+        #
+        assert lst4 == []
+        co4.switch()
+        assert lst4 == [co4]
+
+    def test_copy_continulet_real(self):
+        import new, sys
+        mod = new.module('test_copy_continulet_real')
+        sys.modules['test_copy_continulet_real'] = mod
+        exec '''if 1:
+            from _continuation import continulet
+            import copy
+            def f(co, x):
+                co.switch(x + 1)
+                co.switch(x + 2)
+                return x + 3
+            co = continulet(f, 40)
+            res = co.switch()
+            assert res == 41
+            co2 = copy.deepcopy(co)
+            #
+            res = co2.switch()
+            assert res == 42
+            assert co2.is_pending()
+            res = co2.switch()
+            assert res == 43
+            assert not co2.is_pending()
+            #
+            res = co.switch()
+            assert res == 42
+            assert co.is_pending()
+            res = co.switch()
+            assert res == 43
+            assert not co.is_pending()
+        ''' in mod.__dict__
+
+    def test_copy_continulet_already_finished(self):
+        from _continuation import continulet, error
+        import copy
+        lst = []
+        co = continulet(lst.append)
+        co.switch()
+        co2 = copy.deepcopy(co)
+        assert not co.is_pending()
+        assert not co2.is_pending()
+        raises(error, co.__init__, lst.append)
+        raises(error, co2.__init__, lst.append)
+        raises(error, co.switch)
+        raises(error, co2.switch)
+
+
+class AppTestPickle:
+    version = 0
+
+    def setup_class(cls):
+        cls.space = gettestobjspace(usemodules=('_continuation',),
+                                    CALL_METHOD=True)
+        cls.space.appexec([], """():
+            global continulet, A, __name__
+
+            import sys
+            __name__ = 'test_pickle_continulet'
+            thismodule = type(sys)(__name__)
+            sys.modules[__name__] = thismodule
+
+            from _continuation import continulet
+            class A(continulet):
+                pass
+
+            thismodule.__dict__.update(globals())
+        """)
+        cls.w_version = cls.space.wrap(cls.version)
+
+    def test_pickle_continulet_empty(self):
+        from _continuation import continulet
+        lst = [4]
+        co = continulet.__new__(continulet)
+        import pickle
+        pckl = pickle.dumps(co, self.version)
+        print repr(pckl)
+        co2 = pickle.loads(pckl)
+        assert co2 is not co
+        assert not co.is_pending()
+        assert not co2.is_pending()
+        # the empty unpickled coroutine can still be used:
+        result = [5]
+        co2.__init__(result.append)
+        res = co2.switch()
+        assert res is None
+        assert result == [5, co2]
+
+    def test_pickle_continulet_empty_subclass(self):
+        from test_pickle_continulet import continulet, A
+        lst = [4]
+        co = continulet.__new__(A)
+        co.foo = 'bar'
+        co.bar = 'baz'
+        import pickle
+        pckl = pickle.dumps(co, self.version)
+        print repr(pckl)
+        co2 = pickle.loads(pckl)
+        assert co2 is not co
+        assert not co.is_pending()
+        assert not co2.is_pending()
+        assert type(co) is type(co2) is A
+        assert co.foo == co2.foo == 'bar'
+        assert co.bar == co2.bar == 'baz'
+        # the empty unpickled coroutine can still be used:
+        result = [5]
+        co2.__init__(result.append)
+        res = co2.switch()
+        assert res is None
+        assert result == [5, co2]
+
+    def test_pickle_continulet_not_started(self):
+        from _continuation import continulet, error
+        import pickle
+        lst = []
+        co = continulet(lst.append)
+        pckl = pickle.dumps((co, lst))
+        print pckl
+        del co, lst
+        for i in range(2):
+            print 'resume...'
+            co2, lst2 = pickle.loads(pckl)
+            assert lst2 == []
+            co2.switch()
+            assert lst2 == [co2]
+
+    def test_pickle_continulet_real(self):
+        import new, sys
+        mod = new.module('test_pickle_continulet_real')
+        sys.modules['test_pickle_continulet_real'] = mod
+        mod.version = self.version
+        exec '''if 1:
+            from _continuation import continulet
+            import pickle
+            def f(co, x):
+                co.switch(x + 1)
+                co.switch(x + 2)
+                return x + 3
+            co = continulet(f, 40)
+            res = co.switch()
+            assert res == 41
+            pckl = pickle.dumps(co, version)
+            print repr(pckl)
+            co2 = pickle.loads(pckl)
+            #
+            res = co2.switch()
+            assert res == 42
+            assert co2.is_pending()
+            res = co2.switch()
+            assert res == 43
+            assert not co2.is_pending()
+            #
+            res = co.switch()
+            assert res == 42
+            assert co.is_pending()
+            res = co.switch()
+            assert res == 43
+            assert not co.is_pending()
+        ''' in mod.__dict__
+
+    def test_pickle_continulet_real_subclass(self):
+        import new, sys
+        mod = new.module('test_pickle_continulet_real_subclass')
+        sys.modules['test_pickle_continulet_real_subclass'] = mod
+        mod.version = self.version
+        exec '''if 1:
+            from _continuation import continulet
+            import pickle
+            class A(continulet):
+                def __init__(self):
+                    crash
+            def f(co):
+                co.switch(co.x + 1)
+                co.switch(co.x + 2)
+                return co.x + 3
+            co = A.__new__(A)
+            continulet.__init__(co, f)
+            co.x = 40
+            res = co.switch()
+            assert res == 41
+            pckl = pickle.dumps(co, version)
+            print repr(pckl)
+            co2 = pickle.loads(pckl)
+            #
+            assert type(co2) is A
+            res = co2.switch()
+            assert res == 42
+            assert co2.is_pending()
+            res = co2.switch()
+            assert res == 43
+            assert not co2.is_pending()
+            #
+            res = co.switch()
+            assert res == 42
+            assert co.is_pending()
+            res = co.switch()
+            assert res == 43
+            assert not co.is_pending()
+        ''' in mod.__dict__
+
+
+class AppTestPickle_v1(AppTestPickle):
+    version = 1
+
+class AppTestPickle_v2(AppTestPickle):
+    version = 2
diff --git a/pypy/module/_multiprocessing/interp_connection.py b/pypy/module/_multiprocessing/interp_connection.py
--- a/pypy/module/_multiprocessing/interp_connection.py
+++ b/pypy/module/_multiprocessing/interp_connection.py
@@ -225,7 +225,9 @@
             except OSError:
                 pass
 
-    def __init__(self, fd, flags):
+    def __init__(self, space, fd, flags):
+        if fd == self.INVALID_HANDLE_VALUE or fd < 0:
+            raise OperationError(space.w_IOError, space.wrap("invalid handle %d" % fd))
         W_BaseConnection.__init__(self, flags)
         self.fd = fd
 
@@ -234,7 +236,7 @@
         flags = (readable and READABLE) | (writable and WRITABLE)
 
         self = space.allocate_instance(W_FileConnection, w_subtype)
-        W_FileConnection.__init__(self, fd, flags)
+        W_FileConnection.__init__(self, space, fd, flags)
         return space.wrap(self)
 
     def fileno(self, space):
diff --git a/pypy/module/_multiprocessing/interp_semaphore.py b/pypy/module/_multiprocessing/interp_semaphore.py
--- a/pypy/module/_multiprocessing/interp_semaphore.py
+++ b/pypy/module/_multiprocessing/interp_semaphore.py
@@ -468,6 +468,9 @@
 
         self.count -= 1
 
+    def after_fork(self):
+        self.count = 0
+
     @unwrap_spec(kind=int, maxvalue=int)
     def rebuild(space, w_cls, w_handle, kind, maxvalue):
         self = space.allocate_instance(W_SemLock, w_cls)
@@ -512,6 +515,7 @@
     acquire = interp2app(W_SemLock.acquire),
     release = interp2app(W_SemLock.release),
     _rebuild = interp2app(W_SemLock.rebuild.im_func, as_classmethod=True),
+    _after_fork = interp2app(W_SemLock.after_fork),
     __enter__=interp2app(W_SemLock.enter),
     __exit__=interp2app(W_SemLock.exit),
     SEM_VALUE_MAX=SEM_VALUE_MAX,
diff --git a/pypy/module/_multiprocessing/test/test_connection.py b/pypy/module/_multiprocessing/test/test_connection.py
--- a/pypy/module/_multiprocessing/test/test_connection.py
+++ b/pypy/module/_multiprocessing/test/test_connection.py
@@ -145,3 +145,9 @@
             else:
                 c.close()
         space.delslice(w_connections, space.wrap(0), space.wrap(100))
+
+    def test_bad_fd(self):
+        import _multiprocessing
+
+        raises(IOError, _multiprocessing.Connection, -1)
+        raises(IOError, _multiprocessing.Connection, -15)
\ No newline at end of file
diff --git a/pypy/module/_multiprocessing/test/test_semaphore.py b/pypy/module/_multiprocessing/test/test_semaphore.py
--- a/pypy/module/_multiprocessing/test/test_semaphore.py
+++ b/pypy/module/_multiprocessing/test/test_semaphore.py
@@ -39,6 +39,10 @@
         sem.release()
         assert sem._count() == 0
 
+        sem.acquire()
+        sem._after_fork()
+        assert sem._count() == 0
+
     def test_recursive(self):
         from _multiprocessing import SemLock
         kind = self.RECURSIVE
diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py
--- a/pypy/module/_sre/interp_sre.py
+++ b/pypy/module/_sre/interp_sre.py
@@ -99,6 +99,7 @@
 # SRE_Pattern class
 
 class W_SRE_Pattern(Wrappable):
+    _immutable_fields_ = ["code", "flags"]
 
     def cannot_copy_w(self):
         space = self.space
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -52,7 +52,8 @@
 constants["CERT_OPTIONAL"] = PY_SSL_CERT_OPTIONAL
 constants["CERT_REQUIRED"] = PY_SSL_CERT_REQUIRED
 
-constants["PROTOCOL_SSLv2"]  = PY_SSL_VERSION_SSL2
+if not OPENSSL_NO_SSL2:
+    constants["PROTOCOL_SSLv2"]  = PY_SSL_VERSION_SSL2
 constants["PROTOCOL_SSLv3"]  = PY_SSL_VERSION_SSL3
 constants["PROTOCOL_SSLv23"] = PY_SSL_VERSION_SSL23
 constants["PROTOCOL_TLSv1"]  = PY_SSL_VERSION_TLS1
@@ -673,7 +674,7 @@
         method = libssl_TLSv1_method()
     elif protocol == PY_SSL_VERSION_SSL3:
         method = libssl_SSLv3_method()
-    elif protocol == PY_SSL_VERSION_SSL2:
+    elif protocol == PY_SSL_VERSION_SSL2 and not OPENSSL_NO_SSL2:
         method = libssl_SSLv2_method()
     elif protocol == PY_SSL_VERSION_SSL23:
         method = libssl_SSLv23_method()
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -57,6 +57,9 @@
         compile_extra=['-DPy_BUILD_CORE'],
         )
 
+class CConfig2:
+    _compilation_info_ = CConfig._compilation_info_
+
 class CConfig_constants:
     _compilation_info_ = CConfig._compilation_info_
 
@@ -300,9 +303,13 @@
         return unwrapper_raise # used in 'normal' RPython code.
     return decorate
 
-def cpython_struct(name, fields, forward=None):
+def cpython_struct(name, fields, forward=None, level=1):
     configname = name.replace(' ', '__')
-    setattr(CConfig, configname, rffi_platform.Struct(name, fields))
+    if level == 1:
+        config = CConfig
+    else:
+        config = CConfig2
+    setattr(config, configname, rffi_platform.Struct(name, fields))
     if forward is None:
         forward = lltype.ForwardReference()
     TYPES[configname] = forward
@@ -445,9 +452,10 @@
 #              'int*': rffi.INTP}
 
 def configure_types():
-    for name, TYPE in rffi_platform.configure(CConfig).iteritems():
-        if name in TYPES:
-            TYPES[name].become(TYPE)
+    for config in (CConfig, CConfig2):
+        for name, TYPE in rffi_platform.configure(config).iteritems():
+            if name in TYPES:
+                TYPES[name].become(TYPE)
 
 def build_type_checkers(type_name, cls=None):
     """
diff --git a/pypy/module/cpyext/funcobject.py b/pypy/module/cpyext/funcobject.py
--- a/pypy/module/cpyext/funcobject.py
+++ b/pypy/module/cpyext/funcobject.py
@@ -4,9 +4,21 @@
     cpython_api, bootstrap_function, cpython_struct, build_type_checkers)
 from pypy.module.cpyext.pyobject import (
     PyObject, make_ref, from_ref, Py_DecRef, make_typedescr, borrow_from)
+from pypy.rlib.unroll import unrolling_iterable
 from pypy.interpreter.error import OperationError
 from pypy.interpreter.function import Function, Method
 from pypy.interpreter.pycode import PyCode
+from pypy.interpreter import pycode
+
+CODE_FLAGS = dict(
+    CO_OPTIMIZED   = 0x0001,
+    CO_NEWLOCALS   = 0x0002,
+    CO_VARARGS     = 0x0004,
+    CO_VARKEYWORDS = 0x0008,
+    CO_NESTED      = 0x0010,
+    CO_GENERATOR   = 0x0020,
+)
+ALL_CODE_FLAGS = unrolling_iterable(CODE_FLAGS.items())
 
 PyFunctionObjectStruct = lltype.ForwardReference()
 PyFunctionObject = lltype.Ptr(PyFunctionObjectStruct)
@@ -16,7 +28,12 @@
 
 PyCodeObjectStruct = lltype.ForwardReference()
 PyCodeObject = lltype.Ptr(PyCodeObjectStruct)
-cpython_struct("PyCodeObject", PyObjectFields, PyCodeObjectStruct)
+PyCodeObjectFields = PyObjectFields + \
+    (("co_name", PyObject),
+     ("co_flags", rffi.INT),
+     ("co_argcount", rffi.INT),
+    )
+cpython_struct("PyCodeObject", PyCodeObjectFields, PyCodeObjectStruct)
 
 @bootstrap_function
 def init_functionobject(space):
@@ -24,6 +41,10 @@
                    basestruct=PyFunctionObject.TO,
                    attach=function_attach,
                    dealloc=function_dealloc)
+    make_typedescr(PyCode.typedef,
+                   basestruct=PyCodeObject.TO,
+                   attach=code_attach,
+                   dealloc=code_dealloc)
 
 PyFunction_Check, PyFunction_CheckExact = build_type_checkers("Function", Function)
 PyMethod_Check, PyMethod_CheckExact = build_type_checkers("Method", Method)
@@ -40,6 +61,31 @@
     from pypy.module.cpyext.object import PyObject_dealloc
     PyObject_dealloc(space, py_obj)
 
+def code_attach(space, py_obj, w_obj):
+    py_code = rffi.cast(PyCodeObject, py_obj)
+    assert isinstance(w_obj, PyCode)
+    py_code.c_co_name = make_ref(space, space.wrap(w_obj.co_name))
+    co_flags = 0
+    for name, value in ALL_CODE_FLAGS:
+        if w_obj.co_flags & getattr(pycode, name):
+            co_flags |= value
+    rffi.setintfield(py_code, 'c_co_flags', co_flags)
+    rffi.setintfield(py_code, 'c_co_argcount', w_obj.co_argcount)
+
+ at cpython_api([PyObject], lltype.Void, external=False)
+def code_dealloc(space, py_obj):
+    py_code = rffi.cast(PyCodeObject, py_obj)
+    Py_DecRef(space, py_code.c_co_name)
+    from pypy.module.cpyext.object import PyObject_dealloc
+    PyObject_dealloc(space, py_obj)
+
+ at cpython_api([PyObject], PyObject)
+def PyFunction_GetCode(space, w_func):
+    """Return the code object associated with the function object op."""
+    func = space.interp_w(Function, w_func)
+    w_code = space.wrap(func.code)
+    return borrow_from(w_func, w_code)
+
 @cpython_api([PyObject, PyObject, PyObject], PyObject)
 def PyMethod_New(space, w_func, w_self, w_cls):
     """Return a new method object, with func being any callable object; this is the
diff --git a/pypy/module/cpyext/include/code.h b/pypy/module/cpyext/include/code.h
--- a/pypy/module/cpyext/include/code.h
+++ b/pypy/module/cpyext/include/code.h
@@ -4,7 +4,21 @@
 extern "C" {
 #endif
 
-typedef PyObject PyCodeObject;
+typedef struct {
+    PyObject_HEAD
+    PyObject *co_name;
+    int co_argcount;
+    int co_flags;
+} PyCodeObject;
+
+/* Masks for co_flags above */
+/* These values are also in funcobject.py */
+#define CO_OPTIMIZED	0x0001
+#define CO_NEWLOCALS	0x0002
+#define CO_VARARGS	0x0004
+#define CO_VARKEYWORDS	0x0008
+#define CO_NESTED       0x0010
+#define CO_GENERATOR    0x0020
 
 #ifdef __cplusplus
 }
diff --git a/pypy/module/cpyext/include/funcobject.h b/pypy/module/cpyext/include/funcobject.h
--- a/pypy/module/cpyext/include/funcobject.h
+++ b/pypy/module/cpyext/include/funcobject.h
@@ -12,6 +12,8 @@
     PyObject *func_name;	/* The __name__ attribute, a string object */
 } PyFunctionObject;
 
+#define PyFunction_GET_CODE(obj) PyFunction_GetCode((PyObject*)(obj))
+
 #define PyMethod_GET_FUNCTION(obj) PyMethod_Function((PyObject*)(obj))
 #define PyMethod_GET_SELF(obj) PyMethod_Self((PyObject*)(obj))
 #define PyMethod_GET_CLASS(obj) PyMethod_Class((PyObject*)(obj))
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -321,6 +321,15 @@
 
 } PyTypeObject;
 
+typedef struct {
+    PyTypeObject ht_type;
+    PyNumberMethods as_number;
+    PyMappingMethods as_mapping;
+    PySequenceMethods as_sequence;
+    PyBufferProcs as_buffer;
+    PyObject *ht_name, *ht_slots;
+} PyHeapTypeObject;
+
 /* Flag bits for printing: */
 #define Py_PRINT_RAW	1	/* No string quotes etc. */
 
@@ -501,6 +510,9 @@
 #define PyObject_TypeCheck(ob, tp) \
     ((ob)->ob_type == (tp) || PyType_IsSubtype((ob)->ob_type, (tp)))
 
+#define Py_TRASHCAN_SAFE_BEGIN(pyObj)
+#define Py_TRASHCAN_SAFE_END(pyObj)
+
 /* Copied from CPython ----------------------------- */
 int PyObject_AsReadBuffer(PyObject *, const void **, Py_ssize_t *);
 int PyObject_AsWriteBuffer(PyObject *, void **, Py_ssize_t *);
diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,7 +29,7 @@
 #define PY_VERSION		"2.7.1"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "1.6.0"
+#define PYPY_VERSION "1.6.1"
 
 /* Subversion Revision number of this file (not of the repository).
  * Empty since Mercurial migration. */
diff --git a/pypy/module/cpyext/include/pythonrun.h b/pypy/module/cpyext/include/pythonrun.h
--- a/pypy/module/cpyext/include/pythonrun.h
+++ b/pypy/module/cpyext/include/pythonrun.h
@@ -12,6 +12,7 @@
 #define Py_Py3kWarningFlag 0
 
 #define Py_FrozenFlag 0
+#define Py_VerboseFlag 0
 
 typedef struct {
     int cf_flags;  /* bitmask of CO_xxx flags relevant to future */
diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py
--- a/pypy/module/cpyext/pyobject.py
+++ b/pypy/module/cpyext/pyobject.py
@@ -19,13 +19,42 @@
     basestruct = PyObject.TO
 
     def get_dealloc(self, space):
-        raise NotImplementedError
+        from pypy.module.cpyext.typeobject import subtype_dealloc
+        return llhelper(
+            subtype_dealloc.api_func.functype,
+            subtype_dealloc.api_func.get_wrapper(space))
+
     def allocate(self, space, w_type, itemcount=0):
-        raise NotImplementedError
+        # similar to PyType_GenericAlloc?
+        # except that it's not related to any pypy object.
+
+        pytype = rffi.cast(PyTypeObjectPtr, make_ref(space, w_type))
+        # Don't increase refcount for non-heaptypes
+        if pytype:
+            flags = rffi.cast(lltype.Signed, pytype.c_tp_flags)
+            if not flags & Py_TPFLAGS_HEAPTYPE:
+                Py_DecRef(space, w_type)
+
+        if pytype:
+            size = pytype.c_tp_basicsize
+        else:
+            size = rffi.sizeof(self.basestruct)
+        if itemcount:
+            size += itemcount * pytype.c_tp_itemsize
+        buf = lltype.malloc(rffi.VOIDP.TO, size,
+                            flavor='raw', zero=True)
+        pyobj = rffi.cast(PyObject, buf)
+        pyobj.c_ob_refcnt = 1
+        pyobj.c_ob_type = pytype
+        return pyobj
+
     def attach(self, space, pyobj, w_obj):
-        raise NotImplementedError
+        pass
+
     def realize(self, space, ref):
-        raise NotImplementedError
+        # For most types, a reference cannot exist without
+        # a real interpreter object
+        raise InvalidPointerException(str(ref))
 
 typedescr_cache = {}
 
@@ -40,6 +69,7 @@
     """
 
     tp_basestruct = kw.pop('basestruct', PyObject.TO)
+    tp_alloc      = kw.pop('alloc', None)
     tp_attach     = kw.pop('attach', None)
     tp_realize    = kw.pop('realize', None)
     tp_dealloc    = kw.pop('dealloc', None)
@@ -49,58 +79,24 @@
 
     class CpyTypedescr(BaseCpyTypedescr):
         basestruct = tp_basestruct
-        realize = tp_realize
 
-        def get_dealloc(self, space):
-            if tp_dealloc:
+        if tp_alloc:
+            def allocate(self, space, w_type, itemcount=0):
+                return tp_alloc(space, w_type)
+
+        if tp_dealloc:
+            def get_dealloc(self, space):
                 return llhelper(
                     tp_dealloc.api_func.functype,
                     tp_dealloc.api_func.get_wrapper(space))
-            else:
-                from pypy.module.cpyext.typeobject import subtype_dealloc
-                return llhelper(
-                    subtype_dealloc.api_func.functype,
-                    subtype_dealloc.api_func.get_wrapper(space))
-
-        def allocate(self, space, w_type, itemcount=0):
-            # similar to PyType_GenericAlloc?
-            # except that it's not related to any pypy object.
-
-            pytype = rffi.cast(PyTypeObjectPtr, make_ref(space, w_type))
-            # Don't increase refcount for non-heaptypes
-            if pytype:
-                flags = rffi.cast(lltype.Signed, pytype.c_tp_flags)
-                if not flags & Py_TPFLAGS_HEAPTYPE:
-                    Py_DecRef(space, w_type)
-
-            if pytype:
-                size = pytype.c_tp_basicsize
-            else:
-                size = rffi.sizeof(tp_basestruct)
-            if itemcount:
-                size += itemcount * pytype.c_tp_itemsize
-            buf = lltype.malloc(rffi.VOIDP.TO, size,
-                                flavor='raw', zero=True)
-            pyobj = rffi.cast(PyObject, buf)
-            pyobj.c_ob_refcnt = 1
-            pyobj.c_ob_type = pytype
-            return pyobj
 
         if tp_attach:
             def attach(self, space, pyobj, w_obj):
                 tp_attach(space, pyobj, w_obj)
-        else:
-            def attach(self, space, pyobj, w_obj):
-                pass
 
         if tp_realize:
             def realize(self, space, ref):
                 return tp_realize(space, ref)
-        else:
-            def realize(self, space, ref):
-                # For most types, a reference cannot exist without
-                # a real interpreter object
-                raise InvalidPointerException(str(ref))
     if typedef:
         CpyTypedescr.__name__ = "CpyTypedescr_%s" % (typedef.name,)
 
diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py
--- a/pypy/module/cpyext/stubs.py
+++ b/pypy/module/cpyext/stubs.py
@@ -920,12 +920,6 @@
     raise NotImplementedError
 
 @cpython_api([PyObject], PyObject)
-def PyFunction_GetCode(space, op):
-    """Return the code object associated with the function object op."""
-    borrow_from()
-    raise NotImplementedError
-
- at cpython_api([PyObject], PyObject)
 def PyFunction_GetGlobals(space, op):
     """Return the globals dictionary associated with the function object op."""
     borrow_from()
diff --git a/pypy/module/cpyext/test/foo.c b/pypy/module/cpyext/test/foo.c
--- a/pypy/module/cpyext/test/foo.c
+++ b/pypy/module/cpyext/test/foo.c
@@ -215,36 +215,36 @@
 typedef struct {
     PyUnicodeObject HEAD;
     int val;
-} FuuObject;
+} UnicodeSubclassObject;
 
 
-static int Fuu_init(FuuObject *self, PyObject *args, PyObject *kwargs) {
+static int UnicodeSubclass_init(UnicodeSubclassObject *self, PyObject *args, PyObject *kwargs) {
     self->val = 42;
     return 0;
 }
 
 static PyObject *
-Fuu_escape(PyTypeObject* type, PyObject *args)
+UnicodeSubclass_escape(PyTypeObject* type, PyObject *args)
 {
     Py_RETURN_TRUE;
 }
 
 static PyObject *
-Fuu_get_val(FuuObject *self) {
+UnicodeSubclass_get_val(UnicodeSubclassObject *self) {
     return PyInt_FromLong(self->val);
 }
 
-static PyMethodDef Fuu_methods[] = {
-    {"escape", (PyCFunction) Fuu_escape, METH_VARARGS, NULL},
-    {"get_val", (PyCFunction) Fuu_get_val, METH_NOARGS, NULL},
+static PyMethodDef UnicodeSubclass_methods[] = {
+    {"escape", (PyCFunction) UnicodeSubclass_escape, METH_VARARGS, NULL},
+    {"get_val", (PyCFunction) UnicodeSubclass_get_val, METH_NOARGS, NULL},
     {NULL}  /* Sentinel */
 };
 
-PyTypeObject FuuType = {
+PyTypeObject UnicodeSubtype = {
     PyObject_HEAD_INIT(NULL)
     0,
     "foo.fuu",
-    sizeof(FuuObject),
+    sizeof(UnicodeSubclassObject),
     0,
     0,          /*tp_dealloc*/
     0,          /*tp_print*/
@@ -277,7 +277,7 @@
 
     /* Attribute descriptor and subclassing stuff */
 
-    Fuu_methods,/*tp_methods*/
+    UnicodeSubclass_methods,/*tp_methods*/
     0,          /*tp_members*/
     0,          /*tp_getset*/
     0,          /*tp_base*/
@@ -287,7 +287,7 @@
     0,          /*tp_descr_set*/
     0,          /*tp_dictoffset*/
 
-    (initproc) Fuu_init, /*tp_init*/
+    (initproc) UnicodeSubclass_init, /*tp_init*/
     0,          /*tp_alloc  will be set to PyType_GenericAlloc in module init*/
     0,          /*tp_new*/
     0,          /*tp_free  Low-level free-memory routine */
@@ -299,11 +299,11 @@
     0           /*tp_weaklist*/
 };
 
-PyTypeObject Fuu2Type = {
+PyTypeObject UnicodeSubtype2 = {
     PyObject_HEAD_INIT(NULL)
     0,
     "foo.fuu2",
-    sizeof(FuuObject),
+    sizeof(UnicodeSubclassObject),
     0,
     0,          /*tp_dealloc*/
     0,          /*tp_print*/
@@ -628,15 +628,15 @@
 
     footype.tp_new = PyType_GenericNew;
 
-    FuuType.tp_base = &PyUnicode_Type;
-    Fuu2Type.tp_base = &FuuType;
+    UnicodeSubtype.tp_base = &PyUnicode_Type;
+    UnicodeSubtype2.tp_base = &UnicodeSubtype;
     MetaType.tp_base = &PyType_Type;
 
     if (PyType_Ready(&footype) < 0)
         return;
-    if (PyType_Ready(&FuuType) < 0)
+    if (PyType_Ready(&UnicodeSubtype) < 0)
         return;
-    if (PyType_Ready(&Fuu2Type) < 0)
+    if (PyType_Ready(&UnicodeSubtype2) < 0)
         return;
     if (PyType_Ready(&MetaType) < 0)
         return;
@@ -655,9 +655,9 @@
         return;
     if (PyDict_SetItemString(d, "fooType", (PyObject *)&footype) < 0)
         return;
-    if (PyDict_SetItemString(d, "FuuType", (PyObject *) &FuuType) < 0)
+    if (PyDict_SetItemString(d, "UnicodeSubtype", (PyObject *) &UnicodeSubtype) < 0)
         return;
-    if(PyDict_SetItemString(d, "Fuu2Type", (PyObject *) &Fuu2Type) < 0)
+    if (PyDict_SetItemString(d, "UnicodeSubtype2", (PyObject *) &UnicodeSubtype2) < 0)
         return;
     if (PyDict_SetItemString(d, "MetaType", (PyObject *) &MetaType) < 0)
         return;
diff --git a/pypy/module/cpyext/test/test_funcobject.py b/pypy/module/cpyext/test/test_funcobject.py
--- a/pypy/module/cpyext/test/test_funcobject.py
+++ b/pypy/module/cpyext/test/test_funcobject.py
@@ -2,8 +2,12 @@
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref
-from pypy.module.cpyext.funcobject import PyFunctionObject
+from pypy.module.cpyext.funcobject import (
+    PyFunctionObject, PyCodeObject, CODE_FLAGS)
 from pypy.interpreter.function import Function, Method
+from pypy.interpreter.pycode import PyCode
+
+globals().update(CODE_FLAGS)
 
 class TestFunctionObject(BaseApiTest):
     def test_function(self, space, api):
@@ -36,6 +40,38 @@
         w_method2 = api.PyMethod_New(w_function, w_self, w_class)
         assert space.eq_w(w_method, w_method2)
 
+    def test_getcode(self, space, api):
+        w_function = space.appexec([], """():
+            def func(x, y, z): return x
+            return func
+        """)
+        w_code = api.PyFunction_GetCode(w_function)
+        assert w_code.co_name == "func"
+
+        ref = make_ref(space, w_code)
+        assert (from_ref(space, rffi.cast(PyObject, ref.c_ob_type)) is
+                space.gettypeobject(PyCode.typedef))
+        assert "func" == space.unwrap(
+           from_ref(space, rffi.cast(PyCodeObject, ref).c_co_name))
+        assert 3 == rffi.cast(PyCodeObject, ref).c_co_argcount
+        api.Py_DecRef(ref)
+
+    def test_co_flags(self, space, api):
+        def get_flags(signature, body="pass"):
+            w_code = space.appexec([], """():
+                def func(%s): %s
+                return func.__code__
+            """ % (signature, body))
+            ref = make_ref(space, w_code)
+            co_flags = rffi.cast(PyCodeObject, ref).c_co_flags
+            api.Py_DecRef(ref)
+            return co_flags
+        assert get_flags("x") == CO_NESTED | CO_OPTIMIZED | CO_NEWLOCALS
+        assert get_flags("x", "exec x") == CO_NESTED | CO_NEWLOCALS
+        assert get_flags("x, *args") & CO_VARARGS
+        assert get_flags("x, **kw") & CO_VARKEYWORDS
+        assert get_flags("x", "yield x") & CO_GENERATOR
+
     def test_newcode(self, space, api):
         filename = rffi.str2charp('filename')
         funcname = rffi.str2charp('funcname')
diff --git a/pypy/module/cpyext/test/test_tupleobject.py b/pypy/module/cpyext/test/test_tupleobject.py
--- a/pypy/module/cpyext/test/test_tupleobject.py
+++ b/pypy/module/cpyext/test/test_tupleobject.py
@@ -48,3 +48,4 @@
         w_slice = api.PyTuple_GetSlice(w_tuple, 3, -3)
         assert space.eq_w(w_slice,
                           space.newtuple([space.wrap(i) for i in range(3, 7)]))
+
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -119,16 +119,16 @@
         module = self.import_module(name='foo')
         obj = module.new()
         # call __new__
-        newobj = module.FuuType(u"xyz")
+        newobj = module.UnicodeSubtype(u"xyz")
         assert newobj == u"xyz"
-        assert isinstance(newobj, module.FuuType)
+        assert isinstance(newobj, module.UnicodeSubtype)
 
         assert isinstance(module.fooType(), module.fooType)
         class bar(module.fooType):
             pass
         assert isinstance(bar(), bar)
 
-        fuu = module.FuuType
+        fuu = module.UnicodeSubtype
         class fuu2(fuu):
             def baz(self):
                 return self
@@ -137,20 +137,20 @@
 
     def test_init(self):
         module = self.import_module(name="foo")
-        newobj = module.FuuType()
+        newobj = module.UnicodeSubtype()
         assert newobj.get_val() == 42
 
         # this subtype should inherit tp_init
-        newobj = module.Fuu2Type()
+        newobj = module.UnicodeSubtype2()
         assert newobj.get_val() == 42
 
         # this subclass redefines __init__
-        class Fuu2(module.FuuType):
+        class UnicodeSubclass2(module.UnicodeSubtype):
             def __init__(self):
                 self.foobar = 32
-                super(Fuu2, self).__init__()
+                super(UnicodeSubclass2, self).__init__()
         
-        newobj = Fuu2()
+        newobj = UnicodeSubclass2()
         assert newobj.get_val() == 42
         assert newobj.foobar == 32
 
@@ -268,6 +268,21 @@
         assert type(obj) is foo.Custom
         assert type(foo.Custom) is foo.MetaType
 
+    def test_heaptype(self):
+        module = self.import_extension('foo', [
+           ("name_by_heaptype", "METH_O",
+            '''
+                 PyHeapTypeObject *heaptype = (PyHeapTypeObject *)args;
+                 Py_INCREF(heaptype->ht_name);
+                 return heaptype->ht_name;
+             '''
+             )
+            ])
+        class C(object):
+            pass
+        assert module.name_by_heaptype(C) == "C"
+        
+
 class TestTypes(BaseApiTest):
     def test_type_attributes(self, space, api):
         w_class = space.appexec([], """():
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -11,7 +11,7 @@
     generic_cpy_call, Py_TPFLAGS_READY, Py_TPFLAGS_READYING,
     Py_TPFLAGS_HEAPTYPE, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL,
     Py_TPFLAGS_HAVE_GETCHARBUFFER,
-    build_type_checkers)
+    build_type_checkers, PyObjectFields)
 from pypy.module.cpyext.pyobject import (
     PyObject, make_ref, create_ref, from_ref, get_typedescr, make_typedescr,
     track_reference, RefcountState, borrow_from)
@@ -25,7 +25,7 @@
 from pypy.module.cpyext.structmember import PyMember_GetOne, PyMember_SetOne
 from pypy.module.cpyext.typeobjectdefs import (
     PyTypeObjectPtr, PyTypeObject, PyGetSetDef, PyMemberDef, newfunc,
-    PyNumberMethods, PySequenceMethods, PyBufferProcs)
+    PyNumberMethods, PyMappingMethods, PySequenceMethods, PyBufferProcs)
 from pypy.module.cpyext.slotdefs import (
     slotdefs_for_tp_slots, slotdefs_for_wrappers, get_slot_tp_function)
 from pypy.interpreter.error import OperationError
@@ -39,6 +39,19 @@
 
 PyType_Check, PyType_CheckExact = build_type_checkers("Type", "w_type")
 
+PyHeapTypeObjectStruct = lltype.ForwardReference()
+PyHeapTypeObject = lltype.Ptr(PyHeapTypeObjectStruct)
+PyHeapTypeObjectFields = (
+    ("ht_type", PyTypeObject),
+    ("ht_name", PyObject),
+    ("as_number", PyNumberMethods),
+    ("as_mapping", PyMappingMethods),
+    ("as_sequence", PySequenceMethods),
+    ("as_buffer", PyBufferProcs),
+    )
+cpython_struct("PyHeapTypeObject", PyHeapTypeObjectFields, PyHeapTypeObjectStruct,
+               level=2)
+
 class W_GetSetPropertyEx(GetSetProperty):
     def __init__(self, getset, w_type):
         self.getset = getset
@@ -136,6 +149,8 @@
             assert len(slot_names) == 2
             struct = getattr(pto, slot_names[0])
             if not struct:
+                assert not space.config.translating
+                assert not pto.c_tp_flags & Py_TPFLAGS_HEAPTYPE
                 if slot_names[0] == 'c_tp_as_number':
                     STRUCT_TYPE = PyNumberMethods
                 elif slot_names[0] == 'c_tp_as_sequence':
@@ -301,6 +316,7 @@
 
     make_typedescr(space.w_type.instancetypedef,
                    basestruct=PyTypeObject,
+                   alloc=type_alloc,
                    attach=type_attach,
                    realize=type_realize,
                    dealloc=type_dealloc)
@@ -319,11 +335,13 @@
     track_reference(space, lltype.nullptr(PyObject.TO), space.w_type)
     track_reference(space, lltype.nullptr(PyObject.TO), space.w_object)
     track_reference(space, lltype.nullptr(PyObject.TO), space.w_tuple)
+    track_reference(space, lltype.nullptr(PyObject.TO), space.w_str)
 
     # create the objects
     py_type = create_ref(space, space.w_type)
     py_object = create_ref(space, space.w_object)
     py_tuple = create_ref(space, space.w_tuple)
+    py_str = create_ref(space, space.w_str)
 
     # form cycles
     pto_type = rffi.cast(PyTypeObjectPtr, py_type)
@@ -340,10 +358,15 @@
     pto_object.c_tp_bases.c_ob_type = pto_tuple
     pto_tuple.c_tp_bases.c_ob_type = pto_tuple
 
+    for typ in (py_type, py_object, py_tuple, py_str):
+        heaptype = rffi.cast(PyHeapTypeObject, typ)
+        heaptype.c_ht_name.c_ob_type = pto_type
+
     # Restore the mapping
     track_reference(space, py_type, space.w_type, replace=True)
     track_reference(space, py_object, space.w_object, replace=True)
     track_reference(space, py_tuple, space.w_tuple, replace=True)
+    track_reference(space, py_str, space.w_str, replace=True)
 
 
 @cpython_api([PyObject], lltype.Void, external=False)
@@ -416,17 +439,34 @@
     Py_DecRef(space, obj_pto.c_tp_cache) # let's do it like cpython
     Py_DecRef(space, obj_pto.c_tp_dict)
     if obj_pto.c_tp_flags & Py_TPFLAGS_HEAPTYPE:
-        if obj_pto.c_tp_as_buffer:
-            lltype.free(obj_pto.c_tp_as_buffer, flavor='raw')
-        if obj_pto.c_tp_as_number:
-            lltype.free(obj_pto.c_tp_as_number, flavor='raw')
-        if obj_pto.c_tp_as_sequence:
-            lltype.free(obj_pto.c_tp_as_sequence, flavor='raw')
+        heaptype = rffi.cast(PyHeapTypeObject, obj)
+        Py_DecRef(space, heaptype.c_ht_name)
         Py_DecRef(space, base_pyo)
-        rffi.free_charp(obj_pto.c_tp_name)
         PyObject_dealloc(space, obj)
 
 
+def type_alloc(space, w_metatype):
+    size = rffi.sizeof(PyHeapTypeObject)
+    metatype = rffi.cast(PyTypeObjectPtr, make_ref(space, w_metatype))
+    # Don't increase refcount for non-heaptypes
+    if metatype:
+        flags = rffi.cast(lltype.Signed, metatype.c_tp_flags)
+        if not flags & Py_TPFLAGS_HEAPTYPE:
+            Py_DecRef(space, w_metatype)
+
+    heaptype = lltype.malloc(PyHeapTypeObject.TO,
+                             flavor='raw', zero=True)
+    pto = heaptype.c_ht_type
+    pto.c_ob_refcnt = 1
+    pto.c_ob_type = metatype
+    pto.c_tp_flags |= Py_TPFLAGS_HEAPTYPE
+    pto.c_tp_as_number = heaptype.c_as_number
+    pto.c_tp_as_sequence = heaptype.c_as_sequence
+    pto.c_tp_as_mapping = heaptype.c_as_mapping
+    pto.c_tp_as_buffer = heaptype.c_as_buffer
+    
+    return rffi.cast(PyObject, heaptype)
+
 def type_attach(space, py_obj, w_type):
     """
     Fills a newly allocated PyTypeObject from an existing type.
@@ -445,12 +485,18 @@
     if space.is_w(w_type, space.w_str):
         setup_string_buffer_procs(space, pto)
 
-    pto.c_tp_flags |= Py_TPFLAGS_HEAPTYPE
     pto.c_tp_free = llhelper(PyObject_Del.api_func.functype,
             PyObject_Del.api_func.get_wrapper(space))
     pto.c_tp_alloc = llhelper(PyType_GenericAlloc.api_func.functype,
             PyType_GenericAlloc.api_func.get_wrapper(space))
-    pto.c_tp_name = rffi.str2charp(w_type.getname(space))
+    if pto.c_tp_flags & Py_TPFLAGS_HEAPTYPE:
+        w_typename = space.getattr(w_type, space.wrap('__name__'))
+        heaptype = rffi.cast(PyHeapTypeObject, pto)
+        heaptype.c_ht_name = make_ref(space, w_typename)
+        from pypy.module.cpyext.stringobject import PyString_AsString
+        pto.c_tp_name = PyString_AsString(space, heaptype.c_ht_name)
+    else:
+        pto.c_tp_name = rffi.str2charp(w_type.getname(space))
     pto.c_tp_basicsize = -1 # hopefully this makes malloc bail out
     pto.c_tp_itemsize = 0
     # uninitialized fields:
diff --git a/pypy/module/marshal/interp_marshal.py b/pypy/module/marshal/interp_marshal.py
--- a/pypy/module/marshal/interp_marshal.py
+++ b/pypy/module/marshal/interp_marshal.py
@@ -40,7 +40,7 @@
         reader = FileReader(space, w_f)
     try:
         u = Unmarshaller(space, reader)
-        return u.load_w_obj(False)
+        return u.load_w_obj()
     finally:
         reader.finished()
 
@@ -49,7 +49,7 @@
 ignored."""
     space.timer.start("marshal loads")
     u = StringUnmarshaller(space, w_str)
-    obj = u.load_w_obj(False)
+    obj = u.load_w_obj()
     space.timer.stop("marshal loads")
     return obj
 
@@ -424,7 +424,7 @@
         lng = self.get_lng()
         return self.get(lng)
 
-    def get_w_obj(self, allow_null):
+    def get_w_obj(self, allow_null=False):
         space = self.space
         w_ret = space.w_None # something not None
         tc = self.get1()
@@ -434,9 +434,9 @@
                 'NULL object in marshal data'))
         return w_ret
 
-    def load_w_obj(self, allow_null):
+    def load_w_obj(self):
         try:
-            return self.get_w_obj(allow_null)
+            return self.get_w_obj()
         except rstackovf.StackOverflow:
             rstackovf.check_stack_overflow()
             self._overflow()
diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -26,13 +26,19 @@
         ("copysign", "copysign"),
         ("cos", "cos"),
         ("divide", "divide"),
+        ("equal", "equal"),
         ("exp", "exp"),
         ("fabs", "fabs"),
         ("floor", "floor"),
+        ("greater", "greater"),
+        ("greater_equal", "greater_equal"),
+        ("less", "less"),
+        ("less_equal", "less_equal"),
         ("maximum", "maximum"),
         ("minimum", "minimum"),
         ("multiply", "multiply"),
         ("negative", "negative"),
+        ("not_equal", "not_equal"),
         ("reciprocal", "reciprocal"),
         ("sign", "sign"),
         ("sin", "sin"),
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -129,6 +129,16 @@
         ))
     return impl
 
+def raw_binop(func):
+    # Returns the result unwrapped.
+    @functools.wraps(func)
+    def impl(self, v1, v2):
+        return func(self,
+            self.for_computation(self.unbox(v1)),
+            self.for_computation(self.unbox(v2))
+        )
+    return impl
+
 def unaryop(func):
     @functools.wraps(func)
     def impl(self, v):
@@ -170,8 +180,24 @@
 
     def bool(self, v):
         return bool(self.for_computation(self.unbox(v)))
+    @raw_binop
+    def eq(self, v1, v2):
+        return v1 == v2
+    @raw_binop
     def ne(self, v1, v2):
-        return self.for_computation(self.unbox(v1)) != self.for_computation(self.unbox(v2))
+        return v1 != v2
+    @raw_binop
+    def lt(self, v1, v2):
+        return v1 < v2
+    @raw_binop
+    def le(self, v1, v2):
+        return v1 <= v2
+    @raw_binop
+    def gt(self, v1, v2):
+        return v1 > v2
+    @raw_binop
+    def ge(self, v1, v2):
+        return v1 >= v2
 
 
 class FloatArithmeticDtype(ArithmaticTypeMixin):
@@ -224,7 +250,7 @@
         return math.tan(v)
     @unaryop
     def arcsin(self, v):
-        if v < -1.0 or  v > 1.0:
+        if v < -1.0 or v > 1.0:
             return rfloat.NAN
         return math.asin(v)
     @unaryop
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -74,6 +74,13 @@
     descr_pow = _binop_impl("power")
     descr_mod = _binop_impl("mod")
 
+    descr_eq = _binop_impl("equal")
+    descr_ne = _binop_impl("not_equal")
+    descr_lt = _binop_impl("less")
+    descr_le = _binop_impl("less_equal")
+    descr_gt = _binop_impl("greater")
+    descr_ge = _binop_impl("greater_equal")
+
     def _binop_right_impl(ufunc_name):
         def impl(self, space, w_other):
             w_other = scalar_w(space,
@@ -404,10 +411,11 @@
     """
     Intermediate class for performing binary operations.
     """
-    def __init__(self, signature, res_dtype, left, right):
+    def __init__(self, signature, calc_dtype, res_dtype, left, right):
         VirtualArray.__init__(self, signature, res_dtype)
         self.left = left
         self.right = right
+        self.calc_dtype = calc_dtype
 
     def _del_sources(self):
         self.left = None
@@ -421,14 +429,14 @@
         return self.right.find_size()
 
     def _eval(self, i):
-        lhs = self.left.eval(i).convert_to(self.res_dtype)
-        rhs = self.right.eval(i).convert_to(self.res_dtype)
+        lhs = self.left.eval(i).convert_to(self.calc_dtype)
+        rhs = self.right.eval(i).convert_to(self.calc_dtype)
 
         sig = jit.promote(self.signature)
         assert isinstance(sig, signature.Signature)
         call_sig = sig.components[0]
         assert isinstance(call_sig, signature.Call2)
-        return call_sig.func(self.res_dtype, lhs, rhs)
+        return call_sig.func(self.calc_dtype, lhs, rhs)
 
 class ViewArray(BaseArray):
     """
@@ -573,18 +581,28 @@
     __pos__ = interp2app(BaseArray.descr_pos),
     __neg__ = interp2app(BaseArray.descr_neg),
     __abs__ = interp2app(BaseArray.descr_abs),
+
     __add__ = interp2app(BaseArray.descr_add),
     __sub__ = interp2app(BaseArray.descr_sub),
     __mul__ = interp2app(BaseArray.descr_mul),
     __div__ = interp2app(BaseArray.descr_div),
     __pow__ = interp2app(BaseArray.descr_pow),
     __mod__ = interp2app(BaseArray.descr_mod),
+
     __radd__ = interp2app(BaseArray.descr_radd),
     __rsub__ = interp2app(BaseArray.descr_rsub),
     __rmul__ = interp2app(BaseArray.descr_rmul),
     __rdiv__ = interp2app(BaseArray.descr_rdiv),
     __rpow__ = interp2app(BaseArray.descr_rpow),
     __rmod__ = interp2app(BaseArray.descr_rmod),
+
+    __eq__ = interp2app(BaseArray.descr_eq),
+    __ne__ = interp2app(BaseArray.descr_ne),
+    __lt__ = interp2app(BaseArray.descr_lt),
+    __le__ = interp2app(BaseArray.descr_le),
+    __gt__ = interp2app(BaseArray.descr_gt),
+    __ge__ = interp2app(BaseArray.descr_ge),
+
     __repr__ = interp2app(BaseArray.descr_repr),
     __str__ = interp2app(BaseArray.descr_str),
 
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -113,10 +113,11 @@
     argcount = 2
 
     def __init__(self, func, name, promote_to_float=False, promote_bools=False,
-        identity=None):
+        identity=None, comparison_func=False):
 
         W_Ufunc.__init__(self, name, promote_to_float, promote_bools, identity)
         self.func = func
+        self.comparison_func = comparison_func
         self.signature = signature.Call2(func)
         self.reduce_signature = signature.BaseSignature()
 
@@ -127,18 +128,25 @@
         [w_lhs, w_rhs] = args_w
         w_lhs = convert_to_array(space, w_lhs)
         w_rhs = convert_to_array(space, w_rhs)
-        res_dtype = find_binop_result_dtype(space,
+        calc_dtype = find_binop_result_dtype(space,
             w_lhs.find_dtype(), w_rhs.find_dtype(),
             promote_to_float=self.promote_to_float,
             promote_bools=self.promote_bools,
         )
+        if self.comparison_func:
+            res_dtype = space.fromcache(interp_dtype.W_BoolDtype)
+        else:
+            res_dtype = calc_dtype
         if isinstance(w_lhs, Scalar) and isinstance(w_rhs, Scalar):
-            return self.func(res_dtype, w_lhs.value, w_rhs.value).wrap(space)
+            return self.func(calc_dtype,
+                w_lhs.value.convert_to(calc_dtype),
+                w_rhs.value.convert_to(calc_dtype)
+            ).wrap(space)
 
         new_sig = signature.Signature.find_sig([
             self.signature, w_lhs.signature, w_rhs.signature
         ])
-        w_res = Call2(new_sig, res_dtype, w_lhs, w_rhs)
+        w_res = Call2(new_sig, calc_dtype, res_dtype, w_lhs, w_rhs)
         w_lhs.add_invalidates(w_res)
         w_rhs.add_invalidates(w_res)
         return w_res
@@ -209,13 +217,16 @@
     return space.fromcache(interp_dtype.W_Float64Dtype)
 
 
-def ufunc_dtype_caller(ufunc_name, op_name, argcount):
+def ufunc_dtype_caller(space, ufunc_name, op_name, argcount, comparison_func):
     if argcount == 1:
         def impl(res_dtype, value):
             return getattr(res_dtype, op_name)(value)
     elif argcount == 2:
         def impl(res_dtype, lvalue, rvalue):
-            return getattr(res_dtype, op_name)(lvalue, rvalue)
+            res = getattr(res_dtype, op_name)(lvalue, rvalue)
+            if comparison_func:
+                res = space.fromcache(interp_dtype.W_BoolDtype).box(res)
+            return res
     return func_with_new_name(impl, ufunc_name)
 
 class UfuncState(object):
@@ -229,6 +240,13 @@
             ("mod", "mod", 2, {"promote_bools": True}),
             ("power", "pow", 2, {"promote_bools": True}),
 
+            ("equal", "eq", 2, {"comparison_func": True}),
+            ("not_equal", "ne", 2, {"comparison_func": True}),
+            ("less", "lt", 2, {"comparison_func": True}),
+            ("less_equal", "le", 2, {"comparison_func": True}),
+            ("greater", "gt", 2, {"comparison_func": True}),
+            ("greater_equal", "ge", 2, {"comparison_func": True}),
+
             ("maximum", "max", 2),
             ("minimum", "min", 2),
 
@@ -262,7 +280,9 @@
             identity = space.fromcache(interp_dtype.W_Int64Dtype).adapt_val(identity)
         extra_kwargs["identity"] = identity
 
-        func = ufunc_dtype_caller(ufunc_name, op_name, argcount)
+        func = ufunc_dtype_caller(space, ufunc_name, op_name, argcount,
+            comparison_func=extra_kwargs.get("comparison_func", False)
+        )
         if argcount == 1:
             ufunc = W_Ufunc1(func, ufunc_name, **extra_kwargs)
         elif argcount == 2:
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -557,6 +557,26 @@
         assert array([1.2, 5]).dtype is dtype(float)
         assert array([]).dtype is dtype(float)
 
+    def test_comparison(self):
+        import operator
+        from numpy import array, dtype
+
+        a = array(range(5))
+        b = array(range(5), float)
+        for func in [
+            operator.eq, operator.ne, operator.lt, operator.le, operator.gt,
+            operator.ge
+        ]:
+            c = func(a, 3)
+            assert c.dtype is dtype(bool)
+            for i in xrange(5):
+                assert c[i] == func(a[i], 3)
+
+            c = func(b, 3)
+            assert c.dtype is dtype(bool)
+            for i in xrange(5):
+                assert c[i] == func(b[i], 3)
+
 
 class AppTestSupport(object):
     def setup_class(cls):
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -310,4 +310,30 @@
         assert add.reduce([1, 2, 3]) == 6
         assert maximum.reduce([1]) == 1
         assert maximum.reduce([1, 2, 3]) == 3
-        raises(ValueError, maximum.reduce, [])
\ No newline at end of file
+        raises(ValueError, maximum.reduce, [])
+
+    def test_comparisons(self):
+        import operator
+        from numpy import equal, not_equal, less, less_equal, greater, greater_equal
+
+        for ufunc, func in [
+            (equal, operator.eq),
+            (not_equal, operator.ne),
+            (less, operator.lt),
+            (less_equal, operator.le),
+            (greater, operator.gt),
+            (greater_equal, operator.ge),
+        ]:
+            for a, b in [
+                (3, 3),
+                (3, 4),
+                (4, 3),
+                (3.0, 3.0),
+                (3.0, 3.5),
+                (3.5, 3.0),
+                (3.0, 3),
+                (3, 3.0),
+                (3.5, 3),
+                (3, 3.5),
+            ]:
+                assert ufunc(a, b) is func(a, b)
diff --git a/pypy/module/pwd/__init__.py b/pypy/module/pwd/__init__.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/pwd/__init__.py
@@ -0,0 +1,25 @@
+from pypy.interpreter.mixedmodule import MixedModule
+
+class Module(MixedModule):
+    """
+    This module provides access to the Unix password database.
+    It is available on all Unix versions.
+
+    Password database entries are reported as 7-tuples containing the following
+    items from the password database (see `<pwd.h>'), in order:
+    pw_name, pw_passwd, pw_uid, pw_gid, pw_gecos, pw_dir, pw_shell.
+    The uid and gid items are integers, all others are strings. An
+    exception is raised if the entry asked for cannot be found.
+    """
+
+    interpleveldefs = {
+        'getpwuid': 'interp_pwd.getpwuid',
+        'getpwnam': 'interp_pwd.getpwnam',
+        'getpwall': 'interp_pwd.getpwall',
+    }
+
+    appleveldefs = {
+        'struct_passwd': 'app_pwd.struct_passwd',
+        'struct_pwent': 'app_pwd.struct_passwd',
+    }
+
diff --git a/pypy/module/pwd/app_pwd.py b/pypy/module/pwd/app_pwd.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/pwd/app_pwd.py
@@ -0,0 +1,20 @@
+from _structseq import structseqtype, structseqfield
+
+class struct_passwd:
+    """
+    pwd.struct_passwd: Results from getpw*() routines.
+
+    This object may be accessed either as a tuple of
+      (pw_name,pw_passwd,pw_uid,pw_gid,pw_gecos,pw_dir,pw_shell)
+    or via the object attributes as named in the above tuple.
+    """
+    __metaclass__ = structseqtype
+    name = "pwd.struct_passwd"
+
+    pw_name   = structseqfield(0, "user name")
+    pw_passwd = structseqfield(1, "password")
+    pw_uid    = structseqfield(2, "user id")
+    pw_gid    = structseqfield(3, "group id")
+    pw_gecos  = structseqfield(4, "real name")
+    pw_dir    = structseqfield(5, "home directory")
+    pw_shell  = structseqfield(6, "shell program")
diff --git a/pypy/module/pwd/interp_pwd.py b/pypy/module/pwd/interp_pwd.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/pwd/interp_pwd.py
@@ -0,0 +1,95 @@
+from pypy.translator.tool.cbuild import ExternalCompilationInfo
+from pypy.rpython.tool import rffi_platform
+from pypy.rpython.lltypesystem import rffi, lltype
+from pypy.interpreter.gateway import interp2app, unwrap_spec
+from pypy.interpreter.error import OperationError, operationerrfmt
+from pypy.rlib.rarithmetic import intmask
+
+eci = ExternalCompilationInfo(
+    includes=['pwd.h']
+    )
+
+class CConfig:
+    _compilation_info_ = eci
+
+    uid_t = rffi_platform.SimpleType("uid_t")
+
+    passwd = rffi_platform.Struct(
+        'struct passwd',
+        [('pw_name', rffi.CCHARP),
+         ('pw_passwd', rffi.CCHARP),
+         ('pw_uid', rffi.INT),
+         ('pw_gid', rffi.INT),
+         ('pw_gecos', rffi.CCHARP),
+         ('pw_dir', rffi.CCHARP),
+         ('pw_shell', rffi.CCHARP),
+         ])
+
+config = rffi_platform.configure(CConfig)
+passwd_p = lltype.Ptr(config['passwd'])
+uid_t = config['uid_t']
+
+def external(name, args, result, **kwargs):
+    return rffi.llexternal(name, args, result, compilation_info=eci, **kwargs)
+
+c_getpwuid = external("getpwuid", [uid_t], passwd_p)
+c_getpwnam = external("getpwnam", [rffi.CCHARP], passwd_p)
+c_setpwent = external("setpwent", [], lltype.Void)
+c_getpwent = external("getpwent", [], passwd_p)
+c_endpwent = external("endpwent", [], lltype.Void)
+
+def make_struct_passwd(space, pw):
+    w_passwd_struct = space.getattr(space.getbuiltinmodule('pwd'),
+                                    space.wrap('struct_passwd'))
+    w_tuple = space.newtuple([
+        space.wrap(rffi.charp2str(pw.c_pw_name)),
+        space.wrap(rffi.charp2str(pw.c_pw_passwd)),
+        space.wrap(intmask(pw.c_pw_uid)),
+        space.wrap(intmask(pw.c_pw_gid)),
+        space.wrap(rffi.charp2str(pw.c_pw_gecos)),
+        space.wrap(rffi.charp2str(pw.c_pw_dir)),
+        space.wrap(rffi.charp2str(pw.c_pw_shell)),
+        ])
+    return space.call_function(w_passwd_struct, w_tuple)
+
+ at unwrap_spec(uid=int)
+def getpwuid(space, uid):
+    """
+    getpwuid(uid) -> (pw_name,pw_passwd,pw_uid,
+                      pw_gid,pw_gecos,pw_dir,pw_shell)
+    Return the password database entry for the given numeric user ID.
+    See pwd.__doc__ for more on password database entries.
+    """
+    pw = c_getpwuid(uid)
+    if not pw:
+        raise operationerrfmt(space.w_KeyError,
+            "getpwuid(): uid not found: %d", uid)
+    return make_struct_passwd(space, pw)
+
+ at unwrap_spec(name=str)
+def getpwnam(space, name):
+    """
+    getpwnam(name) -> (pw_name,pw_passwd,pw_uid,
+                        pw_gid,pw_gecos,pw_dir,pw_shell)
+    Return the password database entry for the given user name.
+    See pwd.__doc__ for more on password database entries.
+    """
+    pw = c_getpwnam(name)
+    if not pw:
+        raise operationerrfmt(space.w_KeyError,
+            "getpwnam(): name not found: %s", name)
+    return make_struct_passwd(space, pw)
+
+def getpwall(space):
+    users_w = []
+    c_setpwent()
+    try:
+        while True:
+            pw = c_getpwent()
+            if not pw:
+                break
+            users_w.append(make_struct_passwd(space, pw))
+    finally:
+        c_endpwent()
+    return space.newlist(users_w)
+    
diff --git a/pypy/module/pwd/test/test_pwd.py b/pypy/module/pwd/test/test_pwd.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/pwd/test/test_pwd.py
@@ -0,0 +1,31 @@
+from pypy.conftest import gettestobjspace
+
+class AppTestPwd:
+    def setup_class(cls):
+        cls.space = gettestobjspace(usemodules=['pwd'])
+
+    def test_getpwuid(self):
+        import pwd, sys
+        raises(KeyError, pwd.getpwuid, -1)
+        pw = pwd.getpwuid(0)
+        assert pw.pw_name == 'root'
+        assert isinstance(pw.pw_passwd, str)
+        assert pw.pw_uid == 0
+        assert pw.pw_gid == 0
+        if sys.platform.startswith('linux'):
+            assert pw.pw_dir == '/root'
+        else:
+            assert pw.pw_dir.startswith('/')
+        assert pw.pw_shell.startswith('/')
+        #
+        assert type(pw.pw_uid) is int
+        assert type(pw.pw_gid) is int
+
+    def test_getpwnam(self):
+        import pwd
+        raises(KeyError, pwd.getpwnam, '~invalid~')
+        assert pwd.getpwnam('root').pw_name == 'root'
+
+    def test_getpwall(self):
+        import pwd
+        assert pwd.getpwnam('root') in pwd.getpwall()
diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py
--- a/pypy/module/pyexpat/interp_pyexpat.py
+++ b/pypy/module/pyexpat/interp_pyexpat.py
@@ -12,6 +12,7 @@
 from pypy.translator.platform import platform
 
 import sys
+import weakref
 import py
 
 if sys.platform == "win32":
@@ -180,7 +181,7 @@
 class CallbackData(Wrappable):
     def __init__(self, space, parser):
         self.space = space
-        self.parser = parser
+        self.parser = weakref.ref(parser)
 
 SETTERS = {}
 for index, (name, params) in enumerate(HANDLERS.items()):
@@ -257,7 +258,7 @@
         id = rffi.cast(lltype.Signed, %(ll_id)s)
         userdata = global_storage.get_object(id)
         space = userdata.space
-        parser = userdata.parser
+        parser = userdata.parser()
 
         handler = parser.handlers[%(index)s]
         if not handler:
@@ -292,7 +293,7 @@
     id = rffi.cast(lltype.Signed, ll_userdata)
     userdata = global_storage.get_object(id)
     space = userdata.space
-    parser = userdata.parser
+    parser = userdata.parser()
 
     name = rffi.charp2str(name)
 
@@ -409,8 +410,7 @@
         if XML_ParserFree: # careful with CPython interpreter shutdown
             XML_ParserFree(self.itself)
         if global_storage:
-            global_storage.free_nonmoving_id(
-                rffi.cast(lltype.Signed, self.itself))
+            global_storage.free_nonmoving_id(self.id)
 
     @unwrap_spec(flag=int)
     def SetParamEntityParsing(self, space, flag):
diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py
--- a/pypy/module/pypyjit/interp_jit.py
+++ b/pypy/module/pypyjit/interp_jit.py
@@ -13,7 +13,6 @@
 from pypy.interpreter.pyframe import PyFrame
 from pypy.interpreter.pyopcode import ExitFrame
 from pypy.interpreter.gateway import unwrap_spec
-from pypy.interpreter.baseobjspace import ObjSpace, W_Root
 from opcode import opmap
 from pypy.rlib.nonconst import NonConstant
 from pypy.jit.metainterp.resoperation import rop
@@ -221,7 +220,6 @@
     def __init__(self, space):
         self.w_compile_hook = space.w_None
 
- at unwrap_spec(ObjSpace, W_Root)
 def set_compile_hook(space, w_hook):
     """ set_compile_hook(hook)
 
diff --git a/pypy/module/pypyjit/policy.py b/pypy/module/pypyjit/policy.py
--- a/pypy/module/pypyjit/policy.py
+++ b/pypy/module/pypyjit/policy.py
@@ -16,7 +16,7 @@
         if modname in ['pypyjit', 'signal', 'micronumpy', 'math', 'exceptions',
                        'imp', 'sys', 'array', '_ffi', 'itertools', 'operator',
                        'posix', '_socket', '_sre', '_lsprof', '_weakref',
-                       '__pypy__', 'cStringIO', '_collections']:
+                       '__pypy__', 'cStringIO', '_collections', 'struct']:
             return True
         return False
 
diff --git a/pypy/module/pypyjit/test/test_policy.py b/pypy/module/pypyjit/test/test_policy.py
--- a/pypy/module/pypyjit/test/test_policy.py
+++ b/pypy/module/pypyjit/test/test_policy.py
@@ -3,8 +3,8 @@
 pypypolicy = policy.PyPyJitPolicy()
 
 def test_id_any():
-    from pypy.objspace.std.default import id__ANY
-    assert pypypolicy.look_inside_function(id__ANY)
+    from pypy.objspace.std.intobject import add__Int_Int
+    assert pypypolicy.look_inside_function(add__Int_Int)
 
 def test_bigint():
     from pypy.rlib.rbigint import rbigint
diff --git a/pypy/module/pypyjit/test_pypy_c/model.py b/pypy/module/pypyjit/test_pypy_c/model.py
--- a/pypy/module/pypyjit/test_pypy_c/model.py
+++ b/pypy/module/pypyjit/test_pypy_c/model.py
@@ -2,7 +2,10 @@
 import sys
 import re
 import os.path
-from _pytest.assertion import newinterpret
+try:
+    from _pytest.assertion import newinterpret
+except ImportError:   # e.g. Python 2.5
+    newinterpret = None
 from pypy.tool.jitlogparser.parser import SimpleParser, Function, TraceForOpcode
 from pypy.tool.jitlogparser.storage import LoopStorage
 
@@ -196,7 +199,7 @@
                     source = str(source.deindent()).strip()
         except py.error.ENOENT:
             source = None
-        if source and source.startswith('self._assert('):
+        if source and source.startswith('self._assert(') and newinterpret:
             # transform self._assert(x, 'foo') into assert x, 'foo'
             source = source.replace('self._assert(', 'assert ')
             source = source[:-1] # remove the trailing ')'
diff --git a/pypy/module/pypyjit/test_pypy_c/test_00_model.py b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
--- a/pypy/module/pypyjit/test_pypy_c/test_00_model.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
@@ -1,4 +1,5 @@
-import sys
+from __future__ import with_statement
+import sys, os
 import types
 import subprocess
 import py
diff --git a/pypy/module/pypyjit/test_pypy_c/test__ffi.py b/pypy/module/pypyjit/test_pypy_c/test__ffi.py
--- a/pypy/module/pypyjit/test_pypy_c/test__ffi.py
+++ b/pypy/module/pypyjit/test_pypy_c/test__ffi.py
@@ -29,11 +29,13 @@
         pow_addr, res = log.result
         assert res == 8.0 * 300
         loop, = log.loops_by_filename(self.filepath)
+        if 'ConstClass(pow)' in repr(loop):   # e.g. OS/X
+            pow_addr = 'ConstClass(pow)'
         assert loop.match_by_id('fficall', """
             guard_not_invalidated(descr=...)
             i17 = force_token()
             setfield_gc(p0, i17, descr=<.* .*PyFrame.vable_token .*>)
-            f21 = call_release_gil(%d, 2.000000, 3.000000, descr=<FloatCallDescr>)
+            f21 = call_release_gil(%s, 2.000000, 3.000000, descr=<FloatCallDescr>)
             guard_not_forced(descr=...)
             guard_no_exception(descr=...)
         """ % pow_addr)
@@ -129,4 +131,5 @@
         assert opnames.count('call_release_gil') == 1
         idx = opnames.index('call_release_gil')
         call = ops[idx]
-        assert int(call.args[0]) == fabs_addr
+        assert (call.args[0] == 'ConstClass(fabs)' or    # e.g. OS/X
+                int(call.args[0]) == fabs_addr)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py
--- a/pypy/module/pypyjit/test_pypy_c/test_call.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_call.py
@@ -337,7 +337,9 @@
         assert loop.match_by_id('append', """
             i13 = getfield_gc(p8, descr=<SignedFieldDescr list.length .*>)
             i15 = int_add(i13, 1)
-            call(ConstClass(_ll_list_resize_ge__listPtr_Signed), p8, i15, descr=<VoidCallDescr>)
+            # Will be killed by the backend
+            i17 = arraylen_gc(p7, descr=<GcPtrArrayDescr>)
+            call(ConstClass(_ll_list_resize_ge), p8, i15, descr=<VoidCallDescr>)
             guard_no_exception(descr=...)
             p17 = getfield_gc(p8, descr=<GcPtrFieldDescr list.items .*>)
             p19 = new_with_vtable(ConstClass(W_IntObject))
diff --git a/pypy/module/pypyjit/test_pypy_c/test_containers.py b/pypy/module/pypyjit/test_pypy_c/test_containers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_containers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_containers.py
@@ -40,12 +40,33 @@
         log = self.run(fn, [1000])
         assert log.result == 300
         loop, = log.loops_by_filename(self.filepath)
-        # check that the call to ll_dict_lookup is not a call_may_force
+        # check that the call to ll_dict_lookup is not a call_may_force, the
+        # gc_id call is hoisted out of the loop, the id of a value obviously
+        # can't change ;)
         assert loop.match_by_id("getitem", """
-            i25 = call(ConstClass(_ll_1_gc_identityhash__objectPtr), p6, descr=...)
-            ...
             i28 = call(ConstClass(ll_dict_lookup__dicttablePtr_objectPtr_Signed), p18, p6, i25, descr=...)
             ...
             p33 = call(ConstClass(ll_get_value__dicttablePtr_Signed), p18, i28, descr=...)
             ...
         """)
+
+    def test_list(self):
+        def main(n):
+            i = 0
+            while i < n:
+                z = list(())
+                z.append(1)
+                i += z[-1] / len(z)
+            return i
+
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loop, = log.loops_by_filename(self.filepath)
+        assert loop.match("""
+            i7 = int_lt(i5, i6)
+            guard_true(i7, descr=...)
+            guard_not_invalidated(descr=...)
+            i9 = int_add(i5, 1)
+            --TICK--
+            jump(..., descr=...)
+        """)
\ No newline at end of file
diff --git a/pypy/module/pypyjit/test_pypy_c/test_instance.py b/pypy/module/pypyjit/test_pypy_c/test_instance.py
--- a/pypy/module/pypyjit/test_pypy_c/test_instance.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_instance.py
@@ -142,6 +142,7 @@
             i = 0
             b = B(1)
             while i < 100:
+                b.x
                 v = b.x # ID: loadattr
                 i += v
             return i
@@ -150,8 +151,6 @@
         loop, = log.loops_by_filename(self.filepath)
         assert loop.match_by_id('loadattr',
         '''
-        guard_not_invalidated(descr=...)
-        i16 = arraylen_gc(p10, descr=<GcPtrArrayDescr>)
         i19 = call(ConstClass(ll_dict_lookup), _, _, _, descr=...)
         guard_no_exception(descr=...)
         i21 = int_and(i19, _)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_min_max.py b/pypy/module/pypyjit/test_pypy_c/test_min_max.py
--- a/pypy/module/pypyjit/test_pypy_c/test_min_max.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_min_max.py
@@ -42,7 +42,7 @@
         assert len(guards) < 20
         assert loop.match_by_id('max',"""
             ...
-            p76 = call_may_force(ConstClass(min_max_loop__max), _, _, descr=...)
+            p76 = call_may_force(ConstClass(min_max_trampoline), _, _, descr=...)
             ...
         """)
 
@@ -63,6 +63,6 @@
         assert len(guards) < 20
         assert loop.match_by_id('max',"""
             ...
-            p76 = call_may_force(ConstClass(min_max_loop__max), _, _, descr=...)
+            p76 = call_may_force(ConstClass(min_max_trampoline), _, _, descr=...)
             ...
         """)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_misc.py b/pypy/module/pypyjit/test_pypy_c/test_misc.py
--- a/pypy/module/pypyjit/test_pypy_c/test_misc.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_misc.py
@@ -92,6 +92,43 @@
         """)
 
 
+    def test_cached_pure_func_of_equal_fields(self):
+        def main(n):
+            class A(object):
+                def __init__(self, val):
+                    self.val1 = self.val2 = val
+            a = A(1)
+            b = A(1)
+            sa = 0
+            while n:
+                sa += 2*a.val1
+                sa += 2*b.val2
+                b.val2 = a.val1
+                n -= 1
+            return sa
+        #
+        log = self.run(main, [1000])
+        assert log.result == 4000
+        loop, = log.loops_by_filename(self.filepath)
+        assert loop.match("""
+            i12 = int_is_true(i4)
+            guard_true(i12, descr=...)
+            guard_not_invalidated(descr=...)
+            i13 = int_add_ovf(i8, i9)
+            guard_no_overflow(descr=...)
+            i10p = getfield_gc_pure(p10, descr=...)
+            i10 = int_mul_ovf(2, i10p)
+            guard_no_overflow(descr=...)
+            i14 = int_add_ovf(i13, i10)
+            guard_no_overflow(descr=...)
+            setfield_gc(p7, p11, descr=...)
+            i17 = int_sub_ovf(i4, 1)
+            guard_no_overflow(descr=...)
+            --TICK--
+            jump(..., descr=...)
+            """)
+
+
     def test_range_iter(self):
         def main(n):
             def g(n):
@@ -115,7 +152,6 @@
             i21 = force_token()
             setfield_gc(p4, i20, descr=<.* .*W_AbstractSeqIterObject.inst_index .*>)
             guard_not_invalidated(descr=...)
-            i26 = int_sub(i9, 1)
             i23 = int_lt(i18, 0)
             guard_false(i23, descr=...)
             i25 = int_ge(i18, i9)
@@ -249,3 +285,48 @@
 
         loop, = log.loops_by_id("globalread", is_entry_bridge=True)
         assert len(loop.ops_by_id("globalread")) == 0
+
+    def test_struct_module(self):
+        def main():
+            import struct
+            i = 1
+            while i < 1000:
+                x = struct.unpack("i", struct.pack("i", i))[0] # ID: struct
+                i += x / i
+            return i
+
+        log = self.run(main)
+        assert log.result == main()
+
+        loop, = log.loops_by_id("struct")
+        if sys.maxint == 2 ** 63 - 1:
+            extra = """
+            i8 = int_lt(i4, -2147483648)
+            guard_false(i8, descr=...)
+            """
+        else:
+            extra = ""
+        # This could, of course stand some improvement, to remove all these
+        # arithmatic ops, but we've removed all the core overhead.
+        assert loop.match_by_id("struct", """
+            guard_not_invalidated(descr=...)
+            # struct.pack
+            %(32_bit_only)s
+            i11 = int_and(i4, 255)
+            i13 = int_rshift(i4, 8)
+            i14 = int_and(i13, 255)
+            i16 = int_rshift(i13, 8)
+            i17 = int_and(i16, 255)
+            i19 = int_rshift(i16, 8)
+            i20 = int_and(i19, 255)
+
+            # struct.unpack
+            i22 = int_lshift(i14, 8)
+            i23 = int_or(i11, i22)
+            i25 = int_lshift(i17, 16)
+            i26 = int_or(i23, i25)
+            i28 = int_ge(i20, 128)
+            guard_false(i28, descr=...)
+            i30 = int_lshift(i20, 24)
+            i31 = int_or(i26, i30)
+        """ % {"32_bit_only": extra})
\ No newline at end of file
diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -1,5 +1,6 @@
 from pypy.module.pypyjit.test_pypy_c.test_00_model import BaseTestPyPyC
 
+
 class TestString(BaseTestPyPyC):
     def test_lookup_default_encoding(self):
         def main(n):
@@ -107,3 +108,52 @@
             --TICK--
             jump(p0, p1, p2, p3, p4, p5, i58, i7, descr=<Loop4>)
         """)
+
+    def test_str_mod(self):
+        def main(n):
+            s = 0
+            while n > 0:
+                s += len('%d %d' % (n, n))
+                n -= 1
+            return s
+
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loop, = log.loops_by_filename(self.filepath)
+        assert loop.match("""
+            i7 = int_gt(i4, 0)
+            guard_true(i7, descr=...)
+            guard_not_invalidated(descr=...)
+            p9 = call(ConstClass(ll_int2dec__Signed), i4, descr=<GcPtrCallDescr>)
+            guard_no_exception(descr=...)
+            i10 = strlen(p9)
+            i11 = int_is_true(i10)
+            guard_true(i11, descr=...)
+            i13 = strgetitem(p9, 0)
+            i15 = int_eq(i13, 45)
+            guard_false(i15, descr=...)
+            i17 = int_sub(0, i10)
+            i19 = int_gt(i10, 23)
+            guard_false(i19, descr=...)
+            p21 = newstr(23)
+            copystrcontent(p9, p21, 0, 0, i10)
+            i25 = int_add(1, i10)
+            i26 = int_gt(i25, 23)
+            guard_false(i26, descr=...)
+            strsetitem(p21, i10, 32)
+            i29 = int_add(i10, 1)
+            i30 = int_add(i10, i25)
+            i31 = int_gt(i30, 23)
+            guard_false(i31, descr=...)
+            copystrcontent(p9, p21, 0, i25, i10)
+            i33 = int_eq(i30, 23)
+            guard_false(i33, descr=...)
+            p35 = call(ConstClass(ll_shrink_array__rpy_stringPtr_Signed), p21, i30, descr=<GcPtrCallDescr>)
+            guard_no_exception(descr=...)
+            i37 = strlen(p35)
+            i38 = int_add_ovf(i5, i37)
+            guard_no_overflow(descr=...)
+            i40 = int_sub(i4, 1)
+            --TICK--
+            jump(p0, p1, p2, p3, i40, i38, descr=<Loop0>)
+        """)
\ No newline at end of file
diff --git a/pypy/module/struct/formatiterator.py b/pypy/module/struct/formatiterator.py
--- a/pypy/module/struct/formatiterator.py
+++ b/pypy/module/struct/formatiterator.py
@@ -1,9 +1,9 @@
-from pypy.interpreter.error import OperationError
-
+from pypy.rlib import jit
 from pypy.rlib.objectmodel import specialize
 from pypy.rlib.rstruct.error import StructError
+from pypy.rlib.rstruct.formatiterator import FormatIterator
 from pypy.rlib.rstruct.standardfmttable import PACK_ACCEPTS_BROKEN_INPUT
-from pypy.rlib.rstruct.formatiterator import FormatIterator
+from pypy.interpreter.error import OperationError
 
 
 class PackFormatIterator(FormatIterator):
@@ -14,15 +14,20 @@
         self.args_index = 0
         self.result = []      # list of characters
 
+    # This *should* be always unroll safe, the only way to get here is by
+    # unroll the interpret function, which means the fmt is const, and thus
+    # this should be const (in theory ;)
+    @jit.unroll_safe
+    @specialize.arg(1)
     def operate(self, fmtdesc, repetitions):
         if fmtdesc.needcount:
             fmtdesc.pack(self, repetitions)
         else:
             for i in range(repetitions):
                 fmtdesc.pack(self)
-    operate._annspecialcase_ = 'specialize:arg(1)'
     _operate_is_specialized_ = True
 
+    @jit.unroll_safe
     def align(self, mask):
         pad = (-len(self.result)) & mask
         for i in range(pad):
@@ -130,13 +135,15 @@
         self.inputpos = 0
         self.result_w = []     # list of wrapped objects
 
+    # See above comment on operate.
+    @jit.unroll_safe
+    @specialize.arg(1)
     def operate(self, fmtdesc, repetitions):
         if fmtdesc.needcount:
             fmtdesc.unpack(self, repetitions)
         else:
             for i in range(repetitions):
                 fmtdesc.unpack(self)
-    operate._annspecialcase_ = 'specialize:arg(1)'
     _operate_is_specialized_ = True
 
     def align(self, mask):
@@ -154,7 +161,6 @@
         self.inputpos = end
         return s
 
+    @specialize.argtype(1)
     def appendobj(self, value):
         self.result_w.append(self.space.wrap(value))
-    appendobj._annspecialcase_ = 'specialize:argtype(1)'
-
diff --git a/pypy/module/struct/interp_struct.py b/pypy/module/struct/interp_struct.py
--- a/pypy/module/struct/interp_struct.py
+++ b/pypy/module/struct/interp_struct.py
@@ -3,6 +3,7 @@
 from pypy.rlib.rstruct.error import StructError
 from pypy.rlib.rstruct.formatiterator import CalcSizeFormatIterator
 
+
 @unwrap_spec(format=str)
 def calcsize(space, format):
     fmtiter = CalcSizeFormatIterator()
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -47,6 +47,7 @@
         'pypy_initial_path'     : 'state.pypy_initial_path',
 
         '_getframe'             : 'vm._getframe', 
+        '_current_frames'       : 'vm._current_frames', 
         'setrecursionlimit'     : 'vm.setrecursionlimit', 
         'getrecursionlimit'     : 'vm.getrecursionlimit', 
         'setcheckinterval'      : 'vm.setcheckinterval', 
diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -1,16 +1,16 @@
 # -*- coding: iso-8859-1 -*-
 import autopath
-from pypy.conftest import option
+from pypy.conftest import option, gettestobjspace
 from py.test import raises
 from pypy.interpreter.gateway import app2interp_temp
 import sys
 
 def test_stdin_exists(space):
-    space.sys.get('stdin') 
+    space.sys.get('stdin')
     space.sys.get('__stdin__')
 
 def test_stdout_exists(space):
-    space.sys.get('stdout') 
+    space.sys.get('stdout')
     space.sys.get('__stdout__')
 
 class AppTestAppSysTests:
@@ -25,7 +25,7 @@
         assert 'sys' in modules, ( "An entry for sys "
                                         "is not in sys.modules.")
         sys2 = sys.modules['sys']
-        assert sys is sys2, "import sys is not sys.modules[sys]." 
+        assert sys is sys2, "import sys is not sys.modules[sys]."
     def test_builtin_in_modules(self):
         import sys
         modules = sys.modules
@@ -89,12 +89,12 @@
         else:
             raise AssertionError, "ZeroDivisionError not caught"
 
-    def test_io(self): 
+    def test_io(self):
         import sys
         assert isinstance(sys.__stdout__, file)
         assert isinstance(sys.__stderr__, file)
         assert isinstance(sys.__stdin__, file)
-    
+
         if self.appdirect and not isinstance(sys.stdin, file):
             return
 
@@ -324,7 +324,7 @@
         import sys
         if self.appdirect:
             skip("not worth running appdirect")
-            
+
         encoding = sys.getdefaultencoding()
         try:
             sys.setdefaultencoding("ascii")
@@ -334,11 +334,11 @@
             sys.setdefaultencoding("latin-1")
             assert sys.getdefaultencoding() == 'latin-1'
             assert unicode('\x80') == u'\u0080'
-            
+
         finally:
             sys.setdefaultencoding(encoding)
 
-            
+
     # testing sys.settrace() is done in test_trace.py
     # testing sys.setprofile() is done in test_profile.py
 
@@ -372,6 +372,21 @@
             assert isinstance(v[3], int)
             assert isinstance(v[4], str)
 
+            assert v[0] == v.major
+            assert v[1] == v.minor
+            assert v[2] == v.build
+            assert v[3] == v.platform
+            assert v[4] == v.service_pack
+
+            assert isinstance(v.service_pack_minor, int)
+            assert isinstance(v.service_pack_major, int)
+            assert isinstance(v.suite_mask, int)
+            assert isinstance(v.product_type, int)
+
+            # This is how platform.py calls it. Make sure tuple still has 5
+            # elements
+            maj, min, buildno, plat, csd = sys.getwindowsversion()
+
     def test_winver(self):
         import sys
         if hasattr(sys, "winver"):
@@ -524,3 +539,51 @@
         # If this ever actually becomes a compilation option this test should
         # be changed.
         assert sys.float_repr_style == "short"
+
+class AppTestCurrentFrames:
+
+    def test_current_frames(self):
+        try:
+            import thread
+        except ImportError:
+            pass
+        else:
+            skip('This test requires an intepreter without threads')
+        import sys
+
+        def f():
+            return sys._current_frames()
+        frames = f()
+        assert frames.keys() == [0]
+        assert frames[0].f_code.co_name == 'f'
+
+class AppTestCurrentFramesWithThread(AppTestCurrentFrames):
+    def setup_class(cls):
+        cls.space = gettestobjspace(usemodules=('thread',))
+
+    def test_current_frames(self):
+        import sys
+        import time
+        import thread
+
+        thread_id = thread.get_ident()
+        self.ready = False
+        def other_thread():
+            self.ready = True
+            print "thread started"
+            time.sleep(5)
+        thread.start_new_thread(other_thread, ())
+
+        def f():
+            for i in range(100):
+                if self.ready: break
+                time.sleep(0.1)
+            return sys._current_frames()
+
+        frames = f()
+        thisframe = frames.pop(thread_id)
+        assert thisframe.f_code.co_name == 'f'
+
+        assert len(frames) == 1
+        _, other_frame = frames.popitem()
+        assert other_frame.f_code.co_name == 'other_thread'
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -10,7 +10,7 @@
 CPYTHON_VERSION            = (2, 7, 1, "final", 42)   #XXX # sync patchlevel.h
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 
-PYPY_VERSION               = (1, 6, 0, "dev", 1)    #XXX # sync patchlevel.h
+PYPY_VERSION               = (1, 6, 1, "dev", 0)    #XXX # sync patchlevel.h
 
 if platform.name == 'msvc':
     COMPILER_INFO = 'MSC v.%d 32 bit' % (platform.version * 10 + 600)
diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -1,11 +1,13 @@
 """
 Implementation of interpreter-level 'sys' routines.
 """
+import sys
+
+from pypy.interpreter import gateway
 from pypy.interpreter.error import OperationError
 from pypy.interpreter.gateway import unwrap_spec, NoneNotWrapped
+from pypy.rlib import jit
 from pypy.rlib.runicode import MAXUNICODE
-from pypy.rlib import jit
-import sys
 
 # ____________________________________________________________
 
@@ -43,6 +45,25 @@
     f.mark_as_escaped()
     return space.wrap(f)
 
+def _current_frames(space):
+    """_current_frames() -> dictionary
+
+    Return a dictionary mapping each current thread T's thread id to T's
+    current stack frame.
+
+    This function should be used for specialized purposes only."""
+    raise OperationError(space.w_NotImplementedError,
+        space.wrap("XXX sys._current_frames() incompatible with the JIT"))
+    w_result = space.newdict()
+    ecs = space.threadlocals.getallvalues()
+    for thread_ident, ec in ecs.items():
+        f = ec.gettopframe_nohidden()
+        f.mark_as_escaped()
+        space.setitem(w_result,
+                      space.wrap(thread_ident),
+                      space.wrap(f))
+    return w_result
+
 def setrecursionlimit(space, w_new_limit):
     """setrecursionlimit() sets the maximum number of nested calls that
 can occur before a RuntimeError is raised.  On PyPy the limit is
@@ -107,7 +128,7 @@
     """Set the global debug tracing function.  It will be called on each
 function call.  See the debugger chapter in the library manual."""
     space.getexecutioncontext().settrace(w_func)
-    
+
 def setprofile(space, w_func):
     """Set the profiling function.  It will be called on each function call
 and return.  See the profiler chapter in the library manual."""
@@ -128,14 +149,47 @@
 a debugger from a checkpoint, to recursively debug some other code."""
     return space.getexecutioncontext().call_tracing(w_func, w_args)
 
+
+app = gateway.applevel('''
+"NOT_RPYTHON"
+from _structseq import structseqtype, structseqfield
+
+class windows_version_info:
+    __metaclass__ = structseqtype
+
+    name = "sys.getwindowsversion"
+
+    major = structseqfield(0, "Major version number")
+    minor = structseqfield(1, "Minor version number")
+    build = structseqfield(2, "Build number")
+    platform = structseqfield(3, "Operating system platform")
+    service_pack = structseqfield(4, "Latest Service Pack installed on the system")
+
+    # Because the indices aren't consecutive, they aren't included when
+    # unpacking and other such operations.
+    service_pack_major = structseqfield(10, "Service Pack major version number")
+    service_pack_minor = structseqfield(11, "Service Pack minor version number")
+    suite_mask = structseqfield(12, "Bit mask identifying available product suites")
+    product_type = structseqfield(13, "System product type")
+''')
+
+
 def getwindowsversion(space):
     from pypy.rlib import rwin32
     info = rwin32.GetVersionEx()
-    return space.newtuple([space.wrap(info[0]),
-                           space.wrap(info[1]),
-                           space.wrap(info[2]),
-                           space.wrap(info[3]),
-                           space.wrap(info[4])])
+    w_windows_version_info = app.wget(space, "windows_version_info")
+    raw_version = space.newtuple([
+        space.wrap(info[0]),
+        space.wrap(info[1]),
+        space.wrap(info[2]),
+        space.wrap(info[3]),
+        space.wrap(info[4]),
+        space.wrap(info[5]),
+        space.wrap(info[6]),
+        space.wrap(info[7]),
+        space.wrap(info[8]),
+    ])
+    return space.call_function(w_windows_version_info, raw_version)
 
 @jit.dont_look_inside
 def get_dllhandle(space):
diff --git a/pypy/module/test_lib_pypy/test_greenlet.py b/pypy/module/test_lib_pypy/test_greenlet.py
--- a/pypy/module/test_lib_pypy/test_greenlet.py
+++ b/pypy/module/test_lib_pypy/test_greenlet.py
@@ -3,7 +3,7 @@
 
 class AppTestGreenlet:
     def setup_class(cls):
-        cls.space = gettestobjspace(usemodules=['_continuation'])
+        cls.space = gettestobjspace(usemodules=['_continuation'], continuation=True)
 
     def test_simple(self):
         from greenlet import greenlet
@@ -241,3 +241,42 @@
         g1 = greenlet(f1)
         raises(ValueError, g1.throw, ValueError)
         assert g1.dead
+
+    def test_exc_info_save_restore(self):
+        # sys.exc_info save/restore behaviour is wrong on CPython's greenlet
+        from greenlet import greenlet
+        import sys
+        def f():
+            try:
+                raise ValueError('fun')
+            except:
+                exc_info = sys.exc_info()
+                greenlet(h).switch()
+                assert exc_info == sys.exc_info()
+
+        def h():
+            assert sys.exc_info() == (None, None, None)
+
+        greenlet(f).switch()
+
+    def test_gr_frame(self):
+        from greenlet import greenlet
+        import sys
+        def f2():
+            assert g.gr_frame is None
+            gmain.switch()
+            assert g.gr_frame is None
+        def f1():
+            assert gmain.gr_frame is gmain_frame
+            assert g.gr_frame is None
+            f2()
+            assert g.gr_frame is None
+        gmain = greenlet.getcurrent()
+        assert gmain.gr_frame is None
+        gmain_frame = sys._getframe()
+        g = greenlet(f1)
+        assert g.gr_frame is None
+        g.switch()
+        assert g.gr_frame.f_code.co_name == 'f2'
+        g.switch()
+        assert g.gr_frame is None
diff --git a/pypy/module/test_lib_pypy/test_stackless_pickle.py b/pypy/module/test_lib_pypy/test_stackless_pickle.py
--- a/pypy/module/test_lib_pypy/test_stackless_pickle.py
+++ b/pypy/module/test_lib_pypy/test_stackless_pickle.py
@@ -1,25 +1,27 @@
-import py; py.test.skip("XXX port me")
+import py
+py.test.skip("in-progress, maybe")
 from pypy.conftest import gettestobjspace, option
 
 class AppTest_Stackless:
 
     def setup_class(cls):
-        import py.test
-        py.test.importorskip('greenlet')
-        space = gettestobjspace(usemodules=('_stackless', '_socket'))
+        space = gettestobjspace(usemodules=('_continuation', '_socket'))
         cls.space = space
-        # cannot test the unpickle part on top of py.py
+        if option.runappdirect:
+            cls.w_lev = space.wrap(14)
+        else:
+            cls.w_lev = space.wrap(2)
 
     def test_pickle(self):
         import new, sys
 
         mod = new.module('mod')
         sys.modules['mod'] = mod
+        mod.lev = self.lev
         try:
             exec '''
 import pickle, sys
 import stackless
-lev = 14
 
 ch = stackless.channel()
 seen = []
diff --git a/pypy/module/thread/threadlocals.py b/pypy/module/thread/threadlocals.py
--- a/pypy/module/thread/threadlocals.py
+++ b/pypy/module/thread/threadlocals.py
@@ -43,6 +43,9 @@
         ident = self._mainthreadident
         return self._valuedict.get(ident, None)
 
+    def getallvalues(self):
+        return self._valuedict
+
     def enter_thread(self, space):
         "Notification that the current thread is just starting."
         ec = space.getexecutioncontext()
diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -6,6 +6,7 @@
 from pypy.interpreter.typedef import default_identity_hash
 from pypy.tool.sourcetools import compile2, func_with_new_name
 from pypy.module.__builtin__.interp_classobj import W_InstanceObject
+from pypy.rlib.objectmodel import specialize
 
 def object_getattribute(space):
     "Utility that returns the app-level descriptor object.__getattribute__."
@@ -507,8 +508,9 @@
     def issubtype(space, w_sub, w_type):
         return space._type_issubtype(w_sub, w_type)
 
+    @specialize.arg_or_var(2)
     def isinstance(space, w_inst, w_type):
-        return space._type_isinstance(w_inst, w_type)
+        return space.wrap(space._type_isinstance(w_inst, w_type))
 
     def issubtype_allow_override(space, w_sub, w_type):
         w_check = space.lookup(w_type, "__subclasscheck__")
diff --git a/pypy/objspace/std/boolobject.py b/pypy/objspace/std/boolobject.py
--- a/pypy/objspace/std/boolobject.py
+++ b/pypy/objspace/std/boolobject.py
@@ -1,8 +1,10 @@
+from pypy.rlib.rbigint import rbigint
+from pypy.rlib.rarithmetic import r_uint
+from pypy.interpreter.error import OperationError
 from pypy.objspace.std.model import registerimplementation, W_Object
 from pypy.objspace.std.register_all import register_all
 from pypy.objspace.std.intobject import W_IntObject
 
-
 class W_BoolObject(W_Object):
     from pypy.objspace.std.booltype import bool_typedef as typedef
     _immutable_fields_ = ['boolval']
@@ -20,6 +22,21 @@
     def unwrap(w_self, space):
         return w_self.boolval
 
+    def int_w(w_self, space):
+        return int(w_self.boolval)
+
+    def uint_w(w_self, space):
+        intval = int(w_self.boolval)
+        if intval < 0:
+            raise OperationError(space.w_ValueError,
+                                 space.wrap("cannot convert negative integer to unsigned"))
+        else:
+            return r_uint(intval)
+
+    def bigint_w(w_self, space):
+        return rbigint.fromint(int(w_self.boolval))
+
+
 registerimplementation(W_BoolObject)
 
 W_BoolObject.w_False = W_BoolObject(False)
diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -250,7 +250,8 @@
 def repr__Bytearray(space, w_bytearray):
     s = w_bytearray.data
 
-    buf = StringBuilder(50)
+    # Good default if there are no replacements.
+    buf = StringBuilder(len("bytearray(b'')") + len(s))
 
     buf.append("bytearray(b'")
 
@@ -369,8 +370,8 @@
     newdata = []
     for i in range(len(list_w)):
         w_s = list_w[i]
-        if not (space.is_true(space.isinstance(w_s, space.w_str)) or
-                space.is_true(space.isinstance(w_s, space.w_bytearray))):
+        if not (space.isinstance_w(w_s, space.w_str) or
+                space.isinstance_w(w_s, space.w_bytearray)):
             raise operationerrfmt(
                 space.w_TypeError,
                 "sequence item %d: expected string, %s "
diff --git a/pypy/objspace/std/complextype.py b/pypy/objspace/std/complextype.py
--- a/pypy/objspace/std/complextype.py
+++ b/pypy/objspace/std/complextype.py
@@ -127,8 +127,8 @@
                and space.is_w(space.type(w_real), space.w_complex)):
         return w_real
 
-    if space.is_true(space.isinstance(w_real, space.w_str)) or \
-            space.is_true(space.isinstance(w_real, space.w_unicode)):
+    if space.isinstance_w(w_real, space.w_str) or \
+            space.isinstance_w(w_real, space.w_unicode):
         # a string argument
         if not noarg2:
             raise OperationError(space.w_TypeError,
@@ -203,8 +203,8 @@
         return (w_complex.realval, w_complex.imagval)
     #
     # Check that it is not a string (on which space.float() would succeed).
-    if (space.is_true(space.isinstance(w_complex, space.w_str)) or
-        space.is_true(space.isinstance(w_complex, space.w_unicode))):
+    if (space.isinstance_w(w_complex, space.w_str) or
+        space.isinstance_w(w_complex, space.w_unicode)):
         raise operationerrfmt(space.w_TypeError,
                               "complex number expected, got '%s'",
                               space.type(w_complex).getname(space))
diff --git a/pypy/objspace/std/default.py b/pypy/objspace/std/default.py
--- a/pypy/objspace/std/default.py
+++ b/pypy/objspace/std/default.py
@@ -1,48 +1,16 @@
 """Default implementation for some operation."""
 
-from pypy.interpreter.error import OperationError
+from pypy.interpreter.error import OperationError, typed_unwrap_error_msg
 from pypy.objspace.std.register_all import register_all
-from pypy.rlib import objectmodel
 
 
-# The following default implementations are used before delegation is tried.
-# 'id' is normally the address of the wrapper.
-
-def id__ANY(space, w_obj):
-    #print 'id:', w_obj
-    return space.wrap(objectmodel.compute_unique_id(w_obj))
-
 # __init__ should succeed if called internally as a multimethod
 
 def init__ANY(space, w_obj, __args__):
     pass
 
-def typed_unwrap_error_msg(space, expected, w_obj):
-    type_name = space.type(w_obj).getname(space)
-    return space.wrap("expected %s, got %s object" % (expected, type_name))
-
-def int_w__ANY(space,w_obj):
-    raise OperationError(space.w_TypeError,
-                         typed_unwrap_error_msg(space, "integer", w_obj))
-
-def str_w__ANY(space,w_obj):
-    raise OperationError(space.w_TypeError,
-                         typed_unwrap_error_msg(space, "string", w_obj))
-
 def float_w__ANY(space,w_obj):
     raise OperationError(space.w_TypeError,
                          typed_unwrap_error_msg(space, "float", w_obj))
 
-def uint_w__ANY(space,w_obj):
-    raise OperationError(space.w_TypeError,
-                         typed_unwrap_error_msg(space, "integer", w_obj))
-
-def unicode_w__ANY(space,w_obj):
-    raise OperationError(space.w_TypeError,
-                         typed_unwrap_error_msg(space, "unicode", w_obj))
-
-def bigint_w__ANY(space,w_obj):
-    raise OperationError(space.w_TypeError,
-                         typed_unwrap_error_msg(space, "integer", w_obj))
-
 register_all(vars())
diff --git a/pypy/objspace/std/floattype.py b/pypy/objspace/std/floattype.py
--- a/pypy/objspace/std/floattype.py
+++ b/pypy/objspace/std/floattype.py
@@ -32,14 +32,14 @@
         if space.is_w(w_floattype, space.w_float):
             return w_obj
         value = space.float_w(w_obj)
-    elif space.is_true(space.isinstance(w_value, space.w_str)):
+    elif space.isinstance_w(w_value, space.w_str):
         strvalue = space.str_w(w_value)
         try:
             value = string_to_float(strvalue)
         except ParseStringError, e:
             raise OperationError(space.w_ValueError,
                                  space.wrap(e.msg))
-    elif space.is_true(space.isinstance(w_value, space.w_unicode)):
+    elif space.isinstance_w(w_value, space.w_unicode):
         if space.config.objspace.std.withropeunicode:
             from pypy.objspace.std.ropeunicodeobject import unicode_to_decimal_w
         else:
diff --git a/pypy/objspace/std/formatting.py b/pypy/objspace/std/formatting.py
--- a/pypy/objspace/std/formatting.py
+++ b/pypy/objspace/std/formatting.py
@@ -1,13 +1,15 @@
 """
 String formatting routines.
 """
-from pypy.rlib.unroll import unrolling_iterable
+from pypy.interpreter.error import OperationError
+from pypy.objspace.std.unicodetype import unicode_from_object
+from pypy.rlib import jit
 from pypy.rlib.rarithmetic import ovfcheck
 from pypy.rlib.rfloat import formatd, DTSF_ALT, isnan, isinf
-from pypy.interpreter.error import OperationError
+from pypy.rlib.rstring import StringBuilder, UnicodeBuilder
+from pypy.rlib.unroll import unrolling_iterable
 from pypy.tool.sourcetools import func_with_new_name
-from pypy.rlib.rstring import StringBuilder, UnicodeBuilder
-from pypy.objspace.std.unicodetype import unicode_from_object
+
 
 class BaseStringFormatter(object):
     def __init__(self, space, values_w, w_valuedict):
@@ -173,6 +175,9 @@
                 raise OperationError(space.w_ValueError,
                                      space.wrap("incomplete format"))
 
+        # Only shows up if we've already started inlining format(), so just
+        # unconditionally unroll this.
+        @jit.unroll_safe
         def getmappingkey(self):
             # return the mapping key in a '%(key)s' specifier
             fmt = self.fmt
@@ -233,6 +238,8 @@
 
             return w_value
 
+        # Same as getmappingkey
+        @jit.unroll_safe
         def peel_flags(self):
             self.f_ljust = False
             self.f_sign  = False
@@ -255,6 +262,8 @@
                     break
                 self.forward()
 
+        # Same as getmappingkey
+        @jit.unroll_safe
         def peel_num(self):
             space = self.space
             c = self.peekchr()
@@ -276,6 +285,7 @@
                 c = self.peekchr()
             return result
 
+        @jit.look_inside_iff(lambda self: jit.isconstant(self.fmt))
         def format(self):
             lgt = len(self.fmt) + 4 * len(self.values_w) + 10
             if do_unicode:
@@ -415,15 +425,15 @@
                                      space.wrap("operand does not support "
                                                 "unary str"))
             w_result = space.get_and_call_function(w_impl, w_value)
-            if space.is_true(space.isinstance(w_result,
-                                              space.w_unicode)):
+            if space.isinstance_w(w_result,
+                                              space.w_unicode):
                 raise NeedUnicodeFormattingError
             return space.str_w(w_result)
 
         def fmt_s(self, w_value):
             space = self.space
-            got_unicode = space.is_true(space.isinstance(w_value,
-                                                         space.w_unicode))
+            got_unicode = space.isinstance_w(w_value,
+                                                         space.w_unicode)
             if not do_unicode:
                 if got_unicode:
                     raise NeedUnicodeFormattingError
@@ -442,13 +452,13 @@
         def fmt_c(self, w_value):
             self.prec = -1     # just because
             space = self.space
-            if space.is_true(space.isinstance(w_value, space.w_str)):
+            if space.isinstance_w(w_value, space.w_str):
                 s = space.str_w(w_value)
                 if len(s) != 1:
                     raise OperationError(space.w_TypeError,
                                          space.wrap("%c requires int or char"))
                 self.std_wp(s)
-            elif space.is_true(space.isinstance(w_value, space.w_unicode)):
+            elif space.isinstance_w(w_value, space.w_unicode):
                 if not do_unicode:
                     raise NeedUnicodeFormattingError
                 ustr = space.unicode_w(w_value)
@@ -510,15 +520,15 @@
     return space.wrap(result)
 
 def mod_format(space, w_format, w_values, do_unicode=False):
-    if space.is_true(space.isinstance(w_values, space.w_tuple)):
+    if space.isinstance_w(w_values, space.w_tuple):
         values_w = space.fixedview(w_values)
         return format(space, w_format, values_w, None, do_unicode)
     else:
         # we check directly for dict to avoid obscure checking
         # in simplest case
-        if space.is_true(space.isinstance(w_values, space.w_dict)) or \
+        if space.isinstance_w(w_values, space.w_dict) or \
            (space.lookup(w_values, '__getitem__') and
-           not space.is_true(space.isinstance(w_values, space.w_basestring))):
+           not space.isinstance_w(w_values, space.w_basestring)):
             return format(space, w_format, [w_values], w_values, do_unicode)
         else:
             return format(space, w_format, [w_values], None, do_unicode)
diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py
--- a/pypy/objspace/std/intobject.py
+++ b/pypy/objspace/std/intobject.py
@@ -30,7 +30,18 @@
 
     def unwrap(w_self, space):
         return int(w_self.intval)
+    int_w = unwrap
 
+    def uint_w(w_self, space):
+        intval = w_self.intval
+        if intval < 0:
+            raise OperationError(space.w_ValueError,
+                                 space.wrap("cannot convert negative integer to unsigned"))
+        else:
+            return r_uint(intval)
+
+    def bigint_w(w_self, space):
+        return rbigint.fromint(w_self.intval)
 
 registerimplementation(W_IntObject)
 
@@ -39,20 +50,6 @@
 # alias and then teach copy_multimethods in smallintobject.py to override
 # it. See int__Int for example.
 
-def int_w__Int(space, w_int1):
-    return int(w_int1.intval)
-
-def uint_w__Int(space, w_int1):
-    intval = w_int1.intval
-    if intval < 0:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("cannot convert negative integer to unsigned"))
-    else:
-        return r_uint(intval)

From noreply at buildbot.pypy.org  Sun Oct  2 01:16:37 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Sun,  2 Oct 2011 01:16:37 +0200 (CEST)
Subject: [pypy-commit] pypy inline-dict-ops: don't look inside
 ll_dict_lookup, it's a bit sketchy
Message-ID: <20111001231637.D69C682A01@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: inline-dict-ops
Changeset: r47763:6da395d124d3
Date: 2011-10-01 15:58 -0300
http://bitbucket.org/pypy/pypy/changeset/6da395d124d3/

Log:	don't look inside ll_dict_lookup, it's a bit sketchy

diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py
--- a/pypy/rpython/lltypesystem/rdict.py
+++ b/pypy/rpython/lltypesystem/rdict.py
@@ -528,6 +528,7 @@
 # ------- a port of CPython's dictobject.c's lookdict implementation -------
 PERTURB_SHIFT = 5
 
+ at jit.dont_look_inside
 def ll_dict_lookup(d, key, hash):
     entries = d.entries
     ENTRIES = lltype.typeOf(entries).TO

From noreply at buildbot.pypy.org  Sun Oct  2 08:55:31 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sun,  2 Oct 2011 08:55:31 +0200 (CEST)
Subject: [pypy-commit] pypy default: expose numpy.inf
Message-ID: <20111002065531.AFE9E820D9@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r47764:59460302c713
Date: 2011-10-02 02:55 -0400
http://bitbucket.org/pypy/pypy/changeset/59460302c713/

Log:	expose numpy.inf

diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -50,4 +50,5 @@
     appleveldefs = {
         'average': 'app_numpy.average',
         'mean': 'app_numpy.mean',
+        'inf': 'app_numpy.inf',
     }
diff --git a/pypy/module/micronumpy/app_numpy.py b/pypy/module/micronumpy/app_numpy.py
--- a/pypy/module/micronumpy/app_numpy.py
+++ b/pypy/module/micronumpy/app_numpy.py
@@ -1,5 +1,8 @@
 import numpy
 
+
+inf = float("inf")
+
 def average(a):
     # This implements a weighted average, for now we don't implement the
     # weighting, just the average part!
@@ -8,4 +11,4 @@
 def mean(a):
     if not hasattr(a, "mean"):
         a = numpy.array(a)
-    return a.mean()
\ No newline at end of file
+    return a.mean()
diff --git a/pypy/module/micronumpy/test/test_module.py b/pypy/module/micronumpy/test/test_module.py
--- a/pypy/module/micronumpy/test/test_module.py
+++ b/pypy/module/micronumpy/test/test_module.py
@@ -10,4 +10,9 @@
     def test_average(self):
         from numpy import array, average
         assert average(range(10)) == 4.5
-        assert average(array(range(10))) == 4.5
\ No newline at end of file
+        assert average(array(range(10))) == 4.5
+
+    def test_inf(self):
+        from numpy import inf
+        assert type(inf) is float
+        assert inf == float("inf")
\ No newline at end of file

From noreply at buildbot.pypy.org  Sun Oct  2 09:48:27 2011
From: noreply at buildbot.pypy.org (hakanardo)
Date: Sun,  2 Oct 2011 09:48:27 +0200 (CEST)
Subject: [pypy-commit] pypy jit-optimizeopt-cleanups: make
 Optimizer.newoperations private and provide access methods
Message-ID: <20111002074827.3FF3A820D9@wyvern.cs.uni-duesseldorf.de>

Author: Hakan Ardo <hakan at debian.org>
Branch: jit-optimizeopt-cleanups
Changeset: r47765:3274372f0d94
Date: 2011-10-02 09:47 +0200
http://bitbucket.org/pypy/pypy/changeset/3274372f0d94/

Log:	make Optimizer.newoperations private and provide access methods

diff --git a/pypy/jit/metainterp/optimizeopt/optimizer.py b/pypy/jit/metainterp/optimizeopt/optimizer.py
--- a/pypy/jit/metainterp/optimizeopt/optimizer.py
+++ b/pypy/jit/metainterp/optimizeopt/optimizer.py
@@ -328,7 +328,7 @@
         self.quasi_immutable_deps = None
         self.opaque_pointers = {}
         self.replaces_guard = {}
-        self.newoperations = []
+        self._newoperations = []
         self.optimizer = self
         self.optpure = None
         self.optearlyforce = None
@@ -423,6 +423,13 @@
             return constbox
         return None
 
+    def get_newoperations(self):
+        self.flush()
+        return self._newoperations
+
+    def clear_newoperations(self):
+        self._newoperations = []
+
     def make_equal_to(self, box, value, replace=False):
         assert isinstance(value, OptValue)
         assert replace or box not in self.values
@@ -471,10 +478,10 @@
 
     def propagate_all_forward(self):
         self.exception_might_have_happened = self.bridge
-        self.newoperations = []
+        self.clear_newoperations()
         for op in self.loop.operations:
             self.first_optimization.propagate_forward(op)
-        self.loop.operations = self.newoperations
+        self.loop.operations = self.get_newoperations()
         self.loop.quasi_immutable_deps = self.quasi_immutable_deps
         # accumulate counters
         self.resumedata_memo.update_counters(self.metainterp_sd.profiler)
@@ -513,15 +520,15 @@
                 op = self.store_final_boxes_in_guard(op)
         elif op.can_raise():
             self.exception_might_have_happened = True
-        self.newoperations.append(op)
+        self._newoperations.append(op)
 
     def replace_op(self, old_op, new_op):
         # XXX: Do we want to cache indexes to prevent search?
-        i = len(self.newoperations) 
+        i = len(self._newoperations) 
         while i > 0:
             i -= 1
-            if self.newoperations[i] is old_op:
-                self.newoperations[i] = new_op
+            if self._newoperations[i] is old_op:
+                self._newoperations[i] = new_op
                 break
         else:
             assert False
diff --git a/pypy/jit/metainterp/optimizeopt/unroll.py b/pypy/jit/metainterp/optimizeopt/unroll.py
--- a/pypy/jit/metainterp/optimizeopt/unroll.py
+++ b/pypy/jit/metainterp/optimizeopt/unroll.py
@@ -138,7 +138,7 @@
 
             KillHugeIntBounds(self.optimizer).apply()
             
-            loop.preamble.operations = self.optimizer.newoperations
+            loop.preamble.operations = self.optimizer.get_newoperations()
             jump_args = [self.getvalue(a).get_key_box() for a in jump_args]
 
             start_resumedescr = loop.preamble.start_resumedescr.clone_if_mutable()
@@ -182,7 +182,7 @@
             # operations needed to setup the proper state of those virtuals
             # in the peeled loop
             inputarg_setup_ops = []
-            preamble_optimizer.newoperations = []
+            preamble_optimizer.clear_newoperations()
             seen = {}
             for box in inputargs:
                 if box in seen:
@@ -197,8 +197,7 @@
                 seen[box] = True
                 value = preamble_optimizer.getvalue(box)
                 value.force_box(preamble_optimizer)
-            preamble_optimizer.flush()
-            inputarg_setup_ops += preamble_optimizer.newoperations
+            inputarg_setup_ops += preamble_optimizer.get_newoperations()
 
             # Setup the state of the new optimizer by emiting the
             # short preamble operations and discarding the result
@@ -235,7 +234,7 @@
             jmp.setdescr(loop.token)
             loop.preamble.operations.append(jmp)
 
-            loop.operations = self.optimizer.newoperations
+            loop.operations = self.optimizer.get_newoperations()
             maxguards = self.optimizer.metainterp_sd.warmrunnerdesc.memory_manager.max_retrace_guards
             
             if self.optimizer.emitted_guards > maxguards:
@@ -333,11 +332,11 @@
             newop = self.short_inliner.inline_op(op)
             self.optimizer.send_extra_operation(newop)
         
-        self.optimizer.flush()
+        newoperations = self.optimizer.get_newoperations()
 
         i = j = 0
-        while i < len(self.optimizer.newoperations) or j < len(jumpargs):
-            if i == len(self.optimizer.newoperations):
+        while i < len(newoperations) or j < len(jumpargs):
+            if i == len(newoperations):
                 while j < len(jumpargs):
                     a = jumpargs[j]
                     if self.optimizer.loop.logops:
@@ -346,7 +345,7 @@
                                     jumpargs, short_seen)
                     j += 1
             else:
-                op = self.optimizer.newoperations[i]
+                op = newoperations[i]
 
                 self.boxes_created_this_iteration[op.result] = True
                 args = op.getarglist()
@@ -361,6 +360,7 @@
                     self.import_box(a, inputargs, short, short_jumpargs,
                                     jumpargs, short_seen)
                 i += 1
+            newoperations = self.optimizer.get_newoperations()
 
         jumpop.initarglist(jumpargs)
         self.optimizer.send_extra_operation(jumpop)

From noreply at buildbot.pypy.org  Sun Oct  2 10:23:22 2011
From: noreply at buildbot.pypy.org (hakanardo)
Date: Sun,  2 Oct 2011 10:23:22 +0200 (CEST)
Subject: [pypy-commit] pypy jit-optimizeopt-cleanups: setfields from forced
 virtuals now delayed a bit
Message-ID: <20111002082322.66357820D9@wyvern.cs.uni-duesseldorf.de>

Author: Hakan Ardo <hakan at debian.org>
Branch: jit-optimizeopt-cleanups
Changeset: r47766:70e28f5386c4
Date: 2011-10-02 10:23 +0200
http://bitbucket.org/pypy/pypy/changeset/70e28f5386c4/

Log:	setfields from forced virtuals now delayed a bit

diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py
--- a/pypy/module/pypyjit/test_pypy_c/test_call.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_call.py
@@ -366,12 +366,12 @@
             # make sure that the "block" is not allocated
             ...
             i20 = force_token()
-            setfield_gc(p0, i20, descr=<SignedFieldDescr .*PyFrame.vable_token .*>)
             p22 = new_with_vtable(19511408)
             p24 = new_array(1, descr=<GcPtrArrayDescr>)
             p26 = new_with_vtable(ConstClass(W_ListObject))
             p27 = new(descr=<SizeDescr .*>)
             p29 = new_array(0, descr=<GcPtrArrayDescr>)
+            setfield_gc(p0, i20, descr=<SignedFieldDescr .*PyFrame.vable_token .*>)
             setfield_gc(p27, p29, descr=<GcPtrFieldDescr list.items .*>)
             setfield_gc(p26, p27, descr=<.* .*W_ListObject.inst_wrappeditems .*>)
             setarrayitem_gc(p24, 0, p26, descr=<GcPtrArrayDescr>)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_generators.py b/pypy/module/pypyjit/test_pypy_c/test_generators.py
--- a/pypy/module/pypyjit/test_pypy_c/test_generators.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_generators.py
@@ -19,8 +19,8 @@
         assert loop.match_by_id("generator", """
             i16 = force_token()
             p45 = new_with_vtable(ConstClass(W_IntObject))
-            setfield_gc(p45, i29, descr=<SignedFieldDescr .*>)
             i47 = arraylen_gc(p8, descr=<GcPtrArrayDescr>) # Should be removed by backend
             setarrayitem_gc(p8, 0, p45, descr=<GcPtrArrayDescr>)
+            setfield_gc(p45, i29, descr=<SignedFieldDescr .*>)
             jump(..., descr=...)
             """)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_instance.py b/pypy/module/pypyjit/test_pypy_c/test_instance.py
--- a/pypy/module/pypyjit/test_pypy_c/test_instance.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_instance.py
@@ -125,8 +125,8 @@
             i12 = force_token()
             --TICK--
             p20 = new_with_vtable(ConstClass(W_IntObject))
+            setfield_gc(ConstPtr(ptr21), p20, descr=<GcPtrFieldDescr .*TypeCell.inst_w_value .*>)
             setfield_gc(p20, i11, descr=<SignedFieldDescr.*W_IntObject.inst_intval .*>)
-            setfield_gc(ConstPtr(ptr21), p20, descr=<GcPtrFieldDescr .*TypeCell.inst_w_value .*>)
             jump(p0, p1, p2, p3, p4, p20, p6, i7, p20, descr=<Loop.>)
         """)
 

From noreply at buildbot.pypy.org  Sun Oct  2 16:23:45 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  2 Oct 2011 16:23:45 +0200 (CEST)
Subject: [pypy-commit] pypy gil-improvement: GIL code for Windows. After
 trying for a while I failed to find
Message-ID: <20111002142345.90024820D9@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: gil-improvement
Changeset: r47767:721b57bfd529
Date: 2011-10-02 16:22 +0200
http://bitbucket.org/pypy/pypy/changeset/721b57bfd529/

Log:	GIL code for Windows. After trying for a while I failed to find a
	reasonable fully fool-proof equivalent to the Posix code, so we end
	up with this hackish solution instead.

diff --git a/pypy/translator/c/src/thread_nt.h b/pypy/translator/c/src/thread_nt.h
--- a/pypy/translator/c/src/thread_nt.h
+++ b/pypy/translator/c/src/thread_nt.h
@@ -221,4 +221,57 @@
 #define RPyThreadTLS_Set(key, value)	TlsSetValue(key, value)
 
 
+/************************************************************/
+/* GIL code                                                 */
+/************************************************************/
+
+static volatile LONG pending_acquires = -1;
+static CRITICAL_SECTION mutex_gil;
+static HANDLE cond_gil;
+
+long RPyGilAllocate(void)
+{
+    pending_acquires = 0;
+    InitializeCriticalSection(&mutex_gil);
+    EnterCriticalSection(&mutex_gil);
+    cond_gil = CreateEvent (NULL, FALSE, FALSE, NULL);
+    return 1;
+}
+
+long RPyGilYieldThread(void)
+{
+    /* can be called even before RPyGilAllocate(), but in this case,
+       pending_acquires will be -1 */
+    if (pending_acquires <= 0)
+        return 0;
+    InterlockedIncrement(&pending_acquires);
+
+    /* hack: the three following lines do a pthread_cond_wait(), and
+       normally specifying a timeout of INFINITE would be fine.  But the
+       first and second operations are not done atomically, so there is a
+       (small) risk that PulseEvent misses the WaitForSingleObject().
+       In this case the process will just sleep 1 millisecond. */
+    LeaveCriticalSection(&mutex_gil);
+    WaitForSingleObject(&cond_gil, 1);
+    EnterCriticalSection(&mutex_gil);
+
+    InterlockedDecrement(&pending_acquires);
+    PulseEvent(&cond_gil);
+    return 1;
+}
+
+void RPyGilRelease(void)
+{
+    LeaveCriticalSection(&mutex_gil);
+}
+
+void RPyGilAcquire(void)
+{
+    InterlockedIncrement(&pending_acquires);
+    EnterCriticalSection(&mutex_gil);
+    InterlockedDecrement(&pending_acquires);
+    PulseEvent(&cond_gil);
+}
+
+
 #endif /* PYPY_NOT_MAIN_FILE */

From noreply at buildbot.pypy.org  Sun Oct  2 16:23:47 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  2 Oct 2011 16:23:47 +0200 (CEST)
Subject: [pypy-commit] pypy gil-improvement: merge heads
Message-ID: <20111002142347.117EE820D9@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: gil-improvement
Changeset: r47768:6edcd8cac15a
Date: 2011-10-02 16:23 +0200
http://bitbucket.org/pypy/pypy/changeset/6edcd8cac15a/

Log:	merge heads

diff --git a/pypy/annotation/test/test_annrpython.py b/pypy/annotation/test/test_annrpython.py
--- a/pypy/annotation/test/test_annrpython.py
+++ b/pypy/annotation/test/test_annrpython.py
@@ -3204,6 +3204,8 @@
         s = a.build_types(f, [])
         assert isinstance(s, annmodel.SomeList)
         assert not s.listdef.listitem.resized
+        assert not s.listdef.listitem.immutable
+        assert s.listdef.listitem.mutated
 
     def test_delslice(self):
         def f():
diff --git a/pypy/annotation/unaryop.py b/pypy/annotation/unaryop.py
--- a/pypy/annotation/unaryop.py
+++ b/pypy/annotation/unaryop.py
@@ -352,6 +352,7 @@
         check_negative_slice(s_start, s_stop)
         if not isinstance(s_iterable, SomeList):
             raise Exception("list[start:stop] = x: x must be a list")
+        lst.listdef.mutate()
         lst.listdef.agree(s_iterable.listdef)
         # note that setslice is not allowed to resize a list in RPython
 
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_util.py b/pypy/jit/metainterp/optimizeopt/test/test_util.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_util.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_util.py
@@ -182,7 +182,8 @@
                             EffectInfo.EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE,
                             can_invalidate=True))
     arraycopydescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT,
-             EffectInfo([], [], [], [], oopspecindex=EffectInfo.OS_ARRAYCOPY))
+             EffectInfo([], [arraydescr], [], [arraydescr],
+                        oopspecindex=EffectInfo.OS_ARRAYCOPY))
 
     for _name, _os in [
         ('strconcatdescr',               'OS_STR_CONCAT'),
diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py
--- a/pypy/objspace/std/intobject.py
+++ b/pypy/objspace/std/intobject.py
@@ -1,12 +1,13 @@
 from pypy.interpreter.error import OperationError
 from pypy.objspace.std import newformat
+from pypy.objspace.std.inttype import wrapint
 from pypy.objspace.std.model import registerimplementation, W_Object
-from pypy.objspace.std.register_all import register_all
 from pypy.objspace.std.multimethod import FailedToImplementArgs
 from pypy.objspace.std.noneobject import W_NoneObject
+from pypy.objspace.std.register_all import register_all
+from pypy.rlib import jit
 from pypy.rlib.rarithmetic import ovfcheck, ovfcheck_lshift, LONG_BIT, r_uint
 from pypy.rlib.rbigint import rbigint
-from pypy.objspace.std.inttype import wrapint
 
 """
 In order to have the same behavior running
@@ -169,7 +170,8 @@
 
 
 # helper for pow()
-def _impl_int_int_pow(space, iv, iw, iz=0):
+ at jit.look_inside_iff(lambda space, iv, iw, iz: jit.isconstant(iw) and jit.isconstant(iz))
+def _impl_int_int_pow(space, iv, iw, iz):
     if iw < 0:
         if iz != 0:
             raise OperationError(space.w_TypeError,
@@ -211,7 +213,7 @@
 def pow__Int_Int_None(space, w_int1, w_int2, w_int3):
     x = w_int1.intval
     y = w_int2.intval
-    return space.wrap(_impl_int_int_pow(space, x, y))
+    return space.wrap(_impl_int_int_pow(space, x, y, 0))
 
 def neg__Int(space, w_int1):
     a = w_int1.intval
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -386,7 +386,11 @@
     if len(items)== 0:
         raise OperationError(space.w_IndexError,
                              space.wrap("pop from empty list"))
-    idx = space.int_w(w_idx)
+    if space.isinstance_w(w_idx, space.w_float):
+        raise OperationError(space.w_TypeError,
+            space.wrap("integer argument expected, got float")
+        )
+    idx = space.int_w(space.int(w_idx))
     try:
         return items.pop(idx)
     except IndexError:
diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py
--- a/pypy/objspace/std/test/test_listobject.py
+++ b/pypy/objspace/std/test/test_listobject.py
@@ -705,6 +705,20 @@
         l.pop()
         assert l == range(9)
 
+    def test_pop_custom_int(self):
+        class A(object):
+            def __init__(self, x):
+                self.x = x
+
+            def __int__(self):
+                return self.x
+
+        l = range(10)
+        x = l.pop(A(-1))
+        assert x == 9
+        assert l == range(9)
+        raises(TypeError, range(10).pop, 1.0)
+
     def test_remove(self):
         c = list('hello world')
         c.remove('l')
diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -126,6 +126,12 @@
         raises(TypeError, type, 'test', 42, {})
         raises(TypeError, type, 'test', (object,), 42)
 
+    def test_call_type_subclass(self):
+        class A(type):
+            pass
+
+        assert A("hello") is str
+
     def test_bases(self):
         assert int.__bases__ == (object,)
         class X:
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -822,14 +822,6 @@
 
 def call__Type(space, w_type, __args__):
     promote(w_type)
-    # special case for type(x)
-    if space.is_w(w_type, space.w_type):
-        try:
-            w_obj, = __args__.fixedunpack(1)
-        except ValueError:
-            pass
-        else:
-            return space.type(w_obj)
     # invoke the __new__ of the type
     if not we_are_jitted():
         # note that the annotator will figure out that w_type.w_bltin_new can
diff --git a/pypy/objspace/std/typetype.py b/pypy/objspace/std/typetype.py
--- a/pypy/objspace/std/typetype.py
+++ b/pypy/objspace/std/typetype.py
@@ -1,17 +1,28 @@
-from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.interpreter import gateway
 from pypy.interpreter.argument import Arguments
+from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.interpreter.typedef import (GetSetProperty, descr_get_dict,
                                       weakref_descr)
 from pypy.objspace.std.stdtypedef import StdTypeDef
 
-def descr__new__(space, w_typetype, w_name, w_bases, w_dict):
+
+def descr__new__(space, w_typetype, w_name, w_bases=gateway.NoneNotWrapped,
+    w_dict=gateway.NoneNotWrapped):
+
     "This is used to create user-defined classes only."
     from pypy.objspace.std.typeobject import W_TypeObject
     # XXX check types
 
     w_typetype = _precheck_for_new(space, w_typetype)
 
+    # special case for type(x)
+    if (space.is_w(space.type(w_typetype), space.w_type) and w_bases is None and
+        w_dict is None):
+        return space.type(w_name)
+    elif w_bases is None or w_dict is None:
+        raise OperationError(space.w_TypeError, space.wrap("type() takes 1 or 3 arguments"))
+
+
     bases_w = space.fixedview(w_bases)
 
     w_winner = w_typetype
diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -167,10 +167,7 @@
 
     This is for advanced usage only.
     """
-    # I hate the annotator so much.
-    if NonConstant(False):
-        return True
-    return False
+    return NonConstant(False)
 
 @oopspec("jit.isvirtual(value)")
 @specialize.ll()
diff --git a/pypy/rlib/rgc.py b/pypy/rlib/rgc.py
--- a/pypy/rlib/rgc.py
+++ b/pypy/rlib/rgc.py
@@ -3,6 +3,7 @@
 
 from pypy.rlib import jit
 from pypy.rlib.objectmodel import we_are_translated, enforceargs, specialize
+from pypy.rlib.nonconst import NonConstant
 from pypy.rpython.extregistry import ExtRegistryEntry
 from pypy.rpython.lltypesystem import lltype, llmemory
 
@@ -143,6 +144,10 @@
     from pypy.rpython.lltypesystem.lloperation import llop
     from pypy.rlib.objectmodel import keepalive_until_here
 
+    # XXX: Hack to ensure that we get a proper effectinfo.write_descrs_arrays
+    if NonConstant(False):
+        dest[dest_start] = source[source_start]
+
     # supports non-overlapping copies only
     if not we_are_translated():
         if source == dest:
diff --git a/pypy/rlib/test/test_jit.py b/pypy/rlib/test/test_jit.py
--- a/pypy/rlib/test/test_jit.py
+++ b/pypy/rlib/test/test_jit.py
@@ -1,7 +1,7 @@
 import py
 from pypy.conftest import option
 from pypy.rlib.jit import hint, we_are_jitted, JitDriver, elidable_promote
-from pypy.rlib.jit import JitHintError, oopspec
+from pypy.rlib.jit import JitHintError, oopspec, isconstant
 from pypy.translator.translator import TranslationContext, graphof
 from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin
 from pypy.rpython.lltypesystem import lltype
@@ -137,6 +137,16 @@
             t.view()
         # assert did not raise
 
+    def test_isconstant(self):
+        def f(n):
+            assert n >= 0
+            assert isconstant(n) is False
+            l = []
+            l.append(n)
+            return len(l)
+        res = self.interpret(f, [234])
+        assert res == 1
+
 
 class TestJITLLtype(BaseTestJIT, LLRtypeMixin):
     pass
diff --git a/pypy/translator/c/genc.py b/pypy/translator/c/genc.py
--- a/pypy/translator/c/genc.py
+++ b/pypy/translator/c/genc.py
@@ -563,7 +563,10 @@
             else:
                 mk.definition('PYPY_MAIN_FUNCTION', "main")
 
-            if sys.platform == 'win32':
+            if (py.path.local.sysfind('python') or
+                py.path.local.sysfind('python.exe')):
+                python = 'python '
+            elif sys.platform == 'win32':
                 python = sys.executable.replace('\\', '/') + ' '
             else:
                 python = sys.executable + ' '

From noreply at buildbot.pypy.org  Sun Oct  2 17:42:49 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sun,  2 Oct 2011 17:42:49 +0200 (CEST)
Subject: [pypy-commit] pypy default: change how some code is organized,
 fixes the failing test_pypy_c tests hopefully
Message-ID: <20111002154249.03E94820D9@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r47769:e28ade06eff5
Date: 2011-10-02 11:42 -0400
http://bitbucket.org/pypy/pypy/changeset/e28ade06eff5/

Log:	change how some code is organized, fixes the failing test_pypy_c
	tests hopefully

diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -77,7 +77,7 @@
         for i in range(len(self.lookup_where)):
             self.lookup_where[i] = None_None
 
-# possible values of compares_by_identity_status 
+# possible values of compares_by_identity_status
 UNKNOWN = 0
 COMPARES_BY_IDENTITY = 1
 OVERRIDES_EQ_CMP_OR_HASH = 2
@@ -358,7 +358,7 @@
                 if w_value is not None:
                     return w_value
         return None
-                
+
     @unroll_safe
     def _lookup(w_self, key):
         space = w_self.space
@@ -854,7 +854,8 @@
         single_arg = False
     else:
         single_arg = True
-    if call_init and not (space.is_w(w_type, space.w_type) and single_arg):
+    if (call_init and not (space.is_w(w_type, space.w_type) and
+        not __args__.keywords and len(__args__.arguments_w) == 1)):
         w_descr = space.lookup(w_newobject, '__init__')
         w_result = space.get_and_call_args(w_descr, w_newobject, __args__)
         if not space.is_w(w_result, space.w_None):

From noreply at buildbot.pypy.org  Sun Oct  2 18:13:50 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  2 Oct 2011 18:13:50 +0200 (CEST)
Subject: [pypy-commit] pypy gil-improvement: Move around the PulseEvents.
 According to the test
Message-ID: <20111002161350.E01E2820D9@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: gil-improvement
Changeset: r47770:b659babedfb9
Date: 2011-10-02 18:13 +0200
http://bitbucket.org/pypy/pypy/changeset/b659babedfb9/

Log:	Move around the PulseEvents. According to the test
	(https://bitbucket.org/arigo/arigo/raw/default/hack/stm/misc) it
	still seems to be a correct placement, and it delays the PulseEvents
	as much as possible, minimizing the risks of a missed PulseEvent.

diff --git a/pypy/translator/c/src/thread_nt.h b/pypy/translator/c/src/thread_nt.h
--- a/pypy/translator/c/src/thread_nt.h
+++ b/pypy/translator/c/src/thread_nt.h
@@ -245,6 +245,7 @@
     if (pending_acquires <= 0)
         return 0;
     InterlockedIncrement(&pending_acquires);
+    PulseEvent(&cond_gil);
 
     /* hack: the three following lines do a pthread_cond_wait(), and
        normally specifying a timeout of INFINITE would be fine.  But the
@@ -256,13 +257,13 @@
     EnterCriticalSection(&mutex_gil);
 
     InterlockedDecrement(&pending_acquires);
-    PulseEvent(&cond_gil);
     return 1;
 }
 
 void RPyGilRelease(void)
 {
     LeaveCriticalSection(&mutex_gil);
+    PulseEvent(&cond_gil);
 }
 
 void RPyGilAcquire(void)
@@ -270,7 +271,6 @@
     InterlockedIncrement(&pending_acquires);
     EnterCriticalSection(&mutex_gil);
     InterlockedDecrement(&pending_acquires);
-    PulseEvent(&cond_gil);
 }
 
 

From noreply at buildbot.pypy.org  Sun Oct  2 18:42:44 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sun,  2 Oct 2011 18:42:44 +0200 (CEST)
Subject: [pypy-commit] pypy default: remove this dead code,
 needed for the optimization from the previous commit.
Message-ID: <20111002164244.18CAF820D9@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r47771:e93bb841a63d
Date: 2011-10-02 12:42 -0400
http://bitbucket.org/pypy/pypy/changeset/e93bb841a63d/

Log:	remove this dead code, needed for the optimization from the previous
	commit.

diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -848,12 +848,6 @@
         call_init = space.isinstance_w(w_newobject, w_type)
 
     # maybe invoke the __init__ of the type
-    try:
-        __args__.fixedunpack(1)
-    except ValueError:
-        single_arg = False
-    else:
-        single_arg = True
     if (call_init and not (space.is_w(w_type, space.w_type) and
         not __args__.keywords and len(__args__.arguments_w) == 1)):
         w_descr = space.lookup(w_newobject, '__init__')

From noreply at buildbot.pypy.org  Sun Oct  2 19:01:06 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sun,  2 Oct 2011 19:01:06 +0200 (CEST)
Subject: [pypy-commit] pypy default: add numpy.arcsinh
Message-ID: <20111002170106.82C13820D9@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r47772:7eb1a7979b5e
Date: 2011-10-02 13:00 -0400
http://bitbucket.org/pypy/pypy/changeset/7eb1a7979b5e/

Log:	add numpy.arcsinh

diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -23,6 +23,7 @@
         ("arccos", "arccos"),
         ("arcsin", "arcsin"),
         ("arctan", "arctan"),
+        ("arcsinh", "arcsinh"),
         ("copysign", "copysign"),
         ("cos", "cos"),
         ("divide", "divide"),
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -261,6 +261,9 @@
     @unaryop
     def arctan(self, v):
         return math.atan(v)
+    @unaryop
+    def arcsinh(self, v):
+        return math.asinh(v)
 
 class IntegerArithmeticDtype(ArithmaticTypeMixin):
     _mixin_ = True
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -268,6 +268,7 @@
             ("arcsin", "arcsin", 1, {"promote_to_float": True}),
             ("arccos", "arccos", 1, {"promote_to_float": True}),
             ("arctan", "arctan", 1, {"promote_to_float": True}),
+            ("arcsinh", "arcsinh", 1, {"promote_to_float": True}),
         ]:
             self.add_ufunc(space, *ufunc_def)
 
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -298,6 +298,14 @@
         b = arctan(a)
         assert math.isnan(b[0])
 
+    def test_arcsinh(self):
+        import math
+        from numpy import arcsinh, inf
+
+        for v in [inf, -inf, 1.0, math.e]:
+            assert math.asinh(v) == arcsinh(v)
+        assert math.isnan(arcsinh(float("nan")))
+
     def test_reduce_errors(self):
         from numpy import sin, add
 

From noreply at buildbot.pypy.org  Mon Oct  3 03:20:38 2011
From: noreply at buildbot.pypy.org (timo_jbo)
Date: Mon,  3 Oct 2011 03:20:38 +0200 (CEST)
Subject: [pypy-commit] pypy separate-applevel-numpy: merge default (namely
	add numpy.arcsinh)
Message-ID: <20111003012038.A5D18820D9@wyvern.cs.uni-duesseldorf.de>

Author: Timo Paulssen <timonator at perpetuum-immobile.de>
Branch: separate-applevel-numpy
Changeset: r47773:319239eb7b1d
Date: 2011-10-03 03:19 +0200
http://bitbucket.org/pypy/pypy/changeset/319239eb7b1d/

Log:	merge default (namely add numpy.arcsinh)

diff --git a/dotviewer/graphparse.py b/dotviewer/graphparse.py
--- a/dotviewer/graphparse.py
+++ b/dotviewer/graphparse.py
@@ -36,48 +36,45 @@
     print >> sys.stderr, "Warning: could not guess file type, using 'dot'"
     return 'unknown'
 
-def dot2plain(content, contenttype, use_codespeak=False):
-    if contenttype == 'plain':
-        # already a .plain file
-        return content
+def dot2plain_graphviz(content, contenttype, use_codespeak=False):
+    if contenttype != 'neato':
+        cmdline = 'dot -Tplain'
+    else:
+        cmdline = 'neato -Tplain'
+    #print >> sys.stderr, '* running:', cmdline
+    close_fds = sys.platform != 'win32'
+    p = subprocess.Popen(cmdline, shell=True, close_fds=close_fds,
+                         stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+    (child_in, child_out) = (p.stdin, p.stdout)
+    try:
+        import thread
+    except ImportError:
+        bkgndwrite(child_in, content)
+    else:
+        thread.start_new_thread(bkgndwrite, (child_in, content))
+    plaincontent = child_out.read()
+    child_out.close()
+    if not plaincontent:    # 'dot' is likely not installed
+        raise PlainParseError("no result from running 'dot'")
+    return plaincontent
 
-    if not use_codespeak:
-        if contenttype != 'neato':
-            cmdline = 'dot -Tplain'
-        else:
-            cmdline = 'neato -Tplain'
-        #print >> sys.stderr, '* running:', cmdline
-        close_fds = sys.platform != 'win32'
-        p = subprocess.Popen(cmdline, shell=True, close_fds=close_fds,
-                             stdin=subprocess.PIPE, stdout=subprocess.PIPE)
-        (child_in, child_out) = (p.stdin, p.stdout)
-        try:
-            import thread
-        except ImportError:
-            bkgndwrite(child_in, content)
-        else:
-            thread.start_new_thread(bkgndwrite, (child_in, content))
-        plaincontent = child_out.read()
-        child_out.close()
-        if not plaincontent:    # 'dot' is likely not installed
-            raise PlainParseError("no result from running 'dot'")
-    else:
-        import urllib
-        request = urllib.urlencode({'dot': content})
-        url = 'http://codespeak.net/pypy/convertdot.cgi'
-        print >> sys.stderr, '* posting:', url
-        g = urllib.urlopen(url, data=request)
-        result = []
-        while True:
-            data = g.read(16384)
-            if not data:
-                break
-            result.append(data)
-        g.close()
-        plaincontent = ''.join(result)
-        # very simple-minded way to give a somewhat better error message
-        if plaincontent.startswith('<body'):
-            raise Exception("the dot on codespeak has very likely crashed")
+def dot2plain_codespeak(content, contenttype):
+    import urllib
+    request = urllib.urlencode({'dot': content})
+    url = 'http://codespeak.net/pypy/convertdot.cgi'
+    print >> sys.stderr, '* posting:', url
+    g = urllib.urlopen(url, data=request)
+    result = []
+    while True:
+        data = g.read(16384)
+        if not data:
+            break
+        result.append(data)
+    g.close()
+    plaincontent = ''.join(result)
+    # very simple-minded way to give a somewhat better error message
+    if plaincontent.startswith('<body'):
+        raise Exception("the dot on codespeak has very likely crashed")
     return plaincontent
 
 def bkgndwrite(f, data):
@@ -148,10 +145,13 @@
 
 def parse_dot(graph_id, content, links={}, fixedfont=False):
     contenttype = guess_type(content)
-    try:
-        plaincontent = dot2plain(content, contenttype, use_codespeak=False)
-        return list(parse_plain(graph_id, plaincontent, links, fixedfont))
-    except PlainParseError:
-        # failed, retry via codespeak
-        plaincontent = dot2plain(content, contenttype, use_codespeak=True)
-        return list(parse_plain(graph_id, plaincontent, links, fixedfont))
+    if contenttype == 'plain':
+        plaincontent = content
+    else:
+        try:
+            plaincontent = dot2plain_graphviz(content, contenttype)
+        except PlainParseError, e:
+            print e
+            # failed, retry via codespeak
+            plaincontent = dot2plain_codespeak(content, contenttype)
+    return list(parse_plain(graph_id, plaincontent, links, fixedfont))
diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -317,7 +317,7 @@
     RegrTest('test_multibytecodec.py', usemodules='_multibytecodec'),
     RegrTest('test_multibytecodec_support.py', skip="not a test"),
     RegrTest('test_multifile.py'),
-    RegrTest('test_multiprocessing.py', skip='FIXME leaves subprocesses'),
+    RegrTest('test_multiprocessing.py', skip="FIXME leaves subprocesses"),
     RegrTest('test_mutants.py', core="possibly"),
     RegrTest('test_mutex.py'),
     RegrTest('test_netrc.py'),
diff --git a/lib-python/modified-2.7/gzip.py b/lib-python/modified-2.7/gzip.py
deleted file mode 100644
--- a/lib-python/modified-2.7/gzip.py
+++ /dev/null
@@ -1,514 +0,0 @@
-"""Functions that read and write gzipped files.
-
-The user of the file doesn't have to worry about the compression,
-but random access is not allowed."""
-
-# based on Andrew Kuchling's minigzip.py distributed with the zlib module
-
-import struct, sys, time, os
-import zlib
-import io
-import __builtin__
-
-__all__ = ["GzipFile","open"]
-
-FTEXT, FHCRC, FEXTRA, FNAME, FCOMMENT = 1, 2, 4, 8, 16
-
-READ, WRITE = 1, 2
-
-def write32u(output, value):
-    # The L format writes the bit pattern correctly whether signed
-    # or unsigned.
-    output.write(struct.pack("<L", value))
-
-def read32(input):
-    return struct.unpack("<I", input.read(4))[0]
-
-def open(filename, mode="rb", compresslevel=9):
-    """Shorthand for GzipFile(filename, mode, compresslevel).
-
-    The filename argument is required; mode defaults to 'rb'
-    and compresslevel defaults to 9.
-
-    """
-    return GzipFile(filename, mode, compresslevel)
-
-class GzipFile(io.BufferedIOBase):
-    """The GzipFile class simulates most of the methods of a file object with
-    the exception of the readinto() and truncate() methods.
-
-    """
-
-    myfileobj = None
-    max_read_chunk = 10 * 1024 * 1024   # 10Mb
-
-    def __init__(self, filename=None, mode=None,
-                 compresslevel=9, fileobj=None, mtime=None):
-        """Constructor for the GzipFile class.
-
-        At least one of fileobj and filename must be given a
-        non-trivial value.
-
-        The new class instance is based on fileobj, which can be a regular
-        file, a StringIO object, or any other object which simulates a file.
-        It defaults to None, in which case filename is opened to provide
-        a file object.
-
-        When fileobj is not None, the filename argument is only used to be
-        included in the gzip file header, which may includes the original
-        filename of the uncompressed file.  It defaults to the filename of
-        fileobj, if discernible; otherwise, it defaults to the empty string,
-        and in this case the original filename is not included in the header.
-
-        The mode argument can be any of 'r', 'rb', 'a', 'ab', 'w', or 'wb',
-        depending on whether the file will be read or written.  The default
-        is the mode of fileobj if discernible; otherwise, the default is 'rb'.
-        Be aware that only the 'rb', 'ab', and 'wb' values should be used
-        for cross-platform portability.
-
-        The compresslevel argument is an integer from 1 to 9 controlling the
-        level of compression; 1 is fastest and produces the least compression,
-        and 9 is slowest and produces the most compression.  The default is 9.
-
-        The mtime argument is an optional numeric timestamp to be written
-        to the stream when compressing.  All gzip compressed streams
-        are required to contain a timestamp.  If omitted or None, the
-        current time is used.  This module ignores the timestamp when
-        decompressing; however, some programs, such as gunzip, make use
-        of it.  The format of the timestamp is the same as that of the
-        return value of time.time() and of the st_mtime member of the
-        object returned by os.stat().
-
-        """
-
-        # guarantee the file is opened in binary mode on platforms
-        # that care about that sort of thing
-        if mode and 'b' not in mode:
-            mode += 'b'
-        if fileobj is None:
-            fileobj = self.myfileobj = __builtin__.open(filename, mode or 'rb')
-        if filename is None:
-            if hasattr(fileobj, 'name'): filename = fileobj.name
-            else: filename = ''
-        if mode is None:
-            if hasattr(fileobj, 'mode'): mode = fileobj.mode
-            else: mode = 'rb'
-
-        if mode[0:1] == 'r':
-            self.mode = READ
-            # Set flag indicating start of a new member
-            self._new_member = True
-            # Buffer data read from gzip file. extrastart is offset in
-            # stream where buffer starts. extrasize is number of
-            # bytes remaining in buffer from current stream position.
-            self.extrabuf = ""
-            self.extrasize = 0
-            self.extrastart = 0
-            self.name = filename
-            # Starts small, scales exponentially
-            self.min_readsize = 100
-
-        elif mode[0:1] == 'w' or mode[0:1] == 'a':
-            self.mode = WRITE
-            self._init_write(filename)
-            self.compress = zlib.compressobj(compresslevel,
-                                             zlib.DEFLATED,
-                                             -zlib.MAX_WBITS,
-                                             zlib.DEF_MEM_LEVEL,
-                                             0)
-        else:
-            raise IOError, "Mode " + mode + " not supported"
-
-        self.fileobj = fileobj
-        self.offset = 0
-        self.mtime = mtime
-
-        if self.mode == WRITE:
-            self._write_gzip_header()
-
-    @property
-    def filename(self):
-        import warnings
-        warnings.warn("use the name attribute", DeprecationWarning, 2)
-        if self.mode == WRITE and self.name[-3:] != ".gz":
-            return self.name + ".gz"
-        return self.name
-
-    def __repr__(self):
-        s = repr(self.fileobj)
-        return '<gzip ' + s[1:-1] + ' ' + hex(id(self)) + '>'
-
-    def _check_closed(self):
-        """Raises a ValueError if the underlying file object has been closed.
-
-        """
-        if self.closed:
-            raise ValueError('I/O operation on closed file.')
-
-    def _init_write(self, filename):
-        self.name = filename
-        self.crc = zlib.crc32("") & 0xffffffffL
-        self.size = 0
-        self.writebuf = []
-        self.bufsize = 0
-
-    def _write_gzip_header(self):
-        self.fileobj.write('\037\213')             # magic header
-        self.fileobj.write('\010')                 # compression method
-        fname = os.path.basename(self.name)
-        if fname.endswith(".gz"):
-            fname = fname[:-3]
-        flags = 0
-        if fname:
-            flags = FNAME
-        self.fileobj.write(chr(flags))
-        mtime = self.mtime
-        if mtime is None:
-            mtime = time.time()
-        write32u(self.fileobj, long(mtime))
-        self.fileobj.write('\002')
-        self.fileobj.write('\377')
-        if fname:
-            self.fileobj.write(fname + '\000')
-
-    def _init_read(self):
-        self.crc = zlib.crc32("") & 0xffffffffL
-        self.size = 0
-
-    def _read_gzip_header(self):
-        magic = self.fileobj.read(2)
-        if magic != '\037\213':
-            raise IOError, 'Not a gzipped file'
-        method = ord( self.fileobj.read(1) )
-        if method != 8:
-            raise IOError, 'Unknown compression method'
-        flag = ord( self.fileobj.read(1) )
-        self.mtime = read32(self.fileobj)
-        # extraflag = self.fileobj.read(1)
-        # os = self.fileobj.read(1)
-        self.fileobj.read(2)
-
-        if flag & FEXTRA:
-            # Read & discard the extra field, if present
-            xlen = ord(self.fileobj.read(1))
-            xlen = xlen + 256*ord(self.fileobj.read(1))
-            self.fileobj.read(xlen)
-        if flag & FNAME:
-            # Read and discard a null-terminated string containing the filename
-            while True:
-                s = self.fileobj.read(1)
-                if not s or s=='\000':
-                    break
-        if flag & FCOMMENT:
-            # Read and discard a null-terminated string containing a comment
-            while True:
-                s = self.fileobj.read(1)
-                if not s or s=='\000':
-                    break
-        if flag & FHCRC:
-            self.fileobj.read(2)     # Read & discard the 16-bit header CRC
-
-    def write(self,data):
-        self._check_closed()
-        if self.mode != WRITE:
-            import errno
-            raise IOError(errno.EBADF, "write() on read-only GzipFile object")
-
-        if self.fileobj is None:
-            raise ValueError, "write() on closed GzipFile object"
-
-        # Convert data type if called by io.BufferedWriter.
-        if isinstance(data, memoryview):
-            data = data.tobytes()
-
-        if len(data) > 0:
-            self.size = self.size + len(data)
-            self.crc = zlib.crc32(data, self.crc) & 0xffffffffL
-            self.fileobj.write( self.compress.compress(data) )
-            self.offset += len(data)
-
-        return len(data)
-
-    def read(self, size=-1):
-        self._check_closed()
-        if self.mode != READ:
-            import errno
-            raise IOError(errno.EBADF, "read() on write-only GzipFile object")
-
-        if self.extrasize <= 0 and self.fileobj is None:
-            return ''
-
-        readsize = 1024
-        if size < 0:        # get the whole thing
-            try:
-                while True:
-                    self._read(readsize)
-                    readsize = min(self.max_read_chunk, readsize * 2)
-            except EOFError:
-                size = self.extrasize
-        elif size == 0:
-            return ""
-        else:               # just get some more of it
-            try:
-                while size > self.extrasize:
-                    self._read(readsize)
-                    readsize = min(self.max_read_chunk, readsize * 2)
-            except EOFError:
-                if size > self.extrasize:
-                    size = self.extrasize
-
-        offset = self.offset - self.extrastart
-        chunk = self.extrabuf[offset: offset + size]
-        self.extrasize = self.extrasize - size
-
-        self.offset += size
-        return chunk
-
-    def _unread(self, buf):
-        self.extrasize = len(buf) + self.extrasize
-        self.offset -= len(buf)
-
-    def _read(self, size=1024):
-        if self.fileobj is None:
-            raise EOFError, "Reached EOF"
-
-        if self._new_member:
-            # If the _new_member flag is set, we have to
-            # jump to the next member, if there is one.
-            #
-            # First, check if we're at the end of the file;
-            # if so, it's time to stop; no more members to read.
-            pos = self.fileobj.tell()   # Save current position
-            self.fileobj.seek(0, 2)     # Seek to end of file
-            if pos == self.fileobj.tell():
-                raise EOFError, "Reached EOF"
-            else:
-                self.fileobj.seek( pos ) # Return to original position
-
-            self._init_read()
-            self._read_gzip_header()
-            self.decompress = zlib.decompressobj(-zlib.MAX_WBITS)
-            self._new_member = False
-
-        # Read a chunk of data from the file
-        buf = self.fileobj.read(size)
-
-        # If the EOF has been reached, flush the decompression object
-        # and mark this object as finished.
-
-        if buf == "":
-            uncompress = self.decompress.flush()
-            self._read_eof()
-            self._add_read_data( uncompress )
-            raise EOFError, 'Reached EOF'
-
-        uncompress = self.decompress.decompress(buf)
-        self._add_read_data( uncompress )
-
-        if self.decompress.unused_data != "":
-            # Ending case: we've come to the end of a member in the file,
-            # so seek back to the start of the unused data, finish up
-            # this member, and read a new gzip header.
-            # (The number of bytes to seek back is the length of the unused
-            # data, minus 8 because _read_eof() will rewind a further 8 bytes)
-            self.fileobj.seek( -len(self.decompress.unused_data)+8, 1)
-
-            # Check the CRC and file size, and set the flag so we read
-            # a new member on the next call
-            self._read_eof()
-            self._new_member = True
-
-    def _add_read_data(self, data):
-        self.crc = zlib.crc32(data, self.crc) & 0xffffffffL
-        offset = self.offset - self.extrastart
-        self.extrabuf = self.extrabuf[offset:] + data
-        self.extrasize = self.extrasize + len(data)
-        self.extrastart = self.offset
-        self.size = self.size + len(data)
-
-    def _read_eof(self):
-        # We've read to the end of the file, so we have to rewind in order
-        # to reread the 8 bytes containing the CRC and the file size.
-        # We check the that the computed CRC and size of the
-        # uncompressed data matches the stored values.  Note that the size
-        # stored is the true file size mod 2**32.
-        self.fileobj.seek(-8, 1)
-        crc32 = read32(self.fileobj)
-        isize = read32(self.fileobj)  # may exceed 2GB
-        if crc32 != self.crc:
-            raise IOError("CRC check failed %s != %s" % (hex(crc32),
-                                                         hex(self.crc)))
-        elif isize != (self.size & 0xffffffffL):
-            raise IOError, "Incorrect length of data produced"
-
-        # Gzip files can be padded with zeroes and still have archives.
-        # Consume all zero bytes and set the file position to the first
-        # non-zero byte. See http://www.gzip.org/#faq8
-        c = "\x00"
-        while c == "\x00":
-            c = self.fileobj.read(1)
-        if c:
-            self.fileobj.seek(-1, 1)
-
-    @property
-    def closed(self):
-        return self.fileobj is None
-
-    def close(self):
-        if self.fileobj is None:
-            return
-        if self.mode == WRITE:
-            self.fileobj.write(self.compress.flush())
-            write32u(self.fileobj, self.crc)
-            # self.size may exceed 2GB, or even 4GB
-            write32u(self.fileobj, self.size & 0xffffffffL)
-            self.fileobj = None
-        elif self.mode == READ:
-            self.fileobj = None
-        if self.myfileobj:
-            self.myfileobj.close()
-            self.myfileobj = None
-
-    def flush(self,zlib_mode=zlib.Z_SYNC_FLUSH):
-        self._check_closed()
-        if self.mode == WRITE:
-            # Ensure the compressor's buffer is flushed
-            self.fileobj.write(self.compress.flush(zlib_mode))
-            self.fileobj.flush()
-
-    def fileno(self):
-        """Invoke the underlying file object's fileno() method.
-
-        This will raise AttributeError if the underlying file object
-        doesn't support fileno().
-        """
-        return self.fileobj.fileno()
-
-    def rewind(self):
-        '''Return the uncompressed stream file position indicator to the
-        beginning of the file'''
-        if self.mode != READ:
-            raise IOError("Can't rewind in write mode")
-        self.fileobj.seek(0)
-        self._new_member = True
-        self.extrabuf = ""
-        self.extrasize = 0
-        self.extrastart = 0
-        self.offset = 0
-
-    def readable(self):
-        return self.mode == READ
-
-    def writable(self):
-        return self.mode == WRITE
-
-    def seekable(self):
-        return True
-
-    def seek(self, offset, whence=0):
-        if whence:
-            if whence == 1:
-                offset = self.offset + offset
-            else:
-                raise ValueError('Seek from end not supported')
-        if self.mode == WRITE:
-            if offset < self.offset:
-                raise IOError('Negative seek in write mode')
-            count = offset - self.offset
-            for i in range(count // 1024):
-                self.write(1024 * '\0')
-            self.write((count % 1024) * '\0')
-        elif self.mode == READ:
-            if offset == self.offset:
-                self.read(0) # to make sure that this file is open
-                return self.offset
-            if offset < self.offset:
-                # for negative seek, rewind and do positive seek
-                self.rewind()
-            count = offset - self.offset
-            for i in range(count // 1024):
-                self.read(1024)
-            self.read(count % 1024)
-
-        return self.offset
-
-    def readline(self, size=-1):
-        if size < 0:
-            # Shortcut common case - newline found in buffer.
-            offset = self.offset - self.extrastart
-            i = self.extrabuf.find('\n', offset) + 1
-            if i > 0:
-                self.extrasize -= i - offset
-                self.offset += i - offset
-                return self.extrabuf[offset: i]
-
-            size = sys.maxint
-            readsize = self.min_readsize
-        else:
-            readsize = size
-        bufs = []
-        while size != 0:
-            c = self.read(readsize)
-            i = c.find('\n')
-
-            # We set i=size to break out of the loop under two
-            # conditions: 1) there's no newline, and the chunk is
-            # larger than size, or 2) there is a newline, but the
-            # resulting line would be longer than 'size'.
-            if (size <= i) or (i == -1 and len(c) > size):
-                i = size - 1
-
-            if i >= 0 or c == '':
-                bufs.append(c[:i + 1])    # Add portion of last chunk
-                self._unread(c[i + 1:])   # Push back rest of chunk
-                break
-
-            # Append chunk to list, decrease 'size',
-            bufs.append(c)
-            size = size - len(c)
-            readsize = min(size, readsize * 2)
-        if readsize > self.min_readsize:
-            self.min_readsize = min(readsize, self.min_readsize * 2, 512)
-        return ''.join(bufs) # Return resulting line
-
-
-def _test():
-    # Act like gzip; with -d, act like gunzip.
-    # The input file is not deleted, however, nor are any other gzip
-    # options or features supported.
-    args = sys.argv[1:]
-    decompress = args and args[0] == "-d"
-    if decompress:
-        args = args[1:]
-    if not args:
-        args = ["-"]
-    for arg in args:
-        if decompress:
-            if arg == "-":
-                f = GzipFile(filename="", mode="rb", fileobj=sys.stdin)
-                g = sys.stdout
-            else:
-                if arg[-3:] != ".gz":
-                    print "filename doesn't end in .gz:", repr(arg)
-                    continue
-                f = open(arg, "rb")
-                g = __builtin__.open(arg[:-3], "wb")
-        else:
-            if arg == "-":
-                f = sys.stdin
-                g = GzipFile(filename="", mode="wb", fileobj=sys.stdout)
-            else:
-                f = __builtin__.open(arg, "rb")
-                g = open(arg + ".gz", "wb")
-        while True:
-            chunk = f.read(1024)
-            if not chunk:
-                break
-            g.write(chunk)
-        if g is not sys.stdout:
-            g.close()
-        if f is not sys.stdin:
-            f.close()
-
-if __name__ == '__main__':
-    _test()
diff --git a/lib-python/modified-2.7/tarfile.py b/lib-python/modified-2.7/tarfile.py
--- a/lib-python/modified-2.7/tarfile.py
+++ b/lib-python/modified-2.7/tarfile.py
@@ -252,8 +252,8 @@
        the high bit set. So we calculate two checksums, unsigned and
        signed.
     """
-    unsigned_chksum = 256 + sum(struct.unpack("148B8x356B", buf[:512]))
-    signed_chksum = 256 + sum(struct.unpack("148b8x356b", buf[:512]))
+    unsigned_chksum = 256 + sum(struct.unpack("148B", buf[:148]) + struct.unpack("356B", buf[156:512]))
+    signed_chksum = 256 + sum(struct.unpack("148b", buf[:148]) + struct.unpack("356b", buf[156:512]))
     return unsigned_chksum, signed_chksum
 
 def copyfileobj(src, dst, length=None):
@@ -265,6 +265,7 @@
     if length is None:
         shutil.copyfileobj(src, dst)
         return
+
     BUFSIZE = 16 * 1024
     blocks, remainder = divmod(length, BUFSIZE)
     for b in xrange(blocks):
@@ -801,19 +802,19 @@
         if self.closed:
             raise ValueError("I/O operation on closed file")
 
+        buf = ""
         if self.buffer:
             if size is None:
-                buf = self.buffer + self.fileobj.read()
+                buf = self.buffer
                 self.buffer = ""
             else:
                 buf = self.buffer[:size]
                 self.buffer = self.buffer[size:]
-                buf += self.fileobj.read(size - len(buf))
+
+        if size is None:
+            buf += self.fileobj.read()
         else:
-            if size is None:
-                buf = self.fileobj.read()
-            else:
-                buf = self.fileobj.read(size)
+            buf += self.fileobj.read(size - len(buf))
 
         self.position += len(buf)
         return buf
diff --git a/lib-python/modified-2.7/test/test_multiprocessing.py b/lib-python/modified-2.7/test/test_multiprocessing.py
--- a/lib-python/modified-2.7/test/test_multiprocessing.py
+++ b/lib-python/modified-2.7/test/test_multiprocessing.py
@@ -510,7 +510,6 @@
 
         p.join()
 
-    @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
     def test_qsize(self):
         q = self.Queue()
         try:
@@ -532,7 +531,6 @@
             time.sleep(DELTA)
             q.task_done()
 
-    @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
     def test_task_done(self):
         queue = self.JoinableQueue()
 
@@ -1091,7 +1089,6 @@
 class _TestPoolWorkerLifetime(BaseTestCase):
 
     ALLOWED_TYPES = ('processes', )
-    @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
     def test_pool_worker_lifetime(self):
         p = multiprocessing.Pool(3, maxtasksperchild=10)
         self.assertEqual(3, len(p._pool))
@@ -1280,7 +1277,6 @@
         queue = manager.get_queue()
         queue.put('hello world')
 
-    @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
     def test_rapid_restart(self):
         authkey = os.urandom(32)
         manager = QueueManager(
@@ -1297,6 +1293,7 @@
         queue = manager.get_queue()
         self.assertEqual(queue.get(), 'hello world')
         del queue
+        test_support.gc_collect()
         manager.shutdown()
         manager = QueueManager(
             address=addr, authkey=authkey, serializer=SERIALIZER)
@@ -1573,7 +1570,6 @@
 
     ALLOWED_TYPES = ('processes',)
 
-    @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
     def test_heap(self):
         iterations = 5000
         maxblocks = 50
diff --git a/lib_pypy/_functools.py b/lib_pypy/_functools.py
--- a/lib_pypy/_functools.py
+++ b/lib_pypy/_functools.py
@@ -18,5 +18,5 @@
 
     def __call__(self, *fargs, **fkeywords):
         if self.keywords is not None:
-            fkeywords.update(self.keywords)
+            fkeywords = dict(self.keywords, **fkeywords)
         return self.func(*(self.args + fargs), **fkeywords)
diff --git a/lib_pypy/greenlet.py b/lib_pypy/greenlet.py
--- a/lib_pypy/greenlet.py
+++ b/lib_pypy/greenlet.py
@@ -96,7 +96,16 @@
 
     @property
     def gr_frame(self):
-        raise NotImplementedError("attribute 'gr_frame' of greenlet objects")
+        # xxx this doesn't work when called on either the current or
+        # the main greenlet of another thread
+        if self is getcurrent():
+            return None
+        if self.__main:
+            self = getcurrent()
+        f = _continulet.__reduce__(self)[2][0]
+        if not f:
+            return None
+        return f.f_back.f_back.f_back   # go past start(), __switch(), switch()
 
 # ____________________________________________________________
 # Internal stuff
diff --git a/lib_pypy/pypy_test/test_stackless_pickling.py b/lib_pypy/pypy_test/test_stackless_pickling.py
--- a/lib_pypy/pypy_test/test_stackless_pickling.py
+++ b/lib_pypy/pypy_test/test_stackless_pickling.py
@@ -1,7 +1,3 @@
-"""
-this test should probably not run from CPython or py.py.
-I'm not entirely sure, how to do that.
-"""
 from __future__ import absolute_import
 from py.test import skip
 try:
@@ -16,11 +12,15 @@
 
 class Test_StacklessPickling:
 
+    def test_pickle_main_coroutine(self):
+        import stackless, pickle
+        s = pickle.dumps(stackless.coroutine.getcurrent())
+        print s
+        c = pickle.loads(s)
+        assert c is stackless.coroutine.getcurrent()
+
     def test_basic_tasklet_pickling(self):
-        try:
-            import stackless
-        except ImportError:
-            skip("can't load stackless and don't know why!!!")
+        import stackless
         from stackless import run, schedule, tasklet
         import pickle
 
diff --git a/lib_pypy/pyrepl/completing_reader.py b/lib_pypy/pyrepl/completing_reader.py
--- a/lib_pypy/pyrepl/completing_reader.py
+++ b/lib_pypy/pyrepl/completing_reader.py
@@ -229,7 +229,8 @@
 
     def after_command(self, cmd):
         super(CompletingReader, self).after_command(cmd)
-        if not isinstance(cmd, complete) and not isinstance(cmd, self_insert):
+        if not isinstance(cmd, self.commands['complete']) \
+           and not isinstance(cmd, self.commands['self_insert']):
             self.cmpltn_reset()
 
     def calc_screen(self):
diff --git a/lib_pypy/stackless.py b/lib_pypy/stackless.py
--- a/lib_pypy/stackless.py
+++ b/lib_pypy/stackless.py
@@ -5,7 +5,6 @@
 """
 
 
-import traceback
 import _continuation
 
 class TaskletExit(Exception):
@@ -14,33 +13,46 @@
 CoroutineExit = TaskletExit
 
 
+def _coroutine_getcurrent():
+    "Returns the current coroutine (i.e. the one which called this function)."
+    try:
+        return _tls.current_coroutine
+    except AttributeError:
+        # first call in this thread: current == main
+        return _coroutine_getmain()
+
+def _coroutine_getmain():
+    try:
+        return _tls.main_coroutine
+    except AttributeError:
+        # create the main coroutine for this thread
+        continulet = _continuation.continulet
+        main = coroutine()
+        main._frame = continulet.__new__(continulet)
+        main._is_started = -1
+        _tls.current_coroutine = _tls.main_coroutine = main
+        return _tls.main_coroutine
+
+
 class coroutine(object):
-    "we can't have continulet as a base, because continulets can't be rebound"
+    _is_started = 0      # 0=no, 1=yes, -1=main
 
     def __init__(self):
         self._frame = None
-        self.is_zombie = False
-
-    def __getattr__(self, attr):
-        return getattr(self._frame, attr)
-
-    def __del__(self):
-        self.is_zombie = True
-        del self._frame
-        self._frame = None
 
     def bind(self, func, *argl, **argd):
         """coro.bind(f, *argl, **argd) -> None.
            binds function f to coro. f will be called with
            arguments *argl, **argd
         """
-        if self._frame is None or not self._frame.is_pending():
-            def run(c):
-                _tls.current_coroutine = self
-                return func(*argl, **argd)
-            self._frame = frame = _continuation.continulet(run)
-        else:
+        if self.is_alive:
             raise ValueError("cannot bind a bound coroutine")
+        def run(c):
+            _tls.current_coroutine = self
+            self._is_started = 1
+            return func(*argl, **argd)
+        self._is_started = 0
+        self._frame = _continuation.continulet(run)
 
     def switch(self):
         """coro.switch() -> returnvalue
@@ -48,7 +60,7 @@
            f finishes, the returnvalue is that of f, otherwise
            None is returned
         """
-        current = _getcurrent()
+        current = _coroutine_getcurrent()
         try:
             current._frame.switch(to=self._frame)
         finally:
@@ -56,37 +68,30 @@
 
     def kill(self):
         """coro.kill() : kill coroutine coro"""
-        current = _getcurrent()
+        current = _coroutine_getcurrent()
         try:
             current._frame.throw(CoroutineExit, to=self._frame)
         finally:
             _tls.current_coroutine = current
 
-    def _is_alive(self):
-        if self._frame is None:
-            return False
-        return not self._frame.is_pending()
-    is_alive = property(_is_alive)
-    del _is_alive
+    @property
+    def is_alive(self):
+        return self._is_started < 0 or (
+            self._frame is not None and self._frame.is_pending())
 
-    def getcurrent():
-        """coroutine.getcurrent() -> the currently running coroutine"""
-        return _getcurrent()
-    getcurrent = staticmethod(getcurrent)
+    @property
+    def is_zombie(self):
+        return self._is_started > 0 and not self._frame.is_pending()
+
+    getcurrent = staticmethod(_coroutine_getcurrent)
 
     def __reduce__(self):
-        raise TypeError, 'pickling is not possible based upon continulets'
+        if self._is_started < 0:
+            return _coroutine_getmain, ()
+        else:
+            return type(self), (), self.__dict__
 
 
-def _getcurrent():
-    "Returns the current coroutine (i.e. the one which called this function)."
-    try:
-        return _tls.current_coroutine
-    except AttributeError:
-        # first call in this thread: current == main
-        _coroutine_create_main()
-        return _tls.current_coroutine
-
 try:
     from thread import _local
 except ImportError:
@@ -95,14 +100,8 @@
 
 _tls = _local()
 
-def _coroutine_create_main():
-    # create the main coroutine for this thread
-    _tls.current_coroutine = None
-    main_coroutine = coroutine()
-    typ = _continuation.continulet
-    main_coroutine._frame = typ.__new__(typ)
-    _tls.main_coroutine = main_coroutine
-    _tls.current_coroutine = main_coroutine
+
+# ____________________________________________________________
 
 
 from collections import deque
@@ -148,10 +147,7 @@
     _last_task = next
     assert not next.blocked
     if next is not current:
-        #try:
-            next.switch()
-        #except CoroutineExit:  --- they are the same anyway
-        #    raise TaskletExit
+        next.switch()
     return current
 
 def set_schedule_callback(callback):
@@ -175,34 +171,6 @@
         raise self.type, self.value, self.traceback
 
 #
-# helpers for pickling
-#
-
-_stackless_primitive_registry = {}
-
-def register_stackless_primitive(thang, retval_expr='None'):
-    import types
-    func = thang
-    if isinstance(thang, types.MethodType):
-        func = thang.im_func
-    code = func.func_code
-    _stackless_primitive_registry[code] = retval_expr
-    # It is not too nice to attach info via the code object, but
-    # I can't think of a better solution without a real transform.
-
-def rewrite_stackless_primitive(coro_state, alive, tempval):
-    flags, frame, thunk, parent = coro_state
-    while frame is not None:
-        retval_expr = _stackless_primitive_registry.get(frame.f_code)
-        if retval_expr:
-            # this tasklet needs to stop pickling here and return its value.
-            tempval = eval(retval_expr, globals(), frame.f_locals)
-            coro_state = flags, frame, thunk, parent
-            break
-        frame = frame.f_back
-    return coro_state, alive, tempval
-
-#
 #
 
 class channel(object):
@@ -354,8 +322,6 @@
         """
         return self._channel_action(None, -1)
 
-    register_stackless_primitive(receive, retval_expr='receiver.tempval')
-
     def send_exception(self, exp_type, msg):
         self.send(bomb(exp_type, exp_type(msg)))
 
@@ -372,9 +338,8 @@
         the runnables list.
         """
         return self._channel_action(msg, 1)
-            
-    register_stackless_primitive(send)
-            
+
+
 class tasklet(coroutine):
     """
     A tasklet object represents a tiny task in a Python thread.
@@ -456,7 +421,7 @@
 
         self.func = None
         coroutine.bind(self, _func)
-        back = _getcurrent()
+        back = _coroutine_getcurrent()
         coroutine.switch(self)
         self.alive = True
         _scheduler_append(self)
@@ -480,39 +445,6 @@
             raise RuntimeError, "The current tasklet cannot be removed."
             # not sure if I will revive this  " Use t=tasklet().capture()"
         _scheduler_remove(self)
-        
-    def __reduce__(self):
-        one, two, coro_state = coroutine.__reduce__(self)
-        assert one is coroutine
-        assert two == ()
-        # we want to get rid of the parent thing.
-        # for now, we just drop it
-        a, frame, c, d = coro_state
-
-        # Removing all frames related to stackless.py.
-        # They point to stuff we don't want to be pickled.
-
-        pickleframe = frame
-        while frame is not None:
-            if frame.f_code == schedule.func_code:
-                # Removing everything including and after the
-                # call to stackless.schedule()
-                pickleframe = frame.f_back
-                break
-            frame = frame.f_back
-        if d:
-            assert isinstance(d, coroutine)
-        coro_state = a, pickleframe, c, None
-        coro_state, alive, tempval = rewrite_stackless_primitive(coro_state, self.alive, self.tempval)
-        inst_dict = self.__dict__.copy()
-        inst_dict.pop('tempval', None)
-        return self.__class__, (), (coro_state, alive, tempval, inst_dict)
-
-    def __setstate__(self, (coro_state, alive, tempval, inst_dict)):
-        coroutine.__setstate__(self, coro_state)
-        self.__dict__.update(inst_dict)
-        self.alive = alive
-        self.tempval = tempval
 
 def getmain():
     """
@@ -601,30 +533,7 @@
     global _last_task
     _global_task_id = 0
     _main_tasklet = coroutine.getcurrent()
-    try:
-        _main_tasklet.__class__ = tasklet
-    except TypeError: # we are running pypy-c
-        class TaskletProxy(object):
-            """TaskletProxy is needed to give the _main_coroutine tasklet behaviour"""
-            def __init__(self, coro):
-                self._coro = coro
-
-            def __getattr__(self,attr):
-                return getattr(self._coro,attr)
-
-            def __str__(self):
-                return '<tasklet %s a:%s>' % (self._task_id, self.is_alive)
-
-            def __reduce__(self):
-                return getmain, ()
-
-            __repr__ = __str__
-
-
-        global _main_coroutine
-        _main_coroutine = _main_tasklet
-        _main_tasklet = TaskletProxy(_main_tasklet)
-        assert _main_tasklet.is_alive and not _main_tasklet.is_zombie
+    _main_tasklet.__class__ = tasklet         # XXX HAAAAAAAAAAAAAAAAAAAAACK
     _last_task = _main_tasklet
     tasklet._init.im_func(_main_tasklet, label='main')
     _squeue = deque()
diff --git a/py/_code/source.py b/py/_code/source.py
--- a/py/_code/source.py
+++ b/py/_code/source.py
@@ -139,7 +139,7 @@
                 trysource = self[start:end]
                 if trysource.isparseable():
                     return start, end
-        return start, end
+        return start, len(self)
 
     def getblockend(self, lineno):
         # XXX
diff --git a/pypy/annotation/policy.py b/pypy/annotation/policy.py
--- a/pypy/annotation/policy.py
+++ b/pypy/annotation/policy.py
@@ -1,6 +1,6 @@
 # base annotation policy for specialization
 from pypy.annotation.specialize import default_specialize as default
-from pypy.annotation.specialize import specialize_argvalue, specialize_argtype, specialize_arglistitemtype
+from pypy.annotation.specialize import specialize_argvalue, specialize_argtype, specialize_arglistitemtype, specialize_arg_or_var
 from pypy.annotation.specialize import memo, specialize_call_location
 # for some reason, model must be imported first,
 # or we create a cycle.
@@ -73,6 +73,7 @@
     default_specialize = staticmethod(default)
     specialize__memo = staticmethod(memo)
     specialize__arg = staticmethod(specialize_argvalue) # specialize:arg(N)
+    specialize__arg_or_var = staticmethod(specialize_arg_or_var)
     specialize__argtype = staticmethod(specialize_argtype) # specialize:argtype(N)
     specialize__arglistitemtype = staticmethod(specialize_arglistitemtype)
     specialize__call_location = staticmethod(specialize_call_location)
diff --git a/pypy/annotation/specialize.py b/pypy/annotation/specialize.py
--- a/pypy/annotation/specialize.py
+++ b/pypy/annotation/specialize.py
@@ -353,6 +353,16 @@
     key = tuple(key)
     return maybe_star_args(funcdesc, key, args_s)
 
+def specialize_arg_or_var(funcdesc, args_s, *argindices):
+    for argno in argindices:
+        if not args_s[argno].is_constant():
+            break
+    else:
+        # all constant
+        return specialize_argvalue(funcdesc, args_s, *argindices)
+    # some not constant
+    return maybe_star_args(funcdesc, None, args_s)
+
 def specialize_argtype(funcdesc, args_s, *argindices):
     key = tuple([args_s[i].knowntype for i in argindices])
     for cls in key:
diff --git a/pypy/annotation/test/test_annrpython.py b/pypy/annotation/test/test_annrpython.py
--- a/pypy/annotation/test/test_annrpython.py
+++ b/pypy/annotation/test/test_annrpython.py
@@ -1194,6 +1194,20 @@
         assert len(executedesc._cache[(0, 'star', 2)].startblock.inputargs) == 4
         assert len(executedesc._cache[(1, 'star', 3)].startblock.inputargs) == 5
 
+    def test_specialize_arg_or_var(self):
+        def f(a):
+            return 1
+        f._annspecialcase_ = 'specialize:arg_or_var(0)'
+
+        def fn(a):
+            return f(3) + f(a)
+
+        a = self.RPythonAnnotator()
+        a.build_types(fn, [int])
+        executedesc = a.bookkeeper.getdesc(f)
+        assert sorted(executedesc._cache.keys()) == [None, (3,)]
+        # we got two different special
+
     def test_specialize_call_location(self):
         def g(a):
             return a
@@ -3190,6 +3204,8 @@
         s = a.build_types(f, [])
         assert isinstance(s, annmodel.SomeList)
         assert not s.listdef.listitem.resized
+        assert not s.listdef.listitem.immutable
+        assert s.listdef.listitem.mutated
 
     def test_delslice(self):
         def f():
diff --git a/pypy/annotation/unaryop.py b/pypy/annotation/unaryop.py
--- a/pypy/annotation/unaryop.py
+++ b/pypy/annotation/unaryop.py
@@ -352,6 +352,7 @@
         check_negative_slice(s_start, s_stop)
         if not isinstance(s_iterable, SomeList):
             raise Exception("list[start:stop] = x: x must be a list")
+        lst.listdef.mutate()
         lst.listdef.agree(s_iterable.listdef)
         # note that setslice is not allowed to resize a list in RPython
 
diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -53,6 +53,18 @@
 this is an ideal task to get started, because it does not require any deep
 knowledge of the internals.
 
+Optimized Unicode Representation
+--------------------------------
+
+CPython 3.3 will use an `optimized unicode representation`_ which switches between
+different ways to represent a unicode string, depending on whether the string
+fits into ASCII, has only two-byte characters or needs four-byte characters.
+
+The actual details would be rather differen in PyPy, but we would like to have
+the same optimization implemented.
+
+.. _`optimized unicode representation`: http://www.python.org/dev/peps/pep-0393/
+
 Translation Toolchain
 ---------------------
 
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -3,12 +3,12 @@
 from pypy.interpreter.executioncontext import ExecutionContext, ActionFlag
 from pypy.interpreter.executioncontext import UserDelAction, FrameTraceAction
 from pypy.interpreter.error import OperationError, operationerrfmt
-from pypy.interpreter.error import new_exception_class
+from pypy.interpreter.error import new_exception_class, typed_unwrap_error_msg
 from pypy.interpreter.argument import Arguments
 from pypy.interpreter.miscutils import ThreadLocals
 from pypy.tool.cache import Cache
 from pypy.tool.uid import HUGEVAL_BYTES
-from pypy.rlib.objectmodel import we_are_translated, newlist
+from pypy.rlib.objectmodel import we_are_translated, newlist, compute_unique_id
 from pypy.rlib.debug import make_sure_not_resized
 from pypy.rlib.timer import DummyTimer, Timer
 from pypy.rlib.rarithmetic import r_uint
@@ -186,6 +186,28 @@
     def _set_mapdict_storage_and_map(self, storage, map):
         raise NotImplementedError
 
+    # -------------------------------------------------------------------
+
+    def str_w(self, space):
+        w_msg = typed_unwrap_error_msg(space, "string", self)
+        raise OperationError(space.w_TypeError, w_msg)
+
+    def unicode_w(self, space):
+        raise OperationError(space.w_TypeError,
+                             typed_unwrap_error_msg(space, "unicode", self))
+
+    def int_w(self, space):
+        raise OperationError(space.w_TypeError,
+                             typed_unwrap_error_msg(space, "integer", self))
+    
+    def uint_w(self, space):
+        raise OperationError(space.w_TypeError,
+                             typed_unwrap_error_msg(space, "integer", self))
+    
+    def bigint_w(self, space):
+        raise OperationError(space.w_TypeError,
+                             typed_unwrap_error_msg(space, "integer", self))
+
 
 class Wrappable(W_Root):
     """A subclass of Wrappable is an internal, interpreter-level class
@@ -901,7 +923,7 @@
         ec.c_call_trace(frame, w_func, args)
         try:
             w_res = self.call_args(w_func, args)
-        except OperationError, e:
+        except OperationError:
             ec.c_exception_trace(frame, w_func)
             raise
         ec.c_return_trace(frame, w_func, args)
@@ -947,6 +969,9 @@
     def isinstance_w(self, w_obj, w_type):
         return self.is_true(self.isinstance(w_obj, w_type))
 
+    def id(self, w_obj):
+        return self.wrap(compute_unique_id(w_obj))
+
     # The code below only works
     # for the simple case (new-style instance).
     # These methods are patched with the full logic by the __builtin__
@@ -999,8 +1024,6 @@
 
     def eval(self, expression, w_globals, w_locals, hidden_applevel=False):
         "NOT_RPYTHON: For internal debugging."
-        import types
-        from pypy.interpreter.pycode import PyCode
         if isinstance(expression, str):
             compiler = self.createcompiler()
             expression = compiler.compile(expression, '?', 'eval', 0,
@@ -1012,7 +1035,6 @@
     def exec_(self, statement, w_globals, w_locals, hidden_applevel=False,
               filename=None):
         "NOT_RPYTHON: For internal debugging."
-        import types
         if filename is None:
             filename = '?'
         from pypy.interpreter.pycode import PyCode
@@ -1210,6 +1232,18 @@
             return None
         return self.str_w(w_obj)
 
+    def str_w(self, w_obj):
+        return w_obj.str_w(self)
+
+    def int_w(self, w_obj):
+        return w_obj.int_w(self)
+
+    def uint_w(self, w_obj):
+        return w_obj.uint_w(self)
+
+    def bigint_w(self, w_obj):
+        return w_obj.bigint_w(self)
+
     def realstr_w(self, w_obj):
         # Like str_w, but only works if w_obj is really of type 'str'.
         if not self.is_true(self.isinstance(w_obj, self.w_str)):
@@ -1217,6 +1251,9 @@
                                  self.wrap('argument must be a string'))
         return self.str_w(w_obj)
 
+    def unicode_w(self, w_obj):
+        return w_obj.unicode_w(self)
+
     def realunicode_w(self, w_obj):
         # Like unicode_w, but only works if w_obj is really of type
         # 'unicode'.
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -458,3 +458,7 @@
     if module:
         space.setattr(w_exc, space.wrap("__module__"), space.wrap(module))
     return w_exc
+
+def typed_unwrap_error_msg(space, expected, w_obj):
+    type_name = space.type(w_obj).getname(space)
+    return space.wrap("expected %s, got %s object" % (expected, type_name))
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -66,7 +66,7 @@
         make_sure_not_resized(self.locals_stack_w)
         check_nonneg(self.nlocals)
         #
-        if space.config.objspace.honor__builtins__ and w_globals is not None:
+        if space.config.objspace.honor__builtins__:
             self.builtin = space.builtin.pick_builtin(w_globals)
         # regular functions always have CO_OPTIMIZED and CO_NEWLOCALS.
         # class bodies only have CO_NEWLOCALS.
diff --git a/pypy/jit/metainterp/optimizeopt/optimizer.py b/pypy/jit/metainterp/optimizeopt/optimizer.py
--- a/pypy/jit/metainterp/optimizeopt/optimizer.py
+++ b/pypy/jit/metainterp/optimizeopt/optimizer.py
@@ -223,6 +223,9 @@
     def __init__(self, box):
         self.make_constant(box)
 
+    def __repr__(self):
+        return 'Constant(%r)' % (self.box,)
+
 CONST_0      = ConstInt(0)
 CONST_1      = ConstInt(1)
 CVAL_ZERO    = ConstantValue(CONST_0)
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -4767,6 +4767,26 @@
         # other
         self.optimize_loop(ops, expected)
 
+    def test_plain_virtual_string_copy_content(self):
+        ops = """
+        []
+        p0 = newstr(6)
+        copystrcontent(s"hello!", p0, 0, 0, 6)
+        p1 = call(0, p0, s"abc123", descr=strconcatdescr)
+        i0 = strgetitem(p1, 0)
+        finish(i0)
+        """
+        expected = """
+        []
+        p0 = newstr(6)
+        copystrcontent(s"hello!", p0, 0, 0, 6)
+        p1 = newstr(12)
+        copystrcontent(p0, p1, 0, 0, 6)
+        copystrcontent(s"abc123", p1, 0, 6, 6)
+        i0 = strgetitem(p1, 0)
+        finish(i0)
+        """
+        self.optimize_strunicode_loop(ops, expected)
 
 
 class TestLLtype(BaseTestOptimizeBasic, LLtypeMixin):
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -7220,6 +7220,16 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_setarrayitem_followed_by_arraycopy(self):
+        ops = """
+        [p1, p2]
+        setarrayitem_gc(p1, 2, 10, descr=arraydescr)
+        setarrayitem_gc(p2, 3, 13, descr=arraydescr)        
+        call(0, p1, p2, 0, 0, 10, descr=arraycopydescr)
+        jump(p1, p2)
+        """
+        self.optimize_loop(ops, ops)
+
 class TestLLtype(OptimizeOptTest, LLtypeMixin):
     pass
 
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_util.py b/pypy/jit/metainterp/optimizeopt/test/test_util.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_util.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_util.py
@@ -182,7 +182,8 @@
                             EffectInfo.EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE,
                             can_invalidate=True))
     arraycopydescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT,
-             EffectInfo([], [], [], [], oopspecindex=EffectInfo.OS_ARRAYCOPY))
+             EffectInfo([], [arraydescr], [], [arraydescr],
+                        oopspecindex=EffectInfo.OS_ARRAYCOPY))
 
     for _name, _os in [
         ('strconcatdescr',               'OS_STR_CONCAT'),
diff --git a/pypy/jit/metainterp/optimizeopt/vstring.py b/pypy/jit/metainterp/optimizeopt/vstring.py
--- a/pypy/jit/metainterp/optimizeopt/vstring.py
+++ b/pypy/jit/metainterp/optimizeopt/vstring.py
@@ -141,6 +141,11 @@
                                    for c in self._chars])
 
     def string_copy_parts(self, optimizer, targetbox, offsetbox, mode):
+        if not self.is_virtual() and targetbox is not self.box:
+            lengthbox = self.getstrlen(optimizer, mode)
+            srcbox = self.force_box()
+            return copy_str_content(optimizer, srcbox, targetbox,
+                                CONST_0, offsetbox, lengthbox, mode)
         for i in range(len(self._chars)):
             charbox = self._chars[i].force_box()
             if not (isinstance(charbox, Const) and charbox.same_constant(CONST_0)):
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -3408,6 +3408,75 @@
         assert res == main(1, 10)
         self.check_loops(call=0)
 
+    def test_setarrayitem_followed_by_arraycopy(self):
+        myjitdriver = JitDriver(greens = [], reds = ['n', 'sa', 'x', 'y'])
+        def f(n):
+            sa = 0
+            x = [1,2,n]
+            y = [1,2,3]
+            while n > 0:
+                myjitdriver.jit_merge_point(sa=sa, n=n, x=x, y=y)
+                y[0] = n
+                x[0:3] = y
+                sa += x[0]
+                n -= 1
+            return sa
+        res = self.meta_interp(f, [16])
+        assert res == f(16)
+        
+
 
 class TestLLtype(BaseLLtypeTests, LLJitMixin):
-    pass
+    def test_tagged(self):
+        py.test.skip("implement me")
+        from pypy.rlib.objectmodel import UnboxedValue
+        class Base(object):
+            __slots__ = ()
+
+        class Int(UnboxedValue, Base):
+            __slots__ = ["a"]
+
+            def is_pos(self):
+                return self.a > 0
+
+            def dec(self):
+                return Int(self.a - 1)
+
+
+        class Float(Base):
+            def __init__(self, a):
+                self.a = a
+
+            def is_pos(self):
+                return self.a > 0
+
+            def dec(self):
+                return Float(self.a - 1)
+
+        driver = JitDriver(greens=['pc', 's'], reds=['o'])
+
+        def main(fl, n, s):
+            if s:
+                s = "--j"
+            else:
+                s = "---j"
+            if fl:
+                o = Float(float(n))
+            else:
+                o = Int(n)
+            pc = 0
+            while True:
+                driver.jit_merge_point(s=s, pc=pc, o=o)
+                c = s[pc]
+                if c == "j":
+                    driver.can_enter_jit(s=s, pc=pc, o=o)
+                    if o.is_pos():
+                        pc = 0
+                        continue
+                    else:
+                        break
+                elif c == "-":
+                    o = o.dec()
+                pc += 1
+            return pc
+        res = self.meta_interp(main, [False, 100, True], taggedpointers=True)
diff --git a/pypy/jit/metainterp/test/test_string.py b/pypy/jit/metainterp/test/test_string.py
--- a/pypy/jit/metainterp/test/test_string.py
+++ b/pypy/jit/metainterp/test/test_string.py
@@ -575,4 +575,18 @@
                 n -= 1
             return result
         res = self.meta_interp(main, [9])
-        assert res == main(9)
\ No newline at end of file
+        assert res == main(9)
+
+    def test_virtual_copystringcontent2(self):
+        jitdriver = JitDriver(reds=['n', 'result'], greens=[])
+        def main(n):
+            result = 0
+            while n >= 0:
+                jitdriver.jit_merge_point(n=n, result=result)
+                b = StringBuilder(6)
+                b.append("Hello!")
+                result += ord((b.build() + "xyz")[0])
+                n -= 1
+            return result
+        res = self.meta_interp(main, [9])
+        assert res == main(9)
diff --git a/pypy/jit/metainterp/warmspot.py b/pypy/jit/metainterp/warmspot.py
--- a/pypy/jit/metainterp/warmspot.py
+++ b/pypy/jit/metainterp/warmspot.py
@@ -53,6 +53,8 @@
         extraconfigopts = {'translation.list_comprehension_operations': True}
     else:
         extraconfigopts = {}
+    if kwds.pop("taggedpointers", False):
+        extraconfigopts["translation.taggedpointers"] = True
     interp, graph = get_interpreter(function, args,
                                     backendopt=False,  # will be done below
                                     type_system=type_system,
diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py
--- a/pypy/module/__builtin__/functional.py
+++ b/pypy/module/__builtin__/functional.py
@@ -3,13 +3,13 @@
 
 """
 
+from pypy.interpreter.baseobjspace import Wrappable
 from pypy.interpreter.error import OperationError
-from pypy.interpreter.gateway import NoneNotWrapped
-from pypy.interpreter.gateway import interp2app, unwrap_spec
+from pypy.interpreter.gateway import NoneNotWrapped, interp2app, unwrap_spec
 from pypy.interpreter.typedef import TypeDef
-from pypy.interpreter.baseobjspace import Wrappable
+from pypy.rlib import jit
+from pypy.rlib.objectmodel import specialize
 from pypy.rlib.rarithmetic import r_uint, intmask
-from pypy.rlib.objectmodel import specialize
 from pypy.rlib.rbigint import rbigint
 
 
@@ -134,29 +134,15 @@
 
 
 @specialize.arg(2)
+ at jit.look_inside_iff(lambda space, args, implementation_of:
+    jit.isconstant(len(args.arguments_w)) and
+    len(args.arguments_w) == 2
+)
 def min_max(space, args, implementation_of):
     if implementation_of == "max":
         compare = space.gt
     else:
         compare = space.lt
-
-    args_w = args.arguments_w
-    if len(args_w) == 2 and not args.keywords:
-        # simple case, suitable for the JIT
-        w_arg0, w_arg1 = args_w
-        if space.is_true(compare(w_arg0, w_arg1)):
-            return w_arg0
-        else:
-            return w_arg1
-    else:
-        return min_max_loop(space, args, implementation_of)
-
- at specialize.arg(2)
-def min_max_loop(space, args, implementation_of):
-    if implementation_of == "max":
-        compare = space.gt
-    else:
-        compare = space.lt
     args_w = args.arguments_w
     if len(args_w) > 1:
         w_sequence = space.newtuple(args_w)
diff --git a/pypy/module/_continuation/__init__.py b/pypy/module/_continuation/__init__.py
--- a/pypy/module/_continuation/__init__.py
+++ b/pypy/module/_continuation/__init__.py
@@ -37,4 +37,5 @@
     interpleveldefs = {
         'continulet': 'interp_continuation.W_Continulet',
         'permute': 'interp_continuation.permute',
+        '_p': 'interp_continuation.unpickle',      # pickle support
     }
diff --git a/pypy/module/_continuation/interp_continuation.py b/pypy/module/_continuation/interp_continuation.py
--- a/pypy/module/_continuation/interp_continuation.py
+++ b/pypy/module/_continuation/interp_continuation.py
@@ -6,6 +6,7 @@
 from pypy.interpreter.typedef import TypeDef
 from pypy.interpreter.gateway import interp2app
 from pypy.interpreter.pycode import PyCode
+from pypy.interpreter.pyframe import PyFrame
 
 
 class W_Continulet(Wrappable):
@@ -23,24 +24,27 @@
         if ec.stacklet_thread is not self.sthread:
             global_state.clear()
             raise geterror(self.space, "inter-thread support is missing")
-        return ec
 
     def descr_init(self, w_callable, __args__):
         if self.sthread is not None:
             raise geterror(self.space, "continulet already __init__ialized")
+        #
+        # hackish: build the frame "by hand", passing it the correct arguments
+        space = self.space
+        w_args, w_kwds = __args__.topacked()
+        bottomframe = space.createframe(get_entrypoint_pycode(space),
+                                        get_w_module_dict(space), None)
+        bottomframe.locals_stack_w[0] = space.wrap(self)
+        bottomframe.locals_stack_w[1] = w_callable
+        bottomframe.locals_stack_w[2] = w_args
+        bottomframe.locals_stack_w[3] = w_kwds
+        self.bottomframe = bottomframe
+        #
         global_state.origin = self
-        global_state.w_callable = w_callable
-        global_state.args = __args__
-        self.bottomframe = make_fresh_frame(self.space)
-        self.sthread = build_sthread(self.space)
-        try:
-            self.h = self.sthread.new(new_stacklet_callback)
-            if self.sthread.is_empty_handle(self.h):    # early return
-                raise MemoryError
-        except MemoryError:
-            self.sthread = None
-            global_state.clear()
-            raise getmemoryerror(self.space)
+        sthread = build_sthread(self.space)
+        self.sthread = sthread
+        h = sthread.new(new_stacklet_callback)
+        post_switch(sthread, h)
 
     def switch(self, w_to):
         sthread = self.sthread
@@ -66,7 +70,7 @@
             if sthread.is_empty_handle(to.h):
                 global_state.clear()
                 raise geterror(self.space, "continulet already finished")
-        ec = self.check_sthread()
+        self.check_sthread()
         #
         global_state.origin = self
         if to is None:
@@ -76,13 +80,8 @@
             # double switch: the final destination is to.h
             global_state.destination = to
         #
-        try:
-            do_switch(sthread, global_state.destination.h)
-        except MemoryError:
-            global_state.clear()
-            raise getmemoryerror(self.space)
-        #
-        return get_result()
+        h = sthread.switch(global_state.destination.h)
+        return post_switch(sthread, h)
 
     def descr_switch(self, w_value=None, w_to=None):
         global_state.w_value = w_value
@@ -109,12 +108,26 @@
                  and not self.sthread.is_empty_handle(self.h))
         return self.space.newbool(valid)
 
+    def descr__reduce__(self):
+        from pypy.module._continuation import interp_pickle
+        return interp_pickle.reduce(self)
+
+    def descr__setstate__(self, w_args):
+        from pypy.module._continuation import interp_pickle
+        interp_pickle.setstate(self, w_args)
+
 
 def W_Continulet___new__(space, w_subtype, __args__):
     r = space.allocate_instance(W_Continulet, w_subtype)
     r.__init__(space)
     return space.wrap(r)
 
+def unpickle(space, w_subtype):
+    """Pickle support."""
+    r = space.allocate_instance(W_Continulet, w_subtype)
+    r.__init__(space)
+    return space.wrap(r)
+
 
 W_Continulet.typedef = TypeDef(
     'continulet',
@@ -124,9 +137,10 @@
     switch      = interp2app(W_Continulet.descr_switch),
     throw       = interp2app(W_Continulet.descr_throw),
     is_pending  = interp2app(W_Continulet.descr_is_pending),
+    __reduce__  = interp2app(W_Continulet.descr__reduce__),
+    __setstate__= interp2app(W_Continulet.descr__setstate__),
     )
 
-
 # ____________________________________________________________
 
 # Continulet objects maintain a dummy frame object in order to ensure
@@ -135,27 +149,40 @@
 
 class State:
     def __init__(self, space):
-        from pypy.interpreter.astcompiler.consts import CO_OPTIMIZED
-        self.space = space 
+        self.space = space
         w_module = space.getbuiltinmodule('_continuation')
         self.w_error = space.getattr(w_module, space.wrap('error'))
-        self.w_memoryerror = OperationError(space.w_MemoryError, space.w_None)
-        self.dummy_pycode = PyCode(space, 0, 0, 0, CO_OPTIMIZED,
-                                   '', [], [], [], '',
-                                   '<bottom of continulet>', 0, '', [], [],
-                                   hidden_applevel=True)
+        # the following function switches away immediately, so that
+        # continulet.__init__() doesn't immediately run func(), but it
+        # also has the hidden purpose of making sure we have a single
+        # bottomframe for the whole duration of the continulet's run.
+        # Hackish: only the func_code is used, and used in the context
+        # of w_globals == this module, so we can access the name
+        # 'continulet' directly.
+        w_code = space.appexec([], '''():
+            def start(c, func, args, kwds):
+                if continulet.switch(c) is not None:
+                    raise TypeError(
+                     "can\'t send non-None value to a just-started continulet")
+                return func(c, *args, **kwds)
+            return start.func_code
+        ''')
+        self.entrypoint_pycode = space.interp_w(PyCode, w_code)
+        self.entrypoint_pycode.hidden_applevel = True
+        self.w_unpickle = w_module.get('_p')
+        self.w_module_dict = w_module.getdict(space)
 
 def geterror(space, message):
     cs = space.fromcache(State)
     return OperationError(cs.w_error, space.wrap(message))
 
-def getmemoryerror(space):
+def get_entrypoint_pycode(space):
     cs = space.fromcache(State)
-    return cs.w_memoryerror
+    return cs.entrypoint_pycode
 
-def make_fresh_frame(space):
+def get_w_module_dict(space):
     cs = space.fromcache(State)
-    return space.FrameClass(space, cs.dummy_pycode, None, None)
+    return cs.w_module_dict
 
 # ____________________________________________________________
 
@@ -166,6 +193,9 @@
         StackletThread.__init__(self, space.config)
         self.space = space
         self.ec = ec
+        # for unpickling
+        from pypy.rlib.rweakref import RWeakKeyDictionary
+        self.frame2continulet = RWeakKeyDictionary(PyFrame, W_Continulet)
 
 ExecutionContext.stacklet_thread = None
 
@@ -176,8 +206,6 @@
     def clear(self):
         self.origin = None
         self.destination = None
-        self.w_callable = None
-        self.args = None
         self.w_value = None
         self.propagate_exception = None
 global_state = GlobalState()
@@ -185,27 +213,13 @@
 
 
 def new_stacklet_callback(h, arg):
-    self       = global_state.origin
-    w_callable = global_state.w_callable
-    args       = global_state.args
+    self = global_state.origin
+    self.h = h
     global_state.clear()
-    try:
-        do_switch(self.sthread, h)
-    except MemoryError:
-        return h       # oups!  do an early return in this case
-    #
     space = self.space
     try:
-        assert self.sthread.ec.topframeref() is None
-        self.sthread.ec.topframeref = jit.non_virtual_ref(self.bottomframe)
-        if global_state.propagate_exception is not None:
-            raise global_state.propagate_exception  # just propagate it further
-        if global_state.w_value is not space.w_None:
-            raise OperationError(space.w_TypeError, space.wrap(
-                "can't send non-None value to a just-started continulet"))
-
-        args = args.prepend(self.space.wrap(self))
-        w_result = space.call_args(w_callable, args)
+        frame = self.bottomframe
+        w_result = frame.execute_frame()
     except Exception, e:
         global_state.propagate_exception = e
     else:
@@ -215,9 +229,7 @@
     global_state.destination = self
     return self.h
 
-
-def do_switch(sthread, h):
-    h = sthread.switch(h)
+def post_switch(sthread, h):
     origin = global_state.origin
     self = global_state.destination
     global_state.origin = None
@@ -228,6 +240,8 @@
     sthread.ec.topframeref = self.bottomframe.f_backref
     self.bottomframe.f_backref = origin.bottomframe.f_backref
     origin.bottomframe.f_backref = current
+    #
+    return get_result()
 
 def get_result():
     if global_state.propagate_exception:
diff --git a/pypy/module/_continuation/interp_pickle.py b/pypy/module/_continuation/interp_pickle.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_continuation/interp_pickle.py
@@ -0,0 +1,128 @@
+from pypy.tool import stdlib_opcode as pythonopcode
+from pypy.rlib import jit
+from pypy.interpreter.error import OperationError
+from pypy.interpreter.pyframe import PyFrame
+from pypy.module._continuation.interp_continuation import State, global_state
+from pypy.module._continuation.interp_continuation import build_sthread
+from pypy.module._continuation.interp_continuation import post_switch
+from pypy.module._continuation.interp_continuation import get_result, geterror
+
+
+def getunpickle(space):
+    cs = space.fromcache(State)
+    return cs.w_unpickle
+
+
+def reduce(self):
+    # xxx this is known to be not completely correct with respect
+    # to subclasses, e.g. no __slots__ support, no looking for a
+    # __getnewargs__ or __getstate__ defined in the subclass, etc.
+    # Doing the right thing looks involved, though...
+    space = self.space
+    if self.sthread is None:
+        w_frame = space.w_False
+    elif self.sthread.is_empty_handle(self.h):
+        w_frame = space.w_None
+    else:
+        w_frame = space.wrap(self.bottomframe)
+    w_continulet_type = space.type(space.wrap(self))
+    w_dict = self.getdict(space) or space.w_None
+    args = [getunpickle(space),
+            space.newtuple([w_continulet_type]),
+            space.newtuple([w_frame, w_dict]),
+            ]
+    return space.newtuple(args)
+
+def setstate(self, w_args):
+    space = self.space
+    if self.sthread is not None:
+        raise geterror(space, "continulet.__setstate__() on an already-"
+                              "initialized continulet")
+    w_frame, w_dict = space.fixedview(w_args, expected_length=2)
+    if not space.is_w(w_dict, space.w_None):
+        self.setdict(space, w_dict)
+    if space.is_w(w_frame, space.w_False):
+        return    # not initialized
+    sthread = build_sthread(self.space)
+    self.sthread = sthread
+    self.bottomframe = space.interp_w(PyFrame, w_frame, can_be_None=True)
+    #
+    global_state.origin = self
+    if self.bottomframe is not None:
+        sthread.frame2continulet.set(self.bottomframe, self)
+    self.h = sthread.new(resume_trampoline_callback)
+    get_result()    # propagate the eventual MemoryError
+
+# ____________________________________________________________
+
+def resume_trampoline_callback(h, arg):
+    self = global_state.origin
+    self.h = h
+    space = self.space
+    sthread = self.sthread
+    try:
+        global_state.clear()
+        if self.bottomframe is None:
+            w_result = space.w_None
+        else:
+            h = sthread.switch(self.h)
+            try:
+                w_result = post_switch(sthread, h)
+                operr = None
+            except OperationError, e:
+                w_result = None
+                operr = e
+            #
+            while True:
+                ec = sthread.ec
+                frame = ec.topframeref()
+                assert frame is not None     # XXX better error message
+                exit_continulet = sthread.frame2continulet.get(frame)
+                #
+                continue_after_call(frame)
+                #
+                # small hack: unlink frame out of the execution context,
+                # because execute_frame will add it there again
+                ec.topframeref = frame.f_backref
+                #
+                try:
+                    w_result = frame.execute_frame(w_result, operr)
+                    operr = None
+                except OperationError, e:
+                    w_result = None
+                    operr = e
+                if exit_continulet is not None:
+                    self = exit_continulet
+                    break
+            sthread.ec.topframeref = jit.vref_None
+            if operr:
+                raise operr
+    except Exception, e:
+        global_state.propagate_exception = e
+    else:
+        global_state.w_value = w_result
+    global_state.origin = self
+    global_state.destination = self
+    return self.h
+
+def continue_after_call(frame):
+    code = frame.pycode.co_code
+    instr = frame.last_instr
+    opcode = ord(code[instr])
+    map = pythonopcode.opmap
+    call_ops = [map['CALL_FUNCTION'], map['CALL_FUNCTION_KW'],
+                map['CALL_FUNCTION_VAR'], map['CALL_FUNCTION_VAR_KW'],
+                map['CALL_METHOD']]
+    assert opcode in call_ops   # XXX check better, and complain better
+    instr += 1
+    oparg = ord(code[instr]) | ord(code[instr + 1]) << 8
+    nargs = oparg & 0xff
+    nkwds = (oparg >> 8) & 0xff
+    if nkwds == 0:     # only positional arguments
+        # fast paths leaves things on the stack, pop them
+        if (frame.space.config.objspace.opcodes.CALL_METHOD and
+            opcode == map['CALL_METHOD']):
+            frame.dropvalues(nargs + 2)
+        elif opcode == map['CALL_FUNCTION']:
+            frame.dropvalues(nargs + 1)
+    frame.last_instr = instr + 1    # continue after the call
diff --git a/pypy/module/_continuation/test/test_stacklet.py b/pypy/module/_continuation/test/test_stacklet.py
--- a/pypy/module/_continuation/test/test_stacklet.py
+++ b/pypy/module/_continuation/test/test_stacklet.py
@@ -13,7 +13,7 @@
         from _continuation import continulet
         #
         def empty_callback(c):
-            pass
+            never_called
         #
         c = continulet(empty_callback)
         assert type(c) is continulet
@@ -36,7 +36,7 @@
         from _continuation import continulet, error
         #
         def empty_callback(c1):
-            pass
+            never_called
         #
         c = continulet(empty_callback)
         raises(error, c.__init__, empty_callback)
diff --git a/pypy/module/_continuation/test/test_zpickle.py b/pypy/module/_continuation/test/test_zpickle.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_continuation/test/test_zpickle.py
@@ -0,0 +1,262 @@
+from pypy.conftest import gettestobjspace
+
+
+class AppTestCopy:
+    def setup_class(cls):
+        cls.space = gettestobjspace(usemodules=('_continuation',),
+                                    CALL_METHOD=True)
+        cls.space.config.translation.continuation = True
+
+    def test_basic_setup(self):
+        from _continuation import continulet
+        lst = [4]
+        co = continulet(lst.append)
+        assert lst == [4]
+        res = co.switch()
+        assert res is None
+        assert lst == [4, co]
+
+    def test_copy_continulet_not_started(self):
+        from _continuation import continulet, error
+        import copy
+        lst = []
+        co = continulet(lst.append)
+        co2, lst2 = copy.deepcopy((co, lst))
+        #
+        assert lst == []
+        co.switch()
+        assert lst == [co]
+        #
+        assert lst2 == []
+        co2.switch()
+        assert lst2 == [co2]
+
+    def test_copy_continulet_not_started_multiple(self):
+        from _continuation import continulet, error
+        import copy
+        lst = []
+        co = continulet(lst.append)
+        co2, lst2 = copy.deepcopy((co, lst))
+        co3, lst3 = copy.deepcopy((co, lst))
+        co4, lst4 = copy.deepcopy((co, lst))
+        #
+        assert lst == []
+        co.switch()
+        assert lst == [co]
+        #
+        assert lst2 == []
+        co2.switch()
+        assert lst2 == [co2]
+        #
+        assert lst3 == []
+        co3.switch()
+        assert lst3 == [co3]
+        #
+        assert lst4 == []
+        co4.switch()
+        assert lst4 == [co4]
+
+    def test_copy_continulet_real(self):
+        import new, sys
+        mod = new.module('test_copy_continulet_real')
+        sys.modules['test_copy_continulet_real'] = mod
+        exec '''if 1:
+            from _continuation import continulet
+            import copy
+            def f(co, x):
+                co.switch(x + 1)
+                co.switch(x + 2)
+                return x + 3
+            co = continulet(f, 40)
+            res = co.switch()
+            assert res == 41
+            co2 = copy.deepcopy(co)
+            #
+            res = co2.switch()
+            assert res == 42
+            assert co2.is_pending()
+            res = co2.switch()
+            assert res == 43
+            assert not co2.is_pending()
+            #
+            res = co.switch()
+            assert res == 42
+            assert co.is_pending()
+            res = co.switch()
+            assert res == 43
+            assert not co.is_pending()
+        ''' in mod.__dict__
+
+    def test_copy_continulet_already_finished(self):
+        from _continuation import continulet, error
+        import copy
+        lst = []
+        co = continulet(lst.append)
+        co.switch()
+        co2 = copy.deepcopy(co)
+        assert not co.is_pending()
+        assert not co2.is_pending()
+        raises(error, co.__init__, lst.append)
+        raises(error, co2.__init__, lst.append)
+        raises(error, co.switch)
+        raises(error, co2.switch)
+
+
+class AppTestPickle:
+    version = 0
+
+    def setup_class(cls):
+        cls.space = gettestobjspace(usemodules=('_continuation',),
+                                    CALL_METHOD=True)
+        cls.space.appexec([], """():
+            global continulet, A, __name__
+
+            import sys
+            __name__ = 'test_pickle_continulet'
+            thismodule = type(sys)(__name__)
+            sys.modules[__name__] = thismodule
+
+            from _continuation import continulet
+            class A(continulet):
+                pass
+
+            thismodule.__dict__.update(globals())
+        """)
+        cls.w_version = cls.space.wrap(cls.version)
+
+    def test_pickle_continulet_empty(self):
+        from _continuation import continulet
+        lst = [4]
+        co = continulet.__new__(continulet)
+        import pickle
+        pckl = pickle.dumps(co, self.version)
+        print repr(pckl)
+        co2 = pickle.loads(pckl)
+        assert co2 is not co
+        assert not co.is_pending()
+        assert not co2.is_pending()
+        # the empty unpickled coroutine can still be used:
+        result = [5]
+        co2.__init__(result.append)
+        res = co2.switch()
+        assert res is None
+        assert result == [5, co2]
+
+    def test_pickle_continulet_empty_subclass(self):
+        from test_pickle_continulet import continulet, A
+        lst = [4]
+        co = continulet.__new__(A)
+        co.foo = 'bar'
+        co.bar = 'baz'
+        import pickle
+        pckl = pickle.dumps(co, self.version)
+        print repr(pckl)
+        co2 = pickle.loads(pckl)
+        assert co2 is not co
+        assert not co.is_pending()
+        assert not co2.is_pending()
+        assert type(co) is type(co2) is A
+        assert co.foo == co2.foo == 'bar'
+        assert co.bar == co2.bar == 'baz'
+        # the empty unpickled coroutine can still be used:
+        result = [5]
+        co2.__init__(result.append)
+        res = co2.switch()
+        assert res is None
+        assert result == [5, co2]
+
+    def test_pickle_continulet_not_started(self):
+        from _continuation import continulet, error
+        import pickle
+        lst = []
+        co = continulet(lst.append)
+        pckl = pickle.dumps((co, lst))
+        print pckl
+        del co, lst
+        for i in range(2):
+            print 'resume...'
+            co2, lst2 = pickle.loads(pckl)
+            assert lst2 == []
+            co2.switch()
+            assert lst2 == [co2]
+
+    def test_pickle_continulet_real(self):
+        import new, sys
+        mod = new.module('test_pickle_continulet_real')
+        sys.modules['test_pickle_continulet_real'] = mod
+        mod.version = self.version
+        exec '''if 1:
+            from _continuation import continulet
+            import pickle
+            def f(co, x):
+                co.switch(x + 1)
+                co.switch(x + 2)
+                return x + 3
+            co = continulet(f, 40)
+            res = co.switch()
+            assert res == 41
+            pckl = pickle.dumps(co, version)
+            print repr(pckl)
+            co2 = pickle.loads(pckl)
+            #
+            res = co2.switch()
+            assert res == 42
+            assert co2.is_pending()
+            res = co2.switch()
+            assert res == 43
+            assert not co2.is_pending()
+            #
+            res = co.switch()
+            assert res == 42
+            assert co.is_pending()
+            res = co.switch()
+            assert res == 43
+            assert not co.is_pending()
+        ''' in mod.__dict__
+
+    def test_pickle_continulet_real_subclass(self):
+        import new, sys
+        mod = new.module('test_pickle_continulet_real_subclass')
+        sys.modules['test_pickle_continulet_real_subclass'] = mod
+        mod.version = self.version
+        exec '''if 1:
+            from _continuation import continulet
+            import pickle
+            class A(continulet):
+                def __init__(self):
+                    crash
+            def f(co):
+                co.switch(co.x + 1)
+                co.switch(co.x + 2)
+                return co.x + 3
+            co = A.__new__(A)
+            continulet.__init__(co, f)
+            co.x = 40
+            res = co.switch()
+            assert res == 41
+            pckl = pickle.dumps(co, version)
+            print repr(pckl)
+            co2 = pickle.loads(pckl)
+            #
+            assert type(co2) is A
+            res = co2.switch()
+            assert res == 42
+            assert co2.is_pending()
+            res = co2.switch()
+            assert res == 43
+            assert not co2.is_pending()
+            #
+            res = co.switch()
+            assert res == 42
+            assert co.is_pending()
+            res = co.switch()
+            assert res == 43
+            assert not co.is_pending()
+        ''' in mod.__dict__
+
+
+class AppTestPickle_v1(AppTestPickle):
+    version = 1
+
+class AppTestPickle_v2(AppTestPickle):
+    version = 2
diff --git a/pypy/module/_multiprocessing/interp_connection.py b/pypy/module/_multiprocessing/interp_connection.py
--- a/pypy/module/_multiprocessing/interp_connection.py
+++ b/pypy/module/_multiprocessing/interp_connection.py
@@ -225,7 +225,9 @@
             except OSError:
                 pass
 
-    def __init__(self, fd, flags):
+    def __init__(self, space, fd, flags):
+        if fd == self.INVALID_HANDLE_VALUE or fd < 0:
+            raise OperationError(space.w_IOError, space.wrap("invalid handle %d" % fd))
         W_BaseConnection.__init__(self, flags)
         self.fd = fd
 
@@ -234,7 +236,7 @@
         flags = (readable and READABLE) | (writable and WRITABLE)
 
         self = space.allocate_instance(W_FileConnection, w_subtype)
-        W_FileConnection.__init__(self, fd, flags)
+        W_FileConnection.__init__(self, space, fd, flags)
         return space.wrap(self)
 
     def fileno(self, space):
diff --git a/pypy/module/_multiprocessing/interp_semaphore.py b/pypy/module/_multiprocessing/interp_semaphore.py
--- a/pypy/module/_multiprocessing/interp_semaphore.py
+++ b/pypy/module/_multiprocessing/interp_semaphore.py
@@ -468,6 +468,9 @@
 
         self.count -= 1
 
+    def after_fork(self):
+        self.count = 0
+
     @unwrap_spec(kind=int, maxvalue=int)
     def rebuild(space, w_cls, w_handle, kind, maxvalue):
         self = space.allocate_instance(W_SemLock, w_cls)
@@ -512,6 +515,7 @@
     acquire = interp2app(W_SemLock.acquire),
     release = interp2app(W_SemLock.release),
     _rebuild = interp2app(W_SemLock.rebuild.im_func, as_classmethod=True),
+    _after_fork = interp2app(W_SemLock.after_fork),
     __enter__=interp2app(W_SemLock.enter),
     __exit__=interp2app(W_SemLock.exit),
     SEM_VALUE_MAX=SEM_VALUE_MAX,
diff --git a/pypy/module/_multiprocessing/test/test_connection.py b/pypy/module/_multiprocessing/test/test_connection.py
--- a/pypy/module/_multiprocessing/test/test_connection.py
+++ b/pypy/module/_multiprocessing/test/test_connection.py
@@ -145,3 +145,9 @@
             else:
                 c.close()
         space.delslice(w_connections, space.wrap(0), space.wrap(100))
+
+    def test_bad_fd(self):
+        import _multiprocessing
+
+        raises(IOError, _multiprocessing.Connection, -1)
+        raises(IOError, _multiprocessing.Connection, -15)
\ No newline at end of file
diff --git a/pypy/module/_multiprocessing/test/test_semaphore.py b/pypy/module/_multiprocessing/test/test_semaphore.py
--- a/pypy/module/_multiprocessing/test/test_semaphore.py
+++ b/pypy/module/_multiprocessing/test/test_semaphore.py
@@ -39,6 +39,10 @@
         sem.release()
         assert sem._count() == 0
 
+        sem.acquire()
+        sem._after_fork()
+        assert sem._count() == 0
+
     def test_recursive(self):
         from _multiprocessing import SemLock
         kind = self.RECURSIVE
diff --git a/pypy/module/_numpy/__init__.py b/pypy/module/_numpy/__init__.py
--- a/pypy/module/_numpy/__init__.py
+++ b/pypy/module/_numpy/__init__.py
@@ -23,6 +23,7 @@
         ("arccos", "arccos"),
         ("arcsin", "arcsin"),
         ("arctan", "arctan"),
+        ("arcsinh", "arcsinh"),
         ("copysign", "copysign"),
         ("cos", "cos"),
         ("divide", "divide"),
diff --git a/pypy/module/_numpy/interp_dtype.py b/pypy/module/_numpy/interp_dtype.py
--- a/pypy/module/_numpy/interp_dtype.py
+++ b/pypy/module/_numpy/interp_dtype.py
@@ -262,6 +262,10 @@
     def arctan(self, v):
         return math.atan(v)
 
+    @unaryop
+    def arcsinh(self, v):
+        return math.asinh(v)
+
 class IntegerArithmeticDtype(ArithmaticTypeMixin):
     _mixin_ = True
 
diff --git a/pypy/module/_numpy/interp_ufuncs.py b/pypy/module/_numpy/interp_ufuncs.py
--- a/pypy/module/_numpy/interp_ufuncs.py
+++ b/pypy/module/_numpy/interp_ufuncs.py
@@ -268,6 +268,7 @@
             ("arcsin", "arcsin", 1, {"promote_to_float": True}),
             ("arccos", "arccos", 1, {"promote_to_float": True}),
             ("arctan", "arctan", 1, {"promote_to_float": True}),
+            ("arcsinh", "arcsinh", 1, {"promote_to_float": True}),
         ]:
             self.add_ufunc(space, *ufunc_def)
 
diff --git a/pypy/module/_numpy/test/test_ufuncs.py b/pypy/module/_numpy/test/test_ufuncs.py
--- a/pypy/module/_numpy/test/test_ufuncs.py
+++ b/pypy/module/_numpy/test/test_ufuncs.py
@@ -298,6 +298,14 @@
         b = arctan(a)
         assert math.isnan(b[0])
 
+    def test_arcsinh(self):
+        import math
+        from numpy import arcsinh, inf
+
+        for v in [inf, -inf, 1.0, math.e]:
+            assert math.asinh(v) == arcsinh(v)
+        assert math.isnan(arcsinh(float("nan")))
+
     def test_reduce_errors(self):
         from _numpy import sin, add
 
diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py
--- a/pypy/module/_sre/interp_sre.py
+++ b/pypy/module/_sre/interp_sre.py
@@ -99,6 +99,7 @@
 # SRE_Pattern class
 
 class W_SRE_Pattern(Wrappable):
+    _immutable_fields_ = ["code", "flags"]
 
     def cannot_copy_w(self):
         space = self.space
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -57,6 +57,9 @@
         compile_extra=['-DPy_BUILD_CORE'],
         )
 
+class CConfig2:
+    _compilation_info_ = CConfig._compilation_info_
+
 class CConfig_constants:
     _compilation_info_ = CConfig._compilation_info_
 
@@ -300,9 +303,13 @@
         return unwrapper_raise # used in 'normal' RPython code.
     return decorate
 
-def cpython_struct(name, fields, forward=None):
+def cpython_struct(name, fields, forward=None, level=1):
     configname = name.replace(' ', '__')
-    setattr(CConfig, configname, rffi_platform.Struct(name, fields))
+    if level == 1:
+        config = CConfig
+    else:
+        config = CConfig2
+    setattr(config, configname, rffi_platform.Struct(name, fields))
     if forward is None:
         forward = lltype.ForwardReference()
     TYPES[configname] = forward
@@ -445,9 +452,10 @@
 #              'int*': rffi.INTP}
 
 def configure_types():
-    for name, TYPE in rffi_platform.configure(CConfig).iteritems():
-        if name in TYPES:
-            TYPES[name].become(TYPE)
+    for config in (CConfig, CConfig2):
+        for name, TYPE in rffi_platform.configure(config).iteritems():
+            if name in TYPES:
+                TYPES[name].become(TYPE)
 
 def build_type_checkers(type_name, cls=None):
     """
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -321,6 +321,15 @@
 
 } PyTypeObject;
 
+typedef struct {
+    PyTypeObject ht_type;
+    PyNumberMethods as_number;
+    PyMappingMethods as_mapping;
+    PySequenceMethods as_sequence;
+    PyBufferProcs as_buffer;
+    PyObject *ht_name, *ht_slots;
+} PyHeapTypeObject;
+
 /* Flag bits for printing: */
 #define Py_PRINT_RAW	1	/* No string quotes etc. */
 
diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py
--- a/pypy/module/cpyext/pyobject.py
+++ b/pypy/module/cpyext/pyobject.py
@@ -19,13 +19,42 @@
     basestruct = PyObject.TO
 
     def get_dealloc(self, space):
-        raise NotImplementedError
+        from pypy.module.cpyext.typeobject import subtype_dealloc
+        return llhelper(
+            subtype_dealloc.api_func.functype,
+            subtype_dealloc.api_func.get_wrapper(space))
+
     def allocate(self, space, w_type, itemcount=0):
-        raise NotImplementedError
+        # similar to PyType_GenericAlloc?
+        # except that it's not related to any pypy object.
+
+        pytype = rffi.cast(PyTypeObjectPtr, make_ref(space, w_type))
+        # Don't increase refcount for non-heaptypes
+        if pytype:
+            flags = rffi.cast(lltype.Signed, pytype.c_tp_flags)
+            if not flags & Py_TPFLAGS_HEAPTYPE:
+                Py_DecRef(space, w_type)
+
+        if pytype:
+            size = pytype.c_tp_basicsize
+        else:
+            size = rffi.sizeof(self.basestruct)
+        if itemcount:
+            size += itemcount * pytype.c_tp_itemsize
+        buf = lltype.malloc(rffi.VOIDP.TO, size,
+                            flavor='raw', zero=True)
+        pyobj = rffi.cast(PyObject, buf)
+        pyobj.c_ob_refcnt = 1
+        pyobj.c_ob_type = pytype
+        return pyobj
+
     def attach(self, space, pyobj, w_obj):
-        raise NotImplementedError
+        pass
+
     def realize(self, space, ref):
-        raise NotImplementedError
+        # For most types, a reference cannot exist without
+        # a real interpreter object
+        raise InvalidPointerException(str(ref))
 
 typedescr_cache = {}
 
@@ -40,6 +69,7 @@
     """
 
     tp_basestruct = kw.pop('basestruct', PyObject.TO)
+    tp_alloc      = kw.pop('alloc', None)
     tp_attach     = kw.pop('attach', None)
     tp_realize    = kw.pop('realize', None)
     tp_dealloc    = kw.pop('dealloc', None)
@@ -49,58 +79,24 @@
 
     class CpyTypedescr(BaseCpyTypedescr):
         basestruct = tp_basestruct
-        realize = tp_realize
 
-        def get_dealloc(self, space):
-            if tp_dealloc:
+        if tp_alloc:
+            def allocate(self, space, w_type, itemcount=0):
+                return tp_alloc(space, w_type)
+
+        if tp_dealloc:
+            def get_dealloc(self, space):
                 return llhelper(
                     tp_dealloc.api_func.functype,
                     tp_dealloc.api_func.get_wrapper(space))
-            else:
-                from pypy.module.cpyext.typeobject import subtype_dealloc
-                return llhelper(
-                    subtype_dealloc.api_func.functype,
-                    subtype_dealloc.api_func.get_wrapper(space))
-
-        def allocate(self, space, w_type, itemcount=0):
-            # similar to PyType_GenericAlloc?
-            # except that it's not related to any pypy object.
-
-            pytype = rffi.cast(PyTypeObjectPtr, make_ref(space, w_type))
-            # Don't increase refcount for non-heaptypes
-            if pytype:
-                flags = rffi.cast(lltype.Signed, pytype.c_tp_flags)
-                if not flags & Py_TPFLAGS_HEAPTYPE:
-                    Py_DecRef(space, w_type)
-
-            if pytype:
-                size = pytype.c_tp_basicsize
-            else:
-                size = rffi.sizeof(tp_basestruct)
-            if itemcount:
-                size += itemcount * pytype.c_tp_itemsize
-            buf = lltype.malloc(rffi.VOIDP.TO, size,
-                                flavor='raw', zero=True)
-            pyobj = rffi.cast(PyObject, buf)
-            pyobj.c_ob_refcnt = 1
-            pyobj.c_ob_type = pytype
-            return pyobj
 
         if tp_attach:
             def attach(self, space, pyobj, w_obj):
                 tp_attach(space, pyobj, w_obj)
-        else:
-            def attach(self, space, pyobj, w_obj):
-                pass
 
         if tp_realize:
             def realize(self, space, ref):
                 return tp_realize(space, ref)
-        else:
-            def realize(self, space, ref):
-                # For most types, a reference cannot exist without
-                # a real interpreter object
-                raise InvalidPointerException(str(ref))
     if typedef:
         CpyTypedescr.__name__ = "CpyTypedescr_%s" % (typedef.name,)
 
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -268,6 +268,21 @@
         assert type(obj) is foo.Custom
         assert type(foo.Custom) is foo.MetaType
 
+    def test_heaptype(self):
+        module = self.import_extension('foo', [
+           ("name_by_heaptype", "METH_O",
+            '''
+                 PyHeapTypeObject *heaptype = (PyHeapTypeObject *)args;
+                 Py_INCREF(heaptype->ht_name);
+                 return heaptype->ht_name;
+             '''
+             )
+            ])
+        class C(object):
+            pass
+        assert module.name_by_heaptype(C) == "C"
+        
+
 class TestTypes(BaseApiTest):
     def test_type_attributes(self, space, api):
         w_class = space.appexec([], """():
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -11,7 +11,7 @@
     generic_cpy_call, Py_TPFLAGS_READY, Py_TPFLAGS_READYING,
     Py_TPFLAGS_HEAPTYPE, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL,
     Py_TPFLAGS_HAVE_GETCHARBUFFER,
-    build_type_checkers)
+    build_type_checkers, PyObjectFields)
 from pypy.module.cpyext.pyobject import (
     PyObject, make_ref, create_ref, from_ref, get_typedescr, make_typedescr,
     track_reference, RefcountState, borrow_from)
@@ -25,7 +25,7 @@
 from pypy.module.cpyext.structmember import PyMember_GetOne, PyMember_SetOne
 from pypy.module.cpyext.typeobjectdefs import (
     PyTypeObjectPtr, PyTypeObject, PyGetSetDef, PyMemberDef, newfunc,
-    PyNumberMethods, PySequenceMethods, PyBufferProcs)
+    PyNumberMethods, PyMappingMethods, PySequenceMethods, PyBufferProcs)
 from pypy.module.cpyext.slotdefs import (
     slotdefs_for_tp_slots, slotdefs_for_wrappers, get_slot_tp_function)
 from pypy.interpreter.error import OperationError
@@ -39,6 +39,19 @@
 
 PyType_Check, PyType_CheckExact = build_type_checkers("Type", "w_type")
 
+PyHeapTypeObjectStruct = lltype.ForwardReference()
+PyHeapTypeObject = lltype.Ptr(PyHeapTypeObjectStruct)
+PyHeapTypeObjectFields = (
+    ("ht_type", PyTypeObject),
+    ("ht_name", PyObject),
+    ("as_number", PyNumberMethods),
+    ("as_mapping", PyMappingMethods),
+    ("as_sequence", PySequenceMethods),
+    ("as_buffer", PyBufferProcs),
+    )
+cpython_struct("PyHeapTypeObject", PyHeapTypeObjectFields, PyHeapTypeObjectStruct,
+               level=2)
+
 class W_GetSetPropertyEx(GetSetProperty):
     def __init__(self, getset, w_type):
         self.getset = getset
@@ -136,6 +149,8 @@
             assert len(slot_names) == 2
             struct = getattr(pto, slot_names[0])
             if not struct:
+                assert not space.config.translating
+                assert not pto.c_tp_flags & Py_TPFLAGS_HEAPTYPE
                 if slot_names[0] == 'c_tp_as_number':
                     STRUCT_TYPE = PyNumberMethods
                 elif slot_names[0] == 'c_tp_as_sequence':
@@ -301,6 +316,7 @@
 
     make_typedescr(space.w_type.instancetypedef,
                    basestruct=PyTypeObject,
+                   alloc=type_alloc,
                    attach=type_attach,
                    realize=type_realize,
                    dealloc=type_dealloc)
@@ -319,11 +335,13 @@
     track_reference(space, lltype.nullptr(PyObject.TO), space.w_type)
     track_reference(space, lltype.nullptr(PyObject.TO), space.w_object)
     track_reference(space, lltype.nullptr(PyObject.TO), space.w_tuple)
+    track_reference(space, lltype.nullptr(PyObject.TO), space.w_str)
 
     # create the objects
     py_type = create_ref(space, space.w_type)
     py_object = create_ref(space, space.w_object)
     py_tuple = create_ref(space, space.w_tuple)
+    py_str = create_ref(space, space.w_str)
 
     # form cycles
     pto_type = rffi.cast(PyTypeObjectPtr, py_type)
@@ -340,10 +358,15 @@
     pto_object.c_tp_bases.c_ob_type = pto_tuple
     pto_tuple.c_tp_bases.c_ob_type = pto_tuple
 
+    for typ in (py_type, py_object, py_tuple, py_str):
+        heaptype = rffi.cast(PyHeapTypeObject, typ)
+        heaptype.c_ht_name.c_ob_type = pto_type
+
     # Restore the mapping
     track_reference(space, py_type, space.w_type, replace=True)
     track_reference(space, py_object, space.w_object, replace=True)
     track_reference(space, py_tuple, space.w_tuple, replace=True)
+    track_reference(space, py_str, space.w_str, replace=True)
 
 
 @cpython_api([PyObject], lltype.Void, external=False)
@@ -416,17 +439,34 @@
     Py_DecRef(space, obj_pto.c_tp_cache) # let's do it like cpython
     Py_DecRef(space, obj_pto.c_tp_dict)
     if obj_pto.c_tp_flags & Py_TPFLAGS_HEAPTYPE:
-        if obj_pto.c_tp_as_buffer:
-            lltype.free(obj_pto.c_tp_as_buffer, flavor='raw')
-        if obj_pto.c_tp_as_number:
-            lltype.free(obj_pto.c_tp_as_number, flavor='raw')
-        if obj_pto.c_tp_as_sequence:
-            lltype.free(obj_pto.c_tp_as_sequence, flavor='raw')
+        heaptype = rffi.cast(PyHeapTypeObject, obj)
+        Py_DecRef(space, heaptype.c_ht_name)
         Py_DecRef(space, base_pyo)
-        rffi.free_charp(obj_pto.c_tp_name)
         PyObject_dealloc(space, obj)
 
 
+def type_alloc(space, w_metatype):
+    size = rffi.sizeof(PyHeapTypeObject)
+    metatype = rffi.cast(PyTypeObjectPtr, make_ref(space, w_metatype))
+    # Don't increase refcount for non-heaptypes
+    if metatype:
+        flags = rffi.cast(lltype.Signed, metatype.c_tp_flags)
+        if not flags & Py_TPFLAGS_HEAPTYPE:
+            Py_DecRef(space, w_metatype)
+
+    heaptype = lltype.malloc(PyHeapTypeObject.TO,
+                             flavor='raw', zero=True)
+    pto = heaptype.c_ht_type
+    pto.c_ob_refcnt = 1
+    pto.c_ob_type = metatype
+    pto.c_tp_flags |= Py_TPFLAGS_HEAPTYPE
+    pto.c_tp_as_number = heaptype.c_as_number
+    pto.c_tp_as_sequence = heaptype.c_as_sequence
+    pto.c_tp_as_mapping = heaptype.c_as_mapping
+    pto.c_tp_as_buffer = heaptype.c_as_buffer
+    
+    return rffi.cast(PyObject, heaptype)
+
 def type_attach(space, py_obj, w_type):
     """
     Fills a newly allocated PyTypeObject from an existing type.
@@ -445,12 +485,18 @@
     if space.is_w(w_type, space.w_str):
         setup_string_buffer_procs(space, pto)
 
-    pto.c_tp_flags |= Py_TPFLAGS_HEAPTYPE
     pto.c_tp_free = llhelper(PyObject_Del.api_func.functype,
             PyObject_Del.api_func.get_wrapper(space))
     pto.c_tp_alloc = llhelper(PyType_GenericAlloc.api_func.functype,
             PyType_GenericAlloc.api_func.get_wrapper(space))
-    pto.c_tp_name = rffi.str2charp(w_type.getname(space))
+    if pto.c_tp_flags & Py_TPFLAGS_HEAPTYPE:
+        w_typename = space.getattr(w_type, space.wrap('__name__'))
+        heaptype = rffi.cast(PyHeapTypeObject, pto)
+        heaptype.c_ht_name = make_ref(space, w_typename)
+        from pypy.module.cpyext.stringobject import PyString_AsString
+        pto.c_tp_name = PyString_AsString(space, heaptype.c_ht_name)
+    else:
+        pto.c_tp_name = rffi.str2charp(w_type.getname(space))
     pto.c_tp_basicsize = -1 # hopefully this makes malloc bail out
     pto.c_tp_itemsize = 0
     # uninitialized fields:
diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py
--- a/pypy/module/pypyjit/interp_jit.py
+++ b/pypy/module/pypyjit/interp_jit.py
@@ -13,7 +13,6 @@
 from pypy.interpreter.pyframe import PyFrame
 from pypy.interpreter.pyopcode import ExitFrame
 from pypy.interpreter.gateway import unwrap_spec
-from pypy.interpreter.baseobjspace import ObjSpace, W_Root
 from opcode import opmap
 from pypy.rlib.nonconst import NonConstant
 from pypy.jit.metainterp.resoperation import rop
@@ -221,7 +220,6 @@
     def __init__(self, space):
         self.w_compile_hook = space.w_None
 
- at unwrap_spec(ObjSpace, W_Root)
 def set_compile_hook(space, w_hook):
     """ set_compile_hook(hook)
 
diff --git a/pypy/module/pypyjit/test/test_policy.py b/pypy/module/pypyjit/test/test_policy.py
--- a/pypy/module/pypyjit/test/test_policy.py
+++ b/pypy/module/pypyjit/test/test_policy.py
@@ -3,8 +3,8 @@
 pypypolicy = policy.PyPyJitPolicy()
 
 def test_id_any():
-    from pypy.objspace.std.default import id__ANY
-    assert pypypolicy.look_inside_function(id__ANY)
+    from pypy.objspace.std.intobject import add__Int_Int
+    assert pypypolicy.look_inside_function(add__Int_Int)
 
 def test_bigint():
     from pypy.rlib.rbigint import rbigint
diff --git a/pypy/module/pypyjit/test_pypy_c/test_containers.py b/pypy/module/pypyjit/test_pypy_c/test_containers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_containers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_containers.py
@@ -49,3 +49,24 @@
             p33 = call(ConstClass(ll_get_value__dicttablePtr_Signed), p18, i28, descr=...)
             ...
         """)
+
+    def test_list(self):
+        def main(n):
+            i = 0
+            while i < n:
+                z = list(())
+                z.append(1)
+                i += z[-1] / len(z)
+            return i
+
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loop, = log.loops_by_filename(self.filepath)
+        assert loop.match("""
+            i7 = int_lt(i5, i6)
+            guard_true(i7, descr=...)
+            guard_not_invalidated(descr=...)
+            i9 = int_add(i5, 1)
+            --TICK--
+            jump(..., descr=...)
+        """)
\ No newline at end of file
diff --git a/pypy/module/pypyjit/test_pypy_c/test_min_max.py b/pypy/module/pypyjit/test_pypy_c/test_min_max.py
--- a/pypy/module/pypyjit/test_pypy_c/test_min_max.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_min_max.py
@@ -42,7 +42,7 @@
         assert len(guards) < 20
         assert loop.match_by_id('max',"""
             ...
-            p76 = call_may_force(ConstClass(min_max_loop__max), _, _, descr=...)
+            p76 = call_may_force(ConstClass(min_max_trampoline), _, _, descr=...)
             ...
         """)
 
@@ -63,6 +63,6 @@
         assert len(guards) < 20
         assert loop.match_by_id('max',"""
             ...
-            p76 = call_may_force(ConstClass(min_max_loop__max), _, _, descr=...)
+            p76 = call_may_force(ConstClass(min_max_trampoline), _, _, descr=...)
             ...
         """)
diff --git a/pypy/module/test_lib_pypy/test_greenlet.py b/pypy/module/test_lib_pypy/test_greenlet.py
--- a/pypy/module/test_lib_pypy/test_greenlet.py
+++ b/pypy/module/test_lib_pypy/test_greenlet.py
@@ -258,3 +258,25 @@
             assert sys.exc_info() == (None, None, None)
 
         greenlet(f).switch()
+
+    def test_gr_frame(self):
+        from greenlet import greenlet
+        import sys
+        def f2():
+            assert g.gr_frame is None
+            gmain.switch()
+            assert g.gr_frame is None
+        def f1():
+            assert gmain.gr_frame is gmain_frame
+            assert g.gr_frame is None
+            f2()
+            assert g.gr_frame is None
+        gmain = greenlet.getcurrent()
+        assert gmain.gr_frame is None
+        gmain_frame = sys._getframe()
+        g = greenlet(f1)
+        assert g.gr_frame is None
+        g.switch()
+        assert g.gr_frame.f_code.co_name == 'f2'
+        g.switch()
+        assert g.gr_frame is None
diff --git a/pypy/module/test_lib_pypy/test_stackless_pickle.py b/pypy/module/test_lib_pypy/test_stackless_pickle.py
--- a/pypy/module/test_lib_pypy/test_stackless_pickle.py
+++ b/pypy/module/test_lib_pypy/test_stackless_pickle.py
@@ -1,25 +1,27 @@
-import py; py.test.skip("XXX port me")
+import py
+py.test.skip("in-progress, maybe")
 from pypy.conftest import gettestobjspace, option
 
 class AppTest_Stackless:
 
     def setup_class(cls):
-        import py.test
-        py.test.importorskip('greenlet')
-        space = gettestobjspace(usemodules=('_stackless', '_socket'))
+        space = gettestobjspace(usemodules=('_continuation', '_socket'))
         cls.space = space
-        # cannot test the unpickle part on top of py.py
+        if option.runappdirect:
+            cls.w_lev = space.wrap(14)
+        else:
+            cls.w_lev = space.wrap(2)
 
     def test_pickle(self):
         import new, sys
 
         mod = new.module('mod')
         sys.modules['mod'] = mod
+        mod.lev = self.lev
         try:
             exec '''
 import pickle, sys
 import stackless
-lev = 14
 
 ch = stackless.channel()
 seen = []
diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -6,6 +6,7 @@
 from pypy.interpreter.typedef import default_identity_hash
 from pypy.tool.sourcetools import compile2, func_with_new_name
 from pypy.module.__builtin__.interp_classobj import W_InstanceObject
+from pypy.rlib.objectmodel import specialize
 
 def object_getattribute(space):
     "Utility that returns the app-level descriptor object.__getattribute__."
@@ -257,15 +258,15 @@
             msg = "'%s' has no length" % (name,)
             raise OperationError(space.w_TypeError, space.wrap(msg))
         w_res = space.get_and_call_function(w_descr, w_obj)
-        space._check_len_result(w_res)
-        return w_res
+        return space.wrap(space._check_len_result(w_res))
 
     def _check_len_result(space, w_obj):
         # Will complain if result is too big.
-        result = space.int_w(w_obj)
+        result = space.int_w(space.int(w_obj))
         if result < 0:
             raise OperationError(space.w_ValueError,
                                  space.wrap("__len__() should return >= 0"))
+        return result
 
     def iter(space, w_obj):
         w_descr = space.lookup(w_obj, '__iter__')
@@ -507,6 +508,7 @@
     def issubtype(space, w_sub, w_type):
         return space._type_issubtype(w_sub, w_type)
 
+    @specialize.arg_or_var(2)
     def isinstance(space, w_inst, w_type):
         return space.wrap(space._type_isinstance(w_inst, w_type))
 
diff --git a/pypy/objspace/std/boolobject.py b/pypy/objspace/std/boolobject.py
--- a/pypy/objspace/std/boolobject.py
+++ b/pypy/objspace/std/boolobject.py
@@ -1,8 +1,10 @@
+from pypy.rlib.rbigint import rbigint
+from pypy.rlib.rarithmetic import r_uint
+from pypy.interpreter.error import OperationError
 from pypy.objspace.std.model import registerimplementation, W_Object
 from pypy.objspace.std.register_all import register_all
 from pypy.objspace.std.intobject import W_IntObject
 
-
 class W_BoolObject(W_Object):
     from pypy.objspace.std.booltype import bool_typedef as typedef
     _immutable_fields_ = ['boolval']
@@ -20,6 +22,21 @@
     def unwrap(w_self, space):
         return w_self.boolval
 
+    def int_w(w_self, space):
+        return int(w_self.boolval)
+
+    def uint_w(w_self, space):
+        intval = int(w_self.boolval)
+        if intval < 0:
+            raise OperationError(space.w_ValueError,
+                                 space.wrap("cannot convert negative integer to unsigned"))
+        else:
+            return r_uint(intval)
+
+    def bigint_w(w_self, space):
+        return rbigint.fromint(int(w_self.boolval))
+
+
 registerimplementation(W_BoolObject)
 
 W_BoolObject.w_False = W_BoolObject(False)
diff --git a/pypy/objspace/std/default.py b/pypy/objspace/std/default.py
--- a/pypy/objspace/std/default.py
+++ b/pypy/objspace/std/default.py
@@ -1,48 +1,16 @@
 """Default implementation for some operation."""
 
-from pypy.interpreter.error import OperationError
+from pypy.interpreter.error import OperationError, typed_unwrap_error_msg
 from pypy.objspace.std.register_all import register_all
-from pypy.rlib import objectmodel
 
 
-# The following default implementations are used before delegation is tried.
-# 'id' is normally the address of the wrapper.
-
-def id__ANY(space, w_obj):
-    #print 'id:', w_obj
-    return space.wrap(objectmodel.compute_unique_id(w_obj))
-
 # __init__ should succeed if called internally as a multimethod
 
 def init__ANY(space, w_obj, __args__):
     pass
 
-def typed_unwrap_error_msg(space, expected, w_obj):
-    type_name = space.type(w_obj).getname(space)
-    return space.wrap("expected %s, got %s object" % (expected, type_name))
-
-def int_w__ANY(space,w_obj):
-    raise OperationError(space.w_TypeError,
-                         typed_unwrap_error_msg(space, "integer", w_obj))
-
-def str_w__ANY(space,w_obj):
-    raise OperationError(space.w_TypeError,
-                         typed_unwrap_error_msg(space, "string", w_obj))
-
 def float_w__ANY(space,w_obj):
     raise OperationError(space.w_TypeError,
                          typed_unwrap_error_msg(space, "float", w_obj))
 
-def uint_w__ANY(space,w_obj):
-    raise OperationError(space.w_TypeError,
-                         typed_unwrap_error_msg(space, "integer", w_obj))
-
-def unicode_w__ANY(space,w_obj):
-    raise OperationError(space.w_TypeError,
-                         typed_unwrap_error_msg(space, "unicode", w_obj))
-
-def bigint_w__ANY(space,w_obj):
-    raise OperationError(space.w_TypeError,
-                         typed_unwrap_error_msg(space, "integer", w_obj))
-
 register_all(vars())
diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py
--- a/pypy/objspace/std/intobject.py
+++ b/pypy/objspace/std/intobject.py
@@ -1,12 +1,13 @@
 from pypy.interpreter.error import OperationError
 from pypy.objspace.std import newformat
+from pypy.objspace.std.inttype import wrapint
 from pypy.objspace.std.model import registerimplementation, W_Object
-from pypy.objspace.std.register_all import register_all
 from pypy.objspace.std.multimethod import FailedToImplementArgs
 from pypy.objspace.std.noneobject import W_NoneObject
+from pypy.objspace.std.register_all import register_all
+from pypy.rlib import jit
 from pypy.rlib.rarithmetic import ovfcheck, ovfcheck_lshift, LONG_BIT, r_uint
 from pypy.rlib.rbigint import rbigint
-from pypy.objspace.std.inttype import wrapint
 
 """
 In order to have the same behavior running
@@ -20,7 +21,7 @@
     _immutable_fields_ = ['intval']
 
     from pypy.objspace.std.inttype import int_typedef as typedef
-    
+
     def __init__(w_self, intval):
         w_self.intval = intval
 
@@ -30,7 +31,18 @@
 
     def unwrap(w_self, space):
         return int(w_self.intval)
+    int_w = unwrap
 
+    def uint_w(w_self, space):
+        intval = w_self.intval
+        if intval < 0:
+            raise OperationError(space.w_ValueError,
+                                 space.wrap("cannot convert negative integer to unsigned"))
+        else:
+            return r_uint(intval)
+
+    def bigint_w(w_self, space):
+        return rbigint.fromint(w_self.intval)
 
 registerimplementation(W_IntObject)
 
@@ -39,20 +51,6 @@
 # alias and then teach copy_multimethods in smallintobject.py to override
 # it. See int__Int for example.
 
-def int_w__Int(space, w_int1):
-    return int(w_int1.intval)
-
-def uint_w__Int(space, w_int1):
-    intval = w_int1.intval
-    if intval < 0:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("cannot convert negative integer to unsigned"))
-    else:
-        return r_uint(intval)
-
-def bigint_w__Int(space, w_int1):
-    return rbigint.fromint(w_int1.intval)
-
 def repr__Int(space, w_int1):
     a = w_int1.intval
     res = str(a)
@@ -138,7 +136,7 @@
     x = float(w_int1.intval)
     y = float(w_int2.intval)
     if y == 0.0:
-        raise FailedToImplementArgs(space.w_ZeroDivisionError, space.wrap("float division"))    
+        raise FailedToImplementArgs(space.w_ZeroDivisionError, space.wrap("float division"))
     return space.wrap(x / y)
 
 def mod__Int_Int(space, w_int1, w_int2):
@@ -172,7 +170,8 @@
 
 
 # helper for pow()
-def _impl_int_int_pow(space, iv, iw, iz=0):
+ at jit.look_inside_iff(lambda space, iv, iw, iz: jit.isconstant(iw) and jit.isconstant(iz))
+def _impl_int_int_pow(space, iv, iw, iz):
     if iw < 0:
         if iz != 0:
             raise OperationError(space.w_TypeError,
@@ -200,7 +199,7 @@
     except OverflowError:
         raise FailedToImplementArgs(space.w_OverflowError,
                                 space.wrap("integer exponentiation"))
-    return wrapint(space, ix)
+    return ix
 
 def pow__Int_Int_Int(space, w_int1, w_int2, w_int3):
     x = w_int1.intval
@@ -209,12 +208,12 @@
     if z == 0:
         raise OperationError(space.w_ValueError,
                              space.wrap("pow() 3rd argument cannot be 0"))
-    return _impl_int_int_pow(space, x, y, z)
+    return space.wrap(_impl_int_int_pow(space, x, y, z))
 
 def pow__Int_Int_None(space, w_int1, w_int2, w_int3):
     x = w_int1.intval
     y = w_int2.intval
-    return _impl_int_int_pow(space, x, y)
+    return space.wrap(_impl_int_int_pow(space, x, y, 0))
 
 def neg__Int(space, w_int1):
     a = w_int1.intval
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -386,7 +386,11 @@
     if len(items)== 0:
         raise OperationError(space.w_IndexError,
                              space.wrap("pop from empty list"))
-    idx = space.int_w(w_idx)
+    if space.isinstance_w(w_idx, space.w_float):
+        raise OperationError(space.w_TypeError,
+            space.wrap("integer argument expected, got float")
+        )
+    idx = space.int_w(space.int(w_idx))
     try:
         return items.pop(idx)
     except IndexError:
diff --git a/pypy/objspace/std/longobject.py b/pypy/objspace/std/longobject.py
--- a/pypy/objspace/std/longobject.py
+++ b/pypy/objspace/std/longobject.py
@@ -45,6 +45,26 @@
     fromrarith_int._annspecialcase_ = "specialize:argtype(0)"
     fromrarith_int = staticmethod(fromrarith_int)
 
+    def int_w(w_self, space):
+        try:
+            return w_self.num.toint()
+        except OverflowError:
+            raise OperationError(space.w_OverflowError, space.wrap(
+                "long int too large to convert to int"))
+
+    def uint_w(w_self, space):
+        try:
+            return w_self.num.touint()
+        except ValueError:
+            raise OperationError(space.w_ValueError, space.wrap(
+                "cannot convert negative integer to unsigned int"))
+        except OverflowError:
+            raise OperationError(space.w_OverflowError, space.wrap(
+                "long int too large to convert to unsigned int"))
+
+    def bigint_w(w_self, space):
+        return w_self.num
+
     def __repr__(self):
         return '<W_LongObject(%d)>' % self.num.tolong()
 
@@ -104,27 +124,6 @@
         raise OperationError(space.w_OverflowError,
                              space.wrap("long int too large to convert to float"))
 
-def int_w__Long(space, w_value):
-    try:
-        return w_value.num.toint()
-    except OverflowError:
-        raise OperationError(space.w_OverflowError, space.wrap(
-            "long int too large to convert to int"))
-
-
-def uint_w__Long(space, w_value):
-    try:
-        return w_value.num.touint()
-    except ValueError:
-        raise OperationError(space.w_ValueError, space.wrap(
-            "cannot convert negative integer to unsigned int"))
-    except OverflowError:
-        raise OperationError(space.w_OverflowError, space.wrap(
-            "long int too large to convert to unsigned int"))
-
-def bigint_w__Long(space, w_value):
-    return w_value.num
-
 def repr__Long(space, w_long):
     return space.wrap(w_long.num.repr())
 
diff --git a/pypy/objspace/std/model.py b/pypy/objspace/std/model.py
--- a/pypy/objspace/std/model.py
+++ b/pypy/objspace/std/model.py
@@ -442,6 +442,13 @@
         mm.dispatch_tree = merge(self.dispatch_tree, other.dispatch_tree)
         return mm
 
+NOT_MULTIMETHODS = dict.fromkeys(
+    ['delattr', 'delete', 'get', 'id', 'inplace_div', 'inplace_floordiv',
+     'inplace_lshift', 'inplace_mod', 'inplace_pow', 'inplace_rshift',
+     'inplace_truediv', 'is_', 'set', 'setattr', 'type', 'userdel',
+     'isinstance', 'issubtype'])
+# XXX should we just remove those from the method table or we're happy
+#     with just not having multimethods?
 
 class MM:
     """StdObjSpace multimethods"""
@@ -451,22 +458,17 @@
     init    = StdObjSpaceMultiMethod('__init__', 1, general__args__=True)
     getnewargs = StdObjSpaceMultiMethod('__getnewargs__', 1)
     # special visible multimethods
-    int_w   = StdObjSpaceMultiMethod('int_w', 1, [])     # returns an unwrapped int
-    str_w   = StdObjSpaceMultiMethod('str_w', 1, [])     # returns an unwrapped string
     float_w = StdObjSpaceMultiMethod('float_w', 1, [])   # returns an unwrapped float
-    uint_w  = StdObjSpaceMultiMethod('uint_w', 1, [])    # returns an unwrapped unsigned int (r_uint)
-    unicode_w = StdObjSpaceMultiMethod('unicode_w', 1, [])    # returns an unwrapped list of unicode characters
-    bigint_w = StdObjSpaceMultiMethod('bigint_w', 1, []) # returns an unwrapped rbigint
     # NOTE: when adding more sometype_w() methods, you need to write a
     # stub in default.py to raise a space.w_TypeError
     marshal_w = StdObjSpaceMultiMethod('marshal_w', 1, [], extra_args=['marshaller'])
-    log     = StdObjSpaceMultiMethod('log', 1, [], extra_args=['base'])
 
     # add all regular multimethods here
     for _name, _symbol, _arity, _specialnames in ObjSpace.MethodTable:
-        if _name not in locals():
+        if _name not in locals() or _name in NOT_MULTIMETHODS:
             mm = StdObjSpaceMultiMethod(_symbol, _arity, _specialnames)
             locals()[_name] = mm
             del mm
 
     pow.extras['defaults'] = (None,)
+
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -1,19 +1,17 @@
 import __builtin__
 import types
-from pypy.interpreter import pyframe, function, special
+from pypy.interpreter import special
 from pypy.interpreter.baseobjspace import ObjSpace, Wrappable
 from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.interpreter.typedef import get_unique_interplevel_subclass
 from pypy.objspace.std import (builtinshortcut, stdtypedef, frame, model,
                                transparent, callmethod, proxyobject)
 from pypy.objspace.descroperation import DescrOperation, raiseattrerror
-from pypy.rlib.objectmodel import instantiate, r_dict, specialize
+from pypy.rlib.objectmodel import instantiate, r_dict, specialize, is_annotation_constant
 from pypy.rlib.debug import make_sure_not_resized
 from pypy.rlib.rarithmetic import base_int, widen
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.rlib import jit
-from pypy.rlib.rbigint import rbigint
-from pypy.tool.sourcetools import func_with_new_name
 
 # Object imports
 from pypy.objspace.std.boolobject import W_BoolObject
@@ -85,6 +83,12 @@
         if self.config.objspace.std.withtproxy:
             transparent.setup(self)
 
+        for type, classes in self.model.typeorder.iteritems():
+            if len(classes) >= 3:
+                # W_Root, AnyXxx and actual object
+                self.gettypefor(type).interplevel_cls = classes[0][0]
+
+
     def get_builtin_types(self):
         return self.builtin_types
 
@@ -569,10 +573,19 @@
             return self.wrap(w_sub.issubtype(w_type))
         raise OperationError(self.w_TypeError, self.wrap("need type objects"))
 
+    @specialize.arg_or_var(2)
     def _type_isinstance(self, w_inst, w_type):
-        if isinstance(w_type, W_TypeObject):
-            return self.type(w_inst).issubtype(w_type)
-        raise OperationError(self.w_TypeError, self.wrap("need type object"))
+        if not isinstance(w_type, W_TypeObject):
+            raise OperationError(self.w_TypeError,
+                                 self.wrap("need type object"))
+        if is_annotation_constant(w_type):
+            cls = w_type.interplevel_cls
+            if cls is not None:
+                assert w_inst is not None
+                if isinstance(w_inst, cls):
+                    return True
+        return self.type(w_inst).issubtype(w_type)
 
+    @specialize.arg_or_var(2)
     def isinstance_w(space, w_inst, w_type):
         return space._type_isinstance(w_inst, w_type)
diff --git a/pypy/objspace/std/rangeobject.py b/pypy/objspace/std/rangeobject.py
--- a/pypy/objspace/std/rangeobject.py
+++ b/pypy/objspace/std/rangeobject.py
@@ -23,7 +23,7 @@
 
 class W_RangeListObject(W_Object):
     typedef = listtype.list_typedef
-    
+
     def __init__(w_self, start, step, length):
         assert step != 0
         w_self.start = start
@@ -40,7 +40,7 @@
         if not length:
             w_self.w_list = space.newlist([])
             return w_self.w_list
-        
+
         arr = [None] * length  # this is to avoid using append.
 
         i = start
@@ -146,7 +146,11 @@
     if length == 0:
         raise OperationError(space.w_IndexError,
                              space.wrap("pop from empty list"))
-    idx = space.int_w(w_idx)
+    if space.isinstance_w(w_idx, space.w_float):
+        raise OperationError(space.w_TypeError,
+            space.wrap("integer argument expected, got float")
+        )
+    idx = space.int_w(space.int(w_idx))
     if idx == 0:
         result = w_rangelist.start
         w_rangelist.start += w_rangelist.step
diff --git a/pypy/objspace/std/ropeobject.py b/pypy/objspace/std/ropeobject.py
--- a/pypy/objspace/std/ropeobject.py
+++ b/pypy/objspace/std/ropeobject.py
@@ -34,12 +34,18 @@
 
     def unwrap(w_self, space):
         return w_self._node.flatten_string()
+    str_w = unwrap
 
     def create_if_subclassed(w_self):
         if type(w_self) is W_RopeObject:
             return w_self
         return W_RopeObject(w_self._node)
 
+    def unicode_w(w_self, space):
+        # XXX should this use the default encoding?
+        from pypy.objspace.std.unicodetype import plain_str2unicode
+        return plain_str2unicode(space, w_self._node.flatten_string())
+
 W_RopeObject.EMPTY = W_RopeObject(rope.LiteralStringNode.EMPTY)
 W_RopeObject.PREBUILT = [W_RopeObject(rope.LiteralStringNode.PREBUILT[i])
                              for i in range(256)]
@@ -663,9 +669,6 @@
         return W_RopeObject(rope.concatenate(
             rope.multiply(zero, middle), node))
 
-def str_w__Rope(space, w_str):
-    return w_str._node.flatten_string()
-
 def hash__Rope(space, w_str):
     return wrapint(space, rope.hash_rope(w_str._node))
 
diff --git a/pypy/objspace/std/ropeunicodeobject.py b/pypy/objspace/std/ropeunicodeobject.py
--- a/pypy/objspace/std/ropeunicodeobject.py
+++ b/pypy/objspace/std/ropeunicodeobject.py
@@ -91,11 +91,17 @@
         # for testing
         return w_self._node.flatten_unicode()
 
+    def str_w(w_self, space):
+        return space.str_w(space.str(w_self))
+
     def create_if_subclassed(w_self):
         if type(w_self) is W_RopeUnicodeObject:
             return w_self
         return W_RopeUnicodeObject(w_self._node)
 
+    def unicode_w(self, space):
+        return self._node.flatten_unicode()
+
 W_RopeUnicodeObject.EMPTY = W_RopeUnicodeObject(rope.LiteralStringNode.EMPTY)
 
 registerimplementation(W_RopeUnicodeObject)
@@ -157,12 +163,6 @@
     assert isinstance(w_uni, W_RopeUnicodeObject) # help the annotator!
     return w_uni
 
-def str_w__RopeUnicode(space, w_uni):
-    return space.str_w(space.str(w_uni))
-
-def unicode_w__RopeUnicode(space, w_uni):
-    return w_uni._node.flatten_unicode()
-
 def str__RopeUnicode(space, w_uni):
     return space.call_method(w_uni, 'encode')
 
@@ -185,7 +185,7 @@
 
 def eq__RopeUnicode_Rope(space, w_runi, w_rope):
     from pypy.objspace.std.unicodeobject import _unicode_string_comparison
-    return _unicode_string_comparison(space, w_runi, w_rope, 
+    return _unicode_string_comparison(space, w_runi, w_rope,
                     False,  unicode_from_string)
 
 def ne__RopeUnicode_RopeUnicode(space, w_str1, w_str2):
@@ -193,7 +193,7 @@
 
 def ne__RopeUnicode_Rope(space, w_runi, w_rope):
     from pypy.objspace.std.unicodeobject import _unicode_string_comparison
-    return _unicode_string_comparison(space, w_runi, w_rope, 
+    return _unicode_string_comparison(space, w_runi, w_rope,
                     True, unicode_from_string)
 
 def gt__RopeUnicode_RopeUnicode(space, w_str1, w_str2):
@@ -247,7 +247,7 @@
     if (len(l_w) == 1 and
         space.is_w(space.type(l_w[0]), space.w_unicode)):
         return l_w[0]
-    
+
     values_list = []
     for i in range(len(l_w)):
         w_item = l_w[i]
@@ -320,7 +320,7 @@
 
 
 def make_generic(funcname):
-    def func(space, w_self): 
+    def func(space, w_self):
         node = w_self._node
         if node.length() == 0:
             return space.w_False
@@ -578,7 +578,7 @@
         return w_self.create_if_subclassed()
     resultnode = rope.concatenate(rope.multiply(fillchar, padding), self)
     return W_RopeUnicodeObject(resultnode)
-    
+
 def unicode_zfill__RopeUnicode_ANY(space, w_self, w_width):
     self = w_self._node
     length = self.length()
@@ -744,7 +744,7 @@
     except OverflowError:
         raise OperationError(space.w_OverflowError,
                              space.wrap("string too long"))
-    
+
 
 def unicode_encode__RopeUnicode_ANY_ANY(space, w_unistr,
                                         w_encoding=None,
@@ -821,7 +821,7 @@
         try:
             w_newval = space.getitem(w_table, space.wrap(char))
         except OperationError, e:
-            if e.match(space, space.w_KeyError):
+            if e.match(space, space.w_LookupError):
                 result.append(crope)
             else:
                 raise
@@ -848,7 +848,7 @@
     hexdigits = "0123456789abcdef"
     node = w_unicode._node
     size = node.length()
-    
+
     singlequote = doublequote = False
     iter = rope.ItemIterator(node)
     for i in range(size):
@@ -900,7 +900,7 @@
                                   ])
                     j += 2
                     continue
-                
+
         if code >= 0x100:
             result.extend(['\\', "u",
                            hexdigits[(code >> 12) & 0xf],
@@ -932,7 +932,7 @@
             continue
         if code < ord(' ') or code >= 0x7f:
             result.extend(['\\', "x",
-                           hexdigits[(code >> 4) & 0xf], 
+                           hexdigits[(code >> 4) & 0xf],
                            hexdigits[(code >> 0) & 0xf],
                           ])
             j += 1
@@ -964,15 +964,15 @@
 
 def next__RopeUnicodeIter(space, w_ropeiter):
     if w_ropeiter.node is None:
-        raise OperationError(space.w_StopIteration, space.w_None) 
+        raise OperationError(space.w_StopIteration, space.w_None)
     try:
         unichar = w_ropeiter.item_iter.nextunichar()
         w_item = space.wrap(unichar)
     except StopIteration:
         w_ropeiter.node = None
         w_ropeiter.char_iter = None
-        raise OperationError(space.w_StopIteration, space.w_None) 
-    w_ropeiter.index += 1 
+        raise OperationError(space.w_StopIteration, space.w_None)
+    w_ropeiter.index += 1
     return w_item
 
 # XXX __length_hint__()
diff --git a/pypy/objspace/std/smallintobject.py b/pypy/objspace/std/smallintobject.py
--- a/pypy/objspace/std/smallintobject.py
+++ b/pypy/objspace/std/smallintobject.py
@@ -7,16 +7,30 @@
 from pypy.objspace.std.register_all import register_all
 from pypy.objspace.std.noneobject import W_NoneObject
 from pypy.objspace.std.intobject import W_IntObject
+from pypy.interpreter.error import OperationError
 from pypy.rlib.objectmodel import UnboxedValue
+from pypy.rlib.rbigint import rbigint
+from pypy.rlib.rarithmetic import r_uint
 from pypy.tool.sourcetools import func_with_new_name
 
-
 class W_SmallIntObject(W_Object, UnboxedValue):
     __slots__ = 'intval'
     from pypy.objspace.std.inttype import int_typedef as typedef
 
     def unwrap(w_self, space):
         return int(w_self.intval)
+    int_w = unwrap
+
+    def uint_w(w_self, space):
+        intval = w_self.intval
+        if intval < 0:
+            raise OperationError(space.w_ValueError,
+                                 space.wrap("cannot convert negative integer to unsigned"))
+        else:
+            return r_uint(intval)
+
+    def bigint_w(w_self, space):
+        return rbigint.fromint(w_self.intval)
 
 
 registerimplementation(W_SmallIntObject)
diff --git a/pypy/objspace/std/smalllongobject.py b/pypy/objspace/std/smalllongobject.py
--- a/pypy/objspace/std/smalllongobject.py
+++ b/pypy/objspace/std/smalllongobject.py
@@ -39,6 +39,30 @@
     def __repr__(w_self):
         return '<W_SmallLongObject(%d)>' % w_self.longlong
 
+    def int_w(w_self, space):
+        a = w_self.longlong
+        b = intmask(a)
+        if b == a:
+            return b
+        else:
+            raise OperationError(space.w_OverflowError, space.wrap(
+                "long int too large to convert to int"))
+
+    def uint_w(w_self, space):
+        a = w_self.longlong
+        if a < 0:
+            raise OperationError(space.w_ValueError, space.wrap(
+                "cannot convert negative integer to unsigned int"))
+        b = r_uint(a)
+        if r_longlong(b) == a:
+            return b
+        else:
+            raise OperationError(space.w_OverflowError, space.wrap(
+                "long int too large to convert to unsigned int"))
+
+    def bigint_w(w_self, space):
+        return w_self.asbigint()
+
 registerimplementation(W_SmallLongObject)
 
 # ____________________________________________________________
@@ -102,30 +126,6 @@
 def float__SmallLong(space, w_value):
     return space.newfloat(float(w_value.longlong))
 
-def int_w__SmallLong(space, w_value):
-    a = w_value.longlong
-    b = intmask(a)
-    if b == a:
-        return b
-    else:
-        raise OperationError(space.w_OverflowError, space.wrap(
-            "long int too large to convert to int"))
-
-def uint_w__SmallLong(space, w_value):
-    a = w_value.longlong
-    if a < 0:
-        raise OperationError(space.w_ValueError, space.wrap(
-            "cannot convert negative integer to unsigned int"))
-    b = r_uint(a)
-    if r_longlong(b) == a:
-        return b
-    else:
-        raise OperationError(space.w_OverflowError, space.wrap(
-            "long int too large to convert to unsigned int"))
-
-def bigint_w__SmallLong(space, w_value):
-    return w_value.asbigint()
-
 def lt__SmallLong_SmallLong(space, w_small1, w_small2):
     return space.newbool(w_small1.longlong <  w_small2.longlong)
 def le__SmallLong_SmallLong(space, w_small1, w_small2):
diff --git a/pypy/objspace/std/strbufobject.py b/pypy/objspace/std/strbufobject.py
--- a/pypy/objspace/std/strbufobject.py
+++ b/pypy/objspace/std/strbufobject.py
@@ -32,6 +32,9 @@
     def unwrap(self, space):
         return self.force()
 
+    def str_w(self, space):
+        return self.force()
+
 registerimplementation(W_StringBufferObject)
 
 # ____________________________________________________________
@@ -55,9 +58,6 @@
 def len__StringBuffer(space, w_self):
     return space.wrap(w_self.length)
 
-def str_w__StringBuffer(space, w_strbuf):
-    return w_strbuf.force()
-
 def add__StringBuffer_String(space, w_self, w_other):
     if w_self.builder.getlength() != w_self.length:
         builder = StringBuilder()
diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py
--- a/pypy/objspace/std/stringobject.py
+++ b/pypy/objspace/std/stringobject.py
@@ -33,17 +33,20 @@
     def unwrap(w_self, space):
         return w_self._value
 
+    def str_w(w_self, space):
+        return w_self._value
+
+    def unicode_w(w_self, space):
+        # XXX should this use the default encoding?
+        from pypy.objspace.std.unicodetype import plain_str2unicode
+        return plain_str2unicode(space, w_self._value)
+
 registerimplementation(W_StringObject)
 
 W_StringObject.EMPTY = W_StringObject('')
 W_StringObject.PREBUILT = [W_StringObject(chr(i)) for i in range(256)]
 del i
 
-def unicode_w__String(space, w_self):
-    # XXX should this use the default encoding?
-    from pypy.objspace.std.unicodetype import plain_str2unicode
-    return plain_str2unicode(space, w_self._value)
-
 def _is_generic(space, w_self, fun):
     v = w_self._value
     if len(v) == 0:
@@ -773,8 +776,6 @@
 
     return space.wrap("".join(buf))
 
-def str_w__String(space, w_str):
-    return w_str._value
 
 def hash__String(space, w_str):
     s = w_str._value
diff --git a/pypy/objspace/std/strjoinobject.py b/pypy/objspace/std/strjoinobject.py
--- a/pypy/objspace/std/strjoinobject.py
+++ b/pypy/objspace/std/strjoinobject.py
@@ -29,6 +29,7 @@
 
     def unwrap(w_self, space):
         return w_self.force()
+    str_w = unwrap
 
 registerimplementation(W_StringJoinObject)
 
@@ -45,9 +46,6 @@
         result += len(w_self.joined_strs[i])
     return space.wrap(result)
 
-def str_w__StringJoin(space, w_str):
-    return w_str.force()
-
 def add__StringJoin_StringJoin(space, w_self, w_other):
     if len(w_self.joined_strs) > w_self.until:
         w_self.force(True)
diff --git a/pypy/objspace/std/strsliceobject.py b/pypy/objspace/std/strsliceobject.py
--- a/pypy/objspace/std/strsliceobject.py
+++ b/pypy/objspace/std/strsliceobject.py
@@ -31,6 +31,9 @@
         w_self.stop = len(str)
         return str
 
+    def str_w(w_self, space):
+        return w_self.force()
+
     def __repr__(w_self):
         """ representation for debugging purposes """
         return "%s(%r[%d:%d])" % (w_self.__class__.__name__,
@@ -165,11 +168,6 @@
             return space.w_True
     return space.w_False
 
-
-def str_w__StringSlice(space, w_str):
-    return w_str.force()
-
-
 def getitem__StringSlice_ANY(space, w_str, w_index):
     ival = space.getindex_w(w_index, space.w_IndexError, "string index")
     slen = w_str.stop - w_str.start
diff --git a/pypy/objspace/std/test/test_boolobject.py b/pypy/objspace/std/test/test_boolobject.py
--- a/pypy/objspace/std/test/test_boolobject.py
+++ b/pypy/objspace/std/test/test_boolobject.py
@@ -17,6 +17,12 @@
         
     def test_false(self):
         assert not self.space.is_true(self.false)
+
+    def test_uint_w(self):
+        assert self.space.uint_w(self.true) == 1
+
+    def test_rbigint_w(self):
+        assert self.space.bigint_w(self.true)._digits == [1]
         
 class AppTestAppBoolTest:
     def test_bool_callable(self):
diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py
--- a/pypy/objspace/std/test/test_listobject.py
+++ b/pypy/objspace/std/test/test_listobject.py
@@ -705,6 +705,20 @@
         l.pop()
         assert l == range(9)
 
+    def test_pop_custom_int(self):
+        class A(object):
+            def __init__(self, x):
+                self.x = x
+
+            def __int__(self):
+                return self.x
+
+        l = range(10)
+        x = l.pop(A(-1))
+        assert x == 9
+        assert l == range(9)
+        raises(TypeError, range(10).pop, 1.0)
+
     def test_remove(self):
         c = list('hello world')
         c.remove('l')
diff --git a/pypy/objspace/std/test/test_stdobjspace.py b/pypy/objspace/std/test/test_stdobjspace.py
--- a/pypy/objspace/std/test/test_stdobjspace.py
+++ b/pypy/objspace/std/test/test_stdobjspace.py
@@ -46,3 +46,17 @@
         assert space.sliceindices(w_slice, w(3)) == (1,2,1)
         assert space.sliceindices(w_obj, w(3)) == (1,2,3)
 
+    def test_fastpath_isinstance(self):
+        from pypy.objspace.std.stringobject import W_StringObject
+        from pypy.objspace.std.intobject import W_IntObject
+        
+        space = self.space
+        assert space.w_str.interplevel_cls is W_StringObject
+        assert space.w_int.interplevel_cls is W_IntObject
+        class X(W_StringObject):
+            def __init__(self):
+                pass
+            
+            typedef = None
+
+        assert space.isinstance_w(X(), space.w_str)
diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -126,6 +126,25 @@
         raises(TypeError, type, 'test', 42, {})
         raises(TypeError, type, 'test', (object,), 42)
 
+    def test_call_type_subclass(self):
+        class A(type):
+            pass
+
+        assert A("hello") is str
+
+        # Make sure type(x) doesn't call x.__class__.__init__
+        class T(type):
+            counter = 0
+            def __init__(self, *args):
+                T.counter += 1
+        class C:
+            __metaclass__ = T
+        assert T.counter == 1
+        a = C()
+        assert T.counter == 1
+        assert type(a) is C
+        assert T.counter == 1
+
     def test_bases(self):
         assert int.__bases__ == (object,)
         class X:
diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py
--- a/pypy/objspace/std/test/test_unicodeobject.py
+++ b/pypy/objspace/std/test/test_unicodeobject.py
@@ -443,6 +443,8 @@
         assert u'<i><i><i>c' == u'abababc'.translate({ord('a'):None, ord('b'):u'<i>'})
         assert u'c' == u'abababc'.translate({ord('a'):None, ord('b'):u''})
         assert u'xyyx' == u'xzx'.translate({ord('z'):u'yy'})
+        assert u'abcd' == u'ab\0d'.translate(u'c')
+        assert u'abcd' == u'abcd'.translate(u'')
 
         raises(TypeError, u'hello'.translate)
         raises(TypeError, u'abababc'.translate, {ord('a'):''})
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -77,7 +77,7 @@
         for i in range(len(self.lookup_where)):
             self.lookup_where[i] = None_None
 
-# possible values of compares_by_identity_status 
+# possible values of compares_by_identity_status
 UNKNOWN = 0
 COMPARES_BY_IDENTITY = 1
 OVERRIDES_EQ_CMP_OR_HASH = 2
@@ -115,6 +115,9 @@
     # of the __new__ is an instance of the type
     w_bltin_new = None
 
+    interplevel_cls = None # not None for prebuilt instances of
+                           # interpreter-level types
+
     @dont_look_inside
     def __init__(w_self, space, name, bases_w, dict_w,
                  overridetypedef=None):
@@ -355,7 +358,7 @@
                 if w_value is not None:
                     return w_value
         return None
-                
+
     @unroll_safe
     def _lookup(w_self, key):
         space = w_self.space
@@ -819,14 +822,6 @@
 
 def call__Type(space, w_type, __args__):
     promote(w_type)
-    # special case for type(x)
-    if space.is_w(w_type, space.w_type):
-        try:
-            w_obj, = __args__.fixedunpack(1)
-        except ValueError:
-            pass
-        else:
-            return space.type(w_obj)
     # invoke the __new__ of the type
     if not we_are_jitted():
         # note that the annotator will figure out that w_type.w_bltin_new can
@@ -853,7 +848,8 @@
         call_init = space.isinstance_w(w_newobject, w_type)
 
     # maybe invoke the __init__ of the type
-    if call_init:
+    if (call_init and not (space.is_w(w_type, space.w_type) and
+        not __args__.keywords and len(__args__.arguments_w) == 1)):
         w_descr = space.lookup(w_newobject, '__init__')
         w_result = space.get_and_call_args(w_descr, w_newobject, __args__)
         if not space.is_w(w_result, space.w_None):
diff --git a/pypy/objspace/std/typetype.py b/pypy/objspace/std/typetype.py
--- a/pypy/objspace/std/typetype.py
+++ b/pypy/objspace/std/typetype.py
@@ -1,17 +1,28 @@
-from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.interpreter import gateway
 from pypy.interpreter.argument import Arguments
+from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.interpreter.typedef import (GetSetProperty, descr_get_dict,
                                       weakref_descr)
 from pypy.objspace.std.stdtypedef import StdTypeDef
 
-def descr__new__(space, w_typetype, w_name, w_bases, w_dict):
+
+def descr__new__(space, w_typetype, w_name, w_bases=gateway.NoneNotWrapped,
+    w_dict=gateway.NoneNotWrapped):
+
     "This is used to create user-defined classes only."
     from pypy.objspace.std.typeobject import W_TypeObject
     # XXX check types
 
     w_typetype = _precheck_for_new(space, w_typetype)
 
+    # special case for type(x)
+    if (space.is_w(space.type(w_typetype), space.w_type) and w_bases is None and
+        w_dict is None):
+        return space.type(w_name)
+    elif w_bases is None or w_dict is None:
+        raise OperationError(space.w_TypeError, space.wrap("type() takes 1 or 3 arguments"))
+
+
     bases_w = space.fixedview(w_bases)
 
     w_winner = w_typetype
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -40,6 +40,12 @@
             return w_self
         return W_UnicodeObject(w_self._value)
 
+    def str_w(self, space):
+        return space.str_w(space.str(self))
+
+    def unicode_w(self, space):
+        return self._value
+
 W_UnicodeObject.EMPTY = W_UnicodeObject(u'')
 
 registerimplementation(W_UnicodeObject)
@@ -99,12 +105,6 @@
         return space.not_(result)
     return result
 
-def str_w__Unicode(space, w_uni):
-    return space.str_w(str__Unicode(space, w_uni))
-
-def unicode_w__Unicode(space, w_uni):
-    return w_uni._value
-
 def str__Unicode(space, w_uni):
     from pypy.objspace.std.unicodetype import encode_object
     return encode_object(space, w_uni, None, None)
@@ -893,7 +893,7 @@
         try:
             w_newval = space.getitem(w_table, space.wrap(ord(unichar)))
         except OperationError, e:
-            if e.match(space, space.w_KeyError):
+            if e.match(space, space.w_LookupError):
                 result.append(unichar)
             else:
                 raise
diff --git a/pypy/objspace/test/test_descroperation.py b/pypy/objspace/test/test_descroperation.py
--- a/pypy/objspace/test/test_descroperation.py
+++ b/pypy/objspace/test/test_descroperation.py
@@ -667,5 +667,19 @@
                 return -1L
         raises(ValueError, len, Y())
 
+    def test_len_custom__int__(self):
+        class X(object):
+            def __init__(self, x):
+                self.x = x
+            def __len__(self):
+                return self.x
+            def __int__(self):
+                return self.x
+
+        l = len(X(3.0))
+        assert l == 3 and type(l) is int
+        l = len(X(X(2)))
+        assert l == 2 and type(l) is int
+
 class AppTestWithBuiltinShortcut(AppTest_Descroperation):
     OPTIONS = {'objspace.std.builtinshortcut': True}
diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -167,10 +167,7 @@
 
     This is for advanced usage only.
     """
-    # I hate the annotator so much.
-    if NonConstant(False):
-        return True
-    return False
+    return NonConstant(False)
 
 @oopspec("jit.isvirtual(value)")
 @specialize.ll()
diff --git a/pypy/rlib/objectmodel.py b/pypy/rlib/objectmodel.py
--- a/pypy/rlib/objectmodel.py
+++ b/pypy/rlib/objectmodel.py
@@ -46,6 +46,17 @@
 
         return decorated_func
 
+    def arg_or_var(self, *args):
+        """ Same as arg, but additionally allow for a 'variable' annotation,
+        that would simply be a situation where designated arg is not
+        a constant
+        """
+        def decorated_func(func):
+            func._annspecialcase_ = 'specialize:arg_or_var' + self._wrap(args)
+            return func
+
+        return decorated_func
+
     def argtype(self, *args):
         """ Specialize function based on types of arguments on given positions.
 
@@ -165,6 +176,24 @@
 def keepalive_until_here(*values):
     pass
 
+def is_annotation_constant(thing):
+    """ Returns whether the annotator can prove that the argument is constant.
+    For advanced usage only."""
+    return True
+
+class Entry(ExtRegistryEntry):
+    _about_ = is_annotation_constant
+
+    def compute_result_annotation(self, s_arg):
+        from pypy.annotation import model
+        r = model.SomeBool()
+        r.const = s_arg.is_constant()
+        return r
+
+    def specialize_call(self, hop):
+        from pypy.rpython.lltypesystem import lltype
+        return hop.inputconst(lltype.Bool, hop.s_result.const)
+
 # ____________________________________________________________
 
 class FREED_OBJECT(object):
diff --git a/pypy/rlib/rgc.py b/pypy/rlib/rgc.py
--- a/pypy/rlib/rgc.py
+++ b/pypy/rlib/rgc.py
@@ -3,6 +3,7 @@
 
 from pypy.rlib import jit
 from pypy.rlib.objectmodel import we_are_translated, enforceargs, specialize
+from pypy.rlib.nonconst import NonConstant
 from pypy.rpython.extregistry import ExtRegistryEntry
 from pypy.rpython.lltypesystem import lltype, llmemory
 
@@ -143,6 +144,10 @@
     from pypy.rpython.lltypesystem.lloperation import llop
     from pypy.rlib.objectmodel import keepalive_until_here
 
+    # XXX: Hack to ensure that we get a proper effectinfo.write_descrs_arrays
+    if NonConstant(False):
+        dest[dest_start] = source[source_start]
+
     # supports non-overlapping copies only
     if not we_are_translated():
         if source == dest:
diff --git a/pypy/rlib/ropenssl.py b/pypy/rlib/ropenssl.py
--- a/pypy/rlib/ropenssl.py
+++ b/pypy/rlib/ropenssl.py
@@ -62,8 +62,7 @@
         "OPENSSL_VERSION_NUMBER")
     SSLEAY_VERSION = rffi_platform.DefinedConstantString(
         "SSLEAY_VERSION", "SSLeay_version(SSLEAY_VERSION)")
-    OPENSSL_NO_SSL2 = rffi_platform.DefinedConstantInteger(
-        "OPENSSL_NO_SSL2")
+    OPENSSL_NO_SSL2 = rffi_platform.Defined("OPENSSL_NO_SSL2")
     SSL_FILETYPE_PEM = rffi_platform.ConstantInteger("SSL_FILETYPE_PEM")
     SSL_OP_ALL = rffi_platform.ConstantInteger("SSL_OP_ALL")
     SSL_VERIFY_NONE = rffi_platform.ConstantInteger("SSL_VERIFY_NONE")
diff --git a/pypy/rlib/rstacklet.py b/pypy/rlib/rstacklet.py
--- a/pypy/rlib/rstacklet.py
+++ b/pypy/rlib/rstacklet.py
@@ -99,12 +99,20 @@
         return False
     def add(self, h):
         if not self.sthread.is_empty_handle(h):
+            if h == self.sthread.get_null_handle():
+                raise StackletDebugError("unexpected null handle")
             self.active.append(h)
     def remove(self, h):
         try:
             i = self.active.index(h)
         except ValueError:
-            raise StackletDebugError
+            if self.sthread.is_empty_handle(h):
+                msg = "empty stacklet handle"
+            elif h == self.sthread.get_null_handle():
+                msg = "unexpected null handle"
+            else:
+                msg = "double usage of handle %r" % (h,)
+            raise StackletDebugError(msg)
         del self.active[i]
 debug = Debug()
 
diff --git a/pypy/rlib/test/test_jit.py b/pypy/rlib/test/test_jit.py
--- a/pypy/rlib/test/test_jit.py
+++ b/pypy/rlib/test/test_jit.py
@@ -1,7 +1,7 @@
 import py
 from pypy.conftest import option
 from pypy.rlib.jit import hint, we_are_jitted, JitDriver, elidable_promote
-from pypy.rlib.jit import JitHintError, oopspec
+from pypy.rlib.jit import JitHintError, oopspec, isconstant
 from pypy.translator.translator import TranslationContext, graphof
 from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin
 from pypy.rpython.lltypesystem import lltype
@@ -137,6 +137,16 @@
             t.view()
         # assert did not raise
 
+    def test_isconstant(self):
+        def f(n):
+            assert n >= 0
+            assert isconstant(n) is False
+            l = []
+            l.append(n)
+            return len(l)
+        res = self.interpret(f, [234])
+        assert res == 1
+
 
 class TestJITLLtype(BaseTestJIT, LLRtypeMixin):
     pass
diff --git a/pypy/rlib/test/test_objectmodel.py b/pypy/rlib/test/test_objectmodel.py
--- a/pypy/rlib/test/test_objectmodel.py
+++ b/pypy/rlib/test/test_objectmodel.py
@@ -339,6 +339,19 @@
         res = self.interpret(f, [42])
         assert res == 84
 
+    def test_isconstant(self):
+        from pypy.rlib.objectmodel import is_annotation_constant, specialize
+
+        @specialize.arg_or_var(0)
+        def f(arg):
+            if is_annotation_constant(arg):
+                return 1
+            return 10
+
+        def fn(arg):
+            return f(arg) + f(3)
+
+        assert self.interpret(fn, [15]) == 11
 
 class TestLLtype(BaseTestObjectModel, LLRtypeMixin):
 
@@ -451,5 +464,4 @@
         if llop.opname == 'malloc_varsize':
             break
     assert llop.args[2] is graph.startblock.inputargs[0]
-
     
diff --git a/pypy/rlib/test/test_rstacklet.py b/pypy/rlib/test/test_rstacklet.py
--- a/pypy/rlib/test/test_rstacklet.py
+++ b/pypy/rlib/test/test_rstacklet.py
@@ -264,6 +264,10 @@
     gcrootfinder = 'shadowstack'
 
 
+def test_dont_keep_debug_to_true():
+    assert not rstacklet.DEBUG
+
+
 def target(*args):
     return entry_point, None
 
diff --git a/pypy/rpython/lltypesystem/ll2ctypes.py b/pypy/rpython/lltypesystem/ll2ctypes.py
--- a/pypy/rpython/lltypesystem/ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/ll2ctypes.py
@@ -140,7 +140,8 @@
                 if isinstance(FIELDTYPE, lltype.Ptr):
                     cls = get_ctypes_type(FIELDTYPE, delayed_builders)
                 else:
-                    cls = get_ctypes_type(FIELDTYPE)
+                    cls = get_ctypes_type(FIELDTYPE, delayed_builders,
+                                          cannot_delay=True)
             fields.append((fieldname, cls))
         CStruct._fields_ = fields
 
@@ -169,7 +170,7 @@
         CStruct._normalized_ctype = get_ctypes_type(S)
         builder()    # no need to be lazy here
     else:
-        delayed_builders.append(builder)
+        delayed_builders.append((S, builder))
     return CStruct
 
 def build_ctypes_array(A, delayed_builders, max_n=0):
@@ -252,11 +253,19 @@
     else:
         return get_ctypes_type(FIELDTYPE)
 
-def get_ctypes_type(T, delayed_builders=None):
+def get_ctypes_type(T, delayed_builders=None, cannot_delay=False):
+    # Check delayed builders
+    if cannot_delay and delayed_builders:
+        for T2, builder in delayed_builders:
+            if T2 is T:
+                builder()
+                delayed_builders.remove((T2, builder))
+                return _ctypes_cache[T]
+
     try:
         return _ctypes_cache[T]
     except KeyError:
-        toplevel = delayed_builders is None
+        toplevel = cannot_delay or delayed_builders is None
         if toplevel:
             delayed_builders = []
         cls = build_new_ctypes_type(T, delayed_builders)
@@ -306,9 +315,11 @@
 
 def complete_builders(delayed_builders):
     while delayed_builders:
-        delayed_builders.pop()()
+        T, builder = delayed_builders[0]
+        builder()
+        delayed_builders.pop(0)
 
-def convert_struct(container, cstruct=None):
+def convert_struct(container, cstruct=None, delayed_converters=None):
     STRUCT = container._TYPE
     if cstruct is None:
         # if 'container' is an inlined substructure, convert the whole
@@ -325,23 +336,38 @@
             n = None
         cstruct = cls._malloc(n)
     add_storage(container, _struct_mixin, ctypes.pointer(cstruct))
+
+    if delayed_converters is None:
+        delayed_converters_was_None = True
+        delayed_converters = []
+    else:
+        delayed_converters_was_None = False
     for field_name in STRUCT._names:
         FIELDTYPE = getattr(STRUCT, field_name)
         field_value = getattr(container, field_name)
         if not isinstance(FIELDTYPE, lltype.ContainerType):
             # regular field
             if FIELDTYPE != lltype.Void:
-                setattr(cstruct, field_name, lltype2ctypes(field_value))
+                def convert(field_name=field_name, field_value=field_value):
+                    setattr(cstruct, field_name, lltype2ctypes(field_value))
+                if isinstance(FIELDTYPE, lltype.Ptr):
+                    delayed_converters.append(convert)
+                else:
+                    convert()
         else:
             # inlined substructure/subarray
             if isinstance(FIELDTYPE, lltype.Struct):
                 csubstruct = getattr(cstruct, field_name)
-                convert_struct(field_value, csubstruct)
+                convert_struct(field_value, csubstruct,
+                               delayed_converters=delayed_converters)
             elif field_name == STRUCT._arrayfld:    # inlined var-sized part
                 csubarray = getattr(cstruct, field_name)
                 convert_array(field_value, csubarray)
             else:
                 raise NotImplementedError('inlined field', FIELDTYPE)
+    if delayed_converters_was_None:
+        for converter in delayed_converters:
+            converter()
     remove_regular_struct_content(container)
 
 def remove_regular_struct_content(container):
@@ -358,7 +384,8 @@
         # bigger structure at once
         parent, parentindex = lltype.parentlink(container)
         if parent is not None:
-            convert_struct(parent)
+            if not isinstance(parent, _parentable_mixin):
+                convert_struct(parent)
             return
         # regular case: allocate a new ctypes array of the proper type
         cls = get_ctypes_type(ARRAY)
diff --git a/pypy/rpython/lltypesystem/rlist.py b/pypy/rpython/lltypesystem/rlist.py
--- a/pypy/rpython/lltypesystem/rlist.py
+++ b/pypy/rpython/lltypesystem/rlist.py
@@ -1,15 +1,15 @@
+from pypy.rlib import rgc, jit
+from pypy.rlib.debug import ll_assert
+from pypy.rlib.objectmodel import enforceargs
+from pypy.rpython.lltypesystem import rstr
+from pypy.rpython.lltypesystem.lltype import (GcForwardReference, Ptr, GcArray,
+     GcStruct, Void, Signed, malloc, typeOf, nullptr, typeMethod)
+from pypy.rpython.rlist import (AbstractBaseListRepr, AbstractListRepr,
+    AbstractFixedSizeListRepr, AbstractListIteratorRepr, ll_setitem_nonneg,
+    ADTIList, ADTIFixedList, dum_nocheck)
+from pypy.rpython.rmodel import Repr, inputconst, externalvsinternal
 from pypy.tool.pairtype import pairtype, pair
-from pypy.rpython.rmodel import Repr, inputconst
-from pypy.rpython.rmodel import externalvsinternal
-from pypy.rpython.rlist import AbstractBaseListRepr, AbstractListRepr, \
-        AbstractFixedSizeListRepr, AbstractListIteratorRepr, \
-        ll_setitem_nonneg, ADTIList, ADTIFixedList
-from pypy.rpython.rlist import dum_nocheck
-from pypy.rpython.lltypesystem.lltype import GcForwardReference, Ptr, GcArray,\
-     GcStruct, Void, Signed, malloc, typeOf, nullptr, typeMethod
-from pypy.rpython.lltypesystem import rstr
-from pypy.rlib.debug import ll_assert
-from pypy.rlib import rgc, jit
+
 
 # ____________________________________________________________
 #
@@ -171,6 +171,7 @@
 
 # adapted C code
 
+ at enforceargs(None, int)
 def _ll_list_resize_really(l, newsize):
     """
     Ensure l.items has room for at least newsize elements, and set
@@ -210,7 +211,6 @@
         rgc.ll_arraycopy(items, newitems, 0, 0, p)
     l.length = newsize
     l.items = newitems
-_ll_list_resize_really._annenforceargs_ = (None, int)
 
 # this common case was factored out of _ll_list_resize
 # to see if inlining it gives some speed-up.
diff --git a/pypy/rpython/lltypesystem/test/test_ll2ctypes.py b/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
--- a/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
@@ -82,7 +82,6 @@
         assert not ALLOCATED     # detects memory leaks in the test
 
     def test_get_pointer(self):
-        py.test.skip("FIXME")
         # Equivalent of the C code::
         #     struct S1 { struct S2 *ptr; struct S2 buf; };
         #     struct S1 s1;
diff --git a/pypy/rpython/rlist.py b/pypy/rpython/rlist.py
--- a/pypy/rpython/rlist.py
+++ b/pypy/rpython/rlist.py
@@ -11,7 +11,7 @@
 from pypy.rlib.debug import ll_assert
 from pypy.rlib.rarithmetic import ovfcheck, widen, r_uint, intmask
 from pypy.rpython.annlowlevel import ADTInterface
-from pypy.rlib import rgc
+from pypy.rlib import rgc, jit
 
 ADTIFixedList = ADTInterface(None, {
     'll_newlist':      (['SELF', Signed        ], 'self'),
@@ -912,6 +912,8 @@
     return l
 # no oopspec -- the function is inlined by the JIT
 
+ at jit.look_inside_iff(lambda l, start: jit.isconstant(start) and jit.isvirtual(l))
+ at jit.oopspec('list.delslice_startonly(l, start)')
 def ll_listdelslice_startonly(l, start):
     ll_assert(start >= 0, "del l[start:] with unexpectedly negative start")
     ll_assert(start <= l.ll_length(), "del l[start:] with start > len(l)")
@@ -923,7 +925,6 @@
             l.ll_setitem_fast(j, null)
             j -= 1
     l._ll_resize_le(newlength)
-ll_listdelslice_startonly.oopspec = 'list.delslice_startonly(l, start)'
 
 def ll_listdelslice_startstop(l, start, stop):
     length = l.ll_length()
diff --git a/pypy/tool/pytest/appsupport.py b/pypy/tool/pytest/appsupport.py
--- a/pypy/tool/pytest/appsupport.py
+++ b/pypy/tool/pytest/appsupport.py
@@ -72,7 +72,9 @@
         space = self.space
         retval = []
         for arg in self.code.getargs():
-            w_val = space.getitem(self.w_locals, space.wrap(arg))
+            w_val = space.finditem(self.w_locals, space.wrap(arg))
+            if w_val is None:
+                w_val = space.wrap('<no value found>')
             retval.append((arg, w_val))
         return retval
 
diff --git a/pypy/translator/c/genc.py b/pypy/translator/c/genc.py
--- a/pypy/translator/c/genc.py
+++ b/pypy/translator/c/genc.py
@@ -563,7 +563,10 @@
             else:
                 mk.definition('PYPY_MAIN_FUNCTION', "main")
 
-            if sys.platform == 'win32':
+            if (py.path.local.sysfind('python') or
+                py.path.local.sysfind('python.exe')):
+                python = 'python '
+            elif sys.platform == 'win32':
                 python = sys.executable.replace('\\', '/') + ' '
             else:
                 python = sys.executable + ' '
diff --git a/pypy/translator/c/src/allocator.h b/pypy/translator/c/src/allocator.h
--- a/pypy/translator/c/src/allocator.h
+++ b/pypy/translator/c/src/allocator.h
@@ -6,11 +6,6 @@
 
 
 #ifndef PYPY_NOT_MAIN_FILE
-#ifdef AVR
-   #ifndef NO_OBMALLOC
-     #define NO_OBMALLOC
-   #endif
-#endif
 
 #if defined(TRIVIAL_MALLOC_DEBUG)
   void *PyObject_Malloc(size_t n) { return malloc(n); }
diff --git a/pypy/translator/c/src/g_include.h b/pypy/translator/c/src/g_include.h
--- a/pypy/translator/c/src/g_include.h
+++ b/pypy/translator/c/src/g_include.h
@@ -31,9 +31,7 @@
 #include "src/char.h"
 #include "src/float.h"
 #include "src/address.h"
-#ifndef AVR
 #include "src/unichar.h"
-#endif
 #include "src/llgroup.h"
 
 #include "src/instrument.h"
@@ -48,11 +46,9 @@
 #  include "src/rtyper.h"
 #  include "src/debug_traceback.h"
 #  include "src/debug_alloc.h"
-#ifndef AVR
 #  include "src/ll_os.h"
 #  include "src/ll_strtod.h"
 #endif
-#endif
 
 #ifdef PYPY_STANDALONE
 #  include "src/allocator.h"
diff --git a/pypy/translator/c/src/g_prerequisite.h b/pypy/translator/c/src/g_prerequisite.h
--- a/pypy/translator/c/src/g_prerequisite.h
+++ b/pypy/translator/c/src/g_prerequisite.h
@@ -13,10 +13,8 @@
 #  include <io.h>   /* needed, otherwise _lseeki64 truncates to 32-bits (??) */
 #endif
 
-#ifndef AVR
 #include "thread.h"   /* needs to be included early to define the
                          struct RPyOpaque_ThreadLock */
-#endif
 
 #include <stddef.h>
 
diff --git a/pypy/translator/c/src/main.h b/pypy/translator/c/src/main.h
--- a/pypy/translator/c/src/main.h
+++ b/pypy/translator/c/src/main.h
@@ -75,9 +75,7 @@
  memory_out:
     errmsg = "out of memory";
  error:
-#ifndef AVR
     fprintf(stderr, "Fatal error during initialization: %s\n", errmsg);
-#endif
     abort();
     return 1;
 }

From noreply at buildbot.pypy.org  Mon Oct  3 05:39:55 2011
From: noreply at buildbot.pypy.org (timo_jbo)
Date: Mon,  3 Oct 2011 05:39:55 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-data-buffer: first bit of buffer support,
 supports reading, somewhat.
Message-ID: <20111003033955.95D67820D9@wyvern.cs.uni-duesseldorf.de>

Author: Timo Paulssen <timonator at perpetuum-immobile.de>
Branch: numpy-data-buffer
Changeset: r47774:103fcb594b05
Date: 2011-10-03 05:12 +0200
http://bitbucket.org/pypy/pypy/changeset/103fcb594b05/

Log:	first bit of buffer support, supports reading, somewhat.

diff --git a/pypy/module/_numpy/interp_buffer.py b/pypy/module/_numpy/interp_buffer.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_numpy/interp_buffer.py
@@ -0,0 +1,20 @@
+from pypy.interpreter.buffer import RWBuffer
+from pypy.rpython.lltypesystem import lltype, rffi
+
+CHAR_TP = lltype.Ptr(lltype.Array(lltype.Char, hints={'nolength': True}))
+
+class NumpyBuffer(RWBuffer):
+    def __init__(self, array):
+        RWBuffer.__init__(self)
+        self.array = array
+
+    def getlength(self):
+        return self.array.get_concrete().find_size()
+
+    def getitem(self, index):
+        if index > self.getlength() - 1:
+            raise IndexError("Index out of bounds (0<=index<%d)" % self.getlength())
+        storage = self.array.get_concrete().get_root_storage()
+        char_data = rffi.cast(CHAR_TP, storage)
+        return char_data[index]
+
diff --git a/pypy/module/_numpy/interp_numarray.py b/pypy/module/_numpy/interp_numarray.py
--- a/pypy/module/_numpy/interp_numarray.py
+++ b/pypy/module/_numpy/interp_numarray.py
@@ -7,6 +7,7 @@
 from pypy.rpython.lltypesystem import lltype
 from pypy.tool.sourcetools import func_with_new_name
 
+from pypy.module._numpy.interp_buffer import NumpyBuffer
 
 numpy_driver = jit.JitDriver(greens = ['signature'],
                              reds = ['result_size', 'i', 'self', 'result'])
@@ -19,6 +20,7 @@
 
     def __init__(self):
         self.invalidates = []
+        self._buffer = None
 
     def invalidated(self):
         if self.invalidates:
@@ -292,6 +294,11 @@
             j += 1
             i += step
 
+    def descr_get_data(self, space):
+        if self._buffer is None:
+            self._buffer = NumpyBuffer(self)
+        return space.wrap(self._buffer)
+
 def convert_to_array(space, w_obj):
     if isinstance(w_obj, BaseArray):
         return w_obj
@@ -622,6 +629,8 @@
     dtype = GetSetProperty(BaseArray.descr_get_dtype),
     shape = GetSetProperty(BaseArray.descr_get_shape),
 
+    data = GetSetProperty(BaseArray.descr_get_data),
+
     mean = interp2app(BaseArray.descr_mean),
     sum = interp2app(BaseArray.descr_sum),
     prod = interp2app(BaseArray.descr_prod),
diff --git a/pypy/module/_numpy/test/test_buffer.py b/pypy/module/_numpy/test/test_buffer.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_numpy/test/test_buffer.py
@@ -0,0 +1,41 @@
+from pypy.module._numpy.test.test_base import BaseNumpyAppTest
+
+class AppTestNumArray(BaseNumpyAppTest):
+    def test_access(self):
+        from _numpy import array
+        from _numpy import dtype
+        ar = array(range(5), dtype=dtype("int8"))
+        buf = ar.data
+
+        assert buf[0] == '\0'
+        assert buf[1] == '\1'
+
+        raises(IndexError, "buf[5]")
+
+    def test_mutable(self):
+        from _numpy import array
+        from _numpy import dtype
+        ar = array(range(5), dtype=dtype("int8"))
+        buf = ar.data
+        assert buf[0] == '\0'
+
+        ar[0] = 5
+        assert buf[0] == "\5"
+
+    def test_slice_view(self):
+        skip("buffers on slicing views doesn't work yet")
+        from _numpy import array
+        from _numpy import dtype
+        ar = array(range(5), dtype=dtype("int8"))
+
+        view = ar[1:-1]
+
+        arbuf = ar.data
+        viewbuf = view.data
+
+        ar[1] = 5
+
+        assert ar[1] == view[0] == 5
+
+        assert arbuf[1] == '\5'
+        assert viewbuf[0] == '\5'

From noreply at buildbot.pypy.org  Mon Oct  3 05:39:56 2011
From: noreply at buildbot.pypy.org (timo_jbo)
Date: Mon,  3 Oct 2011 05:39:56 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-data-buffer: implement buffers of
	slice-views.
Message-ID: <20111003033956.C131F820D9@wyvern.cs.uni-duesseldorf.de>

Author: Timo Paulssen <timonator at perpetuum-immobile.de>
Branch: numpy-data-buffer
Changeset: r47775:04889b07a351
Date: 2011-10-03 05:29 +0200
http://bitbucket.org/pypy/pypy/changeset/04889b07a351/

Log:	implement buffers of slice-views.

diff --git a/pypy/module/_numpy/interp_buffer.py b/pypy/module/_numpy/interp_buffer.py
--- a/pypy/module/_numpy/interp_buffer.py
+++ b/pypy/module/_numpy/interp_buffer.py
@@ -16,5 +16,12 @@
             raise IndexError("Index out of bounds (0<=index<%d)" % self.getlength())
         storage = self.array.get_concrete().get_root_storage()
         char_data = rffi.cast(CHAR_TP, storage)
-        return char_data[index]
+        return char_data[self.calc_index(index)]
 
+    def calc_index(self, index):
+        return index
+
+class NumpyViewBuffer(NumpyBuffer):
+    def calc_index(self, index):
+        return self.array.calc_index(index) * self.array.find_dtype().num_bytes
+
diff --git a/pypy/module/_numpy/interp_numarray.py b/pypy/module/_numpy/interp_numarray.py
--- a/pypy/module/_numpy/interp_numarray.py
+++ b/pypy/module/_numpy/interp_numarray.py
@@ -7,7 +7,7 @@
 from pypy.rpython.lltypesystem import lltype
 from pypy.tool.sourcetools import func_with_new_name
 
-from pypy.module._numpy.interp_buffer import NumpyBuffer
+from pypy.module._numpy.interp_buffer import NumpyBuffer, NumpyViewBuffer
 
 numpy_driver = jit.JitDriver(greens = ['signature'],
                              reds = ['result_size', 'i', 'self', 'result'])
@@ -17,6 +17,7 @@
 
 class BaseArray(Wrappable):
     _attrs_ = ["invalidates", "signature"]
+    BufferClass = NumpyBuffer
 
     def __init__(self):
         self.invalidates = []
@@ -296,7 +297,7 @@
 
     def descr_get_data(self, space):
         if self._buffer is None:
-            self._buffer = NumpyBuffer(self)
+            self._buffer = self.BufferClass(self)
         return space.wrap(self._buffer)
 
 def convert_to_array(space, w_obj):
@@ -450,11 +451,15 @@
     Class for representing views of arrays, they will reflect changes of parent
     arrays. Example: slices
     """
+
+    BufferClass = NumpyViewBuffer
+
     def __init__(self, parent, signature):
         BaseArray.__init__(self)
         self.signature = signature
         self.parent = parent
         self.invalidates = parent.invalidates
+        self._buffer = None
 
     def get_concrete(self):
         # in fact, ViewArray never gets "concrete" as it never stores data.
diff --git a/pypy/module/_numpy/test/test_buffer.py b/pypy/module/_numpy/test/test_buffer.py
--- a/pypy/module/_numpy/test/test_buffer.py
+++ b/pypy/module/_numpy/test/test_buffer.py
@@ -23,7 +23,6 @@
         assert buf[0] == "\5"
 
     def test_slice_view(self):
-        skip("buffers on slicing views doesn't work yet")
         from _numpy import array
         from _numpy import dtype
         ar = array(range(5), dtype=dtype("int8"))

From noreply at buildbot.pypy.org  Mon Oct  3 05:39:57 2011
From: noreply at buildbot.pypy.org (timo_jbo)
Date: Mon,  3 Oct 2011 05:39:57 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-data-buffer: implement setting items on
 buffers of arrays and views.
Message-ID: <20111003033957.EBFED820D9@wyvern.cs.uni-duesseldorf.de>

Author: Timo Paulssen <timonator at perpetuum-immobile.de>
Branch: numpy-data-buffer
Changeset: r47776:d577d2a94297
Date: 2011-10-03 05:38 +0200
http://bitbucket.org/pypy/pypy/changeset/d577d2a94297/

Log:	implement setting items on buffers of arrays and views.

diff --git a/pypy/module/_numpy/interp_buffer.py b/pypy/module/_numpy/interp_buffer.py
--- a/pypy/module/_numpy/interp_buffer.py
+++ b/pypy/module/_numpy/interp_buffer.py
@@ -18,6 +18,13 @@
         char_data = rffi.cast(CHAR_TP, storage)
         return char_data[self.calc_index(index)]
 
+    def setitem(self, index, value):
+        if index > self.getlength() - 1:
+            raise IndexError("Index out of bounds (0<=index<%d)" % self.getlength())
+        storage = self.array.get_concrete().get_root_storage()
+        char_ptr = rffi.cast(CHAR_TP, storage)
+        char_ptr[self.calc_index(index)] = value
+
     def calc_index(self, index):
         return index
 
diff --git a/pypy/module/_numpy/test/test_buffer.py b/pypy/module/_numpy/test/test_buffer.py
--- a/pypy/module/_numpy/test/test_buffer.py
+++ b/pypy/module/_numpy/test/test_buffer.py
@@ -38,3 +38,36 @@
 
         assert arbuf[1] == '\5'
         assert viewbuf[0] == '\5'
+
+    def test_buffer_set(self):
+        from _numpy import array
+        from _numpy import dtype
+        ar = array(range(5), dtype=dtype("int8"))
+        buf = ar.data
+
+        buf[0] = '\5'
+        buf[1] = '\0'
+
+        assert ar[0] == 5
+        assert ar[1] == 0
+
+        raises(IndexError, "buf[5] = '\\9'")
+
+    def test_slice_set(self):
+        from _numpy import array
+        from _numpy import dtype
+        ar = array(range(5), dtype=dtype("int8"))
+
+        view = ar[1:-1]
+
+        arbuf = ar.data
+        viewbuf = view.data
+
+        viewbuf[0] = '\5'
+
+        assert view[0] == 5
+
+        arbuf[1] = '\4'
+        assert view[0] == 4
+
+        raises(IndexError, "view[4] = '\\5'")

From noreply at buildbot.pypy.org  Mon Oct  3 05:55:50 2011
From: noreply at buildbot.pypy.org (timo_jbo)
Date: Mon,  3 Oct 2011 05:55:50 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-data-buffer: setting slices on buffers of
 arrays and slice-views
Message-ID: <20111003035550.D943D820D9@wyvern.cs.uni-duesseldorf.de>

Author: Timo Paulssen <timonator at perpetuum-immobile.de>
Branch: numpy-data-buffer
Changeset: r47777:da293bf60612
Date: 2011-10-03 05:55 +0200
http://bitbucket.org/pypy/pypy/changeset/da293bf60612/

Log:	setting slices on buffers of arrays and slice-views

diff --git a/pypy/module/_numpy/interp_buffer.py b/pypy/module/_numpy/interp_buffer.py
--- a/pypy/module/_numpy/interp_buffer.py
+++ b/pypy/module/_numpy/interp_buffer.py
@@ -12,18 +12,27 @@
         return self.array.get_concrete().find_size()
 
     def getitem(self, index):
+        index = self.calc_index(index)
         if index > self.getlength() - 1:
             raise IndexError("Index out of bounds (0<=index<%d)" % self.getlength())
         storage = self.array.get_concrete().get_root_storage()
         char_data = rffi.cast(CHAR_TP, storage)
-        return char_data[self.calc_index(index)]
+        return char_data[index]
 
     def setitem(self, index, value):
+        index = self.calc_index(index)
         if index > self.getlength() - 1:
             raise IndexError("Index out of bounds (0<=index<%d)" % self.getlength())
         storage = self.array.get_concrete().get_root_storage()
         char_ptr = rffi.cast(CHAR_TP, storage)
-        char_ptr[self.calc_index(index)] = value
+        char_ptr[index] = value
+
+    def setslice(self, index, newstring):
+        offset_index = self.calc_index(index)
+        if offset_index + len(newstring) > self.getlength() - 1:
+            raise IndexError("End of slice to set out of bounds (0<=index<%d)" % self.getlength())
+        for idx in range(0, len(newstring)):
+            self.setitem(index + idx, newstring[idx])
 
     def calc_index(self, index):
         return index
diff --git a/pypy/module/_numpy/test/test_buffer.py b/pypy/module/_numpy/test/test_buffer.py
--- a/pypy/module/_numpy/test/test_buffer.py
+++ b/pypy/module/_numpy/test/test_buffer.py
@@ -71,3 +71,28 @@
         assert view[0] == 4
 
         raises(IndexError, "view[4] = '\\5'")
+
+    def test_buffer_setslice(self):
+        from _numpy import array
+        from _numpy import dtype
+        ar = array(range(8), dtype=dtype("int8"))
+        buf = ar.data
+
+        buf[1:4] = '\1\1\1'
+
+        assert ar[1] == 1
+        assert ar[2] == 1
+        assert ar[3] == 1
+
+    def test_view_setslice(self):
+        from _numpy import array
+        from _numpy import dtype
+        ar = array(range(8), dtype=dtype("int8"))
+        view = ar[1:-1]
+
+        viewbuf = view.data
+        viewbuf[1:4] = '\1\1\1'
+
+        assert ar[2] == 1
+        assert ar[3] == 1
+        assert ar[4] == 1

From noreply at buildbot.pypy.org  Mon Oct  3 07:51:46 2011
From: noreply at buildbot.pypy.org (timo_jbo)
Date: Mon,  3 Oct 2011 07:51:46 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-data-buffer: fix obvious off-by-ones,
 assert lens, size of dtype is important for length of buffer.
Message-ID: <20111003055146.7678A820D9@wyvern.cs.uni-duesseldorf.de>

Author: Timo Paulssen <timonator at perpetuum-immobile.de>
Branch: numpy-data-buffer
Changeset: r47778:bceb4ab33edd
Date: 2011-10-03 07:21 +0200
http://bitbucket.org/pypy/pypy/changeset/bceb4ab33edd/

Log:	fix obvious off-by-ones, assert lens, size of dtype is important for
	length of buffer.

diff --git a/pypy/module/_numpy/interp_buffer.py b/pypy/module/_numpy/interp_buffer.py
--- a/pypy/module/_numpy/interp_buffer.py
+++ b/pypy/module/_numpy/interp_buffer.py
@@ -9,7 +9,7 @@
         self.array = array
 
     def getlength(self):
-        return self.array.get_concrete().find_size()
+        return self.array.get_concrete().find_size() * self.array.find_dtype().num_bytes
 
     def getitem(self, index):
         index = self.calc_index(index)
@@ -29,7 +29,7 @@
 
     def setslice(self, index, newstring):
         offset_index = self.calc_index(index)
-        if offset_index + len(newstring) > self.getlength() - 1:
+        if offset_index + len(newstring) > self.getlength():
             raise IndexError("End of slice to set out of bounds (0<=index<%d)" % self.getlength())
         for idx in range(0, len(newstring)):
             self.setitem(index + idx, newstring[idx])
diff --git a/pypy/module/_numpy/test/test_buffer.py b/pypy/module/_numpy/test/test_buffer.py
--- a/pypy/module/_numpy/test/test_buffer.py
+++ b/pypy/module/_numpy/test/test_buffer.py
@@ -12,6 +12,8 @@
 
         raises(IndexError, "buf[5]")
 
+        assert len(buf) == 5
+
     def test_mutable(self):
         from _numpy import array
         from _numpy import dtype
@@ -39,6 +41,8 @@
         assert arbuf[1] == '\5'
         assert viewbuf[0] == '\5'
 
+        assert len(view) == len(ar) - 2 == 3
+
     def test_buffer_set(self):
         from _numpy import array
         from _numpy import dtype

From noreply at buildbot.pypy.org  Mon Oct  3 07:51:47 2011
From: noreply at buildbot.pypy.org (timo_jbo)
Date: Mon,  3 Oct 2011 07:51:47 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-data-buffer: add property "itemsize" to
	dtype objects
Message-ID: <20111003055147.A098E820D9@wyvern.cs.uni-duesseldorf.de>

Author: Timo Paulssen <timonator at perpetuum-immobile.de>
Branch: numpy-data-buffer
Changeset: r47779:7c67ce4f3e43
Date: 2011-10-03 07:22 +0200
http://bitbucket.org/pypy/pypy/changeset/7c67ce4f3e43/

Log:	add property "itemsize" to dtype objects

diff --git a/pypy/module/_numpy/interp_dtype.py b/pypy/module/_numpy/interp_dtype.py
--- a/pypy/module/_numpy/interp_dtype.py
+++ b/pypy/module/_numpy/interp_dtype.py
@@ -400,5 +400,6 @@
     num = interp_attrproperty("num", cls=W_Dtype),
     kind = interp_attrproperty("kind", cls=W_Dtype),
     shape = GetSetProperty(W_Dtype.descr_get_shape),
+    itemsize = interp_attrproperty("num_bytes", cls=W_Dtype),
 )
 W_Dtype.typedef.acceptable_as_base_class = False

From noreply at buildbot.pypy.org  Mon Oct  3 07:51:48 2011
From: noreply at buildbot.pypy.org (timo_jbo)
Date: Mon,  3 Oct 2011 07:51:48 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-data-buffer: move numpy.fromstring to
 applevel, add support for dtypes and count.
Message-ID: <20111003055148.D0833820D9@wyvern.cs.uni-duesseldorf.de>

Author: Timo Paulssen <timonator at perpetuum-immobile.de>
Branch: numpy-data-buffer
Changeset: r47780:f4a7400e9c6e
Date: 2011-10-03 07:22 +0200
http://bitbucket.org/pypy/pypy/changeset/f4a7400e9c6e/

Log:	move numpy.fromstring to applevel, add support for dtypes and count.

diff --git a/lib_pypy/numpy/__init__.py b/lib_pypy/numpy/__init__.py
--- a/lib_pypy/numpy/__init__.py
+++ b/lib_pypy/numpy/__init__.py
@@ -6,7 +6,6 @@
         zeros,
         empty,
         ones,
-        fromstring,
 
         abs,
         absolute,
@@ -87,3 +86,24 @@
 
     return array(result)
 
+def fromstring(s, dtype=float, count=-1, sep=''):
+    from _numpy import dtype as dt
+    if sep:
+        raise NotImplementedError("Cannot use fromstring with a separator yet")
+
+    _dtype = dt(dtype)
+    if count > 0:
+        length = count * _dtype.itemsize
+        if length > len(s):
+            raise ValueError("length of string (%d) not enough for %d %s" % (len(s), count, _dtype))
+        s = s[:length]
+    else:
+        length = len(s)
+        if len(s) % _dtype.itemsize != 0:
+            raise ValueError("length of string (%d) not evenly dividable by size of dtype (%d)" % (len(s), _dtype.itemsize))
+
+    arr = empty(length / _dtype.itemsize, dtype=_dtype)
+    print len(arr.data), len(s), length, _dtype.itemsize
+    arr.data[:length] = s
+
+    return arr
diff --git a/lib_pypy/pypy_test/test_numpy.py b/lib_pypy/pypy_test/test_numpy.py
--- a/lib_pypy/pypy_test/test_numpy.py
+++ b/lib_pypy/pypy_test/test_numpy.py
@@ -74,3 +74,25 @@
         raises(ValueError, "bincount(c, w)")
         raises(ValueError, "bincount([])")
 
+    def test_fromstring(self):
+        from numpy import fromstring
+        import struct
+
+        data = struct.pack('dddd', 0, 1, 2, 3)
+        a = fromstring(data)
+        assert len(a) == 4
+        for i in range(4):
+            assert a[i] == i
+        raises(ValueError, fromstring, "abc")
+
+        data = struct.pack("iiii", 0, 1, 2, 3)
+        assert len(a) == 4
+        a = fromstring(data, dtype="i")
+        for i in range(4):
+            assert a[i] == i
+
+        data = struct.pack("iiii", 0, 1, 2, 3) + "hello world"
+        a = fromstring(data, dtype="i", count=4)
+        assert len(a) == 4
+        for i in range(4):
+            assert a[i] == i
diff --git a/pypy/module/_numpy/__init__.py b/pypy/module/_numpy/__init__.py
--- a/pypy/module/_numpy/__init__.py
+++ b/pypy/module/_numpy/__init__.py
@@ -12,7 +12,6 @@
         'zeros': 'interp_numarray.zeros',
         'empty': 'interp_numarray.zeros',
         'ones': 'interp_numarray.ones',
-        'fromstring': 'interp_support.fromstring',
     }
 
     # ufuncs
diff --git a/pypy/module/_numpy/interp_support.py b/pypy/module/_numpy/interp_support.py
deleted file mode 100644
--- a/pypy/module/_numpy/interp_support.py
+++ /dev/null
@@ -1,34 +0,0 @@
-from pypy.interpreter.error import OperationError
-from pypy.interpreter.gateway import unwrap_spec
-from pypy.module._numpy.interp_dtype import W_Float64Dtype
-from pypy.rlib.rstruct.runpack import runpack
-from pypy.rpython.lltypesystem import lltype, rffi
-
-
-FLOAT_SIZE = rffi.sizeof(lltype.Float)
-
- at unwrap_spec(s=str)
-def fromstring(space, s):
-    from pypy.module._numpy.interp_numarray import SingleDimArray
-    length = len(s)
-
-    if length % FLOAT_SIZE == 0:
-        number = length/FLOAT_SIZE
-    else:
-        raise OperationError(space.w_ValueError, space.wrap(
-            "string length %d not divisable by %d" % (length, FLOAT_SIZE)))
-
-    dtype = space.fromcache(W_Float64Dtype)
-    a = SingleDimArray(number, dtype=dtype)
-
-    start = 0
-    end = FLOAT_SIZE
-    i = 0
-    while i < number:
-        part = s[start:end]
-        a.dtype.setitem(a.storage, i, dtype.box(runpack('d', part)))
-        i += 1
-        start += FLOAT_SIZE
-        end += FLOAT_SIZE
-
-    return space.wrap(a)
diff --git a/pypy/module/_numpy/test/test_numarray.py b/pypy/module/_numpy/test/test_numarray.py
--- a/pypy/module/_numpy/test/test_numarray.py
+++ b/pypy/module/_numpy/test/test_numarray.py
@@ -587,15 +587,3 @@
                 assert c[i] == func(b[i], 3)
 
 
-class AppTestSupport(object):
-    def setup_class(cls):
-        import struct
-        cls.space = gettestobjspace(usemodules=('_numpy',))
-        cls.w_data = cls.space.wrap(struct.pack('dddd', 1, 2, 3, 4))
-
-    def test_fromstring(self):
-        from _numpy import fromstring
-        a = fromstring(self.data)
-        for i in range(4):
-            assert a[i] == i + 1
-        raises(ValueError, fromstring, "abc")

From noreply at buildbot.pypy.org  Mon Oct  3 07:51:50 2011
From: noreply at buildbot.pypy.org (timo_jbo)
Date: Mon,  3 Oct 2011 07:51:50 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-data-buffer: fix off-by-ones,
 fix NumpyViewBuffer.calc_index, test float dtype a bit.
Message-ID: <20111003055150.06C0C820D9@wyvern.cs.uni-duesseldorf.de>

Author: Timo Paulssen <timonator at perpetuum-immobile.de>
Branch: numpy-data-buffer
Changeset: r47781:02416eeae241
Date: 2011-10-03 07:51 +0200
http://bitbucket.org/pypy/pypy/changeset/02416eeae241/

Log:	fix off-by-ones, fix NumpyViewBuffer.calc_index, test float dtype a
	bit.

diff --git a/pypy/module/_numpy/interp_buffer.py b/pypy/module/_numpy/interp_buffer.py
--- a/pypy/module/_numpy/interp_buffer.py
+++ b/pypy/module/_numpy/interp_buffer.py
@@ -13,7 +13,7 @@
 
     def getitem(self, index):
         index = self.calc_index(index)
-        if index > self.getlength() - 1:
+        if index > self.getlength():
             raise IndexError("Index out of bounds (0<=index<%d)" % self.getlength())
         storage = self.array.get_concrete().get_root_storage()
         char_data = rffi.cast(CHAR_TP, storage)
@@ -21,15 +21,14 @@
 
     def setitem(self, index, value):
         index = self.calc_index(index)
-        if index > self.getlength() - 1:
+        if index > self.getlength():
             raise IndexError("Index out of bounds (0<=index<%d)" % self.getlength())
         storage = self.array.get_concrete().get_root_storage()
         char_ptr = rffi.cast(CHAR_TP, storage)
         char_ptr[index] = value
 
     def setslice(self, index, newstring):
-        offset_index = self.calc_index(index)
-        if offset_index + len(newstring) > self.getlength():
+        if index + len(newstring) > self.getlength():
             raise IndexError("End of slice to set out of bounds (0<=index<%d)" % self.getlength())
         for idx in range(0, len(newstring)):
             self.setitem(index + idx, newstring[idx])
@@ -39,5 +38,12 @@
 
 class NumpyViewBuffer(NumpyBuffer):
     def calc_index(self, index):
-        return self.array.calc_index(index) * self.array.find_dtype().num_bytes
+        box_size = self.array.find_dtype().num_bytes
+        # index is a byte-index, calculate the box-index from it
+        box_index = index / box_size
+        # and we need the byte-inside-box index, too.
+        in_box_index = index % box_size
+        # now we use calc_index to get the correct box_index
+        offset_index = self.array.calc_index(box_index)
+        return offset_index * box_size + in_box_index
 
diff --git a/pypy/module/_numpy/test/test_buffer.py b/pypy/module/_numpy/test/test_buffer.py
--- a/pypy/module/_numpy/test/test_buffer.py
+++ b/pypy/module/_numpy/test/test_buffer.py
@@ -4,6 +4,7 @@
     def test_access(self):
         from _numpy import array
         from _numpy import dtype
+        from struct import pack
         ar = array(range(5), dtype=dtype("int8"))
         buf = ar.data
 
@@ -14,9 +15,16 @@
 
         assert len(buf) == 5
 
+        br = array(range(5,10), dtype=float)
+        buf = br.data
+        assert len(buf) == 5 * (64 / 8)
+
+        assert buf[:8] == pack("d", 5)
+
     def test_mutable(self):
         from _numpy import array
         from _numpy import dtype
+        from struct import pack
         ar = array(range(5), dtype=dtype("int8"))
         buf = ar.data
         assert buf[0] == '\0'
@@ -24,9 +32,19 @@
         ar[0] = 5
         assert buf[0] == "\5"
 
+        br = array(range(5,10), dtype=float)
+        buf = br.data
+        assert len(buf) == 5 * (64 / 8)
+
+        assert buf[:8] == pack("d", 5)
+
+        br[0] = 100
+        assert buf[:8] == pack("d", 100)
+
     def test_slice_view(self):
         from _numpy import array
         from _numpy import dtype
+        from struct import pack
         ar = array(range(5), dtype=dtype("int8"))
 
         view = ar[1:-1]
@@ -43,6 +61,13 @@
 
         assert len(view) == len(ar) - 2 == 3
 
+        br = array(range(5,10), dtype=float)
+        buf = br.data
+        viewbuf = br[1:-1].data
+
+        assert len(viewbuf) == len(buf) - 2 * (64 / 8)
+        assert viewbuf[:8] == buf[8:16] == pack("d", 6)
+
     def test_buffer_set(self):
         from _numpy import array
         from _numpy import dtype

From noreply at buildbot.pypy.org  Mon Oct  3 08:05:25 2011
From: noreply at buildbot.pypy.org (timo_jbo)
Date: Mon,  3 Oct 2011 08:05:25 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-data-buffer: more tests for floats,
	setslice, setitem, ...
Message-ID: <20111003060525.3CCA4820D9@wyvern.cs.uni-duesseldorf.de>

Author: Timo Paulssen <timonator at perpetuum-immobile.de>
Branch: numpy-data-buffer
Changeset: r47782:d4bb282ba363
Date: 2011-10-03 07:58 +0200
http://bitbucket.org/pypy/pypy/changeset/d4bb282ba363/

Log:	more tests for floats, setslice, setitem, ...

diff --git a/pypy/module/_numpy/test/test_buffer.py b/pypy/module/_numpy/test/test_buffer.py
--- a/pypy/module/_numpy/test/test_buffer.py
+++ b/pypy/module/_numpy/test/test_buffer.py
@@ -36,9 +36,13 @@
         buf = br.data
         assert len(buf) == 5 * (64 / 8)
 
+        for idx in range(8):
+            assert buf[idx] == pack("d", 5)[idx]
         assert buf[:8] == pack("d", 5)
 
         br[0] = 100
+        for idx in range(8):
+            assert buf[idx] == pack("d", 100)[idx]
         assert buf[:8] == pack("d", 100)
 
     def test_slice_view(self):
@@ -67,10 +71,13 @@
 
         assert len(viewbuf) == len(buf) - 2 * (64 / 8)
         assert viewbuf[:8] == buf[8:16] == pack("d", 6)
+        for idx in range(8):
+            assert viewbuf[idx] == buf[8+idx] == pack("d", 6)[idx]
 
     def test_buffer_set(self):
         from _numpy import array
         from _numpy import dtype
+        from struct import pack
         ar = array(range(5), dtype=dtype("int8"))
         buf = ar.data
 
@@ -82,9 +89,19 @@
 
         raises(IndexError, "buf[5] = '\\9'")
 
+        br = array(range(5), dtype=float)
+        buf = br.data
+
+        for idx in range(8):
+            buf[idx] = pack("d", 99)[idx]
+        assert br[0] == 99
+        buf[:8] = pack("d", 23.42)
+        assert br[0] == 23.42
+
     def test_slice_set(self):
         from _numpy import array
         from _numpy import dtype
+        from struct import pack
         ar = array(range(5), dtype=dtype("int8"))
 
         view = ar[1:-1]
@@ -101,6 +118,21 @@
 
         raises(IndexError, "view[4] = '\\5'")
 
+        br = array(range(5), dtype=float)
+
+        view = br[1:-1]
+        buf = br.data
+        viewbuf = view.data
+
+        viewbuf[:8] = pack("d", 9000)
+        assert buf[8:16] == viewbuf[:8]
+        assert br[1] == view[0] == 9000
+
+        for idx in range(8):
+            viewbuf[idx] = pack("d", -9000)[idx]
+        assert buf[8:16] == viewbuf[:8]
+        assert br[1] == view[0] == -9000
+
     def test_buffer_setslice(self):
         from _numpy import array
         from _numpy import dtype
@@ -113,6 +145,8 @@
         assert ar[2] == 1
         assert ar[3] == 1
 
+        # tests for float dtype already done above
+
     def test_view_setslice(self):
         from _numpy import array
         from _numpy import dtype
@@ -125,3 +159,5 @@
         assert ar[2] == 1
         assert ar[3] == 1
         assert ar[4] == 1
+
+        # tests for float dtype already done above

From noreply at buildbot.pypy.org  Mon Oct  3 08:05:26 2011
From: noreply at buildbot.pypy.org (timo_jbo)
Date: Mon,  3 Oct 2011 08:05:26 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-data-buffer: test str(NumpyBuffer),
 fix incorrect size constraints in views.
Message-ID: <20111003060526.690B7820D9@wyvern.cs.uni-duesseldorf.de>

Author: Timo Paulssen <timonator at perpetuum-immobile.de>
Branch: numpy-data-buffer
Changeset: r47783:b2f6f9ba6200
Date: 2011-10-03 08:05 +0200
http://bitbucket.org/pypy/pypy/changeset/b2f6f9ba6200/

Log:	test str(NumpyBuffer), fix incorrect size constraints in views.

diff --git a/pypy/module/_numpy/interp_buffer.py b/pypy/module/_numpy/interp_buffer.py
--- a/pypy/module/_numpy/interp_buffer.py
+++ b/pypy/module/_numpy/interp_buffer.py
@@ -12,19 +12,19 @@
         return self.array.get_concrete().find_size() * self.array.find_dtype().num_bytes
 
     def getitem(self, index):
-        index = self.calc_index(index)
         if index > self.getlength():
             raise IndexError("Index out of bounds (0<=index<%d)" % self.getlength())
         storage = self.array.get_concrete().get_root_storage()
         char_data = rffi.cast(CHAR_TP, storage)
+        index = self.calc_index(index)
         return char_data[index]
 
     def setitem(self, index, value):
-        index = self.calc_index(index)
         if index > self.getlength():
             raise IndexError("Index out of bounds (0<=index<%d)" % self.getlength())
         storage = self.array.get_concrete().get_root_storage()
         char_ptr = rffi.cast(CHAR_TP, storage)
+        index = self.calc_index(index)
         char_ptr[index] = value
 
     def setslice(self, index, newstring):
diff --git a/pypy/module/_numpy/test/test_buffer.py b/pypy/module/_numpy/test/test_buffer.py
--- a/pypy/module/_numpy/test/test_buffer.py
+++ b/pypy/module/_numpy/test/test_buffer.py
@@ -63,7 +63,7 @@
         assert arbuf[1] == '\5'
         assert viewbuf[0] == '\5'
 
-        assert len(view) == len(ar) - 2 == 3
+        assert len(viewbuf) == len(arbuf) - 2 == 3
 
         br = array(range(5,10), dtype=float)
         buf = br.data
@@ -161,3 +161,39 @@
         assert ar[4] == 1
 
         # tests for float dtype already done above
+
+    def test_convert_to_string(self):
+        from _numpy import array
+        from _numpy import dtype
+        ar = array(range(5,10), dtype=dtype("int8"))
+        buf = ar.data
+        as_str = str(buf)
+
+        for idx in range(len(buf)):
+            assert buf[idx] == as_str[idx]
+
+        br = array(range(5,10), dtype=float)
+        buf = br.data
+        bs_str = str(buf)
+
+        for idx in range(len(buf)):
+            assert buf[idx] == bs_str[idx]
+
+    def test_convert_view_to_string(self):
+        from _numpy import array
+        from _numpy import dtype
+        ar = array(range(5,10), dtype=dtype("int8"))
+        view = ar[1:-1]
+        buf = view.data
+        as_str = str(buf)
+
+        for idx in range(len(buf)):
+            assert buf[idx] == as_str[idx]
+
+        br = array(range(5,10), dtype=float)
+        view = br[1:-1]
+        buf = view.data
+        bs_str = str(buf)
+
+        for idx in range(len(buf)):
+            assert buf[idx] == bs_str[idx]

From noreply at buildbot.pypy.org  Mon Oct  3 08:33:26 2011
From: noreply at buildbot.pypy.org (justinpeel)
Date: Mon,  3 Oct 2011 08:33:26 +0200 (CEST)
Subject: [pypy-commit] pypy unsigned-dtypes: merge in default
Message-ID: <20111003063326.DA536820D9@wyvern.cs.uni-duesseldorf.de>

Author: Justin Peel <notmuchtotell at gmail.com>
Branch: unsigned-dtypes
Changeset: r47784:d6d4d230f8d9
Date: 2011-10-03 00:33 -0600
http://bitbucket.org/pypy/pypy/changeset/d6d4d230f8d9/

Log:	merge in default

diff --git a/pypy/annotation/test/test_annrpython.py b/pypy/annotation/test/test_annrpython.py
--- a/pypy/annotation/test/test_annrpython.py
+++ b/pypy/annotation/test/test_annrpython.py
@@ -3204,6 +3204,8 @@
         s = a.build_types(f, [])
         assert isinstance(s, annmodel.SomeList)
         assert not s.listdef.listitem.resized
+        assert not s.listdef.listitem.immutable
+        assert s.listdef.listitem.mutated
 
     def test_delslice(self):
         def f():
diff --git a/pypy/annotation/unaryop.py b/pypy/annotation/unaryop.py
--- a/pypy/annotation/unaryop.py
+++ b/pypy/annotation/unaryop.py
@@ -352,6 +352,7 @@
         check_negative_slice(s_start, s_stop)
         if not isinstance(s_iterable, SomeList):
             raise Exception("list[start:stop] = x: x must be a list")
+        lst.listdef.mutate()
         lst.listdef.agree(s_iterable.listdef)
         # note that setslice is not allowed to resize a list in RPython
 
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_util.py b/pypy/jit/metainterp/optimizeopt/test/test_util.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_util.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_util.py
@@ -182,7 +182,8 @@
                             EffectInfo.EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE,
                             can_invalidate=True))
     arraycopydescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT,
-             EffectInfo([], [], [], [], oopspecindex=EffectInfo.OS_ARRAYCOPY))
+             EffectInfo([], [arraydescr], [], [arraydescr],
+                        oopspecindex=EffectInfo.OS_ARRAYCOPY))
 
     for _name, _os in [
         ('strconcatdescr',               'OS_STR_CONCAT'),
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -3420,6 +3420,23 @@
         assert res == main(1, 10)
         self.check_loops(call=0)
 
+    def test_setarrayitem_followed_by_arraycopy(self):
+        myjitdriver = JitDriver(greens = [], reds = ['n', 'sa', 'x', 'y'])
+        def f(n):
+            sa = 0
+            x = [1,2,n]
+            y = [1,2,3]
+            while n > 0:
+                myjitdriver.jit_merge_point(sa=sa, n=n, x=x, y=y)
+                y[0] = n
+                x[0:3] = y
+                sa += x[0]
+                n -= 1
+            return sa
+        res = self.meta_interp(f, [16])
+        assert res == f(16)
+        
+
 
 class TestLLtype(BaseLLtypeTests, LLJitMixin):
     def test_tagged(self):
diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -23,6 +23,7 @@
         ("arccos", "arccos"),
         ("arcsin", "arcsin"),
         ("arctan", "arctan"),
+        ("arcsinh", "arcsinh"),
         ("copysign", "copysign"),
         ("cos", "cos"),
         ("divide", "divide"),
@@ -50,4 +51,5 @@
     appleveldefs = {
         'average': 'app_numpy.average',
         'mean': 'app_numpy.mean',
+        'inf': 'app_numpy.inf',
     }
diff --git a/pypy/module/micronumpy/app_numpy.py b/pypy/module/micronumpy/app_numpy.py
--- a/pypy/module/micronumpy/app_numpy.py
+++ b/pypy/module/micronumpy/app_numpy.py
@@ -1,5 +1,8 @@
 import numpy
 
+
+inf = float("inf")
+
 def average(a):
     # This implements a weighted average, for now we don't implement the
     # weighting, just the average part!
@@ -8,4 +11,4 @@
 def mean(a):
     if not hasattr(a, "mean"):
         a = numpy.array(a)
-    return a.mean()
\ No newline at end of file
+    return a.mean()
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -271,6 +271,9 @@
     @unaryop
     def arctan(self, v):
         return math.atan(v)
+    @unaryop
+    def arcsinh(self, v):
+        return math.asinh(v)
 
 class IntegerArithmeticDtype(ArithmeticTypeMixin):
     _mixin_ = True
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -312,6 +312,7 @@
             ("arcsin", "arcsin", 1, {"promote_to_float": True}),
             ("arccos", "arccos", 1, {"promote_to_float": True}),
             ("arctan", "arctan", 1, {"promote_to_float": True}),
+            ("arcsinh", "arcsinh", 1, {"promote_to_float": True}),
         ]:
             self.add_ufunc(space, *ufunc_def)
 
diff --git a/pypy/module/micronumpy/test/test_module.py b/pypy/module/micronumpy/test/test_module.py
--- a/pypy/module/micronumpy/test/test_module.py
+++ b/pypy/module/micronumpy/test/test_module.py
@@ -10,4 +10,9 @@
     def test_average(self):
         from numpy import array, average
         assert average(range(10)) == 4.5
-        assert average(array(range(10))) == 4.5
\ No newline at end of file
+        assert average(array(range(10))) == 4.5
+
+    def test_inf(self):
+        from numpy import inf
+        assert type(inf) is float
+        assert inf == float("inf")
\ No newline at end of file
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -298,6 +298,14 @@
         b = arctan(a)
         assert math.isnan(b[0])
 
+    def test_arcsinh(self):
+        import math
+        from numpy import arcsinh, inf
+
+        for v in [inf, -inf, 1.0, math.e]:
+            assert math.asinh(v) == arcsinh(v)
+        assert math.isnan(arcsinh(float("nan")))
+
     def test_reduce_errors(self):
         from numpy import sin, add
 
diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -258,15 +258,15 @@
             msg = "'%s' has no length" % (name,)
             raise OperationError(space.w_TypeError, space.wrap(msg))
         w_res = space.get_and_call_function(w_descr, w_obj)
-        space._check_len_result(w_res)
-        return w_res
+        return space.wrap(space._check_len_result(w_res))
 
     def _check_len_result(space, w_obj):
         # Will complain if result is too big.
-        result = space.int_w(w_obj)
+        result = space.int_w(space.int(w_obj))
         if result < 0:
             raise OperationError(space.w_ValueError,
                                  space.wrap("__len__() should return >= 0"))
+        return result
 
     def iter(space, w_obj):
         w_descr = space.lookup(w_obj, '__iter__')
diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py
--- a/pypy/objspace/std/intobject.py
+++ b/pypy/objspace/std/intobject.py
@@ -1,12 +1,13 @@
 from pypy.interpreter.error import OperationError
 from pypy.objspace.std import newformat
+from pypy.objspace.std.inttype import wrapint
 from pypy.objspace.std.model import registerimplementation, W_Object
-from pypy.objspace.std.register_all import register_all
 from pypy.objspace.std.multimethod import FailedToImplementArgs
 from pypy.objspace.std.noneobject import W_NoneObject
+from pypy.objspace.std.register_all import register_all
+from pypy.rlib import jit
 from pypy.rlib.rarithmetic import ovfcheck, ovfcheck_lshift, LONG_BIT, r_uint
 from pypy.rlib.rbigint import rbigint
-from pypy.objspace.std.inttype import wrapint
 
 """
 In order to have the same behavior running
@@ -20,7 +21,7 @@
     _immutable_fields_ = ['intval']
 
     from pypy.objspace.std.inttype import int_typedef as typedef
-    
+
     def __init__(w_self, intval):
         w_self.intval = intval
 
@@ -135,7 +136,7 @@
     x = float(w_int1.intval)
     y = float(w_int2.intval)
     if y == 0.0:
-        raise FailedToImplementArgs(space.w_ZeroDivisionError, space.wrap("float division"))    
+        raise FailedToImplementArgs(space.w_ZeroDivisionError, space.wrap("float division"))
     return space.wrap(x / y)
 
 def mod__Int_Int(space, w_int1, w_int2):
@@ -169,7 +170,8 @@
 
 
 # helper for pow()
-def _impl_int_int_pow(space, iv, iw, iz=0):
+ at jit.look_inside_iff(lambda space, iv, iw, iz: jit.isconstant(iw) and jit.isconstant(iz))
+def _impl_int_int_pow(space, iv, iw, iz):
     if iw < 0:
         if iz != 0:
             raise OperationError(space.w_TypeError,
@@ -197,7 +199,7 @@
     except OverflowError:
         raise FailedToImplementArgs(space.w_OverflowError,
                                 space.wrap("integer exponentiation"))
-    return wrapint(space, ix)
+    return ix
 
 def pow__Int_Int_Int(space, w_int1, w_int2, w_int3):
     x = w_int1.intval
@@ -206,12 +208,12 @@
     if z == 0:
         raise OperationError(space.w_ValueError,
                              space.wrap("pow() 3rd argument cannot be 0"))
-    return _impl_int_int_pow(space, x, y, z)
+    return space.wrap(_impl_int_int_pow(space, x, y, z))
 
 def pow__Int_Int_None(space, w_int1, w_int2, w_int3):
     x = w_int1.intval
     y = w_int2.intval
-    return _impl_int_int_pow(space, x, y)
+    return space.wrap(_impl_int_int_pow(space, x, y, 0))
 
 def neg__Int(space, w_int1):
     a = w_int1.intval
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -386,7 +386,11 @@
     if len(items)== 0:
         raise OperationError(space.w_IndexError,
                              space.wrap("pop from empty list"))
-    idx = space.int_w(w_idx)
+    if space.isinstance_w(w_idx, space.w_float):
+        raise OperationError(space.w_TypeError,
+            space.wrap("integer argument expected, got float")
+        )
+    idx = space.int_w(space.int(w_idx))
     try:
         return items.pop(idx)
     except IndexError:
diff --git a/pypy/objspace/std/rangeobject.py b/pypy/objspace/std/rangeobject.py
--- a/pypy/objspace/std/rangeobject.py
+++ b/pypy/objspace/std/rangeobject.py
@@ -23,7 +23,7 @@
 
 class W_RangeListObject(W_Object):
     typedef = listtype.list_typedef
-    
+
     def __init__(w_self, start, step, length):
         assert step != 0
         w_self.start = start
@@ -40,7 +40,7 @@
         if not length:
             w_self.w_list = space.newlist([])
             return w_self.w_list
-        
+
         arr = [None] * length  # this is to avoid using append.
 
         i = start
@@ -146,7 +146,11 @@
     if length == 0:
         raise OperationError(space.w_IndexError,
                              space.wrap("pop from empty list"))
-    idx = space.int_w(w_idx)
+    if space.isinstance_w(w_idx, space.w_float):
+        raise OperationError(space.w_TypeError,
+            space.wrap("integer argument expected, got float")
+        )
+    idx = space.int_w(space.int(w_idx))
     if idx == 0:
         result = w_rangelist.start
         w_rangelist.start += w_rangelist.step
diff --git a/pypy/objspace/std/ropeunicodeobject.py b/pypy/objspace/std/ropeunicodeobject.py
--- a/pypy/objspace/std/ropeunicodeobject.py
+++ b/pypy/objspace/std/ropeunicodeobject.py
@@ -185,7 +185,7 @@
 
 def eq__RopeUnicode_Rope(space, w_runi, w_rope):
     from pypy.objspace.std.unicodeobject import _unicode_string_comparison
-    return _unicode_string_comparison(space, w_runi, w_rope, 
+    return _unicode_string_comparison(space, w_runi, w_rope,
                     False,  unicode_from_string)
 
 def ne__RopeUnicode_RopeUnicode(space, w_str1, w_str2):
@@ -193,7 +193,7 @@
 
 def ne__RopeUnicode_Rope(space, w_runi, w_rope):
     from pypy.objspace.std.unicodeobject import _unicode_string_comparison
-    return _unicode_string_comparison(space, w_runi, w_rope, 
+    return _unicode_string_comparison(space, w_runi, w_rope,
                     True, unicode_from_string)
 
 def gt__RopeUnicode_RopeUnicode(space, w_str1, w_str2):
@@ -247,7 +247,7 @@
     if (len(l_w) == 1 and
         space.is_w(space.type(l_w[0]), space.w_unicode)):
         return l_w[0]
-    
+
     values_list = []
     for i in range(len(l_w)):
         w_item = l_w[i]
@@ -320,7 +320,7 @@
 
 
 def make_generic(funcname):
-    def func(space, w_self): 
+    def func(space, w_self):
         node = w_self._node
         if node.length() == 0:
             return space.w_False
@@ -578,7 +578,7 @@
         return w_self.create_if_subclassed()
     resultnode = rope.concatenate(rope.multiply(fillchar, padding), self)
     return W_RopeUnicodeObject(resultnode)
-    
+
 def unicode_zfill__RopeUnicode_ANY(space, w_self, w_width):
     self = w_self._node
     length = self.length()
@@ -744,7 +744,7 @@
     except OverflowError:
         raise OperationError(space.w_OverflowError,
                              space.wrap("string too long"))
-    
+
 
 def unicode_encode__RopeUnicode_ANY_ANY(space, w_unistr,
                                         w_encoding=None,
@@ -821,7 +821,7 @@
         try:
             w_newval = space.getitem(w_table, space.wrap(char))
         except OperationError, e:
-            if e.match(space, space.w_KeyError):
+            if e.match(space, space.w_LookupError):
                 result.append(crope)
             else:
                 raise
@@ -848,7 +848,7 @@
     hexdigits = "0123456789abcdef"
     node = w_unicode._node
     size = node.length()
-    
+
     singlequote = doublequote = False
     iter = rope.ItemIterator(node)
     for i in range(size):
@@ -900,7 +900,7 @@
                                   ])
                     j += 2
                     continue
-                
+
         if code >= 0x100:
             result.extend(['\\', "u",
                            hexdigits[(code >> 12) & 0xf],
@@ -932,7 +932,7 @@
             continue
         if code < ord(' ') or code >= 0x7f:
             result.extend(['\\', "x",
-                           hexdigits[(code >> 4) & 0xf], 
+                           hexdigits[(code >> 4) & 0xf],
                            hexdigits[(code >> 0) & 0xf],
                           ])
             j += 1
@@ -964,15 +964,15 @@
 
 def next__RopeUnicodeIter(space, w_ropeiter):
     if w_ropeiter.node is None:
-        raise OperationError(space.w_StopIteration, space.w_None) 
+        raise OperationError(space.w_StopIteration, space.w_None)
     try:
         unichar = w_ropeiter.item_iter.nextunichar()
         w_item = space.wrap(unichar)
     except StopIteration:
         w_ropeiter.node = None
         w_ropeiter.char_iter = None
-        raise OperationError(space.w_StopIteration, space.w_None) 
-    w_ropeiter.index += 1 
+        raise OperationError(space.w_StopIteration, space.w_None)
+    w_ropeiter.index += 1
     return w_item
 
 # XXX __length_hint__()
diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py
--- a/pypy/objspace/std/test/test_listobject.py
+++ b/pypy/objspace/std/test/test_listobject.py
@@ -705,6 +705,20 @@
         l.pop()
         assert l == range(9)
 
+    def test_pop_custom_int(self):
+        class A(object):
+            def __init__(self, x):
+                self.x = x
+
+            def __int__(self):
+                return self.x
+
+        l = range(10)
+        x = l.pop(A(-1))
+        assert x == 9
+        assert l == range(9)
+        raises(TypeError, range(10).pop, 1.0)
+
     def test_remove(self):
         c = list('hello world')
         c.remove('l')
diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -126,6 +126,25 @@
         raises(TypeError, type, 'test', 42, {})
         raises(TypeError, type, 'test', (object,), 42)
 
+    def test_call_type_subclass(self):
+        class A(type):
+            pass
+
+        assert A("hello") is str
+
+        # Make sure type(x) doesn't call x.__class__.__init__
+        class T(type):
+            counter = 0
+            def __init__(self, *args):
+                T.counter += 1
+        class C:
+            __metaclass__ = T
+        assert T.counter == 1
+        a = C()
+        assert T.counter == 1
+        assert type(a) is C
+        assert T.counter == 1
+
     def test_bases(self):
         assert int.__bases__ == (object,)
         class X:
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -77,7 +77,7 @@
         for i in range(len(self.lookup_where)):
             self.lookup_where[i] = None_None
 
-# possible values of compares_by_identity_status 
+# possible values of compares_by_identity_status
 UNKNOWN = 0
 COMPARES_BY_IDENTITY = 1
 OVERRIDES_EQ_CMP_OR_HASH = 2
@@ -358,7 +358,7 @@
                 if w_value is not None:
                     return w_value
         return None
-                
+
     @unroll_safe
     def _lookup(w_self, key):
         space = w_self.space
@@ -822,14 +822,6 @@
 
 def call__Type(space, w_type, __args__):
     promote(w_type)
-    # special case for type(x)
-    if space.is_w(w_type, space.w_type):
-        try:
-            w_obj, = __args__.fixedunpack(1)
-        except ValueError:
-            pass
-        else:
-            return space.type(w_obj)
     # invoke the __new__ of the type
     if not we_are_jitted():
         # note that the annotator will figure out that w_type.w_bltin_new can
@@ -856,7 +848,8 @@
         call_init = space.isinstance_w(w_newobject, w_type)
 
     # maybe invoke the __init__ of the type
-    if call_init:
+    if (call_init and not (space.is_w(w_type, space.w_type) and
+        not __args__.keywords and len(__args__.arguments_w) == 1)):
         w_descr = space.lookup(w_newobject, '__init__')
         w_result = space.get_and_call_args(w_descr, w_newobject, __args__)
         if not space.is_w(w_result, space.w_None):
diff --git a/pypy/objspace/std/typetype.py b/pypy/objspace/std/typetype.py
--- a/pypy/objspace/std/typetype.py
+++ b/pypy/objspace/std/typetype.py
@@ -1,17 +1,28 @@
-from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.interpreter import gateway
 from pypy.interpreter.argument import Arguments
+from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.interpreter.typedef import (GetSetProperty, descr_get_dict,
                                       weakref_descr)
 from pypy.objspace.std.stdtypedef import StdTypeDef
 
-def descr__new__(space, w_typetype, w_name, w_bases, w_dict):
+
+def descr__new__(space, w_typetype, w_name, w_bases=gateway.NoneNotWrapped,
+    w_dict=gateway.NoneNotWrapped):
+
     "This is used to create user-defined classes only."
     from pypy.objspace.std.typeobject import W_TypeObject
     # XXX check types
 
     w_typetype = _precheck_for_new(space, w_typetype)
 
+    # special case for type(x)
+    if (space.is_w(space.type(w_typetype), space.w_type) and w_bases is None and
+        w_dict is None):
+        return space.type(w_name)
+    elif w_bases is None or w_dict is None:
+        raise OperationError(space.w_TypeError, space.wrap("type() takes 1 or 3 arguments"))
+
+
     bases_w = space.fixedview(w_bases)
 
     w_winner = w_typetype
diff --git a/pypy/objspace/test/test_descroperation.py b/pypy/objspace/test/test_descroperation.py
--- a/pypy/objspace/test/test_descroperation.py
+++ b/pypy/objspace/test/test_descroperation.py
@@ -667,5 +667,19 @@
                 return -1L
         raises(ValueError, len, Y())
 
+    def test_len_custom__int__(self):
+        class X(object):
+            def __init__(self, x):
+                self.x = x
+            def __len__(self):
+                return self.x
+            def __int__(self):
+                return self.x
+
+        l = len(X(3.0))
+        assert l == 3 and type(l) is int
+        l = len(X(X(2)))
+        assert l == 2 and type(l) is int
+
 class AppTestWithBuiltinShortcut(AppTest_Descroperation):
     OPTIONS = {'objspace.std.builtinshortcut': True}
diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -167,10 +167,7 @@
 
     This is for advanced usage only.
     """
-    # I hate the annotator so much.
-    if NonConstant(False):
-        return True
-    return False
+    return NonConstant(False)
 
 @oopspec("jit.isvirtual(value)")
 @specialize.ll()
diff --git a/pypy/rlib/rgc.py b/pypy/rlib/rgc.py
--- a/pypy/rlib/rgc.py
+++ b/pypy/rlib/rgc.py
@@ -3,6 +3,7 @@
 
 from pypy.rlib import jit
 from pypy.rlib.objectmodel import we_are_translated, enforceargs, specialize
+from pypy.rlib.nonconst import NonConstant
 from pypy.rpython.extregistry import ExtRegistryEntry
 from pypy.rpython.lltypesystem import lltype, llmemory
 
@@ -143,6 +144,10 @@
     from pypy.rpython.lltypesystem.lloperation import llop
     from pypy.rlib.objectmodel import keepalive_until_here
 
+    # XXX: Hack to ensure that we get a proper effectinfo.write_descrs_arrays
+    if NonConstant(False):
+        dest[dest_start] = source[source_start]
+
     # supports non-overlapping copies only
     if not we_are_translated():
         if source == dest:
diff --git a/pypy/rlib/test/test_jit.py b/pypy/rlib/test/test_jit.py
--- a/pypy/rlib/test/test_jit.py
+++ b/pypy/rlib/test/test_jit.py
@@ -1,7 +1,7 @@
 import py
 from pypy.conftest import option
 from pypy.rlib.jit import hint, we_are_jitted, JitDriver, elidable_promote
-from pypy.rlib.jit import JitHintError, oopspec
+from pypy.rlib.jit import JitHintError, oopspec, isconstant
 from pypy.translator.translator import TranslationContext, graphof
 from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin
 from pypy.rpython.lltypesystem import lltype
@@ -137,6 +137,16 @@
             t.view()
         # assert did not raise
 
+    def test_isconstant(self):
+        def f(n):
+            assert n >= 0
+            assert isconstant(n) is False
+            l = []
+            l.append(n)
+            return len(l)
+        res = self.interpret(f, [234])
+        assert res == 1
+
 
 class TestJITLLtype(BaseTestJIT, LLRtypeMixin):
     pass
diff --git a/pypy/rpython/lltypesystem/rlist.py b/pypy/rpython/lltypesystem/rlist.py
--- a/pypy/rpython/lltypesystem/rlist.py
+++ b/pypy/rpython/lltypesystem/rlist.py
@@ -1,15 +1,15 @@
+from pypy.rlib import rgc, jit
+from pypy.rlib.debug import ll_assert
+from pypy.rlib.objectmodel import enforceargs
+from pypy.rpython.lltypesystem import rstr
+from pypy.rpython.lltypesystem.lltype import (GcForwardReference, Ptr, GcArray,
+     GcStruct, Void, Signed, malloc, typeOf, nullptr, typeMethod)
+from pypy.rpython.rlist import (AbstractBaseListRepr, AbstractListRepr,
+    AbstractFixedSizeListRepr, AbstractListIteratorRepr, ll_setitem_nonneg,
+    ADTIList, ADTIFixedList, dum_nocheck)
+from pypy.rpython.rmodel import Repr, inputconst, externalvsinternal
 from pypy.tool.pairtype import pairtype, pair
-from pypy.rpython.rmodel import Repr, inputconst
-from pypy.rpython.rmodel import externalvsinternal
-from pypy.rpython.rlist import AbstractBaseListRepr, AbstractListRepr, \
-        AbstractFixedSizeListRepr, AbstractListIteratorRepr, \
-        ll_setitem_nonneg, ADTIList, ADTIFixedList
-from pypy.rpython.rlist import dum_nocheck
-from pypy.rpython.lltypesystem.lltype import GcForwardReference, Ptr, GcArray,\
-     GcStruct, Void, Signed, malloc, typeOf, nullptr, typeMethod
-from pypy.rpython.lltypesystem import rstr
-from pypy.rlib.debug import ll_assert
-from pypy.rlib import rgc, jit
+
 
 # ____________________________________________________________
 #
@@ -171,6 +171,7 @@
 
 # adapted C code
 
+ at enforceargs(None, int)
 def _ll_list_resize_really(l, newsize):
     """
     Ensure l.items has room for at least newsize elements, and set
@@ -210,7 +211,6 @@
         rgc.ll_arraycopy(items, newitems, 0, 0, p)
     l.length = newsize
     l.items = newitems
-_ll_list_resize_really._annenforceargs_ = (None, int)
 
 # this common case was factored out of _ll_list_resize
 # to see if inlining it gives some speed-up.
diff --git a/pypy/translator/c/genc.py b/pypy/translator/c/genc.py
--- a/pypy/translator/c/genc.py
+++ b/pypy/translator/c/genc.py
@@ -563,7 +563,10 @@
             else:
                 mk.definition('PYPY_MAIN_FUNCTION', "main")
 
-            if sys.platform == 'win32':
+            if (py.path.local.sysfind('python') or
+                py.path.local.sysfind('python.exe')):
+                python = 'python '
+            elif sys.platform == 'win32':
                 python = sys.executable.replace('\\', '/') + ' '
             else:
                 python = sys.executable + ' '

From noreply at buildbot.pypy.org  Mon Oct  3 08:36:37 2011
From: noreply at buildbot.pypy.org (hakanardo)
Date: Mon,  3 Oct 2011 08:36:37 +0200 (CEST)
Subject: [pypy-commit] pypy jit-optimizeopt-cleanups: failing test
Message-ID: <20111003063637.08759820D9@wyvern.cs.uni-duesseldorf.de>

Author: Hakan Ardo <hakan at debian.org>
Branch: jit-optimizeopt-cleanups
Changeset: r47785:8ab3accea25a
Date: 2011-10-03 08:24 +0200
http://bitbucket.org/pypy/pypy/changeset/8ab3accea25a/

Log:	failing test

diff --git a/pypy/jit/metainterp/test/test_string.py b/pypy/jit/metainterp/test/test_string.py
--- a/pypy/jit/metainterp/test/test_string.py
+++ b/pypy/jit/metainterp/test/test_string.py
@@ -484,6 +484,30 @@
             return len(sa.val)
         assert self.meta_interp(f, ['a']) == f('a')
 
+    def test_string_comepare_quasiimmutable(self):
+        class Sys(object):
+            _immutable_fields_ = ["defaultencoding?"]
+            def __init__(self, s):
+                self.defaultencoding = s
+        _str = self._str
+        sys = Sys(_str('ascii'))        
+        mydriver = JitDriver(reds = ['n', 'sa'], greens = [])
+        def f(n):
+            sa = 0
+            sys.defaultencoding = _str('ascii')
+            while n:
+                mydriver.jit_merge_point(n=n, sa=sa)
+                if sys.defaultencoding == _str('ascii'):
+                    sa += 1
+                n -= 1
+            sys.defaultencoding = _str('utf-8')
+            return sa
+        assert self.meta_interp(f, [8]) == f(8)
+        self.check_loops({'int_add': 1, 'guard_true': 1, 'int_sub': 1,
+                          'jump': 1, 'int_is_true': 1,
+                          'guard_not_invalidated': 1})
+
+
 #class TestOOtype(StringTests, OOJitMixin):
 #    CALL = "oosend"
 #    CALL_PURE = "oosend_pure"
diff --git a/pypy/module/pypyjit/test_pypy_c/test_00_model.py b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
--- a/pypy/module/pypyjit/test_pypy_c/test_00_model.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
@@ -50,8 +50,7 @@
         cmdline.append(str(self.filepath))
         #
         print cmdline, logfile
-        env={'PYPYLOG': 'jit-log-opt,jit-summary:' + str(logfile)}
-        #env={'PYPYLOG': ':' + str(logfile)}
+        env={'PYPYLOG': 'jit-log-opt,jit-log-noopt,jit-summary:' + str(logfile)}
         pipe = subprocess.Popen(cmdline,
                                 env=env,
                                 stdout=subprocess.PIPE,

From noreply at buildbot.pypy.org  Mon Oct  3 08:36:38 2011
From: noreply at buildbot.pypy.org (hakanardo)
Date: Mon,  3 Oct 2011 08:36:38 +0200 (CEST)
Subject: [pypy-commit] pypy jit-optimizeopt-cleanups: we want this part of
 optimize_CALL_PURE before OptString in the optimization chain
Message-ID: <20111003063638.36D90820D9@wyvern.cs.uni-duesseldorf.de>

Author: Hakan Ardo <hakan at debian.org>
Branch: jit-optimizeopt-cleanups
Changeset: r47786:a0de46f7ca56
Date: 2011-10-03 08:35 +0200
http://bitbucket.org/pypy/pypy/changeset/a0de46f7ca56/

Log:	we want this part of optimize_CALL_PURE before OptString in the
	optimization chain

diff --git a/pypy/jit/metainterp/optimizeopt/pure.py b/pypy/jit/metainterp/optimizeopt/pure.py
--- a/pypy/jit/metainterp/optimizeopt/pure.py
+++ b/pypy/jit/metainterp/optimizeopt/pure.py
@@ -57,23 +57,6 @@
             self.emit_operation(nextop)
 
     def optimize_CALL_PURE(self, op):
-        arg_consts = []
-        for i in range(op.numargs()):
-            arg = op.getarg(i)
-            const = self.get_constant_box(arg)
-            if const is None:
-                break
-            arg_consts.append(const)
-        else:
-            # all constant arguments: check if we already know the result
-            try:
-                result = self.optimizer.call_pure_results[arg_consts]
-            except KeyError:
-                pass
-            else:
-                self.make_constant(op.result, result)
-                return
-
         args = self.optimizer.make_args_key(op)
         oldop = self.pure_operations.get(args, None)
         if oldop is not None and oldop.getdescr() is op.getdescr():
diff --git a/pypy/jit/metainterp/optimizeopt/rewrite.py b/pypy/jit/metainterp/optimizeopt/rewrite.py
--- a/pypy/jit/metainterp/optimizeopt/rewrite.py
+++ b/pypy/jit/metainterp/optimizeopt/rewrite.py
@@ -421,6 +421,25 @@
             return True # 0-length arraycopy
         return False
 
+    def optimize_CALL_PURE(self, op):
+        arg_consts = []
+        for i in range(op.numargs()):
+            arg = op.getarg(i)
+            const = self.get_constant_box(arg)
+            if const is None:
+                break
+            arg_consts.append(const)
+        else:
+            # all constant arguments: check if we already know the result
+            try:
+                result = self.optimizer.call_pure_results[arg_consts]
+            except KeyError:
+                pass
+            else:
+                self.make_constant(op.result, result)
+                return
+        self.emit_operation(op)
+
     def optimize_INT_FLOORDIV(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))

From noreply at buildbot.pypy.org  Mon Oct  3 08:36:39 2011
From: noreply at buildbot.pypy.org (hakanardo)
Date: Mon,  3 Oct 2011 08:36:39 +0200 (CEST)
Subject: [pypy-commit] pypy jit-optimizeopt-cleanups: hg merge default
Message-ID: <20111003063639.71B0C820D9@wyvern.cs.uni-duesseldorf.de>

Author: Hakan Ardo <hakan at debian.org>
Branch: jit-optimizeopt-cleanups
Changeset: r47787:0c0391d46337
Date: 2011-10-03 08:36 +0200
http://bitbucket.org/pypy/pypy/changeset/0c0391d46337/

Log:	hg merge default

diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -23,6 +23,7 @@
         ("arccos", "arccos"),
         ("arcsin", "arcsin"),
         ("arctan", "arctan"),
+        ("arcsinh", "arcsinh"),
         ("copysign", "copysign"),
         ("cos", "cos"),
         ("divide", "divide"),
@@ -50,4 +51,5 @@
     appleveldefs = {
         'average': 'app_numpy.average',
         'mean': 'app_numpy.mean',
+        'inf': 'app_numpy.inf',
     }
diff --git a/pypy/module/micronumpy/app_numpy.py b/pypy/module/micronumpy/app_numpy.py
--- a/pypy/module/micronumpy/app_numpy.py
+++ b/pypy/module/micronumpy/app_numpy.py
@@ -1,5 +1,8 @@
 import numpy
 
+
+inf = float("inf")
+
 def average(a):
     # This implements a weighted average, for now we don't implement the
     # weighting, just the average part!
@@ -8,4 +11,4 @@
 def mean(a):
     if not hasattr(a, "mean"):
         a = numpy.array(a)
-    return a.mean()
\ No newline at end of file
+    return a.mean()
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -261,6 +261,9 @@
     @unaryop
     def arctan(self, v):
         return math.atan(v)
+    @unaryop
+    def arcsinh(self, v):
+        return math.asinh(v)
 
 class IntegerArithmeticDtype(ArithmaticTypeMixin):
     _mixin_ = True
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -268,6 +268,7 @@
             ("arcsin", "arcsin", 1, {"promote_to_float": True}),
             ("arccos", "arccos", 1, {"promote_to_float": True}),
             ("arctan", "arctan", 1, {"promote_to_float": True}),
+            ("arcsinh", "arcsinh", 1, {"promote_to_float": True}),
         ]:
             self.add_ufunc(space, *ufunc_def)
 
diff --git a/pypy/module/micronumpy/test/test_module.py b/pypy/module/micronumpy/test/test_module.py
--- a/pypy/module/micronumpy/test/test_module.py
+++ b/pypy/module/micronumpy/test/test_module.py
@@ -10,4 +10,9 @@
     def test_average(self):
         from numpy import array, average
         assert average(range(10)) == 4.5
-        assert average(array(range(10))) == 4.5
\ No newline at end of file
+        assert average(array(range(10))) == 4.5
+
+    def test_inf(self):
+        from numpy import inf
+        assert type(inf) is float
+        assert inf == float("inf")
\ No newline at end of file
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -298,6 +298,14 @@
         b = arctan(a)
         assert math.isnan(b[0])
 
+    def test_arcsinh(self):
+        import math
+        from numpy import arcsinh, inf
+
+        for v in [inf, -inf, 1.0, math.e]:
+            assert math.asinh(v) == arcsinh(v)
+        assert math.isnan(arcsinh(float("nan")))
+
     def test_reduce_errors(self):
         from numpy import sin, add
 
diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -258,15 +258,15 @@
             msg = "'%s' has no length" % (name,)
             raise OperationError(space.w_TypeError, space.wrap(msg))
         w_res = space.get_and_call_function(w_descr, w_obj)
-        space._check_len_result(w_res)
-        return w_res
+        return space.wrap(space._check_len_result(w_res))
 
     def _check_len_result(space, w_obj):
         # Will complain if result is too big.
-        result = space.int_w(w_obj)
+        result = space.int_w(space.int(w_obj))
         if result < 0:
             raise OperationError(space.w_ValueError,
                                  space.wrap("__len__() should return >= 0"))
+        return result
 
     def iter(space, w_obj):
         w_descr = space.lookup(w_obj, '__iter__')
diff --git a/pypy/objspace/std/rangeobject.py b/pypy/objspace/std/rangeobject.py
--- a/pypy/objspace/std/rangeobject.py
+++ b/pypy/objspace/std/rangeobject.py
@@ -23,7 +23,7 @@
 
 class W_RangeListObject(W_Object):
     typedef = listtype.list_typedef
-    
+
     def __init__(w_self, start, step, length):
         assert step != 0
         w_self.start = start
@@ -40,7 +40,7 @@
         if not length:
             w_self.w_list = space.newlist([])
             return w_self.w_list
-        
+
         arr = [None] * length  # this is to avoid using append.
 
         i = start
@@ -146,7 +146,11 @@
     if length == 0:
         raise OperationError(space.w_IndexError,
                              space.wrap("pop from empty list"))
-    idx = space.int_w(w_idx)
+    if space.isinstance_w(w_idx, space.w_float):
+        raise OperationError(space.w_TypeError,
+            space.wrap("integer argument expected, got float")
+        )
+    idx = space.int_w(space.int(w_idx))
     if idx == 0:
         result = w_rangelist.start
         w_rangelist.start += w_rangelist.step
diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -132,6 +132,19 @@
 
         assert A("hello") is str
 
+        # Make sure type(x) doesn't call x.__class__.__init__
+        class T(type):
+            counter = 0
+            def __init__(self, *args):
+                T.counter += 1
+        class C:
+            __metaclass__ = T
+        assert T.counter == 1
+        a = C()
+        assert T.counter == 1
+        assert type(a) is C
+        assert T.counter == 1
+
     def test_bases(self):
         assert int.__bases__ == (object,)
         class X:
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -77,7 +77,7 @@
         for i in range(len(self.lookup_where)):
             self.lookup_where[i] = None_None
 
-# possible values of compares_by_identity_status 
+# possible values of compares_by_identity_status
 UNKNOWN = 0
 COMPARES_BY_IDENTITY = 1
 OVERRIDES_EQ_CMP_OR_HASH = 2
@@ -358,7 +358,7 @@
                 if w_value is not None:
                     return w_value
         return None
-                
+
     @unroll_safe
     def _lookup(w_self, key):
         space = w_self.space
@@ -848,7 +848,8 @@
         call_init = space.isinstance_w(w_newobject, w_type)
 
     # maybe invoke the __init__ of the type
-    if call_init:
+    if (call_init and not (space.is_w(w_type, space.w_type) and
+        not __args__.keywords and len(__args__.arguments_w) == 1)):
         w_descr = space.lookup(w_newobject, '__init__')
         w_result = space.get_and_call_args(w_descr, w_newobject, __args__)
         if not space.is_w(w_result, space.w_None):
diff --git a/pypy/objspace/test/test_descroperation.py b/pypy/objspace/test/test_descroperation.py
--- a/pypy/objspace/test/test_descroperation.py
+++ b/pypy/objspace/test/test_descroperation.py
@@ -667,5 +667,19 @@
                 return -1L
         raises(ValueError, len, Y())
 
+    def test_len_custom__int__(self):
+        class X(object):
+            def __init__(self, x):
+                self.x = x
+            def __len__(self):
+                return self.x
+            def __int__(self):
+                return self.x
+
+        l = len(X(3.0))
+        assert l == 3 and type(l) is int
+        l = len(X(X(2)))
+        assert l == 2 and type(l) is int
+
 class AppTestWithBuiltinShortcut(AppTest_Descroperation):
     OPTIONS = {'objspace.std.builtinshortcut': True}
diff --git a/pypy/rpython/lltypesystem/rlist.py b/pypy/rpython/lltypesystem/rlist.py
--- a/pypy/rpython/lltypesystem/rlist.py
+++ b/pypy/rpython/lltypesystem/rlist.py
@@ -1,15 +1,15 @@
+from pypy.rlib import rgc, jit
+from pypy.rlib.debug import ll_assert
+from pypy.rlib.objectmodel import enforceargs
+from pypy.rpython.lltypesystem import rstr
+from pypy.rpython.lltypesystem.lltype import (GcForwardReference, Ptr, GcArray,
+     GcStruct, Void, Signed, malloc, typeOf, nullptr, typeMethod)
+from pypy.rpython.rlist import (AbstractBaseListRepr, AbstractListRepr,
+    AbstractFixedSizeListRepr, AbstractListIteratorRepr, ll_setitem_nonneg,
+    ADTIList, ADTIFixedList, dum_nocheck)
+from pypy.rpython.rmodel import Repr, inputconst, externalvsinternal
 from pypy.tool.pairtype import pairtype, pair
-from pypy.rpython.rmodel import Repr, inputconst
-from pypy.rpython.rmodel import externalvsinternal
-from pypy.rpython.rlist import AbstractBaseListRepr, AbstractListRepr, \
-        AbstractFixedSizeListRepr, AbstractListIteratorRepr, \
-        ll_setitem_nonneg, ADTIList, ADTIFixedList
-from pypy.rpython.rlist import dum_nocheck
-from pypy.rpython.lltypesystem.lltype import GcForwardReference, Ptr, GcArray,\
-     GcStruct, Void, Signed, malloc, typeOf, nullptr, typeMethod
-from pypy.rpython.lltypesystem import rstr
-from pypy.rlib.debug import ll_assert
-from pypy.rlib import rgc, jit
+
 
 # ____________________________________________________________
 #
@@ -171,6 +171,7 @@
 
 # adapted C code
 
+ at enforceargs(None, int)
 def _ll_list_resize_really(l, newsize):
     """
     Ensure l.items has room for at least newsize elements, and set
@@ -210,7 +211,6 @@
         rgc.ll_arraycopy(items, newitems, 0, 0, p)
     l.length = newsize
     l.items = newitems
-_ll_list_resize_really._annenforceargs_ = (None, int)
 
 # this common case was factored out of _ll_list_resize
 # to see if inlining it gives some speed-up.

From noreply at buildbot.pypy.org  Mon Oct  3 08:39:23 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  3 Oct 2011 08:39:23 +0200 (CEST)
Subject: [pypy-commit] pypy gil-improvement: Tweaks.
Message-ID: <20111003063923.B736E820D9@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: gil-improvement
Changeset: r47788:bdeb78cfcd9f
Date: 2011-10-03 08:11 +0200
http://bitbucket.org/pypy/pypy/changeset/bdeb78cfcd9f/

Log:	Tweaks.

diff --git a/pypy/translator/c/src/thread_nt.h b/pypy/translator/c/src/thread_nt.h
--- a/pypy/translator/c/src/thread_nt.h
+++ b/pypy/translator/c/src/thread_nt.h
@@ -251,9 +251,9 @@
        normally specifying a timeout of INFINITE would be fine.  But the
        first and second operations are not done atomically, so there is a
        (small) risk that PulseEvent misses the WaitForSingleObject().
-       In this case the process will just sleep 1 millisecond. */
+       In this case the process will just sleep a few milliseconds. */
     LeaveCriticalSection(&mutex_gil);
-    WaitForSingleObject(&cond_gil, 1);
+    WaitForSingleObject(&cond_gil, 15);
     EnterCriticalSection(&mutex_gil);
 
     InterlockedDecrement(&pending_acquires);
diff --git a/pypy/translator/c/src/thread_pthread.h b/pypy/translator/c/src/thread_pthread.h
--- a/pypy/translator/c/src/thread_pthread.h
+++ b/pypy/translator/c/src/thread_pthread.h
@@ -536,10 +536,10 @@
         return 0;
     atomic_add(&pending_acquires, 1L);
     _debug_print("{");
+    ASSERT_STATUS(pthread_cond_signal(&cond_gil));
     ASSERT_STATUS(pthread_cond_wait(&cond_gil, &mutex_gil));
     _debug_print("}");
     atomic_add(&pending_acquires, -1L);
-    ASSERT_STATUS(pthread_cond_signal(&cond_gil));
     assert_has_the_gil();
     return 1;
 }
@@ -552,6 +552,7 @@
 #endif
     assert_has_the_gil();
     ASSERT_STATUS(pthread_mutex_unlock(&mutex_gil));
+    ASSERT_STATUS(pthread_cond_signal(&cond_gil));
 }
 
 void RPyGilAcquire(void)
@@ -564,7 +565,6 @@
     ASSERT_STATUS(pthread_mutex_lock(&mutex_gil));
     atomic_add(&pending_acquires, -1L);
     assert_has_the_gil();
-    ASSERT_STATUS(pthread_cond_signal(&cond_gil));
     _debug_print("RPyGilAcquire\n");
 }
 

From noreply at buildbot.pypy.org  Mon Oct  3 11:28:31 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon,  3 Oct 2011 11:28:31 +0200 (CEST)
Subject: [pypy-commit] pypy default: Merge gil-improvement: improve the
 balance of threads getting the GIL, 
Message-ID: <20111003092831.5A709820D9@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r47789:99233931c2bf
Date: 2011-10-03 11:28 +0200
http://bitbucket.org/pypy/pypy/changeset/99233931c2bf/

Log:	Merge gil-improvement: improve the balance of threads getting the
	GIL, notably when there is a mixture of CPU-intensive and I/O
	threads.

	Previously, any CPU-intensive thread would release and immediately
	re-acquire the GIL every 100 bytecodes; but most of the time on
	multi-CPU machines this didn't actually gave other threads a chance
	to run. And so when an I/O thread did any operation, chances were
	that they would loose control and not regain it for quite a long
	time.

	Now, the yield-after-N-bytecodes step is different from the
	releasing and reacquiring the GIL around I/O: the former actually
	forces another thread to run (if there is one), while the later does
	not (which, in the common case where the I/O is very short, tends to
	let the same thread continuing to run). Also, using 10000 instead
	of 100 bytecodes as sys.checkinterval makes more sense in this
	version.

	This makes the example in https://bugs.pypy.org/issue884 work
	without any noticable delay at all, at least on Linux.

	The Windows version was tested too, but didn't give so good
	improvements; see comment in thread_nt.c. Will try to tweak a bit
	more, so I'm not closing the gil-improvement branch right now...

diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -307,7 +307,11 @@
         self._nonperiodic_actions = []
         self.has_bytecode_counter = False
         self.fired_actions = None
-        self.checkinterval_scaled = 100 * TICK_COUNTER_STEP
+        # the default value is not 100, unlike CPython 2.7, but a much
+        # larger value, because we use a technique that not only allows
+        # but actually *forces* another thread to run whenever the counter
+        # reaches zero.
+        self.checkinterval_scaled = 10000 * TICK_COUNTER_STEP
         self._rebuild_action_dispatcher()
 
     def fire(self, action):
diff --git a/pypy/module/thread/gil.py b/pypy/module/thread/gil.py
--- a/pypy/module/thread/gil.py
+++ b/pypy/module/thread/gil.py
@@ -16,7 +16,7 @@
 
 class GILThreadLocals(OSThreadLocals):
     """A version of OSThreadLocals that enforces a GIL."""
-    ll_GIL = thread.null_ll_lock
+    gil_ready = False
 
     def initialize(self, space):
         # add the GIL-releasing callback as an action on the space
@@ -25,12 +25,10 @@
 
     def setup_threads(self, space):
         """Enable threads in the object space, if they haven't already been."""
-        if not self.ll_GIL:
-            try:
-                self.ll_GIL = thread.allocate_ll_lock()
-            except thread.error:
+        if not self.gil_ready:
+            if not thread.gil_allocate():
                 raise wrap_thread_error(space, "can't allocate GIL")
-            thread.acquire_NOAUTO(self.ll_GIL, True)
+            self.gil_ready = True
             self.enter_thread(space)   # setup the main thread
             result = True
         else:
@@ -44,19 +42,16 @@
         # test_lock_again after the global state was cleared by
         # test_compile_lock.  As a workaround, we repatch these global
         # fields systematically.
-        spacestate.ll_GIL = self.ll_GIL
         invoke_around_extcall(before_external_call, after_external_call)
         return result
 
     def reinit_threads(self, space):
-        if self.ll_GIL:
-            self.ll_GIL = thread.allocate_ll_lock()
-            thread.acquire_NOAUTO(self.ll_GIL, True)
-            self.enter_thread(space)
+        if self.gil_ready:
+            self.gil_ready = False
+            self.setup_threads(space)
 
     def yield_thread(self):
-        thread.yield_thread()  # explicitly release the gil (used by test_gil)
-
+        do_yield_thread()
 
 class GILReleaseAction(PeriodicAsyncAction):
     """An action called every sys.checkinterval bytecodes.  It releases
@@ -64,16 +59,12 @@
     """
 
     def perform(self, executioncontext, frame):
-        # Other threads can run between the release() and the acquire()
-        # implicit in the following external function call (which has
-        # otherwise no effect).
-        thread.yield_thread()
+        do_yield_thread()
 
 
 class SpaceState:
 
     def _freeze_(self):
-        self.ll_GIL = thread.null_ll_lock
         self.action_after_thread_switch = None
         # ^^^ set by AsyncAction.fire_after_thread_switch()
         return False
@@ -95,14 +86,14 @@
     # this function must not raise, in such a way that the exception
     # transformer knows that it cannot raise!
     e = get_errno()
-    thread.release_NOAUTO(spacestate.ll_GIL)
+    thread.gil_release()
     set_errno(e)
 before_external_call._gctransformer_hint_cannot_collect_ = True
 before_external_call._dont_reach_me_in_del_ = True
 
 def after_external_call():
     e = get_errno()
-    thread.acquire_NOAUTO(spacestate.ll_GIL, True)
+    thread.gil_acquire()
     thread.gc_thread_run()
     spacestate.after_thread_switch()
     set_errno(e)
@@ -115,3 +106,18 @@
 # pointers in the shadow stack.  This is necessary because the GIL is
 # not held after the call to before_external_call() or before the call
 # to after_external_call().
+
+def do_yield_thread():
+    # explicitly release the gil, in a way that tries to give more
+    # priority to other threads (as opposed to continuing to run in
+    # the same thread).
+    if thread.gil_yield_thread():
+        thread.gc_thread_run()
+        spacestate.after_thread_switch()
+do_yield_thread._gctransformer_hint_close_stack_ = True
+do_yield_thread._dont_reach_me_in_del_ = True
+do_yield_thread._dont_inline_ = True
+
+# do_yield_thread() needs a different hint: _gctransformer_hint_close_stack_.
+# The *_external_call() functions are themselves called only from the rffi
+# module from a helper function that also has this hint.
diff --git a/pypy/module/thread/ll_thread.py b/pypy/module/thread/ll_thread.py
--- a/pypy/module/thread/ll_thread.py
+++ b/pypy/module/thread/ll_thread.py
@@ -17,7 +17,8 @@
     include_dirs = [str(py.path.local(autopath.pypydir).join('translator', 'c'))],
     export_symbols = ['RPyThreadGetIdent', 'RPyThreadLockInit',
                       'RPyThreadAcquireLock', 'RPyThreadReleaseLock',
-                      'RPyThreadYield',
+                      'RPyGilAllocate', 'RPyGilYieldThread',
+                      'RPyGilRelease', 'RPyGilAcquire',
                       'RPyThreadGetStackSize', 'RPyThreadSetStackSize',
                       'RPyOpaqueDealloc_ThreadLock',
                       'RPyThreadAfterFork']
@@ -69,8 +70,16 @@
                                          [TLOCKP], lltype.Void,
                                          _nowrapper=True)
 
-# this function does nothing apart from releasing the GIL temporarily.
-yield_thread = llexternal('RPyThreadYield', [], lltype.Void, threadsafe=True)
+# these functions manipulate directly the GIL, whose definition does not
+# escape the C code itself
+gil_allocate     = llexternal('RPyGilAllocate', [], lltype.Signed,
+                              _nowrapper=True)
+gil_yield_thread = llexternal('RPyGilYieldThread', [], lltype.Signed,
+                              _nowrapper=True)
+gil_release      = llexternal('RPyGilRelease', [], lltype.Void,
+                              _nowrapper=True)
+gil_acquire      = llexternal('RPyGilAcquire', [], lltype.Void,
+                              _nowrapper=True)
 
 def allocate_lock():
     return Lock(allocate_ll_lock())
diff --git a/pypy/module/thread/test/test_gil.py b/pypy/module/thread/test/test_gil.py
--- a/pypy/module/thread/test/test_gil.py
+++ b/pypy/module/thread/test/test_gil.py
@@ -30,19 +30,34 @@
     use_threads = True
     bigtest = False
 
-    def test_one_thread(self):
+    def test_one_thread(self, skew=+1):
+        from pypy.rlib.debug import debug_print
         if self.bigtest:
-            N = 1000000
+            N = 100000
+            skew *= 25000
         else:
             N = 100
+            skew *= 25
         space = FakeSpace()
         class State:
             pass
         state = State()
-        def runme():
-            for i in range(N):
+        def runme(main=False):
+            j = 0
+            for i in range(N + [-skew, skew][main]):
+                state.datalen1 += 1   # try to crash if the GIL is not
+                state.datalen2 += 1   # correctly acquired
                 state.data.append((thread.get_ident(), i))
+                state.datalen3 += 1
+                state.datalen4 += 1
+                assert state.datalen1 == len(state.data)
+                assert state.datalen2 == len(state.data)
+                assert state.datalen3 == len(state.data)
+                assert state.datalen4 == len(state.data)
+                debug_print(main, i, state.datalen4)
                 state.threadlocals.yield_thread()
+                assert i == j
+                j += 1
         def bootstrap():
             try:
                 runme()
@@ -50,20 +65,26 @@
                 thread.gc_thread_die()
         def f():
             state.data = []
+            state.datalen1 = 0
+            state.datalen2 = 0
+            state.datalen3 = 0
+            state.datalen4 = 0
             state.threadlocals = gil.GILThreadLocals()
             state.threadlocals.setup_threads(space)
             thread.gc_thread_prepare()
             subident = thread.start_new_thread(bootstrap, ())
             mainident = thread.get_ident()
-            runme()
+            runme(True)
             still_waiting = 3000
             while len(state.data) < 2*N:
+                debug_print(len(state.data))
                 if not still_waiting:
                     raise ValueError("time out")
                 still_waiting -= 1
                 if not we_are_translated(): gil.before_external_call()
                 time.sleep(0.01)
                 if not we_are_translated(): gil.after_external_call()
+            debug_print("leaving!")
             i1 = i2 = 0
             for tid, i in state.data:
                 if tid == mainident:
@@ -72,14 +93,17 @@
                     assert i == i2; i2 += 1
                 else:
                     assert 0
-            assert i1 == N
-            assert i2 == N
+            assert i1 == N + skew
+            assert i2 == N - skew
             return len(state.data)
 
         fn = self.getcompiled(f, [])
         res = fn()
         assert res == 2*N
 
+    def test_one_thread_rev(self):
+        self.test_one_thread(skew=-1)
+
 
 class TestRunDirectly(GILTests):
     def getcompiled(self, f, argtypes):
diff --git a/pypy/module/thread/test/test_thread.py b/pypy/module/thread/test/test_thread.py
--- a/pypy/module/thread/test/test_thread.py
+++ b/pypy/module/thread/test/test_thread.py
@@ -225,7 +225,8 @@
 
         def busy_wait():
             for x in range(1000):
-                time.sleep(0.01)
+                print 'tick...', x  # <-force the GIL to be released, as
+                time.sleep(0.01)    #   time.sleep doesn't do non-translated
 
         # This is normally called by app_main.py
         signal.signal(signal.SIGINT, signal.default_int_handler)
diff --git a/pypy/translator/c/gcc/trackgcroot.py b/pypy/translator/c/gcc/trackgcroot.py
--- a/pypy/translator/c/gcc/trackgcroot.py
+++ b/pypy/translator/c/gcc/trackgcroot.py
@@ -486,6 +486,8 @@
         'paddq', 'pinsr',
         # zero-extending moves should not produce GC pointers
         'movz', 
+        # locked operations should not move GC pointers, at least so far
+        'lock',
         ])
 
     # a partial list is hopefully good enough for now; it's all to support
diff --git a/pypy/translator/c/src/thread.h b/pypy/translator/c/src/thread.h
--- a/pypy/translator/c/src/thread.h
+++ b/pypy/translator/c/src/thread.h
@@ -37,14 +37,9 @@
 
 #endif
 
-/* common helper: this does nothing, but is called with the GIL released.
-   This gives other threads a chance to grab the GIL and run. */
-void RPyThreadYield(void);
-
-#ifndef PYPY_NOT_MAIN_FILE
-void RPyThreadYield(void)
-{
-}
-#endif
+long RPyGilAllocate(void);
+long RPyGilYieldThread(void);
+void RPyGilRelease(void);
+void RPyGilAcquire(void);
 
 #endif
diff --git a/pypy/translator/c/src/thread_nt.h b/pypy/translator/c/src/thread_nt.h
--- a/pypy/translator/c/src/thread_nt.h
+++ b/pypy/translator/c/src/thread_nt.h
@@ -221,4 +221,57 @@
 #define RPyThreadTLS_Set(key, value)	TlsSetValue(key, value)
 
 
+/************************************************************/
+/* GIL code                                                 */
+/************************************************************/
+
+static volatile LONG pending_acquires = -1;
+static CRITICAL_SECTION mutex_gil;
+static HANDLE cond_gil;
+
+long RPyGilAllocate(void)
+{
+    pending_acquires = 0;
+    InitializeCriticalSection(&mutex_gil);
+    EnterCriticalSection(&mutex_gil);
+    cond_gil = CreateEvent (NULL, FALSE, FALSE, NULL);
+    return 1;
+}
+
+long RPyGilYieldThread(void)
+{
+    /* can be called even before RPyGilAllocate(), but in this case,
+       pending_acquires will be -1 */
+    if (pending_acquires <= 0)
+        return 0;
+    InterlockedIncrement(&pending_acquires);
+    PulseEvent(&cond_gil);
+
+    /* hack: the three following lines do a pthread_cond_wait(), and
+       normally specifying a timeout of INFINITE would be fine.  But the
+       first and second operations are not done atomically, so there is a
+       (small) risk that PulseEvent misses the WaitForSingleObject().
+       In this case the process will just sleep a few milliseconds. */
+    LeaveCriticalSection(&mutex_gil);
+    WaitForSingleObject(&cond_gil, 15);
+    EnterCriticalSection(&mutex_gil);
+
+    InterlockedDecrement(&pending_acquires);
+    return 1;
+}
+
+void RPyGilRelease(void)
+{
+    LeaveCriticalSection(&mutex_gil);
+    PulseEvent(&cond_gil);
+}
+
+void RPyGilAcquire(void)
+{
+    InterlockedIncrement(&pending_acquires);
+    EnterCriticalSection(&mutex_gil);
+    InterlockedDecrement(&pending_acquires);
+}
+
+
 #endif /* PYPY_NOT_MAIN_FILE */
diff --git a/pypy/translator/c/src/thread_pthread.h b/pypy/translator/c/src/thread_pthread.h
--- a/pypy/translator/c/src/thread_pthread.h
+++ b/pypy/translator/c/src/thread_pthread.h
@@ -12,6 +12,7 @@
 #include <signal.h>
 #include <stdio.h>
 #include <errno.h>
+#include <assert.h>
 
 /* The following is hopefully equivalent to what CPython does
    (which is trying to compile a snippet of code using it) */
@@ -459,4 +460,113 @@
 #define RPyThreadTLS_Set(key, value)	pthread_setspecific(key, value)
 
 
+/************************************************************/
+/* GIL code                                                 */
+/************************************************************/
+
+#ifdef __llvm__
+#  define HAS_ATOMIC_ADD
+#endif
+
+#ifdef __GNUC__
+#  if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 1)
+#    define HAS_ATOMIC_ADD
+#  endif
+#endif
+
+#ifdef HAS_ATOMIC_ADD
+#  define atomic_add __sync_fetch_and_add
+#else
+#  if defined(__amd64__)
+#    define atomic_add(ptr, value)  asm volatile ("lock addq %0, %1"        \
+                                 : : "ri"(value), "m"(*(ptr)) : "memory")
+#  elif defined(__i386__)
+#    define atomic_add(ptr, value)  asm volatile ("lock addl %0, %1"        \
+                                 : : "ri"(value), "m"(*(ptr)) : "memory")
+#  else
+#    error "Please use gcc >= 4.1 or write a custom 'asm' for your CPU."
+#  endif
+#endif
+
+#define ASSERT_STATUS(call)                             \
+    if (call != 0) {                                    \
+        fprintf(stderr, "Fatal error: " #call "\n");    \
+        abort();                                        \
+    }
+
+static void _debug_print(const char *msg)
+{
+#if 0
+    int col = (int)pthread_self();
+    col = 31 + ((col / 8) % 8);
+    fprintf(stderr, "\033[%dm%s\033[0m", col, msg);
+#endif
+}
+
+static volatile long pending_acquires = -1;
+static pthread_mutex_t mutex_gil = PTHREAD_MUTEX_INITIALIZER;
+static pthread_cond_t cond_gil = PTHREAD_COND_INITIALIZER;
+
+static void assert_has_the_gil(void)
+{
+#ifdef RPY_ASSERT
+    assert(pthread_mutex_trylock(&mutex_gil) != 0);
+    assert(pending_acquires >= 0);
+#endif
+}
+
+long RPyGilAllocate(void)
+{
+    _debug_print("RPyGilAllocate\n");
+    pending_acquires = 0;
+    pthread_mutex_trylock(&mutex_gil);
+    assert_has_the_gil();
+    return 1;
+}
+
+long RPyGilYieldThread(void)
+{
+    /* can be called even before RPyGilAllocate(), but in this case,
+       pending_acquires will be -1 */
+#ifdef RPY_ASSERT
+    if (pending_acquires >= 0)
+        assert_has_the_gil();
+#endif
+    if (pending_acquires <= 0)
+        return 0;
+    atomic_add(&pending_acquires, 1L);
+    _debug_print("{");
+    ASSERT_STATUS(pthread_cond_signal(&cond_gil));
+    ASSERT_STATUS(pthread_cond_wait(&cond_gil, &mutex_gil));
+    _debug_print("}");
+    atomic_add(&pending_acquires, -1L);
+    assert_has_the_gil();
+    return 1;
+}
+
+void RPyGilRelease(void)
+{
+    _debug_print("RPyGilRelease\n");
+#ifdef RPY_ASSERT
+    assert(pending_acquires >= 0);
+#endif
+    assert_has_the_gil();
+    ASSERT_STATUS(pthread_mutex_unlock(&mutex_gil));
+    ASSERT_STATUS(pthread_cond_signal(&cond_gil));
+}
+
+void RPyGilAcquire(void)
+{
+    _debug_print("about to RPyGilAcquire...\n");
+#ifdef RPY_ASSERT
+    assert(pending_acquires >= 0);
+#endif
+    atomic_add(&pending_acquires, 1L);
+    ASSERT_STATUS(pthread_mutex_lock(&mutex_gil));
+    atomic_add(&pending_acquires, -1L);
+    assert_has_the_gil();
+    _debug_print("RPyGilAcquire\n");
+}
+
+
 #endif /* PYPY_NOT_MAIN_FILE */

From noreply at buildbot.pypy.org  Mon Oct  3 13:13:40 2011
From: noreply at buildbot.pypy.org (timo_jbo)
Date: Mon,  3 Oct 2011 13:13:40 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-data-buffer: translation improvement.
Message-ID: <20111003111340.75FCA820D9@wyvern.cs.uni-duesseldorf.de>

Author: Timo Paulssen <timonator at perpetuum-immobile.de>
Branch: numpy-data-buffer
Changeset: r47790:a8fbe3b9f870
Date: 2011-10-03 13:13 +0200
http://bitbucket.org/pypy/pypy/changeset/a8fbe3b9f870/

Log:	translation improvement.

diff --git a/pypy/module/_numpy/interp_numarray.py b/pypy/module/_numpy/interp_numarray.py
--- a/pypy/module/_numpy/interp_numarray.py
+++ b/pypy/module/_numpy/interp_numarray.py
@@ -16,7 +16,7 @@
 slice_driver = jit.JitDriver(greens=['signature'], reds=['i', 'j', 'step', 'stop', 'source', 'dest'])
 
 class BaseArray(Wrappable):
-    _attrs_ = ["invalidates", "signature"]
+    _attrs_ = ["invalidates", "signature", "_buffer"]
     BufferClass = NumpyBuffer
 
     def __init__(self):

From noreply at buildbot.pypy.org  Mon Oct  3 15:50:00 2011
From: noreply at buildbot.pypy.org (timo_jbo)
Date: Mon,  3 Oct 2011 15:50:00 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-data-buffer: fix translation
Message-ID: <20111003135000.BCC3D820D9@wyvern.cs.uni-duesseldorf.de>

Author: Timo Paulssen <timonator at perpetuum-immobile.de>
Branch: numpy-data-buffer
Changeset: r47791:e5cc9556a7bb
Date: 2011-10-03 15:43 +0200
http://bitbucket.org/pypy/pypy/changeset/e5cc9556a7bb/

Log:	fix translation

diff --git a/pypy/module/_numpy/interp_buffer.py b/pypy/module/_numpy/interp_buffer.py
--- a/pypy/module/_numpy/interp_buffer.py
+++ b/pypy/module/_numpy/interp_buffer.py
@@ -5,7 +5,6 @@
 
 class NumpyBuffer(RWBuffer):
     def __init__(self, array):
-        RWBuffer.__init__(self)
         self.array = array
 
     def getlength(self):

From noreply at buildbot.pypy.org  Mon Oct  3 15:50:01 2011
From: noreply at buildbot.pypy.org (timo_jbo)
Date: Mon,  3 Oct 2011 15:50:01 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-data-buffer: fix error messages for index
	out of bounds.
Message-ID: <20111003135001.E6E3B820D9@wyvern.cs.uni-duesseldorf.de>

Author: Timo Paulssen <timonator at perpetuum-immobile.de>
Branch: numpy-data-buffer
Changeset: r47792:103438c04860
Date: 2011-10-03 15:49 +0200
http://bitbucket.org/pypy/pypy/changeset/103438c04860/

Log:	fix error messages for index out of bounds.

diff --git a/pypy/module/_numpy/interp_buffer.py b/pypy/module/_numpy/interp_buffer.py
--- a/pypy/module/_numpy/interp_buffer.py
+++ b/pypy/module/_numpy/interp_buffer.py
@@ -12,7 +12,7 @@
 
     def getitem(self, index):
         if index > self.getlength():
-            raise IndexError("Index out of bounds (0<=index<%d)" % self.getlength())
+            raise IndexError("Index out of bounds (0<=index<=%d)" % self.getlength())
         storage = self.array.get_concrete().get_root_storage()
         char_data = rffi.cast(CHAR_TP, storage)
         index = self.calc_index(index)
@@ -20,7 +20,7 @@
 
     def setitem(self, index, value):
         if index > self.getlength():
-            raise IndexError("Index out of bounds (0<=index<%d)" % self.getlength())
+            raise IndexError("Index out of bounds (0<=index<=%d)" % self.getlength())
         storage = self.array.get_concrete().get_root_storage()
         char_ptr = rffi.cast(CHAR_TP, storage)
         index = self.calc_index(index)
@@ -28,7 +28,7 @@
 
     def setslice(self, index, newstring):
         if index + len(newstring) > self.getlength():
-            raise IndexError("End of slice to set out of bounds (0<=index<%d)" % self.getlength())
+            raise IndexError("End of slice to set out of bounds (0<=index<=%d)" % self.getlength())
         for idx in range(0, len(newstring)):
             self.setitem(index + idx, newstring[idx])
 

From noreply at buildbot.pypy.org  Mon Oct  3 16:23:51 2011
From: noreply at buildbot.pypy.org (timo_jbo)
Date: Mon,  3 Oct 2011 16:23:51 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-data-buffer: implement numpy.frombuffer
Message-ID: <20111003142351.72324820D9@wyvern.cs.uni-duesseldorf.de>

Author: Timo Paulssen <timonator at perpetuum-immobile.de>
Branch: numpy-data-buffer
Changeset: r47793:f30d7a06caf9
Date: 2011-10-03 16:23 +0200
http://bitbucket.org/pypy/pypy/changeset/f30d7a06caf9/

Log:	implement numpy.frombuffer

diff --git a/lib_pypy/numpy/__init__.py b/lib_pypy/numpy/__init__.py
--- a/lib_pypy/numpy/__init__.py
+++ b/lib_pypy/numpy/__init__.py
@@ -86,24 +86,34 @@
 
     return array(result)
 
+def __from_buffer_or_datastring(buf_or_str, dt, count, offset=0):
+    _dtype = dtype(dt)
+
+    if count > 0:
+        length = count * _dtype.itemsize
+        if length + offset > len(buf_or_str):
+            raise ValueError("length of string (%d) not enough for %d %s" %
+                             (len(buf_or_str), count, _dtype))
+
+        buf_or_str = buf_or_str[offset:length+offset]
+    else:
+        length = len(buf_or_str) - offset
+        buf_or_str = buf_or_str[offset:]
+        if len(buf_or_str) % _dtype.itemsize != 0:
+            raise ValueError("length of string (%d) not evenly dividable by size of dtype (%d)" %
+                             (len(buf_or_str), _dtype.itemsize))
+
+    arr = empty(length / _dtype.itemsize, dtype=_dtype)
+    arr.data[:length] = buf_or_str
+
+    return arr
+
+def frombuffer(buf, dtype=float, count=-1, offset=0):
+    return __from_buffer_or_datastring(buf, dtype, count, offset)
+
 def fromstring(s, dtype=float, count=-1, sep=''):
-    from _numpy import dtype as dt
     if sep:
         raise NotImplementedError("Cannot use fromstring with a separator yet")
 
-    _dtype = dt(dtype)
-    if count > 0:
-        length = count * _dtype.itemsize
-        if length > len(s):
-            raise ValueError("length of string (%d) not enough for %d %s" % (len(s), count, _dtype))
-        s = s[:length]
-    else:
-        length = len(s)
-        if len(s) % _dtype.itemsize != 0:
-            raise ValueError("length of string (%d) not evenly dividable by size of dtype (%d)" % (len(s), _dtype.itemsize))
+    return __from_buffer_or_datastring(s, dtype, count)
 
-    arr = empty(length / _dtype.itemsize, dtype=_dtype)
-    print len(arr.data), len(s), length, _dtype.itemsize
-    arr.data[:length] = s
-
-    return arr
diff --git a/lib_pypy/pypy_test/test_numpy.py b/lib_pypy/pypy_test/test_numpy.py
--- a/lib_pypy/pypy_test/test_numpy.py
+++ b/lib_pypy/pypy_test/test_numpy.py
@@ -96,3 +96,24 @@
         assert len(a) == 4
         for i in range(4):
             assert a[i] == i
+
+    def test_frombuffer(self):
+        from numpy import frombuffer
+        import struct
+
+        data = "\0\1\2\3\4"
+        a = frombuffer(data, dtype="int8", count=3)
+        b = frombuffer(data, dtype="int8", count=3, offset=1)
+        # does this depend on the machine architecture? byte-order?
+        assert a[0] == 0
+        assert b[0] == a[1] == 1
+
+        data = struct.pack("iiii", 0, 1, 0, 0)
+        a = frombuffer(data, dtype="i", count=1)[0]
+        b = frombuffer(data, dtype="i", count=1, offset=1)[0]
+        c = frombuffer(data, dtype="i", count=1, offset=2)[0]
+        d = frombuffer(data, dtype="i", count=1, offset=3)[0]
+        assert a == 0
+        assert b == 1 << 24
+        assert c == 1 << 16
+        assert d == 1 << 8

From noreply at buildbot.pypy.org  Mon Oct  3 16:37:18 2011
From: noreply at buildbot.pypy.org (timo_jbo)
Date: Mon,  3 Oct 2011 16:37:18 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-data-buffer: implement using separators
	for fromstring.
Message-ID: <20111003143718.717E6820D9@wyvern.cs.uni-duesseldorf.de>

Author: Timo Paulssen <timonator at perpetuum-immobile.de>
Branch: numpy-data-buffer
Changeset: r47794:fb2e36c041b7
Date: 2011-10-03 16:36 +0200
http://bitbucket.org/pypy/pypy/changeset/fb2e36c041b7/

Log:	implement using separators for fromstring.

diff --git a/lib_pypy/numpy/__init__.py b/lib_pypy/numpy/__init__.py
--- a/lib_pypy/numpy/__init__.py
+++ b/lib_pypy/numpy/__init__.py
@@ -113,7 +113,26 @@
 
 def fromstring(s, dtype=float, count=-1, sep=''):
     if sep:
-        raise NotImplementedError("Cannot use fromstring with a separator yet")
+        import numpy as np
+        dtype = np.dtype(dtype)
+
+        parts = s.split(sep)
+        clean_parts = [part for part in parts if part]
+        if count >= 0:
+            clean_parts = clean_parts[:count]
+
+        if dtype.kind == "f":
+            cast_func = float
+        elif dtype.kind == "i":
+            cast_func = int
+        else:
+            raise TypeError("Can only read int-likes or float-likes from strings.")
+
+        result = empty(len(clean_parts), dtype=dtype)
+        for number, value in enumerate(clean_parts):
+            result[number] = cast_func(value)
+
+        return result
 
     return __from_buffer_or_datastring(s, dtype, count)
 
diff --git a/lib_pypy/pypy_test/test_numpy.py b/lib_pypy/pypy_test/test_numpy.py
--- a/lib_pypy/pypy_test/test_numpy.py
+++ b/lib_pypy/pypy_test/test_numpy.py
@@ -97,6 +97,34 @@
         for i in range(4):
             assert a[i] == i
 
+        data = "0, 1, 2, 3, 4, 5, 6"
+        a = fromstring(data, dtype="i", sep=",")
+        assert len(a) == 7
+        assert list(a) == range(7)
+
+        data = "0,1,2,3,4,5,6"
+        a = fromstring(data, dtype="i", sep=",")
+        assert len(a) == 7
+        assert list(a) == range(7)
+
+        data = "0,  1, 2,    3, 4, 5,        6"
+        a = fromstring(data, dtype="i", sep=",")
+        assert len(a) == 7
+        assert list(a) == range(7)
+
+        data = "0 X 1 X 2 X 3 X 4    X     5   X   6"
+        a = fromstring(data, dtype="i", sep="X", count=4)
+        assert len(a) == 4
+        assert list(a) == range(4)
+
+        fdata = [f / 5.0 for f in range(10)]
+        data = ",".join(str(f) for f in fdata)
+        a = fromstring(data, dtype=float, sep=",")
+        assert list(a) == fdata
+
+        a = fromstring(data, dtype=float, sep=",", count=3)
+        assert list(a) == fdata[:3]
+
     def test_frombuffer(self):
         from numpy import frombuffer
         import struct

From noreply at buildbot.pypy.org  Mon Oct  3 16:48:46 2011
From: noreply at buildbot.pypy.org (timo_jbo)
Date: Mon,  3 Oct 2011 16:48:46 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-data-buffer: use StringBuilder for
	getslice in NumpyBuffer.
Message-ID: <20111003144846.517AC820D9@wyvern.cs.uni-duesseldorf.de>

Author: Timo Paulssen <timonator at perpetuum-immobile.de>
Branch: numpy-data-buffer
Changeset: r47795:fa778e5f7b0f
Date: 2011-10-03 16:48 +0200
http://bitbucket.org/pypy/pypy/changeset/fa778e5f7b0f/

Log:	use StringBuilder for getslice in NumpyBuffer.

diff --git a/pypy/module/_numpy/interp_buffer.py b/pypy/module/_numpy/interp_buffer.py
--- a/pypy/module/_numpy/interp_buffer.py
+++ b/pypy/module/_numpy/interp_buffer.py
@@ -1,5 +1,6 @@
 from pypy.interpreter.buffer import RWBuffer
 from pypy.rpython.lltypesystem import lltype, rffi
+from pypy.rlib.rstring import StringBuilder
 
 CHAR_TP = lltype.Ptr(lltype.Array(lltype.Char, hints={'nolength': True}))
 
@@ -10,27 +11,41 @@
     def getlength(self):
         return self.array.get_concrete().find_size() * self.array.find_dtype().num_bytes
 
-    def getitem(self, index):
-        if index > self.getlength():
-            raise IndexError("Index out of bounds (0<=index<=%d)" % self.getlength())
+    def getitem_noboundcheck(self, index):
         storage = self.array.get_concrete().get_root_storage()
         char_data = rffi.cast(CHAR_TP, storage)
         index = self.calc_index(index)
         return char_data[index]
 
-    def setitem(self, index, value):
+    def getitem(self, index):
         if index > self.getlength():
             raise IndexError("Index out of bounds (0<=index<=%d)" % self.getlength())
+        return self.getitem_noboundcheck(index)
+
+    def setitem_noboundcheck(self, index, value):
         storage = self.array.get_concrete().get_root_storage()
         char_ptr = rffi.cast(CHAR_TP, storage)
         index = self.calc_index(index)
         char_ptr[index] = value
 
+    def setitem(self, index, value):
+        if index > self.getlength():
+            raise IndexError("Index out of bounds (0<=index<=%d)" % self.getlength())
+        self.setitem_noboundcheck(index, value)
+
     def setslice(self, index, newstring):
         if index + len(newstring) > self.getlength():
             raise IndexError("End of slice to set out of bounds (0<=index<=%d)" % self.getlength())
         for idx in range(0, len(newstring)):
-            self.setitem(index + idx, newstring[idx])
+            self.setitem_noboundcheck(index + idx, newstring[idx])
+
+    def getslice(self, start, stop, step, size):
+        builder = StringBuilder(size)
+
+        for index in range(start, stop, step):
+            builder.append(self.getitem_noboundcheck(index))
+
+        return builder.build()
 
     def calc_index(self, index):
         return index

From noreply at buildbot.pypy.org  Mon Oct  3 17:11:06 2011
From: noreply at buildbot.pypy.org (timo_jbo)
Date: Mon,  3 Oct 2011 17:11:06 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-data-buffer: simple implementation of
	fromfile.
Message-ID: <20111003151106.D698B820D9@wyvern.cs.uni-duesseldorf.de>

Author: Timo Paulssen <timonator at perpetuum-immobile.de>
Branch: numpy-data-buffer
Changeset: r47796:60161adfdbba
Date: 2011-10-03 17:01 +0200
http://bitbucket.org/pypy/pypy/changeset/60161adfdbba/

Log:	simple implementation of fromfile.

diff --git a/lib_pypy/numpy/__init__.py b/lib_pypy/numpy/__init__.py
--- a/lib_pypy/numpy/__init__.py
+++ b/lib_pypy/numpy/__init__.py
@@ -136,3 +136,7 @@
 
     return __from_buffer_or_datastring(s, dtype, count)
 
+def fromfile(file, dtype=float, count=-1, sep=''):
+    if isinstance(file, basestring):
+        file = open(file, "r")
+    return fromstring(file.read(), dtype, count, sep)

From noreply at buildbot.pypy.org  Mon Oct  3 17:36:04 2011
From: noreply at buildbot.pypy.org (timo_jbo)
Date: Mon,  3 Oct 2011 17:36:04 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-data-buffer: test numpy.fromfile.
Message-ID: <20111003153604.E9310820D9@wyvern.cs.uni-duesseldorf.de>

Author: Timo Paulssen <timonator at perpetuum-immobile.de>
Branch: numpy-data-buffer
Changeset: r47797:49b174c8db3a
Date: 2011-10-03 17:35 +0200
http://bitbucket.org/pypy/pypy/changeset/49b174c8db3a/

Log:	test numpy.fromfile.

diff --git a/lib_pypy/pypy_test/test_numpy.py b/lib_pypy/pypy_test/test_numpy.py
--- a/lib_pypy/pypy_test/test_numpy.py
+++ b/lib_pypy/pypy_test/test_numpy.py
@@ -145,3 +145,35 @@
         assert b == 1 << 24
         assert c == 1 << 16
         assert d == 1 << 8
+
+    def test_fromfile(self):
+        from numpy import fromfile
+        from tempfile import NamedTemporaryFile
+        from os import remove
+        import cStringIO
+        import struct
+
+        my_file = cStringIO.StringIO(struct.pack("iiiiiiii", *range(8)))
+        a = fromfile(my_file, dtype="i")
+        assert list(a) == range(8)
+        my_file.seek(0)
+        b = fromfile(my_file, dtype="i", count=3)
+        assert list(b) == range(3)
+
+        my_file = cStringIO.StringIO(struct.pack("dddddddd", *[f / 5.0 for f in range(8)]))
+        a = fromfile(my_file, dtype=float)
+        assert list(a) == [f / 5.0 for f in range(8)]
+
+        my_file = cStringIO.StringIO(" ".join(map(str, range(10))))
+        a = fromfile(my_file, dtype="i", sep=" ")
+        assert list(a) == range(10)
+        my_file.seek(0)
+        b = fromfile(my_file, dtype="i", sep=" ", count=3)
+        assert list(b) == range(3)
+
+        tmpf = NamedTemporaryFile(delete=False)
+        tmpf.file.write(struct.pack("dddddddd", *[f / 5.0 for f in range(8)]))
+        tmpf.file.close()
+        a = fromfile(tmpf.name, dtype=float)
+        assert list(a) == [f / 5.0 for f in range(8)]
+        remove(tmpf.name)

From notifications-noreply at bitbucket.org  Mon Oct  3 19:34:41 2011
From: notifications-noreply at bitbucket.org (Bitbucket)
Date: Mon, 03 Oct 2011 17:34:41 -0000
Subject: [pypy-commit] Notification: pypy
Message-ID: <20111003173441.10406.97798@bitbucket12.managed.contegix.com>

You have received a notification from Thiago Avelino.

Hi, I forked pypy. My fork is at https://bitbucket.org/avelino/pypy.

--
Disable notifications at https://bitbucket.org/account/notifications/

From noreply at buildbot.pypy.org  Mon Oct  3 19:55:48 2011
From: noreply at buildbot.pypy.org (justinpeel)
Date: Mon,  3 Oct 2011 19:55:48 +0200 (CEST)
Subject: [pypy-commit] pypy unsigned-dtypes: remove some commented code and
 skip a zjit test until we change llimpl
Message-ID: <20111003175548.486F3820D9@wyvern.cs.uni-duesseldorf.de>

Author: Justin Peel <notmuchtotell at gmail.com>
Branch: unsigned-dtypes
Changeset: r47798:1075b6767e74
Date: 2011-10-03 11:55 -0600
http://bitbucket.org/pypy/pypy/changeset/1075b6767e74/

Log:	remove some commented code and skip a zjit test until we change
	llimpl

diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -459,23 +459,12 @@
 class W_Float64Dtype(FloatArithmeticDtype, W_Float64Dtype):
     pass
 
-#W_Float96Dtype = create_low_level_dtype(
-#    num = 13, kind = FLOATINGLTR, name = "float96",
-#    aliases = ["g", "float96"],
-#    applevel_types = [],
-#    T = lltype.Float, # LongFloat
-#    valtype = float, # r_longfloat
-#    expected_size = 8, # 12
-#)
-#class W_Float96Dtype(FloatArithmeticDtype, W_Float96Dtype):
-#    pass
-
 ALL_DTYPES = [
     W_BoolDtype,
     W_Int8Dtype, W_UInt8Dtype, W_Int16Dtype, W_UInt16Dtype,
     W_Int32Dtype, W_UInt32Dtype, W_LongDtype, W_ULongDtype,
     W_Int64Dtype, W_UInt64Dtype,
-    W_Float32Dtype, W_Float64Dtype, #W_Float96Dtype,
+    W_Float32Dtype, W_Float64Dtype,
 ]
 
 dtypes_by_alias = unrolling_iterable([
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -9,6 +9,8 @@
 from pypy.rpython.annlowlevel import llstr
 from pypy.rpython.test.test_llinterp import interpret
 
+import py
+
 
 class TestNumpyJIt(LLJitMixin):
     def setup_class(cls):
@@ -306,6 +308,9 @@
         assert result == 11.0
 
     def test_int32_sum(self):
+        py.test.skip("pypy/jit/backend/llimpl.py needs to be changed to "
+                     "deal correctly with int dtypes for this test to "
+                     "work. skip for now until someone feels up to the task")
         space = self.space
         float64_dtype = self.float64_dtype
         int32_dtype = self.int32_dtype
diff --git a/pypy/objspace/std/iterobject.py b/pypy/objspace/std/iterobject.py
--- a/pypy/objspace/std/iterobject.py
+++ b/pypy/objspace/std/iterobject.py
@@ -35,16 +35,32 @@
         w_self.listitems = wrappeditems
 
 class W_FastTupleIterObject(W_AbstractSeqIterObject):
-   """Sequence iterator specialized for tuples, accessing
-   directly their RPython-level list of wrapped objects.
-   """ 
-   def __init__(w_self, w_seq, wrappeditems):
+    """Sequence iterator specialized for tuples, accessing
+    directly their RPython-level list of wrapped objects.
+    """
+    def __init__(w_self, w_seq, wrappeditems):
         W_AbstractSeqIterObject.__init__(w_self, w_seq)
         w_self.tupleitems = wrappeditems
 
+class W_FastStringIterObject(W_AbstractSeqIterObject):
+    """Sequence iterator specialized for strings, accessing
+    directly their RPython string.
+    """
+    def __init__(w_self, w_seq, value):
+        W_AbstractSeqIterObject.__init__(w_self, w_seq)
+        w_self.chars = value
+
+class W_FastUnicodeIterObject(W_AbstractSeqIterObject):
+    """Sequence iterator specialized for strings, accessing
+    directly their RPython string.
+    """
+    def __init__(w_self, w_seq, value):
+        W_AbstractSeqIterObject.__init__(w_self, w_seq)
+        w_self.unichars = value
+
 class W_ReverseSeqIterObject(W_Object):
     from pypy.objspace.std.itertype import reverse_iter_typedef as typedef
-    
+
     def __init__(w_self, space, w_seq, index=-1):
         w_self.w_seq = w_seq
         w_self.w_len = space.len(w_seq)
@@ -54,6 +70,8 @@
 registerimplementation(W_SeqIterObject)
 registerimplementation(W_FastListIterObject)
 registerimplementation(W_FastTupleIterObject)
+registerimplementation(W_FastStringIterObject)
+registerimplementation(W_FastUnicodeIterObject)
 registerimplementation(W_ReverseSeqIterObject)
 
 def iter__SeqIter(space, w_seqiter):
@@ -119,6 +137,48 @@
 ##    return w_seqiter.getlength(space)
 
 
+def iter__FastStringIter(space, w_seqiter):
+    return w_seqiter
+
+def next__FastStringIter(space, w_seqiter):
+    if w_seqiter.chars is None:
+        raise OperationError(space.w_StopIteration, space.w_None)
+    index = w_seqiter.index
+    try:
+        s = w_seqiter.chars[index]
+    except IndexError:
+        w_seqiter.chars = None
+        w_seqiter.w_seq = None
+        raise OperationError(space.w_StopIteration, space.w_None) 
+    w_seqiter.index = index + 1
+    return space.wrap(s)
+
+# XXX __length_hint__()
+##def len__FastStringIter(space, w_seqiter):
+##    return w_seqiter.getlength(space)
+
+
+def iter__FastUnicodeIter(space, w_seqiter):
+    return w_seqiter
+
+def next__FastUnicodeIter(space, w_seqiter):
+    if w_seqiter.unichars is None:
+        raise OperationError(space.w_StopIteration, space.w_None)
+    index = w_seqiter.index
+    try:
+        s = w_seqiter.unichars[index]
+    except IndexError:
+        w_seqiter.unichars = None
+        w_seqiter.w_seq = None
+        raise OperationError(space.w_StopIteration, space.w_None) 
+    w_seqiter.index = index + 1
+    return space.wrap(s)
+
+# XXX __length_hint__()
+##def len__FastUnicodeIter(space, w_seqiter):
+##    return w_seqiter.getlength(space)
+
+
 def iter__ReverseSeqIter(space, w_seqiter):
     return w_seqiter
 
diff --git a/pypy/objspace/std/model.py b/pypy/objspace/std/model.py
--- a/pypy/objspace/std/model.py
+++ b/pypy/objspace/std/model.py
@@ -122,6 +122,8 @@
             iterobject.W_SeqIterObject: [],
             iterobject.W_FastListIterObject: [],
             iterobject.W_FastTupleIterObject: [],
+            iterobject.W_FastStringIterObject: [],
+            iterobject.W_FastUnicodeIterObject: [],
             iterobject.W_ReverseSeqIterObject: [],
             unicodeobject.W_UnicodeObject: [],
             dictmultiobject.W_DictViewKeysObject: [],
diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py
--- a/pypy/objspace/std/stringobject.py
+++ b/pypy/objspace/std/stringobject.py
@@ -863,6 +863,10 @@
     else:
         return sliced(space, s, start, stop, w_str)
 
+def iter__String(space, w_str):
+    from pypy.objspace.std import iterobject
+    return iterobject.W_FastStringIterObject(w_str, w_str._value)
+
 def mul_string_times(space, w_str, w_times):
     try:
         mul = space.getindex_w(w_times, space.w_OverflowError)
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -265,6 +265,10 @@
     start, stop = normalize_simple_slice(space, len(uni), w_start, w_stop)
     return W_UnicodeObject(uni[start:stop])
 
+def iter__Unicode(space, w_uni):
+    from pypy.objspace.std import iterobject
+    return iterobject.W_FastUnicodeIterObject(w_uni, w_uni._value)
+
 def mul__Unicode_ANY(space, w_uni, w_times):
     try:
         times = space.getindex_w(w_times, space.w_OverflowError)

From noreply at buildbot.pypy.org  Mon Oct  3 20:07:06 2011
From: noreply at buildbot.pypy.org (justinpeel)
Date: Mon,  3 Oct 2011 20:07:06 +0200 (CEST)
Subject: [pypy-commit] pypy unsigned-dtypes: remove iter stuff that I hadn't
 intended to merge
Message-ID: <20111003180706.C8A31820D9@wyvern.cs.uni-duesseldorf.de>

Author: Justin Peel <notmuchtotell at gmail.com>
Branch: unsigned-dtypes
Changeset: r47799:e08fe0a276cb
Date: 2011-10-03 12:02 -0600
http://bitbucket.org/pypy/pypy/changeset/e08fe0a276cb/

Log:	remove iter stuff that I hadn't intended to merge

diff --git a/pypy/objspace/std/iterobject.py b/pypy/objspace/std/iterobject.py
--- a/pypy/objspace/std/iterobject.py
+++ b/pypy/objspace/std/iterobject.py
@@ -35,32 +35,16 @@
         w_self.listitems = wrappeditems
 
 class W_FastTupleIterObject(W_AbstractSeqIterObject):
-    """Sequence iterator specialized for tuples, accessing
-    directly their RPython-level list of wrapped objects.
-    """
-    def __init__(w_self, w_seq, wrappeditems):
+   """Sequence iterator specialized for tuples, accessing
+   directly their RPython-level list of wrapped objects.
+   """ 
+   def __init__(w_self, w_seq, wrappeditems):
         W_AbstractSeqIterObject.__init__(w_self, w_seq)
         w_self.tupleitems = wrappeditems
 
-class W_FastStringIterObject(W_AbstractSeqIterObject):
-    """Sequence iterator specialized for strings, accessing
-    directly their RPython string.
-    """
-    def __init__(w_self, w_seq, value):
-        W_AbstractSeqIterObject.__init__(w_self, w_seq)
-        w_self.chars = value
-
-class W_FastUnicodeIterObject(W_AbstractSeqIterObject):
-    """Sequence iterator specialized for strings, accessing
-    directly their RPython string.
-    """
-    def __init__(w_self, w_seq, value):
-        W_AbstractSeqIterObject.__init__(w_self, w_seq)
-        w_self.unichars = value
-
 class W_ReverseSeqIterObject(W_Object):
     from pypy.objspace.std.itertype import reverse_iter_typedef as typedef
-
+    
     def __init__(w_self, space, w_seq, index=-1):
         w_self.w_seq = w_seq
         w_self.w_len = space.len(w_seq)
@@ -70,8 +54,6 @@
 registerimplementation(W_SeqIterObject)
 registerimplementation(W_FastListIterObject)
 registerimplementation(W_FastTupleIterObject)
-registerimplementation(W_FastStringIterObject)
-registerimplementation(W_FastUnicodeIterObject)
 registerimplementation(W_ReverseSeqIterObject)
 
 def iter__SeqIter(space, w_seqiter):
@@ -137,48 +119,6 @@
 ##    return w_seqiter.getlength(space)
 
 
-def iter__FastStringIter(space, w_seqiter):
-    return w_seqiter
-
-def next__FastStringIter(space, w_seqiter):
-    if w_seqiter.chars is None:
-        raise OperationError(space.w_StopIteration, space.w_None)
-    index = w_seqiter.index
-    try:
-        s = w_seqiter.chars[index]
-    except IndexError:
-        w_seqiter.chars = None
-        w_seqiter.w_seq = None
-        raise OperationError(space.w_StopIteration, space.w_None) 
-    w_seqiter.index = index + 1
-    return space.wrap(s)
-
-# XXX __length_hint__()
-##def len__FastStringIter(space, w_seqiter):
-##    return w_seqiter.getlength(space)
-
-
-def iter__FastUnicodeIter(space, w_seqiter):
-    return w_seqiter
-
-def next__FastUnicodeIter(space, w_seqiter):
-    if w_seqiter.unichars is None:
-        raise OperationError(space.w_StopIteration, space.w_None)
-    index = w_seqiter.index
-    try:
-        s = w_seqiter.unichars[index]
-    except IndexError:
-        w_seqiter.unichars = None
-        w_seqiter.w_seq = None
-        raise OperationError(space.w_StopIteration, space.w_None) 
-    w_seqiter.index = index + 1
-    return space.wrap(s)
-
-# XXX __length_hint__()
-##def len__FastUnicodeIter(space, w_seqiter):
-##    return w_seqiter.getlength(space)
-
-
 def iter__ReverseSeqIter(space, w_seqiter):
     return w_seqiter
 
diff --git a/pypy/objspace/std/model.py b/pypy/objspace/std/model.py
--- a/pypy/objspace/std/model.py
+++ b/pypy/objspace/std/model.py
@@ -122,8 +122,6 @@
             iterobject.W_SeqIterObject: [],
             iterobject.W_FastListIterObject: [],
             iterobject.W_FastTupleIterObject: [],
-            iterobject.W_FastStringIterObject: [],
-            iterobject.W_FastUnicodeIterObject: [],
             iterobject.W_ReverseSeqIterObject: [],
             unicodeobject.W_UnicodeObject: [],
             dictmultiobject.W_DictViewKeysObject: [],
diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py
--- a/pypy/objspace/std/stringobject.py
+++ b/pypy/objspace/std/stringobject.py
@@ -863,10 +863,6 @@
     else:
         return sliced(space, s, start, stop, w_str)
 
-def iter__String(space, w_str):
-    from pypy.objspace.std import iterobject
-    return iterobject.W_FastStringIterObject(w_str, w_str._value)
-
 def mul_string_times(space, w_str, w_times):
     try:
         mul = space.getindex_w(w_times, space.w_OverflowError)
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -265,10 +265,6 @@
     start, stop = normalize_simple_slice(space, len(uni), w_start, w_stop)
     return W_UnicodeObject(uni[start:stop])
 
-def iter__Unicode(space, w_uni):
-    from pypy.objspace.std import iterobject
-    return iterobject.W_FastUnicodeIterObject(w_uni, w_uni._value)
-
 def mul__Unicode_ANY(space, w_uni, w_times):
     try:
         times = space.getindex_w(w_times, space.w_OverflowError)

From noreply at buildbot.pypy.org  Mon Oct  3 20:09:00 2011
From: noreply at buildbot.pypy.org (justinpeel)
Date: Mon,  3 Oct 2011 20:09:00 +0200 (CEST)
Subject: [pypy-commit] pypy unsigned-dtypes: merge in default
Message-ID: <20111003180900.E4330820D9@wyvern.cs.uni-duesseldorf.de>

Author: Justin Peel <notmuchtotell at gmail.com>
Branch: unsigned-dtypes
Changeset: r47800:6b3e91c0764a
Date: 2011-10-03 12:08 -0600
http://bitbucket.org/pypy/pypy/changeset/6b3e91c0764a/

Log:	merge in default

diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -307,7 +307,11 @@
         self._nonperiodic_actions = []
         self.has_bytecode_counter = False
         self.fired_actions = None
-        self.checkinterval_scaled = 100 * TICK_COUNTER_STEP
+        # the default value is not 100, unlike CPython 2.7, but a much
+        # larger value, because we use a technique that not only allows
+        # but actually *forces* another thread to run whenever the counter
+        # reaches zero.
+        self.checkinterval_scaled = 10000 * TICK_COUNTER_STEP
         self._rebuild_action_dispatcher()
 
     def fire(self, action):
diff --git a/pypy/module/thread/gil.py b/pypy/module/thread/gil.py
--- a/pypy/module/thread/gil.py
+++ b/pypy/module/thread/gil.py
@@ -16,7 +16,7 @@
 
 class GILThreadLocals(OSThreadLocals):
     """A version of OSThreadLocals that enforces a GIL."""
-    ll_GIL = thread.null_ll_lock
+    gil_ready = False
 
     def initialize(self, space):
         # add the GIL-releasing callback as an action on the space
@@ -25,12 +25,10 @@
 
     def setup_threads(self, space):
         """Enable threads in the object space, if they haven't already been."""
-        if not self.ll_GIL:
-            try:
-                self.ll_GIL = thread.allocate_ll_lock()
-            except thread.error:
+        if not self.gil_ready:
+            if not thread.gil_allocate():
                 raise wrap_thread_error(space, "can't allocate GIL")
-            thread.acquire_NOAUTO(self.ll_GIL, True)
+            self.gil_ready = True
             self.enter_thread(space)   # setup the main thread
             result = True
         else:
@@ -44,19 +42,16 @@
         # test_lock_again after the global state was cleared by
         # test_compile_lock.  As a workaround, we repatch these global
         # fields systematically.
-        spacestate.ll_GIL = self.ll_GIL
         invoke_around_extcall(before_external_call, after_external_call)
         return result
 
     def reinit_threads(self, space):
-        if self.ll_GIL:
-            self.ll_GIL = thread.allocate_ll_lock()
-            thread.acquire_NOAUTO(self.ll_GIL, True)
-            self.enter_thread(space)
+        if self.gil_ready:
+            self.gil_ready = False
+            self.setup_threads(space)
 
     def yield_thread(self):
-        thread.yield_thread()  # explicitly release the gil (used by test_gil)
-
+        do_yield_thread()
 
 class GILReleaseAction(PeriodicAsyncAction):
     """An action called every sys.checkinterval bytecodes.  It releases
@@ -64,16 +59,12 @@
     """
 
     def perform(self, executioncontext, frame):
-        # Other threads can run between the release() and the acquire()
-        # implicit in the following external function call (which has
-        # otherwise no effect).
-        thread.yield_thread()
+        do_yield_thread()
 
 
 class SpaceState:
 
     def _freeze_(self):
-        self.ll_GIL = thread.null_ll_lock
         self.action_after_thread_switch = None
         # ^^^ set by AsyncAction.fire_after_thread_switch()
         return False
@@ -95,14 +86,14 @@
     # this function must not raise, in such a way that the exception
     # transformer knows that it cannot raise!
     e = get_errno()
-    thread.release_NOAUTO(spacestate.ll_GIL)
+    thread.gil_release()
     set_errno(e)
 before_external_call._gctransformer_hint_cannot_collect_ = True
 before_external_call._dont_reach_me_in_del_ = True
 
 def after_external_call():
     e = get_errno()
-    thread.acquire_NOAUTO(spacestate.ll_GIL, True)
+    thread.gil_acquire()
     thread.gc_thread_run()
     spacestate.after_thread_switch()
     set_errno(e)
@@ -115,3 +106,18 @@
 # pointers in the shadow stack.  This is necessary because the GIL is
 # not held after the call to before_external_call() or before the call
 # to after_external_call().
+
+def do_yield_thread():
+    # explicitly release the gil, in a way that tries to give more
+    # priority to other threads (as opposed to continuing to run in
+    # the same thread).
+    if thread.gil_yield_thread():
+        thread.gc_thread_run()
+        spacestate.after_thread_switch()
+do_yield_thread._gctransformer_hint_close_stack_ = True
+do_yield_thread._dont_reach_me_in_del_ = True
+do_yield_thread._dont_inline_ = True
+
+# do_yield_thread() needs a different hint: _gctransformer_hint_close_stack_.
+# The *_external_call() functions are themselves called only from the rffi
+# module from a helper function that also has this hint.
diff --git a/pypy/module/thread/ll_thread.py b/pypy/module/thread/ll_thread.py
--- a/pypy/module/thread/ll_thread.py
+++ b/pypy/module/thread/ll_thread.py
@@ -17,7 +17,8 @@
     include_dirs = [str(py.path.local(autopath.pypydir).join('translator', 'c'))],
     export_symbols = ['RPyThreadGetIdent', 'RPyThreadLockInit',
                       'RPyThreadAcquireLock', 'RPyThreadReleaseLock',
-                      'RPyThreadYield',
+                      'RPyGilAllocate', 'RPyGilYieldThread',
+                      'RPyGilRelease', 'RPyGilAcquire',
                       'RPyThreadGetStackSize', 'RPyThreadSetStackSize',
                       'RPyOpaqueDealloc_ThreadLock',
                       'RPyThreadAfterFork']
@@ -69,8 +70,16 @@
                                          [TLOCKP], lltype.Void,
                                          _nowrapper=True)
 
-# this function does nothing apart from releasing the GIL temporarily.
-yield_thread = llexternal('RPyThreadYield', [], lltype.Void, threadsafe=True)
+# these functions manipulate directly the GIL, whose definition does not
+# escape the C code itself
+gil_allocate     = llexternal('RPyGilAllocate', [], lltype.Signed,
+                              _nowrapper=True)
+gil_yield_thread = llexternal('RPyGilYieldThread', [], lltype.Signed,
+                              _nowrapper=True)
+gil_release      = llexternal('RPyGilRelease', [], lltype.Void,
+                              _nowrapper=True)
+gil_acquire      = llexternal('RPyGilAcquire', [], lltype.Void,
+                              _nowrapper=True)
 
 def allocate_lock():
     return Lock(allocate_ll_lock())
diff --git a/pypy/module/thread/test/test_gil.py b/pypy/module/thread/test/test_gil.py
--- a/pypy/module/thread/test/test_gil.py
+++ b/pypy/module/thread/test/test_gil.py
@@ -30,19 +30,34 @@
     use_threads = True
     bigtest = False
 
-    def test_one_thread(self):
+    def test_one_thread(self, skew=+1):
+        from pypy.rlib.debug import debug_print
         if self.bigtest:
-            N = 1000000
+            N = 100000
+            skew *= 25000
         else:
             N = 100
+            skew *= 25
         space = FakeSpace()
         class State:
             pass
         state = State()
-        def runme():
-            for i in range(N):
+        def runme(main=False):
+            j = 0
+            for i in range(N + [-skew, skew][main]):
+                state.datalen1 += 1   # try to crash if the GIL is not
+                state.datalen2 += 1   # correctly acquired
                 state.data.append((thread.get_ident(), i))
+                state.datalen3 += 1
+                state.datalen4 += 1
+                assert state.datalen1 == len(state.data)
+                assert state.datalen2 == len(state.data)
+                assert state.datalen3 == len(state.data)
+                assert state.datalen4 == len(state.data)
+                debug_print(main, i, state.datalen4)
                 state.threadlocals.yield_thread()
+                assert i == j
+                j += 1
         def bootstrap():
             try:
                 runme()
@@ -50,20 +65,26 @@
                 thread.gc_thread_die()
         def f():
             state.data = []
+            state.datalen1 = 0
+            state.datalen2 = 0
+            state.datalen3 = 0
+            state.datalen4 = 0
             state.threadlocals = gil.GILThreadLocals()
             state.threadlocals.setup_threads(space)
             thread.gc_thread_prepare()
             subident = thread.start_new_thread(bootstrap, ())
             mainident = thread.get_ident()
-            runme()
+            runme(True)
             still_waiting = 3000
             while len(state.data) < 2*N:
+                debug_print(len(state.data))
                 if not still_waiting:
                     raise ValueError("time out")
                 still_waiting -= 1
                 if not we_are_translated(): gil.before_external_call()
                 time.sleep(0.01)
                 if not we_are_translated(): gil.after_external_call()
+            debug_print("leaving!")
             i1 = i2 = 0
             for tid, i in state.data:
                 if tid == mainident:
@@ -72,14 +93,17 @@
                     assert i == i2; i2 += 1
                 else:
                     assert 0
-            assert i1 == N
-            assert i2 == N
+            assert i1 == N + skew
+            assert i2 == N - skew
             return len(state.data)
 
         fn = self.getcompiled(f, [])
         res = fn()
         assert res == 2*N
 
+    def test_one_thread_rev(self):
+        self.test_one_thread(skew=-1)
+
 
 class TestRunDirectly(GILTests):
     def getcompiled(self, f, argtypes):
diff --git a/pypy/module/thread/test/test_thread.py b/pypy/module/thread/test/test_thread.py
--- a/pypy/module/thread/test/test_thread.py
+++ b/pypy/module/thread/test/test_thread.py
@@ -225,7 +225,8 @@
 
         def busy_wait():
             for x in range(1000):
-                time.sleep(0.01)
+                print 'tick...', x  # <-force the GIL to be released, as
+                time.sleep(0.01)    #   time.sleep doesn't do non-translated
 
         # This is normally called by app_main.py
         signal.signal(signal.SIGINT, signal.default_int_handler)
diff --git a/pypy/translator/c/gcc/trackgcroot.py b/pypy/translator/c/gcc/trackgcroot.py
--- a/pypy/translator/c/gcc/trackgcroot.py
+++ b/pypy/translator/c/gcc/trackgcroot.py
@@ -486,6 +486,8 @@
         'paddq', 'pinsr',
         # zero-extending moves should not produce GC pointers
         'movz', 
+        # locked operations should not move GC pointers, at least so far
+        'lock',
         ])
 
     # a partial list is hopefully good enough for now; it's all to support
diff --git a/pypy/translator/c/src/thread.h b/pypy/translator/c/src/thread.h
--- a/pypy/translator/c/src/thread.h
+++ b/pypy/translator/c/src/thread.h
@@ -37,14 +37,9 @@
 
 #endif
 
-/* common helper: this does nothing, but is called with the GIL released.
-   This gives other threads a chance to grab the GIL and run. */
-void RPyThreadYield(void);
-
-#ifndef PYPY_NOT_MAIN_FILE
-void RPyThreadYield(void)
-{
-}
-#endif
+long RPyGilAllocate(void);
+long RPyGilYieldThread(void);
+void RPyGilRelease(void);
+void RPyGilAcquire(void);
 
 #endif
diff --git a/pypy/translator/c/src/thread_nt.h b/pypy/translator/c/src/thread_nt.h
--- a/pypy/translator/c/src/thread_nt.h
+++ b/pypy/translator/c/src/thread_nt.h
@@ -221,4 +221,57 @@
 #define RPyThreadTLS_Set(key, value)	TlsSetValue(key, value)
 
 
+/************************************************************/
+/* GIL code                                                 */
+/************************************************************/
+
+static volatile LONG pending_acquires = -1;
+static CRITICAL_SECTION mutex_gil;
+static HANDLE cond_gil;
+
+long RPyGilAllocate(void)
+{
+    pending_acquires = 0;
+    InitializeCriticalSection(&mutex_gil);
+    EnterCriticalSection(&mutex_gil);
+    cond_gil = CreateEvent (NULL, FALSE, FALSE, NULL);
+    return 1;
+}
+
+long RPyGilYieldThread(void)
+{
+    /* can be called even before RPyGilAllocate(), but in this case,
+       pending_acquires will be -1 */
+    if (pending_acquires <= 0)
+        return 0;
+    InterlockedIncrement(&pending_acquires);
+    PulseEvent(&cond_gil);
+
+    /* hack: the three following lines do a pthread_cond_wait(), and
+       normally specifying a timeout of INFINITE would be fine.  But the
+       first and second operations are not done atomically, so there is a
+       (small) risk that PulseEvent misses the WaitForSingleObject().
+       In this case the process will just sleep a few milliseconds. */
+    LeaveCriticalSection(&mutex_gil);
+    WaitForSingleObject(&cond_gil, 15);
+    EnterCriticalSection(&mutex_gil);
+
+    InterlockedDecrement(&pending_acquires);
+    return 1;
+}
+
+void RPyGilRelease(void)
+{
+    LeaveCriticalSection(&mutex_gil);
+    PulseEvent(&cond_gil);
+}
+
+void RPyGilAcquire(void)
+{
+    InterlockedIncrement(&pending_acquires);
+    EnterCriticalSection(&mutex_gil);
+    InterlockedDecrement(&pending_acquires);
+}
+
+
 #endif /* PYPY_NOT_MAIN_FILE */
diff --git a/pypy/translator/c/src/thread_pthread.h b/pypy/translator/c/src/thread_pthread.h
--- a/pypy/translator/c/src/thread_pthread.h
+++ b/pypy/translator/c/src/thread_pthread.h
@@ -12,6 +12,7 @@
 #include <signal.h>
 #include <stdio.h>
 #include <errno.h>
+#include <assert.h>
 
 /* The following is hopefully equivalent to what CPython does
    (which is trying to compile a snippet of code using it) */
@@ -459,4 +460,113 @@
 #define RPyThreadTLS_Set(key, value)	pthread_setspecific(key, value)
 
 
+/************************************************************/
+/* GIL code                                                 */
+/************************************************************/
+
+#ifdef __llvm__
+#  define HAS_ATOMIC_ADD
+#endif
+
+#ifdef __GNUC__
+#  if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 1)
+#    define HAS_ATOMIC_ADD
+#  endif
+#endif
+
+#ifdef HAS_ATOMIC_ADD
+#  define atomic_add __sync_fetch_and_add
+#else
+#  if defined(__amd64__)
+#    define atomic_add(ptr, value)  asm volatile ("lock addq %0, %1"        \
+                                 : : "ri"(value), "m"(*(ptr)) : "memory")
+#  elif defined(__i386__)
+#    define atomic_add(ptr, value)  asm volatile ("lock addl %0, %1"        \
+                                 : : "ri"(value), "m"(*(ptr)) : "memory")
+#  else
+#    error "Please use gcc >= 4.1 or write a custom 'asm' for your CPU."
+#  endif
+#endif
+
+#define ASSERT_STATUS(call)                             \
+    if (call != 0) {                                    \
+        fprintf(stderr, "Fatal error: " #call "\n");    \
+        abort();                                        \
+    }
+
+static void _debug_print(const char *msg)
+{
+#if 0
+    int col = (int)pthread_self();
+    col = 31 + ((col / 8) % 8);
+    fprintf(stderr, "\033[%dm%s\033[0m", col, msg);
+#endif
+}
+
+static volatile long pending_acquires = -1;
+static pthread_mutex_t mutex_gil = PTHREAD_MUTEX_INITIALIZER;
+static pthread_cond_t cond_gil = PTHREAD_COND_INITIALIZER;
+
+static void assert_has_the_gil(void)
+{
+#ifdef RPY_ASSERT
+    assert(pthread_mutex_trylock(&mutex_gil) != 0);
+    assert(pending_acquires >= 0);
+#endif
+}
+
+long RPyGilAllocate(void)
+{
+    _debug_print("RPyGilAllocate\n");
+    pending_acquires = 0;
+    pthread_mutex_trylock(&mutex_gil);
+    assert_has_the_gil();
+    return 1;
+}
+
+long RPyGilYieldThread(void)
+{
+    /* can be called even before RPyGilAllocate(), but in this case,
+       pending_acquires will be -1 */
+#ifdef RPY_ASSERT
+    if (pending_acquires >= 0)
+        assert_has_the_gil();
+#endif
+    if (pending_acquires <= 0)
+        return 0;
+    atomic_add(&pending_acquires, 1L);
+    _debug_print("{");
+    ASSERT_STATUS(pthread_cond_signal(&cond_gil));
+    ASSERT_STATUS(pthread_cond_wait(&cond_gil, &mutex_gil));
+    _debug_print("}");
+    atomic_add(&pending_acquires, -1L);
+    assert_has_the_gil();
+    return 1;
+}
+
+void RPyGilRelease(void)
+{
+    _debug_print("RPyGilRelease\n");
+#ifdef RPY_ASSERT
+    assert(pending_acquires >= 0);
+#endif
+    assert_has_the_gil();
+    ASSERT_STATUS(pthread_mutex_unlock(&mutex_gil));
+    ASSERT_STATUS(pthread_cond_signal(&cond_gil));
+}
+
+void RPyGilAcquire(void)
+{
+    _debug_print("about to RPyGilAcquire...\n");
+#ifdef RPY_ASSERT
+    assert(pending_acquires >= 0);
+#endif
+    atomic_add(&pending_acquires, 1L);
+    ASSERT_STATUS(pthread_mutex_lock(&mutex_gil));
+    atomic_add(&pending_acquires, -1L);
+    assert_has_the_gil();
+    _debug_print("RPyGilAcquire\n");
+}
+
+
 #endif /* PYPY_NOT_MAIN_FILE */

From noreply at buildbot.pypy.org  Mon Oct  3 21:06:47 2011
From: noreply at buildbot.pypy.org (hakanardo)
Date: Mon,  3 Oct 2011 21:06:47 +0200 (CEST)
Subject: [pypy-commit] pypy jit-optimizeopt-cleanups: hg merge default
Message-ID: <20111003190647.5D536820D9@wyvern.cs.uni-duesseldorf.de>

Author: Hakan Ardo <hakan at debian.org>
Branch: jit-optimizeopt-cleanups
Changeset: r47801:58bdbfef9178
Date: 2011-10-03 20:49 +0200
http://bitbucket.org/pypy/pypy/changeset/58bdbfef9178/

Log:	hg merge default

diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -307,7 +307,11 @@
         self._nonperiodic_actions = []
         self.has_bytecode_counter = False
         self.fired_actions = None
-        self.checkinterval_scaled = 100 * TICK_COUNTER_STEP
+        # the default value is not 100, unlike CPython 2.7, but a much
+        # larger value, because we use a technique that not only allows
+        # but actually *forces* another thread to run whenever the counter
+        # reaches zero.
+        self.checkinterval_scaled = 10000 * TICK_COUNTER_STEP
         self._rebuild_action_dispatcher()
 
     def fire(self, action):
diff --git a/pypy/module/thread/gil.py b/pypy/module/thread/gil.py
--- a/pypy/module/thread/gil.py
+++ b/pypy/module/thread/gil.py
@@ -16,7 +16,7 @@
 
 class GILThreadLocals(OSThreadLocals):
     """A version of OSThreadLocals that enforces a GIL."""
-    ll_GIL = thread.null_ll_lock
+    gil_ready = False
 
     def initialize(self, space):
         # add the GIL-releasing callback as an action on the space
@@ -25,12 +25,10 @@
 
     def setup_threads(self, space):
         """Enable threads in the object space, if they haven't already been."""
-        if not self.ll_GIL:
-            try:
-                self.ll_GIL = thread.allocate_ll_lock()
-            except thread.error:
+        if not self.gil_ready:
+            if not thread.gil_allocate():
                 raise wrap_thread_error(space, "can't allocate GIL")
-            thread.acquire_NOAUTO(self.ll_GIL, True)
+            self.gil_ready = True
             self.enter_thread(space)   # setup the main thread
             result = True
         else:
@@ -44,19 +42,16 @@
         # test_lock_again after the global state was cleared by
         # test_compile_lock.  As a workaround, we repatch these global
         # fields systematically.
-        spacestate.ll_GIL = self.ll_GIL
         invoke_around_extcall(before_external_call, after_external_call)
         return result
 
     def reinit_threads(self, space):
-        if self.ll_GIL:
-            self.ll_GIL = thread.allocate_ll_lock()
-            thread.acquire_NOAUTO(self.ll_GIL, True)
-            self.enter_thread(space)
+        if self.gil_ready:
+            self.gil_ready = False
+            self.setup_threads(space)
 
     def yield_thread(self):
-        thread.yield_thread()  # explicitly release the gil (used by test_gil)
-
+        do_yield_thread()
 
 class GILReleaseAction(PeriodicAsyncAction):
     """An action called every sys.checkinterval bytecodes.  It releases
@@ -64,16 +59,12 @@
     """
 
     def perform(self, executioncontext, frame):
-        # Other threads can run between the release() and the acquire()
-        # implicit in the following external function call (which has
-        # otherwise no effect).
-        thread.yield_thread()
+        do_yield_thread()
 
 
 class SpaceState:
 
     def _freeze_(self):
-        self.ll_GIL = thread.null_ll_lock
         self.action_after_thread_switch = None
         # ^^^ set by AsyncAction.fire_after_thread_switch()
         return False
@@ -95,14 +86,14 @@
     # this function must not raise, in such a way that the exception
     # transformer knows that it cannot raise!
     e = get_errno()
-    thread.release_NOAUTO(spacestate.ll_GIL)
+    thread.gil_release()
     set_errno(e)
 before_external_call._gctransformer_hint_cannot_collect_ = True
 before_external_call._dont_reach_me_in_del_ = True
 
 def after_external_call():
     e = get_errno()
-    thread.acquire_NOAUTO(spacestate.ll_GIL, True)
+    thread.gil_acquire()
     thread.gc_thread_run()
     spacestate.after_thread_switch()
     set_errno(e)
@@ -115,3 +106,18 @@
 # pointers in the shadow stack.  This is necessary because the GIL is
 # not held after the call to before_external_call() or before the call
 # to after_external_call().
+
+def do_yield_thread():
+    # explicitly release the gil, in a way that tries to give more
+    # priority to other threads (as opposed to continuing to run in
+    # the same thread).
+    if thread.gil_yield_thread():
+        thread.gc_thread_run()
+        spacestate.after_thread_switch()
+do_yield_thread._gctransformer_hint_close_stack_ = True
+do_yield_thread._dont_reach_me_in_del_ = True
+do_yield_thread._dont_inline_ = True
+
+# do_yield_thread() needs a different hint: _gctransformer_hint_close_stack_.
+# The *_external_call() functions are themselves called only from the rffi
+# module from a helper function that also has this hint.
diff --git a/pypy/module/thread/ll_thread.py b/pypy/module/thread/ll_thread.py
--- a/pypy/module/thread/ll_thread.py
+++ b/pypy/module/thread/ll_thread.py
@@ -17,7 +17,8 @@
     include_dirs = [str(py.path.local(autopath.pypydir).join('translator', 'c'))],
     export_symbols = ['RPyThreadGetIdent', 'RPyThreadLockInit',
                       'RPyThreadAcquireLock', 'RPyThreadReleaseLock',
-                      'RPyThreadYield',
+                      'RPyGilAllocate', 'RPyGilYieldThread',
+                      'RPyGilRelease', 'RPyGilAcquire',
                       'RPyThreadGetStackSize', 'RPyThreadSetStackSize',
                       'RPyOpaqueDealloc_ThreadLock',
                       'RPyThreadAfterFork']
@@ -69,8 +70,16 @@
                                          [TLOCKP], lltype.Void,
                                          _nowrapper=True)
 
-# this function does nothing apart from releasing the GIL temporarily.
-yield_thread = llexternal('RPyThreadYield', [], lltype.Void, threadsafe=True)
+# these functions manipulate directly the GIL, whose definition does not
+# escape the C code itself
+gil_allocate     = llexternal('RPyGilAllocate', [], lltype.Signed,
+                              _nowrapper=True)
+gil_yield_thread = llexternal('RPyGilYieldThread', [], lltype.Signed,
+                              _nowrapper=True)
+gil_release      = llexternal('RPyGilRelease', [], lltype.Void,
+                              _nowrapper=True)
+gil_acquire      = llexternal('RPyGilAcquire', [], lltype.Void,
+                              _nowrapper=True)
 
 def allocate_lock():
     return Lock(allocate_ll_lock())
diff --git a/pypy/module/thread/test/test_gil.py b/pypy/module/thread/test/test_gil.py
--- a/pypy/module/thread/test/test_gil.py
+++ b/pypy/module/thread/test/test_gil.py
@@ -30,19 +30,34 @@
     use_threads = True
     bigtest = False
 
-    def test_one_thread(self):
+    def test_one_thread(self, skew=+1):
+        from pypy.rlib.debug import debug_print
         if self.bigtest:
-            N = 1000000
+            N = 100000
+            skew *= 25000
         else:
             N = 100
+            skew *= 25
         space = FakeSpace()
         class State:
             pass
         state = State()
-        def runme():
-            for i in range(N):
+        def runme(main=False):
+            j = 0
+            for i in range(N + [-skew, skew][main]):
+                state.datalen1 += 1   # try to crash if the GIL is not
+                state.datalen2 += 1   # correctly acquired
                 state.data.append((thread.get_ident(), i))
+                state.datalen3 += 1
+                state.datalen4 += 1
+                assert state.datalen1 == len(state.data)
+                assert state.datalen2 == len(state.data)
+                assert state.datalen3 == len(state.data)
+                assert state.datalen4 == len(state.data)
+                debug_print(main, i, state.datalen4)
                 state.threadlocals.yield_thread()
+                assert i == j
+                j += 1
         def bootstrap():
             try:
                 runme()
@@ -50,20 +65,26 @@
                 thread.gc_thread_die()
         def f():
             state.data = []
+            state.datalen1 = 0
+            state.datalen2 = 0
+            state.datalen3 = 0
+            state.datalen4 = 0
             state.threadlocals = gil.GILThreadLocals()
             state.threadlocals.setup_threads(space)
             thread.gc_thread_prepare()
             subident = thread.start_new_thread(bootstrap, ())
             mainident = thread.get_ident()
-            runme()
+            runme(True)
             still_waiting = 3000
             while len(state.data) < 2*N:
+                debug_print(len(state.data))
                 if not still_waiting:
                     raise ValueError("time out")
                 still_waiting -= 1
                 if not we_are_translated(): gil.before_external_call()
                 time.sleep(0.01)
                 if not we_are_translated(): gil.after_external_call()
+            debug_print("leaving!")
             i1 = i2 = 0
             for tid, i in state.data:
                 if tid == mainident:
@@ -72,14 +93,17 @@
                     assert i == i2; i2 += 1
                 else:
                     assert 0
-            assert i1 == N
-            assert i2 == N
+            assert i1 == N + skew
+            assert i2 == N - skew
             return len(state.data)
 
         fn = self.getcompiled(f, [])
         res = fn()
         assert res == 2*N
 
+    def test_one_thread_rev(self):
+        self.test_one_thread(skew=-1)
+
 
 class TestRunDirectly(GILTests):
     def getcompiled(self, f, argtypes):
diff --git a/pypy/module/thread/test/test_thread.py b/pypy/module/thread/test/test_thread.py
--- a/pypy/module/thread/test/test_thread.py
+++ b/pypy/module/thread/test/test_thread.py
@@ -225,7 +225,8 @@
 
         def busy_wait():
             for x in range(1000):
-                time.sleep(0.01)
+                print 'tick...', x  # <-force the GIL to be released, as
+                time.sleep(0.01)    #   time.sleep doesn't do non-translated
 
         # This is normally called by app_main.py
         signal.signal(signal.SIGINT, signal.default_int_handler)
diff --git a/pypy/translator/c/gcc/trackgcroot.py b/pypy/translator/c/gcc/trackgcroot.py
--- a/pypy/translator/c/gcc/trackgcroot.py
+++ b/pypy/translator/c/gcc/trackgcroot.py
@@ -486,6 +486,8 @@
         'paddq', 'pinsr',
         # zero-extending moves should not produce GC pointers
         'movz', 
+        # locked operations should not move GC pointers, at least so far
+        'lock',
         ])
 
     # a partial list is hopefully good enough for now; it's all to support
diff --git a/pypy/translator/c/src/thread.h b/pypy/translator/c/src/thread.h
--- a/pypy/translator/c/src/thread.h
+++ b/pypy/translator/c/src/thread.h
@@ -37,14 +37,9 @@
 
 #endif
 
-/* common helper: this does nothing, but is called with the GIL released.
-   This gives other threads a chance to grab the GIL and run. */
-void RPyThreadYield(void);
-
-#ifndef PYPY_NOT_MAIN_FILE
-void RPyThreadYield(void)
-{
-}
-#endif
+long RPyGilAllocate(void);
+long RPyGilYieldThread(void);
+void RPyGilRelease(void);
+void RPyGilAcquire(void);
 
 #endif
diff --git a/pypy/translator/c/src/thread_nt.h b/pypy/translator/c/src/thread_nt.h
--- a/pypy/translator/c/src/thread_nt.h
+++ b/pypy/translator/c/src/thread_nt.h
@@ -221,4 +221,57 @@
 #define RPyThreadTLS_Set(key, value)	TlsSetValue(key, value)
 
 
+/************************************************************/
+/* GIL code                                                 */
+/************************************************************/
+
+static volatile LONG pending_acquires = -1;
+static CRITICAL_SECTION mutex_gil;
+static HANDLE cond_gil;
+
+long RPyGilAllocate(void)
+{
+    pending_acquires = 0;
+    InitializeCriticalSection(&mutex_gil);
+    EnterCriticalSection(&mutex_gil);
+    cond_gil = CreateEvent (NULL, FALSE, FALSE, NULL);
+    return 1;
+}
+
+long RPyGilYieldThread(void)
+{
+    /* can be called even before RPyGilAllocate(), but in this case,
+       pending_acquires will be -1 */
+    if (pending_acquires <= 0)
+        return 0;
+    InterlockedIncrement(&pending_acquires);
+    PulseEvent(&cond_gil);
+
+    /* hack: the three following lines do a pthread_cond_wait(), and
+       normally specifying a timeout of INFINITE would be fine.  But the
+       first and second operations are not done atomically, so there is a
+       (small) risk that PulseEvent misses the WaitForSingleObject().
+       In this case the process will just sleep a few milliseconds. */
+    LeaveCriticalSection(&mutex_gil);
+    WaitForSingleObject(&cond_gil, 15);
+    EnterCriticalSection(&mutex_gil);
+
+    InterlockedDecrement(&pending_acquires);
+    return 1;
+}
+
+void RPyGilRelease(void)
+{
+    LeaveCriticalSection(&mutex_gil);
+    PulseEvent(&cond_gil);
+}
+
+void RPyGilAcquire(void)
+{
+    InterlockedIncrement(&pending_acquires);
+    EnterCriticalSection(&mutex_gil);
+    InterlockedDecrement(&pending_acquires);
+}
+
+
 #endif /* PYPY_NOT_MAIN_FILE */
diff --git a/pypy/translator/c/src/thread_pthread.h b/pypy/translator/c/src/thread_pthread.h
--- a/pypy/translator/c/src/thread_pthread.h
+++ b/pypy/translator/c/src/thread_pthread.h
@@ -12,6 +12,7 @@
 #include <signal.h>
 #include <stdio.h>
 #include <errno.h>
+#include <assert.h>
 
 /* The following is hopefully equivalent to what CPython does
    (which is trying to compile a snippet of code using it) */
@@ -459,4 +460,113 @@
 #define RPyThreadTLS_Set(key, value)	pthread_setspecific(key, value)
 
 
+/************************************************************/
+/* GIL code                                                 */
+/************************************************************/
+
+#ifdef __llvm__
+#  define HAS_ATOMIC_ADD
+#endif
+
+#ifdef __GNUC__
+#  if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 1)
+#    define HAS_ATOMIC_ADD
+#  endif
+#endif
+
+#ifdef HAS_ATOMIC_ADD
+#  define atomic_add __sync_fetch_and_add
+#else
+#  if defined(__amd64__)
+#    define atomic_add(ptr, value)  asm volatile ("lock addq %0, %1"        \
+                                 : : "ri"(value), "m"(*(ptr)) : "memory")
+#  elif defined(__i386__)
+#    define atomic_add(ptr, value)  asm volatile ("lock addl %0, %1"        \
+                                 : : "ri"(value), "m"(*(ptr)) : "memory")
+#  else
+#    error "Please use gcc >= 4.1 or write a custom 'asm' for your CPU."
+#  endif
+#endif
+
+#define ASSERT_STATUS(call)                             \
+    if (call != 0) {                                    \
+        fprintf(stderr, "Fatal error: " #call "\n");    \
+        abort();                                        \
+    }
+
+static void _debug_print(const char *msg)
+{
+#if 0
+    int col = (int)pthread_self();
+    col = 31 + ((col / 8) % 8);
+    fprintf(stderr, "\033[%dm%s\033[0m", col, msg);
+#endif
+}
+
+static volatile long pending_acquires = -1;
+static pthread_mutex_t mutex_gil = PTHREAD_MUTEX_INITIALIZER;
+static pthread_cond_t cond_gil = PTHREAD_COND_INITIALIZER;
+
+static void assert_has_the_gil(void)
+{
+#ifdef RPY_ASSERT
+    assert(pthread_mutex_trylock(&mutex_gil) != 0);
+    assert(pending_acquires >= 0);
+#endif
+}
+
+long RPyGilAllocate(void)
+{
+    _debug_print("RPyGilAllocate\n");
+    pending_acquires = 0;
+    pthread_mutex_trylock(&mutex_gil);
+    assert_has_the_gil();
+    return 1;
+}
+
+long RPyGilYieldThread(void)
+{
+    /* can be called even before RPyGilAllocate(), but in this case,
+       pending_acquires will be -1 */
+#ifdef RPY_ASSERT
+    if (pending_acquires >= 0)
+        assert_has_the_gil();
+#endif
+    if (pending_acquires <= 0)
+        return 0;
+    atomic_add(&pending_acquires, 1L);
+    _debug_print("{");
+    ASSERT_STATUS(pthread_cond_signal(&cond_gil));
+    ASSERT_STATUS(pthread_cond_wait(&cond_gil, &mutex_gil));
+    _debug_print("}");
+    atomic_add(&pending_acquires, -1L);
+    assert_has_the_gil();
+    return 1;
+}
+
+void RPyGilRelease(void)
+{
+    _debug_print("RPyGilRelease\n");
+#ifdef RPY_ASSERT
+    assert(pending_acquires >= 0);
+#endif
+    assert_has_the_gil();
+    ASSERT_STATUS(pthread_mutex_unlock(&mutex_gil));
+    ASSERT_STATUS(pthread_cond_signal(&cond_gil));
+}
+
+void RPyGilAcquire(void)
+{
+    _debug_print("about to RPyGilAcquire...\n");
+#ifdef RPY_ASSERT
+    assert(pending_acquires >= 0);
+#endif
+    atomic_add(&pending_acquires, 1L);
+    ASSERT_STATUS(pthread_mutex_lock(&mutex_gil));
+    atomic_add(&pending_acquires, -1L);
+    assert_has_the_gil();
+    _debug_print("RPyGilAcquire\n");
+}
+
+
 #endif /* PYPY_NOT_MAIN_FILE */

From noreply at buildbot.pypy.org  Mon Oct  3 21:06:48 2011
From: noreply at buildbot.pypy.org (hakanardo)
Date: Mon,  3 Oct 2011 21:06:48 +0200 (CEST)
Subject: [pypy-commit] pypy jit-optimizeopt-cleanups: closing branch for
	merge
Message-ID: <20111003190648.95010820D9@wyvern.cs.uni-duesseldorf.de>

Author: Hakan Ardo <hakan at debian.org>
Branch: jit-optimizeopt-cleanups
Changeset: r47802:07785846e0ad
Date: 2011-10-03 20:49 +0200
http://bitbucket.org/pypy/pypy/changeset/07785846e0ad/

Log:	closing branch for merge


From noreply at buildbot.pypy.org  Mon Oct  3 21:06:50 2011
From: noreply at buildbot.pypy.org (hakanardo)
Date: Mon,  3 Oct 2011 21:06:50 +0200 (CEST)
Subject: [pypy-commit] pypy default: Merging jit-optimizeopt-cleanups. It
 makes the the different
Message-ID: <20111003190650.2D99B820D9@wyvern.cs.uni-duesseldorf.de>

Author: Hakan Ardo <hakan at debian.org>
Branch: 
Changeset: r47803:f79884ba173a
Date: 2011-10-03 21:05 +0200
http://bitbucket.org/pypy/pypy/changeset/f79884ba173a/

Log:	Merging jit-optimizeopt-cleanups. It makes the the different
	Optimization stages of optimizeopt more stricly follow the model of
	emitting operations to the next stage instead of manipulating
	Optimizer.newoperations directly. This means fewer optimizations are
	prevented due to unwanted interactions between the different stages
	(for example test_remove_duplicate_pure_op_ovf_with_lazy_setfield).

	Operations produced by forcing boxes are emitted by the stage doing
	the forcing. Two new Optimization stages have been intorduced,
	OptPure and OptEarlyForce. OptEarlyForce forces things early in the
	chain and thus allows the following stages to work on the ops
	produced (moste notable the OptHeap and OptPure stages).

	Operations produced as an effect of forincing a lazy setfiled are
	still not seen by OptHeap. The main task remaining there is to
	refactor OptHeap to make it able to recieve new setfield operations
	during the execution of force_lazy_XXX, force_from_effectinfo,
	force_all_lazy_XXX, ...

diff --git a/pypy/jit/metainterp/optimizeopt/__init__.py b/pypy/jit/metainterp/optimizeopt/__init__.py
--- a/pypy/jit/metainterp/optimizeopt/__init__.py
+++ b/pypy/jit/metainterp/optimizeopt/__init__.py
@@ -7,6 +7,8 @@
 from pypy.jit.metainterp.optimizeopt.unroll import optimize_unroll, OptInlineShortPreamble
 from pypy.jit.metainterp.optimizeopt.fficall import OptFfiCall
 from pypy.jit.metainterp.optimizeopt.simplify import OptSimplify
+from pypy.jit.metainterp.optimizeopt.pure import OptPure
+from pypy.jit.metainterp.optimizeopt.earlyforce import OptEarlyForce
 from pypy.rlib.jit import PARAMETERS
 from pypy.rlib.unroll import unrolling_iterable
 
@@ -14,6 +16,8 @@
             ('rewrite', OptRewrite),
             ('virtualize', OptVirtualize),
             ('string', OptString),
+            ('earlyforce', OptEarlyForce),
+            ('pure', OptPure),
             ('heap', OptHeap),
             ('ffi', None),
             ('unroll', None)]
diff --git a/pypy/jit/metainterp/optimizeopt/earlyforce.py b/pypy/jit/metainterp/optimizeopt/earlyforce.py
new file mode 100644
--- /dev/null
+++ b/pypy/jit/metainterp/optimizeopt/earlyforce.py
@@ -0,0 +1,24 @@
+from pypy.jit.metainterp.optimizeopt.optimizer import Optimization
+from pypy.jit.metainterp.optimizeopt.vstring import VAbstractStringValue
+from pypy.jit.metainterp.resoperation import rop, ResOperation
+
+class OptEarlyForce(Optimization):
+    def propagate_forward(self, op):
+        opnum = op.getopnum()
+        if (opnum != rop.SETFIELD_GC and 
+            opnum != rop.SETARRAYITEM_GC and
+            opnum != rop.QUASIIMMUT_FIELD):
+               
+            for arg in op.getarglist():
+                if arg in self.optimizer.values:
+                    value = self.getvalue(arg)
+                    value.force_box(self)
+        self.emit_operation(op)
+
+    def new(self):
+        return OptEarlyForce()
+
+    def setup(self):
+        self.optimizer.optearlyforce = self
+
+    
diff --git a/pypy/jit/metainterp/optimizeopt/fficall.py b/pypy/jit/metainterp/optimizeopt/fficall.py
--- a/pypy/jit/metainterp/optimizeopt/fficall.py
+++ b/pypy/jit/metainterp/optimizeopt/fficall.py
@@ -177,10 +177,10 @@
             funcinfo.descr is None):
             return [op] # cannot optimize
         funcsymval = self.getvalue(op.getarg(2))
-        arglist = [funcsymval.force_box()]
+        arglist = [funcsymval.get_key_box()]
         for push_op in funcinfo.opargs:
             argval = self.getvalue(push_op.getarg(2))
-            arglist.append(argval.force_box())
+            arglist.append(argval.get_key_box())
         newop = ResOperation(rop.CALL_RELEASE_GIL, arglist, op.result,
                              descr=funcinfo.descr)
         self.commit_optimization()
diff --git a/pypy/jit/metainterp/optimizeopt/heap.py b/pypy/jit/metainterp/optimizeopt/heap.py
--- a/pypy/jit/metainterp/optimizeopt/heap.py
+++ b/pypy/jit/metainterp/optimizeopt/heap.py
@@ -50,6 +50,7 @@
             if not self._lazy_setfield_registered:
                 optheap._lazy_setfields_and_arrayitems.append(self)
                 self._lazy_setfield_registered = True
+            
         else:
             # this is the case where the pending setfield ends up
             # storing precisely the value that is already there,
@@ -158,12 +159,17 @@
         self._lazy_setfields_and_arrayitems = []
         self._remove_guard_not_invalidated = False
         self._seen_guard_not_invalidated = False
+        self.posponedop = None
 
     def force_at_end_of_preamble(self):
         self.force_all_lazy_setfields_and_arrayitems()
 
     def flush(self):
         self.force_all_lazy_setfields_and_arrayitems()
+        if self.posponedop:
+            posponedop = self.posponedop
+            self.posponedop = None
+            self.next_optimization.propagate_forward(posponedop)
 
     def new(self):
         return OptHeap()
@@ -211,7 +217,15 @@
 
     def emit_operation(self, op):
         self.emitting_operation(op)
-        self.next_optimization.propagate_forward(op)
+        if self.posponedop:
+            posponedop = self.posponedop
+            self.posponedop = None
+            self.next_optimization.propagate_forward(posponedop)
+        if (op.is_comparison() or op.getopnum() == rop.CALL_MAY_FORCE
+            or op.is_ovf()):
+            self.posponedop = op
+        else:
+            self.next_optimization.propagate_forward(op)
 
     def emitting_operation(self, op):
         if op.has_no_side_effect():
@@ -293,30 +307,6 @@
             if indexvalue is None or indexvalue.intbound.contains(idx):
                 cf.force_lazy_setfield(self, can_cache)
 
-    def fixup_guard_situation(self):
-        # hackish: reverse the order of the last two operations if it makes
-        # sense to avoid a situation like "int_eq/setfield_gc/guard_true",
-        # which the backend (at least the x86 backend) does not handle well.
-        newoperations = self.optimizer.newoperations
-        if len(newoperations) < 2:
-            return
-        lastop = newoperations[-1]
-        if (lastop.getopnum() != rop.SETFIELD_GC and
-            lastop.getopnum() != rop.SETARRAYITEM_GC):
-            return
-        # - is_comparison() for cases like "int_eq/setfield_gc/guard_true"
-        # - CALL_MAY_FORCE: "call_may_force/setfield_gc/guard_not_forced"
-        # - is_ovf(): "int_add_ovf/setfield_gc/guard_no_overflow"
-        prevop = newoperations[-2]
-        opnum = prevop.getopnum()
-        if not (prevop.is_comparison() or opnum == rop.CALL_MAY_FORCE
-                or prevop.is_ovf()):
-            return
-        if prevop.result in lastop.getarglist():
-            return
-        newoperations[-2] = lastop
-        newoperations[-1] = prevop
-
     def _assert_valid_cf(self, cf):
         # check that 'cf' is in cached_fields or cached_arrayitems
         if not we_are_translated():
@@ -362,7 +352,6 @@
                                       fieldvalue.get_key_box(), itemindex))
             else:
                 cf.force_lazy_setfield(self)
-                self.fixup_guard_situation()
         return pendingfields
 
     def optimize_GETFIELD_GC(self, op):
@@ -374,12 +363,22 @@
             return
         # default case: produce the operation
         structvalue.ensure_nonnull()
-        ###self.optimizer.optimize_default(op)
         self.emit_operation(op)
         # then remember the result of reading the field
         fieldvalue = self.getvalue(op.result)
         cf.remember_field_value(structvalue, fieldvalue, op)
 
+    def optimize_GETFIELD_GC_PURE(self, op):
+        structvalue = self.getvalue(op.getarg(0))
+        cf = self.field_cache(op.getdescr())
+        fieldvalue = cf.getfield_from_cache(self, structvalue)
+        if fieldvalue is not None:
+            self.make_equal_to(op.result, fieldvalue)
+            return
+        # default case: produce the operation
+        structvalue.ensure_nonnull()
+        self.emit_operation(op)
+
     def optimize_SETFIELD_GC(self, op):
         if self.has_pure_result(rop.GETFIELD_GC_PURE, [op.getarg(0)],
                                 op.getdescr()):
@@ -389,6 +388,7 @@
         #
         cf = self.field_cache(op.getdescr())
         cf.do_setfield(self, op)
+        
 
     def optimize_GETARRAYITEM_GC(self, op):
         arrayvalue = self.getvalue(op.getarg(0))
@@ -413,6 +413,25 @@
             fieldvalue = self.getvalue(op.result)
             cf.remember_field_value(arrayvalue, fieldvalue, op)
 
+    def optimize_GETARRAYITEM_GC_PURE(self, op):
+        arrayvalue = self.getvalue(op.getarg(0))
+        indexvalue = self.getvalue(op.getarg(1))
+        cf = None
+        if indexvalue.is_constant():
+            arrayvalue.make_len_gt(MODE_ARRAY, op.getdescr(), indexvalue.box.getint())
+            # use the cache on (arraydescr, index), which is a constant
+            cf = self.arrayitem_cache(op.getdescr(), indexvalue.box.getint())
+            fieldvalue = cf.getfield_from_cache(self, arrayvalue)
+            if fieldvalue is not None:
+                self.make_equal_to(op.result, fieldvalue)
+                return
+        else:
+            # variable index, so make sure the lazy setarrayitems are done
+            self.force_lazy_setarrayitem(op.getdescr(), indexvalue=indexvalue)
+        # default case: produce the operation
+        arrayvalue.ensure_nonnull()
+        self.emit_operation(op)
+
     def optimize_SETARRAYITEM_GC(self, op):
         if self.has_pure_result(rop.GETARRAYITEM_GC_PURE, [op.getarg(0),
                                                            op.getarg(1)],
diff --git a/pypy/jit/metainterp/optimizeopt/optimizer.py b/pypy/jit/metainterp/optimizeopt/optimizer.py
--- a/pypy/jit/metainterp/optimizeopt/optimizer.py
+++ b/pypy/jit/metainterp/optimizeopt/optimizer.py
@@ -31,8 +31,8 @@
 
 class OptValue(object):
     __metaclass__ = extendabletype
-    _attrs_ = ('box', 'known_class', 'last_guard_index', 'level', 'intbound', 'lenbound')
-    last_guard_index = -1
+    _attrs_ = ('box', 'known_class', 'last_guard', 'level', 'intbound', 'lenbound')
+    last_guard = None
 
     level = LEVEL_UNKNOWN
     known_class = None
@@ -100,7 +100,7 @@
             self.make_constant(other.get_key_box())
             optimizer.turned_constant(self)
         elif other.level == LEVEL_KNOWNCLASS:
-            self.make_constant_class(other.known_class, -1)
+            self.make_constant_class(other.known_class, None)
         else:
             if other.level == LEVEL_NONNULL:
                 self.ensure_nonnull()
@@ -114,13 +114,13 @@
                     self.lenbound = other.lenbound.clone()
 
 
-    def force_box(self):
+    def force_box(self, optforce):
         return self.box
 
     def get_key_box(self):
         return self.box
 
-    def force_at_end_of_preamble(self, already_forced):
+    def force_at_end_of_preamble(self, already_forced, optforce):
         return self
 
     def get_args_for_fail(self, modifier):
@@ -162,16 +162,16 @@
         else:
             return None
 
-    def make_constant_class(self, classbox, opindex):
+    def make_constant_class(self, classbox, guardop):
         assert self.level < LEVEL_KNOWNCLASS
         self.known_class = classbox
         self.level = LEVEL_KNOWNCLASS
-        self.last_guard_index = opindex
+        self.last_guard = guardop
 
-    def make_nonnull(self, opindex):
+    def make_nonnull(self, guardop):
         assert self.level < LEVEL_NONNULL
         self.level = LEVEL_NONNULL
-        self.last_guard_index = opindex
+        self.last_guard = guardop
 
     def is_nonnull(self):
         level = self.level
@@ -240,26 +240,12 @@
     def __init__(self):
         pass # make rpython happy
 
-    def propagate_begin_forward(self):
-        if self.next_optimization:
-            self.next_optimization.propagate_begin_forward()
-
-    def propagate_end_forward(self):
-        if self.next_optimization:
-            self.next_optimization.propagate_end_forward()
-
     def propagate_forward(self, op):
         raise NotImplementedError
 
     def emit_operation(self, op):
         self.next_optimization.propagate_forward(op)
 
-    def test_emittable(self, op):
-        return self.is_emittable(op)
-
-    def is_emittable(self, op):
-        return self.next_optimization.test_emittable(op)
-
     # FIXME: Move some of these here?
     def getvalue(self, box):
         return self.optimizer.getvalue(box)
@@ -289,19 +275,19 @@
         return self.optimizer.new_const_item(arraydescr)
 
     def pure(self, opnum, args, result):
-        op = ResOperation(opnum, args, result)
-        key = self.optimizer.make_args_key(op)
-        if key not in self.optimizer.pure_operations:
-            self.optimizer.pure_operations[key] = op
+        if self.optimizer.optpure:
+            self.optimizer.optpure.pure(opnum, args, result)
 
     def has_pure_result(self, opnum, args, descr):
-        op = ResOperation(opnum, args, None, descr)
-        key = self.optimizer.make_args_key(op)
-        op = self.optimizer.pure_operations.get(key, None)
-        if op is None:
-            return False
-        return op.getdescr() is descr
+        if self.optimizer.optpure:
+            return self.optimizer.optpure.has_pure_result(opnum, args, descr)
+        return False
 
+    def get_pure_result(self, key):    
+        if self.optimizer.optpure:
+            return self.optimizer.optpure.get_pure_result(key)
+        return None
+    
     def setup(self):
         pass
 
@@ -322,6 +308,9 @@
     def produce_potential_short_preamble_ops(self, potential_ops):
         pass
 
+    def forget_numberings(self, box):
+        self.optimizer.forget_numberings(box)
+
 class Optimizer(Optimization):
 
     def __init__(self, metainterp_sd, loop, optimizations=None, bridge=False):
@@ -333,14 +322,16 @@
         self.interned_refs = self.cpu.ts.new_ref_dict()
         self.resumedata_memo = resume.ResumeDataLoopMemo(metainterp_sd)
         self.bool_boxes = {}
-        self.pure_operations = args_dict()
         self.producer = {}
         self.pendingfields = []
-        self.posponedop = None
         self.exception_might_have_happened = False
         self.quasi_immutable_deps = None
         self.opaque_pointers = {}
-        self.newoperations = []
+        self.replaces_guard = {}
+        self._newoperations = []
+        self.optimizer = self
+        self.optpure = None
+        self.optearlyforce = None
         if loop is not None:
             self.call_pure_results = loop.call_pure_results
 
@@ -369,21 +360,20 @@
     def flush(self):
         for o in self.optimizations:
             o.flush()
-        assert self.posponedop is None
 
     def new(self):
         new = Optimizer(self.metainterp_sd, self.loop)
         return self._new(new)
 
     def _new(self, new):
-        assert self.posponedop is None
         optimizations = [o.new() for o in self.optimizations]
         new.set_optimizations(optimizations)
         new.quasi_immutable_deps = self.quasi_immutable_deps
         return new
 
     def produce_potential_short_preamble_ops(self, sb):
-        raise NotImplementedError('This is implemented in unroll.UnrollableOptimizer')
+        for opt in self.optimizations:
+            opt.produce_potential_short_preamble_ops(sb)
 
     def turned_constant(self, value):
         for o in self.optimizations:
@@ -433,6 +423,13 @@
             return constbox
         return None
 
+    def get_newoperations(self):
+        self.flush()
+        return self._newoperations
+
+    def clear_newoperations(self):
+        self._newoperations = []
+
     def make_equal_to(self, box, value, replace=False):
         assert isinstance(value, OptValue)
         assert replace or box not in self.values
@@ -481,15 +478,10 @@
 
     def propagate_all_forward(self):
         self.exception_might_have_happened = self.bridge
-        self.newoperations = []
-        self.first_optimization.propagate_begin_forward()
-        self.i = 0
-        while self.i < len(self.loop.operations):
-            op = self.loop.operations[self.i]
+        self.clear_newoperations()
+        for op in self.loop.operations:
             self.first_optimization.propagate_forward(op)
-            self.i += 1
-        self.first_optimization.propagate_end_forward()
-        self.loop.operations = self.newoperations
+        self.loop.operations = self.get_newoperations()
         self.loop.quasi_immutable_deps = self.quasi_immutable_deps
         # accumulate counters
         self.resumedata_memo.update_counters(self.metainterp_sd.profiler)
@@ -501,9 +493,6 @@
         self.producer[op.result] = op
         dispatch_opt(self, op)
 
-    def test_emittable(self, op):
-        return True
-
     def emit_operation(self, op):
         if op.returns_bool_result():
             self.bool_boxes[self.getvalue(op.result)] = None
@@ -519,14 +508,30 @@
                 pass
             else:
                 self.ensure_imported(value)
-                op.setarg(i, value.force_box())
+                op.setarg(i, value.force_box(self))
         self.metainterp_sd.profiler.count(jitprof.OPT_OPS)
         if op.is_guard():
             self.metainterp_sd.profiler.count(jitprof.OPT_GUARDS)
-            op = self.store_final_boxes_in_guard(op)
+            if self.replaces_guard and op in self.replaces_guard:
+                self.replace_op(self.replaces_guard[op], op)
+                del self.replaces_guard[op]
+                return
+            else:
+                op = self.store_final_boxes_in_guard(op)
         elif op.can_raise():
             self.exception_might_have_happened = True
-        self.newoperations.append(op)
+        self._newoperations.append(op)
+
+    def replace_op(self, old_op, new_op):
+        # XXX: Do we want to cache indexes to prevent search?
+        i = len(self._newoperations) 
+        while i > 0:
+            i -= 1
+            if self._newoperations[i] is old_op:
+                self._newoperations[i] = new_op
+                break
+        else:
+            assert False
 
     def store_final_boxes_in_guard(self, op):
         descr = op.getdescr()
@@ -574,51 +579,8 @@
         args[n + 1] = op.getdescr()
         return args
 
-    @specialize.argtype(0)
     def optimize_default(self, op):
-        canfold = op.is_always_pure()
-        if op.is_ovf():
-            self.posponedop = op
-            return
-        if self.posponedop:
-            nextop = op
-            op = self.posponedop
-            self.posponedop = None
-            canfold = nextop.getopnum() == rop.GUARD_NO_OVERFLOW
-        else:
-            nextop = None
-
-        if canfold:
-            for i in range(op.numargs()):
-                if self.get_constant_box(op.getarg(i)) is None:
-                    break
-            else:
-                # all constant arguments: constant-fold away
-                resbox = self.constant_fold(op)
-                # note that INT_xxx_OVF is not done from here, and the
-                # overflows in the INT_xxx operations are ignored
-                self.make_constant(op.result, resbox)
-                return
-
-            # did we do the exact same operation already?
-            args = self.make_args_key(op)
-            oldop = self.pure_operations.get(args, None)
-            if oldop is not None and oldop.getdescr() is op.getdescr():
-                assert oldop.getopnum() == op.getopnum()
-                self.make_equal_to(op.result, self.getvalue(oldop.result),
-                                   True)
-                return
-            else:
-                self.pure_operations[args] = op
-                self.remember_emitting_pure(op)
-
-        # otherwise, the operation remains
         self.emit_operation(op)
-        if nextop:
-            self.emit_operation(nextop)
-
-    def remember_emitting_pure(self, op):
-        pass
 
     def constant_fold(self, op):
         argboxes = [self.get_constant_box(op.getarg(i))
@@ -636,11 +598,6 @@
     def optimize_DEBUG_MERGE_POINT(self, op):
         self.emit_operation(op)
 
-    def optimize_CAST_OPAQUE_PTR(self, op):
-        value = self.getvalue(op.getarg(0))
-        self.opaque_pointers[value] = True
-        self.make_equal_to(op.result, value)
-
     def optimize_GETARRAYITEM_GC_PURE(self, op):
         indexvalue = self.getvalue(op.getarg(1))
         if indexvalue.is_constant():
diff --git a/pypy/jit/metainterp/optimizeopt/pure.py b/pypy/jit/metainterp/optimizeopt/pure.py
new file mode 100644
--- /dev/null
+++ b/pypy/jit/metainterp/optimizeopt/pure.py
@@ -0,0 +1,115 @@
+from pypy.jit.metainterp.optimizeopt.optimizer import Optimization
+from pypy.jit.metainterp.resoperation import rop, ResOperation
+from pypy.jit.metainterp.optimizeopt.util import (make_dispatcher_method,
+    args_dict)
+
+class OptPure(Optimization):
+    def __init__(self):
+        self.posponedop = None
+        self.pure_operations = args_dict()
+        self.emitted_pure_operations = {}
+
+    def propagate_forward(self, op):
+        dispatch_opt(self, op)
+
+    def optimize_default(self, op):
+        canfold = op.is_always_pure()
+        if op.is_ovf():
+            self.posponedop = op
+            return
+        if self.posponedop:
+            nextop = op
+            op = self.posponedop
+            self.posponedop = None
+            canfold = nextop.getopnum() == rop.GUARD_NO_OVERFLOW
+        else:
+            nextop = None
+
+        if canfold:
+            for i in range(op.numargs()):
+                if self.get_constant_box(op.getarg(i)) is None:
+                    break
+            else:
+                # all constant arguments: constant-fold away
+                resbox = self.optimizer.constant_fold(op)
+                # note that INT_xxx_OVF is not done from here, and the
+                # overflows in the INT_xxx operations are ignored
+                self.optimizer.make_constant(op.result, resbox)
+                return
+
+            # did we do the exact same operation already?
+            args = self.optimizer.make_args_key(op)
+            oldop = self.pure_operations.get(args, None)
+            if oldop is not None and oldop.getdescr() is op.getdescr():
+                assert oldop.getopnum() == op.getopnum()
+                self.optimizer.make_equal_to(op.result, self.getvalue(oldop.result),
+                                   True)
+                return
+            else:
+                self.pure_operations[args] = op
+                self.remember_emitting_pure(op)
+
+        # otherwise, the operation remains
+        self.emit_operation(op)
+        if op.returns_bool_result():
+            self.optimizer.bool_boxes[self.getvalue(op.result)] = None        
+        if nextop:
+            self.emit_operation(nextop)
+
+    def optimize_CALL_PURE(self, op):
+        args = self.optimizer.make_args_key(op)
+        oldop = self.pure_operations.get(args, None)
+        if oldop is not None and oldop.getdescr() is op.getdescr():
+            assert oldop.getopnum() == op.getopnum()
+            self.make_equal_to(op.result, self.getvalue(oldop.result))
+            return
+        else:
+            self.pure_operations[args] = op
+            self.remember_emitting_pure(op)
+
+        # replace CALL_PURE with just CALL
+        args = op.getarglist()
+        self.emit_operation(ResOperation(rop.CALL, args, op.result,
+                                         op.getdescr()))
+
+    def flush(self):
+        assert self.posponedop is None
+
+    def new(self):
+        assert self.posponedop is None
+        return OptPure()
+
+    def setup(self):
+        self.optimizer.optpure = self
+
+    def pure(self, opnum, args, result):
+        op = ResOperation(opnum, args, result)
+        key = self.optimizer.make_args_key(op)
+        if key not in self.pure_operations:
+            self.pure_operations[key] = op
+
+    def has_pure_result(self, opnum, args, descr):
+        op = ResOperation(opnum, args, None, descr)
+        key = self.optimizer.make_args_key(op)
+        op = self.pure_operations.get(key, None)
+        if op is None:
+            return False
+        return op.getdescr() is descr
+
+    def get_pure_result(self, key):
+        return self.pure_operations.get(key, None)
+
+    def remember_emitting_pure(self, op):
+        self.emitted_pure_operations[op] = True
+
+    def produce_potential_short_preamble_ops(self, sb):
+        for op in self.emitted_pure_operations:
+            if op.getopnum() == rop.GETARRAYITEM_GC_PURE or \
+               op.getopnum() == rop.STRGETITEM or \
+               op.getopnum() == rop.UNICODEGETITEM:
+                if not self.getvalue(op.getarg(1)).is_constant():
+                    continue
+            sb.add_potential(op)
+
+dispatch_opt = make_dispatcher_method(OptPure, 'optimize_',
+                                      default=OptPure.optimize_default)
diff --git a/pypy/jit/metainterp/optimizeopt/rewrite.py b/pypy/jit/metainterp/optimizeopt/rewrite.py
--- a/pypy/jit/metainterp/optimizeopt/rewrite.py
+++ b/pypy/jit/metainterp/optimizeopt/rewrite.py
@@ -31,21 +31,8 @@
 
         dispatch_opt(self, op)
 
-    def test_emittable(self, op):
-        opnum = op.getopnum()
-        for value, cls, func in optimize_guards:
-            if opnum == value:
-                assert isinstance(op, cls)
-                try:
-                    func(self, op, dryrun=True)
-                    return self.is_emittable(op)
-                except InvalidLoop:
-                    return False
-        return self.is_emittable(op)
-
-
     def try_boolinvers(self, op, targs):
-        oldop = self.optimizer.pure_operations.get(targs, None)
+        oldop = self.get_pure_result(targs)
         if oldop is not None and oldop.getdescr() is op.getdescr():
             value = self.getvalue(oldop.result)
             if value.is_constant():
@@ -73,7 +60,7 @@
             oldopnum = opboolreflex[op.getopnum()] # FIXME: add INT_ADD, INT_MUL
             targs = self.optimizer.make_args_key(ResOperation(oldopnum, [args[1], args[0]],
                                                               None))
-            oldop = self.optimizer.pure_operations.get(targs, None)
+            oldop = self.get_pure_result(targs)
             if oldop is not None and oldop.getdescr() is op.getdescr():
                 self.make_equal_to(op.result, self.getvalue(oldop.result))
                 return True
@@ -214,40 +201,7 @@
         self.emit_operation(op)
         self.pure(rop.FLOAT_NEG, [op.result], v1)
 
-    def optimize_CALL_PURE(self, op):
-        arg_consts = []
-        for i in range(op.numargs()):
-            arg = op.getarg(i)
-            const = self.get_constant_box(arg)
-            if const is None:
-                break
-            arg_consts.append(const)
-        else:
-            # all constant arguments: check if we already know the result
-            try:
-                result = self.optimizer.call_pure_results[arg_consts]
-            except KeyError:
-                pass
-            else:
-                self.make_constant(op.result, result)
-                return
-
-        args = self.optimizer.make_args_key(op)
-        oldop = self.optimizer.pure_operations.get(args, None)
-        if oldop is not None and oldop.getdescr() is op.getdescr():
-            assert oldop.getopnum() == op.getopnum()
-            self.make_equal_to(op.result, self.getvalue(oldop.result))
-            return
-        else:
-            self.optimizer.pure_operations[args] = op
-            self.optimizer.remember_emitting_pure(op)
-
-        # replace CALL_PURE with just CALL
-        args = op.getarglist()
-        self.emit_operation(ResOperation(rop.CALL, args, op.result,
-                                         op.getdescr()))
-
-    def optimize_guard(self, op, constbox, emit_operation=True, dryrun=False):
+    def optimize_guard(self, op, constbox, emit_operation=True):
         value = self.getvalue(op.getarg(0))
         if value.is_constant():
             box = value.box
@@ -255,62 +209,57 @@
             if not box.same_constant(constbox):
                 raise InvalidLoop
             return
-        if dryrun: return
         if emit_operation:
             self.emit_operation(op)
         value.make_constant(constbox)
         self.optimizer.turned_constant(value)
 
-    def optimize_GUARD_ISNULL(self, op, dryrun=False):
+    def optimize_GUARD_ISNULL(self, op):
         value = self.getvalue(op.getarg(0))
         if value.is_null():
             return
         elif value.is_nonnull():
             raise InvalidLoop
-        if dryrun: return
         self.emit_operation(op)
         value.make_constant(self.optimizer.cpu.ts.CONST_NULL)
 
-    def optimize_GUARD_NONNULL(self, op, dryrun=False):
+    def optimize_GUARD_NONNULL(self, op):
         value = self.getvalue(op.getarg(0))
         if value.is_nonnull():
             return
         elif value.is_null():
             raise InvalidLoop
-        if dryrun: return
         self.emit_operation(op)
-        value.make_nonnull(len(self.optimizer.newoperations) - 1)
+        value.make_nonnull(op)
 
-    def optimize_GUARD_VALUE(self, op, dryrun=False):
+    def optimize_GUARD_VALUE(self, op):
         value = self.getvalue(op.getarg(0))
-        emit_operation = True
-        if not dryrun and value.last_guard_index != -1:
+        if value.last_guard:
             # there already has been a guard_nonnull or guard_class or
             # guard_nonnull_class on this value, which is rather silly.
             # replace the original guard with a guard_value
-            old_guard_op = self.optimizer.newoperations[value.last_guard_index]
-            new_guard_op = old_guard_op.copy_and_change(rop.GUARD_VALUE,
-                                             args = [old_guard_op.getarg(0), op.getarg(1)])
-            self.optimizer.newoperations[value.last_guard_index] = new_guard_op
+            old_guard_op = value.last_guard
+            op = old_guard_op.copy_and_change(rop.GUARD_VALUE,
+                                      args = [old_guard_op.getarg(0), op.getarg(1)])
+            self.optimizer.replaces_guard[op] = old_guard_op
             # hack hack hack.  Change the guard_opnum on
             # new_guard_op.getdescr() so that when resuming,
             # the operation is not skipped by pyjitpl.py.
-            descr = new_guard_op.getdescr()
+            descr = op.getdescr()
             assert isinstance(descr, compile.ResumeGuardDescr)
             descr.guard_opnum = rop.GUARD_VALUE
-            descr.make_a_counter_per_value(new_guard_op)
-            emit_operation = False
+            descr.make_a_counter_per_value(op)
         constbox = op.getarg(1)
         assert isinstance(constbox, Const)
-        self.optimize_guard(op, constbox, emit_operation, dryrun)
+        self.optimize_guard(op, constbox)
 
-    def optimize_GUARD_TRUE(self, op, dryrun=False):
-        self.optimize_guard(op, CONST_1, dryrun=dryrun)
+    def optimize_GUARD_TRUE(self, op):
+        self.optimize_guard(op, CONST_1)
 
-    def optimize_GUARD_FALSE(self, op, dryrun=False):
-        self.optimize_guard(op, CONST_0, dryrun=dryrun)
+    def optimize_GUARD_FALSE(self, op):
+        self.optimize_guard(op, CONST_0)
 
-    def optimize_GUARD_CLASS(self, op, dryrun=False):
+    def optimize_GUARD_CLASS(self, op):
         value = self.getvalue(op.getarg(0))
         expectedclassbox = op.getarg(1)
         assert isinstance(expectedclassbox, Const)
@@ -319,38 +268,32 @@
             if realclassbox.same_constant(expectedclassbox):
                 return
             raise InvalidLoop
-        if dryrun: return
-        emit_operation = True
-        if value.last_guard_index != -1:
+        if value.last_guard:
             # there already has been a guard_nonnull or guard_class or
             # guard_nonnull_class on this value.
-            old_guard_op = self.optimizer.newoperations[value.last_guard_index]
+            old_guard_op = value.last_guard
             if old_guard_op.getopnum() == rop.GUARD_NONNULL:
                 # it was a guard_nonnull, which we replace with a
                 # guard_nonnull_class.
-                new_guard_op = old_guard_op.copy_and_change (rop.GUARD_NONNULL_CLASS,
+                op = old_guard_op.copy_and_change (rop.GUARD_NONNULL_CLASS,
                                          args = [old_guard_op.getarg(0), op.getarg(1)])
-                self.optimizer.newoperations[value.last_guard_index] = new_guard_op
+                self.optimizer.replaces_guard[op] = old_guard_op
                 # hack hack hack.  Change the guard_opnum on
                 # new_guard_op.getdescr() so that when resuming,
                 # the operation is not skipped by pyjitpl.py.
-                descr = new_guard_op.getdescr()
+                descr = op.getdescr()
                 assert isinstance(descr, compile.ResumeGuardDescr)
                 descr.guard_opnum = rop.GUARD_NONNULL_CLASS
-                emit_operation = False
-        if emit_operation:
-            self.emit_operation(op)
-            last_guard_index = len(self.optimizer.newoperations) - 1
-        else:
-            last_guard_index = value.last_guard_index
-        value.make_constant_class(expectedclassbox, last_guard_index)
+        self.emit_operation(op)
+        value.make_constant_class(expectedclassbox, op)
 
-    def optimize_GUARD_NONNULL_CLASS(self, op, dryrun=False):
-        self.optimize_GUARD_NONNULL(op, True)
-        self.optimize_GUARD_CLASS(op, dryrun)
+    def optimize_GUARD_NONNULL_CLASS(self, op):
+        value = self.getvalue(op.getarg(0))
+        if value.is_null():
+            raise InvalidLoop
+        self.optimize_GUARD_CLASS(op)
 
-    def optimize_GUARD_NO_EXCEPTION(self, op, dryrun=False):
-        if dryrun: return
+    def optimize_GUARD_NO_EXCEPTION(self, op):
         if not self.optimizer.exception_might_have_happened:
             return
         self.emit_operation(op)
@@ -470,7 +413,7 @@
                     newop = ResOperation(rop.SETARRAYITEM_GC,
                                          [op.getarg(2),
                                           ConstInt(index + dest_start),
-                                          val.force_box()], None,
+                                          val.get_key_box()], None,
                                          descr=source_value.arraydescr)
                     self.emit_operation(newop)
             return True
@@ -478,6 +421,25 @@
             return True # 0-length arraycopy
         return False
 
+    def optimize_CALL_PURE(self, op):
+        arg_consts = []
+        for i in range(op.numargs()):
+            arg = op.getarg(i)
+            const = self.get_constant_box(arg)
+            if const is None:
+                break
+            arg_consts.append(const)
+        else:
+            # all constant arguments: check if we already know the result
+            try:
+                result = self.optimizer.call_pure_results[arg_consts]
+            except KeyError:
+                pass
+            else:
+                self.make_constant(op.result, result)
+                return
+        self.emit_operation(op)
+
     def optimize_INT_FLOORDIV(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
@@ -492,6 +454,10 @@
                                         args = [op.getarg(0), ConstInt(highest_bit(val))])
         self.emit_operation(op)
 
+    def optimize_CAST_OPAQUE_PTR(self, op):
+        value = self.getvalue(op.getarg(0))
+        self.optimizer.opaque_pointers[value] = True
+        self.make_equal_to(op.result, value)
 
 dispatch_opt = make_dispatcher_method(OptRewrite, 'optimize_',
         default=OptRewrite.emit_operation)
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -39,8 +39,9 @@
 
 def test_sharing_field_lists_of_virtual():
     class FakeOptimizer(object):
-        class cpu(object):
-            pass
+        class optimizer(object):
+            class cpu(object):
+                pass
     opt = FakeOptimizer()
     virt1 = virtualize.AbstractVirtualStructValue(opt, None)
     lst1 = virt1._get_field_descr_list()
@@ -69,7 +70,7 @@
     class FakeVirtualValue(virtualize.AbstractVirtualValue):
         def _make_virtual(self, *args):
             return FakeVInfo()
-    v1 = FakeVirtualValue(None, None, None)
+    v1 = FakeVirtualValue(None, None)
     vinfo1 = v1.make_virtual_info(None, [1, 2, 4])
     vinfo2 = v1.make_virtual_info(None, [1, 2, 4])
     assert vinfo1 is vinfo2
@@ -111,7 +112,7 @@
 
 class BaseTestBasic(BaseTest):
 
-    enable_opts = "intbounds:rewrite:virtualize:string:heap"
+    enable_opts = "intbounds:rewrite:virtualize:string:earlyforce:pure:heap"
 
     def optimize_loop(self, ops, optops, call_pure_results=None):
 
@@ -651,8 +652,8 @@
         i3 = getfield_gc(p3, descr=valuedescr)
         escape(i3)
         p1 = new_with_vtable(ConstClass(node_vtable))
+        p1sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p1, i1, descr=valuedescr)
-        p1sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p1sub, i1, descr=valuedescr)
         setfield_gc(p1, p1sub, descr=nextdescr)
         jump(i1, p1, p2)
@@ -667,10 +668,10 @@
         p3sub = getfield_gc(p3, descr=nextdescr)
         i3 = getfield_gc(p3sub, descr=valuedescr)
         escape(i3)
+        p1 = new_with_vtable(ConstClass(node_vtable))
         p2sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p2sub, i1, descr=valuedescr)
         setfield_gc(p2, p2sub, descr=nextdescr)
-        p1 = new_with_vtable(ConstClass(node_vtable))
         jump(i1, p1, p2)
         """
         # The same as test_p123_simple, but in the end the "old" p2 contains
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizefficall.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizefficall.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizefficall.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizefficall.py
@@ -36,7 +36,7 @@
 
 class TestFfiCall(BaseTestBasic, LLtypeMixin):
 
-    enable_opts = "intbounds:rewrite:virtualize:string:heap:ffi"
+    enable_opts = "intbounds:rewrite:virtualize:string:pure:earlyforce:heap:ffi"
 
     class namespace:
         cpu = LLtypeMixin.cpu
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -68,7 +68,7 @@
 
 class BaseTestWithUnroll(BaseTest):
 
-    enable_opts = "intbounds:rewrite:virtualize:string:heap:unroll"
+    enable_opts = "intbounds:rewrite:virtualize:string:earlyforce:pure:heap:unroll"
 
     def optimize_loop(self, ops, expected, expected_preamble=None,
                       call_pure_results=None, expected_short=None):
@@ -825,8 +825,8 @@
         i3 = getfield_gc(p2, descr=valuedescr)
         escape(i3)
         p4 = new_with_vtable(ConstClass(node_vtable))
+        p1sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p4, i1, descr=valuedescr)
-        p1sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p1sub, i1, descr=valuedescr)
         setfield_gc(p4, p1sub, descr=nextdescr)
         jump(i1, p4)
@@ -865,13 +865,7 @@
         p3sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p3sub, i1, descr=valuedescr)
         setfield_gc(p1, p3sub, descr=nextdescr)
-        # XXX: We get two extra operations here because the setfield
-        #      above is the result of forcing p1 and thus not
-        #      registered with the heap optimizer. I've makred tests
-        #      below with VIRTUALHEAP if they suffer from this issue
-        p3sub2 = getfield_gc(p1, descr=nextdescr)
-        guard_nonnull_class(p3sub2, ConstClass(node_vtable2)) []
-        jump(i1, p1, p3sub2)
+        jump(i1, p1, p3sub)
         """
         self.optimize_loop(ops, expected, preamble)
 
@@ -902,9 +896,7 @@
         guard_true(i2b) []
         p3 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3, i2, descr=nextdescr)
-        # XXX: VIRTUALHEAP (see above)
-        i3 = getfield_gc(p3, descr=nextdescr)
-        jump(p3, i3)
+        jump(p3, i2)
         """
         self.optimize_loop(ops, expected, preamble)
 
@@ -1219,7 +1211,15 @@
         setfield_gc(p3, p30, descr=valuedescr)
         jump(i29, p30, p3)
         """
-        expected = preamble
+        expected = """
+        [i0, p1, p3]
+        i28 = int_add(i0, 1)
+        i29 = int_add(i28, 1)
+        p30 = new_with_vtable(ConstClass(node_vtable))
+        setfield_gc(p3, p30, descr=valuedescr)
+        setfield_gc(p30, i28, descr=nextdescr)
+        jump(i29, p30, p3)
+        """
         self.optimize_loop(ops, expected, preamble)
 
     def test_nonvirtual_1(self):
@@ -2408,8 +2408,8 @@
         guard_class(p2, ConstClass(node_vtable)) []
         p3 = getfield_gc(p1, descr=otherdescr)
         guard_class(p3, ConstClass(node_vtable)) []
+        p3a = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3, p2, descr=otherdescr)
-        p3a = new_with_vtable(ConstClass(node_vtable))
         escape(p3a)
         jump(p3a)
         """
@@ -2421,9 +2421,9 @@
         # setfield_gc(p3, p2, descr=otherdescr) # p3a.other = p2a
         # p1a = new_with_vtable(ConstClass(node_vtable2))
         # p2a = new_with_vtable(ConstClass(node_vtable))
+        p3anew = new_with_vtable(ConstClass(node_vtable))
         p2 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3a, p2, descr=otherdescr) # p3a.other = p2a
-        p3anew = new_with_vtable(ConstClass(node_vtable))
         escape(p3anew)
         jump(p3anew)
         """
@@ -2458,9 +2458,9 @@
         p3 = getfield_gc(p1, descr=otherdescr)
         guard_class(p3, ConstClass(node_vtable)) []
         # p1a = new_with_vtable(ConstClass(node_vtable2))
+        p3a = new_with_vtable(ConstClass(node_vtable))
         p2a = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3, p2a, descr=otherdescr)
-        p3a = new_with_vtable(ConstClass(node_vtable))
         escape(p3a)
         # setfield_gc(p1a, p2a, descr=nextdescr)
         # setfield_gc(p1a, p3a, descr=otherdescr)
@@ -2468,9 +2468,9 @@
         """
         expected = """
         [p2, p3]
+        p3a = new_with_vtable(ConstClass(node_vtable))
         p2a = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3, p2a, descr=otherdescr)
-        p3a = new_with_vtable(ConstClass(node_vtable))
         escape(p3a)
         jump(p2a, p3a)
         """
@@ -2790,7 +2790,6 @@
         self.optimize_loop(ops, expected, preamble)
 
     def test_remove_duplicate_pure_op_ovf_with_lazy_setfield(self):
-        py.test.skip('this optimization is not yet supprted')
         ops = """
         [i1, p1]
         i3 = int_add_ovf(i1, 1)
@@ -5961,13 +5960,18 @@
         escape(i0)
         jump(p1)
         """
-        expected = """
+        preamble = """
         [p1]
         i0 = ptr_eq(p1, NULL)
         escape(i0)
-        jump(p1)
-        """
-        self.optimize_strunicode_loop_extradescrs(ops, expected, expected)
+        jump(p1, i0)
+        """
+        expected = """
+        [p1, i0]
+        escape(i0)
+        jump(p1, i0)
+        """
+        self.optimize_strunicode_loop_extradescrs(ops, expected, preamble)
 
     def test_str_equal_none2(self):
         ops = """
@@ -5976,13 +5980,18 @@
         escape(i0)
         jump(p1)
         """
-        expected = """
+        preamble = """
         [p1]
         i0 = ptr_eq(p1, NULL)
         escape(i0)
-        jump(p1)
-        """
-        self.optimize_strunicode_loop_extradescrs(ops, expected, expected)
+        jump(p1, i0)
+        """
+        expected = """
+        [p1, i0]
+        escape(i0)
+        jump(p1, i0)
+        """
+        self.optimize_strunicode_loop_extradescrs(ops, expected, preamble)
 
     def test_str_equal_nonnull1(self):
         ops = """
@@ -7220,6 +7229,29 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_heap_cache_forced_virtuals(self):
+        ops = """
+        [i1, i2, p0]
+        p1 = new(descr=ssize)
+        setfield_gc(p1, i1, descr=adescr)
+        setfield_gc(p1, i2, descr=bdescr)
+        call(p0, p1, descr=writeadescr)
+        i3 = getfield_gc(p1, descr=adescr)
+        i4 = getfield_gc(p1, descr=bdescr)
+        jump(i3, i4, p0)
+        """
+        expected = """
+        [i1, i2, p0]
+        p1 = new(descr=ssize)
+        setfield_gc(p1, i1, descr=adescr)
+        call(p0, p1, descr=writeadescr)
+        i3 = getfield_gc(p1, descr=adescr)
+        setfield_gc(p1, i2, descr=bdescr)
+        jump(i3, i2, p0)
+        """
+        self.optimize_loop(ops, expected)
+
+
     def test_setarrayitem_followed_by_arraycopy(self):
         ops = """
         [p1, p2]
@@ -7230,6 +7262,27 @@
         """
         self.optimize_loop(ops, ops)
 
+    def test_heap_cache_virtuals_forced_by_delayed_setfield(self):
+        py.test.skip('not yet supoprted')
+        ops = """
+        [i1, p0]
+        p1 = new(descr=ssize)
+        setfield_gc(p1, i1, descr=valuedescr)
+        setfield_gc(p0, p1, descr=adescr)
+        call(p0, descr=writeadescr)
+        i2 = getfield_gc(p1, descr=valuedescr)
+        jump(i2, p0)
+        """
+        expected = """
+        [i1, p0]
+        p1 = new(descr=ssize)
+        setfield_gc(p1, i1, descr=valuedescr)
+        setfield_gc(p0, p1, descr=adescr)
+        call(p0, descr=writeadescr)
+        jump(i1, p0)
+        """
+        self.optimize_loop(ops, expected)
+
 class TestLLtype(OptimizeOptTest, LLtypeMixin):
     pass
 
diff --git a/pypy/jit/metainterp/optimizeopt/unroll.py b/pypy/jit/metainterp/optimizeopt/unroll.py
--- a/pypy/jit/metainterp/optimizeopt/unroll.py
+++ b/pypy/jit/metainterp/optimizeopt/unroll.py
@@ -75,7 +75,6 @@
         self.importable_values = {}
         self.emitting_dissabled = False
         self.emitted_guards = 0
-        self.emitted_pure_operations = {}
 
     def ensure_imported(self, value):
         if not self.emitting_dissabled and value in self.importable_values:
@@ -96,21 +95,6 @@
         new = UnrollableOptimizer(self.metainterp_sd, self.loop)
         return self._new(new)
 
-    def remember_emitting_pure(self, op):
-        self.emitted_pure_operations[op] = True
-
-    def produce_potential_short_preamble_ops(self, sb):
-        for op in self.emitted_pure_operations:
-            if op.getopnum() == rop.GETARRAYITEM_GC_PURE or \
-               op.getopnum() == rop.STRGETITEM or \
-               op.getopnum() == rop.UNICODEGETITEM:
-                if not self.getvalue(op.getarg(1)).is_constant():
-                    continue
-            sb.add_potential(op)
-        for opt in self.optimizations:
-            opt.produce_potential_short_preamble_ops(sb)
-
-
 
 class UnrollOptimizer(Optimization):
     """Unroll the loop into two iterations. The first one will
@@ -154,7 +138,7 @@
 
             KillHugeIntBounds(self.optimizer).apply()
             
-            loop.preamble.operations = self.optimizer.newoperations
+            loop.preamble.operations = self.optimizer.get_newoperations()
             jump_args = [self.getvalue(a).get_key_box() for a in jump_args]
 
             start_resumedescr = loop.preamble.start_resumedescr.clone_if_mutable()
@@ -167,8 +151,9 @@
             virtual_state = modifier.get_virtual_state(jump_args)
             
             values = [self.getvalue(arg) for arg in jump_args]
-            inputargs = virtual_state.make_inputargs(values)
-            short_inputargs = virtual_state.make_inputargs(values, keyboxes=True)
+            inputargs = virtual_state.make_inputargs(values, self.optimizer)
+            short_inputargs = virtual_state.make_inputargs(values, self.optimizer,
+                                                           keyboxes=True)
 
             self.constant_inputargs = {}
             for box in jump_args: 
@@ -197,7 +182,7 @@
             # operations needed to setup the proper state of those virtuals
             # in the peeled loop
             inputarg_setup_ops = []
-            preamble_optimizer.newoperations = []
+            preamble_optimizer.clear_newoperations()
             seen = {}
             for box in inputargs:
                 if box in seen:
@@ -211,9 +196,8 @@
                     continue
                 seen[box] = True
                 value = preamble_optimizer.getvalue(box)
-                value.force_box()
-            preamble_optimizer.flush()
-            inputarg_setup_ops += preamble_optimizer.newoperations
+                value.force_box(preamble_optimizer)
+            inputarg_setup_ops += preamble_optimizer.get_newoperations()
 
             # Setup the state of the new optimizer by emiting the
             # short preamble operations and discarding the result
@@ -244,13 +228,13 @@
                                 virtual_state)
             
             loop.inputargs = inputargs
-            args = [preamble_optimizer.getvalue(self.short_boxes.original(a)).force_box()\
+            args = [preamble_optimizer.getvalue(self.short_boxes.original(a)).force_box(preamble_optimizer)\
                     for a in inputargs]
             jmp = ResOperation(rop.JUMP, args, None)
             jmp.setdescr(loop.token)
             loop.preamble.operations.append(jmp)
 
-            loop.operations = self.optimizer.newoperations
+            loop.operations = self.optimizer.get_newoperations()
             maxguards = self.optimizer.metainterp_sd.warmrunnerdesc.memory_manager.max_retrace_guards
             
             if self.optimizer.emitted_guards > maxguards:
@@ -335,9 +319,10 @@
         assert jumpop
         original_jumpargs = jumpop.getarglist()[:]
         values = [self.getvalue(arg) for arg in jumpop.getarglist()]
-        jumpargs = virtual_state.make_inputargs(values)
+        jumpargs = virtual_state.make_inputargs(values, self.optimizer)
         jumpop.initarglist(jumpargs)
-        jmp_to_short_args = virtual_state.make_inputargs(values, keyboxes=True)
+        jmp_to_short_args = virtual_state.make_inputargs(values, self.optimizer,
+                                                         keyboxes=True)
         self.short_inliner = Inliner(short_inputargs, jmp_to_short_args)
         
         for box, const in self.constant_inputargs.items():
@@ -347,11 +332,11 @@
             newop = self.short_inliner.inline_op(op)
             self.optimizer.send_extra_operation(newop)
         
-        self.optimizer.flush()
+        newoperations = self.optimizer.get_newoperations()
 
         i = j = 0
-        while i < len(self.optimizer.newoperations) or j < len(jumpargs):
-            if i == len(self.optimizer.newoperations):
+        while i < len(newoperations) or j < len(jumpargs):
+            if i == len(newoperations):
                 while j < len(jumpargs):
                     a = jumpargs[j]
                     if self.optimizer.loop.logops:
@@ -360,7 +345,7 @@
                                     jumpargs, short_seen)
                     j += 1
             else:
-                op = self.optimizer.newoperations[i]
+                op = newoperations[i]
 
                 self.boxes_created_this_iteration[op.result] = True
                 args = op.getarglist()
@@ -375,6 +360,7 @@
                     self.import_box(a, inputargs, short, short_jumpargs,
                                     jumpargs, short_seen)
                 i += 1
+            newoperations = self.optimizer.get_newoperations()
 
         jumpop.initarglist(jumpargs)
         self.optimizer.send_extra_operation(jumpop)
@@ -468,7 +454,7 @@
         inputargs.append(box)
         box = newresult
         if box in self.optimizer.values:
-            box = self.optimizer.values[box].force_box()
+            box = self.optimizer.values[box].force_box(self.optimizer)
         jumpargs.append(box)
         
 
@@ -483,11 +469,6 @@
         if op.getopnum() == rop.JUMP:
             loop_token = op.getdescr()
             assert isinstance(loop_token, LoopToken)
-            # FIXME: Use a tree, similar to the tree formed by the full
-            # preamble and it's bridges, instead of a list to save time and
-            # memory. This should also allow better behaviour in
-            # situations that the is_emittable() chain currently cant
-            # handle and the inlining fails unexpectedly belwo.
             short = loop_token.short_preamble
             if short:
                 args = op.getarglist()
@@ -523,7 +504,7 @@
 
                         values = [self.getvalue(arg)
                                   for arg in op.getarglist()]
-                        args = sh.virtual_state.make_inputargs(values,
+                        args = sh.virtual_state.make_inputargs(values, self.optimizer,
                                                                keyboxes=True)
                         inliner = Inliner(sh.inputargs, args)
                         
diff --git a/pypy/jit/metainterp/optimizeopt/virtualize.py b/pypy/jit/metainterp/optimizeopt/virtualize.py
--- a/pypy/jit/metainterp/optimizeopt/virtualize.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualize.py
@@ -10,13 +10,12 @@
 
 
 class AbstractVirtualValue(optimizer.OptValue):
-    _attrs_ = ('optimizer', 'keybox', 'source_op', '_cached_vinfo')
+    _attrs_ = ('keybox', 'source_op', '_cached_vinfo')
     box = None
     level = optimizer.LEVEL_NONNULL
     _cached_vinfo = None
 
-    def __init__(self, optimizer, keybox, source_op=None):
-        self.optimizer = optimizer
+    def __init__(self, keybox, source_op=None):
         self.keybox = keybox   # only used as a key in dictionaries
         self.source_op = source_op  # the NEW_WITH_VTABLE/NEW_ARRAY operation
                                     # that builds this box
@@ -29,17 +28,17 @@
             return self.keybox
         return self.box
 
-    def force_box(self):
+    def force_box(self, optforce):
         if self.box is None:
-            self.optimizer.forget_numberings(self.keybox)
-            self._really_force()
+            optforce.forget_numberings(self.keybox)
+            self._really_force(optforce)
         return self.box
 
-    def force_at_end_of_preamble(self, already_forced):
+    def force_at_end_of_preamble(self, already_forced, optforce):
         value = already_forced.get(self, None)
         if value:
             return value
-        return OptValue(self.force_box())
+        return OptValue(self.force_box(optforce))
 
     def make_virtual_info(self, modifier, fieldnums):
         if fieldnums is None:
@@ -55,7 +54,7 @@
     def _make_virtual(self, modifier):
         raise NotImplementedError("abstract base")
 
-    def _really_force(self):
+    def _really_force(self, optforce):
         raise NotImplementedError("abstract base")
 
     def import_from(self, other, optimizer):
@@ -70,10 +69,11 @@
 get_fielddescrlist_cache._annspecialcase_ = "specialize:memo"
 
 class AbstractVirtualStructValue(AbstractVirtualValue):
-    _attrs_ = ('_fields', '_cached_sorted_fields')
+    _attrs_ = ('_fields', 'cpu', '_cached_sorted_fields')
 
-    def __init__(self, optimizer, keybox, source_op=None):
-        AbstractVirtualValue.__init__(self, optimizer, keybox, source_op)
+    def __init__(self, cpu, keybox, source_op=None):
+        AbstractVirtualValue.__init__(self, keybox, source_op)
+        self.cpu = cpu
         self._fields = {}
         self._cached_sorted_fields = None
 
@@ -87,45 +87,44 @@
     def _get_descr(self):
         raise NotImplementedError
 
-    def _is_immutable_and_filled_with_constants(self):
+    def _is_immutable_and_filled_with_constants(self, optforce):
         count = self._get_descr().count_fields_if_immutable()
         if count != len(self._fields):    # always the case if count == -1
             return False
         for value in self._fields.itervalues():
-            subbox = value.force_box()
+            subbox = value.force_box(optforce)
             if not isinstance(subbox, Const):
                 return False
         return True
 
-    def force_at_end_of_preamble(self, already_forced):
+    def force_at_end_of_preamble(self, already_forced, optforce):
         if self in already_forced:
             return self
         already_forced[self] = self
         if self._fields:
             for ofs in self._fields.keys():
-                self._fields[ofs] = self._fields[ofs].force_at_end_of_preamble(already_forced)
+                self._fields[ofs] = self._fields[ofs].force_at_end_of_preamble(already_forced, optforce)
         return self
 
-    def _really_force(self):
+    def _really_force(self, optforce):
         op = self.source_op
         assert op is not None
         # ^^^ This case should not occur any more (see test_bug_3).
         #
         if not we_are_translated():
             op.name = 'FORCE ' + self.source_op.name
-
-        if self._is_immutable_and_filled_with_constants():
-            box = self.optimizer.constant_fold(op)
+            
+        if self._is_immutable_and_filled_with_constants(optforce):
+            box = optforce.optimizer.constant_fold(op)
             self.make_constant(box)
             for ofs, value in self._fields.iteritems():
-                subbox = value.force_box()
+                subbox = value.force_box(optforce)
                 assert isinstance(subbox, Const)
-                execute(self.optimizer.cpu, None, rop.SETFIELD_GC,
+                execute(optforce.optimizer.cpu, None, rop.SETFIELD_GC,
                         ofs, box, subbox)
             # keep self._fields, because it's all immutable anyway
         else:
-            newoperations = self.optimizer.newoperations
-            newoperations.append(op)
+            optforce.emit_operation(op)
             self.box = box = op.result
             #
             iteritems = self._fields.iteritems()
@@ -135,10 +134,11 @@
             for ofs, value in iteritems:
                 if value.is_null():
                     continue
-                subbox = value.force_box()
+                subbox = value.force_box(optforce)
                 op = ResOperation(rop.SETFIELD_GC, [box, subbox], None,
                                   descr=ofs)
-                newoperations.append(op)
+
+                optforce.emit_operation(op)
 
     def _get_field_descr_list(self):
         _cached_sorted_fields = self._cached_sorted_fields
@@ -155,7 +155,7 @@
             else:
                 lst = self._fields.keys()
             sort_descrs(lst)
-            cache = get_fielddescrlist_cache(self.optimizer.cpu)
+            cache = get_fielddescrlist_cache(self.cpu)
             result = cache.get(lst, None)
             if result is None:
                 cache[lst] = lst
@@ -180,8 +180,8 @@
 class VirtualValue(AbstractVirtualStructValue):
     level = optimizer.LEVEL_KNOWNCLASS
 
-    def __init__(self, optimizer, known_class, keybox, source_op=None):
-        AbstractVirtualStructValue.__init__(self, optimizer, keybox, source_op)
+    def __init__(self, cpu, known_class, keybox, source_op=None):
+        AbstractVirtualStructValue.__init__(self, cpu, keybox, source_op)
         assert isinstance(known_class, Const)
         self.known_class = known_class
 
@@ -190,7 +190,7 @@
         return modifier.make_virtual(self.known_class, fielddescrs)
 
     def _get_descr(self):
-        return vtable2descr(self.optimizer.cpu, self.known_class.getint())
+        return vtable2descr(self.cpu, self.known_class.getint())
 
     def __repr__(self):
         cls_name = self.known_class.value.adr.ptr._obj._TYPE._name
@@ -201,8 +201,8 @@
 
 class VStructValue(AbstractVirtualStructValue):
 
-    def __init__(self, optimizer, structdescr, keybox, source_op=None):
-        AbstractVirtualStructValue.__init__(self, optimizer, keybox, source_op)
+    def __init__(self, cpu, structdescr, keybox, source_op=None):
+        AbstractVirtualStructValue.__init__(self, cpu, keybox, source_op)
         self.structdescr = structdescr
 
     def _make_virtual(self, modifier):
@@ -215,10 +215,10 @@
 
 class VArrayValue(AbstractVirtualValue):
 
-    def __init__(self, optimizer, arraydescr, size, keybox, source_op=None):
-        AbstractVirtualValue.__init__(self, optimizer, keybox, source_op)
+    def __init__(self, arraydescr, constvalue, size, keybox, source_op=None):
+        AbstractVirtualValue.__init__(self, keybox, source_op)
         self.arraydescr = arraydescr
-        self.constvalue = optimizer.new_const_item(arraydescr)
+        self.constvalue = constvalue
         self._items = [self.constvalue] * size
 
     def getlength(self):
@@ -232,31 +232,30 @@
         assert isinstance(itemvalue, optimizer.OptValue)
         self._items[index] = itemvalue
 
-    def force_at_end_of_preamble(self, already_forced):
+    def force_at_end_of_preamble(self, already_forced, optforce):
         if self in already_forced:
             return self
         already_forced[self] = self
         for index in range(len(self._items)):
-            self._items[index] = self._items[index].force_at_end_of_preamble(already_forced)
+            self._items[index] = self._items[index].force_at_end_of_preamble(already_forced, optforce)
         return self
     
-    def _really_force(self):
+    def _really_force(self, optforce):
         assert self.source_op is not None
         if not we_are_translated():
             self.source_op.name = 'FORCE ' + self.source_op.name
-        newoperations = self.optimizer.newoperations
-        newoperations.append(self.source_op)
+        optforce.emit_operation(self.source_op)        
         self.box = box = self.source_op.result
         for index in range(len(self._items)):
             subvalue = self._items[index]
             if subvalue is not self.constvalue:
                 if subvalue.is_null():
                     continue
-                subbox = subvalue.force_box()
+                subbox = subvalue.force_box(optforce)
                 op = ResOperation(rop.SETARRAYITEM_GC,
                                   [box, ConstInt(index), subbox], None,
                                   descr=self.arraydescr)
-                newoperations.append(op)
+                optforce.emit_operation(op)
 
     def get_args_for_fail(self, modifier):
         if self.box is None and not modifier.already_seen_virtual(self.keybox):
@@ -279,17 +278,18 @@
         return OptVirtualize()
         
     def make_virtual(self, known_class, box, source_op=None):
-        vvalue = VirtualValue(self.optimizer, known_class, box, source_op)
+        vvalue = VirtualValue(self.optimizer.cpu, known_class, box, source_op)
         self.make_equal_to(box, vvalue)
         return vvalue
 
     def make_varray(self, arraydescr, size, box, source_op=None):
-        vvalue = VArrayValue(self.optimizer, arraydescr, size, box, source_op)
+        constvalue = self.new_const_item(arraydescr)
+        vvalue = VArrayValue(arraydescr, constvalue, size, box, source_op)
         self.make_equal_to(box, vvalue)
         return vvalue
 
     def make_vstruct(self, structdescr, box, source_op=None):
-        vvalue = VStructValue(self.optimizer, structdescr, box, source_op)
+        vvalue = VStructValue(self.optimizer.cpu, structdescr, box, source_op)
         self.make_equal_to(box, vvalue)
         return vvalue
 
@@ -362,7 +362,6 @@
             self.make_equal_to(op.result, fieldvalue)
         else:
             value.ensure_nonnull()
-            ###self.heap_op_optimizer.optimize_GETFIELD_GC(op, value)
             self.emit_operation(op)
 
     # note: the following line does not mean that the two operations are
@@ -377,7 +376,6 @@
             value.setfield(op.getdescr(), fieldvalue)
         else:
             value.ensure_nonnull()
-            ###self.heap_op_optimizer.optimize_SETFIELD_GC(op, value, fieldvalue)
             self.emit_operation(op)
 
     def optimize_NEW_WITH_VTABLE(self, op):
@@ -417,7 +415,6 @@
                 self.make_equal_to(op.result, itemvalue)
                 return
         value.ensure_nonnull()
-        ###self.heap_op_optimizer.optimize_GETARRAYITEM_GC(op, value)
         self.emit_operation(op)
 
     # note: the following line does not mean that the two operations are
@@ -432,7 +429,6 @@
                 value.setitem(indexbox.getint(), self.getvalue(op.getarg(2)))
                 return
         value.ensure_nonnull()
-        ###self.heap_op_optimizer.optimize_SETARRAYITEM_GC(op, value, fieldvalue)
         self.emit_operation(op)
 
 
diff --git a/pypy/jit/metainterp/optimizeopt/virtualstate.py b/pypy/jit/metainterp/optimizeopt/virtualstate.py
--- a/pypy/jit/metainterp/optimizeopt/virtualstate.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualstate.py
@@ -30,7 +30,7 @@
     def _generate_guards(self, other, box, cpu, extra_guards):
         raise InvalidLoop
 
-    def enum_forced_boxes(self, boxes, value):
+    def enum_forced_boxes(self, boxes, value, optimizer):
         raise NotImplementedError
 
     def enum(self, virtual_state):
@@ -100,14 +100,14 @@
     def _generalization_of(self, other):
         raise NotImplementedError
 
-    def enum_forced_boxes(self, boxes, value):
+    def enum_forced_boxes(self, boxes, value, optimizer):
         assert isinstance(value, virtualize.AbstractVirtualStructValue)
         assert value.is_virtual()
         for i in range(len(self.fielddescrs)):
             v = value._fields[self.fielddescrs[i]]
             s = self.fieldstate[i]
             if s.position > self.position:
-                s.enum_forced_boxes(boxes, v)
+                s.enum_forced_boxes(boxes, v, optimizer)
 
     def _enum(self, virtual_state):
         for s in self.fieldstate:
@@ -177,14 +177,14 @@
                 return False
         return True
 
-    def enum_forced_boxes(self, boxes, value):
+    def enum_forced_boxes(self, boxes, value, optimizer):
         assert isinstance(value, virtualize.VArrayValue)
         assert value.is_virtual()
         for i in range(len(self.fieldstate)):
             v = value._items[i]
             s = self.fieldstate[i]
             if s.position > self.position:
-                s.enum_forced_boxes(boxes, v)
+                s.enum_forced_boxes(boxes, v, optimizer)
 
     def _enum(self, virtual_state):
         for s in self.fieldstate:
@@ -316,11 +316,11 @@
             import pdb; pdb.set_trace()
             raise NotImplementedError
 
-    def enum_forced_boxes(self, boxes, value):
+    def enum_forced_boxes(self, boxes, value, optimizer):
         if self.level == LEVEL_CONSTANT:
             return
         assert 0 <= self.position_in_notvirtuals 
-        boxes[self.position_in_notvirtuals] = value.force_box()
+        boxes[self.position_in_notvirtuals] = value.force_box(optimizer)
 
     def _enum(self, virtual_state):
         if self.level == LEVEL_CONSTANT:
@@ -377,11 +377,13 @@
             self.state[i].generate_guards(other.state[i], args[i],
                                           cpu, extra_guards, renum)
 
-    def make_inputargs(self, values, keyboxes=False):
+    def make_inputargs(self, values, optimizer, keyboxes=False):
+        if optimizer.optearlyforce:
+            optimizer = optimizer.optearlyforce
         assert len(values) == len(self.state)
         inputargs = [None] * len(self.notvirtuals)
         for i in range(len(values)):
-            self.state[i].enum_forced_boxes(inputargs, values[i])
+            self.state[i].enum_forced_boxes(inputargs, values[i], optimizer)
 
         if keyboxes:
             for i in range(len(values)):
@@ -434,7 +436,12 @@
     def get_virtual_state(self, jump_args):
         self.optimizer.force_at_end_of_preamble()
         already_forced = {}
-        values = [self.getvalue(box).force_at_end_of_preamble(already_forced)
+        if self.optimizer.optearlyforce:
+            opt = self.optimizer.optearlyforce
+        else:
+            opt = self.optimizer
+        values = [self.getvalue(box).force_at_end_of_preamble(already_forced,
+                                                              opt)
                   for box in jump_args]
 
         for value in values:
diff --git a/pypy/jit/metainterp/optimizeopt/vstring.py b/pypy/jit/metainterp/optimizeopt/vstring.py
--- a/pypy/jit/metainterp/optimizeopt/vstring.py
+++ b/pypy/jit/metainterp/optimizeopt/vstring.py
@@ -43,7 +43,7 @@
 class __extend__(optimizer.OptValue):
     """New methods added to the base class OptValue for this file."""
 
-    def getstrlen(self, optimization, mode):
+    def getstrlen(self, string_optimizer, mode):
         if mode is mode_string:
             s = self.get_constant_string_spec(mode_string)
             if s is not None:
@@ -52,12 +52,12 @@
             s = self.get_constant_string_spec(mode_unicode)
             if s is not None:
                 return ConstInt(len(s))
-        if optimization is None:
+        if string_optimizer is None:
             return None
         self.ensure_nonnull()
-        box = self.force_box()
+        box = self.force_box(string_optimizer)
         lengthbox = BoxInt()
-        optimization.propagate_forward(ResOperation(mode.STRLEN, [box], lengthbox))
+        string_optimizer.emit_operation(ResOperation(mode.STRLEN, [box], lengthbox))
         return lengthbox
 
     @specialize.arg(1)
@@ -68,25 +68,25 @@
         else:
             return None
 
-    def string_copy_parts(self, optimizer, targetbox, offsetbox, mode):
+    def string_copy_parts(self, string_optimizer, targetbox, offsetbox, mode):
         # Copies the pointer-to-string 'self' into the target string
         # given by 'targetbox', at the specified offset.  Returns the offset
         # at the end of the copy.
-        lengthbox = self.getstrlen(optimizer, mode)
-        srcbox = self.force_box()
-        return copy_str_content(optimizer, srcbox, targetbox,
+        lengthbox = self.getstrlen(string_optimizer, mode)
+        srcbox = self.force_box(string_optimizer)
+        return copy_str_content(string_optimizer, srcbox, targetbox,
                                 CONST_0, offsetbox, lengthbox, mode)
 
 
 class VAbstractStringValue(virtualize.AbstractVirtualValue):
     _attrs_ = ('mode',)
 
-    def __init__(self, optimizer, keybox, source_op, mode):
-        virtualize.AbstractVirtualValue.__init__(self, optimizer, keybox,
+    def __init__(self, keybox, source_op, mode):
+        virtualize.AbstractVirtualValue.__init__(self, keybox,
                                                  source_op)
         self.mode = mode
 
-    def _really_force(self):
+    def _really_force(self, optforce):
         if self.mode is mode_string:
             s = self.get_constant_string_spec(mode_string)
             if s is not None:
@@ -101,12 +101,12 @@
                 return
         assert self.source_op is not None
         self.box = box = self.source_op.result
-        lengthbox = self.getstrlen(self.optimizer, self.mode)
+        lengthbox = self.getstrlen(optforce, self.mode)
         op = ResOperation(self.mode.NEWSTR, [lengthbox], box)
         if not we_are_translated():
             op.name = 'FORCE'
-        self.optimizer.emit_operation(op)
-        self.string_copy_parts(self.optimizer, box, CONST_0, self.mode)
+        optforce.emit_operation(op)
+        self.string_copy_parts(optforce, box, CONST_0, self.mode)
 
 
 class VStringPlainValue(VAbstractStringValue):
@@ -140,20 +140,20 @@
         return mode.emptystr.join([mode.chr(c.box.getint())
                                    for c in self._chars])
 
-    def string_copy_parts(self, optimizer, targetbox, offsetbox, mode):
+    def string_copy_parts(self, string_optimizer, targetbox, offsetbox, mode):
         if not self.is_virtual() and targetbox is not self.box:
-            lengthbox = self.getstrlen(optimizer, mode)
-            srcbox = self.force_box()
-            return copy_str_content(optimizer, srcbox, targetbox,
+            lengthbox = self.getstrlen(string_optimizer, mode)
+            srcbox = self.force_box(string_optimizer)
+            return copy_str_content(string_optimizer, srcbox, targetbox,
                                 CONST_0, offsetbox, lengthbox, mode)
         for i in range(len(self._chars)):
-            charbox = self._chars[i].force_box()
+            charbox = self._chars[i].force_box(string_optimizer)
             if not (isinstance(charbox, Const) and charbox.same_constant(CONST_0)):
-                optimizer.emit_operation(ResOperation(mode.STRSETITEM, [targetbox,
-                                                                    offsetbox,
-                                                                    charbox],
+                string_optimizer.emit_operation(ResOperation(mode.STRSETITEM, [targetbox,
+                                                                               offsetbox,
+                                                                               charbox],
                                                   None))
-            offsetbox = _int_add(optimizer, offsetbox, CONST_1)
+            offsetbox = _int_add(string_optimizer, offsetbox, CONST_1)
         return offsetbox
 
     def get_args_for_fail(self, modifier):
@@ -187,16 +187,16 @@
         self.left = left
         self.right = right
 
-    def getstrlen(self, optimizer, mode):
+    def getstrlen(self, string_optimizer, mode):
         if self.lengthbox is None:
-            len1box = self.left.getstrlen(optimizer, mode)
+            len1box = self.left.getstrlen(string_optimizer, mode)
             if len1box is None:
                 return None
-            len2box = self.right.getstrlen(optimizer, mode)
+            len2box = self.right.getstrlen(string_optimizer, mode)
             if len2box is None:
                 return None
-            self.lengthbox = _int_add(optimizer, len1box, len2box)
-            # ^^^ may still be None, if optimizer is None
+            self.lengthbox = _int_add(string_optimizer, len1box, len2box)
+            # ^^^ may still be None, if string_optimizer is None
         return self.lengthbox
 
     @specialize.arg(1)
@@ -209,10 +209,10 @@
             return None
         return s1 + s2
 
-    def string_copy_parts(self, optimizer, targetbox, offsetbox, mode):
-        offsetbox = self.left.string_copy_parts(optimizer, targetbox,
+    def string_copy_parts(self, string_optimizer, targetbox, offsetbox, mode):
+        offsetbox = self.left.string_copy_parts(string_optimizer, targetbox,
                                                 offsetbox, mode)
-        offsetbox = self.right.string_copy_parts(optimizer, targetbox,
+        offsetbox = self.right.string_copy_parts(string_optimizer, targetbox,
                                                  offsetbox, mode)
         return offsetbox
 
@@ -251,8 +251,8 @@
         self.vstart = vstart
         self.vlength = vlength
 
-    def getstrlen(self, _, mode):
-        return self.vlength.force_box()
+    def getstrlen(self, optforce, mode):
+        return self.vlength.force_box(optforce)
 
     @specialize.arg(1)
     def get_constant_string_spec(self, mode):
@@ -267,11 +267,11 @@
             return s1[start : start + length]
         return None
 
-    def string_copy_parts(self, optimizer, targetbox, offsetbox, mode):
-        lengthbox = self.getstrlen(optimizer, mode)
-        return copy_str_content(optimizer,
-                                self.vstr.force_box(), targetbox,
-                                self.vstart.force_box(), offsetbox,
+    def string_copy_parts(self, string_optimizer, targetbox, offsetbox, mode):
+        lengthbox = self.getstrlen(string_optimizer, mode)
+        return copy_str_content(string_optimizer,
+                                self.vstr.force_box(string_optimizer), targetbox,
+                                self.vstart.force_box(string_optimizer), offsetbox,
                                 lengthbox, mode)
 
     def get_args_for_fail(self, modifier):
@@ -300,7 +300,7 @@
         return modifier.make_vstrslice(self.mode is mode_unicode)
 
 
-def copy_str_content(optimizer, srcbox, targetbox,
+def copy_str_content(string_optimizer, srcbox, targetbox,
                      srcoffsetbox, offsetbox, lengthbox, mode, need_next_offset=True):
     if isinstance(srcbox, ConstPtr) and isinstance(srcoffsetbox, Const):
         M = 5
@@ -310,26 +310,26 @@
         # up to M characters are done "inline", i.e. with STRGETITEM/STRSETITEM
         # instead of just a COPYSTRCONTENT.
         for i in range(lengthbox.value):
-            charbox = _strgetitem(optimizer, srcbox, srcoffsetbox, mode)
-            srcoffsetbox = _int_add(optimizer, srcoffsetbox, CONST_1)
-            optimizer.emit_operation(ResOperation(mode.STRSETITEM, [targetbox,
-                                                                offsetbox,
-                                                                charbox],
+            charbox = _strgetitem(string_optimizer, srcbox, srcoffsetbox, mode)
+            srcoffsetbox = _int_add(string_optimizer, srcoffsetbox, CONST_1)
+            string_optimizer.emit_operation(ResOperation(mode.STRSETITEM, [targetbox,
+                                                                           offsetbox,
+                                                                           charbox],
                                               None))
-            offsetbox = _int_add(optimizer, offsetbox, CONST_1)
+            offsetbox = _int_add(string_optimizer, offsetbox, CONST_1)
     else:
         if need_next_offset:
-            nextoffsetbox = _int_add(optimizer, offsetbox, lengthbox)
+            nextoffsetbox = _int_add(string_optimizer, offsetbox, lengthbox)
         else:
             nextoffsetbox = None
         op = ResOperation(mode.COPYSTRCONTENT, [srcbox, targetbox,
                                                 srcoffsetbox, offsetbox,
                                                 lengthbox], None)
-        optimizer.emit_operation(op)
+        string_optimizer.emit_operation(op)
         offsetbox = nextoffsetbox
     return offsetbox
 
-def _int_add(optimizer, box1, box2):
+def _int_add(string_optimizer, box1, box2):
     if isinstance(box1, ConstInt):
         if box1.value == 0:
             return box2
@@ -337,23 +337,23 @@
             return ConstInt(box1.value + box2.value)
     elif isinstance(box2, ConstInt) and box2.value == 0:
         return box1
-    if optimizer is None:
+    if string_optimizer is None:
         return None
     resbox = BoxInt()
-    optimizer.propagate_forward(ResOperation(rop.INT_ADD, [box1, box2], resbox))
+    string_optimizer.emit_operation(ResOperation(rop.INT_ADD, [box1, box2], resbox))
     return resbox
 
-def _int_sub(optimizer, box1, box2):
+def _int_sub(string_optimizer, box1, box2):
     if isinstance(box2, ConstInt):
         if box2.value == 0:
             return box1
         if isinstance(box1, ConstInt):
             return ConstInt(box1.value - box2.value)
     resbox = BoxInt()
-    optimizer.propagate_forward(ResOperation(rop.INT_SUB, [box1, box2], resbox))
+    string_optimizer.emit_operation(ResOperation(rop.INT_SUB, [box1, box2], resbox))
     return resbox
 
-def _strgetitem(optimizer, strbox, indexbox, mode):
+def _strgetitem(string_optimizer, strbox, indexbox, mode):
     if isinstance(strbox, ConstPtr) and isinstance(indexbox, ConstInt):
         if mode is mode_string:
             s = strbox.getref(lltype.Ptr(rstr.STR))
@@ -362,30 +362,28 @@
             s = strbox.getref(lltype.Ptr(rstr.UNICODE))
             return ConstInt(ord(s.chars[indexbox.getint()]))
     resbox = BoxInt()
-    optimizer.propagate_forward(ResOperation(mode.STRGETITEM, [strbox, indexbox],
-                                      resbox))
+    string_optimizer.emit_operation(ResOperation(mode.STRGETITEM, [strbox, indexbox],
+                                                 resbox))
     return resbox
 
 
 class OptString(optimizer.Optimization):
     "Handling of strings and unicodes."
-    enabled = True
-
     def new(self):
         return OptString()
 
     def make_vstring_plain(self, box, source_op, mode):
-        vvalue = VStringPlainValue(self.optimizer, box, source_op, mode)
+        vvalue = VStringPlainValue(box, source_op, mode)
         self.make_equal_to(box, vvalue)
         return vvalue
 
     def make_vstring_concat(self, box, source_op, mode):
-        vvalue = VStringConcatValue(self.optimizer, box, source_op, mode)
+        vvalue = VStringConcatValue(box, source_op, mode)
         self.make_equal_to(box, vvalue)
         return vvalue
 
     def make_vstring_slice(self, box, source_op, mode):
-        vvalue = VStringSliceValue(self.optimizer, box, source_op, mode)
+        vvalue = VStringSliceValue(box, source_op, mode)
         self.make_equal_to(box, vvalue)
         return vvalue
 
@@ -435,9 +433,9 @@
         value.ensure_nonnull()
         #
         if value.is_virtual() and isinstance(value, VStringSliceValue):
-            fullindexbox = _int_add(self.optimizer,
-                                    value.vstart.force_box(),
-                                    vindex.force_box())
+            fullindexbox = _int_add(self,
+                                    value.vstart.force_box(self),
+                                    vindex.force_box(self))
             value = value.vstr
             vindex = self.getvalue(fullindexbox)
         #
@@ -449,7 +447,7 @@
                 if res is not optimizer.CVAL_UNINITIALIZED_ZERO:
                     return res
         #
-        resbox = _strgetitem(self.optimizer, value.force_box(), vindex.force_box(), mode)
+        resbox = _strgetitem(self, value.force_box(self), vindex.force_box(self), mode)
         return self.getvalue(resbox)
 
     def optimize_STRLEN(self, op):
@@ -459,7 +457,7 @@
 
     def _optimize_STRLEN(self, op, mode):
         value = self.getvalue(op.getarg(0))
-        lengthbox = value.getstrlen(self.optimizer, mode)
+        lengthbox = value.getstrlen(self, mode)
         self.make_equal_to(op.result, self.getvalue(lengthbox))
 
     def optimize_COPYSTRCONTENT(self, op):
@@ -477,12 +475,12 @@
 
         if length.is_constant() and length.box.getint() == 0:
             return
-        copy_str_content(self.optimizer,
-            src.force_box(),
-            dst.force_box(),
-            srcstart.force_box(),
-            dststart.force_box(),
-            length.force_box(),
+        copy_str_content(self,
+            src.force_box(self),
+            dst.force_box(self),
+            srcstart.force_box(self),
+            dststart.force_box(self),
+            length.force_box(self),
             mode, need_next_offset=False
         )
 
@@ -508,6 +506,8 @@
                     return
         self.emit_operation(op)
 
+    optimize_CALL_PURE = optimize_CALL
+
     def opt_call_str_STR2UNICODE(self, op):
         # Constant-fold unicode("constant string").
         # More generally, supporting non-constant but virtual cases is
@@ -547,16 +547,16 @@
             return True
         #
         vstr.ensure_nonnull()
-        lengthbox = _int_sub(self.optimizer, vstop.force_box(),
-                                            vstart.force_box())
+        lengthbox = _int_sub(self, vstop.force_box(self),
+                                   vstart.force_box(self))
         #
         if isinstance(vstr, VStringSliceValue):
             # double slicing  s[i:j][k:l]
             vintermediate = vstr
             vstr = vintermediate.vstr
-            startbox = _int_add(self.optimizer,
-                                vintermediate.vstart.force_box(),
-                                vstart.force_box())
+            startbox = _int_add(self,
+                                vintermediate.vstart.force_box(self),
+                                vstart.force_box(self))
             vstart = self.getvalue(startbox)
         #
         value = self.make_vstring_slice(op.result, op, mode)
@@ -594,8 +594,8 @@
                 do = EffectInfo.OS_STREQ_LENGTHOK
             else:
                 do = EffectInfo.OS_STREQ_NONNULL
-            self.generate_modified_call(do, [v1.force_box(),
-                                             v2.force_box()], op.result, mode)
+            self.generate_modified_call(do, [v1.force_box(self),
+                                             v2.force_box(self)], op.result, mode)
             return True
         return False
 
@@ -603,7 +603,7 @@
         l2box = v2.getstrlen(None, mode)
         if isinstance(l2box, ConstInt):
             if l2box.value == 0:
-                lengthbox = v1.getstrlen(self.optimizer, mode)
+                lengthbox = v1.getstrlen(self, mode)
                 seo = self.optimizer.send_extra_operation
                 seo(ResOperation(rop.INT_EQ, [lengthbox, CONST_0], resultbox))
                 return True
@@ -614,17 +614,17 @@
                     vchar1 = self.strgetitem(v1, optimizer.CVAL_ZERO, mode)
                     vchar2 = self.strgetitem(v2, optimizer.CVAL_ZERO, mode)
                     seo = self.optimizer.send_extra_operation
-                    seo(ResOperation(rop.INT_EQ, [vchar1.force_box(),
-                                                  vchar2.force_box()],
+                    seo(ResOperation(rop.INT_EQ, [vchar1.force_box(self),
+                                                  vchar2.force_box(self)],
                                      resultbox))
                     return True
                 if isinstance(v1, VStringSliceValue):
                     vchar = self.strgetitem(v2, optimizer.CVAL_ZERO, mode)
                     do = EffectInfo.OS_STREQ_SLICE_CHAR
-                    self.generate_modified_call(do, [v1.vstr.force_box(),
-                                                     v1.vstart.force_box(),
-                                                     v1.vlength.force_box(),
-                                                     vchar.force_box()],
+                    self.generate_modified_call(do, [v1.vstr.force_box(self),
+                                                     v1.vstart.force_box(self),
+                                                     v1.vlength.force_box(self),
+                                                     vchar.force_box(self)],
                                                 resultbox, mode)
                     return True
         #
@@ -635,10 +635,10 @@
             if v1.is_null():
                 self.make_constant(resultbox, CONST_1)
                 return True
-            op = ResOperation(rop.PTR_EQ, [v1.force_box(),
+            op = ResOperation(rop.PTR_EQ, [v1.force_box(self),
                                            llhelper.CONST_NULL],
                               resultbox)
-            self.optimizer.emit_operation(op)
+            self.emit_operation(op)
             return True
         #
         return False
@@ -652,8 +652,8 @@
                     do = EffectInfo.OS_STREQ_NONNULL_CHAR
                 else:
                     do = EffectInfo.OS_STREQ_CHECKNULL_CHAR
-                self.generate_modified_call(do, [v1.force_box(),
-                                                 vchar.force_box()], resultbox,
+                self.generate_modified_call(do, [v1.force_box(self),
+                                                 vchar.force_box(self)], resultbox,
                                             mode)
                 return True
         #
@@ -662,10 +662,10 @@
                 do = EffectInfo.OS_STREQ_SLICE_NONNULL
             else:
                 do = EffectInfo.OS_STREQ_SLICE_CHECKNULL
-            self.generate_modified_call(do, [v1.vstr.force_box(),
-                                             v1.vstart.force_box(),
-                                             v1.vlength.force_box(),
-                                             v2.force_box()], resultbox, mode)
+            self.generate_modified_call(do, [v1.vstr.force_box(self),
+                                             v1.vstart.force_box(self),
+                                             v1.vlength.force_box(self),
+                                             v2.force_box(self)], resultbox, mode)
             return True
         return False
 
@@ -675,16 +675,11 @@
         calldescr, func = cic.callinfo_for_oopspec(oopspecindex)
         op = ResOperation(rop.CALL, [ConstInt(func)] + args, result,
                           descr=calldescr)
-        self.optimizer.emit_operation(op)
+        self.emit_operation(op)
 
     def propagate_forward(self, op):
-        if not self.enabled:
-            self.emit_operation(op)
-            return
-
         dispatch_opt(self, op)
 
-
 dispatch_opt = make_dispatcher_method(OptString, 'optimize_',
         default=OptString.emit_operation)
 
diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py
--- a/pypy/jit/metainterp/resoperation.py
+++ b/pypy/jit/metainterp/resoperation.py
@@ -221,7 +221,6 @@
         newop.setfailargs(self.getfailargs())
         return newop
 
-
 # ============
 # arity mixins
 # ============
diff --git a/pypy/jit/metainterp/test/test_resume.py b/pypy/jit/metainterp/test/test_resume.py
--- a/pypy/jit/metainterp/test/test_resume.py
+++ b/pypy/jit/metainterp/test/test_resume.py
@@ -576,8 +576,9 @@
 
 
 class FakeOptimizer_VirtualValue(object):
-    class cpu:
-        pass
+    class optimizer:
+        class cpu:
+            pass
 fakeoptimizer = FakeOptimizer_VirtualValue()
 
 def ConstAddr(addr, cpu):   # compatibility
@@ -1135,12 +1136,7 @@
     modifier.liveboxes = {}
     modifier.vfieldboxes = {}
 
-    class FakeOptimizer(object):
-        class cpu:
-            pass
-        def new_const_item(self, descr):
-            return None
-    v2 = VArrayValue(FakeOptimizer(), LLtypeMixin.arraydescr, 2, b2s)
+    v2 = VArrayValue(LLtypeMixin.arraydescr, None, 2, b2s)
     v2._items = [b4s, c1s]
     modifier.register_virtual_fields(b2s, [b4s, c1s])
     liveboxes = []
diff --git a/pypy/jit/metainterp/test/test_string.py b/pypy/jit/metainterp/test/test_string.py
--- a/pypy/jit/metainterp/test/test_string.py
+++ b/pypy/jit/metainterp/test/test_string.py
@@ -484,6 +484,30 @@
             return len(sa.val)
         assert self.meta_interp(f, ['a']) == f('a')
 
+    def test_string_comepare_quasiimmutable(self):
+        class Sys(object):
+            _immutable_fields_ = ["defaultencoding?"]
+            def __init__(self, s):
+                self.defaultencoding = s
+        _str = self._str
+        sys = Sys(_str('ascii'))        
+        mydriver = JitDriver(reds = ['n', 'sa'], greens = [])
+        def f(n):
+            sa = 0
+            sys.defaultencoding = _str('ascii')
+            while n:
+                mydriver.jit_merge_point(n=n, sa=sa)
+                if sys.defaultencoding == _str('ascii'):
+                    sa += 1
+                n -= 1
+            sys.defaultencoding = _str('utf-8')
+            return sa
+        assert self.meta_interp(f, [8]) == f(8)
+        self.check_loops({'int_add': 1, 'guard_true': 1, 'int_sub': 1,
+                          'jump': 1, 'int_is_true': 1,
+                          'guard_not_invalidated': 1})
+
+
 #class TestOOtype(StringTests, OOJitMixin):
 #    CALL = "oosend"
 #    CALL_PURE = "oosend_pure"
diff --git a/pypy/jit/metainterp/test/test_virtualstate.py b/pypy/jit/metainterp/test/test_virtualstate.py
--- a/pypy/jit/metainterp/test/test_virtualstate.py
+++ b/pypy/jit/metainterp/test/test_virtualstate.py
@@ -431,7 +431,7 @@
         
 
 class BaseTestBridges(BaseTest):
-    enable_opts = "intbounds:rewrite:virtualize:string:heap:unroll"
+    enable_opts = "intbounds:rewrite:virtualize:string:pure:heap:unroll"
 
     def _do_optimize_bridge(self, bridge, call_pure_results):
         from pypy.jit.metainterp.optimizeopt import optimize_bridge_1, build_opt_chain
diff --git a/pypy/module/pypyjit/test_pypy_c/test_00_model.py b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
--- a/pypy/module/pypyjit/test_pypy_c/test_00_model.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
@@ -50,8 +50,7 @@
         cmdline.append(str(self.filepath))
         #
         print cmdline, logfile
-        env={'PYPYLOG': 'jit-log-opt,jit-summary:' + str(logfile)}
-        #env={'PYPYLOG': ':' + str(logfile)}
+        env={'PYPYLOG': 'jit-log-opt,jit-log-noopt,jit-summary:' + str(logfile)}
         pipe = subprocess.Popen(cmdline,
                                 env=env,
                                 stdout=subprocess.PIPE,
diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py
--- a/pypy/module/pypyjit/test_pypy_c/test_call.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_call.py
@@ -366,12 +366,12 @@
             # make sure that the "block" is not allocated
             ...
             i20 = force_token()
-            setfield_gc(p0, i20, descr=<SignedFieldDescr .*PyFrame.vable_token .*>)
             p22 = new_with_vtable(19511408)
             p24 = new_array(1, descr=<GcPtrArrayDescr>)
             p26 = new_with_vtable(ConstClass(W_ListObject))
             p27 = new(descr=<SizeDescr .*>)
             p29 = new_array(0, descr=<GcPtrArrayDescr>)
+            setfield_gc(p0, i20, descr=<SignedFieldDescr .*PyFrame.vable_token .*>)
             setfield_gc(p27, p29, descr=<GcPtrFieldDescr list.items .*>)
             setfield_gc(p26, p27, descr=<.* .*W_ListObject.inst_wrappeditems .*>)
             setarrayitem_gc(p24, 0, p26, descr=<GcPtrArrayDescr>)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_generators.py b/pypy/module/pypyjit/test_pypy_c/test_generators.py
--- a/pypy/module/pypyjit/test_pypy_c/test_generators.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_generators.py
@@ -19,8 +19,8 @@
         assert loop.match_by_id("generator", """
             i16 = force_token()
             p45 = new_with_vtable(ConstClass(W_IntObject))
-            setfield_gc(p45, i29, descr=<SignedFieldDescr .*>)
             i47 = arraylen_gc(p8, descr=<GcPtrArrayDescr>) # Should be removed by backend
             setarrayitem_gc(p8, 0, p45, descr=<GcPtrArrayDescr>)
+            setfield_gc(p45, i29, descr=<SignedFieldDescr .*>)
             jump(..., descr=...)
             """)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_instance.py b/pypy/module/pypyjit/test_pypy_c/test_instance.py
--- a/pypy/module/pypyjit/test_pypy_c/test_instance.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_instance.py
@@ -125,8 +125,8 @@
             i12 = force_token()
             --TICK--
             p20 = new_with_vtable(ConstClass(W_IntObject))
+            setfield_gc(ConstPtr(ptr21), p20, descr=<GcPtrFieldDescr .*TypeCell.inst_w_value .*>)
             setfield_gc(p20, i11, descr=<SignedFieldDescr.*W_IntObject.inst_intval .*>)
-            setfield_gc(ConstPtr(ptr21), p20, descr=<GcPtrFieldDescr .*TypeCell.inst_w_value .*>)
             jump(p0, p1, p2, p3, p4, p20, p6, i7, p20, descr=<Loop.>)
         """)
 

From noreply at buildbot.pypy.org  Mon Oct  3 21:44:34 2011
From: noreply at buildbot.pypy.org (justinpeel)
Date: Mon,  3 Oct 2011 21:44:34 +0200 (CEST)
Subject: [pypy-commit] pypy unsigned-dtypes: merge in default
Message-ID: <20111003194434.5D90D820D9@wyvern.cs.uni-duesseldorf.de>

Author: Justin Peel <notmuchtotell at gmail.com>
Branch: unsigned-dtypes
Changeset: r47804:7bc0f79559ca
Date: 2011-10-03 13:42 -0600
http://bitbucket.org/pypy/pypy/changeset/7bc0f79559ca/

Log:	merge in default

diff --git a/pypy/jit/metainterp/optimizeopt/__init__.py b/pypy/jit/metainterp/optimizeopt/__init__.py
--- a/pypy/jit/metainterp/optimizeopt/__init__.py
+++ b/pypy/jit/metainterp/optimizeopt/__init__.py
@@ -7,6 +7,8 @@
 from pypy.jit.metainterp.optimizeopt.unroll import optimize_unroll, OptInlineShortPreamble
 from pypy.jit.metainterp.optimizeopt.fficall import OptFfiCall
 from pypy.jit.metainterp.optimizeopt.simplify import OptSimplify
+from pypy.jit.metainterp.optimizeopt.pure import OptPure
+from pypy.jit.metainterp.optimizeopt.earlyforce import OptEarlyForce
 from pypy.rlib.jit import PARAMETERS
 from pypy.rlib.unroll import unrolling_iterable
 
@@ -14,6 +16,8 @@
             ('rewrite', OptRewrite),
             ('virtualize', OptVirtualize),
             ('string', OptString),
+            ('earlyforce', OptEarlyForce),
+            ('pure', OptPure),
             ('heap', OptHeap),
             ('ffi', None),
             ('unroll', None)]
diff --git a/pypy/jit/metainterp/optimizeopt/earlyforce.py b/pypy/jit/metainterp/optimizeopt/earlyforce.py
new file mode 100644
--- /dev/null
+++ b/pypy/jit/metainterp/optimizeopt/earlyforce.py
@@ -0,0 +1,24 @@
+from pypy.jit.metainterp.optimizeopt.optimizer import Optimization
+from pypy.jit.metainterp.optimizeopt.vstring import VAbstractStringValue
+from pypy.jit.metainterp.resoperation import rop, ResOperation
+
+class OptEarlyForce(Optimization):
+    def propagate_forward(self, op):
+        opnum = op.getopnum()
+        if (opnum != rop.SETFIELD_GC and 
+            opnum != rop.SETARRAYITEM_GC and
+            opnum != rop.QUASIIMMUT_FIELD):
+               
+            for arg in op.getarglist():
+                if arg in self.optimizer.values:
+                    value = self.getvalue(arg)
+                    value.force_box(self)
+        self.emit_operation(op)
+
+    def new(self):
+        return OptEarlyForce()
+
+    def setup(self):
+        self.optimizer.optearlyforce = self
+
+    
diff --git a/pypy/jit/metainterp/optimizeopt/fficall.py b/pypy/jit/metainterp/optimizeopt/fficall.py
--- a/pypy/jit/metainterp/optimizeopt/fficall.py
+++ b/pypy/jit/metainterp/optimizeopt/fficall.py
@@ -177,10 +177,10 @@
             funcinfo.descr is None):
             return [op] # cannot optimize
         funcsymval = self.getvalue(op.getarg(2))
-        arglist = [funcsymval.force_box()]
+        arglist = [funcsymval.get_key_box()]
         for push_op in funcinfo.opargs:
             argval = self.getvalue(push_op.getarg(2))
-            arglist.append(argval.force_box())
+            arglist.append(argval.get_key_box())
         newop = ResOperation(rop.CALL_RELEASE_GIL, arglist, op.result,
                              descr=funcinfo.descr)
         self.commit_optimization()
diff --git a/pypy/jit/metainterp/optimizeopt/heap.py b/pypy/jit/metainterp/optimizeopt/heap.py
--- a/pypy/jit/metainterp/optimizeopt/heap.py
+++ b/pypy/jit/metainterp/optimizeopt/heap.py
@@ -50,6 +50,7 @@
             if not self._lazy_setfield_registered:
                 optheap._lazy_setfields_and_arrayitems.append(self)
                 self._lazy_setfield_registered = True
+            
         else:
             # this is the case where the pending setfield ends up
             # storing precisely the value that is already there,
@@ -158,12 +159,17 @@
         self._lazy_setfields_and_arrayitems = []
         self._remove_guard_not_invalidated = False
         self._seen_guard_not_invalidated = False
+        self.posponedop = None
 
     def force_at_end_of_preamble(self):
         self.force_all_lazy_setfields_and_arrayitems()
 
     def flush(self):
         self.force_all_lazy_setfields_and_arrayitems()
+        if self.posponedop:
+            posponedop = self.posponedop
+            self.posponedop = None
+            self.next_optimization.propagate_forward(posponedop)
 
     def new(self):
         return OptHeap()
@@ -211,7 +217,15 @@
 
     def emit_operation(self, op):
         self.emitting_operation(op)
-        self.next_optimization.propagate_forward(op)
+        if self.posponedop:
+            posponedop = self.posponedop
+            self.posponedop = None
+            self.next_optimization.propagate_forward(posponedop)
+        if (op.is_comparison() or op.getopnum() == rop.CALL_MAY_FORCE
+            or op.is_ovf()):
+            self.posponedop = op
+        else:
+            self.next_optimization.propagate_forward(op)
 
     def emitting_operation(self, op):
         if op.has_no_side_effect():
@@ -293,30 +307,6 @@
             if indexvalue is None or indexvalue.intbound.contains(idx):
                 cf.force_lazy_setfield(self, can_cache)
 
-    def fixup_guard_situation(self):
-        # hackish: reverse the order of the last two operations if it makes
-        # sense to avoid a situation like "int_eq/setfield_gc/guard_true",
-        # which the backend (at least the x86 backend) does not handle well.
-        newoperations = self.optimizer.newoperations
-        if len(newoperations) < 2:
-            return
-        lastop = newoperations[-1]
-        if (lastop.getopnum() != rop.SETFIELD_GC and
-            lastop.getopnum() != rop.SETARRAYITEM_GC):
-            return
-        # - is_comparison() for cases like "int_eq/setfield_gc/guard_true"
-        # - CALL_MAY_FORCE: "call_may_force/setfield_gc/guard_not_forced"
-        # - is_ovf(): "int_add_ovf/setfield_gc/guard_no_overflow"
-        prevop = newoperations[-2]
-        opnum = prevop.getopnum()
-        if not (prevop.is_comparison() or opnum == rop.CALL_MAY_FORCE
-                or prevop.is_ovf()):
-            return
-        if prevop.result in lastop.getarglist():
-            return
-        newoperations[-2] = lastop
-        newoperations[-1] = prevop
-
     def _assert_valid_cf(self, cf):
         # check that 'cf' is in cached_fields or cached_arrayitems
         if not we_are_translated():
@@ -362,7 +352,6 @@
                                       fieldvalue.get_key_box(), itemindex))
             else:
                 cf.force_lazy_setfield(self)
-                self.fixup_guard_situation()
         return pendingfields
 
     def optimize_GETFIELD_GC(self, op):
@@ -374,12 +363,22 @@
             return
         # default case: produce the operation
         structvalue.ensure_nonnull()
-        ###self.optimizer.optimize_default(op)
         self.emit_operation(op)
         # then remember the result of reading the field
         fieldvalue = self.getvalue(op.result)
         cf.remember_field_value(structvalue, fieldvalue, op)
 
+    def optimize_GETFIELD_GC_PURE(self, op):
+        structvalue = self.getvalue(op.getarg(0))
+        cf = self.field_cache(op.getdescr())
+        fieldvalue = cf.getfield_from_cache(self, structvalue)
+        if fieldvalue is not None:
+            self.make_equal_to(op.result, fieldvalue)
+            return
+        # default case: produce the operation
+        structvalue.ensure_nonnull()
+        self.emit_operation(op)
+
     def optimize_SETFIELD_GC(self, op):
         if self.has_pure_result(rop.GETFIELD_GC_PURE, [op.getarg(0)],
                                 op.getdescr()):
@@ -389,6 +388,7 @@
         #
         cf = self.field_cache(op.getdescr())
         cf.do_setfield(self, op)
+        
 
     def optimize_GETARRAYITEM_GC(self, op):
         arrayvalue = self.getvalue(op.getarg(0))
@@ -413,6 +413,25 @@
             fieldvalue = self.getvalue(op.result)
             cf.remember_field_value(arrayvalue, fieldvalue, op)
 
+    def optimize_GETARRAYITEM_GC_PURE(self, op):
+        arrayvalue = self.getvalue(op.getarg(0))
+        indexvalue = self.getvalue(op.getarg(1))
+        cf = None
+        if indexvalue.is_constant():
+            arrayvalue.make_len_gt(MODE_ARRAY, op.getdescr(), indexvalue.box.getint())
+            # use the cache on (arraydescr, index), which is a constant
+            cf = self.arrayitem_cache(op.getdescr(), indexvalue.box.getint())
+            fieldvalue = cf.getfield_from_cache(self, arrayvalue)
+            if fieldvalue is not None:
+                self.make_equal_to(op.result, fieldvalue)
+                return
+        else:
+            # variable index, so make sure the lazy setarrayitems are done
+            self.force_lazy_setarrayitem(op.getdescr(), indexvalue=indexvalue)
+        # default case: produce the operation
+        arrayvalue.ensure_nonnull()
+        self.emit_operation(op)
+
     def optimize_SETARRAYITEM_GC(self, op):
         if self.has_pure_result(rop.GETARRAYITEM_GC_PURE, [op.getarg(0),
                                                            op.getarg(1)],
diff --git a/pypy/jit/metainterp/optimizeopt/optimizer.py b/pypy/jit/metainterp/optimizeopt/optimizer.py
--- a/pypy/jit/metainterp/optimizeopt/optimizer.py
+++ b/pypy/jit/metainterp/optimizeopt/optimizer.py
@@ -31,8 +31,8 @@
 
 class OptValue(object):
     __metaclass__ = extendabletype
-    _attrs_ = ('box', 'known_class', 'last_guard_index', 'level', 'intbound', 'lenbound')
-    last_guard_index = -1
+    _attrs_ = ('box', 'known_class', 'last_guard', 'level', 'intbound', 'lenbound')
+    last_guard = None
 
     level = LEVEL_UNKNOWN
     known_class = None
@@ -100,7 +100,7 @@
             self.make_constant(other.get_key_box())
             optimizer.turned_constant(self)
         elif other.level == LEVEL_KNOWNCLASS:
-            self.make_constant_class(other.known_class, -1)
+            self.make_constant_class(other.known_class, None)
         else:
             if other.level == LEVEL_NONNULL:
                 self.ensure_nonnull()
@@ -114,13 +114,13 @@
                     self.lenbound = other.lenbound.clone()
 
 
-    def force_box(self):
+    def force_box(self, optforce):
         return self.box
 
     def get_key_box(self):
         return self.box
 
-    def force_at_end_of_preamble(self, already_forced):
+    def force_at_end_of_preamble(self, already_forced, optforce):
         return self
 
     def get_args_for_fail(self, modifier):
@@ -162,16 +162,16 @@
         else:
             return None
 
-    def make_constant_class(self, classbox, opindex):
+    def make_constant_class(self, classbox, guardop):
         assert self.level < LEVEL_KNOWNCLASS
         self.known_class = classbox
         self.level = LEVEL_KNOWNCLASS
-        self.last_guard_index = opindex
+        self.last_guard = guardop
 
-    def make_nonnull(self, opindex):
+    def make_nonnull(self, guardop):
         assert self.level < LEVEL_NONNULL
         self.level = LEVEL_NONNULL
-        self.last_guard_index = opindex
+        self.last_guard = guardop
 
     def is_nonnull(self):
         level = self.level
@@ -240,26 +240,12 @@
     def __init__(self):
         pass # make rpython happy
 
-    def propagate_begin_forward(self):
-        if self.next_optimization:
-            self.next_optimization.propagate_begin_forward()
-
-    def propagate_end_forward(self):
-        if self.next_optimization:
-            self.next_optimization.propagate_end_forward()
-
     def propagate_forward(self, op):
         raise NotImplementedError
 
     def emit_operation(self, op):
         self.next_optimization.propagate_forward(op)
 
-    def test_emittable(self, op):
-        return self.is_emittable(op)
-
-    def is_emittable(self, op):
-        return self.next_optimization.test_emittable(op)
-
     # FIXME: Move some of these here?
     def getvalue(self, box):
         return self.optimizer.getvalue(box)
@@ -289,19 +275,19 @@
         return self.optimizer.new_const_item(arraydescr)
 
     def pure(self, opnum, args, result):
-        op = ResOperation(opnum, args, result)
-        key = self.optimizer.make_args_key(op)
-        if key not in self.optimizer.pure_operations:
-            self.optimizer.pure_operations[key] = op
+        if self.optimizer.optpure:
+            self.optimizer.optpure.pure(opnum, args, result)
 
     def has_pure_result(self, opnum, args, descr):
-        op = ResOperation(opnum, args, None, descr)
-        key = self.optimizer.make_args_key(op)
-        op = self.optimizer.pure_operations.get(key, None)
-        if op is None:
-            return False
-        return op.getdescr() is descr
+        if self.optimizer.optpure:
+            return self.optimizer.optpure.has_pure_result(opnum, args, descr)
+        return False
 
+    def get_pure_result(self, key):    
+        if self.optimizer.optpure:
+            return self.optimizer.optpure.get_pure_result(key)
+        return None
+    
     def setup(self):
         pass
 
@@ -322,6 +308,9 @@
     def produce_potential_short_preamble_ops(self, potential_ops):
         pass
 
+    def forget_numberings(self, box):
+        self.optimizer.forget_numberings(box)
+
 class Optimizer(Optimization):
 
     def __init__(self, metainterp_sd, loop, optimizations=None, bridge=False):
@@ -333,14 +322,16 @@
         self.interned_refs = self.cpu.ts.new_ref_dict()
         self.resumedata_memo = resume.ResumeDataLoopMemo(metainterp_sd)
         self.bool_boxes = {}
-        self.pure_operations = args_dict()
         self.producer = {}
         self.pendingfields = []
-        self.posponedop = None
         self.exception_might_have_happened = False
         self.quasi_immutable_deps = None
         self.opaque_pointers = {}
-        self.newoperations = []
+        self.replaces_guard = {}
+        self._newoperations = []
+        self.optimizer = self
+        self.optpure = None
+        self.optearlyforce = None
         if loop is not None:
             self.call_pure_results = loop.call_pure_results
 
@@ -369,21 +360,20 @@
     def flush(self):
         for o in self.optimizations:
             o.flush()
-        assert self.posponedop is None
 
     def new(self):
         new = Optimizer(self.metainterp_sd, self.loop)
         return self._new(new)
 
     def _new(self, new):
-        assert self.posponedop is None
         optimizations = [o.new() for o in self.optimizations]
         new.set_optimizations(optimizations)
         new.quasi_immutable_deps = self.quasi_immutable_deps
         return new
 
     def produce_potential_short_preamble_ops(self, sb):
-        raise NotImplementedError('This is implemented in unroll.UnrollableOptimizer')
+        for opt in self.optimizations:
+            opt.produce_potential_short_preamble_ops(sb)
 
     def turned_constant(self, value):
         for o in self.optimizations:
@@ -433,6 +423,13 @@
             return constbox
         return None
 
+    def get_newoperations(self):
+        self.flush()
+        return self._newoperations
+
+    def clear_newoperations(self):
+        self._newoperations = []
+
     def make_equal_to(self, box, value, replace=False):
         assert isinstance(value, OptValue)
         assert replace or box not in self.values
@@ -481,15 +478,10 @@
 
     def propagate_all_forward(self):
         self.exception_might_have_happened = self.bridge
-        self.newoperations = []
-        self.first_optimization.propagate_begin_forward()
-        self.i = 0
-        while self.i < len(self.loop.operations):
-            op = self.loop.operations[self.i]
+        self.clear_newoperations()
+        for op in self.loop.operations:
             self.first_optimization.propagate_forward(op)
-            self.i += 1
-        self.first_optimization.propagate_end_forward()
-        self.loop.operations = self.newoperations
+        self.loop.operations = self.get_newoperations()
         self.loop.quasi_immutable_deps = self.quasi_immutable_deps
         # accumulate counters
         self.resumedata_memo.update_counters(self.metainterp_sd.profiler)
@@ -501,9 +493,6 @@
         self.producer[op.result] = op
         dispatch_opt(self, op)
 
-    def test_emittable(self, op):
-        return True
-
     def emit_operation(self, op):
         if op.returns_bool_result():
             self.bool_boxes[self.getvalue(op.result)] = None
@@ -519,14 +508,30 @@
                 pass
             else:
                 self.ensure_imported(value)
-                op.setarg(i, value.force_box())
+                op.setarg(i, value.force_box(self))
         self.metainterp_sd.profiler.count(jitprof.OPT_OPS)
         if op.is_guard():
             self.metainterp_sd.profiler.count(jitprof.OPT_GUARDS)
-            op = self.store_final_boxes_in_guard(op)
+            if self.replaces_guard and op in self.replaces_guard:
+                self.replace_op(self.replaces_guard[op], op)
+                del self.replaces_guard[op]
+                return
+            else:
+                op = self.store_final_boxes_in_guard(op)
         elif op.can_raise():
             self.exception_might_have_happened = True
-        self.newoperations.append(op)
+        self._newoperations.append(op)
+
+    def replace_op(self, old_op, new_op):
+        # XXX: Do we want to cache indexes to prevent search?
+        i = len(self._newoperations) 
+        while i > 0:
+            i -= 1
+            if self._newoperations[i] is old_op:
+                self._newoperations[i] = new_op
+                break
+        else:
+            assert False
 
     def store_final_boxes_in_guard(self, op):
         descr = op.getdescr()
@@ -574,51 +579,8 @@
         args[n + 1] = op.getdescr()
         return args
 
-    @specialize.argtype(0)
     def optimize_default(self, op):
-        canfold = op.is_always_pure()
-        if op.is_ovf():
-            self.posponedop = op
-            return
-        if self.posponedop:
-            nextop = op
-            op = self.posponedop
-            self.posponedop = None
-            canfold = nextop.getopnum() == rop.GUARD_NO_OVERFLOW
-        else:
-            nextop = None
-
-        if canfold:
-            for i in range(op.numargs()):
-                if self.get_constant_box(op.getarg(i)) is None:
-                    break
-            else:
-                # all constant arguments: constant-fold away
-                resbox = self.constant_fold(op)
-                # note that INT_xxx_OVF is not done from here, and the
-                # overflows in the INT_xxx operations are ignored
-                self.make_constant(op.result, resbox)
-                return
-
-            # did we do the exact same operation already?
-            args = self.make_args_key(op)
-            oldop = self.pure_operations.get(args, None)
-            if oldop is not None and oldop.getdescr() is op.getdescr():
-                assert oldop.getopnum() == op.getopnum()
-                self.make_equal_to(op.result, self.getvalue(oldop.result),
-                                   True)
-                return
-            else:
-                self.pure_operations[args] = op
-                self.remember_emitting_pure(op)
-
-        # otherwise, the operation remains
         self.emit_operation(op)
-        if nextop:
-            self.emit_operation(nextop)
-
-    def remember_emitting_pure(self, op):
-        pass
 
     def constant_fold(self, op):
         argboxes = [self.get_constant_box(op.getarg(i))
@@ -636,11 +598,6 @@
     def optimize_DEBUG_MERGE_POINT(self, op):
         self.emit_operation(op)
 
-    def optimize_CAST_OPAQUE_PTR(self, op):
-        value = self.getvalue(op.getarg(0))
-        self.opaque_pointers[value] = True
-        self.make_equal_to(op.result, value)
-
     def optimize_GETARRAYITEM_GC_PURE(self, op):
         indexvalue = self.getvalue(op.getarg(1))
         if indexvalue.is_constant():
diff --git a/pypy/jit/metainterp/optimizeopt/pure.py b/pypy/jit/metainterp/optimizeopt/pure.py
new file mode 100644
--- /dev/null
+++ b/pypy/jit/metainterp/optimizeopt/pure.py
@@ -0,0 +1,115 @@
+from pypy.jit.metainterp.optimizeopt.optimizer import Optimization
+from pypy.jit.metainterp.resoperation import rop, ResOperation
+from pypy.jit.metainterp.optimizeopt.util import (make_dispatcher_method,
+    args_dict)
+
+class OptPure(Optimization):
+    def __init__(self):
+        self.posponedop = None
+        self.pure_operations = args_dict()
+        self.emitted_pure_operations = {}
+
+    def propagate_forward(self, op):
+        dispatch_opt(self, op)
+
+    def optimize_default(self, op):
+        canfold = op.is_always_pure()
+        if op.is_ovf():
+            self.posponedop = op
+            return
+        if self.posponedop:
+            nextop = op
+            op = self.posponedop
+            self.posponedop = None
+            canfold = nextop.getopnum() == rop.GUARD_NO_OVERFLOW
+        else:
+            nextop = None
+
+        if canfold:
+            for i in range(op.numargs()):
+                if self.get_constant_box(op.getarg(i)) is None:
+                    break
+            else:
+                # all constant arguments: constant-fold away
+                resbox = self.optimizer.constant_fold(op)
+                # note that INT_xxx_OVF is not done from here, and the
+                # overflows in the INT_xxx operations are ignored
+                self.optimizer.make_constant(op.result, resbox)
+                return
+
+            # did we do the exact same operation already?
+            args = self.optimizer.make_args_key(op)
+            oldop = self.pure_operations.get(args, None)
+            if oldop is not None and oldop.getdescr() is op.getdescr():
+                assert oldop.getopnum() == op.getopnum()
+                self.optimizer.make_equal_to(op.result, self.getvalue(oldop.result),
+                                   True)
+                return
+            else:
+                self.pure_operations[args] = op
+                self.remember_emitting_pure(op)
+
+        # otherwise, the operation remains
+        self.emit_operation(op)
+        if op.returns_bool_result():
+            self.optimizer.bool_boxes[self.getvalue(op.result)] = None        
+        if nextop:
+            self.emit_operation(nextop)
+
+    def optimize_CALL_PURE(self, op):
+        args = self.optimizer.make_args_key(op)
+        oldop = self.pure_operations.get(args, None)
+        if oldop is not None and oldop.getdescr() is op.getdescr():
+            assert oldop.getopnum() == op.getopnum()
+            self.make_equal_to(op.result, self.getvalue(oldop.result))
+            return
+        else:
+            self.pure_operations[args] = op
+            self.remember_emitting_pure(op)
+
+        # replace CALL_PURE with just CALL
+        args = op.getarglist()
+        self.emit_operation(ResOperation(rop.CALL, args, op.result,
+                                         op.getdescr()))
+
+    def flush(self):
+        assert self.posponedop is None
+
+    def new(self):
+        assert self.posponedop is None
+        return OptPure()
+
+    def setup(self):
+        self.optimizer.optpure = self
+
+    def pure(self, opnum, args, result):
+        op = ResOperation(opnum, args, result)
+        key = self.optimizer.make_args_key(op)
+        if key not in self.pure_operations:
+            self.pure_operations[key] = op
+
+    def has_pure_result(self, opnum, args, descr):
+        op = ResOperation(opnum, args, None, descr)
+        key = self.optimizer.make_args_key(op)
+        op = self.pure_operations.get(key, None)
+        if op is None:
+            return False
+        return op.getdescr() is descr
+
+    def get_pure_result(self, key):
+        return self.pure_operations.get(key, None)
+
+    def remember_emitting_pure(self, op):
+        self.emitted_pure_operations[op] = True
+
+    def produce_potential_short_preamble_ops(self, sb):
+        for op in self.emitted_pure_operations:
+            if op.getopnum() == rop.GETARRAYITEM_GC_PURE or \
+               op.getopnum() == rop.STRGETITEM or \
+               op.getopnum() == rop.UNICODEGETITEM:
+                if not self.getvalue(op.getarg(1)).is_constant():
+                    continue
+            sb.add_potential(op)
+
+dispatch_opt = make_dispatcher_method(OptPure, 'optimize_',
+                                      default=OptPure.optimize_default)
diff --git a/pypy/jit/metainterp/optimizeopt/rewrite.py b/pypy/jit/metainterp/optimizeopt/rewrite.py
--- a/pypy/jit/metainterp/optimizeopt/rewrite.py
+++ b/pypy/jit/metainterp/optimizeopt/rewrite.py
@@ -31,21 +31,8 @@
 
         dispatch_opt(self, op)
 
-    def test_emittable(self, op):
-        opnum = op.getopnum()
-        for value, cls, func in optimize_guards:
-            if opnum == value:
-                assert isinstance(op, cls)
-                try:
-                    func(self, op, dryrun=True)
-                    return self.is_emittable(op)
-                except InvalidLoop:
-                    return False
-        return self.is_emittable(op)
-
-
     def try_boolinvers(self, op, targs):
-        oldop = self.optimizer.pure_operations.get(targs, None)
+        oldop = self.get_pure_result(targs)
         if oldop is not None and oldop.getdescr() is op.getdescr():
             value = self.getvalue(oldop.result)
             if value.is_constant():
@@ -73,7 +60,7 @@
             oldopnum = opboolreflex[op.getopnum()] # FIXME: add INT_ADD, INT_MUL
             targs = self.optimizer.make_args_key(ResOperation(oldopnum, [args[1], args[0]],
                                                               None))
-            oldop = self.optimizer.pure_operations.get(targs, None)
+            oldop = self.get_pure_result(targs)
             if oldop is not None and oldop.getdescr() is op.getdescr():
                 self.make_equal_to(op.result, self.getvalue(oldop.result))
                 return True
@@ -214,40 +201,7 @@
         self.emit_operation(op)
         self.pure(rop.FLOAT_NEG, [op.result], v1)
 
-    def optimize_CALL_PURE(self, op):
-        arg_consts = []
-        for i in range(op.numargs()):
-            arg = op.getarg(i)
-            const = self.get_constant_box(arg)
-            if const is None:
-                break
-            arg_consts.append(const)
-        else:
-            # all constant arguments: check if we already know the result
-            try:
-                result = self.optimizer.call_pure_results[arg_consts]
-            except KeyError:
-                pass
-            else:
-                self.make_constant(op.result, result)
-                return
-
-        args = self.optimizer.make_args_key(op)
-        oldop = self.optimizer.pure_operations.get(args, None)
-        if oldop is not None and oldop.getdescr() is op.getdescr():
-            assert oldop.getopnum() == op.getopnum()
-            self.make_equal_to(op.result, self.getvalue(oldop.result))
-            return
-        else:
-            self.optimizer.pure_operations[args] = op
-            self.optimizer.remember_emitting_pure(op)
-
-        # replace CALL_PURE with just CALL
-        args = op.getarglist()
-        self.emit_operation(ResOperation(rop.CALL, args, op.result,
-                                         op.getdescr()))
-
-    def optimize_guard(self, op, constbox, emit_operation=True, dryrun=False):
+    def optimize_guard(self, op, constbox, emit_operation=True):
         value = self.getvalue(op.getarg(0))
         if value.is_constant():
             box = value.box
@@ -255,62 +209,57 @@
             if not box.same_constant(constbox):
                 raise InvalidLoop
             return
-        if dryrun: return
         if emit_operation:
             self.emit_operation(op)
         value.make_constant(constbox)
         self.optimizer.turned_constant(value)
 
-    def optimize_GUARD_ISNULL(self, op, dryrun=False):
+    def optimize_GUARD_ISNULL(self, op):
         value = self.getvalue(op.getarg(0))
         if value.is_null():
             return
         elif value.is_nonnull():
             raise InvalidLoop
-        if dryrun: return
         self.emit_operation(op)
         value.make_constant(self.optimizer.cpu.ts.CONST_NULL)
 
-    def optimize_GUARD_NONNULL(self, op, dryrun=False):
+    def optimize_GUARD_NONNULL(self, op):
         value = self.getvalue(op.getarg(0))
         if value.is_nonnull():
             return
         elif value.is_null():
             raise InvalidLoop
-        if dryrun: return
         self.emit_operation(op)
-        value.make_nonnull(len(self.optimizer.newoperations) - 1)
+        value.make_nonnull(op)
 
-    def optimize_GUARD_VALUE(self, op, dryrun=False):
+    def optimize_GUARD_VALUE(self, op):
         value = self.getvalue(op.getarg(0))
-        emit_operation = True
-        if not dryrun and value.last_guard_index != -1:
+        if value.last_guard:
             # there already has been a guard_nonnull or guard_class or
             # guard_nonnull_class on this value, which is rather silly.
             # replace the original guard with a guard_value
-            old_guard_op = self.optimizer.newoperations[value.last_guard_index]
-            new_guard_op = old_guard_op.copy_and_change(rop.GUARD_VALUE,
-                                             args = [old_guard_op.getarg(0), op.getarg(1)])
-            self.optimizer.newoperations[value.last_guard_index] = new_guard_op
+            old_guard_op = value.last_guard
+            op = old_guard_op.copy_and_change(rop.GUARD_VALUE,
+                                      args = [old_guard_op.getarg(0), op.getarg(1)])
+            self.optimizer.replaces_guard[op] = old_guard_op
             # hack hack hack.  Change the guard_opnum on
             # new_guard_op.getdescr() so that when resuming,
             # the operation is not skipped by pyjitpl.py.
-            descr = new_guard_op.getdescr()
+            descr = op.getdescr()
             assert isinstance(descr, compile.ResumeGuardDescr)
             descr.guard_opnum = rop.GUARD_VALUE
-            descr.make_a_counter_per_value(new_guard_op)
-            emit_operation = False
+            descr.make_a_counter_per_value(op)
         constbox = op.getarg(1)
         assert isinstance(constbox, Const)
-        self.optimize_guard(op, constbox, emit_operation, dryrun)
+        self.optimize_guard(op, constbox)
 
-    def optimize_GUARD_TRUE(self, op, dryrun=False):
-        self.optimize_guard(op, CONST_1, dryrun=dryrun)
+    def optimize_GUARD_TRUE(self, op):
+        self.optimize_guard(op, CONST_1)
 
-    def optimize_GUARD_FALSE(self, op, dryrun=False):
-        self.optimize_guard(op, CONST_0, dryrun=dryrun)
+    def optimize_GUARD_FALSE(self, op):
+        self.optimize_guard(op, CONST_0)
 
-    def optimize_GUARD_CLASS(self, op, dryrun=False):
+    def optimize_GUARD_CLASS(self, op):
         value = self.getvalue(op.getarg(0))
         expectedclassbox = op.getarg(1)
         assert isinstance(expectedclassbox, Const)
@@ -319,38 +268,32 @@
             if realclassbox.same_constant(expectedclassbox):
                 return
             raise InvalidLoop
-        if dryrun: return
-        emit_operation = True
-        if value.last_guard_index != -1:
+        if value.last_guard:
             # there already has been a guard_nonnull or guard_class or
             # guard_nonnull_class on this value.
-            old_guard_op = self.optimizer.newoperations[value.last_guard_index]
+            old_guard_op = value.last_guard
             if old_guard_op.getopnum() == rop.GUARD_NONNULL:
                 # it was a guard_nonnull, which we replace with a
                 # guard_nonnull_class.
-                new_guard_op = old_guard_op.copy_and_change (rop.GUARD_NONNULL_CLASS,
+                op = old_guard_op.copy_and_change (rop.GUARD_NONNULL_CLASS,
                                          args = [old_guard_op.getarg(0), op.getarg(1)])
-                self.optimizer.newoperations[value.last_guard_index] = new_guard_op
+                self.optimizer.replaces_guard[op] = old_guard_op
                 # hack hack hack.  Change the guard_opnum on
                 # new_guard_op.getdescr() so that when resuming,
                 # the operation is not skipped by pyjitpl.py.
-                descr = new_guard_op.getdescr()
+                descr = op.getdescr()
                 assert isinstance(descr, compile.ResumeGuardDescr)
                 descr.guard_opnum = rop.GUARD_NONNULL_CLASS
-                emit_operation = False
-        if emit_operation:
-            self.emit_operation(op)
-            last_guard_index = len(self.optimizer.newoperations) - 1
-        else:
-            last_guard_index = value.last_guard_index
-        value.make_constant_class(expectedclassbox, last_guard_index)
+        self.emit_operation(op)
+        value.make_constant_class(expectedclassbox, op)
 
-    def optimize_GUARD_NONNULL_CLASS(self, op, dryrun=False):
-        self.optimize_GUARD_NONNULL(op, True)
-        self.optimize_GUARD_CLASS(op, dryrun)
+    def optimize_GUARD_NONNULL_CLASS(self, op):
+        value = self.getvalue(op.getarg(0))
+        if value.is_null():
+            raise InvalidLoop
+        self.optimize_GUARD_CLASS(op)
 
-    def optimize_GUARD_NO_EXCEPTION(self, op, dryrun=False):
-        if dryrun: return
+    def optimize_GUARD_NO_EXCEPTION(self, op):
         if not self.optimizer.exception_might_have_happened:
             return
         self.emit_operation(op)
@@ -470,7 +413,7 @@
                     newop = ResOperation(rop.SETARRAYITEM_GC,
                                          [op.getarg(2),
                                           ConstInt(index + dest_start),
-                                          val.force_box()], None,
+                                          val.get_key_box()], None,
                                          descr=source_value.arraydescr)
                     self.emit_operation(newop)
             return True
@@ -478,6 +421,25 @@
             return True # 0-length arraycopy
         return False
 
+    def optimize_CALL_PURE(self, op):
+        arg_consts = []
+        for i in range(op.numargs()):
+            arg = op.getarg(i)
+            const = self.get_constant_box(arg)
+            if const is None:
+                break
+            arg_consts.append(const)
+        else:
+            # all constant arguments: check if we already know the result
+            try:
+                result = self.optimizer.call_pure_results[arg_consts]
+            except KeyError:
+                pass
+            else:
+                self.make_constant(op.result, result)
+                return
+        self.emit_operation(op)
+
     def optimize_INT_FLOORDIV(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
@@ -492,6 +454,10 @@
                                         args = [op.getarg(0), ConstInt(highest_bit(val))])
         self.emit_operation(op)
 
+    def optimize_CAST_OPAQUE_PTR(self, op):
+        value = self.getvalue(op.getarg(0))
+        self.optimizer.opaque_pointers[value] = True
+        self.make_equal_to(op.result, value)
 
 dispatch_opt = make_dispatcher_method(OptRewrite, 'optimize_',
         default=OptRewrite.emit_operation)
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -39,8 +39,9 @@
 
 def test_sharing_field_lists_of_virtual():
     class FakeOptimizer(object):
-        class cpu(object):
-            pass
+        class optimizer(object):
+            class cpu(object):
+                pass
     opt = FakeOptimizer()
     virt1 = virtualize.AbstractVirtualStructValue(opt, None)
     lst1 = virt1._get_field_descr_list()
@@ -69,7 +70,7 @@
     class FakeVirtualValue(virtualize.AbstractVirtualValue):
         def _make_virtual(self, *args):
             return FakeVInfo()
-    v1 = FakeVirtualValue(None, None, None)
+    v1 = FakeVirtualValue(None, None)
     vinfo1 = v1.make_virtual_info(None, [1, 2, 4])
     vinfo2 = v1.make_virtual_info(None, [1, 2, 4])
     assert vinfo1 is vinfo2
@@ -111,7 +112,7 @@
 
 class BaseTestBasic(BaseTest):
 
-    enable_opts = "intbounds:rewrite:virtualize:string:heap"
+    enable_opts = "intbounds:rewrite:virtualize:string:earlyforce:pure:heap"
 
     def optimize_loop(self, ops, optops, call_pure_results=None):
 
@@ -651,8 +652,8 @@
         i3 = getfield_gc(p3, descr=valuedescr)
         escape(i3)
         p1 = new_with_vtable(ConstClass(node_vtable))
+        p1sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p1, i1, descr=valuedescr)
-        p1sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p1sub, i1, descr=valuedescr)
         setfield_gc(p1, p1sub, descr=nextdescr)
         jump(i1, p1, p2)
@@ -667,10 +668,10 @@
         p3sub = getfield_gc(p3, descr=nextdescr)
         i3 = getfield_gc(p3sub, descr=valuedescr)
         escape(i3)
+        p1 = new_with_vtable(ConstClass(node_vtable))
         p2sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p2sub, i1, descr=valuedescr)
         setfield_gc(p2, p2sub, descr=nextdescr)
-        p1 = new_with_vtable(ConstClass(node_vtable))
         jump(i1, p1, p2)
         """
         # The same as test_p123_simple, but in the end the "old" p2 contains
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizefficall.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizefficall.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizefficall.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizefficall.py
@@ -36,7 +36,7 @@
 
 class TestFfiCall(BaseTestBasic, LLtypeMixin):
 
-    enable_opts = "intbounds:rewrite:virtualize:string:heap:ffi"
+    enable_opts = "intbounds:rewrite:virtualize:string:pure:earlyforce:heap:ffi"
 
     class namespace:
         cpu = LLtypeMixin.cpu
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -68,7 +68,7 @@
 
 class BaseTestWithUnroll(BaseTest):
 
-    enable_opts = "intbounds:rewrite:virtualize:string:heap:unroll"
+    enable_opts = "intbounds:rewrite:virtualize:string:earlyforce:pure:heap:unroll"
 
     def optimize_loop(self, ops, expected, expected_preamble=None,
                       call_pure_results=None, expected_short=None):
@@ -825,8 +825,8 @@
         i3 = getfield_gc(p2, descr=valuedescr)
         escape(i3)
         p4 = new_with_vtable(ConstClass(node_vtable))
+        p1sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p4, i1, descr=valuedescr)
-        p1sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p1sub, i1, descr=valuedescr)
         setfield_gc(p4, p1sub, descr=nextdescr)
         jump(i1, p4)
@@ -865,13 +865,7 @@
         p3sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p3sub, i1, descr=valuedescr)
         setfield_gc(p1, p3sub, descr=nextdescr)
-        # XXX: We get two extra operations here because the setfield
-        #      above is the result of forcing p1 and thus not
-        #      registered with the heap optimizer. I've makred tests
-        #      below with VIRTUALHEAP if they suffer from this issue
-        p3sub2 = getfield_gc(p1, descr=nextdescr)
-        guard_nonnull_class(p3sub2, ConstClass(node_vtable2)) []
-        jump(i1, p1, p3sub2)
+        jump(i1, p1, p3sub)
         """
         self.optimize_loop(ops, expected, preamble)
 
@@ -902,9 +896,7 @@
         guard_true(i2b) []
         p3 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3, i2, descr=nextdescr)
-        # XXX: VIRTUALHEAP (see above)
-        i3 = getfield_gc(p3, descr=nextdescr)
-        jump(p3, i3)
+        jump(p3, i2)
         """
         self.optimize_loop(ops, expected, preamble)
 
@@ -1219,7 +1211,15 @@
         setfield_gc(p3, p30, descr=valuedescr)
         jump(i29, p30, p3)
         """
-        expected = preamble
+        expected = """
+        [i0, p1, p3]
+        i28 = int_add(i0, 1)
+        i29 = int_add(i28, 1)
+        p30 = new_with_vtable(ConstClass(node_vtable))
+        setfield_gc(p3, p30, descr=valuedescr)
+        setfield_gc(p30, i28, descr=nextdescr)
+        jump(i29, p30, p3)
+        """
         self.optimize_loop(ops, expected, preamble)
 
     def test_nonvirtual_1(self):
@@ -2408,8 +2408,8 @@
         guard_class(p2, ConstClass(node_vtable)) []
         p3 = getfield_gc(p1, descr=otherdescr)
         guard_class(p3, ConstClass(node_vtable)) []
+        p3a = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3, p2, descr=otherdescr)
-        p3a = new_with_vtable(ConstClass(node_vtable))
         escape(p3a)
         jump(p3a)
         """
@@ -2421,9 +2421,9 @@
         # setfield_gc(p3, p2, descr=otherdescr) # p3a.other = p2a
         # p1a = new_with_vtable(ConstClass(node_vtable2))
         # p2a = new_with_vtable(ConstClass(node_vtable))
+        p3anew = new_with_vtable(ConstClass(node_vtable))
         p2 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3a, p2, descr=otherdescr) # p3a.other = p2a
-        p3anew = new_with_vtable(ConstClass(node_vtable))
         escape(p3anew)
         jump(p3anew)
         """
@@ -2458,9 +2458,9 @@
         p3 = getfield_gc(p1, descr=otherdescr)
         guard_class(p3, ConstClass(node_vtable)) []
         # p1a = new_with_vtable(ConstClass(node_vtable2))
+        p3a = new_with_vtable(ConstClass(node_vtable))
         p2a = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3, p2a, descr=otherdescr)
-        p3a = new_with_vtable(ConstClass(node_vtable))
         escape(p3a)
         # setfield_gc(p1a, p2a, descr=nextdescr)
         # setfield_gc(p1a, p3a, descr=otherdescr)
@@ -2468,9 +2468,9 @@
         """
         expected = """
         [p2, p3]
+        p3a = new_with_vtable(ConstClass(node_vtable))
         p2a = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3, p2a, descr=otherdescr)
-        p3a = new_with_vtable(ConstClass(node_vtable))
         escape(p3a)
         jump(p2a, p3a)
         """
@@ -2790,7 +2790,6 @@
         self.optimize_loop(ops, expected, preamble)
 
     def test_remove_duplicate_pure_op_ovf_with_lazy_setfield(self):
-        py.test.skip('this optimization is not yet supprted')
         ops = """
         [i1, p1]
         i3 = int_add_ovf(i1, 1)
@@ -5961,13 +5960,18 @@
         escape(i0)
         jump(p1)
         """
-        expected = """
+        preamble = """
         [p1]
         i0 = ptr_eq(p1, NULL)
         escape(i0)
-        jump(p1)
-        """
-        self.optimize_strunicode_loop_extradescrs(ops, expected, expected)
+        jump(p1, i0)
+        """
+        expected = """
+        [p1, i0]
+        escape(i0)
+        jump(p1, i0)
+        """
+        self.optimize_strunicode_loop_extradescrs(ops, expected, preamble)
 
     def test_str_equal_none2(self):
         ops = """
@@ -5976,13 +5980,18 @@
         escape(i0)
         jump(p1)
         """
-        expected = """
+        preamble = """
         [p1]
         i0 = ptr_eq(p1, NULL)
         escape(i0)
-        jump(p1)
-        """
-        self.optimize_strunicode_loop_extradescrs(ops, expected, expected)
+        jump(p1, i0)
+        """
+        expected = """
+        [p1, i0]
+        escape(i0)
+        jump(p1, i0)
+        """
+        self.optimize_strunicode_loop_extradescrs(ops, expected, preamble)
 
     def test_str_equal_nonnull1(self):
         ops = """
@@ -7220,6 +7229,29 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_heap_cache_forced_virtuals(self):
+        ops = """
+        [i1, i2, p0]
+        p1 = new(descr=ssize)
+        setfield_gc(p1, i1, descr=adescr)
+        setfield_gc(p1, i2, descr=bdescr)
+        call(p0, p1, descr=writeadescr)
+        i3 = getfield_gc(p1, descr=adescr)
+        i4 = getfield_gc(p1, descr=bdescr)
+        jump(i3, i4, p0)
+        """
+        expected = """
+        [i1, i2, p0]
+        p1 = new(descr=ssize)
+        setfield_gc(p1, i1, descr=adescr)
+        call(p0, p1, descr=writeadescr)
+        i3 = getfield_gc(p1, descr=adescr)
+        setfield_gc(p1, i2, descr=bdescr)
+        jump(i3, i2, p0)
+        """
+        self.optimize_loop(ops, expected)
+
+
     def test_setarrayitem_followed_by_arraycopy(self):
         ops = """
         [p1, p2]
@@ -7230,6 +7262,27 @@
         """
         self.optimize_loop(ops, ops)
 
+    def test_heap_cache_virtuals_forced_by_delayed_setfield(self):
+        py.test.skip('not yet supoprted')
+        ops = """
+        [i1, p0]
+        p1 = new(descr=ssize)
+        setfield_gc(p1, i1, descr=valuedescr)
+        setfield_gc(p0, p1, descr=adescr)
+        call(p0, descr=writeadescr)
+        i2 = getfield_gc(p1, descr=valuedescr)
+        jump(i2, p0)
+        """
+        expected = """
+        [i1, p0]
+        p1 = new(descr=ssize)
+        setfield_gc(p1, i1, descr=valuedescr)
+        setfield_gc(p0, p1, descr=adescr)
+        call(p0, descr=writeadescr)
+        jump(i1, p0)
+        """
+        self.optimize_loop(ops, expected)
+
 class TestLLtype(OptimizeOptTest, LLtypeMixin):
     pass
 
diff --git a/pypy/jit/metainterp/optimizeopt/unroll.py b/pypy/jit/metainterp/optimizeopt/unroll.py
--- a/pypy/jit/metainterp/optimizeopt/unroll.py
+++ b/pypy/jit/metainterp/optimizeopt/unroll.py
@@ -75,7 +75,6 @@
         self.importable_values = {}
         self.emitting_dissabled = False
         self.emitted_guards = 0
-        self.emitted_pure_operations = {}
 
     def ensure_imported(self, value):
         if not self.emitting_dissabled and value in self.importable_values:
@@ -96,21 +95,6 @@
         new = UnrollableOptimizer(self.metainterp_sd, self.loop)
         return self._new(new)
 
-    def remember_emitting_pure(self, op):
-        self.emitted_pure_operations[op] = True
-
-    def produce_potential_short_preamble_ops(self, sb):
-        for op in self.emitted_pure_operations:
-            if op.getopnum() == rop.GETARRAYITEM_GC_PURE or \
-               op.getopnum() == rop.STRGETITEM or \
-               op.getopnum() == rop.UNICODEGETITEM:
-                if not self.getvalue(op.getarg(1)).is_constant():
-                    continue
-            sb.add_potential(op)
-        for opt in self.optimizations:
-            opt.produce_potential_short_preamble_ops(sb)
-
-
 
 class UnrollOptimizer(Optimization):
     """Unroll the loop into two iterations. The first one will
@@ -154,7 +138,7 @@
 
             KillHugeIntBounds(self.optimizer).apply()
             
-            loop.preamble.operations = self.optimizer.newoperations
+            loop.preamble.operations = self.optimizer.get_newoperations()
             jump_args = [self.getvalue(a).get_key_box() for a in jump_args]
 
             start_resumedescr = loop.preamble.start_resumedescr.clone_if_mutable()
@@ -167,8 +151,9 @@
             virtual_state = modifier.get_virtual_state(jump_args)
             
             values = [self.getvalue(arg) for arg in jump_args]
-            inputargs = virtual_state.make_inputargs(values)
-            short_inputargs = virtual_state.make_inputargs(values, keyboxes=True)
+            inputargs = virtual_state.make_inputargs(values, self.optimizer)
+            short_inputargs = virtual_state.make_inputargs(values, self.optimizer,
+                                                           keyboxes=True)
 
             self.constant_inputargs = {}
             for box in jump_args: 
@@ -197,7 +182,7 @@
             # operations needed to setup the proper state of those virtuals
             # in the peeled loop
             inputarg_setup_ops = []
-            preamble_optimizer.newoperations = []
+            preamble_optimizer.clear_newoperations()
             seen = {}
             for box in inputargs:
                 if box in seen:
@@ -211,9 +196,8 @@
                     continue
                 seen[box] = True
                 value = preamble_optimizer.getvalue(box)
-                value.force_box()
-            preamble_optimizer.flush()
-            inputarg_setup_ops += preamble_optimizer.newoperations
+                value.force_box(preamble_optimizer)
+            inputarg_setup_ops += preamble_optimizer.get_newoperations()
 
             # Setup the state of the new optimizer by emiting the
             # short preamble operations and discarding the result
@@ -244,13 +228,13 @@
                                 virtual_state)
             
             loop.inputargs = inputargs
-            args = [preamble_optimizer.getvalue(self.short_boxes.original(a)).force_box()\
+            args = [preamble_optimizer.getvalue(self.short_boxes.original(a)).force_box(preamble_optimizer)\
                     for a in inputargs]
             jmp = ResOperation(rop.JUMP, args, None)
             jmp.setdescr(loop.token)
             loop.preamble.operations.append(jmp)
 
-            loop.operations = self.optimizer.newoperations
+            loop.operations = self.optimizer.get_newoperations()
             maxguards = self.optimizer.metainterp_sd.warmrunnerdesc.memory_manager.max_retrace_guards
             
             if self.optimizer.emitted_guards > maxguards:
@@ -335,9 +319,10 @@
         assert jumpop
         original_jumpargs = jumpop.getarglist()[:]
         values = [self.getvalue(arg) for arg in jumpop.getarglist()]
-        jumpargs = virtual_state.make_inputargs(values)
+        jumpargs = virtual_state.make_inputargs(values, self.optimizer)
         jumpop.initarglist(jumpargs)
-        jmp_to_short_args = virtual_state.make_inputargs(values, keyboxes=True)
+        jmp_to_short_args = virtual_state.make_inputargs(values, self.optimizer,
+                                                         keyboxes=True)
         self.short_inliner = Inliner(short_inputargs, jmp_to_short_args)
         
         for box, const in self.constant_inputargs.items():
@@ -347,11 +332,11 @@
             newop = self.short_inliner.inline_op(op)
             self.optimizer.send_extra_operation(newop)
         
-        self.optimizer.flush()
+        newoperations = self.optimizer.get_newoperations()
 
         i = j = 0
-        while i < len(self.optimizer.newoperations) or j < len(jumpargs):
-            if i == len(self.optimizer.newoperations):
+        while i < len(newoperations) or j < len(jumpargs):
+            if i == len(newoperations):
                 while j < len(jumpargs):
                     a = jumpargs[j]
                     if self.optimizer.loop.logops:
@@ -360,7 +345,7 @@
                                     jumpargs, short_seen)
                     j += 1
             else:
-                op = self.optimizer.newoperations[i]
+                op = newoperations[i]
 
                 self.boxes_created_this_iteration[op.result] = True
                 args = op.getarglist()
@@ -375,6 +360,7 @@
                     self.import_box(a, inputargs, short, short_jumpargs,
                                     jumpargs, short_seen)
                 i += 1
+            newoperations = self.optimizer.get_newoperations()
 
         jumpop.initarglist(jumpargs)
         self.optimizer.send_extra_operation(jumpop)
@@ -468,7 +454,7 @@
         inputargs.append(box)
         box = newresult
         if box in self.optimizer.values:
-            box = self.optimizer.values[box].force_box()
+            box = self.optimizer.values[box].force_box(self.optimizer)
         jumpargs.append(box)
         
 
@@ -483,11 +469,6 @@
         if op.getopnum() == rop.JUMP:
             loop_token = op.getdescr()
             assert isinstance(loop_token, LoopToken)
-            # FIXME: Use a tree, similar to the tree formed by the full
-            # preamble and it's bridges, instead of a list to save time and
-            # memory. This should also allow better behaviour in
-            # situations that the is_emittable() chain currently cant
-            # handle and the inlining fails unexpectedly belwo.
             short = loop_token.short_preamble
             if short:
                 args = op.getarglist()
@@ -523,7 +504,7 @@
 
                         values = [self.getvalue(arg)
                                   for arg in op.getarglist()]
-                        args = sh.virtual_state.make_inputargs(values,
+                        args = sh.virtual_state.make_inputargs(values, self.optimizer,
                                                                keyboxes=True)
                         inliner = Inliner(sh.inputargs, args)
                         
diff --git a/pypy/jit/metainterp/optimizeopt/virtualize.py b/pypy/jit/metainterp/optimizeopt/virtualize.py
--- a/pypy/jit/metainterp/optimizeopt/virtualize.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualize.py
@@ -10,13 +10,12 @@
 
 
 class AbstractVirtualValue(optimizer.OptValue):
-    _attrs_ = ('optimizer', 'keybox', 'source_op', '_cached_vinfo')
+    _attrs_ = ('keybox', 'source_op', '_cached_vinfo')
     box = None
     level = optimizer.LEVEL_NONNULL
     _cached_vinfo = None
 
-    def __init__(self, optimizer, keybox, source_op=None):
-        self.optimizer = optimizer
+    def __init__(self, keybox, source_op=None):
         self.keybox = keybox   # only used as a key in dictionaries
         self.source_op = source_op  # the NEW_WITH_VTABLE/NEW_ARRAY operation
                                     # that builds this box
@@ -29,17 +28,17 @@
             return self.keybox
         return self.box
 
-    def force_box(self):
+    def force_box(self, optforce):
         if self.box is None:
-            self.optimizer.forget_numberings(self.keybox)
-            self._really_force()
+            optforce.forget_numberings(self.keybox)
+            self._really_force(optforce)
         return self.box
 
-    def force_at_end_of_preamble(self, already_forced):
+    def force_at_end_of_preamble(self, already_forced, optforce):
         value = already_forced.get(self, None)
         if value:
             return value
-        return OptValue(self.force_box())
+        return OptValue(self.force_box(optforce))
 
     def make_virtual_info(self, modifier, fieldnums):
         if fieldnums is None:
@@ -55,7 +54,7 @@
     def _make_virtual(self, modifier):
         raise NotImplementedError("abstract base")
 
-    def _really_force(self):
+    def _really_force(self, optforce):
         raise NotImplementedError("abstract base")
 
     def import_from(self, other, optimizer):
@@ -70,10 +69,11 @@
 get_fielddescrlist_cache._annspecialcase_ = "specialize:memo"
 
 class AbstractVirtualStructValue(AbstractVirtualValue):
-    _attrs_ = ('_fields', '_cached_sorted_fields')
+    _attrs_ = ('_fields', 'cpu', '_cached_sorted_fields')
 
-    def __init__(self, optimizer, keybox, source_op=None):
-        AbstractVirtualValue.__init__(self, optimizer, keybox, source_op)
+    def __init__(self, cpu, keybox, source_op=None):
+        AbstractVirtualValue.__init__(self, keybox, source_op)
+        self.cpu = cpu
         self._fields = {}
         self._cached_sorted_fields = None
 
@@ -87,45 +87,44 @@
     def _get_descr(self):
         raise NotImplementedError
 
-    def _is_immutable_and_filled_with_constants(self):
+    def _is_immutable_and_filled_with_constants(self, optforce):
         count = self._get_descr().count_fields_if_immutable()
         if count != len(self._fields):    # always the case if count == -1
             return False
         for value in self._fields.itervalues():
-            subbox = value.force_box()
+            subbox = value.force_box(optforce)
             if not isinstance(subbox, Const):
                 return False
         return True
 
-    def force_at_end_of_preamble(self, already_forced):
+    def force_at_end_of_preamble(self, already_forced, optforce):
         if self in already_forced:
             return self
         already_forced[self] = self
         if self._fields:
             for ofs in self._fields.keys():
-                self._fields[ofs] = self._fields[ofs].force_at_end_of_preamble(already_forced)
+                self._fields[ofs] = self._fields[ofs].force_at_end_of_preamble(already_forced, optforce)
         return self
 
-    def _really_force(self):
+    def _really_force(self, optforce):
         op = self.source_op
         assert op is not None
         # ^^^ This case should not occur any more (see test_bug_3).
         #
         if not we_are_translated():
             op.name = 'FORCE ' + self.source_op.name
-
-        if self._is_immutable_and_filled_with_constants():
-            box = self.optimizer.constant_fold(op)
+            
+        if self._is_immutable_and_filled_with_constants(optforce):
+            box = optforce.optimizer.constant_fold(op)
             self.make_constant(box)
             for ofs, value in self._fields.iteritems():
-                subbox = value.force_box()
+                subbox = value.force_box(optforce)
                 assert isinstance(subbox, Const)
-                execute(self.optimizer.cpu, None, rop.SETFIELD_GC,
+                execute(optforce.optimizer.cpu, None, rop.SETFIELD_GC,
                         ofs, box, subbox)
             # keep self._fields, because it's all immutable anyway
         else:
-            newoperations = self.optimizer.newoperations
-            newoperations.append(op)
+            optforce.emit_operation(op)
             self.box = box = op.result
             #
             iteritems = self._fields.iteritems()
@@ -135,10 +134,11 @@
             for ofs, value in iteritems:
                 if value.is_null():
                     continue
-                subbox = value.force_box()
+                subbox = value.force_box(optforce)
                 op = ResOperation(rop.SETFIELD_GC, [box, subbox], None,
                                   descr=ofs)
-                newoperations.append(op)
+
+                optforce.emit_operation(op)
 
     def _get_field_descr_list(self):
         _cached_sorted_fields = self._cached_sorted_fields
@@ -155,7 +155,7 @@
             else:
                 lst = self._fields.keys()
             sort_descrs(lst)
-            cache = get_fielddescrlist_cache(self.optimizer.cpu)
+            cache = get_fielddescrlist_cache(self.cpu)
             result = cache.get(lst, None)
             if result is None:
                 cache[lst] = lst
@@ -180,8 +180,8 @@
 class VirtualValue(AbstractVirtualStructValue):
     level = optimizer.LEVEL_KNOWNCLASS
 
-    def __init__(self, optimizer, known_class, keybox, source_op=None):
-        AbstractVirtualStructValue.__init__(self, optimizer, keybox, source_op)
+    def __init__(self, cpu, known_class, keybox, source_op=None):
+        AbstractVirtualStructValue.__init__(self, cpu, keybox, source_op)
         assert isinstance(known_class, Const)
         self.known_class = known_class
 
@@ -190,7 +190,7 @@
         return modifier.make_virtual(self.known_class, fielddescrs)
 
     def _get_descr(self):
-        return vtable2descr(self.optimizer.cpu, self.known_class.getint())
+        return vtable2descr(self.cpu, self.known_class.getint())
 
     def __repr__(self):
         cls_name = self.known_class.value.adr.ptr._obj._TYPE._name
@@ -201,8 +201,8 @@
 
 class VStructValue(AbstractVirtualStructValue):
 
-    def __init__(self, optimizer, structdescr, keybox, source_op=None):
-        AbstractVirtualStructValue.__init__(self, optimizer, keybox, source_op)
+    def __init__(self, cpu, structdescr, keybox, source_op=None):
+        AbstractVirtualStructValue.__init__(self, cpu, keybox, source_op)
         self.structdescr = structdescr
 
     def _make_virtual(self, modifier):
@@ -215,10 +215,10 @@
 
 class VArrayValue(AbstractVirtualValue):
 
-    def __init__(self, optimizer, arraydescr, size, keybox, source_op=None):
-        AbstractVirtualValue.__init__(self, optimizer, keybox, source_op)
+    def __init__(self, arraydescr, constvalue, size, keybox, source_op=None):
+        AbstractVirtualValue.__init__(self, keybox, source_op)
         self.arraydescr = arraydescr
-        self.constvalue = optimizer.new_const_item(arraydescr)
+        self.constvalue = constvalue
         self._items = [self.constvalue] * size
 
     def getlength(self):
@@ -232,31 +232,30 @@
         assert isinstance(itemvalue, optimizer.OptValue)
         self._items[index] = itemvalue
 
-    def force_at_end_of_preamble(self, already_forced):
+    def force_at_end_of_preamble(self, already_forced, optforce):
         if self in already_forced:
             return self
         already_forced[self] = self
         for index in range(len(self._items)):
-            self._items[index] = self._items[index].force_at_end_of_preamble(already_forced)
+            self._items[index] = self._items[index].force_at_end_of_preamble(already_forced, optforce)
         return self
     
-    def _really_force(self):
+    def _really_force(self, optforce):
         assert self.source_op is not None
         if not we_are_translated():
             self.source_op.name = 'FORCE ' + self.source_op.name
-        newoperations = self.optimizer.newoperations
-        newoperations.append(self.source_op)
+        optforce.emit_operation(self.source_op)        
         self.box = box = self.source_op.result
         for index in range(len(self._items)):
             subvalue = self._items[index]
             if subvalue is not self.constvalue:
                 if subvalue.is_null():
                     continue
-                subbox = subvalue.force_box()
+                subbox = subvalue.force_box(optforce)
                 op = ResOperation(rop.SETARRAYITEM_GC,
                                   [box, ConstInt(index), subbox], None,
                                   descr=self.arraydescr)
-                newoperations.append(op)
+                optforce.emit_operation(op)
 
     def get_args_for_fail(self, modifier):
         if self.box is None and not modifier.already_seen_virtual(self.keybox):
@@ -279,17 +278,18 @@
         return OptVirtualize()
         
     def make_virtual(self, known_class, box, source_op=None):
-        vvalue = VirtualValue(self.optimizer, known_class, box, source_op)
+        vvalue = VirtualValue(self.optimizer.cpu, known_class, box, source_op)
         self.make_equal_to(box, vvalue)
         return vvalue
 
     def make_varray(self, arraydescr, size, box, source_op=None):
-        vvalue = VArrayValue(self.optimizer, arraydescr, size, box, source_op)
+        constvalue = self.new_const_item(arraydescr)
+        vvalue = VArrayValue(arraydescr, constvalue, size, box, source_op)
         self.make_equal_to(box, vvalue)
         return vvalue
 
     def make_vstruct(self, structdescr, box, source_op=None):
-        vvalue = VStructValue(self.optimizer, structdescr, box, source_op)
+        vvalue = VStructValue(self.optimizer.cpu, structdescr, box, source_op)
         self.make_equal_to(box, vvalue)
         return vvalue
 
@@ -362,7 +362,6 @@
             self.make_equal_to(op.result, fieldvalue)
         else:
             value.ensure_nonnull()
-            ###self.heap_op_optimizer.optimize_GETFIELD_GC(op, value)
             self.emit_operation(op)
 
     # note: the following line does not mean that the two operations are
@@ -377,7 +376,6 @@
             value.setfield(op.getdescr(), fieldvalue)
         else:
             value.ensure_nonnull()
-            ###self.heap_op_optimizer.optimize_SETFIELD_GC(op, value, fieldvalue)
             self.emit_operation(op)
 
     def optimize_NEW_WITH_VTABLE(self, op):
@@ -417,7 +415,6 @@
                 self.make_equal_to(op.result, itemvalue)
                 return
         value.ensure_nonnull()
-        ###self.heap_op_optimizer.optimize_GETARRAYITEM_GC(op, value)
         self.emit_operation(op)
 
     # note: the following line does not mean that the two operations are
@@ -432,7 +429,6 @@
                 value.setitem(indexbox.getint(), self.getvalue(op.getarg(2)))
                 return
         value.ensure_nonnull()
-        ###self.heap_op_optimizer.optimize_SETARRAYITEM_GC(op, value, fieldvalue)
         self.emit_operation(op)
 
 
diff --git a/pypy/jit/metainterp/optimizeopt/virtualstate.py b/pypy/jit/metainterp/optimizeopt/virtualstate.py
--- a/pypy/jit/metainterp/optimizeopt/virtualstate.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualstate.py
@@ -30,7 +30,7 @@
     def _generate_guards(self, other, box, cpu, extra_guards):
         raise InvalidLoop
 
-    def enum_forced_boxes(self, boxes, value):
+    def enum_forced_boxes(self, boxes, value, optimizer):
         raise NotImplementedError
 
     def enum(self, virtual_state):
@@ -100,14 +100,14 @@
     def _generalization_of(self, other):
         raise NotImplementedError
 
-    def enum_forced_boxes(self, boxes, value):
+    def enum_forced_boxes(self, boxes, value, optimizer):
         assert isinstance(value, virtualize.AbstractVirtualStructValue)
         assert value.is_virtual()
         for i in range(len(self.fielddescrs)):
             v = value._fields[self.fielddescrs[i]]
             s = self.fieldstate[i]
             if s.position > self.position:
-                s.enum_forced_boxes(boxes, v)
+                s.enum_forced_boxes(boxes, v, optimizer)
 
     def _enum(self, virtual_state):
         for s in self.fieldstate:
@@ -177,14 +177,14 @@
                 return False
         return True
 
-    def enum_forced_boxes(self, boxes, value):
+    def enum_forced_boxes(self, boxes, value, optimizer):
         assert isinstance(value, virtualize.VArrayValue)
         assert value.is_virtual()
         for i in range(len(self.fieldstate)):
             v = value._items[i]
             s = self.fieldstate[i]
             if s.position > self.position:
-                s.enum_forced_boxes(boxes, v)
+                s.enum_forced_boxes(boxes, v, optimizer)
 
     def _enum(self, virtual_state):
         for s in self.fieldstate:
@@ -316,11 +316,11 @@
             import pdb; pdb.set_trace()
             raise NotImplementedError
 
-    def enum_forced_boxes(self, boxes, value):
+    def enum_forced_boxes(self, boxes, value, optimizer):
         if self.level == LEVEL_CONSTANT:
             return
         assert 0 <= self.position_in_notvirtuals 
-        boxes[self.position_in_notvirtuals] = value.force_box()
+        boxes[self.position_in_notvirtuals] = value.force_box(optimizer)
 
     def _enum(self, virtual_state):
         if self.level == LEVEL_CONSTANT:
@@ -377,11 +377,13 @@
             self.state[i].generate_guards(other.state[i], args[i],
                                           cpu, extra_guards, renum)
 
-    def make_inputargs(self, values, keyboxes=False):
+    def make_inputargs(self, values, optimizer, keyboxes=False):
+        if optimizer.optearlyforce:
+            optimizer = optimizer.optearlyforce
         assert len(values) == len(self.state)
         inputargs = [None] * len(self.notvirtuals)
         for i in range(len(values)):
-            self.state[i].enum_forced_boxes(inputargs, values[i])
+            self.state[i].enum_forced_boxes(inputargs, values[i], optimizer)
 
         if keyboxes:
             for i in range(len(values)):
@@ -434,7 +436,12 @@
     def get_virtual_state(self, jump_args):
         self.optimizer.force_at_end_of_preamble()
         already_forced = {}
-        values = [self.getvalue(box).force_at_end_of_preamble(already_forced)
+        if self.optimizer.optearlyforce:
+            opt = self.optimizer.optearlyforce
+        else:
+            opt = self.optimizer
+        values = [self.getvalue(box).force_at_end_of_preamble(already_forced,
+                                                              opt)
                   for box in jump_args]
 
         for value in values:
diff --git a/pypy/jit/metainterp/optimizeopt/vstring.py b/pypy/jit/metainterp/optimizeopt/vstring.py
--- a/pypy/jit/metainterp/optimizeopt/vstring.py
+++ b/pypy/jit/metainterp/optimizeopt/vstring.py
@@ -43,7 +43,7 @@
 class __extend__(optimizer.OptValue):
     """New methods added to the base class OptValue for this file."""
 
-    def getstrlen(self, optimization, mode):
+    def getstrlen(self, string_optimizer, mode):
         if mode is mode_string:
             s = self.get_constant_string_spec(mode_string)
             if s is not None:
@@ -52,12 +52,12 @@
             s = self.get_constant_string_spec(mode_unicode)
             if s is not None:
                 return ConstInt(len(s))
-        if optimization is None:
+        if string_optimizer is None:
             return None
         self.ensure_nonnull()
-        box = self.force_box()
+        box = self.force_box(string_optimizer)
         lengthbox = BoxInt()
-        optimization.propagate_forward(ResOperation(mode.STRLEN, [box], lengthbox))
+        string_optimizer.emit_operation(ResOperation(mode.STRLEN, [box], lengthbox))
         return lengthbox
 
     @specialize.arg(1)
@@ -68,25 +68,25 @@
         else:
             return None
 
-    def string_copy_parts(self, optimizer, targetbox, offsetbox, mode):
+    def string_copy_parts(self, string_optimizer, targetbox, offsetbox, mode):
         # Copies the pointer-to-string 'self' into the target string
         # given by 'targetbox', at the specified offset.  Returns the offset
         # at the end of the copy.
-        lengthbox = self.getstrlen(optimizer, mode)
-        srcbox = self.force_box()
-        return copy_str_content(optimizer, srcbox, targetbox,
+        lengthbox = self.getstrlen(string_optimizer, mode)
+        srcbox = self.force_box(string_optimizer)
+        return copy_str_content(string_optimizer, srcbox, targetbox,
                                 CONST_0, offsetbox, lengthbox, mode)
 
 
 class VAbstractStringValue(virtualize.AbstractVirtualValue):
     _attrs_ = ('mode',)
 
-    def __init__(self, optimizer, keybox, source_op, mode):
-        virtualize.AbstractVirtualValue.__init__(self, optimizer, keybox,
+    def __init__(self, keybox, source_op, mode):
+        virtualize.AbstractVirtualValue.__init__(self, keybox,
                                                  source_op)
         self.mode = mode
 
-    def _really_force(self):
+    def _really_force(self, optforce):
         if self.mode is mode_string:
             s = self.get_constant_string_spec(mode_string)
             if s is not None:
@@ -101,12 +101,12 @@
                 return
         assert self.source_op is not None
         self.box = box = self.source_op.result
-        lengthbox = self.getstrlen(self.optimizer, self.mode)
+        lengthbox = self.getstrlen(optforce, self.mode)
         op = ResOperation(self.mode.NEWSTR, [lengthbox], box)
         if not we_are_translated():
             op.name = 'FORCE'
-        self.optimizer.emit_operation(op)
-        self.string_copy_parts(self.optimizer, box, CONST_0, self.mode)
+        optforce.emit_operation(op)
+        self.string_copy_parts(optforce, box, CONST_0, self.mode)
 
 
 class VStringPlainValue(VAbstractStringValue):
@@ -140,20 +140,20 @@
         return mode.emptystr.join([mode.chr(c.box.getint())
                                    for c in self._chars])
 
-    def string_copy_parts(self, optimizer, targetbox, offsetbox, mode):
+    def string_copy_parts(self, string_optimizer, targetbox, offsetbox, mode):
         if not self.is_virtual() and targetbox is not self.box:
-            lengthbox = self.getstrlen(optimizer, mode)
-            srcbox = self.force_box()
-            return copy_str_content(optimizer, srcbox, targetbox,
+            lengthbox = self.getstrlen(string_optimizer, mode)
+            srcbox = self.force_box(string_optimizer)
+            return copy_str_content(string_optimizer, srcbox, targetbox,
                                 CONST_0, offsetbox, lengthbox, mode)
         for i in range(len(self._chars)):
-            charbox = self._chars[i].force_box()
+            charbox = self._chars[i].force_box(string_optimizer)
             if not (isinstance(charbox, Const) and charbox.same_constant(CONST_0)):
-                optimizer.emit_operation(ResOperation(mode.STRSETITEM, [targetbox,
-                                                                    offsetbox,
-                                                                    charbox],
+                string_optimizer.emit_operation(ResOperation(mode.STRSETITEM, [targetbox,
+                                                                               offsetbox,
+                                                                               charbox],
                                                   None))
-            offsetbox = _int_add(optimizer, offsetbox, CONST_1)
+            offsetbox = _int_add(string_optimizer, offsetbox, CONST_1)
         return offsetbox
 
     def get_args_for_fail(self, modifier):
@@ -187,16 +187,16 @@
         self.left = left
         self.right = right
 
-    def getstrlen(self, optimizer, mode):
+    def getstrlen(self, string_optimizer, mode):
         if self.lengthbox is None:
-            len1box = self.left.getstrlen(optimizer, mode)
+            len1box = self.left.getstrlen(string_optimizer, mode)
             if len1box is None:
                 return None
-            len2box = self.right.getstrlen(optimizer, mode)
+            len2box = self.right.getstrlen(string_optimizer, mode)
             if len2box is None:
                 return None
-            self.lengthbox = _int_add(optimizer, len1box, len2box)
-            # ^^^ may still be None, if optimizer is None
+            self.lengthbox = _int_add(string_optimizer, len1box, len2box)
+            # ^^^ may still be None, if string_optimizer is None
         return self.lengthbox
 
     @specialize.arg(1)
@@ -209,10 +209,10 @@
             return None
         return s1 + s2
 
-    def string_copy_parts(self, optimizer, targetbox, offsetbox, mode):
-        offsetbox = self.left.string_copy_parts(optimizer, targetbox,
+    def string_copy_parts(self, string_optimizer, targetbox, offsetbox, mode):
+        offsetbox = self.left.string_copy_parts(string_optimizer, targetbox,
                                                 offsetbox, mode)
-        offsetbox = self.right.string_copy_parts(optimizer, targetbox,
+        offsetbox = self.right.string_copy_parts(string_optimizer, targetbox,
                                                  offsetbox, mode)
         return offsetbox
 
@@ -251,8 +251,8 @@
         self.vstart = vstart
         self.vlength = vlength
 
-    def getstrlen(self, _, mode):
-        return self.vlength.force_box()
+    def getstrlen(self, optforce, mode):
+        return self.vlength.force_box(optforce)
 
     @specialize.arg(1)
     def get_constant_string_spec(self, mode):
@@ -267,11 +267,11 @@
             return s1[start : start + length]
         return None
 
-    def string_copy_parts(self, optimizer, targetbox, offsetbox, mode):
-        lengthbox = self.getstrlen(optimizer, mode)
-        return copy_str_content(optimizer,
-                                self.vstr.force_box(), targetbox,
-                                self.vstart.force_box(), offsetbox,
+    def string_copy_parts(self, string_optimizer, targetbox, offsetbox, mode):
+        lengthbox = self.getstrlen(string_optimizer, mode)
+        return copy_str_content(string_optimizer,
+                                self.vstr.force_box(string_optimizer), targetbox,
+                                self.vstart.force_box(string_optimizer), offsetbox,
                                 lengthbox, mode)
 
     def get_args_for_fail(self, modifier):
@@ -300,7 +300,7 @@
         return modifier.make_vstrslice(self.mode is mode_unicode)
 
 
-def copy_str_content(optimizer, srcbox, targetbox,
+def copy_str_content(string_optimizer, srcbox, targetbox,
                      srcoffsetbox, offsetbox, lengthbox, mode, need_next_offset=True):
     if isinstance(srcbox, ConstPtr) and isinstance(srcoffsetbox, Const):
         M = 5
@@ -310,26 +310,26 @@
         # up to M characters are done "inline", i.e. with STRGETITEM/STRSETITEM
         # instead of just a COPYSTRCONTENT.
         for i in range(lengthbox.value):
-            charbox = _strgetitem(optimizer, srcbox, srcoffsetbox, mode)
-            srcoffsetbox = _int_add(optimizer, srcoffsetbox, CONST_1)
-            optimizer.emit_operation(ResOperation(mode.STRSETITEM, [targetbox,
-                                                                offsetbox,
-                                                                charbox],
+            charbox = _strgetitem(string_optimizer, srcbox, srcoffsetbox, mode)
+            srcoffsetbox = _int_add(string_optimizer, srcoffsetbox, CONST_1)
+            string_optimizer.emit_operation(ResOperation(mode.STRSETITEM, [targetbox,
+                                                                           offsetbox,
+                                                                           charbox],
                                               None))
-            offsetbox = _int_add(optimizer, offsetbox, CONST_1)
+            offsetbox = _int_add(string_optimizer, offsetbox, CONST_1)
     else:
         if need_next_offset:
-            nextoffsetbox = _int_add(optimizer, offsetbox, lengthbox)
+            nextoffsetbox = _int_add(string_optimizer, offsetbox, lengthbox)
         else:
             nextoffsetbox = None
         op = ResOperation(mode.COPYSTRCONTENT, [srcbox, targetbox,
                                                 srcoffsetbox, offsetbox,
                                                 lengthbox], None)
-        optimizer.emit_operation(op)
+        string_optimizer.emit_operation(op)
         offsetbox = nextoffsetbox
     return offsetbox
 
-def _int_add(optimizer, box1, box2):
+def _int_add(string_optimizer, box1, box2):
     if isinstance(box1, ConstInt):
         if box1.value == 0:
             return box2
@@ -337,23 +337,23 @@
             return ConstInt(box1.value + box2.value)
     elif isinstance(box2, ConstInt) and box2.value == 0:
         return box1
-    if optimizer is None:
+    if string_optimizer is None:
         return None
     resbox = BoxInt()
-    optimizer.propagate_forward(ResOperation(rop.INT_ADD, [box1, box2], resbox))
+    string_optimizer.emit_operation(ResOperation(rop.INT_ADD, [box1, box2], resbox))
     return resbox
 
-def _int_sub(optimizer, box1, box2):
+def _int_sub(string_optimizer, box1, box2):
     if isinstance(box2, ConstInt):
         if box2.value == 0:
             return box1
         if isinstance(box1, ConstInt):
             return ConstInt(box1.value - box2.value)
     resbox = BoxInt()
-    optimizer.propagate_forward(ResOperation(rop.INT_SUB, [box1, box2], resbox))
+    string_optimizer.emit_operation(ResOperation(rop.INT_SUB, [box1, box2], resbox))
     return resbox
 
-def _strgetitem(optimizer, strbox, indexbox, mode):
+def _strgetitem(string_optimizer, strbox, indexbox, mode):
     if isinstance(strbox, ConstPtr) and isinstance(indexbox, ConstInt):
         if mode is mode_string:
             s = strbox.getref(lltype.Ptr(rstr.STR))
@@ -362,30 +362,28 @@
             s = strbox.getref(lltype.Ptr(rstr.UNICODE))
             return ConstInt(ord(s.chars[indexbox.getint()]))
     resbox = BoxInt()
-    optimizer.propagate_forward(ResOperation(mode.STRGETITEM, [strbox, indexbox],
-                                      resbox))
+    string_optimizer.emit_operation(ResOperation(mode.STRGETITEM, [strbox, indexbox],
+                                                 resbox))
     return resbox
 
 
 class OptString(optimizer.Optimization):
     "Handling of strings and unicodes."
-    enabled = True
-
     def new(self):
         return OptString()
 
     def make_vstring_plain(self, box, source_op, mode):
-        vvalue = VStringPlainValue(self.optimizer, box, source_op, mode)
+        vvalue = VStringPlainValue(box, source_op, mode)
         self.make_equal_to(box, vvalue)
         return vvalue
 
     def make_vstring_concat(self, box, source_op, mode):
-        vvalue = VStringConcatValue(self.optimizer, box, source_op, mode)
+        vvalue = VStringConcatValue(box, source_op, mode)
         self.make_equal_to(box, vvalue)
         return vvalue
 
     def make_vstring_slice(self, box, source_op, mode):
-        vvalue = VStringSliceValue(self.optimizer, box, source_op, mode)
+        vvalue = VStringSliceValue(box, source_op, mode)
         self.make_equal_to(box, vvalue)
         return vvalue
 
@@ -435,9 +433,9 @@
         value.ensure_nonnull()
         #
         if value.is_virtual() and isinstance(value, VStringSliceValue):
-            fullindexbox = _int_add(self.optimizer,
-                                    value.vstart.force_box(),
-                                    vindex.force_box())
+            fullindexbox = _int_add(self,
+                                    value.vstart.force_box(self),
+                                    vindex.force_box(self))
             value = value.vstr
             vindex = self.getvalue(fullindexbox)
         #
@@ -449,7 +447,7 @@
                 if res is not optimizer.CVAL_UNINITIALIZED_ZERO:
                     return res
         #
-        resbox = _strgetitem(self.optimizer, value.force_box(), vindex.force_box(), mode)
+        resbox = _strgetitem(self, value.force_box(self), vindex.force_box(self), mode)
         return self.getvalue(resbox)
 
     def optimize_STRLEN(self, op):
@@ -459,7 +457,7 @@
 
     def _optimize_STRLEN(self, op, mode):
         value = self.getvalue(op.getarg(0))
-        lengthbox = value.getstrlen(self.optimizer, mode)
+        lengthbox = value.getstrlen(self, mode)
         self.make_equal_to(op.result, self.getvalue(lengthbox))
 
     def optimize_COPYSTRCONTENT(self, op):
@@ -477,12 +475,12 @@
 
         if length.is_constant() and length.box.getint() == 0:
             return
-        copy_str_content(self.optimizer,
-            src.force_box(),
-            dst.force_box(),
-            srcstart.force_box(),
-            dststart.force_box(),
-            length.force_box(),
+        copy_str_content(self,
+            src.force_box(self),
+            dst.force_box(self),
+            srcstart.force_box(self),
+            dststart.force_box(self),
+            length.force_box(self),
             mode, need_next_offset=False
         )
 
@@ -508,6 +506,8 @@
                     return
         self.emit_operation(op)
 
+    optimize_CALL_PURE = optimize_CALL
+
     def opt_call_str_STR2UNICODE(self, op):
         # Constant-fold unicode("constant string").
         # More generally, supporting non-constant but virtual cases is
@@ -547,16 +547,16 @@
             return True
         #
         vstr.ensure_nonnull()
-        lengthbox = _int_sub(self.optimizer, vstop.force_box(),
-                                            vstart.force_box())
+        lengthbox = _int_sub(self, vstop.force_box(self),
+                                   vstart.force_box(self))
         #
         if isinstance(vstr, VStringSliceValue):
             # double slicing  s[i:j][k:l]
             vintermediate = vstr
             vstr = vintermediate.vstr
-            startbox = _int_add(self.optimizer,
-                                vintermediate.vstart.force_box(),
-                                vstart.force_box())
+            startbox = _int_add(self,
+                                vintermediate.vstart.force_box(self),
+                                vstart.force_box(self))
             vstart = self.getvalue(startbox)
         #
         value = self.make_vstring_slice(op.result, op, mode)
@@ -594,8 +594,8 @@
                 do = EffectInfo.OS_STREQ_LENGTHOK
             else:
                 do = EffectInfo.OS_STREQ_NONNULL
-            self.generate_modified_call(do, [v1.force_box(),
-                                             v2.force_box()], op.result, mode)
+            self.generate_modified_call(do, [v1.force_box(self),
+                                             v2.force_box(self)], op.result, mode)
             return True
         return False
 
@@ -603,7 +603,7 @@
         l2box = v2.getstrlen(None, mode)
         if isinstance(l2box, ConstInt):
             if l2box.value == 0:
-                lengthbox = v1.getstrlen(self.optimizer, mode)
+                lengthbox = v1.getstrlen(self, mode)
                 seo = self.optimizer.send_extra_operation
                 seo(ResOperation(rop.INT_EQ, [lengthbox, CONST_0], resultbox))
                 return True
@@ -614,17 +614,17 @@
                     vchar1 = self.strgetitem(v1, optimizer.CVAL_ZERO, mode)
                     vchar2 = self.strgetitem(v2, optimizer.CVAL_ZERO, mode)
                     seo = self.optimizer.send_extra_operation
-                    seo(ResOperation(rop.INT_EQ, [vchar1.force_box(),
-                                                  vchar2.force_box()],
+                    seo(ResOperation(rop.INT_EQ, [vchar1.force_box(self),
+                                                  vchar2.force_box(self)],
                                      resultbox))
                     return True
                 if isinstance(v1, VStringSliceValue):
                     vchar = self.strgetitem(v2, optimizer.CVAL_ZERO, mode)
                     do = EffectInfo.OS_STREQ_SLICE_CHAR
-                    self.generate_modified_call(do, [v1.vstr.force_box(),
-                                                     v1.vstart.force_box(),
-                                                     v1.vlength.force_box(),
-                                                     vchar.force_box()],
+                    self.generate_modified_call(do, [v1.vstr.force_box(self),
+                                                     v1.vstart.force_box(self),
+                                                     v1.vlength.force_box(self),
+                                                     vchar.force_box(self)],
                                                 resultbox, mode)
                     return True
         #
@@ -635,10 +635,10 @@
             if v1.is_null():
                 self.make_constant(resultbox, CONST_1)
                 return True
-            op = ResOperation(rop.PTR_EQ, [v1.force_box(),
+            op = ResOperation(rop.PTR_EQ, [v1.force_box(self),
                                            llhelper.CONST_NULL],
                               resultbox)
-            self.optimizer.emit_operation(op)
+            self.emit_operation(op)
             return True
         #
         return False
@@ -652,8 +652,8 @@
                     do = EffectInfo.OS_STREQ_NONNULL_CHAR
                 else:
                     do = EffectInfo.OS_STREQ_CHECKNULL_CHAR
-                self.generate_modified_call(do, [v1.force_box(),
-                                                 vchar.force_box()], resultbox,
+                self.generate_modified_call(do, [v1.force_box(self),
+                                                 vchar.force_box(self)], resultbox,
                                             mode)
                 return True
         #
@@ -662,10 +662,10 @@
                 do = EffectInfo.OS_STREQ_SLICE_NONNULL
             else:
                 do = EffectInfo.OS_STREQ_SLICE_CHECKNULL
-            self.generate_modified_call(do, [v1.vstr.force_box(),
-                                             v1.vstart.force_box(),
-                                             v1.vlength.force_box(),
-                                             v2.force_box()], resultbox, mode)
+            self.generate_modified_call(do, [v1.vstr.force_box(self),
+                                             v1.vstart.force_box(self),
+                                             v1.vlength.force_box(self),
+                                             v2.force_box(self)], resultbox, mode)
             return True
         return False
 
@@ -675,16 +675,11 @@
         calldescr, func = cic.callinfo_for_oopspec(oopspecindex)
         op = ResOperation(rop.CALL, [ConstInt(func)] + args, result,
                           descr=calldescr)
-        self.optimizer.emit_operation(op)
+        self.emit_operation(op)
 
     def propagate_forward(self, op):
-        if not self.enabled:
-            self.emit_operation(op)
-            return
-
         dispatch_opt(self, op)
 
-
 dispatch_opt = make_dispatcher_method(OptString, 'optimize_',
         default=OptString.emit_operation)
 
diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py
--- a/pypy/jit/metainterp/resoperation.py
+++ b/pypy/jit/metainterp/resoperation.py
@@ -221,7 +221,6 @@
         newop.setfailargs(self.getfailargs())
         return newop
 
-
 # ============
 # arity mixins
 # ============
diff --git a/pypy/jit/metainterp/test/test_resume.py b/pypy/jit/metainterp/test/test_resume.py
--- a/pypy/jit/metainterp/test/test_resume.py
+++ b/pypy/jit/metainterp/test/test_resume.py
@@ -576,8 +576,9 @@
 
 
 class FakeOptimizer_VirtualValue(object):
-    class cpu:
-        pass
+    class optimizer:
+        class cpu:
+            pass
 fakeoptimizer = FakeOptimizer_VirtualValue()
 
 def ConstAddr(addr, cpu):   # compatibility
@@ -1135,12 +1136,7 @@
     modifier.liveboxes = {}
     modifier.vfieldboxes = {}
 
-    class FakeOptimizer(object):
-        class cpu:
-            pass
-        def new_const_item(self, descr):
-            return None
-    v2 = VArrayValue(FakeOptimizer(), LLtypeMixin.arraydescr, 2, b2s)
+    v2 = VArrayValue(LLtypeMixin.arraydescr, None, 2, b2s)
     v2._items = [b4s, c1s]
     modifier.register_virtual_fields(b2s, [b4s, c1s])
     liveboxes = []
diff --git a/pypy/jit/metainterp/test/test_string.py b/pypy/jit/metainterp/test/test_string.py
--- a/pypy/jit/metainterp/test/test_string.py
+++ b/pypy/jit/metainterp/test/test_string.py
@@ -484,6 +484,30 @@
             return len(sa.val)
         assert self.meta_interp(f, ['a']) == f('a')
 
+    def test_string_comepare_quasiimmutable(self):
+        class Sys(object):
+            _immutable_fields_ = ["defaultencoding?"]
+            def __init__(self, s):
+                self.defaultencoding = s
+        _str = self._str
+        sys = Sys(_str('ascii'))        
+        mydriver = JitDriver(reds = ['n', 'sa'], greens = [])
+        def f(n):
+            sa = 0
+            sys.defaultencoding = _str('ascii')
+            while n:
+                mydriver.jit_merge_point(n=n, sa=sa)
+                if sys.defaultencoding == _str('ascii'):
+                    sa += 1
+                n -= 1
+            sys.defaultencoding = _str('utf-8')
+            return sa
+        assert self.meta_interp(f, [8]) == f(8)
+        self.check_loops({'int_add': 1, 'guard_true': 1, 'int_sub': 1,
+                          'jump': 1, 'int_is_true': 1,
+                          'guard_not_invalidated': 1})
+
+
 #class TestOOtype(StringTests, OOJitMixin):
 #    CALL = "oosend"
 #    CALL_PURE = "oosend_pure"
diff --git a/pypy/jit/metainterp/test/test_virtualstate.py b/pypy/jit/metainterp/test/test_virtualstate.py
--- a/pypy/jit/metainterp/test/test_virtualstate.py
+++ b/pypy/jit/metainterp/test/test_virtualstate.py
@@ -431,7 +431,7 @@
         
 
 class BaseTestBridges(BaseTest):
-    enable_opts = "intbounds:rewrite:virtualize:string:heap:unroll"
+    enable_opts = "intbounds:rewrite:virtualize:string:pure:heap:unroll"
 
     def _do_optimize_bridge(self, bridge, call_pure_results):
         from pypy.jit.metainterp.optimizeopt import optimize_bridge_1, build_opt_chain
diff --git a/pypy/module/pypyjit/test_pypy_c/test_00_model.py b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
--- a/pypy/module/pypyjit/test_pypy_c/test_00_model.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
@@ -50,8 +50,7 @@
         cmdline.append(str(self.filepath))
         #
         print cmdline, logfile
-        env={'PYPYLOG': 'jit-log-opt,jit-summary:' + str(logfile)}
-        #env={'PYPYLOG': ':' + str(logfile)}
+        env={'PYPYLOG': 'jit-log-opt,jit-log-noopt,jit-summary:' + str(logfile)}
         pipe = subprocess.Popen(cmdline,
                                 env=env,
                                 stdout=subprocess.PIPE,
diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py
--- a/pypy/module/pypyjit/test_pypy_c/test_call.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_call.py
@@ -366,12 +366,12 @@
             # make sure that the "block" is not allocated
             ...
             i20 = force_token()
-            setfield_gc(p0, i20, descr=<SignedFieldDescr .*PyFrame.vable_token .*>)
             p22 = new_with_vtable(19511408)
             p24 = new_array(1, descr=<GcPtrArrayDescr>)
             p26 = new_with_vtable(ConstClass(W_ListObject))
             p27 = new(descr=<SizeDescr .*>)
             p29 = new_array(0, descr=<GcPtrArrayDescr>)
+            setfield_gc(p0, i20, descr=<SignedFieldDescr .*PyFrame.vable_token .*>)
             setfield_gc(p27, p29, descr=<GcPtrFieldDescr list.items .*>)
             setfield_gc(p26, p27, descr=<.* .*W_ListObject.inst_wrappeditems .*>)
             setarrayitem_gc(p24, 0, p26, descr=<GcPtrArrayDescr>)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_generators.py b/pypy/module/pypyjit/test_pypy_c/test_generators.py
--- a/pypy/module/pypyjit/test_pypy_c/test_generators.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_generators.py
@@ -19,8 +19,8 @@
         assert loop.match_by_id("generator", """
             i16 = force_token()
             p45 = new_with_vtable(ConstClass(W_IntObject))
-            setfield_gc(p45, i29, descr=<SignedFieldDescr .*>)
             i47 = arraylen_gc(p8, descr=<GcPtrArrayDescr>) # Should be removed by backend
             setarrayitem_gc(p8, 0, p45, descr=<GcPtrArrayDescr>)
+            setfield_gc(p45, i29, descr=<SignedFieldDescr .*>)
             jump(..., descr=...)
             """)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_instance.py b/pypy/module/pypyjit/test_pypy_c/test_instance.py
--- a/pypy/module/pypyjit/test_pypy_c/test_instance.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_instance.py
@@ -125,8 +125,8 @@
             i12 = force_token()
             --TICK--
             p20 = new_with_vtable(ConstClass(W_IntObject))
+            setfield_gc(ConstPtr(ptr21), p20, descr=<GcPtrFieldDescr .*TypeCell.inst_w_value .*>)
             setfield_gc(p20, i11, descr=<SignedFieldDescr.*W_IntObject.inst_intval .*>)
-            setfield_gc(ConstPtr(ptr21), p20, descr=<GcPtrFieldDescr .*TypeCell.inst_w_value .*>)
             jump(p0, p1, p2, p3, p4, p20, p6, i7, p20, descr=<Loop.>)
         """)
 

From noreply at buildbot.pypy.org  Mon Oct  3 21:44:35 2011
From: noreply at buildbot.pypy.org (justinpeel)
Date: Mon,  3 Oct 2011 21:44:35 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge unsigned-dtypes branch
Message-ID: <20111003194435.D432D820D9@wyvern.cs.uni-duesseldorf.de>

Author: Justin Peel <notmuchtotell at gmail.com>
Branch: 
Changeset: r47805:1255f9205eda
Date: 2011-10-03 13:44 -0600
http://bitbucket.org/pypy/pypy/changeset/1255f9205eda/

Log:	merge unsigned-dtypes branch

diff --git a/pypy/jit/codewriter/effectinfo.py b/pypy/jit/codewriter/effectinfo.py
--- a/pypy/jit/codewriter/effectinfo.py
+++ b/pypy/jit/codewriter/effectinfo.py
@@ -74,6 +74,7 @@
     OS_LLONG_UGE                = 91
     OS_LLONG_URSHIFT            = 92
     OS_LLONG_FROM_UINT          = 93
+    OS_LLONG_U_TO_FLOAT         = 94
     #
     OS_MATH_SQRT                = 100
 
diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -440,6 +440,7 @@
     rewrite_op_ullong_mod_zer      = _do_builtin_call
     rewrite_op_gc_identityhash = _do_builtin_call
     rewrite_op_gc_id           = _do_builtin_call
+    rewrite_op_uint_mod        = _do_builtin_call
 
     # ----------
     # getfield/setfield/mallocs etc.
@@ -883,9 +884,15 @@
                 v = v_arg
                 oplist = []
             if unsigned1:
-                opname = 'cast_uint_to_longlong'
+                if unsigned2:
+                    opname = 'cast_uint_to_ulonglong'
+                else:
+                    opname = 'cast_uint_to_longlong'
             else:
-                opname = 'cast_int_to_longlong'
+                if unsigned2:
+                    opname = 'cast_int_to_ulonglong'
+                else:
+                    opname = 'cast_int_to_longlong'
             op2 = self.rewrite_operation(
                 SpaceOperation(opname, [v], v_result)
             )
@@ -995,6 +1002,21 @@
                 return op2
         ''' % (_op, _oopspec.lower(), _oopspec, _oopspec)).compile()
 
+    for _op, _oopspec in [('cast_int_to_ulonglong',     'FROM_INT'),
+                          ('cast_uint_to_ulonglong',    'FROM_UINT'),
+                          ('cast_float_to_ulonglong',   'FROM_FLOAT'),
+                          ('cast_ulonglong_to_float',   'U_TO_FLOAT'),
+                         ]:
+        exec py.code.Source('''
+            def rewrite_op_%s(self, op):
+                args = op.args
+                op1 = self.prepare_builtin_call(op, "ullong_%s", args)
+                op2 = self._handle_oopspec_call(op1, args,
+                                                EffectInfo.OS_LLONG_%s,
+                                           EffectInfo.EF_ELIDABLE_CANNOT_RAISE)
+                return op2
+        ''' % (_op, _oopspec.lower(), _oopspec)).compile()
+
     def _normalize(self, oplist):
         if isinstance(oplist, SpaceOperation):
             return [oplist]
diff --git a/pypy/jit/codewriter/support.py b/pypy/jit/codewriter/support.py
--- a/pypy/jit/codewriter/support.py
+++ b/pypy/jit/codewriter/support.py
@@ -315,18 +315,30 @@
 def _ll_1_llong_from_int(x):
     return r_longlong(intmask(x))
 
+def _ll_1_ullong_from_int(x):
+    return r_ulonglong(intmask(x))
+
 def _ll_1_llong_from_uint(x):
     return r_longlong(r_uint(x))
 
+def _ll_1_ullong_from_uint(x):
+    return r_ulonglong(r_uint(x))
+
 def _ll_1_llong_to_int(xll):
     return intmask(xll)
 
 def _ll_1_llong_from_float(xf):
     return r_longlong(xf)
 
+def _ll_1_ullong_from_float(xf):
+    return r_ulonglong(xf)
+
 def _ll_1_llong_to_float(xll):
     return float(rffi.cast(lltype.SignedLongLong, xll))
 
+def _ll_1_ullong_u_to_float(xull):
+    return float(rffi.cast(lltype.UnsignedLongLong, xull))
+
 
 def _ll_1_llong_abs(xll):
     if xll < 0:
@@ -351,20 +363,23 @@
     return llop.llong_mod(lltype.SignedLongLong, xll, yll)
 
 def _ll_2_ullong_floordiv(xll, yll):
-    return llop.ullong_floordiv(lltype.SignedLongLong, xll, yll)
+    return llop.ullong_floordiv(lltype.UnsignedLongLong, xll, yll)
 
 def _ll_2_ullong_floordiv_zer(xll, yll):
     if yll == 0:
         raise ZeroDivisionError
-    return llop.ullong_floordiv(lltype.SignedLongLong, xll, yll)
+    return llop.ullong_floordiv(lltype.UnsignedLongLong, xll, yll)
 
 def _ll_2_ullong_mod(xll, yll):
-    return llop.ullong_mod(lltype.SignedLongLong, xll, yll)
+    return llop.ullong_mod(lltype.UnsignedLongLong, xll, yll)
 
 def _ll_2_ullong_mod_zer(xll, yll):
     if yll == 0:
         raise ZeroDivisionError
-    return llop.ullong_mod(lltype.SignedLongLong, xll, yll)
+    return llop.ullong_mod(lltype.UnsignedLongLong, xll, yll)
+
+def _ll_2_uint_mod(xll, yll):
+    return llop.uint_mod(lltype.Unsigned, xll, yll)
 
 
 # libffi support
diff --git a/pypy/jit/codewriter/test/test_flatten.py b/pypy/jit/codewriter/test/test_flatten.py
--- a/pypy/jit/codewriter/test/test_flatten.py
+++ b/pypy/jit/codewriter/test/test_flatten.py
@@ -829,14 +829,15 @@
                     self.encoding_test(f, [rffi.cast(FROM, 42)], expectedstr,
                                        transform=True)
                 elif TO in (rffi.LONG, rffi.ULONG):
+                    if rffi.cast(FROM, -1) < 0:
+                        fnname = "llong_from_int"
+                    else:
+                        fnname = "llong_from_uint"
                     if TO == rffi.LONG:
                         TO = rffi.LONGLONG
                     else:
                         TO = rffi.ULONGLONG
-                    if rffi.cast(FROM, -1) < 0:
-                        fnname = "llong_from_int"
-                    else:
-                        fnname = "llong_from_uint"
+                        fnname = "u" + fnname
                     expected.pop()   # remove int_return
                     expected.append(
                         "residual_call_irf_f $<* fn %s>, <Descr>, I[%s], R[], F[] -> %%f0"
diff --git a/pypy/jit/codewriter/test/test_longlong.py b/pypy/jit/codewriter/test/test_longlong.py
--- a/pypy/jit/codewriter/test/test_longlong.py
+++ b/pypy/jit/codewriter/test/test_longlong.py
@@ -57,7 +57,8 @@
             assert op1.opname == 'residual_call_irf_f'
         else:
             assert op1.opname == 'residual_call_irf_i'
-        gotindex = getattr(EffectInfo, 'OS_' + op1.args[0].value.upper())
+        gotindex = getattr(EffectInfo,
+                           'OS_' + op1.args[0].value.upper().lstrip('U'))
         assert gotindex == oopspecindex
         assert op1.args[1] == 'calldescr-%d' % oopspecindex
         assert list(op1.args[2]) == [v for v in vlist
@@ -192,8 +193,12 @@
                       [lltype.SignedLongLong], lltype.Signed)
         self.do_check('cast_float_to_longlong', EffectInfo.OS_LLONG_FROM_FLOAT,
                       [lltype.Float], lltype.SignedLongLong)
+        self.do_check('cast_float_to_ulonglong', EffectInfo.OS_LLONG_FROM_FLOAT,
+                      [lltype.Float], lltype.UnsignedLongLong)
         self.do_check('cast_longlong_to_float', EffectInfo.OS_LLONG_TO_FLOAT,
                       [lltype.SignedLongLong], lltype.Float)
+        self.do_check('cast_ulonglong_to_float', EffectInfo.OS_LLONG_U_TO_FLOAT,
+                      [lltype.UnsignedLongLong], lltype.Float)
         for T1 in [lltype.SignedLongLong, lltype.UnsignedLongLong]:
             for T2 in [lltype.Signed, lltype.Unsigned]:
                 self.do_check('cast_primitive', EffectInfo.OS_LLONG_TO_INT,
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -2956,6 +2956,18 @@
         assert res == f(32)
         self.check_loops(arraylen_gc=2)
 
+    def test_ulonglong_mod(self):
+        myjitdriver = JitDriver(greens = [], reds = ['n', 'sa', 'i'])
+        def f(n):
+            sa = i = rffi.cast(rffi.ULONGLONG, 1)
+            while i < rffi.cast(rffi.ULONGLONG, n):
+                myjitdriver.jit_merge_point(sa=sa, n=n, i=i)
+                sa += sa % i
+                i += 1
+        res = self.meta_interp(f, [32])
+        assert res == f(32)
+
+
 class TestOOtype(BasicTests, OOJitMixin):
 
     def test_oohash(self):
diff --git a/pypy/jit/metainterp/test/test_longlong.py b/pypy/jit/metainterp/test/test_longlong.py
--- a/pypy/jit/metainterp/test/test_longlong.py
+++ b/pypy/jit/metainterp/test/test_longlong.py
@@ -118,6 +118,26 @@
         res = self.interp_operations(f, [1000000000])
         assert res == 123500000000.0
 
+    def test_floats_negative(self):
+        def f(i):
+            # i == 1000000000
+            f = i * -123.5
+            n = r_longlong(f)
+            compare(n, -29, 1054051584)
+            return float(n)
+        res = self.interp_operations(f, [1000000000])
+        assert res == -123500000000.0
+
+    def test_floats_ulonglong(self):
+        def f(i):
+            # i == 1000000000
+            f = i * 12350000000.0
+            n = r_ulonglong(f)
+            compare(n, -1419508847, 538116096)
+            return float(n)
+        res = self.interp_operations(f, [1000000000])
+        assert res == 12350000000000000000.0
+
     def test_unsigned_compare_ops(self):
         def f(n1, n2):
             # n == 30002000000000
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -7,13 +7,14 @@
 from pypy.interpreter.typedef import TypeDef, interp_attrproperty, GetSetProperty
 from pypy.module.micronumpy import signature
 from pypy.objspace.std.floatobject import float2string
-from pypy.rlib import rfloat
-from pypy.rlib.rarithmetic import widen
+from pypy.rlib import rarithmetic, rfloat
+from pypy.rlib.rarithmetic import LONG_BIT, widen
 from pypy.rlib.objectmodel import specialize, enforceargs
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.rpython.lltypesystem import lltype, rffi
 
 
+UNSIGNEDLTR = "u"
 SIGNEDLTR = "i"
 BOOLLTR = "b"
 FLOATINGLTR = "f"
@@ -61,7 +62,10 @@
             self.val = val
 
         def wrap(self, space):
-            return space.wrap(self.val)
+            val = self.val
+            if valtype is rarithmetic.r_singlefloat:
+                val = float(val)
+            return space.wrap(val)
 
         def convert_to(self, dtype):
             return dtype.adapt_val(self.val)
@@ -145,7 +149,7 @@
         return self.adapt_val(func(self, self.for_computation(self.unbox(v))))
     return impl
 
-class ArithmaticTypeMixin(object):
+class ArithmeticTypeMixin(object):
     _mixin_ = True
 
     @binop
@@ -200,11 +204,17 @@
         return v1 >= v2
 
 
-class FloatArithmeticDtype(ArithmaticTypeMixin):
+class FloatArithmeticDtype(ArithmeticTypeMixin):
     _mixin_ = True
 
+    def unwrap(self, space, w_item):
+        return self.adapt_val(space.float_w(space.float(w_item)))
+
     def for_computation(self, v):
-        return v
+        return float(v)
+
+    def str_format(self, item):
+        return float2string(self.for_computation(self.unbox(item)), 'g', rfloat.DTSF_STR_PRECISION)
 
     @binop
     def mod(self, v1, v2):
@@ -265,7 +275,7 @@
     def arcsinh(self, v):
         return math.asinh(v)
 
-class IntegerArithmeticDtype(ArithmaticTypeMixin):
+class IntegerArithmeticDtype(ArithmeticTypeMixin):
     _mixin_ = True
 
     def unwrap(self, space, w_item):
@@ -274,10 +284,16 @@
     def for_computation(self, v):
         return widen(v)
 
+    def str_format(self, item):
+        return str(widen(self.unbox(item)))
+
     @binop
     def mod(self, v1, v2):
         return v1 % v2
 
+class SignedIntegerArithmeticDtype(IntegerArithmeticDtype):
+    _mixin_ = True
+
     @unaryop
     def sign(self, v):
         if v > 0:
@@ -288,17 +304,22 @@
             assert v == 0
             return 0
 
-    def str_format(self, item):
-        return str(widen(self.unbox(item)))
+class UnsignedIntegerArithmeticDtype(IntegerArithmeticDtype):
+    _mixin_ = True
+
+    @unaryop
+    def sign(self, v):
+        return int(v != 0)
+
 
 W_BoolDtype = create_low_level_dtype(
     num = 0, kind = BOOLLTR, name = "bool",
-    aliases = ["?"],
+    aliases = ["?", "bool", "bool8"],
     applevel_types = ["bool"],
     T = lltype.Bool,
     valtype = bool,
 )
-class W_BoolDtype(IntegerArithmeticDtype, W_BoolDtype):
+class W_BoolDtype(SignedIntegerArithmeticDtype, W_BoolDtype):
     def unwrap(self, space, w_item):
         return self.adapt_val(space.is_true(w_item))
 
@@ -311,67 +332,139 @@
 
 W_Int8Dtype = create_low_level_dtype(
     num = 1, kind = SIGNEDLTR, name = "int8",
-    aliases = ["int8"],
+    aliases = ["b", "int8", "i1"],
     applevel_types = [],
     T = rffi.SIGNEDCHAR,
     valtype = rffi.SIGNEDCHAR._type,
     expected_size = 1,
 )
-class W_Int8Dtype(IntegerArithmeticDtype, W_Int8Dtype):
+class W_Int8Dtype(SignedIntegerArithmeticDtype, W_Int8Dtype):
+    pass
+
+W_UInt8Dtype = create_low_level_dtype(
+    num = 2, kind = UNSIGNEDLTR, name = "uint8",
+    aliases = ["B", "uint8", "I1"],
+    applevel_types = [],
+    T = rffi.UCHAR,
+    valtype = rffi.UCHAR._type,
+    expected_size = 1,
+)
+class W_UInt8Dtype(UnsignedIntegerArithmeticDtype, W_UInt8Dtype):
     pass
 
 W_Int16Dtype = create_low_level_dtype(
     num = 3, kind = SIGNEDLTR, name = "int16",
-    aliases = ["int16"],
+    aliases = ["h", "int16", "i2"],
     applevel_types = [],
     T = rffi.SHORT,
     valtype = rffi.SHORT._type,
     expected_size = 2,
 )
-class W_Int16Dtype(IntegerArithmeticDtype, W_Int16Dtype):
+class W_Int16Dtype(SignedIntegerArithmeticDtype, W_Int16Dtype):
+    pass
+
+W_UInt16Dtype = create_low_level_dtype(
+    num = 4, kind = UNSIGNEDLTR, name = "uint16",
+    aliases = ["H", "uint16", "I2"],
+    applevel_types = [],
+    T = rffi.USHORT,
+    valtype = rffi.USHORT._type,
+    expected_size = 2,
+)
+class W_UInt16Dtype(UnsignedIntegerArithmeticDtype, W_UInt16Dtype):
     pass
 
 W_Int32Dtype = create_low_level_dtype(
     num = 5, kind = SIGNEDLTR, name = "int32",
-    aliases = ["i"],
+    aliases = ["i", "int32", "i4"],
     applevel_types = [],
     T = rffi.INT,
     valtype = rffi.INT._type,
     expected_size = 4,
 )
-class W_Int32Dtype(IntegerArithmeticDtype, W_Int32Dtype):
+class W_Int32Dtype(SignedIntegerArithmeticDtype, W_Int32Dtype):
+    pass
+
+W_UInt32Dtype = create_low_level_dtype(
+    num = 6, kind = UNSIGNEDLTR, name = "uint32",
+    aliases = ["I", "uint32", "I4"],
+    applevel_types = [],
+    T = rffi.UINT,
+    valtype = rffi.UINT._type,
+    expected_size = 4,
+)
+class W_UInt32Dtype(UnsignedIntegerArithmeticDtype, W_UInt32Dtype):
     pass
 
 W_Int64Dtype = create_low_level_dtype(
     num = 9, kind = SIGNEDLTR, name = "int64",
-    aliases = [],
+    aliases = ["q", "int64", "i8"],
     applevel_types = ["long"],
     T = rffi.LONGLONG,
     valtype = rffi.LONGLONG._type,
     expected_size = 8,
 )
-class W_Int64Dtype(IntegerArithmeticDtype, W_Int64Dtype):
+class W_Int64Dtype(SignedIntegerArithmeticDtype, W_Int64Dtype):
+    pass
+
+W_UInt64Dtype = create_low_level_dtype(
+    num = 10, kind = UNSIGNEDLTR, name = "uint64",
+    aliases = ["Q", "uint64", "I8"],
+    applevel_types = [],
+    T = rffi.ULONGLONG,
+    valtype = rffi.ULONGLONG._type,
+    expected_size = 8,
+)
+class W_UInt64Dtype(UnsignedIntegerArithmeticDtype, W_UInt64Dtype):
+    pass
+
+if LONG_BIT == 32:
+    class W_LongDtype(W_Int32Dtype):
+        pass
+
+    class W_ULongDtype(W_UInt32Dtype):
+        pass
+else:
+    class W_LongDtype(W_Int64Dtype):
+        pass
+
+    class W_ULongDtype(W_UInt64Dtype):
+        pass
+
+W_LongDtype.num = 7
+W_LongDtype.aliases = ["l"]
+W_LongDtype.applevel_types = ["int"]
+W_ULongDtype.num = 8
+W_ULongDtype.aliases = ["L"]
+
+W_Float32Dtype = create_low_level_dtype(
+    num = 11, kind = FLOATINGLTR, name = "float32",
+    aliases = ["f", "float32", "f4"],
+    applevel_types = [],
+    T = lltype.SingleFloat,
+    valtype = rarithmetic.r_singlefloat,
+    expected_size = 4,
+)
+class W_Float32Dtype(FloatArithmeticDtype, W_Float32Dtype):
     pass
 
 W_Float64Dtype = create_low_level_dtype(
     num = 12, kind = FLOATINGLTR, name = "float64",
-    aliases = [],
+    aliases = ["d", "float64", "f8"],
     applevel_types = ["float"],
     T = lltype.Float,
     valtype = float,
     expected_size = 8,
 )
 class W_Float64Dtype(FloatArithmeticDtype, W_Float64Dtype):
-    def unwrap(self, space, w_item):
-        return self.adapt_val(space.float_w(space.float(w_item)))
-
-    def str_format(self, item):
-        return float2string(self.unbox(item), 'g', rfloat.DTSF_STR_PRECISION)
+    pass
 
 ALL_DTYPES = [
     W_BoolDtype,
-    W_Int8Dtype, W_Int16Dtype, W_Int32Dtype, W_Int64Dtype,
-    W_Float64Dtype
+    W_Int8Dtype, W_UInt8Dtype, W_Int16Dtype, W_UInt16Dtype,
+    W_Int32Dtype, W_UInt32Dtype, W_LongDtype, W_ULongDtype,
+    W_Int64Dtype, W_UInt64Dtype,
+    W_Float32Dtype, W_Float64Dtype,
 ]
 
 dtypes_by_alias = unrolling_iterable([
@@ -398,6 +491,7 @@
 
     num = interp_attrproperty("num", cls=W_Dtype),
     kind = interp_attrproperty("kind", cls=W_Dtype),
+    itemsize = interp_attrproperty("num_bytes", cls=W_Dtype),
     shape = GetSetProperty(W_Dtype.descr_get_shape),
 )
 W_Dtype.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -4,6 +4,7 @@
 from pypy.interpreter.typedef import TypeDef, GetSetProperty, interp_attrproperty
 from pypy.module.micronumpy import interp_dtype, signature
 from pypy.rlib import jit
+from pypy.rlib.rarithmetic import LONG_BIT
 from pypy.tool.sourcetools import func_with_new_name
 
 
@@ -180,23 +181,56 @@
 
     # Everything promotes to float, and bool promotes to everything.
     if dt2.kind == interp_dtype.FLOATINGLTR or dt1.kind == interp_dtype.BOOLLTR:
+        # Float32 + 8-bit int = Float64
+        if dt2.num == 11 and dt1.num_bytes >= 4:
+            return space.fromcache(interp_dtype.W_Float64Dtype)
         return dt2
 
-    assert False
+    # for now this means mixing signed and unsigned
+    if dt2.kind == interp_dtype.SIGNEDLTR:
+        # if dt2 has a greater number of bytes, then just go with it
+        if dt1.num_bytes < dt2.num_bytes:
+            return dt2
+        # we need to promote both dtypes
+        dtypenum = dt2.num + 2
+    else:
+        # increase to the next signed type (or to float)
+        dtypenum = dt2.num + 1
+        # UInt64 + signed = Float64
+        if dt2.num == 10:
+            dtypenum += 1
+    newdtype = interp_dtype.ALL_DTYPES[dtypenum]
+
+    if newdtype.num_bytes > dt2.num_bytes or newdtype.kind == interp_dtype.FLOATINGLTR:
+        return space.fromcache(newdtype)
+    else:
+        # we only promoted to long on 32-bit or to longlong on 64-bit
+        # this is really for dealing with the Long and Ulong dtypes
+        if LONG_BIT == 32:
+            dtypenum += 2
+        else:
+            dtypenum += 3
+        return space.fromcache(interp_dtype.ALL_DTYPES[dtypenum])
 
 def find_unaryop_result_dtype(space, dt, promote_to_float=False,
     promote_bools=False, promote_to_largest=False):
     if promote_bools and (dt.kind == interp_dtype.BOOLLTR):
         return space.fromcache(interp_dtype.W_Int8Dtype)
     if promote_to_float:
+        if dt.kind == interp_dtype.FLOATINGLTR:
+            return dt
+        if dt.num >= 5:
+            return space.fromcache(interp_dtype.W_Float64Dtype)
         for bytes, dtype in interp_dtype.dtypes_by_num_bytes:
-            if dtype.kind == interp_dtype.FLOATINGLTR and dtype.num_bytes >= dt.num_bytes:
+            if dtype.kind == interp_dtype.FLOATINGLTR and dtype.num_bytes > dt.num_bytes:
                 return space.fromcache(dtype)
     if promote_to_largest:
         if dt.kind == interp_dtype.BOOLLTR or dt.kind == interp_dtype.SIGNEDLTR:
             return space.fromcache(interp_dtype.W_Int64Dtype)
         elif dt.kind == interp_dtype.FLOATINGLTR:
             return space.fromcache(interp_dtype.W_Float64Dtype)
+        elif dt.kind == interp_dtype.UNSIGNEDLTR:
+            return space.fromcache(interp_dtype.W_UInt64Dtype)
         else:
             assert False
     return dt
@@ -205,15 +239,23 @@
     w_type = space.type(w_obj)
 
     bool_dtype = space.fromcache(interp_dtype.W_BoolDtype)
+    long_dtype = space.fromcache(interp_dtype.W_LongDtype)
     int64_dtype = space.fromcache(interp_dtype.W_Int64Dtype)
 
     if space.is_w(w_type, space.w_bool):
-        if current_guess is None:
+        if current_guess is None or current_guess is bool_dtype:
             return bool_dtype
+        return current_guess
     elif space.is_w(w_type, space.w_int):
         if (current_guess is None or current_guess is bool_dtype or
-            current_guess is int64_dtype):
+            current_guess is long_dtype):
+            return long_dtype
+        return current_guess
+    elif space.is_w(w_type, space.w_long):
+        if (current_guess is None or current_guess is bool_dtype or
+            current_guess is long_dtype or current_guess is int64_dtype):
             return int64_dtype
+        return current_guess
     return space.fromcache(interp_dtype.W_Float64Dtype)
 
 
@@ -225,7 +267,9 @@
         def impl(res_dtype, lvalue, rvalue):
             res = getattr(res_dtype, op_name)(lvalue, rvalue)
             if comparison_func:
-                res = space.fromcache(interp_dtype.W_BoolDtype).box(res)
+                booldtype = space.fromcache(interp_dtype.W_BoolDtype)
+                assert isinstance(booldtype, interp_dtype.W_BoolDtype)
+                res = booldtype.box(res)
             return res
     return func_with_new_name(impl, ufunc_name)
 
@@ -278,7 +322,7 @@
 
         identity = extra_kwargs.get("identity")
         if identity is not None:
-            identity = space.fromcache(interp_dtype.W_Int64Dtype).adapt_val(identity)
+            identity = space.fromcache(interp_dtype.W_LongDtype).adapt_val(identity)
         extra_kwargs["identity"] = identity
 
         func = ufunc_dtype_caller(space, ufunc_name, op_name, argcount,
@@ -291,4 +335,4 @@
         setattr(self, ufunc_name, ufunc)
 
 def get(space):
-    return space.fromcache(UfuncState)
\ No newline at end of file
+    return space.fromcache(UfuncState)
diff --git a/pypy/module/micronumpy/test/test_base.py b/pypy/module/micronumpy/test/test_base.py
--- a/pypy/module/micronumpy/test/test_base.py
+++ b/pypy/module/micronumpy/test/test_base.py
@@ -64,18 +64,46 @@
     def test_unaryops(self, space):
         bool_dtype = space.fromcache(interp_dtype.W_BoolDtype)
         int8_dtype = space.fromcache(interp_dtype.W_Int8Dtype)
+        uint8_dtype = space.fromcache(interp_dtype.W_UInt8Dtype)
+        int16_dtype = space.fromcache(interp_dtype.W_Int16Dtype)
+        uint16_dtype = space.fromcache(interp_dtype.W_UInt16Dtype)
         int32_dtype = space.fromcache(interp_dtype.W_Int32Dtype)
+        uint32_dtype = space.fromcache(interp_dtype.W_UInt32Dtype)
+        long_dtype = space.fromcache(interp_dtype.W_LongDtype)
+        ulong_dtype = space.fromcache(interp_dtype.W_ULongDtype)
+        int64_dtype = space.fromcache(interp_dtype.W_Int64Dtype)
+        uint64_dtype = space.fromcache(interp_dtype.W_UInt64Dtype)
+        float32_dtype = space.fromcache(interp_dtype.W_Float32Dtype)
         float64_dtype = space.fromcache(interp_dtype.W_Float64Dtype)
 
-        # Normal rules, everythign returns itself
+        # Normal rules, everything returns itself
         assert find_unaryop_result_dtype(space, bool_dtype) is bool_dtype
         assert find_unaryop_result_dtype(space, int8_dtype) is int8_dtype
+        assert find_unaryop_result_dtype(space, uint8_dtype) is uint8_dtype
+        assert find_unaryop_result_dtype(space, int16_dtype) is int16_dtype
+        assert find_unaryop_result_dtype(space, uint16_dtype) is uint16_dtype
         assert find_unaryop_result_dtype(space, int32_dtype) is int32_dtype
+        assert find_unaryop_result_dtype(space, uint32_dtype) is uint32_dtype
+        assert find_unaryop_result_dtype(space, long_dtype) is long_dtype
+        assert find_unaryop_result_dtype(space, ulong_dtype) is ulong_dtype
+        assert find_unaryop_result_dtype(space, int64_dtype) is int64_dtype
+        assert find_unaryop_result_dtype(space, uint64_dtype) is uint64_dtype
+        assert find_unaryop_result_dtype(space, float32_dtype) is float32_dtype
         assert find_unaryop_result_dtype(space, float64_dtype) is float64_dtype
 
         # Coerce to floats, some of these will eventually be float16, or
         # whatever our smallest float type is.
-        assert find_unaryop_result_dtype(space, bool_dtype, promote_to_float=True) is float64_dtype
-        assert find_unaryop_result_dtype(space, int8_dtype, promote_to_float=True) is float64_dtype
+        assert find_unaryop_result_dtype(space, bool_dtype, promote_to_float=True) is float32_dtype # will be float16 if we ever put that in
+        assert find_unaryop_result_dtype(space, int8_dtype, promote_to_float=True) is float32_dtype # will be float16 if we ever put that in
+        assert find_unaryop_result_dtype(space, uint8_dtype, promote_to_float=True) is float32_dtype # will be float16 if we ever put that in
+        assert find_unaryop_result_dtype(space, int16_dtype, promote_to_float=True) is float32_dtype
+        assert find_unaryop_result_dtype(space, uint16_dtype, promote_to_float=True) is float32_dtype
         assert find_unaryop_result_dtype(space, int32_dtype, promote_to_float=True) is float64_dtype
-        assert find_unaryop_result_dtype(space, float64_dtype, promote_to_float=True) is float64_dtype
\ No newline at end of file
+        assert find_unaryop_result_dtype(space, uint32_dtype, promote_to_float=True) is float64_dtype
+        assert find_unaryop_result_dtype(space, int64_dtype, promote_to_float=True) is float64_dtype
+        assert find_unaryop_result_dtype(space, uint64_dtype, promote_to_float=True) is float64_dtype
+        assert find_unaryop_result_dtype(space, float32_dtype, promote_to_float=True) is float32_dtype
+        assert find_unaryop_result_dtype(space, float64_dtype, promote_to_float=True) is float64_dtype
+
+        # promote bools, happens with sign ufunc
+        assert find_unaryop_result_dtype(space, bool_dtype, promote_bools=True) is int8_dtype
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -17,6 +17,7 @@
         from numpy import dtype
 
         assert dtype(bool).num == 0
+        assert dtype(int).num == 7
         assert dtype(long).num == 9
         assert dtype(float).num == 12
 
@@ -81,6 +82,48 @@
             assert isinstance(a[i], (int, long))
             assert a[1] == 1
 
+    def test_overflow(self):
+        from numpy import array, dtype
+        assert array([128], 'b')[0] == -128
+        assert array([256], 'B')[0] == 0
+        assert array([32768], 'h')[0] == -32768
+        assert array([65536], 'H')[0] == 0
+        if dtype('l').itemsize == 4: # 32-bit
+            raises(OverflowError, "array([2**32/2], 'i')")
+            raises(OverflowError, "array([2**32], 'I')")
+        raises(OverflowError, "array([2**64/2], 'q')")
+        raises(OverflowError, "array([2**64], 'Q')")
+
+    def test_bool_binop_types(self):
+        from numpy import array, dtype
+        types = ('?','b','B','h','H','i','I','l','L','q','Q','f','d')
+        N = len(types)
+        a = array([True], '?')
+        for t in types:
+            assert (a + array([0], t)).dtype is dtype(t)
+
+    def test_binop_types(self):
+        from numpy import array, dtype
+        tests = [('b','B','h'), ('b','h','h'), ('b','H','i'), ('b','i','i'),
+                 ('b','l','l'), ('b','q','q'), ('b','Q','d'), ('B','h','h'),
+                 ('B','H','H'), ('B','i','i'), ('B','I','I'), ('B','l','l'),
+                 ('B','L','L'), ('B','q','q'), ('B','Q','Q'), ('h','H','i'),
+                 ('h','i','i'), ('h','l','l'), ('h','q','q'), ('h','Q','d'),
+                 ('H','i','i'), ('H','I','I'), ('H','l','l'), ('H','L','L'),
+                 ('H','q','q'), ('H','Q','Q'), ('i','l','l'), ('i','q','q'),
+                 ('i','Q','d'), ('I','L','L'), ('I','q','q'), ('I','Q','Q'),
+                 ('q','Q','d'), ('b','f','f'), ('B','f','f'), ('h','f','f'),
+                 ('H','f','f'), ('i','f','d'), ('I','f','d'), ('l','f','d'),
+                 ('L','f','d'), ('q','f','d'), ('Q','f','d'), ('q','d','d')]
+        if dtype('i').itemsize == dtype('l').itemsize: # 32-bit
+            tests.extend([('b','I','q'), ('b','L','q'), ('h','I','q'),
+                          ('h','L','q'), ('i','I','q'), ('i','L','q')])
+        else:
+            tests.extend([('b','I','l'), ('b','L','d'), ('h','I','l'),
+                          ('h','L','d'), ('i','I','l'), ('i','L','d')])
+        for d1, d2, dout in tests:
+            assert (array([1], d1) + array([1], d2)).dtype is dtype(dout)
+
     def test_add_int8(self):
         from numpy import array, dtype
 
@@ -99,6 +142,15 @@
         for i in range(5):
             assert b[i] == i * 2
 
+    def test_add_uint32(self):
+        from numpy import array, dtype
+
+        a = array(range(5), dtype="I")
+        b = a + a
+        assert b.dtype is dtype("I")
+        for i in range(5):
+            assert b[i] == i * 2
+
     def test_shape(self):
         from numpy import dtype
 
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -551,8 +551,10 @@
         from numpy import array, dtype
 
         assert array([True]).dtype is dtype(bool)
-        assert array([True, 1]).dtype is dtype(long)
-        assert array([1, 2, 3]).dtype is dtype(long)
+        assert array([True, False]).dtype is dtype(bool)
+        assert array([True, 1]).dtype is dtype(int)
+        assert array([1, 2, 3]).dtype is dtype(int)
+        assert array([1L, 2, 3]).dtype is dtype(long)
         assert array([1.2, True]).dtype is dtype(float)
         assert array([1.2, 5]).dtype is dtype(float)
         assert array([]).dtype is dtype(float)
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -234,7 +234,7 @@
             assert b[i] == math.sin(a[i])
 
         a = sin(array([True, False], dtype=bool))
-        assert a[0] == sin(1)
+        assert abs(a[0] - sin(1)) < 1e-7 # a[0] will be less precise
         assert a[1] == 0.0
 
     def test_cos(self):
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -1,20 +1,24 @@
 from pypy.jit.metainterp.test.support import LLJitMixin
 from pypy.module.micronumpy import interp_ufuncs, signature
 from pypy.module.micronumpy.compile import (numpy_compile, FakeSpace,
-    FloatObject)
-from pypy.module.micronumpy.interp_dtype import W_Float64Dtype, W_Int64Dtype
+    FloatObject, IntObject)
+from pypy.module.micronumpy.interp_dtype import W_Int32Dtype, W_Float64Dtype, W_Int64Dtype, W_UInt64Dtype
 from pypy.module.micronumpy.interp_numarray import (BaseArray, SingleDimArray,
     SingleDimSlice, scalar_w)
 from pypy.rlib.nonconst import NonConstant
 from pypy.rpython.annlowlevel import llstr
 from pypy.rpython.test.test_llinterp import interpret
 
+import py
+
 
 class TestNumpyJIt(LLJitMixin):
     def setup_class(cls):
         cls.space = FakeSpace()
         cls.float64_dtype = cls.space.fromcache(W_Float64Dtype)
         cls.int64_dtype = cls.space.fromcache(W_Int64Dtype)
+        cls.uint64_dtype = cls.space.fromcache(W_UInt64Dtype)
+        cls.int32_dtype = cls.space.fromcache(W_Int32Dtype)
 
     def test_add(self):
         def f(i):
@@ -303,6 +307,31 @@
                           'int_lt': 1, 'guard_true': 1, 'jump': 1})
         assert result == 11.0
 
+    def test_int32_sum(self):
+        py.test.skip("pypy/jit/backend/llimpl.py needs to be changed to "
+                     "deal correctly with int dtypes for this test to "
+                     "work. skip for now until someone feels up to the task")
+        space = self.space
+        float64_dtype = self.float64_dtype
+        int32_dtype = self.int32_dtype
+
+        def f(n):
+            if NonConstant(False):
+                dtype = float64_dtype
+            else:
+                dtype = int32_dtype
+            ar = SingleDimArray(n, dtype=dtype)
+            i = 0
+            while i < n:
+                ar.get_concrete().setitem(i, int32_dtype.box(7))
+                i += 1
+            v = ar.descr_add(space, ar).descr_sum(space)
+            assert isinstance(v, IntObject)
+            return v.intval
+
+        result = self.meta_interp(f, [5], listops=True, backendopt=True)
+        assert result == f(5)
+
 class TestTranslation(object):
     def test_compile(self):
         x = numpy_compile('aa+f*f/a-', 10)
diff --git a/pypy/rpython/lltypesystem/opimpl.py b/pypy/rpython/lltypesystem/opimpl.py
--- a/pypy/rpython/lltypesystem/opimpl.py
+++ b/pypy/rpython/lltypesystem/opimpl.py
@@ -357,7 +357,7 @@
 
 def op_cast_float_to_uint(f):
     assert type(f) is float
-    return r_uint(int(f))
+    return r_uint(long(f))
 
 def op_cast_float_to_longlong(f):
     assert type(f) is float
@@ -369,7 +369,7 @@
 
 def op_cast_float_to_ulonglong(f):
     assert type(f) is float
-    return r_ulonglong(r_longlong(f))
+    return r_ulonglong(long(f))
 
 def op_cast_char_to_int(b):
     assert type(b) is str and len(b) == 1
diff --git a/pypy/rpython/lltypesystem/test/test_lloperation.py b/pypy/rpython/lltypesystem/test/test_lloperation.py
--- a/pypy/rpython/lltypesystem/test/test_lloperation.py
+++ b/pypy/rpython/lltypesystem/test/test_lloperation.py
@@ -5,6 +5,7 @@
 from pypy.rpython.llinterp import LLFrame
 from pypy.rpython.test.test_llinterp import interpret
 from pypy.rpython import rclass
+from pypy.rlib.rarithmetic import LONGLONG_MASK, r_longlong, r_ulonglong
 
 LL_INTERP_OPERATIONS = [name[3:] for name in LLFrame.__dict__.keys()
                                  if name.startswith('op_')]
@@ -133,6 +134,14 @@
         py.test.raises(TypeError, llop.getinteriorfield,
                        lltype.Signed, s3, 'y')
 
+def test_cast_float_to_ulonglong():
+    f = 12350000000000000000.0
+    py.test.raises(OverflowError, r_longlong, f)
+    r_longlong(f / 2)   # does not raise OverflowError
+    #
+    x = llop.cast_float_to_ulonglong(lltype.UnsignedLongLong, f)
+    assert x == r_ulonglong(f)
+
 # ___________________________________________________________________________
 # This tests that the LLInterpreter and the LL_OPERATIONS tables are in sync.
 

From noreply at buildbot.pypy.org  Mon Oct  3 21:45:32 2011
From: noreply at buildbot.pypy.org (justinpeel)
Date: Mon,  3 Oct 2011 21:45:32 +0200 (CEST)
Subject: [pypy-commit] pypy unsigned-dtypes: close the branch
Message-ID: <20111003194532.0771C820D9@wyvern.cs.uni-duesseldorf.de>

Author: Justin Peel <notmuchtotell at gmail.com>
Branch: unsigned-dtypes
Changeset: r47806:39fb2ab2290b
Date: 2011-10-03 13:45 -0600
http://bitbucket.org/pypy/pypy/changeset/39fb2ab2290b/

Log:	close the branch


From noreply at buildbot.pypy.org  Mon Oct  3 22:24:01 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Mon,  3 Oct 2011 22:24:01 +0200 (CEST)
Subject: [pypy-commit] pypy default: added numpy.arctanh.
Message-ID: <20111003202401.C0687820D9@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r47807:755ade32727d
Date: 2011-10-03 16:23 -0400
http://bitbucket.org/pypy/pypy/changeset/755ade32727d/

Log:	added numpy.arctanh.

diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -24,6 +24,7 @@
         ("arcsin", "arcsin"),
         ("arctan", "arctan"),
         ("arcsinh", "arcsinh"),
+        ("arctanh", "arctanh"),
         ("copysign", "copysign"),
         ("cos", "cos"),
         ("divide", "divide"),
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -260,12 +260,12 @@
         return math.tan(v)
     @unaryop
     def arcsin(self, v):
-        if v < -1.0 or v > 1.0:
+        if not -1.0 <= v <= 1.0:
             return rfloat.NAN
         return math.asin(v)
     @unaryop
     def arccos(self, v):
-        if v < -1.0 or v > 1.0:
+        if not -1.0 <= v <= 1.0:
             return rfloat.NAN
         return math.acos(v)
     @unaryop
@@ -274,6 +274,13 @@
     @unaryop
     def arcsinh(self, v):
         return math.asinh(v)
+    @unaryop
+    def arctanh(self, v):
+        if v == 1.0 or v == -1.0:
+            return math.copysign(rfloat.INFINITY, v)
+        if not -1.0 < v < 1.0:
+            return rfloat.NAN
+        return math.atanh(v)
 
 class IntegerArithmeticDtype(ArithmeticTypeMixin):
     _mixin_ = True
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -313,6 +313,7 @@
             ("arccos", "arccos", 1, {"promote_to_float": True}),
             ("arctan", "arctan", 1, {"promote_to_float": True}),
             ("arcsinh", "arcsinh", 1, {"promote_to_float": True}),
+            ("arctanh", "arctanh", 1, {"promote_to_float": True}),
         ]:
             self.add_ufunc(space, *ufunc_def)
 
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -306,6 +306,17 @@
             assert math.asinh(v) == arcsinh(v)
         assert math.isnan(arcsinh(float("nan")))
 
+    def test_arctanh(self):
+        import math
+        from numpy import arctanh
+
+        for v in [.99, .5, 0, -.5, -.99]:
+            assert math.atanh(v) == arctanh(v)
+        for v in [2.0, -2.0]:
+            assert math.isnan(arctanh(v))
+        for v in [1.0, -1.0]:
+            assert arctanh(v) == math.copysign(float("inf"), v)
+
     def test_reduce_errors(self):
         from numpy import sin, add
 

From noreply at buildbot.pypy.org  Mon Oct  3 23:00:57 2011
From: noreply at buildbot.pypy.org (timo_jbo)
Date: Mon,  3 Oct 2011 23:00:57 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-data-buffer: add jits for setslice and
 getslice, move constants to earlier places.
Message-ID: <20111003210057.2AA7B820D9@wyvern.cs.uni-duesseldorf.de>

Author: Timo Paulssen <timonator at perpetuum-immobile.de>
Branch: numpy-data-buffer
Changeset: r47808:0eb532b36b5d
Date: 2011-10-03 23:00 +0200
http://bitbucket.org/pypy/pypy/changeset/0eb532b36b5d/

Log:	add jits for setslice and getslice, move constants to earlier
	places.

diff --git a/lib_pypy/numpy/__init__.py b/lib_pypy/numpy/__init__.py
--- a/lib_pypy/numpy/__init__.py
+++ b/lib_pypy/numpy/__init__.py
@@ -88,6 +88,7 @@
 
 def __from_buffer_or_datastring(buf_or_str, dt, count, offset=0):
     _dtype = dtype(dt)
+    buf_or_str = buffer(buf_or_str)
 
     if count > 0:
         length = count * _dtype.itemsize
diff --git a/pypy/module/_numpy/interp_buffer.py b/pypy/module/_numpy/interp_buffer.py
--- a/pypy/module/_numpy/interp_buffer.py
+++ b/pypy/module/_numpy/interp_buffer.py
@@ -1,21 +1,28 @@
 from pypy.interpreter.buffer import RWBuffer
 from pypy.rpython.lltypesystem import lltype, rffi
 from pypy.rlib.rstring import StringBuilder
+from pypy.rlib import jit
 
 CHAR_TP = lltype.Ptr(lltype.Array(lltype.Char, hints={'nolength': True}))
 
+setslice_driver = jit.JitDriver(greens = ['index', 'end', 'self', 'newstring'],
+                                reds = ['idx'])
+getslice_driver = jit.JitDriver(greens = ['stop', 'step', 'self', 'builder'],
+                                reds = ['idx'])
 class NumpyBuffer(RWBuffer):
     def __init__(self, array):
         self.array = array
+        storage = self.array.get_concrete().get_root_storage()
+        self.char_data = rffi.cast(CHAR_TP, storage)
+        self.length = self.array.get_concrete().find_size() \
+                          * self.array.find_dtype().num_bytes
 
     def getlength(self):
-        return self.array.get_concrete().find_size() * self.array.find_dtype().num_bytes
+        return self.length
 
     def getitem_noboundcheck(self, index):
-        storage = self.array.get_concrete().get_root_storage()
-        char_data = rffi.cast(CHAR_TP, storage)
         index = self.calc_index(index)
-        return char_data[index]
+        return self.char_data[index]
 
     def getitem(self, index):
         if index > self.getlength():
@@ -23,10 +30,8 @@
         return self.getitem_noboundcheck(index)
 
     def setitem_noboundcheck(self, index, value):
-        storage = self.array.get_concrete().get_root_storage()
-        char_ptr = rffi.cast(CHAR_TP, storage)
         index = self.calc_index(index)
-        char_ptr[index] = value
+        self.char_data[index] = value
 
     def setitem(self, index, value):
         if index > self.getlength():
@@ -36,14 +41,23 @@
     def setslice(self, index, newstring):
         if index + len(newstring) > self.getlength():
             raise IndexError("End of slice to set out of bounds (0<=index<=%d)" % self.getlength())
-        for idx in range(0, len(newstring)):
-            self.setitem_noboundcheck(index + idx, newstring[idx])
+        idx = 0
+        end = len(newstring)
+        while idx < end:
+            setslice_driver.jit_merge_point(self=self, newstring=newstring, index=index, end=end,
+                                            idx=idx)
+            self.setitem_noboundcheck(idx + index, newstring[idx])
+            idx += 1
 
     def getslice(self, start, stop, step, size):
         builder = StringBuilder(size)
 
-        for index in range(start, stop, step):
-            builder.append(self.getitem_noboundcheck(index))
+        idx = start
+        while idx < stop:
+            getslice_driver.jit_merge_point(self=self, builder=builder, stop=stop, step=step,
+                                            idx=idx)
+            builder.append(self.getitem_noboundcheck(idx))
+            idx += step
 
         return builder.build()
 

From noreply at buildbot.pypy.org  Mon Oct  3 23:19:22 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Mon,  3 Oct 2011 23:19:22 +0200 (CEST)
Subject: [pypy-commit] pypy default: added numpy.e
Message-ID: <20111003211922.9C26E820D9@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r47809:d63bd644f1cd
Date: 2011-10-03 17:19 -0400
http://bitbucket.org/pypy/pypy/changeset/d63bd644f1cd/

Log:	added numpy.e

diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -53,4 +53,5 @@
         'average': 'app_numpy.average',
         'mean': 'app_numpy.mean',
         'inf': 'app_numpy.inf',
+        'e': 'app_numpy.e',
     }
diff --git a/pypy/module/micronumpy/app_numpy.py b/pypy/module/micronumpy/app_numpy.py
--- a/pypy/module/micronumpy/app_numpy.py
+++ b/pypy/module/micronumpy/app_numpy.py
@@ -1,7 +1,10 @@
+import math
+
 import numpy
 
 
 inf = float("inf")
+e = math.e
 
 def average(a):
     # This implements a weighted average, for now we don't implement the
diff --git a/pypy/module/micronumpy/test/test_module.py b/pypy/module/micronumpy/test/test_module.py
--- a/pypy/module/micronumpy/test/test_module.py
+++ b/pypy/module/micronumpy/test/test_module.py
@@ -12,7 +12,10 @@
         assert average(range(10)) == 4.5
         assert average(array(range(10))) == 4.5
 
-    def test_inf(self):
-        from numpy import inf
+    def test_constants(self):
+        import math
+        from numpy import inf, e
         assert type(inf) is float
-        assert inf == float("inf")
\ No newline at end of file
+        assert inf == float("inf")
+        assert e == math.e
+        assert type(e) is float
\ No newline at end of file

From noreply at buildbot.pypy.org  Mon Oct  3 23:41:22 2011
From: noreply at buildbot.pypy.org (timo_jbo)
Date: Mon,  3 Oct 2011 23:41:22 +0200 (CEST)
Subject: [pypy-commit] pypy separate-applevel-numpy: merged in
 unsigned-dtypes, apply the same micronumpy -> _numpy/numpy rename
Message-ID: <20111003214122.1582C820D9@wyvern.cs.uni-duesseldorf.de>

Author: Timo Paulssen <timonator at perpetuum-immobile.de>
Branch: separate-applevel-numpy
Changeset: r47810:79cd81925a89
Date: 2011-10-03 23:39 +0200
http://bitbucket.org/pypy/pypy/changeset/79cd81925a89/

Log:	merged in unsigned-dtypes, apply the same micronumpy -> _numpy/numpy
	rename

diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -307,7 +307,11 @@
         self._nonperiodic_actions = []
         self.has_bytecode_counter = False
         self.fired_actions = None
-        self.checkinterval_scaled = 100 * TICK_COUNTER_STEP
+        # the default value is not 100, unlike CPython 2.7, but a much
+        # larger value, because we use a technique that not only allows
+        # but actually *forces* another thread to run whenever the counter
+        # reaches zero.
+        self.checkinterval_scaled = 10000 * TICK_COUNTER_STEP
         self._rebuild_action_dispatcher()
 
     def fire(self, action):
diff --git a/pypy/jit/codewriter/effectinfo.py b/pypy/jit/codewriter/effectinfo.py
--- a/pypy/jit/codewriter/effectinfo.py
+++ b/pypy/jit/codewriter/effectinfo.py
@@ -74,6 +74,7 @@
     OS_LLONG_UGE                = 91
     OS_LLONG_URSHIFT            = 92
     OS_LLONG_FROM_UINT          = 93
+    OS_LLONG_U_TO_FLOAT         = 94
     #
     OS_MATH_SQRT                = 100
 
diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -440,6 +440,7 @@
     rewrite_op_ullong_mod_zer      = _do_builtin_call
     rewrite_op_gc_identityhash = _do_builtin_call
     rewrite_op_gc_id           = _do_builtin_call
+    rewrite_op_uint_mod        = _do_builtin_call
 
     # ----------
     # getfield/setfield/mallocs etc.
@@ -883,9 +884,15 @@
                 v = v_arg
                 oplist = []
             if unsigned1:
-                opname = 'cast_uint_to_longlong'
+                if unsigned2:
+                    opname = 'cast_uint_to_ulonglong'
+                else:
+                    opname = 'cast_uint_to_longlong'
             else:
-                opname = 'cast_int_to_longlong'
+                if unsigned2:
+                    opname = 'cast_int_to_ulonglong'
+                else:
+                    opname = 'cast_int_to_longlong'
             op2 = self.rewrite_operation(
                 SpaceOperation(opname, [v], v_result)
             )
@@ -995,6 +1002,21 @@
                 return op2
         ''' % (_op, _oopspec.lower(), _oopspec, _oopspec)).compile()
 
+    for _op, _oopspec in [('cast_int_to_ulonglong',     'FROM_INT'),
+                          ('cast_uint_to_ulonglong',    'FROM_UINT'),
+                          ('cast_float_to_ulonglong',   'FROM_FLOAT'),
+                          ('cast_ulonglong_to_float',   'U_TO_FLOAT'),
+                         ]:
+        exec py.code.Source('''
+            def rewrite_op_%s(self, op):
+                args = op.args
+                op1 = self.prepare_builtin_call(op, "ullong_%s", args)
+                op2 = self._handle_oopspec_call(op1, args,
+                                                EffectInfo.OS_LLONG_%s,
+                                           EffectInfo.EF_ELIDABLE_CANNOT_RAISE)
+                return op2
+        ''' % (_op, _oopspec.lower(), _oopspec)).compile()
+
     def _normalize(self, oplist):
         if isinstance(oplist, SpaceOperation):
             return [oplist]
diff --git a/pypy/jit/codewriter/support.py b/pypy/jit/codewriter/support.py
--- a/pypy/jit/codewriter/support.py
+++ b/pypy/jit/codewriter/support.py
@@ -315,18 +315,30 @@
 def _ll_1_llong_from_int(x):
     return r_longlong(intmask(x))
 
+def _ll_1_ullong_from_int(x):
+    return r_ulonglong(intmask(x))
+
 def _ll_1_llong_from_uint(x):
     return r_longlong(r_uint(x))
 
+def _ll_1_ullong_from_uint(x):
+    return r_ulonglong(r_uint(x))
+
 def _ll_1_llong_to_int(xll):
     return intmask(xll)
 
 def _ll_1_llong_from_float(xf):
     return r_longlong(xf)
 
+def _ll_1_ullong_from_float(xf):
+    return r_ulonglong(xf)
+
 def _ll_1_llong_to_float(xll):
     return float(rffi.cast(lltype.SignedLongLong, xll))
 
+def _ll_1_ullong_u_to_float(xull):
+    return float(rffi.cast(lltype.UnsignedLongLong, xull))
+
 
 def _ll_1_llong_abs(xll):
     if xll < 0:
@@ -351,20 +363,23 @@
     return llop.llong_mod(lltype.SignedLongLong, xll, yll)
 
 def _ll_2_ullong_floordiv(xll, yll):
-    return llop.ullong_floordiv(lltype.SignedLongLong, xll, yll)
+    return llop.ullong_floordiv(lltype.UnsignedLongLong, xll, yll)
 
 def _ll_2_ullong_floordiv_zer(xll, yll):
     if yll == 0:
         raise ZeroDivisionError
-    return llop.ullong_floordiv(lltype.SignedLongLong, xll, yll)
+    return llop.ullong_floordiv(lltype.UnsignedLongLong, xll, yll)
 
 def _ll_2_ullong_mod(xll, yll):
-    return llop.ullong_mod(lltype.SignedLongLong, xll, yll)
+    return llop.ullong_mod(lltype.UnsignedLongLong, xll, yll)
 
 def _ll_2_ullong_mod_zer(xll, yll):
     if yll == 0:
         raise ZeroDivisionError
-    return llop.ullong_mod(lltype.SignedLongLong, xll, yll)
+    return llop.ullong_mod(lltype.UnsignedLongLong, xll, yll)
+
+def _ll_2_uint_mod(xll, yll):
+    return llop.uint_mod(lltype.Unsigned, xll, yll)
 
 
 # libffi support
diff --git a/pypy/jit/codewriter/test/test_flatten.py b/pypy/jit/codewriter/test/test_flatten.py
--- a/pypy/jit/codewriter/test/test_flatten.py
+++ b/pypy/jit/codewriter/test/test_flatten.py
@@ -829,14 +829,15 @@
                     self.encoding_test(f, [rffi.cast(FROM, 42)], expectedstr,
                                        transform=True)
                 elif TO in (rffi.LONG, rffi.ULONG):
+                    if rffi.cast(FROM, -1) < 0:
+                        fnname = "llong_from_int"
+                    else:
+                        fnname = "llong_from_uint"
                     if TO == rffi.LONG:
                         TO = rffi.LONGLONG
                     else:
                         TO = rffi.ULONGLONG
-                    if rffi.cast(FROM, -1) < 0:
-                        fnname = "llong_from_int"
-                    else:
-                        fnname = "llong_from_uint"
+                        fnname = "u" + fnname
                     expected.pop()   # remove int_return
                     expected.append(
                         "residual_call_irf_f $<* fn %s>, <Descr>, I[%s], R[], F[] -> %%f0"
diff --git a/pypy/jit/codewriter/test/test_longlong.py b/pypy/jit/codewriter/test/test_longlong.py
--- a/pypy/jit/codewriter/test/test_longlong.py
+++ b/pypy/jit/codewriter/test/test_longlong.py
@@ -57,7 +57,8 @@
             assert op1.opname == 'residual_call_irf_f'
         else:
             assert op1.opname == 'residual_call_irf_i'
-        gotindex = getattr(EffectInfo, 'OS_' + op1.args[0].value.upper())
+        gotindex = getattr(EffectInfo,
+                           'OS_' + op1.args[0].value.upper().lstrip('U'))
         assert gotindex == oopspecindex
         assert op1.args[1] == 'calldescr-%d' % oopspecindex
         assert list(op1.args[2]) == [v for v in vlist
@@ -192,8 +193,12 @@
                       [lltype.SignedLongLong], lltype.Signed)
         self.do_check('cast_float_to_longlong', EffectInfo.OS_LLONG_FROM_FLOAT,
                       [lltype.Float], lltype.SignedLongLong)
+        self.do_check('cast_float_to_ulonglong', EffectInfo.OS_LLONG_FROM_FLOAT,
+                      [lltype.Float], lltype.UnsignedLongLong)
         self.do_check('cast_longlong_to_float', EffectInfo.OS_LLONG_TO_FLOAT,
                       [lltype.SignedLongLong], lltype.Float)
+        self.do_check('cast_ulonglong_to_float', EffectInfo.OS_LLONG_U_TO_FLOAT,
+                      [lltype.UnsignedLongLong], lltype.Float)
         for T1 in [lltype.SignedLongLong, lltype.UnsignedLongLong]:
             for T2 in [lltype.Signed, lltype.Unsigned]:
                 self.do_check('cast_primitive', EffectInfo.OS_LLONG_TO_INT,
diff --git a/pypy/jit/metainterp/optimizeopt/__init__.py b/pypy/jit/metainterp/optimizeopt/__init__.py
--- a/pypy/jit/metainterp/optimizeopt/__init__.py
+++ b/pypy/jit/metainterp/optimizeopt/__init__.py
@@ -7,6 +7,8 @@
 from pypy.jit.metainterp.optimizeopt.unroll import optimize_unroll, OptInlineShortPreamble
 from pypy.jit.metainterp.optimizeopt.fficall import OptFfiCall
 from pypy.jit.metainterp.optimizeopt.simplify import OptSimplify
+from pypy.jit.metainterp.optimizeopt.pure import OptPure
+from pypy.jit.metainterp.optimizeopt.earlyforce import OptEarlyForce
 from pypy.rlib.jit import PARAMETERS
 from pypy.rlib.unroll import unrolling_iterable
 
@@ -14,6 +16,8 @@
             ('rewrite', OptRewrite),
             ('virtualize', OptVirtualize),
             ('string', OptString),
+            ('earlyforce', OptEarlyForce),
+            ('pure', OptPure),
             ('heap', OptHeap),
             ('ffi', None),
             ('unroll', None)]
diff --git a/pypy/jit/metainterp/optimizeopt/earlyforce.py b/pypy/jit/metainterp/optimizeopt/earlyforce.py
new file mode 100644
--- /dev/null
+++ b/pypy/jit/metainterp/optimizeopt/earlyforce.py
@@ -0,0 +1,24 @@
+from pypy.jit.metainterp.optimizeopt.optimizer import Optimization
+from pypy.jit.metainterp.optimizeopt.vstring import VAbstractStringValue
+from pypy.jit.metainterp.resoperation import rop, ResOperation
+
+class OptEarlyForce(Optimization):
+    def propagate_forward(self, op):
+        opnum = op.getopnum()
+        if (opnum != rop.SETFIELD_GC and 
+            opnum != rop.SETARRAYITEM_GC and
+            opnum != rop.QUASIIMMUT_FIELD):
+               
+            for arg in op.getarglist():
+                if arg in self.optimizer.values:
+                    value = self.getvalue(arg)
+                    value.force_box(self)
+        self.emit_operation(op)
+
+    def new(self):
+        return OptEarlyForce()
+
+    def setup(self):
+        self.optimizer.optearlyforce = self
+
+    
diff --git a/pypy/jit/metainterp/optimizeopt/fficall.py b/pypy/jit/metainterp/optimizeopt/fficall.py
--- a/pypy/jit/metainterp/optimizeopt/fficall.py
+++ b/pypy/jit/metainterp/optimizeopt/fficall.py
@@ -177,10 +177,10 @@
             funcinfo.descr is None):
             return [op] # cannot optimize
         funcsymval = self.getvalue(op.getarg(2))
-        arglist = [funcsymval.force_box()]
+        arglist = [funcsymval.get_key_box()]
         for push_op in funcinfo.opargs:
             argval = self.getvalue(push_op.getarg(2))
-            arglist.append(argval.force_box())
+            arglist.append(argval.get_key_box())
         newop = ResOperation(rop.CALL_RELEASE_GIL, arglist, op.result,
                              descr=funcinfo.descr)
         self.commit_optimization()
diff --git a/pypy/jit/metainterp/optimizeopt/heap.py b/pypy/jit/metainterp/optimizeopt/heap.py
--- a/pypy/jit/metainterp/optimizeopt/heap.py
+++ b/pypy/jit/metainterp/optimizeopt/heap.py
@@ -50,6 +50,7 @@
             if not self._lazy_setfield_registered:
                 optheap._lazy_setfields_and_arrayitems.append(self)
                 self._lazy_setfield_registered = True
+            
         else:
             # this is the case where the pending setfield ends up
             # storing precisely the value that is already there,
@@ -158,12 +159,17 @@
         self._lazy_setfields_and_arrayitems = []
         self._remove_guard_not_invalidated = False
         self._seen_guard_not_invalidated = False
+        self.posponedop = None
 
     def force_at_end_of_preamble(self):
         self.force_all_lazy_setfields_and_arrayitems()
 
     def flush(self):
         self.force_all_lazy_setfields_and_arrayitems()
+        if self.posponedop:
+            posponedop = self.posponedop
+            self.posponedop = None
+            self.next_optimization.propagate_forward(posponedop)
 
     def new(self):
         return OptHeap()
@@ -211,7 +217,15 @@
 
     def emit_operation(self, op):
         self.emitting_operation(op)
-        self.next_optimization.propagate_forward(op)
+        if self.posponedop:
+            posponedop = self.posponedop
+            self.posponedop = None
+            self.next_optimization.propagate_forward(posponedop)
+        if (op.is_comparison() or op.getopnum() == rop.CALL_MAY_FORCE
+            or op.is_ovf()):
+            self.posponedop = op
+        else:
+            self.next_optimization.propagate_forward(op)
 
     def emitting_operation(self, op):
         if op.has_no_side_effect():
@@ -293,30 +307,6 @@
             if indexvalue is None or indexvalue.intbound.contains(idx):
                 cf.force_lazy_setfield(self, can_cache)
 
-    def fixup_guard_situation(self):
-        # hackish: reverse the order of the last two operations if it makes
-        # sense to avoid a situation like "int_eq/setfield_gc/guard_true",
-        # which the backend (at least the x86 backend) does not handle well.
-        newoperations = self.optimizer.newoperations
-        if len(newoperations) < 2:
-            return
-        lastop = newoperations[-1]
-        if (lastop.getopnum() != rop.SETFIELD_GC and
-            lastop.getopnum() != rop.SETARRAYITEM_GC):
-            return
-        # - is_comparison() for cases like "int_eq/setfield_gc/guard_true"
-        # - CALL_MAY_FORCE: "call_may_force/setfield_gc/guard_not_forced"
-        # - is_ovf(): "int_add_ovf/setfield_gc/guard_no_overflow"
-        prevop = newoperations[-2]
-        opnum = prevop.getopnum()
-        if not (prevop.is_comparison() or opnum == rop.CALL_MAY_FORCE
-                or prevop.is_ovf()):
-            return
-        if prevop.result in lastop.getarglist():
-            return
-        newoperations[-2] = lastop
-        newoperations[-1] = prevop
-
     def _assert_valid_cf(self, cf):
         # check that 'cf' is in cached_fields or cached_arrayitems
         if not we_are_translated():
@@ -362,7 +352,6 @@
                                       fieldvalue.get_key_box(), itemindex))
             else:
                 cf.force_lazy_setfield(self)
-                self.fixup_guard_situation()
         return pendingfields
 
     def optimize_GETFIELD_GC(self, op):
@@ -374,12 +363,22 @@
             return
         # default case: produce the operation
         structvalue.ensure_nonnull()
-        ###self.optimizer.optimize_default(op)
         self.emit_operation(op)
         # then remember the result of reading the field
         fieldvalue = self.getvalue(op.result)
         cf.remember_field_value(structvalue, fieldvalue, op)
 
+    def optimize_GETFIELD_GC_PURE(self, op):
+        structvalue = self.getvalue(op.getarg(0))
+        cf = self.field_cache(op.getdescr())
+        fieldvalue = cf.getfield_from_cache(self, structvalue)
+        if fieldvalue is not None:
+            self.make_equal_to(op.result, fieldvalue)
+            return
+        # default case: produce the operation
+        structvalue.ensure_nonnull()
+        self.emit_operation(op)
+
     def optimize_SETFIELD_GC(self, op):
         if self.has_pure_result(rop.GETFIELD_GC_PURE, [op.getarg(0)],
                                 op.getdescr()):
@@ -389,6 +388,7 @@
         #
         cf = self.field_cache(op.getdescr())
         cf.do_setfield(self, op)
+        
 
     def optimize_GETARRAYITEM_GC(self, op):
         arrayvalue = self.getvalue(op.getarg(0))
@@ -413,6 +413,25 @@
             fieldvalue = self.getvalue(op.result)
             cf.remember_field_value(arrayvalue, fieldvalue, op)
 
+    def optimize_GETARRAYITEM_GC_PURE(self, op):
+        arrayvalue = self.getvalue(op.getarg(0))
+        indexvalue = self.getvalue(op.getarg(1))
+        cf = None
+        if indexvalue.is_constant():
+            arrayvalue.make_len_gt(MODE_ARRAY, op.getdescr(), indexvalue.box.getint())
+            # use the cache on (arraydescr, index), which is a constant
+            cf = self.arrayitem_cache(op.getdescr(), indexvalue.box.getint())
+            fieldvalue = cf.getfield_from_cache(self, arrayvalue)
+            if fieldvalue is not None:
+                self.make_equal_to(op.result, fieldvalue)
+                return
+        else:
+            # variable index, so make sure the lazy setarrayitems are done
+            self.force_lazy_setarrayitem(op.getdescr(), indexvalue=indexvalue)
+        # default case: produce the operation
+        arrayvalue.ensure_nonnull()
+        self.emit_operation(op)
+
     def optimize_SETARRAYITEM_GC(self, op):
         if self.has_pure_result(rop.GETARRAYITEM_GC_PURE, [op.getarg(0),
                                                            op.getarg(1)],
diff --git a/pypy/jit/metainterp/optimizeopt/optimizer.py b/pypy/jit/metainterp/optimizeopt/optimizer.py
--- a/pypy/jit/metainterp/optimizeopt/optimizer.py
+++ b/pypy/jit/metainterp/optimizeopt/optimizer.py
@@ -31,8 +31,8 @@
 
 class OptValue(object):
     __metaclass__ = extendabletype
-    _attrs_ = ('box', 'known_class', 'last_guard_index', 'level', 'intbound', 'lenbound')
-    last_guard_index = -1
+    _attrs_ = ('box', 'known_class', 'last_guard', 'level', 'intbound', 'lenbound')
+    last_guard = None
 
     level = LEVEL_UNKNOWN
     known_class = None
@@ -100,7 +100,7 @@
             self.make_constant(other.get_key_box())
             optimizer.turned_constant(self)
         elif other.level == LEVEL_KNOWNCLASS:
-            self.make_constant_class(other.known_class, -1)
+            self.make_constant_class(other.known_class, None)
         else:
             if other.level == LEVEL_NONNULL:
                 self.ensure_nonnull()
@@ -114,13 +114,13 @@
                     self.lenbound = other.lenbound.clone()
 
 
-    def force_box(self):
+    def force_box(self, optforce):
         return self.box
 
     def get_key_box(self):
         return self.box
 
-    def force_at_end_of_preamble(self, already_forced):
+    def force_at_end_of_preamble(self, already_forced, optforce):
         return self
 
     def get_args_for_fail(self, modifier):
@@ -162,16 +162,16 @@
         else:
             return None
 
-    def make_constant_class(self, classbox, opindex):
+    def make_constant_class(self, classbox, guardop):
         assert self.level < LEVEL_KNOWNCLASS
         self.known_class = classbox
         self.level = LEVEL_KNOWNCLASS
-        self.last_guard_index = opindex
+        self.last_guard = guardop
 
-    def make_nonnull(self, opindex):
+    def make_nonnull(self, guardop):
         assert self.level < LEVEL_NONNULL
         self.level = LEVEL_NONNULL
-        self.last_guard_index = opindex
+        self.last_guard = guardop
 
     def is_nonnull(self):
         level = self.level
@@ -240,26 +240,12 @@
     def __init__(self):
         pass # make rpython happy
 
-    def propagate_begin_forward(self):
-        if self.next_optimization:
-            self.next_optimization.propagate_begin_forward()
-
-    def propagate_end_forward(self):
-        if self.next_optimization:
-            self.next_optimization.propagate_end_forward()
-
     def propagate_forward(self, op):
         raise NotImplementedError
 
     def emit_operation(self, op):
         self.next_optimization.propagate_forward(op)
 
-    def test_emittable(self, op):
-        return self.is_emittable(op)
-
-    def is_emittable(self, op):
-        return self.next_optimization.test_emittable(op)
-
     # FIXME: Move some of these here?
     def getvalue(self, box):
         return self.optimizer.getvalue(box)
@@ -289,19 +275,19 @@
         return self.optimizer.new_const_item(arraydescr)
 
     def pure(self, opnum, args, result):
-        op = ResOperation(opnum, args, result)
-        key = self.optimizer.make_args_key(op)
-        if key not in self.optimizer.pure_operations:
-            self.optimizer.pure_operations[key] = op
+        if self.optimizer.optpure:
+            self.optimizer.optpure.pure(opnum, args, result)
 
     def has_pure_result(self, opnum, args, descr):
-        op = ResOperation(opnum, args, None, descr)
-        key = self.optimizer.make_args_key(op)
-        op = self.optimizer.pure_operations.get(key, None)
-        if op is None:
-            return False
-        return op.getdescr() is descr
+        if self.optimizer.optpure:
+            return self.optimizer.optpure.has_pure_result(opnum, args, descr)
+        return False
 
+    def get_pure_result(self, key):    
+        if self.optimizer.optpure:
+            return self.optimizer.optpure.get_pure_result(key)
+        return None
+    
     def setup(self):
         pass
 
@@ -322,6 +308,9 @@
     def produce_potential_short_preamble_ops(self, potential_ops):
         pass
 
+    def forget_numberings(self, box):
+        self.optimizer.forget_numberings(box)
+
 class Optimizer(Optimization):
 
     def __init__(self, metainterp_sd, loop, optimizations=None, bridge=False):
@@ -333,14 +322,16 @@
         self.interned_refs = self.cpu.ts.new_ref_dict()
         self.resumedata_memo = resume.ResumeDataLoopMemo(metainterp_sd)
         self.bool_boxes = {}
-        self.pure_operations = args_dict()
         self.producer = {}
         self.pendingfields = []
-        self.posponedop = None
         self.exception_might_have_happened = False
         self.quasi_immutable_deps = None
         self.opaque_pointers = {}
-        self.newoperations = []
+        self.replaces_guard = {}
+        self._newoperations = []
+        self.optimizer = self
+        self.optpure = None
+        self.optearlyforce = None
         if loop is not None:
             self.call_pure_results = loop.call_pure_results
 
@@ -369,21 +360,20 @@
     def flush(self):
         for o in self.optimizations:
             o.flush()
-        assert self.posponedop is None
 
     def new(self):
         new = Optimizer(self.metainterp_sd, self.loop)
         return self._new(new)
 
     def _new(self, new):
-        assert self.posponedop is None
         optimizations = [o.new() for o in self.optimizations]
         new.set_optimizations(optimizations)
         new.quasi_immutable_deps = self.quasi_immutable_deps
         return new
 
     def produce_potential_short_preamble_ops(self, sb):
-        raise NotImplementedError('This is implemented in unroll.UnrollableOptimizer')
+        for opt in self.optimizations:
+            opt.produce_potential_short_preamble_ops(sb)
 
     def turned_constant(self, value):
         for o in self.optimizations:
@@ -433,6 +423,13 @@
             return constbox
         return None
 
+    def get_newoperations(self):
+        self.flush()
+        return self._newoperations
+
+    def clear_newoperations(self):
+        self._newoperations = []
+
     def make_equal_to(self, box, value, replace=False):
         assert isinstance(value, OptValue)
         assert replace or box not in self.values
@@ -481,15 +478,10 @@
 
     def propagate_all_forward(self):
         self.exception_might_have_happened = self.bridge
-        self.newoperations = []
-        self.first_optimization.propagate_begin_forward()
-        self.i = 0
-        while self.i < len(self.loop.operations):
-            op = self.loop.operations[self.i]
+        self.clear_newoperations()
+        for op in self.loop.operations:
             self.first_optimization.propagate_forward(op)
-            self.i += 1
-        self.first_optimization.propagate_end_forward()
-        self.loop.operations = self.newoperations
+        self.loop.operations = self.get_newoperations()
         self.loop.quasi_immutable_deps = self.quasi_immutable_deps
         # accumulate counters
         self.resumedata_memo.update_counters(self.metainterp_sd.profiler)
@@ -501,9 +493,6 @@
         self.producer[op.result] = op
         dispatch_opt(self, op)
 
-    def test_emittable(self, op):
-        return True
-
     def emit_operation(self, op):
         if op.returns_bool_result():
             self.bool_boxes[self.getvalue(op.result)] = None
@@ -519,14 +508,30 @@
                 pass
             else:
                 self.ensure_imported(value)
-                op.setarg(i, value.force_box())
+                op.setarg(i, value.force_box(self))
         self.metainterp_sd.profiler.count(jitprof.OPT_OPS)
         if op.is_guard():
             self.metainterp_sd.profiler.count(jitprof.OPT_GUARDS)
-            op = self.store_final_boxes_in_guard(op)
+            if self.replaces_guard and op in self.replaces_guard:
+                self.replace_op(self.replaces_guard[op], op)
+                del self.replaces_guard[op]
+                return
+            else:
+                op = self.store_final_boxes_in_guard(op)
         elif op.can_raise():
             self.exception_might_have_happened = True
-        self.newoperations.append(op)
+        self._newoperations.append(op)
+
+    def replace_op(self, old_op, new_op):
+        # XXX: Do we want to cache indexes to prevent search?
+        i = len(self._newoperations) 
+        while i > 0:
+            i -= 1
+            if self._newoperations[i] is old_op:
+                self._newoperations[i] = new_op
+                break
+        else:
+            assert False
 
     def store_final_boxes_in_guard(self, op):
         descr = op.getdescr()
@@ -574,51 +579,8 @@
         args[n + 1] = op.getdescr()
         return args
 
-    @specialize.argtype(0)
     def optimize_default(self, op):
-        canfold = op.is_always_pure()
-        if op.is_ovf():
-            self.posponedop = op
-            return
-        if self.posponedop:
-            nextop = op
-            op = self.posponedop
-            self.posponedop = None
-            canfold = nextop.getopnum() == rop.GUARD_NO_OVERFLOW
-        else:
-            nextop = None
-
-        if canfold:
-            for i in range(op.numargs()):
-                if self.get_constant_box(op.getarg(i)) is None:
-                    break
-            else:
-                # all constant arguments: constant-fold away
-                resbox = self.constant_fold(op)
-                # note that INT_xxx_OVF is not done from here, and the
-                # overflows in the INT_xxx operations are ignored
-                self.make_constant(op.result, resbox)
-                return
-
-            # did we do the exact same operation already?
-            args = self.make_args_key(op)
-            oldop = self.pure_operations.get(args, None)
-            if oldop is not None and oldop.getdescr() is op.getdescr():
-                assert oldop.getopnum() == op.getopnum()
-                self.make_equal_to(op.result, self.getvalue(oldop.result),
-                                   True)
-                return
-            else:
-                self.pure_operations[args] = op
-                self.remember_emitting_pure(op)
-
-        # otherwise, the operation remains
         self.emit_operation(op)
-        if nextop:
-            self.emit_operation(nextop)
-
-    def remember_emitting_pure(self, op):
-        pass
 
     def constant_fold(self, op):
         argboxes = [self.get_constant_box(op.getarg(i))
@@ -636,11 +598,6 @@
     def optimize_DEBUG_MERGE_POINT(self, op):
         self.emit_operation(op)
 
-    def optimize_CAST_OPAQUE_PTR(self, op):
-        value = self.getvalue(op.getarg(0))
-        self.opaque_pointers[value] = True
-        self.make_equal_to(op.result, value)
-
     def optimize_GETARRAYITEM_GC_PURE(self, op):
         indexvalue = self.getvalue(op.getarg(1))
         if indexvalue.is_constant():
diff --git a/pypy/jit/metainterp/optimizeopt/pure.py b/pypy/jit/metainterp/optimizeopt/pure.py
new file mode 100644
--- /dev/null
+++ b/pypy/jit/metainterp/optimizeopt/pure.py
@@ -0,0 +1,115 @@
+from pypy.jit.metainterp.optimizeopt.optimizer import Optimization
+from pypy.jit.metainterp.resoperation import rop, ResOperation
+from pypy.jit.metainterp.optimizeopt.util import (make_dispatcher_method,
+    args_dict)
+
+class OptPure(Optimization):
+    def __init__(self):
+        self.posponedop = None
+        self.pure_operations = args_dict()
+        self.emitted_pure_operations = {}
+
+    def propagate_forward(self, op):
+        dispatch_opt(self, op)
+
+    def optimize_default(self, op):
+        canfold = op.is_always_pure()
+        if op.is_ovf():
+            self.posponedop = op
+            return
+        if self.posponedop:
+            nextop = op
+            op = self.posponedop
+            self.posponedop = None
+            canfold = nextop.getopnum() == rop.GUARD_NO_OVERFLOW
+        else:
+            nextop = None
+
+        if canfold:
+            for i in range(op.numargs()):
+                if self.get_constant_box(op.getarg(i)) is None:
+                    break
+            else:
+                # all constant arguments: constant-fold away
+                resbox = self.optimizer.constant_fold(op)
+                # note that INT_xxx_OVF is not done from here, and the
+                # overflows in the INT_xxx operations are ignored
+                self.optimizer.make_constant(op.result, resbox)
+                return
+
+            # did we do the exact same operation already?
+            args = self.optimizer.make_args_key(op)
+            oldop = self.pure_operations.get(args, None)
+            if oldop is not None and oldop.getdescr() is op.getdescr():
+                assert oldop.getopnum() == op.getopnum()
+                self.optimizer.make_equal_to(op.result, self.getvalue(oldop.result),
+                                   True)
+                return
+            else:
+                self.pure_operations[args] = op
+                self.remember_emitting_pure(op)
+
+        # otherwise, the operation remains
+        self.emit_operation(op)
+        if op.returns_bool_result():
+            self.optimizer.bool_boxes[self.getvalue(op.result)] = None        
+        if nextop:
+            self.emit_operation(nextop)
+
+    def optimize_CALL_PURE(self, op):
+        args = self.optimizer.make_args_key(op)
+        oldop = self.pure_operations.get(args, None)
+        if oldop is not None and oldop.getdescr() is op.getdescr():
+            assert oldop.getopnum() == op.getopnum()
+            self.make_equal_to(op.result, self.getvalue(oldop.result))
+            return
+        else:
+            self.pure_operations[args] = op
+            self.remember_emitting_pure(op)
+
+        # replace CALL_PURE with just CALL
+        args = op.getarglist()
+        self.emit_operation(ResOperation(rop.CALL, args, op.result,
+                                         op.getdescr()))
+
+    def flush(self):
+        assert self.posponedop is None
+
+    def new(self):
+        assert self.posponedop is None
+        return OptPure()
+
+    def setup(self):
+        self.optimizer.optpure = self
+
+    def pure(self, opnum, args, result):
+        op = ResOperation(opnum, args, result)
+        key = self.optimizer.make_args_key(op)
+        if key not in self.pure_operations:
+            self.pure_operations[key] = op
+
+    def has_pure_result(self, opnum, args, descr):
+        op = ResOperation(opnum, args, None, descr)
+        key = self.optimizer.make_args_key(op)
+        op = self.pure_operations.get(key, None)
+        if op is None:
+            return False
+        return op.getdescr() is descr
+
+    def get_pure_result(self, key):
+        return self.pure_operations.get(key, None)
+
+    def remember_emitting_pure(self, op):
+        self.emitted_pure_operations[op] = True
+
+    def produce_potential_short_preamble_ops(self, sb):
+        for op in self.emitted_pure_operations:
+            if op.getopnum() == rop.GETARRAYITEM_GC_PURE or \
+               op.getopnum() == rop.STRGETITEM or \
+               op.getopnum() == rop.UNICODEGETITEM:
+                if not self.getvalue(op.getarg(1)).is_constant():
+                    continue
+            sb.add_potential(op)
+
+dispatch_opt = make_dispatcher_method(OptPure, 'optimize_',
+                                      default=OptPure.optimize_default)
diff --git a/pypy/jit/metainterp/optimizeopt/rewrite.py b/pypy/jit/metainterp/optimizeopt/rewrite.py
--- a/pypy/jit/metainterp/optimizeopt/rewrite.py
+++ b/pypy/jit/metainterp/optimizeopt/rewrite.py
@@ -31,21 +31,8 @@
 
         dispatch_opt(self, op)
 
-    def test_emittable(self, op):
-        opnum = op.getopnum()
-        for value, cls, func in optimize_guards:
-            if opnum == value:
-                assert isinstance(op, cls)
-                try:
-                    func(self, op, dryrun=True)
-                    return self.is_emittable(op)
-                except InvalidLoop:
-                    return False
-        return self.is_emittable(op)
-
-
     def try_boolinvers(self, op, targs):
-        oldop = self.optimizer.pure_operations.get(targs, None)
+        oldop = self.get_pure_result(targs)
         if oldop is not None and oldop.getdescr() is op.getdescr():
             value = self.getvalue(oldop.result)
             if value.is_constant():
@@ -73,7 +60,7 @@
             oldopnum = opboolreflex[op.getopnum()] # FIXME: add INT_ADD, INT_MUL
             targs = self.optimizer.make_args_key(ResOperation(oldopnum, [args[1], args[0]],
                                                               None))
-            oldop = self.optimizer.pure_operations.get(targs, None)
+            oldop = self.get_pure_result(targs)
             if oldop is not None and oldop.getdescr() is op.getdescr():
                 self.make_equal_to(op.result, self.getvalue(oldop.result))
                 return True
@@ -214,40 +201,7 @@
         self.emit_operation(op)
         self.pure(rop.FLOAT_NEG, [op.result], v1)
 
-    def optimize_CALL_PURE(self, op):
-        arg_consts = []
-        for i in range(op.numargs()):
-            arg = op.getarg(i)
-            const = self.get_constant_box(arg)
-            if const is None:
-                break
-            arg_consts.append(const)
-        else:
-            # all constant arguments: check if we already know the result
-            try:
-                result = self.optimizer.call_pure_results[arg_consts]
-            except KeyError:
-                pass
-            else:
-                self.make_constant(op.result, result)
-                return
-
-        args = self.optimizer.make_args_key(op)
-        oldop = self.optimizer.pure_operations.get(args, None)
-        if oldop is not None and oldop.getdescr() is op.getdescr():
-            assert oldop.getopnum() == op.getopnum()
-            self.make_equal_to(op.result, self.getvalue(oldop.result))
-            return
-        else:
-            self.optimizer.pure_operations[args] = op
-            self.optimizer.remember_emitting_pure(op)
-
-        # replace CALL_PURE with just CALL
-        args = op.getarglist()
-        self.emit_operation(ResOperation(rop.CALL, args, op.result,
-                                         op.getdescr()))
-
-    def optimize_guard(self, op, constbox, emit_operation=True, dryrun=False):
+    def optimize_guard(self, op, constbox, emit_operation=True):
         value = self.getvalue(op.getarg(0))
         if value.is_constant():
             box = value.box
@@ -255,62 +209,57 @@
             if not box.same_constant(constbox):
                 raise InvalidLoop
             return
-        if dryrun: return
         if emit_operation:
             self.emit_operation(op)
         value.make_constant(constbox)
         self.optimizer.turned_constant(value)
 
-    def optimize_GUARD_ISNULL(self, op, dryrun=False):
+    def optimize_GUARD_ISNULL(self, op):
         value = self.getvalue(op.getarg(0))
         if value.is_null():
             return
         elif value.is_nonnull():
             raise InvalidLoop
-        if dryrun: return
         self.emit_operation(op)
         value.make_constant(self.optimizer.cpu.ts.CONST_NULL)
 
-    def optimize_GUARD_NONNULL(self, op, dryrun=False):
+    def optimize_GUARD_NONNULL(self, op):
         value = self.getvalue(op.getarg(0))
         if value.is_nonnull():
             return
         elif value.is_null():
             raise InvalidLoop
-        if dryrun: return
         self.emit_operation(op)
-        value.make_nonnull(len(self.optimizer.newoperations) - 1)
+        value.make_nonnull(op)
 
-    def optimize_GUARD_VALUE(self, op, dryrun=False):
+    def optimize_GUARD_VALUE(self, op):
         value = self.getvalue(op.getarg(0))
-        emit_operation = True
-        if not dryrun and value.last_guard_index != -1:
+        if value.last_guard:
             # there already has been a guard_nonnull or guard_class or
             # guard_nonnull_class on this value, which is rather silly.
             # replace the original guard with a guard_value
-            old_guard_op = self.optimizer.newoperations[value.last_guard_index]
-            new_guard_op = old_guard_op.copy_and_change(rop.GUARD_VALUE,
-                                             args = [old_guard_op.getarg(0), op.getarg(1)])
-            self.optimizer.newoperations[value.last_guard_index] = new_guard_op
+            old_guard_op = value.last_guard
+            op = old_guard_op.copy_and_change(rop.GUARD_VALUE,
+                                      args = [old_guard_op.getarg(0), op.getarg(1)])
+            self.optimizer.replaces_guard[op] = old_guard_op
             # hack hack hack.  Change the guard_opnum on
             # new_guard_op.getdescr() so that when resuming,
             # the operation is not skipped by pyjitpl.py.
-            descr = new_guard_op.getdescr()
+            descr = op.getdescr()
             assert isinstance(descr, compile.ResumeGuardDescr)
             descr.guard_opnum = rop.GUARD_VALUE
-            descr.make_a_counter_per_value(new_guard_op)
-            emit_operation = False
+            descr.make_a_counter_per_value(op)
         constbox = op.getarg(1)
         assert isinstance(constbox, Const)
-        self.optimize_guard(op, constbox, emit_operation, dryrun)
+        self.optimize_guard(op, constbox)
 
-    def optimize_GUARD_TRUE(self, op, dryrun=False):
-        self.optimize_guard(op, CONST_1, dryrun=dryrun)
+    def optimize_GUARD_TRUE(self, op):
+        self.optimize_guard(op, CONST_1)
 
-    def optimize_GUARD_FALSE(self, op, dryrun=False):
-        self.optimize_guard(op, CONST_0, dryrun=dryrun)
+    def optimize_GUARD_FALSE(self, op):
+        self.optimize_guard(op, CONST_0)
 
-    def optimize_GUARD_CLASS(self, op, dryrun=False):
+    def optimize_GUARD_CLASS(self, op):
         value = self.getvalue(op.getarg(0))
         expectedclassbox = op.getarg(1)
         assert isinstance(expectedclassbox, Const)
@@ -319,38 +268,32 @@
             if realclassbox.same_constant(expectedclassbox):
                 return
             raise InvalidLoop
-        if dryrun: return
-        emit_operation = True
-        if value.last_guard_index != -1:
+        if value.last_guard:
             # there already has been a guard_nonnull or guard_class or
             # guard_nonnull_class on this value.
-            old_guard_op = self.optimizer.newoperations[value.last_guard_index]
+            old_guard_op = value.last_guard
             if old_guard_op.getopnum() == rop.GUARD_NONNULL:
                 # it was a guard_nonnull, which we replace with a
                 # guard_nonnull_class.
-                new_guard_op = old_guard_op.copy_and_change (rop.GUARD_NONNULL_CLASS,
+                op = old_guard_op.copy_and_change (rop.GUARD_NONNULL_CLASS,
                                          args = [old_guard_op.getarg(0), op.getarg(1)])
-                self.optimizer.newoperations[value.last_guard_index] = new_guard_op
+                self.optimizer.replaces_guard[op] = old_guard_op
                 # hack hack hack.  Change the guard_opnum on
                 # new_guard_op.getdescr() so that when resuming,
                 # the operation is not skipped by pyjitpl.py.
-                descr = new_guard_op.getdescr()
+                descr = op.getdescr()
                 assert isinstance(descr, compile.ResumeGuardDescr)
                 descr.guard_opnum = rop.GUARD_NONNULL_CLASS
-                emit_operation = False
-        if emit_operation:
-            self.emit_operation(op)
-            last_guard_index = len(self.optimizer.newoperations) - 1
-        else:
-            last_guard_index = value.last_guard_index
-        value.make_constant_class(expectedclassbox, last_guard_index)
+        self.emit_operation(op)
+        value.make_constant_class(expectedclassbox, op)
 
-    def optimize_GUARD_NONNULL_CLASS(self, op, dryrun=False):
-        self.optimize_GUARD_NONNULL(op, True)
-        self.optimize_GUARD_CLASS(op, dryrun)
+    def optimize_GUARD_NONNULL_CLASS(self, op):
+        value = self.getvalue(op.getarg(0))
+        if value.is_null():
+            raise InvalidLoop
+        self.optimize_GUARD_CLASS(op)
 
-    def optimize_GUARD_NO_EXCEPTION(self, op, dryrun=False):
-        if dryrun: return
+    def optimize_GUARD_NO_EXCEPTION(self, op):
         if not self.optimizer.exception_might_have_happened:
             return
         self.emit_operation(op)
@@ -470,7 +413,7 @@
                     newop = ResOperation(rop.SETARRAYITEM_GC,
                                          [op.getarg(2),
                                           ConstInt(index + dest_start),
-                                          val.force_box()], None,
+                                          val.get_key_box()], None,
                                          descr=source_value.arraydescr)
                     self.emit_operation(newop)
             return True
@@ -478,6 +421,25 @@
             return True # 0-length arraycopy
         return False
 
+    def optimize_CALL_PURE(self, op):
+        arg_consts = []
+        for i in range(op.numargs()):
+            arg = op.getarg(i)
+            const = self.get_constant_box(arg)
+            if const is None:
+                break
+            arg_consts.append(const)
+        else:
+            # all constant arguments: check if we already know the result
+            try:
+                result = self.optimizer.call_pure_results[arg_consts]
+            except KeyError:
+                pass
+            else:
+                self.make_constant(op.result, result)
+                return
+        self.emit_operation(op)
+
     def optimize_INT_FLOORDIV(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
@@ -492,6 +454,10 @@
                                         args = [op.getarg(0), ConstInt(highest_bit(val))])
         self.emit_operation(op)
 
+    def optimize_CAST_OPAQUE_PTR(self, op):
+        value = self.getvalue(op.getarg(0))
+        self.optimizer.opaque_pointers[value] = True
+        self.make_equal_to(op.result, value)
 
 dispatch_opt = make_dispatcher_method(OptRewrite, 'optimize_',
         default=OptRewrite.emit_operation)
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -39,8 +39,9 @@
 
 def test_sharing_field_lists_of_virtual():
     class FakeOptimizer(object):
-        class cpu(object):
-            pass
+        class optimizer(object):
+            class cpu(object):
+                pass
     opt = FakeOptimizer()
     virt1 = virtualize.AbstractVirtualStructValue(opt, None)
     lst1 = virt1._get_field_descr_list()
@@ -69,7 +70,7 @@
     class FakeVirtualValue(virtualize.AbstractVirtualValue):
         def _make_virtual(self, *args):
             return FakeVInfo()
-    v1 = FakeVirtualValue(None, None, None)
+    v1 = FakeVirtualValue(None, None)
     vinfo1 = v1.make_virtual_info(None, [1, 2, 4])
     vinfo2 = v1.make_virtual_info(None, [1, 2, 4])
     assert vinfo1 is vinfo2
@@ -111,7 +112,7 @@
 
 class BaseTestBasic(BaseTest):
 
-    enable_opts = "intbounds:rewrite:virtualize:string:heap"
+    enable_opts = "intbounds:rewrite:virtualize:string:earlyforce:pure:heap"
 
     def optimize_loop(self, ops, optops, call_pure_results=None):
 
@@ -651,8 +652,8 @@
         i3 = getfield_gc(p3, descr=valuedescr)
         escape(i3)
         p1 = new_with_vtable(ConstClass(node_vtable))
+        p1sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p1, i1, descr=valuedescr)
-        p1sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p1sub, i1, descr=valuedescr)
         setfield_gc(p1, p1sub, descr=nextdescr)
         jump(i1, p1, p2)
@@ -667,10 +668,10 @@
         p3sub = getfield_gc(p3, descr=nextdescr)
         i3 = getfield_gc(p3sub, descr=valuedescr)
         escape(i3)
+        p1 = new_with_vtable(ConstClass(node_vtable))
         p2sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p2sub, i1, descr=valuedescr)
         setfield_gc(p2, p2sub, descr=nextdescr)
-        p1 = new_with_vtable(ConstClass(node_vtable))
         jump(i1, p1, p2)
         """
         # The same as test_p123_simple, but in the end the "old" p2 contains
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizefficall.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizefficall.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizefficall.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizefficall.py
@@ -36,7 +36,7 @@
 
 class TestFfiCall(BaseTestBasic, LLtypeMixin):
 
-    enable_opts = "intbounds:rewrite:virtualize:string:heap:ffi"
+    enable_opts = "intbounds:rewrite:virtualize:string:pure:earlyforce:heap:ffi"
 
     class namespace:
         cpu = LLtypeMixin.cpu
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -68,7 +68,7 @@
 
 class BaseTestWithUnroll(BaseTest):
 
-    enable_opts = "intbounds:rewrite:virtualize:string:heap:unroll"
+    enable_opts = "intbounds:rewrite:virtualize:string:earlyforce:pure:heap:unroll"
 
     def optimize_loop(self, ops, expected, expected_preamble=None,
                       call_pure_results=None, expected_short=None):
@@ -825,8 +825,8 @@
         i3 = getfield_gc(p2, descr=valuedescr)
         escape(i3)
         p4 = new_with_vtable(ConstClass(node_vtable))
+        p1sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p4, i1, descr=valuedescr)
-        p1sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p1sub, i1, descr=valuedescr)
         setfield_gc(p4, p1sub, descr=nextdescr)
         jump(i1, p4)
@@ -865,13 +865,7 @@
         p3sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p3sub, i1, descr=valuedescr)
         setfield_gc(p1, p3sub, descr=nextdescr)
-        # XXX: We get two extra operations here because the setfield
-        #      above is the result of forcing p1 and thus not
-        #      registered with the heap optimizer. I've makred tests
-        #      below with VIRTUALHEAP if they suffer from this issue
-        p3sub2 = getfield_gc(p1, descr=nextdescr)
-        guard_nonnull_class(p3sub2, ConstClass(node_vtable2)) []
-        jump(i1, p1, p3sub2)
+        jump(i1, p1, p3sub)
         """
         self.optimize_loop(ops, expected, preamble)
 
@@ -902,9 +896,7 @@
         guard_true(i2b) []
         p3 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3, i2, descr=nextdescr)
-        # XXX: VIRTUALHEAP (see above)
-        i3 = getfield_gc(p3, descr=nextdescr)
-        jump(p3, i3)
+        jump(p3, i2)
         """
         self.optimize_loop(ops, expected, preamble)
 
@@ -1219,7 +1211,15 @@
         setfield_gc(p3, p30, descr=valuedescr)
         jump(i29, p30, p3)
         """
-        expected = preamble
+        expected = """
+        [i0, p1, p3]
+        i28 = int_add(i0, 1)
+        i29 = int_add(i28, 1)
+        p30 = new_with_vtable(ConstClass(node_vtable))
+        setfield_gc(p3, p30, descr=valuedescr)
+        setfield_gc(p30, i28, descr=nextdescr)
+        jump(i29, p30, p3)
+        """
         self.optimize_loop(ops, expected, preamble)
 
     def test_nonvirtual_1(self):
@@ -2408,8 +2408,8 @@
         guard_class(p2, ConstClass(node_vtable)) []
         p3 = getfield_gc(p1, descr=otherdescr)
         guard_class(p3, ConstClass(node_vtable)) []
+        p3a = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3, p2, descr=otherdescr)
-        p3a = new_with_vtable(ConstClass(node_vtable))
         escape(p3a)
         jump(p3a)
         """
@@ -2421,9 +2421,9 @@
         # setfield_gc(p3, p2, descr=otherdescr) # p3a.other = p2a
         # p1a = new_with_vtable(ConstClass(node_vtable2))
         # p2a = new_with_vtable(ConstClass(node_vtable))
+        p3anew = new_with_vtable(ConstClass(node_vtable))
         p2 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3a, p2, descr=otherdescr) # p3a.other = p2a
-        p3anew = new_with_vtable(ConstClass(node_vtable))
         escape(p3anew)
         jump(p3anew)
         """
@@ -2458,9 +2458,9 @@
         p3 = getfield_gc(p1, descr=otherdescr)
         guard_class(p3, ConstClass(node_vtable)) []
         # p1a = new_with_vtable(ConstClass(node_vtable2))
+        p3a = new_with_vtable(ConstClass(node_vtable))
         p2a = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3, p2a, descr=otherdescr)
-        p3a = new_with_vtable(ConstClass(node_vtable))
         escape(p3a)
         # setfield_gc(p1a, p2a, descr=nextdescr)
         # setfield_gc(p1a, p3a, descr=otherdescr)
@@ -2468,9 +2468,9 @@
         """
         expected = """
         [p2, p3]
+        p3a = new_with_vtable(ConstClass(node_vtable))
         p2a = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3, p2a, descr=otherdescr)
-        p3a = new_with_vtable(ConstClass(node_vtable))
         escape(p3a)
         jump(p2a, p3a)
         """
@@ -2790,7 +2790,6 @@
         self.optimize_loop(ops, expected, preamble)
 
     def test_remove_duplicate_pure_op_ovf_with_lazy_setfield(self):
-        py.test.skip('this optimization is not yet supprted')
         ops = """
         [i1, p1]
         i3 = int_add_ovf(i1, 1)
@@ -5961,13 +5960,18 @@
         escape(i0)
         jump(p1)
         """
-        expected = """
+        preamble = """
         [p1]
         i0 = ptr_eq(p1, NULL)
         escape(i0)
-        jump(p1)
-        """
-        self.optimize_strunicode_loop_extradescrs(ops, expected, expected)
+        jump(p1, i0)
+        """
+        expected = """
+        [p1, i0]
+        escape(i0)
+        jump(p1, i0)
+        """
+        self.optimize_strunicode_loop_extradescrs(ops, expected, preamble)
 
     def test_str_equal_none2(self):
         ops = """
@@ -5976,13 +5980,18 @@
         escape(i0)
         jump(p1)
         """
-        expected = """
+        preamble = """
         [p1]
         i0 = ptr_eq(p1, NULL)
         escape(i0)
-        jump(p1)
-        """
-        self.optimize_strunicode_loop_extradescrs(ops, expected, expected)
+        jump(p1, i0)
+        """
+        expected = """
+        [p1, i0]
+        escape(i0)
+        jump(p1, i0)
+        """
+        self.optimize_strunicode_loop_extradescrs(ops, expected, preamble)
 
     def test_str_equal_nonnull1(self):
         ops = """
@@ -7220,6 +7229,29 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_heap_cache_forced_virtuals(self):
+        ops = """
+        [i1, i2, p0]
+        p1 = new(descr=ssize)
+        setfield_gc(p1, i1, descr=adescr)
+        setfield_gc(p1, i2, descr=bdescr)
+        call(p0, p1, descr=writeadescr)
+        i3 = getfield_gc(p1, descr=adescr)
+        i4 = getfield_gc(p1, descr=bdescr)
+        jump(i3, i4, p0)
+        """
+        expected = """
+        [i1, i2, p0]
+        p1 = new(descr=ssize)
+        setfield_gc(p1, i1, descr=adescr)
+        call(p0, p1, descr=writeadescr)
+        i3 = getfield_gc(p1, descr=adescr)
+        setfield_gc(p1, i2, descr=bdescr)
+        jump(i3, i2, p0)
+        """
+        self.optimize_loop(ops, expected)
+
+
     def test_setarrayitem_followed_by_arraycopy(self):
         ops = """
         [p1, p2]
@@ -7230,6 +7262,27 @@
         """
         self.optimize_loop(ops, ops)
 
+    def test_heap_cache_virtuals_forced_by_delayed_setfield(self):
+        py.test.skip('not yet supoprted')
+        ops = """
+        [i1, p0]
+        p1 = new(descr=ssize)
+        setfield_gc(p1, i1, descr=valuedescr)
+        setfield_gc(p0, p1, descr=adescr)
+        call(p0, descr=writeadescr)
+        i2 = getfield_gc(p1, descr=valuedescr)
+        jump(i2, p0)
+        """
+        expected = """
+        [i1, p0]
+        p1 = new(descr=ssize)
+        setfield_gc(p1, i1, descr=valuedescr)
+        setfield_gc(p0, p1, descr=adescr)
+        call(p0, descr=writeadescr)
+        jump(i1, p0)
+        """
+        self.optimize_loop(ops, expected)
+
 class TestLLtype(OptimizeOptTest, LLtypeMixin):
     pass
 
diff --git a/pypy/jit/metainterp/optimizeopt/unroll.py b/pypy/jit/metainterp/optimizeopt/unroll.py
--- a/pypy/jit/metainterp/optimizeopt/unroll.py
+++ b/pypy/jit/metainterp/optimizeopt/unroll.py
@@ -75,7 +75,6 @@
         self.importable_values = {}
         self.emitting_dissabled = False
         self.emitted_guards = 0
-        self.emitted_pure_operations = {}
 
     def ensure_imported(self, value):
         if not self.emitting_dissabled and value in self.importable_values:
@@ -96,21 +95,6 @@
         new = UnrollableOptimizer(self.metainterp_sd, self.loop)
         return self._new(new)
 
-    def remember_emitting_pure(self, op):
-        self.emitted_pure_operations[op] = True
-
-    def produce_potential_short_preamble_ops(self, sb):
-        for op in self.emitted_pure_operations:
-            if op.getopnum() == rop.GETARRAYITEM_GC_PURE or \
-               op.getopnum() == rop.STRGETITEM or \
-               op.getopnum() == rop.UNICODEGETITEM:
-                if not self.getvalue(op.getarg(1)).is_constant():
-                    continue
-            sb.add_potential(op)
-        for opt in self.optimizations:
-            opt.produce_potential_short_preamble_ops(sb)
-
-
 
 class UnrollOptimizer(Optimization):
     """Unroll the loop into two iterations. The first one will
@@ -154,7 +138,7 @@
 
             KillHugeIntBounds(self.optimizer).apply()
             
-            loop.preamble.operations = self.optimizer.newoperations
+            loop.preamble.operations = self.optimizer.get_newoperations()
             jump_args = [self.getvalue(a).get_key_box() for a in jump_args]
 
             start_resumedescr = loop.preamble.start_resumedescr.clone_if_mutable()
@@ -167,8 +151,9 @@
             virtual_state = modifier.get_virtual_state(jump_args)
             
             values = [self.getvalue(arg) for arg in jump_args]
-            inputargs = virtual_state.make_inputargs(values)
-            short_inputargs = virtual_state.make_inputargs(values, keyboxes=True)
+            inputargs = virtual_state.make_inputargs(values, self.optimizer)
+            short_inputargs = virtual_state.make_inputargs(values, self.optimizer,
+                                                           keyboxes=True)
 
             self.constant_inputargs = {}
             for box in jump_args: 
@@ -197,7 +182,7 @@
             # operations needed to setup the proper state of those virtuals
             # in the peeled loop
             inputarg_setup_ops = []
-            preamble_optimizer.newoperations = []
+            preamble_optimizer.clear_newoperations()
             seen = {}
             for box in inputargs:
                 if box in seen:
@@ -211,9 +196,8 @@
                     continue
                 seen[box] = True
                 value = preamble_optimizer.getvalue(box)
-                value.force_box()
-            preamble_optimizer.flush()
-            inputarg_setup_ops += preamble_optimizer.newoperations
+                value.force_box(preamble_optimizer)
+            inputarg_setup_ops += preamble_optimizer.get_newoperations()
 
             # Setup the state of the new optimizer by emiting the
             # short preamble operations and discarding the result
@@ -244,13 +228,13 @@
                                 virtual_state)
             
             loop.inputargs = inputargs
-            args = [preamble_optimizer.getvalue(self.short_boxes.original(a)).force_box()\
+            args = [preamble_optimizer.getvalue(self.short_boxes.original(a)).force_box(preamble_optimizer)\
                     for a in inputargs]
             jmp = ResOperation(rop.JUMP, args, None)
             jmp.setdescr(loop.token)
             loop.preamble.operations.append(jmp)
 
-            loop.operations = self.optimizer.newoperations
+            loop.operations = self.optimizer.get_newoperations()
             maxguards = self.optimizer.metainterp_sd.warmrunnerdesc.memory_manager.max_retrace_guards
             
             if self.optimizer.emitted_guards > maxguards:
@@ -335,9 +319,10 @@
         assert jumpop
         original_jumpargs = jumpop.getarglist()[:]
         values = [self.getvalue(arg) for arg in jumpop.getarglist()]
-        jumpargs = virtual_state.make_inputargs(values)
+        jumpargs = virtual_state.make_inputargs(values, self.optimizer)
         jumpop.initarglist(jumpargs)
-        jmp_to_short_args = virtual_state.make_inputargs(values, keyboxes=True)
+        jmp_to_short_args = virtual_state.make_inputargs(values, self.optimizer,
+                                                         keyboxes=True)
         self.short_inliner = Inliner(short_inputargs, jmp_to_short_args)
         
         for box, const in self.constant_inputargs.items():
@@ -347,11 +332,11 @@
             newop = self.short_inliner.inline_op(op)
             self.optimizer.send_extra_operation(newop)
         
-        self.optimizer.flush()
+        newoperations = self.optimizer.get_newoperations()
 
         i = j = 0
-        while i < len(self.optimizer.newoperations) or j < len(jumpargs):
-            if i == len(self.optimizer.newoperations):
+        while i < len(newoperations) or j < len(jumpargs):
+            if i == len(newoperations):
                 while j < len(jumpargs):
                     a = jumpargs[j]
                     if self.optimizer.loop.logops:
@@ -360,7 +345,7 @@
                                     jumpargs, short_seen)
                     j += 1
             else:
-                op = self.optimizer.newoperations[i]
+                op = newoperations[i]
 
                 self.boxes_created_this_iteration[op.result] = True
                 args = op.getarglist()
@@ -375,6 +360,7 @@
                     self.import_box(a, inputargs, short, short_jumpargs,
                                     jumpargs, short_seen)
                 i += 1
+            newoperations = self.optimizer.get_newoperations()
 
         jumpop.initarglist(jumpargs)
         self.optimizer.send_extra_operation(jumpop)
@@ -468,7 +454,7 @@
         inputargs.append(box)
         box = newresult
         if box in self.optimizer.values:
-            box = self.optimizer.values[box].force_box()
+            box = self.optimizer.values[box].force_box(self.optimizer)
         jumpargs.append(box)
         
 
@@ -483,11 +469,6 @@
         if op.getopnum() == rop.JUMP:
             loop_token = op.getdescr()
             assert isinstance(loop_token, LoopToken)
-            # FIXME: Use a tree, similar to the tree formed by the full
-            # preamble and it's bridges, instead of a list to save time and
-            # memory. This should also allow better behaviour in
-            # situations that the is_emittable() chain currently cant
-            # handle and the inlining fails unexpectedly belwo.
             short = loop_token.short_preamble
             if short:
                 args = op.getarglist()
@@ -523,7 +504,7 @@
 
                         values = [self.getvalue(arg)
                                   for arg in op.getarglist()]
-                        args = sh.virtual_state.make_inputargs(values,
+                        args = sh.virtual_state.make_inputargs(values, self.optimizer,
                                                                keyboxes=True)
                         inliner = Inliner(sh.inputargs, args)
                         
diff --git a/pypy/jit/metainterp/optimizeopt/virtualize.py b/pypy/jit/metainterp/optimizeopt/virtualize.py
--- a/pypy/jit/metainterp/optimizeopt/virtualize.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualize.py
@@ -10,13 +10,12 @@
 
 
 class AbstractVirtualValue(optimizer.OptValue):
-    _attrs_ = ('optimizer', 'keybox', 'source_op', '_cached_vinfo')
+    _attrs_ = ('keybox', 'source_op', '_cached_vinfo')
     box = None
     level = optimizer.LEVEL_NONNULL
     _cached_vinfo = None
 
-    def __init__(self, optimizer, keybox, source_op=None):
-        self.optimizer = optimizer
+    def __init__(self, keybox, source_op=None):
         self.keybox = keybox   # only used as a key in dictionaries
         self.source_op = source_op  # the NEW_WITH_VTABLE/NEW_ARRAY operation
                                     # that builds this box
@@ -29,17 +28,17 @@
             return self.keybox
         return self.box
 
-    def force_box(self):
+    def force_box(self, optforce):
         if self.box is None:
-            self.optimizer.forget_numberings(self.keybox)
-            self._really_force()
+            optforce.forget_numberings(self.keybox)
+            self._really_force(optforce)
         return self.box
 
-    def force_at_end_of_preamble(self, already_forced):
+    def force_at_end_of_preamble(self, already_forced, optforce):
         value = already_forced.get(self, None)
         if value:
             return value
-        return OptValue(self.force_box())
+        return OptValue(self.force_box(optforce))
 
     def make_virtual_info(self, modifier, fieldnums):
         if fieldnums is None:
@@ -55,7 +54,7 @@
     def _make_virtual(self, modifier):
         raise NotImplementedError("abstract base")
 
-    def _really_force(self):
+    def _really_force(self, optforce):
         raise NotImplementedError("abstract base")
 
     def import_from(self, other, optimizer):
@@ -70,10 +69,11 @@
 get_fielddescrlist_cache._annspecialcase_ = "specialize:memo"
 
 class AbstractVirtualStructValue(AbstractVirtualValue):
-    _attrs_ = ('_fields', '_cached_sorted_fields')
+    _attrs_ = ('_fields', 'cpu', '_cached_sorted_fields')
 
-    def __init__(self, optimizer, keybox, source_op=None):
-        AbstractVirtualValue.__init__(self, optimizer, keybox, source_op)
+    def __init__(self, cpu, keybox, source_op=None):
+        AbstractVirtualValue.__init__(self, keybox, source_op)
+        self.cpu = cpu
         self._fields = {}
         self._cached_sorted_fields = None
 
@@ -87,45 +87,44 @@
     def _get_descr(self):
         raise NotImplementedError
 
-    def _is_immutable_and_filled_with_constants(self):
+    def _is_immutable_and_filled_with_constants(self, optforce):
         count = self._get_descr().count_fields_if_immutable()
         if count != len(self._fields):    # always the case if count == -1
             return False
         for value in self._fields.itervalues():
-            subbox = value.force_box()
+            subbox = value.force_box(optforce)
             if not isinstance(subbox, Const):
                 return False
         return True
 
-    def force_at_end_of_preamble(self, already_forced):
+    def force_at_end_of_preamble(self, already_forced, optforce):
         if self in already_forced:
             return self
         already_forced[self] = self
         if self._fields:
             for ofs in self._fields.keys():
-                self._fields[ofs] = self._fields[ofs].force_at_end_of_preamble(already_forced)
+                self._fields[ofs] = self._fields[ofs].force_at_end_of_preamble(already_forced, optforce)
         return self
 
-    def _really_force(self):
+    def _really_force(self, optforce):
         op = self.source_op
         assert op is not None
         # ^^^ This case should not occur any more (see test_bug_3).
         #
         if not we_are_translated():
             op.name = 'FORCE ' + self.source_op.name
-
-        if self._is_immutable_and_filled_with_constants():
-            box = self.optimizer.constant_fold(op)
+            
+        if self._is_immutable_and_filled_with_constants(optforce):
+            box = optforce.optimizer.constant_fold(op)
             self.make_constant(box)
             for ofs, value in self._fields.iteritems():
-                subbox = value.force_box()
+                subbox = value.force_box(optforce)
                 assert isinstance(subbox, Const)
-                execute(self.optimizer.cpu, None, rop.SETFIELD_GC,
+                execute(optforce.optimizer.cpu, None, rop.SETFIELD_GC,
                         ofs, box, subbox)
             # keep self._fields, because it's all immutable anyway
         else:
-            newoperations = self.optimizer.newoperations
-            newoperations.append(op)
+            optforce.emit_operation(op)
             self.box = box = op.result
             #
             iteritems = self._fields.iteritems()
@@ -135,10 +134,11 @@
             for ofs, value in iteritems:
                 if value.is_null():
                     continue
-                subbox = value.force_box()
+                subbox = value.force_box(optforce)
                 op = ResOperation(rop.SETFIELD_GC, [box, subbox], None,
                                   descr=ofs)
-                newoperations.append(op)
+
+                optforce.emit_operation(op)
 
     def _get_field_descr_list(self):
         _cached_sorted_fields = self._cached_sorted_fields
@@ -155,7 +155,7 @@
             else:
                 lst = self._fields.keys()
             sort_descrs(lst)
-            cache = get_fielddescrlist_cache(self.optimizer.cpu)
+            cache = get_fielddescrlist_cache(self.cpu)
             result = cache.get(lst, None)
             if result is None:
                 cache[lst] = lst
@@ -180,8 +180,8 @@
 class VirtualValue(AbstractVirtualStructValue):
     level = optimizer.LEVEL_KNOWNCLASS
 
-    def __init__(self, optimizer, known_class, keybox, source_op=None):
-        AbstractVirtualStructValue.__init__(self, optimizer, keybox, source_op)
+    def __init__(self, cpu, known_class, keybox, source_op=None):
+        AbstractVirtualStructValue.__init__(self, cpu, keybox, source_op)
         assert isinstance(known_class, Const)
         self.known_class = known_class
 
@@ -190,7 +190,7 @@
         return modifier.make_virtual(self.known_class, fielddescrs)
 
     def _get_descr(self):
-        return vtable2descr(self.optimizer.cpu, self.known_class.getint())
+        return vtable2descr(self.cpu, self.known_class.getint())
 
     def __repr__(self):
         cls_name = self.known_class.value.adr.ptr._obj._TYPE._name
@@ -201,8 +201,8 @@
 
 class VStructValue(AbstractVirtualStructValue):
 
-    def __init__(self, optimizer, structdescr, keybox, source_op=None):
-        AbstractVirtualStructValue.__init__(self, optimizer, keybox, source_op)
+    def __init__(self, cpu, structdescr, keybox, source_op=None):
+        AbstractVirtualStructValue.__init__(self, cpu, keybox, source_op)
         self.structdescr = structdescr
 
     def _make_virtual(self, modifier):
@@ -215,10 +215,10 @@
 
 class VArrayValue(AbstractVirtualValue):
 
-    def __init__(self, optimizer, arraydescr, size, keybox, source_op=None):
-        AbstractVirtualValue.__init__(self, optimizer, keybox, source_op)
+    def __init__(self, arraydescr, constvalue, size, keybox, source_op=None):
+        AbstractVirtualValue.__init__(self, keybox, source_op)
         self.arraydescr = arraydescr
-        self.constvalue = optimizer.new_const_item(arraydescr)
+        self.constvalue = constvalue
         self._items = [self.constvalue] * size
 
     def getlength(self):
@@ -232,31 +232,30 @@
         assert isinstance(itemvalue, optimizer.OptValue)
         self._items[index] = itemvalue
 
-    def force_at_end_of_preamble(self, already_forced):
+    def force_at_end_of_preamble(self, already_forced, optforce):
         if self in already_forced:
             return self
         already_forced[self] = self
         for index in range(len(self._items)):
-            self._items[index] = self._items[index].force_at_end_of_preamble(already_forced)
+            self._items[index] = self._items[index].force_at_end_of_preamble(already_forced, optforce)
         return self
     
-    def _really_force(self):
+    def _really_force(self, optforce):
         assert self.source_op is not None
         if not we_are_translated():
             self.source_op.name = 'FORCE ' + self.source_op.name
-        newoperations = self.optimizer.newoperations
-        newoperations.append(self.source_op)
+        optforce.emit_operation(self.source_op)        
         self.box = box = self.source_op.result
         for index in range(len(self._items)):
             subvalue = self._items[index]
             if subvalue is not self.constvalue:
                 if subvalue.is_null():
                     continue
-                subbox = subvalue.force_box()
+                subbox = subvalue.force_box(optforce)
                 op = ResOperation(rop.SETARRAYITEM_GC,
                                   [box, ConstInt(index), subbox], None,
                                   descr=self.arraydescr)
-                newoperations.append(op)
+                optforce.emit_operation(op)
 
     def get_args_for_fail(self, modifier):
         if self.box is None and not modifier.already_seen_virtual(self.keybox):
@@ -279,17 +278,18 @@
         return OptVirtualize()
         
     def make_virtual(self, known_class, box, source_op=None):
-        vvalue = VirtualValue(self.optimizer, known_class, box, source_op)
+        vvalue = VirtualValue(self.optimizer.cpu, known_class, box, source_op)
         self.make_equal_to(box, vvalue)
         return vvalue
 
     def make_varray(self, arraydescr, size, box, source_op=None):
-        vvalue = VArrayValue(self.optimizer, arraydescr, size, box, source_op)
+        constvalue = self.new_const_item(arraydescr)
+        vvalue = VArrayValue(arraydescr, constvalue, size, box, source_op)
         self.make_equal_to(box, vvalue)
         return vvalue
 
     def make_vstruct(self, structdescr, box, source_op=None):
-        vvalue = VStructValue(self.optimizer, structdescr, box, source_op)
+        vvalue = VStructValue(self.optimizer.cpu, structdescr, box, source_op)
         self.make_equal_to(box, vvalue)
         return vvalue
 
@@ -362,7 +362,6 @@
             self.make_equal_to(op.result, fieldvalue)
         else:
             value.ensure_nonnull()
-            ###self.heap_op_optimizer.optimize_GETFIELD_GC(op, value)
             self.emit_operation(op)
 
     # note: the following line does not mean that the two operations are
@@ -377,7 +376,6 @@
             value.setfield(op.getdescr(), fieldvalue)
         else:
             value.ensure_nonnull()
-            ###self.heap_op_optimizer.optimize_SETFIELD_GC(op, value, fieldvalue)
             self.emit_operation(op)
 
     def optimize_NEW_WITH_VTABLE(self, op):
@@ -417,7 +415,6 @@
                 self.make_equal_to(op.result, itemvalue)
                 return
         value.ensure_nonnull()
-        ###self.heap_op_optimizer.optimize_GETARRAYITEM_GC(op, value)
         self.emit_operation(op)
 
     # note: the following line does not mean that the two operations are
@@ -432,7 +429,6 @@
                 value.setitem(indexbox.getint(), self.getvalue(op.getarg(2)))
                 return
         value.ensure_nonnull()
-        ###self.heap_op_optimizer.optimize_SETARRAYITEM_GC(op, value, fieldvalue)
         self.emit_operation(op)
 
 
diff --git a/pypy/jit/metainterp/optimizeopt/virtualstate.py b/pypy/jit/metainterp/optimizeopt/virtualstate.py
--- a/pypy/jit/metainterp/optimizeopt/virtualstate.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualstate.py
@@ -30,7 +30,7 @@
     def _generate_guards(self, other, box, cpu, extra_guards):
         raise InvalidLoop
 
-    def enum_forced_boxes(self, boxes, value):
+    def enum_forced_boxes(self, boxes, value, optimizer):
         raise NotImplementedError
 
     def enum(self, virtual_state):
@@ -100,14 +100,14 @@
     def _generalization_of(self, other):
         raise NotImplementedError
 
-    def enum_forced_boxes(self, boxes, value):
+    def enum_forced_boxes(self, boxes, value, optimizer):
         assert isinstance(value, virtualize.AbstractVirtualStructValue)
         assert value.is_virtual()
         for i in range(len(self.fielddescrs)):
             v = value._fields[self.fielddescrs[i]]
             s = self.fieldstate[i]
             if s.position > self.position:
-                s.enum_forced_boxes(boxes, v)
+                s.enum_forced_boxes(boxes, v, optimizer)
 
     def _enum(self, virtual_state):
         for s in self.fieldstate:
@@ -177,14 +177,14 @@
                 return False
         return True
 
-    def enum_forced_boxes(self, boxes, value):
+    def enum_forced_boxes(self, boxes, value, optimizer):
         assert isinstance(value, virtualize.VArrayValue)
         assert value.is_virtual()
         for i in range(len(self.fieldstate)):
             v = value._items[i]
             s = self.fieldstate[i]
             if s.position > self.position:
-                s.enum_forced_boxes(boxes, v)
+                s.enum_forced_boxes(boxes, v, optimizer)
 
     def _enum(self, virtual_state):
         for s in self.fieldstate:
@@ -316,11 +316,11 @@
             import pdb; pdb.set_trace()
             raise NotImplementedError
 
-    def enum_forced_boxes(self, boxes, value):
+    def enum_forced_boxes(self, boxes, value, optimizer):
         if self.level == LEVEL_CONSTANT:
             return
         assert 0 <= self.position_in_notvirtuals 
-        boxes[self.position_in_notvirtuals] = value.force_box()
+        boxes[self.position_in_notvirtuals] = value.force_box(optimizer)
 
     def _enum(self, virtual_state):
         if self.level == LEVEL_CONSTANT:
@@ -377,11 +377,13 @@
             self.state[i].generate_guards(other.state[i], args[i],
                                           cpu, extra_guards, renum)
 
-    def make_inputargs(self, values, keyboxes=False):
+    def make_inputargs(self, values, optimizer, keyboxes=False):
+        if optimizer.optearlyforce:
+            optimizer = optimizer.optearlyforce
         assert len(values) == len(self.state)
         inputargs = [None] * len(self.notvirtuals)
         for i in range(len(values)):
-            self.state[i].enum_forced_boxes(inputargs, values[i])
+            self.state[i].enum_forced_boxes(inputargs, values[i], optimizer)
 
         if keyboxes:
             for i in range(len(values)):
@@ -434,7 +436,12 @@
     def get_virtual_state(self, jump_args):
         self.optimizer.force_at_end_of_preamble()
         already_forced = {}
-        values = [self.getvalue(box).force_at_end_of_preamble(already_forced)
+        if self.optimizer.optearlyforce:
+            opt = self.optimizer.optearlyforce
+        else:
+            opt = self.optimizer
+        values = [self.getvalue(box).force_at_end_of_preamble(already_forced,
+                                                              opt)
                   for box in jump_args]
 
         for value in values:
diff --git a/pypy/jit/metainterp/optimizeopt/vstring.py b/pypy/jit/metainterp/optimizeopt/vstring.py
--- a/pypy/jit/metainterp/optimizeopt/vstring.py
+++ b/pypy/jit/metainterp/optimizeopt/vstring.py
@@ -43,7 +43,7 @@
 class __extend__(optimizer.OptValue):
     """New methods added to the base class OptValue for this file."""
 
-    def getstrlen(self, optimization, mode):
+    def getstrlen(self, string_optimizer, mode):
         if mode is mode_string:
             s = self.get_constant_string_spec(mode_string)
             if s is not None:
@@ -52,12 +52,12 @@
             s = self.get_constant_string_spec(mode_unicode)
             if s is not None:
                 return ConstInt(len(s))
-        if optimization is None:
+        if string_optimizer is None:
             return None
         self.ensure_nonnull()
-        box = self.force_box()
+        box = self.force_box(string_optimizer)
         lengthbox = BoxInt()
-        optimization.propagate_forward(ResOperation(mode.STRLEN, [box], lengthbox))
+        string_optimizer.emit_operation(ResOperation(mode.STRLEN, [box], lengthbox))
         return lengthbox
 
     @specialize.arg(1)
@@ -68,25 +68,25 @@
         else:
             return None
 
-    def string_copy_parts(self, optimizer, targetbox, offsetbox, mode):
+    def string_copy_parts(self, string_optimizer, targetbox, offsetbox, mode):
         # Copies the pointer-to-string 'self' into the target string
         # given by 'targetbox', at the specified offset.  Returns the offset
         # at the end of the copy.
-        lengthbox = self.getstrlen(optimizer, mode)
-        srcbox = self.force_box()
-        return copy_str_content(optimizer, srcbox, targetbox,
+        lengthbox = self.getstrlen(string_optimizer, mode)
+        srcbox = self.force_box(string_optimizer)
+        return copy_str_content(string_optimizer, srcbox, targetbox,
                                 CONST_0, offsetbox, lengthbox, mode)
 
 
 class VAbstractStringValue(virtualize.AbstractVirtualValue):
     _attrs_ = ('mode',)
 
-    def __init__(self, optimizer, keybox, source_op, mode):
-        virtualize.AbstractVirtualValue.__init__(self, optimizer, keybox,
+    def __init__(self, keybox, source_op, mode):
+        virtualize.AbstractVirtualValue.__init__(self, keybox,
                                                  source_op)
         self.mode = mode
 
-    def _really_force(self):
+    def _really_force(self, optforce):
         if self.mode is mode_string:
             s = self.get_constant_string_spec(mode_string)
             if s is not None:
@@ -101,12 +101,12 @@
                 return
         assert self.source_op is not None
         self.box = box = self.source_op.result
-        lengthbox = self.getstrlen(self.optimizer, self.mode)
+        lengthbox = self.getstrlen(optforce, self.mode)
         op = ResOperation(self.mode.NEWSTR, [lengthbox], box)
         if not we_are_translated():
             op.name = 'FORCE'
-        self.optimizer.emit_operation(op)
-        self.string_copy_parts(self.optimizer, box, CONST_0, self.mode)
+        optforce.emit_operation(op)
+        self.string_copy_parts(optforce, box, CONST_0, self.mode)
 
 
 class VStringPlainValue(VAbstractStringValue):
@@ -140,20 +140,20 @@
         return mode.emptystr.join([mode.chr(c.box.getint())
                                    for c in self._chars])
 
-    def string_copy_parts(self, optimizer, targetbox, offsetbox, mode):
+    def string_copy_parts(self, string_optimizer, targetbox, offsetbox, mode):
         if not self.is_virtual() and targetbox is not self.box:
-            lengthbox = self.getstrlen(optimizer, mode)
-            srcbox = self.force_box()
-            return copy_str_content(optimizer, srcbox, targetbox,
+            lengthbox = self.getstrlen(string_optimizer, mode)
+            srcbox = self.force_box(string_optimizer)
+            return copy_str_content(string_optimizer, srcbox, targetbox,
                                 CONST_0, offsetbox, lengthbox, mode)
         for i in range(len(self._chars)):
-            charbox = self._chars[i].force_box()
+            charbox = self._chars[i].force_box(string_optimizer)
             if not (isinstance(charbox, Const) and charbox.same_constant(CONST_0)):
-                optimizer.emit_operation(ResOperation(mode.STRSETITEM, [targetbox,
-                                                                    offsetbox,
-                                                                    charbox],
+                string_optimizer.emit_operation(ResOperation(mode.STRSETITEM, [targetbox,
+                                                                               offsetbox,
+                                                                               charbox],
                                                   None))
-            offsetbox = _int_add(optimizer, offsetbox, CONST_1)
+            offsetbox = _int_add(string_optimizer, offsetbox, CONST_1)
         return offsetbox
 
     def get_args_for_fail(self, modifier):
@@ -187,16 +187,16 @@
         self.left = left
         self.right = right
 
-    def getstrlen(self, optimizer, mode):
+    def getstrlen(self, string_optimizer, mode):
         if self.lengthbox is None:
-            len1box = self.left.getstrlen(optimizer, mode)
+            len1box = self.left.getstrlen(string_optimizer, mode)
             if len1box is None:
                 return None
-            len2box = self.right.getstrlen(optimizer, mode)
+            len2box = self.right.getstrlen(string_optimizer, mode)
             if len2box is None:
                 return None
-            self.lengthbox = _int_add(optimizer, len1box, len2box)
-            # ^^^ may still be None, if optimizer is None
+            self.lengthbox = _int_add(string_optimizer, len1box, len2box)
+            # ^^^ may still be None, if string_optimizer is None
         return self.lengthbox
 
     @specialize.arg(1)
@@ -209,10 +209,10 @@
             return None
         return s1 + s2
 
-    def string_copy_parts(self, optimizer, targetbox, offsetbox, mode):
-        offsetbox = self.left.string_copy_parts(optimizer, targetbox,
+    def string_copy_parts(self, string_optimizer, targetbox, offsetbox, mode):
+        offsetbox = self.left.string_copy_parts(string_optimizer, targetbox,
                                                 offsetbox, mode)
-        offsetbox = self.right.string_copy_parts(optimizer, targetbox,
+        offsetbox = self.right.string_copy_parts(string_optimizer, targetbox,
                                                  offsetbox, mode)
         return offsetbox
 
@@ -251,8 +251,8 @@
         self.vstart = vstart
         self.vlength = vlength
 
-    def getstrlen(self, _, mode):
-        return self.vlength.force_box()
+    def getstrlen(self, optforce, mode):
+        return self.vlength.force_box(optforce)
 
     @specialize.arg(1)
     def get_constant_string_spec(self, mode):
@@ -267,11 +267,11 @@
             return s1[start : start + length]
         return None
 
-    def string_copy_parts(self, optimizer, targetbox, offsetbox, mode):
-        lengthbox = self.getstrlen(optimizer, mode)
-        return copy_str_content(optimizer,
-                                self.vstr.force_box(), targetbox,
-                                self.vstart.force_box(), offsetbox,
+    def string_copy_parts(self, string_optimizer, targetbox, offsetbox, mode):
+        lengthbox = self.getstrlen(string_optimizer, mode)
+        return copy_str_content(string_optimizer,
+                                self.vstr.force_box(string_optimizer), targetbox,
+                                self.vstart.force_box(string_optimizer), offsetbox,
                                 lengthbox, mode)
 
     def get_args_for_fail(self, modifier):
@@ -300,7 +300,7 @@
         return modifier.make_vstrslice(self.mode is mode_unicode)
 
 
-def copy_str_content(optimizer, srcbox, targetbox,
+def copy_str_content(string_optimizer, srcbox, targetbox,
                      srcoffsetbox, offsetbox, lengthbox, mode, need_next_offset=True):
     if isinstance(srcbox, ConstPtr) and isinstance(srcoffsetbox, Const):
         M = 5
@@ -310,26 +310,26 @@
         # up to M characters are done "inline", i.e. with STRGETITEM/STRSETITEM
         # instead of just a COPYSTRCONTENT.
         for i in range(lengthbox.value):
-            charbox = _strgetitem(optimizer, srcbox, srcoffsetbox, mode)
-            srcoffsetbox = _int_add(optimizer, srcoffsetbox, CONST_1)
-            optimizer.emit_operation(ResOperation(mode.STRSETITEM, [targetbox,
-                                                                offsetbox,
-                                                                charbox],
+            charbox = _strgetitem(string_optimizer, srcbox, srcoffsetbox, mode)
+            srcoffsetbox = _int_add(string_optimizer, srcoffsetbox, CONST_1)
+            string_optimizer.emit_operation(ResOperation(mode.STRSETITEM, [targetbox,
+                                                                           offsetbox,
+                                                                           charbox],
                                               None))
-            offsetbox = _int_add(optimizer, offsetbox, CONST_1)
+            offsetbox = _int_add(string_optimizer, offsetbox, CONST_1)
     else:
         if need_next_offset:
-            nextoffsetbox = _int_add(optimizer, offsetbox, lengthbox)
+            nextoffsetbox = _int_add(string_optimizer, offsetbox, lengthbox)
         else:
             nextoffsetbox = None
         op = ResOperation(mode.COPYSTRCONTENT, [srcbox, targetbox,
                                                 srcoffsetbox, offsetbox,
                                                 lengthbox], None)
-        optimizer.emit_operation(op)
+        string_optimizer.emit_operation(op)
         offsetbox = nextoffsetbox
     return offsetbox
 
-def _int_add(optimizer, box1, box2):
+def _int_add(string_optimizer, box1, box2):
     if isinstance(box1, ConstInt):
         if box1.value == 0:
             return box2
@@ -337,23 +337,23 @@
             return ConstInt(box1.value + box2.value)
     elif isinstance(box2, ConstInt) and box2.value == 0:
         return box1
-    if optimizer is None:
+    if string_optimizer is None:
         return None
     resbox = BoxInt()
-    optimizer.propagate_forward(ResOperation(rop.INT_ADD, [box1, box2], resbox))
+    string_optimizer.emit_operation(ResOperation(rop.INT_ADD, [box1, box2], resbox))
     return resbox
 
-def _int_sub(optimizer, box1, box2):
+def _int_sub(string_optimizer, box1, box2):
     if isinstance(box2, ConstInt):
         if box2.value == 0:
             return box1
         if isinstance(box1, ConstInt):
             return ConstInt(box1.value - box2.value)
     resbox = BoxInt()
-    optimizer.propagate_forward(ResOperation(rop.INT_SUB, [box1, box2], resbox))
+    string_optimizer.emit_operation(ResOperation(rop.INT_SUB, [box1, box2], resbox))
     return resbox
 
-def _strgetitem(optimizer, strbox, indexbox, mode):
+def _strgetitem(string_optimizer, strbox, indexbox, mode):
     if isinstance(strbox, ConstPtr) and isinstance(indexbox, ConstInt):
         if mode is mode_string:
             s = strbox.getref(lltype.Ptr(rstr.STR))
@@ -362,30 +362,28 @@
             s = strbox.getref(lltype.Ptr(rstr.UNICODE))
             return ConstInt(ord(s.chars[indexbox.getint()]))
     resbox = BoxInt()
-    optimizer.propagate_forward(ResOperation(mode.STRGETITEM, [strbox, indexbox],
-                                      resbox))
+    string_optimizer.emit_operation(ResOperation(mode.STRGETITEM, [strbox, indexbox],
+                                                 resbox))
     return resbox
 
 
 class OptString(optimizer.Optimization):
     "Handling of strings and unicodes."
-    enabled = True
-
     def new(self):
         return OptString()
 
     def make_vstring_plain(self, box, source_op, mode):
-        vvalue = VStringPlainValue(self.optimizer, box, source_op, mode)
+        vvalue = VStringPlainValue(box, source_op, mode)
         self.make_equal_to(box, vvalue)
         return vvalue
 
     def make_vstring_concat(self, box, source_op, mode):
-        vvalue = VStringConcatValue(self.optimizer, box, source_op, mode)
+        vvalue = VStringConcatValue(box, source_op, mode)
         self.make_equal_to(box, vvalue)
         return vvalue
 
     def make_vstring_slice(self, box, source_op, mode):
-        vvalue = VStringSliceValue(self.optimizer, box, source_op, mode)
+        vvalue = VStringSliceValue(box, source_op, mode)
         self.make_equal_to(box, vvalue)
         return vvalue
 
@@ -435,9 +433,9 @@
         value.ensure_nonnull()
         #
         if value.is_virtual() and isinstance(value, VStringSliceValue):
-            fullindexbox = _int_add(self.optimizer,
-                                    value.vstart.force_box(),
-                                    vindex.force_box())
+            fullindexbox = _int_add(self,
+                                    value.vstart.force_box(self),
+                                    vindex.force_box(self))
             value = value.vstr
             vindex = self.getvalue(fullindexbox)
         #
@@ -449,7 +447,7 @@
                 if res is not optimizer.CVAL_UNINITIALIZED_ZERO:
                     return res
         #
-        resbox = _strgetitem(self.optimizer, value.force_box(), vindex.force_box(), mode)
+        resbox = _strgetitem(self, value.force_box(self), vindex.force_box(self), mode)
         return self.getvalue(resbox)
 
     def optimize_STRLEN(self, op):
@@ -459,7 +457,7 @@
 
     def _optimize_STRLEN(self, op, mode):
         value = self.getvalue(op.getarg(0))
-        lengthbox = value.getstrlen(self.optimizer, mode)
+        lengthbox = value.getstrlen(self, mode)
         self.make_equal_to(op.result, self.getvalue(lengthbox))
 
     def optimize_COPYSTRCONTENT(self, op):
@@ -477,12 +475,12 @@
 
         if length.is_constant() and length.box.getint() == 0:
             return
-        copy_str_content(self.optimizer,
-            src.force_box(),
-            dst.force_box(),
-            srcstart.force_box(),
-            dststart.force_box(),
-            length.force_box(),
+        copy_str_content(self,
+            src.force_box(self),
+            dst.force_box(self),
+            srcstart.force_box(self),
+            dststart.force_box(self),
+            length.force_box(self),
             mode, need_next_offset=False
         )
 
@@ -508,6 +506,8 @@
                     return
         self.emit_operation(op)
 
+    optimize_CALL_PURE = optimize_CALL
+
     def opt_call_str_STR2UNICODE(self, op):
         # Constant-fold unicode("constant string").
         # More generally, supporting non-constant but virtual cases is
@@ -547,16 +547,16 @@
             return True
         #
         vstr.ensure_nonnull()
-        lengthbox = _int_sub(self.optimizer, vstop.force_box(),
-                                            vstart.force_box())
+        lengthbox = _int_sub(self, vstop.force_box(self),
+                                   vstart.force_box(self))
         #
         if isinstance(vstr, VStringSliceValue):
             # double slicing  s[i:j][k:l]
             vintermediate = vstr
             vstr = vintermediate.vstr
-            startbox = _int_add(self.optimizer,
-                                vintermediate.vstart.force_box(),
-                                vstart.force_box())
+            startbox = _int_add(self,
+                                vintermediate.vstart.force_box(self),
+                                vstart.force_box(self))
             vstart = self.getvalue(startbox)
         #
         value = self.make_vstring_slice(op.result, op, mode)
@@ -594,8 +594,8 @@
                 do = EffectInfo.OS_STREQ_LENGTHOK
             else:
                 do = EffectInfo.OS_STREQ_NONNULL
-            self.generate_modified_call(do, [v1.force_box(),
-                                             v2.force_box()], op.result, mode)
+            self.generate_modified_call(do, [v1.force_box(self),
+                                             v2.force_box(self)], op.result, mode)
             return True
         return False
 
@@ -603,7 +603,7 @@
         l2box = v2.getstrlen(None, mode)
         if isinstance(l2box, ConstInt):
             if l2box.value == 0:
-                lengthbox = v1.getstrlen(self.optimizer, mode)
+                lengthbox = v1.getstrlen(self, mode)
                 seo = self.optimizer.send_extra_operation
                 seo(ResOperation(rop.INT_EQ, [lengthbox, CONST_0], resultbox))
                 return True
@@ -614,17 +614,17 @@
                     vchar1 = self.strgetitem(v1, optimizer.CVAL_ZERO, mode)
                     vchar2 = self.strgetitem(v2, optimizer.CVAL_ZERO, mode)
                     seo = self.optimizer.send_extra_operation
-                    seo(ResOperation(rop.INT_EQ, [vchar1.force_box(),
-                                                  vchar2.force_box()],
+                    seo(ResOperation(rop.INT_EQ, [vchar1.force_box(self),
+                                                  vchar2.force_box(self)],
                                      resultbox))
                     return True
                 if isinstance(v1, VStringSliceValue):
                     vchar = self.strgetitem(v2, optimizer.CVAL_ZERO, mode)
                     do = EffectInfo.OS_STREQ_SLICE_CHAR
-                    self.generate_modified_call(do, [v1.vstr.force_box(),
-                                                     v1.vstart.force_box(),
-                                                     v1.vlength.force_box(),
-                                                     vchar.force_box()],
+                    self.generate_modified_call(do, [v1.vstr.force_box(self),
+                                                     v1.vstart.force_box(self),
+                                                     v1.vlength.force_box(self),
+                                                     vchar.force_box(self)],
                                                 resultbox, mode)
                     return True
         #
@@ -635,10 +635,10 @@
             if v1.is_null():
                 self.make_constant(resultbox, CONST_1)
                 return True
-            op = ResOperation(rop.PTR_EQ, [v1.force_box(),
+            op = ResOperation(rop.PTR_EQ, [v1.force_box(self),
                                            llhelper.CONST_NULL],
                               resultbox)
-            self.optimizer.emit_operation(op)
+            self.emit_operation(op)
             return True
         #
         return False
@@ -652,8 +652,8 @@
                     do = EffectInfo.OS_STREQ_NONNULL_CHAR
                 else:
                     do = EffectInfo.OS_STREQ_CHECKNULL_CHAR
-                self.generate_modified_call(do, [v1.force_box(),
-                                                 vchar.force_box()], resultbox,
+                self.generate_modified_call(do, [v1.force_box(self),
+                                                 vchar.force_box(self)], resultbox,
                                             mode)
                 return True
         #
@@ -662,10 +662,10 @@
                 do = EffectInfo.OS_STREQ_SLICE_NONNULL
             else:
                 do = EffectInfo.OS_STREQ_SLICE_CHECKNULL
-            self.generate_modified_call(do, [v1.vstr.force_box(),
-                                             v1.vstart.force_box(),
-                                             v1.vlength.force_box(),
-                                             v2.force_box()], resultbox, mode)
+            self.generate_modified_call(do, [v1.vstr.force_box(self),
+                                             v1.vstart.force_box(self),
+                                             v1.vlength.force_box(self),
+                                             v2.force_box(self)], resultbox, mode)
             return True
         return False
 
@@ -675,16 +675,11 @@
         calldescr, func = cic.callinfo_for_oopspec(oopspecindex)
         op = ResOperation(rop.CALL, [ConstInt(func)] + args, result,
                           descr=calldescr)
-        self.optimizer.emit_operation(op)
+        self.emit_operation(op)
 
     def propagate_forward(self, op):
-        if not self.enabled:
-            self.emit_operation(op)
-            return
-
         dispatch_opt(self, op)
 
-
 dispatch_opt = make_dispatcher_method(OptString, 'optimize_',
         default=OptString.emit_operation)
 
diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py
--- a/pypy/jit/metainterp/resoperation.py
+++ b/pypy/jit/metainterp/resoperation.py
@@ -221,7 +221,6 @@
         newop.setfailargs(self.getfailargs())
         return newop
 
-
 # ============
 # arity mixins
 # ============
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -2956,6 +2956,18 @@
         assert res == f(32)
         self.check_loops(arraylen_gc=2)
 
+    def test_ulonglong_mod(self):
+        myjitdriver = JitDriver(greens = [], reds = ['n', 'sa', 'i'])
+        def f(n):
+            sa = i = rffi.cast(rffi.ULONGLONG, 1)
+            while i < rffi.cast(rffi.ULONGLONG, n):
+                myjitdriver.jit_merge_point(sa=sa, n=n, i=i)
+                sa += sa % i
+                i += 1
+        res = self.meta_interp(f, [32])
+        assert res == f(32)
+
+
 class TestOOtype(BasicTests, OOJitMixin):
 
     def test_oohash(self):
diff --git a/pypy/jit/metainterp/test/test_longlong.py b/pypy/jit/metainterp/test/test_longlong.py
--- a/pypy/jit/metainterp/test/test_longlong.py
+++ b/pypy/jit/metainterp/test/test_longlong.py
@@ -118,6 +118,26 @@
         res = self.interp_operations(f, [1000000000])
         assert res == 123500000000.0
 
+    def test_floats_negative(self):
+        def f(i):
+            # i == 1000000000
+            f = i * -123.5
+            n = r_longlong(f)
+            compare(n, -29, 1054051584)
+            return float(n)
+        res = self.interp_operations(f, [1000000000])
+        assert res == -123500000000.0
+
+    def test_floats_ulonglong(self):
+        def f(i):
+            # i == 1000000000
+            f = i * 12350000000.0
+            n = r_ulonglong(f)
+            compare(n, -1419508847, 538116096)
+            return float(n)
+        res = self.interp_operations(f, [1000000000])
+        assert res == 12350000000000000000.0
+
     def test_unsigned_compare_ops(self):
         def f(n1, n2):
             # n == 30002000000000
diff --git a/pypy/jit/metainterp/test/test_resume.py b/pypy/jit/metainterp/test/test_resume.py
--- a/pypy/jit/metainterp/test/test_resume.py
+++ b/pypy/jit/metainterp/test/test_resume.py
@@ -576,8 +576,9 @@
 
 
 class FakeOptimizer_VirtualValue(object):
-    class cpu:
-        pass
+    class optimizer:
+        class cpu:
+            pass
 fakeoptimizer = FakeOptimizer_VirtualValue()
 
 def ConstAddr(addr, cpu):   # compatibility
@@ -1135,12 +1136,7 @@
     modifier.liveboxes = {}
     modifier.vfieldboxes = {}
 
-    class FakeOptimizer(object):
-        class cpu:
-            pass
-        def new_const_item(self, descr):
-            return None
-    v2 = VArrayValue(FakeOptimizer(), LLtypeMixin.arraydescr, 2, b2s)
+    v2 = VArrayValue(LLtypeMixin.arraydescr, None, 2, b2s)
     v2._items = [b4s, c1s]
     modifier.register_virtual_fields(b2s, [b4s, c1s])
     liveboxes = []
diff --git a/pypy/jit/metainterp/test/test_string.py b/pypy/jit/metainterp/test/test_string.py
--- a/pypy/jit/metainterp/test/test_string.py
+++ b/pypy/jit/metainterp/test/test_string.py
@@ -484,6 +484,30 @@
             return len(sa.val)
         assert self.meta_interp(f, ['a']) == f('a')
 
+    def test_string_comepare_quasiimmutable(self):
+        class Sys(object):
+            _immutable_fields_ = ["defaultencoding?"]
+            def __init__(self, s):
+                self.defaultencoding = s
+        _str = self._str
+        sys = Sys(_str('ascii'))        
+        mydriver = JitDriver(reds = ['n', 'sa'], greens = [])
+        def f(n):
+            sa = 0
+            sys.defaultencoding = _str('ascii')
+            while n:
+                mydriver.jit_merge_point(n=n, sa=sa)
+                if sys.defaultencoding == _str('ascii'):
+                    sa += 1
+                n -= 1
+            sys.defaultencoding = _str('utf-8')
+            return sa
+        assert self.meta_interp(f, [8]) == f(8)
+        self.check_loops({'int_add': 1, 'guard_true': 1, 'int_sub': 1,
+                          'jump': 1, 'int_is_true': 1,
+                          'guard_not_invalidated': 1})
+
+
 #class TestOOtype(StringTests, OOJitMixin):
 #    CALL = "oosend"
 #    CALL_PURE = "oosend_pure"
diff --git a/pypy/jit/metainterp/test/test_virtualstate.py b/pypy/jit/metainterp/test/test_virtualstate.py
--- a/pypy/jit/metainterp/test/test_virtualstate.py
+++ b/pypy/jit/metainterp/test/test_virtualstate.py
@@ -431,7 +431,7 @@
         
 
 class BaseTestBridges(BaseTest):
-    enable_opts = "intbounds:rewrite:virtualize:string:heap:unroll"
+    enable_opts = "intbounds:rewrite:virtualize:string:pure:heap:unroll"
 
     def _do_optimize_bridge(self, bridge, call_pure_results):
         from pypy.jit.metainterp.optimizeopt import optimize_bridge_1, build_opt_chain
diff --git a/pypy/module/_numpy/interp_dtype.py b/pypy/module/_numpy/interp_dtype.py
--- a/pypy/module/_numpy/interp_dtype.py
+++ b/pypy/module/_numpy/interp_dtype.py
@@ -7,13 +7,14 @@
 from pypy.interpreter.typedef import TypeDef, interp_attrproperty, GetSetProperty
 from pypy.module._numpy import signature
 from pypy.objspace.std.floatobject import float2string
-from pypy.rlib import rfloat
-from pypy.rlib.rarithmetic import widen
+from pypy.rlib import rarithmetic, rfloat
+from pypy.rlib.rarithmetic import LONG_BIT, widen
 from pypy.rlib.objectmodel import specialize, enforceargs
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.rpython.lltypesystem import lltype, rffi
 
 
+UNSIGNEDLTR = "u"
 SIGNEDLTR = "i"
 BOOLLTR = "b"
 FLOATINGLTR = "f"
@@ -61,7 +62,10 @@
             self.val = val
 
         def wrap(self, space):
-            return space.wrap(self.val)
+            val = self.val
+            if valtype is rarithmetic.r_singlefloat:
+                val = float(val)
+            return space.wrap(val)
 
         def convert_to(self, dtype):
             return dtype.adapt_val(self.val)
@@ -145,7 +149,7 @@
         return self.adapt_val(func(self, self.for_computation(self.unbox(v))))
     return impl
 
-class ArithmaticTypeMixin(object):
+class ArithmeticTypeMixin(object):
     _mixin_ = True
 
     @binop
@@ -200,11 +204,17 @@
         return v1 >= v2
 
 
-class FloatArithmeticDtype(ArithmaticTypeMixin):
+class FloatArithmeticDtype(ArithmeticTypeMixin):
     _mixin_ = True
 
+    def unwrap(self, space, w_item):
+        return self.adapt_val(space.float_w(space.float(w_item)))
+
     def for_computation(self, v):
-        return v
+        return float(v)
+
+    def str_format(self, item):
+        return float2string(self.for_computation(self.unbox(item)), 'g', rfloat.DTSF_STR_PRECISION)
 
     @binop
     def mod(self, v1, v2):
@@ -266,7 +276,7 @@
     def arcsinh(self, v):
         return math.asinh(v)
 
-class IntegerArithmeticDtype(ArithmaticTypeMixin):
+class IntegerArithmeticDtype(ArithmeticTypeMixin):
     _mixin_ = True
 
     def unwrap(self, space, w_item):
@@ -275,10 +285,16 @@
     def for_computation(self, v):
         return widen(v)
 
+    def str_format(self, item):
+        return str(widen(self.unbox(item)))
+
     @binop
     def mod(self, v1, v2):
         return v1 % v2
 
+class SignedIntegerArithmeticDtype(IntegerArithmeticDtype):
+    _mixin_ = True
+
     @unaryop
     def sign(self, v):
         if v > 0:
@@ -289,17 +305,21 @@
             assert v == 0
             return 0
 
-    def str_format(self, item):
-        return str(widen(self.unbox(item)))
+class UnsignedIntegerArithmeticDtype(IntegerArithmeticDtype):
+    _mixin_ = True
+
+    @unaryop
+    def sign(self, v):
+        return int(v != 0)
 
 W_BoolDtype = create_low_level_dtype(
     num = 0, kind = BOOLLTR, name = "bool",
-    aliases = ["?"],
+    aliases = ["?", "bool", "bool8"],
     applevel_types = ["bool"],
     T = lltype.Bool,
     valtype = bool,
 )
-class W_BoolDtype(IntegerArithmeticDtype, W_BoolDtype):
+class W_BoolDtype(SignedIntegerArithmeticDtype, W_BoolDtype):
     def unwrap(self, space, w_item):
         return self.adapt_val(space.is_true(w_item))
 
@@ -312,69 +332,140 @@
 
 W_Int8Dtype = create_low_level_dtype(
     num = 1, kind = SIGNEDLTR, name = "int8",
-    aliases = ["int8"],
+    aliases = ["b", "int8", "i1"],
     applevel_types = [],
     T = rffi.SIGNEDCHAR,
     valtype = rffi.SIGNEDCHAR._type,
     expected_size = 1,
 )
-class W_Int8Dtype(IntegerArithmeticDtype, W_Int8Dtype):
+class W_Int8Dtype(SignedIntegerArithmeticDtype, W_Int8Dtype):
+    pass
+
+W_UInt8Dtype = create_low_level_dtype(
+    num = 2, kind = UNSIGNEDLTR, name = "uint8",
+    aliases = ["B", "uint8", "I1"],
+    applevel_types = [],
+    T = rffi.UCHAR,
+    valtype = rffi.UCHAR._type,
+    expected_size = 1,
+)
+class W_UInt8Dtype(UnsignedIntegerArithmeticDtype, W_UInt8Dtype):
     pass
 
 W_Int16Dtype = create_low_level_dtype(
     num = 3, kind = SIGNEDLTR, name = "int16",
-    aliases = ["int16"],
+    aliases = ["h", "int16", "i2"],
     applevel_types = [],
     T = rffi.SHORT,
     valtype = rffi.SHORT._type,
     expected_size = 2,
 )
-class W_Int16Dtype(IntegerArithmeticDtype, W_Int16Dtype):
+class W_Int16Dtype(SignedIntegerArithmeticDtype, W_Int16Dtype):
+    pass
+
+W_UInt16Dtype = create_low_level_dtype(
+    num = 4, kind = UNSIGNEDLTR, name = "uint16",
+    aliases = ["H", "uint16", "I2"],
+    applevel_types = [],
+    T = rffi.USHORT,
+    valtype = rffi.USHORT._type,
+    expected_size = 2,
+)
+class W_UInt16Dtype(UnsignedIntegerArithmeticDtype, W_UInt16Dtype):
     pass
 
 W_Int32Dtype = create_low_level_dtype(
     num = 5, kind = SIGNEDLTR, name = "int32",
-    aliases = ["i"],
+    aliases = ["i", "int32", "i4"],
     applevel_types = [],
     T = rffi.INT,
     valtype = rffi.INT._type,
     expected_size = 4,
 )
-class W_Int32Dtype(IntegerArithmeticDtype, W_Int32Dtype):
+class W_Int32Dtype(SignedIntegerArithmeticDtype, W_Int32Dtype):
+    pass
+
+W_UInt32Dtype = create_low_level_dtype(
+    num = 6, kind = UNSIGNEDLTR, name = "uint32",
+    aliases = ["I", "uint32", "I4"],
+    applevel_types = [],
+    T = rffi.UINT,
+    valtype = rffi.UINT._type,
+    expected_size = 4,
+)
+class W_UInt32Dtype(UnsignedIntegerArithmeticDtype, W_UInt32Dtype):
     pass
 
 W_Int64Dtype = create_low_level_dtype(
     num = 9, kind = SIGNEDLTR, name = "int64",
-    aliases = [],
+    aliases = ["q", "int64", "i8"],
     applevel_types = ["long"],
     T = rffi.LONGLONG,
     valtype = rffi.LONGLONG._type,
     expected_size = 8,
 )
-class W_Int64Dtype(IntegerArithmeticDtype, W_Int64Dtype):
+class W_Int64Dtype(SignedIntegerArithmeticDtype, W_Int64Dtype):
+    pass
+
+W_UInt64Dtype = create_low_level_dtype(
+    num = 10, kind = UNSIGNEDLTR, name = "uint64",
+    aliases = ["Q", "uint64", "I8"],
+    applevel_types = [],
+    T = rffi.ULONGLONG,
+    valtype = rffi.ULONGLONG._type,
+    expected_size = 8,
+)
+class W_UInt64Dtype(UnsignedIntegerArithmeticDtype, W_UInt64Dtype):
+    pass
+
+if LONG_BIT == 32:
+    class W_LongDtype(W_Int32Dtype):
+        pass
+
+    class W_ULongDtype(W_UInt32Dtype):
+        pass
+else:
+    class W_LongDtype(W_Int64Dtype):
+        pass
+
+    class W_ULongDtype(W_UInt64Dtype):
+        pass
+
+W_LongDtype.num = 7
+W_LongDtype.aliases = ["l"]
+W_LongDtype.applevel_types = ["int"]
+W_ULongDtype.num = 8
+W_ULongDtype.aliases = ["L"]
+
+W_Float32Dtype = create_low_level_dtype(
+    num = 11, kind = FLOATINGLTR, name = "float32",
+    aliases = ["f", "float32", "f4"],
+    applevel_types = [],
+    T = lltype.SingleFloat,
+    valtype = rarithmetic.r_singlefloat,
+    expected_size = 4,
+)
+class W_Float32Dtype(FloatArithmeticDtype, W_Float32Dtype):
     pass
 
 W_Float64Dtype = create_low_level_dtype(
     num = 12, kind = FLOATINGLTR, name = "float64",
-    aliases = [],
+    aliases = ["d", "float64", "f8"],
     applevel_types = ["float"],
     T = lltype.Float,
     valtype = float,
     expected_size = 8,
 )
 class W_Float64Dtype(FloatArithmeticDtype, W_Float64Dtype):
-    def unwrap(self, space, w_item):
-        return self.adapt_val(space.float_w(space.float(w_item)))
-
-    def str_format(self, item):
-        return float2string(self.unbox(item), 'g', rfloat.DTSF_STR_PRECISION)
+    pass
 
 ALL_DTYPES = [
     W_BoolDtype,
-    W_Int8Dtype, W_Int16Dtype, W_Int32Dtype, W_Int64Dtype,
-    W_Float64Dtype
+    W_Int8Dtype, W_UInt8Dtype, W_Int16Dtype, W_UInt16Dtype,
+    W_Int32Dtype, W_UInt32Dtype, W_LongDtype, W_ULongDtype,
+    W_Int64Dtype, W_UInt64Dtype,
+    W_Float32Dtype, W_Float64Dtype,
 ]
-
 dtypes_by_alias = unrolling_iterable([
     (alias, dtype)
     for dtype in ALL_DTYPES
@@ -399,6 +490,7 @@
 
     num = interp_attrproperty("num", cls=W_Dtype),
     kind = interp_attrproperty("kind", cls=W_Dtype),
+    itemsize = interp_attrproperty("num_bytes", cls=W_Dtype),
     shape = GetSetProperty(W_Dtype.descr_get_shape),
 )
 W_Dtype.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/_numpy/interp_ufuncs.py b/pypy/module/_numpy/interp_ufuncs.py
--- a/pypy/module/_numpy/interp_ufuncs.py
+++ b/pypy/module/_numpy/interp_ufuncs.py
@@ -4,6 +4,7 @@
 from pypy.interpreter.typedef import TypeDef, GetSetProperty, interp_attrproperty
 from pypy.module._numpy import interp_dtype, signature
 from pypy.rlib import jit
+from pypy.rlib.rarithmetic import LONG_BIT
 from pypy.tool.sourcetools import func_with_new_name
 
 
@@ -180,23 +181,56 @@
 
     # Everything promotes to float, and bool promotes to everything.
     if dt2.kind == interp_dtype.FLOATINGLTR or dt1.kind == interp_dtype.BOOLLTR:
+        # Float32 + 8-bit int = Float64
+        if dt2.num == 11 and dt1.num_bytes >= 4:
+            return space.fromcache(interp_dtype.W_Float64Dtype)
         return dt2
 
-    assert False
+    # for now this means mixing signed and unsigned
+    if dt2.kind == interp_dtype.SIGNEDLTR:
+        # if dt2 has a greater number of bytes, then just go with it
+        if dt1.num_bytes < dt2.num_bytes:
+            return dt2
+        # we need to promote both dtypes
+        dtypenum = dt2.num + 2
+    else:
+        # increase to the next signed type (or to float)
+        dtypenum = dt2.num + 1
+        # UInt64 + signed = Float64
+        if dt2.num == 10:
+            dtypenum += 1
+    newdtype = interp_dtype.ALL_DTYPES[dtypenum]
+
+    if newdtype.num_bytes > dt2.num_bytes or newdtype.kind == interp_dtype.FLOATINGLTR:
+        return space.fromcache(newdtype)
+    else:
+        # we only promoted to long on 32-bit or to longlong on 64-bit
+        # this is really for dealing with the Long and Ulong dtypes
+        if LONG_BIT == 32:
+            dtypenum += 2
+        else:
+            dtypenum += 3
+        return space.fromcache(interp_dtype.ALL_DTYPES[dtypenum])
 
 def find_unaryop_result_dtype(space, dt, promote_to_float=False,
     promote_bools=False, promote_to_largest=False):
     if promote_bools and (dt.kind == interp_dtype.BOOLLTR):
         return space.fromcache(interp_dtype.W_Int8Dtype)
     if promote_to_float:
+        if dt.kind == interp_dtype.FLOATINGLTR:
+            return dt
+        if dt.num >= 5:
+            return space.fromcache(interp_dtype.W_Float64Dtype)
         for bytes, dtype in interp_dtype.dtypes_by_num_bytes:
-            if dtype.kind == interp_dtype.FLOATINGLTR and dtype.num_bytes >= dt.num_bytes:
+            if dtype.kind == interp_dtype.FLOATINGLTR and dtype.num_bytes > dt.num_bytes:
                 return space.fromcache(dtype)
     if promote_to_largest:
         if dt.kind == interp_dtype.BOOLLTR or dt.kind == interp_dtype.SIGNEDLTR:
             return space.fromcache(interp_dtype.W_Int64Dtype)
         elif dt.kind == interp_dtype.FLOATINGLTR:
             return space.fromcache(interp_dtype.W_Float64Dtype)
+        elif dt.kind == interp_dtype.UNSIGNEDLTR:
+            return space.fromcache(interp_dtype.W_UInt64Dtype)
         else:
             assert False
     return dt
@@ -205,15 +239,23 @@
     w_type = space.type(w_obj)
 
     bool_dtype = space.fromcache(interp_dtype.W_BoolDtype)
+    long_dtype = space.fromcache(interp_dtype.W_LongDtype)
     int64_dtype = space.fromcache(interp_dtype.W_Int64Dtype)
 
     if space.is_w(w_type, space.w_bool):
-        if current_guess is None:
+        if current_guess is None or current_guess is bool_dtype:
             return bool_dtype
+        return current_guess
     elif space.is_w(w_type, space.w_int):
         if (current_guess is None or current_guess is bool_dtype or
-            current_guess is int64_dtype):
-            return int64_dtype
+            current_guess is long_dtype):
+            return long_dtype
+        return current_guess
+    elif space.is_w(w_type, space.w_long):
+        if (current_guess is None or current_guess is bool_dtype or
+            current_guess is long_dtype or current_guess is int64_dtype):
+             return int64_dtype
+        return current_guess
     return space.fromcache(interp_dtype.W_Float64Dtype)
 
 
@@ -225,7 +267,9 @@
         def impl(res_dtype, lvalue, rvalue):
             res = getattr(res_dtype, op_name)(lvalue, rvalue)
             if comparison_func:
-                res = space.fromcache(interp_dtype.W_BoolDtype).box(res)
+                booldtype = space.fromcache(interp_dtype.W_BoolDtype)
+                assert isinstance(booldtype, interp_dtype.W_BoolDtype)
+                res = booldtype.box(res)
             return res
     return func_with_new_name(impl, ufunc_name)
 
@@ -278,7 +322,7 @@
 
         identity = extra_kwargs.get("identity")
         if identity is not None:
-            identity = space.fromcache(interp_dtype.W_Int64Dtype).adapt_val(identity)
+            identity = space.fromcache(interp_dtype.W_LongDtype).adapt_val(identity)
         extra_kwargs["identity"] = identity
 
         func = ufunc_dtype_caller(space, ufunc_name, op_name, argcount,
diff --git a/pypy/module/_numpy/test/test_base.py b/pypy/module/_numpy/test/test_base.py
--- a/pypy/module/_numpy/test/test_base.py
+++ b/pypy/module/_numpy/test/test_base.py
@@ -64,18 +64,46 @@
     def test_unaryops(self, space):
         bool_dtype = space.fromcache(interp_dtype.W_BoolDtype)
         int8_dtype = space.fromcache(interp_dtype.W_Int8Dtype)
+        uint8_dtype = space.fromcache(interp_dtype.W_UInt8Dtype)
+        int16_dtype = space.fromcache(interp_dtype.W_Int16Dtype)
+        uint16_dtype = space.fromcache(interp_dtype.W_UInt16Dtype)
         int32_dtype = space.fromcache(interp_dtype.W_Int32Dtype)
+        uint32_dtype = space.fromcache(interp_dtype.W_UInt32Dtype)
+        long_dtype = space.fromcache(interp_dtype.W_LongDtype)
+        ulong_dtype = space.fromcache(interp_dtype.W_ULongDtype)
+        int64_dtype = space.fromcache(interp_dtype.W_Int64Dtype)
+        uint64_dtype = space.fromcache(interp_dtype.W_UInt64Dtype)
+        float32_dtype = space.fromcache(interp_dtype.W_Float32Dtype)
         float64_dtype = space.fromcache(interp_dtype.W_Float64Dtype)
 
-        # Normal rules, everythign returns itself
+        # Normal rules, everything returns itself
         assert find_unaryop_result_dtype(space, bool_dtype) is bool_dtype
         assert find_unaryop_result_dtype(space, int8_dtype) is int8_dtype
+        assert find_unaryop_result_dtype(space, uint8_dtype) is uint8_dtype
+        assert find_unaryop_result_dtype(space, int16_dtype) is int16_dtype
+        assert find_unaryop_result_dtype(space, uint16_dtype) is uint16_dtype
         assert find_unaryop_result_dtype(space, int32_dtype) is int32_dtype
+        assert find_unaryop_result_dtype(space, uint32_dtype) is uint32_dtype
+        assert find_unaryop_result_dtype(space, long_dtype) is long_dtype
+        assert find_unaryop_result_dtype(space, ulong_dtype) is ulong_dtype
+        assert find_unaryop_result_dtype(space, int64_dtype) is int64_dtype
+        assert find_unaryop_result_dtype(space, uint64_dtype) is uint64_dtype
+        assert find_unaryop_result_dtype(space, float32_dtype) is float32_dtype
         assert find_unaryop_result_dtype(space, float64_dtype) is float64_dtype
 
         # Coerce to floats, some of these will eventually be float16, or
         # whatever our smallest float type is.
-        assert find_unaryop_result_dtype(space, bool_dtype, promote_to_float=True) is float64_dtype
-        assert find_unaryop_result_dtype(space, int8_dtype, promote_to_float=True) is float64_dtype
+        assert find_unaryop_result_dtype(space, bool_dtype, promote_to_float=True) is float32_dtype # will be float16 if we ever put that in
+        assert find_unaryop_result_dtype(space, int8_dtype, promote_to_float=True) is float32_dtype # will be float16 if we ever put that in
+        assert find_unaryop_result_dtype(space, uint8_dtype, promote_to_float=True) is float32_dtype # will be float16 if we ever put that in
+        assert find_unaryop_result_dtype(space, int16_dtype, promote_to_float=True) is float32_dtype
+        assert find_unaryop_result_dtype(space, uint16_dtype, promote_to_float=True) is float32_dtype
         assert find_unaryop_result_dtype(space, int32_dtype, promote_to_float=True) is float64_dtype
+        assert find_unaryop_result_dtype(space, uint32_dtype, promote_to_float=True) is float64_dtype
+        assert find_unaryop_result_dtype(space, int64_dtype, promote_to_float=True) is float64_dtype
+        assert find_unaryop_result_dtype(space, uint64_dtype, promote_to_float=True) is float64_dtype
+        assert find_unaryop_result_dtype(space, float32_dtype, promote_to_float=True) is float32_dtype
         assert find_unaryop_result_dtype(space, float64_dtype, promote_to_float=True) is float64_dtype
+
+        # promote bools, happens with sign ufunc
+        assert find_unaryop_result_dtype(space, bool_dtype, promote_bools=True) is int8_dtype
diff --git a/pypy/module/_numpy/test/test_dtypes.py b/pypy/module/_numpy/test/test_dtypes.py
--- a/pypy/module/_numpy/test/test_dtypes.py
+++ b/pypy/module/_numpy/test/test_dtypes.py
@@ -17,6 +17,7 @@
         from _numpy import dtype
 
         assert dtype(bool).num == 0
+        assert dtype(int).num == 7
         assert dtype(long).num == 9
         assert dtype(float).num == 12
 
@@ -81,6 +82,48 @@
             assert isinstance(a[i], (int, long))
             assert a[1] == 1
 
+    def test_overflow(self):
+        from _numpy import array, dtype
+        assert array([128], 'b')[0] == -128
+        assert array([256], 'B')[0] == 0
+        assert array([32768], 'h')[0] == -32768
+        assert array([65536], 'H')[0] == 0
+        if dtype('l').itemsize == 4: # 32-bit
+            raises(OverflowError, "array([2**32/2], 'i')")
+            raises(OverflowError, "array([2**32], 'I')")
+        raises(OverflowError, "array([2**64/2], 'q')")
+        raises(OverflowError, "array([2**64], 'Q')")
+
+    def test_bool_binop_types(self):
+        from _numpy import array, dtype
+        types = ('?','b','B','h','H','i','I','l','L','q','Q','f','d')
+        N = len(types)
+        a = array([True], '?')
+        for t in types:
+            assert (a + array([0], t)).dtype is dtype(t)
+
+    def test_binop_types(self):
+        from _numpy import array, dtype
+        tests = [('b','B','h'), ('b','h','h'), ('b','H','i'), ('b','i','i'),
+                 ('b','l','l'), ('b','q','q'), ('b','Q','d'), ('B','h','h'),
+                 ('B','H','H'), ('B','i','i'), ('B','I','I'), ('B','l','l'),
+                 ('B','L','L'), ('B','q','q'), ('B','Q','Q'), ('h','H','i'),
+                 ('h','i','i'), ('h','l','l'), ('h','q','q'), ('h','Q','d'),
+                 ('H','i','i'), ('H','I','I'), ('H','l','l'), ('H','L','L'),
+                 ('H','q','q'), ('H','Q','Q'), ('i','l','l'), ('i','q','q'),
+                 ('i','Q','d'), ('I','L','L'), ('I','q','q'), ('I','Q','Q'),
+                 ('q','Q','d'), ('b','f','f'), ('B','f','f'), ('h','f','f'),
+                 ('H','f','f'), ('i','f','d'), ('I','f','d'), ('l','f','d'),
+                 ('L','f','d'), ('q','f','d'), ('Q','f','d'), ('q','d','d')]
+        if dtype('i').itemsize == dtype('l').itemsize: # 32-bit
+            tests.extend([('b','I','q'), ('b','L','q'), ('h','I','q'),
+                          ('h','L','q'), ('i','I','q'), ('i','L','q')])
+        else:
+            tests.extend([('b','I','l'), ('b','L','d'), ('h','I','l'),
+                          ('h','L','d'), ('i','I','l'), ('i','L','d')])
+        for d1, d2, dout in tests:
+            assert (array([1], d1) + array([1], d2)).dtype is dtype(dout)
+
     def test_add_int8(self):
         from _numpy import array, dtype
 
@@ -99,6 +142,15 @@
         for i in range(5):
             assert b[i] == i * 2
 
+    def test_add_uint32(self):
+        from _numpy import array, dtype
+
+        a = array(range(5), dtype="I")
+        b = a + a
+        assert b.dtype is dtype("I")
+        for i in range(5):
+            assert b[i] == i * 2
+
     def test_shape(self):
         from _numpy import dtype
 
diff --git a/pypy/module/_numpy/test/test_numarray.py b/pypy/module/_numpy/test/test_numarray.py
--- a/pypy/module/_numpy/test/test_numarray.py
+++ b/pypy/module/_numpy/test/test_numarray.py
@@ -18,7 +18,7 @@
         assert a[13] == 5.3
 
     def test_zeros_tuple_shape(self):
-        from numpy import zeros, ones, empty
+        from _numpy import zeros, ones, empty
         a = zeros((15,))
         assert len(a) == 15
         b = ones((3,))
@@ -560,8 +560,10 @@
         from _numpy import array, dtype
 
         assert array([True]).dtype is dtype(bool)
-        assert array([True, 1]).dtype is dtype(long)
-        assert array([1, 2, 3]).dtype is dtype(long)
+        assert array([True, False]).dtype is dtype(bool)
+        assert array([True, 1]).dtype is dtype(int)
+        assert array([1, 2, 3]).dtype is dtype(int)
+        assert array([1L, 2, 3]).dtype is dtype(long)
         assert array([1.2, True]).dtype is dtype(float)
         assert array([1.2, 5]).dtype is dtype(float)
         assert array([]).dtype is dtype(float)
diff --git a/pypy/module/_numpy/test/test_ufuncs.py b/pypy/module/_numpy/test/test_ufuncs.py
--- a/pypy/module/_numpy/test/test_ufuncs.py
+++ b/pypy/module/_numpy/test/test_ufuncs.py
@@ -234,7 +234,7 @@
             assert b[i] == math.sin(a[i])
 
         a = sin(array([True, False], dtype=bool))
-        assert a[0] == sin(1)
+        assert abs(a[0] - sin(1)) < 1e-7 # a[0] will be less precise
         assert a[1] == 0.0
 
     def test_cos(self):
@@ -300,7 +300,7 @@
 
     def test_arcsinh(self):
         import math
-        from numpy import arcsinh, inf
+        from _numpy import arcsinh, inf
 
         for v in [inf, -inf, 1.0, math.e]:
             assert math.asinh(v) == arcsinh(v)
diff --git a/pypy/module/_numpy/test/test_zjit.py b/pypy/module/_numpy/test/test_zjit.py
--- a/pypy/module/_numpy/test/test_zjit.py
+++ b/pypy/module/_numpy/test/test_zjit.py
@@ -1,20 +1,24 @@
 from pypy.jit.metainterp.test.support import LLJitMixin
 from pypy.module._numpy import interp_ufuncs, signature
 from pypy.module._numpy.compile import (numpy_compile, FakeSpace,
-    FloatObject)
-from pypy.module._numpy.interp_dtype import W_Float64Dtype, W_Int64Dtype
+    FloatObject, IntObject)
+from pypy.module._numpy.interp_dtype import W_Int32Dtype, W_Float64Dtype, W_Int64Dtype, W_UInt64Dtype
 from pypy.module._numpy.interp_numarray import (BaseArray, SingleDimArray,
     SingleDimSlice, scalar_w)
 from pypy.rlib.nonconst import NonConstant
 from pypy.rpython.annlowlevel import llstr
 from pypy.rpython.test.test_llinterp import interpret
 
+import py
+
 
 class TestNumpyJIt(LLJitMixin):
     def setup_class(cls):
         cls.space = FakeSpace()
         cls.float64_dtype = cls.space.fromcache(W_Float64Dtype)
         cls.int64_dtype = cls.space.fromcache(W_Int64Dtype)
+        cls.uint64_dtype = cls.space.fromcache(W_UInt64Dtype)
+        cls.int32_dtype = cls.space.fromcache(W_Int32Dtype)
 
     def test_add(self):
         def f(i):
@@ -303,6 +307,31 @@
                           'int_lt': 1, 'guard_true': 1, 'jump': 1})
         assert result == 11.0
 
+    def test_int32_sum(self):
+        py.test.skip("pypy/jit/backend/llimpl.py needs to be changed to "
+                     "deal correctly with int dtypes for this test to "
+                     "work. skip for now until someone feels up to the task")
+        space = self.space
+        float64_dtype = self.float64_dtype
+        int32_dtype = self.int32_dtype
+
+        def f(n):
+            if NonConstant(False):
+                dtype = float64_dtype
+            else:
+                dtype = int32_dtype
+            ar = SingleDimArray(n, dtype=dtype)
+            i = 0
+            while i < n:
+                ar.get_concrete().setitem(i, int32_dtype.box(7))
+                i += 1
+            v = ar.descr_add(space, ar).descr_sum(space)
+            assert isinstance(v, IntObject)
+            return v.intval
+
+        result = self.meta_interp(f, [5], listops=True, backendopt=True)
+        assert result == f(5)
+
 class TestTranslation(object):
     def test_compile(self):
         x = numpy_compile('aa+f*f/a-', 10)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_00_model.py b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
--- a/pypy/module/pypyjit/test_pypy_c/test_00_model.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
@@ -50,8 +50,7 @@
         cmdline.append(str(self.filepath))
         #
         print cmdline, logfile
-        env={'PYPYLOG': 'jit-log-opt,jit-summary:' + str(logfile)}
-        #env={'PYPYLOG': ':' + str(logfile)}
+        env={'PYPYLOG': 'jit-log-opt,jit-log-noopt,jit-summary:' + str(logfile)}
         pipe = subprocess.Popen(cmdline,
                                 env=env,
                                 stdout=subprocess.PIPE,
diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py
--- a/pypy/module/pypyjit/test_pypy_c/test_call.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_call.py
@@ -366,12 +366,12 @@
             # make sure that the "block" is not allocated
             ...
             i20 = force_token()
-            setfield_gc(p0, i20, descr=<SignedFieldDescr .*PyFrame.vable_token .*>)
             p22 = new_with_vtable(19511408)
             p24 = new_array(1, descr=<GcPtrArrayDescr>)
             p26 = new_with_vtable(ConstClass(W_ListObject))
             p27 = new(descr=<SizeDescr .*>)
             p29 = new_array(0, descr=<GcPtrArrayDescr>)
+            setfield_gc(p0, i20, descr=<SignedFieldDescr .*PyFrame.vable_token .*>)
             setfield_gc(p27, p29, descr=<GcPtrFieldDescr list.items .*>)
             setfield_gc(p26, p27, descr=<.* .*W_ListObject.inst_wrappeditems .*>)
             setarrayitem_gc(p24, 0, p26, descr=<GcPtrArrayDescr>)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_generators.py b/pypy/module/pypyjit/test_pypy_c/test_generators.py
--- a/pypy/module/pypyjit/test_pypy_c/test_generators.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_generators.py
@@ -19,8 +19,8 @@
         assert loop.match_by_id("generator", """
             i16 = force_token()
             p45 = new_with_vtable(ConstClass(W_IntObject))
-            setfield_gc(p45, i29, descr=<SignedFieldDescr .*>)
             i47 = arraylen_gc(p8, descr=<GcPtrArrayDescr>) # Should be removed by backend
             setarrayitem_gc(p8, 0, p45, descr=<GcPtrArrayDescr>)
+            setfield_gc(p45, i29, descr=<SignedFieldDescr .*>)
             jump(..., descr=...)
             """)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_instance.py b/pypy/module/pypyjit/test_pypy_c/test_instance.py
--- a/pypy/module/pypyjit/test_pypy_c/test_instance.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_instance.py
@@ -125,8 +125,8 @@
             i12 = force_token()
             --TICK--
             p20 = new_with_vtable(ConstClass(W_IntObject))
+            setfield_gc(ConstPtr(ptr21), p20, descr=<GcPtrFieldDescr .*TypeCell.inst_w_value .*>)
             setfield_gc(p20, i11, descr=<SignedFieldDescr.*W_IntObject.inst_intval .*>)
-            setfield_gc(ConstPtr(ptr21), p20, descr=<GcPtrFieldDescr .*TypeCell.inst_w_value .*>)
             jump(p0, p1, p2, p3, p4, p20, p6, i7, p20, descr=<Loop.>)
         """)
 
diff --git a/pypy/module/thread/gil.py b/pypy/module/thread/gil.py
--- a/pypy/module/thread/gil.py
+++ b/pypy/module/thread/gil.py
@@ -16,7 +16,7 @@
 
 class GILThreadLocals(OSThreadLocals):
     """A version of OSThreadLocals that enforces a GIL."""
-    ll_GIL = thread.null_ll_lock
+    gil_ready = False
 
     def initialize(self, space):
         # add the GIL-releasing callback as an action on the space
@@ -25,12 +25,10 @@
 
     def setup_threads(self, space):
         """Enable threads in the object space, if they haven't already been."""
-        if not self.ll_GIL:
-            try:
-                self.ll_GIL = thread.allocate_ll_lock()
-            except thread.error:
+        if not self.gil_ready:
+            if not thread.gil_allocate():
                 raise wrap_thread_error(space, "can't allocate GIL")
-            thread.acquire_NOAUTO(self.ll_GIL, True)
+            self.gil_ready = True
             self.enter_thread(space)   # setup the main thread
             result = True
         else:
@@ -44,19 +42,16 @@
         # test_lock_again after the global state was cleared by
         # test_compile_lock.  As a workaround, we repatch these global
         # fields systematically.
-        spacestate.ll_GIL = self.ll_GIL
         invoke_around_extcall(before_external_call, after_external_call)
         return result
 
     def reinit_threads(self, space):
-        if self.ll_GIL:
-            self.ll_GIL = thread.allocate_ll_lock()
-            thread.acquire_NOAUTO(self.ll_GIL, True)
-            self.enter_thread(space)
+        if self.gil_ready:
+            self.gil_ready = False
+            self.setup_threads(space)
 
     def yield_thread(self):
-        thread.yield_thread()  # explicitly release the gil (used by test_gil)
-
+        do_yield_thread()
 
 class GILReleaseAction(PeriodicAsyncAction):
     """An action called every sys.checkinterval bytecodes.  It releases
@@ -64,16 +59,12 @@
     """
 
     def perform(self, executioncontext, frame):
-        # Other threads can run between the release() and the acquire()
-        # implicit in the following external function call (which has
-        # otherwise no effect).
-        thread.yield_thread()
+        do_yield_thread()
 
 
 class SpaceState:
 
     def _freeze_(self):
-        self.ll_GIL = thread.null_ll_lock
         self.action_after_thread_switch = None
         # ^^^ set by AsyncAction.fire_after_thread_switch()
         return False
@@ -95,14 +86,14 @@
     # this function must not raise, in such a way that the exception
     # transformer knows that it cannot raise!
     e = get_errno()
-    thread.release_NOAUTO(spacestate.ll_GIL)
+    thread.gil_release()
     set_errno(e)
 before_external_call._gctransformer_hint_cannot_collect_ = True
 before_external_call._dont_reach_me_in_del_ = True
 
 def after_external_call():
     e = get_errno()
-    thread.acquire_NOAUTO(spacestate.ll_GIL, True)
+    thread.gil_acquire()
     thread.gc_thread_run()
     spacestate.after_thread_switch()
     set_errno(e)
@@ -115,3 +106,18 @@
 # pointers in the shadow stack.  This is necessary because the GIL is
 # not held after the call to before_external_call() or before the call
 # to after_external_call().
+
+def do_yield_thread():
+    # explicitly release the gil, in a way that tries to give more
+    # priority to other threads (as opposed to continuing to run in
+    # the same thread).
+    if thread.gil_yield_thread():
+        thread.gc_thread_run()
+        spacestate.after_thread_switch()
+do_yield_thread._gctransformer_hint_close_stack_ = True
+do_yield_thread._dont_reach_me_in_del_ = True
+do_yield_thread._dont_inline_ = True
+
+# do_yield_thread() needs a different hint: _gctransformer_hint_close_stack_.
+# The *_external_call() functions are themselves called only from the rffi
+# module from a helper function that also has this hint.
diff --git a/pypy/module/thread/ll_thread.py b/pypy/module/thread/ll_thread.py
--- a/pypy/module/thread/ll_thread.py
+++ b/pypy/module/thread/ll_thread.py
@@ -17,7 +17,8 @@
     include_dirs = [str(py.path.local(autopath.pypydir).join('translator', 'c'))],
     export_symbols = ['RPyThreadGetIdent', 'RPyThreadLockInit',
                       'RPyThreadAcquireLock', 'RPyThreadReleaseLock',
-                      'RPyThreadYield',
+                      'RPyGilAllocate', 'RPyGilYieldThread',
+                      'RPyGilRelease', 'RPyGilAcquire',
                       'RPyThreadGetStackSize', 'RPyThreadSetStackSize',
                       'RPyOpaqueDealloc_ThreadLock',
                       'RPyThreadAfterFork']
@@ -69,8 +70,16 @@
                                          [TLOCKP], lltype.Void,
                                          _nowrapper=True)
 
-# this function does nothing apart from releasing the GIL temporarily.
-yield_thread = llexternal('RPyThreadYield', [], lltype.Void, threadsafe=True)
+# these functions manipulate directly the GIL, whose definition does not
+# escape the C code itself
+gil_allocate     = llexternal('RPyGilAllocate', [], lltype.Signed,
+                              _nowrapper=True)
+gil_yield_thread = llexternal('RPyGilYieldThread', [], lltype.Signed,
+                              _nowrapper=True)
+gil_release      = llexternal('RPyGilRelease', [], lltype.Void,
+                              _nowrapper=True)
+gil_acquire      = llexternal('RPyGilAcquire', [], lltype.Void,
+                              _nowrapper=True)
 
 def allocate_lock():
     return Lock(allocate_ll_lock())
diff --git a/pypy/module/thread/test/test_gil.py b/pypy/module/thread/test/test_gil.py
--- a/pypy/module/thread/test/test_gil.py
+++ b/pypy/module/thread/test/test_gil.py
@@ -30,19 +30,34 @@
     use_threads = True
     bigtest = False
 
-    def test_one_thread(self):
+    def test_one_thread(self, skew=+1):
+        from pypy.rlib.debug import debug_print
         if self.bigtest:
-            N = 1000000
+            N = 100000
+            skew *= 25000
         else:
             N = 100
+            skew *= 25
         space = FakeSpace()
         class State:
             pass
         state = State()
-        def runme():
-            for i in range(N):
+        def runme(main=False):
+            j = 0
+            for i in range(N + [-skew, skew][main]):
+                state.datalen1 += 1   # try to crash if the GIL is not
+                state.datalen2 += 1   # correctly acquired
                 state.data.append((thread.get_ident(), i))
+                state.datalen3 += 1
+                state.datalen4 += 1
+                assert state.datalen1 == len(state.data)
+                assert state.datalen2 == len(state.data)
+                assert state.datalen3 == len(state.data)
+                assert state.datalen4 == len(state.data)
+                debug_print(main, i, state.datalen4)
                 state.threadlocals.yield_thread()
+                assert i == j
+                j += 1
         def bootstrap():
             try:
                 runme()
@@ -50,20 +65,26 @@
                 thread.gc_thread_die()
         def f():
             state.data = []
+            state.datalen1 = 0
+            state.datalen2 = 0
+            state.datalen3 = 0
+            state.datalen4 = 0
             state.threadlocals = gil.GILThreadLocals()
             state.threadlocals.setup_threads(space)
             thread.gc_thread_prepare()
             subident = thread.start_new_thread(bootstrap, ())
             mainident = thread.get_ident()
-            runme()
+            runme(True)
             still_waiting = 3000
             while len(state.data) < 2*N:
+                debug_print(len(state.data))
                 if not still_waiting:
                     raise ValueError("time out")
                 still_waiting -= 1
                 if not we_are_translated(): gil.before_external_call()
                 time.sleep(0.01)
                 if not we_are_translated(): gil.after_external_call()
+            debug_print("leaving!")
             i1 = i2 = 0
             for tid, i in state.data:
                 if tid == mainident:
@@ -72,14 +93,17 @@
                     assert i == i2; i2 += 1
                 else:
                     assert 0
-            assert i1 == N
-            assert i2 == N
+            assert i1 == N + skew
+            assert i2 == N - skew
             return len(state.data)
 
         fn = self.getcompiled(f, [])
         res = fn()
         assert res == 2*N
 
+    def test_one_thread_rev(self):
+        self.test_one_thread(skew=-1)
+
 
 class TestRunDirectly(GILTests):
     def getcompiled(self, f, argtypes):
diff --git a/pypy/module/thread/test/test_thread.py b/pypy/module/thread/test/test_thread.py
--- a/pypy/module/thread/test/test_thread.py
+++ b/pypy/module/thread/test/test_thread.py
@@ -225,7 +225,8 @@
 
         def busy_wait():
             for x in range(1000):
-                time.sleep(0.01)
+                print 'tick...', x  # <-force the GIL to be released, as
+                time.sleep(0.01)    #   time.sleep doesn't do non-translated
 
         # This is normally called by app_main.py
         signal.signal(signal.SIGINT, signal.default_int_handler)
diff --git a/pypy/rpython/lltypesystem/opimpl.py b/pypy/rpython/lltypesystem/opimpl.py
--- a/pypy/rpython/lltypesystem/opimpl.py
+++ b/pypy/rpython/lltypesystem/opimpl.py
@@ -357,7 +357,7 @@
 
 def op_cast_float_to_uint(f):
     assert type(f) is float
-    return r_uint(int(f))
+    return r_uint(long(f))
 
 def op_cast_float_to_longlong(f):
     assert type(f) is float
@@ -369,7 +369,7 @@
 
 def op_cast_float_to_ulonglong(f):
     assert type(f) is float
-    return r_ulonglong(r_longlong(f))
+    return r_ulonglong(long(f))
 
 def op_cast_char_to_int(b):
     assert type(b) is str and len(b) == 1
diff --git a/pypy/rpython/lltypesystem/test/test_lloperation.py b/pypy/rpython/lltypesystem/test/test_lloperation.py
--- a/pypy/rpython/lltypesystem/test/test_lloperation.py
+++ b/pypy/rpython/lltypesystem/test/test_lloperation.py
@@ -5,6 +5,7 @@
 from pypy.rpython.llinterp import LLFrame
 from pypy.rpython.test.test_llinterp import interpret
 from pypy.rpython import rclass
+from pypy.rlib.rarithmetic import LONGLONG_MASK, r_longlong, r_ulonglong
 
 LL_INTERP_OPERATIONS = [name[3:] for name in LLFrame.__dict__.keys()
                                  if name.startswith('op_')]
@@ -133,6 +134,14 @@
         py.test.raises(TypeError, llop.getinteriorfield,
                        lltype.Signed, s3, 'y')
 
+def test_cast_float_to_ulonglong():
+    f = 12350000000000000000.0
+    py.test.raises(OverflowError, r_longlong, f)
+    r_longlong(f / 2)   # does not raise OverflowError
+    #
+    x = llop.cast_float_to_ulonglong(lltype.UnsignedLongLong, f)
+    assert x == r_ulonglong(f)
+
 # ___________________________________________________________________________
 # This tests that the LLInterpreter and the LL_OPERATIONS tables are in sync.
 
diff --git a/pypy/translator/c/gcc/trackgcroot.py b/pypy/translator/c/gcc/trackgcroot.py
--- a/pypy/translator/c/gcc/trackgcroot.py
+++ b/pypy/translator/c/gcc/trackgcroot.py
@@ -486,6 +486,8 @@
         'paddq', 'pinsr',
         # zero-extending moves should not produce GC pointers
         'movz', 
+        # locked operations should not move GC pointers, at least so far
+        'lock',
         ])
 
     # a partial list is hopefully good enough for now; it's all to support
diff --git a/pypy/translator/c/src/thread.h b/pypy/translator/c/src/thread.h
--- a/pypy/translator/c/src/thread.h
+++ b/pypy/translator/c/src/thread.h
@@ -37,14 +37,9 @@
 
 #endif
 
-/* common helper: this does nothing, but is called with the GIL released.
-   This gives other threads a chance to grab the GIL and run. */
-void RPyThreadYield(void);
-
-#ifndef PYPY_NOT_MAIN_FILE
-void RPyThreadYield(void)
-{
-}
-#endif
+long RPyGilAllocate(void);
+long RPyGilYieldThread(void);
+void RPyGilRelease(void);
+void RPyGilAcquire(void);
 
 #endif
diff --git a/pypy/translator/c/src/thread_nt.h b/pypy/translator/c/src/thread_nt.h
--- a/pypy/translator/c/src/thread_nt.h
+++ b/pypy/translator/c/src/thread_nt.h
@@ -221,4 +221,57 @@
 #define RPyThreadTLS_Set(key, value)	TlsSetValue(key, value)
 
 
+/************************************************************/
+/* GIL code                                                 */
+/************************************************************/
+
+static volatile LONG pending_acquires = -1;
+static CRITICAL_SECTION mutex_gil;
+static HANDLE cond_gil;
+
+long RPyGilAllocate(void)
+{
+    pending_acquires = 0;
+    InitializeCriticalSection(&mutex_gil);
+    EnterCriticalSection(&mutex_gil);
+    cond_gil = CreateEvent (NULL, FALSE, FALSE, NULL);
+    return 1;
+}
+
+long RPyGilYieldThread(void)
+{
+    /* can be called even before RPyGilAllocate(), but in this case,
+       pending_acquires will be -1 */
+    if (pending_acquires <= 0)
+        return 0;
+    InterlockedIncrement(&pending_acquires);
+    PulseEvent(&cond_gil);
+
+    /* hack: the three following lines do a pthread_cond_wait(), and
+       normally specifying a timeout of INFINITE would be fine.  But the
+       first and second operations are not done atomically, so there is a
+       (small) risk that PulseEvent misses the WaitForSingleObject().
+       In this case the process will just sleep a few milliseconds. */
+    LeaveCriticalSection(&mutex_gil);
+    WaitForSingleObject(&cond_gil, 15);
+    EnterCriticalSection(&mutex_gil);
+
+    InterlockedDecrement(&pending_acquires);
+    return 1;
+}
+
+void RPyGilRelease(void)
+{
+    LeaveCriticalSection(&mutex_gil);
+    PulseEvent(&cond_gil);
+}
+
+void RPyGilAcquire(void)
+{
+    InterlockedIncrement(&pending_acquires);
+    EnterCriticalSection(&mutex_gil);
+    InterlockedDecrement(&pending_acquires);
+}
+
+
 #endif /* PYPY_NOT_MAIN_FILE */
diff --git a/pypy/translator/c/src/thread_pthread.h b/pypy/translator/c/src/thread_pthread.h
--- a/pypy/translator/c/src/thread_pthread.h
+++ b/pypy/translator/c/src/thread_pthread.h
@@ -12,6 +12,7 @@
 #include <signal.h>
 #include <stdio.h>
 #include <errno.h>
+#include <assert.h>
 
 /* The following is hopefully equivalent to what CPython does
    (which is trying to compile a snippet of code using it) */
@@ -459,4 +460,113 @@
 #define RPyThreadTLS_Set(key, value)	pthread_setspecific(key, value)
 
 
+/************************************************************/
+/* GIL code                                                 */
+/************************************************************/
+
+#ifdef __llvm__
+#  define HAS_ATOMIC_ADD
+#endif
+
+#ifdef __GNUC__
+#  if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 1)
+#    define HAS_ATOMIC_ADD
+#  endif
+#endif
+
+#ifdef HAS_ATOMIC_ADD
+#  define atomic_add __sync_fetch_and_add
+#else
+#  if defined(__amd64__)
+#    define atomic_add(ptr, value)  asm volatile ("lock addq %0, %1"        \
+                                 : : "ri"(value), "m"(*(ptr)) : "memory")
+#  elif defined(__i386__)
+#    define atomic_add(ptr, value)  asm volatile ("lock addl %0, %1"        \
+                                 : : "ri"(value), "m"(*(ptr)) : "memory")
+#  else
+#    error "Please use gcc >= 4.1 or write a custom 'asm' for your CPU."
+#  endif
+#endif
+
+#define ASSERT_STATUS(call)                             \
+    if (call != 0) {                                    \
+        fprintf(stderr, "Fatal error: " #call "\n");    \
+        abort();                                        \
+    }
+
+static void _debug_print(const char *msg)
+{
+#if 0
+    int col = (int)pthread_self();
+    col = 31 + ((col / 8) % 8);
+    fprintf(stderr, "\033[%dm%s\033[0m", col, msg);
+#endif
+}
+
+static volatile long pending_acquires = -1;
+static pthread_mutex_t mutex_gil = PTHREAD_MUTEX_INITIALIZER;
+static pthread_cond_t cond_gil = PTHREAD_COND_INITIALIZER;
+
+static void assert_has_the_gil(void)
+{
+#ifdef RPY_ASSERT
+    assert(pthread_mutex_trylock(&mutex_gil) != 0);
+    assert(pending_acquires >= 0);
+#endif
+}
+
+long RPyGilAllocate(void)
+{
+    _debug_print("RPyGilAllocate\n");
+    pending_acquires = 0;
+    pthread_mutex_trylock(&mutex_gil);
+    assert_has_the_gil();
+    return 1;
+}
+
+long RPyGilYieldThread(void)
+{
+    /* can be called even before RPyGilAllocate(), but in this case,
+       pending_acquires will be -1 */
+#ifdef RPY_ASSERT
+    if (pending_acquires >= 0)
+        assert_has_the_gil();
+#endif
+    if (pending_acquires <= 0)
+        return 0;
+    atomic_add(&pending_acquires, 1L);
+    _debug_print("{");
+    ASSERT_STATUS(pthread_cond_signal(&cond_gil));
+    ASSERT_STATUS(pthread_cond_wait(&cond_gil, &mutex_gil));
+    _debug_print("}");
+    atomic_add(&pending_acquires, -1L);
+    assert_has_the_gil();
+    return 1;
+}
+
+void RPyGilRelease(void)
+{
+    _debug_print("RPyGilRelease\n");
+#ifdef RPY_ASSERT
+    assert(pending_acquires >= 0);
+#endif
+    assert_has_the_gil();
+    ASSERT_STATUS(pthread_mutex_unlock(&mutex_gil));
+    ASSERT_STATUS(pthread_cond_signal(&cond_gil));
+}
+
+void RPyGilAcquire(void)
+{
+    _debug_print("about to RPyGilAcquire...\n");
+#ifdef RPY_ASSERT
+    assert(pending_acquires >= 0);
+#endif
+    atomic_add(&pending_acquires, 1L);
+    ASSERT_STATUS(pthread_mutex_lock(&mutex_gil));
+    atomic_add(&pending_acquires, -1L);
+    assert_has_the_gil();
+    _debug_print("RPyGilAcquire\n");
+}
+
+
 #endif /* PYPY_NOT_MAIN_FILE */

From noreply at buildbot.pypy.org  Tue Oct  4 00:40:53 2011
From: noreply at buildbot.pypy.org (timo_jbo)
Date: Tue,  4 Oct 2011 00:40:53 +0200 (CEST)
Subject: [pypy-commit] pypy separate-applevel-numpy: inf went missing from
 numpy somewhere along the way
Message-ID: <20111003224053.439D7820D9@wyvern.cs.uni-duesseldorf.de>

Author: Timo Paulssen <timonator at perpetuum-immobile.de>
Branch: separate-applevel-numpy
Changeset: r47811:436d920d8853
Date: 2011-10-04 00:04 +0200
http://bitbucket.org/pypy/pypy/changeset/436d920d8853/

Log:	inf went missing from numpy somewhere along the way

diff --git a/lib_pypy/numpy/__init__.py b/lib_pypy/numpy/__init__.py
--- a/lib_pypy/numpy/__init__.py
+++ b/lib_pypy/numpy/__init__.py
@@ -37,6 +37,8 @@
         tan,
     )
 
+inf = float("inf")
+
 def average(a):
     # This implements a weighted average, for now we don't implement the
     # weighting, just the average part!
diff --git a/pypy/module/_numpy/test/test_ufuncs.py b/pypy/module/_numpy/test/test_ufuncs.py
--- a/pypy/module/_numpy/test/test_ufuncs.py
+++ b/pypy/module/_numpy/test/test_ufuncs.py
@@ -300,7 +300,8 @@
 
     def test_arcsinh(self):
         import math
-        from _numpy import arcsinh, inf
+        from _numpy import arcsinh
+        from numpy import inf
 
         for v in [inf, -inf, 1.0, math.e]:
             assert math.asinh(v) == arcsinh(v)

From noreply at buildbot.pypy.org  Tue Oct  4 00:40:54 2011
From: noreply at buildbot.pypy.org (timo_jbo)
Date: Tue,  4 Oct 2011 00:40:54 +0200 (CEST)
Subject: [pypy-commit] pypy separate-applevel-numpy: remove the app_numpy.py,
 which should've been gone already.
Message-ID: <20111003224054.7279A820D9@wyvern.cs.uni-duesseldorf.de>

Author: Timo Paulssen <timonator at perpetuum-immobile.de>
Branch: separate-applevel-numpy
Changeset: r47812:025efeb947da
Date: 2011-10-04 00:12 +0200
http://bitbucket.org/pypy/pypy/changeset/025efeb947da/

Log:	remove the app_numpy.py, which should've been gone already.

diff --git a/pypy/module/_numpy/app_numpy.py b/pypy/module/_numpy/app_numpy.py
deleted file mode 100644

From noreply at buildbot.pypy.org  Tue Oct  4 00:40:55 2011
From: noreply at buildbot.pypy.org (timo_jbo)
Date: Tue,  4 Oct 2011 00:40:55 +0200 (CEST)
Subject: [pypy-commit] pypy separate-applevel-numpy: merge in default
Message-ID: <20111003224055.9F706820D9@wyvern.cs.uni-duesseldorf.de>

Author: Timo Paulssen <timonator at perpetuum-immobile.de>
Branch: separate-applevel-numpy
Changeset: r47813:b528c5ddc7c1
Date: 2011-10-04 00:16 +0200
http://bitbucket.org/pypy/pypy/changeset/b528c5ddc7c1/

Log:	merge in default

diff --git a/pypy/module/_numpy/__init__.py b/pypy/module/_numpy/__init__.py
--- a/pypy/module/_numpy/__init__.py
+++ b/pypy/module/_numpy/__init__.py
@@ -24,6 +24,7 @@
         ("arcsin", "arcsin"),
         ("arctan", "arctan"),
         ("arcsinh", "arcsinh"),
+        ("arctanh", "arctanh"),
         ("copysign", "copysign"),
         ("cos", "cos"),
         ("divide", "divide"),
diff --git a/pypy/module/_numpy/interp_dtype.py b/pypy/module/_numpy/interp_dtype.py
--- a/pypy/module/_numpy/interp_dtype.py
+++ b/pypy/module/_numpy/interp_dtype.py
@@ -260,12 +260,12 @@
         return math.tan(v)
     @unaryop
     def arcsin(self, v):
-        if v < -1.0 or v > 1.0:
+        if not -1.0 <= v <= 1.0:
             return rfloat.NAN
         return math.asin(v)
     @unaryop
     def arccos(self, v):
-        if v < -1.0 or v > 1.0:
+        if not -1.0 <= v <= 1.0:
             return rfloat.NAN
         return math.acos(v)
     @unaryop
@@ -276,6 +276,14 @@
     def arcsinh(self, v):
         return math.asinh(v)
 
+    @unaryop
+    def arctanh(self, v):
+        if v == 1.0 or v == -1.0:
+            return math.copysign(rfloat.INFINITY, v)
+        if not -1.0 < v < 1.0:
+            return rfloat.NAN
+        return math.atanh(v)
+
 class IntegerArithmeticDtype(ArithmeticTypeMixin):
     _mixin_ = True
 
diff --git a/pypy/module/_numpy/interp_ufuncs.py b/pypy/module/_numpy/interp_ufuncs.py
--- a/pypy/module/_numpy/interp_ufuncs.py
+++ b/pypy/module/_numpy/interp_ufuncs.py
@@ -313,6 +313,7 @@
             ("arccos", "arccos", 1, {"promote_to_float": True}),
             ("arctan", "arctan", 1, {"promote_to_float": True}),
             ("arcsinh", "arcsinh", 1, {"promote_to_float": True}),
+            ("arctanh", "arctanh", 1, {"promote_to_float": True}),
         ]:
             self.add_ufunc(space, *ufunc_def)
 
diff --git a/pypy/module/_numpy/test/test_ufuncs.py b/pypy/module/_numpy/test/test_ufuncs.py
--- a/pypy/module/_numpy/test/test_ufuncs.py
+++ b/pypy/module/_numpy/test/test_ufuncs.py
@@ -307,6 +307,17 @@
             assert math.asinh(v) == arcsinh(v)
         assert math.isnan(arcsinh(float("nan")))
 
+    def test_arctanh(self):
+        import math
+        from numpy import arctanh
+
+        for v in [.99, .5, 0, -.5, -.99]:
+            assert math.atanh(v) == arctanh(v)
+        for v in [2.0, -2.0]:
+            assert math.isnan(arctanh(v))
+        for v in [1.0, -1.0]:
+            assert arctanh(v) == math.copysign(float("inf"), v)
+
     def test_reduce_errors(self):
         from _numpy import sin, add
 

From noreply at buildbot.pypy.org  Tue Oct  4 00:40:57 2011
From: noreply at buildbot.pypy.org (timo_jbo)
Date: Tue,  4 Oct 2011 00:40:57 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-data-buffer: merge in
	separate-applevel-numpy
Message-ID: <20111003224057.26719820D9@wyvern.cs.uni-duesseldorf.de>

Author: Timo Paulssen <timonator at perpetuum-immobile.de>
Branch: numpy-data-buffer
Changeset: r47814:0698ee9b73e2
Date: 2011-10-04 00:35 +0200
http://bitbucket.org/pypy/pypy/changeset/0698ee9b73e2/

Log:	merge in separate-applevel-numpy

diff --git a/lib_pypy/numpy/__init__.py b/lib_pypy/numpy/__init__.py
--- a/lib_pypy/numpy/__init__.py
+++ b/lib_pypy/numpy/__init__.py
@@ -13,6 +13,8 @@
         arccos,
         arcsin,
         arctan,
+        arcsinh,
+        arctanh,
         copysign,
         cos,
         divide,
@@ -36,6 +38,8 @@
         tan,
     )
 
+inf = float("inf")
+
 def average(a):
     # This implements a weighted average, for now we don't implement the
     # weighting, just the average part!
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -307,7 +307,11 @@
         self._nonperiodic_actions = []
         self.has_bytecode_counter = False
         self.fired_actions = None
-        self.checkinterval_scaled = 100 * TICK_COUNTER_STEP
+        # the default value is not 100, unlike CPython 2.7, but a much
+        # larger value, because we use a technique that not only allows
+        # but actually *forces* another thread to run whenever the counter
+        # reaches zero.
+        self.checkinterval_scaled = 10000 * TICK_COUNTER_STEP
         self._rebuild_action_dispatcher()
 
     def fire(self, action):
diff --git a/pypy/jit/codewriter/effectinfo.py b/pypy/jit/codewriter/effectinfo.py
--- a/pypy/jit/codewriter/effectinfo.py
+++ b/pypy/jit/codewriter/effectinfo.py
@@ -74,6 +74,7 @@
     OS_LLONG_UGE                = 91
     OS_LLONG_URSHIFT            = 92
     OS_LLONG_FROM_UINT          = 93
+    OS_LLONG_U_TO_FLOAT         = 94
     #
     OS_MATH_SQRT                = 100
 
diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -440,6 +440,7 @@
     rewrite_op_ullong_mod_zer      = _do_builtin_call
     rewrite_op_gc_identityhash = _do_builtin_call
     rewrite_op_gc_id           = _do_builtin_call
+    rewrite_op_uint_mod        = _do_builtin_call
 
     # ----------
     # getfield/setfield/mallocs etc.
@@ -883,9 +884,15 @@
                 v = v_arg
                 oplist = []
             if unsigned1:
-                opname = 'cast_uint_to_longlong'
+                if unsigned2:
+                    opname = 'cast_uint_to_ulonglong'
+                else:
+                    opname = 'cast_uint_to_longlong'
             else:
-                opname = 'cast_int_to_longlong'
+                if unsigned2:
+                    opname = 'cast_int_to_ulonglong'
+                else:
+                    opname = 'cast_int_to_longlong'
             op2 = self.rewrite_operation(
                 SpaceOperation(opname, [v], v_result)
             )
@@ -995,6 +1002,21 @@
                 return op2
         ''' % (_op, _oopspec.lower(), _oopspec, _oopspec)).compile()
 
+    for _op, _oopspec in [('cast_int_to_ulonglong',     'FROM_INT'),
+                          ('cast_uint_to_ulonglong',    'FROM_UINT'),
+                          ('cast_float_to_ulonglong',   'FROM_FLOAT'),
+                          ('cast_ulonglong_to_float',   'U_TO_FLOAT'),
+                         ]:
+        exec py.code.Source('''
+            def rewrite_op_%s(self, op):
+                args = op.args
+                op1 = self.prepare_builtin_call(op, "ullong_%s", args)
+                op2 = self._handle_oopspec_call(op1, args,
+                                                EffectInfo.OS_LLONG_%s,
+                                           EffectInfo.EF_ELIDABLE_CANNOT_RAISE)
+                return op2
+        ''' % (_op, _oopspec.lower(), _oopspec)).compile()
+
     def _normalize(self, oplist):
         if isinstance(oplist, SpaceOperation):
             return [oplist]
diff --git a/pypy/jit/codewriter/support.py b/pypy/jit/codewriter/support.py
--- a/pypy/jit/codewriter/support.py
+++ b/pypy/jit/codewriter/support.py
@@ -315,18 +315,30 @@
 def _ll_1_llong_from_int(x):
     return r_longlong(intmask(x))
 
+def _ll_1_ullong_from_int(x):
+    return r_ulonglong(intmask(x))
+
 def _ll_1_llong_from_uint(x):
     return r_longlong(r_uint(x))
 
+def _ll_1_ullong_from_uint(x):
+    return r_ulonglong(r_uint(x))
+
 def _ll_1_llong_to_int(xll):
     return intmask(xll)
 
 def _ll_1_llong_from_float(xf):
     return r_longlong(xf)
 
+def _ll_1_ullong_from_float(xf):
+    return r_ulonglong(xf)
+
 def _ll_1_llong_to_float(xll):
     return float(rffi.cast(lltype.SignedLongLong, xll))
 
+def _ll_1_ullong_u_to_float(xull):
+    return float(rffi.cast(lltype.UnsignedLongLong, xull))
+
 
 def _ll_1_llong_abs(xll):
     if xll < 0:
@@ -351,20 +363,23 @@
     return llop.llong_mod(lltype.SignedLongLong, xll, yll)
 
 def _ll_2_ullong_floordiv(xll, yll):
-    return llop.ullong_floordiv(lltype.SignedLongLong, xll, yll)
+    return llop.ullong_floordiv(lltype.UnsignedLongLong, xll, yll)
 
 def _ll_2_ullong_floordiv_zer(xll, yll):
     if yll == 0:
         raise ZeroDivisionError
-    return llop.ullong_floordiv(lltype.SignedLongLong, xll, yll)
+    return llop.ullong_floordiv(lltype.UnsignedLongLong, xll, yll)
 
 def _ll_2_ullong_mod(xll, yll):
-    return llop.ullong_mod(lltype.SignedLongLong, xll, yll)
+    return llop.ullong_mod(lltype.UnsignedLongLong, xll, yll)
 
 def _ll_2_ullong_mod_zer(xll, yll):
     if yll == 0:
         raise ZeroDivisionError
-    return llop.ullong_mod(lltype.SignedLongLong, xll, yll)
+    return llop.ullong_mod(lltype.UnsignedLongLong, xll, yll)
+
+def _ll_2_uint_mod(xll, yll):
+    return llop.uint_mod(lltype.Unsigned, xll, yll)
 
 
 # libffi support
diff --git a/pypy/jit/codewriter/test/test_flatten.py b/pypy/jit/codewriter/test/test_flatten.py
--- a/pypy/jit/codewriter/test/test_flatten.py
+++ b/pypy/jit/codewriter/test/test_flatten.py
@@ -829,14 +829,15 @@
                     self.encoding_test(f, [rffi.cast(FROM, 42)], expectedstr,
                                        transform=True)
                 elif TO in (rffi.LONG, rffi.ULONG):
+                    if rffi.cast(FROM, -1) < 0:
+                        fnname = "llong_from_int"
+                    else:
+                        fnname = "llong_from_uint"
                     if TO == rffi.LONG:
                         TO = rffi.LONGLONG
                     else:
                         TO = rffi.ULONGLONG
-                    if rffi.cast(FROM, -1) < 0:
-                        fnname = "llong_from_int"
-                    else:
-                        fnname = "llong_from_uint"
+                        fnname = "u" + fnname
                     expected.pop()   # remove int_return
                     expected.append(
                         "residual_call_irf_f $<* fn %s>, <Descr>, I[%s], R[], F[] -> %%f0"
diff --git a/pypy/jit/codewriter/test/test_longlong.py b/pypy/jit/codewriter/test/test_longlong.py
--- a/pypy/jit/codewriter/test/test_longlong.py
+++ b/pypy/jit/codewriter/test/test_longlong.py
@@ -57,7 +57,8 @@
             assert op1.opname == 'residual_call_irf_f'
         else:
             assert op1.opname == 'residual_call_irf_i'
-        gotindex = getattr(EffectInfo, 'OS_' + op1.args[0].value.upper())
+        gotindex = getattr(EffectInfo,
+                           'OS_' + op1.args[0].value.upper().lstrip('U'))
         assert gotindex == oopspecindex
         assert op1.args[1] == 'calldescr-%d' % oopspecindex
         assert list(op1.args[2]) == [v for v in vlist
@@ -192,8 +193,12 @@
                       [lltype.SignedLongLong], lltype.Signed)
         self.do_check('cast_float_to_longlong', EffectInfo.OS_LLONG_FROM_FLOAT,
                       [lltype.Float], lltype.SignedLongLong)
+        self.do_check('cast_float_to_ulonglong', EffectInfo.OS_LLONG_FROM_FLOAT,
+                      [lltype.Float], lltype.UnsignedLongLong)
         self.do_check('cast_longlong_to_float', EffectInfo.OS_LLONG_TO_FLOAT,
                       [lltype.SignedLongLong], lltype.Float)
+        self.do_check('cast_ulonglong_to_float', EffectInfo.OS_LLONG_U_TO_FLOAT,
+                      [lltype.UnsignedLongLong], lltype.Float)
         for T1 in [lltype.SignedLongLong, lltype.UnsignedLongLong]:
             for T2 in [lltype.Signed, lltype.Unsigned]:
                 self.do_check('cast_primitive', EffectInfo.OS_LLONG_TO_INT,
diff --git a/pypy/jit/metainterp/optimizeopt/__init__.py b/pypy/jit/metainterp/optimizeopt/__init__.py
--- a/pypy/jit/metainterp/optimizeopt/__init__.py
+++ b/pypy/jit/metainterp/optimizeopt/__init__.py
@@ -7,6 +7,8 @@
 from pypy.jit.metainterp.optimizeopt.unroll import optimize_unroll, OptInlineShortPreamble
 from pypy.jit.metainterp.optimizeopt.fficall import OptFfiCall
 from pypy.jit.metainterp.optimizeopt.simplify import OptSimplify
+from pypy.jit.metainterp.optimizeopt.pure import OptPure
+from pypy.jit.metainterp.optimizeopt.earlyforce import OptEarlyForce
 from pypy.rlib.jit import PARAMETERS
 from pypy.rlib.unroll import unrolling_iterable
 
@@ -14,6 +16,8 @@
             ('rewrite', OptRewrite),
             ('virtualize', OptVirtualize),
             ('string', OptString),
+            ('earlyforce', OptEarlyForce),
+            ('pure', OptPure),
             ('heap', OptHeap),
             ('ffi', None),
             ('unroll', None)]
diff --git a/pypy/jit/metainterp/optimizeopt/earlyforce.py b/pypy/jit/metainterp/optimizeopt/earlyforce.py
new file mode 100644
--- /dev/null
+++ b/pypy/jit/metainterp/optimizeopt/earlyforce.py
@@ -0,0 +1,24 @@
+from pypy.jit.metainterp.optimizeopt.optimizer import Optimization
+from pypy.jit.metainterp.optimizeopt.vstring import VAbstractStringValue
+from pypy.jit.metainterp.resoperation import rop, ResOperation
+
+class OptEarlyForce(Optimization):
+    def propagate_forward(self, op):
+        opnum = op.getopnum()
+        if (opnum != rop.SETFIELD_GC and 
+            opnum != rop.SETARRAYITEM_GC and
+            opnum != rop.QUASIIMMUT_FIELD):
+               
+            for arg in op.getarglist():
+                if arg in self.optimizer.values:
+                    value = self.getvalue(arg)
+                    value.force_box(self)
+        self.emit_operation(op)
+
+    def new(self):
+        return OptEarlyForce()
+
+    def setup(self):
+        self.optimizer.optearlyforce = self
+
+    
diff --git a/pypy/jit/metainterp/optimizeopt/fficall.py b/pypy/jit/metainterp/optimizeopt/fficall.py
--- a/pypy/jit/metainterp/optimizeopt/fficall.py
+++ b/pypy/jit/metainterp/optimizeopt/fficall.py
@@ -177,10 +177,10 @@
             funcinfo.descr is None):
             return [op] # cannot optimize
         funcsymval = self.getvalue(op.getarg(2))
-        arglist = [funcsymval.force_box()]
+        arglist = [funcsymval.get_key_box()]
         for push_op in funcinfo.opargs:
             argval = self.getvalue(push_op.getarg(2))
-            arglist.append(argval.force_box())
+            arglist.append(argval.get_key_box())
         newop = ResOperation(rop.CALL_RELEASE_GIL, arglist, op.result,
                              descr=funcinfo.descr)
         self.commit_optimization()
diff --git a/pypy/jit/metainterp/optimizeopt/heap.py b/pypy/jit/metainterp/optimizeopt/heap.py
--- a/pypy/jit/metainterp/optimizeopt/heap.py
+++ b/pypy/jit/metainterp/optimizeopt/heap.py
@@ -50,6 +50,7 @@
             if not self._lazy_setfield_registered:
                 optheap._lazy_setfields_and_arrayitems.append(self)
                 self._lazy_setfield_registered = True
+            
         else:
             # this is the case where the pending setfield ends up
             # storing precisely the value that is already there,
@@ -158,12 +159,17 @@
         self._lazy_setfields_and_arrayitems = []
         self._remove_guard_not_invalidated = False
         self._seen_guard_not_invalidated = False
+        self.posponedop = None
 
     def force_at_end_of_preamble(self):
         self.force_all_lazy_setfields_and_arrayitems()
 
     def flush(self):
         self.force_all_lazy_setfields_and_arrayitems()
+        if self.posponedop:
+            posponedop = self.posponedop
+            self.posponedop = None
+            self.next_optimization.propagate_forward(posponedop)
 
     def new(self):
         return OptHeap()
@@ -211,7 +217,15 @@
 
     def emit_operation(self, op):
         self.emitting_operation(op)
-        self.next_optimization.propagate_forward(op)
+        if self.posponedop:
+            posponedop = self.posponedop
+            self.posponedop = None
+            self.next_optimization.propagate_forward(posponedop)
+        if (op.is_comparison() or op.getopnum() == rop.CALL_MAY_FORCE
+            or op.is_ovf()):
+            self.posponedop = op
+        else:
+            self.next_optimization.propagate_forward(op)
 
     def emitting_operation(self, op):
         if op.has_no_side_effect():
@@ -293,30 +307,6 @@
             if indexvalue is None or indexvalue.intbound.contains(idx):
                 cf.force_lazy_setfield(self, can_cache)
 
-    def fixup_guard_situation(self):
-        # hackish: reverse the order of the last two operations if it makes
-        # sense to avoid a situation like "int_eq/setfield_gc/guard_true",
-        # which the backend (at least the x86 backend) does not handle well.
-        newoperations = self.optimizer.newoperations
-        if len(newoperations) < 2:
-            return
-        lastop = newoperations[-1]
-        if (lastop.getopnum() != rop.SETFIELD_GC and
-            lastop.getopnum() != rop.SETARRAYITEM_GC):
-            return
-        # - is_comparison() for cases like "int_eq/setfield_gc/guard_true"
-        # - CALL_MAY_FORCE: "call_may_force/setfield_gc/guard_not_forced"
-        # - is_ovf(): "int_add_ovf/setfield_gc/guard_no_overflow"
-        prevop = newoperations[-2]
-        opnum = prevop.getopnum()
-        if not (prevop.is_comparison() or opnum == rop.CALL_MAY_FORCE
-                or prevop.is_ovf()):
-            return
-        if prevop.result in lastop.getarglist():
-            return
-        newoperations[-2] = lastop
-        newoperations[-1] = prevop
-
     def _assert_valid_cf(self, cf):
         # check that 'cf' is in cached_fields or cached_arrayitems
         if not we_are_translated():
@@ -362,7 +352,6 @@
                                       fieldvalue.get_key_box(), itemindex))
             else:
                 cf.force_lazy_setfield(self)
-                self.fixup_guard_situation()
         return pendingfields
 
     def optimize_GETFIELD_GC(self, op):
@@ -374,12 +363,22 @@
             return
         # default case: produce the operation
         structvalue.ensure_nonnull()
-        ###self.optimizer.optimize_default(op)
         self.emit_operation(op)
         # then remember the result of reading the field
         fieldvalue = self.getvalue(op.result)
         cf.remember_field_value(structvalue, fieldvalue, op)
 
+    def optimize_GETFIELD_GC_PURE(self, op):
+        structvalue = self.getvalue(op.getarg(0))
+        cf = self.field_cache(op.getdescr())
+        fieldvalue = cf.getfield_from_cache(self, structvalue)
+        if fieldvalue is not None:
+            self.make_equal_to(op.result, fieldvalue)
+            return
+        # default case: produce the operation
+        structvalue.ensure_nonnull()
+        self.emit_operation(op)
+
     def optimize_SETFIELD_GC(self, op):
         if self.has_pure_result(rop.GETFIELD_GC_PURE, [op.getarg(0)],
                                 op.getdescr()):
@@ -389,6 +388,7 @@
         #
         cf = self.field_cache(op.getdescr())
         cf.do_setfield(self, op)
+        
 
     def optimize_GETARRAYITEM_GC(self, op):
         arrayvalue = self.getvalue(op.getarg(0))
@@ -413,6 +413,25 @@
             fieldvalue = self.getvalue(op.result)
             cf.remember_field_value(arrayvalue, fieldvalue, op)
 
+    def optimize_GETARRAYITEM_GC_PURE(self, op):
+        arrayvalue = self.getvalue(op.getarg(0))
+        indexvalue = self.getvalue(op.getarg(1))
+        cf = None
+        if indexvalue.is_constant():
+            arrayvalue.make_len_gt(MODE_ARRAY, op.getdescr(), indexvalue.box.getint())
+            # use the cache on (arraydescr, index), which is a constant
+            cf = self.arrayitem_cache(op.getdescr(), indexvalue.box.getint())
+            fieldvalue = cf.getfield_from_cache(self, arrayvalue)
+            if fieldvalue is not None:
+                self.make_equal_to(op.result, fieldvalue)
+                return
+        else:
+            # variable index, so make sure the lazy setarrayitems are done
+            self.force_lazy_setarrayitem(op.getdescr(), indexvalue=indexvalue)
+        # default case: produce the operation
+        arrayvalue.ensure_nonnull()
+        self.emit_operation(op)
+
     def optimize_SETARRAYITEM_GC(self, op):
         if self.has_pure_result(rop.GETARRAYITEM_GC_PURE, [op.getarg(0),
                                                            op.getarg(1)],
diff --git a/pypy/jit/metainterp/optimizeopt/optimizer.py b/pypy/jit/metainterp/optimizeopt/optimizer.py
--- a/pypy/jit/metainterp/optimizeopt/optimizer.py
+++ b/pypy/jit/metainterp/optimizeopt/optimizer.py
@@ -31,8 +31,8 @@
 
 class OptValue(object):
     __metaclass__ = extendabletype
-    _attrs_ = ('box', 'known_class', 'last_guard_index', 'level', 'intbound', 'lenbound')
-    last_guard_index = -1
+    _attrs_ = ('box', 'known_class', 'last_guard', 'level', 'intbound', 'lenbound')
+    last_guard = None
 
     level = LEVEL_UNKNOWN
     known_class = None
@@ -100,7 +100,7 @@
             self.make_constant(other.get_key_box())
             optimizer.turned_constant(self)
         elif other.level == LEVEL_KNOWNCLASS:
-            self.make_constant_class(other.known_class, -1)
+            self.make_constant_class(other.known_class, None)
         else:
             if other.level == LEVEL_NONNULL:
                 self.ensure_nonnull()
@@ -114,13 +114,13 @@
                     self.lenbound = other.lenbound.clone()
 
 
-    def force_box(self):
+    def force_box(self, optforce):
         return self.box
 
     def get_key_box(self):
         return self.box
 
-    def force_at_end_of_preamble(self, already_forced):
+    def force_at_end_of_preamble(self, already_forced, optforce):
         return self
 
     def get_args_for_fail(self, modifier):
@@ -162,16 +162,16 @@
         else:
             return None
 
-    def make_constant_class(self, classbox, opindex):
+    def make_constant_class(self, classbox, guardop):
         assert self.level < LEVEL_KNOWNCLASS
         self.known_class = classbox
         self.level = LEVEL_KNOWNCLASS
-        self.last_guard_index = opindex
+        self.last_guard = guardop
 
-    def make_nonnull(self, opindex):
+    def make_nonnull(self, guardop):
         assert self.level < LEVEL_NONNULL
         self.level = LEVEL_NONNULL
-        self.last_guard_index = opindex
+        self.last_guard = guardop
 
     def is_nonnull(self):
         level = self.level
@@ -240,26 +240,12 @@
     def __init__(self):
         pass # make rpython happy
 
-    def propagate_begin_forward(self):
-        if self.next_optimization:
-            self.next_optimization.propagate_begin_forward()
-
-    def propagate_end_forward(self):
-        if self.next_optimization:
-            self.next_optimization.propagate_end_forward()
-
     def propagate_forward(self, op):
         raise NotImplementedError
 
     def emit_operation(self, op):
         self.next_optimization.propagate_forward(op)
 
-    def test_emittable(self, op):
-        return self.is_emittable(op)
-
-    def is_emittable(self, op):
-        return self.next_optimization.test_emittable(op)
-
     # FIXME: Move some of these here?
     def getvalue(self, box):
         return self.optimizer.getvalue(box)
@@ -289,19 +275,19 @@
         return self.optimizer.new_const_item(arraydescr)
 
     def pure(self, opnum, args, result):
-        op = ResOperation(opnum, args, result)
-        key = self.optimizer.make_args_key(op)
-        if key not in self.optimizer.pure_operations:
-            self.optimizer.pure_operations[key] = op
+        if self.optimizer.optpure:
+            self.optimizer.optpure.pure(opnum, args, result)
 
     def has_pure_result(self, opnum, args, descr):
-        op = ResOperation(opnum, args, None, descr)
-        key = self.optimizer.make_args_key(op)
-        op = self.optimizer.pure_operations.get(key, None)
-        if op is None:
-            return False
-        return op.getdescr() is descr
+        if self.optimizer.optpure:
+            return self.optimizer.optpure.has_pure_result(opnum, args, descr)
+        return False
 
+    def get_pure_result(self, key):    
+        if self.optimizer.optpure:
+            return self.optimizer.optpure.get_pure_result(key)
+        return None
+    
     def setup(self):
         pass
 
@@ -322,6 +308,9 @@
     def produce_potential_short_preamble_ops(self, potential_ops):
         pass
 
+    def forget_numberings(self, box):
+        self.optimizer.forget_numberings(box)
+
 class Optimizer(Optimization):
 
     def __init__(self, metainterp_sd, loop, optimizations=None, bridge=False):
@@ -333,14 +322,16 @@
         self.interned_refs = self.cpu.ts.new_ref_dict()
         self.resumedata_memo = resume.ResumeDataLoopMemo(metainterp_sd)
         self.bool_boxes = {}
-        self.pure_operations = args_dict()
         self.producer = {}
         self.pendingfields = []
-        self.posponedop = None
         self.exception_might_have_happened = False
         self.quasi_immutable_deps = None
         self.opaque_pointers = {}
-        self.newoperations = []
+        self.replaces_guard = {}
+        self._newoperations = []
+        self.optimizer = self
+        self.optpure = None
+        self.optearlyforce = None
         if loop is not None:
             self.call_pure_results = loop.call_pure_results
 
@@ -369,21 +360,20 @@
     def flush(self):
         for o in self.optimizations:
             o.flush()
-        assert self.posponedop is None
 
     def new(self):
         new = Optimizer(self.metainterp_sd, self.loop)
         return self._new(new)
 
     def _new(self, new):
-        assert self.posponedop is None
         optimizations = [o.new() for o in self.optimizations]
         new.set_optimizations(optimizations)
         new.quasi_immutable_deps = self.quasi_immutable_deps
         return new
 
     def produce_potential_short_preamble_ops(self, sb):
-        raise NotImplementedError('This is implemented in unroll.UnrollableOptimizer')
+        for opt in self.optimizations:
+            opt.produce_potential_short_preamble_ops(sb)
 
     def turned_constant(self, value):
         for o in self.optimizations:
@@ -433,6 +423,13 @@
             return constbox
         return None
 
+    def get_newoperations(self):
+        self.flush()
+        return self._newoperations
+
+    def clear_newoperations(self):
+        self._newoperations = []
+
     def make_equal_to(self, box, value, replace=False):
         assert isinstance(value, OptValue)
         assert replace or box not in self.values
@@ -481,15 +478,10 @@
 
     def propagate_all_forward(self):
         self.exception_might_have_happened = self.bridge
-        self.newoperations = []
-        self.first_optimization.propagate_begin_forward()
-        self.i = 0
-        while self.i < len(self.loop.operations):
-            op = self.loop.operations[self.i]
+        self.clear_newoperations()
+        for op in self.loop.operations:
             self.first_optimization.propagate_forward(op)
-            self.i += 1
-        self.first_optimization.propagate_end_forward()
-        self.loop.operations = self.newoperations
+        self.loop.operations = self.get_newoperations()
         self.loop.quasi_immutable_deps = self.quasi_immutable_deps
         # accumulate counters
         self.resumedata_memo.update_counters(self.metainterp_sd.profiler)
@@ -501,9 +493,6 @@
         self.producer[op.result] = op
         dispatch_opt(self, op)
 
-    def test_emittable(self, op):
-        return True
-
     def emit_operation(self, op):
         if op.returns_bool_result():
             self.bool_boxes[self.getvalue(op.result)] = None
@@ -519,14 +508,30 @@
                 pass
             else:
                 self.ensure_imported(value)
-                op.setarg(i, value.force_box())
+                op.setarg(i, value.force_box(self))
         self.metainterp_sd.profiler.count(jitprof.OPT_OPS)
         if op.is_guard():
             self.metainterp_sd.profiler.count(jitprof.OPT_GUARDS)
-            op = self.store_final_boxes_in_guard(op)
+            if self.replaces_guard and op in self.replaces_guard:
+                self.replace_op(self.replaces_guard[op], op)
+                del self.replaces_guard[op]
+                return
+            else:
+                op = self.store_final_boxes_in_guard(op)
         elif op.can_raise():
             self.exception_might_have_happened = True
-        self.newoperations.append(op)
+        self._newoperations.append(op)
+
+    def replace_op(self, old_op, new_op):
+        # XXX: Do we want to cache indexes to prevent search?
+        i = len(self._newoperations) 
+        while i > 0:
+            i -= 1
+            if self._newoperations[i] is old_op:
+                self._newoperations[i] = new_op
+                break
+        else:
+            assert False
 
     def store_final_boxes_in_guard(self, op):
         descr = op.getdescr()
@@ -574,51 +579,8 @@
         args[n + 1] = op.getdescr()
         return args
 
-    @specialize.argtype(0)
     def optimize_default(self, op):
-        canfold = op.is_always_pure()
-        if op.is_ovf():
-            self.posponedop = op
-            return
-        if self.posponedop:
-            nextop = op
-            op = self.posponedop
-            self.posponedop = None
-            canfold = nextop.getopnum() == rop.GUARD_NO_OVERFLOW
-        else:
-            nextop = None
-
-        if canfold:
-            for i in range(op.numargs()):
-                if self.get_constant_box(op.getarg(i)) is None:
-                    break
-            else:
-                # all constant arguments: constant-fold away
-                resbox = self.constant_fold(op)
-                # note that INT_xxx_OVF is not done from here, and the
-                # overflows in the INT_xxx operations are ignored
-                self.make_constant(op.result, resbox)
-                return
-
-            # did we do the exact same operation already?
-            args = self.make_args_key(op)
-            oldop = self.pure_operations.get(args, None)
-            if oldop is not None and oldop.getdescr() is op.getdescr():
-                assert oldop.getopnum() == op.getopnum()
-                self.make_equal_to(op.result, self.getvalue(oldop.result),
-                                   True)
-                return
-            else:
-                self.pure_operations[args] = op
-                self.remember_emitting_pure(op)
-
-        # otherwise, the operation remains
         self.emit_operation(op)
-        if nextop:
-            self.emit_operation(nextop)
-
-    def remember_emitting_pure(self, op):
-        pass
 
     def constant_fold(self, op):
         argboxes = [self.get_constant_box(op.getarg(i))
@@ -636,11 +598,6 @@
     def optimize_DEBUG_MERGE_POINT(self, op):
         self.emit_operation(op)
 
-    def optimize_CAST_OPAQUE_PTR(self, op):
-        value = self.getvalue(op.getarg(0))
-        self.opaque_pointers[value] = True
-        self.make_equal_to(op.result, value)
-
     def optimize_GETARRAYITEM_GC_PURE(self, op):
         indexvalue = self.getvalue(op.getarg(1))
         if indexvalue.is_constant():
diff --git a/pypy/jit/metainterp/optimizeopt/pure.py b/pypy/jit/metainterp/optimizeopt/pure.py
new file mode 100644
--- /dev/null
+++ b/pypy/jit/metainterp/optimizeopt/pure.py
@@ -0,0 +1,115 @@
+from pypy.jit.metainterp.optimizeopt.optimizer import Optimization
+from pypy.jit.metainterp.resoperation import rop, ResOperation
+from pypy.jit.metainterp.optimizeopt.util import (make_dispatcher_method,
+    args_dict)
+
+class OptPure(Optimization):
+    def __init__(self):
+        self.posponedop = None
+        self.pure_operations = args_dict()
+        self.emitted_pure_operations = {}
+
+    def propagate_forward(self, op):
+        dispatch_opt(self, op)
+
+    def optimize_default(self, op):
+        canfold = op.is_always_pure()
+        if op.is_ovf():
+            self.posponedop = op
+            return
+        if self.posponedop:
+            nextop = op
+            op = self.posponedop
+            self.posponedop = None
+            canfold = nextop.getopnum() == rop.GUARD_NO_OVERFLOW
+        else:
+            nextop = None
+
+        if canfold:
+            for i in range(op.numargs()):
+                if self.get_constant_box(op.getarg(i)) is None:
+                    break
+            else:
+                # all constant arguments: constant-fold away
+                resbox = self.optimizer.constant_fold(op)
+                # note that INT_xxx_OVF is not done from here, and the
+                # overflows in the INT_xxx operations are ignored
+                self.optimizer.make_constant(op.result, resbox)
+                return
+
+            # did we do the exact same operation already?
+            args = self.optimizer.make_args_key(op)
+            oldop = self.pure_operations.get(args, None)
+            if oldop is not None and oldop.getdescr() is op.getdescr():
+                assert oldop.getopnum() == op.getopnum()
+                self.optimizer.make_equal_to(op.result, self.getvalue(oldop.result),
+                                   True)
+                return
+            else:
+                self.pure_operations[args] = op
+                self.remember_emitting_pure(op)
+
+        # otherwise, the operation remains
+        self.emit_operation(op)
+        if op.returns_bool_result():
+            self.optimizer.bool_boxes[self.getvalue(op.result)] = None        
+        if nextop:
+            self.emit_operation(nextop)
+
+    def optimize_CALL_PURE(self, op):
+        args = self.optimizer.make_args_key(op)
+        oldop = self.pure_operations.get(args, None)
+        if oldop is not None and oldop.getdescr() is op.getdescr():
+            assert oldop.getopnum() == op.getopnum()
+            self.make_equal_to(op.result, self.getvalue(oldop.result))
+            return
+        else:
+            self.pure_operations[args] = op
+            self.remember_emitting_pure(op)
+
+        # replace CALL_PURE with just CALL
+        args = op.getarglist()
+        self.emit_operation(ResOperation(rop.CALL, args, op.result,
+                                         op.getdescr()))
+
+    def flush(self):
+        assert self.posponedop is None
+
+    def new(self):
+        assert self.posponedop is None
+        return OptPure()
+
+    def setup(self):
+        self.optimizer.optpure = self
+
+    def pure(self, opnum, args, result):
+        op = ResOperation(opnum, args, result)
+        key = self.optimizer.make_args_key(op)
+        if key not in self.pure_operations:
+            self.pure_operations[key] = op
+
+    def has_pure_result(self, opnum, args, descr):
+        op = ResOperation(opnum, args, None, descr)
+        key = self.optimizer.make_args_key(op)
+        op = self.pure_operations.get(key, None)
+        if op is None:
+            return False
+        return op.getdescr() is descr
+
+    def get_pure_result(self, key):
+        return self.pure_operations.get(key, None)
+
+    def remember_emitting_pure(self, op):
+        self.emitted_pure_operations[op] = True
+
+    def produce_potential_short_preamble_ops(self, sb):
+        for op in self.emitted_pure_operations:
+            if op.getopnum() == rop.GETARRAYITEM_GC_PURE or \
+               op.getopnum() == rop.STRGETITEM or \
+               op.getopnum() == rop.UNICODEGETITEM:
+                if not self.getvalue(op.getarg(1)).is_constant():
+                    continue
+            sb.add_potential(op)
+
+dispatch_opt = make_dispatcher_method(OptPure, 'optimize_',
+                                      default=OptPure.optimize_default)
diff --git a/pypy/jit/metainterp/optimizeopt/rewrite.py b/pypy/jit/metainterp/optimizeopt/rewrite.py
--- a/pypy/jit/metainterp/optimizeopt/rewrite.py
+++ b/pypy/jit/metainterp/optimizeopt/rewrite.py
@@ -31,21 +31,8 @@
 
         dispatch_opt(self, op)
 
-    def test_emittable(self, op):
-        opnum = op.getopnum()
-        for value, cls, func in optimize_guards:
-            if opnum == value:
-                assert isinstance(op, cls)
-                try:
-                    func(self, op, dryrun=True)
-                    return self.is_emittable(op)
-                except InvalidLoop:
-                    return False
-        return self.is_emittable(op)
-
-
     def try_boolinvers(self, op, targs):
-        oldop = self.optimizer.pure_operations.get(targs, None)
+        oldop = self.get_pure_result(targs)
         if oldop is not None and oldop.getdescr() is op.getdescr():
             value = self.getvalue(oldop.result)
             if value.is_constant():
@@ -73,7 +60,7 @@
             oldopnum = opboolreflex[op.getopnum()] # FIXME: add INT_ADD, INT_MUL
             targs = self.optimizer.make_args_key(ResOperation(oldopnum, [args[1], args[0]],
                                                               None))
-            oldop = self.optimizer.pure_operations.get(targs, None)
+            oldop = self.get_pure_result(targs)
             if oldop is not None and oldop.getdescr() is op.getdescr():
                 self.make_equal_to(op.result, self.getvalue(oldop.result))
                 return True
@@ -214,40 +201,7 @@
         self.emit_operation(op)
         self.pure(rop.FLOAT_NEG, [op.result], v1)
 
-    def optimize_CALL_PURE(self, op):
-        arg_consts = []
-        for i in range(op.numargs()):
-            arg = op.getarg(i)
-            const = self.get_constant_box(arg)
-            if const is None:
-                break
-            arg_consts.append(const)
-        else:
-            # all constant arguments: check if we already know the result
-            try:
-                result = self.optimizer.call_pure_results[arg_consts]
-            except KeyError:
-                pass
-            else:
-                self.make_constant(op.result, result)
-                return
-
-        args = self.optimizer.make_args_key(op)
-        oldop = self.optimizer.pure_operations.get(args, None)
-        if oldop is not None and oldop.getdescr() is op.getdescr():
-            assert oldop.getopnum() == op.getopnum()
-            self.make_equal_to(op.result, self.getvalue(oldop.result))
-            return
-        else:
-            self.optimizer.pure_operations[args] = op
-            self.optimizer.remember_emitting_pure(op)
-
-        # replace CALL_PURE with just CALL
-        args = op.getarglist()
-        self.emit_operation(ResOperation(rop.CALL, args, op.result,
-                                         op.getdescr()))
-
-    def optimize_guard(self, op, constbox, emit_operation=True, dryrun=False):
+    def optimize_guard(self, op, constbox, emit_operation=True):
         value = self.getvalue(op.getarg(0))
         if value.is_constant():
             box = value.box
@@ -255,62 +209,57 @@
             if not box.same_constant(constbox):
                 raise InvalidLoop
             return
-        if dryrun: return
         if emit_operation:
             self.emit_operation(op)
         value.make_constant(constbox)
         self.optimizer.turned_constant(value)
 
-    def optimize_GUARD_ISNULL(self, op, dryrun=False):
+    def optimize_GUARD_ISNULL(self, op):
         value = self.getvalue(op.getarg(0))
         if value.is_null():
             return
         elif value.is_nonnull():
             raise InvalidLoop
-        if dryrun: return
         self.emit_operation(op)
         value.make_constant(self.optimizer.cpu.ts.CONST_NULL)
 
-    def optimize_GUARD_NONNULL(self, op, dryrun=False):
+    def optimize_GUARD_NONNULL(self, op):
         value = self.getvalue(op.getarg(0))
         if value.is_nonnull():
             return
         elif value.is_null():
             raise InvalidLoop
-        if dryrun: return
         self.emit_operation(op)
-        value.make_nonnull(len(self.optimizer.newoperations) - 1)
+        value.make_nonnull(op)
 
-    def optimize_GUARD_VALUE(self, op, dryrun=False):
+    def optimize_GUARD_VALUE(self, op):
         value = self.getvalue(op.getarg(0))
-        emit_operation = True
-        if not dryrun and value.last_guard_index != -1:
+        if value.last_guard:
             # there already has been a guard_nonnull or guard_class or
             # guard_nonnull_class on this value, which is rather silly.
             # replace the original guard with a guard_value
-            old_guard_op = self.optimizer.newoperations[value.last_guard_index]
-            new_guard_op = old_guard_op.copy_and_change(rop.GUARD_VALUE,
-                                             args = [old_guard_op.getarg(0), op.getarg(1)])
-            self.optimizer.newoperations[value.last_guard_index] = new_guard_op
+            old_guard_op = value.last_guard
+            op = old_guard_op.copy_and_change(rop.GUARD_VALUE,
+                                      args = [old_guard_op.getarg(0), op.getarg(1)])
+            self.optimizer.replaces_guard[op] = old_guard_op
             # hack hack hack.  Change the guard_opnum on
             # new_guard_op.getdescr() so that when resuming,
             # the operation is not skipped by pyjitpl.py.
-            descr = new_guard_op.getdescr()
+            descr = op.getdescr()
             assert isinstance(descr, compile.ResumeGuardDescr)
             descr.guard_opnum = rop.GUARD_VALUE
-            descr.make_a_counter_per_value(new_guard_op)
-            emit_operation = False
+            descr.make_a_counter_per_value(op)
         constbox = op.getarg(1)
         assert isinstance(constbox, Const)
-        self.optimize_guard(op, constbox, emit_operation, dryrun)
+        self.optimize_guard(op, constbox)
 
-    def optimize_GUARD_TRUE(self, op, dryrun=False):
-        self.optimize_guard(op, CONST_1, dryrun=dryrun)
+    def optimize_GUARD_TRUE(self, op):
+        self.optimize_guard(op, CONST_1)
 
-    def optimize_GUARD_FALSE(self, op, dryrun=False):
-        self.optimize_guard(op, CONST_0, dryrun=dryrun)
+    def optimize_GUARD_FALSE(self, op):
+        self.optimize_guard(op, CONST_0)
 
-    def optimize_GUARD_CLASS(self, op, dryrun=False):
+    def optimize_GUARD_CLASS(self, op):
         value = self.getvalue(op.getarg(0))
         expectedclassbox = op.getarg(1)
         assert isinstance(expectedclassbox, Const)
@@ -319,38 +268,32 @@
             if realclassbox.same_constant(expectedclassbox):
                 return
             raise InvalidLoop
-        if dryrun: return
-        emit_operation = True
-        if value.last_guard_index != -1:
+        if value.last_guard:
             # there already has been a guard_nonnull or guard_class or
             # guard_nonnull_class on this value.
-            old_guard_op = self.optimizer.newoperations[value.last_guard_index]
+            old_guard_op = value.last_guard
             if old_guard_op.getopnum() == rop.GUARD_NONNULL:
                 # it was a guard_nonnull, which we replace with a
                 # guard_nonnull_class.
-                new_guard_op = old_guard_op.copy_and_change (rop.GUARD_NONNULL_CLASS,
+                op = old_guard_op.copy_and_change (rop.GUARD_NONNULL_CLASS,
                                          args = [old_guard_op.getarg(0), op.getarg(1)])
-                self.optimizer.newoperations[value.last_guard_index] = new_guard_op
+                self.optimizer.replaces_guard[op] = old_guard_op
                 # hack hack hack.  Change the guard_opnum on
                 # new_guard_op.getdescr() so that when resuming,
                 # the operation is not skipped by pyjitpl.py.
-                descr = new_guard_op.getdescr()
+                descr = op.getdescr()
                 assert isinstance(descr, compile.ResumeGuardDescr)
                 descr.guard_opnum = rop.GUARD_NONNULL_CLASS
-                emit_operation = False
-        if emit_operation:
-            self.emit_operation(op)
-            last_guard_index = len(self.optimizer.newoperations) - 1
-        else:
-            last_guard_index = value.last_guard_index
-        value.make_constant_class(expectedclassbox, last_guard_index)
+        self.emit_operation(op)
+        value.make_constant_class(expectedclassbox, op)
 
-    def optimize_GUARD_NONNULL_CLASS(self, op, dryrun=False):
-        self.optimize_GUARD_NONNULL(op, True)
-        self.optimize_GUARD_CLASS(op, dryrun)
+    def optimize_GUARD_NONNULL_CLASS(self, op):
+        value = self.getvalue(op.getarg(0))
+        if value.is_null():
+            raise InvalidLoop
+        self.optimize_GUARD_CLASS(op)
 
-    def optimize_GUARD_NO_EXCEPTION(self, op, dryrun=False):
-        if dryrun: return
+    def optimize_GUARD_NO_EXCEPTION(self, op):
         if not self.optimizer.exception_might_have_happened:
             return
         self.emit_operation(op)
@@ -470,7 +413,7 @@
                     newop = ResOperation(rop.SETARRAYITEM_GC,
                                          [op.getarg(2),
                                           ConstInt(index + dest_start),
-                                          val.force_box()], None,
+                                          val.get_key_box()], None,
                                          descr=source_value.arraydescr)
                     self.emit_operation(newop)
             return True
@@ -478,6 +421,25 @@
             return True # 0-length arraycopy
         return False
 
+    def optimize_CALL_PURE(self, op):
+        arg_consts = []
+        for i in range(op.numargs()):
+            arg = op.getarg(i)
+            const = self.get_constant_box(arg)
+            if const is None:
+                break
+            arg_consts.append(const)
+        else:
+            # all constant arguments: check if we already know the result
+            try:
+                result = self.optimizer.call_pure_results[arg_consts]
+            except KeyError:
+                pass
+            else:
+                self.make_constant(op.result, result)
+                return
+        self.emit_operation(op)
+
     def optimize_INT_FLOORDIV(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
@@ -492,6 +454,10 @@
                                         args = [op.getarg(0), ConstInt(highest_bit(val))])
         self.emit_operation(op)
 
+    def optimize_CAST_OPAQUE_PTR(self, op):
+        value = self.getvalue(op.getarg(0))
+        self.optimizer.opaque_pointers[value] = True
+        self.make_equal_to(op.result, value)
 
 dispatch_opt = make_dispatcher_method(OptRewrite, 'optimize_',
         default=OptRewrite.emit_operation)
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -39,8 +39,9 @@
 
 def test_sharing_field_lists_of_virtual():
     class FakeOptimizer(object):
-        class cpu(object):
-            pass
+        class optimizer(object):
+            class cpu(object):
+                pass
     opt = FakeOptimizer()
     virt1 = virtualize.AbstractVirtualStructValue(opt, None)
     lst1 = virt1._get_field_descr_list()
@@ -69,7 +70,7 @@
     class FakeVirtualValue(virtualize.AbstractVirtualValue):
         def _make_virtual(self, *args):
             return FakeVInfo()
-    v1 = FakeVirtualValue(None, None, None)
+    v1 = FakeVirtualValue(None, None)
     vinfo1 = v1.make_virtual_info(None, [1, 2, 4])
     vinfo2 = v1.make_virtual_info(None, [1, 2, 4])
     assert vinfo1 is vinfo2
@@ -111,7 +112,7 @@
 
 class BaseTestBasic(BaseTest):
 
-    enable_opts = "intbounds:rewrite:virtualize:string:heap"
+    enable_opts = "intbounds:rewrite:virtualize:string:earlyforce:pure:heap"
 
     def optimize_loop(self, ops, optops, call_pure_results=None):
 
@@ -651,8 +652,8 @@
         i3 = getfield_gc(p3, descr=valuedescr)
         escape(i3)
         p1 = new_with_vtable(ConstClass(node_vtable))
+        p1sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p1, i1, descr=valuedescr)
-        p1sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p1sub, i1, descr=valuedescr)
         setfield_gc(p1, p1sub, descr=nextdescr)
         jump(i1, p1, p2)
@@ -667,10 +668,10 @@
         p3sub = getfield_gc(p3, descr=nextdescr)
         i3 = getfield_gc(p3sub, descr=valuedescr)
         escape(i3)
+        p1 = new_with_vtable(ConstClass(node_vtable))
         p2sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p2sub, i1, descr=valuedescr)
         setfield_gc(p2, p2sub, descr=nextdescr)
-        p1 = new_with_vtable(ConstClass(node_vtable))
         jump(i1, p1, p2)
         """
         # The same as test_p123_simple, but in the end the "old" p2 contains
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizefficall.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizefficall.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizefficall.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizefficall.py
@@ -36,7 +36,7 @@
 
 class TestFfiCall(BaseTestBasic, LLtypeMixin):
 
-    enable_opts = "intbounds:rewrite:virtualize:string:heap:ffi"
+    enable_opts = "intbounds:rewrite:virtualize:string:pure:earlyforce:heap:ffi"
 
     class namespace:
         cpu = LLtypeMixin.cpu
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -68,7 +68,7 @@
 
 class BaseTestWithUnroll(BaseTest):
 
-    enable_opts = "intbounds:rewrite:virtualize:string:heap:unroll"
+    enable_opts = "intbounds:rewrite:virtualize:string:earlyforce:pure:heap:unroll"
 
     def optimize_loop(self, ops, expected, expected_preamble=None,
                       call_pure_results=None, expected_short=None):
@@ -825,8 +825,8 @@
         i3 = getfield_gc(p2, descr=valuedescr)
         escape(i3)
         p4 = new_with_vtable(ConstClass(node_vtable))
+        p1sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p4, i1, descr=valuedescr)
-        p1sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p1sub, i1, descr=valuedescr)
         setfield_gc(p4, p1sub, descr=nextdescr)
         jump(i1, p4)
@@ -865,13 +865,7 @@
         p3sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p3sub, i1, descr=valuedescr)
         setfield_gc(p1, p3sub, descr=nextdescr)
-        # XXX: We get two extra operations here because the setfield
-        #      above is the result of forcing p1 and thus not
-        #      registered with the heap optimizer. I've makred tests
-        #      below with VIRTUALHEAP if they suffer from this issue
-        p3sub2 = getfield_gc(p1, descr=nextdescr)
-        guard_nonnull_class(p3sub2, ConstClass(node_vtable2)) []
-        jump(i1, p1, p3sub2)
+        jump(i1, p1, p3sub)
         """
         self.optimize_loop(ops, expected, preamble)
 
@@ -902,9 +896,7 @@
         guard_true(i2b) []
         p3 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3, i2, descr=nextdescr)
-        # XXX: VIRTUALHEAP (see above)
-        i3 = getfield_gc(p3, descr=nextdescr)
-        jump(p3, i3)
+        jump(p3, i2)
         """
         self.optimize_loop(ops, expected, preamble)
 
@@ -1219,7 +1211,15 @@
         setfield_gc(p3, p30, descr=valuedescr)
         jump(i29, p30, p3)
         """
-        expected = preamble
+        expected = """
+        [i0, p1, p3]
+        i28 = int_add(i0, 1)
+        i29 = int_add(i28, 1)
+        p30 = new_with_vtable(ConstClass(node_vtable))
+        setfield_gc(p3, p30, descr=valuedescr)
+        setfield_gc(p30, i28, descr=nextdescr)
+        jump(i29, p30, p3)
+        """
         self.optimize_loop(ops, expected, preamble)
 
     def test_nonvirtual_1(self):
@@ -2408,8 +2408,8 @@
         guard_class(p2, ConstClass(node_vtable)) []
         p3 = getfield_gc(p1, descr=otherdescr)
         guard_class(p3, ConstClass(node_vtable)) []
+        p3a = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3, p2, descr=otherdescr)
-        p3a = new_with_vtable(ConstClass(node_vtable))
         escape(p3a)
         jump(p3a)
         """
@@ -2421,9 +2421,9 @@
         # setfield_gc(p3, p2, descr=otherdescr) # p3a.other = p2a
         # p1a = new_with_vtable(ConstClass(node_vtable2))
         # p2a = new_with_vtable(ConstClass(node_vtable))
+        p3anew = new_with_vtable(ConstClass(node_vtable))
         p2 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3a, p2, descr=otherdescr) # p3a.other = p2a
-        p3anew = new_with_vtable(ConstClass(node_vtable))
         escape(p3anew)
         jump(p3anew)
         """
@@ -2458,9 +2458,9 @@
         p3 = getfield_gc(p1, descr=otherdescr)
         guard_class(p3, ConstClass(node_vtable)) []
         # p1a = new_with_vtable(ConstClass(node_vtable2))
+        p3a = new_with_vtable(ConstClass(node_vtable))
         p2a = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3, p2a, descr=otherdescr)
-        p3a = new_with_vtable(ConstClass(node_vtable))
         escape(p3a)
         # setfield_gc(p1a, p2a, descr=nextdescr)
         # setfield_gc(p1a, p3a, descr=otherdescr)
@@ -2468,9 +2468,9 @@
         """
         expected = """
         [p2, p3]
+        p3a = new_with_vtable(ConstClass(node_vtable))
         p2a = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3, p2a, descr=otherdescr)
-        p3a = new_with_vtable(ConstClass(node_vtable))
         escape(p3a)
         jump(p2a, p3a)
         """
@@ -2790,7 +2790,6 @@
         self.optimize_loop(ops, expected, preamble)
 
     def test_remove_duplicate_pure_op_ovf_with_lazy_setfield(self):
-        py.test.skip('this optimization is not yet supprted')
         ops = """
         [i1, p1]
         i3 = int_add_ovf(i1, 1)
@@ -5961,13 +5960,18 @@
         escape(i0)
         jump(p1)
         """
-        expected = """
+        preamble = """
         [p1]
         i0 = ptr_eq(p1, NULL)
         escape(i0)
-        jump(p1)
-        """
-        self.optimize_strunicode_loop_extradescrs(ops, expected, expected)
+        jump(p1, i0)
+        """
+        expected = """
+        [p1, i0]
+        escape(i0)
+        jump(p1, i0)
+        """
+        self.optimize_strunicode_loop_extradescrs(ops, expected, preamble)
 
     def test_str_equal_none2(self):
         ops = """
@@ -5976,13 +5980,18 @@
         escape(i0)
         jump(p1)
         """
-        expected = """
+        preamble = """
         [p1]
         i0 = ptr_eq(p1, NULL)
         escape(i0)
-        jump(p1)
-        """
-        self.optimize_strunicode_loop_extradescrs(ops, expected, expected)
+        jump(p1, i0)
+        """
+        expected = """
+        [p1, i0]
+        escape(i0)
+        jump(p1, i0)
+        """
+        self.optimize_strunicode_loop_extradescrs(ops, expected, preamble)
 
     def test_str_equal_nonnull1(self):
         ops = """
@@ -7220,6 +7229,29 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_heap_cache_forced_virtuals(self):
+        ops = """
+        [i1, i2, p0]
+        p1 = new(descr=ssize)
+        setfield_gc(p1, i1, descr=adescr)
+        setfield_gc(p1, i2, descr=bdescr)
+        call(p0, p1, descr=writeadescr)
+        i3 = getfield_gc(p1, descr=adescr)
+        i4 = getfield_gc(p1, descr=bdescr)
+        jump(i3, i4, p0)
+        """
+        expected = """
+        [i1, i2, p0]
+        p1 = new(descr=ssize)
+        setfield_gc(p1, i1, descr=adescr)
+        call(p0, p1, descr=writeadescr)
+        i3 = getfield_gc(p1, descr=adescr)
+        setfield_gc(p1, i2, descr=bdescr)
+        jump(i3, i2, p0)
+        """
+        self.optimize_loop(ops, expected)
+
+
     def test_setarrayitem_followed_by_arraycopy(self):
         ops = """
         [p1, p2]
@@ -7230,6 +7262,27 @@
         """
         self.optimize_loop(ops, ops)
 
+    def test_heap_cache_virtuals_forced_by_delayed_setfield(self):
+        py.test.skip('not yet supoprted')
+        ops = """
+        [i1, p0]
+        p1 = new(descr=ssize)
+        setfield_gc(p1, i1, descr=valuedescr)
+        setfield_gc(p0, p1, descr=adescr)
+        call(p0, descr=writeadescr)
+        i2 = getfield_gc(p1, descr=valuedescr)
+        jump(i2, p0)
+        """
+        expected = """
+        [i1, p0]
+        p1 = new(descr=ssize)
+        setfield_gc(p1, i1, descr=valuedescr)
+        setfield_gc(p0, p1, descr=adescr)
+        call(p0, descr=writeadescr)
+        jump(i1, p0)
+        """
+        self.optimize_loop(ops, expected)
+
 class TestLLtype(OptimizeOptTest, LLtypeMixin):
     pass
 
diff --git a/pypy/jit/metainterp/optimizeopt/unroll.py b/pypy/jit/metainterp/optimizeopt/unroll.py
--- a/pypy/jit/metainterp/optimizeopt/unroll.py
+++ b/pypy/jit/metainterp/optimizeopt/unroll.py
@@ -75,7 +75,6 @@
         self.importable_values = {}
         self.emitting_dissabled = False
         self.emitted_guards = 0
-        self.emitted_pure_operations = {}
 
     def ensure_imported(self, value):
         if not self.emitting_dissabled and value in self.importable_values:
@@ -96,21 +95,6 @@
         new = UnrollableOptimizer(self.metainterp_sd, self.loop)
         return self._new(new)
 
-    def remember_emitting_pure(self, op):
-        self.emitted_pure_operations[op] = True
-
-    def produce_potential_short_preamble_ops(self, sb):
-        for op in self.emitted_pure_operations:
-            if op.getopnum() == rop.GETARRAYITEM_GC_PURE or \
-               op.getopnum() == rop.STRGETITEM or \
-               op.getopnum() == rop.UNICODEGETITEM:
-                if not self.getvalue(op.getarg(1)).is_constant():
-                    continue
-            sb.add_potential(op)
-        for opt in self.optimizations:
-            opt.produce_potential_short_preamble_ops(sb)
-
-
 
 class UnrollOptimizer(Optimization):
     """Unroll the loop into two iterations. The first one will
@@ -154,7 +138,7 @@
 
             KillHugeIntBounds(self.optimizer).apply()
             
-            loop.preamble.operations = self.optimizer.newoperations
+            loop.preamble.operations = self.optimizer.get_newoperations()
             jump_args = [self.getvalue(a).get_key_box() for a in jump_args]
 
             start_resumedescr = loop.preamble.start_resumedescr.clone_if_mutable()
@@ -167,8 +151,9 @@
             virtual_state = modifier.get_virtual_state(jump_args)
             
             values = [self.getvalue(arg) for arg in jump_args]
-            inputargs = virtual_state.make_inputargs(values)
-            short_inputargs = virtual_state.make_inputargs(values, keyboxes=True)
+            inputargs = virtual_state.make_inputargs(values, self.optimizer)
+            short_inputargs = virtual_state.make_inputargs(values, self.optimizer,
+                                                           keyboxes=True)
 
             self.constant_inputargs = {}
             for box in jump_args: 
@@ -197,7 +182,7 @@
             # operations needed to setup the proper state of those virtuals
             # in the peeled loop
             inputarg_setup_ops = []
-            preamble_optimizer.newoperations = []
+            preamble_optimizer.clear_newoperations()
             seen = {}
             for box in inputargs:
                 if box in seen:
@@ -211,9 +196,8 @@
                     continue
                 seen[box] = True
                 value = preamble_optimizer.getvalue(box)
-                value.force_box()
-            preamble_optimizer.flush()
-            inputarg_setup_ops += preamble_optimizer.newoperations
+                value.force_box(preamble_optimizer)
+            inputarg_setup_ops += preamble_optimizer.get_newoperations()
 
             # Setup the state of the new optimizer by emiting the
             # short preamble operations and discarding the result
@@ -244,13 +228,13 @@
                                 virtual_state)
             
             loop.inputargs = inputargs
-            args = [preamble_optimizer.getvalue(self.short_boxes.original(a)).force_box()\
+            args = [preamble_optimizer.getvalue(self.short_boxes.original(a)).force_box(preamble_optimizer)\
                     for a in inputargs]
             jmp = ResOperation(rop.JUMP, args, None)
             jmp.setdescr(loop.token)
             loop.preamble.operations.append(jmp)
 
-            loop.operations = self.optimizer.newoperations
+            loop.operations = self.optimizer.get_newoperations()
             maxguards = self.optimizer.metainterp_sd.warmrunnerdesc.memory_manager.max_retrace_guards
             
             if self.optimizer.emitted_guards > maxguards:
@@ -335,9 +319,10 @@
         assert jumpop
         original_jumpargs = jumpop.getarglist()[:]
         values = [self.getvalue(arg) for arg in jumpop.getarglist()]
-        jumpargs = virtual_state.make_inputargs(values)
+        jumpargs = virtual_state.make_inputargs(values, self.optimizer)
         jumpop.initarglist(jumpargs)
-        jmp_to_short_args = virtual_state.make_inputargs(values, keyboxes=True)
+        jmp_to_short_args = virtual_state.make_inputargs(values, self.optimizer,
+                                                         keyboxes=True)
         self.short_inliner = Inliner(short_inputargs, jmp_to_short_args)
         
         for box, const in self.constant_inputargs.items():
@@ -347,11 +332,11 @@
             newop = self.short_inliner.inline_op(op)
             self.optimizer.send_extra_operation(newop)
         
-        self.optimizer.flush()
+        newoperations = self.optimizer.get_newoperations()
 
         i = j = 0
-        while i < len(self.optimizer.newoperations) or j < len(jumpargs):
-            if i == len(self.optimizer.newoperations):
+        while i < len(newoperations) or j < len(jumpargs):
+            if i == len(newoperations):
                 while j < len(jumpargs):
                     a = jumpargs[j]
                     if self.optimizer.loop.logops:
@@ -360,7 +345,7 @@
                                     jumpargs, short_seen)
                     j += 1
             else:
-                op = self.optimizer.newoperations[i]
+                op = newoperations[i]
 
                 self.boxes_created_this_iteration[op.result] = True
                 args = op.getarglist()
@@ -375,6 +360,7 @@
                     self.import_box(a, inputargs, short, short_jumpargs,
                                     jumpargs, short_seen)
                 i += 1
+            newoperations = self.optimizer.get_newoperations()
 
         jumpop.initarglist(jumpargs)
         self.optimizer.send_extra_operation(jumpop)
@@ -468,7 +454,7 @@
         inputargs.append(box)
         box = newresult
         if box in self.optimizer.values:
-            box = self.optimizer.values[box].force_box()
+            box = self.optimizer.values[box].force_box(self.optimizer)
         jumpargs.append(box)
         
 
@@ -483,11 +469,6 @@
         if op.getopnum() == rop.JUMP:
             loop_token = op.getdescr()
             assert isinstance(loop_token, LoopToken)
-            # FIXME: Use a tree, similar to the tree formed by the full
-            # preamble and it's bridges, instead of a list to save time and
-            # memory. This should also allow better behaviour in
-            # situations that the is_emittable() chain currently cant
-            # handle and the inlining fails unexpectedly belwo.
             short = loop_token.short_preamble
             if short:
                 args = op.getarglist()
@@ -523,7 +504,7 @@
 
                         values = [self.getvalue(arg)
                                   for arg in op.getarglist()]
-                        args = sh.virtual_state.make_inputargs(values,
+                        args = sh.virtual_state.make_inputargs(values, self.optimizer,
                                                                keyboxes=True)
                         inliner = Inliner(sh.inputargs, args)
                         
diff --git a/pypy/jit/metainterp/optimizeopt/virtualize.py b/pypy/jit/metainterp/optimizeopt/virtualize.py
--- a/pypy/jit/metainterp/optimizeopt/virtualize.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualize.py
@@ -10,13 +10,12 @@
 
 
 class AbstractVirtualValue(optimizer.OptValue):
-    _attrs_ = ('optimizer', 'keybox', 'source_op', '_cached_vinfo')
+    _attrs_ = ('keybox', 'source_op', '_cached_vinfo')
     box = None
     level = optimizer.LEVEL_NONNULL
     _cached_vinfo = None
 
-    def __init__(self, optimizer, keybox, source_op=None):
-        self.optimizer = optimizer
+    def __init__(self, keybox, source_op=None):
         self.keybox = keybox   # only used as a key in dictionaries
         self.source_op = source_op  # the NEW_WITH_VTABLE/NEW_ARRAY operation
                                     # that builds this box
@@ -29,17 +28,17 @@
             return self.keybox
         return self.box
 
-    def force_box(self):
+    def force_box(self, optforce):
         if self.box is None:
-            self.optimizer.forget_numberings(self.keybox)
-            self._really_force()
+            optforce.forget_numberings(self.keybox)
+            self._really_force(optforce)
         return self.box
 
-    def force_at_end_of_preamble(self, already_forced):
+    def force_at_end_of_preamble(self, already_forced, optforce):
         value = already_forced.get(self, None)
         if value:
             return value
-        return OptValue(self.force_box())
+        return OptValue(self.force_box(optforce))
 
     def make_virtual_info(self, modifier, fieldnums):
         if fieldnums is None:
@@ -55,7 +54,7 @@
     def _make_virtual(self, modifier):
         raise NotImplementedError("abstract base")
 
-    def _really_force(self):
+    def _really_force(self, optforce):
         raise NotImplementedError("abstract base")
 
     def import_from(self, other, optimizer):
@@ -70,10 +69,11 @@
 get_fielddescrlist_cache._annspecialcase_ = "specialize:memo"
 
 class AbstractVirtualStructValue(AbstractVirtualValue):
-    _attrs_ = ('_fields', '_cached_sorted_fields')
+    _attrs_ = ('_fields', 'cpu', '_cached_sorted_fields')
 
-    def __init__(self, optimizer, keybox, source_op=None):
-        AbstractVirtualValue.__init__(self, optimizer, keybox, source_op)
+    def __init__(self, cpu, keybox, source_op=None):
+        AbstractVirtualValue.__init__(self, keybox, source_op)
+        self.cpu = cpu
         self._fields = {}
         self._cached_sorted_fields = None
 
@@ -87,45 +87,44 @@
     def _get_descr(self):
         raise NotImplementedError
 
-    def _is_immutable_and_filled_with_constants(self):
+    def _is_immutable_and_filled_with_constants(self, optforce):
         count = self._get_descr().count_fields_if_immutable()
         if count != len(self._fields):    # always the case if count == -1
             return False
         for value in self._fields.itervalues():
-            subbox = value.force_box()
+            subbox = value.force_box(optforce)
             if not isinstance(subbox, Const):
                 return False
         return True
 
-    def force_at_end_of_preamble(self, already_forced):
+    def force_at_end_of_preamble(self, already_forced, optforce):
         if self in already_forced:
             return self
         already_forced[self] = self
         if self._fields:
             for ofs in self._fields.keys():
-                self._fields[ofs] = self._fields[ofs].force_at_end_of_preamble(already_forced)
+                self._fields[ofs] = self._fields[ofs].force_at_end_of_preamble(already_forced, optforce)
         return self
 
-    def _really_force(self):
+    def _really_force(self, optforce):
         op = self.source_op
         assert op is not None
         # ^^^ This case should not occur any more (see test_bug_3).
         #
         if not we_are_translated():
             op.name = 'FORCE ' + self.source_op.name
-
-        if self._is_immutable_and_filled_with_constants():
-            box = self.optimizer.constant_fold(op)
+            
+        if self._is_immutable_and_filled_with_constants(optforce):
+            box = optforce.optimizer.constant_fold(op)
             self.make_constant(box)
             for ofs, value in self._fields.iteritems():
-                subbox = value.force_box()
+                subbox = value.force_box(optforce)
                 assert isinstance(subbox, Const)
-                execute(self.optimizer.cpu, None, rop.SETFIELD_GC,
+                execute(optforce.optimizer.cpu, None, rop.SETFIELD_GC,
                         ofs, box, subbox)
             # keep self._fields, because it's all immutable anyway
         else:
-            newoperations = self.optimizer.newoperations
-            newoperations.append(op)
+            optforce.emit_operation(op)
             self.box = box = op.result
             #
             iteritems = self._fields.iteritems()
@@ -135,10 +134,11 @@
             for ofs, value in iteritems:
                 if value.is_null():
                     continue
-                subbox = value.force_box()
+                subbox = value.force_box(optforce)
                 op = ResOperation(rop.SETFIELD_GC, [box, subbox], None,
                                   descr=ofs)
-                newoperations.append(op)
+
+                optforce.emit_operation(op)
 
     def _get_field_descr_list(self):
         _cached_sorted_fields = self._cached_sorted_fields
@@ -155,7 +155,7 @@
             else:
                 lst = self._fields.keys()
             sort_descrs(lst)
-            cache = get_fielddescrlist_cache(self.optimizer.cpu)
+            cache = get_fielddescrlist_cache(self.cpu)
             result = cache.get(lst, None)
             if result is None:
                 cache[lst] = lst
@@ -180,8 +180,8 @@
 class VirtualValue(AbstractVirtualStructValue):
     level = optimizer.LEVEL_KNOWNCLASS
 
-    def __init__(self, optimizer, known_class, keybox, source_op=None):
-        AbstractVirtualStructValue.__init__(self, optimizer, keybox, source_op)
+    def __init__(self, cpu, known_class, keybox, source_op=None):
+        AbstractVirtualStructValue.__init__(self, cpu, keybox, source_op)
         assert isinstance(known_class, Const)
         self.known_class = known_class
 
@@ -190,7 +190,7 @@
         return modifier.make_virtual(self.known_class, fielddescrs)
 
     def _get_descr(self):
-        return vtable2descr(self.optimizer.cpu, self.known_class.getint())
+        return vtable2descr(self.cpu, self.known_class.getint())
 
     def __repr__(self):
         cls_name = self.known_class.value.adr.ptr._obj._TYPE._name
@@ -201,8 +201,8 @@
 
 class VStructValue(AbstractVirtualStructValue):
 
-    def __init__(self, optimizer, structdescr, keybox, source_op=None):
-        AbstractVirtualStructValue.__init__(self, optimizer, keybox, source_op)
+    def __init__(self, cpu, structdescr, keybox, source_op=None):
+        AbstractVirtualStructValue.__init__(self, cpu, keybox, source_op)
         self.structdescr = structdescr
 
     def _make_virtual(self, modifier):
@@ -215,10 +215,10 @@
 
 class VArrayValue(AbstractVirtualValue):
 
-    def __init__(self, optimizer, arraydescr, size, keybox, source_op=None):
-        AbstractVirtualValue.__init__(self, optimizer, keybox, source_op)
+    def __init__(self, arraydescr, constvalue, size, keybox, source_op=None):
+        AbstractVirtualValue.__init__(self, keybox, source_op)
         self.arraydescr = arraydescr
-        self.constvalue = optimizer.new_const_item(arraydescr)
+        self.constvalue = constvalue
         self._items = [self.constvalue] * size
 
     def getlength(self):
@@ -232,31 +232,30 @@
         assert isinstance(itemvalue, optimizer.OptValue)
         self._items[index] = itemvalue
 
-    def force_at_end_of_preamble(self, already_forced):
+    def force_at_end_of_preamble(self, already_forced, optforce):
         if self in already_forced:
             return self
         already_forced[self] = self
         for index in range(len(self._items)):
-            self._items[index] = self._items[index].force_at_end_of_preamble(already_forced)
+            self._items[index] = self._items[index].force_at_end_of_preamble(already_forced, optforce)
         return self
     
-    def _really_force(self):
+    def _really_force(self, optforce):
         assert self.source_op is not None
         if not we_are_translated():
             self.source_op.name = 'FORCE ' + self.source_op.name
-        newoperations = self.optimizer.newoperations
-        newoperations.append(self.source_op)
+        optforce.emit_operation(self.source_op)        
         self.box = box = self.source_op.result
         for index in range(len(self._items)):
             subvalue = self._items[index]
             if subvalue is not self.constvalue:
                 if subvalue.is_null():
                     continue
-                subbox = subvalue.force_box()
+                subbox = subvalue.force_box(optforce)
                 op = ResOperation(rop.SETARRAYITEM_GC,
                                   [box, ConstInt(index), subbox], None,
                                   descr=self.arraydescr)
-                newoperations.append(op)
+                optforce.emit_operation(op)
 
     def get_args_for_fail(self, modifier):
         if self.box is None and not modifier.already_seen_virtual(self.keybox):
@@ -279,17 +278,18 @@
         return OptVirtualize()
         
     def make_virtual(self, known_class, box, source_op=None):
-        vvalue = VirtualValue(self.optimizer, known_class, box, source_op)
+        vvalue = VirtualValue(self.optimizer.cpu, known_class, box, source_op)
         self.make_equal_to(box, vvalue)
         return vvalue
 
     def make_varray(self, arraydescr, size, box, source_op=None):
-        vvalue = VArrayValue(self.optimizer, arraydescr, size, box, source_op)
+        constvalue = self.new_const_item(arraydescr)
+        vvalue = VArrayValue(arraydescr, constvalue, size, box, source_op)
         self.make_equal_to(box, vvalue)
         return vvalue
 
     def make_vstruct(self, structdescr, box, source_op=None):
-        vvalue = VStructValue(self.optimizer, structdescr, box, source_op)
+        vvalue = VStructValue(self.optimizer.cpu, structdescr, box, source_op)
         self.make_equal_to(box, vvalue)
         return vvalue
 
@@ -362,7 +362,6 @@
             self.make_equal_to(op.result, fieldvalue)
         else:
             value.ensure_nonnull()
-            ###self.heap_op_optimizer.optimize_GETFIELD_GC(op, value)
             self.emit_operation(op)
 
     # note: the following line does not mean that the two operations are
@@ -377,7 +376,6 @@
             value.setfield(op.getdescr(), fieldvalue)
         else:
             value.ensure_nonnull()
-            ###self.heap_op_optimizer.optimize_SETFIELD_GC(op, value, fieldvalue)
             self.emit_operation(op)
 
     def optimize_NEW_WITH_VTABLE(self, op):
@@ -417,7 +415,6 @@
                 self.make_equal_to(op.result, itemvalue)
                 return
         value.ensure_nonnull()
-        ###self.heap_op_optimizer.optimize_GETARRAYITEM_GC(op, value)
         self.emit_operation(op)
 
     # note: the following line does not mean that the two operations are
@@ -432,7 +429,6 @@
                 value.setitem(indexbox.getint(), self.getvalue(op.getarg(2)))
                 return
         value.ensure_nonnull()
-        ###self.heap_op_optimizer.optimize_SETARRAYITEM_GC(op, value, fieldvalue)
         self.emit_operation(op)
 
 
diff --git a/pypy/jit/metainterp/optimizeopt/virtualstate.py b/pypy/jit/metainterp/optimizeopt/virtualstate.py
--- a/pypy/jit/metainterp/optimizeopt/virtualstate.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualstate.py
@@ -30,7 +30,7 @@
     def _generate_guards(self, other, box, cpu, extra_guards):
         raise InvalidLoop
 
-    def enum_forced_boxes(self, boxes, value):
+    def enum_forced_boxes(self, boxes, value, optimizer):
         raise NotImplementedError
 
     def enum(self, virtual_state):
@@ -100,14 +100,14 @@
     def _generalization_of(self, other):
         raise NotImplementedError
 
-    def enum_forced_boxes(self, boxes, value):
+    def enum_forced_boxes(self, boxes, value, optimizer):
         assert isinstance(value, virtualize.AbstractVirtualStructValue)
         assert value.is_virtual()
         for i in range(len(self.fielddescrs)):
             v = value._fields[self.fielddescrs[i]]
             s = self.fieldstate[i]
             if s.position > self.position:
-                s.enum_forced_boxes(boxes, v)
+                s.enum_forced_boxes(boxes, v, optimizer)
 
     def _enum(self, virtual_state):
         for s in self.fieldstate:
@@ -177,14 +177,14 @@
                 return False
         return True
 
-    def enum_forced_boxes(self, boxes, value):
+    def enum_forced_boxes(self, boxes, value, optimizer):
         assert isinstance(value, virtualize.VArrayValue)
         assert value.is_virtual()
         for i in range(len(self.fieldstate)):
             v = value._items[i]
             s = self.fieldstate[i]
             if s.position > self.position:
-                s.enum_forced_boxes(boxes, v)
+                s.enum_forced_boxes(boxes, v, optimizer)
 
     def _enum(self, virtual_state):
         for s in self.fieldstate:
@@ -316,11 +316,11 @@
             import pdb; pdb.set_trace()
             raise NotImplementedError
 
-    def enum_forced_boxes(self, boxes, value):
+    def enum_forced_boxes(self, boxes, value, optimizer):
         if self.level == LEVEL_CONSTANT:
             return
         assert 0 <= self.position_in_notvirtuals 
-        boxes[self.position_in_notvirtuals] = value.force_box()
+        boxes[self.position_in_notvirtuals] = value.force_box(optimizer)
 
     def _enum(self, virtual_state):
         if self.level == LEVEL_CONSTANT:
@@ -377,11 +377,13 @@
             self.state[i].generate_guards(other.state[i], args[i],
                                           cpu, extra_guards, renum)
 
-    def make_inputargs(self, values, keyboxes=False):
+    def make_inputargs(self, values, optimizer, keyboxes=False):
+        if optimizer.optearlyforce:
+            optimizer = optimizer.optearlyforce
         assert len(values) == len(self.state)
         inputargs = [None] * len(self.notvirtuals)
         for i in range(len(values)):
-            self.state[i].enum_forced_boxes(inputargs, values[i])
+            self.state[i].enum_forced_boxes(inputargs, values[i], optimizer)
 
         if keyboxes:
             for i in range(len(values)):
@@ -434,7 +436,12 @@
     def get_virtual_state(self, jump_args):
         self.optimizer.force_at_end_of_preamble()
         already_forced = {}
-        values = [self.getvalue(box).force_at_end_of_preamble(already_forced)
+        if self.optimizer.optearlyforce:
+            opt = self.optimizer.optearlyforce
+        else:
+            opt = self.optimizer
+        values = [self.getvalue(box).force_at_end_of_preamble(already_forced,
+                                                              opt)
                   for box in jump_args]
 
         for value in values:
diff --git a/pypy/jit/metainterp/optimizeopt/vstring.py b/pypy/jit/metainterp/optimizeopt/vstring.py
--- a/pypy/jit/metainterp/optimizeopt/vstring.py
+++ b/pypy/jit/metainterp/optimizeopt/vstring.py
@@ -43,7 +43,7 @@
 class __extend__(optimizer.OptValue):
     """New methods added to the base class OptValue for this file."""
 
-    def getstrlen(self, optimization, mode):
+    def getstrlen(self, string_optimizer, mode):
         if mode is mode_string:
             s = self.get_constant_string_spec(mode_string)
             if s is not None:
@@ -52,12 +52,12 @@
             s = self.get_constant_string_spec(mode_unicode)
             if s is not None:
                 return ConstInt(len(s))
-        if optimization is None:
+        if string_optimizer is None:
             return None
         self.ensure_nonnull()
-        box = self.force_box()
+        box = self.force_box(string_optimizer)
         lengthbox = BoxInt()
-        optimization.propagate_forward(ResOperation(mode.STRLEN, [box], lengthbox))
+        string_optimizer.emit_operation(ResOperation(mode.STRLEN, [box], lengthbox))
         return lengthbox
 
     @specialize.arg(1)
@@ -68,25 +68,25 @@
         else:
             return None
 
-    def string_copy_parts(self, optimizer, targetbox, offsetbox, mode):
+    def string_copy_parts(self, string_optimizer, targetbox, offsetbox, mode):
         # Copies the pointer-to-string 'self' into the target string
         # given by 'targetbox', at the specified offset.  Returns the offset
         # at the end of the copy.
-        lengthbox = self.getstrlen(optimizer, mode)
-        srcbox = self.force_box()
-        return copy_str_content(optimizer, srcbox, targetbox,
+        lengthbox = self.getstrlen(string_optimizer, mode)
+        srcbox = self.force_box(string_optimizer)
+        return copy_str_content(string_optimizer, srcbox, targetbox,
                                 CONST_0, offsetbox, lengthbox, mode)
 
 
 class VAbstractStringValue(virtualize.AbstractVirtualValue):
     _attrs_ = ('mode',)
 
-    def __init__(self, optimizer, keybox, source_op, mode):
-        virtualize.AbstractVirtualValue.__init__(self, optimizer, keybox,
+    def __init__(self, keybox, source_op, mode):
+        virtualize.AbstractVirtualValue.__init__(self, keybox,
                                                  source_op)
         self.mode = mode
 
-    def _really_force(self):
+    def _really_force(self, optforce):
         if self.mode is mode_string:
             s = self.get_constant_string_spec(mode_string)
             if s is not None:
@@ -101,12 +101,12 @@
                 return
         assert self.source_op is not None
         self.box = box = self.source_op.result
-        lengthbox = self.getstrlen(self.optimizer, self.mode)
+        lengthbox = self.getstrlen(optforce, self.mode)
         op = ResOperation(self.mode.NEWSTR, [lengthbox], box)
         if not we_are_translated():
             op.name = 'FORCE'
-        self.optimizer.emit_operation(op)
-        self.string_copy_parts(self.optimizer, box, CONST_0, self.mode)
+        optforce.emit_operation(op)
+        self.string_copy_parts(optforce, box, CONST_0, self.mode)
 
 
 class VStringPlainValue(VAbstractStringValue):
@@ -140,20 +140,20 @@
         return mode.emptystr.join([mode.chr(c.box.getint())
                                    for c in self._chars])
 
-    def string_copy_parts(self, optimizer, targetbox, offsetbox, mode):
+    def string_copy_parts(self, string_optimizer, targetbox, offsetbox, mode):
         if not self.is_virtual() and targetbox is not self.box:
-            lengthbox = self.getstrlen(optimizer, mode)
-            srcbox = self.force_box()
-            return copy_str_content(optimizer, srcbox, targetbox,
+            lengthbox = self.getstrlen(string_optimizer, mode)
+            srcbox = self.force_box(string_optimizer)
+            return copy_str_content(string_optimizer, srcbox, targetbox,
                                 CONST_0, offsetbox, lengthbox, mode)
         for i in range(len(self._chars)):
-            charbox = self._chars[i].force_box()
+            charbox = self._chars[i].force_box(string_optimizer)
             if not (isinstance(charbox, Const) and charbox.same_constant(CONST_0)):
-                optimizer.emit_operation(ResOperation(mode.STRSETITEM, [targetbox,
-                                                                    offsetbox,
-                                                                    charbox],
+                string_optimizer.emit_operation(ResOperation(mode.STRSETITEM, [targetbox,
+                                                                               offsetbox,
+                                                                               charbox],
                                                   None))
-            offsetbox = _int_add(optimizer, offsetbox, CONST_1)
+            offsetbox = _int_add(string_optimizer, offsetbox, CONST_1)
         return offsetbox
 
     def get_args_for_fail(self, modifier):
@@ -187,16 +187,16 @@
         self.left = left
         self.right = right
 
-    def getstrlen(self, optimizer, mode):
+    def getstrlen(self, string_optimizer, mode):
         if self.lengthbox is None:
-            len1box = self.left.getstrlen(optimizer, mode)
+            len1box = self.left.getstrlen(string_optimizer, mode)
             if len1box is None:
                 return None
-            len2box = self.right.getstrlen(optimizer, mode)
+            len2box = self.right.getstrlen(string_optimizer, mode)
             if len2box is None:
                 return None
-            self.lengthbox = _int_add(optimizer, len1box, len2box)
-            # ^^^ may still be None, if optimizer is None
+            self.lengthbox = _int_add(string_optimizer, len1box, len2box)
+            # ^^^ may still be None, if string_optimizer is None
         return self.lengthbox
 
     @specialize.arg(1)
@@ -209,10 +209,10 @@
             return None
         return s1 + s2
 
-    def string_copy_parts(self, optimizer, targetbox, offsetbox, mode):
-        offsetbox = self.left.string_copy_parts(optimizer, targetbox,
+    def string_copy_parts(self, string_optimizer, targetbox, offsetbox, mode):
+        offsetbox = self.left.string_copy_parts(string_optimizer, targetbox,
                                                 offsetbox, mode)
-        offsetbox = self.right.string_copy_parts(optimizer, targetbox,
+        offsetbox = self.right.string_copy_parts(string_optimizer, targetbox,
                                                  offsetbox, mode)
         return offsetbox
 
@@ -251,8 +251,8 @@
         self.vstart = vstart
         self.vlength = vlength
 
-    def getstrlen(self, _, mode):
-        return self.vlength.force_box()
+    def getstrlen(self, optforce, mode):
+        return self.vlength.force_box(optforce)
 
     @specialize.arg(1)
     def get_constant_string_spec(self, mode):
@@ -267,11 +267,11 @@
             return s1[start : start + length]
         return None
 
-    def string_copy_parts(self, optimizer, targetbox, offsetbox, mode):
-        lengthbox = self.getstrlen(optimizer, mode)
-        return copy_str_content(optimizer,
-                                self.vstr.force_box(), targetbox,
-                                self.vstart.force_box(), offsetbox,
+    def string_copy_parts(self, string_optimizer, targetbox, offsetbox, mode):
+        lengthbox = self.getstrlen(string_optimizer, mode)
+        return copy_str_content(string_optimizer,
+                                self.vstr.force_box(string_optimizer), targetbox,
+                                self.vstart.force_box(string_optimizer), offsetbox,
                                 lengthbox, mode)
 
     def get_args_for_fail(self, modifier):
@@ -300,7 +300,7 @@
         return modifier.make_vstrslice(self.mode is mode_unicode)
 
 
-def copy_str_content(optimizer, srcbox, targetbox,
+def copy_str_content(string_optimizer, srcbox, targetbox,
                      srcoffsetbox, offsetbox, lengthbox, mode, need_next_offset=True):
     if isinstance(srcbox, ConstPtr) and isinstance(srcoffsetbox, Const):
         M = 5
@@ -310,26 +310,26 @@
         # up to M characters are done "inline", i.e. with STRGETITEM/STRSETITEM
         # instead of just a COPYSTRCONTENT.
         for i in range(lengthbox.value):
-            charbox = _strgetitem(optimizer, srcbox, srcoffsetbox, mode)
-            srcoffsetbox = _int_add(optimizer, srcoffsetbox, CONST_1)
-            optimizer.emit_operation(ResOperation(mode.STRSETITEM, [targetbox,
-                                                                offsetbox,
-                                                                charbox],
+            charbox = _strgetitem(string_optimizer, srcbox, srcoffsetbox, mode)
+            srcoffsetbox = _int_add(string_optimizer, srcoffsetbox, CONST_1)
+            string_optimizer.emit_operation(ResOperation(mode.STRSETITEM, [targetbox,
+                                                                           offsetbox,
+                                                                           charbox],
                                               None))
-            offsetbox = _int_add(optimizer, offsetbox, CONST_1)
+            offsetbox = _int_add(string_optimizer, offsetbox, CONST_1)
     else:
         if need_next_offset:
-            nextoffsetbox = _int_add(optimizer, offsetbox, lengthbox)
+            nextoffsetbox = _int_add(string_optimizer, offsetbox, lengthbox)
         else:
             nextoffsetbox = None
         op = ResOperation(mode.COPYSTRCONTENT, [srcbox, targetbox,
                                                 srcoffsetbox, offsetbox,
                                                 lengthbox], None)
-        optimizer.emit_operation(op)
+        string_optimizer.emit_operation(op)
         offsetbox = nextoffsetbox
     return offsetbox
 
-def _int_add(optimizer, box1, box2):
+def _int_add(string_optimizer, box1, box2):
     if isinstance(box1, ConstInt):
         if box1.value == 0:
             return box2
@@ -337,23 +337,23 @@
             return ConstInt(box1.value + box2.value)
     elif isinstance(box2, ConstInt) and box2.value == 0:
         return box1
-    if optimizer is None:
+    if string_optimizer is None:
         return None
     resbox = BoxInt()
-    optimizer.propagate_forward(ResOperation(rop.INT_ADD, [box1, box2], resbox))
+    string_optimizer.emit_operation(ResOperation(rop.INT_ADD, [box1, box2], resbox))
     return resbox
 
-def _int_sub(optimizer, box1, box2):
+def _int_sub(string_optimizer, box1, box2):
     if isinstance(box2, ConstInt):
         if box2.value == 0:
             return box1
         if isinstance(box1, ConstInt):
             return ConstInt(box1.value - box2.value)
     resbox = BoxInt()
-    optimizer.propagate_forward(ResOperation(rop.INT_SUB, [box1, box2], resbox))
+    string_optimizer.emit_operation(ResOperation(rop.INT_SUB, [box1, box2], resbox))
     return resbox
 
-def _strgetitem(optimizer, strbox, indexbox, mode):
+def _strgetitem(string_optimizer, strbox, indexbox, mode):
     if isinstance(strbox, ConstPtr) and isinstance(indexbox, ConstInt):
         if mode is mode_string:
             s = strbox.getref(lltype.Ptr(rstr.STR))
@@ -362,30 +362,28 @@
             s = strbox.getref(lltype.Ptr(rstr.UNICODE))
             return ConstInt(ord(s.chars[indexbox.getint()]))
     resbox = BoxInt()
-    optimizer.propagate_forward(ResOperation(mode.STRGETITEM, [strbox, indexbox],
-                                      resbox))
+    string_optimizer.emit_operation(ResOperation(mode.STRGETITEM, [strbox, indexbox],
+                                                 resbox))
     return resbox
 
 
 class OptString(optimizer.Optimization):
     "Handling of strings and unicodes."
-    enabled = True
-
     def new(self):
         return OptString()
 
     def make_vstring_plain(self, box, source_op, mode):
-        vvalue = VStringPlainValue(self.optimizer, box, source_op, mode)
+        vvalue = VStringPlainValue(box, source_op, mode)
         self.make_equal_to(box, vvalue)
         return vvalue
 
     def make_vstring_concat(self, box, source_op, mode):
-        vvalue = VStringConcatValue(self.optimizer, box, source_op, mode)
+        vvalue = VStringConcatValue(box, source_op, mode)
         self.make_equal_to(box, vvalue)
         return vvalue
 
     def make_vstring_slice(self, box, source_op, mode):
-        vvalue = VStringSliceValue(self.optimizer, box, source_op, mode)
+        vvalue = VStringSliceValue(box, source_op, mode)
         self.make_equal_to(box, vvalue)
         return vvalue
 
@@ -435,9 +433,9 @@
         value.ensure_nonnull()
         #
         if value.is_virtual() and isinstance(value, VStringSliceValue):
-            fullindexbox = _int_add(self.optimizer,
-                                    value.vstart.force_box(),
-                                    vindex.force_box())
+            fullindexbox = _int_add(self,
+                                    value.vstart.force_box(self),
+                                    vindex.force_box(self))
             value = value.vstr
             vindex = self.getvalue(fullindexbox)
         #
@@ -449,7 +447,7 @@
                 if res is not optimizer.CVAL_UNINITIALIZED_ZERO:
                     return res
         #
-        resbox = _strgetitem(self.optimizer, value.force_box(), vindex.force_box(), mode)
+        resbox = _strgetitem(self, value.force_box(self), vindex.force_box(self), mode)
         return self.getvalue(resbox)
 
     def optimize_STRLEN(self, op):
@@ -459,7 +457,7 @@
 
     def _optimize_STRLEN(self, op, mode):
         value = self.getvalue(op.getarg(0))
-        lengthbox = value.getstrlen(self.optimizer, mode)
+        lengthbox = value.getstrlen(self, mode)
         self.make_equal_to(op.result, self.getvalue(lengthbox))
 
     def optimize_COPYSTRCONTENT(self, op):
@@ -477,12 +475,12 @@
 
         if length.is_constant() and length.box.getint() == 0:
             return
-        copy_str_content(self.optimizer,
-            src.force_box(),
-            dst.force_box(),
-            srcstart.force_box(),
-            dststart.force_box(),
-            length.force_box(),
+        copy_str_content(self,
+            src.force_box(self),
+            dst.force_box(self),
+            srcstart.force_box(self),
+            dststart.force_box(self),
+            length.force_box(self),
             mode, need_next_offset=False
         )
 
@@ -508,6 +506,8 @@
                     return
         self.emit_operation(op)
 
+    optimize_CALL_PURE = optimize_CALL
+
     def opt_call_str_STR2UNICODE(self, op):
         # Constant-fold unicode("constant string").
         # More generally, supporting non-constant but virtual cases is
@@ -547,16 +547,16 @@
             return True
         #
         vstr.ensure_nonnull()
-        lengthbox = _int_sub(self.optimizer, vstop.force_box(),
-                                            vstart.force_box())
+        lengthbox = _int_sub(self, vstop.force_box(self),
+                                   vstart.force_box(self))
         #
         if isinstance(vstr, VStringSliceValue):
             # double slicing  s[i:j][k:l]
             vintermediate = vstr
             vstr = vintermediate.vstr
-            startbox = _int_add(self.optimizer,
-                                vintermediate.vstart.force_box(),
-                                vstart.force_box())
+            startbox = _int_add(self,
+                                vintermediate.vstart.force_box(self),
+                                vstart.force_box(self))
             vstart = self.getvalue(startbox)
         #
         value = self.make_vstring_slice(op.result, op, mode)
@@ -594,8 +594,8 @@
                 do = EffectInfo.OS_STREQ_LENGTHOK
             else:
                 do = EffectInfo.OS_STREQ_NONNULL
-            self.generate_modified_call(do, [v1.force_box(),
-                                             v2.force_box()], op.result, mode)
+            self.generate_modified_call(do, [v1.force_box(self),
+                                             v2.force_box(self)], op.result, mode)
             return True
         return False
 
@@ -603,7 +603,7 @@
         l2box = v2.getstrlen(None, mode)
         if isinstance(l2box, ConstInt):
             if l2box.value == 0:
-                lengthbox = v1.getstrlen(self.optimizer, mode)
+                lengthbox = v1.getstrlen(self, mode)
                 seo = self.optimizer.send_extra_operation
                 seo(ResOperation(rop.INT_EQ, [lengthbox, CONST_0], resultbox))
                 return True
@@ -614,17 +614,17 @@
                     vchar1 = self.strgetitem(v1, optimizer.CVAL_ZERO, mode)
                     vchar2 = self.strgetitem(v2, optimizer.CVAL_ZERO, mode)
                     seo = self.optimizer.send_extra_operation
-                    seo(ResOperation(rop.INT_EQ, [vchar1.force_box(),
-                                                  vchar2.force_box()],
+                    seo(ResOperation(rop.INT_EQ, [vchar1.force_box(self),
+                                                  vchar2.force_box(self)],
                                      resultbox))
                     return True
                 if isinstance(v1, VStringSliceValue):
                     vchar = self.strgetitem(v2, optimizer.CVAL_ZERO, mode)
                     do = EffectInfo.OS_STREQ_SLICE_CHAR
-                    self.generate_modified_call(do, [v1.vstr.force_box(),
-                                                     v1.vstart.force_box(),
-                                                     v1.vlength.force_box(),
-                                                     vchar.force_box()],
+                    self.generate_modified_call(do, [v1.vstr.force_box(self),
+                                                     v1.vstart.force_box(self),
+                                                     v1.vlength.force_box(self),
+                                                     vchar.force_box(self)],
                                                 resultbox, mode)
                     return True
         #
@@ -635,10 +635,10 @@
             if v1.is_null():
                 self.make_constant(resultbox, CONST_1)
                 return True
-            op = ResOperation(rop.PTR_EQ, [v1.force_box(),
+            op = ResOperation(rop.PTR_EQ, [v1.force_box(self),
                                            llhelper.CONST_NULL],
                               resultbox)
-            self.optimizer.emit_operation(op)
+            self.emit_operation(op)
             return True
         #
         return False
@@ -652,8 +652,8 @@
                     do = EffectInfo.OS_STREQ_NONNULL_CHAR
                 else:
                     do = EffectInfo.OS_STREQ_CHECKNULL_CHAR
-                self.generate_modified_call(do, [v1.force_box(),
-                                                 vchar.force_box()], resultbox,
+                self.generate_modified_call(do, [v1.force_box(self),
+                                                 vchar.force_box(self)], resultbox,
                                             mode)
                 return True
         #
@@ -662,10 +662,10 @@
                 do = EffectInfo.OS_STREQ_SLICE_NONNULL
             else:
                 do = EffectInfo.OS_STREQ_SLICE_CHECKNULL
-            self.generate_modified_call(do, [v1.vstr.force_box(),
-                                             v1.vstart.force_box(),
-                                             v1.vlength.force_box(),
-                                             v2.force_box()], resultbox, mode)
+            self.generate_modified_call(do, [v1.vstr.force_box(self),
+                                             v1.vstart.force_box(self),
+                                             v1.vlength.force_box(self),
+                                             v2.force_box(self)], resultbox, mode)
             return True
         return False
 
@@ -675,16 +675,11 @@
         calldescr, func = cic.callinfo_for_oopspec(oopspecindex)
         op = ResOperation(rop.CALL, [ConstInt(func)] + args, result,
                           descr=calldescr)
-        self.optimizer.emit_operation(op)
+        self.emit_operation(op)
 
     def propagate_forward(self, op):
-        if not self.enabled:
-            self.emit_operation(op)
-            return
-
         dispatch_opt(self, op)
 
-
 dispatch_opt = make_dispatcher_method(OptString, 'optimize_',
         default=OptString.emit_operation)
 
diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py
--- a/pypy/jit/metainterp/resoperation.py
+++ b/pypy/jit/metainterp/resoperation.py
@@ -221,7 +221,6 @@
         newop.setfailargs(self.getfailargs())
         return newop
 
-
 # ============
 # arity mixins
 # ============
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -2956,6 +2956,18 @@
         assert res == f(32)
         self.check_loops(arraylen_gc=2)
 
+    def test_ulonglong_mod(self):
+        myjitdriver = JitDriver(greens = [], reds = ['n', 'sa', 'i'])
+        def f(n):
+            sa = i = rffi.cast(rffi.ULONGLONG, 1)
+            while i < rffi.cast(rffi.ULONGLONG, n):
+                myjitdriver.jit_merge_point(sa=sa, n=n, i=i)
+                sa += sa % i
+                i += 1
+        res = self.meta_interp(f, [32])
+        assert res == f(32)
+
+
 class TestOOtype(BasicTests, OOJitMixin):
 
     def test_oohash(self):
diff --git a/pypy/jit/metainterp/test/test_longlong.py b/pypy/jit/metainterp/test/test_longlong.py
--- a/pypy/jit/metainterp/test/test_longlong.py
+++ b/pypy/jit/metainterp/test/test_longlong.py
@@ -118,6 +118,26 @@
         res = self.interp_operations(f, [1000000000])
         assert res == 123500000000.0
 
+    def test_floats_negative(self):
+        def f(i):
+            # i == 1000000000
+            f = i * -123.5
+            n = r_longlong(f)
+            compare(n, -29, 1054051584)
+            return float(n)
+        res = self.interp_operations(f, [1000000000])
+        assert res == -123500000000.0
+
+    def test_floats_ulonglong(self):
+        def f(i):
+            # i == 1000000000
+            f = i * 12350000000.0
+            n = r_ulonglong(f)
+            compare(n, -1419508847, 538116096)
+            return float(n)
+        res = self.interp_operations(f, [1000000000])
+        assert res == 12350000000000000000.0
+
     def test_unsigned_compare_ops(self):
         def f(n1, n2):
             # n == 30002000000000
diff --git a/pypy/jit/metainterp/test/test_resume.py b/pypy/jit/metainterp/test/test_resume.py
--- a/pypy/jit/metainterp/test/test_resume.py
+++ b/pypy/jit/metainterp/test/test_resume.py
@@ -576,8 +576,9 @@
 
 
 class FakeOptimizer_VirtualValue(object):
-    class cpu:
-        pass
+    class optimizer:
+        class cpu:
+            pass
 fakeoptimizer = FakeOptimizer_VirtualValue()
 
 def ConstAddr(addr, cpu):   # compatibility
@@ -1135,12 +1136,7 @@
     modifier.liveboxes = {}
     modifier.vfieldboxes = {}
 
-    class FakeOptimizer(object):
-        class cpu:
-            pass
-        def new_const_item(self, descr):
-            return None
-    v2 = VArrayValue(FakeOptimizer(), LLtypeMixin.arraydescr, 2, b2s)
+    v2 = VArrayValue(LLtypeMixin.arraydescr, None, 2, b2s)
     v2._items = [b4s, c1s]
     modifier.register_virtual_fields(b2s, [b4s, c1s])
     liveboxes = []
diff --git a/pypy/jit/metainterp/test/test_string.py b/pypy/jit/metainterp/test/test_string.py
--- a/pypy/jit/metainterp/test/test_string.py
+++ b/pypy/jit/metainterp/test/test_string.py
@@ -484,6 +484,30 @@
             return len(sa.val)
         assert self.meta_interp(f, ['a']) == f('a')
 
+    def test_string_comepare_quasiimmutable(self):
+        class Sys(object):
+            _immutable_fields_ = ["defaultencoding?"]
+            def __init__(self, s):
+                self.defaultencoding = s
+        _str = self._str
+        sys = Sys(_str('ascii'))        
+        mydriver = JitDriver(reds = ['n', 'sa'], greens = [])
+        def f(n):
+            sa = 0
+            sys.defaultencoding = _str('ascii')
+            while n:
+                mydriver.jit_merge_point(n=n, sa=sa)
+                if sys.defaultencoding == _str('ascii'):
+                    sa += 1
+                n -= 1
+            sys.defaultencoding = _str('utf-8')
+            return sa
+        assert self.meta_interp(f, [8]) == f(8)
+        self.check_loops({'int_add': 1, 'guard_true': 1, 'int_sub': 1,
+                          'jump': 1, 'int_is_true': 1,
+                          'guard_not_invalidated': 1})
+
+
 #class TestOOtype(StringTests, OOJitMixin):
 #    CALL = "oosend"
 #    CALL_PURE = "oosend_pure"
diff --git a/pypy/jit/metainterp/test/test_virtualstate.py b/pypy/jit/metainterp/test/test_virtualstate.py
--- a/pypy/jit/metainterp/test/test_virtualstate.py
+++ b/pypy/jit/metainterp/test/test_virtualstate.py
@@ -431,7 +431,7 @@
         
 
 class BaseTestBridges(BaseTest):
-    enable_opts = "intbounds:rewrite:virtualize:string:heap:unroll"
+    enable_opts = "intbounds:rewrite:virtualize:string:pure:heap:unroll"
 
     def _do_optimize_bridge(self, bridge, call_pure_results):
         from pypy.jit.metainterp.optimizeopt import optimize_bridge_1, build_opt_chain
diff --git a/pypy/module/_numpy/__init__.py b/pypy/module/_numpy/__init__.py
--- a/pypy/module/_numpy/__init__.py
+++ b/pypy/module/_numpy/__init__.py
@@ -23,6 +23,7 @@
         ("arcsin", "arcsin"),
         ("arctan", "arctan"),
         ("arcsinh", "arcsinh"),
+        ("arctanh", "arctanh"),
         ("copysign", "copysign"),
         ("cos", "cos"),
         ("divide", "divide"),
diff --git a/pypy/module/_numpy/app_numpy.py b/pypy/module/_numpy/app_numpy.py
deleted file mode 100644
diff --git a/pypy/module/_numpy/interp_dtype.py b/pypy/module/_numpy/interp_dtype.py
--- a/pypy/module/_numpy/interp_dtype.py
+++ b/pypy/module/_numpy/interp_dtype.py
@@ -7,13 +7,14 @@
 from pypy.interpreter.typedef import TypeDef, interp_attrproperty, GetSetProperty
 from pypy.module._numpy import signature
 from pypy.objspace.std.floatobject import float2string
-from pypy.rlib import rfloat
-from pypy.rlib.rarithmetic import widen
+from pypy.rlib import rarithmetic, rfloat
+from pypy.rlib.rarithmetic import LONG_BIT, widen
 from pypy.rlib.objectmodel import specialize, enforceargs
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.rpython.lltypesystem import lltype, rffi
 
 
+UNSIGNEDLTR = "u"
 SIGNEDLTR = "i"
 BOOLLTR = "b"
 FLOATINGLTR = "f"
@@ -61,7 +62,10 @@
             self.val = val
 
         def wrap(self, space):
-            return space.wrap(self.val)
+            val = self.val
+            if valtype is rarithmetic.r_singlefloat:
+                val = float(val)
+            return space.wrap(val)
 
         def convert_to(self, dtype):
             return dtype.adapt_val(self.val)
@@ -145,7 +149,7 @@
         return self.adapt_val(func(self, self.for_computation(self.unbox(v))))
     return impl
 
-class ArithmaticTypeMixin(object):
+class ArithmeticTypeMixin(object):
     _mixin_ = True
 
     @binop
@@ -200,11 +204,17 @@
         return v1 >= v2
 
 
-class FloatArithmeticDtype(ArithmaticTypeMixin):
+class FloatArithmeticDtype(ArithmeticTypeMixin):
     _mixin_ = True
 
+    def unwrap(self, space, w_item):
+        return self.adapt_val(space.float_w(space.float(w_item)))
+
     def for_computation(self, v):
-        return v
+        return float(v)
+
+    def str_format(self, item):
+        return float2string(self.for_computation(self.unbox(item)), 'g', rfloat.DTSF_STR_PRECISION)
 
     @binop
     def mod(self, v1, v2):
@@ -250,12 +260,12 @@
         return math.tan(v)
     @unaryop
     def arcsin(self, v):
-        if v < -1.0 or v > 1.0:
+        if not -1.0 <= v <= 1.0:
             return rfloat.NAN
         return math.asin(v)
     @unaryop
     def arccos(self, v):
-        if v < -1.0 or v > 1.0:
+        if not -1.0 <= v <= 1.0:
             return rfloat.NAN
         return math.acos(v)
     @unaryop
@@ -266,7 +276,15 @@
     def arcsinh(self, v):
         return math.asinh(v)
 
-class IntegerArithmeticDtype(ArithmaticTypeMixin):
+    @unaryop
+    def arctanh(self, v):
+        if v == 1.0 or v == -1.0:
+            return math.copysign(rfloat.INFINITY, v)
+        if not -1.0 < v < 1.0:
+            return rfloat.NAN
+        return math.atanh(v)
+
+class IntegerArithmeticDtype(ArithmeticTypeMixin):
     _mixin_ = True
 
     def unwrap(self, space, w_item):
@@ -275,10 +293,16 @@
     def for_computation(self, v):
         return widen(v)
 
+    def str_format(self, item):
+        return str(widen(self.unbox(item)))
+
     @binop
     def mod(self, v1, v2):
         return v1 % v2
 
+class SignedIntegerArithmeticDtype(IntegerArithmeticDtype):
+    _mixin_ = True
+
     @unaryop
     def sign(self, v):
         if v > 0:
@@ -289,17 +313,21 @@
             assert v == 0
             return 0
 
-    def str_format(self, item):
-        return str(widen(self.unbox(item)))
+class UnsignedIntegerArithmeticDtype(IntegerArithmeticDtype):
+    _mixin_ = True
+
+    @unaryop
+    def sign(self, v):
+        return int(v != 0)
 
 W_BoolDtype = create_low_level_dtype(
     num = 0, kind = BOOLLTR, name = "bool",
-    aliases = ["?"],
+    aliases = ["?", "bool", "bool8"],
     applevel_types = ["bool"],
     T = lltype.Bool,
     valtype = bool,
 )
-class W_BoolDtype(IntegerArithmeticDtype, W_BoolDtype):
+class W_BoolDtype(SignedIntegerArithmeticDtype, W_BoolDtype):
     def unwrap(self, space, w_item):
         return self.adapt_val(space.is_true(w_item))
 
@@ -312,69 +340,140 @@
 
 W_Int8Dtype = create_low_level_dtype(
     num = 1, kind = SIGNEDLTR, name = "int8",
-    aliases = ["int8"],
+    aliases = ["b", "int8", "i1"],
     applevel_types = [],
     T = rffi.SIGNEDCHAR,
     valtype = rffi.SIGNEDCHAR._type,
     expected_size = 1,
 )
-class W_Int8Dtype(IntegerArithmeticDtype, W_Int8Dtype):
+class W_Int8Dtype(SignedIntegerArithmeticDtype, W_Int8Dtype):
+    pass
+
+W_UInt8Dtype = create_low_level_dtype(
+    num = 2, kind = UNSIGNEDLTR, name = "uint8",
+    aliases = ["B", "uint8", "I1"],
+    applevel_types = [],
+    T = rffi.UCHAR,
+    valtype = rffi.UCHAR._type,
+    expected_size = 1,
+)
+class W_UInt8Dtype(UnsignedIntegerArithmeticDtype, W_UInt8Dtype):
     pass
 
 W_Int16Dtype = create_low_level_dtype(
     num = 3, kind = SIGNEDLTR, name = "int16",
-    aliases = ["int16"],
+    aliases = ["h", "int16", "i2"],
     applevel_types = [],
     T = rffi.SHORT,
     valtype = rffi.SHORT._type,
     expected_size = 2,
 )
-class W_Int16Dtype(IntegerArithmeticDtype, W_Int16Dtype):
+class W_Int16Dtype(SignedIntegerArithmeticDtype, W_Int16Dtype):
+    pass
+
+W_UInt16Dtype = create_low_level_dtype(
+    num = 4, kind = UNSIGNEDLTR, name = "uint16",
+    aliases = ["H", "uint16", "I2"],
+    applevel_types = [],
+    T = rffi.USHORT,
+    valtype = rffi.USHORT._type,
+    expected_size = 2,
+)
+class W_UInt16Dtype(UnsignedIntegerArithmeticDtype, W_UInt16Dtype):
     pass
 
 W_Int32Dtype = create_low_level_dtype(
     num = 5, kind = SIGNEDLTR, name = "int32",
-    aliases = ["i"],
+    aliases = ["i", "int32", "i4"],
     applevel_types = [],
     T = rffi.INT,
     valtype = rffi.INT._type,
     expected_size = 4,
 )
-class W_Int32Dtype(IntegerArithmeticDtype, W_Int32Dtype):
+class W_Int32Dtype(SignedIntegerArithmeticDtype, W_Int32Dtype):
+    pass
+
+W_UInt32Dtype = create_low_level_dtype(
+    num = 6, kind = UNSIGNEDLTR, name = "uint32",
+    aliases = ["I", "uint32", "I4"],
+    applevel_types = [],
+    T = rffi.UINT,
+    valtype = rffi.UINT._type,
+    expected_size = 4,
+)
+class W_UInt32Dtype(UnsignedIntegerArithmeticDtype, W_UInt32Dtype):
     pass
 
 W_Int64Dtype = create_low_level_dtype(
     num = 9, kind = SIGNEDLTR, name = "int64",
-    aliases = [],
+    aliases = ["q", "int64", "i8"],
     applevel_types = ["long"],
     T = rffi.LONGLONG,
     valtype = rffi.LONGLONG._type,
     expected_size = 8,
 )
-class W_Int64Dtype(IntegerArithmeticDtype, W_Int64Dtype):
+class W_Int64Dtype(SignedIntegerArithmeticDtype, W_Int64Dtype):
+    pass
+
+W_UInt64Dtype = create_low_level_dtype(
+    num = 10, kind = UNSIGNEDLTR, name = "uint64",
+    aliases = ["Q", "uint64", "I8"],
+    applevel_types = [],
+    T = rffi.ULONGLONG,
+    valtype = rffi.ULONGLONG._type,
+    expected_size = 8,
+)
+class W_UInt64Dtype(UnsignedIntegerArithmeticDtype, W_UInt64Dtype):
+    pass
+
+if LONG_BIT == 32:
+    class W_LongDtype(W_Int32Dtype):
+        pass
+
+    class W_ULongDtype(W_UInt32Dtype):
+        pass
+else:
+    class W_LongDtype(W_Int64Dtype):
+        pass
+
+    class W_ULongDtype(W_UInt64Dtype):
+        pass
+
+W_LongDtype.num = 7
+W_LongDtype.aliases = ["l"]
+W_LongDtype.applevel_types = ["int"]
+W_ULongDtype.num = 8
+W_ULongDtype.aliases = ["L"]
+
+W_Float32Dtype = create_low_level_dtype(
+    num = 11, kind = FLOATINGLTR, name = "float32",
+    aliases = ["f", "float32", "f4"],
+    applevel_types = [],
+    T = lltype.SingleFloat,
+    valtype = rarithmetic.r_singlefloat,
+    expected_size = 4,
+)
+class W_Float32Dtype(FloatArithmeticDtype, W_Float32Dtype):
     pass
 
 W_Float64Dtype = create_low_level_dtype(
     num = 12, kind = FLOATINGLTR, name = "float64",
-    aliases = [],
+    aliases = ["d", "float64", "f8"],
     applevel_types = ["float"],
     T = lltype.Float,
     valtype = float,
     expected_size = 8,
 )
 class W_Float64Dtype(FloatArithmeticDtype, W_Float64Dtype):
-    def unwrap(self, space, w_item):
-        return self.adapt_val(space.float_w(space.float(w_item)))
-
-    def str_format(self, item):
-        return float2string(self.unbox(item), 'g', rfloat.DTSF_STR_PRECISION)
+    pass
 
 ALL_DTYPES = [
     W_BoolDtype,
-    W_Int8Dtype, W_Int16Dtype, W_Int32Dtype, W_Int64Dtype,
-    W_Float64Dtype
+    W_Int8Dtype, W_UInt8Dtype, W_Int16Dtype, W_UInt16Dtype,
+    W_Int32Dtype, W_UInt32Dtype, W_LongDtype, W_ULongDtype,
+    W_Int64Dtype, W_UInt64Dtype,
+    W_Float32Dtype, W_Float64Dtype,
 ]
-
 dtypes_by_alias = unrolling_iterable([
     (alias, dtype)
     for dtype in ALL_DTYPES
@@ -399,7 +498,7 @@
 
     num = interp_attrproperty("num", cls=W_Dtype),
     kind = interp_attrproperty("kind", cls=W_Dtype),
+    itemsize = interp_attrproperty("num_bytes", cls=W_Dtype),
     shape = GetSetProperty(W_Dtype.descr_get_shape),
-    itemsize = interp_attrproperty("num_bytes", cls=W_Dtype),
 )
 W_Dtype.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/_numpy/interp_ufuncs.py b/pypy/module/_numpy/interp_ufuncs.py
--- a/pypy/module/_numpy/interp_ufuncs.py
+++ b/pypy/module/_numpy/interp_ufuncs.py
@@ -4,6 +4,7 @@
 from pypy.interpreter.typedef import TypeDef, GetSetProperty, interp_attrproperty
 from pypy.module._numpy import interp_dtype, signature
 from pypy.rlib import jit
+from pypy.rlib.rarithmetic import LONG_BIT
 from pypy.tool.sourcetools import func_with_new_name
 
 
@@ -180,23 +181,56 @@
 
     # Everything promotes to float, and bool promotes to everything.
     if dt2.kind == interp_dtype.FLOATINGLTR or dt1.kind == interp_dtype.BOOLLTR:
+        # Float32 + 8-bit int = Float64
+        if dt2.num == 11 and dt1.num_bytes >= 4:
+            return space.fromcache(interp_dtype.W_Float64Dtype)
         return dt2
 
-    assert False
+    # for now this means mixing signed and unsigned
+    if dt2.kind == interp_dtype.SIGNEDLTR:
+        # if dt2 has a greater number of bytes, then just go with it
+        if dt1.num_bytes < dt2.num_bytes:
+            return dt2
+        # we need to promote both dtypes
+        dtypenum = dt2.num + 2
+    else:
+        # increase to the next signed type (or to float)
+        dtypenum = dt2.num + 1
+        # UInt64 + signed = Float64
+        if dt2.num == 10:
+            dtypenum += 1
+    newdtype = interp_dtype.ALL_DTYPES[dtypenum]
+
+    if newdtype.num_bytes > dt2.num_bytes or newdtype.kind == interp_dtype.FLOATINGLTR:
+        return space.fromcache(newdtype)
+    else:
+        # we only promoted to long on 32-bit or to longlong on 64-bit
+        # this is really for dealing with the Long and Ulong dtypes
+        if LONG_BIT == 32:
+            dtypenum += 2
+        else:
+            dtypenum += 3
+        return space.fromcache(interp_dtype.ALL_DTYPES[dtypenum])
 
 def find_unaryop_result_dtype(space, dt, promote_to_float=False,
     promote_bools=False, promote_to_largest=False):
     if promote_bools and (dt.kind == interp_dtype.BOOLLTR):
         return space.fromcache(interp_dtype.W_Int8Dtype)
     if promote_to_float:
+        if dt.kind == interp_dtype.FLOATINGLTR:
+            return dt
+        if dt.num >= 5:
+            return space.fromcache(interp_dtype.W_Float64Dtype)
         for bytes, dtype in interp_dtype.dtypes_by_num_bytes:
-            if dtype.kind == interp_dtype.FLOATINGLTR and dtype.num_bytes >= dt.num_bytes:
+            if dtype.kind == interp_dtype.FLOATINGLTR and dtype.num_bytes > dt.num_bytes:
                 return space.fromcache(dtype)
     if promote_to_largest:
         if dt.kind == interp_dtype.BOOLLTR or dt.kind == interp_dtype.SIGNEDLTR:
             return space.fromcache(interp_dtype.W_Int64Dtype)
         elif dt.kind == interp_dtype.FLOATINGLTR:
             return space.fromcache(interp_dtype.W_Float64Dtype)
+        elif dt.kind == interp_dtype.UNSIGNEDLTR:
+            return space.fromcache(interp_dtype.W_UInt64Dtype)
         else:
             assert False
     return dt
@@ -205,15 +239,23 @@
     w_type = space.type(w_obj)
 
     bool_dtype = space.fromcache(interp_dtype.W_BoolDtype)
+    long_dtype = space.fromcache(interp_dtype.W_LongDtype)
     int64_dtype = space.fromcache(interp_dtype.W_Int64Dtype)
 
     if space.is_w(w_type, space.w_bool):
-        if current_guess is None:
+        if current_guess is None or current_guess is bool_dtype:
             return bool_dtype
+        return current_guess
     elif space.is_w(w_type, space.w_int):
         if (current_guess is None or current_guess is bool_dtype or
-            current_guess is int64_dtype):
-            return int64_dtype
+            current_guess is long_dtype):
+            return long_dtype
+        return current_guess
+    elif space.is_w(w_type, space.w_long):
+        if (current_guess is None or current_guess is bool_dtype or
+            current_guess is long_dtype or current_guess is int64_dtype):
+             return int64_dtype
+        return current_guess
     return space.fromcache(interp_dtype.W_Float64Dtype)
 
 
@@ -225,7 +267,9 @@
         def impl(res_dtype, lvalue, rvalue):
             res = getattr(res_dtype, op_name)(lvalue, rvalue)
             if comparison_func:
-                res = space.fromcache(interp_dtype.W_BoolDtype).box(res)
+                booldtype = space.fromcache(interp_dtype.W_BoolDtype)
+                assert isinstance(booldtype, interp_dtype.W_BoolDtype)
+                res = booldtype.box(res)
             return res
     return func_with_new_name(impl, ufunc_name)
 
@@ -269,6 +313,7 @@
             ("arccos", "arccos", 1, {"promote_to_float": True}),
             ("arctan", "arctan", 1, {"promote_to_float": True}),
             ("arcsinh", "arcsinh", 1, {"promote_to_float": True}),
+            ("arctanh", "arctanh", 1, {"promote_to_float": True}),
         ]:
             self.add_ufunc(space, *ufunc_def)
 
@@ -278,7 +323,7 @@
 
         identity = extra_kwargs.get("identity")
         if identity is not None:
-            identity = space.fromcache(interp_dtype.W_Int64Dtype).adapt_val(identity)
+            identity = space.fromcache(interp_dtype.W_LongDtype).adapt_val(identity)
         extra_kwargs["identity"] = identity
 
         func = ufunc_dtype_caller(space, ufunc_name, op_name, argcount,
diff --git a/pypy/module/_numpy/test/test_base.py b/pypy/module/_numpy/test/test_base.py
--- a/pypy/module/_numpy/test/test_base.py
+++ b/pypy/module/_numpy/test/test_base.py
@@ -64,18 +64,46 @@
     def test_unaryops(self, space):
         bool_dtype = space.fromcache(interp_dtype.W_BoolDtype)
         int8_dtype = space.fromcache(interp_dtype.W_Int8Dtype)
+        uint8_dtype = space.fromcache(interp_dtype.W_UInt8Dtype)
+        int16_dtype = space.fromcache(interp_dtype.W_Int16Dtype)
+        uint16_dtype = space.fromcache(interp_dtype.W_UInt16Dtype)
         int32_dtype = space.fromcache(interp_dtype.W_Int32Dtype)
+        uint32_dtype = space.fromcache(interp_dtype.W_UInt32Dtype)
+        long_dtype = space.fromcache(interp_dtype.W_LongDtype)
+        ulong_dtype = space.fromcache(interp_dtype.W_ULongDtype)
+        int64_dtype = space.fromcache(interp_dtype.W_Int64Dtype)
+        uint64_dtype = space.fromcache(interp_dtype.W_UInt64Dtype)
+        float32_dtype = space.fromcache(interp_dtype.W_Float32Dtype)
         float64_dtype = space.fromcache(interp_dtype.W_Float64Dtype)
 
-        # Normal rules, everythign returns itself
+        # Normal rules, everything returns itself
         assert find_unaryop_result_dtype(space, bool_dtype) is bool_dtype
         assert find_unaryop_result_dtype(space, int8_dtype) is int8_dtype
+        assert find_unaryop_result_dtype(space, uint8_dtype) is uint8_dtype
+        assert find_unaryop_result_dtype(space, int16_dtype) is int16_dtype
+        assert find_unaryop_result_dtype(space, uint16_dtype) is uint16_dtype
         assert find_unaryop_result_dtype(space, int32_dtype) is int32_dtype
+        assert find_unaryop_result_dtype(space, uint32_dtype) is uint32_dtype
+        assert find_unaryop_result_dtype(space, long_dtype) is long_dtype
+        assert find_unaryop_result_dtype(space, ulong_dtype) is ulong_dtype
+        assert find_unaryop_result_dtype(space, int64_dtype) is int64_dtype
+        assert find_unaryop_result_dtype(space, uint64_dtype) is uint64_dtype
+        assert find_unaryop_result_dtype(space, float32_dtype) is float32_dtype
         assert find_unaryop_result_dtype(space, float64_dtype) is float64_dtype
 
         # Coerce to floats, some of these will eventually be float16, or
         # whatever our smallest float type is.
-        assert find_unaryop_result_dtype(space, bool_dtype, promote_to_float=True) is float64_dtype
-        assert find_unaryop_result_dtype(space, int8_dtype, promote_to_float=True) is float64_dtype
+        assert find_unaryop_result_dtype(space, bool_dtype, promote_to_float=True) is float32_dtype # will be float16 if we ever put that in
+        assert find_unaryop_result_dtype(space, int8_dtype, promote_to_float=True) is float32_dtype # will be float16 if we ever put that in
+        assert find_unaryop_result_dtype(space, uint8_dtype, promote_to_float=True) is float32_dtype # will be float16 if we ever put that in
+        assert find_unaryop_result_dtype(space, int16_dtype, promote_to_float=True) is float32_dtype
+        assert find_unaryop_result_dtype(space, uint16_dtype, promote_to_float=True) is float32_dtype
         assert find_unaryop_result_dtype(space, int32_dtype, promote_to_float=True) is float64_dtype
+        assert find_unaryop_result_dtype(space, uint32_dtype, promote_to_float=True) is float64_dtype
+        assert find_unaryop_result_dtype(space, int64_dtype, promote_to_float=True) is float64_dtype
+        assert find_unaryop_result_dtype(space, uint64_dtype, promote_to_float=True) is float64_dtype
+        assert find_unaryop_result_dtype(space, float32_dtype, promote_to_float=True) is float32_dtype
         assert find_unaryop_result_dtype(space, float64_dtype, promote_to_float=True) is float64_dtype
+
+        # promote bools, happens with sign ufunc
+        assert find_unaryop_result_dtype(space, bool_dtype, promote_bools=True) is int8_dtype
diff --git a/pypy/module/_numpy/test/test_dtypes.py b/pypy/module/_numpy/test/test_dtypes.py
--- a/pypy/module/_numpy/test/test_dtypes.py
+++ b/pypy/module/_numpy/test/test_dtypes.py
@@ -17,6 +17,7 @@
         from _numpy import dtype
 
         assert dtype(bool).num == 0
+        assert dtype(int).num == 7
         assert dtype(long).num == 9
         assert dtype(float).num == 12
 
@@ -81,6 +82,48 @@
             assert isinstance(a[i], (int, long))
             assert a[1] == 1
 
+    def test_overflow(self):
+        from _numpy import array, dtype
+        assert array([128], 'b')[0] == -128
+        assert array([256], 'B')[0] == 0
+        assert array([32768], 'h')[0] == -32768
+        assert array([65536], 'H')[0] == 0
+        if dtype('l').itemsize == 4: # 32-bit
+            raises(OverflowError, "array([2**32/2], 'i')")
+            raises(OverflowError, "array([2**32], 'I')")
+        raises(OverflowError, "array([2**64/2], 'q')")
+        raises(OverflowError, "array([2**64], 'Q')")
+
+    def test_bool_binop_types(self):
+        from _numpy import array, dtype
+        types = ('?','b','B','h','H','i','I','l','L','q','Q','f','d')
+        N = len(types)
+        a = array([True], '?')
+        for t in types:
+            assert (a + array([0], t)).dtype is dtype(t)
+
+    def test_binop_types(self):
+        from _numpy import array, dtype
+        tests = [('b','B','h'), ('b','h','h'), ('b','H','i'), ('b','i','i'),
+                 ('b','l','l'), ('b','q','q'), ('b','Q','d'), ('B','h','h'),
+                 ('B','H','H'), ('B','i','i'), ('B','I','I'), ('B','l','l'),
+                 ('B','L','L'), ('B','q','q'), ('B','Q','Q'), ('h','H','i'),
+                 ('h','i','i'), ('h','l','l'), ('h','q','q'), ('h','Q','d'),
+                 ('H','i','i'), ('H','I','I'), ('H','l','l'), ('H','L','L'),
+                 ('H','q','q'), ('H','Q','Q'), ('i','l','l'), ('i','q','q'),
+                 ('i','Q','d'), ('I','L','L'), ('I','q','q'), ('I','Q','Q'),
+                 ('q','Q','d'), ('b','f','f'), ('B','f','f'), ('h','f','f'),
+                 ('H','f','f'), ('i','f','d'), ('I','f','d'), ('l','f','d'),
+                 ('L','f','d'), ('q','f','d'), ('Q','f','d'), ('q','d','d')]
+        if dtype('i').itemsize == dtype('l').itemsize: # 32-bit
+            tests.extend([('b','I','q'), ('b','L','q'), ('h','I','q'),
+                          ('h','L','q'), ('i','I','q'), ('i','L','q')])
+        else:
+            tests.extend([('b','I','l'), ('b','L','d'), ('h','I','l'),
+                          ('h','L','d'), ('i','I','l'), ('i','L','d')])
+        for d1, d2, dout in tests:
+            assert (array([1], d1) + array([1], d2)).dtype is dtype(dout)
+
     def test_add_int8(self):
         from _numpy import array, dtype
 
@@ -99,6 +142,15 @@
         for i in range(5):
             assert b[i] == i * 2
 
+    def test_add_uint32(self):
+        from _numpy import array, dtype
+
+        a = array(range(5), dtype="I")
+        b = a + a
+        assert b.dtype is dtype("I")
+        for i in range(5):
+            assert b[i] == i * 2
+
     def test_shape(self):
         from _numpy import dtype
 
diff --git a/pypy/module/_numpy/test/test_numarray.py b/pypy/module/_numpy/test/test_numarray.py
--- a/pypy/module/_numpy/test/test_numarray.py
+++ b/pypy/module/_numpy/test/test_numarray.py
@@ -18,7 +18,7 @@
         assert a[13] == 5.3
 
     def test_zeros_tuple_shape(self):
-        from numpy import zeros, ones, empty
+        from _numpy import zeros, ones, empty
         a = zeros((15,))
         assert len(a) == 15
         b = ones((3,))
@@ -560,8 +560,10 @@
         from _numpy import array, dtype
 
         assert array([True]).dtype is dtype(bool)
-        assert array([True, 1]).dtype is dtype(long)
-        assert array([1, 2, 3]).dtype is dtype(long)
+        assert array([True, False]).dtype is dtype(bool)
+        assert array([True, 1]).dtype is dtype(int)
+        assert array([1, 2, 3]).dtype is dtype(int)
+        assert array([1L, 2, 3]).dtype is dtype(long)
         assert array([1.2, True]).dtype is dtype(float)
         assert array([1.2, 5]).dtype is dtype(float)
         assert array([]).dtype is dtype(float)
diff --git a/pypy/module/_numpy/test/test_ufuncs.py b/pypy/module/_numpy/test/test_ufuncs.py
--- a/pypy/module/_numpy/test/test_ufuncs.py
+++ b/pypy/module/_numpy/test/test_ufuncs.py
@@ -234,7 +234,7 @@
             assert b[i] == math.sin(a[i])
 
         a = sin(array([True, False], dtype=bool))
-        assert a[0] == sin(1)
+        assert abs(a[0] - sin(1)) < 1e-7 # a[0] will be less precise
         assert a[1] == 0.0
 
     def test_cos(self):
@@ -300,12 +300,24 @@
 
     def test_arcsinh(self):
         import math
-        from numpy import arcsinh, inf
+        from _numpy import arcsinh
+        from numpy import inf
 
         for v in [inf, -inf, 1.0, math.e]:
             assert math.asinh(v) == arcsinh(v)
         assert math.isnan(arcsinh(float("nan")))
 
+    def test_arctanh(self):
+        import math
+        from numpy import arctanh
+
+        for v in [.99, .5, 0, -.5, -.99]:
+            assert math.atanh(v) == arctanh(v)
+        for v in [2.0, -2.0]:
+            assert math.isnan(arctanh(v))
+        for v in [1.0, -1.0]:
+            assert arctanh(v) == math.copysign(float("inf"), v)
+
     def test_reduce_errors(self):
         from _numpy import sin, add
 
diff --git a/pypy/module/_numpy/test/test_zjit.py b/pypy/module/_numpy/test/test_zjit.py
--- a/pypy/module/_numpy/test/test_zjit.py
+++ b/pypy/module/_numpy/test/test_zjit.py
@@ -1,20 +1,24 @@
 from pypy.jit.metainterp.test.support import LLJitMixin
 from pypy.module._numpy import interp_ufuncs, signature
 from pypy.module._numpy.compile import (numpy_compile, FakeSpace,
-    FloatObject)
-from pypy.module._numpy.interp_dtype import W_Float64Dtype, W_Int64Dtype
+    FloatObject, IntObject)
+from pypy.module._numpy.interp_dtype import W_Int32Dtype, W_Float64Dtype, W_Int64Dtype, W_UInt64Dtype
 from pypy.module._numpy.interp_numarray import (BaseArray, SingleDimArray,
     SingleDimSlice, scalar_w)
 from pypy.rlib.nonconst import NonConstant
 from pypy.rpython.annlowlevel import llstr
 from pypy.rpython.test.test_llinterp import interpret
 
+import py
+
 
 class TestNumpyJIt(LLJitMixin):
     def setup_class(cls):
         cls.space = FakeSpace()
         cls.float64_dtype = cls.space.fromcache(W_Float64Dtype)
         cls.int64_dtype = cls.space.fromcache(W_Int64Dtype)
+        cls.uint64_dtype = cls.space.fromcache(W_UInt64Dtype)
+        cls.int32_dtype = cls.space.fromcache(W_Int32Dtype)
 
     def test_add(self):
         def f(i):
@@ -303,6 +307,31 @@
                           'int_lt': 1, 'guard_true': 1, 'jump': 1})
         assert result == 11.0
 
+    def test_int32_sum(self):
+        py.test.skip("pypy/jit/backend/llimpl.py needs to be changed to "
+                     "deal correctly with int dtypes for this test to "
+                     "work. skip for now until someone feels up to the task")
+        space = self.space
+        float64_dtype = self.float64_dtype
+        int32_dtype = self.int32_dtype
+
+        def f(n):
+            if NonConstant(False):
+                dtype = float64_dtype
+            else:
+                dtype = int32_dtype
+            ar = SingleDimArray(n, dtype=dtype)
+            i = 0
+            while i < n:
+                ar.get_concrete().setitem(i, int32_dtype.box(7))
+                i += 1
+            v = ar.descr_add(space, ar).descr_sum(space)
+            assert isinstance(v, IntObject)
+            return v.intval
+
+        result = self.meta_interp(f, [5], listops=True, backendopt=True)
+        assert result == f(5)
+
 class TestTranslation(object):
     def test_compile(self):
         x = numpy_compile('aa+f*f/a-', 10)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_00_model.py b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
--- a/pypy/module/pypyjit/test_pypy_c/test_00_model.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
@@ -50,8 +50,7 @@
         cmdline.append(str(self.filepath))
         #
         print cmdline, logfile
-        env={'PYPYLOG': 'jit-log-opt,jit-summary:' + str(logfile)}
-        #env={'PYPYLOG': ':' + str(logfile)}
+        env={'PYPYLOG': 'jit-log-opt,jit-log-noopt,jit-summary:' + str(logfile)}
         pipe = subprocess.Popen(cmdline,
                                 env=env,
                                 stdout=subprocess.PIPE,
diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py
--- a/pypy/module/pypyjit/test_pypy_c/test_call.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_call.py
@@ -366,12 +366,12 @@
             # make sure that the "block" is not allocated
             ...
             i20 = force_token()
-            setfield_gc(p0, i20, descr=<SignedFieldDescr .*PyFrame.vable_token .*>)
             p22 = new_with_vtable(19511408)
             p24 = new_array(1, descr=<GcPtrArrayDescr>)
             p26 = new_with_vtable(ConstClass(W_ListObject))
             p27 = new(descr=<SizeDescr .*>)
             p29 = new_array(0, descr=<GcPtrArrayDescr>)
+            setfield_gc(p0, i20, descr=<SignedFieldDescr .*PyFrame.vable_token .*>)
             setfield_gc(p27, p29, descr=<GcPtrFieldDescr list.items .*>)
             setfield_gc(p26, p27, descr=<.* .*W_ListObject.inst_wrappeditems .*>)
             setarrayitem_gc(p24, 0, p26, descr=<GcPtrArrayDescr>)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_generators.py b/pypy/module/pypyjit/test_pypy_c/test_generators.py
--- a/pypy/module/pypyjit/test_pypy_c/test_generators.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_generators.py
@@ -19,8 +19,8 @@
         assert loop.match_by_id("generator", """
             i16 = force_token()
             p45 = new_with_vtable(ConstClass(W_IntObject))
-            setfield_gc(p45, i29, descr=<SignedFieldDescr .*>)
             i47 = arraylen_gc(p8, descr=<GcPtrArrayDescr>) # Should be removed by backend
             setarrayitem_gc(p8, 0, p45, descr=<GcPtrArrayDescr>)
+            setfield_gc(p45, i29, descr=<SignedFieldDescr .*>)
             jump(..., descr=...)
             """)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_instance.py b/pypy/module/pypyjit/test_pypy_c/test_instance.py
--- a/pypy/module/pypyjit/test_pypy_c/test_instance.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_instance.py
@@ -125,8 +125,8 @@
             i12 = force_token()
             --TICK--
             p20 = new_with_vtable(ConstClass(W_IntObject))
+            setfield_gc(ConstPtr(ptr21), p20, descr=<GcPtrFieldDescr .*TypeCell.inst_w_value .*>)
             setfield_gc(p20, i11, descr=<SignedFieldDescr.*W_IntObject.inst_intval .*>)
-            setfield_gc(ConstPtr(ptr21), p20, descr=<GcPtrFieldDescr .*TypeCell.inst_w_value .*>)
             jump(p0, p1, p2, p3, p4, p20, p6, i7, p20, descr=<Loop.>)
         """)
 
diff --git a/pypy/module/thread/gil.py b/pypy/module/thread/gil.py
--- a/pypy/module/thread/gil.py
+++ b/pypy/module/thread/gil.py
@@ -16,7 +16,7 @@
 
 class GILThreadLocals(OSThreadLocals):
     """A version of OSThreadLocals that enforces a GIL."""
-    ll_GIL = thread.null_ll_lock
+    gil_ready = False
 
     def initialize(self, space):
         # add the GIL-releasing callback as an action on the space
@@ -25,12 +25,10 @@
 
     def setup_threads(self, space):
         """Enable threads in the object space, if they haven't already been."""
-        if not self.ll_GIL:
-            try:
-                self.ll_GIL = thread.allocate_ll_lock()
-            except thread.error:
+        if not self.gil_ready:
+            if not thread.gil_allocate():
                 raise wrap_thread_error(space, "can't allocate GIL")
-            thread.acquire_NOAUTO(self.ll_GIL, True)
+            self.gil_ready = True
             self.enter_thread(space)   # setup the main thread
             result = True
         else:
@@ -44,19 +42,16 @@
         # test_lock_again after the global state was cleared by
         # test_compile_lock.  As a workaround, we repatch these global
         # fields systematically.
-        spacestate.ll_GIL = self.ll_GIL
         invoke_around_extcall(before_external_call, after_external_call)
         return result
 
     def reinit_threads(self, space):
-        if self.ll_GIL:
-            self.ll_GIL = thread.allocate_ll_lock()
-            thread.acquire_NOAUTO(self.ll_GIL, True)
-            self.enter_thread(space)
+        if self.gil_ready:
+            self.gil_ready = False
+            self.setup_threads(space)
 
     def yield_thread(self):
-        thread.yield_thread()  # explicitly release the gil (used by test_gil)
-
+        do_yield_thread()
 
 class GILReleaseAction(PeriodicAsyncAction):
     """An action called every sys.checkinterval bytecodes.  It releases
@@ -64,16 +59,12 @@
     """
 
     def perform(self, executioncontext, frame):
-        # Other threads can run between the release() and the acquire()
-        # implicit in the following external function call (which has
-        # otherwise no effect).
-        thread.yield_thread()
+        do_yield_thread()
 
 
 class SpaceState:
 
     def _freeze_(self):
-        self.ll_GIL = thread.null_ll_lock
         self.action_after_thread_switch = None
         # ^^^ set by AsyncAction.fire_after_thread_switch()
         return False
@@ -95,14 +86,14 @@
     # this function must not raise, in such a way that the exception
     # transformer knows that it cannot raise!
     e = get_errno()
-    thread.release_NOAUTO(spacestate.ll_GIL)
+    thread.gil_release()
     set_errno(e)
 before_external_call._gctransformer_hint_cannot_collect_ = True
 before_external_call._dont_reach_me_in_del_ = True
 
 def after_external_call():
     e = get_errno()
-    thread.acquire_NOAUTO(spacestate.ll_GIL, True)
+    thread.gil_acquire()
     thread.gc_thread_run()
     spacestate.after_thread_switch()
     set_errno(e)
@@ -115,3 +106,18 @@
 # pointers in the shadow stack.  This is necessary because the GIL is
 # not held after the call to before_external_call() or before the call
 # to after_external_call().
+
+def do_yield_thread():
+    # explicitly release the gil, in a way that tries to give more
+    # priority to other threads (as opposed to continuing to run in
+    # the same thread).
+    if thread.gil_yield_thread():
+        thread.gc_thread_run()
+        spacestate.after_thread_switch()
+do_yield_thread._gctransformer_hint_close_stack_ = True
+do_yield_thread._dont_reach_me_in_del_ = True
+do_yield_thread._dont_inline_ = True
+
+# do_yield_thread() needs a different hint: _gctransformer_hint_close_stack_.
+# The *_external_call() functions are themselves called only from the rffi
+# module from a helper function that also has this hint.
diff --git a/pypy/module/thread/ll_thread.py b/pypy/module/thread/ll_thread.py
--- a/pypy/module/thread/ll_thread.py
+++ b/pypy/module/thread/ll_thread.py
@@ -17,7 +17,8 @@
     include_dirs = [str(py.path.local(autopath.pypydir).join('translator', 'c'))],
     export_symbols = ['RPyThreadGetIdent', 'RPyThreadLockInit',
                       'RPyThreadAcquireLock', 'RPyThreadReleaseLock',
-                      'RPyThreadYield',
+                      'RPyGilAllocate', 'RPyGilYieldThread',
+                      'RPyGilRelease', 'RPyGilAcquire',
                       'RPyThreadGetStackSize', 'RPyThreadSetStackSize',
                       'RPyOpaqueDealloc_ThreadLock',
                       'RPyThreadAfterFork']
@@ -69,8 +70,16 @@
                                          [TLOCKP], lltype.Void,
                                          _nowrapper=True)
 
-# this function does nothing apart from releasing the GIL temporarily.
-yield_thread = llexternal('RPyThreadYield', [], lltype.Void, threadsafe=True)
+# these functions manipulate directly the GIL, whose definition does not
+# escape the C code itself
+gil_allocate     = llexternal('RPyGilAllocate', [], lltype.Signed,
+                              _nowrapper=True)
+gil_yield_thread = llexternal('RPyGilYieldThread', [], lltype.Signed,
+                              _nowrapper=True)
+gil_release      = llexternal('RPyGilRelease', [], lltype.Void,
+                              _nowrapper=True)
+gil_acquire      = llexternal('RPyGilAcquire', [], lltype.Void,
+                              _nowrapper=True)
 
 def allocate_lock():
     return Lock(allocate_ll_lock())
diff --git a/pypy/module/thread/test/test_gil.py b/pypy/module/thread/test/test_gil.py
--- a/pypy/module/thread/test/test_gil.py
+++ b/pypy/module/thread/test/test_gil.py
@@ -30,19 +30,34 @@
     use_threads = True
     bigtest = False
 
-    def test_one_thread(self):
+    def test_one_thread(self, skew=+1):
+        from pypy.rlib.debug import debug_print
         if self.bigtest:
-            N = 1000000
+            N = 100000
+            skew *= 25000
         else:
             N = 100
+            skew *= 25
         space = FakeSpace()
         class State:
             pass
         state = State()
-        def runme():
-            for i in range(N):
+        def runme(main=False):
+            j = 0
+            for i in range(N + [-skew, skew][main]):
+                state.datalen1 += 1   # try to crash if the GIL is not
+                state.datalen2 += 1   # correctly acquired
                 state.data.append((thread.get_ident(), i))
+                state.datalen3 += 1
+                state.datalen4 += 1
+                assert state.datalen1 == len(state.data)
+                assert state.datalen2 == len(state.data)
+                assert state.datalen3 == len(state.data)
+                assert state.datalen4 == len(state.data)
+                debug_print(main, i, state.datalen4)
                 state.threadlocals.yield_thread()
+                assert i == j
+                j += 1
         def bootstrap():
             try:
                 runme()
@@ -50,20 +65,26 @@
                 thread.gc_thread_die()
         def f():
             state.data = []
+            state.datalen1 = 0
+            state.datalen2 = 0
+            state.datalen3 = 0
+            state.datalen4 = 0
             state.threadlocals = gil.GILThreadLocals()
             state.threadlocals.setup_threads(space)
             thread.gc_thread_prepare()
             subident = thread.start_new_thread(bootstrap, ())
             mainident = thread.get_ident()
-            runme()
+            runme(True)
             still_waiting = 3000
             while len(state.data) < 2*N:
+                debug_print(len(state.data))
                 if not still_waiting:
                     raise ValueError("time out")
                 still_waiting -= 1
                 if not we_are_translated(): gil.before_external_call()
                 time.sleep(0.01)
                 if not we_are_translated(): gil.after_external_call()
+            debug_print("leaving!")
             i1 = i2 = 0
             for tid, i in state.data:
                 if tid == mainident:
@@ -72,14 +93,17 @@
                     assert i == i2; i2 += 1
                 else:
                     assert 0
-            assert i1 == N
-            assert i2 == N
+            assert i1 == N + skew
+            assert i2 == N - skew
             return len(state.data)
 
         fn = self.getcompiled(f, [])
         res = fn()
         assert res == 2*N
 
+    def test_one_thread_rev(self):
+        self.test_one_thread(skew=-1)
+
 
 class TestRunDirectly(GILTests):
     def getcompiled(self, f, argtypes):
diff --git a/pypy/module/thread/test/test_thread.py b/pypy/module/thread/test/test_thread.py
--- a/pypy/module/thread/test/test_thread.py
+++ b/pypy/module/thread/test/test_thread.py
@@ -225,7 +225,8 @@
 
         def busy_wait():
             for x in range(1000):
-                time.sleep(0.01)
+                print 'tick...', x  # <-force the GIL to be released, as
+                time.sleep(0.01)    #   time.sleep doesn't do non-translated
 
         # This is normally called by app_main.py
         signal.signal(signal.SIGINT, signal.default_int_handler)
diff --git a/pypy/rpython/lltypesystem/opimpl.py b/pypy/rpython/lltypesystem/opimpl.py
--- a/pypy/rpython/lltypesystem/opimpl.py
+++ b/pypy/rpython/lltypesystem/opimpl.py
@@ -357,7 +357,7 @@
 
 def op_cast_float_to_uint(f):
     assert type(f) is float
-    return r_uint(int(f))
+    return r_uint(long(f))
 
 def op_cast_float_to_longlong(f):
     assert type(f) is float
@@ -369,7 +369,7 @@
 
 def op_cast_float_to_ulonglong(f):
     assert type(f) is float
-    return r_ulonglong(r_longlong(f))
+    return r_ulonglong(long(f))
 
 def op_cast_char_to_int(b):
     assert type(b) is str and len(b) == 1
diff --git a/pypy/rpython/lltypesystem/test/test_lloperation.py b/pypy/rpython/lltypesystem/test/test_lloperation.py
--- a/pypy/rpython/lltypesystem/test/test_lloperation.py
+++ b/pypy/rpython/lltypesystem/test/test_lloperation.py
@@ -5,6 +5,7 @@
 from pypy.rpython.llinterp import LLFrame
 from pypy.rpython.test.test_llinterp import interpret
 from pypy.rpython import rclass
+from pypy.rlib.rarithmetic import LONGLONG_MASK, r_longlong, r_ulonglong
 
 LL_INTERP_OPERATIONS = [name[3:] for name in LLFrame.__dict__.keys()
                                  if name.startswith('op_')]
@@ -133,6 +134,14 @@
         py.test.raises(TypeError, llop.getinteriorfield,
                        lltype.Signed, s3, 'y')
 
+def test_cast_float_to_ulonglong():
+    f = 12350000000000000000.0
+    py.test.raises(OverflowError, r_longlong, f)
+    r_longlong(f / 2)   # does not raise OverflowError
+    #
+    x = llop.cast_float_to_ulonglong(lltype.UnsignedLongLong, f)
+    assert x == r_ulonglong(f)
+
 # ___________________________________________________________________________
 # This tests that the LLInterpreter and the LL_OPERATIONS tables are in sync.
 
diff --git a/pypy/translator/c/gcc/trackgcroot.py b/pypy/translator/c/gcc/trackgcroot.py
--- a/pypy/translator/c/gcc/trackgcroot.py
+++ b/pypy/translator/c/gcc/trackgcroot.py
@@ -486,6 +486,8 @@
         'paddq', 'pinsr',
         # zero-extending moves should not produce GC pointers
         'movz', 
+        # locked operations should not move GC pointers, at least so far
+        'lock',
         ])
 
     # a partial list is hopefully good enough for now; it's all to support
diff --git a/pypy/translator/c/src/thread.h b/pypy/translator/c/src/thread.h
--- a/pypy/translator/c/src/thread.h
+++ b/pypy/translator/c/src/thread.h
@@ -37,14 +37,9 @@
 
 #endif
 
-/* common helper: this does nothing, but is called with the GIL released.
-   This gives other threads a chance to grab the GIL and run. */
-void RPyThreadYield(void);
-
-#ifndef PYPY_NOT_MAIN_FILE
-void RPyThreadYield(void)
-{
-}
-#endif
+long RPyGilAllocate(void);
+long RPyGilYieldThread(void);
+void RPyGilRelease(void);
+void RPyGilAcquire(void);
 
 #endif
diff --git a/pypy/translator/c/src/thread_nt.h b/pypy/translator/c/src/thread_nt.h
--- a/pypy/translator/c/src/thread_nt.h
+++ b/pypy/translator/c/src/thread_nt.h
@@ -221,4 +221,57 @@
 #define RPyThreadTLS_Set(key, value)	TlsSetValue(key, value)
 
 
+/************************************************************/
+/* GIL code                                                 */
+/************************************************************/
+
+static volatile LONG pending_acquires = -1;
+static CRITICAL_SECTION mutex_gil;
+static HANDLE cond_gil;
+
+long RPyGilAllocate(void)
+{
+    pending_acquires = 0;
+    InitializeCriticalSection(&mutex_gil);
+    EnterCriticalSection(&mutex_gil);
+    cond_gil = CreateEvent (NULL, FALSE, FALSE, NULL);
+    return 1;
+}
+
+long RPyGilYieldThread(void)
+{
+    /* can be called even before RPyGilAllocate(), but in this case,
+       pending_acquires will be -1 */
+    if (pending_acquires <= 0)
+        return 0;
+    InterlockedIncrement(&pending_acquires);
+    PulseEvent(&cond_gil);
+
+    /* hack: the three following lines do a pthread_cond_wait(), and
+       normally specifying a timeout of INFINITE would be fine.  But the
+       first and second operations are not done atomically, so there is a
+       (small) risk that PulseEvent misses the WaitForSingleObject().
+       In this case the process will just sleep a few milliseconds. */
+    LeaveCriticalSection(&mutex_gil);
+    WaitForSingleObject(&cond_gil, 15);
+    EnterCriticalSection(&mutex_gil);
+
+    InterlockedDecrement(&pending_acquires);
+    return 1;
+}
+
+void RPyGilRelease(void)
+{
+    LeaveCriticalSection(&mutex_gil);
+    PulseEvent(&cond_gil);
+}
+
+void RPyGilAcquire(void)
+{
+    InterlockedIncrement(&pending_acquires);
+    EnterCriticalSection(&mutex_gil);
+    InterlockedDecrement(&pending_acquires);
+}
+
+
 #endif /* PYPY_NOT_MAIN_FILE */
diff --git a/pypy/translator/c/src/thread_pthread.h b/pypy/translator/c/src/thread_pthread.h
--- a/pypy/translator/c/src/thread_pthread.h
+++ b/pypy/translator/c/src/thread_pthread.h
@@ -12,6 +12,7 @@
 #include <signal.h>
 #include <stdio.h>
 #include <errno.h>
+#include <assert.h>
 
 /* The following is hopefully equivalent to what CPython does
    (which is trying to compile a snippet of code using it) */
@@ -459,4 +460,113 @@
 #define RPyThreadTLS_Set(key, value)	pthread_setspecific(key, value)
 
 
+/************************************************************/
+/* GIL code                                                 */
+/************************************************************/
+
+#ifdef __llvm__
+#  define HAS_ATOMIC_ADD
+#endif
+
+#ifdef __GNUC__
+#  if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 1)
+#    define HAS_ATOMIC_ADD
+#  endif
+#endif
+
+#ifdef HAS_ATOMIC_ADD
+#  define atomic_add __sync_fetch_and_add
+#else
+#  if defined(__amd64__)
+#    define atomic_add(ptr, value)  asm volatile ("lock addq %0, %1"        \
+                                 : : "ri"(value), "m"(*(ptr)) : "memory")
+#  elif defined(__i386__)
+#    define atomic_add(ptr, value)  asm volatile ("lock addl %0, %1"        \
+                                 : : "ri"(value), "m"(*(ptr)) : "memory")
+#  else
+#    error "Please use gcc >= 4.1 or write a custom 'asm' for your CPU."
+#  endif
+#endif
+
+#define ASSERT_STATUS(call)                             \
+    if (call != 0) {                                    \
+        fprintf(stderr, "Fatal error: " #call "\n");    \
+        abort();                                        \
+    }
+
+static void _debug_print(const char *msg)
+{
+#if 0
+    int col = (int)pthread_self();
+    col = 31 + ((col / 8) % 8);
+    fprintf(stderr, "\033[%dm%s\033[0m", col, msg);
+#endif
+}
+
+static volatile long pending_acquires = -1;
+static pthread_mutex_t mutex_gil = PTHREAD_MUTEX_INITIALIZER;
+static pthread_cond_t cond_gil = PTHREAD_COND_INITIALIZER;
+
+static void assert_has_the_gil(void)
+{
+#ifdef RPY_ASSERT
+    assert(pthread_mutex_trylock(&mutex_gil) != 0);
+    assert(pending_acquires >= 0);
+#endif
+}
+
+long RPyGilAllocate(void)
+{
+    _debug_print("RPyGilAllocate\n");
+    pending_acquires = 0;
+    pthread_mutex_trylock(&mutex_gil);
+    assert_has_the_gil();
+    return 1;
+}
+
+long RPyGilYieldThread(void)
+{
+    /* can be called even before RPyGilAllocate(), but in this case,
+       pending_acquires will be -1 */
+#ifdef RPY_ASSERT
+    if (pending_acquires >= 0)
+        assert_has_the_gil();
+#endif
+    if (pending_acquires <= 0)
+        return 0;
+    atomic_add(&pending_acquires, 1L);
+    _debug_print("{");
+    ASSERT_STATUS(pthread_cond_signal(&cond_gil));
+    ASSERT_STATUS(pthread_cond_wait(&cond_gil, &mutex_gil));
+    _debug_print("}");
+    atomic_add(&pending_acquires, -1L);
+    assert_has_the_gil();
+    return 1;
+}
+
+void RPyGilRelease(void)
+{
+    _debug_print("RPyGilRelease\n");
+#ifdef RPY_ASSERT
+    assert(pending_acquires >= 0);
+#endif
+    assert_has_the_gil();
+    ASSERT_STATUS(pthread_mutex_unlock(&mutex_gil));
+    ASSERT_STATUS(pthread_cond_signal(&cond_gil));
+}
+
+void RPyGilAcquire(void)
+{
+    _debug_print("about to RPyGilAcquire...\n");
+#ifdef RPY_ASSERT
+    assert(pending_acquires >= 0);
+#endif
+    atomic_add(&pending_acquires, 1L);
+    ASSERT_STATUS(pthread_mutex_lock(&mutex_gil));
+    atomic_add(&pending_acquires, -1L);
+    assert_has_the_gil();
+    _debug_print("RPyGilAcquire\n");
+}
+
+
 #endif /* PYPY_NOT_MAIN_FILE */

From noreply at buildbot.pypy.org  Tue Oct  4 00:40:58 2011
From: noreply at buildbot.pypy.org (timo_jbo)
Date: Tue,  4 Oct 2011 00:40:58 +0200 (CEST)
Subject: [pypy-commit] pypy separate-applevel-numpy: import arctanh and
 arcsinh from _numpy into numpy, too.
Message-ID: <20111003224058.53862820D9@wyvern.cs.uni-duesseldorf.de>

Author: Timo Paulssen <timonator at perpetuum-immobile.de>
Branch: separate-applevel-numpy
Changeset: r47815:f3a0c7efb8c5
Date: 2011-10-04 00:39 +0200
http://bitbucket.org/pypy/pypy/changeset/f3a0c7efb8c5/

Log:	import arctanh and arcsinh from _numpy into numpy, too.

diff --git a/lib_pypy/numpy/__init__.py b/lib_pypy/numpy/__init__.py
--- a/lib_pypy/numpy/__init__.py
+++ b/lib_pypy/numpy/__init__.py
@@ -14,6 +14,8 @@
         arccos,
         arcsin,
         arctan,
+        arctanh,
+        arcsinh,
         copysign,
         cos,
         divide,
diff --git a/pypy/module/_numpy/test/test_ufuncs.py b/pypy/module/_numpy/test/test_ufuncs.py
--- a/pypy/module/_numpy/test/test_ufuncs.py
+++ b/pypy/module/_numpy/test/test_ufuncs.py
@@ -287,7 +287,7 @@
 
     def test_arctan(self):
         import math
-        from _numpy import array, arctan
+        from numpy import array, arctan
 
         a = array([-3, -2, -1, 0, 1, 2, 3, float('inf'), float('-inf')])
         b = arctan(a)
@@ -300,8 +300,7 @@
 
     def test_arcsinh(self):
         import math
-        from _numpy import arcsinh
-        from numpy import inf
+        from numpy import arcsinh, inf
 
         for v in [inf, -inf, 1.0, math.e]:
             assert math.asinh(v) == arcsinh(v)

From noreply at buildbot.pypy.org  Tue Oct  4 04:13:50 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue,  4 Oct 2011 04:13:50 +0200 (CEST)
Subject: [pypy-commit] pypy default: simplify some code
Message-ID: <20111004021350.D20F6820D9@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r47816:71bb26b2f173
Date: 2011-10-03 22:13 -0400
http://bitbucket.org/pypy/pypy/changeset/71bb26b2f173/

Log:	simplify some code

diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -178,9 +178,7 @@
 
     This is for advanced usage only.
     """
-    if NonConstant(False):
-        return True
-    return False
+    return NonConstant(False)
 
 class Entry(ExtRegistryEntry):
     _about_ = hint

From noreply at buildbot.pypy.org  Tue Oct  4 04:20:47 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue,  4 Oct 2011 04:20:47 +0200 (CEST)
Subject: [pypy-commit] pypy default: appears to fix the rlib test_jit test,
 might just be a phase of the moon thing :/
Message-ID: <20111004022047.9F9E5820D9@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r47817:a28e8ce07fdb
Date: 2011-10-03 22:20 -0400
http://bitbucket.org/pypy/pypy/changeset/a28e8ce07fdb/

Log:	appears to fix the rlib test_jit test, might just be a phase of the
	moon thing :/

diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -158,7 +158,7 @@
     return decorator
 
 @oopspec("jit.isconstant(value)")
- at specialize.argtype(0)
+ at specialize.ll()
 def isconstant(value):
     """
     While tracing, returns whether or not the value is currently known to be

From noreply at buildbot.pypy.org  Tue Oct  4 09:28:41 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Tue,  4 Oct 2011 09:28:41 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix test.
Message-ID: <20111004072841.A956A820D9@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r47818:15798ab8cf48
Date: 2011-10-04 09:28 +0200
http://bitbucket.org/pypy/pypy/changeset/15798ab8cf48/

Log:	Fix test.

diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -350,6 +350,7 @@
         elif interval > MAX:
             interval = MAX
         self.checkinterval_scaled = interval * TICK_COUNTER_STEP
+        self.reset_ticker(-1)
 
     def _rebuild_action_dispatcher(self):
         periodic_actions = unrolling_iterable(self._periodic_actions)
diff --git a/pypy/interpreter/test/test_executioncontext.py b/pypy/interpreter/test/test_executioncontext.py
--- a/pypy/interpreter/test/test_executioncontext.py
+++ b/pypy/interpreter/test/test_executioncontext.py
@@ -42,6 +42,7 @@
         assert i == 9
 
     def test_periodic_action(self):
+        from pypy.interpreter.executioncontext import ActionFlag
 
         class DemoAction(executioncontext.PeriodicAsyncAction):
             counter = 0
@@ -53,17 +54,20 @@
 
         space = self.space
         a2 = DemoAction(space)
-        space.actionflag.register_periodic_action(a2, True)
         try:
-            for i in range(500):
-                space.appexec([], """():
-                    n = 5
-                    return n + 2
-                """)
-        except Finished:
-            pass
-        checkinterval = space.actionflag.getcheckinterval()
-        assert checkinterval / 10 < i < checkinterval * 1.1
+            space.actionflag.setcheckinterval(100)
+            space.actionflag.register_periodic_action(a2, True)
+            try:
+                for i in range(500):
+                    space.appexec([], """():
+                        n = 5
+                        return n + 2
+                    """)
+            except Finished:
+                pass
+        finally:
+            space.actionflag = ActionFlag()   # reset to default
+        assert 10 < i < 110
 
     def test_llprofile(self):
         l = []

From noreply at buildbot.pypy.org  Tue Oct  4 15:03:03 2011
From: noreply at buildbot.pypy.org (cfbolz)
Date: Tue,  4 Oct 2011 15:03:03 +0200 (CEST)
Subject: [pypy-commit] pypy list-strategies: specialization always gives
 this functions different names, so just use "_" here.
Message-ID: <20111004130303.19B71820D9@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: list-strategies
Changeset: r47819:fe769d7ec5a2
Date: 2011-10-04 11:51 +0200
http://bitbucket.org/pypy/pypy/changeset/fe769d7ec5a2/

Log:	specialization always gives this functions different names, so just
	use "_" here.

diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py
--- a/pypy/module/pypyjit/test_pypy_c/test_call.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_call.py
@@ -340,7 +340,7 @@
             # Will be killed by the backend
             p15 = getfield_gc(p8, descr=<GcPtrFieldDescr list.items .*>)
             i17 = arraylen_gc(p15, descr=<SignedArrayDescr>)
-            call(ConstClass(_ll_list_resize_ge_trampoline__v232___simple_call__function__), p8, i15, descr=<VoidCallDescr>)
+            call(_, p8, i15, descr=<VoidCallDescr>) # this is a call to _ll_list_resize_ge_trampoline__...
             guard_no_exception(descr=...)
             p17 = getfield_gc(p8, descr=<GcPtrFieldDescr list.items .*>)
             setarrayitem_gc(p17, i13, i12, descr=<SignedArrayDescr>)

From noreply at buildbot.pypy.org  Tue Oct  4 15:03:04 2011
From: noreply at buildbot.pypy.org (cfbolz)
Date: Tue,  4 Oct 2011 15:03:04 +0200 (CEST)
Subject: [pypy-commit] pypy list-strategies: comment
Message-ID: <20111004130304.4DC23820D9@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: list-strategies
Changeset: r47820:caab31c0b915
Date: 2011-10-04 11:52 +0200
http://bitbucket.org/pypy/pypy/changeset/caab31c0b915/

Log:	comment

diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py
--- a/pypy/module/pypyjit/test_pypy_c/test_call.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_call.py
@@ -334,6 +334,7 @@
         log = self.run(main, [1000])
         assert log.result == (1000, 998)
         loop, = log.loops_by_filename(self.filepath)
+        # the int strategy is used here
         assert loop.match_by_id('append', """
             i13 = getfield_gc(p8, descr=<SignedFieldDescr list.length .*>)
             i15 = int_add(i13, 1)

From noreply at buildbot.pypy.org  Tue Oct  4 15:03:06 2011
From: noreply at buildbot.pypy.org (cfbolz)
Date: Tue,  4 Oct 2011 15:03:06 +0200 (CEST)
Subject: [pypy-commit] pypy list-strategies: merge default
Message-ID: <20111004130306.B7ED7820D9@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: list-strategies
Changeset: r47821:3790adb7fde0
Date: 2011-10-04 12:22 +0200
http://bitbucket.org/pypy/pypy/changeset/3790adb7fde0/

Log:	merge default

diff --git a/pypy/annotation/test/test_annrpython.py b/pypy/annotation/test/test_annrpython.py
--- a/pypy/annotation/test/test_annrpython.py
+++ b/pypy/annotation/test/test_annrpython.py
@@ -3204,6 +3204,8 @@
         s = a.build_types(f, [])
         assert isinstance(s, annmodel.SomeList)
         assert not s.listdef.listitem.resized
+        assert not s.listdef.listitem.immutable
+        assert s.listdef.listitem.mutated
 
     def test_delslice(self):
         def f():
diff --git a/pypy/annotation/unaryop.py b/pypy/annotation/unaryop.py
--- a/pypy/annotation/unaryop.py
+++ b/pypy/annotation/unaryop.py
@@ -352,6 +352,7 @@
         check_negative_slice(s_start, s_stop)
         if not isinstance(s_iterable, SomeList):
             raise Exception("list[start:stop] = x: x must be a list")
+        lst.listdef.mutate()
         lst.listdef.agree(s_iterable.listdef)
         # note that setslice is not allowed to resize a list in RPython
 
diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -53,6 +53,18 @@
 this is an ideal task to get started, because it does not require any deep
 knowledge of the internals.
 
+Optimized Unicode Representation
+--------------------------------
+
+CPython 3.3 will use an `optimized unicode representation`_ which switches between
+different ways to represent a unicode string, depending on whether the string
+fits into ASCII, has only two-byte characters or needs four-byte characters.
+
+The actual details would be rather differen in PyPy, but we would like to have
+the same optimization implemented.
+
+.. _`optimized unicode representation`: http://www.python.org/dev/peps/pep-0393/
+
 Translation Toolchain
 ---------------------
 
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -307,7 +307,11 @@
         self._nonperiodic_actions = []
         self.has_bytecode_counter = False
         self.fired_actions = None
-        self.checkinterval_scaled = 100 * TICK_COUNTER_STEP
+        # the default value is not 100, unlike CPython 2.7, but a much
+        # larger value, because we use a technique that not only allows
+        # but actually *forces* another thread to run whenever the counter
+        # reaches zero.
+        self.checkinterval_scaled = 10000 * TICK_COUNTER_STEP
         self._rebuild_action_dispatcher()
 
     def fire(self, action):
@@ -346,6 +350,7 @@
         elif interval > MAX:
             interval = MAX
         self.checkinterval_scaled = interval * TICK_COUNTER_STEP
+        self.reset_ticker(-1)
 
     def _rebuild_action_dispatcher(self):
         periodic_actions = unrolling_iterable(self._periodic_actions)
diff --git a/pypy/interpreter/test/test_executioncontext.py b/pypy/interpreter/test/test_executioncontext.py
--- a/pypy/interpreter/test/test_executioncontext.py
+++ b/pypy/interpreter/test/test_executioncontext.py
@@ -42,6 +42,7 @@
         assert i == 9
 
     def test_periodic_action(self):
+        from pypy.interpreter.executioncontext import ActionFlag
 
         class DemoAction(executioncontext.PeriodicAsyncAction):
             counter = 0
@@ -53,17 +54,20 @@
 
         space = self.space
         a2 = DemoAction(space)
-        space.actionflag.register_periodic_action(a2, True)
         try:
-            for i in range(500):
-                space.appexec([], """():
-                    n = 5
-                    return n + 2
-                """)
-        except Finished:
-            pass
-        checkinterval = space.actionflag.getcheckinterval()
-        assert checkinterval / 10 < i < checkinterval * 1.1
+            space.actionflag.setcheckinterval(100)
+            space.actionflag.register_periodic_action(a2, True)
+            try:
+                for i in range(500):
+                    space.appexec([], """():
+                        n = 5
+                        return n + 2
+                    """)
+            except Finished:
+                pass
+        finally:
+            space.actionflag = ActionFlag()   # reset to default
+        assert 10 < i < 110
 
     def test_llprofile(self):
         l = []
diff --git a/pypy/jit/codewriter/effectinfo.py b/pypy/jit/codewriter/effectinfo.py
--- a/pypy/jit/codewriter/effectinfo.py
+++ b/pypy/jit/codewriter/effectinfo.py
@@ -74,6 +74,7 @@
     OS_LLONG_UGE                = 91
     OS_LLONG_URSHIFT            = 92
     OS_LLONG_FROM_UINT          = 93
+    OS_LLONG_U_TO_FLOAT         = 94
     #
     OS_MATH_SQRT                = 100
 
diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -440,6 +440,7 @@
     rewrite_op_ullong_mod_zer      = _do_builtin_call
     rewrite_op_gc_identityhash = _do_builtin_call
     rewrite_op_gc_id           = _do_builtin_call
+    rewrite_op_uint_mod        = _do_builtin_call
 
     # ----------
     # getfield/setfield/mallocs etc.
@@ -883,9 +884,15 @@
                 v = v_arg
                 oplist = []
             if unsigned1:
-                opname = 'cast_uint_to_longlong'
+                if unsigned2:
+                    opname = 'cast_uint_to_ulonglong'
+                else:
+                    opname = 'cast_uint_to_longlong'
             else:
-                opname = 'cast_int_to_longlong'
+                if unsigned2:
+                    opname = 'cast_int_to_ulonglong'
+                else:
+                    opname = 'cast_int_to_longlong'
             op2 = self.rewrite_operation(
                 SpaceOperation(opname, [v], v_result)
             )
@@ -995,6 +1002,21 @@
                 return op2
         ''' % (_op, _oopspec.lower(), _oopspec, _oopspec)).compile()
 
+    for _op, _oopspec in [('cast_int_to_ulonglong',     'FROM_INT'),
+                          ('cast_uint_to_ulonglong',    'FROM_UINT'),
+                          ('cast_float_to_ulonglong',   'FROM_FLOAT'),
+                          ('cast_ulonglong_to_float',   'U_TO_FLOAT'),
+                         ]:
+        exec py.code.Source('''
+            def rewrite_op_%s(self, op):
+                args = op.args
+                op1 = self.prepare_builtin_call(op, "ullong_%s", args)
+                op2 = self._handle_oopspec_call(op1, args,
+                                                EffectInfo.OS_LLONG_%s,
+                                           EffectInfo.EF_ELIDABLE_CANNOT_RAISE)
+                return op2
+        ''' % (_op, _oopspec.lower(), _oopspec)).compile()
+
     def _normalize(self, oplist):
         if isinstance(oplist, SpaceOperation):
             return [oplist]
diff --git a/pypy/jit/codewriter/support.py b/pypy/jit/codewriter/support.py
--- a/pypy/jit/codewriter/support.py
+++ b/pypy/jit/codewriter/support.py
@@ -315,18 +315,30 @@
 def _ll_1_llong_from_int(x):
     return r_longlong(intmask(x))
 
+def _ll_1_ullong_from_int(x):
+    return r_ulonglong(intmask(x))
+
 def _ll_1_llong_from_uint(x):
     return r_longlong(r_uint(x))
 
+def _ll_1_ullong_from_uint(x):
+    return r_ulonglong(r_uint(x))
+
 def _ll_1_llong_to_int(xll):
     return intmask(xll)
 
 def _ll_1_llong_from_float(xf):
     return r_longlong(xf)
 
+def _ll_1_ullong_from_float(xf):
+    return r_ulonglong(xf)
+
 def _ll_1_llong_to_float(xll):
     return float(rffi.cast(lltype.SignedLongLong, xll))
 
+def _ll_1_ullong_u_to_float(xull):
+    return float(rffi.cast(lltype.UnsignedLongLong, xull))
+
 
 def _ll_1_llong_abs(xll):
     if xll < 0:
@@ -351,20 +363,23 @@
     return llop.llong_mod(lltype.SignedLongLong, xll, yll)
 
 def _ll_2_ullong_floordiv(xll, yll):
-    return llop.ullong_floordiv(lltype.SignedLongLong, xll, yll)
+    return llop.ullong_floordiv(lltype.UnsignedLongLong, xll, yll)
 
 def _ll_2_ullong_floordiv_zer(xll, yll):
     if yll == 0:
         raise ZeroDivisionError
-    return llop.ullong_floordiv(lltype.SignedLongLong, xll, yll)
+    return llop.ullong_floordiv(lltype.UnsignedLongLong, xll, yll)
 
 def _ll_2_ullong_mod(xll, yll):
-    return llop.ullong_mod(lltype.SignedLongLong, xll, yll)
+    return llop.ullong_mod(lltype.UnsignedLongLong, xll, yll)
 
 def _ll_2_ullong_mod_zer(xll, yll):
     if yll == 0:
         raise ZeroDivisionError
-    return llop.ullong_mod(lltype.SignedLongLong, xll, yll)
+    return llop.ullong_mod(lltype.UnsignedLongLong, xll, yll)
+
+def _ll_2_uint_mod(xll, yll):
+    return llop.uint_mod(lltype.Unsigned, xll, yll)
 
 
 # libffi support
diff --git a/pypy/jit/codewriter/test/test_flatten.py b/pypy/jit/codewriter/test/test_flatten.py
--- a/pypy/jit/codewriter/test/test_flatten.py
+++ b/pypy/jit/codewriter/test/test_flatten.py
@@ -829,14 +829,15 @@
                     self.encoding_test(f, [rffi.cast(FROM, 42)], expectedstr,
                                        transform=True)
                 elif TO in (rffi.LONG, rffi.ULONG):
+                    if rffi.cast(FROM, -1) < 0:
+                        fnname = "llong_from_int"
+                    else:
+                        fnname = "llong_from_uint"
                     if TO == rffi.LONG:
                         TO = rffi.LONGLONG
                     else:
                         TO = rffi.ULONGLONG
-                    if rffi.cast(FROM, -1) < 0:
-                        fnname = "llong_from_int"
-                    else:
-                        fnname = "llong_from_uint"
+                        fnname = "u" + fnname
                     expected.pop()   # remove int_return
                     expected.append(
                         "residual_call_irf_f $<* fn %s>, <Descr>, I[%s], R[], F[] -> %%f0"
diff --git a/pypy/jit/codewriter/test/test_longlong.py b/pypy/jit/codewriter/test/test_longlong.py
--- a/pypy/jit/codewriter/test/test_longlong.py
+++ b/pypy/jit/codewriter/test/test_longlong.py
@@ -57,7 +57,8 @@
             assert op1.opname == 'residual_call_irf_f'
         else:
             assert op1.opname == 'residual_call_irf_i'
-        gotindex = getattr(EffectInfo, 'OS_' + op1.args[0].value.upper())
+        gotindex = getattr(EffectInfo,
+                           'OS_' + op1.args[0].value.upper().lstrip('U'))
         assert gotindex == oopspecindex
         assert op1.args[1] == 'calldescr-%d' % oopspecindex
         assert list(op1.args[2]) == [v for v in vlist
@@ -192,8 +193,12 @@
                       [lltype.SignedLongLong], lltype.Signed)
         self.do_check('cast_float_to_longlong', EffectInfo.OS_LLONG_FROM_FLOAT,
                       [lltype.Float], lltype.SignedLongLong)
+        self.do_check('cast_float_to_ulonglong', EffectInfo.OS_LLONG_FROM_FLOAT,
+                      [lltype.Float], lltype.UnsignedLongLong)
         self.do_check('cast_longlong_to_float', EffectInfo.OS_LLONG_TO_FLOAT,
                       [lltype.SignedLongLong], lltype.Float)
+        self.do_check('cast_ulonglong_to_float', EffectInfo.OS_LLONG_U_TO_FLOAT,
+                      [lltype.UnsignedLongLong], lltype.Float)
         for T1 in [lltype.SignedLongLong, lltype.UnsignedLongLong]:
             for T2 in [lltype.Signed, lltype.Unsigned]:
                 self.do_check('cast_primitive', EffectInfo.OS_LLONG_TO_INT,
diff --git a/pypy/jit/metainterp/optimizeopt/__init__.py b/pypy/jit/metainterp/optimizeopt/__init__.py
--- a/pypy/jit/metainterp/optimizeopt/__init__.py
+++ b/pypy/jit/metainterp/optimizeopt/__init__.py
@@ -7,6 +7,8 @@
 from pypy.jit.metainterp.optimizeopt.unroll import optimize_unroll, OptInlineShortPreamble
 from pypy.jit.metainterp.optimizeopt.fficall import OptFfiCall
 from pypy.jit.metainterp.optimizeopt.simplify import OptSimplify
+from pypy.jit.metainterp.optimizeopt.pure import OptPure
+from pypy.jit.metainterp.optimizeopt.earlyforce import OptEarlyForce
 from pypy.rlib.jit import PARAMETERS
 from pypy.rlib.unroll import unrolling_iterable
 
@@ -14,6 +16,8 @@
             ('rewrite', OptRewrite),
             ('virtualize', OptVirtualize),
             ('string', OptString),
+            ('earlyforce', OptEarlyForce),
+            ('pure', OptPure),
             ('heap', OptHeap),
             ('ffi', None),
             ('unroll', None)]
diff --git a/pypy/jit/metainterp/optimizeopt/earlyforce.py b/pypy/jit/metainterp/optimizeopt/earlyforce.py
new file mode 100644
--- /dev/null
+++ b/pypy/jit/metainterp/optimizeopt/earlyforce.py
@@ -0,0 +1,24 @@
+from pypy.jit.metainterp.optimizeopt.optimizer import Optimization
+from pypy.jit.metainterp.optimizeopt.vstring import VAbstractStringValue
+from pypy.jit.metainterp.resoperation import rop, ResOperation
+
+class OptEarlyForce(Optimization):
+    def propagate_forward(self, op):
+        opnum = op.getopnum()
+        if (opnum != rop.SETFIELD_GC and 
+            opnum != rop.SETARRAYITEM_GC and
+            opnum != rop.QUASIIMMUT_FIELD):
+               
+            for arg in op.getarglist():
+                if arg in self.optimizer.values:
+                    value = self.getvalue(arg)
+                    value.force_box(self)
+        self.emit_operation(op)
+
+    def new(self):
+        return OptEarlyForce()
+
+    def setup(self):
+        self.optimizer.optearlyforce = self
+
+    
diff --git a/pypy/jit/metainterp/optimizeopt/fficall.py b/pypy/jit/metainterp/optimizeopt/fficall.py
--- a/pypy/jit/metainterp/optimizeopt/fficall.py
+++ b/pypy/jit/metainterp/optimizeopt/fficall.py
@@ -177,10 +177,10 @@
             funcinfo.descr is None):
             return [op] # cannot optimize
         funcsymval = self.getvalue(op.getarg(2))
-        arglist = [funcsymval.force_box()]
+        arglist = [funcsymval.get_key_box()]
         for push_op in funcinfo.opargs:
             argval = self.getvalue(push_op.getarg(2))
-            arglist.append(argval.force_box())
+            arglist.append(argval.get_key_box())
         newop = ResOperation(rop.CALL_RELEASE_GIL, arglist, op.result,
                              descr=funcinfo.descr)
         self.commit_optimization()
diff --git a/pypy/jit/metainterp/optimizeopt/heap.py b/pypy/jit/metainterp/optimizeopt/heap.py
--- a/pypy/jit/metainterp/optimizeopt/heap.py
+++ b/pypy/jit/metainterp/optimizeopt/heap.py
@@ -50,6 +50,7 @@
             if not self._lazy_setfield_registered:
                 optheap._lazy_setfields_and_arrayitems.append(self)
                 self._lazy_setfield_registered = True
+            
         else:
             # this is the case where the pending setfield ends up
             # storing precisely the value that is already there,
@@ -158,12 +159,17 @@
         self._lazy_setfields_and_arrayitems = []
         self._remove_guard_not_invalidated = False
         self._seen_guard_not_invalidated = False
+        self.posponedop = None
 
     def force_at_end_of_preamble(self):
         self.force_all_lazy_setfields_and_arrayitems()
 
     def flush(self):
         self.force_all_lazy_setfields_and_arrayitems()
+        if self.posponedop:
+            posponedop = self.posponedop
+            self.posponedop = None
+            self.next_optimization.propagate_forward(posponedop)
 
     def new(self):
         return OptHeap()
@@ -211,7 +217,15 @@
 
     def emit_operation(self, op):
         self.emitting_operation(op)
-        self.next_optimization.propagate_forward(op)
+        if self.posponedop:
+            posponedop = self.posponedop
+            self.posponedop = None
+            self.next_optimization.propagate_forward(posponedop)
+        if (op.is_comparison() or op.getopnum() == rop.CALL_MAY_FORCE
+            or op.is_ovf()):
+            self.posponedop = op
+        else:
+            self.next_optimization.propagate_forward(op)
 
     def emitting_operation(self, op):
         if op.has_no_side_effect():
@@ -293,30 +307,6 @@
             if indexvalue is None or indexvalue.intbound.contains(idx):
                 cf.force_lazy_setfield(self, can_cache)
 
-    def fixup_guard_situation(self):
-        # hackish: reverse the order of the last two operations if it makes
-        # sense to avoid a situation like "int_eq/setfield_gc/guard_true",
-        # which the backend (at least the x86 backend) does not handle well.
-        newoperations = self.optimizer.newoperations
-        if len(newoperations) < 2:
-            return
-        lastop = newoperations[-1]
-        if (lastop.getopnum() != rop.SETFIELD_GC and
-            lastop.getopnum() != rop.SETARRAYITEM_GC):
-            return
-        # - is_comparison() for cases like "int_eq/setfield_gc/guard_true"
-        # - CALL_MAY_FORCE: "call_may_force/setfield_gc/guard_not_forced"
-        # - is_ovf(): "int_add_ovf/setfield_gc/guard_no_overflow"
-        prevop = newoperations[-2]
-        opnum = prevop.getopnum()
-        if not (prevop.is_comparison() or opnum == rop.CALL_MAY_FORCE
-                or prevop.is_ovf()):
-            return
-        if prevop.result in lastop.getarglist():
-            return
-        newoperations[-2] = lastop
-        newoperations[-1] = prevop
-
     def _assert_valid_cf(self, cf):
         # check that 'cf' is in cached_fields or cached_arrayitems
         if not we_are_translated():
@@ -362,7 +352,6 @@
                                       fieldvalue.get_key_box(), itemindex))
             else:
                 cf.force_lazy_setfield(self)
-                self.fixup_guard_situation()
         return pendingfields
 
     def optimize_GETFIELD_GC(self, op):
@@ -374,12 +363,22 @@
             return
         # default case: produce the operation
         structvalue.ensure_nonnull()
-        ###self.optimizer.optimize_default(op)
         self.emit_operation(op)
         # then remember the result of reading the field
         fieldvalue = self.getvalue(op.result)
         cf.remember_field_value(structvalue, fieldvalue, op)
 
+    def optimize_GETFIELD_GC_PURE(self, op):
+        structvalue = self.getvalue(op.getarg(0))
+        cf = self.field_cache(op.getdescr())
+        fieldvalue = cf.getfield_from_cache(self, structvalue)
+        if fieldvalue is not None:
+            self.make_equal_to(op.result, fieldvalue)
+            return
+        # default case: produce the operation
+        structvalue.ensure_nonnull()
+        self.emit_operation(op)
+
     def optimize_SETFIELD_GC(self, op):
         if self.has_pure_result(rop.GETFIELD_GC_PURE, [op.getarg(0)],
                                 op.getdescr()):
@@ -389,6 +388,7 @@
         #
         cf = self.field_cache(op.getdescr())
         cf.do_setfield(self, op)
+        
 
     def optimize_GETARRAYITEM_GC(self, op):
         arrayvalue = self.getvalue(op.getarg(0))
@@ -413,6 +413,25 @@
             fieldvalue = self.getvalue(op.result)
             cf.remember_field_value(arrayvalue, fieldvalue, op)
 
+    def optimize_GETARRAYITEM_GC_PURE(self, op):
+        arrayvalue = self.getvalue(op.getarg(0))
+        indexvalue = self.getvalue(op.getarg(1))
+        cf = None
+        if indexvalue.is_constant():
+            arrayvalue.make_len_gt(MODE_ARRAY, op.getdescr(), indexvalue.box.getint())
+            # use the cache on (arraydescr, index), which is a constant
+            cf = self.arrayitem_cache(op.getdescr(), indexvalue.box.getint())
+            fieldvalue = cf.getfield_from_cache(self, arrayvalue)
+            if fieldvalue is not None:
+                self.make_equal_to(op.result, fieldvalue)
+                return
+        else:
+            # variable index, so make sure the lazy setarrayitems are done
+            self.force_lazy_setarrayitem(op.getdescr(), indexvalue=indexvalue)
+        # default case: produce the operation
+        arrayvalue.ensure_nonnull()
+        self.emit_operation(op)
+
     def optimize_SETARRAYITEM_GC(self, op):
         if self.has_pure_result(rop.GETARRAYITEM_GC_PURE, [op.getarg(0),
                                                            op.getarg(1)],
diff --git a/pypy/jit/metainterp/optimizeopt/optimizer.py b/pypy/jit/metainterp/optimizeopt/optimizer.py
--- a/pypy/jit/metainterp/optimizeopt/optimizer.py
+++ b/pypy/jit/metainterp/optimizeopt/optimizer.py
@@ -31,8 +31,8 @@
 
 class OptValue(object):
     __metaclass__ = extendabletype
-    _attrs_ = ('box', 'known_class', 'last_guard_index', 'level', 'intbound', 'lenbound')
-    last_guard_index = -1
+    _attrs_ = ('box', 'known_class', 'last_guard', 'level', 'intbound', 'lenbound')
+    last_guard = None
 
     level = LEVEL_UNKNOWN
     known_class = None
@@ -100,7 +100,7 @@
             self.make_constant(other.get_key_box())
             optimizer.turned_constant(self)
         elif other.level == LEVEL_KNOWNCLASS:
-            self.make_constant_class(other.known_class, -1)
+            self.make_constant_class(other.known_class, None)
         else:
             if other.level == LEVEL_NONNULL:
                 self.ensure_nonnull()
@@ -114,13 +114,13 @@
                     self.lenbound = other.lenbound.clone()
 
 
-    def force_box(self):
+    def force_box(self, optforce):
         return self.box
 
     def get_key_box(self):
         return self.box
 
-    def force_at_end_of_preamble(self, already_forced):
+    def force_at_end_of_preamble(self, already_forced, optforce):
         return self
 
     def get_args_for_fail(self, modifier):
@@ -162,16 +162,16 @@
         else:
             return None
 
-    def make_constant_class(self, classbox, opindex):
+    def make_constant_class(self, classbox, guardop):
         assert self.level < LEVEL_KNOWNCLASS
         self.known_class = classbox
         self.level = LEVEL_KNOWNCLASS
-        self.last_guard_index = opindex
+        self.last_guard = guardop
 
-    def make_nonnull(self, opindex):
+    def make_nonnull(self, guardop):
         assert self.level < LEVEL_NONNULL
         self.level = LEVEL_NONNULL
-        self.last_guard_index = opindex
+        self.last_guard = guardop
 
     def is_nonnull(self):
         level = self.level
@@ -240,26 +240,12 @@
     def __init__(self):
         pass # make rpython happy
 
-    def propagate_begin_forward(self):
-        if self.next_optimization:
-            self.next_optimization.propagate_begin_forward()
-
-    def propagate_end_forward(self):
-        if self.next_optimization:
-            self.next_optimization.propagate_end_forward()
-
     def propagate_forward(self, op):
         raise NotImplementedError
 
     def emit_operation(self, op):
         self.next_optimization.propagate_forward(op)
 
-    def test_emittable(self, op):
-        return self.is_emittable(op)
-
-    def is_emittable(self, op):
-        return self.next_optimization.test_emittable(op)
-
     # FIXME: Move some of these here?
     def getvalue(self, box):
         return self.optimizer.getvalue(box)
@@ -289,19 +275,19 @@
         return self.optimizer.new_const_item(arraydescr)
 
     def pure(self, opnum, args, result):
-        op = ResOperation(opnum, args, result)
-        key = self.optimizer.make_args_key(op)
-        if key not in self.optimizer.pure_operations:
-            self.optimizer.pure_operations[key] = op
+        if self.optimizer.optpure:
+            self.optimizer.optpure.pure(opnum, args, result)
 
     def has_pure_result(self, opnum, args, descr):
-        op = ResOperation(opnum, args, None, descr)
-        key = self.optimizer.make_args_key(op)
-        op = self.optimizer.pure_operations.get(key, None)
-        if op is None:
-            return False
-        return op.getdescr() is descr
+        if self.optimizer.optpure:
+            return self.optimizer.optpure.has_pure_result(opnum, args, descr)
+        return False
 
+    def get_pure_result(self, key):    
+        if self.optimizer.optpure:
+            return self.optimizer.optpure.get_pure_result(key)
+        return None
+    
     def setup(self):
         pass
 
@@ -322,6 +308,9 @@
     def produce_potential_short_preamble_ops(self, potential_ops):
         pass
 
+    def forget_numberings(self, box):
+        self.optimizer.forget_numberings(box)
+
 class Optimizer(Optimization):
 
     def __init__(self, metainterp_sd, loop, optimizations=None, bridge=False):
@@ -333,14 +322,16 @@
         self.interned_refs = self.cpu.ts.new_ref_dict()
         self.resumedata_memo = resume.ResumeDataLoopMemo(metainterp_sd)
         self.bool_boxes = {}
-        self.pure_operations = args_dict()
         self.producer = {}
         self.pendingfields = []
-        self.posponedop = None
         self.exception_might_have_happened = False
         self.quasi_immutable_deps = None
         self.opaque_pointers = {}
-        self.newoperations = []
+        self.replaces_guard = {}
+        self._newoperations = []
+        self.optimizer = self
+        self.optpure = None
+        self.optearlyforce = None
         if loop is not None:
             self.call_pure_results = loop.call_pure_results
 
@@ -369,21 +360,20 @@
     def flush(self):
         for o in self.optimizations:
             o.flush()
-        assert self.posponedop is None
 
     def new(self):
         new = Optimizer(self.metainterp_sd, self.loop)
         return self._new(new)
 
     def _new(self, new):
-        assert self.posponedop is None
         optimizations = [o.new() for o in self.optimizations]
         new.set_optimizations(optimizations)
         new.quasi_immutable_deps = self.quasi_immutable_deps
         return new
 
     def produce_potential_short_preamble_ops(self, sb):
-        raise NotImplementedError('This is implemented in unroll.UnrollableOptimizer')
+        for opt in self.optimizations:
+            opt.produce_potential_short_preamble_ops(sb)
 
     def turned_constant(self, value):
         for o in self.optimizations:
@@ -433,6 +423,13 @@
             return constbox
         return None
 
+    def get_newoperations(self):
+        self.flush()
+        return self._newoperations
+
+    def clear_newoperations(self):
+        self._newoperations = []
+
     def make_equal_to(self, box, value, replace=False):
         assert isinstance(value, OptValue)
         assert replace or box not in self.values
@@ -481,15 +478,10 @@
 
     def propagate_all_forward(self):
         self.exception_might_have_happened = self.bridge
-        self.newoperations = []
-        self.first_optimization.propagate_begin_forward()
-        self.i = 0
-        while self.i < len(self.loop.operations):
-            op = self.loop.operations[self.i]
+        self.clear_newoperations()
+        for op in self.loop.operations:
             self.first_optimization.propagate_forward(op)
-            self.i += 1
-        self.first_optimization.propagate_end_forward()
-        self.loop.operations = self.newoperations
+        self.loop.operations = self.get_newoperations()
         self.loop.quasi_immutable_deps = self.quasi_immutable_deps
         # accumulate counters
         self.resumedata_memo.update_counters(self.metainterp_sd.profiler)
@@ -501,9 +493,6 @@
         self.producer[op.result] = op
         dispatch_opt(self, op)
 
-    def test_emittable(self, op):
-        return True
-
     def emit_operation(self, op):
         if op.returns_bool_result():
             self.bool_boxes[self.getvalue(op.result)] = None
@@ -519,14 +508,30 @@
                 pass
             else:
                 self.ensure_imported(value)
-                op.setarg(i, value.force_box())
+                op.setarg(i, value.force_box(self))
         self.metainterp_sd.profiler.count(jitprof.OPT_OPS)
         if op.is_guard():
             self.metainterp_sd.profiler.count(jitprof.OPT_GUARDS)
-            op = self.store_final_boxes_in_guard(op)
+            if self.replaces_guard and op in self.replaces_guard:
+                self.replace_op(self.replaces_guard[op], op)
+                del self.replaces_guard[op]
+                return
+            else:
+                op = self.store_final_boxes_in_guard(op)
         elif op.can_raise():
             self.exception_might_have_happened = True
-        self.newoperations.append(op)
+        self._newoperations.append(op)
+
+    def replace_op(self, old_op, new_op):
+        # XXX: Do we want to cache indexes to prevent search?
+        i = len(self._newoperations) 
+        while i > 0:
+            i -= 1
+            if self._newoperations[i] is old_op:
+                self._newoperations[i] = new_op
+                break
+        else:
+            assert False
 
     def store_final_boxes_in_guard(self, op):
         descr = op.getdescr()
@@ -574,51 +579,8 @@
         args[n + 1] = op.getdescr()
         return args
 
-    @specialize.argtype(0)
     def optimize_default(self, op):
-        canfold = op.is_always_pure()
-        if op.is_ovf():
-            self.posponedop = op
-            return
-        if self.posponedop:
-            nextop = op
-            op = self.posponedop
-            self.posponedop = None
-            canfold = nextop.getopnum() == rop.GUARD_NO_OVERFLOW
-        else:
-            nextop = None
-
-        if canfold:
-            for i in range(op.numargs()):
-                if self.get_constant_box(op.getarg(i)) is None:
-                    break
-            else:
-                # all constant arguments: constant-fold away
-                resbox = self.constant_fold(op)
-                # note that INT_xxx_OVF is not done from here, and the
-                # overflows in the INT_xxx operations are ignored
-                self.make_constant(op.result, resbox)
-                return
-
-            # did we do the exact same operation already?
-            args = self.make_args_key(op)
-            oldop = self.pure_operations.get(args, None)
-            if oldop is not None and oldop.getdescr() is op.getdescr():
-                assert oldop.getopnum() == op.getopnum()
-                self.make_equal_to(op.result, self.getvalue(oldop.result),
-                                   True)
-                return
-            else:
-                self.pure_operations[args] = op
-                self.remember_emitting_pure(op)
-
-        # otherwise, the operation remains
         self.emit_operation(op)
-        if nextop:
-            self.emit_operation(nextop)
-
-    def remember_emitting_pure(self, op):
-        pass
 
     def constant_fold(self, op):
         argboxes = [self.get_constant_box(op.getarg(i))
@@ -636,11 +598,6 @@
     def optimize_DEBUG_MERGE_POINT(self, op):
         self.emit_operation(op)
 
-    def optimize_CAST_OPAQUE_PTR(self, op):
-        value = self.getvalue(op.getarg(0))
-        self.opaque_pointers[value] = True
-        self.make_equal_to(op.result, value)
-
     def optimize_GETARRAYITEM_GC_PURE(self, op):
         indexvalue = self.getvalue(op.getarg(1))
         if indexvalue.is_constant():
diff --git a/pypy/jit/metainterp/optimizeopt/pure.py b/pypy/jit/metainterp/optimizeopt/pure.py
new file mode 100644
--- /dev/null
+++ b/pypy/jit/metainterp/optimizeopt/pure.py
@@ -0,0 +1,115 @@
+from pypy.jit.metainterp.optimizeopt.optimizer import Optimization
+from pypy.jit.metainterp.resoperation import rop, ResOperation
+from pypy.jit.metainterp.optimizeopt.util import (make_dispatcher_method,
+    args_dict)
+
+class OptPure(Optimization):
+    def __init__(self):
+        self.posponedop = None
+        self.pure_operations = args_dict()
+        self.emitted_pure_operations = {}
+
+    def propagate_forward(self, op):
+        dispatch_opt(self, op)
+
+    def optimize_default(self, op):
+        canfold = op.is_always_pure()
+        if op.is_ovf():
+            self.posponedop = op
+            return
+        if self.posponedop:
+            nextop = op
+            op = self.posponedop
+            self.posponedop = None
+            canfold = nextop.getopnum() == rop.GUARD_NO_OVERFLOW
+        else:
+            nextop = None
+
+        if canfold:
+            for i in range(op.numargs()):
+                if self.get_constant_box(op.getarg(i)) is None:
+                    break
+            else:
+                # all constant arguments: constant-fold away
+                resbox = self.optimizer.constant_fold(op)
+                # note that INT_xxx_OVF is not done from here, and the
+                # overflows in the INT_xxx operations are ignored
+                self.optimizer.make_constant(op.result, resbox)
+                return
+
+            # did we do the exact same operation already?
+            args = self.optimizer.make_args_key(op)
+            oldop = self.pure_operations.get(args, None)
+            if oldop is not None and oldop.getdescr() is op.getdescr():
+                assert oldop.getopnum() == op.getopnum()
+                self.optimizer.make_equal_to(op.result, self.getvalue(oldop.result),
+                                   True)
+                return
+            else:
+                self.pure_operations[args] = op
+                self.remember_emitting_pure(op)
+
+        # otherwise, the operation remains
+        self.emit_operation(op)
+        if op.returns_bool_result():
+            self.optimizer.bool_boxes[self.getvalue(op.result)] = None        
+        if nextop:
+            self.emit_operation(nextop)
+
+    def optimize_CALL_PURE(self, op):
+        args = self.optimizer.make_args_key(op)
+        oldop = self.pure_operations.get(args, None)
+        if oldop is not None and oldop.getdescr() is op.getdescr():
+            assert oldop.getopnum() == op.getopnum()
+            self.make_equal_to(op.result, self.getvalue(oldop.result))
+            return
+        else:
+            self.pure_operations[args] = op
+            self.remember_emitting_pure(op)
+
+        # replace CALL_PURE with just CALL
+        args = op.getarglist()
+        self.emit_operation(ResOperation(rop.CALL, args, op.result,
+                                         op.getdescr()))
+
+    def flush(self):
+        assert self.posponedop is None
+
+    def new(self):
+        assert self.posponedop is None
+        return OptPure()
+
+    def setup(self):
+        self.optimizer.optpure = self
+
+    def pure(self, opnum, args, result):
+        op = ResOperation(opnum, args, result)
+        key = self.optimizer.make_args_key(op)
+        if key not in self.pure_operations:
+            self.pure_operations[key] = op
+
+    def has_pure_result(self, opnum, args, descr):
+        op = ResOperation(opnum, args, None, descr)
+        key = self.optimizer.make_args_key(op)
+        op = self.pure_operations.get(key, None)
+        if op is None:
+            return False
+        return op.getdescr() is descr
+
+    def get_pure_result(self, key):
+        return self.pure_operations.get(key, None)
+
+    def remember_emitting_pure(self, op):
+        self.emitted_pure_operations[op] = True
+
+    def produce_potential_short_preamble_ops(self, sb):
+        for op in self.emitted_pure_operations:
+            if op.getopnum() == rop.GETARRAYITEM_GC_PURE or \
+               op.getopnum() == rop.STRGETITEM or \
+               op.getopnum() == rop.UNICODEGETITEM:
+                if not self.getvalue(op.getarg(1)).is_constant():
+                    continue
+            sb.add_potential(op)
+
+dispatch_opt = make_dispatcher_method(OptPure, 'optimize_',
+                                      default=OptPure.optimize_default)
diff --git a/pypy/jit/metainterp/optimizeopt/rewrite.py b/pypy/jit/metainterp/optimizeopt/rewrite.py
--- a/pypy/jit/metainterp/optimizeopt/rewrite.py
+++ b/pypy/jit/metainterp/optimizeopt/rewrite.py
@@ -31,21 +31,8 @@
 
         dispatch_opt(self, op)
 
-    def test_emittable(self, op):
-        opnum = op.getopnum()
-        for value, cls, func in optimize_guards:
-            if opnum == value:
-                assert isinstance(op, cls)
-                try:
-                    func(self, op, dryrun=True)
-                    return self.is_emittable(op)
-                except InvalidLoop:
-                    return False
-        return self.is_emittable(op)
-
-
     def try_boolinvers(self, op, targs):
-        oldop = self.optimizer.pure_operations.get(targs, None)
+        oldop = self.get_pure_result(targs)
         if oldop is not None and oldop.getdescr() is op.getdescr():
             value = self.getvalue(oldop.result)
             if value.is_constant():
@@ -73,7 +60,7 @@
             oldopnum = opboolreflex[op.getopnum()] # FIXME: add INT_ADD, INT_MUL
             targs = self.optimizer.make_args_key(ResOperation(oldopnum, [args[1], args[0]],
                                                               None))
-            oldop = self.optimizer.pure_operations.get(targs, None)
+            oldop = self.get_pure_result(targs)
             if oldop is not None and oldop.getdescr() is op.getdescr():
                 self.make_equal_to(op.result, self.getvalue(oldop.result))
                 return True
@@ -214,40 +201,7 @@
         self.emit_operation(op)
         self.pure(rop.FLOAT_NEG, [op.result], v1)
 
-    def optimize_CALL_PURE(self, op):
-        arg_consts = []
-        for i in range(op.numargs()):
-            arg = op.getarg(i)
-            const = self.get_constant_box(arg)
-            if const is None:
-                break
-            arg_consts.append(const)
-        else:
-            # all constant arguments: check if we already know the result
-            try:
-                result = self.optimizer.call_pure_results[arg_consts]
-            except KeyError:
-                pass
-            else:
-                self.make_constant(op.result, result)
-                return
-
-        args = self.optimizer.make_args_key(op)
-        oldop = self.optimizer.pure_operations.get(args, None)
-        if oldop is not None and oldop.getdescr() is op.getdescr():
-            assert oldop.getopnum() == op.getopnum()
-            self.make_equal_to(op.result, self.getvalue(oldop.result))
-            return
-        else:
-            self.optimizer.pure_operations[args] = op
-            self.optimizer.remember_emitting_pure(op)
-
-        # replace CALL_PURE with just CALL
-        args = op.getarglist()
-        self.emit_operation(ResOperation(rop.CALL, args, op.result,
-                                         op.getdescr()))
-
-    def optimize_guard(self, op, constbox, emit_operation=True, dryrun=False):
+    def optimize_guard(self, op, constbox, emit_operation=True):
         value = self.getvalue(op.getarg(0))
         if value.is_constant():
             box = value.box
@@ -255,62 +209,57 @@
             if not box.same_constant(constbox):
                 raise InvalidLoop
             return
-        if dryrun: return
         if emit_operation:
             self.emit_operation(op)
         value.make_constant(constbox)
         self.optimizer.turned_constant(value)
 
-    def optimize_GUARD_ISNULL(self, op, dryrun=False):
+    def optimize_GUARD_ISNULL(self, op):
         value = self.getvalue(op.getarg(0))
         if value.is_null():
             return
         elif value.is_nonnull():
             raise InvalidLoop
-        if dryrun: return
         self.emit_operation(op)
         value.make_constant(self.optimizer.cpu.ts.CONST_NULL)
 
-    def optimize_GUARD_NONNULL(self, op, dryrun=False):
+    def optimize_GUARD_NONNULL(self, op):
         value = self.getvalue(op.getarg(0))
         if value.is_nonnull():
             return
         elif value.is_null():
             raise InvalidLoop
-        if dryrun: return
         self.emit_operation(op)
-        value.make_nonnull(len(self.optimizer.newoperations) - 1)
+        value.make_nonnull(op)
 
-    def optimize_GUARD_VALUE(self, op, dryrun=False):
+    def optimize_GUARD_VALUE(self, op):
         value = self.getvalue(op.getarg(0))
-        emit_operation = True
-        if not dryrun and value.last_guard_index != -1:
+        if value.last_guard:
             # there already has been a guard_nonnull or guard_class or
             # guard_nonnull_class on this value, which is rather silly.
             # replace the original guard with a guard_value
-            old_guard_op = self.optimizer.newoperations[value.last_guard_index]
-            new_guard_op = old_guard_op.copy_and_change(rop.GUARD_VALUE,
-                                             args = [old_guard_op.getarg(0), op.getarg(1)])
-            self.optimizer.newoperations[value.last_guard_index] = new_guard_op
+            old_guard_op = value.last_guard
+            op = old_guard_op.copy_and_change(rop.GUARD_VALUE,
+                                      args = [old_guard_op.getarg(0), op.getarg(1)])
+            self.optimizer.replaces_guard[op] = old_guard_op
             # hack hack hack.  Change the guard_opnum on
             # new_guard_op.getdescr() so that when resuming,
             # the operation is not skipped by pyjitpl.py.
-            descr = new_guard_op.getdescr()
+            descr = op.getdescr()
             assert isinstance(descr, compile.ResumeGuardDescr)
             descr.guard_opnum = rop.GUARD_VALUE
-            descr.make_a_counter_per_value(new_guard_op)
-            emit_operation = False
+            descr.make_a_counter_per_value(op)
         constbox = op.getarg(1)
         assert isinstance(constbox, Const)
-        self.optimize_guard(op, constbox, emit_operation, dryrun)
+        self.optimize_guard(op, constbox)
 
-    def optimize_GUARD_TRUE(self, op, dryrun=False):
-        self.optimize_guard(op, CONST_1, dryrun=dryrun)
+    def optimize_GUARD_TRUE(self, op):
+        self.optimize_guard(op, CONST_1)
 
-    def optimize_GUARD_FALSE(self, op, dryrun=False):
-        self.optimize_guard(op, CONST_0, dryrun=dryrun)
+    def optimize_GUARD_FALSE(self, op):
+        self.optimize_guard(op, CONST_0)
 
-    def optimize_GUARD_CLASS(self, op, dryrun=False):
+    def optimize_GUARD_CLASS(self, op):
         value = self.getvalue(op.getarg(0))
         expectedclassbox = op.getarg(1)
         assert isinstance(expectedclassbox, Const)
@@ -319,38 +268,32 @@
             if realclassbox.same_constant(expectedclassbox):
                 return
             raise InvalidLoop
-        if dryrun: return
-        emit_operation = True
-        if value.last_guard_index != -1:
+        if value.last_guard:
             # there already has been a guard_nonnull or guard_class or
             # guard_nonnull_class on this value.
-            old_guard_op = self.optimizer.newoperations[value.last_guard_index]
+            old_guard_op = value.last_guard
             if old_guard_op.getopnum() == rop.GUARD_NONNULL:
                 # it was a guard_nonnull, which we replace with a
                 # guard_nonnull_class.
-                new_guard_op = old_guard_op.copy_and_change (rop.GUARD_NONNULL_CLASS,
+                op = old_guard_op.copy_and_change (rop.GUARD_NONNULL_CLASS,
                                          args = [old_guard_op.getarg(0), op.getarg(1)])
-                self.optimizer.newoperations[value.last_guard_index] = new_guard_op
+                self.optimizer.replaces_guard[op] = old_guard_op
                 # hack hack hack.  Change the guard_opnum on
                 # new_guard_op.getdescr() so that when resuming,
                 # the operation is not skipped by pyjitpl.py.
-                descr = new_guard_op.getdescr()
+                descr = op.getdescr()
                 assert isinstance(descr, compile.ResumeGuardDescr)
                 descr.guard_opnum = rop.GUARD_NONNULL_CLASS
-                emit_operation = False
-        if emit_operation:
-            self.emit_operation(op)
-            last_guard_index = len(self.optimizer.newoperations) - 1
-        else:
-            last_guard_index = value.last_guard_index
-        value.make_constant_class(expectedclassbox, last_guard_index)
+        self.emit_operation(op)
+        value.make_constant_class(expectedclassbox, op)
 
-    def optimize_GUARD_NONNULL_CLASS(self, op, dryrun=False):
-        self.optimize_GUARD_NONNULL(op, True)
-        self.optimize_GUARD_CLASS(op, dryrun)
+    def optimize_GUARD_NONNULL_CLASS(self, op):
+        value = self.getvalue(op.getarg(0))
+        if value.is_null():
+            raise InvalidLoop
+        self.optimize_GUARD_CLASS(op)
 
-    def optimize_GUARD_NO_EXCEPTION(self, op, dryrun=False):
-        if dryrun: return
+    def optimize_GUARD_NO_EXCEPTION(self, op):
         if not self.optimizer.exception_might_have_happened:
             return
         self.emit_operation(op)
@@ -470,7 +413,7 @@
                     newop = ResOperation(rop.SETARRAYITEM_GC,
                                          [op.getarg(2),
                                           ConstInt(index + dest_start),
-                                          val.force_box()], None,
+                                          val.get_key_box()], None,
                                          descr=source_value.arraydescr)
                     self.emit_operation(newop)
             return True
@@ -478,6 +421,25 @@
             return True # 0-length arraycopy
         return False
 
+    def optimize_CALL_PURE(self, op):
+        arg_consts = []
+        for i in range(op.numargs()):
+            arg = op.getarg(i)
+            const = self.get_constant_box(arg)
+            if const is None:
+                break
+            arg_consts.append(const)
+        else:
+            # all constant arguments: check if we already know the result
+            try:
+                result = self.optimizer.call_pure_results[arg_consts]
+            except KeyError:
+                pass
+            else:
+                self.make_constant(op.result, result)
+                return
+        self.emit_operation(op)
+
     def optimize_INT_FLOORDIV(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
@@ -492,6 +454,10 @@
                                         args = [op.getarg(0), ConstInt(highest_bit(val))])
         self.emit_operation(op)
 
+    def optimize_CAST_OPAQUE_PTR(self, op):
+        value = self.getvalue(op.getarg(0))
+        self.optimizer.opaque_pointers[value] = True
+        self.make_equal_to(op.result, value)
 
 dispatch_opt = make_dispatcher_method(OptRewrite, 'optimize_',
         default=OptRewrite.emit_operation)
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -39,8 +39,9 @@
 
 def test_sharing_field_lists_of_virtual():
     class FakeOptimizer(object):
-        class cpu(object):
-            pass
+        class optimizer(object):
+            class cpu(object):
+                pass
     opt = FakeOptimizer()
     virt1 = virtualize.AbstractVirtualStructValue(opt, None)
     lst1 = virt1._get_field_descr_list()
@@ -69,7 +70,7 @@
     class FakeVirtualValue(virtualize.AbstractVirtualValue):
         def _make_virtual(self, *args):
             return FakeVInfo()
-    v1 = FakeVirtualValue(None, None, None)
+    v1 = FakeVirtualValue(None, None)
     vinfo1 = v1.make_virtual_info(None, [1, 2, 4])
     vinfo2 = v1.make_virtual_info(None, [1, 2, 4])
     assert vinfo1 is vinfo2
@@ -111,7 +112,7 @@
 
 class BaseTestBasic(BaseTest):
 
-    enable_opts = "intbounds:rewrite:virtualize:string:heap"
+    enable_opts = "intbounds:rewrite:virtualize:string:earlyforce:pure:heap"
 
     def optimize_loop(self, ops, optops, call_pure_results=None):
 
@@ -651,8 +652,8 @@
         i3 = getfield_gc(p3, descr=valuedescr)
         escape(i3)
         p1 = new_with_vtable(ConstClass(node_vtable))
+        p1sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p1, i1, descr=valuedescr)
-        p1sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p1sub, i1, descr=valuedescr)
         setfield_gc(p1, p1sub, descr=nextdescr)
         jump(i1, p1, p2)
@@ -667,10 +668,10 @@
         p3sub = getfield_gc(p3, descr=nextdescr)
         i3 = getfield_gc(p3sub, descr=valuedescr)
         escape(i3)
+        p1 = new_with_vtable(ConstClass(node_vtable))
         p2sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p2sub, i1, descr=valuedescr)
         setfield_gc(p2, p2sub, descr=nextdescr)
-        p1 = new_with_vtable(ConstClass(node_vtable))
         jump(i1, p1, p2)
         """
         # The same as test_p123_simple, but in the end the "old" p2 contains
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizefficall.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizefficall.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizefficall.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizefficall.py
@@ -36,7 +36,7 @@
 
 class TestFfiCall(BaseTestBasic, LLtypeMixin):
 
-    enable_opts = "intbounds:rewrite:virtualize:string:heap:ffi"
+    enable_opts = "intbounds:rewrite:virtualize:string:pure:earlyforce:heap:ffi"
 
     class namespace:
         cpu = LLtypeMixin.cpu
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -68,7 +68,7 @@
 
 class BaseTestWithUnroll(BaseTest):
 
-    enable_opts = "intbounds:rewrite:virtualize:string:heap:unroll"
+    enable_opts = "intbounds:rewrite:virtualize:string:earlyforce:pure:heap:unroll"
 
     def optimize_loop(self, ops, expected, expected_preamble=None,
                       call_pure_results=None, expected_short=None):
@@ -825,8 +825,8 @@
         i3 = getfield_gc(p2, descr=valuedescr)
         escape(i3)
         p4 = new_with_vtable(ConstClass(node_vtable))
+        p1sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p4, i1, descr=valuedescr)
-        p1sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p1sub, i1, descr=valuedescr)
         setfield_gc(p4, p1sub, descr=nextdescr)
         jump(i1, p4)
@@ -865,13 +865,7 @@
         p3sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p3sub, i1, descr=valuedescr)
         setfield_gc(p1, p3sub, descr=nextdescr)
-        # XXX: We get two extra operations here because the setfield
-        #      above is the result of forcing p1 and thus not
-        #      registered with the heap optimizer. I've makred tests
-        #      below with VIRTUALHEAP if they suffer from this issue
-        p3sub2 = getfield_gc(p1, descr=nextdescr)
-        guard_nonnull_class(p3sub2, ConstClass(node_vtable2)) []
-        jump(i1, p1, p3sub2)
+        jump(i1, p1, p3sub)
         """
         self.optimize_loop(ops, expected, preamble)
 
@@ -902,9 +896,7 @@
         guard_true(i2b) []
         p3 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3, i2, descr=nextdescr)
-        # XXX: VIRTUALHEAP (see above)
-        i3 = getfield_gc(p3, descr=nextdescr)
-        jump(p3, i3)
+        jump(p3, i2)
         """
         self.optimize_loop(ops, expected, preamble)
 
@@ -1219,7 +1211,15 @@
         setfield_gc(p3, p30, descr=valuedescr)
         jump(i29, p30, p3)
         """
-        expected = preamble
+        expected = """
+        [i0, p1, p3]
+        i28 = int_add(i0, 1)
+        i29 = int_add(i28, 1)
+        p30 = new_with_vtable(ConstClass(node_vtable))
+        setfield_gc(p3, p30, descr=valuedescr)
+        setfield_gc(p30, i28, descr=nextdescr)
+        jump(i29, p30, p3)
+        """
         self.optimize_loop(ops, expected, preamble)
 
     def test_nonvirtual_1(self):
@@ -2408,8 +2408,8 @@
         guard_class(p2, ConstClass(node_vtable)) []
         p3 = getfield_gc(p1, descr=otherdescr)
         guard_class(p3, ConstClass(node_vtable)) []
+        p3a = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3, p2, descr=otherdescr)
-        p3a = new_with_vtable(ConstClass(node_vtable))
         escape(p3a)
         jump(p3a)
         """
@@ -2421,9 +2421,9 @@
         # setfield_gc(p3, p2, descr=otherdescr) # p3a.other = p2a
         # p1a = new_with_vtable(ConstClass(node_vtable2))
         # p2a = new_with_vtable(ConstClass(node_vtable))
+        p3anew = new_with_vtable(ConstClass(node_vtable))
         p2 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3a, p2, descr=otherdescr) # p3a.other = p2a
-        p3anew = new_with_vtable(ConstClass(node_vtable))
         escape(p3anew)
         jump(p3anew)
         """
@@ -2458,9 +2458,9 @@
         p3 = getfield_gc(p1, descr=otherdescr)
         guard_class(p3, ConstClass(node_vtable)) []
         # p1a = new_with_vtable(ConstClass(node_vtable2))
+        p3a = new_with_vtable(ConstClass(node_vtable))
         p2a = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3, p2a, descr=otherdescr)
-        p3a = new_with_vtable(ConstClass(node_vtable))
         escape(p3a)
         # setfield_gc(p1a, p2a, descr=nextdescr)
         # setfield_gc(p1a, p3a, descr=otherdescr)
@@ -2468,9 +2468,9 @@
         """
         expected = """
         [p2, p3]
+        p3a = new_with_vtable(ConstClass(node_vtable))
         p2a = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3, p2a, descr=otherdescr)
-        p3a = new_with_vtable(ConstClass(node_vtable))
         escape(p3a)
         jump(p2a, p3a)
         """
@@ -2790,7 +2790,6 @@
         self.optimize_loop(ops, expected, preamble)
 
     def test_remove_duplicate_pure_op_ovf_with_lazy_setfield(self):
-        py.test.skip('this optimization is not yet supprted')
         ops = """
         [i1, p1]
         i3 = int_add_ovf(i1, 1)
@@ -5961,13 +5960,18 @@
         escape(i0)
         jump(p1)
         """
-        expected = """
+        preamble = """
         [p1]
         i0 = ptr_eq(p1, NULL)
         escape(i0)
-        jump(p1)
-        """
-        self.optimize_strunicode_loop_extradescrs(ops, expected, expected)
+        jump(p1, i0)
+        """
+        expected = """
+        [p1, i0]
+        escape(i0)
+        jump(p1, i0)
+        """
+        self.optimize_strunicode_loop_extradescrs(ops, expected, preamble)
 
     def test_str_equal_none2(self):
         ops = """
@@ -5976,13 +5980,18 @@
         escape(i0)
         jump(p1)
         """
-        expected = """
+        preamble = """
         [p1]
         i0 = ptr_eq(p1, NULL)
         escape(i0)
-        jump(p1)
-        """
-        self.optimize_strunicode_loop_extradescrs(ops, expected, expected)
+        jump(p1, i0)
+        """
+        expected = """
+        [p1, i0]
+        escape(i0)
+        jump(p1, i0)
+        """
+        self.optimize_strunicode_loop_extradescrs(ops, expected, preamble)
 
     def test_str_equal_nonnull1(self):
         ops = """
@@ -7220,6 +7229,60 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_heap_cache_forced_virtuals(self):
+        ops = """
+        [i1, i2, p0]
+        p1 = new(descr=ssize)
+        setfield_gc(p1, i1, descr=adescr)
+        setfield_gc(p1, i2, descr=bdescr)
+        call(p0, p1, descr=writeadescr)
+        i3 = getfield_gc(p1, descr=adescr)
+        i4 = getfield_gc(p1, descr=bdescr)
+        jump(i3, i4, p0)
+        """
+        expected = """
+        [i1, i2, p0]
+        p1 = new(descr=ssize)
+        setfield_gc(p1, i1, descr=adescr)
+        call(p0, p1, descr=writeadescr)
+        i3 = getfield_gc(p1, descr=adescr)
+        setfield_gc(p1, i2, descr=bdescr)
+        jump(i3, i2, p0)
+        """
+        self.optimize_loop(ops, expected)
+
+
+    def test_setarrayitem_followed_by_arraycopy(self):
+        ops = """
+        [p1, p2]
+        setarrayitem_gc(p1, 2, 10, descr=arraydescr)
+        setarrayitem_gc(p2, 3, 13, descr=arraydescr)        
+        call(0, p1, p2, 0, 0, 10, descr=arraycopydescr)
+        jump(p1, p2)
+        """
+        self.optimize_loop(ops, ops)
+
+    def test_heap_cache_virtuals_forced_by_delayed_setfield(self):
+        py.test.skip('not yet supoprted')
+        ops = """
+        [i1, p0]
+        p1 = new(descr=ssize)
+        setfield_gc(p1, i1, descr=valuedescr)
+        setfield_gc(p0, p1, descr=adescr)
+        call(p0, descr=writeadescr)
+        i2 = getfield_gc(p1, descr=valuedescr)
+        jump(i2, p0)
+        """
+        expected = """
+        [i1, p0]
+        p1 = new(descr=ssize)
+        setfield_gc(p1, i1, descr=valuedescr)
+        setfield_gc(p0, p1, descr=adescr)
+        call(p0, descr=writeadescr)
+        jump(i1, p0)
+        """
+        self.optimize_loop(ops, expected)
+
 class TestLLtype(OptimizeOptTest, LLtypeMixin):
     pass
 
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_util.py b/pypy/jit/metainterp/optimizeopt/test/test_util.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_util.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_util.py
@@ -182,7 +182,8 @@
                             EffectInfo.EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE,
                             can_invalidate=True))
     arraycopydescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT,
-             EffectInfo([], [], [], [], oopspecindex=EffectInfo.OS_ARRAYCOPY))
+             EffectInfo([], [arraydescr], [], [arraydescr],
+                        oopspecindex=EffectInfo.OS_ARRAYCOPY))
 
     for _name, _os in [
         ('strconcatdescr',               'OS_STR_CONCAT'),
diff --git a/pypy/jit/metainterp/optimizeopt/unroll.py b/pypy/jit/metainterp/optimizeopt/unroll.py
--- a/pypy/jit/metainterp/optimizeopt/unroll.py
+++ b/pypy/jit/metainterp/optimizeopt/unroll.py
@@ -75,7 +75,6 @@
         self.importable_values = {}
         self.emitting_dissabled = False
         self.emitted_guards = 0
-        self.emitted_pure_operations = {}
 
     def ensure_imported(self, value):
         if not self.emitting_dissabled and value in self.importable_values:
@@ -96,21 +95,6 @@
         new = UnrollableOptimizer(self.metainterp_sd, self.loop)
         return self._new(new)
 
-    def remember_emitting_pure(self, op):
-        self.emitted_pure_operations[op] = True
-
-    def produce_potential_short_preamble_ops(self, sb):
-        for op in self.emitted_pure_operations:
-            if op.getopnum() == rop.GETARRAYITEM_GC_PURE or \
-               op.getopnum() == rop.STRGETITEM or \
-               op.getopnum() == rop.UNICODEGETITEM:
-                if not self.getvalue(op.getarg(1)).is_constant():
-                    continue
-            sb.add_potential(op)
-        for opt in self.optimizations:
-            opt.produce_potential_short_preamble_ops(sb)
-
-
 
 class UnrollOptimizer(Optimization):
     """Unroll the loop into two iterations. The first one will
@@ -154,7 +138,7 @@
 
             KillHugeIntBounds(self.optimizer).apply()
             
-            loop.preamble.operations = self.optimizer.newoperations
+            loop.preamble.operations = self.optimizer.get_newoperations()
             jump_args = [self.getvalue(a).get_key_box() for a in jump_args]
 
             start_resumedescr = loop.preamble.start_resumedescr.clone_if_mutable()
@@ -167,8 +151,9 @@
             virtual_state = modifier.get_virtual_state(jump_args)
             
             values = [self.getvalue(arg) for arg in jump_args]
-            inputargs = virtual_state.make_inputargs(values)
-            short_inputargs = virtual_state.make_inputargs(values, keyboxes=True)
+            inputargs = virtual_state.make_inputargs(values, self.optimizer)
+            short_inputargs = virtual_state.make_inputargs(values, self.optimizer,
+                                                           keyboxes=True)
 
             self.constant_inputargs = {}
             for box in jump_args: 
@@ -197,7 +182,7 @@
             # operations needed to setup the proper state of those virtuals
             # in the peeled loop
             inputarg_setup_ops = []
-            preamble_optimizer.newoperations = []
+            preamble_optimizer.clear_newoperations()
             seen = {}
             for box in inputargs:
                 if box in seen:
@@ -211,9 +196,8 @@
                     continue
                 seen[box] = True
                 value = preamble_optimizer.getvalue(box)
-                value.force_box()
-            preamble_optimizer.flush()
-            inputarg_setup_ops += preamble_optimizer.newoperations
+                value.force_box(preamble_optimizer)
+            inputarg_setup_ops += preamble_optimizer.get_newoperations()
 
             # Setup the state of the new optimizer by emiting the
             # short preamble operations and discarding the result
@@ -244,13 +228,13 @@
                                 virtual_state)
             
             loop.inputargs = inputargs
-            args = [preamble_optimizer.getvalue(self.short_boxes.original(a)).force_box()\
+            args = [preamble_optimizer.getvalue(self.short_boxes.original(a)).force_box(preamble_optimizer)\
                     for a in inputargs]
             jmp = ResOperation(rop.JUMP, args, None)
             jmp.setdescr(loop.token)
             loop.preamble.operations.append(jmp)
 
-            loop.operations = self.optimizer.newoperations
+            loop.operations = self.optimizer.get_newoperations()
             maxguards = self.optimizer.metainterp_sd.warmrunnerdesc.memory_manager.max_retrace_guards
             
             if self.optimizer.emitted_guards > maxguards:
@@ -335,9 +319,10 @@
         assert jumpop
         original_jumpargs = jumpop.getarglist()[:]
         values = [self.getvalue(arg) for arg in jumpop.getarglist()]
-        jumpargs = virtual_state.make_inputargs(values)
+        jumpargs = virtual_state.make_inputargs(values, self.optimizer)
         jumpop.initarglist(jumpargs)
-        jmp_to_short_args = virtual_state.make_inputargs(values, keyboxes=True)
+        jmp_to_short_args = virtual_state.make_inputargs(values, self.optimizer,
+                                                         keyboxes=True)
         self.short_inliner = Inliner(short_inputargs, jmp_to_short_args)
         
         for box, const in self.constant_inputargs.items():
@@ -347,11 +332,11 @@
             newop = self.short_inliner.inline_op(op)
             self.optimizer.send_extra_operation(newop)
         
-        self.optimizer.flush()
+        newoperations = self.optimizer.get_newoperations()
 
         i = j = 0
-        while i < len(self.optimizer.newoperations) or j < len(jumpargs):
-            if i == len(self.optimizer.newoperations):
+        while i < len(newoperations) or j < len(jumpargs):
+            if i == len(newoperations):
                 while j < len(jumpargs):
                     a = jumpargs[j]
                     if self.optimizer.loop.logops:
@@ -360,7 +345,7 @@
                                     jumpargs, short_seen)
                     j += 1
             else:
-                op = self.optimizer.newoperations[i]
+                op = newoperations[i]
 
                 self.boxes_created_this_iteration[op.result] = True
                 args = op.getarglist()
@@ -375,6 +360,7 @@
                     self.import_box(a, inputargs, short, short_jumpargs,
                                     jumpargs, short_seen)
                 i += 1
+            newoperations = self.optimizer.get_newoperations()
 
         jumpop.initarglist(jumpargs)
         self.optimizer.send_extra_operation(jumpop)
@@ -468,7 +454,7 @@
         inputargs.append(box)
         box = newresult
         if box in self.optimizer.values:
-            box = self.optimizer.values[box].force_box()
+            box = self.optimizer.values[box].force_box(self.optimizer)
         jumpargs.append(box)
         
 
@@ -483,11 +469,6 @@
         if op.getopnum() == rop.JUMP:
             loop_token = op.getdescr()
             assert isinstance(loop_token, LoopToken)
-            # FIXME: Use a tree, similar to the tree formed by the full
-            # preamble and it's bridges, instead of a list to save time and
-            # memory. This should also allow better behaviour in
-            # situations that the is_emittable() chain currently cant
-            # handle and the inlining fails unexpectedly belwo.
             short = loop_token.short_preamble
             if short:
                 args = op.getarglist()
@@ -523,7 +504,7 @@
 
                         values = [self.getvalue(arg)
                                   for arg in op.getarglist()]
-                        args = sh.virtual_state.make_inputargs(values,
+                        args = sh.virtual_state.make_inputargs(values, self.optimizer,
                                                                keyboxes=True)
                         inliner = Inliner(sh.inputargs, args)
                         
diff --git a/pypy/jit/metainterp/optimizeopt/virtualize.py b/pypy/jit/metainterp/optimizeopt/virtualize.py
--- a/pypy/jit/metainterp/optimizeopt/virtualize.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualize.py
@@ -10,13 +10,12 @@
 
 
 class AbstractVirtualValue(optimizer.OptValue):
-    _attrs_ = ('optimizer', 'keybox', 'source_op', '_cached_vinfo')
+    _attrs_ = ('keybox', 'source_op', '_cached_vinfo')
     box = None
     level = optimizer.LEVEL_NONNULL
     _cached_vinfo = None
 
-    def __init__(self, optimizer, keybox, source_op=None):
-        self.optimizer = optimizer
+    def __init__(self, keybox, source_op=None):
         self.keybox = keybox   # only used as a key in dictionaries
         self.source_op = source_op  # the NEW_WITH_VTABLE/NEW_ARRAY operation
                                     # that builds this box
@@ -29,17 +28,17 @@
             return self.keybox
         return self.box
 
-    def force_box(self):
+    def force_box(self, optforce):
         if self.box is None:
-            self.optimizer.forget_numberings(self.keybox)
-            self._really_force()
+            optforce.forget_numberings(self.keybox)
+            self._really_force(optforce)
         return self.box
 
-    def force_at_end_of_preamble(self, already_forced):
+    def force_at_end_of_preamble(self, already_forced, optforce):
         value = already_forced.get(self, None)
         if value:
             return value
-        return OptValue(self.force_box())
+        return OptValue(self.force_box(optforce))
 
     def make_virtual_info(self, modifier, fieldnums):
         if fieldnums is None:
@@ -55,7 +54,7 @@
     def _make_virtual(self, modifier):
         raise NotImplementedError("abstract base")
 
-    def _really_force(self):
+    def _really_force(self, optforce):
         raise NotImplementedError("abstract base")
 
     def import_from(self, other, optimizer):
@@ -70,10 +69,11 @@
 get_fielddescrlist_cache._annspecialcase_ = "specialize:memo"
 
 class AbstractVirtualStructValue(AbstractVirtualValue):
-    _attrs_ = ('_fields', '_cached_sorted_fields')
+    _attrs_ = ('_fields', 'cpu', '_cached_sorted_fields')
 
-    def __init__(self, optimizer, keybox, source_op=None):
-        AbstractVirtualValue.__init__(self, optimizer, keybox, source_op)
+    def __init__(self, cpu, keybox, source_op=None):
+        AbstractVirtualValue.__init__(self, keybox, source_op)
+        self.cpu = cpu
         self._fields = {}
         self._cached_sorted_fields = None
 
@@ -87,45 +87,44 @@
     def _get_descr(self):
         raise NotImplementedError
 
-    def _is_immutable_and_filled_with_constants(self):
+    def _is_immutable_and_filled_with_constants(self, optforce):
         count = self._get_descr().count_fields_if_immutable()
         if count != len(self._fields):    # always the case if count == -1
             return False
         for value in self._fields.itervalues():
-            subbox = value.force_box()
+            subbox = value.force_box(optforce)
             if not isinstance(subbox, Const):
                 return False
         return True
 
-    def force_at_end_of_preamble(self, already_forced):
+    def force_at_end_of_preamble(self, already_forced, optforce):
         if self in already_forced:
             return self
         already_forced[self] = self
         if self._fields:
             for ofs in self._fields.keys():
-                self._fields[ofs] = self._fields[ofs].force_at_end_of_preamble(already_forced)
+                self._fields[ofs] = self._fields[ofs].force_at_end_of_preamble(already_forced, optforce)
         return self
 
-    def _really_force(self):
+    def _really_force(self, optforce):
         op = self.source_op
         assert op is not None
         # ^^^ This case should not occur any more (see test_bug_3).
         #
         if not we_are_translated():
             op.name = 'FORCE ' + self.source_op.name
-
-        if self._is_immutable_and_filled_with_constants():
-            box = self.optimizer.constant_fold(op)
+            
+        if self._is_immutable_and_filled_with_constants(optforce):
+            box = optforce.optimizer.constant_fold(op)
             self.make_constant(box)
             for ofs, value in self._fields.iteritems():
-                subbox = value.force_box()
+                subbox = value.force_box(optforce)
                 assert isinstance(subbox, Const)
-                execute(self.optimizer.cpu, None, rop.SETFIELD_GC,
+                execute(optforce.optimizer.cpu, None, rop.SETFIELD_GC,
                         ofs, box, subbox)
             # keep self._fields, because it's all immutable anyway
         else:
-            newoperations = self.optimizer.newoperations
-            newoperations.append(op)
+            optforce.emit_operation(op)
             self.box = box = op.result
             #
             iteritems = self._fields.iteritems()
@@ -135,10 +134,11 @@
             for ofs, value in iteritems:
                 if value.is_null():
                     continue
-                subbox = value.force_box()
+                subbox = value.force_box(optforce)
                 op = ResOperation(rop.SETFIELD_GC, [box, subbox], None,
                                   descr=ofs)
-                newoperations.append(op)
+
+                optforce.emit_operation(op)
 
     def _get_field_descr_list(self):
         _cached_sorted_fields = self._cached_sorted_fields
@@ -155,7 +155,7 @@
             else:
                 lst = self._fields.keys()
             sort_descrs(lst)
-            cache = get_fielddescrlist_cache(self.optimizer.cpu)
+            cache = get_fielddescrlist_cache(self.cpu)
             result = cache.get(lst, None)
             if result is None:
                 cache[lst] = lst
@@ -180,8 +180,8 @@
 class VirtualValue(AbstractVirtualStructValue):
     level = optimizer.LEVEL_KNOWNCLASS
 
-    def __init__(self, optimizer, known_class, keybox, source_op=None):
-        AbstractVirtualStructValue.__init__(self, optimizer, keybox, source_op)
+    def __init__(self, cpu, known_class, keybox, source_op=None):
+        AbstractVirtualStructValue.__init__(self, cpu, keybox, source_op)
         assert isinstance(known_class, Const)
         self.known_class = known_class
 
@@ -190,7 +190,7 @@
         return modifier.make_virtual(self.known_class, fielddescrs)
 
     def _get_descr(self):
-        return vtable2descr(self.optimizer.cpu, self.known_class.getint())
+        return vtable2descr(self.cpu, self.known_class.getint())
 
     def __repr__(self):
         cls_name = self.known_class.value.adr.ptr._obj._TYPE._name
@@ -201,8 +201,8 @@
 
 class VStructValue(AbstractVirtualStructValue):
 
-    def __init__(self, optimizer, structdescr, keybox, source_op=None):
-        AbstractVirtualStructValue.__init__(self, optimizer, keybox, source_op)
+    def __init__(self, cpu, structdescr, keybox, source_op=None):
+        AbstractVirtualStructValue.__init__(self, cpu, keybox, source_op)
         self.structdescr = structdescr
 
     def _make_virtual(self, modifier):
@@ -215,10 +215,10 @@
 
 class VArrayValue(AbstractVirtualValue):
 
-    def __init__(self, optimizer, arraydescr, size, keybox, source_op=None):
-        AbstractVirtualValue.__init__(self, optimizer, keybox, source_op)
+    def __init__(self, arraydescr, constvalue, size, keybox, source_op=None):
+        AbstractVirtualValue.__init__(self, keybox, source_op)
         self.arraydescr = arraydescr
-        self.constvalue = optimizer.new_const_item(arraydescr)
+        self.constvalue = constvalue
         self._items = [self.constvalue] * size
 
     def getlength(self):
@@ -232,31 +232,30 @@
         assert isinstance(itemvalue, optimizer.OptValue)
         self._items[index] = itemvalue
 
-    def force_at_end_of_preamble(self, already_forced):
+    def force_at_end_of_preamble(self, already_forced, optforce):
         if self in already_forced:
             return self
         already_forced[self] = self
         for index in range(len(self._items)):
-            self._items[index] = self._items[index].force_at_end_of_preamble(already_forced)
+            self._items[index] = self._items[index].force_at_end_of_preamble(already_forced, optforce)
         return self
     
-    def _really_force(self):
+    def _really_force(self, optforce):
         assert self.source_op is not None
         if not we_are_translated():
             self.source_op.name = 'FORCE ' + self.source_op.name
-        newoperations = self.optimizer.newoperations
-        newoperations.append(self.source_op)
+        optforce.emit_operation(self.source_op)        
         self.box = box = self.source_op.result
         for index in range(len(self._items)):
             subvalue = self._items[index]
             if subvalue is not self.constvalue:
                 if subvalue.is_null():
                     continue
-                subbox = subvalue.force_box()
+                subbox = subvalue.force_box(optforce)
                 op = ResOperation(rop.SETARRAYITEM_GC,
                                   [box, ConstInt(index), subbox], None,
                                   descr=self.arraydescr)
-                newoperations.append(op)
+                optforce.emit_operation(op)
 
     def get_args_for_fail(self, modifier):
         if self.box is None and not modifier.already_seen_virtual(self.keybox):
@@ -279,17 +278,18 @@
         return OptVirtualize()
         
     def make_virtual(self, known_class, box, source_op=None):
-        vvalue = VirtualValue(self.optimizer, known_class, box, source_op)
+        vvalue = VirtualValue(self.optimizer.cpu, known_class, box, source_op)
         self.make_equal_to(box, vvalue)
         return vvalue
 
     def make_varray(self, arraydescr, size, box, source_op=None):
-        vvalue = VArrayValue(self.optimizer, arraydescr, size, box, source_op)
+        constvalue = self.new_const_item(arraydescr)
+        vvalue = VArrayValue(arraydescr, constvalue, size, box, source_op)
         self.make_equal_to(box, vvalue)
         return vvalue
 
     def make_vstruct(self, structdescr, box, source_op=None):
-        vvalue = VStructValue(self.optimizer, structdescr, box, source_op)
+        vvalue = VStructValue(self.optimizer.cpu, structdescr, box, source_op)
         self.make_equal_to(box, vvalue)
         return vvalue
 
@@ -362,7 +362,6 @@
             self.make_equal_to(op.result, fieldvalue)
         else:
             value.ensure_nonnull()
-            ###self.heap_op_optimizer.optimize_GETFIELD_GC(op, value)
             self.emit_operation(op)
 
     # note: the following line does not mean that the two operations are
@@ -377,7 +376,6 @@
             value.setfield(op.getdescr(), fieldvalue)
         else:
             value.ensure_nonnull()
-            ###self.heap_op_optimizer.optimize_SETFIELD_GC(op, value, fieldvalue)
             self.emit_operation(op)
 
     def optimize_NEW_WITH_VTABLE(self, op):
@@ -417,7 +415,6 @@
                 self.make_equal_to(op.result, itemvalue)
                 return
         value.ensure_nonnull()
-        ###self.heap_op_optimizer.optimize_GETARRAYITEM_GC(op, value)
         self.emit_operation(op)
 
     # note: the following line does not mean that the two operations are
@@ -432,7 +429,6 @@
                 value.setitem(indexbox.getint(), self.getvalue(op.getarg(2)))
                 return
         value.ensure_nonnull()
-        ###self.heap_op_optimizer.optimize_SETARRAYITEM_GC(op, value, fieldvalue)
         self.emit_operation(op)
 
 
diff --git a/pypy/jit/metainterp/optimizeopt/virtualstate.py b/pypy/jit/metainterp/optimizeopt/virtualstate.py
--- a/pypy/jit/metainterp/optimizeopt/virtualstate.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualstate.py
@@ -30,7 +30,7 @@
     def _generate_guards(self, other, box, cpu, extra_guards):
         raise InvalidLoop
 
-    def enum_forced_boxes(self, boxes, value):
+    def enum_forced_boxes(self, boxes, value, optimizer):
         raise NotImplementedError
 
     def enum(self, virtual_state):
@@ -100,14 +100,14 @@
     def _generalization_of(self, other):
         raise NotImplementedError
 
-    def enum_forced_boxes(self, boxes, value):
+    def enum_forced_boxes(self, boxes, value, optimizer):
         assert isinstance(value, virtualize.AbstractVirtualStructValue)
         assert value.is_virtual()
         for i in range(len(self.fielddescrs)):
             v = value._fields[self.fielddescrs[i]]
             s = self.fieldstate[i]
             if s.position > self.position:
-                s.enum_forced_boxes(boxes, v)
+                s.enum_forced_boxes(boxes, v, optimizer)
 
     def _enum(self, virtual_state):
         for s in self.fieldstate:
@@ -177,14 +177,14 @@
                 return False
         return True
 
-    def enum_forced_boxes(self, boxes, value):
+    def enum_forced_boxes(self, boxes, value, optimizer):
         assert isinstance(value, virtualize.VArrayValue)
         assert value.is_virtual()
         for i in range(len(self.fieldstate)):
             v = value._items[i]
             s = self.fieldstate[i]
             if s.position > self.position:
-                s.enum_forced_boxes(boxes, v)
+                s.enum_forced_boxes(boxes, v, optimizer)
 
     def _enum(self, virtual_state):
         for s in self.fieldstate:
@@ -316,11 +316,11 @@
             import pdb; pdb.set_trace()
             raise NotImplementedError
 
-    def enum_forced_boxes(self, boxes, value):
+    def enum_forced_boxes(self, boxes, value, optimizer):
         if self.level == LEVEL_CONSTANT:
             return
         assert 0 <= self.position_in_notvirtuals 
-        boxes[self.position_in_notvirtuals] = value.force_box()
+        boxes[self.position_in_notvirtuals] = value.force_box(optimizer)
 
     def _enum(self, virtual_state):
         if self.level == LEVEL_CONSTANT:
@@ -377,11 +377,13 @@
             self.state[i].generate_guards(other.state[i], args[i],
                                           cpu, extra_guards, renum)
 
-    def make_inputargs(self, values, keyboxes=False):
+    def make_inputargs(self, values, optimizer, keyboxes=False):
+        if optimizer.optearlyforce:
+            optimizer = optimizer.optearlyforce
         assert len(values) == len(self.state)
         inputargs = [None] * len(self.notvirtuals)
         for i in range(len(values)):
-            self.state[i].enum_forced_boxes(inputargs, values[i])
+            self.state[i].enum_forced_boxes(inputargs, values[i], optimizer)
 
         if keyboxes:
             for i in range(len(values)):
@@ -434,7 +436,12 @@
     def get_virtual_state(self, jump_args):
         self.optimizer.force_at_end_of_preamble()
         already_forced = {}
-        values = [self.getvalue(box).force_at_end_of_preamble(already_forced)
+        if self.optimizer.optearlyforce:
+            opt = self.optimizer.optearlyforce
+        else:
+            opt = self.optimizer
+        values = [self.getvalue(box).force_at_end_of_preamble(already_forced,
+                                                              opt)
                   for box in jump_args]
 
         for value in values:
diff --git a/pypy/jit/metainterp/optimizeopt/vstring.py b/pypy/jit/metainterp/optimizeopt/vstring.py
--- a/pypy/jit/metainterp/optimizeopt/vstring.py
+++ b/pypy/jit/metainterp/optimizeopt/vstring.py
@@ -43,7 +43,7 @@
 class __extend__(optimizer.OptValue):
     """New methods added to the base class OptValue for this file."""
 
-    def getstrlen(self, optimization, mode):
+    def getstrlen(self, string_optimizer, mode):
         if mode is mode_string:
             s = self.get_constant_string_spec(mode_string)
             if s is not None:
@@ -52,12 +52,12 @@
             s = self.get_constant_string_spec(mode_unicode)
             if s is not None:
                 return ConstInt(len(s))
-        if optimization is None:
+        if string_optimizer is None:
             return None
         self.ensure_nonnull()
-        box = self.force_box()
+        box = self.force_box(string_optimizer)
         lengthbox = BoxInt()
-        optimization.propagate_forward(ResOperation(mode.STRLEN, [box], lengthbox))
+        string_optimizer.emit_operation(ResOperation(mode.STRLEN, [box], lengthbox))
         return lengthbox
 
     @specialize.arg(1)
@@ -68,25 +68,25 @@
         else:
             return None
 
-    def string_copy_parts(self, optimizer, targetbox, offsetbox, mode):
+    def string_copy_parts(self, string_optimizer, targetbox, offsetbox, mode):
         # Copies the pointer-to-string 'self' into the target string
         # given by 'targetbox', at the specified offset.  Returns the offset
         # at the end of the copy.
-        lengthbox = self.getstrlen(optimizer, mode)
-        srcbox = self.force_box()
-        return copy_str_content(optimizer, srcbox, targetbox,
+        lengthbox = self.getstrlen(string_optimizer, mode)
+        srcbox = self.force_box(string_optimizer)
+        return copy_str_content(string_optimizer, srcbox, targetbox,
                                 CONST_0, offsetbox, lengthbox, mode)
 
 
 class VAbstractStringValue(virtualize.AbstractVirtualValue):
     _attrs_ = ('mode',)
 
-    def __init__(self, optimizer, keybox, source_op, mode):
-        virtualize.AbstractVirtualValue.__init__(self, optimizer, keybox,
+    def __init__(self, keybox, source_op, mode):
+        virtualize.AbstractVirtualValue.__init__(self, keybox,
                                                  source_op)
         self.mode = mode
 
-    def _really_force(self):
+    def _really_force(self, optforce):
         if self.mode is mode_string:
             s = self.get_constant_string_spec(mode_string)
             if s is not None:
@@ -101,12 +101,12 @@
                 return
         assert self.source_op is not None
         self.box = box = self.source_op.result
-        lengthbox = self.getstrlen(self.optimizer, self.mode)
+        lengthbox = self.getstrlen(optforce, self.mode)
         op = ResOperation(self.mode.NEWSTR, [lengthbox], box)
         if not we_are_translated():
             op.name = 'FORCE'
-        self.optimizer.emit_operation(op)
-        self.string_copy_parts(self.optimizer, box, CONST_0, self.mode)
+        optforce.emit_operation(op)
+        self.string_copy_parts(optforce, box, CONST_0, self.mode)
 
 
 class VStringPlainValue(VAbstractStringValue):
@@ -140,20 +140,20 @@
         return mode.emptystr.join([mode.chr(c.box.getint())
                                    for c in self._chars])
 
-    def string_copy_parts(self, optimizer, targetbox, offsetbox, mode):
+    def string_copy_parts(self, string_optimizer, targetbox, offsetbox, mode):
         if not self.is_virtual() and targetbox is not self.box:
-            lengthbox = self.getstrlen(optimizer, mode)
-            srcbox = self.force_box()
-            return copy_str_content(optimizer, srcbox, targetbox,
+            lengthbox = self.getstrlen(string_optimizer, mode)
+            srcbox = self.force_box(string_optimizer)
+            return copy_str_content(string_optimizer, srcbox, targetbox,
                                 CONST_0, offsetbox, lengthbox, mode)
         for i in range(len(self._chars)):
-            charbox = self._chars[i].force_box()
+            charbox = self._chars[i].force_box(string_optimizer)
             if not (isinstance(charbox, Const) and charbox.same_constant(CONST_0)):
-                optimizer.emit_operation(ResOperation(mode.STRSETITEM, [targetbox,
-                                                                    offsetbox,
-                                                                    charbox],
+                string_optimizer.emit_operation(ResOperation(mode.STRSETITEM, [targetbox,
+                                                                               offsetbox,
+                                                                               charbox],
                                                   None))
-            offsetbox = _int_add(optimizer, offsetbox, CONST_1)
+            offsetbox = _int_add(string_optimizer, offsetbox, CONST_1)
         return offsetbox
 
     def get_args_for_fail(self, modifier):
@@ -187,16 +187,16 @@
         self.left = left
         self.right = right
 
-    def getstrlen(self, optimizer, mode):
+    def getstrlen(self, string_optimizer, mode):
         if self.lengthbox is None:
-            len1box = self.left.getstrlen(optimizer, mode)
+            len1box = self.left.getstrlen(string_optimizer, mode)
             if len1box is None:
                 return None
-            len2box = self.right.getstrlen(optimizer, mode)
+            len2box = self.right.getstrlen(string_optimizer, mode)
             if len2box is None:
                 return None
-            self.lengthbox = _int_add(optimizer, len1box, len2box)
-            # ^^^ may still be None, if optimizer is None
+            self.lengthbox = _int_add(string_optimizer, len1box, len2box)
+            # ^^^ may still be None, if string_optimizer is None
         return self.lengthbox
 
     @specialize.arg(1)
@@ -209,10 +209,10 @@
             return None
         return s1 + s2
 
-    def string_copy_parts(self, optimizer, targetbox, offsetbox, mode):
-        offsetbox = self.left.string_copy_parts(optimizer, targetbox,
+    def string_copy_parts(self, string_optimizer, targetbox, offsetbox, mode):
+        offsetbox = self.left.string_copy_parts(string_optimizer, targetbox,
                                                 offsetbox, mode)
-        offsetbox = self.right.string_copy_parts(optimizer, targetbox,
+        offsetbox = self.right.string_copy_parts(string_optimizer, targetbox,
                                                  offsetbox, mode)
         return offsetbox
 
@@ -251,8 +251,8 @@
         self.vstart = vstart
         self.vlength = vlength
 
-    def getstrlen(self, _, mode):
-        return self.vlength.force_box()
+    def getstrlen(self, optforce, mode):
+        return self.vlength.force_box(optforce)
 
     @specialize.arg(1)
     def get_constant_string_spec(self, mode):
@@ -267,11 +267,11 @@
             return s1[start : start + length]
         return None
 
-    def string_copy_parts(self, optimizer, targetbox, offsetbox, mode):
-        lengthbox = self.getstrlen(optimizer, mode)
-        return copy_str_content(optimizer,
-                                self.vstr.force_box(), targetbox,
-                                self.vstart.force_box(), offsetbox,
+    def string_copy_parts(self, string_optimizer, targetbox, offsetbox, mode):
+        lengthbox = self.getstrlen(string_optimizer, mode)
+        return copy_str_content(string_optimizer,
+                                self.vstr.force_box(string_optimizer), targetbox,
+                                self.vstart.force_box(string_optimizer), offsetbox,
                                 lengthbox, mode)
 
     def get_args_for_fail(self, modifier):
@@ -300,7 +300,7 @@
         return modifier.make_vstrslice(self.mode is mode_unicode)
 
 
-def copy_str_content(optimizer, srcbox, targetbox,
+def copy_str_content(string_optimizer, srcbox, targetbox,
                      srcoffsetbox, offsetbox, lengthbox, mode, need_next_offset=True):
     if isinstance(srcbox, ConstPtr) and isinstance(srcoffsetbox, Const):
         M = 5
@@ -310,26 +310,26 @@
         # up to M characters are done "inline", i.e. with STRGETITEM/STRSETITEM
         # instead of just a COPYSTRCONTENT.
         for i in range(lengthbox.value):
-            charbox = _strgetitem(optimizer, srcbox, srcoffsetbox, mode)
-            srcoffsetbox = _int_add(optimizer, srcoffsetbox, CONST_1)
-            optimizer.emit_operation(ResOperation(mode.STRSETITEM, [targetbox,
-                                                                offsetbox,
-                                                                charbox],
+            charbox = _strgetitem(string_optimizer, srcbox, srcoffsetbox, mode)
+            srcoffsetbox = _int_add(string_optimizer, srcoffsetbox, CONST_1)
+            string_optimizer.emit_operation(ResOperation(mode.STRSETITEM, [targetbox,
+                                                                           offsetbox,
+                                                                           charbox],
                                               None))
-            offsetbox = _int_add(optimizer, offsetbox, CONST_1)
+            offsetbox = _int_add(string_optimizer, offsetbox, CONST_1)
     else:
         if need_next_offset:
-            nextoffsetbox = _int_add(optimizer, offsetbox, lengthbox)
+            nextoffsetbox = _int_add(string_optimizer, offsetbox, lengthbox)
         else:
             nextoffsetbox = None
         op = ResOperation(mode.COPYSTRCONTENT, [srcbox, targetbox,
                                                 srcoffsetbox, offsetbox,
                                                 lengthbox], None)
-        optimizer.emit_operation(op)
+        string_optimizer.emit_operation(op)
         offsetbox = nextoffsetbox
     return offsetbox
 
-def _int_add(optimizer, box1, box2):
+def _int_add(string_optimizer, box1, box2):
     if isinstance(box1, ConstInt):
         if box1.value == 0:
             return box2
@@ -337,23 +337,23 @@
             return ConstInt(box1.value + box2.value)
     elif isinstance(box2, ConstInt) and box2.value == 0:
         return box1
-    if optimizer is None:
+    if string_optimizer is None:
         return None
     resbox = BoxInt()
-    optimizer.propagate_forward(ResOperation(rop.INT_ADD, [box1, box2], resbox))
+    string_optimizer.emit_operation(ResOperation(rop.INT_ADD, [box1, box2], resbox))
     return resbox
 
-def _int_sub(optimizer, box1, box2):
+def _int_sub(string_optimizer, box1, box2):
     if isinstance(box2, ConstInt):
         if box2.value == 0:
             return box1
         if isinstance(box1, ConstInt):
             return ConstInt(box1.value - box2.value)
     resbox = BoxInt()
-    optimizer.propagate_forward(ResOperation(rop.INT_SUB, [box1, box2], resbox))
+    string_optimizer.emit_operation(ResOperation(rop.INT_SUB, [box1, box2], resbox))
     return resbox
 
-def _strgetitem(optimizer, strbox, indexbox, mode):
+def _strgetitem(string_optimizer, strbox, indexbox, mode):
     if isinstance(strbox, ConstPtr) and isinstance(indexbox, ConstInt):
         if mode is mode_string:
             s = strbox.getref(lltype.Ptr(rstr.STR))
@@ -362,30 +362,28 @@
             s = strbox.getref(lltype.Ptr(rstr.UNICODE))
             return ConstInt(ord(s.chars[indexbox.getint()]))
     resbox = BoxInt()
-    optimizer.propagate_forward(ResOperation(mode.STRGETITEM, [strbox, indexbox],
-                                      resbox))
+    string_optimizer.emit_operation(ResOperation(mode.STRGETITEM, [strbox, indexbox],
+                                                 resbox))
     return resbox
 
 
 class OptString(optimizer.Optimization):
     "Handling of strings and unicodes."
-    enabled = True
-
     def new(self):
         return OptString()
 
     def make_vstring_plain(self, box, source_op, mode):
-        vvalue = VStringPlainValue(self.optimizer, box, source_op, mode)
+        vvalue = VStringPlainValue(box, source_op, mode)
         self.make_equal_to(box, vvalue)
         return vvalue
 
     def make_vstring_concat(self, box, source_op, mode):
-        vvalue = VStringConcatValue(self.optimizer, box, source_op, mode)
+        vvalue = VStringConcatValue(box, source_op, mode)
         self.make_equal_to(box, vvalue)
         return vvalue
 
     def make_vstring_slice(self, box, source_op, mode):
-        vvalue = VStringSliceValue(self.optimizer, box, source_op, mode)
+        vvalue = VStringSliceValue(box, source_op, mode)
         self.make_equal_to(box, vvalue)
         return vvalue
 
@@ -435,9 +433,9 @@
         value.ensure_nonnull()
         #
         if value.is_virtual() and isinstance(value, VStringSliceValue):
-            fullindexbox = _int_add(self.optimizer,
-                                    value.vstart.force_box(),
-                                    vindex.force_box())
+            fullindexbox = _int_add(self,
+                                    value.vstart.force_box(self),
+                                    vindex.force_box(self))
             value = value.vstr
             vindex = self.getvalue(fullindexbox)
         #
@@ -449,7 +447,7 @@
                 if res is not optimizer.CVAL_UNINITIALIZED_ZERO:
                     return res
         #
-        resbox = _strgetitem(self.optimizer, value.force_box(), vindex.force_box(), mode)
+        resbox = _strgetitem(self, value.force_box(self), vindex.force_box(self), mode)
         return self.getvalue(resbox)
 
     def optimize_STRLEN(self, op):
@@ -459,7 +457,7 @@
 
     def _optimize_STRLEN(self, op, mode):
         value = self.getvalue(op.getarg(0))
-        lengthbox = value.getstrlen(self.optimizer, mode)
+        lengthbox = value.getstrlen(self, mode)
         self.make_equal_to(op.result, self.getvalue(lengthbox))
 
     def optimize_COPYSTRCONTENT(self, op):
@@ -477,12 +475,12 @@
 
         if length.is_constant() and length.box.getint() == 0:
             return
-        copy_str_content(self.optimizer,
-            src.force_box(),
-            dst.force_box(),
-            srcstart.force_box(),
-            dststart.force_box(),
-            length.force_box(),
+        copy_str_content(self,
+            src.force_box(self),
+            dst.force_box(self),
+            srcstart.force_box(self),
+            dststart.force_box(self),
+            length.force_box(self),
             mode, need_next_offset=False
         )
 
@@ -508,6 +506,8 @@
                     return
         self.emit_operation(op)
 
+    optimize_CALL_PURE = optimize_CALL
+
     def opt_call_str_STR2UNICODE(self, op):
         # Constant-fold unicode("constant string").
         # More generally, supporting non-constant but virtual cases is
@@ -547,16 +547,16 @@
             return True
         #
         vstr.ensure_nonnull()
-        lengthbox = _int_sub(self.optimizer, vstop.force_box(),
-                                            vstart.force_box())
+        lengthbox = _int_sub(self, vstop.force_box(self),
+                                   vstart.force_box(self))
         #
         if isinstance(vstr, VStringSliceValue):
             # double slicing  s[i:j][k:l]
             vintermediate = vstr
             vstr = vintermediate.vstr
-            startbox = _int_add(self.optimizer,
-                                vintermediate.vstart.force_box(),
-                                vstart.force_box())
+            startbox = _int_add(self,
+                                vintermediate.vstart.force_box(self),
+                                vstart.force_box(self))
             vstart = self.getvalue(startbox)
         #
         value = self.make_vstring_slice(op.result, op, mode)
@@ -594,8 +594,8 @@
                 do = EffectInfo.OS_STREQ_LENGTHOK
             else:
                 do = EffectInfo.OS_STREQ_NONNULL
-            self.generate_modified_call(do, [v1.force_box(),
-                                             v2.force_box()], op.result, mode)
+            self.generate_modified_call(do, [v1.force_box(self),
+                                             v2.force_box(self)], op.result, mode)
             return True
         return False
 
@@ -603,7 +603,7 @@
         l2box = v2.getstrlen(None, mode)
         if isinstance(l2box, ConstInt):
             if l2box.value == 0:
-                lengthbox = v1.getstrlen(self.optimizer, mode)
+                lengthbox = v1.getstrlen(self, mode)
                 seo = self.optimizer.send_extra_operation
                 seo(ResOperation(rop.INT_EQ, [lengthbox, CONST_0], resultbox))
                 return True
@@ -614,17 +614,17 @@
                     vchar1 = self.strgetitem(v1, optimizer.CVAL_ZERO, mode)
                     vchar2 = self.strgetitem(v2, optimizer.CVAL_ZERO, mode)
                     seo = self.optimizer.send_extra_operation
-                    seo(ResOperation(rop.INT_EQ, [vchar1.force_box(),
-                                                  vchar2.force_box()],
+                    seo(ResOperation(rop.INT_EQ, [vchar1.force_box(self),
+                                                  vchar2.force_box(self)],
                                      resultbox))
                     return True
                 if isinstance(v1, VStringSliceValue):
                     vchar = self.strgetitem(v2, optimizer.CVAL_ZERO, mode)
                     do = EffectInfo.OS_STREQ_SLICE_CHAR
-                    self.generate_modified_call(do, [v1.vstr.force_box(),
-                                                     v1.vstart.force_box(),
-                                                     v1.vlength.force_box(),
-                                                     vchar.force_box()],
+                    self.generate_modified_call(do, [v1.vstr.force_box(self),
+                                                     v1.vstart.force_box(self),
+                                                     v1.vlength.force_box(self),
+                                                     vchar.force_box(self)],
                                                 resultbox, mode)
                     return True
         #
@@ -635,10 +635,10 @@
             if v1.is_null():
                 self.make_constant(resultbox, CONST_1)
                 return True
-            op = ResOperation(rop.PTR_EQ, [v1.force_box(),
+            op = ResOperation(rop.PTR_EQ, [v1.force_box(self),
                                            llhelper.CONST_NULL],
                               resultbox)
-            self.optimizer.emit_operation(op)
+            self.emit_operation(op)
             return True
         #
         return False
@@ -652,8 +652,8 @@
                     do = EffectInfo.OS_STREQ_NONNULL_CHAR
                 else:
                     do = EffectInfo.OS_STREQ_CHECKNULL_CHAR
-                self.generate_modified_call(do, [v1.force_box(),
-                                                 vchar.force_box()], resultbox,
+                self.generate_modified_call(do, [v1.force_box(self),
+                                                 vchar.force_box(self)], resultbox,
                                             mode)
                 return True
         #
@@ -662,10 +662,10 @@
                 do = EffectInfo.OS_STREQ_SLICE_NONNULL
             else:
                 do = EffectInfo.OS_STREQ_SLICE_CHECKNULL
-            self.generate_modified_call(do, [v1.vstr.force_box(),
-                                             v1.vstart.force_box(),
-                                             v1.vlength.force_box(),
-                                             v2.force_box()], resultbox, mode)
+            self.generate_modified_call(do, [v1.vstr.force_box(self),
+                                             v1.vstart.force_box(self),
+                                             v1.vlength.force_box(self),
+                                             v2.force_box(self)], resultbox, mode)
             return True
         return False
 
@@ -675,16 +675,11 @@
         calldescr, func = cic.callinfo_for_oopspec(oopspecindex)
         op = ResOperation(rop.CALL, [ConstInt(func)] + args, result,
                           descr=calldescr)
-        self.optimizer.emit_operation(op)
+        self.emit_operation(op)
 
     def propagate_forward(self, op):
-        if not self.enabled:
-            self.emit_operation(op)
-            return
-
         dispatch_opt(self, op)
 
-
 dispatch_opt = make_dispatcher_method(OptString, 'optimize_',
         default=OptString.emit_operation)
 
diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py
--- a/pypy/jit/metainterp/resoperation.py
+++ b/pypy/jit/metainterp/resoperation.py
@@ -221,7 +221,6 @@
         newop.setfailargs(self.getfailargs())
         return newop
 
-
 # ============
 # arity mixins
 # ============
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -2956,6 +2956,18 @@
         assert res == f(32)
         self.check_loops(arraylen_gc=2)
 
+    def test_ulonglong_mod(self):
+        myjitdriver = JitDriver(greens = [], reds = ['n', 'sa', 'i'])
+        def f(n):
+            sa = i = rffi.cast(rffi.ULONGLONG, 1)
+            while i < rffi.cast(rffi.ULONGLONG, n):
+                myjitdriver.jit_merge_point(sa=sa, n=n, i=i)
+                sa += sa % i
+                i += 1
+        res = self.meta_interp(f, [32])
+        assert res == f(32)
+
+
 class TestOOtype(BasicTests, OOJitMixin):
 
     def test_oohash(self):
@@ -3408,6 +3420,75 @@
         assert res == main(1, 10)
         self.check_loops(call=0)
 
+    def test_setarrayitem_followed_by_arraycopy(self):
+        myjitdriver = JitDriver(greens = [], reds = ['n', 'sa', 'x', 'y'])
+        def f(n):
+            sa = 0
+            x = [1,2,n]
+            y = [1,2,3]
+            while n > 0:
+                myjitdriver.jit_merge_point(sa=sa, n=n, x=x, y=y)
+                y[0] = n
+                x[0:3] = y
+                sa += x[0]
+                n -= 1
+            return sa
+        res = self.meta_interp(f, [16])
+        assert res == f(16)
+        
+
 
 class TestLLtype(BaseLLtypeTests, LLJitMixin):
-    pass
+    def test_tagged(self):
+        py.test.skip("implement me")
+        from pypy.rlib.objectmodel import UnboxedValue
+        class Base(object):
+            __slots__ = ()
+
+        class Int(UnboxedValue, Base):
+            __slots__ = ["a"]
+
+            def is_pos(self):
+                return self.a > 0
+
+            def dec(self):
+                return Int(self.a - 1)
+
+
+        class Float(Base):
+            def __init__(self, a):
+                self.a = a
+
+            def is_pos(self):
+                return self.a > 0
+
+            def dec(self):
+                return Float(self.a - 1)
+
+        driver = JitDriver(greens=['pc', 's'], reds=['o'])
+
+        def main(fl, n, s):
+            if s:
+                s = "--j"
+            else:
+                s = "---j"
+            if fl:
+                o = Float(float(n))
+            else:
+                o = Int(n)
+            pc = 0
+            while True:
+                driver.jit_merge_point(s=s, pc=pc, o=o)
+                c = s[pc]
+                if c == "j":
+                    driver.can_enter_jit(s=s, pc=pc, o=o)
+                    if o.is_pos():
+                        pc = 0
+                        continue
+                    else:
+                        break
+                elif c == "-":
+                    o = o.dec()
+                pc += 1
+            return pc
+        res = self.meta_interp(main, [False, 100, True], taggedpointers=True)
diff --git a/pypy/jit/metainterp/test/test_longlong.py b/pypy/jit/metainterp/test/test_longlong.py
--- a/pypy/jit/metainterp/test/test_longlong.py
+++ b/pypy/jit/metainterp/test/test_longlong.py
@@ -118,6 +118,26 @@
         res = self.interp_operations(f, [1000000000])
         assert res == 123500000000.0
 
+    def test_floats_negative(self):
+        def f(i):
+            # i == 1000000000
+            f = i * -123.5
+            n = r_longlong(f)
+            compare(n, -29, 1054051584)
+            return float(n)
+        res = self.interp_operations(f, [1000000000])
+        assert res == -123500000000.0
+
+    def test_floats_ulonglong(self):
+        def f(i):
+            # i == 1000000000
+            f = i * 12350000000.0
+            n = r_ulonglong(f)
+            compare(n, -1419508847, 538116096)
+            return float(n)
+        res = self.interp_operations(f, [1000000000])
+        assert res == 12350000000000000000.0
+
     def test_unsigned_compare_ops(self):
         def f(n1, n2):
             # n == 30002000000000
diff --git a/pypy/jit/metainterp/test/test_resume.py b/pypy/jit/metainterp/test/test_resume.py
--- a/pypy/jit/metainterp/test/test_resume.py
+++ b/pypy/jit/metainterp/test/test_resume.py
@@ -576,8 +576,9 @@
 
 
 class FakeOptimizer_VirtualValue(object):
-    class cpu:
-        pass
+    class optimizer:
+        class cpu:
+            pass
 fakeoptimizer = FakeOptimizer_VirtualValue()
 
 def ConstAddr(addr, cpu):   # compatibility
@@ -1135,12 +1136,7 @@
     modifier.liveboxes = {}
     modifier.vfieldboxes = {}
 
-    class FakeOptimizer(object):
-        class cpu:
-            pass
-        def new_const_item(self, descr):
-            return None
-    v2 = VArrayValue(FakeOptimizer(), LLtypeMixin.arraydescr, 2, b2s)
+    v2 = VArrayValue(LLtypeMixin.arraydescr, None, 2, b2s)
     v2._items = [b4s, c1s]
     modifier.register_virtual_fields(b2s, [b4s, c1s])
     liveboxes = []
diff --git a/pypy/jit/metainterp/test/test_string.py b/pypy/jit/metainterp/test/test_string.py
--- a/pypy/jit/metainterp/test/test_string.py
+++ b/pypy/jit/metainterp/test/test_string.py
@@ -484,6 +484,30 @@
             return len(sa.val)
         assert self.meta_interp(f, ['a']) == f('a')
 
+    def test_string_comepare_quasiimmutable(self):
+        class Sys(object):
+            _immutable_fields_ = ["defaultencoding?"]
+            def __init__(self, s):
+                self.defaultencoding = s
+        _str = self._str
+        sys = Sys(_str('ascii'))        
+        mydriver = JitDriver(reds = ['n', 'sa'], greens = [])
+        def f(n):
+            sa = 0
+            sys.defaultencoding = _str('ascii')
+            while n:
+                mydriver.jit_merge_point(n=n, sa=sa)
+                if sys.defaultencoding == _str('ascii'):
+                    sa += 1
+                n -= 1
+            sys.defaultencoding = _str('utf-8')
+            return sa
+        assert self.meta_interp(f, [8]) == f(8)
+        self.check_loops({'int_add': 1, 'guard_true': 1, 'int_sub': 1,
+                          'jump': 1, 'int_is_true': 1,
+                          'guard_not_invalidated': 1})
+
+
 #class TestOOtype(StringTests, OOJitMixin):
 #    CALL = "oosend"
 #    CALL_PURE = "oosend_pure"
diff --git a/pypy/jit/metainterp/test/test_virtualstate.py b/pypy/jit/metainterp/test/test_virtualstate.py
--- a/pypy/jit/metainterp/test/test_virtualstate.py
+++ b/pypy/jit/metainterp/test/test_virtualstate.py
@@ -431,7 +431,7 @@
         
 
 class BaseTestBridges(BaseTest):
-    enable_opts = "intbounds:rewrite:virtualize:string:heap:unroll"
+    enable_opts = "intbounds:rewrite:virtualize:string:pure:heap:unroll"
 
     def _do_optimize_bridge(self, bridge, call_pure_results):
         from pypy.jit.metainterp.optimizeopt import optimize_bridge_1, build_opt_chain
diff --git a/pypy/jit/metainterp/warmspot.py b/pypy/jit/metainterp/warmspot.py
--- a/pypy/jit/metainterp/warmspot.py
+++ b/pypy/jit/metainterp/warmspot.py
@@ -53,6 +53,8 @@
         extraconfigopts = {'translation.list_comprehension_operations': True}
     else:
         extraconfigopts = {}
+    if kwds.pop("taggedpointers", False):
+        extraconfigopts["translation.taggedpointers"] = True
     interp, graph = get_interpreter(function, args,
                                     backendopt=False,  # will be done below
                                     type_system=type_system,
diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -23,6 +23,8 @@
         ("arccos", "arccos"),
         ("arcsin", "arcsin"),
         ("arctan", "arctan"),
+        ("arcsinh", "arcsinh"),
+        ("arctanh", "arctanh"),
         ("copysign", "copysign"),
         ("cos", "cos"),
         ("divide", "divide"),
@@ -50,4 +52,6 @@
     appleveldefs = {
         'average': 'app_numpy.average',
         'mean': 'app_numpy.mean',
+        'inf': 'app_numpy.inf',
+        'e': 'app_numpy.e',
     }
diff --git a/pypy/module/micronumpy/app_numpy.py b/pypy/module/micronumpy/app_numpy.py
--- a/pypy/module/micronumpy/app_numpy.py
+++ b/pypy/module/micronumpy/app_numpy.py
@@ -1,5 +1,11 @@
+import math
+
 import numpy
 
+
+inf = float("inf")
+e = math.e
+
 def average(a):
     # This implements a weighted average, for now we don't implement the
     # weighting, just the average part!
@@ -8,4 +14,4 @@
 def mean(a):
     if not hasattr(a, "mean"):
         a = numpy.array(a)
-    return a.mean()
\ No newline at end of file
+    return a.mean()
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -7,13 +7,14 @@
 from pypy.interpreter.typedef import TypeDef, interp_attrproperty, GetSetProperty
 from pypy.module.micronumpy import signature
 from pypy.objspace.std.floatobject import float2string
-from pypy.rlib import rfloat
-from pypy.rlib.rarithmetic import widen
+from pypy.rlib import rarithmetic, rfloat
+from pypy.rlib.rarithmetic import LONG_BIT, widen
 from pypy.rlib.objectmodel import specialize, enforceargs
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.rpython.lltypesystem import lltype, rffi
 
 
+UNSIGNEDLTR = "u"
 SIGNEDLTR = "i"
 BOOLLTR = "b"
 FLOATINGLTR = "f"
@@ -61,7 +62,10 @@
             self.val = val
 
         def wrap(self, space):
-            return space.wrap(self.val)
+            val = self.val
+            if valtype is rarithmetic.r_singlefloat:
+                val = float(val)
+            return space.wrap(val)
 
         def convert_to(self, dtype):
             return dtype.adapt_val(self.val)
@@ -145,7 +149,7 @@
         return self.adapt_val(func(self, self.for_computation(self.unbox(v))))
     return impl
 
-class ArithmaticTypeMixin(object):
+class ArithmeticTypeMixin(object):
     _mixin_ = True
 
     @binop
@@ -200,11 +204,17 @@
         return v1 >= v2
 
 
-class FloatArithmeticDtype(ArithmaticTypeMixin):
+class FloatArithmeticDtype(ArithmeticTypeMixin):
     _mixin_ = True
 
+    def unwrap(self, space, w_item):
+        return self.adapt_val(space.float_w(space.float(w_item)))
+
     def for_computation(self, v):
-        return v
+        return float(v)
+
+    def str_format(self, item):
+        return float2string(self.for_computation(self.unbox(item)), 'g', rfloat.DTSF_STR_PRECISION)
 
     @binop
     def mod(self, v1, v2):
@@ -250,19 +260,29 @@
         return math.tan(v)
     @unaryop
     def arcsin(self, v):
-        if v < -1.0 or v > 1.0:
+        if not -1.0 <= v <= 1.0:
             return rfloat.NAN
         return math.asin(v)
     @unaryop
     def arccos(self, v):
-        if v < -1.0 or v > 1.0:
+        if not -1.0 <= v <= 1.0:
             return rfloat.NAN
         return math.acos(v)
     @unaryop
     def arctan(self, v):
         return math.atan(v)
+    @unaryop
+    def arcsinh(self, v):
+        return math.asinh(v)
+    @unaryop
+    def arctanh(self, v):
+        if v == 1.0 or v == -1.0:
+            return math.copysign(rfloat.INFINITY, v)
+        if not -1.0 < v < 1.0:
+            return rfloat.NAN
+        return math.atanh(v)
 
-class IntegerArithmeticDtype(ArithmaticTypeMixin):
+class IntegerArithmeticDtype(ArithmeticTypeMixin):
     _mixin_ = True
 
     def unwrap(self, space, w_item):
@@ -271,10 +291,16 @@
     def for_computation(self, v):
         return widen(v)
 
+    def str_format(self, item):
+        return str(widen(self.unbox(item)))
+
     @binop
     def mod(self, v1, v2):
         return v1 % v2
 
+class SignedIntegerArithmeticDtype(IntegerArithmeticDtype):
+    _mixin_ = True
+
     @unaryop
     def sign(self, v):
         if v > 0:
@@ -285,17 +311,22 @@
             assert v == 0
             return 0
 
-    def str_format(self, item):
-        return str(widen(self.unbox(item)))
+class UnsignedIntegerArithmeticDtype(IntegerArithmeticDtype):
+    _mixin_ = True
+
+    @unaryop
+    def sign(self, v):
+        return int(v != 0)
+
 
 W_BoolDtype = create_low_level_dtype(
     num = 0, kind = BOOLLTR, name = "bool",
-    aliases = ["?"],
+    aliases = ["?", "bool", "bool8"],
     applevel_types = ["bool"],
     T = lltype.Bool,
     valtype = bool,
 )
-class W_BoolDtype(IntegerArithmeticDtype, W_BoolDtype):
+class W_BoolDtype(SignedIntegerArithmeticDtype, W_BoolDtype):
     def unwrap(self, space, w_item):
         return self.adapt_val(space.is_true(w_item))
 
@@ -308,67 +339,139 @@
 
 W_Int8Dtype = create_low_level_dtype(
     num = 1, kind = SIGNEDLTR, name = "int8",
-    aliases = ["int8"],
+    aliases = ["b", "int8", "i1"],
     applevel_types = [],
     T = rffi.SIGNEDCHAR,
     valtype = rffi.SIGNEDCHAR._type,
     expected_size = 1,
 )
-class W_Int8Dtype(IntegerArithmeticDtype, W_Int8Dtype):
+class W_Int8Dtype(SignedIntegerArithmeticDtype, W_Int8Dtype):
+    pass
+
+W_UInt8Dtype = create_low_level_dtype(
+    num = 2, kind = UNSIGNEDLTR, name = "uint8",
+    aliases = ["B", "uint8", "I1"],
+    applevel_types = [],
+    T = rffi.UCHAR,
+    valtype = rffi.UCHAR._type,
+    expected_size = 1,
+)
+class W_UInt8Dtype(UnsignedIntegerArithmeticDtype, W_UInt8Dtype):
     pass
 
 W_Int16Dtype = create_low_level_dtype(
     num = 3, kind = SIGNEDLTR, name = "int16",
-    aliases = ["int16"],
+    aliases = ["h", "int16", "i2"],
     applevel_types = [],
     T = rffi.SHORT,
     valtype = rffi.SHORT._type,
     expected_size = 2,
 )
-class W_Int16Dtype(IntegerArithmeticDtype, W_Int16Dtype):
+class W_Int16Dtype(SignedIntegerArithmeticDtype, W_Int16Dtype):
+    pass
+
+W_UInt16Dtype = create_low_level_dtype(
+    num = 4, kind = UNSIGNEDLTR, name = "uint16",
+    aliases = ["H", "uint16", "I2"],
+    applevel_types = [],
+    T = rffi.USHORT,
+    valtype = rffi.USHORT._type,
+    expected_size = 2,
+)
+class W_UInt16Dtype(UnsignedIntegerArithmeticDtype, W_UInt16Dtype):
     pass
 
 W_Int32Dtype = create_low_level_dtype(
     num = 5, kind = SIGNEDLTR, name = "int32",
-    aliases = ["i"],
+    aliases = ["i", "int32", "i4"],
     applevel_types = [],
     T = rffi.INT,
     valtype = rffi.INT._type,
     expected_size = 4,
 )
-class W_Int32Dtype(IntegerArithmeticDtype, W_Int32Dtype):
+class W_Int32Dtype(SignedIntegerArithmeticDtype, W_Int32Dtype):
+    pass
+
+W_UInt32Dtype = create_low_level_dtype(
+    num = 6, kind = UNSIGNEDLTR, name = "uint32",
+    aliases = ["I", "uint32", "I4"],
+    applevel_types = [],
+    T = rffi.UINT,
+    valtype = rffi.UINT._type,
+    expected_size = 4,
+)
+class W_UInt32Dtype(UnsignedIntegerArithmeticDtype, W_UInt32Dtype):
     pass
 
 W_Int64Dtype = create_low_level_dtype(
     num = 9, kind = SIGNEDLTR, name = "int64",
-    aliases = [],
+    aliases = ["q", "int64", "i8"],
     applevel_types = ["long"],
     T = rffi.LONGLONG,
     valtype = rffi.LONGLONG._type,
     expected_size = 8,
 )
-class W_Int64Dtype(IntegerArithmeticDtype, W_Int64Dtype):
+class W_Int64Dtype(SignedIntegerArithmeticDtype, W_Int64Dtype):
+    pass
+
+W_UInt64Dtype = create_low_level_dtype(
+    num = 10, kind = UNSIGNEDLTR, name = "uint64",
+    aliases = ["Q", "uint64", "I8"],
+    applevel_types = [],
+    T = rffi.ULONGLONG,
+    valtype = rffi.ULONGLONG._type,
+    expected_size = 8,
+)
+class W_UInt64Dtype(UnsignedIntegerArithmeticDtype, W_UInt64Dtype):
+    pass
+
+if LONG_BIT == 32:
+    class W_LongDtype(W_Int32Dtype):
+        pass
+
+    class W_ULongDtype(W_UInt32Dtype):
+        pass
+else:
+    class W_LongDtype(W_Int64Dtype):
+        pass
+
+    class W_ULongDtype(W_UInt64Dtype):
+        pass
+
+W_LongDtype.num = 7
+W_LongDtype.aliases = ["l"]
+W_LongDtype.applevel_types = ["int"]
+W_ULongDtype.num = 8
+W_ULongDtype.aliases = ["L"]
+
+W_Float32Dtype = create_low_level_dtype(
+    num = 11, kind = FLOATINGLTR, name = "float32",
+    aliases = ["f", "float32", "f4"],
+    applevel_types = [],
+    T = lltype.SingleFloat,
+    valtype = rarithmetic.r_singlefloat,
+    expected_size = 4,
+)
+class W_Float32Dtype(FloatArithmeticDtype, W_Float32Dtype):
     pass
 
 W_Float64Dtype = create_low_level_dtype(
     num = 12, kind = FLOATINGLTR, name = "float64",
-    aliases = [],
+    aliases = ["d", "float64", "f8"],
     applevel_types = ["float"],
     T = lltype.Float,
     valtype = float,
     expected_size = 8,
 )
 class W_Float64Dtype(FloatArithmeticDtype, W_Float64Dtype):
-    def unwrap(self, space, w_item):
-        return self.adapt_val(space.float_w(space.float(w_item)))
-
-    def str_format(self, item):
-        return float2string(self.unbox(item), 'g', rfloat.DTSF_STR_PRECISION)
+    pass
 
 ALL_DTYPES = [
     W_BoolDtype,
-    W_Int8Dtype, W_Int16Dtype, W_Int32Dtype, W_Int64Dtype,
-    W_Float64Dtype
+    W_Int8Dtype, W_UInt8Dtype, W_Int16Dtype, W_UInt16Dtype,
+    W_Int32Dtype, W_UInt32Dtype, W_LongDtype, W_ULongDtype,
+    W_Int64Dtype, W_UInt64Dtype,
+    W_Float32Dtype, W_Float64Dtype,
 ]
 
 dtypes_by_alias = unrolling_iterable([
@@ -395,6 +498,7 @@
 
     num = interp_attrproperty("num", cls=W_Dtype),
     kind = interp_attrproperty("kind", cls=W_Dtype),
+    itemsize = interp_attrproperty("num_bytes", cls=W_Dtype),
     shape = GetSetProperty(W_Dtype.descr_get_shape),
 )
 W_Dtype.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -4,6 +4,7 @@
 from pypy.interpreter.typedef import TypeDef, GetSetProperty, interp_attrproperty
 from pypy.module.micronumpy import interp_dtype, signature
 from pypy.rlib import jit
+from pypy.rlib.rarithmetic import LONG_BIT
 from pypy.tool.sourcetools import func_with_new_name
 
 
@@ -180,23 +181,56 @@
 
     # Everything promotes to float, and bool promotes to everything.
     if dt2.kind == interp_dtype.FLOATINGLTR or dt1.kind == interp_dtype.BOOLLTR:
+        # Float32 + 8-bit int = Float64
+        if dt2.num == 11 and dt1.num_bytes >= 4:
+            return space.fromcache(interp_dtype.W_Float64Dtype)
         return dt2
 
-    assert False
+    # for now this means mixing signed and unsigned
+    if dt2.kind == interp_dtype.SIGNEDLTR:
+        # if dt2 has a greater number of bytes, then just go with it
+        if dt1.num_bytes < dt2.num_bytes:
+            return dt2
+        # we need to promote both dtypes
+        dtypenum = dt2.num + 2
+    else:
+        # increase to the next signed type (or to float)
+        dtypenum = dt2.num + 1
+        # UInt64 + signed = Float64
+        if dt2.num == 10:
+            dtypenum += 1
+    newdtype = interp_dtype.ALL_DTYPES[dtypenum]
+
+    if newdtype.num_bytes > dt2.num_bytes or newdtype.kind == interp_dtype.FLOATINGLTR:
+        return space.fromcache(newdtype)
+    else:
+        # we only promoted to long on 32-bit or to longlong on 64-bit
+        # this is really for dealing with the Long and Ulong dtypes
+        if LONG_BIT == 32:
+            dtypenum += 2
+        else:
+            dtypenum += 3
+        return space.fromcache(interp_dtype.ALL_DTYPES[dtypenum])
 
 def find_unaryop_result_dtype(space, dt, promote_to_float=False,
     promote_bools=False, promote_to_largest=False):
     if promote_bools and (dt.kind == interp_dtype.BOOLLTR):
         return space.fromcache(interp_dtype.W_Int8Dtype)
     if promote_to_float:
+        if dt.kind == interp_dtype.FLOATINGLTR:
+            return dt
+        if dt.num >= 5:
+            return space.fromcache(interp_dtype.W_Float64Dtype)
         for bytes, dtype in interp_dtype.dtypes_by_num_bytes:
-            if dtype.kind == interp_dtype.FLOATINGLTR and dtype.num_bytes >= dt.num_bytes:
+            if dtype.kind == interp_dtype.FLOATINGLTR and dtype.num_bytes > dt.num_bytes:
                 return space.fromcache(dtype)
     if promote_to_largest:
         if dt.kind == interp_dtype.BOOLLTR or dt.kind == interp_dtype.SIGNEDLTR:
             return space.fromcache(interp_dtype.W_Int64Dtype)
         elif dt.kind == interp_dtype.FLOATINGLTR:
             return space.fromcache(interp_dtype.W_Float64Dtype)
+        elif dt.kind == interp_dtype.UNSIGNEDLTR:
+            return space.fromcache(interp_dtype.W_UInt64Dtype)
         else:
             assert False
     return dt
@@ -205,15 +239,23 @@
     w_type = space.type(w_obj)
 
     bool_dtype = space.fromcache(interp_dtype.W_BoolDtype)
+    long_dtype = space.fromcache(interp_dtype.W_LongDtype)
     int64_dtype = space.fromcache(interp_dtype.W_Int64Dtype)
 
     if space.is_w(w_type, space.w_bool):
-        if current_guess is None:
+        if current_guess is None or current_guess is bool_dtype:
             return bool_dtype
+        return current_guess
     elif space.is_w(w_type, space.w_int):
         if (current_guess is None or current_guess is bool_dtype or
-            current_guess is int64_dtype):
+            current_guess is long_dtype):
+            return long_dtype
+        return current_guess
+    elif space.is_w(w_type, space.w_long):
+        if (current_guess is None or current_guess is bool_dtype or
+            current_guess is long_dtype or current_guess is int64_dtype):
             return int64_dtype
+        return current_guess
     return space.fromcache(interp_dtype.W_Float64Dtype)
 
 
@@ -225,7 +267,9 @@
         def impl(res_dtype, lvalue, rvalue):
             res = getattr(res_dtype, op_name)(lvalue, rvalue)
             if comparison_func:
-                res = space.fromcache(interp_dtype.W_BoolDtype).box(res)
+                booldtype = space.fromcache(interp_dtype.W_BoolDtype)
+                assert isinstance(booldtype, interp_dtype.W_BoolDtype)
+                res = booldtype.box(res)
             return res
     return func_with_new_name(impl, ufunc_name)
 
@@ -268,6 +312,8 @@
             ("arcsin", "arcsin", 1, {"promote_to_float": True}),
             ("arccos", "arccos", 1, {"promote_to_float": True}),
             ("arctan", "arctan", 1, {"promote_to_float": True}),
+            ("arcsinh", "arcsinh", 1, {"promote_to_float": True}),
+            ("arctanh", "arctanh", 1, {"promote_to_float": True}),
         ]:
             self.add_ufunc(space, *ufunc_def)
 
@@ -277,7 +323,7 @@
 
         identity = extra_kwargs.get("identity")
         if identity is not None:
-            identity = space.fromcache(interp_dtype.W_Int64Dtype).adapt_val(identity)
+            identity = space.fromcache(interp_dtype.W_LongDtype).adapt_val(identity)
         extra_kwargs["identity"] = identity
 
         func = ufunc_dtype_caller(space, ufunc_name, op_name, argcount,
@@ -290,4 +336,4 @@
         setattr(self, ufunc_name, ufunc)
 
 def get(space):
-    return space.fromcache(UfuncState)
\ No newline at end of file
+    return space.fromcache(UfuncState)
diff --git a/pypy/module/micronumpy/test/test_base.py b/pypy/module/micronumpy/test/test_base.py
--- a/pypy/module/micronumpy/test/test_base.py
+++ b/pypy/module/micronumpy/test/test_base.py
@@ -64,18 +64,46 @@
     def test_unaryops(self, space):
         bool_dtype = space.fromcache(interp_dtype.W_BoolDtype)
         int8_dtype = space.fromcache(interp_dtype.W_Int8Dtype)
+        uint8_dtype = space.fromcache(interp_dtype.W_UInt8Dtype)
+        int16_dtype = space.fromcache(interp_dtype.W_Int16Dtype)
+        uint16_dtype = space.fromcache(interp_dtype.W_UInt16Dtype)
         int32_dtype = space.fromcache(interp_dtype.W_Int32Dtype)
+        uint32_dtype = space.fromcache(interp_dtype.W_UInt32Dtype)
+        long_dtype = space.fromcache(interp_dtype.W_LongDtype)
+        ulong_dtype = space.fromcache(interp_dtype.W_ULongDtype)
+        int64_dtype = space.fromcache(interp_dtype.W_Int64Dtype)
+        uint64_dtype = space.fromcache(interp_dtype.W_UInt64Dtype)
+        float32_dtype = space.fromcache(interp_dtype.W_Float32Dtype)
         float64_dtype = space.fromcache(interp_dtype.W_Float64Dtype)
 
-        # Normal rules, everythign returns itself
+        # Normal rules, everything returns itself
         assert find_unaryop_result_dtype(space, bool_dtype) is bool_dtype
         assert find_unaryop_result_dtype(space, int8_dtype) is int8_dtype
+        assert find_unaryop_result_dtype(space, uint8_dtype) is uint8_dtype
+        assert find_unaryop_result_dtype(space, int16_dtype) is int16_dtype
+        assert find_unaryop_result_dtype(space, uint16_dtype) is uint16_dtype
         assert find_unaryop_result_dtype(space, int32_dtype) is int32_dtype
+        assert find_unaryop_result_dtype(space, uint32_dtype) is uint32_dtype
+        assert find_unaryop_result_dtype(space, long_dtype) is long_dtype
+        assert find_unaryop_result_dtype(space, ulong_dtype) is ulong_dtype
+        assert find_unaryop_result_dtype(space, int64_dtype) is int64_dtype
+        assert find_unaryop_result_dtype(space, uint64_dtype) is uint64_dtype
+        assert find_unaryop_result_dtype(space, float32_dtype) is float32_dtype
         assert find_unaryop_result_dtype(space, float64_dtype) is float64_dtype
 
         # Coerce to floats, some of these will eventually be float16, or
         # whatever our smallest float type is.
-        assert find_unaryop_result_dtype(space, bool_dtype, promote_to_float=True) is float64_dtype
-        assert find_unaryop_result_dtype(space, int8_dtype, promote_to_float=True) is float64_dtype
+        assert find_unaryop_result_dtype(space, bool_dtype, promote_to_float=True) is float32_dtype # will be float16 if we ever put that in
+        assert find_unaryop_result_dtype(space, int8_dtype, promote_to_float=True) is float32_dtype # will be float16 if we ever put that in
+        assert find_unaryop_result_dtype(space, uint8_dtype, promote_to_float=True) is float32_dtype # will be float16 if we ever put that in
+        assert find_unaryop_result_dtype(space, int16_dtype, promote_to_float=True) is float32_dtype
+        assert find_unaryop_result_dtype(space, uint16_dtype, promote_to_float=True) is float32_dtype
         assert find_unaryop_result_dtype(space, int32_dtype, promote_to_float=True) is float64_dtype
-        assert find_unaryop_result_dtype(space, float64_dtype, promote_to_float=True) is float64_dtype
\ No newline at end of file
+        assert find_unaryop_result_dtype(space, uint32_dtype, promote_to_float=True) is float64_dtype
+        assert find_unaryop_result_dtype(space, int64_dtype, promote_to_float=True) is float64_dtype
+        assert find_unaryop_result_dtype(space, uint64_dtype, promote_to_float=True) is float64_dtype
+        assert find_unaryop_result_dtype(space, float32_dtype, promote_to_float=True) is float32_dtype
+        assert find_unaryop_result_dtype(space, float64_dtype, promote_to_float=True) is float64_dtype
+
+        # promote bools, happens with sign ufunc
+        assert find_unaryop_result_dtype(space, bool_dtype, promote_bools=True) is int8_dtype
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -17,6 +17,7 @@
         from numpy import dtype
 
         assert dtype(bool).num == 0
+        assert dtype(int).num == 7
         assert dtype(long).num == 9
         assert dtype(float).num == 12
 
@@ -81,6 +82,48 @@
             assert isinstance(a[i], (int, long))
             assert a[1] == 1
 
+    def test_overflow(self):
+        from numpy import array, dtype
+        assert array([128], 'b')[0] == -128
+        assert array([256], 'B')[0] == 0
+        assert array([32768], 'h')[0] == -32768
+        assert array([65536], 'H')[0] == 0
+        if dtype('l').itemsize == 4: # 32-bit
+            raises(OverflowError, "array([2**32/2], 'i')")
+            raises(OverflowError, "array([2**32], 'I')")
+        raises(OverflowError, "array([2**64/2], 'q')")
+        raises(OverflowError, "array([2**64], 'Q')")
+
+    def test_bool_binop_types(self):
+        from numpy import array, dtype
+        types = ('?','b','B','h','H','i','I','l','L','q','Q','f','d')
+        N = len(types)
+        a = array([True], '?')
+        for t in types:
+            assert (a + array([0], t)).dtype is dtype(t)
+
+    def test_binop_types(self):
+        from numpy import array, dtype
+        tests = [('b','B','h'), ('b','h','h'), ('b','H','i'), ('b','i','i'),
+                 ('b','l','l'), ('b','q','q'), ('b','Q','d'), ('B','h','h'),
+                 ('B','H','H'), ('B','i','i'), ('B','I','I'), ('B','l','l'),
+                 ('B','L','L'), ('B','q','q'), ('B','Q','Q'), ('h','H','i'),
+                 ('h','i','i'), ('h','l','l'), ('h','q','q'), ('h','Q','d'),
+                 ('H','i','i'), ('H','I','I'), ('H','l','l'), ('H','L','L'),
+                 ('H','q','q'), ('H','Q','Q'), ('i','l','l'), ('i','q','q'),
+                 ('i','Q','d'), ('I','L','L'), ('I','q','q'), ('I','Q','Q'),
+                 ('q','Q','d'), ('b','f','f'), ('B','f','f'), ('h','f','f'),
+                 ('H','f','f'), ('i','f','d'), ('I','f','d'), ('l','f','d'),
+                 ('L','f','d'), ('q','f','d'), ('Q','f','d'), ('q','d','d')]
+        if dtype('i').itemsize == dtype('l').itemsize: # 32-bit
+            tests.extend([('b','I','q'), ('b','L','q'), ('h','I','q'),
+                          ('h','L','q'), ('i','I','q'), ('i','L','q')])
+        else:
+            tests.extend([('b','I','l'), ('b','L','d'), ('h','I','l'),
+                          ('h','L','d'), ('i','I','l'), ('i','L','d')])
+        for d1, d2, dout in tests:
+            assert (array([1], d1) + array([1], d2)).dtype is dtype(dout)
+
     def test_add_int8(self):
         from numpy import array, dtype
 
@@ -99,6 +142,15 @@
         for i in range(5):
             assert b[i] == i * 2
 
+    def test_add_uint32(self):
+        from numpy import array, dtype
+
+        a = array(range(5), dtype="I")
+        b = a + a
+        assert b.dtype is dtype("I")
+        for i in range(5):
+            assert b[i] == i * 2
+
     def test_shape(self):
         from numpy import dtype
 
diff --git a/pypy/module/micronumpy/test/test_module.py b/pypy/module/micronumpy/test/test_module.py
--- a/pypy/module/micronumpy/test/test_module.py
+++ b/pypy/module/micronumpy/test/test_module.py
@@ -10,4 +10,12 @@
     def test_average(self):
         from numpy import array, average
         assert average(range(10)) == 4.5
-        assert average(array(range(10))) == 4.5
\ No newline at end of file
+        assert average(array(range(10))) == 4.5
+
+    def test_constants(self):
+        import math
+        from numpy import inf, e
+        assert type(inf) is float
+        assert inf == float("inf")
+        assert e == math.e
+        assert type(e) is float
\ No newline at end of file
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -551,8 +551,10 @@
         from numpy import array, dtype
 
         assert array([True]).dtype is dtype(bool)
-        assert array([True, 1]).dtype is dtype(long)
-        assert array([1, 2, 3]).dtype is dtype(long)
+        assert array([True, False]).dtype is dtype(bool)
+        assert array([True, 1]).dtype is dtype(int)
+        assert array([1, 2, 3]).dtype is dtype(int)
+        assert array([1L, 2, 3]).dtype is dtype(long)
         assert array([1.2, True]).dtype is dtype(float)
         assert array([1.2, 5]).dtype is dtype(float)
         assert array([]).dtype is dtype(float)
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -234,7 +234,7 @@
             assert b[i] == math.sin(a[i])
 
         a = sin(array([True, False], dtype=bool))
-        assert a[0] == sin(1)
+        assert abs(a[0] - sin(1)) < 1e-7 # a[0] will be less precise
         assert a[1] == 0.0
 
     def test_cos(self):
@@ -298,6 +298,25 @@
         b = arctan(a)
         assert math.isnan(b[0])
 
+    def test_arcsinh(self):
+        import math
+        from numpy import arcsinh, inf
+
+        for v in [inf, -inf, 1.0, math.e]:
+            assert math.asinh(v) == arcsinh(v)
+        assert math.isnan(arcsinh(float("nan")))
+
+    def test_arctanh(self):
+        import math
+        from numpy import arctanh
+
+        for v in [.99, .5, 0, -.5, -.99]:
+            assert math.atanh(v) == arctanh(v)
+        for v in [2.0, -2.0]:
+            assert math.isnan(arctanh(v))
+        for v in [1.0, -1.0]:
+            assert arctanh(v) == math.copysign(float("inf"), v)
+
     def test_reduce_errors(self):
         from numpy import sin, add
 
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -1,20 +1,24 @@
 from pypy.jit.metainterp.test.support import LLJitMixin
 from pypy.module.micronumpy import interp_ufuncs, signature
 from pypy.module.micronumpy.compile import (numpy_compile, FakeSpace,
-    FloatObject)
-from pypy.module.micronumpy.interp_dtype import W_Float64Dtype, W_Int64Dtype
+    FloatObject, IntObject)
+from pypy.module.micronumpy.interp_dtype import W_Int32Dtype, W_Float64Dtype, W_Int64Dtype, W_UInt64Dtype
 from pypy.module.micronumpy.interp_numarray import (BaseArray, SingleDimArray,
     SingleDimSlice, scalar_w)
 from pypy.rlib.nonconst import NonConstant
 from pypy.rpython.annlowlevel import llstr
 from pypy.rpython.test.test_llinterp import interpret
 
+import py
+
 
 class TestNumpyJIt(LLJitMixin):
     def setup_class(cls):
         cls.space = FakeSpace()
         cls.float64_dtype = cls.space.fromcache(W_Float64Dtype)
         cls.int64_dtype = cls.space.fromcache(W_Int64Dtype)
+        cls.uint64_dtype = cls.space.fromcache(W_UInt64Dtype)
+        cls.int32_dtype = cls.space.fromcache(W_Int32Dtype)
 
     def test_add(self):
         def f(i):
@@ -303,6 +307,31 @@
                           'int_lt': 1, 'guard_true': 1, 'jump': 1})
         assert result == 11.0
 
+    def test_int32_sum(self):
+        py.test.skip("pypy/jit/backend/llimpl.py needs to be changed to "
+                     "deal correctly with int dtypes for this test to "
+                     "work. skip for now until someone feels up to the task")
+        space = self.space
+        float64_dtype = self.float64_dtype
+        int32_dtype = self.int32_dtype
+
+        def f(n):
+            if NonConstant(False):
+                dtype = float64_dtype
+            else:
+                dtype = int32_dtype
+            ar = SingleDimArray(n, dtype=dtype)
+            i = 0
+            while i < n:
+                ar.get_concrete().setitem(i, int32_dtype.box(7))
+                i += 1
+            v = ar.descr_add(space, ar).descr_sum(space)
+            assert isinstance(v, IntObject)
+            return v.intval
+
+        result = self.meta_interp(f, [5], listops=True, backendopt=True)
+        assert result == f(5)
+
 class TestTranslation(object):
     def test_compile(self):
         x = numpy_compile('aa+f*f/a-', 10)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_00_model.py b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
--- a/pypy/module/pypyjit/test_pypy_c/test_00_model.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
@@ -50,8 +50,7 @@
         cmdline.append(str(self.filepath))
         #
         print cmdline, logfile
-        env={'PYPYLOG': 'jit-log-opt,jit-summary:' + str(logfile)}
-        #env={'PYPYLOG': ':' + str(logfile)}
+        env={'PYPYLOG': 'jit-log-opt,jit-log-noopt,jit-summary:' + str(logfile)}
         pipe = subprocess.Popen(cmdline,
                                 env=env,
                                 stdout=subprocess.PIPE,
diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py
--- a/pypy/module/pypyjit/test_pypy_c/test_call.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_call.py
@@ -366,12 +366,12 @@
             # make sure that the "block" is not allocated
             ...
             i20 = force_token()
-            setfield_gc(p0, i20, descr=<SignedFieldDescr .*PyFrame.vable_token .*>)
             p22 = new_with_vtable(19511408)
             p24 = new_array(1, descr=<GcPtrArrayDescr>)
             p26 = new_with_vtable(ConstClass(W_ListObject))
             p27 = new(descr=<SizeDescr .*>)
             p29 = new_array(0, descr=<GcPtrArrayDescr>)
+            setfield_gc(p0, i20, descr=<SignedFieldDescr .*PyFrame.vable_token .*>)
             setfield_gc(p27, p29, descr=<GcPtrFieldDescr list.items .*>)
             setfield_gc(p26, p27, descr=<.* .*W_ListObject.inst_wrappeditems .*>)
             setarrayitem_gc(p24, 0, p26, descr=<GcPtrArrayDescr>)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_generators.py b/pypy/module/pypyjit/test_pypy_c/test_generators.py
--- a/pypy/module/pypyjit/test_pypy_c/test_generators.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_generators.py
@@ -19,8 +19,8 @@
         assert loop.match_by_id("generator", """
             i16 = force_token()
             p45 = new_with_vtable(ConstClass(W_IntObject))
-            setfield_gc(p45, i29, descr=<SignedFieldDescr .*>)
             i47 = arraylen_gc(p8, descr=<GcPtrArrayDescr>) # Should be removed by backend
             setarrayitem_gc(p8, 0, p45, descr=<GcPtrArrayDescr>)
+            setfield_gc(p45, i29, descr=<SignedFieldDescr .*>)
             jump(..., descr=...)
             """)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_instance.py b/pypy/module/pypyjit/test_pypy_c/test_instance.py
--- a/pypy/module/pypyjit/test_pypy_c/test_instance.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_instance.py
@@ -125,8 +125,8 @@
             i12 = force_token()
             --TICK--
             p20 = new_with_vtable(ConstClass(W_IntObject))
+            setfield_gc(ConstPtr(ptr21), p20, descr=<GcPtrFieldDescr .*TypeCell.inst_w_value .*>)
             setfield_gc(p20, i11, descr=<SignedFieldDescr.*W_IntObject.inst_intval .*>)
-            setfield_gc(ConstPtr(ptr21), p20, descr=<GcPtrFieldDescr .*TypeCell.inst_w_value .*>)
             jump(p0, p1, p2, p3, p4, p20, p6, i7, p20, descr=<Loop.>)
         """)
 
diff --git a/pypy/module/thread/gil.py b/pypy/module/thread/gil.py
--- a/pypy/module/thread/gil.py
+++ b/pypy/module/thread/gil.py
@@ -16,7 +16,7 @@
 
 class GILThreadLocals(OSThreadLocals):
     """A version of OSThreadLocals that enforces a GIL."""
-    ll_GIL = thread.null_ll_lock
+    gil_ready = False
 
     def initialize(self, space):
         # add the GIL-releasing callback as an action on the space
@@ -25,12 +25,10 @@
 
     def setup_threads(self, space):
         """Enable threads in the object space, if they haven't already been."""
-        if not self.ll_GIL:
-            try:
-                self.ll_GIL = thread.allocate_ll_lock()
-            except thread.error:
+        if not self.gil_ready:
+            if not thread.gil_allocate():
                 raise wrap_thread_error(space, "can't allocate GIL")
-            thread.acquire_NOAUTO(self.ll_GIL, True)
+            self.gil_ready = True
             self.enter_thread(space)   # setup the main thread
             result = True
         else:
@@ -44,19 +42,16 @@
         # test_lock_again after the global state was cleared by
         # test_compile_lock.  As a workaround, we repatch these global
         # fields systematically.
-        spacestate.ll_GIL = self.ll_GIL
         invoke_around_extcall(before_external_call, after_external_call)
         return result
 
     def reinit_threads(self, space):
-        if self.ll_GIL:
-            self.ll_GIL = thread.allocate_ll_lock()
-            thread.acquire_NOAUTO(self.ll_GIL, True)
-            self.enter_thread(space)
+        if self.gil_ready:
+            self.gil_ready = False
+            self.setup_threads(space)
 
     def yield_thread(self):
-        thread.yield_thread()  # explicitly release the gil (used by test_gil)
-
+        do_yield_thread()
 
 class GILReleaseAction(PeriodicAsyncAction):
     """An action called every sys.checkinterval bytecodes.  It releases
@@ -64,16 +59,12 @@
     """
 
     def perform(self, executioncontext, frame):
-        # Other threads can run between the release() and the acquire()
-        # implicit in the following external function call (which has
-        # otherwise no effect).
-        thread.yield_thread()
+        do_yield_thread()
 
 
 class SpaceState:
 
     def _freeze_(self):
-        self.ll_GIL = thread.null_ll_lock
         self.action_after_thread_switch = None
         # ^^^ set by AsyncAction.fire_after_thread_switch()
         return False
@@ -95,14 +86,14 @@
     # this function must not raise, in such a way that the exception
     # transformer knows that it cannot raise!
     e = get_errno()
-    thread.release_NOAUTO(spacestate.ll_GIL)
+    thread.gil_release()
     set_errno(e)
 before_external_call._gctransformer_hint_cannot_collect_ = True
 before_external_call._dont_reach_me_in_del_ = True
 
 def after_external_call():
     e = get_errno()
-    thread.acquire_NOAUTO(spacestate.ll_GIL, True)
+    thread.gil_acquire()
     thread.gc_thread_run()
     spacestate.after_thread_switch()
     set_errno(e)
@@ -115,3 +106,18 @@
 # pointers in the shadow stack.  This is necessary because the GIL is
 # not held after the call to before_external_call() or before the call
 # to after_external_call().
+
+def do_yield_thread():
+    # explicitly release the gil, in a way that tries to give more
+    # priority to other threads (as opposed to continuing to run in
+    # the same thread).
+    if thread.gil_yield_thread():
+        thread.gc_thread_run()
+        spacestate.after_thread_switch()
+do_yield_thread._gctransformer_hint_close_stack_ = True
+do_yield_thread._dont_reach_me_in_del_ = True
+do_yield_thread._dont_inline_ = True
+
+# do_yield_thread() needs a different hint: _gctransformer_hint_close_stack_.
+# The *_external_call() functions are themselves called only from the rffi
+# module from a helper function that also has this hint.
diff --git a/pypy/module/thread/ll_thread.py b/pypy/module/thread/ll_thread.py
--- a/pypy/module/thread/ll_thread.py
+++ b/pypy/module/thread/ll_thread.py
@@ -17,7 +17,8 @@
     include_dirs = [str(py.path.local(autopath.pypydir).join('translator', 'c'))],
     export_symbols = ['RPyThreadGetIdent', 'RPyThreadLockInit',
                       'RPyThreadAcquireLock', 'RPyThreadReleaseLock',
-                      'RPyThreadYield',
+                      'RPyGilAllocate', 'RPyGilYieldThread',
+                      'RPyGilRelease', 'RPyGilAcquire',
                       'RPyThreadGetStackSize', 'RPyThreadSetStackSize',
                       'RPyOpaqueDealloc_ThreadLock',
                       'RPyThreadAfterFork']
@@ -69,8 +70,16 @@
                                          [TLOCKP], lltype.Void,
                                          _nowrapper=True)
 
-# this function does nothing apart from releasing the GIL temporarily.
-yield_thread = llexternal('RPyThreadYield', [], lltype.Void, threadsafe=True)
+# these functions manipulate directly the GIL, whose definition does not
+# escape the C code itself
+gil_allocate     = llexternal('RPyGilAllocate', [], lltype.Signed,
+                              _nowrapper=True)
+gil_yield_thread = llexternal('RPyGilYieldThread', [], lltype.Signed,
+                              _nowrapper=True)
+gil_release      = llexternal('RPyGilRelease', [], lltype.Void,
+                              _nowrapper=True)
+gil_acquire      = llexternal('RPyGilAcquire', [], lltype.Void,
+                              _nowrapper=True)
 
 def allocate_lock():
     return Lock(allocate_ll_lock())
diff --git a/pypy/module/thread/test/test_gil.py b/pypy/module/thread/test/test_gil.py
--- a/pypy/module/thread/test/test_gil.py
+++ b/pypy/module/thread/test/test_gil.py
@@ -30,19 +30,34 @@
     use_threads = True
     bigtest = False
 
-    def test_one_thread(self):
+    def test_one_thread(self, skew=+1):
+        from pypy.rlib.debug import debug_print
         if self.bigtest:
-            N = 1000000
+            N = 100000
+            skew *= 25000
         else:
             N = 100
+            skew *= 25
         space = FakeSpace()
         class State:
             pass
         state = State()
-        def runme():
-            for i in range(N):
+        def runme(main=False):
+            j = 0
+            for i in range(N + [-skew, skew][main]):
+                state.datalen1 += 1   # try to crash if the GIL is not
+                state.datalen2 += 1   # correctly acquired
                 state.data.append((thread.get_ident(), i))
+                state.datalen3 += 1
+                state.datalen4 += 1
+                assert state.datalen1 == len(state.data)
+                assert state.datalen2 == len(state.data)
+                assert state.datalen3 == len(state.data)
+                assert state.datalen4 == len(state.data)
+                debug_print(main, i, state.datalen4)
                 state.threadlocals.yield_thread()
+                assert i == j
+                j += 1
         def bootstrap():
             try:
                 runme()
@@ -50,20 +65,26 @@
                 thread.gc_thread_die()
         def f():
             state.data = []
+            state.datalen1 = 0
+            state.datalen2 = 0
+            state.datalen3 = 0
+            state.datalen4 = 0
             state.threadlocals = gil.GILThreadLocals()
             state.threadlocals.setup_threads(space)
             thread.gc_thread_prepare()
             subident = thread.start_new_thread(bootstrap, ())
             mainident = thread.get_ident()
-            runme()
+            runme(True)
             still_waiting = 3000
             while len(state.data) < 2*N:
+                debug_print(len(state.data))
                 if not still_waiting:
                     raise ValueError("time out")
                 still_waiting -= 1
                 if not we_are_translated(): gil.before_external_call()
                 time.sleep(0.01)
                 if not we_are_translated(): gil.after_external_call()
+            debug_print("leaving!")
             i1 = i2 = 0
             for tid, i in state.data:
                 if tid == mainident:
@@ -72,14 +93,17 @@
                     assert i == i2; i2 += 1
                 else:
                     assert 0
-            assert i1 == N
-            assert i2 == N
+            assert i1 == N + skew
+            assert i2 == N - skew
             return len(state.data)
 
         fn = self.getcompiled(f, [])
         res = fn()
         assert res == 2*N
 
+    def test_one_thread_rev(self):
+        self.test_one_thread(skew=-1)
+
 
 class TestRunDirectly(GILTests):
     def getcompiled(self, f, argtypes):
diff --git a/pypy/module/thread/test/test_thread.py b/pypy/module/thread/test/test_thread.py
--- a/pypy/module/thread/test/test_thread.py
+++ b/pypy/module/thread/test/test_thread.py
@@ -225,7 +225,8 @@
 
         def busy_wait():
             for x in range(1000):
-                time.sleep(0.01)
+                print 'tick...', x  # <-force the GIL to be released, as
+                time.sleep(0.01)    #   time.sleep doesn't do non-translated
 
         # This is normally called by app_main.py
         signal.signal(signal.SIGINT, signal.default_int_handler)
diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -258,15 +258,15 @@
             msg = "'%s' has no length" % (name,)
             raise OperationError(space.w_TypeError, space.wrap(msg))
         w_res = space.get_and_call_function(w_descr, w_obj)
-        space._check_len_result(w_res)
-        return w_res
+        return space.wrap(space._check_len_result(w_res))
 
     def _check_len_result(space, w_obj):
         # Will complain if result is too big.
-        result = space.int_w(w_obj)
+        result = space.int_w(space.int(w_obj))
         if result < 0:
             raise OperationError(space.w_ValueError,
                                  space.wrap("__len__() should return >= 0"))
+        return result
 
     def iter(space, w_obj):
         w_descr = space.lookup(w_obj, '__iter__')
diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py
--- a/pypy/objspace/std/intobject.py
+++ b/pypy/objspace/std/intobject.py
@@ -1,12 +1,13 @@
 from pypy.interpreter.error import OperationError
 from pypy.objspace.std import newformat
+from pypy.objspace.std.inttype import wrapint
 from pypy.objspace.std.model import registerimplementation, W_Object
-from pypy.objspace.std.register_all import register_all
 from pypy.objspace.std.multimethod import FailedToImplementArgs
 from pypy.objspace.std.noneobject import W_NoneObject
+from pypy.objspace.std.register_all import register_all
+from pypy.rlib import jit
 from pypy.rlib.rarithmetic import ovfcheck, ovfcheck_lshift, LONG_BIT, r_uint
 from pypy.rlib.rbigint import rbigint
-from pypy.objspace.std.inttype import wrapint
 
 """
 In order to have the same behavior running
@@ -20,7 +21,7 @@
     _immutable_fields_ = ['intval']
 
     from pypy.objspace.std.inttype import int_typedef as typedef
-    
+
     def __init__(w_self, intval):
         w_self.intval = intval
 
@@ -135,7 +136,7 @@
     x = float(w_int1.intval)
     y = float(w_int2.intval)
     if y == 0.0:
-        raise FailedToImplementArgs(space.w_ZeroDivisionError, space.wrap("float division"))    
+        raise FailedToImplementArgs(space.w_ZeroDivisionError, space.wrap("float division"))
     return space.wrap(x / y)
 
 def mod__Int_Int(space, w_int1, w_int2):
@@ -169,7 +170,8 @@
 
 
 # helper for pow()
-def _impl_int_int_pow(space, iv, iw, iz=0):
+ at jit.look_inside_iff(lambda space, iv, iw, iz: jit.isconstant(iw) and jit.isconstant(iz))
+def _impl_int_int_pow(space, iv, iw, iz):
     if iw < 0:
         if iz != 0:
             raise OperationError(space.w_TypeError,
@@ -197,7 +199,7 @@
     except OverflowError:
         raise FailedToImplementArgs(space.w_OverflowError,
                                 space.wrap("integer exponentiation"))
-    return wrapint(space, ix)
+    return ix
 
 def pow__Int_Int_Int(space, w_int1, w_int2, w_int3):
     x = w_int1.intval
@@ -206,12 +208,12 @@
     if z == 0:
         raise OperationError(space.w_ValueError,
                              space.wrap("pow() 3rd argument cannot be 0"))
-    return _impl_int_int_pow(space, x, y, z)
+    return space.wrap(_impl_int_int_pow(space, x, y, z))
 
 def pow__Int_Int_None(space, w_int1, w_int2, w_int3):
     x = w_int1.intval
     y = w_int2.intval
-    return _impl_int_int_pow(space, x, y)
+    return space.wrap(_impl_int_int_pow(space, x, y, 0))
 
 def neg__Int(space, w_int1):
     a = w_int1.intval
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -1202,7 +1202,11 @@
     if length == 0:
         raise OperationError(space.w_IndexError,
                              space.wrap("pop from empty list"))
-    idx = space.int_w(w_idx)
+    if space.isinstance_w(w_idx, space.w_float):
+        raise OperationError(space.w_TypeError,
+            space.wrap("integer argument expected, got float")
+        )
+    idx = space.int_w(space.int(w_idx))
     if idx < 0:
         idx += length
     try:
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -84,7 +84,7 @@
             transparent.setup(self)
 
         for type, classes in self.model.typeorder.iteritems():
-            if len(classes) == 3:
+            if len(classes) >= 3:
                 # W_Root, AnyXxx and actual object
                 self.gettypefor(type).interplevel_cls = classes[0][0]
 
diff --git a/pypy/objspace/std/ropeunicodeobject.py b/pypy/objspace/std/ropeunicodeobject.py
--- a/pypy/objspace/std/ropeunicodeobject.py
+++ b/pypy/objspace/std/ropeunicodeobject.py
@@ -185,7 +185,7 @@
 
 def eq__RopeUnicode_Rope(space, w_runi, w_rope):
     from pypy.objspace.std.unicodeobject import _unicode_string_comparison
-    return _unicode_string_comparison(space, w_runi, w_rope, 
+    return _unicode_string_comparison(space, w_runi, w_rope,
                     False,  unicode_from_string)
 
 def ne__RopeUnicode_RopeUnicode(space, w_str1, w_str2):
@@ -193,7 +193,7 @@
 
 def ne__RopeUnicode_Rope(space, w_runi, w_rope):
     from pypy.objspace.std.unicodeobject import _unicode_string_comparison
-    return _unicode_string_comparison(space, w_runi, w_rope, 
+    return _unicode_string_comparison(space, w_runi, w_rope,
                     True, unicode_from_string)
 
 def gt__RopeUnicode_RopeUnicode(space, w_str1, w_str2):
@@ -247,7 +247,7 @@
     if (len(l_w) == 1 and
         space.is_w(space.type(l_w[0]), space.w_unicode)):
         return l_w[0]
-    
+
     values_list = []
     for i in range(len(l_w)):
         w_item = l_w[i]
@@ -320,7 +320,7 @@
 
 
 def make_generic(funcname):
-    def func(space, w_self): 
+    def func(space, w_self):
         node = w_self._node
         if node.length() == 0:
             return space.w_False
@@ -578,7 +578,7 @@
         return w_self.create_if_subclassed()
     resultnode = rope.concatenate(rope.multiply(fillchar, padding), self)
     return W_RopeUnicodeObject(resultnode)
-    
+
 def unicode_zfill__RopeUnicode_ANY(space, w_self, w_width):
     self = w_self._node
     length = self.length()
@@ -744,7 +744,7 @@
     except OverflowError:
         raise OperationError(space.w_OverflowError,
                              space.wrap("string too long"))
-    
+
 
 def unicode_encode__RopeUnicode_ANY_ANY(space, w_unistr,
                                         w_encoding=None,
@@ -821,7 +821,7 @@
         try:
             w_newval = space.getitem(w_table, space.wrap(char))
         except OperationError, e:
-            if e.match(space, space.w_KeyError):
+            if e.match(space, space.w_LookupError):
                 result.append(crope)
             else:
                 raise
@@ -848,7 +848,7 @@
     hexdigits = "0123456789abcdef"
     node = w_unicode._node
     size = node.length()
-    
+
     singlequote = doublequote = False
     iter = rope.ItemIterator(node)
     for i in range(size):
@@ -900,7 +900,7 @@
                                   ])
                     j += 2
                     continue
-                
+
         if code >= 0x100:
             result.extend(['\\', "u",
                            hexdigits[(code >> 12) & 0xf],
@@ -932,7 +932,7 @@
             continue
         if code < ord(' ') or code >= 0x7f:
             result.extend(['\\', "x",
-                           hexdigits[(code >> 4) & 0xf], 
+                           hexdigits[(code >> 4) & 0xf],
                            hexdigits[(code >> 0) & 0xf],
                           ])
             j += 1
@@ -964,15 +964,15 @@
 
 def next__RopeUnicodeIter(space, w_ropeiter):
     if w_ropeiter.node is None:
-        raise OperationError(space.w_StopIteration, space.w_None) 
+        raise OperationError(space.w_StopIteration, space.w_None)
     try:
         unichar = w_ropeiter.item_iter.nextunichar()
         w_item = space.wrap(unichar)
     except StopIteration:
         w_ropeiter.node = None
         w_ropeiter.char_iter = None
-        raise OperationError(space.w_StopIteration, space.w_None) 
-    w_ropeiter.index += 1 
+        raise OperationError(space.w_StopIteration, space.w_None)
+    w_ropeiter.index += 1
     return w_item
 
 # XXX __length_hint__()
diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py
--- a/pypy/objspace/std/test/test_listobject.py
+++ b/pypy/objspace/std/test/test_listobject.py
@@ -791,6 +791,20 @@
         l.pop()
         assert l == range(9)
 
+    def test_pop_custom_int(self):
+        class A(object):
+            def __init__(self, x):
+                self.x = x
+
+            def __int__(self):
+                return self.x
+
+        l = range(10)
+        x = l.pop(A(-1))
+        assert x == 9
+        assert l == range(9)
+        raises(TypeError, range(10).pop, 1.0)
+
     def test_pop_negative(self):
         l1 = [1,2,3,4]
         l2 = ["1", "2", "3", "4"]
@@ -904,7 +918,7 @@
         assert l == [3, 4]
 
     def test_unicode(self):
-        s = u"&#65533;&#65533;&#65533;"
+        s = u"&#65533;&#65533;&#65533;"
         assert s.encode("ascii", "replace") == "???"
         assert s.encode("ascii", "ignore") == ""
 
diff --git a/pypy/objspace/std/test/test_stdobjspace.py b/pypy/objspace/std/test/test_stdobjspace.py
--- a/pypy/objspace/std/test/test_stdobjspace.py
+++ b/pypy/objspace/std/test/test_stdobjspace.py
@@ -46,3 +46,17 @@
         assert space.sliceindices(w_slice, w(3)) == (1,2,1)
         assert space.sliceindices(w_obj, w(3)) == (1,2,3)
 
+    def test_fastpath_isinstance(self):
+        from pypy.objspace.std.stringobject import W_StringObject
+        from pypy.objspace.std.intobject import W_IntObject
+        
+        space = self.space
+        assert space.w_str.interplevel_cls is W_StringObject
+        assert space.w_int.interplevel_cls is W_IntObject
+        class X(W_StringObject):
+            def __init__(self):
+                pass
+            
+            typedef = None
+
+        assert space.isinstance_w(X(), space.w_str)
diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -126,6 +126,25 @@
         raises(TypeError, type, 'test', 42, {})
         raises(TypeError, type, 'test', (object,), 42)
 
+    def test_call_type_subclass(self):
+        class A(type):
+            pass
+
+        assert A("hello") is str
+
+        # Make sure type(x) doesn't call x.__class__.__init__
+        class T(type):
+            counter = 0
+            def __init__(self, *args):
+                T.counter += 1
+        class C:
+            __metaclass__ = T
+        assert T.counter == 1
+        a = C()
+        assert T.counter == 1
+        assert type(a) is C
+        assert T.counter == 1
+
     def test_bases(self):
         assert int.__bases__ == (object,)
         class X:
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -77,7 +77,7 @@
         for i in range(len(self.lookup_where)):
             self.lookup_where[i] = None_None
 
-# possible values of compares_by_identity_status 
+# possible values of compares_by_identity_status
 UNKNOWN = 0
 COMPARES_BY_IDENTITY = 1
 OVERRIDES_EQ_CMP_OR_HASH = 2
@@ -358,7 +358,7 @@
                 if w_value is not None:
                     return w_value
         return None
-                
+
     @unroll_safe
     def _lookup(w_self, key):
         space = w_self.space
@@ -822,14 +822,6 @@
 
 def call__Type(space, w_type, __args__):
     promote(w_type)
-    # special case for type(x)
-    if space.is_w(w_type, space.w_type):
-        try:
-            w_obj, = __args__.fixedunpack(1)
-        except ValueError:
-            pass
-        else:
-            return space.type(w_obj)
     # invoke the __new__ of the type
     if not we_are_jitted():
         # note that the annotator will figure out that w_type.w_bltin_new can
@@ -856,7 +848,8 @@
         call_init = space.isinstance_w(w_newobject, w_type)
 
     # maybe invoke the __init__ of the type
-    if call_init:
+    if (call_init and not (space.is_w(w_type, space.w_type) and
+        not __args__.keywords and len(__args__.arguments_w) == 1)):
         w_descr = space.lookup(w_newobject, '__init__')
         w_result = space.get_and_call_args(w_descr, w_newobject, __args__)
         if not space.is_w(w_result, space.w_None):
diff --git a/pypy/objspace/std/typetype.py b/pypy/objspace/std/typetype.py
--- a/pypy/objspace/std/typetype.py
+++ b/pypy/objspace/std/typetype.py
@@ -1,17 +1,28 @@
-from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.interpreter import gateway
 from pypy.interpreter.argument import Arguments
+from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.interpreter.typedef import (GetSetProperty, descr_get_dict,
                                       weakref_descr)
 from pypy.objspace.std.stdtypedef import StdTypeDef
 
-def descr__new__(space, w_typetype, w_name, w_bases, w_dict):
+
+def descr__new__(space, w_typetype, w_name, w_bases=gateway.NoneNotWrapped,
+    w_dict=gateway.NoneNotWrapped):
+
     "This is used to create user-defined classes only."
     from pypy.objspace.std.typeobject import W_TypeObject
     # XXX check types
 
     w_typetype = _precheck_for_new(space, w_typetype)
 
+    # special case for type(x)
+    if (space.is_w(space.type(w_typetype), space.w_type) and w_bases is None and
+        w_dict is None):
+        return space.type(w_name)
+    elif w_bases is None or w_dict is None:
+        raise OperationError(space.w_TypeError, space.wrap("type() takes 1 or 3 arguments"))
+
+
     bases_w = space.fixedview(w_bases)
 
     w_winner = w_typetype
diff --git a/pypy/objspace/test/test_descroperation.py b/pypy/objspace/test/test_descroperation.py
--- a/pypy/objspace/test/test_descroperation.py
+++ b/pypy/objspace/test/test_descroperation.py
@@ -667,5 +667,19 @@
                 return -1L
         raises(ValueError, len, Y())
 
+    def test_len_custom__int__(self):
+        class X(object):
+            def __init__(self, x):
+                self.x = x
+            def __len__(self):
+                return self.x
+            def __int__(self):
+                return self.x
+
+        l = len(X(3.0))
+        assert l == 3 and type(l) is int
+        l = len(X(X(2)))
+        assert l == 2 and type(l) is int
+
 class AppTestWithBuiltinShortcut(AppTest_Descroperation):
     OPTIONS = {'objspace.std.builtinshortcut': True}
diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -158,7 +158,7 @@
     return decorator
 
 @oopspec("jit.isconstant(value)")
- at specialize.argtype(0)
+ at specialize.ll()
 def isconstant(value):
     """
     While tracing, returns whether or not the value is currently known to be
@@ -167,10 +167,7 @@
 
     This is for advanced usage only.
     """
-    # I hate the annotator so much.
-    if NonConstant(False):
-        return True
-    return False
+    return NonConstant(False)
 
 @oopspec("jit.isvirtual(value)")
 @specialize.ll()
@@ -181,9 +178,7 @@
 
     This is for advanced usage only.
     """
-    if NonConstant(False):
-        return True
-    return False
+    return NonConstant(False)
 
 class Entry(ExtRegistryEntry):
     _about_ = hint
diff --git a/pypy/rlib/rgc.py b/pypy/rlib/rgc.py
--- a/pypy/rlib/rgc.py
+++ b/pypy/rlib/rgc.py
@@ -3,6 +3,7 @@
 
 from pypy.rlib import jit
 from pypy.rlib.objectmodel import we_are_translated, enforceargs, specialize
+from pypy.rlib.nonconst import NonConstant
 from pypy.rpython.extregistry import ExtRegistryEntry
 from pypy.rpython.lltypesystem import lltype, llmemory
 
@@ -143,6 +144,10 @@
     from pypy.rpython.lltypesystem.lloperation import llop
     from pypy.rlib.objectmodel import keepalive_until_here
 
+    # XXX: Hack to ensure that we get a proper effectinfo.write_descrs_arrays
+    if NonConstant(False):
+        dest[dest_start] = source[source_start]
+
     # supports non-overlapping copies only
     if not we_are_translated():
         if source == dest:
diff --git a/pypy/rlib/test/test_jit.py b/pypy/rlib/test/test_jit.py
--- a/pypy/rlib/test/test_jit.py
+++ b/pypy/rlib/test/test_jit.py
@@ -1,7 +1,7 @@
 import py
 from pypy.conftest import option
 from pypy.rlib.jit import hint, we_are_jitted, JitDriver, elidable_promote
-from pypy.rlib.jit import JitHintError, oopspec
+from pypy.rlib.jit import JitHintError, oopspec, isconstant
 from pypy.translator.translator import TranslationContext, graphof
 from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin
 from pypy.rpython.lltypesystem import lltype
@@ -137,6 +137,16 @@
             t.view()
         # assert did not raise
 
+    def test_isconstant(self):
+        def f(n):
+            assert n >= 0
+            assert isconstant(n) is False
+            l = []
+            l.append(n)
+            return len(l)
+        res = self.interpret(f, [234])
+        assert res == 1
+
 
 class TestJITLLtype(BaseTestJIT, LLRtypeMixin):
     pass
diff --git a/pypy/rpython/lltypesystem/opimpl.py b/pypy/rpython/lltypesystem/opimpl.py
--- a/pypy/rpython/lltypesystem/opimpl.py
+++ b/pypy/rpython/lltypesystem/opimpl.py
@@ -357,7 +357,7 @@
 
 def op_cast_float_to_uint(f):
     assert type(f) is float
-    return r_uint(int(f))
+    return r_uint(long(f))
 
 def op_cast_float_to_longlong(f):
     assert type(f) is float
@@ -369,7 +369,7 @@
 
 def op_cast_float_to_ulonglong(f):
     assert type(f) is float
-    return r_ulonglong(r_longlong(f))
+    return r_ulonglong(long(f))
 
 def op_cast_char_to_int(b):
     assert type(b) is str and len(b) == 1
diff --git a/pypy/rpython/lltypesystem/rlist.py b/pypy/rpython/lltypesystem/rlist.py
--- a/pypy/rpython/lltypesystem/rlist.py
+++ b/pypy/rpython/lltypesystem/rlist.py
@@ -1,15 +1,15 @@
+from pypy.rlib import rgc, jit
+from pypy.rlib.debug import ll_assert
+from pypy.rlib.objectmodel import enforceargs
+from pypy.rpython.lltypesystem import rstr
+from pypy.rpython.lltypesystem.lltype import (GcForwardReference, Ptr, GcArray,
+     GcStruct, Void, Signed, malloc, typeOf, nullptr, typeMethod)
+from pypy.rpython.rlist import (AbstractBaseListRepr, AbstractListRepr,
+    AbstractFixedSizeListRepr, AbstractListIteratorRepr, ll_setitem_nonneg,
+    ADTIList, ADTIFixedList, dum_nocheck)
+from pypy.rpython.rmodel import Repr, inputconst, externalvsinternal
 from pypy.tool.pairtype import pairtype, pair
-from pypy.rpython.rmodel import Repr, inputconst
-from pypy.rpython.rmodel import externalvsinternal
-from pypy.rpython.rlist import AbstractBaseListRepr, AbstractListRepr, \
-        AbstractFixedSizeListRepr, AbstractListIteratorRepr, \
-        ll_setitem_nonneg, ADTIList, ADTIFixedList
-from pypy.rpython.rlist import dum_nocheck
-from pypy.rpython.lltypesystem.lltype import GcForwardReference, Ptr, GcArray,\
-     GcStruct, Void, Signed, malloc, typeOf, nullptr, typeMethod
-from pypy.rpython.lltypesystem import rstr
-from pypy.rlib.debug import ll_assert
-from pypy.rlib import rgc, jit
+
 
 # ____________________________________________________________
 #
@@ -171,6 +171,7 @@
 
 # adapted C code
 
+ at enforceargs(None, int)
 def _ll_list_resize_really(l, newsize):
     """
     Ensure l.items has room for at least newsize elements, and set
@@ -210,7 +211,6 @@
         rgc.ll_arraycopy(items, newitems, 0, 0, p)
     l.length = newsize
     l.items = newitems
-_ll_list_resize_really._annenforceargs_ = (None, int)
 
 # this common case was factored out of _ll_list_resize
 # to see if inlining it gives some speed-up.
diff --git a/pypy/rpython/lltypesystem/test/test_lloperation.py b/pypy/rpython/lltypesystem/test/test_lloperation.py
--- a/pypy/rpython/lltypesystem/test/test_lloperation.py
+++ b/pypy/rpython/lltypesystem/test/test_lloperation.py
@@ -5,6 +5,7 @@
 from pypy.rpython.llinterp import LLFrame
 from pypy.rpython.test.test_llinterp import interpret
 from pypy.rpython import rclass
+from pypy.rlib.rarithmetic import LONGLONG_MASK, r_longlong, r_ulonglong
 
 LL_INTERP_OPERATIONS = [name[3:] for name in LLFrame.__dict__.keys()
                                  if name.startswith('op_')]
@@ -133,6 +134,14 @@
         py.test.raises(TypeError, llop.getinteriorfield,
                        lltype.Signed, s3, 'y')
 
+def test_cast_float_to_ulonglong():
+    f = 12350000000000000000.0
+    py.test.raises(OverflowError, r_longlong, f)
+    r_longlong(f / 2)   # does not raise OverflowError
+    #
+    x = llop.cast_float_to_ulonglong(lltype.UnsignedLongLong, f)
+    assert x == r_ulonglong(f)
+
 # ___________________________________________________________________________
 # This tests that the LLInterpreter and the LL_OPERATIONS tables are in sync.
 
diff --git a/pypy/translator/c/gcc/trackgcroot.py b/pypy/translator/c/gcc/trackgcroot.py
--- a/pypy/translator/c/gcc/trackgcroot.py
+++ b/pypy/translator/c/gcc/trackgcroot.py
@@ -486,6 +486,8 @@
         'paddq', 'pinsr',
         # zero-extending moves should not produce GC pointers
         'movz', 
+        # locked operations should not move GC pointers, at least so far
+        'lock',
         ])
 
     # a partial list is hopefully good enough for now; it's all to support
diff --git a/pypy/translator/c/genc.py b/pypy/translator/c/genc.py
--- a/pypy/translator/c/genc.py
+++ b/pypy/translator/c/genc.py
@@ -563,7 +563,10 @@
             else:
                 mk.definition('PYPY_MAIN_FUNCTION', "main")
 
-            if sys.platform == 'win32':
+            if (py.path.local.sysfind('python') or
+                py.path.local.sysfind('python.exe')):
+                python = 'python '
+            elif sys.platform == 'win32':
                 python = sys.executable.replace('\\', '/') + ' '
             else:
                 python = sys.executable + ' '
diff --git a/pypy/translator/c/src/thread.h b/pypy/translator/c/src/thread.h
--- a/pypy/translator/c/src/thread.h
+++ b/pypy/translator/c/src/thread.h
@@ -37,14 +37,9 @@
 
 #endif
 
-/* common helper: this does nothing, but is called with the GIL released.
-   This gives other threads a chance to grab the GIL and run. */
-void RPyThreadYield(void);
-
-#ifndef PYPY_NOT_MAIN_FILE
-void RPyThreadYield(void)
-{
-}
-#endif
+long RPyGilAllocate(void);
+long RPyGilYieldThread(void);
+void RPyGilRelease(void);
+void RPyGilAcquire(void);
 
 #endif
diff --git a/pypy/translator/c/src/thread_nt.h b/pypy/translator/c/src/thread_nt.h
--- a/pypy/translator/c/src/thread_nt.h
+++ b/pypy/translator/c/src/thread_nt.h
@@ -221,4 +221,57 @@
 #define RPyThreadTLS_Set(key, value)	TlsSetValue(key, value)
 
 
+/************************************************************/
+/* GIL code                                                 */
+/************************************************************/
+
+static volatile LONG pending_acquires = -1;
+static CRITICAL_SECTION mutex_gil;
+static HANDLE cond_gil;
+
+long RPyGilAllocate(void)
+{
+    pending_acquires = 0;
+    InitializeCriticalSection(&mutex_gil);
+    EnterCriticalSection(&mutex_gil);
+    cond_gil = CreateEvent (NULL, FALSE, FALSE, NULL);
+    return 1;
+}
+
+long RPyGilYieldThread(void)
+{
+    /* can be called even before RPyGilAllocate(), but in this case,
+       pending_acquires will be -1 */
+    if (pending_acquires <= 0)
+        return 0;
+    InterlockedIncrement(&pending_acquires);
+    PulseEvent(&cond_gil);
+
+    /* hack: the three following lines do a pthread_cond_wait(), and
+       normally specifying a timeout of INFINITE would be fine.  But the
+       first and second operations are not done atomically, so there is a
+       (small) risk that PulseEvent misses the WaitForSingleObject().
+       In this case the process will just sleep a few milliseconds. */
+    LeaveCriticalSection(&mutex_gil);
+    WaitForSingleObject(&cond_gil, 15);
+    EnterCriticalSection(&mutex_gil);
+
+    InterlockedDecrement(&pending_acquires);
+    return 1;
+}
+
+void RPyGilRelease(void)
+{
+    LeaveCriticalSection(&mutex_gil);
+    PulseEvent(&cond_gil);
+}
+
+void RPyGilAcquire(void)
+{
+    InterlockedIncrement(&pending_acquires);
+    EnterCriticalSection(&mutex_gil);
+    InterlockedDecrement(&pending_acquires);
+}
+
+
 #endif /* PYPY_NOT_MAIN_FILE */
diff --git a/pypy/translator/c/src/thread_pthread.h b/pypy/translator/c/src/thread_pthread.h
--- a/pypy/translator/c/src/thread_pthread.h
+++ b/pypy/translator/c/src/thread_pthread.h
@@ -12,6 +12,7 @@
 #include <signal.h>
 #include <stdio.h>
 #include <errno.h>
+#include <assert.h>
 
 /* The following is hopefully equivalent to what CPython does
    (which is trying to compile a snippet of code using it) */
@@ -459,4 +460,113 @@
 #define RPyThreadTLS_Set(key, value)	pthread_setspecific(key, value)
 
 
+/************************************************************/
+/* GIL code                                                 */
+/************************************************************/
+
+#ifdef __llvm__
+#  define HAS_ATOMIC_ADD
+#endif
+
+#ifdef __GNUC__
+#  if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 1)
+#    define HAS_ATOMIC_ADD
+#  endif
+#endif
+
+#ifdef HAS_ATOMIC_ADD
+#  define atomic_add __sync_fetch_and_add
+#else
+#  if defined(__amd64__)
+#    define atomic_add(ptr, value)  asm volatile ("lock addq %0, %1"        \
+                                 : : "ri"(value), "m"(*(ptr)) : "memory")
+#  elif defined(__i386__)
+#    define atomic_add(ptr, value)  asm volatile ("lock addl %0, %1"        \
+                                 : : "ri"(value), "m"(*(ptr)) : "memory")
+#  else
+#    error "Please use gcc >= 4.1 or write a custom 'asm' for your CPU."
+#  endif
+#endif
+
+#define ASSERT_STATUS(call)                             \
+    if (call != 0) {                                    \
+        fprintf(stderr, "Fatal error: " #call "\n");    \
+        abort();                                        \
+    }
+
+static void _debug_print(const char *msg)
+{
+#if 0
+    int col = (int)pthread_self();
+    col = 31 + ((col / 8) % 8);
+    fprintf(stderr, "\033[%dm%s\033[0m", col, msg);
+#endif
+}
+
+static volatile long pending_acquires = -1;
+static pthread_mutex_t mutex_gil = PTHREAD_MUTEX_INITIALIZER;
+static pthread_cond_t cond_gil = PTHREAD_COND_INITIALIZER;
+
+static void assert_has_the_gil(void)
+{
+#ifdef RPY_ASSERT
+    assert(pthread_mutex_trylock(&mutex_gil) != 0);
+    assert(pending_acquires >= 0);
+#endif
+}
+
+long RPyGilAllocate(void)
+{
+    _debug_print("RPyGilAllocate\n");
+    pending_acquires = 0;
+    pthread_mutex_trylock(&mutex_gil);
+    assert_has_the_gil();
+    return 1;
+}
+
+long RPyGilYieldThread(void)
+{
+    /* can be called even before RPyGilAllocate(), but in this case,
+       pending_acquires will be -1 */
+#ifdef RPY_ASSERT
+    if (pending_acquires >= 0)
+        assert_has_the_gil();
+#endif
+    if (pending_acquires <= 0)
+        return 0;
+    atomic_add(&pending_acquires, 1L);
+    _debug_print("{");
+    ASSERT_STATUS(pthread_cond_signal(&cond_gil));
+    ASSERT_STATUS(pthread_cond_wait(&cond_gil, &mutex_gil));
+    _debug_print("}");
+    atomic_add(&pending_acquires, -1L);
+    assert_has_the_gil();
+    return 1;
+}
+
+void RPyGilRelease(void)
+{
+    _debug_print("RPyGilRelease\n");
+#ifdef RPY_ASSERT
+    assert(pending_acquires >= 0);
+#endif
+    assert_has_the_gil();
+    ASSERT_STATUS(pthread_mutex_unlock(&mutex_gil));
+    ASSERT_STATUS(pthread_cond_signal(&cond_gil));
+}
+
+void RPyGilAcquire(void)
+{
+    _debug_print("about to RPyGilAcquire...\n");
+#ifdef RPY_ASSERT
+    assert(pending_acquires >= 0);
+#endif
+    atomic_add(&pending_acquires, 1L);
+    ASSERT_STATUS(pthread_mutex_lock(&mutex_gil));
+    atomic_add(&pending_acquires, -1L);
+    assert_has_the_gil();
+    _debug_print("RPyGilAcquire\n");
+}
+
+
 #endif /* PYPY_NOT_MAIN_FILE */

From noreply at buildbot.pypy.org  Tue Oct  4 15:03:07 2011
From: noreply at buildbot.pypy.org (cfbolz)
Date: Tue,  4 Oct 2011 15:03:07 +0200 (CEST)
Subject: [pypy-commit] pypy list-strategies: slightly annoying:
 cast_opaque_ptr killed the tracing heap cache. I cannot just
Message-ID: <20111004130307.EE851820D9@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: list-strategies
Changeset: r47822:2a6af948db14
Date: 2011-10-04 14:57 +0200
http://bitbucket.org/pypy/pypy/changeset/2a6af948db14/

Log:	slightly annoying: cast_opaque_ptr killed the tracing heap cache. I
	cannot just remove it, seems the short preamble stuff seems to need
	it. Thus I need to consider equivalences of boxes in the heapcache.

diff --git a/pypy/jit/metainterp/heapcache.py b/pypy/jit/metainterp/heapcache.py
--- a/pypy/jit/metainterp/heapcache.py
+++ b/pypy/jit/metainterp/heapcache.py
@@ -29,6 +29,23 @@
         # cache the length of arrays
         self.length_cache = {}
 
+        # equivalences between boxes
+        self.equivalent = {}
+
+    def get_repr(self, box):
+        res = self.equivalent.get(box, box)
+        if res is not box:
+            res2 = self.get_repr(res)
+            # path compression
+            if res2 is not res:
+                self.equivalent[box] = res2
+                res = res2
+        return res
+
+    def same_boxes(self, box1, box2):
+        assert box1 not in self.equivalent
+        self.equivalent[box1] = self.get_repr(box2)
+
     def invalidate_caches(self, opnum, descr, argboxes):
         self.mark_escaped(opnum, argboxes)
         self.clear_caches(opnum, descr, argboxes)
@@ -98,18 +115,23 @@
         self.heap_array_cache.clear()
 
     def is_class_known(self, box):
+        box = self.get_repr(box)
         return box in self.known_class_boxes
 
     def class_now_known(self, box):
+        box = self.get_repr(box)
         self.known_class_boxes[box] = None
 
     def is_nonstandard_virtualizable(self, box):
+        box = self.get_repr(box)
         return box in self.nonstandard_virtualizables
 
     def nonstandard_virtualizables_now_known(self, box):
+        box = self.get_repr(box)
         self.nonstandard_virtualizables[box] = None
 
     def is_unescaped(self, box):
+        box = self.get_repr(box)
         return self.new_boxes.get(box, False)
 
     def new(self, box):
@@ -120,6 +142,7 @@
         self.arraylen_now_known(box, lengthbox)
 
     def getfield(self, box, descr):
+        box = self.get_repr(box)
         d = self.heap_cache.get(descr, None)
         if d:
             tobox = d.get(box, None)
@@ -128,9 +151,11 @@
         return None
 
     def getfield_now_known(self, box, descr, fieldbox):
+        box = self.get_repr(box)
         self.heap_cache.setdefault(descr, {})[box] = fieldbox
 
     def setfield(self, box, descr, fieldbox):
+        box = self.get_repr(box)
         d = self.heap_cache.get(descr, None)
         new_d = self._do_write_with_aliasing(d, box, fieldbox)
         self.heap_cache[descr] = new_d
@@ -154,6 +179,7 @@
         return new_d
 
     def getarrayitem(self, box, descr, indexbox):
+        box = self.get_repr(box)
         if not isinstance(indexbox, ConstInt):
             return
         index = indexbox.getint()
@@ -164,6 +190,7 @@
                 return indexcache.get(box, None)
 
     def getarrayitem_now_known(self, box, descr, indexbox, valuebox):
+        box = self.get_repr(box)
         if not isinstance(indexbox, ConstInt):
             return
         index = indexbox.getint()
@@ -175,6 +202,7 @@
             cache[index] = {box: valuebox}
 
     def setarrayitem(self, box, descr, indexbox, valuebox):
+        box = self.get_repr(box)
         if not isinstance(indexbox, ConstInt):
             cache = self.heap_array_cache.get(descr, None)
             if cache is not None:
@@ -186,9 +214,11 @@
         cache[index] = self._do_write_with_aliasing(indexcache, box, valuebox)
 
     def arraylen(self, box):
+        box = self.get_repr(box)
         return self.length_cache.get(box, None)
 
     def arraylen_now_known(self, box, lengthbox):
+        box = self.get_repr(box)
         self.length_cache[box] = lengthbox
 
     def _replace_box(self, d, oldbox, newbox):
@@ -202,6 +232,7 @@
         return new_d
 
     def replace_box(self, oldbox, newbox):
+        oldbox = self.get_repr(oldbox)
         for descr, d in self.heap_cache.iteritems():
             self.heap_cache[descr] = self._replace_box(d, oldbox, newbox)
         for descr, d in self.heap_array_cache.iteritems():
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -236,7 +236,10 @@
 
     @arguments("box")
     def opimpl_cast_opaque_ptr(self, box):
-        return self.execute(rop.CAST_OPAQUE_PTR, box)
+        res = self.execute(rop.CAST_OPAQUE_PTR, box)
+        self.metainterp.heapcache.same_boxes(res, box)
+        return res
+
 
     @arguments("box")
     def _opimpl_any_return(self, box):
diff --git a/pypy/jit/metainterp/test/test_heapcache.py b/pypy/jit/metainterp/test/test_heapcache.py
--- a/pypy/jit/metainterp/test/test_heapcache.py
+++ b/pypy/jit/metainterp/test/test_heapcache.py
@@ -362,4 +362,16 @@
         h.invalidate_caches(rop.SETARRAYITEM_GC, None, [box1, index1, box2])
         assert h.is_unescaped(box1)
         h.invalidate_caches(rop.SETARRAYITEM_GC, None, [box2, index1, box1])
-        assert not h.is_unescaped(box1)
\ No newline at end of file
+        assert not h.is_unescaped(box1)
+
+    def test_heapcache_fields(self):
+        h = HeapCache()
+        assert h.getfield(box1, descr1) is None
+        assert h.getfield(box1, descr2) is None
+        h.setfield(box1, descr1, box2)
+        assert h.getfield(box1, descr1) is box2
+        assert h.getfield(box1, descr2) is None
+        h.same_boxes(box3, box1)
+        h.same_boxes(box4, box3)
+        assert h.getfield(box4, descr1) is box2
+        assert h.getfield(box4, descr2) is None
diff --git a/pypy/jit/metainterp/test/test_tracingopts.py b/pypy/jit/metainterp/test/test_tracingopts.py
--- a/pypy/jit/metainterp/test/test_tracingopts.py
+++ b/pypy/jit/metainterp/test/test_tracingopts.py
@@ -590,4 +590,32 @@
         assert res == 4
         self.check_operations_history(int_add_ovf=0)
         res = self.interp_operations(fn, [sys.maxint])
-        assert res == 12
\ No newline at end of file
+        assert res == 12
+
+
+
+    def test_opaque_list(self):
+        from pypy.rlib.rerased import new_erasing_pair
+        erase, unerase = new_erasing_pair("test_opaque_list")
+        def fn(n, ca, cb):
+            l1 = [n]
+            l2 = [n]
+            a1 = erase(l1)
+            a2 = erase(l1)
+            a = a1
+            if ca:
+                a = a2
+                if n < -100:
+                    unerase(a).append(5)
+            b = a1
+            if cb:
+                b = a
+            return unerase(a)[0] + unerase(b)[0]
+        res = self.interp_operations(fn, [7, 0, 1])
+        assert res == 7 * 2
+        self.check_operations_history(getarrayitem_gc=0,
+                getfield_gc=0)
+        res = self.interp_operations(fn, [-7, 1, 1])
+        assert res == -7 * 2
+        self.check_operations_history(getarrayitem_gc=0,
+                getfield_gc=0)

From noreply at buildbot.pypy.org  Tue Oct  4 15:03:09 2011
From: noreply at buildbot.pypy.org (cfbolz)
Date: Tue,  4 Oct 2011 15:03:09 +0200 (CEST)
Subject: [pypy-commit] pypy list-strategies: for some reason some of the
 list fields are not cached across the loop. is this
Message-ID: <20111004130309.2C4C0820D9@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: list-strategies
Changeset: r47823:ef71640445f3
Date: 2011-10-04 15:02 +0200
http://bitbucket.org/pypy/pypy/changeset/ef71640445f3/

Log:	for some reason some of the list fields are not cached across the
	loop. is this due to the opaque pointer restriction of the short
	preambles?

diff --git a/pypy/module/pypyjit/test_pypy_c/test_misc.py b/pypy/module/pypyjit/test_pypy_c/test_misc.py
--- a/pypy/module/pypyjit/test_pypy_c/test_misc.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_misc.py
@@ -201,9 +201,11 @@
         assert log.result == 1000000
         loop, = log.loops_by_filename(self.filepath)
         assert loop.match("""
-            i16 = int_ge(i12, i13)
+            i14 = getfield_gc(p12, descr=<SignedFieldDescr list.length 4>)
+            i16 = uint_ge(i12, i14)
             guard_false(i16, descr=...)
-            p17 = getarrayitem_gc(p15, i12, descr=<GcPtrArrayDescr>)
+            p16 = getfield_gc(p12, descr=<GcPtrFieldDescr list.items 8>)
+            p17 = getarrayitem_gc(p16, i12, descr=<GcPtrArrayDescr>)
             i19 = int_add(i12, 1)
             setfield_gc(p9, i19, descr=<SignedFieldDescr .*W_AbstractSeqIterObject.inst_index .*>)
             guard_nonnull_class(p17, 146982464, descr=...)
@@ -217,7 +219,7 @@
             i28 = int_add_ovf(i10, i25)
             guard_no_overflow(descr=...)
             --TICK--
-            jump(p0, p1, p2, p3, p4, p5, p6, i28, i25, p9, p10, p11, i19, i13, p14, p15, descr=<Loop0>)
+            jump(p0, p1, p2, p3, p4, p5, p6, i28, i25, p9, p10, p11, p12, i19, descr=<Loop0>)
         """)
 
 
@@ -329,4 +331,4 @@
             guard_false(i28, descr=...)
             i30 = int_lshift(i20, 24)
             i31 = int_or(i26, i30)
-        """ % {"32_bit_only": extra})
\ No newline at end of file
+        """ % {"32_bit_only": extra})

From notifications-noreply at bitbucket.org  Tue Oct  4 16:19:26 2011
From: notifications-noreply at bitbucket.org (Bitbucket)
Date: Tue, 04 Oct 2011 14:19:26 -0000
Subject: [pypy-commit] Notification: pypy
Message-ID: <20111004141926.10438.51530@bitbucket13.managed.contegix.com>

You have received a notification from Dan Loewenherz.

Hi, I forked pypy. My fork is at https://bitbucket.org/dlo/pypy.

--
Disable notifications at https://bitbucket.org/account/notifications/

From pullrequests-noreply at bitbucket.org  Tue Oct  4 16:40:49 2011
From: pullrequests-noreply at bitbucket.org (Bitbucket)
Date: Tue, 04 Oct 2011 14:40:49 -0000
Subject: [pypy-commit] [OPEN] Pull request #9 for pypy/pypy: Don't perform
 regex search when CFLAGS isn't set in unixcompiler.py
Message-ID: <pullreq-487ae03a6cdf@bitbucket.org>

A new pull request has been opened by Dan Loewenherz.

dlo/pypy has changes to be pulled into pypy/pypy.

https://bitbucket.org/pypy/pypy/pull-request/9/dont-perform-regex-search-when-cflags-isnt

Title: Don't perform regex search when CFLAGS isn't set in unixcompiler.py

I was compiling PIL and ran into the following error on Mac OS X Lion:

http://pastie.org/2638071

I change unixcompiler.py to perform a check that cflags is set before performing the regex search.

Changes to be pulled:



--
This is an issue notification from bitbucket.org.
You are receiving this either because you are the participating
in a pull request, or you are following it.


From notifications-noreply at bitbucket.org  Tue Oct  4 16:42:17 2011
From: notifications-noreply at bitbucket.org (Bitbucket)
Date: Tue, 04 Oct 2011 14:42:17 -0000
Subject: [pypy-commit] [COMMENT] Pull request #9 for pypy/pypy: Don't
 perform regex search when CFLAGS isn't set in unixcompiler.py
Message-ID: <20111004144217.24735.33255@bitbucket05.managed.contegix.com>

New comment on pull request:

https://bitbucket.org/pypy/pypy/pull-request/9/dont-perform-regex-search-when-cflags-isnt#comment-277

Alex Gaynor (alex_gaynor) said:

cflags should always at least be '', and won't re.search just return None then, since there's no match?  That is to say, this seems like effectively no semantic change to me.

--
This is a pull request comment notification from bitbucket.org.
You are receiving this either because you are participating
in a pull request, or you are following it.

From notifications-noreply at bitbucket.org  Tue Oct  4 16:44:53 2011
From: notifications-noreply at bitbucket.org (Bitbucket)
Date: Tue, 04 Oct 2011 14:44:53 -0000
Subject: [pypy-commit] [COMMENT] Pull request #9 for pypy/pypy: Don't
 perform regex search when CFLAGS isn't set in unixcompiler.py
Message-ID: <20111004144453.9053.18200@bitbucket13.managed.contegix.com>

New comment on pull request:

https://bitbucket.org/pypy/pypy/pull-request/9/dont-perform-regex-search-when-cflags-isnt#comment-278

Dan Loewenherz (dlo) said:

cflags was set to None when compiling PIL, so I'm not sure. Maybe it's a PIL problem? Or is there a way to set a default when the environment variable isn't set? E.g.

    cflags = sysconfig.get_config_var('CFLAGS', '')

--
This is a pull request comment notification from bitbucket.org.
You are receiving this either because you are participating
in a pull request, or you are following it.

From notifications-noreply at bitbucket.org  Tue Oct  4 16:46:00 2011
From: notifications-noreply at bitbucket.org (Bitbucket)
Date: Tue, 04 Oct 2011 14:46:00 -0000
Subject: [pypy-commit] [COMMENT] Pull request #9 for pypy/pypy: Don't
 perform regex search when CFLAGS isn't set in unixcompiler.py
Message-ID: <20111004144600.1063.4739@bitbucket12.managed.contegix.com>

New comment on pull request:

https://bitbucket.org/pypy/pypy/pull-request/9/dont-perform-regex-search-when-cflags-isnt#comment-279

Alex Gaynor (alex_gaynor) said:

It's already doing that, effectively:

{{{
#!\
cflags = sysconfig.get_config_var('CFLAGS') or ''
}}}

--
This is a pull request comment notification from bitbucket.org.
You are receiving this either because you are participating
in a pull request, or you are following it.

From notifications-noreply at bitbucket.org  Tue Oct  4 16:49:23 2011
From: notifications-noreply at bitbucket.org (Bitbucket)
Date: Tue, 04 Oct 2011 14:49:23 -0000
Subject: [pypy-commit] [COMMENT] Pull request #9 for pypy/pypy: Don't
 perform regex search when CFLAGS isn't set in unixcompiler.py
Message-ID: <20111004144923.29582.9444@bitbucket02.managed.contegix.com>

New comment on pull request:

https://bitbucket.org/pypy/pypy/pull-request/9/dont-perform-regex-search-when-cflags-isnt#comment-280

Dan Loewenherz (dlo) said:

Haha...wow. That's weird. I thought I just updated my repo and that wasn't there. I see it in the pull request though.

I only get this line locally:

{{{
cflags = sysconfig.get_config_var('CFLAGS')
}}}

I must have an old copy of PyPy. My bad.

--
This is a pull request comment notification from bitbucket.org.
You are receiving this either because you are participating
in a pull request, or you are following it.

From notifications-noreply at bitbucket.org  Tue Oct  4 16:54:38 2011
From: notifications-noreply at bitbucket.org (Bitbucket)
Date: Tue, 04 Oct 2011 14:54:38 -0000
Subject: [pypy-commit] [COMMENT] Pull request #9 for pypy/pypy: Don't
 perform regex search when CFLAGS isn't set in unixcompiler.py
Message-ID: <20111004145438.5259.47232@bitbucket12.managed.contegix.com>

New comment on pull request:

https://bitbucket.org/pypy/pypy/pull-request/9/dont-perform-regex-search-when-cflags-isnt#comment-281

Dan Loewenherz (dlo) said:

Also I think the existing solution is cleaner...we can cancel this pull request.

--
This is a pull request comment notification from bitbucket.org.
You are receiving this either because you are participating
in a pull request, or you are following it.

From pullrequests-noreply at bitbucket.org  Tue Oct  4 16:56:18 2011
From: pullrequests-noreply at bitbucket.org (Bitbucket)
Date: Tue, 04 Oct 2011 14:56:18 -0000
Subject: [pypy-commit] [REJECTED] Pull request #9 for pypy/pypy: Don't
 perform regex search when CFLAGS isn't set in unixcompiler.py
In-Reply-To: <pullreq-487ae03a6cdf@bitbucket.org>
References: <pullreq-487ae03a6cdf@bitbucket.org>
Message-ID: <20111004145618.16168.19923@bitbucket01.managed.contegix.com>

Pull request #9 has been rejected by Alex Gaynor.

dlo/pypy had changes to be pulled into pypy/pypy.

https://bitbucket.org/pypy/pypy/pull-request/9/dont-perform-regex-search-when-cflags-isnt

Determined that the issue is fixed in a new PyPy.

Thanks for taking the time to test and report bugs!

Rejected changes:



The pull request has been closed.

--
This is an issue notification from bitbucket.org.
You are receiving this either because you are the participating
in a pull request, or you are following it.


From noreply at buildbot.pypy.org  Wed Oct  5 02:44:03 2011
From: noreply at buildbot.pypy.org (timo_jbo)
Date: Wed,  5 Oct 2011 02:44:03 +0200 (CEST)
Subject: [pypy-commit] pypy separate-applevel-numpy: merge in default
Message-ID: <20111005004403.D625A820D9@wyvern.cs.uni-duesseldorf.de>

Author: Timo Paulssen <timonator at perpetuum-immobile.de>
Branch: separate-applevel-numpy
Changeset: r47824:73824d8b6491
Date: 2011-10-05 02:43 +0200
http://bitbucket.org/pypy/pypy/changeset/73824d8b6491/

Log:	merge in default

diff --git a/lib_pypy/numpy/__init__.py b/lib_pypy/numpy/__init__.py
--- a/lib_pypy/numpy/__init__.py
+++ b/lib_pypy/numpy/__init__.py
@@ -39,6 +39,8 @@
         tan,
     )
 
+from math import e
+
 inf = float("inf")
 
 def average(a):
diff --git a/lib_pypy/pypy_test/test_numpy.py b/lib_pypy/pypy_test/test_numpy.py
--- a/lib_pypy/pypy_test/test_numpy.py
+++ b/lib_pypy/pypy_test/test_numpy.py
@@ -74,3 +74,11 @@
         raises(ValueError, "bincount(c, w)")
         raises(ValueError, "bincount([])")
 
+    def test_constants(self):
+        import math
+        from numpy import inf, e
+        assert type(inf) is float
+        assert inf == float("inf")
+        assert e == math.e
+        assert type(e) is float
+
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -350,6 +350,7 @@
         elif interval > MAX:
             interval = MAX
         self.checkinterval_scaled = interval * TICK_COUNTER_STEP
+        self.reset_ticker(-1)
 
     def _rebuild_action_dispatcher(self):
         periodic_actions = unrolling_iterable(self._periodic_actions)
diff --git a/pypy/interpreter/test/test_executioncontext.py b/pypy/interpreter/test/test_executioncontext.py
--- a/pypy/interpreter/test/test_executioncontext.py
+++ b/pypy/interpreter/test/test_executioncontext.py
@@ -42,6 +42,7 @@
         assert i == 9
 
     def test_periodic_action(self):
+        from pypy.interpreter.executioncontext import ActionFlag
 
         class DemoAction(executioncontext.PeriodicAsyncAction):
             counter = 0
@@ -53,17 +54,20 @@
 
         space = self.space
         a2 = DemoAction(space)
-        space.actionflag.register_periodic_action(a2, True)
         try:
-            for i in range(500):
-                space.appexec([], """():
-                    n = 5
-                    return n + 2
-                """)
-        except Finished:
-            pass
-        checkinterval = space.actionflag.getcheckinterval()
-        assert checkinterval / 10 < i < checkinterval * 1.1
+            space.actionflag.setcheckinterval(100)
+            space.actionflag.register_periodic_action(a2, True)
+            try:
+                for i in range(500):
+                    space.appexec([], """():
+                        n = 5
+                        return n + 2
+                    """)
+            except Finished:
+                pass
+        finally:
+            space.actionflag = ActionFlag()   # reset to default
+        assert 10 < i < 110
 
     def test_llprofile(self):
         l = []
diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -158,7 +158,7 @@
     return decorator
 
 @oopspec("jit.isconstant(value)")
- at specialize.argtype(0)
+ at specialize.ll()
 def isconstant(value):
     """
     While tracing, returns whether or not the value is currently known to be
@@ -178,9 +178,7 @@
 
     This is for advanced usage only.
     """
-    if NonConstant(False):
-        return True
-    return False
+    return NonConstant(False)
 
 class Entry(ExtRegistryEntry):
     _about_ = hint

From noreply at buildbot.pypy.org  Wed Oct  5 08:14:55 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Wed,  5 Oct 2011 08:14:55 +0200 (CEST)
Subject: [pypy-commit] pypy default: move some code arround to avoid
 swallowing unwanted exceptions.
Message-ID: <20111005061455.D715D82A02@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r47825:cdb51e95efd2
Date: 2011-10-05 02:14 -0400
http://bitbucket.org/pypy/pypy/changeset/cdb51e95efd2/

Log:	move some code arround to avoid swallowing unwanted exceptions.

diff --git a/pypy/jit/metainterp/optimizeopt/rewrite.py b/pypy/jit/metainterp/optimizeopt/rewrite.py
--- a/pypy/jit/metainterp/optimizeopt/rewrite.py
+++ b/pypy/jit/metainterp/optimizeopt/rewrite.py
@@ -49,32 +49,35 @@
     def find_rewritable_bool(self, op, args):
         try:
             oldopnum = opboolinvers[op.getopnum()]
+        except KeyError:
+            pass
+        else:
             targs = self.optimizer.make_args_key(ResOperation(oldopnum, [args[0], args[1]],
                                                               None))
             if self.try_boolinvers(op, targs):
                 return True
-        except KeyError:
-            pass
 
         try:
             oldopnum = opboolreflex[op.getopnum()] # FIXME: add INT_ADD, INT_MUL
+        except KeyError:
+            pass
+        else:
             targs = self.optimizer.make_args_key(ResOperation(oldopnum, [args[1], args[0]],
                                                               None))
             oldop = self.get_pure_result(targs)
             if oldop is not None and oldop.getdescr() is op.getdescr():
                 self.make_equal_to(op.result, self.getvalue(oldop.result))
                 return True
-        except KeyError:
-            pass
 
         try:
             oldopnum = opboolinvers[opboolreflex[op.getopnum()]]
+        except KeyError:
+            pass
+        else:
             targs = self.optimizer.make_args_key(ResOperation(oldopnum, [args[1], args[0]],
                                                               None))
             if self.try_boolinvers(op, targs):
                 return True
-        except KeyError:
-            pass
 
         return False
 

From noreply at buildbot.pypy.org  Wed Oct  5 09:35:06 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Wed,  5 Oct 2011 09:35:06 +0200 (CEST)
Subject: [pypy-commit] pypy.org extradoc: add progressbar
Message-ID: <20111005073506.2E6B482A02@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: extradoc
Changeset: r277:f20036f73b59
Date: 2011-10-05 09:34 +0200
http://bitbucket.org/pypy/pypy.org/changeset/f20036f73b59/

Log:	add progressbar

diff --git a/donate1.html b/donate1.html
--- a/donate1.html
+++ b/donate1.html
@@ -4,6 +4,20 @@
     <a href="#" onclick="general_donate(); return false;">Donate towards general pypy progress</a><br/>
   </li>
   <li>
+<script>
+  $(function() {
+    $("#progressbar").progressbar({
+      value: 1.8
+   });
+  });
+</script>
+
+   $1974 of $105000 (1.8%)
+   <div id="progressbar">
+   </div>
+
+  </li>
+  <li>
     <span id="donate_text">This donation will go towards supporting python 3 in PyPy <a href="py3donate.html">Read proposal</a>
   </li>
   <li>

From noreply at buildbot.pypy.org  Wed Oct  5 10:40:56 2011
From: noreply at buildbot.pypy.org (cfbolz)
Date: Wed,  5 Oct 2011 10:40:56 +0200 (CEST)
Subject: [pypy-commit] pypy list-strategies: The ticker does not necessarily
 decrease by one. it depends on the trace length.
Message-ID: <20111005084056.ED35B82A02@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: list-strategies
Changeset: r47826:4125c33b4c6c
Date: 2011-10-05 10:40 +0200
http://bitbucket.org/pypy/pypy/changeset/4125c33b4c6c/

Log:	The ticker does not necessarily decrease by one. it depends on the
	trace length.

diff --git a/pypy/module/pypyjit/test_pypy_c/model.py b/pypy/module/pypyjit/test_pypy_c/model.py
--- a/pypy/module/pypyjit/test_pypy_c/model.py
+++ b/pypy/module/pypyjit/test_pypy_c/model.py
@@ -259,7 +259,7 @@
         # to repeat it every time
         ticker_check = """
             ticker0 = getfield_raw(ticker_address, descr=<SignedFieldDescr pypysig_long_struct.c_value .*>)
-            ticker1 = int_sub(ticker0, 1)
+            ticker1 = int_sub(ticker0, _)
             setfield_raw(ticker_address, ticker1, descr=<SignedFieldDescr pypysig_long_struct.c_value .*>)
             ticker_cond0 = int_lt(ticker1, 0)
             guard_false(ticker_cond0, descr=...)

From noreply at buildbot.pypy.org  Wed Oct  5 11:46:17 2011
From: noreply at buildbot.pypy.org (cfbolz)
Date: Wed,  5 Oct 2011 11:46:17 +0200 (CEST)
Subject: [pypy-commit] pypy list-strategies: don't rely on the file encoding,
 which seems to be brittle in some situations.
Message-ID: <20111005094617.B29FF82A02@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: list-strategies
Changeset: r47827:d9497f6bf240
Date: 2011-10-05 11:11 +0200
http://bitbucket.org/pypy/pypy/changeset/d9497f6bf240/

Log:	don't rely on the file encoding, which seems to be brittle in some
	situations.

diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py
--- a/pypy/objspace/std/test/test_listobject.py
+++ b/pypy/objspace/std/test/test_listobject.py
@@ -918,7 +918,7 @@
         assert l == [3, 4]
 
     def test_unicode(self):
-        s = u"&#65533;&#65533;&#65533;"
+        s = u"\ufffd\ufffd\ufffd"
         assert s.encode("ascii", "replace") == "???"
         assert s.encode("ascii", "ignore") == ""
 

From noreply at buildbot.pypy.org  Wed Oct  5 11:46:18 2011
From: noreply at buildbot.pypy.org (cfbolz)
Date: Wed,  5 Oct 2011 11:46:18 +0200 (CEST)
Subject: [pypy-commit] pypy list-strategies: making an empty wrapped list
 takes a lot less allocations now.
Message-ID: <20111005094618.DDB5182A02@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: list-strategies
Changeset: r47828:92ccf911530a
Date: 2011-10-05 11:25 +0200
http://bitbucket.org/pypy/pypy/changeset/92ccf911530a/

Log:	making an empty wrapped list takes a lot less allocations now.

diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py
--- a/pypy/module/pypyjit/test_pypy_c/test_call.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_call.py
@@ -350,10 +350,11 @@
     def test_blockstack_virtualizable(self):
         def main(n):
             from pypyjit import residual_call
+            l = len
             i = 0
             while i < n:
                 try:
-                    residual_call(len, [])   # ID: call
+                    residual_call(l, [])   # ID: call
                 except:
                     pass
                 i += 1
@@ -369,11 +370,8 @@
             p22 = new_with_vtable(19511408)
             p24 = new_array(1, descr=<GcPtrArrayDescr>)
             p26 = new_with_vtable(ConstClass(W_ListObject))
-            p27 = new(descr=<SizeDescr .*>)
-            p29 = new_array(0, descr=<GcPtrArrayDescr>)
             setfield_gc(p0, i20, descr=<SignedFieldDescr .*PyFrame.vable_token .*>)
-            setfield_gc(p27, p29, descr=<GcPtrFieldDescr list.items .*>)
-            setfield_gc(p26, p27, descr=<.* .*W_ListObject.inst_wrappeditems .*>)
+            setfield_gc(p26, ConstPtr(ptr22), descr=<GcPtrFieldDescr pypy.objspace.std.listobject.W_ListObject.inst_strategy 12>)
             setarrayitem_gc(p24, 0, p26, descr=<GcPtrArrayDescr>)
             setfield_gc(p22, p24, descr=<GcPtrFieldDescr .*Arguments.inst_arguments_w .*>)
             p32 = call_may_force(11376960, p18, p22, descr=<GcPtrCallDescr>)

From noreply at buildbot.pypy.org  Wed Oct  5 11:46:20 2011
From: noreply at buildbot.pypy.org (cfbolz)
Date: Wed,  5 Oct 2011 11:46:20 +0200 (CEST)
Subject: [pypy-commit] pypy list-strategies: unroll getitem as well,
 if the list has a constant size.
Message-ID: <20111005094620.1360482A02@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: list-strategies
Changeset: r47829:d396dfd92356
Date: 2011-10-05 11:45 +0200
http://bitbucket.org/pypy/pypy/changeset/d396dfd92356/

Log:	unroll getitem as well, if the list has a constant size.

diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -12,6 +12,8 @@
 from pypy.rlib import rerased, jit
 from pypy.interpreter.argument import Signature
 
+UNROLL_CUTOFF = 5
+
 def make_range_list(space, start, step, length):
     if length <= 0:
         strategy = space.fromcache(EmptyListStrategy)
@@ -26,7 +28,7 @@
     storage = strategy.erase(None)
     return W_ListObject.from_storage_and_strategy(space, storage, strategy)
 
- at jit.look_inside_iff(lambda space, list_w: jit.isconstant(len(list_w)) and len(list_w) < 5)
+ at jit.look_inside_iff(lambda space, list_w: jit.isconstant(len(list_w)) and len(list_w) < UNROLL_CUTOFF)
 def get_strategy_from_list_objects(space, list_w):
     if not list_w:
         return space.fromcache(EmptyListStrategy)
@@ -143,13 +145,13 @@
         return self.strategy.getslice(self, start, stop, step, length)
 
     def getitems(self):
-        """Returns a list of all items after wrapping them. Used only for sorting
-        an ObjectList, sorting with custom compare method or switching to ObjectListStrategy."""
+        """Returns a list of all items after wrapping them. The result can
+        share with the storage, if possible."""
         return self.strategy.getitems(self)
 
     def getitems_copy(self):
-        """Returns a copy of all items in the list. Same as getitems except for ObjectListStrategy
-        which has a different getitems method"""
+        """Returns a copy of all items in the list. Same as getitems except for
+        ObjectListStrategy."""
         return self.strategy.getitems_copy(self)
     # ___________________________________________________
 
@@ -581,7 +583,7 @@
         raise NotImplementedError("abstract base class")
 
     @jit.look_inside_iff(lambda space, w_list, list_w:
-        jit.isconstant(len(list_w)) and len(list_w) < 5)
+        jit.isconstant(len(list_w)) and len(list_w) < UNROLL_CUTOFF)
     def init_from_list_w(self, w_list, list_w):
         l = [self.unwrap(w_item) for w_item in list_w]
         w_list.lstorage = self.erase(l)
@@ -620,6 +622,8 @@
             raise
         return self.wrap(r)
 
+    @jit.look_inside_iff(lambda self, w_list:
+            jit.isconstant(w_list.length()) and w_list.length() < UNROLL_CUTOFF)
     def getitems_copy(self, w_list):
         return [self.wrap(item) for item in self.unerase(w_list.lstorage)]
     getitems = getitems_copy

From noreply at buildbot.pypy.org  Wed Oct  5 14:05:42 2011
From: noreply at buildbot.pypy.org (cfbolz)
Date: Wed,  5 Oct 2011 14:05:42 +0200 (CEST)
Subject: [pypy-commit] pypy list-strategies: clean up getitem variants a bit
Message-ID: <20111005120542.272FC82A02@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: list-strategies
Changeset: r47830:623e452781d4
Date: 2011-10-05 11:51 +0200
http://bitbucket.org/pypy/pypy/changeset/623e452781d4/

Log:	clean up getitem variants a bit

diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -240,7 +240,7 @@
         raise NotImplementedError
 
     def getitems(self, w_list):
-        raise NotImplementedError
+        return self.getitems_copy(w_list)
 
     def getitems_copy(self, w_list):
         raise NotImplementedError
@@ -325,7 +325,7 @@
         return self.cached_emptylist_w
 
     def getitems_copy(self, w_list):
-        return self.cached_emptylist_w
+        return []
 
     def getstorage_copy(self, w_list):
         return self.erase(None)
@@ -443,9 +443,8 @@
     def getitem(self, w_list, i):
         return self.wrap(self._getitem_unwrapped(w_list, i))
 
-    def getitems(self, w_list):
+    def getitems_copy(self, w_list):
         return self._getitems_range(w_list, True)
-    getitems_copy = getitems
 
     def getstorage_copy(self, w_list):
         # tuple is unmutable
@@ -626,7 +625,6 @@
             jit.isconstant(w_list.length()) and w_list.length() < UNROLL_CUTOFF)
     def getitems_copy(self, w_list):
         return [self.wrap(item) for item in self.unerase(w_list.lstorage)]
-    getitems = getitems_copy
 
     def getstorage_copy(self, w_list):
         items = self.unerase(w_list.lstorage)[:]

From noreply at buildbot.pypy.org  Wed Oct  5 14:05:43 2011
From: noreply at buildbot.pypy.org (cfbolz)
Date: Wed,  5 Oct 2011 14:05:43 +0200 (CEST)
Subject: [pypy-commit] pypy list-strategies: add a new space method
 listview_str which returns None except if the argument
Message-ID: <20111005120543.65A3282A02@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: list-strategies
Changeset: r47831:547bc8141987
Date: 2011-10-05 13:29 +0200
http://bitbucket.org/pypy/pypy/changeset/547bc8141987/

Log:	add a new space method listview_str which returns None except if the
	argument is a list using the string strategy. then return an
	unwrapped list of unwrapped strings. use this in str.join.

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -835,6 +835,13 @@
         """
         return self.unpackiterable(w_iterable, expected_length)
 
+    def listview_str(self, w_list):
+        """ Return a list of unwrapped strings out of a list of strings. If the
+        argument is not a list or does not contain only strings, return None.
+        May return None anyway.
+        """
+        return None
+
     @jit.unroll_safe
     def exception_match(self, w_exc_type, w_check_class):
         """Checks if the given exception type matches 'w_check_class'."""
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -153,6 +153,11 @@
         """Returns a copy of all items in the list. Same as getitems except for
         ObjectListStrategy."""
         return self.strategy.getitems_copy(self)
+
+    def getitems_str(self):
+        """ Return the items in the list as unwrapped strings. If the list does
+        not use the list strategy, return None. """
+        return self.strategy.getitems_str(self)
     # ___________________________________________________
 
 
@@ -245,6 +250,9 @@
     def getitems_copy(self, w_list):
         raise NotImplementedError
 
+    def getitems_str(self, w_list):
+        return None
+
     def getstorage_copy(self, w_list):
         raise NotImplementedError
 
@@ -923,6 +931,9 @@
         if reverse:
             l.reverse()
 
+    def getitems_str(self, w_list):
+        return self.unerase(w_list.lstorage)
+
 # _______________________________________________________
 
 init_signature = Signature(['sequence'], None, None)
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -436,6 +436,11 @@
             raise self._wrap_expected_length(expected_length, len(t))
         return t
 
+    def listview_str(self, w_obj):
+        if isinstance(w_obj, W_ListObject):
+            return w_obj.getitems_str()
+        return None
+
     def sliceindices(self, w_slice, w_length):
         if isinstance(w_slice, W_SliceObject):
             a, b, c = w_slice.indices3(self, self.int_w(w_length))
diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py
--- a/pypy/objspace/std/stringobject.py
+++ b/pypy/objspace/std/stringobject.py
@@ -347,6 +347,9 @@
                                                        sliced)
 
 def str_join__String_ANY(space, w_self, w_list):
+    l = space.listview_str(w_list)
+    if l is not None:
+        return space.wrap(w_self._value.join(l))
     list_w = space.listview(w_list)
     size = len(list_w)
 
diff --git a/pypy/objspace/std/test/test_liststrategies.py b/pypy/objspace/std/test/test_liststrategies.py
--- a/pypy/objspace/std/test/test_liststrategies.py
+++ b/pypy/objspace/std/test/test_liststrategies.py
@@ -348,6 +348,17 @@
         l3 = W_ListObject(self.space, [self.space.wrap("eins"), self.space.wrap(u"zwei")])
         assert isinstance(l3.strategy, ObjectListStrategy)
 
+    def test_listview_str(self):
+        space = self.space
+        assert space.listview_str(space.wrap("a")) is None
+        w_l = self.space.newlist([self.space.wrap('a'), self.space.wrap('b')])
+        assert space.listview_str(w_l) == ["a", "b"]
+
+    def test_string_uses_listview_str(self):
+        space = self.space
+        w_l = self.space.newlist([self.space.wrap('a'), self.space.wrap('b')])
+        w_l.getitems = None
+        assert space.str_w(space.call_method(space.wrap("c"), "join", w_l)) == "acb"
 
 class TestW_ListStrategiesDisabled:
     def setup_class(cls):

From noreply at buildbot.pypy.org  Wed Oct  5 14:05:44 2011
From: noreply at buildbot.pypy.org (cfbolz)
Date: Wed,  5 Oct 2011 14:05:44 +0200 (CEST)
Subject: [pypy-commit] pypy list-strategies: add a space.newlist_str method
 that can be used to wrap a list of unwrapped
Message-ID: <20111005120544.9C74182A02@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: list-strategies
Changeset: r47832:36b7b2d1a3d8
Date: 2011-10-05 13:59 +0200
http://bitbucket.org/pypy/pypy/changeset/36b7b2d1a3d8/

Log:	add a space.newlist_str method that can be used to wrap a list of
	unwrapped strings. implement a fast path in the std object space.

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -842,6 +842,9 @@
         """
         return None
 
+    def newlist_str(self, list_s):
+        return self.newlist([self.wrap(s) for s in list_s])
+
     @jit.unroll_safe
     def exception_match(self, w_exc_type, w_check_class):
         """Checks if the given exception type matches 'w_check_class'."""
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -77,8 +77,16 @@
         w_self.space = space
         w_self.strategy = strategy
         w_self.lstorage = storage
+        if not space.config.objspace.std.withliststrategies:
+            w_self.switch_to_object_strategy()
         return w_self
 
+    @staticmethod
+    def newlist_str(space, list_s):
+        strategy = space.fromcache(StringListStrategy)
+        storage = strategy.erase(list_s)
+        return W_ListObject.from_storage_and_strategy(space, storage, strategy)
+
     def __repr__(w_self):
         """ representation for debugging purposes """
         return "%s(%s, %s)" % (w_self.__class__.__name__, w_self.strategy, w_self.lstorage._x)
@@ -91,6 +99,7 @@
     def switch_to_object_strategy(self):
         list_w = self.getitems()
         self.strategy = self.space.fromcache(ObjectListStrategy)
+        # XXX this is quite indirect
         self.init_from_list_w(list_w)
 
     def _temporarily_as_objects(self):
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -307,6 +307,9 @@
     def newlist(self, list_w):
         return W_ListObject(self, list_w)
 
+    def newlist_str(self, list_s):
+        return W_ListObject.newlist_str(self, list_s)
+
     def newdict(self, module=False, instance=False, classofinstance=None,
                 strdict=False):
         return W_DictMultiObject.allocate_and_init_instance(
diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py
--- a/pypy/objspace/std/stringobject.py
+++ b/pypy/objspace/std/stringobject.py
@@ -215,7 +215,7 @@
 
 def str_split__String_None_ANY(space, w_self, w_none, w_maxsplit=-1):
     maxsplit = space.int_w(w_maxsplit)
-    res_w = []
+    res = []
     value = w_self._value
     length = len(value)
     i = 0
@@ -238,12 +238,12 @@
             maxsplit -= 1   # NB. if it's already < 0, it stays < 0
 
         # the word is value[i:j]
-        res_w.append(sliced(space, value, i, j, w_self))
+        res.append(value[i:j])
 
         # continue to look from the character following the space after the word
         i = j + 1
 
-    return space.newlist(res_w)
+    return space.newlist_str(res)
 
 def str_split__String_String_ANY(space, w_self, w_by, w_maxsplit=-1):
     maxsplit = space.int_w(w_maxsplit)
@@ -253,20 +253,20 @@
     if bylen == 0:
         raise OperationError(space.w_ValueError, space.wrap("empty separator"))
 
-    res_w = []
+    res = []
     start = 0
     if bylen == 1 and maxsplit < 0:
         # fast path: uses str.rfind(character) and str.count(character)
         by = by[0]    # annotator hack: string -> char
         count = value.count(by)
-        res_w = [None] * (count + 1)
+        res = [None] * (count + 1)
         end = len(value)
         while count >= 0:
             assert end >= 0
             prev = value.rfind(by, 0, end)
             start = prev + 1
             assert start >= 0
-            res_w[count] = sliced(space, value, start, end, w_self)
+            res[count] = value[start:end]
             count -= 1
             end = prev
     else:
@@ -274,12 +274,12 @@
             next = value.find(by, start)
             if next < 0:
                 break
-            res_w.append(sliced(space, value, start, next, w_self))
+            res.append(value[start:next])
             start = next + bylen
             maxsplit -= 1   # NB. if it's already < 0, it stays < 0
-        res_w.append(sliced(space, value, start, len(value), w_self))
+        res.append(value[start:])
 
-    return space.newlist(res_w)
+    return space.newlist_str(res)
 
 def str_rsplit__String_None_ANY(space, w_self, w_none, w_maxsplit=-1):
     maxsplit = space.int_w(w_maxsplit)
diff --git a/pypy/objspace/std/test/test_liststrategies.py b/pypy/objspace/std/test/test_liststrategies.py
--- a/pypy/objspace/std/test/test_liststrategies.py
+++ b/pypy/objspace/std/test/test_liststrategies.py
@@ -360,6 +360,26 @@
         w_l.getitems = None
         assert space.str_w(space.call_method(space.wrap("c"), "join", w_l)) == "acb"
 
+    def test_newlist_str(self):
+        space = self.space
+        l = ['a', 'b']
+        w_l = self.space.newlist_str(l)
+        assert isinstance(w_l.strategy, StringListStrategy)
+        assert space.listview_str(w_l) is l
+
+    def test_string_uses_newlist_str(self):
+        space = self.space
+        w_s = space.wrap("a b c")
+        space.newlist = None
+        try:
+            w_l = space.call_method(w_s, "split")
+            w_l2 = space.call_method(w_s, "split", space.wrap(" "))
+        finally:
+            del space.newlist
+        assert space.listview_str(w_l) == ["a", "b", "c"]
+        assert space.listview_str(w_l2) == ["a", "b", "c"]
+
+
 class TestW_ListStrategiesDisabled:
     def setup_class(cls):
         cls.space = gettestobjspace(**{"objspace.std.withliststrategies" :
diff --git a/pypy/objspace/std/test/test_strsliceobject.py b/pypy/objspace/std/test/test_strsliceobject.py
--- a/pypy/objspace/std/test/test_strsliceobject.py
+++ b/pypy/objspace/std/test/test_strsliceobject.py
@@ -110,12 +110,6 @@
         assert 'W_StringSliceObject' in __pypy__.internal_repr(s)
         assert hash(s) & 0x7fffffff == 0x7e0bce58
 
-    def test_split_produces_strslices(self):
-        import __pypy__
-        l = ("X" * 100 + "," + "Y" * 100).split(",")
-        assert "W_StringSliceObject" in __pypy__.internal_repr(l[0])
-        assert "W_StringSliceObject" in __pypy__.internal_repr(l[1])
-
     def test_strip_produces_strslices(self):
         import __pypy__
         s = ("abc" + "X" * 100 + "," + "Y" * 100 + "abc").strip("abc")

From noreply at buildbot.pypy.org  Wed Oct  5 14:05:45 2011
From: noreply at buildbot.pypy.org (cfbolz)
Date: Wed,  5 Oct 2011 14:05:45 +0200 (CEST)
Subject: [pypy-commit] pypy list-strategies: that means we can now use the
 split in rlib.rstring
Message-ID: <20111005120545.CB6BC82A02@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: list-strategies
Changeset: r47833:05b957a9d0ad
Date: 2011-10-05 14:05 +0200
http://bitbucket.org/pypy/pypy/changeset/05b957a9d0ad/

Log:	that means we can now use the split in rlib.rstring

diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py
--- a/pypy/objspace/std/stringobject.py
+++ b/pypy/objspace/std/stringobject.py
@@ -11,7 +11,7 @@
 from pypy.objspace.std.listobject import W_ListObject
 from pypy.objspace.std.noneobject import W_NoneObject
 from pypy.objspace.std.tupleobject import W_TupleObject
-from pypy.rlib.rstring import StringBuilder
+from pypy.rlib.rstring import StringBuilder, split
 from pypy.interpreter.buffer import StringBuffer
 
 from pypy.objspace.std.stringtype import sliced, wrapstr, wrapchar, \
@@ -253,9 +253,9 @@
     if bylen == 0:
         raise OperationError(space.w_ValueError, space.wrap("empty separator"))
 
-    res = []
-    start = 0
     if bylen == 1 and maxsplit < 0:
+        res = []
+        start = 0
         # fast path: uses str.rfind(character) and str.count(character)
         by = by[0]    # annotator hack: string -> char
         count = value.count(by)
@@ -270,14 +270,7 @@
             count -= 1
             end = prev
     else:
-        while maxsplit != 0:
-            next = value.find(by, start)
-            if next < 0:
-                break
-            res.append(value[start:next])
-            start = next + bylen
-            maxsplit -= 1   # NB. if it's already < 0, it stays < 0
-        res.append(value[start:])
+        res = split(value, by, maxsplit)
 
     return space.newlist_str(res)
 

From noreply at buildbot.pypy.org  Wed Oct  5 22:02:30 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed,  5 Oct 2011 22:02:30 +0200 (CEST)
Subject: [pypy-commit] pypy default: Implement sys.gettrace()
Message-ID: <20111005200230.6C34782A02@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r47834:d71f4eb207d1
Date: 2011-10-05 22:00 +0200
http://bitbucket.org/pypy/pypy/changeset/d71f4eb207d1/

Log:	Implement sys.gettrace()

diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -175,6 +175,9 @@
             self.w_tracefunc = w_func
             self.space.frame_trace_action.fire()
 
+    def gettrace(self):
+        return self.w_tracefunc
+
     def setprofile(self, w_func):
         """Set the global trace function."""
         if self.space.is_w(w_func, self.space.w_None):
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -55,6 +55,7 @@
         'exc_info'              : 'vm.exc_info', 
         'exc_clear'             : 'vm.exc_clear', 
         'settrace'              : 'vm.settrace',
+        'gettrace'              : 'vm.gettrace',
         'setprofile'            : 'vm.setprofile',
         'getprofile'            : 'vm.getprofile',
         'call_tracing'          : 'vm.call_tracing',
diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -478,6 +478,7 @@
         sys.settrace(trace)
         try:
             x()
+            assert sys.gettrace() is trace
         finally:
             sys.settrace(None)
         assert len(counts) == 1
diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -129,6 +129,11 @@
 function call.  See the debugger chapter in the library manual."""
     space.getexecutioncontext().settrace(w_func)
 
+def gettrace(space):
+    """Return the global debug tracing function set with sys.settrace.
+See the debugger chapter in the library manual."""
+    return space.getexecutioncontext().gettrace()
+
 def setprofile(space, w_func):
     """Set the profiling function.  It will be called on each function call
 and return.  See the profiler chapter in the library manual."""

From noreply at buildbot.pypy.org  Wed Oct  5 22:02:31 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed,  5 Oct 2011 22:02:31 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix sys.getprofile() docstring
Message-ID: <20111005200231.A380682A02@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r47835:be4e5bec46c1
Date: 2011-10-05 22:01 +0200
http://bitbucket.org/pypy/pypy/changeset/be4e5bec46c1/

Log:	Fix sys.getprofile() docstring

diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -140,8 +140,8 @@
     space.getexecutioncontext().setprofile(w_func)
 
 def getprofile(space):
-    """Set the profiling function.  It will be called on each function call
-and return.  See the profiler chapter in the library manual."""
+    """Return the profiling function set with sys.setprofile.
+See the profiler chapter in the library manual."""
     w_func = space.getexecutioncontext().getprofile()
     if w_func is not None:
         return w_func

From noreply at buildbot.pypy.org  Wed Oct  5 22:18:26 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed,  5 Oct 2011 22:18:26 +0200 (CEST)
Subject: [pypy-commit] pypy default: Have cpython's test_sys_settrace test
	something.
Message-ID: <20111005201826.9D53782A02@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r47836:383ca802ba07
Date: 2011-10-05 22:17 +0200
http://bitbucket.org/pypy/pypy/changeset/383ca802ba07/

Log:	Have cpython's test_sys_settrace test something.

diff --git a/lib-python/modified-2.7/test/test_sys_settrace.py b/lib-python/modified-2.7/test/test_sys_settrace.py
--- a/lib-python/modified-2.7/test/test_sys_settrace.py
+++ b/lib-python/modified-2.7/test/test_sys_settrace.py
@@ -286,11 +286,11 @@
         self.compare_events(func.func_code.co_firstlineno,
                             tracer.events, func.events)
 
-    def set_and_retrieve_none(self):
+    def test_set_and_retrieve_none(self):
         sys.settrace(None)
         assert sys.gettrace() is None
 
-    def set_and_retrieve_func(self):
+    def test_set_and_retrieve_func(self):
         def fn(*args):
             pass
 

From noreply at buildbot.pypy.org  Thu Oct  6 03:31:58 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Thu,  6 Oct 2011 03:31:58 +0200 (CEST)
Subject: [pypy-commit] pypy.org extradoc: update progress bar
Message-ID: <20111006013158.4E6BD82A02@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: extradoc
Changeset: r278:711a769c80c1
Date: 2011-10-06 03:31 +0200
http://bitbucket.org/pypy/pypy.org/changeset/711a769c80c1/

Log:	update progress bar

diff --git a/donate1.html b/donate1.html
--- a/donate1.html
+++ b/donate1.html
@@ -7,12 +7,12 @@
 <script>
   $(function() {
     $("#progressbar").progressbar({
-      value: 1.8
+      value: 3.1
    });
   });
 </script>
 
-   $1974 of $105000 (1.8%)
+   $3286 of $105000 (3.1%)
    <div id="progressbar">
    </div>
 

From noreply at buildbot.pypy.org  Thu Oct  6 04:05:34 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Thu,  6 Oct 2011 04:05:34 +0200 (CEST)
Subject: [pypy-commit] pypy default: support length of a builder
Message-ID: <20111006020534.94A6E82A02@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r47837:b9e14afe8066
Date: 2011-10-04 16:08 +0200
http://bitbucket.org/pypy/pypy/changeset/b9e14afe8066/

Log:	support length of a builder

diff --git a/pypy/module/__pypy__/interp_builders.py b/pypy/module/__pypy__/interp_builders.py
--- a/pypy/module/__pypy__/interp_builders.py
+++ b/pypy/module/__pypy__/interp_builders.py
@@ -40,6 +40,12 @@
             self.builder = None
             return w_s
 
+        def descr_len(self, space):
+            if self.builder is None:
+                raise OperationError(space.w_ValueError,
+                                     space.wrap('no lenght of built builder'))
+            return space.wrap(self.builder.getlength())
+
     W_Builder.__name__ = "W_%s" % name
     W_Builder.typedef = TypeDef(name,
         __new__ = interp2app(func_with_new_name(
@@ -48,6 +54,7 @@
         append = interp2app(W_Builder.descr_append),
         append_slice = interp2app(W_Builder.descr_append_slice),
         build = interp2app(W_Builder.descr_build),
+        __len__ = interp2app(W_Builder.descr_len),
     )
     W_Builder.typedef.acceptable_as_base_class = False
     return W_Builder
diff --git a/pypy/module/__pypy__/test/test_builders.py b/pypy/module/__pypy__/test/test_builders.py
--- a/pypy/module/__pypy__/test/test_builders.py
+++ b/pypy/module/__pypy__/test/test_builders.py
@@ -38,7 +38,9 @@
         b = StringBuilder()
         b.append("abc")
         b.append("123")
+        assert len(b) == 6
         b.append("you and me")
         s = b.build()
+        raises(ValueError, len, b)
         assert s == "abc123you and me"
-        raises(ValueError, b.build)
\ No newline at end of file
+        raises(ValueError, b.build)

From noreply at buildbot.pypy.org  Thu Oct  6 04:05:35 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Thu,  6 Oct 2011 04:05:35 +0200 (CEST)
Subject: [pypy-commit] pypy lightweight-finalizers: remove one set of hacks
 to another in order for it to work after translation
Message-ID: <20111006020535.CF03382A02@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: lightweight-finalizers
Changeset: r47838:a85b330c7a9c
Date: 2011-10-04 17:58 +0200
http://bitbucket.org/pypy/pypy/changeset/a85b330c7a9c/

Log:	remove one set of hacks to another in order for it to work after
	translation as well (OP_TRACK_ALLOC_STOP will be emitted correctly)

diff --git a/pypy/annotation/builtin.py b/pypy/annotation/builtin.py
--- a/pypy/annotation/builtin.py
+++ b/pypy/annotation/builtin.py
@@ -691,7 +691,7 @@
     assert isinstance(s_size, SomeInteger) #XXX add noneg...?
     return SomeInteger(nonneg=True)
 
-def raw_free(s_addr, s_track_free=None):
+def raw_free(s_addr):
     assert isinstance(s_addr, SomeAddress)
 
 def raw_memclear(s_addr, s_int):
diff --git a/pypy/rpython/lltypesystem/llmemory.py b/pypy/rpython/lltypesystem/llmemory.py
--- a/pypy/rpython/lltypesystem/llmemory.py
+++ b/pypy/rpython/lltypesystem/llmemory.py
@@ -8,7 +8,6 @@
 from pypy.rlib.objectmodel import Symbolic
 from pypy.rpython.lltypesystem import lltype
 from pypy.tool.uid import uid
-from pypy.tool import leakfinder
 
 class AddressOffset(Symbolic):
 
@@ -781,7 +780,7 @@
         raise NotImplementedError(size)
     return size._raw_malloc([], zero=False)
 
-def raw_free(adr, track_free=False):
+def raw_free(adr):
     # try to free the whole object if 'adr' is the address of the header
     from pypy.rpython.memory.gcheader import GCHeaderBuilder
     try:
@@ -791,8 +790,6 @@
     else:
         raw_free(cast_ptr_to_adr(objectptr))
     assert isinstance(adr.ref()._obj, lltype._parentable)
-    if track_free:
-        leakfinder.remember_free(adr.ptr._as_obj())
     adr.ptr._as_obj()._free()
 
 def raw_malloc_usage(size):
diff --git a/pypy/rpython/lltypesystem/lloperation.py b/pypy/rpython/lltypesystem/lloperation.py
--- a/pypy/rpython/lltypesystem/lloperation.py
+++ b/pypy/rpython/lltypesystem/lloperation.py
@@ -400,7 +400,7 @@
     'raw_store':            LLOp(),
     'stack_malloc':         LLOp(), # mmh
     'track_alloc_start':    LLOp(),
-    'track_alloc_stop':     LLOp(),
+    'track_alloc_stop':     LLOp(canrun=True),
     'adr_add':              LLOp(canfold=True),
     'adr_sub':              LLOp(canfold=True),
     'adr_delta':            LLOp(canfold=True),
diff --git a/pypy/rpython/lltypesystem/opimpl.py b/pypy/rpython/lltypesystem/opimpl.py
--- a/pypy/rpython/lltypesystem/opimpl.py
+++ b/pypy/rpython/lltypesystem/opimpl.py
@@ -4,6 +4,7 @@
 from pypy.rpython.lltypesystem import lltype, llmemory
 from pypy.rpython.lltypesystem.lloperation import opimpls
 from pypy.rlib import debug
+from pypy.tool import leakfinder
 
 # ____________________________________________________________
 # Implementation of the 'canfold' operations
@@ -598,6 +599,9 @@
     from pypy.rlib.rtimer import read_timestamp
     return read_timestamp()
 
+def op_track_alloc_stop(addr):
+    leakfinder.remember_free(addr.ptr._obj)
+
 # ____________________________________________________________
 
 def get_op_impl(opname):
diff --git a/pypy/rpython/memory/gc/base.py b/pypy/rpython/memory/gc/base.py
--- a/pypy/rpython/memory/gc/base.py
+++ b/pypy/rpython/memory/gc/base.py
@@ -1,4 +1,5 @@
 from pypy.rpython.lltypesystem import lltype, llmemory, llarena, rffi
+from pypy.rpython.lltypesystem.lloperation import llop
 from pypy.rlib.debug import ll_assert
 from pypy.rpython.memory.gcheader import GCHeaderBuilder
 from pypy.rpython.memory.support import DEFAULT_CHUNK_SIZE
@@ -357,7 +358,8 @@
         typeid = self.get_type_id(addr)
         raw_adr = (addr + self.ofs_to_raw_mem_ptr(typeid)).address[0]
         if raw_adr:
-            llmemory.raw_free(raw_adr, track_free=True)
+            llop.track_alloc_stop(lltype.Void, raw_adr)
+            llmemory.raw_free(raw_adr)
 
 
 class MovingGCBase(GCBase):
diff --git a/pypy/rpython/rbuiltin.py b/pypy/rpython/rbuiltin.py
--- a/pypy/rpython/rbuiltin.py
+++ b/pypy/rpython/rbuiltin.py
@@ -565,11 +565,11 @@
     hop.exception_cannot_occur()
     return hop.genop('raw_malloc_usage', [v_size], resulttype=lltype.Signed)
 
-def rtype_raw_free(hop, i_track_free=None):
+def rtype_raw_free(hop):
     s_addr = hop.args_s[0]
     if s_addr.is_null_address():
         raise TyperError("raw_free(x) where x is the constant NULL")
-    v_addr = hop.inputarg(llmemory.Address, 0)
+    v_addr, = hop.inputargs(llmemory.Address)
     hop.exception_cannot_occur()
     return hop.genop('raw_free', [v_addr])
 

From noreply at buildbot.pypy.org  Thu Oct  6 04:05:37 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Thu,  6 Oct 2011 04:05:37 +0200 (CEST)
Subject: [pypy-commit] pypy default: hopefully fix annenforceargs
Message-ID: <20111006020537.10FC682A02@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r47839:9af0a17b33a8
Date: 2011-10-06 04:04 +0200
http://bitbucket.org/pypy/pypy/changeset/9af0a17b33a8/

Log:	hopefully fix annenforceargs

diff --git a/pypy/rpython/lltypesystem/rstr.py b/pypy/rpython/lltypesystem/rstr.py
--- a/pypy/rpython/lltypesystem/rstr.py
+++ b/pypy/rpython/lltypesystem/rstr.py
@@ -694,8 +694,8 @@
             return -1
         return count
 
+    @enforceargs(int, None)
     @jit.look_inside_iff(lambda length, items: jit.isconstant(length) and length <= 2)
-    @enforceargs(int, None)
     def ll_join_strs(length, items):
         # Special case for length 1 items, helps both the JIT and other code
         if length == 1:

From noreply at buildbot.pypy.org  Thu Oct  6 04:05:38 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Thu,  6 Oct 2011 04:05:38 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge default
Message-ID: <20111006020538.44B7B82A02@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r47840:f4baf68db210
Date: 2011-10-06 04:05 +0200
http://bitbucket.org/pypy/pypy/changeset/f4baf68db210/

Log:	merge default

diff --git a/pypy/module/__pypy__/interp_builders.py b/pypy/module/__pypy__/interp_builders.py
--- a/pypy/module/__pypy__/interp_builders.py
+++ b/pypy/module/__pypy__/interp_builders.py
@@ -40,6 +40,12 @@
             self.builder = None
             return w_s
 
+        def descr_len(self, space):
+            if self.builder is None:
+                raise OperationError(space.w_ValueError,
+                                     space.wrap('no lenght of built builder'))
+            return space.wrap(self.builder.getlength())
+
     W_Builder.__name__ = "W_%s" % name
     W_Builder.typedef = TypeDef(name,
         __new__ = interp2app(func_with_new_name(
@@ -48,6 +54,7 @@
         append = interp2app(W_Builder.descr_append),
         append_slice = interp2app(W_Builder.descr_append_slice),
         build = interp2app(W_Builder.descr_build),
+        __len__ = interp2app(W_Builder.descr_len),
     )
     W_Builder.typedef.acceptable_as_base_class = False
     return W_Builder
diff --git a/pypy/module/__pypy__/test/test_builders.py b/pypy/module/__pypy__/test/test_builders.py
--- a/pypy/module/__pypy__/test/test_builders.py
+++ b/pypy/module/__pypy__/test/test_builders.py
@@ -38,7 +38,9 @@
         b = StringBuilder()
         b.append("abc")
         b.append("123")
+        assert len(b) == 6
         b.append("you and me")
         s = b.build()
+        raises(ValueError, len, b)
         assert s == "abc123you and me"
-        raises(ValueError, b.build)
\ No newline at end of file
+        raises(ValueError, b.build)

From noreply at buildbot.pypy.org  Thu Oct  6 06:31:34 2011
From: noreply at buildbot.pypy.org (justinpeel)
Date: Thu,  6 Oct 2011 06:31:34 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-indexing-by-arrays: merge in default
Message-ID: <20111006043134.76E8082A02@wyvern.cs.uni-duesseldorf.de>

Author: Justin Peel <notmuchtotell at gmail.com>
Branch: numpy-indexing-by-arrays
Changeset: r47841:12c31a30642d
Date: 2011-10-05 22:31 -0600
http://bitbucket.org/pypy/pypy/changeset/12c31a30642d/

Log:	merge in default

diff --git a/dotviewer/graphparse.py b/dotviewer/graphparse.py
--- a/dotviewer/graphparse.py
+++ b/dotviewer/graphparse.py
@@ -36,48 +36,45 @@
     print >> sys.stderr, "Warning: could not guess file type, using 'dot'"
     return 'unknown'
 
-def dot2plain(content, contenttype, use_codespeak=False):
-    if contenttype == 'plain':
-        # already a .plain file
-        return content
+def dot2plain_graphviz(content, contenttype, use_codespeak=False):
+    if contenttype != 'neato':
+        cmdline = 'dot -Tplain'
+    else:
+        cmdline = 'neato -Tplain'
+    #print >> sys.stderr, '* running:', cmdline
+    close_fds = sys.platform != 'win32'
+    p = subprocess.Popen(cmdline, shell=True, close_fds=close_fds,
+                         stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+    (child_in, child_out) = (p.stdin, p.stdout)
+    try:
+        import thread
+    except ImportError:
+        bkgndwrite(child_in, content)
+    else:
+        thread.start_new_thread(bkgndwrite, (child_in, content))
+    plaincontent = child_out.read()
+    child_out.close()
+    if not plaincontent:    # 'dot' is likely not installed
+        raise PlainParseError("no result from running 'dot'")
+    return plaincontent
 
-    if not use_codespeak:
-        if contenttype != 'neato':
-            cmdline = 'dot -Tplain'
-        else:
-            cmdline = 'neato -Tplain'
-        #print >> sys.stderr, '* running:', cmdline
-        close_fds = sys.platform != 'win32'
-        p = subprocess.Popen(cmdline, shell=True, close_fds=close_fds,
-                             stdin=subprocess.PIPE, stdout=subprocess.PIPE)
-        (child_in, child_out) = (p.stdin, p.stdout)
-        try:
-            import thread
-        except ImportError:
-            bkgndwrite(child_in, content)
-        else:
-            thread.start_new_thread(bkgndwrite, (child_in, content))
-        plaincontent = child_out.read()
-        child_out.close()
-        if not plaincontent:    # 'dot' is likely not installed
-            raise PlainParseError("no result from running 'dot'")
-    else:
-        import urllib
-        request = urllib.urlencode({'dot': content})
-        url = 'http://codespeak.net/pypy/convertdot.cgi'
-        print >> sys.stderr, '* posting:', url
-        g = urllib.urlopen(url, data=request)
-        result = []
-        while True:
-            data = g.read(16384)
-            if not data:
-                break
-            result.append(data)
-        g.close()
-        plaincontent = ''.join(result)
-        # very simple-minded way to give a somewhat better error message
-        if plaincontent.startswith('<body'):
-            raise Exception("the dot on codespeak has very likely crashed")
+def dot2plain_codespeak(content, contenttype):
+    import urllib
+    request = urllib.urlencode({'dot': content})
+    url = 'http://codespeak.net/pypy/convertdot.cgi'
+    print >> sys.stderr, '* posting:', url
+    g = urllib.urlopen(url, data=request)
+    result = []
+    while True:
+        data = g.read(16384)
+        if not data:
+            break
+        result.append(data)
+    g.close()
+    plaincontent = ''.join(result)
+    # very simple-minded way to give a somewhat better error message
+    if plaincontent.startswith('<body'):
+        raise Exception("the dot on codespeak has very likely crashed")
     return plaincontent
 
 def bkgndwrite(f, data):
@@ -148,10 +145,13 @@
 
 def parse_dot(graph_id, content, links={}, fixedfont=False):
     contenttype = guess_type(content)
-    try:
-        plaincontent = dot2plain(content, contenttype, use_codespeak=False)
-        return list(parse_plain(graph_id, plaincontent, links, fixedfont))
-    except PlainParseError:
-        # failed, retry via codespeak
-        plaincontent = dot2plain(content, contenttype, use_codespeak=True)
-        return list(parse_plain(graph_id, plaincontent, links, fixedfont))
+    if contenttype == 'plain':
+        plaincontent = content
+    else:
+        try:
+            plaincontent = dot2plain_graphviz(content, contenttype)
+        except PlainParseError, e:
+            print e
+            # failed, retry via codespeak
+            plaincontent = dot2plain_codespeak(content, contenttype)
+    return list(parse_plain(graph_id, plaincontent, links, fixedfont))
diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -317,7 +317,7 @@
     RegrTest('test_multibytecodec.py', usemodules='_multibytecodec'),
     RegrTest('test_multibytecodec_support.py', skip="not a test"),
     RegrTest('test_multifile.py'),
-    RegrTest('test_multiprocessing.py', skip='FIXME leaves subprocesses'),
+    RegrTest('test_multiprocessing.py', skip="FIXME leaves subprocesses"),
     RegrTest('test_mutants.py', core="possibly"),
     RegrTest('test_mutex.py'),
     RegrTest('test_netrc.py'),
diff --git a/lib-python/modified-2.7/gzip.py b/lib-python/modified-2.7/gzip.py
deleted file mode 100644
--- a/lib-python/modified-2.7/gzip.py
+++ /dev/null
@@ -1,514 +0,0 @@
-"""Functions that read and write gzipped files.
-
-The user of the file doesn't have to worry about the compression,
-but random access is not allowed."""
-
-# based on Andrew Kuchling's minigzip.py distributed with the zlib module
-
-import struct, sys, time, os
-import zlib
-import io
-import __builtin__
-
-__all__ = ["GzipFile","open"]
-
-FTEXT, FHCRC, FEXTRA, FNAME, FCOMMENT = 1, 2, 4, 8, 16
-
-READ, WRITE = 1, 2
-
-def write32u(output, value):
-    # The L format writes the bit pattern correctly whether signed
-    # or unsigned.
-    output.write(struct.pack("<L", value))
-
-def read32(input):
-    return struct.unpack("<I", input.read(4))[0]
-
-def open(filename, mode="rb", compresslevel=9):
-    """Shorthand for GzipFile(filename, mode, compresslevel).
-
-    The filename argument is required; mode defaults to 'rb'
-    and compresslevel defaults to 9.
-
-    """
-    return GzipFile(filename, mode, compresslevel)
-
-class GzipFile(io.BufferedIOBase):
-    """The GzipFile class simulates most of the methods of a file object with
-    the exception of the readinto() and truncate() methods.
-
-    """
-
-    myfileobj = None
-    max_read_chunk = 10 * 1024 * 1024   # 10Mb
-
-    def __init__(self, filename=None, mode=None,
-                 compresslevel=9, fileobj=None, mtime=None):
-        """Constructor for the GzipFile class.
-
-        At least one of fileobj and filename must be given a
-        non-trivial value.
-
-        The new class instance is based on fileobj, which can be a regular
-        file, a StringIO object, or any other object which simulates a file.
-        It defaults to None, in which case filename is opened to provide
-        a file object.
-
-        When fileobj is not None, the filename argument is only used to be
-        included in the gzip file header, which may includes the original
-        filename of the uncompressed file.  It defaults to the filename of
-        fileobj, if discernible; otherwise, it defaults to the empty string,
-        and in this case the original filename is not included in the header.
-
-        The mode argument can be any of 'r', 'rb', 'a', 'ab', 'w', or 'wb',
-        depending on whether the file will be read or written.  The default
-        is the mode of fileobj if discernible; otherwise, the default is 'rb'.
-        Be aware that only the 'rb', 'ab', and 'wb' values should be used
-        for cross-platform portability.
-
-        The compresslevel argument is an integer from 1 to 9 controlling the
-        level of compression; 1 is fastest and produces the least compression,
-        and 9 is slowest and produces the most compression.  The default is 9.
-
-        The mtime argument is an optional numeric timestamp to be written
-        to the stream when compressing.  All gzip compressed streams
-        are required to contain a timestamp.  If omitted or None, the
-        current time is used.  This module ignores the timestamp when
-        decompressing; however, some programs, such as gunzip, make use
-        of it.  The format of the timestamp is the same as that of the
-        return value of time.time() and of the st_mtime member of the
-        object returned by os.stat().
-
-        """
-
-        # guarantee the file is opened in binary mode on platforms
-        # that care about that sort of thing
-        if mode and 'b' not in mode:
-            mode += 'b'
-        if fileobj is None:
-            fileobj = self.myfileobj = __builtin__.open(filename, mode or 'rb')
-        if filename is None:
-            if hasattr(fileobj, 'name'): filename = fileobj.name
-            else: filename = ''
-        if mode is None:
-            if hasattr(fileobj, 'mode'): mode = fileobj.mode
-            else: mode = 'rb'
-
-        if mode[0:1] == 'r':
-            self.mode = READ
-            # Set flag indicating start of a new member
-            self._new_member = True
-            # Buffer data read from gzip file. extrastart is offset in
-            # stream where buffer starts. extrasize is number of
-            # bytes remaining in buffer from current stream position.
-            self.extrabuf = ""
-            self.extrasize = 0
-            self.extrastart = 0
-            self.name = filename
-            # Starts small, scales exponentially
-            self.min_readsize = 100
-
-        elif mode[0:1] == 'w' or mode[0:1] == 'a':
-            self.mode = WRITE
-            self._init_write(filename)
-            self.compress = zlib.compressobj(compresslevel,
-                                             zlib.DEFLATED,
-                                             -zlib.MAX_WBITS,
-                                             zlib.DEF_MEM_LEVEL,
-                                             0)
-        else:
-            raise IOError, "Mode " + mode + " not supported"
-
-        self.fileobj = fileobj
-        self.offset = 0
-        self.mtime = mtime
-
-        if self.mode == WRITE:
-            self._write_gzip_header()
-
-    @property
-    def filename(self):
-        import warnings
-        warnings.warn("use the name attribute", DeprecationWarning, 2)
-        if self.mode == WRITE and self.name[-3:] != ".gz":
-            return self.name + ".gz"
-        return self.name
-
-    def __repr__(self):
-        s = repr(self.fileobj)
-        return '<gzip ' + s[1:-1] + ' ' + hex(id(self)) + '>'
-
-    def _check_closed(self):
-        """Raises a ValueError if the underlying file object has been closed.
-
-        """
-        if self.closed:
-            raise ValueError('I/O operation on closed file.')
-
-    def _init_write(self, filename):
-        self.name = filename
-        self.crc = zlib.crc32("") & 0xffffffffL
-        self.size = 0
-        self.writebuf = []
-        self.bufsize = 0
-
-    def _write_gzip_header(self):
-        self.fileobj.write('\037\213')             # magic header
-        self.fileobj.write('\010')                 # compression method
-        fname = os.path.basename(self.name)
-        if fname.endswith(".gz"):
-            fname = fname[:-3]
-        flags = 0
-        if fname:
-            flags = FNAME
-        self.fileobj.write(chr(flags))
-        mtime = self.mtime
-        if mtime is None:
-            mtime = time.time()
-        write32u(self.fileobj, long(mtime))
-        self.fileobj.write('\002')
-        self.fileobj.write('\377')
-        if fname:
-            self.fileobj.write(fname + '\000')
-
-    def _init_read(self):
-        self.crc = zlib.crc32("") & 0xffffffffL
-        self.size = 0
-
-    def _read_gzip_header(self):
-        magic = self.fileobj.read(2)
-        if magic != '\037\213':
-            raise IOError, 'Not a gzipped file'
-        method = ord( self.fileobj.read(1) )
-        if method != 8:
-            raise IOError, 'Unknown compression method'
-        flag = ord( self.fileobj.read(1) )
-        self.mtime = read32(self.fileobj)
-        # extraflag = self.fileobj.read(1)
-        # os = self.fileobj.read(1)
-        self.fileobj.read(2)
-
-        if flag & FEXTRA:
-            # Read & discard the extra field, if present
-            xlen = ord(self.fileobj.read(1))
-            xlen = xlen + 256*ord(self.fileobj.read(1))
-            self.fileobj.read(xlen)
-        if flag & FNAME:
-            # Read and discard a null-terminated string containing the filename
-            while True:
-                s = self.fileobj.read(1)
-                if not s or s=='\000':
-                    break
-        if flag & FCOMMENT:
-            # Read and discard a null-terminated string containing a comment
-            while True:
-                s = self.fileobj.read(1)
-                if not s or s=='\000':
-                    break
-        if flag & FHCRC:
-            self.fileobj.read(2)     # Read & discard the 16-bit header CRC
-
-    def write(self,data):
-        self._check_closed()
-        if self.mode != WRITE:
-            import errno
-            raise IOError(errno.EBADF, "write() on read-only GzipFile object")
-
-        if self.fileobj is None:
-            raise ValueError, "write() on closed GzipFile object"
-
-        # Convert data type if called by io.BufferedWriter.
-        if isinstance(data, memoryview):
-            data = data.tobytes()
-
-        if len(data) > 0:
-            self.size = self.size + len(data)
-            self.crc = zlib.crc32(data, self.crc) & 0xffffffffL
-            self.fileobj.write( self.compress.compress(data) )
-            self.offset += len(data)
-
-        return len(data)
-
-    def read(self, size=-1):
-        self._check_closed()
-        if self.mode != READ:
-            import errno
-            raise IOError(errno.EBADF, "read() on write-only GzipFile object")
-
-        if self.extrasize <= 0 and self.fileobj is None:
-            return ''
-
-        readsize = 1024
-        if size < 0:        # get the whole thing
-            try:
-                while True:
-                    self._read(readsize)
-                    readsize = min(self.max_read_chunk, readsize * 2)
-            except EOFError:
-                size = self.extrasize
-        elif size == 0:
-            return ""
-        else:               # just get some more of it
-            try:
-                while size > self.extrasize:
-                    self._read(readsize)
-                    readsize = min(self.max_read_chunk, readsize * 2)
-            except EOFError:
-                if size > self.extrasize:
-                    size = self.extrasize
-
-        offset = self.offset - self.extrastart
-        chunk = self.extrabuf[offset: offset + size]
-        self.extrasize = self.extrasize - size
-
-        self.offset += size
-        return chunk
-
-    def _unread(self, buf):
-        self.extrasize = len(buf) + self.extrasize
-        self.offset -= len(buf)
-
-    def _read(self, size=1024):
-        if self.fileobj is None:
-            raise EOFError, "Reached EOF"
-
-        if self._new_member:
-            # If the _new_member flag is set, we have to
-            # jump to the next member, if there is one.
-            #
-            # First, check if we're at the end of the file;
-            # if so, it's time to stop; no more members to read.
-            pos = self.fileobj.tell()   # Save current position
-            self.fileobj.seek(0, 2)     # Seek to end of file
-            if pos == self.fileobj.tell():
-                raise EOFError, "Reached EOF"
-            else:
-                self.fileobj.seek( pos ) # Return to original position
-
-            self._init_read()
-            self._read_gzip_header()
-            self.decompress = zlib.decompressobj(-zlib.MAX_WBITS)
-            self._new_member = False
-
-        # Read a chunk of data from the file
-        buf = self.fileobj.read(size)
-
-        # If the EOF has been reached, flush the decompression object
-        # and mark this object as finished.
-
-        if buf == "":
-            uncompress = self.decompress.flush()
-            self._read_eof()
-            self._add_read_data( uncompress )
-            raise EOFError, 'Reached EOF'
-
-        uncompress = self.decompress.decompress(buf)
-        self._add_read_data( uncompress )
-
-        if self.decompress.unused_data != "":
-            # Ending case: we've come to the end of a member in the file,
-            # so seek back to the start of the unused data, finish up
-            # this member, and read a new gzip header.
-            # (The number of bytes to seek back is the length of the unused
-            # data, minus 8 because _read_eof() will rewind a further 8 bytes)
-            self.fileobj.seek( -len(self.decompress.unused_data)+8, 1)
-
-            # Check the CRC and file size, and set the flag so we read
-            # a new member on the next call
-            self._read_eof()
-            self._new_member = True
-
-    def _add_read_data(self, data):
-        self.crc = zlib.crc32(data, self.crc) & 0xffffffffL
-        offset = self.offset - self.extrastart
-        self.extrabuf = self.extrabuf[offset:] + data
-        self.extrasize = self.extrasize + len(data)
-        self.extrastart = self.offset
-        self.size = self.size + len(data)
-
-    def _read_eof(self):
-        # We've read to the end of the file, so we have to rewind in order
-        # to reread the 8 bytes containing the CRC and the file size.
-        # We check the that the computed CRC and size of the
-        # uncompressed data matches the stored values.  Note that the size
-        # stored is the true file size mod 2**32.
-        self.fileobj.seek(-8, 1)
-        crc32 = read32(self.fileobj)
-        isize = read32(self.fileobj)  # may exceed 2GB
-        if crc32 != self.crc:
-            raise IOError("CRC check failed %s != %s" % (hex(crc32),
-                                                         hex(self.crc)))
-        elif isize != (self.size & 0xffffffffL):
-            raise IOError, "Incorrect length of data produced"
-
-        # Gzip files can be padded with zeroes and still have archives.
-        # Consume all zero bytes and set the file position to the first
-        # non-zero byte. See http://www.gzip.org/#faq8
-        c = "\x00"
-        while c == "\x00":
-            c = self.fileobj.read(1)
-        if c:
-            self.fileobj.seek(-1, 1)
-
-    @property
-    def closed(self):
-        return self.fileobj is None
-
-    def close(self):
-        if self.fileobj is None:
-            return
-        if self.mode == WRITE:
-            self.fileobj.write(self.compress.flush())
-            write32u(self.fileobj, self.crc)
-            # self.size may exceed 2GB, or even 4GB
-            write32u(self.fileobj, self.size & 0xffffffffL)
-            self.fileobj = None
-        elif self.mode == READ:
-            self.fileobj = None
-        if self.myfileobj:
-            self.myfileobj.close()
-            self.myfileobj = None
-
-    def flush(self,zlib_mode=zlib.Z_SYNC_FLUSH):
-        self._check_closed()
-        if self.mode == WRITE:
-            # Ensure the compressor's buffer is flushed
-            self.fileobj.write(self.compress.flush(zlib_mode))
-            self.fileobj.flush()
-
-    def fileno(self):
-        """Invoke the underlying file object's fileno() method.
-
-        This will raise AttributeError if the underlying file object
-        doesn't support fileno().
-        """
-        return self.fileobj.fileno()
-
-    def rewind(self):
-        '''Return the uncompressed stream file position indicator to the
-        beginning of the file'''
-        if self.mode != READ:
-            raise IOError("Can't rewind in write mode")
-        self.fileobj.seek(0)
-        self._new_member = True
-        self.extrabuf = ""
-        self.extrasize = 0
-        self.extrastart = 0
-        self.offset = 0
-
-    def readable(self):
-        return self.mode == READ
-
-    def writable(self):
-        return self.mode == WRITE
-
-    def seekable(self):
-        return True
-
-    def seek(self, offset, whence=0):
-        if whence:
-            if whence == 1:
-                offset = self.offset + offset
-            else:
-                raise ValueError('Seek from end not supported')
-        if self.mode == WRITE:
-            if offset < self.offset:
-                raise IOError('Negative seek in write mode')
-            count = offset - self.offset
-            for i in range(count // 1024):
-                self.write(1024 * '\0')
-            self.write((count % 1024) * '\0')
-        elif self.mode == READ:
-            if offset == self.offset:
-                self.read(0) # to make sure that this file is open
-                return self.offset
-            if offset < self.offset:
-                # for negative seek, rewind and do positive seek
-                self.rewind()
-            count = offset - self.offset
-            for i in range(count // 1024):
-                self.read(1024)
-            self.read(count % 1024)
-
-        return self.offset
-
-    def readline(self, size=-1):
-        if size < 0:
-            # Shortcut common case - newline found in buffer.
-            offset = self.offset - self.extrastart
-            i = self.extrabuf.find('\n', offset) + 1
-            if i > 0:
-                self.extrasize -= i - offset
-                self.offset += i - offset
-                return self.extrabuf[offset: i]
-
-            size = sys.maxint
-            readsize = self.min_readsize
-        else:
-            readsize = size
-        bufs = []
-        while size != 0:
-            c = self.read(readsize)
-            i = c.find('\n')
-
-            # We set i=size to break out of the loop under two
-            # conditions: 1) there's no newline, and the chunk is
-            # larger than size, or 2) there is a newline, but the
-            # resulting line would be longer than 'size'.
-            if (size <= i) or (i == -1 and len(c) > size):
-                i = size - 1
-
-            if i >= 0 or c == '':
-                bufs.append(c[:i + 1])    # Add portion of last chunk
-                self._unread(c[i + 1:])   # Push back rest of chunk
-                break
-
-            # Append chunk to list, decrease 'size',
-            bufs.append(c)
-            size = size - len(c)
-            readsize = min(size, readsize * 2)
-        if readsize > self.min_readsize:
-            self.min_readsize = min(readsize, self.min_readsize * 2, 512)
-        return ''.join(bufs) # Return resulting line
-
-
-def _test():
-    # Act like gzip; with -d, act like gunzip.
-    # The input file is not deleted, however, nor are any other gzip
-    # options or features supported.
-    args = sys.argv[1:]
-    decompress = args and args[0] == "-d"
-    if decompress:
-        args = args[1:]
-    if not args:
-        args = ["-"]
-    for arg in args:
-        if decompress:
-            if arg == "-":
-                f = GzipFile(filename="", mode="rb", fileobj=sys.stdin)
-                g = sys.stdout
-            else:
-                if arg[-3:] != ".gz":
-                    print "filename doesn't end in .gz:", repr(arg)
-                    continue
-                f = open(arg, "rb")
-                g = __builtin__.open(arg[:-3], "wb")
-        else:
-            if arg == "-":
-                f = sys.stdin
-                g = GzipFile(filename="", mode="wb", fileobj=sys.stdout)
-            else:
-                f = __builtin__.open(arg, "rb")
-                g = open(arg + ".gz", "wb")
-        while True:
-            chunk = f.read(1024)
-            if not chunk:
-                break
-            g.write(chunk)
-        if g is not sys.stdout:
-            g.close()
-        if f is not sys.stdin:
-            f.close()
-
-if __name__ == '__main__':
-    _test()
diff --git a/lib-python/modified-2.7/tarfile.py b/lib-python/modified-2.7/tarfile.py
--- a/lib-python/modified-2.7/tarfile.py
+++ b/lib-python/modified-2.7/tarfile.py
@@ -252,8 +252,8 @@
        the high bit set. So we calculate two checksums, unsigned and
        signed.
     """
-    unsigned_chksum = 256 + sum(struct.unpack("148B8x356B", buf[:512]))
-    signed_chksum = 256 + sum(struct.unpack("148b8x356b", buf[:512]))
+    unsigned_chksum = 256 + sum(struct.unpack("148B", buf[:148]) + struct.unpack("356B", buf[156:512]))
+    signed_chksum = 256 + sum(struct.unpack("148b", buf[:148]) + struct.unpack("356b", buf[156:512]))
     return unsigned_chksum, signed_chksum
 
 def copyfileobj(src, dst, length=None):
@@ -265,6 +265,7 @@
     if length is None:
         shutil.copyfileobj(src, dst)
         return
+
     BUFSIZE = 16 * 1024
     blocks, remainder = divmod(length, BUFSIZE)
     for b in xrange(blocks):
@@ -801,19 +802,19 @@
         if self.closed:
             raise ValueError("I/O operation on closed file")
 
+        buf = ""
         if self.buffer:
             if size is None:
-                buf = self.buffer + self.fileobj.read()
+                buf = self.buffer
                 self.buffer = ""
             else:
                 buf = self.buffer[:size]
                 self.buffer = self.buffer[size:]
-                buf += self.fileobj.read(size - len(buf))
+
+        if size is None:
+            buf += self.fileobj.read()
         else:
-            if size is None:
-                buf = self.fileobj.read()
-            else:
-                buf = self.fileobj.read(size)
+            buf += self.fileobj.read(size - len(buf))
 
         self.position += len(buf)
         return buf
diff --git a/lib-python/modified-2.7/test/test_multiprocessing.py b/lib-python/modified-2.7/test/test_multiprocessing.py
--- a/lib-python/modified-2.7/test/test_multiprocessing.py
+++ b/lib-python/modified-2.7/test/test_multiprocessing.py
@@ -510,7 +510,6 @@
 
         p.join()
 
-    @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
     def test_qsize(self):
         q = self.Queue()
         try:
@@ -532,7 +531,6 @@
             time.sleep(DELTA)
             q.task_done()
 
-    @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
     def test_task_done(self):
         queue = self.JoinableQueue()
 
@@ -1091,7 +1089,6 @@
 class _TestPoolWorkerLifetime(BaseTestCase):
 
     ALLOWED_TYPES = ('processes', )
-    @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
     def test_pool_worker_lifetime(self):
         p = multiprocessing.Pool(3, maxtasksperchild=10)
         self.assertEqual(3, len(p._pool))
@@ -1280,7 +1277,6 @@
         queue = manager.get_queue()
         queue.put('hello world')
 
-    @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
     def test_rapid_restart(self):
         authkey = os.urandom(32)
         manager = QueueManager(
@@ -1297,6 +1293,7 @@
         queue = manager.get_queue()
         self.assertEqual(queue.get(), 'hello world')
         del queue
+        test_support.gc_collect()
         manager.shutdown()
         manager = QueueManager(
             address=addr, authkey=authkey, serializer=SERIALIZER)
@@ -1573,7 +1570,6 @@
 
     ALLOWED_TYPES = ('processes',)
 
-    @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
     def test_heap(self):
         iterations = 5000
         maxblocks = 50
diff --git a/lib-python/modified-2.7/test/test_sys_settrace.py b/lib-python/modified-2.7/test/test_sys_settrace.py
--- a/lib-python/modified-2.7/test/test_sys_settrace.py
+++ b/lib-python/modified-2.7/test/test_sys_settrace.py
@@ -286,11 +286,11 @@
         self.compare_events(func.func_code.co_firstlineno,
                             tracer.events, func.events)
 
-    def set_and_retrieve_none(self):
+    def test_set_and_retrieve_none(self):
         sys.settrace(None)
         assert sys.gettrace() is None
 
-    def set_and_retrieve_func(self):
+    def test_set_and_retrieve_func(self):
         def fn(*args):
             pass
 
diff --git a/lib_pypy/_functools.py b/lib_pypy/_functools.py
--- a/lib_pypy/_functools.py
+++ b/lib_pypy/_functools.py
@@ -18,5 +18,5 @@
 
     def __call__(self, *fargs, **fkeywords):
         if self.keywords is not None:
-            fkeywords.update(self.keywords)
+            fkeywords = dict(self.keywords, **fkeywords)
         return self.func(*(self.args + fargs), **fkeywords)
diff --git a/lib_pypy/greenlet.py b/lib_pypy/greenlet.py
--- a/lib_pypy/greenlet.py
+++ b/lib_pypy/greenlet.py
@@ -96,7 +96,16 @@
 
     @property
     def gr_frame(self):
-        raise NotImplementedError("attribute 'gr_frame' of greenlet objects")
+        # xxx this doesn't work when called on either the current or
+        # the main greenlet of another thread
+        if self is getcurrent():
+            return None
+        if self.__main:
+            self = getcurrent()
+        f = _continulet.__reduce__(self)[2][0]
+        if not f:
+            return None
+        return f.f_back.f_back.f_back   # go past start(), __switch(), switch()
 
 # ____________________________________________________________
 # Internal stuff
diff --git a/lib_pypy/pypy_test/test_stackless_pickling.py b/lib_pypy/pypy_test/test_stackless_pickling.py
--- a/lib_pypy/pypy_test/test_stackless_pickling.py
+++ b/lib_pypy/pypy_test/test_stackless_pickling.py
@@ -1,7 +1,3 @@
-"""
-this test should probably not run from CPython or py.py.
-I'm not entirely sure, how to do that.
-"""
 from __future__ import absolute_import
 from py.test import skip
 try:
@@ -16,11 +12,15 @@
 
 class Test_StacklessPickling:
 
+    def test_pickle_main_coroutine(self):
+        import stackless, pickle
+        s = pickle.dumps(stackless.coroutine.getcurrent())
+        print s
+        c = pickle.loads(s)
+        assert c is stackless.coroutine.getcurrent()
+
     def test_basic_tasklet_pickling(self):
-        try:
-            import stackless
-        except ImportError:
-            skip("can't load stackless and don't know why!!!")
+        import stackless
         from stackless import run, schedule, tasklet
         import pickle
 
diff --git a/lib_pypy/pyrepl/completing_reader.py b/lib_pypy/pyrepl/completing_reader.py
--- a/lib_pypy/pyrepl/completing_reader.py
+++ b/lib_pypy/pyrepl/completing_reader.py
@@ -229,7 +229,8 @@
 
     def after_command(self, cmd):
         super(CompletingReader, self).after_command(cmd)
-        if not isinstance(cmd, complete) and not isinstance(cmd, self_insert):
+        if not isinstance(cmd, self.commands['complete']) \
+           and not isinstance(cmd, self.commands['self_insert']):
             self.cmpltn_reset()
 
     def calc_screen(self):
diff --git a/lib_pypy/stackless.py b/lib_pypy/stackless.py
--- a/lib_pypy/stackless.py
+++ b/lib_pypy/stackless.py
@@ -5,7 +5,6 @@
 """
 
 
-import traceback
 import _continuation
 
 class TaskletExit(Exception):
@@ -14,33 +13,46 @@
 CoroutineExit = TaskletExit
 
 
+def _coroutine_getcurrent():
+    "Returns the current coroutine (i.e. the one which called this function)."
+    try:
+        return _tls.current_coroutine
+    except AttributeError:
+        # first call in this thread: current == main
+        return _coroutine_getmain()
+
+def _coroutine_getmain():
+    try:
+        return _tls.main_coroutine
+    except AttributeError:
+        # create the main coroutine for this thread
+        continulet = _continuation.continulet
+        main = coroutine()
+        main._frame = continulet.__new__(continulet)
+        main._is_started = -1
+        _tls.current_coroutine = _tls.main_coroutine = main
+        return _tls.main_coroutine
+
+
 class coroutine(object):
-    "we can't have continulet as a base, because continulets can't be rebound"
+    _is_started = 0      # 0=no, 1=yes, -1=main
 
     def __init__(self):
         self._frame = None
-        self.is_zombie = False
-
-    def __getattr__(self, attr):
-        return getattr(self._frame, attr)
-
-    def __del__(self):
-        self.is_zombie = True
-        del self._frame
-        self._frame = None
 
     def bind(self, func, *argl, **argd):
         """coro.bind(f, *argl, **argd) -> None.
            binds function f to coro. f will be called with
            arguments *argl, **argd
         """
-        if self._frame is None or not self._frame.is_pending():
-            def run(c):
-                _tls.current_coroutine = self
-                return func(*argl, **argd)
-            self._frame = frame = _continuation.continulet(run)
-        else:
+        if self.is_alive:
             raise ValueError("cannot bind a bound coroutine")
+        def run(c):
+            _tls.current_coroutine = self
+            self._is_started = 1
+            return func(*argl, **argd)
+        self._is_started = 0
+        self._frame = _continuation.continulet(run)
 
     def switch(self):
         """coro.switch() -> returnvalue
@@ -48,7 +60,7 @@
            f finishes, the returnvalue is that of f, otherwise
            None is returned
         """
-        current = _getcurrent()
+        current = _coroutine_getcurrent()
         try:
             current._frame.switch(to=self._frame)
         finally:
@@ -56,37 +68,30 @@
 
     def kill(self):
         """coro.kill() : kill coroutine coro"""
-        current = _getcurrent()
+        current = _coroutine_getcurrent()
         try:
             current._frame.throw(CoroutineExit, to=self._frame)
         finally:
             _tls.current_coroutine = current
 
-    def _is_alive(self):
-        if self._frame is None:
-            return False
-        return not self._frame.is_pending()
-    is_alive = property(_is_alive)
-    del _is_alive
+    @property
+    def is_alive(self):
+        return self._is_started < 0 or (
+            self._frame is not None and self._frame.is_pending())
 
-    def getcurrent():
-        """coroutine.getcurrent() -> the currently running coroutine"""
-        return _getcurrent()
-    getcurrent = staticmethod(getcurrent)
+    @property
+    def is_zombie(self):
+        return self._is_started > 0 and not self._frame.is_pending()
+
+    getcurrent = staticmethod(_coroutine_getcurrent)
 
     def __reduce__(self):
-        raise TypeError, 'pickling is not possible based upon continulets'
+        if self._is_started < 0:
+            return _coroutine_getmain, ()
+        else:
+            return type(self), (), self.__dict__
 
 
-def _getcurrent():
-    "Returns the current coroutine (i.e. the one which called this function)."
-    try:
-        return _tls.current_coroutine
-    except AttributeError:
-        # first call in this thread: current == main
-        _coroutine_create_main()
-        return _tls.current_coroutine
-
 try:
     from thread import _local
 except ImportError:
@@ -95,14 +100,8 @@
 
 _tls = _local()
 
-def _coroutine_create_main():
-    # create the main coroutine for this thread
-    _tls.current_coroutine = None
-    main_coroutine = coroutine()
-    typ = _continuation.continulet
-    main_coroutine._frame = typ.__new__(typ)
-    _tls.main_coroutine = main_coroutine
-    _tls.current_coroutine = main_coroutine
+
+# ____________________________________________________________
 
 
 from collections import deque
@@ -148,10 +147,7 @@
     _last_task = next
     assert not next.blocked
     if next is not current:
-        #try:
-            next.switch()
-        #except CoroutineExit:  --- they are the same anyway
-        #    raise TaskletExit
+        next.switch()
     return current
 
 def set_schedule_callback(callback):
@@ -175,34 +171,6 @@
         raise self.type, self.value, self.traceback
 
 #
-# helpers for pickling
-#
-
-_stackless_primitive_registry = {}
-
-def register_stackless_primitive(thang, retval_expr='None'):
-    import types
-    func = thang
-    if isinstance(thang, types.MethodType):
-        func = thang.im_func
-    code = func.func_code
-    _stackless_primitive_registry[code] = retval_expr
-    # It is not too nice to attach info via the code object, but
-    # I can't think of a better solution without a real transform.
-
-def rewrite_stackless_primitive(coro_state, alive, tempval):
-    flags, frame, thunk, parent = coro_state
-    while frame is not None:
-        retval_expr = _stackless_primitive_registry.get(frame.f_code)
-        if retval_expr:
-            # this tasklet needs to stop pickling here and return its value.
-            tempval = eval(retval_expr, globals(), frame.f_locals)
-            coro_state = flags, frame, thunk, parent
-            break
-        frame = frame.f_back
-    return coro_state, alive, tempval
-
-#
 #
 
 class channel(object):
@@ -354,8 +322,6 @@
         """
         return self._channel_action(None, -1)
 
-    register_stackless_primitive(receive, retval_expr='receiver.tempval')
-
     def send_exception(self, exp_type, msg):
         self.send(bomb(exp_type, exp_type(msg)))
 
@@ -372,9 +338,8 @@
         the runnables list.
         """
         return self._channel_action(msg, 1)
-            
-    register_stackless_primitive(send)
-            
+
+
 class tasklet(coroutine):
     """
     A tasklet object represents a tiny task in a Python thread.
@@ -456,7 +421,7 @@
 
         self.func = None
         coroutine.bind(self, _func)
-        back = _getcurrent()
+        back = _coroutine_getcurrent()
         coroutine.switch(self)
         self.alive = True
         _scheduler_append(self)
@@ -480,39 +445,6 @@
             raise RuntimeError, "The current tasklet cannot be removed."
             # not sure if I will revive this  " Use t=tasklet().capture()"
         _scheduler_remove(self)
-        
-    def __reduce__(self):
-        one, two, coro_state = coroutine.__reduce__(self)
-        assert one is coroutine
-        assert two == ()
-        # we want to get rid of the parent thing.
-        # for now, we just drop it
-        a, frame, c, d = coro_state
-
-        # Removing all frames related to stackless.py.
-        # They point to stuff we don't want to be pickled.
-
-        pickleframe = frame
-        while frame is not None:
-            if frame.f_code == schedule.func_code:
-                # Removing everything including and after the
-                # call to stackless.schedule()
-                pickleframe = frame.f_back
-                break
-            frame = frame.f_back
-        if d:
-            assert isinstance(d, coroutine)
-        coro_state = a, pickleframe, c, None
-        coro_state, alive, tempval = rewrite_stackless_primitive(coro_state, self.alive, self.tempval)
-        inst_dict = self.__dict__.copy()
-        inst_dict.pop('tempval', None)
-        return self.__class__, (), (coro_state, alive, tempval, inst_dict)
-
-    def __setstate__(self, (coro_state, alive, tempval, inst_dict)):
-        coroutine.__setstate__(self, coro_state)
-        self.__dict__.update(inst_dict)
-        self.alive = alive
-        self.tempval = tempval
 
 def getmain():
     """
@@ -601,30 +533,7 @@
     global _last_task
     _global_task_id = 0
     _main_tasklet = coroutine.getcurrent()
-    try:
-        _main_tasklet.__class__ = tasklet
-    except TypeError: # we are running pypy-c
-        class TaskletProxy(object):
-            """TaskletProxy is needed to give the _main_coroutine tasklet behaviour"""
-            def __init__(self, coro):
-                self._coro = coro
-
-            def __getattr__(self,attr):
-                return getattr(self._coro,attr)
-
-            def __str__(self):
-                return '<tasklet %s a:%s>' % (self._task_id, self.is_alive)
-
-            def __reduce__(self):
-                return getmain, ()
-
-            __repr__ = __str__
-
-
-        global _main_coroutine
-        _main_coroutine = _main_tasklet
-        _main_tasklet = TaskletProxy(_main_tasklet)
-        assert _main_tasklet.is_alive and not _main_tasklet.is_zombie
+    _main_tasklet.__class__ = tasklet         # XXX HAAAAAAAAAAAAAAAAAAAAACK
     _last_task = _main_tasklet
     tasklet._init.im_func(_main_tasklet, label='main')
     _squeue = deque()
diff --git a/py/_code/source.py b/py/_code/source.py
--- a/py/_code/source.py
+++ b/py/_code/source.py
@@ -139,7 +139,7 @@
                 trysource = self[start:end]
                 if trysource.isparseable():
                     return start, end
-        return start, end
+        return start, len(self)
 
     def getblockend(self, lineno):
         # XXX
diff --git a/pypy/annotation/policy.py b/pypy/annotation/policy.py
--- a/pypy/annotation/policy.py
+++ b/pypy/annotation/policy.py
@@ -1,6 +1,6 @@
 # base annotation policy for specialization
 from pypy.annotation.specialize import default_specialize as default
-from pypy.annotation.specialize import specialize_argvalue, specialize_argtype, specialize_arglistitemtype
+from pypy.annotation.specialize import specialize_argvalue, specialize_argtype, specialize_arglistitemtype, specialize_arg_or_var
 from pypy.annotation.specialize import memo, specialize_call_location
 # for some reason, model must be imported first,
 # or we create a cycle.
@@ -73,6 +73,7 @@
     default_specialize = staticmethod(default)
     specialize__memo = staticmethod(memo)
     specialize__arg = staticmethod(specialize_argvalue) # specialize:arg(N)
+    specialize__arg_or_var = staticmethod(specialize_arg_or_var)
     specialize__argtype = staticmethod(specialize_argtype) # specialize:argtype(N)
     specialize__arglistitemtype = staticmethod(specialize_arglistitemtype)
     specialize__call_location = staticmethod(specialize_call_location)
diff --git a/pypy/annotation/specialize.py b/pypy/annotation/specialize.py
--- a/pypy/annotation/specialize.py
+++ b/pypy/annotation/specialize.py
@@ -353,6 +353,16 @@
     key = tuple(key)
     return maybe_star_args(funcdesc, key, args_s)
 
+def specialize_arg_or_var(funcdesc, args_s, *argindices):
+    for argno in argindices:
+        if not args_s[argno].is_constant():
+            break
+    else:
+        # all constant
+        return specialize_argvalue(funcdesc, args_s, *argindices)
+    # some not constant
+    return maybe_star_args(funcdesc, None, args_s)
+
 def specialize_argtype(funcdesc, args_s, *argindices):
     key = tuple([args_s[i].knowntype for i in argindices])
     for cls in key:
diff --git a/pypy/annotation/test/test_annrpython.py b/pypy/annotation/test/test_annrpython.py
--- a/pypy/annotation/test/test_annrpython.py
+++ b/pypy/annotation/test/test_annrpython.py
@@ -1194,6 +1194,20 @@
         assert len(executedesc._cache[(0, 'star', 2)].startblock.inputargs) == 4
         assert len(executedesc._cache[(1, 'star', 3)].startblock.inputargs) == 5
 
+    def test_specialize_arg_or_var(self):
+        def f(a):
+            return 1
+        f._annspecialcase_ = 'specialize:arg_or_var(0)'
+
+        def fn(a):
+            return f(3) + f(a)
+
+        a = self.RPythonAnnotator()
+        a.build_types(fn, [int])
+        executedesc = a.bookkeeper.getdesc(f)
+        assert sorted(executedesc._cache.keys()) == [None, (3,)]
+        # we got two different special
+
     def test_specialize_call_location(self):
         def g(a):
             return a
@@ -3190,6 +3204,8 @@
         s = a.build_types(f, [])
         assert isinstance(s, annmodel.SomeList)
         assert not s.listdef.listitem.resized
+        assert not s.listdef.listitem.immutable
+        assert s.listdef.listitem.mutated
 
     def test_delslice(self):
         def f():
diff --git a/pypy/annotation/unaryop.py b/pypy/annotation/unaryop.py
--- a/pypy/annotation/unaryop.py
+++ b/pypy/annotation/unaryop.py
@@ -352,6 +352,7 @@
         check_negative_slice(s_start, s_stop)
         if not isinstance(s_iterable, SomeList):
             raise Exception("list[start:stop] = x: x must be a list")
+        lst.listdef.mutate()
         lst.listdef.agree(s_iterable.listdef)
         # note that setslice is not allowed to resize a list in RPython
 
diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -53,6 +53,18 @@
 this is an ideal task to get started, because it does not require any deep
 knowledge of the internals.
 
+Optimized Unicode Representation
+--------------------------------
+
+CPython 3.3 will use an `optimized unicode representation`_ which switches between
+different ways to represent a unicode string, depending on whether the string
+fits into ASCII, has only two-byte characters or needs four-byte characters.
+
+The actual details would be rather differen in PyPy, but we would like to have
+the same optimization implemented.
+
+.. _`optimized unicode representation`: http://www.python.org/dev/peps/pep-0393/
+
 Translation Toolchain
 ---------------------
 
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -3,12 +3,12 @@
 from pypy.interpreter.executioncontext import ExecutionContext, ActionFlag
 from pypy.interpreter.executioncontext import UserDelAction, FrameTraceAction
 from pypy.interpreter.error import OperationError, operationerrfmt
-from pypy.interpreter.error import new_exception_class
+from pypy.interpreter.error import new_exception_class, typed_unwrap_error_msg
 from pypy.interpreter.argument import Arguments
 from pypy.interpreter.miscutils import ThreadLocals
 from pypy.tool.cache import Cache
 from pypy.tool.uid import HUGEVAL_BYTES
-from pypy.rlib.objectmodel import we_are_translated, newlist
+from pypy.rlib.objectmodel import we_are_translated, newlist, compute_unique_id
 from pypy.rlib.debug import make_sure_not_resized
 from pypy.rlib.timer import DummyTimer, Timer
 from pypy.rlib.rarithmetic import r_uint
@@ -186,6 +186,28 @@
     def _set_mapdict_storage_and_map(self, storage, map):
         raise NotImplementedError
 
+    # -------------------------------------------------------------------
+
+    def str_w(self, space):
+        w_msg = typed_unwrap_error_msg(space, "string", self)
+        raise OperationError(space.w_TypeError, w_msg)
+
+    def unicode_w(self, space):
+        raise OperationError(space.w_TypeError,
+                             typed_unwrap_error_msg(space, "unicode", self))
+
+    def int_w(self, space):
+        raise OperationError(space.w_TypeError,
+                             typed_unwrap_error_msg(space, "integer", self))
+    
+    def uint_w(self, space):
+        raise OperationError(space.w_TypeError,
+                             typed_unwrap_error_msg(space, "integer", self))
+    
+    def bigint_w(self, space):
+        raise OperationError(space.w_TypeError,
+                             typed_unwrap_error_msg(space, "integer", self))
+
 
 class Wrappable(W_Root):
     """A subclass of Wrappable is an internal, interpreter-level class
@@ -901,7 +923,7 @@
         ec.c_call_trace(frame, w_func, args)
         try:
             w_res = self.call_args(w_func, args)
-        except OperationError, e:
+        except OperationError:
             ec.c_exception_trace(frame, w_func)
             raise
         ec.c_return_trace(frame, w_func, args)
@@ -947,6 +969,9 @@
     def isinstance_w(self, w_obj, w_type):
         return self.is_true(self.isinstance(w_obj, w_type))
 
+    def id(self, w_obj):
+        return self.wrap(compute_unique_id(w_obj))
+
     # The code below only works
     # for the simple case (new-style instance).
     # These methods are patched with the full logic by the __builtin__
@@ -999,8 +1024,6 @@
 
     def eval(self, expression, w_globals, w_locals, hidden_applevel=False):
         "NOT_RPYTHON: For internal debugging."
-        import types
-        from pypy.interpreter.pycode import PyCode
         if isinstance(expression, str):
             compiler = self.createcompiler()
             expression = compiler.compile(expression, '?', 'eval', 0,
@@ -1012,7 +1035,6 @@
     def exec_(self, statement, w_globals, w_locals, hidden_applevel=False,
               filename=None):
         "NOT_RPYTHON: For internal debugging."
-        import types
         if filename is None:
             filename = '?'
         from pypy.interpreter.pycode import PyCode
@@ -1210,6 +1232,18 @@
             return None
         return self.str_w(w_obj)
 
+    def str_w(self, w_obj):
+        return w_obj.str_w(self)
+
+    def int_w(self, w_obj):
+        return w_obj.int_w(self)
+
+    def uint_w(self, w_obj):
+        return w_obj.uint_w(self)
+
+    def bigint_w(self, w_obj):
+        return w_obj.bigint_w(self)
+
     def realstr_w(self, w_obj):
         # Like str_w, but only works if w_obj is really of type 'str'.
         if not self.is_true(self.isinstance(w_obj, self.w_str)):
@@ -1217,6 +1251,9 @@
                                  self.wrap('argument must be a string'))
         return self.str_w(w_obj)
 
+    def unicode_w(self, w_obj):
+        return w_obj.unicode_w(self)
+
     def realunicode_w(self, w_obj):
         # Like unicode_w, but only works if w_obj is really of type
         # 'unicode'.
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -458,3 +458,7 @@
     if module:
         space.setattr(w_exc, space.wrap("__module__"), space.wrap(module))
     return w_exc
+
+def typed_unwrap_error_msg(space, expected, w_obj):
+    type_name = space.type(w_obj).getname(space)
+    return space.wrap("expected %s, got %s object" % (expected, type_name))
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -175,6 +175,9 @@
             self.w_tracefunc = w_func
             self.space.frame_trace_action.fire()
 
+    def gettrace(self):
+        return self.w_tracefunc
+
     def setprofile(self, w_func):
         """Set the global trace function."""
         if self.space.is_w(w_func, self.space.w_None):
@@ -307,7 +310,11 @@
         self._nonperiodic_actions = []
         self.has_bytecode_counter = False
         self.fired_actions = None
-        self.checkinterval_scaled = 100 * TICK_COUNTER_STEP
+        # the default value is not 100, unlike CPython 2.7, but a much
+        # larger value, because we use a technique that not only allows
+        # but actually *forces* another thread to run whenever the counter
+        # reaches zero.
+        self.checkinterval_scaled = 10000 * TICK_COUNTER_STEP
         self._rebuild_action_dispatcher()
 
     def fire(self, action):
@@ -346,6 +353,7 @@
         elif interval > MAX:
             interval = MAX
         self.checkinterval_scaled = interval * TICK_COUNTER_STEP
+        self.reset_ticker(-1)
 
     def _rebuild_action_dispatcher(self):
         periodic_actions = unrolling_iterable(self._periodic_actions)
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -66,7 +66,7 @@
         make_sure_not_resized(self.locals_stack_w)
         check_nonneg(self.nlocals)
         #
-        if space.config.objspace.honor__builtins__ and w_globals is not None:
+        if space.config.objspace.honor__builtins__:
             self.builtin = space.builtin.pick_builtin(w_globals)
         # regular functions always have CO_OPTIMIZED and CO_NEWLOCALS.
         # class bodies only have CO_NEWLOCALS.
diff --git a/pypy/interpreter/test/test_executioncontext.py b/pypy/interpreter/test/test_executioncontext.py
--- a/pypy/interpreter/test/test_executioncontext.py
+++ b/pypy/interpreter/test/test_executioncontext.py
@@ -42,6 +42,7 @@
         assert i == 9
 
     def test_periodic_action(self):
+        from pypy.interpreter.executioncontext import ActionFlag
 
         class DemoAction(executioncontext.PeriodicAsyncAction):
             counter = 0
@@ -53,17 +54,20 @@
 
         space = self.space
         a2 = DemoAction(space)
-        space.actionflag.register_periodic_action(a2, True)
         try:
-            for i in range(500):
-                space.appexec([], """():
-                    n = 5
-                    return n + 2
-                """)
-        except Finished:
-            pass
-        checkinterval = space.actionflag.getcheckinterval()
-        assert checkinterval / 10 < i < checkinterval * 1.1
+            space.actionflag.setcheckinterval(100)
+            space.actionflag.register_periodic_action(a2, True)
+            try:
+                for i in range(500):
+                    space.appexec([], """():
+                        n = 5
+                        return n + 2
+                    """)
+            except Finished:
+                pass
+        finally:
+            space.actionflag = ActionFlag()   # reset to default
+        assert 10 < i < 110
 
     def test_llprofile(self):
         l = []
diff --git a/pypy/jit/codewriter/effectinfo.py b/pypy/jit/codewriter/effectinfo.py
--- a/pypy/jit/codewriter/effectinfo.py
+++ b/pypy/jit/codewriter/effectinfo.py
@@ -74,6 +74,7 @@
     OS_LLONG_UGE                = 91
     OS_LLONG_URSHIFT            = 92
     OS_LLONG_FROM_UINT          = 93
+    OS_LLONG_U_TO_FLOAT         = 94
     #
     OS_MATH_SQRT                = 100
 
diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -440,6 +440,7 @@
     rewrite_op_ullong_mod_zer      = _do_builtin_call
     rewrite_op_gc_identityhash = _do_builtin_call
     rewrite_op_gc_id           = _do_builtin_call
+    rewrite_op_uint_mod        = _do_builtin_call
 
     # ----------
     # getfield/setfield/mallocs etc.
@@ -883,9 +884,15 @@
                 v = v_arg
                 oplist = []
             if unsigned1:
-                opname = 'cast_uint_to_longlong'
+                if unsigned2:
+                    opname = 'cast_uint_to_ulonglong'
+                else:
+                    opname = 'cast_uint_to_longlong'
             else:
-                opname = 'cast_int_to_longlong'
+                if unsigned2:
+                    opname = 'cast_int_to_ulonglong'
+                else:
+                    opname = 'cast_int_to_longlong'
             op2 = self.rewrite_operation(
                 SpaceOperation(opname, [v], v_result)
             )
@@ -995,6 +1002,21 @@
                 return op2
         ''' % (_op, _oopspec.lower(), _oopspec, _oopspec)).compile()
 
+    for _op, _oopspec in [('cast_int_to_ulonglong',     'FROM_INT'),
+                          ('cast_uint_to_ulonglong',    'FROM_UINT'),
+                          ('cast_float_to_ulonglong',   'FROM_FLOAT'),
+                          ('cast_ulonglong_to_float',   'U_TO_FLOAT'),
+                         ]:
+        exec py.code.Source('''
+            def rewrite_op_%s(self, op):
+                args = op.args
+                op1 = self.prepare_builtin_call(op, "ullong_%s", args)
+                op2 = self._handle_oopspec_call(op1, args,
+                                                EffectInfo.OS_LLONG_%s,
+                                           EffectInfo.EF_ELIDABLE_CANNOT_RAISE)
+                return op2
+        ''' % (_op, _oopspec.lower(), _oopspec)).compile()
+
     def _normalize(self, oplist):
         if isinstance(oplist, SpaceOperation):
             return [oplist]
diff --git a/pypy/jit/codewriter/support.py b/pypy/jit/codewriter/support.py
--- a/pypy/jit/codewriter/support.py
+++ b/pypy/jit/codewriter/support.py
@@ -315,18 +315,30 @@
 def _ll_1_llong_from_int(x):
     return r_longlong(intmask(x))
 
+def _ll_1_ullong_from_int(x):
+    return r_ulonglong(intmask(x))
+
 def _ll_1_llong_from_uint(x):
     return r_longlong(r_uint(x))
 
+def _ll_1_ullong_from_uint(x):
+    return r_ulonglong(r_uint(x))
+
 def _ll_1_llong_to_int(xll):
     return intmask(xll)
 
 def _ll_1_llong_from_float(xf):
     return r_longlong(xf)
 
+def _ll_1_ullong_from_float(xf):
+    return r_ulonglong(xf)
+
 def _ll_1_llong_to_float(xll):
     return float(rffi.cast(lltype.SignedLongLong, xll))
 
+def _ll_1_ullong_u_to_float(xull):
+    return float(rffi.cast(lltype.UnsignedLongLong, xull))
+
 
 def _ll_1_llong_abs(xll):
     if xll < 0:
@@ -351,20 +363,23 @@
     return llop.llong_mod(lltype.SignedLongLong, xll, yll)
 
 def _ll_2_ullong_floordiv(xll, yll):
-    return llop.ullong_floordiv(lltype.SignedLongLong, xll, yll)
+    return llop.ullong_floordiv(lltype.UnsignedLongLong, xll, yll)
 
 def _ll_2_ullong_floordiv_zer(xll, yll):
     if yll == 0:
         raise ZeroDivisionError
-    return llop.ullong_floordiv(lltype.SignedLongLong, xll, yll)
+    return llop.ullong_floordiv(lltype.UnsignedLongLong, xll, yll)
 
 def _ll_2_ullong_mod(xll, yll):
-    return llop.ullong_mod(lltype.SignedLongLong, xll, yll)
+    return llop.ullong_mod(lltype.UnsignedLongLong, xll, yll)
 
 def _ll_2_ullong_mod_zer(xll, yll):
     if yll == 0:
         raise ZeroDivisionError
-    return llop.ullong_mod(lltype.SignedLongLong, xll, yll)
+    return llop.ullong_mod(lltype.UnsignedLongLong, xll, yll)
+
+def _ll_2_uint_mod(xll, yll):
+    return llop.uint_mod(lltype.Unsigned, xll, yll)
 
 
 # libffi support
diff --git a/pypy/jit/codewriter/test/test_flatten.py b/pypy/jit/codewriter/test/test_flatten.py
--- a/pypy/jit/codewriter/test/test_flatten.py
+++ b/pypy/jit/codewriter/test/test_flatten.py
@@ -829,14 +829,15 @@
                     self.encoding_test(f, [rffi.cast(FROM, 42)], expectedstr,
                                        transform=True)
                 elif TO in (rffi.LONG, rffi.ULONG):
+                    if rffi.cast(FROM, -1) < 0:
+                        fnname = "llong_from_int"
+                    else:
+                        fnname = "llong_from_uint"
                     if TO == rffi.LONG:
                         TO = rffi.LONGLONG
                     else:
                         TO = rffi.ULONGLONG
-                    if rffi.cast(FROM, -1) < 0:
-                        fnname = "llong_from_int"
-                    else:
-                        fnname = "llong_from_uint"
+                        fnname = "u" + fnname
                     expected.pop()   # remove int_return
                     expected.append(
                         "residual_call_irf_f $<* fn %s>, <Descr>, I[%s], R[], F[] -> %%f0"
diff --git a/pypy/jit/codewriter/test/test_longlong.py b/pypy/jit/codewriter/test/test_longlong.py
--- a/pypy/jit/codewriter/test/test_longlong.py
+++ b/pypy/jit/codewriter/test/test_longlong.py
@@ -57,7 +57,8 @@
             assert op1.opname == 'residual_call_irf_f'
         else:
             assert op1.opname == 'residual_call_irf_i'
-        gotindex = getattr(EffectInfo, 'OS_' + op1.args[0].value.upper())
+        gotindex = getattr(EffectInfo,
+                           'OS_' + op1.args[0].value.upper().lstrip('U'))
         assert gotindex == oopspecindex
         assert op1.args[1] == 'calldescr-%d' % oopspecindex
         assert list(op1.args[2]) == [v for v in vlist
@@ -192,8 +193,12 @@
                       [lltype.SignedLongLong], lltype.Signed)
         self.do_check('cast_float_to_longlong', EffectInfo.OS_LLONG_FROM_FLOAT,
                       [lltype.Float], lltype.SignedLongLong)
+        self.do_check('cast_float_to_ulonglong', EffectInfo.OS_LLONG_FROM_FLOAT,
+                      [lltype.Float], lltype.UnsignedLongLong)
         self.do_check('cast_longlong_to_float', EffectInfo.OS_LLONG_TO_FLOAT,
                       [lltype.SignedLongLong], lltype.Float)
+        self.do_check('cast_ulonglong_to_float', EffectInfo.OS_LLONG_U_TO_FLOAT,
+                      [lltype.UnsignedLongLong], lltype.Float)
         for T1 in [lltype.SignedLongLong, lltype.UnsignedLongLong]:
             for T2 in [lltype.Signed, lltype.Unsigned]:
                 self.do_check('cast_primitive', EffectInfo.OS_LLONG_TO_INT,
diff --git a/pypy/jit/metainterp/optimizeopt/__init__.py b/pypy/jit/metainterp/optimizeopt/__init__.py
--- a/pypy/jit/metainterp/optimizeopt/__init__.py
+++ b/pypy/jit/metainterp/optimizeopt/__init__.py
@@ -7,6 +7,8 @@
 from pypy.jit.metainterp.optimizeopt.unroll import optimize_unroll, OptInlineShortPreamble
 from pypy.jit.metainterp.optimizeopt.fficall import OptFfiCall
 from pypy.jit.metainterp.optimizeopt.simplify import OptSimplify
+from pypy.jit.metainterp.optimizeopt.pure import OptPure
+from pypy.jit.metainterp.optimizeopt.earlyforce import OptEarlyForce
 from pypy.rlib.jit import PARAMETERS
 from pypy.rlib.unroll import unrolling_iterable
 
@@ -14,6 +16,8 @@
             ('rewrite', OptRewrite),
             ('virtualize', OptVirtualize),
             ('string', OptString),
+            ('earlyforce', OptEarlyForce),
+            ('pure', OptPure),
             ('heap', OptHeap),
             ('ffi', None),
             ('unroll', None)]
diff --git a/pypy/jit/metainterp/optimizeopt/earlyforce.py b/pypy/jit/metainterp/optimizeopt/earlyforce.py
new file mode 100644
--- /dev/null
+++ b/pypy/jit/metainterp/optimizeopt/earlyforce.py
@@ -0,0 +1,24 @@
+from pypy.jit.metainterp.optimizeopt.optimizer import Optimization
+from pypy.jit.metainterp.optimizeopt.vstring import VAbstractStringValue
+from pypy.jit.metainterp.resoperation import rop, ResOperation
+
+class OptEarlyForce(Optimization):
+    def propagate_forward(self, op):
+        opnum = op.getopnum()
+        if (opnum != rop.SETFIELD_GC and 
+            opnum != rop.SETARRAYITEM_GC and
+            opnum != rop.QUASIIMMUT_FIELD):
+               
+            for arg in op.getarglist():
+                if arg in self.optimizer.values:
+                    value = self.getvalue(arg)
+                    value.force_box(self)
+        self.emit_operation(op)
+
+    def new(self):
+        return OptEarlyForce()
+
+    def setup(self):
+        self.optimizer.optearlyforce = self
+
+    
diff --git a/pypy/jit/metainterp/optimizeopt/fficall.py b/pypy/jit/metainterp/optimizeopt/fficall.py
--- a/pypy/jit/metainterp/optimizeopt/fficall.py
+++ b/pypy/jit/metainterp/optimizeopt/fficall.py
@@ -177,10 +177,10 @@
             funcinfo.descr is None):
             return [op] # cannot optimize
         funcsymval = self.getvalue(op.getarg(2))
-        arglist = [funcsymval.force_box()]
+        arglist = [funcsymval.get_key_box()]
         for push_op in funcinfo.opargs:
             argval = self.getvalue(push_op.getarg(2))
-            arglist.append(argval.force_box())
+            arglist.append(argval.get_key_box())
         newop = ResOperation(rop.CALL_RELEASE_GIL, arglist, op.result,
                              descr=funcinfo.descr)
         self.commit_optimization()
diff --git a/pypy/jit/metainterp/optimizeopt/heap.py b/pypy/jit/metainterp/optimizeopt/heap.py
--- a/pypy/jit/metainterp/optimizeopt/heap.py
+++ b/pypy/jit/metainterp/optimizeopt/heap.py
@@ -50,6 +50,7 @@
             if not self._lazy_setfield_registered:
                 optheap._lazy_setfields_and_arrayitems.append(self)
                 self._lazy_setfield_registered = True
+            
         else:
             # this is the case where the pending setfield ends up
             # storing precisely the value that is already there,
@@ -158,12 +159,17 @@
         self._lazy_setfields_and_arrayitems = []
         self._remove_guard_not_invalidated = False
         self._seen_guard_not_invalidated = False
+        self.posponedop = None
 
     def force_at_end_of_preamble(self):
         self.force_all_lazy_setfields_and_arrayitems()
 
     def flush(self):
         self.force_all_lazy_setfields_and_arrayitems()
+        if self.posponedop:
+            posponedop = self.posponedop
+            self.posponedop = None
+            self.next_optimization.propagate_forward(posponedop)
 
     def new(self):
         return OptHeap()
@@ -211,7 +217,15 @@
 
     def emit_operation(self, op):
         self.emitting_operation(op)
-        self.next_optimization.propagate_forward(op)
+        if self.posponedop:
+            posponedop = self.posponedop
+            self.posponedop = None
+            self.next_optimization.propagate_forward(posponedop)
+        if (op.is_comparison() or op.getopnum() == rop.CALL_MAY_FORCE
+            or op.is_ovf()):
+            self.posponedop = op
+        else:
+            self.next_optimization.propagate_forward(op)
 
     def emitting_operation(self, op):
         if op.has_no_side_effect():
@@ -293,30 +307,6 @@
             if indexvalue is None or indexvalue.intbound.contains(idx):
                 cf.force_lazy_setfield(self, can_cache)
 
-    def fixup_guard_situation(self):
-        # hackish: reverse the order of the last two operations if it makes
-        # sense to avoid a situation like "int_eq/setfield_gc/guard_true",
-        # which the backend (at least the x86 backend) does not handle well.
-        newoperations = self.optimizer.newoperations
-        if len(newoperations) < 2:
-            return
-        lastop = newoperations[-1]
-        if (lastop.getopnum() != rop.SETFIELD_GC and
-            lastop.getopnum() != rop.SETARRAYITEM_GC):
-            return
-        # - is_comparison() for cases like "int_eq/setfield_gc/guard_true"
-        # - CALL_MAY_FORCE: "call_may_force/setfield_gc/guard_not_forced"
-        # - is_ovf(): "int_add_ovf/setfield_gc/guard_no_overflow"
-        prevop = newoperations[-2]
-        opnum = prevop.getopnum()
-        if not (prevop.is_comparison() or opnum == rop.CALL_MAY_FORCE
-                or prevop.is_ovf()):
-            return
-        if prevop.result in lastop.getarglist():
-            return
-        newoperations[-2] = lastop
-        newoperations[-1] = prevop
-
     def _assert_valid_cf(self, cf):
         # check that 'cf' is in cached_fields or cached_arrayitems
         if not we_are_translated():
@@ -362,7 +352,6 @@
                                       fieldvalue.get_key_box(), itemindex))
             else:
                 cf.force_lazy_setfield(self)
-                self.fixup_guard_situation()
         return pendingfields
 
     def optimize_GETFIELD_GC(self, op):
@@ -374,12 +363,22 @@
             return
         # default case: produce the operation
         structvalue.ensure_nonnull()
-        ###self.optimizer.optimize_default(op)
         self.emit_operation(op)
         # then remember the result of reading the field
         fieldvalue = self.getvalue(op.result)
         cf.remember_field_value(structvalue, fieldvalue, op)
 
+    def optimize_GETFIELD_GC_PURE(self, op):
+        structvalue = self.getvalue(op.getarg(0))
+        cf = self.field_cache(op.getdescr())
+        fieldvalue = cf.getfield_from_cache(self, structvalue)
+        if fieldvalue is not None:
+            self.make_equal_to(op.result, fieldvalue)
+            return
+        # default case: produce the operation
+        structvalue.ensure_nonnull()
+        self.emit_operation(op)
+
     def optimize_SETFIELD_GC(self, op):
         if self.has_pure_result(rop.GETFIELD_GC_PURE, [op.getarg(0)],
                                 op.getdescr()):
@@ -389,6 +388,7 @@
         #
         cf = self.field_cache(op.getdescr())
         cf.do_setfield(self, op)
+        
 
     def optimize_GETARRAYITEM_GC(self, op):
         arrayvalue = self.getvalue(op.getarg(0))
@@ -413,6 +413,25 @@
             fieldvalue = self.getvalue(op.result)
             cf.remember_field_value(arrayvalue, fieldvalue, op)
 
+    def optimize_GETARRAYITEM_GC_PURE(self, op):
+        arrayvalue = self.getvalue(op.getarg(0))
+        indexvalue = self.getvalue(op.getarg(1))
+        cf = None
+        if indexvalue.is_constant():
+            arrayvalue.make_len_gt(MODE_ARRAY, op.getdescr(), indexvalue.box.getint())
+            # use the cache on (arraydescr, index), which is a constant
+            cf = self.arrayitem_cache(op.getdescr(), indexvalue.box.getint())
+            fieldvalue = cf.getfield_from_cache(self, arrayvalue)
+            if fieldvalue is not None:
+                self.make_equal_to(op.result, fieldvalue)
+                return
+        else:
+            # variable index, so make sure the lazy setarrayitems are done
+            self.force_lazy_setarrayitem(op.getdescr(), indexvalue=indexvalue)
+        # default case: produce the operation
+        arrayvalue.ensure_nonnull()
+        self.emit_operation(op)
+
     def optimize_SETARRAYITEM_GC(self, op):
         if self.has_pure_result(rop.GETARRAYITEM_GC_PURE, [op.getarg(0),
                                                            op.getarg(1)],
diff --git a/pypy/jit/metainterp/optimizeopt/optimizer.py b/pypy/jit/metainterp/optimizeopt/optimizer.py
--- a/pypy/jit/metainterp/optimizeopt/optimizer.py
+++ b/pypy/jit/metainterp/optimizeopt/optimizer.py
@@ -31,8 +31,8 @@
 
 class OptValue(object):
     __metaclass__ = extendabletype
-    _attrs_ = ('box', 'known_class', 'last_guard_index', 'level', 'intbound', 'lenbound')
-    last_guard_index = -1
+    _attrs_ = ('box', 'known_class', 'last_guard', 'level', 'intbound', 'lenbound')
+    last_guard = None
 
     level = LEVEL_UNKNOWN
     known_class = None
@@ -100,7 +100,7 @@
             self.make_constant(other.get_key_box())
             optimizer.turned_constant(self)
         elif other.level == LEVEL_KNOWNCLASS:
-            self.make_constant_class(other.known_class, -1)
+            self.make_constant_class(other.known_class, None)
         else:
             if other.level == LEVEL_NONNULL:
                 self.ensure_nonnull()
@@ -114,13 +114,13 @@
                     self.lenbound = other.lenbound.clone()
 
 
-    def force_box(self):
+    def force_box(self, optforce):
         return self.box
 
     def get_key_box(self):
         return self.box
 
-    def force_at_end_of_preamble(self, already_forced):
+    def force_at_end_of_preamble(self, already_forced, optforce):
         return self
 
     def get_args_for_fail(self, modifier):
@@ -162,16 +162,16 @@
         else:
             return None
 
-    def make_constant_class(self, classbox, opindex):
+    def make_constant_class(self, classbox, guardop):
         assert self.level < LEVEL_KNOWNCLASS
         self.known_class = classbox
         self.level = LEVEL_KNOWNCLASS
-        self.last_guard_index = opindex
+        self.last_guard = guardop
 
-    def make_nonnull(self, opindex):
+    def make_nonnull(self, guardop):
         assert self.level < LEVEL_NONNULL
         self.level = LEVEL_NONNULL
-        self.last_guard_index = opindex
+        self.last_guard = guardop
 
     def is_nonnull(self):
         level = self.level
@@ -223,6 +223,9 @@
     def __init__(self, box):
         self.make_constant(box)
 
+    def __repr__(self):
+        return 'Constant(%r)' % (self.box,)
+
 CONST_0      = ConstInt(0)
 CONST_1      = ConstInt(1)
 CVAL_ZERO    = ConstantValue(CONST_0)
@@ -237,26 +240,12 @@
     def __init__(self):
         pass # make rpython happy
 
-    def propagate_begin_forward(self):
-        if self.next_optimization:
-            self.next_optimization.propagate_begin_forward()
-
-    def propagate_end_forward(self):
-        if self.next_optimization:
-            self.next_optimization.propagate_end_forward()
-
     def propagate_forward(self, op):
         raise NotImplementedError
 
     def emit_operation(self, op):
         self.next_optimization.propagate_forward(op)
 
-    def test_emittable(self, op):
-        return self.is_emittable(op)
-
-    def is_emittable(self, op):
-        return self.next_optimization.test_emittable(op)
-
     # FIXME: Move some of these here?
     def getvalue(self, box):
         return self.optimizer.getvalue(box)
@@ -286,19 +275,19 @@
         return self.optimizer.new_const_item(arraydescr)
 
     def pure(self, opnum, args, result):
-        op = ResOperation(opnum, args, result)
-        key = self.optimizer.make_args_key(op)
-        if key not in self.optimizer.pure_operations:
-            self.optimizer.pure_operations[key] = op
+        if self.optimizer.optpure:
+            self.optimizer.optpure.pure(opnum, args, result)
 
     def has_pure_result(self, opnum, args, descr):
-        op = ResOperation(opnum, args, None, descr)
-        key = self.optimizer.make_args_key(op)
-        op = self.optimizer.pure_operations.get(key, None)
-        if op is None:
-            return False
-        return op.getdescr() is descr
+        if self.optimizer.optpure:
+            return self.optimizer.optpure.has_pure_result(opnum, args, descr)
+        return False
 
+    def get_pure_result(self, key):    
+        if self.optimizer.optpure:
+            return self.optimizer.optpure.get_pure_result(key)
+        return None
+    
     def setup(self):
         pass
 
@@ -319,6 +308,9 @@
     def produce_potential_short_preamble_ops(self, potential_ops):
         pass
 
+    def forget_numberings(self, box):
+        self.optimizer.forget_numberings(box)
+
 class Optimizer(Optimization):
 
     def __init__(self, metainterp_sd, loop, optimizations=None, bridge=False):
@@ -330,14 +322,16 @@
         self.interned_refs = self.cpu.ts.new_ref_dict()
         self.resumedata_memo = resume.ResumeDataLoopMemo(metainterp_sd)
         self.bool_boxes = {}
-        self.pure_operations = args_dict()
         self.producer = {}
         self.pendingfields = []
-        self.posponedop = None
         self.exception_might_have_happened = False
         self.quasi_immutable_deps = None
         self.opaque_pointers = {}
-        self.newoperations = []
+        self.replaces_guard = {}
+        self._newoperations = []
+        self.optimizer = self
+        self.optpure = None
+        self.optearlyforce = None
         if loop is not None:
             self.call_pure_results = loop.call_pure_results
 
@@ -366,21 +360,20 @@
     def flush(self):
         for o in self.optimizations:
             o.flush()
-        assert self.posponedop is None
 
     def new(self):
         new = Optimizer(self.metainterp_sd, self.loop)
         return self._new(new)
 
     def _new(self, new):
-        assert self.posponedop is None
         optimizations = [o.new() for o in self.optimizations]
         new.set_optimizations(optimizations)
         new.quasi_immutable_deps = self.quasi_immutable_deps
         return new
 
     def produce_potential_short_preamble_ops(self, sb):
-        raise NotImplementedError('This is implemented in unroll.UnrollableOptimizer')
+        for opt in self.optimizations:
+            opt.produce_potential_short_preamble_ops(sb)
 
     def turned_constant(self, value):
         for o in self.optimizations:
@@ -430,6 +423,13 @@
             return constbox
         return None
 
+    def get_newoperations(self):
+        self.flush()
+        return self._newoperations
+
+    def clear_newoperations(self):
+        self._newoperations = []
+
     def make_equal_to(self, box, value, replace=False):
         assert isinstance(value, OptValue)
         assert replace or box not in self.values
@@ -478,15 +478,10 @@
 
     def propagate_all_forward(self):
         self.exception_might_have_happened = self.bridge
-        self.newoperations = []
-        self.first_optimization.propagate_begin_forward()
-        self.i = 0
-        while self.i < len(self.loop.operations):
-            op = self.loop.operations[self.i]
+        self.clear_newoperations()
+        for op in self.loop.operations:
             self.first_optimization.propagate_forward(op)
-            self.i += 1
-        self.first_optimization.propagate_end_forward()
-        self.loop.operations = self.newoperations
+        self.loop.operations = self.get_newoperations()
         self.loop.quasi_immutable_deps = self.quasi_immutable_deps
         # accumulate counters
         self.resumedata_memo.update_counters(self.metainterp_sd.profiler)
@@ -498,9 +493,6 @@
         self.producer[op.result] = op
         dispatch_opt(self, op)
 
-    def test_emittable(self, op):
-        return True
-
     def emit_operation(self, op):
         if op.returns_bool_result():
             self.bool_boxes[self.getvalue(op.result)] = None
@@ -516,14 +508,30 @@
                 pass
             else:
                 self.ensure_imported(value)
-                op.setarg(i, value.force_box())
+                op.setarg(i, value.force_box(self))
         self.metainterp_sd.profiler.count(jitprof.OPT_OPS)
         if op.is_guard():
             self.metainterp_sd.profiler.count(jitprof.OPT_GUARDS)
-            op = self.store_final_boxes_in_guard(op)
+            if self.replaces_guard and op in self.replaces_guard:
+                self.replace_op(self.replaces_guard[op], op)
+                del self.replaces_guard[op]
+                return
+            else:
+                op = self.store_final_boxes_in_guard(op)
         elif op.can_raise():
             self.exception_might_have_happened = True
-        self.newoperations.append(op)
+        self._newoperations.append(op)
+
+    def replace_op(self, old_op, new_op):
+        # XXX: Do we want to cache indexes to prevent search?
+        i = len(self._newoperations) 
+        while i > 0:
+            i -= 1
+            if self._newoperations[i] is old_op:
+                self._newoperations[i] = new_op
+                break
+        else:
+            assert False
 
     def store_final_boxes_in_guard(self, op):
         descr = op.getdescr()
@@ -571,51 +579,8 @@
         args[n + 1] = op.getdescr()
         return args
 
-    @specialize.argtype(0)
     def optimize_default(self, op):
-        canfold = op.is_always_pure()
-        if op.is_ovf():
-            self.posponedop = op
-            return
-        if self.posponedop:
-            nextop = op
-            op = self.posponedop
-            self.posponedop = None
-            canfold = nextop.getopnum() == rop.GUARD_NO_OVERFLOW
-        else:
-            nextop = None
-
-        if canfold:
-            for i in range(op.numargs()):
-                if self.get_constant_box(op.getarg(i)) is None:
-                    break
-            else:
-                # all constant arguments: constant-fold away
-                resbox = self.constant_fold(op)
-                # note that INT_xxx_OVF is not done from here, and the
-                # overflows in the INT_xxx operations are ignored
-                self.make_constant(op.result, resbox)
-                return
-
-            # did we do the exact same operation already?
-            args = self.make_args_key(op)
-            oldop = self.pure_operations.get(args, None)
-            if oldop is not None and oldop.getdescr() is op.getdescr():
-                assert oldop.getopnum() == op.getopnum()
-                self.make_equal_to(op.result, self.getvalue(oldop.result),
-                                   True)
-                return
-            else:
-                self.pure_operations[args] = op
-                self.remember_emitting_pure(op)
-
-        # otherwise, the operation remains
         self.emit_operation(op)
-        if nextop:
-            self.emit_operation(nextop)
-
-    def remember_emitting_pure(self, op):
-        pass
 
     def constant_fold(self, op):
         argboxes = [self.get_constant_box(op.getarg(i))
@@ -633,11 +598,6 @@
     def optimize_DEBUG_MERGE_POINT(self, op):
         self.emit_operation(op)
 
-    def optimize_CAST_OPAQUE_PTR(self, op):
-        value = self.getvalue(op.getarg(0))
-        self.opaque_pointers[value] = True
-        self.make_equal_to(op.result, value)
-
     def optimize_GETARRAYITEM_GC_PURE(self, op):
         indexvalue = self.getvalue(op.getarg(1))
         if indexvalue.is_constant():
diff --git a/pypy/jit/metainterp/optimizeopt/pure.py b/pypy/jit/metainterp/optimizeopt/pure.py
new file mode 100644
--- /dev/null
+++ b/pypy/jit/metainterp/optimizeopt/pure.py
@@ -0,0 +1,115 @@
+from pypy.jit.metainterp.optimizeopt.optimizer import Optimization
+from pypy.jit.metainterp.resoperation import rop, ResOperation
+from pypy.jit.metainterp.optimizeopt.util import (make_dispatcher_method,
+    args_dict)
+
+class OptPure(Optimization):
+    def __init__(self):
+        self.posponedop = None
+        self.pure_operations = args_dict()
+        self.emitted_pure_operations = {}
+
+    def propagate_forward(self, op):
+        dispatch_opt(self, op)
+
+    def optimize_default(self, op):
+        canfold = op.is_always_pure()
+        if op.is_ovf():
+            self.posponedop = op
+            return
+        if self.posponedop:
+            nextop = op
+            op = self.posponedop
+            self.posponedop = None
+            canfold = nextop.getopnum() == rop.GUARD_NO_OVERFLOW
+        else:
+            nextop = None
+
+        if canfold:
+            for i in range(op.numargs()):
+                if self.get_constant_box(op.getarg(i)) is None:
+                    break
+            else:
+                # all constant arguments: constant-fold away
+                resbox = self.optimizer.constant_fold(op)
+                # note that INT_xxx_OVF is not done from here, and the
+                # overflows in the INT_xxx operations are ignored
+                self.optimizer.make_constant(op.result, resbox)
+                return
+
+            # did we do the exact same operation already?
+            args = self.optimizer.make_args_key(op)
+            oldop = self.pure_operations.get(args, None)
+            if oldop is not None and oldop.getdescr() is op.getdescr():
+                assert oldop.getopnum() == op.getopnum()
+                self.optimizer.make_equal_to(op.result, self.getvalue(oldop.result),
+                                   True)
+                return
+            else:
+                self.pure_operations[args] = op
+                self.remember_emitting_pure(op)
+
+        # otherwise, the operation remains
+        self.emit_operation(op)
+        if op.returns_bool_result():
+            self.optimizer.bool_boxes[self.getvalue(op.result)] = None        
+        if nextop:
+            self.emit_operation(nextop)
+
+    def optimize_CALL_PURE(self, op):
+        args = self.optimizer.make_args_key(op)
+        oldop = self.pure_operations.get(args, None)
+        if oldop is not None and oldop.getdescr() is op.getdescr():
+            assert oldop.getopnum() == op.getopnum()
+            self.make_equal_to(op.result, self.getvalue(oldop.result))
+            return
+        else:
+            self.pure_operations[args] = op
+            self.remember_emitting_pure(op)
+
+        # replace CALL_PURE with just CALL
+        args = op.getarglist()
+        self.emit_operation(ResOperation(rop.CALL, args, op.result,
+                                         op.getdescr()))
+
+    def flush(self):
+        assert self.posponedop is None
+
+    def new(self):
+        assert self.posponedop is None
+        return OptPure()
+
+    def setup(self):
+        self.optimizer.optpure = self
+
+    def pure(self, opnum, args, result):
+        op = ResOperation(opnum, args, result)
+        key = self.optimizer.make_args_key(op)
+        if key not in self.pure_operations:
+            self.pure_operations[key] = op
+
+    def has_pure_result(self, opnum, args, descr):
+        op = ResOperation(opnum, args, None, descr)
+        key = self.optimizer.make_args_key(op)
+        op = self.pure_operations.get(key, None)
+        if op is None:
+            return False
+        return op.getdescr() is descr
+
+    def get_pure_result(self, key):
+        return self.pure_operations.get(key, None)
+
+    def remember_emitting_pure(self, op):
+        self.emitted_pure_operations[op] = True
+
+    def produce_potential_short_preamble_ops(self, sb):
+        for op in self.emitted_pure_operations:
+            if op.getopnum() == rop.GETARRAYITEM_GC_PURE or \
+               op.getopnum() == rop.STRGETITEM or \
+               op.getopnum() == rop.UNICODEGETITEM:
+                if not self.getvalue(op.getarg(1)).is_constant():
+                    continue
+            sb.add_potential(op)
+
+dispatch_opt = make_dispatcher_method(OptPure, 'optimize_',
+                                      default=OptPure.optimize_default)
diff --git a/pypy/jit/metainterp/optimizeopt/rewrite.py b/pypy/jit/metainterp/optimizeopt/rewrite.py
--- a/pypy/jit/metainterp/optimizeopt/rewrite.py
+++ b/pypy/jit/metainterp/optimizeopt/rewrite.py
@@ -31,21 +31,8 @@
 
         dispatch_opt(self, op)
 
-    def test_emittable(self, op):
-        opnum = op.getopnum()
-        for value, cls, func in optimize_guards:
-            if opnum == value:
-                assert isinstance(op, cls)
-                try:
-                    func(self, op, dryrun=True)
-                    return self.is_emittable(op)
-                except InvalidLoop:
-                    return False
-        return self.is_emittable(op)
-
-
     def try_boolinvers(self, op, targs):
-        oldop = self.optimizer.pure_operations.get(targs, None)
+        oldop = self.get_pure_result(targs)
         if oldop is not None and oldop.getdescr() is op.getdescr():
             value = self.getvalue(oldop.result)
             if value.is_constant():
@@ -62,32 +49,35 @@
     def find_rewritable_bool(self, op, args):
         try:
             oldopnum = opboolinvers[op.getopnum()]
+        except KeyError:
+            pass
+        else:
             targs = self.optimizer.make_args_key(ResOperation(oldopnum, [args[0], args[1]],
                                                               None))
             if self.try_boolinvers(op, targs):
                 return True
-        except KeyError:
-            pass
 
         try:
             oldopnum = opboolreflex[op.getopnum()] # FIXME: add INT_ADD, INT_MUL
+        except KeyError:
+            pass
+        else:
             targs = self.optimizer.make_args_key(ResOperation(oldopnum, [args[1], args[0]],
                                                               None))
-            oldop = self.optimizer.pure_operations.get(targs, None)
+            oldop = self.get_pure_result(targs)
             if oldop is not None and oldop.getdescr() is op.getdescr():
                 self.make_equal_to(op.result, self.getvalue(oldop.result))
                 return True
-        except KeyError:
-            pass
 
         try:
             oldopnum = opboolinvers[opboolreflex[op.getopnum()]]
+        except KeyError:
+            pass
+        else:
             targs = self.optimizer.make_args_key(ResOperation(oldopnum, [args[1], args[0]],
                                                               None))
             if self.try_boolinvers(op, targs):
                 return True
-        except KeyError:
-            pass
 
         return False
 
@@ -214,40 +204,7 @@
         self.emit_operation(op)
         self.pure(rop.FLOAT_NEG, [op.result], v1)
 
-    def optimize_CALL_PURE(self, op):
-        arg_consts = []
-        for i in range(op.numargs()):
-            arg = op.getarg(i)
-            const = self.get_constant_box(arg)
-            if const is None:
-                break
-            arg_consts.append(const)
-        else:
-            # all constant arguments: check if we already know the result
-            try:
-                result = self.optimizer.call_pure_results[arg_consts]
-            except KeyError:
-                pass
-            else:
-                self.make_constant(op.result, result)
-                return
-
-        args = self.optimizer.make_args_key(op)
-        oldop = self.optimizer.pure_operations.get(args, None)
-        if oldop is not None and oldop.getdescr() is op.getdescr():
-            assert oldop.getopnum() == op.getopnum()
-            self.make_equal_to(op.result, self.getvalue(oldop.result))
-            return
-        else:
-            self.optimizer.pure_operations[args] = op
-            self.optimizer.remember_emitting_pure(op)
-
-        # replace CALL_PURE with just CALL
-        args = op.getarglist()
-        self.emit_operation(ResOperation(rop.CALL, args, op.result,
-                                         op.getdescr()))
-
-    def optimize_guard(self, op, constbox, emit_operation=True, dryrun=False):
+    def optimize_guard(self, op, constbox, emit_operation=True):
         value = self.getvalue(op.getarg(0))
         if value.is_constant():
             box = value.box
@@ -255,62 +212,57 @@
             if not box.same_constant(constbox):
                 raise InvalidLoop
             return
-        if dryrun: return
         if emit_operation:
             self.emit_operation(op)
         value.make_constant(constbox)
         self.optimizer.turned_constant(value)
 
-    def optimize_GUARD_ISNULL(self, op, dryrun=False):
+    def optimize_GUARD_ISNULL(self, op):
         value = self.getvalue(op.getarg(0))
         if value.is_null():
             return
         elif value.is_nonnull():
             raise InvalidLoop
-        if dryrun: return
         self.emit_operation(op)
         value.make_constant(self.optimizer.cpu.ts.CONST_NULL)
 
-    def optimize_GUARD_NONNULL(self, op, dryrun=False):
+    def optimize_GUARD_NONNULL(self, op):
         value = self.getvalue(op.getarg(0))
         if value.is_nonnull():
             return
         elif value.is_null():
             raise InvalidLoop
-        if dryrun: return
         self.emit_operation(op)
-        value.make_nonnull(len(self.optimizer.newoperations) - 1)
+        value.make_nonnull(op)
 
-    def optimize_GUARD_VALUE(self, op, dryrun=False):
+    def optimize_GUARD_VALUE(self, op):
         value = self.getvalue(op.getarg(0))
-        emit_operation = True
-        if not dryrun and value.last_guard_index != -1:
+        if value.last_guard:
             # there already has been a guard_nonnull or guard_class or
             # guard_nonnull_class on this value, which is rather silly.
             # replace the original guard with a guard_value
-            old_guard_op = self.optimizer.newoperations[value.last_guard_index]
-            new_guard_op = old_guard_op.copy_and_change(rop.GUARD_VALUE,
-                                             args = [old_guard_op.getarg(0), op.getarg(1)])
-            self.optimizer.newoperations[value.last_guard_index] = new_guard_op
+            old_guard_op = value.last_guard
+            op = old_guard_op.copy_and_change(rop.GUARD_VALUE,
+                                      args = [old_guard_op.getarg(0), op.getarg(1)])
+            self.optimizer.replaces_guard[op] = old_guard_op
             # hack hack hack.  Change the guard_opnum on
             # new_guard_op.getdescr() so that when resuming,
             # the operation is not skipped by pyjitpl.py.
-            descr = new_guard_op.getdescr()
+            descr = op.getdescr()
             assert isinstance(descr, compile.ResumeGuardDescr)
             descr.guard_opnum = rop.GUARD_VALUE
-            descr.make_a_counter_per_value(new_guard_op)
-            emit_operation = False
+            descr.make_a_counter_per_value(op)
         constbox = op.getarg(1)
         assert isinstance(constbox, Const)
-        self.optimize_guard(op, constbox, emit_operation, dryrun)
+        self.optimize_guard(op, constbox)
 
-    def optimize_GUARD_TRUE(self, op, dryrun=False):
-        self.optimize_guard(op, CONST_1, dryrun=dryrun)
+    def optimize_GUARD_TRUE(self, op):
+        self.optimize_guard(op, CONST_1)
 
-    def optimize_GUARD_FALSE(self, op, dryrun=False):
-        self.optimize_guard(op, CONST_0, dryrun=dryrun)
+    def optimize_GUARD_FALSE(self, op):
+        self.optimize_guard(op, CONST_0)
 
-    def optimize_GUARD_CLASS(self, op, dryrun=False):
+    def optimize_GUARD_CLASS(self, op):
         value = self.getvalue(op.getarg(0))
         expectedclassbox = op.getarg(1)
         assert isinstance(expectedclassbox, Const)
@@ -319,38 +271,32 @@
             if realclassbox.same_constant(expectedclassbox):
                 return
             raise InvalidLoop
-        if dryrun: return
-        emit_operation = True
-        if value.last_guard_index != -1:
+        if value.last_guard:
             # there already has been a guard_nonnull or guard_class or
             # guard_nonnull_class on this value.
-            old_guard_op = self.optimizer.newoperations[value.last_guard_index]
+            old_guard_op = value.last_guard
             if old_guard_op.getopnum() == rop.GUARD_NONNULL:
                 # it was a guard_nonnull, which we replace with a
                 # guard_nonnull_class.
-                new_guard_op = old_guard_op.copy_and_change (rop.GUARD_NONNULL_CLASS,
+                op = old_guard_op.copy_and_change (rop.GUARD_NONNULL_CLASS,
                                          args = [old_guard_op.getarg(0), op.getarg(1)])
-                self.optimizer.newoperations[value.last_guard_index] = new_guard_op
+                self.optimizer.replaces_guard[op] = old_guard_op
                 # hack hack hack.  Change the guard_opnum on
                 # new_guard_op.getdescr() so that when resuming,
                 # the operation is not skipped by pyjitpl.py.
-                descr = new_guard_op.getdescr()
+                descr = op.getdescr()
                 assert isinstance(descr, compile.ResumeGuardDescr)
                 descr.guard_opnum = rop.GUARD_NONNULL_CLASS
-                emit_operation = False
-        if emit_operation:
-            self.emit_operation(op)
-            last_guard_index = len(self.optimizer.newoperations) - 1
-        else:
-            last_guard_index = value.last_guard_index
-        value.make_constant_class(expectedclassbox, last_guard_index)
+        self.emit_operation(op)
+        value.make_constant_class(expectedclassbox, op)
 
-    def optimize_GUARD_NONNULL_CLASS(self, op, dryrun=False):
-        self.optimize_GUARD_NONNULL(op, True)
-        self.optimize_GUARD_CLASS(op, dryrun)
+    def optimize_GUARD_NONNULL_CLASS(self, op):
+        value = self.getvalue(op.getarg(0))
+        if value.is_null():
+            raise InvalidLoop
+        self.optimize_GUARD_CLASS(op)
 
-    def optimize_GUARD_NO_EXCEPTION(self, op, dryrun=False):
-        if dryrun: return
+    def optimize_GUARD_NO_EXCEPTION(self, op):
         if not self.optimizer.exception_might_have_happened:
             return
         self.emit_operation(op)
@@ -470,7 +416,7 @@
                     newop = ResOperation(rop.SETARRAYITEM_GC,
                                          [op.getarg(2),
                                           ConstInt(index + dest_start),
-                                          val.force_box()], None,
+                                          val.get_key_box()], None,
                                          descr=source_value.arraydescr)
                     self.emit_operation(newop)
             return True
@@ -478,6 +424,25 @@
             return True # 0-length arraycopy
         return False
 
+    def optimize_CALL_PURE(self, op):
+        arg_consts = []
+        for i in range(op.numargs()):
+            arg = op.getarg(i)
+            const = self.get_constant_box(arg)
+            if const is None:
+                break
+            arg_consts.append(const)
+        else:
+            # all constant arguments: check if we already know the result
+            try:
+                result = self.optimizer.call_pure_results[arg_consts]
+            except KeyError:
+                pass
+            else:
+                self.make_constant(op.result, result)
+                return
+        self.emit_operation(op)
+
     def optimize_INT_FLOORDIV(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
@@ -492,6 +457,10 @@
                                         args = [op.getarg(0), ConstInt(highest_bit(val))])
         self.emit_operation(op)
 
+    def optimize_CAST_OPAQUE_PTR(self, op):
+        value = self.getvalue(op.getarg(0))
+        self.optimizer.opaque_pointers[value] = True
+        self.make_equal_to(op.result, value)
 
 dispatch_opt = make_dispatcher_method(OptRewrite, 'optimize_',
         default=OptRewrite.emit_operation)
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -39,8 +39,9 @@
 
 def test_sharing_field_lists_of_virtual():
     class FakeOptimizer(object):
-        class cpu(object):
-            pass
+        class optimizer(object):
+            class cpu(object):
+                pass
     opt = FakeOptimizer()
     virt1 = virtualize.AbstractVirtualStructValue(opt, None)
     lst1 = virt1._get_field_descr_list()
@@ -69,7 +70,7 @@
     class FakeVirtualValue(virtualize.AbstractVirtualValue):
         def _make_virtual(self, *args):
             return FakeVInfo()
-    v1 = FakeVirtualValue(None, None, None)
+    v1 = FakeVirtualValue(None, None)
     vinfo1 = v1.make_virtual_info(None, [1, 2, 4])
     vinfo2 = v1.make_virtual_info(None, [1, 2, 4])
     assert vinfo1 is vinfo2
@@ -111,7 +112,7 @@
 
 class BaseTestBasic(BaseTest):
 
-    enable_opts = "intbounds:rewrite:virtualize:string:heap"
+    enable_opts = "intbounds:rewrite:virtualize:string:earlyforce:pure:heap"
 
     def optimize_loop(self, ops, optops, call_pure_results=None):
 
@@ -651,8 +652,8 @@
         i3 = getfield_gc(p3, descr=valuedescr)
         escape(i3)
         p1 = new_with_vtable(ConstClass(node_vtable))
+        p1sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p1, i1, descr=valuedescr)
-        p1sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p1sub, i1, descr=valuedescr)
         setfield_gc(p1, p1sub, descr=nextdescr)
         jump(i1, p1, p2)
@@ -667,10 +668,10 @@
         p3sub = getfield_gc(p3, descr=nextdescr)
         i3 = getfield_gc(p3sub, descr=valuedescr)
         escape(i3)
+        p1 = new_with_vtable(ConstClass(node_vtable))
         p2sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p2sub, i1, descr=valuedescr)
         setfield_gc(p2, p2sub, descr=nextdescr)
-        p1 = new_with_vtable(ConstClass(node_vtable))
         jump(i1, p1, p2)
         """
         # The same as test_p123_simple, but in the end the "old" p2 contains
@@ -4767,6 +4768,26 @@
         # other
         self.optimize_loop(ops, expected)
 
+    def test_plain_virtual_string_copy_content(self):
+        ops = """
+        []
+        p0 = newstr(6)
+        copystrcontent(s"hello!", p0, 0, 0, 6)
+        p1 = call(0, p0, s"abc123", descr=strconcatdescr)
+        i0 = strgetitem(p1, 0)
+        finish(i0)
+        """
+        expected = """
+        []
+        p0 = newstr(6)
+        copystrcontent(s"hello!", p0, 0, 0, 6)
+        p1 = newstr(12)
+        copystrcontent(p0, p1, 0, 0, 6)
+        copystrcontent(s"abc123", p1, 0, 6, 6)
+        i0 = strgetitem(p1, 0)
+        finish(i0)
+        """
+        self.optimize_strunicode_loop(ops, expected)
 
 
 class TestLLtype(BaseTestOptimizeBasic, LLtypeMixin):
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizefficall.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizefficall.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizefficall.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizefficall.py
@@ -36,7 +36,7 @@
 
 class TestFfiCall(BaseTestBasic, LLtypeMixin):
 
-    enable_opts = "intbounds:rewrite:virtualize:string:heap:ffi"
+    enable_opts = "intbounds:rewrite:virtualize:string:pure:earlyforce:heap:ffi"
 
     class namespace:
         cpu = LLtypeMixin.cpu
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -68,7 +68,7 @@
 
 class BaseTestWithUnroll(BaseTest):
 
-    enable_opts = "intbounds:rewrite:virtualize:string:heap:unroll"
+    enable_opts = "intbounds:rewrite:virtualize:string:earlyforce:pure:heap:unroll"
 
     def optimize_loop(self, ops, expected, expected_preamble=None,
                       call_pure_results=None, expected_short=None):
@@ -825,8 +825,8 @@
         i3 = getfield_gc(p2, descr=valuedescr)
         escape(i3)
         p4 = new_with_vtable(ConstClass(node_vtable))
+        p1sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p4, i1, descr=valuedescr)
-        p1sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p1sub, i1, descr=valuedescr)
         setfield_gc(p4, p1sub, descr=nextdescr)
         jump(i1, p4)
@@ -865,13 +865,7 @@
         p3sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p3sub, i1, descr=valuedescr)
         setfield_gc(p1, p3sub, descr=nextdescr)
-        # XXX: We get two extra operations here because the setfield
-        #      above is the result of forcing p1 and thus not
-        #      registered with the heap optimizer. I've makred tests
-        #      below with VIRTUALHEAP if they suffer from this issue
-        p3sub2 = getfield_gc(p1, descr=nextdescr)
-        guard_nonnull_class(p3sub2, ConstClass(node_vtable2)) []
-        jump(i1, p1, p3sub2)
+        jump(i1, p1, p3sub)
         """
         self.optimize_loop(ops, expected, preamble)
 
@@ -902,9 +896,7 @@
         guard_true(i2b) []
         p3 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3, i2, descr=nextdescr)
-        # XXX: VIRTUALHEAP (see above)
-        i3 = getfield_gc(p3, descr=nextdescr)
-        jump(p3, i3)
+        jump(p3, i2)
         """
         self.optimize_loop(ops, expected, preamble)
 
@@ -1219,7 +1211,15 @@
         setfield_gc(p3, p30, descr=valuedescr)
         jump(i29, p30, p3)
         """
-        expected = preamble
+        expected = """
+        [i0, p1, p3]
+        i28 = int_add(i0, 1)
+        i29 = int_add(i28, 1)
+        p30 = new_with_vtable(ConstClass(node_vtable))
+        setfield_gc(p3, p30, descr=valuedescr)
+        setfield_gc(p30, i28, descr=nextdescr)
+        jump(i29, p30, p3)
+        """
         self.optimize_loop(ops, expected, preamble)
 
     def test_nonvirtual_1(self):
@@ -2408,8 +2408,8 @@
         guard_class(p2, ConstClass(node_vtable)) []
         p3 = getfield_gc(p1, descr=otherdescr)
         guard_class(p3, ConstClass(node_vtable)) []
+        p3a = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3, p2, descr=otherdescr)
-        p3a = new_with_vtable(ConstClass(node_vtable))
         escape(p3a)
         jump(p3a)
         """
@@ -2421,9 +2421,9 @@
         # setfield_gc(p3, p2, descr=otherdescr) # p3a.other = p2a
         # p1a = new_with_vtable(ConstClass(node_vtable2))
         # p2a = new_with_vtable(ConstClass(node_vtable))
+        p3anew = new_with_vtable(ConstClass(node_vtable))
         p2 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3a, p2, descr=otherdescr) # p3a.other = p2a
-        p3anew = new_with_vtable(ConstClass(node_vtable))
         escape(p3anew)
         jump(p3anew)
         """
@@ -2458,9 +2458,9 @@
         p3 = getfield_gc(p1, descr=otherdescr)
         guard_class(p3, ConstClass(node_vtable)) []
         # p1a = new_with_vtable(ConstClass(node_vtable2))
+        p3a = new_with_vtable(ConstClass(node_vtable))
         p2a = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3, p2a, descr=otherdescr)
-        p3a = new_with_vtable(ConstClass(node_vtable))
         escape(p3a)
         # setfield_gc(p1a, p2a, descr=nextdescr)
         # setfield_gc(p1a, p3a, descr=otherdescr)
@@ -2468,9 +2468,9 @@
         """
         expected = """
         [p2, p3]
+        p3a = new_with_vtable(ConstClass(node_vtable))
         p2a = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3, p2a, descr=otherdescr)
-        p3a = new_with_vtable(ConstClass(node_vtable))
         escape(p3a)
         jump(p2a, p3a)
         """
@@ -2790,7 +2790,6 @@
         self.optimize_loop(ops, expected, preamble)
 
     def test_remove_duplicate_pure_op_ovf_with_lazy_setfield(self):
-        py.test.skip('this optimization is not yet supprted')
         ops = """
         [i1, p1]
         i3 = int_add_ovf(i1, 1)
@@ -5961,13 +5960,18 @@
         escape(i0)
         jump(p1)
         """
-        expected = """
+        preamble = """
         [p1]
         i0 = ptr_eq(p1, NULL)
         escape(i0)
-        jump(p1)
-        """
-        self.optimize_strunicode_loop_extradescrs(ops, expected, expected)
+        jump(p1, i0)
+        """
+        expected = """
+        [p1, i0]
+        escape(i0)
+        jump(p1, i0)
+        """
+        self.optimize_strunicode_loop_extradescrs(ops, expected, preamble)
 
     def test_str_equal_none2(self):
         ops = """
@@ -5976,13 +5980,18 @@
         escape(i0)
         jump(p1)
         """
-        expected = """
+        preamble = """
         [p1]
         i0 = ptr_eq(p1, NULL)
         escape(i0)
-        jump(p1)
-        """
-        self.optimize_strunicode_loop_extradescrs(ops, expected, expected)
+        jump(p1, i0)
+        """
+        expected = """
+        [p1, i0]
+        escape(i0)
+        jump(p1, i0)
+        """
+        self.optimize_strunicode_loop_extradescrs(ops, expected, preamble)
 
     def test_str_equal_nonnull1(self):
         ops = """
@@ -7220,6 +7229,60 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_heap_cache_forced_virtuals(self):
+        ops = """
+        [i1, i2, p0]
+        p1 = new(descr=ssize)
+        setfield_gc(p1, i1, descr=adescr)
+        setfield_gc(p1, i2, descr=bdescr)
+        call(p0, p1, descr=writeadescr)
+        i3 = getfield_gc(p1, descr=adescr)
+        i4 = getfield_gc(p1, descr=bdescr)
+        jump(i3, i4, p0)
+        """
+        expected = """
+        [i1, i2, p0]
+        p1 = new(descr=ssize)
+        setfield_gc(p1, i1, descr=adescr)
+        call(p0, p1, descr=writeadescr)
+        i3 = getfield_gc(p1, descr=adescr)
+        setfield_gc(p1, i2, descr=bdescr)
+        jump(i3, i2, p0)
+        """
+        self.optimize_loop(ops, expected)
+
+
+    def test_setarrayitem_followed_by_arraycopy(self):
+        ops = """
+        [p1, p2]
+        setarrayitem_gc(p1, 2, 10, descr=arraydescr)
+        setarrayitem_gc(p2, 3, 13, descr=arraydescr)        
+        call(0, p1, p2, 0, 0, 10, descr=arraycopydescr)
+        jump(p1, p2)
+        """
+        self.optimize_loop(ops, ops)
+
+    def test_heap_cache_virtuals_forced_by_delayed_setfield(self):
+        py.test.skip('not yet supoprted')
+        ops = """
+        [i1, p0]
+        p1 = new(descr=ssize)
+        setfield_gc(p1, i1, descr=valuedescr)
+        setfield_gc(p0, p1, descr=adescr)
+        call(p0, descr=writeadescr)
+        i2 = getfield_gc(p1, descr=valuedescr)
+        jump(i2, p0)
+        """
+        expected = """
+        [i1, p0]
+        p1 = new(descr=ssize)
+        setfield_gc(p1, i1, descr=valuedescr)
+        setfield_gc(p0, p1, descr=adescr)
+        call(p0, descr=writeadescr)
+        jump(i1, p0)
+        """
+        self.optimize_loop(ops, expected)
+
 class TestLLtype(OptimizeOptTest, LLtypeMixin):
     pass
 
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_util.py b/pypy/jit/metainterp/optimizeopt/test/test_util.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_util.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_util.py
@@ -182,7 +182,8 @@
                             EffectInfo.EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE,
                             can_invalidate=True))
     arraycopydescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT,
-             EffectInfo([], [], [], [], oopspecindex=EffectInfo.OS_ARRAYCOPY))
+             EffectInfo([], [arraydescr], [], [arraydescr],
+                        oopspecindex=EffectInfo.OS_ARRAYCOPY))
 
     for _name, _os in [
         ('strconcatdescr',               'OS_STR_CONCAT'),
diff --git a/pypy/jit/metainterp/optimizeopt/unroll.py b/pypy/jit/metainterp/optimizeopt/unroll.py
--- a/pypy/jit/metainterp/optimizeopt/unroll.py
+++ b/pypy/jit/metainterp/optimizeopt/unroll.py
@@ -75,7 +75,6 @@
         self.importable_values = {}
         self.emitting_dissabled = False
         self.emitted_guards = 0
-        self.emitted_pure_operations = {}
 
     def ensure_imported(self, value):
         if not self.emitting_dissabled and value in self.importable_values:
@@ -96,21 +95,6 @@
         new = UnrollableOptimizer(self.metainterp_sd, self.loop)
         return self._new(new)
 
-    def remember_emitting_pure(self, op):
-        self.emitted_pure_operations[op] = True
-
-    def produce_potential_short_preamble_ops(self, sb):
-        for op in self.emitted_pure_operations:
-            if op.getopnum() == rop.GETARRAYITEM_GC_PURE or \
-               op.getopnum() == rop.STRGETITEM or \
-               op.getopnum() == rop.UNICODEGETITEM:
-                if not self.getvalue(op.getarg(1)).is_constant():
-                    continue
-            sb.add_potential(op)
-        for opt in self.optimizations:
-            opt.produce_potential_short_preamble_ops(sb)
-
-
 
 class UnrollOptimizer(Optimization):
     """Unroll the loop into two iterations. The first one will
@@ -154,7 +138,7 @@
 
             KillHugeIntBounds(self.optimizer).apply()
             
-            loop.preamble.operations = self.optimizer.newoperations
+            loop.preamble.operations = self.optimizer.get_newoperations()
             jump_args = [self.getvalue(a).get_key_box() for a in jump_args]
 
             start_resumedescr = loop.preamble.start_resumedescr.clone_if_mutable()
@@ -167,8 +151,9 @@
             virtual_state = modifier.get_virtual_state(jump_args)
             
             values = [self.getvalue(arg) for arg in jump_args]
-            inputargs = virtual_state.make_inputargs(values)
-            short_inputargs = virtual_state.make_inputargs(values, keyboxes=True)
+            inputargs = virtual_state.make_inputargs(values, self.optimizer)
+            short_inputargs = virtual_state.make_inputargs(values, self.optimizer,
+                                                           keyboxes=True)
 
             self.constant_inputargs = {}
             for box in jump_args: 
@@ -197,7 +182,7 @@
             # operations needed to setup the proper state of those virtuals
             # in the peeled loop
             inputarg_setup_ops = []
-            preamble_optimizer.newoperations = []
+            preamble_optimizer.clear_newoperations()
             seen = {}
             for box in inputargs:
                 if box in seen:
@@ -211,9 +196,8 @@
                     continue
                 seen[box] = True
                 value = preamble_optimizer.getvalue(box)
-                value.force_box()
-            preamble_optimizer.flush()
-            inputarg_setup_ops += preamble_optimizer.newoperations
+                value.force_box(preamble_optimizer)
+            inputarg_setup_ops += preamble_optimizer.get_newoperations()
 
             # Setup the state of the new optimizer by emiting the
             # short preamble operations and discarding the result
@@ -244,13 +228,13 @@
                                 virtual_state)
             
             loop.inputargs = inputargs
-            args = [preamble_optimizer.getvalue(self.short_boxes.original(a)).force_box()\
+            args = [preamble_optimizer.getvalue(self.short_boxes.original(a)).force_box(preamble_optimizer)\
                     for a in inputargs]
             jmp = ResOperation(rop.JUMP, args, None)
             jmp.setdescr(loop.token)
             loop.preamble.operations.append(jmp)
 
-            loop.operations = self.optimizer.newoperations
+            loop.operations = self.optimizer.get_newoperations()
             maxguards = self.optimizer.metainterp_sd.warmrunnerdesc.memory_manager.max_retrace_guards
             
             if self.optimizer.emitted_guards > maxguards:
@@ -335,9 +319,10 @@
         assert jumpop
         original_jumpargs = jumpop.getarglist()[:]
         values = [self.getvalue(arg) for arg in jumpop.getarglist()]
-        jumpargs = virtual_state.make_inputargs(values)
+        jumpargs = virtual_state.make_inputargs(values, self.optimizer)
         jumpop.initarglist(jumpargs)
-        jmp_to_short_args = virtual_state.make_inputargs(values, keyboxes=True)
+        jmp_to_short_args = virtual_state.make_inputargs(values, self.optimizer,
+                                                         keyboxes=True)
         self.short_inliner = Inliner(short_inputargs, jmp_to_short_args)
         
         for box, const in self.constant_inputargs.items():
@@ -347,11 +332,11 @@
             newop = self.short_inliner.inline_op(op)
             self.optimizer.send_extra_operation(newop)
         
-        self.optimizer.flush()
+        newoperations = self.optimizer.get_newoperations()
 
         i = j = 0
-        while i < len(self.optimizer.newoperations) or j < len(jumpargs):
-            if i == len(self.optimizer.newoperations):
+        while i < len(newoperations) or j < len(jumpargs):
+            if i == len(newoperations):
                 while j < len(jumpargs):
                     a = jumpargs[j]
                     if self.optimizer.loop.logops:
@@ -360,7 +345,7 @@
                                     jumpargs, short_seen)
                     j += 1
             else:
-                op = self.optimizer.newoperations[i]
+                op = newoperations[i]
 
                 self.boxes_created_this_iteration[op.result] = True
                 args = op.getarglist()
@@ -375,6 +360,7 @@
                     self.import_box(a, inputargs, short, short_jumpargs,
                                     jumpargs, short_seen)
                 i += 1
+            newoperations = self.optimizer.get_newoperations()
 
         jumpop.initarglist(jumpargs)
         self.optimizer.send_extra_operation(jumpop)
@@ -468,7 +454,7 @@
         inputargs.append(box)
         box = newresult
         if box in self.optimizer.values:
-            box = self.optimizer.values[box].force_box()
+            box = self.optimizer.values[box].force_box(self.optimizer)
         jumpargs.append(box)
         
 
@@ -483,11 +469,6 @@
         if op.getopnum() == rop.JUMP:
             loop_token = op.getdescr()
             assert isinstance(loop_token, LoopToken)
-            # FIXME: Use a tree, similar to the tree formed by the full
-            # preamble and it's bridges, instead of a list to save time and
-            # memory. This should also allow better behaviour in
-            # situations that the is_emittable() chain currently cant
-            # handle and the inlining fails unexpectedly belwo.
             short = loop_token.short_preamble
             if short:
                 args = op.getarglist()
@@ -523,7 +504,7 @@
 
                         values = [self.getvalue(arg)
                                   for arg in op.getarglist()]
-                        args = sh.virtual_state.make_inputargs(values,
+                        args = sh.virtual_state.make_inputargs(values, self.optimizer,
                                                                keyboxes=True)
                         inliner = Inliner(sh.inputargs, args)
                         
diff --git a/pypy/jit/metainterp/optimizeopt/virtualize.py b/pypy/jit/metainterp/optimizeopt/virtualize.py
--- a/pypy/jit/metainterp/optimizeopt/virtualize.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualize.py
@@ -10,13 +10,12 @@
 
 
 class AbstractVirtualValue(optimizer.OptValue):
-    _attrs_ = ('optimizer', 'keybox', 'source_op', '_cached_vinfo')
+    _attrs_ = ('keybox', 'source_op', '_cached_vinfo')
     box = None
     level = optimizer.LEVEL_NONNULL
     _cached_vinfo = None
 
-    def __init__(self, optimizer, keybox, source_op=None):
-        self.optimizer = optimizer
+    def __init__(self, keybox, source_op=None):
         self.keybox = keybox   # only used as a key in dictionaries
         self.source_op = source_op  # the NEW_WITH_VTABLE/NEW_ARRAY operation
                                     # that builds this box
@@ -29,17 +28,17 @@
             return self.keybox
         return self.box
 
-    def force_box(self):
+    def force_box(self, optforce):
         if self.box is None:
-            self.optimizer.forget_numberings(self.keybox)
-            self._really_force()
+            optforce.forget_numberings(self.keybox)
+            self._really_force(optforce)
         return self.box
 
-    def force_at_end_of_preamble(self, already_forced):
+    def force_at_end_of_preamble(self, already_forced, optforce):
         value = already_forced.get(self, None)
         if value:
             return value
-        return OptValue(self.force_box())
+        return OptValue(self.force_box(optforce))
 
     def make_virtual_info(self, modifier, fieldnums):
         if fieldnums is None:
@@ -55,7 +54,7 @@
     def _make_virtual(self, modifier):
         raise NotImplementedError("abstract base")
 
-    def _really_force(self):
+    def _really_force(self, optforce):
         raise NotImplementedError("abstract base")
 
     def import_from(self, other, optimizer):
@@ -70,10 +69,11 @@
 get_fielddescrlist_cache._annspecialcase_ = "specialize:memo"
 
 class AbstractVirtualStructValue(AbstractVirtualValue):
-    _attrs_ = ('_fields', '_cached_sorted_fields')
+    _attrs_ = ('_fields', 'cpu', '_cached_sorted_fields')
 
-    def __init__(self, optimizer, keybox, source_op=None):
-        AbstractVirtualValue.__init__(self, optimizer, keybox, source_op)
+    def __init__(self, cpu, keybox, source_op=None):
+        AbstractVirtualValue.__init__(self, keybox, source_op)
+        self.cpu = cpu
         self._fields = {}
         self._cached_sorted_fields = None
 
@@ -87,45 +87,44 @@
     def _get_descr(self):
         raise NotImplementedError
 
-    def _is_immutable_and_filled_with_constants(self):
+    def _is_immutable_and_filled_with_constants(self, optforce):
         count = self._get_descr().count_fields_if_immutable()
         if count != len(self._fields):    # always the case if count == -1
             return False
         for value in self._fields.itervalues():
-            subbox = value.force_box()
+            subbox = value.force_box(optforce)
             if not isinstance(subbox, Const):
                 return False
         return True
 
-    def force_at_end_of_preamble(self, already_forced):
+    def force_at_end_of_preamble(self, already_forced, optforce):
         if self in already_forced:
             return self
         already_forced[self] = self
         if self._fields:
             for ofs in self._fields.keys():
-                self._fields[ofs] = self._fields[ofs].force_at_end_of_preamble(already_forced)
+                self._fields[ofs] = self._fields[ofs].force_at_end_of_preamble(already_forced, optforce)
         return self
 
-    def _really_force(self):
+    def _really_force(self, optforce):
         op = self.source_op
         assert op is not None
         # ^^^ This case should not occur any more (see test_bug_3).
         #
         if not we_are_translated():
             op.name = 'FORCE ' + self.source_op.name
-
-        if self._is_immutable_and_filled_with_constants():
-            box = self.optimizer.constant_fold(op)
+            
+        if self._is_immutable_and_filled_with_constants(optforce):
+            box = optforce.optimizer.constant_fold(op)
             self.make_constant(box)
             for ofs, value in self._fields.iteritems():
-                subbox = value.force_box()
+                subbox = value.force_box(optforce)
                 assert isinstance(subbox, Const)
-                execute(self.optimizer.cpu, None, rop.SETFIELD_GC,
+                execute(optforce.optimizer.cpu, None, rop.SETFIELD_GC,
                         ofs, box, subbox)
             # keep self._fields, because it's all immutable anyway
         else:
-            newoperations = self.optimizer.newoperations
-            newoperations.append(op)
+            optforce.emit_operation(op)
             self.box = box = op.result
             #
             iteritems = self._fields.iteritems()
@@ -135,10 +134,11 @@
             for ofs, value in iteritems:
                 if value.is_null():
                     continue
-                subbox = value.force_box()
+                subbox = value.force_box(optforce)
                 op = ResOperation(rop.SETFIELD_GC, [box, subbox], None,
                                   descr=ofs)
-                newoperations.append(op)
+
+                optforce.emit_operation(op)
 
     def _get_field_descr_list(self):
         _cached_sorted_fields = self._cached_sorted_fields
@@ -155,7 +155,7 @@
             else:
                 lst = self._fields.keys()
             sort_descrs(lst)
-            cache = get_fielddescrlist_cache(self.optimizer.cpu)
+            cache = get_fielddescrlist_cache(self.cpu)
             result = cache.get(lst, None)
             if result is None:
                 cache[lst] = lst
@@ -180,8 +180,8 @@
 class VirtualValue(AbstractVirtualStructValue):
     level = optimizer.LEVEL_KNOWNCLASS
 
-    def __init__(self, optimizer, known_class, keybox, source_op=None):
-        AbstractVirtualStructValue.__init__(self, optimizer, keybox, source_op)
+    def __init__(self, cpu, known_class, keybox, source_op=None):
+        AbstractVirtualStructValue.__init__(self, cpu, keybox, source_op)
         assert isinstance(known_class, Const)
         self.known_class = known_class
 
@@ -190,7 +190,7 @@
         return modifier.make_virtual(self.known_class, fielddescrs)
 
     def _get_descr(self):
-        return vtable2descr(self.optimizer.cpu, self.known_class.getint())
+        return vtable2descr(self.cpu, self.known_class.getint())
 
     def __repr__(self):
         cls_name = self.known_class.value.adr.ptr._obj._TYPE._name
@@ -201,8 +201,8 @@
 
 class VStructValue(AbstractVirtualStructValue):
 
-    def __init__(self, optimizer, structdescr, keybox, source_op=None):
-        AbstractVirtualStructValue.__init__(self, optimizer, keybox, source_op)
+    def __init__(self, cpu, structdescr, keybox, source_op=None):
+        AbstractVirtualStructValue.__init__(self, cpu, keybox, source_op)
         self.structdescr = structdescr
 
     def _make_virtual(self, modifier):
@@ -215,10 +215,10 @@
 
 class VArrayValue(AbstractVirtualValue):
 
-    def __init__(self, optimizer, arraydescr, size, keybox, source_op=None):
-        AbstractVirtualValue.__init__(self, optimizer, keybox, source_op)
+    def __init__(self, arraydescr, constvalue, size, keybox, source_op=None):
+        AbstractVirtualValue.__init__(self, keybox, source_op)
         self.arraydescr = arraydescr
-        self.constvalue = optimizer.new_const_item(arraydescr)
+        self.constvalue = constvalue
         self._items = [self.constvalue] * size
 
     def getlength(self):
@@ -232,31 +232,30 @@
         assert isinstance(itemvalue, optimizer.OptValue)
         self._items[index] = itemvalue
 
-    def force_at_end_of_preamble(self, already_forced):
+    def force_at_end_of_preamble(self, already_forced, optforce):
         if self in already_forced:
             return self
         already_forced[self] = self
         for index in range(len(self._items)):
-            self._items[index] = self._items[index].force_at_end_of_preamble(already_forced)
+            self._items[index] = self._items[index].force_at_end_of_preamble(already_forced, optforce)
         return self
     
-    def _really_force(self):
+    def _really_force(self, optforce):
         assert self.source_op is not None
         if not we_are_translated():
             self.source_op.name = 'FORCE ' + self.source_op.name
-        newoperations = self.optimizer.newoperations
-        newoperations.append(self.source_op)
+        optforce.emit_operation(self.source_op)        
         self.box = box = self.source_op.result
         for index in range(len(self._items)):
             subvalue = self._items[index]
             if subvalue is not self.constvalue:
                 if subvalue.is_null():
                     continue
-                subbox = subvalue.force_box()
+                subbox = subvalue.force_box(optforce)
                 op = ResOperation(rop.SETARRAYITEM_GC,
                                   [box, ConstInt(index), subbox], None,
                                   descr=self.arraydescr)
-                newoperations.append(op)
+                optforce.emit_operation(op)
 
     def get_args_for_fail(self, modifier):
         if self.box is None and not modifier.already_seen_virtual(self.keybox):
@@ -279,17 +278,18 @@
         return OptVirtualize()
         
     def make_virtual(self, known_class, box, source_op=None):
-        vvalue = VirtualValue(self.optimizer, known_class, box, source_op)
+        vvalue = VirtualValue(self.optimizer.cpu, known_class, box, source_op)
         self.make_equal_to(box, vvalue)
         return vvalue
 
     def make_varray(self, arraydescr, size, box, source_op=None):
-        vvalue = VArrayValue(self.optimizer, arraydescr, size, box, source_op)
+        constvalue = self.new_const_item(arraydescr)
+        vvalue = VArrayValue(arraydescr, constvalue, size, box, source_op)
         self.make_equal_to(box, vvalue)
         return vvalue
 
     def make_vstruct(self, structdescr, box, source_op=None):
-        vvalue = VStructValue(self.optimizer, structdescr, box, source_op)
+        vvalue = VStructValue(self.optimizer.cpu, structdescr, box, source_op)
         self.make_equal_to(box, vvalue)
         return vvalue
 
@@ -362,7 +362,6 @@
             self.make_equal_to(op.result, fieldvalue)
         else:
             value.ensure_nonnull()
-            ###self.heap_op_optimizer.optimize_GETFIELD_GC(op, value)
             self.emit_operation(op)
 
     # note: the following line does not mean that the two operations are
@@ -377,7 +376,6 @@
             value.setfield(op.getdescr(), fieldvalue)
         else:
             value.ensure_nonnull()
-            ###self.heap_op_optimizer.optimize_SETFIELD_GC(op, value, fieldvalue)
             self.emit_operation(op)
 
     def optimize_NEW_WITH_VTABLE(self, op):
@@ -417,7 +415,6 @@
                 self.make_equal_to(op.result, itemvalue)
                 return
         value.ensure_nonnull()
-        ###self.heap_op_optimizer.optimize_GETARRAYITEM_GC(op, value)
         self.emit_operation(op)
 
     # note: the following line does not mean that the two operations are
@@ -432,7 +429,6 @@
                 value.setitem(indexbox.getint(), self.getvalue(op.getarg(2)))
                 return
         value.ensure_nonnull()
-        ###self.heap_op_optimizer.optimize_SETARRAYITEM_GC(op, value, fieldvalue)
         self.emit_operation(op)
 
 
diff --git a/pypy/jit/metainterp/optimizeopt/virtualstate.py b/pypy/jit/metainterp/optimizeopt/virtualstate.py
--- a/pypy/jit/metainterp/optimizeopt/virtualstate.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualstate.py
@@ -30,7 +30,7 @@
     def _generate_guards(self, other, box, cpu, extra_guards):
         raise InvalidLoop
 
-    def enum_forced_boxes(self, boxes, value):
+    def enum_forced_boxes(self, boxes, value, optimizer):
         raise NotImplementedError
 
     def enum(self, virtual_state):
@@ -100,14 +100,14 @@
     def _generalization_of(self, other):
         raise NotImplementedError
 
-    def enum_forced_boxes(self, boxes, value):
+    def enum_forced_boxes(self, boxes, value, optimizer):
         assert isinstance(value, virtualize.AbstractVirtualStructValue)
         assert value.is_virtual()
         for i in range(len(self.fielddescrs)):
             v = value._fields[self.fielddescrs[i]]
             s = self.fieldstate[i]
             if s.position > self.position:
-                s.enum_forced_boxes(boxes, v)
+                s.enum_forced_boxes(boxes, v, optimizer)
 
     def _enum(self, virtual_state):
         for s in self.fieldstate:
@@ -177,14 +177,14 @@
                 return False
         return True
 
-    def enum_forced_boxes(self, boxes, value):
+    def enum_forced_boxes(self, boxes, value, optimizer):
         assert isinstance(value, virtualize.VArrayValue)
         assert value.is_virtual()
         for i in range(len(self.fieldstate)):
             v = value._items[i]
             s = self.fieldstate[i]
             if s.position > self.position:
-                s.enum_forced_boxes(boxes, v)
+                s.enum_forced_boxes(boxes, v, optimizer)
 
     def _enum(self, virtual_state):
         for s in self.fieldstate:
@@ -316,11 +316,11 @@
             import pdb; pdb.set_trace()
             raise NotImplementedError
 
-    def enum_forced_boxes(self, boxes, value):
+    def enum_forced_boxes(self, boxes, value, optimizer):
         if self.level == LEVEL_CONSTANT:
             return
         assert 0 <= self.position_in_notvirtuals 
-        boxes[self.position_in_notvirtuals] = value.force_box()
+        boxes[self.position_in_notvirtuals] = value.force_box(optimizer)
 
     def _enum(self, virtual_state):
         if self.level == LEVEL_CONSTANT:
@@ -377,11 +377,13 @@
             self.state[i].generate_guards(other.state[i], args[i],
                                           cpu, extra_guards, renum)
 
-    def make_inputargs(self, values, keyboxes=False):
+    def make_inputargs(self, values, optimizer, keyboxes=False):
+        if optimizer.optearlyforce:
+            optimizer = optimizer.optearlyforce
         assert len(values) == len(self.state)
         inputargs = [None] * len(self.notvirtuals)
         for i in range(len(values)):
-            self.state[i].enum_forced_boxes(inputargs, values[i])
+            self.state[i].enum_forced_boxes(inputargs, values[i], optimizer)
 
         if keyboxes:
             for i in range(len(values)):
@@ -434,7 +436,12 @@
     def get_virtual_state(self, jump_args):
         self.optimizer.force_at_end_of_preamble()
         already_forced = {}
-        values = [self.getvalue(box).force_at_end_of_preamble(already_forced)
+        if self.optimizer.optearlyforce:
+            opt = self.optimizer.optearlyforce
+        else:
+            opt = self.optimizer
+        values = [self.getvalue(box).force_at_end_of_preamble(already_forced,
+                                                              opt)
                   for box in jump_args]
 
         for value in values:
diff --git a/pypy/jit/metainterp/optimizeopt/vstring.py b/pypy/jit/metainterp/optimizeopt/vstring.py
--- a/pypy/jit/metainterp/optimizeopt/vstring.py
+++ b/pypy/jit/metainterp/optimizeopt/vstring.py
@@ -43,7 +43,7 @@
 class __extend__(optimizer.OptValue):
     """New methods added to the base class OptValue for this file."""
 
-    def getstrlen(self, optimization, mode):
+    def getstrlen(self, string_optimizer, mode):
         if mode is mode_string:
             s = self.get_constant_string_spec(mode_string)
             if s is not None:
@@ -52,12 +52,12 @@
             s = self.get_constant_string_spec(mode_unicode)
             if s is not None:
                 return ConstInt(len(s))
-        if optimization is None:
+        if string_optimizer is None:
             return None
         self.ensure_nonnull()
-        box = self.force_box()
+        box = self.force_box(string_optimizer)
         lengthbox = BoxInt()
-        optimization.propagate_forward(ResOperation(mode.STRLEN, [box], lengthbox))
+        string_optimizer.emit_operation(ResOperation(mode.STRLEN, [box], lengthbox))
         return lengthbox
 
     @specialize.arg(1)
@@ -68,25 +68,25 @@
         else:
             return None
 
-    def string_copy_parts(self, optimizer, targetbox, offsetbox, mode):
+    def string_copy_parts(self, string_optimizer, targetbox, offsetbox, mode):
         # Copies the pointer-to-string 'self' into the target string
         # given by 'targetbox', at the specified offset.  Returns the offset
         # at the end of the copy.
-        lengthbox = self.getstrlen(optimizer, mode)
-        srcbox = self.force_box()
-        return copy_str_content(optimizer, srcbox, targetbox,
+        lengthbox = self.getstrlen(string_optimizer, mode)
+        srcbox = self.force_box(string_optimizer)
+        return copy_str_content(string_optimizer, srcbox, targetbox,
                                 CONST_0, offsetbox, lengthbox, mode)
 
 
 class VAbstractStringValue(virtualize.AbstractVirtualValue):
     _attrs_ = ('mode',)
 
-    def __init__(self, optimizer, keybox, source_op, mode):
-        virtualize.AbstractVirtualValue.__init__(self, optimizer, keybox,
+    def __init__(self, keybox, source_op, mode):
+        virtualize.AbstractVirtualValue.__init__(self, keybox,
                                                  source_op)
         self.mode = mode
 
-    def _really_force(self):
+    def _really_force(self, optforce):
         if self.mode is mode_string:
             s = self.get_constant_string_spec(mode_string)
             if s is not None:
@@ -101,12 +101,12 @@
                 return
         assert self.source_op is not None
         self.box = box = self.source_op.result
-        lengthbox = self.getstrlen(self.optimizer, self.mode)
+        lengthbox = self.getstrlen(optforce, self.mode)
         op = ResOperation(self.mode.NEWSTR, [lengthbox], box)
         if not we_are_translated():
             op.name = 'FORCE'
-        self.optimizer.emit_operation(op)
-        self.string_copy_parts(self.optimizer, box, CONST_0, self.mode)
+        optforce.emit_operation(op)
+        self.string_copy_parts(optforce, box, CONST_0, self.mode)
 
 
 class VStringPlainValue(VAbstractStringValue):
@@ -140,15 +140,20 @@
         return mode.emptystr.join([mode.chr(c.box.getint())
                                    for c in self._chars])
 
-    def string_copy_parts(self, optimizer, targetbox, offsetbox, mode):
+    def string_copy_parts(self, string_optimizer, targetbox, offsetbox, mode):
+        if not self.is_virtual() and targetbox is not self.box:
+            lengthbox = self.getstrlen(string_optimizer, mode)
+            srcbox = self.force_box(string_optimizer)
+            return copy_str_content(string_optimizer, srcbox, targetbox,
+                                CONST_0, offsetbox, lengthbox, mode)
         for i in range(len(self._chars)):
-            charbox = self._chars[i].force_box()
+            charbox = self._chars[i].force_box(string_optimizer)
             if not (isinstance(charbox, Const) and charbox.same_constant(CONST_0)):
-                optimizer.emit_operation(ResOperation(mode.STRSETITEM, [targetbox,
-                                                                    offsetbox,
-                                                                    charbox],
+                string_optimizer.emit_operation(ResOperation(mode.STRSETITEM, [targetbox,
+                                                                               offsetbox,
+                                                                               charbox],
                                                   None))
-            offsetbox = _int_add(optimizer, offsetbox, CONST_1)
+            offsetbox = _int_add(string_optimizer, offsetbox, CONST_1)
         return offsetbox
 
     def get_args_for_fail(self, modifier):
@@ -182,16 +187,16 @@
         self.left = left
         self.right = right
 
-    def getstrlen(self, optimizer, mode):
+    def getstrlen(self, string_optimizer, mode):
         if self.lengthbox is None:
-            len1box = self.left.getstrlen(optimizer, mode)
+            len1box = self.left.getstrlen(string_optimizer, mode)
             if len1box is None:
                 return None
-            len2box = self.right.getstrlen(optimizer, mode)
+            len2box = self.right.getstrlen(string_optimizer, mode)
             if len2box is None:
                 return None
-            self.lengthbox = _int_add(optimizer, len1box, len2box)
-            # ^^^ may still be None, if optimizer is None
+            self.lengthbox = _int_add(string_optimizer, len1box, len2box)
+            # ^^^ may still be None, if string_optimizer is None
         return self.lengthbox
 
     @specialize.arg(1)
@@ -204,10 +209,10 @@
             return None
         return s1 + s2
 
-    def string_copy_parts(self, optimizer, targetbox, offsetbox, mode):
-        offsetbox = self.left.string_copy_parts(optimizer, targetbox,
+    def string_copy_parts(self, string_optimizer, targetbox, offsetbox, mode):
+        offsetbox = self.left.string_copy_parts(string_optimizer, targetbox,
                                                 offsetbox, mode)
-        offsetbox = self.right.string_copy_parts(optimizer, targetbox,
+        offsetbox = self.right.string_copy_parts(string_optimizer, targetbox,
                                                  offsetbox, mode)
         return offsetbox
 
@@ -246,8 +251,8 @@
         self.vstart = vstart
         self.vlength = vlength
 
-    def getstrlen(self, _, mode):
-        return self.vlength.force_box()
+    def getstrlen(self, optforce, mode):
+        return self.vlength.force_box(optforce)
 
     @specialize.arg(1)
     def get_constant_string_spec(self, mode):
@@ -262,11 +267,11 @@
             return s1[start : start + length]
         return None
 
-    def string_copy_parts(self, optimizer, targetbox, offsetbox, mode):
-        lengthbox = self.getstrlen(optimizer, mode)
-        return copy_str_content(optimizer,
-                                self.vstr.force_box(), targetbox,
-                                self.vstart.force_box(), offsetbox,
+    def string_copy_parts(self, string_optimizer, targetbox, offsetbox, mode):
+        lengthbox = self.getstrlen(string_optimizer, mode)
+        return copy_str_content(string_optimizer,
+                                self.vstr.force_box(string_optimizer), targetbox,
+                                self.vstart.force_box(string_optimizer), offsetbox,
                                 lengthbox, mode)
 
     def get_args_for_fail(self, modifier):
@@ -295,7 +300,7 @@
         return modifier.make_vstrslice(self.mode is mode_unicode)
 
 
-def copy_str_content(optimizer, srcbox, targetbox,
+def copy_str_content(string_optimizer, srcbox, targetbox,
                      srcoffsetbox, offsetbox, lengthbox, mode, need_next_offset=True):
     if isinstance(srcbox, ConstPtr) and isinstance(srcoffsetbox, Const):
         M = 5
@@ -305,26 +310,26 @@
         # up to M characters are done "inline", i.e. with STRGETITEM/STRSETITEM
         # instead of just a COPYSTRCONTENT.
         for i in range(lengthbox.value):
-            charbox = _strgetitem(optimizer, srcbox, srcoffsetbox, mode)
-            srcoffsetbox = _int_add(optimizer, srcoffsetbox, CONST_1)
-            optimizer.emit_operation(ResOperation(mode.STRSETITEM, [targetbox,
-                                                                offsetbox,
-                                                                charbox],
+            charbox = _strgetitem(string_optimizer, srcbox, srcoffsetbox, mode)
+            srcoffsetbox = _int_add(string_optimizer, srcoffsetbox, CONST_1)
+            string_optimizer.emit_operation(ResOperation(mode.STRSETITEM, [targetbox,
+                                                                           offsetbox,
+                                                                           charbox],
                                               None))
-            offsetbox = _int_add(optimizer, offsetbox, CONST_1)
+            offsetbox = _int_add(string_optimizer, offsetbox, CONST_1)
     else:
         if need_next_offset:
-            nextoffsetbox = _int_add(optimizer, offsetbox, lengthbox)
+            nextoffsetbox = _int_add(string_optimizer, offsetbox, lengthbox)
         else:
             nextoffsetbox = None
         op = ResOperation(mode.COPYSTRCONTENT, [srcbox, targetbox,
                                                 srcoffsetbox, offsetbox,
                                                 lengthbox], None)
-        optimizer.emit_operation(op)
+        string_optimizer.emit_operation(op)
         offsetbox = nextoffsetbox
     return offsetbox
 
-def _int_add(optimizer, box1, box2):
+def _int_add(string_optimizer, box1, box2):
     if isinstance(box1, ConstInt):
         if box1.value == 0:
             return box2
@@ -332,23 +337,23 @@
             return ConstInt(box1.value + box2.value)
     elif isinstance(box2, ConstInt) and box2.value == 0:
         return box1
-    if optimizer is None:
+    if string_optimizer is None:
         return None
     resbox = BoxInt()
-    optimizer.propagate_forward(ResOperation(rop.INT_ADD, [box1, box2], resbox))
+    string_optimizer.emit_operation(ResOperation(rop.INT_ADD, [box1, box2], resbox))
     return resbox
 
-def _int_sub(optimizer, box1, box2):
+def _int_sub(string_optimizer, box1, box2):
     if isinstance(box2, ConstInt):
         if box2.value == 0:
             return box1
         if isinstance(box1, ConstInt):
             return ConstInt(box1.value - box2.value)
     resbox = BoxInt()
-    optimizer.propagate_forward(ResOperation(rop.INT_SUB, [box1, box2], resbox))
+    string_optimizer.emit_operation(ResOperation(rop.INT_SUB, [box1, box2], resbox))
     return resbox
 
-def _strgetitem(optimizer, strbox, indexbox, mode):
+def _strgetitem(string_optimizer, strbox, indexbox, mode):
     if isinstance(strbox, ConstPtr) and isinstance(indexbox, ConstInt):
         if mode is mode_string:
             s = strbox.getref(lltype.Ptr(rstr.STR))
@@ -357,30 +362,28 @@
             s = strbox.getref(lltype.Ptr(rstr.UNICODE))
             return ConstInt(ord(s.chars[indexbox.getint()]))
     resbox = BoxInt()
-    optimizer.propagate_forward(ResOperation(mode.STRGETITEM, [strbox, indexbox],
-                                      resbox))
+    string_optimizer.emit_operation(ResOperation(mode.STRGETITEM, [strbox, indexbox],
+                                                 resbox))
     return resbox
 
 
 class OptString(optimizer.Optimization):
     "Handling of strings and unicodes."
-    enabled = True
-
     def new(self):
         return OptString()
 
     def make_vstring_plain(self, box, source_op, mode):
-        vvalue = VStringPlainValue(self.optimizer, box, source_op, mode)
+        vvalue = VStringPlainValue(box, source_op, mode)
         self.make_equal_to(box, vvalue)
         return vvalue
 
     def make_vstring_concat(self, box, source_op, mode):
-        vvalue = VStringConcatValue(self.optimizer, box, source_op, mode)
+        vvalue = VStringConcatValue(box, source_op, mode)
         self.make_equal_to(box, vvalue)
         return vvalue
 
     def make_vstring_slice(self, box, source_op, mode):
-        vvalue = VStringSliceValue(self.optimizer, box, source_op, mode)
+        vvalue = VStringSliceValue(box, source_op, mode)
         self.make_equal_to(box, vvalue)
         return vvalue
 
@@ -430,9 +433,9 @@
         value.ensure_nonnull()
         #
         if value.is_virtual() and isinstance(value, VStringSliceValue):
-            fullindexbox = _int_add(self.optimizer,
-                                    value.vstart.force_box(),
-                                    vindex.force_box())
+            fullindexbox = _int_add(self,
+                                    value.vstart.force_box(self),
+                                    vindex.force_box(self))
             value = value.vstr
             vindex = self.getvalue(fullindexbox)
         #
@@ -444,7 +447,7 @@
                 if res is not optimizer.CVAL_UNINITIALIZED_ZERO:
                     return res
         #
-        resbox = _strgetitem(self.optimizer, value.force_box(), vindex.force_box(), mode)
+        resbox = _strgetitem(self, value.force_box(self), vindex.force_box(self), mode)
         return self.getvalue(resbox)
 
     def optimize_STRLEN(self, op):
@@ -454,7 +457,7 @@
 
     def _optimize_STRLEN(self, op, mode):
         value = self.getvalue(op.getarg(0))
-        lengthbox = value.getstrlen(self.optimizer, mode)
+        lengthbox = value.getstrlen(self, mode)
         self.make_equal_to(op.result, self.getvalue(lengthbox))
 
     def optimize_COPYSTRCONTENT(self, op):
@@ -472,12 +475,12 @@
 
         if length.is_constant() and length.box.getint() == 0:
             return
-        copy_str_content(self.optimizer,
-            src.force_box(),
-            dst.force_box(),
-            srcstart.force_box(),
-            dststart.force_box(),
-            length.force_box(),
+        copy_str_content(self,
+            src.force_box(self),
+            dst.force_box(self),
+            srcstart.force_box(self),
+            dststart.force_box(self),
+            length.force_box(self),
             mode, need_next_offset=False
         )
 
@@ -503,6 +506,8 @@
                     return
         self.emit_operation(op)
 
+    optimize_CALL_PURE = optimize_CALL
+
     def opt_call_str_STR2UNICODE(self, op):
         # Constant-fold unicode("constant string").
         # More generally, supporting non-constant but virtual cases is
@@ -542,16 +547,16 @@
             return True
         #
         vstr.ensure_nonnull()
-        lengthbox = _int_sub(self.optimizer, vstop.force_box(),
-                                            vstart.force_box())
+        lengthbox = _int_sub(self, vstop.force_box(self),
+                                   vstart.force_box(self))
         #
         if isinstance(vstr, VStringSliceValue):
             # double slicing  s[i:j][k:l]
             vintermediate = vstr
             vstr = vintermediate.vstr
-            startbox = _int_add(self.optimizer,
-                                vintermediate.vstart.force_box(),
-                                vstart.force_box())
+            startbox = _int_add(self,
+                                vintermediate.vstart.force_box(self),
+                                vstart.force_box(self))
             vstart = self.getvalue(startbox)
         #
         value = self.make_vstring_slice(op.result, op, mode)
@@ -589,8 +594,8 @@
                 do = EffectInfo.OS_STREQ_LENGTHOK
             else:
                 do = EffectInfo.OS_STREQ_NONNULL
-            self.generate_modified_call(do, [v1.force_box(),
-                                             v2.force_box()], op.result, mode)
+            self.generate_modified_call(do, [v1.force_box(self),
+                                             v2.force_box(self)], op.result, mode)
             return True
         return False
 
@@ -598,7 +603,7 @@
         l2box = v2.getstrlen(None, mode)
         if isinstance(l2box, ConstInt):
             if l2box.value == 0:
-                lengthbox = v1.getstrlen(self.optimizer, mode)
+                lengthbox = v1.getstrlen(self, mode)
                 seo = self.optimizer.send_extra_operation
                 seo(ResOperation(rop.INT_EQ, [lengthbox, CONST_0], resultbox))
                 return True
@@ -609,17 +614,17 @@
                     vchar1 = self.strgetitem(v1, optimizer.CVAL_ZERO, mode)
                     vchar2 = self.strgetitem(v2, optimizer.CVAL_ZERO, mode)
                     seo = self.optimizer.send_extra_operation
-                    seo(ResOperation(rop.INT_EQ, [vchar1.force_box(),
-                                                  vchar2.force_box()],
+                    seo(ResOperation(rop.INT_EQ, [vchar1.force_box(self),
+                                                  vchar2.force_box(self)],
                                      resultbox))
                     return True
                 if isinstance(v1, VStringSliceValue):
                     vchar = self.strgetitem(v2, optimizer.CVAL_ZERO, mode)
                     do = EffectInfo.OS_STREQ_SLICE_CHAR
-                    self.generate_modified_call(do, [v1.vstr.force_box(),
-                                                     v1.vstart.force_box(),
-                                                     v1.vlength.force_box(),
-                                                     vchar.force_box()],
+                    self.generate_modified_call(do, [v1.vstr.force_box(self),
+                                                     v1.vstart.force_box(self),
+                                                     v1.vlength.force_box(self),
+                                                     vchar.force_box(self)],
                                                 resultbox, mode)
                     return True
         #
@@ -630,10 +635,10 @@
             if v1.is_null():
                 self.make_constant(resultbox, CONST_1)
                 return True
-            op = ResOperation(rop.PTR_EQ, [v1.force_box(),
+            op = ResOperation(rop.PTR_EQ, [v1.force_box(self),
                                            llhelper.CONST_NULL],
                               resultbox)
-            self.optimizer.emit_operation(op)
+            self.emit_operation(op)
             return True
         #
         return False
@@ -647,8 +652,8 @@
                     do = EffectInfo.OS_STREQ_NONNULL_CHAR
                 else:
                     do = EffectInfo.OS_STREQ_CHECKNULL_CHAR
-                self.generate_modified_call(do, [v1.force_box(),
-                                                 vchar.force_box()], resultbox,
+                self.generate_modified_call(do, [v1.force_box(self),
+                                                 vchar.force_box(self)], resultbox,
                                             mode)
                 return True
         #
@@ -657,10 +662,10 @@
                 do = EffectInfo.OS_STREQ_SLICE_NONNULL
             else:
                 do = EffectInfo.OS_STREQ_SLICE_CHECKNULL
-            self.generate_modified_call(do, [v1.vstr.force_box(),
-                                             v1.vstart.force_box(),
-                                             v1.vlength.force_box(),
-                                             v2.force_box()], resultbox, mode)
+            self.generate_modified_call(do, [v1.vstr.force_box(self),
+                                             v1.vstart.force_box(self),
+                                             v1.vlength.force_box(self),
+                                             v2.force_box(self)], resultbox, mode)
             return True
         return False
 
@@ -670,16 +675,11 @@
         calldescr, func = cic.callinfo_for_oopspec(oopspecindex)
         op = ResOperation(rop.CALL, [ConstInt(func)] + args, result,
                           descr=calldescr)
-        self.optimizer.emit_operation(op)
+        self.emit_operation(op)
 
     def propagate_forward(self, op):
-        if not self.enabled:
-            self.emit_operation(op)
-            return
-
         dispatch_opt(self, op)
 
-
 dispatch_opt = make_dispatcher_method(OptString, 'optimize_',
         default=OptString.emit_operation)
 
diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py
--- a/pypy/jit/metainterp/resoperation.py
+++ b/pypy/jit/metainterp/resoperation.py
@@ -221,7 +221,6 @@
         newop.setfailargs(self.getfailargs())
         return newop
 
-
 # ============
 # arity mixins
 # ============
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -2956,6 +2956,18 @@
         assert res == f(32)
         self.check_loops(arraylen_gc=2)
 
+    def test_ulonglong_mod(self):
+        myjitdriver = JitDriver(greens = [], reds = ['n', 'sa', 'i'])
+        def f(n):
+            sa = i = rffi.cast(rffi.ULONGLONG, 1)
+            while i < rffi.cast(rffi.ULONGLONG, n):
+                myjitdriver.jit_merge_point(sa=sa, n=n, i=i)
+                sa += sa % i
+                i += 1
+        res = self.meta_interp(f, [32])
+        assert res == f(32)
+
+
 class TestOOtype(BasicTests, OOJitMixin):
 
     def test_oohash(self):
@@ -3408,6 +3420,75 @@
         assert res == main(1, 10)
         self.check_loops(call=0)
 
+    def test_setarrayitem_followed_by_arraycopy(self):
+        myjitdriver = JitDriver(greens = [], reds = ['n', 'sa', 'x', 'y'])
+        def f(n):
+            sa = 0
+            x = [1,2,n]
+            y = [1,2,3]
+            while n > 0:
+                myjitdriver.jit_merge_point(sa=sa, n=n, x=x, y=y)
+                y[0] = n
+                x[0:3] = y
+                sa += x[0]
+                n -= 1
+            return sa
+        res = self.meta_interp(f, [16])
+        assert res == f(16)
+        
+
 
 class TestLLtype(BaseLLtypeTests, LLJitMixin):
-    pass
+    def test_tagged(self):
+        py.test.skip("implement me")
+        from pypy.rlib.objectmodel import UnboxedValue
+        class Base(object):
+            __slots__ = ()
+
+        class Int(UnboxedValue, Base):
+            __slots__ = ["a"]
+
+            def is_pos(self):
+                return self.a > 0
+
+            def dec(self):
+                return Int(self.a - 1)
+
+
+        class Float(Base):
+            def __init__(self, a):
+                self.a = a
+
+            def is_pos(self):
+                return self.a > 0
+
+            def dec(self):
+                return Float(self.a - 1)
+
+        driver = JitDriver(greens=['pc', 's'], reds=['o'])
+
+        def main(fl, n, s):
+            if s:
+                s = "--j"
+            else:
+                s = "---j"
+            if fl:
+                o = Float(float(n))
+            else:
+                o = Int(n)
+            pc = 0
+            while True:
+                driver.jit_merge_point(s=s, pc=pc, o=o)
+                c = s[pc]
+                if c == "j":
+                    driver.can_enter_jit(s=s, pc=pc, o=o)
+                    if o.is_pos():
+                        pc = 0
+                        continue
+                    else:
+                        break
+                elif c == "-":
+                    o = o.dec()
+                pc += 1
+            return pc
+        res = self.meta_interp(main, [False, 100, True], taggedpointers=True)
diff --git a/pypy/jit/metainterp/test/test_longlong.py b/pypy/jit/metainterp/test/test_longlong.py
--- a/pypy/jit/metainterp/test/test_longlong.py
+++ b/pypy/jit/metainterp/test/test_longlong.py
@@ -118,6 +118,26 @@
         res = self.interp_operations(f, [1000000000])
         assert res == 123500000000.0
 
+    def test_floats_negative(self):
+        def f(i):
+            # i == 1000000000
+            f = i * -123.5
+            n = r_longlong(f)
+            compare(n, -29, 1054051584)
+            return float(n)
+        res = self.interp_operations(f, [1000000000])
+        assert res == -123500000000.0
+
+    def test_floats_ulonglong(self):
+        def f(i):
+            # i == 1000000000
+            f = i * 12350000000.0
+            n = r_ulonglong(f)
+            compare(n, -1419508847, 538116096)
+            return float(n)
+        res = self.interp_operations(f, [1000000000])
+        assert res == 12350000000000000000.0
+
     def test_unsigned_compare_ops(self):
         def f(n1, n2):
             # n == 30002000000000
diff --git a/pypy/jit/metainterp/test/test_resume.py b/pypy/jit/metainterp/test/test_resume.py
--- a/pypy/jit/metainterp/test/test_resume.py
+++ b/pypy/jit/metainterp/test/test_resume.py
@@ -576,8 +576,9 @@
 
 
 class FakeOptimizer_VirtualValue(object):
-    class cpu:
-        pass
+    class optimizer:
+        class cpu:
+            pass
 fakeoptimizer = FakeOptimizer_VirtualValue()
 
 def ConstAddr(addr, cpu):   # compatibility
@@ -1135,12 +1136,7 @@
     modifier.liveboxes = {}
     modifier.vfieldboxes = {}
 
-    class FakeOptimizer(object):
-        class cpu:
-            pass
-        def new_const_item(self, descr):
-            return None
-    v2 = VArrayValue(FakeOptimizer(), LLtypeMixin.arraydescr, 2, b2s)
+    v2 = VArrayValue(LLtypeMixin.arraydescr, None, 2, b2s)
     v2._items = [b4s, c1s]
     modifier.register_virtual_fields(b2s, [b4s, c1s])
     liveboxes = []
diff --git a/pypy/jit/metainterp/test/test_string.py b/pypy/jit/metainterp/test/test_string.py
--- a/pypy/jit/metainterp/test/test_string.py
+++ b/pypy/jit/metainterp/test/test_string.py
@@ -484,6 +484,30 @@
             return len(sa.val)
         assert self.meta_interp(f, ['a']) == f('a')
 
+    def test_string_comepare_quasiimmutable(self):
+        class Sys(object):
+            _immutable_fields_ = ["defaultencoding?"]
+            def __init__(self, s):
+                self.defaultencoding = s
+        _str = self._str
+        sys = Sys(_str('ascii'))        
+        mydriver = JitDriver(reds = ['n', 'sa'], greens = [])
+        def f(n):
+            sa = 0
+            sys.defaultencoding = _str('ascii')
+            while n:
+                mydriver.jit_merge_point(n=n, sa=sa)
+                if sys.defaultencoding == _str('ascii'):
+                    sa += 1
+                n -= 1
+            sys.defaultencoding = _str('utf-8')
+            return sa
+        assert self.meta_interp(f, [8]) == f(8)
+        self.check_loops({'int_add': 1, 'guard_true': 1, 'int_sub': 1,
+                          'jump': 1, 'int_is_true': 1,
+                          'guard_not_invalidated': 1})
+
+
 #class TestOOtype(StringTests, OOJitMixin):
 #    CALL = "oosend"
 #    CALL_PURE = "oosend_pure"
@@ -575,4 +599,18 @@
                 n -= 1
             return result
         res = self.meta_interp(main, [9])
-        assert res == main(9)
\ No newline at end of file
+        assert res == main(9)
+
+    def test_virtual_copystringcontent2(self):
+        jitdriver = JitDriver(reds=['n', 'result'], greens=[])
+        def main(n):
+            result = 0
+            while n >= 0:
+                jitdriver.jit_merge_point(n=n, result=result)
+                b = StringBuilder(6)
+                b.append("Hello!")
+                result += ord((b.build() + "xyz")[0])
+                n -= 1
+            return result
+        res = self.meta_interp(main, [9])
+        assert res == main(9)
diff --git a/pypy/jit/metainterp/test/test_virtualstate.py b/pypy/jit/metainterp/test/test_virtualstate.py
--- a/pypy/jit/metainterp/test/test_virtualstate.py
+++ b/pypy/jit/metainterp/test/test_virtualstate.py
@@ -431,7 +431,7 @@
         
 
 class BaseTestBridges(BaseTest):
-    enable_opts = "intbounds:rewrite:virtualize:string:heap:unroll"
+    enable_opts = "intbounds:rewrite:virtualize:string:pure:heap:unroll"
 
     def _do_optimize_bridge(self, bridge, call_pure_results):
         from pypy.jit.metainterp.optimizeopt import optimize_bridge_1, build_opt_chain
diff --git a/pypy/jit/metainterp/warmspot.py b/pypy/jit/metainterp/warmspot.py
--- a/pypy/jit/metainterp/warmspot.py
+++ b/pypy/jit/metainterp/warmspot.py
@@ -53,6 +53,8 @@
         extraconfigopts = {'translation.list_comprehension_operations': True}
     else:
         extraconfigopts = {}
+    if kwds.pop("taggedpointers", False):
+        extraconfigopts["translation.taggedpointers"] = True
     interp, graph = get_interpreter(function, args,
                                     backendopt=False,  # will be done below
                                     type_system=type_system,
diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py
--- a/pypy/module/__builtin__/functional.py
+++ b/pypy/module/__builtin__/functional.py
@@ -3,13 +3,13 @@
 
 """
 
+from pypy.interpreter.baseobjspace import Wrappable
 from pypy.interpreter.error import OperationError
-from pypy.interpreter.gateway import NoneNotWrapped
-from pypy.interpreter.gateway import interp2app, unwrap_spec
+from pypy.interpreter.gateway import NoneNotWrapped, interp2app, unwrap_spec
 from pypy.interpreter.typedef import TypeDef
-from pypy.interpreter.baseobjspace import Wrappable
+from pypy.rlib import jit
+from pypy.rlib.objectmodel import specialize
 from pypy.rlib.rarithmetic import r_uint, intmask
-from pypy.rlib.objectmodel import specialize
 from pypy.rlib.rbigint import rbigint
 
 
@@ -134,29 +134,15 @@
 
 
 @specialize.arg(2)
+ at jit.look_inside_iff(lambda space, args, implementation_of:
+    jit.isconstant(len(args.arguments_w)) and
+    len(args.arguments_w) == 2
+)
 def min_max(space, args, implementation_of):
     if implementation_of == "max":
         compare = space.gt
     else:
         compare = space.lt
-
-    args_w = args.arguments_w
-    if len(args_w) == 2 and not args.keywords:
-        # simple case, suitable for the JIT
-        w_arg0, w_arg1 = args_w
-        if space.is_true(compare(w_arg0, w_arg1)):
-            return w_arg0
-        else:
-            return w_arg1
-    else:
-        return min_max_loop(space, args, implementation_of)
-
- at specialize.arg(2)
-def min_max_loop(space, args, implementation_of):
-    if implementation_of == "max":
-        compare = space.gt
-    else:
-        compare = space.lt
     args_w = args.arguments_w
     if len(args_w) > 1:
         w_sequence = space.newtuple(args_w)
diff --git a/pypy/module/_continuation/__init__.py b/pypy/module/_continuation/__init__.py
--- a/pypy/module/_continuation/__init__.py
+++ b/pypy/module/_continuation/__init__.py
@@ -37,4 +37,5 @@
     interpleveldefs = {
         'continulet': 'interp_continuation.W_Continulet',
         'permute': 'interp_continuation.permute',
+        '_p': 'interp_continuation.unpickle',      # pickle support
     }
diff --git a/pypy/module/_continuation/interp_continuation.py b/pypy/module/_continuation/interp_continuation.py
--- a/pypy/module/_continuation/interp_continuation.py
+++ b/pypy/module/_continuation/interp_continuation.py
@@ -6,6 +6,7 @@
 from pypy.interpreter.typedef import TypeDef
 from pypy.interpreter.gateway import interp2app
 from pypy.interpreter.pycode import PyCode
+from pypy.interpreter.pyframe import PyFrame
 
 
 class W_Continulet(Wrappable):
@@ -23,24 +24,27 @@
         if ec.stacklet_thread is not self.sthread:
             global_state.clear()
             raise geterror(self.space, "inter-thread support is missing")
-        return ec
 
     def descr_init(self, w_callable, __args__):
         if self.sthread is not None:
             raise geterror(self.space, "continulet already __init__ialized")
+        #
+        # hackish: build the frame "by hand", passing it the correct arguments
+        space = self.space
+        w_args, w_kwds = __args__.topacked()
+        bottomframe = space.createframe(get_entrypoint_pycode(space),
+                                        get_w_module_dict(space), None)
+        bottomframe.locals_stack_w[0] = space.wrap(self)
+        bottomframe.locals_stack_w[1] = w_callable
+        bottomframe.locals_stack_w[2] = w_args
+        bottomframe.locals_stack_w[3] = w_kwds
+        self.bottomframe = bottomframe
+        #
         global_state.origin = self
-        global_state.w_callable = w_callable
-        global_state.args = __args__
-        self.bottomframe = make_fresh_frame(self.space)
-        self.sthread = build_sthread(self.space)
-        try:
-            self.h = self.sthread.new(new_stacklet_callback)
-            if self.sthread.is_empty_handle(self.h):    # early return
-                raise MemoryError
-        except MemoryError:
-            self.sthread = None
-            global_state.clear()
-            raise getmemoryerror(self.space)
+        sthread = build_sthread(self.space)
+        self.sthread = sthread
+        h = sthread.new(new_stacklet_callback)
+        post_switch(sthread, h)
 
     def switch(self, w_to):
         sthread = self.sthread
@@ -66,7 +70,7 @@
             if sthread.is_empty_handle(to.h):
                 global_state.clear()
                 raise geterror(self.space, "continulet already finished")
-        ec = self.check_sthread()
+        self.check_sthread()
         #
         global_state.origin = self
         if to is None:
@@ -76,13 +80,8 @@
             # double switch: the final destination is to.h
             global_state.destination = to
         #
-        try:
-            do_switch(sthread, global_state.destination.h)
-        except MemoryError:
-            global_state.clear()
-            raise getmemoryerror(self.space)
-        #
-        return get_result()
+        h = sthread.switch(global_state.destination.h)
+        return post_switch(sthread, h)
 
     def descr_switch(self, w_value=None, w_to=None):
         global_state.w_value = w_value
@@ -109,12 +108,26 @@
                  and not self.sthread.is_empty_handle(self.h))
         return self.space.newbool(valid)
 
+    def descr__reduce__(self):
+        from pypy.module._continuation import interp_pickle
+        return interp_pickle.reduce(self)
+
+    def descr__setstate__(self, w_args):
+        from pypy.module._continuation import interp_pickle
+        interp_pickle.setstate(self, w_args)
+
 
 def W_Continulet___new__(space, w_subtype, __args__):
     r = space.allocate_instance(W_Continulet, w_subtype)
     r.__init__(space)
     return space.wrap(r)
 
+def unpickle(space, w_subtype):
+    """Pickle support."""
+    r = space.allocate_instance(W_Continulet, w_subtype)
+    r.__init__(space)
+    return space.wrap(r)
+
 
 W_Continulet.typedef = TypeDef(
     'continulet',
@@ -124,9 +137,10 @@
     switch      = interp2app(W_Continulet.descr_switch),
     throw       = interp2app(W_Continulet.descr_throw),
     is_pending  = interp2app(W_Continulet.descr_is_pending),
+    __reduce__  = interp2app(W_Continulet.descr__reduce__),
+    __setstate__= interp2app(W_Continulet.descr__setstate__),
     )
 
-
 # ____________________________________________________________
 
 # Continulet objects maintain a dummy frame object in order to ensure
@@ -135,27 +149,40 @@
 
 class State:
     def __init__(self, space):
-        from pypy.interpreter.astcompiler.consts import CO_OPTIMIZED
-        self.space = space 
+        self.space = space
         w_module = space.getbuiltinmodule('_continuation')
         self.w_error = space.getattr(w_module, space.wrap('error'))
-        self.w_memoryerror = OperationError(space.w_MemoryError, space.w_None)
-        self.dummy_pycode = PyCode(space, 0, 0, 0, CO_OPTIMIZED,
-                                   '', [], [], [], '',
-                                   '<bottom of continulet>', 0, '', [], [],
-                                   hidden_applevel=True)
+        # the following function switches away immediately, so that
+        # continulet.__init__() doesn't immediately run func(), but it
+        # also has the hidden purpose of making sure we have a single
+        # bottomframe for the whole duration of the continulet's run.
+        # Hackish: only the func_code is used, and used in the context
+        # of w_globals == this module, so we can access the name
+        # 'continulet' directly.
+        w_code = space.appexec([], '''():
+            def start(c, func, args, kwds):
+                if continulet.switch(c) is not None:
+                    raise TypeError(
+                     "can\'t send non-None value to a just-started continulet")
+                return func(c, *args, **kwds)
+            return start.func_code
+        ''')
+        self.entrypoint_pycode = space.interp_w(PyCode, w_code)
+        self.entrypoint_pycode.hidden_applevel = True
+        self.w_unpickle = w_module.get('_p')
+        self.w_module_dict = w_module.getdict(space)
 
 def geterror(space, message):
     cs = space.fromcache(State)
     return OperationError(cs.w_error, space.wrap(message))
 
-def getmemoryerror(space):
+def get_entrypoint_pycode(space):
     cs = space.fromcache(State)
-    return cs.w_memoryerror
+    return cs.entrypoint_pycode
 
-def make_fresh_frame(space):
+def get_w_module_dict(space):
     cs = space.fromcache(State)
-    return space.FrameClass(space, cs.dummy_pycode, None, None)
+    return cs.w_module_dict
 
 # ____________________________________________________________
 
@@ -166,6 +193,9 @@
         StackletThread.__init__(self, space.config)
         self.space = space
         self.ec = ec
+        # for unpickling
+        from pypy.rlib.rweakref import RWeakKeyDictionary
+        self.frame2continulet = RWeakKeyDictionary(PyFrame, W_Continulet)
 
 ExecutionContext.stacklet_thread = None
 
@@ -176,8 +206,6 @@
     def clear(self):
         self.origin = None
         self.destination = None
-        self.w_callable = None
-        self.args = None
         self.w_value = None
         self.propagate_exception = None
 global_state = GlobalState()
@@ -185,27 +213,13 @@
 
 
 def new_stacklet_callback(h, arg):
-    self       = global_state.origin
-    w_callable = global_state.w_callable
-    args       = global_state.args
+    self = global_state.origin
+    self.h = h
     global_state.clear()
-    try:
-        do_switch(self.sthread, h)
-    except MemoryError:
-        return h       # oups!  do an early return in this case
-    #
     space = self.space
     try:
-        assert self.sthread.ec.topframeref() is None
-        self.sthread.ec.topframeref = jit.non_virtual_ref(self.bottomframe)
-        if global_state.propagate_exception is not None:
-            raise global_state.propagate_exception  # just propagate it further
-        if global_state.w_value is not space.w_None:
-            raise OperationError(space.w_TypeError, space.wrap(
-                "can't send non-None value to a just-started continulet"))
-
-        args = args.prepend(self.space.wrap(self))
-        w_result = space.call_args(w_callable, args)
+        frame = self.bottomframe
+        w_result = frame.execute_frame()
     except Exception, e:
         global_state.propagate_exception = e
     else:
@@ -215,9 +229,7 @@
     global_state.destination = self
     return self.h
 
-
-def do_switch(sthread, h):
-    h = sthread.switch(h)
+def post_switch(sthread, h):
     origin = global_state.origin
     self = global_state.destination
     global_state.origin = None
@@ -228,6 +240,8 @@
     sthread.ec.topframeref = self.bottomframe.f_backref
     self.bottomframe.f_backref = origin.bottomframe.f_backref
     origin.bottomframe.f_backref = current
+    #
+    return get_result()
 
 def get_result():
     if global_state.propagate_exception:
diff --git a/pypy/module/_continuation/interp_pickle.py b/pypy/module/_continuation/interp_pickle.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_continuation/interp_pickle.py
@@ -0,0 +1,128 @@
+from pypy.tool import stdlib_opcode as pythonopcode
+from pypy.rlib import jit
+from pypy.interpreter.error import OperationError
+from pypy.interpreter.pyframe import PyFrame
+from pypy.module._continuation.interp_continuation import State, global_state
+from pypy.module._continuation.interp_continuation import build_sthread
+from pypy.module._continuation.interp_continuation import post_switch
+from pypy.module._continuation.interp_continuation import get_result, geterror
+
+
+def getunpickle(space):
+    cs = space.fromcache(State)
+    return cs.w_unpickle
+
+
+def reduce(self):
+    # xxx this is known to be not completely correct with respect
+    # to subclasses, e.g. no __slots__ support, no looking for a
+    # __getnewargs__ or __getstate__ defined in the subclass, etc.
+    # Doing the right thing looks involved, though...
+    space = self.space
+    if self.sthread is None:
+        w_frame = space.w_False
+    elif self.sthread.is_empty_handle(self.h):
+        w_frame = space.w_None
+    else:
+        w_frame = space.wrap(self.bottomframe)
+    w_continulet_type = space.type(space.wrap(self))
+    w_dict = self.getdict(space) or space.w_None
+    args = [getunpickle(space),
+            space.newtuple([w_continulet_type]),
+            space.newtuple([w_frame, w_dict]),
+            ]
+    return space.newtuple(args)
+
+def setstate(self, w_args):
+    space = self.space
+    if self.sthread is not None:
+        raise geterror(space, "continulet.__setstate__() on an already-"
+                              "initialized continulet")
+    w_frame, w_dict = space.fixedview(w_args, expected_length=2)
+    if not space.is_w(w_dict, space.w_None):
+        self.setdict(space, w_dict)
+    if space.is_w(w_frame, space.w_False):
+        return    # not initialized
+    sthread = build_sthread(self.space)
+    self.sthread = sthread
+    self.bottomframe = space.interp_w(PyFrame, w_frame, can_be_None=True)
+    #
+    global_state.origin = self
+    if self.bottomframe is not None:
+        sthread.frame2continulet.set(self.bottomframe, self)
+    self.h = sthread.new(resume_trampoline_callback)
+    get_result()    # propagate the eventual MemoryError
+
+# ____________________________________________________________
+
+def resume_trampoline_callback(h, arg):
+    self = global_state.origin
+    self.h = h
+    space = self.space
+    sthread = self.sthread
+    try:
+        global_state.clear()
+        if self.bottomframe is None:
+            w_result = space.w_None
+        else:
+            h = sthread.switch(self.h)
+            try:
+                w_result = post_switch(sthread, h)
+                operr = None
+            except OperationError, e:
+                w_result = None
+                operr = e
+            #
+            while True:
+                ec = sthread.ec
+                frame = ec.topframeref()
+                assert frame is not None     # XXX better error message
+                exit_continulet = sthread.frame2continulet.get(frame)
+                #
+                continue_after_call(frame)
+                #
+                # small hack: unlink frame out of the execution context,
+                # because execute_frame will add it there again
+                ec.topframeref = frame.f_backref
+                #
+                try:
+                    w_result = frame.execute_frame(w_result, operr)
+                    operr = None
+                except OperationError, e:
+                    w_result = None
+                    operr = e
+                if exit_continulet is not None:
+                    self = exit_continulet
+                    break
+            sthread.ec.topframeref = jit.vref_None
+            if operr:
+                raise operr
+    except Exception, e:
+        global_state.propagate_exception = e
+    else:
+        global_state.w_value = w_result
+    global_state.origin = self
+    global_state.destination = self
+    return self.h
+
+def continue_after_call(frame):
+    code = frame.pycode.co_code
+    instr = frame.last_instr
+    opcode = ord(code[instr])
+    map = pythonopcode.opmap
+    call_ops = [map['CALL_FUNCTION'], map['CALL_FUNCTION_KW'],
+                map['CALL_FUNCTION_VAR'], map['CALL_FUNCTION_VAR_KW'],
+                map['CALL_METHOD']]
+    assert opcode in call_ops   # XXX check better, and complain better
+    instr += 1
+    oparg = ord(code[instr]) | ord(code[instr + 1]) << 8
+    nargs = oparg & 0xff
+    nkwds = (oparg >> 8) & 0xff
+    if nkwds == 0:     # only positional arguments
+        # fast paths leaves things on the stack, pop them
+        if (frame.space.config.objspace.opcodes.CALL_METHOD and
+            opcode == map['CALL_METHOD']):
+            frame.dropvalues(nargs + 2)
+        elif opcode == map['CALL_FUNCTION']:
+            frame.dropvalues(nargs + 1)
+    frame.last_instr = instr + 1    # continue after the call
diff --git a/pypy/module/_continuation/test/test_stacklet.py b/pypy/module/_continuation/test/test_stacklet.py
--- a/pypy/module/_continuation/test/test_stacklet.py
+++ b/pypy/module/_continuation/test/test_stacklet.py
@@ -13,7 +13,7 @@
         from _continuation import continulet
         #
         def empty_callback(c):
-            pass
+            never_called
         #
         c = continulet(empty_callback)
         assert type(c) is continulet
@@ -36,7 +36,7 @@
         from _continuation import continulet, error
         #
         def empty_callback(c1):
-            pass
+            never_called
         #
         c = continulet(empty_callback)
         raises(error, c.__init__, empty_callback)
diff --git a/pypy/module/_continuation/test/test_zpickle.py b/pypy/module/_continuation/test/test_zpickle.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_continuation/test/test_zpickle.py
@@ -0,0 +1,262 @@
+from pypy.conftest import gettestobjspace
+
+
+class AppTestCopy:
+    def setup_class(cls):
+        cls.space = gettestobjspace(usemodules=('_continuation',),
+                                    CALL_METHOD=True)
+        cls.space.config.translation.continuation = True
+
+    def test_basic_setup(self):
+        from _continuation import continulet
+        lst = [4]
+        co = continulet(lst.append)
+        assert lst == [4]
+        res = co.switch()
+        assert res is None
+        assert lst == [4, co]
+
+    def test_copy_continulet_not_started(self):
+        from _continuation import continulet, error
+        import copy
+        lst = []
+        co = continulet(lst.append)
+        co2, lst2 = copy.deepcopy((co, lst))
+        #
+        assert lst == []
+        co.switch()
+        assert lst == [co]
+        #
+        assert lst2 == []
+        co2.switch()
+        assert lst2 == [co2]
+
+    def test_copy_continulet_not_started_multiple(self):
+        from _continuation import continulet, error
+        import copy
+        lst = []
+        co = continulet(lst.append)
+        co2, lst2 = copy.deepcopy((co, lst))
+        co3, lst3 = copy.deepcopy((co, lst))
+        co4, lst4 = copy.deepcopy((co, lst))
+        #
+        assert lst == []
+        co.switch()
+        assert lst == [co]
+        #
+        assert lst2 == []
+        co2.switch()
+        assert lst2 == [co2]
+        #
+        assert lst3 == []
+        co3.switch()
+        assert lst3 == [co3]
+        #
+        assert lst4 == []
+        co4.switch()
+        assert lst4 == [co4]
+
+    def test_copy_continulet_real(self):
+        import new, sys
+        mod = new.module('test_copy_continulet_real')
+        sys.modules['test_copy_continulet_real'] = mod
+        exec '''if 1:
+            from _continuation import continulet
+            import copy
+            def f(co, x):
+                co.switch(x + 1)
+                co.switch(x + 2)
+                return x + 3
+            co = continulet(f, 40)
+            res = co.switch()
+            assert res == 41
+            co2 = copy.deepcopy(co)
+            #
+            res = co2.switch()
+            assert res == 42
+            assert co2.is_pending()
+            res = co2.switch()
+            assert res == 43
+            assert not co2.is_pending()
+            #
+            res = co.switch()
+            assert res == 42
+            assert co.is_pending()
+            res = co.switch()
+            assert res == 43
+            assert not co.is_pending()
+        ''' in mod.__dict__
+
+    def test_copy_continulet_already_finished(self):
+        from _continuation import continulet, error
+        import copy
+        lst = []
+        co = continulet(lst.append)
+        co.switch()
+        co2 = copy.deepcopy(co)
+        assert not co.is_pending()
+        assert not co2.is_pending()
+        raises(error, co.__init__, lst.append)
+        raises(error, co2.__init__, lst.append)
+        raises(error, co.switch)
+        raises(error, co2.switch)
+
+
+class AppTestPickle:
+    version = 0
+
+    def setup_class(cls):
+        cls.space = gettestobjspace(usemodules=('_continuation',),
+                                    CALL_METHOD=True)
+        cls.space.appexec([], """():
+            global continulet, A, __name__
+
+            import sys
+            __name__ = 'test_pickle_continulet'
+            thismodule = type(sys)(__name__)
+            sys.modules[__name__] = thismodule
+
+            from _continuation import continulet
+            class A(continulet):
+                pass
+
+            thismodule.__dict__.update(globals())
+        """)
+        cls.w_version = cls.space.wrap(cls.version)
+
+    def test_pickle_continulet_empty(self):
+        from _continuation import continulet
+        lst = [4]
+        co = continulet.__new__(continulet)
+        import pickle
+        pckl = pickle.dumps(co, self.version)
+        print repr(pckl)
+        co2 = pickle.loads(pckl)
+        assert co2 is not co
+        assert not co.is_pending()
+        assert not co2.is_pending()
+        # the empty unpickled coroutine can still be used:
+        result = [5]
+        co2.__init__(result.append)
+        res = co2.switch()
+        assert res is None
+        assert result == [5, co2]
+
+    def test_pickle_continulet_empty_subclass(self):
+        from test_pickle_continulet import continulet, A
+        lst = [4]
+        co = continulet.__new__(A)
+        co.foo = 'bar'
+        co.bar = 'baz'
+        import pickle
+        pckl = pickle.dumps(co, self.version)
+        print repr(pckl)
+        co2 = pickle.loads(pckl)
+        assert co2 is not co
+        assert not co.is_pending()
+        assert not co2.is_pending()
+        assert type(co) is type(co2) is A
+        assert co.foo == co2.foo == 'bar'
+        assert co.bar == co2.bar == 'baz'
+        # the empty unpickled coroutine can still be used:
+        result = [5]
+        co2.__init__(result.append)
+        res = co2.switch()
+        assert res is None
+        assert result == [5, co2]
+
+    def test_pickle_continulet_not_started(self):
+        from _continuation import continulet, error
+        import pickle
+        lst = []
+        co = continulet(lst.append)
+        pckl = pickle.dumps((co, lst))
+        print pckl
+        del co, lst
+        for i in range(2):
+            print 'resume...'
+            co2, lst2 = pickle.loads(pckl)
+            assert lst2 == []
+            co2.switch()
+            assert lst2 == [co2]
+
+    def test_pickle_continulet_real(self):
+        import new, sys
+        mod = new.module('test_pickle_continulet_real')
+        sys.modules['test_pickle_continulet_real'] = mod
+        mod.version = self.version
+        exec '''if 1:
+            from _continuation import continulet
+            import pickle
+            def f(co, x):
+                co.switch(x + 1)
+                co.switch(x + 2)
+                return x + 3
+            co = continulet(f, 40)
+            res = co.switch()
+            assert res == 41
+            pckl = pickle.dumps(co, version)
+            print repr(pckl)
+            co2 = pickle.loads(pckl)
+            #
+            res = co2.switch()
+            assert res == 42
+            assert co2.is_pending()
+            res = co2.switch()
+            assert res == 43
+            assert not co2.is_pending()
+            #
+            res = co.switch()
+            assert res == 42
+            assert co.is_pending()
+            res = co.switch()
+            assert res == 43
+            assert not co.is_pending()
+        ''' in mod.__dict__
+
+    def test_pickle_continulet_real_subclass(self):
+        import new, sys
+        mod = new.module('test_pickle_continulet_real_subclass')
+        sys.modules['test_pickle_continulet_real_subclass'] = mod
+        mod.version = self.version
+        exec '''if 1:
+            from _continuation import continulet
+            import pickle
+            class A(continulet):
+                def __init__(self):
+                    crash
+            def f(co):
+                co.switch(co.x + 1)
+                co.switch(co.x + 2)
+                return co.x + 3
+            co = A.__new__(A)
+            continulet.__init__(co, f)
+            co.x = 40
+            res = co.switch()
+            assert res == 41
+            pckl = pickle.dumps(co, version)
+            print repr(pckl)
+            co2 = pickle.loads(pckl)
+            #
+            assert type(co2) is A
+            res = co2.switch()
+            assert res == 42
+            assert co2.is_pending()
+            res = co2.switch()
+            assert res == 43
+            assert not co2.is_pending()
+            #
+            res = co.switch()
+            assert res == 42
+            assert co.is_pending()
+            res = co.switch()
+            assert res == 43
+            assert not co.is_pending()
+        ''' in mod.__dict__
+
+
+class AppTestPickle_v1(AppTestPickle):
+    version = 1
+
+class AppTestPickle_v2(AppTestPickle):
+    version = 2
diff --git a/pypy/module/_multiprocessing/interp_connection.py b/pypy/module/_multiprocessing/interp_connection.py
--- a/pypy/module/_multiprocessing/interp_connection.py
+++ b/pypy/module/_multiprocessing/interp_connection.py
@@ -225,7 +225,9 @@
             except OSError:
                 pass
 
-    def __init__(self, fd, flags):
+    def __init__(self, space, fd, flags):
+        if fd == self.INVALID_HANDLE_VALUE or fd < 0:
+            raise OperationError(space.w_IOError, space.wrap("invalid handle %d" % fd))
         W_BaseConnection.__init__(self, flags)
         self.fd = fd
 
@@ -234,7 +236,7 @@
         flags = (readable and READABLE) | (writable and WRITABLE)
 
         self = space.allocate_instance(W_FileConnection, w_subtype)
-        W_FileConnection.__init__(self, fd, flags)
+        W_FileConnection.__init__(self, space, fd, flags)
         return space.wrap(self)
 
     def fileno(self, space):
diff --git a/pypy/module/_multiprocessing/interp_semaphore.py b/pypy/module/_multiprocessing/interp_semaphore.py
--- a/pypy/module/_multiprocessing/interp_semaphore.py
+++ b/pypy/module/_multiprocessing/interp_semaphore.py
@@ -468,6 +468,9 @@
 
         self.count -= 1
 
+    def after_fork(self):
+        self.count = 0
+
     @unwrap_spec(kind=int, maxvalue=int)
     def rebuild(space, w_cls, w_handle, kind, maxvalue):
         self = space.allocate_instance(W_SemLock, w_cls)
@@ -512,6 +515,7 @@
     acquire = interp2app(W_SemLock.acquire),
     release = interp2app(W_SemLock.release),
     _rebuild = interp2app(W_SemLock.rebuild.im_func, as_classmethod=True),
+    _after_fork = interp2app(W_SemLock.after_fork),
     __enter__=interp2app(W_SemLock.enter),
     __exit__=interp2app(W_SemLock.exit),
     SEM_VALUE_MAX=SEM_VALUE_MAX,
diff --git a/pypy/module/_multiprocessing/test/test_connection.py b/pypy/module/_multiprocessing/test/test_connection.py
--- a/pypy/module/_multiprocessing/test/test_connection.py
+++ b/pypy/module/_multiprocessing/test/test_connection.py
@@ -145,3 +145,9 @@
             else:
                 c.close()
         space.delslice(w_connections, space.wrap(0), space.wrap(100))
+
+    def test_bad_fd(self):
+        import _multiprocessing
+
+        raises(IOError, _multiprocessing.Connection, -1)
+        raises(IOError, _multiprocessing.Connection, -15)
\ No newline at end of file
diff --git a/pypy/module/_multiprocessing/test/test_semaphore.py b/pypy/module/_multiprocessing/test/test_semaphore.py
--- a/pypy/module/_multiprocessing/test/test_semaphore.py
+++ b/pypy/module/_multiprocessing/test/test_semaphore.py
@@ -39,6 +39,10 @@
         sem.release()
         assert sem._count() == 0
 
+        sem.acquire()
+        sem._after_fork()
+        assert sem._count() == 0
+
     def test_recursive(self):
         from _multiprocessing import SemLock
         kind = self.RECURSIVE
diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py
--- a/pypy/module/_sre/interp_sre.py
+++ b/pypy/module/_sre/interp_sre.py
@@ -99,6 +99,7 @@
 # SRE_Pattern class
 
 class W_SRE_Pattern(Wrappable):
+    _immutable_fields_ = ["code", "flags"]
 
     def cannot_copy_w(self):
         space = self.space
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -57,6 +57,9 @@
         compile_extra=['-DPy_BUILD_CORE'],
         )
 
+class CConfig2:
+    _compilation_info_ = CConfig._compilation_info_
+
 class CConfig_constants:
     _compilation_info_ = CConfig._compilation_info_
 
@@ -300,9 +303,13 @@
         return unwrapper_raise # used in 'normal' RPython code.
     return decorate
 
-def cpython_struct(name, fields, forward=None):
+def cpython_struct(name, fields, forward=None, level=1):
     configname = name.replace(' ', '__')
-    setattr(CConfig, configname, rffi_platform.Struct(name, fields))
+    if level == 1:
+        config = CConfig
+    else:
+        config = CConfig2
+    setattr(config, configname, rffi_platform.Struct(name, fields))
     if forward is None:
         forward = lltype.ForwardReference()
     TYPES[configname] = forward
@@ -445,9 +452,10 @@
 #              'int*': rffi.INTP}
 
 def configure_types():
-    for name, TYPE in rffi_platform.configure(CConfig).iteritems():
-        if name in TYPES:
-            TYPES[name].become(TYPE)
+    for config in (CConfig, CConfig2):
+        for name, TYPE in rffi_platform.configure(config).iteritems():
+            if name in TYPES:
+                TYPES[name].become(TYPE)
 
 def build_type_checkers(type_name, cls=None):
     """
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -321,6 +321,15 @@
 
 } PyTypeObject;
 
+typedef struct {
+    PyTypeObject ht_type;
+    PyNumberMethods as_number;
+    PyMappingMethods as_mapping;
+    PySequenceMethods as_sequence;
+    PyBufferProcs as_buffer;
+    PyObject *ht_name, *ht_slots;
+} PyHeapTypeObject;
+
 /* Flag bits for printing: */
 #define Py_PRINT_RAW	1	/* No string quotes etc. */
 
diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py
--- a/pypy/module/cpyext/pyobject.py
+++ b/pypy/module/cpyext/pyobject.py
@@ -19,13 +19,42 @@
     basestruct = PyObject.TO
 
     def get_dealloc(self, space):
-        raise NotImplementedError
+        from pypy.module.cpyext.typeobject import subtype_dealloc
+        return llhelper(
+            subtype_dealloc.api_func.functype,
+            subtype_dealloc.api_func.get_wrapper(space))
+
     def allocate(self, space, w_type, itemcount=0):
-        raise NotImplementedError
+        # similar to PyType_GenericAlloc?
+        # except that it's not related to any pypy object.
+
+        pytype = rffi.cast(PyTypeObjectPtr, make_ref(space, w_type))
+        # Don't increase refcount for non-heaptypes
+        if pytype:
+            flags = rffi.cast(lltype.Signed, pytype.c_tp_flags)
+            if not flags & Py_TPFLAGS_HEAPTYPE:
+                Py_DecRef(space, w_type)
+
+        if pytype:
+            size = pytype.c_tp_basicsize
+        else:
+            size = rffi.sizeof(self.basestruct)
+        if itemcount:
+            size += itemcount * pytype.c_tp_itemsize
+        buf = lltype.malloc(rffi.VOIDP.TO, size,
+                            flavor='raw', zero=True)
+        pyobj = rffi.cast(PyObject, buf)
+        pyobj.c_ob_refcnt = 1
+        pyobj.c_ob_type = pytype
+        return pyobj
+
     def attach(self, space, pyobj, w_obj):
-        raise NotImplementedError
+        pass
+
     def realize(self, space, ref):
-        raise NotImplementedError
+        # For most types, a reference cannot exist without
+        # a real interpreter object
+        raise InvalidPointerException(str(ref))
 
 typedescr_cache = {}
 
@@ -40,6 +69,7 @@
     """
 
     tp_basestruct = kw.pop('basestruct', PyObject.TO)
+    tp_alloc      = kw.pop('alloc', None)
     tp_attach     = kw.pop('attach', None)
     tp_realize    = kw.pop('realize', None)
     tp_dealloc    = kw.pop('dealloc', None)
@@ -49,58 +79,24 @@
 
     class CpyTypedescr(BaseCpyTypedescr):
         basestruct = tp_basestruct
-        realize = tp_realize
 
-        def get_dealloc(self, space):
-            if tp_dealloc:
+        if tp_alloc:
+            def allocate(self, space, w_type, itemcount=0):
+                return tp_alloc(space, w_type)
+
+        if tp_dealloc:
+            def get_dealloc(self, space):
                 return llhelper(
                     tp_dealloc.api_func.functype,
                     tp_dealloc.api_func.get_wrapper(space))
-            else:
-                from pypy.module.cpyext.typeobject import subtype_dealloc
-                return llhelper(
-                    subtype_dealloc.api_func.functype,
-                    subtype_dealloc.api_func.get_wrapper(space))
-
-        def allocate(self, space, w_type, itemcount=0):
-            # similar to PyType_GenericAlloc?
-            # except that it's not related to any pypy object.
-
-            pytype = rffi.cast(PyTypeObjectPtr, make_ref(space, w_type))
-            # Don't increase refcount for non-heaptypes
-            if pytype:
-                flags = rffi.cast(lltype.Signed, pytype.c_tp_flags)
-                if not flags & Py_TPFLAGS_HEAPTYPE:
-                    Py_DecRef(space, w_type)
-
-            if pytype:
-                size = pytype.c_tp_basicsize
-            else:
-                size = rffi.sizeof(tp_basestruct)
-            if itemcount:
-                size += itemcount * pytype.c_tp_itemsize
-            buf = lltype.malloc(rffi.VOIDP.TO, size,
-                                flavor='raw', zero=True)
-            pyobj = rffi.cast(PyObject, buf)
-            pyobj.c_ob_refcnt = 1
-            pyobj.c_ob_type = pytype
-            return pyobj
 
         if tp_attach:
             def attach(self, space, pyobj, w_obj):
                 tp_attach(space, pyobj, w_obj)
-        else:
-            def attach(self, space, pyobj, w_obj):
-                pass
 
         if tp_realize:
             def realize(self, space, ref):
                 return tp_realize(space, ref)
-        else:
-            def realize(self, space, ref):
-                # For most types, a reference cannot exist without
-                # a real interpreter object
-                raise InvalidPointerException(str(ref))
     if typedef:
         CpyTypedescr.__name__ = "CpyTypedescr_%s" % (typedef.name,)
 
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -268,6 +268,21 @@
         assert type(obj) is foo.Custom
         assert type(foo.Custom) is foo.MetaType
 
+    def test_heaptype(self):
+        module = self.import_extension('foo', [
+           ("name_by_heaptype", "METH_O",
+            '''
+                 PyHeapTypeObject *heaptype = (PyHeapTypeObject *)args;
+                 Py_INCREF(heaptype->ht_name);
+                 return heaptype->ht_name;
+             '''
+             )
+            ])
+        class C(object):
+            pass
+        assert module.name_by_heaptype(C) == "C"
+        
+
 class TestTypes(BaseApiTest):
     def test_type_attributes(self, space, api):
         w_class = space.appexec([], """():
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -11,7 +11,7 @@
     generic_cpy_call, Py_TPFLAGS_READY, Py_TPFLAGS_READYING,
     Py_TPFLAGS_HEAPTYPE, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL,
     Py_TPFLAGS_HAVE_GETCHARBUFFER,
-    build_type_checkers)
+    build_type_checkers, PyObjectFields)
 from pypy.module.cpyext.pyobject import (
     PyObject, make_ref, create_ref, from_ref, get_typedescr, make_typedescr,
     track_reference, RefcountState, borrow_from)
@@ -25,7 +25,7 @@
 from pypy.module.cpyext.structmember import PyMember_GetOne, PyMember_SetOne
 from pypy.module.cpyext.typeobjectdefs import (
     PyTypeObjectPtr, PyTypeObject, PyGetSetDef, PyMemberDef, newfunc,
-    PyNumberMethods, PySequenceMethods, PyBufferProcs)
+    PyNumberMethods, PyMappingMethods, PySequenceMethods, PyBufferProcs)
 from pypy.module.cpyext.slotdefs import (
     slotdefs_for_tp_slots, slotdefs_for_wrappers, get_slot_tp_function)
 from pypy.interpreter.error import OperationError
@@ -39,6 +39,19 @@
 
 PyType_Check, PyType_CheckExact = build_type_checkers("Type", "w_type")
 
+PyHeapTypeObjectStruct = lltype.ForwardReference()
+PyHeapTypeObject = lltype.Ptr(PyHeapTypeObjectStruct)
+PyHeapTypeObjectFields = (
+    ("ht_type", PyTypeObject),
+    ("ht_name", PyObject),
+    ("as_number", PyNumberMethods),
+    ("as_mapping", PyMappingMethods),
+    ("as_sequence", PySequenceMethods),
+    ("as_buffer", PyBufferProcs),
+    )
+cpython_struct("PyHeapTypeObject", PyHeapTypeObjectFields, PyHeapTypeObjectStruct,
+               level=2)
+
 class W_GetSetPropertyEx(GetSetProperty):
     def __init__(self, getset, w_type):
         self.getset = getset
@@ -136,6 +149,8 @@
             assert len(slot_names) == 2
             struct = getattr(pto, slot_names[0])
             if not struct:
+                assert not space.config.translating
+                assert not pto.c_tp_flags & Py_TPFLAGS_HEAPTYPE
                 if slot_names[0] == 'c_tp_as_number':
                     STRUCT_TYPE = PyNumberMethods
                 elif slot_names[0] == 'c_tp_as_sequence':
@@ -301,6 +316,7 @@
 
     make_typedescr(space.w_type.instancetypedef,
                    basestruct=PyTypeObject,
+                   alloc=type_alloc,
                    attach=type_attach,
                    realize=type_realize,
                    dealloc=type_dealloc)
@@ -319,11 +335,13 @@
     track_reference(space, lltype.nullptr(PyObject.TO), space.w_type)
     track_reference(space, lltype.nullptr(PyObject.TO), space.w_object)
     track_reference(space, lltype.nullptr(PyObject.TO), space.w_tuple)
+    track_reference(space, lltype.nullptr(PyObject.TO), space.w_str)
 
     # create the objects
     py_type = create_ref(space, space.w_type)
     py_object = create_ref(space, space.w_object)
     py_tuple = create_ref(space, space.w_tuple)
+    py_str = create_ref(space, space.w_str)
 
     # form cycles
     pto_type = rffi.cast(PyTypeObjectPtr, py_type)
@@ -340,10 +358,15 @@
     pto_object.c_tp_bases.c_ob_type = pto_tuple
     pto_tuple.c_tp_bases.c_ob_type = pto_tuple
 
+    for typ in (py_type, py_object, py_tuple, py_str):
+        heaptype = rffi.cast(PyHeapTypeObject, typ)
+        heaptype.c_ht_name.c_ob_type = pto_type
+
     # Restore the mapping
     track_reference(space, py_type, space.w_type, replace=True)
     track_reference(space, py_object, space.w_object, replace=True)
     track_reference(space, py_tuple, space.w_tuple, replace=True)
+    track_reference(space, py_str, space.w_str, replace=True)
 
 
 @cpython_api([PyObject], lltype.Void, external=False)
@@ -416,17 +439,34 @@
     Py_DecRef(space, obj_pto.c_tp_cache) # let's do it like cpython
     Py_DecRef(space, obj_pto.c_tp_dict)
     if obj_pto.c_tp_flags & Py_TPFLAGS_HEAPTYPE:
-        if obj_pto.c_tp_as_buffer:
-            lltype.free(obj_pto.c_tp_as_buffer, flavor='raw')
-        if obj_pto.c_tp_as_number:
-            lltype.free(obj_pto.c_tp_as_number, flavor='raw')
-        if obj_pto.c_tp_as_sequence:
-            lltype.free(obj_pto.c_tp_as_sequence, flavor='raw')
+        heaptype = rffi.cast(PyHeapTypeObject, obj)
+        Py_DecRef(space, heaptype.c_ht_name)
         Py_DecRef(space, base_pyo)
-        rffi.free_charp(obj_pto.c_tp_name)
         PyObject_dealloc(space, obj)
 
 
+def type_alloc(space, w_metatype):
+    size = rffi.sizeof(PyHeapTypeObject)
+    metatype = rffi.cast(PyTypeObjectPtr, make_ref(space, w_metatype))
+    # Don't increase refcount for non-heaptypes
+    if metatype:
+        flags = rffi.cast(lltype.Signed, metatype.c_tp_flags)
+        if not flags & Py_TPFLAGS_HEAPTYPE:
+            Py_DecRef(space, w_metatype)
+
+    heaptype = lltype.malloc(PyHeapTypeObject.TO,
+                             flavor='raw', zero=True)
+    pto = heaptype.c_ht_type
+    pto.c_ob_refcnt = 1
+    pto.c_ob_type = metatype
+    pto.c_tp_flags |= Py_TPFLAGS_HEAPTYPE
+    pto.c_tp_as_number = heaptype.c_as_number
+    pto.c_tp_as_sequence = heaptype.c_as_sequence
+    pto.c_tp_as_mapping = heaptype.c_as_mapping
+    pto.c_tp_as_buffer = heaptype.c_as_buffer
+    
+    return rffi.cast(PyObject, heaptype)
+
 def type_attach(space, py_obj, w_type):
     """
     Fills a newly allocated PyTypeObject from an existing type.
@@ -445,12 +485,18 @@
     if space.is_w(w_type, space.w_str):
         setup_string_buffer_procs(space, pto)
 
-    pto.c_tp_flags |= Py_TPFLAGS_HEAPTYPE
     pto.c_tp_free = llhelper(PyObject_Del.api_func.functype,
             PyObject_Del.api_func.get_wrapper(space))
     pto.c_tp_alloc = llhelper(PyType_GenericAlloc.api_func.functype,
             PyType_GenericAlloc.api_func.get_wrapper(space))
-    pto.c_tp_name = rffi.str2charp(w_type.getname(space))
+    if pto.c_tp_flags & Py_TPFLAGS_HEAPTYPE:
+        w_typename = space.getattr(w_type, space.wrap('__name__'))
+        heaptype = rffi.cast(PyHeapTypeObject, pto)
+        heaptype.c_ht_name = make_ref(space, w_typename)
+        from pypy.module.cpyext.stringobject import PyString_AsString
+        pto.c_tp_name = PyString_AsString(space, heaptype.c_ht_name)
+    else:
+        pto.c_tp_name = rffi.str2charp(w_type.getname(space))
     pto.c_tp_basicsize = -1 # hopefully this makes malloc bail out
     pto.c_tp_itemsize = 0
     # uninitialized fields:
diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -23,6 +23,8 @@
         ("arccos", "arccos"),
         ("arcsin", "arcsin"),
         ("arctan", "arctan"),
+        ("arcsinh", "arcsinh"),
+        ("arctanh", "arctanh"),
         ("copysign", "copysign"),
         ("cos", "cos"),
         ("divide", "divide"),
@@ -50,4 +52,6 @@
     appleveldefs = {
         'average': 'app_numpy.average',
         'mean': 'app_numpy.mean',
+        'inf': 'app_numpy.inf',
+        'e': 'app_numpy.e',
     }
diff --git a/pypy/module/micronumpy/app_numpy.py b/pypy/module/micronumpy/app_numpy.py
--- a/pypy/module/micronumpy/app_numpy.py
+++ b/pypy/module/micronumpy/app_numpy.py
@@ -1,5 +1,11 @@
+import math
+
 import numpy
 
+
+inf = float("inf")
+e = math.e
+
 def average(a):
     # This implements a weighted average, for now we don't implement the
     # weighting, just the average part!
@@ -8,4 +14,4 @@
 def mean(a):
     if not hasattr(a, "mean"):
         a = numpy.array(a)
-    return a.mean()
\ No newline at end of file
+    return a.mean()
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -7,13 +7,14 @@
 from pypy.interpreter.typedef import TypeDef, interp_attrproperty, GetSetProperty
 from pypy.module.micronumpy import signature
 from pypy.objspace.std.floatobject import float2string
-from pypy.rlib import rfloat
-from pypy.rlib.rarithmetic import widen
+from pypy.rlib import rarithmetic, rfloat
+from pypy.rlib.rarithmetic import LONG_BIT, widen
 from pypy.rlib.objectmodel import specialize, enforceargs
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.rpython.lltypesystem import lltype, rffi
 
 
+UNSIGNEDLTR = "u"
 SIGNEDLTR = "i"
 BOOLLTR = "b"
 FLOATINGLTR = "f"
@@ -61,7 +62,10 @@
             self.val = val
 
         def wrap(self, space):
-            return space.wrap(self.val)
+            val = self.val
+            if valtype is rarithmetic.r_singlefloat:
+                val = float(val)
+            return space.wrap(val)
 
         def convert_to(self, dtype):
             return dtype.adapt_val(self.val)
@@ -145,7 +149,7 @@
         return self.adapt_val(func(self, self.for_computation(self.unbox(v))))
     return impl
 
-class ArithmaticTypeMixin(object):
+class ArithmeticTypeMixin(object):
     _mixin_ = True
 
     @binop
@@ -200,11 +204,17 @@
         return v1 >= v2
 
 
-class FloatArithmeticDtype(ArithmaticTypeMixin):
+class FloatArithmeticDtype(ArithmeticTypeMixin):
     _mixin_ = True
 
+    def unwrap(self, space, w_item):
+        return self.adapt_val(space.float_w(space.float(w_item)))
+
     def for_computation(self, v):
-        return v
+        return float(v)
+
+    def str_format(self, item):
+        return float2string(self.for_computation(self.unbox(item)), 'g', rfloat.DTSF_STR_PRECISION)
 
     @binop
     def mod(self, v1, v2):
@@ -250,19 +260,29 @@
         return math.tan(v)
     @unaryop
     def arcsin(self, v):
-        if v < -1.0 or v > 1.0:
+        if not -1.0 <= v <= 1.0:
             return rfloat.NAN
         return math.asin(v)
     @unaryop
     def arccos(self, v):
-        if v < -1.0 or v > 1.0:
+        if not -1.0 <= v <= 1.0:
             return rfloat.NAN
         return math.acos(v)
     @unaryop
     def arctan(self, v):
         return math.atan(v)
+    @unaryop
+    def arcsinh(self, v):
+        return math.asinh(v)
+    @unaryop
+    def arctanh(self, v):
+        if v == 1.0 or v == -1.0:
+            return math.copysign(rfloat.INFINITY, v)
+        if not -1.0 < v < 1.0:
+            return rfloat.NAN
+        return math.atanh(v)
 
-class IntegerArithmeticDtype(ArithmaticTypeMixin):
+class IntegerArithmeticDtype(ArithmeticTypeMixin):
     _mixin_ = True
 
     def unwrap(self, space, w_item):
@@ -271,10 +291,16 @@
     def for_computation(self, v):
         return widen(v)
 
+    def str_format(self, item):
+        return str(widen(self.unbox(item)))
+
     @binop
     def mod(self, v1, v2):
         return v1 % v2
 
+class SignedIntegerArithmeticDtype(IntegerArithmeticDtype):
+    _mixin_ = True
+
     @unaryop
     def sign(self, v):
         if v > 0:
@@ -285,17 +311,22 @@
             assert v == 0
             return 0
 
-    def str_format(self, item):
-        return str(widen(self.unbox(item)))
+class UnsignedIntegerArithmeticDtype(IntegerArithmeticDtype):
+    _mixin_ = True
+
+    @unaryop
+    def sign(self, v):
+        return int(v != 0)
+
 
 W_BoolDtype = create_low_level_dtype(
     num = 0, kind = BOOLLTR, name = "bool",
-    aliases = ["?"],
+    aliases = ["?", "bool", "bool8"],
     applevel_types = ["bool"],
     T = lltype.Bool,
     valtype = bool,
 )
-class W_BoolDtype(IntegerArithmeticDtype, W_BoolDtype):
+class W_BoolDtype(SignedIntegerArithmeticDtype, W_BoolDtype):
     def unwrap(self, space, w_item):
         return self.adapt_val(space.is_true(w_item))
 
@@ -308,67 +339,139 @@
 
 W_Int8Dtype = create_low_level_dtype(
     num = 1, kind = SIGNEDLTR, name = "int8",
-    aliases = ["int8"],
+    aliases = ["b", "int8", "i1"],
     applevel_types = [],
     T = rffi.SIGNEDCHAR,
     valtype = rffi.SIGNEDCHAR._type,
     expected_size = 1,
 )
-class W_Int8Dtype(IntegerArithmeticDtype, W_Int8Dtype):
+class W_Int8Dtype(SignedIntegerArithmeticDtype, W_Int8Dtype):
+    pass
+
+W_UInt8Dtype = create_low_level_dtype(
+    num = 2, kind = UNSIGNEDLTR, name = "uint8",
+    aliases = ["B", "uint8", "I1"],
+    applevel_types = [],
+    T = rffi.UCHAR,
+    valtype = rffi.UCHAR._type,
+    expected_size = 1,
+)
+class W_UInt8Dtype(UnsignedIntegerArithmeticDtype, W_UInt8Dtype):
     pass
 
 W_Int16Dtype = create_low_level_dtype(
     num = 3, kind = SIGNEDLTR, name = "int16",
-    aliases = ["int16"],
+    aliases = ["h", "int16", "i2"],
     applevel_types = [],
     T = rffi.SHORT,
     valtype = rffi.SHORT._type,
     expected_size = 2,
 )
-class W_Int16Dtype(IntegerArithmeticDtype, W_Int16Dtype):
+class W_Int16Dtype(SignedIntegerArithmeticDtype, W_Int16Dtype):
+    pass
+
+W_UInt16Dtype = create_low_level_dtype(
+    num = 4, kind = UNSIGNEDLTR, name = "uint16",
+    aliases = ["H", "uint16", "I2"],
+    applevel_types = [],
+    T = rffi.USHORT,
+    valtype = rffi.USHORT._type,
+    expected_size = 2,
+)
+class W_UInt16Dtype(UnsignedIntegerArithmeticDtype, W_UInt16Dtype):
     pass
 
 W_Int32Dtype = create_low_level_dtype(
     num = 5, kind = SIGNEDLTR, name = "int32",
-    aliases = ["i"],
+    aliases = ["i", "int32", "i4"],
     applevel_types = [],
     T = rffi.INT,
     valtype = rffi.INT._type,
     expected_size = 4,
 )
-class W_Int32Dtype(IntegerArithmeticDtype, W_Int32Dtype):
+class W_Int32Dtype(SignedIntegerArithmeticDtype, W_Int32Dtype):
+    pass
+
+W_UInt32Dtype = create_low_level_dtype(
+    num = 6, kind = UNSIGNEDLTR, name = "uint32",
+    aliases = ["I", "uint32", "I4"],
+    applevel_types = [],
+    T = rffi.UINT,
+    valtype = rffi.UINT._type,
+    expected_size = 4,
+)
+class W_UInt32Dtype(UnsignedIntegerArithmeticDtype, W_UInt32Dtype):
     pass
 
 W_Int64Dtype = create_low_level_dtype(
     num = 9, kind = SIGNEDLTR, name = "int64",
-    aliases = [],
+    aliases = ["q", "int64", "i8"],
     applevel_types = ["long"],
     T = rffi.LONGLONG,
     valtype = rffi.LONGLONG._type,
     expected_size = 8,
 )
-class W_Int64Dtype(IntegerArithmeticDtype, W_Int64Dtype):
+class W_Int64Dtype(SignedIntegerArithmeticDtype, W_Int64Dtype):
+    pass
+
+W_UInt64Dtype = create_low_level_dtype(
+    num = 10, kind = UNSIGNEDLTR, name = "uint64",
+    aliases = ["Q", "uint64", "I8"],
+    applevel_types = [],
+    T = rffi.ULONGLONG,
+    valtype = rffi.ULONGLONG._type,
+    expected_size = 8,
+)
+class W_UInt64Dtype(UnsignedIntegerArithmeticDtype, W_UInt64Dtype):
+    pass
+
+if LONG_BIT == 32:
+    class W_LongDtype(W_Int32Dtype):
+        pass
+
+    class W_ULongDtype(W_UInt32Dtype):
+        pass
+else:
+    class W_LongDtype(W_Int64Dtype):
+        pass
+
+    class W_ULongDtype(W_UInt64Dtype):
+        pass
+
+W_LongDtype.num = 7
+W_LongDtype.aliases = ["l"]
+W_LongDtype.applevel_types = ["int"]
+W_ULongDtype.num = 8
+W_ULongDtype.aliases = ["L"]
+
+W_Float32Dtype = create_low_level_dtype(
+    num = 11, kind = FLOATINGLTR, name = "float32",
+    aliases = ["f", "float32", "f4"],
+    applevel_types = [],
+    T = lltype.SingleFloat,
+    valtype = rarithmetic.r_singlefloat,
+    expected_size = 4,
+)
+class W_Float32Dtype(FloatArithmeticDtype, W_Float32Dtype):
     pass
 
 W_Float64Dtype = create_low_level_dtype(
     num = 12, kind = FLOATINGLTR, name = "float64",
-    aliases = [],
+    aliases = ["d", "float64", "f8"],
     applevel_types = ["float"],
     T = lltype.Float,
     valtype = float,
     expected_size = 8,
 )
 class W_Float64Dtype(FloatArithmeticDtype, W_Float64Dtype):
-    def unwrap(self, space, w_item):
-        return self.adapt_val(space.float_w(space.float(w_item)))
-
-    def str_format(self, item):
-        return float2string(self.unbox(item), 'g', rfloat.DTSF_STR_PRECISION)
+    pass
 
 ALL_DTYPES = [
     W_BoolDtype,
-    W_Int8Dtype, W_Int16Dtype, W_Int32Dtype, W_Int64Dtype,
-    W_Float64Dtype
+    W_Int8Dtype, W_UInt8Dtype, W_Int16Dtype, W_UInt16Dtype,
+    W_Int32Dtype, W_UInt32Dtype, W_LongDtype, W_ULongDtype,
+    W_Int64Dtype, W_UInt64Dtype,
+    W_Float32Dtype, W_Float64Dtype,
 ]
 
 dtypes_by_alias = unrolling_iterable([
@@ -395,6 +498,7 @@
 
     num = interp_attrproperty("num", cls=W_Dtype),
     kind = interp_attrproperty("kind", cls=W_Dtype),
+    itemsize = interp_attrproperty("num_bytes", cls=W_Dtype),
     shape = GetSetProperty(W_Dtype.descr_get_shape),
 )
 W_Dtype.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -4,6 +4,7 @@
 from pypy.interpreter.typedef import TypeDef, GetSetProperty, interp_attrproperty
 from pypy.module.micronumpy import interp_dtype, signature
 from pypy.rlib import jit
+from pypy.rlib.rarithmetic import LONG_BIT
 from pypy.tool.sourcetools import func_with_new_name
 
 
@@ -180,23 +181,56 @@
 
     # Everything promotes to float, and bool promotes to everything.
     if dt2.kind == interp_dtype.FLOATINGLTR or dt1.kind == interp_dtype.BOOLLTR:
+        # Float32 + 8-bit int = Float64
+        if dt2.num == 11 and dt1.num_bytes >= 4:
+            return space.fromcache(interp_dtype.W_Float64Dtype)
         return dt2
 
-    assert False
+    # for now this means mixing signed and unsigned
+    if dt2.kind == interp_dtype.SIGNEDLTR:
+        # if dt2 has a greater number of bytes, then just go with it
+        if dt1.num_bytes < dt2.num_bytes:
+            return dt2
+        # we need to promote both dtypes
+        dtypenum = dt2.num + 2
+    else:
+        # increase to the next signed type (or to float)
+        dtypenum = dt2.num + 1
+        # UInt64 + signed = Float64
+        if dt2.num == 10:
+            dtypenum += 1
+    newdtype = interp_dtype.ALL_DTYPES[dtypenum]
+
+    if newdtype.num_bytes > dt2.num_bytes or newdtype.kind == interp_dtype.FLOATINGLTR:
+        return space.fromcache(newdtype)
+    else:
+        # we only promoted to long on 32-bit or to longlong on 64-bit
+        # this is really for dealing with the Long and Ulong dtypes
+        if LONG_BIT == 32:
+            dtypenum += 2
+        else:
+            dtypenum += 3
+        return space.fromcache(interp_dtype.ALL_DTYPES[dtypenum])
 
 def find_unaryop_result_dtype(space, dt, promote_to_float=False,
     promote_bools=False, promote_to_largest=False):
     if promote_bools and (dt.kind == interp_dtype.BOOLLTR):
         return space.fromcache(interp_dtype.W_Int8Dtype)
     if promote_to_float:
+        if dt.kind == interp_dtype.FLOATINGLTR:
+            return dt
+        if dt.num >= 5:
+            return space.fromcache(interp_dtype.W_Float64Dtype)
         for bytes, dtype in interp_dtype.dtypes_by_num_bytes:
-            if dtype.kind == interp_dtype.FLOATINGLTR and dtype.num_bytes >= dt.num_bytes:
+            if dtype.kind == interp_dtype.FLOATINGLTR and dtype.num_bytes > dt.num_bytes:
                 return space.fromcache(dtype)
     if promote_to_largest:
         if dt.kind == interp_dtype.BOOLLTR or dt.kind == interp_dtype.SIGNEDLTR:
             return space.fromcache(interp_dtype.W_Int64Dtype)
         elif dt.kind == interp_dtype.FLOATINGLTR:
             return space.fromcache(interp_dtype.W_Float64Dtype)
+        elif dt.kind == interp_dtype.UNSIGNEDLTR:
+            return space.fromcache(interp_dtype.W_UInt64Dtype)
         else:
             assert False
     return dt
@@ -205,15 +239,23 @@
     w_type = space.type(w_obj)
 
     bool_dtype = space.fromcache(interp_dtype.W_BoolDtype)
+    long_dtype = space.fromcache(interp_dtype.W_LongDtype)
     int64_dtype = space.fromcache(interp_dtype.W_Int64Dtype)
 
     if space.is_w(w_type, space.w_bool):
         if current_guess is None or current_guess is bool_dtype:
             return bool_dtype
+        return current_guess
     elif space.is_w(w_type, space.w_int):
         if (current_guess is None or current_guess is bool_dtype or
-            current_guess is int64_dtype):
+            current_guess is long_dtype):
+            return long_dtype
+        return current_guess
+    elif space.is_w(w_type, space.w_long):
+        if (current_guess is None or current_guess is bool_dtype or
+            current_guess is long_dtype or current_guess is int64_dtype):
             return int64_dtype
+        return current_guess
     return space.fromcache(interp_dtype.W_Float64Dtype)
 
 
@@ -225,7 +267,9 @@
         def impl(res_dtype, lvalue, rvalue):
             res = getattr(res_dtype, op_name)(lvalue, rvalue)
             if comparison_func:
-                res = space.fromcache(interp_dtype.W_BoolDtype).box(res)
+                booldtype = space.fromcache(interp_dtype.W_BoolDtype)
+                assert isinstance(booldtype, interp_dtype.W_BoolDtype)
+                res = booldtype.box(res)
             return res
     return func_with_new_name(impl, ufunc_name)
 
@@ -268,6 +312,8 @@
             ("arcsin", "arcsin", 1, {"promote_to_float": True}),
             ("arccos", "arccos", 1, {"promote_to_float": True}),
             ("arctan", "arctan", 1, {"promote_to_float": True}),
+            ("arcsinh", "arcsinh", 1, {"promote_to_float": True}),
+            ("arctanh", "arctanh", 1, {"promote_to_float": True}),
         ]:
             self.add_ufunc(space, *ufunc_def)
 
@@ -277,7 +323,7 @@
 
         identity = extra_kwargs.get("identity")
         if identity is not None:
-            identity = space.fromcache(interp_dtype.W_Int64Dtype).adapt_val(identity)
+            identity = space.fromcache(interp_dtype.W_LongDtype).adapt_val(identity)
         extra_kwargs["identity"] = identity
 
         func = ufunc_dtype_caller(space, ufunc_name, op_name, argcount,
diff --git a/pypy/module/micronumpy/test/test_base.py b/pypy/module/micronumpy/test/test_base.py
--- a/pypy/module/micronumpy/test/test_base.py
+++ b/pypy/module/micronumpy/test/test_base.py
@@ -64,18 +64,46 @@
     def test_unaryops(self, space):
         bool_dtype = space.fromcache(interp_dtype.W_BoolDtype)
         int8_dtype = space.fromcache(interp_dtype.W_Int8Dtype)
+        uint8_dtype = space.fromcache(interp_dtype.W_UInt8Dtype)
+        int16_dtype = space.fromcache(interp_dtype.W_Int16Dtype)
+        uint16_dtype = space.fromcache(interp_dtype.W_UInt16Dtype)
         int32_dtype = space.fromcache(interp_dtype.W_Int32Dtype)
+        uint32_dtype = space.fromcache(interp_dtype.W_UInt32Dtype)
+        long_dtype = space.fromcache(interp_dtype.W_LongDtype)
+        ulong_dtype = space.fromcache(interp_dtype.W_ULongDtype)
+        int64_dtype = space.fromcache(interp_dtype.W_Int64Dtype)
+        uint64_dtype = space.fromcache(interp_dtype.W_UInt64Dtype)
+        float32_dtype = space.fromcache(interp_dtype.W_Float32Dtype)
         float64_dtype = space.fromcache(interp_dtype.W_Float64Dtype)
 
-        # Normal rules, everythign returns itself
+        # Normal rules, everything returns itself
         assert find_unaryop_result_dtype(space, bool_dtype) is bool_dtype
         assert find_unaryop_result_dtype(space, int8_dtype) is int8_dtype
+        assert find_unaryop_result_dtype(space, uint8_dtype) is uint8_dtype
+        assert find_unaryop_result_dtype(space, int16_dtype) is int16_dtype
+        assert find_unaryop_result_dtype(space, uint16_dtype) is uint16_dtype
         assert find_unaryop_result_dtype(space, int32_dtype) is int32_dtype
+        assert find_unaryop_result_dtype(space, uint32_dtype) is uint32_dtype
+        assert find_unaryop_result_dtype(space, long_dtype) is long_dtype
+        assert find_unaryop_result_dtype(space, ulong_dtype) is ulong_dtype
+        assert find_unaryop_result_dtype(space, int64_dtype) is int64_dtype
+        assert find_unaryop_result_dtype(space, uint64_dtype) is uint64_dtype
+        assert find_unaryop_result_dtype(space, float32_dtype) is float32_dtype
         assert find_unaryop_result_dtype(space, float64_dtype) is float64_dtype
 
         # Coerce to floats, some of these will eventually be float16, or
         # whatever our smallest float type is.
-        assert find_unaryop_result_dtype(space, bool_dtype, promote_to_float=True) is float64_dtype
-        assert find_unaryop_result_dtype(space, int8_dtype, promote_to_float=True) is float64_dtype
+        assert find_unaryop_result_dtype(space, bool_dtype, promote_to_float=True) is float32_dtype # will be float16 if we ever put that in
+        assert find_unaryop_result_dtype(space, int8_dtype, promote_to_float=True) is float32_dtype # will be float16 if we ever put that in
+        assert find_unaryop_result_dtype(space, uint8_dtype, promote_to_float=True) is float32_dtype # will be float16 if we ever put that in
+        assert find_unaryop_result_dtype(space, int16_dtype, promote_to_float=True) is float32_dtype
+        assert find_unaryop_result_dtype(space, uint16_dtype, promote_to_float=True) is float32_dtype
         assert find_unaryop_result_dtype(space, int32_dtype, promote_to_float=True) is float64_dtype
-        assert find_unaryop_result_dtype(space, float64_dtype, promote_to_float=True) is float64_dtype
\ No newline at end of file
+        assert find_unaryop_result_dtype(space, uint32_dtype, promote_to_float=True) is float64_dtype
+        assert find_unaryop_result_dtype(space, int64_dtype, promote_to_float=True) is float64_dtype
+        assert find_unaryop_result_dtype(space, uint64_dtype, promote_to_float=True) is float64_dtype
+        assert find_unaryop_result_dtype(space, float32_dtype, promote_to_float=True) is float32_dtype
+        assert find_unaryop_result_dtype(space, float64_dtype, promote_to_float=True) is float64_dtype
+
+        # promote bools, happens with sign ufunc
+        assert find_unaryop_result_dtype(space, bool_dtype, promote_bools=True) is int8_dtype
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -17,6 +17,7 @@
         from numpy import dtype
 
         assert dtype(bool).num == 0
+        assert dtype(int).num == 7
         assert dtype(long).num == 9
         assert dtype(float).num == 12
 
@@ -81,6 +82,48 @@
             assert isinstance(a[i], (int, long))
             assert a[1] == 1
 
+    def test_overflow(self):
+        from numpy import array, dtype
+        assert array([128], 'b')[0] == -128
+        assert array([256], 'B')[0] == 0
+        assert array([32768], 'h')[0] == -32768
+        assert array([65536], 'H')[0] == 0
+        if dtype('l').itemsize == 4: # 32-bit
+            raises(OverflowError, "array([2**32/2], 'i')")
+            raises(OverflowError, "array([2**32], 'I')")
+        raises(OverflowError, "array([2**64/2], 'q')")
+        raises(OverflowError, "array([2**64], 'Q')")
+
+    def test_bool_binop_types(self):
+        from numpy import array, dtype
+        types = ('?','b','B','h','H','i','I','l','L','q','Q','f','d')
+        N = len(types)
+        a = array([True], '?')
+        for t in types:
+            assert (a + array([0], t)).dtype is dtype(t)
+
+    def test_binop_types(self):
+        from numpy import array, dtype
+        tests = [('b','B','h'), ('b','h','h'), ('b','H','i'), ('b','i','i'),
+                 ('b','l','l'), ('b','q','q'), ('b','Q','d'), ('B','h','h'),
+                 ('B','H','H'), ('B','i','i'), ('B','I','I'), ('B','l','l'),
+                 ('B','L','L'), ('B','q','q'), ('B','Q','Q'), ('h','H','i'),
+                 ('h','i','i'), ('h','l','l'), ('h','q','q'), ('h','Q','d'),
+                 ('H','i','i'), ('H','I','I'), ('H','l','l'), ('H','L','L'),
+                 ('H','q','q'), ('H','Q','Q'), ('i','l','l'), ('i','q','q'),
+                 ('i','Q','d'), ('I','L','L'), ('I','q','q'), ('I','Q','Q'),
+                 ('q','Q','d'), ('b','f','f'), ('B','f','f'), ('h','f','f'),
+                 ('H','f','f'), ('i','f','d'), ('I','f','d'), ('l','f','d'),
+                 ('L','f','d'), ('q','f','d'), ('Q','f','d'), ('q','d','d')]
+        if dtype('i').itemsize == dtype('l').itemsize: # 32-bit
+            tests.extend([('b','I','q'), ('b','L','q'), ('h','I','q'),
+                          ('h','L','q'), ('i','I','q'), ('i','L','q')])
+        else:
+            tests.extend([('b','I','l'), ('b','L','d'), ('h','I','l'),
+                          ('h','L','d'), ('i','I','l'), ('i','L','d')])
+        for d1, d2, dout in tests:
+            assert (array([1], d1) + array([1], d2)).dtype is dtype(dout)
+
     def test_add_int8(self):
         from numpy import array, dtype
 
@@ -99,6 +142,15 @@
         for i in range(5):
             assert b[i] == i * 2
 
+    def test_add_uint32(self):
+        from numpy import array, dtype
+
+        a = array(range(5), dtype="I")
+        b = a + a
+        assert b.dtype is dtype("I")
+        for i in range(5):
+            assert b[i] == i * 2
+
     def test_shape(self):
         from numpy import dtype
 
diff --git a/pypy/module/micronumpy/test/test_module.py b/pypy/module/micronumpy/test/test_module.py
--- a/pypy/module/micronumpy/test/test_module.py
+++ b/pypy/module/micronumpy/test/test_module.py
@@ -10,4 +10,12 @@
     def test_average(self):
         from numpy import array, average
         assert average(range(10)) == 4.5
-        assert average(array(range(10))) == 4.5
\ No newline at end of file
+        assert average(array(range(10))) == 4.5
+
+    def test_constants(self):
+        import math
+        from numpy import inf, e
+        assert type(inf) is float
+        assert inf == float("inf")
+        assert e == math.e
+        assert type(e) is float
\ No newline at end of file
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -581,8 +581,10 @@
         from numpy import array, dtype
 
         assert array([True]).dtype is dtype(bool)
-        assert array([True, 1]).dtype is dtype(long)
-        assert array([1, 2, 3]).dtype is dtype(long)
+        assert array([True, False]).dtype is dtype(bool)
+        assert array([True, 1]).dtype is dtype(int)
+        assert array([1, 2, 3]).dtype is dtype(int)
+        assert array([1L, 2, 3]).dtype is dtype(long)
         assert array([1.2, True]).dtype is dtype(float)
         assert array([1.2, 5]).dtype is dtype(float)
         assert array([]).dtype is dtype(float)
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -234,7 +234,7 @@
             assert b[i] == math.sin(a[i])
 
         a = sin(array([True, False], dtype=bool))
-        assert a[0] == sin(1)
+        assert abs(a[0] - sin(1)) < 1e-7 # a[0] will be less precise
         assert a[1] == 0.0
 
     def test_cos(self):
@@ -298,6 +298,25 @@
         b = arctan(a)
         assert math.isnan(b[0])
 
+    def test_arcsinh(self):
+        import math
+        from numpy import arcsinh, inf
+
+        for v in [inf, -inf, 1.0, math.e]:
+            assert math.asinh(v) == arcsinh(v)
+        assert math.isnan(arcsinh(float("nan")))
+
+    def test_arctanh(self):
+        import math
+        from numpy import arctanh
+
+        for v in [.99, .5, 0, -.5, -.99]:
+            assert math.atanh(v) == arctanh(v)
+        for v in [2.0, -2.0]:
+            assert math.isnan(arctanh(v))
+        for v in [1.0, -1.0]:
+            assert arctanh(v) == math.copysign(float("inf"), v)
+
     def test_reduce_errors(self):
         from numpy import sin, add
 
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -1,20 +1,24 @@
 from pypy.jit.metainterp.test.support import LLJitMixin
 from pypy.module.micronumpy import interp_ufuncs, signature
 from pypy.module.micronumpy.compile import (numpy_compile, FakeSpace,
-    FloatObject)
-from pypy.module.micronumpy.interp_dtype import W_Float64Dtype, W_Int64Dtype
+    FloatObject, IntObject)
+from pypy.module.micronumpy.interp_dtype import W_Int32Dtype, W_Float64Dtype, W_Int64Dtype, W_UInt64Dtype
 from pypy.module.micronumpy.interp_numarray import (BaseArray, SingleDimArray,
     SingleDimSlice, scalar_w)
 from pypy.rlib.nonconst import NonConstant
 from pypy.rpython.annlowlevel import llstr
 from pypy.rpython.test.test_llinterp import interpret
 
+import py
+
 
 class TestNumpyJIt(LLJitMixin):
     def setup_class(cls):
         cls.space = FakeSpace()
         cls.float64_dtype = cls.space.fromcache(W_Float64Dtype)
         cls.int64_dtype = cls.space.fromcache(W_Int64Dtype)
+        cls.uint64_dtype = cls.space.fromcache(W_UInt64Dtype)
+        cls.int32_dtype = cls.space.fromcache(W_Int32Dtype)
 
     def test_add(self):
         def f(i):
@@ -303,6 +307,31 @@
                           'int_lt': 1, 'guard_true': 1, 'jump': 1})
         assert result == 11.0
 
+    def test_int32_sum(self):
+        py.test.skip("pypy/jit/backend/llimpl.py needs to be changed to "
+                     "deal correctly with int dtypes for this test to "
+                     "work. skip for now until someone feels up to the task")
+        space = self.space
+        float64_dtype = self.float64_dtype
+        int32_dtype = self.int32_dtype
+
+        def f(n):
+            if NonConstant(False):
+                dtype = float64_dtype
+            else:
+                dtype = int32_dtype
+            ar = SingleDimArray(n, dtype=dtype)
+            i = 0
+            while i < n:
+                ar.get_concrete().setitem(i, int32_dtype.box(7))
+                i += 1
+            v = ar.descr_add(space, ar).descr_sum(space)
+            assert isinstance(v, IntObject)
+            return v.intval
+
+        result = self.meta_interp(f, [5], listops=True, backendopt=True)
+        assert result == f(5)
+
 class TestTranslation(object):
     def test_compile(self):
         x = numpy_compile('aa+f*f/a-', 10)
diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py
--- a/pypy/module/pypyjit/interp_jit.py
+++ b/pypy/module/pypyjit/interp_jit.py
@@ -13,7 +13,6 @@
 from pypy.interpreter.pyframe import PyFrame
 from pypy.interpreter.pyopcode import ExitFrame
 from pypy.interpreter.gateway import unwrap_spec
-from pypy.interpreter.baseobjspace import ObjSpace, W_Root
 from opcode import opmap
 from pypy.rlib.nonconst import NonConstant
 from pypy.jit.metainterp.resoperation import rop
@@ -221,7 +220,6 @@
     def __init__(self, space):
         self.w_compile_hook = space.w_None
 
- at unwrap_spec(ObjSpace, W_Root)
 def set_compile_hook(space, w_hook):
     """ set_compile_hook(hook)
 
diff --git a/pypy/module/pypyjit/test/test_policy.py b/pypy/module/pypyjit/test/test_policy.py
--- a/pypy/module/pypyjit/test/test_policy.py
+++ b/pypy/module/pypyjit/test/test_policy.py
@@ -3,8 +3,8 @@
 pypypolicy = policy.PyPyJitPolicy()
 
 def test_id_any():
-    from pypy.objspace.std.default import id__ANY
-    assert pypypolicy.look_inside_function(id__ANY)
+    from pypy.objspace.std.intobject import add__Int_Int
+    assert pypypolicy.look_inside_function(add__Int_Int)
 
 def test_bigint():
     from pypy.rlib.rbigint import rbigint
diff --git a/pypy/module/pypyjit/test_pypy_c/test_00_model.py b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
--- a/pypy/module/pypyjit/test_pypy_c/test_00_model.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
@@ -50,8 +50,7 @@
         cmdline.append(str(self.filepath))
         #
         print cmdline, logfile
-        env={'PYPYLOG': 'jit-log-opt,jit-summary:' + str(logfile)}
-        #env={'PYPYLOG': ':' + str(logfile)}
+        env={'PYPYLOG': 'jit-log-opt,jit-log-noopt,jit-summary:' + str(logfile)}
         pipe = subprocess.Popen(cmdline,
                                 env=env,
                                 stdout=subprocess.PIPE,
diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py
--- a/pypy/module/pypyjit/test_pypy_c/test_call.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_call.py
@@ -366,12 +366,12 @@
             # make sure that the "block" is not allocated
             ...
             i20 = force_token()
-            setfield_gc(p0, i20, descr=<SignedFieldDescr .*PyFrame.vable_token .*>)
             p22 = new_with_vtable(19511408)
             p24 = new_array(1, descr=<GcPtrArrayDescr>)
             p26 = new_with_vtable(ConstClass(W_ListObject))
             p27 = new(descr=<SizeDescr .*>)
             p29 = new_array(0, descr=<GcPtrArrayDescr>)
+            setfield_gc(p0, i20, descr=<SignedFieldDescr .*PyFrame.vable_token .*>)
             setfield_gc(p27, p29, descr=<GcPtrFieldDescr list.items .*>)
             setfield_gc(p26, p27, descr=<.* .*W_ListObject.inst_wrappeditems .*>)
             setarrayitem_gc(p24, 0, p26, descr=<GcPtrArrayDescr>)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_containers.py b/pypy/module/pypyjit/test_pypy_c/test_containers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_containers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_containers.py
@@ -49,3 +49,24 @@
             p33 = call(ConstClass(ll_get_value__dicttablePtr_Signed), p18, i28, descr=...)
             ...
         """)
+
+    def test_list(self):
+        def main(n):
+            i = 0
+            while i < n:
+                z = list(())
+                z.append(1)
+                i += z[-1] / len(z)
+            return i
+
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loop, = log.loops_by_filename(self.filepath)
+        assert loop.match("""
+            i7 = int_lt(i5, i6)
+            guard_true(i7, descr=...)
+            guard_not_invalidated(descr=...)
+            i9 = int_add(i5, 1)
+            --TICK--
+            jump(..., descr=...)
+        """)
\ No newline at end of file
diff --git a/pypy/module/pypyjit/test_pypy_c/test_generators.py b/pypy/module/pypyjit/test_pypy_c/test_generators.py
--- a/pypy/module/pypyjit/test_pypy_c/test_generators.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_generators.py
@@ -19,8 +19,8 @@
         assert loop.match_by_id("generator", """
             i16 = force_token()
             p45 = new_with_vtable(ConstClass(W_IntObject))
-            setfield_gc(p45, i29, descr=<SignedFieldDescr .*>)
             i47 = arraylen_gc(p8, descr=<GcPtrArrayDescr>) # Should be removed by backend
             setarrayitem_gc(p8, 0, p45, descr=<GcPtrArrayDescr>)
+            setfield_gc(p45, i29, descr=<SignedFieldDescr .*>)
             jump(..., descr=...)
             """)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_instance.py b/pypy/module/pypyjit/test_pypy_c/test_instance.py
--- a/pypy/module/pypyjit/test_pypy_c/test_instance.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_instance.py
@@ -125,8 +125,8 @@
             i12 = force_token()
             --TICK--
             p20 = new_with_vtable(ConstClass(W_IntObject))
+            setfield_gc(ConstPtr(ptr21), p20, descr=<GcPtrFieldDescr .*TypeCell.inst_w_value .*>)
             setfield_gc(p20, i11, descr=<SignedFieldDescr.*W_IntObject.inst_intval .*>)
-            setfield_gc(ConstPtr(ptr21), p20, descr=<GcPtrFieldDescr .*TypeCell.inst_w_value .*>)
             jump(p0, p1, p2, p3, p4, p20, p6, i7, p20, descr=<Loop.>)
         """)
 
diff --git a/pypy/module/pypyjit/test_pypy_c/test_min_max.py b/pypy/module/pypyjit/test_pypy_c/test_min_max.py
--- a/pypy/module/pypyjit/test_pypy_c/test_min_max.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_min_max.py
@@ -42,7 +42,7 @@
         assert len(guards) < 20
         assert loop.match_by_id('max',"""
             ...
-            p76 = call_may_force(ConstClass(min_max_loop__max), _, _, descr=...)
+            p76 = call_may_force(ConstClass(min_max_trampoline), _, _, descr=...)
             ...
         """)
 
@@ -63,6 +63,6 @@
         assert len(guards) < 20
         assert loop.match_by_id('max',"""
             ...
-            p76 = call_may_force(ConstClass(min_max_loop__max), _, _, descr=...)
+            p76 = call_may_force(ConstClass(min_max_trampoline), _, _, descr=...)
             ...
         """)
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -55,6 +55,7 @@
         'exc_info'              : 'vm.exc_info', 
         'exc_clear'             : 'vm.exc_clear', 
         'settrace'              : 'vm.settrace',
+        'gettrace'              : 'vm.gettrace',
         'setprofile'            : 'vm.setprofile',
         'getprofile'            : 'vm.getprofile',
         'call_tracing'          : 'vm.call_tracing',
diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -478,6 +478,7 @@
         sys.settrace(trace)
         try:
             x()
+            assert sys.gettrace() is trace
         finally:
             sys.settrace(None)
         assert len(counts) == 1
diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -129,14 +129,19 @@
 function call.  See the debugger chapter in the library manual."""
     space.getexecutioncontext().settrace(w_func)
 
+def gettrace(space):
+    """Return the global debug tracing function set with sys.settrace.
+See the debugger chapter in the library manual."""
+    return space.getexecutioncontext().gettrace()
+
 def setprofile(space, w_func):
     """Set the profiling function.  It will be called on each function call
 and return.  See the profiler chapter in the library manual."""
     space.getexecutioncontext().setprofile(w_func)
 
 def getprofile(space):
-    """Set the profiling function.  It will be called on each function call
-and return.  See the profiler chapter in the library manual."""
+    """Return the profiling function set with sys.setprofile.
+See the profiler chapter in the library manual."""
     w_func = space.getexecutioncontext().getprofile()
     if w_func is not None:
         return w_func
diff --git a/pypy/module/test_lib_pypy/test_greenlet.py b/pypy/module/test_lib_pypy/test_greenlet.py
--- a/pypy/module/test_lib_pypy/test_greenlet.py
+++ b/pypy/module/test_lib_pypy/test_greenlet.py
@@ -258,3 +258,25 @@
             assert sys.exc_info() == (None, None, None)
 
         greenlet(f).switch()
+
+    def test_gr_frame(self):
+        from greenlet import greenlet
+        import sys
+        def f2():
+            assert g.gr_frame is None
+            gmain.switch()
+            assert g.gr_frame is None
+        def f1():
+            assert gmain.gr_frame is gmain_frame
+            assert g.gr_frame is None
+            f2()
+            assert g.gr_frame is None
+        gmain = greenlet.getcurrent()
+        assert gmain.gr_frame is None
+        gmain_frame = sys._getframe()
+        g = greenlet(f1)
+        assert g.gr_frame is None
+        g.switch()
+        assert g.gr_frame.f_code.co_name == 'f2'
+        g.switch()
+        assert g.gr_frame is None
diff --git a/pypy/module/test_lib_pypy/test_stackless_pickle.py b/pypy/module/test_lib_pypy/test_stackless_pickle.py
--- a/pypy/module/test_lib_pypy/test_stackless_pickle.py
+++ b/pypy/module/test_lib_pypy/test_stackless_pickle.py
@@ -1,25 +1,27 @@
-import py; py.test.skip("XXX port me")
+import py
+py.test.skip("in-progress, maybe")
 from pypy.conftest import gettestobjspace, option
 
 class AppTest_Stackless:
 
     def setup_class(cls):
-        import py.test
-        py.test.importorskip('greenlet')
-        space = gettestobjspace(usemodules=('_stackless', '_socket'))
+        space = gettestobjspace(usemodules=('_continuation', '_socket'))
         cls.space = space
-        # cannot test the unpickle part on top of py.py
+        if option.runappdirect:
+            cls.w_lev = space.wrap(14)
+        else:
+            cls.w_lev = space.wrap(2)
 
     def test_pickle(self):
         import new, sys
 
         mod = new.module('mod')
         sys.modules['mod'] = mod
+        mod.lev = self.lev
         try:
             exec '''
 import pickle, sys
 import stackless
-lev = 14
 
 ch = stackless.channel()
 seen = []
diff --git a/pypy/module/thread/gil.py b/pypy/module/thread/gil.py
--- a/pypy/module/thread/gil.py
+++ b/pypy/module/thread/gil.py
@@ -16,7 +16,7 @@
 
 class GILThreadLocals(OSThreadLocals):
     """A version of OSThreadLocals that enforces a GIL."""
-    ll_GIL = thread.null_ll_lock
+    gil_ready = False
 
     def initialize(self, space):
         # add the GIL-releasing callback as an action on the space
@@ -25,12 +25,10 @@
 
     def setup_threads(self, space):
         """Enable threads in the object space, if they haven't already been."""
-        if not self.ll_GIL:
-            try:
-                self.ll_GIL = thread.allocate_ll_lock()
-            except thread.error:
+        if not self.gil_ready:
+            if not thread.gil_allocate():
                 raise wrap_thread_error(space, "can't allocate GIL")
-            thread.acquire_NOAUTO(self.ll_GIL, True)
+            self.gil_ready = True
             self.enter_thread(space)   # setup the main thread
             result = True
         else:
@@ -44,19 +42,16 @@
         # test_lock_again after the global state was cleared by
         # test_compile_lock.  As a workaround, we repatch these global
         # fields systematically.
-        spacestate.ll_GIL = self.ll_GIL
         invoke_around_extcall(before_external_call, after_external_call)
         return result
 
     def reinit_threads(self, space):
-        if self.ll_GIL:
-            self.ll_GIL = thread.allocate_ll_lock()
-            thread.acquire_NOAUTO(self.ll_GIL, True)
-            self.enter_thread(space)
+        if self.gil_ready:
+            self.gil_ready = False
+            self.setup_threads(space)
 
     def yield_thread(self):
-        thread.yield_thread()  # explicitly release the gil (used by test_gil)
-
+        do_yield_thread()
 
 class GILReleaseAction(PeriodicAsyncAction):
     """An action called every sys.checkinterval bytecodes.  It releases
@@ -64,16 +59,12 @@
     """
 
     def perform(self, executioncontext, frame):
-        # Other threads can run between the release() and the acquire()
-        # implicit in the following external function call (which has
-        # otherwise no effect).
-        thread.yield_thread()
+        do_yield_thread()
 
 
 class SpaceState:
 
     def _freeze_(self):
-        self.ll_GIL = thread.null_ll_lock
         self.action_after_thread_switch = None
         # ^^^ set by AsyncAction.fire_after_thread_switch()
         return False
@@ -95,14 +86,14 @@
     # this function must not raise, in such a way that the exception
     # transformer knows that it cannot raise!
     e = get_errno()
-    thread.release_NOAUTO(spacestate.ll_GIL)
+    thread.gil_release()
     set_errno(e)
 before_external_call._gctransformer_hint_cannot_collect_ = True
 before_external_call._dont_reach_me_in_del_ = True
 
 def after_external_call():
     e = get_errno()
-    thread.acquire_NOAUTO(spacestate.ll_GIL, True)
+    thread.gil_acquire()
     thread.gc_thread_run()
     spacestate.after_thread_switch()
     set_errno(e)
@@ -115,3 +106,18 @@
 # pointers in the shadow stack.  This is necessary because the GIL is
 # not held after the call to before_external_call() or before the call
 # to after_external_call().
+
+def do_yield_thread():
+    # explicitly release the gil, in a way that tries to give more
+    # priority to other threads (as opposed to continuing to run in
+    # the same thread).
+    if thread.gil_yield_thread():
+        thread.gc_thread_run()
+        spacestate.after_thread_switch()
+do_yield_thread._gctransformer_hint_close_stack_ = True
+do_yield_thread._dont_reach_me_in_del_ = True
+do_yield_thread._dont_inline_ = True
+
+# do_yield_thread() needs a different hint: _gctransformer_hint_close_stack_.
+# The *_external_call() functions are themselves called only from the rffi
+# module from a helper function that also has this hint.
diff --git a/pypy/module/thread/ll_thread.py b/pypy/module/thread/ll_thread.py
--- a/pypy/module/thread/ll_thread.py
+++ b/pypy/module/thread/ll_thread.py
@@ -17,7 +17,8 @@
     include_dirs = [str(py.path.local(autopath.pypydir).join('translator', 'c'))],
     export_symbols = ['RPyThreadGetIdent', 'RPyThreadLockInit',
                       'RPyThreadAcquireLock', 'RPyThreadReleaseLock',
-                      'RPyThreadYield',
+                      'RPyGilAllocate', 'RPyGilYieldThread',
+                      'RPyGilRelease', 'RPyGilAcquire',
                       'RPyThreadGetStackSize', 'RPyThreadSetStackSize',
                       'RPyOpaqueDealloc_ThreadLock',
                       'RPyThreadAfterFork']
@@ -69,8 +70,16 @@
                                          [TLOCKP], lltype.Void,
                                          _nowrapper=True)
 
-# this function does nothing apart from releasing the GIL temporarily.
-yield_thread = llexternal('RPyThreadYield', [], lltype.Void, threadsafe=True)
+# these functions manipulate directly the GIL, whose definition does not
+# escape the C code itself
+gil_allocate     = llexternal('RPyGilAllocate', [], lltype.Signed,
+                              _nowrapper=True)
+gil_yield_thread = llexternal('RPyGilYieldThread', [], lltype.Signed,
+                              _nowrapper=True)
+gil_release      = llexternal('RPyGilRelease', [], lltype.Void,
+                              _nowrapper=True)
+gil_acquire      = llexternal('RPyGilAcquire', [], lltype.Void,
+                              _nowrapper=True)
 
 def allocate_lock():
     return Lock(allocate_ll_lock())
diff --git a/pypy/module/thread/test/test_gil.py b/pypy/module/thread/test/test_gil.py
--- a/pypy/module/thread/test/test_gil.py
+++ b/pypy/module/thread/test/test_gil.py
@@ -30,19 +30,34 @@
     use_threads = True
     bigtest = False
 
-    def test_one_thread(self):
+    def test_one_thread(self, skew=+1):
+        from pypy.rlib.debug import debug_print
         if self.bigtest:
-            N = 1000000
+            N = 100000
+            skew *= 25000
         else:
             N = 100
+            skew *= 25
         space = FakeSpace()
         class State:
             pass
         state = State()
-        def runme():
-            for i in range(N):
+        def runme(main=False):
+            j = 0
+            for i in range(N + [-skew, skew][main]):
+                state.datalen1 += 1   # try to crash if the GIL is not
+                state.datalen2 += 1   # correctly acquired
                 state.data.append((thread.get_ident(), i))
+                state.datalen3 += 1
+                state.datalen4 += 1
+                assert state.datalen1 == len(state.data)
+                assert state.datalen2 == len(state.data)
+                assert state.datalen3 == len(state.data)
+                assert state.datalen4 == len(state.data)
+                debug_print(main, i, state.datalen4)
                 state.threadlocals.yield_thread()
+                assert i == j
+                j += 1
         def bootstrap():
             try:
                 runme()
@@ -50,20 +65,26 @@
                 thread.gc_thread_die()
         def f():
             state.data = []
+            state.datalen1 = 0
+            state.datalen2 = 0
+            state.datalen3 = 0
+            state.datalen4 = 0
             state.threadlocals = gil.GILThreadLocals()
             state.threadlocals.setup_threads(space)
             thread.gc_thread_prepare()
             subident = thread.start_new_thread(bootstrap, ())
             mainident = thread.get_ident()
-            runme()
+            runme(True)
             still_waiting = 3000
             while len(state.data) < 2*N:
+                debug_print(len(state.data))
                 if not still_waiting:
                     raise ValueError("time out")
                 still_waiting -= 1
                 if not we_are_translated(): gil.before_external_call()
                 time.sleep(0.01)
                 if not we_are_translated(): gil.after_external_call()
+            debug_print("leaving!")
             i1 = i2 = 0
             for tid, i in state.data:
                 if tid == mainident:
@@ -72,14 +93,17 @@
                     assert i == i2; i2 += 1
                 else:
                     assert 0
-            assert i1 == N
-            assert i2 == N
+            assert i1 == N + skew
+            assert i2 == N - skew
             return len(state.data)
 
         fn = self.getcompiled(f, [])
         res = fn()
         assert res == 2*N
 
+    def test_one_thread_rev(self):
+        self.test_one_thread(skew=-1)
+
 
 class TestRunDirectly(GILTests):
     def getcompiled(self, f, argtypes):
diff --git a/pypy/module/thread/test/test_thread.py b/pypy/module/thread/test/test_thread.py
--- a/pypy/module/thread/test/test_thread.py
+++ b/pypy/module/thread/test/test_thread.py
@@ -225,7 +225,8 @@
 
         def busy_wait():
             for x in range(1000):
-                time.sleep(0.01)
+                print 'tick...', x  # <-force the GIL to be released, as
+                time.sleep(0.01)    #   time.sleep doesn't do non-translated
 
         # This is normally called by app_main.py
         signal.signal(signal.SIGINT, signal.default_int_handler)
diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -6,6 +6,7 @@
 from pypy.interpreter.typedef import default_identity_hash
 from pypy.tool.sourcetools import compile2, func_with_new_name
 from pypy.module.__builtin__.interp_classobj import W_InstanceObject
+from pypy.rlib.objectmodel import specialize
 
 def object_getattribute(space):
     "Utility that returns the app-level descriptor object.__getattribute__."
@@ -257,15 +258,15 @@
             msg = "'%s' has no length" % (name,)
             raise OperationError(space.w_TypeError, space.wrap(msg))
         w_res = space.get_and_call_function(w_descr, w_obj)
-        space._check_len_result(w_res)
-        return w_res
+        return space.wrap(space._check_len_result(w_res))
 
     def _check_len_result(space, w_obj):
         # Will complain if result is too big.
-        result = space.int_w(w_obj)
+        result = space.int_w(space.int(w_obj))
         if result < 0:
             raise OperationError(space.w_ValueError,
                                  space.wrap("__len__() should return >= 0"))
+        return result
 
     def iter(space, w_obj):
         w_descr = space.lookup(w_obj, '__iter__')
@@ -507,6 +508,7 @@
     def issubtype(space, w_sub, w_type):
         return space._type_issubtype(w_sub, w_type)
 
+    @specialize.arg_or_var(2)
     def isinstance(space, w_inst, w_type):
         return space.wrap(space._type_isinstance(w_inst, w_type))
 
diff --git a/pypy/objspace/std/boolobject.py b/pypy/objspace/std/boolobject.py
--- a/pypy/objspace/std/boolobject.py
+++ b/pypy/objspace/std/boolobject.py
@@ -1,8 +1,10 @@
+from pypy.rlib.rbigint import rbigint
+from pypy.rlib.rarithmetic import r_uint
+from pypy.interpreter.error import OperationError
 from pypy.objspace.std.model import registerimplementation, W_Object
 from pypy.objspace.std.register_all import register_all
 from pypy.objspace.std.intobject import W_IntObject
 
-
 class W_BoolObject(W_Object):
     from pypy.objspace.std.booltype import bool_typedef as typedef
     _immutable_fields_ = ['boolval']
@@ -20,6 +22,21 @@
     def unwrap(w_self, space):
         return w_self.boolval
 
+    def int_w(w_self, space):
+        return int(w_self.boolval)
+
+    def uint_w(w_self, space):
+        intval = int(w_self.boolval)
+        if intval < 0:
+            raise OperationError(space.w_ValueError,
+                                 space.wrap("cannot convert negative integer to unsigned"))
+        else:
+            return r_uint(intval)
+
+    def bigint_w(w_self, space):
+        return rbigint.fromint(int(w_self.boolval))
+
+
 registerimplementation(W_BoolObject)
 
 W_BoolObject.w_False = W_BoolObject(False)
diff --git a/pypy/objspace/std/default.py b/pypy/objspace/std/default.py
--- a/pypy/objspace/std/default.py
+++ b/pypy/objspace/std/default.py
@@ -1,48 +1,16 @@
 """Default implementation for some operation."""
 
-from pypy.interpreter.error import OperationError
+from pypy.interpreter.error import OperationError, typed_unwrap_error_msg
 from pypy.objspace.std.register_all import register_all
-from pypy.rlib import objectmodel
 
 
-# The following default implementations are used before delegation is tried.
-# 'id' is normally the address of the wrapper.
-
-def id__ANY(space, w_obj):
-    #print 'id:', w_obj
-    return space.wrap(objectmodel.compute_unique_id(w_obj))
-
 # __init__ should succeed if called internally as a multimethod
 
 def init__ANY(space, w_obj, __args__):
     pass
 
-def typed_unwrap_error_msg(space, expected, w_obj):
-    type_name = space.type(w_obj).getname(space)
-    return space.wrap("expected %s, got %s object" % (expected, type_name))
-
-def int_w__ANY(space,w_obj):
-    raise OperationError(space.w_TypeError,
-                         typed_unwrap_error_msg(space, "integer", w_obj))
-
-def str_w__ANY(space,w_obj):
-    raise OperationError(space.w_TypeError,
-                         typed_unwrap_error_msg(space, "string", w_obj))
-
 def float_w__ANY(space,w_obj):
     raise OperationError(space.w_TypeError,
                          typed_unwrap_error_msg(space, "float", w_obj))
 
-def uint_w__ANY(space,w_obj):
-    raise OperationError(space.w_TypeError,
-                         typed_unwrap_error_msg(space, "integer", w_obj))
-
-def unicode_w__ANY(space,w_obj):
-    raise OperationError(space.w_TypeError,
-                         typed_unwrap_error_msg(space, "unicode", w_obj))
-
-def bigint_w__ANY(space,w_obj):
-    raise OperationError(space.w_TypeError,
-                         typed_unwrap_error_msg(space, "integer", w_obj))
-
 register_all(vars())
diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py
--- a/pypy/objspace/std/intobject.py
+++ b/pypy/objspace/std/intobject.py
@@ -1,12 +1,13 @@
 from pypy.interpreter.error import OperationError
 from pypy.objspace.std import newformat
+from pypy.objspace.std.inttype import wrapint
 from pypy.objspace.std.model import registerimplementation, W_Object
-from pypy.objspace.std.register_all import register_all
 from pypy.objspace.std.multimethod import FailedToImplementArgs
 from pypy.objspace.std.noneobject import W_NoneObject
+from pypy.objspace.std.register_all import register_all
+from pypy.rlib import jit
 from pypy.rlib.rarithmetic import ovfcheck, ovfcheck_lshift, LONG_BIT, r_uint
 from pypy.rlib.rbigint import rbigint
-from pypy.objspace.std.inttype import wrapint
 
 """
 In order to have the same behavior running
@@ -20,7 +21,7 @@
     _immutable_fields_ = ['intval']
 
     from pypy.objspace.std.inttype import int_typedef as typedef
-    
+
     def __init__(w_self, intval):
         w_self.intval = intval
 
@@ -30,7 +31,18 @@
 
     def unwrap(w_self, space):
         return int(w_self.intval)
+    int_w = unwrap
 
+    def uint_w(w_self, space):
+        intval = w_self.intval
+        if intval < 0:
+            raise OperationError(space.w_ValueError,
+                                 space.wrap("cannot convert negative integer to unsigned"))
+        else:
+            return r_uint(intval)
+
+    def bigint_w(w_self, space):
+        return rbigint.fromint(w_self.intval)
 
 registerimplementation(W_IntObject)
 
@@ -39,20 +51,6 @@
 # alias and then teach copy_multimethods in smallintobject.py to override
 # it. See int__Int for example.
 
-def int_w__Int(space, w_int1):
-    return int(w_int1.intval)
-
-def uint_w__Int(space, w_int1):
-    intval = w_int1.intval
-    if intval < 0:
-        raise OperationError(space.w_ValueError,
-                             space.wrap("cannot convert negative integer to unsigned"))
-    else:
-        return r_uint(intval)
-
-def bigint_w__Int(space, w_int1):
-    return rbigint.fromint(w_int1.intval)
-
 def repr__Int(space, w_int1):
     a = w_int1.intval
     res = str(a)
@@ -138,7 +136,7 @@
     x = float(w_int1.intval)
     y = float(w_int2.intval)
     if y == 0.0:
-        raise FailedToImplementArgs(space.w_ZeroDivisionError, space.wrap("float division"))    
+        raise FailedToImplementArgs(space.w_ZeroDivisionError, space.wrap("float division"))
     return space.wrap(x / y)
 
 def mod__Int_Int(space, w_int1, w_int2):
@@ -172,7 +170,8 @@
 
 
 # helper for pow()
-def _impl_int_int_pow(space, iv, iw, iz=0):
+ at jit.look_inside_iff(lambda space, iv, iw, iz: jit.isconstant(iw) and jit.isconstant(iz))
+def _impl_int_int_pow(space, iv, iw, iz):
     if iw < 0:
         if iz != 0:
             raise OperationError(space.w_TypeError,
@@ -200,7 +199,7 @@
     except OverflowError:
         raise FailedToImplementArgs(space.w_OverflowError,
                                 space.wrap("integer exponentiation"))
-    return wrapint(space, ix)
+    return ix
 
 def pow__Int_Int_Int(space, w_int1, w_int2, w_int3):
     x = w_int1.intval
@@ -209,12 +208,12 @@
     if z == 0:
         raise OperationError(space.w_ValueError,
                              space.wrap("pow() 3rd argument cannot be 0"))
-    return _impl_int_int_pow(space, x, y, z)
+    return space.wrap(_impl_int_int_pow(space, x, y, z))
 
 def pow__Int_Int_None(space, w_int1, w_int2, w_int3):
     x = w_int1.intval
     y = w_int2.intval
-    return _impl_int_int_pow(space, x, y)
+    return space.wrap(_impl_int_int_pow(space, x, y, 0))
 
 def neg__Int(space, w_int1):
     a = w_int1.intval
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -386,7 +386,11 @@
     if len(items)== 0:
         raise OperationError(space.w_IndexError,
                              space.wrap("pop from empty list"))
-    idx = space.int_w(w_idx)
+    if space.isinstance_w(w_idx, space.w_float):
+        raise OperationError(space.w_TypeError,
+            space.wrap("integer argument expected, got float")
+        )
+    idx = space.int_w(space.int(w_idx))
     try:
         return items.pop(idx)
     except IndexError:
diff --git a/pypy/objspace/std/longobject.py b/pypy/objspace/std/longobject.py
--- a/pypy/objspace/std/longobject.py
+++ b/pypy/objspace/std/longobject.py
@@ -45,6 +45,26 @@
     fromrarith_int._annspecialcase_ = "specialize:argtype(0)"
     fromrarith_int = staticmethod(fromrarith_int)
 
+    def int_w(w_self, space):
+        try:
+            return w_self.num.toint()
+        except OverflowError:
+            raise OperationError(space.w_OverflowError, space.wrap(
+                "long int too large to convert to int"))
+
+    def uint_w(w_self, space):
+        try:
+            return w_self.num.touint()
+        except ValueError:
+            raise OperationError(space.w_ValueError, space.wrap(
+                "cannot convert negative integer to unsigned int"))
+        except OverflowError:
+            raise OperationError(space.w_OverflowError, space.wrap(
+                "long int too large to convert to unsigned int"))
+
+    def bigint_w(w_self, space):
+        return w_self.num
+
     def __repr__(self):
         return '<W_LongObject(%d)>' % self.num.tolong()
 
@@ -104,27 +124,6 @@
         raise OperationError(space.w_OverflowError,
                              space.wrap("long int too large to convert to float"))
 
-def int_w__Long(space, w_value):
-    try:
-        return w_value.num.toint()
-    except OverflowError:
-        raise OperationError(space.w_OverflowError, space.wrap(
-            "long int too large to convert to int"))
-
-
-def uint_w__Long(space, w_value):
-    try:
-        return w_value.num.touint()
-    except ValueError:
-        raise OperationError(space.w_ValueError, space.wrap(
-            "cannot convert negative integer to unsigned int"))
-    except OverflowError:
-        raise OperationError(space.w_OverflowError, space.wrap(
-            "long int too large to convert to unsigned int"))
-
-def bigint_w__Long(space, w_value):
-    return w_value.num
-
 def repr__Long(space, w_long):
     return space.wrap(w_long.num.repr())
 
diff --git a/pypy/objspace/std/model.py b/pypy/objspace/std/model.py
--- a/pypy/objspace/std/model.py
+++ b/pypy/objspace/std/model.py
@@ -442,6 +442,13 @@
         mm.dispatch_tree = merge(self.dispatch_tree, other.dispatch_tree)
         return mm
 
+NOT_MULTIMETHODS = dict.fromkeys(
+    ['delattr', 'delete', 'get', 'id', 'inplace_div', 'inplace_floordiv',
+     'inplace_lshift', 'inplace_mod', 'inplace_pow', 'inplace_rshift',
+     'inplace_truediv', 'is_', 'set', 'setattr', 'type', 'userdel',
+     'isinstance', 'issubtype'])
+# XXX should we just remove those from the method table or we're happy
+#     with just not having multimethods?
 
 class MM:
     """StdObjSpace multimethods"""
@@ -451,22 +458,17 @@
     init    = StdObjSpaceMultiMethod('__init__', 1, general__args__=True)
     getnewargs = StdObjSpaceMultiMethod('__getnewargs__', 1)
     # special visible multimethods
-    int_w   = StdObjSpaceMultiMethod('int_w', 1, [])     # returns an unwrapped int
-    str_w   = StdObjSpaceMultiMethod('str_w', 1, [])     # returns an unwrapped string
     float_w = StdObjSpaceMultiMethod('float_w', 1, [])   # returns an unwrapped float
-    uint_w  = StdObjSpaceMultiMethod('uint_w', 1, [])    # returns an unwrapped unsigned int (r_uint)
-    unicode_w = StdObjSpaceMultiMethod('unicode_w', 1, [])    # returns an unwrapped list of unicode characters
-    bigint_w = StdObjSpaceMultiMethod('bigint_w', 1, []) # returns an unwrapped rbigint
     # NOTE: when adding more sometype_w() methods, you need to write a
     # stub in default.py to raise a space.w_TypeError
     marshal_w = StdObjSpaceMultiMethod('marshal_w', 1, [], extra_args=['marshaller'])
-    log     = StdObjSpaceMultiMethod('log', 1, [], extra_args=['base'])
 
     # add all regular multimethods here
     for _name, _symbol, _arity, _specialnames in ObjSpace.MethodTable:
-        if _name not in locals():
+        if _name not in locals() or _name in NOT_MULTIMETHODS:
             mm = StdObjSpaceMultiMethod(_symbol, _arity, _specialnames)
             locals()[_name] = mm
             del mm
 
     pow.extras['defaults'] = (None,)
+
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -1,19 +1,17 @@
 import __builtin__
 import types
-from pypy.interpreter import pyframe, function, special
+from pypy.interpreter import special
 from pypy.interpreter.baseobjspace import ObjSpace, Wrappable
 from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.interpreter.typedef import get_unique_interplevel_subclass
 from pypy.objspace.std import (builtinshortcut, stdtypedef, frame, model,
                                transparent, callmethod, proxyobject)
 from pypy.objspace.descroperation import DescrOperation, raiseattrerror
-from pypy.rlib.objectmodel import instantiate, r_dict, specialize
+from pypy.rlib.objectmodel import instantiate, r_dict, specialize, is_annotation_constant
 from pypy.rlib.debug import make_sure_not_resized
 from pypy.rlib.rarithmetic import base_int, widen
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.rlib import jit
-from pypy.rlib.rbigint import rbigint
-from pypy.tool.sourcetools import func_with_new_name
 
 # Object imports
 from pypy.objspace.std.boolobject import W_BoolObject
@@ -85,6 +83,12 @@
         if self.config.objspace.std.withtproxy:
             transparent.setup(self)
 
+        for type, classes in self.model.typeorder.iteritems():
+            if len(classes) >= 3:
+                # W_Root, AnyXxx and actual object
+                self.gettypefor(type).interplevel_cls = classes[0][0]
+
+
     def get_builtin_types(self):
         return self.builtin_types
 
@@ -569,10 +573,19 @@
             return self.wrap(w_sub.issubtype(w_type))
         raise OperationError(self.w_TypeError, self.wrap("need type objects"))
 
+    @specialize.arg_or_var(2)
     def _type_isinstance(self, w_inst, w_type):
-        if isinstance(w_type, W_TypeObject):
-            return self.type(w_inst).issubtype(w_type)
-        raise OperationError(self.w_TypeError, self.wrap("need type object"))
+        if not isinstance(w_type, W_TypeObject):
+            raise OperationError(self.w_TypeError,
+                                 self.wrap("need type object"))
+        if is_annotation_constant(w_type):
+            cls = w_type.interplevel_cls
+            if cls is not None:
+                assert w_inst is not None
+                if isinstance(w_inst, cls):
+                    return True
+        return self.type(w_inst).issubtype(w_type)
 
+    @specialize.arg_or_var(2)
     def isinstance_w(space, w_inst, w_type):
         return space._type_isinstance(w_inst, w_type)
diff --git a/pypy/objspace/std/rangeobject.py b/pypy/objspace/std/rangeobject.py
--- a/pypy/objspace/std/rangeobject.py
+++ b/pypy/objspace/std/rangeobject.py
@@ -23,7 +23,7 @@
 
 class W_RangeListObject(W_Object):
     typedef = listtype.list_typedef
-    
+
     def __init__(w_self, start, step, length):
         assert step != 0
         w_self.start = start
@@ -40,7 +40,7 @@
         if not length:
             w_self.w_list = space.newlist([])
             return w_self.w_list
-        
+
         arr = [None] * length  # this is to avoid using append.
 
         i = start
@@ -146,7 +146,11 @@
     if length == 0:
         raise OperationError(space.w_IndexError,
                              space.wrap("pop from empty list"))
-    idx = space.int_w(w_idx)
+    if space.isinstance_w(w_idx, space.w_float):
+        raise OperationError(space.w_TypeError,
+            space.wrap("integer argument expected, got float")
+        )
+    idx = space.int_w(space.int(w_idx))
     if idx == 0:
         result = w_rangelist.start
         w_rangelist.start += w_rangelist.step
diff --git a/pypy/objspace/std/ropeobject.py b/pypy/objspace/std/ropeobject.py
--- a/pypy/objspace/std/ropeobject.py
+++ b/pypy/objspace/std/ropeobject.py
@@ -34,12 +34,18 @@
 
     def unwrap(w_self, space):
         return w_self._node.flatten_string()
+    str_w = unwrap
 
     def create_if_subclassed(w_self):
         if type(w_self) is W_RopeObject:
             return w_self
         return W_RopeObject(w_self._node)
 
+    def unicode_w(w_self, space):
+        # XXX should this use the default encoding?
+        from pypy.objspace.std.unicodetype import plain_str2unicode
+        return plain_str2unicode(space, w_self._node.flatten_string())
+
 W_RopeObject.EMPTY = W_RopeObject(rope.LiteralStringNode.EMPTY)
 W_RopeObject.PREBUILT = [W_RopeObject(rope.LiteralStringNode.PREBUILT[i])
                              for i in range(256)]
@@ -663,9 +669,6 @@
         return W_RopeObject(rope.concatenate(
             rope.multiply(zero, middle), node))
 
-def str_w__Rope(space, w_str):
-    return w_str._node.flatten_string()
-
 def hash__Rope(space, w_str):
     return wrapint(space, rope.hash_rope(w_str._node))
 
diff --git a/pypy/objspace/std/ropeunicodeobject.py b/pypy/objspace/std/ropeunicodeobject.py
--- a/pypy/objspace/std/ropeunicodeobject.py
+++ b/pypy/objspace/std/ropeunicodeobject.py
@@ -91,11 +91,17 @@
         # for testing
         return w_self._node.flatten_unicode()
 
+    def str_w(w_self, space):
+        return space.str_w(space.str(w_self))
+
     def create_if_subclassed(w_self):
         if type(w_self) is W_RopeUnicodeObject:
             return w_self
         return W_RopeUnicodeObject(w_self._node)
 
+    def unicode_w(self, space):
+        return self._node.flatten_unicode()
+
 W_RopeUnicodeObject.EMPTY = W_RopeUnicodeObject(rope.LiteralStringNode.EMPTY)
 
 registerimplementation(W_RopeUnicodeObject)
@@ -157,12 +163,6 @@
     assert isinstance(w_uni, W_RopeUnicodeObject) # help the annotator!
     return w_uni
 
-def str_w__RopeUnicode(space, w_uni):
-    return space.str_w(space.str(w_uni))
-
-def unicode_w__RopeUnicode(space, w_uni):
-    return w_uni._node.flatten_unicode()
-
 def str__RopeUnicode(space, w_uni):
     return space.call_method(w_uni, 'encode')
 
@@ -185,7 +185,7 @@
 
 def eq__RopeUnicode_Rope(space, w_runi, w_rope):
     from pypy.objspace.std.unicodeobject import _unicode_string_comparison
-    return _unicode_string_comparison(space, w_runi, w_rope, 
+    return _unicode_string_comparison(space, w_runi, w_rope,
                     False,  unicode_from_string)
 
 def ne__RopeUnicode_RopeUnicode(space, w_str1, w_str2):
@@ -193,7 +193,7 @@
 
 def ne__RopeUnicode_Rope(space, w_runi, w_rope):
     from pypy.objspace.std.unicodeobject import _unicode_string_comparison
-    return _unicode_string_comparison(space, w_runi, w_rope, 
+    return _unicode_string_comparison(space, w_runi, w_rope,
                     True, unicode_from_string)
 
 def gt__RopeUnicode_RopeUnicode(space, w_str1, w_str2):
@@ -247,7 +247,7 @@
     if (len(l_w) == 1 and
         space.is_w(space.type(l_w[0]), space.w_unicode)):
         return l_w[0]
-    
+
     values_list = []
     for i in range(len(l_w)):
         w_item = l_w[i]
@@ -320,7 +320,7 @@
 
 
 def make_generic(funcname):
-    def func(space, w_self): 
+    def func(space, w_self):
         node = w_self._node
         if node.length() == 0:
             return space.w_False
@@ -578,7 +578,7 @@
         return w_self.create_if_subclassed()
     resultnode = rope.concatenate(rope.multiply(fillchar, padding), self)
     return W_RopeUnicodeObject(resultnode)
-    
+
 def unicode_zfill__RopeUnicode_ANY(space, w_self, w_width):
     self = w_self._node
     length = self.length()
@@ -744,7 +744,7 @@
     except OverflowError:
         raise OperationError(space.w_OverflowError,
                              space.wrap("string too long"))
-    
+
 
 def unicode_encode__RopeUnicode_ANY_ANY(space, w_unistr,
                                         w_encoding=None,
@@ -821,7 +821,7 @@
         try:
             w_newval = space.getitem(w_table, space.wrap(char))
         except OperationError, e:
-            if e.match(space, space.w_KeyError):
+            if e.match(space, space.w_LookupError):
                 result.append(crope)
             else:
                 raise
@@ -848,7 +848,7 @@
     hexdigits = "0123456789abcdef"
     node = w_unicode._node
     size = node.length()
-    
+
     singlequote = doublequote = False
     iter = rope.ItemIterator(node)
     for i in range(size):
@@ -900,7 +900,7 @@
                                   ])
                     j += 2
                     continue
-                
+
         if code >= 0x100:
             result.extend(['\\', "u",
                            hexdigits[(code >> 12) & 0xf],
@@ -932,7 +932,7 @@
             continue
         if code < ord(' ') or code >= 0x7f:
             result.extend(['\\', "x",
-                           hexdigits[(code >> 4) & 0xf], 
+                           hexdigits[(code >> 4) & 0xf],
                            hexdigits[(code >> 0) & 0xf],
                           ])
             j += 1
@@ -964,15 +964,15 @@
 
 def next__RopeUnicodeIter(space, w_ropeiter):
     if w_ropeiter.node is None:
-        raise OperationError(space.w_StopIteration, space.w_None) 
+        raise OperationError(space.w_StopIteration, space.w_None)
     try:
         unichar = w_ropeiter.item_iter.nextunichar()
         w_item = space.wrap(unichar)
     except StopIteration:
         w_ropeiter.node = None
         w_ropeiter.char_iter = None
-        raise OperationError(space.w_StopIteration, space.w_None) 
-    w_ropeiter.index += 1 
+        raise OperationError(space.w_StopIteration, space.w_None)
+    w_ropeiter.index += 1
     return w_item
 
 # XXX __length_hint__()
diff --git a/pypy/objspace/std/smallintobject.py b/pypy/objspace/std/smallintobject.py
--- a/pypy/objspace/std/smallintobject.py
+++ b/pypy/objspace/std/smallintobject.py
@@ -7,16 +7,30 @@
 from pypy.objspace.std.register_all import register_all
 from pypy.objspace.std.noneobject import W_NoneObject
 from pypy.objspace.std.intobject import W_IntObject
+from pypy.interpreter.error import OperationError
 from pypy.rlib.objectmodel import UnboxedValue
+from pypy.rlib.rbigint import rbigint
+from pypy.rlib.rarithmetic import r_uint
 from pypy.tool.sourcetools import func_with_new_name
 
-
 class W_SmallIntObject(W_Object, UnboxedValue):
     __slots__ = 'intval'
     from pypy.objspace.std.inttype import int_typedef as typedef
 
     def unwrap(w_self, space):
         return int(w_self.intval)
+    int_w = unwrap
+
+    def uint_w(w_self, space):
+        intval = w_self.intval
+        if intval < 0:
+            raise OperationError(space.w_ValueError,
+                                 space.wrap("cannot convert negative integer to unsigned"))
+        else:
+            return r_uint(intval)
+
+    def bigint_w(w_self, space):
+        return rbigint.fromint(w_self.intval)
 
 
 registerimplementation(W_SmallIntObject)
diff --git a/pypy/objspace/std/smalllongobject.py b/pypy/objspace/std/smalllongobject.py
--- a/pypy/objspace/std/smalllongobject.py
+++ b/pypy/objspace/std/smalllongobject.py
@@ -39,6 +39,30 @@
     def __repr__(w_self):
         return '<W_SmallLongObject(%d)>' % w_self.longlong
 
+    def int_w(w_self, space):
+        a = w_self.longlong
+        b = intmask(a)
+        if b == a:
+            return b
+        else:
+            raise OperationError(space.w_OverflowError, space.wrap(
+                "long int too large to convert to int"))
+
+    def uint_w(w_self, space):
+        a = w_self.longlong
+        if a < 0:
+            raise OperationError(space.w_ValueError, space.wrap(
+                "cannot convert negative integer to unsigned int"))
+        b = r_uint(a)
+        if r_longlong(b) == a:
+            return b
+        else:
+            raise OperationError(space.w_OverflowError, space.wrap(
+                "long int too large to convert to unsigned int"))
+
+    def bigint_w(w_self, space):
+        return w_self.asbigint()
+
 registerimplementation(W_SmallLongObject)
 
 # ____________________________________________________________
@@ -102,30 +126,6 @@
 def float__SmallLong(space, w_value):
     return space.newfloat(float(w_value.longlong))
 
-def int_w__SmallLong(space, w_value):
-    a = w_value.longlong
-    b = intmask(a)
-    if b == a:
-        return b
-    else:
-        raise OperationError(space.w_OverflowError, space.wrap(
-            "long int too large to convert to int"))
-
-def uint_w__SmallLong(space, w_value):
-    a = w_value.longlong
-    if a < 0:
-        raise OperationError(space.w_ValueError, space.wrap(
-            "cannot convert negative integer to unsigned int"))
-    b = r_uint(a)
-    if r_longlong(b) == a:
-        return b
-    else:
-        raise OperationError(space.w_OverflowError, space.wrap(
-            "long int too large to convert to unsigned int"))
-
-def bigint_w__SmallLong(space, w_value):
-    return w_value.asbigint()
-
 def lt__SmallLong_SmallLong(space, w_small1, w_small2):
     return space.newbool(w_small1.longlong <  w_small2.longlong)
 def le__SmallLong_SmallLong(space, w_small1, w_small2):
diff --git a/pypy/objspace/std/strbufobject.py b/pypy/objspace/std/strbufobject.py
--- a/pypy/objspace/std/strbufobject.py
+++ b/pypy/objspace/std/strbufobject.py
@@ -32,6 +32,9 @@
     def unwrap(self, space):
         return self.force()
 
+    def str_w(self, space):
+        return self.force()
+
 registerimplementation(W_StringBufferObject)
 
 # ____________________________________________________________
@@ -55,9 +58,6 @@
 def len__StringBuffer(space, w_self):
     return space.wrap(w_self.length)
 
-def str_w__StringBuffer(space, w_strbuf):
-    return w_strbuf.force()
-
 def add__StringBuffer_String(space, w_self, w_other):
     if w_self.builder.getlength() != w_self.length:
         builder = StringBuilder()
diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py
--- a/pypy/objspace/std/stringobject.py
+++ b/pypy/objspace/std/stringobject.py
@@ -33,17 +33,20 @@
     def unwrap(w_self, space):
         return w_self._value
 
+    def str_w(w_self, space):
+        return w_self._value
+
+    def unicode_w(w_self, space):
+        # XXX should this use the default encoding?
+        from pypy.objspace.std.unicodetype import plain_str2unicode
+        return plain_str2unicode(space, w_self._value)
+
 registerimplementation(W_StringObject)
 
 W_StringObject.EMPTY = W_StringObject('')
 W_StringObject.PREBUILT = [W_StringObject(chr(i)) for i in range(256)]
 del i
 
-def unicode_w__String(space, w_self):
-    # XXX should this use the default encoding?
-    from pypy.objspace.std.unicodetype import plain_str2unicode
-    return plain_str2unicode(space, w_self._value)
-
 def _is_generic(space, w_self, fun):
     v = w_self._value
     if len(v) == 0:
@@ -773,8 +776,6 @@
 
     return space.wrap("".join(buf))
 
-def str_w__String(space, w_str):
-    return w_str._value
 
 def hash__String(space, w_str):
     s = w_str._value
diff --git a/pypy/objspace/std/strjoinobject.py b/pypy/objspace/std/strjoinobject.py
--- a/pypy/objspace/std/strjoinobject.py
+++ b/pypy/objspace/std/strjoinobject.py
@@ -29,6 +29,7 @@
 
     def unwrap(w_self, space):
         return w_self.force()
+    str_w = unwrap
 
 registerimplementation(W_StringJoinObject)
 
@@ -45,9 +46,6 @@
         result += len(w_self.joined_strs[i])
     return space.wrap(result)
 
-def str_w__StringJoin(space, w_str):
-    return w_str.force()
-
 def add__StringJoin_StringJoin(space, w_self, w_other):
     if len(w_self.joined_strs) > w_self.until:
         w_self.force(True)
diff --git a/pypy/objspace/std/strsliceobject.py b/pypy/objspace/std/strsliceobject.py
--- a/pypy/objspace/std/strsliceobject.py
+++ b/pypy/objspace/std/strsliceobject.py
@@ -31,6 +31,9 @@
         w_self.stop = len(str)
         return str
 
+    def str_w(w_self, space):
+        return w_self.force()
+
     def __repr__(w_self):
         """ representation for debugging purposes """
         return "%s(%r[%d:%d])" % (w_self.__class__.__name__,
@@ -165,11 +168,6 @@
             return space.w_True
     return space.w_False
 
-
-def str_w__StringSlice(space, w_str):
-    return w_str.force()
-
-
 def getitem__StringSlice_ANY(space, w_str, w_index):
     ival = space.getindex_w(w_index, space.w_IndexError, "string index")
     slen = w_str.stop - w_str.start
diff --git a/pypy/objspace/std/test/test_boolobject.py b/pypy/objspace/std/test/test_boolobject.py
--- a/pypy/objspace/std/test/test_boolobject.py
+++ b/pypy/objspace/std/test/test_boolobject.py
@@ -17,6 +17,12 @@
         
     def test_false(self):
         assert not self.space.is_true(self.false)
+
+    def test_uint_w(self):
+        assert self.space.uint_w(self.true) == 1
+
+    def test_rbigint_w(self):
+        assert self.space.bigint_w(self.true)._digits == [1]
         
 class AppTestAppBoolTest:
     def test_bool_callable(self):
diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py
--- a/pypy/objspace/std/test/test_listobject.py
+++ b/pypy/objspace/std/test/test_listobject.py
@@ -705,6 +705,20 @@
         l.pop()
         assert l == range(9)
 
+    def test_pop_custom_int(self):
+        class A(object):
+            def __init__(self, x):
+                self.x = x
+
+            def __int__(self):
+                return self.x
+
+        l = range(10)
+        x = l.pop(A(-1))
+        assert x == 9
+        assert l == range(9)
+        raises(TypeError, range(10).pop, 1.0)
+
     def test_remove(self):
         c = list('hello world')
         c.remove('l')
diff --git a/pypy/objspace/std/test/test_stdobjspace.py b/pypy/objspace/std/test/test_stdobjspace.py
--- a/pypy/objspace/std/test/test_stdobjspace.py
+++ b/pypy/objspace/std/test/test_stdobjspace.py
@@ -46,3 +46,17 @@
         assert space.sliceindices(w_slice, w(3)) == (1,2,1)
         assert space.sliceindices(w_obj, w(3)) == (1,2,3)
 
+    def test_fastpath_isinstance(self):
+        from pypy.objspace.std.stringobject import W_StringObject
+        from pypy.objspace.std.intobject import W_IntObject
+        
+        space = self.space
+        assert space.w_str.interplevel_cls is W_StringObject
+        assert space.w_int.interplevel_cls is W_IntObject
+        class X(W_StringObject):
+            def __init__(self):
+                pass
+            
+            typedef = None
+
+        assert space.isinstance_w(X(), space.w_str)
diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -126,6 +126,25 @@
         raises(TypeError, type, 'test', 42, {})
         raises(TypeError, type, 'test', (object,), 42)
 
+    def test_call_type_subclass(self):
+        class A(type):
+            pass
+
+        assert A("hello") is str
+
+        # Make sure type(x) doesn't call x.__class__.__init__
+        class T(type):
+            counter = 0
+            def __init__(self, *args):
+                T.counter += 1
+        class C:
+            __metaclass__ = T
+        assert T.counter == 1
+        a = C()
+        assert T.counter == 1
+        assert type(a) is C
+        assert T.counter == 1
+
     def test_bases(self):
         assert int.__bases__ == (object,)
         class X:
diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py
--- a/pypy/objspace/std/test/test_unicodeobject.py
+++ b/pypy/objspace/std/test/test_unicodeobject.py
@@ -443,6 +443,8 @@
         assert u'<i><i><i>c' == u'abababc'.translate({ord('a'):None, ord('b'):u'<i>'})
         assert u'c' == u'abababc'.translate({ord('a'):None, ord('b'):u''})
         assert u'xyyx' == u'xzx'.translate({ord('z'):u'yy'})
+        assert u'abcd' == u'ab\0d'.translate(u'c')
+        assert u'abcd' == u'abcd'.translate(u'')
 
         raises(TypeError, u'hello'.translate)
         raises(TypeError, u'abababc'.translate, {ord('a'):''})
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -77,7 +77,7 @@
         for i in range(len(self.lookup_where)):
             self.lookup_where[i] = None_None
 
-# possible values of compares_by_identity_status 
+# possible values of compares_by_identity_status
 UNKNOWN = 0
 COMPARES_BY_IDENTITY = 1
 OVERRIDES_EQ_CMP_OR_HASH = 2
@@ -115,6 +115,9 @@
     # of the __new__ is an instance of the type
     w_bltin_new = None
 
+    interplevel_cls = None # not None for prebuilt instances of
+                           # interpreter-level types
+
     @dont_look_inside
     def __init__(w_self, space, name, bases_w, dict_w,
                  overridetypedef=None):
@@ -355,7 +358,7 @@
                 if w_value is not None:
                     return w_value
         return None
-                
+
     @unroll_safe
     def _lookup(w_self, key):
         space = w_self.space
@@ -819,14 +822,6 @@
 
 def call__Type(space, w_type, __args__):
     promote(w_type)
-    # special case for type(x)
-    if space.is_w(w_type, space.w_type):
-        try:
-            w_obj, = __args__.fixedunpack(1)
-        except ValueError:
-            pass
-        else:
-            return space.type(w_obj)
     # invoke the __new__ of the type
     if not we_are_jitted():
         # note that the annotator will figure out that w_type.w_bltin_new can
@@ -853,7 +848,8 @@
         call_init = space.isinstance_w(w_newobject, w_type)
 
     # maybe invoke the __init__ of the type
-    if call_init:
+    if (call_init and not (space.is_w(w_type, space.w_type) and
+        not __args__.keywords and len(__args__.arguments_w) == 1)):
         w_descr = space.lookup(w_newobject, '__init__')
         w_result = space.get_and_call_args(w_descr, w_newobject, __args__)
         if not space.is_w(w_result, space.w_None):
diff --git a/pypy/objspace/std/typetype.py b/pypy/objspace/std/typetype.py
--- a/pypy/objspace/std/typetype.py
+++ b/pypy/objspace/std/typetype.py
@@ -1,17 +1,28 @@
-from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.interpreter import gateway
 from pypy.interpreter.argument import Arguments
+from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.interpreter.typedef import (GetSetProperty, descr_get_dict,
                                       weakref_descr)
 from pypy.objspace.std.stdtypedef import StdTypeDef
 
-def descr__new__(space, w_typetype, w_name, w_bases, w_dict):
+
+def descr__new__(space, w_typetype, w_name, w_bases=gateway.NoneNotWrapped,
+    w_dict=gateway.NoneNotWrapped):
+
     "This is used to create user-defined classes only."
     from pypy.objspace.std.typeobject import W_TypeObject
     # XXX check types
 
     w_typetype = _precheck_for_new(space, w_typetype)
 
+    # special case for type(x)
+    if (space.is_w(space.type(w_typetype), space.w_type) and w_bases is None and
+        w_dict is None):
+        return space.type(w_name)
+    elif w_bases is None or w_dict is None:
+        raise OperationError(space.w_TypeError, space.wrap("type() takes 1 or 3 arguments"))
+
+
     bases_w = space.fixedview(w_bases)
 
     w_winner = w_typetype
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -40,6 +40,12 @@
             return w_self
         return W_UnicodeObject(w_self._value)
 
+    def str_w(self, space):
+        return space.str_w(space.str(self))
+
+    def unicode_w(self, space):
+        return self._value
+
 W_UnicodeObject.EMPTY = W_UnicodeObject(u'')
 
 registerimplementation(W_UnicodeObject)
@@ -99,12 +105,6 @@
         return space.not_(result)
     return result
 
-def str_w__Unicode(space, w_uni):
-    return space.str_w(str__Unicode(space, w_uni))
-
-def unicode_w__Unicode(space, w_uni):
-    return w_uni._value
-
 def str__Unicode(space, w_uni):
     from pypy.objspace.std.unicodetype import encode_object
     return encode_object(space, w_uni, None, None)
@@ -893,7 +893,7 @@
         try:
             w_newval = space.getitem(w_table, space.wrap(ord(unichar)))
         except OperationError, e:
-            if e.match(space, space.w_KeyError):
+            if e.match(space, space.w_LookupError):
                 result.append(unichar)
             else:
                 raise
diff --git a/pypy/objspace/test/test_descroperation.py b/pypy/objspace/test/test_descroperation.py
--- a/pypy/objspace/test/test_descroperation.py
+++ b/pypy/objspace/test/test_descroperation.py
@@ -667,5 +667,19 @@
                 return -1L
         raises(ValueError, len, Y())
 
+    def test_len_custom__int__(self):
+        class X(object):
+            def __init__(self, x):
+                self.x = x
+            def __len__(self):
+                return self.x
+            def __int__(self):
+                return self.x
+
+        l = len(X(3.0))
+        assert l == 3 and type(l) is int
+        l = len(X(X(2)))
+        assert l == 2 and type(l) is int
+
 class AppTestWithBuiltinShortcut(AppTest_Descroperation):
     OPTIONS = {'objspace.std.builtinshortcut': True}
diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -158,7 +158,7 @@
     return decorator
 
 @oopspec("jit.isconstant(value)")
- at specialize.argtype(0)
+ at specialize.ll()
 def isconstant(value):
     """
     While tracing, returns whether or not the value is currently known to be
@@ -167,10 +167,7 @@
 
     This is for advanced usage only.
     """
-    # I hate the annotator so much.
-    if NonConstant(False):
-        return True
-    return False
+    return NonConstant(False)
 
 @oopspec("jit.isvirtual(value)")
 @specialize.ll()
@@ -181,9 +178,7 @@
 
     This is for advanced usage only.
     """
-    if NonConstant(False):
-        return True
-    return False
+    return NonConstant(False)
 
 class Entry(ExtRegistryEntry):
     _about_ = hint
diff --git a/pypy/rlib/objectmodel.py b/pypy/rlib/objectmodel.py
--- a/pypy/rlib/objectmodel.py
+++ b/pypy/rlib/objectmodel.py
@@ -46,6 +46,17 @@
 
         return decorated_func
 
+    def arg_or_var(self, *args):
+        """ Same as arg, but additionally allow for a 'variable' annotation,
+        that would simply be a situation where designated arg is not
+        a constant
+        """
+        def decorated_func(func):
+            func._annspecialcase_ = 'specialize:arg_or_var' + self._wrap(args)
+            return func
+
+        return decorated_func
+
     def argtype(self, *args):
         """ Specialize function based on types of arguments on given positions.
 
@@ -165,6 +176,24 @@
 def keepalive_until_here(*values):
     pass
 
+def is_annotation_constant(thing):
+    """ Returns whether the annotator can prove that the argument is constant.
+    For advanced usage only."""
+    return True
+
+class Entry(ExtRegistryEntry):
+    _about_ = is_annotation_constant
+
+    def compute_result_annotation(self, s_arg):
+        from pypy.annotation import model
+        r = model.SomeBool()
+        r.const = s_arg.is_constant()
+        return r
+
+    def specialize_call(self, hop):
+        from pypy.rpython.lltypesystem import lltype
+        return hop.inputconst(lltype.Bool, hop.s_result.const)
+
 # ____________________________________________________________
 
 class FREED_OBJECT(object):
diff --git a/pypy/rlib/rgc.py b/pypy/rlib/rgc.py
--- a/pypy/rlib/rgc.py
+++ b/pypy/rlib/rgc.py
@@ -3,6 +3,7 @@
 
 from pypy.rlib import jit
 from pypy.rlib.objectmodel import we_are_translated, enforceargs, specialize
+from pypy.rlib.nonconst import NonConstant
 from pypy.rpython.extregistry import ExtRegistryEntry
 from pypy.rpython.lltypesystem import lltype, llmemory
 
@@ -143,6 +144,10 @@
     from pypy.rpython.lltypesystem.lloperation import llop
     from pypy.rlib.objectmodel import keepalive_until_here
 
+    # XXX: Hack to ensure that we get a proper effectinfo.write_descrs_arrays
+    if NonConstant(False):
+        dest[dest_start] = source[source_start]
+
     # supports non-overlapping copies only
     if not we_are_translated():
         if source == dest:
diff --git a/pypy/rlib/ropenssl.py b/pypy/rlib/ropenssl.py
--- a/pypy/rlib/ropenssl.py
+++ b/pypy/rlib/ropenssl.py
@@ -62,8 +62,7 @@
         "OPENSSL_VERSION_NUMBER")
     SSLEAY_VERSION = rffi_platform.DefinedConstantString(
         "SSLEAY_VERSION", "SSLeay_version(SSLEAY_VERSION)")
-    OPENSSL_NO_SSL2 = rffi_platform.DefinedConstantInteger(
-        "OPENSSL_NO_SSL2")
+    OPENSSL_NO_SSL2 = rffi_platform.Defined("OPENSSL_NO_SSL2")
     SSL_FILETYPE_PEM = rffi_platform.ConstantInteger("SSL_FILETYPE_PEM")
     SSL_OP_ALL = rffi_platform.ConstantInteger("SSL_OP_ALL")
     SSL_VERIFY_NONE = rffi_platform.ConstantInteger("SSL_VERIFY_NONE")
diff --git a/pypy/rlib/rstacklet.py b/pypy/rlib/rstacklet.py
--- a/pypy/rlib/rstacklet.py
+++ b/pypy/rlib/rstacklet.py
@@ -99,12 +99,20 @@
         return False
     def add(self, h):
         if not self.sthread.is_empty_handle(h):
+            if h == self.sthread.get_null_handle():
+                raise StackletDebugError("unexpected null handle")
             self.active.append(h)
     def remove(self, h):
         try:
             i = self.active.index(h)
         except ValueError:
-            raise StackletDebugError
+            if self.sthread.is_empty_handle(h):
+                msg = "empty stacklet handle"
+            elif h == self.sthread.get_null_handle():
+                msg = "unexpected null handle"
+            else:
+                msg = "double usage of handle %r" % (h,)
+            raise StackletDebugError(msg)
         del self.active[i]
 debug = Debug()
 
diff --git a/pypy/rlib/test/test_jit.py b/pypy/rlib/test/test_jit.py
--- a/pypy/rlib/test/test_jit.py
+++ b/pypy/rlib/test/test_jit.py
@@ -1,7 +1,7 @@
 import py
 from pypy.conftest import option
 from pypy.rlib.jit import hint, we_are_jitted, JitDriver, elidable_promote
-from pypy.rlib.jit import JitHintError, oopspec
+from pypy.rlib.jit import JitHintError, oopspec, isconstant
 from pypy.translator.translator import TranslationContext, graphof
 from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin
 from pypy.rpython.lltypesystem import lltype
@@ -137,6 +137,16 @@
             t.view()
         # assert did not raise
 
+    def test_isconstant(self):
+        def f(n):
+            assert n >= 0
+            assert isconstant(n) is False
+            l = []
+            l.append(n)
+            return len(l)
+        res = self.interpret(f, [234])
+        assert res == 1
+
 
 class TestJITLLtype(BaseTestJIT, LLRtypeMixin):
     pass
diff --git a/pypy/rlib/test/test_objectmodel.py b/pypy/rlib/test/test_objectmodel.py
--- a/pypy/rlib/test/test_objectmodel.py
+++ b/pypy/rlib/test/test_objectmodel.py
@@ -339,6 +339,19 @@
         res = self.interpret(f, [42])
         assert res == 84
 
+    def test_isconstant(self):
+        from pypy.rlib.objectmodel import is_annotation_constant, specialize
+
+        @specialize.arg_or_var(0)
+        def f(arg):
+            if is_annotation_constant(arg):
+                return 1
+            return 10
+
+        def fn(arg):
+            return f(arg) + f(3)
+
+        assert self.interpret(fn, [15]) == 11
 
 class TestLLtype(BaseTestObjectModel, LLRtypeMixin):
 
@@ -451,5 +464,4 @@
         if llop.opname == 'malloc_varsize':
             break
     assert llop.args[2] is graph.startblock.inputargs[0]
-
     
diff --git a/pypy/rlib/test/test_rstacklet.py b/pypy/rlib/test/test_rstacklet.py
--- a/pypy/rlib/test/test_rstacklet.py
+++ b/pypy/rlib/test/test_rstacklet.py
@@ -264,6 +264,10 @@
     gcrootfinder = 'shadowstack'
 
 
+def test_dont_keep_debug_to_true():
+    assert not rstacklet.DEBUG
+
+
 def target(*args):
     return entry_point, None
 
diff --git a/pypy/rpython/lltypesystem/ll2ctypes.py b/pypy/rpython/lltypesystem/ll2ctypes.py
--- a/pypy/rpython/lltypesystem/ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/ll2ctypes.py
@@ -140,7 +140,8 @@
                 if isinstance(FIELDTYPE, lltype.Ptr):
                     cls = get_ctypes_type(FIELDTYPE, delayed_builders)
                 else:
-                    cls = get_ctypes_type(FIELDTYPE)
+                    cls = get_ctypes_type(FIELDTYPE, delayed_builders,
+                                          cannot_delay=True)
             fields.append((fieldname, cls))
         CStruct._fields_ = fields
 
@@ -169,7 +170,7 @@
         CStruct._normalized_ctype = get_ctypes_type(S)
         builder()    # no need to be lazy here
     else:
-        delayed_builders.append(builder)
+        delayed_builders.append((S, builder))
     return CStruct
 
 def build_ctypes_array(A, delayed_builders, max_n=0):
@@ -252,11 +253,19 @@
     else:
         return get_ctypes_type(FIELDTYPE)
 
-def get_ctypes_type(T, delayed_builders=None):
+def get_ctypes_type(T, delayed_builders=None, cannot_delay=False):
+    # Check delayed builders
+    if cannot_delay and delayed_builders:
+        for T2, builder in delayed_builders:
+            if T2 is T:
+                builder()
+                delayed_builders.remove((T2, builder))
+                return _ctypes_cache[T]
+
     try:
         return _ctypes_cache[T]
     except KeyError:
-        toplevel = delayed_builders is None
+        toplevel = cannot_delay or delayed_builders is None
         if toplevel:
             delayed_builders = []
         cls = build_new_ctypes_type(T, delayed_builders)
@@ -306,9 +315,11 @@
 
 def complete_builders(delayed_builders):
     while delayed_builders:
-        delayed_builders.pop()()
+        T, builder = delayed_builders[0]
+        builder()
+        delayed_builders.pop(0)
 
-def convert_struct(container, cstruct=None):
+def convert_struct(container, cstruct=None, delayed_converters=None):
     STRUCT = container._TYPE
     if cstruct is None:
         # if 'container' is an inlined substructure, convert the whole
@@ -325,23 +336,38 @@
             n = None
         cstruct = cls._malloc(n)
     add_storage(container, _struct_mixin, ctypes.pointer(cstruct))
+
+    if delayed_converters is None:
+        delayed_converters_was_None = True
+        delayed_converters = []
+    else:
+        delayed_converters_was_None = False
     for field_name in STRUCT._names:
         FIELDTYPE = getattr(STRUCT, field_name)
         field_value = getattr(container, field_name)
         if not isinstance(FIELDTYPE, lltype.ContainerType):
             # regular field
             if FIELDTYPE != lltype.Void:
-                setattr(cstruct, field_name, lltype2ctypes(field_value))
+                def convert(field_name=field_name, field_value=field_value):
+                    setattr(cstruct, field_name, lltype2ctypes(field_value))
+                if isinstance(FIELDTYPE, lltype.Ptr):
+                    delayed_converters.append(convert)
+                else:
+                    convert()
         else:
             # inlined substructure/subarray
             if isinstance(FIELDTYPE, lltype.Struct):
                 csubstruct = getattr(cstruct, field_name)
-                convert_struct(field_value, csubstruct)
+                convert_struct(field_value, csubstruct,
+                               delayed_converters=delayed_converters)
             elif field_name == STRUCT._arrayfld:    # inlined var-sized part
                 csubarray = getattr(cstruct, field_name)
                 convert_array(field_value, csubarray)
             else:
                 raise NotImplementedError('inlined field', FIELDTYPE)
+    if delayed_converters_was_None:
+        for converter in delayed_converters:
+            converter()
     remove_regular_struct_content(container)
 
 def remove_regular_struct_content(container):
@@ -358,7 +384,8 @@
         # bigger structure at once
         parent, parentindex = lltype.parentlink(container)
         if parent is not None:
-            convert_struct(parent)
+            if not isinstance(parent, _parentable_mixin):
+                convert_struct(parent)
             return
         # regular case: allocate a new ctypes array of the proper type
         cls = get_ctypes_type(ARRAY)
diff --git a/pypy/rpython/lltypesystem/opimpl.py b/pypy/rpython/lltypesystem/opimpl.py
--- a/pypy/rpython/lltypesystem/opimpl.py
+++ b/pypy/rpython/lltypesystem/opimpl.py
@@ -357,7 +357,7 @@
 
 def op_cast_float_to_uint(f):
     assert type(f) is float
-    return r_uint(int(f))
+    return r_uint(long(f))
 
 def op_cast_float_to_longlong(f):
     assert type(f) is float
@@ -369,7 +369,7 @@
 
 def op_cast_float_to_ulonglong(f):
     assert type(f) is float
-    return r_ulonglong(r_longlong(f))
+    return r_ulonglong(long(f))
 
 def op_cast_char_to_int(b):
     assert type(b) is str and len(b) == 1
diff --git a/pypy/rpython/lltypesystem/rlist.py b/pypy/rpython/lltypesystem/rlist.py
--- a/pypy/rpython/lltypesystem/rlist.py
+++ b/pypy/rpython/lltypesystem/rlist.py
@@ -1,15 +1,15 @@
+from pypy.rlib import rgc, jit
+from pypy.rlib.debug import ll_assert
+from pypy.rlib.objectmodel import enforceargs
+from pypy.rpython.lltypesystem import rstr
+from pypy.rpython.lltypesystem.lltype import (GcForwardReference, Ptr, GcArray,
+     GcStruct, Void, Signed, malloc, typeOf, nullptr, typeMethod)
+from pypy.rpython.rlist import (AbstractBaseListRepr, AbstractListRepr,
+    AbstractFixedSizeListRepr, AbstractListIteratorRepr, ll_setitem_nonneg,
+    ADTIList, ADTIFixedList, dum_nocheck)
+from pypy.rpython.rmodel import Repr, inputconst, externalvsinternal
 from pypy.tool.pairtype import pairtype, pair
-from pypy.rpython.rmodel import Repr, inputconst
-from pypy.rpython.rmodel import externalvsinternal
-from pypy.rpython.rlist import AbstractBaseListRepr, AbstractListRepr, \
-        AbstractFixedSizeListRepr, AbstractListIteratorRepr, \
-        ll_setitem_nonneg, ADTIList, ADTIFixedList
-from pypy.rpython.rlist import dum_nocheck
-from pypy.rpython.lltypesystem.lltype import GcForwardReference, Ptr, GcArray,\
-     GcStruct, Void, Signed, malloc, typeOf, nullptr, typeMethod
-from pypy.rpython.lltypesystem import rstr
-from pypy.rlib.debug import ll_assert
-from pypy.rlib import rgc, jit
+
 
 # ____________________________________________________________
 #
@@ -171,6 +171,7 @@
 
 # adapted C code
 
+ at enforceargs(None, int)
 def _ll_list_resize_really(l, newsize):
     """
     Ensure l.items has room for at least newsize elements, and set
@@ -210,7 +211,6 @@
         rgc.ll_arraycopy(items, newitems, 0, 0, p)
     l.length = newsize
     l.items = newitems
-_ll_list_resize_really._annenforceargs_ = (None, int)
 
 # this common case was factored out of _ll_list_resize
 # to see if inlining it gives some speed-up.
diff --git a/pypy/rpython/lltypesystem/rstr.py b/pypy/rpython/lltypesystem/rstr.py
--- a/pypy/rpython/lltypesystem/rstr.py
+++ b/pypy/rpython/lltypesystem/rstr.py
@@ -694,8 +694,8 @@
             return -1
         return count
 
+    @enforceargs(int, None)
     @jit.look_inside_iff(lambda length, items: jit.isconstant(length) and length <= 2)
-    @enforceargs(int, None)
     def ll_join_strs(length, items):
         # Special case for length 1 items, helps both the JIT and other code
         if length == 1:
diff --git a/pypy/rpython/lltypesystem/test/test_ll2ctypes.py b/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
--- a/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
@@ -82,7 +82,6 @@
         assert not ALLOCATED     # detects memory leaks in the test
 
     def test_get_pointer(self):
-        py.test.skip("FIXME")
         # Equivalent of the C code::
         #     struct S1 { struct S2 *ptr; struct S2 buf; };
         #     struct S1 s1;
diff --git a/pypy/rpython/lltypesystem/test/test_lloperation.py b/pypy/rpython/lltypesystem/test/test_lloperation.py
--- a/pypy/rpython/lltypesystem/test/test_lloperation.py
+++ b/pypy/rpython/lltypesystem/test/test_lloperation.py
@@ -5,6 +5,7 @@
 from pypy.rpython.llinterp import LLFrame
 from pypy.rpython.test.test_llinterp import interpret
 from pypy.rpython import rclass
+from pypy.rlib.rarithmetic import LONGLONG_MASK, r_longlong, r_ulonglong
 
 LL_INTERP_OPERATIONS = [name[3:] for name in LLFrame.__dict__.keys()
                                  if name.startswith('op_')]
@@ -133,6 +134,14 @@
         py.test.raises(TypeError, llop.getinteriorfield,
                        lltype.Signed, s3, 'y')
 
+def test_cast_float_to_ulonglong():
+    f = 12350000000000000000.0
+    py.test.raises(OverflowError, r_longlong, f)
+    r_longlong(f / 2)   # does not raise OverflowError
+    #
+    x = llop.cast_float_to_ulonglong(lltype.UnsignedLongLong, f)
+    assert x == r_ulonglong(f)
+
 # ___________________________________________________________________________
 # This tests that the LLInterpreter and the LL_OPERATIONS tables are in sync.
 
diff --git a/pypy/rpython/module/ll_os_stat.py b/pypy/rpython/module/ll_os_stat.py
--- a/pypy/rpython/module/ll_os_stat.py
+++ b/pypy/rpython/module/ll_os_stat.py
@@ -173,7 +173,8 @@
         _compilation_info_ = compilation_info
         STAT_STRUCT = platform.Struct('struct %s' % _name_struct_stat, LL_STAT_FIELDS)
     try:
-        config = platform.configure(CConfig)
+        config = platform.configure(CConfig, ignore_errors=
+                                    try_to_add is not None)
     except platform.CompilationError:
         if try_to_add:
             return    # failed to add this field, give up
diff --git a/pypy/rpython/rlist.py b/pypy/rpython/rlist.py
--- a/pypy/rpython/rlist.py
+++ b/pypy/rpython/rlist.py
@@ -11,7 +11,7 @@
 from pypy.rlib.debug import ll_assert
 from pypy.rlib.rarithmetic import ovfcheck, widen, r_uint, intmask
 from pypy.rpython.annlowlevel import ADTInterface
-from pypy.rlib import rgc
+from pypy.rlib import rgc, jit
 
 ADTIFixedList = ADTInterface(None, {
     'll_newlist':      (['SELF', Signed        ], 'self'),
@@ -912,6 +912,8 @@
     return l
 # no oopspec -- the function is inlined by the JIT
 
+ at jit.look_inside_iff(lambda l, start: jit.isconstant(start) and jit.isvirtual(l))
+ at jit.oopspec('list.delslice_startonly(l, start)')
 def ll_listdelslice_startonly(l, start):
     ll_assert(start >= 0, "del l[start:] with unexpectedly negative start")
     ll_assert(start <= l.ll_length(), "del l[start:] with start > len(l)")
@@ -923,7 +925,6 @@
             l.ll_setitem_fast(j, null)
             j -= 1
     l._ll_resize_le(newlength)
-ll_listdelslice_startonly.oopspec = 'list.delslice_startonly(l, start)'
 
 def ll_listdelslice_startstop(l, start, stop):
     length = l.ll_length()
diff --git a/pypy/rpython/tool/rffi_platform.py b/pypy/rpython/tool/rffi_platform.py
--- a/pypy/rpython/tool/rffi_platform.py
+++ b/pypy/rpython/tool/rffi_platform.py
@@ -171,7 +171,7 @@
         eci = self.config._compilation_info_
         try_compile_cache([self.path], eci)
 
-def configure(CConfig):
+def configure(CConfig, ignore_errors=False):
     """Examine the local system by running the C compiler.
     The CConfig class contains CConfigEntry attribues that describe
     what should be inspected; configure() returns a dict mapping
@@ -199,7 +199,8 @@
         writer.close()
 
         eci = CConfig._compilation_info_
-        infolist = list(run_example_code(writer.path, eci))
+        infolist = list(run_example_code(writer.path, eci,
+                                         ignore_errors=ignore_errors))
         assert len(infolist) == len(entries)
 
         resultinfo = {}
@@ -680,10 +681,10 @@
 }
 """
 
-def run_example_code(filepath, eci):
+def run_example_code(filepath, eci, ignore_errors=False):
     eci = eci.convert_sources_to_files(being_main=True)
     files = [filepath]
-    output = build_executable_cache(files, eci)
+    output = build_executable_cache(files, eci, ignore_errors=ignore_errors)
     section = None
     for line in output.splitlines():
         line = line.strip()
diff --git a/pypy/tool/gcc_cache.py b/pypy/tool/gcc_cache.py
--- a/pypy/tool/gcc_cache.py
+++ b/pypy/tool/gcc_cache.py
@@ -16,7 +16,7 @@
     hash = md5(key).hexdigest()
     return cache_dir.join(hash)
 
-def build_executable_cache(c_files, eci):
+def build_executable_cache(c_files, eci, ignore_errors=False):
     "Builds and run a program; caches the result"
     # Import 'platform' every time, the compiler may have been changed
     from pypy.translator.platform import platform
@@ -24,7 +24,18 @@
     try:
         return path.read()
     except py.error.Error:
-        result = platform.execute(platform.compile(c_files, eci))
+        _previous = platform.log_errors
+        try:
+            if ignore_errors:
+                platform.log_errors = False
+            result = platform.execute(platform.compile(c_files, eci))
+        finally:
+            if ignore_errors:
+                del platform.log_errors
+            # ^^^remove from the instance --- needed so that it can
+            # compare equal to another instance without it
+            if platform.log_errors != _previous:
+                platform.log_errors = _previous
         path.write(result.out)
         return result.out
 
diff --git a/pypy/tool/pytest/appsupport.py b/pypy/tool/pytest/appsupport.py
--- a/pypy/tool/pytest/appsupport.py
+++ b/pypy/tool/pytest/appsupport.py
@@ -72,7 +72,9 @@
         space = self.space
         retval = []
         for arg in self.code.getargs():
-            w_val = space.getitem(self.w_locals, space.wrap(arg))
+            w_val = space.finditem(self.w_locals, space.wrap(arg))
+            if w_val is None:
+                w_val = space.wrap('<no value found>')
             retval.append((arg, w_val))
         return retval
 
diff --git a/pypy/tool/test/test_gcc_cache.py b/pypy/tool/test/test_gcc_cache.py
--- a/pypy/tool/test/test_gcc_cache.py
+++ b/pypy/tool/test/test_gcc_cache.py
@@ -77,3 +77,17 @@
     finally:
         sys.stderr = oldstderr
     assert 'ERROR' not in capture.getvalue().upper()
+
+def test_execute_code_ignore_errors():
+    f = localudir.join('z.c')
+    f.write("""this file is not valid C code\n""")
+    eci = ExternalCompilationInfo()
+    oldstderr = sys.stderr
+    try:
+        sys.stderr = capture = cStringIO.StringIO()
+        py.test.raises(CompilationError, build_executable_cache,
+                       [f], eci, True)
+    finally:
+        sys.stderr = oldstderr
+    assert 'ERROR' not in capture.getvalue().upper()
+    
diff --git a/pypy/translator/c/gcc/trackgcroot.py b/pypy/translator/c/gcc/trackgcroot.py
--- a/pypy/translator/c/gcc/trackgcroot.py
+++ b/pypy/translator/c/gcc/trackgcroot.py
@@ -486,6 +486,8 @@
         'paddq', 'pinsr',
         # zero-extending moves should not produce GC pointers
         'movz', 
+        # locked operations should not move GC pointers, at least so far
+        'lock',
         ])
 
     # a partial list is hopefully good enough for now; it's all to support
diff --git a/pypy/translator/c/genc.py b/pypy/translator/c/genc.py
--- a/pypy/translator/c/genc.py
+++ b/pypy/translator/c/genc.py
@@ -563,7 +563,10 @@
             else:
                 mk.definition('PYPY_MAIN_FUNCTION', "main")
 
-            if sys.platform == 'win32':
+            if (py.path.local.sysfind('python') or
+                py.path.local.sysfind('python.exe')):
+                python = 'python '
+            elif sys.platform == 'win32':
                 python = sys.executable.replace('\\', '/') + ' '
             else:
                 python = sys.executable + ' '
diff --git a/pypy/translator/c/src/allocator.h b/pypy/translator/c/src/allocator.h
--- a/pypy/translator/c/src/allocator.h
+++ b/pypy/translator/c/src/allocator.h
@@ -6,11 +6,6 @@
 
 
 #ifndef PYPY_NOT_MAIN_FILE
-#ifdef AVR
-   #ifndef NO_OBMALLOC
-     #define NO_OBMALLOC
-   #endif
-#endif
 
 #if defined(TRIVIAL_MALLOC_DEBUG)
   void *PyObject_Malloc(size_t n) { return malloc(n); }
diff --git a/pypy/translator/c/src/g_include.h b/pypy/translator/c/src/g_include.h
--- a/pypy/translator/c/src/g_include.h
+++ b/pypy/translator/c/src/g_include.h
@@ -31,9 +31,7 @@
 #include "src/char.h"
 #include "src/float.h"
 #include "src/address.h"
-#ifndef AVR
 #include "src/unichar.h"
-#endif
 #include "src/llgroup.h"
 
 #include "src/instrument.h"
@@ -48,11 +46,9 @@
 #  include "src/rtyper.h"
 #  include "src/debug_traceback.h"
 #  include "src/debug_alloc.h"
-#ifndef AVR
 #  include "src/ll_os.h"
 #  include "src/ll_strtod.h"
 #endif
-#endif
 
 #ifdef PYPY_STANDALONE
 #  include "src/allocator.h"
diff --git a/pypy/translator/c/src/g_prerequisite.h b/pypy/translator/c/src/g_prerequisite.h
--- a/pypy/translator/c/src/g_prerequisite.h
+++ b/pypy/translator/c/src/g_prerequisite.h
@@ -13,10 +13,8 @@
 #  include <io.h>   /* needed, otherwise _lseeki64 truncates to 32-bits (??) */
 #endif
 
-#ifndef AVR
 #include "thread.h"   /* needs to be included early to define the
                          struct RPyOpaque_ThreadLock */
-#endif
 
 #include <stddef.h>
 
diff --git a/pypy/translator/c/src/main.h b/pypy/translator/c/src/main.h
--- a/pypy/translator/c/src/main.h
+++ b/pypy/translator/c/src/main.h
@@ -75,9 +75,7 @@
  memory_out:
     errmsg = "out of memory";
  error:
-#ifndef AVR
     fprintf(stderr, "Fatal error during initialization: %s\n", errmsg);
-#endif
     abort();
     return 1;
 }
diff --git a/pypy/translator/c/src/thread.h b/pypy/translator/c/src/thread.h
--- a/pypy/translator/c/src/thread.h
+++ b/pypy/translator/c/src/thread.h
@@ -37,14 +37,9 @@
 
 #endif
 
-/* common helper: this does nothing, but is called with the GIL released.
-   This gives other threads a chance to grab the GIL and run. */
-void RPyThreadYield(void);
-
-#ifndef PYPY_NOT_MAIN_FILE
-void RPyThreadYield(void)
-{
-}
-#endif
+long RPyGilAllocate(void);
+long RPyGilYieldThread(void);
+void RPyGilRelease(void);
+void RPyGilAcquire(void);
 
 #endif
diff --git a/pypy/translator/c/src/thread_nt.h b/pypy/translator/c/src/thread_nt.h
--- a/pypy/translator/c/src/thread_nt.h
+++ b/pypy/translator/c/src/thread_nt.h
@@ -221,4 +221,57 @@
 #define RPyThreadTLS_Set(key, value)	TlsSetValue(key, value)
 
 
+/************************************************************/
+/* GIL code                                                 */
+/************************************************************/
+
+static volatile LONG pending_acquires = -1;
+static CRITICAL_SECTION mutex_gil;
+static HANDLE cond_gil;
+
+long RPyGilAllocate(void)
+{
+    pending_acquires = 0;
+    InitializeCriticalSection(&mutex_gil);
+    EnterCriticalSection(&mutex_gil);
+    cond_gil = CreateEvent (NULL, FALSE, FALSE, NULL);
+    return 1;
+}
+
+long RPyGilYieldThread(void)
+{
+    /* can be called even before RPyGilAllocate(), but in this case,
+       pending_acquires will be -1 */
+    if (pending_acquires <= 0)
+        return 0;
+    InterlockedIncrement(&pending_acquires);
+    PulseEvent(&cond_gil);
+
+    /* hack: the three following lines do a pthread_cond_wait(), and
+       normally specifying a timeout of INFINITE would be fine.  But the
+       first and second operations are not done atomically, so there is a
+       (small) risk that PulseEvent misses the WaitForSingleObject().
+       In this case the process will just sleep a few milliseconds. */
+    LeaveCriticalSection(&mutex_gil);
+    WaitForSingleObject(&cond_gil, 15);
+    EnterCriticalSection(&mutex_gil);
+
+    InterlockedDecrement(&pending_acquires);
+    return 1;
+}
+
+void RPyGilRelease(void)
+{
+    LeaveCriticalSection(&mutex_gil);
+    PulseEvent(&cond_gil);
+}
+
+void RPyGilAcquire(void)
+{
+    InterlockedIncrement(&pending_acquires);
+    EnterCriticalSection(&mutex_gil);
+    InterlockedDecrement(&pending_acquires);
+}
+
+
 #endif /* PYPY_NOT_MAIN_FILE */
diff --git a/pypy/translator/c/src/thread_pthread.h b/pypy/translator/c/src/thread_pthread.h
--- a/pypy/translator/c/src/thread_pthread.h
+++ b/pypy/translator/c/src/thread_pthread.h
@@ -12,6 +12,7 @@
 #include <signal.h>
 #include <stdio.h>
 #include <errno.h>
+#include <assert.h>
 
 /* The following is hopefully equivalent to what CPython does
    (which is trying to compile a snippet of code using it) */
@@ -459,4 +460,113 @@
 #define RPyThreadTLS_Set(key, value)	pthread_setspecific(key, value)
 
 
+/************************************************************/
+/* GIL code                                                 */
+/************************************************************/
+
+#ifdef __llvm__
+#  define HAS_ATOMIC_ADD
+#endif
+
+#ifdef __GNUC__
+#  if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 1)
+#    define HAS_ATOMIC_ADD
+#  endif
+#endif
+
+#ifdef HAS_ATOMIC_ADD
+#  define atomic_add __sync_fetch_and_add
+#else
+#  if defined(__amd64__)
+#    define atomic_add(ptr, value)  asm volatile ("lock addq %0, %1"        \
+                                 : : "ri"(value), "m"(*(ptr)) : "memory")
+#  elif defined(__i386__)
+#    define atomic_add(ptr, value)  asm volatile ("lock addl %0, %1"        \
+                                 : : "ri"(value), "m"(*(ptr)) : "memory")
+#  else
+#    error "Please use gcc >= 4.1 or write a custom 'asm' for your CPU."
+#  endif
+#endif
+
+#define ASSERT_STATUS(call)                             \
+    if (call != 0) {                                    \
+        fprintf(stderr, "Fatal error: " #call "\n");    \
+        abort();                                        \
+    }
+
+static void _debug_print(const char *msg)
+{
+#if 0
+    int col = (int)pthread_self();
+    col = 31 + ((col / 8) % 8);
+    fprintf(stderr, "\033[%dm%s\033[0m", col, msg);
+#endif
+}
+
+static volatile long pending_acquires = -1;
+static pthread_mutex_t mutex_gil = PTHREAD_MUTEX_INITIALIZER;
+static pthread_cond_t cond_gil = PTHREAD_COND_INITIALIZER;
+
+static void assert_has_the_gil(void)
+{
+#ifdef RPY_ASSERT
+    assert(pthread_mutex_trylock(&mutex_gil) != 0);
+    assert(pending_acquires >= 0);
+#endif
+}
+
+long RPyGilAllocate(void)
+{
+    _debug_print("RPyGilAllocate\n");
+    pending_acquires = 0;
+    pthread_mutex_trylock(&mutex_gil);
+    assert_has_the_gil();
+    return 1;
+}
+
+long RPyGilYieldThread(void)
+{
+    /* can be called even before RPyGilAllocate(), but in this case,
+       pending_acquires will be -1 */
+#ifdef RPY_ASSERT
+    if (pending_acquires >= 0)
+        assert_has_the_gil();
+#endif
+    if (pending_acquires <= 0)
+        return 0;
+    atomic_add(&pending_acquires, 1L);
+    _debug_print("{");
+    ASSERT_STATUS(pthread_cond_signal(&cond_gil));
+    ASSERT_STATUS(pthread_cond_wait(&cond_gil, &mutex_gil));
+    _debug_print("}");
+    atomic_add(&pending_acquires, -1L);
+    assert_has_the_gil();
+    return 1;
+}
+
+void RPyGilRelease(void)
+{
+    _debug_print("RPyGilRelease\n");
+#ifdef RPY_ASSERT
+    assert(pending_acquires >= 0);
+#endif
+    assert_has_the_gil();
+    ASSERT_STATUS(pthread_mutex_unlock(&mutex_gil));
+    ASSERT_STATUS(pthread_cond_signal(&cond_gil));
+}
+
+void RPyGilAcquire(void)
+{
+    _debug_print("about to RPyGilAcquire...\n");
+#ifdef RPY_ASSERT
+    assert(pending_acquires >= 0);
+#endif
+    atomic_add(&pending_acquires, 1L);
+    ASSERT_STATUS(pthread_mutex_lock(&mutex_gil));
+    atomic_add(&pending_acquires, -1L);
+    assert_has_the_gil();
+    _debug_print("RPyGilAcquire\n");
+}
+
+
 #endif /* PYPY_NOT_MAIN_FILE */

From noreply at buildbot.pypy.org  Thu Oct  6 07:07:58 2011
From: noreply at buildbot.pypy.org (justinpeel)
Date: Thu,  6 Oct 2011 07:07:58 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-indexing-by-arrays: bool indexing requires
 a numpy array and matching size. use array's dtype if it is an ind for
 integer indexing.
Message-ID: <20111006050758.5B17182A02@wyvern.cs.uni-duesseldorf.de>

Author: Justin Peel <notmuchtotell at gmail.com>
Branch: numpy-indexing-by-arrays
Changeset: r47842:67245c6b73ee
Date: 2011-10-05 23:07 -0600
http://bitbucket.org/pypy/pypy/changeset/67245c6b73ee/

Log:	bool indexing requires a numpy array and matching size. use array's
	dtype if it is an ind for integer indexing.

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -233,25 +233,35 @@
                                      space.wrap("invalid index"))
             w_idx = space.getitem(w_idx, space.wrap(0))
         elif space.issequence_w(w_idx):
-            w_idx = convert_to_array(space, w_idx)
-            bool_dtype = space.fromcache(interp_dtype.W_BoolDtype)
-            int_dtype = space.fromcache(interp_dtype.W_Int64Dtype)
-            if w_idx.find_dtype() is bool_dtype:
-                # Indexing by boolean array
+            if isinstance(w_idx, BaseArray) and \
+                w_idx.find_dtype().kind == interp_dtype.BOOLLTR:
+                # Indexing by boolean array - must be using a bool numpy
+                # array of the exact same size as self to do this.
+                # Can't use a list.
+                bool_dtype = space.fromcache(interp_dtype.W_BoolDtype)
                 new_sig = signature.Signature.find_sig([
                     IndexedByBoolArray.signature, self.signature
-                ])                
+                ])
+                # will be more complicated with multi-dim arrays
+                if self.find_size() != w_idx.find_size():
+                    raise OperationError(space.w_ValueError,
+                        space.wrap("bool indexing requires matching dims"))
                 res = IndexedByBoolArray(new_sig, bool_dtype, self, w_idx)
                 return space.wrap(res)
             else:
+                w_idx = convert_to_array(space, w_idx)
                 # Indexing by array
-
                 # FIXME: should raise exception if any index in
                 # array is out od bound, but this kills lazy execution
                 new_sig = signature.Signature.find_sig([
                     IndexedByArray.signature, self.signature
-                ])                
-                res = IndexedByArray(new_sig, int_dtype, self, w_idx)
+                ])
+                # Use w_idx's dtype if possible
+                dtype = w_idx.find_dtype()
+                if dtype.kind != interp_dtype.SIGNEDLTR \
+                        and dtype.kind != interp_dtype.UNSIGNEDLTR:
+                    dtype = space.fromcache(interp_dtype.W_Int64Dtype)
+                res = IndexedByArray(new_sig, dtype, self, w_idx)
                 return space.wrap(res)
 
         start, stop, step, slice_length = space.decode_index4(w_idx, self.find_size())
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -136,6 +136,8 @@
     def test_index_by_bool_array(self):
         from numpy import array, dtype
         a = array(range(5))
+        ind = array([False,True])
+        raises(ValueError, "a[ind]")
         ind = array([False, True, False, True, False])
         assert ind.dtype is dtype(bool)
         # get length before actual calculation

From noreply at buildbot.pypy.org  Thu Oct  6 16:20:24 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Thu,  6 Oct 2011 16:20:24 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix-up the docstring for min.
Message-ID: <20111006142024.1D52C82A02@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r47843:2f94f8ae8756
Date: 2011-10-06 10:20 -0400
http://bitbucket.org/pypy/pypy/changeset/2f94f8ae8756/

Log:	Fix-up the docstring for min.

diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py
--- a/pypy/module/__builtin__/functional.py
+++ b/pypy/module/__builtin__/functional.py
@@ -193,9 +193,11 @@
     return min_max(space, __args__, "max")
 
 def min(space, __args__):
-    """Return the smallest item in a sequence.
+    """min(iterable[, key=func]) -> value
+    min(a, b, c, ...[, key=func]) -> value
 
-    If more than one argument is passed, return the minimum of them.
+    With a single iterable argument, return its smallest item.
+    With two or more arguments, return the smallest argument.
     """
     return min_max(space, __args__, "min")
 

From noreply at buildbot.pypy.org  Thu Oct  6 22:12:25 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Thu,  6 Oct 2011 22:12:25 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix indentation (how did it happen?)
Message-ID: <20111006201225.8C02182A02@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r47844:1917da0b25f3
Date: 2011-10-06 16:12 -0400
http://bitbucket.org/pypy/pypy/changeset/1917da0b25f3/

Log:	fix indentation (how did it happen?)

diff --git a/pypy/objspace/std/celldict.py b/pypy/objspace/std/celldict.py
--- a/pypy/objspace/std/celldict.py
+++ b/pypy/objspace/std/celldict.py
@@ -37,10 +37,10 @@
         self.version = VersionTag()
 
     def get_empty_storage(self):
-       return self.erase({})
+        return self.erase({})
 
     def mutated(self):
-       self.version = VersionTag()
+        self.version = VersionTag()
 
     def getdictvalue_no_unwrapping(self, w_dict, key):
         # NB: it's important to promote self here, so that self.version is a

From noreply at buildbot.pypy.org  Fri Oct  7 00:56:29 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri,  7 Oct 2011 00:56:29 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix typo and indentation in error
	messages
Message-ID: <20111006225629.BA7DC82A02@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r47845:167c86d2b1e3
Date: 2011-10-06 22:45 +0200
http://bitbucket.org/pypy/pypy/changeset/167c86d2b1e3/

Log:	Fix typo and indentation in error messages

diff --git a/pypy/module/__pypy__/interp_builders.py b/pypy/module/__pypy__/interp_builders.py
--- a/pypy/module/__pypy__/interp_builders.py
+++ b/pypy/module/__pypy__/interp_builders.py
@@ -16,7 +16,8 @@
 
         def _check_done(self, space):
             if self.builder is None:
-                raise OperationError(space.w_ValueError, space.wrap("Can't operate on a done builder"))
+                raise OperationError(space.w_ValueError, space.wrap(
+                        "Can't operate on a built builder"))
 
         @unwrap_spec(size=int)
         def descr__new__(space, w_subtype, size=-1):
@@ -31,7 +32,8 @@
         def descr_append_slice(self, space, s, start, end):
             self._check_done(space)
             if not 0 <= start <= end <= len(s):
-                raise OperationError(space.w_ValueError, space.wrap("bad start/stop"))
+                raise OperationError(space.w_ValueError, space.wrap(
+                        "bad start/stop"))
             self.builder.append_slice(s, start, end)
 
         def descr_build(self, space):
@@ -42,8 +44,8 @@
 
         def descr_len(self, space):
             if self.builder is None:
-                raise OperationError(space.w_ValueError,
-                                     space.wrap('no lenght of built builder'))
+                raise OperationError(space.w_ValueError, space.wrap(
+                        "no length of built builder"))
             return space.wrap(self.builder.getlength())
 
     W_Builder.__name__ = "W_%s" % name

From noreply at buildbot.pypy.org  Fri Oct  7 00:56:30 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri,  7 Oct 2011 00:56:30 +0200 (CEST)
Subject: [pypy-commit] pypy default: accept unicode arguments in posix.execv
 and posix.execve.
Message-ID: <20111006225630.F216482A86@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r47846:787a29a88622
Date: 2011-10-07 00:55 +0200
http://bitbucket.org/pypy/pypy/changeset/787a29a88622/

Log:	accept unicode arguments in posix.execv and posix.execve. encode
	with sys.getfilesystemencoding()

diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -10,6 +10,7 @@
 from pypy.rpython.lltypesystem import rffi, lltype
 from pypy.rpython.tool import rffi_platform
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
+from pypy.module.sys.interp_encoding import getfilesystemencoding
 
 import os, sys
 _WIN = sys.platform == 'win32'
@@ -32,17 +33,19 @@
             raise OperationError(space.w_OverflowError,
                                  space.wrap("integer out of range"))
 
+def fsencode_w(space, w_obj):
+    if space.isinstance_w(w_obj, space.w_unicode):
+        w_obj = space.call_method(w_obj, 'encode',
+                                  getfilesystemencoding(space))
+    return space.str_w(w_obj)
+
 class FileEncoder(object):
     def __init__(self, space, w_obj):
         self.space = space
         self.w_obj = w_obj
 
     def as_bytes(self):
-        from pypy.module.sys.interp_encoding import getfilesystemencoding
-        space = self.space
-        w_bytes = space.call_method(self.w_obj, 'encode',
-                                    getfilesystemencoding(space))
-        return space.str_w(w_bytes)
+        return fsencode_w(self.space, self.w_obj)
 
     def as_unicode(self):
         return self.space.unicode_w(self.w_obj)
@@ -56,7 +59,6 @@
         return self.space.str_w(self.w_obj)
 
     def as_unicode(self):
-        from pypy.module.sys.interp_encoding import getfilesystemencoding
         space = self.space
         w_unicode = space.call_method(self.w_obj, 'decode',
                                       getfilesystemencoding(space))
@@ -536,7 +538,6 @@
 
 The list is in arbitrary order.  It does not include the special
 entries '.' and '..' even if they are present in the directory."""
-    from pypy.module.sys.interp_encoding import getfilesystemencoding
     try:
         if space.isinstance_w(w_dirname, space.w_unicode):
             dirname = FileEncoder(space, w_dirname)
@@ -734,8 +735,7 @@
 def _exit(space, status):
     os._exit(status)
 
- at unwrap_spec(command=str)
-def execv(space, command, w_args):
+def execv(space, w_command, w_args):
     """ execv(path, args)
 
 Execute an executable path with arguments, replacing current process.
@@ -743,12 +743,13 @@
         path: path of executable file
         args: iterable of strings
     """
+    command = fsencode_w(space, w_command)
     try:
         args_w = space.unpackiterable(w_args)
         if len(args_w) < 1:
             w_msg = space.wrap("execv() must have at least one argument")
             raise OperationError(space.w_ValueError, w_msg)
-        args = [space.str_w(w_arg) for w_arg in args_w]
+        args = [fsencode_w(space, w_arg) for w_arg in args_w]
     except OperationError, e:
         if not e.match(space, space.w_TypeError):
             raise
@@ -759,8 +760,7 @@
     except OSError, e:
         raise wrap_oserror(space, e)
 
- at unwrap_spec(command=str)
-def execve(space, command, w_args, w_env):
+def execve(space, w_command, w_args, w_env):
     """ execve(path, args, env)
 
 Execute a path with arguments and environment, replacing current process.
@@ -769,7 +769,8 @@
         args: iterable of arguments
         env: dictionary of strings mapping to strings
     """
-    args = [space.str_w(w_arg) for w_arg in space.unpackiterable(w_args)]
+    command = fsencode_w(space, w_command)
+    args = [fsencode_w(space, w_arg) for w_arg in space.unpackiterable(w_args)]
     env = {}
     w_keys = space.call_method(w_env, 'keys')
     for w_key in space.unpackiterable(w_keys):
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -415,6 +415,20 @@
                 else:
                     py.test.fail("didn't raise")
 
+        def test_execv_unicode(self):
+            os = self.posix
+            import sys
+            if not hasattr(os, "fork"):
+                skip("Need fork() to test execv()")
+            pid = os.fork()
+            if pid == 0:
+                os.execv(u"/bin/sh", ["sh", "-c",
+                                      u"echo caf\xe9 \u1234 > onefile"])
+            os.waitpid(pid, 0)
+            output = u"caf\xe9 \u1234\n".encode(sys.getfilesystemencoding())
+            assert open("onefile").read() == output
+            os.unlink("onefile")
+
         def test_execve(self):
             os = self.posix
             if not hasattr(os, "fork"):
@@ -425,6 +439,21 @@
             os.waitpid(pid, 0)
             assert open("onefile").read() == "xxx"
             os.unlink("onefile")
+
+        def test_execve_unicode(self):
+            os = self.posix
+            import sys
+            if not hasattr(os, "fork"):
+                skip("Need fork() to test execve()")
+            pid = os.fork()
+            if pid == 0:
+                os.execve(u"/bin/sh", ["sh", "-c",
+                                      u"echo caf\xe9 \u1234 > onefile"],
+                          {'ddd': 'xxx'})
+            os.waitpid(pid, 0)
+            output = u"caf\xe9 \u1234\n".encode(sys.getfilesystemencoding())
+            assert open("onefile").read() == output
+            os.unlink("onefile")
         pass # <- please, inspect.getsource(), don't crash
 
     if hasattr(__import__(os.name), "spawnv"):

From noreply at buildbot.pypy.org  Fri Oct  7 08:25:33 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  7 Oct 2011 08:25:33 +0200 (CEST)
Subject: [pypy-commit] pypy continulet-jit: Found out a problem with the JIT
 and continulets: the JIT's
Message-ID: <20111007062533.31DF982A02@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: continulet-jit
Changeset: r47847:292172639cc1
Date: 2011-10-04 11:50 +0200
http://bitbucket.org/pypy/pypy/changeset/292172639cc1/

Log:	Found out a problem with the JIT and continulets: the JIT's runtime
	code tries occasionally to read stuff out of the stack (via
	"vable_token"), but the corresponding stack might have been put
	away.


From noreply at buildbot.pypy.org  Fri Oct  7 08:25:34 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  7 Oct 2011 08:25:34 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: A branch to play with
 trying out a "mostly-concurrent mark-sweep
Message-ID: <20111007062534.5C9BD82A86@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r47848:1bdf1d93d873
Date: 2011-10-06 14:48 +0200
http://bitbucket.org/pypy/pypy/changeset/1bdf1d93d873/

Log:	A branch to play with trying out a "mostly-concurrent mark-sweep
	garbage collector": the GC would run in a separate thread.


From noreply at buildbot.pypy.org  Fri Oct  7 08:25:35 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  7 Oct 2011 08:25:35 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Started.
Message-ID: <20111007062535.9241A82A87@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r47849:3bcb6e4544b8
Date: 2011-10-06 16:18 +0200
http://bitbucket.org/pypy/pypy/changeset/3bcb6e4544b8/

Log:	Started.

diff --git a/pypy/rpython/memory/gc/concurrentms.py b/pypy/rpython/memory/gc/concurrentms.py
new file mode 100644
--- /dev/null
+++ b/pypy/rpython/memory/gc/concurrentms.py
@@ -0,0 +1,141 @@
+from pypy.rpython.lltypesystem import lltype, llmemory, llarena, rffi
+from pypy.rpython.lltypesystem.lloperation import llop
+from pypy.rpython.lltypesystem.llmemory import raw_malloc_usage
+from pypy.rlib.debug import ll_assert
+from pypy.rlib.rarithmetic import LONG_BIT
+from pypy.rpython.memory.gc.base import GCBase
+
+#
+# A "mostly concurrent" mark&sweep GC.  It can delegate most of the GC
+# operations to a separate thread, which runs concurrently with the
+# mutator (i.e. the rest of the program).  Based on the idea that the
+# concurrent collection should be relatively fast --- 20-25% of the
+# time? after which the collector thread just sleeps --- it uses a
+# snapshot-at-the-beginning technique with a "deletion barrier", i.e. a
+# write barrier that prevents changes to objects that have not been
+# scanned yet (Abraham and Patel, Yuasa).
+#
+# Reference: The Garbage Collection Handbook, Richard Jones and Antony
+# Hosking and Eliot Moss.
+#
+
+WORD = LONG_BIT // 8
+NULL = llmemory.NULL
+WORD_POWER_2 = {32: 2, 64: 3}[LONG_BIT]
+assert 1 << WORD_POWER_2 == WORD
+size_of_addr = llmemory.sizeof(llmemory.Address)
+
+
+class MostlyConcurrentMarkSweepGC(GCBase):
+    _alloc_flavor_ = "raw"
+    inline_simple_malloc = True
+    inline_simple_malloc_varsize = True
+    needs_write_barrier = True
+    prebuilt_gc_objects_are_static_roots = False
+    malloc_zero_filled = True
+    #gcflag_extra = GCFLAG_FINALIZATION_ORDERING
+
+    HDR = lltype.Struct('header', ('tid', lltype.Signed))
+    typeid_is_in_field = 'tid'
+
+    TRANSLATION_PARAMS = {'page_size': 4096,
+                          'small_request_threshold': 35*WORD,
+                          }
+
+    def __init__(self, config, page_size=64, small_request_threshold=24,
+                 **kwds):
+        # 'small_request_threshold' is the largest size that we will
+        # satisfy using our own pages mecanism.  Larger requests just
+        # go to the system malloc().
+        GCBase.__init__(self, config, **kwds)
+        assert small_request_threshold % WORD == 0
+        self.small_request_threshold = small_request_threshold
+        self.page_size = page_size
+        self.free_pages = NULL
+        length = small_request_threshold // WORD + 1
+        self.free_lists = lltype.malloc(rffi.CArray(llmemory.Address),
+                                        length, flavor='raw', zero=True,
+                                        immortal=True)
+
+    def combine(self, typeid16, flags):
+        return llop.combine_ushort(lltype.Signed, typeid16, flags)
+
+    def malloc_fixedsize_clear(self, typeid, size,
+                               needs_finalizer=False, contains_weakptr=False):
+        assert not needs_finalizer  # XXX
+        assert not contains_weakptr # XXX
+        size_gc_header = self.gcheaderbuilder.size_gc_header
+        totalsize = size_gc_header + size
+        rawtotalsize = raw_malloc_usage(totalsize)
+        if rawtotalsize <= self.small_request_threshold:
+            n = (rawtotalsize + WORD - 1) >> WORD_POWER_2
+            result = self.free_lists[n]
+            if result != llmemory.NULL:
+                self.free_lists[n] = result.address[0]
+                #
+                llarena.arena_reset(result, size_of_addr, 0)
+                llarena.arena_reserve(result, totalsize)
+                hdr = llmemory.cast_adr_to_ptr(result, lltype.Ptr(self.HDR))
+                hdr.tid = self.combine(typeid, flags=0)
+                #
+                obj = result + size_gc_header
+                return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
+                #
+        return self._malloc_slowpath(typeid, size)
+
+    def _malloc_slowpath(self, typeid, size):
+        size_gc_header = self.gcheaderbuilder.size_gc_header
+        totalsize = size_gc_header + size
+        rawtotalsize = raw_malloc_usage(totalsize)
+        if rawtotalsize <= self.small_request_threshold:
+            #
+            # Case 1: we have run out of the free list corresponding to
+            # the size.  Grab the next free page.
+            newpage = self.free_pages
+            if newpage == llmemory.NULL:
+                self.allocate_next_arena()
+                newpage = self.free_pages
+            self.free_pages = newpage.address[0]
+            llarena.arena_reset(newpage, size_of_addr, 0)
+            #
+            # Initialize the free page to contain objects of the given
+            # size.  This requires setting up all object locations in the
+            # page, linking them in the free list.
+            n = (rawtotalsize + WORD - 1) >> WORD_POWER_2
+            head = self.free_lists[n]
+            ll_assert(not head, "_malloc_slowpath: unexpected free_lists[n]")
+            i = self.page_size - rawtotalsize
+            while i >= rawtotalsize:
+                llarena.arena_reserve(newpage + i, size_of_addr)
+                (newpage + i).address[0] = head
+                head = newpage + i
+                i -= rawtotalsize
+            self.free_lists[n] = head
+            result = head - rawtotalsize
+            #
+            # Done: all object locations are linked, apart from 'result',
+            # which is the first object location in the page.  Note that
+            # if the size is not a multiple of 2, there are a few wasted
+            # WORDs, which we place at the start of the page rather than
+            # at the end (Hans Boehm, xxx ref).
+            llarena.arena_reserve(result, totalsize)
+            hdr = llmemory.cast_adr_to_ptr(result, lltype.Ptr(self.HDR))
+            hdr.tid = self.combine(typeid, flags=0)
+            #
+            obj = result + size_gc_header
+            return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
+            #
+        else:
+            # Case 2: the object is too big, so allocate it directly
+            # with the system malloc().
+            xxxxx
+    _malloc_slowpath._dont_inline_ = True
+
+    def allocate_next_arena(self):
+        # xxx for now, allocate one page at a time with the system malloc()
+        page = llarena.arena_malloc(self.page_size, 2)     # zero-filled
+        ll_assert(bool(page), "out of memory!")
+        llarena.arena_reserve(page, size_of_addr)
+        page.address[0] = NULL
+        self.free_pages = page
+    allocate_next_arena._dont_inline_ = True
diff --git a/pypy/rpython/memory/gc/test/test_direct.py b/pypy/rpython/memory/gc/test/test_direct.py
--- a/pypy/rpython/memory/gc/test/test_direct.py
+++ b/pypy/rpython/memory/gc/test/test_direct.py
@@ -599,3 +599,7 @@
 
 class TestMiniMarkGCFull(DirectGCTest):
     from pypy.rpython.memory.gc.minimark import MiniMarkGC as GCClass
+
+class TestMostlyConcurrentMarkSweepGC(DirectGCTest):
+    from pypy.rpython.memory.gc.concurrentms \
+            import MostlyConcurrentMarkSweepGC as GCClass

From noreply at buildbot.pypy.org  Fri Oct  7 08:25:36 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  7 Oct 2011 08:25:36 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Marking.
Message-ID: <20111007062536.C63E682A88@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r47850:991e521500e4
Date: 2011-10-07 06:49 +0200
http://bitbucket.org/pypy/pypy/changeset/991e521500e4/

Log:	Marking.

diff --git a/pypy/rpython/lltypesystem/llarena.py b/pypy/rpython/lltypesystem/llarena.py
--- a/pypy/rpython/lltypesystem/llarena.py
+++ b/pypy/rpython/lltypesystem/llarena.py
@@ -77,8 +77,20 @@
         bytes = llmemory.raw_malloc_usage(size)
         if offset + bytes > self.nbytes:
             raise ArenaError("object overflows beyond the end of the arena")
+        # common case: 'size' starts with a GCHeaderOffset.  In this case
+        # we allocate with 'zero' even if the header doesn't have null
+        # bytes, as long as the rest of the object is over null bytes.
+        sz1 = size
+        while isinstance(sz1, RoundedUpForAllocation):
+            sz1 = sz1.basesize
+        if (isinstance(sz1, llmemory.CompositeOffset) and
+            isinstance(sz1.offsets[0], llmemory.GCHeaderOffset)):
+            hdrbytes = llmemory.raw_malloc_usage(sz1.offsets[0])
+            assert bytes >= hdrbytes
+        else:
+            hdrbytes = 0
         zero = True
-        for c in self.usagemap[offset:offset+bytes]:
+        for c in self.usagemap[offset+hdrbytes:offset+bytes]:
             if c == '0':
                 pass
             elif c == '#':
@@ -90,14 +102,10 @@
         pattern = letter.upper() + letter*(bytes-1)
         self.usagemap[offset:offset+bytes] = array.array('c', pattern)
         self.setobject(addr2, offset, bytes)
-        # common case: 'size' starts with a GCHeaderOffset.  In this case
-        # we can also remember that the real object starts after the header.
-        while isinstance(size, RoundedUpForAllocation):
-            size = size.basesize
-        if (isinstance(size, llmemory.CompositeOffset) and
-            isinstance(size.offsets[0], llmemory.GCHeaderOffset)):
-            objaddr = addr2 + size.offsets[0]
-            hdrbytes = llmemory.raw_malloc_usage(size.offsets[0])
+        # in the common case where 'size' starts with a GCHeaderOffset,
+        # we also remember that the real object starts after the header.
+        if hdrbytes > 0:
+            objaddr = addr2 + sz1.offsets[0]
             objoffset = offset + hdrbytes
             self.setobject(objaddr, objoffset, bytes - hdrbytes)
         return addr2
diff --git a/pypy/rpython/memory/gc/concurrentms.py b/pypy/rpython/memory/gc/concurrentms.py
--- a/pypy/rpython/memory/gc/concurrentms.py
+++ b/pypy/rpython/memory/gc/concurrentms.py
@@ -1,9 +1,12 @@
-from pypy.rpython.lltypesystem import lltype, llmemory, llarena, rffi
+import time, sys
+from pypy.rpython.lltypesystem import lltype, llmemory, llarena, llgroup, rffi
 from pypy.rpython.lltypesystem.lloperation import llop
 from pypy.rpython.lltypesystem.llmemory import raw_malloc_usage
+from pypy.rlib.objectmodel import we_are_translated, specialize
 from pypy.rlib.debug import ll_assert
-from pypy.rlib.rarithmetic import LONG_BIT
+from pypy.rlib.rarithmetic import LONG_BIT, r_uint
 from pypy.rpython.memory.gc.base import GCBase
+from pypy.module.thread import ll_thread
 
 #
 # A "mostly concurrent" mark&sweep GC.  It can delegate most of the GC
@@ -24,6 +27,10 @@
 WORD_POWER_2 = {32: 2, 64: 3}[LONG_BIT]
 assert 1 << WORD_POWER_2 == WORD
 size_of_addr = llmemory.sizeof(llmemory.Address)
+first_gcflag = 1 << (LONG_BIT//2)
+
+GCFLAG_MARK_TOGGLE = first_gcflag << 0
+GCFLAG_FINALIZATION_ORDERING = first_gcflag << 1
 
 
 class MostlyConcurrentMarkSweepGC(GCBase):
@@ -33,7 +40,7 @@
     needs_write_barrier = True
     prebuilt_gc_objects_are_static_roots = False
     malloc_zero_filled = True
-    #gcflag_extra = GCFLAG_FINALIZATION_ORDERING
+    gcflag_extra = GCFLAG_FINALIZATION_ORDERING
 
     HDR = lltype.Struct('header', ('tid', lltype.Signed))
     typeid_is_in_field = 'tid'
@@ -56,6 +63,54 @@
         self.free_lists = lltype.malloc(rffi.CArray(llmemory.Address),
                                         length, flavor='raw', zero=True,
                                         immortal=True)
+        self.current_mark = 0
+        #
+        # When the mutator thread wants to trigger the next collection,
+        # it scans its own stack roots and prepares everything, then
+        # sets 'collection_running' to True, and releases
+        # 'ready_to_start_lock'.  This triggers the collector thread,
+        # which re-acquires 'ready_to_start_lock' and does its job.
+        # When done it releases 'finished_lock'.  The mutator thread is
+        # responsible for resetting 'collection_running' to False.
+        self.collection_running = False
+        self.ready_to_start_lock = ll_thread.allocate_lock()
+        self.finished_lock = ll_thread.allocate_lock()
+        #
+        # NOT_RPYTHON: set to non-empty in _teardown()
+        self._teardown_now = []
+        #
+        def collector_start():
+            self.collector_run()
+        self.collector_start = collector_start
+        #
+        self.mutex_lock = ll_thread.allocate_lock()
+        self.gray_objects = self.AddressStack()
+        self.extra_objects_to_mark = self.AddressStack()
+        #
+        # Write barrier: actually a deletion barrier, triggered when there
+        # is a collection running and the mutator tries to change an object
+        # that was not scanned yet.
+        self._init_writebarrier_logic()
+
+    def setup(self):
+        "Start the concurrent collector thread."
+        self.acquire(self.finished_lock)
+        self.acquire(self.ready_to_start_lock)
+        self.collector_ident = ll_thread.start_new_thread(
+            self.collector_start, ())
+
+    def _teardown(self):
+        "NOT_RPYTHON.  Stop the collector thread after tests have run."
+        self.wait_for_the_end_of_collection()
+        #
+        # start the next collection, but with "stop" in _teardown_now,
+        # which should shut down the collector thread
+        self._teardown_now.append("stop")
+        self.collect()
+
+    def get_type_id(self, obj):
+        tid = self.header(obj).tid
+        return llop.extract_ushort(llgroup.HALFWORD, tid)
 
     def combine(self, typeid16, flags):
         return llop.combine_ushort(lltype.Signed, typeid16, flags)
@@ -76,7 +131,7 @@
                 llarena.arena_reset(result, size_of_addr, 0)
                 llarena.arena_reserve(result, totalsize)
                 hdr = llmemory.cast_adr_to_ptr(result, lltype.Ptr(self.HDR))
-                hdr.tid = self.combine(typeid, flags=0)
+                hdr.tid = self.combine(typeid, self.current_mark)
                 #
                 obj = result + size_gc_header
                 return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
@@ -120,7 +175,7 @@
             # at the end (Hans Boehm, xxx ref).
             llarena.arena_reserve(result, totalsize)
             hdr = llmemory.cast_adr_to_ptr(result, lltype.Ptr(self.HDR))
-            hdr.tid = self.combine(typeid, flags=0)
+            hdr.tid = self.combine(typeid, self.current_mark)
             #
             obj = result + size_gc_header
             return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
@@ -138,4 +193,171 @@
         llarena.arena_reserve(page, size_of_addr)
         page.address[0] = NULL
         self.free_pages = page
-    allocate_next_arena._dont_inline_ = True
+
+
+    def write_barrier(self, newvalue, addr_struct):
+        flag = self.header(addr_struct).tid & GCFLAG_MARK_TOGGLE
+        if flag != self.current_mark:
+            self.force_scan(addr_struct)
+
+    def _init_writebarrier_logic(self):
+        #
+        def force_scan(obj):
+            self.mutex_lock.acquire(True)
+            if self.current_mark:
+                self.set_mark_flag(obj, GCFLAG_MARK_TOGGLE)
+            else:
+                self.set_mark_flag(obj, 0)
+            self.trace(obj, self._barrier_add_extra, None)
+            self.mutex_lock.release()
+        #
+        force_scan._dont_inline_ = True
+        self.force_scan = force_scan
+
+    def _barrier_add_extra(self, root, ignored):
+        self.extra_objects_to_mark.append(root.address[0])
+
+
+    def collect(self, gen=0):
+        """Trigger a complete collection, and wait for it to finish."""
+        self.trigger_next_collection()
+        self.wait_for_the_end_of_collection()
+
+    def wait_for_the_end_of_collection(self):
+        """In the mutator thread: wait for the collection currently
+        running (if any) to finish."""
+        if self.collection_running:
+            self.acquire(self.finished_lock)
+            self.collection_running = False
+            #
+            # It's possible that an object was added to 'extra_objects_to_mark'
+            # by the write barrier but not taken out by the collector thread,
+            # because it finished in the meantime.  The result is still
+            # correct, but we need to clear the list.
+            self.extra_objects_to_mark.clear()
+
+    def trigger_next_collection(self):
+        """In the mutator thread: triggers the next collection."""
+        #
+        # In case the previous collection is not over yet, wait for it
+        self.wait_for_the_end_of_collection()
+        #
+        # Scan the stack roots and the refs in non-GC objects
+        self.root_walker.walk_roots(
+            MostlyConcurrentMarkSweepGC._add_stack_root,  # stack roots
+            MostlyConcurrentMarkSweepGC._add_stack_root,  # in prebuilt non-gc
+            None)                         # static in prebuilt gc
+        #
+        # Invert this global variable, which has the effect that on all
+        # objects' state go instantly from "marked" to "non marked"
+        self.current_mark ^= GCFLAG_MARK_TOGGLE
+        #
+        # Start the collector thread
+        self.collection_running = True
+        self.ready_to_start_lock.release()
+
+    def _add_stack_root(self, root):
+        obj = root.address[0]
+        self.gray_objects.append(obj)
+
+    def acquire(self, lock):
+        if we_are_translated():
+            lock.acquire(True)
+        else:
+            while not lock.acquire(False):
+                time.sleep(0.001)
+                # ---------- EXCEPTION FROM THE COLLECTOR THREAD ----------
+                if hasattr(self, '_exc_info'):
+                    self._reraise_from_collector_thread()
+
+    def _reraise_from_collector_thread(self):
+        exc, val, tb = self._exc_info
+        raise exc, val, tb
+
+
+    def collector_run(self):
+        """Main function of the collector's thread."""
+        try:
+            while True:
+                #
+                # Wait for the lock to be released
+                self.acquire(self.ready_to_start_lock)
+                #
+                # For tests: detect when we have to shut down
+                if not we_are_translated():
+                    if self._teardown_now:
+                        self.finished_lock.release()
+                        break
+                #
+                # Mark
+                self.collector_mark()
+                #
+                # Sweep
+                self.collector_sweep()
+        except Exception, e:
+            print 'Crash!', e.__class__.__name__, e
+            self._exc_info = sys.exc_info()
+
+    @specialize.arg(2)
+    def is_marked(self, obj, current_mark):
+        return (self.header(obj).tid & GCFLAG_MARK_TOGGLE) == current_mark
+
+    @specialize.arg(2)
+    def set_mark_flag(self, obj, current_mark):
+        if current_mark:
+            self.header(obj).tid |= GCFLAG_MARK_TOGGLE
+        else:
+            self.header(obj).tid &= ~GCFLAG_MARK_TOGGLE
+
+    def collector_mark(self):
+        while True:
+            #
+            # Do marking.  The following function call is interrupted
+            # if the mutator's write barrier adds new objects to
+            # 'extra_objects_to_mark'.
+            if self.current_mark:
+                self._collect_mark(GCFLAG_MARK_TOGGLE)
+            else:
+                self._collect_mark(0)
+            #
+            # Move the objects from 'extra_objects_to_mark' to
+            # 'gray_objects'.  This requires the mutex lock.
+            # There are typically only a few objects to move here,
+            # unless XXX we've hit the write barrier of a large array
+            self.mutex_lock.acquire(True)
+            while self.extra_objects_to_mark.non_empty():
+                obj = self.extra_objects_to_mark.pop()
+                self.gray_objects.append(obj)
+            self.mutex_lock.release()
+            #
+            # If 'gray_objects' is empty, we are done: there should be
+            # no possible case in which more objects are being added to
+            # 'extra_objects_to_mark' concurrently, because 'gray_objects'
+            # and 'extra_objects_to_mark' were already empty before we
+            # acquired the 'mutex_lock', so all reachable objects have
+            # been marked.
+            if not self.gray_objects.non_empty():
+                return
+
+    @specialize.arg(1)
+    def _collect_mark(self, current_mark):
+        while self.gray_objects.non_empty():
+            obj = self.gray_objects.pop()
+            if not self.is_marked(obj, current_mark):
+                self.set_mark_flag(obj, current_mark)
+                self.trace(obj, self._collect_add_pending, None)
+                #
+                # Interrupt early if the mutator's write barrier adds stuff
+                # to that list.  Note that the check is imprecise because
+                # it is not lock-protected, but that's good enough.  The
+                # idea is that we trace in priority objects flagged with
+                # the write barrier, because they are more likely to
+                # reference further objects that will soon be accessed too.
+                if self.extra_objects_to_mark.non_empty():
+                    return
+
+    def _collect_add_pending(self, root, ignored):
+        self.gray_objects.append(root.address[0])
+
+    def collector_sweep(self):
+        xxx
diff --git a/pypy/rpython/memory/support.py b/pypy/rpython/memory/support.py
--- a/pypy/rpython/memory/support.py
+++ b/pypy/rpython/memory/support.py
@@ -114,6 +114,13 @@
                 self.shrink()
             return result
 
+        def clear(self):
+            while self.chunk.next:
+                next = self.chunk.next
+                unused_chunks.put(self.chunk)
+                self.chunk = next
+            self.used_in_last_chunk = 0
+
         def delete(self):
             cur = self.chunk
             while cur:
diff --git a/pypy/rpython/memory/test/test_support.py b/pypy/rpython/memory/test/test_support.py
--- a/pypy/rpython/memory/test/test_support.py
+++ b/pypy/rpython/memory/test/test_support.py
@@ -94,6 +94,21 @@
             assert a == addrs[i]
         assert not ll.non_empty()
 
+    def test_clear(self):
+        AddressStack = get_address_stack()
+        addrs = [raw_malloc(llmemory.sizeof(lltype.Signed))
+                 for i in range(2200)]
+        ll = AddressStack()
+        for i in range(2200):
+            ll.append(addrs[i])
+        ll.clear()
+        assert not ll.non_empty()
+        ll.append(addrs[0])
+        assert ll.non_empty()
+        x = ll.pop()
+        assert x == addrs[0]
+        assert not ll.non_empty()
+
 
 class TestAddressDeque:
     def test_big_access(self):

From noreply at buildbot.pypy.org  Fri Oct  7 08:34:29 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  7 Oct 2011 08:34:29 +0200 (CEST)
Subject: [pypy-commit] pypy default: Clean up more the win32 logic,
 sharing more code with Posix.
Message-ID: <20111007063429.3D10C82A02@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r47851:3ad9eefabff2
Date: 2011-10-07 08:34 +0200
http://bitbucket.org/pypy/pypy/changeset/3ad9eefabff2/

Log:	Clean up more the win32 logic, sharing more code with Posix. Now it
	should create a "pypy.exe" instead of "pypy-c.exe", which was
	apparently considered confusing by people on pypy-dev.

diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -43,34 +43,29 @@
 def package(basedir, name='pypy-nightly', rename_pypy_c='pypy',
             copy_to_dir = None, override_pypy_c = None):
     basedir = py.path.local(basedir)
+    if override_pypy_c is None:
+        basename = 'pypy-c'
+        if sys.platform == 'win32':
+            basename += '.exe'
+        pypy_c = basedir.join('pypy', 'translator', 'goal', basename)
+    else:
+        pypy_c = py.path.local(override_pypy_c)
+    if not pypy_c.check():
+        print pypy_c
+        raise PyPyCNotFound('Please compile pypy first, using translate.py')
+    binaries = [(pypy_c, rename_pypy_c)]
+    #
     if sys.platform == 'win32':
-        # Can't rename a DLL
-        if override_pypy_c is not None:
-            rename_pypy_c = py.path.local(override_pypy_c).purebasename
-            pypy_c_dir = py.path.local(override_pypy_c).dirname
-        else:
-            pypy_c_dir = basedir.join('pypy', 'translator', 'goal')
-        pypy_c = pypy_c_dir.join(rename_pypy_c + '.exe')
-        libpypy_c = pypy_c_dir.join('lib' + rename_pypy_c + '.dll')
-        binaries = [(pypy_c, pypy_c.basename),
-                    (libpypy_c, libpypy_c.basename)]
-        for extra in ['libexpat.dll', 'sqlite3.dll', 'msvcr90.dll']:
-            p = pypy_c_dir.join(extra)
+        # Can't rename a DLL: it is always called 'libpypy_c.dll'
+        for extra in ['libpypy_c.dll',
+                      'libexpat.dll', 'sqlite3.dll', 'msvcr90.dll']:
+            p = pypy_c.dirpath().join(extra)
             if not p.check():
                 p = py.path.local.sysfind(extra)
                 assert p, "%s not found" % (extra,)
             print "Picking %s" % p
             binaries.append((p, p.basename))
-    else:
-        basename = 'pypy-c'
-        if override_pypy_c is None:
-            pypy_c = basedir.join('pypy', 'translator', 'goal', basename)
-        else:
-            pypy_c = py.path.local(override_pypy_c)
-        binaries = [(pypy_c, rename_pypy_c)]
-    if not pypy_c.check():
-        print pypy_c
-        raise PyPyCNotFound('Please compile pypy first, using translate.py')
+    #
     builddir = udir.ensure("build", dir=True)
     pypydir = builddir.ensure(name, dir=True)
     # Careful: to copy lib_pypy, copying just the svn-tracked files

From noreply at buildbot.pypy.org  Fri Oct  7 14:58:16 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  7 Oct 2011 14:58:16 +0200 (CEST)
Subject: [pypy-commit] pypy stm: long long support.
Message-ID: <20111007125816.35C6582A02@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm
Changeset: r47852:959f6f983474
Date: 2011-10-07 13:14 +0200
http://bitbucket.org/pypy/pypy/changeset/959f6f983474/

Log:	long long support.

diff --git a/pypy/translator/stm/funcgen.py b/pypy/translator/stm/funcgen.py
--- a/pypy/translator/stm/funcgen.py
+++ b/pypy/translator/stm/funcgen.py
@@ -19,12 +19,17 @@
     fieldsize = rffi.sizeof(T)
     if fieldsize >= size_of_voidp:
         assert 1      # xxx assert somehow that the field is aligned
-        assert fieldsize == size_of_voidp     # XXX
+        if fieldsize == size_of_voidp:
+            funcname = 'stm_read_word'
+        elif fieldsize == 8:    # 32-bit only: read a 64-bit field
+            funcname = 'stm_read_doubleword'
+        else:
+            raise NotImplementedError(fieldsize)
         expr = structdef.ptr_access_expr(basename,
                                          fieldname,
                                          baseexpr_is_const)
-        return '%s = (%s)stm_read_word((long*)&%s);' % (
-            newvalue, cfieldtypename, expr)
+        return '%s = (%s)%s((long*)&%s);' % (
+            newvalue, cfieldtypename, funcname, expr)
     else:
         # assume that the object is aligned, and any possible misalignment
         # comes from the field offset, so that it can be resolved at
@@ -48,12 +53,19 @@
     fieldsize = rffi.sizeof(T)
     if fieldsize >= size_of_voidp:
         assert 1      # xxx assert somehow that the field is aligned
-        assert fieldsize == size_of_voidp     # XXX
+        if fieldsize == size_of_voidp:
+            funcname = 'stm_write_word'
+            newtype = 'long'
+        elif fieldsize == 8:    # 32-bit only: read a 64-bit field
+            funcname = 'stm_write_doubleword'
+            newtype = 'long long'
+        else:
+            raise NotImplementedError(fieldsize)
         expr = structdef.ptr_access_expr(basename,
                                          fieldname,
                                          baseexpr_is_const)
-        return 'stm_write_word((long*)&%s, (long)%s);' % (
-            expr, newvalue)
+        return '%s((long*)&%s, (%s)%s);' % (
+            funcname, expr, newtype, newvalue)
     else:
         cfieldtypename = cdecl(fieldtypename, '')
         return ('stm_write_partial_word(sizeof(%s), (char*)%s, '
diff --git a/pypy/translator/stm/rstm.py b/pypy/translator/stm/rstm.py
--- a/pypy/translator/stm/rstm.py
+++ b/pypy/translator/stm/rstm.py
@@ -4,6 +4,7 @@
 from pypy.translator.stm import _rffi_stm
 from pypy.annotation import model as annmodel
 from pypy.objspace.flow.model import Constant
+from pypy.rlib.rarithmetic import r_uint, r_ulonglong
 
 size_of_voidp = rffi.sizeof(rffi.VOIDP)
 assert size_of_voidp & (size_of_voidp - 1) == 0
@@ -22,8 +23,14 @@
     p = rffi.cast(_rffi_stm.SignedP, p - misalignment)
     if fieldsize >= size_of_voidp:
         assert misalignment == 0
-        assert fieldsize == size_of_voidp   # XXX
-        res = _rffi_stm.stm_read_word(p)
+        if fieldsize == size_of_voidp:
+            res = _rffi_stm.stm_read_word(p)
+        elif fieldsize == 8:    # 32-bit only: read a 64-bit field
+            res0 = r_uint(_rffi_stm.stm_read_word(p))
+            res1 = r_uint(_rffi_stm.stm_read_word(rffi.ptradd(p, 1)))
+            res = (r_ulonglong(res1) << 32) | res0
+        else:
+            raise NotImplementedError(fieldsize)
     else:
         assert misalignment + fieldsize <= size_of_voidp
         res = _rffi_stm.stm_read_word(p)
@@ -42,8 +49,15 @@
     p = rffi.cast(_rffi_stm.SignedP, p - misalignment)
     if fieldsize >= size_of_voidp:
         assert misalignment == 0
-        assert fieldsize == size_of_voidp   # XXX
-        _rffi_stm.stm_write_word(p, rffi.cast(lltype.Signed, newvalue))
+        if fieldsize == size_of_voidp:
+            _rffi_stm.stm_write_word(p, rffi.cast(lltype.Signed, newvalue))
+        elif fieldsize == 8:    # 32-bit only: write a 64-bit field
+            _rffi_stm.stm_write_word(p, rffi.cast(lltype.Signed, newvalue))
+            p = rffi.ptradd(p, 1)
+            newvalue = rffi.cast(lltype.SignedLongLong, newvalue) >> 32
+            _rffi_stm.stm_write_word(p, rffi.cast(lltype.Signed, newvalue))
+        else:
+            raise NotImplementedError(fieldsize)
         #print 'ok'
     else:
         # bah, must read the complete word in order to modify only a part
diff --git a/pypy/translator/stm/src_stm/et.c b/pypy/translator/stm/src_stm/et.c
--- a/pypy/translator/stm/src_stm/et.c
+++ b/pypy/translator/stm/src_stm/et.c
@@ -773,3 +773,18 @@
   val = (val & mask) | (word & ~mask);
   stm_write_word(p, val);
 }
+
+long long stm_read_doubleword(long *addr)
+{
+  /* 32-bit only */
+  unsigned long res0 = (unsigned long)stm_read_word(addr);
+  unsigned long res1 = (unsigned long)stm_read_word(addr + 1);
+  return (((unsigned long long)res1) << 32) | res0;
+}
+
+void stm_write_doubleword(long *addr, long long val)
+{
+  /* 32-bit only */
+  stm_write_word(addr, (long)val);
+  stm_write_word(addr + 1, (long)(val >> 32));
+}
diff --git a/pypy/translator/stm/src_stm/et.h b/pypy/translator/stm/src_stm/et.h
--- a/pypy/translator/stm/src_stm/et.h
+++ b/pypy/translator/stm/src_stm/et.h
@@ -35,5 +35,8 @@
 
 void stm_write_partial_word(int fieldsize, char *base, long offset, long nval);
 
+long long stm_read_doubleword(long *addr);
+void stm_write_doubleword(long *addr, long long val);
+
 
 #endif  /* _ET_H */
diff --git a/pypy/translator/stm/test/test_rstm.py b/pypy/translator/stm/test/test_rstm.py
--- a/pypy/translator/stm/test/test_rstm.py
+++ b/pypy/translator/stm/test/test_rstm.py
@@ -2,11 +2,14 @@
 from pypy.translator.stm._rffi_stm import *
 from pypy.translator.stm.rstm import *
 from pypy.rpython.annlowlevel import llhelper
+from pypy.rlib.rarithmetic import r_longlong
 
 
 A = lltype.Struct('A', ('x', lltype.Signed), ('y', lltype.Signed),
                        ('c1', lltype.Char), ('c2', lltype.Char),
-                       ('c3', lltype.Char))
+                       ('c3', lltype.Char), ('l', lltype.SignedLongLong))
+rll1 = r_longlong(-10000000000003)
+rll2 = r_longlong(-300400500600700)
 
 def callback1(a):
     a = rffi.cast(lltype.Ptr(A), a)
@@ -14,10 +17,12 @@
     assert a.c1 == '/'
     assert a.c2 == '\\'
     assert a.c3 == '!'
+    assert a.l == rll1
     assert stm_getfield(a, 'x') == -611
     assert stm_getfield(a, 'c2') == '\\'
     assert stm_getfield(a, 'c1') == '/'
     assert stm_getfield(a, 'c3') == '!'
+    assert stm_getfield(a, 'l') == rll1
     p = lltype.direct_fieldptr(a, 'x')
     p = rffi.cast(SignedP, p)
     stm_write_word(p, 42 * a.y)
@@ -35,6 +40,7 @@
     a.c2 = '\\'
     a.c3 = '!'
     a.y = 0
+    a.l = rll1
     descriptor_init()
     perform_transaction(llhelper(CALLBACK, callback1),
                         rffi.cast(rffi.VOIDP, a))
@@ -43,6 +49,7 @@
     assert a.c1 == '/'
     assert a.c2 == '\\'
     assert a.c3 == '!'
+    assert a.l == rll1
     lltype.free(a, flavor='raw')
 
 def callback2(a):
@@ -51,22 +58,27 @@
     assert a.c1 == '&'
     assert a.c2 == '*'
     assert a.c3 == '#'
+    assert a.l == rll1
     assert stm_getfield(a, 'x') == -611
     assert stm_getfield(a, 'c1') == '&'
     assert stm_getfield(a, 'c2') == '*'
     assert stm_getfield(a, 'c3') == '#'
+    assert stm_getfield(a, 'l') == rll1
     stm_setfield(a, 'x', 42 * a.y)
     stm_setfield(a, 'c1', '(')
     stm_setfield(a, 'c2', '?')
     stm_setfield(a, 'c3', ')')
+    stm_setfield(a, 'l', rll2)
     assert stm_getfield(a, 'x') == 42 * a.y
     assert stm_getfield(a, 'c1') == '('
     assert stm_getfield(a, 'c2') == '?'
     assert stm_getfield(a, 'c3') == ')'
+    assert stm_getfield(a, 'l') == rll2
     assert a.x == -611 # xxx still the old value when reading non-transact.
     assert a.c1 == '&'
     assert a.c2 == '*'
     assert a.c3 == '#'
+    assert a.l == rll1
     if a.y < 10:
         a.y += 1    # non-transactionally
         abort_and_retry()
@@ -79,6 +91,7 @@
     a.c2 = '*'
     a.c3 = '#'
     a.y = 0
+    a.l = rll1
     descriptor_init()
     perform_transaction(llhelper(CALLBACK, callback2),
                         rffi.cast(rffi.VOIDP, a))
@@ -87,6 +100,7 @@
     assert a.c1 == '('
     assert a.c2 == '?'
     assert a.c3 == ')'
+    assert a.l == rll2
     lltype.free(a, flavor='raw')
 
 # ____________________________________________________________

From noreply at buildbot.pypy.org  Fri Oct  7 14:58:17 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  7 Oct 2011 14:58:17 +0200 (CEST)
Subject: [pypy-commit] pypy stm: Support for 'double'.
Message-ID: <20111007125817.64C1E82A86@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm
Changeset: r47853:aae602a1eca9
Date: 2011-10-07 13:30 +0200
http://bitbucket.org/pypy/pypy/changeset/aae602a1eca9/

Log:	Support for 'double'.

diff --git a/pypy/translator/stm/funcgen.py b/pypy/translator/stm/funcgen.py
--- a/pypy/translator/stm/funcgen.py
+++ b/pypy/translator/stm/funcgen.py
@@ -19,7 +19,9 @@
     fieldsize = rffi.sizeof(T)
     if fieldsize >= size_of_voidp:
         assert 1      # xxx assert somehow that the field is aligned
-        if fieldsize == size_of_voidp:
+        if T == lltype.Float:
+            funcname = 'stm_read_double'
+        elif fieldsize == size_of_voidp:
             funcname = 'stm_read_word'
         elif fieldsize == 8:    # 32-bit only: read a 64-bit field
             funcname = 'stm_read_doubleword'
@@ -53,7 +55,10 @@
     fieldsize = rffi.sizeof(T)
     if fieldsize >= size_of_voidp:
         assert 1      # xxx assert somehow that the field is aligned
-        if fieldsize == size_of_voidp:
+        if T == lltype.Float:
+            funcname = 'stm_write_double'
+            newtype = 'double'
+        elif fieldsize == size_of_voidp:
             funcname = 'stm_write_word'
             newtype = 'long'
         elif fieldsize == 8:    # 32-bit only: read a 64-bit field
diff --git a/pypy/translator/stm/rstm.py b/pypy/translator/stm/rstm.py
--- a/pypy/translator/stm/rstm.py
+++ b/pypy/translator/stm/rstm.py
@@ -5,6 +5,7 @@
 from pypy.annotation import model as annmodel
 from pypy.objspace.flow.model import Constant
 from pypy.rlib.rarithmetic import r_uint, r_ulonglong
+from pypy.rlib import longlong2float
 
 size_of_voidp = rffi.sizeof(rffi.VOIDP)
 assert size_of_voidp & (size_of_voidp - 1) == 0
@@ -31,6 +32,8 @@
             res = (r_ulonglong(res1) << 32) | res0
         else:
             raise NotImplementedError(fieldsize)
+        if FIELD == lltype.Float:
+            return longlong2float.longlong2float(rffi.cast(rffi.LONGLONG, res))
     else:
         assert misalignment + fieldsize <= size_of_voidp
         res = _rffi_stm.stm_read_word(p)
@@ -49,6 +52,8 @@
     p = rffi.cast(_rffi_stm.SignedP, p - misalignment)
     if fieldsize >= size_of_voidp:
         assert misalignment == 0
+        if FIELD == lltype.Float:
+            newvalue = longlong2float.float2longlong(newvalue)
         if fieldsize == size_of_voidp:
             _rffi_stm.stm_write_word(p, rffi.cast(lltype.Signed, newvalue))
         elif fieldsize == 8:    # 32-bit only: write a 64-bit field
diff --git a/pypy/translator/stm/src_stm/et.c b/pypy/translator/stm/src_stm/et.c
--- a/pypy/translator/stm/src_stm/et.c
+++ b/pypy/translator/stm/src_stm/et.c
@@ -788,3 +788,27 @@
   stm_write_word(addr, (long)val);
   stm_write_word(addr + 1, (long)(val >> 32));
 }
+
+double stm_read_double(long *addr)
+{
+  long long x;
+  double dd;
+  if (sizeof(double) > sizeof(long))
+    x = stm_read_doubleword(addr);   /* 32 bits */
+  else
+    x = stm_read_word(addr);         /* 64 bits */
+  assert(sizeof(double) == 8 && sizeof(long long) == 8);
+  memcpy(&dd, &x, 8);
+  return dd;
+}
+
+void stm_write_double(long *addr, double val)
+{
+  long long ll;
+  assert(sizeof(double) == 8 && sizeof(long long) == 8);
+  memcpy(&ll, &val, 8);
+  if (sizeof(double) > sizeof(long))
+    stm_write_doubleword(addr, ll);   /* 32 bits */
+  else
+    stm_write_word(addr, ll);         /* 64 bits */
+}
diff --git a/pypy/translator/stm/src_stm/et.h b/pypy/translator/stm/src_stm/et.h
--- a/pypy/translator/stm/src_stm/et.h
+++ b/pypy/translator/stm/src_stm/et.h
@@ -35,6 +35,8 @@
 
 void stm_write_partial_word(int fieldsize, char *base, long offset, long nval);
 
+double stm_read_double(long *addr);
+void stm_write_double(long *addr, double val);
 long long stm_read_doubleword(long *addr);
 void stm_write_doubleword(long *addr, long long val);
 
diff --git a/pypy/translator/stm/test/test_rstm.py b/pypy/translator/stm/test/test_rstm.py
--- a/pypy/translator/stm/test/test_rstm.py
+++ b/pypy/translator/stm/test/test_rstm.py
@@ -7,9 +7,12 @@
 
 A = lltype.Struct('A', ('x', lltype.Signed), ('y', lltype.Signed),
                        ('c1', lltype.Char), ('c2', lltype.Char),
-                       ('c3', lltype.Char), ('l', lltype.SignedLongLong))
+                       ('c3', lltype.Char), ('l', lltype.SignedLongLong),
+                       ('f', lltype.Float))
 rll1 = r_longlong(-10000000000003)
 rll2 = r_longlong(-300400500600700)
+rf1 = -12.38976129
+rf2 = 52.1029
 
 def callback1(a):
     a = rffi.cast(lltype.Ptr(A), a)
@@ -18,11 +21,13 @@
     assert a.c2 == '\\'
     assert a.c3 == '!'
     assert a.l == rll1
+    assert a.f == rf1
     assert stm_getfield(a, 'x') == -611
     assert stm_getfield(a, 'c2') == '\\'
     assert stm_getfield(a, 'c1') == '/'
     assert stm_getfield(a, 'c3') == '!'
     assert stm_getfield(a, 'l') == rll1
+    assert stm_getfield(a, 'f') == rf1
     p = lltype.direct_fieldptr(a, 'x')
     p = rffi.cast(SignedP, p)
     stm_write_word(p, 42 * a.y)
@@ -41,6 +46,7 @@
     a.c3 = '!'
     a.y = 0
     a.l = rll1
+    a.f = rf1
     descriptor_init()
     perform_transaction(llhelper(CALLBACK, callback1),
                         rffi.cast(rffi.VOIDP, a))
@@ -50,6 +56,7 @@
     assert a.c2 == '\\'
     assert a.c3 == '!'
     assert a.l == rll1
+    assert a.f == rf1
     lltype.free(a, flavor='raw')
 
 def callback2(a):
@@ -59,26 +66,31 @@
     assert a.c2 == '*'
     assert a.c3 == '#'
     assert a.l == rll1
+    assert a.f == rf1
     assert stm_getfield(a, 'x') == -611
     assert stm_getfield(a, 'c1') == '&'
     assert stm_getfield(a, 'c2') == '*'
     assert stm_getfield(a, 'c3') == '#'
     assert stm_getfield(a, 'l') == rll1
+    assert stm_getfield(a, 'f') == rf1
     stm_setfield(a, 'x', 42 * a.y)
     stm_setfield(a, 'c1', '(')
     stm_setfield(a, 'c2', '?')
     stm_setfield(a, 'c3', ')')
     stm_setfield(a, 'l', rll2)
+    stm_setfield(a, 'f', rf2)
     assert stm_getfield(a, 'x') == 42 * a.y
     assert stm_getfield(a, 'c1') == '('
     assert stm_getfield(a, 'c2') == '?'
     assert stm_getfield(a, 'c3') == ')'
     assert stm_getfield(a, 'l') == rll2
+    assert stm_getfield(a, 'f') == rf2
     assert a.x == -611 # xxx still the old value when reading non-transact.
     assert a.c1 == '&'
     assert a.c2 == '*'
     assert a.c3 == '#'
     assert a.l == rll1
+    assert a.f == rf1
     if a.y < 10:
         a.y += 1    # non-transactionally
         abort_and_retry()
@@ -92,6 +104,7 @@
     a.c3 = '#'
     a.y = 0
     a.l = rll1
+    a.f = rf1
     descriptor_init()
     perform_transaction(llhelper(CALLBACK, callback2),
                         rffi.cast(rffi.VOIDP, a))
@@ -101,6 +114,7 @@
     assert a.c2 == '?'
     assert a.c3 == ')'
     assert a.l == rll2
+    assert a.f == rf2
     lltype.free(a, flavor='raw')
 
 # ____________________________________________________________

From noreply at buildbot.pypy.org  Fri Oct  7 14:58:18 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  7 Oct 2011 14:58:18 +0200 (CEST)
Subject: [pypy-commit] pypy stm: Single floats.
Message-ID: <20111007125818.93AB182A02@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm
Changeset: r47854:2e13d3c1cb12
Date: 2011-10-07 14:55 +0200
http://bitbucket.org/pypy/pypy/changeset/2e13d3c1cb12/

Log:	Single floats.

diff --git a/pypy/translator/stm/_rffi_stm.py b/pypy/translator/stm/_rffi_stm.py
--- a/pypy/translator/stm/_rffi_stm.py
+++ b/pypy/translator/stm/_rffi_stm.py
@@ -24,7 +24,7 @@
 descriptor_init = llexternal('stm_descriptor_init', [], lltype.Void)
 descriptor_done = llexternal('stm_descriptor_done', [], lltype.Void)
 
-begin_transaction = llexternal('stm_begin_transaction_inline',[], lltype.Void)
+begin_transaction = llexternal('STM_begin_transaction',[], lltype.Void)
 commit_transaction = llexternal('stm_commit_transaction', [], lltype.Signed)
 
 stm_read_word = llexternal('stm_read_word', [SignedP], lltype.Signed)
diff --git a/pypy/translator/stm/funcgen.py b/pypy/translator/stm/funcgen.py
--- a/pypy/translator/stm/funcgen.py
+++ b/pypy/translator/stm/funcgen.py
@@ -17,10 +17,12 @@
     #
     assert T is not lltype.Void     # XXX
     fieldsize = rffi.sizeof(T)
-    if fieldsize >= size_of_voidp:
+    if fieldsize >= size_of_voidp or T == lltype.SingleFloat:
         assert 1      # xxx assert somehow that the field is aligned
         if T == lltype.Float:
             funcname = 'stm_read_double'
+        elif T == lltype.SingleFloat:
+            funcname = 'stm_read_float'
         elif fieldsize == size_of_voidp:
             funcname = 'stm_read_word'
         elif fieldsize == 8:    # 32-bit only: read a 64-bit field
@@ -36,7 +38,7 @@
         # assume that the object is aligned, and any possible misalignment
         # comes from the field offset, so that it can be resolved at
         # compile-time (by using C macros)
-        return '%s = stm_read_partial_word(%s, %s, offsetof(%s, %s));' % (
+        return '%s = STM_read_partial_word(%s, %s, offsetof(%s, %s));' % (
             newvalue, cfieldtypename, basename,
             cdecl(funcgen.db.gettype(STRUCT), ''),
             structdef.c_struct_field_name(fieldname))
@@ -53,11 +55,14 @@
     #
     assert T is not lltype.Void     # XXX
     fieldsize = rffi.sizeof(T)
-    if fieldsize >= size_of_voidp:
+    if fieldsize >= size_of_voidp or T == lltype.SingleFloat:
         assert 1      # xxx assert somehow that the field is aligned
         if T == lltype.Float:
             funcname = 'stm_write_double'
             newtype = 'double'
+        elif T == lltype.SingleFloat:
+            funcname = 'stm_write_float'
+            newtype = 'float'
         elif fieldsize == size_of_voidp:
             funcname = 'stm_write_word'
             newtype = 'long'
diff --git a/pypy/translator/stm/rstm.py b/pypy/translator/stm/rstm.py
--- a/pypy/translator/stm/rstm.py
+++ b/pypy/translator/stm/rstm.py
@@ -38,6 +38,8 @@
         assert misalignment + fieldsize <= size_of_voidp
         res = _rffi_stm.stm_read_word(p)
         res = res >> (misalignment * 8)
+    if FIELD == lltype.SingleFloat:
+        return longlong2float.uint2singlefloat(rffi.cast(rffi.UINT, res))
     return rffi.cast(FIELD, res)
 
 def stm_setfield(structptr, fieldname, newvalue):
@@ -50,6 +52,8 @@
     fieldsize = rffi.sizeof(FIELD)
     #print 'setfield %x size %d:' % (p, fieldsize),
     p = rffi.cast(_rffi_stm.SignedP, p - misalignment)
+    if FIELD == lltype.SingleFloat:
+        newvalue = longlong2float.singlefloat2uint(newvalue)
     if fieldsize >= size_of_voidp:
         assert misalignment == 0
         if FIELD == lltype.Float:
diff --git a/pypy/translator/stm/src_stm/et.c b/pypy/translator/stm/src_stm/et.c
--- a/pypy/translator/stm/src_stm/et.c
+++ b/pypy/translator/stm/src_stm/et.c
@@ -588,7 +588,7 @@
 void* stm_perform_transaction(void*(*callback)(void*), void *arg)
 {
   void *result;
-  stm_begin_transaction_inline();
+  STM_begin_transaction();
   result = callback(arg);
   stm_commit_transaction();
   return result;
@@ -763,13 +763,14 @@
 }
 
 // XXX little-endian only!
-void stm_write_partial_word(int fieldsize, char *base, long offset, long nval)
+void stm_write_partial_word(int fieldsize, char *base, long offset,
+                            unsigned long nval)
 {
   long *p = (long*)(base + (offset & ~(sizeof(void*)-1)));
   int misalignment = offset & (sizeof(void*)-1);
   long val = nval << (misalignment * 8);
   long word = stm_read_word(p);
-  long mask = ((1 << (fieldsize * 8)) - 1) << (misalignment * 8);
+  long mask = ((1L << (fieldsize * 8)) - 1) << (misalignment * 8);
   val = (val & mask) | (word & ~mask);
   stm_write_word(p, val);
 }
@@ -812,3 +813,34 @@
   else
     stm_write_word(addr, ll);         /* 64 bits */
 }
+
+float stm_read_float(long *addr)
+{
+  unsigned int x;
+  float ff;
+  if (sizeof(float) == sizeof(long))
+    x = stm_read_word(addr);         /* 32 bits */
+  else if (((long)(char*)addr) & 7) {
+    addr = (long *)(((char *)addr) - 4);
+    x = (unsigned int)(stm_read_word(addr) >> 32);   /* 64 bits, unaligned */
+  }
+  else
+    x = (unsigned int)stm_read_word(addr);           /* 64 bits, aligned */
+  assert(sizeof(float) == 4 && sizeof(unsigned int) == 4);
+  memcpy(&ff, &x, 4);
+  return ff;
+}
+
+void stm_write_float(long *addr, float val)
+{
+  unsigned int ii;
+  assert(sizeof(float) == 4 && sizeof(unsigned int) == 4);
+  memcpy(&ii, &val, 4);
+  if (sizeof(float) == sizeof(long))
+    stm_write_word(addr, ii);         /* 32 bits */
+  else if (((long)(char*)addr) & 7)
+    stm_write_partial_word(4, (((char *)addr) - 4),
+                           4, ii);                   /* 64 bits, unaligned */
+  else
+    stm_write_partial_word(4, (char *)addr, 0, ii);    /* 64 bits, aligned */
+}
diff --git a/pypy/translator/stm/src_stm/et.h b/pypy/translator/stm/src_stm/et.h
--- a/pypy/translator/stm/src_stm/et.h
+++ b/pypy/translator/stm/src_stm/et.h
@@ -22,21 +22,24 @@
 void stm_begin_inevitable_transaction(void);
 void stm_abort_and_retry(void);
 
-#define stm_begin_transaction_inline()  ; \
+#define STM_begin_transaction()         ; \
        jmp_buf _jmpbuf;                   \
        setjmp(_jmpbuf);                   \
        stm_begin_transaction(&_jmpbuf)
 
 // XXX little-endian only!
-#define stm_read_partial_word(T, base, offset)                          \
+#define STM_read_partial_word(T, base, offset)                          \
     (T)(stm_read_word(                                                  \
            (long*)(((char*)(base)) + ((offset) & ~(sizeof(void*)-1))))  \
         >> (8 * ((offset) & (sizeof(void*)-1))))
 
-void stm_write_partial_word(int fieldsize, char *base, long offset, long nval);
+void stm_write_partial_word(int fieldsize, char *base, long offset,
+                            unsigned long nval);
 
 double stm_read_double(long *addr);
 void stm_write_double(long *addr, double val);
+float stm_read_float(long *addr);
+void stm_write_float(long *addr, float val);
 long long stm_read_doubleword(long *addr);
 void stm_write_doubleword(long *addr, long long val);
 
diff --git a/pypy/translator/stm/test/test_rstm.py b/pypy/translator/stm/test/test_rstm.py
--- a/pypy/translator/stm/test/test_rstm.py
+++ b/pypy/translator/stm/test/test_rstm.py
@@ -2,17 +2,22 @@
 from pypy.translator.stm._rffi_stm import *
 from pypy.translator.stm.rstm import *
 from pypy.rpython.annlowlevel import llhelper
-from pypy.rlib.rarithmetic import r_longlong
+from pypy.rlib.rarithmetic import r_longlong, r_singlefloat
 
 
 A = lltype.Struct('A', ('x', lltype.Signed), ('y', lltype.Signed),
                        ('c1', lltype.Char), ('c2', lltype.Char),
                        ('c3', lltype.Char), ('l', lltype.SignedLongLong),
-                       ('f', lltype.Float))
+                       ('f', lltype.Float), ('sa', lltype.SingleFloat),
+                       ('sb', lltype.SingleFloat))
 rll1 = r_longlong(-10000000000003)
 rll2 = r_longlong(-300400500600700)
 rf1 = -12.38976129
 rf2 = 52.1029
+rs1a = r_singlefloat(-0.598127)
+rs2a = r_singlefloat(0.017634)
+rs1b = r_singlefloat(40.121)
+rs2b = r_singlefloat(-9e9)
 
 def callback1(a):
     a = rffi.cast(lltype.Ptr(A), a)
@@ -22,12 +27,16 @@
     assert a.c3 == '!'
     assert a.l == rll1
     assert a.f == rf1
+    assert float(a.sa) == float(rs1a)
+    assert float(a.sb) == float(rs1b)
     assert stm_getfield(a, 'x') == -611
     assert stm_getfield(a, 'c2') == '\\'
     assert stm_getfield(a, 'c1') == '/'
     assert stm_getfield(a, 'c3') == '!'
     assert stm_getfield(a, 'l') == rll1
     assert stm_getfield(a, 'f') == rf1
+    assert float(stm_getfield(a, 'sa')) == float(rs1a)
+    assert float(stm_getfield(a, 'sb')) == float(rs1b)
     p = lltype.direct_fieldptr(a, 'x')
     p = rffi.cast(SignedP, p)
     stm_write_word(p, 42 * a.y)
@@ -47,6 +56,8 @@
     a.y = 0
     a.l = rll1
     a.f = rf1
+    a.sa = rs1a
+    a.sb = rs1b
     descriptor_init()
     perform_transaction(llhelper(CALLBACK, callback1),
                         rffi.cast(rffi.VOIDP, a))
@@ -57,6 +68,8 @@
     assert a.c3 == '!'
     assert a.l == rll1
     assert a.f == rf1
+    assert float(a.sa) == float(rs1a)
+    assert float(a.sb) == float(rs1b)
     lltype.free(a, flavor='raw')
 
 def callback2(a):
@@ -67,30 +80,40 @@
     assert a.c3 == '#'
     assert a.l == rll1
     assert a.f == rf1
+    assert float(a.sa) == float(rs1a)
+    assert float(a.sb) == float(rs1b)
     assert stm_getfield(a, 'x') == -611
     assert stm_getfield(a, 'c1') == '&'
     assert stm_getfield(a, 'c2') == '*'
     assert stm_getfield(a, 'c3') == '#'
     assert stm_getfield(a, 'l') == rll1
     assert stm_getfield(a, 'f') == rf1
+    assert float(stm_getfield(a, 'sa')) == float(rs1a)
+    assert float(stm_getfield(a, 'sb')) == float(rs1b)
     stm_setfield(a, 'x', 42 * a.y)
     stm_setfield(a, 'c1', '(')
     stm_setfield(a, 'c2', '?')
     stm_setfield(a, 'c3', ')')
     stm_setfield(a, 'l', rll2)
     stm_setfield(a, 'f', rf2)
+    stm_setfield(a, 'sa', rs2a)
+    stm_setfield(a, 'sb', rs2b)
     assert stm_getfield(a, 'x') == 42 * a.y
     assert stm_getfield(a, 'c1') == '('
     assert stm_getfield(a, 'c2') == '?'
     assert stm_getfield(a, 'c3') == ')'
     assert stm_getfield(a, 'l') == rll2
     assert stm_getfield(a, 'f') == rf2
+    assert float(stm_getfield(a, 'sa')) == float(rs2a)
+    assert float(stm_getfield(a, 'sb')) == float(rs2b)
     assert a.x == -611 # xxx still the old value when reading non-transact.
     assert a.c1 == '&'
     assert a.c2 == '*'
     assert a.c3 == '#'
     assert a.l == rll1
     assert a.f == rf1
+    assert float(a.sa) == float(rs1a)
+    assert float(a.sb) == float(rs1b)
     if a.y < 10:
         a.y += 1    # non-transactionally
         abort_and_retry()
@@ -105,6 +128,8 @@
     a.y = 0
     a.l = rll1
     a.f = rf1
+    a.sa = rs1a
+    a.sb = rs1b
     descriptor_init()
     perform_transaction(llhelper(CALLBACK, callback2),
                         rffi.cast(rffi.VOIDP, a))
@@ -115,6 +140,8 @@
     assert a.c3 == ')'
     assert a.l == rll2
     assert a.f == rf2
+    assert float(a.sa) == float(rs2a)
+    assert float(a.sb) == float(rs2b)
     lltype.free(a, flavor='raw')
 
 # ____________________________________________________________

From noreply at buildbot.pypy.org  Fri Oct  7 16:17:33 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  7 Oct 2011 16:17:33 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: The first few tests pass
	:-)
Message-ID: <20111007141733.3E02882A02@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r47855:2ee9ca05d5df
Date: 2011-10-07 15:54 +0200
http://bitbucket.org/pypy/pypy/changeset/2ee9ca05d5df/

Log:	The first few tests pass :-)

diff --git a/pypy/rpython/memory/gc/concurrentms.py b/pypy/rpython/memory/gc/concurrentms.py
--- a/pypy/rpython/memory/gc/concurrentms.py
+++ b/pypy/rpython/memory/gc/concurrentms.py
@@ -1,8 +1,7 @@
 import time, sys
 from pypy.rpython.lltypesystem import lltype, llmemory, llarena, llgroup, rffi
-from pypy.rpython.lltypesystem.lloperation import llop
 from pypy.rpython.lltypesystem.llmemory import raw_malloc_usage
-from pypy.rlib.objectmodel import we_are_translated, specialize
+from pypy.rlib.objectmodel import we_are_translated, running_on_llinterp
 from pypy.rlib.debug import ll_assert
 from pypy.rlib.rarithmetic import LONG_BIT, r_uint
 from pypy.rpython.memory.gc.base import GCBase
@@ -27,10 +26,17 @@
 WORD_POWER_2 = {32: 2, 64: 3}[LONG_BIT]
 assert 1 << WORD_POWER_2 == WORD
 size_of_addr = llmemory.sizeof(llmemory.Address)
-first_gcflag = 1 << (LONG_BIT//2)
 
-GCFLAG_MARK_TOGGLE = first_gcflag << 0
-GCFLAG_FINALIZATION_ORDERING = first_gcflag << 1
+# XXX assumes little-endian machines for now: the byte at offset 0 in
+# the object is either a mark byte (equal to an odd value), or if the
+# location is free, it is the low byte of a pointer to the next free
+# location (and then it is an even value, by pointer alignment).
+assert sys.byteorder == 'little'
+
+
+MARK_VALUE_1      = 'M'     #  77, 0x4D
+MARK_VALUE_2      = 'k'     # 107, 0x6B
+GCFLAG_WITH_HASH  = 0x01
 
 
 class MostlyConcurrentMarkSweepGC(GCBase):
@@ -40,10 +46,13 @@
     needs_write_barrier = True
     prebuilt_gc_objects_are_static_roots = False
     malloc_zero_filled = True
-    gcflag_extra = GCFLAG_FINALIZATION_ORDERING
+    #gcflag_extra = GCFLAG_FINALIZATION_ORDERING
 
-    HDR = lltype.Struct('header', ('tid', lltype.Signed))
-    typeid_is_in_field = 'tid'
+    HDR = lltype.Struct('header', ('mark', lltype.Char),  # MARK_VALUE_{1,2}
+                                  ('flags', lltype.Char),
+                                  ('typeid16', llgroup.HALFWORD))
+    typeid_is_in_field = 'typeid16'
+    withhash_flag_is_in_field = 'flags', GCFLAG_WITH_HASH
 
     TRANSLATION_PARAMS = {'page_size': 4096,
                           'small_request_threshold': 35*WORD,
@@ -59,20 +68,27 @@
         self.small_request_threshold = small_request_threshold
         self.page_size = page_size
         self.free_pages = NULL
-        length = small_request_threshold // WORD + 1
-        self.free_lists = lltype.malloc(rffi.CArray(llmemory.Address),
-                                        length, flavor='raw', zero=True,
-                                        immortal=True)
-        self.current_mark = 0
+        self.pagelists_length = small_request_threshold // WORD + 1
+        #
+        def list_of_addresses_per_small_size():
+            return lltype.malloc(rffi.CArray(llmemory.Address),
+                                 self.pagelists_length, flavor='raw',
+                                 zero=True, immortal=True)
+        self.nonfree_pages = list_of_addresses_per_small_size()
+        self.collect_pages = list_of_addresses_per_small_size()
+        self.free_lists    = list_of_addresses_per_small_size()
+        self.collect_heads = list_of_addresses_per_small_size()
+        self.collect_tails = list_of_addresses_per_small_size()
+        self.current_mark = MARK_VALUE_1
         #
         # When the mutator thread wants to trigger the next collection,
         # it scans its own stack roots and prepares everything, then
-        # sets 'collection_running' to True, and releases
+        # sets 'collection_running' to 1, and releases
         # 'ready_to_start_lock'.  This triggers the collector thread,
         # which re-acquires 'ready_to_start_lock' and does its job.
         # When done it releases 'finished_lock'.  The mutator thread is
-        # responsible for resetting 'collection_running' to False.
-        self.collection_running = False
+        # responsible for resetting 'collection_running' to 0.
+        self.collection_running = 0
         self.ready_to_start_lock = ll_thread.allocate_lock()
         self.finished_lock = ll_thread.allocate_lock()
         #
@@ -109,11 +125,7 @@
         self.collect()
 
     def get_type_id(self, obj):
-        tid = self.header(obj).tid
-        return llop.extract_ushort(llgroup.HALFWORD, tid)
-
-    def combine(self, typeid16, flags):
-        return llop.combine_ushort(lltype.Signed, typeid16, flags)
+        return self.header(obj).typeid16
 
     def malloc_fixedsize_clear(self, typeid, size,
                                needs_finalizer=False, contains_weakptr=False):
@@ -131,7 +143,9 @@
                 llarena.arena_reset(result, size_of_addr, 0)
                 llarena.arena_reserve(result, totalsize)
                 hdr = llmemory.cast_adr_to_ptr(result, lltype.Ptr(self.HDR))
-                hdr.tid = self.combine(typeid, self.current_mark)
+                hdr.typeid16 = typeid
+                hdr.mark = self.current_mark
+                hdr.flags = '\x00'
                 #
                 obj = result + size_gc_header
                 return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
@@ -151,16 +165,21 @@
                 self.allocate_next_arena()
                 newpage = self.free_pages
             self.free_pages = newpage.address[0]
-            llarena.arena_reset(newpage, size_of_addr, 0)
+            #
+            # Put the free page in the list 'nonfree_pages[n]'.  This is
+            # a linked list chained through the first word of each page.
+            n = (rawtotalsize + WORD - 1) >> WORD_POWER_2
+            newpage.address[0] = self.nonfree_pages[n]
+            self.nonfree_pages[n] = newpage
             #
             # Initialize the free page to contain objects of the given
             # size.  This requires setting up all object locations in the
             # page, linking them in the free list.
-            n = (rawtotalsize + WORD - 1) >> WORD_POWER_2
             head = self.free_lists[n]
             ll_assert(not head, "_malloc_slowpath: unexpected free_lists[n]")
             i = self.page_size - rawtotalsize
-            while i >= rawtotalsize:
+            limit = rawtotalsize + raw_malloc_usage(size_of_addr)
+            while i >= limit:
                 llarena.arena_reserve(newpage + i, size_of_addr)
                 (newpage + i).address[0] = head
                 head = newpage + i
@@ -168,14 +187,17 @@
             self.free_lists[n] = head
             result = head - rawtotalsize
             #
-            # Done: all object locations are linked, apart from 'result',
-            # which is the first object location in the page.  Note that
-            # if the size is not a multiple of 2, there are a few wasted
-            # WORDs, which we place at the start of the page rather than
-            # at the end (Hans Boehm, xxx ref).
+            # Done: all object locations are linked, apart from
+            # 'result', which is the first object location in the page.
+            # Note that if the size is not an exact divisor of
+            # 4096-WORD, there are a few wasted WORDs, which we place at
+            # the start of the page rather than at the end (Hans Boehm,
+            # xxx ref).
             llarena.arena_reserve(result, totalsize)
             hdr = llmemory.cast_adr_to_ptr(result, lltype.Ptr(self.HDR))
-            hdr.tid = self.combine(typeid, self.current_mark)
+            hdr.typeid16 = typeid
+            hdr.mark = self.current_mark
+            hdr.flags = '\x00'
             #
             obj = result + size_gc_header
             return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
@@ -196,19 +218,23 @@
 
 
     def write_barrier(self, newvalue, addr_struct):
-        flag = self.header(addr_struct).tid & GCFLAG_MARK_TOGGLE
-        if flag != self.current_mark:
+        mark = self.header(addr_struct).mark
+        if mark != self.current_mark:
             self.force_scan(addr_struct)
 
     def _init_writebarrier_logic(self):
         #
         def force_scan(obj):
             self.mutex_lock.acquire(True)
-            if self.current_mark:
-                self.set_mark_flag(obj, GCFLAG_MARK_TOGGLE)
-            else:
-                self.set_mark_flag(obj, 0)
-            self.trace(obj, self._barrier_add_extra, None)
+            if not self.is_marked(obj, self.current_mark):
+                # it is only possible to reach this point if there is
+                # a collection running in collector_mark(), before it
+                # does mutex_lock itself.  Check this:
+                ll_assert(self.collection_running == 1,
+                          "write barrier: wrong call?")
+                #
+                self.set_mark(obj, self.current_mark)
+                self.trace(obj, self._barrier_add_extra, None)
             self.mutex_lock.release()
         #
         force_scan._dont_inline_ = True
@@ -218,24 +244,35 @@
         self.extra_objects_to_mark.append(root.address[0])
 
 
+    def wait_for_the_end_of_collection(self):
+        """In the mutator thread: wait for the collection currently
+        running (if any) to finish."""
+        if self.collection_running != 0:
+            self.acquire(self.finished_lock)
+            self.collection_running = 0
+            #
+            # Check invariants
+            ll_assert(not self.extra_objects_to_mark.non_empty(),
+                      "objs left behind in extra_objects_to_mark")
+            ll_assert(not self.gray_objects.non_empty(),
+                      "objs left behind in gray_objects")
+            #
+            # Grab the results of the last collection: read the collector's
+            # 'collect_heads/collect_tails' and merge them with the mutator's
+            # 'free_lists'.
+            n = 1
+            while n < self.pagelists_length:
+                if self.collect_tails[n] != NULL:
+                    self.collect_tails[n].address[0] = self.free_lists[n]
+                    self.free_lists[n] = self.collect_heads[n]
+                n += 1
+
+
     def collect(self, gen=0):
         """Trigger a complete collection, and wait for it to finish."""
         self.trigger_next_collection()
         self.wait_for_the_end_of_collection()
 
-    def wait_for_the_end_of_collection(self):
-        """In the mutator thread: wait for the collection currently
-        running (if any) to finish."""
-        if self.collection_running:
-            self.acquire(self.finished_lock)
-            self.collection_running = False
-            #
-            # It's possible that an object was added to 'extra_objects_to_mark'
-            # by the write barrier but not taken out by the collector thread,
-            # because it finished in the meantime.  The result is still
-            # correct, but we need to clear the list.
-            self.extra_objects_to_mark.clear()
-
     def trigger_next_collection(self):
         """In the mutator thread: triggers the next collection."""
         #
@@ -250,10 +287,19 @@
         #
         # Invert this global variable, which has the effect that on all
         # objects' state go instantly from "marked" to "non marked"
-        self.current_mark ^= GCFLAG_MARK_TOGGLE
+        self.current_mark = self.other_mark(self.current_mark)
+        #
+        # Copy a few 'mutator' fields to 'collector' fields:
+        # 'collect_pages' make linked lists of all nonfree pages at the
+        # start of the collection (unlike the 'nonfree_pages' lists, which
+        # the mutator will continue to grow).
+        n = 1
+        while n < self.pagelists_length:
+            self.collect_pages[n] = self.nonfree_pages[n]
+            n += 1
         #
         # Start the collector thread
-        self.collection_running = True
+        self.collection_running = 1
         self.ready_to_start_lock.release()
 
     def _add_stack_root(self, root):
@@ -291,23 +337,29 @@
                 #
                 # Mark
                 self.collector_mark()
+                self.collection_running = 2
                 #
                 # Sweep
                 self.collector_sweep()
+                self.finished_lock.release()
+                #
         except Exception, e:
             print 'Crash!', e.__class__.__name__, e
             self._exc_info = sys.exc_info()
 
-    @specialize.arg(2)
+    def other_mark(self, mark):
+        ll_assert(mark == MARK_VALUE_1 or mark == MARK_VALUE_2,
+                  "bad mark value")
+        return chr(ord(mark) ^ (ord(MARK_VALUE_1) ^ ord(MARK_VALUE_2)))
+
     def is_marked(self, obj, current_mark):
-        return (self.header(obj).tid & GCFLAG_MARK_TOGGLE) == current_mark
+        mark = self.header(obj).mark
+        ll_assert(mark == MARK_VALUE_1 or mark == MARK_VALUE_2,
+                  "bad mark byte in object")
+        return mark == current_mark
 
-    @specialize.arg(2)
-    def set_mark_flag(self, obj, current_mark):
-        if current_mark:
-            self.header(obj).tid |= GCFLAG_MARK_TOGGLE
-        else:
-            self.header(obj).tid &= ~GCFLAG_MARK_TOGGLE
+    def set_mark(self, obj, current_mark):
+        self.header(obj).mark = current_mark
 
     def collector_mark(self):
         while True:
@@ -315,10 +367,7 @@
             # Do marking.  The following function call is interrupted
             # if the mutator's write barrier adds new objects to
             # 'extra_objects_to_mark'.
-            if self.current_mark:
-                self._collect_mark(GCFLAG_MARK_TOGGLE)
-            else:
-                self._collect_mark(0)
+            self._collect_mark()
             #
             # Move the objects from 'extra_objects_to_mark' to
             # 'gray_objects'.  This requires the mutex lock.
@@ -339,12 +388,12 @@
             if not self.gray_objects.non_empty():
                 return
 
-    @specialize.arg(1)
-    def _collect_mark(self, current_mark):
+    def _collect_mark(self):
+        current_mark = self.current_mark
         while self.gray_objects.non_empty():
             obj = self.gray_objects.pop()
             if not self.is_marked(obj, current_mark):
-                self.set_mark_flag(obj, current_mark)
+                self.set_mark(obj, current_mark)
                 self.trace(obj, self._collect_add_pending, None)
                 #
                 # Interrupt early if the mutator's write barrier adds stuff
@@ -360,4 +409,49 @@
         self.gray_objects.append(root.address[0])
 
     def collector_sweep(self):
-        xxx
+        n = 1
+        while n < self.pagelists_length:
+            self._collect_sweep_pages(n)
+            n += 1
+
+    def _collect_sweep_pages(self, n):
+        # sweep all pages from the linked list starting at 'page',
+        # containing objects of fixed size 'object_size'.
+        page = self.collect_pages[n]
+        object_size = n << WORD_POWER_2
+        linked_list = NULL
+        first_freed_object = NULL
+        nonmarked = self.other_mark(self.current_mark)
+        while page != llmemory.NULL:
+            i = self.page_size - object_size
+            limit = raw_malloc_usage(size_of_addr)
+            while i >= limit:
+                hdr = page + i
+                #
+                if maybe_read_mark_byte(hdr) == nonmarked:
+                    # the location contains really an object (and is not just
+                    # part of a linked list of free locations), and moreover
+                    # the object is still not marked.  Free it by inserting
+                    # it into the linked list.
+                    llarena.arena_reset(hdr, object_size, 0)
+                    llarena.arena_reserve(hdr, size_of_addr)
+                    hdr.address[0] = linked_list
+                    linked_list = hdr
+                    if first_freed_object == NULL:
+                        first_freed_object = hdr
+                    # XXX detect when the whole page is freed again
+                #
+                i -= object_size
+            #
+            page = page.address[0]
+        #
+        self.collect_heads[n] = linked_list
+        self.collect_tails[n] = first_freed_object
+
+
+def maybe_read_mark_byte(addr):
+    "NOT_RPYTHON"
+    try:
+        return addr.ptr.mark
+    except AttributeError:
+        return '\x00'

From noreply at buildbot.pypy.org  Fri Oct  7 16:17:34 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  7 Oct 2011 16:17:34 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Do something about
	prebuilt GC objects.
Message-ID: <20111007141734.6C8C082A02@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r47856:ba1f195ed5fe
Date: 2011-10-07 16:16 +0200
http://bitbucket.org/pypy/pypy/changeset/ba1f195ed5fe/

Log:	Do something about prebuilt GC objects.

diff --git a/pypy/rpython/memory/gc/concurrentms.py b/pypy/rpython/memory/gc/concurrentms.py
--- a/pypy/rpython/memory/gc/concurrentms.py
+++ b/pypy/rpython/memory/gc/concurrentms.py
@@ -36,6 +36,7 @@
 
 MARK_VALUE_1      = 'M'     #  77, 0x4D
 MARK_VALUE_2      = 'k'     # 107, 0x6B
+MARK_VALUE_STATIC = 'S'     #  83, 0x53
 GCFLAG_WITH_HASH  = 0x01
 
 
@@ -102,6 +103,7 @@
         self.mutex_lock = ll_thread.allocate_lock()
         self.gray_objects = self.AddressStack()
         self.extra_objects_to_mark = self.AddressStack()
+        self.prebuilt_root_objects = self.AddressStack()
         #
         # Write barrier: actually a deletion barrier, triggered when there
         # is a collection running and the mutator tries to change an object
@@ -127,6 +129,13 @@
     def get_type_id(self, obj):
         return self.header(obj).typeid16
 
+    def init_gc_object_immortal(self, addr, typeid, flags=0):
+        # 'flags' is ignored here
+        hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR))
+        hdr.typeid16 = typeid
+        hdr.mark = MARK_VALUE_STATIC
+        hdr.flags = '\x00'
+
     def malloc_fixedsize_clear(self, typeid, size,
                                needs_finalizer=False, contains_weakptr=False):
         assert not needs_finalizer  # XXX
@@ -226,15 +235,29 @@
         #
         def force_scan(obj):
             self.mutex_lock.acquire(True)
-            if not self.is_marked(obj, self.current_mark):
-                # it is only possible to reach this point if there is
-                # a collection running in collector_mark(), before it
-                # does mutex_lock itself.  Check this:
-                ll_assert(self.collection_running == 1,
-                          "write barrier: wrong call?")
+            mark = self.header(obj).mark
+            if mark != self.current_mark:
                 #
-                self.set_mark(obj, self.current_mark)
-                self.trace(obj, self._barrier_add_extra, None)
+                if mark == MARK_VALUE_STATIC:
+                    # This is the first write into a prebuilt GC object.
+                    # Record it in 'prebuilt_root_objects'.  Even if a
+                    # collection marking phase is running now, we can
+                    # ignore this object, because at the snapshot-at-the-
+                    # beginning it didn't contain any pointer to non-
+                    # prebuilt objects.
+                    self.prebuilt_root_objects.append(obj)
+                    self.set_mark(obj, self.current_mark)
+                    #
+                else:
+                    # it is only possible to reach this point if there is
+                    # a collection running in collector_mark(), before it
+                    # does mutex_lock itself.  Check this:
+                    ll_assert(self.collection_running == 1,
+                              "write barrier: wrong call?")
+                    #
+                    self.set_mark(obj, self.current_mark)
+                    self.trace(obj, self._barrier_add_extra, None)
+                #
             self.mutex_lock.release()
         #
         force_scan._dont_inline_ = True
@@ -285,6 +308,9 @@
             MostlyConcurrentMarkSweepGC._add_stack_root,  # in prebuilt non-gc
             None)                         # static in prebuilt gc
         #
+        # Add the prebuilt root objects that have been written to
+        self.prebuilt_root_objects.foreach(self._add_prebuilt_root, None)
+        #
         # Invert this global variable, which has the effect that on all
         # objects' state go instantly from "marked" to "non marked"
         self.current_mark = self.other_mark(self.current_mark)
@@ -306,6 +332,9 @@
         obj = root.address[0]
         self.gray_objects.append(obj)
 
+    def _add_prebuilt_root(self, obj, ignored):
+        self.gray_objects.append(obj)
+
     def acquire(self, lock):
         if we_are_translated():
             lock.acquire(True)
@@ -354,7 +383,7 @@
 
     def is_marked(self, obj, current_mark):
         mark = self.header(obj).mark
-        ll_assert(mark == MARK_VALUE_1 or mark == MARK_VALUE_2,
+        ll_assert(mark in (MARK_VALUE_1, MARK_VALUE_2, MARK_VALUE_STATIC),
                   "bad mark byte in object")
         return mark == current_mark
 

From noreply at buildbot.pypy.org  Fri Oct  7 17:00:27 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  7 Oct 2011 17:00:27 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: malloc_varsize.
Message-ID: <20111007150027.BBEDD82A02@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r47857:ec32c4165b61
Date: 2011-10-07 16:31 +0200
http://bitbucket.org/pypy/pypy/changeset/ec32c4165b61/

Log:	malloc_varsize.

diff --git a/pypy/rpython/memory/gc/concurrentms.py b/pypy/rpython/memory/gc/concurrentms.py
--- a/pypy/rpython/memory/gc/concurrentms.py
+++ b/pypy/rpython/memory/gc/concurrentms.py
@@ -3,7 +3,7 @@
 from pypy.rpython.lltypesystem.llmemory import raw_malloc_usage
 from pypy.rlib.objectmodel import we_are_translated, running_on_llinterp
 from pypy.rlib.debug import ll_assert
-from pypy.rlib.rarithmetic import LONG_BIT, r_uint
+from pypy.rlib.rarithmetic import ovfcheck, LONG_BIT, r_uint
 from pypy.rpython.memory.gc.base import GCBase
 from pypy.module.thread import ll_thread
 
@@ -144,7 +144,9 @@
         totalsize = size_gc_header + size
         rawtotalsize = raw_malloc_usage(totalsize)
         if rawtotalsize <= self.small_request_threshold:
-            n = (rawtotalsize + WORD - 1) >> WORD_POWER_2
+            ll_assert(rawtotalsize & (WORD - 1) == 0,
+                      "fixedsize not properly rounded")
+            n = rawtotalsize >> WORD_POWER_2
             result = self.free_lists[n]
             if result != llmemory.NULL:
                 self.free_lists[n] = result.address[0]
@@ -161,6 +163,55 @@
                 #
         return self._malloc_slowpath(typeid, size)
 
+    def malloc_varsize_clear(self, typeid, length, size, itemsize,
+                             offset_to_length):
+        size_gc_header = self.gcheaderbuilder.size_gc_header
+        nonvarsize = size_gc_header + size
+        #
+        # Compute the maximal length that makes the object still below
+        # 'small_request_threshold'.  All the following logic is usually
+        # constant-folded because size and itemsize are constants (due
+        # to inlining).
+        maxsize = self.small_request_threshold - raw_malloc_usage(nonvarsize)
+        if maxsize < 0:
+            toobig = r_uint(0)    # the nonvarsize alone is too big
+        elif raw_malloc_usage(itemsize):
+            toobig = r_uint(maxsize // raw_malloc_usage(itemsize)) + 1
+        else:
+            toobig = r_uint(sys.maxint) + 1
+
+        if r_uint(length) < r_uint(toobig):
+            # With the above checks we know now that totalsize cannot be more
+            # than 'small_request_threshold'; in particular, the + and *
+            # cannot overflow.
+            totalsize = nonvarsize + itemsize * length
+            totalsize = llarena.round_up_for_allocation(totalsize)
+            rawtotalsize = raw_malloc_usage(totalsize)
+            ll_assert(rawtotalsize & (WORD - 1) == 0,
+                      "round_up_for_allocation failed")
+            #
+            n = rawtotalsize >> WORD_POWER_2
+            result = self.free_lists[n]
+            if result != llmemory.NULL:
+                self.free_lists[n] = result.address[0]
+                #
+                llarena.arena_reset(result, size_of_addr, 0)
+                llarena.arena_reserve(result, totalsize)
+                hdr = llmemory.cast_adr_to_ptr(result, lltype.Ptr(self.HDR))
+                hdr.typeid16 = typeid
+                hdr.mark = self.current_mark
+                hdr.flags = '\x00'
+                #
+                obj = result + size_gc_header
+                (obj + offset_to_length).signed[0] = length
+                return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
+        #
+        # If the total size of the object would be larger than
+        # 'small_request_threshold', or if the free_list is empty,
+        # then allocate it externally.  We also go there if 'length'
+        # is actually negative.
+        return self._malloc_varsize_slowpath(typeid, length)
+
     def _malloc_slowpath(self, typeid, size):
         size_gc_header = self.gcheaderbuilder.size_gc_header
         totalsize = size_gc_header + size
@@ -217,6 +268,30 @@
             xxxxx
     _malloc_slowpath._dont_inline_ = True
 
+    def _malloc_varsize_slowpath(self, typeid, length):
+        #
+        if length < 0:
+            # negative length!  This likely comes from an overflow
+            # earlier.  We will just raise MemoryError here.
+            raise MemoryError
+        #
+        # Compute the total size, carefully checking for overflows.
+        nonvarsize = self.fixed_size(typeid)
+        itemsize = self.varsize_item_sizes(typeid)
+        try:
+            varsize = ovfcheck(itemsize * length)
+            totalsize = ovfcheck(nonvarsize + varsize)
+        except OverflowError:
+            raise MemoryError
+        #
+        result = self._malloc_slowpath(typeid, totalsize)
+        #
+        offset_to_length = self.varsize_offset_to_length(typeid)
+        obj = llmemory.cast_ptr_to_adr(result)
+        (obj + offset_to_length).signed[0] = length
+        return result
+    _malloc_varsize_slowpath._dont_inline_ = True
+
     def allocate_next_arena(self):
         # xxx for now, allocate one page at a time with the system malloc()
         page = llarena.arena_malloc(self.page_size, 2)     # zero-filled

From noreply at buildbot.pypy.org  Fri Oct  7 17:00:28 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  7 Oct 2011 17:00:28 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: In-progress.
Message-ID: <20111007150028.E7BEC82A02@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r47858:0960781272db
Date: 2011-10-07 17:00 +0200
http://bitbucket.org/pypy/pypy/changeset/0960781272db/

Log:	In-progress.

diff --git a/pypy/rpython/memory/gc/concurrentms.py b/pypy/rpython/memory/gc/concurrentms.py
--- a/pypy/rpython/memory/gc/concurrentms.py
+++ b/pypy/rpython/memory/gc/concurrentms.py
@@ -71,15 +71,32 @@
         self.free_pages = NULL
         self.pagelists_length = small_request_threshold // WORD + 1
         #
+        # The following are arrays of 36 linked lists: the linked lists
+        # at indices 1 to 35 correspond to pages that store objects of
+        # size  1 * WORD  to  35 * WORD,  and the linked list at index 0
+        # is a list of all larger objects.
         def list_of_addresses_per_small_size():
             return lltype.malloc(rffi.CArray(llmemory.Address),
                                  self.pagelists_length, flavor='raw',
                                  zero=True, immortal=True)
+        # 1-35: a linked list of all pages; 0: a linked list of all larger objs
         self.nonfree_pages = list_of_addresses_per_small_size()
+        # a snapshot of 'nonfree_pages' done when the collection starts
         self.collect_pages = list_of_addresses_per_small_size()
+        # 1-35: free list of non-allocated locations; 0: unused
         self.free_lists    = list_of_addresses_per_small_size()
+        # 1-35: head and tail of the free list built by the collector thread
         self.collect_heads = list_of_addresses_per_small_size()
         self.collect_tails = list_of_addresses_per_small_size()
+        #
+        # The following character is either MARK_VALUE_1 or MARK_VALUE_2,
+        # and represents the character that must be in the 'mark' field
+        # of an object header in order for the object to be considered as
+        # marked.  Objects whose 'mark' field have the opposite value are
+        # not marked yet; the collector thread will mark them if they are
+        # still alive, or sweep them away if they are not reachable.
+        # The special value MARK_VALUE_STATIC is initially used in the
+        # 'mark' field of static prebuilt GC objects.
         self.current_mark = MARK_VALUE_1
         #
         # When the mutator thread wants to trigger the next collection,
@@ -253,19 +270,33 @@
             # 4096-WORD, there are a few wasted WORDs, which we place at
             # the start of the page rather than at the end (Hans Boehm,
             # xxx ref).
-            llarena.arena_reserve(result, totalsize)
-            hdr = llmemory.cast_adr_to_ptr(result, lltype.Ptr(self.HDR))
-            hdr.typeid16 = typeid
-            hdr.mark = self.current_mark
-            hdr.flags = '\x00'
-            #
-            obj = result + size_gc_header
-            return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
             #
         else:
-            # Case 2: the object is too big, so allocate it directly
-            # with the system malloc().
-            xxxxx
+            # Case 2: the object is too large, so allocate it directly
+            # with the system malloc().  XXX on 32-bit, we should prefer
+            # 64-bit alignment of the object
+            try:
+                rawtotalsize = ovfcheck(raw_malloc_usage(size_of_addr) +
+                                        rawtotalsize)
+            except OverflowError:
+                raise MemoryError
+            block = llarena.arena_malloc(rawtotalsize, 2)
+            if not block:
+                raise MemoryError
+            llarena.arena_reserve(block, size_of_addr)
+            block.address[0] = self.free_lists[0]
+            self.free_lists[0] = block
+            result = block + size_of_addr
+        #
+        llarena.arena_reserve(result, totalsize)
+        hdr = llmemory.cast_adr_to_ptr(result, lltype.Ptr(self.HDR))
+        hdr.typeid16 = typeid
+        hdr.mark = self.current_mark
+        hdr.flags = '\x00'
+        #
+        obj = result + size_gc_header
+        return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
+        #
     _malloc_slowpath._dont_inline_ = True
 
     def _malloc_varsize_slowpath(self, typeid, length):
@@ -295,7 +326,8 @@
     def allocate_next_arena(self):
         # xxx for now, allocate one page at a time with the system malloc()
         page = llarena.arena_malloc(self.page_size, 2)     # zero-filled
-        ll_assert(bool(page), "out of memory!")
+        if not page:
+            raise MemoryError
         llarena.arena_reserve(page, size_of_addr)
         page.address[0] = NULL
         self.free_pages = page
@@ -394,7 +426,7 @@
         # 'collect_pages' make linked lists of all nonfree pages at the
         # start of the collection (unlike the 'nonfree_pages' lists, which
         # the mutator will continue to grow).
-        n = 1
+        n = 0
         while n < self.pagelists_length:
             self.collect_pages[n] = self.nonfree_pages[n]
             n += 1
@@ -544,6 +576,11 @@
                     if first_freed_object == NULL:
                         first_freed_object = hdr
                     # XXX detect when the whole page is freed again
+                    #
+                    # Clear the data, in prevision for the following
+                    # malloc_fixedsize_clear().
+                    llarena.arena_reset(hdr + size_of_addr,
+                        object_size - raw_malloc_usage(size_of_addr), 2)
                 #
                 i -= object_size
             #

From noreply at buildbot.pypy.org  Fri Oct  7 17:17:17 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  7 Oct 2011 17:17:17 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Also try to sweep the
	large objects.
Message-ID: <20111007151717.22F4182A02@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r47859:513203521539
Date: 2011-10-07 17:17 +0200
http://bitbucket.org/pypy/pypy/changeset/513203521539/

Log:	Also try to sweep the large objects.

diff --git a/pypy/rpython/memory/gc/concurrentms.py b/pypy/rpython/memory/gc/concurrentms.py
--- a/pypy/rpython/memory/gc/concurrentms.py
+++ b/pypy/rpython/memory/gc/concurrentms.py
@@ -86,6 +86,7 @@
         # 1-35: free list of non-allocated locations; 0: unused
         self.free_lists    = list_of_addresses_per_small_size()
         # 1-35: head and tail of the free list built by the collector thread
+        # 0: head and tail of the linked list of surviving large objects
         self.collect_heads = list_of_addresses_per_small_size()
         self.collect_tails = list_of_addresses_per_small_size()
         #
@@ -136,12 +137,16 @@
 
     def _teardown(self):
         "NOT_RPYTHON.  Stop the collector thread after tests have run."
+        if self._teardown_now:
+            return
         self.wait_for_the_end_of_collection()
         #
         # start the next collection, but with "stop" in _teardown_now,
         # which should shut down the collector thread
         self._teardown_now.append("stop")
-        self.collect()
+        self.ready_to_start_lock.release()
+        self.acquire(self.finished_lock)
+        del self.ready_to_start_lock, self.finished_lock
 
     def get_type_id(self, obj):
         return self.header(obj).typeid16
@@ -390,7 +395,7 @@
             # Grab the results of the last collection: read the collector's
             # 'collect_heads/collect_tails' and merge them with the mutator's
             # 'free_lists'.
-            n = 1
+            n = 0
             while n < self.pagelists_length:
                 if self.collect_tails[n] != NULL:
                     self.collect_tails[n].address[0] = self.free_lists[n]
@@ -545,18 +550,40 @@
         self.gray_objects.append(root.address[0])
 
     def collector_sweep(self):
+        self._collect_sweep_large_objects()
         n = 1
         while n < self.pagelists_length:
             self._collect_sweep_pages(n)
             n += 1
 
+    def _collect_sweep_large_objects(self):
+        block = self.collect_pages[0]
+        nonmarked = self.other_mark(self.current_mark)
+        linked_list = NULL
+        first_block_in_linked_list = NULL
+        while block != llmemory.NULL:
+            hdr = block + size_of_addr
+            if maybe_read_mark_byte(hdr) == nonmarked:
+                # the object is still not marked.  Free it.
+                llarena.arena_free(block)
+                #
+            else:
+                # the object was marked: relink it
+                block.address[0] = linked_list
+                linked_list = block
+                if first_block_in_linked_list == NULL:
+                    first_block_in_linked_list = block
+        #
+        self.collect_heads[0] = linked_list
+        self.collect_tails[0] = first_block_in_linked_list
+
     def _collect_sweep_pages(self, n):
         # sweep all pages from the linked list starting at 'page',
         # containing objects of fixed size 'object_size'.
         page = self.collect_pages[n]
         object_size = n << WORD_POWER_2
         linked_list = NULL
-        first_freed_object = NULL
+        first_loc_in_linked_list = NULL
         nonmarked = self.other_mark(self.current_mark)
         while page != llmemory.NULL:
             i = self.page_size - object_size
@@ -573,8 +600,8 @@
                     llarena.arena_reserve(hdr, size_of_addr)
                     hdr.address[0] = linked_list
                     linked_list = hdr
-                    if first_freed_object == NULL:
-                        first_freed_object = hdr
+                    if first_loc_in_linked_list == NULL:
+                        first_loc_in_linked_list = hdr
                     # XXX detect when the whole page is freed again
                     #
                     # Clear the data, in prevision for the following
@@ -587,7 +614,7 @@
             page = page.address[0]
         #
         self.collect_heads[n] = linked_list
-        self.collect_tails[n] = first_freed_object
+        self.collect_tails[n] = first_loc_in_linked_list
 
 
 def maybe_read_mark_byte(addr):

From noreply at buildbot.pypy.org  Fri Oct  7 17:20:07 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Fri,  7 Oct 2011 17:20:07 +0200 (CEST)
Subject: [pypy-commit] pypy default: Clean some random test code up a bit.
Message-ID: <20111007152007.5038E82A02@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r47860:339919452175
Date: 2011-10-07 11:19 -0400
http://bitbucket.org/pypy/pypy/changeset/339919452175/

Log:	Clean some random test code up a bit.

diff --git a/pypy/jit/codewriter/test/test_jtransform.py b/pypy/jit/codewriter/test/test_jtransform.py
--- a/pypy/jit/codewriter/test/test_jtransform.py
+++ b/pypy/jit/codewriter/test/test_jtransform.py
@@ -1,5 +1,7 @@
 import py
+import itertools
 import random
+
 from pypy.objspace.flow.model import FunctionGraph, Block, Link
 from pypy.objspace.flow.model import SpaceOperation, Variable, Constant
 from pypy.jit.codewriter.jtransform import Transformer
@@ -254,26 +256,35 @@
             assert op1.result is None
 
 def test_calls():
-    for RESTYPE in [lltype.Signed, rclass.OBJECTPTR,
-                    lltype.Float, lltype.Void]:
-      for with_void in [False, True]:
-        for with_i in [False, True]:
-          for with_r in [False, True]:
-            for with_f in [False, True]:
-              ARGS = []
-              if with_void: ARGS += [lltype.Void, lltype.Void]
-              if with_i: ARGS += [lltype.Signed, lltype.Char]
-              if with_r: ARGS += [rclass.OBJECTPTR, lltype.Ptr(rstr.STR)]
-              if with_f: ARGS += [lltype.Float, lltype.Float]
-              random.shuffle(ARGS)
-              if RESTYPE == lltype.Float: with_f = True
-              if with_f: expectedkind = 'irf'   # all kinds
-              elif with_i: expectedkind = 'ir'  # integers and references
-              else: expectedkind = 'r'          # only references
-              yield residual_call_test, ARGS, RESTYPE, expectedkind
-              yield direct_call_test, ARGS, RESTYPE, expectedkind
-              yield indirect_residual_call_test, ARGS, RESTYPE, expectedkind
-              yield indirect_regular_call_test, ARGS, RESTYPE, expectedkind
+    for RESTYPE, with_void, with_i, with_r, with_f in itertools.product(
+        [lltype.Signed, rclass.OBJECTPTR, lltype.Float, lltype.Void],
+        [False, True],
+        [False, True],
+        [False, True],
+        [False, True],
+    ):
+        ARGS = []
+        if with_void:
+            ARGS += [lltype.Void, lltype.Void]
+        if with_i:
+            ARGS += [lltype.Signed, lltype.Char]
+        if with_r:
+            ARGS += [rclass.OBJECTPTR, lltype.Ptr(rstr.STR)]
+        if with_f:
+            ARGS += [lltype.Float, lltype.Float]
+        random.shuffle(ARGS)
+        if RESTYPE == lltype.Float:
+            with_f = True
+        if with_f:
+            expectedkind = 'irf'   # all kinds
+        elif with_i:
+            expectedkind = 'ir'  # integers and references
+        else:
+            expectedkind = 'r'          # only references
+        yield residual_call_test, ARGS, RESTYPE, expectedkind
+        yield direct_call_test, ARGS, RESTYPE, expectedkind
+        yield indirect_residual_call_test, ARGS, RESTYPE, expectedkind
+        yield indirect_regular_call_test, ARGS, RESTYPE, expectedkind
 
 def get_direct_call_op(argtypes, restype):
     FUNC = lltype.FuncType(argtypes, restype)

From noreply at buildbot.pypy.org  Fri Oct  7 17:26:48 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Fri,  7 Oct 2011 17:26:48 +0200 (CEST)
Subject: [pypy-commit] pypy default: make it work on 2.5
Message-ID: <20111007152648.8E89282A02@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r47861:a74c03d6d1fb
Date: 2011-10-07 11:26 -0400
http://bitbucket.org/pypy/pypy/changeset/a74c03d6d1fb/

Log:	make it work on 2.5

diff --git a/pypy/jit/codewriter/test/test_jtransform.py b/pypy/jit/codewriter/test/test_jtransform.py
--- a/pypy/jit/codewriter/test/test_jtransform.py
+++ b/pypy/jit/codewriter/test/test_jtransform.py
@@ -1,6 +1,18 @@
 import py
-import itertools
 import random
+try:
+    from itertools import product
+except ImportError:
+    # Python 2.5, this is taken from the CPython docs, but simplified.
+    def product(*args):
+        # product('ABCD', 'xy') --> Ax Ay Bx By Cx Cy Dx Dy
+        # product(range(2), repeat=3) --> 000 001 010 011 100 101 110 111
+        pools = map(tuple, args)
+        result = [[]]
+        for pool in pools:
+            result = [x+[y] for x in result for y in pool]
+        for prod in result:
+            yield tuple(prod)
 
 from pypy.objspace.flow.model import FunctionGraph, Block, Link
 from pypy.objspace.flow.model import SpaceOperation, Variable, Constant
@@ -256,7 +268,7 @@
             assert op1.result is None
 
 def test_calls():
-    for RESTYPE, with_void, with_i, with_r, with_f in itertools.product(
+    for RESTYPE, with_void, with_i, with_r, with_f in product(
         [lltype.Signed, rclass.OBJECTPTR, lltype.Float, lltype.Void],
         [False, True],
         [False, True],

From noreply at buildbot.pypy.org  Fri Oct  7 17:36:20 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  7 Oct 2011 17:36:20 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Fix.
Message-ID: <20111007153620.B3E8E82A02@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r47862:da95bbaf2858
Date: 2011-10-07 17:33 +0200
http://bitbucket.org/pypy/pypy/changeset/da95bbaf2858/

Log:	Fix.

diff --git a/pypy/rpython/memory/gc/concurrentms.py b/pypy/rpython/memory/gc/concurrentms.py
--- a/pypy/rpython/memory/gc/concurrentms.py
+++ b/pypy/rpython/memory/gc/concurrentms.py
@@ -289,8 +289,8 @@
             if not block:
                 raise MemoryError
             llarena.arena_reserve(block, size_of_addr)
-            block.address[0] = self.free_lists[0]
-            self.free_lists[0] = block
+            block.address[0] = self.nonfree_pages[0]
+            self.nonfree_pages[0] = block
             result = block + size_of_addr
         #
         llarena.arena_reserve(result, totalsize)
@@ -395,12 +395,17 @@
             # Grab the results of the last collection: read the collector's
             # 'collect_heads/collect_tails' and merge them with the mutator's
             # 'free_lists'.
-            n = 0
+            n = 1
             while n < self.pagelists_length:
                 if self.collect_tails[n] != NULL:
                     self.collect_tails[n].address[0] = self.free_lists[n]
                     self.free_lists[n] = self.collect_heads[n]
                 n += 1
+            #
+            # Do the same with 'collect_heads[0]/collect_tails[0]'.
+            if self.collect_tails[0] != NULL:
+                self.collect_tails[0].address[0] = self.nonfree_pages[0]
+                self.nonfree_pages[0] = self.collect_heads[0]
 
 
     def collect(self, gen=0):

From noreply at buildbot.pypy.org  Fri Oct  7 17:36:21 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  7 Oct 2011 17:36:21 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Fix.
Message-ID: <20111007153621.DDA5682A02@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r47863:8d18b3b7104d
Date: 2011-10-07 17:34 +0200
http://bitbucket.org/pypy/pypy/changeset/8d18b3b7104d/

Log:	Fix.

diff --git a/pypy/rpython/memory/gc/concurrentms.py b/pypy/rpython/memory/gc/concurrentms.py
--- a/pypy/rpython/memory/gc/concurrentms.py
+++ b/pypy/rpython/memory/gc/concurrentms.py
@@ -567,6 +567,7 @@
         linked_list = NULL
         first_block_in_linked_list = NULL
         while block != llmemory.NULL:
+            nextblock = block.address[0]
             hdr = block + size_of_addr
             if maybe_read_mark_byte(hdr) == nonmarked:
                 # the object is still not marked.  Free it.
@@ -578,6 +579,7 @@
                 linked_list = block
                 if first_block_in_linked_list == NULL:
                     first_block_in_linked_list = block
+            block = nextblock
         #
         self.collect_heads[0] = linked_list
         self.collect_tails[0] = first_block_in_linked_list

From noreply at buildbot.pypy.org  Fri Oct  7 17:56:26 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  7 Oct 2011 17:56:26 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Start test_gc.
Message-ID: <20111007155626.9FD6082A02@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r47864:0ad357ff8c34
Date: 2011-10-07 17:39 +0200
http://bitbucket.org/pypy/pypy/changeset/0ad357ff8c34/

Log:	Start test_gc.

diff --git a/pypy/rpython/memory/gc/concurrentms.py b/pypy/rpython/memory/gc/concurrentms.py
--- a/pypy/rpython/memory/gc/concurrentms.py
+++ b/pypy/rpython/memory/gc/concurrentms.py
@@ -343,6 +343,13 @@
         if mark != self.current_mark:
             self.force_scan(addr_struct)
 
+    def writebarrier_before_copy(self, source_addr, dest_addr,
+                                 source_start, dest_start, length):
+        mark = self.header(dest_addr).mark
+        if mark != self.current_mark:
+            self.force_scan(dest_addr)
+        return True
+
     def _init_writebarrier_logic(self):
         #
         def force_scan(obj):
diff --git a/pypy/rpython/memory/test/test_gc.py b/pypy/rpython/memory/test/test_gc.py
--- a/pypy/rpython/memory/test/test_gc.py
+++ b/pypy/rpython/memory/test/test_gc.py
@@ -926,3 +926,12 @@
 
 class TestMiniMarkGCCardMarking(TestMiniMarkGC):
     GC_PARAMS = {'card_page_indices': 4}
+
+class TestMostlyConcurrentMarkSweepGC(TestMiniMarkGC):
+    from pypy.rpython.memory.gc.concurrentms \
+            import MostlyConcurrentMarkSweepGC as GCClass
+
+    def test_finalizer_order(self):
+        py.test.skip("in-progress")
+    def test_from_objwithfinalizer_to_youngobj(self):
+        py.test.skip("in-progress")

From noreply at buildbot.pypy.org  Fri Oct  7 17:56:27 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  7 Oct 2011 17:56:27 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Fix.
Message-ID: <20111007155627.CBF6E82A02@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r47865:4546a546bf69
Date: 2011-10-07 17:55 +0200
http://bitbucket.org/pypy/pypy/changeset/4546a546bf69/

Log:	Fix.

diff --git a/pypy/rpython/memory/gc/concurrentms.py b/pypy/rpython/memory/gc/concurrentms.py
--- a/pypy/rpython/memory/gc/concurrentms.py
+++ b/pypy/rpython/memory/gc/concurrentms.py
@@ -26,6 +26,7 @@
 WORD_POWER_2 = {32: 2, 64: 3}[LONG_BIT]
 assert 1 << WORD_POWER_2 == WORD
 size_of_addr = llmemory.sizeof(llmemory.Address)
+MAXIMUM_SIZE = sys.maxint - (2*WORD-1)
 
 # XXX assumes little-endian machines for now: the byte at offset 0 in
 # the object is either a mark byte (equal to an odd value), or if the
@@ -168,6 +169,7 @@
         if rawtotalsize <= self.small_request_threshold:
             ll_assert(rawtotalsize & (WORD - 1) == 0,
                       "fixedsize not properly rounded")
+            #
             n = rawtotalsize >> WORD_POWER_2
             result = self.free_lists[n]
             if result != llmemory.NULL:
@@ -235,9 +237,14 @@
         return self._malloc_varsize_slowpath(typeid, length)
 
     def _malloc_slowpath(self, typeid, size):
+        # Slow-path malloc.  Call this with 'size' being a valid and
+        # rounded number, between WORD and up to MAXIMUM_SIZE.
         size_gc_header = self.gcheaderbuilder.size_gc_header
         totalsize = size_gc_header + size
         rawtotalsize = raw_malloc_usage(totalsize)
+        ll_assert(rawtotalsize & (WORD - 1) == 0,
+                  "malloc_slowpath: non-rounded size")
+        #
         if rawtotalsize <= self.small_request_threshold:
             #
             # Case 1: we have run out of the free list corresponding to
@@ -250,7 +257,7 @@
             #
             # Put the free page in the list 'nonfree_pages[n]'.  This is
             # a linked list chained through the first word of each page.
-            n = (rawtotalsize + WORD - 1) >> WORD_POWER_2
+            n = rawtotalsize >> WORD_POWER_2
             newpage.address[0] = self.nonfree_pages[n]
             self.nonfree_pages[n] = newpage
             #
@@ -280,11 +287,7 @@
             # Case 2: the object is too large, so allocate it directly
             # with the system malloc().  XXX on 32-bit, we should prefer
             # 64-bit alignment of the object
-            try:
-                rawtotalsize = ovfcheck(raw_malloc_usage(size_of_addr) +
-                                        rawtotalsize)
-            except OverflowError:
-                raise MemoryError
+            rawtotalsize += raw_malloc_usage(size_of_addr)
             block = llarena.arena_malloc(rawtotalsize, 2)
             if not block:
                 raise MemoryError
@@ -320,6 +323,11 @@
         except OverflowError:
             raise MemoryError
         #
+        # Detect very rare cases of overflows
+        if raw_malloc_usage(totalsize) > MAXIMUM_SIZE:
+            raise MemoryError("rare case of overflow")
+        #
+        totalsize = llarena.round_up_for_allocation(totalsize)
         result = self._malloc_slowpath(typeid, totalsize)
         #
         offset_to_length = self.varsize_offset_to_length(typeid)

From noreply at buildbot.pypy.org  Fri Oct  7 17:56:29 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Fri,  7 Oct 2011 17:56:29 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Use a real thread-local
 storage in lltype. Needed to run the
Message-ID: <20111007155629.0C8E682A02@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r47866:08367abdb6d8
Date: 2011-10-07 17:56 +0200
http://bitbucket.org/pypy/pypy/changeset/08367abdb6d8/

Log:	Use a real thread-local storage in lltype. Needed to run the tests
	of the concurrent GC.

diff --git a/pypy/rpython/lltypesystem/ll2ctypes.py b/pypy/rpython/lltypesystem/ll2ctypes.py
--- a/pypy/rpython/lltypesystem/ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/ll2ctypes.py
@@ -27,11 +27,6 @@
 from pypy.rpython import raddress
 from pypy.translator.platform import platform
 from array import array
-try:
-    from thread import _local as tlsobject
-except ImportError:
-    class tlsobject(object):
-        pass
 
 # ____________________________________________________________
 
@@ -1339,7 +1334,7 @@
 # call to the next.  Otherwise a non-zero value left behind will confuse
 # CPython itself a bit later, and/or CPython will stamp on it before we
 # try to inspect it via rposix.get_errno().
-TLS = tlsobject()
+TLS = lltype.tlsobject()
 
 # helpers to save/restore the C-level errno -- platform-specific because
 # ctypes doesn't just do the right thing and expose it directly :-(
diff --git a/pypy/rpython/lltypesystem/lltype.py b/pypy/rpython/lltypesystem/lltype.py
--- a/pypy/rpython/lltypesystem/lltype.py
+++ b/pypy/rpython/lltypesystem/lltype.py
@@ -10,10 +10,13 @@
 from sys import maxint
 import weakref
 
-class State(object):
-    pass
+try:
+    from thread import _local as tlsobject
+except ImportError:
+    class tlsobject(object):
+        pass
 
-TLS = State()
+TLS = tlsobject()
 
 class WeakValueDictionary(weakref.WeakValueDictionary):
     """A subclass of weakref.WeakValueDictionary

From noreply at buildbot.pypy.org  Fri Oct  7 22:37:40 2011
From: noreply at buildbot.pypy.org (justinpeel)
Date: Fri,  7 Oct 2011 22:37:40 +0200 (CEST)
Subject: [pypy-commit] pypy rgc-mem-pressure: Branch to add manual memory
 pressure. Useful for mallocing OpaquePtrs.
Message-ID: <20111007203740.8D8DF82A02@wyvern.cs.uni-duesseldorf.de>

Author: Justin Peel <notmuchtotell at gmail.com>
Branch: rgc-mem-pressure
Changeset: r47867:d78305bced3f
Date: 2011-10-07 14:37 -0600
http://bitbucket.org/pypy/pypy/changeset/d78305bced3f/

Log:	Branch to add manual memory pressure. Useful for mallocing
	OpaquePtrs.

diff --git a/pypy/rlib/rgc.py b/pypy/rlib/rgc.py
--- a/pypy/rlib/rgc.py
+++ b/pypy/rlib/rgc.py
@@ -255,6 +255,10 @@
     except Exception:
         return False      # don't keep objects whose _freeze_() method explodes
 
+def add_memory_pressure(estimate):
+    """Add memory pressure for OpaquePtrs."""
+    pass
+
 def get_rpy_memory_usage(gcref):
     "NOT_RPYTHON"
     # approximate implementation using CPython's type info
diff --git a/pypy/rpython/llinterp.py b/pypy/rpython/llinterp.py
--- a/pypy/rpython/llinterp.py
+++ b/pypy/rpython/llinterp.py
@@ -172,7 +172,7 @@
 
 def checkadr(addr):
     assert lltype.typeOf(addr) is llmemory.Address
-    
+
 def is_inst(inst):
     return isinstance(lltype.typeOf(inst), (ootype.Instance, ootype.BuiltinType, ootype.StaticMethod))
 
@@ -657,7 +657,7 @@
                 raise TypeError("graph with %r args called with wrong func ptr type: %r" %
                                 (tuple([v.concretetype for v in args_v]), ARGS)) 
         frame = self.newsubframe(graph, args)
-        return frame.eval()        
+        return frame.eval()
 
     def op_direct_call(self, f, *args):
         FTYPE = self.llinterpreter.typer.type_system.derefType(lltype.typeOf(f))
@@ -698,13 +698,13 @@
             return ptr
         except MemoryError:
             self.make_llexception()
-            
+
     def op_malloc_nonmovable(self, TYPE, flags):
         flavor = flags['flavor']
         assert flavor == 'gc'
         zero = flags.get('zero', False)
         return self.heap.malloc_nonmovable(TYPE, zero=zero)
-        
+
     def op_malloc_nonmovable_varsize(self, TYPE, flags, size):
         flavor = flags['flavor']
         assert flavor == 'gc'
@@ -716,6 +716,9 @@
         track_allocation = flags.get('track_allocation', True)
         self.heap.free(obj, flavor='raw', track_allocation=track_allocation)
 
+    def add_memory_pressure(self, size):
+        self.heap.add_memory_pressure(size)
+
     def op_shrink_array(self, obj, smallersize):
         return self.heap.shrink_array(obj, smallersize)
 
@@ -1325,7 +1328,7 @@
         func_graph = fn.graph
     else:
         # obj is an instance, we want to call 'method_name' on it
-        assert fn is None        
+        assert fn is None
         self_arg = [obj]
         func_graph = obj._TYPE._methods[method_name._str].graph
 
diff --git a/pypy/rpython/lltypesystem/llheap.py b/pypy/rpython/lltypesystem/llheap.py
--- a/pypy/rpython/lltypesystem/llheap.py
+++ b/pypy/rpython/lltypesystem/llheap.py
@@ -5,8 +5,7 @@
 
 setfield = setattr
 from operator import setitem as setarrayitem
-from pypy.rlib.rgc import collect
-from pypy.rlib.rgc import can_move
+from pypy.rlib.rgc import can_move, collect, add_memory_pressure
 
 def setinterior(toplevelcontainer, inneraddr, INNERTYPE, newvalue,
                 offsets=None):
diff --git a/pypy/rpython/lltypesystem/lloperation.py b/pypy/rpython/lltypesystem/lloperation.py
--- a/pypy/rpython/lltypesystem/lloperation.py
+++ b/pypy/rpython/lltypesystem/lloperation.py
@@ -519,6 +519,7 @@
     'decode_arg_def':       LLOp(canraise=(Exception,)),
     'getslice':             LLOp(canraise=(Exception,)),
     'check_and_clear_exc':  LLOp(),
+    'add_memory_pressure':  LLOp(),
 
     # __________ debugging __________
     'debug_view':           LLOp(),
diff --git a/pypy/rpython/memory/gctransform/framework.py b/pypy/rpython/memory/gctransform/framework.py
--- a/pypy/rpython/memory/gctransform/framework.py
+++ b/pypy/rpython/memory/gctransform/framework.py
@@ -385,6 +385,14 @@
                 [annmodel.SomeInteger(), annmodel.SomeInteger()],
                 annmodel.s_None, minimal_transform = False)
 
+        if getattr(GCClass, 'add_memory_pressure', False):
+            def add_memory_pressure(size):
+                gcdata.gc.raw_malloc_memory_pressure(size)
+            self.add_memory_pressure_ptr = getfun(
+                add_memory_pressure,
+                [annmodel.SomeInteger()],
+                annmodel.s_None, minimal_transform = False)
+
         self.identityhash_ptr = getfn(GCClass.identityhash.im_func,
                                       [s_gc, s_gcref],
                                       annmodel.SomeInteger(),
diff --git a/pypy/rpython/memory/gctransform/transform.py b/pypy/rpython/memory/gctransform/transform.py
--- a/pypy/rpython/memory/gctransform/transform.py
+++ b/pypy/rpython/memory/gctransform/transform.py
@@ -63,7 +63,7 @@
                     gct.push_alive(v_result, self.llops)
             elif opname not in ('direct_call', 'indirect_call'):
                 gct.push_alive(v_result, self.llops)
-        
+
 
 
     def rename(self, newopname):
@@ -118,7 +118,7 @@
             self.minimalgctransformer = self.MinimalGCTransformer(self)
         else:
             self.minimalgctransformer = None
-            
+
     def get_lltype_of_exception_value(self):
         if self.translator is not None:
             exceptiondata = self.translator.rtyper.getexceptiondata()
@@ -399,7 +399,7 @@
 
     def gct_gc_heap_stats(self, hop):
         from pypy.rpython.memory.gc.base import ARRAY_TYPEID_MAP
-        
+
         return hop.cast_result(rmodel.inputconst(lltype.Ptr(ARRAY_TYPEID_MAP),
                                         lltype.nullptr(ARRAY_TYPEID_MAP)))
 
@@ -427,7 +427,7 @@
         assert flavor == 'raw'
         assert not flags.get('zero')
         return self.parenttransformer.gct_malloc_varsize(hop)
-    
+
     def gct_free(self, hop):
         flags = hop.spaceop.args[1].value
         flavor = flags['flavor']
@@ -502,7 +502,7 @@
         stack_mh = mallocHelpers()
         stack_mh.allocate = lambda size: llop.stack_malloc(llmemory.Address, size)
         ll_stack_malloc_fixedsize = stack_mh._ll_malloc_fixedsize
-        
+
         if self.translator:
             self.raw_malloc_fixedsize_ptr = self.inittime_helper(
                 ll_raw_malloc_fixedsize, [lltype.Signed], llmemory.Address)
@@ -541,7 +541,7 @@
                           resulttype=llmemory.Address)
         if flags.get('zero'):
             hop.genop("raw_memclear", [v_raw, c_size])
-        return v_raw        
+        return v_raw
 
     def gct_malloc_varsize(self, hop, add_flags=None):
         flags = hop.spaceop.args[1].value
@@ -559,6 +559,14 @@
     def gct_malloc_nonmovable_varsize(self, *args, **kwds):
         return self.gct_malloc_varsize(*args, **kwds)
 
+    def gct_add_memory_pressure(self, hop):
+        if hasattr(self, 'add_memory_pressure_ptr'):
+            op = hop.spaceop
+            size = hop.args[0]
+            hop.genop("direct_call",
+                          [self.raw_malloc_memory_pressure_ptr,
+                           size])
+
     def varsize_malloc_helper(self, hop, flags, meth, extraargs):
         def intconst(c): return rmodel.inputconst(lltype.Signed, c)
         op = hop.spaceop
@@ -625,7 +633,7 @@
                 hop.genop("track_alloc_stop", [v])
             hop.genop('raw_free', [v])
         else:
-            assert False, "%s has no support for free with flavor %r" % (self, flavor)           
+            assert False, "%s has no support for free with flavor %r" % (self, flavor)
 
     def gct_gc_can_move(self, hop):
         return hop.cast_result(rmodel.inputconst(lltype.Bool, False))
diff --git a/pypy/rpython/memory/gcwrapper.py b/pypy/rpython/memory/gcwrapper.py
--- a/pypy/rpython/memory/gcwrapper.py
+++ b/pypy/rpython/memory/gcwrapper.py
@@ -65,6 +65,9 @@
             gctypelayout.zero_gc_pointers(result)
         return result
 
+    def add_memory_pressure(self, size):
+        self.gc.add_memory_pressure(self)
+
     def shrink_array(self, p, smallersize):
         if hasattr(self.gc, 'shrink_array'):
             addr = llmemory.cast_ptr_to_adr(p)
diff --git a/pypy/rpython/memory/test/test_gc.py b/pypy/rpython/memory/test/test_gc.py
--- a/pypy/rpython/memory/test/test_gc.py
+++ b/pypy/rpython/memory/test/test_gc.py
@@ -593,7 +593,7 @@
             return rgc.can_move(lltype.malloc(TP, 1))
         assert self.interpret(func, []) == self.GC_CAN_MOVE
 
-    
+
     def test_malloc_nonmovable(self):
         TP = lltype.GcArray(lltype.Char)
         def func():
diff --git a/pypy/rpython/memory/test/test_transformed_gc.py b/pypy/rpython/memory/test/test_transformed_gc.py
--- a/pypy/rpython/memory/test/test_transformed_gc.py
+++ b/pypy/rpython/memory/test/test_transformed_gc.py
@@ -27,7 +27,7 @@
     t.config.set(**extraconfigopts)
     ann = t.buildannotator(policy=annpolicy.StrictAnnotatorPolicy())
     ann.build_types(func, inputtypes)
-                                   
+
     if specialize:
         t.buildrtyper().specialize()
     if backendopt:
@@ -44,7 +44,7 @@
     GC_CAN_MOVE = False
     GC_CAN_MALLOC_NONMOVABLE = True
     taggedpointers = False
-    
+
     def setup_class(cls):
         funcs0 = []
         funcs2 = []
@@ -155,7 +155,7 @@
             return run, gct
         else:
             return run
-        
+
 class GenericGCTests(GCTest):
     GC_CAN_SHRINK_ARRAY = False
 
@@ -190,7 +190,7 @@
                     j += 1
             return 0
         return malloc_a_lot
-    
+
     def test_instances(self):
         run, statistics = self.runner("instances", statistics=True)
         run([])
@@ -276,7 +276,7 @@
         for i in range(1, 5):
             res = run([i, i - 1])
             assert res == i - 1 # crashes if constants are not considered roots
-            
+
     def define_string_concatenation(cls):
         def concat(j, dummy):
             lst = []
@@ -656,7 +656,7 @@
             #    return 2
 
         return func
-    
+
     def test_malloc_nonmovable(self):
         run = self.runner("malloc_nonmovable")
         assert int(self.GC_CAN_MALLOC_NONMOVABLE) == run([])
@@ -676,7 +676,7 @@
                 return 2
 
         return func
-    
+
     def test_malloc_nonmovable_fixsize(self):
         run = self.runner("malloc_nonmovable_fixsize")
         assert run([]) == int(self.GC_CAN_MALLOC_NONMOVABLE)
@@ -757,7 +757,7 @@
             lltype.free(idarray, flavor='raw')
             return 0
         return f
-    
+
     def test_many_ids(self):
         if not self.GC_CAN_TEST_ID:
             py.test.skip("fails for bad reasons in lltype.py :-(")
@@ -812,7 +812,7 @@
             else:
                 assert 0, "oups, not found"
         return f, None, fix_graph_of_g
-            
+
     def test_do_malloc_operations(self):
         run = self.runner("do_malloc_operations")
         run([])
@@ -848,7 +848,7 @@
             else:
                 assert 0, "oups, not found"
         return f, None, fix_graph_of_g
-        
+
     def test_do_malloc_operations_in_call(self):
         run = self.runner("do_malloc_operations_in_call")
         run([])
@@ -859,7 +859,7 @@
         l2 = []
         l3 = []
         l4 = []
-        
+
         def f():
             for i in range(10):
                 s = lltype.malloc(S)
@@ -1079,7 +1079,7 @@
             return 0
 
         return f
-        
+
     def test_many_weakrefs(self):
         run = self.runner("many_weakrefs")
         run([])
@@ -1129,7 +1129,7 @@
     def define_adr_of_nursery(cls):
         class A(object):
             pass
-        
+
         def f():
             # we need at least 1 obj to allocate a nursery
             a = A()
@@ -1145,9 +1145,9 @@
             assert nt1 > nf1
             assert nt1 == nt0
             return 0
-        
+
         return f
-    
+
     def test_adr_of_nursery(self):
         run = self.runner("adr_of_nursery")
         res = run([])        
@@ -1173,7 +1173,7 @@
             def _teardown(self):
                 self.__ready = False # collecting here is expected
                 GenerationGC._teardown(self)
-                
+
             GC_PARAMS = {'space_size': 512*WORD,
                          'nursery_size': 128*WORD,
                          'translated_to_c': False}

From noreply at buildbot.pypy.org  Fri Oct  7 23:10:41 2011
From: noreply at buildbot.pypy.org (justinpeel)
Date: Fri,  7 Oct 2011 23:10:41 +0200 (CEST)
Subject: [pypy-commit] pypy rgc-mem-pressure: change llop name
Message-ID: <20111007211041.1666582A02@wyvern.cs.uni-duesseldorf.de>

Author: Justin Peel <notmuchtotell at gmail.com>
Branch: rgc-mem-pressure
Changeset: r47868:06978e4a4126
Date: 2011-10-07 15:08 -0600
http://bitbucket.org/pypy/pypy/changeset/06978e4a4126/

Log:	change llop name

diff --git a/pypy/rpython/lltypesystem/lloperation.py b/pypy/rpython/lltypesystem/lloperation.py
--- a/pypy/rpython/lltypesystem/lloperation.py
+++ b/pypy/rpython/lltypesystem/lloperation.py
@@ -472,6 +472,7 @@
     'gc_is_rpy_instance'  : LLOp(),
     'gc_dump_rpy_heap'    : LLOp(),
     'gc_typeids_z'        : LLOp(),
+    'gc_add_memory_pressure': LLOp(),
 
     # ------- JIT & GC interaction, only for some GCs ----------
 
@@ -519,7 +520,6 @@
     'decode_arg_def':       LLOp(canraise=(Exception,)),
     'getslice':             LLOp(canraise=(Exception,)),
     'check_and_clear_exc':  LLOp(),
-    'add_memory_pressure':  LLOp(),
 
     # __________ debugging __________
     'debug_view':           LLOp(),
diff --git a/pypy/translator/c/test/test_newgc.py b/pypy/translator/c/test/test_newgc.py
--- a/pypy/translator/c/test/test_newgc.py
+++ b/pypy/translator/c/test/test_newgc.py
@@ -37,7 +37,7 @@
             else:
                 print res
             return 0
-        
+
         t = Translation(main, standalone=True, gc=cls.gcpolicy,
                         policy=annpolicy.StrictAnnotatorPolicy(),
                         taggedpointers=cls.taggedpointers,

From noreply at buildbot.pypy.org  Fri Oct  7 23:10:42 2011
From: noreply at buildbot.pypy.org (justinpeel)
Date: Fri,  7 Oct 2011 23:10:42 +0200 (CEST)
Subject: [pypy-commit] pypy rgc-mem-pressure: fix gc framework code
Message-ID: <20111007211042.4666282A02@wyvern.cs.uni-duesseldorf.de>

Author: Justin Peel <notmuchtotell at gmail.com>
Branch: rgc-mem-pressure
Changeset: r47869:04ba9860f69c
Date: 2011-10-07 15:10 -0600
http://bitbucket.org/pypy/pypy/changeset/04ba9860f69c/

Log:	fix gc framework code

diff --git a/pypy/rpython/memory/gctransform/framework.py b/pypy/rpython/memory/gctransform/framework.py
--- a/pypy/rpython/memory/gctransform/framework.py
+++ b/pypy/rpython/memory/gctransform/framework.py
@@ -374,24 +374,23 @@
             self.malloc_varsize_nonmovable_ptr = None
 
         if getattr(GCClass, 'raw_malloc_memory_pressure', False):
-            def raw_malloc_memory_pressure(length, itemsize):
+            def raw_malloc_memory_pressure_varsize(length, itemsize):
                 totalmem = length * itemsize
                 if totalmem > 0:
                     gcdata.gc.raw_malloc_memory_pressure(totalmem)
                 #else: probably an overflow -- the following rawmalloc
                 #      will fail then
-            self.raw_malloc_memory_pressure_ptr = getfn(
-                raw_malloc_memory_pressure,
+            def raw_malloc_memory_pressure(sizehint):
+                gcdata.gc.raw_malloc_memory_pressure(sizehint)
+            self.raw_malloc_memory_pressure_varsize_ptr = getfn(
+                raw_malloc_memory_pressure_varsize,
                 [annmodel.SomeInteger(), annmodel.SomeInteger()],
                 annmodel.s_None, minimal_transform = False)
+            self.raw_malloc_memory_pressure = getfn(
+                raw_malloc_memory_pressure,
+                [annmodel.SomeInteger()],
+                annmode.s_None, minimal_transform = False)
 
-        if getattr(GCClass, 'add_memory_pressure', False):
-            def add_memory_pressure(size):
-                gcdata.gc.raw_malloc_memory_pressure(size)
-            self.add_memory_pressure_ptr = getfun(
-                add_memory_pressure,
-                [annmodel.SomeInteger()],
-                annmodel.s_None, minimal_transform = False)
 
         self.identityhash_ptr = getfn(GCClass.identityhash.im_func,
                                       [s_gc, s_gcref],

From noreply at buildbot.pypy.org  Sat Oct  8 04:37:58 2011
From: noreply at buildbot.pypy.org (pjenvey)
Date: Sat,  8 Oct 2011 04:37:58 +0200 (CEST)
Subject: [pypy-commit] jitviewer default: have to also include the package
 data here for it to be picked up sdist
Message-ID: <20111008023758.577D982A02@wyvern.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: 
Changeset: r175:8b993824df41
Date: 2011-10-07 19:37 -0700
http://bitbucket.org/pypy/jitviewer/changeset/8b993824df41/

Log:	have to also include the package data here for it to be picked up
	sdist

diff --git a/MANIFEST.in b/MANIFEST.in
new file mode 100644
--- /dev/null
+++ b/MANIFEST.in
@@ -0,0 +1,2 @@
+recursive-include _jitviewer/templates *.html
+recursive-include _jitviewer/static *

From noreply at buildbot.pypy.org  Sat Oct  8 06:42:27 2011
From: noreply at buildbot.pypy.org (justinpeel)
Date: Sat,  8 Oct 2011 06:42:27 +0200 (CEST)
Subject: [pypy-commit] pypy rgc-mem-pressure: add a gc test
Message-ID: <20111008044227.685EE82A02@wyvern.cs.uni-duesseldorf.de>

Author: Justin Peel <notmuchtotell at gmail.com>
Branch: rgc-mem-pressure
Changeset: r47870:dc3a8493fc44
Date: 2011-10-07 22:42 -0600
http://bitbucket.org/pypy/pypy/changeset/dc3a8493fc44/

Log:	add a gc test

diff --git a/pypy/rpython/memory/gctransform/framework.py b/pypy/rpython/memory/gctransform/framework.py
--- a/pypy/rpython/memory/gctransform/framework.py
+++ b/pypy/rpython/memory/gctransform/framework.py
@@ -389,7 +389,7 @@
             self.raw_malloc_memory_pressure = getfn(
                 raw_malloc_memory_pressure,
                 [annmodel.SomeInteger()],
-                annmode.s_None, minimal_transform = False)
+                annmodel.s_None, minimal_transform = False)
 
 
         self.identityhash_ptr = getfn(GCClass.identityhash.im_func,
diff --git a/pypy/translator/c/test/test_newgc.py b/pypy/translator/c/test/test_newgc.py
--- a/pypy/translator/c/test/test_newgc.py
+++ b/pypy/translator/c/test/test_newgc.py
@@ -128,10 +128,10 @@
         if not args:
             args = (-1, )
         res = self.allfuncs(name, *args)
-        num = self.name_to_func[name]        
+        num = self.name_to_func[name]
         if self.funcsstr[num]:
             return res
-        return int(res)        
+        return int(res)
 
     def define_empty_collect(cls):
         def f():
@@ -1458,6 +1458,33 @@
         res = self.run("nongc_attached_to_gc")
         assert res == -99997
 
+    def define_nongc_opaque_attached_to_gc(cls):
+        from pypy.rpython.lltypesystem import rffi
+        from pypy.rlib import ropenssl
+        class A:
+            def __init__(self):
+                self.ctx = lltype.malloc(ropenssl.EVP_MD_CTX.TO,
+                    flavor='raw')
+                digest = ropenssl.EVP_get_digestbyname('sha1')
+                ropenssl.EVP_DigestInit(self.ctx, digest)
+            def __del__(self):
+                ropenssl.EVP_MD_CTX_cleanup(self.ctx)
+                lltype.free(self.ctx, flavor='raw')
+        A()
+        def f():
+            am1 = am2 = am3 = None
+            for i in range(100000):
+                am3 = am2
+                am2 = am1
+                am1 = A()
+            # what can we use for the res?
+            return 0
+        return f
+
+    def test_nongc_opaque_attached_to_gc(self):
+        res = self.run("nongc_opaque_attached_to_gc")
+        assert res == 0
+
 # ____________________________________________________________________
 
 class TaggedPointersTest(object):

From noreply at buildbot.pypy.org  Sat Oct  8 10:25:56 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  8 Oct 2011 10:25:56 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Add the tests to
 test_transformed_gc. Not passing so far, of course.
Message-ID: <20111008082556.AB2B982A02@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r47871:d8e46aa55083
Date: 2011-10-07 18:20 +0200
http://bitbucket.org/pypy/pypy/changeset/d8e46aa55083/

Log:	Add the tests to test_transformed_gc. Not passing so far, of
	course.

diff --git a/pypy/config/translationoption.py b/pypy/config/translationoption.py
--- a/pypy/config/translationoption.py
+++ b/pypy/config/translationoption.py
@@ -58,7 +58,8 @@
     # gc
     ChoiceOption("gc", "Garbage Collection Strategy",
                  ["boehm", "ref", "marksweep", "semispace", "statistics",
-                  "generation", "hybrid", "markcompact", "minimark", "none"],
+                  "generation", "hybrid", "markcompact", "minimark",
+                  "concurrentms", "none"],
                   "ref", requires={
                      "ref": [("translation.rweakref", False), # XXX
                              ("translation.gctransformer", "ref")],
@@ -73,6 +74,7 @@
                                ("translation.continuation", False)],  # breaks
                      "markcompact": [("translation.gctransformer", "framework")],
                      "minimark": [("translation.gctransformer", "framework")],
+                     "concurrentms": [("translation.gctransformer", "framework")],
                      },
                   cmdline="--gc"),
     ChoiceOption("gctransformer", "GC transformer that is used - internal",
diff --git a/pypy/rpython/memory/gc/base.py b/pypy/rpython/memory/gc/base.py
--- a/pypy/rpython/memory/gc/base.py
+++ b/pypy/rpython/memory/gc/base.py
@@ -435,6 +435,7 @@
                "hybrid": "hybrid.HybridGC",
                "markcompact" : "markcompact.MarkCompactGC",
                "minimark" : "minimark.MiniMarkGC",
+               "concurrentms": "concurrentms.MostlyConcurrentMarkSweepGC",
                }
     try:
         modulename, classname = classes[config.translation.gc].split('.')
diff --git a/pypy/rpython/memory/test/test_gc.py b/pypy/rpython/memory/test/test_gc.py
--- a/pypy/rpython/memory/test/test_gc.py
+++ b/pypy/rpython/memory/test/test_gc.py
@@ -927,7 +927,7 @@
 class TestMiniMarkGCCardMarking(TestMiniMarkGC):
     GC_PARAMS = {'card_page_indices': 4}
 
-class TestMostlyConcurrentMarkSweepGC(TestMiniMarkGC):
+class TestMostlyConcurrentMarkSweepGC(GCTest):
     from pypy.rpython.memory.gc.concurrentms \
             import MostlyConcurrentMarkSweepGC as GCClass
 
@@ -935,3 +935,15 @@
         py.test.skip("in-progress")
     def test_from_objwithfinalizer_to_youngobj(self):
         py.test.skip("in-progress")
+    def test_finalizer(self):
+        py.test.skip("in-progress")
+    def test_finalizer_calls_malloc(self):
+        py.test.skip("in-progress")
+    def test_finalizer_calls_collect(self):
+        py.test.skip("in-progress")
+    def test_finalizer_resurrects(self):
+        py.test.skip("in-progress")
+    def test_weakref(self):
+        py.test.skip("in-progress")
+    def test_weakref_to_object_with_finalizer(self):
+        py.test.skip("in-progress")
diff --git a/pypy/rpython/memory/test/test_transformed_gc.py b/pypy/rpython/memory/test/test_transformed_gc.py
--- a/pypy/rpython/memory/test/test_transformed_gc.py
+++ b/pypy/rpython/memory/test/test_transformed_gc.py
@@ -1415,3 +1415,14 @@
             GC_PARAMS = {'space_size': 4096*WORD,
                          'translated_to_c': False}
             root_stack_depth = 200
+
+
+class TestMostlyConcurrentMarkSweepGC(GenericGCTests):
+    gcname = "concurrentms"
+    class gcpolicy(gc.FrameworkGcPolicy):
+        class transformerclass(framework.FrameworkGCTransformer):
+            from pypy.rpython.memory.gc.concurrentms \
+                    import MostlyConcurrentMarkSweepGC as GCClass
+            GC_PARAMS = {'page_size': 128*WORD,
+                         'translated_to_c': False}
+            root_stack_depth = 200

From noreply at buildbot.pypy.org  Sat Oct  8 10:25:57 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  8 Oct 2011 10:25:57 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Progress.
Message-ID: <20111008082557.DD02B82A86@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r47872:fd256b1aaf7c
Date: 2011-10-08 09:02 +0200
http://bitbucket.org/pypy/pypy/changeset/fd256b1aaf7c/

Log:	Progress.

diff --git a/pypy/rpython/memory/gc/concurrentms.py b/pypy/rpython/memory/gc/concurrentms.py
--- a/pypy/rpython/memory/gc/concurrentms.py
+++ b/pypy/rpython/memory/gc/concurrentms.py
@@ -38,7 +38,8 @@
 MARK_VALUE_1      = 'M'     #  77, 0x4D
 MARK_VALUE_2      = 'k'     # 107, 0x6B
 MARK_VALUE_STATIC = 'S'     #  83, 0x53
-GCFLAG_WITH_HASH  = 0x01
+
+FL_WITHHASH = 0x01
 
 
 class MostlyConcurrentMarkSweepGC(GCBase):
@@ -54,7 +55,7 @@
                                   ('flags', lltype.Char),
                                   ('typeid16', llgroup.HALFWORD))
     typeid_is_in_field = 'typeid16'
-    withhash_flag_is_in_field = 'flags', GCFLAG_WITH_HASH
+    withhash_flag_is_in_field = 'flags', FL_WITHHASH
 
     TRANSLATION_PARAMS = {'page_size': 4096,
                           'small_request_threshold': 35*WORD,
@@ -137,17 +138,18 @@
             self.collector_start, ())
 
     def _teardown(self):
-        "NOT_RPYTHON.  Stop the collector thread after tests have run."
+        "Stop the collector thread after tests have run."
         if self._teardown_now:
             return
         self.wait_for_the_end_of_collection()
         #
         # start the next collection, but with "stop" in _teardown_now,
         # which should shut down the collector thread
-        self._teardown_now.append("stop")
+        self._teardown_now.append(-1)
         self.ready_to_start_lock.release()
         self.acquire(self.finished_lock)
-        del self.ready_to_start_lock, self.finished_lock
+        if not we_are_translated():
+            del self.ready_to_start_lock, self.finished_lock
 
     def get_type_id(self, obj):
         return self.header(obj).typeid16
@@ -423,10 +425,23 @@
                 self.nonfree_pages[0] = self.collect_heads[0]
 
 
-    def collect(self, gen=0):
-        """Trigger a complete collection, and wait for it to finish."""
-        self.trigger_next_collection()
-        self.wait_for_the_end_of_collection()
+    def collect(self, gen=2):
+        """
+        gen=0: Trigger a collection if none is running.  Never blocks.
+        
+        gen=1: The same, but if a collection is running, wait for it
+        to finish before triggering the next one.  Guarantees that
+        objects not reachable when collect() is called will soon be
+        freed.
+
+        gen>=2: The same, but wait for the triggered collection to
+        finish.  Guarantees that objects not reachable when collect()
+        is called will be freed by the time collect() returns.
+        """
+        if gen >= 1 or self.collection_running == 0:
+            self.trigger_next_collection()
+            if gen >= 2:
+                self.wait_for_the_end_of_collection()
 
     def trigger_next_collection(self):
         """In the mutator thread: triggers the next collection."""
@@ -639,6 +654,16 @@
         self.collect_tails[n] = first_loc_in_linked_list
 
 
+    def identityhash(self, obj):
+        obj = llmemory.cast_ptr_to_adr(obj)
+        hdr = self.header(obj)
+        if ord(hdr.flags) & FL_WITHHASH:
+            obj += self.get_size(obj)
+            return obj.signed[0]
+        else:
+            return llmemory.cast_adr_to_int(obj)
+
+
 def maybe_read_mark_byte(addr):
     "NOT_RPYTHON"
     try:

From noreply at buildbot.pypy.org  Sat Oct  8 10:25:59 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  8 Oct 2011 10:25:59 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Translation fix.
Message-ID: <20111008082559.1A9FB82A87@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r47873:58cca100f3ed
Date: 2011-10-08 09:08 +0200
http://bitbucket.org/pypy/pypy/changeset/58cca100f3ed/

Log:	Translation fix.

diff --git a/pypy/rpython/memory/gc/concurrentms.py b/pypy/rpython/memory/gc/concurrentms.py
--- a/pypy/rpython/memory/gc/concurrentms.py
+++ b/pypy/rpython/memory/gc/concurrentms.py
@@ -1,6 +1,7 @@
 import time, sys
 from pypy.rpython.lltypesystem import lltype, llmemory, llarena, llgroup, rffi
 from pypy.rpython.lltypesystem.llmemory import raw_malloc_usage
+from pypy.translator.tool.cbuild import ExternalCompilationInfo
 from pypy.rlib.objectmodel import we_are_translated, running_on_llinterp
 from pypy.rlib.debug import ll_assert
 from pypy.rlib.rarithmetic import ovfcheck, LONG_BIT, r_uint
@@ -663,10 +664,24 @@
         else:
             return llmemory.cast_adr_to_int(obj)
 
+# ____________________________________________________________
+#
+# Hack to read the first byte of a location, which may be the
+# "mark" byte in an object or, if the location is free, the lowest
+# byte of the "next" pointer.
 
-def maybe_read_mark_byte(addr):
+def emulate_read_mark_byte(addr):
     "NOT_RPYTHON"
     try:
         return addr.ptr.mark
     except AttributeError:
         return '\x00'
+
+eci = ExternalCompilationInfo(
+    post_include_bits = ["""
+        #define pypy_concurrentms_read_byte(addr)  (*(char*)(addr))
+    """])
+maybe_read_mark_byte = rffi.llexternal("pypy_concurrentms_read_byte",
+                                       [llmemory.Address], lltype.Char,
+                                       compilation_info=eci, _nowrapper=True,
+                                       _callable=emulate_read_mark_byte)

From noreply at buildbot.pypy.org  Sat Oct  8 10:26:00 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  8 Oct 2011 10:26:00 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Add checks; fix a bug.
Message-ID: <20111008082600.49E0F82A88@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r47874:900da077c864
Date: 2011-10-08 09:23 +0200
http://bitbucket.org/pypy/pypy/changeset/900da077c864/

Log:	Add checks; fix a bug.

diff --git a/pypy/rpython/memory/gc/concurrentms.py b/pypy/rpython/memory/gc/concurrentms.py
--- a/pypy/rpython/memory/gc/concurrentms.py
+++ b/pypy/rpython/memory/gc/concurrentms.py
@@ -424,6 +424,9 @@
             if self.collect_tails[0] != NULL:
                 self.collect_tails[0].address[0] = self.nonfree_pages[0]
                 self.nonfree_pages[0] = self.collect_heads[0]
+            #
+            if self.DEBUG:
+                self.debug_check_lists()
 
 
     def collect(self, gen=2):
@@ -471,6 +474,7 @@
         while n < self.pagelists_length:
             self.collect_pages[n] = self.nonfree_pages[n]
             n += 1
+        self.nonfree_pages[0] = NULL
         #
         # Start the collector thread
         self.collection_running = 1
@@ -483,6 +487,22 @@
     def _add_prebuilt_root(self, obj, ignored):
         self.gray_objects.append(obj)
 
+    def debug_check_lists(self):
+        # just check that they are correct, non-infinite linked lists
+        self.debug_check_list(self.nonfree_pages[0])
+        n = 1
+        while n < self.pagelists_length:
+            self.debug_check_list(self.free_lists[n])
+            n += 1
+
+    def debug_check_list(self, page):
+        try:
+            while page != llmemory.NULL:
+                page = page.address[0]
+        except KeyboardInterrupt:
+            ll_assert(False, "interrupted")
+            raise
+
     def acquire(self, lock):
         if we_are_translated():
             lock.acquire(True)
@@ -600,12 +620,15 @@
         while block != llmemory.NULL:
             nextblock = block.address[0]
             hdr = block + size_of_addr
-            if maybe_read_mark_byte(hdr) == nonmarked:
+            mark = maybe_read_mark_byte(hdr)
+            if mark == nonmarked:
                 # the object is still not marked.  Free it.
                 llarena.arena_free(block)
                 #
             else:
                 # the object was marked: relink it
+                ll_assert(mark == self.current_mark,
+                          "bad mark in large object")
                 block.address[0] = linked_list
                 linked_list = block
                 if first_block_in_linked_list == NULL:

From noreply at buildbot.pypy.org  Sat Oct  8 10:26:01 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  8 Oct 2011 10:26:01 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Translation fixes.
Message-ID: <20111008082601.7E20882A89@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r47875:75c893ed80a3
Date: 2011-10-08 10:00 +0200
http://bitbucket.org/pypy/pypy/changeset/75c893ed80a3/

Log:	Translation fixes.

diff --git a/pypy/module/thread/ll_thread.py b/pypy/module/thread/ll_thread.py
--- a/pypy/module/thread/ll_thread.py
+++ b/pypy/module/thread/ll_thread.py
@@ -118,11 +118,10 @@
 
     def release(self):
         # Sanity check: the lock must be locked
-        if self.acquire(False):
-            c_thread_releaselock(self._lock)
+        error = self.acquire(False)
+        c_thread_releaselock(self._lock)
+        if error:
             raise error("bad lock")
-        else:
-            c_thread_releaselock(self._lock)
 
     def __del__(self):
         if free_ll_lock is None:  # happens when tests are shutting down
diff --git a/pypy/rpython/memory/gc/concurrentms.py b/pypy/rpython/memory/gc/concurrentms.py
--- a/pypy/rpython/memory/gc/concurrentms.py
+++ b/pypy/rpython/memory/gc/concurrentms.py
@@ -111,17 +111,25 @@
         # When done it releases 'finished_lock'.  The mutator thread is
         # responsible for resetting 'collection_running' to 0.
         self.collection_running = 0
-        self.ready_to_start_lock = ll_thread.allocate_lock()
-        self.finished_lock = ll_thread.allocate_lock()
+        #self.ready_to_start_lock = ...built in setup()
+        #self.finished_lock = ...built in setup()
         #
         # NOT_RPYTHON: set to non-empty in _teardown()
         self._teardown_now = []
         #
         def collector_start():
-            self.collector_run()
+            if we_are_translated():
+                self.collector_run()
+            else:
+                try:
+                    self.collector_run()
+                except Exception, e:
+                    print 'Crash!', e.__class__.__name__, e
+                    self._exc_info = sys.exc_info()
+        #
         self.collector_start = collector_start
         #
-        self.mutex_lock = ll_thread.allocate_lock()
+        #self.mutex_lock = ...built in setup()
         self.gray_objects = self.AddressStack()
         self.extra_objects_to_mark = self.AddressStack()
         self.prebuilt_root_objects = self.AddressStack()
@@ -133,8 +141,15 @@
 
     def setup(self):
         "Start the concurrent collector thread."
+        GCBase.setup(self)
+        #
+        self.ready_to_start_lock = ll_thread.allocate_ll_lock()
+        self.finished_lock = ll_thread.allocate_ll_lock()
+        self.mutex_lock = ll_thread.allocate_ll_lock()
+        #
         self.acquire(self.finished_lock)
         self.acquire(self.ready_to_start_lock)
+        #
         self.collector_ident = ll_thread.start_new_thread(
             self.collector_start, ())
 
@@ -147,7 +162,7 @@
         # start the next collection, but with "stop" in _teardown_now,
         # which should shut down the collector thread
         self._teardown_now.append(-1)
-        self.ready_to_start_lock.release()
+        self.release(self.ready_to_start_lock)
         self.acquire(self.finished_lock)
         if not we_are_translated():
             del self.ready_to_start_lock, self.finished_lock
@@ -364,7 +379,7 @@
     def _init_writebarrier_logic(self):
         #
         def force_scan(obj):
-            self.mutex_lock.acquire(True)
+            self.acquire(self.mutex_lock)
             mark = self.header(obj).mark
             if mark != self.current_mark:
                 #
@@ -388,7 +403,7 @@
                     self.set_mark(obj, self.current_mark)
                     self.trace(obj, self._barrier_add_extra, None)
                 #
-            self.mutex_lock.release()
+            self.release(self.mutex_lock)
         #
         force_scan._dont_inline_ = True
         self.force_scan = force_scan
@@ -478,7 +493,7 @@
         #
         # Start the collector thread
         self.collection_running = 1
-        self.ready_to_start_lock.release()
+        self.release(self.ready_to_start_lock)
 
     def _add_stack_root(self, root):
         obj = root.address[0]
@@ -498,6 +513,9 @@
     def debug_check_list(self, page):
         try:
             while page != llmemory.NULL:
+                # prevent constant-folding:
+                byte = ord(maybe_read_mark_byte(page))
+                ll_assert((byte & 3) == 0, "misaligned?")
                 page = page.address[0]
         except KeyboardInterrupt:
             ll_assert(False, "interrupted")
@@ -505,14 +523,18 @@
 
     def acquire(self, lock):
         if we_are_translated():
-            lock.acquire(True)
+            ll_thread.c_thread_acquirelock(lock, 1)
         else:
-            while not lock.acquire(False):
+            while rffi.cast(lltype.Signed,
+                            ll_thread.c_thread_acquirelock(lock, 0)) == 0:
                 time.sleep(0.001)
                 # ---------- EXCEPTION FROM THE COLLECTOR THREAD ----------
                 if hasattr(self, '_exc_info'):
                     self._reraise_from_collector_thread()
 
+    def release(self, lock):
+        ll_thread.c_thread_releaselock(lock)
+
     def _reraise_from_collector_thread(self):
         exc, val, tb = self._exc_info
         raise exc, val, tb
@@ -520,29 +542,25 @@
 
     def collector_run(self):
         """Main function of the collector's thread."""
-        try:
-            while True:
-                #
-                # Wait for the lock to be released
-                self.acquire(self.ready_to_start_lock)
-                #
-                # For tests: detect when we have to shut down
-                if not we_are_translated():
-                    if self._teardown_now:
-                        self.finished_lock.release()
-                        break
-                #
-                # Mark
-                self.collector_mark()
-                self.collection_running = 2
-                #
-                # Sweep
-                self.collector_sweep()
-                self.finished_lock.release()
-                #
-        except Exception, e:
-            print 'Crash!', e.__class__.__name__, e
-            self._exc_info = sys.exc_info()
+        while True:
+            #
+            # Wait for the lock to be released
+            self.acquire(self.ready_to_start_lock)
+            #
+            # For tests: detect when we have to shut down
+            if not we_are_translated():
+                if self._teardown_now:
+                    self.release(finished_lock)
+                    break
+            #
+            # Mark
+            self.collector_mark()
+            self.collection_running = 2
+            #
+            # Sweep
+            self.collector_sweep()
+            self.release(self.finished_lock)
+
 
     def other_mark(self, mark):
         ll_assert(mark == MARK_VALUE_1 or mark == MARK_VALUE_2,
@@ -570,11 +588,11 @@
             # 'gray_objects'.  This requires the mutex lock.
             # There are typically only a few objects to move here,
             # unless XXX we've hit the write barrier of a large array
-            self.mutex_lock.acquire(True)
+            self.acquire(self.mutex_lock)
             while self.extra_objects_to_mark.non_empty():
                 obj = self.extra_objects_to_mark.pop()
                 self.gray_objects.append(obj)
-            self.mutex_lock.release()
+            self.release(self.mutex_lock)
             #
             # If 'gray_objects' is empty, we are done: there should be
             # no possible case in which more objects are being added to
diff --git a/pypy/translator/c/test/test_newgc.py b/pypy/translator/c/test/test_newgc.py
--- a/pypy/translator/c/test/test_newgc.py
+++ b/pypy/translator/c/test/test_newgc.py
@@ -1515,3 +1515,6 @@
 
 class TestMiniMarkGCMostCompact(TaggedPointersTest, TestMiniMarkGC):
     removetypeptr = True
+
+class TestMostlyConcurrentMarkSweepGC(TestUsingFramework):
+    gcpolicy = "concurrentms"

From noreply at buildbot.pypy.org  Sat Oct  8 14:01:29 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Sat,  8 Oct 2011 14:01:29 +0200 (CEST)
Subject: [pypy-commit] benchmarks default: add sympy benchmarks
Message-ID: <20111008120129.60A3482A87@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r145:335e29a9381a
Date: 2011-10-08 13:56 +0200
http://bitbucket.org/pypy/benchmarks/changeset/335e29a9381a/

Log:	add sympy benchmarks

diff too long, truncating to 10000 out of 268124 lines

diff --git a/benchmarks.py b/benchmarks.py
--- a/benchmarks.py
+++ b/benchmarks.py
@@ -45,6 +45,12 @@
     'bm_chameleon': {'bm_env': {'PYTHONPATH': relative('lib/chameleon/src')}},
 }
 
+for name in ['expand', 'integrate', 'sum', 'str']:
+    _register_new_bm('bm_sympy', 'sympy_' + name,
+                     globals(), bm_env={'PYTHONPATH': relative('lib/sympy')},
+                     extra_args=['--benchmark=' + name],
+                     iteration_scaling=0.1)
+
 for name in ['float', 'nbody_modified', 'meteor-contest', 'fannkuch',
              'spectral-norm', 'chaos', 'telco', 'go', 'pyflate-fast',
              'raytrace-simple', 'crypto_pyaes', 'bm_mako', 'bm_chameleon']:
diff --git a/lib/sympy/.mailmap b/lib/sympy/.mailmap
new file mode 100644
--- /dev/null
+++ b/lib/sympy/.mailmap
@@ -0,0 +1,19 @@
+Ondrej Certik <ondrej at certik.cz> ondrej.certik <devnull at localhost>
+Fredrik Johansson <fredrik.johansson at gmail.com> fredrik.johansson <devnull at localhost>
+Kirill Smelkov <kirr at landau.phys.spbu.ru> kirill.smelkov <devnull at localhost>
+Kirill Smelkov <kirr at landau.phys.spbu.ru> convert-repo <devnull at localhost>
+Mateusz Paprocki <mattpap at gmail.com> mattpap <devnull at localhost>
+Fabian Pedregosa <fabian at fseoane.net> Fabian Seoane <fabian at fseoane.net>
+Fabian Pedregosa <fabian at fseoane.net> fabian <fabian at debian.(none)>
+Jason Gedge <inferno1386 at gmail.com> inferno1386 <devnull at localhost>
+Robert Schwarz <lethargo at googlemail.com> lethargo <devnull at localhost>
+Sebastian Kr&#228;mer <basti.kr at gmail.com> basti.kr <devnull at localhost>
+David Roberts <dvdr18 at gmail.com> David Roberts (dvdr18 [at] gmail [dot] com) <devnull at localhost>
+Saroj Adhikari <adh.saroj at gmail.com> Saroj <adh.saroj at gmail.com>
+Rizgar Mella <rizgar.mella at gmail.com> RizgarMella rizgar.mella at gmail.com <RizgarMella rizgar.mella at gmail.com>
+Pearu Peterson <pearu.peterson at gmail.com> pearu.peterson <devnull at localhost>
+Brian Jorgensen <brian.jorgensen at gmail.com> brian.jorgensen <devnull at localhost>
+Chris Wu <chris.wu at gmail.com> Chris.Wu <devnull at localhost>
+Jezreel Ng <jezreel at gmail.com>
+James Pearson <xiong.chiamiov at gmail.com>
+Crist&#243;v&#227;o Sousa <crisjss at gmail.com>
diff --git a/lib/sympy/AUTHORS b/lib/sympy/AUTHORS
new file mode 100644
--- /dev/null
+++ b/lib/sympy/AUTHORS
@@ -0,0 +1,135 @@
+All people who contributed to SymPy by sending at least a patch or more (in
+the order of the date of their first contribution), except those who
+explicitely didn't want to be mentioned:
+
+Ondrej Certik <ondrej at certik.cz>
+Fabian Pedregosa <fabian at fseoane.net>
+Jurjen N.E. Bos <jnebos at gmail.com>
+Mateusz Paprocki <mattpap at gmail.com>
+Marc-Etienne M.Leveille <protonyc at gmail.com>
+Brian Jorgensen <brian.jorgensen at gmail.com>
+Jason Gedge <inferno1386 at gmail.com>
+Robert Schwarz <lethargo at googlemail.com>
+Pearu Peterson <pearu.peterson at gmail.com>
+Fredrik Johansson <fredrik.johansson at gmail.com>
+Chris Wu <chris.wu at gmail.com>
+Ulrich Hecht <ulrich.hecht at gmail.com>
+Goutham Lakshminarayan <dl.goutham at gmail.com>
+David Lawrence <dmlawrence at gmail.com>
+Jaroslaw Tworek <dev.jrx at gmail.com>
+David Marek <h4wk.cz at gmail.com>
+Bernhard R. Link <brlink at debian.org>
+Andrej Tokar&#269;&#237;k <androsis at gmail.com>
+Or Dvory <gidesa at gmail.com>
+Saroj Adhikari <adh.saroj at gmail.com>
+Pauli Virtanen <pav at iki.fi>
+Robert Kern <robert.kern at gmail.com>
+James Aspnes <aspnes at cs.yale.edu>
+Nimish Telang <ntelang at gmail.com>
+Abderrahim Kitouni <a.kitouni at gmail.com>
+Pan Peng <pengpanster at gmail.com>
+Friedrich Hagedorn <friedrich_h at gmx.de>
+Elrond der Elbenfuerst <elrond+sympy.org at samba-tng.org>
+Rizgar Mella <rizgar.mella at gmail.com>
+Felix Kaiser <felix.kaiser at fxkr.net>
+Roberto Nobrega <rwnobrega at gmail.com>
+David Roberts <dvdr18 at gmail.com>
+Sebastian Kr&#228;mer <basti.kr at gmail.com>
+Vinzent Steinberg <vinzent.steinberg at gmail.com>
+Riccardo Gori <goriccardo at gmail.com>
+Case Van Horsen <casevh at gmail.com>
+Stepan Roucka <stepan at roucka.eu>
+Ali Raza Syed <arsyed at gmail.com>
+Stefano Maggiolo <s.maggiolo at gmail.com>
+Robert Cimrman <cimrman3 at ntc.zcu.cz>
+Bastian Weber <bastian.weber at gmx-topmail.de>
+Sebastian Krause <sebastian.krause at gmx.de>
+Sebastian Kreft <skreft at gmail.com>
+Dan <coolg49964 at gmail.com>
+Alan Bromborsky <abrombo at verizon.net>
+Boris Timokhin <qoqenator at gmail.com>
+Robert <average.programmer at gmail.com>
+Andy R. Terrel <aterrel at uchicago.edu>
+Hubert Tsang <intsangity at gmail.com>
+Konrad Meyer <konrad.meyer at gmail.com>
+Henrik Johansson <henjo2006 at gmail.com>
+Priit Laes <plaes at plaes.org>
+Freddie Witherden <freddie at witherden.org>
+Brian E. Granger <ellisonbg at gmail.com>
+Andrew Straw <strawman at astraw.com>
+Kaifeng Zhu <cafeeee at gmail.com>
+Ted Horst <ted.horst at earthlink.net>
+Akshay Srinivasan <akshaysrinivasan at gmail.com>
+Aaron Meurer <asmeurer at gmail.com>
+Barry Wardell <barry.wardell at gmail.com>
+Tomasz Buchert <thinred at gmail.com>
+Vinay Kumar <gnulinooks at gmail.com>
+Johann Cohen-Tanugi <johann.cohentanugi at gmail.com>
+Jochen Voss <voss at seehuhn.de>
+Luke Peterson <hazelnusse at gmail.com>
+Chris Smith <smichr at gmail.com>
+Thomas Sidoti <TSidoti at gmail.com>
+Florian Mickler <florian at mickler.org>
+Nicolas Pourcelot <nicolas.pourcelot at gmail.com>
+Ben Goodrich <goodrich.ben at gmail.com>
+Toon Verstraelen <Toon.Verstraelen at UGent.be>
+Ronan Lamy <ronan.lamy at gmail.com>
+James Abbatiello <abbeyj at gmail.com>
+Ryan Krauss <ryanlists at gmail.com>
+Bill Flynn <wflynny at gmail.com>
+Jorn Baayen <jorn.baayen at gmail.com>
+Eh Tan <tan2tan2 at gmail.com>
+Renato Coutinho <renato.coutinho at gmail.com>
+Oscar Benjamin
+&#216;yvind Jensen <jensen.oyvind at gmail.com>
+Julio Idichekop Filho <idichekop at yahoo.com.br>
+&#321;ukasz Pankowski <lukpank at o2.pl>
+Chu-Ching Huang <cchuang at mail.cgu.edu.tw>
+Fernando Perez <Fernando.Perez at berkeley.edu>
+Raffaele De Feo <alberthilbert at gmail.com>
+Christian Muise <christian.muise at gmail.com>
+Matt Curry <mattjcurry at gmail.com>
+Kazuo Thow <kazuo.thow at gmail.com>
+Jezreel Ng <jezreel at gmail.com>
+Matthew Brett <matthew.brett at gmail.com>
+Addison Cugini <ajcugini at gmail.com>
+Nicholas J.S. Kinar <n.kinar at usask.ca>
+Thomas Dixon <thom at thomdixon.org>
+Crist&#243;v&#227;o Sousa <crisjss at gmail.com>
+Andre de Fortier Smit <freevryheid at gmail.com>
+Alexey U. Goodchenko <proga at goodok.ru>
+Gary Kerr <gary.kerr at blueyonder.co.uk>
+Sherjil Ozair <sherjilozair at gmail.com>
+Oleksandr Gituliar <gituliar at gmail.com>
+Sean Vig <sean.v.775 at gmail.com>
+Prafullkumar P. Tale <hector1618 at gmail.com>
+Vladimir Peri&#263; <vlada.peric at gmail.com>
+Tom Bachmann <e_mc_h2 at web.de>
+Yuri Karadzhov <yuri.karadzhov at gmail.com>
+Vladimir Lagunov <werehuman at gmail.com>
+Matthew Rocklin <mrocklin at cs.uchicago.edu>
+Saptarshi Mandal <sapta.iitkgp at gmail.com>
+Gilbert Gede <gilbertgede at gmail.com>
+Anatolii Koval <weralwolf at gmail.com>
+Tomo Lazovich <lazovich at gmail.com>
+Pavel Fedotov <fedotovp at gmail.com>
+Kibeom Kim <kk1674 at nyu.edu>
+Gregory Ksionda <ksiondag846 at gmail.com>
+Tom&#225;&#353; Bambas <tomas.bambas at gmail.com>
+Jeremias Yehdegho <j.yehdegho at gmail.com>
+Jack McCaffery <jpmccaffery at gmail.com>
+Luca Weihs <astronomicalcuriosity at gmail.com>
+Shai 'Deshe' Wyborski <shaide at cs.huji.ac.il>
+Thomas Wiecki <thomas.wiecki at gmail.com>
+&#211;scar N&#225;jera <najera.oscar at gmail.com>
+Mario Pernici <mario.pernici at gmail.com>
+Benjamin McDonald <mcdonald.ben at gmail.com>
+Sam Magura <samtheman132 at gmail.com>
+Stefan Krastanov <krastanov.stefan at gmail.com>
+Bradley Froehle <brad.froehle at gmail.com>
+Min Ragan-Kelley <benjaminrk at gmail.com>
+Emma Hogan <ehogan at gemini.edu>
+Julien Rioux <julien.rioux at gmail.com>
+Roberto Colistete, Jr. <roberto.colistete at gmail.com>
+Raoul Bourquin <raoulb at bluewin.ch>
+Gert-Ludwig Ingold <gert.ingold at physik.uni-augsburg.de>
diff --git a/lib/sympy/LICENSE b/lib/sympy/LICENSE
new file mode 100644
--- /dev/null
+++ b/lib/sympy/LICENSE
@@ -0,0 +1,28 @@
+Copyright (c) 2006, 2007, 2008, 2009, 2010, 2011 SymPy Development Team
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+  a. Redistributions of source code must retain the above copyright notice,
+     this list of conditions and the following disclaimer.
+  b. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+  c. Neither the name of the SymPy nor the names of its contributors
+     may be used to endorse or promote products derived from this software
+     without specific prior written permission.
+
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGE.
diff --git a/lib/sympy/MANIFEST.in b/lib/sympy/MANIFEST.in
new file mode 100644
--- /dev/null
+++ b/lib/sympy/MANIFEST.in
@@ -0,0 +1,10 @@
+recursive-include data *
+recursive-include doc *
+prune doc/_build
+recursive-include examples *.py README
+
+include sympy/utilities/mathml/data/*.xsl
+
+include LICENSE
+include TODO
+include AUTHORS
diff --git a/lib/sympy/Makefile b/lib/sympy/Makefile
new file mode 100644
--- /dev/null
+++ b/lib/sympy/Makefile
@@ -0,0 +1,9 @@
+# build or clean Cythonized modules
+
+all:
+	python build.py build_ext --inplace
+
+clean:
+	rm -rf sympy/polys/*.{pyc,c,so}
+	rm -rf build/
+
diff --git a/lib/sympy/README.rst b/lib/sympy/README.rst
new file mode 100644
--- /dev/null
+++ b/lib/sympy/README.rst
@@ -0,0 +1,138 @@
+SymPy
+=====
+
+A Python library for symbolic mathematics.
+
+http://sympy.org/
+
+See the AUTHORS file for the list of authors.
+
+And many more people helped on the SymPy mailinglist, reported bugs, helped
+organize SymPy's participation in the Google Summer of Code, the Google Highly
+Open Participation Contest, wrote and blogged about SymPy...
+
+License: New BSD License (see the LICENSE file for details)
+covers all files in the sympy repository unless stated otherwise.
+
+0. Download
+-----------
+
+::
+
+    $ git clone git://github.com/sympy/sympy.git
+
+For other options (tarballs, debs, etc.), see the web page of SymPy.
+
+1. Documentation and usage
+--------------------------
+
+Everything is at:
+
+http://docs.sympy.org/
+
+You can generate everything at the above site in your local copy of SymPy by::
+
+    $ cd doc
+    $ make html
+    $ epiphany _build/html/index.html # Or your preferred web browser
+
+If you don't want to read that, here is a short usage:
+
+From this directory, start python and::
+
+    >>> from sympy import Symbol, cos
+    >>> x = Symbol('x')
+    >>> e = 1/cos(x)
+    >>> print e.series(x, 0, 10)
+    1 + (1/2)*x**2 + (5/24)*x**4 + (61/720)*x**6 + (277/8064)*x**8 + O(x**10)
+
+SymPy also comes with a console that is a simple wrapper around the
+classic python console (or ipython when available) that loads the
+sympy namespace and executes some common commands for you.
+
+To start it, issue::
+
+    $ bin/isympy
+
+from this directory if SymPy is not installed or simply::
+
+    $ isympy
+
+if SymPy is installed somewhere in your ``PATH``.
+
+3. Installation
+---------------
+
+To install SymPy, simply run::
+
+    $ python setup.py install
+
+If you install it system-wide, you may need to prefix the previous command with ``sudo``::
+
+    $ sudo python setup.py install
+
+4. Tests
+--------
+
+To execute all tests, run::
+
+    $./setup.py test
+
+in the current directory.
+
+For more fine-grained running of tests, use ``bin/test`` or respectively
+``bin/doctest``.
+
+
+5. Clean
+--------
+
+To clean everything (thus getting the same tree as in the repository)::
+
+    $./setup.py clean
+
+6. Brief History
+----------------
+
+SymPy was started by Ondrej Certik in 2005, he wrote some code during the
+summer, then he wrote some more code during the summer 2006. In February 2007,
+Fabian Pedregosa joined the project and helped fixed many things, contributed
+documentation and made it alive again. 5 students (Mateusz Paprocki, Brian
+Jorgensen, Jason Gedge, Robert Schwarz and Chris Wu) improved SymPy incredibly
+during the summer 2007 as part of the Google Summer of Code. Pearu Peterson
+joined the development during the summer 2007 and he has made SymPy much more
+competitive by rewriting the core from scratch, that has made it from 10x to
+100x faster. Jurjen N.E. Bos has contributed pretty printing and other patches.
+Fredrik Johansson has wrote mpmath and contributed a lot of patches. Since
+then, a lot more people have joined the development and some people have also
+left. You can see the full list in doc/src/aboutus.txt, or online at:
+
+http://docs.sympy.org/aboutus.html#sympy-development-team
+
+For people that don't want to be listed there, see the git history.
+
+
+7. Citation
+-----------
+
+To cite SymPy in publications use::
+
+    SymPy Development Team (2011). SymPy: Python library for symbolic mathematics
+    URL http://www.sympy.org.
+
+A BibTeX entry for LaTeX users is::
+
+    @Manual{,
+    title = {SymPy: Python library for symbolic mathematics},
+    author = {{SymPy Development Team}},
+    year = {2011},
+    url = {http://www.sympy.org},
+    }
+
+SymPy is BSD licensed, so you are free to use it whatever you like, be it
+academic, commercial, creating forks or derivatives, as long as you copy the BSD
+statement if you redistribute it (see the LICENSE file for details).
+That said, although not required by the SymPy license, if it is convenient for
+you, please cite SymPy when using it in your work and also consider
+contributing all your changes back, so that we can incorporate it and all of us
+will benefit in the end.
diff --git a/lib/sympy/TODO b/lib/sympy/TODO
new file mode 100644
--- /dev/null
+++ b/lib/sympy/TODO
@@ -0,0 +1,8 @@
+Current problems are tracked at:
+
+http://code.google.com/p/sympy/issues/list
+
+for ideas what should be done in the more distant future, see:
+http://code.google.com/p/sympy/wiki/Discussion
+
+but generally ask on the SymPy mailinglist for uptodate future plan. :)
diff --git a/lib/sympy/bin/adapt_paths.py b/lib/sympy/bin/adapt_paths.py
new file mode 100644
--- /dev/null
+++ b/lib/sympy/bin/adapt_paths.py
@@ -0,0 +1,40 @@
+"""
+This script adapts import statements in sympy/mpmath/tests to work properly.
+
+We want to have this fully automatic, so that we don't have to do it by hand
+each time we copy files from mpmath to sympy.
+
+Usage:
+
+$ python bin/adapt_paths.py > /tmp/x
+$ patch -p0 < /tmp/x
+
+You can use the "--dry-run" parameter to 'patch' to see if all is ok and you
+should also inspect the /tmp/x that all the changes generated are actually
+correct.
+"""
+
+from glob import glob
+import re
+import difflib
+
+def get_files_mpmath():
+    return glob("sympy/mpmath/tests/test_*.py")
+
+def fix_file(filename):
+    f = open(filename)
+    orig = f.read()
+    # This converts stuff like "mpmath.dps -> sympy.mpmath.dps", but will leave
+    # "sympy.mpmath.dps" untouched.
+    s = re.sub("(?<!sympy\.)mpmath", "sympy.mpmath", orig)
+
+    # print differences in an unified diff format
+    d = difflib.unified_diff(orig.split("\n"), s.split("\n"),
+        fromfile=filename, tofile=filename+".new", lineterm="")
+    import sys
+    for l in d:
+        print l
+
+
+for x in get_files_mpmath():
+    fix_file(x)
diff --git a/lib/sympy/bin/coverage_doctest.py b/lib/sympy/bin/coverage_doctest.py
new file mode 100755
--- /dev/null
+++ b/lib/sympy/bin/coverage_doctest.py
@@ -0,0 +1,161 @@
+#! /usr/bin/env python
+
+"""
+Program to test that all methods/functions have at least one example doctest.
+
+Usage:
+
+bin/coverage_doctest.py sympy/core
+
+or
+
+bin/coverage_doctest.py sympy/core/basic.py
+
+This script is based on the sage-coverage script from Sage written by William
+Stein.
+"""
+
+import os
+import re
+import sys
+from optparse import OptionParser
+
+def parse_file(file, verbose=False):
+    skipped = []
+    missing_docstring = []
+    missing_doctest = []
+    has_doctest = []
+    indirect_doctest = []
+    while True:
+        i = file.find("def ")
+        if i == -1:
+            break
+
+        e = re.compile('\)\s*:')
+        m = e.search(file[i:])
+        if m is None:
+            break
+        j = m.end() + i
+
+        # "j" now points to the end of the function definition.
+
+        function_name = (' '.join(file[i:j].lstrip('def').split()))[:-1]
+        bare_function_name = function_name[:function_name.find("(")]
+
+        skip_this = False
+        for skip in ['__dealloc__', '__new__', '_']:
+            if function_name.startswith(skip + '('):
+                skip_this = True
+                break
+        if function_name.startswith("_"):
+            # For the time being, let's skip all "private" functions, that
+            # beging with "_". Later, when our doctests are in a good shape, we
+            # may doctest those too.
+            skip_this = True
+        if skip_this:
+            if verbose:
+                skipped.append(function_name)
+            file = file[j:]
+            continue
+
+        k = file[j:].find('\n')
+        if k == -1:
+            break
+        k += j
+        kk = file[k+1:].find('\n')
+        if kk == -1:
+            break
+        kk += k+1
+
+        q0 = file[k:kk].find('"""')
+        if q0 == -1:
+            missing_docstring.append(function_name)
+        else:
+            q0 += k
+            q1 = file[q0+3:].find('"""')
+            if q1 == -1:
+                print "ERROR: Error parsing %s" % function_name
+            else:
+                q1 += q0 + 3
+                # the docstring is now between q0:q1
+                d = file[q0:q1].find('>>>')
+                if d == -1:
+                    missing_doctest.append(function_name)
+                else:
+                    has_doctest.append(function_name)
+                    if not (bare_function_name[0:2] == '__' and
+                        bare_function_name[-2:] == '__'):
+                        d = file[q0:q1].find(bare_function_name)
+                        e = file[q0:q1].find('indirect doctest')
+                        if d == -1 and e == -1:
+                            indirect_doctest.append(function_name)
+
+        file = file[j+3:]
+    return skipped, missing_docstring, missing_doctest, has_doctest, \
+            indirect_doctest
+
+
+def coverage(filename, file, verbose=False):
+    skipped, missing_docstring, missing_doctest, has_doctest, \
+        indirect_doctest = parse_file(file, verbose)
+    num_functions = len(missing_docstring + missing_doctest + has_doctest)
+    if num_functions == 0:
+        print "No functions in %s" % filename
+        return
+    print '-'*70
+    print filename
+    score = 100 * float(len(has_doctest)) / num_functions
+    score = int(score)
+
+    if missing_docstring:
+        print "\nMissing documentation:\n\t * %s\n" % \
+                ('\n\t * '.join(missing_docstring))
+    if missing_doctest:
+        print "\nMissing doctests:\n\t * %s\n" % \
+                ('\n\t * '.join(missing_doctest))
+
+    if indirect_doctest:
+        print "\nIndirect doctest (function name doesn't occur in doctests):\n"\
+                "\t * %s\n"%('\n\t * '.join(indirect_doctest))
+        print 'Use "# indirect doctest" in the docstring to surpress this ' \
+                'warning'
+
+    print "SCORE %s: %s%% (%s of %s)" % (filename, score,
+            len(has_doctest), num_functions)
+
+    print '-'*70
+
+
+
+def go(file, verbose=False, exact=True):
+    if os.path.isdir(file):
+        for F in os.listdir(file):
+            go('%s/%s'%(file,F), verbose, exact=False)
+        return
+    if not (file.endswith('.py') or file.endswith('.pyx')) or \
+        not exact and ('test_' in file or 'bench_' in file):
+            return
+    if not os.path.exists(file):
+        print "File %s does not exist."%file
+        sys.exit(1)
+    f = open(file).read()
+    coverage(file, f, verbose)
+
+if __name__ == "__main__":
+    bintest_dir = os.path.abspath(os.path.dirname(__file__))   # bin/cover...
+    sympy_top  = os.path.split(bintest_dir)[0]      # ../
+    sympy_dir  = os.path.join(sympy_top, 'sympy')  # ../sympy/
+    if os.path.isdir(sympy_dir):
+        sys.path.insert(0, sympy_top)
+
+    parser = OptionParser()
+    parser.add_option("-v", "--verbose", action="store_true", dest="verbose",
+            default=False)
+
+    options, args = parser.parse_args()
+
+    if len(args) == 0:
+        parser.print_help()
+    else:
+        for file in args:
+            go(file, options.verbose)
diff --git a/lib/sympy/bin/coverage_report.py b/lib/sympy/bin/coverage_report.py
new file mode 100755
--- /dev/null
+++ b/lib/sympy/bin/coverage_report.py
@@ -0,0 +1,95 @@
+#!/usr/bin/env python
+"""
+Script to generate test coverage reports.
+
+Usage:
+
+$ bin/coverage_report.py
+
+This will create a directory covhtml with the coverage reports.To restrict the analysis
+to a directory, you just need to pass its name as
+argument. For example:
+
+$ bin/coverage_report.py sympy/logic
+
+runs only the tests in sympy/logic/ and reports only on the modules in
+sympy/logic/.  You can also get a report on the parts of the whole sympy code
+covered by the tests in sympy/logic/ by following up the previous command with
+
+$ bin/coverage_report.py -c
+
+"""
+import os, sys, re
+from optparse import OptionParser
+
+try:
+    import coverage
+except ImportError:
+    print "You need to install module coverage. See http://nedbatchelder.com/code/coverage/"
+    sys.exit(-1)
+
+REPORT_DIR = "covhtml"
+REFRESH = False
+
+omit_dir_patterns= ['.*tests', 'benchmark', 'examples',
+                    'mpmath', 'pyglet', 'test_external']
+omit_dir_re = re.compile(r'|'.join(omit_dir_patterns))
+source_re = re.compile(r'.*\.py$')
+
+def generate_covered_files(top_dir):
+    for dirpath, dirnames, filenames in os.walk(top_dir):
+        omit_dirs = [dirn for dirn in dirnames if omit_dir_re.match(dirn)]
+        for x in omit_dirs:
+            dirnames.remove(x)
+        for filename in filenames:
+            if source_re.match(filename):
+                yield os.path.join(dirpath, filename)
+
+
+def make_report(source_dir, report_dir, use_cache=False):
+    #code adapted from /bin/test
+    bin_dir = os.path.abspath(os.path.dirname(__file__))         # bin/
+    sympy_top  = os.path.split(bin_dir)[0]      # ../
+    sympy_dir  = os.path.join(sympy_top, 'sympy')  # ../sympy/
+    if os.path.isdir(sympy_dir):
+        sys.path.insert(0, sympy_top)
+    os.chdir(sympy_top)
+
+    cov = coverage.coverage()
+    cov.exclude("raise NotImplementedError")
+    cov.exclude("def canonize")      #this should be "@decorated"
+    if use_cache:
+        cov.load()
+    else:
+        cov.erase()
+        cov.start()
+        import sympy
+        sympy.test(source_dir)
+        #sympy.doctest()        #coverage doesn't play well with doctests
+        cov.stop()
+        cov.save()
+
+    covered_files = list(generate_covered_files(source_dir))
+
+    if report_dir in os.listdir(os.curdir):
+        for f in os.listdir(report_dir):
+            if f.split('.')[-1] in ['html', 'css', 'js']:
+                os.remove(os.path.join(report_dir, f))
+
+    cov.html_report(morfs=covered_files, directory=report_dir)
+
+if __name__ == '__main__':
+    parser = OptionParser()
+    parser.add_option('-c', '--use-cache', action='store_true', default=False,
+                        help='Use cached data.')
+    parser.add_option('-d', '--report-dir', default='covhtml',
+                        help='Directory to put the generated report in.')
+
+    options, args = parser.parse_args()
+
+    if args:
+        source_dir = args[0]
+    else:
+        source_dir = 'sympy/'
+
+    make_report(source_dir, **options.__dict__)
diff --git a/lib/sympy/bin/doctest b/lib/sympy/bin/doctest
new file mode 100755
--- /dev/null
+++ b/lib/sympy/bin/doctest
@@ -0,0 +1,59 @@
+#! /usr/bin/env python
+
+"""
+Program to execute doctests using the py.test like interface.
+
+The advantage over py.test is that it only depends on sympy and should just
+work in any circumstances. See "sympy.dotest?" for documentation.
+"""
+
+# files listed here can be in unix forward slash format with paths
+# listed relative to sympy (which contains bin, etc...)
+blacklist = []
+
+import sys
+import os
+from optparse import OptionParser
+
+from get_sympy import path_hack
+path_hack()
+
+parser = OptionParser()
+parser.add_option("-v", "--verbose", action="store_true", dest="verbose",
+        default=False)
+
+# if you don't see a -n `default=False`;
+# if you do see a -n `store_true` means to store a True value for it;
+# dest is where in options to put it, options.normal will hold the bool;
+# when the user enters -h or --help, print the `help` text
+parser.add_option("-n", "--normal", action="store_true", dest="normal",
+        help="run normal doctests; do not require explicit imports", default=False)
+parser.add_option('-t', '--types', dest='types', action='store',
+        default=None, choices=['gmpy', 'python', 'sympy'],
+        help='setup ground types: gmpy | python | sympy')
+parser.add_option('-C', '--no-cache', dest='cache', action='store_false',
+        default=True, help='disable caching mechanism')
+parser.set_usage("test [options ...] [files ...]")
+parser.epilog = """\
+"options" are any of the options above. "files" are 0 or more glob strings of \
+files to run doctests on. If no file arguments are given, all doctests will be \
+run. This program runs both doctests in the source and doctests in the Sphinx \
+documentation (doc/src/ directory).\
+"""
+
+options, args = parser.parse_args()
+
+if not options.cache:
+    os.environ['SYMPY_USE_CACHE'] = 'no'
+if options.types:
+    os.environ['SYMPY_GROUND_TYPES'] = options.types
+
+import sympy
+
+ok = sympy.doctest(*args, **{"verbose": options.verbose,
+    "blacklist": blacklist, "normal": options.normal})
+
+if ok:
+    sys.exit(0)
+else:
+    sys.exit(1)
diff --git a/lib/sympy/bin/generate_test_list.py b/lib/sympy/bin/generate_test_list.py
new file mode 100644
--- /dev/null
+++ b/lib/sympy/bin/generate_test_list.py
@@ -0,0 +1,62 @@
+"""
+Execute like this:
+
+$ python bin/generate_test_list.py
+tests = [
+    'sympy.concrete.tests',
+    'sympy.core.tests',
+    'sympy.functions.combinatorial.tests',
+    'sympy.functions.elementary.tests',
+    'sympy.functions.special.tests',
+    'sympy.geometry.tests',
+    'sympy.integrals.tests',
+    'sympy.matrices.tests',
+    'sympy.ntheory.tests',
+    'sympy.numerics.tests',
+    'sympy.parsing.tests',
+    'sympy.physics.tests',
+    'sympy.plotting.tests',
+    'sympy.polynomials.tests',
+    'sympy.printing.tests',
+    'sympy.series.tests',
+    'sympy.simplify.tests',
+    'sympy.solvers.tests',
+    'sympy.specfun.tests',
+    'sympy.statistics.tests',
+    'sympy.test_external',
+    'sympy.utilities.tests',
+    ]
+
+"""
+
+from glob import glob
+
+def get_paths(level=15):
+    """
+    Generates a set of paths for testfiles searching.
+
+    Example:
+    >>> get_paths(2)
+    ['sympy/test_*.py', 'sympy/*/test_*.py', 'sympy/*/*/test_*.py']
+    >>> get_paths(6)
+    ['sympy/test_*.py', 'sympy/*/test_*.py', 'sympy/*/*/test_*.py',
+    'sympy/*/*/*/test_*.py', 'sympy/*/*/*/*/test_*.py',
+    'sympy/*/*/*/*/*/test_*.py', 'sympy/*/*/*/*/*/*/test_*.py']
+
+    """
+    wildcards = ["/"]
+    for i in range(level):
+        wildcards.append(wildcards[-1]+"*/")
+    p = ["sympy"+x+"test_*.py" for x in wildcards]
+    return p
+
+g = []
+for x in get_paths():
+    g.extend(glob(x))
+g = [".".join(x.split("/")[:-1]) for x in g]
+g = list(set(g))
+g.sort()
+print "tests = ["
+for x in g:
+    print "    '%s'," % x
+print "    ]"
diff --git a/lib/sympy/bin/get_sympy.py b/lib/sympy/bin/get_sympy.py
new file mode 100644
--- /dev/null
+++ b/lib/sympy/bin/get_sympy.py
@@ -0,0 +1,13 @@
+"""Functions to get the correct sympy version to run tests."""
+
+import os
+import sys
+
+def path_hack():
+    """
+    Hack sys.path to import correct (local) sympy.
+    """
+    this_file = os.path.abspath(__file__)
+    sympy_dir = os.path.join(os.path.dirname(this_file), "..")
+    sympy_dir = os.path.normpath(sympy_dir)
+    sys.path.insert(0, sympy_dir)
diff --git a/lib/sympy/bin/isympy b/lib/sympy/bin/isympy
new file mode 100755
--- /dev/null
+++ b/lib/sympy/bin/isympy
@@ -0,0 +1,163 @@
+#! /usr/bin/env python
+
+"""Python shell for SymPy.
+
+   This is just a normal Python shell (IPython shell if you have the
+   IPython package installed),  that executes the following commands
+   for the user:
+
+       >>> from __future__ import division
+       >>> from sympy import *
+       >>> x, y, z, t = symbols('x y z t')
+       >>> k, m, n = symbols('k m n', integer=True)
+       >>> f, g, h = symbols('f g h', cls=Function)
+
+   So starting 'isympy' is equivalent to starting Python (or IPython)
+   and executing the above commands by hand.  It is intended for easy
+   and quick experimentation with SymPy.
+
+   COMMAND LINE OPTIONS
+   --------------------
+
+   -c CONSOLE, --console=CONSOLE
+
+     Use the specified Python or IPython shell as console backend instead
+     of the default one (IPython if present or Python otherwise), e.g.:
+
+        isympy -c python
+
+   -p PRETTY, --pretty PRETTY
+
+     Setup pretty printing in SymPy. By default the most pretty,  Unicode
+     printing is enabled. User can use less pretty ASCII printing instead
+     or no pretty printing at all, e.g.:
+
+   -q, --quiet
+
+     Print only Python's and SymPy's versions to stdout at startup.
+
+   -- IPython's options
+
+     Additionally you can pass command line options directly to IPython
+     interpreter (standard Python shell is not supported).  However you
+     need to add '--' separator between two types of options. To run
+     SymPy without startup banner and colors, for example, issue:
+
+        isympy -q -- -colors NoColor
+
+"""
+
+import os, sys
+
+# hook in-tree SymPy into Python path, if possible
+
+isympy_path = os.path.abspath(__file__)
+isympy_dir  = os.path.dirname(isympy_path)
+sympy_top   = os.path.split(isympy_dir)[0]
+sympy_dir   = os.path.join(sympy_top, 'sympy')
+
+if os.path.isdir(sympy_dir):
+    sys.path.insert(0, sympy_top)
+
+def main():
+    from optparse import OptionParser
+
+    usage = 'usage: isympy [options] -- [ipython options]'
+    parser = OptionParser(usage)
+
+    parser.add_option(
+        '-c', '--console',
+        dest='console',
+        action='store',
+        default=None,
+        choices=['ipython', 'python'],
+        help='select type of interactive session: ipython | python')
+
+    parser.add_option(
+        '-p', '--pretty',
+        dest='pretty',
+        action='store',
+        default=None,
+        choices=['unicode', 'ascii', 'no'],
+        help='setup pretty printing: unicode | ascii | no')
+
+    parser.add_option(
+        '-t', '--types',
+        dest='types',
+        action='store',
+        default=None,
+        choices=['gmpy', 'python', 'sympy'],
+        help='setup ground types: gmpy | python | sympy')
+
+    parser.add_option(
+        '-o', '--order',
+        dest='order',
+        action='store',
+        default=None,
+        choices=['lex', 'grlex', 'grevlex', 'rev-lex', 'rev-grlex', 'rev-grevlex', 'old', 'none'],
+        help='setup ordering of terms: [rev-]lex | [rev-]grlex | [rev-]grevlex | old | none')
+
+    parser.add_option(
+        '-q', '--quiet',
+        dest='quiet',
+        action='store_true',
+        default=False,
+        help='print only version information at startup')
+
+    parser.add_option(
+        '-d', '--doctest',
+        dest='doctest',
+        action='store_true',
+        default=False,
+        help='use the doctest format for output (you can just copy and paste it)')
+
+    parser.add_option(
+        '-C', '--no-cache',
+        dest='cache',
+        action='store_false',
+        default=True,
+        help='disable caching mechanism')
+
+    (options, ipy_args) = parser.parse_args()
+
+    if not options.cache:
+        os.environ['SYMPY_USE_CACHE'] = 'no'
+
+    if options.types:
+        os.environ['SYMPY_GROUND_TYPES'] = options.types
+
+    if options.doctest:
+        options.pretty = 'no'
+        options.console = 'python'
+
+    session = options.console
+
+    if session is not None:
+        ipython = session == 'ipython'
+    else:
+        ipython = None
+
+    args = {
+        'pretty_print' : True,
+        'use_unicode'  : None,
+        'order'        : None,
+        'argv'         : ipy_args,
+    }
+
+    if options.pretty == 'unicode':
+        args['use_unicode'] = True
+    elif options.pretty == 'ascii':
+        args['use_unicode'] = False
+    elif options.pretty == 'no':
+        args['pretty_print'] = False
+
+    if options.order is not None:
+        args['order'] = options.order
+
+    args['quiet'] = options.quiet
+
+    from sympy.interactive import init_session
+    init_session(ipython, **args)
+
+if __name__ == "__main__":
+    main()
diff --git a/lib/sympy/bin/py.bench b/lib/sympy/bin/py.bench
new file mode 100755
--- /dev/null
+++ b/lib/sympy/bin/py.bench
@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+
+# hook in-tree SymPy into Python path, if possible
+# TODO this should be shared with isympy
+import os, sys
+
+isympy_dir = os.path.dirname(__file__)         # bin/isympy
+sympy_top  = os.path.split(isympy_dir)[0]      # ../
+sympy_dir  = os.path.join(sympy_top, 'sympy')  # ../sympy/
+
+if os.path.isdir(sympy_dir):
+    sys.path.insert(0, sympy_top)
+
+
+from sympy.utilities import benchmarking
+
+if __name__ == '__main__':
+    benchmarking.main()
diff --git a/lib/sympy/bin/strip_whitespace b/lib/sympy/bin/strip_whitespace
new file mode 100755
--- /dev/null
+++ b/lib/sympy/bin/strip_whitespace
@@ -0,0 +1,79 @@
+#!/usr/bin/env python
+
+import os
+
+def strip_file(filename, write, report):
+    # .readlines() retains \n chars, while .read().splitlines() does not.
+    # Assuming that whitespace errors will be few, we will thus only need
+    # to re-add \n to a few right-stripped lines. The hit flag will keep us
+    # from unecessarily re-writing files with no changes.
+
+    lines = open(filename, 'rb').readlines() # without "b" all lines appear as \n terminated
+    hit = False
+    cr = False
+    for index, line in enumerate(lines):
+        if line.endswith(" \n"):
+            if report:
+                print "%s, line %s" % (filename, index + 1)
+            if write:
+                lines[index] = line.rstrip() + "\n"
+                hit = True
+        if line.endswith("\r\n"):
+            if report and not cr:
+                print "%s, line %s (crlf now silent)" % (filename, index + 1)
+                cr = True
+            if write:
+                lines[index] = line.rstrip() + "\n"
+                hit = True
+
+    # correct no newline at eof
+    if lines and not lines[-1].endswith("\n"):
+        lines[-1] += "\n"
+        if report:
+            print "%s, no newline at eof" % filename
+        if write:
+            hit = True
+
+    if write and hit:
+        f = open(filename, "wb") # without "b" the lines may be written in sys-dep format
+        f.writelines(lines)
+        f.close()
+
+def go(path, write, report):
+    allowed_ext = [
+            ".cpp",
+            ".cc",
+            ".h",
+            ".py",
+            ]
+    for root, dirs, files in os.walk(path):
+        if ".git" in dirs:
+            dirs.remove(".git")
+        for file in files:
+            if os.path.splitext(file)[1] not in allowed_ext:
+                continue
+            filename = os.path.join(root, file)
+            strip_file(filename, write, report)
+
+def main():
+    from optparse import OptionParser
+    p = OptionParser("usage: %prog [options] filename")
+    p.add_option("-d", "--dry", action="store_true", dest="dry",
+                 help="Do not modify files.")
+    p.add_option("-v", "--verbose", action="store_true", dest="verbose",
+                 help="Report all changes.")
+    p.add_option("-r", "--recursive", action="store_true", dest="recursive",
+                 help="Recursively correct all files in a directory.")
+    options, args = p.parse_args()
+    if options.dry:
+        options.verbose = True
+    if len(args) == 1:
+        if options.recursive:
+            go(args[0], not options.dry, options.verbose)
+        else:
+            strip_file(args[0], not options.dry, options.verbose)
+    else:
+        p.print_help()
+
+if __name__ == "__main__":
+    main()
diff --git a/lib/sympy/bin/sympy_time.py b/lib/sympy/bin/sympy_time.py
new file mode 100644
--- /dev/null
+++ b/lib/sympy/bin/sympy_time.py
@@ -0,0 +1,49 @@
+import time
+from get_sympy import path_hack
+path_hack()
+
+seen = set()
+import_order = []
+elapsed_times = {}
+level = 0
+parent = None
+children = {}
+
+def new_import(name, globals={}, locals={}, fromlist=[]):
+    global level, parent
+    if name in seen:
+        return old_import(name, globals, locals, fromlist)
+    seen.add(name)
+    import_order.append((name, level, parent))
+    t1 = time.time()
+    old_parent = parent
+    parent = name
+    level += 1
+    module = old_import(name, globals, locals, fromlist)
+    level -= 1
+    parent = old_parent
+    t2 = time.time()
+    elapsed_times[name] = t2-t1
+    return module
+
+old_import = __builtins__.__import__
+
+__builtins__.__import__ = new_import
+from sympy import *
+
+parents = {}
+is_parent = {}
+for name, level, parent in import_order:
+    parents[name] = parent
+    is_parent[parent] = True
+
+print "== Tree =="
+for name, level, parent in import_order:
+    print "%s%s: %.3f (%s)" % (" "*level, name, elapsed_times.get(name,0),
+            parent)
+
+print "\n"
+print "== Slowest (including children) =="
+slowest = sorted((t, name) for (name, t) in elapsed_times.items())[-50:]
+for elapsed_time, name in slowest[::-1]:
+    print "%.3f %s (%s)" % (elapsed_time, name, parents[name])
diff --git a/lib/sympy/bin/sympy_time_cache.py b/lib/sympy/bin/sympy_time_cache.py
new file mode 100644
--- /dev/null
+++ b/lib/sympy/bin/sympy_time_cache.py
@@ -0,0 +1,128 @@
+import time, timeit
+
+class TreeNode(object):
+    def __init__(self, name):
+        self._name = name
+        self._children = []
+        self._time = 0
+
+    def __str__(self):
+        return "%s: %s"%(self._name, self._time)
+
+    __repr__ = __str__
+
+    def add_child(self, node):
+        self._children.append(node)
+
+    def children(self):
+        return self._children
+
+    def child(self, i):
+        return self.children()[i]
+
+    def set_time(self, time):
+        self._time = time
+
+    def time(self):
+        return self._time
+
+    total_time = time
+
+    def exclusive_time(self):
+        return self.total_time() - sum(child.time() for child in self.children())
+
+    def name(self):
+        return self._name
+
+    def linearize(self):
+        res = [self]
+        for child in self.children():
+            res.extend(child.linearize())
+        return res
+
+    def print_tree(self, level=0, max_depth=None):
+        print "  "*level + str(self)
+        if max_depth is not None and max_depth <= level:
+            return
+        for child in self.children():
+            child.print_tree(level+1, max_depth=max_depth)
+
+    def print_generic(self, n=50, method="time"):
+        slowest = sorted((getattr(node, method)(), node.name()) for node in self.linearize())[-n:]
+        for time, name in slowest[::-1]:
+            print "%s %s"%(time, name)
+
+    def print_slowest(self, n=50):
+        self.print_generic(n=50, method="time")
+
+    def print_slowest_exclusive(self, n=50):
+        self.print_generic(n, method="exclusive_time")
+
+    def write_cachegrind(self, f):
+        if isinstance(f, str):
+            f = open(f, "w")
+            f.write("events: Microseconds\n")
+            f.write("fl=sympyallimport\n")
+            must_close = True
+        else:
+            must_close = False
+
+        f.write("fn=%s\n"%self.name())
+        f.write("1 %s\n"%self.exclusive_time())
+
+        counter = 2
+        for child in self.children():
+            f.write("cfn=%s\n"%child.name())
+            f.write("calls=1 1\n")
+            f.write("%s %s\n"%(counter, child.time()))
+            counter += 1
+
+        f.write("\n\n")
+
+        for child in self.children():
+            child.write_cachegrind(f)
+
+        if must_close:
+            f.close()
+
+
+pp = TreeNode(None)  #We have to use pp since there is a sage function
+                     #called parent that gets imported
+seen = set()
+
+def new_import(name, globals={}, locals={}, fromlist=[]):
+    global pp
+    if name in seen:
+        return old_import(name, globals, locals, fromlist)
+    seen.add(name)
+
+    node = TreeNode(name)
+
+    pp.add_child(node)
+    old_pp = pp
+    pp = node
+
+    #Do the actual import
+    t1 = timeit.default_timer()
+    module = old_import(name, globals, locals, fromlist)
+    t2 = timeit.default_timer()
+    node.set_time(int(1000000*(t2-t1)))
+
+
+    pp = old_pp
+
+    return module
+
+old_import = __builtins__.__import__
+
+__builtins__.__import__ = new_import
+old_sum = sum
+
+from sympy import *
+
+sum = old_sum
+
+sageall = pp.child(0)
+sageall.write_cachegrind("sympy.cachegrind")
+
+print "Timings saved. Do:\n$ kcachegrind sympy.cachegrind"
diff --git a/lib/sympy/bin/test b/lib/sympy/bin/test
new file mode 100755
--- /dev/null
+++ b/lib/sympy/bin/test
@@ -0,0 +1,62 @@
+#! /usr/bin/env python
+
+"""
+Program to execute tests using the py.test like interface.
+
+The advantage over py.test is that it only depends on sympy and should just
+work in any circumstances. See "sympy.test?" for documentation.
+"""
+
+import sys
+import os
+from optparse import OptionParser
+
+from get_sympy import path_hack
+path_hack()
+
+parser = OptionParser()
+parser.add_option("-v", "--verbose", action="store_true", dest="verbose",
+        default=False)
+parser.add_option("--pdb", action="store_true", dest="pdb",
+        default=False, help="Run post mortem pdb on each failure")
+parser.add_option("--no-colors", action="store_false", dest="colors",
+        default=True, help="Do not report colored [OK] and [FAIL]")
+parser.add_option("-k", dest="kw",
+        help="only run tests matching the given keyword expression",
+        metavar="KEYWORD", default="")
+parser.add_option("--tb", dest="tb",
+        help="traceback verboseness (short/no) [default: %default]",
+        metavar="TBSTYLE", default="short")
+parser.add_option("--random", action="store_false", dest="sort", default=True,
+        help="Run tests in random order instead of sorting them")
+parser.add_option("--seed", dest="seed", type="int",
+        help="use this seed for randomised tests",
+        metavar="SEED")
+parser.add_option('-t', '--types', dest='types', action='store',
+        default=None, choices=['gmpy', 'python', 'sympy'],
+        help='setup ground types: gmpy | python | sympy')
+parser.add_option('-C', '--no-cache', dest='cache', action='store_false',
+        default=True, help='disable caching mechanism')
+parser.set_usage("test [options ...] [tests ...]")
+parser.epilog = """\
+"options" are any of the options above. "tests" are 0 or more glob strings of \
+tests to run. If no test arguments are given, all tests will be run.\
+"""
+
+options, args = parser.parse_args()
+
+if not options.cache:
+    os.environ['SYMPY_USE_CACHE'] = 'no'
+if options.types:
+    os.environ['SYMPY_GROUND_TYPES'] = options.types
+
+import sympy
+
+ok = sympy.test(*args, **{"verbose": options.verbose, "kw": options.kw,
+    "tb": options.tb, "pdb": options.pdb, "colors": options.colors,
+    "sort": options.sort, "seed": options.seed})
+
+if ok:
+    sys.exit(0)
+else:
+    sys.exit(1)
diff --git a/lib/sympy/bin/test_import b/lib/sympy/bin/test_import
new file mode 100755
--- /dev/null
+++ b/lib/sympy/bin/test_import
@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+
+"""
+Tests the speed of "import sympy" by measuring it many times in a row and
+averaging the values.
+
+Usage:
+
+$ bin/test_import
+"""
+
+n_tests = 50
+
+from pexpect import run
+from numpy import mean, std
+
+def test():
+    t = run("python bin/test_import.py")
+    t = float(t)
+    return t
+
+tests = [test() for x in range(n_tests+1)]
+print "Note: the first run (warm up) was not included in the average + std dev"
+print "All runs (including warm up):"
+print tests
+# skip the first run (warm up):
+tests = tests[1:]
+print "Number of tests: %d" % (n_tests)
+print 'The speed of "import sympy" is: %f +- %f' % (mean(tests), std(tests))
diff --git a/lib/sympy/bin/test_import.py b/lib/sympy/bin/test_import.py
new file mode 100644
--- /dev/null
+++ b/lib/sympy/bin/test_import.py
@@ -0,0 +1,7 @@
+from timeit import default_timer as clock
+from get_sympy import path_hack
+path_hack()
+t = clock()
+import sympy
+t = clock()-t
+print t
diff --git a/lib/sympy/bin/test_isolated b/lib/sympy/bin/test_isolated
new file mode 100755
--- /dev/null
+++ b/lib/sympy/bin/test_isolated
@@ -0,0 +1,115 @@
+#! /usr/bin/env python
+"""
+Generates a bash script, that executes py.test or nosetest on each file
+individually.
+
+Usage and help:
+
+    $ bin/test_isolated -h
+
+and read the instructions.
+"""
+
+from os import chmod, getcwd
+from glob import glob
+from optparse import OptionParser
+from stat import S_IREAD, S_IWRITE, S_IXUSR, S_IRGRP, S_IROTH, S_IXGRP, S_IXOTH
+
+filemode = S_IREAD | S_IWRITE | S_IXUSR | S_IRGRP | S_IROTH | S_IXGRP | S_IXOTH
+
+def get_paths(level=6):
+    """
+    Generates a set of paths for testfiles searching.
+
+    Example:
+    >>> get_paths(2)
+    ['sympy/test_*.py', 'sympy/*/test_*.py', 'sympy/*/*/test_*.py']
+    >>> get_paths(6)
+    ['sympy/test_*.py', 'sympy/*/test_*.py', 'sympy/*/*/test_*.py',
+    'sympy/*/*/*/test_*.py', 'sympy/*/*/*/*/test_*.py',
+    'sympy/*/*/*/*/*/test_*.py', 'sympy/*/*/*/*/*/*/test_*.py']
+    """
+    wildcards = ["/"]
+    for i in range(level):
+        wildcards.append(wildcards[-1]+"*/")
+    my_dir = getcwd()
+    p = [my_dir+"/sympy"+x+"test_*.py" for x in wildcards]
+    return p
+
+def generate_test_script1(testlib="py.test"):
+    """Generates a bash script for doing the test.
+
+    "testlib" is the name of the executable, that is going to execute the test,
+    for example "py.test" or "nosetests".
+    """
+    g = []
+    for x in get_paths(10):
+        g.extend(glob(x))
+    f = open("/tmp/test_sympy.sh","w")
+    f.write("#! /bin/sh\n")
+    f.write("# Autogenerated script for a reliable test of SymPy.\n")
+    f.write("# Execute with 'sh test_sympy.sh' (in any directory).\n\n")
+    for x in g:
+        f.write(testlib+" " + x + "\n")
+    chmod(f.name, filemode)
+
+def generate_test_script2(testlib="nosetests"):
+    """Generates a bash script for doing the test.
+
+    "testlib" is the name of the executable, that is going to execute the test,
+    for example "py.test" or "nosetests".
+    """
+    g = []
+    for x in get_paths(10):
+        g.extend(glob(x))
+    f = open("/tmp/test_sympy.sh","w")
+    f.write("#! /bin/sh\n")
+    f.write("# Autogenerated script for a reliable test of SymPy.\n")
+    f.write("# Execute with 'sh test_sympy.sh' (in any directory).\n\n")
+    for x in g:
+        f.write(testlib+" " + x + " && \\\n")
+    f.write("echo 'all tests passed, ok to commit'")
+    chmod(f.name, filemode)
+
+usage = """%prog [options]
+
+Generates a bash script, that executes py.test or nosetest on each file
+individually.
+
+Usage:
+
+    $ bin/test_isolated
+    Generating py.test isolated testsuite...
+    Done. Run (search for 'COMMIT' in the less environment):
+    /tmp/test_sympy.sh | less
+    $ /tmp/test_sympy.sh | less
+
+Let the tests run and then search if any test failed (it will write DO NOT
+COMMIT), so search for COMMIT."""
+
+def main():
+    parser = OptionParser(usage=usage)
+    parser.add_option("-p","--py.test", action="store_false", dest="nosetests",
+            help="Use py.test (default)")
+    parser.add_option("-n","--nosetests", action="store_true", dest="nosetests",
+            help="Use nosetests")
+    parser.add_option("-q","--quiet", action="store_false", dest="verbose")
+    parser.set_defaults(nosetests=False, verbose=True)
+
+    options, args = parser.parse_args()
+    if len(args) != 0:
+        parser.error("too many arguments")
+    if options.nosetests:
+        if options.verbose:
+            print "Generating nosetests isolated testsuite..."
+        generate_test_script2("nosetests")
+    else:
+        if options.verbose:
+            print "Generating py.test isolated testsuite..."
+        generate_test_script1("py.test")
+    if options.verbose:
+        print "Done. Run (search for 'COMMIT' in the less environment):"
+        print "/tmp/test_sympy.sh | less"
+
+if __name__ == "__main__":
+    main()
diff --git a/lib/sympy/bin/use2to3 b/lib/sympy/bin/use2to3
new file mode 100755
--- /dev/null
+++ b/lib/sympy/bin/use2to3
@@ -0,0 +1,130 @@
+#!/usr/bin/env python
+
+"""
+This script converts SymPy code to a Python 3-compatible version.
+
+The script copies all files except the ones related to mpmath to a sympy-py3k
+directory, runs 2to3 on them and then copies the vanilla mpmath files over. We
+need this because running 2to3 on mpmath (which is already Python 3 compatible)
+produces errors. You can then use SymPy normally from the sympy-py3k directory
+(installing it or importing it directly).
+
+Because copying and running 2to3 can take a lot of time, we try to do it only on
+files that have been modified since the last run.
+
+Note that the 2to3 shipped with Python 2.6 crashes when converting doctests. It
+is recommended to use the Python 3.2 version (or newer) as it is much faster.
+
+TODO: Add --destination argument (others?)
+ --destination # copy over the source to a user-specified destination
+"""
+import os
+import fnmatch
+import shutil
+
+destination = "sympy-py3k" # directory to copy to
+
+# TODO: build this from .gitignore
+skip_dirs = (
+    '.*',       # skip hidden dirs; .git and .tox in particular can be quite big
+    'mpmath',   # everything related to mpmath, both in doc/ and sympy/
+    '_build',   # files built by Sphinx
+    '__pycache__',
+    'covhtml',  # files produced by bin/test_coverage
+    'my',       # the user can have stuff here we don't want to copy
+    destination # this prevents infinite recursion if the dir already exists
+    )
+
+skip_files = (
+    '*.pyc',
+    '.*',
+    'ast_parser_python25.py', # this files produces doctest errors under py3k
+                              # as we need it only for 2.5, just skip copying it
+    )
+
+modified_files = []
+modified_txt_files = []
+
+# we need to run 2to3 on .txt files; however, not all .txt files are doctests,
+# so we need a list of files we care about
+relevant_txt_files = []
+
+# generate the relevant txt files
+# most of them should be in this directory:
+for root, dirs, files in os.walk('./doc/src/modules'):
+    # NOTE: this will consider mpmath-related files relevant, but it doesn't matter
+    for filename in fnmatch.filter(files, '*.txt'):
+        relevant_txt_files.append(os.path.join(root,filename))
+
+# some files aren't in /doc/src/modules, add them explicitly
+relevant_txt_files.append('./doc/src/tutorial.txt')
+relevant_txt_files.append('./doc/src/gotchas.txt')
+relevant_txt_files.append('./doc/src/guide.txt')
+relevant_txt_files.append('./doc/src/python-comparisons.txt')
+
+# walk the tree and copy over files as necessary
+for root, dirs, files in os.walk('.'):
+    for pattern in skip_dirs:
+        for directory in fnmatch.filter(dirs, pattern):
+            dirs.remove(directory)
+    for pattern in skip_files:
+        for filename in fnmatch.filter(files, pattern):
+            files.remove(filename)
+    for directory in dirs:
+        dstdir = os.path.join(destination, root, directory)
+        if not os.path.exists(dstdir):
+            os.makedirs(dstdir)
+    for filename in files:
+        src = os.path.join(root, filename)
+        dst = os.path.join(destination, root, filename)
+        if os.path.isfile(dst):
+            if os.path.getmtime(src) - os.path.getmtime(dst) < 1:
+                # the file hasn't been modified since the last run, so skip it
+                # we check for one second of difference because Python can be
+                # imprecise (when copying) with smaller time periods
+                continue
+        shutil.copy2(src, dst)
+        # add to the list of files to pass to 2to3 if needed
+        if filename.endswith(".py"):
+            modified_files.append(dst)
+        elif filename.endswith(".txt"):
+            # we need to check the exact path here, not just the filename
+            # as there are eg. multiple index.txt files and not all are relevant
+            if src in relevant_txt_files:
+                modified_txt_files.append(dst)
+
+
+# arguments to call 2to3 with
+args_2to3 = [
+    "-w",         # writes back the changes
+    "-n",         # doesn't write a backup file
+    "--no-diffs", # don't show the diffs for individual files
+]
+
+args_2to3_doctests = args_2to3 + ["-d"] # convert doctests too
+
+# extend the argument list with the list of files that need it
+args_2to3.extend(modified_files)
+args_2to3_doctests.extend(modified_files)
+args_2to3_doctests.extend(modified_txt_files)
+
+# call 2to3, once for regular files and once for doctests
+from lib2to3.main import main as main2to3
+main2to3("lib2to3.fixes", args_2to3)
+main2to3("lib2to3.fixes", args_2to3_doctests)
+
+# once we are finished with everything, we should finally copy over the files
+# provided by mpmath; these should all be in the following two directories
+
+# to skip checking if something has been updated, just copy everything always
+# the main bottleneck is running 2to3, not copying files
+# TODO: only copy updated files; this would need a heavy modification to the
+#       above code, or copy-pasting the relevant part over
+try:
+    shutil.rmtree(os.path.join(destination, "./sympy/mpmath"))
+    shutil.rmtree(os.path.join(destination, "./doc/src/modules/mpmath"))
+except OSError: # directories don't exist
+    pass
+
+shutil.copytree("sympy/mpmath", os.path.join(destination, "./sympy/mpmath"))
+shutil.copytree("doc/src/modules/mpmath", os.path.join(destination, "./doc/src/modules/mpmath"))
diff --git a/lib/sympy/build.py b/lib/sympy/build.py
new file mode 100755
--- /dev/null
+++ b/lib/sympy/build.py
@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+
+import os
+
+from Cython.Compiler.Main import compile
+
+from distutils.core import setup, Extension
+from distutils.command.build_ext import build_ext
+
+source_root = os.path.dirname(__file__)
+
+compiled_modules = [
+    "sympy.polys.densearith",
+    "sympy.polys.densebasic",
+    "sympy.polys.densetools",
+    "sympy.polys.euclidtools",
+    "sympy.polys.factortools",
+    "sympy.polys.galoistools",
+    "sympy.polys.monomialtools",
+    "sympy.polys.orthopolys",
+    "sympy.polys.specialpolys",
+    "sympy.polys.sqfreetools",
+]
+
+extensions = []
+
+for module in compiled_modules:
+    source_file = os.path.join(source_root, *module.split('.')) + ".py"
+
+    print("Compiling module %s ..." % module)
+    result = compile(source_file)
+
+    if result.c_file is None:
+        raise RuntimeError("failed to compile %s" % module)
+
+    extensions.append(
+        Extension(module, sources=[str(result.c_file)],
+            extra_compile_args=['-O2', '-Wall'],
+        )
+    )
+
+setup(
+    name        = "SymPy",
+    packages    = [
+        "sympy",
+        "sympy.polys",
+    ],
+    cmdclass    = {
+        "build_ext": build_ext
+    },
+    ext_modules = extensions
+)
+
diff --git a/lib/sympy/data/IPython/ipythonrc-sympy b/lib/sympy/data/IPython/ipythonrc-sympy
new file mode 100644
--- /dev/null
+++ b/lib/sympy/data/IPython/ipythonrc-sympy
@@ -0,0 +1,31 @@
+# -*- Mode: Shell-Script -*-  Not really, but shows comments correctly
+#*******************************************************************************
+# Configuration file for ipython -- ipythonrc format
+#
+# The format of this file is one of 'key value' lines.
+#
+# Lines containing only whitespace at the beginning and then a '#' are
+# ignored as comments.  But comments can NOT be put on lines with data.
+#*******************************************************************************
+#
+# This configuration file is  a  customization to turn IPython into a
+# very capable environment for high-end purely symbolic and numerical
+# work, using SymPy, similar to Mathematica, AXIOM etc.  but with the
+# beauty and flexibility of the Python language.
+#
+# SymPy is a Python library for symbolic mathematics. It aims to become
+# a  full-featured computer algebra system (CAS) while keeping the code
+# as simple as possible in order to be comprehensible and easily exten-
+# sible.  SymPy is written entirely in  Python and does not require any
+# external libraries.
+#
+# For more information visit www.sympy.org
+#
+
+include ipythonrc
+
+banner 0
+
+import_all sympy.interactive
+
+execute init_session()
diff --git a/lib/sympy/data/TeXmacs/LICENSE b/lib/sympy/data/TeXmacs/LICENSE
new file mode 100644
--- /dev/null
+++ b/lib/sympy/data/TeXmacs/LICENSE
@@ -0,0 +1,28 @@
+Copyright (c) 2006, 2007, 2008, 2009, 2010, 2011 SymPy developers
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+  a. Redistributions of source code must retain the above copyright notice,
+     this list of conditions and the following disclaimer.
+  b. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+  c. Neither the name of the SymPy nor the names of its contributors
+     may be used to endorse or promote products derived from this software
+     without specific prior written permission.
+
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGE.
diff --git a/lib/sympy/data/TeXmacs/bin/tm_sympy b/lib/sympy/data/TeXmacs/bin/tm_sympy
new file mode 100755
--- /dev/null
+++ b/lib/sympy/data/TeXmacs/bin/tm_sympy
@@ -0,0 +1,97 @@
+#!/usr/bin/python
+#
+# TeXmacs interface for SymPy
+#
+# This plugin supports LaTeX printing of SymPy expressions
+# using sympy.printing.latex function. It handles pretty
+# printing of containers. If you wish to display raw
+# Python code, then use 'print' before an expression.
+#
+# TeXmacs encodes new-lines as spaces so we must use
+# heuristics to know where a multi-line expression is
+# broken. As a result you can't use more than one space
+# in a sequence. However you can and must indent your
+# expression using standard Pyhon rules.
+#
+# You can retrive the last output using '_' built-in
+# symbol. If the previous command did not generate
+# any ouput, it will be assigned with None.
+#
+# For a complete Python interface visit:
+#
+#  http://dkbza.org/tmPython.html
+#
+
+import os
+import re
+import traceback
+
+from sympy import __version__
+from sympy.printing import latex
+
+BEGIN, END = chr(2), chr(5)
+
+DATA_COMMAND = chr(16)
+DATA_ESCAPE = chr(27)
+
+def send(kind, output=None):
+    if output is None:
+        output = ""
+    elif kind == "latex":
+        output = latex(output)
+
+    message = "%s:%s" %  (kind, output)
+
+    os.sys.stdout.write(BEGIN)
+    os.sys.stdout.write(message)
+    os.sys.stdout.write(END)
+    os.sys.stdout.flush()
+
+send("verbatim", "Welcome to SymPy " + __version__)
+
+_globals = {}
+
+_init = \
+"""
+from sympy import *
+
+_ = None
+
+x, y, z, t = symbols('x,y,z,t')
+k, i, m, n = symbols('k,i,m,n', integer=True)
+
+f = Function('f')
+
+Gamma, Zeta = gamma, zeta
+
+_greek = 'alpha beta gamma delta epsilon zeta eta '  \
+         'theta iota kappa mu nu xi omicron pi rho ' \
+         'sigma tau upsilon phi chi psi omega'
+
+for _symbol in _greek.split(' '):
+    exec "%s = Symbol('%s')" % (_symbol, _symbol)
+
+del _symbol
+"""
+
+eval(compile(_init, 'tm_sympy', 'exec'), _globals)
+
+while True:
+    line = os.sys.stdin.readline().strip()
+
+    if not line:
+        send("verbatim")
+    elif line[0] != DATA_COMMAND:
+        line = re.sub(r' {2}(\s*)', r'\n \1', line)
+
+        try:
+            output = eval(line, _globals)
+        except:
+            try:
+                output = eval(compile(line, 'tm_sympy', 'exec'), _globals)
+            except:
+                send("verbatim", traceback.format_exc(limit = 0))
+                continue
+
+        _globals['_'] = output
+        send("latex", output)
diff --git a/lib/sympy/data/TeXmacs/progs/init-sympy.scm b/lib/sympy/data/TeXmacs/progs/init-sympy.scm
new file mode 100755
--- /dev/null
+++ b/lib/sympy/data/TeXmacs/progs/init-sympy.scm
@@ -0,0 +1,4 @@
+(plugin-configure sympy
+  (:require (url-exists-in-path? "tm_sympy"))
+  (:launch "tm_sympy --texmacs")
+  (:session "SymPy"))
diff --git a/lib/sympy/doc/Makefile b/lib/sympy/doc/Makefile
new file mode 100644
--- /dev/null
+++ b/lib/sympy/doc/Makefile
@@ -0,0 +1,96 @@
+# Makefile for Sphinx documentation
+#
+PYTHON       = python
+RST2HTML     = rst2html
+
+# You can set these variables from the command line.
+SPHINXOPTS   =
+SPHINXVER    = 0.5
+#SPHINXBUILDpy  = sphinx/Sphinx-$(SPHINXVER)/sphinx-build.py
+SPHINXBUILDpy  = sphinx-build
+SPHINXBUILD  = PYTHONPATH=..:$(PYTHONPATH) $(SPHINXBUILDpy)
+PAPER        =
+
+ALLSPHINXOPTS = -d _build/doctrees -D latex_paper_size=$(PAPER) \
+                $(SPHINXOPTS) src
+
+ALLSPHINXOPTSapi = -d _build/doctreesapi -D latex_paper_size=$(PAPER) \
+                $(SPHINXOPTS) api
+
+.PHONY: help clean html web htmlhelp latex changes linkcheck
+
+help:
+	@echo "Please use \`make <target>' where <target> is one of"
+	@echo "  html      to make standalone HTML files"
+	@echo "  web       to make files usable by Sphinx.web"
+	@echo "  htmlhelp  to make HTML files and a HTML help project"
+	@echo "  latex     to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
+	@echo "  changes   to make an overview over all changed/added/deprecated items"
+	@echo "  linkcheck to check all external links for integrity"
+
+clean:
+	-rm -rf _build
+	-rm -rf sphinx
+
+$(SPHINXBUILDpy):
+	rm -rf sphinx
+#	mkdir sphinx
+#	cd sphinx; wget http://pypi.python.org/packages/source/S/Sphinx/Sphinx-$(SPHINXVER).tar.gz;
+#	cd sphinx; tar xzf Sphinx-$(SPHINXVER).tar.gz
+
+html: $(SPHINXBUILDpy)
+	mkdir -p src/.static
+	mkdir -p _build/html
+	mkdir -p _build/doctrees
+	mkdir -p src/modules
+	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) _build/html
+	cp -r src/pics _build/html/
+	cp -r src/figures _build/html/
+	@echo
+	@echo "Build finished. The HTML pages are in _build/html."
+
+htmlapi: $(SPHINXBUILDpy)
+	mkdir -p api/.static
+	mkdir -p api/modules
+	mkdir -p _build/api _build/doctreesapi
+	rm -f api/modules/sympy*.txt
+	./generate_reference.py
+	$(SPHINXBUILD) -b html $(ALLSPHINXOPTSapi) _build/api
+	@echo
+	@echo "Build finished. The API docs pages are in _build/api."
+
+web:
+	mkdir -p _build/web _build/doctrees
+	$(SPHINXBUILD) -b web $(ALLSPHINXOPTS) _build/web
+	@echo
+	@echo "Build finished; now you can run"
+	@echo "  python -m sphinx.web _build/web"
+	@echo "to start the server."
+
+htmlhelp:
+	mkdir -p _build/htmlhelp _build/doctrees
+	$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) _build/htmlhelp
+	@echo
+	@echo "Build finished; now you can run HTML Help Workshop with the" \
+	      ".hhp project file in _build/htmlhelp."
+
+latex: $(SPHINXBUILDpy)
+	mkdir -p _build/latex _build/doctrees
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) _build/latex
+	@echo
+	@echo "Build finished; the LaTeX files are in _build/latex."
+	@echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \
+	      "run these through (pdf)latex."
+
+changes:
+	mkdir -p _build/changes _build/doctrees
+	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) _build/changes
+	@echo
+	@echo "The overview file is in _build/changes."
+
+linkcheck:
+	mkdir -p _build/linkcheck _build/doctrees
+	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) _build/linkcheck
+	@echo
+	@echo "Link check complete; look for any errors in the above output " \
+	      "or in _build/linkcheck/output.txt."
diff --git a/lib/sympy/doc/README b/lib/sympy/doc/README
new file mode 100644
--- /dev/null
+++ b/lib/sympy/doc/README
@@ -0,0 +1,12 @@
+To make the html documentation, install the prerequisites, e.g. on
+Debian/Ubuntu (similarly for other distributions):
+
+apt-get install python-sphinx texlive-latex-recommended dvipng
+
+and do:
+
+$ make html
+
+and to view it, do:
+
+$ epiphany _build/html/index.html
diff --git a/lib/sympy/doc/api/conf.py b/lib/sympy/doc/api/conf.py
new file mode 100644
--- /dev/null
+++ b/lib/sympy/doc/api/conf.py
@@ -0,0 +1,133 @@
+# -*- coding: utf-8 -*-
+#
+# SymPy documentation build configuration file, created by
+# sphinx-quickstart.py on Sat Mar 22 19:34:32 2008.
+#
+# This file is execfile()d with the current directory set to its containing dir.
+#
+# The contents of this file are pickled, so don't put values in the namespace
+# that aren't pickleable (module imports are okay, they're removed automatically).
+#
+# All configuration values have a default value; values that are commented out
+# serve to show the default value.
+
+import sys
+
+# If your extensions are in another directory, add it here.
+#sys.path.append('some/directory')
+
+# General configuration
+# ---------------------
+
+# Add any Sphinx extension module names here, as strings. They can be extensions
+# coming with Sphinx (named 'sphinx.addons.*') or your custom ones.
+extensions = ['sphinx.ext.autodoc']
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['.templates']
+
+# The suffix of source filenames.
+source_suffix = '.txt'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General substitutions.
+project = 'SymPy'
+copyright = '2011, SymPy Development Team'
+
+# The default replacements for |version| and |release|, also used in various
+# other places throughout the built documents.
+#
+# The short X.Y version.
+version = '0.7.1'
+# The full version, including alpha/beta/rc tags.
+release = '0.7.1-git'
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+#today = ''
+# Else, today_fmt is used as the format for a strftime call.
+today_fmt = '%B %d, %Y'
+
+# List of documents that shouldn't be included in the build.
+#unused_docs = []
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+#add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+#add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+#show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+
+# Options for HTML output
+# -----------------------
+
+# The style sheet to use for HTML and HTML Help pages. A file of that name
+# must exist either in Sphinx' static/ path, or in one of the custom paths
+# given in html_static_path.
+html_style = 'default.css'
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['.static']
+
+# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
+# using the given strftime format.
+html_last_updated_fmt = '%b %d, %Y'
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+#html_use_smartypants = True
+
+# Content template for the index page.
+#html_index = ''
+
+# Custom sidebar templates, maps document names to template names.
+#html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+#html_additional_pages = {}
+
+# If false, no module index is generated.
+#html_use_modindex = True
+
+# If true, the reST sources are included in the HTML build as _sources/<name>.
+#html_copy_source = True
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'SymPydoc'
+
+
+# Options for LaTeX output
+# ------------------------
+
+# The paper size ('letter' or 'a4').
+#latex_paper_size = 'letter'
+
+# The font size ('10pt', '11pt' or '12pt').
+#latex_font_size = '10pt'
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title, author, document class [howto/manual]).
+latex_documents = [('index', 'sympy.tex', 'SymPy Documentation',
+                        'SymPy Development Team', 'manual')]
+
+# Additional stuff for the LaTeX preamble.
+#latex_preamble = ''
+
+# Documents to append as an appendix to all manuals.
+#latex_appendices = []
+
+# If false, no module index is generated.
+#latex_use_modindex = True
diff --git a/lib/sympy/doc/api/index.txt b/lib/sympy/doc/api/index.txt
new file mode 100644
--- /dev/null
+++ b/lib/sympy/doc/api/index.txt
@@ -0,0 +1,10 @@
+Welcome to SymPy API reference
+==============================
+
+This is an automaticaly generated API documentation from SymPy sources.
+
+Click the  "modules" (:ref:`modindex`) link in the top right corner to
+browse the modules.
+
+Or click the "index" to see an index of all SymPy functions, methods and
+classes.
diff --git a/lib/sympy/doc/apidoc.conf b/lib/sympy/doc/apidoc.conf
new file mode 100644
--- /dev/null
+++ b/lib/sympy/doc/apidoc.conf
@@ -0,0 +1,19 @@
+[epydoc]
+
+name : Sympy
+url: http://code.google.com/p/sympy
+
+modules : sympy, sympy.core, sympy.modules
+
+output : html
+target : ../api/
+
+dotpath : /usr/bin/dot
+
+parse: yes
+introspect: yes
+
+private : no
+frames: no
+
+docformat : epytext
diff --git a/lib/sympy/doc/ext/mathjax.py b/lib/sympy/doc/ext/mathjax.py
new file mode 100644
--- /dev/null
+++ b/lib/sympy/doc/ext/mathjax.py
@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*-
+"""
+    sphinx.ext.mathjax
+    ~~~~~~~~~~~~~~~~~~
+
+    Allow `MathJax <http://mathjax.org/>`_ to be used to display math in
+    Sphinx's HTML writer -- requires the MathJax JavaScript library on your
+    webserver/computer.
+
+    :copyright: Copyright 2007-2011 by the Sphinx team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from docutils import nodes
+
+from sphinx.application import ExtensionError
+from sphinx.ext.mathbase import setup_math as mathbase_setup
+
+
+def html_visit_math(self, node):
+    self.body.append(self.starttag(node, 'span', '', CLASS='math'))
+    self.body.append(self.builder.config.mathjax_inline[0] +
+                     self.encode(node['latex']) +
+                     self.builder.config.mathjax_inline[1] + '</span>')
+    raise nodes.SkipNode
+
+def html_visit_displaymath(self, node):
+    self.body.append(self.starttag(node, 'div', CLASS='math'))
+    if node['nowrap']:
+        self.body.append(self.builder.config.mathjax_display[0] +
+                         node['latex'] +
+                         self.builder.config.mathjax_display[1])
+        self.body.append('</div>')
+        raise nodes.SkipNode
+
+    parts = [prt for prt in node['latex'].split('\n\n') if prt.strip()]
+    for i, part in enumerate(parts):
+        part = self.encode(part)
+        if i == 0:
+            # necessary to e.g. set the id property correctly
+            if node['number']:
+                self.body.append('<span class="eqno">(%s)</span>' %
+                                 node['number'])
+        if '&' in part or '\\\\' in part:
+            self.body.append(self.builder.config.mathjax_display[0] +
+                             '\\begin{split}' + part + '\\end{split}' +
+                             self.builder.config.mathjax_display[1])
+        else:
+            self.body.append(self.builder.config.mathjax_display[0] + part +
+                             self.builder.config.mathjax_display[1])
+    self.body.append('</div>\n')
+    raise nodes.SkipNode
+
+def builder_inited(app):
+    if not app.config.mathjax_path:
+        raise ExtensionError('mathjax_path config value must be set for the '
+                             'mathjax extension to work')
+    app.add_javascript(app.config.mathjax_path)
+
+
+def setup(app):
+    mathbase_setup(app, (html_visit_math, None), (html_visit_displaymath, None))
+    app.add_config_value('mathjax_path',
+                         'http://cdn.mathjax.org/mathjax/latest/MathJax.js?'
+                         'config=TeX-AMS-MML_HTMLorMML', False)
+    app.add_config_value('mathjax_inline', [r'\(', r'\)'], 'html')
+    app.add_config_value('mathjax_display', [r'\[', r'\]'], 'html')
+    app.connect('builder-inited', builder_inited)
diff --git a/lib/sympy/doc/generate_reference.py b/lib/sympy/doc/generate_reference.py
new file mode 100755
--- /dev/null
+++ b/lib/sympy/doc/generate_reference.py
@@ -0,0 +1,164 @@
+#! /usr/bin/python2.5
+
+# See the README in this directory how to generate the documentation.
+
+# This script generates the reference, however, it currently doesn't produce a
+# nicely polished documentation.
+
+# You need to run python2.5 with this, because python2.4 has some weird
+# Exception hierarchy classes that causes Exceptions to be ignored by this
+# script
+
+import types
+import os
+import sys
+
+def isclass(x):
+    from inspect import isclass as _isclass
+    return _isclass(x)
+
+def ismethod(x):
+    from inspect import ismethod as _ismethod
+    if _ismethod(x) or isinstance(x, (types.MethodType, types.FunctionType,
+                    types.UnboundMethodType)) or str(type(x)) in [
+                    "<type 'classmethod'>", "<type 'staticmethod'>"]:
+        return True
+    else:
+        return False
+
+def get_method_name(x):
+    if hasattr(x, "__name__"):
+        return x.__name__
+    # This is a static method, don't know how to read the name.
+    return None
+
+def get_method_args(x):
+    from inspect import getsource
+    try:
+        s = getsource(x)
+    except TypeError:
+        return ""
+    s = s[s.find("("):s.find(":")]
+    assert s is not None
+    return s
+
+def getdoc(x):
+    from inspect import getdoc as _getdoc
+    s = _getdoc(x)
+    return s
+
+class Parser(object):
+
+    def __init__(self):
+        self.modules = {}
+
+    def generate_doc(self, module, outdir="/tmp/", importpath = ""):
+        """
+        Takes the "module" string and generates a rst for this module.
+
+        modules is just the name, like "sympy", i.e. without a path.
+        """
+
+        print "Generating API documentation ... (this may take a while)"
+        sys.path.insert(0, importpath)
+        m = __import__(module)
+        self.main_module = module
+        self.handle_module(m)
+        print "  saving..."
+        for x in self.modules:
+            if x.startswith(module):
+                f = open(outdir+x+".txt", "w")
+                f.write(self.modules[x])
+        print "API generated in %s." % (outdir)
+
+
+    def handle_module(self, mod):
+        mname = mod.__name__
+        if mname in self.modules:
+            # we already handled this module
+            return
+        self.modules[mname] = ""
+
+        # if you want to get the top level modules without "sympy.", uncomment
+        # this:
+        #if mname.startswith(self.main_module):
+        #    #strip the "sympy.":
+        #    s = ".. module:: %s\n\n" % mname[len(self.main_module)+1:]
+        #else:
+        #    s = ".. module:: %s\n\n" % mname
+
+        s = "=" * len(mname) + "\n"
+        s += mname + "\n"
+        s += "=" * len(mname) + "\n" + "\n"
+
+        s += ".. module:: %s\n\n" % mname
+        if hasattr(mod, __file__):
+            s += "filename: %s\n" % mod.__file__
+        for x in mod.__dict__.values():
+            if isinstance(x, types.ModuleType):
+                self.handle_module(x)
+            elif x is None:
+                pass
+            elif isinstance(x, (int, float, str, list, dict)):
+                # skip these
+                pass
+            elif str(type(x)) == "<type 'classobj'>":
+                # old style classes
+                pass
+            elif hasattr(x, "__class__"):
+                s += self.handle_class(x)
+            else:
+                print "  Ignored:", type(x), x
+        self.modules[mod.__name__] = s
+
+    def handle_class(self, cls):
+        if hasattr(cls, "__name__"):
+            s = "\n.. class:: %s\n" % cls.__name__
+        else:
+            return ""
+
+        # Uncomment this to generate class docstrings too:
+        # unfortunately, sphinx fails to read them, so we need to fix sympy
+        # first.
+        #doc = getdoc(cls)
+        #if doc is not None:
+        #    s += doc
+        #    s += "\n"
+
+        if hasattr(cls, "__dict__"):
+            for x in cls.__dict__.values():
+                if isinstance(x, types.ModuleType):
+                    self.handle_module(x)
+                elif str(type(x)) == "<type 'classobj'>":
+                    # old style classes
+                    pass
+                elif x is None:
+                    pass
+                elif ismethod(x):
+                    s += self.handle_method(x)
+                elif str(type(x)) == "<type 'property'>":
+                    pass
+                elif isinstance(x, (int, float, str, list, tuple, dict)):
+                    # skip these
+                    pass
+                elif hasattr(x, "__class__"):
+                    # ignore nested classes
+                    pass
+                else:
+                    print "    Ignored in class:", type(x), x
+
+        return s
+
+    def handle_method(self, m):
+        mname = get_method_name(m)
+        if mname is None:
+            s = ""
+        else:
+            s = "\n.. method:: %s%s\n\n" % (mname, get_method_args(m))
+            doc = getdoc(m)
+            if doc is not None:
+                s += doc
+                s += "\n"
+        return s
+
+Parser().generate_doc("sympy", importpath = "..", outdir="api/modules/")
diff --git a/lib/sympy/doc/logo/logo.txt b/lib/sympy/doc/logo/logo.txt
new file mode 100644
--- /dev/null
+++ b/lib/sympy/doc/logo/logo.txt
@@ -0,0 +1,10 @@
+I created this logo and permit its free use on the same terms as the rest of Sympy's code and documentation.
+
+Fredrik Johansson
+
+----------------
+
+This directory only contain the logo, that is distributed in the tarball.
+All files can be downloaded from:
+
+http://sympy.googlecode.com/svn/materials/logo/
diff --git a/lib/sympy/doc/logo/sympy-160px.png b/lib/sympy/doc/logo/sympy-160px.png
new file mode 100644
index 0000000000000000000000000000000000000000..66f923ed1b90005b02b1cb4d980800ffc741b7e4
GIT binary patch

[cut]

diff --git a/lib/sympy/doc/man/isympy.1 b/lib/sympy/doc/man/isympy.1
new file mode 100644
--- /dev/null
+++ b/lib/sympy/doc/man/isympy.1
@@ -0,0 +1,73 @@
+'\" -*- coding: us-ascii -*-
+.if \n(.g .ds T< \\FC
+.if \n(.g .ds T> \\F[\n[.fam]]
+.de URL
+\\$2 \(la\\$1\(ra\\$3
+..
+.if \n(.g .mso www.tmac
+.TH isympy 1 2007-10-8 "" ""
+.SH NAME
+isympy \- interactive shell for SymPy
+.SH SYNOPSIS
+'nh
+.fi
+.ad l
+\fBisympy\fR \kx
+.if (\nx>(\n(.l/2)) .nr x (\n(.l/5)
+'in \n(.iu+\nxu
+[\fB-c\fR | \fB--console\fR]
+'in \n(.iu-\nxu
+.ad b
+'hy
+'nh
+.fi
+.ad l
+\fBisympy\fR \kx
+.if (\nx>(\n(.l/2)) .nr x (\n(.l/5)
+'in \n(.iu+\nxu
+[
+{\fB-h\fR | \fB--help\fR}
+| 
+{\fB-v\fR | \fB--version\fR}
+]
+'in \n(.iu-\nxu
+.ad b
+'hy
+.SH DESCRIPTION
+isympy is a Python shell for SymPy. It is just a normal python shell
+(ipython shell if you have the ipython package installed) that executes
+the following commands so that you don't have to:
+.PP
+.nf
+\*(T<
+>>> from __future__ import division
+>>> from sympy import *
+>>> x, y, z = symbols("xyz")
+>>> k, m, n = symbols("kmn", integer=True)
+    \*(T>
+.fi
+.PP
+So starting isympy is equivalent to starting python (or ipython) and
+executing the above commands by hand. It is intended for easy and quick
+experimentation with SymPy. For more complicated programs, it is recommended
+to write a script and import things explicitly (using the "from sympy
+import sin, log, Symbol, ..." idiom).
+.SH OPTIONS
+.TP 
+\*(T<\fB\-c \fR\*(T>\fIshell\fR, \*(T<\fB\-\-console=\fR\*(T>\fIshell\fR
+Use the specified shell (python or ipython) as
+console backend instead of the default one (ipython
+if present or python otherwise).
+
+Example: isympy -c python
+.SH FILES
+.TP 
+\*(T<\fI${HOME}/.sympy\-history\fR\*(T>
+Saves the history of commands when using the python
+shell as backend.
+.SH BUGS
+The upstreams BTS can be found at \(lahttp://code.google.com/p/sympy/issues/list\(ra
+Please report all bugs that you find in there, this will help improve
+the overall quality of SymPy.
+.SH "SEE ALSO"
+\fBipython\fR(1), \fBpython\fR(1)
diff --git a/lib/sympy/doc/man/isympy.xml b/lib/sympy/doc/man/isympy.xml
new file mode 100644
--- /dev/null
+++ b/lib/sympy/doc/man/isympy.xml
@@ -0,0 +1,193 @@
+<?xml version='1.0' encoding='ISO-8859-1'?>
+<?xml-stylesheet type="text/xsl"
+	href="http://docbook.sourceforge.net/release/xsl/current/manpages/docbook.xsl"?>
+<!DOCTYPE refentry PUBLIC "-//OASIS//DTD DocBook XML V4.4//EN"
+	"http://www.oasis-open.org/docbook/xml/4.4/docbookx.dtd" [
+
+<!-- Process this file docbook2x-man:
+
+docbook2x-man isympy.xml
+
+this will generate isympy.1 in the current directory. -->
+
+  <!-- Fill in your name for FIRSTNAME and SURNAME.              -->
+  <!ENTITY dhfirstname "Fabian">
+  <!ENTITY dhsurname   "Seoane">
+  <!ENTITY dhemail     "fabian at fseoane.net">
+  <!-- dhusername could also be set to "&firstname; &surname;".  -->
+  <!ENTITY dhusername  "&dhfirstname; &dhsurname;">
+
+  <!-- Please adjust the date whenever revising the manpage.     -->
+  <!ENTITY dhdate      "2007-10-8">
+  <!ENTITY dhrelease   "unused">
+
+  <!-- TITLE should be something like "User commands",           -->
+  <!-- "&dhpackage; command-line reference" or similar (see e.g. -->
+  <!-- http://www.tldp.org/HOWTO/Man-Page/q2.html). But limit    -->
+  <!-- the length to 30 chars.                                   -->
+  <!ENTITY dhtitle     "&dhpackage; command-line reference">
+
+  <!ENTITY dhucpackage "isympy">
+  <!ENTITY dhpackage   "isympy">
+
+  <!-- If the application e.g. belongs to a package like X.org,  -->
+  <!-- this should be set to the its name instead.               -->
+  <!ENTITY dhproduct   "python-sympy">
+
+  <!-- SECTION should be 1-8, maybe w/ subsection other          -->
+  <!-- parameters are allowed: see man(7), man(1) and            -->
+  <!-- http://www.tldp.org/HOWTO/Man-Page/q2.html.               -->
+  <!ENTITY dhsection   "1">
+
+]>
+
+<refentry>
+	<refentryinfo>
+		<title>&dhtitle;</title>
+		<productname>&dhpackage;</productname>
+		<releaseinfo role="version">&dhrelease;</releaseinfo>
+		<date>&dhdate;</date>
+		<authorgroup>
+			<author>
+				<firstname>Ondrej</firstname>
+				<surname>Certik</surname>
+				<contrib>Design, programming.</contrib>
+				<address>
+					<email>ondrej at certik.cz</email>
+				</address>
+			</author>
+			<author>
+				<firstname>&dhfirstname;</firstname>
+				<surname>&dhsurname;</surname>
+				<contrib>Programming.</contrib>
+				<address>
+					<email>&dhemail;</email>
+				</address>
+			</author>
+		</authorgroup>
+		<copyright>
+			<year>2011</year>
+			<holder>SymPy Development Team</holder>
+		</copyright>
+		<legalnotice>
+			<para>Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:</para>
+			<orderedlist>
+				<listitem>
+					<para>Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.</para>
+				</listitem>
+				<listitem>
+					<para>Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.</para>
+				</listitem>
+			</orderedlist>
+			<para>THIS SOFTWARE IS PROVIDED BY THE AUTHOR &quot;AS IS&quot; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.</para>
+		</legalnotice>
+	</refentryinfo>
+	<refmeta>
+		<refentrytitle>&dhucpackage;</refentrytitle>
+		<manvolnum>&dhsection;</manvolnum>
+	</refmeta>
+	<refnamediv>
+		<refname>&dhpackage;</refname>
+		<refpurpose>interactive shell for SymPy</refpurpose>
+	</refnamediv>
+	<refsynopsisdiv>
+		<cmdsynopsis>
+			<command>&dhpackage;</command>
+			<group choice="opt">
+				<arg choice="plain"><option>-c</option></arg>
+				<arg choice="plain"><option>--console</option></arg>
+			</group>
+		</cmdsynopsis>
+		<cmdsynopsis>
+			<command>&dhpackage;</command>
+      <!-- Normally the help and version options make the programs stop
+			     right after outputting the requested information. -->
+			<group choice="opt">
+				<arg choice="plain">
+					<group choice="req">
+						<arg choice="plain"><option>-h</option></arg>
+						<arg choice="plain"><option>--help</option></arg>
+					</group>
+				</arg>
+				<arg choice="plain">
+					<group choice="req">
+						<arg choice="plain"><option>-v</option></arg>
+						<arg choice="plain"><option>--version</option></arg>
+					</group>
+				</arg>
+			</group>
+		</cmdsynopsis>
+	</refsynopsisdiv>
+	<refsect1 id="description">
+		<title>DESCRIPTION</title>
+    <para>isympy is a Python shell for SymPy. It is just a normal python shell
+        (ipython shell if you have the ipython package installed) that executes
+        the following commands so that you don't have to:
+    </para>
+
+    <programlisting>
+>>> from __future__ import division
+>>> from sympy import *
+>>> x, y, z = symbols("x,y,z")
+>>> k, m, n = symbols("k,m,n", integer=True)
+    </programlisting>
+
+    <para>So starting isympy is equivalent to starting python (or ipython) and
+    executing the above commands by hand. It is intended for easy and quick
+    experimentation with SymPy. For more complicated programs, it is recommended
+    to write a script and import things explicitly (using the "from sympy
+        import sin, log, Symbol, ..." idiom).</para>
+
+	</refsect1>
+	<refsect1 id="options">
+		<title>OPTIONS</title>
+		<variablelist>
+			<!-- Use the variablelist.term.separator and the
+			     variablelist.term.break.after parameters to
+			     control the term elements. -->
+			<varlistentry>
+				<term><option>-c <replaceable class="parameter">shell</replaceable></option></term>
+				<term><option>--console=<replaceable class="parameter">shell</replaceable></option></term>
+				<listitem>
+					<para>Use the specified shell (python or ipython) as
+                        console backend instead of the default one (ipython
+                        if present or python otherwise).</para>
+					<para>Example: isympy -c python</para>
+				</listitem>
+			</varlistentry>
+		</variablelist>
+	</refsect1>
+	<refsect1 id="files">
+		<title>FILES</title>
+		<variablelist>
+			<varlistentry>
+				<term><filename>${HOME}/.sympy-history</filename></term>
+				<listitem>
+					<para>Saves the history of commands when using the python
+                        shell as backend.</para>
+				</listitem>
+			</varlistentry>
+		</variablelist>
+	</refsect1>
+
+	<refsect1 id="bugs">
+		<!-- Or use this section to tell about upstream BTS. -->
+		<title>BUGS</title>
+		<para>The upstreams <acronym>BTS</acronym> can be found at <ulink
+                url="http://code.google.com/p/sympy/issues/list">http://code.google.com/p/sympy/issues/list</ulink>
+        Please report all bugs that you find in there, this will help improve
+        the overall quality of SymPy.</para>
+	</refsect1>
+	<refsect1 id="see_also">
+		<title>SEE ALSO</title>
+		<!-- In alpabetical order. -->
+		<para><citerefentry>
+				<refentrytitle>ipython</refentrytitle>
+				<manvolnum>1</manvolnum>
+			</citerefentry>, <citerefentry>
+				<refentrytitle>python</refentrytitle>
+				<manvolnum>1</manvolnum>
+			</citerefentry></para>
+	</refsect1>
+</refentry>
+
diff --git a/lib/sympy/doc/src/_static/sympylogo.png b/lib/sympy/doc/src/_static/sympylogo.png
new file mode 100644
index 0000000000000000000000000000000000000000..d0d969e5b6df1c72198888d27470cd466904971b
GIT binary patch

[cut]

diff --git a/lib/sympy/doc/src/aboutus.txt b/lib/sympy/doc/src/aboutus.txt
new file mode 100644
--- /dev/null
+++ b/lib/sympy/doc/src/aboutus.txt
@@ -0,0 +1,172 @@
+About
+=====
+
+SymPy Development Team
+----------------------
+
+SymPy is a team project and it was developed by a lot of people.
+
+Here is a list of contributors together with what they do, (and in some cases links to their
+wiki pages), where they describe in
+more details what they do and what they are interested in (some people didn't
+want to be mentioned here, so see our repository history for a full list).
+
+#. Ond&#345;ej &#268;ert&#237;k: started the project in 2006, on Jan 4, 2011 passed the project leadership to Aaron Meurer
+#. Fabian Pedregosa: everything, reviewing patches, releases, general advice (issues and mailinglist), GSoC 2009
+#. Jurjen N.E. Bos: pretty printing and other patches
+#. Mateusz Paprocki: GSoC 2007, concrete math module, integration module, new core integration, a lot of patches, general advice, new polynomial module, improvements to solvers, simplifications, patch review
+#. Marc-Etienne M.Leveille: matrix patch
+#. Brian Jorgensen: GSoC 2007, plotting module and related things, patches
+#. Jason Gedge: GSoC 2007, geometry module, a lot of patches and fixes, new core integration
+#. Robert Schwarz: GSoC 2007, polynomials module, patches
+#. `Pearu Peterson <https://github.com/sympy/sympy/wiki/Pearu-Peterson>`_: new core, sympycore project, general advice (issues and mailinglist)
+#. `Fredrik Johansson <https://github.com/sympy/sympy/wiki/Fredrik-Johansson>`_: mpmath project and its integration in SymPy, number theory, combinatorial functions, products & summation, statistics, units, patches, documentation, general advice (issues and mailinglist)
+#. Chris Wu: GSoC 2007, linear algebra module
+#. Ulrich Hecht: pattern matching and other patches
+#. Goutham Lakshminarayan: number theory functions
+#. David Lawrence: GHOP, Mathematica parser, square root denesting
+#. Jaroslaw Tworek: GHOP, sympify AST implementation, sqrt() refactoring, maxima parser and other patches
+#. David Marek: GHOP, derivative evaluation patch, int(NumberSymbol) fix
+#. Bernhard R. Link: documentation patch
+#. Andrej Tokar&#269;&#237;k: GHOP, python printer
+#. Or Dvory: GHOP, documentation
+#. Saroj Adhikari: bug fixes
+#. Pauli Virtanen: bug fix
+#. Robert Kern: bug fix, common subexpression elimination
+#. James Aspnes: bug fixes
+#. Nimish Telang: multivariate lambdas
+#. Abderrahim Kitouni: pretty printing + complex expansion bug fixes
+#. Pan Peng: ode solvers patch
+#. Friedrich Hagedorn: many bug fixes, refactorings and new features added
+#. Elrond der Elbenfuerst: pretty printing fix
+#. Rizgar Mella: BBP formula for pi calculating algorithm
+#. Felix Kaiser: documentation + whitespace testing patches
+#. Roberto Nobrega: several pretty printing patches
+#. David Roberts: latex printing patches
+#. Sebastian Kr&#228;mer: implemented lambdify/numpy/mpmath cooperation, bug fixes, refactoring, lambdifying of matrices, large printing refactoring and bugfixes
+#. Vinzent Steinberg: docstring patches, a lot of bug fixes, nsolve (nonlinear equation systems solver), compiling functions to machine code, patches review
+#. Riccardo Gori: improvements and speedups to matrices, many bug fixes
+#. Case Van Horsen: implemented optional support for gmpy in mpmath
+#. &#352;t&#283;p&#225;n Rou&#269;ka: a lot of bug fixes all over sympy (matrix, simplification, limits, series, ...)
+#. Ali Raza Syed: pretty printing/isympy on windows fix
+#. Stefano Maggiolo: many bug fixes, polishings and improvements
+#. Robert Cimrman: matrix patches
+#. Bastian Weber: latex printing patches
+#. Sebastian Krause: match patches
+#. Sebastian Kreft: latex printing patches, Dirac delta function, other fixes
+#. Dan (coolg49964): documentation fixes
+#. Alan Bromborsky: geometric algebra modules
+#. Boris Timokhin: matrix fixes
+#. Robert (average.programmer): initial piecewise function patch
+#. Andy R. Terrel: piecewise function polishing and other patches
+#. Hubert Tsang: LaTeX printing fixes
+#. Konrad Meyer: policy patch
+#. Henrik Johansson: matrix zero finder patch
+#. Priit Laes: line integrals, cleanups in ODE, tests added
+#. Freddie Witherden: trigonometric simplifications fixes, LaTeX printer improvements
+#. Brian E. Granger: second quantization physics module
+#. Andrew Straw: lambdify() improvements
+#. Kaifeng Zhu: factorint() fix
+#. Ted Horst: Basic.__call__() fix, pretty printing fix
+#. Akshay Srinivasan: printing fix, improvements to integration
+#. Aaron Meurer: ODE solvers (GSoC 2009), The Risch Algorithm for integration (GSoC 2010), other fixes, project leader as of Jan 4, 2011
+#. Barry Wardell: implement series as function, several tests added
+#. Tomasz Buchert: ccode printing fixes, code quality concerning exceptions, documentation
+#. Vinay Kumar: polygamma tests
+#. Johann Cohen-Tanugi: commutative diff tests
+#. Jochen Voss: a test for the @cachit decorator and several other fixes
+#. Luke Peterson: improve solve() to handle Derivatives
+#. Chris Smith: improvements to solvers, many bug fixes, documentation and test improvements
+#. Thomas Sidoti: MathML printer improvements
+#. Florian Mickler: reimplementation of convex_hull, several geometry module fixes
+#. Nicolas Pourcelot: Infinity comparison fixes, latex fixes
+#. Ben Goodrich: Matrix.jacobian() function improvements and fixes
+#. Toon Verstraelen: code generation module, latex printing improvements
+#. Ronan Lamy: test coverage script; limit, expansion and other fixes and improvements; cleanup
+#. James Abbatiello: fixes tests on windows
+#. Ryan Krauss: fixes could_extract_minus_sign(), latex fixes and improvements
+#. Bill Flynn: substitution fixes
+#. Jorn Baayen: improvements to piecewise functions and latex printing, bug fixes
+#. Eh Tan: improve trigonometric simplification
+#. Renato Coutinho: derivative improvements
+#. Oscar Benjamin: latex printer fix
+#. &#216;yvind Jensen: implemented coupled cluster expansion and wick theorem, improvements to assumptions, bugfixes
+#. Julio Idichekop Filho: indentation fixes, docstring improvements
+#. &#321;ukasz Pankowski: fix matrix multiplication with numpy scalars
+#. Chu-Ching Huang: fix 3d plotting example
+#. Fernando Perez: symarray() implemented
+#. Raffaele De Feo: fix non-commutative expansion
+#. Christian Muise: fixes to logic module
+#. Matt Curry: GSoC 2010 project (symbolic quantum mechanics)
+#. Kazuo Thow: cleanup pretty printing tests
+#. Jezreel Ng: fix hyperbolic functions rewrite
+#. Matthew Brett: fixes to lambdify
+#. Addison Cugini: GSoC 2010 project (quantum computing)
+#. Nicholas J.S. Kinar: improved documentation about "Immutability of Expressions"
+#. Thomas Dixon: fix a limit bug
+#. Crist&#243;v&#227;o Sousa: implements _sage_ method for sign function.
+#. Andre de Fortier Smit: doctests in matrices improved
+#. Alexey U. Goodchenko: various work in matrices
+#. Gary Kerr: fix examples, docs
+#. Sherjil Ozair: fixes to SparseMatrix
+#. Oleksandr Gituliar: fixes to Matrix
+#. Sean Vig: Quantum improvement
+#. Prafullkumar P. Tale: fixed plotting documentation
+#. Vladimir Peri&#263;: fix some Python 3 issues
+#. Tom Bachmann: fixes to Real
+#. Yuri Karadzhov: improvements to hyperbolic identities
+#. Vladimir Lagunov: improvements to the geometry module
+#. Matthew Rocklin: improvements to Number instantiation
+#. Saptarshi Mandal: Test for an integral
+#. Gilbert Gede: Tests for solvers
+#. Anatolii Koval: Infinite 1D box example
+#. Tomo Lazovich: fixes to quantum operators
+#. Pavel Fedotov: fix to is_number
+#. Kibeom Kim: fixes to cse function and preorder_traversal
+#. Gregory Ksionda: fixes to Integral instantiation
+#. Tom&#225;&#353; Bambas: prettier printing of examples
+#. Jeremias Yehdegho: fixes to the nthoery module
+#. Jack McCaffery: fixes to asin and acos
+#. Luca Weihs: improvements to the geometry module
+#. Shai 'Deshe' Wyborski: fix to numeric evaluation of hypergeometric sums
+#. Thomas Wiecki: Fix Sum.diff
+#. &#211;scar N&#225;jera: better Laguerre polynomial generator
+#. Mario Pernici: faster algorithm for computing Groebner bases
+#. Benjamin McDonald: Fix bug in geometry
+#. Sam Magura: Improvements to Plot.saveimage
+#. Stefan Krastanov: Make Pyglet an external dependency
+#. Bradley Froehle: Fix shell command to generate modules in setup.py
+#. Min Ragan-Kelley: Fix isympy to work with IPython 0.11
+#. Emma Hogan: Fixes to the documentation
+#. Julien Rioux: Fix behavior of some deprecated methods
+#. Roberto Colistete, Jr.: Add num_columns option to the pretty printer
+#. Raoul Bourquin: Implement Euler numbers
+#. Gert-Ludwig Ingold: Correct derivative of coth
+
+Up-to-date list in the order of the first contribution is given in the `AUTHORS
+<https://github.com/sympy/sympy/blob/master/AUTHORS>`_ file.
+
+You can find a brief history of SymPy in the README.
+
+Financial and Infrastructure Support
+------------------------------------
+
+* `Google <http://www.google.com/corporate/>`_: SymPy has received generous financial support from Google in 2007 by financing 5 students through the Google Summer of Code (GSOC 2007) program (`more info <http://code.google.com/p/sympy/wiki/GSoC2007>`_) and 1 student in 2008 (`more info <http://code.google.com/p/sympy/wiki/GSoC2008>`_)
+
+* `Python Software Foundation <http://www.python.org/psf/>`_: PSF hosted 3 GSoC 2007 students (Brian, Robert and Jason), and 1 GSoC 2008 student (Fredrik)
+
+* `the Space Telescope Science Institute <http://www.stsci.edu/portal/>`_: STScI hosted 1 GSoC 2007 student (Mateusz)
+
+* `Portland State University <http://www.pdx.edu/>`_: PSU hosted 1 GSoC 2007 student (Chris)
+
+* `Simula Research Laboratory <http://www.simula.no/>`_: supports Pearu Peterson work in SymPy/Sympy Core projects
+
+License
+-------
+
+Unless stated otherwise, all files in the SymPy project, SymPy's webpage (and
+wiki), all images and all documentation including this User's Guide is licensed
+using the new BSD license:
+
+.. literalinclude:: ../../LICENSE
+
diff --git a/lib/sympy/doc/src/conf.py b/lib/sympy/doc/src/conf.py
new file mode 100644
--- /dev/null
+++ b/lib/sympy/doc/src/conf.py
@@ -0,0 +1,151 @@
+# -*- coding: utf-8 -*-
+#
+# SymPy documentation build configuration file, created by
+# sphinx-quickstart.py on Sat Mar 22 19:34:32 2008.
+#
+# This file is execfile()d with the current directory set to its containing dir.
+#
+# The contents of this file are pickled, so don't put values in the namespace
+# that aren't pickleable (module imports are okay, they're removed automatically).
+#
+# All configuration values have a default value; values that are commented out
+# serve to show the default value.
+
+import sys
+
+# If your extensions are in another directory, add it here.
+sys.path.extend(['../sympy', 'ext'])
+
+# General configuration
+# ---------------------
+
+# Add any Sphinx extension module names here, as strings. They can be extensions
+# coming with Sphinx (named 'sphinx.addons.*') or your custom ones.
+extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode', 'mathjax', ]
+
+# Use this to use pngmath instead
+#extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode', 'sphinx.ext.pngmath', ]
+
+# MathJax file, which is free to use.  See https://bitbucket.org/kevindunn/sphinx-extension-mathjax/wiki/Home
+mathjax_path = 'http://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML-full'
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['.templates']
+
+# The suffix of source filenames.
+source_suffix = '.txt'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General substitutions.
+project = 'SymPy'
+copyright = '2008, 2009, 2010, 2011 SymPy Development Team'
+
+# The default replacements for |version| and |release|, also used in various
+# other places throughout the built documents.
+#
+# The short X.Y version.
+version = '0.7.1'
+# The full version, including alpha/beta/rc tags.
+release = '0.7.1-git'
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+#today = ''
+# Else, today_fmt is used as the format for a strftime call.
+today_fmt = '%B %d, %Y'
+
+# List of documents that shouldn't be included in the build.
+#unused_docs = []
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+#add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+#add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+#show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+
+# Options for HTML output
+# -----------------------
+
+# The style sheet to use for HTML and HTML Help pages. A file of that name
+# must exist either in Sphinx' static/ path, or in one of the custom paths
+# given in html_static_path.
+html_style = 'default.css'
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
+# using the given strftime format.
+html_last_updated_fmt = '%b %d, %Y'
+
+html_logo = '_static/sympylogo.png'
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+#html_use_smartypants = True
+
+# Content template for the index page.
+#html_index = ''
+
+# Custom sidebar templates, maps document names to template names.
+#html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+#html_additional_pages = {}
+
+# If false, no module index is generated.
+#html_use_modindex = True
+
+# If true, the reST sources are included in the HTML build as _sources/<name>.
+#html_copy_source = True
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'SymPydoc'
+
+
+# Options for LaTeX output
+# ------------------------
+
+# The paper size ('letter' or 'a4').
+#latex_paper_size = 'letter'
+
+# The font size ('10pt', '11pt' or '12pt').
+#latex_font_size = '10pt'
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title, author, document class [howto/manual]).
+latex_documents = [('index', 'sympy.tex', 'SymPy Documentation',
+                        'SymPy Development Team', 'manual')]
+
+# Additional stuff for the LaTeX preamble.
+#latex_preamble = ''
+
+# Documents to append as an appendix to all manuals.
+#latex_appendices = []
+
+# If false, no module index is generated.
+#latex_use_modindex = True
+
+default_role = 'math'
+pngmath_divpng_args = ['-gamma 1.5','-D 110']
+# Note, this is ignored by the mathjax extension
+# Any \newcommand should be defined in the file
+pngmath_latex_preamble =  '\\usepackage{amsmath}\n'+\
+              '\\usepackage{bm}\n'+\
+              '\\usepackage{amsfonts}\n'+\
+              '\\usepackage{amssymb}\n'+\
+              '\\setlength{\\parindent}{0pt}\n'
diff --git a/lib/sympy/doc/src/figures/admon-note.png b/lib/sympy/doc/src/figures/admon-note.png
new file mode 100644
index 0000000000000000000000000000000000000000..4ffa19c734b05a4092ffcd79b3219476f1fde4a1
GIT binary patch

[cut]

diff --git a/lib/sympy/doc/src/figures/featured-downloads.png b/lib/sympy/doc/src/figures/featured-downloads.png
new file mode 100644
index 0000000000000000000000000000000000000000..db50c63ed3fb3c3360c067158b1a5e1f05ecad0c
GIT binary patch

[cut]

diff --git a/lib/sympy/doc/src/gotchas.txt b/lib/sympy/doc/src/gotchas.txt
new file mode 100644
--- /dev/null
+++ b/lib/sympy/doc/src/gotchas.txt
@@ -0,0 +1,624 @@
+.. _gotchas:
+
+====================
+Gotchas and Pitfalls
+====================
+
+.. role:: input(strong)
+
+Introduction
+============
+SymPy runs under the `Python Programming Language
+<http://www.python.org/>`_, so there are some things that may behave
+differently than they do in other, independent computer algebra systems
+like Maple or Mathematica.  These are some of the gotchas and pitfalls
+that you may encounter when using SymPy.  See also the `FAQ
+<https://github.com/sympy/sympy/wiki/Faq>`_, the :ref:`Tutorial<tutorial>`, the
+remainder of the SymPy Docs, and the `official Python Tutorial
+<http://docs.python.org/tutorial/>`_.
+
+If you are already familiar with C or Java, you might also want to look
+this `4 minute Python tutorial
+<http://www.nerdparadise.com/tech/python/4minutecrashcourse/>`_.
+
+Ignore ``#doctest: +SKIP`` in the examples.  That has to do with
+internal testing of the examples.
+
+.. _equals-signs:
+
+Equals Signs (=)
+================
+Single Equals Sign
+------------------
+The equals sign (``=``) is the assignment operator, not an equality.  If
+you want to do :math:`x=y`, use ``Eq(x,y)`` for equality.
+Alternatively, all expressions are assumed to equal zero, so you can
+just subtract one side and use ``x - y``.
+
+The proper use of the equals sign is to assign expressions to variables.
+
+For example:
+
+    >>> from sympy.abc import x, y
+    >>> a = x - y
+    >>> print a
+    x - y
+
+Double Equals Signs
+-------------------
+Double equals signs (``==``) are used to test equality.  However, this
+tests expressions exactly, not symbolically.  For example:
+
+    >>> (x + 1)**2 == x**2 + 2*x + 1
+    False
+    >>> (x + 1)**2 == (x + 1)**2
+    True
+
+If you want to test for symbolic equality, one way is to subtract one
+expression from the other and run it through functions like
+:func:`expand`, :func:`simplify`, and :func:`trigsimp` and see if the
+equation reduces to 0.
+
+    >>> from sympy import simplify, cos, sin, expand
+    >>> simplify((x + 1)**2 - (x**2 + 2*x + 1))
+    0
+    >>> simplify(sin(2*x) - 2*sin(x)*cos(x))
+    -2*sin(x)*cos(x) + sin(2*x)
+    >>> expand(sin(2*x) - 2*sin(x)*cos(x), trig=True)
+    0
+
+.. note::
+
+    See also `Why does SymPy say that two equal expressions are unequal? 
+    <https://github.com/sympy/sympy/wiki/Faq>`_ in the FAQ.
+
+
+Variables
+=========
+Variables Assignment does not Create a Relation Between Expressions
+-------------------------------------------------------------------
+When you use ``=`` to do assignment, remember that in Python, as in most
+programming languages, the variable does not change if you change the
+value you assigned to it.  The equations you are typing use the values
+present at the time of creation to "fill in" values, just like regular
+Python definitions. They are not altered by changes made afterwards.
+Consider the following:
+
+    >>> from sympy import Symbol
+    >>> a = Symbol('a') # Create a Symbol named a, that is also stored in the variable "a"
+    >>> b = a + 1       # Create another object, b, that refers to 'a'
+    >>> print b
+    a + 1
+    >>> a = 4           # a now points to the literal integer 4, not Symbol('a')
+    >>> print a
+    4
+    >>> b               # But b is still pointing at Symbol('a')
+    a + 1
+
+Changing quantity :obj:`a` does not change :obj:`b`; you are not working
+with a set of simultaneous equations. It might be helpful to remember
+that the string that gets printed when you print a variable referring to
+a SymPy object is the string that was given to it when it was created;
+that string does not have to be the same as the variable that you assign
+it to.
+
+    >>> from sympy import var
+    >>> r, t, d = var('rate time short_life')
+    >>> d = r*t
+    >>> print d
+    rate*time
+    >>> r=80
+    >>> t=2
+    >>> print d         # We haven't changed d, only r and t
+    rate*time
+    >>> d=r*t
+    >>> print d         # Now d is using the current values of r and t
+    160
+
+
+If you need variables that have dependence on each other, you can define
+functions.  Use the ``def`` operator.  Indent the body of the function.
+See the Python docs for more information on defining functions.
+
+    >>> c, d = var('c d')
+    >>> print c
+    c
+    >>> print d
+    d
+    >>> def ctimesd():
+    ...     """
+    ...     This function returns whatever c is times whatever d is.
+    ...     """
+    ...     return c*d
+    ...
+    >>> ctimesd()
+    c*d
+    >>> c = 2
+    >>> print c
+    2
+    >>> ctimesd()
+    2*d
+
+
+If you define a circular relationship, you will get a
+:exc:`RuntimeError`.
+
+    >>> def a():
+    ...     return b()
+    ...
+    >>> def b():
+    ...     return a()
+    ...
+    >>> a()
+    Traceback (most recent call last):
+      File "...", line ..., in ...
+        compileflags, 1) in test.globs
+      File "<...>", line 1, in <module>
+        a()
+      File "<...>", line 2, in a
+        return b()
+      File "<...>", line 2, in b
+        return a()
+      File "<...>", line 2, in a
+        return b()
+    ...
+    RuntimeError: maximum recursion depth exceeded
+
+
+.. note::
+    See also `Why doesn't changing one variable change another that depends on it? 
+    <https://github.com/sympy/sympy/wiki/Faq>`_ in the FAQ.
+
+.. _symbols:
+
+Symbols
+-------
+Symbols are variables, and like all other variables, they need to be
+assigned before you can use them.  For example:
+
+    >>> import sympy
+    >>> z**2 # z is not defined yet #doctest: +SKIP
+    Traceback (most recent call last):
+      File "<stdin>", line 1, in <module>
+    NameError: name 'z' is not defined
+    >>> sympy.var('z') # This is the easiest way to define z as a standard symbol
+    z
+    >>> z**2
+    z**2
+
+
+If you use :command:`isympy`, it runs the following commands for you,
+giving you some default Symbols and Functions.
+
+    >>> from __future__ import division
+    >>> from sympy import *
+    >>> x, y, z, t = symbols('x y z t')
+    >>> k, m, n = symbols('k m n', integer=True)
+    >>> f, g, h = map(Function, 'fgh')
+
+You can also import common symbol names from :mod:`sympy.abc`.
+
+    >>> from sympy.abc import w
+    >>> w
+    w
+    >>> import sympy
+    >>> dir(sympy.abc) #doctest: +SKIP
+    ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O',
+    'P', 'Q', 'R', 'S', 'Symbol', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z',
+    '__builtins__', '__doc__', '__file__', '__name__', '__package__', '_greek',
+    '_latin', 'a', 'alpha', 'b', 'beta', 'c', 'chi', 'd', 'delta', 'e',
+    'epsilon', 'eta', 'f', 'g', 'gamma', 'h', 'i', 'iota', 'j', 'k', 'kappa',
+    'l', 'm', 'mu', 'n', 'nu', 'o', 'omega', 'omicron', 'p', 'phi', 'pi',
+    'psi', 'q', 'r', 'rho', 's', 'sigma', 't', 'tau', 'theta', 'u', 'upsilon',
+    'v', 'w', 'x', 'xi', 'y', 'z', 'zeta']
+
+If you want control over the assumptions of the variables, use
+:func:`Symbol` and :func:`symbols`.  See :ref:`Keyword
+Arguments<keyword-arguments>` below.
+
+Lastly, it is recommended that you not use :obj:`I`, :obj:`E`, :obj:`S`,
+:obj:`N`, :obj:`C`, :obj:`O`, or :obj:`Q` for variable or symbol names, as those
+are used for the imaginary unit (:math:`i`), the base of the natural
+logarithm (:math:`e`), the :func:`sympify` function (see :ref:`Symbolic
+Expressions<symbolic-expressions>` below), numeric evaluation (:func:`N`
+is equivalent to :ref:`evalf()<evalf-label>` ), the class registry (for
+things like :func:`C.cos`, to prevent cyclic imports in some code),
+the `big O <http://en.wikipedia.org/wiki/Big_O_notation>`_ order symbol
+(as in :math:`O(n\log{n})`), and the assumptions object that holds a list of
+supported ask keys (such as :obj:`Q.real`), respectively.  You can use the
+mnemonic ``QCOSINE`` to remember what Symbols are defined by default in SymPy.
+Or better yet, always use lowercase letters for Symbol names.  Python will
+not prevent you from overriding default SymPy names or functions, so be
+careful.
+
+    >>> cos(pi) # cos and pi are a built-in sympy names.
+    -1
+    >>> pi = 3 # Notice that there is no warning for overriding pi.
+    >>> cos(pi)
+    cos(3)
+    >>> def cos(x): # No warning for overriding built-in functions either.
+    ...     return 5*x
+    ...
+    >>> cos(pi)
+    15
+
+
+
+To get a full list of all default names in SymPy do:
+
+    >>> import sympy
+    >>> dir(sympy) #doctest: +SKIP
+    # A big list of all default sympy names and functions follows.
+    # Ignore everything that starts and ends with __.
+
+If you have `iPython <http://ipython.scipy.org/moin/>`_ installed and
+use :command:`isympy`, you can also press the TAB key to get a list of
+all built-in names and to autocomplete.  Also, see `this page
+<http://kogs-www.informatik.uni-hamburg.de/~meine/python_tricks>`_ for a
+trick for getting tab completion in the regular Python console.
+
+.. note::
+    See also `What is the best way to create symbols? 
+    <https://github.com/sympy/sympy/wiki/Faq>`_ in the FAQ.
+
+.. _symbolic-expressions:
+
+Symbolic Expressions
+====================
+.. _python-vs-sympy-numbers:
+
+Python numbers vs. SymPy Numbers
+--------------------------------
+SymPy uses its own classes for integers, rational numbers, and floating
+point numbers instead of the default Python :obj:`int` and :obj:`float`
+types because it allows for more control.  But you have to be careful.
+If you type an expression that just has numbers in it, it will default
+to a Python expression.  Use the :func:`sympify` function, or just
+:func:`S`, to ensure that something is a SymPy expression.
+
+    >>> 6.2 # Python float. Notice the floating point accuracy problems. #doctest: +SKIP
+    6.2000000000000002
+    >>> type(6.2)
+    <... 'float'>
+    >>> S(6.2) # SymPy Float has no such problems because of arbitrary precision.
+    6.20000000000000
+    >>> type(S(6.2))
+    <class 'sympy.core.numbers.Float'>
+
+If you include numbers in a SymPy expression, they will be sympified
+automatically, but there is one gotcha you should be aware of.  If you
+do ``<number>/<number>`` inside of a SymPy expression, Python will
+evaluate the two numbers before SymPy has a chance to get
+to them.  The solution is to :func:`sympify` one of the numbers, or use
+:mod:`Rational`.
+
+    >>> x**(1/2) # evaluates to x**0 or x**0.5 #doctest: +SKIP
+    x**0.5
+    >>> x**(S(1)/2) # sympyify one of the ints
+    sqrt(x)
+    >>> x**Rational(1, 2) # use the Rational class
+    sqrt(x)
+
+With a power of ``1/2`` you can also use ``sqrt`` shorthand:
+
+    >>> sqrt(x) == x**Rational(1, 2)
+    True
+
+If the two integers are not directly separated by a division sign then
+you don't have to worry about this problem:
+
+    >>> x**(2*x/3)
+    x**(2*x/3)
+
+.. note::
+
+    A common mistake is copying an expression that is printed and
+    reusing it.  If the expression has a :mod:`Rational` (i.e.,
+    ``<number>/<number>``) in it, you will not get the same result,
+    obtaining the Python result for the division rather than a SymPy
+    Rational.
+
+    >>> x = Symbol('x')
+    >>> print solve(7*x -22,x)
+    [22/7]
+    >>> 22/7 # If we just copy and paste we get int 3 or a float #doctest: +SKIP
+    3.142857142857143
+    >>> # One solution is to just assign the expression to a variable
+    >>> # if we need to use it again.
+    >>> a = solve(7*x - 22,x)
+    >>> a
+    [22/7]
+    >>> # The other solution is to put quotes around the expression and run it through S() (sympify)
+    >>> S("22/7")
+    22/7
+
+Also, if you do not use :command:`isympy`, you could use ``from
+__future__ import division`` to prevent the ``/`` sign from performing
+`integer division <http://en.wikipedia.org/wiki/Integer_division>`_.
+
+    >>> from __future__ import division
+    >>> 1/2 # with division imported it evaluates to a python float #doctest: +SKIP
+    0.5
+    >>> 1//2 # You can still achieve integer division with //
+    0
+
+    But be careful: you will now receive floats where you might have desired
+    a Rational:
+
+    >>> x**(1/2) #doctest: +SKIP
+    x**0.5
+
+:mod:`Rational` only works for number/number and is only meant for
+rational numbers.  If you want a fraction with symbols or expressions in
+it, just use ``/``.  If you do number/expression or expression/number,
+then the number will automatically be converted into a SymPy Number.
+You only need to be careful with number/number.
+
+    >>> Rational(2, x)
+    Traceback (most recent call last):
+      File "...", line ..., in ...
+        compileflags, 1) in test.globs
+      File "<...>", line 1, in <module>
+        Rational(2, x)
+      ...
+    TypeError: int() argument must be a string or a number, not 'Symbol'
+    >>> 2/x
+    2/x
+
+
+.. _Immutability-of-Expressions:
+
+Immutability of Expressions
+---------------------------
+
+Expressions in SymPy are immutable, and cannot be modified by an in-place
+operation.  This means that a function will always return an object, and the
+original expression will not be modified. The following example snippet
+demonstrates how this works::
+
+	def main():
+	    var('x y a b')
+	    expr = 3*x + 4*y
+	    print 'original =', expr
+	    expr_modified = expr.subs({x:a, y:b})
+	    print 'modified = ', expr_modified
+
+	if __name__ == "__main__":
+	    main()
+
+The output shows that the :func:`subs` function has replaced variable
+:obj:`x` with variable :obj:`a`, and variable :obj:`y` with variable :obj:`b`::
+
+	original = 3*x + 4*y
+	modified =  3*a + 4*b
+
+The :func:`subs` function does not modify the original expression :obj:`expr`.
+Rather, a modified copy of the expression is returned. This returned object
+is stored in the variable :obj:`expr_modified`. Note that unlike C/C++ and
+other high-level languages, Python does not require you to declare a variable
+before it is used.
+
+
+Mathematical Operators
+----------------------
+SymPy uses the same default operators as Python.  Most of these, like
+``*/+-``, are standard.  Aside from integer division discussed in
+:ref:`Python numbers vs. SymPy Numbers <python-vs-sympy-numbers>` above,
+you should also be aware that implied multiplication is not allowed. You
+need to use ``*`` whenever you wish to multiply something.  Also, to
+raise something to a power, use ``**``, not ``^`` as many computer
+algebra systems use.  Parentheses ``()`` change operator precedence as
+you would normally expect.
+
+In :command:`isympy`, with the :command:`ipython` shell::
+
+    >>> 2x
+    Traceback (most recent call last):
+    ...
+    SyntaxError: invalid syntax
+    >>> 2*x
+    2*x
+    >>> (x+1)^2 # This is not power.  Use ** instead.
+    Traceback (most recent call last):
+    ...
+    TypeError: unsupported operand type(s) for ^: 'Add' and 'int'
+    >>> (x+1)**2
+    (x + 1)**2
+    >>> pprint(3 - x**(2*x)/(x + 1))
+        2*x
+       x
+    - ----- + 3
+      x + 1
+
+
+Inverse Trig Functions
+----------------------
+SymPy uses different names for some functions than most computer algebra
+systems.  In particular, the inverse trig functions use the python names
+of :func:`asin`, :func:`acos` and so on instead of the usual ``arcsin``
+and ``arccos``.  Use the methods described in :ref:`Symbols <symbols>`
+above to see the names of all SymPy functions.
+
+Special Symbols
+===============
+The symbols ``[]``, ``{}``, ``=``, and ``()`` have special meanings in
+Python, and thus in SymPy.  See the Python docs linked to above for
+additional information.
+
+.. _lists:
+
+Lists
+-----
+Square brackets ``[]`` denote a list.  A list is a container that holds
+any number of different objects.  A list can contain anything, including
+items of different types.  Lists are mutable, which means that you can
+change the elements of a list after it has been created.  You access the
+items of a list also using square brackets, placing them after the list
+or list variable.  Items are numbered using the space before the item.
+
+.. note::
+
+    List indexes begin at 0.
+
+Example:
+
+    >>> a = [x, 1] # A simple list of two items
+    >>> a
+    [x, 1]
+    >>> a[0] # This is the first item
+    x
+    >>> a[0] = 2 # You can change values of lists after they have been created
+    >>> print a
+    [2, 1]
+    >>> print solve(x**2+2*x-1,x) # Some functions return lists
+    [-1 + sqrt(2), -sqrt(2) - 1]
+
+
+.. note::
+    See the Python docs for more information on lists and the square
+    bracket notation for accessing elements of a list.
+
+Dictionaries
+------------
+Curly brackets ``{}`` denote a dictionary, or a dict for short.  A
+dictionary is an unordered list of non-duplicate keys and values.  The
+syntax is ``{key:value}``.  You can access values of keys using square
+bracket notation.
+
+    >>> d = {'a':1, 'b':2} # A dictionary.
+    >>> d
+    {'a': 1, 'b': 2}
+    >>> d['a'] # How to access items in a dict
+    1
+    >>> roots((x-1)**2*(x-2),x) # some functions return dicts
+    {1: 2, 2: 1}
+    >>> # Some SymPy functions return dictionaries.  For example,
+    >>> # roots returns a dictionary of root:multiplicity items.
+    >>> roots((x - 5)**2*(x + 3),x)
+    {-3: 1, 5: 2}
+    >>> # This means that the root -3 occurs once and the root 5 occurs twice.
+
+.. note::
+
+    See the Python docs for more information on dictionaries.
+
+Tuples
+------
+Parentheses ``()``, aside from changing operator precedence and their
+use in function calls, (like ``cos(x)``), are also used for tuples.  A
+``tuple`` is identical to a :ref:`list <lists>`, except that it is not
+mutable.  That means that you can not change their values after they
+have been created.  In general, you will not need tuples in SymPy, but
+sometimes it can be more convenient to type parentheses instead of
+square brackets.
+
+    >>> t = (1, 2, x) # Tuples are like lists
+    >>> t
+    (1, 2, x)
+    >>> t[0]
+    1
+    >>> t[0] = 4 # Except you can not change them after they have been created
+    Traceback (most recent call last):
+      File "<console>", line 1, in <module>
+    TypeError: 'tuple' object does not support item assignment
+    >>> (x,) # Single element tuples, unlike lists, must have a comma in them.
+    (x,)
+    >>> (x) # Not a tuple
+    x
+    >>> # integrate takes a tuple as the second argument if you want to integrate with limits.
+    >>> integrate(x**2, (x, 0, 1))
+    1/3
+    >>> integrate(x**2, [x, 0, 1]) # But a list works too.
+    1/3
+
+
+.. note::
+
+    See the Python docs for more information on tuples.
+
+.. _keyword-arguments:
+
+Keyword Arguments
+-----------------
+Aside from the usage described :ref:`above <equals-signs>`, equals signs
+(``=``) are also used to give named arguments to functions.  Any
+function that has ``key=value`` in its parameters list (see below on how
+to find this out), then ``key`` is set to ``value`` by default.  You can
+change the value of the key by supplying your own value using the equals
+sign in the function call.  Also, functions that have ``**`` followed by
+a name in the parameters list (usually ``**kwargs`` or
+``**assumptions``) allow you to add any number of ``key=value`` pairs
+that you want, and they will all be evaluated according to the function.
+
+    >>> # sqrt(x**2) doesn't auto simplify to x because x is assumed to be
+    >>> # complex by default, and, for example, sqrt((-1)**2) == sqrt(1) == 1 != -1.
+    >>> sqrt(x**2)
+    sqrt(x**2)
+    >>> x = Symbol('x', positive=True) # One example of keyword arguments is assumptions for Symbols
+    >>> sqrt(x**2) # only == x if x >= 0
+    x
+    >>> pprint(powsimp(x**n*x**m*y**n*y**m)) # powsimp has a default argument, combine='all'
+         m + n
+    (x*y)
+    >>> # Setting combine to the default value is the same as not setting it.
+    >>> pprint(powsimp(x**n*x**m*y**n*y**m, combine='all'))
+         m + n
+    (x*y)
+    >>> # The non-default options are 'exp', which combines exponents...
+    >>> pprint(powsimp(x**n*x**m*y**n*y**m, combine='exp'))
+     m + n  m + n
+    x     *y
+    >>> # ...and 'base', which combines bases.
+    >>> pprint(powsimp(x**n*x**m*y**n*y**m, combine='base'))
+         m      n
+    (x*y) *(x*y)
+
+.. note::
+
+    See the Python docs for more information on function parameters.
+
+Getting help from within SymPy
+==============================
+help()
+------
+Although all docs are available at `docs.sympy.org <http://docs.sympy.org/>`_ or on the 
+`SymPy Wiki <http://wiki.sympy.org/>`_, you can also get info on functions from within the 
+Python interpreter that runs SymPy.  The easiest way to do this is to do 
+``help(function)``, or ``function?`` if you are using :command:`ipython`::
+
+    In [1]: help(powsimp) # help() works everywhere
+
+    In [2]: # But in ipython, you can also use ?, which is better because it
+    In [3]: # it gives you more information
+    In [4]: powsimp?
+
+These will give you the function parameters and docstring for
+:func:`powsimp`.  The output will look something like this:
+
+.. module:: sympy.simplify.simplify
+.. autofunction:: powsimp
+
+source()
+--------
+Another useful option is the :func:`source` function.  This will print
+the source code of a function, including any docstring that it may have.
+You can also do ``function??`` in :command:`ipython`.  For example,
+from SymPy 0.6.5:
+
+    >>> source(simplify) # simplify() is actually only 2 lines of code. #doctest: +SKIP
+    In file: ./sympy/simplify/simplify.py
+    def simplify(expr):
+        """Naively simplifies the given expression.
+           ...
+           Simplification is not a well defined term and the exact strategies
+           this function tries can change in the future versions of SymPy. If
+           your algorithm relies on "simplification" (whatever it is), try to
+           determine what you need exactly  -  is it powsimp()? radsimp()?
+           together()?, logcombine()?, or something else? And use this particular
+           function directly, because those are well defined and thus your algorithm
+           will be robust.
+           ...
+        """
+        expr = Poly.cancel(powsimp(expr))
+        return powsimp(together(expr.expand()), combine='exp', deep=True)
+
diff --git a/lib/sympy/doc/src/guide.txt b/lib/sympy/doc/src/guide.txt
new file mode 100644
--- /dev/null
+++ b/lib/sympy/doc/src/guide.txt
@@ -0,0 +1,536 @@
+.. _guide:
+
+==================
+SymPy User's Guide
+==================
+
+.. role:: input(strong)
+
+Introduction
+============
+
+If you are new to Sympy, start with the :ref:`Tutorial <tutorial>`. If you went
+through it, now
+it's time to learn how SymPy works internally and this is what this guide is
+about. Once you grasp the idea behind SymPy, you will be able to use it
+effectively and also know how to extend it and fix it.
+You may also be just interested in :ref:`SymPy Modules Reference <module-docs>`.
+
+Learning SymPy
+==============
+
+Everyone has different ways of understanding the code written by others.
+
+Ond&#345;ej's approach
+-----------------
+
+Let's say I'd like to understand how ``x+y+x`` works and how it is possible
+that it gets simplified to ``2*x+y``.
+
+I write a simple script, I usually call it ``t.py`` (I don't remember anymore
+why I call it that way)::
+
+    from sympy.abc import x, y
+
+    e = x + y +x
+
+    print e
+
+And I try if it works
+
+.. parsed-literal::
+
+    $ :input:`python t.py`
+    y + 2*x
+
+Now I start `winpdb <http://winpdb.org/>`_ on it (if you've never used winpdb
+-- it's an excellent multiplatform debugger, works on Linux, Windows and Mac OS
+X):
+
+.. parsed-literal::
+
+    $ :input:`winpdb t.py`
+    y + 2*x
+
+and a winpdb window will popup, I move to the next line using F6:
+
+.. image:: pics/winpdb1.png
+
+Then I step into (F7) and after a little debugging I get for example:
+
+.. image:: pics/winpdb2.png
+
+.. tip:: Make the winpdb window larger on your screen, it was just made smaller to fit in
+         this guide.
+
+I see values of all local variables in the left panel, so it's very easy to see
+what's happening. You can see, that the ``y+2*x`` is emerging in the ``obj``
+variable. Observing that ``obj`` is constructed from ``c_part`` and ``nc_part``
+and seeing what ``c_part`` contains (``y`` and ``2*x``). So looking at the line
+28 (the whole line is not visible on the screenshot, so here it is)::
+
+    c_part, nc_part, lambda_args, order_symbols = cls.flatten(map(_sympify, args))
+
+you can see that the simplification happens in ``cls.flatten``. Now you can set
+the breakpoint on the line 28, quit winpdb (it will remember the breakpoint),
+start it again, hit F5, this will stop at this breakpoint, hit F7, this will go
+into the function ``Add.flatten()``::
+
+    @classmethod
+    def flatten(cls, seq):
+        """
+        Takes the sequence "seq" of nested Adds and returns a flatten list.
+
+        Returns: (commutative_part, noncommutative_part, lambda_args,
+            order_symbols)
+
+        Applies associativity, all terms are commutable with respect to
+        addition.
+        """
+        terms = {}      # term -> coeff
+                        # e.g. x**2 -> 5   for ... + 5*x**2 + ...
+
+        coeff = S.Zero  # standalone term
+                        # e.g. 3 + ...
+        lambda_args = None
+        order_factors = []
+        while seq:
+            o = seq.pop(0)
+
+and then you can study how it works. I am going to stop here, this should be
+enough to get you going -- with the above technique, I am able to understand
+almost any Python code.
+
+
+SymPy's Architecture
+====================
+
+We try to make the sources easily understandable, so you can look into the sources and
+read the doctests, it should be well documented and if you don't understand something,
+ask on the mailinglist_.
+
+You can find all the decisions archived in the issues_, to see rationale for
+doing this and that.
+
+Basics
+------
+
+All symbolic things are implemented using subclasses of the ``Basic`` class.
+First, you need to create symbols using ``Symbol("x")`` or numbers using
+``Integer(5)`` or ``Float(34.3)``. Then you construct the expression using any
+class from SymPy.  For example ``Add(Symbol("a"), Symbol("b"))`` gives an
+instance of the ``Add`` class.  You can call all methods, which the particular
+class supports.
+
+For easier use, there is a syntactic sugar for expressions like:
+
+``cos(x)+1`` is equal to ``cos(x).__add__(1)`` is equal to ``Add(cos(x),Integer(1))``
+
+or
+
+``2/cos(x)`` is equal to ``cos(x).__rdiv__(2)`` is equal to
+``Mul(Rational(2),Pow(cos(x),Rational(-1)))``.
+
+So, you can write normal expressions using python arithmetics like this::
+
+    a=Symbol("a")
+    b=Symbol("b")
+    e=(a+b)**2
+    print e
+
+but from the sympy point of view, we just need the classes ``Add``, ``Mul``, ``Pow``,
+``Rational``, ``Integer``.
+
+Automatic evaluation to canonical form
+--------------------------------------
+
+For computation, all expressions need to be in a
+canonical form, this is done during the creation of the particular instance
+and only inexpensive operations are performed, necessary to put the expression
+in the
+canonical form.  So the canonical form doesn't mean the simplest possible
+expression. The exact list of operations performed depend on the
+implementation.  Obviously, the definition of the canonical form is arbitrary,
+the only requirement is that all equivalent expressions must have the same
+canonical form.  We tried the conversion to a canonical (standard) form to be
+as fast as possible and also in a way so that the result is what you would
+write by hand - so for example ``b*a + -4 + b + a*b + 4 + (a+b)**2`` becomes
+``2*a*b + b + (a+b)**2``.
+
+Whenever you construct an expression, for example ``Add(x, x)``, the
+``Add.__new__()`` is called and it determines what to return. In this case::
+
+    >>> from sympy import Add
+    >>> from sympy.abc import x
+    >>> e = Add(x, x)
+    >>> e
+    2*x
+
+    >>> type(e)
+    <class 'sympy.core.mul.Mul'>
+
+``e`` is actually an instance of ``Mul(2, x)``, because ``Add.__new__()``
+returned ``Mul``.
+
+Comparisons
+-----------
+
+Expressions can be compared using a regular python syntax::
+
+    >>> from sympy.abc import x, y
+    >>> x+y == y+x
+    True
+
+    >>> x+y == y-x
+    False
+
+We made the following decision in SymPy: ``a=Symbol("x")`` and another
+``b=Symbol("x")`` (with the same string "x") is the same thing, i.e ``a==b`` is
+``True``. We chose ``a==b``, because it is more natural - ``exp(x)==exp(x)`` is
+also ``True`` for the same instance of ``x`` but different instances of ``exp``,
+so we chose to have ``exp(x)==exp(x)`` even for different instances of ``x``.
+
+Sometimes, you need to have a unique symbol, for example as a temporary one in
+some calculation, which is going to be substituted for something else at the
+end anyway. This is achieved using ``Dummy("x")``. So, to sum it
+up::
+
+    >>> from  sympy import Symbol, Dummy
+    >>> Symbol("x") == Symbol("x")
+    True
+
+    >>> Dummy("x") == Dummy("x")
+    False
+
+
+Debugging
+---------
+
+Starting with 0.6.4, you can turn on/off debug messages with the environment variable
+SYMPY_DEBUG, which is expected to have the values True or False. For example, to turn on
+debugging, you would issue::
+
+    [user at localhost]: SYMPY_DEBUG=True ./bin/isympy
+
+Functionality
+-------------
+
+There are no given requirements on classes in the library. For example, if they
+don't implement the ``fdiff()`` method and you construct an expression using
+such a class, then trying to use the ``Basic.series()`` method will raise an
+exception of not finding the ``fdiff()`` method in your class.  This "duck
+typing" has an advantage that you just implement the functionality which you
+need.
+
+You can define the ``cos`` class like this::
+
+    class cos(Function):
+        pass
+
+and use it like ``1+cos(x)``, but if you don't implement the ``fdiff()`` method,
+you will not be able to call ``(1+cos(x)).series()``.
+
+The symbolic object is characterized (defined) by the things which it can do,
+so implementing more methods like ``fdiff()``, ``subs()`` etc., you are creating a
+"shape" of the symbolic object. Useful things to implement in new classes are:
+``hash()`` (to use the class in comparisons), ``fdiff()`` (to use it in series
+expansion), ``subs()`` (to use it in expressions, where some parts are being
+substituted) and ``series()`` (if the series cannot be computed using the general
+``Basic.series()`` method). When you create a new class, don't worry about this
+too much - just try to use it in your code and you will realize immediately
+which methods need to be implemented in each situation.
+
+All objects in sympy are immutable - in the sense that any operation just
+returns a new instance (it can return the same instance only if it didn't
+change). This is a common mistake to change the current instance, like
+``self.arg=self.arg +1`` (wrong!). Use ``arg=self.arg + 1; return arg`` instead.
+The object is immutable in the
+sense of the symbolic expression it represents. It can modify itself to keep
+track of, for example, its hash. Or it can recalculate anything regarding the
+expression it contains. But the expression cannot be changed. So you can pass
+any instance to other objects, because you don't have to worry that it will
+change, or that this would break anything.
+
+Conclusion
+----------
+
+Above are the main ideas behind SymPy that we try to obey. The rest
+depends on the current implementation and may possibly change in the future.
+The point of all of this is that the interdependencies inside SymPy should be
+kept to a minimum. If one wants to add new functionality to SymPy, all that is
+necessary is to create a subclass of ``Basic`` and implement what you want.
+
+Functions
+---------
+
+How to create a new function with one variable::
+
+    class sign(Function):
+
+        nargs = 1
+
+        @classmethod
+        def eval(cls, arg):
+            if isinstance(arg, Basic.NaN):
+                return S.NaN
+            if isinstance(arg, Basic.Zero):
+                return S.Zero
+            if arg.is_positive:
+                return S.One
+            if arg.is_negative:
+                return S.NegativeOne
+            if isinstance(arg, Basic.Mul):
+                coeff, terms = arg.as_coeff_terms()
+                if not isinstance(coeff, Basic.One):
+                    return cls(coeff) * cls(Basic.Mul(*terms))
+
+        is_bounded = True
+
+        def _eval_conjugate(self):
+            return self
+
+        def _eval_is_zero(self):
+            return isinstance(self[0], Basic.Zero)
+
+and that's it. The ``_eval_*`` functions are called when something is needed.
+The ``eval`` is called when the class is about to be instantiated and it
+should return either some simplified instance of some other class or if the
+class should be unmodified, return ``None`` (see ``core/function.py`` in
+``Function.__new__`` for implementation details). See also tests in
+`sympy/functions/elementary/tests/test_interface.py
+<https://github.com/sympy/sympy/blob/master/sympy/functions/elementary/tests/test_interface.py>`_ that test this interface. You can use them to create your own new functions.
+
+The applied function ``sign(x)`` is constructed using
+::
+
+    sign(x)
+
+both inside and outside of SymPy. Unapplied functions ``sign`` is just the class itself::
+
+    sign
+
+both inside and outside of SymPy. This is the current structure of classes in SymPy::
+
+    class BasicType(type):
+        pass
+    class MetaBasicMeths(BasicType):
+        ...
+    class BasicMeths(AssumeMeths):
+        __metaclass__ = MetaBasicMeths
+        ...
+    class Basic(BasicMeths):
+        ...
+    class FunctionClass(MetaBasicMeths):
+        ...
+    class Function(Basic, RelMeths, ArithMeths):
+        __metaclass__ = FunctionClass
+        ...
+
+The exact names of the classes and the names of the methods and how they work can be
+changed in the future.
+
+This is how to create a function with two variables::
+
+    class chebyshevt_root(Function):
+        nargs = 2
+
+        @classmethod
+        def eval(cls, n, k):
+            if not 0 <= k < n:
+                raise ValueError("must have 0 <= k < n")
+            return C.cos(S.Pi*(2*k+1)/(2*n))
+
+
+.. note:: the first argument of a @classmethod should be ``cls`` (i.e. not ``self``).
+
+Here it's how to define a derivative of the function::
+
+    >>> from sympy import Function, sympify, cos
+    >>> class my_function(Function):
+    ...     nargs = 1
+    ...
+    ...     def fdiff(self, argindex = 1):
+    ...         return cos(self.args[0])
+    ...
+    ...     @classmethod
+    ...     def eval(cls, arg):
+    ...         arg = sympify(arg)
+    ...         if arg == 0:
+    ...             return sympify(0)
+
+So guess what this ``my_function`` is going to be? Well, it's derivative is
+``cos`` and the function value at 0 is 0, but let's pretend we don't know::
+
+    >>> from sympy import pprint
+    >>> pprint(my_function(x).series(x, 0, 10))
+         3     5     7       9
+        x     x     x       x       / 10\
+    x - -- + --- - ---- + ------ + O\x  /
+        6    120   5040   362880
+
+Looks familiar indeed::
+
+    >>> from sympy import sin
+    >>> pprint(sin(x).series(x, 0, 10))
+         3     5     7       9
+        x     x     x       x       / 10\
+    x - -- + --- - ---- + ------ + O\x  /
+        6    120   5040   362880
+
+Let's try a more complicated example. Let's define the derivative in terms of the
+function itself::
+
+    >>> class what_am_i(Function):
+    ...     nargs = 1
+    ...
+    ...     def fdiff(self, argindex = 1):
+    ...         return 1-what_am_i(self.args[0])**2
+    ...
+    ...     @classmethod
+    ...     def eval(cls, arg):
+    ...         arg = sympify(arg)
+    ...         if arg == 0:
+    ...             return sympify(0)
+
+So what is ``what_am_i``?  Let's try it::
+
+    >>> pprint(what_am_i(x).series(x, 0, 10))
+         3      5       7       9
+        x    2*x    17*x    62*x     / 10\
+    x - -- + ---- - ----- + ----- + O\x  /
+        3     15     315     2835
+
+Well, it's ``tanh``::
+
+    >>> from sympy import tanh
+    >>> pprint(tanh(x).series(x, 0, 10))
+         3      5       7       9
+        x    2*x    17*x    62*x     / 10\
+    x - -- + ---- - ----- + ----- + O\x  /
+        3     15     315     2835
+
+The new functions we just defined are regular SymPy objects, you
+can use them all over SymPy, e.g.::
+
+    >>> from sympy import limit
+    >>> limit(what_am_i(x)/x, x, 0)
+    1
+
+
+common tasks
+------------
+
+Please use the same way as is shown below all across SymPy.
+
+**accessing parameters**::
+
+    >>> from sympy import sign, sin
+    >>> from sympy.abc import x, y, z
+
+    >>> e = sign(x**2)
+    >>> e.args
+    (x**2,)
+
+    >>> e.args[0]
+    x**2
+
+    >>> (x+y*z).args
+    (y*z, x)
+
+    >>> (x+y*z).args[0]
+    y*z
+
+    >>> (x+y*z).args[1]
+    x
+
+    >>> (y*z).args
+    (y, z)
+
+    >>> sin(y*z).args
+    (y*z,)
+
+Never use internal methods or variables, prefixed with "``_``" (example: don't
+use ``_args``, use ``.args`` instead).
+
+**testing the structure of a SymPy expression**
+
+Applied functions::
+
+    >>> from sympy import sign, exp, Function
+    >>> e = sign(x**2)
+
+    >>> isinstance(e, sign)
+    True
+
+    >>> isinstance(e, exp)
+    False
+
+    >>> isinstance(e, Function)
+    True
+
+So ``e`` is a ``sign(z)`` function, but not ``exp(z)`` function.
+
+Unapplied functions::
+
+    >>> from sympy import sign, exp, FunctionClass
+    >>> e = sign
+
+    >>> f = exp
+
+    >>> g = Add
+
+    >>> isinstance(e, FunctionClass)
+    True
+
+    >>> isinstance(f, FunctionClass)
+    True
+
+    >>> isinstance(g, FunctionClass)
+    False
+
+    >>> g is Add
+    True
+
+So ``e`` and ``f`` are functions, ``g`` is not a function.
+
+Contributing
+============
+
+We welcome every SymPy user to participate in it's development. Don't worry if
+you've never contributed to any open source project, we'll help you learn
+anything necessary, just ask on our mailinglist_.
+
+Don't be afraid to ask anything and don't worry that you are wasting our time
+if you are new to SymPy and ask questions that maybe most of the people know
+the answer to -- you are not, because that's exactly what the mailinglist_ is
+for and people answer your emails because they want to. Also we try hard to
+answer every email, so you'll always get some feedback and pointers what to do
+next.
+
+Improving the code
+------------------
+
+Go to issues_ that are sorted by priority and simply find something that you
+would like to get fixed and fix it. If you find something odd, please report it
+into issues_ first before fixing it. Feel free to consult with us on the
+mailinglist_.  Then send your patch either to the issues_ or the mailinglist_. See
+the SympyDevelopment_ wiki, but don't worry about it too much if you find it
+too formal - simply get in touch with us on the mailinglist_ and we'll help you
+get your patch accepted.
+
+.. _issues:             http://code.google.com/p/sympy/issues/list
+.. _mailinglist:        http://groups.google.com/group/sympy
+.. _SympyDevelopment:   http://code.google.com/p/sympy/wiki/SympyDevelopment
+
+Please read our excellent `SymPy Patches Tutorial
+<https://github.com/sympy/sympy/wiki/Development-workflow>`_ at our
+wiki for a guide on how to write patches to SymPy, how to work with Git,
+and how to make your life easier as you get started with SymPy.
+
+
+Improving the docs
+------------------
+
+Please see :ref:`the documentation <module-docs>` how to fix and improve
+SymPy's documentation. All contribution is very welcome.
+
diff --git a/lib/sympy/doc/src/index.txt b/lib/sympy/doc/src/index.txt
new file mode 100644
--- /dev/null
+++ b/lib/sympy/doc/src/index.txt
@@ -0,0 +1,34 @@
+.. SymPy documentation master file, created by sphinx-quickstart.py on Sat Mar 22 19:34:32 2008.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to SymPy's documentation!
+=================================
+
+`SymPy <http://sympy.org>`_ is a Python library for symbolic mathematics.
+If you are new to SymPy, start with the Tutorial.
+
+This is the central page for all of SymPy's documentation.
+
+
+Contents:
+
+.. toctree::
+   :maxdepth: 2
+
+   install.txt
+   tutorial.txt
+   gotchas.txt
+   guide.txt
+   modules/index.txt
+   python-comparisons.txt
+   wiki.txt
+   outreach.txt
+   aboutus.txt
+
+If something cannot be easily accessed from this page, it's a bug (`please
+report it`_).
+
+This documentation is maintained with docutils, so you might see some comments in the form #doctest:... . You can safely ignore them.
+
+.. _please report it: http://code.google.com/p/sympy/issues/list
diff --git a/lib/sympy/doc/src/install.txt b/lib/sympy/doc/src/install.txt
new file mode 100644
--- /dev/null
+++ b/lib/sympy/doc/src/install.txt
@@ -0,0 +1,9 @@
+Installation
+------------
+
+Go to the `Downloads`_ tab on SymPy's webpage for thourough instructions.
+
+You'll find there detailed explanation how to install SymPy using source,
+Git repository, Debian, Ubuntu, Gentoo, Windows and Sage.
+
+.. _Downloads: http://code.google.com/p/sympy/wiki/DownloadInstallation?tm=2
diff --git a/lib/sympy/doc/src/modules/assumptions.txt b/lib/sympy/doc/src/modules/assumptions.txt
new file mode 100644
--- /dev/null
+++ b/lib/sympy/doc/src/modules/assumptions.txt
@@ -0,0 +1,388 @@
+Assumptions module
+******************
+
+.. module:: sympy.assumptions
+
+Queries are used to ask information about expressions. Main method for this
+is ask():
+
+.. automethod:: sympy.assumptions.ask
+
+Querying
+========
+
+ask's optional second argument should be a boolean expression involving
+assumptions about objects in expr. Valid values include:
+
+    * Q.integer(x)
+    * Q.positive(x)
+    * Q.integer(x) & Q.positive(x)
+    * etc.
+
+Q is a class in sympy.assumptions holding known predicates.
+
+See documentation for the logic module for a complete list of valid boolean
+expressions.
+
+You can also define global assumptions so you don't have to pass that argument
+each time to function ask(). This is done by using the global_assumptions
+object from module sympy.assumptions. You can then clear global assumptions
+with global_assumptions.clear()::
+
+     >>> from sympy import *
+     >>> x = Symbol('x')
+     >>> global_assumptions.add(Q.positive(x))
+     >>> ask(Q.positive(x))
+     True
+     >>> global_assumptions.clear()
+
+
+Supported predicates
+====================
+
+bounded
+-------
+
+Test that a function is bounded with respect to its variables. For example,
+sin(x) is a bounded functions, but exp(x) is not.
+
+Examples::
+
+    >>> from sympy import *
+    >>> x = Symbol('x')
+    >>> ask(Q.bounded(exp(x)), ~Q.bounded(x))
+    False
+    >>> ask(Q.bounded(exp(x)) , Q.bounded(x))
+    True
+    >>> ask(Q.bounded(sin(x)), ~Q.bounded(x))
+    True
+
+
+commutative
+-----------
+
+Test that objects are commutative. By default, symbols in SymPy are considered
+commutative except otherwise stated.
+
+Examples::
+
+    >>> from sympy import *
+    >>> x, y = symbols('x,y')
+    >>> ask(Q.commutative(x))
+    True
+    >>> ask(Q.commutative(x), ~Q.commutative(x))
+    False
+    >>> ask(Q.commutative(x*y), ~Q.commutative(x))
+    False
+
+
+complex
+-------
+
+Test that expression belongs to the field of complex numbers.
+
+Examples::
+
+    >>> from sympy import *
+    >>> ask(Q.complex(2))
+    True
+    >>> ask(Q.complex(I))
+    True
+    >>> x, y = symbols('x,y')
+    >>> ask(Q.complex(x+I*y), Q.real(x) & Q.real(y))
+    True
+
+
+even
+----
+
+Test that expression represents an even number, that is, an number that
+can be written in the form 2*n, n integer.
+
+Examples::
+
+    >>> from sympy import *
+    >>> ask(Q.even(2))
+    True
+    >>> n = Symbol('n')
+    >>> ask(Q.even(2*n), Q.integer(n))
+    True
+
+
+extended_real
+-------------
+
+Test that an expression belongs to the field of extended real numbers, that is, real
+numbers union {Infinity, -Infinity}.
+
+Examples::
+
+    >>> from sympy import *
+    >>> ask(Q.extended_real(oo))
+    True
+    >>> ask(Q.extended_real(2))
+    True
+    >>> ask(Q.extended_real(x), Q.real(x))
+    True
+
+
+imaginary
+---------
+
+Test that an expression belongs to the set of imaginary numbers, that is,
+ it can be written as x*I, where x is real and I is the imaginary unit.
+
+Examples::
+
+    >>> from sympy import *
+    >>> ask(Q.imaginary(2*I))
+    True
+    >>> x = Symbol('x')
+    >>> ask(Q.imaginary(x*I), Q.real(x))
+    True
+
+
+infinitesimal
+-------------
+
+Test that an expression is equivalent to an infinitesimal number.
+
+Examples::
+
+    >>> from sympy import *
+    >>> ask(Q.infinitesimal(1/oo))
+    True
+    >>> x, y = symbols('x,y')
+    >>> ask(Q.infinitesimal(2*x), Q.infinitesimal(x))
+    True
+    >>> ask(Q.infinitesimal(x*y), Q.infinitesimal(x) & Q.bounded(y))
+    True
+
+
+integer
+-------
+
+Test that an expression belongs to the set of integer numbers.
+
+Examples::
+
+    >>> from sympy import *
+    >>> ask(Q.integer(2))
+    True
+    >>> ask(Q.integer(sqrt(2)))
+    False
+    >>> x = Symbol('x')
+    >>> ask(Q.integer(x/2), Q.even(x))
+    True
+
+
+irrational
+----------
+
+Test that an expression represents an irrational number.
+
+Examples::
+
+     >>> from sympy import *
+     >>> ask(Q.irrational(pi))
+     True
+     >>> ask(Q.irrational(sqrt(2)))
+     True
+     >>> ask(Q.irrational(x*sqrt(2)), Q.rational(x))
+     True
+
+
+rational
+--------
+
+Test that an expression represents a rational number.
+
+Examples::
+
+     >>> from sympy import *
+     >>> ask(Q.rational(Rational(3, 4)))
+     True
+     >>> x, y = symbols('x,y')
+     >>> ask(Q.rational(x/2), Q.integer(x))
+     True
+     >>> ask(Q.rational(x/y), Q.integer(x) & Q.integer(y))
+     True
+
+
+negative
+--------
+
+Test that an expression is less (strict) than zero.
+
+Examples::
+
+     >>> from sympy import *
+     >>> ask(Q.negative(0.3))
+     False
+     >>> x = Symbol('x')
+     >>> ask(Q.negative(-x), Q.positive(x))
+     True
+
+Remarks
+^^^^^^^
+negative numbers are defined as real numbers that are not zero nor positive, so
+complex numbers (with nontrivial imaginary coefficients) will return False
+for this predicate. The same applies to Q.positive.
+
+
+positive
+--------
+
+Test that a given expression is greater (strict) than zero.
+
+Examples::
+
+     >>> from sympy import *
+     >>> ask(Q.positive(0.3))
+     True
+     >>> x = Symbol('x')
+     >>> ask(Q.positive(-x), Q.negative(x))
+     True
+
+Remarks
+^^^^^^^
+see Remarks for negative
+
+
+prime
+-----
+
+Test that an expression represents a prime number.
+
+Examples::
+
+    >>> from sympy import *
+    >>> ask(Q.prime(13))
+    True
+
+Remarks: Use sympy.ntheory.isprime to test numeric values efficiently.
+
+
+real
+----
+
+Test that an expression belongs to the field of real numbers.
+
+Examples::
+
+    >>> from sympy import *
+    >>> ask(Q.real(sqrt(2)))
+    True
+    >>> x, y = symbols('x,y')
+    >>> ask(Q.real(x*y), Q.real(x) & Q.real(y))
+    True
+
+
+odd
+---
+
+Test that an expression represents an odd number.
+
+Examples::
+
+    >>> from sympy import *
+    >>> ask(Q.odd(3))
+    True
+    >>> n = Symbol('n')
+    >>> ask(Q.odd(2*n + 1), Q.integer(n))
+    True
+
+
+nonzero
+-------
+
+Test that an expression is not zero.
+
+Examples::
+
+     >>> from sympy import *
+     >>> x = Symbol('x')
+     >>> ask(Q.nonzero(x), Q.positive(x) | Q.negative(x))
+     True
+
+
+Design
+======
+
+Each time ask is called, the appropriate Handler for the current key is called. This is
+always a subclass of sympy.assumptions.AskHandler. It's classmethods have the name's of the classes
+it supports. For example, a (simplified) AskHandler for the ask 'positive' would
+look like this::
+
+    class AskPositiveHandler(CommonHandler):
+
+        def Mul(self):
+            # return True if all argument's in self.expr.args are positive
+            ...
+
+        def Add(self):
+            for arg in self.expr.args:
+                if not ask(arg, positive, self.assumptions):
+                    break
+            else:
+                # if all argument's are positive
+                return True
+        ...
+
+The .Mul() method is called when self.expr is an instance of Mul, the Add method
+would be called when self.expr is an instance of Add and so on.
+
+
+Extensibility
+=============
+
+You can define new queries or support new types by subclassing sympy.assumptions.AskHandler
+ and registering that handler for a particular key by calling register_handler:
+
+.. automethod:: sympy.assumptions.register_handler
+
+You can undo this operation by calling remove_handler.
+
+.. automethod:: sympy.assumptions.remove_handler
+
+You can support new types [1]_ by adding a handler to an existing key. In the
+following example, we will create a new type MyType and extend the key 'prime'
+to accept this type (and return True)
+
+.. parsed-literal::
+
+    >>> from sympy.core import Basic
+    >>> from sympy.assumptions import register_handler
+    >>> from sympy.assumptions.handlers import AskHandler
+    >>> class MyType(Basic):
+    ...     pass
+    >>> class MyAskHandler(AskHandler):
+    ...     @staticmethod
+    ...     def MyType(expr, assumptions):
+    ...         return True
+    >>> a = MyType()
+    >>> register_handler('prime', MyAskHandler)
+    >>> ask(Q.prime(a))
+    True
+
+
+Performance improvements
+========================
+
+On queries that involve symbolic coefficients, logical inference is used. Work on
+improving satisfiable function (sympy.logic.inference.satisfiable) should result
+in notable speed improvements.
+
+Logic inference used in one ask could be used to speed up further queries, and
+current system does not take advantage of this. For example, a truth maintenance
+system (http://en.wikipedia.org/wiki/Truth_maintenance_system) could be implemented.
+
+Misc
+====
+
+You can find more examples in the in the form of test under directory
+sympy/assumptions/tests/
+
+.. [1] New type must inherit from Basic, otherwise an exception will be raised.
+   This is a bug and should be fixed.
+
diff --git a/lib/sympy/doc/src/modules/concrete.txt b/lib/sympy/doc/src/modules/concrete.txt
new file mode 100644
--- /dev/null
+++ b/lib/sympy/doc/src/modules/concrete.txt
@@ -0,0 +1,80 @@
+Concrete Mathematics
+====================
+
+Hypergeometric terms
+--------------------
+
+The center stage, in recurrence solving and summations, play hypergeometric
+terms. Formally these are sequences annihilated by first order linear
+recurrence operators. In simple words if we are given term `a(n)` then it is
+hypergeometric if its consecutive term ratio is a rational function in `n`.
+
+To check if a sequence is of this type you can use the ``is_hypergeometric`` method
+which is  available in Basic class. Here is simple example involving a
+polynomial:
+
+    >>> from sympy import *
+    >>> n, k = symbols('n,k')
+    >>> (n**2 + 1).is_hypergeometric(n)
+    True
+
+Of course polynomials are hypergeometric but are there any more complicated
+sequences of this type? Here are some trivial examples:
+
+    >>> factorial(n).is_hypergeometric(n)
+    True
+    >>> binomial(n, k).is_hypergeometric(n)
+    True
+    >>> rf(n, k).is_hypergeometric(n)
+    True
+    >>> ff(n, k).is_hypergeometric(n)
+    True
+    >>> gamma(n).is_hypergeometric(n)
+    True
+    >>> (2**n).is_hypergeometric(n)
+    True
+
+We see that all species used in summations and other parts of concrete
+mathematics are hypergeometric. Note also that binomial coefficients and both
+rising and falling factorials are hypergeometric in both their arguments:
+
+    >>> binomial(n, k).is_hypergeometric(k)
+    True
+    >>> rf(n, k).is_hypergeometric(k)
+    True
+    >>> ff(n, k).is_hypergeometric(k)
+    True
+
+To say more, all previously shown examples are valid for integer linear
+arguments:
+
+    >>> factorial(2*n).is_hypergeometric(n)
+    True
+    >>> binomial(3*n+1, k).is_hypergeometric(n)
+    True
+    >>> rf(n+1, k-1).is_hypergeometric(n)
+    True
+    >>> ff(n-1, k+1).is_hypergeometric(n)
+    True
+    >>> gamma(5*n).is_hypergeometric(n)
+    True
+    >>> (2**(n-7)).is_hypergeometric(n)
+    True
+
+However nonlinear arguments make those sequences fail to be hypergeometric:
+
+    >>> factorial(n**2).is_hypergeometric(n)
+    False
+    >>> (2**(n**3 + 1)).is_hypergeometric(n)
+    False
+
+If not only the knowledge of being hypergeometric or not is needed, you can use
+``hypersimp()`` function. It will try to simplify combinatorial expression and
+if the term given is hypergeometric it will return a quotient of polynomials of
+minimal degree. Otherwise is will return `None` to say that sequence is not
+hypergeometric:
+
+    >>> hypersimp(factorial(2*n), n)
+    4*n**2 + 6*n + 2
+    >>> hypersimp(factorial(n**2), n)
+
diff --git a/lib/sympy/doc/src/modules/core.txt b/lib/sympy/doc/src/modules/core.txt
new file mode 100644
--- /dev/null
+++ b/lib/sympy/doc/src/modules/core.txt
@@ -0,0 +1,351 @@
+SymPy Core
+==========
+
+sympify
+-------
+.. module:: sympy.core.sympify
+.. autofunction:: sympify
+
+cacheit
+-------
+.. module:: sympy.core.cache
+.. autofunction:: cacheit
+
+.. module:: sympy.core.basic
+
+Basic
+-----
+.. autoclass:: Basic
+   :members:
+
+Atom
+----
+.. autoclass:: Atom
+   :members:
+
+C
+-
+.. autoclass:: C
+   :members:
+
+S
+-
+.. module:: sympy.core.singleton
+.. autoclass:: S
+   :members:
+
+.. module:: sympy.core.expr
+
+Expr
+----
+.. autoclass:: Expr
+   :members:
+
+AtomicExpr
+----------
+.. autoclass:: AtomicExpr
+   :members:
+
+.. module:: sympy.core.symbol
+
+Symbol
+------
+.. autoclass:: Symbol
+   :members:
+
+Wild
+----
+.. autoclass:: Wild
+   :members:
+
+Dummy
+-----
+.. autoclass:: Dummy
+   :members:
+
+symbols
+-------
+.. autofunction:: symbols
+
+var
+---
+.. autofunction:: var
+
+.. module:: sympy.core.numbers
+
+Number
+------
+.. autoclass:: Number
+   :members:
+
+Float
+-----
+.. autoclass:: Float
+   :members:
+
+Rational
+--------
+.. autoclass:: Rational
+   :members:
+
+Integer
+-------
+.. autoclass:: Integer
+   :members:
+
+NumberSymbol
+------------
+.. autoclass:: NumberSymbol
+   :members:
+
+RealNumber
+----------
+.. autoclass:: RealNumber
+   :members:
+
+Real
+----
+.. autoclass:: Real
+   :members:
+
+igcd
+----
+.. autofunction:: igcd
+
+ilcm
+----
+.. autofunction:: ilcm
+
+seterr
+------
+.. autofunction:: seterr
+
+E
+-
+.. autoclass:: E
+   :members:
+
+I
+-
+.. autoclass:: I
+   :members:
+
+nan
+---
+.. autofunction:: nan
+
+oo
+--
+.. autofunction:: oo
+
+pi
+--
+.. autofunction:: pi
+
+zoo
+---
+.. autofunction:: zoo
+
+.. module:: sympy.core.power
+
+Pow
+---
+.. autoclass:: Pow
+   :members:
+
+integer_nthroot
+---------------
+.. autofunction:: integer_nthroot
+
+Mul
+---
+.. module:: sympy.core.mul
+.. autoclass:: Mul
+   :members:
+
+Add
+---
+.. module:: sympy.core.add
+.. autoclass:: Add
+   :members:
+
+.. module:: sympy.core.relational
+
+Rel
+---
+.. autoclass:: Rel
+   :members:
+
+Eq
+--
+.. autoclass:: Eq
+   :members:
+
+Ne
+--
+.. autoclass:: Ne
+   :members:
+
+Lt
+--
+.. autoclass:: Lt
+   :members:
+
+Le
+--
+.. autoclass:: Le
+   :members:
+
+Gt
+--
+.. autoclass:: Gt
+   :members:
+
+Ge
+--
+.. autoclass:: Ge
+   :members:
+
+Equality
+--------
+.. autoclass:: Equality
+   :members:
+
+Inequality
+----------
+.. autoclass:: Inequality
+   :members:
+
+Unequality
+----------
+.. autoclass:: Unequality
+   :members:
+
+StrictInequality
+----------------
+.. autoclass:: StrictInequality
+   :members:
+
+vectorize
+---------
+.. module:: sympy.core.multidimensional
+.. autoclass:: vectorize
+   :members:
+
+.. module:: sympy.core.function
+
+Lambda
+------
+.. autoclass:: Lambda
+   :members:
+
+WildFunction
+------------
+.. autoclass:: WildFunction
+   :members:
+
+Derivative
+----------
+.. autoclass:: Derivative
+   :members:
+
+diff
+----
+.. autofunction:: diff
+
+FunctionClass
+-------------
+.. autoclass:: FunctionClass
+   :members:
+
+Function
+--------
+.. autoclass:: Function
+   :members:
+
+Subs
+----
+.. autoclass:: Subs
+   :members:
+
+expand
+------
+.. autofunction:: expand
+
+PoleError
+---------
+.. autoclass:: PoleError
+   :members:
+
+count_ops
+---------
+.. autofunction:: count_ops
+
+expand_mul
+----------
+.. autofunction:: expand_mul
+
+expand_log
+----------
+.. autofunction:: expand_log
+
+expand_func
+-----------
+.. autofunction:: expand_func
+
+expand_trig
+-----------
+.. autofunction:: expand_trig
+
+expand_complex
+--------------
+.. autofunction:: expand_complex
+
+expand_multinomial
+------------------
+.. autofunction:: expand_multinomial
+
+.. module:: sympy.core.sets
+
+Set
+---
+.. autoclass:: Set
+   :members:
+
+Interval
+--------
+.. autoclass:: Interval
+   :members:
+
+Union
+-----
+.. autoclass:: Union
+   :members:
+
+EmptySet
+--------
+.. autoclass:: EmptySet
+   :members:
+
+.. module:: sympy.core.evalf
+
+PrecisionExhausted
+------------------
+.. autoclass:: PrecisionExhausted
+   :members:
+
+N
+-
+.. autoclass:: N
+   :members:
+
+Tuple
+-----
+.. module:: sympy.core.containers
+.. autoclass:: Tuple
+   :members:
+
+gcd_terms
+---------
+.. module:: sympy.core.exprtools
+.. autofunction:: gcd_terms
diff --git a/lib/sympy/doc/src/modules/evalf.txt b/lib/sympy/doc/src/modules/evalf.txt
new file mode 100644
--- /dev/null
+++ b/lib/sympy/doc/src/modules/evalf.txt
@@ -0,0 +1,397 @@
+.. _evalf-label:
+
+Numerical evaluation
+====================
+
+Basics
+------
+
+Exact SymPy expressions can be converted to floating-point approximations
+(decimal numbers) using either the ``.evalf()`` method or the ``N()`` function.
+``N(expr, <args>)`` is equivalent to ``sympify(expr).evalf(<args>)``.
+
+    >>> from sympy import *
+    >>> N(sqrt(2)*pi)
+    4.44288293815837
+    >>> (sqrt(2)*pi).evalf()
+    4.44288293815837
+
+
+By default, numerical evaluation is performed to an accuracy of 15 decimal
+digits. You can optionally pass a desired accuracy (which should be a positive
+integer) as an argument to ``evalf`` or ``N``:
+
+    >>> N(sqrt(2)*pi, 5)
+    4.4429
+    >>> N(sqrt(2)*pi, 50)
+    4.4428829381583662470158809900606936986146216893757
+
+
+Complex numbers are supported:
+
+    >>> N(1/(pi + I), 20)
+    0.28902548222223624241 - 0.091999668350375232456*I
+
+
+If the expression contains symbols or for some other reason cannot be evaluated
+numerically, calling ``.evalf()`` or ``N()`` returns the original expression, or
+ in some cases a partially evaluated expression. For example, when the
+ expression is a polynomial in expanded form, the coefficients are evaluated:
+
+    >>> x = Symbol('x')
+    >>> (pi*x**2 + x/3).evalf()
+    3.14159265358979*x**2 + 0.333333333333333*x
+
+
+You can also use the standard Python functions ``float()``, ``complex()`` to
+convert SymPy expressions to regular Python numbers:
+
+    >>> float(pi) #doctest: +SKIP
+    3.1415926535...
+    >>> complex(pi+E*I) #doctest: +SKIP
+    (3.1415926535...+2.7182818284...j)
+
+
+If these functions are used, failure to evaluate the expression to an explicit
+number (for example if the expression contains symbols) will raise an exception.
+
+There is essentially no upper precision limit. The following command, for
+example, computes the first 100,000 digits of &#960;/e:
+
+    >>> N(pi/E, 100000) #doctest: +SKIP
+    ...
+
+
+This shows digits 999,951 through 1,000,000 of pi:
+
+    >>> str(N(pi, 10**6))[-50:] #doctest: +SKIP
+    '95678796130331164628399634646042209010610577945815'
+
+
+High-precision calculations can be slow. It is recommended (but entirely
+optional) to install gmpy (http://code.google.com/p/gmpy/), which will
+significantly speed up computations such as the one above.
+
+Floating-point numbers
+----------------------
+
+Floating-point numbers in SymPy are instances of the class ``Float``. A ``Float``
+can be created with a custom precision as second argument:
+
+    >>> Float(0.1)
+    0.100000000000000
+    >>> Float(0.1, 10)
+    0.1000000000
+    >>> Float(0.1, 30)
+    0.100000000000000005551115123126
+
+
+As the last example shows, Python floats are only accurate to about 15 digits as
+inputs. To create a ``Float`` from a high-precision decimal number, it is better
+to pass a string or ``evalf`` a ``Rational``:
+
+    >>> Float('0.1', 30)
+    0.100000000000000000000000000000
+    >>> Rational(1,10).evalf(30)
+    0.100000000000000000000000000000
+
+
+The precision of a number determines 1) the precision to use when performing
+arithmetic with the number, and 2) the number of digits to display when printing
+the number. When two numbers with different precision are used together in an
+arithmetic operation, the higher of the precisions is used for the result. Note
+that precision therefore cannot be used as a model of error propagation or
+significance arithmetic; rather, this scheme is designed to ensure stability of
+numerical algorithms.
+
+``N`` and ``evalf`` can be used to change the precision of existing
+floating-point numbers:
+
+    >>> N(3.5)
+    3.50000000000000
+    >>> N(3.5, 5)
+    3.5000
+    >>> N(3.5, 30)
+    3.50000000000000000000000000000
+
+
+Accuracy and error handling
+---------------------------
+
+When the input to ``N`` or ``evalf`` is a complicated expression, numerical
+error propagation becomes a concern. As an example, consider the 100'th
+Fibonacci number and the excellent (but not exact) approximation &#966;^100 / &#8730;5
+where &#966; is the golden ratio. With ordinary floating-point arithmetic,
+subtracting these numbers from each other erroneously results in a complete
+cancellation:
+
+    >>> float(GoldenRatio**1000/sqrt(5)) #doctest: +SKIP
+    4.34665576869...e+208
+    >>> float(fibonacci(1000)) #doctest: +SKIP
+    4.34665576869...e+208
+    >>> float(fibonacci(1000)) - float(GoldenRatio**1000/sqrt(5))
+    0.0
+
+
+``N`` and ``evalf`` keep track of errors and automatically increase the
+precision used internally in order to obtain a correct result:
+
+    >>> N(fibonacci(100) - GoldenRatio**100/sqrt(5))
+    -5.64613129282185e-22
+
+
+Unfortunately, numerical evaluation cannot tell an expression that is exactly
+zero apart from one that is merely very small. The working precision is
+therefore capped, by default to around 100 digits. If we try with the 1000'th
+Fibonacci number, the following happens:
+
+    >>> N(fibonacci(1000) - (GoldenRatio)**1000/sqrt(5))
+    -.0e+84
+
+
+The lack of digits in the returned number indicates that ``N`` failed to achieve
+full accuracy. The result indicates that the magnitude of the expression is something less than 10^84, but that is not a particularly good answer. To force a higher working precision, the ``maxn`` keyword argument can be used:
+
+    >>> N(fibonacci(1000) - (GoldenRatio)**1000/sqrt(5), maxn=500)
+    -4.60123853010113e-210
+
+
+Normally, ``maxn`` can be set very high (thousands of digits), but be aware that
+this may cause significant slowdown in extreme cases. Alternatively, the
+``strict=True`` option can be set to force an exception instead of silently
+returning a value with less than the requested accuracy:
+
+    >>> N(fibonacci(1000) - (GoldenRatio)**1000/sqrt(5), strict=True)
+    Traceback (most recent call last):
+    ...
+    PrecisionExhausted: Failed to distinguish the expression:
+    <BLANKLINE>
+    -sqrt(5)*GoldenRatio**1000/5 + 43466557686937456435688527675040625802564660517371780402481729089536555417949051890403879840079255169295922593080322634775209689623239873322471161642996440906533187938298969649928516003704476137795166849228875
+    <BLANKLINE>
+    from zero. Try simplifying the input, using chop=True, or providing a higher maxn for evalf
+
+
+If we add a term so that the Fibonacci approximation becomes exact (the full
+form of Binet's formula), we get an expression that is exactly zero, but ``N``
+does not know this:
+
+    >>> f = fibonacci(100) - (GoldenRatio**100 - (GoldenRatio-1)**100)/sqrt(5)
+    >>> N(f)
+    .0e-113
+    >>> N(f, maxn=1000)
+    .0e-1345
+
+
+In situations where such cancellations are known to occur, the ``chop`` options
+is useful. This basically replaces very small numbers with exact zeros:
+
+    >>> N(f, chop=True)
+    0
+
+
+The ``chop`` option is very useful for cleaning up small real or imaginary parts:
+
+    >>> N((E+pi*I)*(E-pi*I))
+    17.25866050002 - 1.30104260698261e-17*I
+    >>> N((E+pi*I)*(E-pi*I), chop=True)
+    17.2586605000200
+
+
+Sums and integrals
+------------------
+
+Sums (in particular infinite series) and integrals can be used like regular
+closed-form expressions, and support arbitrary-precision evaluation:
+
+    >>> var('n x')
+    (n, x)
+    >>> Sum(1/n**n, (n, 1, oo)).evalf()
+    1.29128599706266
+    >>> Integral(x**(-x), (x, 0, 1)).evalf()
+    1.29128599706266
+    >>> Sum(1/n**n, (n, 1, oo)).evalf(50)
+    1.2912859970626635404072825905956005414986193682745
+    >>> Integral(x**(-x), (x, 0, 1)).evalf(50)
+    1.2912859970626635404072825905956005414986193682745
+    >>> (Integral(exp(-x**2), (x, -oo, oo)) ** 2).evalf(30)
+    3.14159265358979323846264338328
+
+
+By default, the tanh-sinh quadrature algorithm is used to evaluate integrals.
+This algorithm is very efficient and robust for smooth integrands (and even
+integrals with endpoint singularities), but may struggle with integrals that
+are highly oscillatory or have mid-interval discontinuities. In many cases,
+``evalf``/``N`` will correctly estimate the error. With the following integral,
+the result is accurate but only good to four digits:
+
+    >>> f = abs(sin(x))
+    >>> Integral(abs(sin(x)), (x, 0, 4)).evalf()
+    2.346
+
+
+It is better to split this integral into two pieces:
+
+    >>> (Integral(f, (x, 0, pi)) + Integral(f, (x, pi, 4))).evalf()
+    2.34635637913639
+
+
+A similar example is the following oscillatory integral:
+
+
+    >>> Integral(sin(x)/x**2, (x, 1, oo)).evalf(maxn=20)
+    0.5
+
+
+It can be dealt with much more efficiently by telling ``evalf`` or ``N`` to
+use an oscillatory quadrature algorithm:
+
+    >>> Integral(sin(x)/x**2, (x, 1, oo)).evalf(quad='osc')
+    0.504067061906928
+    >>> Integral(sin(x)/x**2, (x, 1, oo)).evalf(20, quad='osc')
+    0.50406706190692837199
+
+
+Oscillatory quadrature requires an integrand containing a factor cos(ax+b) or
+sin(ax+b). Note that many other oscillatory integrals can be transformed to
+this form with a change of variables:
+
+    >>> init_printing(use_unicode=False, wrap_line=False, no_global=True)
+    >>> intgrl = Integral(sin(1/x), (x, 0, 1)).transform(x, 1/x)
+    >>> intgrl
+     oo
+      /
+     |
+     |  sin(x)
+     |  ------ dx
+     |     2
+     |    x
+     |
+    /
+    1
+    >>> N(intgrl, quad='osc')
+    0.504067061906928
+
+
+Infinite series use direct summation if the series converges quickly enough.
+Otherwise, extrapolation methods (generally the Euler-Maclaurin formula but
+also Richardson extrapolation) are used to speed up convergence. This allows
+high-precision evaluation of slowly convergent series:
+
+    >>> var('k')
+    k
+    >>> Sum(1/k**2, (k, 1, oo)).evalf()
+    1.64493406684823
+    >>> zeta(2).evalf()
+    1.64493406684823
+    >>> Sum(1/k-log(1+1/k), (k, 1, oo)).evalf()
+    0.577215664901533
+    >>> Sum(1/k-log(1+1/k), (k, 1, oo)).evalf(50)
+    0.57721566490153286060651209008240243104215933593992
+    >>> EulerGamma.evalf(50)
+    0.57721566490153286060651209008240243104215933593992
+
+
+The Euler-Maclaurin formula is also used for finite series, allowing them to
+be approximated quickly without evaluating all terms:
+
+    >>> Sum(1/k, (k, 10000000, 20000000)).evalf()
+    0.693147255559946
+
+
+Note that ``evalf`` makes some assumptions that are not always optimal. For
+fine-tuned control over numerical summation, it might be worthwhile to manually
+use the method ``Sum.euler_maclaurin``.
+
+Special optimizations are used for rational hypergeometric series (where the
+term is a product of polynomials, powers, factorials, binomial coefficients and
+the like). ``N``/``evalf`` sum series of this type very rapidly to high
+precision. For example, this Ramanujan formula for pi can be summed to 10,000
+digits in a fraction of a second with a simple command:
+
+    >>> f = factorial
+    >>> n = Symbol('n', integer=True)
+    >>> R = 9801/sqrt(8)/Sum(f(4*n)*(1103+26390*n)/f(n)**4/396**(4*n), (n, 0, oo)) #doctest: +SKIP
+    >>> N(R, 10000) #doctest: +SKIP
+    3.141592653589793238462643383279502884197169399375105820974944592307816406286208
+    99862803482534211706798214808651328230664709384460955058223172535940812848111745
+    02841027019385211055596446229489549303819644288109756659334461284756482337867831
+    ...
+
+
+Numerical simplification
+------------------------
+
+The function ``nsimplify`` attempts to find a formula that is numerically equal
+to the given input. This feature can be used to guess an exact formula for an
+approximate floating-point input, or to guess a simpler formula for a
+complicated symbolic input. The algorithm used by ``nsimplify`` is capable of
+identifying simple fractions, simple algebraic expressions, linear combinations
+of given constants, and certain elementary functional transformations of any of
+the preceding.
+
+Optionally, ``nsimplify`` can be passed a list of constants to include (e.g. pi)
+and a minimum numerical tolerance. Here are some elementary examples:
+
+    >>> nsimplify(0.1)
+    1/10
+    >>> nsimplify(6.28, [pi], tolerance=0.01)
+    2*pi
+    >>> nsimplify(pi, tolerance=0.01)
+    22/7
+    >>> nsimplify(pi, tolerance=0.001)
+    355
+    ---
+    113
+    >>> nsimplify(0.33333, tolerance=1e-4)
+    1/3
+    >>> nsimplify(2.0**(1/3.), tolerance=0.001)
+    635
+    ---
+    504
+    >>> nsimplify(2.0**(1/3.), tolerance=0.001, full=True)
+    3 ___
+    \/ 2
+
+
+Here are several more advanced examples:
+
+    >>> nsimplify(Float('0.130198866629986772369127970337',30), [pi, E])
+        1
+    ----------
+    5*pi
+    ---- + 2*E
+     7
+    >>> nsimplify(cos(atan('1/3')))
+        ____
+    3*\/ 10
+    --------
+       10
+    >>> nsimplify(4/(1+sqrt(5)), [GoldenRatio])
+    -2 + 2*GoldenRatio
+    >>> nsimplify(2 + exp(2*atan('1/4')*I))
+    49   8*I
+    -- + ---
+    17    17
+    >>> nsimplify((1/(exp(3*pi*I/5)+1)))
+               ___________
+              /   ___
+    1        /  \/ 5    1
+    - - I*  /   ----- + -
+    2     \/      10    4
+    >>> nsimplify(I**I, [pi])
+     -pi
+     ---
+      2
+    e
+    >>> n = Symbol('n')
+    >>> nsimplify(Sum(1/n**2, (n, 1, oo)), [pi])
+      2
+    pi
+    ---
+     6
+    >>> nsimplify(gamma('1/4')*gamma('3/4'), [pi])
+      ___
+    \/ 2 *pi
+
+
+
diff --git a/lib/sympy/doc/src/modules/functions.txt b/lib/sympy/doc/src/modules/functions.txt
new file mode 100644
--- /dev/null
+++ b/lib/sympy/doc/src/modules/functions.txt
@@ -0,0 +1,220 @@
+Functions Module
+****************
+
+.. module:: sympy.functions
+
+Elementary
+==========
+
+This module implements elementary functions, as well as functions like Abs, Max, etc.
+
+
+Abs
+---
+
+Returns the absolute value of the argument.
+
+Examples::
+    >>> from sympy.functions import Abs
+    >>> Abs(-1)
+    1
+
+.. autoclass:: sympy.functions.elementary.complexes.Abs
+
+
+arg
+---
+
+Returns the argument (in radians) of a complex number. For a real
+number, the argument is always 0.
+
+Examples::
+    >>> from sympy.functions import arg
+    >>> from sympy import I, sqrt
+    >>> arg(2.0)
+    0
+    >>> arg(I)
+    pi/2
+    >>> arg(sqrt(2) + I*sqrt(2))
+    pi/4
+
+.. autoclass:: sympy.functions.elementary.complexes.arg
+
+conjugate
+---------
+
+Returns the 'complex conjugate <http://en.wikipedia.org/wiki/Complex_conjugation>'_
+of an argument. In mathematics, the complex conjugate of a complex number is given
+by changing the sign of the imaginary part. Thus, the conjugate of the complex number
+
+    a + ib
+
+(where a and b are real numbers) is
+
+    a - ib
+
+Examples::
+    >>> from sympy.functions import conjugate
+    >>> from sympy import I
+    >>> conjugate(2)
+    2
+    >>> conjugate(I)
+    -I
+
+.. autoclass:: sympy.functions.elementary.complexes.conjugate
+
+
+Min
+---
+
+Returns the minimum of two (comparable) expressions.
+
+Examples::
+    >>> from sympy.functions import Min
+    >>> Min(1,2)
+    1
+    >>> from sympy.abc import x
+    >>> Min(1, x)
+    Min(1, x)
+
+It is named Min and not min to avoid conflicts with the built-in function min.
+
+.. autoclass:: sympy.functions.elementary.miscellaneous.Min
+
+
+Max
+---
+
+Returns the maximum of two (comparable) expressions
+
+It is named Max and not max to avoid conflicts with the built-in function max.
+
+.. autoclass:: sympy.functions.elementary.miscellaneous.Max
+
+
+re
+--
+
+Return the real part of an expression
+
+Examples::
+    >>> from sympy.functions import re
+    >>> from sympy import I
+    >>> re(2+3*I)
+    2
+
+.. autoclass:: sympy.functions.elementary.complexes.re
+
+
+sqrt
+----
+
+Returns the square root of an expression. It is equivalent to raise to Rational(1,2)
+
+    >>> from sympy.functions import sqrt
+    >>> from sympy import Rational
+    >>> sqrt(2) == 2**Rational(1,2)
+    True
+
+.. autoclass:: sympy.functions.elementary.miscellaneous.sqrt
+
+
+sign
+----
+
+Returns the sign of an expression, that is:
+    -1 if expr is negative
+     0 if expr is zero
+     1 if expr is positive
+
+     >>> from sympy.functions import sign
+     >>> sign(-1)
+     -1
+     >>> sign(0)
+     0
+
+.. autoclass:: sympy.functions.elementary.complexes.sign
+
+
+Combinatorial
+=============
+
+This module implements various combinatorial functions.
+
+
+Binomial
+--------
+
+.. autoclass:: sympy.functions.combinatorial.factorials.binomial
+
+
+Factorial
+---------
+
+.. autoclass:: sympy.functions.combinatorial.factorials.factorial
+
+
+FallingFactorial
+----------------
+
+.. autoclass:: sympy.functions.combinatorial.factorials.FallingFactorial
+
+
+MultiFactorial
+--------------
+
+.. autoclass:: sympy.functions.combinatorial.factorials.MultiFactorial
+
+
+RisingFactorial
+---------------
+
+.. autoclass:: sympy.functions.combinatorial.factorials.RisingFactorial
+
+
+Special
+=======
+
+DiracDelta
+----------
+.. autoclass:: sympy.functions.special.delta_functions.DiracDelta
+
+Heaviside
+---------
+.. autoclass:: sympy.functions.special.delta_functions.Heaviside
+
+gamma
+-----
+.. autoclass:: sympy.functions.special.gamma_functions.gamma
+
+loggamma
+--------
+.. autoclass:: sympy.functions.special.gamma_functions.loggamma
+
+polygamma
+---------
+.. autoclass:: sympy.functions.special.gamma_functions.polygamma
+
+uppergamma
+----------
+.. autoclass:: sympy.functions.special.gamma_functions.uppergamma
+
+lowergamma
+----------
+.. autoclass:: sympy.functions.special.gamma_functions.lowergamma
+
+Bessel Type Functions
+---------------------
+.. autoclass:: sympy.functions.special.bessel.besselj
+.. autoclass:: sympy.functions.special.bessel.bessely
+.. autoclass:: sympy.functions.special.bessel.besseli
+.. autoclass:: sympy.functions.special.bessel.besselk
+.. autoclass:: sympy.functions.special.bessel.hankel1
+.. autoclass:: sympy.functions.special.bessel.hankel2
+.. autoclass:: sympy.functions.special.bessel.jn
+.. autoclass:: sympy.functions.special.bessel.yn
+
+Hypergeometric Functions
+------------------------
+.. autoclass:: sympy.functions.special.hyper.hyper
+.. autoclass:: sympy.functions.special.hyper.meijerg
diff --git a/lib/sympy/doc/src/modules/galgebra/GA/BasicGAtest.py b/lib/sympy/doc/src/modules/galgebra/GA/BasicGAtest.py
new file mode 100755
--- /dev/null
+++ b/lib/sympy/doc/src/modules/galgebra/GA/BasicGAtest.py
@@ -0,0 +1,28 @@
+        MV.setup('a b c d e')
+        MV.set_str_format(1)
+
+        print 'e|(a^b) =',e|(a^b)
+        print 'e|(a^b^c) =',e|(a^b^c)
+        print 'a*(b^c)-b*(a^c)+c*(a^b) =',a*(b^c)-b*(a^c)+c*(a^b)
+        print 'e|(a^b^c^d) =',e|(a^b^c^d)
+        print -d*(a^b^c)+c*(a^b^d)-b*(a^c^d)+a*(b^c^d)
+
+        print (a^b)|(c^d)
+
+e|(a^b) = {-(b.e)}a
++{(a.e)}b
+
+e|(a^b^c) = {(c.e)}a^b
++{-(b.e)}a^c
++{(a.e)}b^c
+
+a*(b^c)-b*(a^c)+c*(a^b) = {3}a^b^c
+
+e|(a^b^c^d) = {-(d.e)}a^b^c
++{(c.e)}a^b^d
++{-(b.e)}a^c^d
++{(a.e)}b^c^d
+
+{4}a^b^c^d
+
+{(a.d)*(b.c) - (a.c)*(b.d)}1
diff --git a/lib/sympy/doc/src/modules/galgebra/GA/Dirac.py b/lib/sympy/doc/src/modules/galgebra/GA/Dirac.py
new file mode 100755
--- /dev/null
+++ b/lib/sympy/doc/src/modules/galgebra/GA/Dirac.py
@@ -0,0 +1,41 @@
+#!/usr/local/bin/python
+#Dirac.py
+
+from sympy.galgebra.GA import *
+from sympy.galgebra.latex_ex import *
+from sympy import *
+
+set_main(sys.modules[__name__])
+
+if __name__ == '__main__':
+
+    metric = '1  0  0  0,'+\
+             '0 -1  0  0,'+\
+             '0  0 -1  0,'+\
+             '0  0  0 -1'
+
+    vars = make_symbols('t x y z')
+    MV.setup('gamma_t gamma_x gamma_y gamma_z',metric,True,vars)
+    
+    parms = make_symbols('m e')
+    Format('1 1 1 1')
+    I = MV(ONE,'pseudo')
+    nvars = len(vars)
+    psi = MV('psi','spinor',fct=True)
+    A = MV('A','vector',fct=True)    
+    sig_x = gamma_x*gamma_t
+    sig_y = gamma_y*gamma_t
+    sig_z = gamma_z*gamma_t
+    print '$A$ is 4-vector potential'
+    print A
+    print r'$\bm{\psi}$ is 8-component real spinor (even multi-vector)'
+    print psi
+    dirac_eq = psi.grad()*I*sig_z-e*A*psi-m*psi*gamma_t
+    dirac_eq.simplify()
+    print 'Dirac equation in terms of real geometric algebra/calculus '+\
+          r'$\lp\nabla \bm{\psi} I \sigma_{z}-eA\bm{\psi} = m\bm{\psi}\gamma_{t}\rp$'
+    print 'Spin measured with respect to $z$ axis'
+    Format('mv=3')
+    print r'\nabla \bm{\psi} I \sigma_{z}-eA\bm{\psi}-m\bm{\psi}\gamma_{t} = ',dirac_eq,' = 0'
+    xdvi(filename='Dirac.tex')
+    
diff --git a/lib/sympy/doc/src/modules/galgebra/GA/GAsympy.txt b/lib/sympy/doc/src/modules/galgebra/GA/GAsympy.txt
new file mode 100644
--- /dev/null
+++ b/lib/sympy/doc/src/modules/galgebra/GA/GAsympy.txt
@@ -0,0 +1,1776 @@
+**************************************
+  Geometric Algebra Module for Sympy
+**************************************
+
+:Author: Alan Bromborsky
+
+.. |release| replace:: 0.10
+
+.. % Complete documentation on the extended LaTeX markup used for Python
+.. % documentation is available in ``Documenting Python'', which is part
+.. % of the standard documentation for Python.  It may be found online
+.. % at:
+.. %
+.. % http://www.python.org/doc/current/doc/doc.html
+.. % \lstset{language=Python}
+.. % \input{macros}
+.. % This is a template for short or medium-size Python-related documents,
+.. % mostly notably the series of HOWTOs, but it can be used for any
+.. % document you like.
+.. % The title should be descriptive enough for people to be able to find
+.. % the relevant document.
+
+.. % Increment the release number whenever significant changes are made.
+.. % The author and/or editor can define 'significant' however they like.
+
+.. % At minimum, give your name and an email address.  You can include a
+.. % snail-mail address if you like.
+
+.. % This makes the Abstract go on a separate page in the HTML version;
+.. % if a copyright notice is used, it should go immediately after this.
+.. %
+.. % \ifhtml
+.. % \chapter*{Front Matter\label{front}}
+.. % \fi
+.. % Copyright statement should go here, if needed.
+.. % ...
+.. % The abstract should be a paragraph or two long, and describe the
+
+.. % scope of the document.
+
+.. math::
+
+    \newcommand{\bfrac}[2]{\displaystyle\frac{#1}{#2}}
+    \newcommand{\lp}{\left (}
+    \newcommand{\rp}{\right )}
+    \newcommand{\half}{\frac{1}{2}}
+    \newcommand{\llt}{\left <}
+    \newcommand{\rgt}{\right >}
+    \newcommand{\abs}[1]{\left |{#1}\right | }
+    \newcommand{\pdiff}[2]{\bfrac{\partial {#1}}{\partial {#2}}}
+    \newcommand{\lbrc}{\left \{}
+    \newcommand{\rbrc}{\right \}}
+    \newcommand{\W}{\wedge}
+    \newcommand{\R}{\dagger}
+    \newcommand{\lbrk}{\left [}
+    \newcommand{\rbrk}{\right ]}
+    \newcommand{\proj}[2]{\llt {#1} \rgt_{#2}}
+    \newcommand{\bm}{\boldsymbol}
+
+
+
+.. topic:: Abstract
+
+   This document describes the implementation of a geometric algebra module in
+   python  that utilizes the :mod:`sympy` symbolic algebra library.  The   python
+   module :mod:`GA` has been developed for coordinate free calculations using
+   the operations (geometric, outer, and inner products etc.) of geometric algebra.
+   The operations can be defined using a completely arbitrary metric defined
+   by the inner products of a set of arbitrary vectors or the metric  can be
+   restricted to enforce orthogonality and signature constraints on the set of
+   vectors.  In addition the module includes the geometric, outer (curl) and inner
+   (div) derivatives and the ability to define a curvilinear coordinate  system.
+   The module requires the numpy and the sympy modules.
+
+
+What is Geometric Algebra?
+==========================
+
+Geometric algebra is the Clifford algebra of a real finite dimensional vector
+space or the algebra that  results when a real finite dimensional vector space
+is extended with a product of vectors (geometric product) that is associative,
+left and right distributive, and yields a real number for the square (geometric
+product) of any vector [Hestenes,Lasenby].  The elements of the geometric
+algebra are called multivectors and consist of the linear combination of
+scalars, vectros, and the geometric product of two or more vectors. The
+additional axioms  for the geometric algebra are that for any vectors :math:`a`,
+:math:`b`, and :math:`c` in the base vector space:
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+  \begin{array}{c}
+  a\lp bc \rp = \lp ab \rp c \\
+  a\lp b+c \rp = ab+ac \\
+  \lp a + b \rp c = ac+bc \\
+  aa = a^{2} \in \Re
+  \end{array}
+  \end{equation*}
+
+
+By induction these also apply to any multivectors.
+
+Several software packages for numerical geometric algebra calculations are
+available from Doran-Lazenby group and the Dorst group. Symbolic packages for
+Clifford algebra using orthongonal bases such as
+:math:`e_{i}e_{j}+e_{j}e_{i} = 2\eta_{ij}`, where :math:`\eta_{ij}` is a numeric
+array are available in Maple and Mathematica. The symbolic algebra module,
+:mod:`GA`, developed for python does not depend on an orthogonal basis
+representation, but rather is generated from a set of :math:`n` arbitrary
+symbolic vectors,  :math:`a_{1},a_{2},\dots,a_{n}` and a symbolic metric
+tensor :math:`g_{ij} = a_{i}\cdot a_{j}`.
+
+In order not to reinvent the wheel all scalar symbolic algebra is handled by the
+python module  :mod:`sympy`.
+
+The basic geometic algebra operations will be implemented in python by defining
+a multivector  class, MV, and overloading the python operators in Table
+:ref:`1 <table1>` where ``A`` and ``B``  are any two multivectors (In the case of
+``+``, ``-``, ``*``, ``^``, ``|``, ``<<``, and ``>>`` the operation is also defined if ``A`` or
+``B`` is a sympy symbol or a sympy real number).
+
+.. _table1:
+
+.. csv-table::
+    :header: " Operation ", " Result "
+    :widths: 10, 40
+
+    " ``A+B`` ", " sum of multivectors "
+    " ``A-B`` ", " difference of multivectors "
+    " ``A*B`` ", " geometric product "
+    " ``A^B`` ", " outer product of multivectors  "
+    " ``A|B`` ", " inner product of multivectors "
+    " ``A<B`` or ``A<<B`` ", " left contraction of multivectors "
+    " ``A>B`` or ``A>>B`` ", " right contraction of multivectors "
+
+Table :ref:`1 <table1>`. Multivector operations for symbolicGA
+
+The option to use ``<`` or ``<<`` for left contraction and ``>`` or ``>>`` is given since the ``<`` and
+``>`` operators do not have r-forms (there are no *__rlt__()* and *__rlt__()* functions to overload) while
+``<<`` and ``>>`` do have r-forms so that *x << A* and *x >> A* are allowed where *x* is a scalar (symbol or integer)
+and *A* is a multivector. With ``<`` and ``>`` we can only have mixed modes (scalars and multivectors) if the
+multivector is the first operand.
+
+.. note::
+
+    Except for ``<`` and ``>`` all the multivector operators have r-forms so that as long as one of the
+    operands, left or right, is a multivector the other can be a multivector or a scalar (sympy symbol or integer).
+
+.. warning::
+
+    Note that the operator order precedence is determined by python and is not
+    neccessarily that used by  geometric algebra. It is **absolutely essential** to
+    use parenthesis in multivector
+    expressions containing ``^``, ``|``, ``<``, ``>``, ``<<`` and/or ``>>``.  As an example let
+    ``A`` and ``B`` be any two multivectors. Then ``A + A*B = A +(A*B)``, but
+    ``A+A^B = (2*A)^B`` since in python the ``^`` operator has a lower precedence
+    than the '+' operator.  In geometric algebra the outer and inner products and
+    the left and right contractions have a higher precedence than the geometric
+    product and the geometric product has a higher precedence than addition and
+    subtraction.  In python the ``^``, ``|``, ``<``, ``>``, ``<<`` and ``>>`` all have a lower
+    precedence than ``+`` and ``-`` while ``*`` has a higher precedence than
+    ``+`` and ``-``.
+
+
+.. _vbm:
+
+Vector Basis and Metric
+=======================
+
+The two structures that define the :class:`MV` (multivector) class are the
+symbolic basis vectors and the  symbolic metric.  The symbolic basis
+vectors are input as a string with the symbol name separated by spaces.  For
+example if we are calculating the geometric algebra of a system with three
+vectors that we wish to denote as ``a0``, ``a1``, and ``a2`` we would define the
+string variable:
+
+``basis = 'a0 a1 a2'``
+
+that would be input into the multivector setup function.  The next step would be
+to define the symbolic  metric for the geometric algebra of the basis we
+have defined. The default metric is the most general and is the matrix of
+the following symbols
+
+.. _eq1:
+
+
+
+.. math::
+  :label: 1
+  :nowrap:
+
+  \begin{equation*}
+  g = \lbrk
+  \begin{array}{ccc}
+    a0**2 & (a0.a1)  & (a0.a2) \\
+    (a0.a1) & a1**2  & (a1.a2) \\
+    (a0.a2) & (a1.a2) & a2**2 \\
+  \end{array}
+  \rbrk
+  \end{equation*}
+
+
+where each of the :math:`g_{ij}` is a symbol representing all of the dot
+products of the basis vectors. Note that the symbols are named so that
+:math:`g_{ij} = g_{ji}` since for the symbol function
+:math:`(a0.a1) \ne (a1.a0)`.
+
+Note that the strings shown in equation :ref:`1 <eq1>` are only used when the values
+of :math:`g_{ij}` are output (printed).   In the :mod:`GA` module (library)
+the :math:`g_{ij}` symbols are stored in a static member list of the multivector
+class :class:`MV` as the double list *MV.metric* (:math:`g_{ij}` = *MV.metric[i][j]*).
+
+The default definition of :math:`g` can be overwritten by specifying a string
+that will define :math:`g`. As an example  consider a symbolic representation
+for conformal geometry. Define for a basis
+
+``basis = 'a0 a1 a2 n nbar'``
+
+and for a metric
+
+``metric = '# # # 0 0, # # # 0 0, # # # 0 0, 0 0 0 0 2, 0 0 0 2 0'``
+
+then calling `MV.setup(basis,metric)` would initialize
+
+
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+  g = \lbrk
+  \begin{array}{ccccc}
+    a0**2 & (a0.a1)  & (a0.a2) & 0 & 0\\
+    (a0.a1) & a1**2  & (a1.a2) & 0 & 0\\
+    (a0.a2) & (a1.a2) & a2**2 & 0 & 0 \\
+    0 & 0 & 0 & 0 & 2 \\
+    0 & 0 & 0 & 2 & 0
+  \end{array}
+  \rbrk
+  \end{equation*}
+
+
+Here we have specified that :math:`n` and :math:`nbar` are orthonal to all the
+:math:`a`'s, :math:`n**2 = nbar**2 = 0`, and :math:`(n.nbar) = 2`. Using
+:math:`\#` in the metric definition string just tells the program to use the
+default symbol for that value.
+
+When ``MV.setup`` is called multivector representations of the basis local to
+the program are instantiated.  For our first example that means that the
+symbolic vectors named ``a0``, ``a1``, and ``a2`` are created and made available
+to the programmer for future calculations.
+
+In addition to the basis vectors the :math:`g_{ij}` are also made available to
+the programer with the following convention. If ``a0`` and ``a1`` are basis
+vectors, then their dot products are denoted by  ``a0sq``, ``a2sq``, and
+``a0dota1`` for use as python program varibles. If you print  ``a0sq`` the
+output would be ``a0**2`` and the output for ``a0dota1`` would be ``(a0.a1)`` as
+shown in equation :ref:`1 <eq1>`.  If the default value are overridden the new
+values are output by print.  For examle if :math:`g_{00} = 0` then "``print
+a0sq``" would output "0."
+
+More generally, if ``metric`` is not a string, but a list of lists or a two
+dimension numpy array, it is assumed that each element of ``metric`` is symbolic
+variable so that the :math:`g_{ij}` could be defined as symbolic functions as
+well as variables. For example instead of letting :math:`g_{01} = (a0.a1)` we
+could have  :math:`g_{01} = cos(theta)` where we use a symbolic :math:`\cos`
+function.
+
+.. note::
+
+    Additionally ``MV.setup`` has an option for an othogonal basis where the signature
+    of the metric space is defined by a string.  For example if the signature of the
+    vector space is :math:`(1,1,1)` (Euclidian 3-space) set
+
+    ``metric = '[1,1,1]'``
+
+    Likewise if the signature is that of spacetime, :math:`(1,-1,-1,-1)` then define
+
+    ``metric = '[1,-1,-1,-1]'``.
+
+
+Representation and Reduction of Multivector Bases
+=================================================
+
+In our symbolic geometric algebra we assume that all multivectors of interest to
+us can be obtained from the symbolic basis vectors we have input, via the
+different operations available to geometric algebra. The first problem we have
+is representing the general multivector in terms terms of the basis vectors.  To
+do this we  form the ordered geometric products of the basis vectors and develop
+an internal representation of these  products in terms of python classes.  The
+ordered geometric products are all multivectors of the form
+:math:`a_{i_{1}}a_{i_{2}}\dots a_{i_{r}}` where :math:`i_{1}<i_{2}<\dots <i_{r}`
+and :math:`r \le n`. We call these multivectors bases and represent them
+internally with the list of integers
+:math:`\lbrk i_{1},i_{2},\dots,i_{r}\rbrk`. The bases are labeled, for the
+purpose of output display, with strings that are concatenations of the strings
+representing the basis vectors.  So that in our example  ``[1,2]`` would be
+labeled with the string ``'a1a2'`` and represents the geometric product
+``a1*a2``. Thus the list ``[0,1,2]`` represents ``a0*a1*a2``.For our example the
+complete set of bases and labels are shown in Table :ref:`2 <table2>`
+
+.. note::
+
+    The empty list, ``[]``, represents the scalar 1.
+
+.. _table2:
+
+::
+
+   MV.basislabel = ['1', ['a0', 'a1', 'a2'], ['a0a1', 'a0a2', 'a1a2'],
+                    ['a0a1a2']]
+   MV.basis      = [[], [[0], [1], [2]], [[0, 1], [0, 2], [1, 2]],
+                    [[0, 1, 2]]]
+
+Table :ref:`2 <table2>`. Multivector basis labels and internal basis representation.
+
+Since there are :math:`2^{n}` bases and the number of bases with equal list
+lengths is the same as for the  grade decomposition of a dimension :math:`n`
+geometric algebra we will call the collections of bases of equal length
+``psuedogrades``.
+
+The critical operation in setting up the geometric algebra module is reducing
+the geomertric product of any two bases to a linear combination of bases so that
+we can calculate a multiplication table for the bases.   First we represent the
+product as the concatenation of two base lists.  For example ``a1a2*a0a1`` is
+represented by the list ``[1,2]+[0,1] = [1,2,0,1]``. The representation of the
+product is reduced via two operations, contraction and revision. The state of
+the reduction is saved in two lists of equal length.  The first list contains
+symbolic scale factors (symbol or numeric types) for the corresponding interger
+list representing the product of  bases.  If we wish to reduce
+:math:`\lbrk i_{1},\dots,i_{r}\rbrk` the starting point is the coefficient list
+:math:`C = \lbrk 1 \rbrk` and the bases list
+:math:`B = \lbrk \lbrk i_{1},\dots,i_{r}\rbrk \rbrk`.  We now operate  on each
+element of the lists as follows:
+
+* ``contraction``: Consider a basis list :math:`B` with element
+  :math:`B[j] = \lbrk i_{1},\dots,i_{l},i_{l+1},\dots,i_{s}\rbrk` where
+  :math:`i_{l} = i_{l+1}`. Then the product of the :math:`l` and :math:`l+1` terms
+  result in a scalar and :math:`B[j]` is replaced by the new list representation
+  :math:`\lbrk i_{1},\dots,i_{l-1},i_{l+2},\dots,i_{r}\rbrk` which is of  psuedo
+  grade :math:`r-2` and :math:`C[j]` is replaced by the symbol
+  :math:`g_{i_{l}i_{l}}C[j]`.
+
+* ``revision``: Consider a basis list :math:`B` with element
+  :math:`B[j] = \lbrk i_{1},\dots,i_{l},i_{l+1},\dots,i_{s}\rbrk` where
+  :math:`i_{l} > i_{l+1}`. Then the :math:`l` and :math:`l+1` elements must be
+  reversed to be put in normal order, but we have :math:`a_{i_{l}}a_{i_{l+1}} = 2g_{i_{l}i_{l+1}}-a_{i_{l+1}}a_{i_{l}}`
+  (From the geometric  algebra definition of the dot product of two vectors). Thus
+  we append the list representing the reduced element,
+  :math:`\lbrk i_{1},\dots,i_{l-1},i_{l+2},\dots,i_{s}\rbrk`, to the pseudo bases
+  list, :math:`B`, and append :math:`2g_{i{l}i_{l+1}}C[j]` to the coefficients
+  list, then  we replace :math:`B[j]` with
+  :math:`\lbrk i_{1},\dots,i_{l+1},i_{l},\dots,i_{s}\rbrk`  and :math:`C[j]` with
+  :math:`-C[j]`. Both lists are increased by one element if
+  :math:`g_{i_{l}i_{l+1}} \ne 0`.
+
+These processes are repeated untill every basis list in :math:`B` is in normal
+(ascending) order with no repeated elements.  Then the coefficents of equivalent
+bases are summed and the bases sorted according to psuedograde and ascending
+order.  We now have a way of calculating the geometric product of any two bases
+as a symbolic linear combination of all the bases with the coefficients
+determined by :math:`g`. The base multiplication table for our simple example of
+three vectors is given by (the coefficient of each psuedo base is enclosed with
+{} for clarity):
+
+
+.. include:: multable.dat
+   :literal:
+
+
+Base Representation of Multivectors
+===================================
+
+In terms of the bases defined an arbitrary multivector can be represented as a
+list of arrays  (we use the numpy python module to implement arrays).   If we
+have :math:`n` basis vectors we initialize the list ``self.mv = [0,0,...,0]``
+with :math:`n+1` integers all zero.   Each zero is a placeholder for an array of
+python objects (in this case the objects will be sympy  symbol objects). If
+``self.mv[r] = numpy.array([list of symbol objects])`` each entry in the
+``numpy.array`` will be a coefficient of the corresponding psuedo base.
+``self.mv[r] = 0`` indicates that the coefficients of every base of psuedo grade
+:math:`r` are 0.  The length of the array ``self.mv[r]`` is :math:`n \choose r`
+the binomial coefficient. For example the psuedo basis vector ``a1`` would be
+represented as a multivector by the list:
+
+``a1.mv = [0,numpy.array([numeric(0),numeric(1),numeric(0)]),0,0]``
+
+and ``a0a1a2`` by:
+
+``a0a1a2.mv = [0,0,0,numpy.array([numeric(1)])]``
+
+The array is stuffed with sympy numeric objects instead of python integers so
+that we can perform symbolically manipulate sympy expressions that consist of
+scalar algebraic symbols and exact rational numbers which sympy can also
+represent.
+
+The ``numpy.array`` is used because operations of addition, substraction, and
+multiplication by an object are defined for the array if they are defined for
+the objects making up the array, which they are by sympy.  We call this
+representation a base type because the ``r`` index is not a grade index since
+the bases we are using are not blades. In a blade representation the structure
+would be identical, but the bases would be replaced by  blades and
+``self.mv[r]`` would represent the ``r`` grade components of the multivector.
+The first use of the base representation is to store the results of the
+multiplication tabel for the bases in the class variable ``MV.mtabel``.  This
+variable is a group of nested lists so that the geometric product of the
+``igrade`` and  ``ibase`` with the ``jgrade`` and ``jbase`` is
+``MV.mtabel[igrade][ibase][jgrade][jbase]``.  We can then use this table to
+calculate the geometric product of any two multivectors.
+
+
+Blade Representation of Multivectors
+====================================
+
+Since we can now calculate the symbolic geometric product of any two
+multivectors we can also calculate the blades corresponding to the product of
+the symbolic basis vectors using the formula
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+    A_{r}\W b = \half\lp A_{r}b-\lp -1 \rp^{r}bA_{r} \rp,
+  \end{equation*}
+
+
+where :math:`A_{r}` is a multivector of grade :math:`r` and :math:`b` is a
+vector.  For our example basis the result  is shown in Table :ref:`3 <table3>`.
+
+.. _table3:
+
+::
+
+   1 = 1
+   a0 = a0
+   a1 = a1
+   a2 = a2
+   a0^a1 = {-(a0.a1)}1+a0a1
+   a0^a2 = {-(a0.a2)}1+a0a2
+   a1^a2 = {-(a1.a2)}1+a1a2
+   a0^a1^a2 = {-(a1.a2)}a0+{(a0.a2)}a1+{-(a0.a1)}a2+a0a1a2
+
+Table :ref:`3 <table3>`. Bases blades in terms of bases.
+
+The important thing to notice about Table :ref:`3 <table3>` is that it is a
+triagonal (lower triangular) system of equations so that using a simple back
+substitution algorithym we can solve for the psuedo bases in terms of the blades
+giving Table :ref:`4 <table4>`.
+
+.. _table4:
+
+::
+
+   1 = 1
+   a0 = a0
+   a1 = a1
+   a2 = a2
+   a0a1 = {(a0.a1)}1+a0^a1
+   a0a2 = {(a0.a2)}1+a0^a2
+   a1a2 = {(a1.a2)}1+a1^a2
+   a0a1a2 = {(a1.a2)}a0+{-(a0.a2)}a1+{(a0.a1)}a2+a0^a1^a2
+
+Table :ref:`4 <table4>`. Bases in terms of basis blades.
+
+Using Table :ref:`4 <table4>` and simple substitution we can convert from a base
+multivector representation to a blade representation.  Likewise, using Table
+:ref:`3 <table3>` we can convert from  blades to bases.
+
+Using the blade representation it becomes simple to program functions that will
+calculate the grade projection, reverse, even, and odd multivector functions.
+
+Note that in the multivector class ``MV`` there is a class variable for each
+instantiation, ``self.bladeflg``, that is set to zero for a base representation
+and 1 for a blade representation.  One needs to keep track of which
+representation is in use since various multivector operations require conversion
+from one representation to the other.
+
+.. warning::
+
+    When the geometric product of two multivectors is calculated the module looks to
+    see if either multivector is in blade representation.  If either is the result of
+    the geometric product is converted to a blade representation.  One result of this
+    is that if either of the multivectors is a simple vector (which is automatically a
+    blade) the result will be in a blade representation.  If ``a`` and ``b`` are vectors
+    then the result ``a*b`` will be ``(a.b)+a^b`` or simply ``a^b`` if ``(a.b) = 0``.
+
+
+Outer and Inner Products, Left and Right Contractions
+=====================================================
+
+In geometric algebra any general multivector :math:`A` can be decomposed into
+pure grade  multivectors (a linear combination of blades of all the same order)
+so that in a :math:`n`-dimensional vector space
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+  A = \sum_{r = 0}^{n}A_{r}
+  \end{equation*}
+
+
+The geometric product of two pure grade multivectors :math:`A_{r}` and
+:math:`B_{s}` has the form
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+  A_{r}B_{s} = \proj{A_{r}B_{s}}{\abs{r-s}}+\proj{A_{r}B_{s}}{\abs{r-s}+2}+\cdots+\proj{A_{r}B_{s}}{r+s}
+  \end{equation*}
+
+
+where :math:`\proj{}{t}` projects the :math:`t` grade components of the
+multivector argument.  The   inner and outer products of :math:`A_{r}` and
+:math:`B_{s}` are then defined to be
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+  A_{r}\cdot B_{s} = \proj{A_{r}B_{s}}{\abs{r-s}}
+  \end{equation*}
+
+
+
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+  A_{r}\wedge B_{s} = \proj{A_{r}B_{s}}{r+s}
+  \end{equation*}
+
+
+and
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+  A\cdot B = \sum_{r,s}A_{r}\cdot B_{s}
+  \end{equation*}
+
+
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+  A\wedge B = \sum_{r,s}A_{r}\wedge B_{s}
+  \end{equation*}
+
+
+Likewise the right (:math:`\lfloor`) and left (:math:`\rfloor`) contractions are defined as
+
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+  A_{r}\lfloor B_{s} = \left \{ \begin{array}{cc}
+     \proj{A_{r}B_{s}}{r-s} &  r \ge s \\
+               0            &  r < s \end{array} \right \}
+  \end{equation*}
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+  A_{r}\rfloor B_{s} = \left \{ \begin{array}{cc}
+     \proj{A_{r}B_{s}}{s-r} &  s \ge r \\
+               0            &  s < r \end{array} \right \}
+  \end{equation*}
+
+
+and
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+  A\lfloor B = \sum_{r,s}A_{r}\lfloor B_{s}
+  \end{equation*}
+
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+  A\rfloor B = \sum_{r,s}A_{r}\rfloor B_{s}
+  \end{equation*}
+
+
+The ``MV`` class function for the outer product of the multivectors ``mv1`` and
+``mv2`` is ::
+
+   @staticmethod
+   def outer_product(mv1,mv2):
+        product = MV()
+        product.bladeflg = 1
+        mv1.convert_to_blades()
+        mv2.convert_to_blades()
+        for igrade1 in MV.n1rg:
+            if not isint(mv1.mv[igrade1]):
+                pg1 = mv1.project(igrade1)
+                for igrade2 in MV.n1rg:
+                    igrade = igrade1+igrade2
+                    if igrade <= MV.n:
+                        if not isint(mv2.mv[igrade2]):
+                            pg2 = mv2.project(igrade2)
+                            pg1pg2 = pg1*pg2
+                            product.add_in_place(pg1pg2.project(igrade))
+        return(product)
+
+
+The steps for calculating the outer product are:
+
+#. Convert ``mv1`` and ``mv2`` to blade representation if they are not already
+   in that form.
+
+#. Project and loop through each grade ``mv1.mv[i1]`` and ``mv2.mv[i2]``.
+
+#. Calculate the geometric product ``pg1*pg2``.
+
+#. Project the ``i1+i2`` grade from ``pg1*pg2``.
+
+#. Accumulate the results for each pair of grades in the input multivectors.
+
+.. warning::
+
+    In the  ``MV`` class we have overloaded the ``^`` operator to represent the outer
+    product so that instead of calling the outer product function we can write ``mv1^ mv2``.
+    Due to the precedence rules for python it is **absolutely essential** to enclose outer products
+    in parenthesis.
+
+For the inner product of the multivectors ``mv1`` and ``mv2`` the ``MV`` class
+function is  ::
+
+    @staticmethod
+    def inner_product(mv1,mv2,mode='s'):
+        """
+        MV.inner_product(mv1,mv2) calculates the inner
+
+        mode = 's' - symmetic (Doran & Lasenby)
+        mode = 'l' - left contraction (Dorst)
+        mode = 'r' - right contraction (Dorst)
+        """
+        if isinstance(mv1,MV) and isinstance(mv2,MV):
+            product = MV()
+            product.bladeflg = 1
+            mv1.convert_to_blades()
+            mv2.convert_to_blades()
+            for igrade1 in range(MV.n1):
+                if isinstance(mv1.mv[igrade1],numpy.ndarray):
+                    pg1 = mv1.project(igrade1)
+                    for igrade2 in range(MV.n1):
+                        igrade = igrade1-igrade2
+                        if mode == 's':
+                            igrade = igrade.__abs__()
+                        else:
+                            if mode == 'l':
+                                igrade = -igrade
+                        if igrade >= 0:
+                            if isinstance(mv2.mv[igrade2],numpy.ndarray):
+                                pg2 = mv2.project(igrade2)
+                                pg1pg2 = pg1*pg2
+                                product.add_in_place(pg1pg2.project(igrade))
+            return(product)
+        else:
+            if mode == 's':
+                if isinstance(mv1,MV):
+                    product = mv1.scalar_mul(mv2)
+                if isinstance(mv2,MV):
+                    product = mv2.scalar_mul(mv1)
+            else:
+                product = None
+        return(product)
+
+
+The inner product is calculated the same way as the outer product except that in
+step 4, ``i1+i2`` is replaced by ``abs(i1-i2)`` or ``i1-i2`` for the right contraction or
+``i2-i1`` for the left contraction.  If ``i1-i2`` is less than zero there is no contribution
+to the right contraction.  If ``i2-i1`` is less than zero there is no contribution to the
+left contraction.
+
+.. warning::
+
+    In the ``MV`` class we have overloaded the ``|`` operator for the inner product,
+    ``>`` operator for the right contraction, and ``<`` operator for the left contraction.
+    Instead of calling the inner product function we can write ``mv1|mv2``, ``mv1>mv2``, or
+    ``mv1<mv2`` respectively for the inner product, right contraction, or left contraction.
+    Again, due to the precedence rules for python it is **absolutely essential** to enclose inner
+    products and/or contractions in parenthesis.
+
+
+.. _reverse:
+
+Reverse of Multivector
+======================
+
+If :math:`A` is the geometric product of :math:`r` vectors
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+    A = a_{1}\dots a_{r}
+  \end{equation*}
+
+
+then the reverse of :math:`A` designated :math:`A^{\R}` is defined by
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+    A^{\R} \equiv a_{r}\dots a_{1}.
+  \end{equation*}
+
+
+The reverse is simply the product with the order of terms reversed.  The reverse
+of a sum of products is defined as the sum of the reverses so that for a general
+multivector A we have
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+    A^{\R} = \sum_{i=0}^{N} \proj{A}{i}^{\R}
+  \end{equation*}
+
+
+but
+
+.. _eq14:
+
+
+
+.. math::
+  :label: 2
+  :nowrap:
+
+  \begin{equation*}
+    \proj{A}{i}^{\R} = \lp -1\rp^{\frac{i\lp i-1\rp}{2}}\proj{A}{i}
+  \end{equation*}
+
+
+which is proved by expanding the blade bases in terms of orthogonal vectors and
+showing that equation :ref:`2 <eq14>` holds for the geometric product of orthogonal
+vectors.
+
+The reverse is important in the theory of rotations in :math:`n`-dimensions.  If
+:math:`R` is the product of an even number of vectors and :math:`RR^{\R} = 1`
+then :math:`RaR^{\R}` is a composition of rotations of the  vector :math:`a`.
+If :math:`R` is the product of two vectors then the plane that :math:`R` defines
+is the plane of the rotation.  That is to say that :math:`RaR^{\R}` rotates the
+component of :math:`a` that is projected into the plane defined by :math:`a` and
+:math:`b` where :math:`R=ab`.  :math:`R` may be written
+:math:`R = e^{\frac{\theta}{2}U}`, where :math:`\theta` is the angle of rotation
+and :math:`u` is a unit blade :math:`\lp u^{2} = \pm 1\rp` that defines the
+plane of rotation.
+
+
+.. _recframe:
+
+Reciprocal Frames
+=================
+
+If we have :math:`M` linearly independent vectors (a frame),
+:math:`a_{1},\dots,a_{M}`, then the reciprocal frame is
+:math:`a^{1},\dots,a^{M}` where :math:`a_{i}\cdot a^{j} = \delta_{i}^{j}`,
+:math:`\delta_{i}^{j}` is the Kronecker delta (zero if :math:`i \ne j` and one
+if :math:`i = j`). The reciprocal frame is constructed as follows:
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+    E_{M} = a_{1}\W\dots\W a_{M}
+  \end{equation*}
+
+
+
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+    E_{M}^{-1} = \bfrac{E_{M}}{E_{M}^{2}}
+  \end{equation*}
+
+
+Then
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+    a^{i} = \lp -1\rp^{i-1}\lp a_{1}\W\dots\W \breve{a}_{i} \W\dots\W a_{M}\rp E_{M}^{-1}
+  \end{equation*}
+
+
+where :math:`\breve{a}_{i}` indicates that :math:`a_{i}` is to be deleted from
+the product.  In the standard notation if a vector is denoted with a subscript
+the reciprocal vector is denoted with a superscript. The multivector setup
+function ``MV.setup(basis,metric,rframe)`` has the argument ``rframe`` with a
+default value of ``False``.  If it is set to ``True`` the reciprocal frame of
+the basis vectors is  calculated. Additionaly there is the function
+``reciprocal_frame(vlst,names='')`` external to the ``MV`` class that will
+calculate the reciprocal frame of a list, ``vlst``, of vectors.  If the argument
+``names`` is set to a space delimited string of names for the vectors the
+reciprocal vectors will be given these names.
+
+
+.. _deriv:
+
+Geometric Derivative
+====================
+
+If :math:`F` is a multivector field that is a function of a vector
+:math:`x = x^{i}\bm{\gamma}_{i}` (we are using the summation convention that
+pairs of subscripts and superscripts are summed over the dimension of the vector
+space) then the geometric derivative :math:`\nabla F` is given by (in this
+section the summation convention is used):
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+    \nabla F = \bm{\gamma}^{i}\bfrac{\partial F}{\partial x^{i}}
+  \end{equation*}
+
+
+If :math:`F_{R}` is a grade-:math:`R` multivector and
+:math:`F_{R} = F_{R}^{i_{1}\dots i_{R}}\bm{\gamma}_{i_{1}}\W\dots\W \bm{\gamma}_{i_{R}}`
+then
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+    \nabla F_{R} = \bfrac{\partial F_{R}^{i_{1}\dots i_{R}}}{\partial x^{j}}\bm{\gamma}^{j}\lp\bm{\gamma}_{i_{1}}\W
+                 \dots\W \bm{\gamma}_{i_{R}} \rp
+  \end{equation*}
+
+
+Note that
+:math:`\bm{\gamma}^{j}\lp\bm{\gamma}_{i_{1}}\W\dots\W \bm{\gamma}_{i_{R}} \rp`
+can only contain grades :math:`R-1` and :math:`R+1` so that :math:`\nabla F_{R}`
+also can only contain those grades. For a grade-:math:`R` multivector
+:math:`F_{R}` the inner (div) and  outer (curl) derivatives are defined as
+
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+  \nabla\cdot F_{R} = \left < \nabla F_{R}\right >_{R-1}
+  \end{equation*}
+
+
+and
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+  \nabla\W F_{R} = \left < \nabla F_{R}\right >_{R+1}
+  \end{equation*}
+
+
+For a general multivector function :math:`F` the inner and outer derivatives are
+just the sum of the inner and outer  dervatives of each grade of the multivector
+function.
+
+Curvilinear coordinates are derived from a vector function
+:math:`x(\bm{\theta})` where
+:math:`\bm{\theta} = \lp\theta_{1},\dots,\theta_{N}\rp` where the number of
+coordinates is equal to the dimension of the  vector space.  In the case of
+3-dimensional spherical coordinates :math:`\bm{\theta} = \lp r,\theta,\phi \rp`
+and the coordinate generating function :math:`x(\bm{\theta})` is
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+  x =  r \cos\left({\phi}\right) \sin\left({\theta}\right){\bm{{\gamma}_{x}}}+ r \sin\left({\phi}\right) \sin\left({\theta}\right){\bm{{\gamma}_{y}}}+ r \cos\left({\theta}\right){\bm{{\gamma}_{z}}}
+  \end{equation*}
+
+
+A coordinate frame is derived from :math:`x` by
+:math:`\bm{e}_{i} = \pdiff{x}{\theta^{i}}`.  The following show the frame for
+spherical coordinates.
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+  \bm{e}_{r} = \cos\left({\phi}\right) \sin\left({\theta}\right){\bm{{\gamma}_{x}}}+\sin\left({\phi}\right) \sin\left({\theta}\right){\bm{{\gamma}_{y}}}+\cos\left({\theta}\right){\bm{{\gamma}_{z}}}
+  \end{equation*}
+
+
+
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+  \bm{e}_{{\theta}} = \cos\left({\phi}\right) \cos\left({\theta}\right){\bm{{\gamma}_{x}}}+r \cos\left({\theta}\right) \sin\left({\phi}\right){\bm{{\gamma}_{y}}} - r \sin\left({\theta}\right){\bm{{\gamma}_{z}}}
+  \end{equation*}
+
+
+
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+  \bm{e}_{{\phi}} =  - r \sin\left({\phi}\right) \sin\left({\theta}\right){\bm{{\gamma}_{x}}}+r \cos\left({\phi}\right) \sin\left({\theta}\right){\bm{{\gamma}_{y}}}
+  \end{equation*}
+
+
+The coordinate frame generated in this manner is not necessarily normalized so
+define a normalized frame by
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+  \bm{\hat{e}}_{i} = \bfrac{\bm{e}_{i}}{\sqrt{\abs{\bm{e}_{i}^{2}}}} = \bfrac{\bm{e}_{i}}{\abs{\bm{e}_{i}}}
+  \end{equation*}
+
+
+This works for all :math:`\bm{e}_{i}^{2} \neq 0` since we have defined
+:math:`\abs{\bm{e}_{i}} = \sqrt{\abs{\bm{e}_{i}^{2}}}`.   For spherical
+coordinates the normalized frame vectors are
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+  \bm{\hat{e}}_{r} =  \cos\left({\phi}\right) \sin\left({\theta}\right){\bm{{\gamma}_{x}}}+\sin\left({\phi}\right) \sin\left({\theta}\right){\bm{{\gamma}_{y}}}+\cos\left({\theta}\right){\bm{{\gamma}_{z}}}
+  \end{equation*}
+
+
+
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+  \bm{\hat{e}}_{{\theta}} = \cos\left({\phi}\right) \cos\left({\theta}\right){\bm{{\gamma}_{x}}}+\cos\left({\theta}\right) \sin\left({\phi}\right){\bm{{\gamma}_{y}}}- \sin\left({\theta}\right){\bm{{\gamma}_{z}}}
+  \end{equation*}
+
+
+
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+  \bm{\hat{e}}_{{\phi}} = - \sin\left({\phi}\right){\bm{{\gamma}_{x}}}+\cos\left({\phi}\right){\bm{{\gamma}_{y}}}
+  \end{equation*}
+
+
+The geometric derivative in curvilinear coordinates is given by
+
+.. math::
+  :nowrap:
+
+  \begin{align*}
+    \nabla F_{R} & =  \bm{\gamma}^{i}\pdiff{}{x^{i}}\lp F_{R}^{i_{1}\dots i_{R}}
+                     \bm{\hat{e}}_{i_{1}}\W\dots\W\bm{\hat{e}}_{i_{R}}\rp  \\
+                   & =  \bm{e^{j}}\pdiff{}{\theta^{j}}\lp F_{R}^{i_{1}\dots i_{R}}
+                     \bm{\hat{e}}_{i_{1}}\W\dots\W\bm{\hat{e}}_{i_{R}}\rp  \\
+                   & =   \lp\pdiff{}{\theta^{j}} F_{R}^{i_{1}\dots i_{R}}\rp
+                     \bm{e^{j}}\lp\bm{\hat{e}}_{i_{1}}\W\dots\W\bm{\hat{e}}_{i_{R}}\rp+
+                     F_{R}^{i_{1}\dots i_{R}}\bm{e^{j}}
+                     \pdiff{}{\theta^{j}}\lp\bm{\hat{e}}_{i_{1}}\W\dots\W\bm{\hat{e}}_{i_{R}}\rp \\
+                   & =   \lp\pdiff{}{\theta^{j}} F_{R}^{i_{1}\dots i_{R}}\rp
+                     \bm{e^{j}}\lp\bm{\hat{e}}_{i_{1}}\W\dots\W\bm{\hat{e}}_{i_{R}}\rp+
+                     F_{R}^{i_{1}\dots i_{R}}C\lbrc \bm{\hat{e}}_{i_{1}}\W\dots\W\bm{\hat{e}}_{i_{R}}\rbrc
+  \end{align*}
+
+
+where
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+  C\lbrc \bm{\hat{e}}_{i_{1}}\W\dots\W\bm{\hat{e}}_{i_{R}}\rbrc  = \bm{e^{j}}\pdiff{}{\theta^{j}}
+                                                              \lp\bm{\hat{e}}_{i_{1}}\W\dots\W\bm{\hat{e}}_{i_{R}}\rp
+  \end{equation*}
+
+
+are the connection multivectors for the curvilinear coordinate system. For a
+spherical coordinate system they are
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+  C\lbrc\bm{\hat{e}}_{r}\rbrc =  \frac{2}{r}
+  \end{equation*}
+
+
+
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+  C\lbrc\bm{\hat{e}}_{\theta}\rbrc =  \frac{\cos\left({\theta}\right)}{r \sin\left({\theta}\right)}
+                                +\frac{1}{r}\bm{\hat{e}}_{r}\W\bm{\hat{e}}_{\theta}
+  \end{equation*}
+
+
+
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+  C\lbrc\bm{\hat{e}}_{\phi}\rbrc = \frac{1}{r}{\bm{\bm{\hat{e}}_{r}}}\W\bm{\hat{e}}_{{\phi}}+ \frac{\cos\left({\theta}\right)}{r \sin\left({\theta}\right)}\bm{\hat{e}}_{{\theta}}\W\bm{\hat{e}}_{{\phi}}
+  \end{equation*}
+
+
+
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+  C\lbrc\hat{e}_{r}\W\hat{e}_{\theta}\rbrc =  - \frac{\cos\left({\theta}\right)}{r \sin\left({\theta}\right)}
+                                        \bm{\hat{e}}_{r}+\frac{1}{r}\bm{\hat{e}}_{{\theta}}
+  \end{equation*}
+
+
+
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+  C\lbrc\bm{\hat{e}}_{r}\W\bm{\hat{e}}_{\phi}\rbrc = \frac{1}{r}\bm{\hat{e}}_{{\phi}}
+                      - \frac{\cos\left({\theta}\right)}{r \sin\left({\theta}\right)}\bm{\hat{e}}_{r}\W\bm{\hat{e}}_{{\theta}}\W\bm{\hat{e}}_{{\phi}}
+  \end{equation*}
+
+
+
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+  C\lbrc\bm{\hat{e}}_{\theta}\W\bm{\hat{e}}_{\phi}\rbrc =  \frac{2}{r}\bm{\hat{e}}_{r}\W
+                                                \bm{\hat{e}}_{\theta}\W\bm{\hat{e}}_{\phi}
+  \end{equation*}
+
+
+
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+  C\lbrc\bm{\hat{e}}_r\W\bm{\hat{e}}_{\theta}\W\bm{\hat{e}}_{\phi}\rbrc = 0
+  \end{equation*}
+
+
+
+Module Components
+=================
+
+
+Initializing Multivector Class
+------------------------------
+
+The multivector class is initialized with:
+
+
+.. function:: MV.setup(basis,metric='',rframe=False,coords=None,debug=False,offset=0)
+
+   The *basis* and *metric* parameters were described in section :ref:`vbm`. If
+   *rframe=True* the  reciprocal frame of the symbolic bases vectors is calculated.
+   If *debug=True* the data structure required to initialize the :class:`MV` class
+   are printer out. *coords* is a list of :class:`sympy` symbols equal in length to
+   the number of basis vectors.  These symbols are used as the arguments of a
+   multivector field as a function of position and for calculating the derivatives
+   of a multivector field (if *coords* is defined then *rframe* is automatically
+   set equal to *True*). *offset* is an integer that is added to the multivector
+   coefficient index. For example if one wishes to start labeling vector coefficient
+   indexes at one instead of zero then set *offset=1*.  Additionally, :func:`MV.setup`
+   calculates the pseudo scalar, :math:`I` and its inverse, :math:`I^{-1}` and makes
+   them available to the programmer as *MV.I* and *MV.Iinv*.
+
+After :func:`MV.setup` is run one can reinialize the :class:`MV` class with
+curvilinear coordinates using:
+
+
+.. function:: MV.rebase(x,coords,base_name,debug=False,debug_level=0)
+
+   A typical usage of ``MV.rebase`` for generating spherical curvilinear coordinate
+   is::
+
+      metric = '1 0 0,0 1 0,0 0 1'
+      MV.setup('gamma_x gamma_y gamma_z',metric,True)
+      coords = make_symbols('r theta phi')
+      x = r*(sympy.cos(theta)*gamma_z+sympy.sin(theta)*\
+          (sympy.cos(phi)*gamma_x+sympy.sin(phi)*gamma_y))
+      x.set_name('x')
+      MV.rebase(x,coords,'e',True)
+
+   The input parameters for ``MV.rebase`` are
+
+*  ``x``: Vector function of coordinates (derivatives define curvilinear basis)
+
+*  ``coords``: List of sympy symbols for curvilinear coordinates
+
+*  ``debug``: If ``True`` printout (LaTeX) all quantities required for derivative calculation
+
+*  ``debug_level``: Set to 0,1,2, or 3 to stop curvilinear calculation before all quatities are
+   calculated.  This is done when debugging new curvilinear coordinate systems since simplification
+   of expressions is not sufficiently automated to insure success of process of any coordinate system
+   defined by vector function ``x``
+
+
+
+   To date ``MV.rebase`` works for cylindrical and spherical coordinate systems in
+   any number of dimensions  (until the execution time becomes too long).  To make
+   it work for these systems required creating some hacks for expression
+   simplification since both trigsimp and simplify were not general enough to
+   perform the required simplification.
+
+
+.. function:: MV.set_str_format(str_mode=0)
+
+   If ``str_mode=0`` the string representation of the multivector contains no
+   newline characters (prints on one line).   If ``str_mode=1`` the string
+   representation of the multivector places a newline after each grade of the
+   multivector  (prints one grade per line).  If ``str_mode=2`` the string
+   representation of the multivector places a newline after each base of the
+   multivector  (prints one base per line). In both cases bases with zero
+   coefficients are not printed.
+
+    .. note::
+
+        This function directly affects the way multivectors are printed with the
+        print command since it interacts with the :func:`__str__` function for the
+        multivector class which is used by the ``print`` command.
+
+
+        .. csv-table::
+            :header: " ``str_mode`` ", " Effect on print "
+            :widths: 10, 50
+
+            " 0 ","One multivector per line"
+            " 1 ","One grade per line"
+            " 2 ","One base per line"
+
+Instantiating a Multivector
+---------------------------
+
+Now that grades and bases have been described we can show all the ways that a
+multivector can be instantiated. As an example assume that the multivector space
+is initialized with ``MV.setup('e1 e2 e3')``. Then the vectors ``e1``, ``e2``,
+and ``e3`` are made available (broadcast) for use in the program .
+
+.. warning::
+
+    This is only true if the statement  ``set_main(sys.modules[__name__])`` appears
+    immediately after the ``from sympy.galgebra.GA import *`` statement.
+
+So that  multivectors
+could be instantiated with statements such as (``a1``, ``a2``, and ``a3`` are
+``sympy`` symbols)::
+
+   x = a1*e1+a2*e2+a3*e3
+   y = x*e1*e2
+   z = x|y
+   w = x^y
+
+or with the multivector class constructor:
+
+
+.. class:: MV(value='',mvtype='',mvname='',fct=False)
+
+   *mvname* is a  string that defines the name of the multivector for output
+   purposes. *value* and  *type* are defined by the following table and *fct* is a
+   switch that will convert the symbolic coefficients of a multivector to functions
+   if coordinate variables have been defined when :func:`MV.setup` is called:
+
+    .. csv-table::
+        :header: " mvtype ", " value ", " result "
+        :widths: 10, 30, 30
+
+        " default ", " default ", " Zero multivector "
+        " 'basisvector' ", " int i ", " :math:`\mbox{i}^{th}` basis vector "
+        " 'basisbivector' ", " int i ", " :math:`\mbox{i}^{th}` basis bivector "
+        " 'scalar' ", " symbol x ", " symbolic scalar of value x"
+        "   ", " string s ", " symbolic scalar of value Symbol(s) "
+        "  ", " int i ", " sympy integer of value i"
+        " 'grade' ", " [int r, 1-D symbol array A] ", " X.grade(r) = A "
+        "   ", " [int r, string s] ", " symbolic grade r multivector "
+        " 'vector' ", " 1-D symbol array A ", " X.grade(1) = A "
+        "   ", " string s ", " symbolic vector "
+        " 'grade2' ", " 1-D symbol array A ", " X.grade(2) = A "
+        " 'pseudo' ", " symbol x ", " X.grade(n) = x "
+        " 'spinor' ", " string s ", " symbolic even multivector "
+
+
+        " default "," string s ", " symbolic general multivector "
+
+
+   If the *value* argument has the option of being a string s then a general symbolic
+   multivector will be constructed constrained by the value of *mvtype*.  The string
+   s will be the base name of the multivector symbolic coefficients.  If *coords* is
+   not defined in :func:`MV.setup` the indices of the multivector bases are appended to
+   the base name with a double underscore (superscript notation).  If *coords* is defined
+   the coordinate names will replace the indices in the coefficient names.  For example if
+   the base string is *A* and the  coordinates *(x,y,z)* then the
+   coefficients of a spinor in 3d space would be *A*, *A__xy*, *A__xz*, and *A__yz*.  If
+   the :mod:`latex_ex` is used to print the multivector the coefficients would print as
+   :math:`A`, :math:`A^{xy}`, :math:`A^{xz}`, and :math:`A^{yz}`.
+
+   If the *fct* argrument of :func:`MV` is set to *True* and the *coords* argument in
+   :func:`MV.setup` is defined the symbolic coefficients of the multivector are functions
+   of the coordinates.
+
+
+Basic Multivector Class Functions
+---------------------------------
+
+
+.. function:: __call__(self,igrade=0,ibase=0)
+
+   ``__call__`` returns the the ``igrade``, ``ibase`` coefficient of the
+   multivector.  The defaults return the scalar component of the multivector.
+
+
+.. function:: convert_to_blades(self)
+
+   Convert multivector from the base representation to the blade representation.
+   If multivector is already in blade representation nothing is done.
+
+
+.. function:: convert_from_blades(self)
+
+   Convert multivector from the blade representation to the base representation.
+   If multivector is already in base representation nothing is done.
+
+
+.. function:: project(self,r)
+
+   If r is a integer return the grade-:math:`r` components of the multivector. If
+   r is a multivector return the grades of the multivector that correspond to the
+   non-zero grades of r. For example if one is projecting a general multivector and
+   r is a spinor, ``A.project(r)`` will return only the even grades of the multivector
+   A since a spinor only has even grades that are non-zero.
+
+
+.. function:: even(self)
+
+   Return the even grade components of the multivector.
+
+
+.. function:: odd(self)
+
+   Return the odd grade components of the multivector.
+
+
+.. function:: rev(self)
+
+   Return the reverse of the multivector.  See section :ref:`reverse`.
+
+
+.. function:: is_pure(self)
+
+   Return true if multivector is pure grade (all grades but one are zero).
+
+
+.. function:: diff(self,x)
+
+   Return the partial derivative of the multivector function with respect to
+   variable :math:`x`.
+
+
+.. function:: grad(self)
+
+   Return the geometric derivative of the multivector function.
+
+
+.. function:: grad_ext(self)
+
+   Return the outer (curl) derivative of the multivector function. Equivalent to
+   :func:`curl`.
+
+
+.. function:: grad_int(self)
+
+   Return the inner (div) derivative of the multivector function. Equivalent to
+   :func:`div`.
+
+.. warning::
+
+    If *A* is a vector field in three dimensions :math:`\nabla\cdot {\bf A}` = A.grad_int() = A.div(), but
+    :math:`\nabla\times {\bf A}` = -MV.I*A.grad_ext() = -MV.I*A.curl(). Note that grad_int() lowers the grade
+    of all blades by one grade and grad_ext() raises the grade of all blades by one.
+
+.. function:: set_coef(self,grade,base,value)
+
+   Set the multivector coefficient of index *(grade,base)* to *value*.
+
+
+Sympy Functions Applied Inplace to Multivector Coefficients
+-----------------------------------------------------------
+
+All the following fuctions belong to the :class:`MV` class and apply the
+corresponding :mod:`sympy` function to each component of a multivector.   All
+these functions perform the operations inplace (``None`` is returned)  on each
+coefficient.  For example if you wished to simplify all the components of the
+multivector ``A`` you would invoke ``A.simplify()``.  The argument list for each
+function is the same as for the corresponding :mod:`sympy` function.   The only
+function that differs in its action from the :mod:`sympy` version is
+:func:`trigsimp` in its case the function ``TrigSimp is applied`` (see
+documentation on :func:`TrigSimp`).
+
+
+.. function:: collect(self,lst)
+
+
+.. function:: sqrfree(self,lst)
+
+
+.. function:: subs(self,*args)
+
+
+.. function:: simplify(self)
+
+
+.. function:: trigsimp(self)
+
+
+.. function:: cancel(self)
+
+
+.. function:: expand(self)
+
+
+Helper Functions
+----------------
+
+These are functions in :mod:`GA`, but not in the multivector (:class:`MV`)
+class.
+
+
+.. function:: set_main(main_program)
+
+   :func:`set_main` passes the argument *main_program* from the main program to the
+   :mod:`GA` module.  The argument must be *sys.modules[__name__]* and the
+   call should be placed immediately after :mod:`sys` and :mod:`GA` are
+   imported.  The purpose of this call is to allow :mod:`GA` to broadcast to
+   the main program :mod:`sympy`  variables and multivectors created by calls to
+   :mod:`GA`. It is used by :func:`MV.setup` and :func:`make_symbols`.
+
+
+.. function:: make_symbols(symnamelst)
+
+   :func:`make_symbols` creates a list of :mod:`sympy` symbols with names defined
+   by the space delimited string *symnamelst*. In addition to returning the symbol
+   list the function broadcasts the named symbols to the main program.  For example
+   if you make the call::
+
+      syms = make_symbols('x y ab')
+
+   Not only will *syms* contain the symbols, but you can also directly use *x*,
+   *y*, and *ab* as symbols in your program.
+
+    .. warning::
+
+        You can only directly use *x*, *y*, and *ab* as symbols in your program if
+        the statement  ``set_main(sys.modules[__name__])`` appears immediately after
+        the ``from sympy.galgebra.GA import *`` statement.
+
+.. function:: set_names(var_lst,var_str)
+
+   :func:`set_names` allows one to name a list, *var_lst*, of multivectors enmass.
+   The names are in *var_str*, a blank separated string of names.  An error is
+   generated if the number of name is not equal to the length of *var_lst*.
+
+
+.. function:: reciprocal_frame(vlst,names='')
+
+   :func:`reciprocal_frame` implements the proceedure described in section
+   :ref:`recframe`.  *vlst* is a list of independent vectors that you wish the
+   reciprocal frame calculated for. *names* is a blank separated string of names
+   for the reciprocal vectors if names are required by you application.  The
+   function returns a list containing the reciprocal vectors.
+
+
+.. function:: TrigSimp(f)
+
+   In general :func:`sympy.trigsimp` will not catch all the trigonometric
+   simplifications in an :mod:`sympy` expression.  Neither will :func:`TrigSimp`,
+   but it will catch a lot more of them. :func:`TrigSimp` is so simple it is show
+   below in its entirety.  All it does is apply  :func:`sympy.trigsimp` to the
+   expressions generated by :func:`sympy.cse`.  ::
+
+      def TrigSimp(f):
+          (w,g) = sympy.cse(f)
+          g = sympy.trigsimp(g[0])
+          for sub in reversed(w):
+              g = g.subs(sub[0],sub[1])
+              g = sympy.trigsimp(g)
+          return(g)
+
+.. function:: S(x)
+
+    :func:`S` instanciates a scaler multivector of value *x*, where *x* can be a
+    :mod:`sympy` variable or integer.  This is just a shorthand method for
+    constructing scalar multivectors and can be used when there is any ambiguity
+    in a multivector expression as to whether a symbol or constant should be
+    treated as a scalar multivector or not.
+
+Examples
+========
+
+
+Algebra
+-------
+
+The following examples of geometric algebra (not calculus) are all in the file
+:program:`testsymbolicGA.py` which is included in the sympy distribution
+examples  under the galbebra directory.  The section of code in the program for
+each example  is show with the respective output following the code section.
+
+
+Example Header
+^^^^^^^^^^^^^^
+
+This is the header of :program:`testsymbolicGA.py` that allows access to the
+required modules and also allow variables and certain multivectors to be
+broadcast from the :mod:`GA` module to the main program.
+
+
+.. include:: headerGAtest.py
+   :literal:
+
+
+Basic Geometric Algebra Operations
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Example of basic geometric algebra operation of geometric, outer, and inner
+products.
+
+
+.. include:: BasicGAtest.py
+   :literal:
+
+
+Examples of Conformal Geometry
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Examples of comformal geometry [Lasenby,Chapter 10]. The examples show that
+basic geometric entities (lines, circles, planes, and spheres) in three
+dimensions can be represented by blades in a five dimensional (conformal) space.
+
+
+.. include:: conformalgeometryGAtest.py
+   :literal:
+
+
+Calculation of Reciprocal Frame
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Example shows the calculation of the reciprocal frame for three arbitrary
+vectors and  verifies that the calculated reciprocal vectors have the correct
+properties.
+
+
+.. include:: reciprocalframeGAtest.py
+   :literal:
+
+
+Hyperbolic Geometry
+^^^^^^^^^^^^^^^^^^^
+
+Examples of calculation of distance in hyperbolic geometry [Lasenby,pp373-375].
+This is a good example of the utility of not restricting the basis vector to be
+orthogonal. Note that most of the calculation is simplifying a scalar
+expression.
+
+
+.. include:: hyperbolicGAtest.py
+   :literal:
+
+
+Calculus
+--------
+
+The calculus examples all use the extened LaTeXoutput module, ``latex_ex``, for
+clarity.
+
+
+Maxwell's Equations
+^^^^^^^^^^^^^^^^^^^
+
+In geometric calculus the equivalent of the electromagnetic tensor is
+:math:`F = E\gamma_{0}+IB\gamma_{0}` where a spacetime vector is given by
+:math:`x = x^{0}\gamma_{0}+x^{1}\gamma_{1}+x^{2}\gamma_{2}+x^{3}\gamma_{3}`
+where :math:`x^{0} = ct`, the pseudoscalar
+:math:`I = \gamma_{0}\gamma_{1}\gamma_{2}\gamma_{3}`, :math:`E` and :math:`B`
+are four vectors where the time component is zero and the spatial components
+equal to the electric and magnetic field  components.  Then Maxwell's equations
+can be all written as :math:`\nabla F = J` with :math:`J` the four current.
+This example shows that this equations generates all of Maxwell's equations
+correctly  (in our units:math:`c=1`) [Lasenby,pp229-231].
+
+``Begin Program Maxwell.py``
+
+
+.. include:: Maxwell.py
+   :literal:
+
+``End Program Maxwell.py``
+
+``Begin Program Output``
+
+:math:`I` Pseudo-Scalar
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+  I = {\gamma}_{t}{\gamma}_{x}{\gamma}_{y}{\gamma}_{z}
+  \end{equation*}
+
+
+:math:`B` Magnetic Field Bi-Vector
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+  B = - {B^{x}}{\gamma}_{t}{\gamma}_{x}- {B^{y}}{\gamma}_{t}{\gamma}_{y}- {B^{z}}{\gamma}_{t}{\gamma}_{z}
+  \end{equation*}
+
+
+:math:`F` Electric Field Bi-Vector
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+  E = - {E^{x}}{\gamma}_{t}{\gamma}_{x}- {E^{y}}{\gamma}_{t}{\gamma}_{y}- {E^{z}}{\gamma}_{t}{\gamma}_{z}
+  \end{equation*}
+
+
+:math:`E+IB` Electo-Magnetic Field Bi-Vector
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+  F = - {E^{x}}{\gamma}_{t}{\gamma}_{x}- {E^{y}}{\gamma}_{t}{\gamma}_{y}- {B^{z}}{\gamma}_{x}{\gamma}_{y}- {E^{z}}{\gamma}_{t}{\gamma}_{z}+ {B^{y}}{\gamma}_{x}{\gamma}_{z}- {B^{x}}{\gamma}_{y}{\gamma}_{z}
+  \end{equation*}
+
+
+:math:`J` Four Current
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+  J =  {J^{t}}{\gamma}_{t}+ {J^{x}}{\gamma}_{x}+ {J^{y}}{\gamma}_{y}+ {J^{z}}{\gamma}_{z}
+  \end{equation*}
+
+
+Geometric Derivative of Electo-Magnetic Field Bi-Vector
+
+.. math::
+  :nowrap:
+
+  \begin{align*}
+  \nabla F & =   \left(\partial_{z} {E^{z}} + \partial_{y} {E^{y}} + \partial_{x} {E^{x}}\right){\gamma}_{t} \\ & + \left(-\partial_{t} {E^{x}} + \partial_{y} {B^{z}} -\partial_{z} {B^{y}}\right){\gamma}_{x} \\ & + \left(\partial_{z} {B^{x}} -\partial_{t} {E^{y}} -\partial_{x} {B^{z}}\right){\gamma}_{y} \\ & + \left(-\partial_{y} {B^{x}} -\partial_{t} {E^{z}} + \partial_{x} {B^{y}}\right){\gamma}_{z} \\ & + \left(-\partial_{x} {E^{y}} -\partial_{t} {B^{z}} + \partial_{y} {E^{x}}\right){\gamma}_{t}{\gamma}_{x}{\gamma}_{y} \\ & + \left(-\partial_{x} {E^{z}} + \partial_{t} {B^{y}} + \partial_{z} {E^{x}}\right){\gamma}_{t}{\gamma}_{x}{\gamma}_{z} \\ & + \left(-\partial_{t} {B^{x}} -\partial_{y} {E^{z}} + \partial_{z} {E^{y}}\right){\gamma}_{t}{\gamma}_{y}{\gamma}_{z} \\ & + \left(\partial_{y} {B^{y}} + \partial_{z} {B^{z}} + \partial_{x} {B^{x}}\right){\gamma}_{x}{\gamma}_{y}{\gamma}_{z}\end{align*}
+
+
+All Maxwell Equations are
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+  \nabla F = J
+  \end{equation*}
+
+
+Div :math:`E` and Curl :math:`H` Equations
+
+.. math::
+  :nowrap:
+
+  \begin{align*}
+  <\nabla F>_1 -J & =   \left(-{J^{t}} + \partial_{z} {E^{z}} + \partial_{y} {E^{y}} + \partial_{x} {E^{x}}\right){\gamma}_{t} \\ & + \left(-{J^{x}} -\partial_{t} {E^{x}} + \partial_{y} {B^{z}} -\partial_{z} {B^{y}}\right){\gamma}_{x} \\ & + \left(\partial_{z} {B^{x}} -\partial_{t} {E^{y}} -{J^{y}} -\partial_{x} {B^{z}}\right){\gamma}_{y} \\ & + \left(-\partial_{y} {B^{x}} -\partial_{t} {E^{z}} -{J^{z}} + \partial_{x} {B^{y}}\right){\gamma}_{z}\end{align*}
+
+
+
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+    = 0
+  \end{equation*}
+
+
+Curl :math:`E` and Div :math:`B` equations
+
+.. math::
+  :nowrap:
+
+  \begin{align*}
+  <\nabla F>_3 & =   \left(-\partial_{x} {E^{y}} -\partial_{t} {B^{z}} + \partial_{y} {E^{x}}\right){\gamma}_{t}\W {\gamma}_{x}\W {\gamma}_{y} \\ & + \left(-\partial_{x} {E^{z}} + \partial_{t} {B^{y}} + \partial_{z} {E^{x}}\right){\gamma}_{t}\W {\gamma}_{x}\W {\gamma}_{z} \\ & + \left(-\partial_{t} {B^{x}} -\partial_{y} {E^{z}} + \partial_{z} {E^{y}}\right){\gamma}_{t}\W {\gamma}_{y}\W {\gamma}_{z} \\ & + \left(\partial_{y} {B^{y}} + \partial_{z} {B^{z}} + \partial_{x} {B^{x}}\right){\gamma}_{x}\W {\gamma}_{y}\W {\gamma}_{z}\end{align*}
+
+
+
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+    = 0
+  \end{equation*}
+
+
+``End Program Output``
+
+
+Dirac's Equation
+^^^^^^^^^^^^^^^^
+
+The geometric algebra/calculus allows one to formulate the Dirac equation in
+real terms (no :math:`\sqrt{-1}`).  Spinors  :math:`\lp\Psi\rp` are even
+multivectors in space time (Minkowski space with signature (1,-1,-1,-1)) and the
+Dirac equation becomes
+:math:`\nabla \Psi I \sigma_{z}-eA\Psi = m\Psi\gamma_{t}`.  All the terms in the
+real equation are explined  in Doran and Lasenby [Lasenby,pp281-283].
+
+``Begin Program Dirac.py``
+
+
+.. include:: Dirac.py
+   :literal:
+
+``End Program Dirac.py``
+
+``Begin Program Output``
+
+:math:`A` is 4-vector potential
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+  A =  {A^{t}}{\gamma}_{t}+ {A^{x}}{\gamma}_{x}+ {A^{y}}{\gamma}_{y}+ {A^{z}}{\gamma}_{z}
+  \end{equation*}
+
+
+:math:`\bm{\psi}` is 8-component real spinor (even multi-vector)
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+  {}\bm{{\psi}} =  {{\psi}}+ {{\psi}^{tx}}{\gamma}_{t}{\gamma}_{x}+ {{\psi}^{ty}}{\gamma}_{t}{\gamma}_{y}+ {{\psi}^{xy}}{\gamma}_{x}{\gamma}_{y}+ {{\psi}^{tz}}{\gamma}_{t}{\gamma}_{z}+ {{\psi}^{xz}}{\gamma}_{x}{\gamma}_{z}+ {{\psi}^{yz}}{\gamma}_{y}{\gamma}_{z}+ {{\psi}^{txyz}}{\gamma}_{t}{\gamma}_{x}{\gamma}_{y}{\gamma}_{z}
+  \end{equation*}
+
+
+Dirac equation in terms of real geometric algebra/calculus
+:math:`\lp\nabla \bm{\psi} I \sigma_{z}-eA\bm{\psi} = m\bm{\psi}\gamma_{t}\rp`
+Spin measured with respect to :math:`z` axis
+
+.. math::
+  :nowrap:
+
+  \begin{align*}
+  \nabla \bm{\psi} I \sigma_{z}-eA\bm{\psi}-m\bm{\psi}\gamma_{t} & =    \left(-e {A^{y}} {{\psi}^{ty}} -m {{\psi}} -\partial_{z} {{\psi}^{txyz}} -\partial_{y} {{\psi}^{tx}} -e {A^{z}} {{\psi}^{tz}} -e {A^{x}} {{\psi}^{tx}} + \partial_{x} {{\psi}^{ty}} -e {A^{t}} {{\psi}} + \partial_{t} {{\psi}^{xy}}\right){\gamma}_{t} \\ & + \left(-e {A^{y}} {{\psi}^{xy}} + \partial_{z} {{\psi}^{yz}} -e {A^{z}} {{\psi}^{xz}} -e {A^{t}} {{\psi}^{tx}} + m {{\psi}^{tx}} -\partial_{x} {{\psi}^{xy}} + \partial_{y} {{\psi}} -e {A^{x}} {{\psi}} -\partial_{t} {{\psi}^{ty}}\right){\gamma}_{x} \\ & + \left(-e {A^{y}} {{\psi}} + e {A^{x}} {{\psi}^{xy}} -e {A^{t}} {{\psi}^{ty}} -\partial_{x} {{\psi}} + \partial_{t} {{\psi}^{tx}} -\partial_{z} {{\psi}^{xz}} -e {A^{z}} {{\psi}^{yz}} + m {{\psi}^{ty}} -\partial_{y} {{\psi}^{xy}}\right){\gamma}_{y} \\ & + \left(-\partial_{z} {{\psi}^{xy}} + e {A^{x}} {{\psi}^{xz}} -e {A^{t}} {{\psi}^{tz}} -\partial_{x} {{\psi}^{yz}} + \partial_{y} {{\psi}^{xz}} + e {A^{y}} {{\psi}^{yz}} + \partial_{t} {{\psi}^{txyz}} + m {{\psi}^{tz}} -e {A^{z}} {{\psi}}\right){\gamma}_{z} \\ & + \left(\partial_{y} {{\psi}^{ty}} -e {A^{y}} {{\psi}^{tx}} + e {A^{x}} {{\psi}^{ty}} + \partial_{z} {{\psi}^{tz}} -e {A^{z}} {{\psi}^{txyz}} + \partial_{x} {{\psi}^{tx}} -e {A^{t}} {{\psi}^{xy}} -\partial_{t} {{\psi}} -m {{\psi}^{xy}}\right){\gamma}_{t}{\gamma}_{x}{\gamma}_{y} \\ & + \left(-\partial_{y} {{\psi}^{tz}} + \partial_{x} {{\psi}^{txyz}} -e {A^{t}} {{\psi}^{xz}} + e {A^{x}} {{\psi}^{tz}} + e {A^{y}} {{\psi}^{txyz}} + \partial_{z} {{\psi}^{ty}} -m {{\psi}^{xz}} -e {A^{z}} {{\psi}^{tx}} -\partial_{t} {{\psi}^{yz}}\right){\gamma}_{t}{\gamma}_{x}{\gamma}_{z} \\ & + \left(-e {A^{t}} {{\psi}^{yz}} -\partial_{z} {{\psi}^{tx}} + \partial_{x} {{\psi}^{tz}} -m {{\psi}^{yz}} + \partial_{y} {{\psi}^{txyz}} + \partial_{t} {{\psi}^{xz}} -e {A^{z}} {{\psi}^{ty}} -e {A^{x}} {{\psi}^{txyz}} + e {A^{y}} {{\psi}^{tz}}\right){\gamma}_{t}{\gamma}_{y}{\gamma}_{z} \\ & + \left(\partial_{z} {{\psi}} + e {A^{y}} {{\psi}^{xz}} + m {{\psi}^{txyz}} -\partial_{t} {{\psi}^{tz}} -\partial_{x} {{\psi}^{xz}} -e {A^{x}} {{\psi}^{yz}} -e {A^{t}} {{\psi}^{txyz}} -\partial_{y} {{\psi}^{yz}} -e {A^{z}} {{\psi}^{xy}}\right){\gamma}_{x}{\gamma}_{y}{\gamma}_{z}\end{align*}
+
+
+
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+    = 0
+  \end{equation*}
+
+
+``End Program Output``
+
+
+Spherical Coordinates
+^^^^^^^^^^^^^^^^^^^^^
+
+Curvilinear coodinates are implemented as shown in section :ref:`deriv`.  The
+gradient of a scalar function and the divergence and curl of a vector function
+(:math:`-I\lp\nabla\W A\rp` is the curl in three dimensions in the notation of
+geometric algebra) to demonstrate the formulas derived in section :ref:`deriv`.
+
+``Begin Program coords.py``
+
+
+.. include:: coords.py
+   :literal:
+
+``End Program coords.py``
+
+``Begin Program Output``
+
+Gradient of Scalar Function :math:`\psi`
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+  \nabla\psi =  \partial_{r} {{\psi}}{}\bm{e}_{r}+\frac{1 \partial_{{\theta}} {{\psi}}}{r}{}\bm{e}_{{\theta}}+\frac{1 \partial_{{\phi}} {{\psi}}}{r \operatorname{sin}\left({\theta}\right)}{}\bm{e}_{{\phi}}
+  \end{equation*}
+
+
+Div and Curl of Vector Function :math:`A`
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+  A =  {A^{r}}{}\bm{e}_{r}+ {A^{{\theta}}}{}\bm{e}_{{\theta}}+ {A^{{\phi}}}{}\bm{e}_{{\phi}}
+  \end{equation*}
+
+
+
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+  \nabla \cdot A =  \left(\frac{{A^{{\theta}}} \operatorname{cos}\left({\theta}\right)}{r \operatorname{sin}\left({\theta}\right)} + 2 \frac{{A^{r}}}{r} + \partial_{r} {A^{r}} + \frac{\partial_{{\theta}} {A^{{\theta}}}}{r} + \frac{\partial_{{\phi}} {A^{{\phi}}}}{r \operatorname{sin}\left({\theta}\right)}\right)
+  \end{equation*}
+
+
+
+
+.. math::
+  :nowrap:
+
+  \begin{align*}
+  -I\lp\nabla \W A\rp & =   \left(\frac{\partial_{{\theta}} {A^{{\phi}}}}{r} + \frac{{A^{{\phi}}} \operatorname{cos}\left({\theta}\right)}{r \operatorname{sin}\left({\theta}\right)} -\frac{\partial_{{\phi}} {A^{{\theta}}}}{r \operatorname{sin}\left({\theta}\right)}\right){}\bm{e}_{r} \\ & - \left(\frac{{A^{{\phi}}}}{r} + \partial_{r} {A^{{\phi}}} -\frac{\partial_{{\phi}} {A^{r}}}{r \operatorname{sin}\left({\theta}\right)}\right){}\bm{e}_{{\theta}} \\ & + \left(\frac{{A^{{\theta}}}}{r} + \partial_{r} {A^{{\theta}}} -\frac{\partial_{{\theta}} {A^{r}}}{r}\right){}\bm{e}_{{\phi}}\end{align*}
+
+
+``End Program Output``
+
+
+.. seealso::
+
+   `Hestenes <http://geocalc.clas.asu.edu/html/CA_to_GC.html>`_
+      ``Clifford Algebra to Geometric Calculus`` by  D.Hestenes and G. Sobczyk, Kluwer
+      Academic Publishers, 1984.
+
+   `Lasenby <http://www.mrao.cam.ac.uk/~cjld1/pages/book.htm>`_
+      ``Geometric Algebra for Physicists`` by  C. Doran and A. Lasenby, Cambridge
+      University Press, 2003.
+
+
diff --git a/lib/sympy/doc/src/modules/galgebra/GA/Maxwell.py b/lib/sympy/doc/src/modules/galgebra/GA/Maxwell.py
new file mode 100755
--- /dev/null
+++ b/lib/sympy/doc/src/modules/galgebra/GA/Maxwell.py
@@ -0,0 +1,57 @@
+
+from sympy import *
+from sympy.galgebra.GA import *
+from sympy.galgebra.latex_ex import *
+
+set_main(sys.modules[__name__])
+
+if __name__ == '__main__':
+    
+    metric = '1  0  0  0,'+\
+             '0 -1  0  0,'+\
+             '0  0 -1  0,'+\
+             '0  0  0 -1'
+
+    vars = make_symbols('t x y z')
+    MV.setup('gamma_t gamma_x gamma_y gamma_z',metric,True,vars)
+    LatexPrinter.format(1,1,1,1)
+    I = MV(1,'pseudo')
+    print '$I$ Pseudo-Scalar'
+    print 'I =',I
+    B = MV('B','vector',fct=True)
+    E = MV('E','vector',fct=True)
+    B.set_coef(1,0,0)
+    E.set_coef(1,0,0)
+    B *= gamma_t
+    E *= gamma_t
+    J = MV('J','vector',fct=True)
+    F = E+I*B
+    print ' '
+    print '$B$ Magnetic Field Bi-Vector'
+    print 'B = Bvec gamma_0 =',B
+    print '$F$ Electric Field Bi-Vector'
+    print 'E = Evec gamma_0 =',E
+    print '$E+IB$ Electo-Magnetic Field Bi-Vector'
+    print 'F = E+IB =',F
+    print '$J$ Four Current'
+    print 'J =',J
+    gradF = F.grad()
+    print 'Geometric Derivative of Electo-Magnetic Field Bi-Vector'
+    MV_format(3)
+    print '\\nabla F =',gradF
+    print 'All Maxwell Equations are'
+    print '\\nabla F = J'
+    print 'Div $E$ and Curl $H$ Equations'
+    print '<\\nabla F>_1 -J =',gradF.project(1)-J,' = 0'
+    print 'Curl $E$ and Div $B$ equations'
+    print '<\\nabla F>_3 =',gradF.project(3),' = 0'
+    xdvi(filename='Maxwell.tex')
+
+
+
+
+
+
+
+
+
diff --git a/lib/sympy/doc/src/modules/galgebra/GA/conformalgeometryGAtest.py b/lib/sympy/doc/src/modules/galgebra/GA/conformalgeometryGAtest.py
new file mode 100755
--- /dev/null
+++ b/lib/sympy/doc/src/modules/galgebra/GA/conformalgeometryGAtest.py
@@ -0,0 +1,46 @@
+        print '\nExample: Conformal representations of circles, lines, spheres, and planes'
+
+        metric = '1 0 0 0 0,0 1 0 0 0,0 0 1 0 0,0 0 0 0 2,0 0 0 2 0'
+
+        MV.setup('e0 e1 e2 n nbar',metric,debug=0)
+        MV.set_str_format(1)
+        e = n+nbar
+        #conformal representation of points
+
+        A = make_vector(e0)    # point a = (1,0,0)  A = F(a)
+        B = make_vector(e1)    # point b = (0,1,0)  B = F(b)
+        C = make_vector(-1*e0) # point c = (-1,0,0) C = F(c)
+        D = make_vector(e2)    # point d = (0,0,1)  D = F(d)
+        X = make_vector('x',3)
+
+        print 'a = e0, b = e1, c = -e0, and d = e2'
+        print 'A = F(a) = 1/2*(a*a*n+2*a-nbar), etc.'
+        print 'Circle through a, b, and c'
+        print 'Circle: A^B^C^X = 0 =',(A^B^C^X)
+        print 'Line through a and b'
+        print 'Line  : A^B^n^X = 0 =',(A^B^n^X)
+        print 'Sphere through a, b, c, and d'
+        print 'Sphere: A^B^C^D^X = 0 =',(A^B^C^D^X)
+        print 'Plane through a, b, and d'
+        print 'Plane : A^B^n^D^X = 0 =',(A^B^n^D^X)
+
+Example: Conformal representations of circles, lines, spheres, and planes
+a = e0, b = e1, c = -e0, and d = e2
+A = F(a) = 1/2*(a*a*n+2*a-nbar), etc.
+Circle through a, b, and c
+Circle: A^B^C^X = 0 = {-x2}e0^e1^e2^n
++{x2}e0^e1^e2^nbar
++{-1/2 + 1/2*x0**2 + 1/2*x1**2 + 1/2*x2**2}e0^e1^n^nbar
+
+Line through a and b
+Line  : A^B^n^X = 0 = {-x2}e0^e1^e2^n
++{-1/2 + x0/2 + x1/2}e0^e1^n^nbar
++{x2/2}e0^e2^n^nbar
++{-x2/2}e1^e2^n^nbar
+
+Sphere through a, b, c, and d
+Sphere: A^B^C^D^X = 0 = {1/2 - 1/2*x0**2 - 1/2*x1**2 - 1/2*x2**2}e0^e1^e2^n^nbar
+
+Plane through a, b, and d
+Plane : A^B^n^D^X = 0 = {1/2 - x0/2 - x1/2 - x2/2}e0^e1^e2^n^nbar
+
diff --git a/lib/sympy/doc/src/modules/galgebra/GA/coords.py b/lib/sympy/doc/src/modules/galgebra/GA/coords.py
new file mode 100755
--- /dev/null
+++ b/lib/sympy/doc/src/modules/galgebra/GA/coords.py
@@ -0,0 +1,48 @@
+#!/usrlocal/bin/python
+#EandM.py
+
+from sympy.galgebra.GA import *
+from sympy.galgebra.latex_ex import *
+from sympy import *
+
+import sympy,numpy,sys
+
+set_main(sys.modules[__name__])
+
+if __name__ == '__main__':
+    metric = '1 0 0,'+\
+             '0 1 0,'+\
+             '0 0 1'
+
+    MV.setup('gamma_x gamma_y gamma_z',metric,True)
+    Format('1 1 1 1')
+
+    coords = make_symbols('r theta phi')
+    x = r*(sympy.cos(theta)*gamma_z+sympy.sin(theta)*\
+        (sympy.cos(phi)*gamma_x+sympy.sin(phi)*gamma_y))
+    x.set_name('x')
+
+    MV.rebase(x,coords,'e',False)
+
+    #psi = MV.scalar_fct('psi')
+    psi = MV('psi','scalar',fct=True)
+    #psi.name = 'psi'
+    dpsi = psi.grad()
+    print 'Gradient of Scalar Function $\\psi$'
+    print '\\nabla\\psi =',dpsi
+    
+    #A = MV.vector_fct('A')
+    A = MV('A','vector',fct=True)
+    #A.name = 'A'
+    print 'Div and Curl of Vector Function $A$'
+    print A
+        
+    gradA = A.grad()
+    I = MV(ONE,'pseudo')
+    divA = A.grad_int()
+    curlA = -I*A.grad_ext()
+    print '\\nabla \\cdot A =',divA
+    Format('mv=3')
+    print '-I\\lp\\nabla \\W A\\rp =',curlA
+    
+    xdvi(filename='coords.tex')
diff --git a/lib/sympy/doc/src/modules/galgebra/GA/headerGAtest.py b/lib/sympy/doc/src/modules/galgebra/GA/headerGAtest.py
new file mode 100755
--- /dev/null
+++ b/lib/sympy/doc/src/modules/galgebra/GA/headerGAtest.py
@@ -0,0 +1,24 @@
+import os,sys,sympy
+from sympy.galgebra.GA import set_main, make_symbols, types, MV, ZERO, ONE, HALF
+from sympy import collect
+set_main(sys.modules[__name__])
+
+def F(x):
+        """
+        Conformal Mapping Function
+        """
+        Fx = HALF*((x*x)*n+2*x-nbar)
+        return(Fx)
+
+def make_vector(a,n = 3):
+        if type(a) == types.StringType:
+                sym_str = ''
+                for i in range(n):
+                        sym_str += a+str(i)+' '
+                sym_lst = make_symbols(sym_str)
+                sym_lst.append(ZERO)
+                sym_lst.append(ZERO)
+                a = MV(sym_lst,'vector')
+        return(F(a))
+
+if __name__ == '__main__':
diff --git a/lib/sympy/doc/src/modules/galgebra/GA/hyperbolicGAtest.py b/lib/sympy/doc/src/modules/galgebra/GA/hyperbolicGAtest.py
new file mode 100755
--- /dev/null
+++ b/lib/sympy/doc/src/modules/galgebra/GA/hyperbolicGAtest.py
@@ -0,0 +1,141 @@
+        print 'Example: non-euclidian distance calculation'
+
+        metric = '0 # #,# 0 #,# # 1'
+        MV.setup('X Y e',metric)
+        MV.set_str_format(1)
+        L = X^Y^e
+        B = L*e
+        Bsq = (B*B)()
+        print 'L = X^Y^e is a non-euclidian line'
+        print 'B = L*e =',B
+        BeBr =B*e*B.rev()
+        print 'B*e*B.rev() =',BeBr
+        print 'B^2 =',Bsq
+        print 'L^2 =',(L*L)()
+        make_symbols('s c Binv M S C alpha')
+        Bhat = Binv*B # Normalize translation generator
+        R = c+s*Bhat # Rotor R = exp(alpha*Bhat/2)
+        print 's = sinh(alpha/2) and c = cosh(alpha/2)'
+        print 'R = exp(alpha*B/(2*|B|)) =',R
+        Z = R*X*R.rev()
+        Z.expand()
+        Z.collect([Binv,s,c,XdotY])
+        print 'R*X*R.rev() =',Z
+        W = Z|Y
+        W.expand()
+        W.collect([s*Binv])
+        print '(R*X*rev(R)).Y =',W
+        M = 1/Bsq
+        W.subs(Binv**2,M)
+        W.simplify()
+        Bmag = sympy.sqrt(XdotY**2-2*XdotY*Xdote*Ydote)
+        W.collect([Binv*c*s,XdotY])
+
+        W.subs(2*XdotY**2-4*XdotY*Xdote*Ydote,2/(Binv**2))
+        W.subs(2*c*s,S)
+        W.subs(c**2,(C+1)/2)
+        W.subs(s**2,(C-1)/2)
+        W.simplify()
+        W.subs(1/Binv,Bmag)
+        W = W().expand()
+        print '(R*X*R.rev()).Y =',W
+        nl = '\n'
+
+        Wd = collect(W,[C,S],exact=True,evaluate=False)
+        print 'Wd =',Wd
+        Wd_1 = Wd[ONE]
+        Wd_C = Wd[C]
+        Wd_S = Wd[S]
+        print '|B| =',Bmag
+        Wd_1 = Wd_1.subs(Bmag,1/Binv)
+        Wd_C = Wd_C.subs(Bmag,1/Binv)
+        Wd_S = Wd_S.subs(Bmag,1/Binv)
+        print 'Wd[ONE] =',Wd_1
+        print 'Wd[C] =',Wd_C
+        print 'Wd[S] =',Wd_S
+
+
+        lhs = Wd_1+Wd_C*C
+        rhs = -Wd_S*S
+        lhs = lhs**2
+        rhs = rhs**2
+        W = (lhs-rhs).expand()
+        W = (W.subs(1/Binv**2,Bmag**2)).expand()
+        print 'W =',W
+        W = (W.subs(S**2,C**2-1)).expand()
+        print 'W =',W
+        W = collect(W,[C,C**2],evaluate=False)
+        print 'W =',W
+        
+        a = W[C**2]
+        b = W[C]
+        c = W[ONE]
+
+        print 'a =',a
+        print 'b =',b
+        print 'c =',c
+
+        D = (b**2-4*a*c).expand()
+        print 'Setting to 0 and solving for C gives:'
+        print 'Descriminant D = b^2-4*a*c =',D
+        C = (-b/(2*a)).expand()
+        print 'C = cosh(alpha) = -b/(2*a) =',C
+
+Example: non-euclidian distance calculation
+L = X^Y^e is a non-euclidian line
+B = L*e = X^Y
++{-(Y.e)}X^e
++{(X.e)}Y^e
+
+B*e*B.rev() = {2*(X.Y)*(X.e)*(Y.e) - (X.Y)**2}e
+
+B^2 = -2*(X.Y)*(X.e)*(Y.e) + (X.Y)**2
+L^2 = -2*(X.Y)*(X.e)*(Y.e) + (X.Y)**2
+s = sinh(alpha/2) and c = cosh(alpha/2)
+R = exp(alpha*B/(2*|B|)) = {c}1
++{Binv*s}X^Y
++{-(Y.e)*Binv*s}X^e
++{(X.e)*Binv*s}Y^e
+
+R*X*R.rev() = {Binv*(2*(X.Y)*c*s - 2*(X.e)*(Y.e)*c*s) + Binv**2*((X.Y)**2*s**2 
+               - 2*(X.Y)*(X.e)*(Y.e)*s**2) + c**2}X
+             +{2*Binv*c*s*(X.e)**2}Y
+             +{Binv**2*(-2*(X.e)*(X.Y)**2*s**2 + 4*(X.Y)*(Y.e)*(X.e)**2*s**2)
+               - 2*(X.Y)*(X.e)*Binv*c*s}e
+
+(R*X*rev(R)).Y = {Binv*s*(-4*(X.Y)*(X.e)*(Y.e)*c + 2*c*(X.Y)**2) 
+                 + Binv**2*s**2*(-4*(X.e)*(Y.e)*(X.Y)**2 +
+                 4*(X.Y)*(X.e)**2*(Y.e)**2 + (X.Y)**3) + (X.Y)*c**2}1
+
+(R*X*R.rev()).Y = S*(-2*(X.Y)*(X.e)*(Y.e) + (X.Y)**2)**(1/2) 
+                  + (X.Y)*Binv*C*(-2*(X.Y)*(X.e)*(Y.e) + 
+                  (X.Y)**2)**(1/2) + (X.e)*(Y.e)*Binv*(-2*(X.Y)*(X.e)*(Y.e) 
+                  + (X.Y)**2)**(1/2) - 
+                  (X.e)*(Y.e)*Binv*C*(-2*(X.Y)*(X.e)*(Y.e) + (X.Y)**2)**(1/2)
+Wd = {1: (X.e)*(Y.e)*Binv*(-2*(X.Y)*(X.e)*(Y.e) + (X.Y)**2)**(1/2),
+      S: (-2*(X.Y)*(X.e)*(Y.e) + (X.Y)**2)**(1/2),
+      C: (X.Y)*Binv*(-2*(X.Y)*(X.e)*(Y.e) + (X.Y)**2)**(1/2) - 
+         (X.e)*(Y.e)*Binv*(-2*(X.Y)*(X.e)*(Y.e) + (X.Y)**2)**(1/2)}
+|B| = (-2*(X.Y)*(X.e)*(Y.e) + (X.Y)**2)**(1/2)
+
+Wd[ONE] = (X.e)*(Y.e)
+Wd[C] = (X.Y) - (X.e)*(Y.e)
+Wd[S] = 1/Binv
+
+W = 2*(X.Y)*(X.e)*(Y.e)*C + (X.Y)**2*C**2 + (X.e)**2*(Y.e)**2 
+    - (X.Y)**2*S**2 + (X.e)**2*(Y.e)**2*C**2 - 2*C*(X.e)**2*(Y.e)**2 
+    - 2*(X.Y)*(X.e)*(Y.e)*C**2 + 2*(X.Y)*(X.e)*(Y.e)*S**2
+W = -2*(X.Y)*(X.e)*(Y.e) + 2*(X.Y)*(X.e)*(Y.e)*C + (X.Y)**2 
+    + (X.e)**2*(Y.e)**2 + (X.e)**2*(Y.e)**2*C**2 -
+     2*C*(X.e)**2*(Y.e)**2
+W = {1: -2*(X.Y)*(X.e)*(Y.e) + (X.Y)**2 + (X.e)**2*(Y.e)**2,
+     C**2: (X.e)**2*(Y.e)**2,
+     C: 2*(X.Y)*(X.e)*(Y.e) - 2*(X.e)**2*(Y.e)**2}
+
+a = (X.e)**2*(Y.e)**2
+b = 2*(X.Y)*(X.e)*(Y.e) - 2*(X.e)**2*(Y.e)**2
+c = -2*(X.Y)*(X.e)*(Y.e) + (X.Y)**2 + (X.e)**2*(Y.e)**2
+
+Setting to 0 and solving for C gives:
+Descriminant D = b^2-4*a*c = 0
+C = cosh(alpha) = -b/(2*a) = 1 - (X.Y)/((X.e)*(Y.e))
diff --git a/lib/sympy/doc/src/modules/galgebra/GA/multable.dat b/lib/sympy/doc/src/modules/galgebra/GA/multable.dat
new file mode 100644
--- /dev/null
+++ b/lib/sympy/doc/src/modules/galgebra/GA/multable.dat
@@ -0,0 +1,75 @@
+(1)(1) = 1
+(1)(a0) = a0
+(1)(a1) = a1
+(1)(a2) = a2
+(1)(a0a1) = a0a1
+(1)(a0a2) = a0a2
+(1)(a1a2) = a1a2
+(1)(a0a1a2) = a0a1a2
+
+(a0)(1) = a0
+(a0)(a0) = {a0**2}1
+(a0)(a1) = a0a1
+(a0)(a2) = a0a2
+(a0)(a0a1) = {a0**2}a1
+(a0)(a0a2) = {a0**2}a2
+(a0)(a1a2) = a0a1a2
+(a0)(a0a1a2) = {a0**2}a1a2
+
+(a1)(1) = a1
+(a1)(a0) = {2*(a0.a1)}1-a0a1
+(a1)(a1) = {a1**2}1
+(a1)(a2) = a1a2
+(a1)(a0a1) = {-a1**2}a0+{2*(a0.a1)}a1
+(a1)(a0a2) = {2*(a0.a1)}a2-a0a1a2
+(a1)(a1a2) = {a1**2}a2
+(a1)(a0a1a2) = {-a1**2}a0a2+{2*(a0.a1)}a1a2
+
+(a2)(1) = a2
+(a2)(a0) = {2*(a0.a2)}1-a0a2
+(a2)(a1) = {2*(a1.a2)}1-a1a2
+(a2)(a2) = {a2**2}1
+(a2)(a0a1) = {-2*(a1.a2)}a0+{2*(a0.a2)}a1+a0a1a2
+(a2)(a0a2) = {-a2**2}a0+{2*(a0.a2)}a2
+(a2)(a1a2) = {-a2**2}a1+{2*(a1.a2)}a2
+(a2)(a0a1a2) = {a2**2}a0a1+{-2*(a1.a2)}a0a2+{2*(a0.a2)}a1a2
+
+(a0a1)(1) = a0a1
+(a0a1)(a0) = {2*(a0.a1)}a0+{-a0**2}a1
+(a0a1)(a1) = {a1**2}a0
+(a0a1)(a2) = a0a1a2
+(a0a1)(a0a1) = {-a0**2*a1**2}1+{2*(a0.a1)}a0a1
+(a0a1)(a0a2) = {2*(a0.a1)}a0a2+{-a0**2}a1a2
+(a0a1)(a1a2) = {a1**2}a0a2
+(a0a1)(a0a1a2) = {-a0**2*a1**2}a2+{2*(a0.a1)}a0a1a2
+
+(a0a2)(1) = a0a2
+(a0a2)(a0) = {2*(a0.a2)}a0+{-a0**2}a2
+(a0a2)(a1) = {2*(a1.a2)}a0-a0a1a2
+(a0a2)(a2) = {a2**2}a0
+(a0a2)(a0a1) = {-2*a0**2*(a1.a2)}1+{2*(a0.a2)}a0a1+{a0**2}a1a2
+(a0a2)(a0a2) = {-a0**2*a2**2}1+{2*(a0.a2)}a0a2
+(a0a2)(a1a2) = {-a2**2}a0a1+{2*(a1.a2)}a0a2
+(a0a2)(a0a1a2) = {a0**2*a2**2}a1+{-2*a0**2*(a1.a2)}a2+{2*(a0.a2)}a0a1a2
+
+(a1a2)(1) = a1a2
+(a1a2)(a0) = {2*(a0.a2)}a1+{-2*(a0.a1)}a2+a0a1a2
+(a1a2)(a1) = {2*(a1.a2)}a1+{-a1**2}a2
+(a1a2)(a2) = {a2**2}a1
+(a1a2)(a0a1) = {2*a1**2*(a0.a2)-4*(a0.a1)*(a1.a2)}1+{2*(a1.a2)}a0a1+{-a1**2}a0a2
+              +{2*(a0.a1)}a1a2
+(a1a2)(a0a2) = {-2*a2**2*(a0.a1)}1+{a2**2}a0a1+{2*(a0.a2)}a1a2
+(a1a2)(a1a2) = {-a1**2*a2**2}1+{2*(a1.a2)}a1a2
+(a1a2)(a0a1a2) = {-a1**2*a2**2}a0+{2*a2**2*(a0.a1)}a1+{2*a1**2*(a0.a2)
+                -4*(a0.a1)*(a1.a2)}a2+{2*(a1.a2)}a0a1a2
+
+(a0a1a2)(1) = a0a1a2
+(a0a1a2)(a0) = {2*(a0.a2)}a0a1+{-2*(a0.a1)}a0a2+{a0**2}a1a2
+(a0a1a2)(a1) = {2*(a1.a2)}a0a1+{-a1**2}a0a2
+(a0a1a2)(a2) = {a2**2}a0a1
+(a0a1a2)(a0a1) = {2*a1**2*(a0.a2)-4*(a0.a1)*(a1.a2)}a0+{2*a0**2*(a1.a2)}a1
+                +{-a0**2*a1**2}a2+{2*(a0.a1)}a0a1a2
+(a0a1a2)(a0a2) = {-2*a2**2*(a0.a1)}a0+{a0**2*a2**2}a1+{2*(a0.a2)}a0a1a2
+(a0a1a2)(a1a2) = {-a1**2*a2**2}a0+{2*(a1.a2)}a0a1a2
+(a0a1a2)(a0a1a2) = {-a0**2*a1**2*a2**2}1+{2*a2**2*(a0.a1)}a0a1+{2*a1**2*(a0.a2)
+                  -4*(a0.a1)*(a1.a2)}a0a2+{2*a0**2*(a1.a2)}a1a2
\ No newline at end of file
diff --git a/lib/sympy/doc/src/modules/galgebra/GA/reciprocalframeGAtest.py b/lib/sympy/doc/src/modules/galgebra/GA/reciprocalframeGAtest.py
new file mode 100755
--- /dev/null
+++ b/lib/sympy/doc/src/modules/galgebra/GA/reciprocalframeGAtest.py
@@ -0,0 +1,79 @@
+        MV.setup('e1 e2 e3',metric)
+
+        print 'Example: Reciprocal Frames e1, e2, and e3 unit vectors.\n\n'
+
+        E = e1^e2^e3
+        Esq = (E*E)()
+        print 'E =',E
+        print 'E^2 =',Esq
+        Esq_inv = 1/Esq
+
+        E1 = (e2^e3)*E
+        E2 = (-1)*(e1^e3)*E
+        E3 = (e1^e2)*E
+
+        print 'E1 = (e2^e3)*E =',E1
+        print 'E2 =-(e1^e3)*E =',E2
+        print 'E3 = (e1^e2)*E =',E3
+
+        w = (E1|e2)
+        w.collect(MV.g)
+        w = w().expand()
+        print 'E1|e2 =',w
+
+        w = (E1|e3)
+        w.collect(MV.g)
+        w = w().expand()
+        print 'E1|e3 =',w
+
+        w = (E2|e1)
+        w.collect(MV.g)
+        w = w().expand()
+        print 'E2|e1 =',w
+
+        w = (E2|e3)
+        w.collect(MV.g)
+        w = w().expand()
+        print 'E2|e3 =',w
+
+        w = (E3|e1)
+        w.collect(MV.g)
+        w = w().expand()
+        print 'E3|e1 =',w
+
+        w = (E3|e2)
+        w.collect(MV.g)
+        w = w().expand()
+        print 'E3|e2 =',w
+
+        w = (E1|e1)
+        w = w().expand()
+        Esq = Esq.expand()
+        print '(E1|e1)/E^2 =',w/Esq
+
+        w = (E2|e2)
+        w = w().expand()
+        print '(E2|e2)/E^2 =',w/Esq
+
+        w = (E3|e3)
+        w = w().expand()
+        print '(E3|e3)/E^2 =',w/Esq
+
+Example: Reciprocal Frames e1, e2, and e3 unit vectors.
+
+
+E = e1^e2^e3
+E^2 = -1 - 2*(e1.e2)*(e1.e3)*(e2.e3) + (e1.e2)**2 + (e1.e3)**2 + (e2.e3)**2
+E1 = (e2^e3)*E = {-1 + (e2.e3)**2}e1+{(e1.e2) - (e1.e3)*(e2.e3)}e2+{(e1.e3) - (e1.e2)*(e2.e3)}e3
+E2 =-(e1^e3)*E = {(e1.e2) - (e1.e3)*(e2.e3)}e1+{-1 + (e1.e3)**2}e2+{(e2.e3) - (e1.e2)*(e1.e3)}e3
+E3 = (e1^e2)*E = {(e1.e3) - (e1.e2)*(e2.e3)}e1+{(e2.e3) - (e1.e2)*(e1.e3)}e2+{-1 + (e1.e2)**2}e3
+E1|e2 = 0
+E1|e3 = 0
+E2|e1 = 0
+E2|e3 = 0
+E3|e1 = 0
+E3|e2 = 0
+(E1|e1)/E^2 = 1
+(E2|e2)/E^2 = 1
+(E3|e3)/E^2 = 1
+
diff --git a/lib/sympy/doc/src/modules/galgebra/latex_ex/Maxwell.py b/lib/sympy/doc/src/modules/galgebra/latex_ex/Maxwell.py
new file mode 100755
--- /dev/null
+++ b/lib/sympy/doc/src/modules/galgebra/latex_ex/Maxwell.py
@@ -0,0 +1,50 @@
+import sys
+import sympy.galgebra.GAsympy as GA
+import sympy.galgebra.latex_ex as tex
+
+GA.set_main(sys.modules[__name__])
+
+if __name__ == '__main__':
+
+    metric = '1  0  0  0,'+\
+             '0 -1  0  0,'+\
+             '0  0 -1  0,'+\
+             '0  0  0 -1'
+
+    vars = GA.make_symbols('t x y z')
+    GA.MV.setup('gamma_t gamma_x gamma_y gamma_z',metric,True,vars)
+    tex.Format()
+    I = GA.MV(1,'pseudo')
+    I.convert_to_blades()
+    print '$I$ Pseudo-Scalar'
+    print 'I =',I
+    B = GA.MV('B','vector',fct=True)
+    E = GA.MV('E','vector',fct=True)
+    B.set_coef(1,0,0)
+    E.set_coef(1,0,0)
+    B *= gamma_t
+    E *= gamma_t
+    B.convert_to_blades()
+    E.convert_to_blades()
+    J = GA.MV('J','vector',fct=True)
+    print '$B$ Magnetic Field Bi-Vector'
+    print 'B = Bvec gamma_0 =',B
+    print '$E$ Electric Field Bi-Vector'
+    print 'E = Evec gamma_0 =',E
+    F = E+I*B
+    print '$E+IB$ Electo-Magnetic Field Bi-Vector'
+    print 'F = E+IB =',F
+    print '$J$ Four Current'
+    print 'J =',J
+    gradF = F.grad()
+    gradF.convert_to_blades()
+    print 'Geometric Derivative of Electo-Magnetic Field Bi-Vector'
+    tex.MV_format(3)
+    print '\\nabla F =',gradF
+    print 'All Maxwell Equations are'
+    print '\\nabla F = J'
+    print 'Div $E$ and Curl $H$ Equations'
+    print '<\\nabla F>_1 -J =',gradF.project(1)-J,' = 0'
+    print 'Curl $E$ and Div $B$ equations'
+    print '<\\nabla F>_3 =',gradF.project(3),' = 0'
+    tex.xdvi(filename='Maxwell.tex')
diff --git a/lib/sympy/doc/src/modules/galgebra/latex_ex/latex_ex.txt b/lib/sympy/doc/src/modules/galgebra/latex_ex/latex_ex.txt
new file mode 100644
--- /dev/null
+++ b/lib/sympy/doc/src/modules/galgebra/latex_ex/latex_ex.txt
@@ -0,0 +1,425 @@
+.. _extended-latex:
+
+**********************************
+  Extended LaTeXModule for Sympy  
+**********************************
+
+:Author: Alan Bromborsky
+
+.. |release| replace:: 0.10
+
+.. % .. math::
+.. % :nowrap:
+
+.. % Complete documentation on the extended LaTeX markup used for Python
+.. % documentation is available in ``Documenting Python'', which is part
+.. % of the standard documentation for Python.  It may be found online
+.. % at:
+.. % 
+.. % http://www.python.org/doc/current/doc/doc.html
+
+.. % \input{macros}
+.. % This is a template for short or medium-size Python-related documents,
+.. % mostly notably the series of HOWTOs, but it can be used for any
+.. % document you like.
+.. % The title should be descriptive enough for people to be able to find
+.. % the relevant document.
+
+.. % Increment the release number whenever significant changes are made.
+.. % The author and/or editor can define 'significant' however they like.
+
+.. % At minimum, give your name and an email address.  You can include a
+.. % snail-mail address if you like.
+
+.. % This makes the Abstract go on a separate page in the HTML version;
+.. % if a copyright notice is used, it should go immediately after this.
+.. % 
+.. % \ifhtml
+.. % \chapter*{Front Matter\label{front}}
+.. % \fi
+.. % Copyright statement should go here, if needed.
+.. % ...
+.. % The abstract should be a paragraph or two long, and describe the
+.. % scope of the document.
+
+.. math ::
+
+    \newcommand{\W}{\wedge}
+    \newcommand{\bm}{\boldsymbol}
+
+.. topic:: Abstract
+
+   This document describes the extension of the latex module for  :mod:`sympy`. The
+   python module :mod:`latex_ex` extends the  capabilities of the current
+   :mod:`latex` (while preserving the  current capabilities) to geometric algebra
+   multivectors, :mod:`numpy` array's, and extends the ascii formatting of greek
+   symbols, accents, and subscripts and superscripts.   Additionally the module is
+   configured to use the print command to generate a LaTeX output file and display
+   it using xdvi in Linux and yap in Windows.  To get LaTeX displayed text latex and 
+   xdvi must be installed on a Linux system and MikTex on a Windows system.
+
+
+Extended Symbol Coding
+======================
+
+One of the main extensions in :mod:`latex_ex` is the ability to encode complex
+symbols (multiple greek letters with accents and superscripts and subscripts) is
+ascii strings containing only letters, numbers, and underscores.  These
+restrictions allow :mod:`sympy` variable names to represent complex symbols. For
+example if we use the  :mod:`GA` module function :func:`make_symbols` as
+follows::
+
+   make_symbols('xalpha Gammavec__1_rho delta__j_k')
+
+``make_symbols`` creates the three :mod:`sympy` symbols *xalpha*,
+*Gammavec__1_rho*, and *delta__j_k*.  If these symbols are printed with the
+:mod:`latex_ex` modules the results are
+
+
+.. csv-table:: 
+	:header: " Ascii String ", " LaTeX Output "
+	:widths: 15, 15
+
+	 " ``xalpha`` ", " :math:`x\alpha` "   
+	 " ``Gammavec__1_rho`` ", " :math:`\vec{\Gamma}^{1}_{\rho}` "
+	 " ``delta__j_k`` ", " :math:`\delta^{j}_{k}` " 
+
+
+A single underscore denotes a subscript and a double undscore a superscript.
+
+In addition to all normal LaTeX accents boldmath is supported so that
+``omegaomegabm``:math:`\rightarrow \Omega\bm{\Omega}` so an accent (or boldmath)
+only applies to the character (characters in the case of the string representing
+a greek letter) immediately preceeding the accent command.
+
+
+How LatexPrinter Works
+======================
+
+The actual :class:`LatexPrinter` class is hidden with the helper functions
+:func:`Format`,  :func:`LaTeX`, and :func:`xdvi`.  :class:`LatexPrinter` is
+setup when :func:`Format` is called.  In addition to setting format switches in
+:class:`LatexPrinter`, :func:`Format` does two other critical tasks. Firstly,
+the :mod:`sympy` function :func:`Basic.__str__` is redirected to the
+:class:`LatexPrinter` helper function :func:`LaTeX`.  If nothing more that this
+were done the  python print command would output LaTeX code.  Secondly,
+*sys.stdout* is redirected  to a file until :func:`xdvi` is called.  This file
+is then compiled with the :program:`latex` program (if present) and the dvi
+output file is displayed with the :program:`xdvi` program (if present).  Thus
+for :class:`LatexPrinter` to display the output in a window both
+:program:`latex` and :program:`xdvi` must be installed on your system and in the
+execution path.
+
+One problem for :class:`LatexPrinter` is determining when equation mode should be
+use in the output formatting.  To allow for printing output not in equation mode the
+program attemps to determine from the output string context when to use equation mode.  
+The current test is to use equation mode if the string contains an =, \_, \^, or \\. 
+This is not a foolproof method.  The safest thing to do if you wish to print an object, *X*,
+in math mode is to use *print 'X =',X* so the = sign triggers math mode.
+
+
+LatexPrinter Functions
+======================
+
+
+LatexPrinter Class Functions for Extending LatexPrinter Class
+-------------------------------------------------------------
+
+The :class:`LatexPrinter` class functions are not called directly, but rather
+are called when :func:`print`, :func:`LaTeX`, or :func:`str` are called.  The
+two new functions for extending the :mod:`latex` module are
+:func:`_print_ndarray` and :func:`_print_MV`.  Some other functions in
+:class:`LatexPrinter` have been modified to increase their utility.
+
+
+.. function:: _print_ndarray(self,expr)
+
+   :func:`_print_ndarray` returns a latex formatted string for the *expr* equal to
+   a :class:`numpy` array with elements that can be :class:`sympy` expressions.
+   The :class:`numpy` array's can have up to three dimensions.
+
+
+.. function:: _print_MV(self,expr)
+
+   :func:`_print_MV` returns a latex formatted string for the *expr* equal to a
+   :class:`GA` multivector.
+
+
+.. function:: str_basic(in_str)
+
+   :func:`str_basic` returns a string without the latex formatting provided by
+   :class:`LatexPrinter`.  This is needed since :class:`LatexPrinter` takes over
+   the :func:`str` fuction and there are instances when the unformatted string is
+   needed such as during the automatic generation of multivector coefficients and
+   the reduction of multivector  coefficients for printing.
+
+
+Helper Functions for Extending LatexPrinter Class
+-------------------------------------------------
+
+
+.. function:: Format(fmt='1 1 1 1')
+
+   ``Format`` iniailizes ``LatexPrinter`` and set the format for ``sympy`` symbols,
+   functions, and  derivatives and for ``GA`` multivectors. The switch are encoded
+   in the text string argument of ``Format`` as follows.  It is assumed that the
+   text string ``fmt`` always contains four integers separated by blanks.
+
+   .. csv-table:: 
+	:header: " Position ", " Switch ", " Values "    
+	:widths: 4, 6, 40
+
+	"   :math:`1^{st}` ", " symbol ", " 0: Use symbol encoding in ``latex.py`` "
+	"   ", "   ", " 1: Use extended symbol encoding in ``latex_ex.py`` " 
+	"   :math:`2^{nd}` ", " function ", " 0: Use symbol encoding in  ``latex.py``. Print functions args,  use ``\operator{ }`` format. "
+	"   ", "   ", " 1: Do not print function args. Do not use ``\operator{}`` format. Suppress printing of function arguments. "
+	"   :math:`3^{d}` ", " partial derivative", " 0: Use partial derivative format in ``latex.py``.  "
+	"   ", "   ", " 1: Use format :math:`\partial_{x}` instead of :math:`\partial/\partial x`. "
+	"   :math:`4^{th}` ", " multivector ", " 1: Print entire multivector on one line. "
+	"   ", "   ", " 2: Print each grade of multivector on one line. " 
+	"   ", "   ", " 3: Print each base of multivector on one line. "
+
+
+.. function:: LaTeX(expr, inline=True)
+
+   :func:`LaTeX` returns the latex formatted string for the :class:`sympy`,
+   :class:`GA`, or :class:`numpy`  expression *expr*.  This is needed since
+   :class:`numpy` cannot be subclassed and hence cannot be used with the
+   :class:`LatexPrinter` modified :func:`print` command. Thus is *A* is a
+   :class:`numpy` array containing :class:`sympy` expressions one cannot simply
+   code  ::
+
+      print A
+
+   but rather must use  ::
+
+      print LaTeX(A)
+
+
+.. function:: xdvi(filename='tmplatex.tex',debug=False)
+
+   :func:`xdvi` postprocesses the output of the print statements and generates the
+   latex file with name *filename*.  If the :program:`latex` and :program:`xdvi`
+   programs are present on the system they are invoked to display the latex file in
+   a  window.  If *debug=True* the associated output of :program:`latex` is sent to
+   *stdout*, otherwise it is sent to  */dev/null* for linux and *NUL* for Windows.  
+   If :class:`LatexPrinter` has not been initialized :func:`xdvi` does nothing.  After the
+   .dvi file is generated it is displayed with :program:`xdvi` for linux (if latex and xdvi
+   are installed ) and :program:`yap` for Windows (if MikTex is installed). 
+
+The functions :func:`sym_format`, :func:`fct_format`, :func:`pdiff_format`, and
+:func:`MV_format` allow one to change various formatting aspects of the
+:class:`LatexPrinter`.  They do not initialize the class and if the are called
+with the class not initialized they have no effect.  These functions and the
+function :func:`xdvi` are designed so that if the :class:`LatexPrinter` class is
+not initialized the program output is as if the :class:`LatexPrinter` class is
+not used. Thus all one needs to do to get simple ascii output (possibly for
+program debugging) is to comment out the one function call that initializes the
+:class:`LatexPrinter` class.  All other :mod:`latex_ex` function calls can
+remain in the program and have no effect on program output.
+
+
+.. function:: sym_format(sym_fmt)
+
+   :func:`sym_format` allows one to change the latex format options for
+   :class:`sympy` symbol output independent of other format switches (see
+   :math:`1^{st}` switch in Table I).
+
+
+.. function:: fct_format(fct_fmt)
+
+   :func:`fct_format` allows one to change the latex format options for
+   :class:`sympy` function output independent of other format switches (see
+   :math:`2^{nd}` switch in Table I).
+
+
+.. function:: pdiff_format(pdiff_fmt)
+
+   :func:`pdiff_format` allows one to change the latex format options for
+   :class:`sympy` partial derivative output independent of other format switches
+   (see :math:`3^{d}` switch in Table I).
+
+
+.. function:: MV_format(mv_fmt)
+
+   :func:`MV_format` allows one to change the latex format options for
+   :class:`sympy` partial derivative output independent of other format switches
+   (see :math:`3^{d}` switch in Table I).
+
+
+Examples
+========
+
+
+:program:`latexdemo.py` a simple example
+----------------------------------------
+
+:program:`latexdemo.py` example of using :mod:`latex_ex` with :mod:`sympy`
+
+
+.. include:: latexdemo.py
+   :literal:
+
+Start of Program Output
+
+
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+  x = \frac{{\alpha}_{1} {}\bm{x}}{{\delta}^{{\nu}{\gamma}}_{r}}
+  \end{equation*}
+
+
+End of Program Output
+
+The program :program:`latexdemo.py` demonstrates the extended symbol naming
+conventions in :mod:`latex_ex`.   the statment ``Format()`` starts the
+:class:`LatexPrinter` driver with default formatting. Note that on the right
+hand side of the output that *xbm* gives :math:`\bm{x}`, *alpha_1* gives
+:math:`\alpha_{1}` and  *delta__nugamma_r* gives :math:`\delta^{\nu\gamma}_{r}`.
+Also the fraction is  printed correctly.  The statment ``print 'x =',x`` sends
+the string ``'x = '+str(x)`` to the output processor (:func:`xdvi`).  Because
+the string contains an :math:`=` sign the processor treats the string as an
+LaTeX equation (unnumbered).  If ``'x ='``  was not in the print statment a
+LaTeX error would be generated.  In the case of a :class:`GA` multivector one
+does not need the ``'x ='`` if the multivector has been given a name.  In the
+example the :class:`GA` function :func:`make_symbols` has been used to create
+the :class:`sympy` symbols for convenience.  The :class:`sympy` function
+:func:`Symbol` could have also been used.
+
+
+:program:`Maxwell.py` a multivector example
+-------------------------------------------
+
+:program:`Maxwell.py` example of using :mod:`latex_ex` with :mod:`GA`
+
+
+.. include:: Maxwell.py
+   :literal:
+
+Start of Program Output
+
+:math:`I` Pseudo-Scalar  
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+  I = {\gamma}_{t}{\gamma}_{x}{\gamma}_{y}{\gamma}_{z}
+  \end{equation*}
+
+
+:math:`B` Magnetic Field Bi-Vector  
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+  B = - {B^{x}}{\gamma}_{t}{\gamma}_{x}- {B^{y}}{\gamma}_{t}{\gamma}_{y}- {B^{z}}{\gamma}_{t}{\gamma}_{z}
+  \end{equation*}
+
+
+:math:`F` Electric Field Bi-Vector  
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+  E = - {E^{x}}{\gamma}_{t}{\gamma}_{x}- {E^{y}}{\gamma}_{t}{\gamma}_{y}- {E^{z}}{\gamma}_{t}{\gamma}_{z}
+  \end{equation*}
+
+
+:math:`E+IB` Electo-Magnetic Field Bi-Vector  
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+  F = - {E^{x}}{\gamma}_{t}{\gamma}_{x}- {E^{y}}{\gamma}_{t}{\gamma}_{y}- {B^{z}}{\gamma}_{x}{\gamma}_{y}- {E^{z}}{\gamma}_{t}{\gamma}_{z}+ {B^{y}}{\gamma}_{x}{\gamma}_{z}- {B^{x}}{\gamma}_{y}{\gamma}_{z}
+  \end{equation*}
+
+
+:math:`J` Four Current  
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+  J =  {J^{t}}{\gamma}_{t}+ {J^{x}}{\gamma}_{x}+ {J^{y}}{\gamma}_{y}+ {J^{z}}{\gamma}_{z}
+  \end{equation*}
+
+
+Geometric Derivative of Electo-Magnetic Field Bi-Vector  
+
+.. math::
+  :nowrap:
+
+  \begin{align*}
+  \nabla F & =   \left(\partial_{z} {E^{z}} + \partial_{y} {E^{y}} + \partial_{x} {E^{x}}\right){\gamma}_{t} \\ & + \left(-\partial_{t} {E^{x}} + \partial_{y} {B^{z}} -\partial_{z} {B^{y}}\right){\gamma}_{x} \\ & + \left(\partial_{z} {B^{x}} -\partial_{t} {E^{y}} -\partial_{x} {B^{z}}\right){\gamma}_{y} \\ & + \left(-\partial_{y} {B^{x}} -\partial_{t} {E^{z}} + \partial_{x} {B^{y}}\right){\gamma}_{z} \\ & + \left(-\partial_{x} {E^{y}} -\partial_{t} {B^{z}} + \partial_{y} {E^{x}}\right){\gamma}_{t}{\gamma}_{x}{\gamma}_{y} \\ & + \left(-\partial_{x} {E^{z}} + \partial_{t} {B^{y}} + \partial_{z} {E^{x}}\right){\gamma}_{t}{\gamma}_{x}{\gamma}_{z} \\ & + \left(-\partial_{t} {B^{x}} -\partial_{y} {E^{z}} + \partial_{z} {E^{y}}\right){\gamma}_{t}{\gamma}_{y}{\gamma}_{z} \\ & + \left(\partial_{y} {B^{y}} + \partial_{z} {B^{z}} + \partial_{x} {B^{x}}\right){\gamma}_{x}{\gamma}_{y}{\gamma}_{z}\end{align*}
+
+
+All Maxwell Equations are  
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+  \nabla F = J
+  \end{equation*}
+
+
+Div :math:`E` and Curl :math:`H` Equations  
+
+.. math::
+  :nowrap:
+
+  \begin{align*}
+  <\nabla F>_1 -J & =   \left(-{J^{t}} + \partial_{z} {E^{z}} + \partial_{y} {E^{y}} + \partial_{x} {E^{x}}\right){\gamma}_{t} \\ & + \left(-{J^{x}} -\partial_{t} {E^{x}} + \partial_{y} {B^{z}} -\partial_{z} {B^{y}}\right){\gamma}_{x} \\ & + \left(\partial_{z} {B^{x}} -\partial_{t} {E^{y}} -{J^{y}} -\partial_{x} {B^{z}}\right){\gamma}_{y} \\ & + \left(-\partial_{y} {B^{x}} -\partial_{t} {E^{z}} -{J^{z}} + \partial_{x} {B^{y}}\right){\gamma}_{z}\end{align*}
+
+
+
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+    = 0
+  \end{equation*}
+
+
+Curl :math:`E` and Div :math:`B` equations  
+
+.. math::
+  :nowrap:
+
+  \begin{align*}
+  <\nabla F>_3 & =   \left( -\partial_{x} {E^{y}} -\partial_{t} {B^{z}} + \partial_{y} {E^{x}}\right){\gamma}_{t}\W {\gamma}_{x}\W {\gamma}_{y} \\ & + \left( -\partial_{x} {E^{z}} + \partial_{t} {B^{y}} + \partial_{z} {E^{x}}\right){\gamma}_{t}\W {\gamma}_{x}\W {\gamma}_{z} \\ & + \left( -\partial_{t} {B^{x}} -\partial_{y} {E^{z}} + \partial_{z} {E^{y}}\right){\gamma}_{t}\W {\gamma}_{y}\W {\gamma}_{z} \\ & + \left( \partial_{y} {B^{y}} + \partial_{z} {B^{z}} + \partial_{x} {B^{x}}\right){\gamma}_{x}\W {\gamma}_{y}\W {\gamma}_{z}\end{align*}
+
+
+
+
+.. math::
+  :nowrap:
+
+  \begin{equation*}
+    = 0
+  \end{equation*}
+
+
+End of Program Output
+
+The program :program:`Maxwell.py` demonstrates the use of the
+:class:`LatexPrinter` class with the :mod:`GA` module multivector class,
+:class:`MV`.  The :func:`Format` call initializes :class:`LatexPrinter`.  The
+only other explicit :mod:`latex_x`  module formatting statement used is
+``MV_format(3)``.  This statment changes the multivector latex format so that
+instead of printing the entire multivector on one line, which would run off the
+page, each multivector base and its coefficient are printed on individual lines
+using the latex align environment.  Another option used is that the printing of
+function arguments is suppressed since :math:`E`, :math:`B`, :math:`J`, and
+:math:`F` are multivector fields and printing out the argument,
+:math:`(t,x,y,z)`, for every field component would greatly lengthen the output
+and make it more difficult to format in a pleasing way.
+
diff --git a/lib/sympy/doc/src/modules/galgebra/latex_ex/latexdemo.py b/lib/sympy/doc/src/modules/galgebra/latex_ex/latexdemo.py
new file mode 100644
--- /dev/null
+++ b/lib/sympy/doc/src/modules/galgebra/latex_ex/latexdemo.py
@@ -0,0 +1,16 @@
+import sys
+import sympy.galgebra.GA as GA
+import sympy.galgebra.latex_ex as tex
+
+GA.set_main(sys.modules[__name__])
+
+if __name__ == '__main__':
+
+    tex.Format()
+    GA.make_symbols('xbm alpha_1 delta__nugamma_r')
+    
+    x = alpha_1*xbm/delta__nugamma_r
+    
+    print 'x =',x
+    
+    tex.xdvi()
diff --git a/lib/sympy/doc/src/modules/geometry.txt b/lib/sympy/doc/src/modules/geometry.txt
new file mode 100644
--- /dev/null
+++ b/lib/sympy/doc/src/modules/geometry.txt
@@ -0,0 +1,190 @@
+Geometry Module
+===============
+
+.. module:: sympy.geometry
+
+Introduction
+------------
+
+The geometry module for SymPy allows one to create two-dimensional geometrical
+entities, such as lines and circles, and query information about these
+entities. This could include asking the area of an ellipse, checking for
+collinearity of a set of points, or finding the intersection between two lines.
+The primary use case of the module involves entities with numerical values, but
+it is possible to also use symbolic representations.
+
+Available Entities
+------------------
+
+The following entities are currently available in the geometry module:
+
+* Point
+* Line, Ray, Segment
+* Ellipse, Circle
+* Polygon, RegularPolygon, Triangle
+
+Most of the work one will do will be through the properties and methods of
+these entities, but several global methods exist for one's usage:
+
+* intersection(entity1, entity2)
+* are_similar(entity1, entity2)
+* convex_hull(points)
+
+For a full API listing and an explanation of the methods and their return
+values please see the list of classes at the end of this document.
+
+Example Usage
+-------------
+
+The following Python session gives one an idea of how to work with some of the
+geometry module.
+
+    >>> from sympy import *
+    >>> from sympy.geometry import *
+    >>> x = Point(0, 0)
+    >>> y = Point(1, 1)
+    >>> z = Point(2, 2)
+    >>> zp = Point(1, 0)
+    >>> Point.is_collinear(x, y, z)
+    True
+    >>> Point.is_collinear(x, y, zp)
+    False
+    >>> t = Triangle(zp, y, x)
+    >>> t.area
+    1/2
+    >>> t.medians[x]
+    Segment(Point(0, 0), Point(1, 1/2))
+    >>> Segment(Point(1, S(1)/2), Point(0, 0))
+    Segment(Point(0, 0), Point(1, 1/2))
+    >>> m = t.medians
+    >>> intersection(m[x], m[y], m[zp])
+    [Point(2/3, 1/3)]
+    >>> c = Circle(x, 5)
+    >>> l = Line(Point(5, -5), Point(5, 5))
+    >>> c.is_tangent(l) # is l tangent to c?
+    True
+    >>> l = Line(x, y)
+    >>> c.is_tangent(l) # is l tangent to c?
+    False
+    >>> intersection(c, l)
+    [Point(-5*sqrt(2)/2, -5*sqrt(2)/2), Point(5*sqrt(2)/2, 5*sqrt(2)/2)]
+
+Intersection of medians
+-----------------------
+::
+
+    >>> from sympy import Symbol
+    >>> from sympy.geometry import Point, Triangle, intersection
+
+    >>> a = Symbol("a")
+    >>> b = Symbol("b")
+    >>> c = Symbol("c")
+
+    >>> x = Point(0,0)
+    >>> y = Point(c,0)
+    >>> z = Point(a,b)
+    >>> t = Triangle(x,y,z)
+
+    >>> t.area
+    b*c/2
+
+    >>> t.medians #doctest: +NORMALIZE_WHITESPACE
+    {Point(a, b): Segment(Point(a, b), Point(c/2, 0)),
+    Point(c, 0): Segment(Point(c, 0), Point(a/2, b/2)),
+    Point(0, 0): Segment(Point(0, 0), Point(a/2 + c/2, b/2))}
+
+    >>> intersection(t.medians[x], t.medians[y], t.medians[z])
+    [Point(a/3 + c/3, b/3)]
+
+An in-depth example: Pappus' Theorem
+------------------------------------
+::
+
+    >>> from sympy import *
+    >>> from sympy.geometry import *
+    >>>
+    >>> l1 = Line(Point(0, 0), Point(5, 6))
+    >>> l2 = Line(Point(0, 0), Point(2, -2))
+    >>>
+    >>> def subs_point(l, val):
+    ...    """Take an arbitrary point and make it a fixed point."""
+    ...    t = Symbol('t', real=True)
+    ...    ap = l.arbitrary_point()
+    ...    return Point(ap[0].subs(t, val), ap[1].subs(t, val))
+    ...
+    >>> p11 = subs_point(l1, 5)
+    >>> p12 = subs_point(l1, 6)
+    >>> p13 = subs_point(l1, 11)
+    >>>
+    >>> p21 = subs_point(l2, -1)
+    >>> p22 = subs_point(l2, 2)
+    >>> p23 = subs_point(l2, 13)
+    >>>
+    >>> ll1 = Line(p11, p22)
+    >>> ll2 = Line(p11, p23)
+    >>> ll3 = Line(p12, p21)
+    >>> ll4 = Line(p12, p23)
+    >>> ll5 = Line(p13, p21)
+    >>> ll6 = Line(p13, p22)
+    >>>
+    >>> pp1 = intersection(ll1, ll3)[0]
+    >>> pp2 = intersection(ll2, ll5)[0]
+    >>> pp3 = intersection(ll4, ll6)[0]
+    >>>
+    >>> print Point.is_collinear(pp1, pp2, pp3)
+    True
+
+Miscellaneous Notes
+-------------------
+
+* The area property of Polygon and Triangle may return a positive or
+  negative value, depending on whether or not the points are oriented
+  counter-clockwise or clockwise, respectively. If you always want a
+  positive value be sure to use the ``abs`` function.
+* Although Polygon can refer to any type of polygon, the code has been
+  written for simple polygons. Hence, expect potential problems if dealing
+  with complex polygons (overlapping sides).
+* Since !SymPy is still in its infancy some things may not simplify
+  properly and hence some things that should return True (e.g.,
+  Point.is_collinear) may not actually do so. Similarly, attempting to find
+  the intersection of entities that do intersect may result in an empty
+  result.
+
+Future Work
+-----------
+
+Truth Setting Expressions
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+When one deals with symbolic entities, it often happens that an assertion
+cannot be guaranteed. For example, consider the following code:
+
+    >>> from sympy import *
+    >>> from sympy.geometry import *
+    >>> x,y,z = map(Symbol, 'xyz')
+    >>> p1,p2,p3 = Point(x, y), Point(y, z), Point(2*x*y, y)
+    >>> Point.is_collinear(p1, p2, p3)
+    False
+
+Even though the result is currently ``False``, this is not *always* true. If the
+quantity `z - y - 2*y*z + 2*y**2 == 0` then the points will be collinear. It
+would be really nice to inform the user of this because such a quantity may be
+useful to a user for further calculation and, at the very least, being nice to
+know. This could be potentially done by returning an object (e.g.,
+GeometryResult) that the user could use. This actually would not involve an
+extensive amount of work.
+
+Three Dimensions and Beyond
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Currently there are no plans for extending the module to three dimensions, but
+it certainly would be a good addition. This would probably involve a fair
+amount of work since many of the algorithms used are specific to two
+dimensions.
+
+Geometry Visualization
+~~~~~~~~~~~~~~~~~~~~~~
+
+The plotting module is capable of plotting geometric entities. See
+`Plotting Geometric Entities <https://github.com/sympy/sympy/wiki/Plotting-Module>`_ in
+the plotting module entry.
diff --git a/lib/sympy/doc/src/modules/index.txt b/lib/sympy/doc/src/modules/index.txt
new file mode 100644
--- /dev/null
+++ b/lib/sympy/doc/src/modules/index.txt
@@ -0,0 +1,51 @@
+.. _module-docs:
+
+SymPy Modules Reference
+=======================
+
+Because every feature of SymPy must have a test case, when you are not sure how
+to use something, just look into the ``tests/`` directories, find that feature
+and read the tests for it, that will tell you everything you need to know.
+
+Most of the things are already documented though in this document, that is
+automatically generated using SymPy's docstrings.
+
+Click the  "modules" (:ref:`modindex`) link in the top right corner to easily
+access any SymPy module, or use this contens:
+
+.. toctree::
+   :maxdepth: 2
+
+   core.txt
+   concrete.txt
+   evalf.txt
+   functions.txt
+   geometry.txt
+   galgebra/GA/GAsympy.txt
+   galgebra/latex_ex/latex_ex.txt
+   integrals.txt
+   logic.txt
+   matrices.txt
+   mpmath/index.txt
+   polys/index.txt
+   printing.txt
+   plotting.txt
+   assumptions.txt
+   rewriting.txt
+   series.txt
+   simplify/simplify.txt
+   simplify/hyperexpand.txt
+   statistics.txt
+   solvers/ode.txt
+   solvers/solvers.txt
+   tensor.txt
+   utilities/index.txt
+
+Contributions to docs
+---------------------
+
+All contributions are welcome. If you'd like to improve something, look into
+the sources if they contain the information you need (if not, please fix them),
+otherwise the documentation generation needs to be improved (look in the
+``doc/`` directory).
+
diff --git a/lib/sympy/doc/src/modules/integrals.txt b/lib/sympy/doc/src/modules/integrals.txt
new file mode 100644
--- /dev/null
+++ b/lib/sympy/doc/src/modules/integrals.txt
@@ -0,0 +1,75 @@
+Integrals
+==========
+
+.. module:: sympy.integrals
+
+The *integrals* module in SymPy implements methdos calculating definite un undefinite integrals of expressions.
+
+Principal method in this module is integrate()
+
+  - integrate(f, x) returns the indefinite integral :math:`\int f\,dx`
+  - integrate(f, (x, a, b)) returns the definite integral :math:`\int_{a}^{b} f\,dx`
+
+Examples
+--------
+SymPy can integrate a vast array of functions. It can integrate polynomial functions::
+
+    >>> from sympy import *
+    >>> init_printing(use_unicode=False, wrap_line=False, no_global=True)
+    >>> x = Symbol('x')
+    >>> integrate(x**2 + x + 1, x)
+     3    2
+    x    x
+    -- + -- + x
+    3    2
+
+Rational functions::
+
+	>>> integrate(x/(x**2+2*x+1), x)
+	               1
+	log(x + 1) + -----
+	             x + 1
+
+
+Exponential-polynomial functions. Multiplicative combinations of polynomials and the functions exp, cos and sin can be integrated by hand using repeated integration by parts, which is an extremely tedious process. Happily, SymPy will deal with these integrals.
+
+::
+
+    >>> integrate(x**2 * exp(x) * cos(x), x)
+     2  x           2  x                         x           x
+    x *e *sin(x)   x *e *cos(x)      x          e *sin(x)   e *cos(x)
+    ------------ + ------------ - x*e *sin(x) + --------- - ---------
+         2              2                           2           2
+
+
+
+even a few nonelementary integrals (in particular, some integrals involving the error function) can be evaluated::
+
+	>>> integrate(exp(-x**2)*erf(x), x)
+	  ____    2
+	\/ pi *erf (x)
+	--------------
+	      4
+
+
+Internals
+---------
+
+There is a general method for calculating antiderivatives of elementary functions, called the Risch algorithm. The Risch algorithm is a decision procedure that can determine whether an elementary solution exists, and in that case calculate it. It can be extended to handle many nonelementary functions in addition to the elementary ones.
+
+SymPy currently uses a simplified version of the Risch algorithm, called the Risch-Norman algorithm. This algorithm is much faster, but may fail to find an antiderivative, although it is still very powerful. SymPy also uses pattern matching and heuristics to speed up evaluation of some types of integrals, e.g. polynomials.
+
+API reference
+-------------
+
+.. automethod:: sympy.integrals.integrate
+
+Class Integral represents an unevaluated integral and has some methods that help in the integration of an expression.
+
+.. autoclass:: sympy.integrals.Integral
+   :members:
+
+
+TODO and Bugs
+-------------
+There are still lots of functions that sympy does not know how to integrate. For bugs related to this module, see http://code.google.com/p/sympy/issues/list?q=label:Integration
diff --git a/lib/sympy/doc/src/modules/logic.txt b/lib/sympy/doc/src/modules/logic.txt
new file mode 100644
--- /dev/null
+++ b/lib/sympy/doc/src/modules/logic.txt
@@ -0,0 +1,80 @@
+Logic Module
+===============
+
+.. module:: sympy.logic
+
+Introduction
+------------
+
+The logic module for SymPy allows to form and manipulate logic expressions using
+symbolic and boolean values
+
+Forming logical expressions
+---------------------------
+
+You can build boolean expressions with the standard python operators & (And),
+|(Or), ~ (Not):
+
+    >>> from sympy import *
+    >>> x, y = symbols('x,y')
+    >>> y | (x & y)
+    Or(And(x, y), y)
+    >>> x | y
+    Or(x, y)
+    >>> ~x
+    Not(x)
+
+You can also form implications with >> and <<:
+
+    >>> x >> y
+    Implies(x, y)
+    >>> x << y
+    Implies(y, x)
+
+Like most types in SymPy, Boolean expressions inherit from sympy.core.Basic:
+
+    >>> (y & x).subs({x: True, y: True})
+    True
+    >>> (x | y).atoms()
+    set([x, y])
+
+Other boolean functions
+-----------------------
+
+.. automethod:: sympy.logic.boolalg.Xor
+
+.. automethod:: sympy.logic.boolalg.Nand
+
+.. automethod:: sympy.logic.boolalg.Nor
+
+.. automethod:: sympy.logic.boolalg.Equivalent
+
+Inference
+---------
+
+.. module: sympy.logic.inference
+
+This module implements some inference routines in propositional logic.
+
+The function satisfiable will test that a given boolean expression is satisfiable,
+that is, you can assign values to the variables to make the sentence True.
+
+For example, the expression x & ~x is not satisfiable, since there are no values
+for x that make this sentence True. On the other hand, (x | y) & (x | ~y) & (~x | y)
+is satisfiable with both x and y being True.
+
+    >>> from sympy.logic.inference import satisfiable
+    >>> from sympy import Symbol
+    >>> x = Symbol('x')
+    >>> y = Symbol('y')
+    >>> satisfiable(x & ~x)
+    False
+    >>> satisfiable((x | y) & (x | ~y) & (~x | y))
+    {x: True, y: True}
+
+As you see, when a sentence is satisfiable, it returns a model that makes that
+sentence True. If it is not satisfiable it will return False
+
+.. automethod:: sympy.logic.inference.satisfiable
+
+.. TODO: write about CNF file format
diff --git a/lib/sympy/doc/src/modules/matrices.txt b/lib/sympy/doc/src/modules/matrices.txt
new file mode 100644
--- /dev/null
+++ b/lib/sympy/doc/src/modules/matrices.txt
@@ -0,0 +1,491 @@
+Matrices (linear algebra)
+=========================
+
+.. module:: sympy.matrices.matrices
+
+Creating Matrices
+-----------------
+
+The linear algebra module is designed to be as simple as possible. First, we
+import and declare our first Matrix object:
+
+    >>> from sympy.interactive.printing import init_printing
+    >>> init_printing(use_unicode=False, wrap_line=False, no_global=True)
+    >>> from sympy.matrices import *
+    >>> Matrix([[1,0], [0,1]])
+    [1  0]
+    [    ]
+    [0  1]
+    >>> Matrix((
+    ...   Matrix((
+    ...     (1, 0, 0),
+    ...     (0, 0, 0)
+    ...   )),
+    ...   (0, 0, -1)
+    ... ))
+    [1  0  0 ]
+    [        ]
+    [0  0  0 ]
+    [        ]
+    [0  0  -1]
+    >>> Matrix([[1, 2, 3]])
+    [1 2 3]
+    >>> Matrix([1, 2, 3])
+    [1]
+    [ ]
+    [2]
+    [ ]
+    [3]
+
+This is the standard manner one creates a matrix, i.e. with a list of
+appropriately-sizes lists and/or matrices. SymPy also supports more advanced
+methods of matrix creation including a single list of values and dimension
+inputs:
+
+    >>> Matrix(2, 3, [1, 2, 3, 4, 5, 6])
+    [1  2  3]
+    [       ]
+    [4  5  6]
+
+More interestingly (and usefully), we can use a 2-variable function (or lambda)
+to make one. Here we create an indicator function which is 1 on the diagonal
+and then use it to make the identity matrix:
+
+    >>> def f(i,j):
+    ...     if i == j:
+    ...             return 1
+    ...     else:
+    ...             return 0
+    ...
+    >>> Matrix(4, 4, f)
+    [1  0  0  0]
+    [          ]
+    [0  1  0  0]
+    [          ]
+    [0  0  1  0]
+    [          ]
+    [0  0  0  1]
+
+Finally let's use lambda to create a 1-line matrix with 1's in the even
+permutation entries:
+
+    >>> Matrix(3, 4, lambda i,j: 1 - (i+j) % 2)
+    [1  0  1  0]
+    [          ]
+    [0  1  0  1]
+    [          ]
+    [1  0  1  0]
+
+There are also a couple of special constructors for quick matrix construction -
+``eye`` is the identity matrix, ``zeros`` and ``ones`` for matrices of all
+zeros and ones, respectively:
+
+    >>> eye(4)
+    [1  0  0  0]
+    [          ]
+    [0  1  0  0]
+    [          ]
+    [0  0  1  0]
+    [          ]
+    [0  0  0  1]
+    >>> zeros(2)
+    [0  0]
+    [    ]
+    [0  0]
+    >>> zeros(2, 5)
+    [0  0  0  0  0]
+    [             ]
+    [0  0  0  0  0]
+    >>> ones(3)
+    [1  1  1]
+    [       ]
+    [1  1  1]
+    [       ]
+    [1  1  1]
+    >>> ones(1, 3)
+    [1  1  1]
+
+
+Basic Manipulation
+------------------
+
+While learning to work with matrices, let's choose one where the entries are
+readily identifiable. One useful thing to know is that while matrices are
+2-dimensional, the storage is not and so it is allowable - though one should be
+careful - to access the entries as if they were a 1-d list.
+
+    >>> M = Matrix(2, 3, [1, 2, 3, 4, 5, 6])
+    >>> M[4]
+    5
+
+Now, the more standard entry access is a pair of indices:
+
+    >>> M[1,2]
+    6
+    >>> M[0,0]
+    1
+    >>> M[1,1]
+    5
+
+Since this is Python we're also able to slice submatrices::
+
+    >>> M[0:2,0:2]
+    [1  2]
+    [    ]
+    [4  5]
+    >>> M[1:2,2]
+    [6]
+    >>> M[:,2]
+    [3]
+    [ ]
+    [6]
+
+Remember in the 2nd example above that slicing 2:2 gives an empty range and
+that, as in python, a 4 column list is indexed from 0 to 3. In particular, this
+mean a quick way to create a copy of the matrix is:
+
+    >>> M2 = M[:,:]
+    >>> M2[0,0] = 100
+    >>> M
+    [1  2  3]
+    [       ]
+    [4  5  6]
+
+See? Changing M2 didn't change M. Since we can slice, we can also assign
+entries:
+
+    >>> M = Matrix(([1,2,3,4],[5,6,7,8],[9,10,11,12],[13,14,15,16]))
+    >>> M
+    [1   2   3   4 ]
+    [              ]
+    [5   6   7   8 ]
+    [              ]
+    [9   10  11  12]
+    [              ]
+    [13  14  15  16]
+    >>> M[2,2] = M[0,3] = 0
+    >>> M
+    [1   2   3   0 ]
+    [              ]
+    [5   6   7   8 ]
+    [              ]
+    [9   10  0   12]
+    [              ]
+    [13  14  15  16]
+
+as well as assign slices:
+
+    >>> M = Matrix(([1,2,3,4],[5,6,7,8],[9,10,11,12],[13,14,15,16]))
+    >>> M[2:,2:] = Matrix(2,2,lambda i,j: 0)
+    >>> M
+    [1   2   3  4]
+    [            ]
+    [5   6   7  8]
+    [            ]
+    [9   10  0  0]
+    [            ]
+    [13  14  0  0]
+
+All the standard arithmetic operations are supported:
+
+    >>> M = Matrix(([1,2,3],[4,5,6],[7,8,9]))
+    >>> M - M
+    [0  0  0]
+    [       ]
+    [0  0  0]
+    [       ]
+    [0  0  0]
+    >>> M + M
+    [2   4   6 ]
+    [          ]
+    [8   10  12]
+    [          ]
+    [14  16  18]
+    >>> M * M
+    [30   36   42 ]
+    [             ]
+    [66   81   96 ]
+    [             ]
+    [102  126  150]
+    >>> M2 = Matrix(3,1,[1,5,0])
+    >>> M*M2
+    [11]
+    [  ]
+    [29]
+    [  ]
+    [47]
+    >>> M**2
+    [30   36   42 ]
+    [             ]
+    [66   81   96 ]
+    [             ]
+    [102  126  150]
+
+As well as some useful vector operations:
+
+    >>> M.row_del(0)
+    >>> M
+    [4  5  6]
+    [       ]
+    [7  8  9]
+    >>> M.col_del(1)
+    >>> M
+    [4  6]
+    [    ]
+    [7  9]
+    >>> v1 = Matrix([1,2,3])
+    >>> v2 = Matrix([4,5,6])
+    >>> v3 = v1.cross(v2)
+    >>> v1.dot(v2)
+    32
+    >>> v2.dot(v3)
+    0
+    >>> v1.dot(v3)
+    0
+
+Recall that the row_del() and col_del() operations don't return a value - they
+simply change the matrix object. We can also ''glue'' together matrices of the
+appropriate size:
+
+    >>> M1 = eye(3)
+    >>> M2 = zeros(3, 4)
+    >>> M1.row_join(M2)
+    [1  0  0  0  0  0  0]
+    [                   ]
+    [0  1  0  0  0  0  0]
+    [                   ]
+    [0  0  1  0  0  0  0]
+    >>> M3 = zeros(4, 3)
+    >>> M1.col_join(M3)
+    [1  0  0]
+    [       ]
+    [0  1  0]
+    [       ]
+    [0  0  1]
+    [       ]
+    [0  0  0]
+    [       ]
+    [0  0  0]
+    [       ]
+    [0  0  0]
+    [       ]
+    [0  0  0]
+
+
+Operations on entries
+---------------------
+
+We are not restricted to having multiplication between two matrices:
+
+    >>> M = eye(3)
+    >>> 2*M
+    [2  0  0]
+    [       ]
+    [0  2  0]
+    [       ]
+    [0  0  2]
+    >>> 3*M
+    [3  0  0]
+    [       ]
+    [0  3  0]
+    [       ]
+    [0  0  3]
+
+but we can also apply functions to our matrix entries using applyfunc(). Here we'll declare a function that double any input number. Then we apply it to the 3x3 identity matrix:
+
+    >>> f = lambda x: 2*x
+    >>> eye(3).applyfunc(f)
+    [2  0  0]
+    [       ]
+    [0  2  0]
+    [       ]
+    [0  0  2]
+
+One more useful matrix-wide entry application function is the substitution function. Let's declare a matrix with symbolic entries then substitute a value. Remember we can substitute anything - even another symbol!:
+
+    >>> from sympy import Symbol
+    >>> x = Symbol('x')
+    >>> M = eye(3) * x
+    >>> M
+    [x  0  0]
+    [       ]
+    [0  x  0]
+    [       ]
+    [0  0  x]
+    >>> M.subs(x, 4)
+    [4  0  0]
+    [       ]
+    [0  4  0]
+    [       ]
+    [0  0  4]
+    >>> y = Symbol('y')
+    >>> M.subs(x, y)
+    [y  0  0]
+    [       ]
+    [0  y  0]
+    [       ]
+    [0  0  y]
+
+
+Linear algebra
+--------------
+
+Now that we have the basics out of the way, let's see what we can do with the
+actual matrices. Of course the first things that come to mind are the basics
+like the determinant:
+
+    >>> M = Matrix(( [1, 2, 3], [3, 6, 2], [2, 0, 1] ))
+    >>> M.det()
+    -28
+    >>> M2 = eye(3)
+    >>> M2.det()
+    1
+    >>> M3 = Matrix(( [1, 0, 0], [1, 0, 0], [1, 0, 0] ))
+    >>> M3.det()
+    0
+
+and the inverse. In SymPy the inverse is computed by Gaussian elimination by
+default but we can specify it be done by LU decomposition as well:
+
+    >>> M2.inv()
+    [1  0  0]
+    [       ]
+    [0  1  0]
+    [       ]
+    [0  0  1]
+    >>> M2.inv("LU")
+    [1  0  0]
+    [       ]
+    [0  1  0]
+    [       ]
+    [0  0  1]
+    >>> M.inv("LU")
+    [-3/14  1/14  1/2 ]
+    [                 ]
+    [-1/28  5/28  -1/4]
+    [                 ]
+    [ 3/7   -1/7   0  ]
+    >>> M * M.inv("LU")
+    [1  0  0]
+    [       ]
+    [0  1  0]
+    [       ]
+    [0  0  1]
+
+We can perform a QR factorization which is handy for solving systems:
+
+    >>> A = Matrix([[1,1,1],[1,1,3],[2,3,4]])
+    >>> Q, R = A.QRdecomposition()
+    >>> Q
+    [  ___     ___     ___]
+    [\/ 6   -\/ 3   -\/ 2 ]
+    [-----  ------  ------]
+    [  6      3       2   ]
+    [                     ]
+    [  ___     ___    ___ ]
+    [\/ 6   -\/ 3   \/ 2  ]
+    [-----  ------  ----- ]
+    [  6      3       2   ]
+    [                     ]
+    [  ___    ___         ]
+    [\/ 6   \/ 3          ]
+    [-----  -----     0   ]
+    [  3      3           ]
+    >>> R
+    [           ___         ]
+    [  ___  4*\/ 6       ___]
+    [\/ 6   -------  2*\/ 6 ]
+    [          3            ]
+    [                       ]
+    [          ___          ]
+    [        \/ 3           ]
+    [  0     -----      0   ]
+    [          3            ]
+    [                       ]
+    [                   ___ ]
+    [  0       0      \/ 2  ]
+    >>> Q*R
+    [1  1  1]
+    [       ]
+    [1  1  3]
+    [       ]
+    [2  3  4]
+
+
+In addition to the solvers in the solver.py file, we can solve the system Ax=b
+by passing the b vector to the matrix A's LUsolve function. Here we'll cheat a
+little choose A and x then multiply to get b. Then we can solve for x and check
+that it's correct:
+
+    >>> A = Matrix([ [2, 3, 5], [3, 6, 2], [8, 3, 6] ])
+    >>> x = Matrix(3,1,[3,7,5])
+    >>> b = A*x
+    >>> soln = A.LUsolve(b)
+    >>> soln
+    [3]
+    [ ]
+    [7]
+    [ ]
+    [5]
+
+There's also a nice Gram-Schmidt orthogonalizer which will take a set of
+vectors and orthogonalize then with respect to another another. There is an
+optional argument which specifies whether or not the output should also be
+normalized, it defaults to False. Let's take some vectors and orthogonalize
+them - one normalized and one not:
+
+    >>> L = [Matrix([2,3,5]), Matrix([3,6,2]), Matrix([8,3,6])]
+    >>> out1 = GramSchmidt(L)
+    >>> out2 = GramSchmidt(L, True)
+
+Let's take a look at the vectors:
+
+    >>> for i in out1:
+    ...     print i
+    ...
+    [2]
+    [3]
+    [5]
+    [ 23/19]
+    [ 63/19]
+    [-47/19]
+    [ 1692/353]
+    [-1551/706]
+    [ -423/706]
+    >>> for i in out2:
+    ...      print i
+    ...
+    [  sqrt(38)/19]
+    [3*sqrt(38)/38]
+    [5*sqrt(38)/38]
+    [ 23*sqrt(6707)/6707]
+    [ 63*sqrt(6707)/6707]
+    [-47*sqrt(6707)/6707]
+    [ 12*sqrt(706)/353]
+    [-11*sqrt(706)/706]
+    [ -3*sqrt(706)/706]
+
+We can spot-check their orthogonality with dot() and their normality with
+norm():
+
+    >>> out1[0].dot(out1[1])
+    0
+    >>> out1[0].dot(out1[2])
+    0
+    >>> out1[1].dot(out1[2])
+    0
+    >>> out2[0].norm()
+    1
+    >>> out2[1].norm()
+    1
+    >>> out2[2].norm()
+    1
+
+So there is quite a bit that can be done with the module including eigenvalues,
+eigenvectors, nullspace calculation, cofactor expansion tools, and so on. From
+here one might want to look over the matrices.py file for all functionality.
+
+Matrix Class Reference
+----------------------
+.. autoclass:: Matrix
+   :members:
diff --git a/lib/sympy/doc/src/modules/mpmath/basics.txt b/lib/sympy/doc/src/modules/mpmath/basics.txt
new file mode 100644
--- /dev/null
+++ b/lib/sympy/doc/src/modules/mpmath/basics.txt
@@ -0,0 +1,227 @@
+Basic usage
+===========================
+
+In interactive code examples that follow, it will be assumed that
+all items in the ``mpmath`` namespace have been imported::
+
+    >>> from mpmath import *
+
+Importing everything can be convenient, especially when using mpmath interactively, but be
+careful when mixing mpmath with other libraries! To avoid inadvertently overriding
+other functions or objects, explicitly import only the needed objects, or use
+the ``mpmath.`` or ``mp.`` namespaces::
+
+    from mpmath import sin, cos
+    sin(1), cos(1)
+
+    import mpmath
+    mpmath.sin(1), mpmath.cos(1)
+
+    from mpmath import mp    # mp context object -- to be explained
+    mp.sin(1), mp.cos(1)
+
+
+Number types
+------------
+
+Mpmath provides the following numerical types:
+
+    +------------+----------------+
+    | Class      | Description    |
+    +============+================+
+    | ``mpf``    | Real float     |
+    +------------+----------------+
+    | ``mpc``    | Complex float  |
+    +------------+----------------+
+    | ``matrix`` | Matrix         |
+    +------------+----------------+
+
+The following section will provide a very short introduction to the types ``mpf`` and ``mpc``. Intervals and matrices are described further in the documentation chapters on interval arithmetic and matrices / linear algebra.
+
+The ``mpf`` type is analogous to Python's built-in ``float``. It holds a real number or one of the special values ``inf`` (positive infinity), ``-inf`` (negative infinity) and ``nan`` (not-a-number, indicating an indeterminate result). You can create ``mpf`` instances from strings, integers, floats, and other ``mpf`` instances:
+
+    >>> mpf(4)
+    mpf('4.0')
+    >>> mpf(2.5)
+    mpf('2.5')
+    >>> mpf("1.25e6")
+    mpf('1250000.0')
+    >>> mpf(mpf(2))
+    mpf('2.0')
+    >>> mpf("inf")
+    mpf('+inf')
+
+The ``mpc`` type represents a complex number in rectangular form as a pair of ``mpf`` instances. It can be constructed from a Python ``complex``, a real number, or a pair of real numbers:
+
+    >>> mpc(2,3)
+    mpc(real='2.0', imag='3.0')
+    >>> mpc(complex(2,3)).imag
+    mpf('3.0')
+
+You can mix ``mpf`` and ``mpc`` instances with each other and with Python numbers:
+
+    >>> mpf(3) + 2*mpf('2.5') + 1.0
+    mpf('9.0')
+    >>> mp.dps = 15      # Set precision (see below)
+    >>> mpc(1j)**0.5
+    mpc(real='0.70710678118654757', imag='0.70710678118654757')
+
+
+Setting the precision
+---------------------
+
+Mpmath uses a global working precision; it does not keep track of the precision or accuracy of individual numbers. Performing an arithmetic operation or calling ``mpf()`` rounds the result to the current working precision. The working precision is controlled by a context object called ``mp``, which has the following default state:
+
+    >>> print mp
+    Mpmath settings:
+      mp.prec = 53                [default: 53]
+      mp.dps = 15                 [default: 15]
+      mp.trap_complex = False     [default: False]
+
+The term **prec** denotes the binary precision (measured in bits) while **dps** (short for *decimal places*) is the decimal precision. Binary and decimal precision are related roughly according to the formula ``prec = 3.33*dps``. For example, it takes a precision of roughly 333 bits to hold an approximation of pi that is accurate to 100 decimal places (actually slightly more than 333 bits is used).
+
+Changing either precision property of the ``mp`` object automatically updates the other; usually you just want to change the ``dps`` value:
+
+    >>> mp.dps = 100
+    >>> mp.dps
+    100
+    >>> mp.prec
+    336
+
+When the precision has been set, all ``mpf`` operations are carried out at that precision::
+
+    >>> mp.dps = 50
+    >>> mpf(1) / 6
+    mpf('0.16666666666666666666666666666666666666666666666666656')
+    >>> mp.dps = 25
+    >>> mpf(2) ** mpf('0.5')
+    mpf('1.414213562373095048801688713')
+
+The precision of complex arithmetic is also controlled by the ``mp`` object:
+
+    >>> mp.dps = 10
+    >>> mpc(1,2) / 3
+    mpc(real='0.3333333333321', imag='0.6666666666642')
+
+There is no restriction on the magnitude of numbers. An ``mpf`` can for example hold an approximation of a large Mersenne prime:
+
+    >>> mp.dps = 15
+    >>> print mpf(2)**32582657 - 1
+    1.24575026015369e+9808357
+
+Or why not 1 googolplex:
+
+    >>> print mpf(10) ** (10**100)  # doctest:+ELLIPSIS
+    1.0e+100000000000000000000000000000000000000000000000000...
+
+The (binary) exponent is stored exactly and is independent of the precision.
+
+Temporarily changing the precision
+..................................
+
+It is often useful to change the precision during only part of a calculation. A way to temporarily increase the precision and then restore it is as follows:
+
+    >>> mp.prec += 2
+    >>> # do_something()
+    >>> mp.prec -= 2
+
+In Python 2.5, the ``with`` statement along with the mpmath functions ``workprec``, ``workdps``, ``extraprec`` and ``extradps`` can be used to temporarily change precision in a more safe manner:
+
+    >>> from __future__ import with_statement
+    >>> with workdps(20):  # doctest: +SKIP
+    ...     print mpf(1)/7
+    ...     with extradps(10):
+    ...         print mpf(1)/7
+    ...
+    0.14285714285714285714
+    0.142857142857142857142857142857
+    >>> mp.dps
+    15
+
+The ``with`` statement ensures that the precision gets reset when exiting the block, even in the case that an exception is raised. (The effect of the ``with`` statement can be emulated in Python 2.4 by using a ``try/finally`` block.)
+
+The ``workprec`` family of functions can also be used as function decorators:
+
+    >>> @workdps(6)
+    ... def f():
+    ...     return mpf(1)/3
+    ...
+    >>> f()
+    mpf('0.33333331346511841')
+
+
+Some functions accept the ``prec`` and ``dps`` keyword arguments and this will override the global working precision. Note that this will not affect the precision at which the result is printed, so to get all digits, you must either use increase precision afterward when printing or use ``nstr``/``nprint``:
+
+    >>> mp.dps = 15
+    >>> print exp(1)
+    2.71828182845905
+    >>> print exp(1, dps=50)      # Extra digits won't be printed
+    2.71828182845905
+    >>> nprint(exp(1, dps=50), 50)
+    2.7182818284590452353602874713526624977572470937
+
+Finally, instead of using the global context object ``mp``, you can create custom contexts and work with methods of those instances instead of global functions. The working precision will be local to each context object:
+
+    >>> mp2 = mp.clone()
+    >>> mp.dps = 10
+    >>> mp2.dps = 20
+    >>> print mp.mpf(1) / 3
+    0.3333333333
+    >>> print mp2.mpf(1) / 3
+    0.33333333333333333333
+
+**Note**: the ability to create multiple contexts is a new feature that is only partially implemented. Not all mpmath functions are yet available as context-local methods. In the present version, you are likely to encounter bugs if you try mixing different contexts.
+
+Providing correct input
+-----------------------
+
+Note that when creating a new ``mpf``, the value will at most be as accurate as the input. *Be careful when mixing mpmath numbers with Python floats*. When working at high precision, fractional ``mpf`` values should be created from strings or integers:
+
+    >>> mp.dps = 30
+    >>> mpf(10.9)   # bad
+    mpf('10.9000000000000003552713678800501')
+    >>> mpf('10.9')  # good
+    mpf('10.8999999999999999999999999999997')
+    >>> mpf(109) / mpf(10)   # also good
+    mpf('10.8999999999999999999999999999997')
+    >>> mp.dps = 15
+
+(Binary fractions such as 0.5, 1.5, 0.75, 0.125, etc, are generally safe as input, however, since those can be represented exactly by Python floats.)
+
+Printing
+--------
+
+By default, the ``repr()`` of a number includes its type signature. This way ``eval`` can be used to recreate a number from its string representation:
+
+    >>> eval(repr(mpf(2.5)))
+    mpf('2.5')
+
+Prettier output can be obtained by using ``str()`` or ``print``, which hide the ``mpf`` and ``mpc`` signatures and also suppress rounding artifacts in the last few digits:
+
+    >>> mpf("3.14159")
+    mpf('3.1415899999999999')
+    >>> print mpf("3.14159")
+    3.14159
+    >>> print mpc(1j)**0.5
+    (0.707106781186548 + 0.707106781186548j)
+
+Setting the ``mp.pretty`` option will use the ``str()``-style output for ``repr()`` as well:
+
+    >>> mp.pretty = True
+    >>> mpf(0.6)
+    0.6
+    >>> mp.pretty = False
+    >>> mpf(0.6)
+    mpf('0.59999999999999998')
+
+The number of digits with which numbers are printed by default is determined by the working precision. To specify the number of digits to show without changing the working precision, use :func:`mpmath.nstr` and :func:`mpmath.nprint`:
+
+    >>> a = mpf(1) / 6
+    >>> a
+    mpf('0.16666666666666666')
+    >>> nstr(a, 8)
+    '0.16666667'
+    >>> nprint(a, 8)
+    0.16666667
+    >>> nstr(a, 50)
+    '0.16666666666666665741480812812369549646973609924316'
diff --git a/lib/sympy/doc/src/modules/mpmath/calculus/approximation.txt b/lib/sympy/doc/src/modules/mpmath/calculus/approximation.txt
new file mode 100644
--- /dev/null
+++ b/lib/sympy/doc/src/modules/mpmath/calculus/approximation.txt
@@ -0,0 +1,23 @@
+Function approximation
+----------------------
+
+Taylor series (``taylor``)
+..........................
+
+.. autofunction:: mpmath.taylor
+
+Pade approximation (``pade``)
+.............................
+
+.. autofunction:: mpmath.pade
+
+Chebyshev approximation (``chebyfit``)
+......................................
+
+.. autofunction:: mpmath.chebyfit
+
+Fourier series (``fourier``, ``fourierval``)
+............................................
+
+.. autofunction:: mpmath.fourier
+.. autofunction:: mpmath.fourierval
diff --git a/lib/sympy/doc/src/modules/mpmath/calculus/differentiation.txt b/lib/sympy/doc/src/modules/mpmath/calculus/differentiation.txt
new file mode 100644
--- /dev/null
+++ b/lib/sympy/doc/src/modules/mpmath/calculus/differentiation.txt
@@ -0,0 +1,19 @@
+Differentiation
+---------------
+
+Numerical derivatives (``diff``, ``diffs``)
+...........................................
+
+.. autofunction:: mpmath.diff
+.. autofunction:: mpmath.diffs
+
+Composition of derivatives (``diffs_prod``, ``diffs_exp``)
+..........................................................
+
+.. autofunction:: mpmath.diffs_prod
+.. autofunction:: mpmath.diffs_exp
+
+Fractional derivatives / differintegration (``differint``)
+............................................................
+
+.. autofunction:: mpmath.differint
diff --git a/lib/sympy/doc/src/modules/mpmath/calculus/index.txt b/lib/sympy/doc/src/modules/mpmath/calculus/index.txt
new file mode 100644
--- /dev/null
+++ b/lib/sympy/doc/src/modules/mpmath/calculus/index.txt
@@ -0,0 +1,13 @@
+Numerical calculus
+==================
+
+.. toctree::
+   :maxdepth: 2
+
+   polynomials.txt
+   optimization.txt
+   sums_limits.txt
+   differentiation.txt
+   integration.txt
+   odes.txt
+   approximation.txt
diff --git a/lib/sympy/doc/src/modules/mpmath/calculus/integration.txt b/lib/sympy/doc/src/modules/mpmath/calculus/integration.txt
new file mode 100644
--- /dev/null
+++ b/lib/sympy/doc/src/modules/mpmath/calculus/integration.txt
@@ -0,0 +1,31 @@
+Numerical integration (quadrature)
+----------------------------------
+
+Standard quadrature (``quad``)
+..............................
+
+.. autofunction:: mpmath.quad
+
+Oscillatory quadrature (``quadosc``)
+....................................
+
+.. autofunction:: mpmath.quadosc
+
+Quadrature rules
+................
+
+.. autoclass:: mpmath.calculus.quadrature.QuadratureRule
+   :members:
+
+Tanh-sinh rule
+~~~~~~~~~~~~~~
+
+.. autoclass:: mpmath.calculus.quadrature.TanhSinh
+   :members:
+
+
+Gauss-Legendre rule
+~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: mpmath.calculus.quadrature.GaussLegendre
+   :members:
diff --git a/lib/sympy/doc/src/modules/mpmath/calculus/odes.txt b/lib/sympy/doc/src/modules/mpmath/calculus/odes.txt
new file mode 100644
--- /dev/null
+++ b/lib/sympy/doc/src/modules/mpmath/calculus/odes.txt
@@ -0,0 +1,7 @@
+Ordinary differential equations
+-------------------------------
+
+Solving the ODE initial value problem (``odefun``)
+..................................................
+
+.. autofunction:: mpmath.odefun
diff --git a/lib/sympy/doc/src/modules/mpmath/calculus/optimization.txt b/lib/sympy/doc/src/modules/mpmath/calculus/optimization.txt
new file mode 100644
--- /dev/null
+++ b/lib/sympy/doc/src/modules/mpmath/calculus/optimization.txt
@@ -0,0 +1,29 @@
+Root-finding and optimization
+-----------------------------
+
+Root-finding (``findroot``)
+...........................
+
+.. autofunction:: mpmath.findroot(f, x0, solver=Secant, tol=None, verbose=False, verify=True, **kwargs)
+
+Solvers
+^^^^^^^
+
+.. autoclass:: mpmath.calculus.optimization.Secant
+.. autoclass:: mpmath.calculus.optimization.Newton
+.. autoclass:: mpmath.calculus.optimization.MNewton
+.. autoclass:: mpmath.calculus.optimization.Halley
+.. autoclass:: mpmath.calculus.optimization.Muller
+.. autoclass:: mpmath.calculus.optimization.Bisection
+.. autoclass:: mpmath.calculus.optimization.Illinois
+.. autoclass:: mpmath.calculus.optimization.Pegasus
+.. autoclass:: mpmath.calculus.optimization.Anderson
+.. autoclass:: mpmath.calculus.optimization.Ridder
+.. autoclass:: mpmath.calculus.optimization.ANewton
+.. autoclass:: mpmath.calculus.optimization.MDNewton
+
+
+.. Minimization and maximization (``findmin``, ``findmax``)
+.. ........................................................
+
+.. (To be added.)
diff --git a/lib/sympy/doc/src/modules/mpmath/calculus/polynomials.txt b/lib/sympy/doc/src/modules/mpmath/calculus/polynomials.txt
new file mode 100644
--- /dev/null
+++ b/lib/sympy/doc/src/modules/mpmath/calculus/polynomials.txt
@@ -0,0 +1,15 @@
+Polynomials
+-----------
+
+See also :func:`taylor` and :func:`chebyfit` for
+approximation of functions by polynomials.
+
+Polynomial evaluation (``polyval``)
+...................................
+
+.. autofunction:: mpmath.polyval
+
+Polynomial roots (``polyroots``)
+................................
+
+.. autofunction:: mpmath.polyroots
diff --git a/lib/sympy/doc/src/modules/mpmath/calculus/sums_limits.txt b/lib/sympy/doc/src/modules/mpmath/calculus/sums_limits.txt
new file mode 100644
--- /dev/null
+++ b/lib/sympy/doc/src/modules/mpmath/calculus/sums_limits.txt
@@ -0,0 +1,58 @@
+Sums, products, limits and extrapolation
+----------------------------------------
+
+The functions listed here permit approximation of infinite
+sums, products, and other sequence limits.
+Use :func:`mpmath.fsum` and :func:`mpmath.fprod`
+for summation and multiplication of finite sequences.
+
+Summation
+..........................................
+
+:func:`nsum`

From noreply at buildbot.pypy.org  Sat Oct  8 14:40:19 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  8 Oct 2011 14:40:19 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Hack hack hack until the
 translator is sure that the collector thread
Message-ID: <20111008124019.7237582A87@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r47876:49ab73172117
Date: 2011-10-08 14:39 +0200
http://bitbucket.org/pypy/pypy/changeset/49ab73172117/

Log:	Hack hack hack until the translator is sure that the collector
	thread doesn't do anything with exceptions. This lets us keep the
	global exception state non-thread-local.

diff --git a/pypy/module/thread/ll_thread.py b/pypy/module/thread/ll_thread.py
--- a/pypy/module/thread/ll_thread.py
+++ b/pypy/module/thread/ll_thread.py
@@ -39,6 +39,9 @@
     return rffi.cast(rffi.LONG, ident)
 
 CALLBACK = lltype.Ptr(lltype.FuncType([], lltype.Void))
+c_thread_start_nowrapper = llexternal('RPyThreadStart', [CALLBACK], rffi.LONG,
+                                      _callable=_emulated_start_new_thread,
+                                      _nowrapper=True)
 c_thread_start = llexternal('RPyThreadStart', [CALLBACK], rffi.LONG,
                             _callable=_emulated_start_new_thread,
                             threadsafe=True)  # release the GIL, but most
diff --git a/pypy/rlib/debug.py b/pypy/rlib/debug.py
--- a/pypy/rlib/debug.py
+++ b/pypy/rlib/debug.py
@@ -210,7 +210,7 @@
         c_pythonfunction = hop.inputconst(lltype.Void, pythonfunction)
         args_v = [hop.inputarg(hop.args_r[i], arg=i)
                   for i in range(2, hop.nb_args)]
-        hop.exception_is_here()
+        hop.exception_cannot_occur()
         return hop.genop('debug_llinterpcall', [c_pythonfunction] + args_v,
                          resulttype=RESTYPE)
 
diff --git a/pypy/rpython/lltypesystem/lloperation.py b/pypy/rpython/lltypesystem/lloperation.py
--- a/pypy/rpython/lltypesystem/lloperation.py
+++ b/pypy/rpython/lltypesystem/lloperation.py
@@ -530,8 +530,7 @@
     'debug_flush':          LLOp(canrun=True),
     'debug_assert':         LLOp(tryfold=True),
     'debug_fatalerror':     LLOp(),
-    'debug_llinterpcall':   LLOp(canraise=(Exception,)),
-                                    # Python func call 'res=arg[0](*arg[1:])'
+    'debug_llinterpcall':   LLOp(), # Python func call 'res=arg[0](*arg[1:])'
                                     # in backends, abort() or whatever is fine
     'debug_start_traceback':   LLOp(),
     'debug_record_traceback':  LLOp(),
diff --git a/pypy/rpython/memory/gc/base.py b/pypy/rpython/memory/gc/base.py
--- a/pypy/rpython/memory/gc/base.py
+++ b/pypy/rpython/memory/gc/base.py
@@ -231,7 +231,7 @@
                     j += 1
                 item += itemlength
                 length -= 1
-        if self.has_custom_trace(typeid):
+        if self.has_custom_trace(typeid) and 0: # XXX XXX temporarily disabled
             generator = self.get_custom_trace(typeid)
             item = llmemory.NULL
             while True:
diff --git a/pypy/rpython/memory/gc/concurrentms.py b/pypy/rpython/memory/gc/concurrentms.py
--- a/pypy/rpython/memory/gc/concurrentms.py
+++ b/pypy/rpython/memory/gc/concurrentms.py
@@ -1,6 +1,7 @@
 import time, sys
 from pypy.rpython.lltypesystem import lltype, llmemory, llarena, llgroup, rffi
 from pypy.rpython.lltypesystem.llmemory import raw_malloc_usage
+from pypy.rpython.annlowlevel import llhelper
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
 from pypy.rlib.objectmodel import we_are_translated, running_on_llinterp
 from pypy.rlib.debug import ll_assert
@@ -127,6 +128,7 @@
                     print 'Crash!', e.__class__.__name__, e
                     self._exc_info = sys.exc_info()
         #
+        collector_start._should_never_raise_ = True
         self.collector_start = collector_start
         #
         #self.mutex_lock = ...built in setup()
@@ -150,8 +152,9 @@
         self.acquire(self.finished_lock)
         self.acquire(self.ready_to_start_lock)
         #
-        self.collector_ident = ll_thread.start_new_thread(
-            self.collector_start, ())
+        self.collector_ident = ll_thread.c_thread_start_nowrapper(
+            llhelper(ll_thread.CALLBACK, self.collector_start))
+        assert self.collector_ident != -1
 
     def _teardown(self):
         "Stop the collector thread after tests have run."
@@ -542,7 +545,12 @@
 
     def collector_run(self):
         """Main function of the collector's thread."""
-        while True:
+        #
+        # hack: make it an infinite loop, but in a way that the annotator
+        # doesn't notice.  It prevents the caller from ending automatically
+        # in a "raise AssertionError", annoyingly, because we don't want
+        # any exception in this thread
+        while self.collection_running < 99:
             #
             # Wait for the lock to be released
             self.acquire(self.ready_to_start_lock)
@@ -601,7 +609,7 @@
             # acquired the 'mutex_lock', so all reachable objects have
             # been marked.
             if not self.gray_objects.non_empty():
-                return
+                break
 
     def _collect_mark(self):
         current_mark = self.current_mark
diff --git a/pypy/rpython/memory/support.py b/pypy/rpython/memory/support.py
--- a/pypy/rpython/memory/support.py
+++ b/pypy/rpython/memory/support.py
@@ -1,7 +1,7 @@
-from pypy.rpython.lltypesystem import lltype, llmemory
+from pypy.rpython.lltypesystem import lltype, llmemory, llarena
 from pypy.rlib.objectmodel import free_non_gc_object, we_are_translated
 from pypy.rlib.rarithmetic import r_uint, LONG_BIT
-from pypy.rlib.debug import ll_assert
+from pypy.rlib.debug import ll_assert, fatalerror
 from pypy.tool.identity_dict import identity_dict
 
 
@@ -39,10 +39,17 @@
             if not self.free_list:
                 # we zero-initialize the chunks to make the translation
                 # backends happy, but we don't need to do it at run-time.
-                zero = not we_are_translated()
-                return lltype.malloc(CHUNK, flavor="raw", zero=zero,
-                                     track_allocation=False)
-                
+                if we_are_translated():
+                    zero = 0
+                else:
+                    zero = 2
+                size = llmemory.raw_malloc_usage(llmemory.sizeof(CHUNK))
+                addr = llarena.arena_malloc(size, zero)
+                if not addr:
+                    fatalerror("out of memory in GC support code")
+                llarena.arena_reserve(addr, llmemory.sizeof(CHUNK))
+                return llmemory.cast_adr_to_ptr(addr, lltype.Ptr(CHUNK))
+
             result = self.free_list
             self.free_list = result.next
             return result
diff --git a/pypy/translator/backendopt/canraise.py b/pypy/translator/backendopt/canraise.py
--- a/pypy/translator/backendopt/canraise.py
+++ b/pypy/translator/backendopt/canraise.py
@@ -12,20 +12,31 @@
 class RaiseAnalyzer(graphanalyze.BoolGraphAnalyzer):
     def analyze_simple_operation(self, op, graphinfo):
         try:
-            return bool(LL_OPERATIONS[op.opname].canraise)
+            if LL_OPERATIONS[op.opname].canraise:
+                self.reason = op
+                return True
+            return False
         except KeyError:
             log.WARNING("Unknown operation: %s" % op.opname)
+            self.reason = op
             return True
 
     def analyze_external_call(self, op, seen=None):
         fnobj = get_funcobj(op.args[0].value)
-        return getattr(fnobj, 'canraise', True)
+        if getattr(fnobj, 'canraise', True):
+            self.reason = 'external call:', fnobj
+            return True
+        return False
 
     def analyze_external_method(self, op, TYPE, meth):
         assert op.opname == 'oosend'
-        return getattr(meth, '_can_raise', True)
+        if getattr(meth, '_can_raise', True):
+            self.reason = 'external method:', meth
+            return True
+        return False
 
     def analyze_exceptblock(self, block, seen=None):
+        self.reason = "except block:", block
         return True
 
     # backward compatible interface
diff --git a/pypy/translator/backendopt/graphanalyze.py b/pypy/translator/backendopt/graphanalyze.py
--- a/pypy/translator/backendopt/graphanalyze.py
+++ b/pypy/translator/backendopt/graphanalyze.py
@@ -4,6 +4,7 @@
 
 class GraphAnalyzer(object):
     verbose = False
+    reason = None
 
     def __init__(self, translator):
         self.translator = translator
@@ -86,6 +87,7 @@
             if graphs is None:
                 if self.verbose:
                     print '\t%s to unknown' % (op,)
+                self.reason = op
                 return self.top_result()
             x = self.analyze_indirect_call(graphs, seen)
             if self.verbose and x:
diff --git a/pypy/translator/exceptiontransform.py b/pypy/translator/exceptiontransform.py
--- a/pypy/translator/exceptiontransform.py
+++ b/pypy/translator/exceptiontransform.py
@@ -198,8 +198,19 @@
             assert self.same_obj(self.exc_data_ptr, graph.exceptiontransformed)
             return
         else:
-            self.raise_analyzer.analyze_direct_call(graph)
+            # side-effect: perform the analysis of all graphs reachable from
+            # 'graph'
+            can_raise = self.raise_analyzer.analyze_direct_call(graph)
             graph.exceptiontransformed = self.exc_data_ptr
+            func = getattr(graph, 'func', None)
+            if getattr(func, '_should_never_raise_', False):
+                if not can_raise:
+                    return
+                ra = canraise.RaiseAnalyzer(self.translator)
+                ra.analyze_direct_call(graph)
+                raise Exception("%r is marked as _should_never_raise_, "
+                                "but is found to raise.  Reason: %r" % (
+                    graph, ra.reason))
 
         join_blocks(graph)
         # collect the blocks before changing them

From noreply at buildbot.pypy.org  Sat Oct  8 17:00:47 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  8 Oct 2011 17:00:47 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Refactor the header of
 objects: use a single 'Signed' field.
Message-ID: <20111008150047.6520882A87@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r47877:e3a93859d337
Date: 2011-10-08 16:46 +0200
http://bitbucket.org/pypy/pypy/changeset/e3a93859d337/

Log:	Refactor the header of objects: use a single 'Signed' field.
	Prompted to avoid C-level issues: reading a "void*" and then writing
	into a "char" that lives at the same address is supposed to be
	invalid C.

diff --git a/pypy/rpython/lltypesystem/llgroup.py b/pypy/rpython/lltypesystem/llgroup.py
--- a/pypy/rpython/lltypesystem/llgroup.py
+++ b/pypy/rpython/lltypesystem/llgroup.py
@@ -159,3 +159,62 @@
             return self.rest != other.rest
         else:
             return NotImplemented
+
+
+class HighCombinedSymbolic(llmemory.Symbolic):
+    """Same as CombinedSymbolic, but the symbolic half-word is in the high
+    half, and the flags are in the low part.
+    """
+    __slots__ = ['hipart', 'rest']
+    MASK = ~((1<<HALFSHIFT)-1)     # 0xFFFF0000 or 0xFFFFFFFF000000000
+
+    def annotation(self):
+        from pypy.annotation import model
+        return model.SomeInteger()
+
+    def lltype(self):
+        return lltype.Signed
+
+    def __init__(self, hipart, rest):
+        assert (rest & HighCombinedSymbolic.MASK) == 0
+        self.hipart = hipart
+        self.rest = rest
+
+    def __repr__(self):
+        return '<HighCombinedSymbolic %r|%s>' % (self.hipart, self.rest)
+
+    def __nonzero__(self):
+        return True
+
+    def __and__(self, other):
+        if (other & HighCombinedSymbolic.MASK) == 0:
+            return self.rest & other
+        if (other & HighCombinedSymbolic.MASK) == HighCombinedSymbolic.MASK:
+            return HighCombinedSymbolic(self.hipart, self.rest & other)
+        raise Exception("other=0x%x" % other)
+
+    def __or__(self, other):
+        assert (other & HighCombinedSymbolic.MASK) == 0
+        return HighCombinedSymbolic(self.hipart, self.rest | other)
+
+    def __add__(self, other):
+        assert (other & HighCombinedSymbolic.MASK) == 0
+        return HighCombinedSymbolic(self.hipart, self.rest + other)
+
+    def __sub__(self, other):
+        assert (other & HighCombinedSymbolic.MASK) == 0
+        return HighCombinedSymbolic(self.hipart, self.rest - other)
+
+    def __eq__(self, other):
+        if (isinstance(other, HighCombinedSymbolic) and
+            self.hipart is other.hipart):
+            return self.rest == other.rest
+        else:
+            return NotImplemented
+
+    def __ne__(self, other):
+        if (isinstance(other, HighCombinedSymbolic) and
+            self.hipart is other.hipart):
+            return self.rest != other.rest
+        else:
+            return NotImplemented
diff --git a/pypy/rpython/lltypesystem/llmemory.py b/pypy/rpython/lltypesystem/llmemory.py
--- a/pypy/rpython/lltypesystem/llmemory.py
+++ b/pypy/rpython/lltypesystem/llmemory.py
@@ -532,12 +532,28 @@
         return self.adr != cast_int_to_adr(other)
     def __nonzero__(self):
         return bool(self.adr)
+    def __and__(self, other):
+        assert type(other) is int
+        if 0 <= other <= 3:
+            return 0
+        return MultipleOf4()
     def __repr__(self):
         try:
             return '<AddressAsInt %s>' % (self.adr.ptr,)
         except AttributeError:
             return '<AddressAsInt at 0x%x>' % (uid(self),)
 
+class MultipleOf4(Symbolic):
+    # support for "i & 0xFF == <odd value>", which is always False if i is
+    # an AddressAsInt (at least assuming the address is of some word-aligned
+    # location).
+    def __eq__(self, other):
+        assert (other & 3) != 0
+        return False
+    def __ne__(self, other):
+        assert (other & 3) != 0
+        return True
+
 # ____________________________________________________________
 
 class NullAddressError(Exception):
diff --git a/pypy/rpython/lltypesystem/lloperation.py b/pypy/rpython/lltypesystem/lloperation.py
--- a/pypy/rpython/lltypesystem/lloperation.py
+++ b/pypy/rpython/lltypesystem/lloperation.py
@@ -420,6 +420,8 @@
     'is_group_member_nonzero':LLOp(canfold=True),
     'extract_ushort':       LLOp(canfold=True),
     'combine_ushort':       LLOp(canfold=True),
+    'extract_high_ushort':  LLOp(canfold=True),
+    'combine_high_ushort':  LLOp(canfold=True),
     'gc_gettypeptr_group':  LLOp(canfold=True),
     'get_member_index':     LLOp(canfold=True),
 
diff --git a/pypy/rpython/lltypesystem/opimpl.py b/pypy/rpython/lltypesystem/opimpl.py
--- a/pypy/rpython/lltypesystem/opimpl.py
+++ b/pypy/rpython/lltypesystem/opimpl.py
@@ -572,6 +572,15 @@
     from pypy.rpython.lltypesystem import llgroup
     return llgroup.CombinedSymbolic(ushort, rest)
 
+def op_extract_high_ushort(combinedoffset):
+    from pypy.rpython.lltypesystem import llgroup
+    assert isinstance(combinedoffset, llgroup.HighCombinedSymbolic)
+    return combinedoffset.hipart
+
+def op_combine_high_ushort(ushort, rest):
+    from pypy.rpython.lltypesystem import llgroup
+    return llgroup.HighCombinedSymbolic(ushort, rest)
+
 def op_gc_gettypeptr_group(TYPE, obj, grpptr, skipoffset, vtableinfo):
     HDR            = vtableinfo[0]
     size_gc_header = vtableinfo[1]
diff --git a/pypy/rpython/memory/gc/concurrentms.py b/pypy/rpython/memory/gc/concurrentms.py
--- a/pypy/rpython/memory/gc/concurrentms.py
+++ b/pypy/rpython/memory/gc/concurrentms.py
@@ -1,6 +1,7 @@
 import time, sys
 from pypy.rpython.lltypesystem import lltype, llmemory, llarena, llgroup, rffi
 from pypy.rpython.lltypesystem.llmemory import raw_malloc_usage
+from pypy.rpython.lltypesystem.lloperation import llop
 from pypy.rpython.annlowlevel import llhelper
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
 from pypy.rlib.objectmodel import we_are_translated, running_on_llinterp
@@ -24,24 +25,20 @@
 #
 
 WORD = LONG_BIT // 8
-NULL = llmemory.NULL
 WORD_POWER_2 = {32: 2, 64: 3}[LONG_BIT]
 assert 1 << WORD_POWER_2 == WORD
-size_of_addr = llmemory.sizeof(llmemory.Address)
-MAXIMUM_SIZE = sys.maxint - (2*WORD-1)
+MAXIMUM_SIZE = sys.maxint - (3*WORD-1)
 
-# XXX assumes little-endian machines for now: the byte at offset 0 in
-# the object is either a mark byte (equal to an odd value), or if the
-# location is free, it is the low byte of a pointer to the next free
-# location (and then it is an even value, by pointer alignment).
-assert sys.byteorder == 'little'
 
-
-MARK_VALUE_1      = 'M'     #  77, 0x4D
-MARK_VALUE_2      = 'k'     # 107, 0x6B
-MARK_VALUE_STATIC = 'S'     #  83, 0x53
-
-FL_WITHHASH = 0x01
+# Objects start with an integer 'tid', which is decomposed as follows.
+# Lowest byte: one of the the following values (which are all odd, so
+# let us know if the 'tid' is valid or is just a word-aligned address):
+MARK_VALUE_1      = 0x4D    # 'M', 77
+MARK_VALUE_2      = 0x6B    # 'k', 107
+MARK_VALUE_STATIC = 0x53    # 'S', 83
+# Next lower byte: a combination of flags.
+FL_WITHHASH = 0x0100
+# And the high half of the word contains the numeric typeid.
 
 
 class MostlyConcurrentMarkSweepGC(GCBase):
@@ -53,11 +50,12 @@
     malloc_zero_filled = True
     #gcflag_extra = GCFLAG_FINALIZATION_ORDERING
 
-    HDR = lltype.Struct('header', ('mark', lltype.Char),  # MARK_VALUE_{1,2}
-                                  ('flags', lltype.Char),
-                                  ('typeid16', llgroup.HALFWORD))
-    typeid_is_in_field = 'typeid16'
-    withhash_flag_is_in_field = 'flags', FL_WITHHASH
+    HDR = lltype.Struct('header', ('tid', lltype.Signed))
+    HDRPTR = lltype.Ptr(HDR)
+    HDRSIZE = llmemory.sizeof(HDR)
+    NULL = lltype.nullptr(HDR)
+    typeid_is_in_field = 'tid'
+    withhash_flag_is_in_field = 'tid', FL_WITHHASH
 
     TRANSLATION_PARAMS = {'page_size': 4096,
                           'small_request_threshold': 35*WORD,
@@ -72,7 +70,7 @@
         assert small_request_threshold % WORD == 0
         self.small_request_threshold = small_request_threshold
         self.page_size = page_size
-        self.free_pages = NULL
+        self.free_pages = lltype.nullptr(self.HDR)
         self.pagelists_length = small_request_threshold // WORD + 1
         #
         # The following are arrays of 36 linked lists: the linked lists
@@ -80,7 +78,7 @@
         # size  1 * WORD  to  35 * WORD,  and the linked list at index 0
         # is a list of all larger objects.
         def list_of_addresses_per_small_size():
-            return lltype.malloc(rffi.CArray(llmemory.Address),
+            return lltype.malloc(rffi.CArray(self.HDRPTR),
                                  self.pagelists_length, flavor='raw',
                                  zero=True, immortal=True)
         # 1-35: a linked list of all pages; 0: a linked list of all larger objs
@@ -171,14 +169,16 @@
             del self.ready_to_start_lock, self.finished_lock
 
     def get_type_id(self, obj):
-        return self.header(obj).typeid16
+        tid = self.header(obj).tid
+        return llop.extract_high_ushort(llgroup.HALFWORD, tid)
+
+    def combine(self, typeid16, mark, flags):
+        return llop.combine_high_ushort(lltype.Signed, typeid16, mark | flags)
 
     def init_gc_object_immortal(self, addr, typeid, flags=0):
         # 'flags' is ignored here
         hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR))
-        hdr.typeid16 = typeid
-        hdr.mark = MARK_VALUE_STATIC
-        hdr.flags = '\x00'
+        hdr.tid = self.combine(typeid, MARK_VALUE_STATIC, 0)
 
     def malloc_fixedsize_clear(self, typeid, size,
                                needs_finalizer=False, contains_weakptr=False):
@@ -193,17 +193,11 @@
             #
             n = rawtotalsize >> WORD_POWER_2
             result = self.free_lists[n]
-            if result != llmemory.NULL:
-                self.free_lists[n] = result.address[0]
-                #
-                llarena.arena_reset(result, size_of_addr, 0)
-                llarena.arena_reserve(result, totalsize)
-                hdr = llmemory.cast_adr_to_ptr(result, lltype.Ptr(self.HDR))
-                hdr.typeid16 = typeid
-                hdr.mark = self.current_mark
-                hdr.flags = '\x00'
-                #
-                obj = result + size_gc_header
+            if result != self.NULL:
+                self.free_lists[n] = self.cast_int_to_hdrptr(result.tid)
+                obj = self.grow_reservation(result, totalsize)
+                hdr = self.header(obj)
+                hdr.tid = self.combine(typeid, self.current_mark, 0)
                 return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
                 #
         return self._malloc_slowpath(typeid, size)
@@ -237,17 +231,11 @@
             #
             n = rawtotalsize >> WORD_POWER_2
             result = self.free_lists[n]
-            if result != llmemory.NULL:
-                self.free_lists[n] = result.address[0]
-                #
-                llarena.arena_reset(result, size_of_addr, 0)
-                llarena.arena_reserve(result, totalsize)
-                hdr = llmemory.cast_adr_to_ptr(result, lltype.Ptr(self.HDR))
-                hdr.typeid16 = typeid
-                hdr.mark = self.current_mark
-                hdr.flags = '\x00'
-                #
-                obj = result + size_gc_header
+            if result != self.NULL:
+                self.free_lists[n] = self.cast_int_to_hdrptr(result.tid)
+                obj = self.grow_reservation(result, totalsize)
+                hdr = self.header(obj)
+                hdr.tid = self.combine(typeid, self.current_mark, 0)
                 (obj + offset_to_length).signed[0] = length
                 return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
         #
@@ -271,15 +259,15 @@
             # Case 1: we have run out of the free list corresponding to
             # the size.  Grab the next free page.
             newpage = self.free_pages
-            if newpage == llmemory.NULL:
+            if newpage == self.NULL:
                 self.allocate_next_arena()
                 newpage = self.free_pages
-            self.free_pages = newpage.address[0]
+            self.free_pages = self.cast_int_to_hdrptr(newpage.tid)
             #
             # Put the free page in the list 'nonfree_pages[n]'.  This is
             # a linked list chained through the first word of each page.
             n = rawtotalsize >> WORD_POWER_2
-            newpage.address[0] = self.nonfree_pages[n]
+            newpage.tid = self.cast_hdrptr_to_int(self.nonfree_pages[n])
             self.nonfree_pages[n] = newpage
             #
             # Initialize the free page to contain objects of the given
@@ -288,14 +276,18 @@
             head = self.free_lists[n]
             ll_assert(not head, "_malloc_slowpath: unexpected free_lists[n]")
             i = self.page_size - rawtotalsize
-            limit = rawtotalsize + raw_malloc_usage(size_of_addr)
+            limit = rawtotalsize + raw_malloc_usage(self.HDRSIZE)
+            newpageadr = llmemory.cast_ptr_to_adr(newpage)
+            newpageadr = llarena.getfakearenaaddress(newpageadr)
             while i >= limit:
-                llarena.arena_reserve(newpage + i, size_of_addr)
-                (newpage + i).address[0] = head
-                head = newpage + i
+                adr = newpageadr + i
+                llarena.arena_reserve(adr, self.HDRSIZE)
+                p = llmemory.cast_adr_to_ptr(adr, self.HDRPTR)
+                p.tid = self.cast_hdrptr_to_int(head)
+                head = p
                 i -= rawtotalsize
             self.free_lists[n] = head
-            result = head - rawtotalsize
+            result = newpageadr + i
             #
             # Done: all object locations are linked, apart from
             # 'result', which is the first object location in the page.
@@ -306,22 +298,21 @@
             #
         else:
             # Case 2: the object is too large, so allocate it directly
-            # with the system malloc().  XXX on 32-bit, we should prefer
-            # 64-bit alignment of the object
-            rawtotalsize += raw_malloc_usage(size_of_addr)
+            # with the system malloc().  xxx on 32-bit, we'll prefer 64-bit
+            # alignment of the object by always allocating an 8-bytes header
+            rawtotalsize += 8
             block = llarena.arena_malloc(rawtotalsize, 2)
             if not block:
                 raise MemoryError
-            llarena.arena_reserve(block, size_of_addr)
-            block.address[0] = self.nonfree_pages[0]
-            self.nonfree_pages[0] = block
-            result = block + size_of_addr
+            llarena.arena_reserve(block, self.HDRSIZE)
+            blockhdr = llmemory.cast_adr_to_ptr(block, self.HDRPTR)
+            blockhdr.tid = self.cast_hdrptr_to_int(self.nonfree_pages[0])
+            self.nonfree_pages[0] = blockhdr
+            result = block + 8
         #
         llarena.arena_reserve(result, totalsize)
-        hdr = llmemory.cast_adr_to_ptr(result, lltype.Ptr(self.HDR))
-        hdr.typeid16 = typeid
-        hdr.mark = self.current_mark
-        hdr.flags = '\x00'
+        hdr = llmemory.cast_adr_to_ptr(result, self.HDRPTR)
+        hdr.tid = self.combine(typeid, self.current_mark, 0)
         #
         obj = result + size_gc_header
         return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
@@ -362,19 +353,30 @@
         page = llarena.arena_malloc(self.page_size, 2)     # zero-filled
         if not page:
             raise MemoryError
-        llarena.arena_reserve(page, size_of_addr)
-        page.address[0] = NULL
+        llarena.arena_reserve(page, self.HDRSIZE)
+        page = llmemory.cast_adr_to_ptr(page, self.HDRPTR)
+        page.tid = 0
         self.free_pages = page
 
+    def grow_reservation(self, hdr, totalsize):
+        # Transform 'hdr', which used to point to just a HDR,
+        # into a pointer to a full object of size 'totalsize'.
+        # This is a no-op after translation.  Returns the
+        # address of the full object.
+        adr = llmemory.cast_ptr_to_adr(hdr)
+        adr = llarena.getfakearenaaddress(adr)
+        llarena.arena_reset(adr, self.HDRSIZE, 0)
+        llarena.arena_reserve(adr, totalsize)
+        return adr + llmemory.raw_malloc_usage(self.HDRSIZE)
 
     def write_barrier(self, newvalue, addr_struct):
-        mark = self.header(addr_struct).mark
+        mark = self.header(addr_struct).tid & 0xFF
         if mark != self.current_mark:
             self.force_scan(addr_struct)
 
     def writebarrier_before_copy(self, source_addr, dest_addr,
                                  source_start, dest_start, length):
-        mark = self.header(dest_addr).mark
+        mark = self.header(dest_addr).tid & 0xFF
         if mark != self.current_mark:
             self.force_scan(dest_addr)
         return True
@@ -383,7 +385,7 @@
         #
         def force_scan(obj):
             self.acquire(self.mutex_lock)
-            mark = self.header(obj).mark
+            mark = self.header(obj).tid & 0xFF
             if mark != self.current_mark:
                 #
                 if mark == MARK_VALUE_STATIC:
@@ -433,14 +435,16 @@
             # 'free_lists'.
             n = 1
             while n < self.pagelists_length:
-                if self.collect_tails[n] != NULL:
-                    self.collect_tails[n].address[0] = self.free_lists[n]
+                if self.collect_tails[n] != self.NULL:
+                    self.collect_tails[n].tid = self.cast_hdrptr_to_int(
+                        self.free_lists[n])
                     self.free_lists[n] = self.collect_heads[n]
                 n += 1
             #
             # Do the same with 'collect_heads[0]/collect_tails[0]'.
-            if self.collect_tails[0] != NULL:
-                self.collect_tails[0].address[0] = self.nonfree_pages[0]
+            if self.collect_tails[0] != self.NULL:
+                self.collect_tails[0].tid = self.cast_hdrptr_to_int(
+                    self.nonfree_pages[0])
                 self.nonfree_pages[0] = self.collect_heads[0]
             #
             if self.DEBUG:
@@ -492,7 +496,7 @@
         while n < self.pagelists_length:
             self.collect_pages[n] = self.nonfree_pages[n]
             n += 1
-        self.nonfree_pages[0] = NULL
+        self.nonfree_pages[0] = self.NULL
         #
         # Start the collector thread
         self.collection_running = 1
@@ -515,11 +519,12 @@
 
     def debug_check_list(self, page):
         try:
-            while page != llmemory.NULL:
-                # prevent constant-folding:
-                byte = ord(maybe_read_mark_byte(page))
-                ll_assert((byte & 3) == 0, "misaligned?")
-                page = page.address[0]
+            previous_page = self.NULL
+            while page != self.NULL:
+                # prevent constant-folding, and detects loops of length 1
+                ll_assert(page != previous_page, "loop!")
+                previous_page = page
+                page = self.cast_int_to_hdrptr(page.tid)
         except KeyboardInterrupt:
             ll_assert(False, "interrupted")
             raise
@@ -542,6 +547,14 @@
         exc, val, tb = self._exc_info
         raise exc, val, tb
 
+    def cast_int_to_hdrptr(self, tid):
+        return llmemory.cast_adr_to_ptr(llmemory.cast_int_to_adr(tid),
+                                        self.HDRPTR)
+
+    def cast_hdrptr_to_int(self, hdr):
+        return llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(hdr),
+                                        "symbolic")
+
 
     def collector_run(self):
         """Main function of the collector's thread."""
@@ -573,16 +586,16 @@
     def other_mark(self, mark):
         ll_assert(mark == MARK_VALUE_1 or mark == MARK_VALUE_2,
                   "bad mark value")
-        return chr(ord(mark) ^ (ord(MARK_VALUE_1) ^ ord(MARK_VALUE_2)))
+        return mark ^ (MARK_VALUE_1 ^ MARK_VALUE_2)
 
     def is_marked(self, obj, current_mark):
-        mark = self.header(obj).mark
+        mark = self.header(obj).tid & 0xFF
         ll_assert(mark in (MARK_VALUE_1, MARK_VALUE_2, MARK_VALUE_STATIC),
                   "bad mark byte in object")
         return mark == current_mark
 
-    def set_mark(self, obj, current_mark):
-        self.header(obj).mark = current_mark
+    def set_mark(self, obj, newmark):
+        _set_mark(self.header(obj), newmark)
 
     def collector_mark(self):
         while True:
@@ -641,23 +654,25 @@
     def _collect_sweep_large_objects(self):
         block = self.collect_pages[0]
         nonmarked = self.other_mark(self.current_mark)
-        linked_list = NULL
-        first_block_in_linked_list = NULL
-        while block != llmemory.NULL:
-            nextblock = block.address[0]
-            hdr = block + size_of_addr
-            mark = maybe_read_mark_byte(hdr)
+        linked_list = self.NULL
+        first_block_in_linked_list = self.NULL
+        while block != self.NULL:
+            nextblock = self.cast_int_to_hdrptr(block.tid)
+            blockadr = llmemory.cast_ptr_to_adr(block)
+            blockadr = llarena.getfakearenaaddress(blockadr)
+            hdr = llmemory.cast_adr_to_ptr(blockadr + 8, self.HDRPTR)
+            mark = hdr.tid & 0xFF
             if mark == nonmarked:
                 # the object is still not marked.  Free it.
-                llarena.arena_free(block)
+                llarena.arena_free(blockadr)
                 #
             else:
                 # the object was marked: relink it
                 ll_assert(mark == self.current_mark,
                           "bad mark in large object")
-                block.address[0] = linked_list
+                block.tid = self.cast_hdrptr_to_int(linked_list)
                 linked_list = block
-                if first_block_in_linked_list == NULL:
+                if first_block_in_linked_list == self.NULL:
                     first_block_in_linked_list = block
             block = nextblock
         #
@@ -669,36 +684,42 @@
         # containing objects of fixed size 'object_size'.
         page = self.collect_pages[n]
         object_size = n << WORD_POWER_2
-        linked_list = NULL
-        first_loc_in_linked_list = NULL
+        linked_list = self.NULL
+        first_loc_in_linked_list = self.NULL
         nonmarked = self.other_mark(self.current_mark)
-        while page != llmemory.NULL:
+        while page != self.NULL:
             i = self.page_size - object_size
-            limit = raw_malloc_usage(size_of_addr)
+            limit = raw_malloc_usage(self.HDRSIZE)
+            pageadr = llmemory.cast_ptr_to_adr(page)
+            pageadr = llarena.getfakearenaaddress(pageadr)
             while i >= limit:
-                hdr = page + i
+                adr = pageadr + i
+                hdr = llmemory.cast_adr_to_ptr(adr, self.HDRPTR)
                 #
-                if maybe_read_mark_byte(hdr) == nonmarked:
+                if (hdr.tid & 0xFF) == nonmarked:
                     # the location contains really an object (and is not just
                     # part of a linked list of free locations), and moreover
                     # the object is still not marked.  Free it by inserting
                     # it into the linked list.
-                    llarena.arena_reset(hdr, object_size, 0)
-                    llarena.arena_reserve(hdr, size_of_addr)
-                    hdr.address[0] = linked_list
+                    llarena.arena_reset(adr, object_size, 0)
+                    llarena.arena_reserve(adr, self.HDRSIZE)
+                    hdr = llmemory.cast_adr_to_ptr(adr, self.HDRPTR)
+                    hdr.tid = self.cast_hdrptr_to_int(linked_list)
                     linked_list = hdr
-                    if first_loc_in_linked_list == NULL:
+                    if first_loc_in_linked_list == self.NULL:
                         first_loc_in_linked_list = hdr
                     # XXX detect when the whole page is freed again
                     #
                     # Clear the data, in prevision for the following
                     # malloc_fixedsize_clear().
-                    llarena.arena_reset(hdr + size_of_addr,
-                        object_size - raw_malloc_usage(size_of_addr), 2)
+                    size_of_int = raw_malloc_usage(
+                        llmemory.sizeof(lltype.Signed))
+                    llarena.arena_reset(adr + size_of_int,
+                                        object_size - size_of_int, 2)
                 #
                 i -= object_size
             #
-            page = page.address[0]
+            page = self.cast_int_to_hdrptr(page.tid)
         #
         self.collect_heads[n] = linked_list
         self.collect_tails[n] = first_loc_in_linked_list
@@ -706,8 +727,7 @@
 
     def identityhash(self, obj):
         obj = llmemory.cast_ptr_to_adr(obj)
-        hdr = self.header(obj)
-        if ord(hdr.flags) & FL_WITHHASH:
+        if self.header(obj).tid & FL_WITHHASH:
             obj += self.get_size(obj)
             return obj.signed[0]
         else:
@@ -715,22 +735,46 @@
 
 # ____________________________________________________________
 #
-# Hack to read the first byte of a location, which may be the
-# "mark" byte in an object or, if the location is free, the lowest
-# byte of the "next" pointer.
+# Hack to write the 'mark' or the 'flags' bytes of an object header
+# without overwriting the whole word.  Essential in the rare case where
+# the other thread might be concurrently writing the other byte.
 
-def emulate_read_mark_byte(addr):
+concurrent_setter_lock = ll_thread.allocate_lock()
+
+def emulate_set_mark(p, v):
     "NOT_RPYTHON"
-    try:
-        return addr.ptr.mark
-    except AttributeError:
-        return '\x00'
+    assert v in (MARK_VALUE_1, MARK_VALUE_2, MARK_VALUE_STATIC)
+    concurrent_setter_lock.acquire(True)
+    p.tid = (p.tid &~ 0xFF) | v
+    concurrent_setter_lock.release()
 
-eci = ExternalCompilationInfo(
-    post_include_bits = ["""
-        #define pypy_concurrentms_read_byte(addr)  (*(char*)(addr))
-    """])
-maybe_read_mark_byte = rffi.llexternal("pypy_concurrentms_read_byte",
-                                       [llmemory.Address], lltype.Char,
-                                       compilation_info=eci, _nowrapper=True,
-                                       _callable=emulate_read_mark_byte)
+def emulate_set_flags(p, v):
+    "NOT_RPYTHON"
+    assert (v & ~0xFF00) == 0
+    concurrent_setter_lock.acquire(True)
+    p.tid = (p.tid &~ 0xFF00) | v
+    concurrent_setter_lock.release()
+
+if sys.byteorder == 'little':
+    eci = ExternalCompilationInfo(
+        post_include_bits = ["""
+#define pypy_concurrentms_set_mark(p, v)   ((char*)p)[0] = v
+#define pypy_concurrentms_set_flags(p, v)  ((char*)p)[1] = v
+        """])
+elif sys.byteorder == 'big':
+    eci = ExternalCompilationInfo(
+        post_include_bits = [r"""
+#define pypy_concurrentms_set_mark(p, v)   ((char*)p)[sizeof(long)-1] = v
+#define pypy_concurrentms_set_flags(p, v)  ((char*)p)[sizeof(long)-2] = v
+        """])
+else:
+    raise NotImplementedError(sys.byteorder)
+
+_set_mark = rffi.llexternal("pypy_concurrentms_set_mark",
+                           [MostlyConcurrentMarkSweepGC.HDRPTR, lltype.Signed],
+                           lltype.Void, compilation_info=eci, _nowrapper=True,
+                           _callable=emulate_set_mark)
+_set_flags = rffi.llexternal("pypy_concurrentms_set_flags",
+                           [MostlyConcurrentMarkSweepGC.HDRPTR, lltype.Signed],
+                           lltype.Void, compilation_info=eci, _nowrapper=True,
+                           _callable=emulate_set_flags)

From noreply at buildbot.pypy.org  Sat Oct  8 17:00:48 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  8 Oct 2011 17:00:48 +0200 (CEST)
Subject: [pypy-commit] pypy default: Don't use the "python" command even if
 there one in case it
Message-ID: <20111008150048.A0D8A82A87@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r47878:46dc1cc20b33
Date: 2011-10-08 17:00 +0200
http://bitbucket.org/pypy/pypy/changeset/46dc1cc20b33/

Log:	Don't use the "python" command even if there one in case it points
	to python 3.

diff --git a/pypy/translator/c/genc.py b/pypy/translator/c/genc.py
--- a/pypy/translator/c/genc.py
+++ b/pypy/translator/c/genc.py
@@ -8,7 +8,7 @@
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
 from pypy.rpython.lltypesystem import lltype
 from pypy.tool.udir import udir
-from pypy.tool import isolate
+from pypy.tool import isolate, runsubprocess
 from pypy.translator.c.support import log, c_string_constant
 from pypy.rpython.typesystem import getfunctionptr
 from pypy.translator.c import gc
@@ -563,14 +563,19 @@
             else:
                 mk.definition('PYPY_MAIN_FUNCTION', "main")
 
-            if (py.path.local.sysfind('python') or
-                py.path.local.sysfind('python.exe')):
-                python = 'python '
-            elif sys.platform == 'win32':
+            if sys.platform == 'win32':
                 python = sys.executable.replace('\\', '/') + ' '
             else:
                 python = sys.executable + ' '
 
+            # Is there a command 'python' that runs python 2.5-2.7?
+            # If there is, then we can use it instead of sys.executable
+            returncode, stdout, stderr = runsubprocess.run_subprocess(
+                "python", "-V")
+            if (stdout.startswith('Python 2.') or
+                stderr.startswith('Python 2.')):
+                python = 'python '
+
             if self.translator.platform.name == 'msvc':
                 lblofiles = []
                 for cfile in mk.cfiles:

From noreply at buildbot.pypy.org  Sat Oct  8 17:06:07 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sat,  8 Oct 2011 17:06:07 +0200 (CEST)
Subject: [pypy-commit] pypy default: Print this to stderr, like CPython.
Message-ID: <20111008150607.9033182A87@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r47879:3fc8df6a35c0
Date: 2011-10-08 17:05 +0200
http://bitbucket.org/pypy/pypy/changeset/3fc8df6a35c0/

Log:	Print this to stderr, like CPython.

diff --git a/pypy/translator/goal/app_main.py b/pypy/translator/goal/app_main.py
--- a/pypy/translator/goal/app_main.py
+++ b/pypy/translator/goal/app_main.py
@@ -144,7 +144,7 @@
     print '  --jit off                  turn off the JIT'
 
 def print_version(*args):
-    print "Python", sys.version
+    print >> sys.stderr, "Python", sys.version
     raise SystemExit
 
 def set_jit_option(options, jitparam, *args):

From noreply at buildbot.pypy.org  Sat Oct  8 18:51:04 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Sat,  8 Oct 2011 18:51:04 +0200 (CEST)
Subject: [pypy-commit] pypy default: Close underlaying sockets in few
 places. I'm sure this list is not-exhaustive, 
Message-ID: <20111008165104.4C64D82A87@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r47880:ab84e543fe0d
Date: 2011-10-08 18:50 +0200
http://bitbucket.org/pypy/pypy/changeset/ab84e543fe0d/

Log:	Close underlaying sockets in few places. I'm sure this list is not-
	exhaustive, but I'm a bit unsure what to do :/

diff --git a/lib-python/modified-2.7/httplib.py b/lib-python/modified-2.7/httplib.py
new file mode 100644
--- /dev/null
+++ b/lib-python/modified-2.7/httplib.py
@@ -0,0 +1,1377 @@
+"""HTTP/1.1 client library
+
+<intro stuff goes here>
+<other stuff, too>
+
+HTTPConnection goes through a number of "states", which define when a client
+may legally make another request or fetch the response for a particular
+request. This diagram details these state transitions:
+
+    (null)
+      |
+      | HTTPConnection()
+      v
+    Idle
+      |
+      | putrequest()
+      v
+    Request-started
+      |
+      | ( putheader() )*  endheaders()
+      v
+    Request-sent
+      |
+      | response = getresponse()
+      v
+    Unread-response   [Response-headers-read]
+      |\____________________
+      |                     |
+      | response.read()     | putrequest()
+      v                     v
+    Idle                  Req-started-unread-response
+                     ______/|
+                   /        |
+   response.read() |        | ( putheader() )*  endheaders()
+                   v        v
+       Request-started    Req-sent-unread-response
+                            |
+                            | response.read()
+                            v
+                          Request-sent
+
+This diagram presents the following rules:
+  -- a second request may not be started until {response-headers-read}
+  -- a response [object] cannot be retrieved until {request-sent}
+  -- there is no differentiation between an unread response body and a
+     partially read response body
+
+Note: this enforcement is applied by the HTTPConnection class. The
+      HTTPResponse class does not enforce this state machine, which
+      implies sophisticated clients may accelerate the request/response
+      pipeline. Caution should be taken, though: accelerating the states
+      beyond the above pattern may imply knowledge of the server's
+      connection-close behavior for certain requests. For example, it
+      is impossible to tell whether the server will close the connection
+      UNTIL the response headers have been read; this means that further
+      requests cannot be placed into the pipeline until it is known that
+      the server will NOT be closing the connection.
+
+Logical State                  __state            __response
+-------------                  -------            ----------
+Idle                           _CS_IDLE           None
+Request-started                _CS_REQ_STARTED    None
+Request-sent                   _CS_REQ_SENT       None
+Unread-response                _CS_IDLE           <response_class>
+Req-started-unread-response    _CS_REQ_STARTED    <response_class>
+Req-sent-unread-response       _CS_REQ_SENT       <response_class>
+"""
+
+from array import array
+import os
+import socket
+from sys import py3kwarning
+from urlparse import urlsplit
+import warnings
+with warnings.catch_warnings():
+    if py3kwarning:
+        warnings.filterwarnings("ignore", ".*mimetools has been removed",
+                                DeprecationWarning)
+    import mimetools
+
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from StringIO import StringIO
+
+__all__ = ["HTTP", "HTTPResponse", "HTTPConnection",
+           "HTTPException", "NotConnected", "UnknownProtocol",
+           "UnknownTransferEncoding", "UnimplementedFileMode",
+           "IncompleteRead", "InvalidURL", "ImproperConnectionState",
+           "CannotSendRequest", "CannotSendHeader", "ResponseNotReady",
+           "BadStatusLine", "error", "responses"]
+
+HTTP_PORT = 80
+HTTPS_PORT = 443
+
+_UNKNOWN = 'UNKNOWN'
+
+# connection states
+_CS_IDLE = 'Idle'
+_CS_REQ_STARTED = 'Request-started'
+_CS_REQ_SENT = 'Request-sent'
+
+# status codes
+# informational
+CONTINUE = 100
+SWITCHING_PROTOCOLS = 101
+PROCESSING = 102
+
+# successful
+OK = 200
+CREATED = 201
+ACCEPTED = 202
+NON_AUTHORITATIVE_INFORMATION = 203
+NO_CONTENT = 204
+RESET_CONTENT = 205
+PARTIAL_CONTENT = 206
+MULTI_STATUS = 207
+IM_USED = 226
+
+# redirection
+MULTIPLE_CHOICES = 300
+MOVED_PERMANENTLY = 301
+FOUND = 302
+SEE_OTHER = 303
+NOT_MODIFIED = 304
+USE_PROXY = 305
+TEMPORARY_REDIRECT = 307
+
+# client error
+BAD_REQUEST = 400
+UNAUTHORIZED = 401
+PAYMENT_REQUIRED = 402
+FORBIDDEN = 403
+NOT_FOUND = 404
+METHOD_NOT_ALLOWED = 405
+NOT_ACCEPTABLE = 406
+PROXY_AUTHENTICATION_REQUIRED = 407
+REQUEST_TIMEOUT = 408
+CONFLICT = 409
+GONE = 410
+LENGTH_REQUIRED = 411
+PRECONDITION_FAILED = 412
+REQUEST_ENTITY_TOO_LARGE = 413
+REQUEST_URI_TOO_LONG = 414
+UNSUPPORTED_MEDIA_TYPE = 415
+REQUESTED_RANGE_NOT_SATISFIABLE = 416
+EXPECTATION_FAILED = 417
+UNPROCESSABLE_ENTITY = 422
+LOCKED = 423
+FAILED_DEPENDENCY = 424
+UPGRADE_REQUIRED = 426
+
+# server error
+INTERNAL_SERVER_ERROR = 500
+NOT_IMPLEMENTED = 501
+BAD_GATEWAY = 502
+SERVICE_UNAVAILABLE = 503
+GATEWAY_TIMEOUT = 504
+HTTP_VERSION_NOT_SUPPORTED = 505
+INSUFFICIENT_STORAGE = 507
+NOT_EXTENDED = 510
+
+# Mapping status codes to official W3C names
+responses = {
+    100: 'Continue',
+    101: 'Switching Protocols',
+
+    200: 'OK',
+    201: 'Created',
+    202: 'Accepted',
+    203: 'Non-Authoritative Information',
+    204: 'No Content',
+    205: 'Reset Content',
+    206: 'Partial Content',
+
+    300: 'Multiple Choices',
+    301: 'Moved Permanently',
+    302: 'Found',
+    303: 'See Other',
+    304: 'Not Modified',
+    305: 'Use Proxy',
+    306: '(Unused)',
+    307: 'Temporary Redirect',
+
+    400: 'Bad Request',
+    401: 'Unauthorized',
+    402: 'Payment Required',
+    403: 'Forbidden',
+    404: 'Not Found',
+    405: 'Method Not Allowed',
+    406: 'Not Acceptable',
+    407: 'Proxy Authentication Required',
+    408: 'Request Timeout',
+    409: 'Conflict',
+    410: 'Gone',
+    411: 'Length Required',
+    412: 'Precondition Failed',
+    413: 'Request Entity Too Large',
+    414: 'Request-URI Too Long',
+    415: 'Unsupported Media Type',
+    416: 'Requested Range Not Satisfiable',
+    417: 'Expectation Failed',
+
+    500: 'Internal Server Error',
+    501: 'Not Implemented',
+    502: 'Bad Gateway',
+    503: 'Service Unavailable',
+    504: 'Gateway Timeout',
+    505: 'HTTP Version Not Supported',
+}
+
+# maximal amount of data to read at one time in _safe_read
+MAXAMOUNT = 1048576
+
+class HTTPMessage(mimetools.Message):
+
+    def addheader(self, key, value):
+        """Add header for field key handling repeats."""
+        prev = self.dict.get(key)
+        if prev is None:
+            self.dict[key] = value
+        else:
+            combined = ", ".join((prev, value))
+            self.dict[key] = combined
+
+    def addcontinue(self, key, more):
+        """Add more field data from a continuation line."""
+        prev = self.dict[key]
+        self.dict[key] = prev + "\n " + more
+
+    def readheaders(self):
+        """Read header lines.
+
+        Read header lines up to the entirely blank line that terminates them.
+        The (normally blank) line that ends the headers is skipped, but not
+        included in the returned list.  If a non-header line ends the headers,
+        (which is an error), an attempt is made to backspace over it; it is
+        never included in the returned list.
+
+        The variable self.status is set to the empty string if all went well,
+        otherwise it is an error message.  The variable self.headers is a
+        completely uninterpreted list of lines contained in the header (so
+        printing them will reproduce the header exactly as it appears in the
+        file).
+
+        If multiple header fields with the same name occur, they are combined
+        according to the rules in RFC 2616 sec 4.2:
+
+        Appending each subsequent field-value to the first, each separated
+        by a comma. The order in which header fields with the same field-name
+        are received is significant to the interpretation of the combined
+        field value.
+        """
+        # XXX The implementation overrides the readheaders() method of
+        # rfc822.Message.  The base class design isn't amenable to
+        # customized behavior here so the method here is a copy of the
+        # base class code with a few small changes.
+
+        self.dict = {}
+        self.unixfrom = ''
+        self.headers = hlist = []
+        self.status = ''
+        headerseen = ""
+        firstline = 1
+        startofline = unread = tell = None
+        if hasattr(self.fp, 'unread'):
+            unread = self.fp.unread
+        elif self.seekable:
+            tell = self.fp.tell
+        while True:
+            if tell:
+                try:
+                    startofline = tell()
+                except IOError:
+                    startofline = tell = None
+                    self.seekable = 0
+            line = self.fp.readline()
+            if not line:
+                self.status = 'EOF in headers'
+                break
+            # Skip unix From name time lines
+            if firstline and line.startswith('From '):
+                self.unixfrom = self.unixfrom + line
+                continue
+            firstline = 0
+            if headerseen and line[0] in ' \t':
+                # XXX Not sure if continuation lines are handled properly
+                # for http and/or for repeating headers
+                # It's a continuation line.
+                hlist.append(line)
+                self.addcontinue(headerseen, line.strip())
+                continue
+            elif self.iscomment(line):
+                # It's a comment.  Ignore it.
+                continue
+            elif self.islast(line):
+                # Note! No pushback here!  The delimiter line gets eaten.
+                break
+            headerseen = self.isheader(line)
+            if headerseen:
+                # It's a legal header line, save it.
+                hlist.append(line)
+                self.addheader(headerseen, line[len(headerseen)+1:].strip())
+                continue
+            else:
+                # It's not a header line; throw it back and stop here.
+                if not self.dict:
+                    self.status = 'No headers'
+                else:
+                    self.status = 'Non-header line where header expected'
+                # Try to undo the read.
+                if unread:
+                    unread(line)
+                elif tell:
+                    self.fp.seek(startofline)
+                else:
+                    self.status = self.status + '; bad seek'
+                break
+
+class HTTPResponse:
+
+    # strict: If true, raise BadStatusLine if the status line can't be
+    # parsed as a valid HTTP/1.0 or 1.1 status line.  By default it is
+    # false because it prevents clients from talking to HTTP/0.9
+    # servers.  Note that a response with a sufficiently corrupted
+    # status line will look like an HTTP/0.9 response.
+
+    # See RFC 2616 sec 19.6 and RFC 1945 sec 6 for details.
+
+    def __init__(self, sock, debuglevel=0, strict=0, method=None, buffering=False):
+        if buffering:
+            # The caller won't be using any sock.recv() calls, so buffering
+            # is fine and recommended for performance.
+            self.fp = sock.makefile('rb')
+        else:
+            # The buffer size is specified as zero, because the headers of
+            # the response are read with readline().  If the reads were
+            # buffered the readline() calls could consume some of the
+            # response, which make be read via a recv() on the underlying
+            # socket.
+            self.fp = sock.makefile('rb', 0)
+        self.debuglevel = debuglevel
+        self.strict = strict
+        self._method = method
+
+        self.msg = None
+
+        # from the Status-Line of the response
+        self.version = _UNKNOWN # HTTP-Version
+        self.status = _UNKNOWN  # Status-Code
+        self.reason = _UNKNOWN  # Reason-Phrase
+
+        self.chunked = _UNKNOWN         # is "chunked" being used?
+        self.chunk_left = _UNKNOWN      # bytes left to read in current chunk
+        self.length = _UNKNOWN          # number of bytes left in response
+        self.will_close = _UNKNOWN      # conn will close at end of response
+
+    def _read_status(self):
+        # Initialize with Simple-Response defaults
+        line = self.fp.readline()
+        if self.debuglevel > 0:
+            print "reply:", repr(line)
+        if not line:
+            # Presumably, the server closed the connection before
+            # sending a valid response.
+            raise BadStatusLine(line)
+        try:
+            [version, status, reason] = line.split(None, 2)
+        except ValueError:
+            try:
+                [version, status] = line.split(None, 1)
+                reason = ""
+            except ValueError:
+                # empty version will cause next test to fail and status
+                # will be treated as 0.9 response.
+                version = ""
+        if not version.startswith('HTTP/'):
+            if self.strict:
+                self.close()
+                raise BadStatusLine(line)
+            else:
+                # assume it's a Simple-Response from an 0.9 server
+                self.fp = LineAndFileWrapper(line, self.fp)
+                return "HTTP/0.9", 200, ""
+
+        # The status code is a three-digit number
+        try:
+            status = int(status)
+            if status < 100 or status > 999:
+                raise BadStatusLine(line)
+        except ValueError:
+            raise BadStatusLine(line)
+        return version, status, reason
+
+    def begin(self):
+        if self.msg is not None:
+            # we've already started reading the response
+            return
+
+        # read until we get a non-100 response
+        while True:
+            version, status, reason = self._read_status()
+            if status != CONTINUE:
+                break
+            # skip the header from the 100 response
+            while True:
+                skip = self.fp.readline().strip()
+                if not skip:
+                    break
+                if self.debuglevel > 0:
+                    print "header:", skip
+
+        self.status = status
+        self.reason = reason.strip()
+        if version == 'HTTP/1.0':
+            self.version = 10
+        elif version.startswith('HTTP/1.'):
+            self.version = 11   # use HTTP/1.1 code for HTTP/1.x where x>=1
+        elif version == 'HTTP/0.9':
+            self.version = 9
+        else:
+            raise UnknownProtocol(version)
+
+        if self.version == 9:
+            self.length = None
+            self.chunked = 0
+            self.will_close = 1
+            self.msg = HTTPMessage(StringIO())
+            return
+
+        self.msg = HTTPMessage(self.fp, 0)
+        if self.debuglevel > 0:
+            for hdr in self.msg.headers:
+                print "header:", hdr,
+
+        # don't let the msg keep an fp
+        self.msg.fp = None
+
+        # are we using the chunked-style of transfer encoding?
+        tr_enc = self.msg.getheader('transfer-encoding')
+        if tr_enc and tr_enc.lower() == "chunked":
+            self.chunked = 1
+            self.chunk_left = None
+        else:
+            self.chunked = 0
+
+        # will the connection close at the end of the response?
+        self.will_close = self._check_close()
+
+        # do we have a Content-Length?
+        # NOTE: RFC 2616, S4.4, #3 says we ignore this if tr_enc is "chunked"
+        length = self.msg.getheader('content-length')
+        if length and not self.chunked:
+            try:
+                self.length = int(length)
+            except ValueError:
+                self.length = None
+            else:
+                if self.length < 0:  # ignore nonsensical negative lengths
+                    self.length = None
+        else:
+            self.length = None
+
+        # does the body have a fixed length? (of zero)
+        if (status == NO_CONTENT or status == NOT_MODIFIED or
+            100 <= status < 200 or      # 1xx codes
+            self._method == 'HEAD'):
+            self.length = 0
+
+        # if the connection remains open, and we aren't using chunked, and
+        # a content-length was not provided, then assume that the connection
+        # WILL close.
+        if not self.will_close and \
+           not self.chunked and \
+           self.length is None:
+            self.will_close = 1
+
+    def _check_close(self):
+        conn = self.msg.getheader('connection')
+        if self.version == 11:
+            # An HTTP/1.1 proxy is assumed to stay open unless
+            # explicitly closed.
+            conn = self.msg.getheader('connection')
+            if conn and "close" in conn.lower():
+                return True
+            return False
+
+        # Some HTTP/1.0 implementations have support for persistent
+        # connections, using rules different than HTTP/1.1.
+
+        # For older HTTP, Keep-Alive indicates persistent connection.
+        if self.msg.getheader('keep-alive'):
+            return False
+
+        # At least Akamai returns a "Connection: Keep-Alive" header,
+        # which was supposed to be sent by the client.
+        if conn and "keep-alive" in conn.lower():
+            return False
+
+        # Proxy-Connection is a netscape hack.
+        pconn = self.msg.getheader('proxy-connection')
+        if pconn and "keep-alive" in pconn.lower():
+            return False
+
+        # otherwise, assume it will close
+        return True
+
+    def close(self):
+        if self.fp:
+            self.fp.close()
+            self.fp = None
+
+    def isclosed(self):
+        # NOTE: it is possible that we will not ever call self.close(). This
+        #       case occurs when will_close is TRUE, length is None, and we
+        #       read up to the last byte, but NOT past it.
+        #
+        # IMPLIES: if will_close is FALSE, then self.close() will ALWAYS be
+        #          called, meaning self.isclosed() is meaningful.
+        return self.fp is None
+
+    # XXX It would be nice to have readline and __iter__ for this, too.
+
+    def read(self, amt=None):
+        if self.fp is None:
+            return ''
+
+        if self._method == 'HEAD':
+            self.close()
+            return ''
+
+        if self.chunked:
+            return self._read_chunked(amt)
+
+        if amt is None:
+            # unbounded read
+            if self.length is None:
+                s = self.fp.read()
+            else:
+                s = self._safe_read(self.length)
+                self.length = 0
+            self.close()        # we read everything
+            return s
+
+        if self.length is not None:
+            if amt > self.length:
+                # clip the read to the "end of response"
+                amt = self.length
+
+        # we do not use _safe_read() here because this may be a .will_close
+        # connection, and the user is reading more bytes than will be provided
+        # (for example, reading in 1k chunks)
+        s = self.fp.read(amt)
+        if self.length is not None:
+            self.length -= len(s)
+            if not self.length:
+                self.close()
+        return s
+
+    def _read_chunked(self, amt):
+        assert self.chunked != _UNKNOWN
+        chunk_left = self.chunk_left
+        value = []
+        while True:
+            if chunk_left is None:
+                line = self.fp.readline()
+                i = line.find(';')
+                if i >= 0:
+                    line = line[:i] # strip chunk-extensions
+                try:
+                    chunk_left = int(line, 16)
+                except ValueError:
+                    # close the connection as protocol synchronisation is
+                    # probably lost
+                    self.close()
+                    raise IncompleteRead(''.join(value))
+                if chunk_left == 0:
+                    break
+            if amt is None:
+                value.append(self._safe_read(chunk_left))
+            elif amt < chunk_left:
+                value.append(self._safe_read(amt))
+                self.chunk_left = chunk_left - amt
+                return ''.join(value)
+            elif amt == chunk_left:
+                value.append(self._safe_read(amt))
+                self._safe_read(2)  # toss the CRLF at the end of the chunk
+                self.chunk_left = None
+                return ''.join(value)
+            else:
+                value.append(self._safe_read(chunk_left))
+                amt -= chunk_left
+
+            # we read the whole chunk, get another
+            self._safe_read(2)      # toss the CRLF at the end of the chunk
+            chunk_left = None
+
+        # read and discard trailer up to the CRLF terminator
+        ### note: we shouldn't have any trailers!
+        while True:
+            line = self.fp.readline()
+            if not line:
+                # a vanishingly small number of sites EOF without
+                # sending the trailer
+                break
+            if line == '\r\n':
+                break
+
+        # we read everything; close the "file"
+        self.close()
+
+        return ''.join(value)
+
+    def _safe_read(self, amt):
+        """Read the number of bytes requested, compensating for partial reads.
+
+        Normally, we have a blocking socket, but a read() can be interrupted
+        by a signal (resulting in a partial read).
+
+        Note that we cannot distinguish between EOF and an interrupt when zero
+        bytes have been read. IncompleteRead() will be raised in this
+        situation.
+
+        This function should be used when <amt> bytes "should" be present for
+        reading. If the bytes are truly not available (due to EOF), then the
+        IncompleteRead exception can be used to detect the problem.
+        """
+        # NOTE(gps): As of svn r74426 socket._fileobject.read(x) will never
+        # return less than x bytes unless EOF is encountered.  It now handles
+        # signal interruptions (socket.error EINTR) internally.  This code
+        # never caught that exception anyways.  It seems largely pointless.
+        # self.fp.read(amt) will work fine.
+        s = []
+        while amt > 0:
+            chunk = self.fp.read(min(amt, MAXAMOUNT))
+            if not chunk:
+                raise IncompleteRead(''.join(s), amt)
+            s.append(chunk)
+            amt -= len(chunk)
+        return ''.join(s)
+
+    def fileno(self):
+        return self.fp.fileno()
+
+    def getheader(self, name, default=None):
+        if self.msg is None:
+            raise ResponseNotReady()
+        return self.msg.getheader(name, default)
+
+    def getheaders(self):
+        """Return list of (header, value) tuples."""
+        if self.msg is None:
+            raise ResponseNotReady()
+        return self.msg.items()
+
+
+class HTTPConnection:
+
+    _http_vsn = 11
+    _http_vsn_str = 'HTTP/1.1'
+
+    response_class = HTTPResponse
+    default_port = HTTP_PORT
+    auto_open = 1
+    debuglevel = 0
+    strict = 0
+
+    def __init__(self, host, port=None, strict=None,
+                 timeout=socket._GLOBAL_DEFAULT_TIMEOUT, source_address=None):
+        self.timeout = timeout
+        self.source_address = source_address
+        self.sock = None
+        self._buffer = []
+        self.__response = None
+        self.__state = _CS_IDLE
+        self._method = None
+        self._tunnel_host = None
+        self._tunnel_port = None
+        self._tunnel_headers = {}
+
+        self._set_hostport(host, port)
+        if strict is not None:
+            self.strict = strict
+
+    def set_tunnel(self, host, port=None, headers=None):
+        """ Sets up the host and the port for the HTTP CONNECT Tunnelling.
+
+        The headers argument should be a mapping of extra HTTP headers
+        to send with the CONNECT request.
+        """
+        self._tunnel_host = host
+        self._tunnel_port = port
+        if headers:
+            self._tunnel_headers = headers
+        else:
+            self._tunnel_headers.clear()
+
+    def _set_hostport(self, host, port):
+        if port is None:
+            i = host.rfind(':')
+            j = host.rfind(']')         # ipv6 addresses have [...]
+            if i > j:
+                try:
+                    port = int(host[i+1:])
+                except ValueError:
+                    raise InvalidURL("nonnumeric port: '%s'" % host[i+1:])
+                host = host[:i]
+            else:
+                port = self.default_port
+            if host and host[0] == '[' and host[-1] == ']':
+                host = host[1:-1]
+        self.host = host
+        self.port = port
+
+    def set_debuglevel(self, level):
+        self.debuglevel = level
+
+    def _tunnel(self):
+        self._set_hostport(self._tunnel_host, self._tunnel_port)
+        self.send("CONNECT %s:%d HTTP/1.0\r\n" % (self.host, self.port))
+        for header, value in self._tunnel_headers.iteritems():
+            self.send("%s: %s\r\n" % (header, value))
+        self.send("\r\n")
+        response = self.response_class(self.sock, strict = self.strict,
+                                       method = self._method)
+        (version, code, message) = response._read_status()
+
+        if code != 200:
+            self.close()
+            raise socket.error("Tunnel connection failed: %d %s" % (code,
+                                                                    message.strip()))
+        while True:
+            line = response.fp.readline()
+            if line == '\r\n': break
+
+
+    def connect(self):
+        """Connect to the host and port specified in __init__."""
+        self.sock = socket.create_connection((self.host,self.port),
+                                             self.timeout, self.source_address)
+
+        if self._tunnel_host:
+            self._tunnel()
+
+    def close(self):
+        """Close the connection to the HTTP server."""
+        if self.sock:
+            self.sock.close()   # close it manually... there may be other refs
+            self.sock = None
+        if self.__response:
+            self.__response.close()
+            self.__response = None
+        self.__state = _CS_IDLE
+
+    def send(self, data):
+        """Send `data' to the server."""
+        if self.sock is None:
+            if self.auto_open:
+                self.connect()
+            else:
+                raise NotConnected()
+
+        if self.debuglevel > 0:
+            print "send:", repr(data)
+        blocksize = 8192
+        if hasattr(data,'read') and not isinstance(data, array):
+            if self.debuglevel > 0: print "sendIng a read()able"
+            datablock = data.read(blocksize)
+            while datablock:
+                self.sock.sendall(datablock)
+                datablock = data.read(blocksize)
+        else:
+            self.sock.sendall(data)
+
+    def _output(self, s):
+        """Add a line of output to the current request buffer.
+
+        Assumes that the line does *not* end with \\r\\n.
+        """
+        self._buffer.append(s)
+
+    def _send_output(self, message_body=None):
+        """Send the currently buffered request and clear the buffer.
+
+        Appends an extra \\r\\n to the buffer.
+        A message_body may be specified, to be appended to the request.
+        """
+        self._buffer.extend(("", ""))
+        msg = "\r\n".join(self._buffer)
+        del self._buffer[:]
+        # If msg and message_body are sent in a single send() call,
+        # it will avoid performance problems caused by the interaction
+        # between delayed ack and the Nagle algorithim.
+        if isinstance(message_body, str):
+            msg += message_body
+            message_body = None
+        self.send(msg)
+        if message_body is not None:
+            #message_body was not a string (i.e. it is a file) and
+            #we must run the risk of Nagle
+            self.send(message_body)
+
+    def putrequest(self, method, url, skip_host=0, skip_accept_encoding=0):
+        """Send a request to the server.
+
+        `method' specifies an HTTP request method, e.g. 'GET'.
+        `url' specifies the object being requested, e.g. '/index.html'.
+        `skip_host' if True does not add automatically a 'Host:' header
+        `skip_accept_encoding' if True does not add automatically an
+           'Accept-Encoding:' header
+        """
+
+        # if a prior response has been completed, then forget about it.
+        if self.__response and self.__response.isclosed():
+            self.__response = None
+
+
+        # in certain cases, we cannot issue another request on this connection.
+        # this occurs when:
+        #   1) we are in the process of sending a request.   (_CS_REQ_STARTED)
+        #   2) a response to a previous request has signalled that it is going
+        #      to close the connection upon completion.
+        #   3) the headers for the previous response have not been read, thus
+        #      we cannot determine whether point (2) is true.   (_CS_REQ_SENT)
+        #
+        # if there is no prior response, then we can request at will.
+        #
+        # if point (2) is true, then we will have passed the socket to the
+        # response (effectively meaning, "there is no prior response"), and
+        # will open a new one when a new request is made.
+        #
+        # Note: if a prior response exists, then we *can* start a new request.
+        #       We are not allowed to begin fetching the response to this new
+        #       request, however, until that prior response is complete.
+        #
+        if self.__state == _CS_IDLE:
+            self.__state = _CS_REQ_STARTED
+        else:
+            raise CannotSendRequest()
+
+        # Save the method we use, we need it later in the response phase
+        self._method = method
+        if not url:
+            url = '/'
+        hdr = '%s %s %s' % (method, url, self._http_vsn_str)
+
+        self._output(hdr)
+
+        if self._http_vsn == 11:
+            # Issue some standard headers for better HTTP/1.1 compliance
+
+            if not skip_host:
+                # this header is issued *only* for HTTP/1.1
+                # connections. more specifically, this means it is
+                # only issued when the client uses the new
+                # HTTPConnection() class. backwards-compat clients
+                # will be using HTTP/1.0 and those clients may be
+                # issuing this header themselves. we should NOT issue
+                # it twice; some web servers (such as Apache) barf
+                # when they see two Host: headers
+
+                # If we need a non-standard port,include it in the
+                # header.  If the request is going through a proxy,
+                # but the host of the actual URL, not the host of the
+                # proxy.
+
+                netloc = ''
+                if url.startswith('http'):
+                    nil, netloc, nil, nil, nil = urlsplit(url)
+
+                if netloc:
+                    try:
+                        netloc_enc = netloc.encode("ascii")
+                    except UnicodeEncodeError:
+                        netloc_enc = netloc.encode("idna")
+                    self.putheader('Host', netloc_enc)
+                else:
+                    try:
+                        host_enc = self.host.encode("ascii")
+                    except UnicodeEncodeError:
+                        host_enc = self.host.encode("idna")
+                    # Wrap the IPv6 Host Header with [] (RFC 2732)
+                    if host_enc.find(':') >= 0:
+                        host_enc = "[" + host_enc + "]"
+                    if self.port == self.default_port:
+                        self.putheader('Host', host_enc)
+                    else:
+                        self.putheader('Host', "%s:%s" % (host_enc, self.port))
+
+            # note: we are assuming that clients will not attempt to set these
+            #       headers since *this* library must deal with the
+            #       consequences. this also means that when the supporting
+            #       libraries are updated to recognize other forms, then this
+            #       code should be changed (removed or updated).
+
+            # we only want a Content-Encoding of "identity" since we don't
+            # support encodings such as x-gzip or x-deflate.
+            if not skip_accept_encoding:
+                self.putheader('Accept-Encoding', 'identity')
+
+            # we can accept "chunked" Transfer-Encodings, but no others
+            # NOTE: no TE header implies *only* "chunked"
+            #self.putheader('TE', 'chunked')
+
+            # if TE is supplied in the header, then it must appear in a
+            # Connection header.
+            #self.putheader('Connection', 'TE')
+
+        else:
+            # For HTTP/1.0, the server will assume "not chunked"
+            pass
+
+    def putheader(self, header, *values):
+        """Send a request header line to the server.
+
+        For example: h.putheader('Accept', 'text/html')
+        """
+        if self.__state != _CS_REQ_STARTED:
+            raise CannotSendHeader()
+
+        hdr = '%s: %s' % (header, '\r\n\t'.join([str(v) for v in values]))
+        self._output(hdr)
+
+    def endheaders(self, message_body=None):
+        """Indicate that the last header line has been sent to the server.
+
+        This method sends the request to the server.  The optional
+        message_body argument can be used to pass message body
+        associated with the request.  The message body will be sent in
+        the same packet as the message headers if possible.  The
+        message_body should be a string.
+        """
+        if self.__state == _CS_REQ_STARTED:
+            self.__state = _CS_REQ_SENT
+        else:
+            raise CannotSendHeader()
+        self._send_output(message_body)
+
+    def request(self, method, url, body=None, headers={}):
+        """Send a complete request to the server."""
+        self._send_request(method, url, body, headers)
+
+    def _set_content_length(self, body):
+        # Set the content-length based on the body.
+        thelen = None
+        try:
+            thelen = str(len(body))
+        except TypeError, te:
+            # If this is a file-like object, try to
+            # fstat its file descriptor
+            try:
+                thelen = str(os.fstat(body.fileno()).st_size)
+            except (AttributeError, OSError):
+                # Don't send a length if this failed
+                if self.debuglevel > 0: print "Cannot stat!!"
+
+        if thelen is not None:
+            self.putheader('Content-Length', thelen)
+
+    def _send_request(self, method, url, body, headers):
+        # Honor explicitly requested Host: and Accept-Encoding: headers.
+        header_names = dict.fromkeys([k.lower() for k in headers])
+        skips = {}
+        if 'host' in header_names:
+            skips['skip_host'] = 1
+        if 'accept-encoding' in header_names:
+            skips['skip_accept_encoding'] = 1
+
+        self.putrequest(method, url, **skips)
+
+        if body and ('content-length' not in header_names):
+            self._set_content_length(body)
+        for hdr, value in headers.iteritems():
+            self.putheader(hdr, value)
+        self.endheaders(body)
+
+    def getresponse(self, buffering=False):
+        "Get the response from the server."
+
+        # if a prior response has been completed, then forget about it.
+        if self.__response and self.__response.isclosed():
+            self.__response = None
+
+        #
+        # if a prior response exists, then it must be completed (otherwise, we
+        # cannot read this response's header to determine the connection-close
+        # behavior)
+        #
+        # note: if a prior response existed, but was connection-close, then the
+        # socket and response were made independent of this HTTPConnection
+        # object since a new request requires that we open a whole new
+        # connection
+        #
+        # this means the prior response had one of two states:
+        #   1) will_close: this connection was reset and the prior socket and
+        #                  response operate independently
+        #   2) persistent: the response was retained and we await its
+        #                  isclosed() status to become true.
+        #
+        if self.__state != _CS_REQ_SENT or self.__response:
+            raise ResponseNotReady()
+
+        args = (self.sock,)
+        kwds = {"strict":self.strict, "method":self._method}
+        if self.debuglevel > 0:
+            args += (self.debuglevel,)
+        if buffering:
+            #only add this keyword if non-default, for compatibility with
+            #other response_classes.
+            kwds["buffering"] = True;
+        response = self.response_class(*args, **kwds)
+
+        try:
+            response.begin()
+        except:
+            response.close()
+            raise
+        assert response.will_close != _UNKNOWN
+        self.__state = _CS_IDLE
+
+        if response.will_close:
+            # this effectively passes the connection to the response
+            self.close()
+        else:
+            # remember this, so we can tell when it is complete
+            self.__response = response
+
+        return response
+
+
+class HTTP:
+    "Compatibility class with httplib.py from 1.5."
+
+    _http_vsn = 10
+    _http_vsn_str = 'HTTP/1.0'
+
+    debuglevel = 0
+
+    _connection_class = HTTPConnection
+
+    def __init__(self, host='', port=None, strict=None):
+        "Provide a default host, since the superclass requires one."
+
+        # some joker passed 0 explicitly, meaning default port
+        if port == 0:
+            port = None
+
+        # Note that we may pass an empty string as the host; this will throw
+        # an error when we attempt to connect. Presumably, the client code
+        # will call connect before then, with a proper host.
+        self._setup(self._connection_class(host, port, strict))
+
+    def _setup(self, conn):
+        self._conn = conn
+
+        # set up delegation to flesh out interface
+        self.send = conn.send
+        self.putrequest = conn.putrequest
+        self.putheader = conn.putheader
+        self.endheaders = conn.endheaders
+        self.set_debuglevel = conn.set_debuglevel
+
+        conn._http_vsn = self._http_vsn
+        conn._http_vsn_str = self._http_vsn_str
+
+        self.file = None
+
+    def connect(self, host=None, port=None):
+        "Accept arguments to set the host/port, since the superclass doesn't."
+
+        if host is not None:
+            self._conn._set_hostport(host, port)
+        self._conn.connect()
+
+    def getfile(self):
+        "Provide a getfile, since the superclass' does not use this concept."
+        return self.file
+
+    def getreply(self, buffering=False):
+        """Compat definition since superclass does not define it.
+
+        Returns a tuple consisting of:
+        - server status code (e.g. '200' if all goes well)
+        - server "reason" corresponding to status code
+        - any RFC822 headers in the response from the server
+        """
+        try:
+            if not buffering:
+                response = self._conn.getresponse()
+            else:
+                #only add this keyword if non-default for compatibility
+                #with other connection classes
+                response = self._conn.getresponse(buffering)
+        except BadStatusLine, e:
+            ### hmm. if getresponse() ever closes the socket on a bad request,
+            ### then we are going to have problems with self.sock
+
+            ### should we keep this behavior? do people use it?
+            # keep the socket open (as a file), and return it
+            self.file = self._conn.sock.makefile('rb', 0)
+
+            # close our socket -- we want to restart after any protocol error
+            self.close()
+
+            self.headers = None
+            return -1, e.line, None
+
+        self.headers = response.msg
+        self.file = response.fp
+        return response.status, response.reason, response.msg
+
+    def close(self):
+        self._conn.close()
+
+        # note that self.file == response.fp, which gets closed by the
+        # superclass. just clear the object ref here.
+        ### hmm. messy. if status==-1, then self.file is owned by us.
+        ### well... we aren't explicitly closing, but losing this ref will
+        ### do it
+        self.file = None
+
+try:
+    import ssl
+except ImportError:
+    pass
+else:
+    class HTTPSConnection(HTTPConnection):
+        "This class allows communication via SSL."
+
+        default_port = HTTPS_PORT
+
+        def __init__(self, host, port=None, key_file=None, cert_file=None,
+                     strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
+                     source_address=None):
+            HTTPConnection.__init__(self, host, port, strict, timeout,
+                                    source_address)
+            self.key_file = key_file
+            self.cert_file = cert_file
+
+        def connect(self):
+            "Connect to a host on a given (SSL) port."
+
+            sock = socket.create_connection((self.host, self.port),
+                                            self.timeout, self.source_address)
+            if self._tunnel_host:
+                self.sock = sock
+                self._tunnel()
+            self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file)
+
+    __all__.append("HTTPSConnection")
+
+    class HTTPS(HTTP):
+        """Compatibility with 1.5 httplib interface
+
+        Python 1.5.2 did not have an HTTPS class, but it defined an
+        interface for sending http requests that is also useful for
+        https.
+        """
+
+        _connection_class = HTTPSConnection
+
+        def __init__(self, host='', port=None, key_file=None, cert_file=None,
+                     strict=None):
+            # provide a default host, pass the X509 cert info
+
+            # urf. compensate for bad input.
+            if port == 0:
+                port = None
+            self._setup(self._connection_class(host, port, key_file,
+                                               cert_file, strict))
+
+            # we never actually use these for anything, but we keep them
+            # here for compatibility with post-1.5.2 CVS.
+            self.key_file = key_file
+            self.cert_file = cert_file
+
+
+    def FakeSocket (sock, sslobj):
+        warnings.warn("FakeSocket is deprecated, and won't be in 3.x.  " +
+                      "Use the result of ssl.wrap_socket() directly instead.",
+                      DeprecationWarning, stacklevel=2)
+        return sslobj
+
+
+class HTTPException(Exception):
+    # Subclasses that define an __init__ must call Exception.__init__
+    # or define self.args.  Otherwise, str() will fail.
+    pass
+
+class NotConnected(HTTPException):
+    pass
+
+class InvalidURL(HTTPException):
+    pass
+
+class UnknownProtocol(HTTPException):
+    def __init__(self, version):
+        self.args = version,
+        self.version = version
+
+class UnknownTransferEncoding(HTTPException):
+    pass
+
+class UnimplementedFileMode(HTTPException):
+    pass
+
+class IncompleteRead(HTTPException):
+    def __init__(self, partial, expected=None):
+        self.args = partial,
+        self.partial = partial
+        self.expected = expected
+    def __repr__(self):
+        if self.expected is not None:
+            e = ', %i more expected' % self.expected
+        else:
+            e = ''
+        return 'IncompleteRead(%i bytes read%s)' % (len(self.partial), e)
+    def __str__(self):
+        return repr(self)
+
+class ImproperConnectionState(HTTPException):
+    pass
+
+class CannotSendRequest(ImproperConnectionState):
+    pass
+
+class CannotSendHeader(ImproperConnectionState):
+    pass
+
+class ResponseNotReady(ImproperConnectionState):
+    pass
+
+class BadStatusLine(HTTPException):
+    def __init__(self, line):
+        if not line:
+            line = repr(line)
+        self.args = line,
+        self.line = line
+
+# for backwards compatibility
+error = HTTPException
+
+class LineAndFileWrapper:
+    """A limited file-like object for HTTP/0.9 responses."""
+
+    # The status-line parsing code calls readline(), which normally
+    # get the HTTP status line.  For a 0.9 response, however, this is
+    # actually the first line of the body!  Clients need to get a
+    # readable file object that contains that line.
+
+    def __init__(self, line, file):
+        self._line = line
+        self._file = file
+        self._line_consumed = 0
+        self._line_offset = 0
+        self._line_left = len(line)
+
+    def __getattr__(self, attr):
+        return getattr(self._file, attr)
+
+    def _done(self):
+        # called when the last byte is read from the line.  After the
+        # call, all read methods are delegated to the underlying file
+        # object.
+        self._line_consumed = 1
+        self.read = self._file.read
+        self.readline = self._file.readline
+        self.readlines = self._file.readlines
+
+    def read(self, amt=None):
+        if self._line_consumed:
+            return self._file.read(amt)
+        assert self._line_left
+        if amt is None or amt > self._line_left:
+            s = self._line[self._line_offset:]
+            self._done()
+            if amt is None:
+                return s + self._file.read()
+            else:
+                return s + self._file.read(amt - len(s))
+        else:
+            assert amt <= self._line_left
+            i = self._line_offset
+            j = i + amt
+            s = self._line[i:j]
+            self._line_offset = j
+            self._line_left -= amt
+            if self._line_left == 0:
+                self._done()
+            return s
+
+    def readline(self):
+        if self._line_consumed:
+            return self._file.readline()
+        assert self._line_left
+        s = self._line[self._line_offset:]
+        self._done()
+        return s
+
+    def readlines(self, size=None):
+        if self._line_consumed:
+            return self._file.readlines(size)
+        assert self._line_left
+        L = [self._line[self._line_offset:]]
+        self._done()
+        if size is None:
+            return L + self._file.readlines()
+        else:
+            return L + self._file.readlines(size)
+
+def test():
+    """Test this module.
+
+    A hodge podge of tests collected here, because they have too many
+    external dependencies for the regular test suite.
+    """
+
+    import sys
+    import getopt
+    opts, args = getopt.getopt(sys.argv[1:], 'd')
+    dl = 0
+    for o, a in opts:
+        if o == '-d': dl = dl + 1
+    host = 'www.python.org'
+    selector = '/'
+    if args[0:]: host = args[0]
+    if args[1:]: selector = args[1]
+    h = HTTP()
+    h.set_debuglevel(dl)
+    h.connect(host)
+    h.putrequest('GET', selector)
+    h.endheaders()
+    status, reason, headers = h.getreply()
+    print 'status =', status
+    print 'reason =', reason
+    print "read", len(h.getfile().read())
+    print
+    if headers:
+        for header in headers.headers: print header.strip()
+    print
+
+    # minimal test that code to extract host from url works
+    class HTTP11(HTTP):
+        _http_vsn = 11
+        _http_vsn_str = 'HTTP/1.1'
+
+    h = HTTP11('www.python.org')
+    h.putrequest('GET', 'http://www.python.org/~jeremy/')
+    h.endheaders()
+    h.getreply()
+    h.close()
+
+    try:
+        import ssl
+    except ImportError:
+        pass
+    else:
+
+        for host, selector in (('sourceforge.net', '/projects/python'),
+                               ):
+            print "https://%s%s" % (host, selector)
+            hs = HTTPS()
+            hs.set_debuglevel(dl)
+            hs.connect(host)
+            hs.putrequest('GET', selector)
+            hs.endheaders()
+            status, reason, headers = hs.getreply()
+            print 'status =', status
+            print 'reason =', reason
+            print "read", len(hs.getfile().read())
+            print
+            if headers:
+                for header in headers.headers: print header.strip()
+            print
+
+if __name__ == '__main__':
+    test()
diff --git a/lib-python/modified-2.7/test/test_urllib2.py b/lib-python/modified-2.7/test/test_urllib2.py
--- a/lib-python/modified-2.7/test/test_urllib2.py
+++ b/lib-python/modified-2.7/test/test_urllib2.py
@@ -307,6 +307,9 @@
     def getresponse(self):
         return MockHTTPResponse(MockFile(), {}, 200, "OK")
 
+    def close(self):
+        pass
+
 class MockHandler:
     # useful for testing handler machinery
     # see add_ordered_mock_handlers() docstring
diff --git a/lib-python/modified-2.7/urllib2.py b/lib-python/modified-2.7/urllib2.py
new file mode 100644
--- /dev/null
+++ b/lib-python/modified-2.7/urllib2.py
@@ -0,0 +1,1440 @@
+"""An extensible library for opening URLs using a variety of protocols
+
+The simplest way to use this module is to call the urlopen function,
+which accepts a string containing a URL or a Request object (described
+below).  It opens the URL and returns the results as file-like
+object; the returned object has some extra methods described below.
+
+The OpenerDirector manages a collection of Handler objects that do
+all the actual work.  Each Handler implements a particular protocol or
+option.  The OpenerDirector is a composite object that invokes the
+Handlers needed to open the requested URL.  For example, the
+HTTPHandler performs HTTP GET and POST requests and deals with
+non-error returns.  The HTTPRedirectHandler automatically deals with
+HTTP 301, 302, 303 and 307 redirect errors, and the HTTPDigestAuthHandler
+deals with digest authentication.
+
+urlopen(url, data=None) -- Basic usage is the same as original
+urllib.  pass the url and optionally data to post to an HTTP URL, and
+get a file-like object back.  One difference is that you can also pass
+a Request instance instead of URL.  Raises a URLError (subclass of
+IOError); for HTTP errors, raises an HTTPError, which can also be
+treated as a valid response.
+
+build_opener -- Function that creates a new OpenerDirector instance.
+Will install the default handlers.  Accepts one or more Handlers as
+arguments, either instances or Handler classes that it will
+instantiate.  If one of the argument is a subclass of the default
+handler, the argument will be installed instead of the default.
+
+install_opener -- Installs a new opener as the default opener.
+
+objects of interest:
+
+OpenerDirector -- Sets up the User Agent as the Python-urllib client and manages
+the Handler classes, while dealing with requests and responses.
+
+Request -- An object that encapsulates the state of a request.  The
+state can be as simple as the URL.  It can also include extra HTTP
+headers, e.g. a User-Agent.
+
+BaseHandler --
+
+exceptions:
+URLError -- A subclass of IOError, individual protocols have their own
+specific subclass.
+
+HTTPError -- Also a valid HTTP response, so you can treat an HTTP error
+as an exceptional event or valid response.
+
+internals:
+BaseHandler and parent
+_call_chain conventions
+
+Example usage:
+
+import urllib2
+
+# set up authentication info
+authinfo = urllib2.HTTPBasicAuthHandler()
+authinfo.add_password(realm='PDQ Application',
+                      uri='https://mahler:8092/site-updates.py',
+                      user='klem',
+                      passwd='geheim$parole')
+
+proxy_support = urllib2.ProxyHandler({"http" : "http://ahad-haam:3128"})
+
+# build a new opener that adds authentication and caching FTP handlers
+opener = urllib2.build_opener(proxy_support, authinfo, urllib2.CacheFTPHandler)
+
+# install it
+urllib2.install_opener(opener)
+
+f = urllib2.urlopen('http://www.python.org/')
+
+
+"""
+
+# XXX issues:
+# If an authentication error handler that tries to perform
+# authentication for some reason but fails, how should the error be
+# signalled?  The client needs to know the HTTP error code.  But if
+# the handler knows that the problem was, e.g., that it didn't know
+# that hash algo that requested in the challenge, it would be good to
+# pass that information along to the client, too.
+# ftp errors aren't handled cleanly
+# check digest against correct (i.e. non-apache) implementation
+
+# Possible extensions:
+# complex proxies  XXX not sure what exactly was meant by this
+# abstract factory for opener
+
+import base64
+import hashlib
+import httplib
+import mimetools
+import os
+import posixpath
+import random
+import re
+import socket
+import sys
+import time
+import urlparse
+import bisect
+
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from StringIO import StringIO
+
+from urllib import (unwrap, unquote, splittype, splithost, quote,
+     addinfourl, splitport, splittag,
+     splitattr, ftpwrapper, splituser, splitpasswd, splitvalue)
+
+# support for FileHandler, proxies via environment variables
+from urllib import localhost, url2pathname, getproxies, proxy_bypass
+
+# used in User-Agent header sent
+__version__ = sys.version[:3]
+
+_opener = None
+def urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
+    global _opener
+    if _opener is None:
+        _opener = build_opener()
+    return _opener.open(url, data, timeout)
+
+def install_opener(opener):
+    global _opener
+    _opener = opener
+
+# do these error classes make sense?
+# make sure all of the IOError stuff is overridden.  we just want to be
+# subtypes.
+
+class URLError(IOError):
+    # URLError is a sub-type of IOError, but it doesn't share any of
+    # the implementation.  need to override __init__ and __str__.
+    # It sets self.args for compatibility with other EnvironmentError
+    # subclasses, but args doesn't have the typical format with errno in
+    # slot 0 and strerror in slot 1.  This may be better than nothing.
+    def __init__(self, reason):
+        self.args = reason,
+        self.reason = reason
+
+    def __str__(self):
+        return '<urlopen error %s>' % self.reason
+
+class HTTPError(URLError, addinfourl):
+    """Raised when HTTP error occurs, but also acts like non-error return"""
+    __super_init = addinfourl.__init__
+
+    def __init__(self, url, code, msg, hdrs, fp):
+        self.code = code
+        self.msg = msg
+        self.hdrs = hdrs
+        self.fp = fp
+        self.filename = url
+        # The addinfourl classes depend on fp being a valid file
+        # object.  In some cases, the HTTPError may not have a valid
+        # file object.  If this happens, the simplest workaround is to
+        # not initialize the base classes.
+        if fp is not None:
+            self.__super_init(fp, hdrs, url, code)
+
+    def __str__(self):
+        return 'HTTP Error %s: %s' % (self.code, self.msg)
+
+# copied from cookielib.py
+_cut_port_re = re.compile(r":\d+$")
+def request_host(request):
+    """Return request-host, as defined by RFC 2965.
+
+    Variation from RFC: returned value is lowercased, for convenient
+    comparison.
+
+    """
+    url = request.get_full_url()
+    host = urlparse.urlparse(url)[1]
+    if host == "":
+        host = request.get_header("Host", "")
+
+    # remove port, if present
+    host = _cut_port_re.sub("", host, 1)
+    return host.lower()
+
+class Request:
+
+    def __init__(self, url, data=None, headers={},
+                 origin_req_host=None, unverifiable=False):
+        # unwrap('<URL:type://host/path>') --> 'type://host/path'
+        self.__original = unwrap(url)
+        self.__original, fragment = splittag(self.__original)
+        self.type = None
+        # self.__r_type is what's left after doing the splittype
+        self.host = None
+        self.port = None
+        self._tunnel_host = None
+        self.data = data
+        self.headers = {}
+        for key, value in headers.items():
+            self.add_header(key, value)
+        self.unredirected_hdrs = {}
+        if origin_req_host is None:
+            origin_req_host = request_host(self)
+        self.origin_req_host = origin_req_host
+        self.unverifiable = unverifiable
+
+    def __getattr__(self, attr):
+        # XXX this is a fallback mechanism to guard against these
+        # methods getting called in a non-standard order.  this may be
+        # too complicated and/or unnecessary.
+        # XXX should the __r_XXX attributes be public?
+        if attr[:12] == '_Request__r_':
+            name = attr[12:]
+            if hasattr(Request, 'get_' + name):
+                getattr(self, 'get_' + name)()
+                return getattr(self, attr)
+        raise AttributeError, attr
+
+    def get_method(self):
+        if self.has_data():
+            return "POST"
+        else:
+            return "GET"
+
+    # XXX these helper methods are lame
+
+    def add_data(self, data):
+        self.data = data
+
+    def has_data(self):
+        return self.data is not None
+
+    def get_data(self):
+        return self.data
+
+    def get_full_url(self):
+        return self.__original
+
+    def get_type(self):
+        if self.type is None:
+            self.type, self.__r_type = splittype(self.__original)
+            if self.type is None:
+                raise ValueError, "unknown url type: %s" % self.__original
+        return self.type
+
+    def get_host(self):
+        if self.host is None:
+            self.host, self.__r_host = splithost(self.__r_type)
+            if self.host:
+                self.host = unquote(self.host)
+        return self.host
+
+    def get_selector(self):
+        return self.__r_host
+
+    def set_proxy(self, host, type):
+        if self.type == 'https' and not self._tunnel_host:
+            self._tunnel_host = self.host
+        else:
+            self.type = type
+            self.__r_host = self.__original
+
+        self.host = host
+
+    def has_proxy(self):
+        return self.__r_host == self.__original
+
+    def get_origin_req_host(self):
+        return self.origin_req_host
+
+    def is_unverifiable(self):
+        return self.unverifiable
+
+    def add_header(self, key, val):
+        # useful for something like authentication
+        self.headers[key.capitalize()] = val
+
+    def add_unredirected_header(self, key, val):
+        # will not be added to a redirected request
+        self.unredirected_hdrs[key.capitalize()] = val
+
+    def has_header(self, header_name):
+        return (header_name in self.headers or
+                header_name in self.unredirected_hdrs)
+
+    def get_header(self, header_name, default=None):
+        return self.headers.get(
+            header_name,
+            self.unredirected_hdrs.get(header_name, default))
+
+    def header_items(self):
+        hdrs = self.unredirected_hdrs.copy()
+        hdrs.update(self.headers)
+        return hdrs.items()
+
+class OpenerDirector:
+    def __init__(self):
+        client_version = "Python-urllib/%s" % __version__
+        self.addheaders = [('User-agent', client_version)]
+        # manage the individual handlers
+        self.handlers = []
+        self.handle_open = {}
+        self.handle_error = {}
+        self.process_response = {}
+        self.process_request = {}
+
+    def add_handler(self, handler):
+        if not hasattr(handler, "add_parent"):
+            raise TypeError("expected BaseHandler instance, got %r" %
+                            type(handler))
+
+        added = False
+        for meth in dir(handler):
+            if meth in ["redirect_request", "do_open", "proxy_open"]:
+                # oops, coincidental match
+                continue
+
+            i = meth.find("_")
+            protocol = meth[:i]
+            condition = meth[i+1:]
+
+            if condition.startswith("error"):
+                j = condition.find("_") + i + 1
+                kind = meth[j+1:]
+                try:
+                    kind = int(kind)
+                except ValueError:
+                    pass
+                lookup = self.handle_error.get(protocol, {})
+                self.handle_error[protocol] = lookup
+            elif condition == "open":
+                kind = protocol
+                lookup = self.handle_open
+            elif condition == "response":
+                kind = protocol
+                lookup = self.process_response
+            elif condition == "request":
+                kind = protocol
+                lookup = self.process_request
+            else:
+                continue
+
+            handlers = lookup.setdefault(kind, [])
+            if handlers:
+                bisect.insort(handlers, handler)
+            else:
+                handlers.append(handler)
+            added = True
+
+        if added:
+            # the handlers must work in an specific order, the order
+            # is specified in a Handler attribute
+            bisect.insort(self.handlers, handler)
+            handler.add_parent(self)
+
+    def close(self):
+        # Only exists for backwards compatibility.
+        pass
+
+    def _call_chain(self, chain, kind, meth_name, *args):
+        # Handlers raise an exception if no one else should try to handle
+        # the request, or return None if they can't but another handler
+        # could.  Otherwise, they return the response.
+        handlers = chain.get(kind, ())
+        for handler in handlers:
+            func = getattr(handler, meth_name)
+
+            result = func(*args)
+            if result is not None:
+                return result
+
+    def open(self, fullurl, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
+        # accept a URL or a Request object
+        if isinstance(fullurl, basestring):
+            req = Request(fullurl, data)
+        else:
+            req = fullurl
+            if data is not None:
+                req.add_data(data)
+
+        req.timeout = timeout
+        protocol = req.get_type()
+
+        # pre-process request
+        meth_name = protocol+"_request"
+        for processor in self.process_request.get(protocol, []):
+            meth = getattr(processor, meth_name)
+            req = meth(req)
+
+        response = self._open(req, data)
+
+        # post-process response
+        meth_name = protocol+"_response"
+        for processor in self.process_response.get(protocol, []):
+            meth = getattr(processor, meth_name)
+            try:
+                response = meth(req, response)
+            except:
+                response.close()
+                raise
+
+        return response
+
+    def _open(self, req, data=None):
+        result = self._call_chain(self.handle_open, 'default',
+                                  'default_open', req)
+        if result:
+            return result
+
+        protocol = req.get_type()
+        result = self._call_chain(self.handle_open, protocol, protocol +
+                                  '_open', req)
+        if result:
+            return result
+
+        return self._call_chain(self.handle_open, 'unknown',
+                                'unknown_open', req)
+
+    def error(self, proto, *args):
+        if proto in ('http', 'https'):
+            # XXX http[s] protocols are special-cased
+            dict = self.handle_error['http'] # https is not different than http
+            proto = args[2]  # YUCK!
+            meth_name = 'http_error_%s' % proto
+            http_err = 1
+            orig_args = args
+        else:
+            dict = self.handle_error
+            meth_name = proto + '_error'
+            http_err = 0
+        args = (dict, proto, meth_name) + args
+        result = self._call_chain(*args)
+        if result:
+            return result
+
+        if http_err:
+            args = (dict, 'default', 'http_error_default') + orig_args
+            return self._call_chain(*args)
+
+# XXX probably also want an abstract factory that knows when it makes
+# sense to skip a superclass in favor of a subclass and when it might
+# make sense to include both
+
+def build_opener(*handlers):
+    """Create an opener object from a list of handlers.
+
+    The opener will use several default handlers, including support
+    for HTTP, FTP and when applicable, HTTPS.
+
+    If any of the handlers passed as arguments are subclasses of the
+    default handlers, the default handlers will not be used.
+    """
+    import types
+    def isclass(obj):
+        return isinstance(obj, (types.ClassType, type))
+
+    opener = OpenerDirector()
+    default_classes = [ProxyHandler, UnknownHandler, HTTPHandler,
+                       HTTPDefaultErrorHandler, HTTPRedirectHandler,
+                       FTPHandler, FileHandler, HTTPErrorProcessor]
+    if hasattr(httplib, 'HTTPS'):
+        default_classes.append(HTTPSHandler)
+    skip = set()
+    for klass in default_classes:
+        for check in handlers:
+            if isclass(check):
+                if issubclass(check, klass):
+                    skip.add(klass)
+            elif isinstance(check, klass):
+                skip.add(klass)
+    for klass in skip:
+        default_classes.remove(klass)
+
+    for klass in default_classes:
+        opener.add_handler(klass())
+
+    for h in handlers:
+        if isclass(h):
+            h = h()
+        opener.add_handler(h)
+    return opener
+
+class BaseHandler:
+    handler_order = 500
+
+    def add_parent(self, parent):
+        self.parent = parent
+
+    def close(self):
+        # Only exists for backwards compatibility
+        pass
+
+    def __lt__(self, other):
+        if not hasattr(other, "handler_order"):
+            # Try to preserve the old behavior of having custom classes
+            # inserted after default ones (works only for custom user
+            # classes which are not aware of handler_order).
+            return True
+        return self.handler_order < other.handler_order
+
+
+class HTTPErrorProcessor(BaseHandler):
+    """Process HTTP error responses."""
+    handler_order = 1000  # after all other processing
+
+    def http_response(self, request, response):
+        code, msg, hdrs = response.code, response.msg, response.info()
+
+        # According to RFC 2616, "2xx" code indicates that the client's
+        # request was successfully received, understood, and accepted.
+        if not (200 <= code < 300):
+            response = self.parent.error(
+                'http', request, response, code, msg, hdrs)
+
+        return response
+
+    https_response = http_response
+
+class HTTPDefaultErrorHandler(BaseHandler):
+    def http_error_default(self, req, fp, code, msg, hdrs):
+        raise HTTPError(req.get_full_url(), code, msg, hdrs, fp)
+
+class HTTPRedirectHandler(BaseHandler):
+    # maximum number of redirections to any single URL
+    # this is needed because of the state that cookies introduce
+    max_repeats = 4
+    # maximum total number of redirections (regardless of URL) before
+    # assuming we're in a loop
+    max_redirections = 10
+
+    def redirect_request(self, req, fp, code, msg, headers, newurl):
+        """Return a Request or None in response to a redirect.
+
+        This is called by the http_error_30x methods when a
+        redirection response is received.  If a redirection should
+        take place, return a new Request to allow http_error_30x to
+        perform the redirect.  Otherwise, raise HTTPError if no-one
+        else should try to handle this url.  Return None if you can't
+        but another Handler might.
+        """
+        m = req.get_method()
+        if (code in (301, 302, 303, 307) and m in ("GET", "HEAD")
+            or code in (301, 302, 303) and m == "POST"):
+            # Strictly (according to RFC 2616), 301 or 302 in response
+            # to a POST MUST NOT cause a redirection without confirmation
+            # from the user (of urllib2, in this case).  In practice,
+            # essentially all clients do redirect in this case, so we
+            # do the same.
+            # be conciliant with URIs containing a space
+            newurl = newurl.replace(' ', '%20')
+            newheaders = dict((k,v) for k,v in req.headers.items()
+                              if k.lower() not in ("content-length", "content-type")
+                             )
+            return Request(newurl,
+                           headers=newheaders,
+                           origin_req_host=req.get_origin_req_host(),
+                           unverifiable=True)
+        else:
+            raise HTTPError(req.get_full_url(), code, msg, headers, fp)
+
+    # Implementation note: To avoid the server sending us into an
+    # infinite loop, the request object needs to track what URLs we
+    # have already seen.  Do this by adding a handler-specific
+    # attribute to the Request object.
+    def http_error_302(self, req, fp, code, msg, headers):
+        # Some servers (incorrectly) return multiple Location headers
+        # (so probably same goes for URI).  Use first header.
+        if 'location' in headers:
+            newurl = headers.getheaders('location')[0]
+        elif 'uri' in headers:
+            newurl = headers.getheaders('uri')[0]
+        else:
+            return
+
+        # fix a possible malformed URL
+        urlparts = urlparse.urlparse(newurl)
+        if not urlparts.path:
+            urlparts = list(urlparts)
+            urlparts[2] = "/"
+        newurl = urlparse.urlunparse(urlparts)
+
+        newurl = urlparse.urljoin(req.get_full_url(), newurl)
+
+        # XXX Probably want to forget about the state of the current
+        # request, although that might interact poorly with other
+        # handlers that also use handler-specific request attributes
+        new = self.redirect_request(req, fp, code, msg, headers, newurl)
+        if new is None:
+            return
+
+        # loop detection
+        # .redirect_dict has a key url if url was previously visited.
+        if hasattr(req, 'redirect_dict'):
+            visited = new.redirect_dict = req.redirect_dict
+            if (visited.get(newurl, 0) >= self.max_repeats or
+                len(visited) >= self.max_redirections):
+                raise HTTPError(req.get_full_url(), code,
+                                self.inf_msg + msg, headers, fp)
+        else:
+            visited = new.redirect_dict = req.redirect_dict = {}
+        visited[newurl] = visited.get(newurl, 0) + 1
+
+        # Don't close the fp until we are sure that we won't use it
+        # with HTTPError.
+        fp.read()
+        fp.close()
+
+        return self.parent.open(new, timeout=req.timeout)
+
+    http_error_301 = http_error_303 = http_error_307 = http_error_302
+
+    inf_msg = "The HTTP server returned a redirect error that would " \
+              "lead to an infinite loop.\n" \
+              "The last 30x error message was:\n"
+
+
+def _parse_proxy(proxy):
+    """Return (scheme, user, password, host/port) given a URL or an authority.
+
+    If a URL is supplied, it must have an authority (host:port) component.
+    According to RFC 3986, having an authority component means the URL must
+    have two slashes after the scheme:
+
+    >>> _parse_proxy('file:/ftp.example.com/')
+    Traceback (most recent call last):
+    ValueError: proxy URL with no authority: 'file:/ftp.example.com/'
+
+    The first three items of the returned tuple may be None.
+
+    Examples of authority parsing:
+
+    >>> _parse_proxy('proxy.example.com')
+    (None, None, None, 'proxy.example.com')
+    >>> _parse_proxy('proxy.example.com:3128')
+    (None, None, None, 'proxy.example.com:3128')
+
+    The authority component may optionally include userinfo (assumed to be
+    username:password):
+
+    >>> _parse_proxy('joe:password at proxy.example.com')
+    (None, 'joe', 'password', 'proxy.example.com')
+    >>> _parse_proxy('joe:password at proxy.example.com:3128')
+    (None, 'joe', 'password', 'proxy.example.com:3128')
+
+    Same examples, but with URLs instead:
+
+    >>> _parse_proxy('http://proxy.example.com/')
+    ('http', None, None, 'proxy.example.com')
+    >>> _parse_proxy('http://proxy.example.com:3128/')
+    ('http', None, None, 'proxy.example.com:3128')
+    >>> _parse_proxy('http://joe:password at proxy.example.com/')
+    ('http', 'joe', 'password', 'proxy.example.com')
+    >>> _parse_proxy('http://joe:password at proxy.example.com:3128')
+    ('http', 'joe', 'password', 'proxy.example.com:3128')
+
+    Everything after the authority is ignored:
+
+    >>> _parse_proxy('ftp://joe:password at proxy.example.com/rubbish:3128')
+    ('ftp', 'joe', 'password', 'proxy.example.com')
+
+    Test for no trailing '/' case:
+
+    >>> _parse_proxy('http://joe:password at proxy.example.com')
+    ('http', 'joe', 'password', 'proxy.example.com')
+
+    """
+    scheme, r_scheme = splittype(proxy)
+    if not r_scheme.startswith("/"):
+        # authority
+        scheme = None
+        authority = proxy
+    else:
+        # URL
+        if not r_scheme.startswith("//"):
+            raise ValueError("proxy URL with no authority: %r" % proxy)
+        # We have an authority, so for RFC 3986-compliant URLs (by ss 3.
+        # and 3.3.), path is empty or starts with '/'
+        end = r_scheme.find("/", 2)
+        if end == -1:
+            end = None
+        authority = r_scheme[2:end]
+    userinfo, hostport = splituser(authority)
+    if userinfo is not None:
+        user, password = splitpasswd(userinfo)
+    else:
+        user = password = None
+    return scheme, user, password, hostport
+
+class ProxyHandler(BaseHandler):
+    # Proxies must be in front
+    handler_order = 100
+
+    def __init__(self, proxies=None):
+        if proxies is None:
+            proxies = getproxies()
+        assert hasattr(proxies, 'has_key'), "proxies must be a mapping"
+        self.proxies = proxies
+        for type, url in proxies.items():
+            setattr(self, '%s_open' % type,
+                    lambda r, proxy=url, type=type, meth=self.proxy_open: \
+                    meth(r, proxy, type))
+
+    def proxy_open(self, req, proxy, type):
+        orig_type = req.get_type()
+        proxy_type, user, password, hostport = _parse_proxy(proxy)
+
+        if proxy_type is None:
+            proxy_type = orig_type
+
+        if req.host and proxy_bypass(req.host):
+            return None
+
+        if user and password:
+            user_pass = '%s:%s' % (unquote(user), unquote(password))
+            creds = base64.b64encode(user_pass).strip()
+            req.add_header('Proxy-authorization', 'Basic ' + creds)
+        hostport = unquote(hostport)
+        req.set_proxy(hostport, proxy_type)
+
+        if orig_type == proxy_type or orig_type == 'https':
+            # let other handlers take care of it
+            return None
+        else:
+            # need to start over, because the other handlers don't
+            # grok the proxy's URL type
+            # e.g. if we have a constructor arg proxies like so:
+            # {'http': 'ftp://proxy.example.com'}, we may end up turning
+            # a request for http://acme.example.com/a into one for
+            # ftp://proxy.example.com/a
+            return self.parent.open(req, timeout=req.timeout)
+
+class HTTPPasswordMgr:
+
+    def __init__(self):
+        self.passwd = {}
+
+    def add_password(self, realm, uri, user, passwd):
+        # uri could be a single URI or a sequence
+        if isinstance(uri, basestring):
+            uri = [uri]
+        if not realm in self.passwd:
+            self.passwd[realm] = {}
+        for default_port in True, False:
+            reduced_uri = tuple(
+                [self.reduce_uri(u, default_port) for u in uri])
+            self.passwd[realm][reduced_uri] = (user, passwd)
+
+    def find_user_password(self, realm, authuri):
+        domains = self.passwd.get(realm, {})
+        for default_port in True, False:
+            reduced_authuri = self.reduce_uri(authuri, default_port)
+            for uris, authinfo in domains.iteritems():
+                for uri in uris:
+                    if self.is_suburi(uri, reduced_authuri):
+                        return authinfo
+        return None, None
+
+    def reduce_uri(self, uri, default_port=True):
+        """Accept authority or URI and extract only the authority and path."""
+        # note HTTP URLs do not have a userinfo component
+        parts = urlparse.urlsplit(uri)
+        if parts[1]:
+            # URI
+            scheme = parts[0]
+            authority = parts[1]
+            path = parts[2] or '/'
+        else:
+            # host or host:port
+            scheme = None
+            authority = uri
+            path = '/'
+        host, port = splitport(authority)
+        if default_port and port is None and scheme is not None:
+            dport = {"http": 80,
+                     "https": 443,
+                     }.get(scheme)
+            if dport is not None:
+                authority = "%s:%d" % (host, dport)
+        return authority, path
+
+    def is_suburi(self, base, test):
+        """Check if test is below base in a URI tree
+
+        Both args must be URIs in reduced form.
+        """
+        if base == test:
+            return True
+        if base[0] != test[0]:
+            return False
+        common = posixpath.commonprefix((base[1], test[1]))
+        if len(common) == len(base[1]):
+            return True
+        return False
+
+
+class HTTPPasswordMgrWithDefaultRealm(HTTPPasswordMgr):
+
+    def find_user_password(self, realm, authuri):
+        user, password = HTTPPasswordMgr.find_user_password(self, realm,
+                                                            authuri)
+        if user is not None:
+            return user, password
+        return HTTPPasswordMgr.find_user_password(self, None, authuri)
+
+
+class AbstractBasicAuthHandler:
+
+    # XXX this allows for multiple auth-schemes, but will stupidly pick
+    # the last one with a realm specified.
+
+    # allow for double- and single-quoted realm values
+    # (single quotes are a violation of the RFC, but appear in the wild)
+    rx = re.compile('(?:.*,)*[ \t]*([^ \t]+)[ \t]+'
+                    'realm=(["\'])(.*?)\\2', re.I)
+
+    # XXX could pre-emptively send auth info already accepted (RFC 2617,
+    # end of section 2, and section 1.2 immediately after "credentials"
+    # production).
+
+    def __init__(self, password_mgr=None):
+        if password_mgr is None:
+            password_mgr = HTTPPasswordMgr()
+        self.passwd = password_mgr
+        self.add_password = self.passwd.add_password
+        self.retried = 0
+
+    def reset_retry_count(self):
+        self.retried = 0
+
+    def http_error_auth_reqed(self, authreq, host, req, headers):
+        # host may be an authority (without userinfo) or a URL with an
+        # authority
+        # XXX could be multiple headers
+        authreq = headers.get(authreq, None)
+
+        if self.retried > 5:
+            # retry sending the username:password 5 times before failing.
+            raise HTTPError(req.get_full_url(), 401, "basic auth failed",
+                            headers, None)
+        else:
+            self.retried += 1
+
+        if authreq:
+            mo = AbstractBasicAuthHandler.rx.search(authreq)
+            if mo:
+                scheme, quote, realm = mo.groups()
+                if scheme.lower() == 'basic':
+                    response = self.retry_http_basic_auth(host, req, realm)
+                    if response and response.code != 401:
+                        self.retried = 0
+                    return response
+
+    def retry_http_basic_auth(self, host, req, realm):
+        user, pw = self.passwd.find_user_password(realm, host)
+        if pw is not None:
+            raw = "%s:%s" % (user, pw)
+            auth = 'Basic %s' % base64.b64encode(raw).strip()
+            if req.headers.get(self.auth_header, None) == auth:
+                return None
+            req.add_unredirected_header(self.auth_header, auth)
+            return self.parent.open(req, timeout=req.timeout)
+        else:
+            return None
+
+
+class HTTPBasicAuthHandler(AbstractBasicAuthHandler, BaseHandler):
+
+    auth_header = 'Authorization'
+
+    def http_error_401(self, req, fp, code, msg, headers):
+        url = req.get_full_url()
+        response = self.http_error_auth_reqed('www-authenticate',
+                                              url, req, headers)
+        self.reset_retry_count()
+        return response
+
+
+class ProxyBasicAuthHandler(AbstractBasicAuthHandler, BaseHandler):
+
+    auth_header = 'Proxy-authorization'
+
+    def http_error_407(self, req, fp, code, msg, headers):
+        # http_error_auth_reqed requires that there is no userinfo component in
+        # authority.  Assume there isn't one, since urllib2 does not (and
+        # should not, RFC 3986 s. 3.2.1) support requests for URLs containing
+        # userinfo.
+        authority = req.get_host()
+        response = self.http_error_auth_reqed('proxy-authenticate',
+                                          authority, req, headers)
+        self.reset_retry_count()
+        return response
+
+
+def randombytes(n):
+    """Return n random bytes."""
+    # Use /dev/urandom if it is available.  Fall back to random module
+    # if not.  It might be worthwhile to extend this function to use
+    # other platform-specific mechanisms for getting random bytes.
+    if os.path.exists("/dev/urandom"):
+        f = open("/dev/urandom")
+        s = f.read(n)
+        f.close()
+        return s
+    else:
+        L = [chr(random.randrange(0, 256)) for i in range(n)]
+        return "".join(L)
+
+class AbstractDigestAuthHandler:
+    # Digest authentication is specified in RFC 2617.
+
+    # XXX The client does not inspect the Authentication-Info header
+    # in a successful response.
+
+    # XXX It should be possible to test this implementation against
+    # a mock server that just generates a static set of challenges.
+
+    # XXX qop="auth-int" supports is shaky
+
+    def __init__(self, passwd=None):
+        if passwd is None:
+            passwd = HTTPPasswordMgr()
+        self.passwd = passwd
+        self.add_password = self.passwd.add_password
+        self.retried = 0
+        self.nonce_count = 0
+        self.last_nonce = None
+
+    def reset_retry_count(self):
+        self.retried = 0
+
+    def http_error_auth_reqed(self, auth_header, host, req, headers):
+        authreq = headers.get(auth_header, None)
+        if self.retried > 5:
+            # Don't fail endlessly - if we failed once, we'll probably
+            # fail a second time. Hm. Unless the Password Manager is
+            # prompting for the information. Crap. This isn't great
+            # but it's better than the current 'repeat until recursion
+            # depth exceeded' approach <wink>
+            raise HTTPError(req.get_full_url(), 401, "digest auth failed",
+                            headers, None)
+        else:
+            self.retried += 1
+        if authreq:
+            scheme = authreq.split()[0]
+            if scheme.lower() == 'digest':
+                return self.retry_http_digest_auth(req, authreq)
+
+    def retry_http_digest_auth(self, req, auth):
+        token, challenge = auth.split(' ', 1)
+        chal = parse_keqv_list(parse_http_list(challenge))
+        auth = self.get_authorization(req, chal)
+        if auth:
+            auth_val = 'Digest %s' % auth
+            if req.headers.get(self.auth_header, None) == auth_val:
+                return None
+            req.add_unredirected_header(self.auth_header, auth_val)
+            resp = self.parent.open(req, timeout=req.timeout)
+            return resp
+
+    def get_cnonce(self, nonce):
+        # The cnonce-value is an opaque
+        # quoted string value provided by the client and used by both client
+        # and server to avoid chosen plaintext attacks, to provide mutual
+        # authentication, and to provide some message integrity protection.
+        # This isn't a fabulous effort, but it's probably Good Enough.
+        dig = hashlib.sha1("%s:%s:%s:%s" % (self.nonce_count, nonce, time.ctime(),
+                                            randombytes(8))).hexdigest()
+        return dig[:16]
+
+    def get_authorization(self, req, chal):
+        try:
+            realm = chal['realm']
+            nonce = chal['nonce']
+            qop = chal.get('qop')
+            algorithm = chal.get('algorithm', 'MD5')
+            # mod_digest doesn't send an opaque, even though it isn't
+            # supposed to be optional
+            opaque = chal.get('opaque', None)
+        except KeyError:
+            return None
+
+        H, KD = self.get_algorithm_impls(algorithm)
+        if H is None:
+            return None
+
+        user, pw = self.passwd.find_user_password(realm, req.get_full_url())
+        if user is None:
+            return None
+
+        # XXX not implemented yet
+        if req.has_data():
+            entdig = self.get_entity_digest(req.get_data(), chal)
+        else:
+            entdig = None
+
+        A1 = "%s:%s:%s" % (user, realm, pw)
+        A2 = "%s:%s" % (req.get_method(),
+                        # XXX selector: what about proxies and full urls
+                        req.get_selector())
+        if qop == 'auth':
+            if nonce == self.last_nonce:
+                self.nonce_count += 1
+            else:
+                self.nonce_count = 1
+                self.last_nonce = nonce
+
+            ncvalue = '%08x' % self.nonce_count
+            cnonce = self.get_cnonce(nonce)
+            noncebit = "%s:%s:%s:%s:%s" % (nonce, ncvalue, cnonce, qop, H(A2))
+            respdig = KD(H(A1), noncebit)
+        elif qop is None:
+            respdig = KD(H(A1), "%s:%s" % (nonce, H(A2)))
+        else:
+            # XXX handle auth-int.
+            raise URLError("qop '%s' is not supported." % qop)
+
+        # XXX should the partial digests be encoded too?
+
+        base = 'username="%s", realm="%s", nonce="%s", uri="%s", ' \
+               'response="%s"' % (user, realm, nonce, req.get_selector(),
+                                  respdig)
+        if opaque:
+            base += ', opaque="%s"' % opaque
+        if entdig:
+            base += ', digest="%s"' % entdig
+        base += ', algorithm="%s"' % algorithm
+        if qop:
+            base += ', qop=auth, nc=%s, cnonce="%s"' % (ncvalue, cnonce)
+        return base
+
+    def get_algorithm_impls(self, algorithm):
+        # algorithm should be case-insensitive according to RFC2617
+        algorithm = algorithm.upper()
+        # lambdas assume digest modules are imported at the top level
+        if algorithm == 'MD5':
+            H = lambda x: hashlib.md5(x).hexdigest()
+        elif algorithm == 'SHA':
+            H = lambda x: hashlib.sha1(x).hexdigest()
+        # XXX MD5-sess
+        KD = lambda s, d: H("%s:%s" % (s, d))
+        return H, KD
+
+    def get_entity_digest(self, data, chal):
+        # XXX not implemented yet
+        return None
+
+
+class HTTPDigestAuthHandler(BaseHandler, AbstractDigestAuthHandler):
+    """An authentication protocol defined by RFC 2069
+
+    Digest authentication improves on basic authentication because it
+    does not transmit passwords in the clear.
+    """
+
+    auth_header = 'Authorization'
+    handler_order = 490  # before Basic auth
+
+    def http_error_401(self, req, fp, code, msg, headers):
+        host = urlparse.urlparse(req.get_full_url())[1]
+        retry = self.http_error_auth_reqed('www-authenticate',
+                                           host, req, headers)
+        self.reset_retry_count()
+        return retry
+
+
+class ProxyDigestAuthHandler(BaseHandler, AbstractDigestAuthHandler):
+
+    auth_header = 'Proxy-Authorization'
+    handler_order = 490  # before Basic auth
+
+    def http_error_407(self, req, fp, code, msg, headers):
+        host = req.get_host()
+        retry = self.http_error_auth_reqed('proxy-authenticate',
+                                           host, req, headers)
+        self.reset_retry_count()
+        return retry
+
+class AbstractHTTPHandler(BaseHandler):
+
+    def __init__(self, debuglevel=0):
+        self._debuglevel = debuglevel
+
+    def set_http_debuglevel(self, level):
+        self._debuglevel = level
+
+    def do_request_(self, request):
+        host = request.get_host()
+        if not host:
+            raise URLError('no host given')
+
+        if request.has_data():  # POST
+            data = request.get_data()
+            if not request.has_header('Content-type'):
+                request.add_unredirected_header(
+                    'Content-type',
+                    'application/x-www-form-urlencoded')
+            if not request.has_header('Content-length'):
+                request.add_unredirected_header(
+                    'Content-length', '%d' % len(data))
+
+        sel_host = host
+        if request.has_proxy():
+            scheme, sel = splittype(request.get_selector())
+            sel_host, sel_path = splithost(sel)
+
+        if not request.has_header('Host'):
+            request.add_unredirected_header('Host', sel_host)
+        for name, value in self.parent.addheaders:
+            name = name.capitalize()
+            if not request.has_header(name):
+                request.add_unredirected_header(name, value)
+
+        return request
+
+    def do_open(self, http_class, req):
+        """Return an addinfourl object for the request, using http_class.
+
+        http_class must implement the HTTPConnection API from httplib.
+        The addinfourl return value is a file-like object.  It also
+        has methods and attributes including:
+            - info(): return a mimetools.Message object for the headers
+            - geturl(): return the original request URL
+            - code: HTTP status code
+        """
+        host = req.get_host()
+        if not host:
+            raise URLError('no host given')
+
+        h = http_class(host, timeout=req.timeout) # will parse host:port
+        h.set_debuglevel(self._debuglevel)
+
+        headers = dict(req.unredirected_hdrs)
+        headers.update(dict((k, v) for k, v in req.headers.items()
+                            if k not in headers))
+
+        # We want to make an HTTP/1.1 request, but the addinfourl
+        # class isn't prepared to deal with a persistent connection.
+        # It will try to read all remaining data from the socket,
+        # which will block while the server waits for the next request.
+        # So make sure the connection gets closed after the (only)
+        # request.
+        headers["Connection"] = "close"
+        headers = dict(
+            (name.title(), val) for name, val in headers.items())
+
+        if req._tunnel_host:
+            tunnel_headers = {}
+            proxy_auth_hdr = "Proxy-Authorization"
+            if proxy_auth_hdr in headers:
+                tunnel_headers[proxy_auth_hdr] = headers[proxy_auth_hdr]
+                # Proxy-Authorization should not be sent to origin
+                # server.
+                del headers[proxy_auth_hdr]
+            h.set_tunnel(req._tunnel_host, headers=tunnel_headers)
+
+        try:
+            h.request(req.get_method(), req.get_selector(), req.data, headers)
+            try:
+                r = h.getresponse(buffering=True)
+            except TypeError: #buffering kw not supported
+                r = h.getresponse()
+        except socket.error, err: # XXX what error?
+            h.close()
+            raise URLError(err)
+
+        # Pick apart the HTTPResponse object to get the addinfourl
+        # object initialized properly.
+
+        # Wrap the HTTPResponse object in socket's file object adapter
+        # for Windows.  That adapter calls recv(), so delegate recv()
+        # to read().  This weird wrapping allows the returned object to
+        # have readline() and readlines() methods.
+
+        # XXX It might be better to extract the read buffering code
+        # out of socket._fileobject() and into a base class.
+
+        r.recv = r.read
+        fp = socket._fileobject(r, close=True)
+
+        resp = addinfourl(fp, r.msg, req.get_full_url())
+        resp.code = r.status
+        resp.msg = r.reason
+        return resp
+
+
+class HTTPHandler(AbstractHTTPHandler):
+
+    def http_open(self, req):
+        return self.do_open(httplib.HTTPConnection, req)
+
+    http_request = AbstractHTTPHandler.do_request_
+
+if hasattr(httplib, 'HTTPS'):
+    class HTTPSHandler(AbstractHTTPHandler):
+
+        def https_open(self, req):
+            return self.do_open(httplib.HTTPSConnection, req)
+
+        https_request = AbstractHTTPHandler.do_request_
+
+class HTTPCookieProcessor(BaseHandler):
+    def __init__(self, cookiejar=None):
+        import cookielib
+        if cookiejar is None:
+            cookiejar = cookielib.CookieJar()
+        self.cookiejar = cookiejar
+
+    def http_request(self, request):
+        self.cookiejar.add_cookie_header(request)
+        return request
+
+    def http_response(self, request, response):
+        self.cookiejar.extract_cookies(response, request)
+        return response
+
+    https_request = http_request
+    https_response = http_response
+
+class UnknownHandler(BaseHandler):
+    def unknown_open(self, req):
+        type = req.get_type()
+        raise URLError('unknown url type: %s' % type)
+
+def parse_keqv_list(l):
+    """Parse list of key=value strings where keys are not duplicated."""
+    parsed = {}
+    for elt in l:
+        k, v = elt.split('=', 1)
+        if v[0] == '"' and v[-1] == '"':
+            v = v[1:-1]
+        parsed[k] = v
+    return parsed
+
+def parse_http_list(s):
+    """Parse lists as described by RFC 2068 Section 2.
+
+    In particular, parse comma-separated lists where the elements of
+    the list may include quoted-strings.  A quoted-string could
+    contain a comma.  A non-quoted string could have quotes in the
+    middle.  Neither commas nor quotes count if they are escaped.
+    Only double-quotes count, not single-quotes.
+    """
+    res = []
+    part = ''
+
+    escape = quote = False
+    for cur in s:
+        if escape:
+            part += cur
+            escape = False
+            continue
+        if quote:
+            if cur == '\\':
+                escape = True
+                continue
+            elif cur == '"':
+                quote = False
+            part += cur
+            continue
+
+        if cur == ',':
+            res.append(part)
+            part = ''
+            continue
+
+        if cur == '"':
+            quote = True
+
+        part += cur
+
+    # append last part
+    if part:
+        res.append(part)
+
+    return [part.strip() for part in res]
+
+def _safe_gethostbyname(host):
+    try:
+        return socket.gethostbyname(host)
+    except socket.gaierror:
+        return None
+
+class FileHandler(BaseHandler):
+    # Use local file or FTP depending on form of URL
+    def file_open(self, req):
+        url = req.get_selector()
+        if url[:2] == '//' and url[2:3] != '/' and (req.host and
+                req.host != 'localhost'):
+            req.type = 'ftp'
+            return self.parent.open(req)
+        else:
+            return self.open_local_file(req)
+
+    # names for the localhost
+    names = None
+    def get_names(self):
+        if FileHandler.names is None:
+            try:
+                FileHandler.names = tuple(
+                    socket.gethostbyname_ex('localhost')[2] +
+                    socket.gethostbyname_ex(socket.gethostname())[2])
+            except socket.gaierror:
+                FileHandler.names = (socket.gethostbyname('localhost'),)
+        return FileHandler.names
+
+    # not entirely sure what the rules are here
+    def open_local_file(self, req):
+        import email.utils
+        import mimetypes
+        host = req.get_host()
+        filename = req.get_selector()
+        localfile = url2pathname(filename)
+        try:
+            stats = os.stat(localfile)
+            size = stats.st_size
+            modified = email.utils.formatdate(stats.st_mtime, usegmt=True)
+            mtype = mimetypes.guess_type(filename)[0]
+            headers = mimetools.Message(StringIO(
+                'Content-type: %s\nContent-length: %d\nLast-modified: %s\n' %
+                (mtype or 'text/plain', size, modified)))
+            if host:
+                host, port = splitport(host)
+            if not host or \
+                (not port and _safe_gethostbyname(host) in self.get_names()):
+                if host:
+                    origurl = 'file://' + host + filename
+                else:
+                    origurl = 'file://' + filename
+                return addinfourl(open(localfile, 'rb'), headers, origurl)
+        except OSError, msg:
+            # urllib2 users shouldn't expect OSErrors coming from urlopen()
+            raise URLError(msg)
+        raise URLError('file not on local host')
+
+class FTPHandler(BaseHandler):
+    def ftp_open(self, req):
+        import ftplib
+        import mimetypes
+        host = req.get_host()
+        if not host:
+            raise URLError('ftp error: no host given')
+        host, port = splitport(host)
+        if port is None:
+            port = ftplib.FTP_PORT
+        else:
+            port = int(port)
+
+        # username/password handling
+        user, host = splituser(host)
+        if user:
+            user, passwd = splitpasswd(user)
+        else:
+            passwd = None
+        host = unquote(host)
+        user = user or ''
+        passwd = passwd or ''
+
+        try:
+            host = socket.gethostbyname(host)
+        except socket.error, msg:
+            raise URLError(msg)
+        path, attrs = splitattr(req.get_selector())
+        dirs = path.split('/')
+        dirs = map(unquote, dirs)
+        dirs, file = dirs[:-1], dirs[-1]
+        if dirs and not dirs[0]:
+            dirs = dirs[1:]
+        try:
+            fw = self.connect_ftp(user, passwd, host, port, dirs, req.timeout)
+            type = file and 'I' or 'D'
+            for attr in attrs:
+                attr, value = splitvalue(attr)
+                if attr.lower() == 'type' and \
+                   value in ('a', 'A', 'i', 'I', 'd', 'D'):
+                    type = value.upper()
+            fp, retrlen = fw.retrfile(file, type)
+            headers = ""
+            mtype = mimetypes.guess_type(req.get_full_url())[0]
+            if mtype:
+                headers += "Content-type: %s\n" % mtype
+            if retrlen is not None and retrlen >= 0:
+                headers += "Content-length: %d\n" % retrlen
+            sf = StringIO(headers)
+            headers = mimetools.Message(sf)
+            return addinfourl(fp, headers, req.get_full_url())
+        except ftplib.all_errors, msg:
+            raise URLError, ('ftp error: %s' % msg), sys.exc_info()[2]
+
+    def connect_ftp(self, user, passwd, host, port, dirs, timeout):
+        fw = ftpwrapper(user, passwd, host, port, dirs, timeout)
+##        fw.ftp.set_debuglevel(1)
+        return fw
+
+class CacheFTPHandler(FTPHandler):
+    # XXX would be nice to have pluggable cache strategies
+    # XXX this stuff is definitely not thread safe
+    def __init__(self):
+        self.cache = {}
+        self.timeout = {}
+        self.soonest = 0
+        self.delay = 60
+        self.max_conns = 16
+
+    def setTimeout(self, t):
+        self.delay = t
+
+    def setMaxConns(self, m):
+        self.max_conns = m
+
+    def connect_ftp(self, user, passwd, host, port, dirs, timeout):
+        key = user, host, port, '/'.join(dirs), timeout
+        if key in self.cache:
+            self.timeout[key] = time.time() + self.delay
+        else:
+            self.cache[key] = ftpwrapper(user, passwd, host, port, dirs, timeout)
+            self.timeout[key] = time.time() + self.delay
+        self.check_cache()
+        return self.cache[key]
+
+    def check_cache(self):
+        # first check for old ones
+        t = time.time()
+        if self.soonest <= t:
+            for k, v in self.timeout.items():
+                if v < t:
+                    self.cache[k].close()
+                    del self.cache[k]
+                    del self.timeout[k]
+        self.soonest = min(self.timeout.values())
+
+        # then check the size
+        if len(self.cache) == self.max_conns:
+            for k, v in self.timeout.items():
+                if v == self.soonest:
+                    del self.cache[k]
+                    del self.timeout[k]
+                    break
+            self.soonest = min(self.timeout.values())

From noreply at buildbot.pypy.org  Sat Oct  8 18:51:05 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Sat,  8 Oct 2011 18:51:05 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20111008165105.81C5F82A88@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r47881:abb1978e51e0
Date: 2011-10-08 18:50 +0200
http://bitbucket.org/pypy/pypy/changeset/abb1978e51e0/

Log:	merge

diff --git a/pypy/translator/c/genc.py b/pypy/translator/c/genc.py
--- a/pypy/translator/c/genc.py
+++ b/pypy/translator/c/genc.py
@@ -8,7 +8,7 @@
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
 from pypy.rpython.lltypesystem import lltype
 from pypy.tool.udir import udir
-from pypy.tool import isolate
+from pypy.tool import isolate, runsubprocess
 from pypy.translator.c.support import log, c_string_constant
 from pypy.rpython.typesystem import getfunctionptr
 from pypy.translator.c import gc
@@ -563,14 +563,19 @@
             else:
                 mk.definition('PYPY_MAIN_FUNCTION', "main")
 
-            if (py.path.local.sysfind('python') or
-                py.path.local.sysfind('python.exe')):
-                python = 'python '
-            elif sys.platform == 'win32':
+            if sys.platform == 'win32':
                 python = sys.executable.replace('\\', '/') + ' '
             else:
                 python = sys.executable + ' '
 
+            # Is there a command 'python' that runs python 2.5-2.7?
+            # If there is, then we can use it instead of sys.executable
+            returncode, stdout, stderr = runsubprocess.run_subprocess(
+                "python", "-V")
+            if (stdout.startswith('Python 2.') or
+                stderr.startswith('Python 2.')):
+                python = 'python '
+
             if self.translator.platform.name == 'msvc':
                 lblofiles = []
                 for cfile in mk.cfiles:
diff --git a/pypy/translator/goal/app_main.py b/pypy/translator/goal/app_main.py
--- a/pypy/translator/goal/app_main.py
+++ b/pypy/translator/goal/app_main.py
@@ -144,7 +144,7 @@
     print '  --jit off                  turn off the JIT'
 
 def print_version(*args):
-    print "Python", sys.version
+    print >> sys.stderr, "Python", sys.version
     raise SystemExit
 
 def set_jit_option(options, jitparam, *args):

From noreply at buildbot.pypy.org  Sat Oct  8 21:06:29 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Sat,  8 Oct 2011 21:06:29 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix the case of nonblocking sys.stdin
Message-ID: <20111008190629.4D5C982A87@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r47882:3f96afe7cdc2
Date: 2011-10-08 21:06 +0200
http://bitbucket.org/pypy/pypy/changeset/3f96afe7cdc2/

Log:	fix the case of nonblocking sys.stdin

diff --git a/pypy/module/_file/test/test_file.py b/pypy/module/_file/test/test_file.py
--- a/pypy/module/_file/test/test_file.py
+++ b/pypy/module/_file/test/test_file.py
@@ -1,5 +1,5 @@
 from __future__ import with_statement
-import py
+import py, os, errno
 
 from pypy.conftest import gettestobjspace, option
 
@@ -257,6 +257,35 @@
             assert self.temppath in g.getvalue()
 
 
+class AppTestNonblocking(object):
+    def setup_class(cls):
+        from pypy.module._file.interp_file import W_File
+        
+        cls.old_read = os.read
+        state = [0]
+        def read(fd, n=None):
+            if fd != 42:
+                return cls.old_read(fd, n)
+            if state[0] == 0:
+                state[0] += 1
+                return "xyz"
+            if state[0] < 3:
+                state[0] += 1
+                raise OSError(errno.EAGAIN, "xyz")
+            return ''
+        os.read = read
+        stdin = W_File(cls.space)
+        stdin.file_fdopen(42, "r", 1)
+        stdin.name = '<stdin>'
+        cls.w_stream = stdin
+
+    def teardown_class(cls):
+        os.read = cls.old_read
+        
+    def test_nonblocking_file(self):
+        res = self.stream.read()
+        assert res == 'xyz'
+
 class AppTestConcurrency(object):
     # these tests only really make sense on top of a translated pypy-c,
     # because on top of py.py the inner calls to os.write() don't
diff --git a/pypy/rlib/streamio.py b/pypy/rlib/streamio.py
--- a/pypy/rlib/streamio.py
+++ b/pypy/rlib/streamio.py
@@ -37,7 +37,7 @@
 # return value of tell(), but not as argument to read().
 #
 
-import os, sys
+import os, sys, errno
 from pypy.rlib.objectmodel import specialize, we_are_translated
 from pypy.rlib.rarithmetic import r_longlong, intmask
 from pypy.rlib import rposix
@@ -587,12 +587,22 @@
     def readall(self):
         pos = self.pos
         assert pos >= 0
-        chunks = [self.buf[pos:]]
+        if self.buf:
+            chunks = [self.buf[pos:]]
+        else:
+            chunks = []
         self.buf = ""
         self.pos = 0
         bufsize = self.bufsize
         while 1:
-            data = self.do_read(bufsize)
+            try:
+                data = self.do_read(bufsize)
+            except OSError, o:
+                if o.errno != errno.EAGAIN:
+                    raise
+                if not chunks:
+                    raise
+                break
             if not data:
                 break
             chunks.append(data)

From noreply at buildbot.pypy.org  Sat Oct  8 21:35:11 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Sat,  8 Oct 2011 21:35:11 +0200 (CEST)
Subject: [pypy-commit] benchmarks default: run chameleon 3x as long
Message-ID: <20111008193512.0150F82A87@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r146:de46a8229ddd
Date: 2011-10-08 21:35 +0200
http://bitbucket.org/pypy/benchmarks/changeset/de46a8229ddd/

Log:	run chameleon 3x as long

diff --git a/benchmarks.py b/benchmarks.py
--- a/benchmarks.py
+++ b/benchmarks.py
@@ -42,7 +42,8 @@
 opts = {
     'gcbench' : {'iteration_scaling' : .10},
     'bm_mako' : {'bm_env': {'PYTHONPATH': relative('lib/mako')}},
-    'bm_chameleon': {'bm_env': {'PYTHONPATH': relative('lib/chameleon/src')}},
+    'bm_chameleon': {'bm_env': {'PYTHONPATH': relative('lib/chameleon/src')},
+                     'iteration_scaling': 3},
 }
 
 for name in ['expand', 'integrate', 'sum', 'str']:

From notifications-noreply at bitbucket.org  Sun Oct  9 00:17:16 2011
From: notifications-noreply at bitbucket.org (Bitbucket)
Date: Sat, 08 Oct 2011 22:17:16 -0000
Subject: [pypy-commit] Notification: pypy
Message-ID: <20111008221716.8119.11723@bitbucket01.managed.contegix.com>

You have received a notification from Stefano Rivera.

Hi, I forked pypy. My fork is at https://bitbucket.org/stefanor/pypy.

--
Disable notifications at https://bitbucket.org/account/notifications/

From noreply at buildbot.pypy.org  Sun Oct  9 01:28:14 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Sun,  9 Oct 2011 01:28:14 +0200 (CEST)
Subject: [pypy-commit] pypy lightweight-finalizers: leave a comment how this
	is buggy
Message-ID: <20111008232814.1297582A87@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: lightweight-finalizers
Changeset: r47883:66cdca63f106
Date: 2011-10-09 01:25 +0200
http://bitbucket.org/pypy/pypy/changeset/66cdca63f106/

Log:	leave a comment how this is buggy

diff --git a/pypy/rlib/rmmap.py b/pypy/rlib/rmmap.py
--- a/pypy/rlib/rmmap.py
+++ b/pypy/rlib/rmmap.py
@@ -292,6 +292,9 @@
         elif _POSIX:
             self.closed = True
             if self.fd != -1:
+                # XXX this is buggy - raising in an RPython del is not a good
+                #     idea, we should swallow the exception or ignore the
+                #     underlaying close error code
                 os.close(self.fd)
                 self.fd = -1
             if self.size > 0:

From noreply at buildbot.pypy.org  Sun Oct  9 01:28:15 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Sun,  9 Oct 2011 01:28:15 +0200 (CEST)
Subject: [pypy-commit] pypy lightweight-finalizers: add FinalizerAnalyzer
 that checks whether an RPython del is lightweight
Message-ID: <20111008232815.48EF982A87@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: lightweight-finalizers
Changeset: r47884:ef34d4cdce31
Date: 2011-10-09 01:28 +0200
http://bitbucket.org/pypy/pypy/changeset/ef34d4cdce31/

Log:	add FinalizerAnalyzer that checks whether an RPython del is
	lightweight

diff --git a/pypy/translator/backendopt/finalizer.py b/pypy/translator/backendopt/finalizer.py
new file mode 100644
--- /dev/null
+++ b/pypy/translator/backendopt/finalizer.py
@@ -0,0 +1,31 @@
+
+from pypy.translator.backendopt import graphanalyze
+from pypy.rpython.lltypesystem import lltype
+
+class FinalizerAnalyzer(graphanalyze.BoolGraphAnalyzer):
+    """ Analyzer that determines whether a finalizer is lightweight enough
+    so it can be called without all the complicated logic in the garbage
+    collector. The set of operations here is restrictive for a good reason
+    - it's better to be safe. Specifically disallowed operations:
+
+    * anything that escapes self
+    * anything that can allocate
+    """
+    ok_operations = ['getfield', 'ptr_nonzero', 'free', 'same_as',
+                     'direct_ptradd', 'force_cast', 'cast_primitive',
+                     'cast_pointer']
+    
+    def analyze_simple_operation(self, op, graphinfo):
+        if op.opname in self.ok_operations:
+            return self.bottom_result()
+        if op.opname.startswith('int_') or op.opname.startswith('float_'):
+            return self.bottom_result()
+        if op.opname == 'setfield':
+            TP = op.args[2].concretetype
+            if not isinstance(TP, lltype.Ptr) or TP.TO._gckind == 'raw':
+                # primitive type
+                return self.bottom_result()
+        print op
+        import pdb
+        pdb.set_trace()
+        return self.top_result()
diff --git a/pypy/translator/backendopt/test/test_finalizer.py b/pypy/translator/backendopt/test/test_finalizer.py
new file mode 100644
--- /dev/null
+++ b/pypy/translator/backendopt/test/test_finalizer.py
@@ -0,0 +1,94 @@
+
+import py
+from pypy.translator.backendopt.finalizer import FinalizerAnalyzer
+from pypy.translator.translator import TranslationContext, graphof
+from pypy.translator.backendopt.all import backend_optimizations
+from pypy.rpython.lltypesystem import lltype, rffi
+from pypy.conftest import option
+
+
+class BaseFinalizerAnalyzerTests(object):
+    """ Below are typical destructors that we encounter in pypy
+    """
+
+    type_system = None
+    
+    def analyze(self, func, sig, func_to_analyze=None, backendopt=False):
+        if func_to_analyze is None:
+            func_to_analyze = func
+        t = TranslationContext()
+        t.buildannotator().build_types(func, sig)
+        t.buildrtyper(type_system=self.type_system).specialize()
+        if backendopt:
+            backend_optimizations(t)
+        if option.view:
+            t.view()
+        a = FinalizerAnalyzer(t)
+        fgraph = graphof(t, func_to_analyze)
+        result = a.analyze_direct_call(fgraph)
+        return result
+
+    def test_nothing(self):
+        def f():
+            pass
+        r = self.analyze(f, [])
+        assert not r
+
+
+class TestLLType(BaseFinalizerAnalyzerTests):
+    type_system = 'lltype'
+
+    def test_malloc(self):
+        S = lltype.GcStruct('S')
+        
+        def f():
+            return lltype.malloc(S)
+
+        r = self.analyze(f, [])
+        assert r
+
+    def test_raw_free_getfield(self):
+        S = lltype.Struct('S')
+        
+        class A(object):
+            def __init__(self):
+                self.x = lltype.malloc(S, flavor='raw')
+
+            def __del__(self):
+                if self.x:
+                    self.x = lltype.nullptr(S)
+                    lltype.free(self.x, flavor='raw')
+
+        def f():
+            return A()
+
+        r = self.analyze(f, [], A.__del__.im_func)
+        assert not r
+
+    def test_c_call(self):
+        C = rffi.CArray(lltype.Signed)
+        c = rffi.llexternal('x', [lltype.Ptr(C)], lltype.Signed)
+
+        def g():
+            p = lltype.malloc(C, 3, flavor='raw')
+            f(p)
+        
+        def f(p):
+            c(rffi.ptradd(p, 0))
+            lltype.free(p, flavor='raw')
+
+        r = self.analyze(g, [], f, backendopt=True)
+        assert not r
+
+    def test_os_call(self):
+        py.test.skip("can allocate OSError, but also can raise, ignore for now")
+        import os
+        
+        def f(i):
+            os.close(i)
+
+        r = self.analyze(f, [int], backendopt=True)
+        assert not r
+
+class TestOOType(BaseFinalizerAnalyzerTests):
+    type_system = 'ootype'

From noreply at buildbot.pypy.org  Sun Oct  9 01:32:23 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Sun,  9 Oct 2011 01:32:23 +0200 (CEST)
Subject: [pypy-commit] pypy lightweight-finalizers: don't call close on
 socket, instead almost copy the logic. The idea is that
Message-ID: <20111008233223.11FCE82A87@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: lightweight-finalizers
Changeset: r47885:d8f5b3366786
Date: 2011-10-09 01:32 +0200
http://bitbucket.org/pypy/pypy/changeset/d8f5b3366786/

Log:	don't call close on socket, instead almost copy the logic. The idea
	is that we should not raise from an RPython level __del__

diff --git a/pypy/rlib/rsocket.py b/pypy/rlib/rsocket.py
--- a/pypy/rlib/rsocket.py
+++ b/pypy/rlib/rsocket.py
@@ -610,7 +610,10 @@
         self.timeout = defaults.timeout
         
     def __del__(self):
-        self.close()
+        fd = self.fd
+        if fd != _c.INVALID_SOCKET:
+            self.fd = _c.INVALID_SOCKET
+            _c.socketclose(fd)
 
     if hasattr(_c, 'fcntl'):
         def _setblocking(self, block):

From notifications-noreply at bitbucket.org  Sun Oct  9 01:53:43 2011
From: notifications-noreply at bitbucket.org (Bitbucket)
Date: Sat, 08 Oct 2011 23:53:43 -0000
Subject: [pypy-commit] Notification: pypy
Message-ID: <20111008235343.4256.10572@bitbucket13.managed.contegix.com>

You have received a notification from Tim Sally.

Hi, I forked pypy. My fork is at https://bitbucket.org/tsally/pypy.

--
Disable notifications at https://bitbucket.org/account/notifications/

From pullrequests-noreply at bitbucket.org  Sun Oct  9 01:55:54 2011
From: pullrequests-noreply at bitbucket.org (Bitbucket)
Date: Sat, 08 Oct 2011 23:55:54 -0000
Subject: [pypy-commit] [OPEN] Pull request #10 for pypy/pypy: Minor styling
 fixes on in zipfile.py, remove unused variables
Message-ID: <pullreq-683a5aa0aa25@bitbucket.org>

A new pull request has been opened by Dan Loewenherz.

dlo/pypy has changes to be pulled into pypy/pypy.

https://bitbucket.org/pypy/pypy/pull-request/10/minor-styling-fixes-on-in-zipfilepy-remove

Title: Minor styling fixes on in zipfile.py, remove unused variables

The title says it all.

Changes to be pulled:



--
This is an issue notification from bitbucket.org.
You are receiving this either because you are the participating
in a pull request, or you are following it.


From pullrequests-noreply at bitbucket.org  Sun Oct  9 01:57:51 2011
From: pullrequests-noreply at bitbucket.org (Bitbucket)
Date: Sat, 08 Oct 2011 23:57:51 -0000
Subject: [pypy-commit] [REJECTED] Pull request #10 for pypy/pypy: Minor
 styling fixes on in zipfile.py, remove unused variables
In-Reply-To: <pullreq-683a5aa0aa25@bitbucket.org>
References: <pullreq-683a5aa0aa25@bitbucket.org>
Message-ID: <20111008235751.19631.71530@bitbucket01.managed.contegix.com>

Pull request #10 has been rejected by Benjamin Peterson.

dlo/pypy had changes to be pulled into pypy/pypy.

https://bitbucket.org/pypy/pypy/pull-request/10/minor-styling-fixes-on-in-zipfilepy-remove

Please submit to CPython bug tracker.

Rejected changes:



The pull request has been closed.

--
This is an issue notification from bitbucket.org.
You are receiving this either because you are the participating
in a pull request, or you are following it.


From noreply at buildbot.pypy.org  Sun Oct  9 11:31:15 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Sun,  9 Oct 2011 11:31:15 +0200 (CEST)
Subject: [pypy-commit] pypy lightweight-finalizers: change approach - remove
 the owns_raw_memory_decorator, we'll try to detect
Message-ID: <20111009093115.57D7682112@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: lightweight-finalizers
Changeset: r47886:b8be75dae939
Date: 2011-10-09 11:31 +0200
http://bitbucket.org/pypy/pypy/changeset/b8be75dae939/

Log:	change approach - remove the owns_raw_memory_decorator, we'll try to
	detect automatically and act accordingly

diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -3408,30 +3408,6 @@
         assert res == main(1, 10)
         self.check_loops(call=0)
 
-    def test_virtual_lightweight_finalizer(self):
-        py.test.skip("raw mallocs unsupported, otherwise this would be a problem")
-        from pypy.rlib import rgc
-        import gc
-
-        S = lltype.Struct('S', ('x', lltype.Signed))
-        
-        @rgc.owns_raw_memory('p')
-        class A(object):
-            def __init__(self):
-                self.p = lltype.malloc(S, flavor='raw')
-
-        driver = JitDriver(greens = [], reds = ['i', 'a'])
-
-        def f(i):
-            a = None
-            while i > 0:
-                driver.jit_merge_point(i=i, a=a)
-                a = A()
-                i -= 1
-            gc.collect()
-
-        self.meta_interp(f, [10])
-
 
 class TestLLtype(BaseLLtypeTests, LLJitMixin):
     pass
diff --git a/pypy/rlib/rgc.py b/pypy/rlib/rgc.py
--- a/pypy/rlib/rgc.py
+++ b/pypy/rlib/rgc.py
@@ -440,23 +440,3 @@
     def specialize_call(self, hop):
         hop.exception_is_here()
         return hop.genop('gc_typeids_z', [], resulttype = hop.r_result)
-
-# ----------------------------------------------------------------------
-
-def owns_raw_memory(name):
-    """ Declare class as owning raw memory under the attribute name. When
-    object is freed, it'll automatically free the raw memory residing
-    under this attribute
-    """
-    def wrapper(cls):
-        def remove_raw_mem_attr(self):
-            if getattr(self, name):
-                lltype.free(getattr(self, name), flavor='raw')
-            if orig_del is not None:
-                orig_del(self)
-
-        orig_del = getattr(cls, '__del__', None)
-        cls._raw_mem_ptr_name = name
-        cls.__del__ = remove_raw_mem_attr
-        return cls
-    return wrapper
diff --git a/pypy/rlib/rsocket.py b/pypy/rlib/rsocket.py
--- a/pypy/rlib/rsocket.py
+++ b/pypy/rlib/rsocket.py
@@ -15,7 +15,6 @@
 # app-level code for PyPy.
 
 from pypy.rlib.objectmodel import instantiate, keepalive_until_here
-from pypy.rlib.rgc import owns_raw_memory
 from pypy.rlib import _rsocket_rffi as _c
 from pypy.rlib.rarithmetic import intmask
 from pypy.rpython.lltypesystem import lltype, rffi
@@ -57,7 +56,6 @@
 
 _FAMILIES = {}
 
- at owns_raw_memory('addr_p')
 class Address(object):
     """The base class for RPython-level objects representing addresses.
     Fields:  addr    - a _c.sockaddr_ptr (memory owned by the Address instance)
@@ -78,6 +76,10 @@
         self.addr_p = addr
         self.addrlen = addrlen
 
+    def __del__(self):
+        if self.addr_p:
+            lltype.free(self.addr_p, flavor='raw')
+
     def setdata(self, addr, addrlen):
         # initialize self.addr and self.addrlen.  'addr' can be a different
         # pointer type than exactly sockaddr_ptr, and we cast it for you.
diff --git a/pypy/rlib/test/test_rgc.py b/pypy/rlib/test/test_rgc.py
--- a/pypy/rlib/test/test_rgc.py
+++ b/pypy/rlib/test/test_rgc.py
@@ -171,45 +171,3 @@
     x1 = X()
     n = rgc.get_rpy_memory_usage(rgc.cast_instance_to_gcref(x1))
     assert n >= 8 and n <= 64
-
-def test_raw_memory_owner():
-    T = lltype.Struct('X', ('x', lltype.Signed))
-
-    @rgc.owns_raw_memory('p')
-    class X(object):
-        p = lltype.nullptr(T)
-        
-        def __init__(self, arg):
-            if arg:
-                self.p = lltype.malloc(T, flavor='raw')
-    
-    a = X(3)
-    b = X(0)
-    ptr2 = b.p
-    ptr = a.p
-    del a, b
-    gc.collect()
-    assert ptr._was_freed()
-    assert not ptr2
-
-def test_raw_memory_owner_with_del():
-    T = lltype.Struct('X', ('x', lltype.Signed))
-    collected = []
-
-    @rgc.owns_raw_memory('p')
-    class X(object):
-        p = lltype.nullptr(T)
-        
-        def __init__(self, arg):
-            if arg:
-                self.p = lltype.malloc(T, flavor='raw')
-
-        def __del__(self):
-            collected.append(None)
-    
-    a = X(3)
-    ptr = a.p
-    del a
-    gc.collect()
-    assert ptr._was_freed()
-    assert collected
diff --git a/pypy/rpython/lltypesystem/rclass.py b/pypy/rpython/lltypesystem/rclass.py
--- a/pypy/rpython/lltypesystem/rclass.py
+++ b/pypy/rpython/lltypesystem/rclass.py
@@ -391,39 +391,26 @@
     def _setup_repr_final(self):
         AbstractInstanceRepr._setup_repr_final(self)
         if self.gcflavor == 'gc':
-            destrptr = None
-            raw_mem_attr_name = None
-            if self.classdef is not None:
-                classdesc = self.classdef.classdesc
-                if classdesc.lookup('__del__') is not None:
-                    s_func = classdesc.s_read_attribute('__del__')
-                    if classdesc.lookup('_raw_mem_ptr_name'):
-                        # this object has a __del__, but it actually does not
-                        # do much
-                        assert s_func.const.func_name == 'remove_raw_mem_attr',(
-                            "You overloaded __del__ on an object that owns"
-                            " a reference to a low level pointer")
-                        raw_mem_attr_name = classdesc.s_read_attribute(
-                            '_raw_mem_ptr_name').const
-                    else:
-                        source_desc = classdesc.lookup('__del__')
-                        source_classdef = source_desc.getclassdef(None)
-                        source_repr = getinstancerepr(self.rtyper,
-                                                      source_classdef)
-                        assert len(s_func.descriptions) == 1
-                        funcdesc, = s_func.descriptions
-                        graph = funcdesc.getuniquegraph()
-                        self.check_graph_of_del_does_not_call_too_much(graph)
-                        FUNCTYPE = FuncType([Ptr(source_repr.object_type)],
-                                            Void)
-                        destrptr = functionptr(FUNCTYPE, graph.name,
-                                               graph=graph,
-                                               _callable=graph.func)
+            if (self.classdef is not None and
+                self.classdef.classdesc.lookup('__del__') is not None):
+                s_func = self.classdef.classdesc.s_read_attribute('__del__')
+                source_desc = self.classdef.classdesc.lookup('__del__')
+                source_classdef = source_desc.getclassdef(None)
+                source_repr = getinstancerepr(self.rtyper, source_classdef)
+                assert len(s_func.descriptions) == 1
+                funcdesc, = s_func.descriptions
+                graph = funcdesc.getuniquegraph()
+                self.check_graph_of_del_does_not_call_too_much(graph)
+                FUNCTYPE = FuncType([Ptr(source_repr.object_type)], Void)
+                destrptr = functionptr(FUNCTYPE, graph.name,
+                                       graph=graph,
+                                       _callable=graph.func)
+            else:
+                destrptr = None
             OBJECT = OBJECT_BY_FLAVOR[LLFLAVOR[self.gcflavor]]
             self.rtyper.attachRuntimeTypeInfoFunc(self.object_type,
                                                   ll_runtime_type_info,
-                                                  OBJECT, destrptr,
-                                                  raw_mem_attr_name)
+                                                  OBJECT, destrptr)
             vtable = self.rclass.getvtable()
             self.rtyper.set_type_for_typeptr(vtable, self.lowleveltype.TO)
 
diff --git a/pypy/rpython/memory/test/test_gc.py b/pypy/rpython/memory/test/test_gc.py
--- a/pypy/rpython/memory/test/test_gc.py
+++ b/pypy/rpython/memory/test/test_gc.py
@@ -130,37 +130,6 @@
         assert res == concat(100)
         #assert simulator.current_size - curr < 16000 * INT_SIZE / 4
 
-    def test_lightweight_finalizer(self):
-        T = lltype.Struct('T', ('x', lltype.Signed))
-        
-        @rgc.owns_raw_memory('p')
-        class AClass(object):
-            p = lltype.nullptr(T)
-
-            def __init__(self, arg):
-                if arg:
-                    self.p = lltype.malloc(T, flavor='raw')
-
-        class B(AClass):
-            pass
-
-        def f():
-            a = AClass(0)
-            for i in range(30):
-                if i % 2:
-                    a = B(3)
-                else:
-                    a = AClass(3)
-                AClass(0)
-            llop.gc__collect(lltype.Void)
-            assert a.p
-            del a
-            llop.gc__collect(lltype.Void)
-            # assert did not crash with malloc mismatch, ie those things
-            # has been freed
-
-        self.interpret(f, [])
-
     def test_finalizer(self):
         class B(object):
             pass
diff --git a/pypy/rpython/memory/test/test_transformed_gc.py b/pypy/rpython/memory/test/test_transformed_gc.py
--- a/pypy/rpython/memory/test/test_transformed_gc.py
+++ b/pypy/rpython/memory/test/test_transformed_gc.py
@@ -339,36 +339,6 @@
         res = run([5, 42]) #XXX pure lazyness here too
         assert res == 6
 
-    def define_lightweight_finalizer(cls):
-        T = lltype.Struct('T', ('x', lltype.Signed))
-        
-        @rgc.owns_raw_memory('p')
-        class AClass(object):
-            p = lltype.nullptr(T)
-
-            def __init__(self, arg):
-                if arg:
-                    self.p = lltype.malloc(T, flavor='raw')
-
-        def f():
-            a = AClass(0)
-            for i in range(30):
-                a = AClass(3)
-                AClass(0)
-            llop.gc__collect(lltype.Void)
-            assert a.p
-            del a
-            llop.gc__collect(lltype.Void)
-            # assert did not crash with malloc mismatch, ie those things
-            # has been freed
-            return 3
-
-        return f
-
-    def test_lightweight_finalizer(self):
-        run = self.runner("lightweight_finalizer")
-        run([])
-
     def define_finalizer_calls_malloc(cls):
         class B(object):
             pass
diff --git a/pypy/rpython/test/test_rclass.py b/pypy/rpython/test/test_rclass.py
--- a/pypy/rpython/test/test_rclass.py
+++ b/pypy/rpython/test/test_rclass.py
@@ -977,28 +977,6 @@
         destrptr = RTTI._obj.destructor_funcptr
         assert destrptr is not None
 
-    def test_lightweight_del(self):
-        T = Struct('T', ('x', Signed))
-        
-        @rgc.owns_raw_memory('p')
-        class A(object):
-            p = nullptr(T)
-
-            def __init__(self, arg):
-                self.p = malloc(T, flavor='raw')
-
-        def f():
-            A(3)
-
-        t = TranslationContext()
-        t.buildannotator().build_types(f, [])
-        t.buildrtyper().specialize()
-        graph = graphof(t, f)
-        TYPE = graph.startblock.operations[0].args[0].value
-        RTTI = getRuntimeTypeInfo(TYPE)
-        RTTI._obj.query_funcptr # should not raise
-        assert RTTI._obj.raw_mem_attr_name == 'p'
-
     def test_del_inheritance(self):
         from pypy.rlib import rgc
         class State:

From noreply at buildbot.pypy.org  Sun Oct  9 12:31:42 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Sun,  9 Oct 2011 12:31:42 +0200 (CEST)
Subject: [pypy-commit] pypy lightweight-finalizers: kill some unused imports,
	thanks pyflakes
Message-ID: <20111009103142.804BD82112@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: lightweight-finalizers
Changeset: r47887:05247b119662
Date: 2011-10-09 11:37 +0200
http://bitbucket.org/pypy/pypy/changeset/05247b119662/

Log:	kill some unused imports, thanks pyflakes

diff --git a/pypy/rpython/memory/test/test_gc.py b/pypy/rpython/memory/test/test_gc.py
--- a/pypy/rpython/memory/test/test_gc.py
+++ b/pypy/rpython/memory/test/test_gc.py
@@ -5,7 +5,6 @@
 from pypy.rpython.memory.test import snippet
 from pypy.rpython.test.test_llinterp import get_interpreter
 from pypy.rpython.lltypesystem import lltype
-from pypy.rpython.lltypesystem.rstr import STR
 from pypy.rpython.lltypesystem.lloperation import llop
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.rlib.objectmodel import compute_unique_id
@@ -57,7 +56,7 @@
                 while j < 20:
                     j += 1
                     a.append(j)
-        res = self.interpret(malloc_a_lot, [])
+        self.interpret(malloc_a_lot, [])
         #assert simulator.current_size - curr < 16000 * INT_SIZE / 4
         #print "size before: %s, size after %s" % (curr, simulator.current_size)
 
@@ -73,7 +72,7 @@
                 while j < 20:
                     j += 1
                     b.append((1, j, i))
-        res = self.interpret(malloc_a_lot, [])
+        self.interpret(malloc_a_lot, [])
         #assert simulator.current_size - curr < 16000 * INT_SIZE / 4
         #print "size before: %s, size after %s" % (curr, simulator.current_size)
 
@@ -278,7 +277,7 @@
                                self.interpret, f, [])
 
     def test_weakref(self):
-        import weakref, gc
+        import weakref
         class A(object):
             pass
         def g():
@@ -299,7 +298,7 @@
         assert res
 
     def test_weakref_to_object_with_finalizer(self):
-        import weakref, gc
+        import weakref
         class A(object):
             count = 0
         a = A()
@@ -338,7 +337,7 @@
         assert res
 
     def test_cycle_with_weakref_and_del(self):
-        import weakref, gc
+        import weakref
         class A(object):
             count = 0
         a = A()
@@ -367,7 +366,7 @@
         assert res == 11
 
     def test_weakref_to_object_with_finalizer_ordering(self):
-        import weakref, gc
+        import weakref
         class A(object):
             count = 0
         a = A()
@@ -616,7 +615,7 @@
                     assert not rgc.can_move(a)
                     return 1
                 return 0
-            except Exception, e:
+            except Exception:
                 return 2
 
         assert self.interpret(func, []) == int(self.GC_CAN_MALLOC_NONMOVABLE)
@@ -647,8 +646,6 @@
         assert self.interpret(f, [bigsize, 0, flag]) == 0x62024241
 
     def test_tagged_simple(self):
-        from pypy.rlib.objectmodel import UnboxedValue
-
         class Unrelated(object):
             pass
 
@@ -689,8 +686,6 @@
         assert res == -897
 
     def test_tagged_id(self):
-        from pypy.rlib.objectmodel import UnboxedValue, compute_unique_id
-
         class Unrelated(object):
             pass
 

From noreply at buildbot.pypy.org  Sun Oct  9 12:31:43 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Sun,  9 Oct 2011 12:31:43 +0200 (CEST)
Subject: [pypy-commit] pypy lightweight-finalizers: remove most of the
 previous approach. Now attach T_HAS_LIGHT_FINALIZER
Message-ID: <20111009103143.B4FD082112@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: lightweight-finalizers
Changeset: r47888:f53ec1da99cc
Date: 2011-10-09 12:31 +0200
http://bitbucket.org/pypy/pypy/changeset/f53ec1da99cc/

Log:	remove most of the previous approach. Now attach
	T_HAS_LIGHT_FINALIZER to gctypelayout

diff --git a/pypy/rpython/memory/gc/base.py b/pypy/rpython/memory/gc/base.py
--- a/pypy/rpython/memory/gc/base.py
+++ b/pypy/rpython/memory/gc/base.py
@@ -73,9 +73,7 @@
                             is_rpython_class,
                             has_custom_trace,
                             get_custom_trace,
-                            fast_path_tracing,
-                            has_raw_mem_ptr,
-                            ofs_to_raw_mem_ptr):
+                            fast_path_tracing):
         self.getfinalizer = getfinalizer
         self.is_varsize = is_varsize
         self.has_gcptr_in_varsize = has_gcptr_in_varsize
@@ -92,8 +90,6 @@
         self.has_custom_trace = has_custom_trace
         self.get_custom_trace = get_custom_trace
         self.fast_path_tracing = fast_path_tracing
-        self.has_raw_mem_ptr = has_raw_mem_ptr
-        self.ofs_to_raw_mem_ptr = ofs_to_raw_mem_ptr
 
     def get_member_index(self, type_id):
         return self.member_index(type_id)
@@ -354,13 +350,6 @@
         finally:
             self.finalizer_lock_count -= 1
 
-    def _free_raw_mem_from(self, addr):
-        typeid = self.get_type_id(addr)
-        raw_adr = (addr + self.ofs_to_raw_mem_ptr(typeid)).address[0]
-        if raw_adr:
-            llop.track_alloc_stop(lltype.Void, raw_adr)
-            llmemory.raw_free(raw_adr)
-
 
 class MovingGCBase(GCBase):
     moving_gc = True
diff --git a/pypy/rpython/memory/gc/marksweep.py b/pypy/rpython/memory/gc/marksweep.py
--- a/pypy/rpython/memory/gc/marksweep.py
+++ b/pypy/rpython/memory/gc/marksweep.py
@@ -362,8 +362,6 @@
                     obj = gc_info + size_gc_header
                     self.write_free_statistics(typeid, obj)
                     freed_size += estimate
-                    if self.has_raw_mem_ptr(typeid):
-                        self._free_raw_mem_from(addr + size_gc_header)
                     raw_free(addr)
                 hdr = next
             ppnext.address[0] = llmemory.NULL
diff --git a/pypy/rpython/memory/gctypelayout.py b/pypy/rpython/memory/gctypelayout.py
--- a/pypy/rpython/memory/gctypelayout.py
+++ b/pypy/rpython/memory/gctypelayout.py
@@ -1,7 +1,6 @@
 from pypy.rpython.lltypesystem import lltype, llmemory, llarena, llgroup
 from pypy.rpython.lltypesystem import rclass
 from pypy.rpython.lltypesystem.lloperation import llop
-from pypy.rlib.objectmodel import we_are_translated
 from pypy.rlib.debug import ll_assert
 from pypy.rlib.rarithmetic import intmask
 from pypy.tool.identity_dict import identity_dict
@@ -34,8 +33,6 @@
         ("finalizer_or_customtrace", FINALIZER_OR_CT),
         ("fixedsize",      lltype.Signed),
         ("ofstoptrs",      lltype.Ptr(OFFSETS_TO_GC_PTR)),
-        ("ofstorawptr",    lltype.Signed),
-        # XXX merge me with finalizer_or_custometrace
         hints={'immutable': True},
         )
     VARSIZE_TYPE_INFO = lltype.Struct("varsize_type_info",
@@ -138,12 +135,6 @@
         infobits = self.get(typeid).infobits
         return infobits & T_ANY_SLOW_FLAG == 0
 
-    def q_has_raw_mem_ptr(self, typeid):
-        return self.get(typeid).infobits & T_HAS_RAW_MEM_PTR != 0
-
-    def q_ofs_to_raw_mem_ptr(self, typeid):
-        return self.get(typeid).ofstorawptr
-
     def set_query_functions(self, gc):
         gc.set_query_functions(
             self.q_is_varsize,
@@ -161,23 +152,21 @@
             self.q_is_rpython_class,
             self.q_has_custom_trace,
             self.q_get_custom_trace,
-            self.q_fast_path_tracing,
-            self.q_has_raw_mem_ptr,
-            self.q_ofs_to_raw_mem_ptr)
+            self.q_fast_path_tracing)
 
 
 # the lowest 16bits are used to store group member index
-T_MEMBER_INDEX         =   0xffff
-T_IS_VARSIZE           = 0x010000
-T_HAS_GCPTR_IN_VARSIZE = 0x020000
-T_IS_GCARRAY_OF_GCPTR  = 0x040000
-T_IS_WEAKREF           = 0x080000
-T_IS_RPYTHON_INSTANCE  = 0x100000    # the type is a subclass of OBJECT
-T_HAS_FINALIZER        = 0x200000
-T_HAS_CUSTOM_TRACE     = 0x400000
-T_HAS_RAW_MEM_PTR      = 0x800000
-T_KEY_MASK             = intmask(0xFF000000)
-T_KEY_VALUE            = intmask(0x5A000000)    # bug detection only
+T_MEMBER_INDEX              =   0xffff
+T_IS_VARSIZE                = 0x010000
+T_HAS_GCPTR_IN_VARSIZE      = 0x020000
+T_IS_GCARRAY_OF_GCPTR       = 0x040000
+T_IS_WEAKREF                = 0x080000
+T_IS_RPYTHON_INSTANCE       = 0x100000 # the type is a subclass of OBJECT
+T_HAS_FINALIZER             = 0x200000
+T_HAS_CUSTOM_TRACE          = 0x400000
+T_HAS_LIGHTWEIGHT_FINALIZER = 0x800000
+T_KEY_MASK                  = intmask(0xFF000000)
+T_KEY_VALUE                 = intmask(0x5A000000) # bug detection only
 
 def _check_valid_type_info(p):
     ll_assert(p.infobits & T_KEY_MASK == T_KEY_VALUE, "invalid type_id")
@@ -205,6 +194,8 @@
         info.finalizer_or_customtrace = fptr
         if kind == "finalizer":
             infobits |= T_HAS_FINALIZER
+        elif kind == 'light_finalizer':
+            infobits |= T_HAS_FINALIZER | T_HAS_LIGHTWEIGHT_FINALIZER
         elif kind == "custom_trace":
             infobits |= T_HAS_CUSTOM_TRACE
         else:
@@ -247,20 +238,6 @@
         infobits |= T_IS_WEAKREF
     if is_subclass_of_object(TYPE):
         infobits |= T_IS_RPYTHON_INSTANCE
-    if builder.get_raw_mem_attr_name(TYPE):
-        name = builder.get_raw_mem_attr_name(TYPE)
-        infobits |= T_HAS_RAW_MEM_PTR
-        # can be in a superclass
-        T = TYPE
-        l = []
-        while not hasattr(T, 'inst_' + name):
-            l.append(llmemory.offsetof(T, 'super'))
-            T = T.super
-        l.append(llmemory.offsetof(T, 'inst_' + name))
-        ofs = l[0]
-        for i in range(1, len(l)):
-            ofs += l[i]
-        info.ofstorawptr = ofs
     info.infobits = infobits | T_KEY_VALUE
 
 # ____________________________________________________________
@@ -358,13 +335,6 @@
     def is_weakref_type(self, TYPE):
         return TYPE == WEAKREF
 
-    def get_raw_mem_attr_name(self, TYPE):
-        from pypy.rpython.memory.gctransform.support import get_rtti
-        
-        rtti = get_rtti(TYPE)
-        return rtti is not None and getattr(rtti._obj, 'raw_mem_attr_name',
-                                            None)
-
     def encode_type_shapes_now(self):
         if not self.can_encode_type_shape:
             self.can_encode_type_shape = True
@@ -399,12 +369,15 @@
     def special_funcptr_for_type(self, TYPE):
         if TYPE in self._special_funcptrs:
             return self._special_funcptrs[TYPE]
-        fptr1 = self.make_finalizer_funcptr_for_type(TYPE)
+        fptr1, is_lightweight = self.make_finalizer_funcptr_for_type(TYPE)
         fptr2 = self.make_custom_trace_funcptr_for_type(TYPE)
         assert not (fptr1 and fptr2), (
             "type %r needs both a finalizer and a custom tracer" % (TYPE,))
         if fptr1:
-            kind_and_fptr = "finalizer", fptr1
+            if is_lightweight:
+                kind_and_fptr = "light_finalizer", fptr1
+            else:
+                kind_and_fptr = "finalizer", fptr1
         elif fptr2:
             kind_and_fptr = "custom_trace", fptr2
         else:
@@ -414,7 +387,7 @@
 
     def make_finalizer_funcptr_for_type(self, TYPE):
         # must be overridden for proper finalizer support
-        return None
+        return None, False
 
     def make_custom_trace_funcptr_for_type(self, TYPE):
         # must be overridden for proper custom tracer support
diff --git a/pypy/rpython/memory/gcwrapper.py b/pypy/rpython/memory/gcwrapper.py
--- a/pypy/rpython/memory/gcwrapper.py
+++ b/pypy/rpython/memory/gcwrapper.py
@@ -1,3 +1,4 @@
+from pypy.translator.backendopt.finalizer import FinalizerAnalyzer
 from pypy.rpython.lltypesystem import lltype, llmemory, llheap
 from pypy.rpython import llinterp
 from pypy.rpython.annlowlevel import llhelper
@@ -196,9 +197,11 @@
             DESTR_ARG = lltype.typeOf(destrptr).TO.ARGS[0]
             destrgraph = destrptr._obj.graph
         else:
-            return None
+            return None, False
 
         assert not type_contains_pyobjs(TYPE), "not implemented"
+        t = self.llinterp.typer.annotator.translator
+        light = not FinalizerAnalyzer(t).analyze_direct_call(destrgraph)
         def ll_finalizer(addr, dummy):
             assert dummy == llmemory.NULL
             try:
@@ -208,7 +211,7 @@
                 raise RuntimeError(
                     "a finalizer raised an exception, shouldn't happen")
             return llmemory.NULL
-        return llhelper(gctypelayout.GCData.FINALIZER_OR_CT, ll_finalizer)
+        return llhelper(gctypelayout.GCData.FINALIZER_OR_CT, ll_finalizer), light
 
     def make_custom_trace_funcptr_for_type(self, TYPE):
         from pypy.rpython.memory.gctransform.support import get_rtti, \
diff --git a/pypy/translator/backendopt/finalizer.py b/pypy/translator/backendopt/finalizer.py
--- a/pypy/translator/backendopt/finalizer.py
+++ b/pypy/translator/backendopt/finalizer.py
@@ -25,7 +25,4 @@
             if not isinstance(TP, lltype.Ptr) or TP.TO._gckind == 'raw':
                 # primitive type
                 return self.bottom_result()
-        print op
-        import pdb
-        pdb.set_trace()
         return self.top_result()

From noreply at buildbot.pypy.org  Sun Oct  9 13:00:33 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  9 Oct 2011 13:00:33 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Tweaks.
Message-ID: <20111009110033.9662682112@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r47889:31f8caa85b5c
Date: 2011-10-09 12:54 +0200
http://bitbucket.org/pypy/pypy/changeset/31f8caa85b5c/

Log:	Tweaks.

diff --git a/pypy/rpython/memory/gc/concurrentms.py b/pypy/rpython/memory/gc/concurrentms.py
--- a/pypy/rpython/memory/gc/concurrentms.py
+++ b/pypy/rpython/memory/gc/concurrentms.py
@@ -138,6 +138,8 @@
         # is a collection running and the mutator tries to change an object
         # that was not scanned yet.
         self._init_writebarrier_logic()
+        #
+        self.main_thread_ident = ll_thread.get_ident()
 
     def setup(self):
         "Start the concurrent collector thread."
@@ -368,6 +370,7 @@
         llarena.arena_reset(adr, self.HDRSIZE, 0)
         llarena.arena_reserve(adr, totalsize)
         return adr + llmemory.raw_malloc_usage(self.HDRSIZE)
+    grow_reservation._always_inline_ = True
 
     def write_barrier(self, newvalue, addr_struct):
         mark = self.header(addr_struct).tid & 0xFF
@@ -530,7 +533,8 @@
             raise
 
     def acquire(self, lock):
-        if we_are_translated():
+        if (we_are_translated() or
+                ll_thread.get_ident() != self.main_thread_ident):
             ll_thread.c_thread_acquirelock(lock, 1)
         else:
             while rffi.cast(lltype.Signed,
diff --git a/pypy/translator/c/primitive.py b/pypy/translator/c/primitive.py
--- a/pypy/translator/c/primitive.py
+++ b/pypy/translator/c/primitive.py
@@ -1,7 +1,7 @@
 import sys
 from pypy.rlib.objectmodel import Symbolic, ComputedIntSymbolic
 from pypy.rlib.objectmodel import CDefinedIntSymbolic
-from pypy.rlib.rarithmetic import r_longlong
+from pypy.rlib.rarithmetic import r_longlong, LONG_BIT
 from pypy.rlib.rfloat import isinf, isnan
 from pypy.rpython.lltypesystem.lltype import *
 from pypy.rpython.lltypesystem import rffi, llgroup
@@ -62,6 +62,11 @@
             name = name_small_integer(value.lowpart, db)
             assert (value.rest & value.MASK) == 0
             return '(%s+%dL)' % (name, value.rest)
+        elif isinstance(value, llgroup.HighCombinedSymbolic):
+            name = name_small_integer(value.hipart, db)
+            assert (value.rest & value.MASK) == 0
+            return '((((long)%s)<<%d)+%dL)' % (name, LONG_BIT//2,
+                                               value.rest)
         elif isinstance(value, AddressAsInt):
             return '((long)%s)' % name_address(value.adr, db)
         else:
diff --git a/pypy/translator/c/src/llgroup.h b/pypy/translator/c/src/llgroup.h
--- a/pypy/translator/c/src/llgroup.h
+++ b/pypy/translator/c/src/llgroup.h
@@ -58,4 +58,9 @@
 #define OP_EXTRACT_USHORT(value, r)         r = (pypy_halfword_t)value
 #define OP_COMBINE_USHORT(ushort, rest, r)  r = ((long)ushort) | rest
 
+#define OP_EXTRACT_HIGH_USHORT(value, r)         \
+    r = (pypy_halfword_t)(value >> (PYPY_LONG_BIT/2))
+#define OP_COMBINE_HIGH_USHORT(ushort, rest, r)  \
+    r = (((long)ushort) << (PYPY_LONG_BIT/2)) | rest
+
 #endif /* _PYPY_LL_GROUP_H_ */

From noreply at buildbot.pypy.org  Sun Oct  9 13:00:34 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  9 Oct 2011 13:00:34 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: More tweaks.
Message-ID: <20111009110034.C695C82112@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r47890:bffefba2678e
Date: 2011-10-09 13:00 +0200
http://bitbucket.org/pypy/pypy/changeset/bffefba2678e/

Log:	More tweaks.

diff --git a/pypy/rpython/lltypesystem/opimpl.py b/pypy/rpython/lltypesystem/opimpl.py
--- a/pypy/rpython/lltypesystem/opimpl.py
+++ b/pypy/rpython/lltypesystem/opimpl.py
@@ -184,14 +184,16 @@
 def op_int_add(x, y):
     if not isinstance(x, (int, llmemory.AddressOffset)):
         from pypy.rpython.lltypesystem import llgroup
-        assert isinstance(x, llgroup.CombinedSymbolic)
+        assert isinstance(x, (llgroup.CombinedSymbolic,
+                              llgroup.HighCombinedSymbolic))
     assert isinstance(y, (int, llmemory.AddressOffset))
     return intmask(x + y)
 
 def op_int_sub(x, y):
     if not isinstance(x, int):
         from pypy.rpython.lltypesystem import llgroup
-        assert isinstance(x, llgroup.CombinedSymbolic)
+        assert isinstance(x, (llgroup.CombinedSymbolic,
+                              llgroup.HighCombinedSymbolic))
     assert isinstance(y, int)
     return intmask(x - y)
 
@@ -216,14 +218,16 @@
 def op_int_and(x, y):
     if not isinstance(x, int):
         from pypy.rpython.lltypesystem import llgroup
-        assert isinstance(x, llgroup.CombinedSymbolic)
+        assert isinstance(x, (llgroup.CombinedSymbolic,
+                              llgroup.HighCombinedSymbolic))
     assert isinstance(y, int)
     return x & y
 
 def op_int_or(x, y):
     if not isinstance(x, int):
         from pypy.rpython.lltypesystem import llgroup
-        assert isinstance(x, llgroup.CombinedSymbolic)
+        assert isinstance(x, (llgroup.CombinedSymbolic,
+                              llgroup.HighCombinedSymbolic))
     assert isinstance(y, int)
     return x | y
 

From noreply at buildbot.pypy.org  Sun Oct  9 13:41:26 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Sun,  9 Oct 2011 13:41:26 +0200 (CEST)
Subject: [pypy-commit] pypy lightweight-finalizers: chained objects unallowed
Message-ID: <20111009114126.43F1F82112@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: lightweight-finalizers
Changeset: r47891:78904ead20ab
Date: 2011-10-09 13:41 +0200
http://bitbucket.org/pypy/pypy/changeset/78904ead20ab/

Log:	chained objects unallowed

diff --git a/pypy/rpython/memory/gc/base.py b/pypy/rpython/memory/gc/base.py
--- a/pypy/rpython/memory/gc/base.py
+++ b/pypy/rpython/memory/gc/base.py
@@ -63,6 +63,7 @@
     def set_query_functions(self, is_varsize, has_gcptr_in_varsize,
                             is_gcarrayofgcptr,
                             getfinalizer,
+                            getlightfinalizer,
                             offsets_to_gc_pointers,
                             fixed_size, varsize_item_sizes,
                             varsize_offset_to_variable_part,
@@ -75,6 +76,7 @@
                             get_custom_trace,
                             fast_path_tracing):
         self.getfinalizer = getfinalizer
+        self.getlightfinalizer = getlightfinalizer
         self.is_varsize = is_varsize
         self.has_gcptr_in_varsize = has_gcptr_in_varsize
         self.is_gcarrayofgcptr = is_gcarrayofgcptr
@@ -140,6 +142,7 @@
 
         size = self.fixed_size(typeid)
         needs_finalizer = bool(self.getfinalizer(typeid))
+        finalizer_is_light = bool(self.getlightfinalizer(typeid))
         contains_weakptr = self.weakpointer_offset(typeid) >= 0
         assert not (needs_finalizer and contains_weakptr)
         if self.is_varsize(typeid):
@@ -159,6 +162,7 @@
             else:
                 malloc_fixedsize = self.malloc_fixedsize
             ref = malloc_fixedsize(typeid, size, needs_finalizer,
+                                   finalizer_is_light,
                                    contains_weakptr)
         # lots of cast and reverse-cast around...
         return llmemory.cast_ptr_to_adr(ref)
diff --git a/pypy/rpython/memory/gc/semispace.py b/pypy/rpython/memory/gc/semispace.py
--- a/pypy/rpython/memory/gc/semispace.py
+++ b/pypy/rpython/memory/gc/semispace.py
@@ -82,7 +82,7 @@
         self.free = self.tospace
         MovingGCBase.setup(self)
         self.objects_with_finalizers = self.AddressDeque()
-        self.objects_with_raw_mem = self.AddressStack()
+        self.objects_with_light_finalizers = self.AddressStack()
         self.objects_with_weakrefs = self.AddressStack()
 
     def _teardown(self):
@@ -94,7 +94,9 @@
     # because the spaces are filled with zeroes in advance.
 
     def malloc_fixedsize_clear(self, typeid16, size,
-                               has_finalizer=False, contains_weakptr=False):
+                               has_finalizer=False,
+                               has_light_finalizer=False,
+                               contains_weakptr=False):
         size_gc_header = self.gcheaderbuilder.size_gc_header
         totalsize = size_gc_header + size
         result = self.free
@@ -103,10 +105,10 @@
         llarena.arena_reserve(result, totalsize)
         self.init_gc_object(result, typeid16)
         self.free = result + totalsize
-        if has_finalizer:
+        if has_light_finalizer:
+            self.objects_with_light_finalizers.append(result + size_gc_header)
+        elif has_finalizer:
             self.objects_with_finalizers.append(result + size_gc_header)
-        if self.has_raw_mem_ptr(typeid16):
-            self.objects_with_raw_mem.append(result + size_gc_header)
         if contains_weakptr:
             self.objects_with_weakrefs.append(result + size_gc_header)
         return llmemory.cast_adr_to_ptr(result+size_gc_header, llmemory.GCREF)
@@ -266,10 +268,10 @@
         if self.run_finalizers.non_empty():
             self.update_run_finalizers()
         scan = self.scan_copied(scan)
+        if self.objects_with_light_finalizers.non_empty():
+            self.deal_with_objects_with_light_finalizers()
         if self.objects_with_finalizers.non_empty():
             scan = self.deal_with_objects_with_finalizers(scan)
-        if self.objects_with_raw_mem.non_empty():
-            self.deal_with_objects_with_raw_mem()
         if self.objects_with_weakrefs.non_empty():
             self.invalidate_weakrefs()
         self.update_objects_with_id()
@@ -476,6 +478,14 @@
         # immortal objects always have GCFLAG_FORWARDED set;
         # see get_forwarding_address().
 
+    def deal_with_objects_with_light_finalizers(self):
+        """ This is a much simpler version of dealing with finalizers
+        and an optimization - we can reasonably assume that those finalizers
+        don't do anything fancy and *just* call them. Among other things
+        they won't resurrect objects
+        """
+        xxx
+
     def deal_with_objects_with_finalizers(self, scan):
         # walk over list of objects with finalizers
         # if it is not copied, add it to the list of to-be-called finalizers
@@ -528,17 +538,6 @@
         self.objects_with_finalizers = new_with_finalizer
         return scan
 
-    def deal_with_objects_with_raw_mem(self):
-        new_with_raw_mem = self.AddressStack()
-        while self.objects_with_raw_mem.non_empty():
-            addr = self.objects_with_raw_mem.pop()
-            if self.surviving(addr):
-                new_with_raw_mem.append(self.get_forwarding_address(addr))
-            else:
-                self._free_raw_mem_from(addr)
-        self.objects_with_raw_mem = new_with_raw_mem
-
-
     def _append_if_nonnull(pointer, stack):
         stack.append(pointer.address[0])
     _append_if_nonnull = staticmethod(_append_if_nonnull)
diff --git a/pypy/rpython/memory/gctransform/framework.py b/pypy/rpython/memory/gctransform/framework.py
--- a/pypy/rpython/memory/gctransform/framework.py
+++ b/pypy/rpython/memory/gctransform/framework.py
@@ -258,6 +258,7 @@
             [s_gc, s_typeid16,
              annmodel.SomeInteger(nonneg=True),
              annmodel.SomeBool(),
+             annmodel.SomeBool(),
              annmodel.SomeBool()], s_gcref,
             inline = False)
         if hasattr(GCClass, 'malloc_fixedsize'):
diff --git a/pypy/rpython/memory/gctypelayout.py b/pypy/rpython/memory/gctypelayout.py
--- a/pypy/rpython/memory/gctypelayout.py
+++ b/pypy/rpython/memory/gctypelayout.py
@@ -84,6 +84,13 @@
         else:
             return lltype.nullptr(GCData.FINALIZER_OR_CT_FUNC)
 
+    def q_light_finalizer(self, typeid):
+        typeinfo = self.get(typeid)
+        if typeinfo.infobits & T_HAS_LIGHTWEIGHT_FINALIZER:
+            return typeinfo.finalizer_or_customtrace
+        else:
+            return lltype.nullptr(GCData.FINALIZER_OR_CT_FUNC)        
+
     def q_offsets_to_gc_pointers(self, typeid):
         return self.get(typeid).ofstoptrs
 
@@ -141,6 +148,7 @@
             self.q_has_gcptr_in_varsize,
             self.q_is_gcarrayofgcptr,
             self.q_finalizer,
+            self.q_light_finalizer,
             self.q_offsets_to_gc_pointers,
             self.q_fixed_size,
             self.q_varsize_item_sizes,
diff --git a/pypy/translator/backendopt/finalizer.py b/pypy/translator/backendopt/finalizer.py
--- a/pypy/translator/backendopt/finalizer.py
+++ b/pypy/translator/backendopt/finalizer.py
@@ -11,7 +11,7 @@
     * anything that escapes self
     * anything that can allocate
     """
-    ok_operations = ['getfield', 'ptr_nonzero', 'free', 'same_as',
+    ok_operations = ['ptr_nonzero', 'free', 'same_as',
                      'direct_ptradd', 'force_cast', 'cast_primitive',
                      'cast_pointer']
     
@@ -25,4 +25,9 @@
             if not isinstance(TP, lltype.Ptr) or TP.TO._gckind == 'raw':
                 # primitive type
                 return self.bottom_result()
+        if op.opname == 'getfield':
+            TP = op.result.concretetype
+            if not isinstance(TP, lltype.Ptr) or TP.TO._gckind == 'raw':
+                # primitive type
+                return self.bottom_result()            
         return self.top_result()
diff --git a/pypy/translator/backendopt/test/test_finalizer.py b/pypy/translator/backendopt/test/test_finalizer.py
--- a/pypy/translator/backendopt/test/test_finalizer.py
+++ b/pypy/translator/backendopt/test/test_finalizer.py
@@ -80,6 +80,24 @@
         r = self.analyze(g, [], f, backendopt=True)
         assert not r
 
+    def test_chain(self):
+        class B(object):
+            def __init__(self):
+                self.counter = 1
+        
+        class A(object):
+            def __init__(self):
+                self.x = B()
+
+            def __del__(self):
+                self.x.counter += 1
+
+        def f():
+            A()
+
+        r = self.analyze(f, [], A.__del__.im_func)
+        assert r
+
     def test_os_call(self):
         py.test.skip("can allocate OSError, but also can raise, ignore for now")
         import os

From noreply at buildbot.pypy.org  Sun Oct  9 14:06:50 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Sun,  9 Oct 2011 14:06:50 +0200 (CEST)
Subject: [pypy-commit] pypy lightweight-finalizers: call RSocket.__del__
 directly instead of close(). Will not raise
Message-ID: <20111009120650.DD52282112@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: lightweight-finalizers
Changeset: r47892:29b1b092fb7a
Date: 2011-10-09 14:06 +0200
http://bitbucket.org/pypy/pypy/changeset/29b1b092fb7a/

Log:	call RSocket.__del__ directly instead of close(). Will not raise

diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -19,7 +19,7 @@
 class W_RSocket(Wrappable, RSocket):
     def __del__(self):
         self.clear_all_weakrefs()
-        self.close()
+        RSocket.__del__(self)
 
     def accept_w(self, space):
         """accept() -> (socket object, address info)

From noreply at buildbot.pypy.org  Sun Oct  9 14:08:41 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Sun,  9 Oct 2011 14:08:41 +0200 (CEST)
Subject: [pypy-commit] pypy lightweight-finalizers: finish semispace
Message-ID: <20111009120841.61F4A82112@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: lightweight-finalizers
Changeset: r47893:6306cf4fd3ef
Date: 2011-10-09 14:08 +0200
http://bitbucket.org/pypy/pypy/changeset/6306cf4fd3ef/

Log:	finish semispace

diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py
--- a/pypy/jit/backend/llsupport/gc.py
+++ b/pypy/jit/backend/llsupport/gc.py
@@ -678,7 +678,7 @@
             # also use it to allocate varsized objects.  The tid
             # and possibly the length are both set afterward.
             gcref = llop1.do_malloc_fixedsize_clear(llmemory.GCREF,
-                                        0, size, False, False)
+                                        0, size, False, False, False)
             return rffi.cast(lltype.Signed, gcref)
         self.malloc_slowpath = malloc_slowpath
         self.MALLOC_SLOWPATH = lltype.FuncType([lltype.Signed], lltype.Signed)
diff --git a/pypy/jit/backend/llsupport/test/test_gc.py b/pypy/jit/backend/llsupport/test/test_gc.py
--- a/pypy/jit/backend/llsupport/test/test_gc.py
+++ b/pypy/jit/backend/llsupport/test/test_gc.py
@@ -247,7 +247,8 @@
         self.record = []
 
     def do_malloc_fixedsize_clear(self, RESTYPE, type_id, size,
-                                  has_finalizer, contains_weakptr):
+                                  has_finalizer, has_light_finalizer,
+                                  contains_weakptr):
         assert not contains_weakptr
         p = llmemory.raw_malloc(size)
         p = llmemory.cast_adr_to_ptr(p, RESTYPE)
diff --git a/pypy/rpython/memory/gc/generation.py b/pypy/rpython/memory/gc/generation.py
--- a/pypy/rpython/memory/gc/generation.py
+++ b/pypy/rpython/memory/gc/generation.py
@@ -168,7 +168,9 @@
         return self.nursery <= addr < self.nursery_top
 
     def malloc_fixedsize_clear(self, typeid, size,
-                               has_finalizer=False, contains_weakptr=False):
+                               has_finalizer=False,
+                               has_light_finalizer=False,
+                               contains_weakptr=False):
         if (has_finalizer or
             (raw_malloc_usage(size) > self.lb_young_fixedsize and
              raw_malloc_usage(size) > self.largest_young_fixedsize)):
diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -461,7 +461,9 @@
 
 
     def malloc_fixedsize_clear(self, typeid, size,
-                               needs_finalizer=False, contains_weakptr=False):
+                               needs_finalizer=False,
+                               has_light_finalizer=False,
+                               contains_weakptr=False):
         size_gc_header = self.gcheaderbuilder.size_gc_header
         totalsize = size_gc_header + size
         rawtotalsize = raw_malloc_usage(totalsize)
diff --git a/pypy/rpython/memory/gc/semispace.py b/pypy/rpython/memory/gc/semispace.py
--- a/pypy/rpython/memory/gc/semispace.py
+++ b/pypy/rpython/memory/gc/semispace.py
@@ -484,7 +484,15 @@
         don't do anything fancy and *just* call them. Among other things
         they won't resurrect objects
         """
-        xxx
+        new_objects = self.AddressStack()
+        while self.objects_with_light_finalizers.non_empty():
+            obj = self.objects_with_light_finalizers.pop()
+            if self.surviving(obj):
+                new_objects.append(self.get_forwarding_address(obj))
+            else:
+                finalizer = self.getfinalizer(self.get_type_id(obj))
+                finalizer(obj, llmemory.NULL)
+        self.objects_with_light_finalizers = new_objects
 
     def deal_with_objects_with_finalizers(self, scan):
         # walk over list of objects with finalizers

From noreply at buildbot.pypy.org  Sun Oct  9 14:33:45 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Sun,  9 Oct 2011 14:33:45 +0200 (CEST)
Subject: [pypy-commit] pypy lightweight-finalizers: clean up previous
 attempts, make this one work
Message-ID: <20111009123345.71A1682112@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: lightweight-finalizers
Changeset: r47894:b399c49c9fe6
Date: 2011-10-09 14:33 +0200
http://bitbucket.org/pypy/pypy/changeset/b399c49c9fe6/

Log:	clean up previous attempts, make this one work

diff --git a/pypy/rpython/memory/gc/generation.py b/pypy/rpython/memory/gc/generation.py
--- a/pypy/rpython/memory/gc/generation.py
+++ b/pypy/rpython/memory/gc/generation.py
@@ -87,7 +87,6 @@
 
         self.last_generation_root_objects = self.AddressStack()
         self.young_objects_with_id = self.AddressDict()
-        self.young_objects_with_raw_mem = self.AddressStack()
         SemiSpaceGC.setup(self)
         self.set_nursery_size(self.initial_nursery_size)
         # the GC is fully setup now.  The rest can make use of it.
@@ -191,8 +190,6 @@
         llarena.arena_reserve(result, totalsize)
         # GCFLAG_NO_YOUNG_PTRS is never set on young objs
         self.init_gc_object(result, typeid, flags=0)
-        if self.has_raw_mem_ptr(typeid):
-            self.young_objects_with_raw_mem.append(result + size_gc_header)
         self.nursery_free = result + totalsize
         if contains_weakptr:
             self.young_objects_with_weakrefs.append(result + size_gc_header)
@@ -270,7 +267,6 @@
     def semispace_collect(self, size_changing=False):
         self.reset_young_gcflags() # we are doing a full collection anyway
         self.weakrefs_grow_older()
-        self.raw_mem_grow_older()
         self.ids_grow_older()
         self.reset_nursery()
         SemiSpaceGC.semispace_collect(self, size_changing)
@@ -305,11 +301,6 @@
             obj = self.young_objects_with_weakrefs.pop()
             self.objects_with_weakrefs.append(obj)
 
-    def raw_mem_grow_older(self):
-        while self.young_objects_with_raw_mem.non_empty():
-            obj = self.young_objects_with_raw_mem.pop()
-            self.objects_with_raw_mem.append(obj)
-
     def collect_roots(self):
         """GenerationGC: collects all roots.
            HybridGC: collects all roots, excluding the generation 3 ones.
@@ -369,8 +360,6 @@
                 self.invalidate_young_weakrefs()
             if self.young_objects_with_id.length() > 0:
                 self.update_young_objects_with_id()
-            if self.young_objects_with_raw_mem.non_empty():
-                self.deal_with_young_objects_with_raw_mem()
             # mark the nursery as free and fill it with zeroes again
             llarena.arena_reset(self.nursery, self.nursery_size, 2)
             debug_print("survived (fraction of the size):",
@@ -573,15 +562,6 @@
         # minimal size, which is actually a good idea: a large, mostly-empty
         # table is bad for the next call to 'foreach'.
 
-    def deal_with_young_objects_with_raw_mem(self):
-        while self.young_objects_with_raw_mem.non_empty():
-            addr = self.young_objects_with_raw_mem.pop()
-            if self.surviving(addr):
-                self.objects_with_raw_mem.append(
-                    self.get_forwarding_address(addr))
-            else:
-                self._free_raw_mem_from(addr)
-
     def ids_grow_older(self):
         self.young_objects_with_id.foreach(self._id_grow_older, None)
         self.young_objects_with_id.clear()
diff --git a/pypy/rpython/memory/gc/marksweep.py b/pypy/rpython/memory/gc/marksweep.py
--- a/pypy/rpython/memory/gc/marksweep.py
+++ b/pypy/rpython/memory/gc/marksweep.py
@@ -93,7 +93,8 @@
         pass
 
     def malloc_fixedsize(self, typeid16, size,
-                         has_finalizer=False, contains_weakptr=False):
+                         has_finalizer=False, has_light_finalizer=False,
+                         contains_weakptr=False):
         self.maybe_collect()
         size_gc_header = self.gcheaderbuilder.size_gc_header
         try:
@@ -128,7 +129,9 @@
     malloc_fixedsize._dont_inline_ = True
 
     def malloc_fixedsize_clear(self, typeid16, size,
-                               has_finalizer=False, contains_weakptr=False):
+                               has_finalizer=False,
+                               has_light_finalizer=False,
+                               contains_weakptr=False):
         self.maybe_collect()
         size_gc_header = self.gcheaderbuilder.size_gc_header
         try:
diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -297,10 +297,6 @@
         # created after it.
         self.young_objects_with_weakrefs = self.AddressStack()
         self.old_objects_with_weakrefs = self.AddressStack()
-        # This is a list for objects that are in the nursery and
-        # own some raw memory. Note that young objects which are raw_malloced
-        # won't go there
-        self.young_objects_with_raw_mem = self.AddressStack()
         #
         # Support for id and identityhash: map nursery objects with
         # GCFLAG_HAS_SHADOW to their future location at the next
@@ -505,9 +501,6 @@
             # If it is a weakref, record it (check constant-folded).
             if contains_weakptr:
                 self.young_objects_with_weakrefs.append(result+size_gc_header)
-            if self.has_raw_mem_ptr(typeid):
-                self.young_objects_with_raw_mem.append(result + size_gc_header)
-            #
             obj = result + size_gc_header
         #
         return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
@@ -1272,8 +1265,6 @@
         # weakrefs' targets.
         if self.young_objects_with_weakrefs.non_empty():
             self.invalidate_young_weakrefs()
-        if self.young_objects_with_raw_mem.non_empty():
-            self.invalidate_young_raw_mem()
         #
         # Clear this mapping.
         if self.nursery_objects_shadows.length() > 0:
@@ -1658,8 +1649,6 @@
             self.header(obj).tid &= ~GCFLAG_VISITED
             return False     # survives
         else:
-            if self.has_raw_mem_ptr(self.get_type_id(obj)):
-                self._free_raw_mem_from(obj)
             return True      # dies
 
     def _reset_gcflag_visited(self, obj, ignored):
@@ -1689,8 +1678,6 @@
                 arena -= extra_words * WORD
                 allocsize += extra_words * WORD
             #
-            if self.has_raw_mem_ptr(self.get_type_id(obj)):
-                self._free_raw_mem_from(obj)
             llarena.arena_free(arena)
             self.rawmalloced_total_size -= allocsize
 
@@ -1989,12 +1976,6 @@
         self.old_objects_with_weakrefs.delete()
         self.old_objects_with_weakrefs = new_with_weakref
 
-    def invalidate_young_raw_mem(self):
-        while self.young_objects_with_raw_mem.non_empty():
-            addr = self.young_objects_with_raw_mem.pop()
-            if self.header(addr).tid & GCFLAG_VISITED == 0:
-                self._free_raw_mem_from(addr)
-
 
 # ____________________________________________________________
 
diff --git a/pypy/rpython/memory/gctransform/framework.py b/pypy/rpython/memory/gctransform/framework.py
--- a/pypy/rpython/memory/gctransform/framework.py
+++ b/pypy/rpython/memory/gctransform/framework.py
@@ -268,6 +268,7 @@
                 [s_gc, s_typeid16,
                  annmodel.SomeInteger(nonneg=True),
                  annmodel.SomeBool(),
+                 annmodel.SomeBool(),
                  annmodel.SomeBool()], s_gcref,
                 inline = False)
         else:

From noreply at buildbot.pypy.org  Sun Oct  9 18:26:43 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  9 Oct 2011 18:26:43 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Translation fix.
Message-ID: <20111009162643.B62A082A88@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r47896:268922a27ca7
Date: 2011-10-09 15:12 +0200
http://bitbucket.org/pypy/pypy/changeset/268922a27ca7/

Log:	Translation fix.

diff --git a/pypy/rpython/memory/gc/concurrentms.py b/pypy/rpython/memory/gc/concurrentms.py
--- a/pypy/rpython/memory/gc/concurrentms.py
+++ b/pypy/rpython/memory/gc/concurrentms.py
@@ -100,6 +100,10 @@
         collector_start._should_never_raise_ = True
         self.collector_start = collector_start
         #
+        self.gray_objects = self.AddressStack()
+        self.extra_objects_to_mark = self.AddressStack()
+        self.prebuilt_root_objects = self.AddressStack()
+        #
         self._initialize()
         #
         # Write barrier: actually a deletion barrier, triggered when there
@@ -148,9 +152,9 @@
         self._teardown_now = []
         #
         #self.mutex_lock = ...built in setup()
-        self.gray_objects = self.AddressStack()
-        self.extra_objects_to_mark = self.AddressStack()
-        self.prebuilt_root_objects = self.AddressStack()
+        self.gray_objects.clear()
+        self.extra_objects_to_mark.clear()
+        self.prebuilt_root_objects.clear()
 
     def setup(self):
         "Start the concurrent collector thread."

From noreply at buildbot.pypy.org  Sun Oct  9 18:26:42 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  9 Oct 2011 18:26:42 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Progress.
Message-ID: <20111009162642.872F482112@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r47895:294bb3f33706
Date: 2011-10-09 15:10 +0200
http://bitbucket.org/pypy/pypy/changeset/294bb3f33706/

Log:	Progress.

diff --git a/pypy/config/translationoption.py b/pypy/config/translationoption.py
--- a/pypy/config/translationoption.py
+++ b/pypy/config/translationoption.py
@@ -90,6 +90,8 @@
                  }),
     BoolOption("gcremovetypeptr", "Remove the typeptr from every object",
                default=IS_64_BITS, cmdline="--gcremovetypeptr"),
+    BoolOption("gctesttransformed", "Set to True by test_transformed_gc",
+               default=False),
     ChoiceOption("gcrootfinder",
                  "Strategy for finding GC Roots (framework GCs only)",
                  ["n/a", "shadowstack", "asmgcc"],
diff --git a/pypy/rpython/lltypesystem/opimpl.py b/pypy/rpython/lltypesystem/opimpl.py
--- a/pypy/rpython/lltypesystem/opimpl.py
+++ b/pypy/rpython/lltypesystem/opimpl.py
@@ -594,7 +594,13 @@
     hdr = llmemory.cast_adr_to_ptr(hdraddr, lltype.Ptr(HDR))
     typeid = getattr(hdr, fieldname)
     if lltype.typeOf(typeid) == lltype.Signed:
-        typeid = op_extract_ushort(typeid)
+        from pypy.rpython.lltypesystem import llgroup
+        if isinstance(typeid, llgroup.CombinedSymbolic):
+            typeid = op_extract_ushort(typeid)
+        elif isinstance(typeid, llgroup.HighCombinedSymbolic):
+            typeid = op_extract_high_ushort(typeid)
+        else:
+            raise TypeError(typeid)
     return op_get_next_group_member(TYPE, grpptr, typeid, skipoffset)
 op_gc_gettypeptr_group.need_result_type = True
 
diff --git a/pypy/rpython/memory/gc/concurrentms.py b/pypy/rpython/memory/gc/concurrentms.py
--- a/pypy/rpython/memory/gc/concurrentms.py
+++ b/pypy/rpython/memory/gc/concurrentms.py
@@ -5,7 +5,7 @@
 from pypy.rpython.annlowlevel import llhelper
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
 from pypy.rlib.objectmodel import we_are_translated, running_on_llinterp
-from pypy.rlib.debug import ll_assert
+from pypy.rlib.debug import ll_assert, debug_print
 from pypy.rlib.rarithmetic import ovfcheck, LONG_BIT, r_uint
 from pypy.rpython.memory.gc.base import GCBase
 from pypy.module.thread import ll_thread
@@ -70,7 +70,6 @@
         assert small_request_threshold % WORD == 0
         self.small_request_threshold = small_request_threshold
         self.page_size = page_size
-        self.free_pages = lltype.nullptr(self.HDR)
         self.pagelists_length = small_request_threshold // WORD + 1
         #
         # The following are arrays of 36 linked lists: the linked lists
@@ -80,7 +79,7 @@
         def list_of_addresses_per_small_size():
             return lltype.malloc(rffi.CArray(self.HDRPTR),
                                  self.pagelists_length, flavor='raw',
-                                 zero=True, immortal=True)
+                                 immortal=True)
         # 1-35: a linked list of all pages; 0: a linked list of all larger objs
         self.nonfree_pages = list_of_addresses_per_small_size()
         # a snapshot of 'nonfree_pages' done when the collection starts
@@ -92,6 +91,38 @@
         self.collect_heads = list_of_addresses_per_small_size()
         self.collect_tails = list_of_addresses_per_small_size()
         #
+        def collector_start():
+            if we_are_translated():
+                self.collector_run()
+            else:
+                self.collector_run_nontranslated()
+        #
+        collector_start._should_never_raise_ = True
+        self.collector_start = collector_start
+        #
+        self._initialize()
+        #
+        # Write barrier: actually a deletion barrier, triggered when there
+        # is a collection running and the mutator tries to change an object
+        # that was not scanned yet.
+        self._init_writebarrier_logic()
+
+    def _clear_list(self, array):
+        i = 0
+        while i < self.pagelists_length:
+            array[i] = lltype.nullptr(self.HDR)
+            i += 1
+
+    def _initialize(self):
+        self.free_pages = lltype.nullptr(self.HDR)
+        #
+        # Clear the lists
+        self._clear_list(self.nonfree_pages)
+        self._clear_list(self.collect_pages)
+        self._clear_list(self.free_lists)
+        self._clear_list(self.collect_heads)
+        self._clear_list(self.collect_tails)
+        #
         # The following character is either MARK_VALUE_1 or MARK_VALUE_2,
         # and represents the character that must be in the 'mark' field
         # of an object header in order for the object to be considered as
@@ -113,38 +144,19 @@
         #self.ready_to_start_lock = ...built in setup()
         #self.finished_lock = ...built in setup()
         #
-        # NOT_RPYTHON: set to non-empty in _teardown()
+        # set to non-empty in _teardown()
         self._teardown_now = []
         #
-        def collector_start():
-            if we_are_translated():
-                self.collector_run()
-            else:
-                try:
-                    self.collector_run()
-                except Exception, e:
-                    print 'Crash!', e.__class__.__name__, e
-                    self._exc_info = sys.exc_info()
-        #
-        collector_start._should_never_raise_ = True
-        self.collector_start = collector_start
-        #
         #self.mutex_lock = ...built in setup()
         self.gray_objects = self.AddressStack()
         self.extra_objects_to_mark = self.AddressStack()
         self.prebuilt_root_objects = self.AddressStack()
-        #
-        # Write barrier: actually a deletion barrier, triggered when there
-        # is a collection running and the mutator tries to change an object
-        # that was not scanned yet.
-        self._init_writebarrier_logic()
-        #
-        self.main_thread_ident = ll_thread.get_ident()
 
     def setup(self):
         "Start the concurrent collector thread."
         GCBase.setup(self)
         #
+        self.main_thread_ident = ll_thread.get_ident()
         self.ready_to_start_lock = ll_thread.allocate_ll_lock()
         self.finished_lock = ll_thread.allocate_ll_lock()
         self.mutex_lock = ll_thread.allocate_ll_lock()
@@ -166,9 +178,9 @@
         # which should shut down the collector thread
         self._teardown_now.append(-1)
         self.release(self.ready_to_start_lock)
+        print "teardown!"
         self.acquire(self.finished_lock)
-        if not we_are_translated():
-            del self.ready_to_start_lock, self.finished_lock
+        self._initialize()
 
     def get_type_id(self, obj):
         tid = self.header(obj).tid
@@ -533,6 +545,7 @@
             raise
 
     def acquire(self, lock):
+        debug_print("acquire", ll_thread.get_ident(), self.main_thread_ident)
         if (we_are_translated() or
                 ll_thread.get_ident() != self.main_thread_ident):
             ll_thread.c_thread_acquirelock(lock, 1)
@@ -543,6 +556,7 @@
                 # ---------- EXCEPTION FROM THE COLLECTOR THREAD ----------
                 if hasattr(self, '_exc_info'):
                     self._reraise_from_collector_thread()
+        debug_print("ok", ll_thread.get_ident(), self.main_thread_ident)
 
     def release(self, lock):
         ll_thread.c_thread_releaselock(lock)
@@ -560,6 +574,24 @@
                                         "symbolic")
 
 
+    def collector_run_nontranslated(self):
+        if hasattr(self, 'ready_to_start_lock'):      # normal tests
+            try:
+                self.collector_run()
+            except Exception, e:
+                print 'Crash!', e.__class__.__name__, e
+                self._exc_info = sys.exc_info()
+        else:
+            # this case is for test_transformed_gc: we need to spawn
+            # another LLInterpreter for this new thread.
+            from pypy.rpython.llinterp import LLInterpreter
+            prev = LLInterpreter.current_interpreter
+            llinterp = LLInterpreter(prev.typer)
+            # XXX FISH HORRIBLY for the graph...
+            graph = sys._getframe(2).f_locals['self']._obj.graph
+            llinterp.eval_graph(graph)
+
+
     def collector_run(self):
         """Main function of the collector's thread."""
         #
diff --git a/pypy/rpython/memory/gctransform/framework.py b/pypy/rpython/memory/gctransform/framework.py
--- a/pypy/rpython/memory/gctransform/framework.py
+++ b/pypy/rpython/memory/gctransform/framework.py
@@ -230,8 +230,9 @@
         self.frameworkgc_setup_ptr = getfn(frameworkgc_setup, [],
                                            annmodel.s_None)
         # for tests
-        self.frameworkgc__teardown_ptr = getfn(frameworkgc__teardown, [],
-                                               annmodel.s_None)
+        if self.translator.config.translation.gctesttransformed:
+            self.frameworkgc__teardown_ptr = getfn(frameworkgc__teardown, [],
+                                                   annmodel.s_None)
         
         if root_walker.need_root_stack:
             self.incr_stack_ptr = getfn(root_walker.incr_stack,
diff --git a/pypy/rpython/memory/test/test_transformed_gc.py b/pypy/rpython/memory/test/test_transformed_gc.py
--- a/pypy/rpython/memory/test/test_transformed_gc.py
+++ b/pypy/rpython/memory/test/test_transformed_gc.py
@@ -105,6 +105,7 @@
             if fixup:
                 fixup(t)
 
+        t.config.translation.gctesttransformed = True
         cbuild = CStandaloneBuilder(t, entrypoint, config=t.config,
                                     gcpolicy=cls.gcpolicy)
         db = cbuild.generate_graphs_for_llinterp()

From noreply at buildbot.pypy.org  Sun Oct  9 18:26:44 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  9 Oct 2011 18:26:44 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Progress.
Message-ID: <20111009162644.E5A3582112@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r47897:dd1b61d40b11
Date: 2011-10-09 15:23 +0200
http://bitbucket.org/pypy/pypy/changeset/dd1b61d40b11/

Log:	Progress.

diff --git a/pypy/rpython/memory/gc/concurrentms.py b/pypy/rpython/memory/gc/concurrentms.py
--- a/pypy/rpython/memory/gc/concurrentms.py
+++ b/pypy/rpython/memory/gc/concurrentms.py
@@ -148,8 +148,8 @@
         #self.ready_to_start_lock = ...built in setup()
         #self.finished_lock = ...built in setup()
         #
-        # set to non-empty in _teardown()
-        self._teardown_now = []
+        # set to True in _teardown()
+        self._teardown_now = False
         #
         #self.mutex_lock = ...built in setup()
         self.gray_objects.clear()
@@ -174,15 +174,14 @@
 
     def _teardown(self):
         "Stop the collector thread after tests have run."
-        if self._teardown_now:
-            return
+        assert not self._teardown_now
         self.wait_for_the_end_of_collection()
         #
         # start the next collection, but with "stop" in _teardown_now,
         # which should shut down the collector thread
-        self._teardown_now.append(-1)
+        self._teardown_now = True
+        print "teardown!"
         self.release(self.ready_to_start_lock)
-        print "teardown!"
         self.acquire(self.finished_lock)
         self._initialize()
 
@@ -609,10 +608,9 @@
             self.acquire(self.ready_to_start_lock)
             #
             # For tests: detect when we have to shut down
-            if not we_are_translated():
-                if self._teardown_now:
-                    self.release(finished_lock)
-                    break
+            if self._teardown_now:
+                self.release(self.finished_lock)
+                break
             #
             # Mark
             self.collector_mark()

From noreply at buildbot.pypy.org  Sun Oct  9 18:26:46 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  9 Oct 2011 18:26:46 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Simplify.
Message-ID: <20111009162646.2530082112@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r47898:c2474cbcbb67
Date: 2011-10-09 15:27 +0200
http://bitbucket.org/pypy/pypy/changeset/c2474cbcbb67/

Log:	Simplify.

diff --git a/pypy/rpython/memory/gc/concurrentms.py b/pypy/rpython/memory/gc/concurrentms.py
--- a/pypy/rpython/memory/gc/concurrentms.py
+++ b/pypy/rpython/memory/gc/concurrentms.py
@@ -148,9 +148,6 @@
         #self.ready_to_start_lock = ...built in setup()
         #self.finished_lock = ...built in setup()
         #
-        # set to True in _teardown()
-        self._teardown_now = False
-        #
         #self.mutex_lock = ...built in setup()
         self.gray_objects.clear()
         self.extra_objects_to_mark.clear()
@@ -174,13 +171,12 @@
 
     def _teardown(self):
         "Stop the collector thread after tests have run."
-        assert not self._teardown_now
         self.wait_for_the_end_of_collection()
         #
-        # start the next collection, but with "stop" in _teardown_now,
+        # start the next collection, but with collection_running set to 42,
         # which should shut down the collector thread
-        self._teardown_now = True
-        print "teardown!"
+        self.collection_running = 42
+        #print "teardown!"
         self.release(self.ready_to_start_lock)
         self.acquire(self.finished_lock)
         self._initialize()
@@ -548,7 +544,7 @@
             raise
 
     def acquire(self, lock):
-        debug_print("acquire", ll_thread.get_ident(), self.main_thread_ident)
+        #debug_print("acquire", ll_thread.get_ident(), self.main_thread_ident)
         if (we_are_translated() or
                 ll_thread.get_ident() != self.main_thread_ident):
             ll_thread.c_thread_acquirelock(lock, 1)
@@ -559,7 +555,7 @@
                 # ---------- EXCEPTION FROM THE COLLECTOR THREAD ----------
                 if hasattr(self, '_exc_info'):
                     self._reraise_from_collector_thread()
-        debug_print("ok", ll_thread.get_ident(), self.main_thread_ident)
+        #debug_print("ok", ll_thread.get_ident(), self.main_thread_ident)
 
     def release(self, lock):
         ll_thread.c_thread_releaselock(lock)
@@ -598,17 +594,18 @@
     def collector_run(self):
         """Main function of the collector's thread."""
         #
-        # hack: make it an infinite loop, but in a way that the annotator
-        # doesn't notice.  It prevents the caller from ending automatically
-        # in a "raise AssertionError", annoyingly, because we don't want
+        # hack: this is an infinite loop in practice.  During tests it can
+        # be interrupted.  In all cases the annotator must not conclude that
+        # it is an infinite loop: otherwise, the caller would automatically
+        # end in a "raise AssertionError", annoyingly, because we don't want
         # any exception in this thread
-        while self.collection_running < 99:
+        while True:
             #
             # Wait for the lock to be released
             self.acquire(self.ready_to_start_lock)
             #
             # For tests: detect when we have to shut down
-            if self._teardown_now:
+            if self.collection_running == 42:
                 self.release(self.finished_lock)
                 break
             #

From noreply at buildbot.pypy.org  Sun Oct  9 18:26:47 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  9 Oct 2011 18:26:47 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: One more test from
 test_transformed_gc passes.
Message-ID: <20111009162647.56B1C82112@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r47899:c24efc6995c0
Date: 2011-10-09 15:30 +0200
http://bitbucket.org/pypy/pypy/changeset/c24efc6995c0/

Log:	One more test from test_transformed_gc passes.

diff --git a/pypy/rpython/lltypesystem/llmemory.py b/pypy/rpython/lltypesystem/llmemory.py
--- a/pypy/rpython/lltypesystem/llmemory.py
+++ b/pypy/rpython/lltypesystem/llmemory.py
@@ -547,6 +547,11 @@
     # support for "i & 0xFF == <odd value>", which is always False if i is
     # an AddressAsInt (at least assuming the address is of some word-aligned
     # location).
+    def annotation(self):
+        from pypy.annotation import model
+        return model.SomeInteger()
+    def lltype(self):
+        return lltype.Signed
     def __eq__(self, other):
         assert (other & 3) != 0
         return False
diff --git a/pypy/rpython/lltypesystem/opimpl.py b/pypy/rpython/lltypesystem/opimpl.py
--- a/pypy/rpython/lltypesystem/opimpl.py
+++ b/pypy/rpython/lltypesystem/opimpl.py
@@ -197,6 +197,12 @@
     assert isinstance(y, int)
     return intmask(x - y)
 
+def op_int_eq(x, y):
+    # specail case for MultipleOf4
+    assert isinstance(x, (int, llmemory.MultipleOf4))
+    assert isinstance(y, int)
+    return x == y
+
 def op_int_ge(x, y):
     # special case for 'AddressOffset >= 0'
     assert isinstance(x, (int, llmemory.AddressOffset))
@@ -219,7 +225,8 @@
     if not isinstance(x, int):
         from pypy.rpython.lltypesystem import llgroup
         assert isinstance(x, (llgroup.CombinedSymbolic,
-                              llgroup.HighCombinedSymbolic))
+                              llgroup.HighCombinedSymbolic,
+                              llmemory.AddressAsInt))
     assert isinstance(y, int)
     return x & y
 

From noreply at buildbot.pypy.org  Sun Oct  9 18:26:48 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  9 Oct 2011 18:26:48 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Fix for test_newgc.
Message-ID: <20111009162648.86E4082112@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r47900:8b51157e362c
Date: 2011-10-09 15:46 +0200
http://bitbucket.org/pypy/pypy/changeset/8b51157e362c/

Log:	Fix for test_newgc.

diff --git a/pypy/rpython/lltypesystem/opimpl.py b/pypy/rpython/lltypesystem/opimpl.py
--- a/pypy/rpython/lltypesystem/opimpl.py
+++ b/pypy/rpython/lltypesystem/opimpl.py
@@ -198,7 +198,7 @@
     return intmask(x - y)
 
 def op_int_eq(x, y):
-    # specail case for MultipleOf4
+    # special case for MultipleOf4
     assert isinstance(x, (int, llmemory.MultipleOf4))
     assert isinstance(y, int)
     return x == y
diff --git a/pypy/rpython/memory/gc/concurrentms.py b/pypy/rpython/memory/gc/concurrentms.py
--- a/pypy/rpython/memory/gc/concurrentms.py
+++ b/pypy/rpython/memory/gc/concurrentms.py
@@ -380,7 +380,7 @@
         adr = llarena.getfakearenaaddress(adr)
         llarena.arena_reset(adr, self.HDRSIZE, 0)
         llarena.arena_reserve(adr, totalsize)
-        return adr + llmemory.raw_malloc_usage(self.HDRSIZE)
+        return adr + self.gcheaderbuilder.size_gc_header
     grow_reservation._always_inline_ = True
 
     def write_barrier(self, newvalue, addr_struct):
@@ -544,7 +544,6 @@
             raise
 
     def acquire(self, lock):
-        #debug_print("acquire", ll_thread.get_ident(), self.main_thread_ident)
         if (we_are_translated() or
                 ll_thread.get_ident() != self.main_thread_ident):
             ll_thread.c_thread_acquirelock(lock, 1)
@@ -555,7 +554,6 @@
                 # ---------- EXCEPTION FROM THE COLLECTOR THREAD ----------
                 if hasattr(self, '_exc_info'):
                     self._reraise_from_collector_thread()
-        #debug_print("ok", ll_thread.get_ident(), self.main_thread_ident)
 
     def release(self, lock):
         ll_thread.c_thread_releaselock(lock)

From noreply at buildbot.pypy.org  Sun Oct  9 18:26:49 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  9 Oct 2011 18:26:49 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: It seems to work better
 with "pages" that are bit bigger, at least
Message-ID: <20111009162649.B617382112@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r47901:aae94d06d7b5
Date: 2011-10-09 18:26 +0200
http://bitbucket.org/pypy/pypy/changeset/aae94d06d7b5/

Log:	It seems to work better with "pages" that are bit bigger, at least
	on my laptop, running targetgcbench-c.

diff --git a/pypy/rpython/memory/gc/concurrentms.py b/pypy/rpython/memory/gc/concurrentms.py
--- a/pypy/rpython/memory/gc/concurrentms.py
+++ b/pypy/rpython/memory/gc/concurrentms.py
@@ -57,7 +57,7 @@
     typeid_is_in_field = 'tid'
     withhash_flag_is_in_field = 'tid', FL_WITHHASH
 
-    TRANSLATION_PARAMS = {'page_size': 4096,
+    TRANSLATION_PARAMS = {'page_size': 16384,
                           'small_request_threshold': 35*WORD,
                           }
 

From noreply at buildbot.pypy.org  Sun Oct  9 18:44:49 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Sun,  9 Oct 2011 18:44:49 +0200 (CEST)
Subject: [pypy-commit] pypy lightweight-finalizers: fix test_transformed_gc
Message-ID: <20111009164449.E0D1C82112@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: lightweight-finalizers
Changeset: r47902:2a9d83dfaf19
Date: 2011-10-09 18:44 +0200
http://bitbucket.org/pypy/pypy/changeset/2a9d83dfaf19/

Log:	fix test_transformed_gc

diff --git a/pypy/rpython/memory/gctransform/framework.py b/pypy/rpython/memory/gctransform/framework.py
--- a/pypy/rpython/memory/gctransform/framework.py
+++ b/pypy/rpython/memory/gctransform/framework.py
@@ -12,6 +12,7 @@
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.translator.backendopt import graphanalyze
 from pypy.translator.backendopt.support import var_needsgc
+from pypy.translator.backendopt.finalizer import FinalizerAnalyzer
 from pypy.annotation import model as annmodel
 from pypy.rpython import annlowlevel
 from pypy.rpython.rbuiltin import gen_cast
@@ -321,7 +322,7 @@
             raise NotImplementedError("GC needs write barrier, but does not provide writebarrier_before_copy functionality")
 
         # in some GCs we can inline the common case of
-        # malloc_fixedsize(typeid, size, True, False, False)
+        # malloc_fixedsize(typeid, size, False, False, False)
         if getattr(GCClass, 'inline_simple_malloc', False):
             # make a copy of this function so that it gets annotated
             # independently and the constants are folded inside
@@ -339,7 +340,7 @@
                 malloc_fast,
                 [s_gc, s_typeid16,
                  annmodel.SomeInteger(nonneg=True),
-                 s_False, s_False], s_gcref,
+                 s_False, s_False, s_False], s_gcref,
                 inline = True)
         else:
             self.malloc_fast_ptr = None
@@ -670,7 +671,13 @@
         kind_and_fptr = self.special_funcptr_for_type(TYPE)
         has_finalizer = (kind_and_fptr is not None and
                          kind_and_fptr[0] == "finalizer")
+        has_light_finalizer = (kind_and_fptr is not None and
+                               kind_and_fptr[0] == "light_finalizer")
+        if has_light_finalizer:
+            has_finalizer = True
         c_has_finalizer = rmodel.inputconst(lltype.Bool, has_finalizer)
+        c_has_light_finalizer = rmodel.inputconst(lltype.Bool,
+                                                  has_light_finalizer)
 
         if not op.opname.endswith('_varsize') and not flags.get('varsize'):
             #malloc_ptr = self.malloc_fixedsize_ptr
@@ -684,7 +691,8 @@
             else:
                 malloc_ptr = self.malloc_fixedsize_ptr
             args = [self.c_const_gc, c_type_id, c_size,
-                    c_has_finalizer, rmodel.inputconst(lltype.Bool, False)]
+                    c_has_finalizer, c_has_light_finalizer,
+                    rmodel.inputconst(lltype.Bool, False)]
         else:
             assert not c_has_finalizer.value
             info_varsize = self.layoutbuilder.get_info_varsize(type_id)
@@ -849,12 +857,13 @@
         # used by the JIT (see pypy.jit.backend.llsupport.gc)
         op = hop.spaceop
         [v_typeid, v_size,
-         v_has_finalizer, v_contains_weakptr] = op.args
+         v_has_finalizer, v_has_light_finalizer, v_contains_weakptr] = op.args
         livevars = self.push_roots(hop)
         hop.genop("direct_call",
                   [self.malloc_fixedsize_clear_ptr, self.c_const_gc,
                    v_typeid, v_size,
-                   v_has_finalizer, v_contains_weakptr],
+                   v_has_finalizer, v_has_light_finalizer,
+                   v_contains_weakptr],
                   resultvar=op.result)
         self.pop_roots(hop, livevars)
 
@@ -914,10 +923,10 @@
         info = self.layoutbuilder.get_info(type_id)
         c_size = rmodel.inputconst(lltype.Signed, info.fixedsize)
         malloc_ptr = self.malloc_fixedsize_ptr
-        c_has_finalizer = rmodel.inputconst(lltype.Bool, False)
+        c_false = rmodel.inputconst(lltype.Bool, False)
         c_has_weakptr = rmodel.inputconst(lltype.Bool, True)
         args = [self.c_const_gc, c_type_id, c_size,
-                c_has_finalizer, c_has_weakptr]
+                c_false, c_false, c_has_weakptr]
 
         # push and pop the current live variables *including* the argument
         # to the weakref_create operation, which must be kept alive and
@@ -1252,6 +1261,7 @@
             lltype2vtable = translator.rtyper.lltype2vtable
         else:
             lltype2vtable = None
+        self.translator = translator
         super(TransformerLayoutBuilder, self).__init__(GCClass, lltype2vtable)
 
     def has_finalizer(self, TYPE):
@@ -1266,7 +1276,7 @@
 
     def make_finalizer_funcptr_for_type(self, TYPE):
         if not self.has_finalizer(TYPE):
-            return None
+            return None, False
         rtti = get_rtti(TYPE)
         destrptr = rtti._obj.destructor_funcptr
         DESTR_ARG = lltype.typeOf(destrptr).TO.ARGS[0]
@@ -1278,7 +1288,9 @@
             return llmemory.NULL
         fptr = self.transformer.annotate_finalizer(ll_finalizer,
                 [llmemory.Address, llmemory.Address], llmemory.Address)
-        return fptr
+        g = destrptr._obj.graph
+        light = not FinalizerAnalyzer(self.translator).analyze_direct_call(g)
+        return fptr, light
 
     def make_custom_trace_funcptr_for_type(self, TYPE):
         if not self.has_custom_trace(TYPE):
diff --git a/pypy/rpython/memory/test/test_transformed_gc.py b/pypy/rpython/memory/test/test_transformed_gc.py
--- a/pypy/rpython/memory/test/test_transformed_gc.py
+++ b/pypy/rpython/memory/test/test_transformed_gc.py
@@ -345,22 +345,22 @@
         b = B()
         b.nextid = 0
         b.num_deleted = 0
-        class A(object):
+        class AAA(object):
             def __init__(self):
                 self.id = b.nextid
                 b.nextid += 1
             def __del__(self):
                 b.num_deleted += 1
                 C()
-        class C(A):
+        class C(AAA):
             def __del__(self):
                 b.num_deleted += 1
         def f(x, y):
-            a = A()
+            a = AAA()
             i = 0
             while i < x:
                 i += 1
-                a = A()
+                a = AAA()
             llop.gc__collect(lltype.Void)
             llop.gc__collect(lltype.Void)
             return b.num_deleted
@@ -807,6 +807,7 @@
                     op.args = [Constant(type_id, llgroup.HALFWORD),
                                Constant(llmemory.sizeof(P), lltype.Signed),
                                Constant(False, lltype.Bool), # has_finalizer
+                               Constant(False, lltype.Bool), # has_light_finalizer
                                Constant(False, lltype.Bool)] # contains_weakptr
                     break
             else:
@@ -843,6 +844,7 @@
                     op.args = [Constant(type_id, llgroup.HALFWORD),
                                Constant(llmemory.sizeof(P), lltype.Signed),
                                Constant(False, lltype.Bool), # has_finalizer
+                               Constant(False, lltype.Bool), # has_light_finalizer
                                Constant(False, lltype.Bool)] # contains_weakptr
                     break
             else:

From noreply at buildbot.pypy.org  Sun Oct  9 19:04:38 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Sun,  9 Oct 2011 19:04:38 +0200 (CEST)
Subject: [pypy-commit] pypy lightweight-finalizers: add this optimization to
 minimark. unclear if more tests are needed
Message-ID: <20111009170438.A4CE082112@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: lightweight-finalizers
Changeset: r47903:cfa63208b692
Date: 2011-10-09 19:04 +0200
http://bitbucket.org/pypy/pypy/changeset/cfa63208b692/

Log:	add this optimization to minimark. unclear if more tests are needed

diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -290,6 +290,7 @@
         #
         # A list of all objects with finalizers (these are never young).
         self.objects_with_finalizers = self.AddressDeque()
+        self.young_objects_with_light_finalizers = self.AddressStack()
         #
         # Two lists of the objects with weakrefs.  No weakref can be an
         # old object weakly pointing to a young object: indeed, weakrefs
@@ -466,7 +467,7 @@
         #
         # If the object needs a finalizer, ask for a rawmalloc.
         # The following check should be constant-folded.
-        if needs_finalizer:
+        if needs_finalizer and not has_light_finalizer:
             ll_assert(not contains_weakptr,
                      "'needs_finalizer' and 'contains_weakptr' both specified")
             obj = self.external_malloc(typeid, 0, can_make_young=False)
@@ -496,12 +497,14 @@
             #
             # Build the object.
             llarena.arena_reserve(result, totalsize)
+            obj = result + size_gc_header
+            if has_light_finalizer:
+                self.young_objects_with_light_finalizers.append(obj)
             self.init_gc_object(result, typeid, flags=0)
             #
             # If it is a weakref, record it (check constant-folded).
             if contains_weakptr:
-                self.young_objects_with_weakrefs.append(result+size_gc_header)
-            obj = result + size_gc_header
+                self.young_objects_with_weakrefs.append(obj)
         #
         return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
 
@@ -1265,6 +1268,8 @@
         # weakrefs' targets.
         if self.young_objects_with_weakrefs.non_empty():
             self.invalidate_young_weakrefs()
+        if self.young_objects_with_light_finalizers.non_empty():
+            self.deal_with_young_objects_with_finalizers()
         #
         # Clear this mapping.
         if self.nursery_objects_shadows.length() > 0:
@@ -1649,6 +1654,9 @@
             self.header(obj).tid &= ~GCFLAG_VISITED
             return False     # survives
         else:
+            finalizer = self.getlightfinalizer(self.get_type_id(obj))
+            if finalizer:
+                finalizer(obj, llmemory.NULL)
             return True      # dies
 
     def _reset_gcflag_visited(self, obj, ignored):
@@ -1662,6 +1670,9 @@
             size_gc_header = self.gcheaderbuilder.size_gc_header
             totalsize = size_gc_header + self.get_size(obj)
             allocsize = raw_malloc_usage(totalsize)
+            finalizer = self.getlightfinalizer(self.get_type_id(obj))
+            if finalizer:
+                finalizer(obj, llmemory.NULL)            
             arena = llarena.getfakearenaaddress(obj - size_gc_header)
             #
             # Must also include the card marker area, if any
@@ -1828,6 +1839,23 @@
     # ----------
     # Finalizers
 
+    def deal_with_young_objects_with_finalizers(self):
+        """ This is a much simpler version of dealing with finalizers
+        and an optimization - we can reasonably assume that those finalizers
+        don't do anything fancy and *just* call them. Among other things
+        they won't resurrect objects
+        """
+        new_objects = self.AddressStack()
+        while self.young_objects_with_light_finalizers.non_empty():
+            obj = self.young_objects_with_light_finalizers.pop()
+            if self.is_forwarded(obj):
+                new_objects.append(self.get_forwarding_address(obj))
+            else:
+                finalizer = self.getlightfinalizer(self.get_type_id(obj))
+                ll_assert(finalizer, "no light finalizer found")
+                finalizer(obj, llmemory.NULL)
+        self.objects_with_light_finalizers = new_objects
+
     def deal_with_objects_with_finalizers(self):
         # Walk over list of objects with finalizers.
         # If it is not surviving, add it to the list of to-be-called

From noreply at buildbot.pypy.org  Sun Oct  9 19:06:53 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Sun,  9 Oct 2011 19:06:53 +0200 (CEST)
Subject: [pypy-commit] pypy lightweight-finalizers: fix translation
Message-ID: <20111009170653.72D7D82112@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: lightweight-finalizers
Changeset: r47904:5281b5f1566f
Date: 2011-10-09 19:06 +0200
http://bitbucket.org/pypy/pypy/changeset/5281b5f1566f/

Log:	fix translation

diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -1852,7 +1852,7 @@
                 new_objects.append(self.get_forwarding_address(obj))
             else:
                 finalizer = self.getlightfinalizer(self.get_type_id(obj))
-                ll_assert(finalizer, "no light finalizer found")
+                ll_assert(bool(finalizer), "no light finalizer found")
                 finalizer(obj, llmemory.NULL)
         self.objects_with_light_finalizers = new_objects
 

From noreply at buildbot.pypy.org  Sun Oct  9 19:31:31 2011
From: noreply at buildbot.pypy.org (hakanardo)
Date: Sun,  9 Oct 2011 19:31:31 +0200 (CEST)
Subject: [pypy-commit] pypy default: (arigato,
 hakanardo): make pointers behave more correctly prior to translation
Message-ID: <20111009173131.7FBAC82112@wyvern.cs.uni-duesseldorf.de>

Author: Hakan Ardo <hakan at debian.org>
Branch: 
Changeset: r47905:cbee41ad4f50
Date: 2011-10-09 19:31 +0200
http://bitbucket.org/pypy/pypy/changeset/cbee41ad4f50/

Log:	(arigato, hakanardo): make pointers behave more correctly prior to
	translation

diff --git a/pypy/rpython/lltypesystem/ll2ctypes.py b/pypy/rpython/lltypesystem/ll2ctypes.py
--- a/pypy/rpython/lltypesystem/ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/ll2ctypes.py
@@ -1288,6 +1288,9 @@
             return False
         return force_cast(lltype.Signed, other._as_ptr()) == self.intval
 
+    def __hash__(self):
+        return self.intval
+
     def __ne__(self, other):
         return not self == other
 

From noreply at buildbot.pypy.org  Sun Oct  9 19:41:44 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Sun,  9 Oct 2011 19:41:44 +0200 (CEST)
Subject: [pypy-commit] pypy lightweight-finalizers: pass the extra flag in
	the JIT as well
Message-ID: <20111009174144.0516582112@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: lightweight-finalizers
Changeset: r47906:de7e060009e3
Date: 2011-10-09 19:41 +0200
http://bitbucket.org/pypy/pypy/changeset/de7e060009e3/

Log:	pass the extra flag in the JIT as well

diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py
--- a/pypy/jit/backend/llsupport/gc.py
+++ b/pypy/jit/backend/llsupport/gc.py
@@ -604,10 +604,13 @@
         def malloc_basic(size, tid):
             type_id = llop.extract_ushort(llgroup.HALFWORD, tid)
             has_finalizer = bool(tid & (1<<llgroup.HALFSHIFT))
+            has_light_finalizer = bool(tid & (1<<(llgroup.HALFSHIFT + 1)))
             check_typeid(type_id)
             res = llop1.do_malloc_fixedsize_clear(llmemory.GCREF,
                                                   type_id, size,
-                                                  has_finalizer, False)
+                                                  has_finalizer,
+                                                  has_light_finalizer,
+                                                  False)
             # In case the operation above failed, we are returning NULL
             # from this function to assembler.  There is also an RPython
             # exception set, typically MemoryError; but it's easier and
@@ -702,7 +705,9 @@
         type_id = self.layoutbuilder.get_type_id(S)
         assert not self.layoutbuilder.is_weakref_type(S)
         has_finalizer = bool(self.layoutbuilder.has_finalizer(S))
-        flags = int(has_finalizer) << llgroup.HALFSHIFT
+        has_light_finalizer = bool(self.layoutbuilder.has_light_finalizer(S))
+        flags = (int(has_finalizer) << llgroup.HALFSHIFT |
+                 int(has_light_finalizer) << (llgroup.HALFSHIFT + 1))
         descr.tid = llop.combine_ushort(lltype.Signed, type_id, flags)
 
     def init_array_descr(self, A, descr):
diff --git a/pypy/jit/backend/llsupport/test/test_gc.py b/pypy/jit/backend/llsupport/test/test_gc.py
--- a/pypy/jit/backend/llsupport/test/test_gc.py
+++ b/pypy/jit/backend/llsupport/test/test_gc.py
@@ -252,7 +252,7 @@
         assert not contains_weakptr
         p = llmemory.raw_malloc(size)
         p = llmemory.cast_adr_to_ptr(p, RESTYPE)
-        flags = int(has_finalizer) << 16
+        flags = (int(has_finalizer) << 16) | (int(has_light_finalizer) << 17)
         tid = llop.combine_ushort(lltype.Signed, type_id, flags)
         self.record.append(("fixedsize", repr(size), tid, p))
         return p
diff --git a/pypy/rpython/memory/gctransform/framework.py b/pypy/rpython/memory/gctransform/framework.py
--- a/pypy/rpython/memory/gctransform/framework.py
+++ b/pypy/rpython/memory/gctransform/framework.py
@@ -1269,6 +1269,10 @@
         return rtti is not None and getattr(rtti._obj, 'destructor_funcptr',
                                             None)
 
+    def has_light_finalizer(self, TYPE):
+        special = self.special_funcptr_for_type(TYPE)
+        return special is not None and special[0] == 'light_finalizer'
+
     def has_custom_trace(self, TYPE):
         rtti = get_rtti(TYPE)
         return rtti is not None and getattr(rtti._obj, 'custom_trace_funcptr',

From noreply at buildbot.pypy.org  Sun Oct  9 20:00:30 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  9 Oct 2011 20:00:30 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Kill the explicit C code
 manipulating GC strings and lists at
Message-ID: <20111009180030.CC6DB82112@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r47907:c682e7a8ccce
Date: 2011-10-09 19:59 +0200
http://bitbucket.org/pypy/pypy/changeset/c682e7a8ccce/

Log:	Kill the explicit C code manipulating GC strings and lists at start-
	up, and replace it with RPython code.

diff --git a/pypy/translator/c/extfunc.py b/pypy/translator/c/extfunc.py
--- a/pypy/translator/c/extfunc.py
+++ b/pypy/translator/c/extfunc.py
@@ -32,47 +32,19 @@
 
 def predeclare_utility_functions(db, rtyper):
     # Common utility functions
+    # (nowadays we are left with only this one function)
     def RPyString_New(length=lltype.Signed):
         return mallocstr(length)
 
-    # !!!
-    # be extremely careful passing a gc tracked object
-    # from such an helper result to another one
-    # as argument, this could result in leaks
-    # Such result should be only from C code
-    # returned directly as results
-
-    LIST_OF_STR = find_list_of_str(rtyper)
-    if LIST_OF_STR is not None:
-        p = lltype.Ptr(LIST_OF_STR)
-
-        def _RPyListOfString_New(length=lltype.Signed):
-            return LIST_OF_STR.ll_newlist(length)
-
-        def _RPyListOfString_New(length=lltype.Signed):
-            return LIST_OF_STR.ll_newlist(length)
-
-        def _RPyListOfString_SetItem(l=p,
-                                    index=lltype.Signed,
-                                    newstring=lltype.Ptr(STR)):
-            rlist.ll_setitem_nonneg(rlist.dum_nocheck, l, index, newstring)
-
-        def _RPyListOfString_GetItem(l=p,
-                                    index=lltype.Signed):
-            return rlist.ll_getitem_fast(l, index)
-
-        def _RPyListOfString_Length(l=p):
-            return rlist.ll_length(l)
-
     for fname, f in locals().items():
         if isinstance(f, types.FunctionType):
             # XXX this is painful :(
-            if (LIST_OF_STR, fname) in db.helper2ptr:
-                yield (fname, db.helper2ptr[LIST_OF_STR, fname])
+            if ("utility", fname) in db.helper2ptr:
+                yield (fname, db.helper2ptr["utility", fname])
             else:
                 # hack: the defaults give the type of the arguments
                 graph = rtyper.annotate_helper(f, f.func_defaults)
-                db.helper2ptr[LIST_OF_STR, fname] = graph
+                db.helper2ptr["utility", fname] = graph
                 yield (fname, graph)
 
 
diff --git a/pypy/translator/c/genc.py b/pypy/translator/c/genc.py
--- a/pypy/translator/c/genc.py
+++ b/pypy/translator/c/genc.py
@@ -417,6 +417,7 @@
     split = True
     executable_name = None
     shared_library_name = None
+    _entrypoint_wrapper = None
 
     def getprofbased(self):
         profbased = None
@@ -439,8 +440,29 @@
     def getentrypointptr(self):
         # XXX check that the entrypoint has the correct
         # signature:  list-of-strings -> int
-        bk = self.translator.annotator.bookkeeper
-        return getfunctionptr(bk.getdesc(self.entrypoint).getuniquegraph())
+        if self._entrypoint_wrapper is not None:
+            return self._entrypoint_wrapper
+        #
+        from pypy.annotation import model as annmodel
+        from pypy.rpython.lltypesystem import rffi
+        from pypy.rpython.annlowlevel import MixLevelHelperAnnotator
+        entrypoint = self.entrypoint
+        #
+        def entrypoint_wrapper(argc, argv):
+            list = [""] * argc
+            i = 0
+            while i < argc:
+                list[i] = rffi.charp2str(argv[i])
+                i += 1
+            return entrypoint(list)
+        #
+        mix = MixLevelHelperAnnotator(self.translator.rtyper)
+        args_s = [annmodel.SomeInteger(),
+                  annmodel.lltype_to_annotation(rffi.CCHARPP)]
+        s_result = annmodel.SomeInteger()
+        graph = mix.getgraph(entrypoint_wrapper, args_s, s_result)
+        mix.finish()
+        return getfunctionptr(graph)
 
     def cmdexec(self, args='', env=None, err=False, expect_crash=False):
         assert self._compiled
diff --git a/pypy/translator/c/src/main.h b/pypy/translator/c/src/main.h
--- a/pypy/translator/c/src/main.h
+++ b/pypy/translator/c/src/main.h
@@ -51,15 +51,7 @@
     errmsg = RPython_StartupCode();
     if (errmsg) goto error;
 
-    list = _RPyListOfString_New(argc);
-    if (RPyExceptionOccurred()) goto memory_out;
-    for (i=0; i<argc; i++) {
-        RPyString *s = RPyString_FromString(argv[i]);
-        if (RPyExceptionOccurred()) goto memory_out;
-        _RPyListOfString_SetItem(list, i, s);
-    }
-
-    exitcode = STANDALONE_ENTRY_POINT(list);
+    exitcode = STANDALONE_ENTRY_POINT(argc, argv);
 
 #ifdef RPY_ASSERT
     pypy_debug_alloc_results();

From noreply at buildbot.pypy.org  Sun Oct  9 20:00:32 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  9 Oct 2011 20:00:32 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: For now,
 do collections all the time (concurrently).
Message-ID: <20111009180032.0BC1382112@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r47908:fe902da25e54
Date: 2011-10-09 20:00 +0200
http://bitbucket.org/pypy/pypy/changeset/fe902da25e54/

Log:	For now, do collections all the time (concurrently). Various fixes.

diff --git a/pypy/rpython/memory/gc/concurrentms.py b/pypy/rpython/memory/gc/concurrentms.py
--- a/pypy/rpython/memory/gc/concurrentms.py
+++ b/pypy/rpython/memory/gc/concurrentms.py
@@ -176,7 +176,7 @@
         # start the next collection, but with collection_running set to 42,
         # which should shut down the collector thread
         self.collection_running = 42
-        #print "teardown!"
+        debug_print("teardown!")
         self.release(self.ready_to_start_lock)
         self.acquire(self.finished_lock)
         self._initialize()
@@ -261,6 +261,11 @@
     def _malloc_slowpath(self, typeid, size):
         # Slow-path malloc.  Call this with 'size' being a valid and
         # rounded number, between WORD and up to MAXIMUM_SIZE.
+        #
+        # For now, we always start the next collection immediately.
+        if self.collection_running <= 0:
+            self.trigger_next_collection()
+        #
         size_gc_header = self.gcheaderbuilder.size_gc_header
         totalsize = size_gc_header + size
         rawtotalsize = raw_malloc_usage(totalsize)
@@ -269,7 +274,18 @@
         #
         if rawtotalsize <= self.small_request_threshold:
             #
-            # Case 1: we have run out of the free list corresponding to
+            # Case 1: unless trigger_next_collection() happened to get us
+            # more locations in free_lists[n], we have run out of them
+            n = rawtotalsize >> WORD_POWER_2
+            head = self.free_lists[n]
+            if head:
+                self.free_lists[n] = self.cast_int_to_hdrptr(head.tid)
+                obj = self.grow_reservation(head, totalsize)
+                hdr = self.header(obj)
+                hdr.tid = self.combine(typeid, self.current_mark, 0)
+                return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
+            #
+            # We really have run out of the free list corresponding to
             # the size.  Grab the next free page.
             newpage = self.free_pages
             if newpage == self.NULL:
@@ -279,15 +295,12 @@
             #
             # Put the free page in the list 'nonfree_pages[n]'.  This is
             # a linked list chained through the first word of each page.
-            n = rawtotalsize >> WORD_POWER_2
             newpage.tid = self.cast_hdrptr_to_int(self.nonfree_pages[n])
             self.nonfree_pages[n] = newpage
             #
             # Initialize the free page to contain objects of the given
             # size.  This requires setting up all object locations in the
             # page, linking them in the free list.
-            head = self.free_lists[n]
-            ll_assert(not head, "_malloc_slowpath: unexpected free_lists[n]")
             i = self.page_size - rawtotalsize
             limit = rawtotalsize + raw_malloc_usage(self.HDRSIZE)
             newpageadr = llmemory.cast_ptr_to_adr(newpage)
@@ -478,7 +491,7 @@
         finish.  Guarantees that objects not reachable when collect()
         is called will be freed by the time collect() returns.
         """
-        if gen >= 1 or self.collection_running == 0:
+        if gen >= 1 or self.collection_running <= 0:
             self.trigger_next_collection()
             if gen >= 2:
                 self.wait_for_the_end_of_collection()
@@ -550,7 +563,7 @@
         else:
             while rffi.cast(lltype.Signed,
                             ll_thread.c_thread_acquirelock(lock, 0)) == 0:
-                time.sleep(0.001)
+                time.sleep(0.05)
                 # ---------- EXCEPTION FROM THE COLLECTOR THREAD ----------
                 if hasattr(self, '_exc_info'):
                     self._reraise_from_collector_thread()
@@ -572,22 +585,21 @@
 
 
     def collector_run_nontranslated(self):
-        if hasattr(self, 'ready_to_start_lock'):      # normal tests
-            try:
+        try:
+            if hasattr(self, 'ready_to_start_lock'):      # normal tests
                 self.collector_run()
-            except Exception, e:
-                print 'Crash!', e.__class__.__name__, e
-                self._exc_info = sys.exc_info()
-        else:
-            # this case is for test_transformed_gc: we need to spawn
-            # another LLInterpreter for this new thread.
-            from pypy.rpython.llinterp import LLInterpreter
-            prev = LLInterpreter.current_interpreter
-            llinterp = LLInterpreter(prev.typer)
-            # XXX FISH HORRIBLY for the graph...
-            graph = sys._getframe(2).f_locals['self']._obj.graph
-            llinterp.eval_graph(graph)
-
+            else:
+                # this case is for test_transformed_gc: we need to spawn
+                # another LLInterpreter for this new thread.
+                from pypy.rpython.llinterp import LLInterpreter
+                prev = LLInterpreter.current_interpreter
+                llinterp = LLInterpreter(prev.typer)
+                # XXX FISH HORRIBLY for the graph...
+                graph = sys._getframe(2).f_locals['self']._obj.graph
+                llinterp.eval_graph(graph)
+        except Exception, e:
+            print 'Crash!', e.__class__.__name__, e
+            self._exc_info = sys.exc_info()
 
     def collector_run(self):
         """Main function of the collector's thread."""
@@ -613,6 +625,9 @@
             #
             # Sweep
             self.collector_sweep()
+            #
+            # Done!
+            self.collection_running = -1
             self.release(self.finished_lock)
 
 
diff --git a/pypy/rpython/memory/test/test_transformed_gc.py b/pypy/rpython/memory/test/test_transformed_gc.py
--- a/pypy/rpython/memory/test/test_transformed_gc.py
+++ b/pypy/rpython/memory/test/test_transformed_gc.py
@@ -119,21 +119,27 @@
         cls.rtyper = t.rtyper
         cls.db = db
 
+    def do_teardown(self):
+        if self.__class__.__dict__.get('_used', False):
+            from pypy.rpython.llinterp import LLInterpreter
+            llinterp = LLInterpreter(self.rtyper)
+            gct = self.db.gctransformer
+            teardowngraph = gct.frameworkgc__teardown_ptr.value._obj.graph
+            llinterp.eval_graph(teardowngraph, [])
+            self.__class__._used = False
+
     def runner(self, name, statistics=False, transformer=False):
         db = self.db
         name_to_func = self.name_to_func
         entrygraph = self.entrygraph
         from pypy.rpython.llinterp import LLInterpreter
 
+        self.do_teardown()
+        self.__class__._used = True
+
         llinterp = LLInterpreter(self.rtyper)
-
         gct = db.gctransformer
 
-        if self.__class__.__dict__.get('_used', False):
-            teardowngraph = gct.frameworkgc__teardown_ptr.value._obj.graph
-            llinterp.eval_graph(teardowngraph, [])
-        self.__class__._used = True
-
         # FIIIIISH
         setupgraph = gct.frameworkgc_setup_ptr.value._obj.graph
         # setup => resets the gc
@@ -144,6 +150,7 @@
             for i in range(len(args)):
                 ll_args[1+i] = args[i]
             res = llinterp.eval_graph(entrygraph, [ll_args])
+            self.do_teardown()
             return res
 
         if statistics:

From noreply at buildbot.pypy.org  Sun Oct  9 21:09:55 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Sun,  9 Oct 2011 21:09:55 +0200 (CEST)
Subject: [pypy-commit] pypy string-promote-2: a branch to experiment with
	promoting strings
Message-ID: <20111009190955.35B5D82112@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: string-promote-2
Changeset: r47909:8bc71436e96c
Date: 2011-10-09 21:09 +0200
http://bitbucket.org/pypy/pypy/changeset/8bc71436e96c/

Log:	a branch to experiment with promoting strings

diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -455,6 +455,12 @@
             # the special return value None forces op.result to be considered
             # equal to op.args[0]
             return [op0, op1, None]
+        if (hints.get('string_promote') and
+            op.args[0].concretetype is not lltype.Void):
+            assert op.args[0].concretetype == lltype.Ptr(rstr.STR)
+            op0 = SpaceOperation('-live-', [], None)
+            op1 = SpaceOperation('str_guard_value', [op.args[0]], op.result)
+            return [op0, op1, None]
         else:
             log.WARNING('ignoring hint %r at %r' % (hints, self.graph))
 
diff --git a/pypy/jit/codewriter/test/test_jtransform.py b/pypy/jit/codewriter/test/test_jtransform.py
--- a/pypy/jit/codewriter/test/test_jtransform.py
+++ b/pypy/jit/codewriter/test/test_jtransform.py
@@ -844,6 +844,20 @@
     assert op1.args[2] == ListOfKind('ref', [v1, v2])
     assert op1.result == v3
 
+def test_str_promote():
+    PSTR = lltype.Ptr(rstr.STR)
+    v1 = varoftype(PSTR)
+    v2 = varoftype(PSTR)
+    op = SpaceOperation('hint',
+                        [v1, Constant({'string_promote': True}, lltype.Void)],
+                        v2)
+    tr = Transformer(FakeCPU(), FakeBuiltinCallControl())
+    op0, op1, _ = tr.rewrite_operation(op)
+    assert op0.opname == '-live-'
+    assert op1.opname == 'str_guard_value'
+    assert op1.args == [v1]
+    assert op1.result == v2
+
 def test_unicode_concat():
     # test that the oopspec is present and correctly transformed
     PSTR = lltype.Ptr(rstr.UNICODE)
diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -38,6 +38,7 @@
     possible arguments are:
 
     * promote - promote the argument from a variable into a constant
+    * string_promote - same, but promote string by *value*
     * access_directly - directly access a virtualizable, as a structure
                         and don't treat it as a virtualizable
     * fresh_virtualizable - means that virtualizable was just allocated.
@@ -51,6 +52,9 @@
 def promote(x):
     return hint(x, promote=True)
 
+def promote_string(x):
+    return hint(x, promote_string=True)
+
 def dont_look_inside(func):
     """ Make sure the JIT does not trace inside decorated function
     (it becomes a call instead)

From noreply at buildbot.pypy.org  Sun Oct  9 21:28:07 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Sun,  9 Oct 2011 21:28:07 +0200 (CEST)
Subject: [pypy-commit] pypy lightweight-finalizers: all casts are ok
Message-ID: <20111009192807.E11C882112@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: lightweight-finalizers
Changeset: r47910:fd38e80d7844
Date: 2011-10-09 21:18 +0200
http://bitbucket.org/pypy/pypy/changeset/fd38e80d7844/

Log:	all casts are ok

diff --git a/pypy/translator/backendopt/finalizer.py b/pypy/translator/backendopt/finalizer.py
--- a/pypy/translator/backendopt/finalizer.py
+++ b/pypy/translator/backendopt/finalizer.py
@@ -12,13 +12,13 @@
     * anything that can allocate
     """
     ok_operations = ['ptr_nonzero', 'free', 'same_as',
-                     'direct_ptradd', 'force_cast', 'cast_primitive',
-                     'cast_pointer']
+                     'direct_ptradd', 'force_cast']
     
     def analyze_simple_operation(self, op, graphinfo):
         if op.opname in self.ok_operations:
             return self.bottom_result()
-        if op.opname.startswith('int_') or op.opname.startswith('float_'):
+        if (op.opname.startswith('int_') or op.opname.startswith('float_')
+            or op.opname.startswith('cast_')):
             return self.bottom_result()
         if op.opname == 'setfield':
             TP = op.args[2].concretetype

From noreply at buildbot.pypy.org  Sun Oct  9 21:28:09 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Sun,  9 Oct 2011 21:28:09 +0200 (CEST)
Subject: [pypy-commit] pypy lightweight-finalizers: Various operations and a
	test
Message-ID: <20111009192809.246E082112@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: lightweight-finalizers
Changeset: r47911:ad34a15783f9
Date: 2011-10-09 21:26 +0200
http://bitbucket.org/pypy/pypy/changeset/ad34a15783f9/

Log:	Various operations and a test

diff --git a/pypy/translator/backendopt/finalizer.py b/pypy/translator/backendopt/finalizer.py
--- a/pypy/translator/backendopt/finalizer.py
+++ b/pypy/translator/backendopt/finalizer.py
@@ -20,7 +20,7 @@
         if (op.opname.startswith('int_') or op.opname.startswith('float_')
             or op.opname.startswith('cast_')):
             return self.bottom_result()
-        if op.opname == 'setfield':
+        if op.opname == 'setfield' or op.opname == 'bare_setfield':
             TP = op.args[2].concretetype
             if not isinstance(TP, lltype.Ptr) or TP.TO._gckind == 'raw':
                 # primitive type
diff --git a/pypy/translator/backendopt/test/test_finalizer.py b/pypy/translator/backendopt/test/test_finalizer.py
--- a/pypy/translator/backendopt/test/test_finalizer.py
+++ b/pypy/translator/backendopt/test/test_finalizer.py
@@ -3,6 +3,7 @@
 from pypy.translator.backendopt.finalizer import FinalizerAnalyzer
 from pypy.translator.translator import TranslationContext, graphof
 from pypy.translator.backendopt.all import backend_optimizations
+from pypy.translator.unsimplify import varoftype
 from pypy.rpython.lltypesystem import lltype, rffi
 from pypy.conftest import option
 
@@ -34,7 +35,32 @@
         r = self.analyze(f, [])
         assert not r
 
+def test_various_ops():
+    from pypy.objspace.flow.model import SpaceOperation, Constant
 
+    X = lltype.Ptr(lltype.GcStruct('X'))
+    Z = lltype.Ptr(lltype.Struct('Z'))
+    S = lltype.GcStruct('S', ('x', lltype.Signed),
+                        ('y', X),
+                        ('z', Z))
+    v1 = varoftype(lltype.Bool)
+    v2 = varoftype(lltype.Signed)
+    f = FinalizerAnalyzer(None)
+    r = f.analyze(SpaceOperation('cast_int_to_bool', [v2],
+                                                       v1))
+    assert not r
+    v1 = varoftype(lltype.Ptr(S))
+    v2 = varoftype(lltype.Signed)
+    v3 = varoftype(X)
+    v4 = varoftype(Z)
+    assert not f.analyze(SpaceOperation('bare_setfield', [v1, Constant('x'),
+                                                          v2], None))
+    assert     f.analyze(SpaceOperation('bare_setfield', [v1, Constant('y'),
+                                                          v3], None))
+    assert not f.analyze(SpaceOperation('bare_setfield', [v1, Constant('z'),
+                                                          v4], None))
+    
+        
 class TestLLType(BaseFinalizerAnalyzerTests):
     type_system = 'lltype'
 

From noreply at buildbot.pypy.org  Sun Oct  9 21:54:20 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  9 Oct 2011 21:54:20 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Start to write debug info,
 very minimal for now
Message-ID: <20111009195420.ED62982112@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r47912:d999161e4c7a
Date: 2011-10-09 21:33 +0200
http://bitbucket.org/pypy/pypy/changeset/d999161e4c7a/

Log:	Start to write debug info, very minimal for now

diff --git a/pypy/rpython/memory/gc/concurrentms.py b/pypy/rpython/memory/gc/concurrentms.py
--- a/pypy/rpython/memory/gc/concurrentms.py
+++ b/pypy/rpython/memory/gc/concurrentms.py
@@ -5,7 +5,7 @@
 from pypy.rpython.annlowlevel import llhelper
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
 from pypy.rlib.objectmodel import we_are_translated, running_on_llinterp
-from pypy.rlib.debug import ll_assert, debug_print
+from pypy.rlib.debug import ll_assert, debug_print, debug_start, debug_stop
 from pypy.rlib.rarithmetic import ovfcheck, LONG_BIT, r_uint
 from pypy.rpython.memory.gc.base import GCBase
 from pypy.module.thread import ll_thread
@@ -448,6 +448,8 @@
         """In the mutator thread: wait for the collection currently
         running (if any) to finish."""
         if self.collection_running != 0:
+            debug_start("gc-stop")
+            #
             self.acquire(self.finished_lock)
             self.collection_running = 0
             #
@@ -476,6 +478,8 @@
             #
             if self.DEBUG:
                 self.debug_check_lists()
+            #
+            debug_stop("gc-stop")
 
 
     def collect(self, gen=2):
@@ -502,6 +506,8 @@
         # In case the previous collection is not over yet, wait for it
         self.wait_for_the_end_of_collection()
         #
+        debug_start("gc-start")
+        #
         # Scan the stack roots and the refs in non-GC objects
         self.root_walker.walk_roots(
             MostlyConcurrentMarkSweepGC._add_stack_root,  # stack roots
@@ -528,6 +534,8 @@
         # Start the collector thread
         self.collection_running = 1
         self.release(self.ready_to_start_lock)
+        #
+        debug_stop("gc-start")
 
     def _add_stack_root(self, root):
         obj = root.address[0]

From noreply at buildbot.pypy.org  Sun Oct  9 21:54:22 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sun,  9 Oct 2011 21:54:22 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Try to fix
 test_transformed_gc. Still not perfect...
Message-ID: <20111009195422.3C39E82112@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r47913:d13d5d0fc1f5
Date: 2011-10-09 21:54 +0200
http://bitbucket.org/pypy/pypy/changeset/d13d5d0fc1f5/

Log:	Try to fix test_transformed_gc. Still not perfect...

diff --git a/pypy/rpython/memory/gc/concurrentms.py b/pypy/rpython/memory/gc/concurrentms.py
--- a/pypy/rpython/memory/gc/concurrentms.py
+++ b/pypy/rpython/memory/gc/concurrentms.py
@@ -594,14 +594,13 @@
 
     def collector_run_nontranslated(self):
         try:
-            if hasattr(self, 'ready_to_start_lock'):      # normal tests
-                self.collector_run()
+            if not hasattr(self, 'currently_running_in_rtyper'):
+                self.collector_run()     # normal tests
             else:
                 # this case is for test_transformed_gc: we need to spawn
                 # another LLInterpreter for this new thread.
                 from pypy.rpython.llinterp import LLInterpreter
-                prev = LLInterpreter.current_interpreter
-                llinterp = LLInterpreter(prev.typer)
+                llinterp = LLInterpreter(self.currently_running_in_rtyper)
                 # XXX FISH HORRIBLY for the graph...
                 graph = sys._getframe(2).f_locals['self']._obj.graph
                 llinterp.eval_graph(graph)
diff --git a/pypy/rpython/memory/test/test_transformed_gc.py b/pypy/rpython/memory/test/test_transformed_gc.py
--- a/pypy/rpython/memory/test/test_transformed_gc.py
+++ b/pypy/rpython/memory/test/test_transformed_gc.py
@@ -4,6 +4,7 @@
 from pypy.translator.c import gc
 from pypy.annotation import model as annmodel
 from pypy.annotation import policy as annpolicy
+from pypy.annotation.listdef import s_list_of_strings
 from pypy.rpython.lltypesystem import lltype, llmemory, llarena, rffi, llgroup
 from pypy.rpython.memory.gctransform import framework
 from pypy.rpython.lltypesystem.lloperation import llop, void
@@ -37,8 +38,6 @@
         t.viewcg()
     return t
 
-ARGS = lltype.FixedSizeArray(lltype.Signed, 3)
-
 class GCTest(object):
     gcpolicy = None
     GC_CAN_MOVE = False
@@ -83,13 +82,13 @@
             cleanups.append(cleanup)
 
         def entrypoint(args):
-            num = args[0]
+            num = int(args[0])
             func = funcs0[num]
             if func:
                 res = func()
             else:
                 func = funcs2[num]
-                res = func(args[1], args[2])
+                res = func(int(args[1]), int(args[2]))
             cleanup = cleanups[num]
             if cleanup:
                 cleanup()
@@ -97,8 +96,7 @@
 
         from pypy.translator.c.genc import CStandaloneBuilder
 
-        s_args = annmodel.SomePtr(lltype.Ptr(ARGS))
-        t = rtype(entrypoint, [s_args], gcname=cls.gcname,
+        t = rtype(entrypoint, [s_list_of_strings], gcname=cls.gcname,
                   taggedpointers=cls.taggedpointers)
 
         for fixup in mixlevelstuff:
@@ -109,6 +107,8 @@
         cbuild = CStandaloneBuilder(t, entrypoint, config=t.config,
                                     gcpolicy=cls.gcpolicy)
         db = cbuild.generate_graphs_for_llinterp()
+        gc = cbuild.db.gctransformer.gcdata.gc
+        gc.currently_running_in_rtyper = t.rtyper
         entrypointptr = cbuild.getentrypointptr()
         entrygraph = entrypointptr._obj.graph
         if conftest.option.view:
@@ -145,11 +145,14 @@
         # setup => resets the gc
         llinterp.eval_graph(setupgraph, [])
         def run(args):
-            ll_args = lltype.malloc(ARGS, immortal=True)
-            ll_args[0] = name_to_func[name]
+            ll_args = lltype.malloc(rffi.CCHARPP.TO, 1+len(args),
+                                    immortal=True)
+            ll_args[0] = rffi.str2charp(str(name_to_func[name]))
             for i in range(len(args)):
-                ll_args[1+i] = args[i]
-            res = llinterp.eval_graph(entrygraph, [ll_args])
+                ll_args[1+i] = rffi.str2charp(str(args[i]))
+            res = llinterp.eval_graph(entrygraph, [1+len(args), ll_args])
+            for i in range(1+len(args)):
+                rffi.free_charp(ll_args[i])
             self.do_teardown()
             return res
 

From noreply at buildbot.pypy.org  Sun Oct  9 21:54:52 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Sun,  9 Oct 2011 21:54:52 +0200 (CEST)
Subject: [pypy-commit] pypy lightweight-finalizers: track_allocation is just
	fine
Message-ID: <20111009195452.9DB5982112@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: lightweight-finalizers
Changeset: r47914:58152c4eb257
Date: 2011-10-09 21:54 +0200
http://bitbucket.org/pypy/pypy/changeset/58152c4eb257/

Log:	track_allocation is just fine

diff --git a/pypy/translator/backendopt/finalizer.py b/pypy/translator/backendopt/finalizer.py
--- a/pypy/translator/backendopt/finalizer.py
+++ b/pypy/translator/backendopt/finalizer.py
@@ -12,7 +12,7 @@
     * anything that can allocate
     """
     ok_operations = ['ptr_nonzero', 'free', 'same_as',
-                     'direct_ptradd', 'force_cast']
+                     'direct_ptradd', 'force_cast', 'track_allocation']
     
     def analyze_simple_operation(self, op, graphinfo):
         if op.opname in self.ok_operations:

From noreply at buildbot.pypy.org  Sun Oct  9 21:58:00 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Sun,  9 Oct 2011 21:58:00 +0200 (CEST)
Subject: [pypy-commit] pypy lightweight-finalizers: raw_free is also fine
Message-ID: <20111009195800.AA43D82112@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: lightweight-finalizers
Changeset: r47915:0316138a9f44
Date: 2011-10-09 21:57 +0200
http://bitbucket.org/pypy/pypy/changeset/0316138a9f44/

Log:	raw_free is also fine

diff --git a/pypy/translator/backendopt/finalizer.py b/pypy/translator/backendopt/finalizer.py
--- a/pypy/translator/backendopt/finalizer.py
+++ b/pypy/translator/backendopt/finalizer.py
@@ -12,7 +12,8 @@
     * anything that can allocate
     """
     ok_operations = ['ptr_nonzero', 'free', 'same_as',
-                     'direct_ptradd', 'force_cast', 'track_allocation']
+                     'direct_ptradd', 'force_cast', 'track_allocation',
+                     'raw_free']
     
     def analyze_simple_operation(self, op, graphinfo):
         if op.opname in self.ok_operations:

From noreply at buildbot.pypy.org  Sun Oct  9 22:25:03 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Sun,  9 Oct 2011 22:25:03 +0200 (CEST)
Subject: [pypy-commit] pypy lightweight-finalizers: precise name of the llop
Message-ID: <20111009202503.DE93182112@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: lightweight-finalizers
Changeset: r47916:dc6caa304710
Date: 2011-10-09 22:24 +0200
http://bitbucket.org/pypy/pypy/changeset/dc6caa304710/

Log:	precise name of the llop

diff --git a/pypy/translator/backendopt/finalizer.py b/pypy/translator/backendopt/finalizer.py
--- a/pypy/translator/backendopt/finalizer.py
+++ b/pypy/translator/backendopt/finalizer.py
@@ -12,7 +12,7 @@
     * anything that can allocate
     """
     ok_operations = ['ptr_nonzero', 'free', 'same_as',
-                     'direct_ptradd', 'force_cast', 'track_allocation',
+                     'direct_ptradd', 'force_cast', 'track_alloc_stop',
                      'raw_free']
     
     def analyze_simple_operation(self, op, graphinfo):

From noreply at buildbot.pypy.org  Sun Oct  9 22:31:11 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Sun,  9 Oct 2011 22:31:11 +0200 (CEST)
Subject: [pypy-commit] pypy lightweight-finalizers: pointer comparisons
Message-ID: <20111009203111.D550A82112@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: lightweight-finalizers
Changeset: r47917:f59fad7e8cb5
Date: 2011-10-09 22:30 +0200
http://bitbucket.org/pypy/pypy/changeset/f59fad7e8cb5/

Log:	pointer comparisons

diff --git a/pypy/translator/backendopt/finalizer.py b/pypy/translator/backendopt/finalizer.py
--- a/pypy/translator/backendopt/finalizer.py
+++ b/pypy/translator/backendopt/finalizer.py
@@ -11,7 +11,7 @@
     * anything that escapes self
     * anything that can allocate
     """
-    ok_operations = ['ptr_nonzero', 'free', 'same_as',
+    ok_operations = ['ptr_nonzero', 'ptr_eq', 'ptr_ne', 'free', 'same_as',
                      'direct_ptradd', 'force_cast', 'track_alloc_stop',
                      'raw_free']
     

From notifications-noreply at bitbucket.org  Sun Oct  9 23:02:24 2011
From: notifications-noreply at bitbucket.org (Bitbucket)
Date: Sun, 09 Oct 2011 21:02:24 -0000
Subject: [pypy-commit] Notification: pypy
Message-ID: <20111009210224.26668.60188@bitbucket01.managed.contegix.com>

You have received a notification from Tim Sally.

Hi, I forked pypy. My fork is at https://bitbucket.org/tsally/pypy.

--
Disable notifications at https://bitbucket.org/account/notifications/

From noreply at buildbot.pypy.org  Sun Oct  9 23:38:10 2011
From: noreply at buildbot.pypy.org (justinpeel)
Date: Sun,  9 Oct 2011 23:38:10 +0200 (CEST)
Subject: [pypy-commit] pypy rgc-mem-pressure: try to apply some memory
	pressure to _hashlib
Message-ID: <20111009213811.0169D82112@wyvern.cs.uni-duesseldorf.de>

Author: Justin Peel <notmuchtotell at gmail.com>
Branch: rgc-mem-pressure
Changeset: r47918:13feff9d23f3
Date: 2011-10-09 15:37 -0600
http://bitbucket.org/pypy/pypy/changeset/13feff9d23f3/

Log:	try to apply some memory pressure to _hashlib

diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py
--- a/pypy/module/_hashlib/interp_hashlib.py
+++ b/pypy/module/_hashlib/interp_hashlib.py
@@ -4,9 +4,9 @@
 from pypy.interpreter.error import OperationError
 from pypy.tool.sourcetools import func_renamer
 from pypy.interpreter.baseobjspace import Wrappable
-from pypy.rpython.lltypesystem import lltype, rffi
+from pypy.rpython.lltypesystem import lltype, llmemory, rffi
+from pypy.rlib import rgc, ropenssl
 from pypy.rlib.objectmodel import keepalive_until_here
-from pypy.rlib import ropenssl
 from pypy.rlib.rstring import StringBuilder
 from pypy.module.thread.os_lock import Lock
 
@@ -29,6 +29,9 @@
                                  space.wrap("unknown hash function"))
         ctx = lltype.malloc(ropenssl.EVP_MD_CTX.TO, flavor='raw')
         ropenssl.EVP_DigestInit(ctx, digest)
+        rgc.add_memory_pressure(ropenssl.EVP_MD_CTX.TO.hints['getsize']() +
+                                ropenssl.EVP_MD.TO.hints['getsize']() +
+                                self._digest_size())
         self.ctx = ctx
 
     def __del__(self):
diff --git a/pypy/rlib/ropenssl.py b/pypy/rlib/ropenssl.py
--- a/pypy/rlib/ropenssl.py
+++ b/pypy/rlib/ropenssl.py
@@ -25,6 +25,7 @@
     'openssl/err.h',
     'openssl/rand.h',
     'openssl/evp.h',
+    'openssl/ossl_typ.h',
     'openssl/x509v3.h']
 
 eci = ExternalCompilationInfo(
@@ -154,7 +155,7 @@
 ssl_external('CRYPTO_set_id_callback',
              [lltype.Ptr(lltype.FuncType([], rffi.LONG))],
              lltype.Void)
-             
+
 if HAVE_OPENSSL_RAND:
     ssl_external('RAND_add', [rffi.CCHARP, rffi.INT, rffi.DOUBLE], lltype.Void)
     ssl_external('RAND_status', [], rffi.INT)
@@ -255,7 +256,7 @@
              [BIO, rffi.VOIDP, rffi.VOIDP, rffi.VOIDP], X509)
 
 EVP_MD_CTX = rffi.COpaquePtr('EVP_MD_CTX', compilation_info=eci)
-EVP_MD     = rffi.COpaquePtr('EVP_MD')
+EVP_MD     = rffi.COpaquePtr('EVP_MD', compilation_info=eci)
 
 OpenSSL_add_all_digests = external(
     'OpenSSL_add_all_digests', [], lltype.Void)
diff --git a/pypy/rpython/lltypesystem/lltype.py b/pypy/rpython/lltypesystem/lltype.py
--- a/pypy/rpython/lltypesystem/lltype.py
+++ b/pypy/rpython/lltypesystem/lltype.py
@@ -48,7 +48,7 @@
         self.TYPE = TYPE
     def __repr__(self):
         return '<Uninitialized %r>'%(self.TYPE,)
-        
+
 
 def saferecursive(func, defl, TLS=TLS):
     def safe(*args):
@@ -537,9 +537,9 @@
         return "Func ( %s ) -> %s" % (args, self.RESULT)
     __str__ = saferecursive(__str__, '...')
 
-    def _short_name(self):        
+    def _short_name(self):
         args = ', '.join([ARG._short_name() for ARG in self.ARGS])
-        return "Func(%s)->%s" % (args, self.RESULT._short_name())        
+        return "Func(%s)->%s" % (args, self.RESULT._short_name())
     _short_name = saferecursive(_short_name, '...')
 
     def _container_example(self):
@@ -553,7 +553,7 @@
 
 class OpaqueType(ContainerType):
     _gckind = 'raw'
-    
+
     def __init__(self, tag, hints={}):
         """ if hints['render_structure'] is set, the type is internal and not considered
             to come from somewhere else (it should be rendered as a structure) """
@@ -723,10 +723,10 @@
 
     def __str__(self):
         return '* %s' % (self.TO, )
-    
+
     def _short_name(self):
         return 'Ptr %s' % (self.TO._short_name(), )
-    
+
     def _is_atomic(self):
         return self.TO._gckind == 'raw'
 
@@ -1520,7 +1520,7 @@
             and parentindex in (self._parent_type._names[0], 0)
             and self._TYPE._gckind == typeOf(parent)._gckind):
             # keep strong reference to parent, we share the same allocation
-            self._keepparent = parent 
+            self._keepparent = parent
 
     def _parentstructure(self, check=True):
         if self._wrparent is not None:
diff --git a/pypy/translator/c/test/test_newgc.py b/pypy/translator/c/test/test_newgc.py
--- a/pypy/translator/c/test/test_newgc.py
+++ b/pypy/translator/c/test/test_newgc.py
@@ -228,7 +228,7 @@
         T = lltype.GcStruct("T", ('y', lltype.Signed),
                                  ('s', lltype.Ptr(S)))
         ARRAY_Ts = lltype.GcArray(lltype.Ptr(T))
-        
+
         def f():
             r = 0
             for i in range(30):
@@ -250,7 +250,7 @@
     def test_framework_varsized(self):
         res = self.run('framework_varsized')
         assert res == self.run_orig('framework_varsized')
-            
+
     def define_framework_using_lists(cls):
         class A(object):
             pass
@@ -271,7 +271,7 @@
         N = 1000
         res = self.run('framework_using_lists')
         assert res == N*(N - 1)/2
-    
+
     def define_framework_static_roots(cls):
         class A(object):
             def __init__(self, y):
@@ -318,8 +318,8 @@
     def test_framework_void_array(self):
         res = self.run('framework_void_array')
         assert res == 44
-        
-        
+
+
     def define_framework_malloc_failure(cls):
         def f():
             a = [1] * (sys.maxint//2)
@@ -342,7 +342,7 @@
     def test_framework_array_of_void(self):
         res = self.run('framework_array_of_void')
         assert res == 43 + 1000000
-        
+
     def define_framework_opaque(cls):
         A = lltype.GcStruct('A', ('value', lltype.Signed))
         O = lltype.GcOpaqueType('test.framework')
@@ -437,7 +437,7 @@
             b = B()
             return 0
         return func
-    
+
     def test_del_raises(self):
         self.run('del_raises') # does not raise
 
@@ -712,7 +712,7 @@
 
     def test_callback_with_collect(self):
         assert self.run('callback_with_collect')
-    
+
     def define_can_move(cls):
         class A:
             pass
@@ -1255,7 +1255,7 @@
         l1 = []
         l2 = []
         l3 = []
-        
+
         def f():
             for i in range(10):
                 s = lltype.malloc(S)
@@ -1298,7 +1298,7 @@
     def test_string_builder(self):
         res = self.run('string_builder')
         assert res == "aabcbdddd"
-    
+
     def definestr_string_builder_over_allocation(cls):
         import gc
         def fn(_):

From noreply at buildbot.pypy.org  Mon Oct 10 14:14:53 2011
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 10 Oct 2011 14:14:53 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20111010121453.CCE8D82A88@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r47920:9afae5e2f69a
Date: 2011-10-10 13:16 +0200
http://bitbucket.org/pypy/pypy/changeset/9afae5e2f69a/

Log:	merge

diff --git a/pypy/jit/metainterp/heapcache.py b/pypy/jit/metainterp/heapcache.py
--- a/pypy/jit/metainterp/heapcache.py
+++ b/pypy/jit/metainterp/heapcache.py
@@ -54,9 +54,10 @@
         if box in self.new_boxes:
             self.new_boxes[box] = False
         if box in self.dependencies:
-            for dep in self.dependencies[box]:
+            deps = self.dependencies[box]
+            del self.dependencies[box]
+            for dep in deps:
                 self._escape(dep)
-            del self.dependencies[box]
 
     def clear_caches(self, opnum, descr, argboxes):
         if opnum == rop.SETFIELD_GC:
diff --git a/pypy/jit/metainterp/test/test_heapcache.py b/pypy/jit/metainterp/test/test_heapcache.py
--- a/pypy/jit/metainterp/test/test_heapcache.py
+++ b/pypy/jit/metainterp/test/test_heapcache.py
@@ -355,6 +355,14 @@
         assert not h.is_unescaped(box1)
         assert not h.is_unescaped(box2)
 
+    def test_circular_virtuals(self):
+        h = HeapCache()
+        h.new(box1)
+        h.new(box2)
+        h.invalidate_caches(rop.SETFIELD_GC, None, [box1, box2])
+        h.invalidate_caches(rop.SETFIELD_GC, None, [box2, box1])
+        h.invalidate_caches(rop.SETFIELD_GC, None, [box3, box1]) # does not crash
+
     def test_unescaped_array(self):
         h = HeapCache()
         h.new_array(box1, lengthbox1)
@@ -362,4 +370,4 @@
         h.invalidate_caches(rop.SETARRAYITEM_GC, None, [box1, index1, box2])
         assert h.is_unescaped(box1)
         h.invalidate_caches(rop.SETARRAYITEM_GC, None, [box2, index1, box1])
-        assert not h.is_unescaped(box1)
\ No newline at end of file
+        assert not h.is_unescaped(box1)

From noreply at buildbot.pypy.org  Mon Oct 10 14:14:52 2011
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 10 Oct 2011 14:14:52 +0200 (CEST)
Subject: [pypy-commit] pypy default: Bad Alex,
	no cookies: deal with recursive virtuals.
Message-ID: <20111010121452.99FC782112@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r47919:cc2e72d09e18
Date: 2011-10-09 21:10 +0200
http://bitbucket.org/pypy/pypy/changeset/cc2e72d09e18/

Log:	Bad Alex, no cookies: deal with recursive virtuals.

	(you should have read the Allocation Removal paper better: the last
	optimization rule on page 7 *clearly* indicates that you need to
	remove the object from the static heap *before* recursively escaping
	an object :-) )

diff --git a/pypy/jit/metainterp/heapcache.py b/pypy/jit/metainterp/heapcache.py
--- a/pypy/jit/metainterp/heapcache.py
+++ b/pypy/jit/metainterp/heapcache.py
@@ -54,9 +54,10 @@
         if box in self.new_boxes:
             self.new_boxes[box] = False
         if box in self.dependencies:
-            for dep in self.dependencies[box]:
+            deps = self.dependencies[box]
+            del self.dependencies[box]
+            for dep in deps:
                 self._escape(dep)
-            del self.dependencies[box]
 
     def clear_caches(self, opnum, descr, argboxes):
         if opnum == rop.SETFIELD_GC:
diff --git a/pypy/jit/metainterp/test/test_heapcache.py b/pypy/jit/metainterp/test/test_heapcache.py
--- a/pypy/jit/metainterp/test/test_heapcache.py
+++ b/pypy/jit/metainterp/test/test_heapcache.py
@@ -355,6 +355,14 @@
         assert not h.is_unescaped(box1)
         assert not h.is_unescaped(box2)
 
+    def test_circular_virtuals(self):
+        h = HeapCache()
+        h.new(box1)
+        h.new(box2)
+        h.invalidate_caches(rop.SETFIELD_GC, None, [box1, box2])
+        h.invalidate_caches(rop.SETFIELD_GC, None, [box2, box1])
+        h.invalidate_caches(rop.SETFIELD_GC, None, [box3, box1]) # does not crash
+
     def test_unescaped_array(self):
         h = HeapCache()
         h.new_array(box1, lengthbox1)
@@ -362,4 +370,4 @@
         h.invalidate_caches(rop.SETARRAYITEM_GC, None, [box1, index1, box2])
         assert h.is_unescaped(box1)
         h.invalidate_caches(rop.SETARRAYITEM_GC, None, [box2, index1, box1])
-        assert not h.is_unescaped(box1)
\ No newline at end of file
+        assert not h.is_unescaped(box1)

From noreply at buildbot.pypy.org  Mon Oct 10 18:35:09 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 10 Oct 2011 18:35:09 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Try to list all missing
	features.
Message-ID: <20111010163509.F344882112@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r47921:4912f5adc38d
Date: 2011-10-10 18:13 +0200
http://bitbucket.org/pypy/pypy/changeset/4912f5adc38d/

Log:	Try to list all missing features.

diff --git a/pypy/rpython/memory/gc/concurrentms.py b/pypy/rpython/memory/gc/concurrentms.py
--- a/pypy/rpython/memory/gc/concurrentms.py
+++ b/pypy/rpython/memory/gc/concurrentms.py
@@ -37,7 +37,8 @@
 MARK_VALUE_2      = 0x6B    # 'k', 107
 MARK_VALUE_STATIC = 0x53    # 'S', 83
 # Next lower byte: a combination of flags.
-FL_WITHHASH = 0x0100
+FL_WITHHASH              = 0x0100
+FL_EXTRA                 = 0x0200
 # And the high half of the word contains the numeric typeid.
 
 
@@ -48,14 +49,16 @@
     needs_write_barrier = True
     prebuilt_gc_objects_are_static_roots = False
     malloc_zero_filled = True
-    #gcflag_extra = GCFLAG_FINALIZATION_ORDERING
+    gcflag_extra = FL_EXTRA
 
     HDR = lltype.Struct('header', ('tid', lltype.Signed))
     HDRPTR = lltype.Ptr(HDR)
     HDRSIZE = llmemory.sizeof(HDR)
     NULL = lltype.nullptr(HDR)
-    typeid_is_in_field = 'tid'
+    typeid_is_in_field = 'tid', llgroup.HALFSHIFT
     withhash_flag_is_in_field = 'tid', FL_WITHHASH
+    # ^^^ prebuilt objects may have the flag FL_WITHHASH;
+    #     then they are one word longer, the extra word storing the hash.
 
     TRANSLATION_PARAMS = {'page_size': 16384,
                           'small_request_threshold': 35*WORD,
@@ -374,6 +377,33 @@
         return result
     _malloc_varsize_slowpath._dont_inline_ = True
 
+    # ----------
+    # Other functions in the GC API
+
+    #def set_max_heap_size(self, size):
+    #    XXX
+
+    #def raw_malloc_memory_pressure(self, sizehint):
+    #    XXX
+
+    #def shrink_array(self, obj, smallerlength):
+    #    return False
+
+    def enumerate_all_roots(self, callback, arg):
+        self.prebuilt_root_objects.foreach(callback, arg)
+        GCBase.enumerate_all_roots(self, callback, arg)
+    enumerate_all_roots._annspecialcase_ = 'specialize:arg(1)'
+
+    def identityhash(self, obj):
+        obj = llmemory.cast_ptr_to_adr(obj)
+        if self.header(obj).tid & FL_WITHHASH:
+            obj += self.get_size(obj)
+            return obj.signed[0]
+        else:
+            return llmemory.cast_adr_to_int(obj)
+
+    # ----------
+
     def allocate_next_arena(self):
         # xxx for now, allocate one page at a time with the system malloc()
         page = llarena.arena_malloc(self.page_size, 2)     # zero-filled
@@ -408,6 +438,9 @@
             self.force_scan(dest_addr)
         return True
 
+    def assume_young_pointers(self, addr_struct):
+        XXX
+
     def _init_writebarrier_logic(self):
         #
         def force_scan(obj):
@@ -780,14 +813,6 @@
         self.collect_tails[n] = first_loc_in_linked_list
 
 
-    def identityhash(self, obj):
-        obj = llmemory.cast_ptr_to_adr(obj)
-        if self.header(obj).tid & FL_WITHHASH:
-            obj += self.get_size(obj)
-            return obj.signed[0]
-        else:
-            return llmemory.cast_adr_to_int(obj)
-
 # ____________________________________________________________
 #
 # Hack to write the 'mark' or the 'flags' bytes of an object header

From noreply at buildbot.pypy.org  Mon Oct 10 18:35:11 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 10 Oct 2011 18:35:11 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Fixes: custom_trace
 support again. The issue is that it was called
Message-ID: <20111010163511.3CE5E82112@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r47922:295b1a9fe890
Date: 2011-10-10 18:34 +0200
http://bitbucket.org/pypy/pypy/changeset/295b1a9fe890/

Log:	Fixes: custom_trace support again. The issue is that it was called
	by an indirect_call() with no known target set --- and so it was
	assumed to be able to raise.

diff --git a/pypy/rpython/llinterp.py b/pypy/rpython/llinterp.py
--- a/pypy/rpython/llinterp.py
+++ b/pypy/rpython/llinterp.py
@@ -675,6 +675,12 @@
             #log.warn("op_indirect_call with graphs=None:", f)
         return self.op_direct_call(f, *args)
 
+    def op_llcall_cannot_raise(self, llf, *args):
+        try:
+            return self.op_direct_call(llf, *args)
+        except LLException, e:
+            raise AssertionError("llcall_cannot_raise: raised %s" % e)
+
     def op_malloc(self, obj, flags):
         flavor = flags['flavor']
         zero = flags.get('zero', False)
diff --git a/pypy/rpython/lltypesystem/lloperation.py b/pypy/rpython/lltypesystem/lloperation.py
--- a/pypy/rpython/lltypesystem/lloperation.py
+++ b/pypy/rpython/lltypesystem/lloperation.py
@@ -186,6 +186,7 @@
 
     'direct_call':          LLOp(canraise=(Exception,)),
     'indirect_call':        LLOp(canraise=(Exception,)),
+    'llcall_cannot_raise':  LLOp(canrun=True),    # dangerous, use carefully!
 
     # __________ numeric operations __________
 
diff --git a/pypy/rpython/lltypesystem/opimpl.py b/pypy/rpython/lltypesystem/opimpl.py
--- a/pypy/rpython/lltypesystem/opimpl.py
+++ b/pypy/rpython/lltypesystem/opimpl.py
@@ -624,6 +624,13 @@
     from pypy.rlib.rtimer import read_timestamp
     return read_timestamp()
 
+def op_llcall_cannot_raise(llf, *args):
+    try:
+        return llf(*args)
+    except Exception, e:
+        raise AssertionError("llcall_cannot_raise: raised %s: %s" % (
+            e.__class__.__name__, e))
+
 # ____________________________________________________________
 
 def get_op_impl(opname):
diff --git a/pypy/rpython/memory/gc/base.py b/pypy/rpython/memory/gc/base.py
--- a/pypy/rpython/memory/gc/base.py
+++ b/pypy/rpython/memory/gc/base.py
@@ -1,4 +1,5 @@
 from pypy.rpython.lltypesystem import lltype, llmemory, llarena
+from pypy.rpython.lltypesystem.lloperation import llop
 from pypy.rlib.debug import ll_assert
 from pypy.rpython.memory.gcheader import GCHeaderBuilder
 from pypy.rpython.memory.support import DEFAULT_CHUNK_SIZE
@@ -231,11 +232,12 @@
                     j += 1
                 item += itemlength
                 length -= 1
-        if self.has_custom_trace(typeid) and 0: # XXX XXX temporarily disabled
+        if self.has_custom_trace(typeid):
             generator = self.get_custom_trace(typeid)
             item = llmemory.NULL
             while True:
-                item = generator(obj, item)
+                item = llop.llcall_cannot_raise(llmemory.Address,
+                                                generator, obj, item)
                 if not item:
                     break
                 if self.points_to_valid_gc_object(item):
diff --git a/pypy/rpython/memory/gc/concurrentms.py b/pypy/rpython/memory/gc/concurrentms.py
--- a/pypy/rpython/memory/gc/concurrentms.py
+++ b/pypy/rpython/memory/gc/concurrentms.py
@@ -439,7 +439,7 @@
         return True
 
     def assume_young_pointers(self, addr_struct):
-        XXX
+        pass # XXX
 
     def _init_writebarrier_logic(self):
         #
diff --git a/pypy/rpython/memory/gctransform/framework.py b/pypy/rpython/memory/gctransform/framework.py
--- a/pypy/rpython/memory/gctransform/framework.py
+++ b/pypy/rpython/memory/gctransform/framework.py
@@ -1286,6 +1286,7 @@
         fptr = rtti._obj.custom_trace_funcptr
         if not hasattr(fptr._obj, 'graph'):
             ll_func = fptr._obj._callable
+            ll_func._should_never_raise_ = True
             fptr = self.transformer.annotate_finalizer(ll_func,
                     [llmemory.Address, llmemory.Address], llmemory.Address)
         return fptr
diff --git a/pypy/rpython/memory/test/test_gc.py b/pypy/rpython/memory/test/test_gc.py
--- a/pypy/rpython/memory/test/test_gc.py
+++ b/pypy/rpython/memory/test/test_gc.py
@@ -276,6 +276,7 @@
             else:
                 py.test.raises((RuntimeError, ArenaError),
                                self.interpret, f, [])
+                print "NOTE: The previous ArenaError, if any, is expected"
 
     def test_weakref(self):
         import weakref, gc
diff --git a/pypy/translator/c/funcgen.py b/pypy/translator/c/funcgen.py
--- a/pypy/translator/c/funcgen.py
+++ b/pypy/translator/c/funcgen.py
@@ -462,14 +462,11 @@
         return self.generic_call(fn.concretetype, self.expr(fn),
                                  op.args[1:-1], op.result, op.args[-1].value)
 
-    def OP_ADR_CALL(self, op):
-        ARGTYPES = [v.concretetype for v in op.args[1:]]
-        RESTYPE = op.result.concretetype
-        FUNC = Ptr(FuncType(ARGTYPES, RESTYPE))
-        typename = self.db.gettype(FUNC)
-        fnaddr = op.args[0]
-        fnexpr = '((%s)%s)' % (cdecl(typename, ''), self.expr(fnaddr))
-        return self.generic_call(FUNC, fnexpr, op.args[1:], op.result)
+    def OP_LLCALL_CANNOT_RAISE(self, op):
+        fn = op.args[0]
+        res = self.generic_call(fn.concretetype, self.expr(fn),
+                                op.args[1:], op.result)
+        return res + ' /* llcall_cannot_raise */'
 
     # low-level operations
     def generic_get(self, op, sourceexpr):
diff --git a/pypy/translator/c/gc.py b/pypy/translator/c/gc.py
--- a/pypy/translator/c/gc.py
+++ b/pypy/translator/c/gc.py
@@ -378,18 +378,23 @@
         [v_obj, c_grpptr, c_skipoffset, c_vtableinfo] = op.args
         typename = funcgen.db.gettype(op.result.concretetype)
         tid_field = c_vtableinfo.value[2]
+        tid_offset = 0
+        if isinstance(tid_field, tuple):
+            tid_field, tid_offset = tid_field
         # Fish out the C name of the tid field.
         HDR = self.db.gctransformer.HDR
         hdr_node = self.db.gettypedefnode(HDR)
         fieldname = hdr_node.c_struct_field_name(tid_field)
+        #
+        tid = '%s->_gcheader.%s' % (funcgen.expr(v_obj), fieldname)
+        if tid_offset:
+            tid = '(%s >> %d)' % (tid, tid_offset)
         return (
-        '%s = (%s)_OP_GET_NEXT_GROUP_MEMBER(%s, (pypy_halfword_t)%s->'
-            '_gcheader.%s, %s);'
+        '%s = (%s)_OP_GET_NEXT_GROUP_MEMBER(%s, (pypy_halfword_t)%s, %s);'
             % (funcgen.expr(op.result),
                cdecl(typename, ''),
                funcgen.expr(c_grpptr),
-               funcgen.expr(v_obj),
-               fieldname,
+               tid,
                funcgen.expr(c_skipoffset)))
 
     def OP_GC_ASSUME_YOUNG_POINTERS(self, funcgen, op):

From noreply at buildbot.pypy.org  Mon Oct 10 18:52:39 2011
From: noreply at buildbot.pypy.org (hager)
Date: Mon, 10 Oct 2011 18:52:39 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: Implemented further comparison
	operations.
Message-ID: <20111010165239.D504E82112@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r47923:2eac535ed148
Date: 2011-10-10 18:48 +0200
http://bitbucket.org/pypy/pypy/changeset/2eac535ed148/

Log:	Implemented further comparison operations.

diff --git a/pypy/jit/backend/ppc/ppcgen/helper/assembler.py b/pypy/jit/backend/ppc/ppcgen/helper/assembler.py
--- a/pypy/jit/backend/ppc/ppcgen/helper/assembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/helper/assembler.py
@@ -1,17 +1,63 @@
 import pypy.jit.backend.ppc.ppcgen.condition as c
+from pypy.rlib.rarithmetic import r_uint, r_longlong, intmask
 
 def gen_emit_cmp_op(condition):
     def f(self, op, arglocs, regalloc):
         l0, l1, res = arglocs
+        # do the comparison
         if l1.is_imm():
             self.mc.cmpwi(0, l0.value, l1.value)
         else:
             self.mc.cmpw(0, l0.value, l1.value)
 
-        if condition == c.LE:
+        # After the comparison, place the result
+        # in the first bit of the CR
+        if condition == c.LT:
+            self.mc.cror(0, 0, 0)
+        elif condition == c.LE:
             self.mc.cror(0, 0, 2)
+        elif condition == c.EQ:
+            self.mc.cror(0, 2, 2)
+        elif condition == c.GE:
+            self.mc.cror(0, 1, 2)
+        elif condition == c.GT:
+            self.mc.cror(0, 1, 1)
 
-        resval = res.value
-        self.mc.mfcr(resval)
+        resval = res.value 
+        # move the content of the CR to resval
+        self.mc.mfcr(resval)       
+        # zero out everything except of the result
         self.mc.rlwinm(resval, resval, 1, 31, 31)
     return f
+
+def encode32(mem, i, n):
+    mem[i] = chr(n & 0xFF)
+    mem[i+1] = chr((n >> 8) & 0xFF)
+    mem[i+2] = chr((n >> 16) & 0xFF)
+    mem[i+3] = chr((n >> 24) & 0xFF)
+
+def decode32(mem, index):
+    return intmask(ord(mem[index])
+            | ord(mem[index+1]) << 8
+            | ord(mem[index+2]) << 16
+            | ord(mem[index+3]) << 24)
+
+class saved_registers(object):
+    def __init__(self, assembler, regs_to_save, regalloc=None):
+        self.assembler = assembler
+        self.regalloc = regalloc
+        if self.regalloc:
+            self._filter_regs(regs_to_save, vfp_regs_to_save)
+        else:
+            self.regs = regs_to_save
+
+    def __enter__(self):
+        if len(self.regs) > 0:
+            self.assembler.PUSH([r.value for r in self.regs])
+
+    def _filter_regs(self, regs_to_save, vfp_regs_to_save):
+        regs = []
+        for box, reg in self.regalloc.rm.reg_bindings.iteritems():
+            if reg is r.ip or (reg in regs_to_save and self.regalloc.stays_alive(box)):
+                regs.append(reg)
+        self.regs = regs

From noreply at buildbot.pypy.org  Mon Oct 10 18:52:41 2011
From: noreply at buildbot.pypy.org (hager)
Date: Mon, 10 Oct 2011 18:52:41 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: Fixed bug in emit_guard_true,
 removed some comments.
Message-ID: <20111010165241.0FDDE82112@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r47924:9593b3b88300
Date: 2011-10-10 18:50 +0200
http://bitbucket.org/pypy/pypy/changeset/9593b3b88300/

Log:	Fixed bug in emit_guard_true, removed some comments.

diff --git a/pypy/jit/backend/ppc/ppcgen/opassembler.py b/pypy/jit/backend/ppc/ppcgen/opassembler.py
--- a/pypy/jit/backend/ppc/ppcgen/opassembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/opassembler.py
@@ -3,7 +3,18 @@
 import pypy.jit.backend.ppc.ppcgen.register as r
 from pypy.jit.backend.ppc.ppcgen.arch import GPR_SAVE_AREA, IS_PPC_32, WORD
 
-from pypy.jit.metainterp.history import LoopToken
+from pypy.jit.metainterp.history import LoopToken, AbstractFailDescr
+
+class GuardToken(object):
+    def __init__(self, descr, failargs, faillocs, offset, fcond=c.NE,
+                                        save_exc=False, is_invalidate=False):
+        self.descr = descr
+        self.offset = offset
+        self.is_invalidate = is_invalidate
+        self.failargs = failargs
+        self.faillocs = faillocs
+        self.save_exc = save_exc
+        self.fcond=fcond
 
 class OpAssembler(object):
         
@@ -18,31 +29,17 @@
    
     emit_int_le = gen_emit_cmp_op(c.LE)   
 
-    def _guard_epilogue(self, op, failargs):
-        fail_descr = op.getdescr()
-        fail_index = self._get_identifier_from_descr(fail_descr)
-        fail_descr.index = fail_index
-        self.cpu.saved_descr[fail_index] = fail_descr
-        numops = self.mc.get_number_of_ops()
-        self.mc.beq(0)
-        reglist = []
-        for failarg in failargs:
-            if failarg is None:
-                reglist.append(None)
-            else:
-                reglist.append(failarg)
-        self.patch_list.append((numops, fail_index, op, reglist))
-
-    def _emit_guard(self, op, arglocs, save_exc=False,
+    def _emit_guard(self, op, arglocs, fcond, save_exc=False,
             is_guard_not_invalidated=False):
         descr = op.getdescr()
         assert isinstance(descr, AbstractFailDescr)
-        pos = self.get_relative_pos()
-        self.mc.b(0)   # has to be patched later on
+        pos = self.mc.currpos()
+        self.mc.nop()     # has to be patched later on
         self.pending_guards.append(GuardToken(descr,
                                    failargs=op.getfailargs(),
                                    faillocs=arglocs,
                                    offset=pos,
+                                   fcond=fcond,
                                    is_invalidate=is_guard_not_invalidated,
                                    save_exc=save_exc))
 
@@ -50,7 +47,9 @@
         l0 = arglocs[0]
         failargs = arglocs[1:]
         self.mc.cmpi(l0.value, 0)
-        self._guard_epilogue(op, failargs)
+        self._emit_guard(op, failargs, c.opposites[c.EQ])
+        #                        #      ^^^^ If this condition is met,
+        #                        #           then the guard fails.
 
     def emit_finish(self, op, arglocs, regalloc):
         descr = op.getdescr()
@@ -60,7 +59,7 @@
         for index, arg in enumerate(arglocs):
             addr = self.fail_boxes_int.get_addr_for_num(index)
             self.store_reg(arg, addr)
-        self.load_imm(r.RES, identifier) # set return value
+        self.load_imm(r.RES.value, identifier) # set return value
         self.branch_abs(self.exit_code_adr)
 
     def emit_jump(self, op, arglocs, regalloc):
@@ -76,6 +75,6 @@
         self.mc.ori(0, 0, 0)
 
     def branch_abs(self, address):
-        self.load_imm(r.r0, address)
+        self.load_imm(r.r0.value, address)
         self.mc.mtctr(0)
         self.mc.bctr()

From noreply at buildbot.pypy.org  Mon Oct 10 18:52:42 2011
From: noreply at buildbot.pypy.org (hager)
Date: Mon, 10 Oct 2011 18:52:42 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: Added some comparison
	conditions.
Message-ID: <20111010165242.3D02B82112@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r47925:6bcddb937964
Date: 2011-10-10 18:51 +0200
http://bitbucket.org/pypy/pypy/changeset/6bcddb937964/

Log:	Added some comparison conditions.

diff --git a/pypy/jit/backend/ppc/ppcgen/condition.py b/pypy/jit/backend/ppc/ppcgen/condition.py
--- a/pypy/jit/backend/ppc/ppcgen/condition.py
+++ b/pypy/jit/backend/ppc/ppcgen/condition.py
@@ -1,1 +1,8 @@
 LE = 0
+NE = 4
+GT = 2
+LT = 3
+EQ = 12
+GE = 33
+
+opposites = {LE: GT, NE: EQ, LT: GE, GE: LT, EQ: NE, GT: LE}

From noreply at buildbot.pypy.org  Mon Oct 10 19:19:41 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 10 Oct 2011 19:19:41 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Support for weakrefs.
Message-ID: <20111010171941.2C73782112@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r47926:1ff0ad13030d
Date: 2011-10-10 19:19 +0200
http://bitbucket.org/pypy/pypy/changeset/1ff0ad13030d/

Log:	Support for weakrefs.

diff --git a/pypy/rpython/memory/gc/concurrentms.py b/pypy/rpython/memory/gc/concurrentms.py
--- a/pypy/rpython/memory/gc/concurrentms.py
+++ b/pypy/rpython/memory/gc/concurrentms.py
@@ -199,7 +199,7 @@
     def malloc_fixedsize_clear(self, typeid, size,
                                needs_finalizer=False, contains_weakptr=False):
         assert not needs_finalizer  # XXX
-        assert not contains_weakptr # XXX
+        # contains_weakptr: detected during collection
         size_gc_header = self.gcheaderbuilder.size_gc_header
         totalsize = size_gc_header + size
         rawtotalsize = raw_malloc_usage(totalsize)
@@ -733,11 +733,14 @@
         self.gray_objects.append(root.address[0])
 
     def collector_sweep(self):
-        self._collect_sweep_large_objects()
         n = 1
         while n < self.pagelists_length:
             self._collect_sweep_pages(n)
             n += 1
+        # do this *after* the other one, so that objects are not free'd
+        # before we get a chance to inspect if they contain objects that
+        # are still alive (needed for weakrefs)
+        self._collect_sweep_large_objects()
 
     def _collect_sweep_large_objects(self):
         block = self.collect_pages[0]
@@ -774,7 +777,8 @@
         object_size = n << WORD_POWER_2
         linked_list = self.NULL
         first_loc_in_linked_list = self.NULL
-        nonmarked = self.other_mark(self.current_mark)
+        marked = self.current_mark
+        nonmarked = self.other_mark(marked)
         while page != self.NULL:
             i = self.page_size - object_size
             limit = raw_malloc_usage(self.HDRSIZE)
@@ -783,8 +787,9 @@
             while i >= limit:
                 adr = pageadr + i
                 hdr = llmemory.cast_adr_to_ptr(adr, self.HDRPTR)
+                mark = hdr.tid & 0xFF
                 #
-                if (hdr.tid & 0xFF) == nonmarked:
+                if mark == nonmarked:
                     # the location contains really an object (and is not just
                     # part of a linked list of free locations), and moreover
                     # the object is still not marked.  Free it by inserting
@@ -805,6 +810,26 @@
                     llarena.arena_reset(adr + size_of_int,
                                         object_size - size_of_int, 2)
                 #
+                elif mark == marked:
+                    # the location contains really an object, which is marked.
+                    # check the typeid to see if it's a weakref.  XXX could
+                    # be faster
+                    tid = hdr.tid
+                    type_id = llop.extract_high_ushort(llgroup.HALFWORD, tid)
+                    wroffset = self.weakpointer_offset(type_id)
+                    if wroffset >= 0:
+                        size_gc_header = self.gcheaderbuilder.size_gc_header
+                        obj = adr + size_gc_header
+                        pointing_to = (obj + wroffset).address[0]
+                        if pointing_to != llmemory.NULL:
+                            pt_adr = pointing_to - size_gc_header
+                            pt_hdr = llmemory.cast_adr_to_ptr(pt_adr,
+                                                              self.HDRPTR)
+                            if (pt_hdr.tid & 0xFF) == nonmarked:
+                                # this weakref points to an object that is still
+                                # not marked, so clear it
+                                (obj + wroffset).address[0] = llmemory.NULL
+                #
                 i -= object_size
             #
             page = self.cast_int_to_hdrptr(page.tid)
diff --git a/pypy/rpython/memory/test/test_gc.py b/pypy/rpython/memory/test/test_gc.py
--- a/pypy/rpython/memory/test/test_gc.py
+++ b/pypy/rpython/memory/test/test_gc.py
@@ -931,20 +931,3 @@
 class TestMostlyConcurrentMarkSweepGC(GCTest):
     from pypy.rpython.memory.gc.concurrentms \
             import MostlyConcurrentMarkSweepGC as GCClass
-
-    def test_finalizer_order(self):
-        py.test.skip("in-progress")
-    def test_from_objwithfinalizer_to_youngobj(self):
-        py.test.skip("in-progress")
-    def test_finalizer(self):
-        py.test.skip("in-progress")
-    def test_finalizer_calls_malloc(self):
-        py.test.skip("in-progress")
-    def test_finalizer_calls_collect(self):
-        py.test.skip("in-progress")
-    def test_finalizer_resurrects(self):
-        py.test.skip("in-progress")
-    def test_weakref(self):
-        py.test.skip("in-progress")
-    def test_weakref_to_object_with_finalizer(self):
-        py.test.skip("in-progress")

From noreply at buildbot.pypy.org  Mon Oct 10 20:45:03 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 10 Oct 2011 20:45:03 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: (agaynor,
	fijal) tutorial proposal draft
Message-ID: <20111010184503.C32BD82112@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: extradoc
Changeset: r3920:a1d122f1dadf
Date: 2011-10-10 20:43 +0200
http://bitbucket.org/pypy/extradoc/changeset/a1d122f1dadf/

Log:	(agaynor, fijal) tutorial proposal draft

diff --git a/talk/pycon2012/tutorial.rst b/talk/pycon2012/tutorial.rst
new file mode 100644
--- /dev/null
+++ b/talk/pycon2012/tutorial.rst
@@ -0,0 +1,51 @@
+How to get the most out of your PyPy
+====================================
+
+Description:
+
+For many applications PyPy can provide performance benefits right out
+of the box. However, little details can push your application to
+perform much better.  In this tutorial we'll give you insights on how
+to push pypy to it's limites. We'll focus on understanding the
+performance characteristics of PyPy, and learning the analysis tools
+in order to maximize your applications performance. For people new to
+PyPy we'll also briefly cover making sure your application runs on
+PyPy.
+
+Abstract:
+
+We aim to teach people how to use performance tools available for PyPy
+as well as to understand PyPy's performance characteristics. We'll explain
+how different parts of PyPy interact (JIT, the garbage collector, the virtual
+machine runtime) and how to measure which part is eating your time. We'll give
+you a tour with jitviewer which is a crucial tool for understanding how your
+Python got compiled to assembler and whether it's performing well. We also plan
+to show potential pitfalls and usage patterns in the Python language that perform
+better or worse in the case of PyPy.
+
+This tutorial is intended for people familiar with Python who have performance
+problems, no previous experience with PyPy is needed. We ask people to come
+with their own problems and we'll provide some example ones. Attendees should
+have the latest version of PyPy preinstalled on their laptops.
+
+Audience level:
+
+Intermediate, people with Python experience but no prior PyPy experiance.
+No knowledge of profiling tools is necessary. It is also not needed to know
+assembler :)
+
+Additional notes (this goes to reviewers only):
+
+Our intended format is going to be a brief presentation on the architecture of
+PyPy, as well as an overview of the tools. From there we'd like to spend the
+remainder of the session reviewing practical problems, and showing the steps
+to understanding real applications' performance. We'll bring our own example
+problems, but we're also going to collect examples from students in advance.
+
+Both presenters have spoken at many previous PyCons, as well as other
+conferences, they're also both very active PyPy core developers.
+
+We don't think tutorial assistants would be necessary.
+
+Outline: ...
+

From noreply at buildbot.pypy.org  Mon Oct 10 21:04:58 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 10 Oct 2011 21:04:58 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: (agaynor, fijal,
	arigo lurking) talk proposal
Message-ID: <20111010190458.6CE6282112@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: extradoc
Changeset: r3921:fcf000be931f
Date: 2011-10-10 21:03 +0200
http://bitbucket.org/pypy/extradoc/changeset/fcf000be931f/

Log:	(agaynor, fijal, arigo lurking) talk proposal

diff --git a/talk/pycon2012/talk.rst b/talk/pycon2012/talk.rst
new file mode 100644
--- /dev/null
+++ b/talk/pycon2012/talk.rst
@@ -0,0 +1,23 @@
+Why PyPy by example
+===================
+
+Description:
+
+One of the goals of PyPy is to make existing Python code faster, however an
+even broader goal was to make it possible to write things in Python that
+previous would needed to be written in C or other low-level language.
+This talk will show examples of this, and describe how they represent
+the tremendous progress PyPy has made, and what it means for people looking
+to use PyPy.
+
+Abstract:
+
+In this talk we'll talk about PyPy's present status and for what kinds
+of applications it might be useful. We'll also show examples of things
+that are possible with PyPy that were impossible to do with Python before,
+like real time video processing done in pure python. Our objective with
+each of the examples will be to highlight the type of work that is sped up
+and why this represents both a boon for existing Python programmers, as well
+as an opportunity for Python to expand to new audiences.
+
+We'll also explain a bit PyPy's current status, its goals and near future.

From noreply at buildbot.pypy.org  Mon Oct 10 21:06:45 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 10 Oct 2011 21:06:45 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: (fijal, agaynor)  minor adjustments
Message-ID: <20111010190645.EF60782112@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: extradoc
Changeset: r3922:f0322a09c5a5
Date: 2011-10-10 21:05 +0200
http://bitbucket.org/pypy/extradoc/changeset/f0322a09c5a5/

Log:	(fijal, agaynor) minor adjustments

diff --git a/talk/pycon2012/talk.rst b/talk/pycon2012/talk.rst
--- a/talk/pycon2012/talk.rst
+++ b/talk/pycon2012/talk.rst
@@ -20,4 +20,5 @@
 and why this represents both a boon for existing Python programmers, as well
 as an opportunity for Python to expand to new audiences.
 
-We'll also explain a bit PyPy's current status, its goals and near future.
+We'll also explain a bit PyPy's accomplishements in the passing year that
+make all of this possible, current status, its goals and the near future.

From noreply at buildbot.pypy.org  Tue Oct 11 11:07:57 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 11 Oct 2011 11:07:57 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: fix comments.
Message-ID: <20111011090757.0F37E82112@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r47927:8b92fa0fa84b
Date: 2011-10-10 19:32 +0200
http://bitbucket.org/pypy/pypy/changeset/8b92fa0fa84b/

Log:	fix comments.

diff --git a/pypy/rpython/memory/gc/concurrentms.py b/pypy/rpython/memory/gc/concurrentms.py
--- a/pypy/rpython/memory/gc/concurrentms.py
+++ b/pypy/rpython/memory/gc/concurrentms.py
@@ -737,9 +737,9 @@
         while n < self.pagelists_length:
             self._collect_sweep_pages(n)
             n += 1
-        # do this *after* the other one, so that objects are not free'd
-        # before we get a chance to inspect if they contain objects that
-        # are still alive (needed for weakrefs)
+        # do this *after* the other one, so that the blocks are not free'd
+        # before we get a chance to inspect them, to see if they contain
+        # an object that is still alive (needed for weakrefs)
         self._collect_sweep_large_objects()
 
     def _collect_sweep_large_objects(self):
@@ -826,8 +826,8 @@
                             pt_hdr = llmemory.cast_adr_to_ptr(pt_adr,
                                                               self.HDRPTR)
                             if (pt_hdr.tid & 0xFF) == nonmarked:
-                                # this weakref points to an object that is still
-                                # not marked, so clear it
+                                # this weakref points to an object that is
+                                # still not marked, so clear it
                                 (obj + wroffset).address[0] = llmemory.NULL
                 #
                 i -= object_size

From noreply at buildbot.pypy.org  Tue Oct 11 11:07:58 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 11 Oct 2011 11:07:58 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Finalizers, first version.
Message-ID: <20111011090758.44F1382A88@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r47928:2d3fb03f4ad4
Date: 2011-10-11 11:07 +0200
http://bitbucket.org/pypy/pypy/changeset/2d3fb03f4ad4/

Log:	Finalizers, first version.

diff --git a/pypy/doc/discussion/finalizer-order.rst b/pypy/doc/discussion/finalizer-order.rst
--- a/pypy/doc/discussion/finalizer-order.rst
+++ b/pypy/doc/discussion/finalizer-order.rst
@@ -96,7 +96,7 @@
   we just skip it ("continue" line) and so it doesn't get marked.
 
 * if x2 is enumerated before x1, then when we process x2 we mark it and
-  set its state to >= 2 (before x2 is in closure(x2)), and then when we
+  set its state to >= 2 (because x2 is in closure(x2)), and then when we
   process x1 we set state[x2] == 3.  So in the final loop x2 gets
   removed from the "marked" list.
 
diff --git a/pypy/rpython/memory/gc/base.py b/pypy/rpython/memory/gc/base.py
--- a/pypy/rpython/memory/gc/base.py
+++ b/pypy/rpython/memory/gc/base.py
@@ -31,6 +31,7 @@
         self.config = config
         assert isinstance(translated_to_c, bool)
         self.translated_to_c = translated_to_c
+        self.run_finalizers = None
 
     def setup(self):
         # all runtime mutable values' setup should happen here
@@ -297,7 +298,8 @@
         callback2, attrname = _convert_callback_formats(callback)    # :-/
         setattr(self, attrname, arg)
         self.root_walker.walk_roots(callback2, callback2, callback2)
-        self.run_finalizers.foreach(callback, arg)
+        if self.run_finalizers is not None:
+            self.run_finalizers.foreach(callback, arg)
     enumerate_all_roots._annspecialcase_ = 'specialize:arg(1)'
 
     def debug_check_consistency(self):
diff --git a/pypy/rpython/memory/gc/concurrentms.py b/pypy/rpython/memory/gc/concurrentms.py
--- a/pypy/rpython/memory/gc/concurrentms.py
+++ b/pypy/rpython/memory/gc/concurrentms.py
@@ -37,8 +37,8 @@
 MARK_VALUE_2      = 0x6B    # 'k', 107
 MARK_VALUE_STATIC = 0x53    # 'S', 83
 # Next lower byte: a combination of flags.
-FL_WITHHASH              = 0x0100
-FL_EXTRA                 = 0x0200
+FL_WITHHASH       = 0x0100
+FL_EXTRA          = 0x0200
 # And the high half of the word contains the numeric typeid.
 
 
@@ -117,11 +117,11 @@
     def _clear_list(self, array):
         i = 0
         while i < self.pagelists_length:
-            array[i] = lltype.nullptr(self.HDR)
+            array[i] = self.NULL
             i += 1
 
     def _initialize(self):
-        self.free_pages = lltype.nullptr(self.HDR)
+        self.free_pages = self.NULL
         #
         # Clear the lists
         self._clear_list(self.nonfree_pages)
@@ -130,6 +130,13 @@
         self._clear_list(self.collect_heads)
         self._clear_list(self.collect_tails)
         #
+        self.finalizer_pages = self.NULL
+        self.collect_finalizer_pages = self.NULL
+        self.collect_finalizer_tails = self.NULL
+        self.collect_run_finalizers_head = self.NULL
+        self.collect_run_finalizers_tail = self.NULL
+        self.run_finalizers = self.NULL
+        #
         # The following character is either MARK_VALUE_1 or MARK_VALUE_2,
         # and represents the character that must be in the 'mark' field
         # of an object header in order for the object to be considered as
@@ -158,7 +165,9 @@
 
     def setup(self):
         "Start the concurrent collector thread."
-        GCBase.setup(self)
+        # don't call GCBase.setup(self), because we don't need
+        # 'run_finalizers' as a deque
+        self.finalizer_lock_count = 0
         #
         self.main_thread_ident = ll_thread.get_ident()
         self.ready_to_start_lock = ll_thread.allocate_ll_lock()
@@ -198,8 +207,15 @@
 
     def malloc_fixedsize_clear(self, typeid, size,
                                needs_finalizer=False, contains_weakptr=False):
-        assert not needs_finalizer  # XXX
         # contains_weakptr: detected during collection
+        #
+        # Case of finalizers (test constant-folded)
+        if needs_finalizer:
+            ll_assert(not contains_weakptr,
+                     "'needs_finalizer' and 'contains_weakptr' both specified")
+            return self._malloc_with_finalizer(typeid, size)
+        #
+        # Regular case
         size_gc_header = self.gcheaderbuilder.size_gc_header
         totalsize = size_gc_header + size
         rawtotalsize = raw_malloc_usage(totalsize)
@@ -377,6 +393,32 @@
         return result
     _malloc_varsize_slowpath._dont_inline_ = True
 
+    def _malloc_with_finalizer(self, typeid, size):
+        # XXX a lot of copy and paste from _malloc_slowpath()
+        size_gc_header = self.gcheaderbuilder.size_gc_header
+        totalsize = size_gc_header + size
+        rawtotalsize = raw_malloc_usage(totalsize)
+        ll_assert(rawtotalsize & (WORD - 1) == 0,
+                  "malloc_with_finalizer: non-rounded size")
+        rawtotalsize += 8
+        block = llarena.arena_malloc(rawtotalsize, 2)
+        if not block:
+            raise MemoryError
+        llarena.arena_reserve(block, self.HDRSIZE)
+        blockhdr = llmemory.cast_adr_to_ptr(block, self.HDRPTR)
+        # the changes are only in the following two lines: we add the block
+        # to a different linked list
+        blockhdr.tid = self.cast_hdrptr_to_int(self.finalizer_pages)
+        self.finalizer_pages = blockhdr
+        result = block + 8
+        #
+        llarena.arena_reserve(result, totalsize)
+        hdr = llmemory.cast_adr_to_ptr(result, self.HDRPTR)
+        hdr.tid = self.combine(typeid, self.current_mark, 0)
+        #
+        obj = result + size_gc_header
+        return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
+
     # ----------
     # Other functions in the GC API
 
@@ -509,10 +551,50 @@
                     self.nonfree_pages[0])
                 self.nonfree_pages[0] = self.collect_heads[0]
             #
+            # Do the same with 'collect_finalizer_pages/tails'
+            if self.collect_finalizer_tails != self.NULL:
+                self.collect_finalizer_tails.tid = self.cast_hdrptr_to_int(
+                    self.finalizer_pages)
+                self.finalizer_pages = self.collect_finalizer_pages
+            #
+            # Do the same with 'collect_run_finalizers_head/tail'
+            if self.collect_run_finalizers_tail != self.NULL:
+                self.collect_run_finalizers_tail.tid = self.cast_hdrptr_to_int(
+                    self.run_finalizers)
+                self.run_finalizers = self.collect_run_finalizers_head
+            #
             if self.DEBUG:
                 self.debug_check_lists()
             #
             debug_stop("gc-stop")
+            #
+            self.execute_finalizers_ll()
+
+
+    def execute_finalizers_ll(self):
+        self.finalizer_lock_count += 1
+        try:
+##            print 'execute_finalizers_ll: %d' % self.finalizer_lock_count
+##            p = self.run_finalizers
+##            while p != self.NULL:
+##                print p
+##                p = self.cast_int_to_hdrptr(p.tid)
+            
+            while self.run_finalizers != self.NULL:
+                if self.finalizer_lock_count > 1:
+                    # the outer invocation of execute_finalizers() will do it
+                    break
+                #
+                x = llmemory.cast_ptr_to_adr(self.run_finalizers)
+                x = llarena.getfakearenaaddress(x) + 8
+                obj = x + self.gcheaderbuilder.size_gc_header
+                self.run_finalizers = self.cast_int_to_hdrptr(
+                    self.run_finalizers.tid)
+                #
+                finalizer = self.getfinalizer(self.get_type_id(obj))
+                finalizer(obj, llmemory.NULL)
+        finally:
+            self.finalizer_lock_count -= 1
 
 
     def collect(self, gen=2):
@@ -550,6 +632,14 @@
         # Add the prebuilt root objects that have been written to
         self.prebuilt_root_objects.foreach(self._add_prebuilt_root, None)
         #
+        # Add the objects still waiting in 'run_finalizers'
+        p = self.run_finalizers
+        while p != self.NULL:
+            x = llmemory.cast_ptr_to_adr(p)
+            x = llarena.getfakearenaaddress(x) + 8
+            self.gray_objects.append(x + self.gcheaderbuilder.size_gc_header)
+            p = self.cast_int_to_hdrptr(p.tid)
+        #
         # Invert this global variable, which has the effect that on all
         # objects' state go instantly from "marked" to "non marked"
         self.current_mark = self.other_mark(self.current_mark)
@@ -562,7 +652,13 @@
         while n < self.pagelists_length:
             self.collect_pages[n] = self.nonfree_pages[n]
             n += 1
+        self.collect_finalizer_pages = self.finalizer_pages
+        #
+        # Clear the following lists.  When the collector thread finishes,
+        # it will give back (in collect_{pages,tails}[0] and
+        # collect_finalizer_{pages,tails}) all the original items that survive.
         self.nonfree_pages[0] = self.NULL
+        self.finalizer_pages = self.NULL
         #
         # Start the collector thread
         self.collection_running = 1
@@ -584,15 +680,20 @@
         while n < self.pagelists_length:
             self.debug_check_list(self.free_lists[n])
             n += 1
+        self.debug_check_list(self.finalizer_pages)
+        self.debug_check_list(self.run_finalizers)
 
     def debug_check_list(self, page):
         try:
             previous_page = self.NULL
+            count = 0
             while page != self.NULL:
                 # prevent constant-folding, and detects loops of length 1
                 ll_assert(page != previous_page, "loop!")
                 previous_page = page
                 page = self.cast_int_to_hdrptr(page.tid)
+                count += 1
+            return count
         except KeyboardInterrupt:
             ll_assert(False, "interrupted")
             raise
@@ -663,6 +764,8 @@
             self.collector_mark()
             self.collection_running = 2
             #
+            self.deal_with_objects_with_finalizers()
+            #
             # Sweep
             self.collector_sweep()
             #
@@ -837,6 +940,51 @@
         self.collect_heads[n] = linked_list
         self.collect_tails[n] = first_loc_in_linked_list
 
+    # ----------
+    # Finalizers
+
+    def deal_with_objects_with_finalizers(self):
+        # XXX needs to be done correctly; for now we'll call finalizers
+        # in random order
+        size_gc_header = self.gcheaderbuilder.size_gc_header
+        marked = self.current_mark
+        finalizer_page = self.collect_finalizer_pages
+        self.collect_run_finalizers_head = self.NULL
+        self.collect_run_finalizers_tail = self.NULL
+        self.collect_finalizer_pages = self.NULL
+        self.collect_finalizer_tails = self.NULL
+        while finalizer_page != self.NULL:
+            next_page = self.cast_int_to_hdrptr(finalizer_page.tid)
+            #
+            x = llmemory.cast_ptr_to_adr(finalizer_page)
+            x = llarena.getfakearenaaddress(x) + 8
+            hdr = llmemory.cast_adr_to_ptr(x, self.HDRPTR)
+            if (hdr.tid & 0xFF) != marked:
+                # non-marked: add to collect_run_finalizers,
+                # and mark the object and its dependencies
+                finalizer_page.tid = self.cast_hdrptr_to_int(
+                    self.collect_run_finalizers_head)
+                self.collect_run_finalizers_head = finalizer_page
+                if self.collect_run_finalizers_tail == self.NULL:
+                    self.collect_run_finalizers_tail = finalizer_page
+                obj = x + size_gc_header
+                self.gray_objects.append(obj)
+            else:
+                # marked: relink into the collect_finalizer_pages list
+                finalizer_page.tid = self.cast_hdrptr_to_int(
+                    self.collect_finalizer_pages)
+                self.collect_finalizer_pages = finalizer_page
+                if self.collect_finalizer_tails != self.NULL:
+                    self.collect_finalizer_tails = finalizer_page
+            #
+            finalizer_page = next_page
+        #
+        self._collect_mark()
+        #
+        ll_assert(not self.extra_objects_to_mark.non_empty(),
+                  "should not see objects only reachable from finalizers "
+                  "before we run them")
+
 
 # ____________________________________________________________
 #
diff --git a/pypy/rpython/memory/test/test_gc.py b/pypy/rpython/memory/test/test_gc.py
--- a/pypy/rpython/memory/test/test_gc.py
+++ b/pypy/rpython/memory/test/test_gc.py
@@ -931,3 +931,6 @@
 class TestMostlyConcurrentMarkSweepGC(GCTest):
     from pypy.rpython.memory.gc.concurrentms \
             import MostlyConcurrentMarkSweepGC as GCClass
+
+    def test_finalizer_calls_malloc_during_minor_collect(self):
+        py.test.skip("check if this test is valid here")

From noreply at buildbot.pypy.org  Tue Oct 11 11:24:08 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 11 Oct 2011 11:24:08 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Simplify the linked list
	support.
Message-ID: <20111011092408.DAF0582112@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r47929:014eb2e11cf5
Date: 2011-10-11 11:19 +0200
http://bitbucket.org/pypy/pypy/changeset/014eb2e11cf5/

Log:	Simplify the linked list support.

diff --git a/pypy/rpython/memory/gc/concurrentms.py b/pypy/rpython/memory/gc/concurrentms.py
--- a/pypy/rpython/memory/gc/concurrentms.py
+++ b/pypy/rpython/memory/gc/concurrentms.py
@@ -135,7 +135,7 @@
         self.collect_finalizer_tails = self.NULL
         self.collect_run_finalizers_head = self.NULL
         self.collect_run_finalizers_tail = self.NULL
-        self.run_finalizers = self.NULL
+        self.objects_with_finalizers_to_run = self.NULL
         #
         # The following character is either MARK_VALUE_1 or MARK_VALUE_2,
         # and represents the character that must be in the 'mark' field
@@ -226,7 +226,7 @@
             n = rawtotalsize >> WORD_POWER_2
             result = self.free_lists[n]
             if result != self.NULL:
-                self.free_lists[n] = self.cast_int_to_hdrptr(result.tid)
+                self.free_lists[n] = list_next(result)
                 obj = self.grow_reservation(result, totalsize)
                 hdr = self.header(obj)
                 hdr.tid = self.combine(typeid, self.current_mark, 0)
@@ -264,7 +264,7 @@
             n = rawtotalsize >> WORD_POWER_2
             result = self.free_lists[n]
             if result != self.NULL:
-                self.free_lists[n] = self.cast_int_to_hdrptr(result.tid)
+                self.free_lists[n] = list_next(result)
                 obj = self.grow_reservation(result, totalsize)
                 hdr = self.header(obj)
                 hdr.tid = self.combine(typeid, self.current_mark, 0)
@@ -298,7 +298,7 @@
             n = rawtotalsize >> WORD_POWER_2
             head = self.free_lists[n]
             if head:
-                self.free_lists[n] = self.cast_int_to_hdrptr(head.tid)
+                self.free_lists[n] = list_next(head)
                 obj = self.grow_reservation(head, totalsize)
                 hdr = self.header(obj)
                 hdr.tid = self.combine(typeid, self.current_mark, 0)
@@ -310,11 +310,11 @@
             if newpage == self.NULL:
                 self.allocate_next_arena()
                 newpage = self.free_pages
-            self.free_pages = self.cast_int_to_hdrptr(newpage.tid)
+            self.free_pages = list_next(newpage)
             #
             # Put the free page in the list 'nonfree_pages[n]'.  This is
             # a linked list chained through the first word of each page.
-            newpage.tid = self.cast_hdrptr_to_int(self.nonfree_pages[n])
+            set_next(newpage, self.nonfree_pages[n])
             self.nonfree_pages[n] = newpage
             #
             # Initialize the free page to contain objects of the given
@@ -328,7 +328,7 @@
                 adr = newpageadr + i
                 llarena.arena_reserve(adr, self.HDRSIZE)
                 p = llmemory.cast_adr_to_ptr(adr, self.HDRPTR)
-                p.tid = self.cast_hdrptr_to_int(head)
+                set_next(p, head)
                 head = p
                 i -= rawtotalsize
             self.free_lists[n] = head
@@ -351,7 +351,7 @@
                 raise MemoryError
             llarena.arena_reserve(block, self.HDRSIZE)
             blockhdr = llmemory.cast_adr_to_ptr(block, self.HDRPTR)
-            blockhdr.tid = self.cast_hdrptr_to_int(self.nonfree_pages[0])
+            set_next(blockhdr, self.nonfree_pages[0])
             self.nonfree_pages[0] = blockhdr
             result = block + 8
         #
@@ -408,7 +408,7 @@
         blockhdr = llmemory.cast_adr_to_ptr(block, self.HDRPTR)
         # the changes are only in the following two lines: we add the block
         # to a different linked list
-        blockhdr.tid = self.cast_hdrptr_to_int(self.finalizer_pages)
+        set_next(blockhdr, self.finalizer_pages)
         self.finalizer_pages = blockhdr
         result = block + 8
         #
@@ -540,28 +540,26 @@
             n = 1
             while n < self.pagelists_length:
                 if self.collect_tails[n] != self.NULL:
-                    self.collect_tails[n].tid = self.cast_hdrptr_to_int(
-                        self.free_lists[n])
+                    set_next(self.collect_tails[n], self.free_lists[n])
                     self.free_lists[n] = self.collect_heads[n]
                 n += 1
             #
             # Do the same with 'collect_heads[0]/collect_tails[0]'.
             if self.collect_tails[0] != self.NULL:
-                self.collect_tails[0].tid = self.cast_hdrptr_to_int(
-                    self.nonfree_pages[0])
+                set_next(self.collect_tails[0], self.nonfree_pages[0])
                 self.nonfree_pages[0] = self.collect_heads[0]
             #
             # Do the same with 'collect_finalizer_pages/tails'
             if self.collect_finalizer_tails != self.NULL:
-                self.collect_finalizer_tails.tid = self.cast_hdrptr_to_int(
-                    self.finalizer_pages)
+                set_next(self.collect_finalizer_tails, self.finalizer_pages)
                 self.finalizer_pages = self.collect_finalizer_pages
             #
             # Do the same with 'collect_run_finalizers_head/tail'
             if self.collect_run_finalizers_tail != self.NULL:
-                self.collect_run_finalizers_tail.tid = self.cast_hdrptr_to_int(
-                    self.run_finalizers)
-                self.run_finalizers = self.collect_run_finalizers_head
+                set_next(self.collect_run_finalizers_tail,
+                         self.objects_with_finalizers_to_run)
+                self.objects_with_finalizers_to_run = (
+                    self.collect_run_finalizers_head)
             #
             if self.DEBUG:
                 self.debug_check_lists()
@@ -574,22 +572,17 @@
     def execute_finalizers_ll(self):
         self.finalizer_lock_count += 1
         try:
-##            print 'execute_finalizers_ll: %d' % self.finalizer_lock_count
-##            p = self.run_finalizers
-##            while p != self.NULL:
-##                print p
-##                p = self.cast_int_to_hdrptr(p.tid)
-            
-            while self.run_finalizers != self.NULL:
+            while self.objects_with_finalizers_to_run != self.NULL:
                 if self.finalizer_lock_count > 1:
                     # the outer invocation of execute_finalizers() will do it
                     break
                 #
-                x = llmemory.cast_ptr_to_adr(self.run_finalizers)
+                x = llmemory.cast_ptr_to_adr(
+                        self.objects_with_finalizers_to_run)
                 x = llarena.getfakearenaaddress(x) + 8
                 obj = x + self.gcheaderbuilder.size_gc_header
-                self.run_finalizers = self.cast_int_to_hdrptr(
-                    self.run_finalizers.tid)
+                self.objects_with_finalizers_to_run = list_next(
+                    self.objects_with_finalizers_to_run)
                 #
                 finalizer = self.getfinalizer(self.get_type_id(obj))
                 finalizer(obj, llmemory.NULL)
@@ -632,13 +625,13 @@
         # Add the prebuilt root objects that have been written to
         self.prebuilt_root_objects.foreach(self._add_prebuilt_root, None)
         #
-        # Add the objects still waiting in 'run_finalizers'
-        p = self.run_finalizers
+        # Add the objects still waiting in 'objects_with_finalizers_to_run'
+        p = self.objects_with_finalizers_to_run
         while p != self.NULL:
             x = llmemory.cast_ptr_to_adr(p)
             x = llarena.getfakearenaaddress(x) + 8
             self.gray_objects.append(x + self.gcheaderbuilder.size_gc_header)
-            p = self.cast_int_to_hdrptr(p.tid)
+            p = list_next(p)
         #
         # Invert this global variable, which has the effect that on all
         # objects' state go instantly from "marked" to "non marked"
@@ -681,7 +674,7 @@
             self.debug_check_list(self.free_lists[n])
             n += 1
         self.debug_check_list(self.finalizer_pages)
-        self.debug_check_list(self.run_finalizers)
+        self.debug_check_list(self.objects_with_finalizers_to_run)
 
     def debug_check_list(self, page):
         try:
@@ -691,7 +684,7 @@
                 # prevent constant-folding, and detects loops of length 1
                 ll_assert(page != previous_page, "loop!")
                 previous_page = page
-                page = self.cast_int_to_hdrptr(page.tid)
+                page = list_next(page)
                 count += 1
             return count
         except KeyboardInterrupt:
@@ -717,14 +710,6 @@
         exc, val, tb = self._exc_info
         raise exc, val, tb
 
-    def cast_int_to_hdrptr(self, tid):
-        return llmemory.cast_adr_to_ptr(llmemory.cast_int_to_adr(tid),
-                                        self.HDRPTR)
-
-    def cast_hdrptr_to_int(self, hdr):
-        return llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(hdr),
-                                        "symbolic")
-
 
     def collector_run_nontranslated(self):
         try:
@@ -851,7 +836,7 @@
         linked_list = self.NULL
         first_block_in_linked_list = self.NULL
         while block != self.NULL:
-            nextblock = self.cast_int_to_hdrptr(block.tid)
+            nextblock = list_next(block)
             blockadr = llmemory.cast_ptr_to_adr(block)
             blockadr = llarena.getfakearenaaddress(blockadr)
             hdr = llmemory.cast_adr_to_ptr(blockadr + 8, self.HDRPTR)
@@ -864,7 +849,7 @@
                 # the object was marked: relink it
                 ll_assert(mark == self.current_mark,
                           "bad mark in large object")
-                block.tid = self.cast_hdrptr_to_int(linked_list)
+                set_next(block, linked_list)
                 linked_list = block
                 if first_block_in_linked_list == self.NULL:
                     first_block_in_linked_list = block
@@ -900,7 +885,7 @@
                     llarena.arena_reset(adr, object_size, 0)
                     llarena.arena_reserve(adr, self.HDRSIZE)
                     hdr = llmemory.cast_adr_to_ptr(adr, self.HDRPTR)
-                    hdr.tid = self.cast_hdrptr_to_int(linked_list)
+                    set_next(hdr, linked_list)
                     linked_list = hdr
                     if first_loc_in_linked_list == self.NULL:
                         first_loc_in_linked_list = hdr
@@ -935,7 +920,7 @@
                 #
                 i -= object_size
             #
-            page = self.cast_int_to_hdrptr(page.tid)
+            page = list_next(page)
         #
         self.collect_heads[n] = linked_list
         self.collect_tails[n] = first_loc_in_linked_list
@@ -954,7 +939,7 @@
         self.collect_finalizer_pages = self.NULL
         self.collect_finalizer_tails = self.NULL
         while finalizer_page != self.NULL:
-            next_page = self.cast_int_to_hdrptr(finalizer_page.tid)
+            next_page = list_next(finalizer_page)
             #
             x = llmemory.cast_ptr_to_adr(finalizer_page)
             x = llarena.getfakearenaaddress(x) + 8
@@ -962,8 +947,7 @@
             if (hdr.tid & 0xFF) != marked:
                 # non-marked: add to collect_run_finalizers,
                 # and mark the object and its dependencies
-                finalizer_page.tid = self.cast_hdrptr_to_int(
-                    self.collect_run_finalizers_head)
+                set_next(finalizer_page, self.collect_run_finalizers_head)
                 self.collect_run_finalizers_head = finalizer_page
                 if self.collect_run_finalizers_tail == self.NULL:
                     self.collect_run_finalizers_tail = finalizer_page
@@ -971,8 +955,7 @@
                 self.gray_objects.append(obj)
             else:
                 # marked: relink into the collect_finalizer_pages list
-                finalizer_page.tid = self.cast_hdrptr_to_int(
-                    self.collect_finalizer_pages)
+                set_next(finalizer_page, self.collect_finalizer_pages)
                 self.collect_finalizer_pages = finalizer_page
                 if self.collect_finalizer_tails != self.NULL:
                     self.collect_finalizer_tails = finalizer_page
@@ -988,6 +971,19 @@
 
 # ____________________________________________________________
 #
+# Support for linked lists (used here because AddressStack is not thread-safe)
+
+def list_next(hdr):
+    return llmemory.cast_adr_to_ptr(llmemory.cast_int_to_adr(hdr.tid),
+                                    MostlyConcurrentMarkSweepGC.HDRPTR)
+
+def set_next(hdr, nexthdr):
+    hdr.tid = llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(nexthdr),
+                                       "symbolic")
+
+
+# ____________________________________________________________
+#
 # Hack to write the 'mark' or the 'flags' bytes of an object header
 # without overwriting the whole word.  Essential in the rare case where
 # the other thread might be concurrently writing the other byte.

From noreply at buildbot.pypy.org  Tue Oct 11 11:49:15 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 11 Oct 2011 11:49:15 +0200 (CEST)
Subject: [pypy-commit] pypy string-promote-2: codewriter support for
	promoting strings
Message-ID: <20111011094915.57AB182112@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: string-promote-2
Changeset: r47930:b2a87ed6e142
Date: 2011-10-09 22:23 +0200
http://bitbucket.org/pypy/pypy/changeset/b2a87ed6e142/

Log:	codewriter support for promoting strings

diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -455,12 +455,20 @@
             # the special return value None forces op.result to be considered
             # equal to op.args[0]
             return [op0, op1, None]
-        if (hints.get('string_promote') and
+        if (hints.get('promote_string') and
             op.args[0].concretetype is not lltype.Void):
-            assert op.args[0].concretetype == lltype.Ptr(rstr.STR)
-            op0 = SpaceOperation('-live-', [], None)
-            op1 = SpaceOperation('str_guard_value', [op.args[0]], op.result)
-            return [op0, op1, None]
+            S = lltype.Ptr(rstr.STR)
+            assert op.args[0].concretetype == S
+            self._register_extra_helper(EffectInfo.OS_STREQ_NONNULL,
+                                        "str.eq_nonnull",
+                                        [S, S],
+                                        lltype.Signed,
+                                        EffectInfo.EF_ELIDABLE_CANNOT_RAISE)
+            descr = self.callcontrol.callinfo_for_oopspec(
+                EffectInfo.OS_STREQ_NONNULL)
+            op1 = SpaceOperation('str_guard_value', [op.args[0], descr],
+                                 op.result)
+            return op1
         else:
             log.WARNING('ignoring hint %r at %r' % (hints, self.graph))
 
@@ -1431,6 +1439,7 @@
             func = heaptracker.adr2int(
                 llmemory.cast_ptr_to_adr(c_func.value))
         self.callcontrol.callinfocollection.add(oopspecindex, calldescr, func)
+        return calldescr
 
     def _handle_stroruni_call(self, op, oopspec_name, args):
         SoU = args[0].concretetype     # Ptr(STR) or Ptr(UNICODE)
diff --git a/pypy/jit/codewriter/test/test_jtransform.py b/pypy/jit/codewriter/test/test_jtransform.py
--- a/pypy/jit/codewriter/test/test_jtransform.py
+++ b/pypy/jit/codewriter/test/test_jtransform.py
@@ -119,6 +119,7 @@
              EI.OS_STR2UNICODE:([PSTR], PUNICODE),
              EI.OS_STR_CONCAT: ([PSTR, PSTR], PSTR),
              EI.OS_STR_SLICE:  ([PSTR, INT, INT], PSTR),
+             EI.OS_STREQ_NONNULL:  ([PSTR, PSTR], INT),
              EI.OS_UNI_CONCAT: ([PUNICODE, PUNICODE], PUNICODE),
              EI.OS_UNI_SLICE:  ([PUNICODE, INT, INT], PUNICODE),
              EI.OS_UNI_EQUAL:  ([PUNICODE, PUNICODE], lltype.Bool),
@@ -140,6 +141,9 @@
         return 'calldescr-%d' % oopspecindex
     def calldescr_canraise(self, calldescr):
         return False
+    def callinfo_for_oopspec(self, oopspecindex):
+        assert oopspecindex == effectinfo.EffectInfo.OS_STREQ_NONNULL
+        return 'calldescr'
 
 
 def test_optimize_goto_if_not():
@@ -849,14 +853,13 @@
     v1 = varoftype(PSTR)
     v2 = varoftype(PSTR)
     op = SpaceOperation('hint',
-                        [v1, Constant({'string_promote': True}, lltype.Void)],
+                        [v1, Constant({'promote_string': True}, lltype.Void)],
                         v2)
     tr = Transformer(FakeCPU(), FakeBuiltinCallControl())
-    op0, op1, _ = tr.rewrite_operation(op)
-    assert op0.opname == '-live-'
-    assert op1.opname == 'str_guard_value'
-    assert op1.args == [v1]
-    assert op1.result == v2
+    op0 = tr.rewrite_operation(op)
+    assert op0.opname == 'str_guard_value'
+    assert op0.args == [v1, 'calldescr']
+    assert op0.result == v2
 
 def test_unicode_concat():
     # test that the oopspec is present and correctly transformed

From noreply at buildbot.pypy.org  Tue Oct 11 11:49:16 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 11 Oct 2011 11:49:16 +0200 (CEST)
Subject: [pypy-commit] pypy string-promote-2: adapt to interfaces
Message-ID: <20111011094916.8CBD682112@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: string-promote-2
Changeset: r47931:846724c06889
Date: 2011-10-09 22:37 +0200
http://bitbucket.org/pypy/pypy/changeset/846724c06889/

Log:	adapt to interfaces

diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -464,8 +464,8 @@
                                         [S, S],
                                         lltype.Signed,
                                         EffectInfo.EF_ELIDABLE_CANNOT_RAISE)
-            descr = self.callcontrol.callinfo_for_oopspec(
-                EffectInfo.OS_STREQ_NONNULL)
+            descr = self.callcontrol.callinfocollection.callinfo_for_oopspec(
+                EffectInfo.OS_STREQ_NONNULL)[0]
             op1 = SpaceOperation('str_guard_value', [op.args[0], descr],
                                  op.result)
             return op1
diff --git a/pypy/jit/codewriter/test/test_jtransform.py b/pypy/jit/codewriter/test/test_jtransform.py
--- a/pypy/jit/codewriter/test/test_jtransform.py
+++ b/pypy/jit/codewriter/test/test_jtransform.py
@@ -99,6 +99,9 @@
             if i == oopspecindex:
                 return True
         return False
+    def callinfo_for_oopspec(self, oopspecindex):
+        assert oopspecindex == effectinfo.EffectInfo.OS_STREQ_NONNULL
+        return ('calldescr', None)
 
 class FakeBuiltinCallControl:
     def __init__(self):
@@ -141,9 +144,6 @@
         return 'calldescr-%d' % oopspecindex
     def calldescr_canraise(self, calldescr):
         return False
-    def callinfo_for_oopspec(self, oopspecindex):
-        assert oopspecindex == effectinfo.EffectInfo.OS_STREQ_NONNULL
-        return 'calldescr'
 
 
 def test_optimize_goto_if_not():

From noreply at buildbot.pypy.org  Tue Oct 11 11:49:17 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 11 Oct 2011 11:49:17 +0200 (CEST)
Subject: [pypy-commit] pypy string-promote-2: pyjitpl support, unfinished
Message-ID: <20111011094917.D086982112@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: string-promote-2
Changeset: r47932:e7fbc95d83be
Date: 2011-10-11 11:48 +0200
http://bitbucket.org/pypy/pypy/changeset/e7fbc95d83be/

Log:	pyjitpl support, unfinished

diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -464,11 +464,13 @@
                                         [S, S],
                                         lltype.Signed,
                                         EffectInfo.EF_ELIDABLE_CANNOT_RAISE)
-            descr = self.callcontrol.callinfocollection.callinfo_for_oopspec(
-                EffectInfo.OS_STREQ_NONNULL)[0]
-            op1 = SpaceOperation('str_guard_value', [op.args[0], descr],
+            descr, p = self.callcontrol.callinfocollection.callinfo_for_oopspec(
+                EffectInfo.OS_STREQ_NONNULL)
+            # XXX
+            c = Constant(p.adr.ptr, lltype.typeOf(p.adr.ptr))
+            op1 = SpaceOperation('str_guard_value', [op.args[0], c, descr],
                                  op.result)
-            return op1
+            return [SpaceOperation('-live-', [], None), op1, None]
         else:
             log.WARNING('ignoring hint %r at %r' % (hints, self.graph))
 
@@ -1439,7 +1441,6 @@
             func = heaptracker.adr2int(
                 llmemory.cast_ptr_to_adr(c_func.value))
         self.callcontrol.callinfocollection.add(oopspecindex, calldescr, func)
-        return calldescr
 
     def _handle_stroruni_call(self, op, oopspec_name, args):
         SoU = args[0].concretetype     # Ptr(STR) or Ptr(UNICODE)
diff --git a/pypy/jit/codewriter/test/test_jtransform.py b/pypy/jit/codewriter/test/test_jtransform.py
--- a/pypy/jit/codewriter/test/test_jtransform.py
+++ b/pypy/jit/codewriter/test/test_jtransform.py
@@ -858,7 +858,8 @@
     tr = Transformer(FakeCPU(), FakeBuiltinCallControl())
     op0 = tr.rewrite_operation(op)
     assert op0.opname == 'str_guard_value'
-    assert op0.args == [v1, 'calldescr']
+    assert op0.args[0] == v1
+    assert op0.args[2] == 'calldescr'
     assert op0.result == v2
 
 def test_unicode_concat():
diff --git a/pypy/jit/metainterp/blackhole.py b/pypy/jit/metainterp/blackhole.py
--- a/pypy/jit/metainterp/blackhole.py
+++ b/pypy/jit/metainterp/blackhole.py
@@ -523,6 +523,9 @@
     @arguments("f")
     def bhimpl_float_guard_value(a):
         pass
+    @arguments("r")
+    def bhimpl_str_guard_value(a):
+        pass
 
     @arguments("self", "i")
     def bhimpl_int_push(self, a):
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -166,11 +166,12 @@
 
     def make_result_of_lastop(self, resultbox):
         got_type = resultbox.type
-        if not we_are_translated():
-            typeof = {'i': history.INT,
-                      'r': history.REF,
-                      'f': history.FLOAT}
-            assert typeof[self.jitcode._resulttypes[self.pc]] == got_type
+        # XXX disabled for now, conflicts with str_guard_value
+        #if not we_are_translated():
+        #    typeof = {'i': history.INT,
+        #              'r': history.REF,
+        #              'f': history.FLOAT}
+        #    assert typeof[self.jitcode._resulttypes[self.pc]] == got_type
         target_index = ord(self.bytecode[self.pc-1])
         if got_type == history.INT:
             self.registers_i[target_index] = resultbox
@@ -895,6 +896,17 @@
     def _opimpl_guard_value(self, orgpc, box):
         self.implement_guard_value(orgpc, box)
 
+    @arguments("orgpc", "box", "box", "descr")
+    def opimpl_str_guard_value(self, orgpc, box, funcbox, descr):
+        if isinstance(box, Const):
+            return box     # no promotion needed, already a Const
+        else:
+            constbox = box.constbox()
+            resbox = self.do_residual_call(funcbox, descr, [box, constbox])
+            self.generate_guard(rop.GUARD_TRUE, resbox, resumepc=orgpc)
+            self.metainterp.replace_box(box, constbox)
+            return constbox
+
     opimpl_int_guard_value = _opimpl_guard_value
     opimpl_ref_guard_value = _opimpl_guard_value
     opimpl_float_guard_value = _opimpl_guard_value
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -13,7 +13,7 @@
 from pypy.rlib.jit import (JitDriver, we_are_jitted, hint, dont_look_inside,
     loop_invariant, elidable, promote, jit_debug, assert_green,
     AssertGreenFailed, unroll_safe, current_trace_length, look_inside_iff,
-    isconstant, isvirtual)
+    isconstant, isvirtual, promote_string)
 from pypy.rlib.rarithmetic import ovfcheck
 from pypy.rpython.lltypesystem import lltype, llmemory, rffi
 from pypy.rpython.ootypesystem import ootype
diff --git a/pypy/jit/metainterp/test/test_string.py b/pypy/jit/metainterp/test/test_string.py
--- a/pypy/jit/metainterp/test/test_string.py
+++ b/pypy/jit/metainterp/test/test_string.py
@@ -3,7 +3,8 @@
 from pypy.jit.codewriter.policy import StopAtXPolicy
 from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin
 from pypy.rlib.debug import debug_print
-from pypy.rlib.jit import JitDriver, dont_look_inside, we_are_jitted
+from pypy.rlib.jit import JitDriver, dont_look_inside, we_are_jitted,\
+     promote_string
 from pypy.rlib.rstring import StringBuilder
 from pypy.rpython.ootypesystem import ootype
 
@@ -507,6 +508,17 @@
                           'jump': 1, 'int_is_true': 1,
                           'guard_not_invalidated': 1})
 
+    def test_promote_string(self):
+        driver = JitDriver(greens = [], reds = ['n'])
+        
+        def f(n):
+            while n < 12:
+                driver.jit_merge_point(n=n)
+                promote_string(str(n % 3))
+                n += 1
+            return 0
+
+        self.meta_interp(f, [0])
 
 #class TestOOtype(StringTests, OOJitMixin):
 #    CALL = "oosend"
diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -38,7 +38,7 @@
     possible arguments are:
 
     * promote - promote the argument from a variable into a constant
-    * string_promote - same, but promote string by *value*
+    * promote_string - same, but promote string by *value*
     * access_directly - directly access a virtualizable, as a structure
                         and don't treat it as a virtualizable
     * fresh_virtualizable - means that virtualizable was just allocated.

From noreply at buildbot.pypy.org  Tue Oct 11 12:12:16 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 11 Oct 2011 12:12:16 +0200 (CEST)
Subject: [pypy-commit] pypy string-promote-2: make the test check something
Message-ID: <20111011101216.2314F82A88@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: string-promote-2
Changeset: r47934:460767032152
Date: 2011-10-11 12:10 +0200
http://bitbucket.org/pypy/pypy/changeset/460767032152/

Log:	make the test check something

diff --git a/pypy/jit/metainterp/test/test_string.py b/pypy/jit/metainterp/test/test_string.py
--- a/pypy/jit/metainterp/test/test_string.py
+++ b/pypy/jit/metainterp/test/test_string.py
@@ -512,13 +512,14 @@
         driver = JitDriver(greens = [], reds = ['n'])
         
         def f(n):
-            while n < 12:
+            while n < 21:
                 driver.jit_merge_point(n=n)
                 promote_string(str(n % 3))
                 n += 1
             return 0
 
         self.meta_interp(f, [0])
+        self.check_loops(call=3 + 1) # one for int2str
 
 #class TestOOtype(StringTests, OOJitMixin):
 #    CALL = "oosend"

From noreply at buildbot.pypy.org  Tue Oct 11 12:12:14 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 11 Oct 2011 12:12:14 +0200 (CEST)
Subject: [pypy-commit] pypy string-promote-2: (fijal,
	arigo) make the test pass
Message-ID: <20111011101214.E13CA82112@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: string-promote-2
Changeset: r47933:71ea9a22ca51
Date: 2011-10-11 12:07 +0200
http://bitbucket.org/pypy/pypy/changeset/71ea9a22ca51/

Log:	(fijal, arigo) make the test pass

diff --git a/pypy/jit/metainterp/blackhole.py b/pypy/jit/metainterp/blackhole.py
--- a/pypy/jit/metainterp/blackhole.py
+++ b/pypy/jit/metainterp/blackhole.py
@@ -523,9 +523,9 @@
     @arguments("f")
     def bhimpl_float_guard_value(a):
         pass
-    @arguments("r")
-    def bhimpl_str_guard_value(a):
-        pass
+    @arguments("r", "i", "d", returns="r")
+    def bhimpl_str_guard_value(a, i, d):
+        return a
 
     @arguments("self", "i")
     def bhimpl_int_push(self, a):
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -903,7 +903,9 @@
         else:
             constbox = box.constbox()
             resbox = self.do_residual_call(funcbox, descr, [box, constbox])
-            self.generate_guard(rop.GUARD_TRUE, resbox, resumepc=orgpc)
+            promoted_box = resbox.constbox()
+            self.generate_guard(rop.GUARD_VALUE, resbox, [promoted_box],
+                                resumepc=orgpc)
             self.metainterp.replace_box(box, constbox)
             return constbox
 

From noreply at buildbot.pypy.org  Tue Oct 11 12:12:17 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 11 Oct 2011 12:12:17 +0200 (CEST)
Subject: [pypy-commit] pypy string-promote-2: try using string_promote for
	promoting lookups
Message-ID: <20111011101217.55BFA82112@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: string-promote-2
Changeset: r47935:1cd384c6c5e7
Date: 2011-10-11 12:11 +0200
http://bitbucket.org/pypy/pypy/changeset/1cd384c6c5e7/

Log:	try using string_promote for promoting lookups

diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -10,7 +10,8 @@
 from pypy.objspace.std import identitydict
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.rlib.objectmodel import current_object_addr_as_int, compute_hash
-from pypy.rlib.jit import promote, elidable_promote, we_are_jitted
+from pypy.rlib.jit import promote, elidable_promote, we_are_jitted,\
+     promote_string
 from pypy.rlib.jit import elidable, dont_look_inside, unroll_safe
 from pypy.rlib.rarithmetic import intmask, r_uint
 
@@ -399,6 +400,7 @@
         if version_tag is None:
             tup = w_self._lookup_where(name)
             return tup
+        name = promote_string(name)
         w_class, w_value = w_self._pure_lookup_where_with_method_cache(name, version_tag)
         return w_class, unwrap_cell(space, w_value)
 

From noreply at buildbot.pypy.org  Tue Oct 11 14:41:40 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 11 Oct 2011 14:41:40 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Rename the write_barrier
 in concurrentms to "deletion_barrier", 
Message-ID: <20111011124140.D06B682112@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r47936:cbf2c361a9bb
Date: 2011-10-11 14:41 +0200
http://bitbucket.org/pypy/pypy/changeset/cbf2c361a9bb/

Log:	Rename the write_barrier in concurrentms to "deletion_barrier", as
	it has subtly different semantics about when it is ok or not to kill
	a call to it.

diff --git a/pypy/rpython/memory/gc/base.py b/pypy/rpython/memory/gc/base.py
--- a/pypy/rpython/memory/gc/base.py
+++ b/pypy/rpython/memory/gc/base.py
@@ -16,6 +16,7 @@
     _alloc_flavor_ = "raw"
     moving_gc = False
     needs_write_barrier = False
+    needs_deletion_barrier = False
     malloc_zero_filled = False
     prebuilt_gc_objects_are_static_roots = True
     object_minimal_size = 0
diff --git a/pypy/rpython/memory/gc/concurrentms.py b/pypy/rpython/memory/gc/concurrentms.py
--- a/pypy/rpython/memory/gc/concurrentms.py
+++ b/pypy/rpython/memory/gc/concurrentms.py
@@ -46,7 +46,7 @@
     _alloc_flavor_ = "raw"
     inline_simple_malloc = True
     inline_simple_malloc_varsize = True
-    needs_write_barrier = True
+    needs_deletion_barrier = True
     prebuilt_gc_objects_are_static_roots = False
     malloc_zero_filled = True
     gcflag_extra = FL_EXTRA
@@ -468,18 +468,11 @@
         return adr + self.gcheaderbuilder.size_gc_header
     grow_reservation._always_inline_ = True
 
-    def write_barrier(self, newvalue, addr_struct):
+    def deletion_barrier(self, addr_struct):
         mark = self.header(addr_struct).tid & 0xFF
         if mark != self.current_mark:
             self.force_scan(addr_struct)
 
-    def writebarrier_before_copy(self, source_addr, dest_addr,
-                                 source_start, dest_start, length):
-        mark = self.header(dest_addr).tid & 0xFF
-        if mark != self.current_mark:
-            self.force_scan(dest_addr)
-        return True
-
     def assume_young_pointers(self, addr_struct):
         pass # XXX
 
@@ -527,6 +520,7 @@
             #
             self.acquire(self.finished_lock)
             self.collection_running = 0
+            debug_print("collection_running = 0")
             #
             # Check invariants
             ll_assert(not self.extra_objects_to_mark.non_empty(),
@@ -655,6 +649,7 @@
         #
         # Start the collector thread
         self.collection_running = 1
+        debug_print("collection_running = 1")
         self.release(self.ready_to_start_lock)
         #
         debug_stop("gc-start")
@@ -748,6 +743,7 @@
             # Mark
             self.collector_mark()
             self.collection_running = 2
+            debug_print("collection_running = 2")
             #
             self.deal_with_objects_with_finalizers()
             #
@@ -756,6 +752,7 @@
             #
             # Done!
             self.collection_running = -1
+            debug_print("collection_running = -1")
             self.release(self.finished_lock)
 
 
diff --git a/pypy/rpython/memory/gc/test/test_direct.py b/pypy/rpython/memory/gc/test/test_direct.py
--- a/pypy/rpython/memory/gc/test/test_direct.py
+++ b/pypy/rpython/memory/gc/test/test_direct.py
@@ -90,6 +90,9 @@
             newaddr = llmemory.cast_ptr_to_adr(newvalue)
             addr_struct = llmemory.cast_ptr_to_adr(p)
             self.gc.write_barrier(newaddr, addr_struct)
+        elif self.gc.needs_deletion_barrier:
+            addr_struct = llmemory.cast_ptr_to_adr(p)
+            self.gc.deletion_barrier(addr_struct)
         setattr(p, fieldname, newvalue)
 
     def writearray(self, p, index, newvalue):
@@ -100,6 +103,9 @@
                 self.gc.write_barrier_from_array(newaddr, addr_struct, index)
             else:
                 self.gc.write_barrier(newaddr, addr_struct)
+        elif self.gc.needs_deletion_barrier:
+            addr_struct = llmemory.cast_ptr_to_adr(p)
+            self.gc.deletion_barrier(addr_struct)
         p[index] = newvalue
 
     def malloc(self, TYPE, n=None):
diff --git a/pypy/rpython/memory/gctransform/framework.py b/pypy/rpython/memory/gctransform/framework.py
--- a/pypy/rpython/memory/gctransform/framework.py
+++ b/pypy/rpython/memory/gctransform/framework.py
@@ -440,6 +440,7 @@
 
         self.write_barrier_ptr = None
         self.write_barrier_from_array_ptr = None
+        self.deletion_barrier_ptr = None
         if GCClass.needs_write_barrier:
             self.write_barrier_ptr = getfn(GCClass.write_barrier.im_func,
                                            [s_gc,
@@ -475,6 +476,12 @@
                                                     annmodel.SomeInteger(),
                                                     annmodel.SomeAddress()],
                                                    annmodel.s_None)
+        elif GCClass.needs_deletion_barrier:
+            self.deletion_barrier_ptr = getfn(GCClass.deletion_barrier.im_func,
+                                              [s_gc, annmodel.SomeAddress()],
+                                              annmodel.s_None,
+                                              inline=True)
+
         self.statistics_ptr = getfn(GCClass.statistics.im_func,
                                     [s_gc, annmodel.SomeInteger()],
                                     annmodel.SomeInteger())
@@ -551,7 +558,7 @@
         log.info("assigned %s typeids" % (len(group.members), ))
         log.info("added %s push/pop stack root instructions" % (
                      self.num_pushs, ))
-        if self.write_barrier_ptr:
+        if self.write_barrier_ptr or self.deletion_barrier_ptr:
             log.info("inserted %s write barrier calls" % (
                          self.write_barrier_calls, ))
         if self.write_barrier_from_array_ptr:
@@ -1136,6 +1143,18 @@
                                           self.c_const_gc,
                                           v_newvalue,
                                           v_structaddr])
+        elif (self.deletion_barrier_ptr
+              and v_struct.concretetype.TO._gckind == "gc"):
+            # The MostlyConcurrentMarkSweepGC is a case where the
+            # write barrier is best called a "deletion barrier" instead.
+            # It needs different logic here (and at a few other places).
+            # XXX try harder to omit some calls!
+            v_structaddr = hop.genop("cast_ptr_to_adr", [v_struct],
+                                     resulttype = llmemory.Address)
+            self.write_barrier_calls += 1
+            hop.genop("direct_call", [self.deletion_barrier_ptr,
+                                      self.c_const_gc,
+                                      v_structaddr])
         hop.rename('bare_' + opname)
 
     def transform_getfield_typeptr(self, hop):
diff --git a/pypy/rpython/memory/gcwrapper.py b/pypy/rpython/memory/gcwrapper.py
--- a/pypy/rpython/memory/gcwrapper.py
+++ b/pypy/rpython/memory/gcwrapper.py
@@ -144,6 +144,10 @@
             return self.gc.writebarrier_before_copy(source_addr, dest_addr,
                                                     source_start, dest_start,
                                                     length)
+        elif self.gc.needs_deletion_barrier:
+            dest_addr = llmemory.cast_ptr_to_adr(dest)
+            self.gc.deletion_barrier(dest_addr)
+            return True
         else:
             return True
 

From noreply at buildbot.pypy.org  Tue Oct 11 15:40:42 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 11 Oct 2011 15:40:42 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Fix.
Message-ID: <20111011134042.56CAF82112@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r47937:788fb4094a80
Date: 2011-10-11 15:40 +0200
http://bitbucket.org/pypy/pypy/changeset/788fb4094a80/

Log:	Fix.

diff --git a/pypy/translator/c/genc.py b/pypy/translator/c/genc.py
--- a/pypy/translator/c/genc.py
+++ b/pypy/translator/c/genc.py
@@ -462,7 +462,9 @@
         s_result = annmodel.SomeInteger()
         graph = mix.getgraph(entrypoint_wrapper, args_s, s_result)
         mix.finish()
-        return getfunctionptr(graph)
+        fn = getfunctionptr(graph)
+        self._entrypoint_wrapper = fn
+        return fn
 
     def cmdexec(self, args='', env=None, err=False, expect_crash=False):
         assert self._compiled

From noreply at buildbot.pypy.org  Tue Oct 11 15:40:43 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 11 Oct 2011 15:40:43 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Debugging help.
Message-ID: <20111011134043.8BE4B82112@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r47938:f76373db4645
Date: 2011-10-11 15:40 +0200
http://bitbucket.org/pypy/pypy/changeset/f76373db4645/

Log:	Debugging help.

diff --git a/pypy/rpython/lltypesystem/llmemory.py b/pypy/rpython/lltypesystem/llmemory.py
--- a/pypy/rpython/lltypesystem/llmemory.py
+++ b/pypy/rpython/lltypesystem/llmemory.py
@@ -411,14 +411,21 @@
         self.ptr = ptr
 
     def __repr__(self):
+        x = ''
         if self.ptr is None:
             s = 'NULL'
         else:
+            from pypy.rpython.lltypesystem import llarena
+            try:
+                addr = llarena.getfakearenaaddress(self)
+                x = '(%s + %d) ' % (addr.arena, addr.offset)
+            except Exception:
+                pass
             #try:
             #    s = hex(self.ptr._cast_to_int())
             #except:
             s = str(self.ptr)
-        return '<fakeaddr %s>' % (s,)
+        return '<fakeaddr %s%s>' % (x, s)
 
     def __add__(self, other):
         if isinstance(other, AddressOffset):
diff --git a/pypy/rpython/memory/gc/concurrentms.py b/pypy/rpython/memory/gc/concurrentms.py
--- a/pypy/rpython/memory/gc/concurrentms.py
+++ b/pypy/rpython/memory/gc/concurrentms.py
@@ -230,6 +230,7 @@
                 obj = self.grow_reservation(result, totalsize)
                 hdr = self.header(obj)
                 hdr.tid = self.combine(typeid, self.current_mark, 0)
+                #debug_print("malloc_fixedsize_clear", obj)
                 return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
                 #
         return self._malloc_slowpath(typeid, size)
@@ -269,6 +270,7 @@
                 hdr = self.header(obj)
                 hdr.tid = self.combine(typeid, self.current_mark, 0)
                 (obj + offset_to_length).signed[0] = length
+                #debug_print("malloc_varsize_clear", obj)
                 return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
         #
         # If the total size of the object would be larger than
@@ -360,6 +362,7 @@
         hdr.tid = self.combine(typeid, self.current_mark, 0)
         #
         obj = result + size_gc_header
+        #debug_print("malloc_slowpath", obj)
         return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
         #
     _malloc_slowpath._dont_inline_ = True
@@ -472,6 +475,8 @@
         mark = self.header(addr_struct).tid & 0xFF
         if mark != self.current_mark:
             self.force_scan(addr_struct)
+        #else:
+        #    debug_print("deletion_barrier (off)", addr_struct)
 
     def assume_young_pointers(self, addr_struct):
         pass # XXX
@@ -479,7 +484,9 @@
     def _init_writebarrier_logic(self):
         #
         def force_scan(obj):
+            #debug_print("deletion_barrier  ON  ", obj)
             self.acquire(self.mutex_lock)
+            #debug_print("...main thread has mutex_lock")
             mark = self.header(obj).tid & 0xFF
             if mark != self.current_mark:
                 #
@@ -504,6 +511,7 @@
                     self.trace(obj, self._barrier_add_extra, None)
                 #
             self.release(self.mutex_lock)
+            #debug_print("deletion_barrier done ", obj)
         #
         force_scan._dont_inline_ = True
         self.force_scan = force_scan
@@ -520,7 +528,7 @@
             #
             self.acquire(self.finished_lock)
             self.collection_running = 0
-            debug_print("collection_running = 0")
+            #debug_print("collection_running = 0")
             #
             # Check invariants
             ll_assert(not self.extra_objects_to_mark.non_empty(),
@@ -624,7 +632,9 @@
         while p != self.NULL:
             x = llmemory.cast_ptr_to_adr(p)
             x = llarena.getfakearenaaddress(x) + 8
-            self.gray_objects.append(x + self.gcheaderbuilder.size_gc_header)
+            obj = x + self.gcheaderbuilder.size_gc_header
+            #debug_print("_objects_with_finalizers_to_run", obj)
+            self.gray_objects.append(obj)
             p = list_next(p)
         #
         # Invert this global variable, which has the effect that on all
@@ -649,16 +659,18 @@
         #
         # Start the collector thread
         self.collection_running = 1
-        debug_print("collection_running = 1")
+        #debug_print("collection_running = 1")
         self.release(self.ready_to_start_lock)
         #
         debug_stop("gc-start")
 
     def _add_stack_root(self, root):
         obj = root.address[0]
+        #debug_print("_add_stack_root", obj)
         self.gray_objects.append(obj)
 
     def _add_prebuilt_root(self, obj, ignored):
+        #debug_print("_add_prebuilt_root", obj)
         self.gray_objects.append(obj)
 
     def debug_check_lists(self):
@@ -743,7 +755,7 @@
             # Mark
             self.collector_mark()
             self.collection_running = 2
-            debug_print("collection_running = 2")
+            #debug_print("collection_running = 2")
             #
             self.deal_with_objects_with_finalizers()
             #
@@ -752,7 +764,7 @@
             #
             # Done!
             self.collection_running = -1
-            debug_print("collection_running = -1")
+            #debug_print("collection_running = -1")
             self.release(self.finished_lock)
 
 
@@ -783,6 +795,7 @@
             # There are typically only a few objects to move here,
             # unless XXX we've hit the write barrier of a large array
             self.acquire(self.mutex_lock)
+            #debug_print("...collector thread has mutex_lock")
             while self.extra_objects_to_mark.non_empty():
                 obj = self.extra_objects_to_mark.pop()
                 self.gray_objects.append(obj)
@@ -858,6 +871,7 @@
     def _collect_sweep_pages(self, n):
         # sweep all pages from the linked list starting at 'page',
         # containing objects of fixed size 'object_size'.
+        size_gc_header = self.gcheaderbuilder.size_gc_header
         page = self.collect_pages[n]
         object_size = n << WORD_POWER_2
         linked_list = self.NULL
@@ -879,6 +893,7 @@
                     # part of a linked list of free locations), and moreover
                     # the object is still not marked.  Free it by inserting
                     # it into the linked list.
+                    #debug_print("sweeps", adr + size_gc_header)
                     llarena.arena_reset(adr, object_size, 0)
                     llarena.arena_reserve(adr, self.HDRSIZE)
                     hdr = llmemory.cast_adr_to_ptr(adr, self.HDRPTR)
@@ -903,7 +918,6 @@
                     type_id = llop.extract_high_ushort(llgroup.HALFWORD, tid)
                     wroffset = self.weakpointer_offset(type_id)
                     if wroffset >= 0:
-                        size_gc_header = self.gcheaderbuilder.size_gc_header
                         obj = adr + size_gc_header
                         pointing_to = (obj + wroffset).address[0]
                         if pointing_to != llmemory.NULL:

From noreply at buildbot.pypy.org  Tue Oct 11 16:33:10 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 11 Oct 2011 16:33:10 +0200 (CEST)
Subject: [pypy-commit] pypy default: Comment out these lines,
 as they were never of any use as far as I know.
Message-ID: <20111011143310.4C39982112@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r47939:0417bfa18751
Date: 2011-10-11 15:56 +0200
http://bitbucket.org/pypy/pypy/changeset/0417bfa18751/

Log:	Comment out these lines, as they were never of any use as far as I
	know.

diff --git a/pypy/annotation/classdef.py b/pypy/annotation/classdef.py
--- a/pypy/annotation/classdef.py
+++ b/pypy/annotation/classdef.py
@@ -276,8 +276,8 @@
         # create the Attribute and do the generalization asked for
         newattr = Attribute(attr, self.bookkeeper)
         if s_value:
-            if newattr.name == 'intval' and getattr(s_value, 'unsigned', False):
-                import pdb; pdb.set_trace()
+            #if newattr.name == 'intval' and getattr(s_value, 'unsigned', False):
+            #    import pdb; pdb.set_trace()
             newattr.s_value = s_value
 
         # keep all subattributes' values

From noreply at buildbot.pypy.org  Tue Oct 11 16:33:11 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 11 Oct 2011 16:33:11 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Fixes,
	but always for test_transformed_gc
Message-ID: <20111011143311.82A5182A88@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r47940:427dfd51ee6b
Date: 2011-10-11 16:32 +0200
http://bitbucket.org/pypy/pypy/changeset/427dfd51ee6b/

Log:	Fixes, but always for test_transformed_gc

diff --git a/pypy/rpython/lltypesystem/llarena.py b/pypy/rpython/lltypesystem/llarena.py
--- a/pypy/rpython/lltypesystem/llarena.py
+++ b/pypy/rpython/lltypesystem/llarena.py
@@ -97,11 +97,10 @@
                 zero = False
             else:
                 raise ArenaError("new object overlaps a previous object")
-        assert offset not in self.objectptrs
         addr2 = size._raw_malloc([], zero=zero)
         pattern = letter.upper() + letter*(bytes-1)
         self.usagemap[offset:offset+bytes] = array.array('c', pattern)
-        self.setobject(addr2, offset, bytes)
+        self.setobject(addr2, offset, bytes, hdrbytes > 0)
         # in the common case where 'size' starts with a GCHeaderOffset,
         # we also remember that the real object starts after the header.
         if hdrbytes > 0:
@@ -110,10 +109,10 @@
             self.setobject(objaddr, objoffset, bytes - hdrbytes)
         return addr2
 
-    def setobject(self, objaddr, offset, bytes):
+    def setobject(self, objaddr, offset, bytes, can_overwrite=False):
         assert bytes > 0, ("llarena does not support GcStructs with no field"
                            " or empty arrays")
-        assert offset not in self.objectptrs
+        assert (offset not in self.objectptrs) or can_overwrite
         self.objectptrs[offset] = objaddr.ptr
         self.objectsizes[offset] = bytes
         container = objaddr.ptr._obj
@@ -165,7 +164,8 @@
         except KeyError:
             self.arena.check()
             raise ArenaError("don't know yet what type of object "
-                             "is at offset %d" % (self.offset,))
+                             "is in %s at offset %d" % (self.arena,
+                                                        self.offset))
     ptr = property(_getptr)
 
     def __repr__(self):
@@ -323,6 +323,9 @@
     def raw_memcopy(self, srcadr, dstadr):
         self.basesize.raw_memcopy(srcadr, dstadr)
 
+    def __sub__(self, other):
+        return RoundedUpForAllocation(self.basesize - other, self.minsize)
+
 # ____________________________________________________________
 #
 # Public interface: arena_malloc(), arena_free(), arena_reset()
diff --git a/pypy/rpython/lltypesystem/llmemory.py b/pypy/rpython/lltypesystem/llmemory.py
--- a/pypy/rpython/lltypesystem/llmemory.py
+++ b/pypy/rpython/lltypesystem/llmemory.py
@@ -238,6 +238,10 @@
         ofs.reverse()
         return CompositeOffset(*ofs)
 
+    def __sub__(self, other):
+        assert other == self.offsets[0]
+        return CompositeOffset(*self.offsets[1:])
+
     def known_nonneg(self):
         for item in self.offsets:
             if not item.known_nonneg():
diff --git a/pypy/rpython/lltypesystem/opimpl.py b/pypy/rpython/lltypesystem/opimpl.py
--- a/pypy/rpython/lltypesystem/opimpl.py
+++ b/pypy/rpython/lltypesystem/opimpl.py
@@ -599,15 +599,22 @@
     objaddr = llmemory.cast_ptr_to_adr(obj)
     hdraddr = objaddr - size_gc_header
     hdr = llmemory.cast_adr_to_ptr(hdraddr, lltype.Ptr(HDR))
-    typeid = getattr(hdr, fieldname)
-    if lltype.typeOf(typeid) == lltype.Signed:
+    if isinstance(fieldname, tuple):
         from pypy.rpython.lltypesystem import llgroup
-        if isinstance(typeid, llgroup.CombinedSymbolic):
-            typeid = op_extract_ushort(typeid)
-        elif isinstance(typeid, llgroup.HighCombinedSymbolic):
-            typeid = op_extract_high_ushort(typeid)
-        else:
-            raise TypeError(typeid)
+        assert fieldname[1] == llgroup.HALFSHIFT     # xxx for now
+        typeid = getattr(hdr, fieldname[0])
+        assert isinstance(typeid, llgroup.HighCombinedSymbolic)
+        typeid = op_extract_high_ushort(typeid)
+    else:
+        typeid = getattr(hdr, fieldname)
+        if lltype.typeOf(typeid) == lltype.Signed:
+            from pypy.rpython.lltypesystem import llgroup
+            if isinstance(typeid, llgroup.CombinedSymbolic):
+                typeid = op_extract_ushort(typeid)
+            elif isinstance(typeid, llgroup.HighCombinedSymbolic):
+                typeid = op_extract_high_ushort(typeid)
+            else:
+                raise TypeError(typeid)
     return op_get_next_group_member(TYPE, grpptr, typeid, skipoffset)
 op_gc_gettypeptr_group.need_result_type = True
 
diff --git a/pypy/rpython/memory/gc/concurrentms.py b/pypy/rpython/memory/gc/concurrentms.py
--- a/pypy/rpython/memory/gc/concurrentms.py
+++ b/pypy/rpython/memory/gc/concurrentms.py
@@ -466,7 +466,6 @@
         # address of the full object.
         adr = llmemory.cast_ptr_to_adr(hdr)
         adr = llarena.getfakearenaaddress(adr)
-        llarena.arena_reset(adr, self.HDRSIZE, 0)
         llarena.arena_reserve(adr, totalsize)
         return adr + self.gcheaderbuilder.size_gc_header
     grow_reservation._always_inline_ = True
@@ -732,6 +731,7 @@
                 llinterp.eval_graph(graph)
         except Exception, e:
             print 'Crash!', e.__class__.__name__, e
+            #import pdb; pdb.post_mortem(sys.exc_info()[2])
             self._exc_info = sys.exc_info()
 
     def collector_run(self):

From noreply at buildbot.pypy.org  Tue Oct 11 18:35:06 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue, 11 Oct 2011 18:35:06 +0200 (CEST)
Subject: [pypy-commit] pypy default: merged upstream
Message-ID: <20111011163506.C7A4682A88@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r47942:ed90c094b9ed
Date: 2011-10-11 12:34 -0400
http://bitbucket.org/pypy/pypy/changeset/ed90c094b9ed/

Log:	merged upstream

diff --git a/pypy/annotation/classdef.py b/pypy/annotation/classdef.py
--- a/pypy/annotation/classdef.py
+++ b/pypy/annotation/classdef.py
@@ -276,8 +276,8 @@
         # create the Attribute and do the generalization asked for
         newattr = Attribute(attr, self.bookkeeper)
         if s_value:
-            if newattr.name == 'intval' and getattr(s_value, 'unsigned', False):
-                import pdb; pdb.set_trace()
+            #if newattr.name == 'intval' and getattr(s_value, 'unsigned', False):
+            #    import pdb; pdb.set_trace()
             newattr.s_value = s_value
 
         # keep all subattributes' values

From noreply at buildbot.pypy.org  Tue Oct 11 18:35:05 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue, 11 Oct 2011 18:35:05 +0200 (CEST)
Subject: [pypy-commit] pypy default: make str.captitalize not do extra
	copies.
Message-ID: <20111011163505.9969782112@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r47941:c154a210d625
Date: 2011-10-11 12:33 -0400
http://bitbucket.org/pypy/pypy/changeset/c154a210d625/

Log:	make str.captitalize not do extra copies.

diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py
--- a/pypy/objspace/std/stringobject.py
+++ b/pypy/objspace/std/stringobject.py
@@ -178,24 +178,24 @@
 
 def str_capitalize__String(space, w_self):
     input = w_self._value
-    buffer = [' '] * len(input)
+    builder = StringBuilder(len(input))
     if len(input) > 0:
         ch = input[0]
         if ch.islower():
             o = ord(ch) - 32
-            buffer[0] = chr(o)
+            builder.append(chr(o))
         else:
-            buffer[0] = ch
+            builder.append(ch)
 
         for i in range(1, len(input)):
             ch = input[i]
             if ch.isupper():
                 o = ord(ch) + 32
-                buffer[i] = chr(o)
+                builder.append(chr(o))
             else:
-                buffer[i] = ch
+                builder.append(ch)
 
-    return space.wrap("".join(buffer))
+    return space.wrap(builder.build())
 
 def str_title__String(space, w_self):
     input = w_self._value

From noreply at buildbot.pypy.org  Tue Oct 11 21:37:09 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 11 Oct 2011 21:37:09 +0200 (CEST)
Subject: [pypy-commit] pypy default: Skip unicode tests when filesystem
 encoding is not good enough
Message-ID: <20111011193709.9D55982112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r47943:c7c09446a462
Date: 2011-10-11 21:35 +0200
http://bitbucket.org/pypy/pypy/changeset/c7c09446a462/

Log:	Skip unicode tests when filesystem encoding is not good enough

diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -420,12 +420,15 @@
             import sys
             if not hasattr(os, "fork"):
                 skip("Need fork() to test execv()")
+            try:
+                output = u"caf\xe9 \u1234\n".encode(sys.getfilesystemencoding())
+            except UnicodeEncodeError:
+                skip("encoding not good enough")
             pid = os.fork()
             if pid == 0:
                 os.execv(u"/bin/sh", ["sh", "-c",
                                       u"echo caf\xe9 \u1234 > onefile"])
             os.waitpid(pid, 0)
-            output = u"caf\xe9 \u1234\n".encode(sys.getfilesystemencoding())
             assert open("onefile").read() == output
             os.unlink("onefile")
 
@@ -445,13 +448,16 @@
             import sys
             if not hasattr(os, "fork"):
                 skip("Need fork() to test execve()")
+            try:
+                output = u"caf\xe9 \u1234\n".encode(sys.getfilesystemencoding())
+            except UnicodeEncodeError:
+                skip("encoding not good enough")
             pid = os.fork()
             if pid == 0:
                 os.execve(u"/bin/sh", ["sh", "-c",
                                       u"echo caf\xe9 \u1234 > onefile"],
                           {'ddd': 'xxx'})
             os.waitpid(pid, 0)
-            output = u"caf\xe9 \u1234\n".encode(sys.getfilesystemencoding())
             assert open("onefile").read() == output
             os.unlink("onefile")
         pass # <- please, inspect.getsource(), don't crash

From noreply at buildbot.pypy.org  Tue Oct 11 22:58:58 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 11 Oct 2011 22:58:58 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: some more info
Message-ID: <20111011205858.C7D8382112@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: extradoc
Changeset: r3923:478ec0e7dd1a
Date: 2011-10-11 22:58 +0200
http://bitbucket.org/pypy/extradoc/changeset/478ec0e7dd1a/

Log:	some more info

diff --git a/talk/pycon2012/tutorial.rst b/talk/pycon2012/tutorial.rst
--- a/talk/pycon2012/tutorial.rst
+++ b/talk/pycon2012/tutorial.rst
@@ -47,5 +47,6 @@
 
 We don't think tutorial assistants would be necessary.
 
-Outline: ...
-
+It's a bit hard to give a reasonable outline by now. Since the project
+evolves a lot, it's very likely a completely new set of tools will
+be available by the time we conduct the tutorial.

From noreply at buildbot.pypy.org  Tue Oct 11 23:15:38 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue, 11 Oct 2011 23:15:38 +0200 (CEST)
Subject: [pypy-commit] pypy default: (fijal,
 alex) Move map() from interp-level to app-level. fijal benchmarked
 this as between a 2x speedup and 10% slowdown. Apparently the JIT is kind
 of good.
Message-ID: <20111011211538.9341D82112@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r47944:cc47e5aeff35
Date: 2011-10-11 17:15 -0400
http://bitbucket.org/pypy/pypy/changeset/cc47e5aeff35/

Log:	(fijal, alex) Move map() from interp-level to app-level. fijal
	benchmarked this as between a 2x speedup and 10% slowdown.
	Apparently the JIT is kind of good.

diff --git a/pypy/module/__builtin__/__init__.py b/pypy/module/__builtin__/__init__.py
--- a/pypy/module/__builtin__/__init__.py
+++ b/pypy/module/__builtin__/__init__.py
@@ -20,6 +20,7 @@
         'any'           : 'app_functional.any',
         'all'           : 'app_functional.all',
         'sum'           : 'app_functional.sum',
+        'map'           : 'app_functional.map',
         'vars'          : 'app_inspect.vars',
         'dir'           : 'app_inspect.dir',
 
@@ -86,7 +87,6 @@
         'enumerate'     : 'functional.W_Enumerate',
         'min'           : 'functional.min',
         'max'           : 'functional.max',
-        'map'           : 'functional.map',
         'zip'           : 'functional.zip',
         'reduce'        : 'functional.reduce',
         'reversed'      : 'functional.reversed',
diff --git a/pypy/module/__builtin__/app_functional.py b/pypy/module/__builtin__/app_functional.py
--- a/pypy/module/__builtin__/app_functional.py
+++ b/pypy/module/__builtin__/app_functional.py
@@ -48,4 +48,53 @@
         # Very intentionally *not* +=, that would have different semantics if
         # start was a mutable type, such as a list
         last = last + x
-    return last
\ No newline at end of file
+    return last
+
+def map(func, *collections):
+    """map(function, sequence[, sequence, ...]) -> list
+
+Return a list of the results of applying the function to the items of
+the argument sequence(s).  If more than one sequence is given, the
+function is called with an argument list consisting of the corresponding
+item of each sequence, substituting None for missing values when not all
+sequences have the same length.  If the function is None, return a list of
+the items of the sequence (or a list of tuples if more than one sequence)."""
+    if not collections:
+        raise TypeError("map() requires at least two arguments")
+    num_collections = len(collections)
+    none_func = func is None
+    if num_collections == 1:
+        if none_func:
+            return list(collections[0])
+        else:
+            # Special case for the really common case of a single collection,
+            # this can be eliminated if we could unroll that loop that creates
+            # `args` based on whether or not len(collections) was constant
+            result = []
+            for item in collections[0]:
+                result.append(func(item))
+            return result
+    result = []
+    # Pair of (iterator, has_finished)
+    iterators = [(iter(seq), False) for seq in collections]
+    while True:
+        cont = False
+        args = []
+        for idx, (iterator, has_finished) in enumerate(iterators):
+            val = None
+            if not has_finished:
+                try:
+                    val = next(iterator)
+                except StopIteration:
+                    iterators[idx] = (None, True)
+                else:
+                    cont = True
+            args.append(val)
+        args = tuple(args)
+        if cont:
+            if none_func:
+                result.append(args)
+            else:
+                result.append(func(*args))
+        else:
+            return result
diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py
--- a/pypy/module/__builtin__/functional.py
+++ b/pypy/module/__builtin__/functional.py
@@ -201,118 +201,6 @@
     """
     return min_max(space, __args__, "min")
 
- at unwrap_spec(collections_w="args_w")
-def map(space, w_func, collections_w):
-    """does 3 separate things, hence this enormous docstring.
-       1.  if function is None, return a list of tuples, each with one
-           item from each collection.  If the collections have different
-           lengths,  shorter ones are padded with None.
-
-       2.  if function is not None, and there is only one collection,
-           apply function to every item in the collection and return a
-           list of the results.
-
-       3.  if function is not None, and there are several collections,
-           repeatedly call the function with one argument from each
-           collection.  If the collections have different lengths,
-           shorter ones are padded with None
-    """
-    if not collections_w:
-        msg = "map() requires at least two arguments"
-        raise OperationError(space.w_TypeError, space.wrap(msg))
-    none_func = space.is_w(w_func, space.w_None)
-    if len(collections_w) == 1:
-        w_collection = collections_w[0]
-        if none_func:
-            result_w = space.unpackiterable(w_collection)
-        else:
-            result_w = map_single_collection(space, w_func, w_collection)
-    else:
-        result_w = map_multiple_collections(space, w_func, collections_w,
-                                            none_func)
-    return space.newlist(result_w)
-
-def map_single_collection(space, w_func, w_collection):
-    """Special case for 'map(func, coll)', where 'func' is not None and there
-    is only one 'coll' argument."""
-    w_iter = space.iter(w_collection)
-    # xxx special hacks for speed
-    from pypy.interpreter import function, pycode
-    if isinstance(w_func, function.Function):
-        # xxx compatibility issue: what if func_code is modified in the
-        # middle of running map()??  That's far too obscure for me to care...
-        code = w_func.getcode()
-        fast_natural_arity = code.fast_natural_arity
-        if fast_natural_arity == (1|pycode.PyCode.FLATPYCALL):
-            assert isinstance(code, pycode.PyCode)
-            return map_single_user_function(code, w_func, w_iter)
-    # /xxx end of special hacks
-    return map_single_other_callable(space, w_func, w_iter)
-
-def map_single_other_callable(space, w_func, w_iter):
-    result_w = []
-    while True:
-        try:
-            w_item = space.next(w_iter)
-        except OperationError, e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            break
-        result_w.append(space.call_function(w_func, w_item))
-    return result_w
-map_single_other_callable._dont_inline_ = True
-
-from pypy.rlib.jit import JitDriver
-mapjitdriver = JitDriver(greens = ['code'],
-                         reds = ['w_func', 'w_iter', 'result_w'])
-def map_single_user_function(code, w_func, w_iter):
-    result_w = []
-    while True:
-        mapjitdriver.can_enter_jit(code=code, w_func=w_func,
-                                   w_iter=w_iter, result_w=result_w)
-        mapjitdriver.jit_merge_point(code=code, w_func=w_func,
-                                     w_iter=w_iter, result_w=result_w)
-        space = w_func.space
-        try:
-            w_item = space.next(w_iter)
-        except OperationError, e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            break
-        new_frame = space.createframe(code, w_func.w_func_globals,
-                                      w_func)
-        new_frame.locals_stack_w[0] = w_item
-        w_res = new_frame.run()
-        result_w.append(w_res)
-    return result_w
-
-def map_multiple_collections(space, w_func, collections_w, none_func):
-    result_w = []
-    iterators_w = [space.iter(w_seq) for w_seq in collections_w]
-    num_iterators = len(iterators_w)
-    while True:
-        cont = False
-        args_w = [space.w_None] * num_iterators
-        for i in range(num_iterators):
-            if iterators_w[i] is not None:
-                try:
-                    args_w[i] = space.next(iterators_w[i])
-                except OperationError, e:
-                    if not e.match(space, space.w_StopIteration):
-                        raise
-                    iterators_w[i] = None
-                else:
-                    cont = True
-        if not cont:
-            break
-        w_args = space.newtuple(args_w)
-        if none_func:
-            w_res = w_args
-        else:
-            w_res = space.call(w_func, w_args)
-        result_w.append(w_res)
-    return result_w
-
 @unwrap_spec(sequences_w="args_w")
 def zip(space, sequences_w):
     """Return a list of tuples, where the nth tuple contains every nth item of
diff --git a/pypy/module/__builtin__/test/test_functional.py b/pypy/module/__builtin__/test/test_functional.py
--- a/pypy/module/__builtin__/test/test_functional.py
+++ b/pypy/module/__builtin__/test/test_functional.py
@@ -147,7 +147,7 @@
        assert list(xrange(A())) == [0, 1, 2, 3, 4]
        assert list(xrange(0, A())) == [0, 1, 2, 3, 4]
        assert list(xrange(0, 10, A())) == [0, 5]
- 
+
    def test_xrange_float(self):
       assert list(xrange(0.1, 2.0, 1.1)) == [0, 1]
 

From noreply at buildbot.pypy.org  Tue Oct 11 23:26:59 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 11 Oct 2011 23:26:59 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Initiate a py3k branch. Nothing works here,
 the 3.2 stdlib is missing.
Message-ID: <20111011212659.EEC7982112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r47945:186690e4ab43
Date: 2011-10-11 22:42 +0200
http://bitbucket.org/pypy/pypy/changeset/186690e4ab43/

Log:	Initiate a py3k branch. Nothing works here, the 3.2 stdlib is
	missing.

diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -7,7 +7,7 @@
 from pypy.interpreter import gateway
 
 #XXX # the release serial 42 is not in range(16)
-CPYTHON_VERSION            = (2, 7, 1, "final", 42)   #XXX # sync patchlevel.h
+CPYTHON_VERSION            = (3, 2, 2, "final", 0)   #XXX # sync patchlevel.h
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 
 PYPY_VERSION               = (1, 6, 1, "dev", 0)    #XXX # sync patchlevel.h

From noreply at buildbot.pypy.org  Tue Oct 11 23:27:15 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 11 Oct 2011 23:27:15 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Add CPython standard library,
	tag: v3.2.2 (137e45f15c0b)
Message-ID: <20111011212715.863A582112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r47946:5a8684411ae6
Date: 2011-10-11 23:09 +0200
http://bitbucket.org/pypy/pypy/changeset/5a8684411ae6/

Log:	Add CPython standard library, tag: v3.2.2 (137e45f15c0b)

diff too long, truncating to 10000 out of 553449 lines

diff --git a/lib-python/3.2/__future__.py b/lib-python/3.2/__future__.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/__future__.py
@@ -0,0 +1,134 @@
+"""Record of phased-in incompatible language changes.
+
+Each line is of the form:
+
+    FeatureName = "_Feature(" OptionalRelease "," MandatoryRelease ","
+                              CompilerFlag ")"
+
+where, normally, OptionalRelease < MandatoryRelease, and both are 5-tuples
+of the same form as sys.version_info:
+
+    (PY_MAJOR_VERSION, # the 2 in 2.1.0a3; an int
+     PY_MINOR_VERSION, # the 1; an int
+     PY_MICRO_VERSION, # the 0; an int
+     PY_RELEASE_LEVEL, # "alpha", "beta", "candidate" or "final"; string
+     PY_RELEASE_SERIAL # the 3; an int
+    )
+
+OptionalRelease records the first release in which
+
+    from __future__ import FeatureName
+
+was accepted.
+
+In the case of MandatoryReleases that have not yet occurred,
+MandatoryRelease predicts the release in which the feature will become part
+of the language.
+
+Else MandatoryRelease records when the feature became part of the language;
+in releases at or after that, modules no longer need
+
+    from __future__ import FeatureName
+
+to use the feature in question, but may continue to use such imports.
+
+MandatoryRelease may also be None, meaning that a planned feature got
+dropped.
+
+Instances of class _Feature have two corresponding methods,
+.getOptionalRelease() and .getMandatoryRelease().
+
+CompilerFlag is the (bitfield) flag that should be passed in the fourth
+argument to the builtin function compile() to enable the feature in
+dynamically compiled code.  This flag is stored in the .compiler_flag
+attribute on _Future instances.  These values must match the appropriate
+#defines of CO_xxx flags in Include/compile.h.
+
+No feature line is ever to be deleted from this file.
+"""
+
+all_feature_names = [
+    "nested_scopes",
+    "generators",
+    "division",
+    "absolute_import",
+    "with_statement",
+    "print_function",
+    "unicode_literals",
+    "barry_as_FLUFL",
+]
+
+__all__ = ["all_feature_names"] + all_feature_names
+
+# The CO_xxx symbols are defined here under the same names used by
+# compile.h, so that an editor search will find them here.  However,
+# they're not exported in __all__, because they don't really belong to
+# this module.
+CO_NESTED            = 0x0010   # nested_scopes
+CO_GENERATOR_ALLOWED = 0        # generators (obsolete, was 0x1000)
+CO_FUTURE_DIVISION   = 0x2000   # division
+CO_FUTURE_ABSOLUTE_IMPORT = 0x4000 # perform absolute imports by default
+CO_FUTURE_WITH_STATEMENT  = 0x8000   # with statement
+CO_FUTURE_PRINT_FUNCTION  = 0x10000   # print function
+CO_FUTURE_UNICODE_LITERALS = 0x20000 # unicode string literals
+CO_FUTURE_BARRY_AS_BDFL = 0x40000
+
+class _Feature:
+    def __init__(self, optionalRelease, mandatoryRelease, compiler_flag):
+        self.optional = optionalRelease
+        self.mandatory = mandatoryRelease
+        self.compiler_flag = compiler_flag
+
+    def getOptionalRelease(self):
+        """Return first release in which this feature was recognized.
+
+        This is a 5-tuple, of the same form as sys.version_info.
+        """
+
+        return self.optional
+
+    def getMandatoryRelease(self):
+        """Return release in which this feature will become mandatory.
+
+        This is a 5-tuple, of the same form as sys.version_info, or, if
+        the feature was dropped, is None.
+        """
+
+        return self.mandatory
+
+    def __repr__(self):
+        return "_Feature" + repr((self.optional,
+                                  self.mandatory,
+                                  self.compiler_flag))
+
+nested_scopes = _Feature((2, 1, 0, "beta",  1),
+                         (2, 2, 0, "alpha", 0),
+                         CO_NESTED)
+
+generators = _Feature((2, 2, 0, "alpha", 1),
+                      (2, 3, 0, "final", 0),
+                      CO_GENERATOR_ALLOWED)
+
+division = _Feature((2, 2, 0, "alpha", 2),
+                    (3, 0, 0, "alpha", 0),
+                    CO_FUTURE_DIVISION)
+
+absolute_import = _Feature((2, 5, 0, "alpha", 1),
+                           (2, 7, 0, "alpha", 0),
+                           CO_FUTURE_ABSOLUTE_IMPORT)
+
+with_statement = _Feature((2, 5, 0, "alpha", 1),
+                          (2, 6, 0, "alpha", 0),
+                          CO_FUTURE_WITH_STATEMENT)
+
+print_function = _Feature((2, 6, 0, "alpha", 2),
+                          (3, 0, 0, "alpha", 0),
+                          CO_FUTURE_PRINT_FUNCTION)
+
+unicode_literals = _Feature((2, 6, 0, "alpha", 2),
+                            (3, 0, 0, "alpha", 0),
+                            CO_FUTURE_UNICODE_LITERALS)
+
+barry_as_FLUFL = _Feature((3, 1, 0, "alpha", 2),
+                         (3, 9, 0, "alpha", 0),
+                         CO_FUTURE_BARRY_AS_BDFL)
diff --git a/lib-python/3.2/__phello__.foo.py b/lib-python/3.2/__phello__.foo.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/__phello__.foo.py
@@ -0,0 +1,1 @@
+# This file exists as a helper for the test.test_frozen module.
diff --git a/lib-python/3.2/_abcoll.py b/lib-python/3.2/_abcoll.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/_abcoll.py
@@ -0,0 +1,623 @@
+# Copyright 2007 Google, Inc. All Rights Reserved.
+# Licensed to PSF under a Contributor Agreement.
+
+"""Abstract Base Classes (ABCs) for collections, according to PEP 3119.
+
+DON'T USE THIS MODULE DIRECTLY!  The classes here should be imported
+via collections; they are defined here only to alleviate certain
+bootstrapping issues.  Unit tests are in test_collections.
+"""
+
+from abc import ABCMeta, abstractmethod
+import sys
+
+__all__ = ["Hashable", "Iterable", "Iterator",
+           "Sized", "Container", "Callable",
+           "Set", "MutableSet",
+           "Mapping", "MutableMapping",
+           "MappingView", "KeysView", "ItemsView", "ValuesView",
+           "Sequence", "MutableSequence",
+           "ByteString",
+           ]
+
+
+### collection related types which are not exposed through builtin ###
+## iterators ##
+bytes_iterator = type(iter(b''))
+bytearray_iterator = type(iter(bytearray()))
+#callable_iterator = ???
+dict_keyiterator = type(iter({}.keys()))
+dict_valueiterator = type(iter({}.values()))
+dict_itemiterator = type(iter({}.items()))
+list_iterator = type(iter([]))
+list_reverseiterator = type(iter(reversed([])))
+range_iterator = type(iter(range(0)))
+set_iterator = type(iter(set()))
+str_iterator = type(iter(""))
+tuple_iterator = type(iter(()))
+zip_iterator = type(iter(zip()))
+## views ##
+dict_keys = type({}.keys())
+dict_values = type({}.values())
+dict_items = type({}.items())
+## misc ##
+dict_proxy = type(type.__dict__)
+
+
+### ONE-TRICK PONIES ###
+
+class Hashable(metaclass=ABCMeta):
+
+    @abstractmethod
+    def __hash__(self):
+        return 0
+
+    @classmethod
+    def __subclasshook__(cls, C):
+        if cls is Hashable:
+            for B in C.__mro__:
+                if "__hash__" in B.__dict__:
+                    if B.__dict__["__hash__"]:
+                        return True
+                    break
+        return NotImplemented
+
+
+class Iterable(metaclass=ABCMeta):
+
+    @abstractmethod
+    def __iter__(self):
+        while False:
+            yield None
+
+    @classmethod
+    def __subclasshook__(cls, C):
+        if cls is Iterable:
+            if any("__iter__" in B.__dict__ for B in C.__mro__):
+                return True
+        return NotImplemented
+
+
+class Iterator(Iterable):
+
+    @abstractmethod
+    def __next__(self):
+        raise StopIteration
+
+    def __iter__(self):
+        return self
+
+    @classmethod
+    def __subclasshook__(cls, C):
+        if cls is Iterator:
+            if (any("__next__" in B.__dict__ for B in C.__mro__) and
+                any("__iter__" in B.__dict__ for B in C.__mro__)):
+                return True
+        return NotImplemented
+
+Iterator.register(bytes_iterator)
+Iterator.register(bytearray_iterator)
+#Iterator.register(callable_iterator)
+Iterator.register(dict_keyiterator)
+Iterator.register(dict_valueiterator)
+Iterator.register(dict_itemiterator)
+Iterator.register(list_iterator)
+Iterator.register(list_reverseiterator)
+Iterator.register(range_iterator)
+Iterator.register(set_iterator)
+Iterator.register(str_iterator)
+Iterator.register(tuple_iterator)
+Iterator.register(zip_iterator)
+
+class Sized(metaclass=ABCMeta):
+
+    @abstractmethod
+    def __len__(self):
+        return 0
+
+    @classmethod
+    def __subclasshook__(cls, C):
+        if cls is Sized:
+            if any("__len__" in B.__dict__ for B in C.__mro__):
+                return True
+        return NotImplemented
+
+
+class Container(metaclass=ABCMeta):
+
+    @abstractmethod
+    def __contains__(self, x):
+        return False
+
+    @classmethod
+    def __subclasshook__(cls, C):
+        if cls is Container:
+            if any("__contains__" in B.__dict__ for B in C.__mro__):
+                return True
+        return NotImplemented
+
+
+class Callable(metaclass=ABCMeta):
+
+    @abstractmethod
+    def __call__(self, *args, **kwds):
+        return False
+
+    @classmethod
+    def __subclasshook__(cls, C):
+        if cls is Callable:
+            if any("__call__" in B.__dict__ for B in C.__mro__):
+                return True
+        return NotImplemented
+
+
+### SETS ###
+
+
+class Set(Sized, Iterable, Container):
+
+    """A set is a finite, iterable container.
+
+    This class provides concrete generic implementations of all
+    methods except for __contains__, __iter__ and __len__.
+
+    To override the comparisons (presumably for speed, as the
+    semantics are fixed), all you have to do is redefine __le__ and
+    then the other operations will automatically follow suit.
+    """
+
+    def __le__(self, other):
+        if not isinstance(other, Set):
+            return NotImplemented
+        if len(self) > len(other):
+            return False
+        for elem in self:
+            if elem not in other:
+                return False
+        return True
+
+    def __lt__(self, other):
+        if not isinstance(other, Set):
+            return NotImplemented
+        return len(self) < len(other) and self.__le__(other)
+
+    def __gt__(self, other):
+        if not isinstance(other, Set):
+            return NotImplemented
+        return other < self
+
+    def __ge__(self, other):
+        if not isinstance(other, Set):
+            return NotImplemented
+        return other <= self
+
+    def __eq__(self, other):
+        if not isinstance(other, Set):
+            return NotImplemented
+        return len(self) == len(other) and self.__le__(other)
+
+    def __ne__(self, other):
+        return not (self == other)
+
+    @classmethod
+    def _from_iterable(cls, it):
+        '''Construct an instance of the class from any iterable input.
+
+        Must override this method if the class constructor signature
+        does not accept an iterable for an input.
+        '''
+        return cls(it)
+
+    def __and__(self, other):
+        if not isinstance(other, Iterable):
+            return NotImplemented
+        return self._from_iterable(value for value in other if value in self)
+
+    def isdisjoint(self, other):
+        for value in other:
+            if value in self:
+                return False
+        return True
+
+    def __or__(self, other):
+        if not isinstance(other, Iterable):
+            return NotImplemented
+        chain = (e for s in (self, other) for e in s)
+        return self._from_iterable(chain)
+
+    def __sub__(self, other):
+        if not isinstance(other, Set):
+            if not isinstance(other, Iterable):
+                return NotImplemented
+            other = self._from_iterable(other)
+        return self._from_iterable(value for value in self
+                                   if value not in other)
+
+    def __xor__(self, other):
+        if not isinstance(other, Set):
+            if not isinstance(other, Iterable):
+                return NotImplemented
+            other = self._from_iterable(other)
+        return (self - other) | (other - self)
+
+    def _hash(self):
+        """Compute the hash value of a set.
+
+        Note that we don't define __hash__: not all sets are hashable.
+        But if you define a hashable set type, its __hash__ should
+        call this function.
+
+        This must be compatible __eq__.
+
+        All sets ought to compare equal if they contain the same
+        elements, regardless of how they are implemented, and
+        regardless of the order of the elements; so there's not much
+        freedom for __eq__ or __hash__.  We match the algorithm used
+        by the built-in frozenset type.
+        """
+        MAX = sys.maxsize
+        MASK = 2 * MAX + 1
+        n = len(self)
+        h = 1927868237 * (n + 1)
+        h &= MASK
+        for x in self:
+            hx = hash(x)
+            h ^= (hx ^ (hx << 16) ^ 89869747)  * 3644798167
+            h &= MASK
+        h = h * 69069 + 907133923
+        h &= MASK
+        if h > MAX:
+            h -= MASK + 1
+        if h == -1:
+            h = 590923713
+        return h
+
+Set.register(frozenset)
+
+
+class MutableSet(Set):
+
+    @abstractmethod
+    def add(self, value):
+        """Add an element."""
+        raise NotImplementedError
+
+    @abstractmethod
+    def discard(self, value):
+        """Remove an element.  Do not raise an exception if absent."""
+        raise NotImplementedError
+
+    def remove(self, value):
+        """Remove an element. If not a member, raise a KeyError."""
+        if value not in self:
+            raise KeyError(value)
+        self.discard(value)
+
+    def pop(self):
+        """Return the popped value.  Raise KeyError if empty."""
+        it = iter(self)
+        try:
+            value = next(it)
+        except StopIteration:
+            raise KeyError
+        self.discard(value)
+        return value
+
+    def clear(self):
+        """This is slow (creates N new iterators!) but effective."""
+        try:
+            while True:
+                self.pop()
+        except KeyError:
+            pass
+
+    def __ior__(self, it):
+        for value in it:
+            self.add(value)
+        return self
+
+    def __iand__(self, it):
+        for value in (self - it):
+            self.discard(value)
+        return self
+
+    def __ixor__(self, it):
+        if it is self:
+            self.clear()
+        else:
+            if not isinstance(it, Set):
+                it = self._from_iterable(it)
+            for value in it:
+                if value in self:
+                    self.discard(value)
+                else:
+                    self.add(value)
+        return self
+
+    def __isub__(self, it):
+        if it is self:
+            self.clear()
+        else:
+            for value in it:
+                self.discard(value)
+        return self
+
+MutableSet.register(set)
+
+
+### MAPPINGS ###
+
+
+class Mapping(Sized, Iterable, Container):
+
+    @abstractmethod
+    def __getitem__(self, key):
+        raise KeyError
+
+    def get(self, key, default=None):
+        try:
+            return self[key]
+        except KeyError:
+            return default
+
+    def __contains__(self, key):
+        try:
+            self[key]
+        except KeyError:
+            return False
+        else:
+            return True
+
+    def keys(self):
+        return KeysView(self)
+
+    def items(self):
+        return ItemsView(self)
+
+    def values(self):
+        return ValuesView(self)
+
+    def __eq__(self, other):
+        if not isinstance(other, Mapping):
+            return NotImplemented
+        return dict(self.items()) == dict(other.items())
+
+    def __ne__(self, other):
+        return not (self == other)
+
+
+class MappingView(Sized):
+
+    def __init__(self, mapping):
+        self._mapping = mapping
+
+    def __len__(self):
+        return len(self._mapping)
+
+    def __repr__(self):
+        return '{0.__class__.__name__}({0._mapping!r})'.format(self)
+
+
+class KeysView(MappingView, Set):
+
+    @classmethod
+    def _from_iterable(self, it):
+        return set(it)
+
+    def __contains__(self, key):
+        return key in self._mapping
+
+    def __iter__(self):
+        for key in self._mapping:
+            yield key
+
+KeysView.register(dict_keys)
+
+
+class ItemsView(MappingView, Set):
+
+    @classmethod
+    def _from_iterable(self, it):
+        return set(it)
+
+    def __contains__(self, item):
+        key, value = item
+        try:
+            v = self._mapping[key]
+        except KeyError:
+            return False
+        else:
+            return v == value
+
+    def __iter__(self):
+        for key in self._mapping:
+            yield (key, self._mapping[key])
+
+ItemsView.register(dict_items)
+
+
+class ValuesView(MappingView):
+
+    def __contains__(self, value):
+        for key in self._mapping:
+            if value == self._mapping[key]:
+                return True
+        return False
+
+    def __iter__(self):
+        for key in self._mapping:
+            yield self._mapping[key]
+
+ValuesView.register(dict_values)
+
+
+class MutableMapping(Mapping):
+
+    @abstractmethod
+    def __setitem__(self, key, value):
+        raise KeyError
+
+    @abstractmethod
+    def __delitem__(self, key):
+        raise KeyError
+
+    __marker = object()
+
+    def pop(self, key, default=__marker):
+        try:
+            value = self[key]
+        except KeyError:
+            if default is self.__marker:
+                raise
+            return default
+        else:
+            del self[key]
+            return value
+
+    def popitem(self):
+        try:
+            key = next(iter(self))
+        except StopIteration:
+            raise KeyError
+        value = self[key]
+        del self[key]
+        return key, value
+
+    def clear(self):
+        try:
+            while True:
+                self.popitem()
+        except KeyError:
+            pass
+
+    def update(*args, **kwds):
+        if len(args) > 2:
+            raise TypeError("update() takes at most 2 positional "
+                            "arguments ({} given)".format(len(args)))
+        elif not args:
+            raise TypeError("update() takes at least 1 argument (0 given)")
+        self = args[0]
+        other = args[1] if len(args) >= 2 else ()
+
+        if isinstance(other, Mapping):
+            for key in other:
+                self[key] = other[key]
+        elif hasattr(other, "keys"):
+            for key in other.keys():
+                self[key] = other[key]
+        else:
+            for key, value in other:
+                self[key] = value
+        for key, value in kwds.items():
+            self[key] = value
+
+    def setdefault(self, key, default=None):
+        try:
+            return self[key]
+        except KeyError:
+            self[key] = default
+        return default
+
+MutableMapping.register(dict)
+
+
+### SEQUENCES ###
+
+
+class Sequence(Sized, Iterable, Container):
+
+    """All the operations on a read-only sequence.
+
+    Concrete subclasses must override __new__ or __init__,
+    __getitem__, and __len__.
+    """
+
+    @abstractmethod
+    def __getitem__(self, index):
+        raise IndexError
+
+    def __iter__(self):
+        i = 0
+        try:
+            while True:
+                v = self[i]
+                yield v
+                i += 1
+        except IndexError:
+            return
+
+    def __contains__(self, value):
+        for v in self:
+            if v == value:
+                return True
+        return False
+
+    def __reversed__(self):
+        for i in reversed(range(len(self))):
+            yield self[i]
+
+    def index(self, value):
+        for i, v in enumerate(self):
+            if v == value:
+                return i
+        raise ValueError
+
+    def count(self, value):
+        return sum(1 for v in self if v == value)
+
+Sequence.register(tuple)
+Sequence.register(str)
+Sequence.register(range)
+
+
+class ByteString(Sequence):
+
+    """This unifies bytes and bytearray.
+
+    XXX Should add all their methods.
+    """
+
+ByteString.register(bytes)
+ByteString.register(bytearray)
+
+
+class MutableSequence(Sequence):
+
+    @abstractmethod
+    def __setitem__(self, index, value):
+        raise IndexError
+
+    @abstractmethod
+    def __delitem__(self, index):
+        raise IndexError
+
+    @abstractmethod
+    def insert(self, index, value):
+        raise IndexError
+
+    def append(self, value):
+        self.insert(len(self), value)
+
+    def reverse(self):
+        n = len(self)
+        for i in range(n//2):
+            self[i], self[n-i-1] = self[n-i-1], self[i]
+
+    def extend(self, values):
+        for v in values:
+            self.append(v)
+
+    def pop(self, index=-1):
+        v = self[index]
+        del self[index]
+        return v
+
+    def remove(self, value):
+        del self[self.index(value)]
+
+    def __iadd__(self, values):
+        self.extend(values)
+        return self
+
+MutableSequence.register(list)
+MutableSequence.register(bytearray)  # Multiply inheriting, see ByteString
diff --git a/lib-python/3.2/_compat_pickle.py b/lib-python/3.2/_compat_pickle.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/_compat_pickle.py
@@ -0,0 +1,81 @@
+# This module is used to map the old Python 2 names to the new names used in
+# Python 3 for the pickle module.  This needed to make pickle streams
+# generated with Python 2 loadable by Python 3.
+
+# This is a copy of lib2to3.fixes.fix_imports.MAPPING.  We cannot import
+# lib2to3 and use the mapping defined there, because lib2to3 uses pickle.
+# Thus, this could cause the module to be imported recursively.
+IMPORT_MAPPING = {
+    'StringIO':  'io',
+    'cStringIO': 'io',
+    'cPickle': 'pickle',
+    '__builtin__' : 'builtins',
+    'copy_reg': 'copyreg',
+    'Queue': 'queue',
+    'SocketServer': 'socketserver',
+    'ConfigParser': 'configparser',
+    'repr': 'reprlib',
+    'FileDialog': 'tkinter.filedialog',
+    'tkFileDialog': 'tkinter.filedialog',
+    'SimpleDialog': 'tkinter.simpledialog',
+    'tkSimpleDialog': 'tkinter.simpledialog',
+    'tkColorChooser': 'tkinter.colorchooser',
+    'tkCommonDialog': 'tkinter.commondialog',
+    'Dialog': 'tkinter.dialog',
+    'Tkdnd': 'tkinter.dnd',
+    'tkFont': 'tkinter.font',
+    'tkMessageBox': 'tkinter.messagebox',
+    'ScrolledText': 'tkinter.scrolledtext',
+    'Tkconstants': 'tkinter.constants',
+    'Tix': 'tkinter.tix',
+    'ttk': 'tkinter.ttk',
+    'Tkinter': 'tkinter',
+    'markupbase': '_markupbase',
+    '_winreg': 'winreg',
+    'thread': '_thread',
+    'dummy_thread': '_dummy_thread',
+    'dbhash': 'dbm.bsd',
+    'dumbdbm': 'dbm.dumb',
+    'dbm': 'dbm.ndbm',
+    'gdbm': 'dbm.gnu',
+    'xmlrpclib': 'xmlrpc.client',
+    'DocXMLRPCServer': 'xmlrpc.server',
+    'SimpleXMLRPCServer': 'xmlrpc.server',
+    'httplib': 'http.client',
+    'htmlentitydefs' : 'html.entities',
+    'HTMLParser' : 'html.parser',
+    'Cookie': 'http.cookies',
+    'cookielib': 'http.cookiejar',
+    'BaseHTTPServer': 'http.server',
+    'SimpleHTTPServer': 'http.server',
+    'CGIHTTPServer': 'http.server',
+    'test.test_support': 'test.support',
+    'commands': 'subprocess',
+    'UserString' : 'collections',
+    'UserList' : 'collections',
+    'urlparse' : 'urllib.parse',
+    'robotparser' : 'urllib.robotparser',
+    'whichdb': 'dbm',
+    'anydbm': 'dbm'
+}
+
+
+# This contains rename rules that are easy to handle.  We ignore the more
+# complex stuff (e.g. mapping the names in the urllib and types modules).
+# These rules should be run before import names are fixed.
+NAME_MAPPING = {
+    ('__builtin__', 'xrange'):     ('builtins', 'range'),
+    ('__builtin__', 'reduce'):     ('functools', 'reduce'),
+    ('__builtin__', 'intern'):     ('sys', 'intern'),
+    ('__builtin__', 'unichr'):     ('builtins', 'chr'),
+    ('__builtin__', 'basestring'): ('builtins', 'str'),
+    ('__builtin__', 'long'):       ('builtins', 'int'),
+    ('itertools', 'izip'):         ('builtins', 'zip'),
+    ('itertools', 'imap'):         ('builtins', 'map'),
+    ('itertools', 'ifilter'):      ('builtins', 'filter'),
+    ('itertools', 'ifilterfalse'): ('itertools', 'filterfalse'),
+}
+
+# Same, but for 3.x to 2.x
+REVERSE_IMPORT_MAPPING = dict((v, k) for (k, v) in IMPORT_MAPPING.items())
+REVERSE_NAME_MAPPING = dict((v, k) for (k, v) in NAME_MAPPING.items())
diff --git a/lib-python/3.2/_dummy_thread.py b/lib-python/3.2/_dummy_thread.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/_dummy_thread.py
@@ -0,0 +1,155 @@
+"""Drop-in replacement for the thread module.
+
+Meant to be used as a brain-dead substitute so that threaded code does
+not need to be rewritten for when the thread module is not present.
+
+Suggested usage is::
+
+    try:
+        import _thread
+    except ImportError:
+        import _dummy_thread as _thread
+
+"""
+# Exports only things specified by thread documentation;
+# skipping obsolete synonyms allocate(), start_new(), exit_thread().
+__all__ = ['error', 'start_new_thread', 'exit', 'get_ident', 'allocate_lock',
+           'interrupt_main', 'LockType']
+
+# A dummy value
+TIMEOUT_MAX = 2**31
+
+# NOTE: this module can be imported early in the extension building process,
+# and so top level imports of other modules should be avoided.  Instead, all
+# imports are done when needed on a function-by-function basis.  Since threads
+# are disabled, the import lock should not be an issue anyway (??).
+
+class error(Exception):
+    """Dummy implementation of _thread.error."""
+
+    def __init__(self, *args):
+        self.args = args
+
+def start_new_thread(function, args, kwargs={}):
+    """Dummy implementation of _thread.start_new_thread().
+
+    Compatibility is maintained by making sure that ``args`` is a
+    tuple and ``kwargs`` is a dictionary.  If an exception is raised
+    and it is SystemExit (which can be done by _thread.exit()) it is
+    caught and nothing is done; all other exceptions are printed out
+    by using traceback.print_exc().
+
+    If the executed function calls interrupt_main the KeyboardInterrupt will be
+    raised when the function returns.
+
+    """
+    if type(args) != type(tuple()):
+        raise TypeError("2nd arg must be a tuple")
+    if type(kwargs) != type(dict()):
+        raise TypeError("3rd arg must be a dict")
+    global _main
+    _main = False
+    try:
+        function(*args, **kwargs)
+    except SystemExit:
+        pass
+    except:
+        import traceback
+        traceback.print_exc()
+    _main = True
+    global _interrupt
+    if _interrupt:
+        _interrupt = False
+        raise KeyboardInterrupt
+
+def exit():
+    """Dummy implementation of _thread.exit()."""
+    raise SystemExit
+
+def get_ident():
+    """Dummy implementation of _thread.get_ident().
+
+    Since this module should only be used when _threadmodule is not
+    available, it is safe to assume that the current process is the
+    only thread.  Thus a constant can be safely returned.
+    """
+    return -1
+
+def allocate_lock():
+    """Dummy implementation of _thread.allocate_lock()."""
+    return LockType()
+
+def stack_size(size=None):
+    """Dummy implementation of _thread.stack_size()."""
+    if size is not None:
+        raise error("setting thread stack size not supported")
+    return 0
+
+class LockType(object):
+    """Class implementing dummy implementation of _thread.LockType.
+
+    Compatibility is maintained by maintaining self.locked_status
+    which is a boolean that stores the state of the lock.  Pickling of
+    the lock, though, should not be done since if the _thread module is
+    then used with an unpickled ``lock()`` from here problems could
+    occur from this class not having atomic methods.
+
+    """
+
+    def __init__(self):
+        self.locked_status = False
+
+    def acquire(self, waitflag=None, timeout=-1):
+        """Dummy implementation of acquire().
+
+        For blocking calls, self.locked_status is automatically set to
+        True and returned appropriately based on value of
+        ``waitflag``.  If it is non-blocking, then the value is
+        actually checked and not set if it is already acquired.  This
+        is all done so that threading.Condition's assert statements
+        aren't triggered and throw a little fit.
+
+        """
+        if waitflag is None or waitflag:
+            self.locked_status = True
+            return True
+        else:
+            if not self.locked_status:
+                self.locked_status = True
+                return True
+            else:
+                if timeout > 0:
+                    import time
+                    time.sleep(timeout)
+                return False
+
+    __enter__ = acquire
+
+    def __exit__(self, typ, val, tb):
+        self.release()
+
+    def release(self):
+        """Release the dummy lock."""
+        # XXX Perhaps shouldn't actually bother to test?  Could lead
+        #     to problems for complex, threaded code.
+        if not self.locked_status:
+            raise error
+        self.locked_status = False
+        return True
+
+    def locked(self):
+        return self.locked_status
+
+# Used to signal that interrupt_main was called in a "thread"
+_interrupt = False
+# True when not executing in a "thread"
+_main = True
+
+def interrupt_main():
+    """Set _interrupt flag to True to have start_new_thread raise
+    KeyboardInterrupt upon exiting."""
+    if _main:
+        raise KeyboardInterrupt
+    else:
+        global _interrupt
+        _interrupt = True
diff --git a/lib-python/3.2/_markupbase.py b/lib-python/3.2/_markupbase.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/_markupbase.py
@@ -0,0 +1,391 @@
+"""Shared support for scanning document type declarations in HTML and XHTML.
+
+This module is used as a foundation for the html.parser module.  It has no
+documented public API and should not be used directly.
+
+"""
+
+import re
+
+_declname_match = re.compile(r'[a-zA-Z][-_.a-zA-Z0-9]*\s*').match
+_declstringlit_match = re.compile(r'(\'[^\']*\'|"[^"]*")\s*').match
+_commentclose = re.compile(r'--\s*>')
+_markedsectionclose = re.compile(r']\s*]\s*>')
+
+# An analysis of the MS-Word extensions is available at
+# http://www.planetpublish.com/xmlarena/xap/Thursday/WordtoXML.pdf
+
+_msmarkedsectionclose = re.compile(r']\s*>')
+
+del re
+
+
+class ParserBase:
+    """Parser base class which provides some common support methods used
+    by the SGML/HTML and XHTML parsers."""
+
+    def __init__(self):
+        if self.__class__ is ParserBase:
+            raise RuntimeError(
+                "_markupbase.ParserBase must be subclassed")
+
+    def error(self, message):
+        raise NotImplementedError(
+            "subclasses of ParserBase must override error()")
+
+    def reset(self):
+        self.lineno = 1
+        self.offset = 0
+
+    def getpos(self):
+        """Return current line number and offset."""
+        return self.lineno, self.offset
+
+    # Internal -- update line number and offset.  This should be
+    # called for each piece of data exactly once, in order -- in other
+    # words the concatenation of all the input strings to this
+    # function should be exactly the entire input.
+    def updatepos(self, i, j):
+        if i >= j:
+            return j
+        rawdata = self.rawdata
+        nlines = rawdata.count("\n", i, j)
+        if nlines:
+            self.lineno = self.lineno + nlines
+            pos = rawdata.rindex("\n", i, j) # Should not fail
+            self.offset = j-(pos+1)
+        else:
+            self.offset = self.offset + j-i
+        return j
+
+    _decl_otherchars = ''
+
+    # Internal -- parse declaration (for use by subclasses).
+    def parse_declaration(self, i):
+        # This is some sort of declaration; in "HTML as
+        # deployed," this should only be the document type
+        # declaration ("<!DOCTYPE html...>").
+        # ISO 8879:1986, however, has more complex
+        # declaration syntax for elements in <!...>, including:
+        # --comment--
+        # [marked section]
+        # name in the following list: ENTITY, DOCTYPE, ELEMENT,
+        # ATTLIST, NOTATION, SHORTREF, USEMAP,
+        # LINKTYPE, LINK, IDLINK, USELINK, SYSTEM
+        rawdata = self.rawdata
+        j = i + 2
+        assert rawdata[i:j] == "<!", "unexpected call to parse_declaration"
+        if rawdata[j:j+1] == ">":
+            # the empty comment <!>
+            return j + 1
+        if rawdata[j:j+1] in ("-", ""):
+            # Start of comment followed by buffer boundary,
+            # or just a buffer boundary.
+            return -1
+        # A simple, practical version could look like: ((name|stringlit) S*) + '>'
+        n = len(rawdata)
+        if rawdata[j:j+2] == '--': #comment
+            # Locate --.*-- as the body of the comment
+            return self.parse_comment(i)
+        elif rawdata[j] == '[': #marked section
+            # Locate [statusWord [...arbitrary SGML...]] as the body of the marked section
+            # Where statusWord is one of TEMP, CDATA, IGNORE, INCLUDE, RCDATA
+            # Note that this is extended by Microsoft Office "Save as Web" function
+            # to include [if...] and [endif].
+            return self.parse_marked_section(i)
+        else: #all other declaration elements
+            decltype, j = self._scan_name(j, i)
+        if j < 0:
+            return j
+        if decltype == "doctype":
+            self._decl_otherchars = ''
+        while j < n:
+            c = rawdata[j]
+            if c == ">":
+                # end of declaration syntax
+                data = rawdata[i+2:j]
+                if decltype == "doctype":
+                    self.handle_decl(data)
+                else:
+                    self.unknown_decl(data)
+                return j + 1
+            if c in "\"'":
+                m = _declstringlit_match(rawdata, j)
+                if not m:
+                    return -1 # incomplete
+                j = m.end()
+            elif c in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ":
+                name, j = self._scan_name(j, i)
+            elif c in self._decl_otherchars:
+                j = j + 1
+            elif c == "[":
+                # this could be handled in a separate doctype parser
+                if decltype == "doctype":
+                    j = self._parse_doctype_subset(j + 1, i)
+                elif decltype in {"attlist", "linktype", "link", "element"}:
+                    # must tolerate []'d groups in a content model in an element declaration
+                    # also in data attribute specifications of attlist declaration
+                    # also link type declaration subsets in linktype declarations
+                    # also link attribute specification lists in link declarations
+                    self.error("unsupported '[' char in %s declaration" % decltype)
+                else:
+                    self.error("unexpected '[' char in declaration")
+            else:
+                self.error(
+                    "unexpected %r char in declaration" % rawdata[j])
+            if j < 0:
+                return j
+        return -1 # incomplete
+
+    # Internal -- parse a marked section
+    # Override this to handle MS-word extension syntax <![if word]>content<![endif]>
+    def parse_marked_section(self, i, report=1):
+        rawdata= self.rawdata
+        assert rawdata[i:i+3] == '<![', "unexpected call to parse_marked_section()"
+        sectName, j = self._scan_name( i+3, i )
+        if j < 0:
+            return j
+        if sectName in {"temp", "cdata", "ignore", "include", "rcdata"}:
+            # look for standard ]]> ending
+            match= _markedsectionclose.search(rawdata, i+3)
+        elif sectName in {"if", "else", "endif"}:
+            # look for MS Office ]> ending
+            match= _msmarkedsectionclose.search(rawdata, i+3)
+        else:
+            self.error('unknown status keyword %r in marked section' % rawdata[i+3:j])
+        if not match:
+            return -1
+        if report:
+            j = match.start(0)
+            self.unknown_decl(rawdata[i+3: j])
+        return match.end(0)
+
+    # Internal -- parse comment, return length or -1 if not terminated
+    def parse_comment(self, i, report=1):
+        rawdata = self.rawdata
+        if rawdata[i:i+4] != '<!--':
+            self.error('unexpected call to parse_comment()')
+        match = _commentclose.search(rawdata, i+4)
+        if not match:
+            return -1
+        if report:
+            j = match.start(0)
+            self.handle_comment(rawdata[i+4: j])
+        return match.end(0)
+
+    # Internal -- scan past the internal subset in a <!DOCTYPE declaration,
+    # returning the index just past any whitespace following the trailing ']'.
+    def _parse_doctype_subset(self, i, declstartpos):
+        rawdata = self.rawdata
+        n = len(rawdata)
+        j = i
+        while j < n:
+            c = rawdata[j]
+            if c == "<":
+                s = rawdata[j:j+2]
+                if s == "<":
+                    # end of buffer; incomplete
+                    return -1
+                if s != "<!":
+                    self.updatepos(declstartpos, j + 1)
+                    self.error("unexpected char in internal subset (in %r)" % s)
+                if (j + 2) == n:
+                    # end of buffer; incomplete
+                    return -1
+                if (j + 4) > n:
+                    # end of buffer; incomplete
+                    return -1
+                if rawdata[j:j+4] == "<!--":
+                    j = self.parse_comment(j, report=0)
+                    if j < 0:
+                        return j
+                    continue
+                name, j = self._scan_name(j + 2, declstartpos)
+                if j == -1:
+                    return -1
+                if name not in {"attlist", "element", "entity", "notation"}:
+                    self.updatepos(declstartpos, j + 2)
+                    self.error(
+                        "unknown declaration %r in internal subset" % name)
+                # handle the individual names
+                meth = getattr(self, "_parse_doctype_" + name)
+                j = meth(j, declstartpos)
+                if j < 0:
+                    return j
+            elif c == "%":
+                # parameter entity reference
+                if (j + 1) == n:
+                    # end of buffer; incomplete
+                    return -1
+                s, j = self._scan_name(j + 1, declstartpos)
+                if j < 0:
+                    return j
+                if rawdata[j] == ";":
+                    j = j + 1
+            elif c == "]":
+                j = j + 1
+                while j < n and rawdata[j].isspace():
+                    j = j + 1
+                if j < n:
+                    if rawdata[j] == ">":
+                        return j
+                    self.updatepos(declstartpos, j)
+                    self.error("unexpected char after internal subset")
+                else:
+                    return -1
+            elif c.isspace():
+                j = j + 1
+            else:
+                self.updatepos(declstartpos, j)
+                self.error("unexpected char %r in internal subset" % c)
+        # end of buffer reached
+        return -1
+
+    # Internal -- scan past <!ELEMENT declarations
+    def _parse_doctype_element(self, i, declstartpos):
+        name, j = self._scan_name(i, declstartpos)
+        if j == -1:
+            return -1
+        # style content model; just skip until '>'
+        rawdata = self.rawdata
+        if '>' in rawdata[j:]:
+            return rawdata.find(">", j) + 1
+        return -1
+
+    # Internal -- scan past <!ATTLIST declarations
+    def _parse_doctype_attlist(self, i, declstartpos):
+        rawdata = self.rawdata
+        name, j = self._scan_name(i, declstartpos)
+        c = rawdata[j:j+1]
+        if c == "":
+            return -1
+        if c == ">":
+            return j + 1
+        while 1:
+            # scan a series of attribute descriptions; simplified:
+            #   name type [value] [#constraint]
+            name, j = self._scan_name(j, declstartpos)
+            if j < 0:
+                return j
+            c = rawdata[j:j+1]
+            if c == "":
+                return -1
+            if c == "(":
+                # an enumerated type; look for ')'
+                if ")" in rawdata[j:]:
+                    j = rawdata.find(")", j) + 1
+                else:
+                    return -1
+                while rawdata[j:j+1].isspace():
+                    j = j + 1
+                if not rawdata[j:]:
+                    # end of buffer, incomplete
+                    return -1
+            else:
+                name, j = self._scan_name(j, declstartpos)
+            c = rawdata[j:j+1]
+            if not c:
+                return -1
+            if c in "'\"":
+                m = _declstringlit_match(rawdata, j)
+                if m:
+                    j = m.end()
+                else:
+                    return -1
+                c = rawdata[j:j+1]
+                if not c:
+                    return -1
+            if c == "#":
+                if rawdata[j:] == "#":
+                    # end of buffer
+                    return -1
+                name, j = self._scan_name(j + 1, declstartpos)
+                if j < 0:
+                    return j
+                c = rawdata[j:j+1]
+                if not c:
+                    return -1
+            if c == '>':
+                # all done
+                return j + 1
+
+    # Internal -- scan past <!NOTATION declarations
+    def _parse_doctype_notation(self, i, declstartpos):
+        name, j = self._scan_name(i, declstartpos)
+        if j < 0:
+            return j
+        rawdata = self.rawdata
+        while 1:
+            c = rawdata[j:j+1]
+            if not c:
+                # end of buffer; incomplete
+                return -1
+            if c == '>':
+                return j + 1
+            if c in "'\"":
+                m = _declstringlit_match(rawdata, j)
+                if not m:
+                    return -1
+                j = m.end()
+            else:
+                name, j = self._scan_name(j, declstartpos)
+                if j < 0:
+                    return j
+
+    # Internal -- scan past <!ENTITY declarations
+    def _parse_doctype_entity(self, i, declstartpos):
+        rawdata = self.rawdata
+        if rawdata[i:i+1] == "%":
+            j = i + 1
+            while 1:
+                c = rawdata[j:j+1]
+                if not c:
+                    return -1
+                if c.isspace():
+                    j = j + 1
+                else:
+                    break
+        else:
+            j = i
+        name, j = self._scan_name(j, declstartpos)
+        if j < 0:
+            return j
+        while 1:
+            c = self.rawdata[j:j+1]
+            if not c:
+                return -1
+            if c in "'\"":
+                m = _declstringlit_match(rawdata, j)
+                if m:
+                    j = m.end()
+                else:
+                    return -1    # incomplete
+            elif c == ">":
+                return j + 1
+            else:
+                name, j = self._scan_name(j, declstartpos)
+                if j < 0:
+                    return j
+
+    # Internal -- scan a name token and the new position and the token, or
+    # return -1 if we've reached the end of the buffer.
+    def _scan_name(self, i, declstartpos):
+        rawdata = self.rawdata
+        n = len(rawdata)
+        if i == n:
+            return None, -1
+        m = _declname_match(rawdata, i)
+        if m:
+            s = m.group()
+            name = s.strip()
+            if (i + len(s)) == n:
+                return None, -1  # end of buffer
+            return name.lower(), m.end()
+        else:
+            self.updatepos(declstartpos, i)
+            self.error("expected name token at %r"
+                       % rawdata[declstartpos:declstartpos+20])
+
+    # To be overridden -- handlers for unknown objects
+    def unknown_decl(self, data):
+        pass
diff --git a/lib-python/3.2/_pyio.py b/lib-python/3.2/_pyio.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/_pyio.py
@@ -0,0 +1,2034 @@
+"""
+Python implementation of the io module.
+"""
+
+import os
+import abc
+import codecs
+import warnings
+# Import _thread instead of threading to reduce startup cost
+try:
+    from _thread import allocate_lock as Lock
+except ImportError:
+    from _dummy_thread import allocate_lock as Lock
+
+import io
+from io import (__all__, SEEK_SET, SEEK_CUR, SEEK_END)
+from errno import EINTR
+
+# open() uses st_blksize whenever we can
+DEFAULT_BUFFER_SIZE = 8 * 1024  # bytes
+
+# NOTE: Base classes defined here are registered with the "official" ABCs
+# defined in io.py. We don't use real inheritance though, because we don't
+# want to inherit the C implementations.
+
+
+class BlockingIOError(IOError):
+
+    """Exception raised when I/O would block on a non-blocking I/O stream."""
+
+    def __init__(self, errno, strerror, characters_written=0):
+        super().__init__(errno, strerror)
+        if not isinstance(characters_written, int):
+            raise TypeError("characters_written must be a integer")
+        self.characters_written = characters_written
+
+
+def open(file, mode="r", buffering=-1, encoding=None, errors=None,
+         newline=None, closefd=True):
+
+    r"""Open file and return a stream.  Raise IOError upon failure.
+
+    file is either a text or byte string giving the name (and the path
+    if the file isn't in the current working directory) of the file to
+    be opened or an integer file descriptor of the file to be
+    wrapped. (If a file descriptor is given, it is closed when the
+    returned I/O object is closed, unless closefd is set to False.)
+
+    mode is an optional string that specifies the mode in which the file
+    is opened. It defaults to 'r' which means open for reading in text
+    mode.  Other common values are 'w' for writing (truncating the file if
+    it already exists), and 'a' for appending (which on some Unix systems,
+    means that all writes append to the end of the file regardless of the
+    current seek position). In text mode, if encoding is not specified the
+    encoding used is platform dependent. (For reading and writing raw
+    bytes use binary mode and leave encoding unspecified.) The available
+    modes are:
+
+    ========= ===============================================================
+    Character Meaning
+    --------- ---------------------------------------------------------------
+    'r'       open for reading (default)
+    'w'       open for writing, truncating the file first
+    'a'       open for writing, appending to the end of the file if it exists
+    'b'       binary mode
+    't'       text mode (default)
+    '+'       open a disk file for updating (reading and writing)
+    'U'       universal newline mode (for backwards compatibility; unneeded
+              for new code)
+    ========= ===============================================================
+
+    The default mode is 'rt' (open for reading text). For binary random
+    access, the mode 'w+b' opens and truncates the file to 0 bytes, while
+    'r+b' opens the file without truncation.
+
+    Python distinguishes between files opened in binary and text modes,
+    even when the underlying operating system doesn't. Files opened in
+    binary mode (appending 'b' to the mode argument) return contents as
+    bytes objects without any decoding. In text mode (the default, or when
+    't' is appended to the mode argument), the contents of the file are
+    returned as strings, the bytes having been first decoded using a
+    platform-dependent encoding or using the specified encoding if given.
+
+    buffering is an optional integer used to set the buffering policy.
+    Pass 0 to switch buffering off (only allowed in binary mode), 1 to select
+    line buffering (only usable in text mode), and an integer > 1 to indicate
+    the size of a fixed-size chunk buffer.  When no buffering argument is
+    given, the default buffering policy works as follows:
+
+    * Binary files are buffered in fixed-size chunks; the size of the buffer
+      is chosen using a heuristic trying to determine the underlying device's
+      "block size" and falling back on `io.DEFAULT_BUFFER_SIZE`.
+      On many systems, the buffer will typically be 4096 or 8192 bytes long.
+
+    * "Interactive" text files (files for which isatty() returns True)
+      use line buffering.  Other text files use the policy described above
+      for binary files.
+
+    encoding is the str name of the encoding used to decode or encode the
+    file. This should only be used in text mode. The default encoding is
+    platform dependent, but any encoding supported by Python can be
+    passed.  See the codecs module for the list of supported encodings.
+
+    errors is an optional string that specifies how encoding errors are to
+    be handled---this argument should not be used in binary mode. Pass
+    'strict' to raise a ValueError exception if there is an encoding error
+    (the default of None has the same effect), or pass 'ignore' to ignore
+    errors. (Note that ignoring encoding errors can lead to data loss.)
+    See the documentation for codecs.register for a list of the permitted
+    encoding error strings.
+
+    newline is a string controlling how universal newlines works (it only
+    applies to text mode). It can be None, '', '\n', '\r', and '\r\n'.  It works
+    as follows:
+
+    * On input, if newline is None, universal newlines mode is
+      enabled. Lines in the input can end in '\n', '\r', or '\r\n', and
+      these are translated into '\n' before being returned to the
+      caller. If it is '', universal newline mode is enabled, but line
+      endings are returned to the caller untranslated. If it has any of
+      the other legal values, input lines are only terminated by the given
+      string, and the line ending is returned to the caller untranslated.
+
+    * On output, if newline is None, any '\n' characters written are
+      translated to the system default line separator, os.linesep. If
+      newline is '', no translation takes place. If newline is any of the
+      other legal values, any '\n' characters written are translated to
+      the given string.
+
+    closedfd is a bool. If closefd is False, the underlying file descriptor will
+    be kept open when the file is closed. This does not work when a file name is
+    given and must be True in that case.
+
+    open() returns a file object whose type depends on the mode, and
+    through which the standard file operations such as reading and writing
+    are performed. When open() is used to open a file in a text mode ('w',
+    'r', 'wt', 'rt', etc.), it returns a TextIOWrapper. When used to open
+    a file in a binary mode, the returned class varies: in read binary
+    mode, it returns a BufferedReader; in write binary and append binary
+    modes, it returns a BufferedWriter, and in read/write mode, it returns
+    a BufferedRandom.
+
+    It is also possible to use a string or bytearray as a file for both
+    reading and writing. For strings StringIO can be used like a file
+    opened in a text mode, and for bytes a BytesIO can be used like a file
+    opened in a binary mode.
+    """
+    if not isinstance(file, (str, bytes, int)):
+        raise TypeError("invalid file: %r" % file)
+    if not isinstance(mode, str):
+        raise TypeError("invalid mode: %r" % mode)
+    if not isinstance(buffering, int):
+        raise TypeError("invalid buffering: %r" % buffering)
+    if encoding is not None and not isinstance(encoding, str):
+        raise TypeError("invalid encoding: %r" % encoding)
+    if errors is not None and not isinstance(errors, str):
+        raise TypeError("invalid errors: %r" % errors)
+    modes = set(mode)
+    if modes - set("arwb+tU") or len(mode) > len(modes):
+        raise ValueError("invalid mode: %r" % mode)
+    reading = "r" in modes
+    writing = "w" in modes
+    appending = "a" in modes
+    updating = "+" in modes
+    text = "t" in modes
+    binary = "b" in modes
+    if "U" in modes:
+        if writing or appending:
+            raise ValueError("can't use U and writing mode at once")
+        reading = True
+    if text and binary:
+        raise ValueError("can't have text and binary mode at once")
+    if reading + writing + appending > 1:
+        raise ValueError("can't have read/write/append mode at once")
+    if not (reading or writing or appending):
+        raise ValueError("must have exactly one of read/write/append mode")
+    if binary and encoding is not None:
+        raise ValueError("binary mode doesn't take an encoding argument")
+    if binary and errors is not None:
+        raise ValueError("binary mode doesn't take an errors argument")
+    if binary and newline is not None:
+        raise ValueError("binary mode doesn't take a newline argument")
+    raw = FileIO(file,
+                 (reading and "r" or "") +
+                 (writing and "w" or "") +
+                 (appending and "a" or "") +
+                 (updating and "+" or ""),
+                 closefd)
+    line_buffering = False
+    if buffering == 1 or buffering < 0 and raw.isatty():
+        buffering = -1
+        line_buffering = True
+    if buffering < 0:
+        buffering = DEFAULT_BUFFER_SIZE
+        try:
+            bs = os.fstat(raw.fileno()).st_blksize
+        except (os.error, AttributeError):
+            pass
+        else:
+            if bs > 1:
+                buffering = bs
+    if buffering < 0:
+        raise ValueError("invalid buffering size")
+    if buffering == 0:
+        if binary:
+            return raw
+        raise ValueError("can't have unbuffered text I/O")
+    if updating:
+        buffer = BufferedRandom(raw, buffering)
+    elif writing or appending:
+        buffer = BufferedWriter(raw, buffering)
+    elif reading:
+        buffer = BufferedReader(raw, buffering)
+    else:
+        raise ValueError("unknown mode: %r" % mode)
+    if binary:
+        return buffer
+    text = TextIOWrapper(buffer, encoding, errors, newline, line_buffering)
+    text.mode = mode
+    return text
+
+
+class DocDescriptor:
+    """Helper for builtins.open.__doc__
+    """
+    def __get__(self, obj, typ):
+        return (
+            "open(file, mode='r', buffering=-1, encoding=None, "
+                 "errors=None, newline=None, closefd=True)\n\n" +
+            open.__doc__)
+
+class OpenWrapper:
+    """Wrapper for builtins.open
+
+    Trick so that open won't become a bound method when stored
+    as a class variable (as dbm.dumb does).
+
+    See initstdio() in Python/pythonrun.c.
+    """
+    __doc__ = DocDescriptor()
+
+    def __new__(cls, *args, **kwargs):
+        return open(*args, **kwargs)
+
+
+# In normal operation, both `UnsupportedOperation`s should be bound to the
+# same object.
+try:
+    UnsupportedOperation = io.UnsupportedOperation
+except AttributeError:
+    class UnsupportedOperation(ValueError, IOError):
+        pass
+
+
+class IOBase(metaclass=abc.ABCMeta):
+
+    """The abstract base class for all I/O classes, acting on streams of
+    bytes. There is no public constructor.
+
+    This class provides dummy implementations for many methods that
+    derived classes can override selectively; the default implementations
+    represent a file that cannot be read, written or seeked.
+
+    Even though IOBase does not declare read, readinto, or write because
+    their signatures will vary, implementations and clients should
+    consider those methods part of the interface. Also, implementations
+    may raise UnsupportedOperation when operations they do not support are
+    called.
+
+    The basic type used for binary data read from or written to a file is
+    bytes. bytearrays are accepted too, and in some cases (such as
+    readinto) needed. Text I/O classes work with str data.
+
+    Note that calling any method (even inquiries) on a closed stream is
+    undefined. Implementations may raise IOError in this case.
+
+    IOBase (and its subclasses) support the iterator protocol, meaning
+    that an IOBase object can be iterated over yielding the lines in a
+    stream.
+
+    IOBase also supports the :keyword:`with` statement. In this example,
+    fp is closed after the suite of the with statement is complete:
+
+    with open('spam.txt', 'r') as fp:
+        fp.write('Spam and eggs!')
+    """
+
+    ### Internal ###
+
+    def _unsupported(self, name):
+        """Internal: raise an IOError exception for unsupported operations."""
+        raise UnsupportedOperation("%s.%s() not supported" %
+                                   (self.__class__.__name__, name))
+
+    ### Positioning ###
+
+    def seek(self, pos, whence=0):
+        """Change stream position.
+
+        Change the stream position to byte offset offset. offset is
+        interpreted relative to the position indicated by whence.  Values
+        for whence are ints:
+
+        * 0 -- start of stream (the default); offset should be zero or positive
+        * 1 -- current stream position; offset may be negative
+        * 2 -- end of stream; offset is usually negative
+
+        Return an int indicating the new absolute position.
+        """
+        self._unsupported("seek")
+
+    def tell(self):
+        """Return an int indicating the current stream position."""
+        return self.seek(0, 1)
+
+    def truncate(self, pos=None):
+        """Truncate file to size bytes.
+
+        Size defaults to the current IO position as reported by tell().  Return
+        the new size.
+        """
+        self._unsupported("truncate")
+
+    ### Flush and close ###
+
+    def flush(self):
+        """Flush write buffers, if applicable.
+
+        This is not implemented for read-only and non-blocking streams.
+        """
+        self._checkClosed()
+        # XXX Should this return the number of bytes written???
+
+    __closed = False
+
+    def close(self):
+        """Flush and close the IO object.
+
+        This method has no effect if the file is already closed.
+        """
+        if not self.__closed:
+            self.flush()
+            self.__closed = True
+
+    def __del__(self):
+        """Destructor.  Calls close()."""
+        # The try/except block is in case this is called at program
+        # exit time, when it's possible that globals have already been
+        # deleted, and then the close() call might fail.  Since
+        # there's nothing we can do about such failures and they annoy
+        # the end users, we suppress the traceback.
+        try:
+            self.close()
+        except:
+            pass
+
+    ### Inquiries ###
+
+    def seekable(self):
+        """Return a bool indicating whether object supports random access.
+
+        If False, seek(), tell() and truncate() will raise UnsupportedOperation.
+        This method may need to do a test seek().
+        """
+        return False
+
+    def _checkSeekable(self, msg=None):
+        """Internal: raise UnsupportedOperation if file is not seekable
+        """
+        if not self.seekable():
+            raise UnsupportedOperation("File or stream is not seekable."
+                                       if msg is None else msg)
+
+    def readable(self):
+        """Return a bool indicating whether object was opened for reading.
+
+        If False, read() will raise UnsupportedOperation.
+        """
+        return False
+
+    def _checkReadable(self, msg=None):
+        """Internal: raise UnsupportedOperation if file is not readable
+        """
+        if not self.readable():
+            raise UnsupportedOperation("File or stream is not readable."
+                                       if msg is None else msg)
+
+    def writable(self):
+        """Return a bool indicating whether object was opened for writing.
+
+        If False, write() and truncate() will raise UnsupportedOperation.
+        """
+        return False
+
+    def _checkWritable(self, msg=None):
+        """Internal: raise UnsupportedOperation if file is not writable
+        """
+        if not self.writable():
+            raise UnsupportedOperation("File or stream is not writable."
+                                       if msg is None else msg)
+
+    @property
+    def closed(self):
+        """closed: bool.  True iff the file has been closed.
+
+        For backwards compatibility, this is a property, not a predicate.
+        """
+        return self.__closed
+
+    def _checkClosed(self, msg=None):
+        """Internal: raise an ValueError if file is closed
+        """
+        if self.closed:
+            raise ValueError("I/O operation on closed file."
+                             if msg is None else msg)
+
+    ### Context manager ###
+
+    def __enter__(self):  # That's a forward reference
+        """Context management protocol.  Returns self (an instance of IOBase)."""
+        self._checkClosed()
+        return self
+
+    def __exit__(self, *args):
+        """Context management protocol.  Calls close()"""
+        self.close()
+
+    ### Lower-level APIs ###
+
+    # XXX Should these be present even if unimplemented?
+
+    def fileno(self):
+        """Returns underlying file descriptor (an int) if one exists.
+
+        An IOError is raised if the IO object does not use a file descriptor.
+        """
+        self._unsupported("fileno")
+
+    def isatty(self):
+        """Return a bool indicating whether this is an 'interactive' stream.
+
+        Return False if it can't be determined.
+        """
+        self._checkClosed()
+        return False
+
+    ### Readline[s] and writelines ###
+
+    def readline(self, limit=-1):
+        r"""Read and return a line of bytes from the stream.
+
+        If limit is specified, at most limit bytes will be read.
+        Limit should be an int.
+
+        The line terminator is always b'\n' for binary files; for text
+        files, the newlines argument to open can be used to select the line
+        terminator(s) recognized.
+        """
+        # For backwards compatibility, a (slowish) readline().
+        if hasattr(self, "peek"):
+            def nreadahead():
+                readahead = self.peek(1)
+                if not readahead:
+                    return 1
+                n = (readahead.find(b"\n") + 1) or len(readahead)
+                if limit >= 0:
+                    n = min(n, limit)
+                return n
+        else:
+            def nreadahead():
+                return 1
+        if limit is None:
+            limit = -1
+        elif not isinstance(limit, int):
+            raise TypeError("limit must be an integer")
+        res = bytearray()
+        while limit < 0 or len(res) < limit:
+            b = self.read(nreadahead())
+            if not b:
+                break
+            res += b
+            if res.endswith(b"\n"):
+                break
+        return bytes(res)
+
+    def __iter__(self):
+        self._checkClosed()
+        return self
+
+    def __next__(self):
+        line = self.readline()
+        if not line:
+            raise StopIteration
+        return line
+
+    def readlines(self, hint=None):
+        """Return a list of lines from the stream.
+
+        hint can be specified to control the number of lines read: no more
+        lines will be read if the total size (in bytes/characters) of all
+        lines so far exceeds hint.
+        """
+        if hint is None or hint <= 0:
+            return list(self)
+        n = 0
+        lines = []
+        for line in self:
+            lines.append(line)
+            n += len(line)
+            if n >= hint:
+                break
+        return lines
+
+    def writelines(self, lines):
+        self._checkClosed()
+        for line in lines:
+            self.write(line)
+
+io.IOBase.register(IOBase)
+
+
+class RawIOBase(IOBase):
+
+    """Base class for raw binary I/O."""
+
+    # The read() method is implemented by calling readinto(); derived
+    # classes that want to support read() only need to implement
+    # readinto() as a primitive operation.  In general, readinto() can be
+    # more efficient than read().
+
+    # (It would be tempting to also provide an implementation of
+    # readinto() in terms of read(), in case the latter is a more suitable
+    # primitive operation, but that would lead to nasty recursion in case
+    # a subclass doesn't implement either.)
+
+    def read(self, n=-1):
+        """Read and return up to n bytes, where n is an int.
+
+        Returns an empty bytes object on EOF, or None if the object is
+        set not to block and has no data to read.
+        """
+        if n is None:
+            n = -1
+        if n < 0:
+            return self.readall()
+        b = bytearray(n.__index__())
+        n = self.readinto(b)
+        if n is None:
+            return None
+        del b[n:]
+        return bytes(b)
+
+    def readall(self):
+        """Read until EOF, using multiple read() call."""
+        res = bytearray()
+        while True:
+            data = self.read(DEFAULT_BUFFER_SIZE)
+            if not data:
+                break
+            res += data
+        if res:
+            return bytes(res)
+        else:
+            # b'' or None
+            return data
+
+    def readinto(self, b):
+        """Read up to len(b) bytes into bytearray b.
+
+        Returns an int representing the number of bytes read (0 for EOF), or
+        None if the object is set not to block and has no data to read.
+        """
+        self._unsupported("readinto")
+
+    def write(self, b):
+        """Write the given buffer to the IO stream.
+
+        Returns the number of bytes written, which may be less than len(b).
+        """
+        self._unsupported("write")
+
+io.RawIOBase.register(RawIOBase)
+from _io import FileIO
+RawIOBase.register(FileIO)
+
+
+class BufferedIOBase(IOBase):
+
+    """Base class for buffered IO objects.
+
+    The main difference with RawIOBase is that the read() method
+    supports omitting the size argument, and does not have a default
+    implementation that defers to readinto().
+
+    In addition, read(), readinto() and write() may raise
+    BlockingIOError if the underlying raw stream is in non-blocking
+    mode and not ready; unlike their raw counterparts, they will never
+    return None.
+
+    A typical implementation should not inherit from a RawIOBase
+    implementation, but wrap one.
+    """
+
+    def read(self, n=None):
+        """Read and return up to n bytes, where n is an int.
+
+        If the argument is omitted, None, or negative, reads and
+        returns all data until EOF.
+
+        If the argument is positive, and the underlying raw stream is
+        not 'interactive', multiple raw reads may be issued to satisfy
+        the byte count (unless EOF is reached first).  But for
+        interactive raw streams (XXX and for pipes?), at most one raw
+        read will be issued, and a short result does not imply that
+        EOF is imminent.
+
+        Returns an empty bytes array on EOF.
+
+        Raises BlockingIOError if the underlying raw stream has no
+        data at the moment.
+        """
+        self._unsupported("read")
+
+    def read1(self, n=None):
+        """Read up to n bytes with at most one read() system call,
+        where n is an int.
+        """
+        self._unsupported("read1")
+
+    def readinto(self, b):
+        """Read up to len(b) bytes into bytearray b.
+
+        Like read(), this may issue multiple reads to the underlying raw
+        stream, unless the latter is 'interactive'.
+
+        Returns an int representing the number of bytes read (0 for EOF).
+
+        Raises BlockingIOError if the underlying raw stream has no
+        data at the moment.
+        """
+        # XXX This ought to work with anything that supports the buffer API
+        data = self.read(len(b))
+        n = len(data)
+        try:
+            b[:n] = data
+        except TypeError as err:
+            import array
+            if not isinstance(b, array.array):
+                raise err
+            b[:n] = array.array('b', data)
+        return n
+
+    def write(self, b):
+        """Write the given bytes buffer to the IO stream.
+
+        Return the number of bytes written, which is never less than
+        len(b).
+
+        Raises BlockingIOError if the buffer is full and the
+        underlying raw stream cannot accept more data at the moment.
+        """
+        self._unsupported("write")
+
+    def detach(self):
+        """
+        Separate the underlying raw stream from the buffer and return it.
+
+        After the raw stream has been detached, the buffer is in an unusable
+        state.
+        """
+        self._unsupported("detach")
+
+io.BufferedIOBase.register(BufferedIOBase)
+
+
+class _BufferedIOMixin(BufferedIOBase):
+
+    """A mixin implementation of BufferedIOBase with an underlying raw stream.
+
+    This passes most requests on to the underlying raw stream.  It
+    does *not* provide implementations of read(), readinto() or
+    write().
+    """
+
+    def __init__(self, raw):
+        self._raw = raw
+
+    ### Positioning ###
+
+    def seek(self, pos, whence=0):
+        new_position = self.raw.seek(pos, whence)
+        if new_position < 0:
+            raise IOError("seek() returned an invalid position")
+        return new_position
+
+    def tell(self):
+        pos = self.raw.tell()
+        if pos < 0:
+            raise IOError("tell() returned an invalid position")
+        return pos
+
+    def truncate(self, pos=None):
+        # Flush the stream.  We're mixing buffered I/O with lower-level I/O,
+        # and a flush may be necessary to synch both views of the current
+        # file state.
+        self.flush()
+
+        if pos is None:
+            pos = self.tell()
+        # XXX: Should seek() be used, instead of passing the position
+        # XXX  directly to truncate?
+        return self.raw.truncate(pos)
+
+    ### Flush and close ###
+
+    def flush(self):
+        if self.closed:
+            raise ValueError("flush of closed file")
+        self.raw.flush()
+
+    def close(self):
+        if self.raw is not None and not self.closed:
+            self.flush()
+            self.raw.close()
+
+    def detach(self):
+        if self.raw is None:
+            raise ValueError("raw stream already detached")
+        self.flush()
+        raw = self._raw
+        self._raw = None
+        return raw
+
+    ### Inquiries ###
+
+    def seekable(self):
+        return self.raw.seekable()
+
+    def readable(self):
+        return self.raw.readable()
+
+    def writable(self):
+        return self.raw.writable()
+
+    @property
+    def raw(self):
+        return self._raw
+
+    @property
+    def closed(self):
+        return self.raw.closed
+
+    @property
+    def name(self):
+        return self.raw.name
+
+    @property
+    def mode(self):
+        return self.raw.mode
+
+    def __getstate__(self):
+        raise TypeError("can not serialize a '{0}' object"
+                        .format(self.__class__.__name__))
+
+    def __repr__(self):
+        clsname = self.__class__.__name__
+        try:
+            name = self.name
+        except AttributeError:
+            return "<_pyio.{0}>".format(clsname)
+        else:
+            return "<_pyio.{0} name={1!r}>".format(clsname, name)
+
+    ### Lower-level APIs ###
+
+    def fileno(self):
+        return self.raw.fileno()
+
+    def isatty(self):
+        return self.raw.isatty()
+
+
+class BytesIO(BufferedIOBase):
+
+    """Buffered I/O implementation using an in-memory bytes buffer."""
+
+    def __init__(self, initial_bytes=None):
+        buf = bytearray()
+        if initial_bytes is not None:
+            buf += initial_bytes
+        self._buffer = buf
+        self._pos = 0
+
+    def __getstate__(self):
+        if self.closed:
+            raise ValueError("__getstate__ on closed file")
+        return self.__dict__.copy()
+
+    def getvalue(self):
+        """Return the bytes value (contents) of the buffer
+        """
+        if self.closed:
+            raise ValueError("getvalue on closed file")
+        return bytes(self._buffer)
+
+    def getbuffer(self):
+        """Return a readable and writable view of the buffer.
+        """
+        return memoryview(self._buffer)
+
+    def read(self, n=None):
+        if self.closed:
+            raise ValueError("read from closed file")
+        if n is None:
+            n = -1
+        if n < 0:
+            n = len(self._buffer)
+        if len(self._buffer) <= self._pos:
+            return b""
+        newpos = min(len(self._buffer), self._pos + n)
+        b = self._buffer[self._pos : newpos]
+        self._pos = newpos
+        return bytes(b)
+
+    def read1(self, n):
+        """This is the same as read.
+        """
+        return self.read(n)
+
+    def write(self, b):
+        if self.closed:
+            raise ValueError("write to closed file")
+        if isinstance(b, str):
+            raise TypeError("can't write str to binary stream")
+        n = len(b)
+        if n == 0:
+            return 0
+        pos = self._pos
+        if pos > len(self._buffer):
+            # Inserts null bytes between the current end of the file
+            # and the new write position.
+            padding = b'\x00' * (pos - len(self._buffer))
+            self._buffer += padding
+        self._buffer[pos:pos + n] = b
+        self._pos += n
+        return n
+
+    def seek(self, pos, whence=0):
+        if self.closed:
+            raise ValueError("seek on closed file")
+        try:
+            pos.__index__
+        except AttributeError as err:
+            raise TypeError("an integer is required") from err
+        if whence == 0:
+            if pos < 0:
+                raise ValueError("negative seek position %r" % (pos,))
+            self._pos = pos
+        elif whence == 1:
+            self._pos = max(0, self._pos + pos)
+        elif whence == 2:
+            self._pos = max(0, len(self._buffer) + pos)
+        else:
+            raise ValueError("invalid whence value")
+        return self._pos
+
+    def tell(self):
+        if self.closed:
+            raise ValueError("tell on closed file")
+        return self._pos
+
+    def truncate(self, pos=None):
+        if self.closed:
+            raise ValueError("truncate on closed file")
+        if pos is None:
+            pos = self._pos
+        else:
+            try:
+                pos.__index__
+            except AttributeError as err:
+                raise TypeError("an integer is required") from err
+            if pos < 0:
+                raise ValueError("negative truncate position %r" % (pos,))
+        del self._buffer[pos:]
+        return pos
+
+    def readable(self):
+        return True
+
+    def writable(self):
+        return True
+
+    def seekable(self):
+        return True
+
+
+class BufferedReader(_BufferedIOMixin):
+
+    """BufferedReader(raw[, buffer_size])
+
+    A buffer for a readable, sequential BaseRawIO object.
+
+    The constructor creates a BufferedReader for the given readable raw
+    stream and buffer_size. If buffer_size is omitted, DEFAULT_BUFFER_SIZE
+    is used.
+    """
+
+    def __init__(self, raw, buffer_size=DEFAULT_BUFFER_SIZE):
+        """Create a new buffered reader using the given readable raw IO object.
+        """
+        if not raw.readable():
+            raise IOError('"raw" argument must be readable.')
+
+        _BufferedIOMixin.__init__(self, raw)
+        if buffer_size <= 0:
+            raise ValueError("invalid buffer size")
+        self.buffer_size = buffer_size
+        self._reset_read_buf()
+        self._read_lock = Lock()
+
+    def _reset_read_buf(self):
+        self._read_buf = b""
+        self._read_pos = 0
+
+    def read(self, n=None):
+        """Read n bytes.
+
+        Returns exactly n bytes of data unless the underlying raw IO
+        stream reaches EOF or if the call would block in non-blocking
+        mode. If n is negative, read until EOF or until read() would
+        block.
+        """
+        if n is not None and n < -1:
+            raise ValueError("invalid number of bytes to read")
+        with self._read_lock:
+            return self._read_unlocked(n)
+
+    def _read_unlocked(self, n=None):
+        nodata_val = b""
+        empty_values = (b"", None)
+        buf = self._read_buf
+        pos = self._read_pos
+
+        # Special case for when the number of bytes to read is unspecified.
+        if n is None or n == -1:
+            self._reset_read_buf()
+            chunks = [buf[pos:]]  # Strip the consumed bytes.
+            current_size = 0
+            while True:
+                # Read until EOF or until read() would block.
+                try:
+                    chunk = self.raw.read()
+                except IOError as e:
+                    if e.errno != EINTR:
+                        raise
+                    continue
+                if chunk in empty_values:
+                    nodata_val = chunk
+                    break
+                current_size += len(chunk)
+                chunks.append(chunk)
+            return b"".join(chunks) or nodata_val
+
+        # The number of bytes to read is specified, return at most n bytes.
+        avail = len(buf) - pos  # Length of the available buffered data.
+        if n <= avail:
+            # Fast path: the data to read is fully buffered.
+            self._read_pos += n
+            return buf[pos:pos+n]
+        # Slow path: read from the stream until enough bytes are read,
+        # or until an EOF occurs or until read() would block.
+        chunks = [buf[pos:]]
+        wanted = max(self.buffer_size, n)
+        while avail < n:
+            try:
+                chunk = self.raw.read(wanted)
+            except IOError as e:
+                if e.errno != EINTR:
+                    raise
+                continue
+            if chunk in empty_values:
+                nodata_val = chunk
+                break
+            avail += len(chunk)
+            chunks.append(chunk)
+        # n is more then avail only when an EOF occurred or when
+        # read() would have blocked.
+        n = min(n, avail)
+        out = b"".join(chunks)
+        self._read_buf = out[n:]  # Save the extra data in the buffer.
+        self._read_pos = 0
+        return out[:n] if out else nodata_val
+
+    def peek(self, n=0):
+        """Returns buffered bytes without advancing the position.
+
+        The argument indicates a desired minimal number of bytes; we
+        do at most one raw read to satisfy it.  We never return more
+        than self.buffer_size.
+        """
+        with self._read_lock:
+            return self._peek_unlocked(n)
+
+    def _peek_unlocked(self, n=0):
+        want = min(n, self.buffer_size)
+        have = len(self._read_buf) - self._read_pos
+        if have < want or have <= 0:
+            to_read = self.buffer_size - have
+            while True:
+                try:
+                    current = self.raw.read(to_read)
+                except IOError as e:
+                    if e.errno != EINTR:
+                        raise
+                    continue
+                break
+            if current:
+                self._read_buf = self._read_buf[self._read_pos:] + current
+                self._read_pos = 0
+        return self._read_buf[self._read_pos:]
+
+    def read1(self, n):
+        """Reads up to n bytes, with at most one read() system call."""
+        # Returns up to n bytes.  If at least one byte is buffered, we
+        # only return buffered bytes.  Otherwise, we do one raw read.
+        if n < 0:
+            raise ValueError("number of bytes to read must be positive")
+        if n == 0:
+            return b""
+        with self._read_lock:
+            self._peek_unlocked(1)
+            return self._read_unlocked(
+                min(n, len(self._read_buf) - self._read_pos))
+
+    def tell(self):
+        return _BufferedIOMixin.tell(self) - len(self._read_buf) + self._read_pos
+
+    def seek(self, pos, whence=0):
+        if not (0 <= whence <= 2):
+            raise ValueError("invalid whence value")
+        with self._read_lock:
+            if whence == 1:
+                pos -= len(self._read_buf) - self._read_pos
+            pos = _BufferedIOMixin.seek(self, pos, whence)
+            self._reset_read_buf()
+            return pos
+
+class BufferedWriter(_BufferedIOMixin):
+
+    """A buffer for a writeable sequential RawIO object.
+
+    The constructor creates a BufferedWriter for the given writeable raw
+    stream. If the buffer_size is not given, it defaults to
+    DEFAULT_BUFFER_SIZE.
+    """
+
+    _warning_stack_offset = 2
+
+    def __init__(self, raw,
+                 buffer_size=DEFAULT_BUFFER_SIZE, max_buffer_size=None):
+        if not raw.writable():
+            raise IOError('"raw" argument must be writable.')
+
+        _BufferedIOMixin.__init__(self, raw)
+        if buffer_size <= 0:
+            raise ValueError("invalid buffer size")
+        if max_buffer_size is not None:
+            warnings.warn("max_buffer_size is deprecated", DeprecationWarning,
+                          self._warning_stack_offset)
+        self.buffer_size = buffer_size
+        self._write_buf = bytearray()
+        self._write_lock = Lock()
+
+    def write(self, b):
+        if self.closed:
+            raise ValueError("write to closed file")
+        if isinstance(b, str):
+            raise TypeError("can't write str to binary stream")
+        with self._write_lock:
+            # XXX we can implement some more tricks to try and avoid
+            # partial writes
+            if len(self._write_buf) > self.buffer_size:
+                # We're full, so let's pre-flush the buffer
+                try:
+                    self._flush_unlocked()
+                except BlockingIOError as e:
+                    # We can't accept anything else.
+                    # XXX Why not just let the exception pass through?
+                    raise BlockingIOError(e.errno, e.strerror, 0)
+            before = len(self._write_buf)
+            self._write_buf.extend(b)
+            written = len(self._write_buf) - before
+            if len(self._write_buf) > self.buffer_size:
+                try:
+                    self._flush_unlocked()
+                except BlockingIOError as e:
+                    if len(self._write_buf) > self.buffer_size:
+                        # We've hit the buffer_size. We have to accept a partial
+                        # write and cut back our buffer.
+                        overage = len(self._write_buf) - self.buffer_size
+                        written -= overage
+                        self._write_buf = self._write_buf[:self.buffer_size]
+                        raise BlockingIOError(e.errno, e.strerror, written)
+            return written
+
+    def truncate(self, pos=None):
+        with self._write_lock:
+            self._flush_unlocked()
+            if pos is None:
+                pos = self.raw.tell()
+            return self.raw.truncate(pos)
+
+    def flush(self):
+        with self._write_lock:
+            self._flush_unlocked()
+
+    def _flush_unlocked(self):
+        if self.closed:
+            raise ValueError("flush of closed file")
+        written = 0
+        try:
+            while self._write_buf:
+                try:
+                    n = self.raw.write(self._write_buf)
+                except IOError as e:
+                    if e.errno != EINTR:
+                        raise
+                    continue
+                if n > len(self._write_buf) or n < 0:
+                    raise IOError("write() returned incorrect number of bytes")
+                del self._write_buf[:n]
+                written += n
+        except BlockingIOError as e:
+            n = e.characters_written
+            del self._write_buf[:n]
+            written += n
+            raise BlockingIOError(e.errno, e.strerror, written)
+
+    def tell(self):
+        return _BufferedIOMixin.tell(self) + len(self._write_buf)
+
+    def seek(self, pos, whence=0):
+        if not (0 <= whence <= 2):
+            raise ValueError("invalid whence")
+        with self._write_lock:
+            self._flush_unlocked()
+            return _BufferedIOMixin.seek(self, pos, whence)
+
+
+class BufferedRWPair(BufferedIOBase):
+
+    """A buffered reader and writer object together.
+
+    A buffered reader object and buffered writer object put together to
+    form a sequential IO object that can read and write. This is typically
+    used with a socket or two-way pipe.
+
+    reader and writer are RawIOBase objects that are readable and
+    writeable respectively. If the buffer_size is omitted it defaults to
+    DEFAULT_BUFFER_SIZE.
+    """
+
+    # XXX The usefulness of this (compared to having two separate IO
+    # objects) is questionable.
+
+    def __init__(self, reader, writer,
+                 buffer_size=DEFAULT_BUFFER_SIZE, max_buffer_size=None):
+        """Constructor.
+
+        The arguments are two RawIO instances.
+        """
+        if max_buffer_size is not None:
+            warnings.warn("max_buffer_size is deprecated", DeprecationWarning, 2)
+
+        if not reader.readable():
+            raise IOError('"reader" argument must be readable.')
+
+        if not writer.writable():
+            raise IOError('"writer" argument must be writable.')
+
+        self.reader = BufferedReader(reader, buffer_size)
+        self.writer = BufferedWriter(writer, buffer_size)
+
+    def read(self, n=None):
+        if n is None:
+            n = -1
+        return self.reader.read(n)
+
+    def readinto(self, b):
+        return self.reader.readinto(b)
+
+    def write(self, b):
+        return self.writer.write(b)
+
+    def peek(self, n=0):
+        return self.reader.peek(n)
+
+    def read1(self, n):
+        return self.reader.read1(n)
+
+    def readable(self):
+        return self.reader.readable()
+
+    def writable(self):
+        return self.writer.writable()
+
+    def flush(self):
+        return self.writer.flush()
+
+    def close(self):
+        self.writer.close()
+        self.reader.close()
+
+    def isatty(self):
+        return self.reader.isatty() or self.writer.isatty()
+
+    @property
+    def closed(self):
+        return self.writer.closed
+
+
+class BufferedRandom(BufferedWriter, BufferedReader):
+
+    """A buffered interface to random access streams.
+
+    The constructor creates a reader and writer for a seekable stream,
+    raw, given in the first argument. If the buffer_size is omitted it
+    defaults to DEFAULT_BUFFER_SIZE.
+    """
+
+    _warning_stack_offset = 3
+
+    def __init__(self, raw,
+                 buffer_size=DEFAULT_BUFFER_SIZE, max_buffer_size=None):
+        raw._checkSeekable()
+        BufferedReader.__init__(self, raw, buffer_size)
+        BufferedWriter.__init__(self, raw, buffer_size, max_buffer_size)
+
+    def seek(self, pos, whence=0):
+        if not (0 <= whence <= 2):
+            raise ValueError("invalid whence")
+        self.flush()
+        if self._read_buf:
+            # Undo read ahead.
+            with self._read_lock:
+                self.raw.seek(self._read_pos - len(self._read_buf), 1)
+        # First do the raw seek, then empty the read buffer, so that
+        # if the raw seek fails, we don't lose buffered data forever.
+        pos = self.raw.seek(pos, whence)
+        with self._read_lock:
+            self._reset_read_buf()
+        if pos < 0:
+            raise IOError("seek() returned invalid position")
+        return pos
+
+    def tell(self):
+        if self._write_buf:
+            return BufferedWriter.tell(self)
+        else:
+            return BufferedReader.tell(self)
+
+    def truncate(self, pos=None):
+        if pos is None:
+            pos = self.tell()
+        # Use seek to flush the read buffer.
+        return BufferedWriter.truncate(self, pos)
+
+    def read(self, n=None):
+        if n is None:
+            n = -1
+        self.flush()
+        return BufferedReader.read(self, n)
+
+    def readinto(self, b):
+        self.flush()
+        return BufferedReader.readinto(self, b)
+
+    def peek(self, n=0):
+        self.flush()
+        return BufferedReader.peek(self, n)
+
+    def read1(self, n):
+        self.flush()
+        return BufferedReader.read1(self, n)
+
+    def write(self, b):
+        if self._read_buf:
+            # Undo readahead
+            with self._read_lock:
+                self.raw.seek(self._read_pos - len(self._read_buf), 1)
+                self._reset_read_buf()
+        return BufferedWriter.write(self, b)
+
+
+class TextIOBase(IOBase):
+
+    """Base class for text I/O.
+
+    This class provides a character and line based interface to stream
+    I/O. There is no readinto method because Python's character strings
+    are immutable. There is no public constructor.
+    """
+
+    def read(self, n=-1):
+        """Read at most n characters from stream, where n is an int.
+
+        Read from underlying buffer until we have n characters or we hit EOF.
+        If n is negative or omitted, read until EOF.
+
+        Returns a string.
+        """
+        self._unsupported("read")
+
+    def write(self, s):
+        """Write string s to stream and returning an int."""
+        self._unsupported("write")
+
+    def truncate(self, pos=None):
+        """Truncate size to pos, where pos is an int."""
+        self._unsupported("truncate")
+
+    def readline(self):
+        """Read until newline or EOF.
+
+        Returns an empty string if EOF is hit immediately.
+        """
+        self._unsupported("readline")
+
+    def detach(self):
+        """
+        Separate the underlying buffer from the TextIOBase and return it.
+
+        After the underlying buffer has been detached, the TextIO is in an
+        unusable state.
+        """
+        self._unsupported("detach")
+
+    @property
+    def encoding(self):
+        """Subclasses should override."""
+        return None
+
+    @property
+    def newlines(self):
+        """Line endings translated so far.
+
+        Only line endings translated during reading are considered.
+
+        Subclasses should override.
+        """
+        return None
+
+    @property
+    def errors(self):
+        """Error setting of the decoder or encoder.
+
+        Subclasses should override."""
+        return None
+
+io.TextIOBase.register(TextIOBase)
+
+
+class IncrementalNewlineDecoder(codecs.IncrementalDecoder):
+    r"""Codec used when reading a file in universal newlines mode.  It wraps
+    another incremental decoder, translating \r\n and \r into \n.  It also
+    records the types of newlines encountered.  When used with
+    translate=False, it ensures that the newline sequence is returned in
+    one piece.
+    """
+    def __init__(self, decoder, translate, errors='strict'):
+        codecs.IncrementalDecoder.__init__(self, errors=errors)
+        self.translate = translate
+        self.decoder = decoder
+        self.seennl = 0
+        self.pendingcr = False
+
+    def decode(self, input, final=False):
+        # decode input (with the eventual \r from a previous pass)
+        if self.decoder is None:
+            output = input
+        else:
+            output = self.decoder.decode(input, final=final)
+        if self.pendingcr and (output or final):
+            output = "\r" + output
+            self.pendingcr = False
+
+        # retain last \r even when not translating data:
+        # then readline() is sure to get \r\n in one pass
+        if output.endswith("\r") and not final:
+            output = output[:-1]
+            self.pendingcr = True
+
+        # Record which newlines are read
+        crlf = output.count('\r\n')
+        cr = output.count('\r') - crlf
+        lf = output.count('\n') - crlf
+        self.seennl |= (lf and self._LF) | (cr and self._CR) \
+                    | (crlf and self._CRLF)
+
+        if self.translate:
+            if crlf:
+                output = output.replace("\r\n", "\n")
+            if cr:
+                output = output.replace("\r", "\n")
+
+        return output
+
+    def getstate(self):
+        if self.decoder is None:
+            buf = b""
+            flag = 0
+        else:
+            buf, flag = self.decoder.getstate()
+        flag <<= 1
+        if self.pendingcr:
+            flag |= 1
+        return buf, flag
+
+    def setstate(self, state):
+        buf, flag = state
+        self.pendingcr = bool(flag & 1)
+        if self.decoder is not None:
+            self.decoder.setstate((buf, flag >> 1))
+
+    def reset(self):
+        self.seennl = 0
+        self.pendingcr = False
+        if self.decoder is not None:
+            self.decoder.reset()
+
+    _LF = 1
+    _CR = 2
+    _CRLF = 4
+
+    @property
+    def newlines(self):
+        return (None,
+                "\n",
+                "\r",
+                ("\r", "\n"),
+                "\r\n",
+                ("\n", "\r\n"),
+                ("\r", "\r\n"),
+                ("\r", "\n", "\r\n")
+               )[self.seennl]
+
+
+class TextIOWrapper(TextIOBase):
+
+    r"""Character and line based layer over a BufferedIOBase object, buffer.
+
+    encoding gives the name of the encoding that the stream will be
+    decoded or encoded with. It defaults to locale.getpreferredencoding.
+
+    errors determines the strictness of encoding and decoding (see the
+    codecs.register) and defaults to "strict".
+
+    newline can be None, '', '\n', '\r', or '\r\n'.  It controls the
+    handling of line endings. If it is None, universal newlines is
+    enabled.  With this enabled, on input, the lines endings '\n', '\r',
+    or '\r\n' are translated to '\n' before being returned to the
+    caller. Conversely, on output, '\n' is translated to the system
+    default line seperator, os.linesep. If newline is any other of its
+    legal values, that newline becomes the newline when the file is read
+    and it is returned untranslated. On output, '\n' is converted to the
+    newline.
+
+    If line_buffering is True, a call to flush is implied when a call to
+    write contains a newline character.
+    """
+
+    _CHUNK_SIZE = 2048
+
+    def __init__(self, buffer, encoding=None, errors=None, newline=None,
+                 line_buffering=False, write_through=False):
+        if newline is not None and not isinstance(newline, str):
+            raise TypeError("illegal newline type: %r" % (type(newline),))
+        if newline not in (None, "", "\n", "\r", "\r\n"):
+            raise ValueError("illegal newline value: %r" % (newline,))
+        if encoding is None:
+            try:
+                encoding = os.device_encoding(buffer.fileno())
+            except (AttributeError, UnsupportedOperation):
+                pass
+            if encoding is None:
+                try:
+                    import locale
+                except ImportError:
+                    # Importing locale may fail if Python is being built
+                    encoding = "ascii"
+                else:
+                    encoding = locale.getpreferredencoding()
+
+        if not isinstance(encoding, str):
+            raise ValueError("invalid encoding: %r" % encoding)
+
+        if errors is None:
+            errors = "strict"
+        else:
+            if not isinstance(errors, str):
+                raise ValueError("invalid errors: %r" % errors)
+
+        self._buffer = buffer
+        self._line_buffering = line_buffering
+        self._encoding = encoding
+        self._errors = errors
+        self._readuniversal = not newline
+        self._readtranslate = newline is None
+        self._readnl = newline
+        self._writetranslate = newline != ''
+        self._writenl = newline or os.linesep
+        self._encoder = None
+        self._decoder = None
+        self._decoded_chars = ''  # buffer for text returned from decoder
+        self._decoded_chars_used = 0  # offset into _decoded_chars for read()
+        self._snapshot = None  # info for reconstructing decoder state
+        self._seekable = self._telling = self.buffer.seekable()
+        self._has_read1 = hasattr(self.buffer, 'read1')
+
+        if self._seekable and self.writable():
+            position = self.buffer.tell()
+            if position != 0:
+                try:
+                    self._get_encoder().setstate(0)
+                except LookupError:
+                    # Sometimes the encoder doesn't exist
+                    pass
+
+    # self._snapshot is either None, or a tuple (dec_flags, next_input)
+    # where dec_flags is the second (integer) item of the decoder state
+    # and next_input is the chunk of input bytes that comes next after the
+    # snapshot point.  We use this to reconstruct decoder states in tell().
+
+    # Naming convention:
+    #   - "bytes_..." for integer variables that count input bytes
+    #   - "chars_..." for integer variables that count decoded characters
+
+    def __repr__(self):
+        result = "<_pyio.TextIOWrapper"
+        try:
+            name = self.name
+        except AttributeError:
+            pass
+        else:
+            result += " name={0!r}".format(name)
+        try:
+            mode = self.mode
+        except AttributeError:
+            pass
+        else:
+            result += " mode={0!r}".format(mode)
+        return result + " encoding={0!r}>".format(self.encoding)
+
+    @property
+    def encoding(self):
+        return self._encoding
+
+    @property
+    def errors(self):
+        return self._errors
+
+    @property
+    def line_buffering(self):
+        return self._line_buffering
+
+    @property
+    def buffer(self):
+        return self._buffer
+
+    def seekable(self):
+        return self._seekable
+
+    def readable(self):
+        return self.buffer.readable()
+
+    def writable(self):
+        return self.buffer.writable()
+
+    def flush(self):
+        self.buffer.flush()
+        self._telling = self._seekable
+
+    def close(self):
+        if self.buffer is not None and not self.closed:
+            self.flush()
+            self.buffer.close()
+
+    @property
+    def closed(self):
+        return self.buffer.closed
+
+    @property
+    def name(self):
+        return self.buffer.name
+
+    def fileno(self):
+        return self.buffer.fileno()
+
+    def isatty(self):
+        return self.buffer.isatty()
+
+    def write(self, s):
+        'Write data, where s is a str'
+        if self.closed:
+            raise ValueError("write to closed file")
+        if not isinstance(s, str):
+            raise TypeError("can't write %s to text stream" %
+                            s.__class__.__name__)
+        length = len(s)
+        haslf = (self._writetranslate or self._line_buffering) and "\n" in s
+        if haslf and self._writetranslate and self._writenl != "\n":
+            s = s.replace("\n", self._writenl)
+        encoder = self._encoder or self._get_encoder()
+        # XXX What if we were just reading?
+        b = encoder.encode(s)
+        self.buffer.write(b)
+        if self._line_buffering and (haslf or "\r" in s):
+            self.flush()
+        self._snapshot = None
+        if self._decoder:
+            self._decoder.reset()
+        return length
+
+    def _get_encoder(self):
+        make_encoder = codecs.getincrementalencoder(self._encoding)
+        self._encoder = make_encoder(self._errors)
+        return self._encoder
+
+    def _get_decoder(self):
+        make_decoder = codecs.getincrementaldecoder(self._encoding)
+        decoder = make_decoder(self._errors)
+        if self._readuniversal:
+            decoder = IncrementalNewlineDecoder(decoder, self._readtranslate)
+        self._decoder = decoder
+        return decoder
+
+    # The following three methods implement an ADT for _decoded_chars.
+    # Text returned from the decoder is buffered here until the client
+    # requests it by calling our read() or readline() method.
+    def _set_decoded_chars(self, chars):
+        """Set the _decoded_chars buffer."""
+        self._decoded_chars = chars
+        self._decoded_chars_used = 0
+
+    def _get_decoded_chars(self, n=None):
+        """Advance into the _decoded_chars buffer."""
+        offset = self._decoded_chars_used
+        if n is None:
+            chars = self._decoded_chars[offset:]
+        else:
+            chars = self._decoded_chars[offset:offset + n]
+        self._decoded_chars_used += len(chars)
+        return chars
+
+    def _rewind_decoded_chars(self, n):
+        """Rewind the _decoded_chars buffer."""
+        if self._decoded_chars_used < n:
+            raise AssertionError("rewind decoded_chars out of bounds")
+        self._decoded_chars_used -= n
+
+    def _read_chunk(self):
+        """
+        Read and decode the next chunk of data from the BufferedReader.
+        """
+
+        # The return value is True unless EOF was reached.  The decoded
+        # string is placed in self._decoded_chars (replacing its previous
+        # value).  The entire input chunk is sent to the decoder, though
+        # some of it may remain buffered in the decoder, yet to be
+        # converted.
+
+        if self._decoder is None:
+            raise ValueError("no decoder")
+
+        if self._telling:
+            # To prepare for tell(), we need to snapshot a point in the
+            # file where the decoder's input buffer is empty.
+
+            dec_buffer, dec_flags = self._decoder.getstate()
+            # Given this, we know there was a valid snapshot point
+            # len(dec_buffer) bytes ago with decoder state (b'', dec_flags).
+
+        # Read a chunk, decode it, and put the result in self._decoded_chars.
+        if self._has_read1:
+            input_chunk = self.buffer.read1(self._CHUNK_SIZE)
+        else:
+            input_chunk = self.buffer.read(self._CHUNK_SIZE)
+        eof = not input_chunk
+        self._set_decoded_chars(self._decoder.decode(input_chunk, eof))
+
+        if self._telling:
+            # At the snapshot point, len(dec_buffer) bytes before the read,
+            # the next input to be decoded is dec_buffer + input_chunk.
+            self._snapshot = (dec_flags, dec_buffer + input_chunk)
+
+        return not eof
+
+    def _pack_cookie(self, position, dec_flags=0,
+                           bytes_to_feed=0, need_eof=0, chars_to_skip=0):
+        # The meaning of a tell() cookie is: seek to position, set the
+        # decoder flags to dec_flags, read bytes_to_feed bytes, feed them
+        # into the decoder with need_eof as the EOF flag, then skip
+        # chars_to_skip characters of the decoded result.  For most simple
+        # decoders, tell() will often just give a byte offset in the file.
+        return (position | (dec_flags<<64) | (bytes_to_feed<<128) |
+               (chars_to_skip<<192) | bool(need_eof)<<256)
+
+    def _unpack_cookie(self, bigint):
+        rest, position = divmod(bigint, 1<<64)
+        rest, dec_flags = divmod(rest, 1<<64)
+        rest, bytes_to_feed = divmod(rest, 1<<64)
+        need_eof, chars_to_skip = divmod(rest, 1<<64)
+        return position, dec_flags, bytes_to_feed, need_eof, chars_to_skip
+
+    def tell(self):
+        if not self._seekable:
+            raise UnsupportedOperation("underlying stream is not seekable")
+        if not self._telling:
+            raise IOError("telling position disabled by next() call")
+        self.flush()
+        position = self.buffer.tell()
+        decoder = self._decoder
+        if decoder is None or self._snapshot is None:
+            if self._decoded_chars:
+                # This should never happen.
+                raise AssertionError("pending decoded text")
+            return position
+
+        # Skip backward to the snapshot point (see _read_chunk).
+        dec_flags, next_input = self._snapshot
+        position -= len(next_input)
+
+        # How many decoded characters have been used up since the snapshot?
+        chars_to_skip = self._decoded_chars_used
+        if chars_to_skip == 0:
+            # We haven't moved from the snapshot point.
+            return self._pack_cookie(position, dec_flags)
+
+        # Starting from the snapshot position, we will walk the decoder
+        # forward until it gives us enough decoded characters.
+        saved_state = decoder.getstate()
+        try:
+            # Note our initial start point.
+            decoder.setstate((b'', dec_flags))
+            start_pos = position
+            start_flags, bytes_fed, chars_decoded = dec_flags, 0, 0
+            need_eof = 0
+
+            # Feed the decoder one byte at a time.  As we go, note the
+            # nearest "safe start point" before the current location
+            # (a point where the decoder has nothing buffered, so seek()
+            # can safely start from there and advance to this location).
+            next_byte = bytearray(1)
+            for next_byte[0] in next_input:
+                bytes_fed += 1
+                chars_decoded += len(decoder.decode(next_byte))
+                dec_buffer, dec_flags = decoder.getstate()
+                if not dec_buffer and chars_decoded <= chars_to_skip:
+                    # Decoder buffer is empty, so this is a safe start point.
+                    start_pos += bytes_fed
+                    chars_to_skip -= chars_decoded
+                    start_flags, bytes_fed, chars_decoded = dec_flags, 0, 0
+                if chars_decoded >= chars_to_skip:
+                    break
+            else:
+                # We didn't get enough decoded data; signal EOF to get more.
+                chars_decoded += len(decoder.decode(b'', final=True))
+                need_eof = 1
+                if chars_decoded < chars_to_skip:
+                    raise IOError("can't reconstruct logical file position")
+
+            # The returned cookie corresponds to the last safe start point.
+            return self._pack_cookie(
+                start_pos, start_flags, bytes_fed, need_eof, chars_to_skip)
+        finally:
+            decoder.setstate(saved_state)
+
+    def truncate(self, pos=None):
+        self.flush()
+        if pos is None:
+            pos = self.tell()
+        return self.buffer.truncate(pos)
+
+    def detach(self):
+        if self.buffer is None:
+            raise ValueError("buffer is already detached")
+        self.flush()
+        buffer = self._buffer
+        self._buffer = None
+        return buffer
+
+    def seek(self, cookie, whence=0):
+        if self.closed:
+            raise ValueError("tell on closed file")
+        if not self._seekable:
+            raise UnsupportedOperation("underlying stream is not seekable")
+        if whence == 1: # seek relative to current position
+            if cookie != 0:
+                raise UnsupportedOperation("can't do nonzero cur-relative seeks")
+            # Seeking to the current position should attempt to
+            # sync the underlying buffer with the current position.
+            whence = 0
+            cookie = self.tell()
+        if whence == 2: # seek relative to end of file
+            if cookie != 0:
+                raise UnsupportedOperation("can't do nonzero end-relative seeks")
+            self.flush()
+            position = self.buffer.seek(0, 2)
+            self._set_decoded_chars('')
+            self._snapshot = None
+            if self._decoder:
+                self._decoder.reset()
+            return position
+        if whence != 0:
+            raise ValueError("invalid whence (%r, should be 0, 1 or 2)" %
+                             (whence,))
+        if cookie < 0:
+            raise ValueError("negative seek position %r" % (cookie,))
+        self.flush()
+
+        # The strategy of seek() is to go back to the safe start point
+        # and replay the effect of read(chars_to_skip) from there.
+        start_pos, dec_flags, bytes_to_feed, need_eof, chars_to_skip = \
+            self._unpack_cookie(cookie)
+
+        # Seek back to the safe start point.
+        self.buffer.seek(start_pos)
+        self._set_decoded_chars('')
+        self._snapshot = None
+
+        # Restore the decoder to its state from the safe start point.
+        if cookie == 0 and self._decoder:
+            self._decoder.reset()
+        elif self._decoder or dec_flags or chars_to_skip:
+            self._decoder = self._decoder or self._get_decoder()
+            self._decoder.setstate((b'', dec_flags))
+            self._snapshot = (dec_flags, b'')
+
+        if chars_to_skip:
+            # Just like _read_chunk, feed the decoder and save a snapshot.
+            input_chunk = self.buffer.read(bytes_to_feed)
+            self._set_decoded_chars(
+                self._decoder.decode(input_chunk, need_eof))
+            self._snapshot = (dec_flags, input_chunk)
+
+            # Skip chars_to_skip of the decoded characters.
+            if len(self._decoded_chars) < chars_to_skip:
+                raise IOError("can't restore logical file position")
+            self._decoded_chars_used = chars_to_skip
+
+        # Finally, reset the encoder (merely useful for proper BOM handling)
+        try:
+            encoder = self._encoder or self._get_encoder()
+        except LookupError:
+            # Sometimes the encoder doesn't exist
+            pass
+        else:
+            if cookie != 0:
+                encoder.setstate(0)
+            else:
+                encoder.reset()
+        return cookie
+
+    def read(self, n=None):
+        self._checkReadable()
+        if n is None:
+            n = -1
+        decoder = self._decoder or self._get_decoder()
+        try:
+            n.__index__
+        except AttributeError as err:
+            raise TypeError("an integer is required") from err
+        if n < 0:
+            # Read everything.
+            result = (self._get_decoded_chars() +
+                      decoder.decode(self.buffer.read(), final=True))
+            self._set_decoded_chars('')
+            self._snapshot = None
+            return result
+        else:
+            # Keep reading chunks until we have n characters to return.
+            eof = False
+            result = self._get_decoded_chars(n)
+            while len(result) < n and not eof:
+                eof = not self._read_chunk()
+                result += self._get_decoded_chars(n - len(result))
+            return result
+
+    def __next__(self):
+        self._telling = False
+        line = self.readline()
+        if not line:
+            self._snapshot = None
+            self._telling = self._seekable
+            raise StopIteration
+        return line
+
+    def readline(self, limit=None):
+        if self.closed:
+            raise ValueError("read from closed file")
+        if limit is None:
+            limit = -1
+        elif not isinstance(limit, int):
+            raise TypeError("limit must be an integer")
+
+        # Grab all the decoded text (we will rewind any extra bits later).
+        line = self._get_decoded_chars()
+
+        start = 0
+        # Make the decoder if it doesn't already exist.
+        if not self._decoder:
+            self._get_decoder()
+
+        pos = endpos = None
+        while True:
+            if self._readtranslate:
+                # Newlines are already translated, only search for \n
+                pos = line.find('\n', start)
+                if pos >= 0:
+                    endpos = pos + 1
+                    break
+                else:
+                    start = len(line)
+
+            elif self._readuniversal:
+                # Universal newline search. Find any of \r, \r\n, \n
+                # The decoder ensures that \r\n are not split in two pieces
+
+                # In C we'd look for these in parallel of course.
+                nlpos = line.find("\n", start)
+                crpos = line.find("\r", start)
+                if crpos == -1:
+                    if nlpos == -1:
+                        # Nothing found
+                        start = len(line)
+                    else:
+                        # Found \n
+                        endpos = nlpos + 1
+                        break
+                elif nlpos == -1:
+                    # Found lone \r
+                    endpos = crpos + 1
+                    break
+                elif nlpos < crpos:
+                    # Found \n
+                    endpos = nlpos + 1
+                    break
+                elif nlpos == crpos + 1:
+                    # Found \r\n
+                    endpos = crpos + 2
+                    break
+                else:
+                    # Found \r
+                    endpos = crpos + 1
+                    break
+            else:
+                # non-universal
+                pos = line.find(self._readnl)
+                if pos >= 0:
+                    endpos = pos + len(self._readnl)
+                    break
+
+            if limit >= 0 and len(line) >= limit:
+                endpos = limit  # reached length limit
+                break
+
+            # No line ending seen yet - get more data'
+            while self._read_chunk():
+                if self._decoded_chars:
+                    break
+            if self._decoded_chars:
+                line += self._get_decoded_chars()
+            else:
+                # end of file
+                self._set_decoded_chars('')
+                self._snapshot = None
+                return line
+
+        if limit >= 0 and endpos > limit:
+            endpos = limit  # don't exceed limit
+
+        # Rewind _decoded_chars to just after the line ending we found.
+        self._rewind_decoded_chars(len(line) - endpos)
+        return line[:endpos]
+
+    @property
+    def newlines(self):
+        return self._decoder.newlines if self._decoder else None
+
+
+class StringIO(TextIOWrapper):
+    """Text I/O implementation using an in-memory buffer.
+
+    The initial_value argument sets the value of object.  The newline
+    argument is like the one of TextIOWrapper's constructor.
+    """
+
+    def __init__(self, initial_value="", newline="\n"):
+        super(StringIO, self).__init__(BytesIO(),
+                                       encoding="utf-8",
+                                       errors="strict",
+                                       newline=newline)
+        # Issue #5645: make universal newlines semantics the same as in the
+        # C version, even under Windows.
+        if newline is None:
+            self._writetranslate = False
+        if initial_value is not None:
+            if not isinstance(initial_value, str):
+                raise TypeError("initial_value must be str or None, not {0}"
+                                .format(type(initial_value).__name__))
+                initial_value = str(initial_value)
+            self.write(initial_value)
+            self.seek(0)
+
+    def getvalue(self):
+        self.flush()
+        return self.buffer.getvalue().decode(self._encoding, self._errors)
+
+    def __repr__(self):
+        # TextIOWrapper tells the encoding in its repr. In StringIO,
+        # that's a implementation detail.
+        return object.__repr__(self)
+
+    @property
+    def errors(self):
+        return None
+
+    @property
+    def encoding(self):
+        return None
+
+    def detach(self):
+        # This doesn't make sense on StringIO.
+        self._unsupported("detach")
diff --git a/lib-python/3.2/_strptime.py b/lib-python/3.2/_strptime.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/_strptime.py
@@ -0,0 +1,499 @@
+"""Strptime-related classes and functions.
+
+CLASSES:
+    LocaleTime -- Discovers and stores locale-specific time information
+    TimeRE -- Creates regexes for pattern matching a string of text containing
+                time information
+
+FUNCTIONS:
+    _getlang -- Figure out what language is being used for the locale
+    strptime -- Calculates the time struct represented by the passed-in string
+
+"""
+import time
+import locale
+import calendar
+from re import compile as re_compile
+from re import IGNORECASE, ASCII
+from re import escape as re_escape
+from datetime import (date as datetime_date,
+                      timedelta as datetime_timedelta,
+                      timezone as datetime_timezone)
+try:
+    from _thread import allocate_lock as _thread_allocate_lock
+except:
+    from _dummy_thread import allocate_lock as _thread_allocate_lock
+
+__all__ = []
+
+def _getlang():
+    # Figure out what the current language is set to.
+    return locale.getlocale(locale.LC_TIME)
+
+class LocaleTime(object):
+    """Stores and handles locale-specific information related to time.
+
+    ATTRIBUTES:
+        f_weekday -- full weekday names (7-item list)
+        a_weekday -- abbreviated weekday names (7-item list)
+        f_month -- full month names (13-item list; dummy value in [0], which
+                    is added by code)
+        a_month -- abbreviated month names (13-item list, dummy value in
+                    [0], which is added by code)
+        am_pm -- AM/PM representation (2-item list)
+        LC_date_time -- format string for date/time representation (string)
+        LC_date -- format string for date representation (string)
+        LC_time -- format string for time representation (string)
+        timezone -- daylight- and non-daylight-savings timezone representation
+                    (2-item list of sets)
+        lang -- Language used by instance (2-item tuple)
+    """
+
+    def __init__(self):
+        """Set all attributes.
+
+        Order of methods called matters for dependency reasons.
+
+        The locale language is set at the offset and then checked again before
+        exiting.  This is to make sure that the attributes were not set with a
+        mix of information from more than one locale.  This would most likely
+        happen when using threads where one thread calls a locale-dependent
+        function while another thread changes the locale while the function in
+        the other thread is still running.  Proper coding would call for
+        locks to prevent changing the locale while locale-dependent code is
+        running.  The check here is done in case someone does not think about
+        doing this.
+
+        Only other possible issue is if someone changed the timezone and did
+        not call tz.tzset .  That is an issue for the programmer, though,
+        since changing the timezone is worthless without that call.
+
+        """
+        self.lang = _getlang()
+        self.__calc_weekday()
+        self.__calc_month()
+        self.__calc_am_pm()
+        self.__calc_timezone()
+        self.__calc_date_time()
+        if _getlang() != self.lang:
+            raise ValueError("locale changed during initialization")
+
+    def __pad(self, seq, front):
+        # Add '' to seq to either the front (is True), else the back.
+        seq = list(seq)
+        if front:
+            seq.insert(0, '')
+        else:
+            seq.append('')
+        return seq
+
+    def __calc_weekday(self):
+        # Set self.a_weekday and self.f_weekday using the calendar
+        # module.
+        a_weekday = [calendar.day_abbr[i].lower() for i in range(7)]
+        f_weekday = [calendar.day_name[i].lower() for i in range(7)]
+        self.a_weekday = a_weekday
+        self.f_weekday = f_weekday
+
+    def __calc_month(self):
+        # Set self.f_month and self.a_month using the calendar module.
+        a_month = [calendar.month_abbr[i].lower() for i in range(13)]
+        f_month = [calendar.month_name[i].lower() for i in range(13)]
+        self.a_month = a_month
+        self.f_month = f_month
+
+    def __calc_am_pm(self):
+        # Set self.am_pm by using time.strftime().
+
+        # The magic date (1999,3,17,hour,44,55,2,76,0) is not really that
+        # magical; just happened to have used it everywhere else where a
+        # static date was needed.
+        am_pm = []
+        for hour in (1, 22):
+            time_tuple = time.struct_time((1999,3,17,hour,44,55,2,76,0))
+            am_pm.append(time.strftime("%p", time_tuple).lower())
+        self.am_pm = am_pm
+
+    def __calc_date_time(self):
+        # Set self.date_time, self.date, & self.time by using
+        # time.strftime().
+
+        # Use (1999,3,17,22,44,55,2,76,0) for magic date because the amount of
+        # overloaded numbers is minimized.  The order in which searches for
+        # values within the format string is very important; it eliminates
+        # possible ambiguity for what something represents.
+        time_tuple = time.struct_time((1999,3,17,22,44,55,2,76,0))
+        date_time = [None, None, None]
+        date_time[0] = time.strftime("%c", time_tuple).lower()
+        date_time[1] = time.strftime("%x", time_tuple).lower()
+        date_time[2] = time.strftime("%X", time_tuple).lower()
+        replacement_pairs = [('%', '%%'), (self.f_weekday[2], '%A'),
+                    (self.f_month[3], '%B'), (self.a_weekday[2], '%a'),
+                    (self.a_month[3], '%b'), (self.am_pm[1], '%p'),
+                    ('1999', '%Y'), ('99', '%y'), ('22', '%H'),
+                    ('44', '%M'), ('55', '%S'), ('76', '%j'),
+                    ('17', '%d'), ('03', '%m'), ('3', '%m'),
+                    # '3' needed for when no leading zero.
+                    ('2', '%w'), ('10', '%I')]
+        replacement_pairs.extend([(tz, "%Z") for tz_values in self.timezone
+                                                for tz in tz_values])
+        for offset,directive in ((0,'%c'), (1,'%x'), (2,'%X')):
+            current_format = date_time[offset]
+            for old, new in replacement_pairs:
+                # Must deal with possible lack of locale info
+                # manifesting itself as the empty string (e.g., Swedish's
+                # lack of AM/PM info) or a platform returning a tuple of empty
+                # strings (e.g., MacOS 9 having timezone as ('','')).
+                if old:
+                    current_format = current_format.replace(old, new)
+            # If %W is used, then Sunday, 2005-01-03 will fall on week 0 since
+            # 2005-01-03 occurs before the first Monday of the year.  Otherwise
+            # %U is used.
+            time_tuple = time.struct_time((1999,1,3,1,1,1,6,3,0))
+            if '00' in time.strftime(directive, time_tuple):
+                U_W = '%W'
+            else:
+                U_W = '%U'
+            date_time[offset] = current_format.replace('11', U_W)
+        self.LC_date_time = date_time[0]
+        self.LC_date = date_time[1]
+        self.LC_time = date_time[2]
+
+    def __calc_timezone(self):
+        # Set self.timezone by using time.tzname.
+        # Do not worry about possibility of time.tzname[0] == timetzname[1]
+        # and time.daylight; handle that in strptime .
+        try:
+            time.tzset()
+        except AttributeError:
+            pass
+        no_saving = frozenset(["utc", "gmt", time.tzname[0].lower()])
+        if time.daylight:
+            has_saving = frozenset([time.tzname[1].lower()])
+        else:
+            has_saving = frozenset()
+        self.timezone = (no_saving, has_saving)
+
+
+class TimeRE(dict):
+    """Handle conversion from format directives to regexes."""
+
+    def __init__(self, locale_time=None):
+        """Create keys/values.
+
+        Order of execution is important for dependency reasons.
+
+        """
+        if locale_time:
+            self.locale_time = locale_time
+        else:
+            self.locale_time = LocaleTime()
+        base = super()
+        base.__init__({
+            # The " \d" part of the regex is to make %c from ANSI C work
+            'd': r"(?P<d>3[0-1]|[1-2]\d|0[1-9]|[1-9]| [1-9])",
+            'f': r"(?P<f>[0-9]{1,6})",
+            'H': r"(?P<H>2[0-3]|[0-1]\d|\d)",
+            'I': r"(?P<I>1[0-2]|0[1-9]|[1-9])",
+            'j': r"(?P<j>36[0-6]|3[0-5]\d|[1-2]\d\d|0[1-9]\d|00[1-9]|[1-9]\d|0[1-9]|[1-9])",
+            'm': r"(?P<m>1[0-2]|0[1-9]|[1-9])",
+            'M': r"(?P<M>[0-5]\d|\d)",
+            'S': r"(?P<S>6[0-1]|[0-5]\d|\d)",
+            'U': r"(?P<U>5[0-3]|[0-4]\d|\d)",
+            'w': r"(?P<w>[0-6])",
+            # W is set below by using 'U'
+            'y': r"(?P<y>\d\d)",
+            #XXX: Does 'Y' need to worry about having less or more than
+            #     4 digits?
+            'Y': r"(?P<Y>\d\d\d\d)",
+            'z': r"(?P<z>[+-]\d\d[0-5]\d)",
+            'A': self.__seqToRE(self.locale_time.f_weekday, 'A'),
+            'a': self.__seqToRE(self.locale_time.a_weekday, 'a'),
+            'B': self.__seqToRE(self.locale_time.f_month[1:], 'B'),
+            'b': self.__seqToRE(self.locale_time.a_month[1:], 'b'),
+            'p': self.__seqToRE(self.locale_time.am_pm, 'p'),
+            'Z': self.__seqToRE((tz for tz_names in self.locale_time.timezone
+                                        for tz in tz_names),
+                                'Z'),
+            '%': '%'})
+        base.__setitem__('W', base.__getitem__('U').replace('U', 'W'))
+        base.__setitem__('c', self.pattern(self.locale_time.LC_date_time))
+        base.__setitem__('x', self.pattern(self.locale_time.LC_date))
+        base.__setitem__('X', self.pattern(self.locale_time.LC_time))
+
+    def __seqToRE(self, to_convert, directive):
+        """Convert a list to a regex string for matching a directive.
+
+        Want possible matching values to be from longest to shortest.  This
+        prevents the possibility of a match occuring for a value that also
+        a substring of a larger value that should have matched (e.g., 'abc'
+        matching when 'abcdef' should have been the match).
+
+        """
+        to_convert = sorted(to_convert, key=len, reverse=True)
+        for value in to_convert:
+            if value != '':
+                break
+        else:
+            return ''
+        regex = '|'.join(re_escape(stuff) for stuff in to_convert)
+        regex = '(?P<%s>%s' % (directive, regex)
+        return '%s)' % regex
+
+    def pattern(self, format):
+        """Return regex pattern for the format string.
+
+        Need to make sure that any characters that might be interpreted as
+        regex syntax are escaped.
+
+        """
+        processed_format = ''
+        # The sub() call escapes all characters that might be misconstrued
+        # as regex syntax.  Cannot use re.escape since we have to deal with
+        # format directives (%m, etc.).
+        regex_chars = re_compile(r"([\\.^$*+?\(\){}\[\]|])")
+        format = regex_chars.sub(r"\\\1", format)
+        whitespace_replacement = re_compile('\s+')
+        format = whitespace_replacement.sub('\s+', format)
+        while '%' in format:
+            directive_index = format.index('%')+1
+            processed_format = "%s%s%s" % (processed_format,
+                                           format[:directive_index-1],
+                                           self[format[directive_index]])
+            format = format[directive_index+1:]
+        return "%s%s" % (processed_format, format)
+
+    def compile(self, format):
+        """Return a compiled re object for the format string."""
+        return re_compile(self.pattern(format), IGNORECASE)
+
+_cache_lock = _thread_allocate_lock()
+# DO NOT modify _TimeRE_cache or _regex_cache without acquiring the cache lock
+# first!
+_TimeRE_cache = TimeRE()
+_CACHE_MAX_SIZE = 5 # Max number of regexes stored in _regex_cache
+_regex_cache = {}
+
+def _calc_julian_from_U_or_W(year, week_of_year, day_of_week, week_starts_Mon):
+    """Calculate the Julian day based on the year, week of the year, and day of
+    the week, with week_start_day representing whether the week of the year
+    assumes the week starts on Sunday or Monday (6 or 0)."""
+    first_weekday = datetime_date(year, 1, 1).weekday()
+    # If we are dealing with the %U directive (week starts on Sunday), it's
+    # easier to just shift the view to Sunday being the first day of the
+    # week.
+    if not week_starts_Mon:
+        first_weekday = (first_weekday + 1) % 7
+        day_of_week = (day_of_week + 1) % 7
+    # Need to watch out for a week 0 (when the first day of the year is not
+    # the same as that specified by %U or %W).
+    week_0_length = (7 - first_weekday) % 7
+    if week_of_year == 0:
+        return 1 + day_of_week - first_weekday
+    else:
+        days_to_week = week_0_length + (7 * (week_of_year - 1))
+        return 1 + days_to_week + day_of_week
+
+
+def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"):
+    """Return a 2-tuple consisting of a time struct and an int containing
+    the number of microseconds based on the input string and the
+    format string."""
+
+    for index, arg in enumerate([data_string, format]):
+        if not isinstance(arg, str):
+            msg = "strptime() argument {} must be str, not {}"
+            raise TypeError(msg.format(index, type(arg)))
+
+    global _TimeRE_cache, _regex_cache
+    with _cache_lock:
+
+        if _getlang() != _TimeRE_cache.locale_time.lang:
+            _TimeRE_cache = TimeRE()
+            _regex_cache.clear()
+        if len(_regex_cache) > _CACHE_MAX_SIZE:
+            _regex_cache.clear()
+        locale_time = _TimeRE_cache.locale_time
+        format_regex = _regex_cache.get(format)
+        if not format_regex:
+            try:
+                format_regex = _TimeRE_cache.compile(format)
+            # KeyError raised when a bad format is found; can be specified as
+            # \\, in which case it was a stray % but with a space after it
+            except KeyError as err:
+                bad_directive = err.args[0]
+                if bad_directive == "\\":
+                    bad_directive = "%"
+                del err
+                raise ValueError("'%s' is a bad directive in format '%s'" %
+                                    (bad_directive, format))
+            # IndexError only occurs when the format string is "%"
+            except IndexError:
+                raise ValueError("stray %% in format '%s'" % format)
+            _regex_cache[format] = format_regex
+    found = format_regex.match(data_string)
+    if not found:
+        raise ValueError("time data %r does not match format %r" %
+                         (data_string, format))
+    if len(data_string) != found.end():
+        raise ValueError("unconverted data remains: %s" %
+                          data_string[found.end():])
+
+    year = 1900
+    month = day = 1
+    hour = minute = second = fraction = 0
+    tz = -1
+    tzoffset = None
+    # Default to -1 to signify that values not known; not critical to have,
+    # though
+    week_of_year = -1
+    week_of_year_start = -1
+    # weekday and julian defaulted to -1 so as to signal need to calculate
+    # values
+    weekday = julian = -1
+    found_dict = found.groupdict()
+    for group_key in found_dict.keys():
+        # Directives not explicitly handled below:
+        #   c, x, X
+        #      handled by making out of other directives
+        #   U, W
+        #      worthless without day of the week
+        if group_key == 'y':
+            year = int(found_dict['y'])
+            # Open Group specification for strptime() states that a %y
+            #value in the range of [00, 68] is in the century 2000, while
+            #[69,99] is in the century 1900
+            if year <= 68:
+                year += 2000
+            else:
+                year += 1900
+        elif group_key == 'Y':
+            year = int(found_dict['Y'])
+        elif group_key == 'm':
+            month = int(found_dict['m'])
+        elif group_key == 'B':
+            month = locale_time.f_month.index(found_dict['B'].lower())
+        elif group_key == 'b':
+            month = locale_time.a_month.index(found_dict['b'].lower())
+        elif group_key == 'd':
+            day = int(found_dict['d'])
+        elif group_key == 'H':
+            hour = int(found_dict['H'])
+        elif group_key == 'I':
+            hour = int(found_dict['I'])
+            ampm = found_dict.get('p', '').lower()
+            # If there was no AM/PM indicator, we'll treat this like AM
+            if ampm in ('', locale_time.am_pm[0]):
+                # We're in AM so the hour is correct unless we're
+                # looking at 12 midnight.
+                # 12 midnight == 12 AM == hour 0
+                if hour == 12:
+                    hour = 0
+            elif ampm == locale_time.am_pm[1]:
+                # We're in PM so we need to add 12 to the hour unless
+                # we're looking at 12 noon.
+                # 12 noon == 12 PM == hour 12
+                if hour != 12:
+                    hour += 12
+        elif group_key == 'M':
+            minute = int(found_dict['M'])
+        elif group_key == 'S':
+            second = int(found_dict['S'])
+        elif group_key == 'f':
+            s = found_dict['f']
+            # Pad to always return microseconds.
+            s += "0" * (6 - len(s))
+            fraction = int(s)
+        elif group_key == 'A':
+            weekday = locale_time.f_weekday.index(found_dict['A'].lower())
+        elif group_key == 'a':
+            weekday = locale_time.a_weekday.index(found_dict['a'].lower())
+        elif group_key == 'w':
+            weekday = int(found_dict['w'])
+            if weekday == 0:
+                weekday = 6
+            else:
+                weekday -= 1
+        elif group_key == 'j':
+            julian = int(found_dict['j'])
+        elif group_key in ('U', 'W'):
+            week_of_year = int(found_dict[group_key])
+            if group_key == 'U':
+                # U starts week on Sunday.
+                week_of_year_start = 6
+            else:
+                # W starts week on Monday.
+                week_of_year_start = 0
+        elif group_key == 'z':
+            z = found_dict['z']
+            tzoffset = int(z[1:3]) * 60 + int(z[3:5])
+            if z.startswith("-"):
+                tzoffset = -tzoffset
+        elif group_key == 'Z':
+            # Since -1 is default value only need to worry about setting tz if
+            # it can be something other than -1.
+            found_zone = found_dict['Z'].lower()
+            for value, tz_values in enumerate(locale_time.timezone):
+                if found_zone in tz_values:
+                    # Deal with bad locale setup where timezone names are the
+                    # same and yet time.daylight is true; too ambiguous to
+                    # be able to tell what timezone has daylight savings
+                    if (time.tzname[0] == time.tzname[1] and
+                       time.daylight and found_zone not in ("utc", "gmt")):
+                        break
+                    else:
+                        tz = value
+                        break
+    # If we know the week of the year and what day of that week, we can figure
+    # out the Julian day of the year.
+    if julian == -1 and week_of_year != -1 and weekday != -1:
+        week_starts_Mon = True if week_of_year_start == 0 else False
+        julian = _calc_julian_from_U_or_W(year, week_of_year, weekday,
+                                            week_starts_Mon)
+    # Cannot pre-calculate datetime_date() since can change in Julian
+    # calculation and thus could have different value for the day of the week
+    # calculation.
+    if julian == -1:
+        # Need to add 1 to result since first day of the year is 1, not 0.
+        julian = datetime_date(year, month, day).toordinal() - \
+                  datetime_date(year, 1, 1).toordinal() + 1
+    else:  # Assume that if they bothered to include Julian day it will
+           # be accurate.
+        datetime_result = datetime_date.fromordinal((julian - 1) + datetime_date(year, 1, 1).toordinal())
+        year = datetime_result.year
+        month = datetime_result.month
+        day = datetime_result.day
+    if weekday == -1:
+        weekday = datetime_date(year, month, day).weekday()
+    # Add timezone info
+    tzname = found_dict.get("Z")
+    if tzoffset is not None:
+        gmtoff = tzoffset * 60
+    else:
+        gmtoff = None
+
+    return (year, month, day,
+            hour, minute, second,
+            weekday, julian, tz, gmtoff, tzname), fraction
+
+def _strptime_time(data_string, format="%a %b %d %H:%M:%S %Y"):
+    """Return a time struct based on the input string and the
+    format string."""
+    tt = _strptime(data_string, format)[0]
+    return time.struct_time(tt[:9])
+
+def _strptime_datetime(cls, data_string, format="%a %b %d %H:%M:%S %Y"):
+    """Return a class cls instance based on the input string and the
+    format string."""
+    tt, fraction = _strptime(data_string, format)
+    gmtoff, tzname = tt[-2:]
+    args = tt[:6] + (fraction,)
+    if gmtoff is not None:
+        tzdelta = datetime_timedelta(seconds=gmtoff)
+        if tzname:
+            tz = datetime_timezone(tzdelta, tzname)
+        else:
+            tz = datetime_timezone(tzdelta)
+        args += (tz,)
+
+    return cls(*args)
diff --git a/lib-python/3.2/_threading_local.py b/lib-python/3.2/_threading_local.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/_threading_local.py
@@ -0,0 +1,246 @@
+"""Thread-local objects.
+
+(Note that this module provides a Python version of the threading.local
+ class.  Depending on the version of Python you're using, there may be a
+ faster one available.  You should always import the `local` class from
+ `threading`.)
+
+Thread-local objects support the management of thread-local data.
+If you have data that you want to be local to a thread, simply create
+a thread-local object and use its attributes:
+
+  >>> mydata = local()
+  >>> mydata.number = 42
+  >>> mydata.number
+  42
+
+You can also access the local-object's dictionary:
+
+  >>> mydata.__dict__
+  {'number': 42}
+  >>> mydata.__dict__.setdefault('widgets', [])
+  []
+  >>> mydata.widgets
+  []
+
+What's important about thread-local objects is that their data are
+local to a thread. If we access the data in a different thread:
+
+  >>> log = []
+  >>> def f():
+  ...     items = sorted(mydata.__dict__.items())
+  ...     log.append(items)
+  ...     mydata.number = 11
+  ...     log.append(mydata.number)
+
+  >>> import threading
+  >>> thread = threading.Thread(target=f)
+  >>> thread.start()
+  >>> thread.join()
+  >>> log
+  [[], 11]
+
+we get different data.  Furthermore, changes made in the other thread
+don't affect data seen in this thread:
+
+  >>> mydata.number
+  42
+
+Of course, values you get from a local object, including a __dict__
+attribute, are for whatever thread was current at the time the
+attribute was read.  For that reason, you generally don't want to save
+these values across threads, as they apply only to the thread they
+came from.
+
+You can create custom local objects by subclassing the local class:
+
+  >>> class MyLocal(local):
+  ...     number = 2
+  ...     initialized = False
+  ...     def __init__(self, **kw):
+  ...         if self.initialized:
+  ...             raise SystemError('__init__ called too many times')
+  ...         self.initialized = True
+  ...         self.__dict__.update(kw)
+  ...     def squared(self):
+  ...         return self.number ** 2
+
+This can be useful to support default values, methods and
+initialization.  Note that if you define an __init__ method, it will be
+called each time the local object is used in a separate thread.  This
+is necessary to initialize each thread's dictionary.
+
+Now if we create a local object:
+
+  >>> mydata = MyLocal(color='red')
+
+Now we have a default number:
+
+  >>> mydata.number
+  2
+
+an initial color:
+
+  >>> mydata.color
+  'red'
+  >>> del mydata.color
+
+And a method that operates on the data:
+
+  >>> mydata.squared()
+  4
+
+As before, we can access the data in a separate thread:
+
+  >>> log = []
+  >>> thread = threading.Thread(target=f)
+  >>> thread.start()
+  >>> thread.join()
+  >>> log
+  [[('color', 'red'), ('initialized', True)], 11]
+
+without affecting this thread's data:
+
+  >>> mydata.number
+  2
+  >>> mydata.color
+  Traceback (most recent call last):
+  ...
+  AttributeError: 'MyLocal' object has no attribute 'color'
+
+Note that subclasses can define slots, but they are not thread
+local. They are shared across threads:
+
+  >>> class MyLocal(local):
+  ...     __slots__ = 'number'
+
+  >>> mydata = MyLocal()
+  >>> mydata.number = 42
+  >>> mydata.color = 'red'
+
+So, the separate thread:
+
+  >>> thread = threading.Thread(target=f)
+  >>> thread.start()
+  >>> thread.join()
+
+affects what we see:
+
+  >>> mydata.number
+  11
+
+>>> del mydata
+"""
+
+from weakref import ref
+from contextlib import contextmanager
+
+__all__ = ["local"]
+
+# We need to use objects from the threading module, but the threading
+# module may also want to use our `local` class, if support for locals
+# isn't compiled in to the `thread` module.  This creates potential problems
+# with circular imports.  For that reason, we don't import `threading`
+# until the bottom of this file (a hack sufficient to worm around the
+# potential problems).  Note that all platforms on CPython do have support
+# for locals in the `thread` module, and there is no circular import problem
+# then, so problems introduced by fiddling the order of imports here won't
+# manifest.
+
+class _localimpl:
+    """A class managing thread-local dicts"""
+    __slots__ = 'key', 'dicts', 'localargs', 'locallock', '__weakref__'
+
+    def __init__(self):
+        # The key used in the Thread objects' attribute dicts.
+        # We keep it a string for speed but make it unlikely to clash with
+        # a "real" attribute.
+        self.key = '_threading_local._localimpl.' + str(id(self))
+        # { id(Thread) -> (ref(Thread), thread-local dict) }
+        self.dicts = {}
+
+    def get_dict(self):
+        """Return the dict for the current thread. Raises KeyError if none
+        defined."""
+        thread = current_thread()
+        return self.dicts[id(thread)][1]
+
+    def create_dict(self):
+        """Create a new dict for the current thread, and return it."""
+        localdict = {}
+        key = self.key
+        thread = current_thread()
+        idt = id(thread)
+        def local_deleted(_, key=key):
+            # When the localimpl is deleted, remove the thread attribute.
+            thread = wrthread()
+            if thread is not None:
+                del thread.__dict__[key]
+        def thread_deleted(_, idt=idt):
+            # When the thread is deleted, remove the local dict.
+            # Note that this is suboptimal if the thread object gets
+            # caught in a reference loop. We would like to be called
+            # as soon as the OS-level thread ends instead.
+            local = wrlocal()
+            if local is not None:
+                dct = local.dicts.pop(idt)
+        wrlocal = ref(self, local_deleted)
+        wrthread = ref(thread, thread_deleted)
+        thread.__dict__[key] = wrlocal
+        self.dicts[idt] = wrthread, localdict
+        return localdict
+
+
+ at contextmanager
+def _patch(self):
+    impl = object.__getattribute__(self, '_local__impl')
+    try:
+        dct = impl.get_dict()
+    except KeyError:
+        dct = impl.create_dict()
+        args, kw = impl.localargs
+        self.__init__(*args, **kw)
+    with impl.locallock:
+        object.__setattr__(self, '__dict__', dct)
+        yield
+
+
+class local:
+    __slots__ = '_local__impl', '__dict__'
+
+    def __new__(cls, *args, **kw):
+        if (args or kw) and (cls.__init__ is object.__init__):
+            raise TypeError("Initialization arguments are not supported")
+        self = object.__new__(cls)
+        impl = _localimpl()
+        impl.localargs = (args, kw)
+        impl.locallock = RLock()
+        object.__setattr__(self, '_local__impl', impl)
+        # We need to create the thread dict in anticipation of
+        # __init__ being called, to make sure we don't call it
+        # again ourselves.
+        impl.create_dict()
+        return self
+
+    def __getattribute__(self, name):
+        with _patch(self):
+            return object.__getattribute__(self, name)
+
+    def __setattr__(self, name, value):
+        if name == '__dict__':
+            raise AttributeError(
+                "%r object attribute '__dict__' is read-only"
+                % self.__class__.__name__)
+        with _patch(self):
+            return object.__setattr__(self, name, value)
+
+    def __delattr__(self, name):
+        if name == '__dict__':
+            raise AttributeError(
+                "%r object attribute '__dict__' is read-only"
+                % self.__class__.__name__)
+        with _patch(self):
+            return object.__delattr__(self, name)
+
+
+from threading import current_thread, RLock
diff --git a/lib-python/3.2/_weakrefset.py b/lib-python/3.2/_weakrefset.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/_weakrefset.py
@@ -0,0 +1,214 @@
+# Access WeakSet through the weakref module.
+# This code is separated-out because it is needed
+# by abc.py to load everything else at startup.
+
+from _weakref import ref
+
+__all__ = ['WeakSet']
+
+
+class _IterationGuard:
+    # This context manager registers itself in the current iterators of the
+    # weak container, such as to delay all removals until the context manager
+    # exits.
+    # This technique should be relatively thread-safe (since sets are).
+
+    def __init__(self, weakcontainer):
+        # Don't create cycles
+        self.weakcontainer = ref(weakcontainer)
+
+    def __enter__(self):
+        w = self.weakcontainer()
+        if w is not None:
+            w._iterating.add(self)
+        return self
+
+    def __exit__(self, e, t, b):
+        w = self.weakcontainer()
+        if w is not None:
+            s = w._iterating
+            s.remove(self)
+            if not s:
+                w._commit_removals()
+
+
+class WeakSet:
+    def __init__(self, data=None):
+        self.data = set()
+        def _remove(item, selfref=ref(self)):
+            self = selfref()
+            if self is not None:
+                if self._iterating:
+                    self._pending_removals.append(item)
+                else:
+                    self.data.discard(item)
+        self._remove = _remove
+        # A list of keys to be removed
+        self._pending_removals = []
+        self._iterating = set()
+        if data is not None:
+            self.update(data)
+
+    def _commit_removals(self):
+        l = self._pending_removals
+        discard = self.data.discard
+        while l:
+            discard(l.pop())
+
+    def __iter__(self):
+        with _IterationGuard(self):
+            for itemref in self.data:
+                item = itemref()
+                if item is not None:
+                    yield item
+
+    def __len__(self):
+        return sum(x() is not None for x in self.data)
+
+    def __contains__(self, item):
+        try:
+            wr = ref(item)
+        except TypeError:
+            return False
+        return wr in self.data
+
+    def __reduce__(self):
+        return (self.__class__, (list(self),),
+                getattr(self, '__dict__', None))
+
+    def add(self, item):
+        if self._pending_removals:
+            self._commit_removals()
+        self.data.add(ref(item, self._remove))
+
+    def clear(self):
+        if self._pending_removals:
+            self._commit_removals()
+        self.data.clear()
+
+    def copy(self):
+        return self.__class__(self)
+
+    def pop(self):
+        if self._pending_removals:
+            self._commit_removals()
+        while True:
+            try:
+                itemref = self.data.pop()
+            except KeyError:
+                raise KeyError('pop from empty WeakSet')
+            item = itemref()
+            if item is not None:
+                return item
+
+    def remove(self, item):
+        if self._pending_removals:
+            self._commit_removals()
+        self.data.remove(ref(item))
+
+    def discard(self, item):
+        if self._pending_removals:
+            self._commit_removals()
+        self.data.discard(ref(item))
+
+    def update(self, other):
+        if self._pending_removals:
+            self._commit_removals()
+        if isinstance(other, self.__class__):
+            self.data.update(other.data)
+        else:
+            for element in other:
+                self.add(element)
+
+    def __ior__(self, other):
+        self.update(other)
+        return self
+
+    # Helper functions for simple delegating methods.
+    def _apply(self, other, method):
+        if not isinstance(other, self.__class__):
+            other = self.__class__(other)
+        newdata = method(other.data)
+        newset = self.__class__()
+        newset.data = newdata
+        return newset
+
+    def difference(self, other):
+        return self._apply(other, self.data.difference)
+    __sub__ = difference
+
+    def difference_update(self, other):
+        if self._pending_removals:
+            self._commit_removals()
+        if self is other:
+            self.data.clear()
+        else:
+            self.data.difference_update(ref(item) for item in other)
+    def __isub__(self, other):
+        if self._pending_removals:
+            self._commit_removals()
+        if self is other:
+            self.data.clear()
+        else:
+            self.data.difference_update(ref(item) for item in other)
+        return self
+
+    def intersection(self, other):
+        return self._apply(other, self.data.intersection)
+    __and__ = intersection
+
+    def intersection_update(self, other):
+        if self._pending_removals:
+            self._commit_removals()
+        self.data.intersection_update(ref(item) for item in other)
+    def __iand__(self, other):
+        if self._pending_removals:
+            self._commit_removals()
+        self.data.intersection_update(ref(item) for item in other)
+        return self
+
+    def issubset(self, other):
+        return self.data.issubset(ref(item) for item in other)
+    __lt__ = issubset
+
+    def __le__(self, other):
+        return self.data <= set(ref(item) for item in other)
+
+    def issuperset(self, other):
+        return self.data.issuperset(ref(item) for item in other)
+    __gt__ = issuperset
+
+    def __ge__(self, other):
+        return self.data >= set(ref(item) for item in other)
+
+    def __eq__(self, other):
+        if not isinstance(other, self.__class__):
+            return NotImplemented
+        return self.data == set(ref(item) for item in other)
+
+    def symmetric_difference(self, other):
+        return self._apply(other, self.data.symmetric_difference)
+    __xor__ = symmetric_difference
+
+    def symmetric_difference_update(self, other):
+        if self._pending_removals:
+            self._commit_removals()
+        if self is other:
+            self.data.clear()
+        else:
+            self.data.symmetric_difference_update(ref(item) for item in other)
+    def __ixor__(self, other):
+        if self._pending_removals:
+            self._commit_removals()
+        if self is other:
+            self.data.clear()
+        else:
+            self.data.symmetric_difference_update(ref(item) for item in other)
+        return self
+
+    def union(self, other):
+        return self._apply(other, self.data.union)
+    __or__ = union
+
+    def isdisjoint(self, other):
+        return len(self.intersection(other)) == 0
diff --git a/lib-python/3.2/abc.py b/lib-python/3.2/abc.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/abc.py
@@ -0,0 +1,211 @@
+# Copyright 2007 Google, Inc. All Rights Reserved.
+# Licensed to PSF under a Contributor Agreement.
+
+"""Abstract Base Classes (ABCs) according to PEP 3119."""
+
+from _weakrefset import WeakSet
+
+def abstractmethod(funcobj):
+    """A decorator indicating abstract methods.
+
+    Requires that the metaclass is ABCMeta or derived from it.  A
+    class that has a metaclass derived from ABCMeta cannot be
+    instantiated unless all of its abstract methods are overridden.
+    The abstract methods can be called using any of the normal
+    'super' call mechanisms.
+
+    Usage:
+
+        class C(metaclass=ABCMeta):
+            @abstractmethod
+            def my_abstract_method(self, ...):
+                ...
+    """
+    funcobj.__isabstractmethod__ = True
+    return funcobj
+
+
+class abstractclassmethod(classmethod):
+    """A decorator indicating abstract classmethods.
+
+    Similar to abstractmethod.
+
+    Usage:
+
+        class C(metaclass=ABCMeta):
+            @abstractclassmethod
+            def my_abstract_classmethod(cls, ...):
+                ...
+    """
+
+    __isabstractmethod__ = True
+
+    def __init__(self, callable):
+        callable.__isabstractmethod__ = True
+        super().__init__(callable)
+
+
+class abstractstaticmethod(staticmethod):
+    """A decorator indicating abstract staticmethods.
+
+    Similar to abstractmethod.
+
+    Usage:
+
+        class C(metaclass=ABCMeta):
+            @abstractstaticmethod
+            def my_abstract_staticmethod(...):
+                ...
+    """
+
+    __isabstractmethod__ = True
+
+    def __init__(self, callable):
+        callable.__isabstractmethod__ = True
+        super().__init__(callable)
+
+
+class abstractproperty(property):
+    """A decorator indicating abstract properties.
+
+    Requires that the metaclass is ABCMeta or derived from it.  A
+    class that has a metaclass derived from ABCMeta cannot be
+    instantiated unless all of its abstract properties are overridden.
+    The abstract properties can be called using any of the normal
+    'super' call mechanisms.
+
+    Usage:
+
+        class C(metaclass=ABCMeta):
+            @abstractproperty
+            def my_abstract_property(self):
+                ...
+
+    This defines a read-only property; you can also define a read-write
+    abstract property using the 'long' form of property declaration:
+
+        class C(metaclass=ABCMeta):
+            def getx(self): ...
+            def setx(self, value): ...
+            x = abstractproperty(getx, setx)
+    """
+    __isabstractmethod__ = True
+
+
+class ABCMeta(type):
+
+    """Metaclass for defining Abstract Base Classes (ABCs).
+
+    Use this metaclass to create an ABC.  An ABC can be subclassed
+    directly, and then acts as a mix-in class.  You can also register
+    unrelated concrete classes (even built-in classes) and unrelated
+    ABCs as 'virtual subclasses' -- these and their descendants will
+    be considered subclasses of the registering ABC by the built-in
+    issubclass() function, but the registering ABC won't show up in
+    their MRO (Method Resolution Order) nor will method
+    implementations defined by the registering ABC be callable (not
+    even via super()).
+
+    """
+
+    # A global counter that is incremented each time a class is
+    # registered as a virtual subclass of anything.  It forces the
+    # negative cache to be cleared before its next use.
+    _abc_invalidation_counter = 0
+
+    def __new__(mcls, name, bases, namespace):
+        cls = super().__new__(mcls, name, bases, namespace)
+        # Compute set of abstract method names
+        abstracts = {name
+                     for name, value in namespace.items()
+                     if getattr(value, "__isabstractmethod__", False)}
+        for base in bases:
+            for name in getattr(base, "__abstractmethods__", set()):
+                value = getattr(cls, name, None)
+                if getattr(value, "__isabstractmethod__", False):
+                    abstracts.add(name)
+        cls.__abstractmethods__ = frozenset(abstracts)
+        # Set up inheritance registry
+        cls._abc_registry = WeakSet()
+        cls._abc_cache = WeakSet()
+        cls._abc_negative_cache = WeakSet()
+        cls._abc_negative_cache_version = ABCMeta._abc_invalidation_counter
+        return cls
+
+    def register(cls, subclass):
+        """Register a virtual subclass of an ABC."""
+        if not isinstance(subclass, type):
+            raise TypeError("Can only register classes")
+        if issubclass(subclass, cls):
+            return  # Already a subclass
+        # Subtle: test for cycles *after* testing for "already a subclass";
+        # this means we allow X.register(X) and interpret it as a no-op.
+        if issubclass(cls, subclass):
+            # This would create a cycle, which is bad for the algorithm below
+            raise RuntimeError("Refusing to create an inheritance cycle")
+        cls._abc_registry.add(subclass)
+        ABCMeta._abc_invalidation_counter += 1  # Invalidate negative cache
+
+    def _dump_registry(cls, file=None):
+        """Debug helper to print the ABC registry."""
+        print("Class: %s.%s" % (cls.__module__, cls.__name__), file=file)
+        print("Inv.counter: %s" % ABCMeta._abc_invalidation_counter, file=file)
+        for name in sorted(cls.__dict__.keys()):
+            if name.startswith("_abc_"):
+                value = getattr(cls, name)
+                print("%s: %r" % (name, value), file=file)
+
+    def __instancecheck__(cls, instance):
+        """Override for isinstance(instance, cls)."""
+        # Inline the cache checking
+        subclass = instance.__class__
+        if subclass in cls._abc_cache:
+            return True
+        subtype = type(instance)
+        if subtype is subclass:
+            if (cls._abc_negative_cache_version ==
+                ABCMeta._abc_invalidation_counter and
+                subclass in cls._abc_negative_cache):
+                return False
+            # Fall back to the subclass check.
+            return cls.__subclasscheck__(subclass)
+        return any(cls.__subclasscheck__(c) for c in {subclass, subtype})
+
+    def __subclasscheck__(cls, subclass):
+        """Override for issubclass(subclass, cls)."""
+        # Check cache
+        if subclass in cls._abc_cache:
+            return True
+        # Check negative cache; may have to invalidate
+        if cls._abc_negative_cache_version < ABCMeta._abc_invalidation_counter:
+            # Invalidate the negative cache
+            cls._abc_negative_cache = WeakSet()
+            cls._abc_negative_cache_version = ABCMeta._abc_invalidation_counter
+        elif subclass in cls._abc_negative_cache:
+            return False
+        # Check the subclass hook
+        ok = cls.__subclasshook__(subclass)
+        if ok is not NotImplemented:
+            assert isinstance(ok, bool)
+            if ok:
+                cls._abc_cache.add(subclass)
+            else:
+                cls._abc_negative_cache.add(subclass)
+            return ok
+        # Check if it's a direct subclass
+        if cls in getattr(subclass, '__mro__', ()):
+            cls._abc_cache.add(subclass)
+            return True
+        # Check if it's a subclass of a registered class (recursive)
+        for rcls in cls._abc_registry:
+            if issubclass(subclass, rcls):
+                cls._abc_cache.add(subclass)
+                return True
+        # Check if it's a subclass of a subclass (recursive)
+        for scls in cls.__subclasses__():
+            if issubclass(subclass, scls):
+                cls._abc_cache.add(subclass)
+                return True
+        # No dice; update negative cache
+        cls._abc_negative_cache.add(subclass)
+        return False
diff --git a/lib-python/3.2/aifc.py b/lib-python/3.2/aifc.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/aifc.py
@@ -0,0 +1,886 @@
+"""Stuff to parse AIFF-C and AIFF files.
+
+Unless explicitly stated otherwise, the description below is true
+both for AIFF-C files and AIFF files.
+
+An AIFF-C file has the following structure.
+
+  +-----------------+
+  | FORM            |
+  +-----------------+
+  | <size>          |
+  +----+------------+
+  |    | AIFC       |
+  |    +------------+
+  |    | <chunks>   |
+  |    |    .       |
+  |    |    .       |
+  |    |    .       |
+  +----+------------+
+
+An AIFF file has the string "AIFF" instead of "AIFC".
+
+A chunk consists of an identifier (4 bytes) followed by a size (4 bytes,
+big endian order), followed by the data.  The size field does not include
+the size of the 8 byte header.
+
+The following chunk types are recognized.
+
+  FVER
+      <version number of AIFF-C defining document> (AIFF-C only).
+  MARK
+      <# of markers> (2 bytes)
+      list of markers:
+          <marker ID> (2 bytes, must be > 0)
+          <position> (4 bytes)
+          <marker name> ("pstring")
+  COMM
+      <# of channels> (2 bytes)
+      <# of sound frames> (4 bytes)
+      <size of the samples> (2 bytes)
+      <sampling frequency> (10 bytes, IEEE 80-bit extended
+          floating point)
+      in AIFF-C files only:
+      <compression type> (4 bytes)
+      <human-readable version of compression type> ("pstring")
+  SSND
+      <offset> (4 bytes, not used by this program)
+      <blocksize> (4 bytes, not used by this program)
+      <sound data>
+
+A pstring consists of 1 byte length, a string of characters, and 0 or 1
+byte pad to make the total length even.
+
+Usage.
+
+Reading AIFF files:
+  f = aifc.open(file, 'r')
+where file is either the name of a file or an open file pointer.
+The open file pointer must have methods read(), seek(), and close().
+In some types of audio files, if the setpos() method is not used,
+the seek() method is not necessary.
+
+This returns an instance of a class with the following public methods:
+  getnchannels()  -- returns number of audio channels (1 for
+             mono, 2 for stereo)
+  getsampwidth()  -- returns sample width in bytes
+  getframerate()  -- returns sampling frequency
+  getnframes()    -- returns number of audio frames
+  getcomptype()   -- returns compression type ('NONE' for AIFF files)
+  getcompname()   -- returns human-readable version of
+             compression type ('not compressed' for AIFF files)
+  getparams() -- returns a tuple consisting of all of the
+             above in the above order
+  getmarkers()    -- get the list of marks in the audio file or None
+             if there are no marks
+  getmark(id) -- get mark with the specified id (raises an error
+             if the mark does not exist)
+  readframes(n)   -- returns at most n frames of audio
+  rewind()    -- rewind to the beginning of the audio stream
+  setpos(pos) -- seek to the specified position
+  tell()      -- return the current position
+  close()     -- close the instance (make it unusable)
+The position returned by tell(), the position given to setpos() and
+the position of marks are all compatible and have nothing to do with
+the actual position in the file.
+The close() method is called automatically when the class instance
+is destroyed.
+
+Writing AIFF files:
+  f = aifc.open(file, 'w')
+where file is either the name of a file or an open file pointer.
+The open file pointer must have methods write(), tell(), seek(), and
+close().
+
+This returns an instance of a class with the following public methods:
+  aiff()      -- create an AIFF file (AIFF-C default)
+  aifc()      -- create an AIFF-C file
+  setnchannels(n) -- set the number of channels
+  setsampwidth(n) -- set the sample width
+  setframerate(n) -- set the frame rate
+  setnframes(n)   -- set the number of frames
+  setcomptype(type, name)
+          -- set the compression type and the
+             human-readable compression type
+  setparams(tuple)
+          -- set all parameters at once
+  setmark(id, pos, name)
+          -- add specified mark to the list of marks
+  tell()      -- return current position in output file (useful
+             in combination with setmark())
+  writeframesraw(data)
+          -- write audio frames without pathing up the
+             file header
+  writeframes(data)
+          -- write audio frames and patch up the file header
+  close()     -- patch up the file header and close the
+             output file
+You should set the parameters before the first writeframesraw or
+writeframes.  The total number of frames does not need to be set,
+but when it is set to the correct value, the header does not have to
+be patched up.
+It is best to first set all parameters, perhaps possibly the
+compression type, and then write audio frames using writeframesraw.
+When all frames have been written, either call writeframes('') or
+close() to patch up the sizes in the header.
+Marks can be added anytime.  If there are any marks, ypu must call
+close() after all frames have been written.
+The close() method is called automatically when the class instance
+is destroyed.
+
+When a file is opened with the extension '.aiff', an AIFF file is
+written, otherwise an AIFF-C file is written.  This default can be
+changed by calling aiff() or aifc() before the first writeframes or
+writeframesraw.
+"""
+
+import struct
+import builtins
+
+__all__ = ["Error", "open", "openfp"]
+
+class Error(Exception):
+    pass
+
+_AIFC_version = 0xA2805140     # Version 1 of AIFF-C
+
+def _read_long(file):
+    try:
+        return struct.unpack('>l', file.read(4))[0]
+    except struct.error:
+        raise EOFError
+
+def _read_ulong(file):
+    try:
+        return struct.unpack('>L', file.read(4))[0]
+    except struct.error:
+        raise EOFError
+
+def _read_short(file):
+    try:
+        return struct.unpack('>h', file.read(2))[0]
+    except struct.error:
+        raise EOFError
+
+def _read_string(file):
+    length = ord(file.read(1))
+    if length == 0:
+        data = b''
+    else:
+        data = file.read(length)
+    if length & 1 == 0:
+        dummy = file.read(1)
+    return data
+
+_HUGE_VAL = 1.79769313486231e+308 # See <limits.h>
+
+def _read_float(f): # 10 bytes
+    expon = _read_short(f) # 2 bytes
+    sign = 1
+    if expon < 0:
+        sign = -1
+        expon = expon + 0x8000
+    himant = _read_ulong(f) # 4 bytes
+    lomant = _read_ulong(f) # 4 bytes
+    if expon == himant == lomant == 0:
+        f = 0.0
+    elif expon == 0x7FFF:
+        f = _HUGE_VAL
+    else:
+        expon = expon - 16383
+        f = (himant * 0x100000000 + lomant) * pow(2.0, expon - 63)
+    return sign * f
+
+def _write_short(f, x):
+    f.write(struct.pack('>h', x))
+
+def _write_long(f, x):
+    f.write(struct.pack('>L', x))
+
+def _write_string(f, s):
+    if len(s) > 255:
+        raise ValueError("string exceeds maximum pstring length")
+    f.write(struct.pack('b', len(s)))
+    f.write(s)
+    if len(s) & 1 == 0:
+        f.write(b'\x00')
+
+def _write_float(f, x):
+    import math
+    if x < 0:
+        sign = 0x8000
+        x = x * -1
+    else:
+        sign = 0
+    if x == 0:
+        expon = 0
+        himant = 0
+        lomant = 0
+    else:
+        fmant, expon = math.frexp(x)
+        if expon > 16384 or fmant >= 1:     # Infinity or NaN
+            expon = sign|0x7FFF
+            himant = 0
+            lomant = 0
+        else:                   # Finite
+            expon = expon + 16382
+            if expon < 0:           # denormalized
+                fmant = math.ldexp(fmant, expon)
+                expon = 0
+            expon = expon | sign
+            fmant = math.ldexp(fmant, 32)
+            fsmant = math.floor(fmant)
+            himant = int(fsmant)
+            fmant = math.ldexp(fmant - fsmant, 32)
+            fsmant = math.floor(fmant)
+            lomant = int(fsmant)
+    _write_short(f, expon)
+    _write_long(f, himant)
+    _write_long(f, lomant)
+
+from chunk import Chunk
+
+class Aifc_read:
+    # Variables used in this class:
+    #
+    # These variables are available to the user though appropriate
+    # methods of this class:
+    # _file -- the open file with methods read(), close(), and seek()
+    #       set through the __init__() method
+    # _nchannels -- the number of audio channels
+    #       available through the getnchannels() method
+    # _nframes -- the number of audio frames
+    #       available through the getnframes() method
+    # _sampwidth -- the number of bytes per audio sample
+    #       available through the getsampwidth() method
+    # _framerate -- the sampling frequency
+    #       available through the getframerate() method
+    # _comptype -- the AIFF-C compression type ('NONE' if AIFF)
+    #       available through the getcomptype() method
+    # _compname -- the human-readable AIFF-C compression type
+    #       available through the getcomptype() method
+    # _markers -- the marks in the audio file
+    #       available through the getmarkers() and getmark()
+    #       methods
+    # _soundpos -- the position in the audio stream
+    #       available through the tell() method, set through the
+    #       setpos() method
+    #
+    # These variables are used internally only:
+    # _version -- the AIFF-C version number
+    # _decomp -- the decompressor from builtin module cl
+    # _comm_chunk_read -- 1 iff the COMM chunk has been read
+    # _aifc -- 1 iff reading an AIFF-C file
+    # _ssnd_seek_needed -- 1 iff positioned correctly in audio
+    #       file for readframes()
+    # _ssnd_chunk -- instantiation of a chunk class for the SSND chunk
+    # _framesize -- size of one frame in the file
+
+    def initfp(self, file):
+        self._version = 0
+        self._convert = None
+        self._markers = []
+        self._soundpos = 0
+        self._file = file
+        chunk = Chunk(file)
+        if chunk.getname() != b'FORM':
+            raise Error('file does not start with FORM id')
+        formdata = chunk.read(4)
+        if formdata == b'AIFF':
+            self._aifc = 0
+        elif formdata == b'AIFC':
+            self._aifc = 1
+        else:
+            raise Error('not an AIFF or AIFF-C file')
+        self._comm_chunk_read = 0
+        while 1:
+            self._ssnd_seek_needed = 1
+            try:
+                chunk = Chunk(self._file)
+            except EOFError:
+                break
+            chunkname = chunk.getname()
+            if chunkname == b'COMM':
+                self._read_comm_chunk(chunk)
+                self._comm_chunk_read = 1
+            elif chunkname == b'SSND':
+                self._ssnd_chunk = chunk
+                dummy = chunk.read(8)
+                self._ssnd_seek_needed = 0
+            elif chunkname == b'FVER':
+                self._version = _read_ulong(chunk)
+            elif chunkname == b'MARK':
+                self._readmark(chunk)
+            chunk.skip()
+        if not self._comm_chunk_read or not self._ssnd_chunk:
+            raise Error('COMM chunk and/or SSND chunk missing')
+
+    def __init__(self, f):
+        if isinstance(f, str):
+            f = builtins.open(f, 'rb')
+        # else, assume it is an open file object already
+        self.initfp(f)
+
+    #
+    # User visible methods.
+    #
+    def getfp(self):
+        return self._file
+
+    def rewind(self):
+        self._ssnd_seek_needed = 1
+        self._soundpos = 0
+
+    def close(self):
+        self._file.close()
+
+    def tell(self):
+        return self._soundpos
+
+    def getnchannels(self):
+        return self._nchannels
+
+    def getnframes(self):
+        return self._nframes
+
+    def getsampwidth(self):
+        return self._sampwidth
+
+    def getframerate(self):
+        return self._framerate
+
+    def getcomptype(self):
+        return self._comptype
+
+    def getcompname(self):
+        return self._compname
+
+##  def getversion(self):
+##      return self._version
+
+    def getparams(self):
+        return self.getnchannels(), self.getsampwidth(), \
+              self.getframerate(), self.getnframes(), \
+              self.getcomptype(), self.getcompname()
+
+    def getmarkers(self):
+        if len(self._markers) == 0:
+            return None
+        return self._markers
+
+    def getmark(self, id):
+        for marker in self._markers:
+            if id == marker[0]:
+                return marker
+        raise Error('marker {0!r} does not exist'.format(id))
+
+    def setpos(self, pos):
+        if pos < 0 or pos > self._nframes:
+            raise Error('position not in range')
+        self._soundpos = pos
+        self._ssnd_seek_needed = 1
+
+    def readframes(self, nframes):
+        if self._ssnd_seek_needed:
+            self._ssnd_chunk.seek(0)
+            dummy = self._ssnd_chunk.read(8)
+            pos = self._soundpos * self._framesize
+            if pos:
+                self._ssnd_chunk.seek(pos + 8)
+            self._ssnd_seek_needed = 0
+        if nframes == 0:
+            return b''
+        data = self._ssnd_chunk.read(nframes * self._framesize)
+        if self._convert and data:
+            data = self._convert(data)
+        self._soundpos = self._soundpos + len(data) // (self._nchannels
+                                                        * self._sampwidth)
+        return data
+
+    #
+    # Internal methods.
+    #
+
+    def _alaw2lin(self, data):
+        import audioop
+        return audioop.alaw2lin(data, 2)
+
+    def _ulaw2lin(self, data):
+        import audioop
+        return audioop.ulaw2lin(data, 2)
+
+    def _adpcm2lin(self, data):
+        import audioop
+        if not hasattr(self, '_adpcmstate'):
+            # first time
+            self._adpcmstate = None
+        data, self._adpcmstate = audioop.adpcm2lin(data, 2, self._adpcmstate)
+        return data
+
+    def _read_comm_chunk(self, chunk):
+        self._nchannels = _read_short(chunk)
+        self._nframes = _read_long(chunk)
+        self._sampwidth = (_read_short(chunk) + 7) // 8
+        self._framerate = int(_read_float(chunk))
+        self._framesize = self._nchannels * self._sampwidth
+        if self._aifc:
+            #DEBUG: SGI's soundeditor produces a bad size :-(
+            kludge = 0
+            if chunk.chunksize == 18:
+                kludge = 1
+                print('Warning: bad COMM chunk size')
+                chunk.chunksize = 23
+            #DEBUG end
+            self._comptype = chunk.read(4)
+            #DEBUG start
+            if kludge:
+                length = ord(chunk.file.read(1))
+                if length & 1 == 0:
+                    length = length + 1
+                chunk.chunksize = chunk.chunksize + length
+                chunk.file.seek(-1, 1)
+            #DEBUG end
+            self._compname = _read_string(chunk)
+            if self._comptype != b'NONE':
+                if self._comptype == b'G722':
+                    self._convert = self._adpcm2lin
+                    self._framesize = self._framesize // 4
+                elif self._comptype in (b'ulaw', b'ULAW'):
+                    self._convert = self._ulaw2lin
+                    self._framesize = self._framesize // 2
+                elif self._comptype in (b'alaw', b'ALAW'):
+                    self._convert = self._alaw2lin
+                    self._framesize = self._framesize // 2
+                else:
+                    raise Error('unsupported compression type')
+        else:
+            self._comptype = b'NONE'
+            self._compname = b'not compressed'
+
+    def _readmark(self, chunk):
+        nmarkers = _read_short(chunk)
+        # Some files appear to contain invalid counts.
+        # Cope with this by testing for EOF.
+        try:
+            for i in range(nmarkers):
+                id = _read_short(chunk)
+                pos = _read_long(chunk)
+                name = _read_string(chunk)
+                if pos or name:
+                    # some files appear to have
+                    # dummy markers consisting of
+                    # a position 0 and name ''
+                    self._markers.append((id, pos, name))
+        except EOFError:
+            print('Warning: MARK chunk contains only', end=' ')
+            print(len(self._markers), end=' ')
+            if len(self._markers) == 1: print('marker', end=' ')
+            else: print('markers', end=' ')
+            print('instead of', nmarkers)
+
+class Aifc_write:
+    # Variables used in this class:
+    #
+    # These variables are user settable through appropriate methods
+    # of this class:
+    # _file -- the open file with methods write(), close(), tell(), seek()
+    #       set through the __init__() method
+    # _comptype -- the AIFF-C compression type ('NONE' in AIFF)
+    #       set through the setcomptype() or setparams() method
+    # _compname -- the human-readable AIFF-C compression type
+    #       set through the setcomptype() or setparams() method
+    # _nchannels -- the number of audio channels
+    #       set through the setnchannels() or setparams() method
+    # _sampwidth -- the number of bytes per audio sample
+    #       set through the setsampwidth() or setparams() method
+    # _framerate -- the sampling frequency
+    #       set through the setframerate() or setparams() method
+    # _nframes -- the number of audio frames written to the header
+    #       set through the setnframes() or setparams() method
+    # _aifc -- whether we're writing an AIFF-C file or an AIFF file
+    #       set through the aifc() method, reset through the
+    #       aiff() method
+    #
+    # These variables are used internally only:
+    # _version -- the AIFF-C version number
+    # _comp -- the compressor from builtin module cl
+    # _nframeswritten -- the number of audio frames actually written
+    # _datalength -- the size of the audio samples written to the header
+    # _datawritten -- the size of the audio samples actually written
+
+    def __init__(self, f):
+        if isinstance(f, str):
+            filename = f
+            f = builtins.open(f, 'wb')
+        else:
+            # else, assume it is an open file object already
+            filename = '???'
+        self.initfp(f)
+        if filename[-5:] == '.aiff':
+            self._aifc = 0
+        else:
+            self._aifc = 1
+
+    def initfp(self, file):
+        self._file = file
+        self._version = _AIFC_version
+        self._comptype = b'NONE'
+        self._compname = b'not compressed'
+        self._convert = None
+        self._nchannels = 0
+        self._sampwidth = 0
+        self._framerate = 0
+        self._nframes = 0
+        self._nframeswritten = 0
+        self._datawritten = 0
+        self._datalength = 0
+        self._markers = []
+        self._marklength = 0
+        self._aifc = 1      # AIFF-C is default
+
+    def __del__(self):
+        if self._file:
+            self.close()
+
+    #
+    # User visible methods.
+    #
+    def aiff(self):
+        if self._nframeswritten:
+            raise Error('cannot change parameters after starting to write')
+        self._aifc = 0
+
+    def aifc(self):
+        if self._nframeswritten:
+            raise Error('cannot change parameters after starting to write')
+        self._aifc = 1
+
+    def setnchannels(self, nchannels):
+        if self._nframeswritten:
+            raise Error('cannot change parameters after starting to write')
+        if nchannels < 1:
+            raise Error('bad # of channels')
+        self._nchannels = nchannels
+
+    def getnchannels(self):
+        if not self._nchannels:
+            raise Error('number of channels not set')
+        return self._nchannels
+
+    def setsampwidth(self, sampwidth):
+        if self._nframeswritten:
+            raise Error('cannot change parameters after starting to write')
+        if sampwidth < 1 or sampwidth > 4:
+            raise Error('bad sample width')
+        self._sampwidth = sampwidth
+
+    def getsampwidth(self):
+        if not self._sampwidth:
+            raise Error('sample width not set')
+        return self._sampwidth
+
+    def setframerate(self, framerate):
+        if self._nframeswritten:
+            raise Error('cannot change parameters after starting to write')
+        if framerate <= 0:
+            raise Error('bad frame rate')
+        self._framerate = framerate
+
+    def getframerate(self):
+        if not self._framerate:
+            raise Error('frame rate not set')
+        return self._framerate
+
+    def setnframes(self, nframes):
+        if self._nframeswritten:
+            raise Error('cannot change parameters after starting to write')
+        self._nframes = nframes
+
+    def getnframes(self):
+        return self._nframeswritten
+
+    def setcomptype(self, comptype, compname):
+        if self._nframeswritten:
+            raise Error('cannot change parameters after starting to write')
+        if comptype not in (b'NONE', b'ulaw', b'ULAW',
+                            b'alaw', b'ALAW', b'G722'):
+            raise Error('unsupported compression type')
+        self._comptype = comptype
+        self._compname = compname
+
+    def getcomptype(self):
+        return self._comptype
+
+    def getcompname(self):
+        return self._compname
+
+##  def setversion(self, version):
+##      if self._nframeswritten:
+##          raise Error, 'cannot change parameters after starting to write'
+##      self._version = version
+
+    def setparams(self, params):
+        nchannels, sampwidth, framerate, nframes, comptype, compname = params
+        if self._nframeswritten:
+            raise Error('cannot change parameters after starting to write')
+        if comptype not in (b'NONE', b'ulaw', b'ULAW',
+                            b'alaw', b'ALAW', b'G722'):
+            raise Error('unsupported compression type')
+        self.setnchannels(nchannels)
+        self.setsampwidth(sampwidth)
+        self.setframerate(framerate)
+        self.setnframes(nframes)
+        self.setcomptype(comptype, compname)
+
+    def getparams(self):
+        if not self._nchannels or not self._sampwidth or not self._framerate:
+            raise Error('not all parameters set')
+        return self._nchannels, self._sampwidth, self._framerate, \
+              self._nframes, self._comptype, self._compname
+
+    def setmark(self, id, pos, name):
+        if id <= 0:
+            raise Error('marker ID must be > 0')
+        if pos < 0:
+            raise Error('marker position must be >= 0')
+        if not isinstance(name, str):
+            raise Error('marker name must be a string')
+        for i in range(len(self._markers)):
+            if id == self._markers[i][0]:
+                self._markers[i] = id, pos, name
+                return
+        self._markers.append((id, pos, name))
+
+    def getmark(self, id):
+        for marker in self._markers:
+            if id == marker[0]:
+                return marker
+        raise Error('marker {0!r} does not exist'.format(id))
+
+    def getmarkers(self):
+        if len(self._markers) == 0:
+            return None
+        return self._markers
+
+    def tell(self):
+        return self._nframeswritten
+
+    def writeframesraw(self, data):
+        self._ensure_header_written(len(data))
+        nframes = len(data) // (self._sampwidth * self._nchannels)
+        if self._convert:
+            data = self._convert(data)
+        self._file.write(data)
+        self._nframeswritten = self._nframeswritten + nframes
+        self._datawritten = self._datawritten + len(data)
+
+    def writeframes(self, data):
+        self.writeframesraw(data)
+        if self._nframeswritten != self._nframes or \
+              self._datalength != self._datawritten:
+            self._patchheader()
+
+    def close(self):
+        self._ensure_header_written(0)
+        if self._datawritten & 1:
+            # quick pad to even size
+            self._file.write(b'\x00')
+            self._datawritten = self._datawritten + 1
+        self._writemarkers()
+        if self._nframeswritten != self._nframes or \
+              self._datalength != self._datawritten or \
+              self._marklength:
+            self._patchheader()
+        # Prevent ref cycles
+        self._convert = None
+        self._file.close()
+
+    #
+    # Internal methods.
+    #
+
+    def _lin2alaw(self, data):
+        import audioop
+        return audioop.lin2alaw(data, 2)
+
+    def _lin2ulaw(self, data):
+        import audioop
+        return audioop.lin2ulaw(data, 2)
+
+    def _lin2adpcm(self, data):
+        import audioop
+        if not hasattr(self, '_adpcmstate'):
+            self._adpcmstate = None
+        data, self._adpcmstate = audioop.lin2adpcm(data, 2, self._adpcmstate)
+        return data
+
+    def _ensure_header_written(self, datasize):
+        if not self._nframeswritten:
+            if self._comptype in (b'ULAW', b'ALAW'):
+                if not self._sampwidth:
+                    self._sampwidth = 2
+                if self._sampwidth != 2:
+                    raise Error('sample width must be 2 when compressing '
+                                'with ulaw/ULAW or alaw/ALAW')
+            if self._comptype == b'G722':
+                if not self._sampwidth:
+                    self._sampwidth = 2
+                if self._sampwidth != 2:
+                    raise Error('sample width must be 2 when compressing '
+                                'with G7.22 (ADPCM)')
+            if not self._nchannels:
+                raise Error('# channels not specified')
+            if not self._sampwidth:
+                raise Error('sample width not specified')
+            if not self._framerate:
+                raise Error('sampling rate not specified')
+            self._write_header(datasize)
+
+    def _init_compression(self):
+        if self._comptype == b'G722':
+            self._convert = self._lin2adpcm
+        elif self._comptype in (b'ulaw', b'ULAW'):
+            self._convert = self._lin2ulaw
+        elif self._comptype in (b'alaw', b'ALAW'):
+            self._convert = self._lin2alaw
+        else:
+            raise Error('unsupported compression type')
+
+    def _write_header(self, initlength):
+        if self._aifc and self._comptype != b'NONE':
+            self._init_compression()
+        self._file.write(b'FORM')
+        if not self._nframes:
+            self._nframes = initlength // (self._nchannels * self._sampwidth)
+        self._datalength = self._nframes * self._nchannels * self._sampwidth
+        if self._datalength & 1:
+            self._datalength = self._datalength + 1
+        if self._aifc:
+            if self._comptype in (b'ulaw', b'ULAW', b'alaw', b'ALAW'):
+                self._datalength = self._datalength // 2
+                if self._datalength & 1:
+                    self._datalength = self._datalength + 1
+            elif self._comptype == b'G722':
+                self._datalength = (self._datalength + 3) // 4
+                if self._datalength & 1:
+                    self._datalength = self._datalength + 1
+        self._form_length_pos = self._file.tell()
+        commlength = self._write_form_length(self._datalength)
+        if self._aifc:
+            self._file.write(b'AIFC')
+            self._file.write(b'FVER')
+            _write_long(self._file, 4)
+            _write_long(self._file, self._version)
+        else:
+            self._file.write(b'AIFF')
+        self._file.write(b'COMM')
+        _write_long(self._file, commlength)
+        _write_short(self._file, self._nchannels)
+        self._nframes_pos = self._file.tell()
+        _write_long(self._file, self._nframes)
+        _write_short(self._file, self._sampwidth * 8)
+        _write_float(self._file, self._framerate)
+        if self._aifc:
+            self._file.write(self._comptype)
+            _write_string(self._file, self._compname)
+        self._file.write(b'SSND')
+        self._ssnd_length_pos = self._file.tell()
+        _write_long(self._file, self._datalength + 8)
+        _write_long(self._file, 0)
+        _write_long(self._file, 0)
+
+    def _write_form_length(self, datalength):
+        if self._aifc:
+            commlength = 18 + 5 + len(self._compname)
+            if commlength & 1:
+                commlength = commlength + 1
+            verslength = 12
+        else:
+            commlength = 18
+            verslength = 0
+        _write_long(self._file, 4 + verslength + self._marklength + \
+                    8 + commlength + 16 + datalength)
+        return commlength
+
+    def _patchheader(self):
+        curpos = self._file.tell()
+        if self._datawritten & 1:
+            datalength = self._datawritten + 1
+            self._file.write(b'\x00')
+        else:
+            datalength = self._datawritten
+        if datalength == self._datalength and \
+              self._nframes == self._nframeswritten and \
+              self._marklength == 0:
+            self._file.seek(curpos, 0)
+            return
+        self._file.seek(self._form_length_pos, 0)
+        dummy = self._write_form_length(datalength)
+        self._file.seek(self._nframes_pos, 0)
+        _write_long(self._file, self._nframeswritten)
+        self._file.seek(self._ssnd_length_pos, 0)
+        _write_long(self._file, datalength + 8)
+        self._file.seek(curpos, 0)
+        self._nframes = self._nframeswritten
+        self._datalength = datalength
+
+    def _writemarkers(self):
+        if len(self._markers) == 0:
+            return
+        self._file.write(b'MARK')
+        length = 2
+        for marker in self._markers:
+            id, pos, name = marker
+            length = length + len(name) + 1 + 6
+            if len(name) & 1 == 0:
+                length = length + 1
+        _write_long(self._file, length)
+        self._marklength = length + 8
+        _write_short(self._file, len(self._markers))
+        for marker in self._markers:
+            id, pos, name = marker
+            _write_short(self._file, id)
+            _write_long(self._file, pos)
+            _write_string(self._file, name)
+
+def open(f, mode=None):
+    if mode is None:
+        if hasattr(f, 'mode'):
+            mode = f.mode
+        else:
+            mode = 'rb'
+    if mode in ('r', 'rb'):
+        return Aifc_read(f)
+    elif mode in ('w', 'wb'):
+        return Aifc_write(f)
+    else:
+        raise Error("mode must be 'r', 'rb', 'w', or 'wb'")
+
+openfp = open # B/W compatibility
+
+if __name__ == '__main__':
+    import sys
+    if not sys.argv[1:]:
+        sys.argv.append('/usr/demos/data/audio/bach.aiff')
+    fn = sys.argv[1]
+    f = open(fn, 'r')
+    print("Reading", fn)
+    print("nchannels =", f.getnchannels())
+    print("nframes   =", f.getnframes())
+    print("sampwidth =", f.getsampwidth())
+    print("framerate =", f.getframerate())
+    print("comptype  =", f.getcomptype())
+    print("compname  =", f.getcompname())
+    if sys.argv[2:]:
+        gn = sys.argv[2]
+        print("Writing", gn)
+        g = open(gn, 'w')
+        g.setparams(f.getparams())
+        while 1:
+            data = f.readframes(1024)
+            if not data:
+                break
+            g.writeframes(data)
+        g.close()
+        f.close()
+        print("Done.")
diff --git a/lib-python/3.2/antigravity.py b/lib-python/3.2/antigravity.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/antigravity.py
@@ -0,0 +1,17 @@
+
+import webbrowser
+import hashlib
+
+webbrowser.open("http://xkcd.com/353/")
+
+def geohash(latitude, longitude, datedow):
+    '''Compute geohash() using the Munroe algorithm.
+
+    >>> geohash(37.421542, -122.085589, b'2005-05-26-10458.68')
+    37.857713 -122.544543
+
+    '''
+    # http://xkcd.com/426/
+    h = hashlib.md5(datedow).hexdigest()
+    p, q = [('%f' % float.fromhex('0.' + x)) for x in (h[:16], h[16:32])]
+    print('%d%s %d%s' % (latitude, p[1:], longitude, q[1:]))
diff --git a/lib-python/3.2/argparse.py b/lib-python/3.2/argparse.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/argparse.py
@@ -0,0 +1,2366 @@
+# Author: Steven J. Bethard <steven.bethard at gmail.com>.
+
+"""Command-line parsing library
+
+This module is an optparse-inspired command-line parsing library that:
+
+    - handles both optional and positional arguments
+    - produces highly informative usage messages
+    - supports parsers that dispatch to sub-parsers
+
+The following is a simple usage example that sums integers from the
+command-line and writes the result to a file::
+
+    parser = argparse.ArgumentParser(
+        description='sum the integers at the command line')
+    parser.add_argument(
+        'integers', metavar='int', nargs='+', type=int,
+        help='an integer to be summed')
+    parser.add_argument(
+        '--log', default=sys.stdout, type=argparse.FileType('w'),
+        help='the file where the sum should be written')
+    args = parser.parse_args()
+    args.log.write('%s' % sum(args.integers))
+    args.log.close()
+
+The module contains the following public classes:
+
+    - ArgumentParser -- The main entry point for command-line parsing. As the
+        example above shows, the add_argument() method is used to populate
+        the parser with actions for optional and positional arguments. Then
+        the parse_args() method is invoked to convert the args at the
+        command-line into an object with attributes.
+
+    - ArgumentError -- The exception raised by ArgumentParser objects when
+        there are errors with the parser's actions. Errors raised while
+        parsing the command-line are caught by ArgumentParser and emitted
+        as command-line messages.
+
+    - FileType -- A factory for defining types of files to be created. As the
+        example above shows, instances of FileType are typically passed as
+        the type= argument of add_argument() calls.
+
+    - Action -- The base class for parser actions. Typically actions are
+        selected by passing strings like 'store_true' or 'append_const' to
+        the action= argument of add_argument(). However, for greater
+        customization of ArgumentParser actions, subclasses of Action may
+        be defined and passed as the action= argument.
+
+    - HelpFormatter, RawDescriptionHelpFormatter, RawTextHelpFormatter,
+        ArgumentDefaultsHelpFormatter -- Formatter classes which
+        may be passed as the formatter_class= argument to the
+        ArgumentParser constructor. HelpFormatter is the default,
+        RawDescriptionHelpFormatter and RawTextHelpFormatter tell the parser
+        not to change the formatting for help text, and
+        ArgumentDefaultsHelpFormatter adds information about argument defaults
+        to the help.
+
+All other classes in this module are considered implementation details.
+(Also note that HelpFormatter and RawDescriptionHelpFormatter are only
+considered public as object names -- the API of the formatter objects is
+still considered an implementation detail.)
+"""
+
+__version__ = '1.1'
+__all__ = [
+    'ArgumentParser',
+    'ArgumentError',
+    'ArgumentTypeError',
+    'FileType',
+    'HelpFormatter',
+    'ArgumentDefaultsHelpFormatter',
+    'RawDescriptionHelpFormatter',
+    'RawTextHelpFormatter',
+    'Namespace',
+    'Action',
+    'ONE_OR_MORE',
+    'OPTIONAL',
+    'PARSER',
+    'REMAINDER',
+    'SUPPRESS',
+    'ZERO_OR_MORE',
+]
+
+
+import collections as _collections
+import copy as _copy
+import os as _os
+import re as _re
+import sys as _sys
+import textwrap as _textwrap
+
+from gettext import gettext as _, ngettext
+
+
+def _callable(obj):
+    return hasattr(obj, '__call__') or hasattr(obj, '__bases__')
+
+
+SUPPRESS = '==SUPPRESS=='
+
+OPTIONAL = '?'
+ZERO_OR_MORE = '*'
+ONE_OR_MORE = '+'
+PARSER = 'A...'
+REMAINDER = '...'
+_UNRECOGNIZED_ARGS_ATTR = '_unrecognized_args'
+
+# =============================
+# Utility functions and classes
+# =============================
+
+class _AttributeHolder(object):
+    """Abstract base class that provides __repr__.
+
+    The __repr__ method returns a string in the format::
+        ClassName(attr=name, attr=name, ...)
+    The attributes are determined either by a class-level attribute,
+    '_kwarg_names', or by inspecting the instance __dict__.
+    """
+
+    def __repr__(self):
+        type_name = type(self).__name__
+        arg_strings = []
+        for arg in self._get_args():
+            arg_strings.append(repr(arg))
+        for name, value in self._get_kwargs():
+            arg_strings.append('%s=%r' % (name, value))
+        return '%s(%s)' % (type_name, ', '.join(arg_strings))
+
+    def _get_kwargs(self):
+        return sorted(self.__dict__.items())
+
+    def _get_args(self):
+        return []
+
+
+def _ensure_value(namespace, name, value):
+    if getattr(namespace, name, None) is None:
+        setattr(namespace, name, value)
+    return getattr(namespace, name)
+
+
+# ===============
+# Formatting Help
+# ===============
+
+class HelpFormatter(object):
+    """Formatter for generating usage messages and argument help strings.
+
+    Only the name of this class is considered a public API. All the methods
+    provided by the class are considered an implementation detail.
+    """
+
+    def __init__(self,
+                 prog,
+                 indent_increment=2,
+                 max_help_position=24,
+                 width=None):
+
+        # default setting for width
+        if width is None:
+            try:
+                width = int(_os.environ['COLUMNS'])
+            except (KeyError, ValueError):
+                width = 80
+            width -= 2
+
+        self._prog = prog
+        self._indent_increment = indent_increment
+        self._max_help_position = max_help_position
+        self._width = width
+
+        self._current_indent = 0
+        self._level = 0
+        self._action_max_length = 0
+
+        self._root_section = self._Section(self, None)
+        self._current_section = self._root_section
+
+        self._whitespace_matcher = _re.compile(r'\s+')
+        self._long_break_matcher = _re.compile(r'\n\n\n+')
+
+    # ===============================
+    # Section and indentation methods
+    # ===============================
+    def _indent(self):
+        self._current_indent += self._indent_increment
+        self._level += 1
+
+    def _dedent(self):
+        self._current_indent -= self._indent_increment
+        assert self._current_indent >= 0, 'Indent decreased below 0.'
+        self._level -= 1
+
+    class _Section(object):
+
+        def __init__(self, formatter, parent, heading=None):
+            self.formatter = formatter
+            self.parent = parent
+            self.heading = heading
+            self.items = []
+
+        def format_help(self):
+            # format the indented section
+            if self.parent is not None:
+                self.formatter._indent()
+            join = self.formatter._join_parts
+            for func, args in self.items:
+                func(*args)
+            item_help = join([func(*args) for func, args in self.items])
+            if self.parent is not None:
+                self.formatter._dedent()
+
+            # return nothing if the section was empty
+            if not item_help:
+                return ''
+
+            # add the heading if the section was non-empty
+            if self.heading is not SUPPRESS and self.heading is not None:
+                current_indent = self.formatter._current_indent
+                heading = '%*s%s:\n' % (current_indent, '', self.heading)
+            else:
+                heading = ''
+
+            # join the section-initial newline, the heading and the help
+            return join(['\n', heading, item_help, '\n'])
+
+    def _add_item(self, func, args):
+        self._current_section.items.append((func, args))
+
+    # ========================
+    # Message building methods
+    # ========================
+    def start_section(self, heading):
+        self._indent()
+        section = self._Section(self, self._current_section, heading)
+        self._add_item(section.format_help, [])
+        self._current_section = section
+
+    def end_section(self):
+        self._current_section = self._current_section.parent
+        self._dedent()
+
+    def add_text(self, text):
+        if text is not SUPPRESS and text is not None:
+            self._add_item(self._format_text, [text])
+
+    def add_usage(self, usage, actions, groups, prefix=None):
+        if usage is not SUPPRESS:
+            args = usage, actions, groups, prefix
+            self._add_item(self._format_usage, args)
+
+    def add_argument(self, action):
+        if action.help is not SUPPRESS:
+
+            # find all invocations
+            get_invocation = self._format_action_invocation
+            invocations = [get_invocation(action)]
+            for subaction in self._iter_indented_subactions(action):
+                invocations.append(get_invocation(subaction))
+
+            # update the maximum item length
+            invocation_length = max([len(s) for s in invocations])
+            action_length = invocation_length + self._current_indent
+            self._action_max_length = max(self._action_max_length,
+                                          action_length)
+
+            # add the item to the list
+            self._add_item(self._format_action, [action])
+
+    def add_arguments(self, actions):
+        for action in actions:
+            self.add_argument(action)
+
+    # =======================
+    # Help-formatting methods
+    # =======================
+    def format_help(self):
+        help = self._root_section.format_help()
+        if help:
+            help = self._long_break_matcher.sub('\n\n', help)
+            help = help.strip('\n') + '\n'
+        return help
+
+    def _join_parts(self, part_strings):
+        return ''.join([part
+                        for part in part_strings
+                        if part and part is not SUPPRESS])
+
+    def _format_usage(self, usage, actions, groups, prefix):
+        if prefix is None:
+            prefix = _('usage: ')
+
+        # if usage is specified, use that
+        if usage is not None:
+            usage = usage % dict(prog=self._prog)
+
+        # if no optionals or positionals are available, usage is just prog
+        elif usage is None and not actions:
+            usage = '%(prog)s' % dict(prog=self._prog)
+
+        # if optionals and positionals are available, calculate usage
+        elif usage is None:
+            prog = '%(prog)s' % dict(prog=self._prog)
+
+            # split optionals from positionals
+            optionals = []
+            positionals = []
+            for action in actions:
+                if action.option_strings:
+                    optionals.append(action)
+                else:
+                    positionals.append(action)
+
+            # build full usage string
+            format = self._format_actions_usage
+            action_usage = format(optionals + positionals, groups)
+            usage = ' '.join([s for s in [prog, action_usage] if s])
+
+            # wrap the usage parts if it's too long
+            text_width = self._width - self._current_indent
+            if len(prefix) + len(usage) > text_width:
+
+                # break usage into wrappable parts
+                part_regexp = r'\(.*?\)+|\[.*?\]+|\S+'
+                opt_usage = format(optionals, groups)
+                pos_usage = format(positionals, groups)
+                opt_parts = _re.findall(part_regexp, opt_usage)
+                pos_parts = _re.findall(part_regexp, pos_usage)
+                assert ' '.join(opt_parts) == opt_usage
+                assert ' '.join(pos_parts) == pos_usage
+
+                # helper for wrapping lines
+                def get_lines(parts, indent, prefix=None):
+                    lines = []
+                    line = []
+                    if prefix is not None:
+                        line_len = len(prefix) - 1
+                    else:
+                        line_len = len(indent) - 1
+                    for part in parts:
+                        if line_len + 1 + len(part) > text_width:
+                            lines.append(indent + ' '.join(line))
+                            line = []
+                            line_len = len(indent) - 1
+                        line.append(part)
+                        line_len += len(part) + 1
+                    if line:
+                        lines.append(indent + ' '.join(line))
+                    if prefix is not None:
+                        lines[0] = lines[0][len(indent):]
+                    return lines
+
+                # if prog is short, follow it with optionals or positionals
+                if len(prefix) + len(prog) <= 0.75 * text_width:
+                    indent = ' ' * (len(prefix) + len(prog) + 1)
+                    if opt_parts:
+                        lines = get_lines([prog] + opt_parts, indent, prefix)
+                        lines.extend(get_lines(pos_parts, indent))
+                    elif pos_parts:
+                        lines = get_lines([prog] + pos_parts, indent, prefix)
+                    else:
+                        lines = [prog]
+
+                # if prog is long, put it on its own line
+                else:
+                    indent = ' ' * len(prefix)
+                    parts = opt_parts + pos_parts
+                    lines = get_lines(parts, indent)
+                    if len(lines) > 1:
+                        lines = []
+                        lines.extend(get_lines(opt_parts, indent))
+                        lines.extend(get_lines(pos_parts, indent))
+                    lines = [prog] + lines
+
+                # join lines into usage
+                usage = '\n'.join(lines)
+
+        # prefix with 'usage:'
+        return '%s%s\n\n' % (prefix, usage)
+
+    def _format_actions_usage(self, actions, groups):
+        # find group indices and identify actions in groups
+        group_actions = set()
+        inserts = {}
+        for group in groups:
+            try:
+                start = actions.index(group._group_actions[0])
+            except ValueError:
+                continue
+            else:
+                end = start + len(group._group_actions)
+                if actions[start:end] == group._group_actions:
+                    for action in group._group_actions:
+                        group_actions.add(action)
+                    if not group.required:
+                        if start in inserts:
+                            inserts[start] += ' ['
+                        else:
+                            inserts[start] = '['
+                        inserts[end] = ']'
+                    else:
+                        if start in inserts:
+                            inserts[start] += ' ('
+                        else:
+                            inserts[start] = '('
+                        inserts[end] = ')'
+                    for i in range(start + 1, end):
+                        inserts[i] = '|'
+
+        # collect all actions format strings
+        parts = []
+        for i, action in enumerate(actions):
+
+            # suppressed arguments are marked with None
+            # remove | separators for suppressed arguments
+            if action.help is SUPPRESS:
+                parts.append(None)
+                if inserts.get(i) == '|':
+                    inserts.pop(i)
+                elif inserts.get(i + 1) == '|':
+                    inserts.pop(i + 1)
+
+            # produce all arg strings
+            elif not action.option_strings:
+                part = self._format_args(action, action.dest)
+
+                # if it's in a group, strip the outer []
+                if action in group_actions:
+                    if part[0] == '[' and part[-1] == ']':
+                        part = part[1:-1]
+
+                # add the action string to the list
+                parts.append(part)
+
+            # produce the first way to invoke the option in brackets
+            else:
+                option_string = action.option_strings[0]
+
+                # if the Optional doesn't take a value, format is:
+                #    -s or --long
+                if action.nargs == 0:
+                    part = '%s' % option_string
+
+                # if the Optional takes a value, format is:
+                #    -s ARGS or --long ARGS
+                else:
+                    default = action.dest.upper()
+                    args_string = self._format_args(action, default)
+                    part = '%s %s' % (option_string, args_string)
+
+                # make it look optional if it's not required or in a group
+                if not action.required and action not in group_actions:
+                    part = '[%s]' % part
+
+                # add the action string to the list
+                parts.append(part)
+
+        # insert things at the necessary indices
+        for i in sorted(inserts, reverse=True):
+            parts[i:i] = [inserts[i]]
+
+        # join all the action items with spaces
+        text = ' '.join([item for item in parts if item is not None])
+
+        # clean up separators for mutually exclusive groups
+        open = r'[\[(]'
+        close = r'[\])]'
+        text = _re.sub(r'(%s) ' % open, r'\1', text)
+        text = _re.sub(r' (%s)' % close, r'\1', text)
+        text = _re.sub(r'%s *%s' % (open, close), r'', text)
+        text = _re.sub(r'\(([^|]*)\)', r'\1', text)
+        text = text.strip()
+
+        # return the text
+        return text
+
+    def _format_text(self, text):
+        if '%(prog)' in text:
+            text = text % dict(prog=self._prog)
+        text_width = self._width - self._current_indent
+        indent = ' ' * self._current_indent
+        return self._fill_text(text, text_width, indent) + '\n\n'
+
+    def _format_action(self, action):
+        # determine the required width and the entry label
+        help_position = min(self._action_max_length + 2,
+                            self._max_help_position)
+        help_width = self._width - help_position
+        action_width = help_position - self._current_indent - 2
+        action_header = self._format_action_invocation(action)
+
+        # ho nelp; start on same line and add a final newline
+        if not action.help:
+            tup = self._current_indent, '', action_header
+            action_header = '%*s%s\n' % tup
+
+        # short action name; start on the same line and pad two spaces
+        elif len(action_header) <= action_width:
+            tup = self._current_indent, '', action_width, action_header
+            action_header = '%*s%-*s  ' % tup
+            indent_first = 0
+
+        # long action name; start on the next line
+        else:
+            tup = self._current_indent, '', action_header
+            action_header = '%*s%s\n' % tup
+            indent_first = help_position
+
+        # collect the pieces of the action help
+        parts = [action_header]
+
+        # if there was help for the action, add lines of help text
+        if action.help:
+            help_text = self._expand_help(action)
+            help_lines = self._split_lines(help_text, help_width)
+            parts.append('%*s%s\n' % (indent_first, '', help_lines[0]))
+            for line in help_lines[1:]:
+                parts.append('%*s%s\n' % (help_position, '', line))
+
+        # or add a newline if the description doesn't end with one
+        elif not action_header.endswith('\n'):
+            parts.append('\n')
+
+        # if there are any sub-actions, add their help as well
+        for subaction in self._iter_indented_subactions(action):
+            parts.append(self._format_action(subaction))
+
+        # return a single string
+        return self._join_parts(parts)
+
+    def _format_action_invocation(self, action):
+        if not action.option_strings:
+            metavar, = self._metavar_formatter(action, action.dest)(1)
+            return metavar
+
+        else:
+            parts = []
+
+            # if the Optional doesn't take a value, format is:
+            #    -s, --long
+            if action.nargs == 0:
+                parts.extend(action.option_strings)
+
+            # if the Optional takes a value, format is:
+            #    -s ARGS, --long ARGS
+            else:
+                default = action.dest.upper()
+                args_string = self._format_args(action, default)
+                for option_string in action.option_strings:
+                    parts.append('%s %s' % (option_string, args_string))
+
+            return ', '.join(parts)
+
+    def _metavar_formatter(self, action, default_metavar):
+        if action.metavar is not None:
+            result = action.metavar
+        elif action.choices is not None:
+            choice_strs = [str(choice) for choice in action.choices]
+            result = '{%s}' % ','.join(choice_strs)
+        else:
+            result = default_metavar
+
+        def format(tuple_size):
+            if isinstance(result, tuple):
+                return result
+            else:
+                return (result, ) * tuple_size
+        return format
+
+    def _format_args(self, action, default_metavar):
+        get_metavar = self._metavar_formatter(action, default_metavar)
+        if action.nargs is None:
+            result = '%s' % get_metavar(1)
+        elif action.nargs == OPTIONAL:
+            result = '[%s]' % get_metavar(1)
+        elif action.nargs == ZERO_OR_MORE:
+            result = '[%s [%s ...]]' % get_metavar(2)
+        elif action.nargs == ONE_OR_MORE:
+            result = '%s [%s ...]' % get_metavar(2)
+        elif action.nargs == REMAINDER:
+            result = '...'
+        elif action.nargs == PARSER:
+            result = '%s ...' % get_metavar(1)
+        else:
+            formats = ['%s' for _ in range(action.nargs)]
+            result = ' '.join(formats) % get_metavar(action.nargs)
+        return result
+
+    def _expand_help(self, action):
+        params = dict(vars(action), prog=self._prog)
+        for name in list(params):
+            if params[name] is SUPPRESS:
+                del params[name]
+        for name in list(params):
+            if hasattr(params[name], '__name__'):
+                params[name] = params[name].__name__
+        if params.get('choices') is not None:
+            choices_str = ', '.join([str(c) for c in params['choices']])
+            params['choices'] = choices_str
+        return self._get_help_string(action) % params
+
+    def _iter_indented_subactions(self, action):
+        try:
+            get_subactions = action._get_subactions
+        except AttributeError:
+            pass
+        else:
+            self._indent()
+            for subaction in get_subactions():
+                yield subaction
+            self._dedent()
+
+    def _split_lines(self, text, width):
+        text = self._whitespace_matcher.sub(' ', text).strip()
+        return _textwrap.wrap(text, width)
+
+    def _fill_text(self, text, width, indent):
+        text = self._whitespace_matcher.sub(' ', text).strip()
+        return _textwrap.fill(text, width, initial_indent=indent,
+                                           subsequent_indent=indent)
+
+    def _get_help_string(self, action):
+        return action.help
+
+
+class RawDescriptionHelpFormatter(HelpFormatter):
+    """Help message formatter which retains any formatting in descriptions.
+
+    Only the name of this class is considered a public API. All the methods
+    provided by the class are considered an implementation detail.
+    """
+
+    def _fill_text(self, text, width, indent):
+        return ''.join([indent + line for line in text.splitlines(True)])
+
+
+class RawTextHelpFormatter(RawDescriptionHelpFormatter):
+    """Help message formatter which retains formatting of all help text.
+
+    Only the name of this class is considered a public API. All the methods
+    provided by the class are considered an implementation detail.
+    """
+
+    def _split_lines(self, text, width):
+        return text.splitlines()
+
+
+class ArgumentDefaultsHelpFormatter(HelpFormatter):
+    """Help message formatter which adds default values to argument help.
+
+    Only the name of this class is considered a public API. All the methods
+    provided by the class are considered an implementation detail.
+    """
+
+    def _get_help_string(self, action):
+        help = action.help
+        if '%(default)' not in action.help:
+            if action.default is not SUPPRESS:
+                defaulting_nargs = [OPTIONAL, ZERO_OR_MORE]
+                if action.option_strings or action.nargs in defaulting_nargs:
+                    help += ' (default: %(default)s)'
+        return help
+
+
+# =====================
+# Options and Arguments
+# =====================
+
+def _get_action_name(argument):
+    if argument is None:
+        return None
+    elif argument.option_strings:
+        return  '/'.join(argument.option_strings)
+    elif argument.metavar not in (None, SUPPRESS):
+        return argument.metavar
+    elif argument.dest not in (None, SUPPRESS):
+        return argument.dest
+    else:
+        return None
+
+
+class ArgumentError(Exception):
+    """An error from creating or using an argument (optional or positional).
+
+    The string value of this exception is the message, augmented with
+    information about the argument that caused it.
+    """
+
+    def __init__(self, argument, message):
+        self.argument_name = _get_action_name(argument)
+        self.message = message
+
+    def __str__(self):
+        if self.argument_name is None:
+            format = '%(message)s'
+        else:
+            format = 'argument %(argument_name)s: %(message)s'
+        return format % dict(message=self.message,
+                             argument_name=self.argument_name)
+
+
+class ArgumentTypeError(Exception):
+    """An error from trying to convert a command line string to a type."""
+    pass
+
+
+# ==============
+# Action classes
+# ==============
+
+class Action(_AttributeHolder):
+    """Information about how to convert command line strings to Python objects.
+
+    Action objects are used by an ArgumentParser to represent the information
+    needed to parse a single argument from one or more strings from the
+    command line. The keyword arguments to the Action constructor are also
+    all attributes of Action instances.
+
+    Keyword Arguments:
+
+        - option_strings -- A list of command-line option strings which
+            should be associated with this action.
+
+        - dest -- The name of the attribute to hold the created object(s)
+
+        - nargs -- The number of command-line arguments that should be
+            consumed. By default, one argument will be consumed and a single
+            value will be produced.  Other values include:
+                - N (an integer) consumes N arguments (and produces a list)
+                - '?' consumes zero or one arguments
+                - '*' consumes zero or more arguments (and produces a list)
+                - '+' consumes one or more arguments (and produces a list)
+            Note that the difference between the default and nargs=1 is that
+            with the default, a single value will be produced, while with
+            nargs=1, a list containing a single value will be produced.
+
+        - const -- The value to be produced if the option is specified and the
+            option uses an action that takes no values.
+
+        - default -- The value to be produced if the option is not specified.
+
+        - type -- The type which the command-line arguments should be converted
+            to, should be one of 'string', 'int', 'float', 'complex' or a
+            callable object that accepts a single string argument. If None,
+            'string' is assumed.
+
+        - choices -- A container of values that should be allowed. If not None,
+            after a command-line argument has been converted to the appropriate
+            type, an exception will be raised if it is not a member of this
+            collection.
+
+        - required -- True if the action must always be specified at the
+            command line. This is only meaningful for optional command-line
+            arguments.
+
+        - help -- The help string describing the argument.
+
+        - metavar -- The name to be used for the option's argument with the
+            help string. If None, the 'dest' value will be used as the name.
+    """
+
+    def __init__(self,
+                 option_strings,
+                 dest,
+                 nargs=None,
+                 const=None,
+                 default=None,
+                 type=None,
+                 choices=None,
+                 required=False,
+                 help=None,
+                 metavar=None):
+        self.option_strings = option_strings
+        self.dest = dest
+        self.nargs = nargs
+        self.const = const
+        self.default = default
+        self.type = type
+        self.choices = choices
+        self.required = required
+        self.help = help
+        self.metavar = metavar
+
+    def _get_kwargs(self):
+        names = [
+            'option_strings',
+            'dest',
+            'nargs',
+            'const',
+            'default',
+            'type',
+            'choices',
+            'help',
+            'metavar',
+        ]
+        return [(name, getattr(self, name)) for name in names]
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        raise NotImplementedError(_('.__call__() not defined'))
+
+
+class _StoreAction(Action):
+
+    def __init__(self,
+                 option_strings,
+                 dest,
+                 nargs=None,
+                 const=None,
+                 default=None,
+                 type=None,
+                 choices=None,
+                 required=False,
+                 help=None,
+                 metavar=None):
+        if nargs == 0:
+            raise ValueError('nargs for store actions must be > 0; if you '
+                             'have nothing to store, actions such as store '
+                             'true or store const may be more appropriate')
+        if const is not None and nargs != OPTIONAL:
+            raise ValueError('nargs must be %r to supply const' % OPTIONAL)
+        super(_StoreAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            nargs=nargs,
+            const=const,
+            default=default,
+            type=type,
+            choices=choices,
+            required=required,
+            help=help,
+            metavar=metavar)
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        setattr(namespace, self.dest, values)
+
+
+class _StoreConstAction(Action):
+
+    def __init__(self,
+                 option_strings,
+                 dest,
+                 const,
+                 default=None,
+                 required=False,
+                 help=None,
+                 metavar=None):
+        super(_StoreConstAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            nargs=0,
+            const=const,
+            default=default,
+            required=required,
+            help=help)
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        setattr(namespace, self.dest, self.const)
+
+
+class _StoreTrueAction(_StoreConstAction):
+
+    def __init__(self,
+                 option_strings,
+                 dest,
+                 default=False,
+                 required=False,
+                 help=None):
+        super(_StoreTrueAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            const=True,
+            default=default,
+            required=required,
+            help=help)
+
+
+class _StoreFalseAction(_StoreConstAction):
+
+    def __init__(self,
+                 option_strings,
+                 dest,
+                 default=True,
+                 required=False,
+                 help=None):
+        super(_StoreFalseAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            const=False,
+            default=default,
+            required=required,
+            help=help)
+
+
+class _AppendAction(Action):
+
+    def __init__(self,
+                 option_strings,
+                 dest,
+                 nargs=None,
+                 const=None,
+                 default=None,
+                 type=None,
+                 choices=None,
+                 required=False,
+                 help=None,
+                 metavar=None):
+        if nargs == 0:
+            raise ValueError('nargs for append actions must be > 0; if arg '
+                             'strings are not supplying the value to append, '
+                             'the append const action may be more appropriate')
+        if const is not None and nargs != OPTIONAL:
+            raise ValueError('nargs must be %r to supply const' % OPTIONAL)
+        super(_AppendAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            nargs=nargs,
+            const=const,
+            default=default,
+            type=type,
+            choices=choices,
+            required=required,
+            help=help,
+            metavar=metavar)
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        items = _copy.copy(_ensure_value(namespace, self.dest, []))
+        items.append(values)
+        setattr(namespace, self.dest, items)
+
+
+class _AppendConstAction(Action):
+
+    def __init__(self,
+                 option_strings,
+                 dest,
+                 const,
+                 default=None,
+                 required=False,
+                 help=None,
+                 metavar=None):
+        super(_AppendConstAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            nargs=0,
+            const=const,
+            default=default,
+            required=required,
+            help=help,
+            metavar=metavar)
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        items = _copy.copy(_ensure_value(namespace, self.dest, []))
+        items.append(self.const)
+        setattr(namespace, self.dest, items)
+
+
+class _CountAction(Action):
+
+    def __init__(self,
+                 option_strings,
+                 dest,
+                 default=None,
+                 required=False,
+                 help=None):
+        super(_CountAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            nargs=0,
+            default=default,
+            required=required,
+            help=help)
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        new_count = _ensure_value(namespace, self.dest, 0) + 1
+        setattr(namespace, self.dest, new_count)
+
+
+class _HelpAction(Action):
+
+    def __init__(self,
+                 option_strings,
+                 dest=SUPPRESS,
+                 default=SUPPRESS,
+                 help=None):
+        super(_HelpAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            default=default,
+            nargs=0,
+            help=help)
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        parser.print_help()
+        parser.exit()
+
+
+class _VersionAction(Action):
+
+    def __init__(self,
+                 option_strings,
+                 version=None,
+                 dest=SUPPRESS,
+                 default=SUPPRESS,
+                 help="show program's version number and exit"):
+        super(_VersionAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            default=default,
+            nargs=0,
+            help=help)
+        self.version = version
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        version = self.version
+        if version is None:
+            version = parser.version
+        formatter = parser._get_formatter()
+        formatter.add_text(version)
+        parser.exit(message=formatter.format_help())
+
+
+class _SubParsersAction(Action):
+
+    class _ChoicesPseudoAction(Action):
+
+        def __init__(self, name, aliases, help):
+            metavar = dest = name
+            if aliases:
+                metavar += ' (%s)' % ', '.join(aliases)
+            sup = super(_SubParsersAction._ChoicesPseudoAction, self)
+            sup.__init__(option_strings=[], dest=dest, help=help,
+                         metavar=metavar)
+
+    def __init__(self,
+                 option_strings,
+                 prog,
+                 parser_class,
+                 dest=SUPPRESS,
+                 help=None,
+                 metavar=None):
+
+        self._prog_prefix = prog
+        self._parser_class = parser_class
+        self._name_parser_map = _collections.OrderedDict()
+        self._choices_actions = []
+
+        super(_SubParsersAction, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            nargs=PARSER,
+            choices=self._name_parser_map,
+            help=help,
+            metavar=metavar)
+
+    def add_parser(self, name, **kwargs):
+        # set prog from the existing prefix
+        if kwargs.get('prog') is None:
+            kwargs['prog'] = '%s %s' % (self._prog_prefix, name)
+
+        aliases = kwargs.pop('aliases', ())
+
+        # create a pseudo-action to hold the choice help
+        if 'help' in kwargs:
+            help = kwargs.pop('help')
+            choice_action = self._ChoicesPseudoAction(name, aliases, help)
+            self._choices_actions.append(choice_action)
+
+        # create the parser and add it to the map
+        parser = self._parser_class(**kwargs)
+        self._name_parser_map[name] = parser
+
+        # make parser available under aliases also
+        for alias in aliases:
+            self._name_parser_map[alias] = parser
+
+        return parser
+
+    def _get_subactions(self):
+        return self._choices_actions
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        parser_name = values[0]
+        arg_strings = values[1:]
+
+        # set the parser name if requested
+        if self.dest is not SUPPRESS:
+            setattr(namespace, self.dest, parser_name)
+
+        # select the parser
+        try:
+            parser = self._name_parser_map[parser_name]
+        except KeyError:
+            args = {'parser_name': parser_name,
+                    'choices': ', '.join(self._name_parser_map)}
+            msg = _('unknown parser %(parser_name)r (choices: %(choices)s)') % args
+            raise ArgumentError(self, msg)
+
+        # parse all the remaining options into the namespace
+        # store any unrecognized options on the object, so that the top
+        # level parser can decide what to do with them
+        namespace, arg_strings = parser.parse_known_args(arg_strings, namespace)
+        if arg_strings:
+            vars(namespace).setdefault(_UNRECOGNIZED_ARGS_ATTR, [])
+            getattr(namespace, _UNRECOGNIZED_ARGS_ATTR).extend(arg_strings)
+
+
+# ==============
+# Type classes
+# ==============
+
+class FileType(object):
+    """Factory for creating file object types
+
+    Instances of FileType are typically passed as type= arguments to the
+    ArgumentParser add_argument() method.
+
+    Keyword Arguments:
+        - mode -- A string indicating how the file is to be opened. Accepts the
+            same values as the builtin open() function.
+        - bufsize -- The file's desired buffer size. Accepts the same values as
+            the builtin open() function.
+    """
+
+    def __init__(self, mode='r', bufsize=-1):
+        self._mode = mode
+        self._bufsize = bufsize
+
+    def __call__(self, string):
+        # the special argument "-" means sys.std{in,out}
+        if string == '-':
+            if 'r' in self._mode:
+                return _sys.stdin
+            elif 'w' in self._mode:
+                return _sys.stdout
+            else:
+                msg = _('argument "-" with mode %r') % self._mode
+                raise ValueError(msg)
+
+        # all other arguments are used as file names
+        try:
+            return open(string, self._mode, self._bufsize)
+        except IOError as e:
+            message = _("can't open '%s': %s")
+            raise ArgumentTypeError(message % (string, e))
+
+    def __repr__(self):
+        args = self._mode, self._bufsize
+        args_str = ', '.join(repr(arg) for arg in args if arg != -1)
+        return '%s(%s)' % (type(self).__name__, args_str)
+
+# ===========================
+# Optional and Positional Parsing
+# ===========================
+
+class Namespace(_AttributeHolder):
+    """Simple object for storing attributes.
+
+    Implements equality by attribute names and values, and provides a simple
+    string representation.
+    """
+
+    def __init__(self, **kwargs):
+        for name in kwargs:
+            setattr(self, name, kwargs[name])
+
+    def __eq__(self, other):
+        return vars(self) == vars(other)
+
+    def __ne__(self, other):
+        return not (self == other)
+
+    def __contains__(self, key):
+        return key in self.__dict__
+
+
+class _ActionsContainer(object):
+
+    def __init__(self,
+                 description,
+                 prefix_chars,
+                 argument_default,
+                 conflict_handler):
+        super(_ActionsContainer, self).__init__()
+
+        self.description = description
+        self.argument_default = argument_default
+        self.prefix_chars = prefix_chars
+        self.conflict_handler = conflict_handler
+
+        # set up registries
+        self._registries = {}
+
+        # register actions
+        self.register('action', None, _StoreAction)
+        self.register('action', 'store', _StoreAction)
+        self.register('action', 'store_const', _StoreConstAction)
+        self.register('action', 'store_true', _StoreTrueAction)
+        self.register('action', 'store_false', _StoreFalseAction)
+        self.register('action', 'append', _AppendAction)
+        self.register('action', 'append_const', _AppendConstAction)
+        self.register('action', 'count', _CountAction)
+        self.register('action', 'help', _HelpAction)
+        self.register('action', 'version', _VersionAction)
+        self.register('action', 'parsers', _SubParsersAction)
+
+        # raise an exception if the conflict handler is invalid
+        self._get_handler()
+
+        # action storage
+        self._actions = []
+        self._option_string_actions = {}
+
+        # groups
+        self._action_groups = []
+        self._mutually_exclusive_groups = []
+
+        # defaults storage
+        self._defaults = {}
+
+        # determines whether an "option" looks like a negative number
+        self._negative_number_matcher = _re.compile(r'^-\d+$|^-\d*\.\d+$')
+
+        # whether or not there are any optionals that look like negative
+        # numbers -- uses a list so it can be shared and edited
+        self._has_negative_number_optionals = []
+
+    # ====================
+    # Registration methods
+    # ====================
+    def register(self, registry_name, value, object):
+        registry = self._registries.setdefault(registry_name, {})
+        registry[value] = object
+
+    def _registry_get(self, registry_name, value, default=None):
+        return self._registries[registry_name].get(value, default)
+
+    # ==================================
+    # Namespace default accessor methods
+    # ==================================
+    def set_defaults(self, **kwargs):
+        self._defaults.update(kwargs)
+
+        # if these defaults match any existing arguments, replace
+        # the previous default on the object with the new one
+        for action in self._actions:
+            if action.dest in kwargs:
+                action.default = kwargs[action.dest]
+
+    def get_default(self, dest):
+        for action in self._actions:
+            if action.dest == dest and action.default is not None:
+                return action.default
+        return self._defaults.get(dest, None)
+
+
+    # =======================
+    # Adding argument actions
+    # =======================
+    def add_argument(self, *args, **kwargs):
+        """
+        add_argument(dest, ..., name=value, ...)
+        add_argument(option_string, option_string, ..., name=value, ...)
+        """
+
+        # if no positional args are supplied or only one is supplied and
+        # it doesn't look like an option string, parse a positional
+        # argument
+        chars = self.prefix_chars
+        if not args or len(args) == 1 and args[0][0] not in chars:
+            if args and 'dest' in kwargs:
+                raise ValueError('dest supplied twice for positional argument')
+            kwargs = self._get_positional_kwargs(*args, **kwargs)
+
+        # otherwise, we're adding an optional argument
+        else:
+            kwargs = self._get_optional_kwargs(*args, **kwargs)
+
+        # if no default was supplied, use the parser-level default
+        if 'default' not in kwargs:
+            dest = kwargs['dest']
+            if dest in self._defaults:
+                kwargs['default'] = self._defaults[dest]
+            elif self.argument_default is not None:
+                kwargs['default'] = self.argument_default
+
+        # create the action object, and add it to the parser
+        action_class = self._pop_action_class(kwargs)
+        if not _callable(action_class):
+            raise ValueError('unknown action "%s"' % (action_class,))
+        action = action_class(**kwargs)
+
+        # raise an error if the action type is not callable
+        type_func = self._registry_get('type', action.type, action.type)
+        if not _callable(type_func):
+            raise ValueError('%r is not callable' % (type_func,))
+
+        # raise an error if the metavar does not match the type
+        if hasattr(self, "_get_formatter"):
+            try:
+                self._get_formatter()._format_args(action, None)
+            except TypeError:
+                raise ValueError("length of metavar tuple does not match nargs")
+
+        return self._add_action(action)
+
+    def add_argument_group(self, *args, **kwargs):
+        group = _ArgumentGroup(self, *args, **kwargs)
+        self._action_groups.append(group)
+        return group
+
+    def add_mutually_exclusive_group(self, **kwargs):
+        group = _MutuallyExclusiveGroup(self, **kwargs)
+        self._mutually_exclusive_groups.append(group)
+        return group
+
+    def _add_action(self, action):
+        # resolve any conflicts
+        self._check_conflict(action)
+
+        # add to actions list
+        self._actions.append(action)
+        action.container = self
+
+        # index the action by any option strings it has
+        for option_string in action.option_strings:
+            self._option_string_actions[option_string] = action
+
+        # set the flag if any option strings look like negative numbers
+        for option_string in action.option_strings:
+            if self._negative_number_matcher.match(option_string):
+                if not self._has_negative_number_optionals:
+                    self._has_negative_number_optionals.append(True)
+
+        # return the created action
+        return action
+
+    def _remove_action(self, action):
+        self._actions.remove(action)
+
+    def _add_container_actions(self, container):
+        # collect groups by titles
+        title_group_map = {}
+        for group in self._action_groups:
+            if group.title in title_group_map:
+                msg = _('cannot merge actions - two groups are named %r')
+                raise ValueError(msg % (group.title))
+            title_group_map[group.title] = group
+
+        # map each action to its group
+        group_map = {}
+        for group in container._action_groups:
+
+            # if a group with the title exists, use that, otherwise
+            # create a new group matching the container's group
+            if group.title not in title_group_map:
+                title_group_map[group.title] = self.add_argument_group(
+                    title=group.title,
+                    description=group.description,
+                    conflict_handler=group.conflict_handler)
+
+            # map the actions to their new group
+            for action in group._group_actions:
+                group_map[action] = title_group_map[group.title]
+
+        # add container's mutually exclusive groups
+        # NOTE: if add_mutually_exclusive_group ever gains title= and
+        # description= then this code will need to be expanded as above
+        for group in container._mutually_exclusive_groups:
+            mutex_group = self.add_mutually_exclusive_group(
+                required=group.required)
+
+            # map the actions to their new mutex group
+            for action in group._group_actions:
+                group_map[action] = mutex_group
+
+        # add all actions to this container or their group
+        for action in container._actions:
+            group_map.get(action, self)._add_action(action)
+
+    def _get_positional_kwargs(self, dest, **kwargs):
+        # make sure required is not specified
+        if 'required' in kwargs:
+            msg = _("'required' is an invalid argument for positionals")
+            raise TypeError(msg)
+
+        # mark positional arguments as required if at least one is
+        # always required
+        if kwargs.get('nargs') not in [OPTIONAL, ZERO_OR_MORE]:
+            kwargs['required'] = True
+        if kwargs.get('nargs') == ZERO_OR_MORE and 'default' not in kwargs:
+            kwargs['required'] = True
+
+        # return the keyword arguments with no option strings
+        return dict(kwargs, dest=dest, option_strings=[])
+
+    def _get_optional_kwargs(self, *args, **kwargs):
+        # determine short and long option strings
+        option_strings = []
+        long_option_strings = []
+        for option_string in args:
+            # error on strings that don't start with an appropriate prefix
+            if not option_string[0] in self.prefix_chars:
+                args = {'option': option_string,
+                        'prefix_chars': self.prefix_chars}
+                msg = _('invalid option string %(option)r: '
+                        'must start with a character %(prefix_chars)r')
+                raise ValueError(msg % args)
+
+            # strings starting with two prefix characters are long options
+            option_strings.append(option_string)
+            if option_string[0] in self.prefix_chars:
+                if len(option_string) > 1:
+                    if option_string[1] in self.prefix_chars:
+                        long_option_strings.append(option_string)
+
+        # infer destination, '--foo-bar' -> 'foo_bar' and '-x' -> 'x'
+        dest = kwargs.pop('dest', None)
+        if dest is None:
+            if long_option_strings:
+                dest_option_string = long_option_strings[0]
+            else:
+                dest_option_string = option_strings[0]
+            dest = dest_option_string.lstrip(self.prefix_chars)
+            if not dest:
+                msg = _('dest= is required for options like %r')
+                raise ValueError(msg % option_string)
+            dest = dest.replace('-', '_')
+
+        # return the updated keyword arguments
+        return dict(kwargs, dest=dest, option_strings=option_strings)
+
+    def _pop_action_class(self, kwargs, default=None):
+        action = kwargs.pop('action', default)
+        return self._registry_get('action', action, action)
+
+    def _get_handler(self):
+        # determine function from conflict handler string
+        handler_func_name = '_handle_conflict_%s' % self.conflict_handler
+        try:
+            return getattr(self, handler_func_name)
+        except AttributeError:
+            msg = _('invalid conflict_resolution value: %r')
+            raise ValueError(msg % self.conflict_handler)
+
+    def _check_conflict(self, action):
+
+        # find all options that conflict with this option
+        confl_optionals = []
+        for option_string in action.option_strings:
+            if option_string in self._option_string_actions:
+                confl_optional = self._option_string_actions[option_string]
+                confl_optionals.append((option_string, confl_optional))
+
+        # resolve any conflicts
+        if confl_optionals:
+            conflict_handler = self._get_handler()
+            conflict_handler(action, confl_optionals)
+
+    def _handle_conflict_error(self, action, conflicting_actions):
+        message = ngettext('conflicting option string: %s',
+                           'conflicting option strings: %s',
+                           len(conflicting_actions))
+        conflict_string = ', '.join([option_string
+                                     for option_string, action
+                                     in conflicting_actions])
+        raise ArgumentError(action, message % conflict_string)
+
+    def _handle_conflict_resolve(self, action, conflicting_actions):
+
+        # remove all conflicting options
+        for option_string, action in conflicting_actions:
+
+            # remove the conflicting option
+            action.option_strings.remove(option_string)
+            self._option_string_actions.pop(option_string, None)
+
+            # if the option now has no option string, remove it from the
+            # container holding it
+            if not action.option_strings:
+                action.container._remove_action(action)
+
+
+class _ArgumentGroup(_ActionsContainer):
+
+    def __init__(self, container, title=None, description=None, **kwargs):
+        # add any missing keyword arguments by checking the container
+        update = kwargs.setdefault
+        update('conflict_handler', container.conflict_handler)
+        update('prefix_chars', container.prefix_chars)
+        update('argument_default', container.argument_default)
+        super_init = super(_ArgumentGroup, self).__init__
+        super_init(description=description, **kwargs)
+
+        # group attributes
+        self.title = title
+        self._group_actions = []
+
+        # share most attributes with the container
+        self._registries = container._registries
+        self._actions = container._actions
+        self._option_string_actions = container._option_string_actions
+        self._defaults = container._defaults
+        self._has_negative_number_optionals = \
+            container._has_negative_number_optionals
+        self._mutually_exclusive_groups = container._mutually_exclusive_groups
+
+    def _add_action(self, action):
+        action = super(_ArgumentGroup, self)._add_action(action)
+        self._group_actions.append(action)
+        return action
+
+    def _remove_action(self, action):
+        super(_ArgumentGroup, self)._remove_action(action)
+        self._group_actions.remove(action)
+
+
+class _MutuallyExclusiveGroup(_ArgumentGroup):
+
+    def __init__(self, container, required=False):
+        super(_MutuallyExclusiveGroup, self).__init__(container)
+        self.required = required
+        self._container = container
+
+    def _add_action(self, action):
+        if action.required:
+            msg = _('mutually exclusive arguments must be optional')
+            raise ValueError(msg)
+        action = self._container._add_action(action)
+        self._group_actions.append(action)
+        return action
+
+    def _remove_action(self, action):
+        self._container._remove_action(action)
+        self._group_actions.remove(action)
+
+
+class ArgumentParser(_AttributeHolder, _ActionsContainer):
+    """Object for parsing command line strings into Python objects.
+
+    Keyword Arguments:
+        - prog -- The name of the program (default: sys.argv[0])
+        - usage -- A usage message (default: auto-generated from arguments)
+        - description -- A description of what the program does
+        - epilog -- Text following the argument descriptions
+        - parents -- Parsers whose arguments should be copied into this one
+        - formatter_class -- HelpFormatter class for printing help messages
+        - prefix_chars -- Characters that prefix optional arguments
+        - fromfile_prefix_chars -- Characters that prefix files containing
+            additional arguments
+        - argument_default -- The default value for all arguments
+        - conflict_handler -- String indicating how to handle conflicts
+        - add_help -- Add a -h/-help option
+    """
+
+    def __init__(self,
+                 prog=None,
+                 usage=None,
+                 description=None,
+                 epilog=None,
+                 version=None,
+                 parents=[],
+                 formatter_class=HelpFormatter,
+                 prefix_chars='-',
+                 fromfile_prefix_chars=None,
+                 argument_default=None,
+                 conflict_handler='error',
+                 add_help=True):
+
+        if version is not None:
+            import warnings
+            warnings.warn(
+                """The "version" argument to ArgumentParser is deprecated. """
+                """Please use """
+                """"add_argument(..., action='version', version="N", ...)" """
+                """instead""", DeprecationWarning)
+
+        superinit = super(ArgumentParser, self).__init__
+        superinit(description=description,
+                  prefix_chars=prefix_chars,
+                  argument_default=argument_default,
+                  conflict_handler=conflict_handler)
+
+        # default setting for prog
+        if prog is None:
+            prog = _os.path.basename(_sys.argv[0])
+
+        self.prog = prog
+        self.usage = usage
+        self.epilog = epilog
+        self.version = version
+        self.formatter_class = formatter_class
+        self.fromfile_prefix_chars = fromfile_prefix_chars
+        self.add_help = add_help
+
+        add_group = self.add_argument_group
+        self._positionals = add_group(_('positional arguments'))
+        self._optionals = add_group(_('optional arguments'))
+        self._subparsers = None
+
+        # register types
+        def identity(string):
+            return string
+        self.register('type', None, identity)
+
+        # add help and version arguments if necessary
+        # (using explicit default to override global argument_default)
+        default_prefix = '-' if '-' in prefix_chars else prefix_chars[0]
+        if self.add_help:
+            self.add_argument(
+                default_prefix+'h', default_prefix*2+'help',
+                action='help', default=SUPPRESS,
+                help=_('show this help message and exit'))
+        if self.version:
+            self.add_argument(
+                default_prefix+'v', default_prefix*2+'version',
+                action='version', default=SUPPRESS,
+                version=self.version,
+                help=_("show program's version number and exit"))
+
+        # add parent arguments and defaults
+        for parent in parents:
+            self._add_container_actions(parent)
+            try:
+                defaults = parent._defaults
+            except AttributeError:
+                pass
+            else:
+                self._defaults.update(defaults)
+
+    # =======================
+    # Pretty __repr__ methods
+    # =======================
+    def _get_kwargs(self):
+        names = [
+            'prog',
+            'usage',
+            'description',
+            'version',
+            'formatter_class',
+            'conflict_handler',
+            'add_help',
+        ]
+        return [(name, getattr(self, name)) for name in names]
+
+    # ==================================
+    # Optional/Positional adding methods
+    # ==================================
+    def add_subparsers(self, **kwargs):
+        if self._subparsers is not None:
+            self.error(_('cannot have multiple subparser arguments'))
+
+        # add the parser class to the arguments if it's not present
+        kwargs.setdefault('parser_class', type(self))
+
+        if 'title' in kwargs or 'description' in kwargs:
+            title = _(kwargs.pop('title', 'subcommands'))
+            description = _(kwargs.pop('description', None))
+            self._subparsers = self.add_argument_group(title, description)
+        else:
+            self._subparsers = self._positionals
+
+        # prog defaults to the usage message of this parser, skipping
+        # optional arguments and with no "usage:" prefix
+        if kwargs.get('prog') is None:
+            formatter = self._get_formatter()
+            positionals = self._get_positional_actions()
+            groups = self._mutually_exclusive_groups
+            formatter.add_usage(self.usage, positionals, groups, '')
+            kwargs['prog'] = formatter.format_help().strip()
+
+        # create the parsers action and add it to the positionals list
+        parsers_class = self._pop_action_class(kwargs, 'parsers')
+        action = parsers_class(option_strings=[], **kwargs)
+        self._subparsers._add_action(action)
+
+        # return the created parsers action
+        return action
+
+    def _add_action(self, action):
+        if action.option_strings:
+            self._optionals._add_action(action)
+        else:
+            self._positionals._add_action(action)
+        return action
+
+    def _get_optional_actions(self):
+        return [action
+                for action in self._actions
+                if action.option_strings]
+
+    def _get_positional_actions(self):
+        return [action
+                for action in self._actions
+                if not action.option_strings]
+
+    # =====================================
+    # Command line argument parsing methods
+    # =====================================
+    def parse_args(self, args=None, namespace=None):
+        args, argv = self.parse_known_args(args, namespace)
+        if argv:
+            msg = _('unrecognized arguments: %s')
+            self.error(msg % ' '.join(argv))
+        return args
+
+    def parse_known_args(self, args=None, namespace=None):
+        # args default to the system args
+        if args is None:
+            args = _sys.argv[1:]
+
+        # default Namespace built from parser defaults
+        if namespace is None:
+            namespace = Namespace()
+
+        # add any action defaults that aren't present
+        for action in self._actions:
+            if action.dest is not SUPPRESS:
+                if not hasattr(namespace, action.dest):
+                    if action.default is not SUPPRESS:
+                        default = action.default
+                        if isinstance(action.default, str):
+                            default = self._get_value(action, default)
+                        setattr(namespace, action.dest, default)
+
+        # add any parser defaults that aren't present
+        for dest in self._defaults:
+            if not hasattr(namespace, dest):
+                setattr(namespace, dest, self._defaults[dest])
+
+        # parse the arguments and exit if there are any errors
+        try:
+            namespace, args = self._parse_known_args(args, namespace)
+            if hasattr(namespace, _UNRECOGNIZED_ARGS_ATTR):
+                args.extend(getattr(namespace, _UNRECOGNIZED_ARGS_ATTR))
+                delattr(namespace, _UNRECOGNIZED_ARGS_ATTR)
+            return namespace, args
+        except ArgumentError:
+            err = _sys.exc_info()[1]
+            self.error(str(err))
+
+    def _parse_known_args(self, arg_strings, namespace):
+        # replace arg strings that are file references
+        if self.fromfile_prefix_chars is not None:
+            arg_strings = self._read_args_from_files(arg_strings)
+
+        # map all mutually exclusive arguments to the other arguments
+        # they can't occur with
+        action_conflicts = {}
+        for mutex_group in self._mutually_exclusive_groups:
+            group_actions = mutex_group._group_actions
+            for i, mutex_action in enumerate(mutex_group._group_actions):
+                conflicts = action_conflicts.setdefault(mutex_action, [])
+                conflicts.extend(group_actions[:i])
+                conflicts.extend(group_actions[i + 1:])
+
+        # find all option indices, and determine the arg_string_pattern
+        # which has an 'O' if there is an option at an index,
+        # an 'A' if there is an argument, or a '-' if there is a '--'
+        option_string_indices = {}
+        arg_string_pattern_parts = []
+        arg_strings_iter = iter(arg_strings)
+        for i, arg_string in enumerate(arg_strings_iter):
+
+            # all args after -- are non-options
+            if arg_string == '--':
+                arg_string_pattern_parts.append('-')
+                for arg_string in arg_strings_iter:
+                    arg_string_pattern_parts.append('A')
+
+            # otherwise, add the arg to the arg strings
+            # and note the index if it was an option
+            else:
+                option_tuple = self._parse_optional(arg_string)
+                if option_tuple is None:
+                    pattern = 'A'
+                else:
+                    option_string_indices[i] = option_tuple
+                    pattern = 'O'
+                arg_string_pattern_parts.append(pattern)
+
+        # join the pieces together to form the pattern
+        arg_strings_pattern = ''.join(arg_string_pattern_parts)
+
+        # converts arg strings to the appropriate and then takes the action
+        seen_actions = set()
+        seen_non_default_actions = set()
+
+        def take_action(action, argument_strings, option_string=None):
+            seen_actions.add(action)
+            argument_values = self._get_values(action, argument_strings)
+
+            # error if this argument is not allowed with other previously
+            # seen arguments, assuming that actions that use the default
+            # value don't really count as "present"
+            if argument_values is not action.default:
+                seen_non_default_actions.add(action)
+                for conflict_action in action_conflicts.get(action, []):
+                    if conflict_action in seen_non_default_actions:
+                        msg = _('not allowed with argument %s')
+                        action_name = _get_action_name(conflict_action)
+                        raise ArgumentError(action, msg % action_name)
+
+            # take the action if we didn't receive a SUPPRESS value
+            # (e.g. from a default)
+            if argument_values is not SUPPRESS:
+                action(self, namespace, argument_values, option_string)
+
+        # function to convert arg_strings into an optional action
+        def consume_optional(start_index):
+
+            # get the optional identified at this index
+            option_tuple = option_string_indices[start_index]
+            action, option_string, explicit_arg = option_tuple
+
+            # identify additional optionals in the same arg string
+            # (e.g. -xyz is the same as -x -y -z if no args are required)
+            match_argument = self._match_argument
+            action_tuples = []
+            while True:
+
+                # if we found no optional action, skip it
+                if action is None:
+                    extras.append(arg_strings[start_index])
+                    return start_index + 1
+
+                # if there is an explicit argument, try to match the
+                # optional's string arguments to only this
+                if explicit_arg is not None:
+                    arg_count = match_argument(action, 'A')
+
+                    # if the action is a single-dash option and takes no
+                    # arguments, try to parse more single-dash options out
+                    # of the tail of the option string
+                    chars = self.prefix_chars
+                    if arg_count == 0 and option_string[1] not in chars:
+                        action_tuples.append((action, [], option_string))
+                        char = option_string[0]
+                        option_string = char + explicit_arg[0]
+                        new_explicit_arg = explicit_arg[1:] or None
+                        optionals_map = self._option_string_actions
+                        if option_string in optionals_map:
+                            action = optionals_map[option_string]
+                            explicit_arg = new_explicit_arg
+                        else:
+                            msg = _('ignored explicit argument %r')
+                            raise ArgumentError(action, msg % explicit_arg)
+
+                    # if the action expect exactly one argument, we've
+                    # successfully matched the option; exit the loop
+                    elif arg_count == 1:
+                        stop = start_index + 1
+                        args = [explicit_arg]
+                        action_tuples.append((action, args, option_string))
+                        break
+
+                    # error if a double-dash option did not use the
+                    # explicit argument
+                    else:
+                        msg = _('ignored explicit argument %r')
+                        raise ArgumentError(action, msg % explicit_arg)
+
+                # if there is no explicit argument, try to match the
+                # optional's string arguments with the following strings
+                # if successful, exit the loop
+                else:
+                    start = start_index + 1
+                    selected_patterns = arg_strings_pattern[start:]
+                    arg_count = match_argument(action, selected_patterns)
+                    stop = start + arg_count
+                    args = arg_strings[start:stop]
+                    action_tuples.append((action, args, option_string))
+                    break
+
+            # add the Optional to the list and return the index at which
+            # the Optional's string args stopped
+            assert action_tuples
+            for action, args, option_string in action_tuples:
+                take_action(action, args, option_string)
+            return stop
+
+        # the list of Positionals left to be parsed; this is modified
+        # by consume_positionals()
+        positionals = self._get_positional_actions()
+
+        # function to convert arg_strings into positional actions
+        def consume_positionals(start_index):
+            # match as many Positionals as possible
+            match_partial = self._match_arguments_partial
+            selected_pattern = arg_strings_pattern[start_index:]
+            arg_counts = match_partial(positionals, selected_pattern)
+
+            # slice off the appropriate arg strings for each Positional
+            # and add the Positional and its args to the list
+            for action, arg_count in zip(positionals, arg_counts):
+                args = arg_strings[start_index: start_index + arg_count]
+                start_index += arg_count
+                take_action(action, args)
+
+            # slice off the Positionals that we just parsed and return the
+            # index at which the Positionals' string args stopped
+            positionals[:] = positionals[len(arg_counts):]
+            return start_index
+
+        # consume Positionals and Optionals alternately, until we have
+        # passed the last option string
+        extras = []
+        start_index = 0
+        if option_string_indices:
+            max_option_string_index = max(option_string_indices)
+        else:
+            max_option_string_index = -1
+        while start_index <= max_option_string_index:
+
+            # consume any Positionals preceding the next option
+            next_option_string_index = min([
+                index
+                for index in option_string_indices
+                if index >= start_index])
+            if start_index != next_option_string_index:
+                positionals_end_index = consume_positionals(start_index)
+
+                # only try to parse the next optional if we didn't consume
+                # the option string during the positionals parsing
+                if positionals_end_index > start_index:
+                    start_index = positionals_end_index
+                    continue
+                else:
+                    start_index = positionals_end_index
+
+            # if we consumed all the positionals we could and we're not
+            # at the index of an option string, there were extra arguments
+            if start_index not in option_string_indices:
+                strings = arg_strings[start_index:next_option_string_index]
+                extras.extend(strings)
+                start_index = next_option_string_index
+
+            # consume the next optional and any arguments for it
+            start_index = consume_optional(start_index)
+
+        # consume any positionals following the last Optional
+        stop_index = consume_positionals(start_index)
+
+        # if we didn't consume all the argument strings, there were extras
+        extras.extend(arg_strings[stop_index:])
+
+        # if we didn't use all the Positional objects, there were too few
+        # arg strings supplied.
+        if positionals:
+            self.error(_('too few arguments'))
+
+        # make sure all required actions were present
+        for action in self._actions:
+            if action.required:
+                if action not in seen_actions:
+                    name = _get_action_name(action)
+                    self.error(_('argument %s is required') % name)
+
+        # make sure all required groups had one option present
+        for group in self._mutually_exclusive_groups:
+            if group.required:
+                for action in group._group_actions:
+                    if action in seen_non_default_actions:
+                        break
+
+                # if no actions were used, report the error
+                else:
+                    names = [_get_action_name(action)
+                             for action in group._group_actions
+                             if action.help is not SUPPRESS]
+                    msg = _('one of the arguments %s is required')
+                    self.error(msg % ' '.join(names))
+
+        # return the updated namespace and the extra arguments
+        return namespace, extras
+
+    def _read_args_from_files(self, arg_strings):
+        # expand arguments referencing files
+        new_arg_strings = []
+        for arg_string in arg_strings:
+
+            # for regular arguments, just add them back into the list
+            if arg_string[0] not in self.fromfile_prefix_chars:
+                new_arg_strings.append(arg_string)
+
+            # replace arguments referencing files with the file content
+            else:
+                try:
+                    args_file = open(arg_string[1:])
+                    try:
+                        arg_strings = []
+                        for arg_line in args_file.read().splitlines():
+                            for arg in self.convert_arg_line_to_args(arg_line):
+                                arg_strings.append(arg)
+                        arg_strings = self._read_args_from_files(arg_strings)
+                        new_arg_strings.extend(arg_strings)
+                    finally:
+                        args_file.close()
+                except IOError:
+                    err = _sys.exc_info()[1]
+                    self.error(str(err))
+
+        # return the modified argument list
+        return new_arg_strings
+
+    def convert_arg_line_to_args(self, arg_line):
+        return [arg_line]
+
+    def _match_argument(self, action, arg_strings_pattern):
+        # match the pattern for this action to the arg strings
+        nargs_pattern = self._get_nargs_pattern(action)
+        match = _re.match(nargs_pattern, arg_strings_pattern)
+
+        # raise an exception if we weren't able to find a match
+        if match is None:
+            nargs_errors = {
+                None: _('expected one argument'),
+                OPTIONAL: _('expected at most one argument'),
+                ONE_OR_MORE: _('expected at least one argument'),
+            }
+            default = ngettext('expected %s argument',
+                               'expected %s arguments',
+                               action.nargs) % action.nargs
+            msg = nargs_errors.get(action.nargs, default)
+            raise ArgumentError(action, msg)
+
+        # return the number of arguments matched
+        return len(match.group(1))
+
+    def _match_arguments_partial(self, actions, arg_strings_pattern):
+        # progressively shorten the actions list by slicing off the
+        # final actions until we find a match
+        result = []
+        for i in range(len(actions), 0, -1):
+            actions_slice = actions[:i]
+            pattern = ''.join([self._get_nargs_pattern(action)
+                               for action in actions_slice])
+            match = _re.match(pattern, arg_strings_pattern)
+            if match is not None:
+                result.extend([len(string) for string in match.groups()])
+                break
+
+        # return the list of arg string counts
+        return result
+
+    def _parse_optional(self, arg_string):
+        # if it's an empty string, it was meant to be a positional
+        if not arg_string:
+            return None
+
+        # if it doesn't start with a prefix, it was meant to be positional
+        if not arg_string[0] in self.prefix_chars:
+            return None
+
+        # if the option string is present in the parser, return the action
+        if arg_string in self._option_string_actions:
+            action = self._option_string_actions[arg_string]
+            return action, arg_string, None
+
+        # if it's just a single character, it was meant to be positional
+        if len(arg_string) == 1:
+            return None
+
+        # if the option string before the "=" is present, return the action
+        if '=' in arg_string:
+            option_string, explicit_arg = arg_string.split('=', 1)
+            if option_string in self._option_string_actions:
+                action = self._option_string_actions[option_string]
+                return action, option_string, explicit_arg
+
+        # search through all possible prefixes of the option string
+        # and all actions in the parser for possible interpretations
+        option_tuples = self._get_option_tuples(arg_string)
+
+        # if multiple actions match, the option string was ambiguous
+        if len(option_tuples) > 1:
+            options = ', '.join([option_string
+                for action, option_string, explicit_arg in option_tuples])
+            args = {'option': arg_string, 'matches': options}
+            msg = _('ambiguous option: %(option)s could match %(matches)s')
+            self.error(msg % args)
+
+        # if exactly one action matched, this segmentation is good,
+        # so return the parsed action
+        elif len(option_tuples) == 1:
+            option_tuple, = option_tuples
+            return option_tuple
+
+        # if it was not found as an option, but it looks like a negative
+        # number, it was meant to be positional
+        # unless there are negative-number-like options
+        if self._negative_number_matcher.match(arg_string):
+            if not self._has_negative_number_optionals:
+                return None
+
+        # if it contains a space, it was meant to be a positional
+        if ' ' in arg_string:
+            return None
+
+        # it was meant to be an optional but there is no such option
+        # in this parser (though it might be a valid option in a subparser)
+        return None, arg_string, None
+
+    def _get_option_tuples(self, option_string):
+        result = []
+
+        # option strings starting with two prefix characters are only
+        # split at the '='
+        chars = self.prefix_chars
+        if option_string[0] in chars and option_string[1] in chars:
+            if '=' in option_string:
+                option_prefix, explicit_arg = option_string.split('=', 1)
+            else:
+                option_prefix = option_string
+                explicit_arg = None
+            for option_string in self._option_string_actions:
+                if option_string.startswith(option_prefix):
+                    action = self._option_string_actions[option_string]
+                    tup = action, option_string, explicit_arg
+                    result.append(tup)
+
+        # single character options can be concatenated with their arguments
+        # but multiple character options always have to have their argument
+        # separate
+        elif option_string[0] in chars and option_string[1] not in chars:
+            option_prefix = option_string
+            explicit_arg = None
+            short_option_prefix = option_string[:2]
+            short_explicit_arg = option_string[2:]
+
+            for option_string in self._option_string_actions:
+                if option_string == short_option_prefix:
+                    action = self._option_string_actions[option_string]
+                    tup = action, option_string, short_explicit_arg
+                    result.append(tup)
+                elif option_string.startswith(option_prefix):
+                    action = self._option_string_actions[option_string]
+                    tup = action, option_string, explicit_arg
+                    result.append(tup)
+
+        # shouldn't ever get here
+        else:
+            self.error(_('unexpected option string: %s') % option_string)
+
+        # return the collected option tuples
+        return result
+
+    def _get_nargs_pattern(self, action):
+        # in all examples below, we have to allow for '--' args
+        # which are represented as '-' in the pattern
+        nargs = action.nargs
+
+        # the default (None) is assumed to be a single argument
+        if nargs is None:
+            nargs_pattern = '(-*A-*)'
+
+        # allow zero or one arguments
+        elif nargs == OPTIONAL:
+            nargs_pattern = '(-*A?-*)'
+
+        # allow zero or more arguments
+        elif nargs == ZERO_OR_MORE:
+            nargs_pattern = '(-*[A-]*)'
+
+        # allow one or more arguments
+        elif nargs == ONE_OR_MORE:
+            nargs_pattern = '(-*A[A-]*)'
+
+        # allow any number of options or arguments
+        elif nargs == REMAINDER:
+            nargs_pattern = '([-AO]*)'
+
+        # allow one argument followed by any number of options or arguments
+        elif nargs == PARSER:
+            nargs_pattern = '(-*A[-AO]*)'
+
+        # all others should be integers
+        else:
+            nargs_pattern = '(-*%s-*)' % '-*'.join('A' * nargs)
+
+        # if this is an optional action, -- is not allowed
+        if action.option_strings:
+            nargs_pattern = nargs_pattern.replace('-*', '')
+            nargs_pattern = nargs_pattern.replace('-', '')
+
+        # return the pattern
+        return nargs_pattern
+
+    # ========================
+    # Value conversion methods
+    # ========================
+    def _get_values(self, action, arg_strings):
+        # for everything but PARSER args, strip out '--'
+        if action.nargs not in [PARSER, REMAINDER]:
+            arg_strings = [s for s in arg_strings if s != '--']
+
+        # optional argument produces a default when not present
+        if not arg_strings and action.nargs == OPTIONAL:
+            if action.option_strings:
+                value = action.const
+            else:
+                value = action.default
+            if isinstance(value, str):
+                value = self._get_value(action, value)
+                self._check_value(action, value)
+
+        # when nargs='*' on a positional, if there were no command-line
+        # args, use the default if it is anything other than None
+        elif (not arg_strings and action.nargs == ZERO_OR_MORE and
+              not action.option_strings):
+            if action.default is not None:
+                value = action.default
+            else:
+                value = arg_strings
+            self._check_value(action, value)
+
+        # single argument or optional argument produces a single value
+        elif len(arg_strings) == 1 and action.nargs in [None, OPTIONAL]:
+            arg_string, = arg_strings
+            value = self._get_value(action, arg_string)
+            self._check_value(action, value)
+
+        # REMAINDER arguments convert all values, checking none
+        elif action.nargs == REMAINDER:
+            value = [self._get_value(action, v) for v in arg_strings]
+
+        # PARSER arguments convert all values, but check only the first
+        elif action.nargs == PARSER:
+            value = [self._get_value(action, v) for v in arg_strings]
+            self._check_value(action, value[0])
+
+        # all other types of nargs produce a list
+        else:
+            value = [self._get_value(action, v) for v in arg_strings]
+            for v in value:
+                self._check_value(action, v)
+
+        # return the converted value
+        return value
+
+    def _get_value(self, action, arg_string):
+        type_func = self._registry_get('type', action.type, action.type)
+        if not _callable(type_func):
+            msg = _('%r is not callable')
+            raise ArgumentError(action, msg % type_func)
+
+        # convert the value to the appropriate type
+        try:
+            result = type_func(arg_string)
+
+        # ArgumentTypeErrors indicate errors
+        except ArgumentTypeError:
+            name = getattr(action.type, '__name__', repr(action.type))
+            msg = str(_sys.exc_info()[1])
+            raise ArgumentError(action, msg)
+
+        # TypeErrors or ValueErrors also indicate errors
+        except (TypeError, ValueError):
+            name = getattr(action.type, '__name__', repr(action.type))
+            args = {'type': name, 'value': arg_string}
+            msg = _('invalid %(type)s value: %(value)r')
+            raise ArgumentError(action, msg % args)
+
+        # return the converted value
+        return result
+
+    def _check_value(self, action, value):
+        # converted value must be one of the choices (if specified)
+        if action.choices is not None and value not in action.choices:
+            args = {'value': value,
+                    'choices': ', '.join(map(repr, action.choices))}
+            msg = _('invalid choice: %(value)r (choose from %(choices)s)')
+            raise ArgumentError(action, msg % args)
+
+    # =======================
+    # Help-formatting methods
+    # =======================
+    def format_usage(self):
+        formatter = self._get_formatter()
+        formatter.add_usage(self.usage, self._actions,
+                            self._mutually_exclusive_groups)
+        return formatter.format_help()
+
+    def format_help(self):
+        formatter = self._get_formatter()
+
+        # usage
+        formatter.add_usage(self.usage, self._actions,
+                            self._mutually_exclusive_groups)
+
+        # description
+        formatter.add_text(self.description)
+
+        # positionals, optionals and user-defined groups
+        for action_group in self._action_groups:
+            formatter.start_section(action_group.title)
+            formatter.add_text(action_group.description)
+            formatter.add_arguments(action_group._group_actions)
+            formatter.end_section()
+
+        # epilog
+        formatter.add_text(self.epilog)
+
+        # determine help from format above
+        return formatter.format_help()
+
+    def format_version(self):
+        import warnings
+        warnings.warn(
+            'The format_version method is deprecated -- the "version" '
+            'argument to ArgumentParser is no longer supported.',
+            DeprecationWarning)
+        formatter = self._get_formatter()
+        formatter.add_text(self.version)
+        return formatter.format_help()
+
+    def _get_formatter(self):
+        return self.formatter_class(prog=self.prog)
+
+    # =====================
+    # Help-printing methods
+    # =====================
+    def print_usage(self, file=None):
+        if file is None:
+            file = _sys.stdout
+        self._print_message(self.format_usage(), file)
+
+    def print_help(self, file=None):
+        if file is None:
+            file = _sys.stdout
+        self._print_message(self.format_help(), file)
+
+    def print_version(self, file=None):
+        import warnings
+        warnings.warn(
+            'The print_version method is deprecated -- the "version" '
+            'argument to ArgumentParser is no longer supported.',
+            DeprecationWarning)
+        self._print_message(self.format_version(), file)
+
+    def _print_message(self, message, file=None):
+        if message:
+            if file is None:
+                file = _sys.stderr
+            file.write(message)
+
+    # ===============
+    # Exiting methods
+    # ===============
+    def exit(self, status=0, message=None):
+        if message:
+            self._print_message(message, _sys.stderr)
+        _sys.exit(status)
+
+    def error(self, message):
+        """error(message: string)
+
+        Prints a usage message incorporating the message to stderr and
+        exits.
+
+        If you override this in a subclass, it should not return -- it
+        should either exit or raise an exception.
+        """
+        self.print_usage(_sys.stderr)
+        args = {'prog': self.prog, 'message': message}
+        self.exit(2, _('%(prog)s: error: %(message)s\n') % args)
diff --git a/lib-python/3.2/ast.py b/lib-python/3.2/ast.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/ast.py
@@ -0,0 +1,318 @@
+"""
+    ast
+    ~~~
+
+    The `ast` module helps Python applications to process trees of the Python
+    abstract syntax grammar.  The abstract syntax itself might change with
+    each Python release; this module helps to find out programmatically what
+    the current grammar looks like and allows modifications of it.
+
+    An abstract syntax tree can be generated by passing `ast.PyCF_ONLY_AST` as
+    a flag to the `compile()` builtin function or by using the `parse()`
+    function from this module.  The result will be a tree of objects whose
+    classes all inherit from `ast.AST`.
+
+    A modified abstract syntax tree can be compiled into a Python code object
+    using the built-in `compile()` function.
+
+    Additionally various helper functions are provided that make working with
+    the trees simpler.  The main intention of the helper functions and this
+    module in general is to provide an easy to use interface for libraries
+    that work tightly with the python syntax (template engines for example).
+
+
+    :copyright: Copyright 2008 by Armin Ronacher.
+    :license: Python License.
+"""
+from _ast import *
+from _ast import __version__
+
+
+def parse(source, filename='<unknown>', mode='exec'):
+    """
+    Parse the source into an AST node.
+    Equivalent to compile(source, filename, mode, PyCF_ONLY_AST).
+    """
+    return compile(source, filename, mode, PyCF_ONLY_AST)
+
+
+def literal_eval(node_or_string):
+    """
+    Safely evaluate an expression node or a string containing a Python
+    expression.  The string or node provided may only consist of the following
+    Python literal structures: strings, bytes, numbers, tuples, lists, dicts,
+    sets, booleans, and None.
+    """
+    _safe_names = {'None': None, 'True': True, 'False': False}
+    if isinstance(node_or_string, str):
+        node_or_string = parse(node_or_string, mode='eval')
+    if isinstance(node_or_string, Expression):
+        node_or_string = node_or_string.body
+    def _convert(node):
+        if isinstance(node, (Str, Bytes)):
+            return node.s
+        elif isinstance(node, Num):
+            return node.n
+        elif isinstance(node, Tuple):
+            return tuple(map(_convert, node.elts))
+        elif isinstance(node, List):
+            return list(map(_convert, node.elts))
+        elif isinstance(node, Set):
+            return set(map(_convert, node.elts))
+        elif isinstance(node, Dict):
+            return dict((_convert(k), _convert(v)) for k, v
+                        in zip(node.keys, node.values))
+        elif isinstance(node, Name):
+            if node.id in _safe_names:
+                return _safe_names[node.id]
+        elif isinstance(node, UnaryOp) and \
+             isinstance(node.op, (UAdd, USub)) and \
+             isinstance(node.operand, (Num, UnaryOp, BinOp)):
+            operand = _convert(node.operand)
+            if isinstance(node.op, UAdd):
+                return + operand
+            else:
+                return - operand
+        elif isinstance(node, BinOp) and \
+             isinstance(node.op, (Add, Sub)) and \
+             isinstance(node.right, (Num, UnaryOp, BinOp)) and \
+             isinstance(node.left, (Num, UnaryOp, BinOp)):
+            left = _convert(node.left)
+            right = _convert(node.right)
+            if isinstance(node.op, Add):
+                return left + right
+            else:
+                return left - right
+        raise ValueError('malformed node or string: ' + repr(node))
+    return _convert(node_or_string)
+
+
+def dump(node, annotate_fields=True, include_attributes=False):
+    """
+    Return a formatted dump of the tree in *node*.  This is mainly useful for
+    debugging purposes.  The returned string will show the names and the values
+    for fields.  This makes the code impossible to evaluate, so if evaluation is
+    wanted *annotate_fields* must be set to False.  Attributes such as line
+    numbers and column offsets are not dumped by default.  If this is wanted,
+    *include_attributes* can be set to True.
+    """
+    def _format(node):
+        if isinstance(node, AST):
+            fields = [(a, _format(b)) for a, b in iter_fields(node)]
+            rv = '%s(%s' % (node.__class__.__name__, ', '.join(
+                ('%s=%s' % field for field in fields)
+                if annotate_fields else
+                (b for a, b in fields)
+            ))
+            if include_attributes and node._attributes:
+                rv += fields and ', ' or ' '
+                rv += ', '.join('%s=%s' % (a, _format(getattr(node, a)))
+                                for a in node._attributes)
+            return rv + ')'
+        elif isinstance(node, list):
+            return '[%s]' % ', '.join(_format(x) for x in node)
+        return repr(node)
+    if not isinstance(node, AST):
+        raise TypeError('expected AST, got %r' % node.__class__.__name__)
+    return _format(node)
+
+
+def copy_location(new_node, old_node):
+    """
+    Copy source location (`lineno` and `col_offset` attributes) from
+    *old_node* to *new_node* if possible, and return *new_node*.
+    """
+    for attr in 'lineno', 'col_offset':
+        if attr in old_node._attributes and attr in new_node._attributes \
+           and hasattr(old_node, attr):
+            setattr(new_node, attr, getattr(old_node, attr))
+    return new_node
+
+
+def fix_missing_locations(node):
+    """
+    When you compile a node tree with compile(), the compiler expects lineno and
+    col_offset attributes for every node that supports them.  This is rather
+    tedious to fill in for generated nodes, so this helper adds these attributes
+    recursively where not already set, by setting them to the values of the
+    parent node.  It works recursively starting at *node*.
+    """
+    def _fix(node, lineno, col_offset):
+        if 'lineno' in node._attributes:
+            if not hasattr(node, 'lineno'):
+                node.lineno = lineno
+            else:
+                lineno = node.lineno
+        if 'col_offset' in node._attributes:
+            if not hasattr(node, 'col_offset'):
+                node.col_offset = col_offset
+            else:
+                col_offset = node.col_offset
+        for child in iter_child_nodes(node):
+            _fix(child, lineno, col_offset)
+    _fix(node, 1, 0)
+    return node
+
+
+def increment_lineno(node, n=1):
+    """
+    Increment the line number of each node in the tree starting at *node* by *n*.
+    This is useful to "move code" to a different location in a file.
+    """
+    for child in walk(node):
+        if 'lineno' in child._attributes:
+            child.lineno = getattr(child, 'lineno', 0) + n
+    return node
+
+
+def iter_fields(node):
+    """
+    Yield a tuple of ``(fieldname, value)`` for each field in ``node._fields``
+    that is present on *node*.
+    """
+    for field in node._fields:
+        try:
+            yield field, getattr(node, field)
+        except AttributeError:
+            pass
+
+
+def iter_child_nodes(node):
+    """
+    Yield all direct child nodes of *node*, that is, all fields that are nodes
+    and all items of fields that are lists of nodes.
+    """
+    for name, field in iter_fields(node):
+        if isinstance(field, AST):
+            yield field
+        elif isinstance(field, list):
+            for item in field:
+                if isinstance(item, AST):
+                    yield item
+
+
+def get_docstring(node, clean=True):
+    """
+    Return the docstring for the given node or None if no docstring can
+    be found.  If the node provided does not have docstrings a TypeError
+    will be raised.
+    """
+    if not isinstance(node, (FunctionDef, ClassDef, Module)):
+        raise TypeError("%r can't have docstrings" % node.__class__.__name__)
+    if node.body and isinstance(node.body[0], Expr) and \
+       isinstance(node.body[0].value, Str):
+        if clean:
+            import inspect
+            return inspect.cleandoc(node.body[0].value.s)
+        return node.body[0].value.s
+
+
+def walk(node):
+    """
+    Recursively yield all descendant nodes in the tree starting at *node*
+    (including *node* itself), in no specified order.  This is useful if you
+    only want to modify nodes in place and don't care about the context.
+    """
+    from collections import deque
+    todo = deque([node])
+    while todo:
+        node = todo.popleft()
+        todo.extend(iter_child_nodes(node))
+        yield node
+
+
+class NodeVisitor(object):
+    """
+    A node visitor base class that walks the abstract syntax tree and calls a
+    visitor function for every node found.  This function may return a value
+    which is forwarded by the `visit` method.
+
+    This class is meant to be subclassed, with the subclass adding visitor
+    methods.
+
+    Per default the visitor functions for the nodes are ``'visit_'`` +
+    class name of the node.  So a `TryFinally` node visit function would
+    be `visit_TryFinally`.  This behavior can be changed by overriding
+    the `visit` method.  If no visitor function exists for a node
+    (return value `None`) the `generic_visit` visitor is used instead.
+
+    Don't use the `NodeVisitor` if you want to apply changes to nodes during
+    traversing.  For this a special visitor exists (`NodeTransformer`) that
+    allows modifications.
+    """
+
+    def visit(self, node):
+        """Visit a node."""
+        method = 'visit_' + node.__class__.__name__
+        visitor = getattr(self, method, self.generic_visit)
+        return visitor(node)
+
+    def generic_visit(self, node):
+        """Called if no explicit visitor function exists for a node."""
+        for field, value in iter_fields(node):
+            if isinstance(value, list):
+                for item in value:
+                    if isinstance(item, AST):
+                        self.visit(item)
+            elif isinstance(value, AST):
+                self.visit(value)
+
+
+class NodeTransformer(NodeVisitor):
+    """
+    A :class:`NodeVisitor` subclass that walks the abstract syntax tree and
+    allows modification of nodes.
+
+    The `NodeTransformer` will walk the AST and use the return value of the
+    visitor methods to replace or remove the old node.  If the return value of
+    the visitor method is ``None``, the node will be removed from its location,
+    otherwise it is replaced with the return value.  The return value may be the
+    original node in which case no replacement takes place.
+
+    Here is an example transformer that rewrites all occurrences of name lookups
+    (``foo``) to ``data['foo']``::
+
+       class RewriteName(NodeTransformer):
+
+           def visit_Name(self, node):
+               return copy_location(Subscript(
+                   value=Name(id='data', ctx=Load()),
+                   slice=Index(value=Str(s=node.id)),
+                   ctx=node.ctx
+               ), node)
+
+    Keep in mind that if the node you're operating on has child nodes you must
+    either transform the child nodes yourself or call the :meth:`generic_visit`
+    method for the node first.
+
+    For nodes that were part of a collection of statements (that applies to all
+    statement nodes), the visitor may also return a list of nodes rather than
+    just a single node.
+
+    Usually you use the transformer like this::
+
+       node = YourTransformer().visit(node)
+    """
+
+    def generic_visit(self, node):
+        for field, old_value in iter_fields(node):
+            old_value = getattr(node, field, None)
+            if isinstance(old_value, list):
+                new_values = []
+                for value in old_value:
+                    if isinstance(value, AST):
+                        value = self.visit(value)
+                        if value is None:
+                            continue
+                        elif not isinstance(value, AST):
+                            new_values.extend(value)
+                            continue
+                    new_values.append(value)
+                old_value[:] = new_values
+            elif isinstance(old_value, AST):
+                new_node = self.visit(old_value)
+                if new_node is None:
+                    delattr(node, field)
+                else:
+                    setattr(node, field, new_node)
+        return node
diff --git a/lib-python/3.2/asynchat.py b/lib-python/3.2/asynchat.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/asynchat.py
@@ -0,0 +1,336 @@
+# -*- Mode: Python; tab-width: 4 -*-
+#       Id: asynchat.py,v 2.26 2000/09/07 22:29:26 rushing Exp
+#       Author: Sam Rushing <rushing at nightmare.com>
+
+# ======================================================================
+# Copyright 1996 by Sam Rushing
+#
+#                         All Rights Reserved
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose and without fee is hereby
+# granted, provided that the above copyright notice appear in all
+# copies and that both that copyright notice and this permission
+# notice appear in supporting documentation, and that the name of Sam
+# Rushing not be used in advertising or publicity pertaining to
+# distribution of the software without specific, written prior
+# permission.
+#
+# SAM RUSHING DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
+# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN
+# NO EVENT SHALL SAM RUSHING BE LIABLE FOR ANY SPECIAL, INDIRECT OR
+# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
+# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+# ======================================================================
+
+r"""A class supporting chat-style (command/response) protocols.
+
+This class adds support for 'chat' style protocols - where one side
+sends a 'command', and the other sends a response (examples would be
+the common internet protocols - smtp, nntp, ftp, etc..).
+
+The handle_read() method looks at the input stream for the current
+'terminator' (usually '\r\n' for single-line responses, '\r\n.\r\n'
+for multi-line output), calling self.found_terminator() on its
+receipt.
+
+for example:
+Say you build an async nntp client using this class.  At the start
+of the connection, you'll have self.terminator set to '\r\n', in
+order to process the single-line greeting.  Just before issuing a
+'LIST' command you'll set it to '\r\n.\r\n'.  The output of the LIST
+command will be accumulated (using your own 'collect_incoming_data'
+method) up to the terminator, and then control will be returned to
+you - by calling your self.found_terminator() method.
+"""
+import socket
+import asyncore
+from collections import deque
+
+def buffer(obj, start=None, stop=None):
+    # if memoryview objects gain slicing semantics,
+    # this function will change for the better
+    # memoryview used for the TypeError
+    memoryview(obj)
+    if start == None:
+        start = 0
+    if stop == None:
+        stop = len(obj)
+    x = obj[start:stop]
+    ## print("buffer type is: %s"%(type(x),))
+    return x
+
+class async_chat (asyncore.dispatcher):
+    """This is an abstract class.  You must derive from this class, and add
+    the two methods collect_incoming_data() and found_terminator()"""
+
+    # these are overridable defaults
+
+    ac_in_buffer_size       = 4096
+    ac_out_buffer_size      = 4096
+
+    # we don't want to enable the use of encoding by default, because that is a
+    # sign of an application bug that we don't want to pass silently
+
+    use_encoding            = 0
+    encoding                = 'latin1'
+
+    def __init__ (self, sock=None, map=None):
+        # for string terminator matching
+        self.ac_in_buffer = b''
+
+        # we use a list here rather than cStringIO for a few reasons...
+        # del lst[:] is faster than sio.truncate(0)
+        # lst = [] is faster than sio.truncate(0)
+        # cStringIO will be gaining unicode support in py3k, which
+        # will negatively affect the performance of bytes compared to
+        # a ''.join() equivalent
+        self.incoming = []
+
+        # we toss the use of the "simple producer" and replace it with
+        # a pure deque, which the original fifo was a wrapping of
+        self.producer_fifo = deque()
+        asyncore.dispatcher.__init__ (self, sock, map)
+
+    def collect_incoming_data(self, data):
+        raise NotImplementedError("must be implemented in subclass")
+
+    def _collect_incoming_data(self, data):
+        self.incoming.append(data)
+
+    def _get_data(self):
+        d = b''.join(self.incoming)
+        del self.incoming[:]
+        return d
+
+    def found_terminator(self):
+        raise NotImplementedError("must be implemented in subclass")
+
+    def set_terminator (self, term):
+        "Set the input delimiter.  Can be a fixed string of any length, an integer, or None"
+        if isinstance(term, str) and self.use_encoding:
+            term = bytes(term, self.encoding)
+        self.terminator = term
+
+    def get_terminator (self):
+        return self.terminator
+
+    # grab some more data from the socket,
+    # throw it to the collector method,
+    # check for the terminator,
+    # if found, transition to the next state.
+
+    def handle_read (self):
+
+        try:
+            data = self.recv (self.ac_in_buffer_size)
+        except socket.error as why:
+            self.handle_error()
+            return
+
+        if isinstance(data, str) and self.use_encoding:
+            data = bytes(str, self.encoding)
+        self.ac_in_buffer = self.ac_in_buffer + data
+
+        # Continue to search for self.terminator in self.ac_in_buffer,
+        # while calling self.collect_incoming_data.  The while loop
+        # is necessary because we might read several data+terminator
+        # combos with a single recv(4096).
+
+        while self.ac_in_buffer:
+            lb = len(self.ac_in_buffer)
+            terminator = self.get_terminator()
+            if not terminator:
+                # no terminator, collect it all
+                self.collect_incoming_data (self.ac_in_buffer)
+                self.ac_in_buffer = b''
+            elif isinstance(terminator, int):
+                # numeric terminator
+                n = terminator
+                if lb < n:
+                    self.collect_incoming_data (self.ac_in_buffer)
+                    self.ac_in_buffer = b''
+                    self.terminator = self.terminator - lb
+                else:
+                    self.collect_incoming_data (self.ac_in_buffer[:n])
+                    self.ac_in_buffer = self.ac_in_buffer[n:]
+                    self.terminator = 0
+                    self.found_terminator()
+            else:
+                # 3 cases:
+                # 1) end of buffer matches terminator exactly:
+                #    collect data, transition
+                # 2) end of buffer matches some prefix:
+                #    collect data to the prefix
+                # 3) end of buffer does not match any prefix:
+                #    collect data
+                terminator_len = len(terminator)
+                index = self.ac_in_buffer.find(terminator)
+                if index != -1:
+                    # we found the terminator
+                    if index > 0:
+                        # don't bother reporting the empty string (source of subtle bugs)
+                        self.collect_incoming_data (self.ac_in_buffer[:index])
+                    self.ac_in_buffer = self.ac_in_buffer[index+terminator_len:]
+                    # This does the Right Thing if the terminator is changed here.
+                    self.found_terminator()
+                else:
+                    # check for a prefix of the terminator
+                    index = find_prefix_at_end (self.ac_in_buffer, terminator)
+                    if index:
+                        if index != lb:
+                            # we found a prefix, collect up to the prefix
+                            self.collect_incoming_data (self.ac_in_buffer[:-index])
+                            self.ac_in_buffer = self.ac_in_buffer[-index:]
+                        break
+                    else:
+                        # no prefix, collect it all
+                        self.collect_incoming_data (self.ac_in_buffer)
+                        self.ac_in_buffer = b''
+
+    def handle_write (self):
+        self.initiate_send()
+
+    def handle_close (self):
+        self.close()
+
+    def push (self, data):
+        sabs = self.ac_out_buffer_size
+        if len(data) > sabs:
+            for i in range(0, len(data), sabs):
+                self.producer_fifo.append(data[i:i+sabs])
+        else:
+            self.producer_fifo.append(data)
+        self.initiate_send()
+
+    def push_with_producer (self, producer):
+        self.producer_fifo.append(producer)
+        self.initiate_send()
+
+    def readable (self):
+        "predicate for inclusion in the readable for select()"
+        # cannot use the old predicate, it violates the claim of the
+        # set_terminator method.
+
+        # return (len(self.ac_in_buffer) <= self.ac_in_buffer_size)
+        return 1
+
+    def writable (self):
+        "predicate for inclusion in the writable for select()"
+        return self.producer_fifo or (not self.connected)
+
+    def close_when_done (self):
+        "automatically close this channel once the outgoing queue is empty"
+        self.producer_fifo.append(None)
+
+    def initiate_send(self):
+        while self.producer_fifo and self.connected:
+            first = self.producer_fifo[0]
+            # handle empty string/buffer or None entry
+            if not first:
+                del self.producer_fifo[0]
+                if first is None:
+                    ## print("first is None")
+                    self.handle_close()
+                    return
+                ## print("first is not None")
+
+            # handle classic producer behavior
+            obs = self.ac_out_buffer_size
+            try:
+                data = buffer(first, 0, obs)
+            except TypeError:
+                data = first.more()
+                if data:
+                    self.producer_fifo.appendleft(data)
+                else:
+                    del self.producer_fifo[0]
+                continue
+
+            if isinstance(data, str) and self.use_encoding:
+                data = bytes(data, self.encoding)
+
+            # send the data
+            try:
+                num_sent = self.send(data)
+            except socket.error:
+                self.handle_error()
+                return
+
+            if num_sent:
+                if num_sent < len(data) or obs < len(first):
+                    self.producer_fifo[0] = first[num_sent:]
+                else:
+                    del self.producer_fifo[0]
+            # we tried to send some actual data
+            return
+
+    def discard_buffers (self):
+        # Emergencies only!
+        self.ac_in_buffer = b''
+        del self.incoming[:]
+        self.producer_fifo.clear()
+
+class simple_producer:
+
+    def __init__ (self, data, buffer_size=512):
+        self.data = data
+        self.buffer_size = buffer_size
+
+    def more (self):
+        if len (self.data) > self.buffer_size:
+            result = self.data[:self.buffer_size]
+            self.data = self.data[self.buffer_size:]
+            return result
+        else:
+            result = self.data
+            self.data = b''
+            return result
+
+class fifo:
+    def __init__ (self, list=None):
+        if not list:
+            self.list = deque()
+        else:
+            self.list = deque(list)
+
+    def __len__ (self):
+        return len(self.list)
+
+    def is_empty (self):
+        return not self.list
+
+    def first (self):
+        return self.list[0]
+
+    def push (self, data):
+        self.list.append(data)
+
+    def pop (self):
+        if self.list:
+            return (1, self.list.popleft())
+        else:
+            return (0, None)
+
+# Given 'haystack', see if any prefix of 'needle' is at its end.  This
+# assumes an exact match has already been checked.  Return the number of
+# characters matched.
+# for example:
+# f_p_a_e ("qwerty\r", "\r\n") => 1
+# f_p_a_e ("qwertydkjf", "\r\n") => 0
+# f_p_a_e ("qwerty\r\n", "\r\n") => <undefined>
+
+# this could maybe be made faster with a computed regex?
+# [answer: no; circa Python-2.0, Jan 2001]
+# new python:   28961/s
+# old python:   18307/s
+# re:        12820/s
+# regex:     14035/s
+
+def find_prefix_at_end (haystack, needle):
+    l = len(needle) - 1
+    while l and not haystack.endswith(needle[:l]):
+        l -= 1
+    return l
diff --git a/lib-python/3.2/asyncore.py b/lib-python/3.2/asyncore.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/asyncore.py
@@ -0,0 +1,663 @@
+# -*- Mode: Python -*-
+#   Id: asyncore.py,v 2.51 2000/09/07 22:29:26 rushing Exp
+#   Author: Sam Rushing <rushing at nightmare.com>
+
+# ======================================================================
+# Copyright 1996 by Sam Rushing
+#
+#                         All Rights Reserved
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose and without fee is hereby
+# granted, provided that the above copyright notice appear in all
+# copies and that both that copyright notice and this permission
+# notice appear in supporting documentation, and that the name of Sam
+# Rushing not be used in advertising or publicity pertaining to
+# distribution of the software without specific, written prior
+# permission.
+#
+# SAM RUSHING DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
+# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN
+# NO EVENT SHALL SAM RUSHING BE LIABLE FOR ANY SPECIAL, INDIRECT OR
+# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
+# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+# ======================================================================
+
+"""Basic infrastructure for asynchronous socket service clients and servers.
+
+There are only two ways to have a program on a single processor do "more
+than one thing at a time".  Multi-threaded programming is the simplest and
+most popular way to do it, but there is another very different technique,
+that lets you have nearly all the advantages of multi-threading, without
+actually using multiple threads. it's really only practical if your program
+is largely I/O bound. If your program is CPU bound, then pre-emptive
+scheduled threads are probably what you really need. Network servers are
+rarely CPU-bound, however.
+
+If your operating system supports the select() system call in its I/O
+library (and nearly all do), then you can use it to juggle multiple
+communication channels at once; doing other work while your I/O is taking
+place in the "background."  Although this strategy can seem strange and
+complex, especially at first, it is in many ways easier to understand and
+control than multi-threaded programming. The module documented here solves
+many of the difficult problems for you, making the task of building
+sophisticated high-performance network servers and clients a snap.
+"""
+
+import select
+import socket
+import sys
+import time
+import warnings
+
+import os
+from errno import EALREADY, EINPROGRESS, EWOULDBLOCK, ECONNRESET, EINVAL, \
+     ENOTCONN, ESHUTDOWN, EINTR, EISCONN, EBADF, ECONNABORTED, EPIPE, EAGAIN, \
+     errorcode
+
+_DISCONNECTED = frozenset((ECONNRESET, ENOTCONN, ESHUTDOWN, ECONNABORTED, EPIPE,
+                           EBADF))
+
+try:
+    socket_map
+except NameError:
+    socket_map = {}
+
+def _strerror(err):
+    try:
+        return os.strerror(err)
+    except (ValueError, OverflowError, NameError):
+        if err in errorcode:
+            return errorcode[err]
+        return "Unknown error %s" %err
+
+class ExitNow(Exception):
+    pass
+
+_reraised_exceptions = (ExitNow, KeyboardInterrupt, SystemExit)
+
+def read(obj):
+    try:
+        obj.handle_read_event()
+    except _reraised_exceptions:
+        raise
+    except:
+        obj.handle_error()
+
+def write(obj):
+    try:
+        obj.handle_write_event()
+    except _reraised_exceptions:
+        raise
+    except:
+        obj.handle_error()
+
+def _exception(obj):
+    try:
+        obj.handle_expt_event()
+    except _reraised_exceptions:
+        raise
+    except:
+        obj.handle_error()
+
+def readwrite(obj, flags):
+    try:
+        if flags & select.POLLIN:
+            obj.handle_read_event()
+        if flags & select.POLLOUT:
+            obj.handle_write_event()
+        if flags & select.POLLPRI:
+            obj.handle_expt_event()
+        if flags & (select.POLLHUP | select.POLLERR | select.POLLNVAL):
+            obj.handle_close()
+    except socket.error as e:
+        if e.args[0] not in _DISCONNECTED:
+            obj.handle_error()
+        else:
+            obj.handle_close()
+    except _reraised_exceptions:
+        raise
+    except:
+        obj.handle_error()
+
+def poll(timeout=0.0, map=None):
+    if map is None:
+        map = socket_map
+    if map:
+        r = []; w = []; e = []
+        for fd, obj in list(map.items()):
+            is_r = obj.readable()
+            is_w = obj.writable()
+            if is_r:
+                r.append(fd)
+            # accepting sockets should not be writable
+            if is_w and not obj.accepting:
+                w.append(fd)
+            if is_r or is_w:
+                e.append(fd)
+        if [] == r == w == e:
+            time.sleep(timeout)
+            return
+
+        try:
+            r, w, e = select.select(r, w, e, timeout)
+        except select.error as err:
+            if err.args[0] != EINTR:
+                raise
+            else:
+                return
+
+        for fd in r:
+            obj = map.get(fd)
+            if obj is None:
+                continue
+            read(obj)
+
+        for fd in w:
+            obj = map.get(fd)
+            if obj is None:
+                continue
+            write(obj)
+
+        for fd in e:
+            obj = map.get(fd)
+            if obj is None:
+                continue
+            _exception(obj)
+
+def poll2(timeout=0.0, map=None):
+    # Use the poll() support added to the select module in Python 2.0
+    if map is None:
+        map = socket_map
+    if timeout is not None:
+        # timeout is in milliseconds
+        timeout = int(timeout*1000)
+    pollster = select.poll()
+    if map:
+        for fd, obj in list(map.items()):
+            flags = 0
+            if obj.readable():
+                flags |= select.POLLIN | select.POLLPRI
+            # accepting sockets should not be writable
+            if obj.writable() and not obj.accepting:
+                flags |= select.POLLOUT
+            if flags:
+                # Only check for exceptions if object was either readable
+                # or writable.
+                flags |= select.POLLERR | select.POLLHUP | select.POLLNVAL
+                pollster.register(fd, flags)
+        try:
+            r = pollster.poll(timeout)
+        except select.error as err:
+            if err.args[0] != EINTR:
+                raise
+            r = []
+        for fd, flags in r:
+            obj = map.get(fd)
+            if obj is None:
+                continue
+            readwrite(obj, flags)
+
+poll3 = poll2                           # Alias for backward compatibility
+
+def loop(timeout=30.0, use_poll=False, map=None, count=None):
+    if map is None:
+        map = socket_map
+
+    if use_poll and hasattr(select, 'poll'):
+        poll_fun = poll2
+    else:
+        poll_fun = poll
+
+    if count is None:
+        while map:
+            poll_fun(timeout, map)
+
+    else:
+        while map and count > 0:
+            poll_fun(timeout, map)
+            count = count - 1
+
+class dispatcher:
+
+    debug = False
+    connected = False
+    accepting = False
+    closing = False
+    addr = None
+    ignore_log_types = frozenset(['warning'])
+
+    def __init__(self, sock=None, map=None):
+        if map is None:
+            self._map = socket_map
+        else:
+            self._map = map
+
+        self._fileno = None
+
+        if sock:
+            # Set to nonblocking just to make sure for cases where we
+            # get a socket from a blocking source.
+            sock.setblocking(0)
+            self.set_socket(sock, map)
+            self.connected = True
+            # The constructor no longer requires that the socket
+            # passed be connected.
+            try:
+                self.addr = sock.getpeername()
+            except socket.error as err:
+                if err.args[0] == ENOTCONN:
+                    # To handle the case where we got an unconnected
+                    # socket.
+                    self.connected = False
+                else:
+                    # The socket is broken in some unknown way, alert
+                    # the user and remove it from the map (to prevent
+                    # polling of broken sockets).
+                    self.del_channel(map)
+                    raise
+        else:
+            self.socket = None
+
+    def __repr__(self):
+        status = [self.__class__.__module__+"."+self.__class__.__name__]
+        if self.accepting and self.addr:
+            status.append('listening')
+        elif self.connected:
+            status.append('connected')
+        if self.addr is not None:
+            try:
+                status.append('%s:%d' % self.addr)
+            except TypeError:
+                status.append(repr(self.addr))
+        return '<%s at %#x>' % (' '.join(status), id(self))
+
+    __str__ = __repr__
+
+    def add_channel(self, map=None):
+        #self.log_info('adding channel %s' % self)
+        if map is None:
+            map = self._map
+        map[self._fileno] = self
+
+    def del_channel(self, map=None):
+        fd = self._fileno
+        if map is None:
+            map = self._map
+        if fd in map:
+            #self.log_info('closing channel %d:%s' % (fd, self))
+            del map[fd]
+        self._fileno = None
+
+    def create_socket(self, family, type):
+        self.family_and_type = family, type
+        sock = socket.socket(family, type)
+        sock.setblocking(0)
+        self.set_socket(sock)
+
+    def set_socket(self, sock, map=None):
+        self.socket = sock
+##        self.__dict__['socket'] = sock
+        self._fileno = sock.fileno()
+        self.add_channel(map)
+
+    def set_reuse_addr(self):
+        # try to re-use a server port if possible
+        try:
+            self.socket.setsockopt(
+                socket.SOL_SOCKET, socket.SO_REUSEADDR,
+                self.socket.getsockopt(socket.SOL_SOCKET,
+                                       socket.SO_REUSEADDR) | 1
+                )
+        except socket.error:
+            pass
+
+    # ==================================================
+    # predicates for select()
+    # these are used as filters for the lists of sockets
+    # to pass to select().
+    # ==================================================
+
+    def readable(self):
+        return True
+
+    def writable(self):
+        return True
+
+    # ==================================================
+    # socket object methods.
+    # ==================================================
+
+    def listen(self, num):
+        self.accepting = True
+        if os.name == 'nt' and num > 5:
+            num = 5
+        return self.socket.listen(num)
+
+    def bind(self, addr):
+        self.addr = addr
+        return self.socket.bind(addr)
+
+    def connect(self, address):
+        self.connected = False
+        err = self.socket.connect_ex(address)
+        if err in (EINPROGRESS, EALREADY, EWOULDBLOCK) \
+        or err == EINVAL and os.name in ('nt', 'ce'):
+            return
+        if err in (0, EISCONN):
+            self.addr = address
+            self.handle_connect_event()
+        else:
+            raise socket.error(err, errorcode[err])
+
+    def accept(self):
+        # XXX can return either an address pair or None
+        try:
+            conn, addr = self.socket.accept()
+        except TypeError:
+            return None
+        except socket.error as why:
+            if why.args[0] in (EWOULDBLOCK, ECONNABORTED, EAGAIN):
+                return None
+            else:
+                raise
+        else:
+            return conn, addr
+
+    def send(self, data):
+        try:
+            result = self.socket.send(data)
+            return result
+        except socket.error as why:
+            if why.args[0] == EWOULDBLOCK:
+                return 0
+            elif why.args[0] in _DISCONNECTED:
+                self.handle_close()
+                return 0
+            else:
+                raise
+
+    def recv(self, buffer_size):
+        try:
+            data = self.socket.recv(buffer_size)
+            if not data:
+                # a closed connection is indicated by signaling
+                # a read condition, and having recv() return 0.
+                self.handle_close()
+                return b''
+            else:
+                return data
+        except socket.error as why:
+            # winsock sometimes throws ENOTCONN
+            if why.args[0] in _DISCONNECTED:
+                self.handle_close()
+                return b''
+            else:
+                raise
+
+    def close(self):
+        self.connected = False
+        self.accepting = False
+        self.del_channel()
+        try:
+            self.socket.close()
+        except socket.error as why:
+            if why.args[0] not in (ENOTCONN, EBADF):
+                raise
+
+    # cheap inheritance, used to pass all other attribute
+    # references to the underlying socket object.
+    def __getattr__(self, attr):
+        try:
+            retattr = getattr(self.socket, attr)
+        except AttributeError:
+            raise AttributeError("%s instance has no attribute '%s'"
+                                 %(self.__class__.__name__, attr))
+        else:
+            msg = "%(me)s.%(attr)s is deprecated; use %(me)s.socket.%(attr)s " \
+                  "instead" % {'me' : self.__class__.__name__, 'attr' : attr}
+            warnings.warn(msg, DeprecationWarning, stacklevel=2)
+            return retattr
+
+    # log and log_info may be overridden to provide more sophisticated
+    # logging and warning methods. In general, log is for 'hit' logging
+    # and 'log_info' is for informational, warning and error logging.
+
+    def log(self, message):
+        sys.stderr.write('log: %s\n' % str(message))
+
+    def log_info(self, message, type='info'):
+        if type not in self.ignore_log_types:
+            print('%s: %s' % (type, message))
+
+    def handle_read_event(self):
+        if self.accepting:
+            # accepting sockets are never connected, they "spawn" new
+            # sockets that are connected
+            self.handle_accept()
+        elif not self.connected:
+            self.handle_connect_event()
+            self.handle_read()
+        else:
+            self.handle_read()
+
+    def handle_connect_event(self):
+        err = self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
+        if err != 0:
+            raise socket.error(err, _strerror(err))
+        self.handle_connect()
+        self.connected = True
+
+    def handle_write_event(self):
+        if self.accepting:
+            # Accepting sockets shouldn't get a write event.
+            # We will pretend it didn't happen.
+            return
+
+        if not self.connected:
+            #check for errors
+            err = self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
+            if err != 0:
+                raise socket.error(err, _strerror(err))
+
+            self.handle_connect_event()
+        self.handle_write()
+
+    def handle_expt_event(self):
+        # handle_expt_event() is called if there might be an error on the
+        # socket, or if there is OOB data
+        # check for the error condition first
+        err = self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
+        if err != 0:
+            # we can get here when select.select() says that there is an
+            # exceptional condition on the socket
+            # since there is an error, we'll go ahead and close the socket
+            # like we would in a subclassed handle_read() that received no
+            # data
+            self.handle_close()
+        else:
+            self.handle_expt()
+
+    def handle_error(self):
+        nil, t, v, tbinfo = compact_traceback()
+
+        # sometimes a user repr method will crash.
+        try:
+            self_repr = repr(self)
+        except:
+            self_repr = '<__repr__(self) failed for object at %0x>' % id(self)
+
+        self.log_info(
+            'uncaptured python exception, closing channel %s (%s:%s %s)' % (
+                self_repr,
+                t,
+                v,
+                tbinfo
+                ),
+            'error'
+            )
+        self.handle_close()
+
+    def handle_expt(self):
+        self.log_info('unhandled incoming priority event', 'warning')
+
+    def handle_read(self):
+        self.log_info('unhandled read event', 'warning')
+
+    def handle_write(self):
+        self.log_info('unhandled write event', 'warning')
+
+    def handle_connect(self):
+        self.log_info('unhandled connect event', 'warning')
+
+    def handle_accept(self):
+        pair = self.accept()
+        if pair is not None:
+            self.handle_accepted(*pair)
+
+    def handle_accepted(self, sock, addr):
+        sock.close()
+        self.log_info('unhandled accepted event', 'warning')
+
+    def handle_close(self):
+        self.log_info('unhandled close event', 'warning')
+        self.close()
+
+# ---------------------------------------------------------------------------
+# adds simple buffered output capability, useful for simple clients.
+# [for more sophisticated usage use asynchat.async_chat]
+# ---------------------------------------------------------------------------
+
+class dispatcher_with_send(dispatcher):
+
+    def __init__(self, sock=None, map=None):
+        dispatcher.__init__(self, sock, map)
+        self.out_buffer = b''
+
+    def initiate_send(self):
+        num_sent = 0
+        num_sent = dispatcher.send(self, self.out_buffer[:512])
+        self.out_buffer = self.out_buffer[num_sent:]
+
+    def handle_write(self):
+        self.initiate_send()
+
+    def writable(self):
+        return (not self.connected) or len(self.out_buffer)
+
+    def send(self, data):
+        if self.debug:
+            self.log_info('sending %s' % repr(data))
+        self.out_buffer = self.out_buffer + data
+        self.initiate_send()
+
+# ---------------------------------------------------------------------------
+# used for debugging.
+# ---------------------------------------------------------------------------
+
+def compact_traceback():
+    t, v, tb = sys.exc_info()
+    tbinfo = []
+    if not tb: # Must have a traceback
+        raise AssertionError("traceback does not exist")
+    while tb:
+        tbinfo.append((
+            tb.tb_frame.f_code.co_filename,
+            tb.tb_frame.f_code.co_name,
+            str(tb.tb_lineno)
+            ))
+        tb = tb.tb_next
+
+    # just to be safe
+    del tb
+
+    file, function, line = tbinfo[-1]
+    info = ' '.join(['[%s|%s|%s]' % x for x in tbinfo])
+    return (file, function, line), t, v, info
+
+def close_all(map=None, ignore_all=False):
+    if map is None:
+        map = socket_map
+    for x in list(map.values()):
+        try:
+            x.close()
+        except OSError as x:
+            if x.args[0] == EBADF:
+                pass
+            elif not ignore_all:
+                raise
+        except _reraised_exceptions:
+            raise
+        except:
+            if not ignore_all:
+                raise
+    map.clear()
+
+# Asynchronous File I/O:
+#
+# After a little research (reading man pages on various unixen, and
+# digging through the linux kernel), I've determined that select()
+# isn't meant for doing asynchronous file i/o.
+# Heartening, though - reading linux/mm/filemap.c shows that linux
+# supports asynchronous read-ahead.  So _MOST_ of the time, the data
+# will be sitting in memory for us already when we go to read it.
+#
+# What other OS's (besides NT) support async file i/o?  [VMS?]
+#
+# Regardless, this is useful for pipes, and stdin/stdout...
+
+if os.name == 'posix':
+    import fcntl
+
+    class file_wrapper:
+        # Here we override just enough to make a file
+        # look like a socket for the purposes of asyncore.
+        # The passed fd is automatically os.dup()'d
+
+        def __init__(self, fd):
+            self.fd = os.dup(fd)
+
+        def recv(self, *args):
+            return os.read(self.fd, *args)
+
+        def send(self, *args):
+            return os.write(self.fd, *args)
+
+        def getsockopt(self, level, optname, buflen=None):
+            if (level == socket.SOL_SOCKET and
+                optname == socket.SO_ERROR and
+                not buflen):
+                return 0
+            raise NotImplementedError("Only asyncore specific behaviour "
+                                      "implemented.")
+
+        read = recv
+        write = send
+
+        def close(self):
+            os.close(self.fd)
+
+        def fileno(self):
+            return self.fd
+
+    class file_dispatcher(dispatcher):
+
+        def __init__(self, fd, map=None):
+            dispatcher.__init__(self, None, map)
+            self.connected = True
+            try:
+                fd = fd.fileno()
+            except AttributeError:
+                pass
+            self.set_file(fd)
+            # set it to non-blocking mode
+            flags = fcntl.fcntl(fd, fcntl.F_GETFL, 0)
+            flags = flags | os.O_NONBLOCK
+            fcntl.fcntl(fd, fcntl.F_SETFL, flags)
+
+        def set_file(self, fd):
+            self.socket = file_wrapper(fd)
+            self._fileno = self.socket.fileno()
+            self.add_channel()
diff --git a/lib-python/3.2/base64.py b/lib-python/3.2/base64.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/base64.py
@@ -0,0 +1,408 @@
+#! /usr/bin/env python3
+
+"""RFC 3548: Base16, Base32, Base64 Data Encodings"""
+
+# Modified 04-Oct-1995 by Jack Jansen to use binascii module
+# Modified 30-Dec-2003 by Barry Warsaw to add full RFC 3548 support
+# Modified 22-May-2007 by Guido van Rossum to use bytes everywhere
+
+import re
+import struct
+import binascii
+
+
+__all__ = [
+    # Legacy interface exports traditional RFC 1521 Base64 encodings
+    'encode', 'decode', 'encodebytes', 'decodebytes',
+    # Generalized interface for other encodings
+    'b64encode', 'b64decode', 'b32encode', 'b32decode',
+    'b16encode', 'b16decode',
+    # Standard Base64 encoding
+    'standard_b64encode', 'standard_b64decode',
+    # Some common Base64 alternatives.  As referenced by RFC 3458, see thread
+    # starting at:
+    #
+    # http://zgp.org/pipermail/p2p-hackers/2001-September/000316.html
+    'urlsafe_b64encode', 'urlsafe_b64decode',
+    ]
+
+
+bytes_types = (bytes, bytearray)  # Types acceptable as binary data
+
+
+def _translate(s, altchars):
+    if not isinstance(s, bytes_types):
+        raise TypeError("expected bytes, not %s" % s.__class__.__name__)
+    translation = bytearray(range(256))
+    for k, v in altchars.items():
+        translation[ord(k)] = v[0]
+    return s.translate(translation)
+
+
+
+# Base64 encoding/decoding uses binascii
+
+def b64encode(s, altchars=None):
+    """Encode a byte string using Base64.
+
+    s is the byte string to encode.  Optional altchars must be a byte
+    string of length 2 which specifies an alternative alphabet for the
+    '+' and '/' characters.  This allows an application to
+    e.g. generate url or filesystem safe Base64 strings.
+
+    The encoded byte string is returned.
+    """
+    if not isinstance(s, bytes_types):
+        raise TypeError("expected bytes, not %s" % s.__class__.__name__)
+    # Strip off the trailing newline
+    encoded = binascii.b2a_base64(s)[:-1]
+    if altchars is not None:
+        if not isinstance(altchars, bytes_types):
+            raise TypeError("expected bytes, not %s"
+                            % altchars.__class__.__name__)
+        assert len(altchars) == 2, repr(altchars)
+        return _translate(encoded, {'+': altchars[0:1], '/': altchars[1:2]})
+    return encoded
+
+
+def b64decode(s, altchars=None, validate=False):
+    """Decode a Base64 encoded byte string.
+
+    s is the byte string to decode.  Optional altchars must be a
+    string of length 2 which specifies the alternative alphabet used
+    instead of the '+' and '/' characters.
+
+    The decoded string is returned.  A binascii.Error is raised if s is
+    incorrectly padded.
+
+    If validate is False (the default), non-base64-alphabet characters are
+    discarded prior to the padding check.  If validate is True,
+    non-base64-alphabet characters in the input result in a binascii.Error.
+    """
+    if not isinstance(s, bytes_types):
+        raise TypeError("expected bytes, not %s" % s.__class__.__name__)
+    if altchars is not None:
+        if not isinstance(altchars, bytes_types):
+            raise TypeError("expected bytes, not %s"
+                            % altchars.__class__.__name__)
+        assert len(altchars) == 2, repr(altchars)
+        s = _translate(s, {chr(altchars[0]): b'+', chr(altchars[1]): b'/'})
+    if validate and not re.match(b'^[A-Za-z0-9+/]*={0,2}$', s):
+        raise binascii.Error('Non-base64 digit found')
+    return binascii.a2b_base64(s)
+
+
+def standard_b64encode(s):
+    """Encode a byte string using the standard Base64 alphabet.
+
+    s is the byte string to encode.  The encoded byte string is returned.
+    """
+    return b64encode(s)
+
+def standard_b64decode(s):
+    """Decode a byte string encoded with the standard Base64 alphabet.
+
+    s is the byte string to decode.  The decoded byte string is
+    returned.  binascii.Error is raised if the input is incorrectly
+    padded or if there are non-alphabet characters present in the
+    input.
+    """
+    return b64decode(s)
+
+def urlsafe_b64encode(s):
+    """Encode a byte string using a url-safe Base64 alphabet.
+
+    s is the byte string to encode.  The encoded byte string is
+    returned.  The alphabet uses '-' instead of '+' and '_' instead of
+    '/'.
+    """
+    return b64encode(s, b'-_')
+
+def urlsafe_b64decode(s):
+    """Decode a byte string encoded with the standard Base64 alphabet.
+
+    s is the byte string to decode.  The decoded byte string is
+    returned.  binascii.Error is raised if the input is incorrectly
+    padded or if there are non-alphabet characters present in the
+    input.
+
+    The alphabet uses '-' instead of '+' and '_' instead of '/'.
+    """
+    return b64decode(s, b'-_')
+
+
+
+# Base32 encoding/decoding must be done in Python
+_b32alphabet = {
+    0: b'A',  9: b'J', 18: b'S', 27: b'3',
+    1: b'B', 10: b'K', 19: b'T', 28: b'4',
+    2: b'C', 11: b'L', 20: b'U', 29: b'5',
+    3: b'D', 12: b'M', 21: b'V', 30: b'6',
+    4: b'E', 13: b'N', 22: b'W', 31: b'7',
+    5: b'F', 14: b'O', 23: b'X',
+    6: b'G', 15: b'P', 24: b'Y',
+    7: b'H', 16: b'Q', 25: b'Z',
+    8: b'I', 17: b'R', 26: b'2',
+    }
+
+_b32tab = [v[0] for k, v in sorted(_b32alphabet.items())]
+_b32rev = dict([(v[0], k) for k, v in _b32alphabet.items()])
+
+
+def b32encode(s):
+    """Encode a byte string using Base32.
+
+    s is the byte string to encode.  The encoded byte string is returned.
+    """
+    if not isinstance(s, bytes_types):
+        raise TypeError("expected bytes, not %s" % s.__class__.__name__)
+    quanta, leftover = divmod(len(s), 5)
+    # Pad the last quantum with zero bits if necessary
+    if leftover:
+        s = s + bytes(5 - leftover)  # Don't use += !
+        quanta += 1
+    encoded = bytes()
+    for i in range(quanta):
+        # c1 and c2 are 16 bits wide, c3 is 8 bits wide.  The intent of this
+        # code is to process the 40 bits in units of 5 bits.  So we take the 1
+        # leftover bit of c1 and tack it onto c2.  Then we take the 2 leftover
+        # bits of c2 and tack them onto c3.  The shifts and masks are intended
+        # to give us values of exactly 5 bits in width.
+        c1, c2, c3 = struct.unpack('!HHB', s[i*5:(i+1)*5])
+        c2 += (c1 & 1) << 16 # 17 bits wide
+        c3 += (c2 & 3) << 8  # 10 bits wide
+        encoded += bytes([_b32tab[c1 >> 11],         # bits 1 - 5
+                          _b32tab[(c1 >> 6) & 0x1f], # bits 6 - 10
+                          _b32tab[(c1 >> 1) & 0x1f], # bits 11 - 15
+                          _b32tab[c2 >> 12],         # bits 16 - 20 (1 - 5)
+                          _b32tab[(c2 >> 7) & 0x1f], # bits 21 - 25 (6 - 10)
+                          _b32tab[(c2 >> 2) & 0x1f], # bits 26 - 30 (11 - 15)
+                          _b32tab[c3 >> 5],          # bits 31 - 35 (1 - 5)
+                          _b32tab[c3 & 0x1f],        # bits 36 - 40 (1 - 5)
+                          ])
+    # Adjust for any leftover partial quanta
+    if leftover == 1:
+        return encoded[:-6] + b'======'
+    elif leftover == 2:
+        return encoded[:-4] + b'===='
+    elif leftover == 3:
+        return encoded[:-3] + b'==='
+    elif leftover == 4:
+        return encoded[:-1] + b'='
+    return encoded
+
+
+def b32decode(s, casefold=False, map01=None):
+    """Decode a Base32 encoded byte string.
+
+    s is the byte string to decode.  Optional casefold is a flag
+    specifying whether a lowercase alphabet is acceptable as input.
+    For security purposes, the default is False.
+
+    RFC 3548 allows for optional mapping of the digit 0 (zero) to the
+    letter O (oh), and for optional mapping of the digit 1 (one) to
+    either the letter I (eye) or letter L (el).  The optional argument
+    map01 when not None, specifies which letter the digit 1 should be
+    mapped to (when map01 is not None, the digit 0 is always mapped to
+    the letter O).  For security purposes the default is None, so that
+    0 and 1 are not allowed in the input.
+
+    The decoded byte string is returned.  binascii.Error is raised if
+    the input is incorrectly padded or if there are non-alphabet
+    characters present in the input.
+    """
+    if not isinstance(s, bytes_types):
+        raise TypeError("expected bytes, not %s" % s.__class__.__name__)
+    quanta, leftover = divmod(len(s), 8)
+    if leftover:
+        raise binascii.Error('Incorrect padding')
+    # Handle section 2.4 zero and one mapping.  The flag map01 will be either
+    # False, or the character to map the digit 1 (one) to.  It should be
+    # either L (el) or I (eye).
+    if map01 is not None:
+        if not isinstance(map01, bytes_types):
+            raise TypeError("expected bytes, not %s" % map01.__class__.__name__)
+        assert len(map01) == 1, repr(map01)
+        s = _translate(s, {b'0': b'O', b'1': map01})
+    if casefold:
+        s = s.upper()
+    # Strip off pad characters from the right.  We need to count the pad
+    # characters because this will tell us how many null bytes to remove from
+    # the end of the decoded string.
+    padchars = 0
+    mo = re.search(b'(?P<pad>[=]*)$', s)
+    if mo:
+        padchars = len(mo.group('pad'))
+        if padchars > 0:
+            s = s[:-padchars]
+    # Now decode the full quanta
+    parts = []
+    acc = 0
+    shift = 35
+    for c in s:
+        val = _b32rev.get(c)
+        if val is None:
+            raise TypeError('Non-base32 digit found')
+        acc += _b32rev[c] << shift
+        shift -= 5
+        if shift < 0:
+            parts.append(binascii.unhexlify(bytes('%010x' % acc, "ascii")))
+            acc = 0
+            shift = 35
+    # Process the last, partial quanta
+    last = binascii.unhexlify(bytes('%010x' % acc, "ascii"))
+    if padchars == 0:
+        last = b''                      # No characters
+    elif padchars == 1:
+        last = last[:-1]
+    elif padchars == 3:
+        last = last[:-2]
+    elif padchars == 4:
+        last = last[:-3]
+    elif padchars == 6:
+        last = last[:-4]
+    else:
+        raise binascii.Error('Incorrect padding')
+    parts.append(last)
+    return b''.join(parts)
+
+
+
+# RFC 3548, Base 16 Alphabet specifies uppercase, but hexlify() returns
+# lowercase.  The RFC also recommends against accepting input case
+# insensitively.
+def b16encode(s):
+    """Encode a byte string using Base16.
+
+    s is the byte string to encode.  The encoded byte string is returned.
+    """
+    if not isinstance(s, bytes_types):
+        raise TypeError("expected bytes, not %s" % s.__class__.__name__)
+    return binascii.hexlify(s).upper()
+
+
+def b16decode(s, casefold=False):
+    """Decode a Base16 encoded byte string.
+
+    s is the byte string to decode.  Optional casefold is a flag
+    specifying whether a lowercase alphabet is acceptable as input.
+    For security purposes, the default is False.
+
+    The decoded byte string is returned.  binascii.Error is raised if
+    s were incorrectly padded or if there are non-alphabet characters
+    present in the string.
+    """
+    if not isinstance(s, bytes_types):
+        raise TypeError("expected bytes, not %s" % s.__class__.__name__)
+    if casefold:
+        s = s.upper()
+    if re.search(b'[^0-9A-F]', s):
+        raise binascii.Error('Non-base16 digit found')
+    return binascii.unhexlify(s)
+
+
+
+# Legacy interface.  This code could be cleaned up since I don't believe
+# binascii has any line length limitations.  It just doesn't seem worth it
+# though.  The files should be opened in binary mode.
+
+MAXLINESIZE = 76 # Excluding the CRLF
+MAXBINSIZE = (MAXLINESIZE//4)*3
+
+def encode(input, output):
+    """Encode a file; input and output are binary files."""
+    while True:
+        s = input.read(MAXBINSIZE)
+        if not s:
+            break
+        while len(s) < MAXBINSIZE:
+            ns = input.read(MAXBINSIZE-len(s))
+            if not ns:
+                break
+            s += ns
+        line = binascii.b2a_base64(s)
+        output.write(line)
+
+
+def decode(input, output):
+    """Decode a file; input and output are binary files."""
+    while True:
+        line = input.readline()
+        if not line:
+            break
+        s = binascii.a2b_base64(line)
+        output.write(s)
+
+
+def encodebytes(s):
+    """Encode a bytestring into a bytestring containing multiple lines
+    of base-64 data."""
+    if not isinstance(s, bytes_types):
+        raise TypeError("expected bytes, not %s" % s.__class__.__name__)
+    pieces = []
+    for i in range(0, len(s), MAXBINSIZE):
+        chunk = s[i : i + MAXBINSIZE]
+        pieces.append(binascii.b2a_base64(chunk))
+    return b"".join(pieces)
+
+def encodestring(s):
+    """Legacy alias of encodebytes()."""
+    import warnings
+    warnings.warn("encodestring() is a deprecated alias, use encodebytes()",
+                  DeprecationWarning, 2)
+    return encodebytes(s)
+
+
+def decodebytes(s):
+    """Decode a bytestring of base-64 data into a bytestring."""
+    if not isinstance(s, bytes_types):
+        raise TypeError("expected bytes, not %s" % s.__class__.__name__)
+    return binascii.a2b_base64(s)
+
+def decodestring(s):
+    """Legacy alias of decodebytes()."""
+    import warnings
+    warnings.warn("decodestring() is a deprecated alias, use decodebytes()",
+                  DeprecationWarning, 2)
+    return decodebytes(s)
+
+
+# Usable as a script...
+def main():
+    """Small main program"""
+    import sys, getopt
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], 'deut')
+    except getopt.error as msg:
+        sys.stdout = sys.stderr
+        print(msg)
+        print("""usage: %s [-d|-e|-u|-t] [file|-]
+        -d, -u: decode
+        -e: encode (default)
+        -t: encode and decode string 'Aladdin:open sesame'"""%sys.argv[0])
+        sys.exit(2)
+    func = encode
+    for o, a in opts:
+        if o == '-e': func = encode
+        if o == '-d': func = decode
+        if o == '-u': func = decode
+        if o == '-t': test(); return
+    if args and args[0] != '-':
+        with open(args[0], 'rb') as f:
+            func(f, sys.stdout.buffer)
+    else:
+        func(sys.stdin.buffer, sys.stdout.buffer)
+
+
+def test():
+    s0 = b"Aladdin:open sesame"
+    print(repr(s0))
+    s1 = encodebytes(s0)
+    print(repr(s1))
+    s2 = decodebytes(s1)
+    print(repr(s2))
+    assert s0 == s2
+
+
+if __name__ == '__main__':
+    main()
diff --git a/lib-python/3.2/bdb.py b/lib-python/3.2/bdb.py
new file mode 100644
--- /dev/null
+++ b/lib-python/3.2/bdb.py
@@ -0,0 +1,634 @@
+"""Debugger basics"""
+
+import fnmatch
+import sys
+import os
+
+__all__ = ["BdbQuit", "Bdb", "Breakpoint"]
+
+class BdbQuit(Exception):
+    """Exception to give up completely."""
+
+
+class Bdb:
+    """Generic Python debugger base class.
+
+    This class takes care of details of the trace facility;
+    a derived class should implement user interaction.
+    The standard debugger class (pdb.Pdb) is an example.
+    """
+
+    def __init__(self, skip=None):
+        self.skip = set(skip) if skip else None
+        self.breaks = {}
+        self.fncache = {}
+
+    def canonic(self, filename):
+        if filename == "<" + filename[1:-1] + ">":
+            return filename
+        canonic = self.fncache.get(filename)
+        if not canonic:
+            canonic = os.path.abspath(filename)
+            canonic = os.path.normcase(canonic)
+            self.fncache[filename] = canonic
+        return canonic
+
+    def reset(self):
+        import linecache
+        linecache.checkcache()
+        self.botframe = None
+        self._set_stopinfo(None, None)
+
+    def trace_dispatch(self, frame, event, arg):
+        if self.quitting:
+            return # None
+        if event == 'line':
+            return self.dispatch_line(frame)
+        if event == 'call':
+            return self.dispatch_call(frame, arg)
+        if event == 'return':
+            return self.dispatch_return(frame, arg)
+        if event == 'exception':
+            return self.dispatch_exception(frame, arg)
+        if event == 'c_call':
+            return self.trace_dispatch
+        if event == 'c_exception':
+            return self.trace_dispatch
+        if event == 'c_return':
+            return self.trace_dispatch
+        print('bdb.Bdb.dispatch: unknown debugging event:', repr(event))
+        return self.trace_dispatch
+
+    def dispatch_line(self, frame):
+        if self.stop_here(frame) or self.break_here(frame):
+            self.user_line(frame)
+            if self.quitting: raise BdbQuit
+        return self.trace_dispatch
+
+    def dispatch_call(self, frame, arg):
+        # XXX 'arg' is no longer used
+        if self.botframe is None:
+            # First call of dispatch since reset()
+            self.botframe = frame.f_back # (CT) Note that this may also be None!
+            return self.trace_dispatch
+        if not (self.stop_here(frame) or self.break_anywhere(frame)):
+            # No need to trace this function
+            return # None
+        self.user_call(frame, arg)
+        if self.quitting: raise BdbQuit
+        return self.trace_dispatch
+
+    def dispatch_return(self, frame, arg):
+        if self.stop_here(frame) or frame == self.returnframe:
+            self.user_return(frame, arg)
+            if self.quitting: raise BdbQuit
+        return self.trace_dispatch
+
+    def dispatch_exception(self, frame, arg):
+        if self.stop_here(frame):
+            self.user_exception(frame, arg)
+            if self.quitting: raise BdbQuit
+        return self.trace_dispatch
+
+    # Normally derived classes don't override the following
+    # methods, but they may if they want to redefine the
+    # definition of stopping and breakpoints.
+
+    def is_skipped_module(self, module_name):
+        for pattern in self.skip:
+            if fnmatch.fnmatch(module_name, pattern):
+                return True
+        return False
+
+    def stop_here(self, frame):
+        # (CT) stopframe may now also be None, see dispatch_call.
+        # (CT) the former test for None is therefore removed from here.
+        if self.skip and \
+               self.is_skipped_module(frame.f_globals.get('__name__')):
+            return False
+        if frame is self.stopframe:
+            if self.stoplineno == -1:
+                return False
+            return frame.f_lineno >= self.stoplineno
+        while frame is not None and frame is not self.stopframe:
+            if frame is self.botframe:
+                return True
+            frame = frame.f_back
+        return False
+
+    def break_here(self, frame):
+        filename = self.canonic(frame.f_code.co_filename)
+        if filename not in self.breaks:
+            return False
+        lineno = frame.f_lineno
+        if lineno not in self.breaks[filename]:
+            # The line itself has no breakpoint, but maybe the line is the
+            # first line of a function with breakpoint set by function name.
+            lineno = frame.f_code.co_firstlineno
+            if lineno not in self.breaks[filename]:
+                return False
+
+        # flag says ok to delete temp. bp
+        (bp, flag) = effective(filename, lineno, frame)
+        if bp:
+            self.currentbp = bp.number
+            if (flag and bp.temporary):
+                self.do_clear(str(bp.number))
+            return True
+        else:
+            return False
+
+    def do_clear(self, arg):
+        raise NotImplementedError("subclass of bdb must implement do_clear()")
+
+    def break_anywhere(self, frame):
+        return self.canonic(frame.f_code.co_filename) in self.breaks
+
+    # Derived classes should override the user_* methods
+    # to gain control.
+
+    def user_call(self, frame, argument_list):
+        """This method is called when there is the remote possibility
+        that we ever need to stop in this function."""
+        pass
+
+    def user_line(self, frame):
+        """This method is called when we stop or break at this line."""
+        pass
+
+    def user_return(self, frame, return_value):
+        """This method is called when a return trap is set here."""
+        pass
+
+    def user_exception(self, frame, exc_info):
+        """This method is called if an exception occurs,
+        but only if we are to stop at or just below this level."""
+        pass
+
+    def _set_stopinfo(self, stopframe, returnframe, stoplineno=0):
+        self.stopframe = stopframe
+        self.returnframe = returnframe
+        self.quitting = False
+        # stoplineno >= 0 means: stop at line >= the stoplineno
+        # stoplineno -1 means: don't stop at all
+        self.stoplineno = stoplineno
+
+    # Derived classes and clients can call the following methods
+    # to affect the stepping state.
+
+    def set_until(self, frame, lineno=None):
+        """Stop when the line with the line no greater than the current one is
+        reached or when returning from current frame"""
+        # the name "until" is borrowed from gdb
+        if lineno is None:
+            lineno = frame.f_lineno + 1
+        self._set_stopinfo(frame, frame, lineno)
+
+    def set_step(self):
+        """Stop after one line of code."""
+        self._set_stopinfo(None, None)
+
+    def set_next(self, frame):
+        """Stop on the next line in or below the given frame."""
+        self._set_stopinfo(frame, None)
+
+    def set_return(self, frame):
+        """Stop when returning from the given frame."""
+        self._set_stopinfo(frame.f_back, frame)
+
+    def set_trace(self, frame=None):
+        """Start debugging from `frame`.
+
+        If frame is not specified, debugging starts from caller's frame.
+        """
+        if frame is None:
+            frame = sys._getframe().f_back
+        self.reset()
+        while frame:
+            frame.f_trace = self.trace_dispatch
+            self.botframe = frame
+            frame = frame.f_back
+        self.set_step()
+        sys.settrace(self.trace_dispatch)
+
+    def set_continue(self):
+        # Don't stop except at breakpoints or when finished
+        self._set_stopinfo(self.botframe, None, -1)
+        if not self.breaks:
+            # no breakpoints; run without debugger overhead
+            sys.settrace(None)
+            frame = sys._getframe().f_back
+            while frame and frame is not self.botframe:
+                del frame.f_trace
+                frame = frame.f_back
+
+    def set_quit(self):
+        self.stopframe = self.botframe
+        self.returnframe = None
+        self.quitting = True
+        sys.settrace(None)
+
+    # Derived classes and clients can call the following methods
+    # to manipulate breakpoints.  These methods return an
+    # error message is something went wrong, None if all is well.
+    # Set_break prints out the breakpoint line and file:lineno.
+    # Call self.get_*break*() to see the breakpoints or better
+    # for bp in Breakpoint.bpbynumber: if bp: bp.bpprint().
+
+    def set_break(self, filename, lineno, temporary=False, cond=None,
+                  funcname=None):
+        filename = self.canonic(filename)
+        import linecache # Import as late as possible
+        line = linecache.getline(filename, lineno)
+        if not line:
+            return 'Line %s:%d does not exist' % (filename, lineno)
+        list = self.breaks.setdefault(filename, [])
+        if lineno not in list:
+            list.append(lineno)
+        bp = Breakpoint(filename, lineno, temporary, cond, funcname)
+
+    def _prune_breaks(self, filename, lineno):
+        if (filename, lineno) not in Breakpoint.bplist:
+            self.breaks[filename].remove(lineno)
+        if not self.breaks[filename]:
+            del self.breaks[filename]
+
+    def clear_break(self, filename, lineno):
+        filename = self.canonic(filename)
+        if filename not in self.breaks:
+            return 'There are no breakpoints in %s' % filename
+        if lineno not in self.breaks[filename]:
+            return 'There is no breakpoint at %s:%d' % (filename, lineno)
+        # If there's only one bp in the list for that file,line
+        # pair, then remove the breaks entry
+        for bp in Breakpoint.bplist[filename, lineno][:]:
+            bp.deleteMe()
+        self._prune_breaks(filename, lineno)
+
+    def clear_bpbynumber(self, arg):
+        try:
+            bp = self.get_bpbynumber(arg)
+        except ValueError as err:
+            return str(err)
+        bp.deleteMe()
+        self._prune_breaks(bp.file, bp.line)
+
+    def clear_all_file_breaks(self, filename):
+        filename = self.canonic(filename)
+        if filename not in self.breaks:
+            return 'There are no breakpoints in %s' % filename
+        for line in self.breaks[filename]:
+            blist = Breakpoint.bplist[filename, line]
+            for bp in blist:
+                bp.deleteMe()
+        del self.breaks[filename]
+
+    def clear_all_breaks(self):
+        if not self.breaks:
+            return 'There are no breakpoints'
+        for bp in Breakpoint.bpbynumber:
+            if bp:
+                bp.deleteMe()
+        self.breaks = {}
+
+    def get_bpbynumber(self, arg):
+        if not arg:
+            raise ValueError('Breakpoint number expected')
+        try:
+            number = int(arg)
+        except ValueError:
+            raise ValueError('Non-numeric breakpoint number %s' % arg)
+        try:
+            bp = Breakpoint.bpbynumber[number]
+        except IndexError:
+            raise ValueError('Breakpoint number %d out of range' % number)
+        if bp is None:
+            raise ValueError('Breakpoint %d already deleted' % number)
+        return bp
+
+    def get_break(self, filename, lineno):
+        filename = self.canonic(filename)
+        return filename in self.breaks and \
+            lineno in self.breaks[filename]
+
+    def get_breaks(self, filename, lineno):
+        filename = self.canonic(filename)
+        return filename in self.breaks and \
+            lineno in self.breaks[filename] and \
+            Breakpoint.bplist[filename, lineno] or []
+
+    def get_file_breaks(self, filename):
+        filename = self.canonic(filename)
+        if filename in self.breaks:

From noreply at buildbot.pypy.org  Tue Oct 11 23:27:16 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 11 Oct 2011 23:27:16 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Add a modifiable copy of opcode.py
Message-ID: <20111011212716.C42DD82112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r47947:838f7aaf8802
Date: 2011-10-11 23:12 +0200
http://bitbucket.org/pypy/pypy/changeset/838f7aaf8802/

Log:	Add a modifiable copy of opcode.py

diff --git a/lib-python/3.2/opcode.py b/lib-python/modified-3.2/opcode.py
copy from lib-python/3.2/opcode.py
copy to lib-python/modified-3.2/opcode.py

From noreply at buildbot.pypy.org  Tue Oct 11 23:27:18 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 11 Oct 2011 23:27:18 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: These SLICE opcodes don't exist anymore
Message-ID: <20111011212718.0501182112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r47948:3cb89066c84a
Date: 2011-10-11 23:13 +0200
http://bitbucket.org/pypy/pypy/changeset/3cb89066c84a/

Log:	These SLICE opcodes don't exist anymore

diff --git a/pypy/tool/stdlib_opcode.py b/pypy/tool/stdlib_opcode.py
--- a/pypy/tool/stdlib_opcode.py
+++ b/pypy/tool/stdlib_opcode.py
@@ -88,9 +88,6 @@
         """NOT_RPYTHON. Add individual opcodes to the module constants."""
         g = globals()
         g.update(self.opmap)
-        g['SLICE'] = self.opmap["SLICE+0"]
-        g['STORE_SLICE'] = self.opmap["STORE_SLICE+0"]
-        g['DELETE_SLICE'] = self.opmap["DELETE_SLICE+0"]
 
     def __str__(self):
         return "<%s bytecode>" % (self.name,)

From noreply at buildbot.pypy.org  Tue Oct 11 23:27:19 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 11 Oct 2011 23:27:19 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Remove stack effects for deleted opcodes.
Message-ID: <20111011212719.3ADE782112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r47949:4f33a53d0e6c
Date: 2011-10-11 23:20 +0200
http://bitbucket.org/pypy/pypy/changeset/4f33a53d0e6c/

Log:	Remove stack effects for deleted opcodes. (I'll add the missing ones
	when the compiler emits them) Restore 2 pypy-specific opcodes

diff --git a/lib-python/modified-3.2/opcode.py b/lib-python/modified-3.2/opcode.py
--- a/lib-python/modified-3.2/opcode.py
+++ b/lib-python/modified-3.2/opcode.py
@@ -177,4 +177,9 @@
 def_op('EXTENDED_ARG', 144)
 EXTENDED_ARG = 144
 
+# pypy modification, experimental bytecode
+def_op('LOOKUP_METHOD', 201)          # Index in name list
+hasname.append(201)
+def_op('CALL_METHOD', 202)            # #args not including 'self'
+
 del def_op, name_op, jrel_op, jabs_op
diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -489,13 +489,11 @@
     ops.POP_TOP : -1,
     ops.ROT_TWO : 0,
     ops.ROT_THREE : 0,
-    ops.ROT_FOUR : 0,
     ops.DUP_TOP : 1,
 
     ops.UNARY_POSITIVE : 0,
     ops.UNARY_NEGATIVE : 0,
     ops.UNARY_NOT : 0,
-    ops.UNARY_CONVERT : 0,
     ops.UNARY_INVERT : 0,
 
     ops.LIST_APPEND : -1,
@@ -505,7 +503,6 @@
 
     ops.BINARY_POWER : -1,
     ops.BINARY_MULTIPLY : -1,
-    ops.BINARY_DIVIDE : -1,
     ops.BINARY_MODULO : -1,
     ops.BINARY_ADD : -1,
     ops.BINARY_SUBTRACT : -1,
@@ -523,7 +520,6 @@
     ops.INPLACE_ADD : -1,
     ops.INPLACE_SUBTRACT : -1,
     ops.INPLACE_MULTIPLY : -1,
-    ops.INPLACE_DIVIDE : -1,
     ops.INPLACE_MODULO : -1,
     ops.INPLACE_POWER : -1,
     ops.INPLACE_LSHIFT : -1,
@@ -532,19 +528,6 @@
     ops.INPLACE_OR : -1,
     ops.INPLACE_XOR : -1,
 
-    ops.SLICE+0 : 1,
-    ops.SLICE+1 : 0,
-    ops.SLICE+2 : 0,
-    ops.SLICE+3 : -1,
-    ops.STORE_SLICE+0 : -2,
-    ops.STORE_SLICE+1 : -3,
-    ops.STORE_SLICE+2 : -3,
-    ops.STORE_SLICE+3 : -4,
-    ops.DELETE_SLICE+0 : -1,
-    ops.DELETE_SLICE+1 : -2,
-    ops.DELETE_SLICE+2 : -2,
-    ops.DELETE_SLICE+3 : -3,
-
     ops.STORE_SUBSCR : -2,
     ops.DELETE_SUBSCR : -2,
 
@@ -555,10 +538,6 @@
     ops.SETUP_LOOP : 0,
 
     ops.PRINT_EXPR : -1,
-    ops.PRINT_ITEM : -1,
-    ops.PRINT_NEWLINE : 0,
-    ops.PRINT_ITEM_TO : -2,
-    ops.PRINT_NEWLINE_TO : -1,
 
     ops.WITH_CLEANUP : -1,
     ops.POP_BLOCK : 0,
@@ -567,11 +546,8 @@
     ops.SETUP_FINALLY : 0,
     ops.SETUP_EXCEPT : 0,
 
-    ops.LOAD_LOCALS : 1,
     ops.RETURN_VALUE : -1,
-    ops.EXEC_STMT : -3,
     ops.YIELD_VALUE : 0,
-    ops.BUILD_CLASS : -2,
     ops.BUILD_MAP : 1,
     ops.BUILD_SET : 1,
     ops.COMPARE_OP : -1,
@@ -616,9 +592,6 @@
 def _compute_UNPACK_SEQUENCE(arg):
     return arg + 1
 
-def _compute_DUP_TOPX(arg):
-    return arg
-
 def _compute_BUILD_TUPLE(arg):
     return 1 - arg
 

From noreply at buildbot.pypy.org  Tue Oct 11 23:32:23 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 11 Oct 2011 23:32:23 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: write one more thing
Message-ID: <20111011213223.CBBDB82112@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: extradoc
Changeset: r3924:c499dbf497df
Date: 2011-10-11 23:32 +0200
http://bitbucket.org/pypy/extradoc/changeset/c499dbf497df/

Log:	write one more thing

diff --git a/blog/draft/numpy-statups-update.rst b/blog/draft/numpy-statups-update.rst
--- a/blog/draft/numpy-statups-update.rst
+++ b/blog/draft/numpy-statups-update.rst
@@ -20,6 +20,7 @@
 
 * More dtypes, including unsigned integers, singe floats, and complex doubles.
 * Subscripting arrays by other array as indices, and by bool arrays as masks.
+* Starting to reuse python-level original numpy
 
 Some of the things on the near horizon are:
 
@@ -29,4 +30,4 @@
 * Multi-dimensional array support.
 
 If you're interested in helping out, we always love more contributors,
-Alex, Maciej, Justin, and the whole PyPy team
\ No newline at end of file
+Alex, Maciej, Justin, and the whole PyPy team

From noreply at buildbot.pypy.org  Tue Oct 11 23:42:27 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 11 Oct 2011 23:42:27 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Remove all references to the __getslice__
 methods and opcodes
Message-ID: <20111011214227.2D50082112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r47950:459148c0f080
Date: 2011-10-11 23:41 +0200
http://bitbucket.org/pypy/pypy/changeset/459148c0f080/

Log:	Remove all references to the __getslice__ methods and opcodes
	space.getslice() still exists, and delegates to getitem.

diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -100,14 +100,6 @@
     ast.Del : ops.DELETE_SUBSCR
 })
 
-slice_operations = misc.dict_to_switch({
-    ast.AugLoad : ops.SLICE,
-    ast.Load : ops.SLICE,
-    ast.AugStore : ops.STORE_SLICE,
-    ast.Store : ops.STORE_SLICE,
-    ast.Del : ops.DELETE_SLICE
-})
-
 
 class __extend__(ast.GeneratorExp):
 
@@ -1081,35 +1073,6 @@
         else:
             raise AssertionError("unknown context")
 
-    def _simple_slice(self, slc, ctx):
-        slice_offset = 0
-        stack_count = 0
-        if slc.lower:
-            slice_offset += 1
-            stack_count += 1
-            if ctx != ast.AugStore:
-                slc.lower.walkabout(self)
-        if slc.upper:
-            slice_offset += 2
-            stack_count += 1
-            if ctx != ast.AugStore:
-                slc.upper.walkabout(self)
-        if ctx == ast.AugLoad:
-            if stack_count == 0:
-                self.emit_op(ops.DUP_TOP)
-            elif stack_count == 1:
-                self.emit_op_arg(ops.DUP_TOPX, 2)
-            elif stack_count == 2:
-                self.emit_op_arg(ops.DUP_TOPX, 3)
-        elif ctx == ast.AugStore:
-            if stack_count == 0:
-                self.emit_op(ops.ROT_TWO)
-            elif stack_count == 1:
-                self.emit_op(ops.ROT_THREE)
-            elif stack_count == 2:
-                self.emit_op(ops.ROT_FOUR)
-        self.emit_op(slice_operations(ctx) + slice_offset)
-
     def _complex_slice(self, slc, ctx):
         if slc.lower:
             slc.lower.walkabout(self)
@@ -1146,10 +1109,7 @@
                 self.load_const(self.space.w_Ellipsis)
         elif isinstance(slc, ast.Slice):
             kind = "slice"
-            if not slc.step:
-                self._simple_slice(slc, ctx)
-                return
-            elif ctx != ast.AugStore:
+            if ctx != ast.AugStore:
                 self._complex_slice(slc, ctx)
         elif isinstance(slc, ast.ExtSlice):
             kind = "extended slice"
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1435,9 +1435,6 @@
     ('getitem',         'getitem',   2, ['__getitem__']),
     ('setitem',         'setitem',   3, ['__setitem__']),
     ('delitem',         'delitem',   2, ['__delitem__']),
-    ('getslice',        'getslice',  3, ['__getslice__']),
-    ('setslice',        'setslice',  4, ['__setslice__']),
-    ('delslice',        'delslice',  3, ['__delslice__']),
     ('trunc',           'trunc',     1, ['__trunc__']),
     ('pos',             'pos',       1, ['__pos__']),
     ('neg',             'neg',       1, ['__neg__']),
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -427,68 +427,6 @@
     INPLACE_XOR = binaryoperation("inplace_xor")
     INPLACE_OR  = binaryoperation("inplace_or")
 
-    def slice(self, w_start, w_end):
-        w_obj = self.popvalue()
-        w_result = self.space.getslice(w_obj, w_start, w_end)
-        self.pushvalue(w_result)
-
-    def SLICE_0(self, oparg, next_instr):
-        self.slice(self.space.w_None, self.space.w_None)
-
-    def SLICE_1(self, oparg, next_instr):
-        w_start = self.popvalue()
-        self.slice(w_start, self.space.w_None)
-
-    def SLICE_2(self, oparg, next_instr):
-        w_end = self.popvalue()
-        self.slice(self.space.w_None, w_end)
-
-    def SLICE_3(self, oparg, next_instr):
-        w_end = self.popvalue()
-        w_start = self.popvalue()
-        self.slice(w_start, w_end)
-
-    def storeslice(self, w_start, w_end):
-        w_obj = self.popvalue()
-        w_newvalue = self.popvalue()
-        self.space.setslice(w_obj, w_start, w_end, w_newvalue)
-
-    def STORE_SLICE_0(self, oparg, next_instr):
-        self.storeslice(self.space.w_None, self.space.w_None)
-
-    def STORE_SLICE_1(self, oparg, next_instr):
-        w_start = self.popvalue()
-        self.storeslice(w_start, self.space.w_None)
-
-    def STORE_SLICE_2(self, oparg, next_instr):
-        w_end = self.popvalue()
-        self.storeslice(self.space.w_None, w_end)
-
-    def STORE_SLICE_3(self, oparg, next_instr):
-        w_end = self.popvalue()
-        w_start = self.popvalue()
-        self.storeslice(w_start, w_end)
-
-    def deleteslice(self, w_start, w_end):
-        w_obj = self.popvalue()
-        self.space.delslice(w_obj, w_start, w_end)
-
-    def DELETE_SLICE_0(self, oparg, next_instr):
-        self.deleteslice(self.space.w_None, self.space.w_None)
-
-    def DELETE_SLICE_1(self, oparg, next_instr):
-        w_start = self.popvalue()
-        self.deleteslice(w_start, self.space.w_None)
-
-    def DELETE_SLICE_2(self, oparg, next_instr):
-        w_end = self.popvalue()
-        self.deleteslice(self.space.w_None, w_end)
-
-    def DELETE_SLICE_3(self, oparg, next_instr):
-        w_end = self.popvalue()
-        w_start = self.popvalue()
-        self.deleteslice(w_start, w_end)
-
     def STORE_SUBSCR(self, oparg, next_instr):
         "obj[subscr] = newvalue"
         w_subscr = self.popvalue()
diff --git a/pypy/module/__builtin__/interp_classobj.py b/pypy/module/__builtin__/interp_classobj.py
--- a/pypy/module/__builtin__/interp_classobj.py
+++ b/pypy/module/__builtin__/interp_classobj.py
@@ -499,28 +499,6 @@
     #XXX do I really need a next method? the old implementation had one, but I
     # don't see the point
 
-    def descr_getslice(self, space, w_i, w_j):
-        w_meth = self.getattr(space, '__getslice__', False)
-        if w_meth is not None:
-            return space.call_function(w_meth, w_i, w_j)
-        else:
-            return space.getitem(self, space.newslice(w_i, w_j, space.w_None))
-
-    def descr_setslice(self, space, w_i, w_j, w_sequence):
-        w_meth = self.getattr(space, '__setslice__', False)
-        if w_meth is not None:
-            space.call_function(w_meth, w_i, w_j, w_sequence)
-        else:
-            space.setitem(self, space.newslice(w_i, w_j, space.w_None),
-                          w_sequence)
-
-    def descr_delslice(self, space, w_i, w_j):
-        w_meth = self.getattr(space, '__delslice__', False)
-        if w_meth is not None:
-            space.call_function(w_meth, w_i, w_j)
-        else:
-            return space.delitem(self, space.newslice(w_i, w_j, space.w_None))
-
     def descr_call(self, space, __args__):
         w_meth = self.getattr(space, '__call__')
         return space.call_args(w_meth, __args__)
@@ -771,9 +749,6 @@
     __setitem__ = interp2app(W_InstanceObject.descr_setitem),
     __delitem__ = interp2app(W_InstanceObject.descr_delitem),
     __iter__ = interp2app(W_InstanceObject.descr_iter),
-    __getslice__ = interp2app(W_InstanceObject.descr_getslice),
-    __setslice__ = interp2app(W_InstanceObject.descr_setslice),
-    __delslice__ = interp2app(W_InstanceObject.descr_delslice),
     __call__ = interp2app(W_InstanceObject.descr_call),
     __nonzero__ = interp2app(W_InstanceObject.descr_nonzero),
     __cmp__ = interp2app(W_InstanceObject.descr_cmp),
diff --git a/pypy/module/__builtin__/test/test_classobj.py b/pypy/module/__builtin__/test/test_classobj.py
--- a/pypy/module/__builtin__/test/test_classobj.py
+++ b/pypy/module/__builtin__/test/test_classobj.py
@@ -606,22 +606,6 @@
             skip("__rpow__(self, other, mod) seems not to work on CPython")
         assert pow(4, a, 5) == 625
 
-    def test_getsetdelslice(self):
-
-        class A:
-            def __getslice__(self, i, j):
-                return i + j
-            def __setslice__(self, i, j, seq):
-                self.last = (i, j, seq)
-            def __delslice__(self, i, j):
-                self.last = (i, j, None)
-        a = A()
-        assert a[1:3] == 4
-        a[1:3] = [1, 2, 3]
-        assert a.last == (1, 3, [1, 2, 3])
-        del a[1:4]
-        assert a.last == (1, 4, None)
-
     def test_contains_bug(self):
         class A:
             def __iter__(self):
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -337,9 +337,6 @@
             j += 1
         return w_a
 
-    def getslice__Array_ANY_ANY(space, self, w_i, w_j):
-        return space.getitem(self, space.newslice(w_i, w_j, space.w_None))
-
     def setitem__Array_ANY_ANY(space, self, w_idx, w_item):
         idx, stop, step = space.decode_index(w_idx, self.len)
         if step != 0:
@@ -372,9 +369,6 @@
                     self.buffer[i] = w_item.buffer[j]
                     j += 1
 
-    def setslice__Array_ANY_ANY_ANY(space, self, w_i, w_j, w_x):
-        space.setitem(self, space.newslice(w_i, w_j, space.w_None), w_x)
-
     def array_append__Array_ANY(space, self, w_x):
         x = self.item_w(w_x)
         self.setlen(self.len + 1)
@@ -447,9 +441,6 @@
         self.setlen(0)
         self.fromsequence(w_lst)
 
-    def delslice__Array_ANY_ANY(space, self, w_i, w_j):
-        return space.delitem(self, space.newslice(w_i, w_j, space.w_None))
-
     # Add and mul methods
 
     def add__Array_Array(space, self, other):
diff --git a/pypy/module/array/test/test_array.py b/pypy/module/array/test/test_array.py
--- a/pypy/module/array/test/test_array.py
+++ b/pypy/module/array/test/test_array.py
@@ -307,11 +307,6 @@
         raises(TypeError, "a[1:3] = self.array('I', [5, 6])")
         raises(TypeError, "a[1:3] = [5, 6]")
 
-        a = self.array('i', [1, 2, 3])
-        assert a.__getslice__(1, 2) == a[1:2]
-        a.__setslice__(1, 2, self.array('i', (7,)))
-        assert a[0] == 1 and a[1] == 7 and a[2] == 3
-
     def test_resizingslice(self):
         a = self.array('i', [1, 2, 3])
         a[1:2] = self.array('i', [7, 8, 9])
@@ -666,9 +661,6 @@
         del a[1:3]
         assert repr(a) == "array('i', [1, 4, 5])"
 
-        a.__delslice__(0, 2)
-        assert repr(a) == "array('i', [5])"
-
     def test_iter(self):
         a = self.array('i', [1, 2, 3])
         assert 1 in a
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -214,9 +214,9 @@
 	binaryfunc sq_concat;
 	ssizeargfunc sq_repeat;
 	ssizeargfunc sq_item;
-	ssizessizeargfunc sq_slice;
+	void *was_sq_slice;
 	ssizeobjargproc sq_ass_item;
-	ssizessizeobjargproc sq_ass_slice;
+	void *was_sq_ass_slice;
 	objobjproc sq_contains;
 	/* Added in release 2.0 */
 	binaryfunc sq_inplace_concat;
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -418,23 +418,10 @@
           "x.__rmul__(n) <==> n*x"),
         SQSLOT("__getitem__", sq_item, slot_sq_item, wrap_sq_item,
                "x.__getitem__(y) <==> x[y]"),
-        SQSLOT("__getslice__", sq_slice, slot_sq_slice, wrap_ssizessizeargfunc,
-               "x.__getslice__(i, j) <==> x[i:j]\n\
-               \n\
-               Use of negative indices is not supported."),
         SQSLOT("__setitem__", sq_ass_item, slot_sq_ass_item, wrap_sq_setitem,
                "x.__setitem__(i, y) <==> x[i]=y"),
         SQSLOT("__delitem__", sq_ass_item, slot_sq_ass_item, wrap_sq_delitem,
                "x.__delitem__(y) <==> del x[y]"),
-        SQSLOT("__setslice__", sq_ass_slice, slot_sq_ass_slice,
-               wrap_ssizessizeobjargproc,
-               "x.__setslice__(i, j, y) <==> x[i:j]=y\n\
-               \n\
-               Use  of negative indices is not supported."),
-        SQSLOT("__delslice__", sq_ass_slice, slot_sq_ass_slice, wrap_delslice,
-               "x.__delslice__(i, j) <==> del x[i:j]\n\
-               \n\
-               Use of negative indices is not supported."),
         SQSLOT("__contains__", sq_contains, slot_sq_contains, wrap_objobjproc,
                "x.__contains__(y) <==> y in x"),
         SQSLOT("__iadd__", sq_inplace_concat, NULL,
diff --git a/pypy/module/cpyext/typeobjectdefs.py b/pypy/module/cpyext/typeobjectdefs.py
--- a/pypy/module/cpyext/typeobjectdefs.py
+++ b/pypy/module/cpyext/typeobjectdefs.py
@@ -120,9 +120,7 @@
     ("sq_concat", binaryfunc),
     ("sq_repeat", ssizeargfunc),
     ("sq_item", ssizeargfunc),
-    ("sq_slice", ssizessizeargfunc),
     ("sq_ass_item", ssizeobjargproc),
-    ("sq_ass_slice", ssizessizeobjargproc),
     ("sq_contains", objobjproc),
     ("sq_inplace_concat", binaryfunc),
     ("sq_inplace_repeat", ssizeargfunc),
diff --git a/pypy/module/operator/__init__.py b/pypy/module/operator/__init__.py
--- a/pypy/module/operator/__init__.py
+++ b/pypy/module/operator/__init__.py
@@ -17,11 +17,9 @@
 
     appleveldefs = {} 
     
-    app_names = ['__delslice__', '__getslice__', '__repeat__', '__setslice__',
-                 'countOf', 'delslice', 'getslice', 'indexOf',
+    app_names = ['__repeat__', 'countOf', 'indexOf',
                  'isMappingType', 'isNumberType', 'isSequenceType',
-                 'repeat', 'setslice',
-                 'attrgetter', 'itemgetter', 'methodcaller',
+                 'repeat', 'attrgetter', 'itemgetter', 'methodcaller',
              ]
 
     for name in app_names:
diff --git a/pypy/module/operator/app_operator.py b/pypy/module/operator/app_operator.py
--- a/pypy/module/operator/app_operator.py
+++ b/pypy/module/operator/app_operator.py
@@ -14,20 +14,6 @@
             count += 1
     return count
 
-def delslice(obj, start, end):
-    'delslice(a, b, c) -- Same as del a[b:c].'
-    if not isinstance(start, int) or not isinstance(end, int):
-        raise TypeError("an integer is expected")
-    del obj[start:end]
-__delslice__ = delslice
-
-def getslice(a, start, end):
-    'getslice(a, b, c) -- Same as a[b:c].'
-    if not isinstance(start, int) or not isinstance(end, int):
-        raise TypeError("an integer is expected")
-    return a[start:end] 
-__getslice__ = getslice
-
 def indexOf(a, b):
     'indexOf(a, b) -- Return the first index of b in a.'
     index = 0
@@ -62,11 +48,6 @@
 
 __repeat__ = repeat
 
-def setslice(a, b, c, d):
-    'setslice(a, b, c, d) -- Same as a[b:c] = d.'
-    a[b:c] = d 
-__setslice__ = setslice
-
 
 def attrgetter(attr, *attrs):
     if attrs:
diff --git a/pypy/module/operator/interp_operator.py b/pypy/module/operator/interp_operator.py
--- a/pypy/module/operator/interp_operator.py
+++ b/pypy/module/operator/interp_operator.py
@@ -33,8 +33,6 @@
     'delitem(a,b) -- Same as del a[b]'
     space.delitem(w_obj, w_key)
 
-# delslice
-
 def div(space, w_a, w_b):
     'div(a, b) -- Same as a / b when __future__.division is no in effect'
     return space.div(w_a, w_b)
@@ -55,8 +53,6 @@
     'getitem(a, b) -- Same as a[b].'
     return space.getitem(w_a, w_b)
 
-# getslice
-
 def gt(space, w_a, w_b):
     'gt(a, b) -- Same as a>b.'
     return space.gt(w_a, w_b)
@@ -133,7 +129,7 @@
     'pow(a, b) -- Same as a**b.'
     return space.pow(w_a, w_b, space.w_None)
 
-# reapeat
+# repeat
 
 def rshift(space, w_a, w_b):
     'rshift(a, b) -- Same as a >> b.'
@@ -145,8 +141,6 @@
     'setitem(a, b, c) -- Same as a[b] = c.'
     space.setitem(w_obj, w_key, w_value)
 
-# setslice
-
 def sub(space, w_a, w_b):
     'sub(a, b) -- Same as a - b.'
     return space.sub(w_a, w_b) 
diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -322,28 +322,16 @@
         return space.get_and_call_function(w_descr, w_obj, w_key)
 
     def getslice(space, w_obj, w_start, w_stop):
-        w_descr = space.lookup(w_obj, '__getslice__')
-        if w_descr is None:
-            w_slice = space.newslice(w_start, w_stop, space.w_None)
-            return space.getitem(w_obj, w_slice)
-        w_start, w_stop = old_slice_range(space, w_obj, w_start, w_stop)
-        return space.get_and_call_function(w_descr, w_obj, w_start, w_stop)
+        w_slice = space.newslice(w_start, w_stop, space.w_None)
+        return space.getitem(w_obj, w_slice)
 
     def setslice(space, w_obj, w_start, w_stop, w_sequence):
-        w_descr = space.lookup(w_obj, '__setslice__')
-        if w_descr is None:
-            w_slice = space.newslice(w_start, w_stop, space.w_None)
-            return space.setitem(w_obj, w_slice, w_sequence)
-        w_start, w_stop = old_slice_range(space, w_obj, w_start, w_stop)
-        return space.get_and_call_function(w_descr, w_obj, w_start, w_stop, w_sequence)
+        w_slice = space.newslice(w_start, w_stop, space.w_None)
+        return space.setitem(w_obj, w_slice, w_sequence)
 
     def delslice(space, w_obj, w_start, w_stop):
-        w_descr = space.lookup(w_obj, '__delslice__')
-        if w_descr is None:
-            w_slice = space.newslice(w_start, w_stop, space.w_None)
-            return space.delitem(w_obj, w_slice)
-        w_start, w_stop = old_slice_range(space, w_obj, w_start, w_stop)
-        return space.get_and_call_function(w_descr, w_obj, w_start, w_stop)
+        w_slice = space.newslice(w_start, w_stop, space.w_None)
+        return space.delitem(w_obj, w_slice)
 
     def format(space, w_obj, w_format_spec):
         w_descr = space.lookup(w_obj, '__format__')
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -91,25 +91,6 @@
         start += step
     return w_res
 
-def getslice__List_ANY_ANY(space, w_list, w_start, w_stop):
-    length = len(w_list.wrappeditems)
-    start, stop = normalize_simple_slice(space, length, w_start, w_stop)
-    return W_ListObject(w_list.wrappeditems[start:stop])
-
-def setslice__List_ANY_ANY_ANY(space, w_list, w_start, w_stop, w_sequence):
-    length = len(w_list.wrappeditems)
-    start, stop = normalize_simple_slice(space, length, w_start, w_stop)
-
-    sequence2 = space.listview(w_sequence)
-    items = w_list.wrappeditems
-    _setitem_slice_helper(space, items, start, 1, stop-start, sequence2,
-                          empty_elem=None)
-
-def delslice__List_ANY_ANY(space, w_list, w_start, w_stop):
-    length = len(w_list.wrappeditems)
-    start, stop = normalize_simple_slice(space, length, w_start, w_stop)
-    _delitem_slice_helper(space, w_list.wrappeditems, start, 1, stop-start)
-
 def contains__List_ANY(space, w_list, w_obj):
     # needs to be safe against eq_w() mutating the w_list behind our back
     i = 0
diff --git a/pypy/objspace/std/rangeobject.py b/pypy/objspace/std/rangeobject.py
--- a/pypy/objspace/std/rangeobject.py
+++ b/pypy/objspace/std/rangeobject.py
@@ -102,17 +102,6 @@
     rangestep = w_rangelist.step * step
     return W_RangeListObject(rangestart, rangestep, slicelength)
 
-def getslice__RangeList_ANY_ANY(space, w_rangelist, w_start, w_stop):
-    if w_rangelist.w_list is not None:
-        return space.getslice(w_rangelist.w_list, w_start, w_stop)
-    length = w_rangelist.length
-    start, stop = normalize_simple_slice(space, length, w_start, w_stop)
-    slicelength = stop - start
-    assert slicelength >= 0
-    rangestart = w_rangelist.getitem_unchecked(start)
-    rangestep = w_rangelist.step
-    return W_RangeListObject(rangestart, rangestep, slicelength)
-
 def iter__RangeList(space, w_rangelist):
     return W_RangeIterObject(w_rangelist)
 
diff --git a/pypy/objspace/std/ropeobject.py b/pypy/objspace/std/ropeobject.py
--- a/pypy/objspace/std/ropeobject.py
+++ b/pypy/objspace/std/ropeobject.py
@@ -721,15 +721,6 @@
         return W_RopeObject.EMPTY
     return W_RopeObject(rope.getslice(node, start, stop, step, sl))
 
-def getslice__Rope_ANY_ANY(space, w_str, w_start, w_stop):
-    node = w_str._node
-    length = node.length()
-    start, stop = normalize_simple_slice(space, length, w_start, w_stop)
-    sl = stop - start
-    if sl == 0:
-        return W_RopeObject.EMPTY
-    return W_RopeObject(rope.getslice(node, start, stop, 1, sl))
-
 def mul_string_times(space, w_str, w_times):
     try:
         mul = space.getindex_w(w_times, space.w_OverflowError)
diff --git a/pypy/objspace/std/ropeunicodeobject.py b/pypy/objspace/std/ropeunicodeobject.py
--- a/pypy/objspace/std/ropeunicodeobject.py
+++ b/pypy/objspace/std/ropeunicodeobject.py
@@ -292,15 +292,6 @@
         return W_RopeUnicodeObject.EMPTY
     return W_RopeUnicodeObject(rope.getslice(node, start, stop, step, sl))
 
-def getslice__RopeUnicode_ANY_ANY(space, w_uni, w_start, w_stop):
-    node = w_uni._node
-    length = node.length()
-    start, stop = normalize_simple_slice(space, length, w_start, w_stop)
-    sl = stop - start
-    if sl == 0:
-        return W_RopeUnicodeObject.EMPTY
-    return W_RopeUnicodeObject(rope.getslice(node, start, stop, 1, sl))
-
 def mul__RopeUnicode_ANY(space, w_uni, w_times):
     try:
         times = space.getindex_w(w_times, space.w_OverflowError)
diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py
--- a/pypy/objspace/std/stringobject.py
+++ b/pypy/objspace/std/stringobject.py
@@ -855,14 +855,6 @@
         str = "".join([s[start + i*step] for i in range(sl)])
     return wrapstr(space, str)
 
-def getslice__String_ANY_ANY(space, w_str, w_start, w_stop):
-    s = w_str._value
-    start, stop = normalize_simple_slice(space, len(s), w_start, w_stop)
-    if start == stop:
-        return W_StringObject.EMPTY
-    else:
-        return sliced(space, s, start, stop, w_str)
-
 def mul_string_times(space, w_str, w_times):
     try:
         mul = space.getindex_w(w_times, space.w_OverflowError)
diff --git a/pypy/objspace/std/strsliceobject.py b/pypy/objspace/std/strsliceobject.py
--- a/pypy/objspace/std/strsliceobject.py
+++ b/pypy/objspace/std/strsliceobject.py
@@ -196,18 +196,6 @@
             str = "".join([s[start + i*step] for i in range(sl)])
     return wrapstr(space, str)
 
-def getslice__StringSlice_ANY_ANY(space, w_str, w_start, w_stop):
-    length = w_str.stop - w_str.start
-    start, stop = normalize_simple_slice(space, length, w_start, w_stop)
-    sl = stop - start
-    if sl == 0:
-        return W_StringObject.EMPTY
-    else:
-        s = w_str.str
-        start = w_str.start + start
-        stop = w_str.start + stop
-        return W_StringSliceObject(s, start, stop)
-
 def len__StringSlice(space, w_str):
     return space.wrap(w_str.stop - w_str.start)
 
diff --git a/pypy/objspace/std/tupleobject.py b/pypy/objspace/std/tupleobject.py
--- a/pypy/objspace/std/tupleobject.py
+++ b/pypy/objspace/std/tupleobject.py
@@ -58,11 +58,6 @@
         start += step
     return space.newtuple(subitems)
 
-def getslice__Tuple_ANY_ANY(space, w_tuple, w_start, w_stop):
-    length = len(w_tuple.wrappeditems)
-    start, stop = normalize_simple_slice(space, length, w_start, w_stop)
-    return space.newtuple(w_tuple.wrappeditems[start:stop])
-
 def contains__Tuple_ANY(space, w_tuple, w_obj):
     for w_item in w_tuple.wrappeditems:
         if space.eq_w(w_item, w_obj):
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -260,11 +260,6 @@
         r = u"".join([uni[start + i*step] for i in range(sl)])
     return W_UnicodeObject(r)
 
-def getslice__Unicode_ANY_ANY(space, w_uni, w_start, w_stop):
-    uni = w_uni._value
-    start, stop = normalize_simple_slice(space, len(uni), w_start, w_stop)
-    return W_UnicodeObject(uni[start:stop])
-
 def mul__Unicode_ANY(space, w_uni, w_times):
     try:
         times = space.getindex_w(w_times, space.w_OverflowError)
diff --git a/pypy/objspace/test/test_descroperation.py b/pypy/objspace/test/test_descroperation.py
--- a/pypy/objspace/test/test_descroperation.py
+++ b/pypy/objspace/test/test_descroperation.py
@@ -127,10 +127,9 @@
 
     def test_getslice(self):
         class Sq(object):
-            def __getslice__(self, start, stop):
+            def __getitem__(self, key):
+                start, stop = key
                 return (start, stop)
-            def __getitem__(self, key):
-                return "booh"
             def __len__(self):
                 return 100
 
@@ -152,10 +151,9 @@
 
     def test_setslice(self):
         class Sq(object):
-            def __setslice__(self, start, stop, sequence):
-                ops.append((start, stop, sequence))
             def __setitem__(self, key, value):
-                raise AssertionError, key
+                start, stop = key
+                ops.append((start, stop, value))
             def __len__(self):
                 return 100
 
@@ -177,10 +175,9 @@
 
     def test_delslice(self):
         class Sq(object):
-            def __delslice__(self, start, stop):
+            def __delitem__(self, key):
+                start, stop = key
                 ops.append((start, stop))
-            def __delitem__(self, key):
-                raise AssertionError, key
             def __len__(self):
                 return 100
 
@@ -202,10 +199,9 @@
 
     def test_getslice_nolength(self):
         class Sq(object):
-            def __getslice__(self, start, stop):
+            def __getitem__(self, key):
+                start, stop = key
                 return (start, stop)
-            def __getitem__(self, key):
-                return "booh"
 
         sq = Sq()
 

From noreply at buildbot.pypy.org  Wed Oct 12 00:16:36 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 12 Oct 2011 00:16:36 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: A slightly better error message when an
	opcode is missing
Message-ID: <20111011221636.48A8F82112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r47951:a0af6c1d5591
Date: 2011-10-11 23:45 +0200
http://bitbucket.org/pypy/pypy/changeset/a0af6c1d5591/

Log:	A slightly better error message when an opcode is missing in the
	"stack effect" map.

diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -660,4 +660,8 @@
         try:
             return _static_opcode_stack_effects[op]
         except KeyError:
-            return _stack_effect_computers[op](arg)
+            try:
+                return _stack_effect_computers[op](arg)
+            except KeyError:
+                raise KeyError("Unknown stack effect for %s (%s)" %
+                               (ops.opname[op], op))

From noreply at buildbot.pypy.org  Wed Oct 12 00:16:37 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 12 Oct 2011 00:16:37 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: update ast for ClassDef statement
Message-ID: <20111011221637.9100A82112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r47952:773f10116499
Date: 2011-10-11 23:51 +0200
http://bitbucket.org/pypy/pypy/changeset/773f10116499/

Log:	update ast for ClassDef statement

diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -292,19 +292,23 @@
 
 class ClassDef(stmt):
 
-    _lineno_mask = 16
-    _col_offset_mask = 32
-
-    def __init__(self, name, bases, body, decorator_list, lineno, col_offset):
+    _lineno_mask = 128
+    _col_offset_mask = 256
+
+    def __init__(self, name, bases, keywords, starargs, kwargs, body, decorator_list, lineno, col_offset):
         self.name = name
         self.bases = bases
         self.w_bases = None
+        self.keywords = keywords
+        self.w_keywords = None
+        self.starargs = starargs
+        self.kwargs = kwargs
         self.body = body
         self.w_body = None
         self.decorator_list = decorator_list
         self.w_decorator_list = None
         stmt.__init__(self, lineno, col_offset)
-        self.initialization_state = 63
+        self.initialization_state = 511
 
     def walkabout(self, visitor):
         visitor.visit_ClassDef(self)
@@ -312,6 +316,12 @@
     def mutate_over(self, visitor):
         if self.bases:
             visitor._mutate_sequence(self.bases)
+        if self.keywords:
+            visitor._mutate_sequence(self.keywords)
+        if self.starargs:
+            self.starargs = self.starargs.mutate_over(visitor)
+        if self.kwargs:
+            self.kwargs = self.kwargs.mutate_over(visitor)
         if self.body:
             visitor._mutate_sequence(self.body)
         if self.decorator_list:
@@ -319,10 +329,13 @@
         return visitor.visit_ClassDef(self)
 
     def sync_app_attrs(self, space):
-        if (self.initialization_state & ~0) ^ 63:
-            missing_field(space, self.initialization_state, ['name', 'bases', 'body', 'decorator_list', 'lineno', 'col_offset'], 'ClassDef')
+        if (self.initialization_state & ~24) ^ 487:
+            missing_field(space, self.initialization_state, ['name', 'bases', 'keywords', None, None, 'body', 'decorator_list', 'lineno', 'col_offset'], 'ClassDef')
         else:
-            pass
+            if not self.initialization_state & 8:
+                self.starargs = None
+            if not self.initialization_state & 16:
+                self.kwargs = None
         w_list = self.w_bases
         if w_list is not None:
             list_w = space.listview(w_list)
@@ -333,6 +346,20 @@
         if self.bases is not None:
             for node in self.bases:
                 node.sync_app_attrs(space)
+        w_list = self.w_keywords
+        if w_list is not None:
+            list_w = space.listview(w_list)
+            if list_w:
+                self.keywords = [space.interp_w(keyword, w_obj) for w_obj in list_w]
+            else:
+                self.keywords = None
+        if self.keywords is not None:
+            for node in self.keywords:
+                node.sync_app_attrs(space)
+        if self.starargs:
+            self.starargs.sync_app_attrs(space)
+        if self.kwargs:
+            self.kwargs.sync_app_attrs(space)
         w_list = self.w_body
         if w_list is not None:
             list_w = space.listview(w_list)
@@ -2692,6 +2719,11 @@
 
     def visit_ClassDef(self, node):
         self.visit_sequence(node.bases)
+        self.visit_sequence(node.keywords)
+        if node.starargs:
+            node.starargs.walkabout(self)
+        if node.kwargs:
+            node.kwargs.walkabout(self)
         self.visit_sequence(node.body)
         self.visit_sequence(node.decorator_list)
 
@@ -3299,8 +3331,70 @@
     w_self.w_bases = w_new_value
     w_self.initialization_state |= 2
 
+def ClassDef_get_keywords(space, w_self):
+    if not w_self.initialization_state & 4:
+        typename = space.type(w_self).getname(space)
+        w_err = space.wrap("'%s' object has no attribute 'keywords'" % typename)
+        raise OperationError(space.w_AttributeError, w_err)
+    if w_self.w_keywords is None:
+        if w_self.keywords is None:
+            w_list = space.newlist([])
+        else:
+            list_w = [space.wrap(node) for node in w_self.keywords]
+            w_list = space.newlist(list_w)
+        w_self.w_keywords = w_list
+    return w_self.w_keywords
+
+def ClassDef_set_keywords(space, w_self, w_new_value):
+    w_self.w_keywords = w_new_value
+    w_self.initialization_state |= 4
+
+def ClassDef_get_starargs(space, w_self):
+    if w_self.w_dict is not None:
+        w_obj = w_self.getdictvalue(space, 'starargs')
+        if w_obj is not None:
+            return w_obj
+    if not w_self.initialization_state & 8:
+        typename = space.type(w_self).getname(space)
+        w_err = space.wrap("'%s' object has no attribute 'starargs'" % typename)
+        raise OperationError(space.w_AttributeError, w_err)
+    return space.wrap(w_self.starargs)
+
+def ClassDef_set_starargs(space, w_self, w_new_value):
+    try:
+        w_self.starargs = space.interp_w(expr, w_new_value, True)
+    except OperationError, e:
+        if not e.match(space, space.w_TypeError):
+            raise
+        w_self.setdictvalue(space, 'starargs', w_new_value)
+        return
+    w_self.deldictvalue(space, 'starargs')
+    w_self.initialization_state |= 8
+
+def ClassDef_get_kwargs(space, w_self):
+    if w_self.w_dict is not None:
+        w_obj = w_self.getdictvalue(space, 'kwargs')
+        if w_obj is not None:
+            return w_obj
+    if not w_self.initialization_state & 16:
+        typename = space.type(w_self).getname(space)
+        w_err = space.wrap("'%s' object has no attribute 'kwargs'" % typename)
+        raise OperationError(space.w_AttributeError, w_err)
+    return space.wrap(w_self.kwargs)
+
+def ClassDef_set_kwargs(space, w_self, w_new_value):
+    try:
+        w_self.kwargs = space.interp_w(expr, w_new_value, True)
+    except OperationError, e:
+        if not e.match(space, space.w_TypeError):
+            raise
+        w_self.setdictvalue(space, 'kwargs', w_new_value)
+        return
+    w_self.deldictvalue(space, 'kwargs')
+    w_self.initialization_state |= 16
+
 def ClassDef_get_body(space, w_self):
-    if not w_self.initialization_state & 4:
+    if not w_self.initialization_state & 32:
         typename = space.type(w_self).getname(space)
         w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
         raise OperationError(space.w_AttributeError, w_err)
@@ -3315,10 +3409,10 @@
 
 def ClassDef_set_body(space, w_self, w_new_value):
     w_self.w_body = w_new_value
-    w_self.initialization_state |= 4
+    w_self.initialization_state |= 32
 
 def ClassDef_get_decorator_list(space, w_self):
-    if not w_self.initialization_state & 8:
+    if not w_self.initialization_state & 64:
         typename = space.type(w_self).getname(space)
         w_err = space.wrap("'%s' object has no attribute 'decorator_list'" % typename)
         raise OperationError(space.w_AttributeError, w_err)
@@ -3333,18 +3427,19 @@
 
 def ClassDef_set_decorator_list(space, w_self, w_new_value):
     w_self.w_decorator_list = w_new_value
-    w_self.initialization_state |= 8
-
-_ClassDef_field_unroller = unrolling_iterable(['name', 'bases', 'body', 'decorator_list'])
+    w_self.initialization_state |= 64
+
+_ClassDef_field_unroller = unrolling_iterable(['name', 'bases', 'keywords', 'starargs', 'kwargs', 'body', 'decorator_list'])
 def ClassDef_init(space, w_self, __args__):
     w_self = space.descr_self_interp_w(ClassDef, w_self)
     w_self.w_bases = None
+    w_self.w_keywords = None
     w_self.w_body = None
     w_self.w_decorator_list = None
     args_w, kwargs_w = __args__.unpack()
     if args_w:
-        if len(args_w) != 4:
-            w_err = space.wrap("ClassDef constructor takes either 0 or 4 positional arguments")
+        if len(args_w) != 7:
+            w_err = space.wrap("ClassDef constructor takes either 0 or 7 positional arguments")
             raise OperationError(space.w_TypeError, w_err)
         i = 0
         for field in _ClassDef_field_unroller:
@@ -3356,9 +3451,12 @@
 ClassDef.typedef = typedef.TypeDef("ClassDef",
     stmt.typedef,
     __module__='_ast',
-    _fields=_FieldsWrapper(['name', 'bases', 'body', 'decorator_list']),
+    _fields=_FieldsWrapper(['name', 'bases', 'keywords', 'starargs', 'kwargs', 'body', 'decorator_list']),
     name=typedef.GetSetProperty(ClassDef_get_name, ClassDef_set_name, cls=ClassDef),
     bases=typedef.GetSetProperty(ClassDef_get_bases, ClassDef_set_bases, cls=ClassDef),
+    keywords=typedef.GetSetProperty(ClassDef_get_keywords, ClassDef_set_keywords, cls=ClassDef),
+    starargs=typedef.GetSetProperty(ClassDef_get_starargs, ClassDef_set_starargs, cls=ClassDef),
+    kwargs=typedef.GetSetProperty(ClassDef_get_kwargs, ClassDef_set_kwargs, cls=ClassDef),
     body=typedef.GetSetProperty(ClassDef_get_body, ClassDef_set_body, cls=ClassDef),
     decorator_list=typedef.GetSetProperty(ClassDef_get_decorator_list, ClassDef_set_decorator_list, cls=ClassDef),
     __new__=interp2app(get_AST_new(ClassDef)),
diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -461,16 +461,16 @@
         self.check_forbidden_name(name, name_node)
         if len(classdef_node.children) == 4:
             body = self.handle_suite(classdef_node.children[3])
-            return ast.ClassDef(name, None, body, decorators,
+            return ast.ClassDef(name, None, None, None, None, body, decorators,
                                 classdef_node.lineno, classdef_node.column)
         if classdef_node.children[3].type == tokens.RPAR:
             body = self.handle_suite(classdef_node.children[5])
-            return ast.ClassDef(name, None, body, decorators,
+            return ast.ClassDef(name, None, None, None, None, body, decorators,
                                 classdef_node.lineno, classdef_node.column)
         bases = self.handle_class_bases(classdef_node.children[3])
         body = self.handle_suite(classdef_node.children[6])
-        return ast.ClassDef(name, bases, body, decorators, classdef_node.lineno,
-                            classdef_node.column)
+        return ast.ClassDef(name, bases, None, None, None, body, decorators,
+                            classdef_node.lineno, classdef_node.column)
 
     def handle_class_bases(self, bases_node):
         if len(bases_node.children) == 1:
diff --git a/pypy/interpreter/astcompiler/tools/Python.asdl b/pypy/interpreter/astcompiler/tools/Python.asdl
--- a/pypy/interpreter/astcompiler/tools/Python.asdl
+++ b/pypy/interpreter/astcompiler/tools/Python.asdl
@@ -11,7 +11,13 @@
 
 	stmt = FunctionDef(identifier name, arguments args, 
                             stmt* body, expr* decorator_list)
-	      | ClassDef(identifier name, expr* bases, stmt* body, expr* decorator_list)
+	      | ClassDef(identifier name, 
+			 expr* bases,
+			 keyword* keywords,
+			 expr? starargs,
+			 expr? kwargs,
+			 stmt* body,
+			 expr* decorator_list)
 	      | Return(expr? value)
 
 	      | Delete(expr* targets)

From noreply at buildbot.pypy.org  Wed Oct 12 00:16:38 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 12 Oct 2011 00:16:38 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: New class definition protocol:
	__build_class__, __prepare__
Message-ID: <20111011221638.CFB0F82112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r47953:444ebc97d236
Date: 2011-10-12 00:13 +0200
http://bitbucket.org/pypy/pypy/changeset/444ebc97d236/

Log:	New class definition protocol: __build_class__, __prepare__ For now
	it's not possible to use it, the grammar does not allow the new
	syntax: class X(metaclass=meta)

diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -540,6 +540,8 @@
     ops.PRINT_EXPR : -1,
 
     ops.WITH_CLEANUP : -1,
+    ops.LOAD_BUILD_CLASS : 1,
+    ops.STORE_LOCALS : -1,
     ops.POP_BLOCK : 0,
     ops.END_FINALLY : -3,
     ops.SETUP_WITH : 1,
diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -460,17 +460,25 @@
         name = name_node.value
         self.check_forbidden_name(name, name_node)
         if len(classdef_node.children) == 4:
+            # class NAME ':' suite
             body = self.handle_suite(classdef_node.children[3])
             return ast.ClassDef(name, None, None, None, None, body, decorators,
                                 classdef_node.lineno, classdef_node.column)
         if classdef_node.children[3].type == tokens.RPAR:
+            # class NAME '(' ')' ':' suite
             body = self.handle_suite(classdef_node.children[5])
             return ast.ClassDef(name, None, None, None, None, body, decorators,
                                 classdef_node.lineno, classdef_node.column)
-        bases = self.handle_class_bases(classdef_node.children[3])
+
+        # class NAME '(' arglist ')' ':' suite
+        # build up a fake Call node so we can extract its pieces
+        call_name = ast.Name(name, ast.Load, classdef_node.lineno,
+                             classdef_node.column)
+        call = self.handle_call(classdef_node.children[3], call_name)
         body = self.handle_suite(classdef_node.children[6])
-        return ast.ClassDef(name, bases, None, None, None, body, decorators,
-                            classdef_node.lineno, classdef_node.column)
+        return ast.ClassDef(
+            name, call.args, call.keywords, call.starargs, call.kwargs,
+            body, decorators, classdef_node.lineno, classdef_node.column)
 
     def handle_class_bases(self, bases_node):
         if len(bases_node.children) == 1:
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -313,17 +313,23 @@
     def visit_ClassDef(self, cls):
         self.update_position(cls.lineno, True)
         self.visit_sequence(cls.decorator_list)
+        # 1. compile the class body into a code object
+        code = self.sub_scope(ClassCodeGenerator, cls.name, cls, cls.lineno)
+        # 2. load the 'build_class' function
+        self.emit_op(ops.LOAD_BUILD_CLASS)
+        # 3. load a function (or closure) made from the code object
+        self._make_function(code, 0)
+        # 4. load class name
         self.load_const(self.space.wrap(cls.name))
-        self.visit_sequence(cls.bases)
-        bases_count = len(cls.bases) if cls.bases is not None else 0
-        self.emit_op_arg(ops.BUILD_TUPLE, bases_count)
-        code = self.sub_scope(ClassCodeGenerator, cls.name, cls, cls.lineno)
-        self._make_function(code, 0)
-        self.emit_op_arg(ops.CALL_FUNCTION, 0)
-        self.emit_op(ops.BUILD_CLASS)
+        # 5. generate the rest of the code for the call
+        self._make_call(2,
+                        cls.bases, cls.keywords,
+                        cls.starargs, cls.kwargs)
+        # 6. apply decorators
         if cls.decorator_list:
             for i in range(len(cls.decorator_list)):
                 self.emit_op_arg(ops.CALL_FUNCTION, 1)
+        # 7. store into <name>
         self.name_op(cls.name, ast.Store)
 
     def _op_for_augassign(self, op):
@@ -907,22 +913,22 @@
         self.load_const(self.space.wrap(keyword.arg))
         keyword.value.walkabout(self)
 
-    def visit_Call(self, call):
-        self.update_position(call.lineno)
-        if self._optimize_method_call(call):
-            return
-        call.func.walkabout(self)
-        arg = len(call.args) if call.args is not None else 0
+    def _make_call(self, n, # args already pushed
+                   args, keywords, starargs, kwargs):
+        if args is not None:
+            arg = len(args) + n
+        else:
+            arg = n
         call_type = 0
-        self.visit_sequence(call.args)
-        if call.keywords:
-            self.visit_sequence(call.keywords)
-            arg |= len(call.keywords) << 8
-        if call.starargs:
-            call.starargs.walkabout(self)
+        self.visit_sequence(args)
+        if keywords:
+            self.visit_sequence(keywords)
+            arg |= len(keywords) << 8
+        if starargs:
+            starargs.walkabout(self)
             call_type |= 1
-        if call.kwargs:
-            call.kwargs.walkabout(self)
+        if kwargs:
+            kwargs.walkabout(self)
             call_type |= 2
         op = 0
         if call_type == 0:
@@ -934,6 +940,15 @@
         elif call_type == 3:
             op = ops.CALL_FUNCTION_VAR_KW
         self.emit_op_arg(op, arg)
+
+    def visit_Call(self, call):
+        self.update_position(call.lineno)
+        if self._optimize_method_call(call):
+            return
+        call.func.walkabout(self)
+        self._make_call(0,
+                        call.args, call.keywords,
+                        call.starargs, call.kwargs)
     
     def _call_has_no_star_args(self, call):
         return not call.starargs and not call.kwargs
@@ -1242,8 +1257,23 @@
     def _compile(self, cls):
         assert isinstance(cls, ast.ClassDef)
         self.lineno = self.first_lineno
+        self.argcount = 1
+        # load the first argument (__locals__) ...
+        self.emit_op_arg(ops.LOAD_FAST, 0)
+        # ...and store it into f_locals.
+        self.emit_op(ops.STORE_LOCALS)
+        # load (global) __name__ ...
         self.name_op("__name__", ast.Load)
+        # ... and store it as __module__
         self.name_op("__module__", ast.Store)
+        # compile the body proper
         self._handle_body(cls.body)
-        self.emit_op(ops.LOAD_LOCALS)
+        # return the (empty) __class__ cell
+        scope = self.scope.lookup("@__class__")
+        if scope == symtable.SCOPE_UNKNOWN:
+            # This happens when nobody references the cell
+            self.load_const(self.space.w_None)
+        else:
+            # Return the cell where to store __class__
+            self.emit_op_arg(ops.LOAD_CLOSURE, self.cell_vars["@__class__"])
         self.emit_op(ops.RETURN_VALUE)
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -511,6 +511,9 @@
     def LOAD_LOCALS(self, oparg, next_instr):
         self.pushvalue(self.w_locals)
 
+    def STORE_LOCALS(self, oparg, next_instr):
+        self.w_locals = self.popvalue()
+
     def EXEC_STMT(self, oparg, next_instr):
         w_locals = self.popvalue()
         w_globals = self.popvalue()
@@ -550,16 +553,13 @@
         unroller = self.space.interpclass_w(w_unroller)
         return unroller
 
-    def BUILD_CLASS(self, oparg, next_instr):
-        w_methodsdict = self.popvalue()
-        w_bases = self.popvalue()
-        w_name = self.popvalue()
-        w_metaclass = find_metaclass(self.space, w_bases,
-                                     w_methodsdict, self.w_globals,
-                                     self.space.wrap(self.get_builtin()))
-        w_newclass = self.space.call_function(w_metaclass, w_name,
-                                              w_bases, w_methodsdict)
-        self.pushvalue(w_newclass)
+    def LOAD_BUILD_CLASS(self, oparg, next_instr):
+        w_build_class = self.get_builtin().getdictvalue(
+            self.space, '__build_class__')
+        if w_build_class is None:
+            raise OperationError(self.space.w_ImportError,
+                                 self.space.wrap("__build_class__ not found"))
+        self.pushvalue(w_build_class)
 
     def STORE_NAME(self, varindex, next_instr):
         varname = self.getname_u(varindex)
diff --git a/pypy/module/__builtin__/__init__.py b/pypy/module/__builtin__/__init__.py
--- a/pypy/module/__builtin__/__init__.py
+++ b/pypy/module/__builtin__/__init__.py
@@ -77,6 +77,7 @@
 
         'compile'       : 'compiling.compile',
         'eval'          : 'compiling.eval',
+        '__build_class__': 'compiling.build_class',
 
         '__import__'    : 'pypy.module.imp.importing.importhook',
         'reload'        : 'pypy.module.imp.importing.reload',
diff --git a/pypy/module/__builtin__/compiling.py b/pypy/module/__builtin__/compiling.py
--- a/pypy/module/__builtin__/compiling.py
+++ b/pypy/module/__builtin__/compiling.py
@@ -6,6 +6,8 @@
 from pypy.interpreter.error import OperationError
 from pypy.interpreter.astcompiler import consts, ast
 from pypy.interpreter.gateway import unwrap_spec
+from pypy.interpreter.argument import Arguments
+from pypy.interpreter.nestedscope import Cell
 
 @unwrap_spec(filename=str, mode=str, flags=int, dont_inherit=int)
 def compile(space, w_source, filename, mode, flags=0, dont_inherit=0):
@@ -107,3 +109,35 @@
             space.setitem(w_globals, space.wrap('__builtins__'), w_builtin)
 
     return codeobj.exec_code(space, w_globals, w_locals)
+
+def build_class(space, w_func, w_name, __args__):
+    bases_w, kwds_w = __args__.unpack()
+    w_bases = space.newtuple(bases_w)
+    w_meta = kwds_w.pop('metaclass', None)
+    if w_meta is None:
+        if bases_w:
+            w_meta = space.type(bases_w[0])
+        else:
+            w_meta = space.w_type
+    
+    try:
+        w_prep = space.getattr(w_meta, space.wrap("__prepare__"))
+    except OperationError, e:
+        if not e.match(space, space.w_AttributeError):
+            raise
+        w_namespace = space.newdict()
+    else:
+        args = Arguments(space, 
+                         args_w=[w_name, w_bases],
+                         keywords=kwds_w.keys(),
+                         keywords_w=kwds_w.values())
+        w_namespace = space.call_args(w_prep, args)
+    w_cell = space.call_function(w_func, w_namespace)
+    args = Arguments(space,
+                     args_w=[w_name, w_bases, w_namespace],
+                     keywords=kwds_w.keys(),
+                     keywords_w=kwds_w.values())
+    w_class = space.call_args(w_meta, args)
+    if isinstance(w_cell, Cell):
+        w_cell.set(w_class)
+    return w_class

From noreply at buildbot.pypy.org  Wed Oct 12 00:16:40 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 12 Oct 2011 00:16:40 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: There is only one True Division.
Message-ID: <20111011221640.1558682112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r47954:4c46f703ae28
Date: 2011-10-12 00:16 +0200
http://bitbucket.org/pypy/pypy/changeset/4c46f703ae28/

Log:	There is only one True Division.

diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -55,6 +55,7 @@
     ast.Add : ops.BINARY_ADD,
     ast.Sub : ops.BINARY_SUBTRACT,
     ast.Mult : ops.BINARY_MULTIPLY,
+    ast.Div : ops.BINARY_TRUE_DIVIDE,
     ast.Mod : ops.BINARY_MODULO,
     ast.Pow : ops.BINARY_POWER,
     ast.LShift : ops.BINARY_LSHIFT,
@@ -69,6 +70,7 @@
     ast.Add : ops.INPLACE_ADD,
     ast.Sub : ops.INPLACE_SUBTRACT,
     ast.Mult : ops.INPLACE_MULTIPLY,
+    ast.Div : ops.INPLACE_TRUE_DIVIDE,
     ast.Mod : ops.INPLACE_MODULO,
     ast.Pow : ops.INPLACE_POWER,
     ast.LShift : ops.INPLACE_LSHIFT,
@@ -333,11 +335,6 @@
         self.name_op(cls.name, ast.Store)
 
     def _op_for_augassign(self, op):
-        if op == ast.Div:
-            if self.compile_info.flags & consts.CO_FUTURE_DIVISION:
-                return ops.INPLACE_TRUE_DIVIDE
-            else:
-                return ops.INPLACE_DIVIDE
         return inplace_operations(op)
 
     def visit_AugAssign(self, assign):
@@ -380,11 +377,6 @@
         self.use_next_block(end)
 
     def _binop(self, op):
-        if op == ast.Div:
-            if self.compile_info.flags & consts.CO_FUTURE_DIVISION:
-                return ops.BINARY_TRUE_DIVIDE
-            else:
-                return ops.BINARY_DIVIDE
         return binary_operations(op)
 
     def visit_BinOp(self, binop):
diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py
--- a/pypy/interpreter/astcompiler/optimize.py
+++ b/pypy/interpreter/astcompiler/optimize.py
@@ -175,12 +175,6 @@
             right = binop.right.as_constant()
             if right is not None:
                 op = binop.op
-                # Can't fold straight division without "from __future_ import
-                # division" because it might be affected at runtime by the -Q
-                # flag.
-                if op == ast.Div and \
-                        not self.compile_info.flags & consts.CO_FUTURE_DIVISION:
-                    return binop
                 try:
                     for op_kind, folder in unrolling_binary_folders:
                         if op_kind == op:
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -1,3 +1,4 @@
+from __future__ import division
 import py
 from pypy.interpreter.astcompiler import codegen, astbuilder, symtable, optimize
 from pypy.interpreter.pyparser import pyparse

From noreply at buildbot.pypy.org  Wed Oct 12 00:32:43 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 12 Oct 2011 00:32:43 +0200 (CEST)
Subject: [pypy-commit] pypy string-promote-2: fix test
Message-ID: <20111011223243.3236A82112@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: string-promote-2
Changeset: r47955:749638fb1863
Date: 2011-10-12 00:32 +0200
http://bitbucket.org/pypy/pypy/changeset/749638fb1863/

Log:	fix test

diff --git a/pypy/jit/codewriter/test/test_jtransform.py b/pypy/jit/codewriter/test/test_jtransform.py
--- a/pypy/jit/codewriter/test/test_jtransform.py
+++ b/pypy/jit/codewriter/test/test_jtransform.py
@@ -101,7 +101,10 @@
         return False
     def callinfo_for_oopspec(self, oopspecindex):
         assert oopspecindex == effectinfo.EffectInfo.OS_STREQ_NONNULL
-        return ('calldescr', None)
+        class c:
+            class adr:
+                ptr = 1
+        return ('calldescr', c)
 
 class FakeBuiltinCallControl:
     def __init__(self):
@@ -856,11 +859,12 @@
                         [v1, Constant({'promote_string': True}, lltype.Void)],
                         v2)
     tr = Transformer(FakeCPU(), FakeBuiltinCallControl())
-    op0 = tr.rewrite_operation(op)
-    assert op0.opname == 'str_guard_value'
-    assert op0.args[0] == v1
-    assert op0.args[2] == 'calldescr'
-    assert op0.result == v2
+    op0, op1, _ = tr.rewrite_operation(op)
+    assert op1.opname == 'str_guard_value'
+    assert op1.args[0] == v1
+    assert op1.args[2] == 'calldescr'
+    assert op1.result == v2
+    assert op0.opname == '-live-'
 
 def test_unicode_concat():
     # test that the oopspec is present and correctly transformed

From noreply at buildbot.pypy.org  Wed Oct 12 00:33:17 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 12 Oct 2011 00:33:17 +0200 (CEST)
Subject: [pypy-commit] pypy string-promote-2: explain XXX
Message-ID: <20111011223317.8447382112@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: string-promote-2
Changeset: r47956:9b85dbd68f25
Date: 2011-10-12 00:33 +0200
http://bitbucket.org/pypy/pypy/changeset/9b85dbd68f25/

Log:	explain XXX

diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -466,7 +466,8 @@
                                         EffectInfo.EF_ELIDABLE_CANNOT_RAISE)
             descr, p = self.callcontrol.callinfocollection.callinfo_for_oopspec(
                 EffectInfo.OS_STREQ_NONNULL)
-            # XXX
+            # XXX this is fairly ugly way of creating a constant,
+            #     however, callinfocollection has no better interface
             c = Constant(p.adr.ptr, lltype.typeOf(p.adr.ptr))
             op1 = SpaceOperation('str_guard_value', [op.args[0], c, descr],
                                  op.result)

From noreply at buildbot.pypy.org  Wed Oct 12 00:34:13 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 12 Oct 2011 00:34:13 +0200 (CEST)
Subject: [pypy-commit] pypy string-promote-2: explain
Message-ID: <20111011223413.E559082112@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: string-promote-2
Changeset: r47957:944836e6101d
Date: 2011-10-12 00:34 +0200
http://bitbucket.org/pypy/pypy/changeset/944836e6101d/

Log:	explain

diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -904,6 +904,8 @@
             constbox = box.constbox()
             resbox = self.do_residual_call(funcbox, descr, [box, constbox])
             promoted_box = resbox.constbox()
+            # This is GUARD_VALUE because GUARD_TRUE assumes the existance
+            # of a label when computing resumepc
             self.generate_guard(rop.GUARD_VALUE, resbox, [promoted_box],
                                 resumepc=orgpc)
             self.metainterp.replace_box(box, constbox)

From noreply at buildbot.pypy.org  Wed Oct 12 01:45:35 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 12 Oct 2011 01:45:35 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Copy Grammar2.7 into Grammar3.2. No change
	so far
Message-ID: <20111011234535.5395082112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r47958:cf11d95048c8
Date: 2011-10-12 01:13 +0200
http://bitbucket.org/pypy/pypy/changeset/cf11d95048c8/

Log:	Copy Grammar2.7 into Grammar3.2. No change so far

diff --git a/pypy/interpreter/pyparser/data/Grammar2.7 b/pypy/interpreter/pyparser/data/Grammar3.2
copy from pypy/interpreter/pyparser/data/Grammar2.7
copy to pypy/interpreter/pyparser/data/Grammar3.2
diff --git a/pypy/interpreter/pyparser/pygram.py b/pypy/interpreter/pyparser/pygram.py
--- a/pypy/interpreter/pyparser/pygram.py
+++ b/pypy/interpreter/pyparser/pygram.py
@@ -9,7 +9,7 @@
 
 def _get_python_grammar():
     here = os.path.dirname(__file__)
-    fp = open(os.path.join(here, "data", "Grammar2.7"))
+    fp = open(os.path.join(here, "data", "Grammar3.2"))
     try:
         gram_source = fp.read()
     finally:

From noreply at buildbot.pypy.org  Wed Oct 12 01:45:36 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 12 Oct 2011 01:45:36 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Fix grammar for class statement,
 and allow classes to have bases :-/
Message-ID: <20111011234536.8ABF082112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r47959:529d4c4eecfd
Date: 2011-10-12 01:14 +0200
http://bitbucket.org/pypy/pypy/changeset/529d4c4eecfd/

Log:	Fix grammar for class statement, and allow classes to have bases :-/

diff --git a/pypy/interpreter/pyparser/data/Grammar3.2 b/pypy/interpreter/pyparser/data/Grammar3.2
--- a/pypy/interpreter/pyparser/data/Grammar3.2
+++ b/pypy/interpreter/pyparser/data/Grammar3.2
@@ -118,7 +118,7 @@
 dictorsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) |
                   (test (comp_for | (',' test)* [','])) )
 
-classdef: 'class' NAME ['(' [testlist] ')'] ':' suite
+classdef: 'class' NAME ['(' [arglist] ')'] ':' suite
 
 arglist: (argument ',')* (argument [',']
                          |'*' test (',' argument)* [',' '**' test] 

From noreply at buildbot.pypy.org  Wed Oct 12 01:45:37 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 12 Oct 2011 01:45:37 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Add FunctionType.__globals__,
 and update some tests because "import new" fails with py3k.
Message-ID: <20111011234537.C498582112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r47960:b9757b3532e9
Date: 2011-10-12 01:15 +0200
http://bitbucket.org/pypy/pypy/changeset/b9757b3532e9/

Log:	Add FunctionType.__globals__, and update some tests because "import
	new" fails with py3k.

diff --git a/pypy/interpreter/test/test_function.py b/pypy/interpreter/test/test_function.py
--- a/pypy/interpreter/test/test_function.py
+++ b/pypy/interpreter/test/test_function.py
@@ -34,6 +34,7 @@
         assert f.__dict__ is f.func_dict
         assert f.__code__ is f.func_code
         assert f.__defaults__ is f.func_defaults
+        assert f.__globals__ is f.func_globals
         assert hasattr(f, '__class__')
 
     def test_classmethod(self):
@@ -438,16 +439,16 @@
         class A(object):
             def __call__(self, x):
                 return x
-        import new
-        im = new.instancemethod(A(), 3)
+        import types
+        im = types.MethodType(A(), 3)
         assert im() == 3
 
     def test_method_w_callable_call_function(self):
         class A(object):
             def __call__(self, x, y):
                 return x+y
-        import new
-        im = new.instancemethod(A(), 3)
+        import types
+        im = types.MethodType(A(), 3)
         assert map(im, [4]) == [7]
 
     def test_unbound_typecheck(self):
@@ -474,15 +475,15 @@
         assert Fun[:2] == ('Fun', ())
 
     def test_unbound_abstract_typecheck(self):
-        import new
+        import types
         def f(*args):
             return args
-        m = new.instancemethod(f, None, "foobar")
+        m = types.MethodType(f, None, "foobar")
         raises(TypeError, m)
         raises(TypeError, m, None)
         raises(TypeError, m, "egg")
 
-        m = new.instancemethod(f, None, (str, int))     # really obscure...
+        m = types.MethodType(f, None, (str, int))     # really obscure...
         assert m(4) == (4,)
         assert m("uh") == ("uh",)
         raises(TypeError, m, [])
@@ -506,7 +507,7 @@
         BSub1.__bases__ = (BBase,)
         BSub2.__bases__ = (BBase,)
         x = MyInst(BSub1)
-        m = new.instancemethod(f, None, BSub1)
+        m = types.MethodType(f, None, BSub1)
         assert m(x) == (x,)
         raises(TypeError, m, MyInst(BBase))
         raises(TypeError, m, MyInst(BSub2))
@@ -514,9 +515,9 @@
         raises(TypeError, m, MyInst(42))
 
     def test_invalid_creation(self):
-        import new
+        import types
         def f(): pass
-        raises(TypeError, new.instancemethod, f, None)
+        raises(TypeError, types.MethodType, f, None)
 
     def test_empty_arg_kwarg_call(self):
         def f():
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -768,6 +768,7 @@
     __name__ = getset_func_name,
     __dict__ = getset_func_dict,
     __defaults__ = getset_func_defaults,
+    __globals__ = interp_attrproperty_w('w_func_globals', cls=Function),
     __module__ = getset___module__,
     __weakref__ = make_weakref_descr(Function),
     )

From noreply at buildbot.pypy.org  Wed Oct 12 01:45:39 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 12 Oct 2011 01:45:39 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Remove the exec statement,
	and try to make more tests pass
Message-ID: <20111011234539.1241782112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r47961:ffc302d9fec2
Date: 2011-10-12 01:44 +0200
http://bitbucket.org/pypy/pypy/changeset/ffc302d9fec2/

Log:	Remove the exec statement, and try to make more tests pass

diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -272,18 +272,6 @@
                  for i in range(1, len(global_node.children), 2)]
         return ast.Global(names, global_node.lineno, global_node.column)
 
-    def handle_exec_stmt(self, exec_node):
-        child_count = len(exec_node.children)
-        globs = None
-        locs = None
-        to_execute = self.handle_expr(exec_node.children[1])
-        if child_count >= 4:
-            globs = self.handle_expr(exec_node.children[3])
-        if child_count == 6:
-            locs = self.handle_expr(exec_node.children[5])
-        return ast.Exec(to_execute, globs, locs, exec_node.lineno,
-                        exec_node.column)
-
     def handle_assert_stmt(self, assert_node):
         child_count = len(assert_node.children)
         expr = self.handle_expr(assert_node.children[1])
@@ -647,8 +635,6 @@
                 return self.handle_global_stmt(stmt)
             elif stmt_type == syms.assert_stmt:
                 return self.handle_assert_stmt(stmt)
-            elif stmt_type == syms.exec_stmt:
-                return self.handle_exec_stmt(stmt)
             else:
                 raise AssertionError("unhandled small statement")
         elif stmt_type == syms.compound_stmt:
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -757,20 +757,6 @@
                     arg += 1
         self.emit_op_arg(ops.RAISE_VARARGS, arg)
 
-    def visit_Exec(self, exc):
-        self.update_position(exc.lineno, True)
-        exc.body.walkabout(self)
-        if exc.globals:
-            exc.globals.walkabout(self)
-            if exc.locals:
-                exc.locals.walkabout(self)
-            else:
-                self.emit_op(ops.DUP_TOP)
-        else:
-            self.load_const(self.space.w_None)
-            self.emit_op(ops.DUP_TOP)
-        self.emit_op(ops.EXEC_STMT)
-
     def visit_Global(self, glob):
         # Handled in symbol table building.
         pass
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -383,7 +383,7 @@
             ('''
                 class Foo(object): pass
                 foo = Foo()
-                exec "'moduledoc'" in foo.__dict__
+                exec("'moduledoc'", foo.__dict__)
              ''',                            "moduledoc"),
             ]:
             yield self.simple_test, source, "foo.__doc__", expected
@@ -782,29 +782,18 @@
 class AppTestCompiler:
 
     def test_docstring_not_loaded(self):
-        import StringIO, dis, sys
+        import io, dis, sys
         ns = {}
-        exec "def f():\n    'hi'" in ns
+        exec("def f():\n    'hi'", ns)
         f = ns["f"]
         save = sys.stdout
-        sys.stdout = output = StringIO.StringIO()
+        sys.stdout = output = io.BytesIO()
         try:
             dis.dis(f)
         finally:
             sys.stdout = save
         assert "0 ('hi')" not in output.getvalue()
 
-    def test_print_to(self):
-         exec """if 1:
-         from StringIO import StringIO
-         s = StringIO()
-         print >> s, "hi", "lovely!"
-         assert s.getvalue() == "hi lovely!\\n"
-         s = StringIO()
-         print >> s, "hi", "lovely!",
-         assert s.getvalue() == "hi lovely!"
-         """ in {}
-
 class TestOptimizations:
     def count_instructions(self, source):
         code, blocks = generate_function_code(source, self.space)
@@ -870,10 +859,10 @@
         space = self.space
         w_generator = space.appexec([], """():
             d = {}
-            exec '''def f(x):
+            exec('''def f(x):
                 return
                 yield 6
-            ''' in d
+            ''', d)
             return d['f'](5)
         """)
         assert 'generator' in space.str_w(space.repr(w_generator))
diff --git a/pypy/interpreter/astcompiler/tools/Python.asdl b/pypy/interpreter/astcompiler/tools/Python.asdl
--- a/pypy/interpreter/astcompiler/tools/Python.asdl
+++ b/pypy/interpreter/astcompiler/tools/Python.asdl
@@ -42,11 +42,6 @@
 	      | Import(alias* names)
 	      | ImportFrom(identifier? module, alias* names, int? level)
 
-	      -- Doesn't capture requirement that locals must be
-	      -- defined if globals is
-	      -- still supports use as a function!
-	      | Exec(expr body, expr? globals, expr? locals)
-
 	      | Global(identifier* names)
 	      | Expr(expr value)
 	      | Pass | Break | Continue
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -514,10 +514,8 @@
     def STORE_LOCALS(self, oparg, next_instr):
         self.w_locals = self.popvalue()
 
-    def EXEC_STMT(self, oparg, next_instr):
-        w_locals = self.popvalue()
-        w_globals = self.popvalue()
-        w_prog = self.popvalue()
+    def exec_(self, w_prog, w_locals, w_globals):
+        """The ___builtin__.exec function."""
         ec = self.space.getexecutioncontext()
         flags = ec.compiler.getcodeflags(self.pycode)
         w_compile_flags = self.space.wrap(flags)
diff --git a/pypy/interpreter/pyparser/data/Grammar3.2 b/pypy/interpreter/pyparser/data/Grammar3.2
--- a/pypy/interpreter/pyparser/data/Grammar3.2
+++ b/pypy/interpreter/pyparser/data/Grammar3.2
@@ -33,7 +33,7 @@
 stmt: simple_stmt | compound_stmt
 simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
 small_stmt: (expr_stmt | print_stmt  | del_stmt | pass_stmt | flow_stmt |
-             import_stmt | global_stmt | exec_stmt | assert_stmt)
+             import_stmt | global_stmt | assert_stmt)
 expr_stmt: testlist (augassign (yield_expr|testlist) |
                      ('=' (yield_expr|testlist))*)
 augassign: ('+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' |
@@ -59,7 +59,6 @@
 dotted_as_names: dotted_as_name (',' dotted_as_name)*
 dotted_name: NAME ('.' NAME)*
 global_stmt: 'global' NAME (',' NAME)*
-exec_stmt: 'exec' expr ['in' test [',' test]]
 assert_stmt: 'assert' test [',' test]
 
 compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated
diff --git a/pypy/module/__builtin__/__init__.py b/pypy/module/__builtin__/__init__.py
--- a/pypy/module/__builtin__/__init__.py
+++ b/pypy/module/__builtin__/__init__.py
@@ -77,6 +77,7 @@
 
         'compile'       : 'compiling.compile',
         'eval'          : 'compiling.eval',
+        'exec'          : 'compiling.exec_',
         '__build_class__': 'compiling.build_class',
 
         '__import__'    : 'pypy.module.imp.importing.importhook',
diff --git a/pypy/module/__builtin__/compiling.py b/pypy/module/__builtin__/compiling.py
--- a/pypy/module/__builtin__/compiling.py
+++ b/pypy/module/__builtin__/compiling.py
@@ -110,6 +110,11 @@
 
     return codeobj.exec_code(space, w_globals, w_locals)
 
+def exec_(space, w_prog, w_globals=None, w_locals=None):
+    ec = space.getexecutioncontext()
+    frame = ec.gettopframe_nohidden()
+    frame.exec_(w_prog, w_globals, w_locals)
+
 def build_class(space, w_func, w_name, __args__):
     bases_w, kwds_w = __args__.unpack()
     w_bases = space.newtuple(bases_w)

From noreply at buildbot.pypy.org  Wed Oct 12 14:59:11 2011
From: noreply at buildbot.pypy.org (hager)
Date: Wed, 12 Oct 2011 14:59:11 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: Added a few helper methods.
Message-ID: <20111012125911.D389182112@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r47962:546e04f68771
Date: 2011-10-12 14:50 +0200
http://bitbucket.org/pypy/pypy/changeset/546e04f68771/

Log:	Added a few helper methods.

diff --git a/pypy/jit/backend/ppc/ppcgen/codebuilder.py b/pypy/jit/backend/ppc/ppcgen/codebuilder.py
--- a/pypy/jit/backend/ppc/ppcgen/codebuilder.py
+++ b/pypy/jit/backend/ppc/ppcgen/codebuilder.py
@@ -964,8 +964,20 @@
         else:
             self.std(source_reg.value, 0, 0)
 
-    def prepare_insts_blocks(self):
-        self.assemble(True)
+    def b_cond_offset(self, offset, condition):
+        pos = self.currpos()
+        target_ofs = offset - pos
+        #self.trap()
+        #import pdb; pdb.set_trace()
+        self.bc(condition, 0, target_ofs)
+
+    def b_abs(self, address):
+        self.load_imm(r.r0, address)
+        self.mtctr(0)
+        self.bctr()
+
+    def prepare_insts_blocks(self, show=False):
+        self.assemble(show)
         insts = self.insts
         for inst in insts:
             self.write32(inst.assemble())
@@ -990,6 +1002,9 @@
     def currpos(self):
         return self.get_rel_pos()
 
+    def copy_to_raw_memory(self, addr):
+        self._copy_to_raw_memory(addr)
+
 class BranchUpdater(PPCAssembler):
     def __init__(self):
         PPCAssembler.__init__(self)

From noreply at buildbot.pypy.org  Wed Oct 12 14:59:13 2011
From: noreply at buildbot.pypy.org (hager)
Date: Wed, 12 Oct 2011 14:59:13 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: Fixed bug in emit_guard_true.
Message-ID: <20111012125913.1277E82A88@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r47963:b4d0f25e4cf8
Date: 2011-10-12 14:56 +0200
http://bitbucket.org/pypy/pypy/changeset/b4d0f25e4cf8/

Log:	Fixed bug in emit_guard_true.

diff --git a/pypy/jit/backend/ppc/ppcgen/opassembler.py b/pypy/jit/backend/ppc/ppcgen/opassembler.py
--- a/pypy/jit/backend/ppc/ppcgen/opassembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/opassembler.py
@@ -47,7 +47,7 @@
         l0 = arglocs[0]
         failargs = arglocs[1:]
         self.mc.cmpi(l0.value, 0)
-        self._emit_guard(op, failargs, c.opposites[c.EQ])
+        self._emit_guard(op, failargs, c.EQ)
         #                        #      ^^^^ If this condition is met,
         #                        #           then the guard fails.
 

From noreply at buildbot.pypy.org  Wed Oct 12 14:59:14 2011
From: noreply at buildbot.pypy.org (hager)
Date: Wed, 12 Oct 2011 14:59:14 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: Some steps more to be more
 similar to the ARm backend.
Message-ID: <20111012125914.4742482A89@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r47964:e5cfb842c91b
Date: 2011-10-12 14:58 +0200
http://bitbucket.org/pypy/pypy/changeset/e5cfb842c91b/

Log:	Some steps more to be more similar to the ARm backend.

diff --git a/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py b/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py
--- a/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py
@@ -10,12 +10,15 @@
 from pypy.jit.backend.ppc.ppcgen.codebuilder import PPCBuilder
 from pypy.jit.backend.ppc.ppcgen.arch import (IS_PPC_32, WORD, NONVOLATILES,
                                               GPR_SAVE_AREA)
-from pypy.jit.backend.ppc.ppcgen.helper.assembler import gen_emit_cmp_op
+from pypy.jit.backend.ppc.ppcgen.helper.assembler import (gen_emit_cmp_op, 
+                                                          encode32)
 import pypy.jit.backend.ppc.ppcgen.register as r
 import pypy.jit.backend.ppc.ppcgen.condition as c
 from pypy.jit.metainterp.history import (Const, ConstPtr, LoopToken,
                                          AbstractFailDescr)
-from pypy.jit.backend.llsupport.asmmemmgr import (BlockBuilderMixin, AsmMemoryManager, MachineDataBlockWrapper)
+from pypy.jit.backend.llsupport.asmmemmgr import (BlockBuilderMixin, 
+                                                  AsmMemoryManager,
+                                                  MachineDataBlockWrapper)
 from pypy.jit.backend.llsupport.regalloc import (RegisterManager, 
                                                  compute_vars_longevity)
 from pypy.jit.backend.llsupport import symbolic
@@ -58,6 +61,16 @@
 
 class AssemblerPPC(OpAssembler):
 
+    FLOAT_TYPE = '\xED'
+    REF_TYPE   = '\xEE'
+    INT_TYPE   = '\xEF'
+
+    STACK_LOC = '\xFC'
+    IMM_LOC = '\xFD'
+    # REG_LOC is empty
+    EMPTY_LOC = '\xFE'
+    END_OF_LOCS = '\xFF'
+
     def __init__(self, cpu, failargs_limit=1000):
         self.cpu = cpu
         self.fail_boxes_int = values_array(lltype.Signed, failargs_limit)
@@ -66,7 +79,6 @@
         self.memcpy_addr = 0
 
     def load_imm(self, rD, word):
-        rD = rD.as_key()
         if word <= 32767 and word >= -32768:
             self.mc.li(rD, word)
         elif IS_PPC_32 or (word <= 2147483647 and word >= -2147483648):
@@ -89,7 +101,7 @@
             self.mc.ld(rD, rD, 0)
 
     def store_reg(self, source_reg, addr):
-        self.load_imm(r.r0, addr)
+        self.load_imm(r.r0.value, addr)
         if IS_PPC_32:
             self.mc.stwx(source_reg.value, 0, 0)
         else:
@@ -137,9 +149,10 @@
     # XXX adjust for 64 bit
     def _make_prologue(self, target_pos, frame_depth):
         if IS_PPC_32:
+            # save it in previous frame (Backchain)
             self.mc.stwu(r.SP.value, r.SP.value, -frame_depth)
             self.mc.mflr(r.r0.value)  # move old link register
-            # save it in previous frame (Backchain)
+            # save old link register in previous frame
             self.mc.stw(r.r0.value, r.SP.value, frame_depth + WORD) 
             # save r31 at the bottom of the stack frame
             self.mc.stw(r.SPP.value, r.SP.value, WORD)
@@ -198,7 +211,20 @@
     #        #self.mc.li(r.r3.value, fail_index)            
     #        #self.mc.blr()
 
-    def gen_exit_code(self):
+    def _gen_leave_jitted_hook_code(self, save_exc=False):
+        mc = PPCBuilder()
+        ### XXX add a check if cpu supports floats
+        #with saved_registers(mc, r.caller_resp + [r.ip], r.caller_vfp_resp):
+        #    addr = self.cpu.get_on_leave_jitted_int(save_exception=save_exc)
+        #    mc.BL(addr)
+        #assert self._exit_code_addr != 0
+        #mc.B(self._exit_code_addr)
+        mc.b_abs(self.exit_code_adr)
+        mc.prepare_insts_blocks()
+        return mc.materialize(self.cpu.asmmemmgr, [],
+                               self.cpu.gc_ll_descr.gcrootmap)
+
+    def _gen_exit_path(self):
         mc = PPCBuilder()
         # save SPP in r5
         # (assume that r5 has been written to failboxes)
@@ -242,12 +268,14 @@
         self.pending_guards = []
         assert self.datablockwrapper is None
         allblocks = self.get_asmmemmgr_blocks(looptoken)
-        self.exit_code_adr = self.gen_exit_code()
         self.datablockwrapper = MachineDataBlockWrapper(self.cpu.asmmemmgr,
                                                         allblocks)
 
     def setup_once(self):
         self.memcpy_addr = self.cpu.cast_ptr_to_int(memcpy_fn)
+        self.exit_code_adr = self._gen_exit_path()
+        #self._leave_jitted_hook_save_exc = self._gen_leave_jitted_hook_code(True)
+        self._leave_jitted_hook = self._gen_leave_jitted_hook_code(False)
 
     def assemble_loop(self, inputargs, operations, looptoken, log):
 
@@ -255,8 +283,6 @@
         looptoken.compiled_loop_token = clt
 
         self.setup(looptoken, operations)
-        self.patch_list = []
-        self.pending_guards = []
         self.startpos = self.mc.currpos()
 
         longevity = compute_vars_longevity(inputargs, operations)
@@ -267,7 +293,7 @@
         regalloc_head = self.mc.currpos()
         self.gen_bootstrap_code(nonfloatlocs, inputargs)
 
-        loophead = self.mc.currpos()
+        loophead = self.mc.currpos()            # address of actual loop
         looptoken._ppc_loop_code = loophead
         looptoken._ppc_arglocs = [nonfloatlocs]
         looptoken._ppc_bootstrap_code = 0
@@ -277,9 +303,11 @@
         start_pos = self.mc.currpos()
         self.framesize = frame_depth = self.compute_frame_depth(regalloc)
         self._make_prologue(regalloc_head, frame_depth)
-        
-        loop_start = self.materialize_loop(looptoken)
+     
+        self.write_pending_failure_recoveries()
+        loop_start = self.materialize_loop(looptoken, True)
         looptoken.ppc_code = loop_start + start_pos
+        self.process_pending_guards(loop_start)
         self._teardown()
 
     # For an explanation of the encoding, see
@@ -365,14 +393,60 @@
                        + regalloc.frame_manager.frame_depth * WORD)
         return frame_depth
     
-    def materialize_loop(self, looptoken):
-        self.mc.prepare_insts_blocks()
+    def materialize_loop(self, looptoken, show):
+        self.mc.prepare_insts_blocks(show)
         self.datablockwrapper.done()
         self.datablockwrapper = None
         allblocks = self.get_asmmemmgr_blocks(looptoken)
         return self.mc.materialize(self.cpu.asmmemmgr, allblocks, 
                                    self.cpu.gc_ll_descr.gcrootmap)
 
+    def write_pending_failure_recoveries(self):
+        for tok in self.pending_guards:
+            descr = tok.descr
+            #generate the exit stub and the encoded representation
+            pos = self.mc.currpos()
+            tok.pos_recovery_stub = pos 
+
+            memaddr = self.gen_exit_stub(descr, tok.failargs,
+                                            tok.faillocs, save_exc=tok.save_exc)
+            # store info on the descr
+            descr._ppc_frame_depth = tok.faillocs[0].getint()
+            descr._failure_recovery_code = memaddr
+            descr._ppc_guard_pos = pos
+
+    def gen_exit_stub(self, descr, args, arglocs, fcond=c.NE,
+                               save_exc=False):
+        memaddr = self.gen_descr_encoding(descr, args, arglocs)
+
+        # store addr in force index field
+        self.mc.load_imm(r.r0, memaddr)
+        self.mc.stw(r.r0.value, r.SPP.value, 0)
+
+        if save_exc:
+            path = self._leave_jitted_hook_save_exc
+        else:
+            path = self._leave_jitted_hook
+        self.mc.trap()
+        #self.mc.ba(path)
+        self.branch_abs(path)
+        return memaddr
+
+    def process_pending_guards(self, block_start):
+        clt = self.current_clt
+        for tok in self.pending_guards:
+            descr = tok.descr
+            assert isinstance(descr, AbstractFailDescr)
+            descr._ppc_block_start = block_start
+
+            if not tok.is_invalidate:
+                mc = PPCBuilder()
+                mc.b_cond_offset(descr._ppc_guard_pos - tok.offset, tok.fcond)
+                mc.prepare_insts_blocks(True)
+                mc.copy_to_raw_memory(block_start + tok.offset)
+            else:
+                assert 0, "not implemented yet"
+
     def get_asmmemmgr_blocks(self, looptoken):
         clt = looptoken.compiled_loop_token
         if clt.asmmemmgr_blocks is None:
diff --git a/pypy/jit/backend/ppc/ppcgen/regalloc.py b/pypy/jit/backend/ppc/ppcgen/regalloc.py
--- a/pypy/jit/backend/ppc/ppcgen/regalloc.py
+++ b/pypy/jit/backend/ppc/ppcgen/regalloc.py
@@ -213,7 +213,7 @@
     def _prepare_guard(self, op, args=None):
         if args is None:
             args = []
-        #args.append(imm(self.frame_manager.frame_depth))
+        args.append(imm(self.frame_manager.frame_depth))
         for arg in op.getfailargs():
             if arg:
                 args.append(self.loc(arg))

From noreply at buildbot.pypy.org  Wed Oct 12 16:52:40 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Wed, 12 Oct 2011 16:52:40 +0200 (CEST)
Subject: [pypy-commit] pypy default: random beautification
Message-ID: <20111012145240.F2E1482112@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r47965:22f84e5ca124
Date: 2011-10-12 10:52 -0400
http://bitbucket.org/pypy/pypy/changeset/22f84e5ca124/

Log:	random beautification

diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -426,23 +426,22 @@
     pass
 
 if LONG_BIT == 32:
-    class W_LongDtype(W_Int32Dtype):
-        pass
+    long_dtype = W_Int32Dtype
+    ulong_dtype = W_UInt32Dtype
+elif LONG_BIT == 64:
+    long_dtype = W_Int64Dtype
+    ulong_dtype = W_UInt64Dtype
+else:
+    assert False
 
-    class W_ULongDtype(W_UInt32Dtype):
-        pass
-else:
-    class W_LongDtype(W_Int64Dtype):
-        pass
+class W_LongDtype(long_dtype):
+    num = 7
+    aliases = ["l"]
+    applevel_types = ["int"]
 
-    class W_ULongDtype(W_UInt64Dtype):
-        pass
-
-W_LongDtype.num = 7
-W_LongDtype.aliases = ["l"]
-W_LongDtype.applevel_types = ["int"]
-W_ULongDtype.num = 8
-W_ULongDtype.aliases = ["L"]
+class W_ULongDtype(ulong_dtype):
+    num = 8
+    aliases = ["L"]
 
 W_Float32Dtype = create_low_level_dtype(
     num = 11, kind = FLOATINGLTR, name = "float32",

From noreply at buildbot.pypy.org  Wed Oct 12 17:12:43 2011
From: noreply at buildbot.pypy.org (hager)
Date: Wed, 12 Oct 2011 17:12:43 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: AAAAARRRRGH fixed bug in
	conditional branch.
Message-ID: <20111012151243.1782C82112@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r47966:aa6d1d104d80
Date: 2011-10-12 17:12 +0200
http://bitbucket.org/pypy/pypy/changeset/aa6d1d104d80/

Log:	AAAAARRRRGH fixed bug in conditional branch.

diff --git a/pypy/jit/backend/ppc/ppcgen/codebuilder.py b/pypy/jit/backend/ppc/ppcgen/codebuilder.py
--- a/pypy/jit/backend/ppc/ppcgen/codebuilder.py
+++ b/pypy/jit/backend/ppc/ppcgen/codebuilder.py
@@ -967,9 +967,7 @@
     def b_cond_offset(self, offset, condition):
         pos = self.currpos()
         target_ofs = offset - pos
-        #self.trap()
-        #import pdb; pdb.set_trace()
-        self.bc(condition, 0, target_ofs)
+        self.bc(condition, 2, target_ofs)
 
     def b_abs(self, address):
         self.load_imm(r.r0, address)

From noreply at buildbot.pypy.org  Wed Oct 12 17:19:08 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Wed, 12 Oct 2011 17:19:08 +0200 (CEST)
Subject: [pypy-commit] pypy default: Move reduce and filter to app-level,
 in my simple benchmarks (reduce(operator.add,
 xrange(1000000)) and filter(lambda x: x % 2,
 xrange(1000000)) they are 30 and 7x faster, respectively.
Message-ID: <20111012151908.B956382112@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r47967:205d254a977b
Date: 2011-10-12 11:18 -0400
http://bitbucket.org/pypy/pypy/changeset/205d254a977b/

Log:	Move reduce and filter to app-level, in my simple benchmarks
	(reduce(operator.add, xrange(1000000)) and filter(lambda x: x % 2,
	xrange(1000000)) they are 30 and 7x faster, respectively.

diff --git a/pypy/module/__builtin__/__init__.py b/pypy/module/__builtin__/__init__.py
--- a/pypy/module/__builtin__/__init__.py
+++ b/pypy/module/__builtin__/__init__.py
@@ -21,6 +21,8 @@
         'all'           : 'app_functional.all',
         'sum'           : 'app_functional.sum',
         'map'           : 'app_functional.map',
+        'reduce'        : 'app_functional.reduce',
+        'filter'        : 'app_functional.filter',
         'vars'          : 'app_inspect.vars',
         'dir'           : 'app_inspect.dir',
 
@@ -88,9 +90,7 @@
         'min'           : 'functional.min',
         'max'           : 'functional.max',
         'zip'           : 'functional.zip',
-        'reduce'        : 'functional.reduce',
         'reversed'      : 'functional.reversed',
-        'filter'        : 'functional.filter',
         'super'         : 'descriptor.W_Super',
         'staticmethod'  : 'descriptor.StaticMethod',
         'classmethod'   : 'descriptor.ClassMethod',
diff --git a/pypy/module/__builtin__/app_functional.py b/pypy/module/__builtin__/app_functional.py
--- a/pypy/module/__builtin__/app_functional.py
+++ b/pypy/module/__builtin__/app_functional.py
@@ -98,3 +98,68 @@
                 result.append(func(*args))
         else:
             return result
+
+sentinel = object()
+
+def reduce(func, sequence, initial=sentinel):
+    """reduce(function, sequence[, initial]) -> value
+
+Apply a function of two arguments cumulatively to the items of a sequence,
+from left to right, so as to reduce the sequence to a single value.
+For example, reduce(lambda x, y: x+y, [1, 2, 3, 4, 5]) calculates
+((((1+2)+3)+4)+5).  If initial is present, it is placed before the items
+of the sequence in the calculation, and serves as a default when the
+sequence is empty."""
+    iterator = iter(sequence)
+    if initial is sentinel:
+        try:
+            initial = next(iterator)
+        except StopIteration:
+            raise TypeError("reduce() of empty sequence with no initial value")
+    result = initial
+    for item in iterator:
+        result = func(result, item)
+    return result
+
+def filter(func, seq):
+    """filter(function or None, sequence) -> list, tuple, or string
+
+Return those items of sequence for which function(item) is true.  If
+function is None, return the items that are true.  If sequence is a tuple
+or string, return the same type, else return a list."""
+    if func is None:
+        func = bool
+    if isinstance(seq, str):
+        return _filter_string(func, seq, str)
+    elif isinstance(seq, unicode):
+        return _filter_string(func, seq, unicode)
+    elif isinstance(seq, tuple):
+        return _filter_tuple(func, seq)
+    result = []
+    for item in seq:
+        if func(item):
+            result.append(item)
+    return result
+
+def _filter_string(func, string, str_type):
+    if func is bool and type(string) is str_type:
+        return string
+    result = []
+    for i in range(len(string)):
+        # You must call __getitem__ on the strings, simply iterating doesn't
+        # work :/
+        item = string[i]
+        if func(item):
+            if not isinstance(item, str_type):
+                raise TypeError("__getitem__ returned a non-string type")
+            result.append(item)
+    return str_type().join(result)
+
+def _filter_tuple(func, seq):
+    result = []
+    for i in range(len(seq)):
+        # Again, must call __getitem__, at least there are tests.
+        item = seq[i]
+        if func(item):
+            result.append(item)
+    return tuple(result)
\ No newline at end of file
diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py
--- a/pypy/module/__builtin__/functional.py
+++ b/pypy/module/__builtin__/functional.py
@@ -222,90 +222,6 @@
             return space.newlist(result_w)
         result_w.append(space.newtuple(items_w))
 
-def reduce(space, w_func, w_sequence, w_initial=NoneNotWrapped):
-    """ Apply function of two arguments cumulatively to the items of sequence,
-        from left to right, so as to reduce the sequence to a single value.
-        Optionally begin with an initial value.
-    """
-    w_iter = space.iter(w_sequence)
-    if w_initial is None:
-        try:
-            w_initial = space.next(w_iter)
-        except OperationError, e:
-            if e.match(space, space.w_StopIteration):
-                msg = "reduce() of empty sequence with no initial value"
-                raise OperationError(space.w_TypeError, space.wrap(msg))
-            raise
-    w_result = w_initial
-    while True:
-        try:
-            w_next = space.next(w_iter)
-        except OperationError, e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            break
-        w_result = space.call_function(w_func, w_result, w_next)
-    return w_result
-
-def filter(space, w_func, w_seq):
-    """construct a list of those elements of collection for which function
-       is True.  If function is None, then return the items in the sequence
-       which are True.
-    """
-    if space.is_true(space.isinstance(w_seq, space.w_str)):
-        return _filter_string(space, w_func, w_seq, space.w_str)
-    if space.is_true(space.isinstance(w_seq, space.w_unicode)):
-        return _filter_string(space, w_func, w_seq, space.w_unicode)
-    if space.is_true(space.isinstance(w_seq, space.w_tuple)):
-        return _filter_tuple(space, w_func, w_seq)
-    w_iter = space.iter(w_seq)
-    result_w = []
-    none_func = space.is_w(w_func, space.w_None)
-    while True:
-        try:
-            w_next = space.next(w_iter)
-        except OperationError, e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            break
-        if none_func:
-            w_keep = w_next
-        else:
-            w_keep = space.call_function(w_func, w_next)
-        if space.is_true(w_keep):
-            result_w.append(w_next)
-    return space.newlist(result_w)
-
-def _filter_tuple(space, w_func, w_tuple):
-    none_func = space.is_w(w_func, space.w_None)
-    length = space.len_w(w_tuple)
-    result_w = []
-    for i in range(length):
-        w_item = space.getitem(w_tuple, space.wrap(i))
-        if none_func:
-            w_keep = w_item
-        else:
-            w_keep = space.call_function(w_func, w_item)
-        if space.is_true(w_keep):
-            result_w.append(w_item)
-    return space.newtuple(result_w)
-
-def _filter_string(space, w_func, w_string, w_str_type):
-    none_func = space.is_w(w_func, space.w_None)
-    if none_func and space.is_w(space.type(w_string), w_str_type):
-        return w_string
-    length = space.len_w(w_string)
-    result_w = []
-    for i in range(length):
-        w_item = space.getitem(w_string, space.wrap(i))
-        if none_func or space.is_true(space.call_function(w_func, w_item)):
-            if not space.is_true(space.isinstance(w_item, w_str_type)):
-                msg = "__getitem__ returned a non-string type"
-                raise OperationError(space.w_TypeError, space.wrap(msg))
-            result_w.append(w_item)
-    w_empty = space.call_function(w_str_type)
-    return space.call_method(w_empty, "join", space.newlist(result_w))
-
 class W_Enumerate(Wrappable):
 
     def __init__(self, w_iter, w_start):

From noreply at buildbot.pypy.org  Wed Oct 12 17:22:28 2011
From: noreply at buildbot.pypy.org (gutworth)
Date: Wed, 12 Oct 2011 17:22:28 +0200 (CEST)
Subject: [pypy-commit] pypy default: remove taint objspace
Message-ID: <20111012152228.A892482112@wyvern.cs.uni-duesseldorf.de>

Author: Benjamin Peterson <benjamin at python.org>
Branch: 
Changeset: r47968:ffa26bfd476e
Date: 2011-10-12 11:22 -0400
http://bitbucket.org/pypy/pypy/changeset/ffa26bfd476e/

Log:	remove taint objspace

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -127,7 +127,7 @@
 
 pypy_optiondescription = OptionDescription("objspace", "Object Space Options", [
     ChoiceOption("name", "Object Space name",
-                 ["std", "flow", "thunk", "dump", "taint"],
+                 ["std", "flow", "thunk", "dump"],
                  "std",
                  cmdline='--objspace -o'),
 
diff --git a/pypy/doc/__pypy__-module.rst b/pypy/doc/__pypy__-module.rst
--- a/pypy/doc/__pypy__-module.rst
+++ b/pypy/doc/__pypy__-module.rst
@@ -37,29 +37,6 @@
 .. _`thunk object space docs`: objspace-proxies.html#thunk
 .. _`interface section of the thunk object space docs`: objspace-proxies.html#thunk-interface
 
-.. broken:
-
-    Taint Object Space Functionality
-    ================================
-
-    When the taint object space is used (choose with :config:`objspace.name`),
-    the following names are put into ``__pypy__``:
-
-     - ``taint``
-     - ``is_tainted``
-     - ``untaint``
-     - ``taint_atomic``
-     - ``_taint_debug``
-     - ``_taint_look``
-     - ``TaintError``
-
-    Those are all described in the `interface section of the taint object space
-    docs`_.
-
-    For more detailed explanations and examples see the `taint object space docs`_.
-
-    .. _`taint object space docs`: objspace-proxies.html#taint
-    .. _`interface section of the taint object space docs`: objspace-proxies.html#taint-interface
 
 Transparent Proxy Functionality
 ===============================
diff --git a/pypy/doc/config/objspace.name.txt b/pypy/doc/config/objspace.name.txt
--- a/pypy/doc/config/objspace.name.txt
+++ b/pypy/doc/config/objspace.name.txt
@@ -4,7 +4,6 @@
 for normal usage):
 
   * thunk_: The thunk object space adds lazy evaluation to PyPy.
-  * taint_: The taint object space adds soft security features.
   * dump_:  Using this object spaces results in the dumpimp of all operations
     to a log.
 
@@ -12,5 +11,4 @@
 .. _`Object Space Proxies`: ../objspace-proxies.html
 .. _`Standard Object Space`: ../objspace.html#standard-object-space
 .. _thunk: ../objspace-proxies.html#thunk
-.. _taint: ../objspace-proxies.html#taint
 .. _dump: ../objspace-proxies.html#dump
diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst
--- a/pypy/doc/index.rst
+++ b/pypy/doc/index.rst
@@ -309,7 +309,6 @@
 .. _`object space`: objspace.html
 .. _FlowObjSpace: objspace.html#the-flow-object-space 
 .. _`trace object space`: objspace.html#the-trace-object-space 
-.. _`taint object space`: objspace-proxies.html#taint
 .. _`thunk object space`: objspace-proxies.html#thunk
 .. _`transparent proxies`: objspace-proxies.html#tproxy
 .. _`Differences between PyPy and CPython`: cpython_differences.html
diff --git a/pypy/doc/objspace-proxies.rst b/pypy/doc/objspace-proxies.rst
--- a/pypy/doc/objspace-proxies.rst
+++ b/pypy/doc/objspace-proxies.rst
@@ -129,297 +129,6 @@
    function behaves lazily: all calls to it return a thunk object.
 
 
-.. broken right now:
-
-    .. _taint:
-
-    The Taint Object Space
-    ======================
-
-    Motivation
-    ----------
-
-    The Taint Object Space provides a form of security: "tainted objects",
-    inspired by various sources, see [D12.1]_ for a more detailed discussion. 
-
-    The basic idea of this kind of security is not to protect against
-    malicious code but to help with handling and boxing sensitive data. 
-    It covers two kinds of sensitive data: secret data which should not leak, 
-    and untrusted data coming from an external source and that must be 
-    validated before it is used.
-
-    The idea is that, considering a large application that handles these
-    kinds of sensitive data, there are typically only a small number of
-    places that need to explicitly manipulate that sensitive data; all the
-    other places merely pass it around, or do entirely unrelated things.
-
-    Nevertheless, if a large application needs to be reviewed for security,
-    it must be entirely carefully checked, because it is possible that a
-    bug at some apparently unrelated place could lead to a leak of sensitive
-    information in a way that an external attacker could exploit.  For
-    example, if any part of the application provides web services, an
-    attacker might be able to issue unexpected requests with a regular web
-    browser and deduce secret information from the details of the answers he
-    gets.  Another example is the common CGI attack where an attacker sends
-    malformed inputs and causes the CGI script to do unintended things.
-
-    An approach like that of the Taint Object Space allows the small parts
-    of the program that manipulate sensitive data to be explicitly marked.
-    The effect of this is that although these small parts still need a
-    careful security review, the rest of the application no longer does,
-    because even a bug would be unable to leak the information.
-
-    We have implemented a simple two-level model: objects are either
-    regular (untainted), or sensitive (tainted).  Objects are marked as
-    sensitive if they are secret or untrusted, and only declassified at
-    carefully-checked positions (e.g. where the secret data is needed, or
-    after the untrusted data has been fully validated).
-
-    It would be simple to extend the code for more fine-grained scales of
-    secrecy.  For example it is typical in the literature to consider
-    user-specified lattices of secrecy levels, corresponding to multiple
-    "owners" that cannot access data belonging to another "owner" unless
-    explicitly authorized to do so.
-
-    Tainting and untainting
-    -----------------------
-
-    Start a py.py with the Taint Object Space and try the following example::
-
-        $ py.py -o taint
-        >>>> from __pypy__ import taint
-        >>>> x = taint(6)
-
-        # x is hidden from now on.  We can pass it around and
-        # even operate on it, but not inspect it.  Taintness
-        # is propagated to operation results.
-
-        >>>> x
-        TaintError
-
-        >>>> if x > 5: y = 2   # see below
-        TaintError
-
-        >>>> y = x + 5         # ok
-        >>>> lst = [x, y]
-        >>>> z = lst.pop()
-        >>>> t = type(z)       # type() works too, tainted answer
-        >>>> t
-        TaintError
-        >>>> u = t is int      # even 'is' works
-        >>>> u
-        TaintError
-
-    Notice that using a tainted boolean like ``x > 5`` in an ``if``
-    statement is forbidden.  This is because knowing which path is followed
-    would give away a hint about ``x``; in the example above, if the
-    statement ``if x > 5: y = 2`` was allowed to run, we would know
-    something about the value of ``x`` by looking at the (untainted) value
-    in the variable ``y``.
-
-    Of course, there is a way to inspect tainted objects.  The basic way is
-    to explicitly "declassify" it with the ``untaint()`` function.  In an
-    application, the places that use ``untaint()`` are the places that need
-    careful security review.  To avoid unexpected objects showing up, the
-    ``untaint()`` function must be called with the exact type of the object
-    to declassify.  It will raise ``TaintError`` if the type doesn't match::
-
-        >>>> from __pypy__ import taint
-        >>>> untaint(int, x)
-        6
-        >>>> untaint(int, z)
-        11
-        >>>> untaint(bool, x > 5)
-        True
-        >>>> untaint(int, x > 5)
-        TaintError
-
-
-    Taint Bombs
-    -----------
-
-    In this area, a common problem is what to do about failing operations.
-    If an operation raises an exception when manipulating a tainted object,
-    then the very presence of the exception can leak information about the
-    tainted object itself.  Consider::
-
-        >>>> 5 / (x-6)
-
-    By checking if this raises ``ZeroDivisionError`` or not, we would know
-    if ``x`` was equal to 6 or not.  The solution to this problem in the
-    Taint Object Space is to introduce *Taint Bombs*.  They are a kind of
-    tainted object that doesn't contain a real object, but a pending
-    exception.  Taint Bombs are indistinguishable from normal tainted
-    objects to unprivileged code. See::
-
-        >>>> x = taint(6)
-        >>>> i = 5 / (x-6)     # no exception here
-        >>>> j = i + 1         # nor here
-        >>>> k = j + 5         # nor here
-        >>>> untaint(int, k)
-        TaintError
-
-    In the above example, all of ``i``, ``j`` and ``k`` contain a Taint
-    Bomb.  Trying to untaint it raises an exception - a generic
-    ``TaintError``.  What we win is that the exception gives little away,
-    and most importantly it occurs at the point where ``untaint()`` is
-    called, not where the operation failed.  This means that all calls to
-    ``untaint()`` - but not the rest of the code - must be carefully
-    reviewed for what occurs if they receive a Taint Bomb; they might catch
-    the ``TaintError`` and give the user a generic message that something
-    went wrong, if we are reasonably careful that the message or even its
-    presence doesn't give information away.  This might be a
-    problem by itself, but there is no satisfying general solution here:
-    it must be considered on a case-by-case basis.  Again, what the
-    Taint Object Space approach achieves is not solving these problems, but
-    localizing them to well-defined small parts of the application - namely,
-    around calls to ``untaint()``.
-
-    The ``TaintError`` exception deliberately does not include any
-    useful error messages, because they might give information away.
-    Of course, this makes debugging quite a bit harder; a difficult
-    problem to solve properly.  So far we have implemented a way to peek in a Taint
-    Box or Bomb, ``__pypy__._taint_look(x)``, and a "debug mode" that
-    prints the exception as soon as a Bomb is created - both write
-    information to the low-level stderr of the application, where we hope
-    that it is unlikely to be seen by anyone but the application
-    developer.
-
-
-    Taint Atomic functions
-    ----------------------
-
-    Occasionally, a more complicated computation must be performed on a
-    tainted object.  This requires first untainting the object, performing the
-    computations, and then carefully tainting the result again (including
-    hiding all exceptions into Bombs).
-
-    There is a built-in decorator that does this for you::
-
-        >>>> @__pypy__.taint_atomic
-        >>>> def myop(x, y):
-        ....     while x > 0:
-        ....         x -= y
-        ....     return x
-        ....
-        >>>> myop(42, 10)
-        -8
-        >>>> z = myop(taint(42), 10)
-        >>>> z
-        TaintError
-        >>>> untaint(int, z)
-        -8
-
-    The decorator makes a whole function behave like a built-in operation.
-    If no tainted argument is passed in, the function behaves normally.  But
-    if any of the arguments is tainted, it is automatically untainted - so
-    the function body always sees untainted arguments - and the eventual
-    result is tainted again (possibly in a Taint Bomb).
-
-    It is important for the function marked as ``taint_atomic`` to have no
-    visible side effects, as these could cause information leakage.
-    This is currently not enforced, which means that all ``taint_atomic``
-    functions have to be carefully reviewed for security (but not the
-    callers of ``taint_atomic`` functions).
-
-    A possible future extension would be to forbid side-effects on
-    non-tainted objects from all ``taint_atomic`` functions.
-
-    An example of usage: given a tainted object ``passwords_db`` that
-    references a database of passwords, we can write a function
-    that checks if a password is valid as follows::
-
-        @taint_atomic
-        def validate(passwords_db, username, password):
-            assert type(passwords_db) is PasswordDatabase
-            assert type(username) is str
-            assert type(password) is str
-            ...load username entry from passwords_db...
-            return expected_password == password
-
-    It returns a tainted boolean answer, or a Taint Bomb if something
-    went wrong.  A caller can do::
-
-        ok = validate(passwords_db, 'john', '1234')
-        ok = untaint(bool, ok)
-
-    This can give three outcomes: ``True``, ``False``, or a ``TaintError``
-    exception (with no information on it) if anything went wrong.  If even
-    this is considered giving too much information away, the ``False`` case
-    can be made indistinguishable from the ``TaintError`` case (simply by
-    raising an exception in ``validate()`` if the password is wrong).
-
-    In the above example, the security results achieved are the following:
-    as long as ``validate()`` does not leak information, no other part of
-    the code can obtain more information about a passwords database than a
-    Yes/No answer to a precise query.
-
-    A possible extension of the ``taint_atomic`` decorator would be to check
-    the argument types, as ``untaint()`` does, for the same reason: to
-    prevent bugs where a function like ``validate()`` above is accidentally
-    called with the wrong kind of tainted object, which would make it
-    misbehave.  For now, all ``taint_atomic`` functions should be
-    conservative and carefully check all assumptions on their input
-    arguments.
-
-
-    .. _`taint-interface`:
-
-    Interface
-    ---------
-
-    .. _`like a built-in operation`:
-
-    The basic rule of the Tainted Object Space is that it introduces two new
-    kinds of objects, Tainted Boxes and Tainted Bombs (which are not types
-    in the Python sense).  Each box internally contains a regular object;
-    each bomb internally contains an exception object.  An operation
-    involving Tainted Boxes is performed on the objects contained in the
-    boxes, and gives a Tainted Box or a Tainted Bomb as a result (such an
-    operation does not let an exception be raised).  An operation called
-    with a Tainted Bomb argument immediately returns the same Tainted Bomb.
-
-    In a PyPy running with (or translated with) the Taint Object Space,
-    the ``__pypy__`` module exposes the following interface:
-
-    * ``taint(obj)``
-
-        Return a new Tainted Box wrapping ``obj``.  Return ``obj`` itself
-        if it is already tainted (a Box or a Bomb).
-
-    * ``is_tainted(obj)``
-
-        Check if ``obj`` is tainted (a Box or a Bomb).
-
-    * ``untaint(type, obj)``
-
-        Untaints ``obj`` if it is tainted.  Raise ``TaintError`` if the type
-        of the untainted object is not exactly ``type``, or if ``obj`` is a
-        Bomb.
-
-    * ``taint_atomic(func)``
-
-        Return a wrapper function around the callable ``func``.  The wrapper
-        behaves `like a built-in operation`_ with respect to untainting the
-        arguments, tainting the result, and returning a Bomb.
-
-    * ``TaintError``
-
-        Exception.  On purpose, it provides no attribute or error message.
-
-    * ``_taint_debug(level)``
-
-        Set the debugging level to ``level`` (0=off).  At level 1 or above,
-        all Taint Bombs print a diagnostic message to stderr when they are
-        created.
-
-    * ``_taint_look(obj)``
-
-        For debugging purposes: prints (to stderr) the type and address of
-        the object in a Tainted Box, or prints the exception if ``obj`` is
-        a Taint Bomb.
-
-
 .. _dump:
 
 The Dump Object Space
diff --git a/pypy/objspace/std/objecttype.py b/pypy/objspace/std/objecttype.py
--- a/pypy/objspace/std/objecttype.py
+++ b/pypy/objspace/std/objecttype.py
@@ -44,7 +44,6 @@
         raise OperationError(space.w_TypeError,
                              space.wrap("__class__ assignment: only for heap types"))
     w_oldcls = space.type(w_obj)
-    # XXX taint space should raise a TaintError here if w_oldcls is tainted
     assert isinstance(w_oldcls, W_TypeObject)
     if w_oldcls.get_full_instance_layout() == w_newcls.get_full_instance_layout():
         w_obj.setclass(space, w_newcls)
diff --git a/pypy/objspace/taint.py b/pypy/objspace/taint.py
deleted file mode 100644
--- a/pypy/objspace/taint.py
+++ /dev/null
@@ -1,294 +0,0 @@
-"""
-Just an experiment.
-"""
-import os
-from pypy.objspace.std.objspace import StdObjSpace
-from pypy.objspace.proxy import patch_space_in_place
-from pypy.objspace.thunk import nb_forcing_args
-from pypy.interpreter.error import OperationError
-from pypy.interpreter import baseobjspace, gateway, executioncontext
-from pypy.interpreter.function import Method
-from pypy.interpreter.pyframe import PyFrame
-from pypy.tool.sourcetools import func_with_new_name
-from pypy.rlib.unroll import unrolling_iterable
-
-
-class W_Tainted(baseobjspace.W_Root):
-    def __init__(self, w_obj):
-        self.w_obj = w_obj
-
-##    def getdict(self, space):
-##        return taint(self.w_obj.getdict(space))
-
-##    def getdictvalue(self, space, attr):
-##        return taint(self.w_obj.getdictvalue(space, attr))
-
-##    def setdictvalue(self, space, attr, w_value):
-##        return self.w_obj.setdictvalue(space, attr, w_value)
-
-##    ...
-
-class W_TaintBomb(baseobjspace.W_Root):
-    filename = '?'
-    codename = '?'
-    codeline = 0
-
-    def __init__(self, space, operr):
-        self.space = space
-        self.operr = operr
-        self.record_debug_info()
-
-    def record_debug_info(self):
-        ec = self.space.getexecutioncontext()
-        frame = ec.gettopframe_nohidden()
-        if isinstance(frame, PyFrame):     # and, in particular, frame != None
-            self.filename = frame.pycode.co_filename
-            self.codename = frame.pycode.co_name
-            self.codeline = frame.get_last_lineno()
-        if get_debug_level(self.space) > 0:
-            self.debug_dump()
-
-    def debug_dump(self):
-        os.write(2, 'Taint Bomb from file "%s", line %d, in %s\n    %s\n' % (
-            self.filename, self.codeline, self.codename,
-            self.operr.errorstr(self.space)))
-
-    def explode(self):
-        #msg = self.operr.errorstr(space)
-        raise OperationError(self.space.w_TaintError, self.space.w_None)
-
-
-def taint(w_obj):
-    """Return a tainted version of the argument."""
-    if w_obj is None or isinstance(w_obj, W_Tainted):
-        return w_obj
-    else:
-        return W_Tainted(w_obj)
-app_taint = gateway.interp2app(taint)
-
-def is_tainted(space, w_obj):
-    """Return whether the argument is tainted."""
-    res = isinstance(w_obj, W_Tainted) or isinstance(w_obj, W_TaintBomb)
-    return space.wrap(res)
-app_is_tainted = gateway.interp2app(is_tainted)
-
-def untaint(space, w_expectedtype, w_obj):
-    """untaint(expectedtype, tainted_obj) -> obj
-Untaint untainted_obj and return it. If the result is not of expectedtype,
-raise a type error."""
-    if (isinstance(w_expectedtype, W_Tainted) or
-        isinstance(w_expectedtype, W_TaintBomb)):
-        raise OperationError(space.w_TypeError,
-                  space.wrap("untaint() arg 1 must be an untainted type"))
-    if not space.is_true(space.isinstance(w_expectedtype, space.w_type)):
-        raise OperationError(space.w_TypeError,
-                             space.wrap("untaint() arg 1 must be a type"))
-    if isinstance(w_obj, W_Tainted):
-        w_obj = w_obj.w_obj
-    elif isinstance(w_obj, W_TaintBomb):
-        w_obj.explode()
-    #if isinstance(w_expectedtype, W_Tainted):
-    #    w_expectedtype = w_expectedtype.w_obj
-    w_realtype = space.type(w_obj)
-    if not space.is_w(w_realtype, w_expectedtype):
-        #msg = "expected an object of type '%s'" % (
-        #    w_expectedtype.getname(space),)
-        #    #w_realtype.getname(space))
-        raise OperationError(space.w_TaintError, space.w_None)
-    return w_obj
-app_untaint = gateway.interp2app(untaint)
-
-# ____________________________________________________________
-
- at gateway.unwrap_spec(args_w='args_w')
-def taint_atomic_function(space, w_func, args_w):
-    newargs_w = []
-    tainted = False
-    for w_arg in args_w:
-        if isinstance(w_arg, W_Tainted):
-            tainted = True
-            w_arg = w_arg.w_obj
-        elif isinstance(w_arg, W_TaintBomb):
-            return w_arg
-        newargs_w.append(w_arg)
-    w_newargs = space.newtuple(newargs_w)
-    try:
-        w_res = space.call(w_func, w_newargs)
-    except OperationError, operr:
-        if not tainted:
-            raise
-        return W_TaintBomb(space, operr)
-    if tainted:
-        w_res = taint(w_res)
-    return w_res
-
-app_taint_atomic_function = gateway.interp2app(taint_atomic_function)
-
-def taint_atomic(space, w_callable):
-    """decorator to make a callable "taint-atomic": if the function is called
-with tainted arguments, those are untainted. The result of the function is
-tainted again.  All exceptions that the callable raises are turned into
-taint bombs."""
-    meth = Method(space, space.w_fn_taint_atomic_function,
-                  w_callable, space.type(w_callable))
-    return space.wrap(meth)
-app_taint_atomic = gateway.interp2app(taint_atomic)
-
-# ____________________________________________________________
-
-executioncontext.ExecutionContext.taint_debug = 0
-
- at gateway.unwrap_spec(level=int)
-def taint_debug(space, level):
-    """Set the debug level. If the debug level is greater than 0, the creation
-of taint bombs will print debug information. For debugging purposes
-only!"""
-    space.getexecutioncontext().taint_debug = level
-app_taint_debug = gateway.interp2app(taint_debug)
-
-def taint_look(space, w_obj):
-    """Print some info about the taintedness of an object. For debugging
-purposes only!"""
-    if isinstance(w_obj, W_Tainted):
-        info = space.type(w_obj.w_obj).getname(space)
-        msg = space.str_w(w_obj.w_obj.getrepr(space, info))
-        msg = 'Taint Box %s\n' % msg
-        os.write(2, msg)
-    elif isinstance(w_obj, W_TaintBomb):
-        w_obj.debug_dump()
-    else:
-        os.write(2, 'not tainted\n')
-app_taint_look = gateway.interp2app(taint_look)
-
-def get_debug_level(space):
-    return space.getexecutioncontext().taint_debug
-
-def debug_bomb(space, operr):
-    ec = space.getexecutioncontext()
-    filename = '?'
-    codename = '?'
-    codeline = 0
-    frame = ec.gettopframe_nohidden()
-    if isinstance(frame, PyFrame):     # and, in particular, frame != None
-        filename = frame.pycode.co_filename
-        codename = frame.pycode.co_name
-        codeline = frame.get_last_lineno()
-    os.write(2, 'Taint Bomb in file "%s", line %d, in %s\n    %s\n' % (
-        filename, codeline, codename, operr.errorstr(space)))
-
-# ____________________________________________________________
-
-
-class TaintSpace(StdObjSpace):
-
-    def __init__(self, *args, **kwds):
-        StdObjSpace.__init__(self, *args, **kwds)
-        w_dict = self.newdict()
-        self.setitem(w_dict, self.wrap("__doc__"), self.wrap("""\
-Exception that is raised when an operation revealing information on a tainted
-object is performed."""))
-        self.w_TaintError = self.call_function(
-            self.w_type,
-            self.wrap("TaintError"),
-            self.newtuple([self.w_Exception]),
-            w_dict
-            )
-        w___pypy__ = self.getbuiltinmodule("__pypy__")
-        self.setattr(w___pypy__, self.wrap('taint'),
-                     self.wrap(app_taint))
-        self.setattr(w___pypy__, self.wrap('is_tainted'),
-                     self.wrap(app_is_tainted))
-        self.setattr(w___pypy__, self.wrap('untaint'),
-                     self.wrap(app_untaint))
-        self.w_fn_taint_atomic_function = self.wrap(app_taint_atomic_function)
-        self.setattr(w___pypy__, self.wrap('taint_atomic'),
-                     self.wrap(app_taint_atomic))
-        self.setattr(w___pypy__, self.wrap('TaintError'),
-                     self.w_TaintError)
-        self.setattr(w___pypy__, self.wrap('_taint_debug'),
-                     self.wrap(app_taint_debug))
-        self.setattr(w___pypy__, self.wrap('_taint_look'),
-                     self.wrap(app_taint_look))
-        patch_space_in_place(self, 'taint', proxymaker)
-
-        # XXX may leak info, perfomance hit, what about taint bombs?
-        from pypy.objspace.std.typeobject import W_TypeObject
-
-        def taint_lookup(w_obj, name):
-            if isinstance(w_obj, W_Tainted):
-                w_obj = w_obj.w_obj
-            w_type = self.type(w_obj)
-            assert isinstance(w_type, W_TypeObject)
-            return w_type.lookup(name)
-
-        def taint_lookup_in_type_where(w_obj, name):
-            if isinstance(w_obj, W_Tainted):
-                w_type = w_obj.w_obj
-            else:
-                w_type = w_obj
-            assert isinstance(w_type, W_TypeObject)
-            return w_type.lookup_where(name)
-
-        self.lookup = taint_lookup
-        self.lookup_in_type_where = taint_lookup_in_type_where
-
-
-Space = TaintSpace
-
-
-def tainted_error(space, name):
-    #msg = "operation '%s' forbidden on tainted object" % (name,)
-    raise OperationError(space.w_TaintError, space.w_None)# space.wrap(msg))
-
-
-RegularMethods = dict.fromkeys(
-    [name for name, _, _, _ in baseobjspace.ObjSpace.MethodTable])
-
-TaintResultIrregularMethods = dict.fromkeys(
-    ['wrap', 'call_args'] +
-    [name for name in baseobjspace.ObjSpace.IrregularOpTable
-          if name.startswith('new')])
-
-def proxymaker(space, name, parentfn):
-    arity = nb_forcing_args[name]
-    indices = unrolling_iterable(range(arity))
-    if name in RegularMethods:
-
-        def proxy(*args_w):
-            newargs_w = ()
-            tainted = False
-            for i in indices:
-                w_arg = args_w[i]
-                if isinstance(w_arg, W_Tainted):
-                    tainted = True
-                    w_arg = w_arg.w_obj
-                elif isinstance(w_arg, W_TaintBomb):
-                    return w_arg
-                newargs_w += (w_arg,)
-            newargs_w += args_w[arity:]
-            try:
-                w_res = parentfn(*newargs_w)
-            except OperationError, operr:
-                if not tainted:
-                    raise
-                return W_TaintBomb(space, operr)
-            if tainted:
-                w_res = taint(w_res)
-            return w_res
-
-    elif arity == 0:
-        return None
-
-    else:
-
-        def proxy(*args_w):
-            for i in indices:
-                w_arg = args_w[i]
-                if isinstance(w_arg, W_Tainted):
-                    tainted_error(space, name)
-                elif isinstance(w_arg, W_TaintBomb):
-                    w_arg.explode()
-            return parentfn(*args_w)
-
-    proxy = func_with_new_name(proxy, '%s_proxy' % name)
-    return proxy
diff --git a/pypy/objspace/test/test_taintobjspace.py b/pypy/objspace/test/test_taintobjspace.py
deleted file mode 100644
--- a/pypy/objspace/test/test_taintobjspace.py
+++ /dev/null
@@ -1,77 +0,0 @@
-from pypy.conftest import gettestobjspace
-
-class AppTest_Taint:
-
-    def setup_class(cls):
-        cls.space = gettestobjspace('taint')
-
-    def test_simple(self):
-        from __pypy__ import taint, untaint, TaintError
-        x = taint(6)
-        x = x * 7
-        raises(TaintError, "if x: y = 1")
-        t = type(x)
-        raises(TaintError, "if t is int: y = 1")
-        assert untaint(int, x) == 42
-        raises(TaintError, "untaint(float, x)")
-
-    def test_bomb(self):
-        from __pypy__ import taint, untaint, TaintError
-        x = taint(6)
-        x = x / 0
-        raises(TaintError, "if x: y = 1")
-        t = type(x)
-        raises(TaintError, "if t is int: y = 1")
-        raises(TaintError, "untaint(int, x)")
-        raises(TaintError, "untaint(float, x)")
-
-    def test_taint_atomic(self):
-        from __pypy__ import taint, untaint, TaintError, taint_atomic
-        x = taint(6)
-        x *= 7
-
-        def dummy(x):
-            if x > 40:
-                return 5
-            else:
-                return 3
-        dummy = taint_atomic(dummy)
-
-        y = dummy(x)
-        raises(TaintError, "if y == 3: z = 1")
-        assert untaint(int, y) == 5
-
-    def test_taint_atomic_exception(self):
-        from __pypy__ import taint, untaint, TaintError, taint_atomic
-        x = taint(6)
-        x *= 7
-
-        def dummy(x):
-            if x + "world" == "hello world":
-                return 5
-            else:
-                return 3
-        dummy = taint_atomic(dummy)
-
-        y = dummy(x)
-        raises(TaintError, "if y == 3: z = 1")
-        raises(TaintError, "untaint(int, y)")
-
-    def test_taint_atomic_incoming_bomb(self):
-        from __pypy__ import taint, untaint, TaintError, taint_atomic
-        x = taint(6)
-        x /= 0
-        lst = []
-
-        def dummy(x):
-            lst.append("running!")
-            if x > 40:
-                return 5
-            else:
-                return 3
-        dummy = taint_atomic(dummy)
-
-        y = dummy(x)
-        raises(TaintError, "if y == 3: z = 1")
-        assert lst == []
-        raises(TaintError, "untaint(int, y)")

From noreply at buildbot.pypy.org  Wed Oct 12 17:22:29 2011
From: noreply at buildbot.pypy.org (gutworth)
Date: Wed, 12 Oct 2011 17:22:29 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20111012152229.D832782112@wyvern.cs.uni-duesseldorf.de>

Author: Benjamin Peterson <benjamin at python.org>
Branch: 
Changeset: r47969:da18a97b974d
Date: 2011-10-12 11:22 -0400
http://bitbucket.org/pypy/pypy/changeset/da18a97b974d/

Log:	merge heads

diff --git a/pypy/module/__builtin__/__init__.py b/pypy/module/__builtin__/__init__.py
--- a/pypy/module/__builtin__/__init__.py
+++ b/pypy/module/__builtin__/__init__.py
@@ -21,6 +21,8 @@
         'all'           : 'app_functional.all',
         'sum'           : 'app_functional.sum',
         'map'           : 'app_functional.map',
+        'reduce'        : 'app_functional.reduce',
+        'filter'        : 'app_functional.filter',
         'vars'          : 'app_inspect.vars',
         'dir'           : 'app_inspect.dir',
 
@@ -88,9 +90,7 @@
         'min'           : 'functional.min',
         'max'           : 'functional.max',
         'zip'           : 'functional.zip',
-        'reduce'        : 'functional.reduce',
         'reversed'      : 'functional.reversed',
-        'filter'        : 'functional.filter',
         'super'         : 'descriptor.W_Super',
         'staticmethod'  : 'descriptor.StaticMethod',
         'classmethod'   : 'descriptor.ClassMethod',
diff --git a/pypy/module/__builtin__/app_functional.py b/pypy/module/__builtin__/app_functional.py
--- a/pypy/module/__builtin__/app_functional.py
+++ b/pypy/module/__builtin__/app_functional.py
@@ -98,3 +98,68 @@
                 result.append(func(*args))
         else:
             return result
+
+sentinel = object()
+
+def reduce(func, sequence, initial=sentinel):
+    """reduce(function, sequence[, initial]) -> value
+
+Apply a function of two arguments cumulatively to the items of a sequence,
+from left to right, so as to reduce the sequence to a single value.
+For example, reduce(lambda x, y: x+y, [1, 2, 3, 4, 5]) calculates
+((((1+2)+3)+4)+5).  If initial is present, it is placed before the items
+of the sequence in the calculation, and serves as a default when the
+sequence is empty."""
+    iterator = iter(sequence)
+    if initial is sentinel:
+        try:
+            initial = next(iterator)
+        except StopIteration:
+            raise TypeError("reduce() of empty sequence with no initial value")
+    result = initial
+    for item in iterator:
+        result = func(result, item)
+    return result
+
+def filter(func, seq):
+    """filter(function or None, sequence) -> list, tuple, or string
+
+Return those items of sequence for which function(item) is true.  If
+function is None, return the items that are true.  If sequence is a tuple
+or string, return the same type, else return a list."""
+    if func is None:
+        func = bool
+    if isinstance(seq, str):
+        return _filter_string(func, seq, str)
+    elif isinstance(seq, unicode):
+        return _filter_string(func, seq, unicode)
+    elif isinstance(seq, tuple):
+        return _filter_tuple(func, seq)
+    result = []
+    for item in seq:
+        if func(item):
+            result.append(item)
+    return result
+
+def _filter_string(func, string, str_type):
+    if func is bool and type(string) is str_type:
+        return string
+    result = []
+    for i in range(len(string)):
+        # You must call __getitem__ on the strings, simply iterating doesn't
+        # work :/
+        item = string[i]
+        if func(item):
+            if not isinstance(item, str_type):
+                raise TypeError("__getitem__ returned a non-string type")
+            result.append(item)
+    return str_type().join(result)
+
+def _filter_tuple(func, seq):
+    result = []
+    for i in range(len(seq)):
+        # Again, must call __getitem__, at least there are tests.
+        item = seq[i]
+        if func(item):
+            result.append(item)
+    return tuple(result)
\ No newline at end of file
diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py
--- a/pypy/module/__builtin__/functional.py
+++ b/pypy/module/__builtin__/functional.py
@@ -222,90 +222,6 @@
             return space.newlist(result_w)
         result_w.append(space.newtuple(items_w))
 
-def reduce(space, w_func, w_sequence, w_initial=NoneNotWrapped):
-    """ Apply function of two arguments cumulatively to the items of sequence,
-        from left to right, so as to reduce the sequence to a single value.
-        Optionally begin with an initial value.
-    """
-    w_iter = space.iter(w_sequence)
-    if w_initial is None:
-        try:
-            w_initial = space.next(w_iter)
-        except OperationError, e:
-            if e.match(space, space.w_StopIteration):
-                msg = "reduce() of empty sequence with no initial value"
-                raise OperationError(space.w_TypeError, space.wrap(msg))
-            raise
-    w_result = w_initial
-    while True:
-        try:
-            w_next = space.next(w_iter)
-        except OperationError, e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            break
-        w_result = space.call_function(w_func, w_result, w_next)
-    return w_result
-
-def filter(space, w_func, w_seq):
-    """construct a list of those elements of collection for which function
-       is True.  If function is None, then return the items in the sequence
-       which are True.
-    """
-    if space.is_true(space.isinstance(w_seq, space.w_str)):
-        return _filter_string(space, w_func, w_seq, space.w_str)
-    if space.is_true(space.isinstance(w_seq, space.w_unicode)):
-        return _filter_string(space, w_func, w_seq, space.w_unicode)
-    if space.is_true(space.isinstance(w_seq, space.w_tuple)):
-        return _filter_tuple(space, w_func, w_seq)
-    w_iter = space.iter(w_seq)
-    result_w = []
-    none_func = space.is_w(w_func, space.w_None)
-    while True:
-        try:
-            w_next = space.next(w_iter)
-        except OperationError, e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            break
-        if none_func:
-            w_keep = w_next
-        else:
-            w_keep = space.call_function(w_func, w_next)
-        if space.is_true(w_keep):
-            result_w.append(w_next)
-    return space.newlist(result_w)
-
-def _filter_tuple(space, w_func, w_tuple):
-    none_func = space.is_w(w_func, space.w_None)
-    length = space.len_w(w_tuple)
-    result_w = []
-    for i in range(length):
-        w_item = space.getitem(w_tuple, space.wrap(i))
-        if none_func:
-            w_keep = w_item
-        else:
-            w_keep = space.call_function(w_func, w_item)
-        if space.is_true(w_keep):
-            result_w.append(w_item)
-    return space.newtuple(result_w)
-
-def _filter_string(space, w_func, w_string, w_str_type):
-    none_func = space.is_w(w_func, space.w_None)
-    if none_func and space.is_w(space.type(w_string), w_str_type):
-        return w_string
-    length = space.len_w(w_string)
-    result_w = []
-    for i in range(length):
-        w_item = space.getitem(w_string, space.wrap(i))
-        if none_func or space.is_true(space.call_function(w_func, w_item)):
-            if not space.is_true(space.isinstance(w_item, w_str_type)):
-                msg = "__getitem__ returned a non-string type"
-                raise OperationError(space.w_TypeError, space.wrap(msg))
-            result_w.append(w_item)
-    w_empty = space.call_function(w_str_type)
-    return space.call_method(w_empty, "join", space.newlist(result_w))
-
 class W_Enumerate(Wrappable):
 
     def __init__(self, w_iter, w_start):

From noreply at buildbot.pypy.org  Wed Oct 12 19:06:04 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 12 Oct 2011 19:06:04 +0200 (CEST)
Subject: [pypy-commit] pypy.org extradoc: fight copyright sign
Message-ID: <20111012170604.B3D1482112@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: extradoc
Changeset: r279:bc1b15fa4fb0
Date: 2011-10-12 19:05 +0200
http://bitbucket.org/pypy/pypy.org/changeset/bc1b15fa4fb0/

Log:	fight copyright sign

diff --git a/py3donate.html b/py3donate.html
--- a/py3donate.html
+++ b/py3donate.html
@@ -74,7 +74,7 @@
 at the latest, we will try our best to make PyPy support Python 3 anyway.  We
 however reserve the right to shift any unused funds to other PyPy activities
 when that date is reached.  Of course, since the Conservancy is a
-501&copy;(3) charitable organization incorporated in NY, USA, all funds will,
+501(c)(3) charitable organization incorporated in NY, USA, all funds will,
 regardless of their use, be spent in a way that benefits the general
 public, the advancement of Open Source and Free Software,
 and in particular the PyPy community and the PyPy codebase.</p>

From noreply at buildbot.pypy.org  Wed Oct 12 19:48:52 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Wed, 12 Oct 2011 19:48:52 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: english and fact-check update.
Message-ID: <20111012174852.077CE82112@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: extradoc
Changeset: r3925:ad23b3cb300d
Date: 2011-10-12 13:48 -0400
http://bitbucket.org/pypy/extradoc/changeset/ad23b3cb300d/

Log:	english and fact-check update.

diff --git a/blog/draft/numpy-statups-update.rst b/blog/draft/numpy-statups-update.rst
--- a/blog/draft/numpy-statups-update.rst
+++ b/blog/draft/numpy-statups-update.rst
@@ -18,9 +18,9 @@
 
 Some of the things we're working on at the moment:
 
-* More dtypes, including unsigned integers, singe floats, and complex doubles.
+* More dtypes, including complex values and user-defined dtypes.
 * Subscripting arrays by other array as indices, and by bool arrays as masks.
-* Starting to reuse python-level original numpy
+* Starting to reuse Python code from the original numpy.
 
 Some of the things on the near horizon are:
 

From noreply at buildbot.pypy.org  Wed Oct 12 21:04:12 2011
From: noreply at buildbot.pypy.org (hakanardo)
Date: Wed, 12 Oct 2011 21:04:12 +0200 (CEST)
Subject: [pypy-commit] pypy default: Kill history.Const.__eq__ to get
 consistant behaviour before and after translation
Message-ID: <20111012190412.07D0E82112@wyvern.cs.uni-duesseldorf.de>

Author: Hakan Ardo <hakan at debian.org>
Branch: 
Changeset: r47970:80901cc39f5d
Date: 2011-10-12 21:03 +0200
http://bitbucket.org/pypy/pypy/changeset/80901cc39f5d/

Log:	Kill history.Const.__eq__ to get consistant behaviour before and
	after translation

diff --git a/pypy/jit/metainterp/history.py b/pypy/jit/metainterp/history.py
--- a/pypy/jit/metainterp/history.py
+++ b/pypy/jit/metainterp/history.py
@@ -9,6 +9,7 @@
 
 from pypy.jit.metainterp.resoperation import ResOperation, rop
 from pypy.jit.codewriter import heaptracker, longlong
+from pypy.rlib.objectmodel import compute_identity_hash
 
 # ____________________________________________________________
 
@@ -104,7 +105,7 @@
     getref._annspecialcase_ = 'specialize:arg(1)'
 
     def _get_hash_(self):
-        raise NotImplementedError
+        return compute_identity_hash(self)
 
     def clonebox(self):
         raise NotImplementedError
@@ -133,6 +134,9 @@
     def _get_str(self):
         raise NotImplementedError
 
+    def same_box(self, other):
+        return self is other
+
 class AbstractDescr(AbstractValue):
     __slots__ = ()
 
@@ -241,32 +245,15 @@
     def constbox(self):
         return self
 
+    def same_box(self, other):
+        return self.same_constant(other)
+
     def same_constant(self, other):
         raise NotImplementedError
 
     def __repr__(self):
         return 'Const(%s)' % self._getrepr_()
 
-    def __eq__(self, other):
-        "NOT_RPYTHON"
-        # Remember that you should not compare Consts with '==' in RPython.
-        # Consts have no special __hash__, in order to force different Consts
-        # from being considered as different keys when stored in dicts
-        # (as they always are after translation).  Use a dict_equal_consts()
-        # to get the other behavior (i.e. using this __eq__).
-        if self.__class__ is not other.__class__:
-            return False
-        try:
-            return self.value == other.value
-        except TypeError:
-            if (isinstance(self.value, Symbolic) and
-                isinstance(other.value, Symbolic)):
-                return self.value is other.value
-            raise
-
-    def __ne__(self, other):
-        return not (self == other)
-
 
 class ConstInt(Const):
     type = INT
@@ -688,33 +675,6 @@
 
 # ____________________________________________________________
 
-def dict_equal_consts():
-    "NOT_RPYTHON"
-    # Returns a dict in which Consts that compare as equal
-    # are identified when used as keys.
-    return r_dict(dc_eq, dc_hash)
-
-def dc_eq(c1, c2):
-    return c1 == c2
-
-def dc_hash(c):
-    "NOT_RPYTHON"
-    # This is called during translation only.  Avoid using identityhash(),
-    # to avoid forcing a hash, at least on lltype objects.
-    if not isinstance(c, Const):
-        return hash(c)
-    if isinstance(c.value, Symbolic):
-        return id(c.value)
-    try:
-        if isinstance(c, ConstPtr):
-            p = lltype.normalizeptr(c.value)
-            if p is not None:
-                return hash(p._obj)
-            else:
-                return 0
-        return c._get_hash_()
-    except lltype.DelayedPointer:
-        return -2      # xxx risk of changing hash...
 
 def make_hashable_int(i):
     from pypy.rpython.lltypesystem.ll2ctypes import NotCtypesAllocatedStructure
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -2329,7 +2329,7 @@
         def _variables_equal(box, varname, strict):
             if varname not in virtuals:
                 if strict:
-                    assert box == oparse.getvar(varname)
+                    assert box.same_box(oparse.getvar(varname))
                 else:
                     assert box.value == oparse.getvar(varname).value
             else:
diff --git a/pypy/jit/metainterp/optimizeopt/util.py b/pypy/jit/metainterp/optimizeopt/util.py
--- a/pypy/jit/metainterp/optimizeopt/util.py
+++ b/pypy/jit/metainterp/optimizeopt/util.py
@@ -90,14 +90,11 @@
     for i in range(len(args1)):
         arg1 = args1[i]
         arg2 = args2[i]
-        if isinstance(arg1, history.Const):
-            if arg1.__class__ is not arg2.__class__:
+        if arg1 is None:
+            if arg2 is not None:
                 return False
-            if not arg1.same_constant(arg2):
-                return False
-        else:
-            if not arg1 is arg2:
-                return False
+        elif not arg1.same_box(arg2):
+            return False
     return True
 
 def args_hash(args):
@@ -106,10 +103,8 @@
     for arg in args:
         if arg is None:
             y = 17
-        elif isinstance(arg, history.Const):
+        else:
             y = arg._get_hash_()
-        else:
-            y = compute_identity_hash(arg)
         res = intmask((1000003 * res) ^ y)
     return res
 
@@ -145,9 +140,12 @@
         for i in range(op1.numargs()):
             x = op1.getarg(i)
             y = op2.getarg(i)
-            assert x == remap.get(y, y)
+            assert x.same_box(remap.get(y, y))
         if op2.result in remap:
-            assert op1.result == remap[op2.result]
+            if op2.result is None:
+                assert op1.result == remap[op2.result]
+            else:
+                assert op1.result.same_box(remap[op2.result])
         else:
             remap[op2.result] = op1.result
         if op1.getopnum() != rop.JUMP:      # xxx obscure
@@ -156,11 +154,20 @@
             assert len(op1.getfailargs()) == len(op2.getfailargs())
             if strict_fail_args:
                 for x, y in zip(op1.getfailargs(), op2.getfailargs()):
-                    assert x == remap.get(y, y)
+                    if x is None:
+                        assert remap.get(y, y) is None
+                    else:
+                        assert x.same_box(remap.get(y, y))
             else:
                 fail_args1 = set(op1.getfailargs())
                 fail_args2 = set([remap.get(y, y) for y in op2.getfailargs()])
-                assert fail_args1 == fail_args2
+                for x in fail_args1:
+                    for y in fail_args2:
+                        if x.same_box(y):
+                            fail_args2.remove(y)
+                            break
+                    else:
+                        assert False
     assert len(oplist1) == len(oplist2)
     print '-'*totwidth
     return True
diff --git a/pypy/jit/metainterp/optimizeopt/vstring.py b/pypy/jit/metainterp/optimizeopt/vstring.py
--- a/pypy/jit/metainterp/optimizeopt/vstring.py
+++ b/pypy/jit/metainterp/optimizeopt/vstring.py
@@ -587,10 +587,7 @@
             return True
         #
         if v1.is_nonnull() and v2.is_nonnull():
-            if l1box is not None and l2box is not None and (
-                l1box == l2box or (isinstance(l1box, ConstInt) and
-                                   isinstance(l2box, ConstInt) and
-                                   l1box.value == l2box.value)):
+            if l1box is not None and l2box is not None and l1box.same_box(l2box):
                 do = EffectInfo.OS_STREQ_LENGTHOK
             else:
                 do = EffectInfo.OS_STREQ_NONNULL
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -162,7 +162,9 @@
             if registers[i] is oldbox:
                 registers[i] = newbox
         if not we_are_translated():
-            assert oldbox not in registers[count:]
+            for b in registers[count:]:
+                assert not oldbox.same_box(b)
+                
 
     def make_result_of_lastop(self, resultbox):
         got_type = resultbox.type
diff --git a/pypy/jit/metainterp/test/test_resume.py b/pypy/jit/metainterp/test/test_resume.py
--- a/pypy/jit/metainterp/test/test_resume.py
+++ b/pypy/jit/metainterp/test/test_resume.py
@@ -180,22 +180,27 @@
     reader.consume_boxes(info, bi, br, bf)
     b1s = reader.liveboxes[0]
     b2s = reader.liveboxes[1]
-    assert bi == [b1s, ConstInt(111), b1s]
-    assert br == [ConstPtr(gcrefnull), b2s]
+    assert_same(bi, [b1s, ConstInt(111), b1s])
+    assert_same(br, [ConstPtr(gcrefnull), b2s])
     bi, br, bf = [None]*2, [None]*0, [None]*0
     info = MyBlackholeInterp([lltype.Signed,
                               lltype.Signed]).get_current_position_info()
     reader.consume_boxes(info, bi, br, bf)
-    assert bi == [ConstInt(222), ConstInt(333)]
+    assert_same(bi, [ConstInt(222), ConstInt(333)])
     bi, br, bf = [None]*2, [None]*1, [None]*0
     info = MyBlackholeInterp([lltype.Signed, llmemory.GCREF,
                               lltype.Signed]).get_current_position_info()
     reader.consume_boxes(info, bi, br, bf)
     b3s = reader.liveboxes[2]
-    assert bi == [b1s, b3s]
-    assert br == [b2s]
+    assert_same(bi, [b1s, b3s])
+    assert_same(br, [b2s])
     #
 
+def assert_same(list1, list2):
+    assert len(list1) == len(list2)
+    for b1, b2 in zip(list1, list2):
+        assert b1.same_box(b2)
+
 def test_simple_read_tagged_ints():
     storage = Storage()
     storage.rd_consts = []
@@ -1023,11 +1028,11 @@
     metainterp = MyMetaInterp()
     reader = ResumeDataFakeReader(storage, newboxes, metainterp)
     lst = reader.consume_boxes()
-    assert lst == [b1t, b1t, b3t]
+    assert_same(lst, [b1t, b1t, b3t])
     lst = reader.consume_boxes()
-    assert lst == [ConstInt(2), ConstInt(3)]
+    assert_same(lst, [ConstInt(2), ConstInt(3)])
     lst = reader.consume_boxes()
-    assert lst == [b1t, ConstInt(1), b1t, b1t]
+    assert_same(lst, [b1t, ConstInt(1), b1t, b1t])
     assert metainterp.trace == []    
 
 
@@ -1044,11 +1049,11 @@
     reader = ResumeDataFakeReader(storage, newboxes, metainterp)
     lst = reader.consume_boxes()
     c1t = ConstInt(111)
-    assert lst == [c1t, b2t, b3t]
+    assert_same(lst, [c1t, b2t, b3t])
     lst = reader.consume_boxes()
-    assert lst == [ConstInt(2), ConstInt(3)]
+    assert_same(lst, [ConstInt(2), ConstInt(3)])
     lst = reader.consume_boxes()
-    assert lst == [c1t, ConstInt(1), c1t, b2t]
+    assert_same(lst, [c1t, ConstInt(1), c1t, b2t])
     assert metainterp.trace == []
 
 
@@ -1114,9 +1119,11 @@
     # check that we get the operations in 'expected', in a possibly different
     # order.
     assert len(trace) == len(expected)
-    for x in trace:
-        assert x in expected
-        expected.remove(x)
+    with CompareableConsts():
+        for x in trace:
+            assert x in expected
+            expected.remove(x)
+
     ptr = b2t.value._obj.container._as_ptr()
     assert lltype.typeOf(ptr) == lltype.Ptr(LLtypeMixin.NODE)
     assert ptr.value == 111
@@ -1126,6 +1133,18 @@
     assert ptr2.parent.value == 33
     assert ptr2.parent.next == ptr
 
+class CompareableConsts(object):
+    def __init__(self):
+        self.oldeq = None
+        
+    def __enter__(self):
+        assert self.oldeq is None
+        self.oldeq = Const.__eq__
+        Const.__eq__ = Const.same_box
+        
+    def __exit__(self, type, value, traceback):
+        Const.__eq__ = self.oldeq
+
 def test_virtual_adder_make_varray():
     b2s, b4s = [BoxPtr(), BoxInt(4)]
     c1s = ConstInt(111)
@@ -1163,8 +1182,9 @@
         (rop.SETARRAYITEM_GC, [b2t,ConstInt(1), c1s], None,
                               LLtypeMixin.arraydescr),
         ]
-    for x, y in zip(expected, trace):
-        assert x == y
+    with CompareableConsts():
+        for x, y in zip(expected, trace):
+            assert x == y
     #
     ptr = b2t.value._obj.container._as_ptr()
     assert lltype.typeOf(ptr) == lltype.Ptr(lltype.GcArray(lltype.Signed))
@@ -1207,8 +1227,9 @@
         (rop.SETFIELD_GC, [b2t, c1s],  None, LLtypeMixin.adescr),
         (rop.SETFIELD_GC, [b2t, b4t], None, LLtypeMixin.bdescr),
         ]
-    for x, y in zip(expected, trace):
-        assert x == y
+    with CompareableConsts():
+        for x, y in zip(expected, trace):
+            assert x == y
     #
     ptr = b2t.value._obj.container._as_ptr()
     assert lltype.typeOf(ptr) == lltype.Ptr(LLtypeMixin.S)

From noreply at buildbot.pypy.org  Wed Oct 12 21:23:09 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 12 Oct 2011 21:23:09 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: typo in the name
Message-ID: <20111012192309.2F04182112@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: extradoc
Changeset: r3926:5533b3925882
Date: 2011-10-12 21:22 +0200
http://bitbucket.org/pypy/extradoc/changeset/5533b3925882/

Log:	typo in the name

diff --git a/blog/draft/numpy-statups-update.rst b/blog/draft/numpy-status-update.rst
rename from blog/draft/numpy-statups-update.rst
rename to blog/draft/numpy-status-update.rst
diff --git a/talk/iwtc11/benchmarks/image/sobel.py b/talk/iwtc11/benchmarks/image/sobel.py
--- a/talk/iwtc11/benchmarks/image/sobel.py
+++ b/talk/iwtc11/benchmarks/image/sobel.py
@@ -44,7 +44,7 @@
              -1.0 * img[p + (-1, 1)] + 1.0 * img[p + (1, 1)]
         dy = -1.0*img[p + (-1,-1)] -2.0*img[p + (0,-1)] -1.0*img[p + (1,-1)] + \
               1.0*img[p + (-1, 1)] +2.0*img[p + (0, 1)] +1.0*img[p + (1, 1)]
-        res[p] = min(int(sqrt(dx*dx + dy*dy) / 4.0), 255)
+        res[p] = min(int(sqrt(dx*dx + dy*dy) / 2.0), 255)
     return res
 
 def main(args):

From noreply at buildbot.pypy.org  Wed Oct 12 21:37:04 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 12 Oct 2011 21:37:04 +0200 (CEST)
Subject: [pypy-commit] pypy minimark-no-mass-free: Comments.
Message-ID: <20111012193704.2051282112@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: minimark-no-mass-free
Changeset: r47971:3e89774d7da6
Date: 2011-10-12 14:20 +0200
http://bitbucket.org/pypy/pypy/changeset/3e89774d7da6/

Log:	Comments.

diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -183,7 +183,7 @@
         # that are larger are just allocated with raw_malloc().  Note that
         # the size limit for being first allocated in the nursery is much
         # larger; see below.
-        "small_request_threshold": 35*WORD,
+        "small_request_threshold": 43*WORD,
 
         # Full collection threshold: after a major collection, we record
         # the total size consumed; and after every minor collection, if the
diff --git a/pypy/rpython/memory/gc/minimarkpage.py b/pypy/rpython/memory/gc/minimarkpage.py
--- a/pypy/rpython/memory/gc/minimarkpage.py
+++ b/pypy/rpython/memory/gc/minimarkpage.py
@@ -21,20 +21,36 @@
 #        already marked.
 #
 #    reset_stop()
-#        Leaves the "resetting" state.  All objects on which still_in_use()
-#        was not called are considered as free.
+#        Leaves the "resetting" state.  All objects on which
+#        mark_still_in_use() was not called are considered as free.
 #
 #    total_memory_used
 #        A r_uint.  Not valid in the resetting state.
 #
-# This is done by keeping one bit per object, in off-object bitfields
-# (to speed up looking for a free bit and to be more fork()-friendly).
-# Initially OFF=0 and ON=1.  malloc() finds an old location which is
-# OFF, and mark it as ON.  reset_start() finds all locations still at
-# OFF and returns to the OS completely OFF arenas, and mark all
-# remaining bits as ON.  Then it switches the meaning of ON and OFF, so
-# that all bits are instantly OFF.  mark_still_in_use() marks the bit as
-# ON again.
+# This is done by keeping one bit per object in off-object bitfields (to
+# speed up looking for a free bit and to be more fork()-friendly).
+# mark_still_in_use() marks the bit as ON ("mark" phase).  Then later
+# malloc() lazily "sweeps" the bits: any location about which the bit is
+# ON is reset to OFF, and the first location about which the bit is
+# already OFF is returned as the result of malloc().  The lazy sweep is
+# completed in reset_start(), which if necessary finishes the sweep to
+# reset ON bits to OFF and to return to the OS arenas that were already
+# completely OFF.
+#
+# A "page" is 1024 WORDs, and an "arena" is 131072 WORDs, i.e. 128
+# pages.  Each page is either free or contains objects all of the same
+# size.  The sizes handled by the ArenaCollection are between WORD and
+# 42*WORD.  The mapping from an object position to a bit is done at a
+# granularity of 2*WORD bytes, so that we need 1024/2 = 512 bits = 64
+# bytes per page or 8192 bytes per arena.  We reserve the first 1 or 2
+# pages of the arena for the bits, leaving 64 or 128 bytes free at the
+# start of the arena for random fields.
+#
+# The granularity of two WORDs is ok because a single-WORD object cannot
+# possibly contain further pointers.  We choose this fixed granularity
+# instead of, more naturally, one bit per object, because computing this
+# would involve dividing the offset by the size.
+#
 # ____________________________________________________________
 #
 
@@ -50,25 +66,20 @@
 # The actual allocation occurs in whole arenas, which are then subdivided
 # into pages.  For each arena we allocate one of the following structures:
 
-def get_arena_type(num_words):
-    ARENA_PTR = lltype.Ptr(lltype.ForwardReference())
-    ARENA = lltype.Struct('ArenaReference',
-        # --- One bit per "basic unit" of size in the whole arena, set if
-        # --- this basic unit contains the start of a live object.  "Basic
-        # --- unit" is 2 WORDs by default.
-        ('usage_bits', lltype.FixedSizeArray(lltype.Unsigned, num_words)),
-        # -- The address of the arena, as returned by malloc()
-        ('base', llmemory.Address),
-        # -- The number of free and the total number of pages in the arena
-        ('nfreepages', lltype.Signed),
-        ('totalpages', lltype.Signed),
-        # -- A chained list of free pages in the arena.  Ends with NULL.
-        ('freepages', llmemory.Address),
-        # -- A linked list of arenas.  See below.
-        ('nextarena', ARENA_PTR),
-        )
-    ARENA_PTR.TO.become(ARENA)
-    return ARENA
+ARENA_PTR = lltype.Ptr(lltype.ForwardReference())
+ARENA = lltype.Struct('ArenaReference',
+    # -- The address of the arena, as returned by malloc()
+    ('base', llmemory.Address),
+    # -- The number of free and the total number of pages in the arena
+    ('nfreepages', lltype.Signed),
+    ('totalpages', lltype.Signed),
+    # -- A chained list of free pages in the arena.  Ends with NULL.
+    ('freepages', llmemory.Address),
+    # -- A linked list of arenas.  See below.
+    ('nextarena', ARENA_PTR),
+    )
+ARENA_PTR.TO.become(ARENA)
+ARENA_NULL = lltype.nullptr(ARENA)
 
 # The idea is that when we need a free page, we take it from the arena
 # which currently has the *lowest* number of free pages.  This allows

From noreply at buildbot.pypy.org  Wed Oct 12 21:37:05 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 12 Oct 2011 21:37:05 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Merge 3f26c4717b82 and
 7cd9093a53f9 from branch 'lltrace'.
Message-ID: <20111012193705.643E082112@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r47972:f1e3e612ab8a
Date: 2011-10-12 14:30 +0200
http://bitbucket.org/pypy/pypy/changeset/f1e3e612ab8a/

Log:	Merge 3f26c4717b82 and 7cd9093a53f9 from branch 'lltrace'.

diff --git a/pypy/translator/c/funcgen.py b/pypy/translator/c/funcgen.py
--- a/pypy/translator/c/funcgen.py
+++ b/pypy/translator/c/funcgen.py
@@ -477,13 +477,18 @@
             result = '/* %s */' % result
         return result
 
-    def generic_set(self, op, targetexpr):
+    def generic_set(self, op, targetexpr, gckind):
         newvalue = self.expr(op.args[-1], special_case_void=False)
         result = '%s = %s;' % (targetexpr, newvalue)
         T = self.lltypemap(op.args[-1])
         if T is Void:
             result = '/* %s */' % result
+        elif gckind == 'gc':
+            mark = FunctionCodeGenerator._TRACE_MARK + 1
+            FunctionCodeGenerator._TRACE_MARK = mark
+            result = '%s RPyTraceSet(%s, %d);' % (result, targetexpr, mark)
         return result
+    _TRACE_MARK = 0
 
     def OP_GETFIELD(self, op, ampersand=''):
         assert isinstance(op.args[1], Constant)
@@ -503,7 +508,7 @@
         expr = structdef.ptr_access_expr(self.expr(op.args[0]),
                                          op.args[1].value,
                                          baseexpr_is_const)
-        return self.generic_set(op, expr)
+        return self.generic_set(op, expr, STRUCT._gckind)
 
     def OP_GETSUBSTRUCT(self, op):
         RESULT = self.lltypemap(op.result).TO
@@ -533,7 +538,8 @@
         ptr = self.expr(op.args[0])
         index = self.expr(op.args[1])
         arraydef = self.db.gettypedefnode(ARRAY)
-        return self.generic_set(op, arraydef.itemindex_access_expr(ptr, index))
+        return self.generic_set(op, arraydef.itemindex_access_expr(ptr, index),
+                                ARRAY._gckind)
     OP_BARE_SETARRAYITEM = OP_SETARRAYITEM
 
     def OP_GETARRAYSUBSTRUCT(self, op):
@@ -575,7 +581,9 @@
         return self.generic_get(op, self.interior_expr(op.args))
 
     def OP_BARE_SETINTERIORFIELD(self, op):
-        return self.generic_set(op, self.interior_expr(op.args[:-1]))
+        STRUCT = self.lltypemap(op.args[0]).TO
+        return self.generic_set(op, self.interior_expr(op.args[:-1]),
+                                STRUCT._gckind)
 
     def OP_GETINTERIORARRAYSIZE(self, op):
         expr, ARRAY = self.interior_expr(op.args, True)
diff --git a/pypy/translator/c/genc.py b/pypy/translator/c/genc.py
--- a/pypy/translator/c/genc.py
+++ b/pypy/translator/c/genc.py
@@ -552,6 +552,7 @@
             ('linuxmemchk', '', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -DRPY_ASSERT -DLINUXMEMCHK" $(TARGET)'),
             ('llsafer', '', '$(MAKE) CFLAGS="-O2 -DRPY_LL_ASSERT" $(TARGET)'),
             ('lldebug', '', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -DRPY_ASSERT -DRPY_LL_ASSERT" $(TARGET)'),
+            ('lltrace', '', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -DRPY_ASSERT -DRPY_LL_ASSERT -DRPY_LL_TRACE" $(TARGET)'),
             ('profile', '', '$(MAKE) CFLAGS="-g -O1 -pg $(CFLAGS) -fno-omit-frame-pointer" LDFLAGS="-pg $(LDFLAGS)" $(TARGET)'),
             ]
         if self.has_profopt():
diff --git a/pypy/translator/c/src/debug_lltrace.h b/pypy/translator/c/src/debug_lltrace.h
new file mode 100644
--- /dev/null
+++ b/pypy/translator/c/src/debug_lltrace.h
@@ -0,0 +1,65 @@
+
+void _RPyTraceSet(void *addr, long newvalue, long mark);
+
+
+#ifndef RPY_LL_TRACE /****************************************/
+
+
+#  define RPY_IS_TRACING           0
+#  define RPyTraceSet(ptr, mark)   /* nothing */
+#  ifndef PYPY_NOT_MAIN_FILE
+void _RPyTraceSet(void *addr, long newvalue, long mark) { }
+#  endif
+
+
+#else /*******************************************************/
+
+
+#  define RPY_IS_TRACING           1
+#  define RPyTraceSet(ptr, mark)   _RPyTraceSet(&(ptr), (long)(ptr), mark)
+
+#  ifndef PYPY_NOT_MAIN_FILE
+
+struct _RPyTrace_s {
+  long mark;
+  void *addr;
+  long newvalue;
+};
+
+static struct _RPyTrace_s *_RPyTrace_start   = NULL;
+static struct _RPyTrace_s *_RPyTrace_stop    = NULL;
+static struct _RPyTrace_s *_RPyTrace_current = NULL;
+static const long _RPyTrace_default_size = 134217728;
+
+void _RPyTrace_WrapAround(void)
+{
+  if (_RPyTrace_start == NULL)
+    {
+      char *csize = getenv("PYPYTRACEBUF");
+      long size = csize ? atol(csize) : 0;
+      if (size <= 1)
+        size = _RPyTrace_default_size;
+      _RPyTrace_start = malloc(size * sizeof(struct _RPyTrace_s));
+      RPyAssert(_RPyTrace_start, "not enough memory to allocate the trace");
+      _RPyTrace_stop = _RPyTrace_start + size;
+    }
+  _RPyTrace_current = _RPyTrace_start;
+  fprintf(stderr, "lltrace: buffer from %p to %p, size %ld entries\n",
+          _RPyTrace_start, _RPyTrace_stop,
+          (long)(_RPyTrace_stop - _RPyTrace_start));
+}
+
+void _RPyTraceSet(void *addr, long newvalue, long mark)
+{
+  if (_RPyTrace_current == _RPyTrace_stop)
+    _RPyTrace_WrapAround();
+  _RPyTrace_current->mark = mark;
+  _RPyTrace_current->addr = addr;
+  _RPyTrace_current->newvalue = newvalue;
+  ++_RPyTrace_current;
+}
+
+#  endif
+
+
+#endif /******************************************************/
diff --git a/pypy/translator/c/src/g_include.h b/pypy/translator/c/src/g_include.h
--- a/pypy/translator/c/src/g_include.h
+++ b/pypy/translator/c/src/g_include.h
@@ -46,6 +46,7 @@
 #  include "src/rtyper.h"
 #  include "src/debug_traceback.h"
 #  include "src/debug_alloc.h"
+#  include "src/debug_lltrace.h"
 #  include "src/ll_os.h"
 #  include "src/ll_strtod.h"
 #endif

From noreply at buildbot.pypy.org  Wed Oct 12 21:37:06 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 12 Oct 2011 21:37:06 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Trace all set{field,
 arrayitem, interiorfield}, even to non-gc objects.
Message-ID: <20111012193706.9AB7082112@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r47973:787da47ce792
Date: 2011-10-12 14:35 +0200
http://bitbucket.org/pypy/pypy/changeset/787da47ce792/

Log:	Trace all set{field,arrayitem,interiorfield}, even to non-gc
	objects.

diff --git a/pypy/translator/c/funcgen.py b/pypy/translator/c/funcgen.py
--- a/pypy/translator/c/funcgen.py
+++ b/pypy/translator/c/funcgen.py
@@ -477,13 +477,13 @@
             result = '/* %s */' % result
         return result
 
-    def generic_set(self, op, targetexpr, gckind):
+    def generic_set(self, op, targetexpr):
         newvalue = self.expr(op.args[-1], special_case_void=False)
         result = '%s = %s;' % (targetexpr, newvalue)
         T = self.lltypemap(op.args[-1])
         if T is Void:
             result = '/* %s */' % result
-        elif gckind == 'gc':
+        else:
             mark = FunctionCodeGenerator._TRACE_MARK + 1
             FunctionCodeGenerator._TRACE_MARK = mark
             result = '%s RPyTraceSet(%s, %d);' % (result, targetexpr, mark)
@@ -508,7 +508,7 @@
         expr = structdef.ptr_access_expr(self.expr(op.args[0]),
                                          op.args[1].value,
                                          baseexpr_is_const)
-        return self.generic_set(op, expr, STRUCT._gckind)
+        return self.generic_set(op, expr)
 
     def OP_GETSUBSTRUCT(self, op):
         RESULT = self.lltypemap(op.result).TO
@@ -538,8 +538,7 @@
         ptr = self.expr(op.args[0])
         index = self.expr(op.args[1])
         arraydef = self.db.gettypedefnode(ARRAY)
-        return self.generic_set(op, arraydef.itemindex_access_expr(ptr, index),
-                                ARRAY._gckind)
+        return self.generic_set(op, arraydef.itemindex_access_expr(ptr, index))
     OP_BARE_SETARRAYITEM = OP_SETARRAYITEM
 
     def OP_GETARRAYSUBSTRUCT(self, op):
@@ -582,8 +581,7 @@
 
     def OP_BARE_SETINTERIORFIELD(self, op):
         STRUCT = self.lltypemap(op.args[0]).TO
-        return self.generic_set(op, self.interior_expr(op.args[:-1]),
-                                STRUCT._gckind)
+        return self.generic_set(op, self.interior_expr(op.args[:-1]))
 
     def OP_GETINTERIORARRAYSIZE(self, op):
         expr, ARRAY = self.interior_expr(op.args, True)

From noreply at buildbot.pypy.org  Wed Oct 12 21:37:07 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 12 Oct 2011 21:37:07 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Improve the test coverage
 by running the same tests several times in
Message-ID: <20111012193707.D11C082112@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r47974:6201872a4e62
Date: 2011-10-12 15:28 +0200
http://bitbucket.org/pypy/pypy/changeset/6201872a4e62/

Log:	Improve the test coverage by running the same tests several times in
	the same process. This is more likely to catch subtle bugs in the
	GC.

diff --git a/pypy/translator/c/test/test_newgc.py b/pypy/translator/c/test/test_newgc.py
--- a/pypy/translator/c/test/test_newgc.py
+++ b/pypy/translator/c/test/test_newgc.py
@@ -24,18 +24,34 @@
 
     _isolated_func = None
     c_allfuncs = None
+    repetitions = 3
 
     @classmethod
     def _makefunc_str_int(cls, f):
+        repetitions = cls.repetitions
         def main(argv):
             arg0 = argv[1]
-            arg1 = int(argv[2])
+            if len(argv) > 2:
+                arg1 = int(argv[2])
+            else:
+                arg1 = -1
             try:
                 res = f(arg0, arg1)
             except MemoryError:
                 print "MEMORY-ERROR"
             else:
                 print res
+                if len(argv) > 3:
+                    i = int(argv[3])
+                else:
+                    i = repetitions
+                while i > 1:
+                    res2 = f(arg0, arg1)
+                    if res != res2:
+                        print 'GOT A DIFFERENT RESULT:'
+                        print '\told result: "%s"' % res
+                        print '\tnew result: "%s"' % res2
+                    i -= 1
             return 0
         
         t = Translation(main, standalone=True, gc=cls.gcpolicy,
@@ -49,8 +65,9 @@
             t.viewcg()
         exename = t.compile()
 
-        def run(s, i):
-            data = py.process.cmdexec("%s %s %d" % (exename, s, i))
+        def run(s, i, repetitions=None):
+            data = py.process.cmdexec("%s %s %d %s" % (exename, s, i,
+                                                       repetitions or ""))
             data = data.strip()
             if data == 'MEMORY-ERROR':
                 raise MemoryError
@@ -114,11 +131,11 @@
             cls.c_allfuncs.close_isolate()
             cls.c_allfuncs = None
 
-    def run(self, name, *args):
+    def run(self, name, *args, **kwds):
         if not args:
             args = (-1, )
         print 'Running %r)' % name
-        res = self.c_allfuncs(name, *args)
+        res = self.c_allfuncs(name, *args, **kwds)
         num = self.name_to_func[name]
         if self.funcsstr[num]:
             return res
@@ -375,7 +392,6 @@
             pass
         b = B()
         b.nextid = 0
-        b.num_deleted = 0
         class A(object):
             def __init__(self):
                 self.id = b.nextid
@@ -383,6 +399,7 @@
             def __del__(self):
                 b.num_deleted += 1
         def f():
+            b.num_deleted = 0
             a = A()
             i = 0
             while i < 5:
@@ -609,6 +626,8 @@
         # all objects returned by the GC should be properly aligned.
         from pypy.rpython.lltypesystem import rffi
         mylist = ['a', 'bc', '84139871', 'ajkdh', '876']
+        from pypy.rpython.tool import rffi_platform
+        expected_alignment = rffi_platform.memory_alignment()
         def f():
             result = 0
             buffer = ""
@@ -617,15 +636,13 @@
                     buffer += s
                     addr = rffi.cast(lltype.Signed, buffer)
                     result |= addr
-            return result
+            return result & (expected_alignment-1)
 
         return f
 
     def test_object_alignment(self):
         res = self.run('object_alignment')
-        from pypy.rpython.tool import rffi_platform
-        expected_alignment = rffi_platform.memory_alignment()
-        assert (res & (expected_alignment-1)) == 0
+        assert res == 0
 
     def define_void_list(cls):
         class E:
@@ -883,13 +900,14 @@
         s = lltype.malloc(S, 3, zero=True)
         h_s = lltype.identityhash(s)
         def f():
-            return lltype.identityhash(s) - h_s    # != 0 (so far),
+            got = lltype.identityhash(s) - h_s    # != 0 (so far),
                                 # because S is a varsized structure.
+            return (got != 0)
         return f
 
     def test_hash_varsized(self):
         res = self.run('hash_varsized')
-        assert res != 0
+        assert res == 1
 
 
     def define_arraycopy_writebarrier_int(cls):
@@ -1247,7 +1265,7 @@
         return fn
 
     def test_gc_set_max_heap_size(self):
-        res = self.run('gc_set_max_heap_size')
+        res = self.run('gc_set_max_heap_size', repetitions=1)
         assert res == 2
 
     def define_gc_heap_stats(cls):
@@ -1334,12 +1352,15 @@
             for i in range(len(objects)):
                 assert compute_identity_hash(objects[i]) == hashes[i]
                 unique[hashes[i]] = None
-            return len(unique)
+            if len(unique) >= 195:
+                return 9999       # ok
+            else:
+                return len(unique)
         return fn
 
     def test_nursery_hash_base(self):
         res = self.run('nursery_hash_base')
-        assert res >= 195
+        assert res == 9999
 
 
 class TestGenerationalGC(TestSemiSpaceGC):
@@ -1518,3 +1539,4 @@
 
 class TestMostlyConcurrentMarkSweepGC(TestUsingFramework):
     gcpolicy = "concurrentms"
+    repetitions = 10

From noreply at buildbot.pypy.org  Wed Oct 12 21:37:09 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 12 Oct 2011 21:37:09 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Fix a subtle bug.
Message-ID: <20111012193709.1224D82112@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r47975:a38dcb8b3c18
Date: 2011-10-12 15:28 +0200
http://bitbucket.org/pypy/pypy/changeset/a38dcb8b3c18/

Log:	Fix a subtle bug.

diff --git a/pypy/rpython/memory/gc/concurrentms.py b/pypy/rpython/memory/gc/concurrentms.py
--- a/pypy/rpython/memory/gc/concurrentms.py
+++ b/pypy/rpython/memory/gc/concurrentms.py
@@ -567,7 +567,10 @@
             #
             debug_stop("gc-stop")
             #
-            self.execute_finalizers_ll()
+            # We must *not* run execute_finalizers_ll() here, because it
+            # can start the next collection, and then this function returns
+            # with a collection in progress, which it should not.  Be careful
+            # to call execute_finalizers_ll() in the caller somewhere.
 
 
     def execute_finalizers_ll(self):
@@ -608,6 +611,7 @@
             self.trigger_next_collection()
             if gen >= 2:
                 self.wait_for_the_end_of_collection()
+        self.execute_finalizers_ll()
 
     def trigger_next_collection(self):
         """In the mutator thread: triggers the next collection."""
@@ -662,6 +666,8 @@
         self.release(self.ready_to_start_lock)
         #
         debug_stop("gc-start")
+        #
+        self.execute_finalizers_ll()
 
     def _add_stack_root(self, root):
         obj = root.address[0]

From noreply at buildbot.pypy.org  Wed Oct 12 21:37:10 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 12 Oct 2011 21:37:10 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Bah. A few hours of
 gdb'ing a pypy-c turned out this bug, that
Message-ID: <20111012193710.49FA582112@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r47976:89b12cccb3b1
Date: 2011-10-12 19:09 +0200
http://bitbucket.org/pypy/pypy/changeset/89b12cccb3b1/

Log:	Bah. A few hours of gdb'ing a pypy-c turned out this bug, that I
	explain in a one-page comment.

diff --git a/pypy/rpython/memory/gc/concurrentms.py b/pypy/rpython/memory/gc/concurrentms.py
--- a/pypy/rpython/memory/gc/concurrentms.py
+++ b/pypy/rpython/memory/gc/concurrentms.py
@@ -432,7 +432,9 @@
     #    XXX
 
     #def shrink_array(self, obj, smallerlength):
-    #    return False
+    #    no real point in supporting this, but if you think it's a good
+    #    idea, remember that changing the array length at run-time needs
+    #    extra care for the collector thread
 
     def enumerate_all_roots(self, callback, arg):
         self.prebuilt_root_objects.foreach(callback, arg)
@@ -471,6 +473,7 @@
     grow_reservation._always_inline_ = True
 
     def deletion_barrier(self, addr_struct):
+        # XXX check the assembler
         mark = self.header(addr_struct).tid & 0xFF
         if mark != self.current_mark:
             self.force_scan(addr_struct)
@@ -506,8 +509,16 @@
                     ll_assert(self.collection_running == 1,
                               "write barrier: wrong call?")
                     #
+                    # It's fine to set the mark before tracing, because
+                    # we are anyway in a 'mutex_lock' critical section.
+                    # The collector thread will not exist the phase
+                    # 'collection_running == 1' here.
                     self.set_mark(obj, self.current_mark)
                     self.trace(obj, self._barrier_add_extra, None)
+                    #
+                    # Still at 1:
+                    ll_assert(self.collection_running == 1,
+                              "write barrier: oups!?")
                 #
             self.release(self.mutex_lock)
             #debug_print("deletion_barrier done ", obj)
@@ -516,7 +527,9 @@
         self.force_scan = force_scan
 
     def _barrier_add_extra(self, root, ignored):
-        self.extra_objects_to_mark.append(root.address[0])
+        obj = root.address[0]
+        self.get_mark(obj)
+        self.extra_objects_to_mark.append(obj)
 
 
     def wait_for_the_end_of_collection(self):
@@ -571,6 +584,8 @@
             # can start the next collection, and then this function returns
             # with a collection in progress, which it should not.  Be careful
             # to call execute_finalizers_ll() in the caller somewhere.
+            ll_assert(self.collection_running == 0,
+                      "collector thread not paused?")
 
 
     def execute_finalizers_ll(self):
@@ -637,6 +652,7 @@
             x = llarena.getfakearenaaddress(x) + 8
             obj = x + self.gcheaderbuilder.size_gc_header
             #debug_print("_objects_with_finalizers_to_run", obj)
+            self.get_mark(obj)
             self.gray_objects.append(obj)
             p = list_next(p)
         #
@@ -672,10 +688,12 @@
     def _add_stack_root(self, root):
         obj = root.address[0]
         #debug_print("_add_stack_root", obj)
+        self.get_mark(obj)
         self.gray_objects.append(obj)
 
     def _add_prebuilt_root(self, obj, ignored):
         #debug_print("_add_prebuilt_root", obj)
+        self.get_mark(obj)
         self.gray_objects.append(obj)
 
     def debug_check_lists(self):
@@ -780,14 +798,17 @@
         return mark ^ (MARK_VALUE_1 ^ MARK_VALUE_2)
 
     def is_marked(self, obj, current_mark):
-        mark = self.header(obj).tid & 0xFF
-        ll_assert(mark in (MARK_VALUE_1, MARK_VALUE_2, MARK_VALUE_STATIC),
-                  "bad mark byte in object")
-        return mark == current_mark
+        return self.get_mark(obj) == current_mark
 
     def set_mark(self, obj, newmark):
         _set_mark(self.header(obj), newmark)
 
+    def get_mark(self, obj):
+        mark = self.header(obj).tid & 0xFF
+        ll_assert(mark == MARK_VALUE_1 or mark == MARK_VALUE_2 or
+                  mark == MARK_VALUE_STATIC, "bad mark byte in object")
+        return mark
+
     def collector_mark(self):
         while True:
             #
@@ -804,6 +825,7 @@
             #debug_print("...collector thread has mutex_lock")
             while self.extra_objects_to_mark.non_empty():
                 obj = self.extra_objects_to_mark.pop()
+                self.get_mark(obj)
                 self.gray_objects.append(obj)
             self.release(self.mutex_lock)
             #
@@ -821,8 +843,30 @@
         while self.gray_objects.non_empty():
             obj = self.gray_objects.pop()
             if not self.is_marked(obj, current_mark):
+                #
+                # Scan the content of 'obj'.  We use a snapshot-at-the-
+                # beginning order, meaning that we want to scan the state
+                # of the object as it was at the beginning of the current
+                # collection --- and not the current state, which might have
+                # been modified.  That's why we have a deletion barrier:
+                # when the mutator thread is about to change an object that
+                # is not yet marked, it will itself do the scanning of just
+                # this object, and mark the object.  But this function is not
+                # synchronized, which means that in some rare cases it's
+                # possible that the object is scanned a second time here
+                # (harmlessly).
+                #
+                # The order of the next two lines is essential!  *First*
+                # scan the object, adding all objects found to gray_objects;
+                # and *only then* set the mark.  This is essential, because
+                # otherwise, we might set the mark, then the main thread
+                # thinks a force_scan() is not necessary and modifies the
+                # content of 'obj', and then here in the collector thread
+                # we scan a modified content --- and the original content
+                # is never scanned.
+                #
+                self.trace(obj, self._collect_add_pending, None)
                 self.set_mark(obj, current_mark)
-                self.trace(obj, self._collect_add_pending, None)
                 #
                 # Interrupt early if the mutator's write barrier adds stuff
                 # to that list.  Note that the check is imprecise because
@@ -834,7 +878,9 @@
                     return
 
     def _collect_add_pending(self, root, ignored):
-        self.gray_objects.append(root.address[0])
+        obj = root.address[0]
+        self.get_mark(obj)
+        self.gray_objects.append(obj)
 
     def collector_sweep(self):
         n = 1
@@ -969,6 +1015,7 @@
                 if self.collect_run_finalizers_tail == self.NULL:
                     self.collect_run_finalizers_tail = finalizer_page
                 obj = x + size_gc_header
+                self.get_mark(obj)
                 self.gray_objects.append(obj)
             else:
                 # marked: relink into the collect_finalizer_pages list
diff --git a/pypy/translator/c/src/debug_lltrace.h b/pypy/translator/c/src/debug_lltrace.h
--- a/pypy/translator/c/src/debug_lltrace.h
+++ b/pypy/translator/c/src/debug_lltrace.h
@@ -14,6 +14,8 @@
 
 #else /*******************************************************/
 
+#  include "src/atomic_ops.h"
+
 
 #  define RPY_IS_TRACING           1
 #  define RPyTraceSet(ptr, mark)   _RPyTraceSet(&(ptr), (long)(ptr), mark)
@@ -28,35 +30,55 @@
 
 static struct _RPyTrace_s *_RPyTrace_start   = NULL;
 static struct _RPyTrace_s *_RPyTrace_stop    = NULL;
-static struct _RPyTrace_s *_RPyTrace_current = NULL;
-static const long _RPyTrace_default_size = 134217728;
+static struct _RPyTrace_s * volatile _RPyTrace_current = NULL;
+static const long _RPyTrace_default_size = 0x4000000;
 
-void _RPyTrace_WrapAround(void)
+void _RPyTrace_Setup(void)
 {
-  if (_RPyTrace_start == NULL)
-    {
-      char *csize = getenv("PYPYTRACEBUF");
-      long size = csize ? atol(csize) : 0;
-      if (size <= 1)
+    /* not thread-safe, but assume that it's called very early anyway,
+       when there is only one thread */
+    char *csize = getenv("PYPYTRACEBUF");
+    long size = csize ? atol(csize) : 0;
+    if (size <= 1)
         size = _RPyTrace_default_size;
-      _RPyTrace_start = malloc(size * sizeof(struct _RPyTrace_s));
-      RPyAssert(_RPyTrace_start, "not enough memory to allocate the trace");
-      _RPyTrace_stop = _RPyTrace_start + size;
-    }
-  _RPyTrace_current = _RPyTrace_start;
-  fprintf(stderr, "lltrace: buffer from %p to %p, size %ld entries\n",
-          _RPyTrace_start, _RPyTrace_stop,
-          (long)(_RPyTrace_stop - _RPyTrace_start));
+    _RPyTrace_start = calloc(size, sizeof(struct _RPyTrace_s));
+    RPyAssert(_RPyTrace_start, "not enough memory to allocate the trace");
+    _RPyTrace_stop = _RPyTrace_start + size;
+
+    _RPyTrace_current = _RPyTrace_start;
+    fprintf(stderr, "lltrace: buffer from %p to %p, size %ld entries\n",
+            _RPyTrace_start, _RPyTrace_stop,
+            (long)(_RPyTrace_stop - _RPyTrace_start));
 }
 
 void _RPyTraceSet(void *addr, long newvalue, long mark)
 {
-  if (_RPyTrace_current == _RPyTrace_stop)
-    _RPyTrace_WrapAround();
-  _RPyTrace_current->mark = mark;
-  _RPyTrace_current->addr = addr;
-  _RPyTrace_current->newvalue = newvalue;
-  ++_RPyTrace_current;
+    struct _RPyTrace_s *current, *next;
+    do {
+        current = _RPyTrace_current;
+        if (current == NULL) {
+            _RPyTrace_Setup();
+            current = _RPyTrace_current;
+        }
+        next = current + 1;
+        if (next == _RPyTrace_stop) next = _RPyTrace_start;
+    } while (!bool_cas((volatile unsigned long*)&_RPyTrace_current,
+                       (unsigned long)current,
+                       (unsigned long)next));
+    current->mark = mark;
+    current->addr = addr;
+    current->newvalue = newvalue;
+}
+
+void _RPyTraceDump(void)
+{
+    char *start   = _RPyTrace_start;
+    char *stop    = _RPyTrace_stop;
+    char *current = (char *)_RPyTrace_current;
+    FILE *f       = fopen("trace.dump", "wb");
+    fwrite(current, 1, stop - current, f);
+    fwrite(start,   1, current - start, f);
+    fclose(f);
 }
 
 #  endif
diff --git a/pypy/translator/c/test/test_newgc.py b/pypy/translator/c/test/test_newgc.py
--- a/pypy/translator/c/test/test_newgc.py
+++ b/pypy/translator/c/test/test_newgc.py
@@ -597,7 +597,8 @@
             p = lltype.malloc(B)
             llop.gc__collect(lltype.Void)
             p.a = lltype.malloc(A)
-            return p.a.x
+            n = p.a.x
+            return n + (~n)    # always -1
         return f
 
     def test_framework_late_filling_pointers(self):
@@ -1300,7 +1301,7 @@
         return f
 
     def test_gc_heap_stats(self):
-        res = self.run("gc_heap_stats")
+        res = self.run("gc_heap_stats", repetitions=1)
         assert res == 3011 or res == 3021
 
     def definestr_string_builder(cls):
@@ -1539,4 +1540,4 @@
 
 class TestMostlyConcurrentMarkSweepGC(TestUsingFramework):
     gcpolicy = "concurrentms"
-    repetitions = 10
+    repetitions = 100

From noreply at buildbot.pypy.org  Wed Oct 12 21:37:11 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 12 Oct 2011 21:37:11 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Comment. A test in
 test_newgc sometimes fails, because of this.
Message-ID: <20111012193711.7F09682112@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r47977:7e06efc4ec2c
Date: 2011-10-12 19:56 +0200
http://bitbucket.org/pypy/pypy/changeset/7e06efc4ec2c/

Log:	Comment. A test in test_newgc sometimes fails, because of this.

diff --git a/pypy/rpython/memory/gc/concurrentms.py b/pypy/rpython/memory/gc/concurrentms.py
--- a/pypy/rpython/memory/gc/concurrentms.py
+++ b/pypy/rpython/memory/gc/concurrentms.py
@@ -966,6 +966,10 @@
                     # the location contains really an object, which is marked.
                     # check the typeid to see if it's a weakref.  XXX could
                     # be faster
+                    # XXX actually it's very wrong: if we read a weakref out
+                    # of an object during collection_running==1, or during
+                    # collection_running==2 but before we reach that point,
+                    # the target object will never be marked
                     tid = hdr.tid
                     type_id = llop.extract_high_ushort(llgroup.HALFWORD, tid)
                     wroffset = self.weakpointer_offset(type_id)

From noreply at buildbot.pypy.org  Wed Oct 12 22:23:15 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 12 Oct 2011 22:23:15 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Remove print statement
Message-ID: <20111012202315.3C2B982112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r47978:36b998dd9966
Date: 2011-10-12 01:52 +0200
http://bitbucket.org/pypy/pypy/changeset/36b998dd9966/

Log:	Remove print statement

diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -514,49 +514,6 @@
         self.value.sync_app_attrs(space)
 
 
-class Print(stmt):
-
-    _lineno_mask = 8
-    _col_offset_mask = 16
-
-    def __init__(self, dest, values, nl, lineno, col_offset):
-        self.dest = dest
-        self.values = values
-        self.w_values = None
-        self.nl = nl
-        stmt.__init__(self, lineno, col_offset)
-        self.initialization_state = 31
-
-    def walkabout(self, visitor):
-        visitor.visit_Print(self)
-
-    def mutate_over(self, visitor):
-        if self.dest:
-            self.dest = self.dest.mutate_over(visitor)
-        if self.values:
-            visitor._mutate_sequence(self.values)
-        return visitor.visit_Print(self)
-
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~1) ^ 30:
-            missing_field(space, self.initialization_state, [None, 'values', 'nl', 'lineno', 'col_offset'], 'Print')
-        else:
-            if not self.initialization_state & 1:
-                self.dest = None
-        if self.dest:
-            self.dest.sync_app_attrs(space)
-        w_list = self.w_values
-        if w_list is not None:
-            list_w = space.listview(w_list)
-            if list_w:
-                self.values = [space.interp_w(expr, w_obj) for w_obj in list_w]
-            else:
-                self.values = None
-        if self.values is not None:
-            for node in self.values:
-                node.sync_app_attrs(space)
-
-
 class For(stmt):
 
     _lineno_mask = 16
@@ -1028,44 +985,6 @@
                 node.sync_app_attrs(space)
 
 
-class Exec(stmt):
-
-    _lineno_mask = 8
-    _col_offset_mask = 16
-
-    def __init__(self, body, globals, locals, lineno, col_offset):
-        self.body = body
-        self.globals = globals
-        self.locals = locals
-        stmt.__init__(self, lineno, col_offset)
-        self.initialization_state = 31
-
-    def walkabout(self, visitor):
-        visitor.visit_Exec(self)
-
-    def mutate_over(self, visitor):
-        self.body = self.body.mutate_over(visitor)
-        if self.globals:
-            self.globals = self.globals.mutate_over(visitor)
-        if self.locals:
-            self.locals = self.locals.mutate_over(visitor)
-        return visitor.visit_Exec(self)
-
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~6) ^ 25:
-            missing_field(space, self.initialization_state, ['body', None, None, 'lineno', 'col_offset'], 'Exec')
-        else:
-            if not self.initialization_state & 2:
-                self.globals = None
-            if not self.initialization_state & 4:
-                self.locals = None
-        self.body.sync_app_attrs(space)
-        if self.globals:
-            self.globals.sync_app_attrs(space)
-        if self.locals:
-            self.locals.sync_app_attrs(space)
-
-
 class Global(stmt):
 
     _lineno_mask = 2
@@ -2599,8 +2518,6 @@
         return self.default_visitor(node)
     def visit_AugAssign(self, node):
         return self.default_visitor(node)
-    def visit_Print(self, node):
-        return self.default_visitor(node)
     def visit_For(self, node):
         return self.default_visitor(node)
     def visit_While(self, node):
@@ -2621,8 +2538,6 @@
         return self.default_visitor(node)
     def visit_ImportFrom(self, node):
         return self.default_visitor(node)
-    def visit_Exec(self, node):
-        return self.default_visitor(node)
     def visit_Global(self, node):
         return self.default_visitor(node)
     def visit_Expr(self, node):
@@ -2742,11 +2657,6 @@
         node.target.walkabout(self)
         node.value.walkabout(self)
 
-    def visit_Print(self, node):
-        if node.dest:
-            node.dest.walkabout(self)
-        self.visit_sequence(node.values)
-
     def visit_For(self, node):
         node.target.walkabout(self)
         node.iter.walkabout(self)
@@ -2797,13 +2707,6 @@
     def visit_ImportFrom(self, node):
         self.visit_sequence(node.names)
 
-    def visit_Exec(self, node):
-        node.body.walkabout(self)
-        if node.globals:
-            node.globals.walkabout(self)
-        if node.locals:
-            node.locals.walkabout(self)
-
     def visit_Global(self, node):
         pass
 
@@ -3712,95 +3615,6 @@
     __init__=interp2app(AugAssign_init),
 )
 
-def Print_get_dest(space, w_self):
-    if w_self.w_dict is not None:
-        w_obj = w_self.getdictvalue(space, 'dest')
-        if w_obj is not None:
-            return w_obj
-    if not w_self.initialization_state & 1:
-        typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'dest'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
-    return space.wrap(w_self.dest)
-
-def Print_set_dest(space, w_self, w_new_value):
-    try:
-        w_self.dest = space.interp_w(expr, w_new_value, True)
-    except OperationError, e:
-        if not e.match(space, space.w_TypeError):
-            raise
-        w_self.setdictvalue(space, 'dest', w_new_value)
-        return
-    w_self.deldictvalue(space, 'dest')
-    w_self.initialization_state |= 1
-
-def Print_get_values(space, w_self):
-    if not w_self.initialization_state & 2:
-        typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'values'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
-    if w_self.w_values is None:
-        if w_self.values is None:
-            w_list = space.newlist([])
-        else:
-            list_w = [space.wrap(node) for node in w_self.values]
-            w_list = space.newlist(list_w)
-        w_self.w_values = w_list
-    return w_self.w_values
-
-def Print_set_values(space, w_self, w_new_value):
-    w_self.w_values = w_new_value
-    w_self.initialization_state |= 2
-
-def Print_get_nl(space, w_self):
-    if w_self.w_dict is not None:
-        w_obj = w_self.getdictvalue(space, 'nl')
-        if w_obj is not None:
-            return w_obj
-    if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'nl'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
-    return space.wrap(w_self.nl)
-
-def Print_set_nl(space, w_self, w_new_value):
-    try:
-        w_self.nl = space.bool_w(w_new_value)
-    except OperationError, e:
-        if not e.match(space, space.w_TypeError):
-            raise
-        w_self.setdictvalue(space, 'nl', w_new_value)
-        return
-    w_self.deldictvalue(space, 'nl')
-    w_self.initialization_state |= 4
-
-_Print_field_unroller = unrolling_iterable(['dest', 'values', 'nl'])
-def Print_init(space, w_self, __args__):
-    w_self = space.descr_self_interp_w(Print, w_self)
-    w_self.w_values = None
-    args_w, kwargs_w = __args__.unpack()
-    if args_w:
-        if len(args_w) != 3:
-            w_err = space.wrap("Print constructor takes either 0 or 3 positional arguments")
-            raise OperationError(space.w_TypeError, w_err)
-        i = 0
-        for field in _Print_field_unroller:
-            space.setattr(w_self, space.wrap(field), args_w[i])
-            i += 1
-    for field, w_value in kwargs_w.iteritems():
-        space.setattr(w_self, space.wrap(field), w_value)
-
-Print.typedef = typedef.TypeDef("Print",
-    stmt.typedef,
-    __module__='_ast',
-    _fields=_FieldsWrapper(['dest', 'values', 'nl']),
-    dest=typedef.GetSetProperty(Print_get_dest, Print_set_dest, cls=Print),
-    values=typedef.GetSetProperty(Print_get_values, Print_set_values, cls=Print),
-    nl=typedef.GetSetProperty(Print_get_nl, Print_set_nl, cls=Print),
-    __new__=interp2app(get_AST_new(Print)),
-    __init__=interp2app(Print_init),
-)
-
 def For_get_target(space, w_self):
     if w_self.w_dict is not None:
         w_obj = w_self.getdictvalue(space, 'target')
@@ -4613,98 +4427,6 @@
     __init__=interp2app(ImportFrom_init),
 )
 
-def Exec_get_body(space, w_self):
-    if w_self.w_dict is not None:
-        w_obj = w_self.getdictvalue(space, 'body')
-        if w_obj is not None:
-            return w_obj
-    if not w_self.initialization_state & 1:
-        typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
-    return space.wrap(w_self.body)
-
-def Exec_set_body(space, w_self, w_new_value):
-    try:
-        w_self.body = space.interp_w(expr, w_new_value, False)
-    except OperationError, e:
-        if not e.match(space, space.w_TypeError):
-            raise
-        w_self.setdictvalue(space, 'body', w_new_value)
-        return
-    w_self.deldictvalue(space, 'body')
-    w_self.initialization_state |= 1
-
-def Exec_get_globals(space, w_self):
-    if w_self.w_dict is not None:
-        w_obj = w_self.getdictvalue(space, 'globals')
-        if w_obj is not None:
-            return w_obj
-    if not w_self.initialization_state & 2:
-        typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'globals'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
-    return space.wrap(w_self.globals)
-
-def Exec_set_globals(space, w_self, w_new_value):
-    try:
-        w_self.globals = space.interp_w(expr, w_new_value, True)
-    except OperationError, e:
-        if not e.match(space, space.w_TypeError):
-            raise
-        w_self.setdictvalue(space, 'globals', w_new_value)
-        return
-    w_self.deldictvalue(space, 'globals')
-    w_self.initialization_state |= 2
-
-def Exec_get_locals(space, w_self):
-    if w_self.w_dict is not None:
-        w_obj = w_self.getdictvalue(space, 'locals')
-        if w_obj is not None:
-            return w_obj
-    if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'locals'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
-    return space.wrap(w_self.locals)
-
-def Exec_set_locals(space, w_self, w_new_value):
-    try:
-        w_self.locals = space.interp_w(expr, w_new_value, True)
-    except OperationError, e:
-        if not e.match(space, space.w_TypeError):
-            raise
-        w_self.setdictvalue(space, 'locals', w_new_value)
-        return
-    w_self.deldictvalue(space, 'locals')
-    w_self.initialization_state |= 4
-
-_Exec_field_unroller = unrolling_iterable(['body', 'globals', 'locals'])
-def Exec_init(space, w_self, __args__):
-    w_self = space.descr_self_interp_w(Exec, w_self)
-    args_w, kwargs_w = __args__.unpack()
-    if args_w:
-        if len(args_w) != 3:
-            w_err = space.wrap("Exec constructor takes either 0 or 3 positional arguments")
-            raise OperationError(space.w_TypeError, w_err)
-        i = 0
-        for field in _Exec_field_unroller:
-            space.setattr(w_self, space.wrap(field), args_w[i])
-            i += 1
-    for field, w_value in kwargs_w.iteritems():
-        space.setattr(w_self, space.wrap(field), w_value)
-
-Exec.typedef = typedef.TypeDef("Exec",
-    stmt.typedef,
-    __module__='_ast',
-    _fields=_FieldsWrapper(['body', 'globals', 'locals']),
-    body=typedef.GetSetProperty(Exec_get_body, Exec_set_body, cls=Exec),
-    globals=typedef.GetSetProperty(Exec_get_globals, Exec_set_globals, cls=Exec),
-    locals=typedef.GetSetProperty(Exec_get_locals, Exec_set_locals, cls=Exec),
-    __new__=interp2app(get_AST_new(Exec)),
-    __init__=interp2app(Exec_init),
-)
-
 def Global_get_names(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -127,23 +127,6 @@
         except misc.ForbiddenNameAssignment, e:
             self.error_ast("cannot assign to %s" % (e.name,), e.node)
 
-    def handle_print_stmt(self, print_node):
-        dest = None
-        expressions = None
-        newline = True
-        start = 1
-        child_count = len(print_node.children)
-        if child_count > 2 and print_node.children[1].type == tokens.RIGHTSHIFT:
-            dest = self.handle_expr(print_node.children[2])
-            start = 4
-        if (child_count + 1 - start) // 2:
-            expressions = [self.handle_expr(print_node.children[i])
-                           for i in range(start, child_count, 2)]
-        if print_node.children[-1].type == tokens.COMMA:
-            newline = False
-        return ast.Print(dest, expressions, newline, print_node.lineno,
-                         print_node.column)
-
     def handle_del_stmt(self, del_node):
         targets = self.handle_exprlist(del_node.children[1], ast.Del)
         return ast.Delete(targets, del_node.lineno, del_node.column)
@@ -621,8 +604,6 @@
             stmt_type = stmt.type
             if stmt_type == syms.expr_stmt:
                 return self.handle_expr_stmt(stmt)
-            elif stmt_type == syms.print_stmt:
-                return self.handle_print_stmt(stmt)
             elif stmt_type == syms.del_stmt:
                 return self.handle_del_stmt(stmt)
             elif stmt_type == syms.pass_stmt:
diff --git a/pypy/interpreter/astcompiler/tools/Python.asdl b/pypy/interpreter/astcompiler/tools/Python.asdl
--- a/pypy/interpreter/astcompiler/tools/Python.asdl
+++ b/pypy/interpreter/astcompiler/tools/Python.asdl
@@ -24,9 +24,6 @@
 	      | Assign(expr* targets, expr value)
 	      | AugAssign(expr target, operator op, expr value)
 
-	      -- not sure if bool is allowed, can always use int
- 	      | Print(expr? dest, expr* values, bool nl)
-
 	      -- use 'orelse' because else is a keyword in target languages
 	      | For(expr target, expr iter, stmt* body, stmt* orelse)
 	      | While(expr test, stmt* body, stmt* orelse)
diff --git a/pypy/interpreter/pyparser/data/Grammar3.2 b/pypy/interpreter/pyparser/data/Grammar3.2
--- a/pypy/interpreter/pyparser/data/Grammar3.2
+++ b/pypy/interpreter/pyparser/data/Grammar3.2
@@ -32,15 +32,13 @@
 
 stmt: simple_stmt | compound_stmt
 simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
-small_stmt: (expr_stmt | print_stmt  | del_stmt | pass_stmt | flow_stmt |
+small_stmt: (expr_stmt | del_stmt | pass_stmt | flow_stmt |
              import_stmt | global_stmt | assert_stmt)
 expr_stmt: testlist (augassign (yield_expr|testlist) |
                      ('=' (yield_expr|testlist))*)
 augassign: ('+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' |
             '<<=' | '>>=' | '**=' | '//=')
 # For normal assignments, additional restrictions enforced by the interpreter
-print_stmt: 'print' ( [ test (',' test)* [','] ] |
-                      '>>' test [ (',' test)+ [','] ] )
 del_stmt: 'del' exprlist
 pass_stmt: 'pass'
 flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt
diff --git a/pypy/interpreter/pyparser/pygram.py b/pypy/interpreter/pyparser/pygram.py
--- a/pypy/interpreter/pyparser/pygram.py
+++ b/pypy/interpreter/pyparser/pygram.py
@@ -19,9 +19,6 @@
 
 
 python_grammar = _get_python_grammar()
-python_grammar_no_print = python_grammar.shared_copy()
-python_grammar_no_print.keyword_ids = python_grammar_no_print.keyword_ids.copy()
-del python_grammar_no_print.keyword_ids["print"]
 
 class _Tokens(object):
     pass
diff --git a/pypy/interpreter/pyparser/pyparse.py b/pypy/interpreter/pyparser/pyparse.py
--- a/pypy/interpreter/pyparser/pyparse.py
+++ b/pypy/interpreter/pyparser/pyparse.py
@@ -141,10 +141,7 @@
 
         flags = compile_info.flags
 
-        if flags & consts.CO_FUTURE_PRINT_FUNCTION:
-            self.grammar = pygram.python_grammar_no_print
-        else:
-            self.grammar = pygram.python_grammar
+        self.grammar = pygram.python_grammar
 
         # The tokenizer is very picky about how it wants its input.
         source_lines = textsrc.splitlines(True)

From noreply at buildbot.pypy.org  Wed Oct 12 22:23:16 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 12 Oct 2011 22:23:16 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: More tests pass
Message-ID: <20111012202316.786E682112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r47979:a61b0fdafd6a
Date: 2011-10-12 02:15 +0200
http://bitbucket.org/pypy/pypy/changeset/a61b0fdafd6a/

Log:	More tests pass

diff --git a/pypy/module/sys/app.py b/pypy/module/sys/app.py
--- a/pypy/module/sys/app.py
+++ b/pypy/module/sys/app.py
@@ -85,8 +85,7 @@
 
 
 # This is tested in test_app_main.py
-class sysflags:
-    __metaclass__ = structseqtype
+class sysflags(metaclass=structseqtype):
 
     name = "sys.flags"
 

From noreply at buildbot.pypy.org  Wed Oct 12 22:23:17 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 12 Oct 2011 22:23:17 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: fix test_descriptor
Message-ID: <20111012202317.AF82C82112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r47980:85551fcf1cf8
Date: 2011-10-12 02:18 +0200
http://bitbucket.org/pypy/pypy/changeset/85551fcf1cf8/

Log:	fix test_descriptor

diff --git a/pypy/module/__builtin__/app_inspect.py b/pypy/module/__builtin__/app_inspect.py
--- a/pypy/module/__builtin__/app_inspect.py
+++ b/pypy/module/__builtin__/app_inspect.py
@@ -64,14 +64,7 @@
 
     obj = args[0]
 
-    dir_meth = None
-    if isinstance(obj, types.InstanceType):
-        try:
-            dir_meth = getattr(obj, "__dir__")
-        except AttributeError:
-            pass
-    else:
-        dir_meth = lookup_special(obj, "__dir__")
+    dir_meth = lookup_special(obj, "__dir__")
     if dir_meth is not None:
         result = dir_meth()
         if not isinstance(result, list):
@@ -87,7 +80,7 @@
         except AttributeError:
             return []
 
-    elif isinstance(obj, (types.TypeType, types.ClassType)):
+    elif isinstance(obj, type):
         #Don't look at __class__, as metaclass methods would be confusing.
         result = _classdir(obj).keys()
         result.sort()

From noreply at buildbot.pypy.org  Wed Oct 12 22:23:18 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 12 Oct 2011 22:23:18 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Implement super() and all its magic:
Message-ID: <20111012202318.EF5D582112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r47981:a8cfd510a295
Date: 2011-10-12 22:19 +0200
http://bitbucket.org/pypy/pypy/changeset/a8cfd510a295/

Log:	Implement super() and all its magic: There is a hidden __class__
	cell, and self is the first argument of the function.

diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -201,7 +201,7 @@
                           filename=self.compile_info.filename)
 
     def name_op(self, identifier, ctx):
-        """Generate an operation appropiate for the scope of the identifier."""
+        """Generate an operation appropriate for the scope of the identifier."""
         scope = self.scope.lookup(identifier)
         op = ops.NOP
         container = self.names
@@ -1248,10 +1248,10 @@
         self._handle_body(cls.body)
         # return the (empty) __class__ cell
         scope = self.scope.lookup("@__class__")
-        if scope == symtable.SCOPE_UNKNOWN:
+        if scope == symtable.SCOPE_CELL:
+            # Return the cell where to store __class__
+            self.emit_op_arg(ops.LOAD_CLOSURE, self.cell_vars["@__class__"])
+        else:
             # This happens when nobody references the cell
             self.load_const(self.space.w_None)
-        else:
-            # Return the cell where to store __class__
-            self.emit_op_arg(ops.LOAD_CLOSURE, self.cell_vars["@__class__"])
         self.emit_op(ops.RETURN_VALUE)
diff --git a/pypy/interpreter/astcompiler/symtable.py b/pypy/interpreter/astcompiler/symtable.py
--- a/pypy/interpreter/astcompiler/symtable.py
+++ b/pypy/interpreter/astcompiler/symtable.py
@@ -163,6 +163,8 @@
             self._finalize_name(name, flags, local, bound, free, globs)
         if not self._hide_bound_from_nested_scopes:
             self._pass_on_bindings(local, bound, globs, new_bound, new_globs)
+        else:
+            self._pass_special_names(local, new_bound)
         child_frees = {}
         for child in self.children:
             # Symbol dictionaries are copied to avoid having child scopes
@@ -214,6 +216,12 @@
         self.import_star = None
         self.bare_exec = None
 
+    def note_symbol(self, identifier, role):
+        # Special-case super: it counts as a use of __class__
+        if role == SYM_USED and identifier == 'super':
+            self.note_symbol('@__class__', SYM_USED)
+        Scope.note_symbol(self, identifier, role)
+
     def note_yield(self, yield_node):
         if self.return_with_value:
             raise SyntaxError("'return' with argument inside generator",
@@ -295,6 +303,16 @@
     def mangle(self, name):
         return misc.mangle(name, self.name)
 
+    def _pass_special_names(self, local, new_bound):
+        assert '@__class__' in local
+        new_bound['@__class__'] = None
+
+    def _finalize_cells(self, free):
+        for name, role in self.symbols.iteritems():
+            if role == SCOPE_LOCAL and name in free and name == '@__class__':
+                self.symbols[name] = SCOPE_CELL
+                del free[name]
+
 
 class SymtableBuilder(ast.GenericASTVisitor):
     """Find symbol information from AST."""
@@ -373,6 +391,8 @@
         self.visit_sequence(clsdef.bases)
         self.visit_sequence(clsdef.decorator_list)
         self.push_scope(ClassScope(clsdef), clsdef)
+        self.note_symbol('@__class__', SYM_ASSIGNED)
+        self.note_symbol('__locals__', SYM_PARAM)
         self.visit_sequence(clsdef.body)
         self.pop_scope()
 
diff --git a/pypy/module/__builtin__/descriptor.py b/pypy/module/__builtin__/descriptor.py
--- a/pypy/module/__builtin__/descriptor.py
+++ b/pypy/module/__builtin__/descriptor.py
@@ -47,7 +47,36 @@
         return space.call_function(object_getattribute(space),
                                    w(self), w(name))
 
-def descr_new_super(space, w_subtype, w_starttype, w_obj_or_type=None):
+def descr_new_super(space, w_subtype, w_starttype=None, w_obj_or_type=None):
+    if space.is_w(w_starttype, space.w_None):
+        # Call super(), without args -- fill in from __class__
+        # and first local variable on the stack.
+        ec = space.getexecutioncontext()
+        frame = ec.gettopframe()
+        code = frame.pycode
+        if not code:
+            raise OperationError(space.w_SystemError, space.wrap(
+                    "super(): no code object"))
+        if code.co_argcount == 0:
+            raise OperationError(space.w_SystemError, space.wrap(
+                    "super(): no arguments"))
+        w_obj = frame.locals_stack_w[0]
+        if not w_obj:
+            raise OperationError(space.w_SystemError, space.wrap(
+                    "super(): arg[0] deleted"))
+        index = 0
+        for name in code.co_freevars:
+            if name == "@__class__":
+                break
+            index += 1
+        else:
+            raise OperationError(space.w_SystemError, space.wrap(
+                    "super(): __class__ cell not found"))
+        # a kind of LOAD_DEREF
+        cell = frame.cells[len(code.co_cellvars) + index]
+        w_starttype = cell.get()
+        w_obj_or_type = w_obj
+
     if space.is_w(w_obj_or_type, space.w_None):
         w_type = None  # unbound super object
     else:
diff --git a/pypy/module/__builtin__/test/test_descriptor.py b/pypy/module/__builtin__/test/test_descriptor.py
--- a/pypy/module/__builtin__/test/test_descriptor.py
+++ b/pypy/module/__builtin__/test/test_descriptor.py
@@ -73,6 +73,23 @@
         assert d.f() == "DBCA"
         assert D.__mro__ == (D, B, C, A, object)
 
+    def test_super_magic(self):
+        class A(object):
+            def f(self):
+                return 'A'
+        class B(A):
+            def f(self):
+                return 'B' + super().f()
+        class C(A):
+            def f(self):
+                return 'C' + super().f()
+        class D(B, C):
+            def f(self):
+                return 'D' + super().f()
+        d = D()
+        assert d.f() == "DBCA"
+        assert D.__mro__ == (D, B, C, A, object)
+
     def test_super_metaclass(self):
         class xtype(type):
             def __init__(self, name, bases, dict):

From noreply at buildbot.pypy.org  Wed Oct 12 22:23:20 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 12 Oct 2011 22:23:20 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Another test, which passes
Message-ID: <20111012202320.3416882112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r47982:f2c302086506
Date: 2011-10-12 22:19 +0200
http://bitbucket.org/pypy/pypy/changeset/f2c302086506/

Log:	Another test, which passes

diff --git a/pypy/module/__builtin__/test/test_descriptor.py b/pypy/module/__builtin__/test/test_descriptor.py
--- a/pypy/module/__builtin__/test/test_descriptor.py
+++ b/pypy/module/__builtin__/test/test_descriptor.py
@@ -110,6 +110,17 @@
             f = classmethod(f)
         assert B().f() == [B, B]
 
+    def test_super_magic_classmethod(self):
+        class A(object):
+            def f(cls):
+                return cls
+            f = classmethod(f)
+        class B(A):
+            def f(cls):
+                return [cls, super().f()]
+            f = classmethod(f)
+        assert B().f() == [B, B]
+
     def test_super_fail(self):
         try:
             super(list, 2)

From noreply at buildbot.pypy.org  Wed Oct 12 22:23:21 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 12 Oct 2011 22:23:21 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Fix some metaclasses,
 and the sys module can now be imported
Message-ID: <20111012202321.72EA382112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r47983:be8493f31b60
Date: 2011-10-12 22:19 +0200
http://bitbucket.org/pypy/pypy/changeset/be8493f31b60/

Log:	Fix some metaclasses, and the sys module can now be imported

diff --git a/pypy/module/sys/system.py b/pypy/module/sys/system.py
--- a/pypy/module/sys/system.py
+++ b/pypy/module/sys/system.py
@@ -7,8 +7,7 @@
 app = gateway.applevel("""
 "NOT_RPYTHON"
 from _structseq import structseqtype, structseqfield
-class float_info:
-    __metaclass__ = structseqtype
+class float_info(metaclass=structseqtype):
 
     max = structseqfield(0)
     max_exp = structseqfield(1)
@@ -22,8 +21,7 @@
     radix = structseqfield(9)
     rounds = structseqfield(10)
 
-class long_info:
-    __metaclass__ = structseqtype
+class long_info(metaclass=structseqtype):
     bits_per_digit = structseqfield(0)
     sizeof_digit = structseqfield(1)
 """)
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -41,8 +41,7 @@
 app = gateway.applevel('''
 "NOT_RPYTHON"
 from _structseq import structseqtype, structseqfield
-class version_info:
-    __metaclass__ = structseqtype
+class version_info(metaclass=structseqtype):
 
     major        = structseqfield(0, "Major release number")
     minor        = structseqfield(1, "Minor release number")
diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -159,8 +159,7 @@
 "NOT_RPYTHON"
 from _structseq import structseqtype, structseqfield
 
-class windows_version_info:
-    __metaclass__ = structseqtype
+class windows_version_info(metaclass=structseqtype):
 
     name = "sys.getwindowsversion"
 

From noreply at buildbot.pypy.org  Wed Oct 12 22:23:22 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 12 Oct 2011 22:23:22 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Another py3k syntax at applevel
Message-ID: <20111012202322.A512882112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r47984:bd26d84b3e84
Date: 2011-10-12 22:19 +0200
http://bitbucket.org/pypy/pypy/changeset/bd26d84b3e84/

Log:	Another py3k syntax at applevel

diff --git a/pypy/module/__builtin__/app_io.py b/pypy/module/__builtin__/app_io.py
--- a/pypy/module/__builtin__/app_io.py
+++ b/pypy/module/__builtin__/app_io.py
@@ -25,7 +25,7 @@
         f.close()
     #Don't exec the source directly, as this loses the filename info
     co = compile(source.rstrip()+"\n", filename, 'exec')
-    exec co in glob, loc
+    exec(co, glob, loc)
 
 def raw_input(prompt=None):
     """raw_input([prompt]) -> string

From noreply at buildbot.pypy.org  Wed Oct 12 22:23:23 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 12 Oct 2011 22:23:23 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Remove `backquote`
Message-ID: <20111012202323.ECFA882112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r47985:8282c696b2ab
Date: 2011-10-12 22:19 +0200
http://bitbucket.org/pypy/pypy/changeset/8282c696b2ab/

Log:	Remove `backquote`

diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -1650,31 +1650,6 @@
             self.kwargs.sync_app_attrs(space)
 
 
-class Repr(expr):
-
-    _lineno_mask = 2
-    _col_offset_mask = 4
-
-    def __init__(self, value, lineno, col_offset):
-        self.value = value
-        expr.__init__(self, lineno, col_offset)
-        self.initialization_state = 7
-
-    def walkabout(self, visitor):
-        visitor.visit_Repr(self)
-
-    def mutate_over(self, visitor):
-        self.value = self.value.mutate_over(visitor)
-        return visitor.visit_Repr(self)
-
-    def sync_app_attrs(self, space):
-        if (self.initialization_state & ~0) ^ 7:
-            missing_field(space, self.initialization_state, ['value', 'lineno', 'col_offset'], 'Repr')
-        else:
-            pass
-        self.value.sync_app_attrs(space)
-
-
 class Num(expr):
 
     _lineno_mask = 2
@@ -2576,8 +2551,6 @@
         return self.default_visitor(node)
     def visit_Call(self, node):
         return self.default_visitor(node)
-    def visit_Repr(self, node):
-        return self.default_visitor(node)
     def visit_Num(self, node):
         return self.default_visitor(node)
     def visit_Str(self, node):
@@ -2782,9 +2755,6 @@
         if node.kwargs:
             node.kwargs.walkabout(self)
 
-    def visit_Repr(self, node):
-        node.value.walkabout(self)
-
     def visit_Num(self, node):
         pass
 
@@ -5671,52 +5641,6 @@
     __init__=interp2app(Call_init),
 )
 
-def Repr_get_value(space, w_self):
-    if w_self.w_dict is not None:
-        w_obj = w_self.getdictvalue(space, 'value')
-        if w_obj is not None:
-            return w_obj
-    if not w_self.initialization_state & 1:
-        typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
-    return space.wrap(w_self.value)
-
-def Repr_set_value(space, w_self, w_new_value):
-    try:
-        w_self.value = space.interp_w(expr, w_new_value, False)
-    except OperationError, e:
-        if not e.match(space, space.w_TypeError):
-            raise
-        w_self.setdictvalue(space, 'value', w_new_value)
-        return
-    w_self.deldictvalue(space, 'value')
-    w_self.initialization_state |= 1
-
-_Repr_field_unroller = unrolling_iterable(['value'])
-def Repr_init(space, w_self, __args__):
-    w_self = space.descr_self_interp_w(Repr, w_self)
-    args_w, kwargs_w = __args__.unpack()
-    if args_w:
-        if len(args_w) != 1:
-            w_err = space.wrap("Repr constructor takes either 0 or 1 positional argument")
-            raise OperationError(space.w_TypeError, w_err)
-        i = 0
-        for field in _Repr_field_unroller:
-            space.setattr(w_self, space.wrap(field), args_w[i])
-            i += 1
-    for field, w_value in kwargs_w.iteritems():
-        space.setattr(w_self, space.wrap(field), w_value)
-
-Repr.typedef = typedef.TypeDef("Repr",
-    expr.typedef,
-    __module__='_ast',
-    _fields=_FieldsWrapper(['value']),
-    value=typedef.GetSetProperty(Repr_get_value, Repr_set_value, cls=Repr),
-    __new__=interp2app(get_AST_new(Repr)),
-    __init__=interp2app(Repr_init),
-)
-
 def Num_get_n(space, w_self):
     if w_self.w_dict is not None:
         w_obj = w_self.getdictvalue(space, 'n')
diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -1123,9 +1123,6 @@
                 keys.append(self.handle_expr(maker.children[i]))
                 values.append(self.handle_expr(maker.children[i + 2]))
             return ast.Dict(keys, values, atom_node.lineno, atom_node.column)
-        elif first_child_type == tokens.BACKQUOTE:
-            expr = self.handle_testlist(atom_node.children[1])
-            return ast.Repr(expr, atom_node.lineno, atom_node.column)
         else:
             raise AssertionError("unknown atom")
 
diff --git a/pypy/interpreter/astcompiler/asthelpers.py b/pypy/interpreter/astcompiler/asthelpers.py
--- a/pypy/interpreter/astcompiler/asthelpers.py
+++ b/pypy/interpreter/astcompiler/asthelpers.py
@@ -139,11 +139,6 @@
     _description = "conditional expression"
 
 
-class __extend__(ast.Repr):
-
-    _description = "repr"
-
-
 class __extend__(ast.Const):
 
     constant = True
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -1040,11 +1040,6 @@
         self._compile_comprehension(dictcomp, "<dictcomp>",
                                     ComprehensionCodeGenerator)
 
-    def visit_Repr(self, rep):
-        self.update_position(rep.lineno)
-        rep.value.walkabout(self)
-        self.emit_op(ops.UNARY_CONVERT)
-
     def visit_Attribute(self, attr):
         self.update_position(attr.lineno)
         names = self.names
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -763,7 +763,7 @@
         yield self.st, "y = lambda x: x", "y(4)", 4
 
     def test_backquote_repr(self):
-        yield self.st, "x = None; y = `x`", "y", "None"
+        py.test.raises(SyntaxError, self.simple_test, "y = `0`", None, None)
 
     def test_deleting_attributes(self):
         test = """if 1:
diff --git a/pypy/interpreter/astcompiler/tools/Python.asdl b/pypy/interpreter/astcompiler/tools/Python.asdl
--- a/pypy/interpreter/astcompiler/tools/Python.asdl
+++ b/pypy/interpreter/astcompiler/tools/Python.asdl
@@ -66,7 +66,6 @@
 	     | Compare(expr left, cmpop* ops, expr* comparators)
 	     | Call(expr func, expr* args, keyword* keywords,
 			 expr? starargs, expr? kwargs)
-	     | Repr(expr value)
 	     | Num(object n) -- a number as a PyObject.
 	     | Str(string s) -- need to specify raw, unicode, etc?
 	     -- other literals? bools?
diff --git a/pypy/interpreter/pyparser/data/Grammar3.2 b/pypy/interpreter/pyparser/data/Grammar3.2
--- a/pypy/interpreter/pyparser/data/Grammar3.2
+++ b/pypy/interpreter/pyparser/data/Grammar3.2
@@ -100,7 +100,6 @@
 atom: ('(' [yield_expr|testlist_comp] ')' |
        '[' [listmaker] ']' |
        '{' [dictorsetmaker] '}' |
-       '`' testlist1 '`' |
        NAME | NUMBER | STRING+)
 listmaker: test ( list_for | (',' test)* [','] )
 testlist_comp: test ( comp_for | (',' test)* [','] )
diff --git a/pypy/interpreter/pyparser/pytoken.py b/pypy/interpreter/pyparser/pytoken.py
--- a/pypy/interpreter/pyparser/pytoken.py
+++ b/pypy/interpreter/pyparser/pytoken.py
@@ -35,7 +35,6 @@
 _add_tok('EQUAL', "=" )
 _add_tok('DOT', "." )
 _add_tok('PERCENT', "%" )
-_add_tok('BACKQUOTE', "`" )
 _add_tok('LBRACE', "{" )
 _add_tok('RBRACE', "}" )
 _add_tok('EQEQUAL', "==" )

From noreply at buildbot.pypy.org  Wed Oct 12 22:23:25 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 12 Oct 2011 22:23:25 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Fix another test
Message-ID: <20111012202325.369F282112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r47986:f1e581823bb9
Date: 2011-10-12 22:19 +0200
http://bitbucket.org/pypy/pypy/changeset/f1e581823bb9/

Log:	Fix another test

diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -377,7 +377,7 @@
                     return a
              ''',                            "foo docstring"),
             ('''def foo():
-                    """doc"""; print 1
+                    """doc"""; assert 1
                     a=1
              ''',                            "doc"),
             ('''

From noreply at buildbot.pypy.org  Wed Oct 12 22:23:26 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 12 Oct 2011 22:23:26 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: New print() syntax for applevel code
Message-ID: <20111012202326.6EF3A82112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r47987:2caefaa6e24d
Date: 2011-10-12 22:19 +0200
http://bitbucket.org/pypy/pypy/changeset/2caefaa6e24d/

Log:	New print() syntax for applevel code

diff --git a/pypy/bin/py.py b/pypy/bin/py.py
--- a/pypy/bin/py.py
+++ b/pypy/bin/py.py
@@ -52,7 +52,7 @@
             import site
         except:
             import sys
-            print >> sys.stderr, "import site\' failed"
+            print("'import site' failed", file=sys.stderr)
 ''').interphook('pypy_init')
 
 

From noreply at buildbot.pypy.org  Wed Oct 12 22:23:27 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 12 Oct 2011 22:23:27 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Add a way to run applevel tests with
 python 3: --appdirect=/path/to/python3
Message-ID: <20111012202327.AAC3F82112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r47988:1b96f59860c8
Date: 2011-10-12 22:19 +0200
http://bitbucket.org/pypy/pypy/changeset/1b96f59860c8/

Log:	Add a way to run applevel tests with python 3:
	--appdirect=/path/to/python3 This implies -A, of course

	Fix range() tests to pass on python3.

diff --git a/pypy/conftest.py b/pypy/conftest.py
--- a/pypy/conftest.py
+++ b/pypy/conftest.py
@@ -8,7 +8,7 @@
 from inspect import isclass, getmro
 from pypy.tool.udir import udir
 from pypy.tool.autopath import pypydir
-from pypy.tool import leakfinder
+from pypy.tool import leakfinder, runsubprocess
 
 # pytest settings
 rsyncdirs = ['.', '../lib-python', '../lib_pypy', '../demo']
@@ -20,11 +20,13 @@
 option = None
 
 def pytest_report_header():
-    return "pytest-%s from %s" %(pytest.__version__, pytest.__file__)
+    return "pytest-%s from %s" % (pytest.__version__, pytest.__file__)
 
 def pytest_configure(config):
     global option
     option = config.option
+    if option.appdirect:
+        option.runappdirect = True
 
 def _set_platform(opt, opt_str, value, parser):
     from pypy.config.translationoption import PLATFORMS
@@ -40,6 +42,8 @@
     group.addoption('-A', '--runappdirect', action="store_true",
            default=False, dest="runappdirect",
            help="run applevel tests directly on python interpreter (not through PyPy)")
+    group.addoption('--appdirect', type="string",
+           help="run applevel tests directly with the specified interpreter")
     group.addoption('--direct', action="store_true",
            default=False, dest="rundirect",
            help="run pexpect tests directly")
@@ -190,6 +194,24 @@
     if option.runappdirect:
         py.test.skip("translation test, skipped for appdirect")
 
+def run_with_python(python, target):
+    helpers = """if 1:
+    def raises(exc, func, *args, **kwargs):
+        try:
+            func(*args, **kwargs)
+        except exc:
+            pass
+        else:
+            raise AssertionError("DID NOT RAISE")
+"""
+    source = py.code.Source(target)[1:].deindent()
+    res, stdout, stderr = runsubprocess.run_subprocess(
+        python, ["-c", helpers + str(source)])
+    print source
+    print >> sys.stdout, stdout
+    print >> sys.stderr, stderr
+    if res > 0:
+        raise AssertionError("Subprocess failed")
 
 class OpErrKeyboardInterrupt(KeyboardInterrupt):
     pass
@@ -398,6 +420,8 @@
     def runtest(self):
         target = self.obj
         if self.config.option.runappdirect:
+            if self.config.option.appdirect:
+                return run_with_python(self.config.option.appdirect, target)
             return target()
         space = gettestobjspace()
         filename = self._getdynfilename(target)
@@ -441,6 +465,8 @@
     def runtest(self):
         target = self.obj
         if self.config.option.runappdirect:
+            if self.config.option.appdirect:
+                return run_with_python(self.config.option.appdirect, target)
             return target()
         space = target.im_self.space
         filename = self._getdynfilename(target)
diff --git a/pypy/module/__builtin__/test/test_range.py b/pypy/module/__builtin__/test/test_range.py
--- a/pypy/module/__builtin__/test/test_range.py
+++ b/pypy/module/__builtin__/test/test_range.py
@@ -9,52 +9,52 @@
       raises(TypeError, range,  1, 2, 3, 4)
 
    def test_range_one(self):
-      assert range(1) == [0]
+      assert list(range(1)) == [0]
 
    def test_range_posstartisstop(self):
-      assert range(1, 1) == []
+      assert list(range(1, 1)) == []
 
    def test_range_negstartisstop(self):
-      assert range(-1, -1) == []
+      assert list(range(-1, -1)) == []
 
    def test_range_zero(self):
-      assert range(0) == []
+      assert list(range(0)) == []
 
    def test_range_twoargs(self):
-      assert range(1, 2) == [1]
+      assert list(range(1, 2)) == [1]
       
    def test_range_decreasingtwoargs(self):
-      assert range(3, 1) == []
+      assert list(range(3, 1)) == []
 
    def test_range_negatives(self):
-      assert range(-3) == []
+      assert list(range(-3)) == []
 
    def test_range_decreasing_negativestep(self):
-      assert range(5, -2, -1) == [5, 4, 3, 2, 1, 0 , -1]
+      assert list(range(5, -2, -1)) == [5, 4, 3, 2, 1, 0 , -1]
 
    def test_range_posfencepost1(self):
-       assert range (1, 10, 3) == [1, 4, 7]
+       assert list(range(1, 10, 3)) == [1, 4, 7]
 
    def test_range_posfencepost2(self):
-       assert range (1, 11, 3) == [1, 4, 7, 10]
+       assert list(range(1, 11, 3)) == [1, 4, 7, 10]
 
    def test_range_posfencepost3(self):
-       assert range (1, 12, 3) == [1, 4, 7, 10]
+       assert list(range(1, 12, 3)) == [1, 4, 7, 10]
 
    def test_range_negfencepost1(self):
-       assert range (-1, -10, -3) == [-1, -4, -7]
+       assert list(range(-1, -10, -3)) == [-1, -4, -7]
 
    def test_range_negfencepost2(self):
-       assert range (-1, -11, -3) == [-1, -4, -7, -10]
+       assert list(range(-1, -11, -3)) == [-1, -4, -7, -10]
 
    def test_range_negfencepost3(self):
-       assert range (-1, -12, -3) == [-1, -4, -7, -10]
+       assert list(range(-1, -12, -3)) == [-1, -4, -7, -10]
 
    def test_range_decreasing_negativelargestep(self):
-       assert range(5, -2, -3) == [5, 2, -1]
+       assert list(range(5, -2, -3)) == [5, 2, -1]
 
    def test_range_increasing_positivelargestep(self):
-       assert range(-5, 2, 3) == [-5, -2, 1]
+       assert list(range(-5, 2, 3)) == [-5, -2, 1]
 
    def test_range_zerostep(self):
        raises(ValueError, range, 1, 5, 0)
@@ -70,34 +70,34 @@
    def test_range_wrong_type(self):
        raises(TypeError, range, "42")
 
-   def test_range_object_with___int__(self):
+   def test_range_object_with___index__(self):
        class A(object):
-           def __int__(self):
+           def __index__(self):
                return 5
 
-       assert range(A()) == [0, 1, 2, 3, 4]
-       assert range(0, A()) == [0, 1, 2, 3, 4]
-       assert range(0, 10, A()) == [0, 5]
+       assert list(range(A())) == [0, 1, 2, 3, 4]
+       assert list(range(0, A())) == [0, 1, 2, 3, 4]
+       assert list(range(0, 10, A())) == [0, 5]
 
    def test_range_long(self):
        import sys
-       assert range(-2**100) == []
-       assert range(0, -2**100) == []
-       assert range(0, 2**100, -1) == []
-       assert range(0, 2**100, -1) == []
+       assert list(range(-2**100)) == []
+       assert list(range(0, -2**100)) == []
+       assert list(range(0, 2**100, -1)) == []
+       assert list(range(0, 2**100, -1)) == []
 
-       a = long(10 * sys.maxint)
-       assert range(a, a+2) == [a, a+1]
-       assert range(a+2, a, -1L) == [a+2, a+1]
-       assert range(a+4, a, -2) == [a+4, a+2]
-       assert range(a, a*5, a) == [a, 2*a, 3*a, 4*a]
+       a = 10 * sys.maxsize
+       assert list(range(a, a+2)) == [a, a+1]
+       assert list(range(a+2, a, -1)) == [a+2, a+1]
+       assert list(range(a+4, a, -2)) == [a+4, a+2]
+       assert list(range(a, a*5, a)) == [a, 2*a, 3*a, 4*a]
 
    def test_range_cases(self):
        import sys
-       for start in [10, 10 * sys.maxint]:
+       for start in [10, 10 * sys.maxsize]:
            for stop in [start-4, start-1, start, start+1, start+4]:
               for step in [1, 2, 3, 4]:
-                  lst = range(start, stop, step)
+                  lst = list(range(start, stop, step))
                   expected = []
                   a = start
                   while a < stop:
@@ -105,7 +105,7 @@
                       a += step
                   assert lst == expected
               for step in [-1, -2, -3, -4]:
-                  lst = range(start, stop, step)
+                  lst = list(range(start, stop, step))
                   expected = []
                   a = start
                   while a > stop:
diff --git a/pypy/tool/pytest/test/test_conftest1.py b/pypy/tool/pytest/test/test_conftest1.py
--- a/pypy/tool/pytest/test/test_conftest1.py
+++ b/pypy/tool/pytest/test/test_conftest1.py
@@ -1,5 +1,6 @@
 
 import py
+import sys
 
 innertest = py.path.local(__file__).dirpath('conftest1_innertest.py')
 pytest_plugins = "pytester"
@@ -21,7 +22,7 @@
         assert "app_test_something" in passed[0].nodeid
         assert "test_method_app" in passed[1].nodeid
 
-    def test_appdirect(self, testdir):
+    def test_runappdirect(self, testdir):
         sorter = testdir.inline_run(innertest, '-k', 'applevel', '--runappdirect')
         passed, skipped, failed = sorter.listoutcomes()
         assert len(passed) == 2
@@ -29,3 +30,12 @@
         assert "app_test_something" in passed[0].nodeid
         assert "test_method_app" in passed[1].nodeid
         
+    def test_appdirect(self, testdir):
+        sorter = testdir.inline_run(innertest, '-k', 'applevel',
+                                    '--appdirect=%s' % (sys.executable,))
+        passed, skipped, failed = sorter.listoutcomes()
+        assert len(passed) == 2
+        print passed
+        assert "app_test_something" in passed[0].nodeid
+        assert "test_method_app" in passed[1].nodeid
+        

From noreply at buildbot.pypy.org  Wed Oct 12 22:23:29 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 12 Oct 2011 22:23:29 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: rename xrange() to range(),
	and remove pypy's RangeList
Message-ID: <20111012202329.103E882112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r47989:fea44485fe72
Date: 2011-10-12 22:19 +0200
http://bitbucket.org/pypy/pypy/changeset/fea44485fe72/

Log:	rename xrange() to range(), and remove pypy's RangeList

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -275,12 +275,6 @@
                              ("objspace.std.withmethodcache", True),
                        ]),
 
-        BoolOption("withrangelist",
-                   "enable special range list implementation that does not "
-                   "actually create the full list until the resulting "
-                   "list is mutated",
-                   default=False),
-
         BoolOption("withtypeversion",
                    "version type objects when changing them",
                    cmdline=None,
diff --git a/pypy/doc/config/objspace.std.withrangelist.txt b/pypy/doc/config/objspace.std.withrangelist.txt
deleted file mode 100644
--- a/pypy/doc/config/objspace.std.withrangelist.txt
+++ /dev/null
@@ -1,11 +0,0 @@
-Enable "range list" objects. They are an additional implementation of the Python
-``list`` type, indistinguishable for the normal user. Whenever the ``range``
-builtin is called, an range list is returned. As long as this list is not
-mutated (and for example only iterated over), it uses only enough memory to
-store the start, stop and step of the range. This makes using ``range`` as
-efficient as ``xrange``, as long as the result is only used in a ``for``-loop.
-
-See the section in `Standard Interpreter Optimizations`_ for more details.
-
-.. _`Standard Interpreter Optimizations`: ../interpreter-optimizations.html#range-lists
-
diff --git a/pypy/module/__builtin__/__init__.py b/pypy/module/__builtin__/__init__.py
--- a/pypy/module/__builtin__/__init__.py
+++ b/pypy/module/__builtin__/__init__.py
@@ -83,8 +83,7 @@
         '__import__'    : 'pypy.module.imp.importing.importhook',
         'reload'        : 'pypy.module.imp.importing.reload',
 
-        'range'         : 'functional.range_int',
-        'xrange'        : 'functional.W_XRange',
+        'range'         : 'functional.W_Range',
         'enumerate'     : 'functional.W_Enumerate',
         'min'           : 'functional.min',
         'max'           : 'functional.max',
diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py
--- a/pypy/module/__builtin__/functional.py
+++ b/pypy/module/__builtin__/functional.py
@@ -15,7 +15,7 @@
 
 def get_len_of_range(space, lo, hi, step):
     """
-    Return number of items in range/xrange (lo, hi, step).
+    Return number of items in range (lo, hi, step).
     Raise ValueError if step == 0 and OverflowError if the true value is too
     large to fit in a signed long.
     """
@@ -47,91 +47,6 @@
         n = 0
     return n
 
-def range_int(space, w_x, w_y=NoneNotWrapped, w_step=1):
-    """Return a list of integers in arithmetic position from start (defaults
-to zero) to stop - 1 by step (defaults to 1).  Use a negative step to
-get a list in decending order."""
-
-    if w_y is None:
-        w_start = space.wrap(0)
-        w_stop = w_x
-    else:
-        w_start = w_x
-        w_stop = w_y
-
-    if space.is_true(space.isinstance(w_stop, space.w_float)):
-        raise OperationError(space.w_TypeError,
-            space.wrap("range() integer end argument expected, got float."))
-    if space.is_true(space.isinstance(w_start, space.w_float)):
-        raise OperationError(space.w_TypeError,
-            space.wrap("range() integer start argument expected, got float."))
-    if space.is_true(space.isinstance(w_step, space.w_float)):
-        raise OperationError(space.w_TypeError,
-            space.wrap("range() integer step argument expected, got float."))
-
-    w_start = space.int(w_start)
-    w_stop  = space.int(w_stop)
-    w_step  = space.int(w_step)
-
-    try:
-        start = space.int_w(w_start)
-        stop  = space.int_w(w_stop)
-        step  = space.int_w(w_step)
-    except OperationError, e:
-        if not e.match(space, space.w_OverflowError):
-            raise
-        return range_with_longs(space, w_start, w_stop, w_step)
-
-    howmany = get_len_of_range(space, start, stop, step)
-
-    if space.config.objspace.std.withrangelist:
-        return range_withspecialized_implementation(space, start,
-                                                    step, howmany)
-    res_w = [None] * howmany
-    v = start
-    for idx in range(howmany):
-        res_w[idx] = space.wrap(v)
-        v += step
-    return space.newlist(res_w)
-
-
-def range_withspecialized_implementation(space, start, step, howmany):
-    assert space.config.objspace.std.withrangelist
-    from pypy.objspace.std.rangeobject import W_RangeListObject
-    return W_RangeListObject(start, step, howmany)
-
-bigint_one = rbigint.fromint(1)
-
-def range_with_longs(space, w_start, w_stop, w_step):
-
-    start = lo = space.bigint_w(w_start)
-    stop  = hi = space.bigint_w(w_stop)
-    step  = st = space.bigint_w(w_step)
-
-    if not step.tobool():
-        raise OperationError(space.w_ValueError,
-                             space.wrap("step argument must not be zero"))
-    elif step.sign < 0:
-        lo, hi, st = hi, lo, st.neg()
-
-    if lo.lt(hi):
-        diff = hi.sub(lo).sub(bigint_one)
-        n = diff.floordiv(st).add(bigint_one)
-        try:
-            howmany = n.toint()
-        except OverflowError:
-            raise OperationError(space.w_OverflowError,
-                                 space.wrap("result has too many items"))
-    else:
-        howmany = 0
-
-    res_w = [None] * howmany
-    v = start
-    for idx in range(howmany):
-        res_w[idx] = space.newlong_from_rbigint(v)
-        v = v.add(step)
-    return space.newlist(res_w)
-
 
 @specialize.arg(2)
 @jit.look_inside_iff(lambda space, args, implementation_of:
@@ -528,7 +443,7 @@
 
 
 
-class W_XRange(Wrappable):
+class W_Range(Wrappable):
     def __init__(self, space, start, len, step):
         self.space = space
         self.start = start
@@ -543,18 +458,18 @@
         else:
             stop = _toint(space, w_stop)
         howmany = get_len_of_range(space, start, stop, step)
-        obj = space.allocate_instance(W_XRange, w_subtype)
-        W_XRange.__init__(obj, space, start, howmany, step)
+        obj = space.allocate_instance(W_Range, w_subtype)
+        W_Range.__init__(obj, space, start, howmany, step)
         return space.wrap(obj)
 
     def descr_repr(self):
         stop = self.start + self.len * self.step
         if self.start == 0 and self.step == 1:
-            s = "xrange(%d)" % (stop,)
+            s = "range(%d)" % (stop,)
         elif self.step == 1:
-            s = "xrange(%d, %d)" % (self.start, stop)
+            s = "range(%d, %d)" % (self.start, stop)
         else:
-            s = "xrange(%d, %d, %d)" %(self.start, stop, self.step)
+            s = "range(%d, %d, %d)" %(self.start, stop, self.step)
         return self.space.wrap(s)
 
     def descr_len(self):
@@ -562,7 +477,7 @@
 
     @unwrap_spec(i='index')
     def descr_getitem(self, i):
-        # xrange does NOT support slicing
+        # range does NOT support slicing
         space = self.space
         len = self.len
         if i < 0:
@@ -570,15 +485,15 @@
         if 0 <= i < len:
             return space.wrap(self.start + i * self.step)
         raise OperationError(space.w_IndexError,
-                             space.wrap("xrange object index out of range"))
+                             space.wrap("range object index out of range"))
 
     def descr_iter(self):
-        return self.space.wrap(W_XRangeIterator(self.space, self.start,
+        return self.space.wrap(W_RangeIterator(self.space, self.start,
                                                 self.len, self.step))
 
     def descr_reversed(self):
         lastitem = self.start + (self.len-1) * self.step
-        return self.space.wrap(W_XRangeIterator(self.space, lastitem,
+        return self.space.wrap(W_RangeIterator(self.space, lastitem,
                                                 self.len, -self.step))
 
     def descr_reduce(self):
@@ -591,21 +506,19 @@
              ])
 
 def _toint(space, w_obj):
-    # this also supports float arguments.  CPython still does, too.
-    # needs a bit more thinking in general...
-    return space.int_w(space.int(w_obj))
+    return space.int_w(space.index(w_obj))
 
-W_XRange.typedef = TypeDef("xrange",
-    __new__          = interp2app(W_XRange.descr_new.im_func),
-    __repr__         = interp2app(W_XRange.descr_repr),
-    __getitem__      = interp2app(W_XRange.descr_getitem),
-    __iter__         = interp2app(W_XRange.descr_iter),
-    __len__          = interp2app(W_XRange.descr_len),
-    __reversed__     = interp2app(W_XRange.descr_reversed),
-    __reduce__       = interp2app(W_XRange.descr_reduce),
+W_Range.typedef = TypeDef("range",
+    __new__          = interp2app(W_Range.descr_new.im_func),
+    __repr__         = interp2app(W_Range.descr_repr),
+    __getitem__      = interp2app(W_Range.descr_getitem),
+    __iter__         = interp2app(W_Range.descr_iter),
+    __len__          = interp2app(W_Range.descr_len),
+    __reversed__     = interp2app(W_Range.descr_reversed),
+    __reduce__       = interp2app(W_Range.descr_reduce),
 )
 
-class W_XRangeIterator(Wrappable):
+class W_RangeIterator(Wrappable):
     def __init__(self, space, current, remaining, step):
         self.space = space
         self.current = current
@@ -631,17 +544,17 @@
         space    = self.space
         w_mod    = space.getbuiltinmodule('_pickle_support')
         mod      = space.interp_w(MixedModule, w_mod)
-        new_inst = mod.get('xrangeiter_new')
+        new_inst = mod.get('rangeiter_new')
         w        = space.wrap
         nt = space.newtuple
 
         tup = [w(self.current), w(self.remaining), w(self.step)]
         return nt([new_inst, nt(tup)])
 
-W_XRangeIterator.typedef = TypeDef("rangeiterator",
-    __iter__        = interp2app(W_XRangeIterator.descr_iter),
+W_RangeIterator.typedef = TypeDef("rangeiterator",
+    __iter__        = interp2app(W_RangeIterator.descr_iter),
 # XXX __length_hint__()
-##    __len__         = interp2app(W_XRangeIterator.descr_len),
-    next            = interp2app(W_XRangeIterator.descr_next),
-    __reduce__      = interp2app(W_XRangeIterator.descr_reduce),
+##    __len__         = interp2app(W_RangeIterator.descr_len),
+    next            = interp2app(W_RangeIterator.descr_next),
+    __reduce__      = interp2app(W_RangeIterator.descr_reduce),
 )
diff --git a/pypy/module/__builtin__/test/test_builtin.py b/pypy/module/__builtin__/test/test_builtin.py
--- a/pypy/module/__builtin__/test/test_builtin.py
+++ b/pypy/module/__builtin__/test/test_builtin.py
@@ -275,84 +275,84 @@
         assert next(x) == 2
         assert next(x) == 3
 
-    def test_xrange_args(self):
-##        # xrange() attributes are deprecated and were removed in Python 2.3.
-##        x = xrange(2)
+    def test_range_args(self):
+##        # range() attributes are deprecated and were removed in Python 2.3.
+##        x = range(2)
 ##        assert x.start == 0
 ##        assert x.stop == 2
 ##        assert x.step == 1
 
-##        x = xrange(2,10,2)
+##        x = range(2,10,2)
 ##        assert x.start == 2
 ##        assert x.stop == 10
 ##        assert x.step == 2
 
-##        x = xrange(2.3, 10.5, 2.4)
+##        x = range(2.3, 10.5, 2.4)
 ##        assert x.start == 2
 ##        assert x.stop == 10
 ##        assert x.step == 2
 
-        raises(ValueError, xrange, 0, 1, 0)
+        raises(ValueError, range, 0, 1, 0)
 
-    def test_xrange_repr(self): 
-        assert repr(xrange(1)) == 'xrange(1)'
-        assert repr(xrange(1,2)) == 'xrange(1, 2)'
-        assert repr(xrange(1,2,3)) == 'xrange(1, 4, 3)'
+    def test_range_repr(self): 
+        assert repr(range(1)) == 'range(1)'
+        assert repr(range(1,2)) == 'range(1, 2)'
+        assert repr(range(1,2,3)) == 'range(1, 4, 3)'
 
-    def test_xrange_up(self):
-        x = xrange(2)
+    def test_range_up(self):
+        x = range(2)
         iter_x = iter(x)
         assert iter_x.next() == 0
         assert iter_x.next() == 1
         raises(StopIteration, iter_x.next)
 
-    def test_xrange_down(self):
-        x = xrange(4,2,-1)
+    def test_range_down(self):
+        x = range(4,2,-1)
 
         iter_x = iter(x)
         assert iter_x.next() == 4
         assert iter_x.next() == 3
         raises(StopIteration, iter_x.next)
 
-    def test_xrange_has_type_identity(self):
-        assert type(xrange(1)) == type(xrange(1))
+    def test_range_has_type_identity(self):
+        assert type(range(1)) == type(range(1))
 
-    def test_xrange_len(self):
-        x = xrange(33)
+    def test_range_len(self):
+        x = range(33)
         assert len(x) == 33
-        x = xrange(33.2)
+        x = range(33.2)
         assert len(x) == 33
-        x = xrange(33,0,-1)
+        x = range(33,0,-1)
         assert len(x) == 33
-        x = xrange(33,0)
+        x = range(33,0)
         assert len(x) == 0
-        x = xrange(33,0.2)
+        x = range(33,0.2)
         assert len(x) == 0
-        x = xrange(0,33)
+        x = range(0,33)
         assert len(x) == 33
-        x = xrange(0,33,-1)
+        x = range(0,33,-1)
         assert len(x) == 0
-        x = xrange(0,33,2)
+        x = range(0,33,2)
         assert len(x) == 17
-        x = xrange(0,32,2)
+        x = range(0,32,2)
         assert len(x) == 16
 
-    def test_xrange_indexing(self):
-        x = xrange(0,33,2)
+    def test_range_indexing(self):
+        x = range(0,33,2)
         assert x[7] == 14
         assert x[-7] == 20
         raises(IndexError, x.__getitem__, 17)
         raises(IndexError, x.__getitem__, -18)
         raises(TypeError, x.__getitem__, slice(0,3,1))
 
-    def test_xrange_bad_args(self):
-        raises(TypeError, xrange, '1')
-        raises(TypeError, xrange, None)
-        raises(TypeError, xrange, 3+2j)
-        raises(TypeError, xrange, 1, '1')
-        raises(TypeError, xrange, 1, 3+2j)
-        raises(TypeError, xrange, 1, 2, '1')
-        raises(TypeError, xrange, 1, 2, 3+2j)
+    def test_range_bad_args(self):
+        raises(TypeError, range, '1')
+        raises(TypeError, range, None)
+        raises(TypeError, range, 3+2j)
+        raises(TypeError, range, 1, '1')
+        raises(TypeError, range, 1, 3+2j)
+        raises(TypeError, range, 1, 2, '1')
+        raises(TypeError, range, 1, 2, 3+2j)
     
     def test_sorted(self):
         l = []
diff --git a/pypy/objspace/std/model.py b/pypy/objspace/std/model.py
--- a/pypy/objspace/std/model.py
+++ b/pypy/objspace/std/model.py
@@ -25,8 +25,6 @@
                         "ropeobject.W_RopeIterObject"],
     "withropeunicode": ["ropeunicodeobject.W_RopeUnicodeObject",
                         "ropeunicodeobject.W_RopeUnicodeIterObject"],
-    "withrangelist"  : ["rangeobject.W_RangeListObject",
-                        "rangeobject.W_RangeIterObject"],
     "withtproxy" : ["proxyobject.W_TransparentList",
                     "proxyobject.W_TransparentDict"],
 }
@@ -89,7 +87,6 @@
         from pypy.objspace.std import iterobject
         from pypy.objspace.std import unicodeobject
         from pypy.objspace.std import dictproxyobject
-        from pypy.objspace.std import rangeobject
         from pypy.objspace.std import proxyobject
         from pypy.objspace.std import fake
         import pypy.objspace.std.default # register a few catch-all multimethods
@@ -250,11 +247,6 @@
                 (unicodeobject.W_UnicodeObject,
                                        strbufobject.delegate_buf2unicode)
                 ]
-        if config.objspace.std.withrangelist:
-            self.typeorder[rangeobject.W_RangeListObject] += [
-                (listobject.W_ListObject,
-                                       rangeobject.delegate_range2list),
-                ]
         if config.objspace.std.withsmalltuple:
             self.typeorder[smalltupleobject.W_SmallTupleObject] += [
                 (tupleobject.W_TupleObject, smalltupleobject.delegate_SmallTuple2Tuple)]
diff --git a/pypy/objspace/std/rangeobject.py b/pypy/objspace/std/rangeobject.py
deleted file mode 100644
--- a/pypy/objspace/std/rangeobject.py
+++ /dev/null
@@ -1,226 +0,0 @@
-from pypy.interpreter.error import OperationError
-from pypy.objspace.std.model import registerimplementation, W_Object
-from pypy.objspace.std.register_all import register_all
-from pypy.objspace.std.multimethod import FailedToImplement
-from pypy.objspace.std.noneobject import W_NoneObject
-from pypy.objspace.std.inttype import wrapint
-from pypy.objspace.std.sliceobject import W_SliceObject, normalize_simple_slice
-from pypy.objspace.std.listobject import W_ListObject
-from pypy.objspace.std import listtype, iterobject, slicetype
-from pypy.interpreter import gateway, baseobjspace
-
-def length(start, stop, step):
-    if step > 0:
-        if stop <= start:
-            return 0
-        return (stop - start + step - 1)/step
-
-    else:  # step must be < 0
-        if stop >= start:
-            return 0
-        return (start - stop - step  - 1)/-step
-
-
-class W_RangeListObject(W_Object):
-    typedef = listtype.list_typedef
-
-    def __init__(w_self, start, step, length):
-        assert step != 0
-        w_self.start = start
-        w_self.step = step
-        w_self.length = length
-        w_self.w_list = None
-
-    def force(w_self, space):
-        if w_self.w_list is not None:
-            return w_self.w_list
-        start = w_self.start
-        step = w_self.step
-        length = w_self.length
-        if not length:
-            w_self.w_list = space.newlist([])
-            return w_self.w_list
-
-        arr = [None] * length  # this is to avoid using append.
-
-        i = start
-        n = 0
-        while n < length:
-            arr[n] = wrapint(space, i)
-            i += step
-            n += 1
-
-        w_self.w_list = space.newlist(arr)
-        return w_self.w_list
-
-    def getitem(w_self, i):
-        if i < 0:
-            i += w_self.length
-            if i < 0:
-                raise IndexError
-        elif i >= w_self.length:
-            raise IndexError
-        return w_self.start + i * w_self.step
-
-    def getitem_unchecked(w_self, i):
-        # bounds not checked, on purpose
-        return w_self.start + i * w_self.step
-
-    def __repr__(w_self):
-        if w_self.w_list is None:
-            return "W_RangeListObject(%s, %s, %s)" % (
-                w_self.start, w_self.step, w_self.length)
-        else:
-            return "W_RangeListObject(%r)" % (w_self.w_list, )
-
-def delegate_range2list(space, w_rangelist):
-    return w_rangelist.force(space)
-
-def len__RangeList(space, w_rangelist):
-    if w_rangelist.w_list is not None:
-        return space.len(w_rangelist.w_list)
-    return wrapint(space, w_rangelist.length)
-
-
-def getitem__RangeList_ANY(space, w_rangelist, w_index):
-    if w_rangelist.w_list is not None:
-        return space.getitem(w_rangelist.w_list, w_index)
-    idx = space.getindex_w(w_index, space.w_IndexError, "list index")
-    try:
-        return wrapint(space, w_rangelist.getitem(idx))
-    except IndexError:
-        raise OperationError(space.w_IndexError,
-                             space.wrap("list index out of range"))
-
-def getitem__RangeList_Slice(space, w_rangelist, w_slice):
-    if w_rangelist.w_list is not None:
-        return space.getitem(w_rangelist.w_list, w_slice)
-    length = w_rangelist.length
-    start, stop, step, slicelength = w_slice.indices4(space, length)
-    assert slicelength >= 0
-    rangestart = w_rangelist.getitem_unchecked(start)
-    rangestep = w_rangelist.step * step
-    return W_RangeListObject(rangestart, rangestep, slicelength)
-
-def iter__RangeList(space, w_rangelist):
-    return W_RangeIterObject(w_rangelist)
-
-def repr__RangeList(space, w_rangelist):
-    if w_rangelist.w_list is not None:
-        return space.repr(w_rangelist.w_list)
-    if w_rangelist.length == 0:
-        return space.wrap('[]')
-    result = [''] * w_rangelist.length
-    i = w_rangelist.start
-    n = 0
-    while n < w_rangelist.length:
-        result[n] = str(i)
-        i += w_rangelist.step
-        n += 1
-    return space.wrap("[" + ", ".join(result) + "]")
-
-def inplace_add__RangeList_ANY(space, w_rangelist, w_iterable2):
-    space.inplace_add(w_rangelist.force(space), w_iterable2)
-    return w_rangelist
-
-def inplace_mul__RangeList_ANY(space, w_rangelist, w_number):
-    space.inplace_mul(w_rangelist.force(space), w_number)
-    return w_rangelist
-
-
-def list_pop__RangeList_ANY(space, w_rangelist, w_idx=-1):
-    if w_rangelist.w_list is not None:
-        raise FailedToImplement
-    length = w_rangelist.length
-    if length == 0:
-        raise OperationError(space.w_IndexError,
-                             space.wrap("pop from empty list"))
-    if space.isinstance_w(w_idx, space.w_float):
-        raise OperationError(space.w_TypeError,
-            space.wrap("integer argument expected, got float")
-        )
-    idx = space.int_w(space.int(w_idx))
-    if idx == 0:
-        result = w_rangelist.start
-        w_rangelist.start += w_rangelist.step
-        w_rangelist.length -= 1
-        return wrapint(space, result)
-    if idx == -1 or idx == length - 1:
-        w_rangelist.length -= 1
-        return wrapint(
-            space, w_rangelist.start + (length - 1) * w_rangelist.step)
-    if idx >= w_rangelist.length:
-        raise OperationError(space.w_IndexError,
-                             space.wrap("pop index out of range"))
-    raise FailedToImplement
-
-def list_reverse__RangeList(space, w_rangelist):
-    # probably somewhat useless, but well...
-    if w_rangelist.w_list is not None:
-        raise FailedToImplement
-    w_rangelist.start = w_rangelist.getitem_unchecked(w_rangelist.length-1)
-    w_rangelist.step = -w_rangelist.step
-
-def list_sort__RangeList_None_None_ANY(space, w_rangelist, w_cmp,
-                                       w_keyfunc, w_reverse):
-    # even more useless but fun
-    has_reverse = space.is_true(w_reverse)
-    if w_rangelist.w_list is not None:
-        raise FailedToImplement
-    if has_reverse:
-        factor = -1
-    else:
-        factor = 1
-    reverse = w_rangelist.step * factor < 0
-    if reverse:
-        w_rangelist.start = w_rangelist.getitem_unchecked(w_rangelist.length-1)
-        w_rangelist.step = -w_rangelist.step
-    return space.w_None
-
-
-class W_RangeIterObject(iterobject.W_AbstractSeqIterObject):
-    pass
-
-def iter__RangeIter(space, w_rangeiter):
-    return w_rangeiter
-
-def next__RangeIter(space, w_rangeiter):
-    w_rangelist = w_rangeiter.w_seq
-    if w_rangelist is None:
-        raise OperationError(space.w_StopIteration, space.w_None)
-    assert isinstance(w_rangelist, W_RangeListObject)
-    index = w_rangeiter.index
-    if w_rangelist.w_list is not None:
-        try:
-            w_item = space.getitem(w_rangelist.w_list,
-                                   wrapint(space, index))
-        except OperationError, e:
-            w_rangeiter.w_seq = None
-            if not e.match(space, space.w_IndexError):
-                raise
-            raise OperationError(space.w_StopIteration, space.w_None)
-    else:
-        if index >= w_rangelist.length:
-            w_rangeiter.w_seq = None
-            raise OperationError(space.w_StopIteration, space.w_None)
-        w_item = wrapint(
-            space,
-            w_rangelist.getitem_unchecked(index))
-    w_rangeiter.index = index + 1
-    return w_item
-
-# XXX __length_hint__()
-##def len__RangeIter(space,  w_rangeiter):
-##    if w_rangeiter.w_seq is None:
-##        return wrapint(space, 0)
-##    index = w_rangeiter.index
-##    w_length = space.len(w_rangeiter.w_seq)
-##    w_len = space.sub(w_length, wrapint(space, index))
-##    if space.is_true(space.lt(w_len, wrapint(space, 0))):
-##        w_len = wrapint(space, 0)
-##    return w_len
-
-registerimplementation(W_RangeListObject)
-registerimplementation(W_RangeIterObject)
-
-register_all(vars(), listtype)
diff --git a/pypy/objspace/std/test/test_rangeobject.py b/pypy/objspace/std/test/test_rangeobject.py
deleted file mode 100644
--- a/pypy/objspace/std/test/test_rangeobject.py
+++ /dev/null
@@ -1,153 +0,0 @@
-import py
-
-from pypy.conftest import gettestobjspace, option
-
-class AppTestRangeListObject(object):
-
-    def setup_class(cls):
-        if option.runappdirect:
-            py.test.skip("__pypy__.internal_repr() cannot be used to see "
-                         "if a range list was forced on top of pypy-c")
-        cls.space = gettestobjspace(**{"objspace.std.withrangelist": True})
-        cls.w_not_forced = cls.space.appexec([], """():
-            import __pypy__
-            def f(r):
-                return (isinstance(r, list) and
-                        "W_ListObject" not in __pypy__.internal_repr(r))
-            return f
-        """)
-        cls.w_SORT_FORCES_LISTS = cls.space.wrap(False)
-
-    def test_simple(self):
-        result = []
-        r = range(1, 8, 2)
-        for i in r:
-            result.append(i)
-        assert result == [1, 3, 5, 7]
-        assert self.not_forced(r)
-
-    def test_getitem_slice(self):
-        result = []
-        r = range(1, 100, 2)
-        for i in r[10:15]:
-            result.append(i)
-        assert result == [21, 23, 25, 27, 29]
-        assert self.not_forced(r)
-
-    def test_getitem_extended_slice(self):
-        result = []
-        r = range(1, 100, 2)
-        for i in r[40:30:-2]:
-            result.append(i)
-        assert result == [81, 77, 73, 69, 65]
-        assert self.not_forced(r)
-
-    def test_empty_range(self):
-        r = range(10, 10)
-        if not self.SORT_FORCES_LISTS:
-            r.sort(reverse=True)
-        assert len(r) == 0
-        assert list(reversed(r)) == []
-        assert r[:] == []
-        assert self.not_forced(r)
-
-    def test_repr(self):
-        r = range(5)
-        assert repr(r) == "[0, 1, 2, 3, 4]"
-        assert self.not_forced(r)
-
-    def test_force(self):
-        r = range(10)
-        r[0] = 42
-        assert not self.not_forced(r)
-        assert r == [42, 1, 2, 3, 4, 5, 6, 7, 8, 9]
-
-    def test_reverse(self):
-        r = range(10)
-        r.reverse()
-        assert self.not_forced(r)
-        assert r == range(9, -1, -1)
-        r = range(3)
-        r[0] = 1
-        assert r == [1, 1, 2]
-        r.reverse()
-        assert r == [2, 1, 1]
-
-    def test_sort(self):
-        if self.SORT_FORCES_LISTS:
-            skip("sort() forces these lists")
-        r = range(10, -1, -1)
-        r.sort()
-        assert self.not_forced(r)
-        assert r == range(11)
-        r = range(11)
-        r.sort(reverse=True)
-        assert self.not_forced(r)
-        assert r == range(10, -1, -1)
-        r = range(100)
-        r[0] = 999
-        assert not self.not_forced(r)
-        r.sort()
-        assert r == range(1, 100) + [999]
-        r = range(10)
-        r.sort(key=lambda x: -x)
-        assert r == range(9, -1, -1)
-
-    def test_pop(self):
-        r = range(10)
-        res = r.pop()
-        assert res == 9
-        assert self.not_forced(r)
-        assert repr(r) == repr(range(9))
-        res = r.pop(0)
-        assert res == 0
-        assert self.not_forced(r)
-        assert repr(r) == repr(range(1, 9))
-        res = r.pop(len(r) - 1)
-        assert res == 8
-        assert self.not_forced(r)
-        assert repr(r) == repr(range(1, 8))
-        res = r.pop(2)
-        assert res == 3
-        assert not self.not_forced(r)
-        assert r == [1, 2, 4, 5, 6, 7]
-        res = r.pop(2)
-        assert res == 4
-        assert not self.not_forced(r)
-        assert r == [1, 2, 5, 6, 7]
-       
-    def test_reduce(self):
-        it = iter(range(10))
-        assert it.next() == 0
-        assert it.next() == 1
-        assert it.next() == 2
-        assert it.next() == 3
-        seqiter_new, args = it.__reduce__()
-        assert it.next() == 4
-        assert it.next() == 5
-        it2 = seqiter_new(*args)
-        assert it2.next() == 4
-        assert it2.next() == 5
-        it3 = seqiter_new(*args)
-        assert it3.next() == 4
-        assert it3.next() == 5
-
-    def test_no_len_on_range_iter(self):
-        iterable = range(10)
-        raises(TypeError, len, iter(iterable))
-
-    def test_inplace_add(self):
-        r1 = r2 = range(5)
-        assert r1 is r2
-        r1 += [15]
-        assert r1 is r2
-        assert r1 == [0, 1, 2, 3, 4, 15]
-        assert r2 == [0, 1, 2, 3, 4, 15]
-
-    def test_inplace_mul(self):
-        r1 = r2 = range(3)
-        assert r1 is r2
-        r1 *= 2
-        assert r1 is r2
-        assert r1 == [0, 1, 2, 0, 1, 2]
-        assert r2 == [0, 1, 2, 0, 1, 2]

From noreply at buildbot.pypy.org  Wed Oct 12 22:23:30 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 12 Oct 2011 22:23:30 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Fix usage of a removed opcode
Message-ID: <20111012202330.4D0DC82112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r47990:a522088e48aa
Date: 2011-10-12 22:19 +0200
http://bitbucket.org/pypy/pypy/changeset/a522088e48aa/

Log:	Fix usage of a removed opcode

diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -490,6 +490,7 @@
     ops.ROT_TWO : 0,
     ops.ROT_THREE : 0,
     ops.DUP_TOP : 1,
+    ops.DUP_TOP_TWO : 2,
 
     ops.UNARY_POSITIVE : 0,
     ops.UNARY_NEGATIVE : 0,
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -1108,7 +1108,7 @@
         else:
             raise AssertionError("unknown slice type")
         if ctx == ast.AugLoad:
-            self.emit_op_arg(ops.DUP_TOPX, 2)
+            self.emit_op(ops.DUP_TOP_TWO)
         elif ctx == ast.AugStore:
             self.emit_op(ops.ROT_THREE)
         self.emit_op(subscr_operations(ctx))

From noreply at buildbot.pypy.org  Wed Oct 12 22:23:31 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 12 Oct 2011 22:23:31 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: More xrange->range renames
Message-ID: <20111012202331.80D3E82112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r47991:c109778c3305
Date: 2011-10-12 22:19 +0200
http://bitbucket.org/pypy/pypy/changeset/c109778c3305/

Log:	More xrange->range renames

diff --git a/pypy/module/__builtin__/test/test_functional.py b/pypy/module/__builtin__/test/test_functional.py
--- a/pypy/module/__builtin__/test/test_functional.py
+++ b/pypy/module/__builtin__/test/test_functional.py
@@ -119,47 +119,47 @@
                return i * 10
        assert filter(lambda x: x != 20, T("abcd")) == (0, 10, 30)
 
-class AppTestXRange:
-   def test_xrange(self):
-      x = xrange(2, 9, 3)
+class AppTestRange:
+   def test_range(self):
+      x = range(2, 9, 3)
       assert x[1] == 5
       assert len(x) == 3
       assert list(x) == [2, 5, 8]
-      # test again, to make sure that xrange() is not its own iterator
+      # test again, to make sure that range() is not its own iterator
       assert list(x) == [2, 5, 8]
 
-   def test_xrange_iter(self):
-      x = xrange(2, 9, 3)
+   def test_range_iter(self):
+      x = range(2, 9, 3)
       it = iter(x)
       assert iter(it) is it
       assert it.next() == 2
       assert it.next() == 5
       assert it.next() == 8
       raises(StopIteration, it.next)
-      # test again, to make sure that xrange() is not its own iterator
+      # test again, to make sure that range() is not its own iterator
       assert iter(x).next() == 2
 
-   def test_xrange_object_with___int__(self):
+   def test_range_object_with___int__(self):
        class A(object):
           def __int__(self):
              return 5
 
-       assert list(xrange(A())) == [0, 1, 2, 3, 4]
-       assert list(xrange(0, A())) == [0, 1, 2, 3, 4]
-       assert list(xrange(0, 10, A())) == [0, 5]
+       assert list(range(A())) == [0, 1, 2, 3, 4]
+       assert list(range(0, A())) == [0, 1, 2, 3, 4]
+       assert list(range(0, 10, A())) == [0, 5]
  
-   def test_xrange_float(self):
-      assert list(xrange(0.1, 2.0, 1.1)) == [0, 1]
+   def test_range_float(self):
+      assert list(range(0.1, 2.0, 1.1)) == [0, 1]
 
-   def test_xrange_long(self):
+   def test_range_long(self):
        import sys
        a = long(10 * sys.maxint)
-       raises(OverflowError, xrange, a)
-       raises(OverflowError, xrange, 0, a)
-       raises(OverflowError, xrange, 0, 1, a)
+       raises(OverflowError, range, a)
+       raises(OverflowError, range, 0, a)
+       raises(OverflowError, range, 0, 1, a)
 
-   def test_xrange_reduce(self):
-      x = xrange(2, 9, 3)
+   def test_range_reduce(self):
+      x = range(2, 9, 3)
       callable, args = x.__reduce__()
       y = callable(*args)
       assert list(y) == list(x)
diff --git a/pypy/module/_pickle_support/__init__.py b/pypy/module/_pickle_support/__init__.py
--- a/pypy/module/_pickle_support/__init__.py
+++ b/pypy/module/_pickle_support/__init__.py
@@ -19,7 +19,7 @@
         'frame_new'    : 'maker.frame_new',
         'traceback_new' : 'maker.traceback_new',
         'generator_new' : 'maker.generator_new',
-        'xrangeiter_new': 'maker.xrangeiter_new',
+        'rangeiter_new': 'maker.rangeiter_new',
         'builtin_code': 'maker.builtin_code',
         'builtin_function' : 'maker.builtin_function',
         'enumerate_new': 'maker.enumerate_new',
diff --git a/pypy/module/_pickle_support/maker.py b/pypy/module/_pickle_support/maker.py
--- a/pypy/module/_pickle_support/maker.py
+++ b/pypy/module/_pickle_support/maker.py
@@ -67,9 +67,9 @@
     return space.wrap(new_generator)
 
 @unwrap_spec(current=int, remaining=int, step=int)
-def xrangeiter_new(space, current, remaining, step):
-    from pypy.module.__builtin__.functional import W_XRangeIterator
-    new_iter = W_XRangeIterator(space, current, remaining, step)
+def rangeiter_new(space, current, remaining, step):
+    from pypy.module.__builtin__.functional import W_RangeIterator
+    new_iter = W_RangeIterator(space, current, remaining, step)
     return space.wrap(new_iter)
 
 @unwrap_spec(identifier=str)
diff --git a/pypy/objspace/std/test/test_index.py b/pypy/objspace/std/test/test_index.py
--- a/pypy/objspace/std/test/test_index.py
+++ b/pypy/objspace/std/test/test_index.py
@@ -269,16 +269,16 @@
         self.w_const = self.space.appexec([], """(): return unicode""")
 
 
-class AppTest_XRangeTestCase:
+class AppTest_RangeTestCase:
 
-    def test_xrange(self):
+    def test_range(self):
         class newstyle(object):
             def __index__(self):
                 return self.ind
         n = newstyle()
         n.ind = 5
-        assert xrange(1, 20)[n] == 6
-        assert xrange(1, 20).__getitem__(n) == 6
+        assert range(1, 20)[n] == 6
+        assert range(1, 20).__getitem__(n) == 6
 
 class AppTest_OverflowTestCase:
 

From noreply at buildbot.pypy.org  Wed Oct 12 22:23:32 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 12 Oct 2011 22:23:32 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Rename '__builtin__' to 'builtins'
Message-ID: <20111012202332.BEAB182112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r47992:f64e3bd01d1f
Date: 2011-10-12 22:19 +0200
http://bitbucket.org/pypy/pypy/changeset/f64e3bd01d1f/

Log:	Rename '__builtin__' to 'builtins' Don't move the files: it will be
	easier to merge with the default branch

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -13,7 +13,7 @@
                and not p.basename.startswith('test')]
 
 essential_modules = dict.fromkeys(
-    ["exceptions", "_file", "sys", "__builtin__", "posix"]
+    ["exceptions", "_file", "sys", "builtins", "posix"]
 )
 
 default_modules = essential_modules.copy()
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -501,13 +501,13 @@
         mod.install()
 
         from pypy.module.__builtin__ import Module
-        w_name = self.wrap('__builtin__')
+        w_name = self.wrap('builtins')
         self.builtin = Module(self, w_name)
         w_builtin = self.wrap(self.builtin)
         w_builtin.install()
         self.setitem(self.builtin.w_dict, self.wrap('__builtins__'), w_builtin)
 
-        bootstrap_modules = set(('sys', 'imp', '__builtin__', 'exceptions'))
+        bootstrap_modules = set(('sys', 'imp', 'builtins', 'exceptions'))
         installed_builtin_modules = list(bootstrap_modules)
 
         exception_types_w = self.export_builtin_exceptions()
@@ -593,7 +593,7 @@
             self.fromcache(State).build_api(self)
         self.getbuiltinmodule('sys')
         self.getbuiltinmodule('imp')
-        self.getbuiltinmodule('__builtin__')
+        self.getbuiltinmodule('builtins')
         for mod in self.builtin_modules.values():
             mod.setup_after_space_initialization()
 
@@ -974,7 +974,7 @@
 
     # The code below only works
     # for the simple case (new-style instance).
-    # These methods are patched with the full logic by the __builtin__
+    # These methods are patched with the full logic by the builtins
     # module when it is loaded
 
     def abstract_issubclass_w(self, w_cls1, w_cls2):
@@ -1495,7 +1495,7 @@
     ]
 
 ObjSpace.BuiltinModuleTable = [
-    '__builtin__',
+    'builtins',
     'sys',
     ]
 
diff --git a/pypy/module/__builtin__/__init__.py b/pypy/module/__builtin__/__init__.py
--- a/pypy/module/__builtin__/__init__.py
+++ b/pypy/module/__builtin__/__init__.py
@@ -7,6 +7,8 @@
 
 class Module(MixedModule):
     """Built-in functions, exceptions, and other objects."""
+    applevel_name = 'builtins'
+
     expose__file__attribute = False
 
     appleveldefs = {

From noreply at buildbot.pypy.org  Wed Oct 12 22:23:33 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 12 Oct 2011 22:23:33 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Try to replace all strings with unicode
Message-ID: <20111012202333.F10AF82112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r47993:caa6a0233682
Date: 2011-10-12 22:19 +0200
http://bitbucket.org/pypy/pypy/changeset/caa6a0233682/

Log:	Try to replace all strings with unicode


From noreply at buildbot.pypy.org  Wed Oct 12 22:23:35 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 12 Oct 2011 22:23:35 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Update grammar and compiler to recognize
	(and skip for now)
Message-ID: <20111012202335.3B62B82112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r47994:16f4dc189f4e
Date: 2011-10-12 22:19 +0200
http://bitbucket.org/pypy/pypy/changeset/16f4dc189f4e/

Log:	Update grammar and compiler to recognize (and skip for now) the
	single * used to separate keywords-only arguments

diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -517,7 +517,7 @@
         while i < child_count:
             argument = arguments_node.children[i]
             arg_type = argument.type
-            if arg_type == syms.fpdef:
+            if arg_type == syms.tfpdef:
                 parenthesized = False
                 complex_args = False
                 while True:
@@ -554,12 +554,16 @@
                     break
             elif arg_type == tokens.STAR:
                 name_node = arguments_node.children[i + 1]
-                variable_arg = name_node.value
-                self.check_forbidden_name(variable_arg, name_node)
-                i += 3
+                if name_node.type == tokens.COMMA:
+                    # XXX for now
+                    i += 2
+                else:
+                    variable_arg = name_node.children[0].value
+                    self.check_forbidden_name(variable_arg, name_node)
+                    i += 3
             elif arg_type == tokens.DOUBLESTAR:
                 name_node = arguments_node.children[i + 1]
-                keywords_arg = name_node.value
+                keywords_arg = name_node.children[0].value
                 self.check_forbidden_name(keywords_arg, name_node)
                 i += 3
             else:
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -89,12 +89,12 @@
         yield self.st, func, "f(0)", 0
 
     def test_argtuple(self):
-        yield (self.simple_test, "def f( x, (y,z) ): return x,y,z",
-               "f((1,2),(3,4))", ((1,2),3,4))
-        yield (self.simple_test, "def f( x, (y,(z,t)) ): return x,y,z,t",
-               "f(1,(2,(3,4)))", (1,2,3,4))
-        yield (self.simple_test, "def f(((((x,),y),z),t),u): return x,y,z,t,u",
-               "f(((((1,),2),3),4),5)", (1,2,3,4,5))
+        yield (self.error_test, "def f( x, (y,z) ): return x,y,z",
+               SyntaxError)
+        yield (self.error_test, "def f( x, (y,(z,t)) ): return x,y,z,t",
+               SyntaxError)
+        yield (self.error_test, "def f(((((x,),y),z),t),u): return x,y,z,t,u",
+               SyntaxError)
 
     def test_constants(self):
         for c in expressions.constants:
@@ -236,6 +236,14 @@
         yield self.st, decl + "x=f(5, b=2, **{'a': 8})", "x", [5, ('a', 8),
                                                                   ('b', 2)]
 
+    def test_kwonly(self):
+        decl = py.code.Source("""
+            def f(a, *, b):
+                return a, b
+        """)
+        decl = str(decl) + '\n'
+        yield self.st, decl + "x=f(1, b=2)", "x", (1, 2)
+
     def test_listmakers(self):
         yield (self.st,
                "l = [(j, i) for j in range(10) for i in range(j)"
diff --git a/pypy/interpreter/pyparser/data/Grammar3.2 b/pypy/interpreter/pyparser/data/Grammar3.2
--- a/pypy/interpreter/pyparser/data/Grammar3.2
+++ b/pypy/interpreter/pyparser/data/Grammar3.2
@@ -23,12 +23,15 @@
 decorators: decorator+
 decorated: decorators (classdef | funcdef)
 funcdef: 'def' NAME parameters ':' suite
-parameters: '(' [varargslist] ')'
-varargslist: ((fpdef ['=' test] ',')*
-              ('*' NAME [',' '**' NAME] | '**' NAME) |
-              fpdef ['=' test] (',' fpdef ['=' test])* [','])
-fpdef: NAME | '(' fplist ')'
-fplist: fpdef (',' fpdef)* [',']
+parameters: '(' [typedargslist] ')'
+typedargslist: (tfpdef ['=' test] (',' tfpdef ['=' test])* [','
+       ['*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef]]
+     |  '*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef)
+tfpdef: NAME [':' test]
+varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [','
+       ['*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef]]
+     |  '*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef)
+vfpdef: NAME
 
 stmt: simple_stmt | compound_stmt
 simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE

From noreply at buildbot.pypy.org  Wed Oct 12 22:23:36 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 12 Oct 2011 22:23:36 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Fix almost all appdirect tests in
	test_unicodeobject.py
Message-ID: <20111012202336.7E60E82112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r47995:4da383a4bed3
Date: 2011-10-12 22:19 +0200
http://bitbucket.org/pypy/pypy/changeset/4da383a4bed3/

Log:	Fix almost all appdirect tests in test_unicodeobject.py

diff --git a/pypy/conftest.py b/pypy/conftest.py
--- a/pypy/conftest.py
+++ b/pypy/conftest.py
@@ -196,6 +196,9 @@
 
 def run_with_python(python, target):
     helpers = """if 1:
+    def skip(message):
+        print(message)
+        raise SystemExit(0)
     def raises(exc, func, *args, **kwargs):
         try:
             func(*args, **kwargs)
diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py
--- a/pypy/objspace/std/test/test_unicodeobject.py
+++ b/pypy/objspace/std/test/test_unicodeobject.py
@@ -2,46 +2,13 @@
 import sys
 from pypy.conftest import gettestobjspace
 
-class TestUnicodeObject:
-
-    def test_comparison_warning(self):
-        warnings = []
-        def my_warn(msg, warningscls):
-            warnings.append(msg)
-            prev_warn(msg, warningscls)
-        space = self.space
-        prev_warn = space.warn
-        try:
-            space.warn = my_warn
-            space.appexec([], """():
-                chr(128) == unichr(128)
-                chr(128) != unichr(128)
-                chr(127) == unichr(127) # no warnings
-            """)
-        finally:
-            space.warn = prev_warn
-        print self.space.config.objspace.std.withrope
-        assert len(warnings) == 2
-
-
 class AppTestUnicodeStringStdOnly:
     def test_compares(self):
-        assert u'a' == 'a'
-        assert 'a' == u'a'
-        assert not u'a' == 'b'
-        assert not 'a'  == u'b'
-        assert u'a' != 'b'
-        assert 'a'  != u'b'
-        assert not (u'a' == 5)
-        assert u'a' != 5
-        assert u'a' < 5 or u'a' > 5
-
-        s = chr(128)
-        u = unichr(128)
-        assert not s == u # UnicodeWarning
-        assert s != u
-        assert not u == s
-        assert u != s
+        assert 'a' != b'a'
+        assert b'a' != 'a'
+        assert not ('a' == 5)
+        assert 'a' != 5
+        raises(TypeError, "'a' < 5")
         
 
 class AppTestUnicodeString:
@@ -51,415 +18,401 @@
         cls.w_version_info = cls.space.wrap(sys.version_info)
 
     def test_addition(self):
-        def check(a, b):
-            assert a == b
-            assert type(a) == type(b)
-        check(u'a' + 'b', u'ab')
-        check('a' + u'b', u'ab')
+        import operator
+        assert 'a' + 'b' == 'ab'
+        raises(TypeError, operator.add, b'a', 'b')
 
     def test_join(self):
         def check(a, b):
             assert a == b
             assert type(a) == type(b)
-        check(', '.join([u'a']), u'a')
-        check(', '.join(['a', u'b']), u'a, b')
-        check(u', '.join(['a', 'b']), u'a, b')
-
-    if sys.version_info >= (2,3):
-        def test_contains_ex(self):
-            assert u'' in 'abc'
-            assert u'bc' in 'abc'
-            assert 'bc' in 'abc'
-        pass   # workaround for inspect.py bug in some Python 2.4s
+        check(', '.join(['a']), 'a')
 
     def test_contains(self):
-        assert u'a' in 'abc'
-        assert 'a' in u'abc'
+        assert '' in 'abc'
+        assert 'a' in 'abc'
+        assert 'bc' in 'abc'
 
     def test_splitlines(self):
-        assert u''.splitlines() == []
-        assert u''.splitlines(1) == []
-        assert u'\n'.splitlines() == [u'']
-        assert u'a'.splitlines() == [u'a']
-        assert u'one\ntwo'.splitlines() == [u'one', u'two']
-        assert u'\ntwo\nthree'.splitlines() == [u'', u'two', u'three']
-        assert u'\n\n'.splitlines() == [u'', u'']
-        assert u'a\nb\nc'.splitlines(1) == [u'a\n', u'b\n', u'c']
-        assert u'\na\nb\n'.splitlines(1) == [u'\n', u'a\n', u'b\n']
+        assert ''.splitlines() == []
+        assert ''.splitlines(1) == []
+        assert '\n'.splitlines() == ['']
+        assert 'a'.splitlines() == ['a']
+        assert 'one\ntwo'.splitlines() == ['one', 'two']
+        assert '\ntwo\nthree'.splitlines() == ['', 'two', 'three']
+        assert '\n\n'.splitlines() == ['', '']
+        assert 'a\nb\nc'.splitlines(1) == ['a\n', 'b\n', 'c']
+        assert '\na\nb\n'.splitlines(1) == ['\n', 'a\n', 'b\n']
 
     def test_zfill(self):
-        assert u'123'.zfill(2) == u'123'
-        assert u'123'.zfill(3) == u'123'
-        assert u'123'.zfill(4) == u'0123'
-        assert u'123'.zfill(6) == u'000123'
-        assert u'+123'.zfill(2) == u'+123'
-        assert u'+123'.zfill(3) == u'+123'
-        assert u'+123'.zfill(4) == u'+123'
-        assert u'+123'.zfill(5) == u'+0123'
-        assert u'+123'.zfill(6) == u'+00123'
-        assert u'-123'.zfill(3) == u'-123'
-        assert u'-123'.zfill(4) == u'-123'
-        assert u'-123'.zfill(5) == u'-0123'
-        assert u''.zfill(3) == u'000'
-        assert u'34'.zfill(1) == u'34'
-        assert u'34'.zfill(4) == u'0034'
+        assert '123'.zfill(2) == '123'
+        assert '123'.zfill(3) == '123'
+        assert '123'.zfill(4) == '0123'
+        assert '123'.zfill(6) == '000123'
+        assert '+123'.zfill(2) == '+123'
+        assert '+123'.zfill(3) == '+123'
+        assert '+123'.zfill(4) == '+123'
+        assert '+123'.zfill(5) == '+0123'
+        assert '+123'.zfill(6) == '+00123'
+        assert '-123'.zfill(3) == '-123'
+        assert '-123'.zfill(4) == '-123'
+        assert '-123'.zfill(5) == '-0123'
+        assert ''.zfill(3) == '000'
+        assert '34'.zfill(1) == '34'
+        assert '34'.zfill(4) == '0034'
 
     def test_split(self):
-        assert u"".split() == []
-        assert u"".split(u'x') == ['']
-        assert u" ".split() == []
-        assert u"a".split() == [u'a']
-        assert u"a".split(u"a", 1) == [u'', u'']
-        assert u" ".split(u" ", 1) == [u'', u'']
-        assert u"aa".split(u"a", 2) == [u'', u'', u'']
-        assert u" a ".split() == [u'a']
-        assert u"a b c".split() == [u'a',u'b',u'c']
-        assert u'this is the split function'.split() == [u'this', u'is', u'the', u'split', u'function']
-        assert u'a|b|c|d'.split(u'|') == [u'a', u'b', u'c', u'd']
-        assert 'a|b|c|d'.split(u'|') == [u'a', u'b', u'c', u'd']
-        assert u'a|b|c|d'.split('|') == [u'a', u'b', u'c', u'd']
-        assert u'a|b|c|d'.split(u'|', 2) == [u'a', u'b', u'c|d']
-        assert u'a b c d'.split(None, 1) == [u'a', u'b c d']
-        assert u'a b c d'.split(None, 2) == [u'a', u'b', u'c d']
-        assert u'a b c d'.split(None, 3) == [u'a', u'b', u'c', u'd']
-        assert u'a b c d'.split(None, 4) == [u'a', u'b', u'c', u'd']
-        assert u'a b c d'.split(None, 0) == [u'a b c d']
-        assert u'a  b  c  d'.split(None, 2) == [u'a', u'b', u'c  d']
-        assert u'a b c d '.split() == [u'a', u'b', u'c', u'd']
-        assert u'a//b//c//d'.split(u'//') == [u'a', u'b', u'c', u'd']
-        assert u'endcase test'.split(u'test') == [u'endcase ', u'']
-        raises(ValueError, u'abc'.split, '')
-        raises(ValueError, u'abc'.split, u'')
-        raises(ValueError, 'abc'.split, u'')
-        assert u'   a b c d'.split(None, 0) == [u'a b c d']
+        assert "".split() == []
+        assert "".split('x') == ['']
+        assert " ".split() == []
+        assert "a".split() == ['a']
+        assert "a".split("a", 1) == ['', '']
+        assert " ".split(" ", 1) == ['', '']
+        assert "aa".split("a", 2) == ['', '', '']
+        assert " a ".split() == ['a']
+        assert "a b c".split() == ['a','b','c']
+        assert 'this is the split function'.split() == ['this', 'is', 'the', 'split', 'function']
+        assert 'a|b|c|d'.split('|') == ['a', 'b', 'c', 'd']
+        assert 'a|b|c|d'.split('|') == ['a', 'b', 'c', 'd']
+        assert 'a|b|c|d'.split('|') == ['a', 'b', 'c', 'd']
+        assert 'a|b|c|d'.split('|', 2) == ['a', 'b', 'c|d']
+        assert 'a b c d'.split(None, 1) == ['a', 'b c d']
+        assert 'a b c d'.split(None, 2) == ['a', 'b', 'c d']
+        assert 'a b c d'.split(None, 3) == ['a', 'b', 'c', 'd']
+        assert 'a b c d'.split(None, 4) == ['a', 'b', 'c', 'd']
+        assert 'a b c d'.split(None, 0) == ['a b c d']
+        assert 'a  b  c  d'.split(None, 2) == ['a', 'b', 'c  d']
+        assert 'a b c d '.split() == ['a', 'b', 'c', 'd']
+        assert 'a//b//c//d'.split('//') == ['a', 'b', 'c', 'd']
+        assert 'endcase test'.split('test') == ['endcase ', '']
+        raises(ValueError, 'abc'.split, '')
+        raises(ValueError, 'abc'.split, '')
+        raises(ValueError, 'abc'.split, '')
+        assert '   a b c d'.split(None, 0) == ['a b c d']
 
     def test_rsplit(self):
-        assert u"".rsplit() == []
-        assert u" ".rsplit() == []
-        assert u"a".rsplit() == [u'a']
-        assert u"a".rsplit(u"a", 1) == [u'', u'']
-        assert u" ".rsplit(u" ", 1) == [u'', u'']
-        assert u"aa".rsplit(u"a", 2) == [u'', u'', u'']
-        assert u" a ".rsplit() == [u'a']
-        assert u"a b c".rsplit() == [u'a',u'b',u'c']
-        assert u'this is the rsplit function'.rsplit() == [u'this', u'is', u'the', u'rsplit', u'function']
-        assert u'a|b|c|d'.rsplit(u'|') == [u'a', u'b', u'c', u'd']
-        assert u'a|b|c|d'.rsplit('|') == [u'a', u'b', u'c', u'd']
-        assert 'a|b|c|d'.rsplit(u'|') == [u'a', u'b', u'c', u'd']
-        assert u'a|b|c|d'.rsplit(u'|', 2) == [u'a|b', u'c', u'd']
-        assert u'a b c d'.rsplit(None, 1) == [u'a b c', u'd']
-        assert u'a b c d'.rsplit(None, 2) == [u'a b', u'c', u'd']
-        assert u'a b c d'.rsplit(None, 3) == [u'a', u'b', u'c', u'd']
-        assert u'a b c d'.rsplit(None, 4) == [u'a', u'b', u'c', u'd']
-        assert u'a b c d'.rsplit(None, 0) == [u'a b c d']
-        assert u'a  b  c  d'.rsplit(None, 2) == [u'a  b', u'c', u'd']
-        assert u'a b c d '.rsplit() == [u'a', u'b', u'c', u'd']
-        assert u'a//b//c//d'.rsplit(u'//') == [u'a', u'b', u'c', u'd']
-        assert u'endcase test'.rsplit(u'test') == [u'endcase ', u'']
-        raises(ValueError, u'abc'.rsplit, u'')
-        raises(ValueError, u'abc'.rsplit, '')
-        raises(ValueError, 'abc'.rsplit, u'')
-        assert u'  a b c  '.rsplit(None, 0) == [u'  a b c']
-        assert u''.rsplit('aaa') == [u'']
+        assert "".rsplit() == []
+        assert " ".rsplit() == []
+        assert "a".rsplit() == ['a']
+        assert "a".rsplit("a", 1) == ['', '']
+        assert " ".rsplit(" ", 1) == ['', '']
+        assert "aa".rsplit("a", 2) == ['', '', '']
+        assert " a ".rsplit() == ['a']
+        assert "a b c".rsplit() == ['a','b','c']
+        assert 'this is the rsplit function'.rsplit() == ['this', 'is', 'the', 'rsplit', 'function']
+        assert 'a|b|c|d'.rsplit('|') == ['a', 'b', 'c', 'd']
+        assert 'a|b|c|d'.rsplit('|') == ['a', 'b', 'c', 'd']
+        assert 'a|b|c|d'.rsplit('|') == ['a', 'b', 'c', 'd']
+        assert 'a|b|c|d'.rsplit('|', 2) == ['a|b', 'c', 'd']
+        assert 'a b c d'.rsplit(None, 1) == ['a b c', 'd']
+        assert 'a b c d'.rsplit(None, 2) == ['a b', 'c', 'd']
+        assert 'a b c d'.rsplit(None, 3) == ['a', 'b', 'c', 'd']
+        assert 'a b c d'.rsplit(None, 4) == ['a', 'b', 'c', 'd']
+        assert 'a b c d'.rsplit(None, 0) == ['a b c d']
+        assert 'a  b  c  d'.rsplit(None, 2) == ['a  b', 'c', 'd']
+        assert 'a b c d '.rsplit() == ['a', 'b', 'c', 'd']
+        assert 'a//b//c//d'.rsplit('//') == ['a', 'b', 'c', 'd']
+        assert 'endcase test'.rsplit('test') == ['endcase ', '']
+        raises(ValueError, 'abc'.rsplit, '')
+        raises(ValueError, 'abc'.rsplit, '')
+        raises(ValueError, 'abc'.rsplit, '')
+        assert '  a b c  '.rsplit(None, 0) == ['  a b c']
+        assert ''.rsplit('aaa') == ['']
 
     def test_center(self):
-        s=u"a b"
-        assert s.center(0) == u"a b"
-        assert s.center(1) == u"a b"
-        assert s.center(2) == u"a b"
-        assert s.center(3) == u"a b"
-        assert s.center(4) == u"a b "
-        assert s.center(5) == u" a b "
-        assert s.center(6) == u" a b  "
-        assert s.center(7) == u"  a b  "
-        assert s.center(8) == u"  a b   "
-        assert s.center(9) == u"   a b   "
-        assert u'abc'.center(10) == u'   abc    '
-        assert u'abc'.center(6) == u' abc  '
-        assert u'abc'.center(3) == u'abc'
-        assert u'abc'.center(2) == u'abc'
-        assert u'abc'.center(5, u'*') == u'*abc*'    # Python 2.4
-        assert u'abc'.center(5, '*') == u'*abc*'     # Python 2.4
-        raises(TypeError, u'abc'.center, 4, u'cba')
+        s="a b"
+        assert s.center(0) == "a b"
+        assert s.center(1) == "a b"
+        assert s.center(2) == "a b"
+        assert s.center(3) == "a b"
+        assert s.center(4) == "a b "
+        assert s.center(5) == " a b "
+        assert s.center(6) == " a b  "
+        assert s.center(7) == "  a b  "
+        assert s.center(8) == "  a b   "
+        assert s.center(9) == "   a b   "
+        assert 'abc'.center(10) == '   abc    '
+        assert 'abc'.center(6) == ' abc  '
+        assert 'abc'.center(3) == 'abc'
+        assert 'abc'.center(2) == 'abc'
+        assert 'abc'.center(5, '*') == '*abc*'    # Python 2.4
+        assert 'abc'.center(5, '*') == '*abc*'     # Python 2.4
+        raises(TypeError, 'abc'.center, 4, 'cba')
 
     def test_title(self):
-        assert u"brown fox".title() == u"Brown Fox"
-        assert u"!brown fox".title() == u"!Brown Fox"
-        assert u"bROWN fOX".title() == u"Brown Fox"
-        assert u"Brown Fox".title() == u"Brown Fox"
-        assert u"bro!wn fox".title() == u"Bro!Wn Fox"
+        assert "brown fox".title() == "Brown Fox"
+        assert "!brown fox".title() == "!Brown Fox"
+        assert "bROWN fOX".title() == "Brown Fox"
+        assert "Brown Fox".title() == "Brown Fox"
+        assert "bro!wn fox".title() == "Bro!Wn Fox"
 
     def test_istitle(self):
-        assert u"".istitle() == False
-        assert u"!".istitle() == False
-        assert u"!!".istitle() == False
-        assert u"brown fox".istitle() == False
-        assert u"!brown fox".istitle() == False
-        assert u"bROWN fOX".istitle() == False
-        assert u"Brown Fox".istitle() == True
-        assert u"bro!wn fox".istitle() == False
-        assert u"Bro!wn fox".istitle() == False
-        assert u"!brown Fox".istitle() == False
-        assert u"!Brown Fox".istitle() == True
-        assert u"Brow&&&&N Fox".istitle() == True
-        assert u"!Brow&&&&n Fox".istitle() == False
+        assert "".istitle() == False
+        assert "!".istitle() == False
+        assert "!!".istitle() == False
+        assert "brown fox".istitle() == False
+        assert "!brown fox".istitle() == False
+        assert "bROWN fOX".istitle() == False
+        assert "Brown Fox".istitle() == True
+        assert "bro!wn fox".istitle() == False
+        assert "Bro!wn fox".istitle() == False
+        assert "!brown Fox".istitle() == False
+        assert "!Brown Fox".istitle() == True
+        assert "Brow&&&&N Fox".istitle() == True
+        assert "!Brow&&&&n Fox".istitle() == False
         
     def test_capitalize(self):
-        assert u"brown fox".capitalize() == u"Brown fox"
-        assert u' hello '.capitalize() == u' hello '
-        assert u'Hello '.capitalize() == u'Hello '
-        assert u'hello '.capitalize() == u'Hello '
-        assert u'aaaa'.capitalize() == u'Aaaa'
-        assert u'AaAa'.capitalize() == u'Aaaa'
+        assert "brown fox".capitalize() == "Brown fox"
+        assert ' hello '.capitalize() == ' hello '
+        assert 'Hello '.capitalize() == 'Hello '
+        assert 'hello '.capitalize() == 'Hello '
+        assert 'aaaa'.capitalize() == 'Aaaa'
+        assert 'AaAa'.capitalize() == 'Aaaa'
         # check that titlecased chars are lowered correctly
         # \u1ffc is the titlecased char
-        assert (u'\u1ff3\u1ff3\u1ffc\u1ffc'.capitalize() ==
-                u'\u1ffc\u1ff3\u1ff3\u1ff3')
+        assert ('\u1ff3\u1ff3\u1ffc\u1ffc'.capitalize() ==
+                '\u1ffc\u1ff3\u1ff3\u1ff3')
         # check with cased non-letter chars
-        assert (u'\u24c5\u24ce\u24c9\u24bd\u24c4\u24c3'.capitalize() ==
-                u'\u24c5\u24e8\u24e3\u24d7\u24de\u24dd')
-        assert (u'\u24df\u24e8\u24e3\u24d7\u24de\u24dd'.capitalize() ==
-                u'\u24c5\u24e8\u24e3\u24d7\u24de\u24dd')
-        assert u'\u2160\u2161\u2162'.capitalize() == u'\u2160\u2171\u2172'
-        assert u'\u2170\u2171\u2172'.capitalize() == u'\u2160\u2171\u2172'
+        assert ('\u24c5\u24ce\u24c9\u24bd\u24c4\u24c3'.capitalize() ==
+                '\u24c5\u24e8\u24e3\u24d7\u24de\u24dd')
+        assert ('\u24df\u24e8\u24e3\u24d7\u24de\u24dd'.capitalize() ==
+                '\u24c5\u24e8\u24e3\u24d7\u24de\u24dd')
+        assert '\u2160\u2161\u2162'.capitalize() == '\u2160\u2171\u2172'
+        assert '\u2170\u2171\u2172'.capitalize() == '\u2160\u2171\u2172'
         # check with Ll chars with no upper - nothing changes here
-        assert (u'\u019b\u1d00\u1d86\u0221\u1fb7'.capitalize() ==
-                u'\u019b\u1d00\u1d86\u0221\u1fb7')
+        assert ('\u019b\u1d00\u1d86\u0221\u1fb7'.capitalize() ==
+                '\u019b\u1d00\u1d86\u0221\u1fb7')
 
     def test_rjust(self):
-        s = u"abc"
+        s = "abc"
         assert s.rjust(2) == s
         assert s.rjust(3) == s
-        assert s.rjust(4) == u" " + s
-        assert s.rjust(5) == u"  " + s
-        assert u'abc'.rjust(10) == u'       abc'
-        assert u'abc'.rjust(6) == u'   abc'
-        assert u'abc'.rjust(3) == u'abc'
-        assert u'abc'.rjust(2) == u'abc'
-        assert u'abc'.rjust(5, u'*') == u'**abc'    # Python 2.4
-        assert u'abc'.rjust(5, '*') == u'**abc'     # Python 2.4
-        raises(TypeError, u'abc'.rjust, 5, u'xx')
+        assert s.rjust(4) == " " + s
+        assert s.rjust(5) == "  " + s
+        assert 'abc'.rjust(10) == '       abc'
+        assert 'abc'.rjust(6) == '   abc'
+        assert 'abc'.rjust(3) == 'abc'
+        assert 'abc'.rjust(2) == 'abc'
+        assert 'abc'.rjust(5, '*') == '**abc'    # Python 2.4
+        assert 'abc'.rjust(5, '*') == '**abc'     # Python 2.4
+        raises(TypeError, 'abc'.rjust, 5, 'xx')
 
     def test_ljust(self):
-        s = u"abc"
+        s = "abc"
         assert s.ljust(2) == s
         assert s.ljust(3) == s
-        assert s.ljust(4) == s + u" "
-        assert s.ljust(5) == s + u"  "
-        assert u'abc'.ljust(10) == u'abc       '
-        assert u'abc'.ljust(6) == u'abc   '
-        assert u'abc'.ljust(3) == u'abc'
-        assert u'abc'.ljust(2) == u'abc'
-        assert u'abc'.ljust(5, u'*') == u'abc**'    # Python 2.4
-        assert u'abc'.ljust(5, '*') == u'abc**'     # Python 2.4
-        raises(TypeError, u'abc'.ljust, 6, u'')
+        assert s.ljust(4) == s + " "
+        assert s.ljust(5) == s + "  "
+        assert 'abc'.ljust(10) == 'abc       '
+        assert 'abc'.ljust(6) == 'abc   '
+        assert 'abc'.ljust(3) == 'abc'
+        assert 'abc'.ljust(2) == 'abc'
+        assert 'abc'.ljust(5, '*') == 'abc**'    # Python 2.4
+        assert 'abc'.ljust(5, '*') == 'abc**'     # Python 2.4
+        raises(TypeError, 'abc'.ljust, 6, '')
 
     def test_replace(self):
-        assert u'one!two!three!'.replace(u'!', '@', 1) == u'one at two!three!'
-        assert u'one!two!three!'.replace('!', u'') == u'onetwothree'
-        assert u'one!two!three!'.replace(u'!', u'@', 2) == u'one at two@three!'
-        assert u'one!two!three!'.replace('!', '@', 3) == u'one at two@three@'
-        assert u'one!two!three!'.replace(u'!', '@', 4) == u'one at two@three@'
-        assert u'one!two!three!'.replace('!', u'@', 0) == u'one!two!three!'
-        assert u'one!two!three!'.replace(u'!', u'@') == u'one at two@three@'
-        assert u'one!two!three!'.replace('x', '@') == u'one!two!three!'
-        assert u'one!two!three!'.replace(u'x', '@', 2) == u'one!two!three!'
-        assert u'abc'.replace('', u'-') == u'-a-b-c-'
-        assert u'abc'.replace(u'', u'-', 3) == u'-a-b-c'
-        assert u'abc'.replace('', '-', 0) == u'abc'
-        assert u''.replace(u'', '') == u''
-        assert u''.replace('', u'a') == u'a'
-        assert u'abc'.replace(u'ab', u'--', 0) == u'abc'
-        assert u'abc'.replace('xy', '--') == u'abc'
-        assert u'123'.replace(u'123', '') == u''
-        assert u'123123'.replace('123', u'') == u''
-        assert u'123x123'.replace(u'123', u'') == u'x'
-
-    def test_replace_buffer(self):
-        assert u'one!two!three'.replace(buffer('!'), buffer('@')) == u'one at two@three'
+        assert 'one!two!three!'.replace('!', '@', 1) == 'one at two!three!'
+        assert 'one!two!three!'.replace('!', '') == 'onetwothree'
+        assert 'one!two!three!'.replace('!', '@', 2) == 'one at two@three!'
+        assert 'one!two!three!'.replace('!', '@', 3) == 'one at two@three@'
+        assert 'one!two!three!'.replace('!', '@', 4) == 'one at two@three@'
+        assert 'one!two!three!'.replace('!', '@', 0) == 'one!two!three!'
+        assert 'one!two!three!'.replace('!', '@') == 'one at two@three@'
+        assert 'one!two!three!'.replace('x', '@') == 'one!two!three!'
+        assert 'one!two!three!'.replace('x', '@', 2) == 'one!two!three!'
+        assert 'abc'.replace('', '-') == '-a-b-c-'
+        assert 'abc'.replace('', '-', 3) == '-a-b-c'
+        assert 'abc'.replace('', '-', 0) == 'abc'
+        assert ''.replace('', '') == ''
+        assert ''.replace('', 'a') == 'a'
+        assert 'abc'.replace('ab', '--', 0) == 'abc'
+        assert 'abc'.replace('xy', '--') == 'abc'
+        assert '123'.replace('123', '') == ''
+        assert '123123'.replace('123', '') == ''
+        assert '123x123'.replace('123', '') == 'x'
 
     def test_replace_overflow(self):
         import sys
-        if sys.maxint > 2**31-1:
+        if sys.maxsize > 2**31-1:
             skip("Wrong platform")
-        s = u"a" * (2**16)
-        raises(OverflowError, s.replace, u"", s)
+        s = "a" * (2**16)
+        raises(OverflowError, s.replace, "", s)
 
     def test_strip(self):
-        s = u" a b "
-        assert s.strip() == u"a b"
-        assert s.rstrip() == u" a b"
-        assert s.lstrip() == u"a b "
-        assert u'xyzzyhelloxyzzy'.strip(u'xyz') == u'hello'
-        assert u'xyzzyhelloxyzzy'.lstrip('xyz') == u'helloxyzzy'
-        assert u'xyzzyhelloxyzzy'.rstrip(u'xyz') == u'xyzzyhello'
+        s = " a b "
+        assert s.strip() == "a b"
+        assert s.rstrip() == " a b"
+        assert s.lstrip() == "a b "
+        assert 'xyzzyhelloxyzzy'.strip('xyz') == 'hello'
+        assert 'xyzzyhelloxyzzy'.lstrip('xyz') == 'helloxyzzy'
+        assert 'xyzzyhelloxyzzy'.rstrip('xyz') == 'xyzzyhello'
 
 
     def test_long_from_unicode(self):
-        assert long(u'12345678901234567890') == 12345678901234567890
-        assert int(u'12345678901234567890') == 12345678901234567890
+        assert int('12345678901234567890') == 12345678901234567890
 
     def test_int_from_unicode(self):
-        assert int(u'12345') == 12345
+        assert int('12345') == 12345
 
     def test_float_from_unicode(self):
-        assert float(u'123.456e89') == float('123.456e89')
+        assert float('123.456e89') == float('123.456e89')
 
     def test_repr_16bits(self):
         # this used to fail when run on a CPython host with 16-bit unicodes
-        s = repr(u'\U00101234')
-        assert s == "u'\\U00101234'"
+        s = repr('\U00101234')
+        assert s == "'\\U00101234'"
 
     def test_repr(self):
-        for ustr in [u"", u"a", u"'", u"\'", u"\"", u"\t", u"\\", u'',
-                     u'a', u'"', u'\'', u'\"', u'\t', u'\\', u"'''\"",
-                     unichr(19), unichr(2), u'\u1234', u'\U00101234']:
+        for ustr in ["", "a", "'", "\'", "\"", "\t", "\\", '',
+                     'a', '"', '\'', '\"', '\t', '\\', "'''\"",
+                     chr(19), chr(2), '\u1234', '\U00101234']:
             assert eval(repr(ustr)) == ustr
             
     def test_getnewargs(self):
-        class X(unicode):
+        class X(str):
             pass
-        x = X(u"foo\u1234")
+        x = X("foo\u1234")
         a = x.__getnewargs__()
-        assert a == (u"foo\u1234",)
-        assert type(a[0]) is unicode
+        assert a == ("foo\u1234",)
+        assert type(a[0]) is str
 
     def test_call_unicode(self):
-        assert unicode() == u''
-        assert unicode(None) == u'None'
-        assert unicode(123) == u'123'
-        assert unicode([2, 3]) == u'[2, 3]'
-        class U(unicode):
+        assert str() == ''
+        assert str(None) == 'None'
+        assert str(123) == '123'
+        assert str([2, 3]) == '[2, 3]'
+        class U(str):
             pass
-        assert unicode(U()).__class__ is unicode
-        assert U(u'test') == u'test'
-        assert U(u'test').__class__ is U
+        assert str(U()).__class__ is str
+        assert U('test') == 'test'
+        assert U('test').__class__ is U
 
     def test_call_unicode_2(self):
         class X(object):
-            def __unicode__(self):
-                return u'x'
-        raises(TypeError, unicode, X(), 'ascii')
+            def __bytes__(self):
+                return b'x'
+        raises(TypeError, str, X(), 'ascii')
 
     def test_startswith(self):
-        assert u'ab'.startswith(u'ab') is True
-        assert u'ab'.startswith(u'a') is True
-        assert u'ab'.startswith(u'') is True
-        assert u'x'.startswith(u'a') is False
-        assert u'x'.startswith(u'x') is True
-        assert u''.startswith(u'') is True
-        assert u''.startswith(u'a') is False
-        assert u'x'.startswith(u'xx') is False
-        assert u'y'.startswith(u'xx') is False
+        assert 'ab'.startswith('ab') is True
+        assert 'ab'.startswith('a') is True
+        assert 'ab'.startswith('') is True
+        assert 'x'.startswith('a') is False
+        assert 'x'.startswith('x') is True
+        assert ''.startswith('') is True
+        assert ''.startswith('a') is False
+        assert 'x'.startswith('xx') is False
+        assert 'y'.startswith('xx') is False
 
     def test_startswith_more(self):
-        assert u'ab'.startswith(u'a', 0) is True
-        assert u'ab'.startswith(u'a', 1) is False
-        assert u'ab'.startswith(u'b', 1) is True
-        assert u'abc'.startswith(u'bc', 1, 2) is False
-        assert u'abc'.startswith(u'c', -1, 4) is True
+        assert 'ab'.startswith('a', 0) is True
+        assert 'ab'.startswith('a', 1) is False
+        assert 'ab'.startswith('b', 1) is True
+        assert 'abc'.startswith('bc', 1, 2) is False
+        assert 'abc'.startswith('c', -1, 4) is True
 
     def test_startswith_tuples(self):
-        assert u'hello'.startswith((u'he', u'ha'))
-        assert not u'hello'.startswith((u'lo', u'llo'))
-        assert u'hello'.startswith((u'hellox', u'hello'))
-        assert not u'hello'.startswith(())
-        assert u'helloworld'.startswith((u'hellowo', u'rld', u'lowo'), 3)
-        assert not u'helloworld'.startswith((u'hellowo', u'ello', u'rld'), 3)
-        assert u'hello'.startswith((u'lo', u'he'), 0, -1)
-        assert not u'hello'.startswith((u'he', u'hel'), 0, 1)
-        assert u'hello'.startswith((u'he', u'hel'), 0, 2)
-        raises(TypeError, u'hello'.startswith, (42,))
+        assert 'hello'.startswith(('he', 'ha'))
+        assert not 'hello'.startswith(('lo', 'llo'))
+        assert 'hello'.startswith(('hellox', 'hello'))
+        assert not 'hello'.startswith(())
+        assert 'helloworld'.startswith(('hellowo', 'rld', 'lowo'), 3)
+        assert not 'helloworld'.startswith(('hellowo', 'ello', 'rld'), 3)
+        assert 'hello'.startswith(('lo', 'he'), 0, -1)
+        assert not 'hello'.startswith(('he', 'hel'), 0, 1)
+        assert 'hello'.startswith(('he', 'hel'), 0, 2)
+        raises(TypeError, 'hello'.startswith, (42,))
 
     def test_startswith_endswith_convert(self):
-        assert 'hello'.startswith((u'he\u1111', u'he'))
-        assert not 'hello'.startswith((u'lo\u1111', u'llo'))
-        assert 'hello'.startswith((u'hellox\u1111', u'hello'))
-        assert not 'hello'.startswith((u'lo', u'he\u1111'), 0, -1)
-        assert not 'hello'.endswith((u'he\u1111', u'he'))
-        assert 'hello'.endswith((u'\u1111lo', u'llo'))
-        assert 'hello'.endswith((u'\u1111hellox', u'hello'))
+        assert 'hello'.startswith(('he\u1111', 'he'))
+        assert not 'hello'.startswith(('lo\u1111', 'llo'))
+        assert 'hello'.startswith(('hellox\u1111', 'hello'))
+        assert not 'hello'.startswith(('lo', 'he\u1111'), 0, -1)
+        assert not 'hello'.endswith(('he\u1111', 'he'))
+        assert 'hello'.endswith(('\u1111lo', 'llo'))
+        assert 'hello'.endswith(('\u1111hellox', 'hello'))
     
     def test_endswith(self):
-        assert u'ab'.endswith(u'ab') is True
-        assert u'ab'.endswith(u'b') is True
-        assert u'ab'.endswith(u'') is True
-        assert u'x'.endswith(u'a') is False
-        assert u'x'.endswith(u'x') is True
-        assert u''.endswith(u'') is True
-        assert u''.endswith(u'a') is False
-        assert u'x'.endswith(u'xx') is False
-        assert u'y'.endswith(u'xx') is False
+        assert 'ab'.endswith('ab') is True
+        assert 'ab'.endswith('b') is True
+        assert 'ab'.endswith('') is True
+        assert 'x'.endswith('a') is False
+        assert 'x'.endswith('x') is True
+        assert ''.endswith('') is True
+        assert ''.endswith('a') is False
+        assert 'x'.endswith('xx') is False
+        assert 'y'.endswith('xx') is False
 
     def test_endswith_more(self):
-        assert u'abc'.endswith(u'ab', 0, 2) is True
-        assert u'abc'.endswith(u'bc', 1) is True
-        assert u'abc'.endswith(u'bc', 2) is False
-        assert u'abc'.endswith(u'b', -3, -1) is True
+        assert 'abc'.endswith('ab', 0, 2) is True
+        assert 'abc'.endswith('bc', 1) is True
+        assert 'abc'.endswith('bc', 2) is False
+        assert 'abc'.endswith('b', -3, -1) is True
 
     def test_endswith_tuple(self):
-        assert not u'hello'.endswith((u'he', u'ha'))
-        assert u'hello'.endswith((u'lo', u'llo'))
-        assert u'hello'.endswith((u'hellox', u'hello'))
-        assert not u'hello'.endswith(())
-        assert u'helloworld'.endswith((u'hellowo', u'rld', u'lowo'), 3)
-        assert not u'helloworld'.endswith((u'hellowo', u'ello', u'rld'), 3, -1)
-        assert u'hello'.endswith((u'hell', u'ell'), 0, -1)
-        assert not u'hello'.endswith((u'he', u'hel'), 0, 1)
-        assert u'hello'.endswith((u'he', u'hell'), 0, 4)
-        raises(TypeError, u'hello'.endswith, (42,))
+        assert not 'hello'.endswith(('he', 'ha'))
+        assert 'hello'.endswith(('lo', 'llo'))
+        assert 'hello'.endswith(('hellox', 'hello'))
+        assert not 'hello'.endswith(())
+        assert 'helloworld'.endswith(('hellowo', 'rld', 'lowo'), 3)
+        assert not 'helloworld'.endswith(('hellowo', 'ello', 'rld'), 3, -1)
+        assert 'hello'.endswith(('hell', 'ell'), 0, -1)
+        assert not 'hello'.endswith(('he', 'hel'), 0, 1)
+        assert 'hello'.endswith(('he', 'hell'), 0, 4)
+        raises(TypeError, 'hello'.endswith, (42,))
 
     def test_expandtabs(self):
-        assert u'abc\rab\tdef\ng\thi'.expandtabs() ==    u'abc\rab      def\ng       hi'
-        assert u'abc\rab\tdef\ng\thi'.expandtabs(8) ==   u'abc\rab      def\ng       hi'
-        assert u'abc\rab\tdef\ng\thi'.expandtabs(4) ==   u'abc\rab  def\ng   hi'
-        assert u'abc\r\nab\tdef\ng\thi'.expandtabs(4) == u'abc\r\nab  def\ng   hi'
-        assert u'abc\rab\tdef\ng\thi'.expandtabs() ==    u'abc\rab      def\ng       hi'
-        assert u'abc\rab\tdef\ng\thi'.expandtabs(8) ==   u'abc\rab      def\ng       hi'
-        assert u'abc\r\nab\r\ndef\ng\r\nhi'.expandtabs(4) == u'abc\r\nab\r\ndef\ng\r\nhi'
+        assert 'abc\rab\tdef\ng\thi'.expandtabs() ==    'abc\rab      def\ng       hi'
+        assert 'abc\rab\tdef\ng\thi'.expandtabs(8) ==   'abc\rab      def\ng       hi'
+        assert 'abc\rab\tdef\ng\thi'.expandtabs(4) ==   'abc\rab  def\ng   hi'
+        assert 'abc\r\nab\tdef\ng\thi'.expandtabs(4) == 'abc\r\nab  def\ng   hi'
+        assert 'abc\rab\tdef\ng\thi'.expandtabs() ==    'abc\rab      def\ng       hi'
+        assert 'abc\rab\tdef\ng\thi'.expandtabs(8) ==   'abc\rab      def\ng       hi'
+        assert 'abc\r\nab\r\ndef\ng\r\nhi'.expandtabs(4) == 'abc\r\nab\r\ndef\ng\r\nhi'
 
-        s = u'xy\t'
+        s = 'xy\t'
         assert s.expandtabs() =='xy      '
         
-        s = u'\txy\t'
+        s = '\txy\t'
         assert s.expandtabs() =='        xy      '
         assert s.expandtabs(1) ==' xy '
         assert s.expandtabs(2) =='  xy  '
         assert s.expandtabs(3) =='   xy '
         
-        assert u'xy'.expandtabs() =='xy'
-        assert u''.expandtabs() ==''
+        assert 'xy'.expandtabs() =='xy'
+        assert ''.expandtabs() ==''
 
     def test_expandtabs_overflows_gracefully(self):
         import sys
-        if sys.maxint > (1 << 32):
+        if sys.maxsize > (1 << 32):
             skip("Wrong platform")
-        raises((OverflowError, MemoryError), u't\tt\t'.expandtabs, sys.maxint)
+        raises((OverflowError, MemoryError), 't\tt\t'.expandtabs, sys.maxsize)
         
     def test_translate(self):
-        assert u'bbbc' == u'abababc'.translate({ord('a'):None})
-        assert u'iiic' == u'abababc'.translate({ord('a'):None, ord('b'):ord('i')})
-        assert u'iiix' == u'abababc'.translate({ord('a'):None, ord('b'):ord('i'), ord('c'):u'x'})
-        assert u'<i><i><i>c' == u'abababc'.translate({ord('a'):None, ord('b'):u'<i>'})
-        assert u'c' == u'abababc'.translate({ord('a'):None, ord('b'):u''})
-        assert u'xyyx' == u'xzx'.translate({ord('z'):u'yy'})
-        assert u'abcd' == u'ab\0d'.translate(u'c')
-        assert u'abcd' == u'abcd'.translate(u'')
+        assert 'bbbc' == 'abababc'.translate({ord('a'):None})
+        assert 'iiic' == 'abababc'.translate({ord('a'):None, ord('b'):ord('i')})
+        assert 'iiix' == 'abababc'.translate({ord('a'):None, ord('b'):ord('i'), ord('c'):'x'})
+        assert '<i><i><i>c' == 'abababc'.translate({ord('a'):None, ord('b'):'<i>'})
+        assert 'c' == 'abababc'.translate({ord('a'):None, ord('b'):''})
+        assert 'xyyx' == 'xzx'.translate({ord('z'):'yy'})
+        assert 'abcd' == 'ab\0d'.translate('c')
+        assert 'abcd' == 'abcd'.translate('')
+        assert 'abababc'.translate({ord('a'): ''}) == 'bbbc'
 
-        raises(TypeError, u'hello'.translate)
-        raises(TypeError, u'abababc'.translate, {ord('a'):''})
+        raises(TypeError, 'hello'.translate)
 
     def test_unicode_form_encoded_object(self):
-        assert unicode('x', 'utf-8') == u'x'
-        assert unicode('x', 'utf-8', 'strict') == u'x'
+        assert str(b'x', 'utf-8') == 'x'
+        assert str(b'x', 'utf-8', 'strict') == 'x'
         
     def test_unicode_startswith_tuple(self):
-        assert u'xxx'.startswith(('x', 'y', 'z'), 0)
-        assert u'xxx'.endswith(('x', 'y', 'z'), 0)
+        assert 'xxx'.startswith(('x', 'y', 'z'), 0)
+        assert 'xxx'.endswith(('x', 'y', 'z'), 0)
 
     def test_missing_cases(self):
         # some random cases, which are discovered to not be tested during annotation
-        assert u'xxx'[1:1] == u''
+        assert 'xxx'[1:1] == ''
 
     # these tests test lots of encodings, so they really belong to the _codecs
     # module. however, they test useful unicode methods too
@@ -467,174 +420,172 @@
 
     def test_codecs_utf7(self):
         utfTests = [
-            (u'A\u2262\u0391.', 'A+ImIDkQ.'),             # RFC2152 example
-            (u'Hi Mom -\u263a-!', 'Hi Mom -+Jjo--!'),     # RFC2152 example
-            (u'\u65E5\u672C\u8A9E', '+ZeVnLIqe-'),        # RFC2152 example
-            (u'Item 3 is \u00a31.', 'Item 3 is +AKM-1.'), # RFC2152 example
-            (u'+', '+-'),
-            (u'+-', '+--'),
-            (u'+?', '+-?'),
-            (u'\?', '+AFw?'),
-            (u'+?', '+-?'),
-            (ur'\\?', '+AFwAXA?'),
-            (ur'\\\?', '+AFwAXABc?'),
-            (ur'++--', '+-+---')
+            ('A\u2262\u0391.', b'A+ImIDkQ.'),             # RFC2152 example
+            ('Hi Mom -\u263a-!', b'Hi Mom -+Jjo--!'),     # RFC2152 example
+            ('\u65E5\u672C\u8A9E', b'+ZeVnLIqe-'),        # RFC2152 example
+            ('Item 3 is \u00a31.', b'Item 3 is +AKM-1.'), # RFC2152 example
+            ('+', b'+-'),
+            ('+-', b'+--'),
+            ('+?', b'+-?'),
+            ('\?', b'+AFw?'),
+            ('+?', b'+-?'),
+            (r'\\?', b'+AFwAXA?'),
+            (r'\\\?', b'+AFwAXABc?'),
+            (r'++--', b'+-+---')
         ]
 
         for (x, y) in utfTests:
             assert x.encode('utf-7') == y
 
         # surrogates not supported
-        raises(UnicodeError, unicode, '+3ADYAA-', 'utf-7')
+        raises(UnicodeError, str, b'+3ADYAA-', 'utf-7')
 
-        assert unicode('+3ADYAA-', 'utf-7', 'replace') == u'\ufffd\ufffd'
+        assert str(b'+3ADYAA-', 'utf-7', 'replace') == '\ufffd\ufffd'
 
     def test_codecs_utf8(self):
-        assert u''.encode('utf-8') == ''
-        assert u'\u20ac'.encode('utf-8') == '\xe2\x82\xac'
-        assert u'\ud800\udc02'.encode('utf-8') == '\xf0\x90\x80\x82'
-        assert u'\ud84d\udc56'.encode('utf-8') == '\xf0\xa3\x91\x96'
-        assert u'\ud800'.encode('utf-8') == '\xed\xa0\x80'
-        assert u'\udc00'.encode('utf-8') == '\xed\xb0\x80'
-        assert (u'\ud800\udc02'*1000).encode('utf-8') == '\xf0\x90\x80\x82'*1000
+        assert ''.encode('utf-8') == b''
+        assert '\u20ac'.encode('utf-8') == b'\xe2\x82\xac'
+        assert '\ud800\udc02'.encode('utf-8') == b'\xf0\x90\x80\x82'
+        assert '\ud84d\udc56'.encode('utf-8') == b'\xf0\xa3\x91\x96'
+        raises(UnicodeEncodeError, '\ud800'.encode, 'utf-8')
+        raises(UnicodeEncodeError, '\udc00'.encode, 'utf-8')
+        assert ('\ud800\udc02'*1000).encode('utf-8') == b'\xf0\x90\x80\x82'*1000
         assert (
-            u'\u6b63\u78ba\u306b\u8a00\u3046\u3068\u7ffb\u8a33\u306f'
-            u'\u3055\u308c\u3066\u3044\u307e\u305b\u3093\u3002\u4e00'
-            u'\u90e8\u306f\u30c9\u30a4\u30c4\u8a9e\u3067\u3059\u304c'
-            u'\u3001\u3042\u3068\u306f\u3067\u305f\u3089\u3081\u3067'
-            u'\u3059\u3002\u5b9f\u969b\u306b\u306f\u300cWenn ist das'
-            u' Nunstuck git und'.encode('utf-8') == 
-            '\xe6\xad\xa3\xe7\xa2\xba\xe3\x81\xab\xe8\xa8\x80\xe3\x81'
-            '\x86\xe3\x81\xa8\xe7\xbf\xbb\xe8\xa8\xb3\xe3\x81\xaf\xe3'
-            '\x81\x95\xe3\x82\x8c\xe3\x81\xa6\xe3\x81\x84\xe3\x81\xbe'
-            '\xe3\x81\x9b\xe3\x82\x93\xe3\x80\x82\xe4\xb8\x80\xe9\x83'
-            '\xa8\xe3\x81\xaf\xe3\x83\x89\xe3\x82\xa4\xe3\x83\x84\xe8'
-            '\xaa\x9e\xe3\x81\xa7\xe3\x81\x99\xe3\x81\x8c\xe3\x80\x81'
-            '\xe3\x81\x82\xe3\x81\xa8\xe3\x81\xaf\xe3\x81\xa7\xe3\x81'
-            '\x9f\xe3\x82\x89\xe3\x82\x81\xe3\x81\xa7\xe3\x81\x99\xe3'
-            '\x80\x82\xe5\xae\x9f\xe9\x9a\x9b\xe3\x81\xab\xe3\x81\xaf'
-            '\xe3\x80\x8cWenn ist das Nunstuck git und'
+            '\u6b63\u78ba\u306b\u8a00\u3046\u3068\u7ffb\u8a33\u306f'
+            '\u3055\u308c\u3066\u3044\u307e\u305b\u3093\u3002\u4e00'
+            '\u90e8\u306f\u30c9\u30a4\u30c4\u8a9e\u3067\u3059\u304c'
+            '\u3001\u3042\u3068\u306f\u3067\u305f\u3089\u3081\u3067'
+            '\u3059\u3002\u5b9f\u969b\u306b\u306f\u300cWenn ist das'
+            ' Nunstuck git und'.encode('utf-8') == 
+            b'\xe6\xad\xa3\xe7\xa2\xba\xe3\x81\xab\xe8\xa8\x80\xe3\x81'
+            b'\x86\xe3\x81\xa8\xe7\xbf\xbb\xe8\xa8\xb3\xe3\x81\xaf\xe3'
+            b'\x81\x95\xe3\x82\x8c\xe3\x81\xa6\xe3\x81\x84\xe3\x81\xbe'
+            b'\xe3\x81\x9b\xe3\x82\x93\xe3\x80\x82\xe4\xb8\x80\xe9\x83'
+            b'\xa8\xe3\x81\xaf\xe3\x83\x89\xe3\x82\xa4\xe3\x83\x84\xe8'
+            b'\xaa\x9e\xe3\x81\xa7\xe3\x81\x99\xe3\x81\x8c\xe3\x80\x81'
+            b'\xe3\x81\x82\xe3\x81\xa8\xe3\x81\xaf\xe3\x81\xa7\xe3\x81'
+            b'\x9f\xe3\x82\x89\xe3\x82\x81\xe3\x81\xa7\xe3\x81\x99\xe3'
+            b'\x80\x82\xe5\xae\x9f\xe9\x9a\x9b\xe3\x81\xab\xe3\x81\xaf'
+            b'\xe3\x80\x8cWenn ist das Nunstuck git und'
         )
 
         # UTF-8 specific decoding tests
-        assert unicode('\xf0\xa3\x91\x96', 'utf-8') == u'\U00023456' 
-        assert unicode('\xf0\x90\x80\x82', 'utf-8') == u'\U00010002' 
-        assert unicode('\xe2\x82\xac', 'utf-8') == u'\u20ac' 
+        assert str(b'\xf0\xa3\x91\x96', 'utf-8') == '\U00023456' 
+        assert str(b'\xf0\x90\x80\x82', 'utf-8') == '\U00010002' 
+        assert str(b'\xe2\x82\xac', 'utf-8') == '\u20ac' 
 
     def test_codecs_errors(self):
         # Error handling (encoding)
-        raises(UnicodeError, u'Andr\202 x'.encode, 'ascii')
-        raises(UnicodeError, u'Andr\202 x'.encode, 'ascii','strict')
-        assert u'Andr\202 x'.encode('ascii','ignore') == "Andr x"
-        assert u'Andr\202 x'.encode('ascii','replace') == "Andr? x"
+        raises(UnicodeError, 'Andr\202 x'.encode, 'ascii')
+        raises(UnicodeError, 'Andr\202 x'.encode, 'ascii','strict')
+        assert 'Andr\202 x'.encode('ascii','ignore') == b"Andr x"
+        assert 'Andr\202 x'.encode('ascii','replace') == b"Andr? x"
 
         # Error handling (decoding)
-        raises(UnicodeError, unicode, 'Andr\202 x', 'ascii')
-        raises(UnicodeError, unicode, 'Andr\202 x', 'ascii','strict')
-        assert unicode('Andr\202 x','ascii','ignore') == u"Andr x"
-        assert unicode('Andr\202 x','ascii','replace') == u'Andr\uFFFD x'
+        raises(UnicodeError, str, b'Andr\202 x', 'ascii')
+        raises(UnicodeError, str, b'Andr\202 x', 'ascii','strict')
+        assert str(b'Andr\202 x','ascii','ignore') == "Andr x"
+        assert str(b'Andr\202 x','ascii','replace') == 'Andr\uFFFD x'
 
         # Error handling (unknown character names)
-        assert "\\N{foo}xx".decode("unicode-escape", "ignore") == u"xx"
+        assert b"\\N{foo}xx".decode("unicode-escape", "ignore") == "xx"
 
         # Error handling (truncated escape sequence)
-        raises(UnicodeError, "\\".decode, "unicode-escape")
+        raises(UnicodeError, b"\\".decode, "unicode-escape")
 
-        raises(UnicodeError, "\xc2".decode, "utf-8")
-        assert '\xe1\x80'.decode('utf-8', 'replace') == u"\ufffd"
+        raises(UnicodeError, b"\xc2".decode, "utf-8")
+        assert b'\xe1\x80'.decode('utf-8', 'replace') == "\ufffd"
 
     def test_repr_bug(self):
-        assert (repr(u'\U00090418\u027d\U000582b9\u54c3\U000fcb6e') == 
-                "u'\\U00090418\\u027d\\U000582b9\\u54c3\\U000fcb6e'")
-        assert (repr(u'\n') == 
-                "u'\\n'")
+        assert (repr('\U00090418\u027d\U000582b9\u54c3\U000fcb6e') == 
+                "'\\U00090418\u027d\\U000582b9\u54c3\\U000fcb6e'")
+        assert (repr('\n') == 
+                "'\\n'")
 
 
     def test_partition(self):
 
-        assert (u'this is the par', u'ti', u'tion method') == \
-            u'this is the partition method'.partition(u'ti')
+        assert ('this is the par', 'ti', 'tion method') == \
+            'this is the partition method'.partition('ti')
 
         # from raymond's original specification
-        S = u'http://www.python.org'
-        assert (u'http', u'://', u'www.python.org') == S.partition(u'://')
-        assert (u'http://www.python.org', u'', u'') == S.partition(u'?')
-        assert (u'', u'http://', u'www.python.org') == S.partition(u'http://')
-        assert (u'http://www.python.', u'org', u'') == S.partition(u'org')
+        S = 'http://www.python.org'
+        assert ('http', '://', 'www.python.org') == S.partition('://')
+        assert ('http://www.python.org', '', '') == S.partition('?')
+        assert ('', 'http://', 'www.python.org') == S.partition('http://')
+        assert ('http://www.python.', 'org', '') == S.partition('org')
 
-        raises(ValueError, S.partition, u'')
+        raises(ValueError, S.partition, '')
         raises(TypeError, S.partition, None)
 
     def test_rpartition(self):
 
-        assert (u'this is the rparti', u'ti', u'on method') == \
-            u'this is the rpartition method'.rpartition(u'ti')
+        assert ('this is the rparti', 'ti', 'on method') == \
+            'this is the rpartition method'.rpartition('ti')
 
         # from raymond's original specification
-        S = u'http://www.python.org'
-        assert (u'http', u'://', u'www.python.org') == S.rpartition(u'://')
-        assert (u'', u'', u'http://www.python.org') == S.rpartition(u'?')
-        assert (u'', u'http://', u'www.python.org') == S.rpartition(u'http://')
-        assert (u'http://www.python.', u'org', u'') == S.rpartition(u'org')
+        S = 'http://www.python.org'
+        assert ('http', '://', 'www.python.org') == S.rpartition('://')
+        assert ('', '', 'http://www.python.org') == S.rpartition('?')
+        assert ('', 'http://', 'www.python.org') == S.rpartition('http://')
+        assert ('http://www.python.', 'org', '') == S.rpartition('org')
 
-        raises(ValueError, S.rpartition, u'')
+        raises(ValueError, S.rpartition, '')
         raises(TypeError, S.rpartition, None)
 
 
     def test_mul(self):
         zero = 0
-        assert type(u'' * zero) == type(zero * u'') == unicode
-        assert u'' * zero == zero * u'' == u''
-        assert u'x' * zero == zero * u'x' == u''
-        assert type(u'x' * zero) == type(zero * u'x') == unicode
-        assert u'123' * zero == zero * u'123' == u''
-        assert type(u'123' * zero) == type(zero * u'123') == unicode
+        assert type('' * zero) == type(zero * '') is str
+        assert '' * zero == zero * '' == ''
+        assert 'x' * zero == zero * 'x' == ''
+        assert type('x' * zero) == type(zero * 'x') is str
+        assert '123' * zero == zero * '123' == ''
+        assert type('123' * zero) == type(zero * '123') is str
         for i in range(10):
-            u = u'123' * i
+            u = '123' * i
             assert len(u) == 3*i
             for j in range(0, i, 3):
-                assert u[j+0] == u'1'
-                assert u[j+1] == u'2'
-                assert u[j+2] == u'3'
-            assert u'123' * i == i * u'123'
+                assert u[j+0] == '1'
+                assert u[j+1] == '2'
+                assert u[j+2] == '3'
+            assert '123' * i == i * '123'
 
     def test_index(self):
-        assert u"rrarrrrrrrrra".index(u'a', 4, None) == 12
-        assert u"rrarrrrrrrrra".index(u'a', None, 6) == 2
+        assert "rrarrrrrrrrra".index('a', 4, None) == 12
+        assert "rrarrrrrrrrra".index('a', None, 6) == 2
 
     def test_rindex(self):
-        from sys import maxint
-        assert u'abcdefghiabc'.rindex(u'') == 12
-        assert u'abcdefghiabc'.rindex(u'def') == 3
-        assert u'abcdefghiabc'.rindex(u'abc') == 9
-        assert u'abcdefghiabc'.rindex(u'abc', 0, -1) == 0
-        assert u'abcdefghiabc'.rindex(u'abc', -4*maxint, 4*maxint) == 9
-        assert u'rrarrrrrrrrra'.rindex(u'a', 4, None) == 12
+        from sys import maxsize
+        assert 'abcdefghiabc'.rindex('') == 12
+        assert 'abcdefghiabc'.rindex('def') == 3
+        assert 'abcdefghiabc'.rindex('abc') == 9
+        assert 'abcdefghiabc'.rindex('abc', 0, -1) == 0
+        assert 'abcdefghiabc'.rindex('abc', -4*maxsize, 4*maxsize) == 9
+        assert 'rrarrrrrrrrra'.rindex('a', 4, None) == 12
 
-        raises(ValueError, u'abcdefghiabc'.rindex, u'hib')
-        raises(ValueError, u'defghiabc'.rindex, u'def', 1)
-        raises(ValueError, u'defghiabc'.rindex, u'abc', 0, -1)
-        raises(ValueError, u'abcdefghi'.rindex, u'ghi', 0, 8)
-        raises(ValueError, u'abcdefghi'.rindex, u'ghi', 0, -1)
-        raises(TypeError, u'abcdefghijklmn'.rindex, u'abc', 0, 0.0)
-        raises(TypeError, u'abcdefghijklmn'.rindex, u'abc', -10.0, 30)
+        raises(ValueError, 'abcdefghiabc'.rindex, 'hib')
+        raises(ValueError, 'defghiabc'.rindex, 'def', 1)
+        raises(ValueError, 'defghiabc'.rindex, 'abc', 0, -1)
+        raises(ValueError, 'abcdefghi'.rindex, 'ghi', 0, 8)
+        raises(ValueError, 'abcdefghi'.rindex, 'ghi', 0, -1)
+        raises(TypeError, 'abcdefghijklmn'.rindex, 'abc', 0, 0.0)
+        raises(TypeError, 'abcdefghijklmn'.rindex, 'abc', -10.0, 30)
 
     def test_rfind(self):
-        assert u'abcdefghiabc'.rfind(u'abc') == 9
-        assert u'abcdefghiabc'.rfind(u'') == 12
-        assert u'abcdefghiabc'.rfind(u'abcd') == 0
-        assert u'abcdefghiabc'.rfind(u'abcz') == -1
+        assert 'abcdefghiabc'.rfind('abc') == 9
+        assert 'abcdefghiabc'.rfind('') == 12
+        assert 'abcdefghiabc'.rfind('abcd') == 0
+        assert 'abcdefghiabc'.rfind('abcz') == -1
 
     def test_rfind_corner_case(self):
-        if self.version_info < (2, 5):
-            skip("fails on top of CPython <= 2.4")
-        assert u'abc'.rfind('', 4) == -1
+        assert 'abc'.rfind('', 4) == -1
 
     def test_count(self):
-        assert u"".count(u"x") ==0
-        assert u"".count(u"") ==1
-        assert u"Python".count(u"") ==7
-        assert u"ab aaba".count(u"ab") ==2
+        assert "".count("x") ==0
+        assert "".count("") ==1
+        assert "Python".count("") ==7
+        assert "ab aaba".count("ab") ==2
         assert 'aaa'.count('a') == 3
         assert 'aaa'.count('b') == 0
         assert 'aaa'.count('a', -1) == 1
@@ -644,20 +595,13 @@
         assert 'ababa'.count('aba') == 1
 
     def test_swapcase(self):
-        assert u'\xe4\xc4\xdf'.swapcase() == u'\xc4\xe4\xdf'
-
-    def test_buffer(self):
-        buf = buffer(u'XY')
-        assert str(buf) in ['X\x00Y\x00',
-                            '\x00X\x00Y',
-                            'X\x00\x00\x00Y\x00\x00\x00',
-                            '\x00\x00\x00X\x00\x00\x00Y']
+        assert '\xe4\xc4\xdf'.swapcase() == '\xc4\xe4\xdf'
 
     def test_call_special_methods(self):
         # xxx not completely clear if these are implementation details or not
-        assert 'abc'.__add__(u'def') == u'abcdef'
-        assert u'abc'.__add__(u'def') == u'abcdef'
-        assert u'abc'.__add__('def') == u'abcdef'
+        assert 'abc'.__add__('def') == 'abcdef'
+        assert 'abc'.__add__('def') == 'abcdef'
+        assert 'abc'.__add__('def') == 'abcdef'
         # xxx CPython has no str.__radd__ and no unicode.__radd__
 
     def test_str_unicode_concat_overrides(self):
@@ -668,15 +612,11 @@
             assert value == expected
 
         def _test_concat(t1, t2):
-            tprecedent = str
-            if issubclass(t1, unicode) or issubclass(t2, unicode):
-                tprecedent = unicode
-
             class SubclassB(t2):
                 def __add__(self, other):
                     return SubclassB(t2(self) + t2(other))
             check(SubclassB('py') + SubclassB('thon'), SubclassB('python'))
-            check(t1('python') + SubclassB('3'), tprecedent('python3'))
+            check(t1('python') + SubclassB('3'), str('python3'))
             check(SubclassB('py') + t1('py'), SubclassB('pypy'))
 
             class SubclassC(t2):
@@ -684,7 +624,7 @@
                     return SubclassC(t2(other) + t2(self))
             check(SubclassC('stack') + SubclassC('less'), t2('stackless'))
             check(t1('iron') + SubclassC('python'), SubclassC('ironpython'))
-            check(SubclassC('tiny') + t1('py'), tprecedent('tinypy'))
+            check(SubclassC('tiny') + t1('py'), str('tinypy'))
 
             class SubclassD(t2):
                 def __add__(self, other):
@@ -697,24 +637,20 @@
             check(SubclassD('dun') + t1('der'), SubclassD('dunder'))
 
         _test_concat(str, str)
-        _test_concat(unicode, unicode)
-        # the following two cases are really there to emulate a CPython bug.
-        _test_concat(str, unicode)   # uses hack in add__String_Unicode()
-        _test_concat(unicode, str)   # uses hack in descroperation.binop_impl()
 
     def test_returns_subclass(self):
-        class X(unicode):
+        class X(str):
             pass
 
-        class Y(object):
-            def __unicode__(self):
+        class Y(str):
+            def __str__(self):
                 return X("stuff")
 
-        assert unicode(Y()).__class__ is X
+        assert str(Y()).__class__ is X
     
     def test_getslice(self):
-        assert u'123456'.__getslice__(1, 5) == u'2345'
-        s = u"abc"
+        assert '123456'[1:5] == '2345'
+        s = "abc"
         assert s[:] == "abc"
         assert s[1:] == "bc"
         assert s[:2] == "ab"
@@ -726,155 +662,127 @@
         assert s[-2:-1] == "b"
 
     def test_no_len_on_str_iter(self):
-        iterable = u"hello"
+        iterable = "hello"
         raises(TypeError, len, iter(iterable))
 
     def test_encode_raw_unicode_escape(self):
-        u = unicode('\\', 'raw_unicode_escape')
-        assert u == u'\\'
+        u = str(b'\\', 'raw_unicode_escape')
+        assert u == '\\'
 
     def test_decode_from_buffer(self):
-        buf = buffer('character buffers are decoded to unicode')
-        u = unicode(buf, 'utf-8', 'strict')
-        assert u == u'character buffers are decoded to unicode'
+        buf = b'character buffers are decoded to unicode'
+        u = str(buf, 'utf-8', 'strict')
+        assert u == 'character buffers are decoded to unicode'
 
-    def test_unicode_conversion_with__unicode__(self):
-        class A(unicode):
-            def __unicode__(self):
+    def test_unicode_conversion_with__str__(self):
+        class A(str):
+            def __str__(self):
                 return "foo"
-        class B(unicode):
+        class B(str):
             pass
         a = A('bar')
         assert a == 'bar'
-        assert unicode(a) == 'foo'
+        assert str(a) == 'foo'
         b = B('bar')
         assert b == 'bar'
-        assert unicode(b) == 'bar'
+        assert str(b) == 'bar'
 
     def test_unicode_conversion_with__str__(self):
         # new-style classes
         class A(object):
             def __str__(self):
-                return u'\u1234'
-        s = unicode(A())
-        assert type(s) is unicode
-        assert s == u'\u1234'
+                return '\u1234'
+        s = str(A())
+        assert type(s) is str
+        assert s == '\u1234'
         # with old-style classes, it's different, but it should work as well
         class A:
             def __str__(self):
-                return u'\u1234'
-        s = unicode(A())
-        assert type(s) is unicode
-        assert s == u'\u1234'
+                return '\u1234'
+        s = str(A())
+        assert type(s) is str
+        assert s == '\u1234'
 
     def test_formatting_unicode__str__(self):
         class A:
             def __init__(self, num):
                 self.num = num
             def __str__(self):
-                return unichr(self.num)
+                return chr(self.num)
 
         s = '%s' % A(111)    # this is ASCII
-        assert type(s) is unicode
+        assert type(s) is str
         assert s == chr(111)
 
         s = '%s' % A(0x1234)    # this is not ASCII
-        assert type(s) is unicode
-        assert s == u'\u1234'
+        assert type(s) is str
+        assert s == '\u1234'
 
         # now the same with a new-style class...
         class A(object):
             def __init__(self, num):
                 self.num = num
             def __str__(self):
-                return unichr(self.num)
+                return chr(self.num)
 
         s = '%s' % A(111)    # this is ASCII
-        assert type(s) is unicode
+        assert type(s) is str
         assert s == chr(111)
 
         s = '%s' % A(0x1234)    # this is not ASCII
-        assert type(s) is unicode
-        assert s == u'\u1234'
+        assert type(s) is str
+        assert s == '\u1234'
 
     def test_formatting_unicode__str__2(self):
         class A:
             def __str__(self):
-                return u'baz'
+                return 'baz'
 
         class B:
             def __str__(self):
-                return 'foo'
-
-            def __unicode__(self):
-                return u'bar'
+                return 'bar'
     
         a = A()
         b = B()
         s = '%s %s' % (a, b)
-        assert s == u'baz bar'
-
-        skip("but this case here is completely insane")
-        s = '%s %s' % (b, a)
-        assert s == u'foo baz'
+        assert s == 'baz bar'
 
     def test_formatting_unicode__str__3(self):
         # "bah" is all I can say
         class X(object):
             def __repr__(self):
-                return u'\u1234'
+                return '\u1234'
         '%s' % X()
         #
         class X(object):
             def __str__(self):
-                return u'\u1234'
+                return '\u1234'
         '%s' % X()
 
     def test_str_subclass(self):
         class Foo9(str):
-            def __unicode__(self):
-                return u"world"
-        assert unicode(Foo9("hello")) == u"world"
-
-    def test_class_with_both_str_and_unicode(self):
-        class A(object):
             def __str__(self):
-                return 'foo'
-
-            def __unicode__(self):
-                return u'bar'
-
-        assert unicode(A()) == u'bar'
-
-        class A:
-            def __str__(self):
-                return 'foo'
-
-            def __unicode__(self):
-                return u'bar'
-
-        assert unicode(A()) == u'bar'
+                return "world"
+        assert str(Foo9("hello")) == "world"
 
     def test_format_unicode_subclass(self):
-        class U(unicode):
-            def __unicode__(self):
-                return u'__unicode__ overridden'
-        u = U(u'xxx')
-        assert repr("%s" % u) == "u'__unicode__ overridden'"
-        assert repr("{}".format(u)) == "'__unicode__ overridden'"
+        class U(str):
+            def __str__(self):
+                return '__str__ overridden'
+        u = U('xxx')
+        assert repr("%s" % u) == "'__str__ overridden'"
+        assert repr("{}".format(u)) == "'__str__ overridden'"
 
     def test_replace_with_buffer(self):
-        assert u'abc'.replace(buffer('b'), buffer('e')) == u'aec'
-        assert u'abc'.replace(buffer('b'), u'e') == u'aec'
-        assert u'abc'.replace(u'b', buffer('e')) == u'aec'
+        raises(TypeError, 'abc'.replace, b'b', b'e')
 
     def test_unicode_subclass(self):
-        class S(unicode):
+        class S(str):
             pass
 
-        a = S(u'hello \u1234')
-        b = unicode(a)
-        assert type(b) is unicode
-        assert b == u'hello \u1234'
+        a = S('hello \u1234')
+        b = str(a)
+        assert type(b) is str
+        assert b == 'hello \u1234'
 
-        assert u'%s' % S(u'mar\xe7') == u'mar\xe7'
+        assert '%s' % S('mar\xe7') == 'mar\xe7'

From noreply at buildbot.pypy.org  Wed Oct 12 22:23:37 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 12 Oct 2011 22:23:37 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Remove str -> unicode delegation
Message-ID: <20111012202337.C0C7F82112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r47996:83e0cb6106b6
Date: 2011-10-12 22:19 +0200
http://bitbucket.org/pypy/pypy/changeset/83e0cb6106b6/

Log:	Remove str -> unicode delegation

diff --git a/pypy/objspace/std/model.py b/pypy/objspace/std/model.py
--- a/pypy/objspace/std/model.py
+++ b/pypy/objspace/std/model.py
@@ -212,40 +212,20 @@
         self.typeorder[setobject.W_FrozensetObject] += [
             (setobject.W_BaseSetObject, None)
             ]
-        if not config.objspace.std.withrope:
-            self.typeorder[stringobject.W_StringObject] += [
-             (unicodeobject.W_UnicodeObject, unicodeobject.delegate_String2Unicode),
-                ]
-        else:
-            if config.objspace.std.withropeunicode:
-                self.typeorder[ropeobject.W_RopeObject] += [
-                 (ropeunicodeobject.W_RopeUnicodeObject,
-                     ropeunicodeobject.delegate_Rope2RopeUnicode),
-                 ]
-            else:
-                self.typeorder[ropeobject.W_RopeObject] += [
-                 (unicodeobject.W_UnicodeObject, unicodeobject.delegate_String2Unicode),
-                    ]
         if config.objspace.std.withstrslice:
             self.typeorder[strsliceobject.W_StringSliceObject] += [
                 (stringobject.W_StringObject,
                                        strsliceobject.delegate_slice2str),
-                (unicodeobject.W_UnicodeObject,
-                                       strsliceobject.delegate_slice2unicode),
                 ]
         if config.objspace.std.withstrjoin:
             self.typeorder[strjoinobject.W_StringJoinObject] += [
                 (stringobject.W_StringObject,
                                        strjoinobject.delegate_join2str),
-                (unicodeobject.W_UnicodeObject,
-                                       strjoinobject.delegate_join2unicode)
                 ]
         elif config.objspace.std.withstrbuf:
             self.typeorder[strbufobject.W_StringBufferObject] += [
                 (stringobject.W_StringObject,
                                        strbufobject.delegate_buf2str),
-                (unicodeobject.W_UnicodeObject,
-                                       strbufobject.delegate_buf2unicode)
                 ]
         if config.objspace.std.withsmalltuple:
             self.typeorder[smalltupleobject.W_SmallTupleObject] += [
diff --git a/pypy/objspace/std/strbufobject.py b/pypy/objspace/std/strbufobject.py
--- a/pypy/objspace/std/strbufobject.py
+++ b/pypy/objspace/std/strbufobject.py
@@ -1,7 +1,6 @@
 from pypy.objspace.std.model import registerimplementation, W_Object
 from pypy.objspace.std.register_all import register_all
 from pypy.objspace.std.stringobject import W_StringObject
-from pypy.objspace.std.unicodeobject import delegate_String2Unicode
 from pypy.rlib.rstring import StringBuilder
 from pypy.interpreter.buffer import Buffer
 
@@ -51,10 +50,6 @@
     w_strbuf.force()
     return w_strbuf.w_str
 
-def delegate_buf2unicode(space, w_strbuf):
-    w_strbuf.force()
-    return delegate_String2Unicode(space, w_strbuf.w_str)
-
 def len__StringBuffer(space, w_self):
     return space.wrap(w_self.length)
 
diff --git a/pypy/objspace/std/strjoinobject.py b/pypy/objspace/std/strjoinobject.py
--- a/pypy/objspace/std/strjoinobject.py
+++ b/pypy/objspace/std/strjoinobject.py
@@ -1,7 +1,6 @@
 from pypy.objspace.std.model import registerimplementation, W_Object
 from pypy.objspace.std.register_all import register_all
 from pypy.objspace.std.stringobject import W_StringObject
-from pypy.objspace.std.unicodeobject import delegate_String2Unicode
 
 from pypy.objspace.std.stringtype import wrapstr
 
@@ -36,10 +35,6 @@
 def delegate_join2str(space, w_strjoin):
     return wrapstr(space, w_strjoin.force())
 
-def delegate_join2unicode(space, w_strjoin):
-    w_str = wrapstr(space, w_strjoin.force())
-    return delegate_String2Unicode(space, w_str)
-
 def len__StringJoin(space, w_self):
     result = 0
     for i in range(w_self.until):
diff --git a/pypy/objspace/std/strsliceobject.py b/pypy/objspace/std/strsliceobject.py
--- a/pypy/objspace/std/strsliceobject.py
+++ b/pypy/objspace/std/strsliceobject.py
@@ -2,7 +2,6 @@
 from pypy.objspace.std.model import registerimplementation, W_Object
 from pypy.objspace.std.register_all import register_all
 from pypy.objspace.std.stringobject import W_StringObject
-from pypy.objspace.std.unicodeobject import delegate_String2Unicode
 from pypy.objspace.std.sliceobject import W_SliceObject, normalize_simple_slice
 from pypy.objspace.std.tupleobject import W_TupleObject
 from pypy.objspace.std import slicetype
@@ -46,10 +45,6 @@
 def delegate_slice2str(space, w_strslice):
     return wrapstr(space, w_strslice.force())
 
-def delegate_slice2unicode(space, w_strslice):
-    w_str = wrapstr(space, w_strslice.force())
-    return delegate_String2Unicode(space, w_str)
-
 # ____________________________________________________________
 
 def contains__StringSlice_String(space, w_self, w_sub):
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -78,13 +78,6 @@
                 raise OperationError(space.w_UnicodeEncodeError, space.newtuple([w_encoding, w_unistr, w_start, w_end, w_reason]))
     return ''.join(result)
 
-# string-to-unicode delegation
-def delegate_String2Unicode(space, w_str):
-    from pypy.objspace.std.unicodetype import unicode_from_string
-    w_uni = unicode_from_string(space, w_str)
-    assert isinstance(w_uni, W_UnicodeObject) # help the annotator!
-    return w_uni
-
 # checks if should trigger an unicode warning
 def _unicode_string_comparison(space, w_uni, w_str, inverse, uni_from_str):
     try:

From noreply at buildbot.pypy.org  Wed Oct 12 22:23:39 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 12 Oct 2011 22:23:39 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Enable unicode literals by default
Message-ID: <20111012202339.04D1082112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r47997:1b2a92468026
Date: 2011-10-12 22:19 +0200
http://bitbucket.org/pypy/pypy/changeset/1b2a92468026/

Log:	Enable unicode literals by default

diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -1065,10 +1065,8 @@
             space = self.space
             encoding = self.compile_info.encoding
             flags = self.compile_info.flags
-            unicode_literals = flags & consts.CO_FUTURE_UNICODE_LITERALS
             try:
-                sub_strings_w = [parsestring.parsestr(space, encoding, s.value,
-                                                      unicode_literals)
+                sub_strings_w = [parsestring.parsestr(space, encoding, s.value)
                                  for s in atom_node.children]
             except error.OperationError, e:
                 if not e.match(space, space.w_UnicodeError):
diff --git a/pypy/interpreter/pyparser/parsestring.py b/pypy/interpreter/pyparser/parsestring.py
--- a/pypy/interpreter/pyparser/parsestring.py
+++ b/pypy/interpreter/pyparser/parsestring.py
@@ -1,7 +1,7 @@
 from pypy.interpreter.error import OperationError
 from pypy.interpreter import unicodehelper
 
-def parsestr(space, encoding, s, unicode_literals=False):
+def parsestr(space, encoding, s):
     # compiler.transformer.Transformer.decode_literal depends on what 
     # might seem like minor details of this function -- changes here 
     # must be reflected there.
@@ -11,7 +11,7 @@
     ps = 0
     quote = s[ps]
     rawmode = False
-    unicode = unicode_literals
+    unicode = True
 
     # string decoration handling
     o = ord(quote)

From noreply at buildbot.pypy.org  Wed Oct 12 22:23:40 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 12 Oct 2011 22:23:40 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Remove mixed operations between str and
	unicode
Message-ID: <20111012202340.4AC1382112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r47998:2693bfc6f0ac
Date: 2011-10-12 22:19 +0200
http://bitbucket.org/pypy/pypy/changeset/2693bfc6f0ac/

Log:	Remove mixed operations between str and unicode

diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -675,15 +675,6 @@
             # Note that space.is_w() is potentially not happy if one of them
             # is None (e.g. with the thunk space)...
             if w_left_src is not w_right_src:    # XXX
-                # -- cpython bug compatibility: see objspace/std/test/
-                # -- test_unicodeobject.test_str_unicode_concat_overrides.
-                # -- The following handles "unicode + string subclass" by
-                # -- pretending that the unicode is a superclass of the
-                # -- string, thus giving priority to the string subclass'
-                # -- __radd__() method.  The case "string + unicode subclass"
-                # -- is handled directly by add__String_Unicode().
-                if symbol == '+' and space.is_w(w_typ1, space.w_unicode):
-                    w_typ1 = space.w_basestring
                 # -- end of bug compatibility
                 if space.is_true(space.issubtype(w_typ2, w_typ1)):
                     if (w_left_src and w_right_src and
diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -196,21 +196,9 @@
 def eq__Bytearray_String(space, w_bytearray, w_other):
     return space.eq(str__Bytearray(space, w_bytearray), w_other)
 
-def eq__Bytearray_Unicode(space, w_bytearray, w_other):
-    return space.w_False
-
-def eq__Unicode_Bytearray(space, w_other, w_bytearray):
-    return space.w_False
-
 def ne__Bytearray_String(space, w_bytearray, w_other):
     return space.ne(str__Bytearray(space, w_bytearray), w_other)
 
-def ne__Bytearray_Unicode(space, w_bytearray, w_other):
-    return space.w_True
-
-def ne__Unicode_Bytearray(space, w_other, w_bytearray):
-    return space.w_True
-
 def _min(a, b):
     if a < b:
         return a
diff --git a/pypy/objspace/std/ropeunicodeobject.py b/pypy/objspace/std/ropeunicodeobject.py
--- a/pypy/objspace/std/ropeunicodeobject.py
+++ b/pypy/objspace/std/ropeunicodeobject.py
@@ -24,19 +24,6 @@
 def wrapunicode(space, uni):
     return W_RopeUnicodeObject(rope.rope_from_unicode(uni))
 
-def unicode_from_string(space, w_str):
-    from pypy.objspace.std.unicodetype import getdefaultencoding
-    assert isinstance(w_str, W_RopeObject)
-    encoding = getdefaultencoding(space)
-    w_retval = decode_string(space, w_str, encoding, "strict")
-    if not space.isinstance_w(w_retval, space.w_unicode):
-        raise operationerrfmt(
-            space.w_TypeError,
-            "decoder did not return an unicode object (type '%s')",
-            space.type(w_retval).getname(space))
-    assert isinstance(w_retval, W_RopeUnicodeObject)
-    return w_retval
-
 def decode_string(space, w_str, encoding, errors):
     from pypy.objspace.std.unicodetype import decode_object
     if errors is None or errors == "strict":
@@ -112,8 +99,6 @@
 def ropeunicode_w(space, w_str):
     if isinstance(w_str, W_RopeUnicodeObject):
         return w_str._node
-    if isinstance(w_str, W_RopeObject):
-        return unicode_from_string(space, w_str)._node
     return rope.LiteralUnicodeNode(space.unicode_w(w_str))
 
 
@@ -157,12 +142,6 @@
                 raise OperationError(space.w_UnicodeEncodeError, space.newtuple([w_encoding, w_unistr, w_start, w_end, w_reason]))
     return ''.join(result)
 
-# string-to-unicode delegation
-def delegate_Rope2RopeUnicode(space, w_rope):
-    w_uni = unicode_from_string(space, w_rope)
-    assert isinstance(w_uni, W_RopeUnicodeObject) # help the annotator!
-    return w_uni
-
 def str__RopeUnicode(space, w_uni):
     return space.call_method(w_uni, 'encode')
 
@@ -183,19 +162,9 @@
 def eq__RopeUnicode_RopeUnicode(space, w_str1, w_str2):
     return space.newbool(_eq(w_str1, w_str2))
 
-def eq__RopeUnicode_Rope(space, w_runi, w_rope):
-    from pypy.objspace.std.unicodeobject import _unicode_string_comparison
-    return _unicode_string_comparison(space, w_runi, w_rope,
-                    False,  unicode_from_string)
-
 def ne__RopeUnicode_RopeUnicode(space, w_str1, w_str2):
     return space.newbool(not _eq(w_str1, w_str2))
 
-def ne__RopeUnicode_Rope(space, w_runi, w_rope):
-    from pypy.objspace.std.unicodeobject import _unicode_string_comparison
-    return _unicode_string_comparison(space, w_runi, w_rope,
-                    True, unicode_from_string)
-
 def gt__RopeUnicode_RopeUnicode(space, w_str1, w_str2):
     n1 = w_str1._node
     n2 = w_str2._node
@@ -224,20 +193,11 @@
         raise OperationError(space.w_OverflowError,
                              space.wrap("string too long"))
 
-def add__Rope_RopeUnicode(space, w_left, w_right):
-    return space.add(unicode_from_string(space, w_left) , w_right)
-
-def add__RopeUnicode_Rope(space, w_left, w_right):
-    return space.add(w_left, unicode_from_string(space, w_right))
-
 def contains__RopeUnicode_RopeUnicode(space, w_container, w_item):
     item = w_item._node
     container = w_container._node
     return space.newbool(rope.find(container, item) != -1)
 
-def contains__Rope_RopeUnicode(space, w_container, w_item):
-    return space.contains(unicode_from_string(space, w_container), w_item )
-
 def unicode_join__RopeUnicode_ANY(space, w_self, w_list):
     l_w = space.listview(w_list)
     delim = w_self._node
@@ -254,10 +214,8 @@
         if isinstance(w_item, W_RopeUnicodeObject):
             # shortcut for performane
             item = w_item._node
-        elif space.isinstance_w(w_item, space.w_str):
-            item = unicode_from_string(space, w_item)._node
         else:
-            msg = 'sequence item %d: expected string or Unicode'
+            msg = 'sequence item %d: expected string'
             raise operationerrfmt(space.w_TypeError, msg, i)
         values_list.append(item)
     try:
@@ -388,27 +346,17 @@
     return W_RopeUnicodeObject(rope.strip(w_self._node, True, True, _contains,
                                w_chars._node.flatten_unicode()))
 
-def unicode_strip__RopeUnicode_Rope(space, w_self, w_chars):
-    return space.call_method(w_self, 'strip',
-                             unicode_from_string(space, w_chars))
-
 def unicode_lstrip__RopeUnicode_None(space, w_self, w_chars):
     return W_RopeUnicodeObject(rope.strip(w_self._node, True, False, _isspace))
 def unicode_lstrip__RopeUnicode_RopeUnicode(space, w_self, w_chars):
     return W_RopeUnicodeObject(rope.strip(w_self._node, True, False, _contains,
                                w_chars._node.flatten_unicode()))
-def unicode_lstrip__RopeUnicode_Rope(space, w_self, w_chars):
-    return space.call_method(w_self, 'lstrip',
-                             unicode_from_string(space, w_chars))
 
 def unicode_rstrip__RopeUnicode_None(space, w_self, w_chars):
     return W_RopeUnicodeObject(rope.strip(w_self._node, False, True, _isspace))
 def unicode_rstrip__RopeUnicode_RopeUnicode(space, w_self, w_chars):
     return W_RopeUnicodeObject(rope.strip(w_self._node, False, True, _contains,
                                w_chars._node.flatten_unicode()))
-def unicode_rstrip__RopeUnicode_Rope(space, w_self, w_chars):
-    return space.call_method(w_self, 'rstrip',
-                             unicode_from_string(space, w_chars))
 
 def unicode_capitalize__RopeUnicode(space, w_self):
     input = w_self._node
@@ -979,49 +927,3 @@
 
 from pypy.objspace.std import unicodetype
 register_all(vars(), unicodetype)
-
-# str.strip(unicode) needs to convert self to unicode and call unicode.strip we
-# use the following magic to register strip_string_unicode as a String
-# multimethod.
-
-# XXX couldn't string and unicode _share_ the multimethods that make up their
-# methods?
-
-class str_methods:
-    from pypy.objspace.std import stringtype
-    W_RopeUnicodeObject = W_RopeUnicodeObject
-    from pypy.objspace.std.ropeobject import W_RopeObject
-    def str_strip__Rope_RopeUnicode(space, w_self, w_chars):
-        return space.call_method(unicode_from_string(space, w_self),
-                                 'strip', w_chars)
-    def str_lstrip__Rope_RopeUnicode(space, w_self, w_chars):
-        return space.call_method(unicode_from_string(space, w_self),
-                                 'lstrip', w_chars)
-    def str_rstrip__Rope_RopeUnicode(space, w_self, w_chars):
-        return space.call_method(unicode_from_string(space, w_self),
-                                 'rstrip', w_chars)
-    def str_count__Rope_RopeUnicode_ANY_ANY(space, w_self, w_substr, w_start, w_end):
-        return space.call_method(unicode_from_string(space, w_self),
-                                 'count', w_substr, w_start, w_end)
-    def str_find__Rope_RopeUnicode_ANY_ANY(space, w_self, w_substr, w_start, w_end):
-        return space.call_method(unicode_from_string(space, w_self),
-                                 'find', w_substr, w_start, w_end)
-    def str_rfind__Rope_RopeUnicode_ANY_ANY(space, w_self, w_substr, w_start, w_end):
-        return space.call_method(unicode_from_string(space, w_self),
-                                 'rfind', w_substr, w_start, w_end)
-    def str_index__Rope_RopeUnicode_ANY_ANY(space, w_self, w_substr, w_start, w_end):
-        return space.call_method(unicode_from_string(space, w_self),
-                                 'index', w_substr, w_start, w_end)
-    def str_rindex__Rope_RopeUnicode_ANY_ANY(space, w_self, w_substr, w_start, w_end):
-        return space.call_method(unicode_from_string(space, w_self),
-                                 'rindex', w_substr, w_start, w_end)
-    def str_replace__Rope_RopeUnicode_RopeUnicode_ANY(space, w_self, w_old, w_new, w_maxsplit):
-        return space.call_method(unicode_from_string(space, w_self),
-                                 'replace', w_old, w_new, w_maxsplit)
-    def str_split__Rope_RopeUnicode_ANY(space, w_self, w_delim, w_maxsplit):
-        return space.call_method(unicode_from_string(space, w_self),
-                                 'split', w_delim, w_maxsplit)
-    def str_rsplit__Rope_RopeUnicode_ANY(space, w_self, w_delim, w_maxsplit):
-        return space.call_method(unicode_from_string(space, w_self),
-                                 'rsplit', w_delim, w_maxsplit)
-    register_all(vars(), stringtype)
diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py
--- a/pypy/objspace/std/stringobject.py
+++ b/pypy/objspace/std/stringobject.py
@@ -996,10 +996,8 @@
 
 def str_decode__String_ANY_ANY(space, w_string, w_encoding=None, w_errors=None):
     from pypy.objspace.std.unicodetype import _get_encoding_and_errors, \
-        unicode_from_string, decode_object
+        decode_object
     encoding, errors = _get_encoding_and_errors(space, w_encoding, w_errors)
-    if encoding is None and errors is None:
-        return unicode_from_string(space, w_string)
     return decode_object(space, w_string, encoding, errors)
 
 def str_encode__String_ANY_ANY(space, w_string, w_encoding=None, w_errors=None):
diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py
--- a/pypy/objspace/std/test/test_unicodeobject.py
+++ b/pypy/objspace/std/test/test_unicodeobject.py
@@ -4,6 +4,7 @@
 
 class AppTestUnicodeStringStdOnly:
     def test_compares(self):
+        assert type('a') != type(b'a')
         assert 'a' != b'a'
         assert b'a' != 'a'
         assert not ('a' == 5)
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -4,7 +4,6 @@
 from pypy.interpreter import gateway
 from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.objspace.std.stringobject import W_StringObject, make_rsplit_with_delim
-from pypy.objspace.std.ropeobject import W_RopeObject
 from pypy.objspace.std.noneobject import W_NoneObject
 from pypy.objspace.std.sliceobject import W_SliceObject, normalize_simple_slice
 from pypy.objspace.std import slicetype, newformat
@@ -78,26 +77,6 @@
                 raise OperationError(space.w_UnicodeEncodeError, space.newtuple([w_encoding, w_unistr, w_start, w_end, w_reason]))
     return ''.join(result)
 
-# checks if should trigger an unicode warning
-def _unicode_string_comparison(space, w_uni, w_str, inverse, uni_from_str):
-    try:
-        w_uni2 = uni_from_str(space, w_str)
-    except OperationError, e:
-        if e.match(space, space.w_UnicodeDecodeError):
-            if inverse:
-                msg = "Unicode unequal comparison failed to convert both "  \
-                      "arguments to Unicode - interpreting them as being unequal"
-            else :
-                msg = "Unicode equal comparison failed to convert both "    \
-                      "arguments to Unicode - interpreting them as being unequal"
-            space.warn(msg, space.w_UnicodeWarning)
-            return space.newbool(inverse)
-        raise
-    result = space.eq(w_uni, w_uni2)
-    if inverse:
-        return space.not_(result)
-    return result
-
 def str__Unicode(space, w_uni):
     from pypy.objspace.std.unicodetype import encode_object
     return encode_object(space, w_uni, None, None)
@@ -105,20 +84,6 @@
 def eq__Unicode_Unicode(space, w_left, w_right):
     return space.newbool(w_left._value == w_right._value)
 
-def eq__Unicode_String(space, w_uni, w_str):
-    from pypy.objspace.std.unicodetype import unicode_from_string
-    return _unicode_string_comparison(space, w_uni, w_str,
-                    False, unicode_from_string)
-
-eq__Unicode_Rope = eq__Unicode_String
-
-def ne__Unicode_String(space, w_uni, w_str):
-    from pypy.objspace.std.unicodetype import unicode_from_string
-    return _unicode_string_comparison(space, w_uni, w_str,
-                    True, unicode_from_string)
-
-ne__Unicode_Rope = ne__Unicode_String
-
 def lt__Unicode_Unicode(space, w_left, w_right):
     left = w_left._value
     right = w_right._value
@@ -137,38 +102,6 @@
 def add__Unicode_Unicode(space, w_left, w_right):
     return W_UnicodeObject(w_left._value + w_right._value)
 
-def add__String_Unicode(space, w_left, w_right):
-    # this function is needed to make 'abc'.__add__(u'def') return
-    # u'abcdef' instead of NotImplemented.  This is what occurs on
-    # top of CPython.
-    from pypy.objspace.std.unicodetype import unicode_from_string
-    # XXX fragile implementation detail: for "string + unicode subclass",
-    # if the unicode subclass overrides __radd__(), then it will be
-    # called (see test_str_unicode_concat_overrides).  This occurs as a
-    # result of the following call to space.add() in which the first
-    # argument is a unicode and the second argument a subclass of unicode
-    # (and thus the usual logic about calling __radd__() first applies).
-    return space.add(unicode_from_string(space, w_left) , w_right)
-
-add__Rope_Unicode = add__String_Unicode
-
-def add__Unicode_String(space, w_left, w_right):
-    # this function is needed to make 'abc'.__radd__(u'def') return
-    # u'defabc', although it's completely unclear if that's necessary
-    # given that CPython doesn't even have a method str.__radd__().
-    from pypy.objspace.std.unicodetype import unicode_from_string
-    return space.add(w_left, unicode_from_string(space, w_right))
-    # Note about "unicode + string subclass": look for
-    # "cpython bug compatibility" in descroperation.py
-
-add__Unicode_Rope = add__Unicode_String
-
-def contains__String_Unicode(space, w_container, w_item):
-    from pypy.objspace.std.unicodetype import unicode_from_string
-    return space.contains(unicode_from_string(space, w_container), w_item )
-contains__Rope_Unicode = contains__String_Unicode
-
-
 def contains__Unicode_Unicode(space, w_container, w_item):
     item = w_item._value
     container = w_container._value
@@ -373,33 +306,16 @@
     return _strip_none(space, w_self, 1, 1)
 def unicode_strip__Unicode_Unicode(space, w_self, w_chars):
     return _strip(space, w_self, w_chars, 1, 1)
-def unicode_strip__Unicode_String(space, w_self, w_chars):
-    from pypy.objspace.std.unicodetype import unicode_from_string
-    return space.call_method(w_self, 'strip',
-                             unicode_from_string(space, w_chars))
-unicode_strip__Unicode_Rope = unicode_strip__Unicode_String
 
 def unicode_lstrip__Unicode_None(space, w_self, w_chars):
     return _strip_none(space, w_self, 1, 0)
 def unicode_lstrip__Unicode_Unicode(space, w_self, w_chars):
     return _strip(space, w_self, w_chars, 1, 0)
-def unicode_lstrip__Unicode_String(space, w_self, w_chars):
-    from pypy.objspace.std.unicodetype import unicode_from_string
-    return space.call_method(w_self, 'lstrip',
-                             unicode_from_string(space, w_chars))
-
-unicode_lstrip__Unicode_Rope = unicode_lstrip__Unicode_String
 
 def unicode_rstrip__Unicode_None(space, w_self, w_chars):
     return _strip_none(space, w_self, 0, 1)
 def unicode_rstrip__Unicode_Unicode(space, w_self, w_chars):
     return _strip(space, w_self, w_chars, 0, 1)
-def unicode_rstrip__Unicode_String(space, w_self, w_chars):
-    from pypy.objspace.std.unicodetype import unicode_from_string
-    return space.call_method(w_self, 'rstrip',
-                             unicode_from_string(space, w_chars))
-
-unicode_rstrip__Unicode_Rope = unicode_rstrip__Unicode_String
 
 def unicode_capitalize__Unicode(space, w_self):
     input = w_self._value
@@ -928,71 +844,3 @@
 from pypy.objspace.std import unicodetype
 register_all(vars(), unicodetype)
 
-# str.strip(unicode) needs to convert self to unicode and call unicode.strip we
-# use the following magic to register strip_string_unicode as a String
-# multimethod.
-
-# XXX couldn't string and unicode _share_ the multimethods that make up their
-# methods?
-
-class str_methods:
-    from pypy.objspace.std import stringtype
-    W_UnicodeObject = W_UnicodeObject
-    from pypy.objspace.std.stringobject import W_StringObject
-    from pypy.objspace.std.ropeobject import W_RopeObject
-    def str_strip__String_Unicode(space, w_self, w_chars):
-        from pypy.objspace.std.unicodetype import unicode_from_string
-        return space.call_method(unicode_from_string(space, w_self),
-                                 'strip', w_chars)
-    str_strip__Rope_Unicode = str_strip__String_Unicode
-    def str_lstrip__String_Unicode(space, w_self, w_chars):
-        from pypy.objspace.std.unicodetype import unicode_from_string
-        return space.call_method(unicode_from_string(space, w_self),
-                                 'lstrip', w_chars)
-    str_lstrip__Rope_Unicode = str_lstrip__String_Unicode
-    def str_rstrip__String_Unicode(space, w_self, w_chars):
-        from pypy.objspace.std.unicodetype import unicode_from_string
-        return space.call_method(unicode_from_string(space, w_self),
-                                 'rstrip', w_chars)
-    str_rstrip__Rope_Unicode = str_rstrip__String_Unicode
-    def str_count__String_Unicode_ANY_ANY(space, w_self, w_substr, w_start, w_end):
-        from pypy.objspace.std.unicodetype import unicode_from_string
-        return space.call_method(unicode_from_string(space, w_self),
-                                 'count', w_substr, w_start, w_end)
-    str_count__Rope_Unicode_ANY_ANY = str_count__String_Unicode_ANY_ANY
-    def str_find__String_Unicode_ANY_ANY(space, w_self, w_substr, w_start, w_end):
-        from pypy.objspace.std.unicodetype import unicode_from_string
-        return space.call_method(unicode_from_string(space, w_self),
-                                 'find', w_substr, w_start, w_end)
-    str_find__Rope_Unicode_ANY_ANY = str_find__String_Unicode_ANY_ANY
-    def str_rfind__String_Unicode_ANY_ANY(space, w_self, w_substr, w_start, w_end):
-        from pypy.objspace.std.unicodetype import unicode_from_string
-        return space.call_method(unicode_from_string(space, w_self),
-                                 'rfind', w_substr, w_start, w_end)
-    str_rfind__Rope_Unicode_ANY_ANY = str_rfind__String_Unicode_ANY_ANY
-    def str_index__String_Unicode_ANY_ANY(space, w_self, w_substr, w_start, w_end):
-        from pypy.objspace.std.unicodetype import unicode_from_string
-        return space.call_method(unicode_from_string(space, w_self),
-                                 'index', w_substr, w_start, w_end)
-    str_index__Rope_Unicode_ANY_ANY = str_index__String_Unicode_ANY_ANY
-    def str_rindex__String_Unicode_ANY_ANY(space, w_self, w_substr, w_start, w_end):
-        from pypy.objspace.std.unicodetype import unicode_from_string
-        return space.call_method(unicode_from_string(space, w_self),
-                                 'rindex', w_substr, w_start, w_end)
-    str_rindex__Rope_Unicode_ANY_ANY = str_rindex__String_Unicode_ANY_ANY
-    def str_replace__String_Unicode_Unicode_ANY(space, w_self, w_old, w_new, w_maxsplit):
-        from pypy.objspace.std.unicodetype import unicode_from_string
-        return space.call_method(unicode_from_string(space, w_self),
-                                 'replace', w_old, w_new, w_maxsplit)
-    str_replace__Rope_Unicode_Unicode_ANY = str_replace__String_Unicode_Unicode_ANY
-    def str_split__String_Unicode_ANY(space, w_self, w_delim, w_maxsplit):
-        from pypy.objspace.std.unicodetype import unicode_from_string
-        return space.call_method(unicode_from_string(space, w_self),
-                                 'split', w_delim, w_maxsplit)
-    str_split__Rope_Unicode_ANY = str_split__String_Unicode_ANY
-    def str_rsplit__String_Unicode_ANY(space, w_self, w_delim, w_maxsplit):
-        from pypy.objspace.std.unicodetype import unicode_from_string
-        return space.call_method(unicode_from_string(space, w_self),
-                                 'rsplit', w_delim, w_maxsplit)
-    str_rsplit__Rope_Unicode_ANY = str_rsplit__String_Unicode_ANY
-    register_all(vars(), stringtype)
diff --git a/pypy/objspace/std/unicodetype.py b/pypy/objspace/std/unicodetype.py
--- a/pypy/objspace/std/unicodetype.py
+++ b/pypy/objspace/std/unicodetype.py
@@ -313,22 +313,6 @@
             return w_res
     return unicode_from_encoded_object(space, w_res, None, "strict")
 
-def unicode_from_string(space, w_str):
-    # this is a performance and bootstrapping hack
-    if space.config.objspace.std.withropeunicode:
-        from pypy.objspace.std.ropeunicodeobject import unicode_from_string
-        return unicode_from_string(space, w_str)
-    encoding = getdefaultencoding(space)
-    from pypy.objspace.std.unicodeobject import W_UnicodeObject
-    if encoding != 'ascii':
-        return unicode_from_encoded_object(space, w_str, encoding, "strict")
-    s = space.str_w(w_str)
-    try:
-        return W_UnicodeObject(s.decode("ascii"))
-    except UnicodeDecodeError:
-        # raising UnicodeDecodeError is messy, "please crash for me"
-        return unicode_from_encoded_object(space, w_str, "ascii", "strict")
-
 def unicode_decode__unitypedef_ANY_ANY(space, w_unicode, w_encoding=None,
                                        w_errors=None):
     return space.call_method(space.str(w_unicode), 'decode',

From noreply at buildbot.pypy.org  Wed Oct 12 22:23:41 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 12 Oct 2011 22:23:41 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: space.wrap("xxx") now returns a unicode
	string!
Message-ID: <20111012202341.AB3C982112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r47999:ef504fcb59bb
Date: 2011-10-12 22:19 +0200
http://bitbucket.org/pypy/pypy/changeset/ef504fcb59bb/

Log:	space.wrap("xxx") now returns a unicode string!
	space.str_w(w_someunicode) return a RPython bytestring! (use
	space.wrapbytes and space.bytes_w to get the previous behavior)

diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -1758,6 +1758,7 @@
     _col_offset_mask = 8
 
     def __init__(self, id, ctx, lineno, col_offset):
+        assert isinstance(id, str)
         self.id = id
         self.ctx = ctx
         expr.__init__(self, lineno, col_offset)
diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -503,6 +503,8 @@
         return name
 
     def handle_arguments(self, arguments_node):
+        # This function handles both typedargslist (function definition)
+        # and varargslist (lambda definition).
         if arguments_node.type == syms.parameters:
             if len(arguments_node.children) == 2:
                 return ast.arguments(None, None, None, None)
@@ -517,7 +519,7 @@
         while i < child_count:
             argument = arguments_node.children[i]
             arg_type = argument.type
-            if arg_type == syms.tfpdef:
+            if arg_type == syms.tfpdef or arg_type == syms.vfpdef:
                 parenthesized = False
                 complex_args = False
                 while True:
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -188,13 +188,13 @@
 
     # -------------------------------------------------------------------
 
-    def str_w(self, space):
-        w_msg = typed_unwrap_error_msg(space, "string", self)
+    def bytes_w(self, space):
+        w_msg = typed_unwrap_error_msg(space, "bytes", self)
         raise OperationError(space.w_TypeError, w_msg)
 
     def unicode_w(self, space):
         raise OperationError(space.w_TypeError,
-                             typed_unwrap_error_msg(space, "unicode", self))
+                             typed_unwrap_error_msg(space, "string", self))
 
     def int_w(self, space):
         raise OperationError(space.w_TypeError,
@@ -1233,7 +1233,10 @@
         return self.str_w(w_obj)
 
     def str_w(self, w_obj):
-        return w_obj.str_w(self)
+        return self.unicode_w(w_obj).encode('ascii')
+
+    def bytes_w(self, w_obj):
+        return w_obj.bytes_w(self)
 
     def int_w(self, w_obj):
         return w_obj.int_w(self)
@@ -1561,7 +1564,7 @@
 
 ObjSpace.IrregularOpTable = [
     'wrap',
-    'str_w',
+    'bytes_w',
     'int_w',
     'float_w',
     'uint_w',
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -66,7 +66,7 @@
             exc_value    = str(w_value)
         else:
             w = space.wrap
-            if space.is_w(space.type(self.w_type), space.w_str):
+            if space.is_w(space.type(self.w_type), space.w_text):
                 exc_typename = space.str_w(self.w_type)
             else:
                 exc_typename = space.str_w(
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -721,7 +721,6 @@
     def IMPORT_NAME(self, nameindex, next_instr):
         space = self.space
         w_modulename = self.getname_w(nameindex)
-        modulename = self.space.str_w(w_modulename)
         w_fromlist = self.popvalue()
 
         w_flag = self.popvalue()
@@ -739,7 +738,6 @@
         w_locals = self.w_locals
         if w_locals is None:            # CPython does this
             w_locals = space.w_None
-        w_modulename = space.wrap(modulename)
         w_globals = self.w_globals
         if w_flag is None:
             w_obj = space.call_function(w_import, w_modulename, w_globals,
diff --git a/pypy/interpreter/pyparser/parsestring.py b/pypy/interpreter/pyparser/parsestring.py
--- a/pypy/interpreter/pyparser/parsestring.py
+++ b/pypy/interpreter/pyparser/parsestring.py
@@ -13,6 +13,8 @@
     rawmode = False
     unicode = True
 
+    assert isinstance(s, str)
+
     # string decoration handling
     o = ord(quote)
     isalpha = (o>=97 and o<=122) or (o>=65 and o<=90)
@@ -95,13 +97,13 @@
             w_v = unicodehelper.PyUnicode_AsEncodedString(space, w_u, space.wrap(encoding))
             return w_v
         else:
-            return space.wrap(substr)
+            return space.wrapbytes(substr)
 
     enc = None
     if need_encoding:
          enc = encoding
     v = PyString_DecodeEscape(space, substr, enc)
-    return space.wrap(v)
+    return space.wrapbytes(v)
 
 def hexbyte(val):
     result = "%x" % val
diff --git a/pypy/interpreter/pyparser/pyparse.py b/pypy/interpreter/pyparser/pyparse.py
--- a/pypy/interpreter/pyparser/pyparse.py
+++ b/pypy/interpreter/pyparser/pyparse.py
@@ -4,14 +4,16 @@
 from pypy.interpreter.astcompiler import consts
 
 
-_recode_to_utf8 = gateway.applevel(r'''
-    def _recode_to_utf8(text, encoding):
-        return unicode(text, encoding).encode("utf-8")
-''').interphook('_recode_to_utf8')
+def decode_source(space, bytes, encoding=None):
+    if encoding is None:
+        encoding = 'utf-8'
+    if encoding == 'utf-8':
+        return bytes
+    text = space.unicode_w(space.call_function(space.w_unicode,
+                                               space.wrapbytes(bytes),
+                                               space.wrap(encoding)))
+    return text.encode('utf-8')
 
-def recode_to_utf8(space, text, encoding):
-    return space.str_w(_recode_to_utf8(space, space.wrap(text),
-                                          space.wrap(encoding)))
 def _normalize_encoding(encoding):
     """returns normalized name for <encoding>
 
@@ -94,7 +96,7 @@
         self.space = space
         self.future_flags = future_flags
 
-    def parse_source(self, textsrc, compile_info):
+    def parse_source(self, bytessrc, compile_info):
         """Main entry point for parsing Python source.
 
         Everything from decoding the source to tokenizing to building the parse
@@ -102,38 +104,39 @@
         """
         # Detect source encoding.
         enc = None
-        if textsrc.startswith("\xEF\xBB\xBF"):
-            textsrc = textsrc[3:]
+        if bytessrc.startswith("\xEF\xBB\xBF"):
+            bytessrc = bytessrc[3:]
             enc = 'utf-8'
             # If an encoding is explicitly given check that it is utf-8.
-            decl_enc = _check_for_encoding(textsrc)
+            decl_enc = _check_for_encoding(bytessrc)
             if decl_enc and decl_enc != "utf-8":
                 raise error.SyntaxError("UTF-8 BOM with non-utf8 coding cookie",
                                         filename=compile_info.filename)
+            textsrc = decode_source(self.space, bytessrc, enc)
         elif compile_info.flags & consts.PyCF_SOURCE_IS_UTF8:
             enc = 'utf-8'
-            if _check_for_encoding(textsrc) is not None:
+            if _check_for_encoding(bytessrc) is not None:
                 raise error.SyntaxError("coding declaration in unicode string",
                                         filename=compile_info.filename)
+            textsrc = decode_source(self.space, bytessrc, enc)
         else:
-            enc = _normalize_encoding(_check_for_encoding(textsrc))
-            if enc is not None and enc not in ('utf-8', 'iso-8859-1'):
-                try:
-                    textsrc = recode_to_utf8(self.space, textsrc, enc)
-                except OperationError, e:
-                    # if the codec is not found, LookupError is raised.  we
-                    # check using 'is_w' not to mask potential IndexError or
-                    # KeyError
-                    space = self.space
-                    if e.match(space, space.w_LookupError):
-                        raise error.SyntaxError("Unknown encoding: %s" % enc,
-                                                filename=compile_info.filename)
-                    # Transform unicode errors into SyntaxError
-                    if e.match(space, space.w_UnicodeDecodeError):
-                        e.normalize_exception(space)
-                        w_message = space.str(e.get_w_value(space))
-                        raise error.SyntaxError(space.str_w(w_message))
-                    raise
+            enc = _normalize_encoding(_check_for_encoding(bytessrc))
+            try:
+                textsrc = decode_source(self.space, bytessrc, enc)
+            except OperationError, e:
+                # if the codec is not found, LookupError is raised.  we
+                # check using 'is_w' not to mask potential IndexError or
+                # KeyError
+                space = self.space
+                if e.match(space, space.w_LookupError):
+                    raise error.SyntaxError("Unknown encoding: %s" % enc,
+                                            filename=compile_info.filename)
+                # Transform unicode errors into SyntaxError
+                if e.match(space, space.w_UnicodeDecodeError):
+                    e.normalize_exception(space)
+                    w_message = space.str(e.get_w_value(space))
+                    raise error.SyntaxError(space.text_w(w_message))
+                raise
 
         f_flags, future_info = future.get_futures(self.future_flags, textsrc)
         compile_info.last_future_import = future_info
diff --git a/pypy/module/__builtin__/compiling.py b/pypy/module/__builtin__/compiling.py
--- a/pypy/module/__builtin__/compiling.py
+++ b/pypy/module/__builtin__/compiling.py
@@ -26,18 +26,14 @@
 
     ast_node = None
     w_ast_type = space.gettypeobject(ast.AST.typedef)
-    str_ = None
+    source_str = None
     if space.is_true(space.isinstance(w_source, w_ast_type)):
         ast_node = space.interp_w(ast.mod, w_source)
         ast_node.sync_app_attrs(space)
-    elif space.is_true(space.isinstance(w_source, space.w_unicode)):
-        w_utf_8_source = space.call_method(w_source, "encode",
-                                           space.wrap("utf-8"))
-        str_ = space.str_w(w_utf_8_source)
+    else:
+        source_str = space.str_w(w_source)
         # This flag tells the parser to reject any coding cookies it sees.
         flags |= consts.PyCF_SOURCE_IS_UTF8
-    else:
-        str_ = space.str_w(w_source)
 
     ec = space.getexecutioncontext()
     if flags & ~(ec.compiler.compiler_flags | consts.PyCF_ONLY_AST |
@@ -56,10 +52,10 @@
 
     if ast_node is None:
         if flags & consts.PyCF_ONLY_AST:
-            mod = ec.compiler.compile_to_ast(str_, filename, mode, flags)
+            mod = ec.compiler.compile_to_ast(source_str, filename, mode, flags)
             return space.wrap(mod)
         else:
-            code = ec.compiler.compile(str_, filename, mode, flags)
+            code = ec.compiler.compile(source_str, filename, mode, flags)
     else:
         code = ec.compiler.compile_ast(ast_node, filename, mode, flags)
     return space.wrap(code)
diff --git a/pypy/module/__builtin__/operation.py b/pypy/module/__builtin__/operation.py
--- a/pypy/module/__builtin__/operation.py
+++ b/pypy/module/__builtin__/operation.py
@@ -47,8 +47,8 @@
     # space.{get,set,del}attr()...
     # Note that if w_name is already a string (or a subclass of str),
     # it must be returned unmodified (and not e.g. unwrapped-rewrapped).
-    if not space.is_w(space.type(w_name), space.w_str):
-        name = space.str_w(w_name)    # typecheck
+    if not space.is_w(space.type(w_name), space.w_text):
+        name = space.text_w(w_name)   # typecheck
         w_name = space.wrap(name)     # rewrap as a real string
     return w_name
 
diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -386,7 +386,7 @@
         state = space.fromcache(CodecState)
         func = getattr(runicode, rname)
         result = func(uni, len(uni), errors, state.encode_error_handler)
-        return space.newtuple([space.wrap(result), space.wrap(len(uni))])
+        return space.newtuple([space.wrapbytes(result), space.wrap(len(uni))])
     wrap_encoder.func_name = rname
     globals()[name] = wrap_encoder
 
diff --git a/pypy/module/exceptions/interp_exceptions.py b/pypy/module/exceptions/interp_exceptions.py
--- a/pypy/module/exceptions/interp_exceptions.py
+++ b/pypy/module/exceptions/interp_exceptions.py
@@ -627,11 +627,11 @@
 
     def descr_init(self, space, w_encoding, w_object, w_start, w_end, w_reason):
         # typechecking
-        space.realstr_w(w_encoding)
-        space.realstr_w(w_object)
+        space.text_w(w_encoding)
+        space.str_w(w_object)
         space.int_w(w_start)
         space.int_w(w_end)
-        space.realstr_w(w_reason)
+        space.text_w(w_reason)
         # assign attributes
         self.w_encoding = w_encoding
         self.w_object = w_object
diff --git a/pypy/module/marshal/interp_marshal.py b/pypy/module/marshal/interp_marshal.py
--- a/pypy/module/marshal/interp_marshal.py
+++ b/pypy/module/marshal/interp_marshal.py
@@ -28,7 +28,7 @@
 by dump(data, file)."""
     m = StringMarshaller(space, space.int_w(w_version))
     m.dump_w_obj(w_data)
-    return space.wrap(m.get_value())
+    return space.wrapbytes(m.get_value())
 
 def load(space, w_f):
     """Read one value from the file 'f' and return it."""
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -42,7 +42,7 @@
         'argv'                  : 'state.get(space).w_argv',
         'py3kwarning'           : 'space.w_False',
         'warnoptions'           : 'state.get(space).w_warnoptions', 
-        'builtin_module_names'  : 'state.w_None',
+        'builtin_module_names'  : 'space.w_None',
         'pypy_getudir'          : 'state.pypy_getudir',    # not translated
         'pypy_initial_path'     : 'state.pypy_initial_path',
 
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -431,7 +431,7 @@
 
     def is_correct_type(self, w_obj):
         space = self.space
-        return space.is_w(space.type(w_obj), space.w_str)
+        return space.is_w(space.type(w_obj), space.w_text)
 
     def get_empty_storage(self):
         res = {}
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -68,6 +68,7 @@
             w_type = self.gettypeobject(typedef)
             self.builtin_types[typedef.name] = w_type
             setattr(self, 'w_' + typedef.name, w_type)
+        self.w_text = self.w_unicode
         self.builtin_types["NotImplemented"] = self.w_NotImplemented
         self.builtin_types["Ellipsis"] = self.w_Ellipsis
 
@@ -149,6 +150,9 @@
         assert typedef is not None
         return self.fromcache(stdtypedef.TypeCache).getorbuild(typedef)
 
+    def wrapbytes(self, bytes):
+        return wrapstr(self, bytes)
+
     def wrap(self, x):
         "Wraps the Python value 'x' into one of the wrapper classes."
         # You might notice that this function is rather conspicuously
@@ -170,7 +174,7 @@
             else:
                 return self.newint(x)
         if isinstance(x, str):
-            return wrapstr(self, x)
+            return wrapunicode(self, x.decode('ascii'))
         if isinstance(x, unicode):
             return wrapunicode(self, x)
         if isinstance(x, float):
diff --git a/pypy/objspace/std/ropeunicodeobject.py b/pypy/objspace/std/ropeunicodeobject.py
--- a/pypy/objspace/std/ropeunicodeobject.py
+++ b/pypy/objspace/std/ropeunicodeobject.py
@@ -78,9 +78,6 @@
         # for testing
         return w_self._node.flatten_unicode()
 
-    def str_w(w_self, space):
-        return space.str_w(space.str(w_self))
-
     def create_if_subclassed(w_self):
         if type(w_self) is W_RopeUnicodeObject:
             return w_self
diff --git a/pypy/objspace/std/strsliceobject.py b/pypy/objspace/std/strsliceobject.py
--- a/pypy/objspace/std/strsliceobject.py
+++ b/pypy/objspace/std/strsliceobject.py
@@ -30,7 +30,7 @@
         w_self.stop = len(str)
         return str
 
-    def str_w(w_self, space):
+    def bytes_w(w_self, space):
         return w_self.force()
 
     def __repr__(w_self):
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -39,9 +39,6 @@
             return w_self
         return W_UnicodeObject(w_self._value)
 
-    def str_w(self, space):
-        return space.str_w(space.str(self))
-
     def unicode_w(self, space):
         return self._value
 
diff --git a/pypy/objspace/std/unicodetype.py b/pypy/objspace/std/unicodetype.py
--- a/pypy/objspace/std/unicodetype.py
+++ b/pypy/objspace/std/unicodetype.py
@@ -25,7 +25,7 @@
                     space.w_UnicodeDecodeError,
                     space.newtuple([
                     space.wrap('ascii'),
-                    space.wrap(s),
+                    space.wrapbytes(s),
                     space.wrap(i),
                     space.wrap(i+1),
                     space.wrap("ordinal not in range(128)")]))
@@ -191,7 +191,7 @@
                                        startingpos, endingpos):
         raise OperationError(space.w_UnicodeDecodeError,
                              space.newtuple([space.wrap(encoding),
-                                             space.wrap(s),
+                                             space.wrapbytes(s),
                                              space.wrap(startingpos),
                                              space.wrap(endingpos),
                                              space.wrap(msg)]))

From noreply at buildbot.pypy.org  Wed Oct 12 22:39:07 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 12 Oct 2011 22:39:07 +0200 (CEST)
Subject: [pypy-commit] pypy.org extradoc: numpy donation campaign
Message-ID: <20111012203907.0E10682112@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: extradoc
Changeset: r280:e9995d9c9546
Date: 2011-10-12 22:38 +0200
http://bitbucket.org/pypy/pypy.org/changeset/e9995d9c9546/

Log:	numpy donation campaign

diff --git a/numpydonate.html b/numpydonate.html
new file mode 100644
--- /dev/null
+++ b/numpydonate.html
@@ -0,0 +1,201 @@
+<!DOCTYPE html>
+<html>
+<head>
+	<title>PyPy :: Call for donations - PyPy to support Numpy!</title>
+	<meta http-equiv="content-language" content="en" />
+	<meta http-equiv="content-type" content="text/html; charset=utf-8" />
+	<meta name="author" content="PyPy Team" />
+	<meta name="description" content="PyPy" />
+	<meta name="copyright" content="MIT" />
+	<meta name="document-rating" content="general" />
+	<link rel="stylesheet" type="text/css" media="screen" title="default" href="css/site.css" />
+	<link rel="alternate" type="application/rss+xml" title="RSS Feed for PyPy" href="http://feeds.feedburner.com/PyPyStatusBlog" />
+  <link rel="stylesheet" type="text/css" href="css/jquery-ui-1.8.14.custom.css" />
+	<script type="text/javascript" src="http://use.typekit.com/hdt8sni.js"></script>
+	<script type="text/javascript">try{Typekit.load();}catch(e){}</script>
+	<script type="text/javascript" src="http://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script>
+  <script type="text/javascript" src="js/jquery-ui-1.8.14.custom.min.js"></script>
+  <script type="text/javascript" src="js/detect.js"></script>
+  <script type="text/javascript" src="js/script.js"></script>
+</head>
+<body>
+<script type="text/javascript">
+	var _gaq = [['_setAccount', 'UA-7778406-3'], ['_trackPageview']];
+	if (document.location.protocol !== 'file:') {
+		(function() {
+			var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
+			ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
+			(document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(ga);
+		})();
+	}
+</script>
+<div id="body-outer"><div id="body-inner"><div id="body" class="clearfix">
+<div id="header">
+	<div id="menu-follow">
+		<div><a href="http://search.twitter.com/search?q=%23pypy" title="Follow the conversation on Twitter"><img src="http://static.ampify.it/icon.twitter.gif" alt="Follow the conversation on Twitter" width="14px" height="14px" /></a></div>
+    <div><a href="https://bitbucket.org/pypy/pypy"><img src="http://www.selenic.com/hg-logo/logo-droplets-25.png" width="14px" height="14px" /></a></div>
+		<div><a href="http://feeds.feedburner.com/PyPyStatusBlog" title="Subscribe to the RSS Feed"><img src="http://static.ampify.it/icon.rss.png" alt="Subscribe to the RSS Feed" width="14px" height="14px" /></a></div>
+	</div>
+	<div id="logo"><a href="http://pypy.org"><img src="image/pypy-logo.png" alt="PyPy" height="110px" /></a></div>
+	<hr class="clear-left" />
+	<div id="menu-sub"><a href="index.html">Home</a><span class="menu-sub-sep"> | </span><a href="features.html">Features</a><span class="menu-sub-sep"> | </span><a href="download.html">Download</a><span class="menu-sub-sep"> | </span><a href="compat.html">Compatibility</a><span class="menu-sub-sep"> | </span><a href="http://speed.pypy.org">Performance</a><span class="menu-sub-sep"> | </span><a href="http://doc.pypy.org">Dev Documentation</a><span class="menu-sub-sep"> | </span><a href="http://morepypy.blogspot.com">Blog</a><span class="menu-sub-sep"> | </span><a href="people.html">People</a><span class="menu-sub-sep"> | </span><a href="contact.html">Contact</a><span class="menu-sub-sep"> | </span><a href="py3donate.html">Py3k donations</a></div>
+	<hr class="clear" />
+</div>
+<div id="content">
+<div>
+<div id="main">
+<h1 class="title">Call for donations - PyPy to support Numpy!</h1>
+<div class="section" id="make-numpy-for-pypy-happen">
+<h1>Make NumPy for PyPy happen</h1>
+<p>This is a proposal to provide a fully compatible working <a class="reference external" href="http://numpy.scipy.org/">NumPy</a> implementation
+for PyPy. This has long been a very commonly <a class="reference external" href="http://morepypy.blogspot.com/2011/06/report-back-from-our-survey.html">requested feature</a> for PyPy
+as well as a worthy goal given that PyPy performs extremely well on numeric
+workloads.</p>
+<p>We already had some success providing a very basic NumPy implementation,
+however we believe that raising funds can significantly speed up its
+development, by paying people to work full time on it.</p>
+<p>Below you'll find the <a class="reference internal" href="#work-plan">work plan</a> and the associated
+fundraising targets we need to make things happen.  Once we reach the
+necessary target for each stage, we will start contracting
+developers.  Contracts and money are managed by the non-profit
+<a class="reference external" href="http://sfconservancy.org/">Software Freedom Conservancy</a> of which the PyPy project is a member.
+The current elected representatives are Carl Friedrich Bolz, Holger
+Krekel and Jacob Hallen and they will &#8211; in close collaboration with
+Conservancy and the core developers &#8211; select the best developers for
+implementing NumPy among well known PyPy contributors.</p>
+<p>Should we not receive enough donations to complete all stages by 1st March 2012
+at the latest, we will try our best to make PyPy support NumPy anyway.  We
+however reserve the right to shift any unused funds to other PyPy activities
+when that date is reached.  Of course, since the Conservancy is a
+501(c)(3) charitable organization incorporated in NY, USA, all funds will,
+regardless of their use, be spent in a way that benefits the general
+public, the advancement of Open Source and Free Software,
+and in particular the PyPy community and the PyPy codebase.</p>
+<p><strong>Note</strong> For donations higher than $1,000, we can arrange for an invoice
+and a different payment method to avoid the high Paypal fees.  Please
+contact pypy at sfconservancy.org if you want to know details on how
+to donate via other means.</p>
+<div class="section" id="what-is-numpy">
+<h2>What is NumPy?</h2>
+<p><a class="reference external" href="http://numpy.scipy.org/">NumPy</a> is a framework for doing numerical calculations in Python. It has
+become the de-facto standard for doing any kinds of computations that involve
+n-dimensional arrays. Please consult the NumPy website for more details.</p>
+</div>
+<div class="section" id="why-does-numpy-on-pypy-makes-sense">
+<h2>Why does NumPy on PyPy makes sense?</h2>
+<p>NumPy on PyPy makes sense for a couple of reasons: Firstly, it is by
+far the most requested feature from PyPy. Secondly, PyPy <a class="reference external" href="http://morepypy.blogspot.com/2011/07/realtime-image-processing-in-python.html">performs
+well</a> on numerical loads already.  Therefore bringing NumPy into the
+equation is a reasonable next step - as it's a very convenient and popular tool
+for doing this kind of work. The resulting implementation should move
+Python in scientific world from being a merely &ldquo;glue&rdquo; language into
+being the main implementation language for a lot of people in
+the scientific/numeric worlds. This will benefit current users of NumPy as
+well as people who so far have to cope with lower level languages like C
+or Fortran for speed purposes.</p>
+</div>
+<div class="section" id="speed">
+<h2>Speed</h2>
+<p>The current implementation of NumPy on PyPy is reasonably fast - it ranges
+from roughly the same speed to 2-5x faster for stacked large array operations
+to 100-300x for accessing NumPy array elements one by one. The exact speed
+depends very much how NumPy is used, but the target would be to be within an
+order of magnitude from handwritten C.
+To achieve this, we would
+need to teach our JIT backends how to use modern vector instructions, like SSE
+or AVM. Hence, we split the proposal into two parts, first part covers compatibility
+with a reasonable approach to keeping current speed achievements, second part
+is about teaching the JIT how to vectorize certain operations, which should bring
+PyPy's NumPy as a very competitive tool compared to other available
+solutions for numerical computations, like matlab or C++ array libraries.</p>
+</div>
+</div>
+<div class="section" id="id1">
+<span id="work-plan"></span><h1>Work plan</h1>
+<div class="section" id="about-estimates-and-costs">
+<h2>About estimates and costs</h2>
+<p>For each step, we estimated the time that it would take to complete for an
+experienced developer who is already familiar with the PyPy codebase.  From
+this number, the money is calculated considering a hourly rate of $60, and a
+5% general donation which goes to the <a class="reference external" href="http://sfconservancy.org/">Software Freedom Conservancy</a> itself, the non-profit
+organization of which the PyPy project is a member and which manages all the
+issues related to donations, payments, and tax-exempt status.</p>
+<p>We split the proposal into two parts - we plan to implement them in the same
+order, starting by the time we raise the corresponding funding targets:</p>
+<ul>
+<li><p class="first"><strong>Compatibility</strong>:</p>
+<p>This part covers the core NumPy Python API. We'll implement all NumPy APIs
+that are officially documented and we'll pass all of NumPy's tests that
+cover documented APIs and are not implementation details.
+Specifically, we don't plan to:</p>
+<ul class="simple">
+<li>implement NumPy's C API</li>
+<li>implement other scientific libraries, like SciPy, matplotlib or biopython</li>
+</ul>
+<p>Estimated costs: USD$30,000. Estimated duration: 3 months.</p>
+</li>
+<li><p class="first"><strong>Speed</strong>:</p>
+<p>This part will cover significant speed improvements in the JIT that would
+make numeric computations faster. This includes, but is not necesarilly
+limited to:</p>
+<ul class="simple">
+<li>write a set of benchmarks covering various use cases</li>
+<li>teaching the JIT backend (or multiple backends) how to deal with vector
+operations, like SSE</li>
+<li>experiments with automatic parallelization using multiple threads, akin
+to numexpr</li>
+<li>improving the JIT register allocator that will make a difference, especially
+for tight loops</li>
+</ul>
+<p>As with all speed improvements, it's relatively hard to predict exactly
+how it'll cope, however we expect the results to be in the range of
+XXX I (cfbolz) find this claim too strong
+handwritten C equivalent, sometimes outperforming it in cases where static
+compilation of C prevents some advanced optimization techniques.</p>
+<p>Estimated costs: USD$30,000. Estimated duration: 3 months.</p>
+</li>
+</ul>
+</div>
+</div>
+<div class="section" id="benefits-of-this-work-to-the-python-community-and-the-general-public">
+<h1>Benefits of This Work to the Python Community and the General Public</h1>
+<p>Python has become one of the most popular dynamic programming languages in
+the world.  Web developers, educators, and scientific programmers alike
+all value Python because Python code is often more readable and because
+Python often increases programmer productivity.</p>
+<p>Traditionally, languages like Python ran more slowly than static, compiled
+languages; Python developers chose to sacrifice execution speed for ease
+of programming.  The PyPy project created a substantially improved Python
+language implementation, including a fast Just-in-time (JIT) compiler.
+The increased execution speed that PyPy provides has attracted many users,
+who now find their Python code runs up to four times faster under PyPy
+than under the reference implementation written in C.</p>
+<p>Meanwhile, adoption of Python is already underway for those researchers
+and developers who work specifically on computing that requires fast
+numeric operations.  Numpy support in PyPy will allow for Python's use by
+those developers and researchers who want the ease of programming that
+Python provides, the speed of PyPy, <em>and</em> the speedups for numerical work
+that Numpy can provide.</p>
+<p>PyPy's developers make all PyPy software available to the public without
+charge, under PyPy's Open Source copyright license, the permissive MIT
+License.  PyPy's license assures that PyPy is equally available to
+everyone freely on terms that allow both non-commercial and commercial
+activity.  This license allows for academics, for-profit software
+developers, volunteers and enthusiasts alike to collaborate together to
+make a better Python implementation for everyone.</p>
+<p>NumPy support for PyPy will be licensed similarly, and therefore NumPy
+support can directly help researchers and developers who seek to do
+numeric computing but want an easier programming language to use than
+Fortan or C, which is typically used for these applications.  Being
+licensed freely to the general public means that opportunities to use,
+improve and learn about how NumPy works itself will be generally available
+to everyone.</p>
+</div>
+</div>
+<div id="sidebar">
+</div>
+</div>
+</div>
+</div></div></div>
+</body>
+</html>
\ No newline at end of file
diff --git a/source/numpydonate.txt b/source/numpydonate.txt
new file mode 100644
--- /dev/null
+++ b/source/numpydonate.txt
@@ -0,0 +1,181 @@
+---
+layout: page
+title: Call for donations - PyPy to support Numpy!
+---
+
+Make NumPy for PyPy happen
+==========================
+
+This is a proposal to provide a fully compatible working `NumPy`_ implementation
+for PyPy. This has long been a very commonly `requested feature`_ for PyPy
+as well as a worthy goal given that PyPy performs extremely well on numeric
+workloads.
+
+We already had some success providing a very basic NumPy implementation,
+however we believe that raising funds can significantly speed up its
+development, by paying people to work full time on it.
+
+Below you'll find the `work plan`_ and the associated
+fundraising targets we need to make things happen.  Once we reach the
+necessary target for each stage, we will start contracting
+developers.  Contracts and money are managed by the non-profit
+`Software Freedom Conservancy`_ of which the PyPy project is a member.
+The current elected representatives are Carl Friedrich Bolz, Holger
+Krekel and Jacob Hallen and they will &#8211; in close collaboration with
+Conservancy and the core developers &#8211; select the best developers for
+implementing NumPy among well known PyPy contributors.
+
+Should we not receive enough donations to complete all stages by 1st March 2012
+at the latest, we will try our best to make PyPy support NumPy anyway.  We
+however reserve the right to shift any unused funds to other PyPy activities
+when that date is reached.  Of course, since the Conservancy is a
+501(c)(3) charitable organization incorporated in NY, USA, all funds will,
+regardless of their use, be spent in a way that benefits the general
+public, the advancement of Open Source and Free Software,
+and in particular the PyPy community and the PyPy codebase. 
+
+**Note** For donations higher than $1,000, we can arrange for an invoice
+and a different payment method to avoid the high Paypal fees.  Please
+contact pypy at sfconservancy.org if you want to know details on how
+to donate via other means.
+
+.. _`Software Freedom Conservancy`: http://sfconservancy.org/
+
+What is NumPy?
+--------------
+
+`NumPy`_ is a framework for doing numerical calculations in Python. It has
+become the de-facto standard for doing any kinds of computations that involve
+n-dimensional arrays. Please consult the NumPy website for more details.
+
+Why does NumPy on PyPy makes sense?
+-----------------------------------
+
+NumPy on PyPy makes sense for a couple of reasons: Firstly, it is by
+far the most requested feature from PyPy. Secondly, PyPy `performs
+well`_ on numerical loads already.  Therefore bringing NumPy into the
+equation is a reasonable next step - as it's a very convenient and popular tool
+for doing this kind of work. The resulting implementation should move
+Python in scientific world from being a merely "glue" language into
+being the main implementation language for a lot of people in
+the scientific/numeric worlds. This will benefit current users of NumPy as
+well as people who so far have to cope with lower level languages like C
+or Fortran for speed purposes.
+
+.. _`NumPy`: http://numpy.scipy.org/
+.. _`requested feature`: http://morepypy.blogspot.com/2011/06/report-back-from-our-survey.html
+.. _`NumPy imlpementation`: http://morepypy.blogspot.com/2011/05/numpy-in-pypy-status-and-roadmap.html
+.. _`performs well`: http://morepypy.blogspot.com/2011/07/realtime-image-processing-in-python.html
+
+Speed
+-----
+
+The current implementation of NumPy on PyPy is reasonably fast - it ranges
+from roughly the same speed to 2-5x faster for stacked large array operations
+to 100-300x for accessing NumPy array elements one by one. The exact speed
+depends very much how NumPy is used, but the target would be to be within an
+order of magnitude from handwritten C.
+To achieve this, we would
+need to teach our JIT backends how to use modern vector instructions, like SSE
+or AVM. Hence, we split the proposal into two parts, first part covers compatibility
+with a reasonable approach to keeping current speed achievements, second part
+is about teaching the JIT how to vectorize certain operations, which should bring
+PyPy's NumPy as a very competitive tool compared to other available
+solutions for numerical computations, like matlab or C++ array libraries.
+
+.. _`work plan`:
+
+Work plan
+=========
+
+About estimates and costs
+-------------------------
+
+For each step, we estimated the time that it would take to complete for an
+experienced developer who is already familiar with the PyPy codebase.  From
+this number, the money is calculated considering a hourly rate of $60, and a
+5% general donation which goes to the `Software Freedom Conservancy`_ itself, the non-profit
+organization of which the PyPy project is a member and which manages all the
+issues related to donations, payments, and tax-exempt status.
+
+.. _`Software Freedom Conservancy`: http://sfconservancy.org/
+
+We split the proposal into two parts - we plan to implement them in the same
+order, starting by the time we raise the corresponding funding targets:
+
+* **Compatibility**:
+
+  This part covers the core NumPy Python API. We'll implement all NumPy APIs
+  that are officially documented and we'll pass all of NumPy's tests that
+  cover documented APIs and are not implementation details.
+  Specifically, we don't plan to:
+
+  * implement NumPy's C API
+
+  * implement other scientific libraries, like SciPy, matplotlib or biopython
+
+  Estimated costs: USD$30,000. Estimated duration: 3 months.
+
+* **Speed**:
+
+  This part will cover significant speed improvements in the JIT that would
+  make numeric computations faster. This includes, but is not necesarilly
+  limited to:
+
+  * write a set of benchmarks covering various use cases
+
+  * teaching the JIT backend (or multiple backends) how to deal with vector
+    operations, like SSE
+
+  * experiments with automatic parallelization using multiple threads, akin
+    to numexpr
+
+  * improving the JIT register allocator that will make a difference, especially
+    for tight loops
+
+  As with all speed improvements, it's relatively hard to predict exactly
+  how it'll cope, however we expect the results to be in the range of
+  XXX I (cfbolz) find this claim too strong
+  handwritten C equivalent, sometimes outperforming it in cases where static
+  compilation of C prevents some advanced optimization techniques.
+
+  Estimated costs: USD$30,000. Estimated duration: 3 months.
+
+Benefits of This Work to the Python Community and the General Public
+====================================================================
+
+Python has become one of the most popular dynamic programming languages in
+the world.  Web developers, educators, and scientific programmers alike
+all value Python because Python code is often more readable and because
+Python often increases programmer productivity.
+
+Traditionally, languages like Python ran more slowly than static, compiled
+languages; Python developers chose to sacrifice execution speed for ease
+of programming.  The PyPy project created a substantially improved Python
+language implementation, including a fast Just-in-time (JIT) compiler.
+The increased execution speed that PyPy provides has attracted many users,
+who now find their Python code runs up to four times faster under PyPy
+than under the reference implementation written in C.
+
+Meanwhile, adoption of Python is already underway for those researchers
+and developers who work specifically on computing that requires fast
+numeric operations.  Numpy support in PyPy will allow for Python's use by
+those developers and researchers who want the ease of programming that
+Python provides, the speed of PyPy, *and* the speedups for numerical work
+that Numpy can provide.
+
+PyPy's developers make all PyPy software available to the public without
+charge, under PyPy's Open Source copyright license, the permissive MIT
+License.  PyPy's license assures that PyPy is equally available to
+everyone freely on terms that allow both non-commercial and commercial
+activity.  This license allows for academics, for-profit software
+developers, volunteers and enthusiasts alike to collaborate together to
+make a better Python implementation for everyone.
+
+NumPy support for PyPy will be licensed similarly, and therefore NumPy
+support can directly help researchers and developers who seek to do
+numeric computing but want an easier programming language to use than
+Fortan or C, which is typically used for these applications.  Being
+licensed freely to the general public means that opportunities to use,
+improve and learn about how NumPy works itself will be generally available
+to everyone.

From noreply at buildbot.pypy.org  Wed Oct 12 22:40:33 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 12 Oct 2011 22:40:33 +0200 (CEST)
Subject: [pypy-commit] pypy.org extradoc: no need for two titles
Message-ID: <20111012204033.073AD82112@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: extradoc
Changeset: r281:6035b3c9d743
Date: 2011-10-12 22:40 +0200
http://bitbucket.org/pypy/pypy.org/changeset/6035b3c9d743/

Log:	no need for two titles

diff --git a/numpydonate.html b/numpydonate.html
--- a/numpydonate.html
+++ b/numpydonate.html
@@ -45,8 +45,6 @@
 <div>
 <div id="main">
 <h1 class="title">Call for donations - PyPy to support Numpy!</h1>
-<div class="section" id="make-numpy-for-pypy-happen">
-<h1>Make NumPy for PyPy happen</h1>
 <p>This is a proposal to provide a fully compatible working <a class="reference external" href="http://numpy.scipy.org/">NumPy</a> implementation
 for PyPy. This has long been a very commonly <a class="reference external" href="http://morepypy.blogspot.com/2011/06/report-back-from-our-survey.html">requested feature</a> for PyPy
 as well as a worthy goal given that PyPy performs extremely well on numeric
@@ -76,13 +74,13 @@
 contact pypy at sfconservancy.org if you want to know details on how
 to donate via other means.</p>
 <div class="section" id="what-is-numpy">
-<h2>What is NumPy?</h2>
+<h1>What is NumPy?</h1>
 <p><a class="reference external" href="http://numpy.scipy.org/">NumPy</a> is a framework for doing numerical calculations in Python. It has
 become the de-facto standard for doing any kinds of computations that involve
 n-dimensional arrays. Please consult the NumPy website for more details.</p>
 </div>
 <div class="section" id="why-does-numpy-on-pypy-makes-sense">
-<h2>Why does NumPy on PyPy makes sense?</h2>
+<h1>Why does NumPy on PyPy makes sense?</h1>
 <p>NumPy on PyPy makes sense for a couple of reasons: Firstly, it is by
 far the most requested feature from PyPy. Secondly, PyPy <a class="reference external" href="http://morepypy.blogspot.com/2011/07/realtime-image-processing-in-python.html">performs
 well</a> on numerical loads already.  Therefore bringing NumPy into the
@@ -95,7 +93,7 @@
 or Fortran for speed purposes.</p>
 </div>
 <div class="section" id="speed">
-<h2>Speed</h2>
+<h1>Speed</h1>
 <p>The current implementation of NumPy on PyPy is reasonably fast - it ranges
 from roughly the same speed to 2-5x faster for stacked large array operations
 to 100-300x for accessing NumPy array elements one by one. The exact speed
@@ -108,12 +106,12 @@
 is about teaching the JIT how to vectorize certain operations, which should bring
 PyPy's NumPy as a very competitive tool compared to other available
 solutions for numerical computations, like matlab or C++ array libraries.</p>
+<div class="section" id="id1">
+<span id="work-plan"></span><h2>Work plan</h2>
 </div>
 </div>
-<div class="section" id="id1">
-<span id="work-plan"></span><h1>Work plan</h1>
 <div class="section" id="about-estimates-and-costs">
-<h2>About estimates and costs</h2>
+<h1>About estimates and costs</h1>
 <p>For each step, we estimated the time that it would take to complete for an
 experienced developer who is already familiar with the PyPy codebase.  From
 this number, the money is calculated considering a hourly rate of $60, and a
@@ -155,10 +153,8 @@
 <p>Estimated costs: USD$30,000. Estimated duration: 3 months.</p>
 </li>
 </ul>
-</div>
-</div>
 <div class="section" id="benefits-of-this-work-to-the-python-community-and-the-general-public">
-<h1>Benefits of This Work to the Python Community and the General Public</h1>
+<h2>Benefits of This Work to the Python Community and the General Public</h2>
 <p>Python has become one of the most popular dynamic programming languages in
 the world.  Web developers, educators, and scientific programmers alike
 all value Python because Python code is often more readable and because
@@ -192,6 +188,7 @@
 to everyone.</p>
 </div>
 </div>
+</div>
 <div id="sidebar">
 </div>
 </div>
diff --git a/source/numpydonate.txt b/source/numpydonate.txt
--- a/source/numpydonate.txt
+++ b/source/numpydonate.txt
@@ -3,9 +3,6 @@
 title: Call for donations - PyPy to support Numpy!
 ---
 
-Make NumPy for PyPy happen
-==========================
-
 This is a proposal to provide a fully compatible working `NumPy`_ implementation
 for PyPy. This has long been a very commonly `requested feature`_ for PyPy
 as well as a worthy goal given that PyPy performs extremely well on numeric

From noreply at buildbot.pypy.org  Wed Oct 12 22:45:06 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 12 Oct 2011 22:45:06 +0200 (CEST)
Subject: [pypy-commit] pypy.org extradoc: uh,
	there were two places mentioning it
Message-ID: <20111012204506.52F4482112@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: extradoc
Changeset: r282:ccaa5c2d87c4
Date: 2011-10-12 22:44 +0200
http://bitbucket.org/pypy/pypy.org/changeset/ccaa5c2d87c4/

Log:	uh, there were two places mentioning it

diff --git a/numpydonate.html b/numpydonate.html
--- a/numpydonate.html
+++ b/numpydonate.html
@@ -146,10 +146,8 @@
 for tight loops</li>
 </ul>
 <p>As with all speed improvements, it's relatively hard to predict exactly
-how it'll cope, however we expect the results to be in the range of
-XXX I (cfbolz) find this claim too strong
-handwritten C equivalent, sometimes outperforming it in cases where static
-compilation of C prevents some advanced optimization techniques.</p>
+how it'll cope, however we expect the results to be withing an order
+of magnitude of handwritten C equivalent.</p>
 <p>Estimated costs: USD$30,000. Estimated duration: 3 months.</p>
 </li>
 </ul>
diff --git a/source/numpydonate.txt b/source/numpydonate.txt
--- a/source/numpydonate.txt
+++ b/source/numpydonate.txt
@@ -131,10 +131,8 @@
     for tight loops
 
   As with all speed improvements, it's relatively hard to predict exactly
-  how it'll cope, however we expect the results to be in the range of
-  XXX I (cfbolz) find this claim too strong
-  handwritten C equivalent, sometimes outperforming it in cases where static
-  compilation of C prevents some advanced optimization techniques.
+  how it'll cope, however we expect the results to be withing an order
+  of magnitude of handwritten C equivalent.
 
   Estimated costs: USD$30,000. Estimated duration: 3 months.
 

From noreply at buildbot.pypy.org  Wed Oct 12 22:52:20 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 12 Oct 2011 22:52:20 +0200 (CEST)
Subject: [pypy-commit] pypy.org extradoc: html/js abomination
Message-ID: <20111012205220.E839782112@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: extradoc
Changeset: r283:928413d43ac4
Date: 2011-10-12 22:52 +0200
http://bitbucket.org/pypy/pypy.org/changeset/928413d43ac4/

Log:	html/js abomination

diff --git a/donate1.html b/donate1.html
--- a/donate1.html
+++ b/donate1.html
@@ -2,6 +2,7 @@
   <li>
     <a href="#" onclick="py3k_donate(); return false;"><b>Donate towards py3k in pypy</b></a><br/>
     <a href="#" onclick="general_donate(); return false;">Donate towards general pypy progress</a><br/>
+    <a href="#" onclick="numpy_donate(); return false;">Donate towards NumPy in pypy</a><br/>
   </li>
   <li>
 <script>
diff --git a/donate2.html b/donate2.html
--- a/donate2.html
+++ b/donate2.html
@@ -2,6 +2,7 @@
   <li>
     <a href="#" onclick="py3k_donate(); return false;">Donate towards py3k in pypy</a><br/>
     <a href="#" onclick="general_donate(); return false;"><b>Donate towards general pypy progress</b></a><br/>
+    <a href="#" onclick="numpy_donate(); return false;">Donate towards NumPy in pypy</a><br/>
   </li>
   <li>
     <span id="donate_text">This donation will go towards general PyPy progress
diff --git a/donate3.html b/donate3.html
new file mode 100644
--- /dev/null
+++ b/donate3.html
@@ -0,0 +1,61 @@
+<ul>
+  <li>
+    <a href="#" onclick="py3k_donate(); return false;">Donate towards py3k in pypy</a><br/>
+    <a href="#" onclick="general_donate(); return false;">Donate towards general pypy progress</a><br/>
+    <a href="#" onclick="numpy_donate(); return false;"><b>Donate towards NumPy in pypy</b></a><br/>
+  </li>
+  <li>
+    <span id="donate_text">This donation will go towards supporting NumPy in PyPy <a href="numpydonate.html">Read proposal</a>
+  </li>
+  <li>
+    <div class="sidepic">
+      <form action="https://www.paypal.com/cgi-bin/webscr" method="post">
+        <input type="hidden" name="cmd" value="_s-xclick">
+        <input type="hidden" name="hosted_button_id" value="7SMJUEAP83RFQ">
+        <input type="image" src="https://www.paypalobjects.com/en_US/i/btn/btn_donateCC_LG.gif" border="0" name="submit" alt="PayPal - The safer, easier way to pay online!">
+        <img alt="" border="0" src="https://www.paypalobjects.com/en_US/i/scr/pixel.gif" width="1" height="1">
+      </form>
+    </div>
+  </li>
+  <li>
+    <div class="sidepic">
+      
+      <script type="text/javascript">
+        function validateAmount(amount){
+        if(amount.value.match( /^[0-9]+(\.([0-9]+))?$/)){
+        return true;
+        }else{
+        alert('You must enter a valid donation.');
+        amount.focus();
+        return false;
+        }
+        }
+      </script>
+      <form action="https://checkout.google.com/cws/v2/Donations/622836985124940/checkoutForm" id="BB_BuyButtonForm" method="post" name="BB_BuyButtonForm" onSubmit="return validateAmount(this.item_price_1)" target="_top">
+        <input name="item_name_1" type="hidden" value="Directed Donation for PyPy's Numpy project"/>
+        <input name="item_description_1" type="hidden" value="Donation to Software Freedom Conservancy directed for PyPy's Numpy project: http://pypy.org/numpydonate.html"/>
+        <input name="item_quantity_1" type="hidden" value="1"/>
+        <input name="item_currency_1" type="hidden" value="USD"/>
+        <input name="item_is_modifiable_1" type="hidden" value="true"/>
+        <input name="item_min_price_1" type="hidden" value="10.0"/>
+        <input name="item_max_price_1" type="hidden" value="25000.0"/>
+        <input name="_charset_" type="hidden" value="utf-8"/>
+        <table cellpadding="5" cellspacing="0" width="1%">
+          <tr>
+            <td align="right" nowrap="nowrap" width="1%">&#x24; <input id="item_price_1" name="item_price_1" onfocus="this.style.color='black'; this.value='';" size="11" style="color:grey;" type="text" value="Enter Amount"/>
+            </td>
+            <td align="left" width="1%">
+              <input alt="Donate" src="https://checkout.google.com/buttons/donateNow.gif?merchant_id=622836985124940&amp;w=115&amp;h=50&amp;style=white&amp;variant=text&amp;loc=en_US" type="image"/>
+            </td>
+          </tr>
+        </table>
+      </form>
+    </div>
+  </li>
+  <li>
+    <div class="sidepic">
+      <a id="main_download" href="download.html">Download PyPy</a>
+    </div>
+  </li>
+</ul>
+
diff --git a/js/script.js b/js/script.js
--- a/js/script.js
+++ b/js/script.js
@@ -11,6 +11,12 @@
     });
 }
 
+function numpy_donate() {
+    $.get("donate3.html", function (html) {
+        $("#sidebar").html(html);
+    });
+}
+
 $(document).ready(function() {
-    py3k_donate();
+    numpy_donate();
 });
\ No newline at end of file

From noreply at buildbot.pypy.org  Wed Oct 12 22:54:50 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 12 Oct 2011 22:54:50 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: add a donation info
Message-ID: <20111012205450.14AC482112@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: extradoc
Changeset: r3927:5a99d6d5f2e5
Date: 2011-10-12 22:54 +0200
http://bitbucket.org/pypy/extradoc/changeset/5a99d6d5f2e5/

Log:	add a donation info

diff --git a/blog/draft/numpy-status-update.rst b/blog/draft/numpy-status-update.rst
--- a/blog/draft/numpy-status-update.rst
+++ b/blog/draft/numpy-status-update.rst
@@ -1,11 +1,14 @@
-NumPy Status Update
--------------------
+NumPy funding and status update
+-------------------------------
 
 Hi everyone,
 
-It's been a little while since we wrote about NumPy on PyPy, so I wanted to
+It's been a little while since we wrote about NumPy on PyPy, so we wanted to
 give everyone an update on what we've been up to, and what's up next for us.
 
+We would also like to note that we're launching a **funding campaign**
+for NumPy support in PyPy. Details can be found on the `donation page`_.
+
 Some of the things that have happened since last we wrote are:
 
 * We added ``dtype`` support, meaning you can now create arrays of a bunch of
@@ -31,3 +34,5 @@
 
 If you're interested in helping out, we always love more contributors,
 Alex, Maciej, Justin, and the whole PyPy team
+
+.. _`donation page`: http://pypy.org/numpy_donate.html

From noreply at buildbot.pypy.org  Wed Oct 12 22:55:32 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 12 Oct 2011 22:55:32 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Forgot to rename these methods
Message-ID: <20111012205532.E14CA82112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48000:49f61e76309c
Date: 2011-10-12 22:31 +0200
http://bitbucket.org/pypy/pypy/changeset/49f61e76309c/

Log:	Forgot to rename these methods

diff --git a/pypy/objspace/std/strbufobject.py b/pypy/objspace/std/strbufobject.py
--- a/pypy/objspace/std/strbufobject.py
+++ b/pypy/objspace/std/strbufobject.py
@@ -31,7 +31,7 @@
     def unwrap(self, space):
         return self.force()
 
-    def str_w(self, space):
+    def bytes_w(self, space):
         return self.force()
 
 registerimplementation(W_StringBufferObject)
diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py
--- a/pypy/objspace/std/stringobject.py
+++ b/pypy/objspace/std/stringobject.py
@@ -33,7 +33,7 @@
     def unwrap(w_self, space):
         return w_self._value
 
-    def str_w(w_self, space):
+    def bytes_w(w_self, space):
         return w_self._value
 
     def unicode_w(w_self, space):

From noreply at buildbot.pypy.org  Wed Oct 12 22:55:34 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 12 Oct 2011 22:55:34 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Implement the nonlocal statement
Message-ID: <20111012205534.35FB182112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48001:d6d9ea7ba306
Date: 2011-10-12 22:55 +0200
http://bitbucket.org/pypy/pypy/changeset/d6d9ea7ba306/

Log:	Implement the nonlocal statement

diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -1016,6 +1016,37 @@
                 self.names = None
 
 
+class Nonlocal(stmt):
+
+    _lineno_mask = 2
+    _col_offset_mask = 4
+
+    def __init__(self, names, lineno, col_offset):
+        self.names = names
+        self.w_names = None
+        stmt.__init__(self, lineno, col_offset)
+        self.initialization_state = 7
+
+    def walkabout(self, visitor):
+        visitor.visit_Nonlocal(self)
+
+    def mutate_over(self, visitor):
+        return visitor.visit_Nonlocal(self)
+
+    def sync_app_attrs(self, space):
+        if (self.initialization_state & ~0) ^ 7:
+            missing_field(space, self.initialization_state, ['names', 'lineno', 'col_offset'], 'Nonlocal')
+        else:
+            pass
+        w_list = self.w_names
+        if w_list is not None:
+            list_w = space.listview(w_list)
+            if list_w:
+                self.names = [space.str_w(w_obj) for w_obj in list_w]
+            else:
+                self.names = None
+
+
 class Expr(stmt):
 
     _lineno_mask = 2
@@ -1758,7 +1789,6 @@
     _col_offset_mask = 8
 
     def __init__(self, id, ctx, lineno, col_offset):
-        assert isinstance(id, str)
         self.id = id
         self.ctx = ctx
         expr.__init__(self, lineno, col_offset)
@@ -2516,6 +2546,8 @@
         return self.default_visitor(node)
     def visit_Global(self, node):
         return self.default_visitor(node)
+    def visit_Nonlocal(self, node):
+        return self.default_visitor(node)
     def visit_Expr(self, node):
         return self.default_visitor(node)
     def visit_Pass(self, node):
@@ -2684,6 +2716,9 @@
     def visit_Global(self, node):
         pass
 
+    def visit_Nonlocal(self, node):
+        pass
+
     def visit_Expr(self, node):
         node.value.walkabout(self)
 
@@ -4441,6 +4476,49 @@
     __init__=interp2app(Global_init),
 )
 
+def Nonlocal_get_names(space, w_self):
+    if not w_self.initialization_state & 1:
+        typename = space.type(w_self).getname(space)
+        w_err = space.wrap("'%s' object has no attribute 'names'" % typename)
+        raise OperationError(space.w_AttributeError, w_err)
+    if w_self.w_names is None:
+        if w_self.names is None:
+            w_list = space.newlist([])
+        else:
+            list_w = [space.wrap(node) for node in w_self.names]
+            w_list = space.newlist(list_w)
+        w_self.w_names = w_list
+    return w_self.w_names
+
+def Nonlocal_set_names(space, w_self, w_new_value):
+    w_self.w_names = w_new_value
+    w_self.initialization_state |= 1
+
+_Nonlocal_field_unroller = unrolling_iterable(['names'])
+def Nonlocal_init(space, w_self, __args__):
+    w_self = space.descr_self_interp_w(Nonlocal, w_self)
+    w_self.w_names = None
+    args_w, kwargs_w = __args__.unpack()
+    if args_w:
+        if len(args_w) != 1:
+            w_err = space.wrap("Nonlocal constructor takes either 0 or 1 positional argument")
+            raise OperationError(space.w_TypeError, w_err)
+        i = 0
+        for field in _Nonlocal_field_unroller:
+            space.setattr(w_self, space.wrap(field), args_w[i])
+            i += 1
+    for field, w_value in kwargs_w.iteritems():
+        space.setattr(w_self, space.wrap(field), w_value)
+
+Nonlocal.typedef = typedef.TypeDef("Nonlocal",
+    stmt.typedef,
+    __module__='_ast',
+    _fields=_FieldsWrapper(['names']),
+    names=typedef.GetSetProperty(Nonlocal_get_names, Nonlocal_set_names, cls=Nonlocal),
+    __new__=interp2app(get_AST_new(Nonlocal)),
+    __init__=interp2app(Nonlocal_init),
+)
+
 def Expr_get_value(space, w_self):
     if w_self.w_dict is not None:
         w_obj = w_self.getdictvalue(space, 'value')
diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -255,6 +255,11 @@
                  for i in range(1, len(global_node.children), 2)]
         return ast.Global(names, global_node.lineno, global_node.column)
 
+    def handle_nonlocal_stmt(self, nonlocal_node):
+        names = [nonlocal_node.children[i].value
+                 for i in range(1, len(nonlocal_node.children), 2)]
+        return ast.Nonlocal(names, nonlocal_node.lineno, nonlocal_node.column)
+
     def handle_assert_stmt(self, assert_node):
         child_count = len(assert_node.children)
         expr = self.handle_expr(assert_node.children[1])
@@ -620,6 +625,8 @@
                 return self.handle_import_stmt(stmt)
             elif stmt_type == syms.global_stmt:
                 return self.handle_global_stmt(stmt)
+            elif stmt_type == syms.nonlocal_stmt:
+                return self.handle_nonlocal_stmt(stmt)
             elif stmt_type == syms.assert_stmt:
                 return self.handle_assert_stmt(stmt)
             else:
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -761,6 +761,10 @@
         # Handled in symbol table building.
         pass
 
+    def visit_Nonlocal(self, glob):
+        # Handled in symbol table building.
+        pass
+
     def visit_Pass(self, pas):
         self.update_position(pas.lineno, True)
 
diff --git a/pypy/interpreter/astcompiler/symtable.py b/pypy/interpreter/astcompiler/symtable.py
--- a/pypy/interpreter/astcompiler/symtable.py
+++ b/pypy/interpreter/astcompiler/symtable.py
@@ -10,7 +10,8 @@
 SYM_GLOBAL = 1
 SYM_ASSIGNED = 2 # Or deleted actually.
 SYM_PARAM = 2 << 1
-SYM_USED = 2 << 2
+SYM_NONLOCAL = 2 << 2
+SYM_USED = 2 << 3
 SYM_BOUND = (SYM_PARAM | SYM_ASSIGNED)
 
 # codegen.py actually deals with these:
@@ -107,7 +108,10 @@
         """Decide on the scope of a name."""
         if flags & SYM_GLOBAL:
             if flags & SYM_PARAM:
-                err = "name '%s' is local and global" % (name,)
+                err = "name '%s' is parameter and global" % (name,)
+                raise SyntaxError(err, self.lineno, self.col_offset)
+            if flags & SYM_NONLOCAL:
+                err = "name '%s' is nonlocal and global" % (name,)
                 raise SyntaxError(err, self.lineno, self.col_offset)
             self.symbols[name] = SCOPE_GLOBAL_EXPLICIT
             globs[name] = None
@@ -116,6 +120,18 @@
                     del bound[name]
                 except KeyError:
                     pass
+        elif flags & SYM_NONLOCAL:
+            if flags & SYM_PARAM:
+                err = "name '%s' is parameter and nonlocal" % (name,)
+                raise SyntaxError(err, self.lineno, self.col_offset)
+            if bound is None:
+                err = "nonlocal declaration not allowed at module level"
+                raise SyntaxError(err, self.lineno, self.col_offset)
+            if name not in bound:
+                err = "no binding for nonlocal '%s' found" % (name,)
+                raise SyntaxError(err, self.lineno, self.col_offset)
+            self.symbols[name] = SCOPE_FREE
+            free[name] = None
         elif flags & SYM_BOUND:
             self.symbols[name] = SCOPE_LOCAL
             local[name] = None
@@ -444,6 +460,21 @@
                                     glob.lineno, glob.col_offset)
             self.note_symbol(name, SYM_GLOBAL)
 
+    def visit_Nonlocal(self, nonl):
+        for name in nonl.names:
+            old_role = self.scope.lookup_role(name)
+            if old_role & (SYM_USED | SYM_ASSIGNED):
+                if old_role & SYM_ASSIGNED:
+                    msg = "name '%s' is assigned to before nonlocal declaration" \
+                        % (name,)
+                else:
+                    msg = "name '%s' is used prior to nonlocal declaration" % \
+                        (name,)
+                misc.syntax_warning(self.space, msg, self.compile_info.filename,
+                                    nonl.lineno, nonl.col_offset)
+            self.note_symbol(name, SYM_NONLOCAL)
+            
+
     def visit_Lambda(self, lamb):
         args = lamb.args
         assert isinstance(args, ast.arguments)
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -786,6 +786,17 @@
             raise AssertionError("attribute not removed")"""
         yield self.st, test, "X.__name__", "X"
 
+    def test_nonlocal(self):
+        test = """if 1:
+        def f():
+            y = 0
+            def g(x):
+                nonlocal y
+                y = x + 1
+            g(3)
+            return y"""
+        yield self.st, test, "f()", 4
+
 
 class AppTestCompiler:
 
diff --git a/pypy/interpreter/astcompiler/tools/Python.asdl b/pypy/interpreter/astcompiler/tools/Python.asdl
--- a/pypy/interpreter/astcompiler/tools/Python.asdl
+++ b/pypy/interpreter/astcompiler/tools/Python.asdl
@@ -40,6 +40,7 @@
 	      | ImportFrom(identifier? module, alias* names, int? level)
 
 	      | Global(identifier* names)
+	      | Nonlocal(identifier* names)
 	      | Expr(expr value)
 	      | Pass | Break | Continue
 
diff --git a/pypy/interpreter/pyparser/data/Grammar3.2 b/pypy/interpreter/pyparser/data/Grammar3.2
--- a/pypy/interpreter/pyparser/data/Grammar3.2
+++ b/pypy/interpreter/pyparser/data/Grammar3.2
@@ -36,7 +36,7 @@
 stmt: simple_stmt | compound_stmt
 simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
 small_stmt: (expr_stmt | del_stmt | pass_stmt | flow_stmt |
-             import_stmt | global_stmt | assert_stmt)
+             import_stmt | global_stmt | nonlocal_stmt | assert_stmt)
 expr_stmt: testlist (augassign (yield_expr|testlist) |
                      ('=' (yield_expr|testlist))*)
 augassign: ('+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' |
@@ -60,6 +60,7 @@
 dotted_as_names: dotted_as_name (',' dotted_as_name)*
 dotted_name: NAME ('.' NAME)*
 global_stmt: 'global' NAME (',' NAME)*
+nonlocal_stmt: 'nonlocal' NAME (',' NAME)*
 assert_stmt: 'assert' test [',' test]
 
 compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated

From noreply at buildbot.pypy.org  Wed Oct 12 22:55:45 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 12 Oct 2011 22:55:45 +0200 (CEST)
Subject: [pypy-commit] pypy.org extradoc: cheat;
Message-ID: <20111012205545.E86CA82112@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: extradoc
Changeset: r284:f19079f27b7e
Date: 2011-10-12 22:55 +0200
http://bitbucket.org/pypy/pypy.org/changeset/f19079f27b7e/

Log:	cheat;

diff --git a/donate1.html b/don1.html
rename from donate1.html
rename to don1.html
diff --git a/donate2.html b/don2.html
rename from donate2.html
rename to don2.html
diff --git a/donate3.html b/don3.html
rename from donate3.html
rename to don3.html
diff --git a/js/script.js b/js/script.js
--- a/js/script.js
+++ b/js/script.js
@@ -1,18 +1,18 @@
 
 function py3k_donate() {
-    $.get("donate1.html", function (html) {
+    $.get("don1.html", function (html) {
         $("#sidebar").html(html);
     });
 }
 
 function general_donate() {
-    $.get("donate2.html", function (html) {
+    $.get("don2.html", function (html) {
         $("#sidebar").html(html);
     });
 }
 
 function numpy_donate() {
-    $.get("donate3.html", function (html) {
+    $.get("don3.html", function (html) {
         $("#sidebar").html(html);
     });
 }

From noreply at buildbot.pypy.org  Wed Oct 12 22:59:31 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 12 Oct 2011 22:59:31 +0200 (CEST)
Subject: [pypy-commit] pypy.org extradoc: cheat some more, rename JS
Message-ID: <20111012205931.F04CA82112@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: extradoc
Changeset: r285:e3c4256a08cb
Date: 2011-10-12 22:59 +0200
http://bitbucket.org/pypy/pypy.org/changeset/e3c4256a08cb/

Log:	cheat some more, rename JS

diff --git a/archive.html b/archive.html
--- a/archive.html
+++ b/archive.html
@@ -16,7 +16,7 @@
 	<script type="text/javascript" src="http://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script>
   <script type="text/javascript" src="js/jquery-ui-1.8.14.custom.min.js"></script>
   <script type="text/javascript" src="js/detect.js"></script>
-  <script type="text/javascript" src="js/script.js"></script>
+  <script type="text/javascript" src="js/script2.js"></script>
 </head>
 <body>
 <script type="text/javascript">
diff --git a/compat.html b/compat.html
--- a/compat.html
+++ b/compat.html
@@ -16,7 +16,7 @@
 	<script type="text/javascript" src="http://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script>
   <script type="text/javascript" src="js/jquery-ui-1.8.14.custom.min.js"></script>
   <script type="text/javascript" src="js/detect.js"></script>
-  <script type="text/javascript" src="js/script.js"></script>
+  <script type="text/javascript" src="js/script2.js"></script>
 </head>
 <body>
 <script type="text/javascript">
diff --git a/contact.html b/contact.html
--- a/contact.html
+++ b/contact.html
@@ -16,7 +16,7 @@
 	<script type="text/javascript" src="http://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script>
   <script type="text/javascript" src="js/jquery-ui-1.8.14.custom.min.js"></script>
   <script type="text/javascript" src="js/detect.js"></script>
-  <script type="text/javascript" src="js/script.js"></script>
+  <script type="text/javascript" src="js/script2.js"></script>
 </head>
 <body>
 <script type="text/javascript">
diff --git a/download.html b/download.html
--- a/download.html
+++ b/download.html
@@ -16,7 +16,7 @@
 	<script type="text/javascript" src="http://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script>
   <script type="text/javascript" src="js/jquery-ui-1.8.14.custom.min.js"></script>
   <script type="text/javascript" src="js/detect.js"></script>
-  <script type="text/javascript" src="js/script.js"></script>
+  <script type="text/javascript" src="js/script2.js"></script>
 </head>
 <body>
 <script type="text/javascript">
diff --git a/features.html b/features.html
--- a/features.html
+++ b/features.html
@@ -16,7 +16,7 @@
 	<script type="text/javascript" src="http://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script>
   <script type="text/javascript" src="js/jquery-ui-1.8.14.custom.min.js"></script>
   <script type="text/javascript" src="js/detect.js"></script>
-  <script type="text/javascript" src="js/script.js"></script>
+  <script type="text/javascript" src="js/script2.js"></script>
 </head>
 <body>
 <script type="text/javascript">
diff --git a/howtohelp.html b/howtohelp.html
--- a/howtohelp.html
+++ b/howtohelp.html
@@ -16,7 +16,7 @@
 	<script type="text/javascript" src="http://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script>
   <script type="text/javascript" src="js/jquery-ui-1.8.14.custom.min.js"></script>
   <script type="text/javascript" src="js/detect.js"></script>
-  <script type="text/javascript" src="js/script.js"></script>
+  <script type="text/javascript" src="js/script2.js"></script>
 </head>
 <body>
 <script type="text/javascript">
diff --git a/index.html b/index.html
--- a/index.html
+++ b/index.html
@@ -16,7 +16,7 @@
 	<script type="text/javascript" src="http://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script>
   <script type="text/javascript" src="js/jquery-ui-1.8.14.custom.min.js"></script>
   <script type="text/javascript" src="js/detect.js"></script>
-  <script type="text/javascript" src="js/script.js"></script>
+  <script type="text/javascript" src="js/script2.js"></script>
 </head>
 <body>
 <script type="text/javascript">
diff --git a/js/script.js b/js/script2.js
rename from js/script.js
rename to js/script2.js
diff --git a/numpydonate.html b/numpydonate.html
--- a/numpydonate.html
+++ b/numpydonate.html
@@ -16,7 +16,7 @@
 	<script type="text/javascript" src="http://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script>
   <script type="text/javascript" src="js/jquery-ui-1.8.14.custom.min.js"></script>
   <script type="text/javascript" src="js/detect.js"></script>
-  <script type="text/javascript" src="js/script.js"></script>
+  <script type="text/javascript" src="js/script2.js"></script>
 </head>
 <body>
 <script type="text/javascript">
diff --git a/people.html b/people.html
--- a/people.html
+++ b/people.html
@@ -16,7 +16,7 @@
 	<script type="text/javascript" src="http://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script>
   <script type="text/javascript" src="js/jquery-ui-1.8.14.custom.min.js"></script>
   <script type="text/javascript" src="js/detect.js"></script>
-  <script type="text/javascript" src="js/script.js"></script>
+  <script type="text/javascript" src="js/script2.js"></script>
 </head>
 <body>
 <script type="text/javascript">
diff --git a/py3donate.html b/py3donate.html
--- a/py3donate.html
+++ b/py3donate.html
@@ -16,7 +16,7 @@
 	<script type="text/javascript" src="http://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script>
   <script type="text/javascript" src="js/jquery-ui-1.8.14.custom.min.js"></script>
   <script type="text/javascript" src="js/detect.js"></script>
-  <script type="text/javascript" src="js/script.js"></script>
+  <script type="text/javascript" src="js/script2.js"></script>
 </head>
 <body>
 <script type="text/javascript">
diff --git a/source/_layouts/site.genshi b/source/_layouts/site.genshi
--- a/source/_layouts/site.genshi
+++ b/source/_layouts/site.genshi
@@ -45,7 +45,7 @@
 	<script type="text/javascript" src="http://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script>
   <script type="text/javascript" src="js/jquery-ui-1.8.14.custom.min.js"/>
   <script type="text/javascript" src="js/detect.js"></script>
-  <script type="text/javascript" src="js/script.js"></script>
+  <script type="text/javascript" src="js/script2.js"></script>
 </head>
 <body>
 <script type="text/javascript">
diff --git a/sponsor.html b/sponsor.html
--- a/sponsor.html
+++ b/sponsor.html
@@ -16,7 +16,7 @@
 	<script type="text/javascript" src="http://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script>
   <script type="text/javascript" src="js/jquery-ui-1.8.14.custom.min.js"></script>
   <script type="text/javascript" src="js/detect.js"></script>
-  <script type="text/javascript" src="js/script.js"></script>
+  <script type="text/javascript" src="js/script2.js"></script>
 </head>
 <body>
 <script type="text/javascript">
diff --git a/success.html b/success.html
--- a/success.html
+++ b/success.html
@@ -16,7 +16,7 @@
 	<script type="text/javascript" src="http://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script>
   <script type="text/javascript" src="js/jquery-ui-1.8.14.custom.min.js"></script>
   <script type="text/javascript" src="js/detect.js"></script>
-  <script type="text/javascript" src="js/script.js"></script>
+  <script type="text/javascript" src="js/script2.js"></script>
 </head>
 <body>
 <script type="text/javascript">

From noreply at buildbot.pypy.org  Thu Oct 13 00:01:36 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 13 Oct 2011 00:01:36 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: update builtins: 'str' is the unicode type,
 'bytes' is the 8bit string
Message-ID: <20111012220136.3FE8382112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48002:435fe1660753
Date: 2011-10-12 23:58 +0200
http://bitbucket.org/pypy/pypy/changeset/435fe1660753/

Log:	update builtins: 'str' is the unicode type, 'bytes' is the 8bit
	string

diff --git a/pypy/module/__builtin__/__init__.py b/pypy/module/__builtin__/__init__.py
--- a/pypy/module/__builtin__/__init__.py
+++ b/pypy/module/__builtin__/__init__.py
@@ -37,8 +37,6 @@
         '__debug__'     : '(space.w_True)',      # XXX
         'type'          : '(space.w_type)',
         'object'        : '(space.w_object)',
-        'bytes'         : '(space.w_str)',
-        'unicode'       : '(space.w_unicode)',
         'buffer'        : 'interp_memoryview.W_Buffer',
         'memoryview'    : 'interp_memoryview.W_MemoryView',
 
diff --git a/pypy/module/__builtin__/app_io.py b/pypy/module/__builtin__/app_io.py
--- a/pypy/module/__builtin__/app_io.py
+++ b/pypy/module/__builtin__/app_io.py
@@ -76,39 +76,23 @@
     if fp is None:
         return
     def write(data):
-        if not isinstance(data, basestring):
+        if not isinstance(data, str):
             data = str(data)
         fp.write(data)
-    want_unicode = False
     sep = kwargs.pop("sep", None)
     if sep is not None:
-        if isinstance(sep, unicode):
-            want_unicode = True
-        elif not isinstance(sep, str):
+        if not isinstance(sep, str):
             raise TypeError("sep must be None or a string")
     end = kwargs.pop("end", None)
     if end is not None:
-        if isinstance(end, unicode):
-            want_unicode = True
-        elif not isinstance(end, str):
+        if not isinstance(end, str):
             raise TypeError("end must be None or a string")
     if kwargs:
         raise TypeError("invalid keyword arguments to print()")
-    if not want_unicode:
-        for arg in args:
-            if isinstance(arg, unicode):
-                want_unicode = True
-                break
-    if want_unicode:
-        newline = u"\n"
-        space = u" "
-    else:
-        newline = "\n"
-        space = " "
     if sep is None:
-        sep = space
+        sep = " "
     if end is None:
-        end = newline
+        end = "\n"
     for i, arg in enumerate(args):
         if i:
             write(sep)
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -68,6 +68,9 @@
             w_type = self.gettypeobject(typedef)
             self.builtin_types[typedef.name] = w_type
             setattr(self, 'w_' + typedef.name, w_type)
+        del self.builtin_types['unicode']
+        self.builtin_types['str'] = self.w_unicode
+        self.builtin_types['bytes'] = self.w_str
         self.w_text = self.w_unicode
         self.builtin_types["NotImplemented"] = self.w_NotImplemented
         self.builtin_types["Ellipsis"] = self.w_Ellipsis

From noreply at buildbot.pypy.org  Thu Oct 13 00:01:37 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 13 Oct 2011 00:01:37 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: It works much better this way
Message-ID: <20111012220137.7F8FE82112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48003:fd4678cd5916
Date: 2011-10-13 00:01 +0200
http://bitbucket.org/pypy/pypy/changeset/fd4678cd5916/

Log:	It works much better this way

diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -514,7 +514,7 @@
     def STORE_LOCALS(self, oparg, next_instr):
         self.w_locals = self.popvalue()
 
-    def exec_(self, w_prog, w_locals, w_globals):
+    def exec_(self, w_prog, w_globals, w_locals):
         """The ___builtin__.exec function."""
         ec = self.space.getexecutioncontext()
         flags = ec.compiler.getcodeflags(self.pycode)

From noreply at buildbot.pypy.org  Thu Oct 13 00:21:11 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 13 Oct 2011 00:21:11 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Fix str.__name__ and remove basestring
Message-ID: <20111012222111.ABE7082112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48004:30686d08358a
Date: 2011-10-13 00:20 +0200
http://bitbucket.org/pypy/pypy/changeset/30686d08358a/

Log:	Fix str.__name__ and remove basestring

diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -228,7 +228,7 @@
     def getdocstring(self, space):
         if self.co_consts_w:   # it is probably never empty
             w_first = self.co_consts_w[0]
-            if space.is_true(space.isinstance(w_first, space.w_basestring)):
+            if space.is_true(space.isinstance(w_first, space.w_unicode)):
                 return w_first
         return space.w_None
 
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1457,7 +1457,7 @@
 
         if not isinstance(prog, codetype):
             filename = '<string>'
-            if not isinstance(prog, basestring):
+            if not isinstance(prog, str):
                 if isinstance(prog, file):
                     filename = prog.name
                     prog = prog.read()
diff --git a/pypy/module/_io/interp_io.py b/pypy/module/_io/interp_io.py
--- a/pypy/module/_io/interp_io.py
+++ b/pypy/module/_io/interp_io.py
@@ -39,9 +39,10 @@
     from pypy.module._io.interp_bufferedio import (W_BufferedRandom,
         W_BufferedWriter, W_BufferedReader)
 
-    if not (space.isinstance_w(w_file, space.w_basestring) or
-        space.isinstance_w(w_file, space.w_int) or
-        space.isinstance_w(w_file, space.w_long)):
+    if not (space.isinstance_w(w_file, space.w_unicode) or
+            space.isinstance_w(w_file, space.w_str) or
+            space.isinstance_w(w_file, space.w_int) or
+            space.isinstance_w(w_file, space.w_long)):
         raise operationerrfmt(space.w_TypeError,
             "invalid file: %s", space.str_w(space.repr(w_file))
         )
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -374,7 +374,6 @@
         "Type": "space.w_type",
         "String": "space.w_str",
         "Unicode": "space.w_unicode",
-        "BaseString": "space.w_basestring",
         "Dict": "space.w_dict",
         "Tuple": "space.w_tuple",
         "List": "space.w_list",
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -1067,7 +1067,7 @@
 
 def confname_w(space, w_name, namespace):
     # XXX slightly non-nice, reuses the sysconf of the underlying os module
-    if space.is_true(space.isinstance(w_name, space.w_basestring)):
+    if space.is_true(space.isinstance(w_name, space.w_unicode)):
         try:
             num = namespace[space.str_w(w_name)]
         except KeyError:
diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -341,11 +341,11 @@
                                   "'%s' object does not define __format__",
                                   typename)
         w_res = space.get_and_call_function(w_descr, w_obj, w_format_spec)
-        if not space.is_true(space.isinstance(w_res, space.w_basestring)):
+        if not space.is_true(space.isinstance(w_res, space.w_unicode)):
             typename = space.type(w_obj).getname(space)
             restypename = space.type(w_res).getname(space)
             raise operationerrfmt(space.w_TypeError,
-                "%s.__format__ must return string or unicode, not %s",
+                "%s.__format__ must return a string, not %s",
                                   typename, restypename)
         return w_res
 
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -39,7 +39,6 @@
     w_long           = W_Object()
     w_tuple          = W_Object()
     w_str            = W_Object()
-    w_basestring     = W_Object()
     w_unicode        = W_Object()
     w_type           = W_Object()
     w_instance       = W_Object()
diff --git a/pypy/objspace/std/basestringtype.py b/pypy/objspace/std/basestringtype.py
deleted file mode 100644
--- a/pypy/objspace/std/basestringtype.py
+++ /dev/null
@@ -1,7 +0,0 @@
-from pypy.objspace.std.stdtypedef import StdTypeDef
-
-
-basestring_typedef = StdTypeDef("basestring",
-    __doc__ =  ("basestring cannot be instantiated; "
-                "it is the base for str and unicode.")
-    )
diff --git a/pypy/objspace/std/fake.py b/pypy/objspace/std/fake.py
--- a/pypy/objspace/std/fake.py
+++ b/pypy/objspace/std/fake.py
@@ -100,15 +100,9 @@
     fake__new__.func_name = "fake__new__" + cpy_type.__name__
 
     kw['__new__'] = gateway.interp2app(fake__new__)
-    if cpy_type.__base__ is not object and not issubclass(cpy_type, Exception):
-        assert cpy_type.__base__ is basestring, cpy_type
-        from pypy.objspace.std.basestringtype import basestring_typedef
-        base = basestring_typedef
-    else:
-        base = None
     class W_Fake(W_Object):
         typedef = StdTypeDef(
-            cpy_type.__name__, base, **kw)
+            cpy_type.__name__, **kw)
         def __init__(w_self, space, val):
             w_self.val = val
             w_self.space = space
diff --git a/pypy/objspace/std/formatting.py b/pypy/objspace/std/formatting.py
--- a/pypy/objspace/std/formatting.py
+++ b/pypy/objspace/std/formatting.py
@@ -528,7 +528,7 @@
         # in simplest case
         if space.isinstance_w(w_values, space.w_dict) or \
            (space.lookup(w_values, '__getitem__') and
-           not space.isinstance_w(w_values, space.w_basestring)):
+           not space.isinstance_w(w_values, space.w_unicode)):
             return format(space, w_format, [w_values], w_values, do_unicode)
         else:
             return format(space, w_format, [w_values], None, do_unicode)
diff --git a/pypy/objspace/std/model.py b/pypy/objspace/std/model.py
--- a/pypy/objspace/std/model.py
+++ b/pypy/objspace/std/model.py
@@ -46,7 +46,6 @@
             from pypy.objspace.std.tupletype  import tuple_typedef
             from pypy.objspace.std.listtype   import list_typedef
             from pypy.objspace.std.dicttype   import dict_typedef
-            from pypy.objspace.std.basestringtype import basestring_typedef
             from pypy.objspace.std.stringtype import str_typedef
             from pypy.objspace.std.bytearraytype import bytearray_typedef
             from pypy.objspace.std.typetype   import type_typedef
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -68,10 +68,11 @@
             w_type = self.gettypeobject(typedef)
             self.builtin_types[typedef.name] = w_type
             setattr(self, 'w_' + typedef.name, w_type)
-        del self.builtin_types['unicode']
+        self.w_unicode = self.w_str
+        self.w_text = self.w_str
+        self.w_str = self.w_bytes
         self.builtin_types['str'] = self.w_unicode
         self.builtin_types['bytes'] = self.w_str
-        self.w_text = self.w_unicode
         self.builtin_types["NotImplemented"] = self.w_NotImplemented
         self.builtin_types["Ellipsis"] = self.w_Ellipsis
 
diff --git a/pypy/objspace/std/stringtype.py b/pypy/objspace/std/stringtype.py
--- a/pypy/objspace/std/stringtype.py
+++ b/pypy/objspace/std/stringtype.py
@@ -1,6 +1,5 @@
 from pypy.interpreter import gateway
 from pypy.objspace.std.stdtypedef import StdTypeDef, SMM
-from pypy.objspace.std.basestringtype import basestring_typedef
 from pypy.objspace.std.register_all import register_all
 
 
@@ -310,7 +309,7 @@
 
 # ____________________________________________________________
 
-str_typedef = StdTypeDef("str", basestring_typedef,
+str_typedef = StdTypeDef("bytes",
     __new__ = gateway.interp2app(descr__new__),
     __doc__ = '''str(object) -> string
 
diff --git a/pypy/objspace/std/unicodetype.py b/pypy/objspace/std/unicodetype.py
--- a/pypy/objspace/std/unicodetype.py
+++ b/pypy/objspace/std/unicodetype.py
@@ -2,7 +2,6 @@
 from pypy.interpreter import gateway
 from pypy.objspace.std.stdtypedef import StdTypeDef, SMM
 from pypy.objspace.std.register_all import register_all
-from pypy.objspace.std.basestringtype import basestring_typedef
 from pypy.rlib.runicode import str_decode_utf_8, str_decode_ascii,\
      unicode_encode_utf_8, unicode_encode_ascii
 
@@ -358,7 +357,7 @@
 
 # ____________________________________________________________
 
-unicode_typedef = StdTypeDef("unicode", basestring_typedef,
+unicode_typedef = StdTypeDef("str",
     __new__ = gateway.interp2app(descr_new_),
     __doc__ = '''unicode(string [, encoding[, errors]]) -> object
 

From noreply at buildbot.pypy.org  Thu Oct 13 01:33:41 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 13 Oct 2011 01:33:41 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Add token.__all__,
 and fix applevel raises when a string is passed to be compiled.
Message-ID: <20111012233341.5FCC382112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48005:565ca4631cdf
Date: 2011-10-13 01:08 +0200
http://bitbucket.org/pypy/pypy/changeset/565ca4631cdf/

Log:	Add token.__all__, and fix applevel raises when a string is passed
	to be compiled.

diff --git a/pypy/module/token/__init__.py b/pypy/module/token/__init__.py
--- a/pypy/module/token/__init__.py
+++ b/pypy/module/token/__init__.py
@@ -21,6 +21,8 @@
         tok_name[id] = tok
     Module.interpleveldefs["tok_name"] = "space.wrap(%r)" % (tok_name,)
     Module.interpleveldefs["N_TOKENS"] = "space.wrap(%d)" % len(tok_name)
+    all_names = Module.interpleveldefs.keys()
+    Module.interpleveldefs["__all__"] = "space.wrap(%r)" % (all_names,)
 
 _init_tokens()
 
diff --git a/pypy/tool/pytest/appsupport.py b/pypy/tool/pytest/appsupport.py
--- a/pypy/tool/pytest/appsupport.py
+++ b/pypy/tool/pytest/appsupport.py
@@ -213,7 +213,7 @@
 def pypyraises(space, w_ExpectedException, w_expr, __args__):
     """A built-in function providing the equivalent of py.test.raises()."""
     args_w, kwds_w = __args__.unpack()
-    if space.is_true(space.isinstance(w_expr, space.w_str)):
+    if space.is_true(space.isinstance(w_expr, space.w_unicode)):
         if args_w:
             raise OperationError(space.w_TypeError,
                                  space.wrap("raises() takes no argument "

From noreply at buildbot.pypy.org  Thu Oct 13 01:33:42 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 13 Oct 2011 01:33:42 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Remove unbound methods. see CPython change
	48af6375207e
Message-ID: <20111012233342.9F0F382112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48006:9d18583f7fc4
Date: 2011-10-13 01:09 +0200
http://bitbucket.org/pypy/pypy/changeset/9d18583f7fc4/

Log:	Remove unbound methods. see CPython change 48af6375207e

diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -420,87 +420,40 @@
     """functionobject.__get__(obj[, type]) -> method"""
     # this is not defined as a method on Function because it's generally
     # useful logic: w_function can be any callable.  It is used by Method too.
-    asking_for_bound = (space.is_w(w_cls, space.w_None) or
-                        not space.is_w(w_obj, space.w_None) or
-                        space.is_w(w_cls, space.type(space.w_None)))
-    if asking_for_bound:
-        return space.wrap(Method(space, w_function, w_obj, w_cls))
+    if w_obj is None or space.is_w(w_obj, space.w_None):
+        return w_function
     else:
-        return space.wrap(Method(space, w_function, None, w_cls))
+        return space.wrap(Method(space, w_function, w_obj))
 
 
 class Method(Wrappable):
-    """A method is a function bound to a specific instance or class."""
-    _immutable_fields_ = ['w_function', 'w_instance', 'w_class']
+    """A method is a function bound to a specific instance."""
+    _immutable_fields_ = ['w_function', 'w_instance']
 
-    def __init__(self, space, w_function, w_instance, w_class):
+    def __init__(self, space, w_function, w_instance):
         self.space = space
         self.w_function = w_function
-        self.w_instance = w_instance   # or None
-        self.w_class = w_class         # possibly space.w_None
+        self.w_instance = w_instance
 
-    def descr_method__new__(space, w_subtype, w_function, w_instance, w_class=None):
+    def descr_method__new__(space, w_subtype, w_function, w_instance):
         if space.is_w(w_instance, space.w_None):
             w_instance = None
-        if w_instance is None and space.is_w(w_class, space.w_None):
+        if w_instance is None:
             raise OperationError(space.w_TypeError,
-                                 space.wrap("unbound methods must have class"))
+                                 space.wrap("self must not be None"))
         method = space.allocate_instance(Method, w_subtype)
-        Method.__init__(method, space, w_function, w_instance, w_class)
+        Method.__init__(method, space, w_function, w_instance)
         return space.wrap(method)
 
     def __repr__(self):
-        if self.w_instance:
-            pre = "bound"
-        else:
-            pre = "unbound"
-        return "%s method %s" % (pre, self.w_function.getname(self.space))
+        return "bound method %s" % (self.w_function.getname(self.space),)
 
     def call_args(self, args):
         space = self.space
-        if self.w_instance is not None:
-            # bound method
-            return space.call_obj_args(self.w_function, self.w_instance, args)
-
-        # unbound method
-        w_firstarg = args.firstarg()
-        if w_firstarg is not None and (
-                space.abstract_isinstance_w(w_firstarg, self.w_class)):
-            pass  # ok
-        else:
-            myname = self.getname(space, "")
-            clsdescr = self.w_class.getname(space, "")
-            if clsdescr:
-                clsdescr += " instance"
-            else:
-                clsdescr = "instance"
-            if w_firstarg is None:
-                instdescr = "nothing"
-            else:
-                instname = space.abstract_getclass(w_firstarg).getname(space,
-                                                                       "")
-                if instname:
-                    instdescr = instname + " instance"
-                else:
-                    instdescr = "instance"
-            msg = ("unbound method %s() must be called with %s "
-                   "as first argument (got %s instead)")
-            raise operationerrfmt(space.w_TypeError, msg,
-                                  myname, clsdescr, instdescr)
-        return space.call_args(self.w_function, args)
+        return space.call_obj_args(self.w_function, self.w_instance, args)
 
     def descr_method_get(self, w_obj, w_cls=None):
-        space = self.space
-        if self.w_instance is not None:
-            return space.wrap(self)    # already bound
-        else:
-            # only allow binding to a more specific class than before
-            if (w_cls is not None and
-                not space.is_w(w_cls, space.w_None) and
-                not space.abstract_issubclass_w(w_cls, self.w_class)):
-                return space.wrap(self)    # subclass test failed
-            else:
-                return descr_function_get(space, self.w_function, w_obj, w_cls)
+        return self.space.wrap(self)    # already bound
 
     def descr_method_call(self, __args__):
         return self.call_args(__args__)
@@ -508,19 +461,11 @@
     def descr_method_repr(self):
         space = self.space
         name = self.w_function.getname(self.space)
-        # XXX do we handle all cases sanely here?
-        if space.is_w(self.w_class, space.w_None):
-            w_class = space.type(self.w_instance)
-        else:
-            w_class = self.w_class
+        w_class = space.type(self.w_instance)
         typename = w_class.getname(self.space)
-        if self.w_instance is None:
-            s = "<unbound method %s.%s>" % (typename, name)
-            return space.wrap(s)
-        else:
-            objrepr = space.str_w(space.repr(self.w_instance))
-            s = '<bound method %s.%s of %s>' % (typename, name, objrepr)
-            return space.wrap(s)
+        objrepr = space.str_w(space.repr(self.w_instance))
+        s = '<bound method %s.%s of %s>' % (typename, name, objrepr)
+        return space.wrap(s)
 
     def descr_method_getattribute(self, w_attr):
         space = self.space
@@ -539,21 +484,14 @@
         other = space.interpclass_w(w_other)
         if not isinstance(other, Method):
             return space.w_NotImplemented
-        if self.w_instance is None:
-            if other.w_instance is not None:
-                return space.w_False
-        else:
-            if other.w_instance is None:
-                return space.w_False
-            if not space.eq_w(self.w_instance, other.w_instance):
-                return space.w_False
+        if not space.eq_w(self.w_instance, other.w_instance):
+            return space.w_False
         return space.eq(self.w_function, other.w_function)
 
     def descr_method_hash(self):
         space = self.space
         w_result = space.hash(self.w_function)
-        if self.w_instance is not None:
-            w_result = space.xor(w_result, space.hash(self.w_instance))
+        w_result = space.xor(w_result, space.hash(self.w_instance))
         return w_result
 
     def descr_method__reduce__(self, space):
@@ -561,20 +499,15 @@
         from pypy.interpreter.gateway import BuiltinCode
         w_mod    = space.getbuiltinmodule('_pickle_support')
         mod      = space.interp_w(MixedModule, w_mod)
-        new_inst = mod.get('method_new')
         w        = space.wrap
         w_instance = self.w_instance or space.w_None
         function = space.interpclass_w(self.w_function)
         if isinstance(function, Function) and isinstance(function.code, BuiltinCode):
             new_inst = mod.get('builtin_method_new')
-            if space.is_w(w_instance, space.w_None):
-                tup = [self.w_class, space.wrap(function.name)]
-            else:
-                tup = [w_instance, space.wrap(function.name)]
-        elif space.is_w( self.w_class, space.w_None ):
+            tup = [w_instance, space.wrap(function.name)]
+        else:
+            new_inst = mod.get('method_new')
             tup = [self.w_function, w_instance]
-        else:
-            tup = [self.w_function, w_instance, self.w_class]
         return space.newtuple([new_inst, space.newtuple(tup)])
 
 class StaticMethod(Wrappable):
@@ -603,7 +536,7 @@
     def descr_classmethod_get(self, space, w_obj, w_klass=None):
         if space.is_w(w_klass, space.w_None):
             w_klass = space.type(w_obj)
-        return space.wrap(Method(space, self.w_function, w_klass, space.w_None))
+        return space.wrap(Method(space, self.w_function, w_klass))
 
     def descr_classmethod__new__(space, w_subtype, w_function):
         instance = space.allocate_instance(ClassMethod, w_subtype)
diff --git a/pypy/interpreter/test/test_function.py b/pypy/interpreter/test/test_function.py
--- a/pypy/interpreter/test/test_function.py
+++ b/pypy/interpreter/test/test_function.py
@@ -101,11 +101,11 @@
 
     def test_set_module_to_name_eagerly(self):
         skip("fails on PyPy but works on CPython.  Unsure we want to care")
-        exec '''if 1:
+        exec('''if 1:
             __name__ = "foo"
             def f(): pass
             __name__ = "bar"
-            assert f.__module__ == "foo"''' in {}
+            assert f.__module__ == "foo"''')
 
 
 class AppTestFunction:
@@ -295,9 +295,9 @@
 
     def test_unicode_docstring(self):
         def f():
-            u"hi"
-        assert f.__doc__ == u"hi"
-        assert type(f.__doc__) is unicode
+            "hi"
+        assert f.__doc__ == "hi"
+        assert type(f.__doc__) is str
 
     def test_subclassing(self):
         # cannot subclass 'function' or 'builtin_function'
@@ -418,13 +418,11 @@
         class A(object):
             def f(self):
                 pass
-        assert repr(A.f) == "<unbound method A.f>"
         assert repr(A().f).startswith("<bound method A.f of <")
         assert repr(A().f).endswith(">>")
         class B:
             def f(self):
                 pass
-        assert repr(B.f) == "<unbound method B.f>"
         assert repr(B().f).startswith("<bound method B.f of <")
         assert repr(A().f).endswith(">>")
 
@@ -451,69 +449,6 @@
         im = types.MethodType(A(), 3)
         assert map(im, [4]) == [7]
 
-    def test_unbound_typecheck(self):
-        class A(object):
-            def foo(self, *args):
-                return args
-        class B(A):
-            pass
-        class C(A):
-            pass
-
-        assert A.foo(A(), 42) == (42,)
-        assert A.foo(B(), 42) == (42,)
-        raises(TypeError, A.foo, 5)
-        raises(TypeError, B.foo, C())
-        try:
-            class Fun:
-                __metaclass__ = A.foo
-            assert 0  # should have raised
-        except TypeError:
-            pass
-        class Fun:
-            __metaclass__ = A().foo
-        assert Fun[:2] == ('Fun', ())
-
-    def test_unbound_abstract_typecheck(self):
-        import types
-        def f(*args):
-            return args
-        m = types.MethodType(f, None, "foobar")
-        raises(TypeError, m)
-        raises(TypeError, m, None)
-        raises(TypeError, m, "egg")
-
-        m = types.MethodType(f, None, (str, int))     # really obscure...
-        assert m(4) == (4,)
-        assert m("uh") == ("uh",)
-        raises(TypeError, m, [])
-
-        class MyBaseInst(object):
-            pass
-        class MyInst(MyBaseInst):
-            def __init__(self, myclass):
-                self.myclass = myclass
-            def __class__(self):
-                if self.myclass is None:
-                    raise AttributeError
-                return self.myclass
-            __class__ = property(__class__)
-        class MyClass(object):
-            pass
-        BBase = MyClass()
-        BSub1 = MyClass()
-        BSub2 = MyClass()
-        BBase.__bases__ = ()
-        BSub1.__bases__ = (BBase,)
-        BSub2.__bases__ = (BBase,)
-        x = MyInst(BSub1)
-        m = types.MethodType(f, None, BSub1)
-        assert m(x) == (x,)
-        raises(TypeError, m, MyInst(BBase))
-        raises(TypeError, m, MyInst(BSub2))
-        raises(TypeError, m, MyInst(None))
-        raises(TypeError, m, MyInst(42))
-
     def test_invalid_creation(self):
         import types
         def f(): pass
@@ -590,14 +525,7 @@
         # Check method returned from unbound_method.__get__()
         w_meth3 = descr_function_get(space, func, None, space.type(obj2))
         meth3 = space.unwrap(w_meth3)
-        w_meth4 = meth3.descr_method_get(obj2, space.w_None)
-        meth4 = space.unwrap(w_meth4)
-        assert isinstance(meth4, Method)
-        assert meth4.call_args(args) == obj2
-        # Check method returned from unbound_method.__get__()
-        # --- with an incompatible class
-        w_meth5 = meth3.descr_method_get(space.wrap('hello'), space.w_str)
-        assert space.is_w(w_meth5, w_meth3)
+        assert meth3 is func
 
 class TestShortcuts(object):
 
diff --git a/pypy/module/cpyext/funcobject.py b/pypy/module/cpyext/funcobject.py
--- a/pypy/module/cpyext/funcobject.py
+++ b/pypy/module/cpyext/funcobject.py
@@ -109,13 +109,6 @@
     return borrow_from(w_method, w_method.w_instance)
 
 @cpython_api([PyObject], PyObject)
-def PyMethod_Class(space, w_method):
-    """Return the class object from which the method meth was created; if this was
-    created from an instance, it will be the class of the instance."""
-    assert isinstance(w_method, Method)
-    return borrow_from(w_method, w_method.w_class)
-
- at cpython_api([PyObject], PyObject)
 def PyClassMethod_New(space, w_function):
     return space.call_method(space.builtin, "classmethod", w_function)
 
diff --git a/pypy/module/cpyext/include/funcobject.h b/pypy/module/cpyext/include/funcobject.h
--- a/pypy/module/cpyext/include/funcobject.h
+++ b/pypy/module/cpyext/include/funcobject.h
@@ -16,7 +16,6 @@
 
 #define PyMethod_GET_FUNCTION(obj) PyMethod_Function((PyObject*)(obj))
 #define PyMethod_GET_SELF(obj) PyMethod_Self((PyObject*)(obj))
-#define PyMethod_GET_CLASS(obj) PyMethod_Class((PyObject*)(obj))
 
 #ifdef __cplusplus
 }

From noreply at buildbot.pypy.org  Thu Oct 13 01:33:43 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 13 Oct 2011 01:33:43 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Add ResourceWarning. Not used yet,
 but now "import warnings" passes!
Message-ID: <20111012233343.D453D82112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48007:2ad1a5d0adfb
Date: 2011-10-13 01:19 +0200
http://bitbucket.org/pypy/pypy/changeset/2ad1a5d0adfb/

Log:	Add ResourceWarning. Not used yet, but now "import warnings" passes!

diff --git a/pypy/module/exceptions/__init__.py b/pypy/module/exceptions/__init__.py
--- a/pypy/module/exceptions/__init__.py
+++ b/pypy/module/exceptions/__init__.py
@@ -12,6 +12,7 @@
         'BufferError' : 'interp_exceptions.W_BufferError',
         'BytesWarning'  : 'interp_exceptions.W_BytesWarning',
         'DeprecationWarning' : 'interp_exceptions.W_DeprecationWarning',
+        'ResourceWarning'  : 'interp_exceptions.W_ResourceWarning',
         'EOFError' : 'interp_exceptions.W_EOFError',
         'EnvironmentError' : 'interp_exceptions.W_EnvironmentError',
         'Exception' : 'interp_exceptions.W_Exception',
diff --git a/pypy/module/exceptions/interp_exceptions.py b/pypy/module/exceptions/interp_exceptions.py
--- a/pypy/module/exceptions/interp_exceptions.py
+++ b/pypy/module/exceptions/interp_exceptions.py
@@ -70,6 +70,7 @@
            +-- ImportWarning
            +-- UnicodeWarning
            +-- BytesWarning
+           +-- ResourceWarning
 """
 
 from pypy.interpreter.baseobjspace import Wrappable
@@ -463,6 +464,9 @@
 W_DeprecationWarning = _new_exception('DeprecationWarning', W_Warning,
                         """Base class for warnings about deprecated features.""")
 
+W_ResourceWarning = _new_exception('ResourceWarning', W_Warning,
+         """Base class for warnings about resource usage.""")
+
 W_ArithmeticError = _new_exception('ArithmeticError', W_StandardError,
                          """Base class for arithmetic errors.""")
 

From noreply at buildbot.pypy.org  Thu Oct 13 01:33:45 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 13 Oct 2011 01:33:45 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Fix more __metaclass__ constructs
Message-ID: <20111012233345.19E2B82112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48008:6876c3ff9b9a
Date: 2011-10-13 01:30 +0200
http://bitbucket.org/pypy/pypy/changeset/6876c3ff9b9a/

Log:	Fix more __metaclass__ constructs

diff --git a/pypy/module/posix/app_posix.py b/pypy/module/posix/app_posix.py
--- a/pypy/module/posix/app_posix.py
+++ b/pypy/module/posix/app_posix.py
@@ -16,8 +16,7 @@
 error = OSError
 
 
-class stat_result:
-    __metaclass__ = structseqtype
+class stat_result(metaclass=structseqtype):
 
     name = "posix.stat_result"
 
diff --git a/pypy/module/pwd/app_pwd.py b/pypy/module/pwd/app_pwd.py
--- a/pypy/module/pwd/app_pwd.py
+++ b/pypy/module/pwd/app_pwd.py
@@ -1,6 +1,6 @@
 from _structseq import structseqtype, structseqfield
 
-class struct_passwd:
+class struct_passwd(metaclass=structseqtype):
     """
     pwd.struct_passwd: Results from getpw*() routines.
 
@@ -8,7 +8,6 @@
       (pw_name,pw_passwd,pw_uid,pw_gid,pw_gecos,pw_dir,pw_shell)
     or via the object attributes as named in the above tuple.
     """
-    __metaclass__ = structseqtype
     name = "pwd.struct_passwd"
 
     pw_name   = structseqfield(0, "user name")
diff --git a/pypy/module/rctime/app_time.py b/pypy/module/rctime/app_time.py
--- a/pypy/module/rctime/app_time.py
+++ b/pypy/module/rctime/app_time.py
@@ -2,8 +2,7 @@
 
 from _structseq import structseqtype, structseqfield
 
-class struct_time:
-    __metaclass__ = structseqtype
+class struct_time(metaclass=structseqtype):
     __module__ = 'time'
     name = 'time.struct_time'
 

From noreply at buildbot.pypy.org  Thu Oct 13 01:53:01 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 13 Oct 2011 01:53:01 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: chr() now returns unicode
Message-ID: <20111012235301.5FAD182112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48009:b42bce22fe28
Date: 2011-10-13 01:37 +0200
http://bitbucket.org/pypy/pypy/changeset/b42bce22fe28/

Log:	chr() now returns unicode

diff --git a/pypy/module/__builtin__/__init__.py b/pypy/module/__builtin__/__init__.py
--- a/pypy/module/__builtin__/__init__.py
+++ b/pypy/module/__builtin__/__init__.py
@@ -49,7 +49,6 @@
         # interp-level function definitions
         'abs'           : 'operation.abs',
         'chr'           : 'operation.chr',
-        'unichr'        : 'operation.unichr',
         'len'           : 'operation.len',
         'ord'           : 'operation.ord',
         'pow'           : 'operation.pow',
diff --git a/pypy/module/__builtin__/operation.py b/pypy/module/__builtin__/operation.py
--- a/pypy/module/__builtin__/operation.py
+++ b/pypy/module/__builtin__/operation.py
@@ -15,19 +15,9 @@
     "abs(number) -> number\n\nReturn the absolute value of the argument."
     return space.abs(w_val)
 
-def chr(space, w_ascii):
-    "Return a string of one character with the given ascii code."
-    try:
-        char = __builtin__.chr(space.int_w(w_ascii))
-    except ValueError:  # chr(out-of-range)
-        raise OperationError(space.w_ValueError,
-                             space.wrap("character code not in range(256)"))
-    return space.wrap(char)
-
 @unwrap_spec(code=int)
-def unichr(space, code):
+def chr(space, code):
     "Return a Unicode string of one character with the given ordinal."
-    # XXX range checking!
     try:
         c = UNICHR(code)
     except ValueError:
diff --git a/pypy/module/__builtin__/test/test_builtin.py b/pypy/module/__builtin__/test/test_builtin.py
--- a/pypy/module/__builtin__/test/test_builtin.py
+++ b/pypy/module/__builtin__/test/test_builtin.py
@@ -37,11 +37,6 @@
         raises(ImportError, __import__, 'spamspam')
         raises(TypeError, __import__, 1, 2, 3, 4)
 
-    def test_chr(self):
-        assert chr(65) == 'A'
-        raises(ValueError, chr, -1)
-        raises(TypeError, chr, 'a')
-
     def test_bin(self):
         assert bin(0) == "0b0"
         assert bin(-1) == "-0b1"
@@ -49,18 +44,18 @@
         assert bin(-2L) == "-0b10"
         raises(TypeError, bin, 0.)
 
-    def test_unichr(self):
+    def test_chr(self):
         import sys
-        assert unichr(65) == u'A'
-        assert type(unicode(65)) is unicode
-        assert unichr(0x9876) == u'\u9876'
+        assert chr(65) == 'A'
+        assert type(str(65)) is str
+        assert chr(0x9876) == '\u9876'
         if sys.maxunicode > 0xFFFF:
-            assert unichr(sys.maxunicode) == u'\U0010FFFF'
-            assert unichr(0x10000) == u'\U00010000'
+            assert chr(sys.maxunicode) == '\U0010FFFF'
+            assert chr(0x10000) == '\U00010000'
         else:
-            assert unichr(sys.maxunicode) == u'\uFFFF'
-        raises(ValueError, unichr, -1)
-        raises(ValueError, unichr, sys.maxunicode+1)
+            assert chr(sys.maxunicode) == '\uFFFF'
+        raises(ValueError, chr, -1)
+        raises(ValueError, chr, sys.maxunicode+1)
 
     def test_intern(self):
         raises(TypeError, intern)

From noreply at buildbot.pypy.org  Thu Oct 13 01:53:02 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 13 Oct 2011 01:53:02 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: marshal does not intern string anymore
Message-ID: <20111012235302.9C0EF82112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48010:bfb4fd55c1ed
Date: 2011-10-13 01:48 +0200
http://bitbucket.org/pypy/pypy/changeset/bfb4fd55c1ed/

Log:	marshal does not intern string anymore

diff --git a/pypy/objspace/std/marshal_impl.py b/pypy/objspace/std/marshal_impl.py
--- a/pypy/objspace/std/marshal_impl.py
+++ b/pypy/objspace/std/marshal_impl.py
@@ -47,7 +47,6 @@
 TYPE_BINARY_COMPLEX = 'y'
 TYPE_LONG      = 'l'
 TYPE_STRING    = 's'
-TYPE_INTERNED  = 't'
 TYPE_STRINGREF = 'R'
 TYPE_TUPLE     = '('
 TYPE_LIST      = '['
@@ -243,30 +242,9 @@
     return w_long
 register(TYPE_LONG, unmarshal_Long)
 
-# XXX currently, intern() is at applevel,
-# and there is no interface to get at the
-# internal table.
-# Move intern to interplevel and add a flag
-# to strings.
-def PySTRING_CHECK_INTERNED(w_str):
-    return False
-
 def marshal_w__String(space, w_str, m):
-    # using the fastest possible access method here
-    # that does not touch the internal representation,
-    # which might change (array of bytes?)
     s = w_str.unwrap(space)
-    if m.version >= 1 and PySTRING_CHECK_INTERNED(w_str):
-        # we use a native rtyper stringdict for speed
-        idx = m.stringtable.get(s, -1)
-        if idx >= 0:
-            m.atom_int(TYPE_STRINGREF, idx)
-        else:
-            idx = len(m.stringtable)
-            m.stringtable[s] = idx
-            m.atom_str(TYPE_INTERNED, s)
-    else:
-        m.atom_str(TYPE_STRING, s)
+    m.atom_str(TYPE_STRING, s)
 
 marshal_w__Rope = marshal_w__String
 
@@ -274,14 +252,6 @@
     return space.wrap(u.get_str())
 register(TYPE_STRING, unmarshal_String)
 
-def unmarshal_interned(space, u, tc):
-    w_ret = space.wrap(u.get_str())
-    u.stringtable_w.append(w_ret)
-    w_intern = space.builtin.get('intern')
-    space.call_function(w_intern, w_ret)
-    return w_ret
-register(TYPE_INTERNED, unmarshal_interned)
-
 def unmarshal_stringref(space, u, tc):
     idx = u.get_int()
     try:
@@ -348,12 +318,12 @@
     m.put_int(x.co_flags)
     m.atom_str(TYPE_STRING, x.co_code)
     m.put_tuple_w(TYPE_TUPLE, x.co_consts_w[:])
-    m.atom_strlist(TYPE_TUPLE, TYPE_INTERNED, [space.str_w(w_name) for w_name in x.co_names_w])
-    m.atom_strlist(TYPE_TUPLE, TYPE_INTERNED, x.co_varnames)
-    m.atom_strlist(TYPE_TUPLE, TYPE_INTERNED, x.co_freevars)
-    m.atom_strlist(TYPE_TUPLE, TYPE_INTERNED, x.co_cellvars)
-    m.atom_str(TYPE_INTERNED, x.co_filename)
-    m.atom_str(TYPE_INTERNED, x.co_name)
+    m.atom_strlist(TYPE_TUPLE, TYPE_STRING, [space.str_w(w_name) for w_name in x.co_names_w])
+    m.atom_strlist(TYPE_TUPLE, TYPE_STRING, x.co_varnames)
+    m.atom_strlist(TYPE_TUPLE, TYPE_STRING, x.co_freevars)
+    m.atom_strlist(TYPE_TUPLE, TYPE_STRING, x.co_cellvars)
+    m.atom_str(TYPE_STRING, x.co_filename)
+    m.atom_str(TYPE_STRING, x.co_name)
     m.put_int(x.co_firstlineno)
     m.atom_str(TYPE_STRING, x.co_lnotab)
 

From noreply at buildbot.pypy.org  Thu Oct 13 01:53:03 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 13 Oct 2011 01:53:03 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Slowly transform string into a bytestring.
Message-ID: <20111012235303.DA88082112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48011:1e27363024ac
Date: 2011-10-13 01:52 +0200
http://bitbucket.org/pypy/pypy/changeset/1e27363024ac/

Log:	Slowly transform string into a bytestring. First step: getitem
	return integers!

diff --git a/lib_pypy/binascii.py b/lib_pypy/binascii.py
--- a/lib_pypy/binascii.py
+++ b/lib_pypy/binascii.py
@@ -703,7 +703,7 @@
     def pairs_gen(s):
         while s:
             try:
-                yield table_hex[ord(s[0])], table_hex[ord(s[1])]
+                yield table_hex[s[0]], table_hex[s[1]]
             except IndexError:
                 if len(s):
                     raise TypeError('Odd-length string')
diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py
--- a/pypy/objspace/std/stringobject.py
+++ b/pypy/objspace/std/stringobject.py
@@ -839,7 +839,7 @@
     if ival < 0 or ival >= slen:
         raise OperationError(space.w_IndexError,
                              space.wrap("string index out of range"))
-    return wrapchar(space, str[ival])
+    return space.wrap(ord(str[ival]))
 
 def getitem__String_Slice(space, w_str, w_slice):
     w = space.wrap

From noreply at buildbot.pypy.org  Thu Oct 13 02:03:42 2011
From: noreply at buildbot.pypy.org (pjenvey)
Date: Thu, 13 Oct 2011 02:03:42 +0200 (CEST)
Subject: [pypy-commit] pypy default: improve subprocess error handling
Message-ID: <20111013000342.8B4D582112@wyvern.cs.uni-duesseldorf.de>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: 
Changeset: r48012:c041e29173a9
Date: 2011-10-12 17:02 -0700
http://bitbucket.org/pypy/pypy/changeset/c041e29173a9/

Log:	improve subprocess error handling

diff --git a/pypy/jit/backend/x86/tool/viewcode.py b/pypy/jit/backend/x86/tool/viewcode.py
--- a/pypy/jit/backend/x86/tool/viewcode.py
+++ b/pypy/jit/backend/x86/tool/viewcode.py
@@ -9,7 +9,12 @@
 """
 
 import autopath
-import operator, sys, os, re, py, new
+import new
+import operator
+import py
+import re
+import sys
+import subprocess
 from bisect import bisect_left
 
 # don't use pypy.tool.udir here to avoid removing old usessions which
@@ -44,14 +49,16 @@
     f = open(tmpfile, 'wb')
     f.write(data)
     f.close()
-    g = os.popen(objdump % {
+    p = subprocess.Popen(objdump % {
         'file': tmpfile,
         'origin': originaddr,
         'backend': objdump_backend_option[backend_name],
-    }, 'r')
-    result = g.readlines()
-    g.close()
-    lines = result[6:]   # drop some objdump cruft
+    }, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    stdout, stderr = p.communicate()
+    assert not p.returncode, ('Encountered an error running objdump: %s' %
+                              stderr)
+    # drop some objdump cruft
+    lines = stdout.splitlines()[6:]
     return format_code_dump_with_labels(originaddr, lines, label_list)
 
 def format_code_dump_with_labels(originaddr, lines, label_list):
@@ -85,8 +92,12 @@
     #
     print 'loading symbols from %s...' % (filename,)
     symbols = {}
-    g = os.popen(symbollister % filename, "r")
-    for line in g:
+    p = subprocess.Popen(symbollister % filename, shell=True,
+                         stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    stdout, stderr = p.communicate()
+    assert not p.returncode, ('Encountered an error running nm: %s' %
+                              stderr)
+    for line in stdout.splitlines():
         match = re_symbolentry.match(line)
         if match:
             addr = long(match.group(1), 16)
@@ -94,7 +105,6 @@
             if name.startswith('pypy_g_'):
                 name = '\xb7' + name[7:]
             symbols[addr] = name
-    g.close()
     print '%d symbols found' % (len(symbols),)
     return symbols
 

From noreply at buildbot.pypy.org  Thu Oct 13 05:24:22 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Thu, 13 Oct 2011 05:24:22 +0200 (CEST)
Subject: [pypy-commit] pypy default: skip this test when run directly
Message-ID: <20111013032422.7D09C82112@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r48013:c6973e36b7c5
Date: 2011-10-12 23:24 -0400
http://bitbucket.org/pypy/pypy/changeset/c6973e36b7c5/

Log:	skip this test when run directly

diff --git a/pypy/module/_file/test/test_file.py b/pypy/module/_file/test/test_file.py
--- a/pypy/module/_file/test/test_file.py
+++ b/pypy/module/_file/test/test_file.py
@@ -260,8 +260,12 @@
 class AppTestNonblocking(object):
     def setup_class(cls):
         from pypy.module._file.interp_file import W_File
-        
+
         cls.old_read = os.read
+
+        if option.runappdirect:
+            py.test.skip("works with internals of _file impl on py.py")
+
         state = [0]
         def read(fd, n=None):
             if fd != 42:
@@ -281,7 +285,7 @@
 
     def teardown_class(cls):
         os.read = cls.old_read
-        
+
     def test_nonblocking_file(self):
         res = self.stream.read()
         assert res == 'xyz'
@@ -403,24 +407,24 @@
         with self.file(self.temppath, 'w') as f:
             f.write('foo')
         assert f.closed
-        
+
         with self.file(self.temppath, 'r') as f:
             s = f.readline()
 
         assert s == "foo"
         assert f.closed
-    
+
     def test_subclass_with(self):
         file = self.file
         class C(file):
             def __init__(self, *args, **kwargs):
                 self.subclass_closed = False
                 file.__init__(self, *args, **kwargs)
-            
+
             def close(self):
                 self.subclass_closed = True
                 file.close(self)
-        
+
         with C(self.temppath, 'w') as f:
             pass
         assert f.subclass_closed

From noreply at buildbot.pypy.org  Thu Oct 13 09:09:31 2011
From: noreply at buildbot.pypy.org (hager)
Date: Thu, 13 Oct 2011 09:09:31 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: Adjusted the tests in
 test_ppc.py to the new framework.
Message-ID: <20111013070931.676CD82112@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r48014:0333d746965b
Date: 2011-10-13 09:09 +0200
http://bitbucket.org/pypy/pypy/changeset/0333d746965b/

Log:	Adjusted the tests in test_ppc.py to the new framework.

diff --git a/pypy/jit/backend/ppc/ppcgen/test/test_ppc.py b/pypy/jit/backend/ppc/ppcgen/test/test_ppc.py
--- a/pypy/jit/backend/ppc/ppcgen/test/test_ppc.py
+++ b/pypy/jit/backend/ppc/ppcgen/test/test_ppc.py
@@ -1,9 +1,10 @@
 import py
 import random, sys, os
 
-from pypy.jit.backend.ppc.ppcgen.ppc_assembler import BasicPPCAssembler, PPCBuilder
+from pypy.jit.backend.ppc.ppcgen.codebuilder import BasicPPCAssembler, PPCBuilder
 from pypy.jit.backend.ppc.ppcgen.symbol_lookup import lookup
 from pypy.jit.backend.ppc.ppcgen.regname import *
+from pypy.jit.backend.ppc.ppcgen.register import *
 from pypy.jit.backend.ppc.ppcgen import form, pystructs
 from pypy.jit.backend.detect_cpu import autodetect_main_model
 from pypy.jit.backend.ppc.ppcgen.arch import IS_PPC_32, IS_PPC_64, WORD
@@ -22,7 +23,7 @@
 
 """
 Creates the boilerplate code for the tests.
-- Make an PPCBuilder object
+- Make a PPCBuilder object
 - Let the given test create the machine code
 - Create a function and call it
 - Compare the return value with the expected result
@@ -80,7 +81,7 @@
     """
 
     @asmtest(expected=200)
-    def test_load_imm(self, a):
+    def test_li(self, a):
         a.li(3, 200)
         a.blr()
 
@@ -91,8 +92,8 @@
         a.blr()
 
     @asmtest(expected=12341234)
-    def test_load_word(self, a):
-        a.load_word(10, 12341234)
+    def test_load_imm(self, a):
+        a.load_imm(r10, 12341234)
         a.mtctr(10)
         a.mfctr(11)
         a.mr(3, 11)
@@ -100,16 +101,16 @@
 
     @asmtest(expected=33333333)
     def test_add_reg(self, a):
-        a.load_word(10, 11111111)
-        a.load_word(11, 22222222)
+        a.load_imm(r10, 11111111)
+        a.load_imm(r11, 22222222)
         a.add(12, 10, 11)
         a.mr(3, 12)
         a.blr()
 
     @asmtest(expected=-1000)
     def test_add_pos_and_neg(self, a):
-        a.load_word(10, 2000)
-        a.load_word(11, -3000)
+        a.load_imm(r10, 2000)
+        a.load_imm(r11, -3000)
         a.add(3, 10, 11)
         a.blr()
 
@@ -121,14 +122,14 @@
 
     @asmtest(expected=(123435 - 76457))
     def test_sub_reg(self, a):
-        a.load_word(5, 123435)
-        a.load_word(6, 76457)
+        a.load_imm(r5, 123435)
+        a.load_imm(r6, 76457)
         a.sub(3, 5, 6)
         a.blr()
 
     @asmtest(expected=(10000 * 5000))
     def test_mul_imm(self, a):
-        a.load_word(3, 10000)
+        a.load_imm(r3, 10000)
         a.mulli(3, 3, 5000)
         a.blr()
 
@@ -137,8 +138,8 @@
     @asmtest(expected=bits_to_signed_int('11010100101001010001000000000000'))
     def test_mullw(self, a):
         word = 1000000
-        a.load_word(5, word)
-        a.load_word(6, word)
+        a.load_imm(r5, word)
+        a.load_imm(r6, word)
         a.mullw(3, 5, 6)
         if IS_PPC_64:
             a.extsw(3, 3)
@@ -149,8 +150,8 @@
     @asmtest(expected=int('11101000', 2))
     def test_mulhw(self, a):
         word = 1000000
-        a.load_word(5, word)
-        a.load_word(6, word)
+        a.load_imm(r5, word)
+        a.load_imm(r6, word)
         a.mulhw(3, 5, 6)
         if IS_PPC_64:
             a.extsw(3, 3)
@@ -161,8 +162,8 @@
     @asmtest(expected=int('11101000', 2))
     def test_mulhwu(self, a):
         word = 1000000
-        a.load_word(5, word)
-        a.load_word(6, word)
+        a.load_imm(r5, word)
+        a.load_imm(r6, word)
         a.mulhwu(3, 5, 6)
         if IS_PPC_64:
             a.extsw(3, 3)
@@ -172,8 +173,8 @@
     def test_divw(self, a):
         divident = 1000000
         divisor = 100
-        a.load_word(10, divident)
-        a.load_word(11, divisor)
+        a.load_imm(r10, divident)
+        a.load_imm(r11, divisor)
         a.divw(3, 10, 11)
         a.blr()
 
@@ -191,7 +192,7 @@
 
         a.li(3, 50)
         if IS_PPC_32:
-            a.load_word(10, call_addr)
+            a.load_imm(r10, call_addr)
         else:
             a.load_from(10, call_addr)
             a.load_from(2, call_addr+WORD)
@@ -205,60 +206,60 @@
 
     @asmtest(expected=0)
     def test_and(self, a):
-        a.load_word(10, 8)
-        a.load_word(11, 7)
+        a.load_imm(r10, 8)
+        a.load_imm(r11, 7)
         a.and_(3, 10, 11)
         a.blr()
 
     @asmtest(expected=15)
     def test_or(self, a):
-        a.load_word(10, 8)
-        a.load_word(11, 7)
+        a.load_imm(r10, 8)
+        a.load_imm(r11, 7)
         a.or_(3, 10, 11)
         a.blr()
 
     @asmtest(expected=15)
     def test_nand(self, a):
-        a.load_word(10, 8)
-        a.load_word(11, 7)
+        a.load_imm(r10, 8)
+        a.load_imm(r11, 7)
         a.nand(3, 10, 11)
-        a.load_word(12, 0x0000000F) # zero out first 28 bits
+        a.load_imm(r12, 0x0000000F) # zero out first 28 bits
         a.and_(3, 3, 12)            # 
         a.blr()
 
     @asmtest(expected=1)
     def test_nor(self, a):
-        a.load_word(10, 10)
-        a.load_word(11, 6)
+        a.load_imm(r10, 10)
+        a.load_imm(r11, 6)
         a.nor(3, 10, 11)
-        a.load_word(12, 0x0000000F) # zero out first 28 bits
+        a.load_imm(r12, 0x0000000F) # zero out first 28 bits
         a.and_(3, 3, 12)            # 
         a.blr()
 
     @asmtest(expected=5)
     def test_xor(self, a):
-        a.load_word(10, 15)
-        a.load_word(11, 10)
+        a.load_imm(r10, 15)
+        a.load_imm(r11, 10)
         a.xor(3, 10, 11)
         a.blr()
 
     @asmtest(expected=0x120)
     def test_slw(self, a):
-        a.load_word(10, 9)
-        a.load_word(11, 5)
+        a.load_imm(r10, 9)
+        a.load_imm(r11, 5)
         a.slw(3, 10, 11)
         a.blr()
 
     @asmtest(expected=9)
     def test_srw(self, a):
-        a.load_word(10, 0x120)
-        a.load_word(11, 5)
+        a.load_imm(r10, 0x120)
+        a.load_imm(r11, 5)
         a.srw(3, 10, 11)
         a.blr()
 
     def test_neg(self):
         a = PPCBuilder()
-        a.load_word(10, 0x0000F0F0)
+        a.load_imm(r10, 0x0000F0F0)
         a.neg(3, 10)
         a.blr()
         f = a.assemble()
@@ -270,11 +271,11 @@
         word2 = 2000
         p = lltype.malloc(rffi.CArray(lltype.Signed), 2, flavor="raw")
 
-        a.load_word(10, word1)
-        a.load_word(11, word2)
+        a.load_imm(r10, word1)
+        a.load_imm(r11, word2)
 
-        a.load_word(8, rffi.cast(lltype.Signed, p))
-        a.load_word(9, rffi.cast(lltype.Signed, p) + WORD)
+        a.load_imm(r8, rffi.cast(lltype.Signed, p))
+        a.load_imm(r9, rffi.cast(lltype.Signed, p) + WORD)
 
         a.stw(10, 8, 0)
         a.stw(11, 9, 0)
@@ -293,7 +294,7 @@
         addr = rffi.cast(lltype.Signed, p)
         p[0] = rffi.cast(rffi.LONG, 200)
 
-        a.load_from(3, addr)
+        a.load_from_addr(3, addr)
         a.blr()
         f = a.assemble()
         assert f() == 200
@@ -338,7 +339,7 @@
         a = MyPPCAssembler()
 
         a.lwz(3, 4, pystructs.PyVarObject.ob_size)
-        a.load_word(5, lookup("PyInt_FromLong"))
+        a.load_imm(5, lookup("PyInt_FromLong"))
         a.mtctr(5)
         a.bctr()
 
@@ -355,10 +356,10 @@
         a.stw(0, 1, 8)
         a.stwu(1, 1, -80)
         a.mr(3, 4)
-        a.load_word(5, lookup("PyTuple_Size"))
+        a.load_imm(5, lookup("PyTuple_Size"))
         a.mtctr(5)
         a.bctrl()
-        a.load_word(5, lookup("PyInt_FromLong"))
+        a.load_imm(5, lookup("PyInt_FromLong"))
         a.mtctr(5)
         a.bctrl()
         a.lwz(0, 1, 88)
@@ -381,7 +382,7 @@
         a.bne("not_one")
         a.lwz(r5, r4, pystructs.PyTupleObject.ob_item + 0*4)
         a.lwz(r5, r5, 4)
-        a.load_word(r6, lookup("PyInt_Type"))
+        a.load_imm(r6, lookup("PyInt_Type"))
         a.cmpw(r5, r6)
         a.bne("not_int")
         a.li(r3, 1)
@@ -392,7 +393,7 @@
         a.label("not_one")
         a.li(r3, 2)
         a.label("exit")
-        a.load_word(r5, lookup("PyInt_FromLong"))
+        a.load_imm(r5, lookup("PyInt_FromLong"))
         a.mtctr(r5)
         a.bctr()
 
@@ -414,7 +415,7 @@
         err_set = lookup("PyErr_SetObject")
         exc = lookup("PyExc_ValueError")
 
-        a.load_word(5, err_set)
+        a.load_imm(5, err_set)
         a.mtctr(5)
         a.load_from(3, exc)
         a.mr(4, 3)
@@ -435,7 +436,7 @@
 
         a.li(r3, 0)
         a.li(r4, 0)
-        a.load_word(r5, lookup("PyString_FromStringAndSize"))
+        a.load_imm(r5, lookup("PyString_FromStringAndSize"))
         a.mtctr(r5)
         a.bctr()
 
@@ -453,7 +454,7 @@
         a.lwz(r3, r4, 12)
         a.lwz(r4, r4, 16)
 
-        a.load_word(r5, lookup("PyNumber_Add"))
+        a.load_imm(r5, lookup("PyNumber_Add"))
         a.mtctr(r5)
         a.bctr()
 
@@ -466,7 +467,7 @@
         err_set = lookup("PyErr_SetObject")
         exc = lookup("PyExc_TypeError")
 
-        a.load_word(r5, err_set)
+        a.load_imm(r5, err_set)
         a.mtctr(r5)
         a.load_from(r3, exc)
         a.mr(r4, r3)

From noreply at buildbot.pypy.org  Thu Oct 13 16:07:07 2011
From: noreply at buildbot.pypy.org (lac)
Date: Thu, 13 Oct 2011 16:07:07 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: First Draft of the sprint
	announcement
Message-ID: <20111013140707.76C0382112@wyvern.cs.uni-duesseldorf.de>

Author: Laura Creighton <lac at openend.se>
Branch: extradoc
Changeset: r3928:56fa2ee352fe
Date: 2011-10-13 16:02 +0200
http://bitbucket.org/pypy/extradoc/changeset/56fa2ee352fe/

Log:	First Draft of the sprint announcement

	needs adding -- what we want to do at the sprint and whether it is
	at my house or at OE. Note that I am fine with you eating and
	hacking at my house, it is the sleeping over that is not working due
	to the lack of showers. I suppose it is possible that somebody will
	want to begin work on the bathroom building that week, in which case
	we would have to have the sprint at OE on the grounds of it being
	too noisy and dirty at our place, but I think the chances of this is
	small.

diff --git a/sprintinfo/gothenburg-2011-2/announce.txt b/sprintinfo/gothenburg-2011-2/announce.txt
new file mode 100644
--- /dev/null
+++ b/sprintinfo/gothenburg-2011-2/announce.txt
@@ -0,0 +1,112 @@
+PyPy G&#65533;teborg Post-Hallowe'en Sprint Nov 2nd - Nov 9th
+=========================================================
+
+The next PyPy sprint will be in Gothenburg, Sweden. It is a public sprint, 
+suitable for newcomers.  We'll focus on making a public kick off for
+both the numpy/pypy integration project and the py3k interpreter project,
+as well as whatever interests the Sprint attendees.
+
+Topics and goals
+----------------
+
+The main goal is to XXX
+
+Other topics:
+
+- Helping people get their code running with PyPy
+
+- maybe work on FSCon talk
+
+- state of the STM Vinnova project (We most likely, but not for certain will
+  know whether or not we are approved by this date.)
+
+Other Useful dates
+------------------
+GothPyCon_ - Saturday Oct 29.
+
+.. _GothPyCon: http://www.meetup.com/GothPy/events/32864862/
+
+FSCONS_ Friday Nov 11 - Sunday Nov 12.
+
+.. _FSCONS: http://fscons.org/
+
+Location
+--------
+
+XXX I am fine with having the sprint here.  Or we could have it at OE.
+    It's the 'sleeping at my house' part that isn't working.
+
+The sprint will be held in the apartment of Laura Creighton and Jacob Hall&#65533;n
+which is at G&#65533;tabergsgatan 22 in Gothenburg, Sweden.  Here is a map_.  This is
+in central Gothenburg.  It is between the tram_ stops of Vasaplatsen and 
+Valand, (a distance of 4 blocks) where many lines call -- the 2, 3, 4, 5,
+7, 10 and 13.
+
+.. _tram: http://www.vasttrafik.se/en/
+
+.. _map: http://bit.ly/grRuQe
+
+Probably cheapest and not too far away is to book accomodation at `SGS
+Veckobostader`_. The  `Elite Park Avenyn Hotel`_ is a luxury hotel just a 
+few blocks away. There are scores of hotels a short walk away from the
+sprint location, suitable for every budget, desire for luxury, and desire
+for the unusual.  You could, for instance, stay on a `boat`_.  Options are
+too numerous to go into here. Just ask in the mailing list or on the blog.
+
+.. _`SGS Veckobostader`: http://www.sgsveckobostader.se/en
+.. _`Elite Park Avenyn Hotel`: http://www.elite.se/hotell/goteborg/park/
+.. _`boat`: http://www.liseberg.se/en/home/Accommodation/Hotel/Hotel-Barken-Viking/
+
+Hours will be
+from 10:00 until people have had enough.  It's a good idea to arrive a
+day before the sprint starts and leave a day later.  In the middle of
+the sprint there usually is a break day and it's usually ok to take
+half-days off if you feel like it.
+
+
+Good to Know
+------------
+
+Sweden is not part of the Euro zone. One SEK (krona in singular, kronor
+in plural) is roughly 1/10th of a Euro (9.36 SEK to 1 Euro).
+
+The venue is central in Gothenburg.  There is a large selection of
+places to get food nearby, from edible-and-cheap to outstanding.  We
+often cook meals together, so let us know if you have any food allergies,
+dislikes, or special requirements.
+
+Sweden uses the same kind of plugs as Germany. 230V AC.
+
+
+Getting Here
+------------
+If are coming train, you will arrive at the `Central Station`_.  It is
+about 12 blocks to the site from there, or you can take a tram_. 
+
+There are two airports which are local to G&#65533;teborg, `Landvetter`_ (the main
+one) and `Gothenburg City Airport`_ (where some budget airlines fly).
+If you arrive at `Landvetter`_  the airport bus stops right downtown at
+`Elite Park Avenyn Hotel`_ which is the second stop, 4 blocks from the
+Sprint site, as well as the end of the line, which is the `Central Station`_.
+If you arrive at `Gothenburg City Airport`_ take the bus to the end of the
+line.  You will be at the  `Central Station`_.
+
+You can also arrive by ferry_, from either Kiel in Germany or Frederikshavn
+in Denmark.
+
+.. _`Central Station`: http://bit.ly/fON43p
+
+.. _`Landvetter`: http://swedavia.se/en/Goteborg/Traveller-information/Traffic-information/
+.. _`Gothenburg City Airport`: http://www.goteborgairport.se/eng.asp
+
+.. _ferry: http://www.stenaline.nl/en/ferry/
+
+Who's Coming?
+--------------
+
+If you'd like to come, please let us know when you will be arriving and 
+leaving, as well as letting us know your interests  We'll keep a list
+of `people`_ which we'll update (which you can do so yourself if you
+have bitbucket pypy commit rights).
+
+.. _`people`: https://bitbucket.org/pypy/extradoc/src/tip/sprintinfo/gothenburg-2011/people.txt

From noreply at buildbot.pypy.org  Thu Oct 13 18:46:11 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 13 Oct 2011 18:46:11 +0200 (CEST)
Subject: [pypy-commit] pypy.org extradoc: update links
Message-ID: <20111013164611.E6CC182112@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: extradoc
Changeset: r286:f9eb2bf02c17
Date: 2011-10-13 18:45 +0200
http://bitbucket.org/pypy/pypy.org/changeset/f9eb2bf02c17/

Log:	update links

diff --git a/archive.html b/archive.html
--- a/archive.html
+++ b/archive.html
@@ -38,7 +38,7 @@
 	</div>
 	<div id="logo"><a href="http://pypy.org"><img src="image/pypy-logo.png" alt="PyPy" height="110px" /></a></div>
 	<hr class="clear-left" />
-	<div id="menu-sub"><a href="index.html">Home</a><span class="menu-sub-sep"> | </span><a href="features.html">Features</a><span class="menu-sub-sep"> | </span><a href="download.html">Download</a><span class="menu-sub-sep"> | </span><a href="compat.html">Compatibility</a><span class="menu-sub-sep"> | </span><a href="http://speed.pypy.org">Performance</a><span class="menu-sub-sep"> | </span><a href="http://doc.pypy.org">Dev Documentation</a><span class="menu-sub-sep"> | </span><a href="http://morepypy.blogspot.com">Blog</a><span class="menu-sub-sep"> | </span><a href="people.html">People</a><span class="menu-sub-sep"> | </span><a href="contact.html">Contact</a><span class="menu-sub-sep"> | </span><a href="py3donate.html">Py3k donations</a></div>
+	<div id="menu-sub"><a href="index.html">Home</a><span class="menu-sub-sep"> | </span><a href="features.html">Features</a><span class="menu-sub-sep"> | </span><a href="download.html">Download</a><span class="menu-sub-sep"> | </span><a href="compat.html">Compatibility</a><span class="menu-sub-sep"> | </span><a href="http://speed.pypy.org">Performance</a><span class="menu-sub-sep"> | </span><a href="http://doc.pypy.org">Dev Documentation</a><span class="menu-sub-sep"> | </span><a href="http://morepypy.blogspot.com">Blog</a><span class="menu-sub-sep"> | </span><a href="people.html">People</a><span class="menu-sub-sep"> | </span><a href="contact.html">Contact</a><span class="menu-sub-sep"> | </span><a href="py3donate.html">Py3k donations</a><span class="menu-sub-sep"> | </span><a href="numpydonate.html">NumPy donations</a></div>
 	<hr class="clear" />
 </div>
 <div id="content">
diff --git a/compat.html b/compat.html
--- a/compat.html
+++ b/compat.html
@@ -38,7 +38,7 @@
 	</div>
 	<div id="logo"><a href="http://pypy.org"><img src="image/pypy-logo.png" alt="PyPy" height="110px" /></a></div>
 	<hr class="clear-left" />
-	<div id="menu-sub"><a href="index.html">Home</a><span class="menu-sub-sep"> | </span><a href="features.html">Features</a><span class="menu-sub-sep"> | </span><a href="download.html">Download</a><span class="menu-sub-sep"> | </span><a href="compat.html">Compatibility</a><span class="menu-sub-sep"> | </span><a href="http://speed.pypy.org">Performance</a><span class="menu-sub-sep"> | </span><a href="http://doc.pypy.org">Dev Documentation</a><span class="menu-sub-sep"> | </span><a href="http://morepypy.blogspot.com">Blog</a><span class="menu-sub-sep"> | </span><a href="people.html">People</a><span class="menu-sub-sep"> | </span><a href="contact.html">Contact</a><span class="menu-sub-sep"> | </span><a href="py3donate.html">Py3k donations</a></div>
+	<div id="menu-sub"><a href="index.html">Home</a><span class="menu-sub-sep"> | </span><a href="features.html">Features</a><span class="menu-sub-sep"> | </span><a href="download.html">Download</a><span class="menu-sub-sep"> | </span><a href="compat.html">Compatibility</a><span class="menu-sub-sep"> | </span><a href="http://speed.pypy.org">Performance</a><span class="menu-sub-sep"> | </span><a href="http://doc.pypy.org">Dev Documentation</a><span class="menu-sub-sep"> | </span><a href="http://morepypy.blogspot.com">Blog</a><span class="menu-sub-sep"> | </span><a href="people.html">People</a><span class="menu-sub-sep"> | </span><a href="contact.html">Contact</a><span class="menu-sub-sep"> | </span><a href="py3donate.html">Py3k donations</a><span class="menu-sub-sep"> | </span><a href="numpydonate.html">NumPy donations</a></div>
 	<hr class="clear" />
 </div>
 <div id="content">
diff --git a/contact.html b/contact.html
--- a/contact.html
+++ b/contact.html
@@ -38,7 +38,7 @@
 	</div>
 	<div id="logo"><a href="http://pypy.org"><img src="image/pypy-logo.png" alt="PyPy" height="110px" /></a></div>
 	<hr class="clear-left" />
-	<div id="menu-sub"><a href="index.html">Home</a><span class="menu-sub-sep"> | </span><a href="features.html">Features</a><span class="menu-sub-sep"> | </span><a href="download.html">Download</a><span class="menu-sub-sep"> | </span><a href="compat.html">Compatibility</a><span class="menu-sub-sep"> | </span><a href="http://speed.pypy.org">Performance</a><span class="menu-sub-sep"> | </span><a href="http://doc.pypy.org">Dev Documentation</a><span class="menu-sub-sep"> | </span><a href="http://morepypy.blogspot.com">Blog</a><span class="menu-sub-sep"> | </span><a href="people.html">People</a><span class="menu-sub-sep"> | </span><a href="contact.html">Contact</a><span class="menu-sub-sep"> | </span><a href="py3donate.html">Py3k donations</a></div>
+	<div id="menu-sub"><a href="index.html">Home</a><span class="menu-sub-sep"> | </span><a href="features.html">Features</a><span class="menu-sub-sep"> | </span><a href="download.html">Download</a><span class="menu-sub-sep"> | </span><a href="compat.html">Compatibility</a><span class="menu-sub-sep"> | </span><a href="http://speed.pypy.org">Performance</a><span class="menu-sub-sep"> | </span><a href="http://doc.pypy.org">Dev Documentation</a><span class="menu-sub-sep"> | </span><a href="http://morepypy.blogspot.com">Blog</a><span class="menu-sub-sep"> | </span><a href="people.html">People</a><span class="menu-sub-sep"> | </span><a href="contact.html">Contact</a><span class="menu-sub-sep"> | </span><a href="py3donate.html">Py3k donations</a><span class="menu-sub-sep"> | </span><a href="numpydonate.html">NumPy donations</a></div>
 	<hr class="clear" />
 </div>
 <div id="content">
diff --git a/download.html b/download.html
--- a/download.html
+++ b/download.html
@@ -38,7 +38,7 @@
 	</div>
 	<div id="logo"><a href="http://pypy.org"><img src="image/pypy-logo.png" alt="PyPy" height="110px" /></a></div>
 	<hr class="clear-left" />
-	<div id="menu-sub"><a href="index.html">Home</a><span class="menu-sub-sep"> | </span><a href="features.html">Features</a><span class="menu-sub-sep"> | </span><a href="download.html">Download</a><span class="menu-sub-sep"> | </span><a href="compat.html">Compatibility</a><span class="menu-sub-sep"> | </span><a href="http://speed.pypy.org">Performance</a><span class="menu-sub-sep"> | </span><a href="http://doc.pypy.org">Dev Documentation</a><span class="menu-sub-sep"> | </span><a href="http://morepypy.blogspot.com">Blog</a><span class="menu-sub-sep"> | </span><a href="people.html">People</a><span class="menu-sub-sep"> | </span><a href="contact.html">Contact</a><span class="menu-sub-sep"> | </span><a href="py3donate.html">Py3k donations</a></div>
+	<div id="menu-sub"><a href="index.html">Home</a><span class="menu-sub-sep"> | </span><a href="features.html">Features</a><span class="menu-sub-sep"> | </span><a href="download.html">Download</a><span class="menu-sub-sep"> | </span><a href="compat.html">Compatibility</a><span class="menu-sub-sep"> | </span><a href="http://speed.pypy.org">Performance</a><span class="menu-sub-sep"> | </span><a href="http://doc.pypy.org">Dev Documentation</a><span class="menu-sub-sep"> | </span><a href="http://morepypy.blogspot.com">Blog</a><span class="menu-sub-sep"> | </span><a href="people.html">People</a><span class="menu-sub-sep"> | </span><a href="contact.html">Contact</a><span class="menu-sub-sep"> | </span><a href="py3donate.html">Py3k donations</a><span class="menu-sub-sep"> | </span><a href="numpydonate.html">NumPy donations</a></div>
 	<hr class="clear" />
 </div>
 <div id="content">
diff --git a/features.html b/features.html
--- a/features.html
+++ b/features.html
@@ -38,7 +38,7 @@
 	</div>
 	<div id="logo"><a href="http://pypy.org"><img src="image/pypy-logo.png" alt="PyPy" height="110px" /></a></div>
 	<hr class="clear-left" />
-	<div id="menu-sub"><a href="index.html">Home</a><span class="menu-sub-sep"> | </span><a href="features.html">Features</a><span class="menu-sub-sep"> | </span><a href="download.html">Download</a><span class="menu-sub-sep"> | </span><a href="compat.html">Compatibility</a><span class="menu-sub-sep"> | </span><a href="http://speed.pypy.org">Performance</a><span class="menu-sub-sep"> | </span><a href="http://doc.pypy.org">Dev Documentation</a><span class="menu-sub-sep"> | </span><a href="http://morepypy.blogspot.com">Blog</a><span class="menu-sub-sep"> | </span><a href="people.html">People</a><span class="menu-sub-sep"> | </span><a href="contact.html">Contact</a><span class="menu-sub-sep"> | </span><a href="py3donate.html">Py3k donations</a></div>
+	<div id="menu-sub"><a href="index.html">Home</a><span class="menu-sub-sep"> | </span><a href="features.html">Features</a><span class="menu-sub-sep"> | </span><a href="download.html">Download</a><span class="menu-sub-sep"> | </span><a href="compat.html">Compatibility</a><span class="menu-sub-sep"> | </span><a href="http://speed.pypy.org">Performance</a><span class="menu-sub-sep"> | </span><a href="http://doc.pypy.org">Dev Documentation</a><span class="menu-sub-sep"> | </span><a href="http://morepypy.blogspot.com">Blog</a><span class="menu-sub-sep"> | </span><a href="people.html">People</a><span class="menu-sub-sep"> | </span><a href="contact.html">Contact</a><span class="menu-sub-sep"> | </span><a href="py3donate.html">Py3k donations</a><span class="menu-sub-sep"> | </span><a href="numpydonate.html">NumPy donations</a></div>
 	<hr class="clear" />
 </div>
 <div id="content">
diff --git a/howtohelp.html b/howtohelp.html
--- a/howtohelp.html
+++ b/howtohelp.html
@@ -38,7 +38,7 @@
 	</div>
 	<div id="logo"><a href="http://pypy.org"><img src="image/pypy-logo.png" alt="PyPy" height="110px" /></a></div>
 	<hr class="clear-left" />
-	<div id="menu-sub"><a href="index.html">Home</a><span class="menu-sub-sep"> | </span><a href="features.html">Features</a><span class="menu-sub-sep"> | </span><a href="download.html">Download</a><span class="menu-sub-sep"> | </span><a href="compat.html">Compatibility</a><span class="menu-sub-sep"> | </span><a href="http://speed.pypy.org">Performance</a><span class="menu-sub-sep"> | </span><a href="http://doc.pypy.org">Dev Documentation</a><span class="menu-sub-sep"> | </span><a href="http://morepypy.blogspot.com">Blog</a><span class="menu-sub-sep"> | </span><a href="people.html">People</a><span class="menu-sub-sep"> | </span><a href="contact.html">Contact</a><span class="menu-sub-sep"> | </span><a href="py3donate.html">Py3k donations</a></div>
+	<div id="menu-sub"><a href="index.html">Home</a><span class="menu-sub-sep"> | </span><a href="features.html">Features</a><span class="menu-sub-sep"> | </span><a href="download.html">Download</a><span class="menu-sub-sep"> | </span><a href="compat.html">Compatibility</a><span class="menu-sub-sep"> | </span><a href="http://speed.pypy.org">Performance</a><span class="menu-sub-sep"> | </span><a href="http://doc.pypy.org">Dev Documentation</a><span class="menu-sub-sep"> | </span><a href="http://morepypy.blogspot.com">Blog</a><span class="menu-sub-sep"> | </span><a href="people.html">People</a><span class="menu-sub-sep"> | </span><a href="contact.html">Contact</a><span class="menu-sub-sep"> | </span><a href="py3donate.html">Py3k donations</a><span class="menu-sub-sep"> | </span><a href="numpydonate.html">NumPy donations</a></div>
 	<hr class="clear" />
 </div>
 <div id="content">
diff --git a/index.html b/index.html
--- a/index.html
+++ b/index.html
@@ -38,7 +38,7 @@
 	</div>
 	<div id="logo"><a href="http://pypy.org"><img src="image/pypy-logo.png" alt="PyPy" height="110px" /></a></div>
 	<hr class="clear-left" />
-	<div id="menu-sub"><a href="index.html">Home</a><span class="menu-sub-sep"> | </span><a href="features.html">Features</a><span class="menu-sub-sep"> | </span><a href="download.html">Download</a><span class="menu-sub-sep"> | </span><a href="compat.html">Compatibility</a><span class="menu-sub-sep"> | </span><a href="http://speed.pypy.org">Performance</a><span class="menu-sub-sep"> | </span><a href="http://doc.pypy.org">Dev Documentation</a><span class="menu-sub-sep"> | </span><a href="http://morepypy.blogspot.com">Blog</a><span class="menu-sub-sep"> | </span><a href="people.html">People</a><span class="menu-sub-sep"> | </span><a href="contact.html">Contact</a><span class="menu-sub-sep"> | </span><a href="py3donate.html">Py3k donations</a></div>
+	<div id="menu-sub"><a href="index.html">Home</a><span class="menu-sub-sep"> | </span><a href="features.html">Features</a><span class="menu-sub-sep"> | </span><a href="download.html">Download</a><span class="menu-sub-sep"> | </span><a href="compat.html">Compatibility</a><span class="menu-sub-sep"> | </span><a href="http://speed.pypy.org">Performance</a><span class="menu-sub-sep"> | </span><a href="http://doc.pypy.org">Dev Documentation</a><span class="menu-sub-sep"> | </span><a href="http://morepypy.blogspot.com">Blog</a><span class="menu-sub-sep"> | </span><a href="people.html">People</a><span class="menu-sub-sep"> | </span><a href="contact.html">Contact</a><span class="menu-sub-sep"> | </span><a href="py3donate.html">Py3k donations</a><span class="menu-sub-sep"> | </span><a href="numpydonate.html">NumPy donations</a></div>
 	<hr class="clear" />
 </div>
 <div id="content">
diff --git a/numpydonate.html b/numpydonate.html
--- a/numpydonate.html
+++ b/numpydonate.html
@@ -38,7 +38,7 @@
 	</div>
 	<div id="logo"><a href="http://pypy.org"><img src="image/pypy-logo.png" alt="PyPy" height="110px" /></a></div>
 	<hr class="clear-left" />
-	<div id="menu-sub"><a href="index.html">Home</a><span class="menu-sub-sep"> | </span><a href="features.html">Features</a><span class="menu-sub-sep"> | </span><a href="download.html">Download</a><span class="menu-sub-sep"> | </span><a href="compat.html">Compatibility</a><span class="menu-sub-sep"> | </span><a href="http://speed.pypy.org">Performance</a><span class="menu-sub-sep"> | </span><a href="http://doc.pypy.org">Dev Documentation</a><span class="menu-sub-sep"> | </span><a href="http://morepypy.blogspot.com">Blog</a><span class="menu-sub-sep"> | </span><a href="people.html">People</a><span class="menu-sub-sep"> | </span><a href="contact.html">Contact</a><span class="menu-sub-sep"> | </span><a href="py3donate.html">Py3k donations</a></div>
+	<div id="menu-sub"><a href="index.html">Home</a><span class="menu-sub-sep"> | </span><a href="features.html">Features</a><span class="menu-sub-sep"> | </span><a href="download.html">Download</a><span class="menu-sub-sep"> | </span><a href="compat.html">Compatibility</a><span class="menu-sub-sep"> | </span><a href="http://speed.pypy.org">Performance</a><span class="menu-sub-sep"> | </span><a href="http://doc.pypy.org">Dev Documentation</a><span class="menu-sub-sep"> | </span><a href="http://morepypy.blogspot.com">Blog</a><span class="menu-sub-sep"> | </span><a href="people.html">People</a><span class="menu-sub-sep"> | </span><a href="contact.html">Contact</a><span class="menu-sub-sep"> | </span><a href="py3donate.html">Py3k donations</a><span class="menu-sub-sep"> | </span><a href="numpydonate.html">NumPy donations</a></div>
 	<hr class="clear" />
 </div>
 <div id="content">
diff --git a/people.html b/people.html
--- a/people.html
+++ b/people.html
@@ -38,7 +38,7 @@
 	</div>
 	<div id="logo"><a href="http://pypy.org"><img src="image/pypy-logo.png" alt="PyPy" height="110px" /></a></div>
 	<hr class="clear-left" />
-	<div id="menu-sub"><a href="index.html">Home</a><span class="menu-sub-sep"> | </span><a href="features.html">Features</a><span class="menu-sub-sep"> | </span><a href="download.html">Download</a><span class="menu-sub-sep"> | </span><a href="compat.html">Compatibility</a><span class="menu-sub-sep"> | </span><a href="http://speed.pypy.org">Performance</a><span class="menu-sub-sep"> | </span><a href="http://doc.pypy.org">Dev Documentation</a><span class="menu-sub-sep"> | </span><a href="http://morepypy.blogspot.com">Blog</a><span class="menu-sub-sep"> | </span><a href="people.html">People</a><span class="menu-sub-sep"> | </span><a href="contact.html">Contact</a><span class="menu-sub-sep"> | </span><a href="py3donate.html">Py3k donations</a></div>
+	<div id="menu-sub"><a href="index.html">Home</a><span class="menu-sub-sep"> | </span><a href="features.html">Features</a><span class="menu-sub-sep"> | </span><a href="download.html">Download</a><span class="menu-sub-sep"> | </span><a href="compat.html">Compatibility</a><span class="menu-sub-sep"> | </span><a href="http://speed.pypy.org">Performance</a><span class="menu-sub-sep"> | </span><a href="http://doc.pypy.org">Dev Documentation</a><span class="menu-sub-sep"> | </span><a href="http://morepypy.blogspot.com">Blog</a><span class="menu-sub-sep"> | </span><a href="people.html">People</a><span class="menu-sub-sep"> | </span><a href="contact.html">Contact</a><span class="menu-sub-sep"> | </span><a href="py3donate.html">Py3k donations</a><span class="menu-sub-sep"> | </span><a href="numpydonate.html">NumPy donations</a></div>
 	<hr class="clear" />
 </div>
 <div id="content">
diff --git a/py3donate.html b/py3donate.html
--- a/py3donate.html
+++ b/py3donate.html
@@ -38,7 +38,7 @@
 	</div>
 	<div id="logo"><a href="http://pypy.org"><img src="image/pypy-logo.png" alt="PyPy" height="110px" /></a></div>
 	<hr class="clear-left" />
-	<div id="menu-sub"><a href="index.html">Home</a><span class="menu-sub-sep"> | </span><a href="features.html">Features</a><span class="menu-sub-sep"> | </span><a href="download.html">Download</a><span class="menu-sub-sep"> | </span><a href="compat.html">Compatibility</a><span class="menu-sub-sep"> | </span><a href="http://speed.pypy.org">Performance</a><span class="menu-sub-sep"> | </span><a href="http://doc.pypy.org">Dev Documentation</a><span class="menu-sub-sep"> | </span><a href="http://morepypy.blogspot.com">Blog</a><span class="menu-sub-sep"> | </span><a href="people.html">People</a><span class="menu-sub-sep"> | </span><a href="contact.html">Contact</a><span class="menu-sub-sep"> | </span><a href="py3donate.html">Py3k donations</a></div>
+	<div id="menu-sub"><a href="index.html">Home</a><span class="menu-sub-sep"> | </span><a href="features.html">Features</a><span class="menu-sub-sep"> | </span><a href="download.html">Download</a><span class="menu-sub-sep"> | </span><a href="compat.html">Compatibility</a><span class="menu-sub-sep"> | </span><a href="http://speed.pypy.org">Performance</a><span class="menu-sub-sep"> | </span><a href="http://doc.pypy.org">Dev Documentation</a><span class="menu-sub-sep"> | </span><a href="http://morepypy.blogspot.com">Blog</a><span class="menu-sub-sep"> | </span><a href="people.html">People</a><span class="menu-sub-sep"> | </span><a href="contact.html">Contact</a><span class="menu-sub-sep"> | </span><a href="py3donate.html">Py3k donations</a><span class="menu-sub-sep"> | </span><a href="numpydonate.html">NumPy donations</a></div>
 	<hr class="clear" />
 </div>
 <div id="content">
diff --git a/source/_layouts/site.genshi b/source/_layouts/site.genshi
--- a/source/_layouts/site.genshi
+++ b/source/_layouts/site.genshi
@@ -14,6 +14,7 @@
       ('People', 'people.html'),
       ('Contact', 'contact.html'),
       ('Py3k donations', 'py3donate.html'),
+      ('NumPy donations', 'numpydonate.html'),
       ],
     }
 
diff --git a/sponsor.html b/sponsor.html
--- a/sponsor.html
+++ b/sponsor.html
@@ -38,7 +38,7 @@
 	</div>
 	<div id="logo"><a href="http://pypy.org"><img src="image/pypy-logo.png" alt="PyPy" height="110px" /></a></div>
 	<hr class="clear-left" />
-	<div id="menu-sub"><a href="index.html">Home</a><span class="menu-sub-sep"> | </span><a href="features.html">Features</a><span class="menu-sub-sep"> | </span><a href="download.html">Download</a><span class="menu-sub-sep"> | </span><a href="compat.html">Compatibility</a><span class="menu-sub-sep"> | </span><a href="http://speed.pypy.org">Performance</a><span class="menu-sub-sep"> | </span><a href="http://doc.pypy.org">Dev Documentation</a><span class="menu-sub-sep"> | </span><a href="http://morepypy.blogspot.com">Blog</a><span class="menu-sub-sep"> | </span><a href="people.html">People</a><span class="menu-sub-sep"> | </span><a href="contact.html">Contact</a><span class="menu-sub-sep"> | </span><a href="py3donate.html">Py3k donations</a></div>
+	<div id="menu-sub"><a href="index.html">Home</a><span class="menu-sub-sep"> | </span><a href="features.html">Features</a><span class="menu-sub-sep"> | </span><a href="download.html">Download</a><span class="menu-sub-sep"> | </span><a href="compat.html">Compatibility</a><span class="menu-sub-sep"> | </span><a href="http://speed.pypy.org">Performance</a><span class="menu-sub-sep"> | </span><a href="http://doc.pypy.org">Dev Documentation</a><span class="menu-sub-sep"> | </span><a href="http://morepypy.blogspot.com">Blog</a><span class="menu-sub-sep"> | </span><a href="people.html">People</a><span class="menu-sub-sep"> | </span><a href="contact.html">Contact</a><span class="menu-sub-sep"> | </span><a href="py3donate.html">Py3k donations</a><span class="menu-sub-sep"> | </span><a href="numpydonate.html">NumPy donations</a></div>
 	<hr class="clear" />
 </div>
 <div id="content">
diff --git a/success.html b/success.html
--- a/success.html
+++ b/success.html
@@ -38,7 +38,7 @@
 	</div>
 	<div id="logo"><a href="http://pypy.org"><img src="image/pypy-logo.png" alt="PyPy" height="110px" /></a></div>
 	<hr class="clear-left" />
-	<div id="menu-sub"><a href="index.html">Home</a><span class="menu-sub-sep"> | </span><a href="features.html">Features</a><span class="menu-sub-sep"> | </span><a href="download.html">Download</a><span class="menu-sub-sep"> | </span><a href="compat.html">Compatibility</a><span class="menu-sub-sep"> | </span><a href="http://speed.pypy.org">Performance</a><span class="menu-sub-sep"> | </span><a href="http://doc.pypy.org">Dev Documentation</a><span class="menu-sub-sep"> | </span><a href="http://morepypy.blogspot.com">Blog</a><span class="menu-sub-sep"> | </span><a href="people.html">People</a><span class="menu-sub-sep"> | </span><a href="contact.html">Contact</a><span class="menu-sub-sep"> | </span><a href="py3donate.html">Py3k donations</a></div>
+	<div id="menu-sub"><a href="index.html">Home</a><span class="menu-sub-sep"> | </span><a href="features.html">Features</a><span class="menu-sub-sep"> | </span><a href="download.html">Download</a><span class="menu-sub-sep"> | </span><a href="compat.html">Compatibility</a><span class="menu-sub-sep"> | </span><a href="http://speed.pypy.org">Performance</a><span class="menu-sub-sep"> | </span><a href="http://doc.pypy.org">Dev Documentation</a><span class="menu-sub-sep"> | </span><a href="http://morepypy.blogspot.com">Blog</a><span class="menu-sub-sep"> | </span><a href="people.html">People</a><span class="menu-sub-sep"> | </span><a href="contact.html">Contact</a><span class="menu-sub-sep"> | </span><a href="py3donate.html">Py3k donations</a><span class="menu-sub-sep"> | </span><a href="numpydonate.html">NumPy donations</a></div>
 	<hr class="clear" />
 </div>
 <div id="content">

From noreply at buildbot.pypy.org  Thu Oct 13 18:59:43 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 13 Oct 2011 18:59:43 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix for "-Ojit --gc=boehm" translations.
Message-ID: <20111013165943.7C9B382112@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48015:d296fda58621
Date: 2011-10-13 18:59 +0200
http://bitbucket.org/pypy/pypy/changeset/d296fda58621/

Log:	Fix for "-Ojit --gc=boehm" translations.

diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py
--- a/pypy/jit/backend/llsupport/gc.py
+++ b/pypy/jit/backend/llsupport/gc.py
@@ -45,6 +45,14 @@
     def freeing_block(self, start, stop):
         pass
 
+    def record_constptrs(self, op, gcrefs_output_list):
+        for i in range(op.numargs()):
+            v = op.getarg(i)
+            if isinstance(v, ConstPtr) and bool(v.value):
+                p = v.value
+                rgc._make_sure_does_not_move(p)
+                gcrefs_output_list.append(p)
+
 # ____________________________________________________________
 
 class GcLLDescr_boehm(GcLLDescription):
@@ -141,6 +149,14 @@
     get_funcptr_for_newstr = None
     get_funcptr_for_newunicode = None
 
+    def rewrite_assembler(self, cpu, operations, gcrefs_output_list):
+        # record all GCREFs too, because Boehm cannot see them and keep them
+        # alive if they end up as constants in the assembler
+        for op in operations:
+            self.record_constptrs(op, gcrefs_output_list)
+        return GcLLDescription.rewrite_assembler(self, cpu, operations,
+                                                 gcrefs_output_list)
+
 
 # ____________________________________________________________
 # All code below is for the hybrid or minimark GC
@@ -757,14 +773,6 @@
             funcptr(llmemory.cast_ptr_to_adr(gcref_struct),
                     llmemory.cast_ptr_to_adr(gcref_newptr))
 
-    def record_constptrs(self, op, gcrefs_output_list):
-        for i in range(op.numargs()):
-            v = op.getarg(i)
-            if isinstance(v, ConstPtr) and bool(v.value):
-                p = v.value
-                rgc._make_sure_does_not_move(p)
-                gcrefs_output_list.append(p)
-
     def rewrite_assembler(self, cpu, operations, gcrefs_output_list):
         # Perform two kinds of rewrites in parallel:
         #

From noreply at buildbot.pypy.org  Thu Oct 13 21:59:03 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 13 Oct 2011 21:59:03 +0200 (CEST)
Subject: [pypy-commit] pypy string-promote-2: write a test_pypy_c
Message-ID: <20111013195903.A2AB182112@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: string-promote-2
Changeset: r48016:74aea6b312d3
Date: 2011-10-13 21:58 +0200
http://bitbucket.org/pypy/pypy/changeset/74aea6b312d3/

Log:	write a test_pypy_c

diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -156,4 +156,41 @@
             i40 = int_sub(i4, 1)
             --TICK--
             jump(p0, p1, p2, p3, i40, i38, descr=<Loop0>)
-        """)
\ No newline at end of file
+        """)
+
+    def test_getattr_promote(self):
+        def main(n):
+            class A(object):
+                def meth_a(self):
+                    return 1
+                def meth_b(self):
+                    return 2
+            a = A()
+
+            l = ['a', 'b']
+            s = 0
+            for i in range(n):
+                name = 'meth_' + l[i & 1]
+                meth = getattr(a, name) # ID: getattr
+                s += meth()
+            return s
+
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loops = log.loops_by_filename(self.filepath)
+        assert len(loops) == 2
+        for loop in loops:
+            loop.match_by_id('getattr','''
+            guard_not_invalidated(descr=...)
+            i32 = strlen(p31)
+            i34 = int_add(5, i32)
+            p35 = newstr(i34)
+            strsetitem(p35, 0, 109)
+            strsetitem(p35, 1, 101)
+            strsetitem(p35, 2, 116)
+            strsetitem(p35, 3, 104)
+            strsetitem(p35, 4, 95)
+            copystrcontent(p31, p35, 0, 5, i32)
+            i49 = call(ConstClass(_ll_2_str_eq_nonnull__rpy_stringPtr_rpy_stringPtr), p35, ConstPtr(ptr48), descr=<SignedCallDescr>)
+            guard_value(i49, 1, descr=<Guard8>)
+            ''')

From noreply at buildbot.pypy.org  Thu Oct 13 22:01:57 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 13 Oct 2011 22:01:57 +0200 (CEST)
Subject: [pypy-commit] pypy default: (fijal,
 reviewed by agaynor) Merge string-promote-2 branch. This branch lets
Message-ID: <20111013200157.28EE082112@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r48017:85ccfa8d2773
Date: 2011-10-13 22:01 +0200
http://bitbucket.org/pypy/pypy/changeset/85ccfa8d2773/

Log:	(fijal, reviewed by agaynor) Merge string-promote-2 branch. This
	branch lets you promote strings by value by using
	jit.promote_string(s). It's used in typeobject for promoting getattr
	on types.

diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -455,6 +455,23 @@
             # the special return value None forces op.result to be considered
             # equal to op.args[0]
             return [op0, op1, None]
+        if (hints.get('promote_string') and
+            op.args[0].concretetype is not lltype.Void):
+            S = lltype.Ptr(rstr.STR)
+            assert op.args[0].concretetype == S
+            self._register_extra_helper(EffectInfo.OS_STREQ_NONNULL,
+                                        "str.eq_nonnull",
+                                        [S, S],
+                                        lltype.Signed,
+                                        EffectInfo.EF_ELIDABLE_CANNOT_RAISE)
+            descr, p = self.callcontrol.callinfocollection.callinfo_for_oopspec(
+                EffectInfo.OS_STREQ_NONNULL)
+            # XXX this is fairly ugly way of creating a constant,
+            #     however, callinfocollection has no better interface
+            c = Constant(p.adr.ptr, lltype.typeOf(p.adr.ptr))
+            op1 = SpaceOperation('str_guard_value', [op.args[0], c, descr],
+                                 op.result)
+            return [SpaceOperation('-live-', [], None), op1, None]
         else:
             log.WARNING('ignoring hint %r at %r' % (hints, self.graph))
 
diff --git a/pypy/jit/codewriter/test/test_jtransform.py b/pypy/jit/codewriter/test/test_jtransform.py
--- a/pypy/jit/codewriter/test/test_jtransform.py
+++ b/pypy/jit/codewriter/test/test_jtransform.py
@@ -99,6 +99,12 @@
             if i == oopspecindex:
                 return True
         return False
+    def callinfo_for_oopspec(self, oopspecindex):
+        assert oopspecindex == effectinfo.EffectInfo.OS_STREQ_NONNULL
+        class c:
+            class adr:
+                ptr = 1
+        return ('calldescr', c)
 
 class FakeBuiltinCallControl:
     def __init__(self):
@@ -119,6 +125,7 @@
              EI.OS_STR2UNICODE:([PSTR], PUNICODE),
              EI.OS_STR_CONCAT: ([PSTR, PSTR], PSTR),
              EI.OS_STR_SLICE:  ([PSTR, INT, INT], PSTR),
+             EI.OS_STREQ_NONNULL:  ([PSTR, PSTR], INT),
              EI.OS_UNI_CONCAT: ([PUNICODE, PUNICODE], PUNICODE),
              EI.OS_UNI_SLICE:  ([PUNICODE, INT, INT], PUNICODE),
              EI.OS_UNI_EQUAL:  ([PUNICODE, PUNICODE], lltype.Bool),
@@ -844,6 +851,21 @@
     assert op1.args[2] == ListOfKind('ref', [v1, v2])
     assert op1.result == v3
 
+def test_str_promote():
+    PSTR = lltype.Ptr(rstr.STR)
+    v1 = varoftype(PSTR)
+    v2 = varoftype(PSTR)
+    op = SpaceOperation('hint',
+                        [v1, Constant({'promote_string': True}, lltype.Void)],
+                        v2)
+    tr = Transformer(FakeCPU(), FakeBuiltinCallControl())
+    op0, op1, _ = tr.rewrite_operation(op)
+    assert op1.opname == 'str_guard_value'
+    assert op1.args[0] == v1
+    assert op1.args[2] == 'calldescr'
+    assert op1.result == v2
+    assert op0.opname == '-live-'
+
 def test_unicode_concat():
     # test that the oopspec is present and correctly transformed
     PSTR = lltype.Ptr(rstr.UNICODE)
diff --git a/pypy/jit/metainterp/blackhole.py b/pypy/jit/metainterp/blackhole.py
--- a/pypy/jit/metainterp/blackhole.py
+++ b/pypy/jit/metainterp/blackhole.py
@@ -523,6 +523,9 @@
     @arguments("f")
     def bhimpl_float_guard_value(a):
         pass
+    @arguments("r", "i", "d", returns="r")
+    def bhimpl_str_guard_value(a, i, d):
+        return a
 
     @arguments("self", "i")
     def bhimpl_int_push(self, a):
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -168,11 +168,12 @@
 
     def make_result_of_lastop(self, resultbox):
         got_type = resultbox.type
-        if not we_are_translated():
-            typeof = {'i': history.INT,
-                      'r': history.REF,
-                      'f': history.FLOAT}
-            assert typeof[self.jitcode._resulttypes[self.pc]] == got_type
+        # XXX disabled for now, conflicts with str_guard_value
+        #if not we_are_translated():
+        #    typeof = {'i': history.INT,
+        #              'r': history.REF,
+        #              'f': history.FLOAT}
+        #    assert typeof[self.jitcode._resulttypes[self.pc]] == got_type
         target_index = ord(self.bytecode[self.pc-1])
         if got_type == history.INT:
             self.registers_i[target_index] = resultbox
@@ -897,6 +898,21 @@
     def _opimpl_guard_value(self, orgpc, box):
         self.implement_guard_value(orgpc, box)
 
+    @arguments("orgpc", "box", "box", "descr")
+    def opimpl_str_guard_value(self, orgpc, box, funcbox, descr):
+        if isinstance(box, Const):
+            return box     # no promotion needed, already a Const
+        else:
+            constbox = box.constbox()
+            resbox = self.do_residual_call(funcbox, descr, [box, constbox])
+            promoted_box = resbox.constbox()
+            # This is GUARD_VALUE because GUARD_TRUE assumes the existance
+            # of a label when computing resumepc
+            self.generate_guard(rop.GUARD_VALUE, resbox, [promoted_box],
+                                resumepc=orgpc)
+            self.metainterp.replace_box(box, constbox)
+            return constbox
+
     opimpl_int_guard_value = _opimpl_guard_value
     opimpl_ref_guard_value = _opimpl_guard_value
     opimpl_float_guard_value = _opimpl_guard_value
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -13,7 +13,7 @@
 from pypy.rlib.jit import (JitDriver, we_are_jitted, hint, dont_look_inside,
     loop_invariant, elidable, promote, jit_debug, assert_green,
     AssertGreenFailed, unroll_safe, current_trace_length, look_inside_iff,
-    isconstant, isvirtual)
+    isconstant, isvirtual, promote_string)
 from pypy.rlib.rarithmetic import ovfcheck
 from pypy.rpython.lltypesystem import lltype, llmemory, rffi
 from pypy.rpython.ootypesystem import ootype
diff --git a/pypy/jit/metainterp/test/test_string.py b/pypy/jit/metainterp/test/test_string.py
--- a/pypy/jit/metainterp/test/test_string.py
+++ b/pypy/jit/metainterp/test/test_string.py
@@ -3,7 +3,8 @@
 from pypy.jit.codewriter.policy import StopAtXPolicy
 from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin
 from pypy.rlib.debug import debug_print
-from pypy.rlib.jit import JitDriver, dont_look_inside, we_are_jitted
+from pypy.rlib.jit import JitDriver, dont_look_inside, we_are_jitted,\
+     promote_string
 from pypy.rlib.rstring import StringBuilder
 from pypy.rpython.ootypesystem import ootype
 
@@ -507,6 +508,18 @@
                           'jump': 1, 'int_is_true': 1,
                           'guard_not_invalidated': 1})
 
+    def test_promote_string(self):
+        driver = JitDriver(greens = [], reds = ['n'])
+        
+        def f(n):
+            while n < 21:
+                driver.jit_merge_point(n=n)
+                promote_string(str(n % 3))
+                n += 1
+            return 0
+
+        self.meta_interp(f, [0])
+        self.check_loops(call=3 + 1) # one for int2str
 
 #class TestOOtype(StringTests, OOJitMixin):
 #    CALL = "oosend"
diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -156,4 +156,41 @@
             i40 = int_sub(i4, 1)
             --TICK--
             jump(p0, p1, p2, p3, i40, i38, descr=<Loop0>)
-        """)
\ No newline at end of file
+        """)
+
+    def test_getattr_promote(self):
+        def main(n):
+            class A(object):
+                def meth_a(self):
+                    return 1
+                def meth_b(self):
+                    return 2
+            a = A()
+
+            l = ['a', 'b']
+            s = 0
+            for i in range(n):
+                name = 'meth_' + l[i & 1]
+                meth = getattr(a, name) # ID: getattr
+                s += meth()
+            return s
+
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loops = log.loops_by_filename(self.filepath)
+        assert len(loops) == 2
+        for loop in loops:
+            loop.match_by_id('getattr','''
+            guard_not_invalidated(descr=...)
+            i32 = strlen(p31)
+            i34 = int_add(5, i32)
+            p35 = newstr(i34)
+            strsetitem(p35, 0, 109)
+            strsetitem(p35, 1, 101)
+            strsetitem(p35, 2, 116)
+            strsetitem(p35, 3, 104)
+            strsetitem(p35, 4, 95)
+            copystrcontent(p31, p35, 0, 5, i32)
+            i49 = call(ConstClass(_ll_2_str_eq_nonnull__rpy_stringPtr_rpy_stringPtr), p35, ConstPtr(ptr48), descr=<SignedCallDescr>)
+            guard_value(i49, 1, descr=<Guard8>)
+            ''')
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -10,7 +10,8 @@
 from pypy.objspace.std import identitydict
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.rlib.objectmodel import current_object_addr_as_int, compute_hash
-from pypy.rlib.jit import promote, elidable_promote, we_are_jitted
+from pypy.rlib.jit import promote, elidable_promote, we_are_jitted,\
+     promote_string
 from pypy.rlib.jit import elidable, dont_look_inside, unroll_safe
 from pypy.rlib.rarithmetic import intmask, r_uint
 
@@ -399,6 +400,7 @@
         if version_tag is None:
             tup = w_self._lookup_where(name)
             return tup
+        name = promote_string(name)
         w_class, w_value = w_self._pure_lookup_where_with_method_cache(name, version_tag)
         return w_class, unwrap_cell(space, w_value)
 
diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -38,6 +38,7 @@
     possible arguments are:
 
     * promote - promote the argument from a variable into a constant
+    * promote_string - same, but promote string by *value*
     * access_directly - directly access a virtualizable, as a structure
                         and don't treat it as a virtualizable
     * fresh_virtualizable - means that virtualizable was just allocated.
@@ -51,6 +52,9 @@
 def promote(x):
     return hint(x, promote=True)
 
+def promote_string(x):
+    return hint(x, promote_string=True)
+
 def dont_look_inside(func):
     """ Make sure the JIT does not trace inside decorated function
     (it becomes a call instead)

From noreply at buildbot.pypy.org  Thu Oct 13 22:02:49 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 13 Oct 2011 22:02:49 +0200 (CEST)
Subject: [pypy-commit] pypy string-promote-2: close merged branch
Message-ID: <20111013200249.47F5782112@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: string-promote-2
Changeset: r48018:f3b267864355
Date: 2011-10-13 22:02 +0200
http://bitbucket.org/pypy/pypy/changeset/f3b267864355/

Log:	close merged branch


From noreply at buildbot.pypy.org  Thu Oct 13 22:40:10 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 13 Oct 2011 22:40:10 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Fix test_newformat: bytes.format always
 return a unicode string
Message-ID: <20111013204010.96FA282112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48019:8e1f509bd8dd
Date: 2011-10-13 22:37 +0200
http://bitbucket.org/pypy/pypy/changeset/8e1f509bd8dd/

Log:	Fix test_newformat: bytes.format always return a unicode string

diff --git a/pypy/objspace/std/test/test_newformat.py b/pypy/objspace/std/test/test_newformat.py
--- a/pypy/objspace/std/test/test_newformat.py
+++ b/pypy/objspace/std/test/test_newformat.py
@@ -7,18 +7,17 @@
     """Test format and __format__ methods of string objects."""
 
     def test_escape(self):
-        assert self.s("{{").format() == self.s("{")
-        assert self.s("}}").format() == self.s("}")
-        assert self.s("{} {{ {}").format(1, 2) == self.s("1 { 2")
-        assert self.s("{{}}").format() == self.s("{}")
+        assert self.s("{{").format() == "{"
+        assert self.s("}}").format() == "}"
+        assert self.s("{} {{ {}").format(1, 2) == "1 { 2"
+        assert self.s("{{}}").format() == "{}"
 
     def test_empty(self):
-        assert self.s().format() == self.s()
-        assert self.s("x").format() == self.s("x")
+        assert self.s().format() == ""
+        assert self.s("x").format() == "x"
 
     def test_several(self):
-        res = self.s("x32 stuff 42m")
-        assert self.s("x{} stuff {}m").format(32, 42) == res
+        assert self.s("x{} stuff {}m").format(32, 42) == "x32 stuff 42m"
 
     def test_stray_brackets(self):
         raises(ValueError, self.s("}").format, 3)
@@ -26,44 +25,43 @@
         raises(ValueError, self.s("{}}").format, 3)
 
     def test_auto_numbering(self):
-        res = "1 3 2"
-        assert self.s("{} {name} {}").format(1, 2, name=3) == self.s(res)
+        assert self.s("{} {name} {}").format(1, 2, name=3) == "1 3 2"
         raises(ValueError, self.s("{} {2}").format, 2, 3)
         raises(ValueError, self.s("{0} {}").format, 2, 3)
 
     def test_positional_args(self):
-        assert self.s("{1}{0}").format(2, 3) == self.s("32")
+        assert self.s("{1}{0}").format(2, 3) == "32"
         raises(IndexError, self.s("{2}").format, 2)
         big = self.s("{123476028570192873049182730984172039840712934}")
         raises(ValueError, big.format)
 
     def test_kwargs(self):
-        assert self.s("{what}").format(what=42) == self.s("42")
+        assert self.s("{what}").format(what=42) == "42"
         raises(KeyError, self.s("{nothing}").format)
 
     def test_attr(self):
         class x:
             apple = 42
-        assert self.s("{.apple}").format(x) == self.s("42")
+        assert self.s("{.apple}").format(x) == "42"
         #
         raises(ValueError, self.s("{.}").format, x)
 
     def test_index(self):
         seq = (1, 42)
-        assert self.s("{[1]}").format(seq) == self.s("42")
+        assert self.s("{[1]}").format(seq) == "42"
         big = self.s("{[1092837041982035981720398471029384012937412]}")
         raises(ValueError, big.format, [0])
 
     def test_getitem(self):
         d = {"hi" : 32}
-        assert self.s("{[hi]}").format(d) == self.s("32")
+        assert self.s("{[hi]}").format(d) == "32"
 
     def test_chained(self):
         class x:
             y = [1, 2, 3]
-        assert self.s("{.y[1]}").format(x) == self.s("2")
+        assert self.s("{.y[1]}").format(x) == "2"
         l = [1, x]
-        assert self.s("{[1].y[2]}").format(l) == self.s("3")
+        assert self.s("{[1].y[2]}").format(l) == "3"
 
     def test_invalid_field_name(self):
         raises(ValueError, self.s("{[x]y}").format, {"x" : 2})
@@ -72,8 +70,8 @@
         class x(object):
             def __repr__(self):
                 return "32"
-        assert self.s("{!r}").format(x()) == self.s("32")
-        assert self.s("{!s}").format(x()) == self.s("32")
+        assert self.s("{!r}").format(x()) == "32"
+        assert self.s("{!s}").format(x()) == "32"
 
     def test_format_spec(self):
         assert self.s('{0!s:}').format('Hello') == 'Hello'
@@ -87,27 +85,27 @@
         raises(ValueError, self.s("{!}").format)
 
     def test_recursive(self):
-        assert self.s("{:{}}").format(42, "#o") == self.s("0o52")
+        assert self.s("{:{}}").format(42, "#o") == "0o52"
         raises(ValueError, self.s("{{}:s}").format)
         raises(ValueError, self.s("{:{:{}}}").format, 1, 2, 3)
 
     def test_presentation(self):
-        assert format(self.s("blah"), "s") == self.s("blah")
-        assert format(self.s("blah")) == self.s("blah")
+        assert format(self.s("blah"), "s") == "blah"
+        assert format(self.s("blah")) == "blah"
         for pres in "bcdoxXeEfFgGn%":
             raises(ValueError, format, self.s("blah"), pres)
 
     def test_padding(self):
-        assert format(self.s("h"), "3") == self.s("h  ")
-        assert format(self.s("h"), "<3") == self.s("h  ")
-        assert format(self.s("h"), ">3") == self.s("  h")
-        assert format(self.s("h"), "^3") == self.s(" h ")
-        assert format(self.s("h"), "^4") == self.s(" h  ")
-        assert format(self.s("h"), "c<3") == self.s("hcc")
+        assert format(self.s("h"), "3") == "h  "
+        assert format(self.s("h"), "<3") == "h  "
+        assert format(self.s("h"), ">3") == "  h"
+        assert format(self.s("h"), "^3") == " h "
+        assert format(self.s("h"), "^4") == " h  "
+        assert format(self.s("h"), "c<3") == "hcc"
         raises(ValueError, format, self.s("blah"), "=12")
 
     def test_precision(self):
-        assert format(self.s("abcdef"), ".3") == self.s("abc")
+        assert format(self.s("abcdef"), ".3") == "abc"
 
     def test_non_ascii_presentation(self):
         raises(ValueError, format, self.s(""), "\x234")
@@ -140,26 +138,26 @@
                     return 'G(' + self.x + ')'
                 return object.__format__(self, format_spec)
 
-        assert self.s("{1}{0}").format(D(10), D(20)) == self.s("2010")
-        assert self.s("{0._x.x}").format(C(D("abc"))) == self.s("abc")
-        assert self.s("{0[1][0].x}").format(["abc", [D("def")]]) == self.s("def")
-        assert self.s("{0}").format(E("data")) == self.s("E(data)")
-        assert self.s("{0:d}").format(G("data")) == self.s("G(data)")
-        assert self.s("{0!s}").format(G("data")) == self.s("string is data")
+        assert self.s("{1}{0}").format(D(10), D(20)) == "2010"
+        assert self.s("{0._x.x}").format(C(D("abc"))) == "abc"
+        assert self.s("{0[1][0].x}").format(["abc", [D("def")]]) == "def"
+        assert self.s("{0}").format(E("data")) == "E(data)"
+        assert self.s("{0:d}").format(G("data")) == "G(data)"
+        assert self.s("{0!s}").format(G("data")) == "string is data"
 
         msg = "object.__format__ with a non-empty format string is deprecated",
         with warnings.catch_warnings(record=True) as log:
             # This is ok because warnings.catch_warnings resets the filters
             warnings.simplefilter("always", PendingDeprecationWarning)
-            assert self.s("{0:^10}").format(E("data")) == self.s(" E(data)  ")
+            assert self.s("{0:^10}").format(E("data")) == " E(data)  "
             assert log[0].message.args == msg
             assert type(log[0].message) is PendingDeprecationWarning
 
-            assert self.s("{0:^10s}").format(E("data")) == self.s(" E(data)  ")
+            assert self.s("{0:^10s}").format(E("data")) == " E(data)  "
             assert log[1].message.args == msg
             assert type(log[1].message) is PendingDeprecationWarning
 
-            assert self.s("{0:>15s}").format(G("data")) == self.s(" string is data")
+            assert self.s("{0:>15s}").format(G("data")) == " string is data"
             assert log[2].message.args == msg
             assert type(log[2].message) is PendingDeprecationWarning
         assert len(log) == 3
@@ -183,8 +181,8 @@
                 return "18"
             def __unicode__(self):
                 return "42"
-        assert self.s("{!s}").format(x()) == self.s("42")
-        assert self.s("{!r}").format(x()) == self.s("32")
+        assert self.s("{!s}").format(x()) == "42"
+        assert self.s("{!r}").format(x()) == "32"
 
     def test_non_latin1_key(self):
         raises(KeyError, self.s("{\u1000}").format)
@@ -204,8 +202,8 @@
                 return "18"
             def __unicode__(self):
                 return "42"
-        assert self.s("{!s}").format(x()) == self.s("18")
-        assert self.s("{!r}").format(x()) == self.s("32")
+        assert self.s("{!s}").format(x()) == "18"
+        assert self.s("{!r}").format(x()) == "32"
 
 
 class BaseIntegralFormattingTest:

From noreply at buildbot.pypy.org  Thu Oct 13 22:40:11 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 13 Oct 2011 22:40:11 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Remove the u in front of str.__repr__
Message-ID: <20111013204011.D929482A88@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48020:76589f214f51
Date: 2011-10-13 22:37 +0200
http://bitbucket.org/pypy/pypy/changeset/76589f214f51/

Log:	Remove the u in front of str.__repr__

diff --git a/pypy/rlib/runicode.py b/pypy/rlib/runicode.py
--- a/pypy/rlib/runicode.py
+++ b/pypy/rlib/runicode.py
@@ -1212,10 +1212,10 @@
     if quotes:
         if s.find(u'\'') != -1 and s.find(u'\"') == -1:
             quote = ord('\"')
-            result.append('u"')
+            result.append('"')
         else:
             quote = ord('\'')
-            result.append('u\'')
+            result.append('\'')
     else:
         quote = 0
 

From noreply at buildbot.pypy.org  Thu Oct 13 22:40:13 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 13 Oct 2011 22:40:13 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: The methods str._formatter_parser and
 formatter_field_name_split have moved to a new _string module.
Message-ID: <20111013204013.1CA8282112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48021:ec3004f1eb9e
Date: 2011-10-13 22:37 +0200
http://bitbucket.org/pypy/pypy/changeset/ec3004f1eb9e/

Log:	The methods str._formatter_parser and formatter_field_name_split
	have moved to a new _string module.

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -21,7 +21,7 @@
     ["_codecs", "gc", "_weakref", "marshal", "errno", "imp",
      "math", "cmath", "_sre", "_pickle_support", "operator",
      "parser", "symbol", "token", "_ast",  "_io", "_random", "__pypy__",
-     "_testing"]))
+     "_string", "_testing"]))
 
 
 # --allworkingmodules
diff --git a/pypy/module/_string/__init__.py b/pypy/module/_string/__init__.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_string/__init__.py
@@ -0,0 +1,17 @@
+"""A _string module, to export formatter_parser and
+   formatter_field_name_split to the string.Formatter class
+   implemented in Python."""
+
+
+from pypy.interpreter.mixedmodule import MixedModule
+
+class Module(MixedModule):
+    "string helper module"
+
+    interpleveldefs = {
+        'formatter_field_name_split': 'formatter.formatter_field_name_split',
+        'formatter_parser': 'formatter.formatter_parser',
+    }
+
+    appleveldefs = {}
+
diff --git a/pypy/module/_string/formatter.py b/pypy/module/_string/formatter.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_string/formatter.py
@@ -0,0 +1,10 @@
+def formatter_parser(space, w_unicode):
+    from pypy.objspace.std.newformat import unicode_template_formatter
+    tformat = unicode_template_formatter(space, space.unicode_w(w_unicode))
+    return tformat.formatter_parser()
+
+def formatter_field_name_split(space, w_unicode):
+    from pypy.objspace.std.newformat import unicode_template_formatter
+    tformat = unicode_template_formatter(space, space.unicode_w(w_unicode))
+    return tformat.formatter_field_name_split()
+
diff --git a/pypy/objspace/std/stringtype.py b/pypy/objspace/std/stringtype.py
--- a/pypy/objspace/std/stringtype.py
+++ b/pypy/objspace/std/stringtype.py
@@ -272,19 +272,6 @@
                          ' with\ncodecs.register_error that is able to handle'
                          ' UnicodeEncodeErrors.')
 
-str_formatter_parser           = SMM('_formatter_parser', 1)
-str_formatter_field_name_split = SMM('_formatter_field_name_split', 1)
-
-def str_formatter_parser__ANY(space, w_str):
-    from pypy.objspace.std.newformat import str_template_formatter
-    tformat = str_template_formatter(space, space.str_w(w_str))
-    return tformat.formatter_parser()
-
-def str_formatter_field_name_split__ANY(space, w_str):
-    from pypy.objspace.std.newformat import str_template_formatter
-    tformat = str_template_formatter(space, space.str_w(w_str))
-    return tformat.formatter_field_name_split()
-
 register_all(vars(), globals())
 
 # ____________________________________________________________
diff --git a/pypy/objspace/std/unicodetype.py b/pypy/objspace/std/unicodetype.py
--- a/pypy/objspace/std/unicodetype.py
+++ b/pypy/objspace/std/unicodetype.py
@@ -151,19 +151,6 @@
                              ' field\nof the specified width. The string x is'
                              ' never truncated.')
 
-unicode_formatter_parser           = SMM('_formatter_parser', 1)
-unicode_formatter_field_name_split = SMM('_formatter_field_name_split', 1)
-
-def unicode_formatter_parser__ANY(space, w_unicode):
-    from pypy.objspace.std.newformat import unicode_template_formatter
-    tformat = unicode_template_formatter(space, space.unicode_w(w_unicode))
-    return tformat.formatter_parser()
-
-def unicode_formatter_field_name_split__ANY(space, w_unicode):
-    from pypy.objspace.std.newformat import unicode_template_formatter
-    tformat = unicode_template_formatter(space, space.unicode_w(w_unicode))
-    return tformat.formatter_field_name_split()
-
 # stuff imported from stringtype for interoperability
 
 from pypy.objspace.std.stringtype import str_endswith as unicode_endswith

From noreply at buildbot.pypy.org  Thu Oct 13 22:40:14 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 13 Oct 2011 22:40:14 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: The atexit module now replaces
	sys.exitfunc.
Message-ID: <20111013204014.5D55182112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48022:fac760084b0d
Date: 2011-10-13 22:38 +0200
http://bitbucket.org/pypy/pypy/changeset/fac760084b0d/

Log:	The atexit module now replaces sys.exitfunc.

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -18,7 +18,7 @@
 
 default_modules = essential_modules.copy()
 default_modules.update(dict.fromkeys(
-    ["_codecs", "gc", "_weakref", "marshal", "errno", "imp",
+    ["_codecs", "atexit", "gc", "_weakref", "marshal", "errno", "imp",
      "math", "cmath", "_sre", "_pickle_support", "operator",
      "parser", "symbol", "token", "_ast",  "_io", "_random", "__pypy__",
      "_string", "_testing"]))
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -330,9 +330,8 @@
 
     def finish(self):
         self.wait_for_thread_shutdown()
-        w_exitfunc = self.sys.getdictvalue(self, 'exitfunc')
-        if w_exitfunc is not None:
-            self.call_function(w_exitfunc)
+        w_atexit = self.getbuiltinmodule('atexit')
+        self.call_method(w_atexit, '_run_exitfuncs')
         from pypy.interpreter.module import Module
         for w_mod in self.builtin_modules.values():
             mod = self.interpclass_w(w_mod)
diff --git a/pypy/interpreter/module.py b/pypy/interpreter/module.py
--- a/pypy/interpreter/module.py
+++ b/pypy/interpreter/module.py
@@ -57,7 +57,7 @@
 
     def shutdown(self, space):
         """This is called when the space is shut down, just after
-        sys.exitfunc(), if the module has been imported.
+        atexit functions, if the module has been imported.
         """
 
     def getdict(self, space):
diff --git a/pypy/module/atexit/__init__.py b/pypy/module/atexit/__init__.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/atexit/__init__.py
@@ -0,0 +1,24 @@
+"""A _string module, to export formatter_parser and
+   formatter_field_name_split to the string.Formatter class
+   implemented in Python."""
+
+
+from pypy.interpreter.mixedmodule import MixedModule
+
+class Module(MixedModule):
+    """Allow programmer to define multiple exit functions to be
+    executed upon normal program termination.
+
+    Two public functions, register and unregister, are defined.
+    """
+
+    interpleveldefs = {
+        }
+
+    appleveldefs = {
+        'register': 'app_atexit.register',
+        'unregister': 'app_atexit.unregister',
+        '_clear': 'app_atexit.clear',
+        '_run_exitfuncs': 'app_atexit.run_exitfuncs',
+        }
+
diff --git a/pypy/module/atexit/app_atexit.py b/pypy/module/atexit/app_atexit.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/atexit/app_atexit.py
@@ -0,0 +1,48 @@
+atexit_callbacks = []
+
+def register(func, *args, **kwargs):
+    """Register a function to be executed upon normal program termination.
+
+    func - function to be called at exit
+    args - optional arguments to pass to func
+    kwargs - optional keyword arguments to pass to func
+
+    func is returned to facilitate usage as a decorator."""
+    
+    if not callable(func):
+        raise TypeError("func must be callable")
+
+    atexit_callbacks.append((func, args, kwargs))
+    return func
+    
+def run_exitfuncs():
+    "Run all registered exit functions."
+    # Maintain the last exception
+    last_exc, last_tb = None, None
+    for (func, args, kwargs) in atexit_callbacks:
+        if func is None:
+            # unregistered slot
+            continue
+        try:
+            func(*args, **kwargs)
+        except BaseException, e:
+            if not isinstance(e, SystemExit):
+                import traceback
+                last_type, last_exc, last_tb = sys.exc_info()
+                traceback.print_exception(last_type, last_exc, last_tb)
+
+    clear()
+
+    if last_exc is not None:
+        raise last_exc.with_traceback(last_tb)
+
+def clear():
+    "Clear the list of previously registered exit functions."
+    del atexit_callbacks[:]
+
+def unregister(func):
+    """Unregister a exit function which was previously registered using
+    atexit.register"""
+    for i, (f, _, _) in enumerate(atexit_callbacks):
+        if f == func:
+            atexit_callbacks[i] = (None, None, None)
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -94,7 +94,6 @@
         'excepthook'            : 'app.excepthook', 
         '__excepthook__'        : 'app.excepthook', 
         'exit'                  : 'app.exit', 
-        'exitfunc'              : 'app.exitfunc',
         'callstats'             : 'app.callstats',
         'copyright'             : 'app.copyright_str',
         'flags'                 : 'app.null_sysflags',
diff --git a/pypy/module/sys/app.py b/pypy/module/sys/app.py
--- a/pypy/module/sys/app.py
+++ b/pypy/module/sys/app.py
@@ -56,9 +56,6 @@
     # in normalize_exception, which is exactly like CPython's.
     raise SystemExit, exitcode
 
-def exitfunc():
-    """Placeholder for sys.exitfunc(), which is called when PyPy exits."""
-
 #import __builtin__
 
 def callstats():

From noreply at buildbot.pypy.org  Thu Oct 13 22:40:15 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 13 Oct 2011 22:40:15 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Add "raise ex2 from ex1" syntax (the from
 clause is not really used yet)
Message-ID: <20111013204015.BAFA782112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48023:689baa45b402
Date: 2011-10-13 22:38 +0200
http://bitbucket.org/pypy/pypy/changeset/689baa45b402/

Log:	Add "raise ex2 from ex1" syntax (the from clause is not really used
	yet)
	- Replace old "raise type, message" syntax
	- various fixes around string conversions

diff --git a/lib_pypy/_collections.py b/lib_pypy/_collections.py
--- a/lib_pypy/_collections.py
+++ b/lib_pypy/_collections.py
@@ -94,7 +94,7 @@
 
     def pop(self):
         if self.left is self.right and self.leftndx > self.rightndx:
-            raise IndexError, "pop from an empty deque"
+            raise IndexError("pop from an empty deque")
         x = self.right[self.rightndx]
         self.right[self.rightndx] = None
         self.length -= 1
@@ -115,7 +115,7 @@
 
     def popleft(self):
         if self.left is self.right and self.leftndx > self.rightndx:
-            raise IndexError, "pop from an empty deque"
+            raise IndexError("pop from an empty deque")
         x = self.left[self.leftndx]
         self.left[self.leftndx] = None
         self.length -= 1
@@ -313,7 +313,7 @@
         return type(self), (list(self), self.maxlen)
 
     def __hash__(self):
-        raise TypeError, "deque objects are unhashable"
+        raise TypeError("deque objects are unhashable")
 
     def __copy__(self):
         return self.__class__(self, self.maxlen)
@@ -365,7 +365,7 @@
         self.counter = len(deq)
         def giveup():
             self.counter = 0
-            raise RuntimeError, "deque mutated during iteration"
+            raise RuntimeError("deque mutated during iteration")
         self._gen = itergen(deq.state, giveup)
 
     def next(self):
diff --git a/lib_pypy/itertools.py b/lib_pypy/itertools.py
--- a/lib_pypy/itertools.py
+++ b/lib_pypy/itertools.py
@@ -377,7 +377,7 @@
             self.step = 1
         if self.start<0 or (self.stop is not None and self.stop<0
            ) or self.step<=0:
-            raise ValueError, "indices for islice() must be positive"
+            raise ValueError("indices for islice() must be positive")
         self.it = iter(iterable)
         self.donext = None
         self.cnt = 0
diff --git a/lib_pypy/struct.py b/lib_pypy/struct.py
--- a/lib_pypy/struct.py
+++ b/lib_pypy/struct.py
@@ -81,18 +81,18 @@
 
 def pack_signed_int(number,size,le):
     if not isinstance(number, (int,long)):
-        raise StructError,"argument for i,I,l,L,q,Q,h,H must be integer"
+        raise StructError("argument for i,I,l,L,q,Q,h,H must be integer")
     if  number > 2**(8*size-1)-1 or number < -1*2**(8*size-1):
-        raise OverflowError,"Number:%i too large to convert" % number
+        raise OverflowError("Number:%i too large to convert" % number)
     return pack_int(number,size,le)
 
 def pack_unsigned_int(number,size,le):
     if not isinstance(number, (int,long)):
-        raise StructError,"argument for i,I,l,L,q,Q,h,H must be integer"
+        raise StructError("argument for i,I,l,L,q,Q,h,H must be integer")
     if number < 0:
-        raise TypeError,"can't convert negative long to unsigned"
+        raise TypeError("can't convert negative long to unsigned")
     if number > 2**(8*size)-1:
-        raise OverflowError,"Number:%i too large to convert" % number
+        raise OverflowError("Number:%i too large to convert" % number)
     return pack_int(number,size,le)
 
 def pack_char(char,size,le):
@@ -296,7 +296,7 @@
         try:
             format = formatdef[cur]
         except KeyError:
-            raise StructError,"%s is not a valid format"%cur
+            raise StructError("%s is not a valid format" % cur)
         if num != None :
             result += num*format['size']
         else:
@@ -319,7 +319,7 @@
         try:
             format = formatdef[cur]
         except KeyError:
-            raise StructError,"%s is not a valid format"%cur
+            raise StructError("%s is not a valid format" % cur)
         if num == None :
             num_s = 0
             num = 1
@@ -334,7 +334,7 @@
                 result += [args[0][:num] + '\0'*padding]
                 args.pop(0)
             else:
-                raise StructError,"arg for string format not a string"
+                raise StructError("arg for string format not a string")
         elif cur == 'p':
             if isinstance(args[0], str):
                 padding = num - len(args[0]) - 1
@@ -348,18 +348,18 @@
                         result += [chr(255) + args[0][:num-1]]
                 args.pop(0)
             else:
-                raise StructError,"arg for string format not a string"
+                raise StructError("arg for string format not a string")
 
         else:
             if len(args) < num:
-                raise StructError,"insufficient arguments to pack"
+                raise StructError("insufficient arguments to pack")
             for var in args[:num]:
                 result += [format['pack'](var,format['size'],endianness)]
             args=args[num:]
         num = None
         i += 1
     if len(args) != 0:
-        raise StructError,"too many arguments for pack format"
+        raise StructError("too many arguments for pack format")
     return ''.join(result)
 
 def unpack(fmt,data):
@@ -373,7 +373,7 @@
     result = []
     length= calcsize(fmt)
     if length != len (data):
-        raise StructError,"unpack str size does not match format"
+        raise StructError("unpack str size does not match format")
     while i<len(fmt):
         num,i=getNum(fmt,i)
         cur = fmt[i]
@@ -381,7 +381,7 @@
         try:
             format = formatdef[cur]
         except KeyError:
-            raise StructError,"%s is not a valid format"%cur
+            raise StructError("%s is not a valid format" % cur)
 
         if not num :
             num = 1
diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -723,44 +723,37 @@
 
 class Raise(stmt):
 
-    _lineno_mask = 8
-    _col_offset_mask = 16
-
-    def __init__(self, type, inst, tback, lineno, col_offset):
-        self.type = type
-        self.inst = inst
-        self.tback = tback
+    _lineno_mask = 4
+    _col_offset_mask = 8
+
+    def __init__(self, exc, cause, lineno, col_offset):
+        self.exc = exc
+        self.cause = cause
         stmt.__init__(self, lineno, col_offset)
-        self.initialization_state = 31
+        self.initialization_state = 15
 
     def walkabout(self, visitor):
         visitor.visit_Raise(self)
 
     def mutate_over(self, visitor):
-        if self.type:
-            self.type = self.type.mutate_over(visitor)
-        if self.inst:
-            self.inst = self.inst.mutate_over(visitor)
-        if self.tback:
-            self.tback = self.tback.mutate_over(visitor)
+        if self.exc:
+            self.exc = self.exc.mutate_over(visitor)
+        if self.cause:
+            self.cause = self.cause.mutate_over(visitor)
         return visitor.visit_Raise(self)
 
     def sync_app_attrs(self, space):
-        if (self.initialization_state & ~7) ^ 24:
-            missing_field(space, self.initialization_state, [None, None, None, 'lineno', 'col_offset'], 'Raise')
+        if (self.initialization_state & ~3) ^ 12:
+            missing_field(space, self.initialization_state, [None, None, 'lineno', 'col_offset'], 'Raise')
         else:
             if not self.initialization_state & 1:
-                self.type = None
+                self.exc = None
             if not self.initialization_state & 2:
-                self.inst = None
-            if not self.initialization_state & 4:
-                self.tback = None
-        if self.type:
-            self.type.sync_app_attrs(space)
-        if self.inst:
-            self.inst.sync_app_attrs(space)
-        if self.tback:
-            self.tback.sync_app_attrs(space)
+                self.cause = None
+        if self.exc:
+            self.exc.sync_app_attrs(space)
+        if self.cause:
+            self.cause.sync_app_attrs(space)
 
 
 class TryExcept(stmt):
@@ -2686,12 +2679,10 @@
         self.visit_sequence(node.body)
 
     def visit_Raise(self, node):
-        if node.type:
-            node.type.walkabout(self)
-        if node.inst:
-            node.inst.walkabout(self)
-        if node.tback:
-            node.tback.walkabout(self)
+        if node.exc:
+            node.exc.walkabout(self)
+        if node.cause:
+            node.cause.walkabout(self)
 
     def visit_TryExcept(self, node):
         self.visit_sequence(node.body)
@@ -3991,79 +3982,57 @@
     __init__=interp2app(With_init),
 )
 
-def Raise_get_type(space, w_self):
+def Raise_get_exc(space, w_self):
     if w_self.w_dict is not None:
-        w_obj = w_self.getdictvalue(space, 'type')
+        w_obj = w_self.getdictvalue(space, 'exc')
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'type'" % typename)
+        w_err = space.wrap("'%s' object has no attribute 'exc'" % typename)
         raise OperationError(space.w_AttributeError, w_err)
-    return space.wrap(w_self.type)
-
-def Raise_set_type(space, w_self, w_new_value):
+    return space.wrap(w_self.exc)
+
+def Raise_set_exc(space, w_self, w_new_value):
     try:
-        w_self.type = space.interp_w(expr, w_new_value, True)
+        w_self.exc = space.interp_w(expr, w_new_value, True)
     except OperationError, e:
         if not e.match(space, space.w_TypeError):
             raise
-        w_self.setdictvalue(space, 'type', w_new_value)
+        w_self.setdictvalue(space, 'exc', w_new_value)
         return
-    w_self.deldictvalue(space, 'type')
+    w_self.deldictvalue(space, 'exc')
     w_self.initialization_state |= 1
 
-def Raise_get_inst(space, w_self):
+def Raise_get_cause(space, w_self):
     if w_self.w_dict is not None:
-        w_obj = w_self.getdictvalue(space, 'inst')
+        w_obj = w_self.getdictvalue(space, 'cause')
         if w_obj is not None:
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'inst'" % typename)
+        w_err = space.wrap("'%s' object has no attribute 'cause'" % typename)
         raise OperationError(space.w_AttributeError, w_err)
-    return space.wrap(w_self.inst)
-
-def Raise_set_inst(space, w_self, w_new_value):
+    return space.wrap(w_self.cause)
+
+def Raise_set_cause(space, w_self, w_new_value):
     try:
-        w_self.inst = space.interp_w(expr, w_new_value, True)
+        w_self.cause = space.interp_w(expr, w_new_value, True)
     except OperationError, e:
         if not e.match(space, space.w_TypeError):
             raise
-        w_self.setdictvalue(space, 'inst', w_new_value)
+        w_self.setdictvalue(space, 'cause', w_new_value)
         return
-    w_self.deldictvalue(space, 'inst')
+    w_self.deldictvalue(space, 'cause')
     w_self.initialization_state |= 2
 
-def Raise_get_tback(space, w_self):
-    if w_self.w_dict is not None:
-        w_obj = w_self.getdictvalue(space, 'tback')
-        if w_obj is not None:
-            return w_obj
-    if not w_self.initialization_state & 4:
-        typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'tback'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
-    return space.wrap(w_self.tback)
-
-def Raise_set_tback(space, w_self, w_new_value):
-    try:
-        w_self.tback = space.interp_w(expr, w_new_value, True)
-    except OperationError, e:
-        if not e.match(space, space.w_TypeError):
-            raise
-        w_self.setdictvalue(space, 'tback', w_new_value)
-        return
-    w_self.deldictvalue(space, 'tback')
-    w_self.initialization_state |= 4
-
-_Raise_field_unroller = unrolling_iterable(['type', 'inst', 'tback'])
+_Raise_field_unroller = unrolling_iterable(['exc', 'cause'])
 def Raise_init(space, w_self, __args__):
     w_self = space.descr_self_interp_w(Raise, w_self)
     args_w, kwargs_w = __args__.unpack()
     if args_w:
-        if len(args_w) != 3:
-            w_err = space.wrap("Raise constructor takes either 0 or 3 positional arguments")
+        if len(args_w) != 2:
+            w_err = space.wrap("Raise constructor takes either 0 or 2 positional arguments")
             raise OperationError(space.w_TypeError, w_err)
         i = 0
         for field in _Raise_field_unroller:
@@ -4075,10 +4044,9 @@
 Raise.typedef = typedef.TypeDef("Raise",
     stmt.typedef,
     __module__='_ast',
-    _fields=_FieldsWrapper(['type', 'inst', 'tback']),
-    type=typedef.GetSetProperty(Raise_get_type, Raise_set_type, cls=Raise),
-    inst=typedef.GetSetProperty(Raise_get_inst, Raise_set_inst, cls=Raise),
-    tback=typedef.GetSetProperty(Raise_get_tback, Raise_set_tback, cls=Raise),
+    _fields=_FieldsWrapper(['exc', 'cause']),
+    exc=typedef.GetSetProperty(Raise_get_exc, Raise_set_exc, cls=Raise),
+    cause=typedef.GetSetProperty(Raise_get_cause, Raise_set_cause, cls=Raise),
     __new__=interp2app(get_AST_new(Raise)),
     __init__=interp2app(Raise_init),
 )
diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -149,17 +149,13 @@
             return ast.Return(values, flow_node.lineno, flow_node.column)
         elif first_child_type == syms.raise_stmt:
             exc = None
-            value = None
-            traceback = None
+            cause = None
             child_count = len(first_child.children)
             if child_count >= 2:
                 exc = self.handle_expr(first_child.children[1])
             if child_count >= 4:
-                value = self.handle_expr(first_child.children[3])
-            if child_count == 6:
-                traceback = self.handle_expr(first_child.children[5])
-            return ast.Raise(exc, value, traceback, flow_node.lineno,
-                             flow_node.column)
+                cause = self.handle_expr(first_child.children[3])
+            return ast.Raise(exc, cause, flow_node.lineno, flow_node.column)
         else:
             raise AssertionError("unknown flow statement")
 
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -746,15 +746,12 @@
     def visit_Raise(self, rais):
         self.update_position(rais.lineno, True)
         arg = 0
-        if rais.type:
-            rais.type.walkabout(self)
+        if rais.exc:
+            rais.exc.walkabout(self)
             arg += 1
-            if rais.inst:
-                rais.inst.walkabout(self)
+            if rais.cause:
+                rais.cause.walkabout(self)
                 arg += 1
-                if rais.tback:
-                    rais.tback.walkabout(self)
-                    arg += 1
         self.emit_op_arg(ops.RAISE_VARARGS, arg)
 
     def visit_Global(self, glob):
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -797,6 +797,17 @@
             return y"""
         yield self.st, test, "f()", 4
 
+    def test_raise_from(self):
+        test = """if 1:
+        def f():
+            try:
+                raise TypeError() from ValueError()
+            except TypeError:
+                return 42
+        """
+        yield self.st, test, "f()", 42
+    # This line is needed for py.code to find the source.
+
 
 class AppTestCompiler:
 
diff --git a/pypy/interpreter/astcompiler/tools/Python.asdl b/pypy/interpreter/astcompiler/tools/Python.asdl
--- a/pypy/interpreter/astcompiler/tools/Python.asdl
+++ b/pypy/interpreter/astcompiler/tools/Python.asdl
@@ -30,8 +30,7 @@
 	      | If(expr test, stmt* body, stmt* orelse)
 	      | With(expr context_expr, expr? optional_vars, stmt* body)
 
-	      -- 'type' is a bad name
-	      | Raise(expr? type, expr? inst, expr? tback)
+	      | Raise(expr? exc, expr? cause)
 	      | TryExcept(stmt* body, excepthandler* handlers, stmt* orelse)
 	      | TryFinally(stmt* body, stmt* finalbody)
 	      | Assert(expr test, expr? msg)
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1219,7 +1219,7 @@
         # unclear if there is any use at all for getting the bytes in
         # the unicode buffer.)
         try:
-            return self.str_w(w_obj)
+            return self.bytes_w(w_obj)
         except OperationError, e:
             if not e.match(self, self.w_TypeError):
                 raise
@@ -1254,6 +1254,10 @@
         return self.str_w(w_obj)
 
     def unicode_w(self, w_obj):
+        if not we_are_translated():
+            if self.isinstance_w(w_obj, self.w_bytes):
+                import pdb; pdb.set_trace()
+                assert False, "unicode_w was called with a bytes string"
         return w_obj.unicode_w(self)
 
     def realunicode_w(self, w_obj):
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -489,15 +489,14 @@
             self.last_exception = operror
             raise Reraise
 
-        w_value = w_traceback = space.w_None
-        if nbargs >= 3:
-            w_traceback = self.popvalue()
+        w_value = w_cause = space.w_None
         if nbargs >= 2:
+            w_cause = self.popvalue() # XXX cause?
+        if 1:
             w_value = self.popvalue()
-        if 1:
-            w_type = self.popvalue()
-        operror = OperationError(w_type, w_value)
+        operror = OperationError(space.type(w_value), w_value)
         operror.normalize_exception(space)
+        w_traceback = space.w_None # XXX with_traceback?
         if not space.full_exceptions or space.is_w(w_traceback, space.w_None):
             # common case
             raise operror
diff --git a/pypy/interpreter/pyparser/data/Grammar3.2 b/pypy/interpreter/pyparser/data/Grammar3.2
--- a/pypy/interpreter/pyparser/data/Grammar3.2
+++ b/pypy/interpreter/pyparser/data/Grammar3.2
@@ -49,7 +49,7 @@
 continue_stmt: 'continue'
 return_stmt: 'return' [testlist]
 yield_stmt: yield_expr
-raise_stmt: 'raise' [test [',' test [',' test]]]
+raise_stmt: 'raise' [test ['from' test]]
 import_stmt: import_name | import_from
 import_name: 'import' dotted_as_names
 import_from: ('from' ('.'* dotted_name | '.'+)
diff --git a/pypy/interpreter/pyparser/pyparse.py b/pypy/interpreter/pyparser/pyparse.py
--- a/pypy/interpreter/pyparser/pyparse.py
+++ b/pypy/interpreter/pyparser/pyparse.py
@@ -135,7 +135,7 @@
                 if e.match(space, space.w_UnicodeDecodeError):
                     e.normalize_exception(space)
                     w_message = space.str(e.get_w_value(space))
-                    raise error.SyntaxError(space.text_w(w_message))
+                    raise error.SyntaxError(space.str_w(w_message))
                 raise
 
         f_flags, future_info = future.get_futures(self.future_flags, textsrc)
diff --git a/pypy/module/__builtin__/app_inspect.py b/pypy/module/__builtin__/app_inspect.py
--- a/pypy/module/__builtin__/app_inspect.py
+++ b/pypy/module/__builtin__/app_inspect.py
@@ -19,12 +19,12 @@
     if len(obj) == 0:
         return _caller_locals()
     elif len(obj) != 1:
-        raise TypeError, "vars() takes at most 1 argument."
+        raise TypeError("vars() takes at most 1 argument.")
     else:
         try:
             return obj[0].__dict__
         except AttributeError:
-            raise TypeError, "vars() argument must have __dict__ attribute"
+            raise TypeError("vars() argument must have __dict__ attribute")
 
 # Replaced by the interp-level helper space.callable(): 
 ##def callable(ob):
diff --git a/pypy/module/exceptions/interp_exceptions.py b/pypy/module/exceptions/interp_exceptions.py
--- a/pypy/module/exceptions/interp_exceptions.py
+++ b/pypy/module/exceptions/interp_exceptions.py
@@ -631,11 +631,11 @@
 
     def descr_init(self, space, w_encoding, w_object, w_start, w_end, w_reason):
         # typechecking
-        space.text_w(w_encoding)
-        space.str_w(w_object)
+        space.str_w(w_encoding)
+        space.bytes_w(w_object)
         space.int_w(w_start)
         space.int_w(w_end)
-        space.text_w(w_reason)
+        space.str_w(w_reason)
         # assign attributes
         self.w_encoding = w_encoding
         self.w_object = w_object
@@ -650,7 +650,7 @@
             if self.end == self.start + 1:
                 return "'%s' codec can't decode byte 0x%02x in position %d: %s"%(
                     self.encoding,
-                    ord(self.object[self.start]), self.start, self.reason)
+                    self.object[self.start], self.start, self.reason)
             return "'%s' codec can't decode bytes in position %d-%d: %s" % (
                 self.encoding, self.start, self.end - 1, self.reason)
         """)
diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -964,7 +964,7 @@
     """ Read a code object from a file and check it for validity """
 
     w_marshal = space.getbuiltinmodule('marshal')
-    w_code = space.call_method(w_marshal, 'loads', space.wrap(strbuf))
+    w_code = space.call_method(w_marshal, 'loads', space.wrapbytes(strbuf))
     pycode = space.interpclass_w(w_code)
     if pycode is None or not isinstance(pycode, Code):
         raise operationerrfmt(space.w_ImportError,
@@ -1008,9 +1008,9 @@
     """
     w_marshal = space.getbuiltinmodule('marshal')
     try:
-        w_str = space.call_method(w_marshal, 'dumps', space.wrap(co),
-                                  space.wrap(MARSHAL_VERSION_FOR_PYC))
-        strbuf = space.str_w(w_str)
+        w_bytes = space.call_method(w_marshal, 'dumps', space.wrap(co),
+                                    space.wrap(MARSHAL_VERSION_FOR_PYC))
+        strbuf = space.bytes_w(w_bytes)
     except OperationError, e:
         if e.async(space):
             raise
diff --git a/pypy/module/operator/app_operator.py b/pypy/module/operator/app_operator.py
--- a/pypy/module/operator/app_operator.py
+++ b/pypy/module/operator/app_operator.py
@@ -21,7 +21,7 @@
         if x == b:
             return index
         index += 1
-    raise ValueError, 'sequence.index(x): x not in sequence'
+    raise ValueError('sequence.index(x): x not in sequence')
 
 # XXX the following is approximative
 def isMappingType(obj,):
@@ -40,9 +40,9 @@
 def repeat(obj, num):
     'repeat(a, b) -- Return a * b, where a is a sequence, and b is an integer.'
     if not isinstance(num, (int, long)):
-        raise TypeError, 'an integer is required'
+        raise TypeError('an integer is required')
     if not isSequenceType(obj):
-        raise TypeError, "non-sequence object can't be repeated"
+        raise TypeError("non-sequence object can't be repeated")
 
     return obj * num
 
diff --git a/pypy/module/posix/app_posix.py b/pypy/module/posix/app_posix.py
--- a/pypy/module/posix/app_posix.py
+++ b/pypy/module/posix/app_posix.py
@@ -191,7 +191,7 @@
         except Exception, e:
             try_close(write_end)
             try_close(read_end)
-            raise Exception, e     # bare 'raise' does not work here :-(
+            raise Exception() from e
 
     def wait():
         """ wait() -> (pid, status)
diff --git a/pypy/module/sys/app.py b/pypy/module/sys/app.py
--- a/pypy/module/sys/app.py
+++ b/pypy/module/sys/app.py
@@ -54,7 +54,7 @@
     # note that we cannot use SystemExit(exitcode) here.
     # The comma version leads to an extra de-tupelizing
     # in normalize_exception, which is exactly like CPython's.
-    raise SystemExit, exitcode
+    raise SystemExit(exitcode)
 
 #import __builtin__
 
diff --git a/pypy/objspace/std/marshal_impl.py b/pypy/objspace/std/marshal_impl.py
--- a/pypy/objspace/std/marshal_impl.py
+++ b/pypy/objspace/std/marshal_impl.py
@@ -249,7 +249,7 @@
 marshal_w__Rope = marshal_w__String
 
 def unmarshal_String(space, u, tc):
-    return space.wrap(u.get_str())
+    return space.wrapbytes(u.get_str())
 register(TYPE_STRING, unmarshal_String)
 
 def unmarshal_stringref(space, u, tc):
@@ -336,7 +336,7 @@
 def unmarshal_str(u):
     w_obj = u.get_w_obj()
     try:
-        return u.space.str_w(w_obj)
+        return u.space.bytes_w(w_obj)
     except OperationError, e:
         if e.match(u.space, u.space.w_TypeError):
             u.raise_exc('invalid marshal data for code object')
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -75,8 +75,7 @@
     return ''.join(result)
 
 def str__Unicode(space, w_uni):
-    from pypy.objspace.std.unicodetype import encode_object
-    return encode_object(space, w_uni, None, None)
+    return w_uni
 
 def eq__Unicode_Unicode(space, w_left, w_right):
     return space.newbool(w_left._value == w_right._value)

From noreply at buildbot.pypy.org  Thu Oct 13 22:40:17 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 13 Oct 2011 22:40:17 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: There is no such things as str.encode()
 and unicode.decode()!
Message-ID: <20111013204017.041CF82112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48024:8c6f39bc08a4
Date: 2011-10-13 22:38 +0200
http://bitbucket.org/pypy/pypy/changeset/8c6f39bc08a4/

Log:	There is no such things as str.encode() and unicode.decode()!

diff --git a/pypy/objspace/std/ropeobject.py b/pypy/objspace/std/ropeobject.py
--- a/pypy/objspace/std/ropeobject.py
+++ b/pypy/objspace/std/ropeobject.py
@@ -867,12 +867,6 @@
         encoding = getdefaultencoding(space)
     return decode_string(space, w_string, encoding, errors)
 
-def str_encode__Rope_ANY_ANY(space, w_string, w_encoding=None, w_errors=None):
-    from pypy.objspace.std.unicodetype import _get_encoding_and_errors, \
-        encode_object
-    encoding, errors = _get_encoding_and_errors(space, w_encoding, w_errors)
-    return encode_object(space, w_string, encoding, errors)
-
 def buffer__Rope(space, w_string):
     from pypy.interpreter.buffer import StringBuffer
     value = w_string._node.flatten_string()      # XXX inefficient
diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py
--- a/pypy/objspace/std/stringobject.py
+++ b/pypy/objspace/std/stringobject.py
@@ -1000,12 +1000,6 @@
     encoding, errors = _get_encoding_and_errors(space, w_encoding, w_errors)
     return decode_object(space, w_string, encoding, errors)
 
-def str_encode__String_ANY_ANY(space, w_string, w_encoding=None, w_errors=None):
-    from pypy.objspace.std.unicodetype import _get_encoding_and_errors, \
-        encode_object
-    encoding, errors = _get_encoding_and_errors(space, w_encoding, w_errors)
-    return encode_object(space, w_string, encoding, errors)
-
 # CPython's logic for deciding if  ""%values  is
 # an error (1 value, 0 %-formatters) or not
 # (values is of a mapping type)
diff --git a/pypy/objspace/std/stringtype.py b/pypy/objspace/std/stringtype.py
--- a/pypy/objspace/std/stringtype.py
+++ b/pypy/objspace/std/stringtype.py
@@ -259,18 +259,6 @@
                          " are 'ignore' and 'replace'\nas well as any other"
                          ' name registerd with codecs.register_error that'
                          ' is\nable to handle UnicodeDecodeErrors.')
-str_encode     = SMM('encode', 3, defaults=(None, None),
-                     argnames=['encoding', 'errors'],
-                     doc='S.encode([encoding[,errors]]) -> object\n\nEncodes S'
-                         ' using the codec registered for encoding. encoding'
-                         ' defaults\nto the default encoding. errors may be'
-                         ' given to set a different error\nhandling scheme.'
-                         " Default is 'strict' meaning that encoding errors"
-                         ' raise\na UnicodeEncodeError. Other possible values'
-                         " are 'ignore', 'replace' and\n'xmlcharrefreplace' as"
-                         ' well as any other name registered'
-                         ' with\ncodecs.register_error that is able to handle'
-                         ' UnicodeEncodeErrors.')
 
 register_all(vars(), globals())
 
diff --git a/pypy/objspace/std/unicodetype.py b/pypy/objspace/std/unicodetype.py
--- a/pypy/objspace/std/unicodetype.py
+++ b/pypy/objspace/std/unicodetype.py
@@ -168,7 +168,6 @@
 from pypy.objspace.std.stringtype import str_strip as unicode_strip
 from pypy.objspace.std.stringtype import str_rstrip as unicode_rstrip
 from pypy.objspace.std.stringtype import str_lstrip as unicode_lstrip
-from pypy.objspace.std.stringtype import str_decode as unicode_decode
 
 # ____________________________________________________________
 
@@ -299,11 +298,6 @@
             return w_res
     return unicode_from_encoded_object(space, w_res, None, "strict")
 
-def unicode_decode__unitypedef_ANY_ANY(space, w_unicode, w_encoding=None,
-                                       w_errors=None):
-    return space.call_method(space.str(w_unicode), 'decode',
-                             w_encoding, w_errors)
-
 
 def descr_new_(space, w_unicodetype, w_string='', w_encoding=None, w_errors=None):
     # NB. the default value of w_obj is really a *wrapped* empty string:

From noreply at buildbot.pypy.org  Thu Oct 13 22:40:18 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 13 Oct 2011 22:40:18 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: unicode_encode should return bytes
Message-ID: <20111013204018.4387182112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48025:f6ede46962ce
Date: 2011-10-13 22:38 +0200
http://bitbucket.org/pypy/pypy/changeset/f6ede46962ce/

Log:	unicode_encode should return bytes

diff --git a/lib_pypy/_md5.py b/lib_pypy/_md5.py
--- a/lib_pypy/_md5.py
+++ b/lib_pypy/_md5.py
@@ -112,7 +112,7 @@
     return res & 0xffffffffL
 
 
-class MD5Type:
+class md5:
     "An implementation of the MD5 hash function in pure Python."
 
     digest_size = digestsize = 16
@@ -366,23 +366,3 @@
         clone.C = self.C
         clone.D = self.D
         return clone
-
-
-# ======================================================================
-# Mimic Python top-level functions from standard library API
-# for consistency with the _md5 module of the standard library.
-# ======================================================================
-
-digest_size = 16
-
-def new(arg=None):
-    """Return a new md5 crypto object.
-    If arg is present, the method call update(arg) is made.
-    """
-
-    crypto = MD5Type()
-    if arg:
-        crypto.update(arg)
-
-    return crypto
-
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -825,11 +825,17 @@
     def _getdefaults(self, space):
         "NOT_RPYTHON"
         defs_w = []
-        for val in self._staticdefs:
+        
+        unwrap_spec = self._code._unwrap_spec[-len(self._staticdefs):]
+        for i, val in enumerate(self._staticdefs):
             if val is NoneNotWrapped:
                 defs_w.append(None)
             else:
-                defs_w.append(space.wrap(val))
+                spec = unwrap_spec[i]
+                if spec in ['bufferstr']:
+                    defs_w.append(space.wrapbytes(val))
+                else:
+                    defs_w.append(space.wrap(val))
         return defs_w
 
     # lazy binding to space
diff --git a/pypy/module/_md5/__init__.py b/pypy/module/_md5/__init__.py
--- a/pypy/module/_md5/__init__.py
+++ b/pypy/module/_md5/__init__.py
@@ -19,9 +19,7 @@
 far using the digest() method."""
 
     interpleveldefs = {
-        'new': 'interp_md5.W_MD5',
-        'MD5Type': 'interp_md5.W_MD5',
-        'digest_size': 'space.wrap(16)',
+        'md5': 'interp_md5.W_MD5',
         }
 
     appleveldefs = {
diff --git a/pypy/module/_md5/interp_md5.py b/pypy/module/_md5/interp_md5.py
--- a/pypy/module/_md5/interp_md5.py
+++ b/pypy/module/_md5/interp_md5.py
@@ -18,7 +18,7 @@
         self.update(string)
 
     def digest_w(self):
-        return self.space.wrap(self.digest())
+        return self.space.wrapbytes(self.digest())
 
     def hexdigest_w(self):
         return self.space.wrap(self.hexdigest())
diff --git a/pypy/module/_md5/test/test_md5.py b/pypy/module/_md5/test/test_md5.py
--- a/pypy/module/_md5/test/test_md5.py
+++ b/pypy/module/_md5/test/test_md5.py
@@ -1,5 +1,5 @@
 """
-Tests for the md5 module implemented at interp-level in pypy/module/md5.
+Tests for the md5 module implemented at interp-level in pypy/module/_md5.
 """
 
 import py, sys
@@ -15,8 +15,8 @@
         """
         cls.space = gettestobjspace(usemodules=['_md5'])
         cls.w_md5 = cls.space.appexec([], """():
-            import md5
-            return md5
+            import _md5
+            return _md5
         """)
 
 
@@ -25,8 +25,6 @@
         md5.digest_size should be 16.
         """
         import sys
-        assert self.md5.digest_size == 16
-        #assert self.md5.digestsize == 16        -- not on CPython
         assert self.md5.md5().digest_size == 16
         if sys.version >= (2, 5):
             assert self.md5.blocksize == 1
@@ -35,15 +33,10 @@
 
     def test_MD5Type(self):
         """
-        Test the two ways to construct an md5 object.
+        Test the construction of an md5 object.
         """
         md5 = self.md5
         d = md5.md5()
-        if not hasattr(md5, 'MD5Type'):
-            skip("no md5.MD5Type on CPython")
-        assert isinstance(d, md5.MD5Type)
-        d = md5.new()
-        assert isinstance(d, md5.MD5Type)
 
 
     def test_md5object(self):
@@ -52,26 +45,27 @@
         hexdigest.
         """
         md5 = self.md5
+        import binascii
         cases = (
-          ("",
+          (b"",
            "d41d8cd98f00b204e9800998ecf8427e"),
-          ("a",
+          (b"a",
            "0cc175b9c0f1b6a831c399e269772661"),
-          ("abc",
+          (b"abc",
            "900150983cd24fb0d6963f7d28e17f72"),
-          ("message digest",
+          (b"message digest",
            "f96b697d7cb7938d525a2f31aaf161d0"),
-          ("abcdefghijklmnopqrstuvwxyz",
+          (b"abcdefghijklmnopqrstuvwxyz",
            "c3fcd3d76192e4007dfb496cca67e13b"),
-          ("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789",
+          (b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789",
            "d174ab98d277d9f5a5611c2c9f419d9f"),
-          ("1234567890"*8,
+          (b"1234567890"*8,
            "57edf4a22be3c955ac49da2e2107b67a"),
         )
         for input, expected in cases:
-            d = md5.new(input)
+            d = md5.md5(input)
             assert d.hexdigest() == expected
-            assert d.digest() == expected.decode('hex')
+            assert d.digest() == binascii.hexlify(expected.encode('ascii'))
 
 
     def test_copy(self):
@@ -80,10 +74,10 @@
         """
         md5 = self.md5
         d1 = md5.md5()
-        d1.update("abcde")
+        d1.update(b"abcde")
         d2 = d1.copy()
-        d2.update("fgh")
-        d1.update("jkl")
+        d2.update(b"fgh")
+        d1.update(b"jkl")
         assert d1.hexdigest() == 'e570e7110ecef72fcb772a9c05d03373'
         assert d2.hexdigest() == 'e8dc4081b13434b45189a720b77b6818'
 
diff --git a/pypy/objspace/std/unicodetype.py b/pypy/objspace/std/unicodetype.py
--- a/pypy/objspace/std/unicodetype.py
+++ b/pypy/objspace/std/unicodetype.py
@@ -221,13 +221,13 @@
             if encoding == 'ascii':
                 u = space.unicode_w(w_object)
                 eh = encode_error_handler(space)
-                return space.wrap(unicode_encode_ascii(u, len(u), None,
-                                                       errorhandler=eh))
+                return space.wrapbytes(unicode_encode_ascii(
+                        u, len(u), None, errorhandler=eh))
             if encoding == 'utf-8':
                 u = space.unicode_w(w_object)
                 eh = encode_error_handler(space)
-                return space.wrap(unicode_encode_utf_8(u, len(u), None,
-                                                       errorhandler=eh))
+                return space.wrapbytes(unicode_encode_utf_8(
+                        u, len(u), None, errorhandler=eh))
         from pypy.module._codecs.interp_codecs import lookup_codec
         w_encoder = space.getitem(lookup_codec(space, encoding), space.wrap(0))
     if errors is None:
@@ -236,9 +236,9 @@
         w_errors = space.wrap(errors)
     w_restuple = space.call_function(w_encoder, w_object, w_errors)
     w_retval = space.getitem(w_restuple, space.wrap(0))
-    if not space.isinstance_w(w_retval, space.w_str):
+    if not space.isinstance_w(w_retval, space.w_bytes):
         raise operationerrfmt(space.w_TypeError,
-            "encoder did not return an string object (type '%s')",
+            "encoder did not return a bytes string (type '%s')",
             space.type(w_retval).getname(space))
     return w_retval
 

From noreply at buildbot.pypy.org  Thu Oct 13 22:40:19 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 13 Oct 2011 22:40:19 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: bytes() now accept a list of integers
Message-ID: <20111013204019.82F4F82112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48026:897c563f0090
Date: 2011-10-13 22:38 +0200
http://bitbucket.org/pypy/pypy/changeset/897c563f0090/

Log:	bytes() now accept a list of integers

diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -22,10 +22,8 @@
 from pypy.interpreter import gateway
 from pypy.interpreter.argument import Signature
 from pypy.interpreter.buffer import RWBuffer
-from pypy.objspace.std.bytearraytype import (
-    makebytearraydata_w, getbytevalue,
-    new_bytearray
-)
+from pypy.objspace.std.stringtype import makebytesdata_w, getbytevalue
+from pypy.objspace.std.bytearraytype import new_bytearray
 from pypy.tool.sourcetools import func_with_new_name
 
 
@@ -81,7 +79,7 @@
         w_bytearray.data = ['\0'] * count
         return
 
-    data = makebytearraydata_w(space, w_source)
+    data = makebytesdata_w(space, w_source)
     w_bytearray.data = data
 
 def len__Bytearray(space, w_bytearray):
@@ -519,7 +517,7 @@
     w_str = str__Bytearray(space, w_bytearray)
     w_result = stringobject.str_splitlines__String_ANY(space, w_str, w_keepends)
     return space.newlist([
-        new_bytearray(space, space.w_bytearray, makebytearraydata_w(space, w_entry))
+        new_bytearray(space, space.w_bytearray, makebytesdata_w(space, w_entry))
                         for w_entry in space.unpackiterable(w_result)
     ])
 
@@ -576,7 +574,7 @@
     w_bytearray.data += w_other.data
 
 def list_extend__Bytearray_ANY(space, w_bytearray, w_other):
-    w_bytearray.data += makebytearraydata_w(space, w_other)
+    w_bytearray.data += makebytesdata_w(space, w_other)
 
 def inplace_add__Bytearray_Bytearray(space, w_bytearray1, w_bytearray2):
     list_extend__Bytearray_Bytearray(space, w_bytearray1, w_bytearray2)
@@ -598,7 +596,7 @@
 def setitem__Bytearray_Slice_ANY(space, w_bytearray, w_slice, w_other):
     oldsize = len(w_bytearray.data)
     start, stop, step, slicelength = w_slice.indices4(space, oldsize)
-    sequence2 = makebytearraydata_w(space, w_other)
+    sequence2 = makebytesdata_w(space, w_other)
     setitem_slice_helper(space, w_bytearray.data, start, step, slicelength, sequence2, empty_elem='\x00')
 
 def delitem__Bytearray_ANY(space, w_bytearray, w_idx):
diff --git a/pypy/objspace/std/bytearraytype.py b/pypy/objspace/std/bytearraytype.py
--- a/pypy/objspace/std/bytearraytype.py
+++ b/pypy/objspace/std/bytearraytype.py
@@ -51,21 +51,6 @@
                     "bytes contained in the argument.\nIf the argument is "
                     "omitted, strip trailing ASCII whitespace.")
 
-def getbytevalue(space, w_value):
-    if space.isinstance_w(w_value, space.w_str):
-        string = space.str_w(w_value)
-        if len(string) != 1:
-            raise OperationError(space.w_ValueError, space.wrap(
-                "string must be of size 1"))
-        return string[0]
-
-    value = space.getindex_w(w_value, None)
-    if not 0 <= value < 256:
-        # this includes the OverflowError in case the long is too large
-        raise OperationError(space.w_ValueError, space.wrap(
-            "byte must be in range(0, 256)"))
-    return chr(value)
-
 def new_bytearray(space, w_bytearraytype, data):
     from pypy.objspace.std.bytearrayobject import W_BytearrayObject
     w_obj = space.allocate_instance(W_BytearrayObject, w_bytearraytype)
@@ -77,30 +62,6 @@
     return new_bytearray(space,w_bytearraytype, [])
 
 
-def makebytearraydata_w(space, w_source):
-    # String-like argument
-    try:
-        string = space.bufferstr_new_w(w_source)
-    except OperationError, e:
-        if not e.match(space, space.w_TypeError):
-            raise
-    else:
-        return [c for c in string]
-
-    # sequence of bytes
-    data = []
-    w_iter = space.iter(w_source)
-    while True:
-        try:
-            w_item = space.next(w_iter)
-        except OperationError, e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            break
-        value = getbytevalue(space, w_item)
-        data.append(value)
-    return data
-
 def descr_bytearray__reduce__(space, w_self):
     from pypy.objspace.std.bytearrayobject import W_BytearrayObject
     assert isinstance(w_self, W_BytearrayObject)
diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py
--- a/pypy/objspace/std/stringobject.py
+++ b/pypy/objspace/std/stringobject.py
@@ -385,7 +385,7 @@
         if self and i != 0:
             sb.append(self)
         sb.append(space.str_w(list_w[i]))
-    return space.wrap(sb.build())
+    return space.wrapbytes(sb.build())
 
 def str_rjust__String_ANY_ANY(space, w_self, w_arg, w_fillchar):
     u_arg = space.int_w(w_arg)
diff --git a/pypy/objspace/std/stringtype.py b/pypy/objspace/std/stringtype.py
--- a/pypy/objspace/std/stringtype.py
+++ b/pypy/objspace/std/stringtype.py
@@ -1,8 +1,8 @@
 from pypy.interpreter import gateway
+from pypy.interpreter.error import OperationError
 from pypy.objspace.std.stdtypedef import StdTypeDef, SMM
 from pypy.objspace.std.register_all import register_all
 
-
 from sys import maxint
 from pypy.rlib.objectmodel import specialize
 from pypy.rlib.jit import we_are_jitted
@@ -264,20 +264,50 @@
 
 # ____________________________________________________________
 
-def descr__new__(space, w_stringtype, w_object=''):
-    # NB. the default value of w_object is really a *wrapped* empty string:
-    #     there is gateway magic at work
-    from pypy.objspace.std.stringobject import W_StringObject
-    w_obj = space.str(w_object)
-    if space.is_w(w_stringtype, space.w_str):
-        return w_obj  # XXX might be reworked when space.str() typechecks
-    value = space.str_w(w_obj)
+def getbytevalue(space, w_value):
+    value = space.getindex_w(w_value, None)
+    if not 0 <= value < 256:
+        # this includes the OverflowError in case the long is too large
+        raise OperationError(space.w_ValueError, space.wrap(
+            "byte must be in range(0, 256)"))
+    return chr(value)
+
+def makebytesdata_w(space, w_source):
+    # String-like argument
+    try:
+        string = space.bufferstr_new_w(w_source)
+    except OperationError, e:
+        if not e.match(space, space.w_TypeError):
+            raise
+    else:
+        return [c for c in string]
+
+    # sequence of bytes
+    data = []
+    w_iter = space.iter(w_source)
+    while True:
+        try:
+            w_item = space.next(w_iter)
+        except OperationError, e:
+            if not e.match(space, space.w_StopIteration):
+                raise
+            break
+        value = getbytevalue(space, w_item)
+        data.append(value)
+    return data
+
+def descr__new__(space, w_stringtype, w_source=gateway.NoneNotWrapped):
+    if (w_source and space.is_w(space.type(w_source), space.w_bytes) and
+        space.is_w(w_stringtype, space.w_bytes)):
+        return w_source
+    value = ''.join(makebytesdata_w(space, w_source))
     if space.config.objspace.std.withrope:
         from pypy.objspace.std.ropeobject import rope, W_RopeObject
         w_obj = space.allocate_instance(W_RopeObject, w_stringtype)
         W_RopeObject.__init__(w_obj, rope.LiteralStringNode(value))
         return w_obj
     else:
+        from pypy.objspace.std.stringobject import W_StringObject
         w_obj = space.allocate_instance(W_StringObject, w_stringtype)
         W_StringObject.__init__(w_obj, value)
         return w_obj

From noreply at buildbot.pypy.org  Thu Oct 13 22:40:20 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 13 Oct 2011 22:40:20 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Fix "raise TypeError",
 exception type vs. exception instance.
Message-ID: <20111013204020.C16B182112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48027:82ba9ff70fc2
Date: 2011-10-13 22:38 +0200
http://bitbucket.org/pypy/pypy/changeset/82ba9ff70fc2/

Log:	Fix "raise TypeError", exception type vs. exception instance.

diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -494,7 +494,12 @@
             w_cause = self.popvalue() # XXX cause?
         if 1:
             w_value = self.popvalue()
-        operror = OperationError(space.type(w_value), w_value)
+        if space.exception_is_valid_obj_as_class_w(w_value):
+            w_type = w_value
+            w_value = space.call_function(w_type)
+        else:
+            w_type = space.type(w_value)
+        operror = OperationError(w_type, w_value)
         operror.normalize_exception(space)
         w_traceback = space.w_None # XXX with_traceback?
         if not space.full_exceptions or space.is_w(w_traceback, space.w_None):

From noreply at buildbot.pypy.org  Thu Oct 13 22:40:22 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 13 Oct 2011 22:40:22 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: test_md5 now passes
Message-ID: <20111013204022.0F6E882112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48028:f76981eb4678
Date: 2011-10-13 22:38 +0200
http://bitbucket.org/pypy/pypy/changeset/f76981eb4678/

Log:	test_md5 now passes

diff --git a/lib_pypy/binascii.py b/lib_pypy/binascii.py
--- a/lib_pypy/binascii.py
+++ b/lib_pypy/binascii.py
@@ -656,7 +656,7 @@
     result = 0
     crc = ~long(crc) & 0xffffffffL
     for c in s:
-        crc = crc_32_tab[(crc ^ long(ord(c))) & 0xffL] ^ (crc >> 8)
+        crc = crc_32_tab[(crc ^ c) & 0xffL] ^ (crc >> 8)
         #/* Note:  (crc >> 8) MUST zero fill on left
 
     result = crc ^ 0xffffffffL
@@ -669,19 +669,19 @@
 def b2a_hex(s):
     result = []
     for char in s:
-        c = (ord(char) >> 4) & 0xf
+        c = (char >> 4) & 0xf
         if c > 9:
             c = c + ord('a') - 10
         else:
             c = c + ord('0')
         result.append(chr(c))
-        c = ord(char) & 0xf
+        c = char & 0xf
         if c > 9:
             c = c + ord('a') - 10
         else:
             c = c + ord('0')
-        result.append(chr(c))
-    return ''.join(result)
+        result.append(c)
+    return bytes(result)
 
 hexlify = b2a_hex
 
@@ -713,8 +713,8 @@
     for a, b in pairs_gen(t):
         if a < 0 or b < 0:
             raise TypeError('Non-hexadecimal digit found')
-        result.append(chr((a << 4) + b))
-    return ''.join(result)
+        result.append((a << 4) + b)
+    return bytes(result)
     
 
 unhexlify = a2b_hex
diff --git a/pypy/module/_md5/test/test_md5.py b/pypy/module/_md5/test/test_md5.py
--- a/pypy/module/_md5/test/test_md5.py
+++ b/pypy/module/_md5/test/test_md5.py
@@ -65,7 +65,7 @@
         for input, expected in cases:
             d = md5.md5(input)
             assert d.hexdigest() == expected
-            assert d.digest() == binascii.hexlify(expected.encode('ascii'))
+            assert d.digest() == binascii.unhexlify(expected.encode('ascii'))
 
 
     def test_copy(self):
@@ -87,8 +87,8 @@
         Test passing a buffer object.
         """
         md5 = self.md5
-        d1 = md5.md5(buffer("abcde"))
-        d1.update(buffer("jkl"))
+        d1 = md5.md5(buffer(b"abcde"))
+        d1.update(buffer(b"jkl"))
         assert d1.hexdigest() == 'e570e7110ecef72fcb772a9c05d03373'
 
 
@@ -97,7 +97,6 @@
         Test passing unicode strings.
         """
         md5 = self.md5
-        d1 = md5.md5(u"abcde")
-        d1.update(u"jkl")
-        assert d1.hexdigest() == 'e570e7110ecef72fcb772a9c05d03373'
-        raises(UnicodeEncodeError, d1.update, u'\xe9')
+        raises(TypeError, md5.md5, "abcde")
+        d1 = md5.md5()
+        raises(TypeError, d1.update, "jkl")

From noreply at buildbot.pypy.org  Thu Oct 13 22:40:23 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 13 Oct 2011 22:40:23 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Fix most of the tests for the exception
	module
Message-ID: <20111013204023.49D0A82112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48029:f01df61615da
Date: 2011-10-13 22:38 +0200
http://bitbucket.org/pypy/pypy/changeset/f01df61615da/

Log:	Fix most of the tests for the exception module

diff --git a/pypy/module/exceptions/interp_exceptions.py b/pypy/module/exceptions/interp_exceptions.py
--- a/pypy/module/exceptions/interp_exceptions.py
+++ b/pypy/module/exceptions/interp_exceptions.py
@@ -288,7 +288,7 @@
         space.realunicode_w(w_object)
         space.int_w(w_start)
         space.int_w(w_end)
-        space.realstr_w(w_reason)
+        space.str_w(w_reason)
         # assign attributes
         self.w_object = w_object
         self.w_start = w_start
@@ -720,11 +720,11 @@
 
     def descr_init(self, space, w_encoding, w_object, w_start, w_end, w_reason):
         # typechecking
-        space.realstr_w(w_encoding)
+        space.str_w(w_encoding)
         space.realunicode_w(w_object)
         space.int_w(w_start)
         space.int_w(w_end)
-        space.realstr_w(w_reason)
+        space.str_w(w_reason)
         # assign attributes
         self.w_encoding = w_encoding
         self.w_object = w_object
diff --git a/pypy/module/exceptions/test/test_exc.py b/pypy/module/exceptions/test/test_exc.py
--- a/pypy/module/exceptions/test/test_exc.py
+++ b/pypy/module/exceptions/test/test_exc.py
@@ -34,14 +34,6 @@
         del x.message
         assert not hasattr(x, "message")
 
-    def test_unicode_message(self):
-        assert unicode(Exception(u"\xe1")) == u"\xe1"
-        class E(BaseException):
-            def __str__(self):
-                return u"\xe1"
-        e = E()
-        assert unicode(e) == u"\xe1"
-
     def test_kwargs(self):
         from exceptions import Exception
         class X(Exception):
@@ -152,13 +144,13 @@
 
     def test_unicode_decode_error(self):
         from exceptions import UnicodeDecodeError
-        ud = UnicodeDecodeError("x", "y", 1, 5, "bah")
+        ud = UnicodeDecodeError("x", b"y", 1, 5, "bah")
         assert ud.encoding == 'x'
-        assert ud.object == 'y'
+        assert ud.object == b'y'
         assert ud.start == 1
         assert ud.end == 5
         assert ud.reason == 'bah'
-        assert ud.args == ('x', 'y', 1, 5, 'bah')
+        assert ud.args == ('x', b'y', 1, 5, 'bah')
         assert ud.message == ''
         ud.object = 'z9'
         assert ud.object == 'z9'
@@ -183,8 +175,7 @@
         assert str(ue) == "'x' codec can't encode character u'\\x39' in position 1: bah"
         ue.object = []
         assert ue.object == []
-        raises(TypeError, UnicodeEncodeError, "x", "y", 1, 5, "bah")
-        raises(TypeError, UnicodeEncodeError, u"x", u"y", 1, 5, "bah")
+        raises(TypeError, UnicodeEncodeError, "x", b"y", 1, 5, "bah")
 
     def test_multiple_inheritance(self):
         from exceptions import LookupError, ValueError, Exception, IOError

From noreply at buildbot.pypy.org  Thu Oct 13 22:40:24 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 13 Oct 2011 22:40:24 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Fix remaining failures in module/exception
Message-ID: <20111013204024.8318482112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48030:9824a8b7789a
Date: 2011-10-13 22:38 +0200
http://bitbucket.org/pypy/pypy/changeset/9824a8b7789a/

Log:	Fix remaining failures in module/exception

diff --git a/pypy/module/exceptions/test/test_exc.py b/pypy/module/exceptions/test/test_exc.py
--- a/pypy/module/exceptions/test/test_exc.py
+++ b/pypy/module/exceptions/test/test_exc.py
@@ -152,8 +152,8 @@
         assert ud.reason == 'bah'
         assert ud.args == ('x', b'y', 1, 5, 'bah')
         assert ud.message == ''
-        ud.object = 'z9'
-        assert ud.object == 'z9'
+        ud.object = b'z9'
+        assert ud.object == b'z9'
         assert str(ud) == "'x' codec can't decode bytes in position 1-4: bah"
         ud.end = 2
         assert str(ud) == "'x' codec can't decode byte 0x39 in position 1: bah"
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -512,7 +512,7 @@
             # hack...
             if ('__module__' in w_self.dict_w and
                 space.isinstance_w(w_self.getdictvalue(space, '__module__'),
-                                               space.w_str)):
+                                               space.w_unicode)):
                 return w_self.getdictvalue(space, '__module__')
             return space.wrap('__builtin__')
 

From noreply at buildbot.pypy.org  Thu Oct 13 22:40:25 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 13 Oct 2011 22:40:25 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: bytes[i] is already a number
Message-ID: <20111013204025.BB1FC82112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48031:005e74019b69
Date: 2011-10-13 22:38 +0200
http://bitbucket.org/pypy/pypy/changeset/005e74019b69/

Log:	bytes[i] is already a number

diff --git a/lib_pypy/struct.py b/lib_pypy/struct.py
--- a/lib_pypy/struct.py
+++ b/lib_pypy/struct.py
@@ -118,7 +118,7 @@
         binary.reverse()
     unsigned = 0
     for i in range(8):
-        unsigned |= ord(binary[i]) << (i * 8)
+        unsigned |= binary[i] << (i * 8)
     return float_unpack(unsigned, size, le)
 
 def round_to_nearest(x):

From noreply at buildbot.pypy.org  Thu Oct 13 22:40:27 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 13 Oct 2011 22:40:27 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Add Exception.__cause__ and __context__
Message-ID: <20111013204027.0210582112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48032:47a171031fd3
Date: 2011-10-13 22:38 +0200
http://bitbucket.org/pypy/pypy/changeset/47a171031fd3/

Log:	Add Exception.__cause__ and __context__

diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -491,7 +491,7 @@
 
         w_value = w_cause = space.w_None
         if nbargs >= 2:
-            w_cause = self.popvalue() # XXX cause?
+            w_cause = self.popvalue()
         if 1:
             w_value = self.popvalue()
         if space.exception_is_valid_obj_as_class_w(w_value):
@@ -499,6 +499,7 @@
             w_value = space.call_function(w_type)
         else:
             w_type = space.type(w_value)
+        w_value.w_cause = w_cause
         operror = OperationError(w_type, w_value)
         operror.normalize_exception(space)
         w_traceback = space.w_None # XXX with_traceback?
diff --git a/pypy/module/exceptions/interp_exceptions.py b/pypy/module/exceptions/interp_exceptions.py
--- a/pypy/module/exceptions/interp_exceptions.py
+++ b/pypy/module/exceptions/interp_exceptions.py
@@ -95,6 +95,8 @@
     """
     w_dict = None
     args_w = []
+    w_cause = None
+    w_context = None
 
     def __init__(self, space):
         self.w_message = space.w_None
@@ -145,6 +147,28 @@
     def descr_setargs(self, space, w_newargs):
         self.args_w = space.fixedview(w_newargs)
 
+    def descr_getcause(self, space):
+        return self.w_cause
+
+    def descr_setcause(self, space, w_newcause):
+        if not (space.is_w(w_newcause, space.w_None) or
+                space.exception_is_valid_class_w(space.type(w_newcause))):
+            raise OperationError(space.w_TypeError, space.wrap(
+                    "exception cause must be None or "
+                    "derive from BaseException"))
+        self.w_cause = w_newcause
+
+    def descr_getcontext(self, space):
+        return self.w_context
+
+    def descr_setcontext(self, space, w_newcontext):
+        if not (space.is_w(w_newcontext, space.w_None) or
+                space.exception_is_valid_class_w(space.type(w_newcontext))):
+            raise OperationError(space.w_TypeError, space.wrap(
+                    "exception context must be None or "
+                    "derive from BaseException"))
+        self.w_context = w_newcontext
+
     def descr_getitem(self, space, w_index):
         return space.getitem(space.newtuple(self.args_w), w_index)
 
@@ -223,6 +247,10 @@
                             W_BaseException.descr_message_del),
     args = GetSetProperty(W_BaseException.descr_getargs,
                           W_BaseException.descr_setargs),
+    __cause__ = GetSetProperty(W_BaseException.descr_getcause,
+                               W_BaseException.descr_setcause),
+    __context__ = GetSetProperty(W_BaseException.descr_getcontext,
+                                 W_BaseException.descr_setcontext),
 )
 
 def _new_exception(name, base, docstring, **kwargs):
diff --git a/pypy/module/exceptions/test/test_exc.py b/pypy/module/exceptions/test/test_exc.py
--- a/pypy/module/exceptions/test/test_exc.py
+++ b/pypy/module/exceptions/test/test_exc.py
@@ -241,3 +241,23 @@
         assert fw.z == 1
         assert fw.xyz == (1, 2)
 
+    def test_cause(self):
+        e1 = TypeError()
+        e2 = ValueError()
+        assert e1.__cause__ is None
+        e1.__cause__ = e2
+        assert e1.__cause__ is e2
+        e1.__cause__ = None
+        raises(TypeError, setattr, e1, '__cause__', 1)
+        raises(AttributeError, delattr, e1, '__cause__')
+
+    def test_context(self):
+        e1 = TypeError()
+        e2 = ValueError()
+        assert e1.__context__ is None
+        e1.__context__ = e2
+        assert e1.__context__ is e2
+        e1.__context__ = None
+        raises(TypeError, setattr, e1, '__context__', 1)
+        raises(AttributeError, delattr, e1, '__context__')
+

From noreply at buildbot.pypy.org  Thu Oct 13 22:40:28 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 13 Oct 2011 22:40:28 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: A few more bytes conversion.
Message-ID: <20111013204028.4087882112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48033:eca0000f43ac
Date: 2011-10-13 22:38 +0200
http://bitbucket.org/pypy/pypy/changeset/eca0000f43ac/

Log:	A few more bytes conversion. Have tokenize.py use the io module
	consistently, to be reverted when pypy installs io.open in builtins

diff --git a/lib-python/3.2/tokenize.py b/lib-python/3.2/tokenize.py
--- a/lib-python/3.2/tokenize.py
+++ b/lib-python/3.2/tokenize.py
@@ -30,7 +30,7 @@
 from token import *
 from codecs import lookup, BOM_UTF8
 import collections
-from io import TextIOWrapper
+import io
 cookie_re = re.compile("coding[:=]\s*([-\w.]+)")
 
 import token
@@ -340,10 +340,10 @@
     """Open a file in read only mode using the encoding detected by
     detect_encoding().
     """
-    buffer = builtins.open(filename, 'rb')
+    buffer = io.open(filename, 'rb')
     encoding, lines = detect_encoding(buffer.readline)
     buffer.seek(0)
-    text = TextIOWrapper(buffer, encoding, line_buffering=True)
+    text = io.TextIOWrapper(buffer, encoding, line_buffering=True)
     text.mode = 'r'
     return text
 
diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py
--- a/pypy/module/_io/interp_fileio.py
+++ b/pypy/module/_io/interp_fileio.py
@@ -352,7 +352,7 @@
             raise wrap_oserror(space, e,
                                exception_name='w_IOError')
 
-        return space.wrap(s)
+        return space.wrapbytes(s)
 
     def readinto_w(self, space, w_buffer):
         self._check_closed(space)
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -37,7 +37,7 @@
     if space.isinstance_w(w_obj, space.w_unicode):
         w_obj = space.call_method(w_obj, 'encode',
                                   getfilesystemencoding(space))
-    return space.str_w(w_obj)
+    return space.bytes_w(w_obj)
 
 class FileEncoder(object):
     def __init__(self, space, w_obj):

From noreply at buildbot.pypy.org  Thu Oct 13 22:40:29 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 13 Oct 2011 22:40:29 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Implement a basic version of
	long.from_bytes, 
Message-ID: <20111013204029.815D382112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48034:6d32eeb3fc38
Date: 2011-10-13 22:38 +0200
http://bitbucket.org/pypy/pypy/changeset/6d32eeb3fc38/

Log:	Implement a basic version of long.from_bytes, it's enough for the
	random module. Also attach it to 'int', until these types are
	unified...

diff --git a/pypy/objspace/std/inttype.py b/pypy/objspace/std/inttype.py
--- a/pypy/objspace/std/inttype.py
+++ b/pypy/objspace/std/inttype.py
@@ -33,6 +33,10 @@
         val >>= 1
     return space.wrap(bits)
 
+ at gateway.unwrap_spec(s='bufferstr', byteorder=str)
+def descr_from_bytes(space, w_cls, s, byteorder):
+    from pypy.objspace.std.longtype import descr_from_bytes
+    return descr_from_bytes(space, space.w_long, s)
 
 def wrapint(space, x):
     if space.config.objspace.std.withsmallint:
@@ -205,5 +209,6 @@
     denominator = typedef.GetSetProperty(descr_get_denominator),
     real = typedef.GetSetProperty(descr_get_real),
     imag = typedef.GetSetProperty(descr_get_imag),
+    from_bytes = gateway.interp2app(descr_from_bytes, as_classmethod=True),
 )
 int_typedef.registermethods(globals())
diff --git a/pypy/objspace/std/longtype.py b/pypy/objspace/std/longtype.py
--- a/pypy/objspace/std/longtype.py
+++ b/pypy/objspace/std/longtype.py
@@ -113,6 +113,15 @@
         raise OperationError(space.w_OverflowError,
                              space.wrap("too many digits in integer"))
 
+ at gateway.unwrap_spec(s='bufferstr', byteorder=str)
+def descr_from_bytes(space, w_cls, s, byteorder):
+    from pypy.rlib.rbigint import rbigint
+    bigint = rbigint.frombytes(s)
+    from pypy.objspace.std.longobject import W_LongObject
+    w_obj = space.allocate_instance(W_LongObject, w_cls)
+    W_LongObject.__init__(w_obj, bigint)
+    return w_obj
+
 # ____________________________________________________________
 
 long_typedef = StdTypeDef("long",
@@ -130,5 +139,6 @@
     real = typedef.GetSetProperty(descr_get_real),
     imag = typedef.GetSetProperty(descr_get_imag),
     bit_length = gateway.interp2app(bit_length),
+    from_bytes = gateway.interp2app(descr_from_bytes, as_classmethod=True),
 )
 long_typedef.registermethods(globals())
diff --git a/pypy/objspace/std/test/test_longobject.py b/pypy/objspace/std/test/test_longobject.py
--- a/pypy/objspace/std/test/test_longobject.py
+++ b/pypy/objspace/std/test/test_longobject.py
@@ -310,6 +310,9 @@
         assert (-1<<40).bit_length() == 41
         assert ((2**31)-1).bit_length() == 31
 
+    def test_from_bytes(self):
+        assert long.from_bytes(b'c', 'little') == 99
+        assert long.from_bytes(b'\x01\x01', 'little') == 257
 
     def test_negative_zero(self):
         x = eval("-0L")
diff --git a/pypy/rlib/rbigint.py b/pypy/rlib/rbigint.py
--- a/pypy/rlib/rbigint.py
+++ b/pypy/rlib/rbigint.py
@@ -215,6 +215,24 @@
         # then modify the result.
         return _decimalstr_to_bigint(s)
 
+    @staticmethod
+    def frombytes(s):
+        accum = 0
+        accumbits = 0
+        digits = []
+        for ch in s:
+            c = ord(ch)
+            accum <<= 8
+            accum |= c
+            accumbits += 8
+            if accumbits >= SHIFT:
+                digits.append(_store_digit(accum & MASK))
+                accum >>= SHIFT
+                accumbits -= SHIFT
+        if accumbits:
+            digits.append(_store_digit(accum))
+        return rbigint(digits, 1)
+
     @jit.elidable
     def toint(self):
         """

From noreply at buildbot.pypy.org  Thu Oct 13 22:40:30 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 13 Oct 2011 22:40:30 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: posix.read() return bytes
Message-ID: <20111013204030.C04DC82112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48035:37c28ac1e0f2
Date: 2011-10-13 22:38 +0200
http://bitbucket.org/pypy/pypy/changeset/37c28ac1e0f2/

Log:	posix.read() return bytes

diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -138,7 +138,7 @@
     except OSError, e:
         raise wrap_oserror(space, e)
     else:
-        return space.wrap(s)
+        return space.wrapbytes(s)
 
 @unwrap_spec(fd=c_int, data='bufferstr')
 def write(space, fd, data):

From noreply at buildbot.pypy.org  Thu Oct 13 22:40:32 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 13 Oct 2011 22:40:32 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Fix typo
Message-ID: <20111013204032.09DC082112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48036:1795d200c7eb
Date: 2011-10-13 22:38 +0200
http://bitbucket.org/pypy/pypy/changeset/1795d200c7eb/

Log:	Fix typo

diff --git a/pypy/objspace/std/inttype.py b/pypy/objspace/std/inttype.py
--- a/pypy/objspace/std/inttype.py
+++ b/pypy/objspace/std/inttype.py
@@ -36,7 +36,7 @@
 @gateway.unwrap_spec(s='bufferstr', byteorder=str)
 def descr_from_bytes(space, w_cls, s, byteorder):
     from pypy.objspace.std.longtype import descr_from_bytes
-    return descr_from_bytes(space, space.w_long, s)
+    return descr_from_bytes(space, space.w_long, s, byteorder)
 
 def wrapint(space, x):
     if space.config.objspace.std.withsmallint:

From noreply at buildbot.pypy.org  Thu Oct 13 23:13:01 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 13 Oct 2011 23:13:01 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: upgrade our pwd.py to new print syntax
Message-ID: <20111013211301.64E0082112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48037:e15fddeec745
Date: 2011-10-13 22:33 +0200
http://bitbucket.org/pypy/pypy/changeset/e15fddeec745/

Log:	upgrade our pwd.py to new print syntax

diff --git a/lib_pypy/pwd.py b/lib_pypy/pwd.py
--- a/lib_pypy/pwd.py
+++ b/lib_pypy/pwd.py
@@ -167,10 +167,10 @@
     from os import getuid
     uid = getuid()
     pw = getpwuid(uid)
-    print "uid %s: %s" % (pw.pw_uid, pw)
+    print("uid %s: %s" % (pw.pw_uid, pw))
     name = pw.pw_name
-    print "name %r: %s" % (name, getpwnam(name))
-    print "All:"
+    print("name %r: %s" % (name, getpwnam(name)))
+    print("All:")
     for pw in getpwall():
-        print pw
+        print(pw)
 

From noreply at buildbot.pypy.org  Thu Oct 13 23:13:02 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Thu, 13 Oct 2011 23:13:02 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: bytes/str stuff in codec errorhandler
Message-ID: <20111013211302.A25E582112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48038:7735d20e8f56
Date: 2011-10-13 23:12 +0200
http://bitbucket.org/pypy/pypy/changeset/7735d20e8f56/

Log:	bytes/str stuff in codec errorhandler

diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -21,12 +21,14 @@
             w_errorhandler = lookup_error(space, errors)
             if decode:
                 w_cls = space.w_UnicodeDecodeError
+                w_input = space.wrapbytes(input)
             else:
                 w_cls = space.w_UnicodeEncodeError
+                w_input = space.wrap(input)
             w_exc =  space.call_function(
                 w_cls,
                 space.wrap(encoding),
-                space.wrap(input),
+                w_input,
                 space.wrap(startpos),
                 space.wrap(endpos),
                 space.wrap(reason))
@@ -59,7 +61,7 @@
             else:
                 from pypy.objspace.std.unicodetype import encode_object
                 w_str = encode_object(space, w_replace, encoding, None)
-                replace = space.str_w(w_str)
+                replace = space.bytes_w(w_str)
                 return replace, newpos
         return unicode_call_errorhandler
 

From noreply at buildbot.pypy.org  Fri Oct 14 00:16:23 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri, 14 Oct 2011 00:16:23 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: utf-8 codec now disallows lone surrogates
Message-ID: <20111013221623.6981A82112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48039:3f9d7e9aa6b6
Date: 2011-10-14 00:12 +0200
http://bitbucket.org/pypy/pypy/changeset/3f9d7e9aa6b6/

Log:	utf-8 codec now disallows lone surrogates the surrogateescape error
	handler can be used for round-tripping unicode strings with lone
	surrogates.

diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -33,11 +33,14 @@
                 space.wrap(endpos),
                 space.wrap(reason))
             w_res = space.call_function(w_errorhandler, w_exc)
-            if (not space.is_true(space.isinstance(w_res, space.w_tuple))
+            if (not space.isinstance_w(w_res, space.w_tuple)
                 or space.len_w(w_res) != 2
-                or not space.is_true(space.isinstance(
-                                 space.getitem(w_res, space.wrap(0)),
-                                 space.w_unicode))):
+                or not (space.isinstance_w(
+                            space.getitem(w_res, space.wrap(0)),
+                            space.w_unicode) or
+                        (not decode and space.isinstance_w(
+                            space.getitem(w_res, space.wrap(0)),
+                            space.w_bytes)))):
                 if decode:
                     msg = ("decoding error handler must return "
                            "(unicode, int) tuple, not %s")
@@ -60,8 +63,9 @@
                 return replace, newpos
             else:
                 from pypy.objspace.std.unicodetype import encode_object
-                w_str = encode_object(space, w_replace, encoding, None)
-                replace = space.bytes_w(w_str)
+                if space.isinstance_w(w_replace, space.w_unicode):
+                    w_replace = encode_object(space, w_replace, encoding, None)
+                replace = space.bytes_w(w_replace)
                 return replace, newpos
         return unicode_call_errorhandler
 
@@ -246,11 +250,51 @@
         raise operationerrfmt(space.w_TypeError,
             "don't know how to handle %s in error callback", typename)
 
+def surrogateescape_errors(space, w_exc):
+    check_exception(space, w_exc)
+    if space.isinstance_w(w_exc, space.w_UnicodeEncodeError):
+        obj = space.realunicode_w(space.getattr(w_exc, space.wrap('object')))
+        start = space.int_w(space.getattr(w_exc, space.wrap('start')))
+        w_end = space.getattr(w_exc, space.wrap('end'))
+        end = space.int_w(w_end)
+        res = ''
+        pos = start
+        while pos < end:
+            ch = ord(obj[pos])
+            pos += 1
+            if ch < 0xdc80 or ch > 0xdcff:
+                # Not a UTF-8b surrogate, fail with original exception
+                raise OperationError(space.type(w_exc), w_exc)
+            res += chr(ch - 0xdc00)
+        return space.newtuple([space.wrapbytes(res), w_end])
+    elif space.isinstance_w(w_exc, space.w_UnicodeDecodeError):
+        consumed = 0
+        start = space.int_w(space.getattr(w_exc, space.wrap('start')))
+        end = space.int_w(space.getattr(w_exc, space.wrap('end')))
+        obj = space.bytes_w(space.getattr(w_exc, space.wrap('object')))
+        replace = u''
+        while consumed < 4 and consumed < end - start:
+            c = ord(obj[start+consumed])
+            if c < 128:
+                # Refuse to escape ASCII bytes.
+                break
+            replace += unichr(0xdc00 + c)
+            consumed += 1
+        if not consumed:
+            # codec complained about ASCII byte.
+            raise OperationError(space.type(w_exc), w_exc)
+        return space.newtuple([space.wrap(replace),
+                               space.wrap(start + consumed)])
+    else:
+        typename = space.type(w_exc).getname(space)
+        raise operationerrfmt(space.w_TypeError,
+            "don't know how to handle %s in error callback", typename)
+
 def register_builtin_error_handlers(space):
     "NOT_RPYTHON"
     state = space.fromcache(CodecState)
     for error in ("strict", "ignore", "replace", "xmlcharrefreplace",
-                  "backslashreplace"):
+                  "backslashreplace", "surrogateescape"):
         name = error + "_errors"
         state.codec_error_registry[error] = space.wrap(interp2app(globals()[name]))
 
diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py
--- a/pypy/module/_codecs/test/test_codecs.py
+++ b/pypy/module/_codecs/test/test_codecs.py
@@ -483,6 +483,10 @@
     def test_backslahreplace(self):
         assert u'a\xac\u1234\u20ac\u8000'.encode('ascii', 'backslashreplace') == 'a\\xac\u1234\u20ac\u8000'
 
+    def test_surrogateescape(self):
+        assert b'a\x80b'.decode('utf-8', 'surrogateescape') == 'a\udc80b'
+        assert 'a\udc80b'.encode('utf-8', 'surrogateescape') == b'a\x80b'
+
     def test_badhandler(self):
         import codecs
         results = ( 42, u"foo", (1,2,3), (u"foo", 1, 3), (u"foo", None), (u"foo",), ("foo", 1, 3), ("foo", None), ("foo",) )
diff --git a/pypy/rlib/runicode.py b/pypy/rlib/runicode.py
--- a/pypy/rlib/runicode.py
+++ b/pypy/rlib/runicode.py
@@ -255,10 +255,10 @@
 def unicode_encode_utf_8(s, size, errors, errorhandler=None):
     assert(size >= 0)
     result = StringBuilder(size)
-    i = 0
-    while i < size:
-        ch = ord(s[i])
-        i += 1
+    pos = 0
+    while pos < size:
+        ch = ord(s[pos])
+        pos += 1
         if ch < 0x80:
             # Encode ASCII
             result.append(chr(ch))
@@ -270,19 +270,19 @@
             # Encode UCS2 Unicode ordinals
             if ch < 0x10000:
                 # Special case: check for high surrogate
-                if 0xD800 <= ch <= 0xDBFF and i != size:
-                    ch2 = ord(s[i])
+                if 0xD800 <= ch <= 0xDBFF and pos != size:
+                    ch2 = ord(s[pos])
                     # Check for low surrogate and combine the two to
                     # form a UCS4 value
                     if 0xDC00 <= ch2 <= 0xDFFF:
                         ch3 = ((ch - 0xD800) << 10 | (ch2 - 0xDC00)) + 0x10000
-                        i += 1
+                        pos += 1
                         _encodeUCS4(result, ch3)
                         continue
-                # Fall through: handles isolated high surrogates
-                result.append((chr((0xe0 | (ch >> 12)))))
-                result.append((chr((0x80 | ((ch >> 6) & 0x3f)))))
-                result.append((chr((0x80 | (ch & 0x3f)))))
+                r, pos = errorhandler(errors, 'utf-8',
+                                      'surrogates not allowed',
+                                      s, pos-1, pos)
+                result.append(r)
                 continue
             else:
                 _encodeUCS4(result, ch)

From noreply at buildbot.pypy.org  Fri Oct 14 00:52:45 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri, 14 Oct 2011 00:52:45 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: iterators now must implement the __next__
	method.
Message-ID: <20111013225245.2FD0E82112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48040:f403ac4c1c5c
Date: 2011-10-14 00:39 +0200
http://bitbucket.org/pypy/pypy/changeset/f403ac4c1c5c/

Log:	iterators now must implement the __next__ method.

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1491,7 +1491,7 @@
     ('coerce',          'coerce',    2, ['__coerce__', '__coerce__']),
     ('contains',        'contains',  2, ['__contains__']),
     ('iter',            'iter',      1, ['__iter__']),
-    ('next',            'next',      1, ['next']),
+    ('next',            'next',      1, ['__next__']),
 #    ('call',            'call',      3, ['__call__']),
     ('get',             'get',       3, ['__get__']),
     ('set',             'set',       3, ['__set__']),
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -861,8 +861,8 @@
 GeneratorIterator.typedef = TypeDef("generator",
     __repr__   = interp2app(GeneratorIterator.descr__repr__),
     __reduce__   = interp2app(GeneratorIterator.descr__reduce__),
-    next       = interp2app(GeneratorIterator.descr_next,
-                            descrmismatch='next'),
+    __next__   = interp2app(GeneratorIterator.descr_next,
+                            descrmismatch='__next__'),
     send       = interp2app(GeneratorIterator.descr_send,
                             descrmismatch='send'),
     throw      = interp2app(GeneratorIterator.descr_throw,
diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py
--- a/pypy/module/__builtin__/functional.py
+++ b/pypy/module/__builtin__/functional.py
@@ -372,7 +372,7 @@
 W_Enumerate.typedef = TypeDef("enumerate",
     __new__=interp2app(W_Enumerate.descr___new__.im_func),
     __iter__=interp2app(W_Enumerate.descr___iter__),
-    next=interp2app(W_Enumerate.descr_next),
+    __next__=interp2app(W_Enumerate.descr_next),
     __reduce__=interp2app(W_Enumerate.descr___reduce__),
 )
 
@@ -429,7 +429,7 @@
 
 W_ReversedIterator.typedef = TypeDef("reversed",
     __iter__=interp2app(W_ReversedIterator.descr___iter__),
-    next=interp2app(W_ReversedIterator.descr_next),
+    __next__=interp2app(W_ReversedIterator.descr_next),
     __reduce__=interp2app(W_ReversedIterator.descr___reduce__),
 )
 
@@ -555,6 +555,6 @@
     __iter__        = interp2app(W_RangeIterator.descr_iter),
 # XXX __length_hint__()
 ##    __len__         = interp2app(W_RangeIterator.descr_len),
-    next            = interp2app(W_RangeIterator.descr_next),
+    __next__        = interp2app(W_RangeIterator.descr_next),
     __reduce__      = interp2app(W_RangeIterator.descr_reduce),
 )
diff --git a/pypy/module/__builtin__/interp_classobj.py b/pypy/module/__builtin__/interp_classobj.py
--- a/pypy/module/__builtin__/interp_classobj.py
+++ b/pypy/module/__builtin__/interp_classobj.py
@@ -496,8 +496,8 @@
                 space.w_TypeError,
                 space.wrap("iteration over non-sequence"))
         return space.newseqiter(self)
-    #XXX do I really need a next method? the old implementation had one, but I
-    # don't see the point
+    # XXX do I really need a __next__ method? the old implementation
+    # had one, but I don't see the point
 
     def descr_call(self, space, __args__):
         w_meth = self.getattr(space, '__call__')
@@ -674,10 +674,10 @@
             return space.w_NotImplemented
 
     def descr_next(self, space):
-        w_func = self.getattr(space, 'next', False)
+        w_func = self.getattr(space, '__next__', False)
         if w_func is None:
-            raise OperationError(space.w_TypeError,
-                                 space.wrap("instance has no next() method"))
+            raise OperationError(space.w_TypeError, space.wrap(
+                    "instance has no __next__() method"))
         return space.call_function(w_func)
 
     def descr_del(self, space):
@@ -759,7 +759,7 @@
     __contains__ = interp2app(W_InstanceObject.descr_contains),
     __pow__ = interp2app(W_InstanceObject.descr_pow),
     __rpow__ = interp2app(W_InstanceObject.descr_rpow),
-    next = interp2app(W_InstanceObject.descr_next),
+    __next__ = interp2app(W_InstanceObject.descr_next),
     __del__ = interp2app(W_InstanceObject.descr_del),
     __exit__ = interp2app(W_InstanceObject.descr_exit),
     __dict__ = dict_descr,
diff --git a/pypy/module/_collections/interp_deque.py b/pypy/module/_collections/interp_deque.py
--- a/pypy/module/_collections/interp_deque.py
+++ b/pypy/module/_collections/interp_deque.py
@@ -533,7 +533,7 @@
 
 W_DequeIter.typedef = TypeDef("deque_iterator",
     __iter__ = interp2app(W_DequeIter.iter),
-    next = interp2app(W_DequeIter.next),
+    __next__ = interp2app(W_DequeIter.next),
 )
 W_DequeIter.typedef.acceptable_as_base_class = False
 
@@ -568,7 +568,7 @@
 
 W_DequeRevIter.typedef = TypeDef("deque_reverse_iterator",
     __iter__ = interp2app(W_DequeRevIter.iter),
-    next = interp2app(W_DequeRevIter.next),
+    __next__ = interp2app(W_DequeRevIter.next),
 )
 W_DequeRevIter.typedef.acceptable_as_base_class = False
 
diff --git a/pypy/module/_file/interp_file.py b/pypy/module/_file/interp_file.py
--- a/pypy/module/_file/interp_file.py
+++ b/pypy/module/_file/interp_file.py
@@ -158,7 +158,7 @@
     def direct_flush(self):
         self.getstream().flush()
 
-    def direct_next(self):
+    def direct___next__(self):
         line = self.getstream().readline()
         if line == '':
             raise OperationError(self.space.w_StopIteration, self.space.w_None)
@@ -344,8 +344,8 @@
     _decl(locals(), "isatty",
         """isatty() -> true or false.  True if the file is connected to a tty device.""")
 
-    _decl(locals(), "next",
-        """next() -> the next line in the file, or raise StopIteration""")
+    _decl(locals(), "__next__",
+        """__next__() -> the next line in the file, or raise StopIteration""")
 
     _decl(locals(), "read",
         """read([size]) -> read at most size bytes, returned as a string.
diff --git a/pypy/module/_io/interp_iobase.py b/pypy/module/_io/interp_iobase.py
--- a/pypy/module/_io/interp_iobase.py
+++ b/pypy/module/_io/interp_iobase.py
@@ -241,7 +241,7 @@
     __enter__ = interp2app(W_IOBase.enter_w),
     __exit__ = interp2app(W_IOBase.exit_w),
     __iter__ = interp2app(W_IOBase.iter_w),
-    next = interp2app(W_IOBase.next_w),
+    __next__ = interp2app(W_IOBase.next_w),
     close = interp2app(W_IOBase.close_w),
     flush = interp2app(W_IOBase.flush_w),
     tell = interp2app(W_IOBase.tell_w),
diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py
--- a/pypy/module/_io/interp_textio.py
+++ b/pypy/module/_io/interp_textio.py
@@ -980,7 +980,7 @@
     __repr__ = interp2app(W_TextIOWrapper.descr_repr),
     __module__ = "_io",
 
-    next = interp2app(W_TextIOWrapper.next_w),
+    __next__ = interp2app(W_TextIOWrapper.next_w),
     read = interp2app(W_TextIOWrapper.read_w),
     readline = interp2app(W_TextIOWrapper.readline_w),
     write = interp2app(W_TextIOWrapper.write_w),
diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py
--- a/pypy/module/_sre/interp_sre.py
+++ b/pypy/module/_sre/interp_sre.py
@@ -542,7 +542,7 @@
 W_SRE_Scanner.typedef = TypeDef(
     'SRE_Scanner',
     __iter__ = interp2app(W_SRE_Scanner.iter_w),
-    next     = interp2app(W_SRE_Scanner.next_w),
+    __next__ = interp2app(W_SRE_Scanner.next_w),
     match    = interp2app(W_SRE_Scanner.match_w),
     search   = interp2app(W_SRE_Scanner.search_w),
     pattern  = interp_attrproperty('srepat', W_SRE_Scanner),
diff --git a/pypy/module/cStringIO/interp_stringio.py b/pypy/module/cStringIO/interp_stringio.py
--- a/pypy/module/cStringIO/interp_stringio.py
+++ b/pypy/module/cStringIO/interp_stringio.py
@@ -32,7 +32,7 @@
         self.check_closed()
         return self.space.w_False
 
-    def descr_next(self):
+    def descr___next__(self):
         space = self.space
         self.check_closed()
         line = self.readline()
@@ -207,11 +207,11 @@
 
 common_descrs = {
     '__iter__':     interp2app(W_InputOutputType.descr___iter__),
+    '__next__':     interp2app(W_InputOutputType.descr___next__),
     'close':        interp2app(W_InputOutputType.descr_close),
     'flush':        interp2app(W_InputOutputType.descr_flush),
     'getvalue':     interp2app(W_InputOutputType.descr_getvalue),
     'isatty':       interp2app(W_InputOutputType.descr_isatty),
-    'next':         interp2app(W_InputOutputType.descr_next),
     'read':         interp2app(W_InputOutputType.descr_read),
     'readline':     interp2app(W_InputOutputType.descr_readline),
     'readlines':    interp2app(W_InputOutputType.descr_readlines),
diff --git a/pypy/module/cpyext/dictobject.py b/pypy/module/cpyext/dictobject.py
--- a/pypy/module/cpyext/dictobject.py
+++ b/pypy/module/cpyext/dictobject.py
@@ -177,10 +177,10 @@
         w_iter = space.call_method(w_dict, "iteritems")
         pos = ppos[0]
         while pos:
-            space.call_method(w_iter, "next")
+            space.next(w_iter)
             pos -= 1
 
-        w_item = space.call_method(w_iter, "next")
+        w_item = space.next(w_iter)
         w_key, w_value = space.fixedview(w_item, 2)
         state = space.fromcache(RefcountState)
         pkey[0]   = state.make_borrowed(w_dict, w_key)
diff --git a/pypy/module/cpyext/iterator.py b/pypy/module/cpyext/iterator.py
--- a/pypy/module/cpyext/iterator.py
+++ b/pypy/module/cpyext/iterator.py
@@ -40,7 +40,7 @@
 def PyIter_Check(space, w_obj):
     """Return true if the object o supports the iterator protocol."""
     try:
-        w_attr = space.getattr(space.type(w_obj), space.wrap("next"))
+        w_attr = space.getattr(space.type(w_obj), space.wrap("__next__"))
     except:
         return False
     else:
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -568,8 +568,8 @@
                "x.__ge__(y) <==> x>=y"),
         TPSLOT("__iter__", tp_iter, slot_tp_iter, wrap_unaryfunc,
                "x.__iter__() <==> iter(x)"),
-        TPSLOT("next", tp_iternext, slot_tp_iternext, wrap_next,
-               "x.next() -> the next value, or raise StopIteration"),
+        TPSLOT("__next__", tp_iternext, slot_tp_iternext, wrap_next,
+               "x.__next__() -> the next value, or raise StopIteration"),
         TPSLOT("__get__", tp_descr_get, slot_tp_descr_get, wrap_descr_get,
                "descr.__get__(obj[, type]) -> value"),
         TPSLOT("__set__", tp_descr_set, slot_tp_descr_set, wrap_descr_set,
diff --git a/pypy/module/exceptions/interp_exceptions.py b/pypy/module/exceptions/interp_exceptions.py
--- a/pypy/module/exceptions/interp_exceptions.py
+++ b/pypy/module/exceptions/interp_exceptions.py
@@ -367,7 +367,7 @@
                             __str__ = key_error_str)
 
 W_StopIteration = _new_exception('StopIteration', W_Exception,
-                                 """Signal the end from iterator.next().""")
+                                 """Signal the end from iterator.__next__().""")
 
 W_Warning = _new_exception('Warning', W_Exception,
                            """Base class for warning categories.""")
diff --git a/pypy/module/itertools/interp_itertools.py b/pypy/module/itertools/interp_itertools.py
--- a/pypy/module/itertools/interp_itertools.py
+++ b/pypy/module/itertools/interp_itertools.py
@@ -61,7 +61,7 @@
         __module__ = 'itertools',
         __new__ = interp2app(W_Count___new__),
         __iter__ = interp2app(W_Count.iter_w),
-        next = interp2app(W_Count.next_w),
+        __next__ = interp2app(W_Count.next_w),
         __reduce__ = interp2app(W_Count.reduce_w),
         __repr__ = interp2app(W_Count.repr_w),
         __doc__ = """Make an iterator that returns consecutive integers starting
@@ -122,7 +122,7 @@
         __module__ = 'itertools',
         __new__  = interp2app(W_Repeat___new__),
         __iter__ = interp2app(W_Repeat.iter_w),
-        next     = interp2app(W_Repeat.next_w),
+        __next__ = interp2app(W_Repeat.next_w),
         __repr__ = interp2app(W_Repeat.repr_w),
         __doc__  = """Make an iterator that returns object over and over again.
     Runs indefinitely unless the times argument is specified.  Used
@@ -175,7 +175,7 @@
         __module__ = 'itertools',
         __new__  = interp2app(W_TakeWhile___new__),
         __iter__ = interp2app(W_TakeWhile.iter_w),
-        next     = interp2app(W_TakeWhile.next_w),
+        __next__ = interp2app(W_TakeWhile.next_w),
         __doc__  = """Make an iterator that returns elements from the iterable as
     long as the predicate is true.
 
@@ -224,7 +224,7 @@
         __module__ = 'itertools',
         __new__  = interp2app(W_DropWhile___new__),
         __iter__ = interp2app(W_DropWhile.iter_w),
-        next     = interp2app(W_DropWhile.next_w),
+        __next__ = interp2app(W_DropWhile.next_w),
         __doc__  = """Make an iterator that drops elements from the iterable as long
     as the predicate is true; afterwards, returns every
     element. Note, the iterator does not produce any output until the
@@ -281,7 +281,7 @@
         __module__ = 'itertools',
         __new__  = interp2app(W_IFilter___new__),
         __iter__ = interp2app(W_IFilter.iter_w),
-        next     = interp2app(W_IFilter.next_w),
+        __next__ = interp2app(W_IFilter.next_w),
         __doc__  = """Make an iterator that filters elements from iterable returning
     only those for which the predicate is True.  If predicate is
     None, return the items that are true.
@@ -309,7 +309,7 @@
         __module__ = 'itertools',
         __new__  = interp2app(W_IFilterFalse___new__),
         __iter__ = interp2app(W_IFilterFalse.iter_w),
-        next     = interp2app(W_IFilterFalse.next_w),
+        __next__ = interp2app(W_IFilterFalse.next_w),
         __doc__  = """Make an iterator that filters elements from iterable returning
     only those for which the predicate is False.  If predicate is
     None, return the items that are false.
@@ -411,7 +411,7 @@
         __module__ = 'itertools',
         __new__  = interp2app(W_ISlice___new__),
         __iter__ = interp2app(W_ISlice.iter_w),
-        next     = interp2app(W_ISlice.next_w),
+        __next__ = interp2app(W_ISlice.next_w),
         __doc__  = """Make an iterator that returns selected elements from the
     iterable.  If start is non-zero, then elements from the iterable
     are skipped until start is reached. Afterward, elements are
@@ -479,7 +479,7 @@
         __module__ = 'itertools',
         __new__  = interp2app(W_Chain___new__),
         __iter__ = interp2app(W_Chain.iter_w),
-        next     = interp2app(W_Chain.next_w),
+        __next__ = interp2app(W_Chain.next_w),
         from_iterable = interp2app(chain_from_iterable, as_classmethod=True),
         __doc__  = """Make an iterator that returns elements from the first iterable
     until it is exhausted, then proceeds to the next iterable, until
@@ -560,7 +560,7 @@
         __module__ = 'itertools',
         __new__  = interp2app(W_IMap___new__),
         __iter__ = interp2app(W_IMap.iter_w),
-        next     = interp2app(W_IMap.next_w),
+        __next__ = interp2app(W_IMap.next_w),
         __doc__  = """Make an iterator that computes the function using arguments
     from each of the iterables. If function is set to None, then
     imap() returns the arguments as a tuple. Like map() but stops
@@ -605,7 +605,7 @@
         __module__ = 'itertools',
         __new__  = interp2app(W_IZip___new__),
         __iter__ = interp2app(W_IZip.iter_w),
-        next     = interp2app(W_IZip.next_w),
+        __next__ = interp2app(W_IZip.next_w),
         __doc__  = """Make an iterator that aggregates elements from each of the
     iterables.  Like zip() except that it returns an iterator instead
     of a list. Used for lock-step iteration over several iterables at
@@ -674,7 +674,7 @@
         __module__ = 'itertools',
         __new__  = interp2app(W_IZipLongest___new__),
         __iter__ = interp2app(W_IZipLongest.iter_w),
-        next     = interp2app(W_IZipLongest.next_w),
+        __next__ = interp2app(W_IZipLongest.next_w),
         __doc__  = """Return an izip_longest object whose .next() method returns a tuple where
     the i-th element comes from the i-th iterable argument.  The .next()
     method continues until the longest iterable in the argument sequence
@@ -734,7 +734,7 @@
         __module__ = 'itertools',
         __new__  = interp2app(W_Cycle___new__),
         __iter__ = interp2app(W_Cycle.iter_w),
-        next     = interp2app(W_Cycle.next_w),
+        __next__ = interp2app(W_Cycle.next_w),
         __doc__  = """Make an iterator returning elements from the iterable and
     saving a copy of each. When the iterable is exhausted, return
     elements from the saved copy. Repeats indefinitely.
@@ -775,7 +775,7 @@
         __module__ = 'itertools',
         __new__  = interp2app(W_StarMap___new__),
         __iter__ = interp2app(W_StarMap.iter_w),
-        next     = interp2app(W_StarMap.next_w),
+        __next__ = interp2app(W_StarMap.next_w),
         __doc__  = """Make an iterator that computes the function using arguments
     tuples obtained from the iterable. Used instead of imap() when
     argument parameters are already grouped in tuples from a single
@@ -880,7 +880,7 @@
         __module__ = 'itertools',
         __new__ = interp2app(W_TeeIterable___new__),
         __iter__ = interp2app(W_TeeIterable.iter_w),
-        next     = interp2app(W_TeeIterable.next_w),
+        __next__ = interp2app(W_TeeIterable.next_w),
         __weakref__ = make_weakref_descr(W_TeeIterable),
         )
 W_TeeIterable.typedef.acceptable_as_base_class = False
@@ -983,7 +983,7 @@
         __module__ = 'itertools',
         __new__  = interp2app(W_GroupBy___new__),
         __iter__ = interp2app(W_GroupBy.iter_w),
-        next     = interp2app(W_GroupBy.next_w),
+        __next__ = interp2app(W_GroupBy.next_w),
         __doc__  = """Make an iterator that returns consecutive keys and groups from the
     iterable. The key is a function computing a key value for each
     element. If not specified or is None, key defaults to an identity
@@ -1029,7 +1029,7 @@
         '_groupby',
         __module__ = 'itertools',
         __iter__ = interp2app(W_GroupByIterator.iter_w),
-        next     = interp2app(W_GroupByIterator.next_w))
+        __next__ = interp2app(W_GroupByIterator.next_w))
 W_GroupByIterator.typedef.acceptable_as_base_class = False
 
 
@@ -1062,7 +1062,7 @@
     __module__ = 'itertools',
     __new__ = interp2app(W_Compress__new__),
     __iter__ = interp2app(W_Compress.iter_w),
-    next     = interp2app(W_Compress.next_w),
+    __next__ = interp2app(W_Compress.next_w),
     __doc__ = """Make an iterator that filters elements from *data* returning
    only those that have a corresponding element in *selectors* that evaluates to
    ``True``.  Stops when either the *data* or *selectors* iterables has been
@@ -1152,7 +1152,7 @@
     __module__ = 'itertools',
     __new__ = interp2app(W_Product__new__),
     __iter__ = interp2app(W_Product.iter_w),
-    next = interp2app(W_Product.next_w),
+    __next__ = interp2app(W_Product.next_w),
     __doc__ = """
    Cartesian product of input iterables.
 
@@ -1256,7 +1256,7 @@
     __module__ = 'itertools',
     __new__ = interp2app(W_Combinations__new__),
     __iter__ = interp2app(W_Combinations.descr__iter__),
-    next = interp2app(W_Combinations.descr_next),
+    __next__ = interp2app(W_Combinations.descr_next),
     __doc__ = """\
 combinations(iterable, r) --> combinations object
 
@@ -1291,7 +1291,7 @@
     __module__ = 'itertools',
     __new__ = interp2app(W_CombinationsWithReplacement__new__),
     __iter__ = interp2app(W_CombinationsWithReplacement.descr__iter__),
-    next = interp2app(W_CombinationsWithReplacement.descr_next),
+    __next__ = interp2app(W_CombinationsWithReplacement.descr_next),
     __doc__ = """\
 combinations_with_replacement(iterable, r) --> combinations_with_replacement object
 
@@ -1357,7 +1357,7 @@
     __module__ = 'itertools',
     __new__ = interp2app(W_Permutations__new__),
     __iter__ = interp2app(W_Permutations.descr__iter__),
-    next = interp2app(W_Permutations.descr_next),
+    __next__ = interp2app(W_Permutations.descr_next),
     __doc__ = """\
 permutations(iterable[, r]) --> permutations object
 
diff --git a/pypy/module/oracle/interp_cursor.py b/pypy/module/oracle/interp_cursor.py
--- a/pypy/module/oracle/interp_cursor.py
+++ b/pypy/module/oracle/interp_cursor.py
@@ -1083,7 +1083,7 @@
     setoutputsize = interp2app(W_Cursor.setoutputsize),
 
     __iter__ = interp2app(W_Cursor.descr_iter),
-    next = interp2app(W_Cursor.descr_next),
+    __next__ = interp2app(W_Cursor.descr_next),
 
     arraysize = GetSetProperty(W_Cursor.arraysize_get,
                                W_Cursor.arraysize_set),
diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -279,14 +279,14 @@
                                       typename)
             return space.newseqiter(w_obj)
         w_iter = space.get_and_call_function(w_descr, w_obj)
-        w_next = space.lookup(w_iter, 'next')
+        w_next = space.lookup(w_iter, '__next__')
         if w_next is None:
             raise OperationError(space.w_TypeError,
                                  space.wrap("iter() returned non-iterator"))
         return w_iter
 
     def next(space, w_obj):
-        w_descr = space.lookup(w_obj, 'next')
+        w_descr = space.lookup(w_obj, '__next__')
         if w_descr is None:
             typename = space.type(w_obj).getname(space)
             raise operationerrfmt(space.w_TypeError,

From noreply at buildbot.pypy.org  Fri Oct 14 00:52:46 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri, 14 Oct 2011 00:52:46 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: More wrapbytes in the io module.
Message-ID: <20111013225246.6987E82A88@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48041:29eb69170a85
Date: 2011-10-14 00:52 +0200
http://bitbucket.org/pypy/pypy/changeset/29eb69170a85/

Log:	More wrapbytes in the io module.

diff --git a/pypy/module/_io/interp_bufferedio.py b/pypy/module/_io/interp_bufferedio.py
--- a/pypy/module/_io/interp_bufferedio.py
+++ b/pypy/module/_io/interp_bufferedio.py
@@ -314,7 +314,7 @@
         self._writer_reset_buf()
 
     def _write(self, space, data):
-        w_data = space.wrap(data)
+        w_data = space.wrapbytes(data)
         w_written = space.call_method(self.w_raw, "write", w_data)
         written = space.getindex_w(w_written, space.w_IOError)
         if not 0 <= written <= len(data):
@@ -379,7 +379,7 @@
         else:
             raise OperationError(space.w_ValueError, space.wrap(
                 "read length must be positive or -1"))
-        return space.wrap(res)
+        return space.wrapbytes(res)
 
     @unwrap_spec(size=int)
     def peek_w(self, space, size=0):
@@ -396,7 +396,7 @@
             have = self._readahead()
             if have > 0:
                 data = ''.join(self.buffer[self.pos:self.pos+have])
-                return space.wrap(data)
+                return space.wrapbytes(data)
 
             # Fill the buffer from the raw stream, and copy it to the result
             self._reader_reset_buf()
@@ -406,7 +406,7 @@
                 size = 0
             self.pos = 0
             data = ''.join(self.buffer[:size])
-            return space.wrap(data)
+            return space.wrapbytes(data)
 
     @unwrap_spec(size=int)
     def read1_w(self, space, size):
@@ -417,7 +417,7 @@
             raise OperationError(space.w_ValueError, space.wrap(
                 "read length must be positive"))
         if size == 0:
-            return space.wrap("")
+            return space.wrapbytes("")
 
         with self.lock:
             if self.writable:
@@ -445,7 +445,7 @@
             endpos = self.pos + size
             data = ''.join(self.buffer[self.pos:endpos])
             self.pos = endpos
-            return space.wrap(data)
+            return space.wrapbytes(data)
 
     def _read_all(self, space):
         "Read all the file, don't update the cache"
@@ -476,7 +476,7 @@
             current_size += size
             if self.abs_pos != -1:
                 self.abs_pos += size
-        return space.wrap(builder.build())
+        return space.wrapbytes(builder.build())
 
     def _raw_read(self, space, buffer, start, length):
         length = intmask(length)
diff --git a/pypy/module/_io/interp_iobase.py b/pypy/module/_io/interp_iobase.py
--- a/pypy/module/_io/interp_iobase.py
+++ b/pypy/module/_io/interp_iobase.py
@@ -164,7 +164,7 @@
                 length = space.len_w(w_readahead)
                 if length > 0:
                     n = 0
-                    buf = space.str_w(w_readahead)
+                    buf = space.bytes_w(w_readahead)
                     if limit >= 0:
                         while True:
                             if n >= length or n >= limit:
@@ -187,7 +187,7 @@
                     space.w_IOError,
                     "peek() should have returned a bytes object, "
                     "not '%s'", space.type(w_read).getname(space))
-            read = space.str_w(w_read)
+            read = space.bytes_w(w_read)
             if not read:
                 break
 
@@ -197,7 +197,7 @@
             if read[-1] == '\n':
                 break
 
-        return space.wrap(builder.build())
+        return space.wrapbytes(builder.build())
 
     def readlines_w(self, space, w_hint=None):
         hint = convert_size(space, w_hint)
@@ -286,11 +286,11 @@
             if not space.isinstance_w(w_data, space.w_str):
                 raise OperationError(space.w_TypeError, space.wrap(
                     "read() should return bytes"))
-            data = space.str_w(w_data)
+            data = space.bytes_w(w_data)
             if not data:
                 break
             builder.append(data)
-        return space.wrap(builder.build())
+        return space.wrapbytes(builder.build())
 
 W_RawIOBase.typedef = TypeDef(
     '_RawIOBase', W_IOBase.typedef,

From noreply at buildbot.pypy.org  Fri Oct 14 01:59:32 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri, 14 Oct 2011 01:59:32 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: popen() and fdopen() are now implemented
	in os.py
Message-ID: <20111013235932.B9A5D82112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48042:ed169efaf72c
Date: 2011-10-14 01:22 +0200
http://bitbucket.org/pypy/pypy/changeset/ed169efaf72c/

Log:	popen() and fdopen() are now implemented in os.py

diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py
--- a/pypy/module/posix/__init__.py
+++ b/pypy/module/posix/__init__.py
@@ -35,9 +35,7 @@
     appleveldefs = {
     'error'      : 'app_posix.error',
     'stat_result': 'app_posix.stat_result',
-    'fdopen'     : 'app_posix.fdopen',
     'tmpfile'    : 'app_posix.tmpfile',
-    'popen'      : 'app_posix.popen',
     'tmpnam'     : 'app_posix.tmpnam',
     'tempnam'    : 'app_posix.tempnam',
     }
diff --git a/pypy/module/posix/app_posix.py b/pypy/module/posix/app_posix.py
--- a/pypy/module/posix/app_posix.py
+++ b/pypy/module/posix/app_posix.py
@@ -78,18 +78,6 @@
         # XXX for the moment
         return
 
-# Capture file.fdopen at import time, as some code replaces
-# __builtins__.file with a custom function.
-_fdopen = file.fdopen
-
-def fdopen(fd, mode='r', buffering=-1):
-    """fdopen(fd [, mode='r' [, buffering]]) -> file_object
-
-    Return an open file object connected to a file descriptor."""
-    _validate_fd(fd)
-    return _fdopen(fd, mode, buffering)
-
-
 def tmpfile():
     """Create a temporary file.
 
@@ -123,76 +111,7 @@
     return tempfile.mktemp('', prefix or 'tmp', dir)
 
 
-# Implement popen() for platforms which have os.fork()
 if osname == 'posix':
-
-    class popenfile(file):
-        _childpid = None
-
-        def close(self):
-            import os
-            super(popenfile, self).close()
-            pid = self._childpid
-            if pid is not None:
-                self._childpid = None
-                sts = os.waitpid(pid, 0)[1]
-                if sts != 0:
-                    return sts
-        __del__ = close     # as in CPython, __del__ may call os.waitpid()
-
-    def popen(command, mode='r', bufsize=-1):
-        """popen(command [, mode='r' [, bufsize]]) -> pipe
-
-        Open a pipe to/from a command returning a file object."""
-
-        from popen2 import MAXFD
-        import os, gc
-
-        def try_close(fd):
-            try:
-                os.close(fd)
-            except OSError:
-                pass
-
-        if not mode.startswith('r') and not mode.startswith('w'):
-            raise ValueError("invalid mode %r" % (mode,))
-        read_end, write_end = os.pipe()
-        try:
-            gc.disable_finalizers()
-            try:
-                childpid = os.fork()
-                if childpid == 0:
-                    # in the child
-                    try:
-                        if mode.startswith('r'):
-                            os.dup2(write_end, 1)
-                            os.close(read_end)
-                        else:
-                            os.dup2(read_end, 0)
-                            os.close(write_end)
-                        os.closerange(3, MAXFD)
-                        cmd = ['/bin/sh', '-c', command]
-                        os.execvp(cmd[0], cmd)
-                    finally:
-                        os._exit(1)
-            finally:
-                gc.enable_finalizers()
-
-            if mode.startswith('r'):
-                os.close(write_end)
-                fd = read_end
-            else:
-                os.close(read_end)
-                fd = write_end
-            g = popenfile.fdopen(fd, mode, bufsize)
-            g._childpid = childpid
-            return g
-
-        except Exception, e:
-            try_close(write_end)
-            try_close(read_end)
-            raise Exception() from e
-
     def wait():
         """ wait() -> (pid, status)
 
@@ -219,36 +138,6 @@
 else:
     # Windows implementations
 
-    # Supply os.popen() based on subprocess
-    def popen(cmd, mode="r", bufsize=-1):
-        """popen(command [, mode='r' [, bufsize]]) -> pipe
-
-        Open a pipe to/from a command returning a file object."""
-
-        cmd = _makecmd_string(cmd)
-
-        if not mode.startswith('r') and not mode.startswith('w'):
-            raise ValueError("invalid mode %r" % (mode,))
-
-        univ_nl = ('b' not in mode)
-
-        import subprocess
-
-        if mode.startswith('r'):
-            proc = subprocess.Popen(cmd,
-                                    shell=True,
-                                    stdout=subprocess.PIPE,
-                                    bufsize=bufsize,
-                                    universal_newlines=univ_nl)
-            return _wrap_close(proc.stdout, proc)
-        else:
-            proc = subprocess.Popen(cmd,
-                                    shell=True,
-                                    stdin=subprocess.PIPE,
-                                    bufsize=bufsize,
-                                    universal_newlines=univ_nl)
-            return _wrap_close(proc.stdin, proc)
-
     def popen2(cmd, mode="t", bufsize=-1):
         ""
 

From noreply at buildbot.pypy.org  Fri Oct 14 01:59:33 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri, 14 Oct 2011 01:59:33 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: more bytes/str distinction in _io
Message-ID: <20111013235933.F2DCC82112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48043:a7d7bcda8830
Date: 2011-10-14 01:58 +0200
http://bitbucket.org/pypy/pypy/changeset/a7d7bcda8830/

Log:	more bytes/str distinction in _io

diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py
--- a/pypy/module/_io/interp_textio.py
+++ b/pypy/module/_io/interp_textio.py
@@ -161,7 +161,7 @@
             w_buffer, w_flag = space.unpackiterable(w_state, 2)
             flag = space.r_longlong_w(w_flag)
         else:
-            w_buffer = space.wrap("")
+            w_buffer = space.wrapbytes("")
             flag = 0
         flag <<= 1
         if self.pendingcr:
@@ -355,7 +355,7 @@
                     raise
                 self.w_encoding = space.wrap("ascii")
             else:
-                if not space.isinstance_w(self.w_encoding, space.w_str):
+                if not space.isinstance_w(self.w_encoding, space.w_unicode):
                     self.w_encoding = None
         if self.w_encoding:
             pass
@@ -542,7 +542,7 @@
             # Given this, we know there was a valid snapshot point
             # len(dec_buffer) bytes ago with decoder state (b'', dec_flags).
             w_dec_buffer, w_dec_flags = space.unpackiterable(w_state, 2)
-            dec_buffer = space.str_w(w_dec_buffer)
+            dec_buffer = space.bytes_w(w_dec_buffer)
             dec_flags = space.int_w(w_dec_flags)
         else:
             dec_buffer = None
@@ -561,7 +561,7 @@
         if self.telling:
             # At the snapshot point, len(dec_buffer) bytes before the read,
             # the next input to be decoded is dec_buffer + input_chunk.
-            next_input = dec_buffer + space.str_w(w_input)
+            next_input = dec_buffer + space.bytes_w(w_input)
             self.snapshot = PositionSnapshot(dec_flags, next_input)
 
         return not eof
@@ -729,7 +729,7 @@
         else:
             w_bytes = space.call_method(self.w_encoder, "encode", w_text)
 
-        b = space.str_w(w_bytes)
+        b = space.bytes_w(w_bytes)
         if not self.pending_bytes:
             self.pending_bytes = []
             self.pending_bytes_count = 0
@@ -757,7 +757,8 @@
         self.pending_bytes = None
         self.pending_bytes_count = 0
 
-        space.call_method(self.w_buffer, "write", space.wrap(pending_bytes))
+        space.call_method(self.w_buffer, "write",
+                          space.wrapbytes(pending_bytes))
 
     def detach_w(self, space):
         self._check_init(space)
@@ -780,7 +781,7 @@
             space.call_method(self.w_decoder, "reset")
         else:
             space.call_method(self.w_decoder, "setstate",
-                              space.newtuple([space.wrap(""),
+                              space.newtuple([space.wrapbytes(""),
                                               space.wrap(cookie.dec_flags)]))
 
     def _encoder_setstate(self, space, cookie):
@@ -851,7 +852,7 @@
             w_chunk = space.call_method(self.w_buffer, "read",
                                         space.wrap(cookie.bytes_to_feed))
             self.snapshot = PositionSnapshot(cookie.dec_flags,
-                                             space.str_w(w_chunk))
+                                             space.bytes_w(w_chunk))
 
             w_decoded = space.call_method(self.w_decoder, "decode",
                                           w_chunk, space.wrap(cookie.need_eof))
@@ -929,7 +930,7 @@
 
                 w_state = space.call_method(self.w_decoder, "getstate")
                 w_dec_buffer, w_flags = space.unpackiterable(w_state, 2)
-                dec_buffer_len = len(space.str_w(w_dec_buffer))
+                dec_buffer_len = space.len_w(w_dec_buffer)
 
                 if dec_buffer_len == 0 and chars_decoded <= chars_to_skip:
                     # Decoder buffer is empty, so this is a safe start point.

From noreply at buildbot.pypy.org  Fri Oct 14 01:59:35 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri, 14 Oct 2011 01:59:35 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Installs io.open in builtins!
Message-ID: <20111013235935.3B2E982112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48044:a0e19ad2f880
Date: 2011-10-14 01:59 +0200
http://bitbucket.org/pypy/pypy/changeset/a0e19ad2f880/

Log:	Installs io.open in builtins! Todo: ensure it is also used for
	sys.stdout

diff --git a/pypy/module/__builtin__/__init__.py b/pypy/module/__builtin__/__init__.py
--- a/pypy/module/__builtin__/__init__.py
+++ b/pypy/module/__builtin__/__init__.py
@@ -40,8 +40,7 @@
         'buffer'        : 'interp_memoryview.W_Buffer',
         'memoryview'    : 'interp_memoryview.W_MemoryView',
 
-        'file'          : 'state.get(space).w_file',
-        'open'          : 'state.get(space).w_file',
+        'open'          : 'state.get(space).w_open',
 
         # default __metaclass__: old-style class
         '__metaclass__' : 'interp_classobj.W_ClassObject',
diff --git a/pypy/module/__builtin__/state.py b/pypy/module/__builtin__/state.py
--- a/pypy/module/__builtin__/state.py
+++ b/pypy/module/__builtin__/state.py
@@ -1,9 +1,9 @@
 
 class State:
     def __init__(self, space):
-        self.w_file = space.appexec([], """():
-                import _file;
-                return _file.file""")
+        self.w_open = space.appexec([], """():
+                import io
+                return io.open""")
         
 def get(space):
     return space.fromcache(State)
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -544,13 +544,13 @@
             result = rposix.listdir(dirname)
             w_fs_encoding = getfilesystemencoding(space)
             result_w = [
-                space.call_method(space.wrap(s), "decode", w_fs_encoding)
+                space.call_method(space.wrapbytes(s), "decode", w_fs_encoding)
                 for s in result
             ]
         else:
             dirname = space.str_w(w_dirname)
             result = rposix.listdir(dirname)
-            result_w = [space.wrap(s) for s in result]
+            result_w = [space.wrapbytes(s) for s in result]
     except OSError, e:
         raise wrap_oserror2(space, e, w_dirname)
     return space.newlist(result_w)

From noreply at buildbot.pypy.org  Fri Oct 14 02:16:45 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Fri, 14 Oct 2011 02:16:45 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: posix.environ is a dict of bytes,
 os.environ decode these with the fsencoding.
Message-ID: <20111014001645.2431182112@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48045:d59c68a96b3b
Date: 2011-10-14 02:04 +0200
http://bitbucket.org/pypy/pypy/changeset/d59c68a96b3b/

Log:	posix.environ is a dict of bytes, os.environ decode these with the
	fsencoding.

diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -510,7 +510,7 @@
 def _convertenviron(space, w_env):
     space.call_method(w_env, 'clear')
     for key, value in os.environ.items():
-        space.setitem(w_env, space.wrap(key), space.wrap(value))
+        space.setitem(w_env, space.wrapbytes(key), space.wrapbytes(value))
 
 @unwrap_spec(name=str, value=str)
 def putenv(space, name, value):

From pullrequests-noreply at bitbucket.org  Fri Oct 14 06:50:55 2011
From: pullrequests-noreply at bitbucket.org (Bitbucket)
Date: Fri, 14 Oct 2011 04:50:55 -0000
Subject: [pypy-commit] [OPEN] Pull request #11 for pypy/pypy: Remove
 unnecessary import and unused variable
Message-ID: <pullreq-c77dd69c7542@bitbucket.org>

A new pull request has been opened by Dan Loewenherz.

dlo/pypy has changes to be pulled into pypy/pypy.

https://bitbucket.org/pypy/pypy/pull-request/11/remove-unnecessary-import-and-unused

Title: Remove unnecessary import and unused variable

Changes to be pulled:



--
This is an issue notification from bitbucket.org.
You are receiving this either because you are the participating
in a pull request, or you are following it.


From noreply at buildbot.pypy.org  Fri Oct 14 07:05:00 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Fri, 14 Oct 2011 07:05:00 +0200 (CEST)
Subject: [pypy-commit] pypy default: make a bunch of random str and unicode
	methods faster
Message-ID: <20111014050500.2D44C82A88@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r48046:cc2c0908716e
Date: 2011-10-14 01:04 -0400
http://bitbucket.org/pypy/pypy/changeset/cc2c0908716e/

Log:	make a bunch of random str and unicode methods faster

diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py
--- a/pypy/objspace/std/stringobject.py
+++ b/pypy/objspace/std/stringobject.py
@@ -161,19 +161,19 @@
 
 def str_swapcase__String(space, w_self):
     self = w_self._value
-    res = [' '] * len(self)
+    builder = StringBuilder(len(self))
     for i in range(len(self)):
         ch = self[i]
         if ch.isupper():
             o = ord(ch) + 32
-            res[i] = chr(o)
+            builder.append(chr(o))
         elif ch.islower():
             o = ord(ch) - 32
-            res[i] = chr(o)
+            builder.append(chr(o))
         else:
-            res[i] = ch
+            builder.append(ch)
 
-    return space.wrap("".join(res))
+    return space.wrap(builder.build())
 
 
 def str_capitalize__String(space, w_self):
@@ -199,19 +199,21 @@
 
 def str_title__String(space, w_self):
     input = w_self._value
-    buffer = [' '] * len(input)
+    builder = StringBuilder(len(input))
     prev_letter=' '
 
-    for pos in range(0, len(input)):
+    for pos in range(len(input)):
         ch = input[pos]
         if not prev_letter.isalpha():
-            buffer[pos] = _upper(ch)
+            ch = _upper(ch)
+            builder.append(ch)
         else:
-            buffer[pos] = _lower(ch)
+            ch = _lower(ch)
+            builder.append(ch)
 
-        prev_letter = buffer[pos]
+        prev_letter = ch
 
-    return space.wrap("".join(buffer))
+    return space.wrap(builder.build())
 
 def str_split__String_None_ANY(space, w_self, w_none, w_maxsplit=-1):
     maxsplit = space.int_w(w_maxsplit)
@@ -758,23 +760,18 @@
         # cannot return w_self, in case it is a subclass of str
         return space.wrap(input)
 
-    buf = [' '] * width
+    builder = StringBuilder(width)
     if len(input) > 0 and (input[0] == '+' or input[0] == '-'):
-        buf[0] = input[0]
+        builder.append(input[0])
         start = 1
         middle = width - len(input) + 1
     else:
         start = 0
         middle = width - len(input)
 
-    for i in range(start, middle):
-        buf[i] = '0'
-
-    for i in range(middle, width):
-        buf[i] = input[start]
-        start = start + 1
-
-    return space.wrap("".join(buf))
+    builder.append_multiple_char('0', middle - start)
+    builder.append(input[start:start + (width - middle)])
+    return space.wrap(builder.build())
 
 
 def hash__String(space, w_str):
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -417,54 +417,54 @@
     input = w_self._value
     if len(input) == 0:
         return W_UnicodeObject.EMPTY
-    result = [u'\0'] * len(input)
-    result[0] = unichr(unicodedb.toupper(ord(input[0])))
+    builder = UnicodeBuilder(len(input))
+    builder.append(unichr(unicodedb.toupper(ord(input[0]))))
     for i in range(1, len(input)):
-        result[i] = unichr(unicodedb.tolower(ord(input[i])))
-    return W_UnicodeObject(u''.join(result))
+        builder.append(unichr(unicodedb.tolower(ord(input[i]))))
+    return W_UnicodeObject(builder.build())
 
 def unicode_title__Unicode(space, w_self):
     input = w_self._value
     if len(input) == 0:
         return w_self
-    result = [u'\0'] * len(input)
+    builder = UnicodeBuilder(len(input))
 
     previous_is_cased = False
     for i in range(len(input)):
         unichar = ord(input[i])
         if previous_is_cased:
-            result[i] = unichr(unicodedb.tolower(unichar))
+            builder.append(unichr(unicodedb.tolower(unichar)))
         else:
-            result[i] = unichr(unicodedb.totitle(unichar))
+            builder.append(unichr(unicodedb.totitle(unichar)))
         previous_is_cased = unicodedb.iscased(unichar)
-    return W_UnicodeObject(u''.join(result))
+    return W_UnicodeObject(builder.build())
 
 def unicode_lower__Unicode(space, w_self):
     input = w_self._value
-    result = [u'\0'] * len(input)
+    builder = UnicodeBuilder(len(input))
     for i in range(len(input)):
-        result[i] = unichr(unicodedb.tolower(ord(input[i])))
-    return W_UnicodeObject(u''.join(result))
+        builder.append(unichr(unicodedb.tolower(ord(input[i]))))
+    return W_UnicodeObject(builder.build())
 
 def unicode_upper__Unicode(space, w_self):
     input = w_self._value
-    result = [u'\0'] * len(input)
+    builder = UnicodeBuilder(len(input))
     for i in range(len(input)):
-        result[i] = unichr(unicodedb.toupper(ord(input[i])))
-    return W_UnicodeObject(u''.join(result))
+        builder.append(unichr(unicodedb.toupper(ord(input[i]))))
+    return W_UnicodeObject(builder.build())
 
 def unicode_swapcase__Unicode(space, w_self):
     input = w_self._value
-    result = [u'\0'] * len(input)
+    builder = UnicodeBuilder(len(input))
     for i in range(len(input)):
         unichar = ord(input[i])
         if unicodedb.islower(unichar):
-            result[i] = unichr(unicodedb.toupper(unichar))
+            builder.append(unichr(unicodedb.toupper(unichar)))
         elif unicodedb.isupper(unichar):
-            result[i] = unichr(unicodedb.tolower(unichar))
+            builder.append(unichr(unicodedb.tolower(unichar)))
         else:
-            result[i] = input[i]
-    return W_UnicodeObject(u''.join(result))
+            builder.append(input[i])
+    return W_UnicodeObject(builder.build())
 
 def _normalize_index(length, index):
     if index < 0:

From noreply at buildbot.pypy.org  Fri Oct 14 07:05:48 2011
From: noreply at buildbot.pypy.org (dlo)
Date: Fri, 14 Oct 2011 07:05:48 +0200 (CEST)
Subject: [pypy-commit] pypy default: Remove unnecessary import and unused
	variable
Message-ID: <20111014050548.4038982A88@wyvern.cs.uni-duesseldorf.de>

Author: Dan Loewenherz <dan at dlo.me>
Branch: 
Changeset: r48047:c77dd69c7542
Date: 2011-10-13 21:46 -0700
http://bitbucket.org/pypy/pypy/changeset/c77dd69c7542/

Log:	Remove unnecessary import and unused variable

diff --git a/lib_pypy/resource.py b/lib_pypy/resource.py
--- a/lib_pypy/resource.py
+++ b/lib_pypy/resource.py
@@ -7,7 +7,7 @@
 
 from ctypes_support import standard_c_lib as libc
 from ctypes_support import get_errno
-from ctypes import Structure, c_int, c_long, byref, sizeof, POINTER
+from ctypes import Structure, c_int, c_long, byref, POINTER
 from errno import EINVAL, EPERM
 import _structseq
 
@@ -165,7 +165,6 @@
 
 @builtinify
 def getpagesize():
-    pagesize = 0
     if _getpagesize:
         return _getpagesize()
     else:

From pullrequests-noreply at bitbucket.org  Fri Oct 14 07:05:49 2011
From: pullrequests-noreply at bitbucket.org (Bitbucket)
Date: Fri, 14 Oct 2011 05:05:49 -0000
Subject: [pypy-commit] [ACCEPTED] Pull request #11 for pypy/pypy: Remove
 unnecessary import and unused variable
In-Reply-To: <pullreq-c77dd69c7542@bitbucket.org>
References: <pullreq-c77dd69c7542@bitbucket.org>
Message-ID: <20111014050549.26553.93890@bitbucket12.managed.contegix.com>

Pull request #11 has been accepted by Alex Gaynor.

Changes in dlo/pypy have been pulled into pypy/pypy.

https://bitbucket.org/pypy/pypy/pull-request/11/remove-unnecessary-import-and-unused
--
This is an issue notification from bitbucket.org.
You are receiving this either because you are the participating
in a pull request, or you are following it.


From pullrequests-noreply at bitbucket.org  Fri Oct 14 07:05:56 2011
From: pullrequests-noreply at bitbucket.org (Bitbucket)
Date: Fri, 14 Oct 2011 05:05:56 -0000
Subject: [pypy-commit] [ACCEPTED] Pull request #11 for pypy/pypy: Remove
 unnecessary import and unused variable
In-Reply-To: <pullreq-c77dd69c7542@bitbucket.org>
References: <pullreq-c77dd69c7542@bitbucket.org>
Message-ID: <20111014050556.26553.60893@bitbucket12.managed.contegix.com>

Pull request #11 has been accepted by Alex Gaynor.

Changes in dlo/pypy have been pulled into pypy/pypy.

https://bitbucket.org/pypy/pypy/pull-request/11/remove-unnecessary-import-and-unused
--
This is an issue notification from bitbucket.org.
You are receiving this either because you are the participating
in a pull request, or you are following it.


From noreply at buildbot.pypy.org  Fri Oct 14 10:47:50 2011
From: noreply at buildbot.pypy.org (hager)
Date: Fri, 14 Oct 2011 10:47:50 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: Fixed bug in INT_ADD
	implementation.
Message-ID: <20111014084750.B4B7C82A89@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r48050:5cb205efd4da
Date: 2011-10-14 10:47 +0200
http://bitbucket.org/pypy/pypy/changeset/5cb205efd4da/

Log:	Fixed bug in INT_ADD implementation.

diff --git a/pypy/jit/backend/ppc/ppcgen/opassembler.py b/pypy/jit/backend/ppc/ppcgen/opassembler.py
--- a/pypy/jit/backend/ppc/ppcgen/opassembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/opassembler.py
@@ -25,7 +25,7 @@
         elif l1.is_imm():
             self.mc.addi(res.value, l0.value, l1.value)
         else:
-            self.add(res.value, l0.value, l1.value)
+            self.mc.add(res.value, l0.value, l1.value)
    
     emit_int_le = gen_emit_cmp_op(c.LE)   
 

From noreply at buildbot.pypy.org  Fri Oct 14 10:47:49 2011
From: noreply at buildbot.pypy.org (hager)
Date: Fri, 14 Oct 2011 10:47:49 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: (bivab,
 shager): Started correct implementation of FINISH operation.
Message-ID: <20111014084749.87DE782A88@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r48049:67b568567990
Date: 2011-10-14 10:46 +0200
http://bitbucket.org/pypy/pypy/changeset/67b568567990/

Log:	(bivab, shager): Started correct implementation of FINISH operation.

diff --git a/pypy/jit/backend/ppc/ppcgen/opassembler.py b/pypy/jit/backend/ppc/ppcgen/opassembler.py
--- a/pypy/jit/backend/ppc/ppcgen/opassembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/opassembler.py
@@ -52,15 +52,7 @@
         #                        #           then the guard fails.
 
     def emit_finish(self, op, arglocs, regalloc):
-        descr = op.getdescr()
-        identifier = self._get_identifier_from_descr(descr)
-        self.cpu.saved_descr[identifier] = descr
-        args = op.getarglist()
-        for index, arg in enumerate(arglocs):
-            addr = self.fail_boxes_int.get_addr_for_num(index)
-            self.store_reg(arg, addr)
-        self.load_imm(r.RES.value, identifier) # set return value
-        self.branch_abs(self.exit_code_adr)
+        self.gen_exit_stub(op.getdescr(), op.getarglist(), arglocs)
 
     def emit_jump(self, op, arglocs, regalloc):
         descr = op.getdescr()
diff --git a/pypy/jit/backend/ppc/ppcgen/regalloc.py b/pypy/jit/backend/ppc/ppcgen/regalloc.py
--- a/pypy/jit/backend/ppc/ppcgen/regalloc.py
+++ b/pypy/jit/backend/ppc/ppcgen/regalloc.py
@@ -199,8 +199,7 @@
         return locs + [res]
 
     def prepare_finish(self, op):
-        #args = [locations.imm(self.frame_manager.frame_depth)]
-        args = []
+        args = [locations.imm(self.frame_manager.frame_depth)]
         for i in range(op.numargs()):
             arg = op.getarg(i)
             if arg:

From noreply at buildbot.pypy.org  Fri Oct 14 10:57:34 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 14 Oct 2011 10:57:34 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Proper weakref support.
	In-progress.
Message-ID: <20111014085734.B031B82A88@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r48051:62ab2327b1d5
Date: 2011-10-14 10:57 +0200
http://bitbucket.org/pypy/pypy/changeset/62ab2327b1d5/

Log:	Proper weakref support. In-progress.

diff --git a/pypy/rpython/memory/gc/base.py b/pypy/rpython/memory/gc/base.py
--- a/pypy/rpython/memory/gc/base.py
+++ b/pypy/rpython/memory/gc/base.py
@@ -17,6 +17,7 @@
     moving_gc = False
     needs_write_barrier = False
     needs_deletion_barrier = False
+    needs_weakref_read_barrier = False
     malloc_zero_filled = False
     prebuilt_gc_objects_are_static_roots = True
     object_minimal_size = 0
diff --git a/pypy/rpython/memory/gc/concurrentms.py b/pypy/rpython/memory/gc/concurrentms.py
--- a/pypy/rpython/memory/gc/concurrentms.py
+++ b/pypy/rpython/memory/gc/concurrentms.py
@@ -8,6 +8,7 @@
 from pypy.rlib.debug import ll_assert, debug_print, debug_start, debug_stop
 from pypy.rlib.rarithmetic import ovfcheck, LONG_BIT, r_uint
 from pypy.rpython.memory.gc.base import GCBase
+from pypy.rpython.memory import gctypelayout
 from pypy.module.thread import ll_thread
 
 #
@@ -31,7 +32,7 @@
 
 
 # Objects start with an integer 'tid', which is decomposed as follows.
-# Lowest byte: one of the the following values (which are all odd, so
+# Lowest byte: one of the following values (which are all odd, so
 # let us know if the 'tid' is valid or is just a word-aligned address):
 MARK_VALUE_1      = 0x4D    # 'M', 77
 MARK_VALUE_2      = 0x6B    # 'k', 107
@@ -47,6 +48,7 @@
     inline_simple_malloc = True
     inline_simple_malloc_varsize = True
     needs_deletion_barrier = True
+    needs_weakref_read_barrier = True
     prebuilt_gc_objects_are_static_roots = False
     malloc_zero_filled = True
     gcflag_extra = FL_EXTRA
@@ -137,6 +139,10 @@
         self.collect_run_finalizers_tail = self.NULL
         self.objects_with_finalizers_to_run = self.NULL
         #
+        self.weakref_pages = self.NULL
+        self.collect_weakref_pages = self.NULL
+        self.collect_weakref_tails = self.NULL
+        #
         # The following character is either MARK_VALUE_1 or MARK_VALUE_2,
         # and represents the character that must be in the 'mark' field
         # of an object header in order for the object to be considered as
@@ -154,6 +160,15 @@
         # which re-acquires 'ready_to_start_lock' and does its job.
         # When done it releases 'finished_lock'.  The mutator thread is
         # responsible for resetting 'collection_running' to 0.
+        #
+        # The collector thread's state can be found (with careful locking)
+        # by inspecting the same variable from the mutator thread:
+        #   * collection_running == 1: Marking.  [Deletion barrier active.]
+        #   * collection_running == 2: Clearing weakrefs.
+        #   * collection_running == 3: Marking from unreachable finalizers.
+        #   * collection_running == 4: Sweeping.
+        #   * collection_running == -1: Done.
+        # The mutex_lock is acquired to go from 1 to 2, and from 2 to 3.
         self.collection_running = 0
         #self.ready_to_start_lock = ...built in setup()
         #self.finished_lock = ...built in setup()
@@ -215,6 +230,10 @@
                      "'needs_finalizer' and 'contains_weakptr' both specified")
             return self._malloc_with_finalizer(typeid, size)
         #
+        # Case of weakreferences (test constant-folded)
+        if contains_weakptr:
+            return self._malloc_weakref(typeid, size)
+        #
         # Regular case
         size_gc_header = self.gcheaderbuilder.size_gc_header
         totalsize = size_gc_header + size
@@ -290,13 +309,13 @@
         size_gc_header = self.gcheaderbuilder.size_gc_header
         totalsize = size_gc_header + size
         rawtotalsize = raw_malloc_usage(totalsize)
-        ll_assert(rawtotalsize & (WORD - 1) == 0,
-                  "malloc_slowpath: non-rounded size")
         #
         if rawtotalsize <= self.small_request_threshold:
             #
             # Case 1: unless trigger_next_collection() happened to get us
             # more locations in free_lists[n], we have run out of them
+            ll_assert(rawtotalsize & (WORD - 1) == 0,
+                      "malloc_slowpath: non-rounded size")
             n = rawtotalsize >> WORD_POWER_2
             head = self.free_lists[n]
             if head:
@@ -343,29 +362,47 @@
             # the start of the page rather than at the end (Hans Boehm,
             # xxx ref).
             #
+            return self._malloc_result(typeid, totalsize, result)
         else:
             # Case 2: the object is too large, so allocate it directly
-            # with the system malloc().  xxx on 32-bit, we'll prefer 64-bit
-            # alignment of the object by always allocating an 8-bytes header
-            rawtotalsize += 8
-            block = llarena.arena_malloc(rawtotalsize, 2)
-            if not block:
-                raise MemoryError
-            llarena.arena_reserve(block, self.HDRSIZE)
-            blockhdr = llmemory.cast_adr_to_ptr(block, self.HDRPTR)
-            set_next(blockhdr, self.nonfree_pages[0])
-            self.nonfree_pages[0] = blockhdr
-            result = block + 8
+            # with the system malloc().
+            return self._malloc_large_object(typeid, size, 0)
         #
+    _malloc_slowpath._dont_inline_ = True
+
+    def _malloc_result(self, typeid, totalsize, result):
         llarena.arena_reserve(result, totalsize)
         hdr = llmemory.cast_adr_to_ptr(result, self.HDRPTR)
         hdr.tid = self.combine(typeid, self.current_mark, 0)
-        #
-        obj = result + size_gc_header
+        obj = result + self.gcheaderbuilder.size_gc_header
         #debug_print("malloc_slowpath", obj)
         return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
-        #
-    _malloc_slowpath._dont_inline_ = True
+
+    def _malloc_large_object(self, typeid, size, linked_list):
+        # xxx on 32-bit, we'll prefer 64-bit alignment of the object by
+        # always allocating an 8-bytes header
+        totalsize = self.gcheaderbuilder.size_gc_header + size
+        rawtotalsize = raw_malloc_usage(totalsize)
+        rawtotalsize += 8
+        block = llarena.arena_malloc(rawtotalsize, 2)
+        if not block:
+            raise MemoryError
+        llarena.arena_reserve(block, self.HDRSIZE)
+        blockhdr = llmemory.cast_adr_to_ptr(block, self.HDRPTR)
+        if linked_list == 0:
+            set_next(blockhdr, self.nonfree_pages[0])
+            self.nonfree_pages[0] = blockhdr
+        elif linked_list == 1:
+            set_next(blockhdr, self.finalizer_pages)
+            self.finalizer_pages = blockhdr
+        elif linked_list == 2:
+            set_next(blockhdr, self.weakref_pages)
+            self.weakref_pages = blockhdr
+        else:
+            ll_assert(0, "bad linked_list")
+        return self._malloc_result(typeid, totalsize, block + 8)
+    _malloc_large_object._annspecialcase_ = 'specialize:arg(3)'
+    _malloc_large_object._dont_inline_ = True
 
     def _malloc_varsize_slowpath(self, typeid, length):
         #
@@ -397,30 +434,10 @@
     _malloc_varsize_slowpath._dont_inline_ = True
 
     def _malloc_with_finalizer(self, typeid, size):
-        # XXX a lot of copy and paste from _malloc_slowpath()
-        size_gc_header = self.gcheaderbuilder.size_gc_header
-        totalsize = size_gc_header + size
-        rawtotalsize = raw_malloc_usage(totalsize)
-        ll_assert(rawtotalsize & (WORD - 1) == 0,
-                  "malloc_with_finalizer: non-rounded size")
-        rawtotalsize += 8
-        block = llarena.arena_malloc(rawtotalsize, 2)
-        if not block:
-            raise MemoryError
-        llarena.arena_reserve(block, self.HDRSIZE)
-        blockhdr = llmemory.cast_adr_to_ptr(block, self.HDRPTR)
-        # the changes are only in the following two lines: we add the block
-        # to a different linked list
-        set_next(blockhdr, self.finalizer_pages)
-        self.finalizer_pages = blockhdr
-        result = block + 8
-        #
-        llarena.arena_reserve(result, totalsize)
-        hdr = llmemory.cast_adr_to_ptr(result, self.HDRPTR)
-        hdr.tid = self.combine(typeid, self.current_mark, 0)
-        #
-        obj = result + size_gc_header
-        return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
+        return self._malloc_large_object(typeid, size, 1)
+
+    def _malloc_weakref(self, typeid, size):
+        return self._malloc_large_object(typeid, size, 2)
 
     # ----------
     # Other functions in the GC API
@@ -553,27 +570,31 @@
             # 'free_lists'.
             n = 1
             while n < self.pagelists_length:
-                if self.collect_tails[n] != self.NULL:
-                    set_next(self.collect_tails[n], self.free_lists[n])
-                    self.free_lists[n] = self.collect_heads[n]
+                self.free_lists[n] = self.join_lists(self.free_lists[n],
+                                                     self.collect_heads[n],
+                                                     self.collect_tails[n])
                 n += 1
             #
             # Do the same with 'collect_heads[0]/collect_tails[0]'.
-            if self.collect_tails[0] != self.NULL:
-                set_next(self.collect_tails[0], self.nonfree_pages[0])
-                self.nonfree_pages[0] = self.collect_heads[0]
+            self.nonfree_pages[0] = self.join_lists(self.nonfree_pages[0],
+                                                    self.collect_heads[0],
+                                                    self.collect_tails[0])
+            #
+            # Do the same with 'collect_weakref_pages/tails'
+            self.weakref_pages = self.join_lists(self.weakref_pages,
+                                                 self.collect_weakref_pages,
+                                                 self.collect_weakref_tails)
             #
             # Do the same with 'collect_finalizer_pages/tails'
-            if self.collect_finalizer_tails != self.NULL:
-                set_next(self.collect_finalizer_tails, self.finalizer_pages)
-                self.finalizer_pages = self.collect_finalizer_pages
+            self.finalizer_pages = self.join_lists(self.finalizer_pages,
+                                                  self.collect_finalizer_pages,
+                                                  self.collect_finalizer_tails)
             #
             # Do the same with 'collect_run_finalizers_head/tail'
-            if self.collect_run_finalizers_tail != self.NULL:
-                set_next(self.collect_run_finalizers_tail,
-                         self.objects_with_finalizers_to_run)
-                self.objects_with_finalizers_to_run = (
-                    self.collect_run_finalizers_head)
+            self.objects_with_finalizers_to_run = self.join_lists(
+                self.objects_with_finalizers_to_run,
+                self.collect_run_finalizers_head,
+                self.collect_run_finalizers_tail)
             #
             if self.DEBUG:
                 self.debug_check_lists()
@@ -587,6 +608,15 @@
             ll_assert(self.collection_running == 0,
                       "collector thread not paused?")
 
+    def join_lists(self, list1, list2head, list2tail):
+        if list2tail == self.NULL:
+            ll_assert(list2head == self.NULL, "join_lists/1")
+            return list1
+        else:
+            ll_assert(list2head != self.NULL, "join_lists/2")
+            set_next(list2tail, list1)
+            return list2head
+
 
     def execute_finalizers_ll(self):
         self.finalizer_lock_count += 1
@@ -668,12 +698,14 @@
         while n < self.pagelists_length:
             self.collect_pages[n] = self.nonfree_pages[n]
             n += 1
+        self.collect_weakref_pages = self.weakref_pages
         self.collect_finalizer_pages = self.finalizer_pages
         #
         # Clear the following lists.  When the collector thread finishes,
         # it will give back (in collect_{pages,tails}[0] and
         # collect_finalizer_{pages,tails}) all the original items that survive.
         self.nonfree_pages[0] = self.NULL
+        self.weakref_pages = self.NULL
         self.finalizer_pages = self.NULL
         #
         # Start the collector thread
@@ -703,6 +735,7 @@
         while n < self.pagelists_length:
             self.debug_check_list(self.free_lists[n])
             n += 1
+        self.debug_check_list(self.weakref_pages)
         self.debug_check_list(self.finalizer_pages)
         self.debug_check_list(self.objects_with_finalizers_to_run)
 
@@ -776,19 +809,15 @@
                 self.release(self.finished_lock)
                 break
             #
-            # Mark
+            # Mark                                # collection_running == 1
             self.collector_mark()
-            self.collection_running = 2
-            #debug_print("collection_running = 2")
-            #
+            #                                     # collection_running == 2
+            self.deal_with_weakrefs()
+            #                                     # collection_running == 3
             self.deal_with_objects_with_finalizers()
-            #
-            # Sweep
+            # Sweep                               # collection_running == 4
             self.collector_sweep()
-            #
-            # Done!
-            self.collection_running = -1
-            #debug_print("collection_running = -1")
+            # Done!                               # collection_running == -1
             self.release(self.finished_lock)
 
 
@@ -797,8 +826,8 @@
                   "bad mark value")
         return mark ^ (MARK_VALUE_1 ^ MARK_VALUE_2)
 
-    def is_marked(self, obj, current_mark):
-        return self.get_mark(obj) == current_mark
+    def is_marked_or_static(self, obj, current_mark):
+        return self.get_mark(obj) != self.other_mark(current_mark)
 
     def set_mark(self, obj, newmark):
         _set_mark(self.header(obj), newmark)
@@ -827,7 +856,6 @@
                 obj = self.extra_objects_to_mark.pop()
                 self.get_mark(obj)
                 self.gray_objects.append(obj)
-            self.release(self.mutex_lock)
             #
             # If 'gray_objects' is empty, we are done: there should be
             # no possible case in which more objects are being added to
@@ -837,12 +865,19 @@
             # been marked.
             if not self.gray_objects.non_empty():
                 break
+            #
+            # Else release mutex_lock and try again.
+            self.release(self.mutex_lock)
+        #
+        self.collection_running = 2
+        #debug_print("collection_running = 2")
+        self.release(self.mutex_lock)
 
     def _collect_mark(self):
         current_mark = self.current_mark
         while self.gray_objects.non_empty():
             obj = self.gray_objects.pop()
-            if not self.is_marked(obj, current_mark):
+            if not self.is_marked_or_static(obj, current_mark):
                 #
                 # Scan the content of 'obj'.  We use a snapshot-at-the-
                 # beginning order, meaning that we want to scan the state
@@ -883,14 +918,15 @@
         self.gray_objects.append(obj)
 
     def collector_sweep(self):
+        self._collect_sweep_large_objects()
+        #
         n = 1
         while n < self.pagelists_length:
             self._collect_sweep_pages(n)
             n += 1
-        # do this *after* the other one, so that the blocks are not free'd
-        # before we get a chance to inspect them, to see if they contain
-        # an object that is still alive (needed for weakrefs)
-        self._collect_sweep_large_objects()
+        #
+        self.collection_running = -1
+        #debug_print("collection_running = -1")
 
     def _collect_sweep_large_objects(self):
         block = self.collect_pages[0]
@@ -962,29 +998,6 @@
                     llarena.arena_reset(adr + size_of_int,
                                         object_size - size_of_int, 2)
                 #
-                elif mark == marked:
-                    # the location contains really an object, which is marked.
-                    # check the typeid to see if it's a weakref.  XXX could
-                    # be faster
-                    # XXX actually it's very wrong: if we read a weakref out
-                    # of an object during collection_running==1, or during
-                    # collection_running==2 but before we reach that point,
-                    # the target object will never be marked
-                    tid = hdr.tid
-                    type_id = llop.extract_high_ushort(llgroup.HALFWORD, tid)
-                    wroffset = self.weakpointer_offset(type_id)
-                    if wroffset >= 0:
-                        obj = adr + size_gc_header
-                        pointing_to = (obj + wroffset).address[0]
-                        if pointing_to != llmemory.NULL:
-                            pt_adr = pointing_to - size_gc_header
-                            pt_hdr = llmemory.cast_adr_to_ptr(pt_adr,
-                                                              self.HDRPTR)
-                            if (pt_hdr.tid & 0xFF) == nonmarked:
-                                # this weakref points to an object that is
-                                # still not marked, so clear it
-                                (obj + wroffset).address[0] = llmemory.NULL
-                #
                 i -= object_size
             #
             page = list_next(page)
@@ -992,6 +1005,87 @@
         self.collect_heads[n] = linked_list
         self.collect_tails[n] = first_loc_in_linked_list
 
+
+    # ----------
+    # Weakrefs
+
+    def weakref_deref(self, wrobj):
+        # Weakrefs need some care.  This code acts as a read barrier.
+        # The only way I found is to acquire the mutex_lock to prevent
+        # the collection thread from going from collection_running==1
+        # to collection_running==2, or from collection_running==2 to
+        # collection_running==3.
+        #
+        self.acquire(self.mutex_lock)
+        #
+        targetobj = gctypelayout.ll_weakref_deref(wrobj)
+        if targetobj != llmemory.NULL:
+            #
+            if self.collection_running == 1:
+                # If we are in the phase collection_running==1, we don't
+                # know if the object will be scanned a bit later or
+                # not; so we have to assume that it survives, and
+                # force it to be scanned.
+                self.get_mark(targetobj)
+                self.extra_objects_to_mark.append(targetobj)
+                #
+            elif self.collection_running == 2:
+                # In the phase collection_running==2, if the object is
+                # not marked it's too late; we have to detect that case
+                # and return NULL instead here, as if the corresponding
+                # collector phase was already finished (deal_with_weakrefs).
+                # Otherwise we would be returning an object that is about to
+                # be swept away.
+                if not self.is_marked_or_static(targetobj, self.current_mark):
+                    targetobj = llmemory.NULL
+                #
+            else:
+                # In other phases we are fine.
+                pass
+        #
+        self.release(self.mutex_lock)
+        #
+        return targetobj
+
+    def deal_with_weakrefs(self):
+        size_gc_header = self.gcheaderbuilder.size_gc_header
+        current_mark = self.current_mark
+        weakref_page = self.collect_weakref_pages
+        self.collect_weakref_pages = self.NULL
+        self.collect_weakref_tails = self.NULL
+        while weakref_page != self.NULL:
+            next_page = list_next(weakref_page)
+            #
+            # If the weakref points to a dead object, make it point to NULL.
+            x = llmemory.cast_ptr_to_adr(weakref_page)
+            x = llarena.getfakearenaaddress(x) + 8
+            hdr = llmemory.cast_adr_to_ptr(x, self.HDRPTR)
+            type_id = llop.extract_high_ushort(llgroup.HALFWORD, hdr.tid)
+            offset = self.weakpointer_offset(type_id)
+            ll_assert(offset >= 0, "bad weakref")
+            obj = x + size_gc_header
+            pointing_to = (obj + offset).address[0]
+            if (pointing_to == llmemory.NULL or
+                not self.is_marked_or_static(pointing_to, current_mark)):
+                # 'pointing_to' dies: relink to self.collect_pages[0]
+                (obj + offset).address[0] = llmemory.NULL
+                set_next(weakref_page, self.collect_pages[0])
+                self.collect_pages[0] = weakref_page
+            else:
+                # the weakref stays alive
+                set_next(weakref_page, self.collect_weakref_pages)
+                self.collect_weakref_pages = weakref_page
+                if self.collect_weakref_tails == self.NULL:
+                    self.collect_weakref_tails = weakref_page
+            #
+            weakref_page = next_page
+        #
+        self.acquire(self.mutex_lock)
+        self.collection_running = 3
+        #debug_print("collection_running = 3")
+        self.release(self.mutex_lock)
+
+
     # ----------
     # Finalizers
 
@@ -1025,7 +1119,7 @@
                 # marked: relink into the collect_finalizer_pages list
                 set_next(finalizer_page, self.collect_finalizer_pages)
                 self.collect_finalizer_pages = finalizer_page
-                if self.collect_finalizer_tails != self.NULL:
+                if self.collect_finalizer_tails == self.NULL:
                     self.collect_finalizer_tails = finalizer_page
             #
             finalizer_page = next_page
@@ -1035,6 +1129,9 @@
         ll_assert(not self.extra_objects_to_mark.non_empty(),
                   "should not see objects only reachable from finalizers "
                   "before we run them")
+        #
+        self.collection_running = 4
+        #debug_print("collection_running = 4")
 
 
 # ____________________________________________________________
diff --git a/pypy/rpython/memory/gctransform/framework.py b/pypy/rpython/memory/gctransform/framework.py
--- a/pypy/rpython/memory/gctransform/framework.py
+++ b/pypy/rpython/memory/gctransform/framework.py
@@ -246,13 +246,20 @@
         else:
             self.incr_stack_ptr = None
             self.decr_stack_ptr = None
-        self.weakref_deref_ptr = self.inittime_helper(
-            ll_weakref_deref, [llmemory.WeakRefPtr], llmemory.Address)
-        
+
         classdef = bk.getuniqueclassdef(GCClass)
         s_gc = annmodel.SomeInstance(classdef)
         s_gcref = annmodel.SomePtr(llmemory.GCREF)
 
+        if GCClass.needs_weakref_read_barrier:
+            self.gc_weakref_deref_ptr = getfn(
+                GCClass.weakref_deref.im_func,
+                [s_gc, annmodel.SomePtr(llmemory.WeakRefPtr)],
+                annmodel.SomeAddress())
+        else:
+            self.weakref_deref_ptr = self.inittime_helper(
+                ll_weakref_deref, [llmemory.WeakRefPtr], llmemory.Address)
+
         malloc_fixedsize_clear_meth = GCClass.malloc_fixedsize_clear.im_func
         self.malloc_fixedsize_clear_ptr = getfn(
             malloc_fixedsize_clear_meth,
@@ -947,9 +954,15 @@
 
     def gct_weakref_deref(self, hop):
         v_wref, = hop.spaceop.args
-        v_addr = hop.genop("direct_call",
-                           [self.weakref_deref_ptr, v_wref],
-                           resulttype=llmemory.Address)
+        if hasattr(self, 'gc_weakref_deref_ptr'):
+            v_addr = hop.genop("direct_call",
+                               [self.gc_weakref_deref_ptr,
+                                self.c_const_gc, v_wref],
+                               resulttype=llmemory.Address)
+        else:
+            v_addr = hop.genop("direct_call",
+                               [self.weakref_deref_ptr, v_wref],
+                               resulttype=llmemory.Address)
         hop.cast_result(v_addr)
 
     def gct_gc_identityhash(self, hop):
diff --git a/pypy/rpython/memory/gcwrapper.py b/pypy/rpython/memory/gcwrapper.py
--- a/pypy/rpython/memory/gcwrapper.py
+++ b/pypy/rpython/memory/gcwrapper.py
@@ -129,8 +129,12 @@
         return llmemory.cast_ptr_to_weakrefptr(result)
     
     def weakref_deref(self, PTRTYPE, obj):
-        addr = gctypelayout.ll_weakref_deref(obj)
-        return llmemory.cast_adr_to_ptr(addr, PTRTYPE)
+        if self.gc.needs_weakref_read_barrier:
+            addr = self.gc.weakref_deref(obj)
+            return llmemory.cast_adr_to_ptr(addr, PTRTYPE)
+        else:
+            addr = gctypelayout.ll_weakref_deref(obj)
+            return llmemory.cast_adr_to_ptr(addr, PTRTYPE)
 
     def gc_id(self, ptr):
         ptr = lltype.cast_opaque_ptr(llmemory.GCREF, ptr)
diff --git a/pypy/rpython/memory/test/test_gc.py b/pypy/rpython/memory/test/test_gc.py
--- a/pypy/rpython/memory/test/test_gc.py
+++ b/pypy/rpython/memory/test/test_gc.py
@@ -934,3 +934,6 @@
 
     def test_finalizer_calls_malloc_during_minor_collect(self):
         py.test.skip("check if this test is valid here")
+
+    def test_weakref_to_object_with_finalizer_ordering(self):
+        py.test.skip("frees weakrefs before calling finalizers")

From noreply at buildbot.pypy.org  Fri Oct 14 12:14:42 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 14 Oct 2011 12:14:42 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Locking is needed in the
 AddressStack class, to access the pool
Message-ID: <20111014101442.E9C0282A88@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r48052:08bcc238a411
Date: 2011-10-14 12:14 +0200
http://bitbucket.org/pypy/pypy/changeset/08bcc238a411/

Log:	Locking is needed in the AddressStack class, to access the pool of
	free pages (which is shared between all AddressStack instances, so
	accessed from the two threads concurrently).

diff --git a/pypy/rpython/memory/gc/base.py b/pypy/rpython/memory/gc/base.py
--- a/pypy/rpython/memory/gc/base.py
+++ b/pypy/rpython/memory/gc/base.py
@@ -24,10 +24,10 @@
     gcflag_extra = 0   # or a real GC flag that is always 0 when not collecting
 
     def __init__(self, config, chunk_size=DEFAULT_CHUNK_SIZE,
-                 translated_to_c=True):
+                 translated_to_c=True, lock=None):
         self.gcheaderbuilder = GCHeaderBuilder(self.HDR)
-        self.AddressStack = get_address_stack(chunk_size)
-        self.AddressDeque = get_address_deque(chunk_size)
+        self.AddressStack = get_address_stack(chunk_size, lock=lock)
+        self.AddressDeque = get_address_deque(chunk_size, lock=lock)
         self.AddressDict = AddressDict
         self.null_address_dict = null_address_dict
         self.config = config
diff --git a/pypy/rpython/memory/gc/concurrentms.py b/pypy/rpython/memory/gc/concurrentms.py
--- a/pypy/rpython/memory/gc/concurrentms.py
+++ b/pypy/rpython/memory/gc/concurrentms.py
@@ -71,6 +71,8 @@
         # 'small_request_threshold' is the largest size that we will
         # satisfy using our own pages mecanism.  Larger requests just
         # go to the system malloc().
+        self.addressstack_lock_object = SyncLock()
+        kwds['lock'] = self.addressstack_lock_object
         GCBase.__init__(self, config, **kwds)
         assert small_request_threshold % WORD == 0
         self.small_request_threshold = small_request_threshold
@@ -188,6 +190,7 @@
         self.ready_to_start_lock = ll_thread.allocate_ll_lock()
         self.finished_lock = ll_thread.allocate_ll_lock()
         self.mutex_lock = ll_thread.allocate_ll_lock()
+        self.addressstack_lock_object.setup()
         #
         self.acquire(self.finished_lock)
         self.acquire(self.ready_to_start_lock)
@@ -1065,8 +1068,8 @@
             ll_assert(offset >= 0, "bad weakref")
             obj = x + size_gc_header
             pointing_to = (obj + offset).address[0]
-            if (pointing_to == llmemory.NULL or
-                not self.is_marked_or_static(pointing_to, current_mark)):
+            ll_assert(pointing_to != llmemory.NULL, "null weakref?")
+            if not self.is_marked_or_static(pointing_to, current_mark):
                 # 'pointing_to' dies: relink to self.collect_pages[0]
                 (obj + offset).address[0] = llmemory.NULL
                 set_next(weakref_page, self.collect_pages[0])
@@ -1192,3 +1195,19 @@
                            [MostlyConcurrentMarkSweepGC.HDRPTR, lltype.Signed],
                            lltype.Void, compilation_info=eci, _nowrapper=True,
                            _callable=emulate_set_flags)
+
+# ____________________________________________________________
+#
+# A lock to synchronize access to AddressStack's free pages
+
+class SyncLock:
+    _alloc_flavor_ = "raw"
+    _lock = lltype.nullptr(ll_thread.TLOCKP.TO)
+    def setup(self):
+        self._lock = ll_thread.allocate_ll_lock()
+    def acquire(self):
+        if self._lock:
+            ll_thread.c_thread_acquirelock(self._lock, 1)
+    def release(self):
+        if self._lock:
+            ll_thread.c_thread_releaselock(self._lock)
diff --git a/pypy/rpython/memory/support.py b/pypy/rpython/memory/support.py
--- a/pypy/rpython/memory/support.py
+++ b/pypy/rpython/memory/support.py
@@ -16,9 +16,9 @@
 DEFAULT_CHUNK_SIZE = 1019
 
 
-def get_chunk_manager(chunk_size=DEFAULT_CHUNK_SIZE, cache={}):
+def get_chunk_manager(chunk_size=DEFAULT_CHUNK_SIZE, cache={}, lock=None):
     try:
-        return cache[chunk_size]
+        return cache[chunk_size, lock]
     except KeyError:
         pass
 
@@ -36,7 +36,9 @@
             self.free_list = null_chunk
 
         def get(self):
+            self._lock()
             if not self.free_list:
+                self._unlock()
                 # we zero-initialize the chunks to make the translation
                 # backends happy, but we don't need to do it at run-time.
                 if we_are_translated():
@@ -52,9 +54,11 @@
 
             result = self.free_list
             self.free_list = result.next
+            self._unlock()
             return result
 
         def put(self, chunk):
+            self._lock()
             if we_are_translated():
                 chunk.next = self.free_list
                 self.free_list = chunk
@@ -63,19 +67,27 @@
                 # Helps debugging, and avoids that old chunks full of
                 # addresses left behind by a test end up in genc...
                 lltype.free(chunk, flavor="raw", track_allocation=False)
+            self._unlock()
+
+        if lock is not None:
+            def _lock(self):   lock.acquire()
+            def _unlock(self): lock.release()
+        else:
+            def _lock(self):   pass
+            def _unlock(self): pass
 
     unused_chunks = FreeList()
-    cache[chunk_size] = unused_chunks, null_chunk
+    cache[chunk_size, lock] = unused_chunks, null_chunk
     return unused_chunks, null_chunk
 
 
-def get_address_stack(chunk_size=DEFAULT_CHUNK_SIZE, cache={}):
+def get_address_stack(chunk_size=DEFAULT_CHUNK_SIZE, cache={}, lock=None):
     try:
-        return cache[chunk_size]
+        return cache[chunk_size, lock]
     except KeyError:
         pass
 
-    unused_chunks, null_chunk = get_chunk_manager(chunk_size)
+    unused_chunks, null_chunk = get_chunk_manager(chunk_size, lock=lock)
 
     class AddressStack(object):
         _alloc_flavor_ = "raw"
@@ -187,20 +199,20 @@
                 chunk.items[count] = got
                 got = next
 
-    cache[chunk_size] = AddressStack
+    cache[chunk_size, lock] = AddressStack
     return AddressStack
 
 def _add_in_dict(item, d):
     d.add(item)
 
 
-def get_address_deque(chunk_size=DEFAULT_CHUNK_SIZE, cache={}):
+def get_address_deque(chunk_size=DEFAULT_CHUNK_SIZE, cache={}, lock=None):
     try:
-        return cache[chunk_size]
+        return cache[chunk_size, lock]
     except KeyError:
         pass
 
-    unused_chunks, null_chunk = get_chunk_manager(chunk_size)
+    unused_chunks, null_chunk = get_chunk_manager(chunk_size, lock=lock)
 
     class AddressDeque(object):
         _alloc_flavor_ = "raw"
@@ -275,7 +287,7 @@
                 cur = next
             free_non_gc_object(self)
 
-    cache[chunk_size] = AddressDeque
+    cache[chunk_size, lock] = AddressDeque
     return AddressDeque
 
 # ____________________________________________________________

From noreply at buildbot.pypy.org  Fri Oct 14 14:54:09 2011
From: noreply at buildbot.pypy.org (cfbolz)
Date: Fri, 14 Oct 2011 14:54:09 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix untranslatable total nonsense in
	zfill
Message-ID: <20111014125409.C6AA582A88@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r48053:c29a0415adb7
Date: 2011-10-14 14:53 +0200
http://bitbucket.org/pypy/pypy/changeset/c29a0415adb7/

Log:	fix untranslatable total nonsense in zfill

diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py
--- a/pypy/objspace/std/stringobject.py
+++ b/pypy/objspace/std/stringobject.py
@@ -756,7 +756,8 @@
     input = w_self._value
     width = space.int_w(w_width)
 
-    if len(input) >= width:
+    num_zeros = width - len(input)
+    if num_zeros <= 0:
         # cannot return w_self, in case it is a subclass of str
         return space.wrap(input)
 
@@ -764,13 +765,11 @@
     if len(input) > 0 and (input[0] == '+' or input[0] == '-'):
         builder.append(input[0])
         start = 1
-        middle = width - len(input) + 1
     else:
         start = 0
-        middle = width - len(input)
 
-    builder.append_multiple_char('0', middle - start)
-    builder.append(input[start:start + (width - middle)])
+    builder.append_multiple_char('0', num_zeros)
+    builder.append_slice(input, start, len(input))
     return space.wrap(builder.build())
 
 

From noreply at buildbot.pypy.org  Fri Oct 14 15:01:03 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 14 Oct 2011 15:01:03 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix.
Message-ID: <20111014130103.584F982A88@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48054:91de2938fbfc
Date: 2011-10-14 15:00 +0200
http://bitbucket.org/pypy/pypy/changeset/91de2938fbfc/

Log:	Fix.

diff --git a/pypy/tool/logparser.py b/pypy/tool/logparser.py
--- a/pypy/tool/logparser.py
+++ b/pypy/tool/logparser.py
@@ -298,6 +298,8 @@
         image.paste(textpercent, (t1x, 5), textpercent)
         image.paste(textlabel,   (t2x, 5), textlabel)
         images.append(image)
+    if not images:
+        return None
     return combine(images, spacing=0, border=1, horizontal=False)
 
 def get_timesummary_single_image(totaltimes, totaltime0, componentdict,
@@ -333,6 +335,8 @@
         del totaltimes[None]
     img2 = render_histogram(totaltimes, totaltime0, {},
                             width, summarybarheight)
+    if img2 is None:
+        return img1
     return combine([img1, img2], spacing=spacing, horizontal=True)
 
 # ----------

From noreply at buildbot.pypy.org  Fri Oct 14 15:01:04 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 14 Oct 2011 15:01:04 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20111014130104.8AEB082A88@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48055:be5a6fc3a6b4
Date: 2011-10-14 15:00 +0200
http://bitbucket.org/pypy/pypy/changeset/be5a6fc3a6b4/

Log:	merge heads

diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py
--- a/pypy/objspace/std/stringobject.py
+++ b/pypy/objspace/std/stringobject.py
@@ -756,7 +756,8 @@
     input = w_self._value
     width = space.int_w(w_width)
 
-    if len(input) >= width:
+    num_zeros = width - len(input)
+    if num_zeros <= 0:
         # cannot return w_self, in case it is a subclass of str
         return space.wrap(input)
 
@@ -764,13 +765,11 @@
     if len(input) > 0 and (input[0] == '+' or input[0] == '-'):
         builder.append(input[0])
         start = 1
-        middle = width - len(input) + 1
     else:
         start = 0
-        middle = width - len(input)
 
-    builder.append_multiple_char('0', middle - start)
-    builder.append(input[start:start + (width - middle)])
+    builder.append_multiple_char('0', num_zeros)
+    builder.append_slice(input, start, len(input))
     return space.wrap(builder.build())
 
 

From noreply at buildbot.pypy.org  Fri Oct 14 16:25:23 2011
From: noreply at buildbot.pypy.org (hager)
Date: Fri, 14 Oct 2011 16:25:23 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: (bivab,
 shager): Implemented the encoding/decoding of states (registers, memory,
 descrs, immediates).
Message-ID: <20111014142523.C51A182A88@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r48056:e95d2d9863da
Date: 2011-10-14 16:24 +0200
http://bitbucket.org/pypy/pypy/changeset/e95d2d9863da/

Log:	(bivab, shager): Implemented the encoding/decoding of states
	(registers, memory, descrs, immediates). First tests pass, added a
	test regarding decode32 and encode32.

diff --git a/pypy/jit/backend/arm/test/test_helper.py b/pypy/jit/backend/arm/test/test_helper.py
--- a/pypy/jit/backend/arm/test/test_helper.py
+++ b/pypy/jit/backend/arm/test/test_helper.py
@@ -33,6 +33,8 @@
     assert decode32(mem, 0) ==  1234567
     mem = list('\x00\x00\x0F\x00')
     assert decode32(mem, 0) == 983040
+    mem = list("\x03\x00\x00\x00")
+    assert decode32(mem, 0) == 3
 
 def test_decode64():
     mem = list('\x87\xd6\x12\x00\x00\x00\x0F\x00')
diff --git a/pypy/jit/backend/ppc/ppcgen/helper/assembler.py b/pypy/jit/backend/ppc/ppcgen/helper/assembler.py
--- a/pypy/jit/backend/ppc/ppcgen/helper/assembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/helper/assembler.py
@@ -31,16 +31,16 @@
     return f
 
 def encode32(mem, i, n):
-    mem[i] = chr(n & 0xFF)
-    mem[i+1] = chr((n >> 8) & 0xFF)
-    mem[i+2] = chr((n >> 16) & 0xFF)
-    mem[i+3] = chr((n >> 24) & 0xFF)
+    mem[i+3] = chr(n & 0xFF)
+    mem[i+2] = chr((n >> 8) & 0xFF)
+    mem[i+1] = chr((n >> 16) & 0xFF)
+    mem[i] = chr((n >> 24) & 0xFF)
 
 def decode32(mem, index):
-    return intmask(ord(mem[index])
-            | ord(mem[index+1]) << 8
-            | ord(mem[index+2]) << 16
-            | ord(mem[index+3]) << 24)
+    return intmask(ord(mem[index+3])
+            | ord(mem[index+2]) << 8
+            | ord(mem[index+1]) << 16
+            | ord(mem[index]) << 24)
 
 class saved_registers(object):
     def __init__(self, assembler, regs_to_save, regalloc=None):
diff --git a/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py b/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py
--- a/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py
@@ -11,7 +11,7 @@
 from pypy.jit.backend.ppc.ppcgen.arch import (IS_PPC_32, WORD, NONVOLATILES,
                                               GPR_SAVE_AREA)
 from pypy.jit.backend.ppc.ppcgen.helper.assembler import (gen_emit_cmp_op, 
-                                                          encode32)
+                                                          encode32, decode32)
 import pypy.jit.backend.ppc.ppcgen.register as r
 import pypy.jit.backend.ppc.ppcgen.condition as c
 from pypy.jit.metainterp.history import (Const, ConstPtr, LoopToken,
@@ -28,6 +28,8 @@
 from pypy.jit.metainterp.history import (BoxInt, ConstInt, ConstPtr,
                                          ConstFloat, Box, INT, REF, FLOAT)
 from pypy.jit.backend.x86.support import values_array
+from pypy.rlib import rgc
+from pypy.rpython.annlowlevel import llhelper
 
 memcpy_fn = rffi.llexternal('memcpy', [llmemory.Address, llmemory.Address,
                                        rffi.SIZE_T], lltype.Void,
@@ -173,59 +175,133 @@
         offset = target_pos - curpos
         self.mc.b(offset)
 
-    #def _make_epilogue(self):
-    #    for op_index, fail_index, guard, reglist in self.patch_list:
-    #        curpos = self.mc.get_rel_pos()
-    #        offset = curpos - (4 * op_index)
-    #        assert (1 << 15) > offset
-    #        self.mc.beq(offset)
-    #        self.mc.patch_op(op_index)
+    def setup_failure_recovery(self):
 
-    #        # store return parameters in memory
-    #        used_mem_indices = []
-    #        for index, reg in enumerate(reglist):
-    #            # if reg is None, then there is a hole in the failargs
-    #            if reg is not None:
-    #                addr = self.fail_boxes_int.get_addr_for_num(index)
-    #                self.store_reg(reg, addr)
-    #                used_mem_indices.append(index)
+        @rgc.no_collect
+        def failure_recovery_func(mem_loc, frame_pointer, stack_pointer):
+            """mem_loc is a structure in memory describing where the values for
+            the failargs are stored.
+            frame loc is the address of the frame pointer for the frame to be
+            decoded frame """
+            return self.decode_registers_and_descr(mem_loc, frame_pointer, stack_pointer)
 
-    #        patch_op = self.mc.get_number_of_ops()
-    #        patch_pos = self.mc.get_rel_pos()
-    #        descr = self.cpu.saved_descr[fail_index]
-    #        descr.patch_op = patch_op
-    #        descr.patch_pos = patch_pos
-    #        descr.used_mem_indices = used_mem_indices
+        self.failure_recovery_func = failure_recovery_func
 
-    #        self.mc.li(r.r3.value, fail_index)
+    recovery_func_sign = lltype.Ptr(lltype.FuncType([lltype.Signed, 
+            lltype.Signed, lltype.Signed], lltype.Signed))
 
-    #        #self._restore_nonvolatiles()
+    @rgc.no_collect
+    def decode_registers_and_descr(self, mem_loc, stack_loc, spp_loc):
+        ''' 
+            mem_loc     : pointer to encoded state
+            stack_loc   : pointer to top of the stack
+            spp_loc     : pointer to begin of the spilling area
+            '''
+        enc = rffi.cast(rffi.CCHARP, mem_loc)
+        managed_size = WORD * len(r.MANAGED_REGS)
+        spilling_depth = spp_loc - stack_loc + managed_size
+        spilling_area = rffi.cast(rffi.CCHARP, stack_loc + managed_size)
+        assert spilling_depth >= 0
 
-    #        #self.mc.lwz(0, 1, self.framesize + 4)
-    #        #if IS_PPC_32:
-    #        #    self.mc.lwz(0, 1, self.framesize + WORD) # 36
-    #        #else:
-    #        #    self.mc.ld(0, 1, self.framesize + WORD) # 36
-    #        #self.mc.mtlr(0)
-    #        #self.mc.addi(1, 1, self.framesize)
-    #        #self.mc.li(r.r3.value, fail_index)            
-    #        #self.mc.blr()
+        regs = rffi.cast(rffi.CCHARP, stack_loc)
+        i = -1
+        fail_index = -1
+        while(True):
+            i += 1
+            fail_index += 1
+            res = enc[i]
+            if res == self.END_OF_LOCS:
+                break
+            if res == self.EMPTY_LOC:
+                continue
+
+            group = res
+            i += 1
+            res = enc[i]
+            if res == self.IMM_LOC:
+               # imm value
+                if group == self.INT_TYPE or group == self.REF_TYPE:
+                    value = decode32(enc, i+1)
+                    i += 4
+                else:
+                    assert group == self.FLOAT_TYPE
+                    adr = decode32(enc, i+1)
+                    value = rffi.cast(rffi.CArrayPtr(longlong.FLOATSTORAGE), adr)[0]
+                    self.fail_boxes_float.setitem(fail_index, value)
+                    i += 4
+                    continue
+            elif res == self.STACK_LOC:
+                stack_loc = decode32(enc, i+1)
+                i += 4
+                if group == self.FLOAT_TYPE:
+                    value = decode64(stack, frame_depth - stack_loc*WORD)
+                    self.fail_boxes_float.setitem(fail_index, value)
+                    continue
+                else:
+                    value = decode32(spilling_area, spilling_area - stack_loc * WORD)
+            else: # REG_LOC
+                reg = ord(enc[i])
+                if group == self.FLOAT_TYPE:
+                    value = decode64(vfp_regs, reg*2*WORD)
+                    self.fail_boxes_float.setitem(fail_index, value)
+                    continue
+                else:
+                    value = decode32(regs, reg*WORD - 2 * WORD)
+
+            if group == self.INT_TYPE:
+                self.fail_boxes_int.setitem(fail_index, value)
+            elif group == self.REF_TYPE:
+                tgt = self.fail_boxes_ptr.get_addr_for_num(fail_index)
+                rffi.cast(rffi.LONGP, tgt)[0] = value
+            else:
+                assert 0, 'unknown type'
+
+
+        assert enc[i] == self.END_OF_LOCS
+        descr = decode32(enc, i+1)
+        self.fail_boxes_count = fail_index
+        self.fail_force_index = spp_loc
+        return descr
 
     def _gen_leave_jitted_hook_code(self, save_exc=False):
         mc = PPCBuilder()
-        ### XXX add a check if cpu supports floats
-        #with saved_registers(mc, r.caller_resp + [r.ip], r.caller_vfp_resp):
-        #    addr = self.cpu.get_on_leave_jitted_int(save_exception=save_exc)
-        #    mc.BL(addr)
-        #assert self._exit_code_addr != 0
-        #mc.B(self._exit_code_addr)
+
+        # PLAN:
+        # =====
+        # save caller save registers AND(!) r0 
+        # (r0 contains address of state encoding)
+
         mc.b_abs(self.exit_code_adr)
         mc.prepare_insts_blocks()
         return mc.materialize(self.cpu.asmmemmgr, [],
                                self.cpu.gc_ll_descr.gcrootmap)
 
+    # XXX 64 bit adjustment needed
     def _gen_exit_path(self):
-        mc = PPCBuilder()
+        mc = PPCBuilder() 
+        #
+        self._save_managed_regs(mc)
+        # adjust SP (r1)
+        size = WORD * len(r.MANAGED_REGS)
+        mc.addi(r.SP.value, r.SP.value, -size)
+        #
+        decode_func_addr = llhelper(self.recovery_func_sign,
+                self.failure_recovery_func)
+        addr = rffi.cast(lltype.Signed, decode_func_addr)
+        #
+        # load parameters into parameter registers
+        mc.lwz(r.r3.value, r.SPP.value, 0)
+        #mc.mr(r.r3.value, r.r0.value)          # address of state encoding 
+        mc.mr(r.r4.value, r.SP.value)          # load stack pointer
+        mc.mr(r.r5.value, r.SPP.value)         # load spilling pointer
+        #
+        # load address of decoding function into r0
+        mc.load_imm(r.r0, addr)
+        # ... and branch there
+        mc.mtctr(r.r0.value)
+        mc.bctrl()
+        #
+        mc.addi(r.SP.value, r.SP.value, size)
         # save SPP in r5
         # (assume that r5 has been written to failboxes)
         mc.mr(r.r5.value, r.SPP.value)
@@ -242,6 +318,17 @@
         return mc.materialize(self.cpu.asmmemmgr, [],
                                    self.cpu.gc_ll_descr.gcrootmap)
 
+    # Save all registers which are managed by the register
+    # allocator on top of the stack before decoding.
+    # XXX adjust for 64 bit
+    def _save_managed_regs(self, mc):
+        for i in range(len(r.MANAGED_REGS) - 1, -1, -1):
+            reg = r.MANAGED_REGS[i]
+            if IS_PPC_32:
+                mc.stw(reg.value, r.SP.value, -(len(r.MANAGED_REGS) - i) * WORD)
+            else:
+                assert 0, "not implemented yet"
+
     def gen_bootstrap_code(self, nonfloatlocs, inputargs):
         for i in range(len(nonfloatlocs)):
             loc = nonfloatlocs[i]
@@ -273,6 +360,7 @@
 
     def setup_once(self):
         self.memcpy_addr = self.cpu.cast_ptr_to_int(memcpy_fn)
+        self.setup_failure_recovery()
         self.exit_code_adr = self._gen_exit_path()
         #self._leave_jitted_hook_save_exc = self._gen_leave_jitted_hook_code(True)
         self._leave_jitted_hook = self._gen_leave_jitted_hook_code(False)
@@ -352,6 +440,7 @@
                 mem[j] = self.EMPTY_LOC
                 j += 1
             i += 1
+                # XXX 64 bit adjustment needed
 
         mem[j] = chr(0xFF)
 
@@ -427,8 +516,6 @@
             path = self._leave_jitted_hook_save_exc
         else:
             path = self._leave_jitted_hook
-        self.mc.trap()
-        #self.mc.ba(path)
         self.branch_abs(path)
         return memaddr
 
diff --git a/pypy/jit/backend/ppc/ppcgen/regalloc.py b/pypy/jit/backend/ppc/ppcgen/regalloc.py
--- a/pypy/jit/backend/ppc/ppcgen/regalloc.py
+++ b/pypy/jit/backend/ppc/ppcgen/regalloc.py
@@ -26,7 +26,7 @@
         return "<TempPtr at %s>" % (id(self),)
 
 class PPCRegisterManager(RegisterManager):
-    all_regs              = r.ALL_REGS[:-1]
+    all_regs              = r.MANAGED_REGS
     box_types             = None       # or a list of acceptable types
     no_lower_byte_regs    = all_regs
     save_around_call_regs = r.VOLATILES
diff --git a/pypy/jit/backend/ppc/ppcgen/register.py b/pypy/jit/backend/ppc/ppcgen/register.py
--- a/pypy/jit/backend/ppc/ppcgen/register.py
+++ b/pypy/jit/backend/ppc/ppcgen/register.py
@@ -13,3 +13,8 @@
 SPP = r31
 SP  = r1
 RES = r3
+
+MANAGED_REGS = [r2, r3, r4, r5, r6, r7, r8, r9, r10,
+                r11, r12, r13, r14, r15, r16, r17, r18, 
+                r19, r20, r21, r22, r23, r24, r25, r26,
+                r27, r28, r29, r30]
diff --git a/pypy/jit/backend/ppc/ppcgen/test/test_helper.py b/pypy/jit/backend/ppc/ppcgen/test/test_helper.py
new file mode 100644
--- /dev/null
+++ b/pypy/jit/backend/ppc/ppcgen/test/test_helper.py
@@ -0,0 +1,25 @@
+from pypy.jit.backend.ppc.ppcgen.helper.assembler import (encode32, decode32)
+                                                          #encode64, decode64)
+
+def test_encode32():
+    mem = [None]*4
+    encode32(mem, 0, 1234567)
+    assert ''.join(mem) == '\x00\x12\xd6\x87'
+    mem = [None]*4
+    encode32(mem, 0, 983040)
+    assert ''.join(mem) == '\x00\x0F\x00\x00'
+
+def test_decode32():
+    mem = list('\x00\x12\xd6\x87')
+    assert decode32(mem, 0) ==  1234567
+    mem = list('\x00\x0F\x00\x00')
+    assert decode32(mem, 0) == 983040
+    mem = list("\x00\x00\x00\x03")
+    assert decode32(mem, 0) == 3
+
+def test_encode32_and_decode32():
+    mem = [None] * 4
+    for val in [1, 45654, -456456, 123, 99999]:
+        encode32(mem, 0, val)
+        assert decode32(mem, 0) == val
+
diff --git a/pypy/jit/backend/ppc/runner.py b/pypy/jit/backend/ppc/runner.py
--- a/pypy/jit/backend/ppc/runner.py
+++ b/pypy/jit/backend/ppc/runner.py
@@ -102,7 +102,7 @@
         addr = looptoken.ppc_code
         func = rffi.cast(lltype.Ptr(self.BOOTSTRAP_TP), addr)
         fail_index = self._execute_call(func)
-        return self.saved_descr[fail_index]
+        return self.get_fail_descr_from_number(fail_index)
 
     def _execute_call(self, func):
         prev_interpreter = None
diff --git a/pypy/jit/backend/ppc/test/test_runner.py b/pypy/jit/backend/ppc/test/test_runner.py
--- a/pypy/jit/backend/ppc/test/test_runner.py
+++ b/pypy/jit/backend/ppc/test/test_runner.py
@@ -5,7 +5,7 @@
     pass
 
 class TestPPC(LLtypeBackendTest):
-    
-    def setup_method(self, method):
-        self.cpu = PPC_64_CPU(rtyper=None, stats=FakeStats())
-        self.cpu.setup_once()
+   
+    def setup_class(cls):
+        cls.cpu = PPC_64_CPU(rtyper=None, stats=FakeStats())
+        cls.cpu.setup_once()
diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -155,7 +155,7 @@
         i1 = BoxInt()
         i2 = BoxInt()
         looptoken = LoopToken()
-        operations = [
+        operations = [ 
             ResOperation(rop.INT_ADD, [i0, ConstInt(1)], i1),
             ResOperation(rop.INT_LE, [i1, ConstInt(9)], i2),
             ResOperation(rop.GUARD_TRUE, [i2], None, descr=BasicFailDescr(2)),

From noreply at buildbot.pypy.org  Fri Oct 14 17:37:27 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 14 Oct 2011 17:37:27 +0200 (CEST)
Subject: [pypy-commit] pypy jit-tagged-2: Revive the 'jit-tagged' branch
	from scratch.
Message-ID: <20111014153727.672B482A88@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: jit-tagged-2
Changeset: r48057:dbec089a432d
Date: 2011-10-14 16:24 +0200
http://bitbucket.org/pypy/pypy/changeset/dbec089a432d/

Log:	Revive the 'jit-tagged' branch from scratch.


From noreply at buildbot.pypy.org  Fri Oct 14 17:37:28 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 14 Oct 2011 17:37:28 +0200 (CEST)
Subject: [pypy-commit] pypy jit-tagged-2: Enough to have the test pass.
Message-ID: <20111014153728.B608E82A88@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: jit-tagged-2
Changeset: r48058:3b36dff8a61d
Date: 2011-10-14 17:12 +0200
http://bitbucket.org/pypy/pypy/changeset/3b36dff8a61d/

Log:	Enough to have the test pass.

diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -800,8 +800,7 @@
 
     def rewrite_op_cast_ptr_to_int(self, op):
         if self._is_gc(op.args[0]):
-            #return op
-            raise NotImplementedError("cast_ptr_to_int")
+            return op
 
     def rewrite_op_force_cast(self, op):
         v_arg = op.args[0]
diff --git a/pypy/jit/metainterp/blackhole.py b/pypy/jit/metainterp/blackhole.py
--- a/pypy/jit/metainterp/blackhole.py
+++ b/pypy/jit/metainterp/blackhole.py
@@ -2,7 +2,7 @@
 from pypy.rlib.rtimer import read_timestamp
 from pypy.rlib.rarithmetic import intmask, LONG_BIT, r_uint, ovfcheck
 from pypy.rlib.objectmodel import we_are_translated
-from pypy.rlib.debug import debug_start, debug_stop
+from pypy.rlib.debug import debug_start, debug_stop, ll_assert
 from pypy.rlib.debug import make_sure_not_resized
 from pypy.rpython.lltypesystem import lltype, llmemory, rclass
 from pypy.rpython.lltypesystem.lloperation import llop
@@ -503,6 +503,15 @@
     @arguments("r", returns="r")
     def bhimpl_cast_opaque_ptr(a):
         return a
+    @arguments("r", returns="i")
+    def bhimpl_cast_ptr_to_int(a):
+        i = lltype.cast_ptr_to_int(a)
+        ll_assert((i & 1) == 1, "bhimpl_cast_ptr_to_int: not an odd int")
+        return i
+    @arguments("i", returns="r")
+    def bhimpl_cast_int_to_ptr(i):
+        ll_assert((i & 1) == 1, "bhimpl_cast_int_to_ptr: not an odd int")
+        return lltype.cast_int_to_ptr(llmemory.GCREF, i)
 
     @arguments("i", returns="i")
     def bhimpl_int_copy(a):
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -222,6 +222,7 @@
                     'cast_float_to_int', 'cast_int_to_float',
                     'cast_float_to_singlefloat', 'cast_singlefloat_to_float',
                     'float_neg', 'float_abs',
+                    'cast_ptr_to_int', 'cast_int_to_ptr',
                     ]:
         exec py.code.Source('''
             @arguments("box")
diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py
--- a/pypy/jit/metainterp/resoperation.py
+++ b/pypy/jit/metainterp/resoperation.py
@@ -431,6 +431,8 @@
     'INT_IS_TRUE/1b',
     'INT_NEG/1',
     'INT_INVERT/1',
+    'CAST_PTR_TO_INT/1',
+    'CAST_INT_TO_PTR/1',
     #
     'SAME_AS/1',      # gets a Const or a Box, turns it into another Box
     #
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -3440,7 +3440,6 @@
 
 class TestLLtype(BaseLLtypeTests, LLJitMixin):
     def test_tagged(self):
-        py.test.skip("implement me")
         from pypy.rlib.objectmodel import UnboxedValue
         class Base(object):
             __slots__ = ()
diff --git a/pypy/rpython/lltypesystem/llmemory.py b/pypy/rpython/lltypesystem/llmemory.py
--- a/pypy/rpython/lltypesystem/llmemory.py
+++ b/pypy/rpython/lltypesystem/llmemory.py
@@ -498,6 +498,8 @@
 
     def _cast_to_int(self, symbolic=False):
         if self:
+            if isinstance(self.ptr._obj0, int):    # tagged integer
+                return self.ptr._obj0
             if symbolic:
                 return AddressAsInt(self)
             else:

From noreply at buildbot.pypy.org  Fri Oct 14 17:37:30 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 14 Oct 2011 17:37:30 +0200 (CEST)
Subject: [pypy-commit] pypy jit-tagged-2: Support the necessary casts in the
 backend, and hit ll2ctypes
Message-ID: <20111014153730.17FCF82A88@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: jit-tagged-2
Changeset: r48059:559236779712
Date: 2011-10-14 17:36 +0200
http://bitbucket.org/pypy/pypy/changeset/559236779712/

Log:	Support the necessary casts in the backend, and hit ll2ctypes until
	it is happy to run tests.

diff --git a/pypy/jit/backend/llgraph/llimpl.py b/pypy/jit/backend/llgraph/llimpl.py
--- a/pypy/jit/backend/llgraph/llimpl.py
+++ b/pypy/jit/backend/llgraph/llimpl.py
@@ -165,6 +165,7 @@
     'unicodegetitem'  : (('ref', 'int'), 'int'),
     'unicodesetitem'  : (('ref', 'int', 'int'), 'int'),
     'cast_ptr_to_int' : (('ref',), 'int'),
+    'cast_int_to_ptr' : (('int',), 'ref'),
     'debug_merge_point': (('ref', 'int'), None),
     'force_token'     : ((), 'int'),
     'call_may_force'  : (('int', 'varargs'), 'intorptr'),
@@ -875,9 +876,6 @@
     def op_new_array(self, arraydescr, count):
         return do_new_array(arraydescr.ofs, count)
 
-    def op_cast_ptr_to_int(self, descr, ptr):
-        return cast_to_int(ptr)
-
     def op_force_token(self, descr):
         opaque_frame = _to_opaque(self)
         return llmemory.cast_ptr_to_adr(opaque_frame)
diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -1468,20 +1468,16 @@
         return u''.join(u.chars)
 
 
-    def test_casts(self):
-        py.test.skip("xxx fix or kill")
-        from pypy.rpython.lltypesystem import lltype, llmemory
-        TP = lltype.GcStruct('x')
-        x = lltype.malloc(TP)        
-        x = lltype.cast_opaque_ptr(llmemory.GCREF, x)
+    def test_cast_int_to_ptr(self):
+        res = self.execute_operation(rop.CAST_INT_TO_PTR,
+                                     [BoxInt(-17)],  'ref').value
+        assert lltype.cast_ptr_to_int(res) == -17
+
+    def test_cast_ptr_to_int(self):
+        x = lltype.cast_int_to_ptr(llmemory.GCREF, -19)
         res = self.execute_operation(rop.CAST_PTR_TO_INT,
-                                     [BoxPtr(x)],  'int').value
-        expected = self.cpu.cast_adr_to_int(llmemory.cast_ptr_to_adr(x))
-        assert rffi.get_real_int(res) == rffi.get_real_int(expected)
-        res = self.execute_operation(rop.CAST_PTR_TO_INT,
-                                     [ConstPtr(x)],  'int').value
-        expected = self.cpu.cast_adr_to_int(llmemory.cast_ptr_to_adr(x))
-        assert rffi.get_real_int(res) == rffi.get_real_int(expected)
+                                     [BoxPtr(x)], 'int').value
+        assert res == -19
 
     def test_ooops_non_gc(self):
         x = lltype.malloc(lltype.Struct('x'), flavor='raw')
@@ -2299,13 +2295,6 @@
         #
         cpu.bh_strsetitem(x, 4, ord('/'))
         assert str.chars[4] == '/'
-        #
-##        x = cpu.bh_newstr(5)
-##        y = cpu.bh_cast_ptr_to_int(x)
-##        z = cpu.bh_cast_ptr_to_int(x)
-##        y = rffi.get_real_int(y)
-##        z = rffi.get_real_int(z)
-##        assert type(y) == type(z) == int and y == z
 
     def test_sorting_of_fields(self):
         S = self.S
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1387,7 +1387,8 @@
 
     def genop_same_as(self, op, arglocs, resloc):
         self.mov(arglocs[0], resloc)
-    #genop_cast_ptr_to_int = genop_same_as
+    genop_cast_ptr_to_int = genop_same_as
+    genop_cast_int_to_ptr = genop_same_as
 
     def genop_int_mod(self, op, arglocs, resloc):
         if IS_X86_32:
diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py
--- a/pypy/jit/backend/x86/regalloc.py
+++ b/pypy/jit/backend/x86/regalloc.py
@@ -1152,7 +1152,8 @@
         self.possibly_free_var(op.getarg(0))
         resloc = self.force_allocate_reg(op.result)
         self.Perform(op, [argloc], resloc)
-    #consider_cast_ptr_to_int = consider_same_as
+    consider_cast_ptr_to_int = consider_same_as
+    consider_cast_int_to_ptr = consider_same_as
 
     def consider_strlen(self, op):
         args = op.getarglist()
diff --git a/pypy/rpython/lltypesystem/ll2ctypes.py b/pypy/rpython/lltypesystem/ll2ctypes.py
--- a/pypy/rpython/lltypesystem/ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/ll2ctypes.py
@@ -658,6 +658,8 @@
         if T == llmemory.GCREF:
             if isinstance(llobj._obj, _llgcopaque):
                 return ctypes.c_void_p(llobj._obj.intval)
+            if isinstance(llobj._obj, int):    # tagged pointer
+                return ctypes.c_void_p(llobj._obj)
             container = llobj._obj.container
             T = lltype.Ptr(lltype.typeOf(container))
             # otherwise it came from integer and we want a c_void_p with
@@ -1268,6 +1270,7 @@
 class _llgcopaque(lltype._container):
     _TYPE = llmemory.GCREF.TO
     _name = "_llgcopaque"
+    _read_directly_intval = True     # for _ptr._cast_to_int()
 
     def __init__(self, void_p):
         if isinstance(void_p, (int, long)):
@@ -1299,11 +1302,6 @@
             return _opaque_objs[self.intval // 2]
         return force_cast(PTRTYPE, self.intval)
 
-##     def _cast_to_int(self):
-##         return self.intval
-
-##     def _cast_to_adr(self):
-##         return _lladdress(self.intval)
 
 def cast_adr_to_int(addr):
     if isinstance(addr, llmemory.fakeaddress):
diff --git a/pypy/rpython/lltypesystem/lltype.py b/pypy/rpython/lltypesystem/lltype.py
--- a/pypy/rpython/lltypesystem/lltype.py
+++ b/pypy/rpython/lltypesystem/lltype.py
@@ -1360,6 +1360,8 @@
         obj = normalizeptr(self, check)._getobj(check)
         if isinstance(obj, int):
             return obj     # special case for cast_int_to_ptr() results put into opaques
+        if getattr(obj, '_read_directly_intval', False):
+            return obj.intval   # special case for _llgcopaque
         result = intmask(obj._getid())
         # assume that id() returns an addressish value which is
         # not zero and aligned to at least a multiple of 4
diff --git a/pypy/rpython/lltypesystem/test/test_ll2ctypes.py b/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
--- a/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
@@ -1123,6 +1123,9 @@
 
         #assert lltype.cast_ptr_to_int(ref1) == intval
 
+        x = rffi.cast(llmemory.GCREF, -17)
+        assert lltype.cast_ptr_to_int(x) == -17
+
     def test_ptr_truth(self):
         abc = rffi.cast(lltype.Ptr(lltype.FuncType([], lltype.Void)), 0)
         assert not abc

From noreply at buildbot.pypy.org  Fri Oct 14 18:06:57 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 14 Oct 2011 18:06:57 +0200 (CEST)
Subject: [pypy-commit] pypy jit-tagged-2: Remove an apparently-superflous
 cast_opaque_ptr here.
Message-ID: <20111014160657.DE69182A88@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: jit-tagged-2
Changeset: r48060:839eed159d0a
Date: 2011-10-14 17:54 +0200
http://bitbucket.org/pypy/pypy/changeset/839eed159d0a/

Log:	Remove an apparently-superflous cast_opaque_ptr here.

diff --git a/pypy/rlib/rerased.py b/pypy/rlib/rerased.py
--- a/pypy/rlib/rerased.py
+++ b/pypy/rlib/rerased.py
@@ -224,9 +224,7 @@
                          resulttype = lltype.Signed)
         v_instance = hop.genop('cast_int_to_ptr', [v2p1],
                                resulttype=self.lowleveltype)
-        v = hop.genop('cast_opaque_ptr', [v_instance],
-                      resulttype=self.lowleveltype)
-        return v
+        return v_instance
 
     def convert_const(self, value):
         if value._identity is _identity_for_ints:

From noreply at buildbot.pypy.org  Fri Oct 14 18:06:59 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 14 Oct 2011 18:06:59 +0200 (CEST)
Subject: [pypy-commit] pypy jit-tagged-2: Add a test, which passes.
Message-ID: <20111014160659.2002182A88@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: jit-tagged-2
Changeset: r48061:1ab0b8bf4eaa
Date: 2011-10-14 18:02 +0200
http://bitbucket.org/pypy/pypy/changeset/1ab0b8bf4eaa/

Log:	Add a test, which passes.

diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -3491,3 +3491,35 @@
                 pc += 1
             return pc
         res = self.meta_interp(main, [False, 100, True], taggedpointers=True)
+
+    def test_rerased(self):
+        from pypy.rlib.rerased import erase_int, unerase_int, new_erasing_pair
+        eraseX, uneraseX = new_erasing_pair("X")
+        #
+        class X:
+            def __init__(self, a, b):
+                self.a = a
+                self.b = b
+        #
+        def f(i, j):
+            # 'j' should be 0 or 1, not other values
+            if j > 0:
+                e = eraseX(X(i, j))
+            else:
+                try:
+                    e = erase_int(i)
+                except OverflowError:
+                    return -42
+            if j & 1:
+                x = uneraseX(e)
+                return x.a - x.b
+            else:
+                return unerase_int(e)
+        #
+        x = self.interp_operations(f, [-128, 0], taggedpointers=True)
+        assert x == -128
+        bigint = sys.maxint//2 + 1
+        x = self.interp_operations(f, [bigint, 0], taggedpointers=True)
+        assert x == -42
+        x = self.interp_operations(f, [1000, 1], taggedpointers=True)
+        assert x == 999

From noreply at buildbot.pypy.org  Fri Oct 14 18:30:22 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Fri, 14 Oct 2011 18:30:22 +0200 (CEST)
Subject: [pypy-commit] pypy default: interplevel_cls on type objects is
	immutable
Message-ID: <20111014163022.0E29B82A88@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r48062:8e5fb2c551cf
Date: 2011-10-14 12:30 -0400
http://bitbucket.org/pypy/pypy/changeset/8e5fb2c551cf/

Log:	interplevel_cls on type objects is immutable

diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -102,6 +102,7 @@
                           'instancetypedef',
                           'terminator',
                           '_version_tag?',
+                          'interplevel_cls',
                           ]
 
     # for config.objspace.std.getattributeshortcut

From noreply at buildbot.pypy.org  Fri Oct 14 19:33:34 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Fri, 14 Oct 2011 19:33:34 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix these tests,
 assertRaises deosn't eval like py.test.raises
Message-ID: <20111014173334.63A3282A88@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r48063:3ea73d1d6fb1
Date: 2011-10-14 13:33 -0400
http://bitbucket.org/pypy/pypy/changeset/3ea73d1d6fb1/

Log:	fix these tests, assertRaises deosn't eval like py.test.raises

diff --git a/lib-python/modified-2.7/test/test_array.py b/lib-python/modified-2.7/test/test_array.py
--- a/lib-python/modified-2.7/test/test_array.py
+++ b/lib-python/modified-2.7/test/test_array.py
@@ -295,9 +295,10 @@
         )
 
         b = array.array(self.badtypecode())
-        self.assertRaises(TypeError, "a + b")
-
-        self.assertRaises(TypeError, "a + 'bad'")
+        with self.assertRaises(TypeError):
+            a + b
+        with self.assertRaises(TypeError):
+            a + 'bad'
 
     def test_iadd(self):
         a = array.array(self.typecode, self.example[::-1])
@@ -316,9 +317,10 @@
         )
 
         b = array.array(self.badtypecode())
-        self.assertRaises(TypeError, "a += b")
-
-        self.assertRaises(TypeError, "a += 'bad'")
+        with self.assertRaises(TypeError):
+            a += b
+        with self.assertRaises(TypeError):
+            a += 'bad'
 
     def test_mul(self):
         a = 5*array.array(self.typecode, self.example)
@@ -345,7 +347,8 @@
             array.array(self.typecode)
         )
 
-        self.assertRaises(TypeError, "a * 'bad'")
+        with self.assertRaises(TypeError):
+            a * 'bad'
 
     def test_imul(self):
         a = array.array(self.typecode, self.example)
@@ -374,7 +377,8 @@
         a *= -1
         self.assertEqual(a, array.array(self.typecode))
 
-        self.assertRaises(TypeError, "a *= 'bad'")
+        with self.assertRaises(TypeError):
+            a *= 'bad'
 
     def test_getitem(self):
         a = array.array(self.typecode, self.example)

From noreply at buildbot.pypy.org  Fri Oct 14 21:29:18 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Fri, 14 Oct 2011 21:29:18 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-complex: A branch to add numpy complex
	dtypes.
Message-ID: <20111014192918.CC5F282A88@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: numpy-complex
Changeset: r48064:72841b4aea0c
Date: 2011-10-14 14:29 -0400
http://bitbucket.org/pypy/pypy/changeset/72841b4aea0c/

Log:	A branch to add numpy complex dtypes.


From noreply at buildbot.pypy.org  Fri Oct 14 21:29:20 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Fri, 14 Oct 2011 21:29:20 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-complex: Initial start. It leaks memory
 like a seive. We need lltype.malloc(flavor="stack") I think.
Message-ID: <20111014192920.0E15182A89@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: numpy-complex
Changeset: r48065:a973209a2d26
Date: 2011-10-14 15:28 -0400
http://bitbucket.org/pypy/pypy/changeset/a973209a2d26/

Log:	Initial start. It leaks memory like a seive. We need
	lltype.malloc(flavor="stack") I think.

diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -18,6 +18,7 @@
 SIGNEDLTR = "i"
 BOOLLTR = "b"
 FLOATINGLTR = "f"
+COMPLEXLTR = "c"
 
 class W_Dtype(Wrappable):
     def __init__(self, space):
@@ -118,9 +119,19 @@
     W_LowLevelDtype.name = name
     W_LowLevelDtype.aliases = aliases
     W_LowLevelDtype.applevel_types = applevel_types
-    W_LowLevelDtype.num_bytes = rffi.sizeof(T)
-    if expected_size is not None:
-        assert W_LowLevelDtype.num_bytes == expected_size
+
+    if hasattr(T, "_arrayfld"):
+        # Hack for structs (read: complex), rffi.sizeof(Struct) returns a
+        # symbolic, which can't be compared
+        assert T._arrayfld is None
+        primitive_size = sum(rffi.sizeof(VT) for VT in T._flds.itervalues())
+    else:
+        primitive_size = rffi.sizeof(T)
+    if expected_size is None:
+        expected_size = primitive_size
+    assert expected_size == primitive_size
+    W_LowLevelDtype.num_bytes = expected_size
+
     return W_LowLevelDtype
 
 
@@ -465,12 +476,55 @@
 class W_Float64Dtype(FloatArithmeticDtype, W_Float64Dtype):
     pass
 
+ComplexDouble = lltype.Struct("Complex Double",
+    ("real", lltype.Float),
+    ("imag", lltype.Float),
+)
+W_Complex128Dtype = create_low_level_dtype(
+    num = 15, kind = COMPLEXLTR, name = "complex128",
+    aliases = [],
+    applevel_types = ["complex"],
+    T = ComplexDouble,
+    valtype = ComplexDouble,
+    expected_size = 16,
+)
+class W_Complex128Dtype(W_Complex128Dtype):
+    def _create_complex_struct(self, real, imag):
+        # Really we want to stack alloc it, doesn't seem possible though.
+        # Unforutnately a raw alloc means we'll leak memory (no gc) and it
+        # can't be a GcStruct because the array storage isn't a GcArray.
+        # So basically we need stack allocs.
+        c = lltype.malloc(ComplexDouble, flavor="raw")
+        c.real = real
+        c.imag = imag
+        return c
+
+    @specialize.argtype(1)
+    def adapt_val(self, val):
+        if hasattr(val, "_T"):
+            assert val._T is ComplexDouble
+        else:
+            val = self._create_complex_struct(rffi.cast(lltype.Float, val), 0.0)
+        return self.box(val)
+
+    def unwrap(self, space, w_item):
+        real, imag = space.unpackcomplex(w_item)
+        return self.adapt_val(self._create_complex_struct(real, imag))
+
+    def setitem(self, storage, i, item):
+        val = self.unbox(item)
+        # You can't set a full struct, gotta do it one field at a time.
+        self.unerase(storage)[i].real = val.real
+        self.unerase(storage)[i].imag = val.imag
+
+
 ALL_DTYPES = [
     W_BoolDtype,
     W_Int8Dtype, W_UInt8Dtype, W_Int16Dtype, W_UInt16Dtype,
     W_Int32Dtype, W_UInt32Dtype, W_LongDtype, W_ULongDtype,
     W_Int64Dtype, W_UInt64Dtype,
     W_Float32Dtype, W_Float64Dtype,
+    W_Complex128Dtype,
 ]
 
 dtypes_by_alias = unrolling_iterable([
@@ -496,6 +550,7 @@
     __str__ = interp2app(W_Dtype.descr_str),
 
     num = interp_attrproperty("num", cls=W_Dtype),
+    name = interp_attrproperty("name", cls=W_Dtype),
     kind = interp_attrproperty("kind", cls=W_Dtype),
     itemsize = interp_attrproperty("num_bytes", cls=W_Dtype),
     shape = GetSetProperty(W_Dtype.descr_get_shape),
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -38,8 +38,6 @@
             w_dtype = None
             for w_item in l:
                 w_dtype = interp_ufuncs.find_dtype_for_scalar(space, w_item, w_dtype)
-                if w_dtype is space.fromcache(interp_dtype.W_Float64Dtype):
-                    break
             if w_dtype is None:
                 w_dtype = space.w_None
 
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -241,6 +241,7 @@
     bool_dtype = space.fromcache(interp_dtype.W_BoolDtype)
     long_dtype = space.fromcache(interp_dtype.W_LongDtype)
     int64_dtype = space.fromcache(interp_dtype.W_Int64Dtype)
+    complex128_dtype = space.fromcache(interp_dtype.W_Complex128Dtype)
 
     if space.is_w(w_type, space.w_bool):
         if current_guess is None or current_guess is bool_dtype:
@@ -256,6 +257,8 @@
             current_guess is long_dtype or current_guess is int64_dtype):
             return int64_dtype
         return current_guess
+    elif space.is_w(w_type, space.w_complex):
+        return complex128_dtype
     return space.fromcache(interp_dtype.W_Float64Dtype)
 
 
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -161,3 +161,12 @@
 
         # You can't subclass dtype
         raises(TypeError, type, "Foo", (dtype,), {})
+
+    def test_complex_dtype(self):
+        from numpy import dtype
+
+        d = dtype(complex)
+        assert d.kind == "c"
+        assert d.name == "complex128"
+        assert d.num == 15
+        assert d.itemsize == 16
\ No newline at end of file
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -558,6 +558,8 @@
         assert array([1.2, True]).dtype is dtype(float)
         assert array([1.2, 5]).dtype is dtype(float)
         assert array([]).dtype is dtype(float)
+        assert array([1 + 2j]).dtype is dtype(complex)
+        assert array([1.0, 1 + 2j]).dtype is dtype(complex)
 
     def test_comparison(self):
         import operator

From noreply at buildbot.pypy.org  Fri Oct 14 22:29:19 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Fri, 14 Oct 2011 22:29:19 +0200 (CEST)
Subject: [pypy-commit] pypy malloc-value: a branch to allow mallocing
 structs with a flavor of "value"m which means stack allocated
Message-ID: <20111014202919.68CF082A88@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: malloc-value
Changeset: r48066:96f75f789ae7
Date: 2011-10-14 16:02 -0400
http://bitbucket.org/pypy/pypy/changeset/96f75f789ae7/

Log:	a branch to allow mallocing structs with a flavor of "value"m which
	means stack allocated


From noreply at buildbot.pypy.org  Fri Oct 14 22:29:20 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Fri, 14 Oct 2011 22:29:20 +0200 (CEST)
Subject: [pypy-commit] pypy malloc-value: malloc flavor = "value" now works.
Message-ID: <20111014202920.A914E82A88@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: malloc-value
Changeset: r48067:4dd646606d28
Date: 2011-10-14 16:02 -0400
http://bitbucket.org/pypy/pypy/changeset/4dd646606d28/

Log:	malloc flavor = "value" now works.

diff --git a/pypy/rpython/lltypesystem/lltype.py b/pypy/rpython/lltypesystem/lltype.py
--- a/pypy/rpython/lltypesystem/lltype.py
+++ b/pypy/rpython/lltypesystem/lltype.py
@@ -48,7 +48,7 @@
         self.TYPE = TYPE
     def __repr__(self):
         return '<Uninitialized %r>'%(self.TYPE,)
-        
+
 
 def saferecursive(func, defl, TLS=TLS):
     def safe(*args):
@@ -302,7 +302,7 @@
     def _names_without_voids(self):
         names_without_voids = [name for name in self._names if self._flds[name] is not Void]
         return names_without_voids
-    
+
     def _str_fields_without_voids(self):
         return ', '.join(['%s: %s' % (name, self._flds[name])
                           for name in self._names_without_voids(False)])
@@ -410,7 +410,7 @@
     _gckind = 'raw'
     __name__ = 'array'
     _anonym_struct = False
-    
+
     def __init__(self, *fields, **kwds):
         if len(fields) == 1 and isinstance(fields[0], LowLevelType):
             self.OF = fields[0]
@@ -537,9 +537,9 @@
         return "Func ( %s ) -> %s" % (args, self.RESULT)
     __str__ = saferecursive(__str__, '...')
 
-    def _short_name(self):        
+    def _short_name(self):
         args = ', '.join([ARG._short_name() for ARG in self.ARGS])
-        return "Func(%s)->%s" % (args, self.RESULT._short_name())        
+        return "Func(%s)->%s" % (args, self.RESULT._short_name())
     _short_name = saferecursive(_short_name, '...')
 
     def _container_example(self):
@@ -553,7 +553,7 @@
 
 class OpaqueType(ContainerType):
     _gckind = 'raw'
-    
+
     def __init__(self, tag, hints={}):
         """ if hints['render_structure'] is set, the type is internal and not considered
             to come from somewhere else (it should be rendered as a structure) """
@@ -663,7 +663,7 @@
 
     def normalized(self):
         return build_number(None, normalizedinttype(self._type))
-        
+
 
 _numbertypes = {int: Number("Signed", int, intmask)}
 _numbertypes[r_int] = _numbertypes[int]
@@ -723,10 +723,10 @@
 
     def __str__(self):
         return '* %s' % (self.TO, )
-    
+
     def _short_name(self):
         return 'Ptr %s' % (self.TO._short_name(), )
-    
+
     def _is_atomic(self):
         return self.TO._gckind == 'raw'
 
@@ -753,7 +753,7 @@
                          adtmeths=TO._adtmeths)
         else:
             R = GcStruct("Interior", ('ptr', self), ('index', Signed),
-                         hints={'interior_ptr_type':True})            
+                         hints={'interior_ptr_type':True})
         return R
 
 class InteriorPtr(LowLevelType):
@@ -898,7 +898,7 @@
             return dwn
         OUTSIDE = getattr(OUTSIDE, first)
     return -1
- 
+
 def castable(PTRTYPE, CURTYPE):
     if CURTYPE.TO._gckind != PTRTYPE.TO._gckind:
         raise TypeError("cast_pointer() cannot change the gc status: %s to %s"
@@ -1107,7 +1107,7 @@
 
     # _setobj, _getobj and _obj0 are really _internal_ implementations details of _ptr,
     # use _obj if necessary instead !
-    def _setobj(self, pointing_to, solid=False):        
+    def _setobj(self, pointing_to, solid=False):
         if pointing_to is None:
             obj0 = None
         elif (solid or self._T._gckind != 'raw' or
@@ -1118,7 +1118,7 @@
             obj0 = weakref.ref(pointing_to)
         self._set_solid(solid)
         self._set_obj0(obj0)
-        
+
     def _getobj(self, check=True):
         obj = self._obj0
         if obj is not None:
@@ -1294,7 +1294,7 @@
             return result
 
 class _ptr(_abstract_ptr):
-    __slots__ = ('_TYPE', 
+    __slots__ = ('_TYPE',
                  '_weak', '_solid',
                  '_obj0', '__weakref__')
 
@@ -1447,9 +1447,9 @@
             assert T._gckind == 'raw'
             val = _interior_ptr(T, self._parent, self._offsets + [offset])
         return val
-    
-    
-        
+
+
+
 assert not '__dict__' in dir(_interior_ptr)
 
 class _container(object):
@@ -1520,7 +1520,7 @@
             and parentindex in (self._parent_type._names[0], 0)
             and self._TYPE._gckind == typeOf(parent)._gckind):
             # keep strong reference to parent, we share the same allocation
-            self._keepparent = parent 
+            self._keepparent = parent
 
     def _parentstructure(self, check=True):
         if self._wrparent is not None:
@@ -1566,7 +1566,7 @@
             __slots__ = flds
         cache[tag] = _struct1
         return _struct1
- 
+
 #for pickling support:
 def _get_empty_instance_of_struct_variety(flds):
     cls = _struct_variety(flds)
@@ -1629,7 +1629,7 @@
         return r
 
     # for FixedSizeArray kind of structs:
-    
+
     def getlength(self):
         assert isinstance(self._TYPE, FixedSizeArray)
         return self._TYPE.length
@@ -1928,7 +1928,7 @@
         # if we are an opaque containing a normal Struct/GcStruct,
         # unwrap it
         if hasattr(self, 'container'):
-            # an integer, cast to a ptr, cast to an opaque    
+            # an integer, cast to a ptr, cast to an opaque
             if type(self.container) is int:
                 return self.container
             if getattr(self.container, '_carry_around_for_tests', False):
@@ -1955,7 +1955,7 @@
 
 def malloc(T, n=None, flavor='gc', immortal=False, zero=False,
            track_allocation=True, add_memory_pressure=False):
-    assert flavor in ('gc', 'raw')
+    assert flavor in ('gc', 'raw', 'value')
     if zero or immortal:
         initialization = 'example'
     elif flavor == 'raw':
@@ -2045,7 +2045,7 @@
 
 def pyobjectptr(obj):
     o = _pyobject(obj)
-    return _ptr(Ptr(PyObject), o) 
+    return _ptr(Ptr(PyObject), o)
 
 def cast_ptr_to_int(ptr):
     return ptr._cast_to_int()
@@ -2069,7 +2069,7 @@
     if not isinstance(GCSTRUCT, RttiStruct):
         raise TypeError, "expected a RttiStruct: %s" % GCSTRUCT
     if GCSTRUCT._runtime_type_info is None:
-        raise ValueError, ("no attached runtime type info for GcStruct %s" % 
+        raise ValueError, ("no attached runtime type info for GcStruct %s" %
                            GCSTRUCT._name)
     return _ptr(Ptr(RuntimeTypeInfo), GCSTRUCT._runtime_type_info)
 
diff --git a/pypy/rpython/lltypesystem/test/test_lltype.py b/pypy/rpython/lltypesystem/test/test_lltype.py
--- a/pypy/rpython/lltypesystem/test/test_lltype.py
+++ b/pypy/rpython/lltypesystem/test/test_lltype.py
@@ -235,7 +235,7 @@
     del p1
     import gc
     gc.collect()
-    py.test.raises(RuntimeError, "p1_5.v")        
+    py.test.raises(RuntimeError, "p1_5.v")
 
 def test_examples():
     A1 = GcArray(('v', Signed))
@@ -388,11 +388,11 @@
         s.x = 1
     def type_info_S(p):
         return getRuntimeTypeInfo(S)
-    qp = functionptr(FuncType([Ptr(S)], Ptr(RuntimeTypeInfo)), 
-                     "type_info_S", 
+    qp = functionptr(FuncType([Ptr(S)], Ptr(RuntimeTypeInfo)),
+                     "type_info_S",
                      _callable=type_info_S)
-    dp = functionptr(FuncType([Ptr(S)], Void), 
-                     "destructor_funcptr", 
+    dp = functionptr(FuncType([Ptr(S)], Void),
+                     "destructor_funcptr",
                      _callable=f)
     pinf0 = attachRuntimeTypeInfo(S, qp, destrptr=dp)
     assert pinf0._obj.about == S
@@ -422,8 +422,8 @@
             return getRuntimeTypeInfo(S)
         else:
             return getRuntimeTypeInfo(S1)
-    fp = functionptr(FuncType([Ptr(S)], Ptr(RuntimeTypeInfo)), 
-                     "dynamic_type_info_S", 
+    fp = functionptr(FuncType([Ptr(S)], Ptr(RuntimeTypeInfo)),
+                     "dynamic_type_info_S",
                      _callable=dynamic_type_info_S)
     attachRuntimeTypeInfo(S, fp)
     assert s.x == 0
@@ -434,7 +434,7 @@
     py.test.raises(RuntimeError, "runtime_type_info(s1.sub)")
     s1.sub.x = 1
     assert runtime_type_info(s1.sub) == getRuntimeTypeInfo(S1)
-    
+
 def test_flavor_malloc():
     def isweak(p, T):
         return p._weak and typeOf(p).TO == T
@@ -450,7 +450,7 @@
     p = malloc(T, flavor="gc")
     assert typeOf(p).TO == T
     assert not isweak(p, T)
-    
+
 def test_opaque():
     O = OpaqueType('O')
     p1 = opaqueptr(O, 'p1', hello="world")
@@ -520,8 +520,8 @@
 def test_adtmeths():
     def h_newstruct():
         return malloc(S)
-    
-    S = GcStruct('s', ('x', Signed), 
+
+    S = GcStruct('s', ('x', Signed),
                  adtmeths={"h_newstruct": h_newstruct})
 
     s = S.h_newstruct()
@@ -553,15 +553,15 @@
     def h_newstruct(S):
         return malloc(S)
     h_newstruct = typeMethod(h_newstruct)
-    
-    S = GcStruct('s', ('x', Signed), 
+
+    S = GcStruct('s', ('x', Signed),
                  adtmeths={"h_newstruct": h_newstruct})
 
     s = S.h_newstruct()
 
     assert typeOf(s) == Ptr(S)
 
-    Sprime = GcStruct('s', ('x', Signed), 
+    Sprime = GcStruct('s', ('x', Signed),
                       adtmeths={"h_newstruct": h_newstruct})
 
     assert S == Sprime
@@ -592,7 +592,7 @@
     PA = Ptr(A)
     a = malloc(A, 2)
     assert cast_pointer(PA, a) == a
-        
+
 def test_array_with_no_length():
     A = GcArray(Signed, hints={'nolength': True})
     a = malloc(A, 10)
@@ -604,7 +604,7 @@
     s = malloc(GcS)
     s.x = 1
     assert list(dissect_ll_instance(s)) == [(Ptr(GcS), s), (GcS, s._obj), (Signed, 1)]
-    
+
     A = GcArray(('x', Signed))
     a = malloc(A, 10)
     for i in range(10):
@@ -812,6 +812,14 @@
     assert F.RESULT == Signed
     assert F.ARGS == (Signed,)
 
+def test_malloc_value():
+    T = Struct("complex", ("real", Float), ("imag", Float))
+    c = malloc(T, flavor="value")
+    c.real = 4.0
+    c.imag = 5.0
+    assert c.real == 4.0
+    assert c.imag == 5.0
+
 
 class TestTrackAllocation:
     def test_automatic_tracking(self):

From noreply at buildbot.pypy.org  Fri Oct 14 22:29:21 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Fri, 14 Oct 2011 22:29:21 +0200 (CEST)
Subject: [pypy-commit] pypy malloc-value: failing test,
 not sure where in the annotator/rtyper I need to modify
Message-ID: <20111014202921.EA48982A88@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: malloc-value
Changeset: r48068:b2ed57719eb3
Date: 2011-10-14 16:28 -0400
http://bitbucket.org/pypy/pypy/changeset/b2ed57719eb3/

Log:	failing test, not sure where in the annotator/rtyper I need to
	modify

diff --git a/pypy/rpython/lltypesystem/lltype.py b/pypy/rpython/lltypesystem/lltype.py
--- a/pypy/rpython/lltypesystem/lltype.py
+++ b/pypy/rpython/lltypesystem/lltype.py
@@ -1960,6 +1960,8 @@
         initialization = 'example'
     elif flavor == 'raw':
         initialization = 'raw'
+    elif flavor == 'value':
+        initialization = 'value'
     else:
         initialization = 'malloc'
     if isinstance(T, Struct):
@@ -1973,6 +1975,8 @@
         raise TypeError, "malloc for Structs and Arrays only"
     if T._gckind != 'gc' and not immortal and flavor.startswith('gc'):
         raise TypeError, "gc flavor malloc of a non-GC non-immortal structure"
+    if T._gckind == 'gc' and flavor == 'value':
+        raise TypeError("Can't malloc with flavor value on a gc struct/array")
     if flavor == "raw" and not immortal and track_allocation:
         leakfinder.remember_malloc(o, framedepth=2)
     solid = immortal or not flavor.startswith('gc') # immortal or non-gc case
diff --git a/pypy/rpython/lltypesystem/test/test_lltype.py b/pypy/rpython/lltypesystem/test/test_lltype.py
--- a/pypy/rpython/lltypesystem/test/test_lltype.py
+++ b/pypy/rpython/lltypesystem/test/test_lltype.py
@@ -820,6 +820,9 @@
     assert c.real == 4.0
     assert c.imag == 5.0
 
+    S = GcStruct("S", ("x", Signed))
+    py.test.raises(TypeError, malloc, S, flavor="value")
+
 
 class TestTrackAllocation:
     def test_automatic_tracking(self):
diff --git a/pypy/translator/c/test/test_lltyped.py b/pypy/translator/c/test/test_lltyped.py
--- a/pypy/translator/c/test/test_lltyped.py
+++ b/pypy/translator/c/test/test_lltyped.py
@@ -312,14 +312,14 @@
         from pypy.rpython.lltypesystem.rstr import STR
         from pypy.rpython.lltypesystem import rffi, llmemory, lltype
         P = lltype.Ptr(lltype.FixedSizeArray(lltype.Char, 1))
-        
+
         def f():
             a = llstr("xyz")
             b = (llmemory.cast_ptr_to_adr(a) + llmemory.offsetof(STR, 'chars')
                  + llmemory.itemoffsetof(STR.chars, 0))
             buf = rffi.cast(rffi.VOIDP, b)
             return buf[2]
-        
+
         fn = self.getcompiled(f, [])
         res = fn()
         assert res == 'z'
@@ -436,6 +436,24 @@
         res = fn(100)
         assert res == 42
 
+    def test_malloc_value(self):
+        T = Struct('complex', ('real', Float), ('imag', Float))
+        def f(n):
+            x = malloc(T, flavor="value")
+            x.real = 0.0
+            x.imag = 0.0
+            for i in range(n):
+                z = malloc(T, flavor="value")
+                z.real = x.real
+                z.imag = x.imag
+                x = z
+            return x.real + x.real
+
+        fn = self.getcompiled(f, [int])
+        res = fn(100)
+        assert res == 200
+
+
     def test_arithmetic_cornercases(self):
         import operator, sys
         from pypy.rlib.unroll import unrolling_iterable

From noreply at buildbot.pypy.org  Sat Oct 15 14:43:54 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 15 Oct 2011 14:43:54 +0200 (CEST)
Subject: [pypy-commit] pypy default: Force a reinit during translation,
 otherwise we might be left with a
Message-ID: <20111015124354.578AC82A90@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48070:6650d6cdcdbe
Date: 2011-10-15 11:11 +0200
http://bitbucket.org/pypy/pypy/changeset/6650d6cdcdbe/

Log:	Force a reinit during translation, otherwise we might be left with a
	broken minicache entry.

diff --git a/pypy/module/thread/threadlocals.py b/pypy/module/thread/threadlocals.py
--- a/pypy/module/thread/threadlocals.py
+++ b/pypy/module/thread/threadlocals.py
@@ -12,6 +12,10 @@
         self._mostrecentkey = 0        # fast minicaching for the common case
         self._mostrecentvalue = None   # fast minicaching for the common case
 
+    def _freeze_(self):
+        self.__init__()
+        return False
+
     def getvalue(self):
         ident = thread.get_ident()
         if ident == self._mostrecentkey:

From noreply at buildbot.pypy.org  Sat Oct 15 14:43:55 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 15 Oct 2011 14:43:55 +0200 (CEST)
Subject: [pypy-commit] pypy default: Baaah. Fought for a while before
 realizing that it was because
Message-ID: <20111015124355.89C3982A90@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48071:1f3d56c3352f
Date: 2011-10-15 14:04 +0200
http://bitbucket.org/pypy/pypy/changeset/1f3d56c3352f/

Log:	Baaah. Fought for a while before realizing that it was because I
	mistakenly wrote "_immutable_ = ['field']". Grr. Should not occur
	again with this extra check.

diff --git a/pypy/rpython/rclass.py b/pypy/rpython/rclass.py
--- a/pypy/rpython/rclass.py
+++ b/pypy/rpython/rclass.py
@@ -191,6 +191,10 @@
                     "class %r inherits from its parent _immutable_=True, "
                     "so it should also declare _immutable_=True" % (
                     self.classdef,))
+            if loc.classdict.get('_immutable_').value is not True:
+                raise TyperError(
+                    "class %r: _immutable_ = something else than True" % (
+                    self.classdef,))
             hints = hints.copy()
             hints['immutable'] = True
         self.immutable_field_set = set()  # unless overwritten below

From noreply at buildbot.pypy.org  Sat Oct 15 14:43:56 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 15 Oct 2011 14:43:56 +0200 (CEST)
Subject: [pypy-commit] pypy default: Use a quasi-immutable field to record
 whether we started using
Message-ID: <20111015124356.BB4CD82A90@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48072:51224696c078
Date: 2011-10-15 14:43 +0200
http://bitbucket.org/pypy/pypy/changeset/51224696c078/

Log:	Use a quasi-immutable field to record whether we started using
	threads or not so far. As long as we didn't, we can generate
	assembler loops that don't need to decrement the GIL counter.

diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -391,8 +391,11 @@
     def decrement_ticker(self, by):
         value = self._ticker
         if self.has_bytecode_counter:    # this 'if' is constant-folded
-            value -= by
-            self._ticker = value
+            if jit.isconstant(by) and by == 0:
+                pass     # normally constant-folded too
+            else:
+                value -= by
+                self._ticker = value
         return value
 
 
diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py
--- a/pypy/module/pypyjit/interp_jit.py
+++ b/pypy/module/pypyjit/interp_jit.py
@@ -137,20 +137,15 @@
 
     def jump_absolute(self, jumpto, _, ec=None):
         if we_are_jitted():
-            # Normally, the tick counter is decremented by 100 for every
-            # Python opcode.  Here, to better support JIT compilation of
-            # small loops, we decrement it by a possibly smaller constant.
-            # We get the maximum 100 when the (unoptimized) trace length
-            # is at least 3200 (a bit randomly).
-            trace_length = r_uint(current_trace_length())
-            decr_by = trace_length // 32
-            if decr_by < 1:
-                decr_by = 1
-            elif decr_by > 100:    # also if current_trace_length() returned -1
-                decr_by = 100
+            #
+            # assume that only threads are using the bytecode counter
+            decr_by = 0
+            if self.space.actionflag.has_bytecode_counter:   # constant-folded
+                if self.space.threadlocals.gil_ready:   # quasi-immutable field
+                    decr_by = _get_adapted_tick_counter()
             #
             self.last_instr = intmask(jumpto)
-            ec.bytecode_trace(self, intmask(decr_by))
+            ec.bytecode_trace(self, decr_by)
             jumpto = r_uint(self.last_instr)
         #
         pypyjitdriver.can_enter_jit(frame=self, ec=ec, next_instr=jumpto,
@@ -158,6 +153,20 @@
                                     is_being_profiled=self.is_being_profiled)
         return jumpto
 
+def _get_adapted_tick_counter():
+    # Normally, the tick counter is decremented by 100 for every
+    # Python opcode.  Here, to better support JIT compilation of
+    # small loops, we decrement it by a possibly smaller constant.
+    # We get the maximum 100 when the (unoptimized) trace length
+    # is at least 3200 (a bit randomly).
+    trace_length = r_uint(current_trace_length())
+    decr_by = trace_length // 32
+    if decr_by < 1:
+        decr_by = 1
+    elif decr_by > 100:    # also if current_trace_length() returned -1
+        decr_by = 100
+    return intmask(decr_by)
+
 
 PyCode__initialize = PyCode._initialize
 
diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py
--- a/pypy/module/signal/interp_signal.py
+++ b/pypy/module/signal/interp_signal.py
@@ -109,8 +109,11 @@
         p = pypysig_getaddr_occurred()
         value = p.c_value
         if self.has_bytecode_counter:    # this 'if' is constant-folded
-            value -= by
-            p.c_value = value
+            if jit.isconstant(by) and by == 0:
+                pass     # normally constant-folded too
+            else:
+                value -= by
+                p.c_value = value
         return value
 
 
diff --git a/pypy/module/thread/gil.py b/pypy/module/thread/gil.py
--- a/pypy/module/thread/gil.py
+++ b/pypy/module/thread/gil.py
@@ -17,6 +17,7 @@
 class GILThreadLocals(OSThreadLocals):
     """A version of OSThreadLocals that enforces a GIL."""
     gil_ready = False
+    _immutable_fields_ = ['gil_ready?']
 
     def initialize(self, space):
         # add the GIL-releasing callback as an action on the space
diff --git a/pypy/module/thread/threadlocals.py b/pypy/module/thread/threadlocals.py
--- a/pypy/module/thread/threadlocals.py
+++ b/pypy/module/thread/threadlocals.py
@@ -8,12 +8,13 @@
 
     def __init__(self):
         self._valuedict = {}   # {thread_ident: ExecutionContext()}
+        self._freeze_()
+
+    def _freeze_(self):
+        self._valuedict.clear()
         self._mainthreadident = 0
         self._mostrecentkey = 0        # fast minicaching for the common case
         self._mostrecentvalue = None   # fast minicaching for the common case
-
-    def _freeze_(self):
-        self.__init__()
         return False
 
     def getvalue(self):

From noreply at buildbot.pypy.org  Sat Oct 15 14:43:53 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 15 Oct 2011 14:43:53 +0200 (CEST)
Subject: [pypy-commit] pypy jit-tagged-2: Replace int_lshift_ovf(x,
 1) with int_add_ovf(x, x).
Message-ID: <20111015124353.275BF82A8B@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: jit-tagged-2
Changeset: r48069:b05d1b6f5dbd
Date: 2011-10-14 18:38 +0200
http://bitbucket.org/pypy/pypy/changeset/b05d1b6f5dbd/

Log:	Replace int_lshift_ovf(x, 1) with int_add_ovf(x, x). It's roughly
	equivalent for the C backend, but maybe a bit better for the JIT.

diff --git a/pypy/rlib/rerased.py b/pypy/rlib/rerased.py
--- a/pypy/rlib/rerased.py
+++ b/pypy/rlib/rerased.py
@@ -218,7 +218,7 @@
         [v_value] = hop.inputargs(lltype.Signed)
         c_one = hop.inputconst(lltype.Signed, 1)
         hop.exception_is_here()
-        v2 = hop.genop('int_lshift_ovf', [v_value, c_one],
+        v2 = hop.genop('int_add_ovf', [v_value, v_value],
                        resulttype = lltype.Signed)
         v2p1 = hop.genop('int_add', [v2, c_one],
                          resulttype = lltype.Signed)
@@ -264,10 +264,10 @@
         return hop.genop('int_rshift', [v2, c_one], resulttype=lltype.Signed)
 
     def rtype_erase_int(self, hop):
-        hop.exception_is_here()
         [v_value] = hop.inputargs(lltype.Signed)
         c_one = hop.inputconst(lltype.Signed, 1)
-        v2 = hop.genop('int_lshift_ovf', [v_value, c_one],
+        hop.exception_is_here()
+        v2 = hop.genop('int_add_ovf', [v_value, v_value],
                        resulttype = lltype.Signed)
         v2p1 = hop.genop('int_add', [v2, c_one],
                          resulttype = lltype.Signed)
diff --git a/pypy/rpython/lltypesystem/rtagged.py b/pypy/rpython/lltypesystem/rtagged.py
--- a/pypy/rpython/lltypesystem/rtagged.py
+++ b/pypy/rpython/lltypesystem/rtagged.py
@@ -43,7 +43,7 @@
         v_value = hop.inputarg(lltype.Signed, arg=1)
         c_one = hop.inputconst(lltype.Signed, 1)
         hop.exception_is_here()
-        v2 = hop.genop('int_lshift_ovf', [v_value, c_one],
+        v2 = hop.genop('int_add_ovf', [v_value, v_value],
                        resulttype = lltype.Signed)
         v2p1 = hop.genop('int_add', [v2, c_one],
                          resulttype = lltype.Signed)

From noreply at buildbot.pypy.org  Sat Oct 15 14:56:45 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 15 Oct 2011 14:56:45 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix the test. This is mostly covering
	for an issue with
Message-ID: <20111015125645.4827A82A8B@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48073:659180bd3030
Date: 2011-10-15 14:56 +0200
http://bitbucket.org/pypy/pypy/changeset/659180bd3030/

Log:	Fix the test. This is mostly covering for an issue with
	jit.isconstant(), but it is known.

diff --git a/pypy/rlib/test/test_jit.py b/pypy/rlib/test/test_jit.py
--- a/pypy/rlib/test/test_jit.py
+++ b/pypy/rlib/test/test_jit.py
@@ -139,12 +139,11 @@
 
     def test_isconstant(self):
         def f(n):
-            assert n >= 0
             assert isconstant(n) is False
             l = []
             l.append(n)
             return len(l)
-        res = self.interpret(f, [234])
+        res = self.interpret(f, [-234])
         assert res == 1
 
 

From noreply at buildbot.pypy.org  Sat Oct 15 15:50:39 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 15 Oct 2011 15:50:39 +0200 (CEST)
Subject: [pypy-commit] pypy default: Baaaah.  Thanks zseil.
Message-ID: <20111015135039.B1A1982A8B@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48074:759db9b281eb
Date: 2011-10-15 15:50 +0200
http://bitbucket.org/pypy/pypy/changeset/759db9b281eb/

Log:	Baaaah. Thanks zseil.

diff --git a/pypy/translator/c/src/thread_nt.h b/pypy/translator/c/src/thread_nt.h
--- a/pypy/translator/c/src/thread_nt.h
+++ b/pypy/translator/c/src/thread_nt.h
@@ -245,7 +245,7 @@
     if (pending_acquires <= 0)
         return 0;
     InterlockedIncrement(&pending_acquires);
-    PulseEvent(&cond_gil);
+    PulseEvent(cond_gil);
 
     /* hack: the three following lines do a pthread_cond_wait(), and
        normally specifying a timeout of INFINITE would be fine.  But the
@@ -253,7 +253,7 @@
        (small) risk that PulseEvent misses the WaitForSingleObject().
        In this case the process will just sleep a few milliseconds. */
     LeaveCriticalSection(&mutex_gil);
-    WaitForSingleObject(&cond_gil, 15);
+    WaitForSingleObject(cond_gil, 15);
     EnterCriticalSection(&mutex_gil);
 
     InterlockedDecrement(&pending_acquires);
@@ -263,7 +263,7 @@
 void RPyGilRelease(void)
 {
     LeaveCriticalSection(&mutex_gil);
-    PulseEvent(&cond_gil);
+    PulseEvent(cond_gil);
 }
 
 void RPyGilAcquire(void)

From noreply at buildbot.pypy.org  Sun Oct 16 10:00:00 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 16 Oct 2011 10:00:00 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix the test_pypy_c tests for
	51224696c078.
Message-ID: <20111016080000.47D86820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48075:979996a75c9a
Date: 2011-10-16 09:59 +0200
http://bitbucket.org/pypy/pypy/changeset/979996a75c9a/

Log:	Fix the test_pypy_c tests for 51224696c078.

diff --git a/pypy/module/pypyjit/test_pypy_c/model.py b/pypy/module/pypyjit/test_pypy_c/model.py
--- a/pypy/module/pypyjit/test_pypy_c/model.py
+++ b/pypy/module/pypyjit/test_pypy_c/model.py
@@ -225,6 +225,8 @@
         # strip comment
         if '#' in line:
             line = line[:line.index('#')]
+        if line.strip() == 'guard_not_invalidated?':
+            return 'guard_not_invalidated', None, [], '...', False
         # find the resvar, if any
         if ' = ' in line:
             resvar, _, line = line.partition(' = ')
@@ -249,7 +251,7 @@
             descr = descr[len('descr='):]
         else:
             descr = None
-        return opname, resvar, args, descr
+        return opname, resvar, args, descr, True
 
     @classmethod
     def preprocess_expected_src(cls, src):
@@ -258,13 +260,23 @@
         # replaced with the corresponding operations, so that tests don't have
         # to repeat it every time
         ticker_check = """
+            guard_not_invalidated?
+            ticker0 = getfield_raw(ticker_address, descr=<SignedFieldDescr pypysig_long_struct.c_value .*>)
+            ticker_cond0 = int_lt(ticker0, 0)
+            guard_false(ticker_cond0, descr=...)
+        """
+        src = src.replace('--TICK--', ticker_check)
+        #
+        # this is the ticker check generated if we have threads
+        thread_ticker_check = """
+            guard_not_invalidated?
             ticker0 = getfield_raw(ticker_address, descr=<SignedFieldDescr pypysig_long_struct.c_value .*>)
             ticker1 = int_sub(ticker0, 1)
             setfield_raw(ticker_address, ticker1, descr=<SignedFieldDescr pypysig_long_struct.c_value .*>)
             ticker_cond0 = int_lt(ticker1, 0)
             guard_false(ticker_cond0, descr=...)
         """
-        src = src.replace('--TICK--', ticker_check)
+        src = src.replace('--THREAD-TICK--', thread_ticker_check)
         #
         # this is the ticker check generated in PyFrame.handle_operation_error
         exc_ticker_check = """
@@ -298,7 +310,7 @@
         if not cond:
             raise InvalidMatch(message, frame=sys._getframe(1))
 
-    def match_op(self, op, (exp_opname, exp_res, exp_args, exp_descr)):
+    def match_op(self, op, (exp_opname, exp_res, exp_args, exp_descr, _)):
         self._assert(op.name == exp_opname, "operation mismatch")
         self.match_var(op.res, exp_res)
         if exp_args != ['...']:
@@ -341,7 +353,7 @@
         what is after the '...'
         """
         iter_exp_ops = iter(expected_ops)
-        iter_ops = iter(self.ops)
+        iter_ops = RevertableIterator(self.ops)
         for opindex, exp_op in enumerate(iter_exp_ops):
             try:
                 if exp_op == '...':
@@ -360,6 +372,9 @@
                             break
                 self.match_op(op, exp_op)
             except InvalidMatch, e:
+                if exp_op[4] is False:    # optional operation
+                    iter_ops.revert_one()
+                    continue       # try to match with the next exp_op
                 e.opindex = opindex
                 raise
         #
@@ -398,3 +413,18 @@
         else:
             return True
 
+
+class RevertableIterator(object):
+    def __init__(self, sequence):
+        self.sequence = sequence
+        self.index = 0
+    def __iter__(self):
+        return self
+    def next(self):
+        index = self.index
+        if index == len(self.sequence):
+            raise StopIteration
+        self.index = index + 1
+        return self.sequence[index]
+    def revert_one(self):
+        self.index -= 1
diff --git a/pypy/module/pypyjit/test_pypy_c/test_00_model.py b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
--- a/pypy/module/pypyjit/test_pypy_c/test_00_model.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
@@ -145,15 +145,17 @@
 
     def test_parse_op(self):
         res = OpMatcher.parse_op("  a =   int_add(  b,  3 ) # foo")
-        assert res == ("int_add", "a", ["b", "3"], None)
+        assert res == ("int_add", "a", ["b", "3"], None, True)
         res = OpMatcher.parse_op("guard_true(a)")
-        assert res == ("guard_true", None, ["a"], None)
+        assert res == ("guard_true", None, ["a"], None, True)
         res = OpMatcher.parse_op("setfield_gc(p0, i0, descr=<foobar>)")
-        assert res == ("setfield_gc", None, ["p0", "i0"], "<foobar>")
+        assert res == ("setfield_gc", None, ["p0", "i0"], "<foobar>", True)
         res = OpMatcher.parse_op("i1 = getfield_gc(p0, descr=<foobar>)")
-        assert res == ("getfield_gc", "i1", ["p0"], "<foobar>")
+        assert res == ("getfield_gc", "i1", ["p0"], "<foobar>", True)
         res = OpMatcher.parse_op("p0 = force_token()")
-        assert res == ("force_token", "p0", [], None)
+        assert res == ("force_token", "p0", [], None, True)
+        res = OpMatcher.parse_op("guard_not_invalidated?")
+        assert res == ("guard_not_invalidated", None, [], '...', False)
 
     def test_exact_match(self):
         loop = """
@@ -341,7 +343,7 @@
             # this is the actual loop
             'int_lt', 'guard_true', 'int_add',
             # this is the signal checking stuff
-            'getfield_raw', 'int_sub', 'setfield_raw', 'int_lt', 'guard_false',
+            'guard_not_invalidated', 'getfield_raw', 'int_lt', 'guard_false',
             'jump'
             ]
 
@@ -407,7 +409,7 @@
             # this is the actual loop
             'int_lt', 'guard_true', 'force_token', 'int_add',
             # this is the signal checking stuff
-            'getfield_raw', 'int_sub', 'setfield_raw', 'int_lt', 'guard_false',
+            'guard_not_invalidated', 'getfield_raw', 'int_lt', 'guard_false',
             'jump'
             ]
 
@@ -425,10 +427,9 @@
             guard_true(i6, descr=...)
             i8 = int_add(i4, 1)
             # signal checking stuff
+            guard_not_invalidated(descr=...)
             i10 = getfield_raw(37212896, descr=<.* pypysig_long_struct.c_value .*>)
-            i12 = int_sub(i10, 1)
-            setfield_raw(37212896, i12, descr=<.* pypysig_long_struct.c_value .*>)
-            i14 = int_lt(i12, 0)
+            i14 = int_lt(i10, 0)
             guard_false(i14, descr=...)
             jump(p0, p1, p2, p3, i8, descr=...)
         """)

From noreply at buildbot.pypy.org  Sun Oct 16 10:04:38 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 16 Oct 2011 10:04:38 +0200 (CEST)
Subject: [pypy-commit] pypy default: Add a test for the thread-aware ticker.
Message-ID: <20111016080438.F0069820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48076:da46d9c97e58
Date: 2011-10-16 10:04 +0200
http://bitbucket.org/pypy/pypy/changeset/da46d9c97e58/

Log:	Add a test for the thread-aware ticker.

diff --git a/pypy/module/pypyjit/test_pypy_c/test_thread.py b/pypy/module/pypyjit/test_pypy_c/test_thread.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/pypyjit/test_pypy_c/test_thread.py
@@ -0,0 +1,28 @@
+from pypy.module.pypyjit.test_pypy_c.test_00_model import BaseTestPyPyC
+
+
+class TestThread(BaseTestPyPyC):
+    def test_simple(self):
+        def main(n):
+            import thread
+            def f():
+                i = 0
+                while i < n:
+                    i += 1
+                done.release()
+
+            done = thread.allocate_lock()
+            done.acquire()
+            thread.start_new_thread(f, ())
+            done.acquire()
+            return 0
+        log = self.run(main, [500])
+        assert round(log.result, 6) == round(main(500), 6)
+        loop, = log.loops_by_filename(self.filepath)
+        assert loop.match("""
+            i2 = int_lt(i0, i1)
+            guard_true(i2, descr=...)
+            i3 = int_add(i0, 1)
+            --THREAD-TICK--
+            jump(..., descr=<Loop0>)
+        """)

From noreply at buildbot.pypy.org  Sun Oct 16 11:49:19 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 16 Oct 2011 11:49:19 +0200 (CEST)
Subject: [pypy-commit] pypy continulet-jit: In-progress: add some notion of
	stacklet id.
Message-ID: <20111016094919.A095A820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: continulet-jit
Changeset: r48077:b0e1da0656a9
Date: 2011-10-16 11:49 +0200
http://bitbucket.org/pypy/pypy/changeset/b0e1da0656a9/

Log:	In-progress: add some notion of stacklet id. Not composable at all,
	but seems to be needed for the JIT's virtualizables and
	virtualrefs...

diff --git a/pypy/translator/c/src/stacklet/stacklet.c b/pypy/translator/c/src/stacklet/stacklet.c
--- a/pypy/translator/c/src/stacklet/stacklet.c
+++ b/pypy/translator/c/src/stacklet/stacklet.c
@@ -34,6 +34,8 @@
 /************************************************************/
 
 struct stacklet_s {
+    stacklet_id id;    /* first field */
+
     /* The portion of the real stack claimed by this paused tealet. */
     char *stack_start;                /* the "near" end of the stack */
     char *stack_stop;                 /* the "far" end of the stack */
@@ -60,6 +62,7 @@
                               stacklet_run_fn, void *) = NULL;
 
 struct stacklet_thread_s {
+    stacklet_id g_current_id;             /* first field */
     struct stacklet_s *g_stack_chain_head;  /* NULL <=> running main */
     char *g_current_stack_stop;
     char *g_current_stack_marker;
@@ -67,6 +70,10 @@
     struct stacklet_s *g_target;
 };
 
+struct stacklet_id_s {
+    stacklet_handle stacklet;
+};
+
 /***************************************************************/
 
 static void g_save(struct stacklet_s* g, char* stop
@@ -128,6 +135,9 @@
         return -1;
 
     stacklet = thrd->g_source;
+    stacklet->id = thrd->g_current_id;
+    if (stacklet->id != NULL)
+        stacklet->id->stacklet = stacklet;
     stacklet->stack_start = old_stack_pointer;
     stacklet->stack_stop  = thrd->g_current_stack_stop;
     stacklet->stack_saved = 0;
@@ -227,6 +237,9 @@
     memcpy(g->stack_start - stack_saved, g+1, stack_saved);
 #endif
     thrd->g_current_stack_stop = g->stack_stop;
+    thrd->g_current_id = g->id;
+    if (thrd->g_current_id != NULL)
+        thrd->g_current_id->stacklet = NULL;
     free(g);
     return EMPTY_STACKLET_HANDLE;
 }
@@ -235,18 +248,33 @@
                           stacklet_run_fn run, void *run_arg)
 {
     struct stacklet_s *result;
+    stacklet_id sid1 = thrd->g_current_id;
+    stacklet_id sid = malloc(sizeof(struct stacklet_id_s));
+    if (sid == NULL) {
+        thrd->g_source = NULL;
+        return;
+    }
 
     /* The following call returns twice! */
     result = (struct stacklet_s *) _stacklet_switchstack(g_initial_save_state,
                                                          g_restore_state,
                                                          thrd);
-    if (result == NULL && thrd->g_source != NULL) {
-        /* First time it returns.  Only g_initial_save_state() has run
-           and has created 'g_source'.  Call run(). */
+    if (result == NULL) {
+        /* First time it returns. */
+        if (thrd->g_source == NULL) {     /* out of memory */
+            free(sid);
+            return;
+        }
+        /* Only g_initial_save_state() has run and has created 'g_source'.
+           Call run(). */
+        sid->stacklet = NULL;
+        thrd->g_current_id = sid;
+        fprintf(stderr, "new sid: %p\n", sid);
         thrd->g_current_stack_stop = thrd->g_current_stack_marker;
         result = run(thrd->g_source, run_arg);
 
         /* Then switch to 'result'. */
+        free(sid);
         thrd->g_target = result;
         _stacklet_switchstack(g_destroy_state, g_restore_state, thrd);
 
@@ -254,6 +282,8 @@
         abort();
     }
     /* The second time it returns. */
+    assert(thrd->g_current_id == sid1);
+    fprintf(stderr, "continue with sid: %p\n", sid1);
 }
 
 /************************************************************/
diff --git a/pypy/translator/c/src/stacklet/stacklet.h b/pypy/translator/c/src/stacklet/stacklet.h
--- a/pypy/translator/c/src/stacklet/stacklet.h
+++ b/pypy/translator/c/src/stacklet/stacklet.h
@@ -59,4 +59,14 @@
  */
 char **_stacklet_translate_pointer(stacklet_handle context, char **ptr);
 
+/* The "stacklet id" is NULL for main stacklets; for other stacklets it is a
+ * value that remain valid and unchanged if the stacklet is suspended and
+ * resumed.  WARNING: DON'T USE unless you have no other choice, because
+ * it is not "composable" at all.
+ */
+typedef struct stacklet_id_s *stacklet_id;
+#define _stacklet_id_of_stacklet(stacklet) (*(stacklet_id*)(stacklet))
+#define _stacklet_id_current(thrd) (*(stacklet_id*)(thrd))
+stacklet_handle _stacklet_with_id(stacklet_thread_handle thrd, stacklet_id id);
+
 #endif /* _STACKLET_H_ */
diff --git a/pypy/translator/c/src/stacklet/tests.c b/pypy/translator/c/src/stacklet/tests.c
--- a/pypy/translator/c/src/stacklet/tests.c
+++ b/pypy/translator/c/src/stacklet/tests.c
@@ -627,6 +627,66 @@
 #endif
 /************************************************************/
 
+struct test_id_s {
+    stacklet_id id1;
+    stacklet_id id2;
+} tid;
+
+stacklet_handle stacklet_id_callback_1(stacklet_handle h, void *arg)
+{
+    stacklet_id myid = _stacklet_id_current(thrd);
+    h = stacklet_switch(thrd, h);
+    
+    tid.id1 = _stacklet_id_current(thrd);
+    assert(tid.id1 != NULL);
+    assert(tid.id1 == myid);
+    assert(status == 0);
+    status = 1;
+
+    return stacklet_switch(thrd, h);
+}
+
+stacklet_handle stacklet_id_callback_2(stacklet_handle h, void *arg)
+{
+    stacklet_id myid = _stacklet_id_current(thrd);
+    h = stacklet_switch(thrd, h);
+
+    tid.id2 = _stacklet_id_current(thrd);
+    assert(tid.id2 != NULL);
+    assert(tid.id2 == myid);
+    assert(status == 1);
+    status = 2;
+
+    return stacklet_switch(thrd, h);
+}
+
+void test_stacklet_id(void)
+{
+    status = 0;
+    stacklet_handle h1 = stacklet_new(thrd, stacklet_id_callback_1, NULL);
+    stacklet_handle h2 = stacklet_new(thrd, stacklet_id_callback_2, NULL);
+
+    assert(_stacklet_id_current(thrd) == NULL);
+
+    assert(status == 0);
+    h1 = stacklet_switch(thrd, h1);
+
+    assert(status == 1);
+    h2 = stacklet_switch(thrd, h2);
+
+    assert(status == 2);
+    assert(_stacklet_id_of_stacklet(h1) == tid.id1);
+    assert(_stacklet_id_of_stacklet(h2) == tid.id2);
+    assert(_stacklet_id_current(thrd) == NULL);
+
+    h1 = stacklet_switch(thrd, h1);
+    assert(h1 == EMPTY_STACKLET_HANDLE);
+    h2 = stacklet_switch(thrd, h2);
+    assert(h2 == EMPTY_STACKLET_HANDLE);
+}
+
+/************************************************************/
+
 #define TEST(name)   { name, #name }
 
 typedef struct {
@@ -649,6 +709,7 @@
   TEST(test_double),
   TEST(test_random),
 #endif
+  TEST(test_stacklet_id),
   { NULL, NULL }
 };
 

From noreply at buildbot.pypy.org  Sun Oct 16 15:10:01 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 16 Oct 2011 15:10:01 +0200 (CEST)
Subject: [pypy-commit] pypy continulet-jit: Finish the part in the C code.
Message-ID: <20111016131001.873B7820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: continulet-jit
Changeset: r48078:ff490d93d8b4
Date: 2011-10-16 13:31 +0200
http://bitbucket.org/pypy/pypy/changeset/ff490d93d8b4/

Log:	Finish the part in the C code.

diff --git a/pypy/translator/c/src/stacklet/stacklet.c b/pypy/translator/c/src/stacklet/stacklet.c
--- a/pypy/translator/c/src/stacklet/stacklet.c
+++ b/pypy/translator/c/src/stacklet/stacklet.c
@@ -61,6 +61,10 @@
 void (*_stacklet_initialstub)(struct stacklet_thread_s *,
                               stacklet_run_fn, void *) = NULL;
 
+struct stacklet_id_s {
+    stacklet_handle stacklet;
+};
+
 struct stacklet_thread_s {
     stacklet_id g_current_id;             /* first field */
     struct stacklet_s *g_stack_chain_head;  /* NULL <=> running main */
@@ -68,10 +72,7 @@
     char *g_current_stack_marker;
     struct stacklet_s *g_source;
     struct stacklet_s *g_target;
-};
-
-struct stacklet_id_s {
-    stacklet_handle stacklet;
+    struct stacklet_id_s g_main_id;
 };
 
 /***************************************************************/
@@ -136,8 +137,7 @@
 
     stacklet = thrd->g_source;
     stacklet->id = thrd->g_current_id;
-    if (stacklet->id != NULL)
-        stacklet->id->stacklet = stacklet;
+    stacklet->id->stacklet = stacklet;
     stacklet->stack_start = old_stack_pointer;
     stacklet->stack_stop  = thrd->g_current_stack_stop;
     stacklet->stack_saved = 0;
@@ -238,8 +238,7 @@
 #endif
     thrd->g_current_stack_stop = g->stack_stop;
     thrd->g_current_id = g->id;
-    if (thrd->g_current_id != NULL)
-        thrd->g_current_id->stacklet = NULL;
+    thrd->g_current_id->stacklet = NULL;
     free(g);
     return EMPTY_STACKLET_HANDLE;
 }
@@ -300,8 +299,10 @@
     }
 
     thrd = malloc(sizeof(struct stacklet_thread_s));
-    if (thrd != NULL)
+    if (thrd != NULL) {
         memset(thrd, 0, sizeof(struct stacklet_thread_s));
+        thrd->g_current_id = &thrd->g_main_id;
+    }
     return thrd;
 }
 
@@ -369,3 +370,10 @@
   }
   return ptr;
 }
+
+stacklet_handle _stacklet_with_id(stacklet_thread_handle thrd, stacklet_id id)
+{
+    if (id == NULL)
+        id = &thrd->g_main_id;
+    return id->stacklet;
+}
diff --git a/pypy/translator/c/src/stacklet/stacklet.h b/pypy/translator/c/src/stacklet/stacklet.h
--- a/pypy/translator/c/src/stacklet/stacklet.h
+++ b/pypy/translator/c/src/stacklet/stacklet.h
@@ -59,14 +59,15 @@
  */
 char **_stacklet_translate_pointer(stacklet_handle context, char **ptr);
 
-/* The "stacklet id" is NULL for main stacklets; for other stacklets it is a
- * value that remain valid and unchanged if the stacklet is suspended and
- * resumed.  WARNING: DON'T USE unless you have no other choice, because
- * it is not "composable" at all.
+/* The "stacklet id" is a value that remain valid and unchanged if the
+ * stacklet is suspended and resumed.  WARNING: DON'T USE unless you have
+ * no other choice, because it is not "composable" at all.
  */
 typedef struct stacklet_id_s *stacklet_id;
 #define _stacklet_id_of_stacklet(stacklet) (*(stacklet_id*)(stacklet))
 #define _stacklet_id_current(thrd) (*(stacklet_id*)(thrd))
+/* Returns the current stacklet with the given id.
+   If 'id' == NULL, returns the main stacklet in the thread. */
 stacklet_handle _stacklet_with_id(stacklet_thread_handle thrd, stacklet_id id);
 
 #endif /* _STACKLET_H_ */
diff --git a/pypy/translator/c/src/stacklet/tests.c b/pypy/translator/c/src/stacklet/tests.c
--- a/pypy/translator/c/src/stacklet/tests.c
+++ b/pypy/translator/c/src/stacklet/tests.c
@@ -628,17 +628,27 @@
 /************************************************************/
 
 struct test_id_s {
+    stacklet_id idmain;
     stacklet_id id1;
     stacklet_id id2;
+    stacklet_handle hmain;
+    stacklet_handle h1;
+    stacklet_handle h2;
 } tid;
 
 stacklet_handle stacklet_id_callback_1(stacklet_handle h, void *arg)
 {
     stacklet_id myid = _stacklet_id_current(thrd);
+    assert(_stacklet_with_id(thrd, myid) == NULL);
+    tid.hmain = h;
+    tid.h1 = NULL;
+
     h = stacklet_switch(thrd, h);
-    
+    tid.hmain = h;
+    tid.h1 = NULL;
+
     tid.id1 = _stacklet_id_current(thrd);
-    assert(tid.id1 != NULL);
+    assert(tid.id1 != tid.idmain);
     assert(tid.id1 == myid);
     assert(status == 0);
     status = 1;
@@ -649,11 +659,22 @@
 stacklet_handle stacklet_id_callback_2(stacklet_handle h, void *arg)
 {
     stacklet_id myid = _stacklet_id_current(thrd);
+    assert(_stacklet_with_id(thrd, myid) == NULL);
+    tid.hmain = h;
+    tid.h2 = NULL;
+
     h = stacklet_switch(thrd, h);
+    tid.hmain = h;
+    tid.h2 = NULL;
 
     tid.id2 = _stacklet_id_current(thrd);
-    assert(tid.id2 != NULL);
+    assert(tid.id2 != tid.idmain);
+    assert(tid.id2 != tid.id1);
     assert(tid.id2 == myid);
+    assert(_stacklet_with_id(thrd, tid.idmain) == tid.hmain);
+    assert(_stacklet_with_id(thrd, tid.id1) == tid.h1);
+    assert(_stacklet_with_id(thrd, tid.id2) == tid.h2);
+
     assert(status == 1);
     status = 2;
 
@@ -665,19 +686,30 @@
     status = 0;
     stacklet_handle h1 = stacklet_new(thrd, stacklet_id_callback_1, NULL);
     stacklet_handle h2 = stacklet_new(thrd, stacklet_id_callback_2, NULL);
+    tid.hmain = NULL;
+    tid.h1 = h1;
+    tid.h2 = h2;
 
-    assert(_stacklet_id_current(thrd) == NULL);
+    tid.idmain = _stacklet_id_current(thrd);
+    assert(_stacklet_with_id(thrd, tid.idmain) == NULL);
 
     assert(status == 0);
     h1 = stacklet_switch(thrd, h1);
+    tid.hmain = NULL;
+    tid.h1 = h1;
 
     assert(status == 1);
     h2 = stacklet_switch(thrd, h2);
+    tid.hmain = NULL;
+    tid.h2 = h2;
 
     assert(status == 2);
     assert(_stacklet_id_of_stacklet(h1) == tid.id1);
     assert(_stacklet_id_of_stacklet(h2) == tid.id2);
-    assert(_stacklet_id_current(thrd) == NULL);
+    assert(_stacklet_id_current(thrd) == tid.idmain);
+    assert(_stacklet_with_id(thrd, tid.idmain) == NULL);
+    assert(_stacklet_with_id(thrd, tid.id1) == tid.h1);
+    assert(_stacklet_with_id(thrd, tid.id2) == tid.h2);
 
     h1 = stacklet_switch(thrd, h1);
     assert(h1 == EMPTY_STACKLET_HANDLE);

From noreply at buildbot.pypy.org  Sun Oct 16 15:10:02 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 16 Oct 2011 15:10:02 +0200 (CEST)
Subject: [pypy-commit] pypy continulet-jit: Kill debugging prints.
Message-ID: <20111016131002.B7D8A82A8B@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: continulet-jit
Changeset: r48079:24ab27e99c85
Date: 2011-10-16 13:32 +0200
http://bitbucket.org/pypy/pypy/changeset/24ab27e99c85/

Log:	Kill debugging prints.

diff --git a/pypy/translator/c/src/stacklet/stacklet.c b/pypy/translator/c/src/stacklet/stacklet.c
--- a/pypy/translator/c/src/stacklet/stacklet.c
+++ b/pypy/translator/c/src/stacklet/stacklet.c
@@ -268,7 +268,6 @@
            Call run(). */
         sid->stacklet = NULL;
         thrd->g_current_id = sid;
-        fprintf(stderr, "new sid: %p\n", sid);
         thrd->g_current_stack_stop = thrd->g_current_stack_marker;
         result = run(thrd->g_source, run_arg);
 
@@ -282,7 +281,6 @@
     }
     /* The second time it returns. */
     assert(thrd->g_current_id == sid1);
-    fprintf(stderr, "continue with sid: %p\n", sid1);
 }
 
 /************************************************************/

From noreply at buildbot.pypy.org  Sun Oct 16 15:10:03 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 16 Oct 2011 15:10:03 +0200 (CEST)
Subject: [pypy-commit] pypy continulet-jit: Forgot a free() in a corner case.
Message-ID: <20111016131003.E7BE4820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: continulet-jit
Changeset: r48080:5d59522520df
Date: 2011-10-16 13:42 +0200
http://bitbucket.org/pypy/pypy/changeset/5d59522520df/

Log:	Forgot a free() in a corner case.

diff --git a/pypy/translator/c/src/stacklet/stacklet.c b/pypy/translator/c/src/stacklet/stacklet.c
--- a/pypy/translator/c/src/stacklet/stacklet.c
+++ b/pypy/translator/c/src/stacklet/stacklet.c
@@ -343,6 +343,8 @@
             *pp = target->stack_prev;
             break;
         }
+    assert(target->id->stacklet == target);
+    free(target->id);
     free(target);
 }
 

From noreply at buildbot.pypy.org  Sun Oct 16 15:10:05 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 16 Oct 2011 15:10:05 +0200 (CEST)
Subject: [pypy-commit] pypy continulet-jit: Random progress,
 but I still didn't find the "correct" way...
Message-ID: <20111016131005.260B2820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: continulet-jit
Changeset: r48081:057d18c508bb
Date: 2011-10-16 14:54 +0200
http://bitbucket.org/pypy/pypy/changeset/057d18c508bb/

Log:	Random progress, but I still didn't find the "correct" way...

diff --git a/pypy/rlib/_rffi_stacklet.py b/pypy/rlib/_rffi_stacklet.py
--- a/pypy/rlib/_rffi_stacklet.py
+++ b/pypy/rlib/_rffi_stacklet.py
@@ -25,6 +25,7 @@
 thread_handle = rffi.COpaquePtr(typedef='stacklet_thread_handle',
                                 compilation_info=eci)
 run_fn = lltype.Ptr(lltype.FuncType([handle, llmemory.Address], handle))
+id = rffi.COpaquePtr(typedef='stacklet_id', compilation_info=eci)
 
 # ----- constants -----
 
@@ -47,3 +48,5 @@
 _translate_pointer = llexternal("_stacklet_translate_pointer",
                                 [llmemory.Address, llmemory.Address],
                                 llmemory.Address)
+_stacklet_with_id = llexternal("_stacklet_with_id", [thread_handle, id],
+                               handle)
diff --git a/pypy/rlib/rstacklet.py b/pypy/rlib/rstacklet.py
--- a/pypy/rlib/rstacklet.py
+++ b/pypy/rlib/rstacklet.py
@@ -10,6 +10,7 @@
 
     @jit.dont_look_inside
     def __init__(self, config):
+        stacklet_thread_global.enable()
         self._gcrootfinder = _getgcrootfinder(config, we_are_translated())
         self._thrd = _c.newthread()
         if not self._thrd:
@@ -66,6 +67,19 @@
             self._thrd = lltype.nullptr(_c.thread_handle.TO)
             _c.deletethread(thrd)
 
+class StackletThreadGlobal(object):
+    _immutable_fields_ = ['enabled?']
+
+    def _freeze_(self):
+        self.enabled = False
+
+    def enable(self):
+        if not self.enabled:
+            self.enabled = True    # change the quasi-immutable field
+
+stacklet_thread_global = StackletThreadGlobal()
+stacklet_thread_global._freeze_()
+
 # ____________________________________________________________
 
 def _getgcrootfinder(config, translated):
diff --git a/pypy/translator/c/src/stacklet/stacklet.c b/pypy/translator/c/src/stacklet/stacklet.c
--- a/pypy/translator/c/src/stacklet/stacklet.c
+++ b/pypy/translator/c/src/stacklet/stacklet.c
@@ -344,7 +344,8 @@
             break;
         }
     assert(target->id->stacklet == target);
-    free(target->id);
+    if (target->id != &thrd->g_main_id)
+        free(target->id);
     free(target);
 }
 
@@ -370,10 +371,3 @@
   }
   return ptr;
 }
-
-stacklet_handle _stacklet_with_id(stacklet_thread_handle thrd, stacklet_id id)
-{
-    if (id == NULL)
-        id = &thrd->g_main_id;
-    return id->stacklet;
-}
diff --git a/pypy/translator/c/src/stacklet/stacklet.h b/pypy/translator/c/src/stacklet/stacklet.h
--- a/pypy/translator/c/src/stacklet/stacklet.h
+++ b/pypy/translator/c/src/stacklet/stacklet.h
@@ -67,7 +67,9 @@
 #define _stacklet_id_of_stacklet(stacklet) (*(stacklet_id*)(stacklet))
 #define _stacklet_id_current(thrd) (*(stacklet_id*)(thrd))
 /* Returns the current stacklet with the given id.
-   If 'id' == NULL, returns the main stacklet in the thread. */
+   If 'id' == NULL, returns the main stacklet in the thread.
+   In both cases the return value is NULL if the id specifies the currently
+   running "stacklet". */
 stacklet_handle _stacklet_with_id(stacklet_thread_handle thrd, stacklet_id id);
 
 #endif /* _STACKLET_H_ */

From noreply at buildbot.pypy.org  Sun Oct 16 15:10:06 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 16 Oct 2011 15:10:06 +0200 (CEST)
Subject: [pypy-commit] pypy default: Add a bad workaround that should
	hopefully fix the JIT
Message-ID: <20111016131006.57AEB820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48082:f671d242b164
Date: 2011-10-16 15:09 +0200
http://bitbucket.org/pypy/pypy/changeset/f671d242b164/

Log:	Add a bad workaround that should hopefully fix the JIT in the
	presence of _continuations --- mostly by killing the performance of
	the JIT for any loop that involves _continuations.

diff --git a/pypy/module/_continuation/interp_continuation.py b/pypy/module/_continuation/interp_continuation.py
--- a/pypy/module/_continuation/interp_continuation.py
+++ b/pypy/module/_continuation/interp_continuation.py
@@ -28,6 +28,8 @@
     def descr_init(self, w_callable, __args__):
         if self.sthread is not None:
             raise geterror(self.space, "continulet already __init__ialized")
+        sthread = build_sthread(self.space)
+        workaround_disable_jit(sthread)
         #
         # hackish: build the frame "by hand", passing it the correct arguments
         space = self.space
@@ -41,7 +43,6 @@
         self.bottomframe = bottomframe
         #
         global_state.origin = self
-        sthread = build_sthread(self.space)
         self.sthread = sthread
         h = sthread.new(new_stacklet_callback)
         post_switch(sthread, h)
@@ -71,6 +72,7 @@
                 global_state.clear()
                 raise geterror(self.space, "continulet already finished")
         self.check_sthread()
+        workaround_disable_jit(self.sthread)
         #
         global_state.origin = self
         if to is None:
@@ -259,6 +261,16 @@
         sthread = ec.stacklet_thread = SThread(space, ec)
     return sthread
 
+def workaround_disable_jit(sthread):
+    # A bad workaround to kill the JIT anywhere in this thread.
+    # This forces all the frames.  It's a bad workaround because
+    # it takes O(depth) time, and it will cause some "abort:
+    # vable escape" in the JIT.  The goal is to prevent any frame
+    # from being still virtuals, because the JIT generates code
+    # to un-virtualizable them "on demand" by loading values based
+    # on FORCE_TOKEN, which is an address in the stack.
+    sthread.ec.force_all_frames()
+
 # ____________________________________________________________
 
 def permute(space, args_w):

From noreply at buildbot.pypy.org  Sun Oct 16 17:31:18 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 16 Oct 2011 17:31:18 +0200 (CEST)
Subject: [pypy-commit] pypy jit-tagged-2: Close branch, to be merged.
Message-ID: <20111016153118.2CD50820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: jit-tagged-2
Changeset: r48083:bdc101d37e6d
Date: 2011-10-16 16:08 +0200
http://bitbucket.org/pypy/pypy/changeset/bdc101d37e6d/

Log:	Close branch, to be merged.


From noreply at buildbot.pypy.org  Sun Oct 16 17:31:19 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 16 Oct 2011 17:31:19 +0200 (CEST)
Subject: [pypy-commit] pypy default: Merge 'jit-tagged-2': add JIT support
 for the "rerased" module
Message-ID: <20111016153119.E3B7E820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48084:6281b527f712
Date: 2011-10-16 16:09 +0200
http://bitbucket.org/pypy/pypy/changeset/6281b527f712/

Log:	Merge 'jit-tagged-2': add JIT support for the "rerased" module to
	erase integers as odd-valued pseudo-pointers.

diff --git a/pypy/jit/backend/llgraph/llimpl.py b/pypy/jit/backend/llgraph/llimpl.py
--- a/pypy/jit/backend/llgraph/llimpl.py
+++ b/pypy/jit/backend/llgraph/llimpl.py
@@ -165,6 +165,7 @@
     'unicodegetitem'  : (('ref', 'int'), 'int'),
     'unicodesetitem'  : (('ref', 'int', 'int'), 'int'),
     'cast_ptr_to_int' : (('ref',), 'int'),
+    'cast_int_to_ptr' : (('int',), 'ref'),
     'debug_merge_point': (('ref', 'int'), None),
     'force_token'     : ((), 'int'),
     'call_may_force'  : (('int', 'varargs'), 'intorptr'),
@@ -875,9 +876,6 @@
     def op_new_array(self, arraydescr, count):
         return do_new_array(arraydescr.ofs, count)
 
-    def op_cast_ptr_to_int(self, descr, ptr):
-        return cast_to_int(ptr)
-
     def op_force_token(self, descr):
         opaque_frame = _to_opaque(self)
         return llmemory.cast_ptr_to_adr(opaque_frame)
diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -1468,20 +1468,16 @@
         return u''.join(u.chars)
 
 
-    def test_casts(self):
-        py.test.skip("xxx fix or kill")
-        from pypy.rpython.lltypesystem import lltype, llmemory
-        TP = lltype.GcStruct('x')
-        x = lltype.malloc(TP)        
-        x = lltype.cast_opaque_ptr(llmemory.GCREF, x)
+    def test_cast_int_to_ptr(self):
+        res = self.execute_operation(rop.CAST_INT_TO_PTR,
+                                     [BoxInt(-17)],  'ref').value
+        assert lltype.cast_ptr_to_int(res) == -17
+
+    def test_cast_ptr_to_int(self):
+        x = lltype.cast_int_to_ptr(llmemory.GCREF, -19)
         res = self.execute_operation(rop.CAST_PTR_TO_INT,
-                                     [BoxPtr(x)],  'int').value
-        expected = self.cpu.cast_adr_to_int(llmemory.cast_ptr_to_adr(x))
-        assert rffi.get_real_int(res) == rffi.get_real_int(expected)
-        res = self.execute_operation(rop.CAST_PTR_TO_INT,
-                                     [ConstPtr(x)],  'int').value
-        expected = self.cpu.cast_adr_to_int(llmemory.cast_ptr_to_adr(x))
-        assert rffi.get_real_int(res) == rffi.get_real_int(expected)
+                                     [BoxPtr(x)], 'int').value
+        assert res == -19
 
     def test_ooops_non_gc(self):
         x = lltype.malloc(lltype.Struct('x'), flavor='raw')
@@ -2299,13 +2295,6 @@
         #
         cpu.bh_strsetitem(x, 4, ord('/'))
         assert str.chars[4] == '/'
-        #
-##        x = cpu.bh_newstr(5)
-##        y = cpu.bh_cast_ptr_to_int(x)
-##        z = cpu.bh_cast_ptr_to_int(x)
-##        y = rffi.get_real_int(y)
-##        z = rffi.get_real_int(z)
-##        assert type(y) == type(z) == int and y == z
 
     def test_sorting_of_fields(self):
         S = self.S
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1387,7 +1387,8 @@
 
     def genop_same_as(self, op, arglocs, resloc):
         self.mov(arglocs[0], resloc)
-    #genop_cast_ptr_to_int = genop_same_as
+    genop_cast_ptr_to_int = genop_same_as
+    genop_cast_int_to_ptr = genop_same_as
 
     def genop_int_mod(self, op, arglocs, resloc):
         if IS_X86_32:
diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py
--- a/pypy/jit/backend/x86/regalloc.py
+++ b/pypy/jit/backend/x86/regalloc.py
@@ -1152,7 +1152,8 @@
         self.possibly_free_var(op.getarg(0))
         resloc = self.force_allocate_reg(op.result)
         self.Perform(op, [argloc], resloc)
-    #consider_cast_ptr_to_int = consider_same_as
+    consider_cast_ptr_to_int = consider_same_as
+    consider_cast_int_to_ptr = consider_same_as
 
     def consider_strlen(self, op):
         args = op.getarglist()
diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -800,8 +800,7 @@
 
     def rewrite_op_cast_ptr_to_int(self, op):
         if self._is_gc(op.args[0]):
-            #return op
-            raise NotImplementedError("cast_ptr_to_int")
+            return op
 
     def rewrite_op_force_cast(self, op):
         v_arg = op.args[0]
diff --git a/pypy/jit/metainterp/blackhole.py b/pypy/jit/metainterp/blackhole.py
--- a/pypy/jit/metainterp/blackhole.py
+++ b/pypy/jit/metainterp/blackhole.py
@@ -2,7 +2,7 @@
 from pypy.rlib.rtimer import read_timestamp
 from pypy.rlib.rarithmetic import intmask, LONG_BIT, r_uint, ovfcheck
 from pypy.rlib.objectmodel import we_are_translated
-from pypy.rlib.debug import debug_start, debug_stop
+from pypy.rlib.debug import debug_start, debug_stop, ll_assert
 from pypy.rlib.debug import make_sure_not_resized
 from pypy.rpython.lltypesystem import lltype, llmemory, rclass
 from pypy.rpython.lltypesystem.lloperation import llop
@@ -503,6 +503,15 @@
     @arguments("r", returns="r")
     def bhimpl_cast_opaque_ptr(a):
         return a
+    @arguments("r", returns="i")
+    def bhimpl_cast_ptr_to_int(a):
+        i = lltype.cast_ptr_to_int(a)
+        ll_assert((i & 1) == 1, "bhimpl_cast_ptr_to_int: not an odd int")
+        return i
+    @arguments("i", returns="r")
+    def bhimpl_cast_int_to_ptr(i):
+        ll_assert((i & 1) == 1, "bhimpl_cast_int_to_ptr: not an odd int")
+        return lltype.cast_int_to_ptr(llmemory.GCREF, i)
 
     @arguments("i", returns="i")
     def bhimpl_int_copy(a):
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -222,6 +222,7 @@
                     'cast_float_to_int', 'cast_int_to_float',
                     'cast_float_to_singlefloat', 'cast_singlefloat_to_float',
                     'float_neg', 'float_abs',
+                    'cast_ptr_to_int', 'cast_int_to_ptr',
                     ]:
         exec py.code.Source('''
             @arguments("box")
diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py
--- a/pypy/jit/metainterp/resoperation.py
+++ b/pypy/jit/metainterp/resoperation.py
@@ -431,6 +431,8 @@
     'INT_IS_TRUE/1b',
     'INT_NEG/1',
     'INT_INVERT/1',
+    'CAST_PTR_TO_INT/1',
+    'CAST_INT_TO_PTR/1',
     #
     'SAME_AS/1',      # gets a Const or a Box, turns it into another Box
     #
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -3440,7 +3440,6 @@
 
 class TestLLtype(BaseLLtypeTests, LLJitMixin):
     def test_tagged(self):
-        py.test.skip("implement me")
         from pypy.rlib.objectmodel import UnboxedValue
         class Base(object):
             __slots__ = ()
@@ -3492,3 +3491,35 @@
                 pc += 1
             return pc
         res = self.meta_interp(main, [False, 100, True], taggedpointers=True)
+
+    def test_rerased(self):
+        from pypy.rlib.rerased import erase_int, unerase_int, new_erasing_pair
+        eraseX, uneraseX = new_erasing_pair("X")
+        #
+        class X:
+            def __init__(self, a, b):
+                self.a = a
+                self.b = b
+        #
+        def f(i, j):
+            # 'j' should be 0 or 1, not other values
+            if j > 0:
+                e = eraseX(X(i, j))
+            else:
+                try:
+                    e = erase_int(i)
+                except OverflowError:
+                    return -42
+            if j & 1:
+                x = uneraseX(e)
+                return x.a - x.b
+            else:
+                return unerase_int(e)
+        #
+        x = self.interp_operations(f, [-128, 0], taggedpointers=True)
+        assert x == -128
+        bigint = sys.maxint//2 + 1
+        x = self.interp_operations(f, [bigint, 0], taggedpointers=True)
+        assert x == -42
+        x = self.interp_operations(f, [1000, 1], taggedpointers=True)
+        assert x == 999
diff --git a/pypy/rlib/rerased.py b/pypy/rlib/rerased.py
--- a/pypy/rlib/rerased.py
+++ b/pypy/rlib/rerased.py
@@ -218,15 +218,13 @@
         [v_value] = hop.inputargs(lltype.Signed)
         c_one = hop.inputconst(lltype.Signed, 1)
         hop.exception_is_here()
-        v2 = hop.genop('int_lshift_ovf', [v_value, c_one],
+        v2 = hop.genop('int_add_ovf', [v_value, v_value],
                        resulttype = lltype.Signed)
         v2p1 = hop.genop('int_add', [v2, c_one],
                          resulttype = lltype.Signed)
         v_instance = hop.genop('cast_int_to_ptr', [v2p1],
                                resulttype=self.lowleveltype)
-        v = hop.genop('cast_opaque_ptr', [v_instance],
-                      resulttype=self.lowleveltype)
-        return v
+        return v_instance
 
     def convert_const(self, value):
         if value._identity is _identity_for_ints:
@@ -266,10 +264,10 @@
         return hop.genop('int_rshift', [v2, c_one], resulttype=lltype.Signed)
 
     def rtype_erase_int(self, hop):
-        hop.exception_is_here()
         [v_value] = hop.inputargs(lltype.Signed)
         c_one = hop.inputconst(lltype.Signed, 1)
-        v2 = hop.genop('int_lshift_ovf', [v_value, c_one],
+        hop.exception_is_here()
+        v2 = hop.genop('int_add_ovf', [v_value, v_value],
                        resulttype = lltype.Signed)
         v2p1 = hop.genop('int_add', [v2, c_one],
                          resulttype = lltype.Signed)
diff --git a/pypy/rpython/lltypesystem/ll2ctypes.py b/pypy/rpython/lltypesystem/ll2ctypes.py
--- a/pypy/rpython/lltypesystem/ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/ll2ctypes.py
@@ -658,6 +658,8 @@
         if T == llmemory.GCREF:
             if isinstance(llobj._obj, _llgcopaque):
                 return ctypes.c_void_p(llobj._obj.intval)
+            if isinstance(llobj._obj, int):    # tagged pointer
+                return ctypes.c_void_p(llobj._obj)
             container = llobj._obj.container
             T = lltype.Ptr(lltype.typeOf(container))
             # otherwise it came from integer and we want a c_void_p with
@@ -1268,6 +1270,7 @@
 class _llgcopaque(lltype._container):
     _TYPE = llmemory.GCREF.TO
     _name = "_llgcopaque"
+    _read_directly_intval = True     # for _ptr._cast_to_int()
 
     def __init__(self, void_p):
         if isinstance(void_p, (int, long)):
@@ -1299,11 +1302,6 @@
             return _opaque_objs[self.intval // 2]
         return force_cast(PTRTYPE, self.intval)
 
-##     def _cast_to_int(self):
-##         return self.intval
-
-##     def _cast_to_adr(self):
-##         return _lladdress(self.intval)
 
 def cast_adr_to_int(addr):
     if isinstance(addr, llmemory.fakeaddress):
diff --git a/pypy/rpython/lltypesystem/llmemory.py b/pypy/rpython/lltypesystem/llmemory.py
--- a/pypy/rpython/lltypesystem/llmemory.py
+++ b/pypy/rpython/lltypesystem/llmemory.py
@@ -498,6 +498,8 @@
 
     def _cast_to_int(self, symbolic=False):
         if self:
+            if isinstance(self.ptr._obj0, int):    # tagged integer
+                return self.ptr._obj0
             if symbolic:
                 return AddressAsInt(self)
             else:
diff --git a/pypy/rpython/lltypesystem/lltype.py b/pypy/rpython/lltypesystem/lltype.py
--- a/pypy/rpython/lltypesystem/lltype.py
+++ b/pypy/rpython/lltypesystem/lltype.py
@@ -1360,6 +1360,8 @@
         obj = normalizeptr(self, check)._getobj(check)
         if isinstance(obj, int):
             return obj     # special case for cast_int_to_ptr() results put into opaques
+        if getattr(obj, '_read_directly_intval', False):
+            return obj.intval   # special case for _llgcopaque
         result = intmask(obj._getid())
         # assume that id() returns an addressish value which is
         # not zero and aligned to at least a multiple of 4
diff --git a/pypy/rpython/lltypesystem/rtagged.py b/pypy/rpython/lltypesystem/rtagged.py
--- a/pypy/rpython/lltypesystem/rtagged.py
+++ b/pypy/rpython/lltypesystem/rtagged.py
@@ -43,7 +43,7 @@
         v_value = hop.inputarg(lltype.Signed, arg=1)
         c_one = hop.inputconst(lltype.Signed, 1)
         hop.exception_is_here()
-        v2 = hop.genop('int_lshift_ovf', [v_value, c_one],
+        v2 = hop.genop('int_add_ovf', [v_value, v_value],
                        resulttype = lltype.Signed)
         v2p1 = hop.genop('int_add', [v2, c_one],
                          resulttype = lltype.Signed)
diff --git a/pypy/rpython/lltypesystem/test/test_ll2ctypes.py b/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
--- a/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
@@ -1123,6 +1123,9 @@
 
         #assert lltype.cast_ptr_to_int(ref1) == intval
 
+        x = rffi.cast(llmemory.GCREF, -17)
+        assert lltype.cast_ptr_to_int(x) == -17
+
     def test_ptr_truth(self):
         abc = rffi.cast(lltype.Ptr(lltype.FuncType([], lltype.Void)), 0)
         assert not abc

From noreply at buildbot.pypy.org  Sun Oct 16 17:31:21 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 16 Oct 2011 17:31:21 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix the test: calling sleep() may not
 release the GIL before translation.
Message-ID: <20111016153121.1FD26820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48085:6048a0bf9e33
Date: 2011-10-16 16:13 +0200
http://bitbucket.org/pypy/pypy/changeset/6048a0bf9e33/

Log:	Fix the test: calling sleep() may not release the GIL before
	translation.

diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -568,20 +568,22 @@
         import thread
 
         thread_id = thread.get_ident()
-        self.ready = False
         def other_thread():
-            self.ready = True
             print "thread started"
-            time.sleep(5)
+            lock2.release()
+            lock1.acquire()
+        lock1 = thread.allocate_lock()
+        lock2 = thread.allocate_lock()
+        lock1.acquire()
+        lock2.acquire()
         thread.start_new_thread(other_thread, ())
 
         def f():
-            for i in range(100):
-                if self.ready: break
-                time.sleep(0.1)
+            lock2.acquire()
             return sys._current_frames()
 
         frames = f()
+        lock1.release()
         thisframe = frames.pop(thread_id)
         assert thisframe.f_code.co_name == 'f'
 

From noreply at buildbot.pypy.org  Sun Oct 16 17:31:22 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 16 Oct 2011 17:31:22 +0200 (CEST)
Subject: [pypy-commit] pypy default: Attempt to fix sys._current_frames() by
 returning fake frame
Message-ID: <20111016153122.6069D820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48086:d48eaa036796
Date: 2011-10-16 16:50 +0200
http://bitbucket.org/pypy/pypy/changeset/d48eaa036796/

Log:	Attempt to fix sys._current_frames() by returning fake frame
	objects.

diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -1542,6 +1542,10 @@
     def rewrite_op_jit_force_virtual(self, op):
         return self._do_builtin_call(op)
 
+    def rewrite_op_jit_is_virtual(self, op):
+        raise Exception, (
+            "'vref.virtual' should not be used from jit-visible code")
+
     def rewrite_op_jit_force_virtualizable(self, op):
         # this one is for virtualizables
         vinfo = self.get_vinfo(op.args[0])
diff --git a/pypy/jit/metainterp/test/test_virtualref.py b/pypy/jit/metainterp/test/test_virtualref.py
--- a/pypy/jit/metainterp/test/test_virtualref.py
+++ b/pypy/jit/metainterp/test/test_virtualref.py
@@ -3,6 +3,7 @@
 from pypy.rpython.llinterp import LLException
 from pypy.rlib.jit import JitDriver, dont_look_inside, vref_None
 from pypy.rlib.jit import virtual_ref, virtual_ref_finish, InvalidVirtualRef
+from pypy.rlib.jit import non_virtual_ref
 from pypy.rlib.objectmodel import compute_unique_id
 from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin, _get_jitcodes
 from pypy.jit.metainterp.resoperation import rop
@@ -595,6 +596,65 @@
         res = self.meta_interp(fn, [10])
         assert res == 6
 
+    def test_is_virtual(self):
+        myjitdriver = JitDriver(greens=[], reds=['n', 'res1'])
+        class X:
+            pass
+        @dont_look_inside
+        def residual(vref):
+            return vref.virtual
+        #
+        def f(n):
+            res1 = -42
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, res1=res1)
+                x = X()
+                vref = virtual_ref(x)
+                res1 = residual(vref)
+                virtual_ref_finish(vref, x)
+                n -= 1
+            return res1
+        #
+        res = self.meta_interp(f, [10])
+        assert res == 1
+
+    def test_is_not_virtual_none(self):
+        myjitdriver = JitDriver(greens=[], reds=['n', 'res1'])
+        @dont_look_inside
+        def residual(vref):
+            return vref.virtual
+        #
+        def f(n):
+            res1 = -42
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, res1=res1)
+                res1 = residual(vref_None)
+                n -= 1
+            return res1
+        #
+        res = self.meta_interp(f, [10])
+        assert res == 0
+
+    def test_is_not_virtual_non_none(self):
+        myjitdriver = JitDriver(greens=[], reds=['n', 'res1'])
+        class X:
+            pass
+        @dont_look_inside
+        def residual(vref):
+            return vref.virtual
+        #
+        def f(n):
+            res1 = -42
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, res1=res1)
+                x = X()
+                res1 = residual(non_virtual_ref(x))
+                n -= 1
+            return res1
+        #
+        res = self.meta_interp(f, [10])
+        assert res == 0
+
 
 class TestLLtype(VRefTests, LLJitMixin):
     pass
diff --git a/pypy/jit/metainterp/virtualref.py b/pypy/jit/metainterp/virtualref.py
--- a/pypy/jit/metainterp/virtualref.py
+++ b/pypy/jit/metainterp/virtualref.py
@@ -39,6 +39,7 @@
     def replace_force_virtual_with_call(self, graphs):
         # similar to rvirtualizable2.replace_force_virtualizable_with_call().
         c_force_virtual_ptr = None
+        c_is_virtual_ptr = None
         force_virtual_count = 0
         for graph in graphs:
             for block in graph.iterblocks():
@@ -52,6 +53,13 @@
                         op.opname = 'direct_call'
                         op.args = [c_force_virtual_ptr, op.args[0]]
                         force_virtual_count += 1
+                    #
+                    if op.opname == 'jit_is_virtual':
+                        if c_is_virtual_ptr is None:
+                            c_is_virtual_ptr = self.get_is_virtual_fnptr()
+                        #
+                        op.opname = 'direct_call'
+                        op.args = [c_is_virtual_ptr, op.args[0]]
         #
         if c_force_virtual_ptr is not None:
             log("replaced %d 'jit_force_virtual' with %r" % (force_virtual_count,
@@ -129,6 +137,17 @@
             force_virtual_if_necessary)
         return inputconst(lltype.typeOf(funcptr), funcptr)
 
+    def get_is_virtual_fnptr(self):
+        #
+        def is_virtual(inst):
+            if not inst:
+                return False
+            return inst.typeptr == self.jit_virtual_ref_vtable
+        #
+        FUNC = lltype.FuncType([rclass.OBJECTPTR], lltype.Bool)
+        funcptr = self.warmrunnerdesc.helper_func(lltype.Ptr(FUNC), is_virtual)
+        return inputconst(lltype.typeOf(funcptr), funcptr)
+
     def force_virtual(self, inst):
         vref = lltype.cast_pointer(lltype.Ptr(self.JIT_VIRTUAL_REF), inst)
         token = vref.virtual_token
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -47,7 +47,7 @@
         'pypy_initial_path'     : 'state.pypy_initial_path',
 
         '_getframe'             : 'vm._getframe', 
-        '_current_frames'       : 'vm._current_frames', 
+        '_current_frames'       : 'currentframes._current_frames', 
         'setrecursionlimit'     : 'vm.setrecursionlimit', 
         'getrecursionlimit'     : 'vm.getrecursionlimit', 
         'setcheckinterval'      : 'vm.setcheckinterval', 
diff --git a/pypy/module/sys/currentframes.py b/pypy/module/sys/currentframes.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/sys/currentframes.py
@@ -0,0 +1,78 @@
+"""
+Implementation of the 'sys._current_frames()' routine.
+"""
+from pypy.interpreter import gateway
+
+app = gateway.applevel('''
+"NOT_RPYTHON"
+import __builtin__
+
+class fake_code(object):
+    co_name = "?"
+    co_filename = "?"
+    co_firstlineno = 0
+
+class fake_frame(object):
+    f_back = None
+    f_builtins = __builtin__.__dict__
+    f_code = fake_code()
+    f_exc_traceback = None
+    f_exc_type = None
+    f_exc_value = None
+    f_globals = {}
+    f_lasti = -1
+    f_lineno = 0
+    f_locals = {}
+    f_restricted = False
+    f_trace = None
+
+    def __init__(self, f):
+        if f is not None:
+            for name in ["f_builtins", "f_code", "f_globals", "f_lasti",
+                         "f_lineno"]:
+                setattr(self, name, getattr(f, name))
+''')
+
+def _current_frames(space):
+    """_current_frames() -> dictionary
+
+    Return a dictionary mapping each current thread T's thread id to T's
+    current stack "frame".  Functions in the traceback module can build the
+    call stack given such a frame.
+
+    Note that in PyPy this returns fake frame objects, to avoid a runtime
+    penalty everywhere with the JIT.  (So far these fake frames can be
+    completely uninformative depending on the JIT state; we could return
+    more with more efforts.)
+
+    This function should be used for specialized purposes only."""
+    w_result = space.newdict()
+    w_fake_frame = app.wget(space, "fake_frame")
+    w_fake_code  = app.wget(space, "fake_code")
+    ecs = space.threadlocals.getallvalues()
+    for thread_ident, ec in ecs.items():
+        vref = ec.topframeref
+        frames = []
+        while not vref.virtual:
+            f = vref()
+            if f is None:
+                break
+            frames.append(f)
+            vref = f.f_backref
+        else:
+            frames.append(None)
+        #
+        w_topframe = space.wrap(None)
+        w_prevframe = None
+        for f in frames:
+            w_nextframe = space.call_function(w_fake_frame, space.wrap(f))
+            if w_prevframe is None:
+                w_topframe = w_nextframe
+            else:
+                space.setattr(w_prevframe, space.wrap('f_back'), w_nextframe)
+            w_prevframe = w_nextframe
+        #
+        space.setitem(w_result,
+                      space.wrap(thread_ident),
+                      w_topframe)
+    return w_result
diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -556,7 +556,7 @@
             return sys._current_frames()
         frames = f()
         assert frames.keys() == [0]
-        assert frames[0].f_code.co_name == 'f'
+        assert frames[0].f_code.co_name in ('f', '?')
 
 class AppTestCurrentFramesWithThread(AppTestCurrentFrames):
     def setup_class(cls):
@@ -585,8 +585,8 @@
         frames = f()
         lock1.release()
         thisframe = frames.pop(thread_id)
-        assert thisframe.f_code.co_name == 'f'
+        assert thisframe.f_code.co_name in ('f', '?')
 
         assert len(frames) == 1
         _, other_frame = frames.popitem()
-        assert other_frame.f_code.co_name == 'other_thread'
+        assert other_frame.f_code.co_name in ('other_thread', '?')
diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -45,25 +45,6 @@
     f.mark_as_escaped()
     return space.wrap(f)
 
-def _current_frames(space):
-    """_current_frames() -> dictionary
-
-    Return a dictionary mapping each current thread T's thread id to T's
-    current stack frame.
-
-    This function should be used for specialized purposes only."""
-    raise OperationError(space.w_NotImplementedError,
-        space.wrap("XXX sys._current_frames() incompatible with the JIT"))
-    w_result = space.newdict()
-    ecs = space.threadlocals.getallvalues()
-    for thread_ident, ec in ecs.items():
-        f = ec.gettopframe_nohidden()
-        f.mark_as_escaped()
-        space.setitem(w_result,
-                      space.wrap(thread_ident),
-                      space.wrap(f))
-    return w_result
-
 def setrecursionlimit(space, w_new_limit):
     """setrecursionlimit() sets the maximum number of nested calls that
 can occur before a RuntimeError is raised.  On PyPy the limit is
diff --git a/pypy/rlib/_jit_vref.py b/pypy/rlib/_jit_vref.py
--- a/pypy/rlib/_jit_vref.py
+++ b/pypy/rlib/_jit_vref.py
@@ -25,6 +25,10 @@
     def simple_call(self):
         return self.s_instance
 
+    def getattr(self, s_attr):
+        assert s_attr.const == 'virtual'
+        return annmodel.s_Bool
+
     def rtyper_makerepr(self, rtyper):
         if rtyper.type_system.name == 'lltypesystem':
             return vrefrepr
@@ -61,6 +65,13 @@
                              " prebuilt virtual_ref")
         return lltype.nullptr(OBJECTPTR.TO)
 
+    def rtype_getattr(self, hop):
+        s_attr = hop.args_s[1]
+        assert s_attr.const == 'virtual'
+        v = hop.inputarg(self, arg=0)
+        hop.exception_cannot_occur()
+        return hop.genop('jit_is_virtual', [v], resulttype = lltype.Bool)
+
 from pypy.rpython.ootypesystem.rclass import OBJECT
 
 class OOVRefRepr(VRefRepr):
diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -317,6 +317,12 @@
             raise InvalidVirtualRef
         return self._x
 
+    @property
+    def virtual(self):
+        """A property that is True if the vref contains a virtual that would
+        be forced by the '()' operator."""
+        return self._state == 'non-forced'
+
     def _finish(self):
         if self._state == 'non-forced':
             self._state = 'invalid'
diff --git a/pypy/rlib/test/test__jit_vref.py b/pypy/rlib/test/test__jit_vref.py
--- a/pypy/rlib/test/test__jit_vref.py
+++ b/pypy/rlib/test/test__jit_vref.py
@@ -27,10 +27,13 @@
     x1 = X()
     vref = virtual_ref(x1)
     assert vref._state == 'non-forced'
+    assert vref.virtual is True
     assert vref() is x1
     assert vref._state == 'forced'
+    assert vref.virtual is False
     virtual_ref_finish(vref, x1)
     assert vref._state == 'forced'
+    assert vref.virtual is False
     assert vref() is x1
 
 def test_direct_invalid():
@@ -135,6 +138,13 @@
         x = self.interpret(f, [])
         assert x == 42
 
+    def test_rtype_virtualattr(self):
+        def f():
+            vref = virtual_ref(X())
+            return vref.virtual
+        x = self.interpret(f, [])
+        assert x is False
+
 
 class TestLLtype(BaseTestVRef, LLRtypeMixin):
     OBJECTTYPE = OBJECTPTR
diff --git a/pypy/rpython/lltypesystem/lloperation.py b/pypy/rpython/lltypesystem/lloperation.py
--- a/pypy/rpython/lltypesystem/lloperation.py
+++ b/pypy/rpython/lltypesystem/lloperation.py
@@ -428,6 +428,7 @@
     'jit_marker':           LLOp(),
     'jit_force_virtualizable':LLOp(canrun=True),
     'jit_force_virtual':    LLOp(canrun=True),
+    'jit_is_virtual':       LLOp(canrun=True),
     'jit_force_quasi_immutable': LLOp(canrun=True),
     'get_exception_addr':   LLOp(),
     'get_exc_value_addr':   LLOp(),
diff --git a/pypy/rpython/lltypesystem/opimpl.py b/pypy/rpython/lltypesystem/opimpl.py
--- a/pypy/rpython/lltypesystem/opimpl.py
+++ b/pypy/rpython/lltypesystem/opimpl.py
@@ -538,6 +538,9 @@
 def op_jit_force_virtual(x):
     return x
 
+def op_jit_is_virtual(x):
+    return False
+
 def op_jit_force_quasi_immutable(*args):
     pass
 
diff --git a/pypy/translator/c/funcgen.py b/pypy/translator/c/funcgen.py
--- a/pypy/translator/c/funcgen.py
+++ b/pypy/translator/c/funcgen.py
@@ -833,7 +833,7 @@
         return 'INSTRUMENT_COUNT(%s);' % counter_label
             
     def OP_IS_EARLY_CONSTANT(self, op):
-        return self.expr(op.result)  + ' = 0;' # Allways false
+        return '%s = 0; /* IS_EARLY_CONSTANT */' % (self.expr(op.result),)
 
     def OP_JIT_MARKER(self, op):
         return '/* JIT_MARKER %s */' % op
@@ -845,6 +845,9 @@
         return '%s = %s; /* JIT_FORCE_VIRTUAL */' % (self.expr(op.result),
                                                      self.expr(op.args[0]))
 
+    def OP_JIT_IS_VIRTUAL(self, op):
+        return '%s = 0; /* JIT_IS_VIRTUAL */' % (self.expr(op.result),)
+
     def OP_JIT_FORCE_QUASI_IMMUTABLE(self, op):
         return '/* JIT_FORCE_QUASI_IMMUTABLE %s */' % op
 

From noreply at buildbot.pypy.org  Sun Oct 16 18:07:56 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sun, 16 Oct 2011 18:07:56 +0200 (CEST)
Subject: [pypy-commit] pypy malloc-value: close branch, isn't going anywhere.
Message-ID: <20111016160756.E1923820D7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: malloc-value
Changeset: r48087:dfa764e18840
Date: 2011-10-16 12:07 -0400
http://bitbucket.org/pypy/pypy/changeset/dfa764e18840/

Log:	close branch, isn't going anywhere.


From noreply at buildbot.pypy.org  Sun Oct 16 18:15:43 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sun, 16 Oct 2011 18:15:43 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-complex: switch to a tuple representation,
 wish we had namedtuples
Message-ID: <20111016161543.CEDA8820D7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: numpy-complex
Changeset: r48088:f097c04396bf
Date: 2011-10-16 12:15 -0400
http://bitbucket.org/pypy/pypy/changeset/f097c04396bf/

Log:	switch to a tuple representation, wish we had namedtuples

diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -485,37 +485,25 @@
     aliases = [],
     applevel_types = ["complex"],
     T = ComplexDouble,
-    valtype = ComplexDouble,
+    valtype = (float, float),
     expected_size = 16,
 )
 class W_Complex128Dtype(W_Complex128Dtype):
-    def _create_complex_struct(self, real, imag):
-        # Really we want to stack alloc it, doesn't seem possible though.
-        # Unforutnately a raw alloc means we'll leak memory (no gc) and it
-        # can't be a GcStruct because the array storage isn't a GcArray.
-        # So basically we need stack allocs.
-        c = lltype.malloc(ComplexDouble, flavor="raw")
-        c.real = real
-        c.imag = imag
-        return c
-
     @specialize.argtype(1)
     def adapt_val(self, val):
-        if hasattr(val, "_T"):
-            assert val._T is ComplexDouble
-        else:
-            val = self._create_complex_struct(rffi.cast(lltype.Float, val), 0.0)
+        if not isinstance(val, tuple):
+            val = (rffi.cast(lltype.Float, val), 0.0)
         return self.box(val)
 
     def unwrap(self, space, w_item):
         real, imag = space.unpackcomplex(w_item)
-        return self.adapt_val(self._create_complex_struct(real, imag))
+        return self.adapt_val((real, imag))
 
     def setitem(self, storage, i, item):
         val = self.unbox(item)
         # You can't set a full struct, gotta do it one field at a time.
-        self.unerase(storage)[i].real = val.real
-        self.unerase(storage)[i].imag = val.imag
+        self.unerase(storage)[i].real = val[0]
+        self.unerase(storage)[i].imag = val[1]
 
 
 ALL_DTYPES = [

From noreply at buildbot.pypy.org  Sun Oct 16 18:23:59 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sun, 16 Oct 2011 18:23:59 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-complex: enough to get tests passing,
 test_zjit blows up though
Message-ID: <20111016162359.2C0DC820D7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: numpy-complex
Changeset: r48089:78921804c629
Date: 2011-10-16 12:23 -0400
http://bitbucket.org/pypy/pypy/changeset/78921804c629/

Log:	enough to get tests passing, test_zjit blows up though

diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -64,6 +64,9 @@
 
         def wrap(self, space):
             val = self.val
+
+            if valtype == (float, float):
+                return space.newcomplex(val[0], val[1])
             if valtype is rarithmetic.r_singlefloat:
                 val = float(val)
             return space.wrap(val)
@@ -101,7 +104,7 @@
             ))
 
         def getitem(self, storage, i):
-            return Box(self.unerase(storage)[i])
+            return self.box(self.unerase(storage)[i])
 
         def setitem(self, storage, i, item):
             self.unerase(storage)[i] = self.unbox(item)
@@ -505,6 +508,12 @@
         self.unerase(storage)[i].real = val[0]
         self.unerase(storage)[i].imag = val[1]
 
+    def getitem(self, storage, i):
+        return (
+            self.unerase(storage)[i].real,
+            self.unerase(storage)[i].imag,
+        )
+
 
 ALL_DTYPES = [
     W_BoolDtype,

From noreply at buildbot.pypy.org  Sun Oct 16 18:42:21 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sun, 16 Oct 2011 18:42:21 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-complex: progress via-hacks.
Message-ID: <20111016164221.08B7D820D7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: numpy-complex
Changeset: r48090:dfe5d22615a6
Date: 2011-10-16 12:42 -0400
http://bitbucket.org/pypy/pypy/changeset/dfe5d22615a6/

Log:	progress via-hacks.

diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -39,6 +39,9 @@
             return IntObject(obj)
         raise Exception
 
+    def newcomplex(self, realval, imagval):
+        return ComplexObject(realval, imagval)
+
     def float(self, w_obj):
         assert isinstance(w_obj, FloatObject)
         return w_obj
@@ -59,6 +62,11 @@
     def __init__(self, intval):
         self.intval = intval
 
+class ComplexObject(W_Root):
+    def __init__(self, realval, imagval):
+        self.realval = realval
+        self.imagval = imagval
+
 
 space = FakeSpace()
 
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -50,13 +50,16 @@
         return space.newtuple([])
 
 
+class UnsupportedOperation(Exception):
+    pass
+
 class BaseBox(object):
     pass
 
 VOID_TP = lltype.Ptr(lltype.Array(lltype.Void, hints={'nolength': True, "uncast_on_llgraph": True}))
 
-def create_low_level_dtype(num, kind, name, aliases, applevel_types, T, valtype,
-    expected_size=None):
+def create_low_level_dtype(num, kind, name, aliases, applevel_types, T,
+    valtype, expected_size=None, exclude_methods=[]):
 
     class Box(BaseBox):
         def __init__(self, val):
@@ -103,8 +106,9 @@
                 track_allocation=False, add_memory_pressure=True
             ))
 
-        def getitem(self, storage, i):
-            return self.box(self.unerase(storage)[i])
+        if "getitem" not in exclude_methods:
+            def getitem(self, storage, i):
+                return self.box(self.unerase(storage)[i])
 
         def setitem(self, storage, i, item):
             self.unerase(storage)[i] = self.unbox(item)
@@ -112,9 +116,12 @@
         def setitem_w(self, space, storage, i, w_item):
             self.setitem(storage, i, self.unwrap(space, w_item))
 
-        @specialize.argtype(1)
-        def adapt_val(self, val):
-            return self.box(rffi.cast(TP.TO.OF, val))
+        if "adapt_val" not in exclude_methods:
+            @specialize.argtype(1)
+            def adapt_val(self, val):
+                if isinstance(val, tuple):
+                    raise UnsupportedOperation("Can't convert a complex to a %s" % self.name)
+                return self.box(rffi.cast(TP.TO.OF, val))
 
     W_LowLevelDtype.__name__ = "W_%sDtype" % name.capitalize()
     W_LowLevelDtype.num = num
@@ -490,6 +497,7 @@
     T = ComplexDouble,
     valtype = (float, float),
     expected_size = 16,
+    exclude_methods = ["adapt_val", "getitem"],
 )
 class W_Complex128Dtype(W_Complex128Dtype):
     @specialize.argtype(1)
@@ -509,10 +517,10 @@
         self.unerase(storage)[i].imag = val[1]
 
     def getitem(self, storage, i):
-        return (
+        return self.box((
             self.unerase(storage)[i].real,
             self.unerase(storage)[i].imag,
-        )
+        ))
 
 
 ALL_DTYPES = [

From noreply at buildbot.pypy.org  Sun Oct 16 19:39:31 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sun, 16 Oct 2011 19:39:31 +0200 (CEST)
Subject: [pypy-commit] pypy inline-dict-ops: merged in default.
Message-ID: <20111016173931.0939E820D7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: inline-dict-ops
Changeset: r48091:1ce87839e2bc
Date: 2011-10-16 13:25 -0400
http://bitbucket.org/pypy/pypy/changeset/1ce87839e2bc/

Log:	merged in default.

diff too long, truncating to 10000 out of 11245 lines

diff --git a/lib-python/modified-2.7/httplib.py b/lib-python/modified-2.7/httplib.py
new file mode 100644
--- /dev/null
+++ b/lib-python/modified-2.7/httplib.py
@@ -0,0 +1,1377 @@
+"""HTTP/1.1 client library
+
+<intro stuff goes here>
+<other stuff, too>
+
+HTTPConnection goes through a number of "states", which define when a client
+may legally make another request or fetch the response for a particular
+request. This diagram details these state transitions:
+
+    (null)
+      |
+      | HTTPConnection()
+      v
+    Idle
+      |
+      | putrequest()
+      v
+    Request-started
+      |
+      | ( putheader() )*  endheaders()
+      v
+    Request-sent
+      |
+      | response = getresponse()
+      v
+    Unread-response   [Response-headers-read]
+      |\____________________
+      |                     |
+      | response.read()     | putrequest()
+      v                     v
+    Idle                  Req-started-unread-response
+                     ______/|
+                   /        |
+   response.read() |        | ( putheader() )*  endheaders()
+                   v        v
+       Request-started    Req-sent-unread-response
+                            |
+                            | response.read()
+                            v
+                          Request-sent
+
+This diagram presents the following rules:
+  -- a second request may not be started until {response-headers-read}
+  -- a response [object] cannot be retrieved until {request-sent}
+  -- there is no differentiation between an unread response body and a
+     partially read response body
+
+Note: this enforcement is applied by the HTTPConnection class. The
+      HTTPResponse class does not enforce this state machine, which
+      implies sophisticated clients may accelerate the request/response
+      pipeline. Caution should be taken, though: accelerating the states
+      beyond the above pattern may imply knowledge of the server's
+      connection-close behavior for certain requests. For example, it
+      is impossible to tell whether the server will close the connection
+      UNTIL the response headers have been read; this means that further
+      requests cannot be placed into the pipeline until it is known that
+      the server will NOT be closing the connection.
+
+Logical State                  __state            __response
+-------------                  -------            ----------
+Idle                           _CS_IDLE           None
+Request-started                _CS_REQ_STARTED    None
+Request-sent                   _CS_REQ_SENT       None
+Unread-response                _CS_IDLE           <response_class>
+Req-started-unread-response    _CS_REQ_STARTED    <response_class>
+Req-sent-unread-response       _CS_REQ_SENT       <response_class>
+"""
+
+from array import array
+import os
+import socket
+from sys import py3kwarning
+from urlparse import urlsplit
+import warnings
+with warnings.catch_warnings():
+    if py3kwarning:
+        warnings.filterwarnings("ignore", ".*mimetools has been removed",
+                                DeprecationWarning)
+    import mimetools
+
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from StringIO import StringIO
+
+__all__ = ["HTTP", "HTTPResponse", "HTTPConnection",
+           "HTTPException", "NotConnected", "UnknownProtocol",
+           "UnknownTransferEncoding", "UnimplementedFileMode",
+           "IncompleteRead", "InvalidURL", "ImproperConnectionState",
+           "CannotSendRequest", "CannotSendHeader", "ResponseNotReady",
+           "BadStatusLine", "error", "responses"]
+
+HTTP_PORT = 80
+HTTPS_PORT = 443
+
+_UNKNOWN = 'UNKNOWN'
+
+# connection states
+_CS_IDLE = 'Idle'
+_CS_REQ_STARTED = 'Request-started'
+_CS_REQ_SENT = 'Request-sent'
+
+# status codes
+# informational
+CONTINUE = 100
+SWITCHING_PROTOCOLS = 101
+PROCESSING = 102
+
+# successful
+OK = 200
+CREATED = 201
+ACCEPTED = 202
+NON_AUTHORITATIVE_INFORMATION = 203
+NO_CONTENT = 204
+RESET_CONTENT = 205
+PARTIAL_CONTENT = 206
+MULTI_STATUS = 207
+IM_USED = 226
+
+# redirection
+MULTIPLE_CHOICES = 300
+MOVED_PERMANENTLY = 301
+FOUND = 302
+SEE_OTHER = 303
+NOT_MODIFIED = 304
+USE_PROXY = 305
+TEMPORARY_REDIRECT = 307
+
+# client error
+BAD_REQUEST = 400
+UNAUTHORIZED = 401
+PAYMENT_REQUIRED = 402
+FORBIDDEN = 403
+NOT_FOUND = 404
+METHOD_NOT_ALLOWED = 405
+NOT_ACCEPTABLE = 406
+PROXY_AUTHENTICATION_REQUIRED = 407
+REQUEST_TIMEOUT = 408
+CONFLICT = 409
+GONE = 410
+LENGTH_REQUIRED = 411
+PRECONDITION_FAILED = 412
+REQUEST_ENTITY_TOO_LARGE = 413
+REQUEST_URI_TOO_LONG = 414
+UNSUPPORTED_MEDIA_TYPE = 415
+REQUESTED_RANGE_NOT_SATISFIABLE = 416
+EXPECTATION_FAILED = 417
+UNPROCESSABLE_ENTITY = 422
+LOCKED = 423
+FAILED_DEPENDENCY = 424
+UPGRADE_REQUIRED = 426
+
+# server error
+INTERNAL_SERVER_ERROR = 500
+NOT_IMPLEMENTED = 501
+BAD_GATEWAY = 502
+SERVICE_UNAVAILABLE = 503
+GATEWAY_TIMEOUT = 504
+HTTP_VERSION_NOT_SUPPORTED = 505
+INSUFFICIENT_STORAGE = 507
+NOT_EXTENDED = 510
+
+# Mapping status codes to official W3C names
+responses = {
+    100: 'Continue',
+    101: 'Switching Protocols',
+
+    200: 'OK',
+    201: 'Created',
+    202: 'Accepted',
+    203: 'Non-Authoritative Information',
+    204: 'No Content',
+    205: 'Reset Content',
+    206: 'Partial Content',
+
+    300: 'Multiple Choices',
+    301: 'Moved Permanently',
+    302: 'Found',
+    303: 'See Other',
+    304: 'Not Modified',
+    305: 'Use Proxy',
+    306: '(Unused)',
+    307: 'Temporary Redirect',
+
+    400: 'Bad Request',
+    401: 'Unauthorized',
+    402: 'Payment Required',
+    403: 'Forbidden',
+    404: 'Not Found',
+    405: 'Method Not Allowed',
+    406: 'Not Acceptable',
+    407: 'Proxy Authentication Required',
+    408: 'Request Timeout',
+    409: 'Conflict',
+    410: 'Gone',
+    411: 'Length Required',
+    412: 'Precondition Failed',
+    413: 'Request Entity Too Large',
+    414: 'Request-URI Too Long',
+    415: 'Unsupported Media Type',
+    416: 'Requested Range Not Satisfiable',
+    417: 'Expectation Failed',
+
+    500: 'Internal Server Error',
+    501: 'Not Implemented',
+    502: 'Bad Gateway',
+    503: 'Service Unavailable',
+    504: 'Gateway Timeout',
+    505: 'HTTP Version Not Supported',
+}
+
+# maximal amount of data to read at one time in _safe_read
+MAXAMOUNT = 1048576
+
+class HTTPMessage(mimetools.Message):
+
+    def addheader(self, key, value):
+        """Add header for field key handling repeats."""
+        prev = self.dict.get(key)
+        if prev is None:
+            self.dict[key] = value
+        else:
+            combined = ", ".join((prev, value))
+            self.dict[key] = combined
+
+    def addcontinue(self, key, more):
+        """Add more field data from a continuation line."""
+        prev = self.dict[key]
+        self.dict[key] = prev + "\n " + more
+
+    def readheaders(self):
+        """Read header lines.
+
+        Read header lines up to the entirely blank line that terminates them.
+        The (normally blank) line that ends the headers is skipped, but not
+        included in the returned list.  If a non-header line ends the headers,
+        (which is an error), an attempt is made to backspace over it; it is
+        never included in the returned list.
+
+        The variable self.status is set to the empty string if all went well,
+        otherwise it is an error message.  The variable self.headers is a
+        completely uninterpreted list of lines contained in the header (so
+        printing them will reproduce the header exactly as it appears in the
+        file).
+
+        If multiple header fields with the same name occur, they are combined
+        according to the rules in RFC 2616 sec 4.2:
+
+        Appending each subsequent field-value to the first, each separated
+        by a comma. The order in which header fields with the same field-name
+        are received is significant to the interpretation of the combined
+        field value.
+        """
+        # XXX The implementation overrides the readheaders() method of
+        # rfc822.Message.  The base class design isn't amenable to
+        # customized behavior here so the method here is a copy of the
+        # base class code with a few small changes.
+
+        self.dict = {}
+        self.unixfrom = ''
+        self.headers = hlist = []
+        self.status = ''
+        headerseen = ""
+        firstline = 1
+        startofline = unread = tell = None
+        if hasattr(self.fp, 'unread'):
+            unread = self.fp.unread
+        elif self.seekable:
+            tell = self.fp.tell
+        while True:
+            if tell:
+                try:
+                    startofline = tell()
+                except IOError:
+                    startofline = tell = None
+                    self.seekable = 0
+            line = self.fp.readline()
+            if not line:
+                self.status = 'EOF in headers'
+                break
+            # Skip unix From name time lines
+            if firstline and line.startswith('From '):
+                self.unixfrom = self.unixfrom + line
+                continue
+            firstline = 0
+            if headerseen and line[0] in ' \t':
+                # XXX Not sure if continuation lines are handled properly
+                # for http and/or for repeating headers
+                # It's a continuation line.
+                hlist.append(line)
+                self.addcontinue(headerseen, line.strip())
+                continue
+            elif self.iscomment(line):
+                # It's a comment.  Ignore it.
+                continue
+            elif self.islast(line):
+                # Note! No pushback here!  The delimiter line gets eaten.
+                break
+            headerseen = self.isheader(line)
+            if headerseen:
+                # It's a legal header line, save it.
+                hlist.append(line)
+                self.addheader(headerseen, line[len(headerseen)+1:].strip())
+                continue
+            else:
+                # It's not a header line; throw it back and stop here.
+                if not self.dict:
+                    self.status = 'No headers'
+                else:
+                    self.status = 'Non-header line where header expected'
+                # Try to undo the read.
+                if unread:
+                    unread(line)
+                elif tell:
+                    self.fp.seek(startofline)
+                else:
+                    self.status = self.status + '; bad seek'
+                break
+
+class HTTPResponse:
+
+    # strict: If true, raise BadStatusLine if the status line can't be
+    # parsed as a valid HTTP/1.0 or 1.1 status line.  By default it is
+    # false because it prevents clients from talking to HTTP/0.9
+    # servers.  Note that a response with a sufficiently corrupted
+    # status line will look like an HTTP/0.9 response.
+
+    # See RFC 2616 sec 19.6 and RFC 1945 sec 6 for details.
+
+    def __init__(self, sock, debuglevel=0, strict=0, method=None, buffering=False):
+        if buffering:
+            # The caller won't be using any sock.recv() calls, so buffering
+            # is fine and recommended for performance.
+            self.fp = sock.makefile('rb')
+        else:
+            # The buffer size is specified as zero, because the headers of
+            # the response are read with readline().  If the reads were
+            # buffered the readline() calls could consume some of the
+            # response, which make be read via a recv() on the underlying
+            # socket.
+            self.fp = sock.makefile('rb', 0)
+        self.debuglevel = debuglevel
+        self.strict = strict
+        self._method = method
+
+        self.msg = None
+
+        # from the Status-Line of the response
+        self.version = _UNKNOWN # HTTP-Version
+        self.status = _UNKNOWN  # Status-Code
+        self.reason = _UNKNOWN  # Reason-Phrase
+
+        self.chunked = _UNKNOWN         # is "chunked" being used?
+        self.chunk_left = _UNKNOWN      # bytes left to read in current chunk
+        self.length = _UNKNOWN          # number of bytes left in response
+        self.will_close = _UNKNOWN      # conn will close at end of response
+
+    def _read_status(self):
+        # Initialize with Simple-Response defaults
+        line = self.fp.readline()
+        if self.debuglevel > 0:
+            print "reply:", repr(line)
+        if not line:
+            # Presumably, the server closed the connection before
+            # sending a valid response.
+            raise BadStatusLine(line)
+        try:
+            [version, status, reason] = line.split(None, 2)
+        except ValueError:
+            try:
+                [version, status] = line.split(None, 1)
+                reason = ""
+            except ValueError:
+                # empty version will cause next test to fail and status
+                # will be treated as 0.9 response.
+                version = ""
+        if not version.startswith('HTTP/'):
+            if self.strict:
+                self.close()
+                raise BadStatusLine(line)
+            else:
+                # assume it's a Simple-Response from an 0.9 server
+                self.fp = LineAndFileWrapper(line, self.fp)
+                return "HTTP/0.9", 200, ""
+
+        # The status code is a three-digit number
+        try:
+            status = int(status)
+            if status < 100 or status > 999:
+                raise BadStatusLine(line)
+        except ValueError:
+            raise BadStatusLine(line)
+        return version, status, reason
+
+    def begin(self):
+        if self.msg is not None:
+            # we've already started reading the response
+            return
+
+        # read until we get a non-100 response
+        while True:
+            version, status, reason = self._read_status()
+            if status != CONTINUE:
+                break
+            # skip the header from the 100 response
+            while True:
+                skip = self.fp.readline().strip()
+                if not skip:
+                    break
+                if self.debuglevel > 0:
+                    print "header:", skip
+
+        self.status = status
+        self.reason = reason.strip()
+        if version == 'HTTP/1.0':
+            self.version = 10
+        elif version.startswith('HTTP/1.'):
+            self.version = 11   # use HTTP/1.1 code for HTTP/1.x where x>=1
+        elif version == 'HTTP/0.9':
+            self.version = 9
+        else:
+            raise UnknownProtocol(version)
+
+        if self.version == 9:
+            self.length = None
+            self.chunked = 0
+            self.will_close = 1
+            self.msg = HTTPMessage(StringIO())
+            return
+
+        self.msg = HTTPMessage(self.fp, 0)
+        if self.debuglevel > 0:
+            for hdr in self.msg.headers:
+                print "header:", hdr,
+
+        # don't let the msg keep an fp
+        self.msg.fp = None
+
+        # are we using the chunked-style of transfer encoding?
+        tr_enc = self.msg.getheader('transfer-encoding')
+        if tr_enc and tr_enc.lower() == "chunked":
+            self.chunked = 1
+            self.chunk_left = None
+        else:
+            self.chunked = 0
+
+        # will the connection close at the end of the response?
+        self.will_close = self._check_close()
+
+        # do we have a Content-Length?
+        # NOTE: RFC 2616, S4.4, #3 says we ignore this if tr_enc is "chunked"
+        length = self.msg.getheader('content-length')
+        if length and not self.chunked:
+            try:
+                self.length = int(length)
+            except ValueError:
+                self.length = None
+            else:
+                if self.length < 0:  # ignore nonsensical negative lengths
+                    self.length = None
+        else:
+            self.length = None
+
+        # does the body have a fixed length? (of zero)
+        if (status == NO_CONTENT or status == NOT_MODIFIED or
+            100 <= status < 200 or      # 1xx codes
+            self._method == 'HEAD'):
+            self.length = 0
+
+        # if the connection remains open, and we aren't using chunked, and
+        # a content-length was not provided, then assume that the connection
+        # WILL close.
+        if not self.will_close and \
+           not self.chunked and \
+           self.length is None:
+            self.will_close = 1
+
+    def _check_close(self):
+        conn = self.msg.getheader('connection')
+        if self.version == 11:
+            # An HTTP/1.1 proxy is assumed to stay open unless
+            # explicitly closed.
+            conn = self.msg.getheader('connection')
+            if conn and "close" in conn.lower():
+                return True
+            return False
+
+        # Some HTTP/1.0 implementations have support for persistent
+        # connections, using rules different than HTTP/1.1.
+
+        # For older HTTP, Keep-Alive indicates persistent connection.
+        if self.msg.getheader('keep-alive'):
+            return False
+
+        # At least Akamai returns a "Connection: Keep-Alive" header,
+        # which was supposed to be sent by the client.
+        if conn and "keep-alive" in conn.lower():
+            return False
+
+        # Proxy-Connection is a netscape hack.
+        pconn = self.msg.getheader('proxy-connection')
+        if pconn and "keep-alive" in pconn.lower():
+            return False
+
+        # otherwise, assume it will close
+        return True
+
+    def close(self):
+        if self.fp:
+            self.fp.close()
+            self.fp = None
+
+    def isclosed(self):
+        # NOTE: it is possible that we will not ever call self.close(). This
+        #       case occurs when will_close is TRUE, length is None, and we
+        #       read up to the last byte, but NOT past it.
+        #
+        # IMPLIES: if will_close is FALSE, then self.close() will ALWAYS be
+        #          called, meaning self.isclosed() is meaningful.
+        return self.fp is None
+
+    # XXX It would be nice to have readline and __iter__ for this, too.
+
+    def read(self, amt=None):
+        if self.fp is None:
+            return ''
+
+        if self._method == 'HEAD':
+            self.close()
+            return ''
+
+        if self.chunked:
+            return self._read_chunked(amt)
+
+        if amt is None:
+            # unbounded read
+            if self.length is None:
+                s = self.fp.read()
+            else:
+                s = self._safe_read(self.length)
+                self.length = 0
+            self.close()        # we read everything
+            return s
+
+        if self.length is not None:
+            if amt > self.length:
+                # clip the read to the "end of response"
+                amt = self.length
+
+        # we do not use _safe_read() here because this may be a .will_close
+        # connection, and the user is reading more bytes than will be provided
+        # (for example, reading in 1k chunks)
+        s = self.fp.read(amt)
+        if self.length is not None:
+            self.length -= len(s)
+            if not self.length:
+                self.close()
+        return s
+
+    def _read_chunked(self, amt):
+        assert self.chunked != _UNKNOWN
+        chunk_left = self.chunk_left
+        value = []
+        while True:
+            if chunk_left is None:
+                line = self.fp.readline()
+                i = line.find(';')
+                if i >= 0:
+                    line = line[:i] # strip chunk-extensions
+                try:
+                    chunk_left = int(line, 16)
+                except ValueError:
+                    # close the connection as protocol synchronisation is
+                    # probably lost
+                    self.close()
+                    raise IncompleteRead(''.join(value))
+                if chunk_left == 0:
+                    break
+            if amt is None:
+                value.append(self._safe_read(chunk_left))
+            elif amt < chunk_left:
+                value.append(self._safe_read(amt))
+                self.chunk_left = chunk_left - amt
+                return ''.join(value)
+            elif amt == chunk_left:
+                value.append(self._safe_read(amt))
+                self._safe_read(2)  # toss the CRLF at the end of the chunk
+                self.chunk_left = None
+                return ''.join(value)
+            else:
+                value.append(self._safe_read(chunk_left))
+                amt -= chunk_left
+
+            # we read the whole chunk, get another
+            self._safe_read(2)      # toss the CRLF at the end of the chunk
+            chunk_left = None
+
+        # read and discard trailer up to the CRLF terminator
+        ### note: we shouldn't have any trailers!
+        while True:
+            line = self.fp.readline()
+            if not line:
+                # a vanishingly small number of sites EOF without
+                # sending the trailer
+                break
+            if line == '\r\n':
+                break
+
+        # we read everything; close the "file"
+        self.close()
+
+        return ''.join(value)
+
+    def _safe_read(self, amt):
+        """Read the number of bytes requested, compensating for partial reads.
+
+        Normally, we have a blocking socket, but a read() can be interrupted
+        by a signal (resulting in a partial read).
+
+        Note that we cannot distinguish between EOF and an interrupt when zero
+        bytes have been read. IncompleteRead() will be raised in this
+        situation.
+
+        This function should be used when <amt> bytes "should" be present for
+        reading. If the bytes are truly not available (due to EOF), then the
+        IncompleteRead exception can be used to detect the problem.
+        """
+        # NOTE(gps): As of svn r74426 socket._fileobject.read(x) will never
+        # return less than x bytes unless EOF is encountered.  It now handles
+        # signal interruptions (socket.error EINTR) internally.  This code
+        # never caught that exception anyways.  It seems largely pointless.
+        # self.fp.read(amt) will work fine.
+        s = []
+        while amt > 0:
+            chunk = self.fp.read(min(amt, MAXAMOUNT))
+            if not chunk:
+                raise IncompleteRead(''.join(s), amt)
+            s.append(chunk)
+            amt -= len(chunk)
+        return ''.join(s)
+
+    def fileno(self):
+        return self.fp.fileno()
+
+    def getheader(self, name, default=None):
+        if self.msg is None:
+            raise ResponseNotReady()
+        return self.msg.getheader(name, default)
+
+    def getheaders(self):
+        """Return list of (header, value) tuples."""
+        if self.msg is None:
+            raise ResponseNotReady()
+        return self.msg.items()
+
+
+class HTTPConnection:
+
+    _http_vsn = 11
+    _http_vsn_str = 'HTTP/1.1'
+
+    response_class = HTTPResponse
+    default_port = HTTP_PORT
+    auto_open = 1
+    debuglevel = 0
+    strict = 0
+
+    def __init__(self, host, port=None, strict=None,
+                 timeout=socket._GLOBAL_DEFAULT_TIMEOUT, source_address=None):
+        self.timeout = timeout
+        self.source_address = source_address
+        self.sock = None
+        self._buffer = []
+        self.__response = None
+        self.__state = _CS_IDLE
+        self._method = None
+        self._tunnel_host = None
+        self._tunnel_port = None
+        self._tunnel_headers = {}
+
+        self._set_hostport(host, port)
+        if strict is not None:
+            self.strict = strict
+
+    def set_tunnel(self, host, port=None, headers=None):
+        """ Sets up the host and the port for the HTTP CONNECT Tunnelling.
+
+        The headers argument should be a mapping of extra HTTP headers
+        to send with the CONNECT request.
+        """
+        self._tunnel_host = host
+        self._tunnel_port = port
+        if headers:
+            self._tunnel_headers = headers
+        else:
+            self._tunnel_headers.clear()
+
+    def _set_hostport(self, host, port):
+        if port is None:
+            i = host.rfind(':')
+            j = host.rfind(']')         # ipv6 addresses have [...]
+            if i > j:
+                try:
+                    port = int(host[i+1:])
+                except ValueError:
+                    raise InvalidURL("nonnumeric port: '%s'" % host[i+1:])
+                host = host[:i]
+            else:
+                port = self.default_port
+            if host and host[0] == '[' and host[-1] == ']':
+                host = host[1:-1]
+        self.host = host
+        self.port = port
+
+    def set_debuglevel(self, level):
+        self.debuglevel = level
+
+    def _tunnel(self):
+        self._set_hostport(self._tunnel_host, self._tunnel_port)
+        self.send("CONNECT %s:%d HTTP/1.0\r\n" % (self.host, self.port))
+        for header, value in self._tunnel_headers.iteritems():
+            self.send("%s: %s\r\n" % (header, value))
+        self.send("\r\n")
+        response = self.response_class(self.sock, strict = self.strict,
+                                       method = self._method)
+        (version, code, message) = response._read_status()
+
+        if code != 200:
+            self.close()
+            raise socket.error("Tunnel connection failed: %d %s" % (code,
+                                                                    message.strip()))
+        while True:
+            line = response.fp.readline()
+            if line == '\r\n': break
+
+
+    def connect(self):
+        """Connect to the host and port specified in __init__."""
+        self.sock = socket.create_connection((self.host,self.port),
+                                             self.timeout, self.source_address)
+
+        if self._tunnel_host:
+            self._tunnel()
+
+    def close(self):
+        """Close the connection to the HTTP server."""
+        if self.sock:
+            self.sock.close()   # close it manually... there may be other refs
+            self.sock = None
+        if self.__response:
+            self.__response.close()
+            self.__response = None
+        self.__state = _CS_IDLE
+
+    def send(self, data):
+        """Send `data' to the server."""
+        if self.sock is None:
+            if self.auto_open:
+                self.connect()
+            else:
+                raise NotConnected()
+
+        if self.debuglevel > 0:
+            print "send:", repr(data)
+        blocksize = 8192
+        if hasattr(data,'read') and not isinstance(data, array):
+            if self.debuglevel > 0: print "sendIng a read()able"
+            datablock = data.read(blocksize)
+            while datablock:
+                self.sock.sendall(datablock)
+                datablock = data.read(blocksize)
+        else:
+            self.sock.sendall(data)
+
+    def _output(self, s):
+        """Add a line of output to the current request buffer.
+
+        Assumes that the line does *not* end with \\r\\n.
+        """
+        self._buffer.append(s)
+
+    def _send_output(self, message_body=None):
+        """Send the currently buffered request and clear the buffer.
+
+        Appends an extra \\r\\n to the buffer.
+        A message_body may be specified, to be appended to the request.
+        """
+        self._buffer.extend(("", ""))
+        msg = "\r\n".join(self._buffer)
+        del self._buffer[:]
+        # If msg and message_body are sent in a single send() call,
+        # it will avoid performance problems caused by the interaction
+        # between delayed ack and the Nagle algorithim.
+        if isinstance(message_body, str):
+            msg += message_body
+            message_body = None
+        self.send(msg)
+        if message_body is not None:
+            #message_body was not a string (i.e. it is a file) and
+            #we must run the risk of Nagle
+            self.send(message_body)
+
+    def putrequest(self, method, url, skip_host=0, skip_accept_encoding=0):
+        """Send a request to the server.
+
+        `method' specifies an HTTP request method, e.g. 'GET'.
+        `url' specifies the object being requested, e.g. '/index.html'.
+        `skip_host' if True does not add automatically a 'Host:' header
+        `skip_accept_encoding' if True does not add automatically an
+           'Accept-Encoding:' header
+        """
+
+        # if a prior response has been completed, then forget about it.
+        if self.__response and self.__response.isclosed():
+            self.__response = None
+
+
+        # in certain cases, we cannot issue another request on this connection.
+        # this occurs when:
+        #   1) we are in the process of sending a request.   (_CS_REQ_STARTED)
+        #   2) a response to a previous request has signalled that it is going
+        #      to close the connection upon completion.
+        #   3) the headers for the previous response have not been read, thus
+        #      we cannot determine whether point (2) is true.   (_CS_REQ_SENT)
+        #
+        # if there is no prior response, then we can request at will.
+        #
+        # if point (2) is true, then we will have passed the socket to the
+        # response (effectively meaning, "there is no prior response"), and
+        # will open a new one when a new request is made.
+        #
+        # Note: if a prior response exists, then we *can* start a new request.
+        #       We are not allowed to begin fetching the response to this new
+        #       request, however, until that prior response is complete.
+        #
+        if self.__state == _CS_IDLE:
+            self.__state = _CS_REQ_STARTED
+        else:
+            raise CannotSendRequest()
+
+        # Save the method we use, we need it later in the response phase
+        self._method = method
+        if not url:
+            url = '/'
+        hdr = '%s %s %s' % (method, url, self._http_vsn_str)
+
+        self._output(hdr)
+
+        if self._http_vsn == 11:
+            # Issue some standard headers for better HTTP/1.1 compliance
+
+            if not skip_host:
+                # this header is issued *only* for HTTP/1.1
+                # connections. more specifically, this means it is
+                # only issued when the client uses the new
+                # HTTPConnection() class. backwards-compat clients
+                # will be using HTTP/1.0 and those clients may be
+                # issuing this header themselves. we should NOT issue
+                # it twice; some web servers (such as Apache) barf
+                # when they see two Host: headers
+
+                # If we need a non-standard port,include it in the
+                # header.  If the request is going through a proxy,
+                # but the host of the actual URL, not the host of the
+                # proxy.
+
+                netloc = ''
+                if url.startswith('http'):
+                    nil, netloc, nil, nil, nil = urlsplit(url)
+
+                if netloc:
+                    try:
+                        netloc_enc = netloc.encode("ascii")
+                    except UnicodeEncodeError:
+                        netloc_enc = netloc.encode("idna")
+                    self.putheader('Host', netloc_enc)
+                else:
+                    try:
+                        host_enc = self.host.encode("ascii")
+                    except UnicodeEncodeError:
+                        host_enc = self.host.encode("idna")
+                    # Wrap the IPv6 Host Header with [] (RFC 2732)
+                    if host_enc.find(':') >= 0:
+                        host_enc = "[" + host_enc + "]"
+                    if self.port == self.default_port:
+                        self.putheader('Host', host_enc)
+                    else:
+                        self.putheader('Host', "%s:%s" % (host_enc, self.port))
+
+            # note: we are assuming that clients will not attempt to set these
+            #       headers since *this* library must deal with the
+            #       consequences. this also means that when the supporting
+            #       libraries are updated to recognize other forms, then this
+            #       code should be changed (removed or updated).
+
+            # we only want a Content-Encoding of "identity" since we don't
+            # support encodings such as x-gzip or x-deflate.
+            if not skip_accept_encoding:
+                self.putheader('Accept-Encoding', 'identity')
+
+            # we can accept "chunked" Transfer-Encodings, but no others
+            # NOTE: no TE header implies *only* "chunked"
+            #self.putheader('TE', 'chunked')
+
+            # if TE is supplied in the header, then it must appear in a
+            # Connection header.
+            #self.putheader('Connection', 'TE')
+
+        else:
+            # For HTTP/1.0, the server will assume "not chunked"
+            pass
+
+    def putheader(self, header, *values):
+        """Send a request header line to the server.
+
+        For example: h.putheader('Accept', 'text/html')
+        """
+        if self.__state != _CS_REQ_STARTED:
+            raise CannotSendHeader()
+
+        hdr = '%s: %s' % (header, '\r\n\t'.join([str(v) for v in values]))
+        self._output(hdr)
+
+    def endheaders(self, message_body=None):
+        """Indicate that the last header line has been sent to the server.
+
+        This method sends the request to the server.  The optional
+        message_body argument can be used to pass message body
+        associated with the request.  The message body will be sent in
+        the same packet as the message headers if possible.  The
+        message_body should be a string.
+        """
+        if self.__state == _CS_REQ_STARTED:
+            self.__state = _CS_REQ_SENT
+        else:
+            raise CannotSendHeader()
+        self._send_output(message_body)
+
+    def request(self, method, url, body=None, headers={}):
+        """Send a complete request to the server."""
+        self._send_request(method, url, body, headers)
+
+    def _set_content_length(self, body):
+        # Set the content-length based on the body.
+        thelen = None
+        try:
+            thelen = str(len(body))
+        except TypeError, te:
+            # If this is a file-like object, try to
+            # fstat its file descriptor
+            try:
+                thelen = str(os.fstat(body.fileno()).st_size)
+            except (AttributeError, OSError):
+                # Don't send a length if this failed
+                if self.debuglevel > 0: print "Cannot stat!!"
+
+        if thelen is not None:
+            self.putheader('Content-Length', thelen)
+
+    def _send_request(self, method, url, body, headers):
+        # Honor explicitly requested Host: and Accept-Encoding: headers.
+        header_names = dict.fromkeys([k.lower() for k in headers])
+        skips = {}
+        if 'host' in header_names:
+            skips['skip_host'] = 1
+        if 'accept-encoding' in header_names:
+            skips['skip_accept_encoding'] = 1
+
+        self.putrequest(method, url, **skips)
+
+        if body and ('content-length' not in header_names):
+            self._set_content_length(body)
+        for hdr, value in headers.iteritems():
+            self.putheader(hdr, value)
+        self.endheaders(body)
+
+    def getresponse(self, buffering=False):
+        "Get the response from the server."
+
+        # if a prior response has been completed, then forget about it.
+        if self.__response and self.__response.isclosed():
+            self.__response = None
+
+        #
+        # if a prior response exists, then it must be completed (otherwise, we
+        # cannot read this response's header to determine the connection-close
+        # behavior)
+        #
+        # note: if a prior response existed, but was connection-close, then the
+        # socket and response were made independent of this HTTPConnection
+        # object since a new request requires that we open a whole new
+        # connection
+        #
+        # this means the prior response had one of two states:
+        #   1) will_close: this connection was reset and the prior socket and
+        #                  response operate independently
+        #   2) persistent: the response was retained and we await its
+        #                  isclosed() status to become true.
+        #
+        if self.__state != _CS_REQ_SENT or self.__response:
+            raise ResponseNotReady()
+
+        args = (self.sock,)
+        kwds = {"strict":self.strict, "method":self._method}
+        if self.debuglevel > 0:
+            args += (self.debuglevel,)
+        if buffering:
+            #only add this keyword if non-default, for compatibility with
+            #other response_classes.
+            kwds["buffering"] = True;
+        response = self.response_class(*args, **kwds)
+
+        try:
+            response.begin()
+        except:
+            response.close()
+            raise
+        assert response.will_close != _UNKNOWN
+        self.__state = _CS_IDLE
+
+        if response.will_close:
+            # this effectively passes the connection to the response
+            self.close()
+        else:
+            # remember this, so we can tell when it is complete
+            self.__response = response
+
+        return response
+
+
+class HTTP:
+    "Compatibility class with httplib.py from 1.5."
+
+    _http_vsn = 10
+    _http_vsn_str = 'HTTP/1.0'
+
+    debuglevel = 0
+
+    _connection_class = HTTPConnection
+
+    def __init__(self, host='', port=None, strict=None):
+        "Provide a default host, since the superclass requires one."
+
+        # some joker passed 0 explicitly, meaning default port
+        if port == 0:
+            port = None
+
+        # Note that we may pass an empty string as the host; this will throw
+        # an error when we attempt to connect. Presumably, the client code
+        # will call connect before then, with a proper host.
+        self._setup(self._connection_class(host, port, strict))
+
+    def _setup(self, conn):
+        self._conn = conn
+
+        # set up delegation to flesh out interface
+        self.send = conn.send
+        self.putrequest = conn.putrequest
+        self.putheader = conn.putheader
+        self.endheaders = conn.endheaders
+        self.set_debuglevel = conn.set_debuglevel
+
+        conn._http_vsn = self._http_vsn
+        conn._http_vsn_str = self._http_vsn_str
+
+        self.file = None
+
+    def connect(self, host=None, port=None):
+        "Accept arguments to set the host/port, since the superclass doesn't."
+
+        if host is not None:
+            self._conn._set_hostport(host, port)
+        self._conn.connect()
+
+    def getfile(self):
+        "Provide a getfile, since the superclass' does not use this concept."
+        return self.file
+
+    def getreply(self, buffering=False):
+        """Compat definition since superclass does not define it.
+
+        Returns a tuple consisting of:
+        - server status code (e.g. '200' if all goes well)
+        - server "reason" corresponding to status code
+        - any RFC822 headers in the response from the server
+        """
+        try:
+            if not buffering:
+                response = self._conn.getresponse()
+            else:
+                #only add this keyword if non-default for compatibility
+                #with other connection classes
+                response = self._conn.getresponse(buffering)
+        except BadStatusLine, e:
+            ### hmm. if getresponse() ever closes the socket on a bad request,
+            ### then we are going to have problems with self.sock
+
+            ### should we keep this behavior? do people use it?
+            # keep the socket open (as a file), and return it
+            self.file = self._conn.sock.makefile('rb', 0)
+
+            # close our socket -- we want to restart after any protocol error
+            self.close()
+
+            self.headers = None
+            return -1, e.line, None
+
+        self.headers = response.msg
+        self.file = response.fp
+        return response.status, response.reason, response.msg
+
+    def close(self):
+        self._conn.close()
+
+        # note that self.file == response.fp, which gets closed by the
+        # superclass. just clear the object ref here.
+        ### hmm. messy. if status==-1, then self.file is owned by us.
+        ### well... we aren't explicitly closing, but losing this ref will
+        ### do it
+        self.file = None
+
+try:
+    import ssl
+except ImportError:
+    pass
+else:
+    class HTTPSConnection(HTTPConnection):
+        "This class allows communication via SSL."
+
+        default_port = HTTPS_PORT
+
+        def __init__(self, host, port=None, key_file=None, cert_file=None,
+                     strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
+                     source_address=None):
+            HTTPConnection.__init__(self, host, port, strict, timeout,
+                                    source_address)
+            self.key_file = key_file
+            self.cert_file = cert_file
+
+        def connect(self):
+            "Connect to a host on a given (SSL) port."
+
+            sock = socket.create_connection((self.host, self.port),
+                                            self.timeout, self.source_address)
+            if self._tunnel_host:
+                self.sock = sock
+                self._tunnel()
+            self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file)
+
+    __all__.append("HTTPSConnection")
+
+    class HTTPS(HTTP):
+        """Compatibility with 1.5 httplib interface
+
+        Python 1.5.2 did not have an HTTPS class, but it defined an
+        interface for sending http requests that is also useful for
+        https.
+        """
+
+        _connection_class = HTTPSConnection
+
+        def __init__(self, host='', port=None, key_file=None, cert_file=None,
+                     strict=None):
+            # provide a default host, pass the X509 cert info
+
+            # urf. compensate for bad input.
+            if port == 0:
+                port = None
+            self._setup(self._connection_class(host, port, key_file,
+                                               cert_file, strict))
+
+            # we never actually use these for anything, but we keep them
+            # here for compatibility with post-1.5.2 CVS.
+            self.key_file = key_file
+            self.cert_file = cert_file
+
+
+    def FakeSocket (sock, sslobj):
+        warnings.warn("FakeSocket is deprecated, and won't be in 3.x.  " +
+                      "Use the result of ssl.wrap_socket() directly instead.",
+                      DeprecationWarning, stacklevel=2)
+        return sslobj
+
+
+class HTTPException(Exception):
+    # Subclasses that define an __init__ must call Exception.__init__
+    # or define self.args.  Otherwise, str() will fail.
+    pass
+
+class NotConnected(HTTPException):
+    pass
+
+class InvalidURL(HTTPException):
+    pass
+
+class UnknownProtocol(HTTPException):
+    def __init__(self, version):
+        self.args = version,
+        self.version = version
+
+class UnknownTransferEncoding(HTTPException):
+    pass
+
+class UnimplementedFileMode(HTTPException):
+    pass
+
+class IncompleteRead(HTTPException):
+    def __init__(self, partial, expected=None):
+        self.args = partial,
+        self.partial = partial
+        self.expected = expected
+    def __repr__(self):
+        if self.expected is not None:
+            e = ', %i more expected' % self.expected
+        else:
+            e = ''
+        return 'IncompleteRead(%i bytes read%s)' % (len(self.partial), e)
+    def __str__(self):
+        return repr(self)
+
+class ImproperConnectionState(HTTPException):
+    pass
+
+class CannotSendRequest(ImproperConnectionState):
+    pass
+
+class CannotSendHeader(ImproperConnectionState):
+    pass
+
+class ResponseNotReady(ImproperConnectionState):
+    pass
+
+class BadStatusLine(HTTPException):
+    def __init__(self, line):
+        if not line:
+            line = repr(line)
+        self.args = line,
+        self.line = line
+
+# for backwards compatibility
+error = HTTPException
+
+class LineAndFileWrapper:
+    """A limited file-like object for HTTP/0.9 responses."""
+
+    # The status-line parsing code calls readline(), which normally
+    # get the HTTP status line.  For a 0.9 response, however, this is
+    # actually the first line of the body!  Clients need to get a
+    # readable file object that contains that line.
+
+    def __init__(self, line, file):
+        self._line = line
+        self._file = file
+        self._line_consumed = 0
+        self._line_offset = 0
+        self._line_left = len(line)
+
+    def __getattr__(self, attr):
+        return getattr(self._file, attr)
+
+    def _done(self):
+        # called when the last byte is read from the line.  After the
+        # call, all read methods are delegated to the underlying file
+        # object.
+        self._line_consumed = 1
+        self.read = self._file.read
+        self.readline = self._file.readline
+        self.readlines = self._file.readlines
+
+    def read(self, amt=None):
+        if self._line_consumed:
+            return self._file.read(amt)
+        assert self._line_left
+        if amt is None or amt > self._line_left:
+            s = self._line[self._line_offset:]
+            self._done()
+            if amt is None:
+                return s + self._file.read()
+            else:
+                return s + self._file.read(amt - len(s))
+        else:
+            assert amt <= self._line_left
+            i = self._line_offset
+            j = i + amt
+            s = self._line[i:j]
+            self._line_offset = j
+            self._line_left -= amt
+            if self._line_left == 0:
+                self._done()
+            return s
+
+    def readline(self):
+        if self._line_consumed:
+            return self._file.readline()
+        assert self._line_left
+        s = self._line[self._line_offset:]
+        self._done()
+        return s
+
+    def readlines(self, size=None):
+        if self._line_consumed:
+            return self._file.readlines(size)
+        assert self._line_left
+        L = [self._line[self._line_offset:]]
+        self._done()
+        if size is None:
+            return L + self._file.readlines()
+        else:
+            return L + self._file.readlines(size)
+
+def test():
+    """Test this module.
+
+    A hodge podge of tests collected here, because they have too many
+    external dependencies for the regular test suite.
+    """
+
+    import sys
+    import getopt
+    opts, args = getopt.getopt(sys.argv[1:], 'd')
+    dl = 0
+    for o, a in opts:
+        if o == '-d': dl = dl + 1
+    host = 'www.python.org'
+    selector = '/'
+    if args[0:]: host = args[0]
+    if args[1:]: selector = args[1]
+    h = HTTP()
+    h.set_debuglevel(dl)
+    h.connect(host)
+    h.putrequest('GET', selector)
+    h.endheaders()
+    status, reason, headers = h.getreply()
+    print 'status =', status
+    print 'reason =', reason
+    print "read", len(h.getfile().read())
+    print
+    if headers:
+        for header in headers.headers: print header.strip()
+    print
+
+    # minimal test that code to extract host from url works
+    class HTTP11(HTTP):
+        _http_vsn = 11
+        _http_vsn_str = 'HTTP/1.1'
+
+    h = HTTP11('www.python.org')
+    h.putrequest('GET', 'http://www.python.org/~jeremy/')
+    h.endheaders()
+    h.getreply()
+    h.close()
+
+    try:
+        import ssl
+    except ImportError:
+        pass
+    else:
+
+        for host, selector in (('sourceforge.net', '/projects/python'),
+                               ):
+            print "https://%s%s" % (host, selector)
+            hs = HTTPS()
+            hs.set_debuglevel(dl)
+            hs.connect(host)
+            hs.putrequest('GET', selector)
+            hs.endheaders()
+            status, reason, headers = hs.getreply()
+            print 'status =', status
+            print 'reason =', reason
+            print "read", len(hs.getfile().read())
+            print
+            if headers:
+                for header in headers.headers: print header.strip()
+            print
+
+if __name__ == '__main__':
+    test()
diff --git a/lib-python/modified-2.7/test/test_array.py b/lib-python/modified-2.7/test/test_array.py
--- a/lib-python/modified-2.7/test/test_array.py
+++ b/lib-python/modified-2.7/test/test_array.py
@@ -295,9 +295,10 @@
         )
 
         b = array.array(self.badtypecode())
-        self.assertRaises(TypeError, "a + b")
-
-        self.assertRaises(TypeError, "a + 'bad'")
+        with self.assertRaises(TypeError):
+            a + b
+        with self.assertRaises(TypeError):
+            a + 'bad'
 
     def test_iadd(self):
         a = array.array(self.typecode, self.example[::-1])
@@ -316,9 +317,10 @@
         )
 
         b = array.array(self.badtypecode())
-        self.assertRaises(TypeError, "a += b")
-
-        self.assertRaises(TypeError, "a += 'bad'")
+        with self.assertRaises(TypeError):
+            a += b
+        with self.assertRaises(TypeError):
+            a += 'bad'
 
     def test_mul(self):
         a = 5*array.array(self.typecode, self.example)
@@ -345,7 +347,8 @@
             array.array(self.typecode)
         )
 
-        self.assertRaises(TypeError, "a * 'bad'")
+        with self.assertRaises(TypeError):
+            a * 'bad'
 
     def test_imul(self):
         a = array.array(self.typecode, self.example)
@@ -374,7 +377,8 @@
         a *= -1
         self.assertEqual(a, array.array(self.typecode))
 
-        self.assertRaises(TypeError, "a *= 'bad'")
+        with self.assertRaises(TypeError):
+            a *= 'bad'
 
     def test_getitem(self):
         a = array.array(self.typecode, self.example)
diff --git a/lib-python/modified-2.7/test/test_sys_settrace.py b/lib-python/modified-2.7/test/test_sys_settrace.py
--- a/lib-python/modified-2.7/test/test_sys_settrace.py
+++ b/lib-python/modified-2.7/test/test_sys_settrace.py
@@ -286,11 +286,11 @@
         self.compare_events(func.func_code.co_firstlineno,
                             tracer.events, func.events)
 
-    def set_and_retrieve_none(self):
+    def test_set_and_retrieve_none(self):
         sys.settrace(None)
         assert sys.gettrace() is None
 
-    def set_and_retrieve_func(self):
+    def test_set_and_retrieve_func(self):
         def fn(*args):
             pass
 
diff --git a/lib-python/modified-2.7/test/test_urllib2.py b/lib-python/modified-2.7/test/test_urllib2.py
--- a/lib-python/modified-2.7/test/test_urllib2.py
+++ b/lib-python/modified-2.7/test/test_urllib2.py
@@ -307,6 +307,9 @@
     def getresponse(self):
         return MockHTTPResponse(MockFile(), {}, 200, "OK")
 
+    def close(self):
+        pass
+
 class MockHandler:
     # useful for testing handler machinery
     # see add_ordered_mock_handlers() docstring
diff --git a/lib-python/modified-2.7/urllib2.py b/lib-python/modified-2.7/urllib2.py
new file mode 100644
--- /dev/null
+++ b/lib-python/modified-2.7/urllib2.py
@@ -0,0 +1,1440 @@
+"""An extensible library for opening URLs using a variety of protocols
+
+The simplest way to use this module is to call the urlopen function,
+which accepts a string containing a URL or a Request object (described
+below).  It opens the URL and returns the results as file-like
+object; the returned object has some extra methods described below.
+
+The OpenerDirector manages a collection of Handler objects that do
+all the actual work.  Each Handler implements a particular protocol or
+option.  The OpenerDirector is a composite object that invokes the
+Handlers needed to open the requested URL.  For example, the
+HTTPHandler performs HTTP GET and POST requests and deals with
+non-error returns.  The HTTPRedirectHandler automatically deals with
+HTTP 301, 302, 303 and 307 redirect errors, and the HTTPDigestAuthHandler
+deals with digest authentication.
+
+urlopen(url, data=None) -- Basic usage is the same as original
+urllib.  pass the url and optionally data to post to an HTTP URL, and
+get a file-like object back.  One difference is that you can also pass
+a Request instance instead of URL.  Raises a URLError (subclass of
+IOError); for HTTP errors, raises an HTTPError, which can also be
+treated as a valid response.
+
+build_opener -- Function that creates a new OpenerDirector instance.
+Will install the default handlers.  Accepts one or more Handlers as
+arguments, either instances or Handler classes that it will
+instantiate.  If one of the argument is a subclass of the default
+handler, the argument will be installed instead of the default.
+
+install_opener -- Installs a new opener as the default opener.
+
+objects of interest:
+
+OpenerDirector -- Sets up the User Agent as the Python-urllib client and manages
+the Handler classes, while dealing with requests and responses.
+
+Request -- An object that encapsulates the state of a request.  The
+state can be as simple as the URL.  It can also include extra HTTP
+headers, e.g. a User-Agent.
+
+BaseHandler --
+
+exceptions:
+URLError -- A subclass of IOError, individual protocols have their own
+specific subclass.
+
+HTTPError -- Also a valid HTTP response, so you can treat an HTTP error
+as an exceptional event or valid response.
+
+internals:
+BaseHandler and parent
+_call_chain conventions
+
+Example usage:
+
+import urllib2
+
+# set up authentication info
+authinfo = urllib2.HTTPBasicAuthHandler()
+authinfo.add_password(realm='PDQ Application',
+                      uri='https://mahler:8092/site-updates.py',
+                      user='klem',
+                      passwd='geheim$parole')
+
+proxy_support = urllib2.ProxyHandler({"http" : "http://ahad-haam:3128"})
+
+# build a new opener that adds authentication and caching FTP handlers
+opener = urllib2.build_opener(proxy_support, authinfo, urllib2.CacheFTPHandler)
+
+# install it
+urllib2.install_opener(opener)
+
+f = urllib2.urlopen('http://www.python.org/')
+
+
+"""
+
+# XXX issues:
+# If an authentication error handler that tries to perform
+# authentication for some reason but fails, how should the error be
+# signalled?  The client needs to know the HTTP error code.  But if
+# the handler knows that the problem was, e.g., that it didn't know
+# that hash algo that requested in the challenge, it would be good to
+# pass that information along to the client, too.
+# ftp errors aren't handled cleanly
+# check digest against correct (i.e. non-apache) implementation
+
+# Possible extensions:
+# complex proxies  XXX not sure what exactly was meant by this
+# abstract factory for opener
+
+import base64
+import hashlib
+import httplib
+import mimetools
+import os
+import posixpath
+import random
+import re
+import socket
+import sys
+import time
+import urlparse
+import bisect
+
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from StringIO import StringIO
+
+from urllib import (unwrap, unquote, splittype, splithost, quote,
+     addinfourl, splitport, splittag,
+     splitattr, ftpwrapper, splituser, splitpasswd, splitvalue)
+
+# support for FileHandler, proxies via environment variables
+from urllib import localhost, url2pathname, getproxies, proxy_bypass
+
+# used in User-Agent header sent
+__version__ = sys.version[:3]
+
+_opener = None
+def urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
+    global _opener
+    if _opener is None:
+        _opener = build_opener()
+    return _opener.open(url, data, timeout)
+
+def install_opener(opener):
+    global _opener
+    _opener = opener
+
+# do these error classes make sense?
+# make sure all of the IOError stuff is overridden.  we just want to be
+# subtypes.
+
+class URLError(IOError):
+    # URLError is a sub-type of IOError, but it doesn't share any of
+    # the implementation.  need to override __init__ and __str__.
+    # It sets self.args for compatibility with other EnvironmentError
+    # subclasses, but args doesn't have the typical format with errno in
+    # slot 0 and strerror in slot 1.  This may be better than nothing.
+    def __init__(self, reason):
+        self.args = reason,
+        self.reason = reason
+
+    def __str__(self):
+        return '<urlopen error %s>' % self.reason
+
+class HTTPError(URLError, addinfourl):
+    """Raised when HTTP error occurs, but also acts like non-error return"""
+    __super_init = addinfourl.__init__
+
+    def __init__(self, url, code, msg, hdrs, fp):
+        self.code = code
+        self.msg = msg
+        self.hdrs = hdrs
+        self.fp = fp
+        self.filename = url
+        # The addinfourl classes depend on fp being a valid file
+        # object.  In some cases, the HTTPError may not have a valid
+        # file object.  If this happens, the simplest workaround is to
+        # not initialize the base classes.
+        if fp is not None:
+            self.__super_init(fp, hdrs, url, code)
+
+    def __str__(self):
+        return 'HTTP Error %s: %s' % (self.code, self.msg)
+
+# copied from cookielib.py
+_cut_port_re = re.compile(r":\d+$")
+def request_host(request):
+    """Return request-host, as defined by RFC 2965.
+
+    Variation from RFC: returned value is lowercased, for convenient
+    comparison.
+
+    """
+    url = request.get_full_url()
+    host = urlparse.urlparse(url)[1]
+    if host == "":
+        host = request.get_header("Host", "")
+
+    # remove port, if present
+    host = _cut_port_re.sub("", host, 1)
+    return host.lower()
+
+class Request:
+
+    def __init__(self, url, data=None, headers={},
+                 origin_req_host=None, unverifiable=False):
+        # unwrap('<URL:type://host/path>') --> 'type://host/path'
+        self.__original = unwrap(url)
+        self.__original, fragment = splittag(self.__original)
+        self.type = None
+        # self.__r_type is what's left after doing the splittype
+        self.host = None
+        self.port = None
+        self._tunnel_host = None
+        self.data = data
+        self.headers = {}
+        for key, value in headers.items():
+            self.add_header(key, value)
+        self.unredirected_hdrs = {}
+        if origin_req_host is None:
+            origin_req_host = request_host(self)
+        self.origin_req_host = origin_req_host
+        self.unverifiable = unverifiable
+
+    def __getattr__(self, attr):
+        # XXX this is a fallback mechanism to guard against these
+        # methods getting called in a non-standard order.  this may be
+        # too complicated and/or unnecessary.
+        # XXX should the __r_XXX attributes be public?
+        if attr[:12] == '_Request__r_':
+            name = attr[12:]
+            if hasattr(Request, 'get_' + name):
+                getattr(self, 'get_' + name)()
+                return getattr(self, attr)
+        raise AttributeError, attr
+
+    def get_method(self):
+        if self.has_data():
+            return "POST"
+        else:
+            return "GET"
+
+    # XXX these helper methods are lame
+
+    def add_data(self, data):
+        self.data = data
+
+    def has_data(self):
+        return self.data is not None
+
+    def get_data(self):
+        return self.data
+
+    def get_full_url(self):
+        return self.__original
+
+    def get_type(self):
+        if self.type is None:
+            self.type, self.__r_type = splittype(self.__original)
+            if self.type is None:
+                raise ValueError, "unknown url type: %s" % self.__original
+        return self.type
+
+    def get_host(self):
+        if self.host is None:
+            self.host, self.__r_host = splithost(self.__r_type)
+            if self.host:
+                self.host = unquote(self.host)
+        return self.host
+
+    def get_selector(self):
+        return self.__r_host
+
+    def set_proxy(self, host, type):
+        if self.type == 'https' and not self._tunnel_host:
+            self._tunnel_host = self.host
+        else:
+            self.type = type
+            self.__r_host = self.__original
+
+        self.host = host
+
+    def has_proxy(self):
+        return self.__r_host == self.__original
+
+    def get_origin_req_host(self):
+        return self.origin_req_host
+
+    def is_unverifiable(self):
+        return self.unverifiable
+
+    def add_header(self, key, val):
+        # useful for something like authentication
+        self.headers[key.capitalize()] = val
+
+    def add_unredirected_header(self, key, val):
+        # will not be added to a redirected request
+        self.unredirected_hdrs[key.capitalize()] = val
+
+    def has_header(self, header_name):
+        return (header_name in self.headers or
+                header_name in self.unredirected_hdrs)
+
+    def get_header(self, header_name, default=None):
+        return self.headers.get(
+            header_name,
+            self.unredirected_hdrs.get(header_name, default))
+
+    def header_items(self):
+        hdrs = self.unredirected_hdrs.copy()
+        hdrs.update(self.headers)
+        return hdrs.items()
+
+class OpenerDirector:
+    def __init__(self):
+        client_version = "Python-urllib/%s" % __version__
+        self.addheaders = [('User-agent', client_version)]
+        # manage the individual handlers
+        self.handlers = []
+        self.handle_open = {}
+        self.handle_error = {}
+        self.process_response = {}
+        self.process_request = {}
+
+    def add_handler(self, handler):
+        if not hasattr(handler, "add_parent"):
+            raise TypeError("expected BaseHandler instance, got %r" %
+                            type(handler))
+
+        added = False
+        for meth in dir(handler):
+            if meth in ["redirect_request", "do_open", "proxy_open"]:
+                # oops, coincidental match
+                continue
+
+            i = meth.find("_")
+            protocol = meth[:i]
+            condition = meth[i+1:]
+
+            if condition.startswith("error"):
+                j = condition.find("_") + i + 1
+                kind = meth[j+1:]
+                try:
+                    kind = int(kind)
+                except ValueError:
+                    pass
+                lookup = self.handle_error.get(protocol, {})
+                self.handle_error[protocol] = lookup
+            elif condition == "open":
+                kind = protocol
+                lookup = self.handle_open
+            elif condition == "response":
+                kind = protocol
+                lookup = self.process_response
+            elif condition == "request":
+                kind = protocol
+                lookup = self.process_request
+            else:
+                continue
+
+            handlers = lookup.setdefault(kind, [])
+            if handlers:
+                bisect.insort(handlers, handler)
+            else:
+                handlers.append(handler)
+            added = True
+
+        if added:
+            # the handlers must work in an specific order, the order
+            # is specified in a Handler attribute
+            bisect.insort(self.handlers, handler)
+            handler.add_parent(self)
+
+    def close(self):
+        # Only exists for backwards compatibility.
+        pass
+
+    def _call_chain(self, chain, kind, meth_name, *args):
+        # Handlers raise an exception if no one else should try to handle
+        # the request, or return None if they can't but another handler
+        # could.  Otherwise, they return the response.
+        handlers = chain.get(kind, ())
+        for handler in handlers:
+            func = getattr(handler, meth_name)
+
+            result = func(*args)
+            if result is not None:
+                return result
+
+    def open(self, fullurl, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
+        # accept a URL or a Request object
+        if isinstance(fullurl, basestring):
+            req = Request(fullurl, data)
+        else:
+            req = fullurl
+            if data is not None:
+                req.add_data(data)
+
+        req.timeout = timeout
+        protocol = req.get_type()
+
+        # pre-process request
+        meth_name = protocol+"_request"
+        for processor in self.process_request.get(protocol, []):
+            meth = getattr(processor, meth_name)
+            req = meth(req)
+
+        response = self._open(req, data)
+
+        # post-process response
+        meth_name = protocol+"_response"
+        for processor in self.process_response.get(protocol, []):
+            meth = getattr(processor, meth_name)
+            try:
+                response = meth(req, response)
+            except:
+                response.close()
+                raise
+
+        return response
+
+    def _open(self, req, data=None):
+        result = self._call_chain(self.handle_open, 'default',
+                                  'default_open', req)
+        if result:
+            return result
+
+        protocol = req.get_type()
+        result = self._call_chain(self.handle_open, protocol, protocol +
+                                  '_open', req)
+        if result:
+            return result
+
+        return self._call_chain(self.handle_open, 'unknown',
+                                'unknown_open', req)
+
+    def error(self, proto, *args):
+        if proto in ('http', 'https'):
+            # XXX http[s] protocols are special-cased
+            dict = self.handle_error['http'] # https is not different than http
+            proto = args[2]  # YUCK!
+            meth_name = 'http_error_%s' % proto
+            http_err = 1
+            orig_args = args
+        else:
+            dict = self.handle_error
+            meth_name = proto + '_error'
+            http_err = 0
+        args = (dict, proto, meth_name) + args
+        result = self._call_chain(*args)
+        if result:
+            return result
+
+        if http_err:
+            args = (dict, 'default', 'http_error_default') + orig_args
+            return self._call_chain(*args)
+
+# XXX probably also want an abstract factory that knows when it makes
+# sense to skip a superclass in favor of a subclass and when it might
+# make sense to include both
+
+def build_opener(*handlers):
+    """Create an opener object from a list of handlers.
+
+    The opener will use several default handlers, including support
+    for HTTP, FTP and when applicable, HTTPS.
+
+    If any of the handlers passed as arguments are subclasses of the
+    default handlers, the default handlers will not be used.
+    """
+    import types
+    def isclass(obj):
+        return isinstance(obj, (types.ClassType, type))
+
+    opener = OpenerDirector()
+    default_classes = [ProxyHandler, UnknownHandler, HTTPHandler,
+                       HTTPDefaultErrorHandler, HTTPRedirectHandler,
+                       FTPHandler, FileHandler, HTTPErrorProcessor]
+    if hasattr(httplib, 'HTTPS'):
+        default_classes.append(HTTPSHandler)
+    skip = set()
+    for klass in default_classes:
+        for check in handlers:
+            if isclass(check):
+                if issubclass(check, klass):
+                    skip.add(klass)
+            elif isinstance(check, klass):
+                skip.add(klass)
+    for klass in skip:
+        default_classes.remove(klass)
+
+    for klass in default_classes:
+        opener.add_handler(klass())
+
+    for h in handlers:
+        if isclass(h):
+            h = h()
+        opener.add_handler(h)
+    return opener
+
+class BaseHandler:
+    handler_order = 500
+
+    def add_parent(self, parent):
+        self.parent = parent
+
+    def close(self):
+        # Only exists for backwards compatibility
+        pass
+
+    def __lt__(self, other):
+        if not hasattr(other, "handler_order"):
+            # Try to preserve the old behavior of having custom classes
+            # inserted after default ones (works only for custom user
+            # classes which are not aware of handler_order).
+            return True
+        return self.handler_order < other.handler_order
+
+
+class HTTPErrorProcessor(BaseHandler):
+    """Process HTTP error responses."""
+    handler_order = 1000  # after all other processing
+
+    def http_response(self, request, response):
+        code, msg, hdrs = response.code, response.msg, response.info()
+
+        # According to RFC 2616, "2xx" code indicates that the client's
+        # request was successfully received, understood, and accepted.
+        if not (200 <= code < 300):
+            response = self.parent.error(
+                'http', request, response, code, msg, hdrs)
+
+        return response
+
+    https_response = http_response
+
+class HTTPDefaultErrorHandler(BaseHandler):
+    def http_error_default(self, req, fp, code, msg, hdrs):
+        raise HTTPError(req.get_full_url(), code, msg, hdrs, fp)
+
+class HTTPRedirectHandler(BaseHandler):
+    # maximum number of redirections to any single URL
+    # this is needed because of the state that cookies introduce
+    max_repeats = 4
+    # maximum total number of redirections (regardless of URL) before
+    # assuming we're in a loop
+    max_redirections = 10
+
+    def redirect_request(self, req, fp, code, msg, headers, newurl):
+        """Return a Request or None in response to a redirect.
+
+        This is called by the http_error_30x methods when a
+        redirection response is received.  If a redirection should
+        take place, return a new Request to allow http_error_30x to
+        perform the redirect.  Otherwise, raise HTTPError if no-one
+        else should try to handle this url.  Return None if you can't
+        but another Handler might.
+        """
+        m = req.get_method()
+        if (code in (301, 302, 303, 307) and m in ("GET", "HEAD")
+            or code in (301, 302, 303) and m == "POST"):
+            # Strictly (according to RFC 2616), 301 or 302 in response
+            # to a POST MUST NOT cause a redirection without confirmation
+            # from the user (of urllib2, in this case).  In practice,
+            # essentially all clients do redirect in this case, so we
+            # do the same.
+            # be conciliant with URIs containing a space
+            newurl = newurl.replace(' ', '%20')
+            newheaders = dict((k,v) for k,v in req.headers.items()
+                              if k.lower() not in ("content-length", "content-type")
+                             )
+            return Request(newurl,
+                           headers=newheaders,
+                           origin_req_host=req.get_origin_req_host(),
+                           unverifiable=True)
+        else:
+            raise HTTPError(req.get_full_url(), code, msg, headers, fp)
+
+    # Implementation note: To avoid the server sending us into an
+    # infinite loop, the request object needs to track what URLs we
+    # have already seen.  Do this by adding a handler-specific
+    # attribute to the Request object.
+    def http_error_302(self, req, fp, code, msg, headers):
+        # Some servers (incorrectly) return multiple Location headers
+        # (so probably same goes for URI).  Use first header.
+        if 'location' in headers:
+            newurl = headers.getheaders('location')[0]
+        elif 'uri' in headers:
+            newurl = headers.getheaders('uri')[0]
+        else:
+            return
+
+        # fix a possible malformed URL
+        urlparts = urlparse.urlparse(newurl)
+        if not urlparts.path:
+            urlparts = list(urlparts)
+            urlparts[2] = "/"
+        newurl = urlparse.urlunparse(urlparts)
+
+        newurl = urlparse.urljoin(req.get_full_url(), newurl)
+
+        # XXX Probably want to forget about the state of the current
+        # request, although that might interact poorly with other
+        # handlers that also use handler-specific request attributes
+        new = self.redirect_request(req, fp, code, msg, headers, newurl)
+        if new is None:
+            return
+
+        # loop detection
+        # .redirect_dict has a key url if url was previously visited.
+        if hasattr(req, 'redirect_dict'):
+            visited = new.redirect_dict = req.redirect_dict
+            if (visited.get(newurl, 0) >= self.max_repeats or
+                len(visited) >= self.max_redirections):
+                raise HTTPError(req.get_full_url(), code,
+                                self.inf_msg + msg, headers, fp)
+        else:
+            visited = new.redirect_dict = req.redirect_dict = {}
+        visited[newurl] = visited.get(newurl, 0) + 1
+
+        # Don't close the fp until we are sure that we won't use it
+        # with HTTPError.
+        fp.read()
+        fp.close()
+
+        return self.parent.open(new, timeout=req.timeout)
+
+    http_error_301 = http_error_303 = http_error_307 = http_error_302
+
+    inf_msg = "The HTTP server returned a redirect error that would " \
+              "lead to an infinite loop.\n" \
+              "The last 30x error message was:\n"
+
+
+def _parse_proxy(proxy):
+    """Return (scheme, user, password, host/port) given a URL or an authority.
+
+    If a URL is supplied, it must have an authority (host:port) component.
+    According to RFC 3986, having an authority component means the URL must
+    have two slashes after the scheme:
+
+    >>> _parse_proxy('file:/ftp.example.com/')
+    Traceback (most recent call last):
+    ValueError: proxy URL with no authority: 'file:/ftp.example.com/'
+
+    The first three items of the returned tuple may be None.
+
+    Examples of authority parsing:
+
+    >>> _parse_proxy('proxy.example.com')
+    (None, None, None, 'proxy.example.com')
+    >>> _parse_proxy('proxy.example.com:3128')
+    (None, None, None, 'proxy.example.com:3128')
+
+    The authority component may optionally include userinfo (assumed to be
+    username:password):
+
+    >>> _parse_proxy('joe:password at proxy.example.com')
+    (None, 'joe', 'password', 'proxy.example.com')
+    >>> _parse_proxy('joe:password at proxy.example.com:3128')
+    (None, 'joe', 'password', 'proxy.example.com:3128')
+
+    Same examples, but with URLs instead:
+
+    >>> _parse_proxy('http://proxy.example.com/')
+    ('http', None, None, 'proxy.example.com')
+    >>> _parse_proxy('http://proxy.example.com:3128/')
+    ('http', None, None, 'proxy.example.com:3128')
+    >>> _parse_proxy('http://joe:password at proxy.example.com/')
+    ('http', 'joe', 'password', 'proxy.example.com')
+    >>> _parse_proxy('http://joe:password at proxy.example.com:3128')
+    ('http', 'joe', 'password', 'proxy.example.com:3128')
+
+    Everything after the authority is ignored:
+
+    >>> _parse_proxy('ftp://joe:password at proxy.example.com/rubbish:3128')
+    ('ftp', 'joe', 'password', 'proxy.example.com')
+
+    Test for no trailing '/' case:
+
+    >>> _parse_proxy('http://joe:password at proxy.example.com')
+    ('http', 'joe', 'password', 'proxy.example.com')
+
+    """
+    scheme, r_scheme = splittype(proxy)
+    if not r_scheme.startswith("/"):
+        # authority
+        scheme = None
+        authority = proxy
+    else:
+        # URL
+        if not r_scheme.startswith("//"):
+            raise ValueError("proxy URL with no authority: %r" % proxy)
+        # We have an authority, so for RFC 3986-compliant URLs (by ss 3.
+        # and 3.3.), path is empty or starts with '/'
+        end = r_scheme.find("/", 2)
+        if end == -1:
+            end = None
+        authority = r_scheme[2:end]
+    userinfo, hostport = splituser(authority)
+    if userinfo is not None:
+        user, password = splitpasswd(userinfo)
+    else:
+        user = password = None
+    return scheme, user, password, hostport
+
+class ProxyHandler(BaseHandler):
+    # Proxies must be in front
+    handler_order = 100
+
+    def __init__(self, proxies=None):
+        if proxies is None:
+            proxies = getproxies()
+        assert hasattr(proxies, 'has_key'), "proxies must be a mapping"
+        self.proxies = proxies
+        for type, url in proxies.items():
+            setattr(self, '%s_open' % type,
+                    lambda r, proxy=url, type=type, meth=self.proxy_open: \
+                    meth(r, proxy, type))
+
+    def proxy_open(self, req, proxy, type):
+        orig_type = req.get_type()
+        proxy_type, user, password, hostport = _parse_proxy(proxy)
+
+        if proxy_type is None:
+            proxy_type = orig_type
+
+        if req.host and proxy_bypass(req.host):
+            return None
+
+        if user and password:
+            user_pass = '%s:%s' % (unquote(user), unquote(password))
+            creds = base64.b64encode(user_pass).strip()
+            req.add_header('Proxy-authorization', 'Basic ' + creds)
+        hostport = unquote(hostport)
+        req.set_proxy(hostport, proxy_type)
+
+        if orig_type == proxy_type or orig_type == 'https':
+            # let other handlers take care of it
+            return None
+        else:
+            # need to start over, because the other handlers don't
+            # grok the proxy's URL type
+            # e.g. if we have a constructor arg proxies like so:
+            # {'http': 'ftp://proxy.example.com'}, we may end up turning
+            # a request for http://acme.example.com/a into one for
+            # ftp://proxy.example.com/a
+            return self.parent.open(req, timeout=req.timeout)
+
+class HTTPPasswordMgr:
+
+    def __init__(self):
+        self.passwd = {}
+
+    def add_password(self, realm, uri, user, passwd):
+        # uri could be a single URI or a sequence
+        if isinstance(uri, basestring):
+            uri = [uri]
+        if not realm in self.passwd:
+            self.passwd[realm] = {}
+        for default_port in True, False:
+            reduced_uri = tuple(
+                [self.reduce_uri(u, default_port) for u in uri])
+            self.passwd[realm][reduced_uri] = (user, passwd)
+
+    def find_user_password(self, realm, authuri):
+        domains = self.passwd.get(realm, {})
+        for default_port in True, False:
+            reduced_authuri = self.reduce_uri(authuri, default_port)
+            for uris, authinfo in domains.iteritems():
+                for uri in uris:
+                    if self.is_suburi(uri, reduced_authuri):
+                        return authinfo
+        return None, None
+
+    def reduce_uri(self, uri, default_port=True):
+        """Accept authority or URI and extract only the authority and path."""
+        # note HTTP URLs do not have a userinfo component
+        parts = urlparse.urlsplit(uri)
+        if parts[1]:
+            # URI
+            scheme = parts[0]
+            authority = parts[1]
+            path = parts[2] or '/'
+        else:
+            # host or host:port
+            scheme = None
+            authority = uri
+            path = '/'
+        host, port = splitport(authority)
+        if default_port and port is None and scheme is not None:
+            dport = {"http": 80,
+                     "https": 443,
+                     }.get(scheme)
+            if dport is not None:
+                authority = "%s:%d" % (host, dport)
+        return authority, path
+
+    def is_suburi(self, base, test):
+        """Check if test is below base in a URI tree
+
+        Both args must be URIs in reduced form.
+        """
+        if base == test:
+            return True
+        if base[0] != test[0]:
+            return False
+        common = posixpath.commonprefix((base[1], test[1]))
+        if len(common) == len(base[1]):
+            return True
+        return False
+
+
+class HTTPPasswordMgrWithDefaultRealm(HTTPPasswordMgr):
+
+    def find_user_password(self, realm, authuri):
+        user, password = HTTPPasswordMgr.find_user_password(self, realm,
+                                                            authuri)
+        if user is not None:
+            return user, password
+        return HTTPPasswordMgr.find_user_password(self, None, authuri)
+
+
+class AbstractBasicAuthHandler:
+
+    # XXX this allows for multiple auth-schemes, but will stupidly pick
+    # the last one with a realm specified.
+
+    # allow for double- and single-quoted realm values
+    # (single quotes are a violation of the RFC, but appear in the wild)
+    rx = re.compile('(?:.*,)*[ \t]*([^ \t]+)[ \t]+'
+                    'realm=(["\'])(.*?)\\2', re.I)
+
+    # XXX could pre-emptively send auth info already accepted (RFC 2617,
+    # end of section 2, and section 1.2 immediately after "credentials"
+    # production).
+
+    def __init__(self, password_mgr=None):
+        if password_mgr is None:
+            password_mgr = HTTPPasswordMgr()
+        self.passwd = password_mgr
+        self.add_password = self.passwd.add_password
+        self.retried = 0
+
+    def reset_retry_count(self):
+        self.retried = 0
+
+    def http_error_auth_reqed(self, authreq, host, req, headers):
+        # host may be an authority (without userinfo) or a URL with an
+        # authority
+        # XXX could be multiple headers
+        authreq = headers.get(authreq, None)
+
+        if self.retried > 5:
+            # retry sending the username:password 5 times before failing.
+            raise HTTPError(req.get_full_url(), 401, "basic auth failed",
+                            headers, None)
+        else:
+            self.retried += 1
+
+        if authreq:
+            mo = AbstractBasicAuthHandler.rx.search(authreq)
+            if mo:
+                scheme, quote, realm = mo.groups()
+                if scheme.lower() == 'basic':
+                    response = self.retry_http_basic_auth(host, req, realm)
+                    if response and response.code != 401:
+                        self.retried = 0
+                    return response
+
+    def retry_http_basic_auth(self, host, req, realm):
+        user, pw = self.passwd.find_user_password(realm, host)
+        if pw is not None:
+            raw = "%s:%s" % (user, pw)
+            auth = 'Basic %s' % base64.b64encode(raw).strip()
+            if req.headers.get(self.auth_header, None) == auth:
+                return None
+            req.add_unredirected_header(self.auth_header, auth)
+            return self.parent.open(req, timeout=req.timeout)
+        else:
+            return None
+
+
+class HTTPBasicAuthHandler(AbstractBasicAuthHandler, BaseHandler):
+
+    auth_header = 'Authorization'
+
+    def http_error_401(self, req, fp, code, msg, headers):
+        url = req.get_full_url()
+        response = self.http_error_auth_reqed('www-authenticate',
+                                              url, req, headers)
+        self.reset_retry_count()
+        return response
+
+
+class ProxyBasicAuthHandler(AbstractBasicAuthHandler, BaseHandler):
+
+    auth_header = 'Proxy-authorization'
+
+    def http_error_407(self, req, fp, code, msg, headers):
+        # http_error_auth_reqed requires that there is no userinfo component in
+        # authority.  Assume there isn't one, since urllib2 does not (and
+        # should not, RFC 3986 s. 3.2.1) support requests for URLs containing
+        # userinfo.
+        authority = req.get_host()
+        response = self.http_error_auth_reqed('proxy-authenticate',
+                                          authority, req, headers)
+        self.reset_retry_count()
+        return response
+
+
+def randombytes(n):
+    """Return n random bytes."""
+    # Use /dev/urandom if it is available.  Fall back to random module
+    # if not.  It might be worthwhile to extend this function to use
+    # other platform-specific mechanisms for getting random bytes.
+    if os.path.exists("/dev/urandom"):
+        f = open("/dev/urandom")
+        s = f.read(n)
+        f.close()
+        return s
+    else:
+        L = [chr(random.randrange(0, 256)) for i in range(n)]
+        return "".join(L)
+
+class AbstractDigestAuthHandler:
+    # Digest authentication is specified in RFC 2617.
+
+    # XXX The client does not inspect the Authentication-Info header
+    # in a successful response.
+
+    # XXX It should be possible to test this implementation against
+    # a mock server that just generates a static set of challenges.
+
+    # XXX qop="auth-int" supports is shaky
+
+    def __init__(self, passwd=None):
+        if passwd is None:
+            passwd = HTTPPasswordMgr()
+        self.passwd = passwd
+        self.add_password = self.passwd.add_password
+        self.retried = 0
+        self.nonce_count = 0
+        self.last_nonce = None
+
+    def reset_retry_count(self):
+        self.retried = 0
+
+    def http_error_auth_reqed(self, auth_header, host, req, headers):
+        authreq = headers.get(auth_header, None)
+        if self.retried > 5:
+            # Don't fail endlessly - if we failed once, we'll probably
+            # fail a second time. Hm. Unless the Password Manager is
+            # prompting for the information. Crap. This isn't great
+            # but it's better than the current 'repeat until recursion
+            # depth exceeded' approach <wink>
+            raise HTTPError(req.get_full_url(), 401, "digest auth failed",
+                            headers, None)
+        else:
+            self.retried += 1
+        if authreq:
+            scheme = authreq.split()[0]
+            if scheme.lower() == 'digest':
+                return self.retry_http_digest_auth(req, authreq)
+
+    def retry_http_digest_auth(self, req, auth):
+        token, challenge = auth.split(' ', 1)
+        chal = parse_keqv_list(parse_http_list(challenge))
+        auth = self.get_authorization(req, chal)
+        if auth:
+            auth_val = 'Digest %s' % auth
+            if req.headers.get(self.auth_header, None) == auth_val:
+                return None
+            req.add_unredirected_header(self.auth_header, auth_val)
+            resp = self.parent.open(req, timeout=req.timeout)
+            return resp
+
+    def get_cnonce(self, nonce):
+        # The cnonce-value is an opaque
+        # quoted string value provided by the client and used by both client
+        # and server to avoid chosen plaintext attacks, to provide mutual
+        # authentication, and to provide some message integrity protection.
+        # This isn't a fabulous effort, but it's probably Good Enough.
+        dig = hashlib.sha1("%s:%s:%s:%s" % (self.nonce_count, nonce, time.ctime(),
+                                            randombytes(8))).hexdigest()
+        return dig[:16]
+
+    def get_authorization(self, req, chal):
+        try:
+            realm = chal['realm']
+            nonce = chal['nonce']
+            qop = chal.get('qop')
+            algorithm = chal.get('algorithm', 'MD5')
+            # mod_digest doesn't send an opaque, even though it isn't
+            # supposed to be optional
+            opaque = chal.get('opaque', None)
+        except KeyError:
+            return None
+
+        H, KD = self.get_algorithm_impls(algorithm)
+        if H is None:
+            return None
+
+        user, pw = self.passwd.find_user_password(realm, req.get_full_url())
+        if user is None:
+            return None
+
+        # XXX not implemented yet
+        if req.has_data():
+            entdig = self.get_entity_digest(req.get_data(), chal)
+        else:
+            entdig = None
+
+        A1 = "%s:%s:%s" % (user, realm, pw)
+        A2 = "%s:%s" % (req.get_method(),
+                        # XXX selector: what about proxies and full urls
+                        req.get_selector())
+        if qop == 'auth':
+            if nonce == self.last_nonce:
+                self.nonce_count += 1
+            else:
+                self.nonce_count = 1
+                self.last_nonce = nonce
+
+            ncvalue = '%08x' % self.nonce_count
+            cnonce = self.get_cnonce(nonce)
+            noncebit = "%s:%s:%s:%s:%s" % (nonce, ncvalue, cnonce, qop, H(A2))
+            respdig = KD(H(A1), noncebit)
+        elif qop is None:
+            respdig = KD(H(A1), "%s:%s" % (nonce, H(A2)))
+        else:
+            # XXX handle auth-int.
+            raise URLError("qop '%s' is not supported." % qop)
+
+        # XXX should the partial digests be encoded too?
+
+        base = 'username="%s", realm="%s", nonce="%s", uri="%s", ' \
+               'response="%s"' % (user, realm, nonce, req.get_selector(),
+                                  respdig)
+        if opaque:
+            base += ', opaque="%s"' % opaque
+        if entdig:
+            base += ', digest="%s"' % entdig
+        base += ', algorithm="%s"' % algorithm
+        if qop:
+            base += ', qop=auth, nc=%s, cnonce="%s"' % (ncvalue, cnonce)
+        return base
+
+    def get_algorithm_impls(self, algorithm):
+        # algorithm should be case-insensitive according to RFC2617
+        algorithm = algorithm.upper()
+        # lambdas assume digest modules are imported at the top level
+        if algorithm == 'MD5':
+            H = lambda x: hashlib.md5(x).hexdigest()
+        elif algorithm == 'SHA':
+            H = lambda x: hashlib.sha1(x).hexdigest()
+        # XXX MD5-sess
+        KD = lambda s, d: H("%s:%s" % (s, d))
+        return H, KD
+
+    def get_entity_digest(self, data, chal):
+        # XXX not implemented yet
+        return None
+
+
+class HTTPDigestAuthHandler(BaseHandler, AbstractDigestAuthHandler):
+    """An authentication protocol defined by RFC 2069
+
+    Digest authentication improves on basic authentication because it
+    does not transmit passwords in the clear.
+    """
+
+    auth_header = 'Authorization'
+    handler_order = 490  # before Basic auth
+
+    def http_error_401(self, req, fp, code, msg, headers):
+        host = urlparse.urlparse(req.get_full_url())[1]
+        retry = self.http_error_auth_reqed('www-authenticate',
+                                           host, req, headers)
+        self.reset_retry_count()
+        return retry
+
+
+class ProxyDigestAuthHandler(BaseHandler, AbstractDigestAuthHandler):
+
+    auth_header = 'Proxy-Authorization'
+    handler_order = 490  # before Basic auth
+
+    def http_error_407(self, req, fp, code, msg, headers):
+        host = req.get_host()
+        retry = self.http_error_auth_reqed('proxy-authenticate',
+                                           host, req, headers)
+        self.reset_retry_count()
+        return retry
+
+class AbstractHTTPHandler(BaseHandler):
+
+    def __init__(self, debuglevel=0):
+        self._debuglevel = debuglevel
+
+    def set_http_debuglevel(self, level):
+        self._debuglevel = level
+
+    def do_request_(self, request):
+        host = request.get_host()
+        if not host:
+            raise URLError('no host given')
+
+        if request.has_data():  # POST
+            data = request.get_data()
+            if not request.has_header('Content-type'):
+                request.add_unredirected_header(
+                    'Content-type',
+                    'application/x-www-form-urlencoded')
+            if not request.has_header('Content-length'):
+                request.add_unredirected_header(
+                    'Content-length', '%d' % len(data))
+
+        sel_host = host
+        if request.has_proxy():
+            scheme, sel = splittype(request.get_selector())
+            sel_host, sel_path = splithost(sel)
+
+        if not request.has_header('Host'):
+            request.add_unredirected_header('Host', sel_host)
+        for name, value in self.parent.addheaders:
+            name = name.capitalize()
+            if not request.has_header(name):
+                request.add_unredirected_header(name, value)
+
+        return request
+
+    def do_open(self, http_class, req):
+        """Return an addinfourl object for the request, using http_class.
+
+        http_class must implement the HTTPConnection API from httplib.
+        The addinfourl return value is a file-like object.  It also
+        has methods and attributes including:
+            - info(): return a mimetools.Message object for the headers
+            - geturl(): return the original request URL
+            - code: HTTP status code
+        """
+        host = req.get_host()
+        if not host:
+            raise URLError('no host given')
+
+        h = http_class(host, timeout=req.timeout) # will parse host:port
+        h.set_debuglevel(self._debuglevel)
+
+        headers = dict(req.unredirected_hdrs)
+        headers.update(dict((k, v) for k, v in req.headers.items()
+                            if k not in headers))
+
+        # We want to make an HTTP/1.1 request, but the addinfourl
+        # class isn't prepared to deal with a persistent connection.
+        # It will try to read all remaining data from the socket,
+        # which will block while the server waits for the next request.
+        # So make sure the connection gets closed after the (only)
+        # request.
+        headers["Connection"] = "close"
+        headers = dict(
+            (name.title(), val) for name, val in headers.items())
+
+        if req._tunnel_host:
+            tunnel_headers = {}
+            proxy_auth_hdr = "Proxy-Authorization"
+            if proxy_auth_hdr in headers:
+                tunnel_headers[proxy_auth_hdr] = headers[proxy_auth_hdr]
+                # Proxy-Authorization should not be sent to origin
+                # server.
+                del headers[proxy_auth_hdr]
+            h.set_tunnel(req._tunnel_host, headers=tunnel_headers)
+
+        try:
+            h.request(req.get_method(), req.get_selector(), req.data, headers)
+            try:
+                r = h.getresponse(buffering=True)
+            except TypeError: #buffering kw not supported
+                r = h.getresponse()
+        except socket.error, err: # XXX what error?
+            h.close()
+            raise URLError(err)
+
+        # Pick apart the HTTPResponse object to get the addinfourl
+        # object initialized properly.
+
+        # Wrap the HTTPResponse object in socket's file object adapter
+        # for Windows.  That adapter calls recv(), so delegate recv()
+        # to read().  This weird wrapping allows the returned object to
+        # have readline() and readlines() methods.
+
+        # XXX It might be better to extract the read buffering code
+        # out of socket._fileobject() and into a base class.
+
+        r.recv = r.read
+        fp = socket._fileobject(r, close=True)
+
+        resp = addinfourl(fp, r.msg, req.get_full_url())
+        resp.code = r.status
+        resp.msg = r.reason
+        return resp
+
+
+class HTTPHandler(AbstractHTTPHandler):
+
+    def http_open(self, req):
+        return self.do_open(httplib.HTTPConnection, req)
+
+    http_request = AbstractHTTPHandler.do_request_
+
+if hasattr(httplib, 'HTTPS'):
+    class HTTPSHandler(AbstractHTTPHandler):
+
+        def https_open(self, req):
+            return self.do_open(httplib.HTTPSConnection, req)
+
+        https_request = AbstractHTTPHandler.do_request_
+
+class HTTPCookieProcessor(BaseHandler):
+    def __init__(self, cookiejar=None):
+        import cookielib
+        if cookiejar is None:
+            cookiejar = cookielib.CookieJar()
+        self.cookiejar = cookiejar
+
+    def http_request(self, request):
+        self.cookiejar.add_cookie_header(request)
+        return request
+
+    def http_response(self, request, response):
+        self.cookiejar.extract_cookies(response, request)
+        return response
+
+    https_request = http_request
+    https_response = http_response
+
+class UnknownHandler(BaseHandler):
+    def unknown_open(self, req):
+        type = req.get_type()
+        raise URLError('unknown url type: %s' % type)
+
+def parse_keqv_list(l):
+    """Parse list of key=value strings where keys are not duplicated."""
+    parsed = {}
+    for elt in l:
+        k, v = elt.split('=', 1)
+        if v[0] == '"' and v[-1] == '"':
+            v = v[1:-1]
+        parsed[k] = v
+    return parsed
+
+def parse_http_list(s):
+    """Parse lists as described by RFC 2068 Section 2.
+
+    In particular, parse comma-separated lists where the elements of
+    the list may include quoted-strings.  A quoted-string could
+    contain a comma.  A non-quoted string could have quotes in the
+    middle.  Neither commas nor quotes count if they are escaped.
+    Only double-quotes count, not single-quotes.
+    """
+    res = []
+    part = ''
+
+    escape = quote = False
+    for cur in s:
+        if escape:
+            part += cur
+            escape = False
+            continue
+        if quote:
+            if cur == '\\':
+                escape = True
+                continue
+            elif cur == '"':
+                quote = False
+            part += cur
+            continue
+
+        if cur == ',':
+            res.append(part)
+            part = ''
+            continue
+
+        if cur == '"':
+            quote = True
+
+        part += cur
+
+    # append last part
+    if part:
+        res.append(part)
+
+    return [part.strip() for part in res]
+
+def _safe_gethostbyname(host):
+    try:
+        return socket.gethostbyname(host)
+    except socket.gaierror:
+        return None
+
+class FileHandler(BaseHandler):
+    # Use local file or FTP depending on form of URL
+    def file_open(self, req):
+        url = req.get_selector()
+        if url[:2] == '//' and url[2:3] != '/' and (req.host and
+                req.host != 'localhost'):
+            req.type = 'ftp'
+            return self.parent.open(req)
+        else:
+            return self.open_local_file(req)
+
+    # names for the localhost
+    names = None
+    def get_names(self):
+        if FileHandler.names is None:
+            try:
+                FileHandler.names = tuple(
+                    socket.gethostbyname_ex('localhost')[2] +
+                    socket.gethostbyname_ex(socket.gethostname())[2])
+            except socket.gaierror:
+                FileHandler.names = (socket.gethostbyname('localhost'),)
+        return FileHandler.names
+
+    # not entirely sure what the rules are here
+    def open_local_file(self, req):
+        import email.utils
+        import mimetypes
+        host = req.get_host()
+        filename = req.get_selector()
+        localfile = url2pathname(filename)
+        try:
+            stats = os.stat(localfile)
+            size = stats.st_size
+            modified = email.utils.formatdate(stats.st_mtime, usegmt=True)
+            mtype = mimetypes.guess_type(filename)[0]
+            headers = mimetools.Message(StringIO(
+                'Content-type: %s\nContent-length: %d\nLast-modified: %s\n' %
+                (mtype or 'text/plain', size, modified)))
+            if host:
+                host, port = splitport(host)
+            if not host or \
+                (not port and _safe_gethostbyname(host) in self.get_names()):
+                if host:
+                    origurl = 'file://' + host + filename
+                else:
+                    origurl = 'file://' + filename
+                return addinfourl(open(localfile, 'rb'), headers, origurl)
+        except OSError, msg:
+            # urllib2 users shouldn't expect OSErrors coming from urlopen()
+            raise URLError(msg)
+        raise URLError('file not on local host')
+
+class FTPHandler(BaseHandler):
+    def ftp_open(self, req):
+        import ftplib
+        import mimetypes
+        host = req.get_host()
+        if not host:
+            raise URLError('ftp error: no host given')
+        host, port = splitport(host)
+        if port is None:
+            port = ftplib.FTP_PORT
+        else:
+            port = int(port)
+
+        # username/password handling
+        user, host = splituser(host)
+        if user:
+            user, passwd = splitpasswd(user)
+        else:
+            passwd = None
+        host = unquote(host)
+        user = user or ''
+        passwd = passwd or ''
+
+        try:
+            host = socket.gethostbyname(host)
+        except socket.error, msg:
+            raise URLError(msg)
+        path, attrs = splitattr(req.get_selector())
+        dirs = path.split('/')
+        dirs = map(unquote, dirs)
+        dirs, file = dirs[:-1], dirs[-1]
+        if dirs and not dirs[0]:
+            dirs = dirs[1:]
+        try:
+            fw = self.connect_ftp(user, passwd, host, port, dirs, req.timeout)
+            type = file and 'I' or 'D'
+            for attr in attrs:
+                attr, value = splitvalue(attr)
+                if attr.lower() == 'type' and \
+                   value in ('a', 'A', 'i', 'I', 'd', 'D'):
+                    type = value.upper()
+            fp, retrlen = fw.retrfile(file, type)
+            headers = ""
+            mtype = mimetypes.guess_type(req.get_full_url())[0]
+            if mtype:
+                headers += "Content-type: %s\n" % mtype
+            if retrlen is not None and retrlen >= 0:
+                headers += "Content-length: %d\n" % retrlen
+            sf = StringIO(headers)
+            headers = mimetools.Message(sf)
+            return addinfourl(fp, headers, req.get_full_url())
+        except ftplib.all_errors, msg:
+            raise URLError, ('ftp error: %s' % msg), sys.exc_info()[2]
+
+    def connect_ftp(self, user, passwd, host, port, dirs, timeout):
+        fw = ftpwrapper(user, passwd, host, port, dirs, timeout)
+##        fw.ftp.set_debuglevel(1)
+        return fw
+
+class CacheFTPHandler(FTPHandler):
+    # XXX would be nice to have pluggable cache strategies
+    # XXX this stuff is definitely not thread safe
+    def __init__(self):
+        self.cache = {}
+        self.timeout = {}
+        self.soonest = 0
+        self.delay = 60
+        self.max_conns = 16
+
+    def setTimeout(self, t):
+        self.delay = t
+
+    def setMaxConns(self, m):
+        self.max_conns = m
+
+    def connect_ftp(self, user, passwd, host, port, dirs, timeout):
+        key = user, host, port, '/'.join(dirs), timeout
+        if key in self.cache:
+            self.timeout[key] = time.time() + self.delay
+        else:
+            self.cache[key] = ftpwrapper(user, passwd, host, port, dirs, timeout)
+            self.timeout[key] = time.time() + self.delay
+        self.check_cache()
+        return self.cache[key]
+
+    def check_cache(self):
+        # first check for old ones
+        t = time.time()
+        if self.soonest <= t:
+            for k, v in self.timeout.items():
+                if v < t:
+                    self.cache[k].close()
+                    del self.cache[k]
+                    del self.timeout[k]
+        self.soonest = min(self.timeout.values())
+
+        # then check the size
+        if len(self.cache) == self.max_conns:
+            for k, v in self.timeout.items():
+                if v == self.soonest:
+                    del self.cache[k]
+                    del self.timeout[k]
+                    break
+            self.soonest = min(self.timeout.values())
diff --git a/lib_pypy/resource.py b/lib_pypy/resource.py
--- a/lib_pypy/resource.py
+++ b/lib_pypy/resource.py
@@ -7,7 +7,7 @@
 
 from ctypes_support import standard_c_lib as libc
 from ctypes_support import get_errno
-from ctypes import Structure, c_int, c_long, byref, sizeof, POINTER
+from ctypes import Structure, c_int, c_long, byref, POINTER
 from errno import EINVAL, EPERM
 import _structseq
 
@@ -165,7 +165,6 @@
 
 @builtinify
 def getpagesize():
-    pagesize = 0
     if _getpagesize:
         return _getpagesize()
     else:
diff --git a/pypy/annotation/classdef.py b/pypy/annotation/classdef.py
--- a/pypy/annotation/classdef.py
+++ b/pypy/annotation/classdef.py
@@ -276,8 +276,8 @@
         # create the Attribute and do the generalization asked for
         newattr = Attribute(attr, self.bookkeeper)
         if s_value:
-            if newattr.name == 'intval' and getattr(s_value, 'unsigned', False):
-                import pdb; pdb.set_trace()
+            #if newattr.name == 'intval' and getattr(s_value, 'unsigned', False):
+            #    import pdb; pdb.set_trace()
             newattr.s_value = s_value
 
         # keep all subattributes' values
diff --git a/pypy/annotation/test/test_annrpython.py b/pypy/annotation/test/test_annrpython.py
--- a/pypy/annotation/test/test_annrpython.py
+++ b/pypy/annotation/test/test_annrpython.py
@@ -3204,6 +3204,8 @@
         s = a.build_types(f, [])
         assert isinstance(s, annmodel.SomeList)
         assert not s.listdef.listitem.resized
+        assert not s.listdef.listitem.immutable
+        assert s.listdef.listitem.mutated
 
     def test_delslice(self):
         def f():
diff --git a/pypy/annotation/unaryop.py b/pypy/annotation/unaryop.py
--- a/pypy/annotation/unaryop.py
+++ b/pypy/annotation/unaryop.py
@@ -352,6 +352,7 @@
         check_negative_slice(s_start, s_stop)
         if not isinstance(s_iterable, SomeList):
             raise Exception("list[start:stop] = x: x must be a list")
+        lst.listdef.mutate()
         lst.listdef.agree(s_iterable.listdef)
         # note that setslice is not allowed to resize a list in RPython
 
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -127,7 +127,7 @@
 
 pypy_optiondescription = OptionDescription("objspace", "Object Space Options", [
     ChoiceOption("name", "Object Space name",
-                 ["std", "flow", "thunk", "dump", "taint"],
+                 ["std", "flow", "thunk", "dump"],
                  "std",
                  cmdline='--objspace -o'),
 
diff --git a/pypy/doc/__pypy__-module.rst b/pypy/doc/__pypy__-module.rst
--- a/pypy/doc/__pypy__-module.rst
+++ b/pypy/doc/__pypy__-module.rst
@@ -37,29 +37,6 @@
 .. _`thunk object space docs`: objspace-proxies.html#thunk
 .. _`interface section of the thunk object space docs`: objspace-proxies.html#thunk-interface
 
-.. broken:
-
-    Taint Object Space Functionality
-    ================================
-
-    When the taint object space is used (choose with :config:`objspace.name`),
-    the following names are put into ``__pypy__``:
-
-     - ``taint``
-     - ``is_tainted``
-     - ``untaint``
-     - ``taint_atomic``
-     - ``_taint_debug``
-     - ``_taint_look``
-     - ``TaintError``
-
-    Those are all described in the `interface section of the taint object space
-    docs`_.
-
-    For more detailed explanations and examples see the `taint object space docs`_.
-
-    .. _`taint object space docs`: objspace-proxies.html#taint
-    .. _`interface section of the taint object space docs`: objspace-proxies.html#taint-interface
 
 Transparent Proxy Functionality
 ===============================
diff --git a/pypy/doc/config/objspace.name.txt b/pypy/doc/config/objspace.name.txt
--- a/pypy/doc/config/objspace.name.txt
+++ b/pypy/doc/config/objspace.name.txt
@@ -4,7 +4,6 @@
 for normal usage):
 
   * thunk_: The thunk object space adds lazy evaluation to PyPy.
-  * taint_: The taint object space adds soft security features.
   * dump_:  Using this object spaces results in the dumpimp of all operations
     to a log.
 
@@ -12,5 +11,4 @@
 .. _`Object Space Proxies`: ../objspace-proxies.html
 .. _`Standard Object Space`: ../objspace.html#standard-object-space
 .. _thunk: ../objspace-proxies.html#thunk
-.. _taint: ../objspace-proxies.html#taint
 .. _dump: ../objspace-proxies.html#dump
diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst
--- a/pypy/doc/index.rst
+++ b/pypy/doc/index.rst
@@ -309,7 +309,6 @@
 .. _`object space`: objspace.html
 .. _FlowObjSpace: objspace.html#the-flow-object-space 
 .. _`trace object space`: objspace.html#the-trace-object-space 
-.. _`taint object space`: objspace-proxies.html#taint
 .. _`thunk object space`: objspace-proxies.html#thunk
 .. _`transparent proxies`: objspace-proxies.html#tproxy
 .. _`Differences between PyPy and CPython`: cpython_differences.html
diff --git a/pypy/doc/objspace-proxies.rst b/pypy/doc/objspace-proxies.rst
--- a/pypy/doc/objspace-proxies.rst
+++ b/pypy/doc/objspace-proxies.rst
@@ -129,297 +129,6 @@
    function behaves lazily: all calls to it return a thunk object.
 
 
-.. broken right now:
-
-    .. _taint:
-
-    The Taint Object Space
-    ======================
-
-    Motivation
-    ----------
-
-    The Taint Object Space provides a form of security: "tainted objects",
-    inspired by various sources, see [D12.1]_ for a more detailed discussion. 
-
-    The basic idea of this kind of security is not to protect against
-    malicious code but to help with handling and boxing sensitive data. 
-    It covers two kinds of sensitive data: secret data which should not leak, 
-    and untrusted data coming from an external source and that must be 
-    validated before it is used.
-
-    The idea is that, considering a large application that handles these
-    kinds of sensitive data, there are typically only a small number of
-    places that need to explicitly manipulate that sensitive data; all the
-    other places merely pass it around, or do entirely unrelated things.
-
-    Nevertheless, if a large application needs to be reviewed for security,
-    it must be entirely carefully checked, because it is possible that a
-    bug at some apparently unrelated place could lead to a leak of sensitive
-    information in a way that an external attacker could exploit.  For
-    example, if any part of the application provides web services, an
-    attacker might be able to issue unexpected requests with a regular web
-    browser and deduce secret information from the details of the answers he
-    gets.  Another example is the common CGI attack where an attacker sends
-    malformed inputs and causes the CGI script to do unintended things.
-
-    An approach like that of the Taint Object Space allows the small parts
-    of the program that manipulate sensitive data to be explicitly marked.
-    The effect of this is that although these small parts still need a
-    careful security review, the rest of the application no longer does,
-    because even a bug would be unable to leak the information.
-
-    We have implemented a simple two-level model: objects are either
-    regular (untainted), or sensitive (tainted).  Objects are marked as
-    sensitive if they are secret or untrusted, and only declassified at
-    carefully-checked positions (e.g. where the secret data is needed, or
-    after the untrusted data has been fully validated).
-
-    It would be simple to extend the code for more fine-grained scales of
-    secrecy.  For example it is typical in the literature to consider
-    user-specified lattices of secrecy levels, corresponding to multiple
-    "owners" that cannot access data belonging to another "owner" unless
-    explicitly authorized to do so.
-
-    Tainting and untainting
-    -----------------------
-
-    Start a py.py with the Taint Object Space and try the following example::
-
-        $ py.py -o taint
-        >>>> from __pypy__ import taint
-        >>>> x = taint(6)
-
-        # x is hidden from now on.  We can pass it around and
-        # even operate on it, but not inspect it.  Taintness
-        # is propagated to operation results.
-
-        >>>> x
-        TaintError
-
-        >>>> if x > 5: y = 2   # see below
-        TaintError
-
-        >>>> y = x + 5         # ok
-        >>>> lst = [x, y]
-        >>>> z = lst.pop()
-        >>>> t = type(z)       # type() works too, tainted answer
-        >>>> t
-        TaintError
-        >>>> u = t is int      # even 'is' works
-        >>>> u
-        TaintError
-
-    Notice that using a tainted boolean like ``x > 5`` in an ``if``
-    statement is forbidden.  This is because knowing which path is followed
-    would give away a hint about ``x``; in the example above, if the
-    statement ``if x > 5: y = 2`` was allowed to run, we would know
-    something about the value of ``x`` by looking at the (untainted) value
-    in the variable ``y``.
-
-    Of course, there is a way to inspect tainted objects.  The basic way is
-    to explicitly "declassify" it with the ``untaint()`` function.  In an
-    application, the places that use ``untaint()`` are the places that need
-    careful security review.  To avoid unexpected objects showing up, the
-    ``untaint()`` function must be called with the exact type of the object
-    to declassify.  It will raise ``TaintError`` if the type doesn't match::
-
-        >>>> from __pypy__ import taint
-        >>>> untaint(int, x)
-        6
-        >>>> untaint(int, z)
-        11
-        >>>> untaint(bool, x > 5)
-        True
-        >>>> untaint(int, x > 5)
-        TaintError
-
-
-    Taint Bombs
-    -----------
-
-    In this area, a common problem is what to do about failing operations.
-    If an operation raises an exception when manipulating a tainted object,
-    then the very presence of the exception can leak information about the
-    tainted object itself.  Consider::
-
-        >>>> 5 / (x-6)
-
-    By checking if this raises ``ZeroDivisionError`` or not, we would know
-    if ``x`` was equal to 6 or not.  The solution to this problem in the
-    Taint Object Space is to introduce *Taint Bombs*.  They are a kind of
-    tainted object that doesn't contain a real object, but a pending
-    exception.  Taint Bombs are indistinguishable from normal tainted
-    objects to unprivileged code. See::
-
-        >>>> x = taint(6)
-        >>>> i = 5 / (x-6)     # no exception here
-        >>>> j = i + 1         # nor here
-        >>>> k = j + 5         # nor here
-        >>>> untaint(int, k)
-        TaintError
-
-    In the above example, all of ``i``, ``j`` and ``k`` contain a Taint
-    Bomb.  Trying to untaint it raises an exception - a generic
-    ``TaintError``.  What we win is that the exception gives little away,
-    and most importantly it occurs at the point where ``untaint()`` is
-    called, not where the operation failed.  This means that all calls to
-    ``untaint()`` - but not the rest of the code - must be carefully
-    reviewed for what occurs if they receive a Taint Bomb; they might catch
-    the ``TaintError`` and give the user a generic message that something
-    went wrong, if we are reasonably careful that the message or even its
-    presence doesn't give information away.  This might be a
-    problem by itself, but there is no satisfying general solution here:
-    it must be considered on a case-by-case basis.  Again, what the
-    Taint Object Space approach achieves is not solving these problems, but
-    localizing them to well-defined small parts of the application - namely,
-    around calls to ``untaint()``.
-
-    The ``TaintError`` exception deliberately does not include any
-    useful error messages, because they might give information away.
-    Of course, this makes debugging quite a bit harder; a difficult
-    problem to solve properly.  So far we have implemented a way to peek in a Taint
-    Box or Bomb, ``__pypy__._taint_look(x)``, and a "debug mode" that
-    prints the exception as soon as a Bomb is created - both write
-    information to the low-level stderr of the application, where we hope
-    that it is unlikely to be seen by anyone but the application
-    developer.
-
-
-    Taint Atomic functions
-    ----------------------
-
-    Occasionally, a more complicated computation must be performed on a
-    tainted object.  This requires first untainting the object, performing the
-    computations, and then carefully tainting the result again (including
-    hiding all exceptions into Bombs).
-
-    There is a built-in decorator that does this for you::
-
-        >>>> @__pypy__.taint_atomic
-        >>>> def myop(x, y):
-        ....     while x > 0:
-        ....         x -= y
-        ....     return x
-        ....
-        >>>> myop(42, 10)
-        -8
-        >>>> z = myop(taint(42), 10)
-        >>>> z
-        TaintError
-        >>>> untaint(int, z)
-        -8
-
-    The decorator makes a whole function behave like a built-in operation.
-    If no tainted argument is passed in, the function behaves normally.  But
-    if any of the arguments is tainted, it is automatically untainted - so
-    the function body always sees untainted arguments - and the eventual
-    result is tainted again (possibly in a Taint Bomb).
-
-    It is important for the function marked as ``taint_atomic`` to have no
-    visible side effects, as these could cause information leakage.
-    This is currently not enforced, which means that all ``taint_atomic``
-    functions have to be carefully reviewed for security (but not the
-    callers of ``taint_atomic`` functions).
-
-    A possible future extension would be to forbid side-effects on
-    non-tainted objects from all ``taint_atomic`` functions.
-
-    An example of usage: given a tainted object ``passwords_db`` that
-    references a database of passwords, we can write a function
-    that checks if a password is valid as follows::
-
-        @taint_atomic
-        def validate(passwords_db, username, password):
-            assert type(passwords_db) is PasswordDatabase
-            assert type(username) is str
-            assert type(password) is str
-            ...load username entry from passwords_db...
-            return expected_password == password
-
-    It returns a tainted boolean answer, or a Taint Bomb if something
-    went wrong.  A caller can do::
-
-        ok = validate(passwords_db, 'john', '1234')
-        ok = untaint(bool, ok)
-
-    This can give three outcomes: ``True``, ``False``, or a ``TaintError``
-    exception (with no information on it) if anything went wrong.  If even
-    this is considered giving too much information away, the ``False`` case
-    can be made indistinguishable from the ``TaintError`` case (simply by
-    raising an exception in ``validate()`` if the password is wrong).
-
-    In the above example, the security results achieved are the following:
-    as long as ``validate()`` does not leak information, no other part of
-    the code can obtain more information about a passwords database than a
-    Yes/No answer to a precise query.
-
-    A possible extension of the ``taint_atomic`` decorator would be to check
-    the argument types, as ``untaint()`` does, for the same reason: to
-    prevent bugs where a function like ``validate()`` above is accidentally
-    called with the wrong kind of tainted object, which would make it
-    misbehave.  For now, all ``taint_atomic`` functions should be
-    conservative and carefully check all assumptions on their input
-    arguments.
-
-
-    .. _`taint-interface`:
-
-    Interface
-    ---------
-
-    .. _`like a built-in operation`:
-
-    The basic rule of the Tainted Object Space is that it introduces two new
-    kinds of objects, Tainted Boxes and Tainted Bombs (which are not types
-    in the Python sense).  Each box internally contains a regular object;
-    each bomb internally contains an exception object.  An operation
-    involving Tainted Boxes is performed on the objects contained in the
-    boxes, and gives a Tainted Box or a Tainted Bomb as a result (such an
-    operation does not let an exception be raised).  An operation called
-    with a Tainted Bomb argument immediately returns the same Tainted Bomb.
-
-    In a PyPy running with (or translated with) the Taint Object Space,
-    the ``__pypy__`` module exposes the following interface:
-
-    * ``taint(obj)``
-
-        Return a new Tainted Box wrapping ``obj``.  Return ``obj`` itself
-        if it is already tainted (a Box or a Bomb).
-
-    * ``is_tainted(obj)``
-
-        Check if ``obj`` is tainted (a Box or a Bomb).
-
-    * ``untaint(type, obj)``
-
-        Untaints ``obj`` if it is tainted.  Raise ``TaintError`` if the type
-        of the untainted object is not exactly ``type``, or if ``obj`` is a
-        Bomb.
-
-    * ``taint_atomic(func)``
-
-        Return a wrapper function around the callable ``func``.  The wrapper
-        behaves `like a built-in operation`_ with respect to untainting the
-        arguments, tainting the result, and returning a Bomb.
-
-    * ``TaintError``
-
-        Exception.  On purpose, it provides no attribute or error message.
-
-    * ``_taint_debug(level)``
-
-        Set the debugging level to ``level`` (0=off).  At level 1 or above,
-        all Taint Bombs print a diagnostic message to stderr when they are
-        created.
-
-    * ``_taint_look(obj)``
-
-        For debugging purposes: prints (to stderr) the type and address of
-        the object in a Tainted Box, or prints the exception if ``obj`` is
-        a Taint Bomb.
-
-
 .. _dump:
 
 The Dump Object Space
diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -53,6 +53,18 @@
 this is an ideal task to get started, because it does not require any deep
 knowledge of the internals.
 
+Optimized Unicode Representation
+--------------------------------
+
+CPython 3.3 will use an `optimized unicode representation`_ which switches between
+different ways to represent a unicode string, depending on whether the string
+fits into ASCII, has only two-byte characters or needs four-byte characters.
+
+The actual details would be rather differen in PyPy, but we would like to have
+the same optimization implemented.
+
+.. _`optimized unicode representation`: http://www.python.org/dev/peps/pep-0393/
+
 Translation Toolchain
 ---------------------
 
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -175,6 +175,9 @@
             self.w_tracefunc = w_func
             self.space.frame_trace_action.fire()
 
+    def gettrace(self):
+        return self.w_tracefunc
+
     def setprofile(self, w_func):
         """Set the global trace function."""
         if self.space.is_w(w_func, self.space.w_None):
@@ -307,7 +310,11 @@
         self._nonperiodic_actions = []
         self.has_bytecode_counter = False
         self.fired_actions = None
-        self.checkinterval_scaled = 100 * TICK_COUNTER_STEP
+        # the default value is not 100, unlike CPython 2.7, but a much
+        # larger value, because we use a technique that not only allows
+        # but actually *forces* another thread to run whenever the counter
+        # reaches zero.
+        self.checkinterval_scaled = 10000 * TICK_COUNTER_STEP
         self._rebuild_action_dispatcher()
 
     def fire(self, action):
@@ -346,6 +353,7 @@
         elif interval > MAX:
             interval = MAX
         self.checkinterval_scaled = interval * TICK_COUNTER_STEP
+        self.reset_ticker(-1)
 
     def _rebuild_action_dispatcher(self):
         periodic_actions = unrolling_iterable(self._periodic_actions)
@@ -383,8 +391,11 @@
     def decrement_ticker(self, by):
         value = self._ticker
         if self.has_bytecode_counter:    # this 'if' is constant-folded
-            value -= by
-            self._ticker = value
+            if jit.isconstant(by) and by == 0:
+                pass     # normally constant-folded too
+            else:
+                value -= by
+                self._ticker = value
         return value
 
 
diff --git a/pypy/interpreter/test/test_executioncontext.py b/pypy/interpreter/test/test_executioncontext.py
--- a/pypy/interpreter/test/test_executioncontext.py
+++ b/pypy/interpreter/test/test_executioncontext.py
@@ -42,6 +42,7 @@
         assert i == 9
 
     def test_periodic_action(self):
+        from pypy.interpreter.executioncontext import ActionFlag
 
         class DemoAction(executioncontext.PeriodicAsyncAction):
             counter = 0
@@ -53,17 +54,20 @@
 
         space = self.space
         a2 = DemoAction(space)
-        space.actionflag.register_periodic_action(a2, True)
         try:
-            for i in range(500):
-                space.appexec([], """():
-                    n = 5
-                    return n + 2
-                """)
-        except Finished:
-            pass
-        checkinterval = space.actionflag.getcheckinterval()
-        assert checkinterval / 10 < i < checkinterval * 1.1
+            space.actionflag.setcheckinterval(100)
+            space.actionflag.register_periodic_action(a2, True)
+            try:
+                for i in range(500):
+                    space.appexec([], """():
+                        n = 5
+                        return n + 2
+                    """)
+            except Finished:
+                pass
+        finally:
+            space.actionflag = ActionFlag()   # reset to default
+        assert 10 < i < 110
 
     def test_llprofile(self):
         l = []
diff --git a/pypy/jit/backend/llgraph/llimpl.py b/pypy/jit/backend/llgraph/llimpl.py
--- a/pypy/jit/backend/llgraph/llimpl.py
+++ b/pypy/jit/backend/llgraph/llimpl.py
@@ -163,6 +163,7 @@
     'unicodegetitem'  : (('ref', 'int'), 'int'),
     'unicodesetitem'  : (('ref', 'int', 'int'), 'int'),
     'cast_ptr_to_int' : (('ref',), 'int'),
+    'cast_int_to_ptr' : (('int',), 'ref'),
     'debug_merge_point': (('ref', 'int'), None),
     'force_token'     : ((), 'int'),
     'call_may_force'  : (('int', 'varargs'), 'intorptr'),
@@ -906,9 +907,6 @@
     def op_new_array(self, arraydescr, count):
         return do_new_array(arraydescr.ofs, count)
 
-    def op_cast_ptr_to_int(self, descr, ptr):
-        return cast_to_int(ptr)
-
     def op_force_token(self, descr):
         opaque_frame = _to_opaque(self)
         return llmemory.cast_ptr_to_adr(opaque_frame)
diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py
--- a/pypy/jit/backend/llsupport/gc.py
+++ b/pypy/jit/backend/llsupport/gc.py
@@ -45,6 +45,14 @@
     def freeing_block(self, start, stop):
         pass
 
+    def record_constptrs(self, op, gcrefs_output_list):
+        for i in range(op.numargs()):
+            v = op.getarg(i)
+            if isinstance(v, ConstPtr) and bool(v.value):
+                p = v.value
+                rgc._make_sure_does_not_move(p)
+                gcrefs_output_list.append(p)
+
 # ____________________________________________________________
 
 class GcLLDescr_boehm(GcLLDescription):
@@ -141,6 +149,14 @@
     get_funcptr_for_newstr = None
     get_funcptr_for_newunicode = None
 
+    def rewrite_assembler(self, cpu, operations, gcrefs_output_list):
+        # record all GCREFs too, because Boehm cannot see them and keep them
+        # alive if they end up as constants in the assembler
+        for op in operations:
+            self.record_constptrs(op, gcrefs_output_list)
+        return GcLLDescription.rewrite_assembler(self, cpu, operations,
+                                                 gcrefs_output_list)
+
 
 # ____________________________________________________________
 # All code below is for the hybrid or minimark GC
@@ -757,14 +773,6 @@
             funcptr(llmemory.cast_ptr_to_adr(gcref_struct),
                     llmemory.cast_ptr_to_adr(gcref_newptr))
 
-    def record_constptrs(self, op, gcrefs_output_list):
-        for i in range(op.numargs()):
-            v = op.getarg(i)
-            if isinstance(v, ConstPtr) and bool(v.value):
-                p = v.value
-                rgc._make_sure_does_not_move(p)
-                gcrefs_output_list.append(p)
-
     def rewrite_assembler(self, cpu, operations, gcrefs_output_list):
         # Perform two kinds of rewrites in parallel:
         #
diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -1506,20 +1506,16 @@
         return u''.join(u.chars)
 
 
-    def test_casts(self):
-        py.test.skip("xxx fix or kill")
-        from pypy.rpython.lltypesystem import lltype, llmemory
-        TP = lltype.GcStruct('x')
-        x = lltype.malloc(TP)        
-        x = lltype.cast_opaque_ptr(llmemory.GCREF, x)
+    def test_cast_int_to_ptr(self):
+        res = self.execute_operation(rop.CAST_INT_TO_PTR,
+                                     [BoxInt(-17)],  'ref').value
+        assert lltype.cast_ptr_to_int(res) == -17
+
+    def test_cast_ptr_to_int(self):
+        x = lltype.cast_int_to_ptr(llmemory.GCREF, -19)
         res = self.execute_operation(rop.CAST_PTR_TO_INT,
-                                     [BoxPtr(x)],  'int').value
-        expected = self.cpu.cast_adr_to_int(llmemory.cast_ptr_to_adr(x))
-        assert rffi.get_real_int(res) == rffi.get_real_int(expected)
-        res = self.execute_operation(rop.CAST_PTR_TO_INT,
-                                     [ConstPtr(x)],  'int').value
-        expected = self.cpu.cast_adr_to_int(llmemory.cast_ptr_to_adr(x))
-        assert rffi.get_real_int(res) == rffi.get_real_int(expected)
+                                     [BoxPtr(x)], 'int').value
+        assert res == -19
 
     def test_ooops_non_gc(self):
         x = lltype.malloc(lltype.Struct('x'), flavor='raw')
@@ -2337,13 +2333,6 @@
         #
         cpu.bh_strsetitem(x, 4, ord('/'))
         assert str.chars[4] == '/'
-        #
-##        x = cpu.bh_newstr(5)
-##        y = cpu.bh_cast_ptr_to_int(x)
-##        z = cpu.bh_cast_ptr_to_int(x)
-##        y = rffi.get_real_int(y)
-##        z = rffi.get_real_int(z)
-##        assert type(y) == type(z) == int and y == z
 
     def test_sorting_of_fields(self):
         S = self.S
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1386,7 +1386,8 @@
 
     def genop_same_as(self, op, arglocs, resloc):
         self.mov(arglocs[0], resloc)
-    #genop_cast_ptr_to_int = genop_same_as
+    genop_cast_ptr_to_int = genop_same_as
+    genop_cast_int_to_ptr = genop_same_as
 
     def genop_int_mod(self, op, arglocs, resloc):
         if IS_X86_32:
diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py
--- a/pypy/jit/backend/x86/regalloc.py
+++ b/pypy/jit/backend/x86/regalloc.py
@@ -1196,7 +1196,8 @@
         self.possibly_free_var(op.getarg(0))
         resloc = self.force_allocate_reg(op.result)
         self.Perform(op, [argloc], resloc)
-    #consider_cast_ptr_to_int = consider_same_as
+    consider_cast_ptr_to_int = consider_same_as
+    consider_cast_int_to_ptr = consider_same_as
 
     def consider_strlen(self, op):
         args = op.getarglist()
diff --git a/pypy/jit/backend/x86/tool/viewcode.py b/pypy/jit/backend/x86/tool/viewcode.py
--- a/pypy/jit/backend/x86/tool/viewcode.py
+++ b/pypy/jit/backend/x86/tool/viewcode.py
@@ -9,7 +9,12 @@
 """
 
 import autopath
-import operator, sys, os, re, py, new
+import new
+import operator
+import py
+import re
+import sys
+import subprocess
 from bisect import bisect_left
 
 # don't use pypy.tool.udir here to avoid removing old usessions which
@@ -44,14 +49,16 @@
     f = open(tmpfile, 'wb')
     f.write(data)
     f.close()
-    g = os.popen(objdump % {
+    p = subprocess.Popen(objdump % {
         'file': tmpfile,
         'origin': originaddr,
         'backend': objdump_backend_option[backend_name],
-    }, 'r')
-    result = g.readlines()
-    g.close()
-    lines = result[6:]   # drop some objdump cruft
+    }, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    stdout, stderr = p.communicate()
+    assert not p.returncode, ('Encountered an error running objdump: %s' %
+                              stderr)
+    # drop some objdump cruft
+    lines = stdout.splitlines()[6:]
     return format_code_dump_with_labels(originaddr, lines, label_list)
 
 def format_code_dump_with_labels(originaddr, lines, label_list):
@@ -85,8 +92,12 @@
     #
     print 'loading symbols from %s...' % (filename,)
     symbols = {}
-    g = os.popen(symbollister % filename, "r")
-    for line in g:
+    p = subprocess.Popen(symbollister % filename, shell=True,
+                         stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    stdout, stderr = p.communicate()
+    assert not p.returncode, ('Encountered an error running nm: %s' %
+                              stderr)
+    for line in stdout.splitlines():
         match = re_symbolentry.match(line)
         if match:
             addr = long(match.group(1), 16)
@@ -94,7 +105,6 @@
             if name.startswith('pypy_g_'):
                 name = '\xb7' + name[7:]
             symbols[addr] = name
-    g.close()
     print '%d symbols found' % (len(symbols),)
     return symbols
 
diff --git a/pypy/jit/codewriter/effectinfo.py b/pypy/jit/codewriter/effectinfo.py
--- a/pypy/jit/codewriter/effectinfo.py
+++ b/pypy/jit/codewriter/effectinfo.py
@@ -74,6 +74,7 @@
     OS_LLONG_UGE                = 91
     OS_LLONG_URSHIFT            = 92
     OS_LLONG_FROM_UINT          = 93
+    OS_LLONG_U_TO_FLOAT         = 94
     #
     OS_MATH_SQRT                = 100
 
diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -442,6 +442,7 @@
     rewrite_op_ullong_mod_zer      = _do_builtin_call
     rewrite_op_gc_identityhash = _do_builtin_call
     rewrite_op_gc_id           = _do_builtin_call
+    rewrite_op_uint_mod        = _do_builtin_call
 
     # ----------
     # getfield/setfield/mallocs etc.
@@ -456,6 +457,23 @@
             # the special return value None forces op.result to be considered
             # equal to op.args[0]
             return [op0, op1, None]
+        if (hints.get('promote_string') and
+            op.args[0].concretetype is not lltype.Void):
+            S = lltype.Ptr(rstr.STR)
+            assert op.args[0].concretetype == S
+            self._register_extra_helper(EffectInfo.OS_STREQ_NONNULL,
+                                        "str.eq_nonnull",
+                                        [S, S],
+                                        lltype.Signed,
+                                        EffectInfo.EF_ELIDABLE_CANNOT_RAISE)
+            descr, p = self.callcontrol.callinfocollection.callinfo_for_oopspec(
+                EffectInfo.OS_STREQ_NONNULL)
+            # XXX this is fairly ugly way of creating a constant,
+            #     however, callinfocollection has no better interface
+            c = Constant(p.adr.ptr, lltype.typeOf(p.adr.ptr))
+            op1 = SpaceOperation('str_guard_value', [op.args[0], c, descr],
+                                 op.result)
+            return [SpaceOperation('-live-', [], None), op1, None]
         else:
             log.WARNING('ignoring hint %r at %r' % (hints, self.graph))
 
@@ -810,8 +828,7 @@
 
     def rewrite_op_cast_ptr_to_int(self, op):
         if self._is_gc(op.args[0]):
-            #return op
-            raise NotImplementedError("cast_ptr_to_int")
+            return op
 
     def rewrite_op_force_cast(self, op):
         v_arg = op.args[0]
@@ -911,9 +928,15 @@
                 v = v_arg
                 oplist = []
             if unsigned1:
-                opname = 'cast_uint_to_longlong'
+                if unsigned2:
+                    opname = 'cast_uint_to_ulonglong'
+                else:
+                    opname = 'cast_uint_to_longlong'
             else:
-                opname = 'cast_int_to_longlong'
+                if unsigned2:
+                    opname = 'cast_int_to_ulonglong'
+                else:
+                    opname = 'cast_int_to_longlong'
             op2 = self.rewrite_operation(
                 SpaceOperation(opname, [v], v_result)
             )
@@ -1023,6 +1046,21 @@
                 return op2
         ''' % (_op, _oopspec.lower(), _oopspec, _oopspec)).compile()
 
+    for _op, _oopspec in [('cast_int_to_ulonglong',     'FROM_INT'),
+                          ('cast_uint_to_ulonglong',    'FROM_UINT'),
+                          ('cast_float_to_ulonglong',   'FROM_FLOAT'),
+                          ('cast_ulonglong_to_float',   'U_TO_FLOAT'),
+                         ]:
+        exec py.code.Source('''
+            def rewrite_op_%s(self, op):
+                args = op.args
+                op1 = self.prepare_builtin_call(op, "ullong_%s", args)
+                op2 = self._handle_oopspec_call(op1, args,
+                                                EffectInfo.OS_LLONG_%s,
+                                           EffectInfo.EF_ELIDABLE_CANNOT_RAISE)
+                return op2
+        ''' % (_op, _oopspec.lower(), _oopspec)).compile()
+
     def _normalize(self, oplist):
         if isinstance(oplist, SpaceOperation):
             return [oplist]
@@ -1532,6 +1570,10 @@
     def rewrite_op_jit_force_virtual(self, op):
         return self._do_builtin_call(op)
 
+    def rewrite_op_jit_is_virtual(self, op):
+        raise Exception, (
+            "'vref.virtual' should not be used from jit-visible code")
+
     def rewrite_op_jit_force_virtualizable(self, op):
         # this one is for virtualizables
         vinfo = self.get_vinfo(op.args[0])
diff --git a/pypy/jit/codewriter/support.py b/pypy/jit/codewriter/support.py
--- a/pypy/jit/codewriter/support.py
+++ b/pypy/jit/codewriter/support.py
@@ -310,18 +310,30 @@
 def _ll_1_llong_from_int(x):
     return r_longlong(intmask(x))
 
+def _ll_1_ullong_from_int(x):
+    return r_ulonglong(intmask(x))
+
 def _ll_1_llong_from_uint(x):
     return r_longlong(r_uint(x))
 
+def _ll_1_ullong_from_uint(x):
+    return r_ulonglong(r_uint(x))
+
 def _ll_1_llong_to_int(xll):
     return intmask(xll)
 
 def _ll_1_llong_from_float(xf):
     return r_longlong(xf)
 
+def _ll_1_ullong_from_float(xf):
+    return r_ulonglong(xf)
+
 def _ll_1_llong_to_float(xll):
     return float(rffi.cast(lltype.SignedLongLong, xll))
 
+def _ll_1_ullong_u_to_float(xull):
+    return float(rffi.cast(lltype.UnsignedLongLong, xull))
+
 
 def _ll_1_llong_abs(xll):
     if xll < 0:
@@ -346,20 +358,23 @@
     return llop.llong_mod(lltype.SignedLongLong, xll, yll)
 
 def _ll_2_ullong_floordiv(xll, yll):
-    return llop.ullong_floordiv(lltype.SignedLongLong, xll, yll)
+    return llop.ullong_floordiv(lltype.UnsignedLongLong, xll, yll)
 
 def _ll_2_ullong_floordiv_zer(xll, yll):
     if yll == 0:
         raise ZeroDivisionError
-    return llop.ullong_floordiv(lltype.SignedLongLong, xll, yll)
+    return llop.ullong_floordiv(lltype.UnsignedLongLong, xll, yll)
 
 def _ll_2_ullong_mod(xll, yll):
-    return llop.ullong_mod(lltype.SignedLongLong, xll, yll)
+    return llop.ullong_mod(lltype.UnsignedLongLong, xll, yll)
 
 def _ll_2_ullong_mod_zer(xll, yll):
     if yll == 0:
         raise ZeroDivisionError
-    return llop.ullong_mod(lltype.SignedLongLong, xll, yll)
+    return llop.ullong_mod(lltype.UnsignedLongLong, xll, yll)
+
+def _ll_2_uint_mod(xll, yll):
+    return llop.uint_mod(lltype.Unsigned, xll, yll)
 
 
 # libffi support
diff --git a/pypy/jit/codewriter/test/test_flatten.py b/pypy/jit/codewriter/test/test_flatten.py
--- a/pypy/jit/codewriter/test/test_flatten.py
+++ b/pypy/jit/codewriter/test/test_flatten.py
@@ -829,14 +829,15 @@
                     self.encoding_test(f, [rffi.cast(FROM, 42)], expectedstr,
                                        transform=True)
                 elif TO in (rffi.LONG, rffi.ULONG):
+                    if rffi.cast(FROM, -1) < 0:
+                        fnname = "llong_from_int"
+                    else:
+                        fnname = "llong_from_uint"
                     if TO == rffi.LONG:
                         TO = rffi.LONGLONG
                     else:
                         TO = rffi.ULONGLONG
-                    if rffi.cast(FROM, -1) < 0:
-                        fnname = "llong_from_int"
-                    else:
-                        fnname = "llong_from_uint"
+                        fnname = "u" + fnname
                     expected.pop()   # remove int_return
                     expected.append(
                         "residual_call_irf_f $<* fn %s>, <Descr>, I[%s], R[], F[] -> %%f0"
diff --git a/pypy/jit/codewriter/test/test_jtransform.py b/pypy/jit/codewriter/test/test_jtransform.py
--- a/pypy/jit/codewriter/test/test_jtransform.py
+++ b/pypy/jit/codewriter/test/test_jtransform.py
@@ -1,5 +1,19 @@
 import random
-from pypy.objspace.flow.model import Block, Link
+try:
+    from itertools import product
+except ImportError:
+    # Python 2.5, this is taken from the CPython docs, but simplified.
+    def product(*args):
+        # product('ABCD', 'xy') --> Ax Ay Bx By Cx Cy Dx Dy
+        # product(range(2), repeat=3) --> 000 001 010 011 100 101 110 111
+        pools = map(tuple, args)
+        result = [[]]
+        for pool in pools:
+            result = [x+[y] for x in result for y in pool]
+        for prod in result:
+            yield tuple(prod)
+
+from pypy.objspace.flow.model import FunctionGraph, Block, Link
 from pypy.objspace.flow.model import SpaceOperation, Variable, Constant
 from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr
 from pypy.rpython.lltypesystem.module import ll_math
@@ -86,6 +100,12 @@
             if i == oopspecindex:
                 return True
         return False
+    def callinfo_for_oopspec(self, oopspecindex):
+        assert oopspecindex == effectinfo.EffectInfo.OS_STREQ_NONNULL
+        class c:
+            class adr:
+                ptr = 1
+        return ('calldescr', c)
 
 class FakeBuiltinCallControl:
     def __init__(self):
@@ -106,6 +126,7 @@
              EI.OS_STR2UNICODE:([PSTR], PUNICODE),
              EI.OS_STR_CONCAT: ([PSTR, PSTR], PSTR),
              EI.OS_STR_SLICE:  ([PSTR, INT, INT], PSTR),
+             EI.OS_STREQ_NONNULL:  ([PSTR, PSTR], INT),
              EI.OS_UNI_CONCAT: ([PUNICODE, PUNICODE], PUNICODE),
              EI.OS_UNI_SLICE:  ([PUNICODE, INT, INT], PUNICODE),
              EI.OS_UNI_EQUAL:  ([PUNICODE, PUNICODE], lltype.Bool),
@@ -255,26 +276,35 @@
             assert op1.result is None
 
 def test_calls():
-    for RESTYPE in [lltype.Signed, rclass.OBJECTPTR,
-                    lltype.Float, lltype.Void]:
-      for with_void in [False, True]:
-        for with_i in [False, True]:
-          for with_r in [False, True]:
-            for with_f in [False, True]:
-              ARGS = []
-              if with_void: ARGS += [lltype.Void, lltype.Void]
-              if with_i: ARGS += [lltype.Signed, lltype.Char]
-              if with_r: ARGS += [rclass.OBJECTPTR, lltype.Ptr(rstr.STR)]
-              if with_f: ARGS += [lltype.Float, lltype.Float]
-              random.shuffle(ARGS)
-              if RESTYPE == lltype.Float: with_f = True
-              if with_f: expectedkind = 'irf'   # all kinds
-              elif with_i: expectedkind = 'ir'  # integers and references
-              else: expectedkind = 'r'          # only references
-              yield residual_call_test, ARGS, RESTYPE, expectedkind
-              yield direct_call_test, ARGS, RESTYPE, expectedkind
-              yield indirect_residual_call_test, ARGS, RESTYPE, expectedkind
-              yield indirect_regular_call_test, ARGS, RESTYPE, expectedkind
+    for RESTYPE, with_void, with_i, with_r, with_f in product(
+        [lltype.Signed, rclass.OBJECTPTR, lltype.Float, lltype.Void],
+        [False, True],
+        [False, True],
+        [False, True],
+        [False, True],
+    ):
+        ARGS = []
+        if with_void:
+            ARGS += [lltype.Void, lltype.Void]
+        if with_i:
+            ARGS += [lltype.Signed, lltype.Char]
+        if with_r:
+            ARGS += [rclass.OBJECTPTR, lltype.Ptr(rstr.STR)]
+        if with_f:
+            ARGS += [lltype.Float, lltype.Float]
+        random.shuffle(ARGS)
+        if RESTYPE == lltype.Float:
+            with_f = True
+        if with_f:
+            expectedkind = 'irf'   # all kinds
+        elif with_i:
+            expectedkind = 'ir'  # integers and references
+        else:
+            expectedkind = 'r'          # only references
+        yield residual_call_test, ARGS, RESTYPE, expectedkind
+        yield direct_call_test, ARGS, RESTYPE, expectedkind
+        yield indirect_residual_call_test, ARGS, RESTYPE, expectedkind
+        yield indirect_regular_call_test, ARGS, RESTYPE, expectedkind
 
 def get_direct_call_op(argtypes, restype):
     FUNC = lltype.FuncType(argtypes, restype)
@@ -855,6 +885,21 @@
     assert op1.args[2] == ListOfKind('ref', [v1, v2])
     assert op1.result == v3
 
+def test_str_promote():
+    PSTR = lltype.Ptr(rstr.STR)
+    v1 = varoftype(PSTR)
+    v2 = varoftype(PSTR)
+    op = SpaceOperation('hint',
+                        [v1, Constant({'promote_string': True}, lltype.Void)],
+                        v2)
+    tr = Transformer(FakeCPU(), FakeBuiltinCallControl())
+    op0, op1, _ = tr.rewrite_operation(op)
+    assert op1.opname == 'str_guard_value'
+    assert op1.args[0] == v1
+    assert op1.args[2] == 'calldescr'
+    assert op1.result == v2
+    assert op0.opname == '-live-'
+
 def test_unicode_concat():
     # test that the oopspec is present and correctly transformed
     PSTR = lltype.Ptr(rstr.UNICODE)
diff --git a/pypy/jit/codewriter/test/test_longlong.py b/pypy/jit/codewriter/test/test_longlong.py
--- a/pypy/jit/codewriter/test/test_longlong.py
+++ b/pypy/jit/codewriter/test/test_longlong.py
@@ -57,7 +57,8 @@
             assert op1.opname == 'residual_call_irf_f'
         else:
             assert op1.opname == 'residual_call_irf_i'
-        gotindex = getattr(EffectInfo, 'OS_' + op1.args[0].value.upper())
+        gotindex = getattr(EffectInfo,
+                           'OS_' + op1.args[0].value.upper().lstrip('U'))
         assert gotindex == oopspecindex
         assert op1.args[1] == 'calldescr-%d' % oopspecindex
         assert list(op1.args[2]) == [v for v in vlist
@@ -192,8 +193,12 @@
                       [lltype.SignedLongLong], lltype.Signed)
         self.do_check('cast_float_to_longlong', EffectInfo.OS_LLONG_FROM_FLOAT,
                       [lltype.Float], lltype.SignedLongLong)
+        self.do_check('cast_float_to_ulonglong', EffectInfo.OS_LLONG_FROM_FLOAT,
+                      [lltype.Float], lltype.UnsignedLongLong)
         self.do_check('cast_longlong_to_float', EffectInfo.OS_LLONG_TO_FLOAT,
                       [lltype.SignedLongLong], lltype.Float)
+        self.do_check('cast_ulonglong_to_float', EffectInfo.OS_LLONG_U_TO_FLOAT,
+                      [lltype.UnsignedLongLong], lltype.Float)
         for T1 in [lltype.SignedLongLong, lltype.UnsignedLongLong]:
             for T2 in [lltype.Signed, lltype.Unsigned]:
                 self.do_check('cast_primitive', EffectInfo.OS_LLONG_TO_INT,
diff --git a/pypy/jit/metainterp/blackhole.py b/pypy/jit/metainterp/blackhole.py
--- a/pypy/jit/metainterp/blackhole.py
+++ b/pypy/jit/metainterp/blackhole.py
@@ -2,7 +2,7 @@
 from pypy.rlib.rtimer import read_timestamp
 from pypy.rlib.rarithmetic import intmask, LONG_BIT, r_uint, ovfcheck
 from pypy.rlib.objectmodel import we_are_translated
-from pypy.rlib.debug import debug_start, debug_stop
+from pypy.rlib.debug import debug_start, debug_stop, ll_assert
 from pypy.rlib.debug import make_sure_not_resized
 from pypy.rpython.lltypesystem import lltype, llmemory, rclass
 from pypy.rpython.lltypesystem.lloperation import llop
@@ -502,6 +502,15 @@
     @arguments("r", returns="r")
     def bhimpl_cast_opaque_ptr(a):
         return a
+    @arguments("r", returns="i")
+    def bhimpl_cast_ptr_to_int(a):
+        i = lltype.cast_ptr_to_int(a)
+        ll_assert((i & 1) == 1, "bhimpl_cast_ptr_to_int: not an odd int")
+        return i
+    @arguments("i", returns="r")
+    def bhimpl_cast_int_to_ptr(i):
+        ll_assert((i & 1) == 1, "bhimpl_cast_int_to_ptr: not an odd int")
+        return lltype.cast_int_to_ptr(llmemory.GCREF, i)
 
     @arguments("i", returns="i")
     def bhimpl_int_copy(a):
@@ -522,6 +531,9 @@
     @arguments("f")
     def bhimpl_float_guard_value(a):
         pass
+    @arguments("r", "i", "d", returns="r")
+    def bhimpl_str_guard_value(a, i, d):
+        return a
 
     @arguments("self", "i")
     def bhimpl_int_push(self, a):
diff --git a/pypy/jit/metainterp/heapcache.py b/pypy/jit/metainterp/heapcache.py
--- a/pypy/jit/metainterp/heapcache.py
+++ b/pypy/jit/metainterp/heapcache.py
@@ -54,9 +54,10 @@
         if box in self.new_boxes:
             self.new_boxes[box] = False
         if box in self.dependencies:
-            for dep in self.dependencies[box]:
+            deps = self.dependencies[box]
+            del self.dependencies[box]
+            for dep in deps:
                 self._escape(dep)
-            del self.dependencies[box]
 
     def clear_caches(self, opnum, descr, argboxes):
         if opnum == rop.SETFIELD_GC:
diff --git a/pypy/jit/metainterp/history.py b/pypy/jit/metainterp/history.py
--- a/pypy/jit/metainterp/history.py
+++ b/pypy/jit/metainterp/history.py
@@ -9,6 +9,7 @@
 
 from pypy.jit.metainterp.resoperation import ResOperation, rop
 from pypy.jit.codewriter import heaptracker, longlong
+from pypy.rlib.objectmodel import compute_identity_hash
 
 # ____________________________________________________________
 
@@ -104,7 +105,7 @@
     getref._annspecialcase_ = 'specialize:arg(1)'
 
     def _get_hash_(self):
-        raise NotImplementedError
+        return compute_identity_hash(self)
 
     def clonebox(self):
         raise NotImplementedError
@@ -133,6 +134,9 @@
     def _get_str(self):
         raise NotImplementedError
 
+    def same_box(self, other):
+        return self is other
+
 class AbstractDescr(AbstractValue):
     __slots__ = ()
 
@@ -241,32 +245,15 @@
     def constbox(self):
         return self
 
+    def same_box(self, other):
+        return self.same_constant(other)
+
     def same_constant(self, other):
         raise NotImplementedError
 
     def __repr__(self):
         return 'Const(%s)' % self._getrepr_()
 
-    def __eq__(self, other):
-        "NOT_RPYTHON"
-        # Remember that you should not compare Consts with '==' in RPython.
-        # Consts have no special __hash__, in order to force different Consts
-        # from being considered as different keys when stored in dicts
-        # (as they always are after translation).  Use a dict_equal_consts()
-        # to get the other behavior (i.e. using this __eq__).
-        if self.__class__ is not other.__class__:
-            return False
-        try:
-            return self.value == other.value
-        except TypeError:
-            if (isinstance(self.value, Symbolic) and
-                isinstance(other.value, Symbolic)):
-                return self.value is other.value
-            raise
-
-    def __ne__(self, other):
-        return not (self == other)
-
 
 class ConstInt(Const):
     type = INT
@@ -688,33 +675,6 @@
 
 # ____________________________________________________________
 
-def dict_equal_consts():
-    "NOT_RPYTHON"
-    # Returns a dict in which Consts that compare as equal
-    # are identified when used as keys.
-    return r_dict(dc_eq, dc_hash)
-
-def dc_eq(c1, c2):
-    return c1 == c2
-
-def dc_hash(c):
-    "NOT_RPYTHON"
-    # This is called during translation only.  Avoid using identityhash(),
-    # to avoid forcing a hash, at least on lltype objects.
-    if not isinstance(c, Const):
-        return hash(c)
-    if isinstance(c.value, Symbolic):
-        return id(c.value)
-    try:
-        if isinstance(c, ConstPtr):
-            p = lltype.normalizeptr(c.value)
-            if p is not None:
-                return hash(p._obj)
-            else:
-                return 0
-        return c._get_hash_()
-    except lltype.DelayedPointer:
-        return -2      # xxx risk of changing hash...
 
 def make_hashable_int(i):
     from pypy.rpython.lltypesystem.ll2ctypes import NotCtypesAllocatedStructure
diff --git a/pypy/jit/metainterp/optimizeopt/__init__.py b/pypy/jit/metainterp/optimizeopt/__init__.py
--- a/pypy/jit/metainterp/optimizeopt/__init__.py
+++ b/pypy/jit/metainterp/optimizeopt/__init__.py
@@ -7,6 +7,8 @@
 from pypy.jit.metainterp.optimizeopt.unroll import optimize_unroll, OptInlineShortPreamble
 from pypy.jit.metainterp.optimizeopt.fficall import OptFfiCall
 from pypy.jit.metainterp.optimizeopt.simplify import OptSimplify
+from pypy.jit.metainterp.optimizeopt.pure import OptPure
+from pypy.jit.metainterp.optimizeopt.earlyforce import OptEarlyForce
 from pypy.rlib.jit import PARAMETERS
 from pypy.rlib.unroll import unrolling_iterable
 
@@ -14,6 +16,8 @@
             ('rewrite', OptRewrite),
             ('virtualize', OptVirtualize),
             ('string', OptString),
+            ('earlyforce', OptEarlyForce),
+            ('pure', OptPure),
             ('heap', OptHeap),
             ('ffi', None),
             ('unroll', None)]
diff --git a/pypy/jit/metainterp/optimizeopt/earlyforce.py b/pypy/jit/metainterp/optimizeopt/earlyforce.py
new file mode 100644
--- /dev/null
+++ b/pypy/jit/metainterp/optimizeopt/earlyforce.py
@@ -0,0 +1,24 @@
+from pypy.jit.metainterp.optimizeopt.optimizer import Optimization
+from pypy.jit.metainterp.optimizeopt.vstring import VAbstractStringValue
+from pypy.jit.metainterp.resoperation import rop, ResOperation
+
+class OptEarlyForce(Optimization):
+    def propagate_forward(self, op):
+        opnum = op.getopnum()
+        if (opnum != rop.SETFIELD_GC and 
+            opnum != rop.SETARRAYITEM_GC and
+            opnum != rop.QUASIIMMUT_FIELD):
+               
+            for arg in op.getarglist():
+                if arg in self.optimizer.values:
+                    value = self.getvalue(arg)
+                    value.force_box(self)
+        self.emit_operation(op)
+
+    def new(self):
+        return OptEarlyForce()
+
+    def setup(self):
+        self.optimizer.optearlyforce = self
+
+    
diff --git a/pypy/jit/metainterp/optimizeopt/fficall.py b/pypy/jit/metainterp/optimizeopt/fficall.py
--- a/pypy/jit/metainterp/optimizeopt/fficall.py
+++ b/pypy/jit/metainterp/optimizeopt/fficall.py
@@ -177,10 +177,10 @@
             funcinfo.descr is None):
             return [op] # cannot optimize
         funcsymval = self.getvalue(op.getarg(2))
-        arglist = [funcsymval.force_box()]
+        arglist = [funcsymval.get_key_box()]
         for push_op in funcinfo.opargs:
             argval = self.getvalue(push_op.getarg(2))
-            arglist.append(argval.force_box())
+            arglist.append(argval.get_key_box())
         newop = ResOperation(rop.CALL_RELEASE_GIL, arglist, op.result,
                              descr=funcinfo.descr)
         self.commit_optimization()
diff --git a/pypy/jit/metainterp/optimizeopt/heap.py b/pypy/jit/metainterp/optimizeopt/heap.py
--- a/pypy/jit/metainterp/optimizeopt/heap.py
+++ b/pypy/jit/metainterp/optimizeopt/heap.py
@@ -50,6 +50,7 @@
             if not self._lazy_setfield_registered:
                 optheap._lazy_setfields_and_arrayitems.append(self)
                 self._lazy_setfield_registered = True
+            
         else:
             # this is the case where the pending setfield ends up
             # storing precisely the value that is already there,
@@ -158,12 +159,17 @@
         self._lazy_setfields_and_arrayitems = []
         self._remove_guard_not_invalidated = False
         self._seen_guard_not_invalidated = False
+        self.posponedop = None
 
     def force_at_end_of_preamble(self):
         self.force_all_lazy_setfields_and_arrayitems()
 
     def flush(self):
         self.force_all_lazy_setfields_and_arrayitems()
+        if self.posponedop:
+            posponedop = self.posponedop
+            self.posponedop = None
+            self.next_optimization.propagate_forward(posponedop)
 
     def new(self):
         return OptHeap()
@@ -211,7 +217,15 @@
 
     def emit_operation(self, op):
         self.emitting_operation(op)
-        self.next_optimization.propagate_forward(op)
+        if self.posponedop:
+            posponedop = self.posponedop
+            self.posponedop = None
+            self.next_optimization.propagate_forward(posponedop)
+        if (op.is_comparison() or op.getopnum() == rop.CALL_MAY_FORCE
+            or op.is_ovf()):
+            self.posponedop = op
+        else:
+            self.next_optimization.propagate_forward(op)
 
     def emitting_operation(self, op):
         if op.has_no_side_effect():
@@ -293,30 +307,6 @@
             if indexvalue is None or indexvalue.intbound.contains(idx):
                 cf.force_lazy_setfield(self, can_cache)
 
-    def fixup_guard_situation(self):
-        # hackish: reverse the order of the last two operations if it makes
-        # sense to avoid a situation like "int_eq/setfield_gc/guard_true",
-        # which the backend (at least the x86 backend) does not handle well.
-        newoperations = self.optimizer.newoperations
-        if len(newoperations) < 2:
-            return
-        lastop = newoperations[-1]
-        if (lastop.getopnum() != rop.SETFIELD_GC and
-            lastop.getopnum() != rop.SETARRAYITEM_GC):
-            return
-        # - is_comparison() for cases like "int_eq/setfield_gc/guard_true"
-        # - CALL_MAY_FORCE: "call_may_force/setfield_gc/guard_not_forced"
-        # - is_ovf(): "int_add_ovf/setfield_gc/guard_no_overflow"
-        prevop = newoperations[-2]
-        opnum = prevop.getopnum()
-        if not (prevop.is_comparison() or opnum == rop.CALL_MAY_FORCE
-                or prevop.is_ovf()):
-            return
-        if prevop.result in lastop.getarglist():
-            return
-        newoperations[-2] = lastop
-        newoperations[-1] = prevop
-
     def _assert_valid_cf(self, cf):
         # check that 'cf' is in cached_fields or cached_arrayitems
         if not we_are_translated():
@@ -362,7 +352,6 @@
                                       fieldvalue.get_key_box(), itemindex))
             else:
                 cf.force_lazy_setfield(self)
-                self.fixup_guard_situation()
         return pendingfields
 
     def optimize_GETFIELD_GC(self, op):
@@ -374,12 +363,22 @@
             return
         # default case: produce the operation
         structvalue.ensure_nonnull()
-        ###self.optimizer.optimize_default(op)
         self.emit_operation(op)
         # then remember the result of reading the field
         fieldvalue = self.getvalue(op.result)
         cf.remember_field_value(structvalue, fieldvalue, op)
 
+    def optimize_GETFIELD_GC_PURE(self, op):
+        structvalue = self.getvalue(op.getarg(0))
+        cf = self.field_cache(op.getdescr())
+        fieldvalue = cf.getfield_from_cache(self, structvalue)
+        if fieldvalue is not None:
+            self.make_equal_to(op.result, fieldvalue)
+            return
+        # default case: produce the operation
+        structvalue.ensure_nonnull()
+        self.emit_operation(op)
+
     def optimize_SETFIELD_GC(self, op):
         if self.has_pure_result(rop.GETFIELD_GC_PURE, [op.getarg(0)],
                                 op.getdescr()):
@@ -389,6 +388,7 @@
         #
         cf = self.field_cache(op.getdescr())
         cf.do_setfield(self, op)
+        
 
     def optimize_GETARRAYITEM_GC(self, op):
         arrayvalue = self.getvalue(op.getarg(0))
@@ -413,6 +413,25 @@
             fieldvalue = self.getvalue(op.result)
             cf.remember_field_value(arrayvalue, fieldvalue, op)
 
+    def optimize_GETARRAYITEM_GC_PURE(self, op):
+        arrayvalue = self.getvalue(op.getarg(0))
+        indexvalue = self.getvalue(op.getarg(1))
+        cf = None
+        if indexvalue.is_constant():
+            arrayvalue.make_len_gt(MODE_ARRAY, op.getdescr(), indexvalue.box.getint())
+            # use the cache on (arraydescr, index), which is a constant
+            cf = self.arrayitem_cache(op.getdescr(), indexvalue.box.getint())
+            fieldvalue = cf.getfield_from_cache(self, arrayvalue)
+            if fieldvalue is not None:
+                self.make_equal_to(op.result, fieldvalue)
+                return
+        else:
+            # variable index, so make sure the lazy setarrayitems are done
+            self.force_lazy_setarrayitem(op.getdescr(), indexvalue=indexvalue)
+        # default case: produce the operation
+        arrayvalue.ensure_nonnull()
+        self.emit_operation(op)
+
     def optimize_SETARRAYITEM_GC(self, op):
         if self.has_pure_result(rop.GETARRAYITEM_GC_PURE, [op.getarg(0),
                                                            op.getarg(1)],
diff --git a/pypy/jit/metainterp/optimizeopt/optimizer.py b/pypy/jit/metainterp/optimizeopt/optimizer.py
--- a/pypy/jit/metainterp/optimizeopt/optimizer.py
+++ b/pypy/jit/metainterp/optimizeopt/optimizer.py
@@ -31,8 +31,8 @@
 
 class OptValue(object):
     __metaclass__ = extendabletype
-    _attrs_ = ('box', 'known_class', 'last_guard_index', 'level', 'intbound', 'lenbound')
-    last_guard_index = -1
+    _attrs_ = ('box', 'known_class', 'last_guard', 'level', 'intbound', 'lenbound')
+    last_guard = None
 
     level = LEVEL_UNKNOWN
     known_class = None
@@ -100,7 +100,7 @@
             self.make_constant(other.get_key_box())
             optimizer.turned_constant(self)
         elif other.level == LEVEL_KNOWNCLASS:
-            self.make_constant_class(other.known_class, -1)
+            self.make_constant_class(other.known_class, None)
         else:
             if other.level == LEVEL_NONNULL:
                 self.ensure_nonnull()
@@ -114,13 +114,13 @@
                     self.lenbound = other.lenbound.clone()
 
 
-    def force_box(self):
+    def force_box(self, optforce):
         return self.box
 
     def get_key_box(self):
         return self.box
 
-    def force_at_end_of_preamble(self, already_forced):
+    def force_at_end_of_preamble(self, already_forced, optforce):
         return self
 
     def get_args_for_fail(self, modifier):
@@ -162,16 +162,16 @@
         else:
             return None
 
-    def make_constant_class(self, classbox, opindex):
+    def make_constant_class(self, classbox, guardop):
         assert self.level < LEVEL_KNOWNCLASS
         self.known_class = classbox
         self.level = LEVEL_KNOWNCLASS
-        self.last_guard_index = opindex
+        self.last_guard = guardop
 
-    def make_nonnull(self, opindex):
+    def make_nonnull(self, guardop):
         assert self.level < LEVEL_NONNULL
         self.level = LEVEL_NONNULL
-        self.last_guard_index = opindex
+        self.last_guard = guardop
 
     def is_nonnull(self):
         level = self.level
@@ -240,26 +240,12 @@
     def __init__(self):
         pass # make rpython happy
 
-    def propagate_begin_forward(self):
-        if self.next_optimization:
-            self.next_optimization.propagate_begin_forward()
-
-    def propagate_end_forward(self):
-        if self.next_optimization:
-            self.next_optimization.propagate_end_forward()
-
     def propagate_forward(self, op):
         raise NotImplementedError
 
     def emit_operation(self, op):
         self.next_optimization.propagate_forward(op)
 
-    def test_emittable(self, op):
-        return self.is_emittable(op)
-
-    def is_emittable(self, op):
-        return self.next_optimization.test_emittable(op)
-
     # FIXME: Move some of these here?
     def getvalue(self, box):
         return self.optimizer.getvalue(box)
@@ -289,19 +275,19 @@
         return self.optimizer.new_const_item(arraydescr)
 
     def pure(self, opnum, args, result):
-        op = ResOperation(opnum, args, result)
-        key = self.optimizer.make_args_key(op)
-        if key not in self.optimizer.pure_operations:
-            self.optimizer.pure_operations[key] = op
+        if self.optimizer.optpure:
+            self.optimizer.optpure.pure(opnum, args, result)
 
     def has_pure_result(self, opnum, args, descr):
-        op = ResOperation(opnum, args, None, descr)
-        key = self.optimizer.make_args_key(op)
-        op = self.optimizer.pure_operations.get(key, None)
-        if op is None:
-            return False
-        return op.getdescr() is descr
+        if self.optimizer.optpure:
+            return self.optimizer.optpure.has_pure_result(opnum, args, descr)
+        return False
 
+    def get_pure_result(self, key):    
+        if self.optimizer.optpure:
+            return self.optimizer.optpure.get_pure_result(key)
+        return None
+    
     def setup(self):
         pass
 
@@ -322,6 +308,9 @@
     def produce_potential_short_preamble_ops(self, potential_ops):
         pass
 
+    def forget_numberings(self, box):
+        self.optimizer.forget_numberings(box)
+
 class Optimizer(Optimization):
 
     def __init__(self, metainterp_sd, loop, optimizations=None, bridge=False):
@@ -333,14 +322,16 @@
         self.interned_refs = self.cpu.ts.new_ref_dict()
         self.resumedata_memo = resume.ResumeDataLoopMemo(metainterp_sd)
         self.bool_boxes = {}
-        self.pure_operations = args_dict()
         self.producer = {}
         self.pendingfields = []
-        self.posponedop = None
         self.exception_might_have_happened = False
         self.quasi_immutable_deps = None
         self.opaque_pointers = {}
-        self.newoperations = []
+        self.replaces_guard = {}
+        self._newoperations = []
+        self.optimizer = self
+        self.optpure = None
+        self.optearlyforce = None
         if loop is not None:
             self.call_pure_results = loop.call_pure_results
 
@@ -369,21 +360,20 @@
     def flush(self):
         for o in self.optimizations:
             o.flush()
-        assert self.posponedop is None
 
     def new(self):
         new = Optimizer(self.metainterp_sd, self.loop)
         return self._new(new)
 
     def _new(self, new):
-        assert self.posponedop is None
         optimizations = [o.new() for o in self.optimizations]
         new.set_optimizations(optimizations)
         new.quasi_immutable_deps = self.quasi_immutable_deps
         return new
 
     def produce_potential_short_preamble_ops(self, sb):
-        raise NotImplementedError('This is implemented in unroll.UnrollableOptimizer')
+        for opt in self.optimizations:
+            opt.produce_potential_short_preamble_ops(sb)
 
     def turned_constant(self, value):
         for o in self.optimizations:
@@ -433,6 +423,13 @@
             return constbox
         return None
 
+    def get_newoperations(self):
+        self.flush()
+        return self._newoperations
+
+    def clear_newoperations(self):
+        self._newoperations = []
+
     def make_equal_to(self, box, value, replace=False):
         assert isinstance(value, OptValue)
         assert replace or box not in self.values
@@ -481,15 +478,10 @@
 
     def propagate_all_forward(self):
         self.exception_might_have_happened = self.bridge
-        self.newoperations = []
-        self.first_optimization.propagate_begin_forward()
-        self.i = 0
-        while self.i < len(self.loop.operations):
-            op = self.loop.operations[self.i]
+        self.clear_newoperations()
+        for op in self.loop.operations:
             self.first_optimization.propagate_forward(op)
-            self.i += 1
-        self.first_optimization.propagate_end_forward()
-        self.loop.operations = self.newoperations
+        self.loop.operations = self.get_newoperations()
         self.loop.quasi_immutable_deps = self.quasi_immutable_deps
         # accumulate counters
         self.resumedata_memo.update_counters(self.metainterp_sd.profiler)
@@ -501,9 +493,6 @@
         self.producer[op.result] = op
         dispatch_opt(self, op)
 
-    def test_emittable(self, op):
-        return True
-
     def emit_operation(self, op):
         if op.returns_bool_result():
             self.bool_boxes[self.getvalue(op.result)] = None
@@ -519,14 +508,30 @@
                 pass
             else:
                 self.ensure_imported(value)
-                op.setarg(i, value.force_box())
+                op.setarg(i, value.force_box(self))
         self.metainterp_sd.profiler.count(jitprof.OPT_OPS)
         if op.is_guard():
             self.metainterp_sd.profiler.count(jitprof.OPT_GUARDS)
-            op = self.store_final_boxes_in_guard(op)
+            if self.replaces_guard and op in self.replaces_guard:
+                self.replace_op(self.replaces_guard[op], op)
+                del self.replaces_guard[op]
+                return
+            else:
+                op = self.store_final_boxes_in_guard(op)
         elif op.can_raise():
             self.exception_might_have_happened = True
-        self.newoperations.append(op)
+        self._newoperations.append(op)
+
+    def replace_op(self, old_op, new_op):
+        # XXX: Do we want to cache indexes to prevent search?
+        i = len(self._newoperations) 
+        while i > 0:
+            i -= 1
+            if self._newoperations[i] is old_op:
+                self._newoperations[i] = new_op
+                break
+        else:
+            assert False
 
     def store_final_boxes_in_guard(self, op):
         descr = op.getdescr()
@@ -574,51 +579,8 @@
         args[n + 1] = op.getdescr()
         return args
 
-    @specialize.argtype(0)
     def optimize_default(self, op):
-        canfold = op.is_always_pure()
-        if op.is_ovf():
-            self.posponedop = op
-            return
-        if self.posponedop:
-            nextop = op
-            op = self.posponedop
-            self.posponedop = None
-            canfold = nextop.getopnum() == rop.GUARD_NO_OVERFLOW
-        else:
-            nextop = None
-
-        if canfold:
-            for i in range(op.numargs()):
-                if self.get_constant_box(op.getarg(i)) is None:
-                    break
-            else:
-                # all constant arguments: constant-fold away
-                resbox = self.constant_fold(op)
-                # note that INT_xxx_OVF is not done from here, and the
-                # overflows in the INT_xxx operations are ignored
-                self.make_constant(op.result, resbox)
-                return
-
-            # did we do the exact same operation already?
-            args = self.make_args_key(op)
-            oldop = self.pure_operations.get(args, None)
-            if oldop is not None and oldop.getdescr() is op.getdescr():
-                assert oldop.getopnum() == op.getopnum()
-                self.make_equal_to(op.result, self.getvalue(oldop.result),
-                                   True)
-                return
-            else:
-                self.pure_operations[args] = op
-                self.remember_emitting_pure(op)
-
-        # otherwise, the operation remains
         self.emit_operation(op)
-        if nextop:
-            self.emit_operation(nextop)
-
-    def remember_emitting_pure(self, op):
-        pass
 
     def constant_fold(self, op):
         argboxes = [self.get_constant_box(op.getarg(i))
@@ -636,11 +598,6 @@
     def optimize_DEBUG_MERGE_POINT(self, op):
         self.emit_operation(op)
 
-    def optimize_CAST_OPAQUE_PTR(self, op):
-        value = self.getvalue(op.getarg(0))
-        self.opaque_pointers[value] = True
-        self.make_equal_to(op.result, value)
-
     def optimize_GETARRAYITEM_GC_PURE(self, op):
         indexvalue = self.getvalue(op.getarg(1))
         if indexvalue.is_constant():
diff --git a/pypy/jit/metainterp/optimizeopt/pure.py b/pypy/jit/metainterp/optimizeopt/pure.py
new file mode 100644
--- /dev/null
+++ b/pypy/jit/metainterp/optimizeopt/pure.py
@@ -0,0 +1,115 @@
+from pypy.jit.metainterp.optimizeopt.optimizer import Optimization
+from pypy.jit.metainterp.resoperation import rop, ResOperation
+from pypy.jit.metainterp.optimizeopt.util import (make_dispatcher_method,
+    args_dict)
+
+class OptPure(Optimization):
+    def __init__(self):
+        self.posponedop = None
+        self.pure_operations = args_dict()
+        self.emitted_pure_operations = {}
+
+    def propagate_forward(self, op):
+        dispatch_opt(self, op)
+
+    def optimize_default(self, op):
+        canfold = op.is_always_pure()
+        if op.is_ovf():
+            self.posponedop = op
+            return
+        if self.posponedop:
+            nextop = op
+            op = self.posponedop
+            self.posponedop = None
+            canfold = nextop.getopnum() == rop.GUARD_NO_OVERFLOW
+        else:
+            nextop = None
+
+        if canfold:
+            for i in range(op.numargs()):
+                if self.get_constant_box(op.getarg(i)) is None:
+                    break
+            else:
+                # all constant arguments: constant-fold away
+                resbox = self.optimizer.constant_fold(op)
+                # note that INT_xxx_OVF is not done from here, and the
+                # overflows in the INT_xxx operations are ignored
+                self.optimizer.make_constant(op.result, resbox)
+                return
+
+            # did we do the exact same operation already?
+            args = self.optimizer.make_args_key(op)
+            oldop = self.pure_operations.get(args, None)
+            if oldop is not None and oldop.getdescr() is op.getdescr():
+                assert oldop.getopnum() == op.getopnum()
+                self.optimizer.make_equal_to(op.result, self.getvalue(oldop.result),
+                                   True)
+                return
+            else:
+                self.pure_operations[args] = op
+                self.remember_emitting_pure(op)
+
+        # otherwise, the operation remains
+        self.emit_operation(op)
+        if op.returns_bool_result():
+            self.optimizer.bool_boxes[self.getvalue(op.result)] = None        
+        if nextop:
+            self.emit_operation(nextop)
+
+    def optimize_CALL_PURE(self, op):
+        args = self.optimizer.make_args_key(op)
+        oldop = self.pure_operations.get(args, None)
+        if oldop is not None and oldop.getdescr() is op.getdescr():
+            assert oldop.getopnum() == op.getopnum()
+            self.make_equal_to(op.result, self.getvalue(oldop.result))
+            return
+        else:
+            self.pure_operations[args] = op
+            self.remember_emitting_pure(op)
+
+        # replace CALL_PURE with just CALL
+        args = op.getarglist()
+        self.emit_operation(ResOperation(rop.CALL, args, op.result,
+                                         op.getdescr()))
+
+    def flush(self):
+        assert self.posponedop is None
+
+    def new(self):
+        assert self.posponedop is None
+        return OptPure()
+
+    def setup(self):
+        self.optimizer.optpure = self
+
+    def pure(self, opnum, args, result):
+        op = ResOperation(opnum, args, result)
+        key = self.optimizer.make_args_key(op)
+        if key not in self.pure_operations:
+            self.pure_operations[key] = op
+
+    def has_pure_result(self, opnum, args, descr):
+        op = ResOperation(opnum, args, None, descr)
+        key = self.optimizer.make_args_key(op)
+        op = self.pure_operations.get(key, None)
+        if op is None:
+            return False
+        return op.getdescr() is descr
+
+    def get_pure_result(self, key):
+        return self.pure_operations.get(key, None)
+
+    def remember_emitting_pure(self, op):
+        self.emitted_pure_operations[op] = True
+
+    def produce_potential_short_preamble_ops(self, sb):
+        for op in self.emitted_pure_operations:
+            if op.getopnum() == rop.GETARRAYITEM_GC_PURE or \
+               op.getopnum() == rop.STRGETITEM or \
+               op.getopnum() == rop.UNICODEGETITEM:
+                if not self.getvalue(op.getarg(1)).is_constant():
+                    continue
+            sb.add_potential(op)
+
+dispatch_opt = make_dispatcher_method(OptPure, 'optimize_',
+                                      default=OptPure.optimize_default)
diff --git a/pypy/jit/metainterp/optimizeopt/rewrite.py b/pypy/jit/metainterp/optimizeopt/rewrite.py
--- a/pypy/jit/metainterp/optimizeopt/rewrite.py
+++ b/pypy/jit/metainterp/optimizeopt/rewrite.py
@@ -31,21 +31,8 @@
 
         dispatch_opt(self, op)
 
-    def test_emittable(self, op):
-        opnum = op.getopnum()
-        for value, cls, func in optimize_guards:
-            if opnum == value:
-                assert isinstance(op, cls)
-                try:
-                    func(self, op, dryrun=True)
-                    return self.is_emittable(op)
-                except InvalidLoop:
-                    return False
-        return self.is_emittable(op)
-
-
     def try_boolinvers(self, op, targs):
-        oldop = self.optimizer.pure_operations.get(targs, None)
+        oldop = self.get_pure_result(targs)
         if oldop is not None and oldop.getdescr() is op.getdescr():
             value = self.getvalue(oldop.result)
             if value.is_constant():
@@ -62,32 +49,35 @@
     def find_rewritable_bool(self, op, args):
         try:
             oldopnum = opboolinvers[op.getopnum()]
+        except KeyError:
+            pass
+        else:
             targs = self.optimizer.make_args_key(ResOperation(oldopnum, [args[0], args[1]],
                                                               None))
             if self.try_boolinvers(op, targs):
                 return True
-        except KeyError:
-            pass
 
         try:
             oldopnum = opboolreflex[op.getopnum()] # FIXME: add INT_ADD, INT_MUL
+        except KeyError:
+            pass
+        else:
             targs = self.optimizer.make_args_key(ResOperation(oldopnum, [args[1], args[0]],
                                                               None))
-            oldop = self.optimizer.pure_operations.get(targs, None)
+            oldop = self.get_pure_result(targs)
             if oldop is not None and oldop.getdescr() is op.getdescr():
                 self.make_equal_to(op.result, self.getvalue(oldop.result))
                 return True
-        except KeyError:
-            pass
 
         try:
             oldopnum = opboolinvers[opboolreflex[op.getopnum()]]
+        except KeyError:
+            pass
+        else:
             targs = self.optimizer.make_args_key(ResOperation(oldopnum, [args[1], args[0]],
                                                               None))
             if self.try_boolinvers(op, targs):
                 return True
-        except KeyError:
-            pass
 
         return False
 
@@ -214,40 +204,7 @@
         self.emit_operation(op)
         self.pure(rop.FLOAT_NEG, [op.result], v1)
 
-    def optimize_CALL_PURE(self, op):
-        arg_consts = []
-        for i in range(op.numargs()):
-            arg = op.getarg(i)
-            const = self.get_constant_box(arg)
-            if const is None:
-                break
-            arg_consts.append(const)
-        else:
-            # all constant arguments: check if we already know the result
-            try:
-                result = self.optimizer.call_pure_results[arg_consts]
-            except KeyError:
-                pass
-            else:
-                self.make_constant(op.result, result)
-                return
-
-        args = self.optimizer.make_args_key(op)
-        oldop = self.optimizer.pure_operations.get(args, None)
-        if oldop is not None and oldop.getdescr() is op.getdescr():
-            assert oldop.getopnum() == op.getopnum()
-            self.make_equal_to(op.result, self.getvalue(oldop.result))
-            return
-        else:
-            self.optimizer.pure_operations[args] = op
-            self.optimizer.remember_emitting_pure(op)
-
-        # replace CALL_PURE with just CALL
-        args = op.getarglist()
-        self.emit_operation(ResOperation(rop.CALL, args, op.result,
-                                         op.getdescr()))
-
-    def optimize_guard(self, op, constbox, emit_operation=True, dryrun=False):
+    def optimize_guard(self, op, constbox, emit_operation=True):
         value = self.getvalue(op.getarg(0))
         if value.is_constant():
             box = value.box
@@ -255,62 +212,57 @@
             if not box.same_constant(constbox):
                 raise InvalidLoop
             return
-        if dryrun: return
         if emit_operation:
             self.emit_operation(op)
         value.make_constant(constbox)
         self.optimizer.turned_constant(value)
 
-    def optimize_GUARD_ISNULL(self, op, dryrun=False):
+    def optimize_GUARD_ISNULL(self, op):
         value = self.getvalue(op.getarg(0))
         if value.is_null():
             return
         elif value.is_nonnull():
             raise InvalidLoop
-        if dryrun: return
         self.emit_operation(op)
         value.make_constant(self.optimizer.cpu.ts.CONST_NULL)
 
-    def optimize_GUARD_NONNULL(self, op, dryrun=False):
+    def optimize_GUARD_NONNULL(self, op):
         value = self.getvalue(op.getarg(0))
         if value.is_nonnull():
             return
         elif value.is_null():
             raise InvalidLoop
-        if dryrun: return
         self.emit_operation(op)
-        value.make_nonnull(len(self.optimizer.newoperations) - 1)
+        value.make_nonnull(op)
 
-    def optimize_GUARD_VALUE(self, op, dryrun=False):
+    def optimize_GUARD_VALUE(self, op):
         value = self.getvalue(op.getarg(0))
-        emit_operation = True
-        if not dryrun and value.last_guard_index != -1:
+        if value.last_guard:
             # there already has been a guard_nonnull or guard_class or
             # guard_nonnull_class on this value, which is rather silly.
             # replace the original guard with a guard_value
-            old_guard_op = self.optimizer.newoperations[value.last_guard_index]
-            new_guard_op = old_guard_op.copy_and_change(rop.GUARD_VALUE,
-                                             args = [old_guard_op.getarg(0), op.getarg(1)])
-            self.optimizer.newoperations[value.last_guard_index] = new_guard_op
+            old_guard_op = value.last_guard
+            op = old_guard_op.copy_and_change(rop.GUARD_VALUE,
+                                      args = [old_guard_op.getarg(0), op.getarg(1)])
+            self.optimizer.replaces_guard[op] = old_guard_op
             # hack hack hack.  Change the guard_opnum on
             # new_guard_op.getdescr() so that when resuming,
             # the operation is not skipped by pyjitpl.py.
-            descr = new_guard_op.getdescr()
+            descr = op.getdescr()
             assert isinstance(descr, compile.ResumeGuardDescr)
             descr.guard_opnum = rop.GUARD_VALUE
-            descr.make_a_counter_per_value(new_guard_op)
-            emit_operation = False
+            descr.make_a_counter_per_value(op)
         constbox = op.getarg(1)
         assert isinstance(constbox, Const)
-        self.optimize_guard(op, constbox, emit_operation, dryrun)
+        self.optimize_guard(op, constbox)
 
-    def optimize_GUARD_TRUE(self, op, dryrun=False):
-        self.optimize_guard(op, CONST_1, dryrun=dryrun)
+    def optimize_GUARD_TRUE(self, op):
+        self.optimize_guard(op, CONST_1)
 
-    def optimize_GUARD_FALSE(self, op, dryrun=False):
-        self.optimize_guard(op, CONST_0, dryrun=dryrun)
+    def optimize_GUARD_FALSE(self, op):
+        self.optimize_guard(op, CONST_0)
 
-    def optimize_GUARD_CLASS(self, op, dryrun=False):
+    def optimize_GUARD_CLASS(self, op):
         value = self.getvalue(op.getarg(0))
         expectedclassbox = op.getarg(1)
         assert isinstance(expectedclassbox, Const)
@@ -319,38 +271,32 @@
             if realclassbox.same_constant(expectedclassbox):
                 return
             raise InvalidLoop
-        if dryrun: return
-        emit_operation = True
-        if value.last_guard_index != -1:
+        if value.last_guard:
             # there already has been a guard_nonnull or guard_class or
             # guard_nonnull_class on this value.
-            old_guard_op = self.optimizer.newoperations[value.last_guard_index]
+            old_guard_op = value.last_guard
             if old_guard_op.getopnum() == rop.GUARD_NONNULL:
                 # it was a guard_nonnull, which we replace with a
                 # guard_nonnull_class.
-                new_guard_op = old_guard_op.copy_and_change (rop.GUARD_NONNULL_CLASS,
+                op = old_guard_op.copy_and_change (rop.GUARD_NONNULL_CLASS,
                                          args = [old_guard_op.getarg(0), op.getarg(1)])
-                self.optimizer.newoperations[value.last_guard_index] = new_guard_op
+                self.optimizer.replaces_guard[op] = old_guard_op
                 # hack hack hack.  Change the guard_opnum on
                 # new_guard_op.getdescr() so that when resuming,
                 # the operation is not skipped by pyjitpl.py.
-                descr = new_guard_op.getdescr()
+                descr = op.getdescr()
                 assert isinstance(descr, compile.ResumeGuardDescr)
                 descr.guard_opnum = rop.GUARD_NONNULL_CLASS
-                emit_operation = False
-        if emit_operation:
-            self.emit_operation(op)
-            last_guard_index = len(self.optimizer.newoperations) - 1
-        else:
-            last_guard_index = value.last_guard_index
-        value.make_constant_class(expectedclassbox, last_guard_index)
+        self.emit_operation(op)
+        value.make_constant_class(expectedclassbox, op)
 
-    def optimize_GUARD_NONNULL_CLASS(self, op, dryrun=False):
-        self.optimize_GUARD_NONNULL(op, True)
-        self.optimize_GUARD_CLASS(op, dryrun)
+    def optimize_GUARD_NONNULL_CLASS(self, op):
+        value = self.getvalue(op.getarg(0))
+        if value.is_null():
+            raise InvalidLoop
+        self.optimize_GUARD_CLASS(op)
 
-    def optimize_GUARD_NO_EXCEPTION(self, op, dryrun=False):
-        if dryrun: return
+    def optimize_GUARD_NO_EXCEPTION(self, op):
         if not self.optimizer.exception_might_have_happened:
             return
         self.emit_operation(op)
@@ -470,7 +416,7 @@
                     newop = ResOperation(rop.SETARRAYITEM_GC,
                                          [op.getarg(2),
                                           ConstInt(index + dest_start),
-                                          val.force_box()], None,
+                                          val.get_key_box()], None,
                                          descr=source_value.arraydescr)
                     self.emit_operation(newop)
             return True
@@ -478,6 +424,25 @@
             return True # 0-length arraycopy
         return False
 
+    def optimize_CALL_PURE(self, op):
+        arg_consts = []
+        for i in range(op.numargs()):
+            arg = op.getarg(i)
+            const = self.get_constant_box(arg)
+            if const is None:
+                break
+            arg_consts.append(const)
+        else:
+            # all constant arguments: check if we already know the result
+            try:
+                result = self.optimizer.call_pure_results[arg_consts]
+            except KeyError:
+                pass
+            else:
+                self.make_constant(op.result, result)
+                return
+        self.emit_operation(op)
+
     def optimize_INT_FLOORDIV(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
@@ -492,6 +457,10 @@
                                         args = [op.getarg(0), ConstInt(highest_bit(val))])
         self.emit_operation(op)
 
+    def optimize_CAST_OPAQUE_PTR(self, op):
+        value = self.getvalue(op.getarg(0))
+        self.optimizer.opaque_pointers[value] = True
+        self.make_equal_to(op.result, value)
 
 dispatch_opt = make_dispatcher_method(OptRewrite, 'optimize_',
         default=OptRewrite.emit_operation)
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -39,8 +39,9 @@
 
 def test_sharing_field_lists_of_virtual():
     class FakeOptimizer(object):
-        class cpu(object):
-            pass
+        class optimizer(object):
+            class cpu(object):
+                pass
     opt = FakeOptimizer()
     virt1 = virtualize.AbstractVirtualStructValue(opt, None)
     lst1 = virt1._get_field_descr_list()
@@ -69,7 +70,7 @@
     class FakeVirtualValue(virtualize.AbstractVirtualValue):
         def _make_virtual(self, *args):
             return FakeVInfo()
-    v1 = FakeVirtualValue(None, None, None)
+    v1 = FakeVirtualValue(None, None)
     vinfo1 = v1.make_virtual_info(None, [1, 2, 4])
     vinfo2 = v1.make_virtual_info(None, [1, 2, 4])
     assert vinfo1 is vinfo2
@@ -111,7 +112,7 @@
 
 class BaseTestBasic(BaseTest):
 
-    enable_opts = "intbounds:rewrite:virtualize:string:heap"
+    enable_opts = "intbounds:rewrite:virtualize:string:earlyforce:pure:heap"
 
     def optimize_loop(self, ops, optops, call_pure_results=None):
 
@@ -651,8 +652,8 @@
         i3 = getfield_gc(p3, descr=valuedescr)
         escape(i3)
         p1 = new_with_vtable(ConstClass(node_vtable))
+        p1sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p1, i1, descr=valuedescr)
-        p1sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p1sub, i1, descr=valuedescr)
         setfield_gc(p1, p1sub, descr=nextdescr)
         jump(i1, p1, p2)
@@ -667,10 +668,10 @@
         p3sub = getfield_gc(p3, descr=nextdescr)
         i3 = getfield_gc(p3sub, descr=valuedescr)
         escape(i3)
+        p1 = new_with_vtable(ConstClass(node_vtable))
         p2sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p2sub, i1, descr=valuedescr)
         setfield_gc(p2, p2sub, descr=nextdescr)
-        p1 = new_with_vtable(ConstClass(node_vtable))
         jump(i1, p1, p2)
         """
         # The same as test_p123_simple, but in the end the "old" p2 contains
@@ -2328,7 +2329,7 @@
         def _variables_equal(box, varname, strict):
             if varname not in virtuals:
                 if strict:
-                    assert box == oparse.getvar(varname)
+                    assert box.same_box(oparse.getvar(varname))
                 else:
                     assert box.value == oparse.getvar(varname).value
             else:
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizefficall.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizefficall.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizefficall.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizefficall.py
@@ -36,7 +36,7 @@
 
 class TestFfiCall(BaseTestBasic, LLtypeMixin):
 
-    enable_opts = "intbounds:rewrite:virtualize:string:heap:ffi"
+    enable_opts = "intbounds:rewrite:virtualize:string:pure:earlyforce:heap:ffi"
 
     class namespace:
         cpu = LLtypeMixin.cpu
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -68,7 +68,7 @@
 
 class BaseTestWithUnroll(BaseTest):
 
-    enable_opts = "intbounds:rewrite:virtualize:string:heap:unroll"
+    enable_opts = "intbounds:rewrite:virtualize:string:earlyforce:pure:heap:unroll"
 
     def optimize_loop(self, ops, expected, expected_preamble=None,
                       call_pure_results=None, expected_short=None):
@@ -825,8 +825,8 @@
         i3 = getfield_gc(p2, descr=valuedescr)
         escape(i3)
         p4 = new_with_vtable(ConstClass(node_vtable))
+        p1sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p4, i1, descr=valuedescr)
-        p1sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p1sub, i1, descr=valuedescr)
         setfield_gc(p4, p1sub, descr=nextdescr)
         jump(i1, p4)
@@ -865,13 +865,7 @@
         p3sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p3sub, i1, descr=valuedescr)
         setfield_gc(p1, p3sub, descr=nextdescr)
-        # XXX: We get two extra operations here because the setfield
-        #      above is the result of forcing p1 and thus not
-        #      registered with the heap optimizer. I've makred tests
-        #      below with VIRTUALHEAP if they suffer from this issue
-        p3sub2 = getfield_gc(p1, descr=nextdescr)
-        guard_nonnull_class(p3sub2, ConstClass(node_vtable2)) []
-        jump(i1, p1, p3sub2)
+        jump(i1, p1, p3sub)
         """
         self.optimize_loop(ops, expected, preamble)
 
@@ -902,9 +896,7 @@
         guard_true(i2b) []
         p3 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3, i2, descr=nextdescr)
-        # XXX: VIRTUALHEAP (see above)
-        i3 = getfield_gc(p3, descr=nextdescr)
-        jump(p3, i3)
+        jump(p3, i2)
         """
         self.optimize_loop(ops, expected, preamble)
 
@@ -1219,7 +1211,15 @@
         setfield_gc(p3, p30, descr=valuedescr)
         jump(i29, p30, p3)
         """
-        expected = preamble
+        expected = """
+        [i0, p1, p3]
+        i28 = int_add(i0, 1)
+        i29 = int_add(i28, 1)
+        p30 = new_with_vtable(ConstClass(node_vtable))
+        setfield_gc(p3, p30, descr=valuedescr)
+        setfield_gc(p30, i28, descr=nextdescr)
+        jump(i29, p30, p3)
+        """
         self.optimize_loop(ops, expected, preamble)
 
     def test_nonvirtual_1(self):
@@ -2408,8 +2408,8 @@
         guard_class(p2, ConstClass(node_vtable)) []
         p3 = getfield_gc(p1, descr=otherdescr)
         guard_class(p3, ConstClass(node_vtable)) []
+        p3a = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3, p2, descr=otherdescr)
-        p3a = new_with_vtable(ConstClass(node_vtable))
         escape(p3a)
         jump(p3a)
         """
@@ -2421,9 +2421,9 @@
         # setfield_gc(p3, p2, descr=otherdescr) # p3a.other = p2a
         # p1a = new_with_vtable(ConstClass(node_vtable2))
         # p2a = new_with_vtable(ConstClass(node_vtable))
+        p3anew = new_with_vtable(ConstClass(node_vtable))
         p2 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3a, p2, descr=otherdescr) # p3a.other = p2a
-        p3anew = new_with_vtable(ConstClass(node_vtable))
         escape(p3anew)
         jump(p3anew)
         """
@@ -2458,9 +2458,9 @@
         p3 = getfield_gc(p1, descr=otherdescr)
         guard_class(p3, ConstClass(node_vtable)) []
         # p1a = new_with_vtable(ConstClass(node_vtable2))
+        p3a = new_with_vtable(ConstClass(node_vtable))
         p2a = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3, p2a, descr=otherdescr)
-        p3a = new_with_vtable(ConstClass(node_vtable))
         escape(p3a)
         # setfield_gc(p1a, p2a, descr=nextdescr)
         # setfield_gc(p1a, p3a, descr=otherdescr)
@@ -2468,9 +2468,9 @@
         """
         expected = """
         [p2, p3]
+        p3a = new_with_vtable(ConstClass(node_vtable))
         p2a = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3, p2a, descr=otherdescr)
-        p3a = new_with_vtable(ConstClass(node_vtable))
         escape(p3a)
         jump(p2a, p3a)
         """
@@ -2790,7 +2790,6 @@
         self.optimize_loop(ops, expected, preamble)
 
     def test_remove_duplicate_pure_op_ovf_with_lazy_setfield(self):
-        py.test.skip('this optimization is not yet supprted')
         ops = """
         [i1, p1]
         i3 = int_add_ovf(i1, 1)
@@ -5961,13 +5960,18 @@
         escape(i0)
         jump(p1)
         """
-        expected = """
+        preamble = """
         [p1]
         i0 = ptr_eq(p1, NULL)
         escape(i0)
-        jump(p1)
-        """
-        self.optimize_strunicode_loop_extradescrs(ops, expected, expected)
+        jump(p1, i0)
+        """
+        expected = """
+        [p1, i0]
+        escape(i0)
+        jump(p1, i0)
+        """
+        self.optimize_strunicode_loop_extradescrs(ops, expected, preamble)
 
     def test_str_equal_none2(self):
         ops = """
@@ -5976,13 +5980,18 @@
         escape(i0)
         jump(p1)
         """
-        expected = """
+        preamble = """
         [p1]
         i0 = ptr_eq(p1, NULL)
         escape(i0)
-        jump(p1)
-        """
-        self.optimize_strunicode_loop_extradescrs(ops, expected, expected)
+        jump(p1, i0)
+        """
+        expected = """
+        [p1, i0]
+        escape(i0)
+        jump(p1, i0)
+        """
+        self.optimize_strunicode_loop_extradescrs(ops, expected, preamble)
 
     def test_str_equal_nonnull1(self):
         ops = """
@@ -7220,6 +7229,60 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_heap_cache_forced_virtuals(self):
+        ops = """
+        [i1, i2, p0]
+        p1 = new(descr=ssize)
+        setfield_gc(p1, i1, descr=adescr)
+        setfield_gc(p1, i2, descr=bdescr)
+        call(p0, p1, descr=writeadescr)
+        i3 = getfield_gc(p1, descr=adescr)
+        i4 = getfield_gc(p1, descr=bdescr)
+        jump(i3, i4, p0)
+        """
+        expected = """
+        [i1, i2, p0]
+        p1 = new(descr=ssize)
+        setfield_gc(p1, i1, descr=adescr)
+        call(p0, p1, descr=writeadescr)
+        i3 = getfield_gc(p1, descr=adescr)
+        setfield_gc(p1, i2, descr=bdescr)
+        jump(i3, i2, p0)
+        """
+        self.optimize_loop(ops, expected)
+
+
+    def test_setarrayitem_followed_by_arraycopy(self):
+        ops = """
+        [p1, p2]
+        setarrayitem_gc(p1, 2, 10, descr=arraydescr)
+        setarrayitem_gc(p2, 3, 13, descr=arraydescr)        
+        call(0, p1, p2, 0, 0, 10, descr=arraycopydescr)
+        jump(p1, p2)
+        """
+        self.optimize_loop(ops, ops)
+
+    def test_heap_cache_virtuals_forced_by_delayed_setfield(self):
+        py.test.skip('not yet supoprted')
+        ops = """
+        [i1, p0]
+        p1 = new(descr=ssize)
+        setfield_gc(p1, i1, descr=valuedescr)
+        setfield_gc(p0, p1, descr=adescr)
+        call(p0, descr=writeadescr)
+        i2 = getfield_gc(p1, descr=valuedescr)
+        jump(i2, p0)
+        """
+        expected = """
+        [i1, p0]
+        p1 = new(descr=ssize)
+        setfield_gc(p1, i1, descr=valuedescr)
+        setfield_gc(p0, p1, descr=adescr)
+        call(p0, descr=writeadescr)
+        jump(i1, p0)
+        """
+        self.optimize_loop(ops, expected)
+
 class TestLLtype(OptimizeOptTest, LLtypeMixin):
     pass
 
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_util.py b/pypy/jit/metainterp/optimizeopt/test/test_util.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_util.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_util.py
@@ -182,7 +182,8 @@
                             EffectInfo.EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE,
                             can_invalidate=True))
     arraycopydescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT,
-             EffectInfo([], [], [], [], oopspecindex=EffectInfo.OS_ARRAYCOPY))
+             EffectInfo([], [arraydescr], [], [arraydescr],
+                        oopspecindex=EffectInfo.OS_ARRAYCOPY))
 
     for _name, _os in [
         ('strconcatdescr',               'OS_STR_CONCAT'),
diff --git a/pypy/jit/metainterp/optimizeopt/unroll.py b/pypy/jit/metainterp/optimizeopt/unroll.py
--- a/pypy/jit/metainterp/optimizeopt/unroll.py
+++ b/pypy/jit/metainterp/optimizeopt/unroll.py
@@ -75,7 +75,6 @@
         self.importable_values = {}
         self.emitting_dissabled = False
         self.emitted_guards = 0
-        self.emitted_pure_operations = {}
 
     def ensure_imported(self, value):
         if not self.emitting_dissabled and value in self.importable_values:
@@ -96,21 +95,6 @@
         new = UnrollableOptimizer(self.metainterp_sd, self.loop)
         return self._new(new)
 
-    def remember_emitting_pure(self, op):
-        self.emitted_pure_operations[op] = True
-
-    def produce_potential_short_preamble_ops(self, sb):
-        for op in self.emitted_pure_operations:
-            if op.getopnum() == rop.GETARRAYITEM_GC_PURE or \
-               op.getopnum() == rop.STRGETITEM or \
-               op.getopnum() == rop.UNICODEGETITEM:
-                if not self.getvalue(op.getarg(1)).is_constant():
-                    continue
-            sb.add_potential(op)
-        for opt in self.optimizations:
-            opt.produce_potential_short_preamble_ops(sb)
-
-
 
 class UnrollOptimizer(Optimization):
     """Unroll the loop into two iterations. The first one will
@@ -154,7 +138,7 @@
 
             KillHugeIntBounds(self.optimizer).apply()
             
-            loop.preamble.operations = self.optimizer.newoperations
+            loop.preamble.operations = self.optimizer.get_newoperations()
             jump_args = [self.getvalue(a).get_key_box() for a in jump_args]
 
             start_resumedescr = loop.preamble.start_resumedescr.clone_if_mutable()
@@ -167,8 +151,9 @@
             virtual_state = modifier.get_virtual_state(jump_args)
             
             values = [self.getvalue(arg) for arg in jump_args]
-            inputargs = virtual_state.make_inputargs(values)
-            short_inputargs = virtual_state.make_inputargs(values, keyboxes=True)
+            inputargs = virtual_state.make_inputargs(values, self.optimizer)
+            short_inputargs = virtual_state.make_inputargs(values, self.optimizer,
+                                                           keyboxes=True)
 
             self.constant_inputargs = {}
             for box in jump_args: 
@@ -197,7 +182,7 @@
             # operations needed to setup the proper state of those virtuals
             # in the peeled loop
             inputarg_setup_ops = []
-            preamble_optimizer.newoperations = []
+            preamble_optimizer.clear_newoperations()
             seen = {}
             for box in inputargs:
                 if box in seen:
@@ -211,9 +196,8 @@
                     continue
                 seen[box] = True
                 value = preamble_optimizer.getvalue(box)
-                value.force_box()
-            preamble_optimizer.flush()
-            inputarg_setup_ops += preamble_optimizer.newoperations
+                value.force_box(preamble_optimizer)
+            inputarg_setup_ops += preamble_optimizer.get_newoperations()
 
             # Setup the state of the new optimizer by emiting the
             # short preamble operations and discarding the result
@@ -244,13 +228,13 @@
                                 virtual_state)
             
             loop.inputargs = inputargs
-            args = [preamble_optimizer.getvalue(self.short_boxes.original(a)).force_box()\
+            args = [preamble_optimizer.getvalue(self.short_boxes.original(a)).force_box(preamble_optimizer)\
                     for a in inputargs]
             jmp = ResOperation(rop.JUMP, args, None)
             jmp.setdescr(loop.token)
             loop.preamble.operations.append(jmp)
 
-            loop.operations = self.optimizer.newoperations
+            loop.operations = self.optimizer.get_newoperations()
             maxguards = self.optimizer.metainterp_sd.warmrunnerdesc.memory_manager.max_retrace_guards
             
             if self.optimizer.emitted_guards > maxguards:
@@ -335,9 +319,10 @@
         assert jumpop
         original_jumpargs = jumpop.getarglist()[:]
         values = [self.getvalue(arg) for arg in jumpop.getarglist()]
-        jumpargs = virtual_state.make_inputargs(values)
+        jumpargs = virtual_state.make_inputargs(values, self.optimizer)
         jumpop.initarglist(jumpargs)
-        jmp_to_short_args = virtual_state.make_inputargs(values, keyboxes=True)
+        jmp_to_short_args = virtual_state.make_inputargs(values, self.optimizer,
+                                                         keyboxes=True)
         self.short_inliner = Inliner(short_inputargs, jmp_to_short_args)
         
         for box, const in self.constant_inputargs.items():
@@ -347,11 +332,11 @@
             newop = self.short_inliner.inline_op(op)
             self.optimizer.send_extra_operation(newop)
         
-        self.optimizer.flush()
+        newoperations = self.optimizer.get_newoperations()
 
         i = j = 0
-        while i < len(self.optimizer.newoperations) or j < len(jumpargs):
-            if i == len(self.optimizer.newoperations):
+        while i < len(newoperations) or j < len(jumpargs):
+            if i == len(newoperations):
                 while j < len(jumpargs):
                     a = jumpargs[j]
                     if self.optimizer.loop.logops:
@@ -360,7 +345,7 @@
                                     jumpargs, short_seen)
                     j += 1
             else:
-                op = self.optimizer.newoperations[i]
+                op = newoperations[i]
 
                 self.boxes_created_this_iteration[op.result] = True
                 args = op.getarglist()
@@ -375,6 +360,7 @@
                     self.import_box(a, inputargs, short, short_jumpargs,
                                     jumpargs, short_seen)
                 i += 1
+            newoperations = self.optimizer.get_newoperations()
 
         jumpop.initarglist(jumpargs)
         self.optimizer.send_extra_operation(jumpop)
@@ -468,7 +454,7 @@
         inputargs.append(box)
         box = newresult
         if box in self.optimizer.values:
-            box = self.optimizer.values[box].force_box()
+            box = self.optimizer.values[box].force_box(self.optimizer)
         jumpargs.append(box)
         
 
@@ -483,11 +469,6 @@
         if op.getopnum() == rop.JUMP:
             loop_token = op.getdescr()
             assert isinstance(loop_token, LoopToken)
-            # FIXME: Use a tree, similar to the tree formed by the full
-            # preamble and it's bridges, instead of a list to save time and
-            # memory. This should also allow better behaviour in
-            # situations that the is_emittable() chain currently cant
-            # handle and the inlining fails unexpectedly belwo.
             short = loop_token.short_preamble
             if short:
                 args = op.getarglist()
@@ -523,7 +504,7 @@
 
                         values = [self.getvalue(arg)
                                   for arg in op.getarglist()]
-                        args = sh.virtual_state.make_inputargs(values,
+                        args = sh.virtual_state.make_inputargs(values, self.optimizer,
                                                                keyboxes=True)
                         inliner = Inliner(sh.inputargs, args)
                         
diff --git a/pypy/jit/metainterp/optimizeopt/util.py b/pypy/jit/metainterp/optimizeopt/util.py
--- a/pypy/jit/metainterp/optimizeopt/util.py
+++ b/pypy/jit/metainterp/optimizeopt/util.py
@@ -90,14 +90,11 @@
     for i in range(len(args1)):
         arg1 = args1[i]
         arg2 = args2[i]
-        if isinstance(arg1, history.Const):
-            if arg1.__class__ is not arg2.__class__:
+        if arg1 is None:
+            if arg2 is not None:
                 return False
-            if not arg1.same_constant(arg2):
-                return False
-        else:
-            if not arg1 is arg2:
-                return False
+        elif not arg1.same_box(arg2):
+            return False
     return True
 
 def args_hash(args):
@@ -106,10 +103,8 @@
     for arg in args:
         if arg is None:
             y = 17
-        elif isinstance(arg, history.Const):
+        else:
             y = arg._get_hash_()
-        else:
-            y = compute_identity_hash(arg)
         res = intmask((1000003 * res) ^ y)
     return res
 
@@ -145,9 +140,12 @@
         for i in range(op1.numargs()):
             x = op1.getarg(i)
             y = op2.getarg(i)
-            assert x == remap.get(y, y)
+            assert x.same_box(remap.get(y, y))
         if op2.result in remap:
-            assert op1.result == remap[op2.result]
+            if op2.result is None:
+                assert op1.result == remap[op2.result]
+            else:
+                assert op1.result.same_box(remap[op2.result])
         else:
             remap[op2.result] = op1.result
         if op1.getopnum() != rop.JUMP:      # xxx obscure
@@ -156,11 +154,20 @@
             assert len(op1.getfailargs()) == len(op2.getfailargs())
             if strict_fail_args:
                 for x, y in zip(op1.getfailargs(), op2.getfailargs()):
-                    assert x == remap.get(y, y)
+                    if x is None:
+                        assert remap.get(y, y) is None
+                    else:
+                        assert x.same_box(remap.get(y, y))
             else:
                 fail_args1 = set(op1.getfailargs())
                 fail_args2 = set([remap.get(y, y) for y in op2.getfailargs()])
-                assert fail_args1 == fail_args2
+                for x in fail_args1:
+                    for y in fail_args2:
+                        if x.same_box(y):
+                            fail_args2.remove(y)
+                            break
+                    else:
+                        assert False
     assert len(oplist1) == len(oplist2)
     print '-'*totwidth
     return True
diff --git a/pypy/jit/metainterp/optimizeopt/virtualize.py b/pypy/jit/metainterp/optimizeopt/virtualize.py
--- a/pypy/jit/metainterp/optimizeopt/virtualize.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualize.py
@@ -10,13 +10,12 @@
 
 
 class AbstractVirtualValue(optimizer.OptValue):
-    _attrs_ = ('optimizer', 'keybox', 'source_op', '_cached_vinfo')
+    _attrs_ = ('keybox', 'source_op', '_cached_vinfo')
     box = None
     level = optimizer.LEVEL_NONNULL
     _cached_vinfo = None
 
-    def __init__(self, optimizer, keybox, source_op=None):
-        self.optimizer = optimizer
+    def __init__(self, keybox, source_op=None):
         self.keybox = keybox   # only used as a key in dictionaries
         self.source_op = source_op  # the NEW_WITH_VTABLE/NEW_ARRAY operation
                                     # that builds this box
@@ -29,17 +28,17 @@
             return self.keybox
         return self.box
 
-    def force_box(self):
+    def force_box(self, optforce):
         if self.box is None:
-            self.optimizer.forget_numberings(self.keybox)
-            self._really_force()
+            optforce.forget_numberings(self.keybox)
+            self._really_force(optforce)
         return self.box
 
-    def force_at_end_of_preamble(self, already_forced):
+    def force_at_end_of_preamble(self, already_forced, optforce):
         value = already_forced.get(self, None)
         if value:
             return value
-        return OptValue(self.force_box())
+        return OptValue(self.force_box(optforce))
 
     def make_virtual_info(self, modifier, fieldnums):
         if fieldnums is None:
@@ -55,7 +54,7 @@
     def _make_virtual(self, modifier):
         raise NotImplementedError("abstract base")
 
-    def _really_force(self):
+    def _really_force(self, optforce):
         raise NotImplementedError("abstract base")
 
     def import_from(self, other, optimizer):
@@ -70,10 +69,11 @@
 get_fielddescrlist_cache._annspecialcase_ = "specialize:memo"
 
 class AbstractVirtualStructValue(AbstractVirtualValue):
-    _attrs_ = ('_fields', '_cached_sorted_fields')
+    _attrs_ = ('_fields', 'cpu', '_cached_sorted_fields')
 
-    def __init__(self, optimizer, keybox, source_op=None):
-        AbstractVirtualValue.__init__(self, optimizer, keybox, source_op)
+    def __init__(self, cpu, keybox, source_op=None):
+        AbstractVirtualValue.__init__(self, keybox, source_op)
+        self.cpu = cpu
         self._fields = {}
         self._cached_sorted_fields = None
 
@@ -87,45 +87,44 @@
     def _get_descr(self):
         raise NotImplementedError
 
-    def _is_immutable_and_filled_with_constants(self):
+    def _is_immutable_and_filled_with_constants(self, optforce):
         count = self._get_descr().count_fields_if_immutable()
         if count != len(self._fields):    # always the case if count == -1
             return False
         for value in self._fields.itervalues():
-            subbox = value.force_box()
+            subbox = value.force_box(optforce)
             if not isinstance(subbox, Const):
                 return False
         return True
 
-    def force_at_end_of_preamble(self, already_forced):
+    def force_at_end_of_preamble(self, already_forced, optforce):
         if self in already_forced:
             return self
         already_forced[self] = self
         if self._fields:
             for ofs in self._fields.keys():
-                self._fields[ofs] = self._fields[ofs].force_at_end_of_preamble(already_forced)
+                self._fields[ofs] = self._fields[ofs].force_at_end_of_preamble(already_forced, optforce)
         return self
 
-    def _really_force(self):
+    def _really_force(self, optforce):
         op = self.source_op
         assert op is not None
         # ^^^ This case should not occur any more (see test_bug_3).
         #
         if not we_are_translated():
             op.name = 'FORCE ' + self.source_op.name
-
-        if self._is_immutable_and_filled_with_constants():
-            box = self.optimizer.constant_fold(op)
+            
+        if self._is_immutable_and_filled_with_constants(optforce):
+            box = optforce.optimizer.constant_fold(op)
             self.make_constant(box)
             for ofs, value in self._fields.iteritems():
-                subbox = value.force_box()
+                subbox = value.force_box(optforce)
                 assert isinstance(subbox, Const)
-                execute(self.optimizer.cpu, None, rop.SETFIELD_GC,
+                execute(optforce.optimizer.cpu, None, rop.SETFIELD_GC,
                         ofs, box, subbox)
             # keep self._fields, because it's all immutable anyway
         else:
-            newoperations = self.optimizer.newoperations
-            newoperations.append(op)
+            optforce.emit_operation(op)
             self.box = box = op.result
             #
             iteritems = self._fields.iteritems()
@@ -135,10 +134,11 @@
             for ofs, value in iteritems:
                 if value.is_null():
                     continue
-                subbox = value.force_box()
+                subbox = value.force_box(optforce)
                 op = ResOperation(rop.SETFIELD_GC, [box, subbox], None,
                                   descr=ofs)
-                newoperations.append(op)
+
+                optforce.emit_operation(op)
 
     def _get_field_descr_list(self):
         _cached_sorted_fields = self._cached_sorted_fields
@@ -155,7 +155,7 @@
             else:
                 lst = self._fields.keys()
             sort_descrs(lst)
-            cache = get_fielddescrlist_cache(self.optimizer.cpu)
+            cache = get_fielddescrlist_cache(self.cpu)
             result = cache.get(lst, None)
             if result is None:
                 cache[lst] = lst
@@ -180,8 +180,8 @@
 class VirtualValue(AbstractVirtualStructValue):
     level = optimizer.LEVEL_KNOWNCLASS
 
-    def __init__(self, optimizer, known_class, keybox, source_op=None):
-        AbstractVirtualStructValue.__init__(self, optimizer, keybox, source_op)
+    def __init__(self, cpu, known_class, keybox, source_op=None):
+        AbstractVirtualStructValue.__init__(self, cpu, keybox, source_op)
         assert isinstance(known_class, Const)
         self.known_class = known_class
 
@@ -190,7 +190,7 @@
         return modifier.make_virtual(self.known_class, fielddescrs)
 
     def _get_descr(self):
-        return vtable2descr(self.optimizer.cpu, self.known_class.getint())
+        return vtable2descr(self.cpu, self.known_class.getint())
 
     def __repr__(self):
         cls_name = self.known_class.value.adr.ptr._obj._TYPE._name
@@ -201,8 +201,8 @@
 
 class VStructValue(AbstractVirtualStructValue):
 
-    def __init__(self, optimizer, structdescr, keybox, source_op=None):
-        AbstractVirtualStructValue.__init__(self, optimizer, keybox, source_op)
+    def __init__(self, cpu, structdescr, keybox, source_op=None):
+        AbstractVirtualStructValue.__init__(self, cpu, keybox, source_op)
         self.structdescr = structdescr
 
     def _make_virtual(self, modifier):
@@ -215,10 +215,10 @@
 
 class VArrayValue(AbstractVirtualValue):
 
-    def __init__(self, optimizer, arraydescr, size, keybox, source_op=None):
-        AbstractVirtualValue.__init__(self, optimizer, keybox, source_op)
+    def __init__(self, arraydescr, constvalue, size, keybox, source_op=None):
+        AbstractVirtualValue.__init__(self, keybox, source_op)
         self.arraydescr = arraydescr
-        self.constvalue = optimizer.new_const_item(arraydescr)
+        self.constvalue = constvalue
         self._items = [self.constvalue] * size
 
     def getlength(self):
@@ -232,31 +232,30 @@
         assert isinstance(itemvalue, optimizer.OptValue)
         self._items[index] = itemvalue
 
-    def force_at_end_of_preamble(self, already_forced):
+    def force_at_end_of_preamble(self, already_forced, optforce):
         if self in already_forced:
             return self
         already_forced[self] = self
         for index in range(len(self._items)):
-            self._items[index] = self._items[index].force_at_end_of_preamble(already_forced)
+            self._items[index] = self._items[index].force_at_end_of_preamble(already_forced, optforce)
         return self
     
-    def _really_force(self):
+    def _really_force(self, optforce):
         assert self.source_op is not None
         if not we_are_translated():
             self.source_op.name = 'FORCE ' + self.source_op.name
-        newoperations = self.optimizer.newoperations
-        newoperations.append(self.source_op)
+        optforce.emit_operation(self.source_op)        
         self.box = box = self.source_op.result
         for index in range(len(self._items)):
             subvalue = self._items[index]
             if subvalue is not self.constvalue:
                 if subvalue.is_null():
                     continue
-                subbox = subvalue.force_box()
+                subbox = subvalue.force_box(optforce)
                 op = ResOperation(rop.SETARRAYITEM_GC,
                                   [box, ConstInt(index), subbox], None,
                                   descr=self.arraydescr)
-                newoperations.append(op)
+                optforce.emit_operation(op)
 
     def get_args_for_fail(self, modifier):
         if self.box is None and not modifier.already_seen_virtual(self.keybox):
@@ -279,17 +278,18 @@
         return OptVirtualize()
         
     def make_virtual(self, known_class, box, source_op=None):
-        vvalue = VirtualValue(self.optimizer, known_class, box, source_op)
+        vvalue = VirtualValue(self.optimizer.cpu, known_class, box, source_op)
         self.make_equal_to(box, vvalue)
         return vvalue
 
     def make_varray(self, arraydescr, size, box, source_op=None):
-        vvalue = VArrayValue(self.optimizer, arraydescr, size, box, source_op)
+        constvalue = self.new_const_item(arraydescr)
+        vvalue = VArrayValue(arraydescr, constvalue, size, box, source_op)
         self.make_equal_to(box, vvalue)
         return vvalue
 
     def make_vstruct(self, structdescr, box, source_op=None):
-        vvalue = VStructValue(self.optimizer, structdescr, box, source_op)
+        vvalue = VStructValue(self.optimizer.cpu, structdescr, box, source_op)
         self.make_equal_to(box, vvalue)
         return vvalue
 
@@ -362,7 +362,6 @@
             self.make_equal_to(op.result, fieldvalue)
         else:
             value.ensure_nonnull()
-            ###self.heap_op_optimizer.optimize_GETFIELD_GC(op, value)
             self.emit_operation(op)
 
     # note: the following line does not mean that the two operations are
@@ -377,7 +376,6 @@
             value.setfield(op.getdescr(), fieldvalue)
         else:
             value.ensure_nonnull()
-            ###self.heap_op_optimizer.optimize_SETFIELD_GC(op, value, fieldvalue)
             self.emit_operation(op)
 
     def optimize_NEW_WITH_VTABLE(self, op):
@@ -417,7 +415,6 @@
                 self.make_equal_to(op.result, itemvalue)
                 return
         value.ensure_nonnull()
-        ###self.heap_op_optimizer.optimize_GETARRAYITEM_GC(op, value)
         self.emit_operation(op)
 
     # note: the following line does not mean that the two operations are
@@ -432,7 +429,6 @@
                 value.setitem(indexbox.getint(), self.getvalue(op.getarg(2)))
                 return
         value.ensure_nonnull()
-        ###self.heap_op_optimizer.optimize_SETARRAYITEM_GC(op, value, fieldvalue)
         self.emit_operation(op)
 
 
diff --git a/pypy/jit/metainterp/optimizeopt/virtualstate.py b/pypy/jit/metainterp/optimizeopt/virtualstate.py
--- a/pypy/jit/metainterp/optimizeopt/virtualstate.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualstate.py
@@ -30,7 +30,7 @@
     def _generate_guards(self, other, box, cpu, extra_guards):
         raise InvalidLoop
 
-    def enum_forced_boxes(self, boxes, value):
+    def enum_forced_boxes(self, boxes, value, optimizer):
         raise NotImplementedError
 
     def enum(self, virtual_state):
@@ -100,14 +100,14 @@
     def _generalization_of(self, other):
         raise NotImplementedError
 
-    def enum_forced_boxes(self, boxes, value):
+    def enum_forced_boxes(self, boxes, value, optimizer):
         assert isinstance(value, virtualize.AbstractVirtualStructValue)
         assert value.is_virtual()
         for i in range(len(self.fielddescrs)):
             v = value._fields[self.fielddescrs[i]]
             s = self.fieldstate[i]
             if s.position > self.position:
-                s.enum_forced_boxes(boxes, v)
+                s.enum_forced_boxes(boxes, v, optimizer)
 
     def _enum(self, virtual_state):
         for s in self.fieldstate:
@@ -177,14 +177,14 @@
                 return False
         return True
 
-    def enum_forced_boxes(self, boxes, value):
+    def enum_forced_boxes(self, boxes, value, optimizer):
         assert isinstance(value, virtualize.VArrayValue)
         assert value.is_virtual()
         for i in range(len(self.fieldstate)):
             v = value._items[i]
             s = self.fieldstate[i]
             if s.position > self.position:
-                s.enum_forced_boxes(boxes, v)
+                s.enum_forced_boxes(boxes, v, optimizer)
 
     def _enum(self, virtual_state):
         for s in self.fieldstate:
@@ -316,11 +316,11 @@
             import pdb; pdb.set_trace()
             raise NotImplementedError
 
-    def enum_forced_boxes(self, boxes, value):
+    def enum_forced_boxes(self, boxes, value, optimizer):
         if self.level == LEVEL_CONSTANT:
             return
         assert 0 <= self.position_in_notvirtuals 
-        boxes[self.position_in_notvirtuals] = value.force_box()
+        boxes[self.position_in_notvirtuals] = value.force_box(optimizer)
 
     def _enum(self, virtual_state):
         if self.level == LEVEL_CONSTANT:
@@ -377,11 +377,13 @@
             self.state[i].generate_guards(other.state[i], args[i],
                                           cpu, extra_guards, renum)
 
-    def make_inputargs(self, values, keyboxes=False):
+    def make_inputargs(self, values, optimizer, keyboxes=False):
+        if optimizer.optearlyforce:
+            optimizer = optimizer.optearlyforce
         assert len(values) == len(self.state)
         inputargs = [None] * len(self.notvirtuals)
         for i in range(len(values)):
-            self.state[i].enum_forced_boxes(inputargs, values[i])
+            self.state[i].enum_forced_boxes(inputargs, values[i], optimizer)
 
         if keyboxes:
             for i in range(len(values)):
@@ -434,7 +436,12 @@
     def get_virtual_state(self, jump_args):
         self.optimizer.force_at_end_of_preamble()
         already_forced = {}
-        values = [self.getvalue(box).force_at_end_of_preamble(already_forced)
+        if self.optimizer.optearlyforce:
+            opt = self.optimizer.optearlyforce
+        else:
+            opt = self.optimizer
+        values = [self.getvalue(box).force_at_end_of_preamble(already_forced,
+                                                              opt)
                   for box in jump_args]
 
         for value in values:
diff --git a/pypy/jit/metainterp/optimizeopt/vstring.py b/pypy/jit/metainterp/optimizeopt/vstring.py
--- a/pypy/jit/metainterp/optimizeopt/vstring.py
+++ b/pypy/jit/metainterp/optimizeopt/vstring.py
@@ -43,7 +43,7 @@
 class __extend__(optimizer.OptValue):
     """New methods added to the base class OptValue for this file."""
 
-    def getstrlen(self, optimization, mode):
+    def getstrlen(self, string_optimizer, mode):
         if mode is mode_string:
             s = self.get_constant_string_spec(mode_string)
             if s is not None:
@@ -52,12 +52,12 @@
             s = self.get_constant_string_spec(mode_unicode)
             if s is not None:
                 return ConstInt(len(s))
-        if optimization is None:
+        if string_optimizer is None:
             return None
         self.ensure_nonnull()
-        box = self.force_box()
+        box = self.force_box(string_optimizer)
         lengthbox = BoxInt()
-        optimization.propagate_forward(ResOperation(mode.STRLEN, [box], lengthbox))
+        string_optimizer.emit_operation(ResOperation(mode.STRLEN, [box], lengthbox))
         return lengthbox
 
     @specialize.arg(1)
@@ -68,25 +68,25 @@
         else:
             return None
 
-    def string_copy_parts(self, optimizer, targetbox, offsetbox, mode):
+    def string_copy_parts(self, string_optimizer, targetbox, offsetbox, mode):
         # Copies the pointer-to-string 'self' into the target string
         # given by 'targetbox', at the specified offset.  Returns the offset
         # at the end of the copy.
-        lengthbox = self.getstrlen(optimizer, mode)
-        srcbox = self.force_box()
-        return copy_str_content(optimizer, srcbox, targetbox,
+        lengthbox = self.getstrlen(string_optimizer, mode)
+        srcbox = self.force_box(string_optimizer)
+        return copy_str_content(string_optimizer, srcbox, targetbox,
                                 CONST_0, offsetbox, lengthbox, mode)
 
 
 class VAbstractStringValue(virtualize.AbstractVirtualValue):
     _attrs_ = ('mode',)
 
-    def __init__(self, optimizer, keybox, source_op, mode):
-        virtualize.AbstractVirtualValue.__init__(self, optimizer, keybox,
+    def __init__(self, keybox, source_op, mode):
+        virtualize.AbstractVirtualValue.__init__(self, keybox,
                                                  source_op)
         self.mode = mode
 
-    def _really_force(self):
+    def _really_force(self, optforce):
         if self.mode is mode_string:
             s = self.get_constant_string_spec(mode_string)
             if s is not None:
@@ -101,12 +101,12 @@
                 return
         assert self.source_op is not None
         self.box = box = self.source_op.result
-        lengthbox = self.getstrlen(self.optimizer, self.mode)
+        lengthbox = self.getstrlen(optforce, self.mode)
         op = ResOperation(self.mode.NEWSTR, [lengthbox], box)
         if not we_are_translated():
             op.name = 'FORCE'
-        self.optimizer.emit_operation(op)
-        self.string_copy_parts(self.optimizer, box, CONST_0, self.mode)
+        optforce.emit_operation(op)
+        self.string_copy_parts(optforce, box, CONST_0, self.mode)
 
 
 class VStringPlainValue(VAbstractStringValue):
@@ -140,20 +140,20 @@
         return mode.emptystr.join([mode.chr(c.box.getint())
                                    for c in self._chars])
 
-    def string_copy_parts(self, optimizer, targetbox, offsetbox, mode):
+    def string_copy_parts(self, string_optimizer, targetbox, offsetbox, mode):
         if not self.is_virtual() and targetbox is not self.box:
-            lengthbox = self.getstrlen(optimizer, mode)
-            srcbox = self.force_box()
-            return copy_str_content(optimizer, srcbox, targetbox,
+            lengthbox = self.getstrlen(string_optimizer, mode)
+            srcbox = self.force_box(string_optimizer)
+            return copy_str_content(string_optimizer, srcbox, targetbox,
                                 CONST_0, offsetbox, lengthbox, mode)
         for i in range(len(self._chars)):
-            charbox = self._chars[i].force_box()
+            charbox = self._chars[i].force_box(string_optimizer)
             if not (isinstance(charbox, Const) and charbox.same_constant(CONST_0)):
-                optimizer.emit_operation(ResOperation(mode.STRSETITEM, [targetbox,
-                                                                    offsetbox,
-                                                                    charbox],
+                string_optimizer.emit_operation(ResOperation(mode.STRSETITEM, [targetbox,
+                                                                               offsetbox,
+                                                                               charbox],
                                                   None))
-            offsetbox = _int_add(optimizer, offsetbox, CONST_1)
+            offsetbox = _int_add(string_optimizer, offsetbox, CONST_1)
         return offsetbox
 
     def get_args_for_fail(self, modifier):
@@ -187,16 +187,16 @@
         self.left = left
         self.right = right
 
-    def getstrlen(self, optimizer, mode):
+    def getstrlen(self, string_optimizer, mode):
         if self.lengthbox is None:
-            len1box = self.left.getstrlen(optimizer, mode)
+            len1box = self.left.getstrlen(string_optimizer, mode)
             if len1box is None:
                 return None
-            len2box = self.right.getstrlen(optimizer, mode)
+            len2box = self.right.getstrlen(string_optimizer, mode)
             if len2box is None:
                 return None
-            self.lengthbox = _int_add(optimizer, len1box, len2box)
-            # ^^^ may still be None, if optimizer is None
+            self.lengthbox = _int_add(string_optimizer, len1box, len2box)
+            # ^^^ may still be None, if string_optimizer is None
         return self.lengthbox
 
     @specialize.arg(1)
@@ -209,10 +209,10 @@
             return None
         return s1 + s2
 
-    def string_copy_parts(self, optimizer, targetbox, offsetbox, mode):
-        offsetbox = self.left.string_copy_parts(optimizer, targetbox,
+    def string_copy_parts(self, string_optimizer, targetbox, offsetbox, mode):
+        offsetbox = self.left.string_copy_parts(string_optimizer, targetbox,
                                                 offsetbox, mode)
-        offsetbox = self.right.string_copy_parts(optimizer, targetbox,
+        offsetbox = self.right.string_copy_parts(string_optimizer, targetbox,
                                                  offsetbox, mode)
         return offsetbox
 
@@ -251,8 +251,8 @@
         self.vstart = vstart
         self.vlength = vlength
 
-    def getstrlen(self, _, mode):
-        return self.vlength.force_box()
+    def getstrlen(self, optforce, mode):
+        return self.vlength.force_box(optforce)
 
     @specialize.arg(1)
     def get_constant_string_spec(self, mode):
@@ -267,11 +267,11 @@
             return s1[start : start + length]
         return None
 
-    def string_copy_parts(self, optimizer, targetbox, offsetbox, mode):
-        lengthbox = self.getstrlen(optimizer, mode)
-        return copy_str_content(optimizer,
-                                self.vstr.force_box(), targetbox,
-                                self.vstart.force_box(), offsetbox,
+    def string_copy_parts(self, string_optimizer, targetbox, offsetbox, mode):
+        lengthbox = self.getstrlen(string_optimizer, mode)
+        return copy_str_content(string_optimizer,
+                                self.vstr.force_box(string_optimizer), targetbox,
+                                self.vstart.force_box(string_optimizer), offsetbox,
                                 lengthbox, mode)
 
     def get_args_for_fail(self, modifier):
@@ -300,7 +300,7 @@
         return modifier.make_vstrslice(self.mode is mode_unicode)
 
 
-def copy_str_content(optimizer, srcbox, targetbox,
+def copy_str_content(string_optimizer, srcbox, targetbox,
                      srcoffsetbox, offsetbox, lengthbox, mode, need_next_offset=True):
     if isinstance(srcbox, ConstPtr) and isinstance(srcoffsetbox, Const):
         M = 5
@@ -310,26 +310,26 @@
         # up to M characters are done "inline", i.e. with STRGETITEM/STRSETITEM
         # instead of just a COPYSTRCONTENT.
         for i in range(lengthbox.value):
-            charbox = _strgetitem(optimizer, srcbox, srcoffsetbox, mode)
-            srcoffsetbox = _int_add(optimizer, srcoffsetbox, CONST_1)
-            optimizer.emit_operation(ResOperation(mode.STRSETITEM, [targetbox,
-                                                                offsetbox,
-                                                                charbox],
+            charbox = _strgetitem(string_optimizer, srcbox, srcoffsetbox, mode)
+            srcoffsetbox = _int_add(string_optimizer, srcoffsetbox, CONST_1)
+            string_optimizer.emit_operation(ResOperation(mode.STRSETITEM, [targetbox,
+                                                                           offsetbox,
+                                                                           charbox],
                                               None))
-            offsetbox = _int_add(optimizer, offsetbox, CONST_1)
+            offsetbox = _int_add(string_optimizer, offsetbox, CONST_1)
     else:
         if need_next_offset:
-            nextoffsetbox = _int_add(optimizer, offsetbox, lengthbox)
+            nextoffsetbox = _int_add(string_optimizer, offsetbox, lengthbox)
         else:
             nextoffsetbox = None
         op = ResOperation(mode.COPYSTRCONTENT, [srcbox, targetbox,
                                                 srcoffsetbox, offsetbox,
                                                 lengthbox], None)
-        optimizer.emit_operation(op)
+        string_optimizer.emit_operation(op)
         offsetbox = nextoffsetbox
     return offsetbox
 
-def _int_add(optimizer, box1, box2):
+def _int_add(string_optimizer, box1, box2):
     if isinstance(box1, ConstInt):
         if box1.value == 0:
             return box2
@@ -337,23 +337,23 @@
             return ConstInt(box1.value + box2.value)
     elif isinstance(box2, ConstInt) and box2.value == 0:
         return box1
-    if optimizer is None:
+    if string_optimizer is None:
         return None
     resbox = BoxInt()
-    optimizer.propagate_forward(ResOperation(rop.INT_ADD, [box1, box2], resbox))
+    string_optimizer.emit_operation(ResOperation(rop.INT_ADD, [box1, box2], resbox))
     return resbox
 
-def _int_sub(optimizer, box1, box2):
+def _int_sub(string_optimizer, box1, box2):
     if isinstance(box2, ConstInt):
         if box2.value == 0:
             return box1
         if isinstance(box1, ConstInt):
             return ConstInt(box1.value - box2.value)
     resbox = BoxInt()
-    optimizer.propagate_forward(ResOperation(rop.INT_SUB, [box1, box2], resbox))
+    string_optimizer.emit_operation(ResOperation(rop.INT_SUB, [box1, box2], resbox))
     return resbox
 
-def _strgetitem(optimizer, strbox, indexbox, mode):
+def _strgetitem(string_optimizer, strbox, indexbox, mode):
     if isinstance(strbox, ConstPtr) and isinstance(indexbox, ConstInt):
         if mode is mode_string:
             s = strbox.getref(lltype.Ptr(rstr.STR))
@@ -362,30 +362,28 @@
             s = strbox.getref(lltype.Ptr(rstr.UNICODE))
             return ConstInt(ord(s.chars[indexbox.getint()]))
     resbox = BoxInt()
-    optimizer.propagate_forward(ResOperation(mode.STRGETITEM, [strbox, indexbox],
-                                      resbox))
+    string_optimizer.emit_operation(ResOperation(mode.STRGETITEM, [strbox, indexbox],
+                                                 resbox))
     return resbox
 
 
 class OptString(optimizer.Optimization):
     "Handling of strings and unicodes."
-    enabled = True
-
     def new(self):
         return OptString()
 
     def make_vstring_plain(self, box, source_op, mode):
-        vvalue = VStringPlainValue(self.optimizer, box, source_op, mode)
+        vvalue = VStringPlainValue(box, source_op, mode)
         self.make_equal_to(box, vvalue)
         return vvalue
 
     def make_vstring_concat(self, box, source_op, mode):
-        vvalue = VStringConcatValue(self.optimizer, box, source_op, mode)
+        vvalue = VStringConcatValue(box, source_op, mode)
         self.make_equal_to(box, vvalue)
         return vvalue
 
     def make_vstring_slice(self, box, source_op, mode):
-        vvalue = VStringSliceValue(self.optimizer, box, source_op, mode)
+        vvalue = VStringSliceValue(box, source_op, mode)
         self.make_equal_to(box, vvalue)
         return vvalue
 
@@ -435,9 +433,9 @@
         value.ensure_nonnull()
         #
         if value.is_virtual() and isinstance(value, VStringSliceValue):
-            fullindexbox = _int_add(self.optimizer,
-                                    value.vstart.force_box(),
-                                    vindex.force_box())
+            fullindexbox = _int_add(self,
+                                    value.vstart.force_box(self),
+                                    vindex.force_box(self))
             value = value.vstr
             vindex = self.getvalue(fullindexbox)
         #
@@ -449,7 +447,7 @@
                 if res is not optimizer.CVAL_UNINITIALIZED_ZERO:
                     return res
         #
-        resbox = _strgetitem(self.optimizer, value.force_box(), vindex.force_box(), mode)
+        resbox = _strgetitem(self, value.force_box(self), vindex.force_box(self), mode)
         return self.getvalue(resbox)
 
     def optimize_STRLEN(self, op):
@@ -459,7 +457,7 @@
 
     def _optimize_STRLEN(self, op, mode):
         value = self.getvalue(op.getarg(0))
-        lengthbox = value.getstrlen(self.optimizer, mode)
+        lengthbox = value.getstrlen(self, mode)
         self.make_equal_to(op.result, self.getvalue(lengthbox))
 
     def optimize_COPYSTRCONTENT(self, op):
@@ -477,12 +475,12 @@
 
         if length.is_constant() and length.box.getint() == 0:
             return
-        copy_str_content(self.optimizer,
-            src.force_box(),
-            dst.force_box(),
-            srcstart.force_box(),
-            dststart.force_box(),
-            length.force_box(),
+        copy_str_content(self,
+            src.force_box(self),
+            dst.force_box(self),
+            srcstart.force_box(self),
+            dststart.force_box(self),
+            length.force_box(self),
             mode, need_next_offset=False
         )
 
@@ -508,6 +506,8 @@
                     return
         self.emit_operation(op)
 
+    optimize_CALL_PURE = optimize_CALL
+
     def opt_call_str_STR2UNICODE(self, op):
         # Constant-fold unicode("constant string").
         # More generally, supporting non-constant but virtual cases is
@@ -547,16 +547,16 @@
             return True
         #
         vstr.ensure_nonnull()
-        lengthbox = _int_sub(self.optimizer, vstop.force_box(),
-                                            vstart.force_box())
+        lengthbox = _int_sub(self, vstop.force_box(self),
+                                   vstart.force_box(self))
         #
         if isinstance(vstr, VStringSliceValue):
             # double slicing  s[i:j][k:l]
             vintermediate = vstr
             vstr = vintermediate.vstr
-            startbox = _int_add(self.optimizer,
-                                vintermediate.vstart.force_box(),
-                                vstart.force_box())
+            startbox = _int_add(self,
+                                vintermediate.vstart.force_box(self),
+                                vstart.force_box(self))
             vstart = self.getvalue(startbox)
         #
         value = self.make_vstring_slice(op.result, op, mode)
@@ -587,15 +587,12 @@
             return True
         #
         if v1.is_nonnull() and v2.is_nonnull():
-            if l1box is not None and l2box is not None and (
-                l1box == l2box or (isinstance(l1box, ConstInt) and
-                                   isinstance(l2box, ConstInt) and
-                                   l1box.value == l2box.value)):
+            if l1box is not None and l2box is not None and l1box.same_box(l2box):
                 do = EffectInfo.OS_STREQ_LENGTHOK
             else:
                 do = EffectInfo.OS_STREQ_NONNULL
-            self.generate_modified_call(do, [v1.force_box(),
-                                             v2.force_box()], op.result, mode)
+            self.generate_modified_call(do, [v1.force_box(self),
+                                             v2.force_box(self)], op.result, mode)
             return True
         return False
 
@@ -603,7 +600,7 @@
         l2box = v2.getstrlen(None, mode)
         if isinstance(l2box, ConstInt):
             if l2box.value == 0:
-                lengthbox = v1.getstrlen(self.optimizer, mode)
+                lengthbox = v1.getstrlen(self, mode)
                 seo = self.optimizer.send_extra_operation
                 seo(ResOperation(rop.INT_EQ, [lengthbox, CONST_0], resultbox))
                 return True
@@ -614,17 +611,17 @@
                     vchar1 = self.strgetitem(v1, optimizer.CVAL_ZERO, mode)
                     vchar2 = self.strgetitem(v2, optimizer.CVAL_ZERO, mode)
                     seo = self.optimizer.send_extra_operation
-                    seo(ResOperation(rop.INT_EQ, [vchar1.force_box(),
-                                                  vchar2.force_box()],
+                    seo(ResOperation(rop.INT_EQ, [vchar1.force_box(self),
+                                                  vchar2.force_box(self)],
                                      resultbox))
                     return True
                 if isinstance(v1, VStringSliceValue):
                     vchar = self.strgetitem(v2, optimizer.CVAL_ZERO, mode)
                     do = EffectInfo.OS_STREQ_SLICE_CHAR
-                    self.generate_modified_call(do, [v1.vstr.force_box(),
-                                                     v1.vstart.force_box(),
-                                                     v1.vlength.force_box(),
-                                                     vchar.force_box()],
+                    self.generate_modified_call(do, [v1.vstr.force_box(self),
+                                                     v1.vstart.force_box(self),
+                                                     v1.vlength.force_box(self),
+                                                     vchar.force_box(self)],
                                                 resultbox, mode)
                     return True
         #
@@ -635,10 +632,10 @@
             if v1.is_null():
                 self.make_constant(resultbox, CONST_1)
                 return True
-            op = ResOperation(rop.PTR_EQ, [v1.force_box(),
+            op = ResOperation(rop.PTR_EQ, [v1.force_box(self),
                                            llhelper.CONST_NULL],
                               resultbox)
-            self.optimizer.emit_operation(op)
+            self.emit_operation(op)
             return True
         #
         return False
@@ -652,8 +649,8 @@
                     do = EffectInfo.OS_STREQ_NONNULL_CHAR
                 else:
                     do = EffectInfo.OS_STREQ_CHECKNULL_CHAR
-                self.generate_modified_call(do, [v1.force_box(),
-                                                 vchar.force_box()], resultbox,
+                self.generate_modified_call(do, [v1.force_box(self),
+                                                 vchar.force_box(self)], resultbox,
                                             mode)
                 return True
         #
@@ -662,10 +659,10 @@
                 do = EffectInfo.OS_STREQ_SLICE_NONNULL
             else:
                 do = EffectInfo.OS_STREQ_SLICE_CHECKNULL
-            self.generate_modified_call(do, [v1.vstr.force_box(),
-                                             v1.vstart.force_box(),
-                                             v1.vlength.force_box(),
-                                             v2.force_box()], resultbox, mode)
+            self.generate_modified_call(do, [v1.vstr.force_box(self),
+                                             v1.vstart.force_box(self),
+                                             v1.vlength.force_box(self),
+                                             v2.force_box(self)], resultbox, mode)
             return True
         return False
 
@@ -675,16 +672,11 @@
         calldescr, func = cic.callinfo_for_oopspec(oopspecindex)
         op = ResOperation(rop.CALL, [ConstInt(func)] + args, result,
                           descr=calldescr)
-        self.optimizer.emit_operation(op)
+        self.emit_operation(op)
 
     def propagate_forward(self, op):
-        if not self.enabled:
-            self.emit_operation(op)
-            return
-
         dispatch_opt(self, op)
 
-
 dispatch_opt = make_dispatcher_method(OptString, 'optimize_',
         default=OptString.emit_operation)
 
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -163,15 +163,18 @@
             if registers[i] is oldbox:
                 registers[i] = newbox
         if not we_are_translated():
-            assert oldbox not in registers[count:]
+            for b in registers[count:]:
+                assert not oldbox.same_box(b)
+                
 
     def make_result_of_lastop(self, resultbox):
         got_type = resultbox.type
-        if not we_are_translated():
-            typeof = {'i': history.INT,
-                      'r': history.REF,
-                      'f': history.FLOAT}
-            assert typeof[self.jitcode._resulttypes[self.pc]] == got_type
+        # XXX disabled for now, conflicts with str_guard_value
+        #if not we_are_translated():
+        #    typeof = {'i': history.INT,
+        #              'r': history.REF,
+        #              'f': history.FLOAT}
+        #    assert typeof[self.jitcode._resulttypes[self.pc]] == got_type
         target_index = ord(self.bytecode[self.pc-1])
         if got_type == history.INT:
             self.registers_i[target_index] = resultbox
@@ -220,6 +223,7 @@
                     'cast_float_to_int', 'cast_int_to_float',
                     'cast_float_to_singlefloat', 'cast_singlefloat_to_float',
                     'float_neg', 'float_abs',
+                    'cast_ptr_to_int', 'cast_int_to_ptr',
                     ]:
         exec py.code.Source('''
             @arguments("box")
@@ -913,6 +917,21 @@
     def _opimpl_guard_value(self, orgpc, box):
         self.implement_guard_value(orgpc, box)
 
+    @arguments("orgpc", "box", "box", "descr")
+    def opimpl_str_guard_value(self, orgpc, box, funcbox, descr):
+        if isinstance(box, Const):
+            return box     # no promotion needed, already a Const
+        else:
+            constbox = box.constbox()
+            resbox = self.do_residual_call(funcbox, descr, [box, constbox])
+            promoted_box = resbox.constbox()
+            # This is GUARD_VALUE because GUARD_TRUE assumes the existance
+            # of a label when computing resumepc
+            self.generate_guard(rop.GUARD_VALUE, resbox, [promoted_box],
+                                resumepc=orgpc)
+            self.metainterp.replace_box(box, constbox)
+            return constbox
+
     opimpl_int_guard_value = _opimpl_guard_value
     opimpl_ref_guard_value = _opimpl_guard_value
     opimpl_float_guard_value = _opimpl_guard_value
diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py
--- a/pypy/jit/metainterp/resoperation.py
+++ b/pypy/jit/metainterp/resoperation.py
@@ -220,7 +220,6 @@
         newop.setfailargs(self.getfailargs())
         return newop
 
-
 # ============
 # arity mixins
 # ============
@@ -431,6 +430,8 @@
     'INT_IS_TRUE/1b',
     'INT_NEG/1',
     'INT_INVERT/1',
+    'CAST_PTR_TO_INT/1',
+    'CAST_INT_TO_PTR/1',
     #
     'SAME_AS/1',      # gets a Const or a Box, turns it into another Box
     #
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -13,7 +13,7 @@
 from pypy.rlib.jit import (JitDriver, we_are_jitted, hint, dont_look_inside,
     loop_invariant, elidable, promote, jit_debug, assert_green,
     AssertGreenFailed, unroll_safe, current_trace_length, look_inside_iff,
-    isconstant, isvirtual)
+    isconstant, isvirtual, promote_string)
 from pypy.rlib.rarithmetic import ovfcheck
 from pypy.rpython.lltypesystem import lltype, llmemory, rffi
 from pypy.rpython.ootypesystem import ootype
@@ -2956,6 +2956,18 @@
         assert res == f(32)
         self.check_loops(arraylen_gc=2)
 
+    def test_ulonglong_mod(self):
+        myjitdriver = JitDriver(greens = [], reds = ['n', 'sa', 'i'])
+        def f(n):
+            sa = i = rffi.cast(rffi.ULONGLONG, 1)
+            while i < rffi.cast(rffi.ULONGLONG, n):
+                myjitdriver.jit_merge_point(sa=sa, n=n, i=i)
+                sa += sa % i
+                i += 1
+        res = self.meta_interp(f, [32])
+        assert res == f(32)
+
+
 class TestOOtype(BasicTests, OOJitMixin):
 
     def test_oohash(self):
@@ -3408,6 +3420,106 @@
         assert res == main(1, 10)
         self.check_loops(call=0)
 
+    def test_setarrayitem_followed_by_arraycopy(self):
+        myjitdriver = JitDriver(greens = [], reds = ['n', 'sa', 'x', 'y'])
+        def f(n):
+            sa = 0
+            x = [1,2,n]
+            y = [1,2,3]
+            while n > 0:
+                myjitdriver.jit_merge_point(sa=sa, n=n, x=x, y=y)
+                y[0] = n
+                x[0:3] = y
+                sa += x[0]
+                n -= 1
+            return sa
+        res = self.meta_interp(f, [16])
+        assert res == f(16)
+        
+
 
 class TestLLtype(BaseLLtypeTests, LLJitMixin):
-    pass
+    def test_tagged(self):
+        from pypy.rlib.objectmodel import UnboxedValue
+        class Base(object):
+            __slots__ = ()
+
+        class Int(UnboxedValue, Base):
+            __slots__ = ["a"]
+
+            def is_pos(self):
+                return self.a > 0
+
+            def dec(self):
+                return Int(self.a - 1)
+
+
+        class Float(Base):
+            def __init__(self, a):
+                self.a = a
+
+            def is_pos(self):
+                return self.a > 0
+
+            def dec(self):
+                return Float(self.a - 1)
+
+        driver = JitDriver(greens=['pc', 's'], reds=['o'])
+
+        def main(fl, n, s):
+            if s:
+                s = "--j"
+            else:
+                s = "---j"
+            if fl:
+                o = Float(float(n))
+            else:
+                o = Int(n)
+            pc = 0
+            while True:
+                driver.jit_merge_point(s=s, pc=pc, o=o)
+                c = s[pc]
+                if c == "j":
+                    driver.can_enter_jit(s=s, pc=pc, o=o)
+                    if o.is_pos():
+                        pc = 0
+                        continue
+                    else:
+                        break
+                elif c == "-":
+                    o = o.dec()
+                pc += 1
+            return pc
+        res = self.meta_interp(main, [False, 100, True], taggedpointers=True)
+
+    def test_rerased(self):
+        from pypy.rlib.rerased import erase_int, unerase_int, new_erasing_pair
+        eraseX, uneraseX = new_erasing_pair("X")
+        #
+        class X:
+            def __init__(self, a, b):
+                self.a = a
+                self.b = b
+        #
+        def f(i, j):
+            # 'j' should be 0 or 1, not other values
+            if j > 0:
+                e = eraseX(X(i, j))
+            else:
+                try:
+                    e = erase_int(i)
+                except OverflowError:
+                    return -42
+            if j & 1:
+                x = uneraseX(e)
+                return x.a - x.b
+            else:
+                return unerase_int(e)
+        #
+        x = self.interp_operations(f, [-128, 0], taggedpointers=True)
+        assert x == -128
+        bigint = sys.maxint//2 + 1
+        x = self.interp_operations(f, [bigint, 0], taggedpointers=True)
+        assert x == -42
+        x = self.interp_operations(f, [1000, 1], taggedpointers=True)
+        assert x == 999
diff --git a/pypy/jit/metainterp/test/test_heapcache.py b/pypy/jit/metainterp/test/test_heapcache.py
--- a/pypy/jit/metainterp/test/test_heapcache.py
+++ b/pypy/jit/metainterp/test/test_heapcache.py
@@ -355,6 +355,14 @@
         assert not h.is_unescaped(box1)
         assert not h.is_unescaped(box2)
 
+    def test_circular_virtuals(self):
+        h = HeapCache()
+        h.new(box1)
+        h.new(box2)
+        h.invalidate_caches(rop.SETFIELD_GC, None, [box1, box2])
+        h.invalidate_caches(rop.SETFIELD_GC, None, [box2, box1])
+        h.invalidate_caches(rop.SETFIELD_GC, None, [box3, box1]) # does not crash
+
     def test_unescaped_array(self):
         h = HeapCache()
         h.new_array(box1, lengthbox1)
@@ -362,4 +370,4 @@
         h.invalidate_caches(rop.SETARRAYITEM_GC, None, [box1, index1, box2])
         assert h.is_unescaped(box1)
         h.invalidate_caches(rop.SETARRAYITEM_GC, None, [box2, index1, box1])
-        assert not h.is_unescaped(box1)
\ No newline at end of file
+        assert not h.is_unescaped(box1)
diff --git a/pypy/jit/metainterp/test/test_longlong.py b/pypy/jit/metainterp/test/test_longlong.py
--- a/pypy/jit/metainterp/test/test_longlong.py
+++ b/pypy/jit/metainterp/test/test_longlong.py
@@ -118,6 +118,26 @@
         res = self.interp_operations(f, [1000000000])
         assert res == 123500000000.0
 
+    def test_floats_negative(self):
+        def f(i):
+            # i == 1000000000
+            f = i * -123.5
+            n = r_longlong(f)
+            compare(n, -29, 1054051584)
+            return float(n)
+        res = self.interp_operations(f, [1000000000])
+        assert res == -123500000000.0
+
+    def test_floats_ulonglong(self):
+        def f(i):
+            # i == 1000000000
+            f = i * 12350000000.0
+            n = r_ulonglong(f)
+            compare(n, -1419508847, 538116096)
+            return float(n)
+        res = self.interp_operations(f, [1000000000])
+        assert res == 12350000000000000000.0
+
     def test_unsigned_compare_ops(self):
         def f(n1, n2):
             # n == 30002000000000
diff --git a/pypy/jit/metainterp/test/test_resume.py b/pypy/jit/metainterp/test/test_resume.py
--- a/pypy/jit/metainterp/test/test_resume.py
+++ b/pypy/jit/metainterp/test/test_resume.py
@@ -180,22 +180,27 @@
     reader.consume_boxes(info, bi, br, bf)
     b1s = reader.liveboxes[0]
     b2s = reader.liveboxes[1]
-    assert bi == [b1s, ConstInt(111), b1s]
-    assert br == [ConstPtr(gcrefnull), b2s]
+    assert_same(bi, [b1s, ConstInt(111), b1s])
+    assert_same(br, [ConstPtr(gcrefnull), b2s])
     bi, br, bf = [None]*2, [None]*0, [None]*0
     info = MyBlackholeInterp([lltype.Signed,
                               lltype.Signed]).get_current_position_info()
     reader.consume_boxes(info, bi, br, bf)
-    assert bi == [ConstInt(222), ConstInt(333)]
+    assert_same(bi, [ConstInt(222), ConstInt(333)])
     bi, br, bf = [None]*2, [None]*1, [None]*0
     info = MyBlackholeInterp([lltype.Signed, llmemory.GCREF,
                               lltype.Signed]).get_current_position_info()
     reader.consume_boxes(info, bi, br, bf)
     b3s = reader.liveboxes[2]
-    assert bi == [b1s, b3s]
-    assert br == [b2s]
+    assert_same(bi, [b1s, b3s])
+    assert_same(br, [b2s])
     #
 
+def assert_same(list1, list2):
+    assert len(list1) == len(list2)
+    for b1, b2 in zip(list1, list2):
+        assert b1.same_box(b2)
+
 def test_simple_read_tagged_ints():
     storage = Storage()
     storage.rd_consts = []
@@ -576,8 +581,9 @@
 
 
 class FakeOptimizer_VirtualValue(object):
-    class cpu:
-        pass
+    class optimizer:
+        class cpu:
+            pass
 fakeoptimizer = FakeOptimizer_VirtualValue()
 
 def ConstAddr(addr, cpu):   # compatibility
@@ -1022,11 +1028,11 @@
     metainterp = MyMetaInterp()
     reader = ResumeDataFakeReader(storage, newboxes, metainterp)
     lst = reader.consume_boxes()
-    assert lst == [b1t, b1t, b3t]
+    assert_same(lst, [b1t, b1t, b3t])
     lst = reader.consume_boxes()
-    assert lst == [ConstInt(2), ConstInt(3)]
+    assert_same(lst, [ConstInt(2), ConstInt(3)])
     lst = reader.consume_boxes()
-    assert lst == [b1t, ConstInt(1), b1t, b1t]
+    assert_same(lst, [b1t, ConstInt(1), b1t, b1t])
     assert metainterp.trace == []    
 
 
@@ -1043,11 +1049,11 @@
     reader = ResumeDataFakeReader(storage, newboxes, metainterp)
     lst = reader.consume_boxes()
     c1t = ConstInt(111)
-    assert lst == [c1t, b2t, b3t]
+    assert_same(lst, [c1t, b2t, b3t])
     lst = reader.consume_boxes()
-    assert lst == [ConstInt(2), ConstInt(3)]
+    assert_same(lst, [ConstInt(2), ConstInt(3)])
     lst = reader.consume_boxes()
-    assert lst == [c1t, ConstInt(1), c1t, b2t]
+    assert_same(lst, [c1t, ConstInt(1), c1t, b2t])
     assert metainterp.trace == []
 
 
@@ -1113,9 +1119,11 @@
     # check that we get the operations in 'expected', in a possibly different
     # order.
     assert len(trace) == len(expected)
-    for x in trace:
-        assert x in expected
-        expected.remove(x)
+    with CompareableConsts():
+        for x in trace:
+            assert x in expected
+            expected.remove(x)
+
     ptr = b2t.value._obj.container._as_ptr()
     assert lltype.typeOf(ptr) == lltype.Ptr(LLtypeMixin.NODE)
     assert ptr.value == 111
@@ -1125,6 +1133,18 @@
     assert ptr2.parent.value == 33
     assert ptr2.parent.next == ptr
 
+class CompareableConsts(object):
+    def __init__(self):
+        self.oldeq = None
+        
+    def __enter__(self):
+        assert self.oldeq is None
+        self.oldeq = Const.__eq__
+        Const.__eq__ = Const.same_box
+        
+    def __exit__(self, type, value, traceback):
+        Const.__eq__ = self.oldeq
+
 def test_virtual_adder_make_varray():
     b2s, b4s = [BoxPtr(), BoxInt(4)]
     c1s = ConstInt(111)
@@ -1135,12 +1155,7 @@
     modifier.liveboxes = {}
     modifier.vfieldboxes = {}
 
-    class FakeOptimizer(object):
-        class cpu:
-            pass
-        def new_const_item(self, descr):
-            return None
-    v2 = VArrayValue(FakeOptimizer(), LLtypeMixin.arraydescr, 2, b2s)
+    v2 = VArrayValue(LLtypeMixin.arraydescr, None, 2, b2s)
     v2._items = [b4s, c1s]
     modifier.register_virtual_fields(b2s, [b4s, c1s])
     liveboxes = []
@@ -1167,8 +1182,9 @@
         (rop.SETARRAYITEM_GC, [b2t,ConstInt(1), c1s], None,
                               LLtypeMixin.arraydescr),
         ]
-    for x, y in zip(expected, trace):
-        assert x == y
+    with CompareableConsts():
+        for x, y in zip(expected, trace):
+            assert x == y
     #
     ptr = b2t.value._obj.container._as_ptr()
     assert lltype.typeOf(ptr) == lltype.Ptr(lltype.GcArray(lltype.Signed))
@@ -1211,8 +1227,9 @@
         (rop.SETFIELD_GC, [b2t, c1s],  None, LLtypeMixin.adescr),
         (rop.SETFIELD_GC, [b2t, b4t], None, LLtypeMixin.bdescr),
         ]
-    for x, y in zip(expected, trace):
-        assert x == y
+    with CompareableConsts():
+        for x, y in zip(expected, trace):
+            assert x == y
     #
     ptr = b2t.value._obj.container._as_ptr()
     assert lltype.typeOf(ptr) == lltype.Ptr(LLtypeMixin.S)
diff --git a/pypy/jit/metainterp/test/test_string.py b/pypy/jit/metainterp/test/test_string.py
--- a/pypy/jit/metainterp/test/test_string.py
+++ b/pypy/jit/metainterp/test/test_string.py
@@ -3,7 +3,8 @@
 from pypy.jit.codewriter.policy import StopAtXPolicy
 from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin
 from pypy.rlib.debug import debug_print
-from pypy.rlib.jit import JitDriver, dont_look_inside, we_are_jitted
+from pypy.rlib.jit import JitDriver, dont_look_inside, we_are_jitted,\
+     promote_string
 from pypy.rlib.rstring import StringBuilder
 from pypy.rpython.ootypesystem import ootype
 
@@ -484,6 +485,42 @@
             return len(sa.val)
         assert self.meta_interp(f, ['a']) == f('a')
 
+    def test_string_comepare_quasiimmutable(self):
+        class Sys(object):
+            _immutable_fields_ = ["defaultencoding?"]
+            def __init__(self, s):
+                self.defaultencoding = s
+        _str = self._str
+        sys = Sys(_str('ascii'))        
+        mydriver = JitDriver(reds = ['n', 'sa'], greens = [])
+        def f(n):
+            sa = 0
+            sys.defaultencoding = _str('ascii')
+            while n:
+                mydriver.jit_merge_point(n=n, sa=sa)
+                if sys.defaultencoding == _str('ascii'):
+                    sa += 1
+                n -= 1
+            sys.defaultencoding = _str('utf-8')
+            return sa
+        assert self.meta_interp(f, [8]) == f(8)
+        self.check_loops({'int_add': 1, 'guard_true': 1, 'int_sub': 1,
+                          'jump': 1, 'int_is_true': 1,
+                          'guard_not_invalidated': 1})
+
+    def test_promote_string(self):
+        driver = JitDriver(greens = [], reds = ['n'])
+        
+        def f(n):
+            while n < 21:
+                driver.jit_merge_point(n=n)
+                promote_string(str(n % 3))
+                n += 1
+            return 0
+
+        self.meta_interp(f, [0])
+        self.check_loops(call=3 + 1) # one for int2str
+
 #class TestOOtype(StringTests, OOJitMixin):
 #    CALL = "oosend"
 #    CALL_PURE = "oosend_pure"
diff --git a/pypy/jit/metainterp/test/test_virtualref.py b/pypy/jit/metainterp/test/test_virtualref.py
--- a/pypy/jit/metainterp/test/test_virtualref.py
+++ b/pypy/jit/metainterp/test/test_virtualref.py
@@ -3,6 +3,7 @@
 from pypy.rpython.llinterp import LLException
 from pypy.rlib.jit import JitDriver, dont_look_inside, vref_None
 from pypy.rlib.jit import virtual_ref, virtual_ref_finish, InvalidVirtualRef
+from pypy.rlib.jit import non_virtual_ref
 from pypy.rlib.objectmodel import compute_unique_id
 from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin, _get_jitcodes
 from pypy.jit.metainterp.resoperation import rop
@@ -595,6 +596,65 @@
         res = self.meta_interp(fn, [10])
         assert res == 6
 
+    def test_is_virtual(self):
+        myjitdriver = JitDriver(greens=[], reds=['n', 'res1'])
+        class X:
+            pass
+        @dont_look_inside
+        def residual(vref):
+            return vref.virtual
+        #
+        def f(n):
+            res1 = -42
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, res1=res1)
+                x = X()
+                vref = virtual_ref(x)
+                res1 = residual(vref)
+                virtual_ref_finish(vref, x)
+                n -= 1
+            return res1
+        #
+        res = self.meta_interp(f, [10])
+        assert res == 1
+
+    def test_is_not_virtual_none(self):
+        myjitdriver = JitDriver(greens=[], reds=['n', 'res1'])
+        @dont_look_inside
+        def residual(vref):
+            return vref.virtual
+        #
+        def f(n):
+            res1 = -42
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, res1=res1)
+                res1 = residual(vref_None)
+                n -= 1
+            return res1
+        #
+        res = self.meta_interp(f, [10])
+        assert res == 0
+
+    def test_is_not_virtual_non_none(self):
+        myjitdriver = JitDriver(greens=[], reds=['n', 'res1'])
+        class X:
+            pass
+        @dont_look_inside
+        def residual(vref):
+            return vref.virtual
+        #
+        def f(n):
+            res1 = -42
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, res1=res1)
+                x = X()
+                res1 = residual(non_virtual_ref(x))
+                n -= 1
+            return res1
+        #
+        res = self.meta_interp(f, [10])
+        assert res == 0
+
 
 class TestLLtype(VRefTests, LLJitMixin):
     pass
diff --git a/pypy/jit/metainterp/test/test_virtualstate.py b/pypy/jit/metainterp/test/test_virtualstate.py
--- a/pypy/jit/metainterp/test/test_virtualstate.py
+++ b/pypy/jit/metainterp/test/test_virtualstate.py
@@ -431,7 +431,7 @@
         
 
 class BaseTestBridges(BaseTest):
-    enable_opts = "intbounds:rewrite:virtualize:string:heap:unroll"
+    enable_opts = "intbounds:rewrite:virtualize:string:pure:heap:unroll"
 
     def _do_optimize_bridge(self, bridge, call_pure_results):
         from pypy.jit.metainterp.optimizeopt import optimize_bridge_1, build_opt_chain
diff --git a/pypy/jit/metainterp/virtualref.py b/pypy/jit/metainterp/virtualref.py
--- a/pypy/jit/metainterp/virtualref.py
+++ b/pypy/jit/metainterp/virtualref.py
@@ -39,6 +39,7 @@
     def replace_force_virtual_with_call(self, graphs):
         # similar to rvirtualizable2.replace_force_virtualizable_with_call().
         c_force_virtual_ptr = None
+        c_is_virtual_ptr = None
         force_virtual_count = 0
         for graph in graphs:
             for block in graph.iterblocks():
@@ -52,6 +53,13 @@
                         op.opname = 'direct_call'
                         op.args = [c_force_virtual_ptr, op.args[0]]
                         force_virtual_count += 1
+                    #
+                    if op.opname == 'jit_is_virtual':
+                        if c_is_virtual_ptr is None:
+                            c_is_virtual_ptr = self.get_is_virtual_fnptr()
+                        #
+                        op.opname = 'direct_call'
+                        op.args = [c_is_virtual_ptr, op.args[0]]
         #
         if c_force_virtual_ptr is not None:
             log("replaced %d 'jit_force_virtual' with %r" % (force_virtual_count,
@@ -129,6 +137,17 @@
             force_virtual_if_necessary)
         return inputconst(lltype.typeOf(funcptr), funcptr)
 
+    def get_is_virtual_fnptr(self):
+        #
+        def is_virtual(inst):
+            if not inst:
+                return False
+            return inst.typeptr == self.jit_virtual_ref_vtable
+        #
+        FUNC = lltype.FuncType([rclass.OBJECTPTR], lltype.Bool)
+        funcptr = self.warmrunnerdesc.helper_func(lltype.Ptr(FUNC), is_virtual)
+        return inputconst(lltype.typeOf(funcptr), funcptr)
+
     def force_virtual(self, inst):
         vref = lltype.cast_pointer(lltype.Ptr(self.JIT_VIRTUAL_REF), inst)
         token = vref.virtual_token
diff --git a/pypy/jit/metainterp/warmspot.py b/pypy/jit/metainterp/warmspot.py
--- a/pypy/jit/metainterp/warmspot.py
+++ b/pypy/jit/metainterp/warmspot.py
@@ -53,6 +53,8 @@
         extraconfigopts = {'translation.list_comprehension_operations': True}
     else:
         extraconfigopts = {}
+    if kwds.pop("taggedpointers", False):
+        extraconfigopts["translation.taggedpointers"] = True
     interp, graph = get_interpreter(function, args,
                                     backendopt=False,  # will be done below
                                     type_system=type_system,
diff --git a/pypy/module/__builtin__/__init__.py b/pypy/module/__builtin__/__init__.py
--- a/pypy/module/__builtin__/__init__.py
+++ b/pypy/module/__builtin__/__init__.py
@@ -20,6 +20,9 @@
         'any'           : 'app_functional.any',
         'all'           : 'app_functional.all',
         'sum'           : 'app_functional.sum',
+        'map'           : 'app_functional.map',
+        'reduce'        : 'app_functional.reduce',
+        'filter'        : 'app_functional.filter',
         'vars'          : 'app_inspect.vars',
         'dir'           : 'app_inspect.dir',
 
@@ -86,11 +89,8 @@
         'enumerate'     : 'functional.W_Enumerate',
         'min'           : 'functional.min',
         'max'           : 'functional.max',
-        'map'           : 'functional.map',
         'zip'           : 'functional.zip',
-        'reduce'        : 'functional.reduce',
         'reversed'      : 'functional.reversed',
-        'filter'        : 'functional.filter',
         'super'         : 'descriptor.W_Super',
         'staticmethod'  : 'descriptor.StaticMethod',
         'classmethod'   : 'descriptor.ClassMethod',
diff --git a/pypy/module/__builtin__/app_functional.py b/pypy/module/__builtin__/app_functional.py
--- a/pypy/module/__builtin__/app_functional.py
+++ b/pypy/module/__builtin__/app_functional.py
@@ -48,4 +48,118 @@
         # Very intentionally *not* +=, that would have different semantics if
         # start was a mutable type, such as a list
         last = last + x
-    return last
\ No newline at end of file
+    return last
+
+def map(func, *collections):
+    """map(function, sequence[, sequence, ...]) -> list
+
+Return a list of the results of applying the function to the items of
+the argument sequence(s).  If more than one sequence is given, the
+function is called with an argument list consisting of the corresponding
+item of each sequence, substituting None for missing values when not all
+sequences have the same length.  If the function is None, return a list of
+the items of the sequence (or a list of tuples if more than one sequence)."""
+    if not collections:
+        raise TypeError("map() requires at least two arguments")
+    num_collections = len(collections)
+    none_func = func is None
+    if num_collections == 1:
+        if none_func:
+            return list(collections[0])
+        else:
+            # Special case for the really common case of a single collection,
+            # this can be eliminated if we could unroll that loop that creates
+            # `args` based on whether or not len(collections) was constant
+            result = []
+            for item in collections[0]:
+                result.append(func(item))
+            return result
+    result = []
+    # Pair of (iterator, has_finished)
+    iterators = [(iter(seq), False) for seq in collections]
+    while True:
+        cont = False
+        args = []
+        for idx, (iterator, has_finished) in enumerate(iterators):
+            val = None
+            if not has_finished:
+                try:
+                    val = next(iterator)
+                except StopIteration:
+                    iterators[idx] = (None, True)
+                else:
+                    cont = True
+            args.append(val)
+        args = tuple(args)
+        if cont:
+            if none_func:
+                result.append(args)
+            else:
+                result.append(func(*args))
+        else:
+            return result
+
+sentinel = object()
+
+def reduce(func, sequence, initial=sentinel):
+    """reduce(function, sequence[, initial]) -> value
+
+Apply a function of two arguments cumulatively to the items of a sequence,
+from left to right, so as to reduce the sequence to a single value.
+For example, reduce(lambda x, y: x+y, [1, 2, 3, 4, 5]) calculates
+((((1+2)+3)+4)+5).  If initial is present, it is placed before the items
+of the sequence in the calculation, and serves as a default when the
+sequence is empty."""
+    iterator = iter(sequence)
+    if initial is sentinel:
+        try:
+            initial = next(iterator)
+        except StopIteration:
+            raise TypeError("reduce() of empty sequence with no initial value")
+    result = initial
+    for item in iterator:
+        result = func(result, item)
+    return result
+
+def filter(func, seq):
+    """filter(function or None, sequence) -> list, tuple, or string
+
+Return those items of sequence for which function(item) is true.  If
+function is None, return the items that are true.  If sequence is a tuple
+or string, return the same type, else return a list."""
+    if func is None:
+        func = bool
+    if isinstance(seq, str):
+        return _filter_string(func, seq, str)
+    elif isinstance(seq, unicode):
+        return _filter_string(func, seq, unicode)
+    elif isinstance(seq, tuple):
+        return _filter_tuple(func, seq)
+    result = []
+    for item in seq:
+        if func(item):
+            result.append(item)
+    return result
+
+def _filter_string(func, string, str_type):
+    if func is bool and type(string) is str_type:
+        return string
+    result = []
+    for i in range(len(string)):
+        # You must call __getitem__ on the strings, simply iterating doesn't
+        # work :/
+        item = string[i]
+        if func(item):
+            if not isinstance(item, str_type):
+                raise TypeError("__getitem__ returned a non-string type")
+            result.append(item)
+    return str_type().join(result)
+
+def _filter_tuple(func, seq):
+    result = []
+    for i in range(len(seq)):
+        # Again, must call __getitem__, at least there are tests.
+        item = seq[i]
+        if func(item):
+            result.append(item)
+    return tuple(result)
\ No newline at end of file
diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py
--- a/pypy/module/__builtin__/functional.py
+++ b/pypy/module/__builtin__/functional.py
@@ -193,124 +193,14 @@
     return min_max(space, __args__, "max")
 
 def min(space, __args__):
-    """Return the smallest item in a sequence.
+    """min(iterable[, key=func]) -> value
+    min(a, b, c, ...[, key=func]) -> value
 
-    If more than one argument is passed, return the minimum of them.
+    With a single iterable argument, return its smallest item.
+    With two or more arguments, return the smallest argument.
     """
     return min_max(space, __args__, "min")
 
- at unwrap_spec(collections_w="args_w")
-def map(space, w_func, collections_w):
-    """does 3 separate things, hence this enormous docstring.
-       1.  if function is None, return a list of tuples, each with one
-           item from each collection.  If the collections have different
-           lengths,  shorter ones are padded with None.
-
-       2.  if function is not None, and there is only one collection,
-           apply function to every item in the collection and return a
-           list of the results.
-
-       3.  if function is not None, and there are several collections,
-           repeatedly call the function with one argument from each
-           collection.  If the collections have different lengths,
-           shorter ones are padded with None
-    """
-    if not collections_w:
-        msg = "map() requires at least two arguments"
-        raise OperationError(space.w_TypeError, space.wrap(msg))
-    none_func = space.is_w(w_func, space.w_None)
-    if len(collections_w) == 1:
-        w_collection = collections_w[0]
-        if none_func:
-            result_w = space.unpackiterable(w_collection)
-        else:
-            result_w = map_single_collection(space, w_func, w_collection)
-    else:
-        result_w = map_multiple_collections(space, w_func, collections_w,
-                                            none_func)
-    return space.newlist(result_w)
-
-def map_single_collection(space, w_func, w_collection):
-    """Special case for 'map(func, coll)', where 'func' is not None and there
-    is only one 'coll' argument."""
-    w_iter = space.iter(w_collection)
-    # xxx special hacks for speed
-    from pypy.interpreter import function, pycode
-    if isinstance(w_func, function.Function):
-        # xxx compatibility issue: what if func_code is modified in the
-        # middle of running map()??  That's far too obscure for me to care...
-        code = w_func.getcode()
-        fast_natural_arity = code.fast_natural_arity
-        if fast_natural_arity == (1|pycode.PyCode.FLATPYCALL):
-            assert isinstance(code, pycode.PyCode)
-            return map_single_user_function(code, w_func, w_iter)
-    # /xxx end of special hacks
-    return map_single_other_callable(space, w_func, w_iter)
-
-def map_single_other_callable(space, w_func, w_iter):
-    result_w = []
-    while True:
-        try:
-            w_item = space.next(w_iter)
-        except OperationError, e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            break
-        result_w.append(space.call_function(w_func, w_item))
-    return result_w
-map_single_other_callable._dont_inline_ = True
-
-from pypy.rlib.jit import JitDriver
-mapjitdriver = JitDriver(greens = ['code'],
-                         reds = ['w_func', 'w_iter', 'result_w'])
-def map_single_user_function(code, w_func, w_iter):
-    result_w = []
-    while True:
-        mapjitdriver.can_enter_jit(code=code, w_func=w_func,
-                                   w_iter=w_iter, result_w=result_w)
-        mapjitdriver.jit_merge_point(code=code, w_func=w_func,
-                                     w_iter=w_iter, result_w=result_w)
-        space = w_func.space
-        try:
-            w_item = space.next(w_iter)
-        except OperationError, e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            break
-        new_frame = space.createframe(code, w_func.w_func_globals,
-                                      w_func)
-        new_frame.locals_stack_w[0] = w_item
-        w_res = new_frame.run()
-        result_w.append(w_res)
-    return result_w
-
-def map_multiple_collections(space, w_func, collections_w, none_func):
-    result_w = []
-    iterators_w = [space.iter(w_seq) for w_seq in collections_w]
-    num_iterators = len(iterators_w)
-    while True:
-        cont = False
-        args_w = [space.w_None] * num_iterators
-        for i in range(num_iterators):
-            if iterators_w[i] is not None:
-                try:
-                    args_w[i] = space.next(iterators_w[i])
-                except OperationError, e:
-                    if not e.match(space, space.w_StopIteration):
-                        raise
-                    iterators_w[i] = None
-                else:
-                    cont = True
-        if not cont:
-            break
-        w_args = space.newtuple(args_w)
-        if none_func:
-            w_res = w_args
-        else:
-            w_res = space.call(w_func, w_args)
-        result_w.append(w_res)
-    return result_w
-
 @unwrap_spec(sequences_w="args_w")
 def zip(space, sequences_w):
     """Return a list of tuples, where the nth tuple contains every nth item of
@@ -332,90 +222,6 @@
             return space.newlist(result_w)
         result_w.append(space.newtuple(items_w))
 
-def reduce(space, w_func, w_sequence, w_initial=NoneNotWrapped):
-    """ Apply function of two arguments cumulatively to the items of sequence,
-        from left to right, so as to reduce the sequence to a single value.
-        Optionally begin with an initial value.
-    """
-    w_iter = space.iter(w_sequence)
-    if w_initial is None:
-        try:
-            w_initial = space.next(w_iter)
-        except OperationError, e:
-            if e.match(space, space.w_StopIteration):
-                msg = "reduce() of empty sequence with no initial value"
-                raise OperationError(space.w_TypeError, space.wrap(msg))
-            raise
-    w_result = w_initial
-    while True:
-        try:
-            w_next = space.next(w_iter)
-        except OperationError, e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            break
-        w_result = space.call_function(w_func, w_result, w_next)
-    return w_result
-
-def filter(space, w_func, w_seq):
-    """construct a list of those elements of collection for which function
-       is True.  If function is None, then return the items in the sequence
-       which are True.
-    """
-    if space.is_true(space.isinstance(w_seq, space.w_str)):
-        return _filter_string(space, w_func, w_seq, space.w_str)
-    if space.is_true(space.isinstance(w_seq, space.w_unicode)):
-        return _filter_string(space, w_func, w_seq, space.w_unicode)
-    if space.is_true(space.isinstance(w_seq, space.w_tuple)):
-        return _filter_tuple(space, w_func, w_seq)
-    w_iter = space.iter(w_seq)
-    result_w = []
-    none_func = space.is_w(w_func, space.w_None)
-    while True:
-        try:
-            w_next = space.next(w_iter)
-        except OperationError, e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            break
-        if none_func:
-            w_keep = w_next
-        else:
-            w_keep = space.call_function(w_func, w_next)
-        if space.is_true(w_keep):
-            result_w.append(w_next)
-    return space.newlist(result_w)
-
-def _filter_tuple(space, w_func, w_tuple):
-    none_func = space.is_w(w_func, space.w_None)
-    length = space.len_w(w_tuple)
-    result_w = []
-    for i in range(length):
-        w_item = space.getitem(w_tuple, space.wrap(i))
-        if none_func:
-            w_keep = w_item
-        else:
-            w_keep = space.call_function(w_func, w_item)
-        if space.is_true(w_keep):
-            result_w.append(w_item)
-    return space.newtuple(result_w)
-
-def _filter_string(space, w_func, w_string, w_str_type):
-    none_func = space.is_w(w_func, space.w_None)
-    if none_func and space.is_w(space.type(w_string), w_str_type):
-        return w_string
-    length = space.len_w(w_string)
-    result_w = []
-    for i in range(length):
-        w_item = space.getitem(w_string, space.wrap(i))
-        if none_func or space.is_true(space.call_function(w_func, w_item)):
-            if not space.is_true(space.isinstance(w_item, w_str_type)):
-                msg = "__getitem__ returned a non-string type"
-                raise OperationError(space.w_TypeError, space.wrap(msg))
-            result_w.append(w_item)
-    w_empty = space.call_function(w_str_type)
-    return space.call_method(w_empty, "join", space.newlist(result_w))
-
 class W_Enumerate(Wrappable):
 
     def __init__(self, w_iter, w_start):
diff --git a/pypy/module/__builtin__/test/test_functional.py b/pypy/module/__builtin__/test/test_functional.py
--- a/pypy/module/__builtin__/test/test_functional.py
+++ b/pypy/module/__builtin__/test/test_functional.py
@@ -147,7 +147,7 @@
        assert list(xrange(A())) == [0, 1, 2, 3, 4]
        assert list(xrange(0, A())) == [0, 1, 2, 3, 4]
        assert list(xrange(0, 10, A())) == [0, 5]
- 
+
    def test_xrange_float(self):
       assert list(xrange(0.1, 2.0, 1.1)) == [0, 1]
 
diff --git a/pypy/module/__pypy__/interp_builders.py b/pypy/module/__pypy__/interp_builders.py
--- a/pypy/module/__pypy__/interp_builders.py
+++ b/pypy/module/__pypy__/interp_builders.py
@@ -16,7 +16,8 @@
 
         def _check_done(self, space):
             if self.builder is None:
-                raise OperationError(space.w_ValueError, space.wrap("Can't operate on a done builder"))
+                raise OperationError(space.w_ValueError, space.wrap(
+                        "Can't operate on a built builder"))
 
         @unwrap_spec(size=int)
         def descr__new__(space, w_subtype, size=-1):
@@ -31,7 +32,8 @@
         def descr_append_slice(self, space, s, start, end):
             self._check_done(space)
             if not 0 <= start <= end <= len(s):
-                raise OperationError(space.w_ValueError, space.wrap("bad start/stop"))
+                raise OperationError(space.w_ValueError, space.wrap(
+                        "bad start/stop"))
             self.builder.append_slice(s, start, end)
 
         def descr_build(self, space):
@@ -40,6 +42,12 @@
             self.builder = None
             return w_s
 
+        def descr_len(self, space):
+            if self.builder is None:
+                raise OperationError(space.w_ValueError, space.wrap(
+                        "no length of built builder"))
+            return space.wrap(self.builder.getlength())
+
     W_Builder.__name__ = "W_%s" % name
     W_Builder.typedef = TypeDef(name,
         __new__ = interp2app(func_with_new_name(
@@ -48,6 +56,7 @@
         append = interp2app(W_Builder.descr_append),
         append_slice = interp2app(W_Builder.descr_append_slice),
         build = interp2app(W_Builder.descr_build),
+        __len__ = interp2app(W_Builder.descr_len),
     )
     W_Builder.typedef.acceptable_as_base_class = False
     return W_Builder
diff --git a/pypy/module/__pypy__/test/test_builders.py b/pypy/module/__pypy__/test/test_builders.py
--- a/pypy/module/__pypy__/test/test_builders.py
+++ b/pypy/module/__pypy__/test/test_builders.py
@@ -38,7 +38,9 @@
         b = StringBuilder()
         b.append("abc")
         b.append("123")
+        assert len(b) == 6
         b.append("you and me")
         s = b.build()
+        raises(ValueError, len, b)
         assert s == "abc123you and me"
-        raises(ValueError, b.build)
\ No newline at end of file
+        raises(ValueError, b.build)
diff --git a/pypy/module/_continuation/interp_continuation.py b/pypy/module/_continuation/interp_continuation.py
--- a/pypy/module/_continuation/interp_continuation.py
+++ b/pypy/module/_continuation/interp_continuation.py
@@ -28,6 +28,8 @@
     def descr_init(self, w_callable, __args__):
         if self.sthread is not None:
             raise geterror(self.space, "continulet already __init__ialized")
+        sthread = build_sthread(self.space)
+        workaround_disable_jit(sthread)
         #
         # hackish: build the frame "by hand", passing it the correct arguments
         space = self.space
@@ -41,7 +43,6 @@
         self.bottomframe = bottomframe
         #
         global_state.origin = self
-        sthread = build_sthread(self.space)
         self.sthread = sthread
         h = sthread.new(new_stacklet_callback)
         post_switch(sthread, h)
@@ -71,6 +72,7 @@
                 global_state.clear()
                 raise geterror(self.space, "continulet already finished")
         self.check_sthread()
+        workaround_disable_jit(self.sthread)
         #
         global_state.origin = self
         if to is None:
@@ -259,6 +261,16 @@
         sthread = ec.stacklet_thread = SThread(space, ec)
     return sthread
 
+def workaround_disable_jit(sthread):
+    # A bad workaround to kill the JIT anywhere in this thread.
+    # This forces all the frames.  It's a bad workaround because
+    # it takes O(depth) time, and it will cause some "abort:
+    # vable escape" in the JIT.  The goal is to prevent any frame
+    # from being still virtuals, because the JIT generates code
+    # to un-virtualizable them "on demand" by loading values based
+    # on FORCE_TOKEN, which is an address in the stack.
+    sthread.ec.force_all_frames()
+
 # ____________________________________________________________
 
 def permute(space, args_w):
diff --git a/pypy/module/_file/test/test_file.py b/pypy/module/_file/test/test_file.py
--- a/pypy/module/_file/test/test_file.py
+++ b/pypy/module/_file/test/test_file.py
@@ -1,5 +1,5 @@
 from __future__ import with_statement
-import py
+import py, os, errno
 
 from pypy.conftest import gettestobjspace, option
 
@@ -257,6 +257,39 @@
             assert self.temppath in g.getvalue()
 
 
+class AppTestNonblocking(object):
+    def setup_class(cls):
+        from pypy.module._file.interp_file import W_File
+
+        cls.old_read = os.read
+
+        if option.runappdirect:
+            py.test.skip("works with internals of _file impl on py.py")
+
+        state = [0]
+        def read(fd, n=None):
+            if fd != 42:
+                return cls.old_read(fd, n)
+            if state[0] == 0:
+                state[0] += 1
+                return "xyz"
+            if state[0] < 3:
+                state[0] += 1
+                raise OSError(errno.EAGAIN, "xyz")
+            return ''
+        os.read = read
+        stdin = W_File(cls.space)
+        stdin.file_fdopen(42, "r", 1)
+        stdin.name = '<stdin>'
+        cls.w_stream = stdin
+
+    def teardown_class(cls):
+        os.read = cls.old_read
+
+    def test_nonblocking_file(self):
+        res = self.stream.read()
+        assert res == 'xyz'
+
 class AppTestConcurrency(object):
     # these tests only really make sense on top of a translated pypy-c,
     # because on top of py.py the inner calls to os.write() don't
@@ -374,24 +407,24 @@
         with self.file(self.temppath, 'w') as f:
             f.write('foo')
         assert f.closed
-        
+
         with self.file(self.temppath, 'r') as f:
             s = f.readline()
 
         assert s == "foo"
         assert f.closed
-    
+
     def test_subclass_with(self):
         file = self.file
         class C(file):
             def __init__(self, *args, **kwargs):
                 self.subclass_closed = False
                 file.__init__(self, *args, **kwargs)
-            
+
             def close(self):
                 self.subclass_closed = True
                 file.close(self)
-        
+
         with C(self.temppath, 'w') as f:
             pass
         assert f.subclass_closed
diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -23,6 +23,8 @@
         ("arccos", "arccos"),
         ("arcsin", "arcsin"),
         ("arctan", "arctan"),
+        ("arcsinh", "arcsinh"),
+        ("arctanh", "arctanh"),
         ("copysign", "copysign"),
         ("cos", "cos"),
         ("divide", "divide"),
@@ -50,4 +52,6 @@
     appleveldefs = {
         'average': 'app_numpy.average',
         'mean': 'app_numpy.mean',
+        'inf': 'app_numpy.inf',
+        'e': 'app_numpy.e',
     }
diff --git a/pypy/module/micronumpy/app_numpy.py b/pypy/module/micronumpy/app_numpy.py
--- a/pypy/module/micronumpy/app_numpy.py
+++ b/pypy/module/micronumpy/app_numpy.py
@@ -1,5 +1,11 @@
+import math
+
 import numpy
 
+
+inf = float("inf")
+e = math.e
+
 def average(a):
     # This implements a weighted average, for now we don't implement the
     # weighting, just the average part!
@@ -8,4 +14,4 @@
 def mean(a):
     if not hasattr(a, "mean"):
         a = numpy.array(a)
-    return a.mean()
\ No newline at end of file
+    return a.mean()
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -7,13 +7,14 @@
 from pypy.interpreter.typedef import TypeDef, interp_attrproperty, GetSetProperty
 from pypy.module.micronumpy import signature
 from pypy.objspace.std.floatobject import float2string
-from pypy.rlib import rfloat
-from pypy.rlib.rarithmetic import widen
+from pypy.rlib import rarithmetic, rfloat
+from pypy.rlib.rarithmetic import LONG_BIT, widen
 from pypy.rlib.objectmodel import specialize, enforceargs
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.rpython.lltypesystem import lltype, rffi
 
 
+UNSIGNEDLTR = "u"
 SIGNEDLTR = "i"
 BOOLLTR = "b"
 FLOATINGLTR = "f"
@@ -61,7 +62,10 @@
             self.val = val
 
         def wrap(self, space):
-            return space.wrap(self.val)
+            val = self.val
+            if valtype is rarithmetic.r_singlefloat:
+                val = float(val)
+            return space.wrap(val)
 
         def convert_to(self, dtype):
             return dtype.adapt_val(self.val)
@@ -145,7 +149,7 @@
         return self.adapt_val(func(self, self.for_computation(self.unbox(v))))
     return impl
 
-class ArithmaticTypeMixin(object):
+class ArithmeticTypeMixin(object):
     _mixin_ = True
 
     @binop
@@ -200,11 +204,17 @@
         return v1 >= v2
 
 
-class FloatArithmeticDtype(ArithmaticTypeMixin):
+class FloatArithmeticDtype(ArithmeticTypeMixin):
     _mixin_ = True
 
+    def unwrap(self, space, w_item):
+        return self.adapt_val(space.float_w(space.float(w_item)))
+
     def for_computation(self, v):
-        return v
+        return float(v)
+
+    def str_format(self, item):
+        return float2string(self.for_computation(self.unbox(item)), 'g', rfloat.DTSF_STR_PRECISION)
 
     @binop
     def mod(self, v1, v2):
@@ -250,19 +260,29 @@
         return math.tan(v)
     @unaryop
     def arcsin(self, v):
-        if v < -1.0 or v > 1.0:
+        if not -1.0 <= v <= 1.0:
             return rfloat.NAN
         return math.asin(v)
     @unaryop
     def arccos(self, v):
-        if v < -1.0 or v > 1.0:
+        if not -1.0 <= v <= 1.0:
             return rfloat.NAN
         return math.acos(v)
     @unaryop
     def arctan(self, v):
         return math.atan(v)
+    @unaryop
+    def arcsinh(self, v):
+        return math.asinh(v)
+    @unaryop
+    def arctanh(self, v):
+        if v == 1.0 or v == -1.0:
+            return math.copysign(rfloat.INFINITY, v)
+        if not -1.0 < v < 1.0:
+            return rfloat.NAN
+        return math.atanh(v)
 
-class IntegerArithmeticDtype(ArithmaticTypeMixin):
+class IntegerArithmeticDtype(ArithmeticTypeMixin):
     _mixin_ = True
 
     def unwrap(self, space, w_item):
@@ -271,10 +291,16 @@
     def for_computation(self, v):
         return widen(v)
 
+    def str_format(self, item):
+        return str(widen(self.unbox(item)))
+
     @binop
     def mod(self, v1, v2):
         return v1 % v2
 
+class SignedIntegerArithmeticDtype(IntegerArithmeticDtype):
+    _mixin_ = True
+
     @unaryop
     def sign(self, v):
         if v > 0:
@@ -285,17 +311,22 @@
             assert v == 0
             return 0
 
-    def str_format(self, item):
-        return str(widen(self.unbox(item)))
+class UnsignedIntegerArithmeticDtype(IntegerArithmeticDtype):
+    _mixin_ = True
+
+    @unaryop
+    def sign(self, v):
+        return int(v != 0)
+
 
 W_BoolDtype = create_low_level_dtype(
     num = 0, kind = BOOLLTR, name = "bool",
-    aliases = ["?"],
+    aliases = ["?", "bool", "bool8"],
     applevel_types = ["bool"],
     T = lltype.Bool,
     valtype = bool,
 )
-class W_BoolDtype(IntegerArithmeticDtype, W_BoolDtype):
+class W_BoolDtype(SignedIntegerArithmeticDtype, W_BoolDtype):
     def unwrap(self, space, w_item):
         return self.adapt_val(space.is_true(w_item))
 
@@ -308,67 +339,138 @@
 
 W_Int8Dtype = create_low_level_dtype(
     num = 1, kind = SIGNEDLTR, name = "int8",
-    aliases = ["int8"],
+    aliases = ["b", "int8", "i1"],
     applevel_types = [],
     T = rffi.SIGNEDCHAR,
     valtype = rffi.SIGNEDCHAR._type,
     expected_size = 1,
 )
-class W_Int8Dtype(IntegerArithmeticDtype, W_Int8Dtype):
+class W_Int8Dtype(SignedIntegerArithmeticDtype, W_Int8Dtype):
+    pass
+
+W_UInt8Dtype = create_low_level_dtype(
+    num = 2, kind = UNSIGNEDLTR, name = "uint8",
+    aliases = ["B", "uint8", "I1"],
+    applevel_types = [],
+    T = rffi.UCHAR,
+    valtype = rffi.UCHAR._type,
+    expected_size = 1,
+)
+class W_UInt8Dtype(UnsignedIntegerArithmeticDtype, W_UInt8Dtype):
     pass
 
 W_Int16Dtype = create_low_level_dtype(
     num = 3, kind = SIGNEDLTR, name = "int16",
-    aliases = ["int16"],
+    aliases = ["h", "int16", "i2"],
     applevel_types = [],
     T = rffi.SHORT,
     valtype = rffi.SHORT._type,
     expected_size = 2,
 )
-class W_Int16Dtype(IntegerArithmeticDtype, W_Int16Dtype):
+class W_Int16Dtype(SignedIntegerArithmeticDtype, W_Int16Dtype):
+    pass
+
+W_UInt16Dtype = create_low_level_dtype(
+    num = 4, kind = UNSIGNEDLTR, name = "uint16",
+    aliases = ["H", "uint16", "I2"],
+    applevel_types = [],
+    T = rffi.USHORT,
+    valtype = rffi.USHORT._type,
+    expected_size = 2,
+)
+class W_UInt16Dtype(UnsignedIntegerArithmeticDtype, W_UInt16Dtype):
     pass
 
 W_Int32Dtype = create_low_level_dtype(
     num = 5, kind = SIGNEDLTR, name = "int32",
-    aliases = ["i"],
+    aliases = ["i", "int32", "i4"],
     applevel_types = [],
     T = rffi.INT,
     valtype = rffi.INT._type,
     expected_size = 4,
 )
-class W_Int32Dtype(IntegerArithmeticDtype, W_Int32Dtype):
+class W_Int32Dtype(SignedIntegerArithmeticDtype, W_Int32Dtype):
+    pass
+
+W_UInt32Dtype = create_low_level_dtype(
+    num = 6, kind = UNSIGNEDLTR, name = "uint32",
+    aliases = ["I", "uint32", "I4"],
+    applevel_types = [],
+    T = rffi.UINT,
+    valtype = rffi.UINT._type,
+    expected_size = 4,
+)
+class W_UInt32Dtype(UnsignedIntegerArithmeticDtype, W_UInt32Dtype):
     pass
 
 W_Int64Dtype = create_low_level_dtype(
     num = 9, kind = SIGNEDLTR, name = "int64",
-    aliases = [],
+    aliases = ["q", "int64", "i8"],
     applevel_types = ["long"],
     T = rffi.LONGLONG,
     valtype = rffi.LONGLONG._type,
     expected_size = 8,
 )
-class W_Int64Dtype(IntegerArithmeticDtype, W_Int64Dtype):
+class W_Int64Dtype(SignedIntegerArithmeticDtype, W_Int64Dtype):
+    pass
+
+W_UInt64Dtype = create_low_level_dtype(
+    num = 10, kind = UNSIGNEDLTR, name = "uint64",
+    aliases = ["Q", "uint64", "I8"],
+    applevel_types = [],
+    T = rffi.ULONGLONG,
+    valtype = rffi.ULONGLONG._type,
+    expected_size = 8,
+)
+class W_UInt64Dtype(UnsignedIntegerArithmeticDtype, W_UInt64Dtype):
+    pass
+
+if LONG_BIT == 32:
+    long_dtype = W_Int32Dtype
+    ulong_dtype = W_UInt32Dtype
+elif LONG_BIT == 64:
+    long_dtype = W_Int64Dtype
+    ulong_dtype = W_UInt64Dtype
+else:
+    assert False
+
+class W_LongDtype(long_dtype):
+    num = 7
+    aliases = ["l"]
+    applevel_types = ["int"]
+
+class W_ULongDtype(ulong_dtype):
+    num = 8
+    aliases = ["L"]
+
+W_Float32Dtype = create_low_level_dtype(
+    num = 11, kind = FLOATINGLTR, name = "float32",
+    aliases = ["f", "float32", "f4"],
+    applevel_types = [],
+    T = lltype.SingleFloat,
+    valtype = rarithmetic.r_singlefloat,
+    expected_size = 4,
+)
+class W_Float32Dtype(FloatArithmeticDtype, W_Float32Dtype):
     pass
 
 W_Float64Dtype = create_low_level_dtype(
     num = 12, kind = FLOATINGLTR, name = "float64",
-    aliases = [],
+    aliases = ["d", "float64", "f8"],
     applevel_types = ["float"],
     T = lltype.Float,
     valtype = float,
     expected_size = 8,
 )
 class W_Float64Dtype(FloatArithmeticDtype, W_Float64Dtype):
-    def unwrap(self, space, w_item):
-        return self.adapt_val(space.float_w(space.float(w_item)))
-
-    def str_format(self, item):
-        return float2string(self.unbox(item), 'g', rfloat.DTSF_STR_PRECISION)
+    pass
 
 ALL_DTYPES = [
     W_BoolDtype,
-    W_Int8Dtype, W_Int16Dtype, W_Int32Dtype, W_Int64Dtype,
-    W_Float64Dtype
+    W_Int8Dtype, W_UInt8Dtype, W_Int16Dtype, W_UInt16Dtype,
+    W_Int32Dtype, W_UInt32Dtype, W_LongDtype, W_ULongDtype,
+    W_Int64Dtype, W_UInt64Dtype,
+    W_Float32Dtype, W_Float64Dtype,
 ]
 
 dtypes_by_alias = unrolling_iterable([
@@ -395,6 +497,7 @@
 
     num = interp_attrproperty("num", cls=W_Dtype),
     kind = interp_attrproperty("kind", cls=W_Dtype),
+    itemsize = interp_attrproperty("num_bytes", cls=W_Dtype),
     shape = GetSetProperty(W_Dtype.descr_get_shape),
 )
 W_Dtype.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -4,6 +4,7 @@
 from pypy.interpreter.typedef import TypeDef, GetSetProperty, interp_attrproperty
 from pypy.module.micronumpy import interp_dtype, signature
 from pypy.rlib import jit
+from pypy.rlib.rarithmetic import LONG_BIT
 from pypy.tool.sourcetools import func_with_new_name
 
 
@@ -180,23 +181,56 @@
 
     # Everything promotes to float, and bool promotes to everything.
     if dt2.kind == interp_dtype.FLOATINGLTR or dt1.kind == interp_dtype.BOOLLTR:
+        # Float32 + 8-bit int = Float64
+        if dt2.num == 11 and dt1.num_bytes >= 4:
+            return space.fromcache(interp_dtype.W_Float64Dtype)
         return dt2
 
-    assert False
+    # for now this means mixing signed and unsigned
+    if dt2.kind == interp_dtype.SIGNEDLTR:
+        # if dt2 has a greater number of bytes, then just go with it
+        if dt1.num_bytes < dt2.num_bytes:
+            return dt2
+        # we need to promote both dtypes
+        dtypenum = dt2.num + 2
+    else:
+        # increase to the next signed type (or to float)
+        dtypenum = dt2.num + 1
+        # UInt64 + signed = Float64
+        if dt2.num == 10:
+            dtypenum += 1
+    newdtype = interp_dtype.ALL_DTYPES[dtypenum]
+
+    if newdtype.num_bytes > dt2.num_bytes or newdtype.kind == interp_dtype.FLOATINGLTR:
+        return space.fromcache(newdtype)
+    else:
+        # we only promoted to long on 32-bit or to longlong on 64-bit
+        # this is really for dealing with the Long and Ulong dtypes
+        if LONG_BIT == 32:
+            dtypenum += 2
+        else:
+            dtypenum += 3
+        return space.fromcache(interp_dtype.ALL_DTYPES[dtypenum])
 
 def find_unaryop_result_dtype(space, dt, promote_to_float=False,
     promote_bools=False, promote_to_largest=False):
     if promote_bools and (dt.kind == interp_dtype.BOOLLTR):
         return space.fromcache(interp_dtype.W_Int8Dtype)
     if promote_to_float:
+        if dt.kind == interp_dtype.FLOATINGLTR:
+            return dt
+        if dt.num >= 5:
+            return space.fromcache(interp_dtype.W_Float64Dtype)
         for bytes, dtype in interp_dtype.dtypes_by_num_bytes:
-            if dtype.kind == interp_dtype.FLOATINGLTR and dtype.num_bytes >= dt.num_bytes:
+            if dtype.kind == interp_dtype.FLOATINGLTR and dtype.num_bytes > dt.num_bytes:
                 return space.fromcache(dtype)
     if promote_to_largest:
         if dt.kind == interp_dtype.BOOLLTR or dt.kind == interp_dtype.SIGNEDLTR:
             return space.fromcache(interp_dtype.W_Int64Dtype)
         elif dt.kind == interp_dtype.FLOATINGLTR:
             return space.fromcache(interp_dtype.W_Float64Dtype)
+        elif dt.kind == interp_dtype.UNSIGNEDLTR:
+            return space.fromcache(interp_dtype.W_UInt64Dtype)
         else:
             assert False
     return dt
@@ -205,15 +239,23 @@
     w_type = space.type(w_obj)
 
     bool_dtype = space.fromcache(interp_dtype.W_BoolDtype)
+    long_dtype = space.fromcache(interp_dtype.W_LongDtype)
     int64_dtype = space.fromcache(interp_dtype.W_Int64Dtype)
 
     if space.is_w(w_type, space.w_bool):
-        if current_guess is None:
+        if current_guess is None or current_guess is bool_dtype:
             return bool_dtype
+        return current_guess
     elif space.is_w(w_type, space.w_int):
         if (current_guess is None or current_guess is bool_dtype or
-            current_guess is int64_dtype):
+            current_guess is long_dtype):
+            return long_dtype
+        return current_guess
+    elif space.is_w(w_type, space.w_long):
+        if (current_guess is None or current_guess is bool_dtype or
+            current_guess is long_dtype or current_guess is int64_dtype):
             return int64_dtype
+        return current_guess
     return space.fromcache(interp_dtype.W_Float64Dtype)
 
 
@@ -225,7 +267,9 @@
         def impl(res_dtype, lvalue, rvalue):
             res = getattr(res_dtype, op_name)(lvalue, rvalue)
             if comparison_func:
-                res = space.fromcache(interp_dtype.W_BoolDtype).box(res)
+                booldtype = space.fromcache(interp_dtype.W_BoolDtype)
+                assert isinstance(booldtype, interp_dtype.W_BoolDtype)
+                res = booldtype.box(res)
             return res
     return func_with_new_name(impl, ufunc_name)
 
@@ -268,6 +312,8 @@
             ("arcsin", "arcsin", 1, {"promote_to_float": True}),
             ("arccos", "arccos", 1, {"promote_to_float": True}),
             ("arctan", "arctan", 1, {"promote_to_float": True}),
+            ("arcsinh", "arcsinh", 1, {"promote_to_float": True}),
+            ("arctanh", "arctanh", 1, {"promote_to_float": True}),
         ]:
             self.add_ufunc(space, *ufunc_def)
 
@@ -277,7 +323,7 @@
 
         identity = extra_kwargs.get("identity")
         if identity is not None:
-            identity = space.fromcache(interp_dtype.W_Int64Dtype).adapt_val(identity)
+            identity = space.fromcache(interp_dtype.W_LongDtype).adapt_val(identity)
         extra_kwargs["identity"] = identity
 
         func = ufunc_dtype_caller(space, ufunc_name, op_name, argcount,
@@ -290,4 +336,4 @@
         setattr(self, ufunc_name, ufunc)
 
 def get(space):
-    return space.fromcache(UfuncState)
\ No newline at end of file
+    return space.fromcache(UfuncState)
diff --git a/pypy/module/micronumpy/test/test_base.py b/pypy/module/micronumpy/test/test_base.py
--- a/pypy/module/micronumpy/test/test_base.py
+++ b/pypy/module/micronumpy/test/test_base.py
@@ -64,18 +64,46 @@
     def test_unaryops(self, space):
         bool_dtype = space.fromcache(interp_dtype.W_BoolDtype)
         int8_dtype = space.fromcache(interp_dtype.W_Int8Dtype)
+        uint8_dtype = space.fromcache(interp_dtype.W_UInt8Dtype)
+        int16_dtype = space.fromcache(interp_dtype.W_Int16Dtype)
+        uint16_dtype = space.fromcache(interp_dtype.W_UInt16Dtype)
         int32_dtype = space.fromcache(interp_dtype.W_Int32Dtype)
+        uint32_dtype = space.fromcache(interp_dtype.W_UInt32Dtype)
+        long_dtype = space.fromcache(interp_dtype.W_LongDtype)
+        ulong_dtype = space.fromcache(interp_dtype.W_ULongDtype)
+        int64_dtype = space.fromcache(interp_dtype.W_Int64Dtype)
+        uint64_dtype = space.fromcache(interp_dtype.W_UInt64Dtype)
+        float32_dtype = space.fromcache(interp_dtype.W_Float32Dtype)
         float64_dtype = space.fromcache(interp_dtype.W_Float64Dtype)
 
-        # Normal rules, everythign returns itself
+        # Normal rules, everything returns itself
         assert find_unaryop_result_dtype(space, bool_dtype) is bool_dtype
         assert find_unaryop_result_dtype(space, int8_dtype) is int8_dtype
+        assert find_unaryop_result_dtype(space, uint8_dtype) is uint8_dtype
+        assert find_unaryop_result_dtype(space, int16_dtype) is int16_dtype
+        assert find_unaryop_result_dtype(space, uint16_dtype) is uint16_dtype
         assert find_unaryop_result_dtype(space, int32_dtype) is int32_dtype
+        assert find_unaryop_result_dtype(space, uint32_dtype) is uint32_dtype
+        assert find_unaryop_result_dtype(space, long_dtype) is long_dtype
+        assert find_unaryop_result_dtype(space, ulong_dtype) is ulong_dtype
+        assert find_unaryop_result_dtype(space, int64_dtype) is int64_dtype
+        assert find_unaryop_result_dtype(space, uint64_dtype) is uint64_dtype
+        assert find_unaryop_result_dtype(space, float32_dtype) is float32_dtype
         assert find_unaryop_result_dtype(space, float64_dtype) is float64_dtype
 
         # Coerce to floats, some of these will eventually be float16, or
         # whatever our smallest float type is.
-        assert find_unaryop_result_dtype(space, bool_dtype, promote_to_float=True) is float64_dtype
-        assert find_unaryop_result_dtype(space, int8_dtype, promote_to_float=True) is float64_dtype
+        assert find_unaryop_result_dtype(space, bool_dtype, promote_to_float=True) is float32_dtype # will be float16 if we ever put that in
+        assert find_unaryop_result_dtype(space, int8_dtype, promote_to_float=True) is float32_dtype # will be float16 if we ever put that in
+        assert find_unaryop_result_dtype(space, uint8_dtype, promote_to_float=True) is float32_dtype # will be float16 if we ever put that in
+        assert find_unaryop_result_dtype(space, int16_dtype, promote_to_float=True) is float32_dtype
+        assert find_unaryop_result_dtype(space, uint16_dtype, promote_to_float=True) is float32_dtype
         assert find_unaryop_result_dtype(space, int32_dtype, promote_to_float=True) is float64_dtype
-        assert find_unaryop_result_dtype(space, float64_dtype, promote_to_float=True) is float64_dtype
\ No newline at end of file
+        assert find_unaryop_result_dtype(space, uint32_dtype, promote_to_float=True) is float64_dtype
+        assert find_unaryop_result_dtype(space, int64_dtype, promote_to_float=True) is float64_dtype
+        assert find_unaryop_result_dtype(space, uint64_dtype, promote_to_float=True) is float64_dtype
+        assert find_unaryop_result_dtype(space, float32_dtype, promote_to_float=True) is float32_dtype
+        assert find_unaryop_result_dtype(space, float64_dtype, promote_to_float=True) is float64_dtype
+
+        # promote bools, happens with sign ufunc
+        assert find_unaryop_result_dtype(space, bool_dtype, promote_bools=True) is int8_dtype
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -17,6 +17,7 @@
         from numpy import dtype
 
         assert dtype(bool).num == 0
+        assert dtype(int).num == 7
         assert dtype(long).num == 9
         assert dtype(float).num == 12
 
@@ -81,6 +82,48 @@
             assert isinstance(a[i], (int, long))
             assert a[1] == 1
 
+    def test_overflow(self):
+        from numpy import array, dtype
+        assert array([128], 'b')[0] == -128
+        assert array([256], 'B')[0] == 0
+        assert array([32768], 'h')[0] == -32768
+        assert array([65536], 'H')[0] == 0
+        if dtype('l').itemsize == 4: # 32-bit
+            raises(OverflowError, "array([2**32/2], 'i')")
+            raises(OverflowError, "array([2**32], 'I')")
+        raises(OverflowError, "array([2**64/2], 'q')")
+        raises(OverflowError, "array([2**64], 'Q')")
+
+    def test_bool_binop_types(self):
+        from numpy import array, dtype
+        types = ('?','b','B','h','H','i','I','l','L','q','Q','f','d')
+        N = len(types)
+        a = array([True], '?')
+        for t in types:
+            assert (a + array([0], t)).dtype is dtype(t)
+
+    def test_binop_types(self):
+        from numpy import array, dtype
+        tests = [('b','B','h'), ('b','h','h'), ('b','H','i'), ('b','i','i'),
+                 ('b','l','l'), ('b','q','q'), ('b','Q','d'), ('B','h','h'),
+                 ('B','H','H'), ('B','i','i'), ('B','I','I'), ('B','l','l'),
+                 ('B','L','L'), ('B','q','q'), ('B','Q','Q'), ('h','H','i'),
+                 ('h','i','i'), ('h','l','l'), ('h','q','q'), ('h','Q','d'),
+                 ('H','i','i'), ('H','I','I'), ('H','l','l'), ('H','L','L'),
+                 ('H','q','q'), ('H','Q','Q'), ('i','l','l'), ('i','q','q'),
+                 ('i','Q','d'), ('I','L','L'), ('I','q','q'), ('I','Q','Q'),
+                 ('q','Q','d'), ('b','f','f'), ('B','f','f'), ('h','f','f'),
+                 ('H','f','f'), ('i','f','d'), ('I','f','d'), ('l','f','d'),
+                 ('L','f','d'), ('q','f','d'), ('Q','f','d'), ('q','d','d')]
+        if dtype('i').itemsize == dtype('l').itemsize: # 32-bit
+            tests.extend([('b','I','q'), ('b','L','q'), ('h','I','q'),
+                          ('h','L','q'), ('i','I','q'), ('i','L','q')])
+        else:
+            tests.extend([('b','I','l'), ('b','L','d'), ('h','I','l'),
+                          ('h','L','d'), ('i','I','l'), ('i','L','d')])
+        for d1, d2, dout in tests:
+            assert (array([1], d1) + array([1], d2)).dtype is dtype(dout)
+
     def test_add_int8(self):
         from numpy import array, dtype
 
@@ -99,6 +142,15 @@
         for i in range(5):
             assert b[i] == i * 2
 
+    def test_add_uint32(self):
+        from numpy import array, dtype
+
+        a = array(range(5), dtype="I")
+        b = a + a
+        assert b.dtype is dtype("I")
+        for i in range(5):
+            assert b[i] == i * 2
+
     def test_shape(self):
         from numpy import dtype
 
diff --git a/pypy/module/micronumpy/test/test_module.py b/pypy/module/micronumpy/test/test_module.py
--- a/pypy/module/micronumpy/test/test_module.py
+++ b/pypy/module/micronumpy/test/test_module.py
@@ -10,4 +10,12 @@
     def test_average(self):
         from numpy import array, average
         assert average(range(10)) == 4.5
-        assert average(array(range(10))) == 4.5
\ No newline at end of file
+        assert average(array(range(10))) == 4.5
+
+    def test_constants(self):
+        import math
+        from numpy import inf, e
+        assert type(inf) is float
+        assert inf == float("inf")
+        assert e == math.e
+        assert type(e) is float
\ No newline at end of file
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -551,8 +551,10 @@
         from numpy import array, dtype
 
         assert array([True]).dtype is dtype(bool)
-        assert array([True, 1]).dtype is dtype(long)
-        assert array([1, 2, 3]).dtype is dtype(long)
+        assert array([True, False]).dtype is dtype(bool)
+        assert array([True, 1]).dtype is dtype(int)
+        assert array([1, 2, 3]).dtype is dtype(int)
+        assert array([1L, 2, 3]).dtype is dtype(long)
         assert array([1.2, True]).dtype is dtype(float)
         assert array([1.2, 5]).dtype is dtype(float)
         assert array([]).dtype is dtype(float)
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -234,7 +234,7 @@
             assert b[i] == math.sin(a[i])
 
         a = sin(array([True, False], dtype=bool))
-        assert a[0] == sin(1)
+        assert abs(a[0] - sin(1)) < 1e-7 # a[0] will be less precise
         assert a[1] == 0.0
 
     def test_cos(self):
@@ -298,6 +298,25 @@
         b = arctan(a)
         assert math.isnan(b[0])
 
+    def test_arcsinh(self):
+        import math
+        from numpy import arcsinh, inf
+
+        for v in [inf, -inf, 1.0, math.e]:
+            assert math.asinh(v) == arcsinh(v)
+        assert math.isnan(arcsinh(float("nan")))
+
+    def test_arctanh(self):
+        import math
+        from numpy import arctanh
+
+        for v in [.99, .5, 0, -.5, -.99]:
+            assert math.atanh(v) == arctanh(v)
+        for v in [2.0, -2.0]:
+            assert math.isnan(arctanh(v))
+        for v in [1.0, -1.0]:
+            assert arctanh(v) == math.copysign(float("inf"), v)
+
     def test_reduce_errors(self):
         from numpy import sin, add
 
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -1,20 +1,24 @@
 from pypy.jit.metainterp.test.support import LLJitMixin
 from pypy.module.micronumpy import interp_ufuncs, signature
 from pypy.module.micronumpy.compile import (numpy_compile, FakeSpace,
-    FloatObject)
-from pypy.module.micronumpy.interp_dtype import W_Float64Dtype, W_Int64Dtype
+    FloatObject, IntObject)
+from pypy.module.micronumpy.interp_dtype import W_Int32Dtype, W_Float64Dtype, W_Int64Dtype, W_UInt64Dtype
 from pypy.module.micronumpy.interp_numarray import (BaseArray, SingleDimArray,
     SingleDimSlice, scalar_w)
 from pypy.rlib.nonconst import NonConstant
 from pypy.rpython.annlowlevel import llstr
 from pypy.rpython.test.test_llinterp import interpret
 
+import py
+
 
 class TestNumpyJIt(LLJitMixin):
     def setup_class(cls):
         cls.space = FakeSpace()
         cls.float64_dtype = cls.space.fromcache(W_Float64Dtype)
         cls.int64_dtype = cls.space.fromcache(W_Int64Dtype)
+        cls.uint64_dtype = cls.space.fromcache(W_UInt64Dtype)
+        cls.int32_dtype = cls.space.fromcache(W_Int32Dtype)
 
     def test_add(self):
         def f(i):
@@ -303,6 +307,31 @@
                           'int_lt': 1, 'guard_true': 1, 'jump': 1})
         assert result == 11.0
 
+    def test_int32_sum(self):
+        py.test.skip("pypy/jit/backend/llimpl.py needs to be changed to "
+                     "deal correctly with int dtypes for this test to "
+                     "work. skip for now until someone feels up to the task")
+        space = self.space
+        float64_dtype = self.float64_dtype
+        int32_dtype = self.int32_dtype
+
+        def f(n):
+            if NonConstant(False):
+                dtype = float64_dtype
+            else:
+                dtype = int32_dtype
+            ar = SingleDimArray(n, dtype=dtype)
+            i = 0
+            while i < n:
+                ar.get_concrete().setitem(i, int32_dtype.box(7))
+                i += 1
+            v = ar.descr_add(space, ar).descr_sum(space)
+            assert isinstance(v, IntObject)
+            return v.intval
+
+        result = self.meta_interp(f, [5], listops=True, backendopt=True)
+        assert result == f(5)
+
 class TestTranslation(object):
     def test_compile(self):
         x = numpy_compile('aa+f*f/a-', 10)
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -10,6 +10,7 @@
 from pypy.rpython.lltypesystem import rffi, lltype
 from pypy.rpython.tool import rffi_platform
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
+from pypy.module.sys.interp_encoding import getfilesystemencoding
 
 import os, sys
 _WIN = sys.platform == 'win32'
@@ -32,17 +33,19 @@
             raise OperationError(space.w_OverflowError,
                                  space.wrap("integer out of range"))
 
+def fsencode_w(space, w_obj):
+    if space.isinstance_w(w_obj, space.w_unicode):
+        w_obj = space.call_method(w_obj, 'encode',
+                                  getfilesystemencoding(space))
+    return space.str_w(w_obj)
+
 class FileEncoder(object):
     def __init__(self, space, w_obj):
         self.space = space
         self.w_obj = w_obj
 
     def as_bytes(self):
-        from pypy.module.sys.interp_encoding import getfilesystemencoding
-        space = self.space
-        w_bytes = space.call_method(self.w_obj, 'encode',
-                                    getfilesystemencoding(space))
-        return space.str_w(w_bytes)
+        return fsencode_w(self.space, self.w_obj)
 
     def as_unicode(self):
         return self.space.unicode_w(self.w_obj)
@@ -56,7 +59,6 @@
         return self.space.str_w(self.w_obj)
 
     def as_unicode(self):
-        from pypy.module.sys.interp_encoding import getfilesystemencoding
         space = self.space
         w_unicode = space.call_method(self.w_obj, 'decode',
                                       getfilesystemencoding(space))
@@ -536,7 +538,6 @@
 
 The list is in arbitrary order.  It does not include the special
 entries '.' and '..' even if they are present in the directory."""
-    from pypy.module.sys.interp_encoding import getfilesystemencoding
     try:
         if space.isinstance_w(w_dirname, space.w_unicode):
             dirname = FileEncoder(space, w_dirname)
@@ -734,8 +735,7 @@
 def _exit(space, status):
     os._exit(status)
 
- at unwrap_spec(command=str)
-def execv(space, command, w_args):
+def execv(space, w_command, w_args):
     """ execv(path, args)
 
 Execute an executable path with arguments, replacing current process.
@@ -743,12 +743,13 @@
         path: path of executable file
         args: iterable of strings
     """
+    command = fsencode_w(space, w_command)
     try:
         args_w = space.unpackiterable(w_args)
         if len(args_w) < 1:
             w_msg = space.wrap("execv() must have at least one argument")
             raise OperationError(space.w_ValueError, w_msg)
-        args = [space.str_w(w_arg) for w_arg in args_w]
+        args = [fsencode_w(space, w_arg) for w_arg in args_w]
     except OperationError, e:
         if not e.match(space, space.w_TypeError):
             raise
@@ -759,8 +760,7 @@
     except OSError, e:
         raise wrap_oserror(space, e)
 
- at unwrap_spec(command=str)
-def execve(space, command, w_args, w_env):
+def execve(space, w_command, w_args, w_env):
     """ execve(path, args, env)
 
 Execute a path with arguments and environment, replacing current process.
@@ -769,7 +769,8 @@
         args: iterable of arguments
         env: dictionary of strings mapping to strings
     """
-    args = [space.str_w(w_arg) for w_arg in space.unpackiterable(w_args)]
+    command = fsencode_w(space, w_command)
+    args = [fsencode_w(space, w_arg) for w_arg in space.unpackiterable(w_args)]
     env = {}
     w_keys = space.call_method(w_env, 'keys')
     for w_key in space.unpackiterable(w_keys):
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -415,6 +415,23 @@
                 else:
                     py.test.fail("didn't raise")
 
+        def test_execv_unicode(self):
+            os = self.posix
+            import sys
+            if not hasattr(os, "fork"):
+                skip("Need fork() to test execv()")
+            try:
+                output = u"caf\xe9 \u1234\n".encode(sys.getfilesystemencoding())
+            except UnicodeEncodeError:
+                skip("encoding not good enough")
+            pid = os.fork()
+            if pid == 0:
+                os.execv(u"/bin/sh", ["sh", "-c",
+                                      u"echo caf\xe9 \u1234 > onefile"])
+            os.waitpid(pid, 0)
+            assert open("onefile").read() == output
+            os.unlink("onefile")
+
         def test_execve(self):
             os = self.posix
             if not hasattr(os, "fork"):
@@ -425,6 +442,24 @@
             os.waitpid(pid, 0)
             assert open("onefile").read() == "xxx"
             os.unlink("onefile")
+
+        def test_execve_unicode(self):
+            os = self.posix
+            import sys
+            if not hasattr(os, "fork"):
+                skip("Need fork() to test execve()")
+            try:
+                output = u"caf\xe9 \u1234\n".encode(sys.getfilesystemencoding())
+            except UnicodeEncodeError:
+                skip("encoding not good enough")
+            pid = os.fork()
+            if pid == 0:
+                os.execve(u"/bin/sh", ["sh", "-c",
+                                      u"echo caf\xe9 \u1234 > onefile"],
+                          {'ddd': 'xxx'})
+            os.waitpid(pid, 0)
+            assert open("onefile").read() == output
+            os.unlink("onefile")
         pass # <- please, inspect.getsource(), don't crash
 
     if hasattr(__import__(os.name), "spawnv"):
diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py
--- a/pypy/module/pypyjit/interp_jit.py
+++ b/pypy/module/pypyjit/interp_jit.py
@@ -137,20 +137,15 @@
 
     def jump_absolute(self, jumpto, _, ec=None):
         if we_are_jitted():
-            # Normally, the tick counter is decremented by 100 for every
-            # Python opcode.  Here, to better support JIT compilation of
-            # small loops, we decrement it by a possibly smaller constant.
-            # We get the maximum 100 when the (unoptimized) trace length
-            # is at least 3200 (a bit randomly).
-            trace_length = r_uint(current_trace_length())
-            decr_by = trace_length // 32
-            if decr_by < 1:
-                decr_by = 1
-            elif decr_by > 100:    # also if current_trace_length() returned -1
-                decr_by = 100
+            #
+            # assume that only threads are using the bytecode counter
+            decr_by = 0
+            if self.space.actionflag.has_bytecode_counter:   # constant-folded
+                if self.space.threadlocals.gil_ready:   # quasi-immutable field
+                    decr_by = _get_adapted_tick_counter()
             #
             self.last_instr = intmask(jumpto)
-            ec.bytecode_trace(self, intmask(decr_by))
+            ec.bytecode_trace(self, decr_by)
             jumpto = r_uint(self.last_instr)
         #
         pypyjitdriver.can_enter_jit(frame=self, ec=ec, next_instr=jumpto,
@@ -158,6 +153,20 @@
                                     is_being_profiled=self.is_being_profiled)
         return jumpto
 
+def _get_adapted_tick_counter():
+    # Normally, the tick counter is decremented by 100 for every
+    # Python opcode.  Here, to better support JIT compilation of
+    # small loops, we decrement it by a possibly smaller constant.
+    # We get the maximum 100 when the (unoptimized) trace length
+    # is at least 3200 (a bit randomly).
+    trace_length = r_uint(current_trace_length())
+    decr_by = trace_length // 32
+    if decr_by < 1:
+        decr_by = 1
+    elif decr_by > 100:    # also if current_trace_length() returned -1
+        decr_by = 100
+    return intmask(decr_by)
+
 
 PyCode__initialize = PyCode._initialize
 
diff --git a/pypy/module/pypyjit/test_pypy_c/model.py b/pypy/module/pypyjit/test_pypy_c/model.py
--- a/pypy/module/pypyjit/test_pypy_c/model.py
+++ b/pypy/module/pypyjit/test_pypy_c/model.py
@@ -225,6 +225,8 @@
         # strip comment
         if '#' in line:
             line = line[:line.index('#')]
+        if line.strip() == 'guard_not_invalidated?':
+            return 'guard_not_invalidated', None, [], '...', False
         # find the resvar, if any
         if ' = ' in line:
             resvar, _, line = line.partition(' = ')
@@ -249,7 +251,7 @@
             descr = descr[len('descr='):]
         else:
             descr = None
-        return opname, resvar, args, descr
+        return opname, resvar, args, descr, True
 
     @classmethod
     def preprocess_expected_src(cls, src):
@@ -258,13 +260,23 @@
         # replaced with the corresponding operations, so that tests don't have
         # to repeat it every time
         ticker_check = """
+            guard_not_invalidated?
+            ticker0 = getfield_raw(ticker_address, descr=<SignedFieldDescr pypysig_long_struct.c_value .*>)
+            ticker_cond0 = int_lt(ticker0, 0)
+            guard_false(ticker_cond0, descr=...)
+        """
+        src = src.replace('--TICK--', ticker_check)
+        #
+        # this is the ticker check generated if we have threads
+        thread_ticker_check = """
+            guard_not_invalidated?
             ticker0 = getfield_raw(ticker_address, descr=<SignedFieldDescr pypysig_long_struct.c_value .*>)
             ticker1 = int_sub(ticker0, 1)
             setfield_raw(ticker_address, ticker1, descr=<SignedFieldDescr pypysig_long_struct.c_value .*>)
             ticker_cond0 = int_lt(ticker1, 0)
             guard_false(ticker_cond0, descr=...)
         """
-        src = src.replace('--TICK--', ticker_check)
+        src = src.replace('--THREAD-TICK--', thread_ticker_check)
         #
         # this is the ticker check generated in PyFrame.handle_operation_error
         exc_ticker_check = """
@@ -298,7 +310,7 @@
         if not cond:
             raise InvalidMatch(message, frame=sys._getframe(1))
 
-    def match_op(self, op, (exp_opname, exp_res, exp_args, exp_descr)):
+    def match_op(self, op, (exp_opname, exp_res, exp_args, exp_descr, _)):
         self._assert(op.name == exp_opname, "operation mismatch")
         self.match_var(op.res, exp_res)
         if exp_args != ['...']:
@@ -341,7 +353,7 @@
         what is after the '...'
         """
         iter_exp_ops = iter(expected_ops)
-        iter_ops = iter(self.ops)
+        iter_ops = RevertableIterator(self.ops)
         for opindex, exp_op in enumerate(iter_exp_ops):
             try:
                 if exp_op == '...':
@@ -360,6 +372,9 @@
                             break
                 self.match_op(op, exp_op)
             except InvalidMatch, e:
+                if exp_op[4] is False:    # optional operation
+                    iter_ops.revert_one()
+                    continue       # try to match with the next exp_op
                 e.opindex = opindex
                 raise
         #
@@ -398,3 +413,18 @@
         else:
             return True
 
+
+class RevertableIterator(object):
+    def __init__(self, sequence):
+        self.sequence = sequence
+        self.index = 0
+    def __iter__(self):
+        return self
+    def next(self):
+        index = self.index
+        if index == len(self.sequence):
+            raise StopIteration
+        self.index = index + 1
+        return self.sequence[index]
+    def revert_one(self):
+        self.index -= 1
diff --git a/pypy/module/pypyjit/test_pypy_c/test_00_model.py b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
--- a/pypy/module/pypyjit/test_pypy_c/test_00_model.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
@@ -50,8 +50,7 @@
         cmdline.append(str(self.filepath))
         #
         print cmdline, logfile
-        env={'PYPYLOG': 'jit-log-opt,jit-summary:' + str(logfile)}
-        #env={'PYPYLOG': ':' + str(logfile)}
+        env={'PYPYLOG': 'jit-log-opt,jit-log-noopt,jit-summary:' + str(logfile)}
         pipe = subprocess.Popen(cmdline,
                                 env=env,
                                 stdout=subprocess.PIPE,
@@ -146,15 +145,17 @@
 
     def test_parse_op(self):
         res = OpMatcher.parse_op("  a =   int_add(  b,  3 ) # foo")
-        assert res == ("int_add", "a", ["b", "3"], None)
+        assert res == ("int_add", "a", ["b", "3"], None, True)
         res = OpMatcher.parse_op("guard_true(a)")
-        assert res == ("guard_true", None, ["a"], None)
+        assert res == ("guard_true", None, ["a"], None, True)
         res = OpMatcher.parse_op("setfield_gc(p0, i0, descr=<foobar>)")
-        assert res == ("setfield_gc", None, ["p0", "i0"], "<foobar>")
+        assert res == ("setfield_gc", None, ["p0", "i0"], "<foobar>", True)
         res = OpMatcher.parse_op("i1 = getfield_gc(p0, descr=<foobar>)")
-        assert res == ("getfield_gc", "i1", ["p0"], "<foobar>")
+        assert res == ("getfield_gc", "i1", ["p0"], "<foobar>", True)
         res = OpMatcher.parse_op("p0 = force_token()")
-        assert res == ("force_token", "p0", [], None)
+        assert res == ("force_token", "p0", [], None, True)
+        res = OpMatcher.parse_op("guard_not_invalidated?")
+        assert res == ("guard_not_invalidated", None, [], '...', False)
 
     def test_exact_match(self):
         loop = """
@@ -342,7 +343,7 @@
             # this is the actual loop
             'int_lt', 'guard_true', 'int_add',
             # this is the signal checking stuff
-            'getfield_raw', 'int_sub', 'setfield_raw', 'int_lt', 'guard_false',
+            'guard_not_invalidated', 'getfield_raw', 'int_lt', 'guard_false',
             'jump'
             ]
 
@@ -408,7 +409,7 @@
             # this is the actual loop
             'int_lt', 'guard_true', 'force_token', 'int_add',
             # this is the signal checking stuff
-            'getfield_raw', 'int_sub', 'setfield_raw', 'int_lt', 'guard_false',
+            'guard_not_invalidated', 'getfield_raw', 'int_lt', 'guard_false',
             'jump'
             ]
 
@@ -426,10 +427,9 @@
             guard_true(i6, descr=...)
             i8 = int_add(i4, 1)
             # signal checking stuff
+            guard_not_invalidated(descr=...)
             i10 = getfield_raw(37212896, descr=<.* pypysig_long_struct.c_value .*>)
-            i12 = int_sub(i10, 1)
-            setfield_raw(37212896, i12, descr=<.* pypysig_long_struct.c_value .*>)
-            i14 = int_lt(i12, 0)
+            i14 = int_lt(i10, 0)
             guard_false(i14, descr=...)
             jump(p0, p1, p2, p3, i8, descr=...)
         """)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py
--- a/pypy/module/pypyjit/test_pypy_c/test_call.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_call.py
@@ -366,12 +366,12 @@
             # make sure that the "block" is not allocated
             ...
             i20 = force_token()
-            setfield_gc(p0, i20, descr=<SignedFieldDescr .*PyFrame.vable_token .*>)
             p22 = new_with_vtable(19511408)
             p24 = new_array(1, descr=<GcPtrArrayDescr>)
             p26 = new_with_vtable(ConstClass(W_ListObject))
             p27 = new(descr=<SizeDescr .*>)
             p29 = new_array(0, descr=<GcPtrArrayDescr>)
+            setfield_gc(p0, i20, descr=<SignedFieldDescr .*PyFrame.vable_token .*>)
             setfield_gc(p27, p29, descr=<GcPtrFieldDescr list.items .*>)
             setfield_gc(p26, p27, descr=<.* .*W_ListObject.inst_wrappeditems .*>)
             setarrayitem_gc(p24, 0, p26, descr=<GcPtrArrayDescr>)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_generators.py b/pypy/module/pypyjit/test_pypy_c/test_generators.py
--- a/pypy/module/pypyjit/test_pypy_c/test_generators.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_generators.py
@@ -19,8 +19,8 @@
         assert loop.match_by_id("generator", """
             i16 = force_token()
             p45 = new_with_vtable(ConstClass(W_IntObject))
-            setfield_gc(p45, i29, descr=<SignedFieldDescr .*>)
             i47 = arraylen_gc(p8, descr=<GcPtrArrayDescr>) # Should be removed by backend
             setarrayitem_gc(p8, 0, p45, descr=<GcPtrArrayDescr>)
+            setfield_gc(p45, i29, descr=<SignedFieldDescr .*>)
             jump(..., descr=...)
             """)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_instance.py b/pypy/module/pypyjit/test_pypy_c/test_instance.py
--- a/pypy/module/pypyjit/test_pypy_c/test_instance.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_instance.py
@@ -125,8 +125,8 @@
             i12 = force_token()
             --TICK--
             p20 = new_with_vtable(ConstClass(W_IntObject))
+            setfield_gc(ConstPtr(ptr21), p20, descr=<GcPtrFieldDescr .*TypeCell.inst_w_value .*>)
             setfield_gc(p20, i11, descr=<SignedFieldDescr.*W_IntObject.inst_intval .*>)
-            setfield_gc(ConstPtr(ptr21), p20, descr=<GcPtrFieldDescr .*TypeCell.inst_w_value .*>)
             jump(p0, p1, p2, p3, p4, p20, p6, i7, p20, descr=<Loop.>)
         """)
 
diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -156,4 +156,41 @@
             i40 = int_sub(i4, 1)
             --TICK--
             jump(p0, p1, p2, p3, i40, i38, descr=<Loop0>)
-        """)
\ No newline at end of file
+        """)
+
+    def test_getattr_promote(self):
+        def main(n):
+            class A(object):
+                def meth_a(self):
+                    return 1
+                def meth_b(self):
+                    return 2
+            a = A()
+
+            l = ['a', 'b']
+            s = 0
+            for i in range(n):
+                name = 'meth_' + l[i & 1]
+                meth = getattr(a, name) # ID: getattr
+                s += meth()
+            return s
+
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loops = log.loops_by_filename(self.filepath)
+        assert len(loops) == 2
+        for loop in loops:
+            loop.match_by_id('getattr','''
+            guard_not_invalidated(descr=...)
+            i32 = strlen(p31)
+            i34 = int_add(5, i32)
+            p35 = newstr(i34)
+            strsetitem(p35, 0, 109)
+            strsetitem(p35, 1, 101)
+            strsetitem(p35, 2, 116)
+            strsetitem(p35, 3, 104)
+            strsetitem(p35, 4, 95)
+            copystrcontent(p31, p35, 0, 5, i32)
+            i49 = call(ConstClass(_ll_2_str_eq_nonnull__rpy_stringPtr_rpy_stringPtr), p35, ConstPtr(ptr48), descr=<SignedCallDescr>)
+            guard_value(i49, 1, descr=<Guard8>)
+            ''')
diff --git a/pypy/module/pypyjit/test_pypy_c/test_thread.py b/pypy/module/pypyjit/test_pypy_c/test_thread.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/pypyjit/test_pypy_c/test_thread.py
@@ -0,0 +1,28 @@
+from pypy.module.pypyjit.test_pypy_c.test_00_model import BaseTestPyPyC
+
+
+class TestThread(BaseTestPyPyC):
+    def test_simple(self):
+        def main(n):
+            import thread
+            def f():
+                i = 0
+                while i < n:
+                    i += 1
+                done.release()
+
+            done = thread.allocate_lock()
+            done.acquire()
+            thread.start_new_thread(f, ())
+            done.acquire()
+            return 0
+        log = self.run(main, [500])
+        assert round(log.result, 6) == round(main(500), 6)
+        loop, = log.loops_by_filename(self.filepath)
+        assert loop.match("""
+            i2 = int_lt(i0, i1)
+            guard_true(i2, descr=...)
+            i3 = int_add(i0, 1)
+            --THREAD-TICK--
+            jump(..., descr=<Loop0>)
+        """)
diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py
--- a/pypy/module/signal/interp_signal.py
+++ b/pypy/module/signal/interp_signal.py
@@ -109,8 +109,11 @@
         p = pypysig_getaddr_occurred()
         value = p.c_value
         if self.has_bytecode_counter:    # this 'if' is constant-folded
-            value -= by
-            p.c_value = value
+            if jit.isconstant(by) and by == 0:
+                pass     # normally constant-folded too
+            else:
+                value -= by
+                p.c_value = value
         return value
 
 
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -47,7 +47,7 @@
         'pypy_initial_path'     : 'state.pypy_initial_path',
 
         '_getframe'             : 'vm._getframe', 
-        '_current_frames'       : 'vm._current_frames', 
+        '_current_frames'       : 'currentframes._current_frames', 
         'setrecursionlimit'     : 'vm.setrecursionlimit', 
         'getrecursionlimit'     : 'vm.getrecursionlimit', 
         'setcheckinterval'      : 'vm.setcheckinterval', 
@@ -55,6 +55,7 @@
         'exc_info'              : 'vm.exc_info', 
         'exc_clear'             : 'vm.exc_clear', 
         'settrace'              : 'vm.settrace',
+        'gettrace'              : 'vm.gettrace',
         'setprofile'            : 'vm.setprofile',
         'getprofile'            : 'vm.getprofile',
         'call_tracing'          : 'vm.call_tracing',
diff --git a/pypy/module/sys/currentframes.py b/pypy/module/sys/currentframes.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/sys/currentframes.py
@@ -0,0 +1,78 @@
+"""
+Implementation of the 'sys._current_frames()' routine.
+"""
+from pypy.interpreter import gateway
+
+app = gateway.applevel('''
+"NOT_RPYTHON"
+import __builtin__
+
+class fake_code(object):
+    co_name = "?"
+    co_filename = "?"
+    co_firstlineno = 0
+
+class fake_frame(object):
+    f_back = None
+    f_builtins = __builtin__.__dict__
+    f_code = fake_code()
+    f_exc_traceback = None
+    f_exc_type = None
+    f_exc_value = None
+    f_globals = {}
+    f_lasti = -1
+    f_lineno = 0
+    f_locals = {}
+    f_restricted = False
+    f_trace = None
+
+    def __init__(self, f):
+        if f is not None:
+            for name in ["f_builtins", "f_code", "f_globals", "f_lasti",
+                         "f_lineno"]:
+                setattr(self, name, getattr(f, name))
+''')
+
+def _current_frames(space):
+    """_current_frames() -> dictionary
+
+    Return a dictionary mapping each current thread T's thread id to T's
+    current stack "frame".  Functions in the traceback module can build the
+    call stack given such a frame.
+
+    Note that in PyPy this returns fake frame objects, to avoid a runtime
+    penalty everywhere with the JIT.  (So far these fake frames can be
+    completely uninformative depending on the JIT state; we could return
+    more with more efforts.)
+
+    This function should be used for specialized purposes only."""
+    w_result = space.newdict()
+    w_fake_frame = app.wget(space, "fake_frame")
+    w_fake_code  = app.wget(space, "fake_code")
+    ecs = space.threadlocals.getallvalues()
+    for thread_ident, ec in ecs.items():
+        vref = ec.topframeref
+        frames = []
+        while not vref.virtual:
+            f = vref()
+            if f is None:
+                break
+            frames.append(f)
+            vref = f.f_backref
+        else:
+            frames.append(None)
+        #
+        w_topframe = space.wrap(None)
+        w_prevframe = None
+        for f in frames:
+            w_nextframe = space.call_function(w_fake_frame, space.wrap(f))
+            if w_prevframe is None:
+                w_topframe = w_nextframe
+            else:
+                space.setattr(w_prevframe, space.wrap('f_back'), w_nextframe)
+            w_prevframe = w_nextframe
+        #
+        space.setitem(w_result,
+                      space.wrap(thread_ident),
+                      w_topframe)
+    return w_result
diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -478,6 +478,7 @@
         sys.settrace(trace)
         try:
             x()
+            assert sys.gettrace() is trace
         finally:
             sys.settrace(None)
         assert len(counts) == 1
@@ -555,7 +556,7 @@
             return sys._current_frames()
         frames = f()
         assert frames.keys() == [0]
-        assert frames[0].f_code.co_name == 'f'
+        assert frames[0].f_code.co_name in ('f', '?')
 
 class AppTestCurrentFramesWithThread(AppTestCurrentFrames):
     def setup_class(cls):
@@ -567,23 +568,25 @@
         import thread
 
         thread_id = thread.get_ident()
-        self.ready = False
         def other_thread():
-            self.ready = True
             print "thread started"
-            time.sleep(5)
+            lock2.release()
+            lock1.acquire()
+        lock1 = thread.allocate_lock()
+        lock2 = thread.allocate_lock()
+        lock1.acquire()
+        lock2.acquire()
         thread.start_new_thread(other_thread, ())
 
         def f():
-            for i in range(100):
-                if self.ready: break
-                time.sleep(0.1)
+            lock2.acquire()
             return sys._current_frames()
 
         frames = f()
+        lock1.release()
         thisframe = frames.pop(thread_id)
-        assert thisframe.f_code.co_name == 'f'
+        assert thisframe.f_code.co_name in ('f', '?')
 
         assert len(frames) == 1
         _, other_frame = frames.popitem()
-        assert other_frame.f_code.co_name == 'other_thread'
+        assert other_frame.f_code.co_name in ('other_thread', '?')
diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -45,25 +45,6 @@
     f.mark_as_escaped()
     return space.wrap(f)
 
-def _current_frames(space):
-    """_current_frames() -> dictionary
-
-    Return a dictionary mapping each current thread T's thread id to T's
-    current stack frame.
-
-    This function should be used for specialized purposes only."""
-    raise OperationError(space.w_NotImplementedError,
-        space.wrap("XXX sys._current_frames() incompatible with the JIT"))
-    w_result = space.newdict()
-    ecs = space.threadlocals.getallvalues()
-    for thread_ident, ec in ecs.items():
-        f = ec.gettopframe_nohidden()
-        f.mark_as_escaped()
-        space.setitem(w_result,
-                      space.wrap(thread_ident),
-                      space.wrap(f))
-    return w_result
-
 def setrecursionlimit(space, w_new_limit):
     """setrecursionlimit() sets the maximum number of nested calls that
 can occur before a RuntimeError is raised.  On PyPy the limit is
@@ -129,14 +110,19 @@
 function call.  See the debugger chapter in the library manual."""
     space.getexecutioncontext().settrace(w_func)
 
+def gettrace(space):
+    """Return the global debug tracing function set with sys.settrace.
+See the debugger chapter in the library manual."""
+    return space.getexecutioncontext().gettrace()
+
 def setprofile(space, w_func):
     """Set the profiling function.  It will be called on each function call
 and return.  See the profiler chapter in the library manual."""
     space.getexecutioncontext().setprofile(w_func)
 
 def getprofile(space):
-    """Set the profiling function.  It will be called on each function call
-and return.  See the profiler chapter in the library manual."""
+    """Return the profiling function set with sys.setprofile.
+See the profiler chapter in the library manual."""
     w_func = space.getexecutioncontext().getprofile()
     if w_func is not None:
         return w_func
diff --git a/pypy/module/thread/gil.py b/pypy/module/thread/gil.py
--- a/pypy/module/thread/gil.py
+++ b/pypy/module/thread/gil.py
@@ -16,7 +16,8 @@
 
 class GILThreadLocals(OSThreadLocals):
     """A version of OSThreadLocals that enforces a GIL."""
-    ll_GIL = thread.null_ll_lock
+    gil_ready = False
+    _immutable_fields_ = ['gil_ready?']
 
     def initialize(self, space):
         # add the GIL-releasing callback as an action on the space
@@ -25,12 +26,10 @@
 
     def setup_threads(self, space):
         """Enable threads in the object space, if they haven't already been."""
-        if not self.ll_GIL:
-            try:
-                self.ll_GIL = thread.allocate_ll_lock()
-            except thread.error:
+        if not self.gil_ready:
+            if not thread.gil_allocate():
                 raise wrap_thread_error(space, "can't allocate GIL")
-            thread.acquire_NOAUTO(self.ll_GIL, True)
+            self.gil_ready = True
             self.enter_thread(space)   # setup the main thread
             result = True
         else:
@@ -44,19 +43,16 @@
         # test_lock_again after the global state was cleared by
         # test_compile_lock.  As a workaround, we repatch these global
         # fields systematically.
-        spacestate.ll_GIL = self.ll_GIL
         invoke_around_extcall(before_external_call, after_external_call)
         return result
 
     def reinit_threads(self, space):
-        if self.ll_GIL:
-            self.ll_GIL = thread.allocate_ll_lock()
-            thread.acquire_NOAUTO(self.ll_GIL, True)
-            self.enter_thread(space)
+        if self.gil_ready:
+            self.gil_ready = False
+            self.setup_threads(space)
 
     def yield_thread(self):
-        thread.yield_thread()  # explicitly release the gil (used by test_gil)
-
+        do_yield_thread()
 
 class GILReleaseAction(PeriodicAsyncAction):
     """An action called every sys.checkinterval bytecodes.  It releases
@@ -64,16 +60,12 @@
     """
 
     def perform(self, executioncontext, frame):
-        # Other threads can run between the release() and the acquire()
-        # implicit in the following external function call (which has
-        # otherwise no effect).
-        thread.yield_thread()
+        do_yield_thread()
 
 
 class SpaceState:
 
     def _freeze_(self):
-        self.ll_GIL = thread.null_ll_lock
         self.action_after_thread_switch = None
         # ^^^ set by AsyncAction.fire_after_thread_switch()
         return False
@@ -95,14 +87,14 @@
     # this function must not raise, in such a way that the exception
     # transformer knows that it cannot raise!
     e = get_errno()
-    thread.release_NOAUTO(spacestate.ll_GIL)
+    thread.gil_release()
     set_errno(e)
 before_external_call._gctransformer_hint_cannot_collect_ = True
 before_external_call._dont_reach_me_in_del_ = True
 
 def after_external_call():
     e = get_errno()
-    thread.acquire_NOAUTO(spacestate.ll_GIL, True)
+    thread.gil_acquire()
     thread.gc_thread_run()
     spacestate.after_thread_switch()
     set_errno(e)
@@ -115,3 +107,18 @@
 # pointers in the shadow stack.  This is necessary because the GIL is
 # not held after the call to before_external_call() or before the call
 # to after_external_call().
+
+def do_yield_thread():
+    # explicitly release the gil, in a way that tries to give more
+    # priority to other threads (as opposed to continuing to run in
+    # the same thread).
+    if thread.gil_yield_thread():
+        thread.gc_thread_run()
+        spacestate.after_thread_switch()
+do_yield_thread._gctransformer_hint_close_stack_ = True
+do_yield_thread._dont_reach_me_in_del_ = True
+do_yield_thread._dont_inline_ = True
+
+# do_yield_thread() needs a different hint: _gctransformer_hint_close_stack_.
+# The *_external_call() functions are themselves called only from the rffi
+# module from a helper function that also has this hint.
diff --git a/pypy/module/thread/ll_thread.py b/pypy/module/thread/ll_thread.py
--- a/pypy/module/thread/ll_thread.py
+++ b/pypy/module/thread/ll_thread.py
@@ -17,7 +17,8 @@
     include_dirs = [str(py.path.local(autopath.pypydir).join('translator', 'c'))],
     export_symbols = ['RPyThreadGetIdent', 'RPyThreadLockInit',
                       'RPyThreadAcquireLock', 'RPyThreadReleaseLock',
-                      'RPyThreadYield',
+                      'RPyGilAllocate', 'RPyGilYieldThread',
+                      'RPyGilRelease', 'RPyGilAcquire',
                       'RPyThreadGetStackSize', 'RPyThreadSetStackSize',
                       'RPyOpaqueDealloc_ThreadLock',
                       'RPyThreadAfterFork']
@@ -69,8 +70,16 @@
                                          [TLOCKP], lltype.Void,
                                          _nowrapper=True)
 
-# this function does nothing apart from releasing the GIL temporarily.
-yield_thread = llexternal('RPyThreadYield', [], lltype.Void, threadsafe=True)
+# these functions manipulate directly the GIL, whose definition does not
+# escape the C code itself
+gil_allocate     = llexternal('RPyGilAllocate', [], lltype.Signed,
+                              _nowrapper=True)
+gil_yield_thread = llexternal('RPyGilYieldThread', [], lltype.Signed,
+                              _nowrapper=True)
+gil_release      = llexternal('RPyGilRelease', [], lltype.Void,
+                              _nowrapper=True)
+gil_acquire      = llexternal('RPyGilAcquire', [], lltype.Void,
+                              _nowrapper=True)
 
 def allocate_lock():
     return Lock(allocate_ll_lock())
diff --git a/pypy/module/thread/test/test_gil.py b/pypy/module/thread/test/test_gil.py
--- a/pypy/module/thread/test/test_gil.py
+++ b/pypy/module/thread/test/test_gil.py
@@ -30,19 +30,34 @@
     use_threads = True
     bigtest = False
 
-    def test_one_thread(self):
+    def test_one_thread(self, skew=+1):
+        from pypy.rlib.debug import debug_print
         if self.bigtest:
-            N = 1000000
+            N = 100000
+            skew *= 25000
         else:
             N = 100
+            skew *= 25
         space = FakeSpace()
         class State:
             pass
         state = State()
-        def runme():
-            for i in range(N):
+        def runme(main=False):
+            j = 0
+            for i in range(N + [-skew, skew][main]):
+                state.datalen1 += 1   # try to crash if the GIL is not
+                state.datalen2 += 1   # correctly acquired
                 state.data.append((thread.get_ident(), i))
+                state.datalen3 += 1
+                state.datalen4 += 1
+                assert state.datalen1 == len(state.data)
+                assert state.datalen2 == len(state.data)
+                assert state.datalen3 == len(state.data)
+                assert state.datalen4 == len(state.data)
+                debug_print(main, i, state.datalen4)
                 state.threadlocals.yield_thread()
+                assert i == j
+                j += 1
         def bootstrap():
             try:
                 runme()
@@ -50,20 +65,26 @@
                 thread.gc_thread_die()
         def f():
             state.data = []
+            state.datalen1 = 0
+            state.datalen2 = 0
+            state.datalen3 = 0
+            state.datalen4 = 0
             state.threadlocals = gil.GILThreadLocals()
             state.threadlocals.setup_threads(space)
             thread.gc_thread_prepare()
             subident = thread.start_new_thread(bootstrap, ())
             mainident = thread.get_ident()
-            runme()
+            runme(True)
             still_waiting = 3000
             while len(state.data) < 2*N:
+                debug_print(len(state.data))
                 if not still_waiting:
                     raise ValueError("time out")
                 still_waiting -= 1
                 if not we_are_translated(): gil.before_external_call()
                 time.sleep(0.01)
                 if not we_are_translated(): gil.after_external_call()
+            debug_print("leaving!")
             i1 = i2 = 0
             for tid, i in state.data:
                 if tid == mainident:
@@ -72,14 +93,17 @@
                     assert i == i2; i2 += 1
                 else:
                     assert 0
-            assert i1 == N
-            assert i2 == N
+            assert i1 == N + skew
+            assert i2 == N - skew
             return len(state.data)
 
         fn = self.getcompiled(f, [])
         res = fn()
         assert res == 2*N
 
+    def test_one_thread_rev(self):
+        self.test_one_thread(skew=-1)
+
 
 class TestRunDirectly(GILTests):
     def getcompiled(self, f, argtypes):
diff --git a/pypy/module/thread/test/test_thread.py b/pypy/module/thread/test/test_thread.py
--- a/pypy/module/thread/test/test_thread.py
+++ b/pypy/module/thread/test/test_thread.py
@@ -225,7 +225,8 @@
 
         def busy_wait():
             for x in range(1000):
-                time.sleep(0.01)
+                print 'tick...', x  # <-force the GIL to be released, as
+                time.sleep(0.01)    #   time.sleep doesn't do non-translated
 
         # This is normally called by app_main.py
         signal.signal(signal.SIGINT, signal.default_int_handler)
diff --git a/pypy/module/thread/threadlocals.py b/pypy/module/thread/threadlocals.py
--- a/pypy/module/thread/threadlocals.py
+++ b/pypy/module/thread/threadlocals.py
@@ -8,9 +8,14 @@
 
     def __init__(self):
         self._valuedict = {}   # {thread_ident: ExecutionContext()}
+        self._freeze_()
+
+    def _freeze_(self):
+        self._valuedict.clear()
         self._mainthreadident = 0
         self._mostrecentkey = 0        # fast minicaching for the common case
         self._mostrecentvalue = None   # fast minicaching for the common case
+        return False
 
     def getvalue(self):
         ident = thread.get_ident()
diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -258,15 +258,15 @@
             msg = "'%s' has no length" % (name,)
             raise OperationError(space.w_TypeError, space.wrap(msg))
         w_res = space.get_and_call_function(w_descr, w_obj)
-        space._check_len_result(w_res)
-        return w_res
+        return space.wrap(space._check_len_result(w_res))
 
     def _check_len_result(space, w_obj):
         # Will complain if result is too big.
-        result = space.int_w(w_obj)
+        result = space.int_w(space.int(w_obj))
         if result < 0:
             raise OperationError(space.w_ValueError,
                                  space.wrap("__len__() should return >= 0"))
+        return result
 
     def iter(space, w_obj):
         w_descr = space.lookup(w_obj, '__iter__')
diff --git a/pypy/objspace/std/celldict.py b/pypy/objspace/std/celldict.py
--- a/pypy/objspace/std/celldict.py
+++ b/pypy/objspace/std/celldict.py
@@ -37,10 +37,10 @@
         self.version = VersionTag()
 
     def get_empty_storage(self):
-       return self.erase({})
+        return self.erase({})
 
     def mutated(self):
-       self.version = VersionTag()
+        self.version = VersionTag()
 
     def getdictvalue_no_unwrapping(self, w_dict, key):
         # NB: it's important to promote self here, so that self.version is a
diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py
--- a/pypy/objspace/std/intobject.py
+++ b/pypy/objspace/std/intobject.py
@@ -1,12 +1,13 @@
 from pypy.interpreter.error import OperationError
 from pypy.objspace.std import newformat
+from pypy.objspace.std.inttype import wrapint
 from pypy.objspace.std.model import registerimplementation, W_Object
-from pypy.objspace.std.register_all import register_all
 from pypy.objspace.std.multimethod import FailedToImplementArgs
 from pypy.objspace.std.noneobject import W_NoneObject
+from pypy.objspace.std.register_all import register_all
+from pypy.rlib import jit
 from pypy.rlib.rarithmetic import ovfcheck, ovfcheck_lshift, LONG_BIT, r_uint
 from pypy.rlib.rbigint import rbigint
-from pypy.objspace.std.inttype import wrapint
 
 """
 In order to have the same behavior running
@@ -20,7 +21,7 @@
     _immutable_fields_ = ['intval']
 
     from pypy.objspace.std.inttype import int_typedef as typedef
-    
+
     def __init__(w_self, intval):
         w_self.intval = intval
 
@@ -135,7 +136,7 @@
     x = float(w_int1.intval)
     y = float(w_int2.intval)
     if y == 0.0:
-        raise FailedToImplementArgs(space.w_ZeroDivisionError, space.wrap("float division"))    
+        raise FailedToImplementArgs(space.w_ZeroDivisionError, space.wrap("float division"))
     return space.wrap(x / y)
 
 def mod__Int_Int(space, w_int1, w_int2):
@@ -169,7 +170,8 @@
 
 
 # helper for pow()
-def _impl_int_int_pow(space, iv, iw, iz=0):
+ at jit.look_inside_iff(lambda space, iv, iw, iz: jit.isconstant(iw) and jit.isconstant(iz))
+def _impl_int_int_pow(space, iv, iw, iz):
     if iw < 0:
         if iz != 0:
             raise OperationError(space.w_TypeError,
@@ -197,7 +199,7 @@
     except OverflowError:
         raise FailedToImplementArgs(space.w_OverflowError,
                                 space.wrap("integer exponentiation"))
-    return wrapint(space, ix)
+    return ix
 
 def pow__Int_Int_Int(space, w_int1, w_int2, w_int3):
     x = w_int1.intval
@@ -206,12 +208,12 @@
     if z == 0:
         raise OperationError(space.w_ValueError,
                              space.wrap("pow() 3rd argument cannot be 0"))
-    return _impl_int_int_pow(space, x, y, z)
+    return space.wrap(_impl_int_int_pow(space, x, y, z))
 
 def pow__Int_Int_None(space, w_int1, w_int2, w_int3):
     x = w_int1.intval
     y = w_int2.intval
-    return _impl_int_int_pow(space, x, y)
+    return space.wrap(_impl_int_int_pow(space, x, y, 0))
 
 def neg__Int(space, w_int1):
     a = w_int1.intval
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -386,7 +386,11 @@
     if len(items)== 0:
         raise OperationError(space.w_IndexError,
                              space.wrap("pop from empty list"))
-    idx = space.int_w(w_idx)
+    if space.isinstance_w(w_idx, space.w_float):
+        raise OperationError(space.w_TypeError,
+            space.wrap("integer argument expected, got float")
+        )
+    idx = space.int_w(space.int(w_idx))
     try:
         return items.pop(idx)
     except IndexError:
diff --git a/pypy/objspace/std/objecttype.py b/pypy/objspace/std/objecttype.py
--- a/pypy/objspace/std/objecttype.py
+++ b/pypy/objspace/std/objecttype.py
@@ -44,7 +44,6 @@
         raise OperationError(space.w_TypeError,
                              space.wrap("__class__ assignment: only for heap types"))
     w_oldcls = space.type(w_obj)
-    # XXX taint space should raise a TaintError here if w_oldcls is tainted
     assert isinstance(w_oldcls, W_TypeObject)
     if w_oldcls.get_full_instance_layout() == w_newcls.get_full_instance_layout():
         w_obj.setclass(space, w_newcls)
diff --git a/pypy/objspace/std/rangeobject.py b/pypy/objspace/std/rangeobject.py
--- a/pypy/objspace/std/rangeobject.py
+++ b/pypy/objspace/std/rangeobject.py
@@ -23,7 +23,7 @@
 
 class W_RangeListObject(W_Object):
     typedef = listtype.list_typedef
-    
+
     def __init__(w_self, start, step, length):
         assert step != 0
         w_self.start = start
@@ -40,7 +40,7 @@
         if not length:
             w_self.w_list = space.newlist([])
             return w_self.w_list
-        
+
         arr = [None] * length  # this is to avoid using append.
 
         i = start
@@ -146,7 +146,11 @@
     if length == 0:
         raise OperationError(space.w_IndexError,
                              space.wrap("pop from empty list"))
-    idx = space.int_w(w_idx)
+    if space.isinstance_w(w_idx, space.w_float):
+        raise OperationError(space.w_TypeError,
+            space.wrap("integer argument expected, got float")
+        )
+    idx = space.int_w(space.int(w_idx))
     if idx == 0:
         result = w_rangelist.start
         w_rangelist.start += w_rangelist.step
diff --git a/pypy/objspace/std/ropeunicodeobject.py b/pypy/objspace/std/ropeunicodeobject.py
--- a/pypy/objspace/std/ropeunicodeobject.py
+++ b/pypy/objspace/std/ropeunicodeobject.py
@@ -185,7 +185,7 @@
 
 def eq__RopeUnicode_Rope(space, w_runi, w_rope):
     from pypy.objspace.std.unicodeobject import _unicode_string_comparison
-    return _unicode_string_comparison(space, w_runi, w_rope, 
+    return _unicode_string_comparison(space, w_runi, w_rope,
                     False,  unicode_from_string)
 
 def ne__RopeUnicode_RopeUnicode(space, w_str1, w_str2):
@@ -193,7 +193,7 @@
 
 def ne__RopeUnicode_Rope(space, w_runi, w_rope):
     from pypy.objspace.std.unicodeobject import _unicode_string_comparison
-    return _unicode_string_comparison(space, w_runi, w_rope, 
+    return _unicode_string_comparison(space, w_runi, w_rope,
                     True, unicode_from_string)
 
 def gt__RopeUnicode_RopeUnicode(space, w_str1, w_str2):
@@ -247,7 +247,7 @@
     if (len(l_w) == 1 and
         space.is_w(space.type(l_w[0]), space.w_unicode)):
         return l_w[0]
-    
+
     values_list = []
     for i in range(len(l_w)):
         w_item = l_w[i]
@@ -320,7 +320,7 @@
 
 
 def make_generic(funcname):
-    def func(space, w_self): 
+    def func(space, w_self):
         node = w_self._node
         if node.length() == 0:
             return space.w_False
@@ -578,7 +578,7 @@
         return w_self.create_if_subclassed()
     resultnode = rope.concatenate(rope.multiply(fillchar, padding), self)
     return W_RopeUnicodeObject(resultnode)
-    
+
 def unicode_zfill__RopeUnicode_ANY(space, w_self, w_width):
     self = w_self._node
     length = self.length()
@@ -744,7 +744,7 @@
     except OverflowError:
         raise OperationError(space.w_OverflowError,
                              space.wrap("string too long"))
-    
+
 
 def unicode_encode__RopeUnicode_ANY_ANY(space, w_unistr,
                                         w_encoding=None,
@@ -821,7 +821,7 @@
         try:
             w_newval = space.getitem(w_table, space.wrap(char))
         except OperationError, e:
-            if e.match(space, space.w_KeyError):
+            if e.match(space, space.w_LookupError):
                 result.append(crope)
             else:
                 raise
@@ -848,7 +848,7 @@
     hexdigits = "0123456789abcdef"
     node = w_unicode._node
     size = node.length()
-    
+
     singlequote = doublequote = False
     iter = rope.ItemIterator(node)
     for i in range(size):
@@ -900,7 +900,7 @@
                                   ])
                     j += 2
                     continue
-                
+
         if code >= 0x100:
             result.extend(['\\', "u",
                            hexdigits[(code >> 12) & 0xf],
@@ -932,7 +932,7 @@
             continue
         if code < ord(' ') or code >= 0x7f:
             result.extend(['\\', "x",
-                           hexdigits[(code >> 4) & 0xf], 
+                           hexdigits[(code >> 4) & 0xf],
                            hexdigits[(code >> 0) & 0xf],
                           ])
             j += 1
@@ -964,15 +964,15 @@
 
 def next__RopeUnicodeIter(space, w_ropeiter):
     if w_ropeiter.node is None:
-        raise OperationError(space.w_StopIteration, space.w_None) 
+        raise OperationError(space.w_StopIteration, space.w_None)
     try:
         unichar = w_ropeiter.item_iter.nextunichar()
         w_item = space.wrap(unichar)
     except StopIteration:
         w_ropeiter.node = None
         w_ropeiter.char_iter = None
-        raise OperationError(space.w_StopIteration, space.w_None) 
-    w_ropeiter.index += 1 
+        raise OperationError(space.w_StopIteration, space.w_None)
+    w_ropeiter.index += 1
     return w_item
 
 # XXX __length_hint__()
diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py
--- a/pypy/objspace/std/stringobject.py
+++ b/pypy/objspace/std/stringobject.py
@@ -161,57 +161,59 @@
 
 def str_swapcase__String(space, w_self):
     self = w_self._value
-    res = [' '] * len(self)
+    builder = StringBuilder(len(self))
     for i in range(len(self)):
         ch = self[i]
         if ch.isupper():
             o = ord(ch) + 32
-            res[i] = chr(o)
+            builder.append(chr(o))
         elif ch.islower():
             o = ord(ch) - 32
-            res[i] = chr(o)
+            builder.append(chr(o))
         else:
-            res[i] = ch
+            builder.append(ch)
 
-    return space.wrap("".join(res))
+    return space.wrap(builder.build())
 
 
 def str_capitalize__String(space, w_self):
     input = w_self._value
-    buffer = [' '] * len(input)
+    builder = StringBuilder(len(input))
     if len(input) > 0:
         ch = input[0]
         if ch.islower():
             o = ord(ch) - 32
-            buffer[0] = chr(o)
+            builder.append(chr(o))
         else:
-            buffer[0] = ch
+            builder.append(ch)
 
         for i in range(1, len(input)):
             ch = input[i]
             if ch.isupper():
                 o = ord(ch) + 32
-                buffer[i] = chr(o)
+                builder.append(chr(o))
             else:
-                buffer[i] = ch
+                builder.append(ch)
 
-    return space.wrap("".join(buffer))
+    return space.wrap(builder.build())
 
 def str_title__String(space, w_self):
     input = w_self._value
-    buffer = [' '] * len(input)
+    builder = StringBuilder(len(input))
     prev_letter=' '
 
-    for pos in range(0, len(input)):
+    for pos in range(len(input)):
         ch = input[pos]
         if not prev_letter.isalpha():
-            buffer[pos] = _upper(ch)
+            ch = _upper(ch)
+            builder.append(ch)
         else:
-            buffer[pos] = _lower(ch)
+            ch = _lower(ch)
+            builder.append(ch)
 
-        prev_letter = buffer[pos]
+        prev_letter = ch
 
-    return space.wrap("".join(buffer))
+    return space.wrap(builder.build())
 
 def str_split__String_None_ANY(space, w_self, w_none, w_maxsplit=-1):
     maxsplit = space.int_w(w_maxsplit)
@@ -754,27 +756,21 @@
     input = w_self._value
     width = space.int_w(w_width)
 
-    if len(input) >= width:
+    num_zeros = width - len(input)
+    if num_zeros <= 0:
         # cannot return w_self, in case it is a subclass of str
         return space.wrap(input)
 
-    buf = [' '] * width
+    builder = StringBuilder(width)
     if len(input) > 0 and (input[0] == '+' or input[0] == '-'):
-        buf[0] = input[0]
+        builder.append(input[0])
         start = 1
-        middle = width - len(input) + 1
     else:
         start = 0
-        middle = width - len(input)
 
-    for i in range(start, middle):
-        buf[i] = '0'
-
-    for i in range(middle, width):
-        buf[i] = input[start]
-        start = start + 1
-
-    return space.wrap("".join(buf))
+    builder.append_multiple_char('0', num_zeros)
+    builder.append_slice(input, start, len(input))
+    return space.wrap(builder.build())
 
 
 def hash__String(space, w_str):
diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py
--- a/pypy/objspace/std/test/test_listobject.py
+++ b/pypy/objspace/std/test/test_listobject.py
@@ -705,6 +705,20 @@
         l.pop()
         assert l == range(9)
 
+    def test_pop_custom_int(self):
+        class A(object):
+            def __init__(self, x):
+                self.x = x
+
+            def __int__(self):
+                return self.x
+
+        l = range(10)
+        x = l.pop(A(-1))
+        assert x == 9
+        assert l == range(9)
+        raises(TypeError, range(10).pop, 1.0)
+
     def test_remove(self):
         c = list('hello world')
         c.remove('l')
diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -126,6 +126,25 @@
         raises(TypeError, type, 'test', 42, {})
         raises(TypeError, type, 'test', (object,), 42)
 
+    def test_call_type_subclass(self):
+        class A(type):
+            pass
+
+        assert A("hello") is str
+
+        # Make sure type(x) doesn't call x.__class__.__init__
+        class T(type):
+            counter = 0
+            def __init__(self, *args):
+                T.counter += 1
+        class C:
+            __metaclass__ = T
+        assert T.counter == 1
+        a = C()
+        assert T.counter == 1
+        assert type(a) is C
+        assert T.counter == 1
+
     def test_bases(self):
         assert int.__bases__ == (object,)
         class X:
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -10,7 +10,8 @@
 from pypy.objspace.std import identitydict
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.rlib.objectmodel import current_object_addr_as_int, compute_hash
-from pypy.rlib.jit import promote, elidable_promote, we_are_jitted
+from pypy.rlib.jit import promote, elidable_promote, we_are_jitted,\
+     promote_string
 from pypy.rlib.jit import elidable, dont_look_inside, unroll_safe
 from pypy.rlib.rarithmetic import intmask, r_uint
 
@@ -77,7 +78,7 @@
         for i in range(len(self.lookup_where)):
             self.lookup_where[i] = None_None
 
-# possible values of compares_by_identity_status 
+# possible values of compares_by_identity_status
 UNKNOWN = 0
 COMPARES_BY_IDENTITY = 1
 OVERRIDES_EQ_CMP_OR_HASH = 2
@@ -101,6 +102,7 @@
                           'instancetypedef',
                           'terminator',
                           '_version_tag?',
+                          'interplevel_cls',
                           ]
 
     # for config.objspace.std.getattributeshortcut
@@ -358,7 +360,7 @@
                 if w_value is not None:
                     return w_value
         return None
-                
+
     @unroll_safe
     def _lookup(w_self, key):
         space = w_self.space
@@ -399,6 +401,7 @@
         if version_tag is None:
             tup = w_self._lookup_where(name)
             return tup
+        name = promote_string(name)
         w_class, w_value = w_self._pure_lookup_where_with_method_cache(name, version_tag)
         return w_class, unwrap_cell(space, w_value)
 
@@ -822,14 +825,6 @@
 
 def call__Type(space, w_type, __args__):
     promote(w_type)
-    # special case for type(x)
-    if space.is_w(w_type, space.w_type):
-        try:
-            w_obj, = __args__.fixedunpack(1)
-        except ValueError:
-            pass
-        else:
-            return space.type(w_obj)
     # invoke the __new__ of the type
     if not we_are_jitted():
         # note that the annotator will figure out that w_type.w_bltin_new can
@@ -856,7 +851,8 @@
         call_init = space.isinstance_w(w_newobject, w_type)
 
     # maybe invoke the __init__ of the type
-    if call_init:
+    if (call_init and not (space.is_w(w_type, space.w_type) and
+        not __args__.keywords and len(__args__.arguments_w) == 1)):
         w_descr = space.lookup(w_newobject, '__init__')
         w_result = space.get_and_call_args(w_descr, w_newobject, __args__)
         if not space.is_w(w_result, space.w_None):
diff --git a/pypy/objspace/std/typetype.py b/pypy/objspace/std/typetype.py
--- a/pypy/objspace/std/typetype.py
+++ b/pypy/objspace/std/typetype.py
@@ -1,17 +1,28 @@
-from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.interpreter import gateway
 from pypy.interpreter.argument import Arguments
+from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.interpreter.typedef import (GetSetProperty, descr_get_dict,
                                       weakref_descr)
 from pypy.objspace.std.stdtypedef import StdTypeDef
 
-def descr__new__(space, w_typetype, w_name, w_bases, w_dict):
+
+def descr__new__(space, w_typetype, w_name, w_bases=gateway.NoneNotWrapped,
+    w_dict=gateway.NoneNotWrapped):
+
     "This is used to create user-defined classes only."
     from pypy.objspace.std.typeobject import W_TypeObject
     # XXX check types
 
     w_typetype = _precheck_for_new(space, w_typetype)
 
+    # special case for type(x)
+    if (space.is_w(space.type(w_typetype), space.w_type) and w_bases is None and
+        w_dict is None):
+        return space.type(w_name)
+    elif w_bases is None or w_dict is None:
+        raise OperationError(space.w_TypeError, space.wrap("type() takes 1 or 3 arguments"))
+
+
     bases_w = space.fixedview(w_bases)
 
     w_winner = w_typetype
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -417,54 +417,54 @@
     input = w_self._value
     if len(input) == 0:
         return W_UnicodeObject.EMPTY
-    result = [u'\0'] * len(input)
-    result[0] = unichr(unicodedb.toupper(ord(input[0])))
+    builder = UnicodeBuilder(len(input))
+    builder.append(unichr(unicodedb.toupper(ord(input[0]))))
     for i in range(1, len(input)):
-        result[i] = unichr(unicodedb.tolower(ord(input[i])))
-    return W_UnicodeObject(u''.join(result))
+        builder.append(unichr(unicodedb.tolower(ord(input[i]))))
+    return W_UnicodeObject(builder.build())
 
 def unicode_title__Unicode(space, w_self):
     input = w_self._value
     if len(input) == 0:
         return w_self
-    result = [u'\0'] * len(input)
+    builder = UnicodeBuilder(len(input))
 
     previous_is_cased = False
     for i in range(len(input)):
         unichar = ord(input[i])
         if previous_is_cased:
-            result[i] = unichr(unicodedb.tolower(unichar))
+            builder.append(unichr(unicodedb.tolower(unichar)))
         else:
-            result[i] = unichr(unicodedb.totitle(unichar))
+            builder.append(unichr(unicodedb.totitle(unichar)))
         previous_is_cased = unicodedb.iscased(unichar)
-    return W_UnicodeObject(u''.join(result))
+    return W_UnicodeObject(builder.build())
 
 def unicode_lower__Unicode(space, w_self):
     input = w_self._value
-    result = [u'\0'] * len(input)
+    builder = UnicodeBuilder(len(input))
     for i in range(len(input)):
-        result[i] = unichr(unicodedb.tolower(ord(input[i])))
-    return W_UnicodeObject(u''.join(result))
+        builder.append(unichr(unicodedb.tolower(ord(input[i]))))
+    return W_UnicodeObject(builder.build())
 
 def unicode_upper__Unicode(space, w_self):
     input = w_self._value
-    result = [u'\0'] * len(input)
+    builder = UnicodeBuilder(len(input))
     for i in range(len(input)):
-        result[i] = unichr(unicodedb.toupper(ord(input[i])))
-    return W_UnicodeObject(u''.join(result))
+        builder.append(unichr(unicodedb.toupper(ord(input[i]))))
+    return W_UnicodeObject(builder.build())
 
 def unicode_swapcase__Unicode(space, w_self):
     input = w_self._value
-    result = [u'\0'] * len(input)
+    builder = UnicodeBuilder(len(input))
     for i in range(len(input)):
         unichar = ord(input[i])
         if unicodedb.islower(unichar):
-            result[i] = unichr(unicodedb.toupper(unichar))
+            builder.append(unichr(unicodedb.toupper(unichar)))
         elif unicodedb.isupper(unichar):
-            result[i] = unichr(unicodedb.tolower(unichar))
+            builder.append(unichr(unicodedb.tolower(unichar)))
         else:
-            result[i] = input[i]

From noreply at buildbot.pypy.org  Sun Oct 16 19:39:32 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sun, 16 Oct 2011 19:39:32 +0200 (CEST)
Subject: [pypy-commit] pypy inline-dict-ops: move stuff around a bit.
Message-ID: <20111016173932.447D9820D7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: inline-dict-ops
Changeset: r48092:8c21856a5724
Date: 2011-10-16 13:39 -0400
http://bitbucket.org/pypy/pypy/changeset/8c21856a5724/

Log:	move stuff around a bit.

diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py
--- a/pypy/rpython/lltypesystem/rdict.py
+++ b/pypy/rpython/lltypesystem/rdict.py
@@ -1,15 +1,14 @@
 from pypy.tool.pairtype import pairtype
 from pypy.objspace.flow.model import Constant
-from pypy.rpython.rdict import AbstractDictRepr, AbstractDictIteratorRepr,\
-     rtype_newdict
+from pypy.rpython.rdict import (AbstractDictRepr, AbstractDictIteratorRepr,
+     rtype_newdict)
 from pypy.rpython.lltypesystem import lltype
+from pypy.rlib import objectmodel, jit
 from pypy.rlib.rarithmetic import r_uint, intmask, LONG_BIT
-from pypy.rlib.objectmodel import hlinvoke
-from pypy.rlib import objectmodel
-from pypy.rlib import jit
 from pypy.rpython import rmodel
 from pypy.rpython.error import TyperError
 
+
 HIGHEST_BIT = intmask(1 << (LONG_BIT - 1))
 MASK = intmask(HIGHEST_BIT - 1)
 
@@ -421,11 +420,11 @@
 
 def ll_keyhash_custom(d, key):
     DICT = lltype.typeOf(d).TO
-    return hlinvoke(DICT.r_rdict_hashfn, d.fnkeyhash, key)
+    return objectmodel.hlinvoke(DICT.r_rdict_hashfn, d.fnkeyhash, key)
 
 def ll_keyeq_custom(d, key1, key2):
     DICT = lltype.typeOf(d).TO
-    return hlinvoke(DICT.r_rdict_eqfn, d.fnkeyeq, key1, key2)
+    return objectmodel.hlinvoke(DICT.r_rdict_eqfn, d.fnkeyeq, key1, key2)
 
 def ll_dict_len(d):
     return d.num_items
@@ -446,6 +445,9 @@
     i = ll_dict_lookup(d, key, hash)
     return _ll_dict_setitem_lookup_done(d, key, value, hash, i)
 
+# Leaving as dont_look_inside ATM, it has a few branches which could lead to
+# many bridges if we don't consider their possible frequency.
+ at jit.dont_look_inside
 def _ll_dict_setitem_lookup_done(d, key, value, hash, i):
     valid = (i & HIGHEST_BIT) == 0
     i = i & MASK
@@ -489,6 +491,9 @@
         raise KeyError
     _ll_dict_del(d, i)
 
+# XXX: Move the size checking and resize into a single call which is opauqe to
+# the JIT to avoid extra branches.
+ at jit.dont_look_inside
 def _ll_dict_del(d, i):
     d.entries.mark_deleted(i)
     d.num_items -= 1
@@ -556,18 +561,11 @@
         freeslot = i
     else:
         return i | HIGHEST_BIT # pristine entry -- lookup failed
-    return _ll_dict_lookup_slowpath(d, key, hash, freeslot)
 
-def _ll_dict_lookup_slowpath(d, key, hash, freeslot):
     # In the loop, a deleted entry (everused and not valid) is by far
     # (factor of 100s) the least likely outcome, so test for that last.
-    perturb = r_uint(hash) 
-    entries = d.entries
-    ENTRIES = lltype.typeOf(entries).TO
-    direct_compare = not hasattr(ENTRIES, 'no_direct_compare')
-    mask = len(entries) - 1
-    i = hash & mask
-    while 1: 
+    perturb = r_uint(hash)
+    while 1:
         # compute the next index using unsigned arithmetic
         i = r_uint(i)
         i = (i << 2) + i + perturb + 1

From noreply at buildbot.pypy.org  Sun Oct 16 23:08:52 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 16 Oct 2011 23:08:52 +0200 (CEST)
Subject: [pypy-commit] pypy default: Move this logic here,
 so that it is also used for Linux64.
Message-ID: <20111016210852.2C634820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48093:ad955890a446
Date: 2011-10-16 17:47 +0200
http://bitbucket.org/pypy/pypy/changeset/ad955890a446/

Log:	Move this logic here, so that it is also used for Linux64.

diff --git a/pypy/translator/platform/linux.py b/pypy/translator/platform/linux.py
--- a/pypy/translator/platform/linux.py
+++ b/pypy/translator/platform/linux.py
@@ -24,15 +24,14 @@
         return self._pkg_config("libffi", "--libs-only-L",
                                 ['/usr/lib/libffi'])
 
+    def library_dirs_for_libffi_a(self):
+        # places where we need to look for libffi.a
+        return self.library_dirs_for_libffi() + ['/usr/lib']
+
 
 class Linux(BaseLinux):
     shared_only = ()    # it seems that on 32-bit linux, compiling with -fPIC
                         # gives assembler that asmgcc is not happy about.
 
-    def library_dirs_for_libffi_a(self):
-        # places where we need to look for libffi.a
-        return self.library_dirs_for_libffi() + ['/usr/lib']
-
-
 class Linux64(BaseLinux):
     pass

From noreply at buildbot.pypy.org  Sun Oct 16 23:08:53 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 16 Oct 2011 23:08:53 +0200 (CEST)
Subject: [pypy-commit] pypy default: Forgot that destroy() must be called
	explicitly.
Message-ID: <20111016210853.5F431820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48094:244cffb0fff0
Date: 2011-10-16 22:13 +0200
http://bitbucket.org/pypy/pypy/changeset/244cffb0fff0/

Log:	Forgot that destroy() must be called explicitly.

diff --git a/pypy/module/_continuation/interp_continuation.py b/pypy/module/_continuation/interp_continuation.py
--- a/pypy/module/_continuation/interp_continuation.py
+++ b/pypy/module/_continuation/interp_continuation.py
@@ -19,6 +19,11 @@
         #  - normal:      self.sthread != None, not is_empty_handle(self.h)
         #  - finished:    self.sthread != None, is_empty_handle(self.h)
 
+    def __del__(self):
+        sthread = self.sthread
+        if sthread is not None and not sthread.is_empty_handle(self.h):
+            sthread.destroy(self.h)
+
     def check_sthread(self):
         ec = self.space.getexecutioncontext()
         if ec.stacklet_thread is not self.sthread:

From noreply at buildbot.pypy.org  Sun Oct 16 23:08:54 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 16 Oct 2011 23:08:54 +0200 (CEST)
Subject: [pypy-commit] pypy default: Move the gettopframe_nohidden() out of
 the way: if we call
Message-ID: <20111016210854.8EA12820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48095:0a33c12bc697
Date: 2011-10-16 23:02 +0200
http://bitbucket.org/pypy/pypy/changeset/0a33c12bc697/

Log:	Move the gettopframe_nohidden() out of the way: if we call
	eval(a,glob,loc) with a 'glob' specified, then we don't need it.
	Helps the JIT in this case.

diff --git a/pypy/module/__builtin__/compiling.py b/pypy/module/__builtin__/compiling.py
--- a/pypy/module/__builtin__/compiling.py
+++ b/pypy/module/__builtin__/compiling.py
@@ -84,8 +84,8 @@
         raise OperationError(space.w_TypeError,
               w('eval() arg 1 must be a string or code object'))
 
-    caller = space.getexecutioncontext().gettopframe_nohidden()
     if space.is_w(w_globals, space.w_None):
+        caller = space.getexecutioncontext().gettopframe_nohidden()
         if caller is None:
             w_globals = space.newdict()
             if space.is_w(w_locals, space.w_None):
@@ -97,13 +97,6 @@
     elif space.is_w(w_locals, space.w_None):
         w_locals = w_globals
 
-    try:
-        space.getitem(w_globals, space.wrap('__builtins__'))
-    except OperationError, e:
-        if not e.match(space, space.w_KeyError):
-            raise
-        if caller is not None:
-            w_builtin = space.builtin.pick_builtin(caller.w_globals)
-            space.setitem(w_globals, space.wrap('__builtins__'), w_builtin)
+    space.builtin.pick_builtin(w_globals)
 
     return codeobj.exec_code(space, w_globals, w_locals)

From noreply at buildbot.pypy.org  Sun Oct 16 23:08:55 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 16 Oct 2011 23:08:55 +0200 (CEST)
Subject: [pypy-commit] pypy default: Checking in a test for the previous
 fix. It fails with tonight's
Message-ID: <20111016210855.BF496820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48096:07c72584eb47
Date: 2011-10-16 23:08 +0200
http://bitbucket.org/pypy/pypy/changeset/07c72584eb47/

Log:	Checking in a test for the previous fix. It fails with tonight's
	pypy-c; hopefully it will pass with next night's.

diff --git a/pypy/module/pypyjit/test_pypy_c/test_misc.py b/pypy/module/pypyjit/test_pypy_c/test_misc.py
--- a/pypy/module/pypyjit/test_pypy_c/test_misc.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_misc.py
@@ -329,4 +329,20 @@
             guard_false(i28, descr=...)
             i30 = int_lshift(i20, 24)
             i31 = int_or(i26, i30)
-        """ % {"32_bit_only": extra})
\ No newline at end of file
+        """ % {"32_bit_only": extra})
+
+    def test_eval(self):
+        def main():
+            i = 1
+            a = compile('x+x+x+x+x+x', 'eval', 'eval')
+            b = {'x': 7}
+            while i < 1000:
+                y = eval(a,b,b)  # ID: eval
+                i += 1
+            return y
+
+        log = self.run(main)
+        assert log.result == 42
+        # the following assertion fails if the loop was cancelled due
+        # to "abort: vable escape"
+        assert len(log.loops_by_id("eval")) == 1

From noreply at buildbot.pypy.org  Sun Oct 16 23:26:46 2011
From: noreply at buildbot.pypy.org (mattip)
Date: Sun, 16 Oct 2011 23:26:46 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-dtype: Add NDim array to numpy,
	still in first stages
Message-ID: <20111016212646.4242A820D7@wyvern.cs.uni-duesseldorf.de>

Author: mattip
Branch: numpy-dtype
Changeset: r48097:2fb493207222
Date: 2011-10-15 23:04 +0200
http://bitbucket.org/pypy/pypy/changeset/2fb493207222/

Log:	Add NDim array to numpy, still in first stages

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -225,7 +225,7 @@
         return space.wrap(self.find_dtype())
 
     def descr_get_shape(self, space):
-        return space.newtuple([self.descr_len(space)])
+        return space.newtuple([self.descr_shape(space)])
 
     def descr_len(self, space):
         return self.get_concrete().descr_len(space)
@@ -550,6 +550,9 @@
     def descr_len(self, space):
         return space.wrap(self.find_size())
 
+    def descr_shape(self,space):
+        return space.wrap(self.find_shape())
+
     def calc_index(self, item):
         raise NotImplementedError
 
@@ -579,6 +582,9 @@
     def find_size(self):
         return self.size
 
+    def find_shape(self):
+        return self.shape
+
     def setslice(self, start, stop, step, slice_length, arr):
         start = self.calc_index(start)
         if stop != -1:
@@ -627,6 +633,9 @@
         def descr_len(self, space):
             return space.wrap(self.size)
 
+        def descr_shape(self, space):
+            return space.wrap(self.shape)
+
         def get_root_storage(self):
             return self.storage
 
@@ -700,15 +709,160 @@
     TypedSingleDimArray.__name__ = 'SingleDimArray_' + _dtype.name
     return TypedSingleDimArray
 
+def make_ndclass(_dtype):
+    class TypedNDimArray(BaseArray):
+        signature = Signature()
+        dtype = _dtype
+        def __init__(self, shape):
+            BaseArray.__init__(self)
+            self.shape = shape
+            self.size = 1
+            for v in shape:
+                self.size *= v
+            self.storage = lltype.malloc(_dtype.TP, self.size, zero=True,
+                                     flavor='raw', track_allocation=False,
+                                     add_memory_pressure=True)
+            # XXX find out why test_zjit explodes with trackign of allocations
+
+        def get_concrete(self):
+            return self
+
+        def find_size(self):
+            return self.size
+
+        def descr_len(self, space):
+            return space.wrap(self.size)
+
+        def descr_shape(self, space):
+            return space.wrap(self.shape)
+
+        def get_root_storage(self):
+            return self.storage
+
+        def eval(self, i):
+            return self.storage[i]
+
+        if _dtype.kind == 'b':
+            def getitem(self, space, i):
+                return space.wrap(bool(self.storage[i]))
+        elif _dtype.kind == 'f':
+            def getitem(self, space, i):
+                return space.wrap(float(self.storage[i]))
+        elif _dtype.num < 8 or (LONG_BIT == 64 and _dtype.num == 9):
+            def getitem(self, space, i):
+                return space.wrap(rffi.cast(lltype.Signed, self.storage[i]))
+        elif LONG_BIT == 64 or _dtype.num == 8:
+            def getitem(self, space, i):
+                return space.wrap(self.storage[i])
+        else: # unsigned longlong and signed longlong for 32-bit
+            def getitem(self, space, i):
+                return newlong(space, fromlong(self.storage[i]))
+
+        def setitem(self, item, value):
+            assert isinstance(value, _dtype.valtype)
+            if issequence(item):
+                assert len(item) == len(self.shape)
+                pp = [0]+self.shape
+                indx = 0
+                for v,p in zip(item,pp):
+                    index += indx*p+v
+                self.storage[indx]=value
+            else:    
+                self.storage[item] = value
+
+        #def setitem_cast(self, item, value):
+        #    self.storage[item] = rffi.cast(_dtype.TP.OF, value)
+
+        def _sliceloop1(self, start, stop, step, source):
+            i = start
+            j = 0
+            conv = _dtype.convval
+            while i < stop:
+                #slice_driver1.jit_merge_point(signature=source.signature,
+                        #dtype=_dtype,
+                        #step=step, stop=stop, i=i, j=j, source=source,
+                        #self=self)
+                self.storage[i] = _dtype.convval(source.eval(j))
+                j += 1
+                i += step
+
+        def _sliceloop2(self, start, stop, step, source):
+            i = start
+            j = 0
+            conv = _dtype.convval
+            while i > stop:
+                #slice_driver2.jit_merge_point(signature=source.signature,
+                #        dtype=_dtype,
+                #        step=step, stop=stop, i=i, j=j, source=source,
+                #        self=self)
+                self.storage[i] = _dtype.convval(source.eval(j))
+                j += 1
+                i += step
+
+        def setslice(self, start, stop, step, slice_length, arr):
+            if step > 0:
+                self._sliceloop1(start, stop, step, arr)
+            else:
+                self._sliceloop2(start, stop, step, arr)
+
+        def setitem_w(self, space, item, value):
+            if space.issequence_w(item):
+                assert len(item) == len(self.shape)
+                pp = [0]+self.shape
+                indx = 0
+                for v,p in zip(item,pp):
+                    index += indx*p+v
+                self.storage[indx] = rffi.cast(_dtype.TP.OF, _dtype.unwrap(space, value))
+            else:    
+                self.storage[item] = rffi.cast(_dtype.TP.OF, _dtype.unwrap(space, value))
+
+        def find_dtype(self):
+            return self.dtype
+
+        def __del__(self):
+            lltype.free(self.storage, flavor='raw', track_allocation=False)
+
+    TypedNDimArray.__name__ = 'NDimArray_' + _dtype.name
+    return TypedNDimArray
+
 _array_classes = [make_class(d) for d in _dtype_list]
+_ndarray_classes = [make_ndclass(d) for d in _dtype_list]
 
 def create_sdarray(L, dtype):
     arr_type = _array_classes[dtype.num]
     return arr_type(L)
+def create_ndarray(L, dtype):
+    arr_type = _ndarray_classes[dtype.num]
+    return arr_type(L)
 
 def new_numarray(space, iterable, dtype):
+    #shape = []
+    #while True:
+    #   try:
+    #       # Try to get the length, a good proxy for iterability.
+    #       length = space.len_w(w_input)
+    #   except OperationError, e:
+           # If it raised a TypeError it's not an iteratble, however if it raises
+           # some other error we propogate it, that means __len__ raised something.
+    #      if not e.matches(space, space.w_TypeError):
+    #        raise
+    #    break
+    #else:
+    #    shape.append(length)
+    #    w_input = space.getitem(w_input, space.wrap(0)
     l = space.listview(iterable)
     dtype = get_dtype(space, dtype)
+    w_elem = space.getitem(iterable, space.wrap(0))
+    if space.issequence_w(w_elem):
+        #Determine the size
+        shape =[len(l)]
+        while space.issequence_w(w_elem):
+            shape.append(space.len_w(w_elem))
+            w_elem = space.getitem(w_elem, space.wrap(0))
+        arr = create_ndarray(shape,dtype)
+        depth = 0
+        #arr.setitem_w(space,shape,l)
+        return arr
     arr = create_sdarray(len(l), dtype)
     i = 0
     for w_elem in l:

From noreply at buildbot.pypy.org  Sun Oct 16 23:26:47 2011
From: noreply at buildbot.pypy.org (mattip)
Date: Sun, 16 Oct 2011 23:26:47 +0200 (CEST)
Subject: [pypy-commit] pypy numpy NDimArray: start
Message-ID: <20111016212647.74245820D7@wyvern.cs.uni-duesseldorf.de>

Author: mattip
Branch: numpy NDimArray
Changeset: r48098:9d6ec72a5681
Date: 2011-10-16 08:36 +0200
http://bitbucket.org/pypy/pypy/changeset/9d6ec72a5681/

Log:	start

diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -163,8 +163,10 @@
         return v1 * v2
     @binop
     def div(self, v1, v2):
-        return v1 / v2
-
+        try:
+            return v1 / v2
+        except ZeroDivisionError:
+            return (1 if v1>=0 else -1) *(1 if v2>=0 else -1)*float('inf')
     @unaryop
     def pos(self, v):
         return +v
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -32,8 +32,50 @@
     def add_invalidates(self, other):
         self.invalidates.append(other)
 
+    def find_dtype(space,w_iterable_or_scalar, w_dtype):
+        if w_dtype is space.fromcache(interp_dtype.W_Float64Dtype):
+            return w_dtype
+        if space.issequence_w(w_iterable_or_scalar):
+            w_iterator = space.iter(w_iterable_or_scalar)
+            while True:
+                try:
+                    w_item = space.next(w_iterator)
+                except OperationError, e:
+                    if not e.match(space, space.w_StopIteration):
+                        raise
+                    return w_dtype
+                w_dtype = BaseArray.find_dtype.im_func(space,w_item, w_dtype)
+        else:
+            w_dtype = interp_ufuncs.find_dtype_for_scalar(space,w_iterable_or_scalar, w_dtype)
+        return w_dtype
+                  
     def descr__new__(space, w_subtype, w_size_or_iterable, w_dtype=None):
         l = space.listview(w_size_or_iterable)
+        w_elem = space.getitem(w_size_or_iterable, space.wrap(0))
+        if space.issequence_w(w_elem):
+            shape = [len(l)]
+            while space.issequence_w(w_elem):
+                shape.append(space.len_w(w_elem))
+                w_elem = space.getitem(w_elem, space.wrap(0))
+            if space.is_w(w_dtype, space.w_None):
+                w_dtype = None
+                w_dtype = BaseArray.find_dtype.im_func(space, w_size_or_iterable, w_dtype)
+                if w_dtype is None:
+                    w_dtype = space.w_None
+
+            dtype = space.interp_w(interp_dtype.W_Dtype,
+                space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype)
+            )
+            arr = NDimArray(shape, dtype=dtype)
+            #Assign all the values
+            assigner = []
+            for s in shape[1:]:
+                assigner.append(':')
+            i = 0
+            for w_elem in l:
+                dtype.setitem_w(space, arr.storage, [i]+assigner, w_elem)
+                i += 1
+            return arr
         if space.is_w(w_dtype, space.w_None):
             w_dtype = None
             for w_item in l:
@@ -199,7 +241,8 @@
         return space.wrap(self.find_dtype())
 
     def descr_get_shape(self, space):
-        return space.newtuple([self.descr_len(space)])
+        #return space.newtuple([self.descr_len(space)])
+        return self.get_concrete().descr_shape(space)
 
     def descr_copy(self, space):
         return space.call_function(space.gettypefor(BaseArray), self, self.find_dtype())
@@ -499,6 +542,9 @@
     def find_dtype(self):
         return self.parent.find_dtype()
 
+    def descr_shape(self,space):
+        return space.newtuple([space.wrap(self.size)])
+
     def setslice(self, space, start, stop, step, slice_length, arr):
         start = self.calc_index(start)
         if stop != -1:
@@ -533,6 +579,55 @@
     def eval(self, i):
         return self.dtype.getitem(self.storage, i)
 
+    def descr_shape(self, space):
+        return space.newtuple([space.wrap(self.size)])
+
+    def descr_len(self, space):
+        return space.wrap(self.size)
+
+    def setitem_w(self, space, item, w_value):
+        self.invalidated()
+        self.dtype.setitem_w(space, self.storage, item, w_value)
+
+    def setitem(self, item, value):
+        self.invalidated()
+        self.dtype.setitem(self.storage, item, value)
+
+    def setslice(self, space, start, stop, step, slice_length, arr):
+        self._sliceloop(start, stop, step, arr, self)
+
+    def __del__(self):
+        lltype.free(self.storage, flavor='raw', track_allocation=False)
+
+class NDimArray(BaseArray):
+    def __init__(self, shape, dtype):
+        BaseArray.__init__(self)
+        self.size = 1
+        for s in shape:
+            self.size *= s
+        self.shape = shape
+        self.dtype = dtype
+        self.storage = dtype.malloc(self.size)
+        self.signature = dtype.signature
+
+    def get_concrete(self):
+        return self
+
+    def get_root_storage(self):
+        return self.storage
+
+    def find_size(self):
+        return self.size
+
+    def find_dtype(self):
+        return self.dtype
+
+    def eval(self, i):
+        return self.dtype.getitem(self.storage, i)
+
+    def descr_shape(self, space):
+        return space.newtuple(space.wrap(self.shape))
+
     def descr_len(self, space):
         return space.wrap(self.size)
 

From noreply at buildbot.pypy.org  Sun Oct 16 23:26:48 2011
From: noreply at buildbot.pypy.org (mattip)
Date: Sun, 16 Oct 2011 23:26:48 +0200 (CEST)
Subject: [pypy-commit] pypy numpy NDimArray: can construct NDimArray
Message-ID: <20111016212648.A479C820D7@wyvern.cs.uni-duesseldorf.de>

Author: mattip
Branch: numpy NDimArray
Changeset: r48099:c967a5f84d60
Date: 2011-10-16 22:54 +0200
http://bitbucket.org/pypy/pypy/changeset/c967a5f84d60/

Log:	can construct NDimArray

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -6,6 +6,7 @@
 from pypy.rlib import jit
 from pypy.rpython.lltypesystem import lltype
 from pypy.tool.sourcetools import func_with_new_name
+from traceback import print_exc
 
 
 numpy_driver = jit.JitDriver(greens = ['signature'],
@@ -68,13 +69,13 @@
             )
             arr = NDimArray(shape, dtype=dtype)
             #Assign all the values
-            assigner = []
-            for s in shape[1:]:
-                assigner.append(':')
-            i = 0
-            for w_elem in l:
-                dtype.setitem_w(space, arr.storage, [i]+assigner, w_elem)
-                i += 1
+            try:
+                i = 0
+                for w_elem in l:
+                    arr.setitem_recurse_w(space,i,0,w_elem)
+                    i += 1
+            except:
+                print_exc()
             return arr
         if space.is_w(w_dtype, space.w_None):
             w_dtype = None
@@ -251,15 +252,16 @@
         return self.get_concrete().descr_len(space)
 
     def descr_repr(self, space):
+        return self.get_concrete().descr_repr(space)
         # Simple implementation so that we can see the array. Needs work.
-        concrete = self.get_concrete()
-        res = "array([" + ", ".join(concrete._getnums(False)) + "]"
-        dtype = concrete.find_dtype()
-        if (dtype is not space.fromcache(interp_dtype.W_Float64Dtype) and
-            dtype is not space.fromcache(interp_dtype.W_Int64Dtype)) or not self.find_size():
-            res += ", dtype=" + dtype.name
-        res += ")"
-        return space.wrap(res)
+        #concrete = self.get_concrete()
+        #res = "array([" + ", ".join(concrete._getnums(False)) + "]"
+        #dtype = concrete.find_dtype()
+        #if (dtype is not space.fromcache(interp_dtype.W_Float64Dtype) and
+        #    dtype is not space.fromcache(interp_dtype.W_Int64Dtype)) or not self.find_size():
+        #    res += ", dtype=" + dtype.name
+        #res += ")"
+        #return space.wrap(res)
 
     def descr_str(self, space):
         # Simple implementation so that we can see the array. Needs work.
@@ -555,6 +557,16 @@
     def calc_index(self, item):
         return (self.start + item * self.step)
 
+    def descr_repr(self, space):
+        # Simple implementation so that we can see the array. Needs work.
+        concrete = self.get_concrete()
+        res = "array([" + ", ".join(concrete._getnums(False)) + "]"
+        dtype = concrete.find_dtype()
+        if (dtype is not space.fromcache(interp_dtype.W_Float64Dtype) and
+            dtype is not space.fromcache(interp_dtype.W_Int64Dtype)) or not self.find_size():
+            res += ", dtype=" + dtype.name
+        res += ")"
+        return space.wrap(res)
 
 class SingleDimArray(BaseArray):
     def __init__(self, size, dtype):
@@ -596,6 +608,17 @@
     def setslice(self, space, start, stop, step, slice_length, arr):
         self._sliceloop(start, stop, step, arr, self)
 
+    def descr_repr(self, space):
+        # Simple implementation so that we can see the array. Needs work.
+        concrete = self.get_concrete()
+        res = "array([" + ", ".join(concrete._getnums(False)) + "]"
+        dtype = concrete.find_dtype()
+        if (dtype is not space.fromcache(interp_dtype.W_Float64Dtype) and
+            dtype is not space.fromcache(interp_dtype.W_Int64Dtype)) or not self.find_size():
+            res += ", dtype=" + dtype.name
+        res += ")"
+        return space.wrap(res)
+
     def __del__(self):
         lltype.free(self.storage, flavor='raw', track_allocation=False)
 
@@ -626,7 +649,7 @@
         return self.dtype.getitem(self.storage, i)
 
     def descr_shape(self, space):
-        return space.newtuple(space.wrap(self.shape))
+        return self.shape
 
     def descr_len(self, space):
         return space.wrap(self.size)
@@ -635,6 +658,22 @@
         self.invalidated()
         self.dtype.setitem_w(space, self.storage, item, w_value)
 
+    def setitem_recurse_w(self,space,index,depth,w_value):
+        self.invalidated()
+        if space.issequence_w(w_value):
+            i=0
+            w_iterator = space.iter(w_value)
+            while True:
+                try:
+                    w_item = space.next(w_iterator)
+                except OperationError, e:
+                    if not e.match(space, space.w_StopIteration):
+                        raise
+                    return 
+                self.setitem_recurse_w(space,i+index*self.shape[depth], depth+1, w_item)
+                i+=1
+        else:
+            self.setitem_w(space,index,w_value) 
     def setitem(self, item, value):
         self.invalidated()
         self.dtype.setitem(self.storage, item, value)
@@ -642,6 +681,17 @@
     def setslice(self, space, start, stop, step, slice_length, arr):
         self._sliceloop(start, stop, step, arr, self)
 
+    def descr_repr(self, space):
+        # Simple implementation so that we can see the array. Needs work.
+        concrete = self.get_concrete()
+        res = "ndarray([" + ", ".join(concrete._getnums(False)) + "]"
+        dtype = concrete.find_dtype()
+        if (dtype is not space.fromcache(interp_dtype.W_Float64Dtype) and
+            dtype is not space.fromcache(interp_dtype.W_Int64Dtype)) or not self.find_size():
+            res += ", dtype=" + dtype.name
+        res += ")"
+        return space.wrap(res)
+
     def __del__(self):
         lltype.free(self.storage, flavor='raw', track_allocation=False)
 

From noreply at buildbot.pypy.org  Sun Oct 16 23:26:49 2011
From: noreply at buildbot.pypy.org (mattip)
Date: Sun, 16 Oct 2011 23:26:49 +0200 (CEST)
Subject: [pypy-commit] pypy numpy NDimArray: add test for construct NDimArray
Message-ID: <20111016212649.D51DE820D7@wyvern.cs.uni-duesseldorf.de>

Author: mattip
Branch: numpy NDimArray
Changeset: r48100:b90c8d28d864
Date: 2011-10-16 22:56 +0200
http://bitbucket.org/pypy/pypy/changeset/b90c8d28d864/

Log:	add test for construct NDimArray

diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -48,6 +48,12 @@
         a = array(range(5))
         assert a[3] == 3
 
+    def test_iterator_init(self):
+        from numpy import array
+        a = array((range(5),range(5))
+        assert a[3] == 3
+        assert a[8] == 3
+
     def test_repr(self):
         from numpy import array, zeros
         a = array(range(5), float)

From noreply at buildbot.pypy.org  Sun Oct 16 23:26:51 2011
From: noreply at buildbot.pypy.org (mattip)
Date: Sun, 16 Oct 2011 23:26:51 +0200 (CEST)
Subject: [pypy-commit] pypy numpy NDimArray: fix iterative indexing
Message-ID: <20111016212651.12B27820D7@wyvern.cs.uni-duesseldorf.de>

Author: mattip
Branch: numpy NDimArray
Changeset: r48101:26c000b7e0f5
Date: 2011-10-16 23:07 +0200
http://bitbucket.org/pypy/pypy/changeset/26c000b7e0f5/

Log:	fix iterative indexing

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -72,7 +72,7 @@
             try:
                 i = 0
                 for w_elem in l:
-                    arr.setitem_recurse_w(space,i,0,w_elem)
+                    arr.setitem_recurse_w(space,i,1,w_elem)
                     i += 1
             except:
                 print_exc()
@@ -670,9 +670,10 @@
                     if not e.match(space, space.w_StopIteration):
                         raise
                     return 
-                self.setitem_recurse_w(space,i+index*self.shape[depth], depth+1, w_item)
+                self.setitem_recurse_w(space,i+index*self.shape[-depth], depth+1, w_item)
                 i+=1
         else:
+            print 'setting',index,'to',w_value
             self.setitem_w(space,index,w_value) 
     def setitem(self, item, value):
         self.invalidated()

From noreply at buildbot.pypy.org  Sun Oct 16 23:40:04 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 16 Oct 2011 23:40:04 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix the position of the "unmatched ')'"
	errors.
Message-ID: <20111016214004.04ED8820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48102:2c0f4e7c8383
Date: 2011-10-16 23:39 +0200
http://bitbucket.org/pypy/pypy/changeset/2c0f4e7c8383/

Log:	Fix the position of the "unmatched ')'" errors.

diff --git a/pypy/interpreter/pyparser/pytokenizer.py b/pypy/interpreter/pyparser/pytokenizer.py
--- a/pypy/interpreter/pyparser/pytokenizer.py
+++ b/pypy/interpreter/pyparser/pytokenizer.py
@@ -226,7 +226,7 @@
                         parenlev = parenlev - 1
                         if parenlev < 0:
                             raise TokenError("unmatched '%s'" % initial, line,
-                                             lnum-1, 0, token_list)
+                                             lnum, start + 1, token_list)
                     if token in python_opmap:
                         punct = python_opmap[token]
                     else:
diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py
--- a/pypy/interpreter/pyparser/test/test_pyparse.py
+++ b/pypy/interpreter/pyparser/test/test_pyparse.py
@@ -87,6 +87,10 @@
         assert exc.lineno == 1
         assert exc.offset == 5
         assert exc.lastlineno == 5
+        exc = py.test.raises(SyntaxError, parse, "abc)").value
+        assert exc.msg == "unmatched ')'"
+        assert exc.lineno == 1
+        assert exc.offset == 4
 
     def test_is(self):
         self.parse("x is y")

From noreply at buildbot.pypy.org  Mon Oct 17 02:29:34 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Mon, 17 Oct 2011 02:29:34 +0200 (CEST)
Subject: [pypy-commit] pypy default: handle division by zero in numpy
	correctly
Message-ID: <20111017002934.C5A67820D7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r48103:bcab5565a59a
Date: 2011-10-16 20:29 -0400
http://bitbucket.org/pypy/pypy/changeset/bcab5565a59a/

Log:	handle division by zero in numpy correctly

diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -161,9 +161,6 @@
     @binop
     def mul(self, v1, v2):
         return v1 * v2
-    @binop
-    def div(self, v1, v2):
-        return v1 / v2
 
     @unaryop
     def pos(self, v):
@@ -217,6 +214,14 @@
         return float2string(self.for_computation(self.unbox(item)), 'g', rfloat.DTSF_STR_PRECISION)
 
     @binop
+    def div(self, v1, v2):
+        try:
+            return v1 / v2
+        except ZeroDivisionError:
+            if v1 == v2 == 0.0:
+                return rfloat.NAN
+            return rfloat.copysign(rfloat.INFINITY, v1 * v2)
+    @binop
     def mod(self, v1, v2):
         return math.fmod(v1, v2)
     @binop
@@ -295,6 +300,11 @@
         return str(widen(self.unbox(item)))
 
     @binop
+    def div(self, v1, v2):
+        if v2 == 0:
+            return 0
+        return v1 / v2
+    @binop
     def mod(self, v1, v2):
         return v1 % v2
 
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -285,7 +285,9 @@
             assert b[i] == i * 5
 
     def test_div(self):
-        from numpy import array, dtype
+        from math import isnan
+        from numpy import array, dtype, inf
+
         a = array(range(1, 6))
         b = a / a
         for i in range(5):
@@ -297,6 +299,24 @@
         for i in range(5):
             assert b[i] == 1
 
+        a = array([-1, 0, 1])
+        b = array([0, 0, 0])
+        c = a / b
+        assert (c == [0, 0, 0]).all()
+
+        a = array([-1.0, 0.0, 1.0])
+        b = array([0.0, 0.0, 0.0])
+        c = a / b
+        assert c[0] == -inf
+        assert isnan(c[1])
+        assert c[2] == inf
+
+        b = array([-0.0, -0.0, -0.0])
+        c = a / b
+        assert c[0] == inf
+        assert isnan(c[1])
+        assert c[2] == -inf
+
     def test_div_other(self):
         from numpy import array
         a = array(range(5))

From noreply at buildbot.pypy.org  Mon Oct 17 02:33:52 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Mon, 17 Oct 2011 02:33:52 +0200 (CEST)
Subject: [pypy-commit] pypy numpy NDimArray: merged default in
Message-ID: <20111017003352.5B54C820D7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: numpy NDimArray
Changeset: r48104:cb0cdef747f2
Date: 2011-10-16 20:31 -0400
http://bitbucket.org/pypy/pypy/changeset/cb0cdef747f2/

Log:	merged default in

diff --git a/pypy/interpreter/pyparser/pytokenizer.py b/pypy/interpreter/pyparser/pytokenizer.py
--- a/pypy/interpreter/pyparser/pytokenizer.py
+++ b/pypy/interpreter/pyparser/pytokenizer.py
@@ -226,7 +226,7 @@
                         parenlev = parenlev - 1
                         if parenlev < 0:
                             raise TokenError("unmatched '%s'" % initial, line,
-                                             lnum-1, 0, token_list)
+                                             lnum, start + 1, token_list)
                     if token in python_opmap:
                         punct = python_opmap[token]
                     else:
diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py
--- a/pypy/interpreter/pyparser/test/test_pyparse.py
+++ b/pypy/interpreter/pyparser/test/test_pyparse.py
@@ -87,6 +87,10 @@
         assert exc.lineno == 1
         assert exc.offset == 5
         assert exc.lastlineno == 5
+        exc = py.test.raises(SyntaxError, parse, "abc)").value
+        assert exc.msg == "unmatched ')'"
+        assert exc.lineno == 1
+        assert exc.offset == 4
 
     def test_is(self):
         self.parse("x is y")
diff --git a/pypy/jit/backend/llgraph/llimpl.py b/pypy/jit/backend/llgraph/llimpl.py
--- a/pypy/jit/backend/llgraph/llimpl.py
+++ b/pypy/jit/backend/llgraph/llimpl.py
@@ -165,6 +165,7 @@
     'unicodegetitem'  : (('ref', 'int'), 'int'),
     'unicodesetitem'  : (('ref', 'int', 'int'), 'int'),
     'cast_ptr_to_int' : (('ref',), 'int'),
+    'cast_int_to_ptr' : (('int',), 'ref'),
     'debug_merge_point': (('ref', 'int'), None),
     'force_token'     : ((), 'int'),
     'call_may_force'  : (('int', 'varargs'), 'intorptr'),
@@ -875,9 +876,6 @@
     def op_new_array(self, arraydescr, count):
         return do_new_array(arraydescr.ofs, count)
 
-    def op_cast_ptr_to_int(self, descr, ptr):
-        return cast_to_int(ptr)
-
     def op_force_token(self, descr):
         opaque_frame = _to_opaque(self)
         return llmemory.cast_ptr_to_adr(opaque_frame)
diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -1468,20 +1468,16 @@
         return u''.join(u.chars)
 
 
-    def test_casts(self):
-        py.test.skip("xxx fix or kill")
-        from pypy.rpython.lltypesystem import lltype, llmemory
-        TP = lltype.GcStruct('x')
-        x = lltype.malloc(TP)        
-        x = lltype.cast_opaque_ptr(llmemory.GCREF, x)
+    def test_cast_int_to_ptr(self):
+        res = self.execute_operation(rop.CAST_INT_TO_PTR,
+                                     [BoxInt(-17)],  'ref').value
+        assert lltype.cast_ptr_to_int(res) == -17
+
+    def test_cast_ptr_to_int(self):
+        x = lltype.cast_int_to_ptr(llmemory.GCREF, -19)
         res = self.execute_operation(rop.CAST_PTR_TO_INT,
-                                     [BoxPtr(x)],  'int').value
-        expected = self.cpu.cast_adr_to_int(llmemory.cast_ptr_to_adr(x))
-        assert rffi.get_real_int(res) == rffi.get_real_int(expected)
-        res = self.execute_operation(rop.CAST_PTR_TO_INT,
-                                     [ConstPtr(x)],  'int').value
-        expected = self.cpu.cast_adr_to_int(llmemory.cast_ptr_to_adr(x))
-        assert rffi.get_real_int(res) == rffi.get_real_int(expected)
+                                     [BoxPtr(x)], 'int').value
+        assert res == -19
 
     def test_ooops_non_gc(self):
         x = lltype.malloc(lltype.Struct('x'), flavor='raw')
@@ -2299,13 +2295,6 @@
         #
         cpu.bh_strsetitem(x, 4, ord('/'))
         assert str.chars[4] == '/'
-        #
-##        x = cpu.bh_newstr(5)
-##        y = cpu.bh_cast_ptr_to_int(x)
-##        z = cpu.bh_cast_ptr_to_int(x)
-##        y = rffi.get_real_int(y)
-##        z = rffi.get_real_int(z)
-##        assert type(y) == type(z) == int and y == z
 
     def test_sorting_of_fields(self):
         S = self.S
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1387,7 +1387,8 @@
 
     def genop_same_as(self, op, arglocs, resloc):
         self.mov(arglocs[0], resloc)
-    #genop_cast_ptr_to_int = genop_same_as
+    genop_cast_ptr_to_int = genop_same_as
+    genop_cast_int_to_ptr = genop_same_as
 
     def genop_int_mod(self, op, arglocs, resloc):
         if IS_X86_32:
diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py
--- a/pypy/jit/backend/x86/regalloc.py
+++ b/pypy/jit/backend/x86/regalloc.py
@@ -1152,7 +1152,8 @@
         self.possibly_free_var(op.getarg(0))
         resloc = self.force_allocate_reg(op.result)
         self.Perform(op, [argloc], resloc)
-    #consider_cast_ptr_to_int = consider_same_as
+    consider_cast_ptr_to_int = consider_same_as
+    consider_cast_int_to_ptr = consider_same_as
 
     def consider_strlen(self, op):
         args = op.getarglist()
diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -800,8 +800,7 @@
 
     def rewrite_op_cast_ptr_to_int(self, op):
         if self._is_gc(op.args[0]):
-            #return op
-            raise NotImplementedError("cast_ptr_to_int")
+            return op
 
     def rewrite_op_force_cast(self, op):
         v_arg = op.args[0]
@@ -1543,6 +1542,10 @@
     def rewrite_op_jit_force_virtual(self, op):
         return self._do_builtin_call(op)
 
+    def rewrite_op_jit_is_virtual(self, op):
+        raise Exception, (
+            "'vref.virtual' should not be used from jit-visible code")
+
     def rewrite_op_jit_force_virtualizable(self, op):
         # this one is for virtualizables
         vinfo = self.get_vinfo(op.args[0])
diff --git a/pypy/jit/metainterp/blackhole.py b/pypy/jit/metainterp/blackhole.py
--- a/pypy/jit/metainterp/blackhole.py
+++ b/pypy/jit/metainterp/blackhole.py
@@ -2,7 +2,7 @@
 from pypy.rlib.rtimer import read_timestamp
 from pypy.rlib.rarithmetic import intmask, LONG_BIT, r_uint, ovfcheck
 from pypy.rlib.objectmodel import we_are_translated
-from pypy.rlib.debug import debug_start, debug_stop
+from pypy.rlib.debug import debug_start, debug_stop, ll_assert
 from pypy.rlib.debug import make_sure_not_resized
 from pypy.rpython.lltypesystem import lltype, llmemory, rclass
 from pypy.rpython.lltypesystem.lloperation import llop
@@ -503,6 +503,15 @@
     @arguments("r", returns="r")
     def bhimpl_cast_opaque_ptr(a):
         return a
+    @arguments("r", returns="i")
+    def bhimpl_cast_ptr_to_int(a):
+        i = lltype.cast_ptr_to_int(a)
+        ll_assert((i & 1) == 1, "bhimpl_cast_ptr_to_int: not an odd int")
+        return i
+    @arguments("i", returns="r")
+    def bhimpl_cast_int_to_ptr(i):
+        ll_assert((i & 1) == 1, "bhimpl_cast_int_to_ptr: not an odd int")
+        return lltype.cast_int_to_ptr(llmemory.GCREF, i)
 
     @arguments("i", returns="i")
     def bhimpl_int_copy(a):
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -222,6 +222,7 @@
                     'cast_float_to_int', 'cast_int_to_float',
                     'cast_float_to_singlefloat', 'cast_singlefloat_to_float',
                     'float_neg', 'float_abs',
+                    'cast_ptr_to_int', 'cast_int_to_ptr',
                     ]:
         exec py.code.Source('''
             @arguments("box")
diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py
--- a/pypy/jit/metainterp/resoperation.py
+++ b/pypy/jit/metainterp/resoperation.py
@@ -431,6 +431,8 @@
     'INT_IS_TRUE/1b',
     'INT_NEG/1',
     'INT_INVERT/1',
+    'CAST_PTR_TO_INT/1',
+    'CAST_INT_TO_PTR/1',
     #
     'SAME_AS/1',      # gets a Const or a Box, turns it into another Box
     #
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -3440,7 +3440,6 @@
 
 class TestLLtype(BaseLLtypeTests, LLJitMixin):
     def test_tagged(self):
-        py.test.skip("implement me")
         from pypy.rlib.objectmodel import UnboxedValue
         class Base(object):
             __slots__ = ()
@@ -3492,3 +3491,35 @@
                 pc += 1
             return pc
         res = self.meta_interp(main, [False, 100, True], taggedpointers=True)
+
+    def test_rerased(self):
+        from pypy.rlib.rerased import erase_int, unerase_int, new_erasing_pair
+        eraseX, uneraseX = new_erasing_pair("X")
+        #
+        class X:
+            def __init__(self, a, b):
+                self.a = a
+                self.b = b
+        #
+        def f(i, j):
+            # 'j' should be 0 or 1, not other values
+            if j > 0:
+                e = eraseX(X(i, j))
+            else:
+                try:
+                    e = erase_int(i)
+                except OverflowError:
+                    return -42
+            if j & 1:
+                x = uneraseX(e)
+                return x.a - x.b
+            else:
+                return unerase_int(e)
+        #
+        x = self.interp_operations(f, [-128, 0], taggedpointers=True)
+        assert x == -128
+        bigint = sys.maxint//2 + 1
+        x = self.interp_operations(f, [bigint, 0], taggedpointers=True)
+        assert x == -42
+        x = self.interp_operations(f, [1000, 1], taggedpointers=True)
+        assert x == 999
diff --git a/pypy/jit/metainterp/test/test_virtualref.py b/pypy/jit/metainterp/test/test_virtualref.py
--- a/pypy/jit/metainterp/test/test_virtualref.py
+++ b/pypy/jit/metainterp/test/test_virtualref.py
@@ -3,6 +3,7 @@
 from pypy.rpython.llinterp import LLException
 from pypy.rlib.jit import JitDriver, dont_look_inside, vref_None
 from pypy.rlib.jit import virtual_ref, virtual_ref_finish, InvalidVirtualRef
+from pypy.rlib.jit import non_virtual_ref
 from pypy.rlib.objectmodel import compute_unique_id
 from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin, _get_jitcodes
 from pypy.jit.metainterp.resoperation import rop
@@ -595,6 +596,65 @@
         res = self.meta_interp(fn, [10])
         assert res == 6
 
+    def test_is_virtual(self):
+        myjitdriver = JitDriver(greens=[], reds=['n', 'res1'])
+        class X:
+            pass
+        @dont_look_inside
+        def residual(vref):
+            return vref.virtual
+        #
+        def f(n):
+            res1 = -42
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, res1=res1)
+                x = X()
+                vref = virtual_ref(x)
+                res1 = residual(vref)
+                virtual_ref_finish(vref, x)
+                n -= 1
+            return res1
+        #
+        res = self.meta_interp(f, [10])
+        assert res == 1
+
+    def test_is_not_virtual_none(self):
+        myjitdriver = JitDriver(greens=[], reds=['n', 'res1'])
+        @dont_look_inside
+        def residual(vref):
+            return vref.virtual
+        #
+        def f(n):
+            res1 = -42
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, res1=res1)
+                res1 = residual(vref_None)
+                n -= 1
+            return res1
+        #
+        res = self.meta_interp(f, [10])
+        assert res == 0
+
+    def test_is_not_virtual_non_none(self):
+        myjitdriver = JitDriver(greens=[], reds=['n', 'res1'])
+        class X:
+            pass
+        @dont_look_inside
+        def residual(vref):
+            return vref.virtual
+        #
+        def f(n):
+            res1 = -42
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, res1=res1)
+                x = X()
+                res1 = residual(non_virtual_ref(x))
+                n -= 1
+            return res1
+        #
+        res = self.meta_interp(f, [10])
+        assert res == 0
+
 
 class TestLLtype(VRefTests, LLJitMixin):
     pass
diff --git a/pypy/jit/metainterp/virtualref.py b/pypy/jit/metainterp/virtualref.py
--- a/pypy/jit/metainterp/virtualref.py
+++ b/pypy/jit/metainterp/virtualref.py
@@ -39,6 +39,7 @@
     def replace_force_virtual_with_call(self, graphs):
         # similar to rvirtualizable2.replace_force_virtualizable_with_call().
         c_force_virtual_ptr = None
+        c_is_virtual_ptr = None
         force_virtual_count = 0
         for graph in graphs:
             for block in graph.iterblocks():
@@ -52,6 +53,13 @@
                         op.opname = 'direct_call'
                         op.args = [c_force_virtual_ptr, op.args[0]]
                         force_virtual_count += 1
+                    #
+                    if op.opname == 'jit_is_virtual':
+                        if c_is_virtual_ptr is None:
+                            c_is_virtual_ptr = self.get_is_virtual_fnptr()
+                        #
+                        op.opname = 'direct_call'
+                        op.args = [c_is_virtual_ptr, op.args[0]]
         #
         if c_force_virtual_ptr is not None:
             log("replaced %d 'jit_force_virtual' with %r" % (force_virtual_count,
@@ -129,6 +137,17 @@
             force_virtual_if_necessary)
         return inputconst(lltype.typeOf(funcptr), funcptr)
 
+    def get_is_virtual_fnptr(self):
+        #
+        def is_virtual(inst):
+            if not inst:
+                return False
+            return inst.typeptr == self.jit_virtual_ref_vtable
+        #
+        FUNC = lltype.FuncType([rclass.OBJECTPTR], lltype.Bool)
+        funcptr = self.warmrunnerdesc.helper_func(lltype.Ptr(FUNC), is_virtual)
+        return inputconst(lltype.typeOf(funcptr), funcptr)
+
     def force_virtual(self, inst):
         vref = lltype.cast_pointer(lltype.Ptr(self.JIT_VIRTUAL_REF), inst)
         token = vref.virtual_token
diff --git a/pypy/module/__builtin__/compiling.py b/pypy/module/__builtin__/compiling.py
--- a/pypy/module/__builtin__/compiling.py
+++ b/pypy/module/__builtin__/compiling.py
@@ -84,8 +84,8 @@
         raise OperationError(space.w_TypeError,
               w('eval() arg 1 must be a string or code object'))
 
-    caller = space.getexecutioncontext().gettopframe_nohidden()
     if space.is_w(w_globals, space.w_None):
+        caller = space.getexecutioncontext().gettopframe_nohidden()
         if caller is None:
             w_globals = space.newdict()
             if space.is_w(w_locals, space.w_None):
@@ -97,13 +97,6 @@
     elif space.is_w(w_locals, space.w_None):
         w_locals = w_globals
 
-    try:
-        space.getitem(w_globals, space.wrap('__builtins__'))
-    except OperationError, e:
-        if not e.match(space, space.w_KeyError):
-            raise
-        if caller is not None:
-            w_builtin = space.builtin.pick_builtin(caller.w_globals)
-            space.setitem(w_globals, space.wrap('__builtins__'), w_builtin)
+    space.builtin.pick_builtin(w_globals)
 
     return codeobj.exec_code(space, w_globals, w_locals)
diff --git a/pypy/module/_continuation/interp_continuation.py b/pypy/module/_continuation/interp_continuation.py
--- a/pypy/module/_continuation/interp_continuation.py
+++ b/pypy/module/_continuation/interp_continuation.py
@@ -19,6 +19,11 @@
         #  - normal:      self.sthread != None, not is_empty_handle(self.h)
         #  - finished:    self.sthread != None, is_empty_handle(self.h)
 
+    def __del__(self):
+        sthread = self.sthread
+        if sthread is not None and not sthread.is_empty_handle(self.h):
+            sthread.destroy(self.h)
+
     def check_sthread(self):
         ec = self.space.getexecutioncontext()
         if ec.stacklet_thread is not self.sthread:
@@ -28,6 +33,8 @@
     def descr_init(self, w_callable, __args__):
         if self.sthread is not None:
             raise geterror(self.space, "continulet already __init__ialized")
+        sthread = build_sthread(self.space)
+        workaround_disable_jit(sthread)
         #
         # hackish: build the frame "by hand", passing it the correct arguments
         space = self.space
@@ -41,7 +48,6 @@
         self.bottomframe = bottomframe
         #
         global_state.origin = self
-        sthread = build_sthread(self.space)
         self.sthread = sthread
         h = sthread.new(new_stacklet_callback)
         post_switch(sthread, h)
@@ -71,6 +77,7 @@
                 global_state.clear()
                 raise geterror(self.space, "continulet already finished")
         self.check_sthread()
+        workaround_disable_jit(self.sthread)
         #
         global_state.origin = self
         if to is None:
@@ -259,6 +266,16 @@
         sthread = ec.stacklet_thread = SThread(space, ec)
     return sthread
 
+def workaround_disable_jit(sthread):
+    # A bad workaround to kill the JIT anywhere in this thread.
+    # This forces all the frames.  It's a bad workaround because
+    # it takes O(depth) time, and it will cause some "abort:
+    # vable escape" in the JIT.  The goal is to prevent any frame
+    # from being still virtuals, because the JIT generates code
+    # to un-virtualizable them "on demand" by loading values based
+    # on FORCE_TOKEN, which is an address in the stack.
+    sthread.ec.force_all_frames()
+
 # ____________________________________________________________
 
 def permute(space, args_w):
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -161,12 +161,7 @@
     @binop
     def mul(self, v1, v2):
         return v1 * v2
-    @binop
-    def div(self, v1, v2):
-        try:
-            return v1 / v2
-        except ZeroDivisionError:
-            return (1 if v1>=0 else -1) *(1 if v2>=0 else -1)*float('inf')
+
     @unaryop
     def pos(self, v):
         return +v
@@ -219,6 +214,14 @@
         return float2string(self.for_computation(self.unbox(item)), 'g', rfloat.DTSF_STR_PRECISION)
 
     @binop
+    def div(self, v1, v2):
+        try:
+            return v1 / v2
+        except ZeroDivisionError:
+            if v1 == v2 == 0.0:
+                return rfloat.NAN
+            return rfloat.copysign(rfloat.INFINITY, v1 * v2)
+    @binop
     def mod(self, v1, v2):
         return math.fmod(v1, v2)
     @binop
@@ -297,6 +300,11 @@
         return str(widen(self.unbox(item)))
 
     @binop
+    def div(self, v1, v2):
+        if v2 == 0:
+            return 0
+        return v1 / v2
+    @binop
     def mod(self, v1, v2):
         return v1 % v2
 
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -291,7 +291,9 @@
             assert b[i] == i * 5
 
     def test_div(self):
-        from numpy import array, dtype
+        from math import isnan
+        from numpy import array, dtype, inf
+
         a = array(range(1, 6))
         b = a / a
         for i in range(5):
@@ -303,6 +305,24 @@
         for i in range(5):
             assert b[i] == 1
 
+        a = array([-1, 0, 1])
+        b = array([0, 0, 0])
+        c = a / b
+        assert (c == [0, 0, 0]).all()
+
+        a = array([-1.0, 0.0, 1.0])
+        b = array([0.0, 0.0, 0.0])
+        c = a / b
+        assert c[0] == -inf
+        assert isnan(c[1])
+        assert c[2] == inf
+
+        b = array([-0.0, -0.0, -0.0])
+        c = a / b
+        assert c[0] == inf
+        assert isnan(c[1])
+        assert c[2] == -inf
+
     def test_div_other(self):
         from numpy import array
         a = array(range(5))
diff --git a/pypy/module/pypyjit/test_pypy_c/model.py b/pypy/module/pypyjit/test_pypy_c/model.py
--- a/pypy/module/pypyjit/test_pypy_c/model.py
+++ b/pypy/module/pypyjit/test_pypy_c/model.py
@@ -225,6 +225,8 @@
         # strip comment
         if '#' in line:
             line = line[:line.index('#')]
+        if line.strip() == 'guard_not_invalidated?':
+            return 'guard_not_invalidated', None, [], '...', False
         # find the resvar, if any
         if ' = ' in line:
             resvar, _, line = line.partition(' = ')
@@ -249,7 +251,7 @@
             descr = descr[len('descr='):]
         else:
             descr = None
-        return opname, resvar, args, descr
+        return opname, resvar, args, descr, True
 
     @classmethod
     def preprocess_expected_src(cls, src):
@@ -258,13 +260,23 @@
         # replaced with the corresponding operations, so that tests don't have
         # to repeat it every time
         ticker_check = """
+            guard_not_invalidated?
+            ticker0 = getfield_raw(ticker_address, descr=<SignedFieldDescr pypysig_long_struct.c_value .*>)
+            ticker_cond0 = int_lt(ticker0, 0)
+            guard_false(ticker_cond0, descr=...)
+        """
+        src = src.replace('--TICK--', ticker_check)
+        #
+        # this is the ticker check generated if we have threads
+        thread_ticker_check = """
+            guard_not_invalidated?
             ticker0 = getfield_raw(ticker_address, descr=<SignedFieldDescr pypysig_long_struct.c_value .*>)
             ticker1 = int_sub(ticker0, 1)
             setfield_raw(ticker_address, ticker1, descr=<SignedFieldDescr pypysig_long_struct.c_value .*>)
             ticker_cond0 = int_lt(ticker1, 0)
             guard_false(ticker_cond0, descr=...)
         """
-        src = src.replace('--TICK--', ticker_check)
+        src = src.replace('--THREAD-TICK--', thread_ticker_check)
         #
         # this is the ticker check generated in PyFrame.handle_operation_error
         exc_ticker_check = """
@@ -298,7 +310,7 @@
         if not cond:
             raise InvalidMatch(message, frame=sys._getframe(1))
 
-    def match_op(self, op, (exp_opname, exp_res, exp_args, exp_descr)):
+    def match_op(self, op, (exp_opname, exp_res, exp_args, exp_descr, _)):
         self._assert(op.name == exp_opname, "operation mismatch")
         self.match_var(op.res, exp_res)
         if exp_args != ['...']:
@@ -341,7 +353,7 @@
         what is after the '...'
         """
         iter_exp_ops = iter(expected_ops)
-        iter_ops = iter(self.ops)
+        iter_ops = RevertableIterator(self.ops)
         for opindex, exp_op in enumerate(iter_exp_ops):
             try:
                 if exp_op == '...':
@@ -360,6 +372,9 @@
                             break
                 self.match_op(op, exp_op)
             except InvalidMatch, e:
+                if exp_op[4] is False:    # optional operation
+                    iter_ops.revert_one()
+                    continue       # try to match with the next exp_op
                 e.opindex = opindex
                 raise
         #
@@ -398,3 +413,18 @@
         else:
             return True
 
+
+class RevertableIterator(object):
+    def __init__(self, sequence):
+        self.sequence = sequence
+        self.index = 0
+    def __iter__(self):
+        return self
+    def next(self):
+        index = self.index
+        if index == len(self.sequence):
+            raise StopIteration
+        self.index = index + 1
+        return self.sequence[index]
+    def revert_one(self):
+        self.index -= 1
diff --git a/pypy/module/pypyjit/test_pypy_c/test_00_model.py b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
--- a/pypy/module/pypyjit/test_pypy_c/test_00_model.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
@@ -145,15 +145,17 @@
 
     def test_parse_op(self):
         res = OpMatcher.parse_op("  a =   int_add(  b,  3 ) # foo")
-        assert res == ("int_add", "a", ["b", "3"], None)
+        assert res == ("int_add", "a", ["b", "3"], None, True)
         res = OpMatcher.parse_op("guard_true(a)")
-        assert res == ("guard_true", None, ["a"], None)
+        assert res == ("guard_true", None, ["a"], None, True)
         res = OpMatcher.parse_op("setfield_gc(p0, i0, descr=<foobar>)")
-        assert res == ("setfield_gc", None, ["p0", "i0"], "<foobar>")
+        assert res == ("setfield_gc", None, ["p0", "i0"], "<foobar>", True)
         res = OpMatcher.parse_op("i1 = getfield_gc(p0, descr=<foobar>)")
-        assert res == ("getfield_gc", "i1", ["p0"], "<foobar>")
+        assert res == ("getfield_gc", "i1", ["p0"], "<foobar>", True)
         res = OpMatcher.parse_op("p0 = force_token()")
-        assert res == ("force_token", "p0", [], None)
+        assert res == ("force_token", "p0", [], None, True)
+        res = OpMatcher.parse_op("guard_not_invalidated?")
+        assert res == ("guard_not_invalidated", None, [], '...', False)
 
     def test_exact_match(self):
         loop = """
@@ -341,7 +343,7 @@
             # this is the actual loop
             'int_lt', 'guard_true', 'int_add',
             # this is the signal checking stuff
-            'getfield_raw', 'int_sub', 'setfield_raw', 'int_lt', 'guard_false',
+            'guard_not_invalidated', 'getfield_raw', 'int_lt', 'guard_false',
             'jump'
             ]
 
@@ -407,7 +409,7 @@
             # this is the actual loop
             'int_lt', 'guard_true', 'force_token', 'int_add',
             # this is the signal checking stuff
-            'getfield_raw', 'int_sub', 'setfield_raw', 'int_lt', 'guard_false',
+            'guard_not_invalidated', 'getfield_raw', 'int_lt', 'guard_false',
             'jump'
             ]
 
@@ -425,10 +427,9 @@
             guard_true(i6, descr=...)
             i8 = int_add(i4, 1)
             # signal checking stuff
+            guard_not_invalidated(descr=...)
             i10 = getfield_raw(37212896, descr=<.* pypysig_long_struct.c_value .*>)
-            i12 = int_sub(i10, 1)
-            setfield_raw(37212896, i12, descr=<.* pypysig_long_struct.c_value .*>)
-            i14 = int_lt(i12, 0)
+            i14 = int_lt(i10, 0)
             guard_false(i14, descr=...)
             jump(p0, p1, p2, p3, i8, descr=...)
         """)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_misc.py b/pypy/module/pypyjit/test_pypy_c/test_misc.py
--- a/pypy/module/pypyjit/test_pypy_c/test_misc.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_misc.py
@@ -329,4 +329,20 @@
             guard_false(i28, descr=...)
             i30 = int_lshift(i20, 24)
             i31 = int_or(i26, i30)
-        """ % {"32_bit_only": extra})
\ No newline at end of file
+        """ % {"32_bit_only": extra})
+
+    def test_eval(self):
+        def main():
+            i = 1
+            a = compile('x+x+x+x+x+x', 'eval', 'eval')
+            b = {'x': 7}
+            while i < 1000:
+                y = eval(a,b,b)  # ID: eval
+                i += 1
+            return y
+
+        log = self.run(main)
+        assert log.result == 42
+        # the following assertion fails if the loop was cancelled due
+        # to "abort: vable escape"
+        assert len(log.loops_by_id("eval")) == 1
diff --git a/pypy/module/pypyjit/test_pypy_c/test_thread.py b/pypy/module/pypyjit/test_pypy_c/test_thread.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/pypyjit/test_pypy_c/test_thread.py
@@ -0,0 +1,28 @@
+from pypy.module.pypyjit.test_pypy_c.test_00_model import BaseTestPyPyC
+
+
+class TestThread(BaseTestPyPyC):
+    def test_simple(self):
+        def main(n):
+            import thread
+            def f():
+                i = 0
+                while i < n:
+                    i += 1
+                done.release()
+
+            done = thread.allocate_lock()
+            done.acquire()
+            thread.start_new_thread(f, ())
+            done.acquire()
+            return 0
+        log = self.run(main, [500])
+        assert round(log.result, 6) == round(main(500), 6)
+        loop, = log.loops_by_filename(self.filepath)
+        assert loop.match("""
+            i2 = int_lt(i0, i1)
+            guard_true(i2, descr=...)
+            i3 = int_add(i0, 1)
+            --THREAD-TICK--
+            jump(..., descr=<Loop0>)
+        """)
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -47,7 +47,7 @@
         'pypy_initial_path'     : 'state.pypy_initial_path',
 
         '_getframe'             : 'vm._getframe', 
-        '_current_frames'       : 'vm._current_frames', 
+        '_current_frames'       : 'currentframes._current_frames', 
         'setrecursionlimit'     : 'vm.setrecursionlimit', 
         'getrecursionlimit'     : 'vm.getrecursionlimit', 
         'setcheckinterval'      : 'vm.setcheckinterval', 
diff --git a/pypy/module/sys/currentframes.py b/pypy/module/sys/currentframes.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/sys/currentframes.py
@@ -0,0 +1,78 @@
+"""
+Implementation of the 'sys._current_frames()' routine.
+"""
+from pypy.interpreter import gateway
+
+app = gateway.applevel('''
+"NOT_RPYTHON"
+import __builtin__
+
+class fake_code(object):
+    co_name = "?"
+    co_filename = "?"
+    co_firstlineno = 0
+
+class fake_frame(object):
+    f_back = None
+    f_builtins = __builtin__.__dict__
+    f_code = fake_code()
+    f_exc_traceback = None
+    f_exc_type = None
+    f_exc_value = None
+    f_globals = {}
+    f_lasti = -1
+    f_lineno = 0
+    f_locals = {}
+    f_restricted = False
+    f_trace = None
+
+    def __init__(self, f):
+        if f is not None:
+            for name in ["f_builtins", "f_code", "f_globals", "f_lasti",
+                         "f_lineno"]:
+                setattr(self, name, getattr(f, name))
+''')
+
+def _current_frames(space):
+    """_current_frames() -> dictionary
+
+    Return a dictionary mapping each current thread T's thread id to T's
+    current stack "frame".  Functions in the traceback module can build the
+    call stack given such a frame.
+
+    Note that in PyPy this returns fake frame objects, to avoid a runtime
+    penalty everywhere with the JIT.  (So far these fake frames can be
+    completely uninformative depending on the JIT state; we could return
+    more with more efforts.)
+
+    This function should be used for specialized purposes only."""
+    w_result = space.newdict()
+    w_fake_frame = app.wget(space, "fake_frame")
+    w_fake_code  = app.wget(space, "fake_code")
+    ecs = space.threadlocals.getallvalues()
+    for thread_ident, ec in ecs.items():
+        vref = ec.topframeref
+        frames = []
+        while not vref.virtual:
+            f = vref()
+            if f is None:
+                break
+            frames.append(f)
+            vref = f.f_backref
+        else:
+            frames.append(None)
+        #
+        w_topframe = space.wrap(None)
+        w_prevframe = None
+        for f in frames:
+            w_nextframe = space.call_function(w_fake_frame, space.wrap(f))
+            if w_prevframe is None:
+                w_topframe = w_nextframe
+            else:
+                space.setattr(w_prevframe, space.wrap('f_back'), w_nextframe)
+            w_prevframe = w_nextframe
+        #
+        space.setitem(w_result,
+                      space.wrap(thread_ident),
+                      w_topframe)
+    return w_result
diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -556,7 +556,7 @@
             return sys._current_frames()
         frames = f()
         assert frames.keys() == [0]
-        assert frames[0].f_code.co_name == 'f'
+        assert frames[0].f_code.co_name in ('f', '?')
 
 class AppTestCurrentFramesWithThread(AppTestCurrentFrames):
     def setup_class(cls):
@@ -568,23 +568,25 @@
         import thread
 
         thread_id = thread.get_ident()
-        self.ready = False
         def other_thread():
-            self.ready = True
             print "thread started"
-            time.sleep(5)
+            lock2.release()
+            lock1.acquire()
+        lock1 = thread.allocate_lock()
+        lock2 = thread.allocate_lock()
+        lock1.acquire()
+        lock2.acquire()
         thread.start_new_thread(other_thread, ())
 
         def f():
-            for i in range(100):
-                if self.ready: break
-                time.sleep(0.1)
+            lock2.acquire()
             return sys._current_frames()
 
         frames = f()
+        lock1.release()
         thisframe = frames.pop(thread_id)
-        assert thisframe.f_code.co_name == 'f'
+        assert thisframe.f_code.co_name in ('f', '?')
 
         assert len(frames) == 1
         _, other_frame = frames.popitem()
-        assert other_frame.f_code.co_name == 'other_thread'
+        assert other_frame.f_code.co_name in ('other_thread', '?')
diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -45,25 +45,6 @@
     f.mark_as_escaped()
     return space.wrap(f)
 
-def _current_frames(space):
-    """_current_frames() -> dictionary
-
-    Return a dictionary mapping each current thread T's thread id to T's
-    current stack frame.
-
-    This function should be used for specialized purposes only."""
-    raise OperationError(space.w_NotImplementedError,
-        space.wrap("XXX sys._current_frames() incompatible with the JIT"))
-    w_result = space.newdict()
-    ecs = space.threadlocals.getallvalues()
-    for thread_ident, ec in ecs.items():
-        f = ec.gettopframe_nohidden()
-        f.mark_as_escaped()
-        space.setitem(w_result,
-                      space.wrap(thread_ident),
-                      space.wrap(f))
-    return w_result
-
 def setrecursionlimit(space, w_new_limit):
     """setrecursionlimit() sets the maximum number of nested calls that
 can occur before a RuntimeError is raised.  On PyPy the limit is
diff --git a/pypy/rlib/_jit_vref.py b/pypy/rlib/_jit_vref.py
--- a/pypy/rlib/_jit_vref.py
+++ b/pypy/rlib/_jit_vref.py
@@ -25,6 +25,10 @@
     def simple_call(self):
         return self.s_instance
 
+    def getattr(self, s_attr):
+        assert s_attr.const == 'virtual'
+        return annmodel.s_Bool
+
     def rtyper_makerepr(self, rtyper):
         if rtyper.type_system.name == 'lltypesystem':
             return vrefrepr
@@ -61,6 +65,13 @@
                              " prebuilt virtual_ref")
         return lltype.nullptr(OBJECTPTR.TO)
 
+    def rtype_getattr(self, hop):
+        s_attr = hop.args_s[1]
+        assert s_attr.const == 'virtual'
+        v = hop.inputarg(self, arg=0)
+        hop.exception_cannot_occur()
+        return hop.genop('jit_is_virtual', [v], resulttype = lltype.Bool)
+
 from pypy.rpython.ootypesystem.rclass import OBJECT
 
 class OOVRefRepr(VRefRepr):
diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -317,6 +317,12 @@
             raise InvalidVirtualRef
         return self._x
 
+    @property
+    def virtual(self):
+        """A property that is True if the vref contains a virtual that would
+        be forced by the '()' operator."""
+        return self._state == 'non-forced'
+
     def _finish(self):
         if self._state == 'non-forced':
             self._state = 'invalid'
diff --git a/pypy/rlib/rerased.py b/pypy/rlib/rerased.py
--- a/pypy/rlib/rerased.py
+++ b/pypy/rlib/rerased.py
@@ -218,15 +218,13 @@
         [v_value] = hop.inputargs(lltype.Signed)
         c_one = hop.inputconst(lltype.Signed, 1)
         hop.exception_is_here()
-        v2 = hop.genop('int_lshift_ovf', [v_value, c_one],
+        v2 = hop.genop('int_add_ovf', [v_value, v_value],
                        resulttype = lltype.Signed)
         v2p1 = hop.genop('int_add', [v2, c_one],
                          resulttype = lltype.Signed)
         v_instance = hop.genop('cast_int_to_ptr', [v2p1],
                                resulttype=self.lowleveltype)
-        v = hop.genop('cast_opaque_ptr', [v_instance],
-                      resulttype=self.lowleveltype)
-        return v
+        return v_instance
 
     def convert_const(self, value):
         if value._identity is _identity_for_ints:
@@ -266,10 +264,10 @@
         return hop.genop('int_rshift', [v2, c_one], resulttype=lltype.Signed)
 
     def rtype_erase_int(self, hop):
-        hop.exception_is_here()
         [v_value] = hop.inputargs(lltype.Signed)
         c_one = hop.inputconst(lltype.Signed, 1)
-        v2 = hop.genop('int_lshift_ovf', [v_value, c_one],
+        hop.exception_is_here()
+        v2 = hop.genop('int_add_ovf', [v_value, v_value],
                        resulttype = lltype.Signed)
         v2p1 = hop.genop('int_add', [v2, c_one],
                          resulttype = lltype.Signed)
diff --git a/pypy/rlib/test/test__jit_vref.py b/pypy/rlib/test/test__jit_vref.py
--- a/pypy/rlib/test/test__jit_vref.py
+++ b/pypy/rlib/test/test__jit_vref.py
@@ -27,10 +27,13 @@
     x1 = X()
     vref = virtual_ref(x1)
     assert vref._state == 'non-forced'
+    assert vref.virtual is True
     assert vref() is x1
     assert vref._state == 'forced'
+    assert vref.virtual is False
     virtual_ref_finish(vref, x1)
     assert vref._state == 'forced'
+    assert vref.virtual is False
     assert vref() is x1
 
 def test_direct_invalid():
@@ -135,6 +138,13 @@
         x = self.interpret(f, [])
         assert x == 42
 
+    def test_rtype_virtualattr(self):
+        def f():
+            vref = virtual_ref(X())
+            return vref.virtual
+        x = self.interpret(f, [])
+        assert x is False
+
 
 class TestLLtype(BaseTestVRef, LLRtypeMixin):
     OBJECTTYPE = OBJECTPTR
diff --git a/pypy/rpython/lltypesystem/ll2ctypes.py b/pypy/rpython/lltypesystem/ll2ctypes.py
--- a/pypy/rpython/lltypesystem/ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/ll2ctypes.py
@@ -658,6 +658,8 @@
         if T == llmemory.GCREF:
             if isinstance(llobj._obj, _llgcopaque):
                 return ctypes.c_void_p(llobj._obj.intval)
+            if isinstance(llobj._obj, int):    # tagged pointer
+                return ctypes.c_void_p(llobj._obj)
             container = llobj._obj.container
             T = lltype.Ptr(lltype.typeOf(container))
             # otherwise it came from integer and we want a c_void_p with
@@ -1268,6 +1270,7 @@
 class _llgcopaque(lltype._container):
     _TYPE = llmemory.GCREF.TO
     _name = "_llgcopaque"
+    _read_directly_intval = True     # for _ptr._cast_to_int()
 
     def __init__(self, void_p):
         if isinstance(void_p, (int, long)):
@@ -1299,11 +1302,6 @@
             return _opaque_objs[self.intval // 2]
         return force_cast(PTRTYPE, self.intval)
 
-##     def _cast_to_int(self):
-##         return self.intval
-
-##     def _cast_to_adr(self):
-##         return _lladdress(self.intval)
 
 def cast_adr_to_int(addr):
     if isinstance(addr, llmemory.fakeaddress):
diff --git a/pypy/rpython/lltypesystem/llmemory.py b/pypy/rpython/lltypesystem/llmemory.py
--- a/pypy/rpython/lltypesystem/llmemory.py
+++ b/pypy/rpython/lltypesystem/llmemory.py
@@ -498,6 +498,8 @@
 
     def _cast_to_int(self, symbolic=False):
         if self:
+            if isinstance(self.ptr._obj0, int):    # tagged integer
+                return self.ptr._obj0
             if symbolic:
                 return AddressAsInt(self)
             else:
diff --git a/pypy/rpython/lltypesystem/lloperation.py b/pypy/rpython/lltypesystem/lloperation.py
--- a/pypy/rpython/lltypesystem/lloperation.py
+++ b/pypy/rpython/lltypesystem/lloperation.py
@@ -428,6 +428,7 @@
     'jit_marker':           LLOp(),
     'jit_force_virtualizable':LLOp(canrun=True),
     'jit_force_virtual':    LLOp(canrun=True),
+    'jit_is_virtual':       LLOp(canrun=True),
     'jit_force_quasi_immutable': LLOp(canrun=True),
     'get_exception_addr':   LLOp(),
     'get_exc_value_addr':   LLOp(),
diff --git a/pypy/rpython/lltypesystem/lltype.py b/pypy/rpython/lltypesystem/lltype.py
--- a/pypy/rpython/lltypesystem/lltype.py
+++ b/pypy/rpython/lltypesystem/lltype.py
@@ -1360,6 +1360,8 @@
         obj = normalizeptr(self, check)._getobj(check)
         if isinstance(obj, int):
             return obj     # special case for cast_int_to_ptr() results put into opaques
+        if getattr(obj, '_read_directly_intval', False):
+            return obj.intval   # special case for _llgcopaque
         result = intmask(obj._getid())
         # assume that id() returns an addressish value which is
         # not zero and aligned to at least a multiple of 4
diff --git a/pypy/rpython/lltypesystem/opimpl.py b/pypy/rpython/lltypesystem/opimpl.py
--- a/pypy/rpython/lltypesystem/opimpl.py
+++ b/pypy/rpython/lltypesystem/opimpl.py
@@ -538,6 +538,9 @@
 def op_jit_force_virtual(x):
     return x
 
+def op_jit_is_virtual(x):
+    return False
+
 def op_jit_force_quasi_immutable(*args):
     pass
 
diff --git a/pypy/rpython/lltypesystem/rtagged.py b/pypy/rpython/lltypesystem/rtagged.py
--- a/pypy/rpython/lltypesystem/rtagged.py
+++ b/pypy/rpython/lltypesystem/rtagged.py
@@ -43,7 +43,7 @@
         v_value = hop.inputarg(lltype.Signed, arg=1)
         c_one = hop.inputconst(lltype.Signed, 1)
         hop.exception_is_here()
-        v2 = hop.genop('int_lshift_ovf', [v_value, c_one],
+        v2 = hop.genop('int_add_ovf', [v_value, v_value],
                        resulttype = lltype.Signed)
         v2p1 = hop.genop('int_add', [v2, c_one],
                          resulttype = lltype.Signed)
diff --git a/pypy/rpython/lltypesystem/test/test_ll2ctypes.py b/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
--- a/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
@@ -1123,6 +1123,9 @@
 
         #assert lltype.cast_ptr_to_int(ref1) == intval
 
+        x = rffi.cast(llmemory.GCREF, -17)
+        assert lltype.cast_ptr_to_int(x) == -17
+
     def test_ptr_truth(self):
         abc = rffi.cast(lltype.Ptr(lltype.FuncType([], lltype.Void)), 0)
         assert not abc
diff --git a/pypy/translator/c/funcgen.py b/pypy/translator/c/funcgen.py
--- a/pypy/translator/c/funcgen.py
+++ b/pypy/translator/c/funcgen.py
@@ -833,7 +833,7 @@
         return 'INSTRUMENT_COUNT(%s);' % counter_label
             
     def OP_IS_EARLY_CONSTANT(self, op):
-        return self.expr(op.result)  + ' = 0;' # Allways false
+        return '%s = 0; /* IS_EARLY_CONSTANT */' % (self.expr(op.result),)
 
     def OP_JIT_MARKER(self, op):
         return '/* JIT_MARKER %s */' % op
@@ -845,6 +845,9 @@
         return '%s = %s; /* JIT_FORCE_VIRTUAL */' % (self.expr(op.result),
                                                      self.expr(op.args[0]))
 
+    def OP_JIT_IS_VIRTUAL(self, op):
+        return '%s = 0; /* JIT_IS_VIRTUAL */' % (self.expr(op.result),)
+
     def OP_JIT_FORCE_QUASI_IMMUTABLE(self, op):
         return '/* JIT_FORCE_QUASI_IMMUTABLE %s */' % op
 
diff --git a/pypy/translator/platform/linux.py b/pypy/translator/platform/linux.py
--- a/pypy/translator/platform/linux.py
+++ b/pypy/translator/platform/linux.py
@@ -24,15 +24,14 @@
         return self._pkg_config("libffi", "--libs-only-L",
                                 ['/usr/lib/libffi'])
 
+    def library_dirs_for_libffi_a(self):
+        # places where we need to look for libffi.a
+        return self.library_dirs_for_libffi() + ['/usr/lib']
+
 
 class Linux(BaseLinux):
     shared_only = ()    # it seems that on 32-bit linux, compiling with -fPIC
                         # gives assembler that asmgcc is not happy about.
 
-    def library_dirs_for_libffi_a(self):
-        # places where we need to look for libffi.a
-        return self.library_dirs_for_libffi() + ['/usr/lib']
-
-
 class Linux64(BaseLinux):
     pass

From noreply at buildbot.pypy.org  Mon Oct 17 04:25:07 2011
From: noreply at buildbot.pypy.org (justinpeel)
Date: Mon, 17 Oct 2011 04:25:07 +0200 (CEST)
Subject: [pypy-commit] pypy rgc-mem-pressure: add memory pressure with a
 more fixed size for testing
Message-ID: <20111017022507.749F0820D7@wyvern.cs.uni-duesseldorf.de>

Author: Justin Peel <notmuchtotell at gmail.com>
Branch: rgc-mem-pressure
Changeset: r48105:dca685fec278
Date: 2011-10-16 20:24 -0600
http://bitbucket.org/pypy/pypy/changeset/dca685fec278/

Log:	add memory pressure with a more fixed size for testing

diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py
--- a/pypy/module/_hashlib/interp_hashlib.py
+++ b/pypy/module/_hashlib/interp_hashlib.py
@@ -29,9 +29,7 @@
                                  space.wrap("unknown hash function"))
         ctx = lltype.malloc(ropenssl.EVP_MD_CTX.TO, flavor='raw')
         ropenssl.EVP_DigestInit(ctx, digest)
-        rgc.add_memory_pressure(ropenssl.EVP_MD_CTX.TO.hints['getsize']() +
-                                ropenssl.EVP_MD.TO.hints['getsize']() +
-                                self._digest_size())
+        rgc.add_memory_pressure(184 + self._digest_size())
         self.ctx = ctx
 
     def __del__(self):

From noreply at buildbot.pypy.org  Mon Oct 17 04:25:09 2011
From: noreply at buildbot.pypy.org (justinpeel)
Date: Mon, 17 Oct 2011 04:25:09 +0200 (CEST)
Subject: [pypy-commit] pypy rgc-mem-pressure: merge in default
Message-ID: <20111017022509.4A4FE820D7@wyvern.cs.uni-duesseldorf.de>

Author: Justin Peel <notmuchtotell at gmail.com>
Branch: rgc-mem-pressure
Changeset: r48106:c26127bd0a39
Date: 2011-10-16 20:24 -0600
http://bitbucket.org/pypy/pypy/changeset/c26127bd0a39/

Log:	merge in default

diff --git a/lib-python/modified-2.7/httplib.py b/lib-python/modified-2.7/httplib.py
new file mode 100644
--- /dev/null
+++ b/lib-python/modified-2.7/httplib.py
@@ -0,0 +1,1377 @@
+"""HTTP/1.1 client library
+
+<intro stuff goes here>
+<other stuff, too>
+
+HTTPConnection goes through a number of "states", which define when a client
+may legally make another request or fetch the response for a particular
+request. This diagram details these state transitions:
+
+    (null)
+      |
+      | HTTPConnection()
+      v
+    Idle
+      |
+      | putrequest()
+      v
+    Request-started
+      |
+      | ( putheader() )*  endheaders()
+      v
+    Request-sent
+      |
+      | response = getresponse()
+      v
+    Unread-response   [Response-headers-read]
+      |\____________________
+      |                     |
+      | response.read()     | putrequest()
+      v                     v
+    Idle                  Req-started-unread-response
+                     ______/|
+                   /        |
+   response.read() |        | ( putheader() )*  endheaders()
+                   v        v
+       Request-started    Req-sent-unread-response
+                            |
+                            | response.read()
+                            v
+                          Request-sent
+
+This diagram presents the following rules:
+  -- a second request may not be started until {response-headers-read}
+  -- a response [object] cannot be retrieved until {request-sent}
+  -- there is no differentiation between an unread response body and a
+     partially read response body
+
+Note: this enforcement is applied by the HTTPConnection class. The
+      HTTPResponse class does not enforce this state machine, which
+      implies sophisticated clients may accelerate the request/response
+      pipeline. Caution should be taken, though: accelerating the states
+      beyond the above pattern may imply knowledge of the server's
+      connection-close behavior for certain requests. For example, it
+      is impossible to tell whether the server will close the connection
+      UNTIL the response headers have been read; this means that further
+      requests cannot be placed into the pipeline until it is known that
+      the server will NOT be closing the connection.
+
+Logical State                  __state            __response
+-------------                  -------            ----------
+Idle                           _CS_IDLE           None
+Request-started                _CS_REQ_STARTED    None
+Request-sent                   _CS_REQ_SENT       None
+Unread-response                _CS_IDLE           <response_class>
+Req-started-unread-response    _CS_REQ_STARTED    <response_class>
+Req-sent-unread-response       _CS_REQ_SENT       <response_class>
+"""
+
+from array import array
+import os
+import socket
+from sys import py3kwarning
+from urlparse import urlsplit
+import warnings
+with warnings.catch_warnings():
+    if py3kwarning:
+        warnings.filterwarnings("ignore", ".*mimetools has been removed",
+                                DeprecationWarning)
+    import mimetools
+
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from StringIO import StringIO
+
+__all__ = ["HTTP", "HTTPResponse", "HTTPConnection",
+           "HTTPException", "NotConnected", "UnknownProtocol",
+           "UnknownTransferEncoding", "UnimplementedFileMode",
+           "IncompleteRead", "InvalidURL", "ImproperConnectionState",
+           "CannotSendRequest", "CannotSendHeader", "ResponseNotReady",
+           "BadStatusLine", "error", "responses"]
+
+HTTP_PORT = 80
+HTTPS_PORT = 443
+
+_UNKNOWN = 'UNKNOWN'
+
+# connection states
+_CS_IDLE = 'Idle'
+_CS_REQ_STARTED = 'Request-started'
+_CS_REQ_SENT = 'Request-sent'
+
+# status codes
+# informational
+CONTINUE = 100
+SWITCHING_PROTOCOLS = 101
+PROCESSING = 102
+
+# successful
+OK = 200
+CREATED = 201
+ACCEPTED = 202
+NON_AUTHORITATIVE_INFORMATION = 203
+NO_CONTENT = 204
+RESET_CONTENT = 205
+PARTIAL_CONTENT = 206
+MULTI_STATUS = 207
+IM_USED = 226
+
+# redirection
+MULTIPLE_CHOICES = 300
+MOVED_PERMANENTLY = 301
+FOUND = 302
+SEE_OTHER = 303
+NOT_MODIFIED = 304
+USE_PROXY = 305
+TEMPORARY_REDIRECT = 307
+
+# client error
+BAD_REQUEST = 400
+UNAUTHORIZED = 401
+PAYMENT_REQUIRED = 402
+FORBIDDEN = 403
+NOT_FOUND = 404
+METHOD_NOT_ALLOWED = 405
+NOT_ACCEPTABLE = 406
+PROXY_AUTHENTICATION_REQUIRED = 407
+REQUEST_TIMEOUT = 408
+CONFLICT = 409
+GONE = 410
+LENGTH_REQUIRED = 411
+PRECONDITION_FAILED = 412
+REQUEST_ENTITY_TOO_LARGE = 413
+REQUEST_URI_TOO_LONG = 414
+UNSUPPORTED_MEDIA_TYPE = 415
+REQUESTED_RANGE_NOT_SATISFIABLE = 416
+EXPECTATION_FAILED = 417
+UNPROCESSABLE_ENTITY = 422
+LOCKED = 423
+FAILED_DEPENDENCY = 424
+UPGRADE_REQUIRED = 426
+
+# server error
+INTERNAL_SERVER_ERROR = 500
+NOT_IMPLEMENTED = 501
+BAD_GATEWAY = 502
+SERVICE_UNAVAILABLE = 503
+GATEWAY_TIMEOUT = 504
+HTTP_VERSION_NOT_SUPPORTED = 505
+INSUFFICIENT_STORAGE = 507
+NOT_EXTENDED = 510
+
+# Mapping status codes to official W3C names
+responses = {
+    100: 'Continue',
+    101: 'Switching Protocols',
+
+    200: 'OK',
+    201: 'Created',
+    202: 'Accepted',
+    203: 'Non-Authoritative Information',
+    204: 'No Content',
+    205: 'Reset Content',
+    206: 'Partial Content',
+
+    300: 'Multiple Choices',
+    301: 'Moved Permanently',
+    302: 'Found',
+    303: 'See Other',
+    304: 'Not Modified',
+    305: 'Use Proxy',
+    306: '(Unused)',
+    307: 'Temporary Redirect',
+
+    400: 'Bad Request',
+    401: 'Unauthorized',
+    402: 'Payment Required',
+    403: 'Forbidden',
+    404: 'Not Found',
+    405: 'Method Not Allowed',
+    406: 'Not Acceptable',
+    407: 'Proxy Authentication Required',
+    408: 'Request Timeout',
+    409: 'Conflict',
+    410: 'Gone',
+    411: 'Length Required',
+    412: 'Precondition Failed',
+    413: 'Request Entity Too Large',
+    414: 'Request-URI Too Long',
+    415: 'Unsupported Media Type',
+    416: 'Requested Range Not Satisfiable',
+    417: 'Expectation Failed',
+
+    500: 'Internal Server Error',
+    501: 'Not Implemented',
+    502: 'Bad Gateway',
+    503: 'Service Unavailable',
+    504: 'Gateway Timeout',
+    505: 'HTTP Version Not Supported',
+}
+
+# maximal amount of data to read at one time in _safe_read
+MAXAMOUNT = 1048576
+
+class HTTPMessage(mimetools.Message):
+
+    def addheader(self, key, value):
+        """Add header for field key handling repeats."""
+        prev = self.dict.get(key)
+        if prev is None:
+            self.dict[key] = value
+        else:
+            combined = ", ".join((prev, value))
+            self.dict[key] = combined
+
+    def addcontinue(self, key, more):
+        """Add more field data from a continuation line."""
+        prev = self.dict[key]
+        self.dict[key] = prev + "\n " + more
+
+    def readheaders(self):
+        """Read header lines.
+
+        Read header lines up to the entirely blank line that terminates them.
+        The (normally blank) line that ends the headers is skipped, but not
+        included in the returned list.  If a non-header line ends the headers,
+        (which is an error), an attempt is made to backspace over it; it is
+        never included in the returned list.
+
+        The variable self.status is set to the empty string if all went well,
+        otherwise it is an error message.  The variable self.headers is a
+        completely uninterpreted list of lines contained in the header (so
+        printing them will reproduce the header exactly as it appears in the
+        file).
+
+        If multiple header fields with the same name occur, they are combined
+        according to the rules in RFC 2616 sec 4.2:
+
+        Appending each subsequent field-value to the first, each separated
+        by a comma. The order in which header fields with the same field-name
+        are received is significant to the interpretation of the combined
+        field value.
+        """
+        # XXX The implementation overrides the readheaders() method of
+        # rfc822.Message.  The base class design isn't amenable to
+        # customized behavior here so the method here is a copy of the
+        # base class code with a few small changes.
+
+        self.dict = {}
+        self.unixfrom = ''
+        self.headers = hlist = []
+        self.status = ''
+        headerseen = ""
+        firstline = 1
+        startofline = unread = tell = None
+        if hasattr(self.fp, 'unread'):
+            unread = self.fp.unread
+        elif self.seekable:
+            tell = self.fp.tell
+        while True:
+            if tell:
+                try:
+                    startofline = tell()
+                except IOError:
+                    startofline = tell = None
+                    self.seekable = 0
+            line = self.fp.readline()
+            if not line:
+                self.status = 'EOF in headers'
+                break
+            # Skip unix From name time lines
+            if firstline and line.startswith('From '):
+                self.unixfrom = self.unixfrom + line
+                continue
+            firstline = 0
+            if headerseen and line[0] in ' \t':
+                # XXX Not sure if continuation lines are handled properly
+                # for http and/or for repeating headers
+                # It's a continuation line.
+                hlist.append(line)
+                self.addcontinue(headerseen, line.strip())
+                continue
+            elif self.iscomment(line):
+                # It's a comment.  Ignore it.
+                continue
+            elif self.islast(line):
+                # Note! No pushback here!  The delimiter line gets eaten.
+                break
+            headerseen = self.isheader(line)
+            if headerseen:
+                # It's a legal header line, save it.
+                hlist.append(line)
+                self.addheader(headerseen, line[len(headerseen)+1:].strip())
+                continue
+            else:
+                # It's not a header line; throw it back and stop here.
+                if not self.dict:
+                    self.status = 'No headers'
+                else:
+                    self.status = 'Non-header line where header expected'
+                # Try to undo the read.
+                if unread:
+                    unread(line)
+                elif tell:
+                    self.fp.seek(startofline)
+                else:
+                    self.status = self.status + '; bad seek'
+                break
+
+class HTTPResponse:
+
+    # strict: If true, raise BadStatusLine if the status line can't be
+    # parsed as a valid HTTP/1.0 or 1.1 status line.  By default it is
+    # false because it prevents clients from talking to HTTP/0.9
+    # servers.  Note that a response with a sufficiently corrupted
+    # status line will look like an HTTP/0.9 response.
+
+    # See RFC 2616 sec 19.6 and RFC 1945 sec 6 for details.
+
+    def __init__(self, sock, debuglevel=0, strict=0, method=None, buffering=False):
+        if buffering:
+            # The caller won't be using any sock.recv() calls, so buffering
+            # is fine and recommended for performance.
+            self.fp = sock.makefile('rb')
+        else:
+            # The buffer size is specified as zero, because the headers of
+            # the response are read with readline().  If the reads were
+            # buffered the readline() calls could consume some of the
+            # response, which make be read via a recv() on the underlying
+            # socket.
+            self.fp = sock.makefile('rb', 0)
+        self.debuglevel = debuglevel
+        self.strict = strict
+        self._method = method
+
+        self.msg = None
+
+        # from the Status-Line of the response
+        self.version = _UNKNOWN # HTTP-Version
+        self.status = _UNKNOWN  # Status-Code
+        self.reason = _UNKNOWN  # Reason-Phrase
+
+        self.chunked = _UNKNOWN         # is "chunked" being used?
+        self.chunk_left = _UNKNOWN      # bytes left to read in current chunk
+        self.length = _UNKNOWN          # number of bytes left in response
+        self.will_close = _UNKNOWN      # conn will close at end of response
+
+    def _read_status(self):
+        # Initialize with Simple-Response defaults
+        line = self.fp.readline()
+        if self.debuglevel > 0:
+            print "reply:", repr(line)
+        if not line:
+            # Presumably, the server closed the connection before
+            # sending a valid response.
+            raise BadStatusLine(line)
+        try:
+            [version, status, reason] = line.split(None, 2)
+        except ValueError:
+            try:
+                [version, status] = line.split(None, 1)
+                reason = ""
+            except ValueError:
+                # empty version will cause next test to fail and status
+                # will be treated as 0.9 response.
+                version = ""
+        if not version.startswith('HTTP/'):
+            if self.strict:
+                self.close()
+                raise BadStatusLine(line)
+            else:
+                # assume it's a Simple-Response from an 0.9 server
+                self.fp = LineAndFileWrapper(line, self.fp)
+                return "HTTP/0.9", 200, ""
+
+        # The status code is a three-digit number
+        try:
+            status = int(status)
+            if status < 100 or status > 999:
+                raise BadStatusLine(line)
+        except ValueError:
+            raise BadStatusLine(line)
+        return version, status, reason
+
+    def begin(self):
+        if self.msg is not None:
+            # we've already started reading the response
+            return
+
+        # read until we get a non-100 response
+        while True:
+            version, status, reason = self._read_status()
+            if status != CONTINUE:
+                break
+            # skip the header from the 100 response
+            while True:
+                skip = self.fp.readline().strip()
+                if not skip:
+                    break
+                if self.debuglevel > 0:
+                    print "header:", skip
+
+        self.status = status
+        self.reason = reason.strip()
+        if version == 'HTTP/1.0':
+            self.version = 10
+        elif version.startswith('HTTP/1.'):
+            self.version = 11   # use HTTP/1.1 code for HTTP/1.x where x>=1
+        elif version == 'HTTP/0.9':
+            self.version = 9
+        else:
+            raise UnknownProtocol(version)
+
+        if self.version == 9:
+            self.length = None
+            self.chunked = 0
+            self.will_close = 1
+            self.msg = HTTPMessage(StringIO())
+            return
+
+        self.msg = HTTPMessage(self.fp, 0)
+        if self.debuglevel > 0:
+            for hdr in self.msg.headers:
+                print "header:", hdr,
+
+        # don't let the msg keep an fp
+        self.msg.fp = None
+
+        # are we using the chunked-style of transfer encoding?
+        tr_enc = self.msg.getheader('transfer-encoding')
+        if tr_enc and tr_enc.lower() == "chunked":
+            self.chunked = 1
+            self.chunk_left = None
+        else:
+            self.chunked = 0
+
+        # will the connection close at the end of the response?
+        self.will_close = self._check_close()
+
+        # do we have a Content-Length?
+        # NOTE: RFC 2616, S4.4, #3 says we ignore this if tr_enc is "chunked"
+        length = self.msg.getheader('content-length')
+        if length and not self.chunked:
+            try:
+                self.length = int(length)
+            except ValueError:
+                self.length = None
+            else:
+                if self.length < 0:  # ignore nonsensical negative lengths
+                    self.length = None
+        else:
+            self.length = None
+
+        # does the body have a fixed length? (of zero)
+        if (status == NO_CONTENT or status == NOT_MODIFIED or
+            100 <= status < 200 or      # 1xx codes
+            self._method == 'HEAD'):
+            self.length = 0
+
+        # if the connection remains open, and we aren't using chunked, and
+        # a content-length was not provided, then assume that the connection
+        # WILL close.
+        if not self.will_close and \
+           not self.chunked and \
+           self.length is None:
+            self.will_close = 1
+
+    def _check_close(self):
+        conn = self.msg.getheader('connection')
+        if self.version == 11:
+            # An HTTP/1.1 proxy is assumed to stay open unless
+            # explicitly closed.
+            conn = self.msg.getheader('connection')
+            if conn and "close" in conn.lower():
+                return True
+            return False
+
+        # Some HTTP/1.0 implementations have support for persistent
+        # connections, using rules different than HTTP/1.1.
+
+        # For older HTTP, Keep-Alive indicates persistent connection.
+        if self.msg.getheader('keep-alive'):
+            return False
+
+        # At least Akamai returns a "Connection: Keep-Alive" header,
+        # which was supposed to be sent by the client.
+        if conn and "keep-alive" in conn.lower():
+            return False
+
+        # Proxy-Connection is a netscape hack.
+        pconn = self.msg.getheader('proxy-connection')
+        if pconn and "keep-alive" in pconn.lower():
+            return False
+
+        # otherwise, assume it will close
+        return True
+
+    def close(self):
+        if self.fp:
+            self.fp.close()
+            self.fp = None
+
+    def isclosed(self):
+        # NOTE: it is possible that we will not ever call self.close(). This
+        #       case occurs when will_close is TRUE, length is None, and we
+        #       read up to the last byte, but NOT past it.
+        #
+        # IMPLIES: if will_close is FALSE, then self.close() will ALWAYS be
+        #          called, meaning self.isclosed() is meaningful.
+        return self.fp is None
+
+    # XXX It would be nice to have readline and __iter__ for this, too.
+
+    def read(self, amt=None):
+        if self.fp is None:
+            return ''
+
+        if self._method == 'HEAD':
+            self.close()
+            return ''
+
+        if self.chunked:
+            return self._read_chunked(amt)
+
+        if amt is None:
+            # unbounded read
+            if self.length is None:
+                s = self.fp.read()
+            else:
+                s = self._safe_read(self.length)
+                self.length = 0
+            self.close()        # we read everything
+            return s
+
+        if self.length is not None:
+            if amt > self.length:
+                # clip the read to the "end of response"
+                amt = self.length
+
+        # we do not use _safe_read() here because this may be a .will_close
+        # connection, and the user is reading more bytes than will be provided
+        # (for example, reading in 1k chunks)
+        s = self.fp.read(amt)
+        if self.length is not None:
+            self.length -= len(s)
+            if not self.length:
+                self.close()
+        return s
+
+    def _read_chunked(self, amt):
+        assert self.chunked != _UNKNOWN
+        chunk_left = self.chunk_left
+        value = []
+        while True:
+            if chunk_left is None:
+                line = self.fp.readline()
+                i = line.find(';')
+                if i >= 0:
+                    line = line[:i] # strip chunk-extensions
+                try:
+                    chunk_left = int(line, 16)
+                except ValueError:
+                    # close the connection as protocol synchronisation is
+                    # probably lost
+                    self.close()
+                    raise IncompleteRead(''.join(value))
+                if chunk_left == 0:
+                    break
+            if amt is None:
+                value.append(self._safe_read(chunk_left))
+            elif amt < chunk_left:
+                value.append(self._safe_read(amt))
+                self.chunk_left = chunk_left - amt
+                return ''.join(value)
+            elif amt == chunk_left:
+                value.append(self._safe_read(amt))
+                self._safe_read(2)  # toss the CRLF at the end of the chunk
+                self.chunk_left = None
+                return ''.join(value)
+            else:
+                value.append(self._safe_read(chunk_left))
+                amt -= chunk_left
+
+            # we read the whole chunk, get another
+            self._safe_read(2)      # toss the CRLF at the end of the chunk
+            chunk_left = None
+
+        # read and discard trailer up to the CRLF terminator
+        ### note: we shouldn't have any trailers!
+        while True:
+            line = self.fp.readline()
+            if not line:
+                # a vanishingly small number of sites EOF without
+                # sending the trailer
+                break
+            if line == '\r\n':
+                break
+
+        # we read everything; close the "file"
+        self.close()
+
+        return ''.join(value)
+
+    def _safe_read(self, amt):
+        """Read the number of bytes requested, compensating for partial reads.
+
+        Normally, we have a blocking socket, but a read() can be interrupted
+        by a signal (resulting in a partial read).
+
+        Note that we cannot distinguish between EOF and an interrupt when zero
+        bytes have been read. IncompleteRead() will be raised in this
+        situation.
+
+        This function should be used when <amt> bytes "should" be present for
+        reading. If the bytes are truly not available (due to EOF), then the
+        IncompleteRead exception can be used to detect the problem.
+        """
+        # NOTE(gps): As of svn r74426 socket._fileobject.read(x) will never
+        # return less than x bytes unless EOF is encountered.  It now handles
+        # signal interruptions (socket.error EINTR) internally.  This code
+        # never caught that exception anyways.  It seems largely pointless.
+        # self.fp.read(amt) will work fine.
+        s = []
+        while amt > 0:
+            chunk = self.fp.read(min(amt, MAXAMOUNT))
+            if not chunk:
+                raise IncompleteRead(''.join(s), amt)
+            s.append(chunk)
+            amt -= len(chunk)
+        return ''.join(s)
+
+    def fileno(self):
+        return self.fp.fileno()
+
+    def getheader(self, name, default=None):
+        if self.msg is None:
+            raise ResponseNotReady()
+        return self.msg.getheader(name, default)
+
+    def getheaders(self):
+        """Return list of (header, value) tuples."""
+        if self.msg is None:
+            raise ResponseNotReady()
+        return self.msg.items()
+
+
+class HTTPConnection:
+
+    _http_vsn = 11
+    _http_vsn_str = 'HTTP/1.1'
+
+    response_class = HTTPResponse
+    default_port = HTTP_PORT
+    auto_open = 1
+    debuglevel = 0
+    strict = 0
+
+    def __init__(self, host, port=None, strict=None,
+                 timeout=socket._GLOBAL_DEFAULT_TIMEOUT, source_address=None):
+        self.timeout = timeout
+        self.source_address = source_address
+        self.sock = None
+        self._buffer = []
+        self.__response = None
+        self.__state = _CS_IDLE
+        self._method = None
+        self._tunnel_host = None
+        self._tunnel_port = None
+        self._tunnel_headers = {}
+
+        self._set_hostport(host, port)
+        if strict is not None:
+            self.strict = strict
+
+    def set_tunnel(self, host, port=None, headers=None):
+        """ Sets up the host and the port for the HTTP CONNECT Tunnelling.
+
+        The headers argument should be a mapping of extra HTTP headers
+        to send with the CONNECT request.
+        """
+        self._tunnel_host = host
+        self._tunnel_port = port
+        if headers:
+            self._tunnel_headers = headers
+        else:
+            self._tunnel_headers.clear()
+
+    def _set_hostport(self, host, port):
+        if port is None:
+            i = host.rfind(':')
+            j = host.rfind(']')         # ipv6 addresses have [...]
+            if i > j:
+                try:
+                    port = int(host[i+1:])
+                except ValueError:
+                    raise InvalidURL("nonnumeric port: '%s'" % host[i+1:])
+                host = host[:i]
+            else:
+                port = self.default_port
+            if host and host[0] == '[' and host[-1] == ']':
+                host = host[1:-1]
+        self.host = host
+        self.port = port
+
+    def set_debuglevel(self, level):
+        self.debuglevel = level
+
+    def _tunnel(self):
+        self._set_hostport(self._tunnel_host, self._tunnel_port)
+        self.send("CONNECT %s:%d HTTP/1.0\r\n" % (self.host, self.port))
+        for header, value in self._tunnel_headers.iteritems():
+            self.send("%s: %s\r\n" % (header, value))
+        self.send("\r\n")
+        response = self.response_class(self.sock, strict = self.strict,
+                                       method = self._method)
+        (version, code, message) = response._read_status()
+
+        if code != 200:
+            self.close()
+            raise socket.error("Tunnel connection failed: %d %s" % (code,
+                                                                    message.strip()))
+        while True:
+            line = response.fp.readline()
+            if line == '\r\n': break
+
+
+    def connect(self):
+        """Connect to the host and port specified in __init__."""
+        self.sock = socket.create_connection((self.host,self.port),
+                                             self.timeout, self.source_address)
+
+        if self._tunnel_host:
+            self._tunnel()
+
+    def close(self):
+        """Close the connection to the HTTP server."""
+        if self.sock:
+            self.sock.close()   # close it manually... there may be other refs
+            self.sock = None
+        if self.__response:
+            self.__response.close()
+            self.__response = None
+        self.__state = _CS_IDLE
+
+    def send(self, data):
+        """Send `data' to the server."""
+        if self.sock is None:
+            if self.auto_open:
+                self.connect()
+            else:
+                raise NotConnected()
+
+        if self.debuglevel > 0:
+            print "send:", repr(data)
+        blocksize = 8192
+        if hasattr(data,'read') and not isinstance(data, array):
+            if self.debuglevel > 0: print "sendIng a read()able"
+            datablock = data.read(blocksize)
+            while datablock:
+                self.sock.sendall(datablock)
+                datablock = data.read(blocksize)
+        else:
+            self.sock.sendall(data)
+
+    def _output(self, s):
+        """Add a line of output to the current request buffer.
+
+        Assumes that the line does *not* end with \\r\\n.
+        """
+        self._buffer.append(s)
+
+    def _send_output(self, message_body=None):
+        """Send the currently buffered request and clear the buffer.
+
+        Appends an extra \\r\\n to the buffer.
+        A message_body may be specified, to be appended to the request.
+        """
+        self._buffer.extend(("", ""))
+        msg = "\r\n".join(self._buffer)
+        del self._buffer[:]
+        # If msg and message_body are sent in a single send() call,
+        # it will avoid performance problems caused by the interaction
+        # between delayed ack and the Nagle algorithim.
+        if isinstance(message_body, str):
+            msg += message_body
+            message_body = None
+        self.send(msg)
+        if message_body is not None:
+            #message_body was not a string (i.e. it is a file) and
+            #we must run the risk of Nagle
+            self.send(message_body)
+
+    def putrequest(self, method, url, skip_host=0, skip_accept_encoding=0):
+        """Send a request to the server.
+
+        `method' specifies an HTTP request method, e.g. 'GET'.
+        `url' specifies the object being requested, e.g. '/index.html'.
+        `skip_host' if True does not add automatically a 'Host:' header
+        `skip_accept_encoding' if True does not add automatically an
+           'Accept-Encoding:' header
+        """
+
+        # if a prior response has been completed, then forget about it.
+        if self.__response and self.__response.isclosed():
+            self.__response = None
+
+
+        # in certain cases, we cannot issue another request on this connection.
+        # this occurs when:
+        #   1) we are in the process of sending a request.   (_CS_REQ_STARTED)
+        #   2) a response to a previous request has signalled that it is going
+        #      to close the connection upon completion.
+        #   3) the headers for the previous response have not been read, thus
+        #      we cannot determine whether point (2) is true.   (_CS_REQ_SENT)
+        #
+        # if there is no prior response, then we can request at will.
+        #
+        # if point (2) is true, then we will have passed the socket to the
+        # response (effectively meaning, "there is no prior response"), and
+        # will open a new one when a new request is made.
+        #
+        # Note: if a prior response exists, then we *can* start a new request.
+        #       We are not allowed to begin fetching the response to this new
+        #       request, however, until that prior response is complete.
+        #
+        if self.__state == _CS_IDLE:
+            self.__state = _CS_REQ_STARTED
+        else:
+            raise CannotSendRequest()
+
+        # Save the method we use, we need it later in the response phase
+        self._method = method
+        if not url:
+            url = '/'
+        hdr = '%s %s %s' % (method, url, self._http_vsn_str)
+
+        self._output(hdr)
+
+        if self._http_vsn == 11:
+            # Issue some standard headers for better HTTP/1.1 compliance
+
+            if not skip_host:
+                # this header is issued *only* for HTTP/1.1
+                # connections. more specifically, this means it is
+                # only issued when the client uses the new
+                # HTTPConnection() class. backwards-compat clients
+                # will be using HTTP/1.0 and those clients may be
+                # issuing this header themselves. we should NOT issue
+                # it twice; some web servers (such as Apache) barf
+                # when they see two Host: headers
+
+                # If we need a non-standard port,include it in the
+                # header.  If the request is going through a proxy,
+                # but the host of the actual URL, not the host of the
+                # proxy.
+
+                netloc = ''
+                if url.startswith('http'):
+                    nil, netloc, nil, nil, nil = urlsplit(url)
+
+                if netloc:
+                    try:
+                        netloc_enc = netloc.encode("ascii")
+                    except UnicodeEncodeError:
+                        netloc_enc = netloc.encode("idna")
+                    self.putheader('Host', netloc_enc)
+                else:
+                    try:
+                        host_enc = self.host.encode("ascii")
+                    except UnicodeEncodeError:
+                        host_enc = self.host.encode("idna")
+                    # Wrap the IPv6 Host Header with [] (RFC 2732)
+                    if host_enc.find(':') >= 0:
+                        host_enc = "[" + host_enc + "]"
+                    if self.port == self.default_port:
+                        self.putheader('Host', host_enc)
+                    else:
+                        self.putheader('Host', "%s:%s" % (host_enc, self.port))
+
+            # note: we are assuming that clients will not attempt to set these
+            #       headers since *this* library must deal with the
+            #       consequences. this also means that when the supporting
+            #       libraries are updated to recognize other forms, then this
+            #       code should be changed (removed or updated).
+
+            # we only want a Content-Encoding of "identity" since we don't
+            # support encodings such as x-gzip or x-deflate.
+            if not skip_accept_encoding:
+                self.putheader('Accept-Encoding', 'identity')
+
+            # we can accept "chunked" Transfer-Encodings, but no others
+            # NOTE: no TE header implies *only* "chunked"
+            #self.putheader('TE', 'chunked')
+
+            # if TE is supplied in the header, then it must appear in a
+            # Connection header.
+            #self.putheader('Connection', 'TE')
+
+        else:
+            # For HTTP/1.0, the server will assume "not chunked"
+            pass
+
+    def putheader(self, header, *values):
+        """Send a request header line to the server.
+
+        For example: h.putheader('Accept', 'text/html')
+        """
+        if self.__state != _CS_REQ_STARTED:
+            raise CannotSendHeader()
+
+        hdr = '%s: %s' % (header, '\r\n\t'.join([str(v) for v in values]))
+        self._output(hdr)
+
+    def endheaders(self, message_body=None):
+        """Indicate that the last header line has been sent to the server.
+
+        This method sends the request to the server.  The optional
+        message_body argument can be used to pass message body
+        associated with the request.  The message body will be sent in
+        the same packet as the message headers if possible.  The
+        message_body should be a string.
+        """
+        if self.__state == _CS_REQ_STARTED:
+            self.__state = _CS_REQ_SENT
+        else:
+            raise CannotSendHeader()
+        self._send_output(message_body)
+
+    def request(self, method, url, body=None, headers={}):
+        """Send a complete request to the server."""
+        self._send_request(method, url, body, headers)
+
+    def _set_content_length(self, body):
+        # Set the content-length based on the body.
+        thelen = None
+        try:
+            thelen = str(len(body))
+        except TypeError, te:
+            # If this is a file-like object, try to
+            # fstat its file descriptor
+            try:
+                thelen = str(os.fstat(body.fileno()).st_size)
+            except (AttributeError, OSError):
+                # Don't send a length if this failed
+                if self.debuglevel > 0: print "Cannot stat!!"
+
+        if thelen is not None:
+            self.putheader('Content-Length', thelen)
+
+    def _send_request(self, method, url, body, headers):
+        # Honor explicitly requested Host: and Accept-Encoding: headers.
+        header_names = dict.fromkeys([k.lower() for k in headers])
+        skips = {}
+        if 'host' in header_names:
+            skips['skip_host'] = 1
+        if 'accept-encoding' in header_names:
+            skips['skip_accept_encoding'] = 1
+
+        self.putrequest(method, url, **skips)
+
+        if body and ('content-length' not in header_names):
+            self._set_content_length(body)
+        for hdr, value in headers.iteritems():
+            self.putheader(hdr, value)
+        self.endheaders(body)
+
+    def getresponse(self, buffering=False):
+        "Get the response from the server."
+
+        # if a prior response has been completed, then forget about it.
+        if self.__response and self.__response.isclosed():
+            self.__response = None
+
+        #
+        # if a prior response exists, then it must be completed (otherwise, we
+        # cannot read this response's header to determine the connection-close
+        # behavior)
+        #
+        # note: if a prior response existed, but was connection-close, then the
+        # socket and response were made independent of this HTTPConnection
+        # object since a new request requires that we open a whole new
+        # connection
+        #
+        # this means the prior response had one of two states:
+        #   1) will_close: this connection was reset and the prior socket and
+        #                  response operate independently
+        #   2) persistent: the response was retained and we await its
+        #                  isclosed() status to become true.
+        #
+        if self.__state != _CS_REQ_SENT or self.__response:
+            raise ResponseNotReady()
+
+        args = (self.sock,)
+        kwds = {"strict":self.strict, "method":self._method}
+        if self.debuglevel > 0:
+            args += (self.debuglevel,)
+        if buffering:
+            #only add this keyword if non-default, for compatibility with
+            #other response_classes.
+            kwds["buffering"] = True;
+        response = self.response_class(*args, **kwds)
+
+        try:
+            response.begin()
+        except:
+            response.close()
+            raise
+        assert response.will_close != _UNKNOWN
+        self.__state = _CS_IDLE
+
+        if response.will_close:
+            # this effectively passes the connection to the response
+            self.close()
+        else:
+            # remember this, so we can tell when it is complete
+            self.__response = response
+
+        return response
+
+
+class HTTP:
+    "Compatibility class with httplib.py from 1.5."
+
+    _http_vsn = 10
+    _http_vsn_str = 'HTTP/1.0'
+
+    debuglevel = 0
+
+    _connection_class = HTTPConnection
+
+    def __init__(self, host='', port=None, strict=None):
+        "Provide a default host, since the superclass requires one."
+
+        # some joker passed 0 explicitly, meaning default port
+        if port == 0:
+            port = None
+
+        # Note that we may pass an empty string as the host; this will throw
+        # an error when we attempt to connect. Presumably, the client code
+        # will call connect before then, with a proper host.
+        self._setup(self._connection_class(host, port, strict))
+
+    def _setup(self, conn):
+        self._conn = conn
+
+        # set up delegation to flesh out interface
+        self.send = conn.send
+        self.putrequest = conn.putrequest
+        self.putheader = conn.putheader
+        self.endheaders = conn.endheaders
+        self.set_debuglevel = conn.set_debuglevel
+
+        conn._http_vsn = self._http_vsn
+        conn._http_vsn_str = self._http_vsn_str
+
+        self.file = None
+
+    def connect(self, host=None, port=None):
+        "Accept arguments to set the host/port, since the superclass doesn't."
+
+        if host is not None:
+            self._conn._set_hostport(host, port)
+        self._conn.connect()
+
+    def getfile(self):
+        "Provide a getfile, since the superclass' does not use this concept."
+        return self.file
+
+    def getreply(self, buffering=False):
+        """Compat definition since superclass does not define it.
+
+        Returns a tuple consisting of:
+        - server status code (e.g. '200' if all goes well)
+        - server "reason" corresponding to status code
+        - any RFC822 headers in the response from the server
+        """
+        try:
+            if not buffering:
+                response = self._conn.getresponse()
+            else:
+                #only add this keyword if non-default for compatibility
+                #with other connection classes
+                response = self._conn.getresponse(buffering)
+        except BadStatusLine, e:
+            ### hmm. if getresponse() ever closes the socket on a bad request,
+            ### then we are going to have problems with self.sock
+
+            ### should we keep this behavior? do people use it?
+            # keep the socket open (as a file), and return it
+            self.file = self._conn.sock.makefile('rb', 0)
+
+            # close our socket -- we want to restart after any protocol error
+            self.close()
+
+            self.headers = None
+            return -1, e.line, None
+
+        self.headers = response.msg
+        self.file = response.fp
+        return response.status, response.reason, response.msg
+
+    def close(self):
+        self._conn.close()
+
+        # note that self.file == response.fp, which gets closed by the
+        # superclass. just clear the object ref here.
+        ### hmm. messy. if status==-1, then self.file is owned by us.
+        ### well... we aren't explicitly closing, but losing this ref will
+        ### do it
+        self.file = None
+
+try:
+    import ssl
+except ImportError:
+    pass
+else:
+    class HTTPSConnection(HTTPConnection):
+        "This class allows communication via SSL."
+
+        default_port = HTTPS_PORT
+
+        def __init__(self, host, port=None, key_file=None, cert_file=None,
+                     strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
+                     source_address=None):
+            HTTPConnection.__init__(self, host, port, strict, timeout,
+                                    source_address)
+            self.key_file = key_file
+            self.cert_file = cert_file
+
+        def connect(self):
+            "Connect to a host on a given (SSL) port."
+
+            sock = socket.create_connection((self.host, self.port),
+                                            self.timeout, self.source_address)
+            if self._tunnel_host:
+                self.sock = sock
+                self._tunnel()
+            self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file)
+
+    __all__.append("HTTPSConnection")
+
+    class HTTPS(HTTP):
+        """Compatibility with 1.5 httplib interface
+
+        Python 1.5.2 did not have an HTTPS class, but it defined an
+        interface for sending http requests that is also useful for
+        https.
+        """
+
+        _connection_class = HTTPSConnection
+
+        def __init__(self, host='', port=None, key_file=None, cert_file=None,
+                     strict=None):
+            # provide a default host, pass the X509 cert info
+
+            # urf. compensate for bad input.
+            if port == 0:
+                port = None
+            self._setup(self._connection_class(host, port, key_file,
+                                               cert_file, strict))
+
+            # we never actually use these for anything, but we keep them
+            # here for compatibility with post-1.5.2 CVS.
+            self.key_file = key_file
+            self.cert_file = cert_file
+
+
+    def FakeSocket (sock, sslobj):
+        warnings.warn("FakeSocket is deprecated, and won't be in 3.x.  " +
+                      "Use the result of ssl.wrap_socket() directly instead.",
+                      DeprecationWarning, stacklevel=2)
+        return sslobj
+
+
+class HTTPException(Exception):
+    # Subclasses that define an __init__ must call Exception.__init__
+    # or define self.args.  Otherwise, str() will fail.
+    pass
+
+class NotConnected(HTTPException):
+    pass
+
+class InvalidURL(HTTPException):
+    pass
+
+class UnknownProtocol(HTTPException):
+    def __init__(self, version):
+        self.args = version,
+        self.version = version
+
+class UnknownTransferEncoding(HTTPException):
+    pass
+
+class UnimplementedFileMode(HTTPException):
+    pass
+
+class IncompleteRead(HTTPException):
+    def __init__(self, partial, expected=None):
+        self.args = partial,
+        self.partial = partial
+        self.expected = expected
+    def __repr__(self):
+        if self.expected is not None:
+            e = ', %i more expected' % self.expected
+        else:
+            e = ''
+        return 'IncompleteRead(%i bytes read%s)' % (len(self.partial), e)
+    def __str__(self):
+        return repr(self)
+
+class ImproperConnectionState(HTTPException):
+    pass
+
+class CannotSendRequest(ImproperConnectionState):
+    pass
+
+class CannotSendHeader(ImproperConnectionState):
+    pass
+
+class ResponseNotReady(ImproperConnectionState):
+    pass
+
+class BadStatusLine(HTTPException):
+    def __init__(self, line):
+        if not line:
+            line = repr(line)
+        self.args = line,
+        self.line = line
+
+# for backwards compatibility
+error = HTTPException
+
+class LineAndFileWrapper:
+    """A limited file-like object for HTTP/0.9 responses."""
+
+    # The status-line parsing code calls readline(), which normally
+    # get the HTTP status line.  For a 0.9 response, however, this is
+    # actually the first line of the body!  Clients need to get a
+    # readable file object that contains that line.
+
+    def __init__(self, line, file):
+        self._line = line
+        self._file = file
+        self._line_consumed = 0
+        self._line_offset = 0
+        self._line_left = len(line)
+
+    def __getattr__(self, attr):
+        return getattr(self._file, attr)
+
+    def _done(self):
+        # called when the last byte is read from the line.  After the
+        # call, all read methods are delegated to the underlying file
+        # object.
+        self._line_consumed = 1
+        self.read = self._file.read
+        self.readline = self._file.readline
+        self.readlines = self._file.readlines
+
+    def read(self, amt=None):
+        if self._line_consumed:
+            return self._file.read(amt)
+        assert self._line_left
+        if amt is None or amt > self._line_left:
+            s = self._line[self._line_offset:]
+            self._done()
+            if amt is None:
+                return s + self._file.read()
+            else:
+                return s + self._file.read(amt - len(s))
+        else:
+            assert amt <= self._line_left
+            i = self._line_offset
+            j = i + amt
+            s = self._line[i:j]
+            self._line_offset = j
+            self._line_left -= amt
+            if self._line_left == 0:
+                self._done()
+            return s
+
+    def readline(self):
+        if self._line_consumed:
+            return self._file.readline()
+        assert self._line_left
+        s = self._line[self._line_offset:]
+        self._done()
+        return s
+
+    def readlines(self, size=None):
+        if self._line_consumed:
+            return self._file.readlines(size)
+        assert self._line_left
+        L = [self._line[self._line_offset:]]
+        self._done()
+        if size is None:
+            return L + self._file.readlines()
+        else:
+            return L + self._file.readlines(size)
+
+def test():
+    """Test this module.
+
+    A hodge podge of tests collected here, because they have too many
+    external dependencies for the regular test suite.
+    """
+
+    import sys
+    import getopt
+    opts, args = getopt.getopt(sys.argv[1:], 'd')
+    dl = 0
+    for o, a in opts:
+        if o == '-d': dl = dl + 1
+    host = 'www.python.org'
+    selector = '/'
+    if args[0:]: host = args[0]
+    if args[1:]: selector = args[1]
+    h = HTTP()
+    h.set_debuglevel(dl)
+    h.connect(host)
+    h.putrequest('GET', selector)
+    h.endheaders()
+    status, reason, headers = h.getreply()
+    print 'status =', status
+    print 'reason =', reason
+    print "read", len(h.getfile().read())
+    print
+    if headers:
+        for header in headers.headers: print header.strip()
+    print
+
+    # minimal test that code to extract host from url works
+    class HTTP11(HTTP):
+        _http_vsn = 11
+        _http_vsn_str = 'HTTP/1.1'
+
+    h = HTTP11('www.python.org')
+    h.putrequest('GET', 'http://www.python.org/~jeremy/')
+    h.endheaders()
+    h.getreply()
+    h.close()
+
+    try:
+        import ssl
+    except ImportError:
+        pass
+    else:
+
+        for host, selector in (('sourceforge.net', '/projects/python'),
+                               ):
+            print "https://%s%s" % (host, selector)
+            hs = HTTPS()
+            hs.set_debuglevel(dl)
+            hs.connect(host)
+            hs.putrequest('GET', selector)
+            hs.endheaders()
+            status, reason, headers = hs.getreply()
+            print 'status =', status
+            print 'reason =', reason
+            print "read", len(hs.getfile().read())
+            print
+            if headers:
+                for header in headers.headers: print header.strip()
+            print
+
+if __name__ == '__main__':
+    test()
diff --git a/lib-python/modified-2.7/test/test_array.py b/lib-python/modified-2.7/test/test_array.py
--- a/lib-python/modified-2.7/test/test_array.py
+++ b/lib-python/modified-2.7/test/test_array.py
@@ -295,9 +295,10 @@
         )
 
         b = array.array(self.badtypecode())
-        self.assertRaises(TypeError, "a + b")
-
-        self.assertRaises(TypeError, "a + 'bad'")
+        with self.assertRaises(TypeError):
+            a + b
+        with self.assertRaises(TypeError):
+            a + 'bad'
 
     def test_iadd(self):
         a = array.array(self.typecode, self.example[::-1])
@@ -316,9 +317,10 @@
         )
 
         b = array.array(self.badtypecode())
-        self.assertRaises(TypeError, "a += b")
-
-        self.assertRaises(TypeError, "a += 'bad'")
+        with self.assertRaises(TypeError):
+            a += b
+        with self.assertRaises(TypeError):
+            a += 'bad'
 
     def test_mul(self):
         a = 5*array.array(self.typecode, self.example)
@@ -345,7 +347,8 @@
             array.array(self.typecode)
         )
 
-        self.assertRaises(TypeError, "a * 'bad'")
+        with self.assertRaises(TypeError):
+            a * 'bad'
 
     def test_imul(self):
         a = array.array(self.typecode, self.example)
@@ -374,7 +377,8 @@
         a *= -1
         self.assertEqual(a, array.array(self.typecode))
 
-        self.assertRaises(TypeError, "a *= 'bad'")
+        with self.assertRaises(TypeError):
+            a *= 'bad'
 
     def test_getitem(self):
         a = array.array(self.typecode, self.example)
diff --git a/lib-python/modified-2.7/test/test_urllib2.py b/lib-python/modified-2.7/test/test_urllib2.py
--- a/lib-python/modified-2.7/test/test_urllib2.py
+++ b/lib-python/modified-2.7/test/test_urllib2.py
@@ -307,6 +307,9 @@
     def getresponse(self):
         return MockHTTPResponse(MockFile(), {}, 200, "OK")
 
+    def close(self):
+        pass
+
 class MockHandler:
     # useful for testing handler machinery
     # see add_ordered_mock_handlers() docstring
diff --git a/lib-python/modified-2.7/urllib2.py b/lib-python/modified-2.7/urllib2.py
new file mode 100644
--- /dev/null
+++ b/lib-python/modified-2.7/urllib2.py
@@ -0,0 +1,1440 @@
+"""An extensible library for opening URLs using a variety of protocols
+
+The simplest way to use this module is to call the urlopen function,
+which accepts a string containing a URL or a Request object (described
+below).  It opens the URL and returns the results as file-like
+object; the returned object has some extra methods described below.
+
+The OpenerDirector manages a collection of Handler objects that do
+all the actual work.  Each Handler implements a particular protocol or
+option.  The OpenerDirector is a composite object that invokes the
+Handlers needed to open the requested URL.  For example, the
+HTTPHandler performs HTTP GET and POST requests and deals with
+non-error returns.  The HTTPRedirectHandler automatically deals with
+HTTP 301, 302, 303 and 307 redirect errors, and the HTTPDigestAuthHandler
+deals with digest authentication.
+
+urlopen(url, data=None) -- Basic usage is the same as original
+urllib.  pass the url and optionally data to post to an HTTP URL, and
+get a file-like object back.  One difference is that you can also pass
+a Request instance instead of URL.  Raises a URLError (subclass of
+IOError); for HTTP errors, raises an HTTPError, which can also be
+treated as a valid response.
+
+build_opener -- Function that creates a new OpenerDirector instance.
+Will install the default handlers.  Accepts one or more Handlers as
+arguments, either instances or Handler classes that it will
+instantiate.  If one of the argument is a subclass of the default
+handler, the argument will be installed instead of the default.
+
+install_opener -- Installs a new opener as the default opener.
+
+objects of interest:
+
+OpenerDirector -- Sets up the User Agent as the Python-urllib client and manages
+the Handler classes, while dealing with requests and responses.
+
+Request -- An object that encapsulates the state of a request.  The
+state can be as simple as the URL.  It can also include extra HTTP
+headers, e.g. a User-Agent.
+
+BaseHandler --
+
+exceptions:
+URLError -- A subclass of IOError, individual protocols have their own
+specific subclass.
+
+HTTPError -- Also a valid HTTP response, so you can treat an HTTP error
+as an exceptional event or valid response.
+
+internals:
+BaseHandler and parent
+_call_chain conventions
+
+Example usage:
+
+import urllib2
+
+# set up authentication info
+authinfo = urllib2.HTTPBasicAuthHandler()
+authinfo.add_password(realm='PDQ Application',
+                      uri='https://mahler:8092/site-updates.py',
+                      user='klem',
+                      passwd='geheim$parole')
+
+proxy_support = urllib2.ProxyHandler({"http" : "http://ahad-haam:3128"})
+
+# build a new opener that adds authentication and caching FTP handlers
+opener = urllib2.build_opener(proxy_support, authinfo, urllib2.CacheFTPHandler)
+
+# install it
+urllib2.install_opener(opener)
+
+f = urllib2.urlopen('http://www.python.org/')
+
+
+"""
+
+# XXX issues:
+# If an authentication error handler that tries to perform
+# authentication for some reason but fails, how should the error be
+# signalled?  The client needs to know the HTTP error code.  But if
+# the handler knows that the problem was, e.g., that it didn't know
+# that hash algo that requested in the challenge, it would be good to
+# pass that information along to the client, too.
+# ftp errors aren't handled cleanly
+# check digest against correct (i.e. non-apache) implementation
+
+# Possible extensions:
+# complex proxies  XXX not sure what exactly was meant by this
+# abstract factory for opener
+
+import base64
+import hashlib
+import httplib
+import mimetools
+import os
+import posixpath
+import random
+import re
+import socket
+import sys
+import time
+import urlparse
+import bisect
+
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from StringIO import StringIO
+
+from urllib import (unwrap, unquote, splittype, splithost, quote,
+     addinfourl, splitport, splittag,
+     splitattr, ftpwrapper, splituser, splitpasswd, splitvalue)
+
+# support for FileHandler, proxies via environment variables
+from urllib import localhost, url2pathname, getproxies, proxy_bypass
+
+# used in User-Agent header sent
+__version__ = sys.version[:3]
+
+_opener = None
+def urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
+    global _opener
+    if _opener is None:
+        _opener = build_opener()
+    return _opener.open(url, data, timeout)
+
+def install_opener(opener):
+    global _opener
+    _opener = opener
+
+# do these error classes make sense?
+# make sure all of the IOError stuff is overridden.  we just want to be
+# subtypes.
+
+class URLError(IOError):
+    # URLError is a sub-type of IOError, but it doesn't share any of
+    # the implementation.  need to override __init__ and __str__.
+    # It sets self.args for compatibility with other EnvironmentError
+    # subclasses, but args doesn't have the typical format with errno in
+    # slot 0 and strerror in slot 1.  This may be better than nothing.
+    def __init__(self, reason):
+        self.args = reason,
+        self.reason = reason
+
+    def __str__(self):
+        return '<urlopen error %s>' % self.reason
+
+class HTTPError(URLError, addinfourl):
+    """Raised when HTTP error occurs, but also acts like non-error return"""
+    __super_init = addinfourl.__init__
+
+    def __init__(self, url, code, msg, hdrs, fp):
+        self.code = code
+        self.msg = msg
+        self.hdrs = hdrs
+        self.fp = fp
+        self.filename = url
+        # The addinfourl classes depend on fp being a valid file
+        # object.  In some cases, the HTTPError may not have a valid
+        # file object.  If this happens, the simplest workaround is to
+        # not initialize the base classes.
+        if fp is not None:
+            self.__super_init(fp, hdrs, url, code)
+
+    def __str__(self):
+        return 'HTTP Error %s: %s' % (self.code, self.msg)
+
+# copied from cookielib.py
+_cut_port_re = re.compile(r":\d+$")
+def request_host(request):
+    """Return request-host, as defined by RFC 2965.
+
+    Variation from RFC: returned value is lowercased, for convenient
+    comparison.
+
+    """
+    url = request.get_full_url()
+    host = urlparse.urlparse(url)[1]
+    if host == "":
+        host = request.get_header("Host", "")
+
+    # remove port, if present
+    host = _cut_port_re.sub("", host, 1)
+    return host.lower()
+
+class Request:
+
+    def __init__(self, url, data=None, headers={},
+                 origin_req_host=None, unverifiable=False):
+        # unwrap('<URL:type://host/path>') --> 'type://host/path'
+        self.__original = unwrap(url)
+        self.__original, fragment = splittag(self.__original)
+        self.type = None
+        # self.__r_type is what's left after doing the splittype
+        self.host = None
+        self.port = None
+        self._tunnel_host = None
+        self.data = data
+        self.headers = {}
+        for key, value in headers.items():
+            self.add_header(key, value)
+        self.unredirected_hdrs = {}
+        if origin_req_host is None:
+            origin_req_host = request_host(self)
+        self.origin_req_host = origin_req_host
+        self.unverifiable = unverifiable
+
+    def __getattr__(self, attr):
+        # XXX this is a fallback mechanism to guard against these
+        # methods getting called in a non-standard order.  this may be
+        # too complicated and/or unnecessary.
+        # XXX should the __r_XXX attributes be public?
+        if attr[:12] == '_Request__r_':
+            name = attr[12:]
+            if hasattr(Request, 'get_' + name):
+                getattr(self, 'get_' + name)()
+                return getattr(self, attr)
+        raise AttributeError, attr
+
+    def get_method(self):
+        if self.has_data():
+            return "POST"
+        else:
+            return "GET"
+
+    # XXX these helper methods are lame
+
+    def add_data(self, data):
+        self.data = data
+
+    def has_data(self):
+        return self.data is not None
+
+    def get_data(self):
+        return self.data
+
+    def get_full_url(self):
+        return self.__original
+
+    def get_type(self):
+        if self.type is None:
+            self.type, self.__r_type = splittype(self.__original)
+            if self.type is None:
+                raise ValueError, "unknown url type: %s" % self.__original
+        return self.type
+
+    def get_host(self):
+        if self.host is None:
+            self.host, self.__r_host = splithost(self.__r_type)
+            if self.host:
+                self.host = unquote(self.host)
+        return self.host
+
+    def get_selector(self):
+        return self.__r_host
+
+    def set_proxy(self, host, type):
+        if self.type == 'https' and not self._tunnel_host:
+            self._tunnel_host = self.host
+        else:
+            self.type = type
+            self.__r_host = self.__original
+
+        self.host = host
+
+    def has_proxy(self):
+        return self.__r_host == self.__original
+
+    def get_origin_req_host(self):
+        return self.origin_req_host
+
+    def is_unverifiable(self):
+        return self.unverifiable
+
+    def add_header(self, key, val):
+        # useful for something like authentication
+        self.headers[key.capitalize()] = val
+
+    def add_unredirected_header(self, key, val):
+        # will not be added to a redirected request
+        self.unredirected_hdrs[key.capitalize()] = val
+
+    def has_header(self, header_name):
+        return (header_name in self.headers or
+                header_name in self.unredirected_hdrs)
+
+    def get_header(self, header_name, default=None):
+        return self.headers.get(
+            header_name,
+            self.unredirected_hdrs.get(header_name, default))
+
+    def header_items(self):
+        hdrs = self.unredirected_hdrs.copy()
+        hdrs.update(self.headers)
+        return hdrs.items()
+
+class OpenerDirector:
+    def __init__(self):
+        client_version = "Python-urllib/%s" % __version__
+        self.addheaders = [('User-agent', client_version)]
+        # manage the individual handlers
+        self.handlers = []
+        self.handle_open = {}
+        self.handle_error = {}
+        self.process_response = {}
+        self.process_request = {}
+
+    def add_handler(self, handler):
+        if not hasattr(handler, "add_parent"):
+            raise TypeError("expected BaseHandler instance, got %r" %
+                            type(handler))
+
+        added = False
+        for meth in dir(handler):
+            if meth in ["redirect_request", "do_open", "proxy_open"]:
+                # oops, coincidental match
+                continue
+
+            i = meth.find("_")
+            protocol = meth[:i]
+            condition = meth[i+1:]
+
+            if condition.startswith("error"):
+                j = condition.find("_") + i + 1
+                kind = meth[j+1:]
+                try:
+                    kind = int(kind)
+                except ValueError:
+                    pass
+                lookup = self.handle_error.get(protocol, {})
+                self.handle_error[protocol] = lookup
+            elif condition == "open":
+                kind = protocol
+                lookup = self.handle_open
+            elif condition == "response":
+                kind = protocol
+                lookup = self.process_response
+            elif condition == "request":
+                kind = protocol
+                lookup = self.process_request
+            else:
+                continue
+
+            handlers = lookup.setdefault(kind, [])
+            if handlers:
+                bisect.insort(handlers, handler)
+            else:
+                handlers.append(handler)
+            added = True
+
+        if added:
+            # the handlers must work in an specific order, the order
+            # is specified in a Handler attribute
+            bisect.insort(self.handlers, handler)
+            handler.add_parent(self)
+
+    def close(self):
+        # Only exists for backwards compatibility.
+        pass
+
+    def _call_chain(self, chain, kind, meth_name, *args):
+        # Handlers raise an exception if no one else should try to handle
+        # the request, or return None if they can't but another handler
+        # could.  Otherwise, they return the response.
+        handlers = chain.get(kind, ())
+        for handler in handlers:
+            func = getattr(handler, meth_name)
+
+            result = func(*args)
+            if result is not None:
+                return result
+
+    def open(self, fullurl, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
+        # accept a URL or a Request object
+        if isinstance(fullurl, basestring):
+            req = Request(fullurl, data)
+        else:
+            req = fullurl
+            if data is not None:
+                req.add_data(data)
+
+        req.timeout = timeout
+        protocol = req.get_type()
+
+        # pre-process request
+        meth_name = protocol+"_request"
+        for processor in self.process_request.get(protocol, []):
+            meth = getattr(processor, meth_name)
+            req = meth(req)
+
+        response = self._open(req, data)
+
+        # post-process response
+        meth_name = protocol+"_response"
+        for processor in self.process_response.get(protocol, []):
+            meth = getattr(processor, meth_name)
+            try:
+                response = meth(req, response)
+            except:
+                response.close()
+                raise
+
+        return response
+
+    def _open(self, req, data=None):
+        result = self._call_chain(self.handle_open, 'default',
+                                  'default_open', req)
+        if result:
+            return result
+
+        protocol = req.get_type()
+        result = self._call_chain(self.handle_open, protocol, protocol +
+                                  '_open', req)
+        if result:
+            return result
+
+        return self._call_chain(self.handle_open, 'unknown',
+                                'unknown_open', req)
+
+    def error(self, proto, *args):
+        if proto in ('http', 'https'):
+            # XXX http[s] protocols are special-cased
+            dict = self.handle_error['http'] # https is not different than http
+            proto = args[2]  # YUCK!
+            meth_name = 'http_error_%s' % proto
+            http_err = 1
+            orig_args = args
+        else:
+            dict = self.handle_error
+            meth_name = proto + '_error'
+            http_err = 0
+        args = (dict, proto, meth_name) + args
+        result = self._call_chain(*args)
+        if result:
+            return result
+
+        if http_err:
+            args = (dict, 'default', 'http_error_default') + orig_args
+            return self._call_chain(*args)
+
+# XXX probably also want an abstract factory that knows when it makes
+# sense to skip a superclass in favor of a subclass and when it might
+# make sense to include both
+
+def build_opener(*handlers):
+    """Create an opener object from a list of handlers.
+
+    The opener will use several default handlers, including support
+    for HTTP, FTP and when applicable, HTTPS.
+
+    If any of the handlers passed as arguments are subclasses of the
+    default handlers, the default handlers will not be used.
+    """
+    import types
+    def isclass(obj):
+        return isinstance(obj, (types.ClassType, type))
+
+    opener = OpenerDirector()
+    default_classes = [ProxyHandler, UnknownHandler, HTTPHandler,
+                       HTTPDefaultErrorHandler, HTTPRedirectHandler,
+                       FTPHandler, FileHandler, HTTPErrorProcessor]
+    if hasattr(httplib, 'HTTPS'):
+        default_classes.append(HTTPSHandler)
+    skip = set()
+    for klass in default_classes:
+        for check in handlers:
+            if isclass(check):
+                if issubclass(check, klass):
+                    skip.add(klass)
+            elif isinstance(check, klass):
+                skip.add(klass)
+    for klass in skip:
+        default_classes.remove(klass)
+
+    for klass in default_classes:
+        opener.add_handler(klass())
+
+    for h in handlers:
+        if isclass(h):
+            h = h()
+        opener.add_handler(h)
+    return opener
+
+class BaseHandler:
+    handler_order = 500
+
+    def add_parent(self, parent):
+        self.parent = parent
+
+    def close(self):
+        # Only exists for backwards compatibility
+        pass
+
+    def __lt__(self, other):
+        if not hasattr(other, "handler_order"):
+            # Try to preserve the old behavior of having custom classes
+            # inserted after default ones (works only for custom user
+            # classes which are not aware of handler_order).
+            return True
+        return self.handler_order < other.handler_order
+
+
+class HTTPErrorProcessor(BaseHandler):
+    """Process HTTP error responses."""
+    handler_order = 1000  # after all other processing
+
+    def http_response(self, request, response):
+        code, msg, hdrs = response.code, response.msg, response.info()
+
+        # According to RFC 2616, "2xx" code indicates that the client's
+        # request was successfully received, understood, and accepted.
+        if not (200 <= code < 300):
+            response = self.parent.error(
+                'http', request, response, code, msg, hdrs)
+
+        return response
+
+    https_response = http_response
+
+class HTTPDefaultErrorHandler(BaseHandler):
+    def http_error_default(self, req, fp, code, msg, hdrs):
+        raise HTTPError(req.get_full_url(), code, msg, hdrs, fp)
+
+class HTTPRedirectHandler(BaseHandler):
+    # maximum number of redirections to any single URL
+    # this is needed because of the state that cookies introduce
+    max_repeats = 4
+    # maximum total number of redirections (regardless of URL) before
+    # assuming we're in a loop
+    max_redirections = 10
+
+    def redirect_request(self, req, fp, code, msg, headers, newurl):
+        """Return a Request or None in response to a redirect.
+
+        This is called by the http_error_30x methods when a
+        redirection response is received.  If a redirection should
+        take place, return a new Request to allow http_error_30x to
+        perform the redirect.  Otherwise, raise HTTPError if no-one
+        else should try to handle this url.  Return None if you can't
+        but another Handler might.
+        """
+        m = req.get_method()
+        if (code in (301, 302, 303, 307) and m in ("GET", "HEAD")
+            or code in (301, 302, 303) and m == "POST"):
+            # Strictly (according to RFC 2616), 301 or 302 in response
+            # to a POST MUST NOT cause a redirection without confirmation
+            # from the user (of urllib2, in this case).  In practice,
+            # essentially all clients do redirect in this case, so we
+            # do the same.
+            # be conciliant with URIs containing a space
+            newurl = newurl.replace(' ', '%20')
+            newheaders = dict((k,v) for k,v in req.headers.items()
+                              if k.lower() not in ("content-length", "content-type")
+                             )
+            return Request(newurl,
+                           headers=newheaders,
+                           origin_req_host=req.get_origin_req_host(),
+                           unverifiable=True)
+        else:
+            raise HTTPError(req.get_full_url(), code, msg, headers, fp)
+
+    # Implementation note: To avoid the server sending us into an
+    # infinite loop, the request object needs to track what URLs we
+    # have already seen.  Do this by adding a handler-specific
+    # attribute to the Request object.
+    def http_error_302(self, req, fp, code, msg, headers):
+        # Some servers (incorrectly) return multiple Location headers
+        # (so probably same goes for URI).  Use first header.
+        if 'location' in headers:
+            newurl = headers.getheaders('location')[0]
+        elif 'uri' in headers:
+            newurl = headers.getheaders('uri')[0]
+        else:
+            return
+
+        # fix a possible malformed URL
+        urlparts = urlparse.urlparse(newurl)
+        if not urlparts.path:
+            urlparts = list(urlparts)
+            urlparts[2] = "/"
+        newurl = urlparse.urlunparse(urlparts)
+
+        newurl = urlparse.urljoin(req.get_full_url(), newurl)
+
+        # XXX Probably want to forget about the state of the current
+        # request, although that might interact poorly with other
+        # handlers that also use handler-specific request attributes
+        new = self.redirect_request(req, fp, code, msg, headers, newurl)
+        if new is None:
+            return
+
+        # loop detection
+        # .redirect_dict has a key url if url was previously visited.
+        if hasattr(req, 'redirect_dict'):
+            visited = new.redirect_dict = req.redirect_dict
+            if (visited.get(newurl, 0) >= self.max_repeats or
+                len(visited) >= self.max_redirections):
+                raise HTTPError(req.get_full_url(), code,
+                                self.inf_msg + msg, headers, fp)
+        else:
+            visited = new.redirect_dict = req.redirect_dict = {}
+        visited[newurl] = visited.get(newurl, 0) + 1
+
+        # Don't close the fp until we are sure that we won't use it
+        # with HTTPError.
+        fp.read()
+        fp.close()
+
+        return self.parent.open(new, timeout=req.timeout)
+
+    http_error_301 = http_error_303 = http_error_307 = http_error_302
+
+    inf_msg = "The HTTP server returned a redirect error that would " \
+              "lead to an infinite loop.\n" \
+              "The last 30x error message was:\n"
+
+
+def _parse_proxy(proxy):
+    """Return (scheme, user, password, host/port) given a URL or an authority.
+
+    If a URL is supplied, it must have an authority (host:port) component.
+    According to RFC 3986, having an authority component means the URL must
+    have two slashes after the scheme:
+
+    >>> _parse_proxy('file:/ftp.example.com/')
+    Traceback (most recent call last):
+    ValueError: proxy URL with no authority: 'file:/ftp.example.com/'
+
+    The first three items of the returned tuple may be None.
+
+    Examples of authority parsing:
+
+    >>> _parse_proxy('proxy.example.com')
+    (None, None, None, 'proxy.example.com')
+    >>> _parse_proxy('proxy.example.com:3128')
+    (None, None, None, 'proxy.example.com:3128')
+
+    The authority component may optionally include userinfo (assumed to be
+    username:password):
+
+    >>> _parse_proxy('joe:password at proxy.example.com')
+    (None, 'joe', 'password', 'proxy.example.com')
+    >>> _parse_proxy('joe:password at proxy.example.com:3128')
+    (None, 'joe', 'password', 'proxy.example.com:3128')
+
+    Same examples, but with URLs instead:
+
+    >>> _parse_proxy('http://proxy.example.com/')
+    ('http', None, None, 'proxy.example.com')
+    >>> _parse_proxy('http://proxy.example.com:3128/')
+    ('http', None, None, 'proxy.example.com:3128')
+    >>> _parse_proxy('http://joe:password at proxy.example.com/')
+    ('http', 'joe', 'password', 'proxy.example.com')
+    >>> _parse_proxy('http://joe:password at proxy.example.com:3128')
+    ('http', 'joe', 'password', 'proxy.example.com:3128')
+
+    Everything after the authority is ignored:
+
+    >>> _parse_proxy('ftp://joe:password at proxy.example.com/rubbish:3128')
+    ('ftp', 'joe', 'password', 'proxy.example.com')
+
+    Test for no trailing '/' case:
+
+    >>> _parse_proxy('http://joe:password at proxy.example.com')
+    ('http', 'joe', 'password', 'proxy.example.com')
+
+    """
+    scheme, r_scheme = splittype(proxy)
+    if not r_scheme.startswith("/"):
+        # authority
+        scheme = None
+        authority = proxy
+    else:
+        # URL
+        if not r_scheme.startswith("//"):
+            raise ValueError("proxy URL with no authority: %r" % proxy)
+        # We have an authority, so for RFC 3986-compliant URLs (by ss 3.
+        # and 3.3.), path is empty or starts with '/'
+        end = r_scheme.find("/", 2)
+        if end == -1:
+            end = None
+        authority = r_scheme[2:end]
+    userinfo, hostport = splituser(authority)
+    if userinfo is not None:
+        user, password = splitpasswd(userinfo)
+    else:
+        user = password = None
+    return scheme, user, password, hostport
+
+class ProxyHandler(BaseHandler):
+    # Proxies must be in front
+    handler_order = 100
+
+    def __init__(self, proxies=None):
+        if proxies is None:
+            proxies = getproxies()
+        assert hasattr(proxies, 'has_key'), "proxies must be a mapping"
+        self.proxies = proxies
+        for type, url in proxies.items():
+            setattr(self, '%s_open' % type,
+                    lambda r, proxy=url, type=type, meth=self.proxy_open: \
+                    meth(r, proxy, type))
+
+    def proxy_open(self, req, proxy, type):
+        orig_type = req.get_type()
+        proxy_type, user, password, hostport = _parse_proxy(proxy)
+
+        if proxy_type is None:
+            proxy_type = orig_type
+
+        if req.host and proxy_bypass(req.host):
+            return None
+
+        if user and password:
+            user_pass = '%s:%s' % (unquote(user), unquote(password))
+            creds = base64.b64encode(user_pass).strip()
+            req.add_header('Proxy-authorization', 'Basic ' + creds)
+        hostport = unquote(hostport)
+        req.set_proxy(hostport, proxy_type)
+
+        if orig_type == proxy_type or orig_type == 'https':
+            # let other handlers take care of it
+            return None
+        else:
+            # need to start over, because the other handlers don't
+            # grok the proxy's URL type
+            # e.g. if we have a constructor arg proxies like so:
+            # {'http': 'ftp://proxy.example.com'}, we may end up turning
+            # a request for http://acme.example.com/a into one for
+            # ftp://proxy.example.com/a
+            return self.parent.open(req, timeout=req.timeout)
+
+class HTTPPasswordMgr:
+
+    def __init__(self):
+        self.passwd = {}
+
+    def add_password(self, realm, uri, user, passwd):
+        # uri could be a single URI or a sequence
+        if isinstance(uri, basestring):
+            uri = [uri]
+        if not realm in self.passwd:
+            self.passwd[realm] = {}
+        for default_port in True, False:
+            reduced_uri = tuple(
+                [self.reduce_uri(u, default_port) for u in uri])
+            self.passwd[realm][reduced_uri] = (user, passwd)
+
+    def find_user_password(self, realm, authuri):
+        domains = self.passwd.get(realm, {})
+        for default_port in True, False:
+            reduced_authuri = self.reduce_uri(authuri, default_port)
+            for uris, authinfo in domains.iteritems():
+                for uri in uris:
+                    if self.is_suburi(uri, reduced_authuri):
+                        return authinfo
+        return None, None
+
+    def reduce_uri(self, uri, default_port=True):
+        """Accept authority or URI and extract only the authority and path."""
+        # note HTTP URLs do not have a userinfo component
+        parts = urlparse.urlsplit(uri)
+        if parts[1]:
+            # URI
+            scheme = parts[0]
+            authority = parts[1]
+            path = parts[2] or '/'
+        else:
+            # host or host:port
+            scheme = None
+            authority = uri
+            path = '/'
+        host, port = splitport(authority)
+        if default_port and port is None and scheme is not None:
+            dport = {"http": 80,
+                     "https": 443,
+                     }.get(scheme)
+            if dport is not None:
+                authority = "%s:%d" % (host, dport)
+        return authority, path
+
+    def is_suburi(self, base, test):
+        """Check if test is below base in a URI tree
+
+        Both args must be URIs in reduced form.
+        """
+        if base == test:
+            return True
+        if base[0] != test[0]:
+            return False
+        common = posixpath.commonprefix((base[1], test[1]))
+        if len(common) == len(base[1]):
+            return True
+        return False
+
+
+class HTTPPasswordMgrWithDefaultRealm(HTTPPasswordMgr):
+
+    def find_user_password(self, realm, authuri):
+        user, password = HTTPPasswordMgr.find_user_password(self, realm,
+                                                            authuri)
+        if user is not None:
+            return user, password
+        return HTTPPasswordMgr.find_user_password(self, None, authuri)
+
+
+class AbstractBasicAuthHandler:
+
+    # XXX this allows for multiple auth-schemes, but will stupidly pick
+    # the last one with a realm specified.
+
+    # allow for double- and single-quoted realm values
+    # (single quotes are a violation of the RFC, but appear in the wild)
+    rx = re.compile('(?:.*,)*[ \t]*([^ \t]+)[ \t]+'
+                    'realm=(["\'])(.*?)\\2', re.I)
+
+    # XXX could pre-emptively send auth info already accepted (RFC 2617,
+    # end of section 2, and section 1.2 immediately after "credentials"
+    # production).
+
+    def __init__(self, password_mgr=None):
+        if password_mgr is None:
+            password_mgr = HTTPPasswordMgr()
+        self.passwd = password_mgr
+        self.add_password = self.passwd.add_password
+        self.retried = 0
+
+    def reset_retry_count(self):
+        self.retried = 0
+
+    def http_error_auth_reqed(self, authreq, host, req, headers):
+        # host may be an authority (without userinfo) or a URL with an
+        # authority
+        # XXX could be multiple headers
+        authreq = headers.get(authreq, None)
+
+        if self.retried > 5:
+            # retry sending the username:password 5 times before failing.
+            raise HTTPError(req.get_full_url(), 401, "basic auth failed",
+                            headers, None)
+        else:
+            self.retried += 1
+
+        if authreq:
+            mo = AbstractBasicAuthHandler.rx.search(authreq)
+            if mo:
+                scheme, quote, realm = mo.groups()
+                if scheme.lower() == 'basic':
+                    response = self.retry_http_basic_auth(host, req, realm)
+                    if response and response.code != 401:
+                        self.retried = 0
+                    return response
+
+    def retry_http_basic_auth(self, host, req, realm):
+        user, pw = self.passwd.find_user_password(realm, host)
+        if pw is not None:
+            raw = "%s:%s" % (user, pw)
+            auth = 'Basic %s' % base64.b64encode(raw).strip()
+            if req.headers.get(self.auth_header, None) == auth:
+                return None
+            req.add_unredirected_header(self.auth_header, auth)
+            return self.parent.open(req, timeout=req.timeout)
+        else:
+            return None
+
+
+class HTTPBasicAuthHandler(AbstractBasicAuthHandler, BaseHandler):
+
+    auth_header = 'Authorization'
+
+    def http_error_401(self, req, fp, code, msg, headers):
+        url = req.get_full_url()
+        response = self.http_error_auth_reqed('www-authenticate',
+                                              url, req, headers)
+        self.reset_retry_count()
+        return response
+
+
+class ProxyBasicAuthHandler(AbstractBasicAuthHandler, BaseHandler):
+
+    auth_header = 'Proxy-authorization'
+
+    def http_error_407(self, req, fp, code, msg, headers):
+        # http_error_auth_reqed requires that there is no userinfo component in
+        # authority.  Assume there isn't one, since urllib2 does not (and
+        # should not, RFC 3986 s. 3.2.1) support requests for URLs containing
+        # userinfo.
+        authority = req.get_host()
+        response = self.http_error_auth_reqed('proxy-authenticate',
+                                          authority, req, headers)
+        self.reset_retry_count()
+        return response
+
+
+def randombytes(n):
+    """Return n random bytes."""
+    # Use /dev/urandom if it is available.  Fall back to random module
+    # if not.  It might be worthwhile to extend this function to use
+    # other platform-specific mechanisms for getting random bytes.
+    if os.path.exists("/dev/urandom"):
+        f = open("/dev/urandom")
+        s = f.read(n)
+        f.close()
+        return s
+    else:
+        L = [chr(random.randrange(0, 256)) for i in range(n)]
+        return "".join(L)
+
+class AbstractDigestAuthHandler:
+    # Digest authentication is specified in RFC 2617.
+
+    # XXX The client does not inspect the Authentication-Info header
+    # in a successful response.
+
+    # XXX It should be possible to test this implementation against
+    # a mock server that just generates a static set of challenges.
+
+    # XXX qop="auth-int" supports is shaky
+
+    def __init__(self, passwd=None):
+        if passwd is None:
+            passwd = HTTPPasswordMgr()
+        self.passwd = passwd
+        self.add_password = self.passwd.add_password
+        self.retried = 0
+        self.nonce_count = 0
+        self.last_nonce = None
+
+    def reset_retry_count(self):
+        self.retried = 0
+
+    def http_error_auth_reqed(self, auth_header, host, req, headers):
+        authreq = headers.get(auth_header, None)
+        if self.retried > 5:
+            # Don't fail endlessly - if we failed once, we'll probably
+            # fail a second time. Hm. Unless the Password Manager is
+            # prompting for the information. Crap. This isn't great
+            # but it's better than the current 'repeat until recursion
+            # depth exceeded' approach <wink>
+            raise HTTPError(req.get_full_url(), 401, "digest auth failed",
+                            headers, None)
+        else:
+            self.retried += 1
+        if authreq:
+            scheme = authreq.split()[0]
+            if scheme.lower() == 'digest':
+                return self.retry_http_digest_auth(req, authreq)
+
+    def retry_http_digest_auth(self, req, auth):
+        token, challenge = auth.split(' ', 1)
+        chal = parse_keqv_list(parse_http_list(challenge))
+        auth = self.get_authorization(req, chal)
+        if auth:
+            auth_val = 'Digest %s' % auth
+            if req.headers.get(self.auth_header, None) == auth_val:
+                return None
+            req.add_unredirected_header(self.auth_header, auth_val)
+            resp = self.parent.open(req, timeout=req.timeout)
+            return resp
+
+    def get_cnonce(self, nonce):
+        # The cnonce-value is an opaque
+        # quoted string value provided by the client and used by both client
+        # and server to avoid chosen plaintext attacks, to provide mutual
+        # authentication, and to provide some message integrity protection.
+        # This isn't a fabulous effort, but it's probably Good Enough.
+        dig = hashlib.sha1("%s:%s:%s:%s" % (self.nonce_count, nonce, time.ctime(),
+                                            randombytes(8))).hexdigest()
+        return dig[:16]
+
+    def get_authorization(self, req, chal):
+        try:
+            realm = chal['realm']
+            nonce = chal['nonce']
+            qop = chal.get('qop')
+            algorithm = chal.get('algorithm', 'MD5')
+            # mod_digest doesn't send an opaque, even though it isn't
+            # supposed to be optional
+            opaque = chal.get('opaque', None)
+        except KeyError:
+            return None
+
+        H, KD = self.get_algorithm_impls(algorithm)
+        if H is None:
+            return None
+
+        user, pw = self.passwd.find_user_password(realm, req.get_full_url())
+        if user is None:
+            return None
+
+        # XXX not implemented yet
+        if req.has_data():
+            entdig = self.get_entity_digest(req.get_data(), chal)
+        else:
+            entdig = None
+
+        A1 = "%s:%s:%s" % (user, realm, pw)
+        A2 = "%s:%s" % (req.get_method(),
+                        # XXX selector: what about proxies and full urls
+                        req.get_selector())
+        if qop == 'auth':
+            if nonce == self.last_nonce:
+                self.nonce_count += 1
+            else:
+                self.nonce_count = 1
+                self.last_nonce = nonce
+
+            ncvalue = '%08x' % self.nonce_count
+            cnonce = self.get_cnonce(nonce)
+            noncebit = "%s:%s:%s:%s:%s" % (nonce, ncvalue, cnonce, qop, H(A2))
+            respdig = KD(H(A1), noncebit)
+        elif qop is None:
+            respdig = KD(H(A1), "%s:%s" % (nonce, H(A2)))
+        else:
+            # XXX handle auth-int.
+            raise URLError("qop '%s' is not supported." % qop)
+
+        # XXX should the partial digests be encoded too?
+
+        base = 'username="%s", realm="%s", nonce="%s", uri="%s", ' \
+               'response="%s"' % (user, realm, nonce, req.get_selector(),
+                                  respdig)
+        if opaque:
+            base += ', opaque="%s"' % opaque
+        if entdig:
+            base += ', digest="%s"' % entdig
+        base += ', algorithm="%s"' % algorithm
+        if qop:
+            base += ', qop=auth, nc=%s, cnonce="%s"' % (ncvalue, cnonce)
+        return base
+
+    def get_algorithm_impls(self, algorithm):
+        # algorithm should be case-insensitive according to RFC2617
+        algorithm = algorithm.upper()
+        # lambdas assume digest modules are imported at the top level
+        if algorithm == 'MD5':
+            H = lambda x: hashlib.md5(x).hexdigest()
+        elif algorithm == 'SHA':
+            H = lambda x: hashlib.sha1(x).hexdigest()
+        # XXX MD5-sess
+        KD = lambda s, d: H("%s:%s" % (s, d))
+        return H, KD
+
+    def get_entity_digest(self, data, chal):
+        # XXX not implemented yet
+        return None
+
+
+class HTTPDigestAuthHandler(BaseHandler, AbstractDigestAuthHandler):
+    """An authentication protocol defined by RFC 2069
+
+    Digest authentication improves on basic authentication because it
+    does not transmit passwords in the clear.
+    """
+
+    auth_header = 'Authorization'
+    handler_order = 490  # before Basic auth
+
+    def http_error_401(self, req, fp, code, msg, headers):
+        host = urlparse.urlparse(req.get_full_url())[1]
+        retry = self.http_error_auth_reqed('www-authenticate',
+                                           host, req, headers)
+        self.reset_retry_count()
+        return retry
+
+
+class ProxyDigestAuthHandler(BaseHandler, AbstractDigestAuthHandler):
+
+    auth_header = 'Proxy-Authorization'
+    handler_order = 490  # before Basic auth
+
+    def http_error_407(self, req, fp, code, msg, headers):
+        host = req.get_host()
+        retry = self.http_error_auth_reqed('proxy-authenticate',
+                                           host, req, headers)
+        self.reset_retry_count()
+        return retry
+
+class AbstractHTTPHandler(BaseHandler):
+
+    def __init__(self, debuglevel=0):
+        self._debuglevel = debuglevel
+
+    def set_http_debuglevel(self, level):
+        self._debuglevel = level
+
+    def do_request_(self, request):
+        host = request.get_host()
+        if not host:
+            raise URLError('no host given')
+
+        if request.has_data():  # POST
+            data = request.get_data()
+            if not request.has_header('Content-type'):
+                request.add_unredirected_header(
+                    'Content-type',
+                    'application/x-www-form-urlencoded')
+            if not request.has_header('Content-length'):
+                request.add_unredirected_header(
+                    'Content-length', '%d' % len(data))
+
+        sel_host = host
+        if request.has_proxy():
+            scheme, sel = splittype(request.get_selector())
+            sel_host, sel_path = splithost(sel)
+
+        if not request.has_header('Host'):
+            request.add_unredirected_header('Host', sel_host)
+        for name, value in self.parent.addheaders:
+            name = name.capitalize()
+            if not request.has_header(name):
+                request.add_unredirected_header(name, value)
+
+        return request
+
+    def do_open(self, http_class, req):
+        """Return an addinfourl object for the request, using http_class.
+
+        http_class must implement the HTTPConnection API from httplib.
+        The addinfourl return value is a file-like object.  It also
+        has methods and attributes including:
+            - info(): return a mimetools.Message object for the headers
+            - geturl(): return the original request URL
+            - code: HTTP status code
+        """
+        host = req.get_host()
+        if not host:
+            raise URLError('no host given')
+
+        h = http_class(host, timeout=req.timeout) # will parse host:port
+        h.set_debuglevel(self._debuglevel)
+
+        headers = dict(req.unredirected_hdrs)
+        headers.update(dict((k, v) for k, v in req.headers.items()
+                            if k not in headers))
+
+        # We want to make an HTTP/1.1 request, but the addinfourl
+        # class isn't prepared to deal with a persistent connection.
+        # It will try to read all remaining data from the socket,
+        # which will block while the server waits for the next request.
+        # So make sure the connection gets closed after the (only)
+        # request.
+        headers["Connection"] = "close"
+        headers = dict(
+            (name.title(), val) for name, val in headers.items())
+
+        if req._tunnel_host:
+            tunnel_headers = {}
+            proxy_auth_hdr = "Proxy-Authorization"
+            if proxy_auth_hdr in headers:
+                tunnel_headers[proxy_auth_hdr] = headers[proxy_auth_hdr]
+                # Proxy-Authorization should not be sent to origin
+                # server.
+                del headers[proxy_auth_hdr]
+            h.set_tunnel(req._tunnel_host, headers=tunnel_headers)
+
+        try:
+            h.request(req.get_method(), req.get_selector(), req.data, headers)
+            try:
+                r = h.getresponse(buffering=True)
+            except TypeError: #buffering kw not supported
+                r = h.getresponse()
+        except socket.error, err: # XXX what error?
+            h.close()
+            raise URLError(err)
+
+        # Pick apart the HTTPResponse object to get the addinfourl
+        # object initialized properly.
+
+        # Wrap the HTTPResponse object in socket's file object adapter
+        # for Windows.  That adapter calls recv(), so delegate recv()
+        # to read().  This weird wrapping allows the returned object to
+        # have readline() and readlines() methods.
+
+        # XXX It might be better to extract the read buffering code
+        # out of socket._fileobject() and into a base class.
+
+        r.recv = r.read
+        fp = socket._fileobject(r, close=True)
+
+        resp = addinfourl(fp, r.msg, req.get_full_url())
+        resp.code = r.status
+        resp.msg = r.reason
+        return resp
+
+
+class HTTPHandler(AbstractHTTPHandler):
+
+    def http_open(self, req):
+        return self.do_open(httplib.HTTPConnection, req)
+
+    http_request = AbstractHTTPHandler.do_request_
+
+if hasattr(httplib, 'HTTPS'):
+    class HTTPSHandler(AbstractHTTPHandler):
+
+        def https_open(self, req):
+            return self.do_open(httplib.HTTPSConnection, req)
+
+        https_request = AbstractHTTPHandler.do_request_
+
+class HTTPCookieProcessor(BaseHandler):
+    def __init__(self, cookiejar=None):
+        import cookielib
+        if cookiejar is None:
+            cookiejar = cookielib.CookieJar()
+        self.cookiejar = cookiejar
+
+    def http_request(self, request):
+        self.cookiejar.add_cookie_header(request)
+        return request
+
+    def http_response(self, request, response):
+        self.cookiejar.extract_cookies(response, request)
+        return response
+
+    https_request = http_request
+    https_response = http_response
+
+class UnknownHandler(BaseHandler):
+    def unknown_open(self, req):
+        type = req.get_type()
+        raise URLError('unknown url type: %s' % type)
+
+def parse_keqv_list(l):
+    """Parse list of key=value strings where keys are not duplicated."""
+    parsed = {}
+    for elt in l:
+        k, v = elt.split('=', 1)
+        if v[0] == '"' and v[-1] == '"':
+            v = v[1:-1]
+        parsed[k] = v
+    return parsed
+
+def parse_http_list(s):
+    """Parse lists as described by RFC 2068 Section 2.
+
+    In particular, parse comma-separated lists where the elements of
+    the list may include quoted-strings.  A quoted-string could
+    contain a comma.  A non-quoted string could have quotes in the
+    middle.  Neither commas nor quotes count if they are escaped.
+    Only double-quotes count, not single-quotes.
+    """
+    res = []
+    part = ''
+
+    escape = quote = False
+    for cur in s:
+        if escape:
+            part += cur
+            escape = False
+            continue
+        if quote:
+            if cur == '\\':
+                escape = True
+                continue
+            elif cur == '"':
+                quote = False
+            part += cur
+            continue
+
+        if cur == ',':
+            res.append(part)
+            part = ''
+            continue
+
+        if cur == '"':
+            quote = True
+
+        part += cur
+
+    # append last part
+    if part:
+        res.append(part)
+
+    return [part.strip() for part in res]
+
+def _safe_gethostbyname(host):
+    try:
+        return socket.gethostbyname(host)
+    except socket.gaierror:
+        return None
+
+class FileHandler(BaseHandler):
+    # Use local file or FTP depending on form of URL
+    def file_open(self, req):
+        url = req.get_selector()
+        if url[:2] == '//' and url[2:3] != '/' and (req.host and
+                req.host != 'localhost'):
+            req.type = 'ftp'
+            return self.parent.open(req)
+        else:
+            return self.open_local_file(req)
+
+    # names for the localhost
+    names = None
+    def get_names(self):
+        if FileHandler.names is None:
+            try:
+                FileHandler.names = tuple(
+                    socket.gethostbyname_ex('localhost')[2] +
+                    socket.gethostbyname_ex(socket.gethostname())[2])
+            except socket.gaierror:
+                FileHandler.names = (socket.gethostbyname('localhost'),)
+        return FileHandler.names
+
+    # not entirely sure what the rules are here
+    def open_local_file(self, req):
+        import email.utils
+        import mimetypes
+        host = req.get_host()
+        filename = req.get_selector()
+        localfile = url2pathname(filename)
+        try:
+            stats = os.stat(localfile)
+            size = stats.st_size
+            modified = email.utils.formatdate(stats.st_mtime, usegmt=True)
+            mtype = mimetypes.guess_type(filename)[0]
+            headers = mimetools.Message(StringIO(
+                'Content-type: %s\nContent-length: %d\nLast-modified: %s\n' %
+                (mtype or 'text/plain', size, modified)))
+            if host:
+                host, port = splitport(host)
+            if not host or \
+                (not port and _safe_gethostbyname(host) in self.get_names()):
+                if host:
+                    origurl = 'file://' + host + filename
+                else:
+                    origurl = 'file://' + filename
+                return addinfourl(open(localfile, 'rb'), headers, origurl)
+        except OSError, msg:
+            # urllib2 users shouldn't expect OSErrors coming from urlopen()
+            raise URLError(msg)
+        raise URLError('file not on local host')
+
+class FTPHandler(BaseHandler):
+    def ftp_open(self, req):
+        import ftplib
+        import mimetypes
+        host = req.get_host()
+        if not host:
+            raise URLError('ftp error: no host given')
+        host, port = splitport(host)
+        if port is None:
+            port = ftplib.FTP_PORT
+        else:
+            port = int(port)
+
+        # username/password handling
+        user, host = splituser(host)
+        if user:
+            user, passwd = splitpasswd(user)
+        else:
+            passwd = None
+        host = unquote(host)
+        user = user or ''
+        passwd = passwd or ''
+
+        try:
+            host = socket.gethostbyname(host)
+        except socket.error, msg:
+            raise URLError(msg)
+        path, attrs = splitattr(req.get_selector())
+        dirs = path.split('/')
+        dirs = map(unquote, dirs)
+        dirs, file = dirs[:-1], dirs[-1]
+        if dirs and not dirs[0]:
+            dirs = dirs[1:]
+        try:
+            fw = self.connect_ftp(user, passwd, host, port, dirs, req.timeout)
+            type = file and 'I' or 'D'
+            for attr in attrs:
+                attr, value = splitvalue(attr)
+                if attr.lower() == 'type' and \
+                   value in ('a', 'A', 'i', 'I', 'd', 'D'):
+                    type = value.upper()
+            fp, retrlen = fw.retrfile(file, type)
+            headers = ""
+            mtype = mimetypes.guess_type(req.get_full_url())[0]
+            if mtype:
+                headers += "Content-type: %s\n" % mtype
+            if retrlen is not None and retrlen >= 0:
+                headers += "Content-length: %d\n" % retrlen
+            sf = StringIO(headers)
+            headers = mimetools.Message(sf)
+            return addinfourl(fp, headers, req.get_full_url())
+        except ftplib.all_errors, msg:
+            raise URLError, ('ftp error: %s' % msg), sys.exc_info()[2]
+
+    def connect_ftp(self, user, passwd, host, port, dirs, timeout):
+        fw = ftpwrapper(user, passwd, host, port, dirs, timeout)
+##        fw.ftp.set_debuglevel(1)
+        return fw
+
+class CacheFTPHandler(FTPHandler):
+    # XXX would be nice to have pluggable cache strategies
+    # XXX this stuff is definitely not thread safe
+    def __init__(self):
+        self.cache = {}
+        self.timeout = {}
+        self.soonest = 0
+        self.delay = 60
+        self.max_conns = 16
+
+    def setTimeout(self, t):
+        self.delay = t
+
+    def setMaxConns(self, m):
+        self.max_conns = m
+
+    def connect_ftp(self, user, passwd, host, port, dirs, timeout):
+        key = user, host, port, '/'.join(dirs), timeout
+        if key in self.cache:
+            self.timeout[key] = time.time() + self.delay
+        else:
+            self.cache[key] = ftpwrapper(user, passwd, host, port, dirs, timeout)
+            self.timeout[key] = time.time() + self.delay
+        self.check_cache()
+        return self.cache[key]
+
+    def check_cache(self):
+        # first check for old ones
+        t = time.time()
+        if self.soonest <= t:
+            for k, v in self.timeout.items():
+                if v < t:
+                    self.cache[k].close()
+                    del self.cache[k]
+                    del self.timeout[k]
+        self.soonest = min(self.timeout.values())
+
+        # then check the size
+        if len(self.cache) == self.max_conns:
+            for k, v in self.timeout.items():
+                if v == self.soonest:
+                    del self.cache[k]
+                    del self.timeout[k]
+                    break
+            self.soonest = min(self.timeout.values())
diff --git a/lib_pypy/resource.py b/lib_pypy/resource.py
--- a/lib_pypy/resource.py
+++ b/lib_pypy/resource.py
@@ -7,7 +7,7 @@
 
 from ctypes_support import standard_c_lib as libc
 from ctypes_support import get_errno
-from ctypes import Structure, c_int, c_long, byref, sizeof, POINTER
+from ctypes import Structure, c_int, c_long, byref, POINTER
 from errno import EINVAL, EPERM
 import _structseq
 
@@ -165,7 +165,6 @@
 
 @builtinify
 def getpagesize():
-    pagesize = 0
     if _getpagesize:
         return _getpagesize()
     else:
diff --git a/pypy/annotation/classdef.py b/pypy/annotation/classdef.py
--- a/pypy/annotation/classdef.py
+++ b/pypy/annotation/classdef.py
@@ -276,8 +276,8 @@
         # create the Attribute and do the generalization asked for
         newattr = Attribute(attr, self.bookkeeper)
         if s_value:
-            if newattr.name == 'intval' and getattr(s_value, 'unsigned', False):
-                import pdb; pdb.set_trace()
+            #if newattr.name == 'intval' and getattr(s_value, 'unsigned', False):
+            #    import pdb; pdb.set_trace()
             newattr.s_value = s_value
 
         # keep all subattributes' values
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -127,7 +127,7 @@
 
 pypy_optiondescription = OptionDescription("objspace", "Object Space Options", [
     ChoiceOption("name", "Object Space name",
-                 ["std", "flow", "thunk", "dump", "taint"],
+                 ["std", "flow", "thunk", "dump"],
                  "std",
                  cmdline='--objspace -o'),
 
diff --git a/pypy/doc/__pypy__-module.rst b/pypy/doc/__pypy__-module.rst
--- a/pypy/doc/__pypy__-module.rst
+++ b/pypy/doc/__pypy__-module.rst
@@ -37,29 +37,6 @@
 .. _`thunk object space docs`: objspace-proxies.html#thunk
 .. _`interface section of the thunk object space docs`: objspace-proxies.html#thunk-interface
 
-.. broken:
-
-    Taint Object Space Functionality
-    ================================
-
-    When the taint object space is used (choose with :config:`objspace.name`),
-    the following names are put into ``__pypy__``:
-
-     - ``taint``
-     - ``is_tainted``
-     - ``untaint``
-     - ``taint_atomic``
-     - ``_taint_debug``
-     - ``_taint_look``
-     - ``TaintError``
-
-    Those are all described in the `interface section of the taint object space
-    docs`_.
-
-    For more detailed explanations and examples see the `taint object space docs`_.
-
-    .. _`taint object space docs`: objspace-proxies.html#taint
-    .. _`interface section of the taint object space docs`: objspace-proxies.html#taint-interface
 
 Transparent Proxy Functionality
 ===============================
diff --git a/pypy/doc/config/objspace.name.txt b/pypy/doc/config/objspace.name.txt
--- a/pypy/doc/config/objspace.name.txt
+++ b/pypy/doc/config/objspace.name.txt
@@ -4,7 +4,6 @@
 for normal usage):
 
   * thunk_: The thunk object space adds lazy evaluation to PyPy.
-  * taint_: The taint object space adds soft security features.
   * dump_:  Using this object spaces results in the dumpimp of all operations
     to a log.
 
@@ -12,5 +11,4 @@
 .. _`Object Space Proxies`: ../objspace-proxies.html
 .. _`Standard Object Space`: ../objspace.html#standard-object-space
 .. _thunk: ../objspace-proxies.html#thunk
-.. _taint: ../objspace-proxies.html#taint
 .. _dump: ../objspace-proxies.html#dump
diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst
--- a/pypy/doc/index.rst
+++ b/pypy/doc/index.rst
@@ -309,7 +309,6 @@
 .. _`object space`: objspace.html
 .. _FlowObjSpace: objspace.html#the-flow-object-space 
 .. _`trace object space`: objspace.html#the-trace-object-space 
-.. _`taint object space`: objspace-proxies.html#taint
 .. _`thunk object space`: objspace-proxies.html#thunk
 .. _`transparent proxies`: objspace-proxies.html#tproxy
 .. _`Differences between PyPy and CPython`: cpython_differences.html
diff --git a/pypy/doc/objspace-proxies.rst b/pypy/doc/objspace-proxies.rst
--- a/pypy/doc/objspace-proxies.rst
+++ b/pypy/doc/objspace-proxies.rst
@@ -129,297 +129,6 @@
    function behaves lazily: all calls to it return a thunk object.
 
 
-.. broken right now:
-
-    .. _taint:
-
-    The Taint Object Space
-    ======================
-
-    Motivation
-    ----------
-
-    The Taint Object Space provides a form of security: "tainted objects",
-    inspired by various sources, see [D12.1]_ for a more detailed discussion. 
-
-    The basic idea of this kind of security is not to protect against
-    malicious code but to help with handling and boxing sensitive data. 
-    It covers two kinds of sensitive data: secret data which should not leak, 
-    and untrusted data coming from an external source and that must be 
-    validated before it is used.
-
-    The idea is that, considering a large application that handles these
-    kinds of sensitive data, there are typically only a small number of
-    places that need to explicitly manipulate that sensitive data; all the
-    other places merely pass it around, or do entirely unrelated things.
-
-    Nevertheless, if a large application needs to be reviewed for security,
-    it must be entirely carefully checked, because it is possible that a
-    bug at some apparently unrelated place could lead to a leak of sensitive
-    information in a way that an external attacker could exploit.  For
-    example, if any part of the application provides web services, an
-    attacker might be able to issue unexpected requests with a regular web
-    browser and deduce secret information from the details of the answers he
-    gets.  Another example is the common CGI attack where an attacker sends
-    malformed inputs and causes the CGI script to do unintended things.
-
-    An approach like that of the Taint Object Space allows the small parts
-    of the program that manipulate sensitive data to be explicitly marked.
-    The effect of this is that although these small parts still need a
-    careful security review, the rest of the application no longer does,
-    because even a bug would be unable to leak the information.
-
-    We have implemented a simple two-level model: objects are either
-    regular (untainted), or sensitive (tainted).  Objects are marked as
-    sensitive if they are secret or untrusted, and only declassified at
-    carefully-checked positions (e.g. where the secret data is needed, or
-    after the untrusted data has been fully validated).
-
-    It would be simple to extend the code for more fine-grained scales of
-    secrecy.  For example it is typical in the literature to consider
-    user-specified lattices of secrecy levels, corresponding to multiple
-    "owners" that cannot access data belonging to another "owner" unless
-    explicitly authorized to do so.
-
-    Tainting and untainting
-    -----------------------
-
-    Start a py.py with the Taint Object Space and try the following example::
-
-        $ py.py -o taint
-        >>>> from __pypy__ import taint
-        >>>> x = taint(6)
-
-        # x is hidden from now on.  We can pass it around and
-        # even operate on it, but not inspect it.  Taintness
-        # is propagated to operation results.
-
-        >>>> x
-        TaintError
-
-        >>>> if x > 5: y = 2   # see below
-        TaintError
-
-        >>>> y = x + 5         # ok
-        >>>> lst = [x, y]
-        >>>> z = lst.pop()
-        >>>> t = type(z)       # type() works too, tainted answer
-        >>>> t
-        TaintError
-        >>>> u = t is int      # even 'is' works
-        >>>> u
-        TaintError
-
-    Notice that using a tainted boolean like ``x > 5`` in an ``if``
-    statement is forbidden.  This is because knowing which path is followed
-    would give away a hint about ``x``; in the example above, if the
-    statement ``if x > 5: y = 2`` was allowed to run, we would know
-    something about the value of ``x`` by looking at the (untainted) value
-    in the variable ``y``.
-
-    Of course, there is a way to inspect tainted objects.  The basic way is
-    to explicitly "declassify" it with the ``untaint()`` function.  In an
-    application, the places that use ``untaint()`` are the places that need
-    careful security review.  To avoid unexpected objects showing up, the
-    ``untaint()`` function must be called with the exact type of the object
-    to declassify.  It will raise ``TaintError`` if the type doesn't match::
-
-        >>>> from __pypy__ import taint
-        >>>> untaint(int, x)
-        6
-        >>>> untaint(int, z)
-        11
-        >>>> untaint(bool, x > 5)
-        True
-        >>>> untaint(int, x > 5)
-        TaintError
-
-
-    Taint Bombs
-    -----------
-
-    In this area, a common problem is what to do about failing operations.
-    If an operation raises an exception when manipulating a tainted object,
-    then the very presence of the exception can leak information about the
-    tainted object itself.  Consider::
-
-        >>>> 5 / (x-6)
-
-    By checking if this raises ``ZeroDivisionError`` or not, we would know
-    if ``x`` was equal to 6 or not.  The solution to this problem in the
-    Taint Object Space is to introduce *Taint Bombs*.  They are a kind of
-    tainted object that doesn't contain a real object, but a pending
-    exception.  Taint Bombs are indistinguishable from normal tainted
-    objects to unprivileged code. See::
-
-        >>>> x = taint(6)
-        >>>> i = 5 / (x-6)     # no exception here
-        >>>> j = i + 1         # nor here
-        >>>> k = j + 5         # nor here
-        >>>> untaint(int, k)
-        TaintError
-
-    In the above example, all of ``i``, ``j`` and ``k`` contain a Taint
-    Bomb.  Trying to untaint it raises an exception - a generic
-    ``TaintError``.  What we win is that the exception gives little away,
-    and most importantly it occurs at the point where ``untaint()`` is
-    called, not where the operation failed.  This means that all calls to
-    ``untaint()`` - but not the rest of the code - must be carefully
-    reviewed for what occurs if they receive a Taint Bomb; they might catch
-    the ``TaintError`` and give the user a generic message that something
-    went wrong, if we are reasonably careful that the message or even its
-    presence doesn't give information away.  This might be a
-    problem by itself, but there is no satisfying general solution here:
-    it must be considered on a case-by-case basis.  Again, what the
-    Taint Object Space approach achieves is not solving these problems, but
-    localizing them to well-defined small parts of the application - namely,
-    around calls to ``untaint()``.
-
-    The ``TaintError`` exception deliberately does not include any
-    useful error messages, because they might give information away.
-    Of course, this makes debugging quite a bit harder; a difficult
-    problem to solve properly.  So far we have implemented a way to peek in a Taint
-    Box or Bomb, ``__pypy__._taint_look(x)``, and a "debug mode" that
-    prints the exception as soon as a Bomb is created - both write
-    information to the low-level stderr of the application, where we hope
-    that it is unlikely to be seen by anyone but the application
-    developer.
-
-
-    Taint Atomic functions
-    ----------------------
-
-    Occasionally, a more complicated computation must be performed on a
-    tainted object.  This requires first untainting the object, performing the
-    computations, and then carefully tainting the result again (including
-    hiding all exceptions into Bombs).
-
-    There is a built-in decorator that does this for you::
-
-        >>>> @__pypy__.taint_atomic
-        >>>> def myop(x, y):
-        ....     while x > 0:
-        ....         x -= y
-        ....     return x
-        ....
-        >>>> myop(42, 10)
-        -8
-        >>>> z = myop(taint(42), 10)
-        >>>> z
-        TaintError
-        >>>> untaint(int, z)
-        -8
-
-    The decorator makes a whole function behave like a built-in operation.
-    If no tainted argument is passed in, the function behaves normally.  But
-    if any of the arguments is tainted, it is automatically untainted - so
-    the function body always sees untainted arguments - and the eventual
-    result is tainted again (possibly in a Taint Bomb).
-
-    It is important for the function marked as ``taint_atomic`` to have no
-    visible side effects, as these could cause information leakage.
-    This is currently not enforced, which means that all ``taint_atomic``
-    functions have to be carefully reviewed for security (but not the
-    callers of ``taint_atomic`` functions).
-
-    A possible future extension would be to forbid side-effects on
-    non-tainted objects from all ``taint_atomic`` functions.
-
-    An example of usage: given a tainted object ``passwords_db`` that
-    references a database of passwords, we can write a function
-    that checks if a password is valid as follows::
-
-        @taint_atomic
-        def validate(passwords_db, username, password):
-            assert type(passwords_db) is PasswordDatabase
-            assert type(username) is str
-            assert type(password) is str
-            ...load username entry from passwords_db...
-            return expected_password == password
-
-    It returns a tainted boolean answer, or a Taint Bomb if something
-    went wrong.  A caller can do::
-
-        ok = validate(passwords_db, 'john', '1234')
-        ok = untaint(bool, ok)
-
-    This can give three outcomes: ``True``, ``False``, or a ``TaintError``
-    exception (with no information on it) if anything went wrong.  If even
-    this is considered giving too much information away, the ``False`` case
-    can be made indistinguishable from the ``TaintError`` case (simply by
-    raising an exception in ``validate()`` if the password is wrong).
-
-    In the above example, the security results achieved are the following:
-    as long as ``validate()`` does not leak information, no other part of
-    the code can obtain more information about a passwords database than a
-    Yes/No answer to a precise query.
-
-    A possible extension of the ``taint_atomic`` decorator would be to check
-    the argument types, as ``untaint()`` does, for the same reason: to
-    prevent bugs where a function like ``validate()`` above is accidentally
-    called with the wrong kind of tainted object, which would make it
-    misbehave.  For now, all ``taint_atomic`` functions should be
-    conservative and carefully check all assumptions on their input
-    arguments.
-
-
-    .. _`taint-interface`:
-
-    Interface
-    ---------
-
-    .. _`like a built-in operation`:
-
-    The basic rule of the Tainted Object Space is that it introduces two new
-    kinds of objects, Tainted Boxes and Tainted Bombs (which are not types
-    in the Python sense).  Each box internally contains a regular object;
-    each bomb internally contains an exception object.  An operation
-    involving Tainted Boxes is performed on the objects contained in the
-    boxes, and gives a Tainted Box or a Tainted Bomb as a result (such an
-    operation does not let an exception be raised).  An operation called
-    with a Tainted Bomb argument immediately returns the same Tainted Bomb.
-
-    In a PyPy running with (or translated with) the Taint Object Space,
-    the ``__pypy__`` module exposes the following interface:
-
-    * ``taint(obj)``
-
-        Return a new Tainted Box wrapping ``obj``.  Return ``obj`` itself
-        if it is already tainted (a Box or a Bomb).
-
-    * ``is_tainted(obj)``
-
-        Check if ``obj`` is tainted (a Box or a Bomb).
-
-    * ``untaint(type, obj)``
-
-        Untaints ``obj`` if it is tainted.  Raise ``TaintError`` if the type
-        of the untainted object is not exactly ``type``, or if ``obj`` is a
-        Bomb.
-
-    * ``taint_atomic(func)``
-
-        Return a wrapper function around the callable ``func``.  The wrapper
-        behaves `like a built-in operation`_ with respect to untainting the
-        arguments, tainting the result, and returning a Bomb.
-
-    * ``TaintError``
-
-        Exception.  On purpose, it provides no attribute or error message.
-
-    * ``_taint_debug(level)``
-
-        Set the debugging level to ``level`` (0=off).  At level 1 or above,
-        all Taint Bombs print a diagnostic message to stderr when they are
-        created.
-
-    * ``_taint_look(obj)``
-
-        For debugging purposes: prints (to stderr) the type and address of
-        the object in a Tainted Box, or prints the exception if ``obj`` is
-        a Taint Bomb.
-
-
 .. _dump:
 
 The Dump Object Space
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -391,8 +391,11 @@
     def decrement_ticker(self, by):
         value = self._ticker
         if self.has_bytecode_counter:    # this 'if' is constant-folded
-            value -= by
-            self._ticker = value
+            if jit.isconstant(by) and by == 0:
+                pass     # normally constant-folded too
+            else:
+                value -= by
+                self._ticker = value
         return value
 
 
diff --git a/pypy/interpreter/pyparser/pytokenizer.py b/pypy/interpreter/pyparser/pytokenizer.py
--- a/pypy/interpreter/pyparser/pytokenizer.py
+++ b/pypy/interpreter/pyparser/pytokenizer.py
@@ -226,7 +226,7 @@
                         parenlev = parenlev - 1
                         if parenlev < 0:
                             raise TokenError("unmatched '%s'" % initial, line,
-                                             lnum-1, 0, token_list)
+                                             lnum, start + 1, token_list)
                     if token in python_opmap:
                         punct = python_opmap[token]
                     else:
diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py
--- a/pypy/interpreter/pyparser/test/test_pyparse.py
+++ b/pypy/interpreter/pyparser/test/test_pyparse.py
@@ -87,6 +87,10 @@
         assert exc.lineno == 1
         assert exc.offset == 5
         assert exc.lastlineno == 5
+        exc = py.test.raises(SyntaxError, parse, "abc)").value
+        assert exc.msg == "unmatched ')'"
+        assert exc.lineno == 1
+        assert exc.offset == 4
 
     def test_is(self):
         self.parse("x is y")
diff --git a/pypy/jit/backend/llgraph/llimpl.py b/pypy/jit/backend/llgraph/llimpl.py
--- a/pypy/jit/backend/llgraph/llimpl.py
+++ b/pypy/jit/backend/llgraph/llimpl.py
@@ -165,6 +165,7 @@
     'unicodegetitem'  : (('ref', 'int'), 'int'),
     'unicodesetitem'  : (('ref', 'int', 'int'), 'int'),
     'cast_ptr_to_int' : (('ref',), 'int'),
+    'cast_int_to_ptr' : (('int',), 'ref'),
     'debug_merge_point': (('ref', 'int'), None),
     'force_token'     : ((), 'int'),
     'call_may_force'  : (('int', 'varargs'), 'intorptr'),
@@ -875,9 +876,6 @@
     def op_new_array(self, arraydescr, count):
         return do_new_array(arraydescr.ofs, count)
 
-    def op_cast_ptr_to_int(self, descr, ptr):
-        return cast_to_int(ptr)
-
     def op_force_token(self, descr):
         opaque_frame = _to_opaque(self)
         return llmemory.cast_ptr_to_adr(opaque_frame)
diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py
--- a/pypy/jit/backend/llsupport/gc.py
+++ b/pypy/jit/backend/llsupport/gc.py
@@ -45,6 +45,14 @@
     def freeing_block(self, start, stop):
         pass
 
+    def record_constptrs(self, op, gcrefs_output_list):
+        for i in range(op.numargs()):
+            v = op.getarg(i)
+            if isinstance(v, ConstPtr) and bool(v.value):
+                p = v.value
+                rgc._make_sure_does_not_move(p)
+                gcrefs_output_list.append(p)
+
 # ____________________________________________________________
 
 class GcLLDescr_boehm(GcLLDescription):
@@ -141,6 +149,14 @@
     get_funcptr_for_newstr = None
     get_funcptr_for_newunicode = None
 
+    def rewrite_assembler(self, cpu, operations, gcrefs_output_list):
+        # record all GCREFs too, because Boehm cannot see them and keep them
+        # alive if they end up as constants in the assembler
+        for op in operations:
+            self.record_constptrs(op, gcrefs_output_list)
+        return GcLLDescription.rewrite_assembler(self, cpu, operations,
+                                                 gcrefs_output_list)
+
 
 # ____________________________________________________________
 # All code below is for the hybrid or minimark GC
@@ -757,14 +773,6 @@
             funcptr(llmemory.cast_ptr_to_adr(gcref_struct),
                     llmemory.cast_ptr_to_adr(gcref_newptr))
 
-    def record_constptrs(self, op, gcrefs_output_list):
-        for i in range(op.numargs()):
-            v = op.getarg(i)
-            if isinstance(v, ConstPtr) and bool(v.value):
-                p = v.value
-                rgc._make_sure_does_not_move(p)
-                gcrefs_output_list.append(p)
-
     def rewrite_assembler(self, cpu, operations, gcrefs_output_list):
         # Perform two kinds of rewrites in parallel:
         #
diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -1468,20 +1468,16 @@
         return u''.join(u.chars)
 
 
-    def test_casts(self):
-        py.test.skip("xxx fix or kill")
-        from pypy.rpython.lltypesystem import lltype, llmemory
-        TP = lltype.GcStruct('x')
-        x = lltype.malloc(TP)        
-        x = lltype.cast_opaque_ptr(llmemory.GCREF, x)
+    def test_cast_int_to_ptr(self):
+        res = self.execute_operation(rop.CAST_INT_TO_PTR,
+                                     [BoxInt(-17)],  'ref').value
+        assert lltype.cast_ptr_to_int(res) == -17
+
+    def test_cast_ptr_to_int(self):
+        x = lltype.cast_int_to_ptr(llmemory.GCREF, -19)
         res = self.execute_operation(rop.CAST_PTR_TO_INT,
-                                     [BoxPtr(x)],  'int').value
-        expected = self.cpu.cast_adr_to_int(llmemory.cast_ptr_to_adr(x))
-        assert rffi.get_real_int(res) == rffi.get_real_int(expected)
-        res = self.execute_operation(rop.CAST_PTR_TO_INT,
-                                     [ConstPtr(x)],  'int').value
-        expected = self.cpu.cast_adr_to_int(llmemory.cast_ptr_to_adr(x))
-        assert rffi.get_real_int(res) == rffi.get_real_int(expected)
+                                     [BoxPtr(x)], 'int').value
+        assert res == -19
 
     def test_ooops_non_gc(self):
         x = lltype.malloc(lltype.Struct('x'), flavor='raw')
@@ -2299,13 +2295,6 @@
         #
         cpu.bh_strsetitem(x, 4, ord('/'))
         assert str.chars[4] == '/'
-        #
-##        x = cpu.bh_newstr(5)
-##        y = cpu.bh_cast_ptr_to_int(x)
-##        z = cpu.bh_cast_ptr_to_int(x)
-##        y = rffi.get_real_int(y)
-##        z = rffi.get_real_int(z)
-##        assert type(y) == type(z) == int and y == z
 
     def test_sorting_of_fields(self):
         S = self.S
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1387,7 +1387,8 @@
 
     def genop_same_as(self, op, arglocs, resloc):
         self.mov(arglocs[0], resloc)
-    #genop_cast_ptr_to_int = genop_same_as
+    genop_cast_ptr_to_int = genop_same_as
+    genop_cast_int_to_ptr = genop_same_as
 
     def genop_int_mod(self, op, arglocs, resloc):
         if IS_X86_32:
diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py
--- a/pypy/jit/backend/x86/regalloc.py
+++ b/pypy/jit/backend/x86/regalloc.py
@@ -1152,7 +1152,8 @@
         self.possibly_free_var(op.getarg(0))
         resloc = self.force_allocate_reg(op.result)
         self.Perform(op, [argloc], resloc)
-    #consider_cast_ptr_to_int = consider_same_as
+    consider_cast_ptr_to_int = consider_same_as
+    consider_cast_int_to_ptr = consider_same_as
 
     def consider_strlen(self, op):
         args = op.getarglist()
diff --git a/pypy/jit/backend/x86/tool/viewcode.py b/pypy/jit/backend/x86/tool/viewcode.py
--- a/pypy/jit/backend/x86/tool/viewcode.py
+++ b/pypy/jit/backend/x86/tool/viewcode.py
@@ -9,7 +9,12 @@
 """
 
 import autopath
-import operator, sys, os, re, py, new
+import new
+import operator
+import py
+import re
+import sys
+import subprocess
 from bisect import bisect_left
 
 # don't use pypy.tool.udir here to avoid removing old usessions which
@@ -44,14 +49,16 @@
     f = open(tmpfile, 'wb')
     f.write(data)
     f.close()
-    g = os.popen(objdump % {
+    p = subprocess.Popen(objdump % {
         'file': tmpfile,
         'origin': originaddr,
         'backend': objdump_backend_option[backend_name],
-    }, 'r')
-    result = g.readlines()
-    g.close()
-    lines = result[6:]   # drop some objdump cruft
+    }, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    stdout, stderr = p.communicate()
+    assert not p.returncode, ('Encountered an error running objdump: %s' %
+                              stderr)
+    # drop some objdump cruft
+    lines = stdout.splitlines()[6:]
     return format_code_dump_with_labels(originaddr, lines, label_list)
 
 def format_code_dump_with_labels(originaddr, lines, label_list):
@@ -85,8 +92,12 @@
     #
     print 'loading symbols from %s...' % (filename,)
     symbols = {}
-    g = os.popen(symbollister % filename, "r")
-    for line in g:
+    p = subprocess.Popen(symbollister % filename, shell=True,
+                         stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    stdout, stderr = p.communicate()
+    assert not p.returncode, ('Encountered an error running nm: %s' %
+                              stderr)
+    for line in stdout.splitlines():
         match = re_symbolentry.match(line)
         if match:
             addr = long(match.group(1), 16)
@@ -94,7 +105,6 @@
             if name.startswith('pypy_g_'):
                 name = '\xb7' + name[7:]
             symbols[addr] = name
-    g.close()
     print '%d symbols found' % (len(symbols),)
     return symbols
 
diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -455,6 +455,23 @@
             # the special return value None forces op.result to be considered
             # equal to op.args[0]
             return [op0, op1, None]
+        if (hints.get('promote_string') and
+            op.args[0].concretetype is not lltype.Void):
+            S = lltype.Ptr(rstr.STR)
+            assert op.args[0].concretetype == S
+            self._register_extra_helper(EffectInfo.OS_STREQ_NONNULL,
+                                        "str.eq_nonnull",
+                                        [S, S],
+                                        lltype.Signed,
+                                        EffectInfo.EF_ELIDABLE_CANNOT_RAISE)
+            descr, p = self.callcontrol.callinfocollection.callinfo_for_oopspec(
+                EffectInfo.OS_STREQ_NONNULL)
+            # XXX this is fairly ugly way of creating a constant,
+            #     however, callinfocollection has no better interface
+            c = Constant(p.adr.ptr, lltype.typeOf(p.adr.ptr))
+            op1 = SpaceOperation('str_guard_value', [op.args[0], c, descr],
+                                 op.result)
+            return [SpaceOperation('-live-', [], None), op1, None]
         else:
             log.WARNING('ignoring hint %r at %r' % (hints, self.graph))
 
@@ -783,8 +800,7 @@
 
     def rewrite_op_cast_ptr_to_int(self, op):
         if self._is_gc(op.args[0]):
-            #return op
-            raise NotImplementedError("cast_ptr_to_int")
+            return op
 
     def rewrite_op_force_cast(self, op):
         v_arg = op.args[0]
@@ -1526,6 +1542,10 @@
     def rewrite_op_jit_force_virtual(self, op):
         return self._do_builtin_call(op)
 
+    def rewrite_op_jit_is_virtual(self, op):
+        raise Exception, (
+            "'vref.virtual' should not be used from jit-visible code")
+
     def rewrite_op_jit_force_virtualizable(self, op):
         # this one is for virtualizables
         vinfo = self.get_vinfo(op.args[0])
diff --git a/pypy/jit/codewriter/test/test_jtransform.py b/pypy/jit/codewriter/test/test_jtransform.py
--- a/pypy/jit/codewriter/test/test_jtransform.py
+++ b/pypy/jit/codewriter/test/test_jtransform.py
@@ -99,6 +99,12 @@
             if i == oopspecindex:
                 return True
         return False
+    def callinfo_for_oopspec(self, oopspecindex):
+        assert oopspecindex == effectinfo.EffectInfo.OS_STREQ_NONNULL
+        class c:
+            class adr:
+                ptr = 1
+        return ('calldescr', c)
 
 class FakeBuiltinCallControl:
     def __init__(self):
@@ -119,6 +125,7 @@
              EI.OS_STR2UNICODE:([PSTR], PUNICODE),
              EI.OS_STR_CONCAT: ([PSTR, PSTR], PSTR),
              EI.OS_STR_SLICE:  ([PSTR, INT, INT], PSTR),
+             EI.OS_STREQ_NONNULL:  ([PSTR, PSTR], INT),
              EI.OS_UNI_CONCAT: ([PUNICODE, PUNICODE], PUNICODE),
              EI.OS_UNI_SLICE:  ([PUNICODE, INT, INT], PUNICODE),
              EI.OS_UNI_EQUAL:  ([PUNICODE, PUNICODE], lltype.Bool),
@@ -844,6 +851,21 @@
     assert op1.args[2] == ListOfKind('ref', [v1, v2])
     assert op1.result == v3
 
+def test_str_promote():
+    PSTR = lltype.Ptr(rstr.STR)
+    v1 = varoftype(PSTR)
+    v2 = varoftype(PSTR)
+    op = SpaceOperation('hint',
+                        [v1, Constant({'promote_string': True}, lltype.Void)],
+                        v2)
+    tr = Transformer(FakeCPU(), FakeBuiltinCallControl())
+    op0, op1, _ = tr.rewrite_operation(op)
+    assert op1.opname == 'str_guard_value'
+    assert op1.args[0] == v1
+    assert op1.args[2] == 'calldescr'
+    assert op1.result == v2
+    assert op0.opname == '-live-'
+
 def test_unicode_concat():
     # test that the oopspec is present and correctly transformed
     PSTR = lltype.Ptr(rstr.UNICODE)
diff --git a/pypy/jit/metainterp/blackhole.py b/pypy/jit/metainterp/blackhole.py
--- a/pypy/jit/metainterp/blackhole.py
+++ b/pypy/jit/metainterp/blackhole.py
@@ -2,7 +2,7 @@
 from pypy.rlib.rtimer import read_timestamp
 from pypy.rlib.rarithmetic import intmask, LONG_BIT, r_uint, ovfcheck
 from pypy.rlib.objectmodel import we_are_translated
-from pypy.rlib.debug import debug_start, debug_stop
+from pypy.rlib.debug import debug_start, debug_stop, ll_assert
 from pypy.rlib.debug import make_sure_not_resized
 from pypy.rpython.lltypesystem import lltype, llmemory, rclass
 from pypy.rpython.lltypesystem.lloperation import llop
@@ -503,6 +503,15 @@
     @arguments("r", returns="r")
     def bhimpl_cast_opaque_ptr(a):
         return a
+    @arguments("r", returns="i")
+    def bhimpl_cast_ptr_to_int(a):
+        i = lltype.cast_ptr_to_int(a)
+        ll_assert((i & 1) == 1, "bhimpl_cast_ptr_to_int: not an odd int")
+        return i
+    @arguments("i", returns="r")
+    def bhimpl_cast_int_to_ptr(i):
+        ll_assert((i & 1) == 1, "bhimpl_cast_int_to_ptr: not an odd int")
+        return lltype.cast_int_to_ptr(llmemory.GCREF, i)
 
     @arguments("i", returns="i")
     def bhimpl_int_copy(a):
@@ -523,6 +532,9 @@
     @arguments("f")
     def bhimpl_float_guard_value(a):
         pass
+    @arguments("r", "i", "d", returns="r")
+    def bhimpl_str_guard_value(a, i, d):
+        return a
 
     @arguments("self", "i")
     def bhimpl_int_push(self, a):
diff --git a/pypy/jit/metainterp/heapcache.py b/pypy/jit/metainterp/heapcache.py
--- a/pypy/jit/metainterp/heapcache.py
+++ b/pypy/jit/metainterp/heapcache.py
@@ -54,9 +54,10 @@
         if box in self.new_boxes:
             self.new_boxes[box] = False
         if box in self.dependencies:
-            for dep in self.dependencies[box]:
+            deps = self.dependencies[box]
+            del self.dependencies[box]
+            for dep in deps:
                 self._escape(dep)
-            del self.dependencies[box]
 
     def clear_caches(self, opnum, descr, argboxes):
         if opnum == rop.SETFIELD_GC:
diff --git a/pypy/jit/metainterp/history.py b/pypy/jit/metainterp/history.py
--- a/pypy/jit/metainterp/history.py
+++ b/pypy/jit/metainterp/history.py
@@ -9,6 +9,7 @@
 
 from pypy.jit.metainterp.resoperation import ResOperation, rop
 from pypy.jit.codewriter import heaptracker, longlong
+from pypy.rlib.objectmodel import compute_identity_hash
 
 # ____________________________________________________________
 
@@ -104,7 +105,7 @@
     getref._annspecialcase_ = 'specialize:arg(1)'
 
     def _get_hash_(self):
-        raise NotImplementedError
+        return compute_identity_hash(self)
 
     def clonebox(self):
         raise NotImplementedError
@@ -133,6 +134,9 @@
     def _get_str(self):
         raise NotImplementedError
 
+    def same_box(self, other):
+        return self is other
+
 class AbstractDescr(AbstractValue):
     __slots__ = ()
 
@@ -241,32 +245,15 @@
     def constbox(self):
         return self
 
+    def same_box(self, other):
+        return self.same_constant(other)
+
     def same_constant(self, other):
         raise NotImplementedError
 
     def __repr__(self):
         return 'Const(%s)' % self._getrepr_()
 
-    def __eq__(self, other):
-        "NOT_RPYTHON"
-        # Remember that you should not compare Consts with '==' in RPython.
-        # Consts have no special __hash__, in order to force different Consts
-        # from being considered as different keys when stored in dicts
-        # (as they always are after translation).  Use a dict_equal_consts()
-        # to get the other behavior (i.e. using this __eq__).
-        if self.__class__ is not other.__class__:
-            return False
-        try:
-            return self.value == other.value
-        except TypeError:
-            if (isinstance(self.value, Symbolic) and
-                isinstance(other.value, Symbolic)):
-                return self.value is other.value
-            raise
-
-    def __ne__(self, other):
-        return not (self == other)
-
 
 class ConstInt(Const):
     type = INT
@@ -688,33 +675,6 @@
 
 # ____________________________________________________________
 
-def dict_equal_consts():
-    "NOT_RPYTHON"
-    # Returns a dict in which Consts that compare as equal
-    # are identified when used as keys.
-    return r_dict(dc_eq, dc_hash)
-
-def dc_eq(c1, c2):
-    return c1 == c2
-
-def dc_hash(c):
-    "NOT_RPYTHON"
-    # This is called during translation only.  Avoid using identityhash(),
-    # to avoid forcing a hash, at least on lltype objects.
-    if not isinstance(c, Const):
-        return hash(c)
-    if isinstance(c.value, Symbolic):
-        return id(c.value)
-    try:
-        if isinstance(c, ConstPtr):
-            p = lltype.normalizeptr(c.value)
-            if p is not None:
-                return hash(p._obj)
-            else:
-                return 0
-        return c._get_hash_()
-    except lltype.DelayedPointer:
-        return -2      # xxx risk of changing hash...
 
 def make_hashable_int(i):
     from pypy.rpython.lltypesystem.ll2ctypes import NotCtypesAllocatedStructure
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -2329,7 +2329,7 @@
         def _variables_equal(box, varname, strict):
             if varname not in virtuals:
                 if strict:
-                    assert box == oparse.getvar(varname)
+                    assert box.same_box(oparse.getvar(varname))
                 else:
                     assert box.value == oparse.getvar(varname).value
             else:
diff --git a/pypy/jit/metainterp/optimizeopt/util.py b/pypy/jit/metainterp/optimizeopt/util.py
--- a/pypy/jit/metainterp/optimizeopt/util.py
+++ b/pypy/jit/metainterp/optimizeopt/util.py
@@ -90,14 +90,11 @@
     for i in range(len(args1)):
         arg1 = args1[i]
         arg2 = args2[i]
-        if isinstance(arg1, history.Const):
-            if arg1.__class__ is not arg2.__class__:
+        if arg1 is None:
+            if arg2 is not None:
                 return False
-            if not arg1.same_constant(arg2):
-                return False
-        else:
-            if not arg1 is arg2:
-                return False
+        elif not arg1.same_box(arg2):
+            return False
     return True
 
 def args_hash(args):
@@ -106,10 +103,8 @@
     for arg in args:
         if arg is None:
             y = 17
-        elif isinstance(arg, history.Const):
+        else:
             y = arg._get_hash_()
-        else:
-            y = compute_identity_hash(arg)
         res = intmask((1000003 * res) ^ y)
     return res
 
@@ -145,9 +140,12 @@
         for i in range(op1.numargs()):
             x = op1.getarg(i)
             y = op2.getarg(i)
-            assert x == remap.get(y, y)
+            assert x.same_box(remap.get(y, y))
         if op2.result in remap:
-            assert op1.result == remap[op2.result]
+            if op2.result is None:
+                assert op1.result == remap[op2.result]
+            else:
+                assert op1.result.same_box(remap[op2.result])
         else:
             remap[op2.result] = op1.result
         if op1.getopnum() != rop.JUMP:      # xxx obscure
@@ -156,11 +154,20 @@
             assert len(op1.getfailargs()) == len(op2.getfailargs())
             if strict_fail_args:
                 for x, y in zip(op1.getfailargs(), op2.getfailargs()):
-                    assert x == remap.get(y, y)
+                    if x is None:
+                        assert remap.get(y, y) is None
+                    else:
+                        assert x.same_box(remap.get(y, y))
             else:
                 fail_args1 = set(op1.getfailargs())
                 fail_args2 = set([remap.get(y, y) for y in op2.getfailargs()])
-                assert fail_args1 == fail_args2
+                for x in fail_args1:
+                    for y in fail_args2:
+                        if x.same_box(y):
+                            fail_args2.remove(y)
+                            break
+                    else:
+                        assert False
     assert len(oplist1) == len(oplist2)
     print '-'*totwidth
     return True
diff --git a/pypy/jit/metainterp/optimizeopt/vstring.py b/pypy/jit/metainterp/optimizeopt/vstring.py
--- a/pypy/jit/metainterp/optimizeopt/vstring.py
+++ b/pypy/jit/metainterp/optimizeopt/vstring.py
@@ -587,10 +587,7 @@
             return True
         #
         if v1.is_nonnull() and v2.is_nonnull():
-            if l1box is not None and l2box is not None and (
-                l1box == l2box or (isinstance(l1box, ConstInt) and
-                                   isinstance(l2box, ConstInt) and
-                                   l1box.value == l2box.value)):
+            if l1box is not None and l2box is not None and l1box.same_box(l2box):
                 do = EffectInfo.OS_STREQ_LENGTHOK
             else:
                 do = EffectInfo.OS_STREQ_NONNULL
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -162,15 +162,18 @@
             if registers[i] is oldbox:
                 registers[i] = newbox
         if not we_are_translated():
-            assert oldbox not in registers[count:]
+            for b in registers[count:]:
+                assert not oldbox.same_box(b)
+                
 
     def make_result_of_lastop(self, resultbox):
         got_type = resultbox.type
-        if not we_are_translated():
-            typeof = {'i': history.INT,
-                      'r': history.REF,
-                      'f': history.FLOAT}
-            assert typeof[self.jitcode._resulttypes[self.pc]] == got_type
+        # XXX disabled for now, conflicts with str_guard_value
+        #if not we_are_translated():
+        #    typeof = {'i': history.INT,
+        #              'r': history.REF,
+        #              'f': history.FLOAT}
+        #    assert typeof[self.jitcode._resulttypes[self.pc]] == got_type
         target_index = ord(self.bytecode[self.pc-1])
         if got_type == history.INT:
             self.registers_i[target_index] = resultbox
@@ -219,6 +222,7 @@
                     'cast_float_to_int', 'cast_int_to_float',
                     'cast_float_to_singlefloat', 'cast_singlefloat_to_float',
                     'float_neg', 'float_abs',
+                    'cast_ptr_to_int', 'cast_int_to_ptr',
                     ]:
         exec py.code.Source('''
             @arguments("box")
@@ -895,6 +899,21 @@
     def _opimpl_guard_value(self, orgpc, box):
         self.implement_guard_value(orgpc, box)
 
+    @arguments("orgpc", "box", "box", "descr")
+    def opimpl_str_guard_value(self, orgpc, box, funcbox, descr):
+        if isinstance(box, Const):
+            return box     # no promotion needed, already a Const
+        else:
+            constbox = box.constbox()
+            resbox = self.do_residual_call(funcbox, descr, [box, constbox])
+            promoted_box = resbox.constbox()
+            # This is GUARD_VALUE because GUARD_TRUE assumes the existance
+            # of a label when computing resumepc
+            self.generate_guard(rop.GUARD_VALUE, resbox, [promoted_box],
+                                resumepc=orgpc)
+            self.metainterp.replace_box(box, constbox)
+            return constbox
+
     opimpl_int_guard_value = _opimpl_guard_value
     opimpl_ref_guard_value = _opimpl_guard_value
     opimpl_float_guard_value = _opimpl_guard_value
diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py
--- a/pypy/jit/metainterp/resoperation.py
+++ b/pypy/jit/metainterp/resoperation.py
@@ -431,6 +431,8 @@
     'INT_IS_TRUE/1b',
     'INT_NEG/1',
     'INT_INVERT/1',
+    'CAST_PTR_TO_INT/1',
+    'CAST_INT_TO_PTR/1',
     #
     'SAME_AS/1',      # gets a Const or a Box, turns it into another Box
     #
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -13,7 +13,7 @@
 from pypy.rlib.jit import (JitDriver, we_are_jitted, hint, dont_look_inside,
     loop_invariant, elidable, promote, jit_debug, assert_green,
     AssertGreenFailed, unroll_safe, current_trace_length, look_inside_iff,
-    isconstant, isvirtual)
+    isconstant, isvirtual, promote_string)
 from pypy.rlib.rarithmetic import ovfcheck
 from pypy.rpython.lltypesystem import lltype, llmemory, rffi
 from pypy.rpython.ootypesystem import ootype
@@ -3440,7 +3440,6 @@
 
 class TestLLtype(BaseLLtypeTests, LLJitMixin):
     def test_tagged(self):
-        py.test.skip("implement me")
         from pypy.rlib.objectmodel import UnboxedValue
         class Base(object):
             __slots__ = ()
@@ -3492,3 +3491,35 @@
                 pc += 1
             return pc
         res = self.meta_interp(main, [False, 100, True], taggedpointers=True)
+
+    def test_rerased(self):
+        from pypy.rlib.rerased import erase_int, unerase_int, new_erasing_pair
+        eraseX, uneraseX = new_erasing_pair("X")
+        #
+        class X:
+            def __init__(self, a, b):
+                self.a = a
+                self.b = b
+        #
+        def f(i, j):
+            # 'j' should be 0 or 1, not other values
+            if j > 0:
+                e = eraseX(X(i, j))
+            else:
+                try:
+                    e = erase_int(i)
+                except OverflowError:
+                    return -42
+            if j & 1:
+                x = uneraseX(e)
+                return x.a - x.b
+            else:
+                return unerase_int(e)
+        #
+        x = self.interp_operations(f, [-128, 0], taggedpointers=True)
+        assert x == -128
+        bigint = sys.maxint//2 + 1
+        x = self.interp_operations(f, [bigint, 0], taggedpointers=True)
+        assert x == -42
+        x = self.interp_operations(f, [1000, 1], taggedpointers=True)
+        assert x == 999
diff --git a/pypy/jit/metainterp/test/test_heapcache.py b/pypy/jit/metainterp/test/test_heapcache.py
--- a/pypy/jit/metainterp/test/test_heapcache.py
+++ b/pypy/jit/metainterp/test/test_heapcache.py
@@ -355,6 +355,14 @@
         assert not h.is_unescaped(box1)
         assert not h.is_unescaped(box2)
 
+    def test_circular_virtuals(self):
+        h = HeapCache()
+        h.new(box1)
+        h.new(box2)
+        h.invalidate_caches(rop.SETFIELD_GC, None, [box1, box2])
+        h.invalidate_caches(rop.SETFIELD_GC, None, [box2, box1])
+        h.invalidate_caches(rop.SETFIELD_GC, None, [box3, box1]) # does not crash
+
     def test_unescaped_array(self):
         h = HeapCache()
         h.new_array(box1, lengthbox1)
@@ -362,4 +370,4 @@
         h.invalidate_caches(rop.SETARRAYITEM_GC, None, [box1, index1, box2])
         assert h.is_unescaped(box1)
         h.invalidate_caches(rop.SETARRAYITEM_GC, None, [box2, index1, box1])
-        assert not h.is_unescaped(box1)
\ No newline at end of file
+        assert not h.is_unescaped(box1)
diff --git a/pypy/jit/metainterp/test/test_resume.py b/pypy/jit/metainterp/test/test_resume.py
--- a/pypy/jit/metainterp/test/test_resume.py
+++ b/pypy/jit/metainterp/test/test_resume.py
@@ -180,22 +180,27 @@
     reader.consume_boxes(info, bi, br, bf)
     b1s = reader.liveboxes[0]
     b2s = reader.liveboxes[1]
-    assert bi == [b1s, ConstInt(111), b1s]
-    assert br == [ConstPtr(gcrefnull), b2s]
+    assert_same(bi, [b1s, ConstInt(111), b1s])
+    assert_same(br, [ConstPtr(gcrefnull), b2s])
     bi, br, bf = [None]*2, [None]*0, [None]*0
     info = MyBlackholeInterp([lltype.Signed,
                               lltype.Signed]).get_current_position_info()
     reader.consume_boxes(info, bi, br, bf)
-    assert bi == [ConstInt(222), ConstInt(333)]
+    assert_same(bi, [ConstInt(222), ConstInt(333)])
     bi, br, bf = [None]*2, [None]*1, [None]*0
     info = MyBlackholeInterp([lltype.Signed, llmemory.GCREF,
                               lltype.Signed]).get_current_position_info()
     reader.consume_boxes(info, bi, br, bf)
     b3s = reader.liveboxes[2]
-    assert bi == [b1s, b3s]
-    assert br == [b2s]
+    assert_same(bi, [b1s, b3s])
+    assert_same(br, [b2s])
     #
 
+def assert_same(list1, list2):
+    assert len(list1) == len(list2)
+    for b1, b2 in zip(list1, list2):
+        assert b1.same_box(b2)
+
 def test_simple_read_tagged_ints():
     storage = Storage()
     storage.rd_consts = []
@@ -1023,11 +1028,11 @@
     metainterp = MyMetaInterp()
     reader = ResumeDataFakeReader(storage, newboxes, metainterp)
     lst = reader.consume_boxes()
-    assert lst == [b1t, b1t, b3t]
+    assert_same(lst, [b1t, b1t, b3t])
     lst = reader.consume_boxes()
-    assert lst == [ConstInt(2), ConstInt(3)]
+    assert_same(lst, [ConstInt(2), ConstInt(3)])
     lst = reader.consume_boxes()
-    assert lst == [b1t, ConstInt(1), b1t, b1t]
+    assert_same(lst, [b1t, ConstInt(1), b1t, b1t])
     assert metainterp.trace == []    
 
 
@@ -1044,11 +1049,11 @@
     reader = ResumeDataFakeReader(storage, newboxes, metainterp)
     lst = reader.consume_boxes()
     c1t = ConstInt(111)
-    assert lst == [c1t, b2t, b3t]
+    assert_same(lst, [c1t, b2t, b3t])
     lst = reader.consume_boxes()
-    assert lst == [ConstInt(2), ConstInt(3)]
+    assert_same(lst, [ConstInt(2), ConstInt(3)])
     lst = reader.consume_boxes()
-    assert lst == [c1t, ConstInt(1), c1t, b2t]
+    assert_same(lst, [c1t, ConstInt(1), c1t, b2t])
     assert metainterp.trace == []
 
 
@@ -1114,9 +1119,11 @@
     # check that we get the operations in 'expected', in a possibly different
     # order.
     assert len(trace) == len(expected)
-    for x in trace:
-        assert x in expected
-        expected.remove(x)
+    with CompareableConsts():
+        for x in trace:
+            assert x in expected
+            expected.remove(x)
+
     ptr = b2t.value._obj.container._as_ptr()
     assert lltype.typeOf(ptr) == lltype.Ptr(LLtypeMixin.NODE)
     assert ptr.value == 111
@@ -1126,6 +1133,18 @@
     assert ptr2.parent.value == 33
     assert ptr2.parent.next == ptr
 
+class CompareableConsts(object):
+    def __init__(self):
+        self.oldeq = None
+        
+    def __enter__(self):
+        assert self.oldeq is None
+        self.oldeq = Const.__eq__
+        Const.__eq__ = Const.same_box
+        
+    def __exit__(self, type, value, traceback):
+        Const.__eq__ = self.oldeq
+
 def test_virtual_adder_make_varray():
     b2s, b4s = [BoxPtr(), BoxInt(4)]
     c1s = ConstInt(111)
@@ -1163,8 +1182,9 @@
         (rop.SETARRAYITEM_GC, [b2t,ConstInt(1), c1s], None,
                               LLtypeMixin.arraydescr),
         ]
-    for x, y in zip(expected, trace):
-        assert x == y
+    with CompareableConsts():
+        for x, y in zip(expected, trace):
+            assert x == y
     #
     ptr = b2t.value._obj.container._as_ptr()
     assert lltype.typeOf(ptr) == lltype.Ptr(lltype.GcArray(lltype.Signed))
@@ -1207,8 +1227,9 @@
         (rop.SETFIELD_GC, [b2t, c1s],  None, LLtypeMixin.adescr),
         (rop.SETFIELD_GC, [b2t, b4t], None, LLtypeMixin.bdescr),
         ]
-    for x, y in zip(expected, trace):
-        assert x == y
+    with CompareableConsts():
+        for x, y in zip(expected, trace):
+            assert x == y
     #
     ptr = b2t.value._obj.container._as_ptr()
     assert lltype.typeOf(ptr) == lltype.Ptr(LLtypeMixin.S)
diff --git a/pypy/jit/metainterp/test/test_string.py b/pypy/jit/metainterp/test/test_string.py
--- a/pypy/jit/metainterp/test/test_string.py
+++ b/pypy/jit/metainterp/test/test_string.py
@@ -3,7 +3,8 @@
 from pypy.jit.codewriter.policy import StopAtXPolicy
 from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin
 from pypy.rlib.debug import debug_print
-from pypy.rlib.jit import JitDriver, dont_look_inside, we_are_jitted
+from pypy.rlib.jit import JitDriver, dont_look_inside, we_are_jitted,\
+     promote_string
 from pypy.rlib.rstring import StringBuilder
 from pypy.rpython.ootypesystem import ootype
 
@@ -507,6 +508,18 @@
                           'jump': 1, 'int_is_true': 1,
                           'guard_not_invalidated': 1})
 
+    def test_promote_string(self):
+        driver = JitDriver(greens = [], reds = ['n'])
+        
+        def f(n):
+            while n < 21:
+                driver.jit_merge_point(n=n)
+                promote_string(str(n % 3))
+                n += 1
+            return 0
+
+        self.meta_interp(f, [0])
+        self.check_loops(call=3 + 1) # one for int2str
 
 #class TestOOtype(StringTests, OOJitMixin):
 #    CALL = "oosend"
diff --git a/pypy/jit/metainterp/test/test_virtualref.py b/pypy/jit/metainterp/test/test_virtualref.py
--- a/pypy/jit/metainterp/test/test_virtualref.py
+++ b/pypy/jit/metainterp/test/test_virtualref.py
@@ -3,6 +3,7 @@
 from pypy.rpython.llinterp import LLException
 from pypy.rlib.jit import JitDriver, dont_look_inside, vref_None
 from pypy.rlib.jit import virtual_ref, virtual_ref_finish, InvalidVirtualRef
+from pypy.rlib.jit import non_virtual_ref
 from pypy.rlib.objectmodel import compute_unique_id
 from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin, _get_jitcodes
 from pypy.jit.metainterp.resoperation import rop
@@ -595,6 +596,65 @@
         res = self.meta_interp(fn, [10])
         assert res == 6
 
+    def test_is_virtual(self):
+        myjitdriver = JitDriver(greens=[], reds=['n', 'res1'])
+        class X:
+            pass
+        @dont_look_inside
+        def residual(vref):
+            return vref.virtual
+        #
+        def f(n):
+            res1 = -42
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, res1=res1)
+                x = X()
+                vref = virtual_ref(x)
+                res1 = residual(vref)
+                virtual_ref_finish(vref, x)
+                n -= 1
+            return res1
+        #
+        res = self.meta_interp(f, [10])
+        assert res == 1
+
+    def test_is_not_virtual_none(self):
+        myjitdriver = JitDriver(greens=[], reds=['n', 'res1'])
+        @dont_look_inside
+        def residual(vref):
+            return vref.virtual
+        #
+        def f(n):
+            res1 = -42
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, res1=res1)
+                res1 = residual(vref_None)
+                n -= 1
+            return res1
+        #
+        res = self.meta_interp(f, [10])
+        assert res == 0
+
+    def test_is_not_virtual_non_none(self):
+        myjitdriver = JitDriver(greens=[], reds=['n', 'res1'])
+        class X:
+            pass
+        @dont_look_inside
+        def residual(vref):
+            return vref.virtual
+        #
+        def f(n):
+            res1 = -42
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, res1=res1)
+                x = X()
+                res1 = residual(non_virtual_ref(x))
+                n -= 1
+            return res1
+        #
+        res = self.meta_interp(f, [10])
+        assert res == 0
+
 
 class TestLLtype(VRefTests, LLJitMixin):
     pass
diff --git a/pypy/jit/metainterp/virtualref.py b/pypy/jit/metainterp/virtualref.py
--- a/pypy/jit/metainterp/virtualref.py
+++ b/pypy/jit/metainterp/virtualref.py
@@ -39,6 +39,7 @@
     def replace_force_virtual_with_call(self, graphs):
         # similar to rvirtualizable2.replace_force_virtualizable_with_call().
         c_force_virtual_ptr = None
+        c_is_virtual_ptr = None
         force_virtual_count = 0
         for graph in graphs:
             for block in graph.iterblocks():
@@ -52,6 +53,13 @@
                         op.opname = 'direct_call'
                         op.args = [c_force_virtual_ptr, op.args[0]]
                         force_virtual_count += 1
+                    #
+                    if op.opname == 'jit_is_virtual':
+                        if c_is_virtual_ptr is None:
+                            c_is_virtual_ptr = self.get_is_virtual_fnptr()
+                        #
+                        op.opname = 'direct_call'
+                        op.args = [c_is_virtual_ptr, op.args[0]]
         #
         if c_force_virtual_ptr is not None:
             log("replaced %d 'jit_force_virtual' with %r" % (force_virtual_count,
@@ -129,6 +137,17 @@
             force_virtual_if_necessary)
         return inputconst(lltype.typeOf(funcptr), funcptr)
 
+    def get_is_virtual_fnptr(self):
+        #
+        def is_virtual(inst):
+            if not inst:
+                return False
+            return inst.typeptr == self.jit_virtual_ref_vtable
+        #
+        FUNC = lltype.FuncType([rclass.OBJECTPTR], lltype.Bool)
+        funcptr = self.warmrunnerdesc.helper_func(lltype.Ptr(FUNC), is_virtual)
+        return inputconst(lltype.typeOf(funcptr), funcptr)
+
     def force_virtual(self, inst):
         vref = lltype.cast_pointer(lltype.Ptr(self.JIT_VIRTUAL_REF), inst)
         token = vref.virtual_token
diff --git a/pypy/module/__builtin__/__init__.py b/pypy/module/__builtin__/__init__.py
--- a/pypy/module/__builtin__/__init__.py
+++ b/pypy/module/__builtin__/__init__.py
@@ -20,6 +20,9 @@
         'any'           : 'app_functional.any',
         'all'           : 'app_functional.all',
         'sum'           : 'app_functional.sum',
+        'map'           : 'app_functional.map',
+        'reduce'        : 'app_functional.reduce',
+        'filter'        : 'app_functional.filter',
         'vars'          : 'app_inspect.vars',
         'dir'           : 'app_inspect.dir',
 
@@ -86,11 +89,8 @@
         'enumerate'     : 'functional.W_Enumerate',
         'min'           : 'functional.min',
         'max'           : 'functional.max',
-        'map'           : 'functional.map',
         'zip'           : 'functional.zip',
-        'reduce'        : 'functional.reduce',
         'reversed'      : 'functional.reversed',
-        'filter'        : 'functional.filter',
         'super'         : 'descriptor.W_Super',
         'staticmethod'  : 'descriptor.StaticMethod',
         'classmethod'   : 'descriptor.ClassMethod',
diff --git a/pypy/module/__builtin__/app_functional.py b/pypy/module/__builtin__/app_functional.py
--- a/pypy/module/__builtin__/app_functional.py
+++ b/pypy/module/__builtin__/app_functional.py
@@ -48,4 +48,118 @@
         # Very intentionally *not* +=, that would have different semantics if
         # start was a mutable type, such as a list
         last = last + x
-    return last
\ No newline at end of file
+    return last
+
+def map(func, *collections):
+    """map(function, sequence[, sequence, ...]) -> list
+
+Return a list of the results of applying the function to the items of
+the argument sequence(s).  If more than one sequence is given, the
+function is called with an argument list consisting of the corresponding
+item of each sequence, substituting None for missing values when not all
+sequences have the same length.  If the function is None, return a list of
+the items of the sequence (or a list of tuples if more than one sequence)."""
+    if not collections:
+        raise TypeError("map() requires at least two arguments")
+    num_collections = len(collections)
+    none_func = func is None
+    if num_collections == 1:
+        if none_func:
+            return list(collections[0])
+        else:
+            # Special case for the really common case of a single collection,
+            # this can be eliminated if we could unroll that loop that creates
+            # `args` based on whether or not len(collections) was constant
+            result = []
+            for item in collections[0]:
+                result.append(func(item))
+            return result
+    result = []
+    # Pair of (iterator, has_finished)
+    iterators = [(iter(seq), False) for seq in collections]
+    while True:
+        cont = False
+        args = []
+        for idx, (iterator, has_finished) in enumerate(iterators):
+            val = None
+            if not has_finished:
+                try:
+                    val = next(iterator)
+                except StopIteration:
+                    iterators[idx] = (None, True)
+                else:
+                    cont = True
+            args.append(val)
+        args = tuple(args)
+        if cont:
+            if none_func:
+                result.append(args)
+            else:
+                result.append(func(*args))
+        else:
+            return result
+
+sentinel = object()
+
+def reduce(func, sequence, initial=sentinel):
+    """reduce(function, sequence[, initial]) -> value
+
+Apply a function of two arguments cumulatively to the items of a sequence,
+from left to right, so as to reduce the sequence to a single value.
+For example, reduce(lambda x, y: x+y, [1, 2, 3, 4, 5]) calculates
+((((1+2)+3)+4)+5).  If initial is present, it is placed before the items
+of the sequence in the calculation, and serves as a default when the
+sequence is empty."""
+    iterator = iter(sequence)
+    if initial is sentinel:
+        try:
+            initial = next(iterator)
+        except StopIteration:
+            raise TypeError("reduce() of empty sequence with no initial value")
+    result = initial
+    for item in iterator:
+        result = func(result, item)
+    return result
+
+def filter(func, seq):
+    """filter(function or None, sequence) -> list, tuple, or string
+
+Return those items of sequence for which function(item) is true.  If
+function is None, return the items that are true.  If sequence is a tuple
+or string, return the same type, else return a list."""
+    if func is None:
+        func = bool
+    if isinstance(seq, str):
+        return _filter_string(func, seq, str)
+    elif isinstance(seq, unicode):
+        return _filter_string(func, seq, unicode)
+    elif isinstance(seq, tuple):
+        return _filter_tuple(func, seq)
+    result = []
+    for item in seq:
+        if func(item):
+            result.append(item)
+    return result
+
+def _filter_string(func, string, str_type):
+    if func is bool and type(string) is str_type:
+        return string
+    result = []
+    for i in range(len(string)):
+        # You must call __getitem__ on the strings, simply iterating doesn't
+        # work :/
+        item = string[i]
+        if func(item):
+            if not isinstance(item, str_type):
+                raise TypeError("__getitem__ returned a non-string type")
+            result.append(item)
+    return str_type().join(result)
+
+def _filter_tuple(func, seq):
+    result = []
+    for i in range(len(seq)):
+        # Again, must call __getitem__, at least there are tests.
+        item = seq[i]
+        if func(item):
+            result.append(item)
+    return tuple(result)
\ No newline at end of file
diff --git a/pypy/module/__builtin__/compiling.py b/pypy/module/__builtin__/compiling.py
--- a/pypy/module/__builtin__/compiling.py
+++ b/pypy/module/__builtin__/compiling.py
@@ -84,8 +84,8 @@
         raise OperationError(space.w_TypeError,
               w('eval() arg 1 must be a string or code object'))
 
-    caller = space.getexecutioncontext().gettopframe_nohidden()
     if space.is_w(w_globals, space.w_None):
+        caller = space.getexecutioncontext().gettopframe_nohidden()
         if caller is None:
             w_globals = space.newdict()
             if space.is_w(w_locals, space.w_None):
@@ -97,13 +97,6 @@
     elif space.is_w(w_locals, space.w_None):
         w_locals = w_globals
 
-    try:
-        space.getitem(w_globals, space.wrap('__builtins__'))
-    except OperationError, e:
-        if not e.match(space, space.w_KeyError):
-            raise
-        if caller is not None:
-            w_builtin = space.builtin.pick_builtin(caller.w_globals)
-            space.setitem(w_globals, space.wrap('__builtins__'), w_builtin)
+    space.builtin.pick_builtin(w_globals)
 
     return codeobj.exec_code(space, w_globals, w_locals)
diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py
--- a/pypy/module/__builtin__/functional.py
+++ b/pypy/module/__builtin__/functional.py
@@ -201,118 +201,6 @@
     """
     return min_max(space, __args__, "min")
 
- at unwrap_spec(collections_w="args_w")
-def map(space, w_func, collections_w):
-    """does 3 separate things, hence this enormous docstring.
-       1.  if function is None, return a list of tuples, each with one
-           item from each collection.  If the collections have different
-           lengths,  shorter ones are padded with None.
-
-       2.  if function is not None, and there is only one collection,
-           apply function to every item in the collection and return a
-           list of the results.
-
-       3.  if function is not None, and there are several collections,
-           repeatedly call the function with one argument from each
-           collection.  If the collections have different lengths,
-           shorter ones are padded with None
-    """
-    if not collections_w:
-        msg = "map() requires at least two arguments"
-        raise OperationError(space.w_TypeError, space.wrap(msg))
-    none_func = space.is_w(w_func, space.w_None)
-    if len(collections_w) == 1:
-        w_collection = collections_w[0]
-        if none_func:
-            result_w = space.unpackiterable(w_collection)
-        else:
-            result_w = map_single_collection(space, w_func, w_collection)
-    else:
-        result_w = map_multiple_collections(space, w_func, collections_w,
-                                            none_func)
-    return space.newlist(result_w)
-
-def map_single_collection(space, w_func, w_collection):
-    """Special case for 'map(func, coll)', where 'func' is not None and there
-    is only one 'coll' argument."""
-    w_iter = space.iter(w_collection)
-    # xxx special hacks for speed
-    from pypy.interpreter import function, pycode
-    if isinstance(w_func, function.Function):
-        # xxx compatibility issue: what if func_code is modified in the
-        # middle of running map()??  That's far too obscure for me to care...
-        code = w_func.getcode()
-        fast_natural_arity = code.fast_natural_arity
-        if fast_natural_arity == (1|pycode.PyCode.FLATPYCALL):
-            assert isinstance(code, pycode.PyCode)
-            return map_single_user_function(code, w_func, w_iter)
-    # /xxx end of special hacks
-    return map_single_other_callable(space, w_func, w_iter)
-
-def map_single_other_callable(space, w_func, w_iter):
-    result_w = []
-    while True:
-        try:
-            w_item = space.next(w_iter)
-        except OperationError, e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            break
-        result_w.append(space.call_function(w_func, w_item))
-    return result_w
-map_single_other_callable._dont_inline_ = True
-
-from pypy.rlib.jit import JitDriver
-mapjitdriver = JitDriver(greens = ['code'],
-                         reds = ['w_func', 'w_iter', 'result_w'])
-def map_single_user_function(code, w_func, w_iter):
-    result_w = []
-    while True:
-        mapjitdriver.can_enter_jit(code=code, w_func=w_func,
-                                   w_iter=w_iter, result_w=result_w)
-        mapjitdriver.jit_merge_point(code=code, w_func=w_func,
-                                     w_iter=w_iter, result_w=result_w)
-        space = w_func.space
-        try:
-            w_item = space.next(w_iter)
-        except OperationError, e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            break
-        new_frame = space.createframe(code, w_func.w_func_globals,
-                                      w_func)
-        new_frame.locals_stack_w[0] = w_item
-        w_res = new_frame.run()
-        result_w.append(w_res)
-    return result_w
-
-def map_multiple_collections(space, w_func, collections_w, none_func):
-    result_w = []
-    iterators_w = [space.iter(w_seq) for w_seq in collections_w]
-    num_iterators = len(iterators_w)
-    while True:
-        cont = False
-        args_w = [space.w_None] * num_iterators
-        for i in range(num_iterators):
-            if iterators_w[i] is not None:
-                try:
-                    args_w[i] = space.next(iterators_w[i])
-                except OperationError, e:
-                    if not e.match(space, space.w_StopIteration):
-                        raise
-                    iterators_w[i] = None
-                else:
-                    cont = True
-        if not cont:
-            break
-        w_args = space.newtuple(args_w)
-        if none_func:
-            w_res = w_args
-        else:
-            w_res = space.call(w_func, w_args)
-        result_w.append(w_res)
-    return result_w
-
 @unwrap_spec(sequences_w="args_w")
 def zip(space, sequences_w):
     """Return a list of tuples, where the nth tuple contains every nth item of
@@ -334,90 +222,6 @@
             return space.newlist(result_w)
         result_w.append(space.newtuple(items_w))
 
-def reduce(space, w_func, w_sequence, w_initial=NoneNotWrapped):
-    """ Apply function of two arguments cumulatively to the items of sequence,
-        from left to right, so as to reduce the sequence to a single value.
-        Optionally begin with an initial value.
-    """
-    w_iter = space.iter(w_sequence)
-    if w_initial is None:
-        try:
-            w_initial = space.next(w_iter)
-        except OperationError, e:
-            if e.match(space, space.w_StopIteration):
-                msg = "reduce() of empty sequence with no initial value"
-                raise OperationError(space.w_TypeError, space.wrap(msg))
-            raise
-    w_result = w_initial
-    while True:
-        try:
-            w_next = space.next(w_iter)
-        except OperationError, e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            break
-        w_result = space.call_function(w_func, w_result, w_next)
-    return w_result
-
-def filter(space, w_func, w_seq):
-    """construct a list of those elements of collection for which function
-       is True.  If function is None, then return the items in the sequence
-       which are True.
-    """
-    if space.is_true(space.isinstance(w_seq, space.w_str)):
-        return _filter_string(space, w_func, w_seq, space.w_str)
-    if space.is_true(space.isinstance(w_seq, space.w_unicode)):
-        return _filter_string(space, w_func, w_seq, space.w_unicode)
-    if space.is_true(space.isinstance(w_seq, space.w_tuple)):
-        return _filter_tuple(space, w_func, w_seq)
-    w_iter = space.iter(w_seq)
-    result_w = []
-    none_func = space.is_w(w_func, space.w_None)
-    while True:
-        try:
-            w_next = space.next(w_iter)
-        except OperationError, e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            break
-        if none_func:
-            w_keep = w_next
-        else:
-            w_keep = space.call_function(w_func, w_next)
-        if space.is_true(w_keep):
-            result_w.append(w_next)
-    return space.newlist(result_w)
-
-def _filter_tuple(space, w_func, w_tuple):
-    none_func = space.is_w(w_func, space.w_None)
-    length = space.len_w(w_tuple)
-    result_w = []
-    for i in range(length):
-        w_item = space.getitem(w_tuple, space.wrap(i))
-        if none_func:
-            w_keep = w_item
-        else:
-            w_keep = space.call_function(w_func, w_item)
-        if space.is_true(w_keep):
-            result_w.append(w_item)
-    return space.newtuple(result_w)
-
-def _filter_string(space, w_func, w_string, w_str_type):
-    none_func = space.is_w(w_func, space.w_None)
-    if none_func and space.is_w(space.type(w_string), w_str_type):
-        return w_string
-    length = space.len_w(w_string)
-    result_w = []
-    for i in range(length):
-        w_item = space.getitem(w_string, space.wrap(i))
-        if none_func or space.is_true(space.call_function(w_func, w_item)):
-            if not space.is_true(space.isinstance(w_item, w_str_type)):
-                msg = "__getitem__ returned a non-string type"
-                raise OperationError(space.w_TypeError, space.wrap(msg))
-            result_w.append(w_item)
-    w_empty = space.call_function(w_str_type)
-    return space.call_method(w_empty, "join", space.newlist(result_w))
-
 class W_Enumerate(Wrappable):
 
     def __init__(self, w_iter, w_start):
diff --git a/pypy/module/__builtin__/test/test_functional.py b/pypy/module/__builtin__/test/test_functional.py
--- a/pypy/module/__builtin__/test/test_functional.py
+++ b/pypy/module/__builtin__/test/test_functional.py
@@ -147,7 +147,7 @@
        assert list(xrange(A())) == [0, 1, 2, 3, 4]
        assert list(xrange(0, A())) == [0, 1, 2, 3, 4]
        assert list(xrange(0, 10, A())) == [0, 5]
- 
+
    def test_xrange_float(self):
       assert list(xrange(0.1, 2.0, 1.1)) == [0, 1]
 
diff --git a/pypy/module/_continuation/interp_continuation.py b/pypy/module/_continuation/interp_continuation.py
--- a/pypy/module/_continuation/interp_continuation.py
+++ b/pypy/module/_continuation/interp_continuation.py
@@ -19,6 +19,11 @@
         #  - normal:      self.sthread != None, not is_empty_handle(self.h)
         #  - finished:    self.sthread != None, is_empty_handle(self.h)
 
+    def __del__(self):
+        sthread = self.sthread
+        if sthread is not None and not sthread.is_empty_handle(self.h):
+            sthread.destroy(self.h)
+
     def check_sthread(self):
         ec = self.space.getexecutioncontext()
         if ec.stacklet_thread is not self.sthread:
@@ -28,6 +33,8 @@
     def descr_init(self, w_callable, __args__):
         if self.sthread is not None:
             raise geterror(self.space, "continulet already __init__ialized")
+        sthread = build_sthread(self.space)
+        workaround_disable_jit(sthread)
         #
         # hackish: build the frame "by hand", passing it the correct arguments
         space = self.space
@@ -41,7 +48,6 @@
         self.bottomframe = bottomframe
         #
         global_state.origin = self
-        sthread = build_sthread(self.space)
         self.sthread = sthread
         h = sthread.new(new_stacklet_callback)
         post_switch(sthread, h)
@@ -71,6 +77,7 @@
                 global_state.clear()
                 raise geterror(self.space, "continulet already finished")
         self.check_sthread()
+        workaround_disable_jit(self.sthread)
         #
         global_state.origin = self
         if to is None:
@@ -259,6 +266,16 @@
         sthread = ec.stacklet_thread = SThread(space, ec)
     return sthread
 
+def workaround_disable_jit(sthread):
+    # A bad workaround to kill the JIT anywhere in this thread.
+    # This forces all the frames.  It's a bad workaround because
+    # it takes O(depth) time, and it will cause some "abort:
+    # vable escape" in the JIT.  The goal is to prevent any frame
+    # from being still virtuals, because the JIT generates code
+    # to un-virtualizable them "on demand" by loading values based
+    # on FORCE_TOKEN, which is an address in the stack.
+    sthread.ec.force_all_frames()
+
 # ____________________________________________________________
 
 def permute(space, args_w):
diff --git a/pypy/module/_file/test/test_file.py b/pypy/module/_file/test/test_file.py
--- a/pypy/module/_file/test/test_file.py
+++ b/pypy/module/_file/test/test_file.py
@@ -1,5 +1,5 @@
 from __future__ import with_statement
-import py
+import py, os, errno
 
 from pypy.conftest import gettestobjspace, option
 
@@ -257,6 +257,39 @@
             assert self.temppath in g.getvalue()
 
 
+class AppTestNonblocking(object):
+    def setup_class(cls):
+        from pypy.module._file.interp_file import W_File
+
+        cls.old_read = os.read
+
+        if option.runappdirect:
+            py.test.skip("works with internals of _file impl on py.py")
+
+        state = [0]
+        def read(fd, n=None):
+            if fd != 42:
+                return cls.old_read(fd, n)
+            if state[0] == 0:
+                state[0] += 1
+                return "xyz"
+            if state[0] < 3:
+                state[0] += 1
+                raise OSError(errno.EAGAIN, "xyz")
+            return ''
+        os.read = read
+        stdin = W_File(cls.space)
+        stdin.file_fdopen(42, "r", 1)
+        stdin.name = '<stdin>'
+        cls.w_stream = stdin
+
+    def teardown_class(cls):
+        os.read = cls.old_read
+
+    def test_nonblocking_file(self):
+        res = self.stream.read()
+        assert res == 'xyz'
+
 class AppTestConcurrency(object):
     # these tests only really make sense on top of a translated pypy-c,
     # because on top of py.py the inner calls to os.write() don't
@@ -374,24 +407,24 @@
         with self.file(self.temppath, 'w') as f:
             f.write('foo')
         assert f.closed
-        
+
         with self.file(self.temppath, 'r') as f:
             s = f.readline()
 
         assert s == "foo"
         assert f.closed
-    
+
     def test_subclass_with(self):
         file = self.file
         class C(file):
             def __init__(self, *args, **kwargs):
                 self.subclass_closed = False
                 file.__init__(self, *args, **kwargs)
-            
+
             def close(self):
                 self.subclass_closed = True
                 file.close(self)
-        
+
         with C(self.temppath, 'w') as f:
             pass
         assert f.subclass_closed
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -161,9 +161,6 @@
     @binop
     def mul(self, v1, v2):
         return v1 * v2
-    @binop
-    def div(self, v1, v2):
-        return v1 / v2
 
     @unaryop
     def pos(self, v):
@@ -217,6 +214,14 @@
         return float2string(self.for_computation(self.unbox(item)), 'g', rfloat.DTSF_STR_PRECISION)
 
     @binop
+    def div(self, v1, v2):
+        try:
+            return v1 / v2
+        except ZeroDivisionError:
+            if v1 == v2 == 0.0:
+                return rfloat.NAN
+            return rfloat.copysign(rfloat.INFINITY, v1 * v2)
+    @binop
     def mod(self, v1, v2):
         return math.fmod(v1, v2)
     @binop
@@ -295,6 +300,11 @@
         return str(widen(self.unbox(item)))
 
     @binop
+    def div(self, v1, v2):
+        if v2 == 0:
+            return 0
+        return v1 / v2
+    @binop
     def mod(self, v1, v2):
         return v1 % v2
 
@@ -426,23 +436,22 @@
     pass
 
 if LONG_BIT == 32:
-    class W_LongDtype(W_Int32Dtype):
-        pass
+    long_dtype = W_Int32Dtype
+    ulong_dtype = W_UInt32Dtype
+elif LONG_BIT == 64:
+    long_dtype = W_Int64Dtype
+    ulong_dtype = W_UInt64Dtype
+else:
+    assert False
 
-    class W_ULongDtype(W_UInt32Dtype):
-        pass
-else:
-    class W_LongDtype(W_Int64Dtype):
-        pass
+class W_LongDtype(long_dtype):
+    num = 7
+    aliases = ["l"]
+    applevel_types = ["int"]
 
-    class W_ULongDtype(W_UInt64Dtype):
-        pass
-
-W_LongDtype.num = 7
-W_LongDtype.aliases = ["l"]
-W_LongDtype.applevel_types = ["int"]
-W_ULongDtype.num = 8
-W_ULongDtype.aliases = ["L"]
+class W_ULongDtype(ulong_dtype):
+    num = 8
+    aliases = ["L"]
 
 W_Float32Dtype = create_low_level_dtype(
     num = 11, kind = FLOATINGLTR, name = "float32",
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -285,7 +285,9 @@
             assert b[i] == i * 5
 
     def test_div(self):
-        from numpy import array, dtype
+        from math import isnan
+        from numpy import array, dtype, inf
+
         a = array(range(1, 6))
         b = a / a
         for i in range(5):
@@ -297,6 +299,24 @@
         for i in range(5):
             assert b[i] == 1
 
+        a = array([-1, 0, 1])
+        b = array([0, 0, 0])
+        c = a / b
+        assert (c == [0, 0, 0]).all()
+
+        a = array([-1.0, 0.0, 1.0])
+        b = array([0.0, 0.0, 0.0])
+        c = a / b
+        assert c[0] == -inf
+        assert isnan(c[1])
+        assert c[2] == inf
+
+        b = array([-0.0, -0.0, -0.0])
+        c = a / b
+        assert c[0] == inf
+        assert isnan(c[1])
+        assert c[2] == -inf
+
     def test_div_other(self):
         from numpy import array
         a = array(range(5))
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -420,12 +420,15 @@
             import sys
             if not hasattr(os, "fork"):
                 skip("Need fork() to test execv()")
+            try:
+                output = u"caf\xe9 \u1234\n".encode(sys.getfilesystemencoding())
+            except UnicodeEncodeError:
+                skip("encoding not good enough")
             pid = os.fork()
             if pid == 0:
                 os.execv(u"/bin/sh", ["sh", "-c",
                                       u"echo caf\xe9 \u1234 > onefile"])
             os.waitpid(pid, 0)
-            output = u"caf\xe9 \u1234\n".encode(sys.getfilesystemencoding())
             assert open("onefile").read() == output
             os.unlink("onefile")
 
@@ -445,13 +448,16 @@
             import sys
             if not hasattr(os, "fork"):
                 skip("Need fork() to test execve()")
+            try:
+                output = u"caf\xe9 \u1234\n".encode(sys.getfilesystemencoding())
+            except UnicodeEncodeError:
+                skip("encoding not good enough")
             pid = os.fork()
             if pid == 0:
                 os.execve(u"/bin/sh", ["sh", "-c",
                                       u"echo caf\xe9 \u1234 > onefile"],
                           {'ddd': 'xxx'})
             os.waitpid(pid, 0)
-            output = u"caf\xe9 \u1234\n".encode(sys.getfilesystemencoding())
             assert open("onefile").read() == output
             os.unlink("onefile")
         pass # <- please, inspect.getsource(), don't crash
diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py
--- a/pypy/module/pypyjit/interp_jit.py
+++ b/pypy/module/pypyjit/interp_jit.py
@@ -137,20 +137,15 @@
 
     def jump_absolute(self, jumpto, _, ec=None):
         if we_are_jitted():
-            # Normally, the tick counter is decremented by 100 for every
-            # Python opcode.  Here, to better support JIT compilation of
-            # small loops, we decrement it by a possibly smaller constant.
-            # We get the maximum 100 when the (unoptimized) trace length
-            # is at least 3200 (a bit randomly).
-            trace_length = r_uint(current_trace_length())
-            decr_by = trace_length // 32
-            if decr_by < 1:
-                decr_by = 1
-            elif decr_by > 100:    # also if current_trace_length() returned -1
-                decr_by = 100
+            #
+            # assume that only threads are using the bytecode counter
+            decr_by = 0
+            if self.space.actionflag.has_bytecode_counter:   # constant-folded
+                if self.space.threadlocals.gil_ready:   # quasi-immutable field
+                    decr_by = _get_adapted_tick_counter()
             #
             self.last_instr = intmask(jumpto)
-            ec.bytecode_trace(self, intmask(decr_by))
+            ec.bytecode_trace(self, decr_by)
             jumpto = r_uint(self.last_instr)
         #
         pypyjitdriver.can_enter_jit(frame=self, ec=ec, next_instr=jumpto,
@@ -158,6 +153,20 @@
                                     is_being_profiled=self.is_being_profiled)
         return jumpto
 
+def _get_adapted_tick_counter():
+    # Normally, the tick counter is decremented by 100 for every
+    # Python opcode.  Here, to better support JIT compilation of
+    # small loops, we decrement it by a possibly smaller constant.
+    # We get the maximum 100 when the (unoptimized) trace length
+    # is at least 3200 (a bit randomly).
+    trace_length = r_uint(current_trace_length())
+    decr_by = trace_length // 32
+    if decr_by < 1:
+        decr_by = 1
+    elif decr_by > 100:    # also if current_trace_length() returned -1
+        decr_by = 100
+    return intmask(decr_by)
+
 
 PyCode__initialize = PyCode._initialize
 
diff --git a/pypy/module/pypyjit/test_pypy_c/model.py b/pypy/module/pypyjit/test_pypy_c/model.py
--- a/pypy/module/pypyjit/test_pypy_c/model.py
+++ b/pypy/module/pypyjit/test_pypy_c/model.py
@@ -225,6 +225,8 @@
         # strip comment
         if '#' in line:
             line = line[:line.index('#')]
+        if line.strip() == 'guard_not_invalidated?':
+            return 'guard_not_invalidated', None, [], '...', False
         # find the resvar, if any
         if ' = ' in line:
             resvar, _, line = line.partition(' = ')
@@ -249,7 +251,7 @@
             descr = descr[len('descr='):]
         else:
             descr = None
-        return opname, resvar, args, descr
+        return opname, resvar, args, descr, True
 
     @classmethod
     def preprocess_expected_src(cls, src):
@@ -258,13 +260,23 @@
         # replaced with the corresponding operations, so that tests don't have
         # to repeat it every time
         ticker_check = """
+            guard_not_invalidated?
+            ticker0 = getfield_raw(ticker_address, descr=<SignedFieldDescr pypysig_long_struct.c_value .*>)
+            ticker_cond0 = int_lt(ticker0, 0)
+            guard_false(ticker_cond0, descr=...)
+        """
+        src = src.replace('--TICK--', ticker_check)
+        #
+        # this is the ticker check generated if we have threads
+        thread_ticker_check = """
+            guard_not_invalidated?
             ticker0 = getfield_raw(ticker_address, descr=<SignedFieldDescr pypysig_long_struct.c_value .*>)
             ticker1 = int_sub(ticker0, 1)
             setfield_raw(ticker_address, ticker1, descr=<SignedFieldDescr pypysig_long_struct.c_value .*>)
             ticker_cond0 = int_lt(ticker1, 0)
             guard_false(ticker_cond0, descr=...)
         """
-        src = src.replace('--TICK--', ticker_check)
+        src = src.replace('--THREAD-TICK--', thread_ticker_check)
         #
         # this is the ticker check generated in PyFrame.handle_operation_error
         exc_ticker_check = """
@@ -298,7 +310,7 @@
         if not cond:
             raise InvalidMatch(message, frame=sys._getframe(1))
 
-    def match_op(self, op, (exp_opname, exp_res, exp_args, exp_descr)):
+    def match_op(self, op, (exp_opname, exp_res, exp_args, exp_descr, _)):
         self._assert(op.name == exp_opname, "operation mismatch")
         self.match_var(op.res, exp_res)
         if exp_args != ['...']:
@@ -341,7 +353,7 @@
         what is after the '...'
         """
         iter_exp_ops = iter(expected_ops)
-        iter_ops = iter(self.ops)
+        iter_ops = RevertableIterator(self.ops)
         for opindex, exp_op in enumerate(iter_exp_ops):
             try:
                 if exp_op == '...':
@@ -360,6 +372,9 @@
                             break
                 self.match_op(op, exp_op)
             except InvalidMatch, e:
+                if exp_op[4] is False:    # optional operation
+                    iter_ops.revert_one()
+                    continue       # try to match with the next exp_op
                 e.opindex = opindex
                 raise
         #
@@ -398,3 +413,18 @@
         else:
             return True
 
+
+class RevertableIterator(object):
+    def __init__(self, sequence):
+        self.sequence = sequence
+        self.index = 0
+    def __iter__(self):
+        return self
+    def next(self):
+        index = self.index
+        if index == len(self.sequence):
+            raise StopIteration
+        self.index = index + 1
+        return self.sequence[index]
+    def revert_one(self):
+        self.index -= 1
diff --git a/pypy/module/pypyjit/test_pypy_c/test_00_model.py b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
--- a/pypy/module/pypyjit/test_pypy_c/test_00_model.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
@@ -145,15 +145,17 @@
 
     def test_parse_op(self):
         res = OpMatcher.parse_op("  a =   int_add(  b,  3 ) # foo")
-        assert res == ("int_add", "a", ["b", "3"], None)
+        assert res == ("int_add", "a", ["b", "3"], None, True)
         res = OpMatcher.parse_op("guard_true(a)")
-        assert res == ("guard_true", None, ["a"], None)
+        assert res == ("guard_true", None, ["a"], None, True)
         res = OpMatcher.parse_op("setfield_gc(p0, i0, descr=<foobar>)")
-        assert res == ("setfield_gc", None, ["p0", "i0"], "<foobar>")
+        assert res == ("setfield_gc", None, ["p0", "i0"], "<foobar>", True)
         res = OpMatcher.parse_op("i1 = getfield_gc(p0, descr=<foobar>)")
-        assert res == ("getfield_gc", "i1", ["p0"], "<foobar>")
+        assert res == ("getfield_gc", "i1", ["p0"], "<foobar>", True)
         res = OpMatcher.parse_op("p0 = force_token()")
-        assert res == ("force_token", "p0", [], None)
+        assert res == ("force_token", "p0", [], None, True)
+        res = OpMatcher.parse_op("guard_not_invalidated?")
+        assert res == ("guard_not_invalidated", None, [], '...', False)
 
     def test_exact_match(self):
         loop = """
@@ -341,7 +343,7 @@
             # this is the actual loop
             'int_lt', 'guard_true', 'int_add',
             # this is the signal checking stuff
-            'getfield_raw', 'int_sub', 'setfield_raw', 'int_lt', 'guard_false',
+            'guard_not_invalidated', 'getfield_raw', 'int_lt', 'guard_false',
             'jump'
             ]
 
@@ -407,7 +409,7 @@
             # this is the actual loop
             'int_lt', 'guard_true', 'force_token', 'int_add',
             # this is the signal checking stuff
-            'getfield_raw', 'int_sub', 'setfield_raw', 'int_lt', 'guard_false',
+            'guard_not_invalidated', 'getfield_raw', 'int_lt', 'guard_false',
             'jump'
             ]
 
@@ -425,10 +427,9 @@
             guard_true(i6, descr=...)
             i8 = int_add(i4, 1)
             # signal checking stuff
+            guard_not_invalidated(descr=...)
             i10 = getfield_raw(37212896, descr=<.* pypysig_long_struct.c_value .*>)
-            i12 = int_sub(i10, 1)
-            setfield_raw(37212896, i12, descr=<.* pypysig_long_struct.c_value .*>)
-            i14 = int_lt(i12, 0)
+            i14 = int_lt(i10, 0)
             guard_false(i14, descr=...)
             jump(p0, p1, p2, p3, i8, descr=...)
         """)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_misc.py b/pypy/module/pypyjit/test_pypy_c/test_misc.py
--- a/pypy/module/pypyjit/test_pypy_c/test_misc.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_misc.py
@@ -329,4 +329,20 @@
             guard_false(i28, descr=...)
             i30 = int_lshift(i20, 24)
             i31 = int_or(i26, i30)
-        """ % {"32_bit_only": extra})
\ No newline at end of file
+        """ % {"32_bit_only": extra})
+
+    def test_eval(self):
+        def main():
+            i = 1
+            a = compile('x+x+x+x+x+x', 'eval', 'eval')
+            b = {'x': 7}
+            while i < 1000:
+                y = eval(a,b,b)  # ID: eval
+                i += 1
+            return y
+
+        log = self.run(main)
+        assert log.result == 42
+        # the following assertion fails if the loop was cancelled due
+        # to "abort: vable escape"
+        assert len(log.loops_by_id("eval")) == 1
diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -156,4 +156,41 @@
             i40 = int_sub(i4, 1)
             --TICK--
             jump(p0, p1, p2, p3, i40, i38, descr=<Loop0>)
-        """)
\ No newline at end of file
+        """)
+
+    def test_getattr_promote(self):
+        def main(n):
+            class A(object):
+                def meth_a(self):
+                    return 1
+                def meth_b(self):
+                    return 2
+            a = A()
+
+            l = ['a', 'b']
+            s = 0
+            for i in range(n):
+                name = 'meth_' + l[i & 1]
+                meth = getattr(a, name) # ID: getattr
+                s += meth()
+            return s
+
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loops = log.loops_by_filename(self.filepath)
+        assert len(loops) == 2
+        for loop in loops:
+            loop.match_by_id('getattr','''
+            guard_not_invalidated(descr=...)
+            i32 = strlen(p31)
+            i34 = int_add(5, i32)
+            p35 = newstr(i34)
+            strsetitem(p35, 0, 109)
+            strsetitem(p35, 1, 101)
+            strsetitem(p35, 2, 116)
+            strsetitem(p35, 3, 104)
+            strsetitem(p35, 4, 95)
+            copystrcontent(p31, p35, 0, 5, i32)
+            i49 = call(ConstClass(_ll_2_str_eq_nonnull__rpy_stringPtr_rpy_stringPtr), p35, ConstPtr(ptr48), descr=<SignedCallDescr>)
+            guard_value(i49, 1, descr=<Guard8>)
+            ''')
diff --git a/pypy/module/pypyjit/test_pypy_c/test_thread.py b/pypy/module/pypyjit/test_pypy_c/test_thread.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/pypyjit/test_pypy_c/test_thread.py
@@ -0,0 +1,28 @@
+from pypy.module.pypyjit.test_pypy_c.test_00_model import BaseTestPyPyC
+
+
+class TestThread(BaseTestPyPyC):
+    def test_simple(self):
+        def main(n):
+            import thread
+            def f():
+                i = 0
+                while i < n:
+                    i += 1
+                done.release()
+
+            done = thread.allocate_lock()
+            done.acquire()
+            thread.start_new_thread(f, ())
+            done.acquire()
+            return 0
+        log = self.run(main, [500])
+        assert round(log.result, 6) == round(main(500), 6)
+        loop, = log.loops_by_filename(self.filepath)
+        assert loop.match("""
+            i2 = int_lt(i0, i1)
+            guard_true(i2, descr=...)
+            i3 = int_add(i0, 1)
+            --THREAD-TICK--
+            jump(..., descr=<Loop0>)
+        """)
diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py
--- a/pypy/module/signal/interp_signal.py
+++ b/pypy/module/signal/interp_signal.py
@@ -109,8 +109,11 @@
         p = pypysig_getaddr_occurred()
         value = p.c_value
         if self.has_bytecode_counter:    # this 'if' is constant-folded
-            value -= by
-            p.c_value = value
+            if jit.isconstant(by) and by == 0:
+                pass     # normally constant-folded too
+            else:
+                value -= by
+                p.c_value = value
         return value
 
 
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -47,7 +47,7 @@
         'pypy_initial_path'     : 'state.pypy_initial_path',
 
         '_getframe'             : 'vm._getframe', 
-        '_current_frames'       : 'vm._current_frames', 
+        '_current_frames'       : 'currentframes._current_frames', 
         'setrecursionlimit'     : 'vm.setrecursionlimit', 
         'getrecursionlimit'     : 'vm.getrecursionlimit', 
         'setcheckinterval'      : 'vm.setcheckinterval', 
diff --git a/pypy/module/sys/currentframes.py b/pypy/module/sys/currentframes.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/sys/currentframes.py
@@ -0,0 +1,78 @@
+"""
+Implementation of the 'sys._current_frames()' routine.
+"""
+from pypy.interpreter import gateway
+
+app = gateway.applevel('''
+"NOT_RPYTHON"
+import __builtin__
+
+class fake_code(object):
+    co_name = "?"
+    co_filename = "?"
+    co_firstlineno = 0
+
+class fake_frame(object):
+    f_back = None
+    f_builtins = __builtin__.__dict__
+    f_code = fake_code()
+    f_exc_traceback = None
+    f_exc_type = None
+    f_exc_value = None
+    f_globals = {}
+    f_lasti = -1
+    f_lineno = 0
+    f_locals = {}
+    f_restricted = False
+    f_trace = None
+
+    def __init__(self, f):
+        if f is not None:
+            for name in ["f_builtins", "f_code", "f_globals", "f_lasti",
+                         "f_lineno"]:
+                setattr(self, name, getattr(f, name))
+''')
+
+def _current_frames(space):
+    """_current_frames() -> dictionary
+
+    Return a dictionary mapping each current thread T's thread id to T's
+    current stack "frame".  Functions in the traceback module can build the
+    call stack given such a frame.
+
+    Note that in PyPy this returns fake frame objects, to avoid a runtime
+    penalty everywhere with the JIT.  (So far these fake frames can be
+    completely uninformative depending on the JIT state; we could return
+    more with more efforts.)
+
+    This function should be used for specialized purposes only."""
+    w_result = space.newdict()
+    w_fake_frame = app.wget(space, "fake_frame")
+    w_fake_code  = app.wget(space, "fake_code")
+    ecs = space.threadlocals.getallvalues()
+    for thread_ident, ec in ecs.items():
+        vref = ec.topframeref
+        frames = []
+        while not vref.virtual:
+            f = vref()
+            if f is None:
+                break
+            frames.append(f)
+            vref = f.f_backref
+        else:
+            frames.append(None)
+        #
+        w_topframe = space.wrap(None)
+        w_prevframe = None
+        for f in frames:
+            w_nextframe = space.call_function(w_fake_frame, space.wrap(f))
+            if w_prevframe is None:
+                w_topframe = w_nextframe
+            else:
+                space.setattr(w_prevframe, space.wrap('f_back'), w_nextframe)
+            w_prevframe = w_nextframe
+        #
+        space.setitem(w_result,
+                      space.wrap(thread_ident),
+                      w_topframe)
+    return w_result
diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -556,7 +556,7 @@
             return sys._current_frames()
         frames = f()
         assert frames.keys() == [0]
-        assert frames[0].f_code.co_name == 'f'
+        assert frames[0].f_code.co_name in ('f', '?')
 
 class AppTestCurrentFramesWithThread(AppTestCurrentFrames):
     def setup_class(cls):
@@ -568,23 +568,25 @@
         import thread
 
         thread_id = thread.get_ident()
-        self.ready = False
         def other_thread():
-            self.ready = True
             print "thread started"
-            time.sleep(5)
+            lock2.release()
+            lock1.acquire()
+        lock1 = thread.allocate_lock()
+        lock2 = thread.allocate_lock()
+        lock1.acquire()
+        lock2.acquire()
         thread.start_new_thread(other_thread, ())
 
         def f():
-            for i in range(100):
-                if self.ready: break
-                time.sleep(0.1)
+            lock2.acquire()
             return sys._current_frames()
 
         frames = f()
+        lock1.release()
         thisframe = frames.pop(thread_id)
-        assert thisframe.f_code.co_name == 'f'
+        assert thisframe.f_code.co_name in ('f', '?')
 
         assert len(frames) == 1
         _, other_frame = frames.popitem()
-        assert other_frame.f_code.co_name == 'other_thread'
+        assert other_frame.f_code.co_name in ('other_thread', '?')
diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -45,25 +45,6 @@
     f.mark_as_escaped()
     return space.wrap(f)
 
-def _current_frames(space):
-    """_current_frames() -> dictionary
-
-    Return a dictionary mapping each current thread T's thread id to T's
-    current stack frame.
-
-    This function should be used for specialized purposes only."""
-    raise OperationError(space.w_NotImplementedError,
-        space.wrap("XXX sys._current_frames() incompatible with the JIT"))
-    w_result = space.newdict()
-    ecs = space.threadlocals.getallvalues()
-    for thread_ident, ec in ecs.items():
-        f = ec.gettopframe_nohidden()
-        f.mark_as_escaped()
-        space.setitem(w_result,
-                      space.wrap(thread_ident),
-                      space.wrap(f))
-    return w_result
-
 def setrecursionlimit(space, w_new_limit):
     """setrecursionlimit() sets the maximum number of nested calls that
 can occur before a RuntimeError is raised.  On PyPy the limit is
diff --git a/pypy/module/thread/gil.py b/pypy/module/thread/gil.py
--- a/pypy/module/thread/gil.py
+++ b/pypy/module/thread/gil.py
@@ -17,6 +17,7 @@
 class GILThreadLocals(OSThreadLocals):
     """A version of OSThreadLocals that enforces a GIL."""
     gil_ready = False
+    _immutable_fields_ = ['gil_ready?']
 
     def initialize(self, space):
         # add the GIL-releasing callback as an action on the space
diff --git a/pypy/module/thread/threadlocals.py b/pypy/module/thread/threadlocals.py
--- a/pypy/module/thread/threadlocals.py
+++ b/pypy/module/thread/threadlocals.py
@@ -8,9 +8,14 @@
 
     def __init__(self):
         self._valuedict = {}   # {thread_ident: ExecutionContext()}
+        self._freeze_()
+
+    def _freeze_(self):
+        self._valuedict.clear()
         self._mainthreadident = 0
         self._mostrecentkey = 0        # fast minicaching for the common case
         self._mostrecentvalue = None   # fast minicaching for the common case
+        return False
 
     def getvalue(self):
         ident = thread.get_ident()
diff --git a/pypy/objspace/std/objecttype.py b/pypy/objspace/std/objecttype.py
--- a/pypy/objspace/std/objecttype.py
+++ b/pypy/objspace/std/objecttype.py
@@ -44,7 +44,6 @@
         raise OperationError(space.w_TypeError,
                              space.wrap("__class__ assignment: only for heap types"))
     w_oldcls = space.type(w_obj)
-    # XXX taint space should raise a TaintError here if w_oldcls is tainted
     assert isinstance(w_oldcls, W_TypeObject)
     if w_oldcls.get_full_instance_layout() == w_newcls.get_full_instance_layout():
         w_obj.setclass(space, w_newcls)
diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py
--- a/pypy/objspace/std/stringobject.py
+++ b/pypy/objspace/std/stringobject.py
@@ -161,57 +161,59 @@
 
 def str_swapcase__String(space, w_self):
     self = w_self._value
-    res = [' '] * len(self)
+    builder = StringBuilder(len(self))
     for i in range(len(self)):
         ch = self[i]
         if ch.isupper():
             o = ord(ch) + 32
-            res[i] = chr(o)
+            builder.append(chr(o))
         elif ch.islower():
             o = ord(ch) - 32
-            res[i] = chr(o)
+            builder.append(chr(o))
         else:
-            res[i] = ch
+            builder.append(ch)
 
-    return space.wrap("".join(res))
+    return space.wrap(builder.build())
 
 
 def str_capitalize__String(space, w_self):
     input = w_self._value
-    buffer = [' '] * len(input)
+    builder = StringBuilder(len(input))
     if len(input) > 0:
         ch = input[0]
         if ch.islower():
             o = ord(ch) - 32
-            buffer[0] = chr(o)
+            builder.append(chr(o))
         else:
-            buffer[0] = ch
+            builder.append(ch)
 
         for i in range(1, len(input)):
             ch = input[i]
             if ch.isupper():
                 o = ord(ch) + 32
-                buffer[i] = chr(o)
+                builder.append(chr(o))
             else:
-                buffer[i] = ch
+                builder.append(ch)
 
-    return space.wrap("".join(buffer))
+    return space.wrap(builder.build())
 
 def str_title__String(space, w_self):
     input = w_self._value
-    buffer = [' '] * len(input)
+    builder = StringBuilder(len(input))
     prev_letter=' '
 
-    for pos in range(0, len(input)):
+    for pos in range(len(input)):
         ch = input[pos]
         if not prev_letter.isalpha():
-            buffer[pos] = _upper(ch)
+            ch = _upper(ch)
+            builder.append(ch)
         else:
-            buffer[pos] = _lower(ch)
+            ch = _lower(ch)
+            builder.append(ch)
 
-        prev_letter = buffer[pos]
+        prev_letter = ch
 
-    return space.wrap("".join(buffer))
+    return space.wrap(builder.build())
 
 def str_split__String_None_ANY(space, w_self, w_none, w_maxsplit=-1):
     maxsplit = space.int_w(w_maxsplit)
@@ -754,27 +756,21 @@
     input = w_self._value
     width = space.int_w(w_width)
 
-    if len(input) >= width:
+    num_zeros = width - len(input)
+    if num_zeros <= 0:
         # cannot return w_self, in case it is a subclass of str
         return space.wrap(input)
 
-    buf = [' '] * width
+    builder = StringBuilder(width)
     if len(input) > 0 and (input[0] == '+' or input[0] == '-'):
-        buf[0] = input[0]
+        builder.append(input[0])
         start = 1
-        middle = width - len(input) + 1
     else:
         start = 0
-        middle = width - len(input)
 
-    for i in range(start, middle):
-        buf[i] = '0'
-
-    for i in range(middle, width):
-        buf[i] = input[start]
-        start = start + 1
-
-    return space.wrap("".join(buf))
+    builder.append_multiple_char('0', num_zeros)
+    builder.append_slice(input, start, len(input))
+    return space.wrap(builder.build())
 
 
 def hash__String(space, w_str):
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -10,7 +10,8 @@
 from pypy.objspace.std import identitydict
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.rlib.objectmodel import current_object_addr_as_int, compute_hash
-from pypy.rlib.jit import promote, elidable_promote, we_are_jitted
+from pypy.rlib.jit import promote, elidable_promote, we_are_jitted,\
+     promote_string
 from pypy.rlib.jit import elidable, dont_look_inside, unroll_safe
 from pypy.rlib.rarithmetic import intmask, r_uint
 
@@ -101,6 +102,7 @@
                           'instancetypedef',
                           'terminator',
                           '_version_tag?',
+                          'interplevel_cls',
                           ]
 
     # for config.objspace.std.getattributeshortcut
@@ -399,6 +401,7 @@
         if version_tag is None:
             tup = w_self._lookup_where(name)
             return tup
+        name = promote_string(name)
         w_class, w_value = w_self._pure_lookup_where_with_method_cache(name, version_tag)
         return w_class, unwrap_cell(space, w_value)
 
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -417,54 +417,54 @@
     input = w_self._value
     if len(input) == 0:
         return W_UnicodeObject.EMPTY
-    result = [u'\0'] * len(input)
-    result[0] = unichr(unicodedb.toupper(ord(input[0])))
+    builder = UnicodeBuilder(len(input))
+    builder.append(unichr(unicodedb.toupper(ord(input[0]))))
     for i in range(1, len(input)):
-        result[i] = unichr(unicodedb.tolower(ord(input[i])))
-    return W_UnicodeObject(u''.join(result))
+        builder.append(unichr(unicodedb.tolower(ord(input[i]))))
+    return W_UnicodeObject(builder.build())
 
 def unicode_title__Unicode(space, w_self):
     input = w_self._value
     if len(input) == 0:
         return w_self
-    result = [u'\0'] * len(input)
+    builder = UnicodeBuilder(len(input))
 
     previous_is_cased = False
     for i in range(len(input)):
         unichar = ord(input[i])
         if previous_is_cased:
-            result[i] = unichr(unicodedb.tolower(unichar))
+            builder.append(unichr(unicodedb.tolower(unichar)))
         else:
-            result[i] = unichr(unicodedb.totitle(unichar))
+            builder.append(unichr(unicodedb.totitle(unichar)))
         previous_is_cased = unicodedb.iscased(unichar)
-    return W_UnicodeObject(u''.join(result))
+    return W_UnicodeObject(builder.build())
 
 def unicode_lower__Unicode(space, w_self):
     input = w_self._value
-    result = [u'\0'] * len(input)
+    builder = UnicodeBuilder(len(input))
     for i in range(len(input)):
-        result[i] = unichr(unicodedb.tolower(ord(input[i])))
-    return W_UnicodeObject(u''.join(result))
+        builder.append(unichr(unicodedb.tolower(ord(input[i]))))
+    return W_UnicodeObject(builder.build())
 
 def unicode_upper__Unicode(space, w_self):
     input = w_self._value
-    result = [u'\0'] * len(input)
+    builder = UnicodeBuilder(len(input))
     for i in range(len(input)):
-        result[i] = unichr(unicodedb.toupper(ord(input[i])))
-    return W_UnicodeObject(u''.join(result))
+        builder.append(unichr(unicodedb.toupper(ord(input[i]))))
+    return W_UnicodeObject(builder.build())
 
 def unicode_swapcase__Unicode(space, w_self):
     input = w_self._value
-    result = [u'\0'] * len(input)
+    builder = UnicodeBuilder(len(input))
     for i in range(len(input)):
         unichar = ord(input[i])
         if unicodedb.islower(unichar):
-            result[i] = unichr(unicodedb.toupper(unichar))
+            builder.append(unichr(unicodedb.toupper(unichar)))
         elif unicodedb.isupper(unichar):
-            result[i] = unichr(unicodedb.tolower(unichar))
+            builder.append(unichr(unicodedb.tolower(unichar)))
         else:
-            result[i] = input[i]
-    return W_UnicodeObject(u''.join(result))
+            builder.append(input[i])
+    return W_UnicodeObject(builder.build())
 
 def _normalize_index(length, index):
     if index < 0:
diff --git a/pypy/objspace/taint.py b/pypy/objspace/taint.py
deleted file mode 100644
--- a/pypy/objspace/taint.py
+++ /dev/null
@@ -1,294 +0,0 @@
-"""
-Just an experiment.
-"""
-import os
-from pypy.objspace.std.objspace import StdObjSpace
-from pypy.objspace.proxy import patch_space_in_place
-from pypy.objspace.thunk import nb_forcing_args
-from pypy.interpreter.error import OperationError
-from pypy.interpreter import baseobjspace, gateway, executioncontext
-from pypy.interpreter.function import Method
-from pypy.interpreter.pyframe import PyFrame
-from pypy.tool.sourcetools import func_with_new_name
-from pypy.rlib.unroll import unrolling_iterable
-
-
-class W_Tainted(baseobjspace.W_Root):
-    def __init__(self, w_obj):
-        self.w_obj = w_obj
-
-##    def getdict(self, space):
-##        return taint(self.w_obj.getdict(space))
-
-##    def getdictvalue(self, space, attr):
-##        return taint(self.w_obj.getdictvalue(space, attr))
-
-##    def setdictvalue(self, space, attr, w_value):
-##        return self.w_obj.setdictvalue(space, attr, w_value)
-
-##    ...
-
-class W_TaintBomb(baseobjspace.W_Root):
-    filename = '?'
-    codename = '?'
-    codeline = 0
-
-    def __init__(self, space, operr):
-        self.space = space
-        self.operr = operr
-        self.record_debug_info()
-
-    def record_debug_info(self):
-        ec = self.space.getexecutioncontext()
-        frame = ec.gettopframe_nohidden()
-        if isinstance(frame, PyFrame):     # and, in particular, frame != None
-            self.filename = frame.pycode.co_filename
-            self.codename = frame.pycode.co_name
-            self.codeline = frame.get_last_lineno()
-        if get_debug_level(self.space) > 0:
-            self.debug_dump()
-
-    def debug_dump(self):
-        os.write(2, 'Taint Bomb from file "%s", line %d, in %s\n    %s\n' % (
-            self.filename, self.codeline, self.codename,
-            self.operr.errorstr(self.space)))
-
-    def explode(self):
-        #msg = self.operr.errorstr(space)
-        raise OperationError(self.space.w_TaintError, self.space.w_None)
-
-
-def taint(w_obj):
-    """Return a tainted version of the argument."""
-    if w_obj is None or isinstance(w_obj, W_Tainted):
-        return w_obj
-    else:
-        return W_Tainted(w_obj)
-app_taint = gateway.interp2app(taint)
-
-def is_tainted(space, w_obj):
-    """Return whether the argument is tainted."""
-    res = isinstance(w_obj, W_Tainted) or isinstance(w_obj, W_TaintBomb)
-    return space.wrap(res)
-app_is_tainted = gateway.interp2app(is_tainted)
-
-def untaint(space, w_expectedtype, w_obj):
-    """untaint(expectedtype, tainted_obj) -> obj
-Untaint untainted_obj and return it. If the result is not of expectedtype,
-raise a type error."""
-    if (isinstance(w_expectedtype, W_Tainted) or
-        isinstance(w_expectedtype, W_TaintBomb)):
-        raise OperationError(space.w_TypeError,
-                  space.wrap("untaint() arg 1 must be an untainted type"))
-    if not space.is_true(space.isinstance(w_expectedtype, space.w_type)):
-        raise OperationError(space.w_TypeError,
-                             space.wrap("untaint() arg 1 must be a type"))
-    if isinstance(w_obj, W_Tainted):
-        w_obj = w_obj.w_obj
-    elif isinstance(w_obj, W_TaintBomb):
-        w_obj.explode()
-    #if isinstance(w_expectedtype, W_Tainted):
-    #    w_expectedtype = w_expectedtype.w_obj
-    w_realtype = space.type(w_obj)
-    if not space.is_w(w_realtype, w_expectedtype):
-        #msg = "expected an object of type '%s'" % (
-        #    w_expectedtype.getname(space),)
-        #    #w_realtype.getname(space))
-        raise OperationError(space.w_TaintError, space.w_None)
-    return w_obj
-app_untaint = gateway.interp2app(untaint)
-
-# ____________________________________________________________
-
- at gateway.unwrap_spec(args_w='args_w')
-def taint_atomic_function(space, w_func, args_w):
-    newargs_w = []
-    tainted = False
-    for w_arg in args_w:
-        if isinstance(w_arg, W_Tainted):
-            tainted = True
-            w_arg = w_arg.w_obj
-        elif isinstance(w_arg, W_TaintBomb):
-            return w_arg
-        newargs_w.append(w_arg)
-    w_newargs = space.newtuple(newargs_w)
-    try:
-        w_res = space.call(w_func, w_newargs)
-    except OperationError, operr:
-        if not tainted:
-            raise
-        return W_TaintBomb(space, operr)
-    if tainted:
-        w_res = taint(w_res)
-    return w_res
-
-app_taint_atomic_function = gateway.interp2app(taint_atomic_function)
-
-def taint_atomic(space, w_callable):
-    """decorator to make a callable "taint-atomic": if the function is called
-with tainted arguments, those are untainted. The result of the function is
-tainted again.  All exceptions that the callable raises are turned into
-taint bombs."""
-    meth = Method(space, space.w_fn_taint_atomic_function,
-                  w_callable, space.type(w_callable))
-    return space.wrap(meth)
-app_taint_atomic = gateway.interp2app(taint_atomic)
-
-# ____________________________________________________________
-
-executioncontext.ExecutionContext.taint_debug = 0
-
- at gateway.unwrap_spec(level=int)
-def taint_debug(space, level):
-    """Set the debug level. If the debug level is greater than 0, the creation
-of taint bombs will print debug information. For debugging purposes
-only!"""
-    space.getexecutioncontext().taint_debug = level
-app_taint_debug = gateway.interp2app(taint_debug)
-
-def taint_look(space, w_obj):
-    """Print some info about the taintedness of an object. For debugging
-purposes only!"""
-    if isinstance(w_obj, W_Tainted):
-        info = space.type(w_obj.w_obj).getname(space)
-        msg = space.str_w(w_obj.w_obj.getrepr(space, info))
-        msg = 'Taint Box %s\n' % msg
-        os.write(2, msg)
-    elif isinstance(w_obj, W_TaintBomb):
-        w_obj.debug_dump()
-    else:
-        os.write(2, 'not tainted\n')
-app_taint_look = gateway.interp2app(taint_look)
-
-def get_debug_level(space):
-    return space.getexecutioncontext().taint_debug
-
-def debug_bomb(space, operr):
-    ec = space.getexecutioncontext()
-    filename = '?'
-    codename = '?'
-    codeline = 0
-    frame = ec.gettopframe_nohidden()
-    if isinstance(frame, PyFrame):     # and, in particular, frame != None
-        filename = frame.pycode.co_filename
-        codename = frame.pycode.co_name
-        codeline = frame.get_last_lineno()
-    os.write(2, 'Taint Bomb in file "%s", line %d, in %s\n    %s\n' % (
-        filename, codeline, codename, operr.errorstr(space)))
-
-# ____________________________________________________________
-
-
-class TaintSpace(StdObjSpace):
-
-    def __init__(self, *args, **kwds):
-        StdObjSpace.__init__(self, *args, **kwds)
-        w_dict = self.newdict()
-        self.setitem(w_dict, self.wrap("__doc__"), self.wrap("""\
-Exception that is raised when an operation revealing information on a tainted
-object is performed."""))
-        self.w_TaintError = self.call_function(
-            self.w_type,
-            self.wrap("TaintError"),
-            self.newtuple([self.w_Exception]),
-            w_dict
-            )
-        w___pypy__ = self.getbuiltinmodule("__pypy__")
-        self.setattr(w___pypy__, self.wrap('taint'),
-                     self.wrap(app_taint))
-        self.setattr(w___pypy__, self.wrap('is_tainted'),
-                     self.wrap(app_is_tainted))
-        self.setattr(w___pypy__, self.wrap('untaint'),
-                     self.wrap(app_untaint))
-        self.w_fn_taint_atomic_function = self.wrap(app_taint_atomic_function)
-        self.setattr(w___pypy__, self.wrap('taint_atomic'),
-                     self.wrap(app_taint_atomic))
-        self.setattr(w___pypy__, self.wrap('TaintError'),
-                     self.w_TaintError)
-        self.setattr(w___pypy__, self.wrap('_taint_debug'),
-                     self.wrap(app_taint_debug))
-        self.setattr(w___pypy__, self.wrap('_taint_look'),
-                     self.wrap(app_taint_look))
-        patch_space_in_place(self, 'taint', proxymaker)
-
-        # XXX may leak info, perfomance hit, what about taint bombs?
-        from pypy.objspace.std.typeobject import W_TypeObject
-
-        def taint_lookup(w_obj, name):
-            if isinstance(w_obj, W_Tainted):
-                w_obj = w_obj.w_obj
-            w_type = self.type(w_obj)
-            assert isinstance(w_type, W_TypeObject)
-            return w_type.lookup(name)
-
-        def taint_lookup_in_type_where(w_obj, name):
-            if isinstance(w_obj, W_Tainted):
-                w_type = w_obj.w_obj
-            else:
-                w_type = w_obj
-            assert isinstance(w_type, W_TypeObject)
-            return w_type.lookup_where(name)
-
-        self.lookup = taint_lookup
-        self.lookup_in_type_where = taint_lookup_in_type_where
-
-
-Space = TaintSpace
-
-
-def tainted_error(space, name):
-    #msg = "operation '%s' forbidden on tainted object" % (name,)
-    raise OperationError(space.w_TaintError, space.w_None)# space.wrap(msg))
-
-
-RegularMethods = dict.fromkeys(
-    [name for name, _, _, _ in baseobjspace.ObjSpace.MethodTable])
-
-TaintResultIrregularMethods = dict.fromkeys(
-    ['wrap', 'call_args'] +
-    [name for name in baseobjspace.ObjSpace.IrregularOpTable
-          if name.startswith('new')])
-
-def proxymaker(space, name, parentfn):
-    arity = nb_forcing_args[name]
-    indices = unrolling_iterable(range(arity))
-    if name in RegularMethods:
-
-        def proxy(*args_w):
-            newargs_w = ()
-            tainted = False
-            for i in indices:
-                w_arg = args_w[i]
-                if isinstance(w_arg, W_Tainted):
-                    tainted = True
-                    w_arg = w_arg.w_obj
-                elif isinstance(w_arg, W_TaintBomb):
-                    return w_arg
-                newargs_w += (w_arg,)
-            newargs_w += args_w[arity:]
-            try:
-                w_res = parentfn(*newargs_w)
-            except OperationError, operr:
-                if not tainted:
-                    raise
-                return W_TaintBomb(space, operr)
-            if tainted:
-                w_res = taint(w_res)
-            return w_res
-
-    elif arity == 0:
-        return None
-
-    else:
-
-        def proxy(*args_w):
-            for i in indices:
-                w_arg = args_w[i]
-                if isinstance(w_arg, W_Tainted):
-                    tainted_error(space, name)
-                elif isinstance(w_arg, W_TaintBomb):
-                    w_arg.explode()
-            return parentfn(*args_w)
-
-    proxy = func_with_new_name(proxy, '%s_proxy' % name)
-    return proxy
diff --git a/pypy/objspace/test/test_taintobjspace.py b/pypy/objspace/test/test_taintobjspace.py
deleted file mode 100644
--- a/pypy/objspace/test/test_taintobjspace.py
+++ /dev/null
@@ -1,77 +0,0 @@
-from pypy.conftest import gettestobjspace
-
-class AppTest_Taint:
-
-    def setup_class(cls):
-        cls.space = gettestobjspace('taint')
-
-    def test_simple(self):
-        from __pypy__ import taint, untaint, TaintError
-        x = taint(6)
-        x = x * 7
-        raises(TaintError, "if x: y = 1")
-        t = type(x)
-        raises(TaintError, "if t is int: y = 1")
-        assert untaint(int, x) == 42
-        raises(TaintError, "untaint(float, x)")
-
-    def test_bomb(self):
-        from __pypy__ import taint, untaint, TaintError
-        x = taint(6)
-        x = x / 0
-        raises(TaintError, "if x: y = 1")
-        t = type(x)
-        raises(TaintError, "if t is int: y = 1")
-        raises(TaintError, "untaint(int, x)")
-        raises(TaintError, "untaint(float, x)")
-
-    def test_taint_atomic(self):
-        from __pypy__ import taint, untaint, TaintError, taint_atomic
-        x = taint(6)
-        x *= 7
-
-        def dummy(x):
-            if x > 40:
-                return 5
-            else:
-                return 3
-        dummy = taint_atomic(dummy)
-
-        y = dummy(x)
-        raises(TaintError, "if y == 3: z = 1")
-        assert untaint(int, y) == 5
-
-    def test_taint_atomic_exception(self):
-        from __pypy__ import taint, untaint, TaintError, taint_atomic
-        x = taint(6)
-        x *= 7
-
-        def dummy(x):
-            if x + "world" == "hello world":
-                return 5
-            else:
-                return 3
-        dummy = taint_atomic(dummy)
-
-        y = dummy(x)
-        raises(TaintError, "if y == 3: z = 1")
-        raises(TaintError, "untaint(int, y)")
-
-    def test_taint_atomic_incoming_bomb(self):
-        from __pypy__ import taint, untaint, TaintError, taint_atomic
-        x = taint(6)
-        x /= 0
-        lst = []
-
-        def dummy(x):
-            lst.append("running!")
-            if x > 40:
-                return 5
-            else:
-                return 3
-        dummy = taint_atomic(dummy)
-
-        y = dummy(x)
-        raises(TaintError, "if y == 3: z = 1")
-        assert lst == []
-        raises(TaintError, "untaint(int, y)")
diff --git a/pypy/rlib/_jit_vref.py b/pypy/rlib/_jit_vref.py
--- a/pypy/rlib/_jit_vref.py
+++ b/pypy/rlib/_jit_vref.py
@@ -25,6 +25,10 @@
     def simple_call(self):
         return self.s_instance
 
+    def getattr(self, s_attr):
+        assert s_attr.const == 'virtual'
+        return annmodel.s_Bool
+
     def rtyper_makerepr(self, rtyper):
         if rtyper.type_system.name == 'lltypesystem':
             return vrefrepr
@@ -61,6 +65,13 @@
                              " prebuilt virtual_ref")
         return lltype.nullptr(OBJECTPTR.TO)
 
+    def rtype_getattr(self, hop):
+        s_attr = hop.args_s[1]
+        assert s_attr.const == 'virtual'
+        v = hop.inputarg(self, arg=0)
+        hop.exception_cannot_occur()
+        return hop.genop('jit_is_virtual', [v], resulttype = lltype.Bool)
+
 from pypy.rpython.ootypesystem.rclass import OBJECT
 
 class OOVRefRepr(VRefRepr):
diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -38,6 +38,7 @@
     possible arguments are:
 
     * promote - promote the argument from a variable into a constant
+    * promote_string - same, but promote string by *value*
     * access_directly - directly access a virtualizable, as a structure
                         and don't treat it as a virtualizable
     * fresh_virtualizable - means that virtualizable was just allocated.
@@ -51,6 +52,9 @@
 def promote(x):
     return hint(x, promote=True)
 
+def promote_string(x):
+    return hint(x, promote_string=True)
+
 def dont_look_inside(func):
     """ Make sure the JIT does not trace inside decorated function
     (it becomes a call instead)
@@ -313,6 +317,12 @@
             raise InvalidVirtualRef
         return self._x
 
+    @property
+    def virtual(self):
+        """A property that is True if the vref contains a virtual that would
+        be forced by the '()' operator."""
+        return self._state == 'non-forced'
+
     def _finish(self):
         if self._state == 'non-forced':
             self._state = 'invalid'
diff --git a/pypy/rlib/rerased.py b/pypy/rlib/rerased.py
--- a/pypy/rlib/rerased.py
+++ b/pypy/rlib/rerased.py
@@ -218,15 +218,13 @@
         [v_value] = hop.inputargs(lltype.Signed)
         c_one = hop.inputconst(lltype.Signed, 1)
         hop.exception_is_here()
-        v2 = hop.genop('int_lshift_ovf', [v_value, c_one],
+        v2 = hop.genop('int_add_ovf', [v_value, v_value],
                        resulttype = lltype.Signed)
         v2p1 = hop.genop('int_add', [v2, c_one],
                          resulttype = lltype.Signed)
         v_instance = hop.genop('cast_int_to_ptr', [v2p1],
                                resulttype=self.lowleveltype)
-        v = hop.genop('cast_opaque_ptr', [v_instance],
-                      resulttype=self.lowleveltype)
-        return v
+        return v_instance
 
     def convert_const(self, value):
         if value._identity is _identity_for_ints:
@@ -266,10 +264,10 @@
         return hop.genop('int_rshift', [v2, c_one], resulttype=lltype.Signed)
 
     def rtype_erase_int(self, hop):
-        hop.exception_is_here()
         [v_value] = hop.inputargs(lltype.Signed)
         c_one = hop.inputconst(lltype.Signed, 1)
-        v2 = hop.genop('int_lshift_ovf', [v_value, c_one],
+        hop.exception_is_here()
+        v2 = hop.genop('int_add_ovf', [v_value, v_value],
                        resulttype = lltype.Signed)
         v2p1 = hop.genop('int_add', [v2, c_one],
                          resulttype = lltype.Signed)
diff --git a/pypy/rlib/streamio.py b/pypy/rlib/streamio.py
--- a/pypy/rlib/streamio.py
+++ b/pypy/rlib/streamio.py
@@ -37,7 +37,7 @@
 # return value of tell(), but not as argument to read().
 #
 
-import os, sys
+import os, sys, errno
 from pypy.rlib.objectmodel import specialize, we_are_translated
 from pypy.rlib.rarithmetic import r_longlong, intmask
 from pypy.rlib import rposix
@@ -587,12 +587,22 @@
     def readall(self):
         pos = self.pos
         assert pos >= 0
-        chunks = [self.buf[pos:]]
+        if self.buf:
+            chunks = [self.buf[pos:]]
+        else:
+            chunks = []
         self.buf = ""
         self.pos = 0
         bufsize = self.bufsize
         while 1:
-            data = self.do_read(bufsize)
+            try:
+                data = self.do_read(bufsize)
+            except OSError, o:
+                if o.errno != errno.EAGAIN:
+                    raise
+                if not chunks:
+                    raise
+                break
             if not data:
                 break
             chunks.append(data)
diff --git a/pypy/rlib/test/test__jit_vref.py b/pypy/rlib/test/test__jit_vref.py
--- a/pypy/rlib/test/test__jit_vref.py
+++ b/pypy/rlib/test/test__jit_vref.py
@@ -27,10 +27,13 @@
     x1 = X()
     vref = virtual_ref(x1)
     assert vref._state == 'non-forced'
+    assert vref.virtual is True
     assert vref() is x1
     assert vref._state == 'forced'
+    assert vref.virtual is False
     virtual_ref_finish(vref, x1)
     assert vref._state == 'forced'
+    assert vref.virtual is False
     assert vref() is x1
 
 def test_direct_invalid():
@@ -135,6 +138,13 @@
         x = self.interpret(f, [])
         assert x == 42
 
+    def test_rtype_virtualattr(self):
+        def f():
+            vref = virtual_ref(X())
+            return vref.virtual
+        x = self.interpret(f, [])
+        assert x is False
+
 
 class TestLLtype(BaseTestVRef, LLRtypeMixin):
     OBJECTTYPE = OBJECTPTR
diff --git a/pypy/rlib/test/test_jit.py b/pypy/rlib/test/test_jit.py
--- a/pypy/rlib/test/test_jit.py
+++ b/pypy/rlib/test/test_jit.py
@@ -139,12 +139,11 @@
 
     def test_isconstant(self):
         def f(n):
-            assert n >= 0
             assert isconstant(n) is False
             l = []
             l.append(n)
             return len(l)
-        res = self.interpret(f, [234])
+        res = self.interpret(f, [-234])
         assert res == 1
 
 
diff --git a/pypy/rpython/lltypesystem/ll2ctypes.py b/pypy/rpython/lltypesystem/ll2ctypes.py
--- a/pypy/rpython/lltypesystem/ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/ll2ctypes.py
@@ -658,6 +658,8 @@
         if T == llmemory.GCREF:
             if isinstance(llobj._obj, _llgcopaque):
                 return ctypes.c_void_p(llobj._obj.intval)
+            if isinstance(llobj._obj, int):    # tagged pointer
+                return ctypes.c_void_p(llobj._obj)
             container = llobj._obj.container
             T = lltype.Ptr(lltype.typeOf(container))
             # otherwise it came from integer and we want a c_void_p with
@@ -1268,6 +1270,7 @@
 class _llgcopaque(lltype._container):
     _TYPE = llmemory.GCREF.TO
     _name = "_llgcopaque"
+    _read_directly_intval = True     # for _ptr._cast_to_int()
 
     def __init__(self, void_p):
         if isinstance(void_p, (int, long)):
@@ -1288,6 +1291,9 @@
             return False
         return force_cast(lltype.Signed, other._as_ptr()) == self.intval
 
+    def __hash__(self):
+        return self.intval
+
     def __ne__(self, other):
         return not self == other
 
@@ -1296,11 +1302,6 @@
             return _opaque_objs[self.intval // 2]
         return force_cast(PTRTYPE, self.intval)
 
-##     def _cast_to_int(self):
-##         return self.intval
-
-##     def _cast_to_adr(self):
-##         return _lladdress(self.intval)
 
 def cast_adr_to_int(addr):
     if isinstance(addr, llmemory.fakeaddress):
diff --git a/pypy/rpython/lltypesystem/llmemory.py b/pypy/rpython/lltypesystem/llmemory.py
--- a/pypy/rpython/lltypesystem/llmemory.py
+++ b/pypy/rpython/lltypesystem/llmemory.py
@@ -498,6 +498,8 @@
 
     def _cast_to_int(self, symbolic=False):
         if self:
+            if isinstance(self.ptr._obj0, int):    # tagged integer
+                return self.ptr._obj0
             if symbolic:
                 return AddressAsInt(self)
             else:
diff --git a/pypy/rpython/lltypesystem/lloperation.py b/pypy/rpython/lltypesystem/lloperation.py
--- a/pypy/rpython/lltypesystem/lloperation.py
+++ b/pypy/rpython/lltypesystem/lloperation.py
@@ -428,6 +428,7 @@
     'jit_marker':           LLOp(),
     'jit_force_virtualizable':LLOp(canrun=True),
     'jit_force_virtual':    LLOp(canrun=True),
+    'jit_is_virtual':       LLOp(canrun=True),
     'jit_force_quasi_immutable': LLOp(canrun=True),
     'get_exception_addr':   LLOp(),
     'get_exc_value_addr':   LLOp(),
diff --git a/pypy/rpython/lltypesystem/lltype.py b/pypy/rpython/lltypesystem/lltype.py
--- a/pypy/rpython/lltypesystem/lltype.py
+++ b/pypy/rpython/lltypesystem/lltype.py
@@ -1360,6 +1360,8 @@
         obj = normalizeptr(self, check)._getobj(check)
         if isinstance(obj, int):
             return obj     # special case for cast_int_to_ptr() results put into opaques
+        if getattr(obj, '_read_directly_intval', False):
+            return obj.intval   # special case for _llgcopaque
         result = intmask(obj._getid())
         # assume that id() returns an addressish value which is
         # not zero and aligned to at least a multiple of 4
diff --git a/pypy/rpython/lltypesystem/opimpl.py b/pypy/rpython/lltypesystem/opimpl.py
--- a/pypy/rpython/lltypesystem/opimpl.py
+++ b/pypy/rpython/lltypesystem/opimpl.py
@@ -538,6 +538,9 @@
 def op_jit_force_virtual(x):
     return x
 
+def op_jit_is_virtual(x):
+    return False
+
 def op_jit_force_quasi_immutable(*args):
     pass
 
diff --git a/pypy/rpython/lltypesystem/rtagged.py b/pypy/rpython/lltypesystem/rtagged.py
--- a/pypy/rpython/lltypesystem/rtagged.py
+++ b/pypy/rpython/lltypesystem/rtagged.py
@@ -43,7 +43,7 @@
         v_value = hop.inputarg(lltype.Signed, arg=1)
         c_one = hop.inputconst(lltype.Signed, 1)
         hop.exception_is_here()
-        v2 = hop.genop('int_lshift_ovf', [v_value, c_one],
+        v2 = hop.genop('int_add_ovf', [v_value, v_value],
                        resulttype = lltype.Signed)
         v2p1 = hop.genop('int_add', [v2, c_one],
                          resulttype = lltype.Signed)
diff --git a/pypy/rpython/lltypesystem/test/test_ll2ctypes.py b/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
--- a/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
@@ -1123,6 +1123,9 @@
 
         #assert lltype.cast_ptr_to_int(ref1) == intval
 
+        x = rffi.cast(llmemory.GCREF, -17)
+        assert lltype.cast_ptr_to_int(x) == -17
+
     def test_ptr_truth(self):
         abc = rffi.cast(lltype.Ptr(lltype.FuncType([], lltype.Void)), 0)
         assert not abc
diff --git a/pypy/rpython/rclass.py b/pypy/rpython/rclass.py
--- a/pypy/rpython/rclass.py
+++ b/pypy/rpython/rclass.py
@@ -191,6 +191,10 @@
                     "class %r inherits from its parent _immutable_=True, "
                     "so it should also declare _immutable_=True" % (
                     self.classdef,))
+            if loc.classdict.get('_immutable_').value is not True:
+                raise TyperError(
+                    "class %r: _immutable_ = something else than True" % (
+                    self.classdef,))
             hints = hints.copy()
             hints['immutable'] = True
         self.immutable_field_set = set()  # unless overwritten below
diff --git a/pypy/tool/logparser.py b/pypy/tool/logparser.py
--- a/pypy/tool/logparser.py
+++ b/pypy/tool/logparser.py
@@ -298,6 +298,8 @@
         image.paste(textpercent, (t1x, 5), textpercent)
         image.paste(textlabel,   (t2x, 5), textlabel)
         images.append(image)
+    if not images:
+        return None
     return combine(images, spacing=0, border=1, horizontal=False)
 
 def get_timesummary_single_image(totaltimes, totaltime0, componentdict,
@@ -333,6 +335,8 @@
         del totaltimes[None]
     img2 = render_histogram(totaltimes, totaltime0, {},
                             width, summarybarheight)
+    if img2 is None:
+        return img1
     return combine([img1, img2], spacing=spacing, horizontal=True)
 
 # ----------
diff --git a/pypy/translator/c/funcgen.py b/pypy/translator/c/funcgen.py
--- a/pypy/translator/c/funcgen.py
+++ b/pypy/translator/c/funcgen.py
@@ -833,7 +833,7 @@
         return 'INSTRUMENT_COUNT(%s);' % counter_label
             
     def OP_IS_EARLY_CONSTANT(self, op):
-        return self.expr(op.result)  + ' = 0;' # Allways false
+        return '%s = 0; /* IS_EARLY_CONSTANT */' % (self.expr(op.result),)
 
     def OP_JIT_MARKER(self, op):
         return '/* JIT_MARKER %s */' % op
@@ -845,6 +845,9 @@
         return '%s = %s; /* JIT_FORCE_VIRTUAL */' % (self.expr(op.result),
                                                      self.expr(op.args[0]))
 
+    def OP_JIT_IS_VIRTUAL(self, op):
+        return '%s = 0; /* JIT_IS_VIRTUAL */' % (self.expr(op.result),)
+
     def OP_JIT_FORCE_QUASI_IMMUTABLE(self, op):
         return '/* JIT_FORCE_QUASI_IMMUTABLE %s */' % op
 
diff --git a/pypy/translator/c/genc.py b/pypy/translator/c/genc.py
--- a/pypy/translator/c/genc.py
+++ b/pypy/translator/c/genc.py
@@ -8,7 +8,7 @@
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
 from pypy.rpython.lltypesystem import lltype
 from pypy.tool.udir import udir
-from pypy.tool import isolate
+from pypy.tool import isolate, runsubprocess
 from pypy.translator.c.support import log, c_string_constant
 from pypy.rpython.typesystem import getfunctionptr
 from pypy.translator.c import gc
@@ -563,14 +563,19 @@
             else:
                 mk.definition('PYPY_MAIN_FUNCTION', "main")
 
-            if (py.path.local.sysfind('python') or
-                py.path.local.sysfind('python.exe')):
-                python = 'python '
-            elif sys.platform == 'win32':
+            if sys.platform == 'win32':
                 python = sys.executable.replace('\\', '/') + ' '
             else:
                 python = sys.executable + ' '
 
+            # Is there a command 'python' that runs python 2.5-2.7?
+            # If there is, then we can use it instead of sys.executable
+            returncode, stdout, stderr = runsubprocess.run_subprocess(
+                "python", "-V")
+            if (stdout.startswith('Python 2.') or
+                stderr.startswith('Python 2.')):
+                python = 'python '
+
             if self.translator.platform.name == 'msvc':
                 lblofiles = []
                 for cfile in mk.cfiles:
diff --git a/pypy/translator/c/src/thread_nt.h b/pypy/translator/c/src/thread_nt.h
--- a/pypy/translator/c/src/thread_nt.h
+++ b/pypy/translator/c/src/thread_nt.h
@@ -245,7 +245,7 @@
     if (pending_acquires <= 0)
         return 0;
     InterlockedIncrement(&pending_acquires);
-    PulseEvent(&cond_gil);
+    PulseEvent(cond_gil);
 
     /* hack: the three following lines do a pthread_cond_wait(), and
        normally specifying a timeout of INFINITE would be fine.  But the
@@ -253,7 +253,7 @@
        (small) risk that PulseEvent misses the WaitForSingleObject().
        In this case the process will just sleep a few milliseconds. */
     LeaveCriticalSection(&mutex_gil);
-    WaitForSingleObject(&cond_gil, 15);
+    WaitForSingleObject(cond_gil, 15);
     EnterCriticalSection(&mutex_gil);
 
     InterlockedDecrement(&pending_acquires);
@@ -263,7 +263,7 @@
 void RPyGilRelease(void)
 {
     LeaveCriticalSection(&mutex_gil);
-    PulseEvent(&cond_gil);
+    PulseEvent(cond_gil);
 }
 
 void RPyGilAcquire(void)
diff --git a/pypy/translator/goal/app_main.py b/pypy/translator/goal/app_main.py
--- a/pypy/translator/goal/app_main.py
+++ b/pypy/translator/goal/app_main.py
@@ -144,7 +144,7 @@
     print '  --jit off                  turn off the JIT'
 
 def print_version(*args):
-    print "Python", sys.version
+    print >> sys.stderr, "Python", sys.version
     raise SystemExit
 
 def set_jit_option(options, jitparam, *args):
diff --git a/pypy/translator/platform/linux.py b/pypy/translator/platform/linux.py
--- a/pypy/translator/platform/linux.py
+++ b/pypy/translator/platform/linux.py
@@ -24,15 +24,14 @@
         return self._pkg_config("libffi", "--libs-only-L",
                                 ['/usr/lib/libffi'])
 
+    def library_dirs_for_libffi_a(self):
+        # places where we need to look for libffi.a
+        return self.library_dirs_for_libffi() + ['/usr/lib']
+
 
 class Linux(BaseLinux):
     shared_only = ()    # it seems that on 32-bit linux, compiling with -fPIC
                         # gives assembler that asmgcc is not happy about.
 
-    def library_dirs_for_libffi_a(self):
-        # places where we need to look for libffi.a
-        return self.library_dirs_for_libffi() + ['/usr/lib']
-
-
 class Linux64(BaseLinux):
     pass

From noreply at buildbot.pypy.org  Mon Oct 17 04:32:25 2011
From: noreply at buildbot.pypy.org (justinpeel)
Date: Mon, 17 Oct 2011 04:32:25 +0200 (CEST)
Subject: [pypy-commit] pypy rgc-mem-pressure: small fix
Message-ID: <20111017023225.BC147820D7@wyvern.cs.uni-duesseldorf.de>

Author: Justin Peel <notmuchtotell at gmail.com>
Branch: rgc-mem-pressure
Changeset: r48107:89bac1cc8a60
Date: 2011-10-16 20:32 -0600
http://bitbucket.org/pypy/pypy/changeset/89bac1cc8a60/

Log:	small fix

diff --git a/pypy/rpython/memory/gctransform/framework.py b/pypy/rpython/memory/gctransform/framework.py
--- a/pypy/rpython/memory/gctransform/framework.py
+++ b/pypy/rpython/memory/gctransform/framework.py
@@ -386,7 +386,7 @@
                 raw_malloc_memory_pressure_varsize,
                 [annmodel.SomeInteger(), annmodel.SomeInteger()],
                 annmodel.s_None, minimal_transform = False)
-            self.raw_malloc_memory_pressure = getfn(
+            self.raw_malloc_memory_pressure_ptr = getfn(
                 raw_malloc_memory_pressure,
                 [annmodel.SomeInteger()],
                 annmodel.s_None, minimal_transform = False)

From noreply at buildbot.pypy.org  Mon Oct 17 04:52:47 2011
From: noreply at buildbot.pypy.org (justinpeel)
Date: Mon, 17 Oct 2011 04:52:47 +0200 (CEST)
Subject: [pypy-commit] pypy rgc-mem-pressure: add memory pressure entry
Message-ID: <20111017025247.33C7C820D7@wyvern.cs.uni-duesseldorf.de>

Author: Justin Peel <notmuchtotell at gmail.com>
Branch: rgc-mem-pressure
Changeset: r48108:ac989ce70d94
Date: 2011-10-16 20:52 -0600
http://bitbucket.org/pypy/pypy/changeset/ac989ce70d94/

Log:	add memory pressure entry

diff --git a/pypy/rlib/rgc.py b/pypy/rlib/rgc.py
--- a/pypy/rlib/rgc.py
+++ b/pypy/rlib/rgc.py
@@ -259,6 +259,20 @@
     """Add memory pressure for OpaquePtrs."""
     pass
 
+class AddMemoryPressureEntry(ExtRegistryEntry):
+    _about_ = add_memory_pressure
+
+    def compute_result_annotation(self, s_nbytes):
+        from pypy.annotation import model as annmodel
+        return annmodel.s_None
+
+    def specialize_call(self, hop):
+        [v_size] = hop.inputargs(lltype.Signed)
+        hop.exception_cannot_occur()
+        return hop.genop('gc_add_memory_pressure', [v_size],
+                         resulttype=lltype.Void)
+
+
 def get_rpy_memory_usage(gcref):
     "NOT_RPYTHON"
     # approximate implementation using CPython's type info

From noreply at buildbot.pypy.org  Mon Oct 17 05:49:18 2011
From: noreply at buildbot.pypy.org (justinpeel)
Date: Mon, 17 Oct 2011 05:49:18 +0200 (CEST)
Subject: [pypy-commit] pypy rgc-mem-pressure: more fixes
Message-ID: <20111017034918.A92F5820D7@wyvern.cs.uni-duesseldorf.de>

Author: Justin Peel <notmuchtotell at gmail.com>
Branch: rgc-mem-pressure
Changeset: r48109:4bfcf92709a4
Date: 2011-10-16 21:49 -0600
http://bitbucket.org/pypy/pypy/changeset/4bfcf92709a4/

Log:	more fixes

diff --git a/pypy/rpython/memory/gctransform/transform.py b/pypy/rpython/memory/gctransform/transform.py
--- a/pypy/rpython/memory/gctransform/transform.py
+++ b/pypy/rpython/memory/gctransform/transform.py
@@ -562,7 +562,7 @@
     def gct_add_memory_pressure(self, hop):
         if hasattr(self, 'add_memory_pressure_ptr'):
             op = hop.spaceop
-            size = hop.args[0]
+            size = op.args[0]
             hop.genop("direct_call",
                           [self.raw_malloc_memory_pressure_ptr,
                            size])
@@ -598,9 +598,9 @@
     def gct_fv_raw_malloc_varsize(self, hop, flags, TYPE, v_length, c_const_size, c_item_size,
                                                                     c_offset_to_length):
         if flags.get('add_memory_pressure', False):
-            if hasattr(self, 'raw_malloc_memory_pressure_ptr'):
+            if hasattr(self, 'raw_malloc_memory_pressure_varsize_ptr'):
                 hop.genop("direct_call",
-                          [self.raw_malloc_memory_pressure_ptr,
+                          [self.raw_malloc_memory_pressure_varsize_ptr,
                            v_length, c_item_size])
         if c_offset_to_length is None:
             if flags.get('zero'):

From noreply at buildbot.pypy.org  Mon Oct 17 07:22:25 2011
From: noreply at buildbot.pypy.org (justinpeel)
Date: Mon, 17 Oct 2011 07:22:25 +0200 (CEST)
Subject: [pypy-commit] pypy rgc-mem-pressure: small fix
Message-ID: <20111017052225.A497F820D7@wyvern.cs.uni-duesseldorf.de>

Author: Justin Peel <notmuchtotell at gmail.com>
Branch: rgc-mem-pressure
Changeset: r48110:8b7a07081692
Date: 2011-10-16 23:22 -0600
http://bitbucket.org/pypy/pypy/changeset/8b7a07081692/

Log:	small fix

diff --git a/pypy/rpython/llinterp.py b/pypy/rpython/llinterp.py
--- a/pypy/rpython/llinterp.py
+++ b/pypy/rpython/llinterp.py
@@ -716,7 +716,7 @@
         track_allocation = flags.get('track_allocation', True)
         self.heap.free(obj, flavor='raw', track_allocation=track_allocation)
 
-    def add_memory_pressure(self, size):
+    def op_gc_add_memory_pressure(self, size):
         self.heap.add_memory_pressure(size)
 
     def op_shrink_array(self, obj, smallersize):

From noreply at buildbot.pypy.org  Mon Oct 17 09:56:53 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Oct 2011 09:56:53 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix a test in test_optimizebasic,
	a bit obscurely.
Message-ID: <20111017075653.9F451820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48111:2bda02052847
Date: 2011-10-17 09:56 +0200
http://bitbucket.org/pypy/pypy/changeset/2bda02052847/

Log:	Fix a test in test_optimizebasic, a bit obscurely.

diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py
--- a/pypy/jit/metainterp/resoperation.py
+++ b/pypy/jit/metainterp/resoperation.py
@@ -431,11 +431,11 @@
     'INT_IS_TRUE/1b',
     'INT_NEG/1',
     'INT_INVERT/1',
+    #
+    'SAME_AS/1',      # gets a Const or a Box, turns it into another Box
     'CAST_PTR_TO_INT/1',
     'CAST_INT_TO_PTR/1',
     #
-    'SAME_AS/1',      # gets a Const or a Box, turns it into another Box
-    #
     'PTR_EQ/2b',
     'PTR_NE/2b',
     'CAST_OPAQUE_PTR/1b',

From noreply at buildbot.pypy.org  Mon Oct 17 09:56:54 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Oct 2011 09:56:54 +0200 (CEST)
Subject: [pypy-commit] pypy default: Python 2.5 compat
Message-ID: <20111017075654.D7C7B820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48112:d1b067df71bd
Date: 2011-10-17 09:56 +0200
http://bitbucket.org/pypy/pypy/changeset/d1b067df71bd/

Log:	Python 2.5 compat

diff --git a/pypy/jit/metainterp/test/test_resume.py b/pypy/jit/metainterp/test/test_resume.py
--- a/pypy/jit/metainterp/test/test_resume.py
+++ b/pypy/jit/metainterp/test/test_resume.py
@@ -1,3 +1,4 @@
+from __future__ import with_statement
 import py
 from pypy.rpython.lltypesystem import lltype, llmemory, rffi
 from pypy.jit.metainterp.optimizeopt.optimizer import OptValue

From noreply at buildbot.pypy.org  Mon Oct 17 10:16:57 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Oct 2011 10:16:57 +0200 (CEST)
Subject: [pypy-commit] pypy default: Ah,
 I misread pick_builtin(): it doesn't have the side-effect of adding
Message-ID: <20111017081657.32F22820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48113:e813668eef85
Date: 2011-10-17 10:16 +0200
http://bitbucket.org/pypy/pypy/changeset/e813668eef85/

Log:	Ah, I misread pick_builtin(): it doesn't have the side-effect of
	adding '__builtins__' in the provided w_globals dictionary. Remove,
	and comment.

diff --git a/pypy/module/__builtin__/__init__.py b/pypy/module/__builtin__/__init__.py
--- a/pypy/module/__builtin__/__init__.py
+++ b/pypy/module/__builtin__/__init__.py
@@ -119,7 +119,7 @@
            builtin = space.interpclass_w(w_builtin)
            if isinstance(builtin, module.Module):
                return builtin
-       # no builtin! make a default one.  Given them None, at least.
+       # no builtin! make a default one.  Give them None, at least.
        builtin = module.Module(space, None)
        space.setitem(builtin.w_dict, space.wrap('None'), space.w_None)
        return builtin
diff --git a/pypy/module/__builtin__/compiling.py b/pypy/module/__builtin__/compiling.py
--- a/pypy/module/__builtin__/compiling.py
+++ b/pypy/module/__builtin__/compiling.py
@@ -97,6 +97,9 @@
     elif space.is_w(w_locals, space.w_None):
         w_locals = w_globals
 
-    space.builtin.pick_builtin(w_globals)
+    # xxx removed: adding '__builtins__' to the w_globals dict, if there
+    # is none.  This logic was removed as costly (it requires to get at
+    # the gettopframe_nohidden()).  I bet no test fails, and it's a really
+    # obscure case.
 
     return codeobj.exec_code(space, w_globals, w_locals)

From noreply at buildbot.pypy.org  Mon Oct 17 10:39:20 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Oct 2011 10:39:20 +0200 (CEST)
Subject: [pypy-commit] pypy rgc-mem-pressure: Fixes (untested).
Message-ID: <20111017083920.F0535820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: rgc-mem-pressure
Changeset: r48114:0ec2ab675a61
Date: 2011-10-17 10:38 +0200
http://bitbucket.org/pypy/pypy/changeset/0ec2ab675a61/

Log:	Fixes (untested).

diff --git a/pypy/rpython/memory/gctransform/transform.py b/pypy/rpython/memory/gctransform/transform.py
--- a/pypy/rpython/memory/gctransform/transform.py
+++ b/pypy/rpython/memory/gctransform/transform.py
@@ -559,8 +559,8 @@
     def gct_malloc_nonmovable_varsize(self, *args, **kwds):
         return self.gct_malloc_varsize(*args, **kwds)
 
-    def gct_add_memory_pressure(self, hop):
-        if hasattr(self, 'add_memory_pressure_ptr'):
+    def gct_gc_add_memory_pressure(self, hop):
+        if hasattr(self, 'raw_malloc_memory_pressure_ptr'):
             op = hop.spaceop
             size = op.args[0]
             hop.genop("direct_call",
diff --git a/pypy/rpython/memory/gcwrapper.py b/pypy/rpython/memory/gcwrapper.py
--- a/pypy/rpython/memory/gcwrapper.py
+++ b/pypy/rpython/memory/gcwrapper.py
@@ -66,7 +66,8 @@
         return result
 
     def add_memory_pressure(self, size):
-        self.gc.add_memory_pressure(self)
+        if hasattr(self.gc, 'raw_malloc_memory_pressure'):
+            self.gc.raw_malloc_memory_pressure(self)
 
     def shrink_array(self, p, smallersize):
         if hasattr(self.gc, 'shrink_array'):

From noreply at buildbot.pypy.org  Mon Oct 17 11:46:28 2011
From: noreply at buildbot.pypy.org (timo_jbo)
Date: Mon, 17 Oct 2011 11:46:28 +0200 (CEST)
Subject: [pypy-commit] pypy separate-applevel-numpy: merge in default
Message-ID: <20111017094628.6DCFA820D7@wyvern.cs.uni-duesseldorf.de>

Author: Timo Paulssen <timonator at perpetuum-immobile.de>
Branch: separate-applevel-numpy
Changeset: r48115:fd6ff475c392
Date: 2011-10-17 11:42 +0200
http://bitbucket.org/pypy/pypy/changeset/fd6ff475c392/

Log:	merge in default

diff --git a/lib-python/modified-2.7/httplib.py b/lib-python/modified-2.7/httplib.py
new file mode 100644
--- /dev/null
+++ b/lib-python/modified-2.7/httplib.py
@@ -0,0 +1,1377 @@
+"""HTTP/1.1 client library
+
+<intro stuff goes here>
+<other stuff, too>
+
+HTTPConnection goes through a number of "states", which define when a client
+may legally make another request or fetch the response for a particular
+request. This diagram details these state transitions:
+
+    (null)
+      |
+      | HTTPConnection()
+      v
+    Idle
+      |
+      | putrequest()
+      v
+    Request-started
+      |
+      | ( putheader() )*  endheaders()
+      v
+    Request-sent
+      |
+      | response = getresponse()
+      v
+    Unread-response   [Response-headers-read]
+      |\____________________
+      |                     |
+      | response.read()     | putrequest()
+      v                     v
+    Idle                  Req-started-unread-response
+                     ______/|
+                   /        |
+   response.read() |        | ( putheader() )*  endheaders()
+                   v        v
+       Request-started    Req-sent-unread-response
+                            |
+                            | response.read()
+                            v
+                          Request-sent
+
+This diagram presents the following rules:
+  -- a second request may not be started until {response-headers-read}
+  -- a response [object] cannot be retrieved until {request-sent}
+  -- there is no differentiation between an unread response body and a
+     partially read response body
+
+Note: this enforcement is applied by the HTTPConnection class. The
+      HTTPResponse class does not enforce this state machine, which
+      implies sophisticated clients may accelerate the request/response
+      pipeline. Caution should be taken, though: accelerating the states
+      beyond the above pattern may imply knowledge of the server's
+      connection-close behavior for certain requests. For example, it
+      is impossible to tell whether the server will close the connection
+      UNTIL the response headers have been read; this means that further
+      requests cannot be placed into the pipeline until it is known that
+      the server will NOT be closing the connection.
+
+Logical State                  __state            __response
+-------------                  -------            ----------
+Idle                           _CS_IDLE           None
+Request-started                _CS_REQ_STARTED    None
+Request-sent                   _CS_REQ_SENT       None
+Unread-response                _CS_IDLE           <response_class>
+Req-started-unread-response    _CS_REQ_STARTED    <response_class>
+Req-sent-unread-response       _CS_REQ_SENT       <response_class>
+"""
+
+from array import array
+import os
+import socket
+from sys import py3kwarning
+from urlparse import urlsplit
+import warnings
+with warnings.catch_warnings():
+    if py3kwarning:
+        warnings.filterwarnings("ignore", ".*mimetools has been removed",
+                                DeprecationWarning)
+    import mimetools
+
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from StringIO import StringIO
+
+__all__ = ["HTTP", "HTTPResponse", "HTTPConnection",
+           "HTTPException", "NotConnected", "UnknownProtocol",
+           "UnknownTransferEncoding", "UnimplementedFileMode",
+           "IncompleteRead", "InvalidURL", "ImproperConnectionState",
+           "CannotSendRequest", "CannotSendHeader", "ResponseNotReady",
+           "BadStatusLine", "error", "responses"]
+
+HTTP_PORT = 80
+HTTPS_PORT = 443
+
+_UNKNOWN = 'UNKNOWN'
+
+# connection states
+_CS_IDLE = 'Idle'
+_CS_REQ_STARTED = 'Request-started'
+_CS_REQ_SENT = 'Request-sent'
+
+# status codes
+# informational
+CONTINUE = 100
+SWITCHING_PROTOCOLS = 101
+PROCESSING = 102
+
+# successful
+OK = 200
+CREATED = 201
+ACCEPTED = 202
+NON_AUTHORITATIVE_INFORMATION = 203
+NO_CONTENT = 204
+RESET_CONTENT = 205
+PARTIAL_CONTENT = 206
+MULTI_STATUS = 207
+IM_USED = 226
+
+# redirection
+MULTIPLE_CHOICES = 300
+MOVED_PERMANENTLY = 301
+FOUND = 302
+SEE_OTHER = 303
+NOT_MODIFIED = 304
+USE_PROXY = 305
+TEMPORARY_REDIRECT = 307
+
+# client error
+BAD_REQUEST = 400
+UNAUTHORIZED = 401
+PAYMENT_REQUIRED = 402
+FORBIDDEN = 403
+NOT_FOUND = 404
+METHOD_NOT_ALLOWED = 405
+NOT_ACCEPTABLE = 406
+PROXY_AUTHENTICATION_REQUIRED = 407
+REQUEST_TIMEOUT = 408
+CONFLICT = 409
+GONE = 410
+LENGTH_REQUIRED = 411
+PRECONDITION_FAILED = 412
+REQUEST_ENTITY_TOO_LARGE = 413
+REQUEST_URI_TOO_LONG = 414
+UNSUPPORTED_MEDIA_TYPE = 415
+REQUESTED_RANGE_NOT_SATISFIABLE = 416
+EXPECTATION_FAILED = 417
+UNPROCESSABLE_ENTITY = 422
+LOCKED = 423
+FAILED_DEPENDENCY = 424
+UPGRADE_REQUIRED = 426
+
+# server error
+INTERNAL_SERVER_ERROR = 500
+NOT_IMPLEMENTED = 501
+BAD_GATEWAY = 502
+SERVICE_UNAVAILABLE = 503
+GATEWAY_TIMEOUT = 504
+HTTP_VERSION_NOT_SUPPORTED = 505
+INSUFFICIENT_STORAGE = 507
+NOT_EXTENDED = 510
+
+# Mapping status codes to official W3C names
+responses = {
+    100: 'Continue',
+    101: 'Switching Protocols',
+
+    200: 'OK',
+    201: 'Created',
+    202: 'Accepted',
+    203: 'Non-Authoritative Information',
+    204: 'No Content',
+    205: 'Reset Content',
+    206: 'Partial Content',
+
+    300: 'Multiple Choices',
+    301: 'Moved Permanently',
+    302: 'Found',
+    303: 'See Other',
+    304: 'Not Modified',
+    305: 'Use Proxy',
+    306: '(Unused)',
+    307: 'Temporary Redirect',
+
+    400: 'Bad Request',
+    401: 'Unauthorized',
+    402: 'Payment Required',
+    403: 'Forbidden',
+    404: 'Not Found',
+    405: 'Method Not Allowed',
+    406: 'Not Acceptable',
+    407: 'Proxy Authentication Required',
+    408: 'Request Timeout',
+    409: 'Conflict',
+    410: 'Gone',
+    411: 'Length Required',
+    412: 'Precondition Failed',
+    413: 'Request Entity Too Large',
+    414: 'Request-URI Too Long',
+    415: 'Unsupported Media Type',
+    416: 'Requested Range Not Satisfiable',
+    417: 'Expectation Failed',
+
+    500: 'Internal Server Error',
+    501: 'Not Implemented',
+    502: 'Bad Gateway',
+    503: 'Service Unavailable',
+    504: 'Gateway Timeout',
+    505: 'HTTP Version Not Supported',
+}
+
+# maximal amount of data to read at one time in _safe_read
+MAXAMOUNT = 1048576
+
+class HTTPMessage(mimetools.Message):
+
+    def addheader(self, key, value):
+        """Add header for field key handling repeats."""
+        prev = self.dict.get(key)
+        if prev is None:
+            self.dict[key] = value
+        else:
+            combined = ", ".join((prev, value))
+            self.dict[key] = combined
+
+    def addcontinue(self, key, more):
+        """Add more field data from a continuation line."""
+        prev = self.dict[key]
+        self.dict[key] = prev + "\n " + more
+
+    def readheaders(self):
+        """Read header lines.
+
+        Read header lines up to the entirely blank line that terminates them.
+        The (normally blank) line that ends the headers is skipped, but not
+        included in the returned list.  If a non-header line ends the headers,
+        (which is an error), an attempt is made to backspace over it; it is
+        never included in the returned list.
+
+        The variable self.status is set to the empty string if all went well,
+        otherwise it is an error message.  The variable self.headers is a
+        completely uninterpreted list of lines contained in the header (so
+        printing them will reproduce the header exactly as it appears in the
+        file).
+
+        If multiple header fields with the same name occur, they are combined
+        according to the rules in RFC 2616 sec 4.2:
+
+        Appending each subsequent field-value to the first, each separated
+        by a comma. The order in which header fields with the same field-name
+        are received is significant to the interpretation of the combined
+        field value.
+        """
+        # XXX The implementation overrides the readheaders() method of
+        # rfc822.Message.  The base class design isn't amenable to
+        # customized behavior here so the method here is a copy of the
+        # base class code with a few small changes.
+
+        self.dict = {}
+        self.unixfrom = ''
+        self.headers = hlist = []
+        self.status = ''
+        headerseen = ""
+        firstline = 1
+        startofline = unread = tell = None
+        if hasattr(self.fp, 'unread'):
+            unread = self.fp.unread
+        elif self.seekable:
+            tell = self.fp.tell
+        while True:
+            if tell:
+                try:
+                    startofline = tell()
+                except IOError:
+                    startofline = tell = None
+                    self.seekable = 0
+            line = self.fp.readline()
+            if not line:
+                self.status = 'EOF in headers'
+                break
+            # Skip unix From name time lines
+            if firstline and line.startswith('From '):
+                self.unixfrom = self.unixfrom + line
+                continue
+            firstline = 0
+            if headerseen and line[0] in ' \t':
+                # XXX Not sure if continuation lines are handled properly
+                # for http and/or for repeating headers
+                # It's a continuation line.
+                hlist.append(line)
+                self.addcontinue(headerseen, line.strip())
+                continue
+            elif self.iscomment(line):
+                # It's a comment.  Ignore it.
+                continue
+            elif self.islast(line):
+                # Note! No pushback here!  The delimiter line gets eaten.
+                break
+            headerseen = self.isheader(line)
+            if headerseen:
+                # It's a legal header line, save it.
+                hlist.append(line)
+                self.addheader(headerseen, line[len(headerseen)+1:].strip())
+                continue
+            else:
+                # It's not a header line; throw it back and stop here.
+                if not self.dict:
+                    self.status = 'No headers'
+                else:
+                    self.status = 'Non-header line where header expected'
+                # Try to undo the read.
+                if unread:
+                    unread(line)
+                elif tell:
+                    self.fp.seek(startofline)
+                else:
+                    self.status = self.status + '; bad seek'
+                break
+
+class HTTPResponse:
+
+    # strict: If true, raise BadStatusLine if the status line can't be
+    # parsed as a valid HTTP/1.0 or 1.1 status line.  By default it is
+    # false because it prevents clients from talking to HTTP/0.9
+    # servers.  Note that a response with a sufficiently corrupted
+    # status line will look like an HTTP/0.9 response.
+
+    # See RFC 2616 sec 19.6 and RFC 1945 sec 6 for details.
+
+    def __init__(self, sock, debuglevel=0, strict=0, method=None, buffering=False):
+        if buffering:
+            # The caller won't be using any sock.recv() calls, so buffering
+            # is fine and recommended for performance.
+            self.fp = sock.makefile('rb')
+        else:
+            # The buffer size is specified as zero, because the headers of
+            # the response are read with readline().  If the reads were
+            # buffered the readline() calls could consume some of the
+            # response, which make be read via a recv() on the underlying
+            # socket.
+            self.fp = sock.makefile('rb', 0)
+        self.debuglevel = debuglevel
+        self.strict = strict
+        self._method = method
+
+        self.msg = None
+
+        # from the Status-Line of the response
+        self.version = _UNKNOWN # HTTP-Version
+        self.status = _UNKNOWN  # Status-Code
+        self.reason = _UNKNOWN  # Reason-Phrase
+
+        self.chunked = _UNKNOWN         # is "chunked" being used?
+        self.chunk_left = _UNKNOWN      # bytes left to read in current chunk
+        self.length = _UNKNOWN          # number of bytes left in response
+        self.will_close = _UNKNOWN      # conn will close at end of response
+
+    def _read_status(self):
+        # Initialize with Simple-Response defaults
+        line = self.fp.readline()
+        if self.debuglevel > 0:
+            print "reply:", repr(line)
+        if not line:
+            # Presumably, the server closed the connection before
+            # sending a valid response.
+            raise BadStatusLine(line)
+        try:
+            [version, status, reason] = line.split(None, 2)
+        except ValueError:
+            try:
+                [version, status] = line.split(None, 1)
+                reason = ""
+            except ValueError:
+                # empty version will cause next test to fail and status
+                # will be treated as 0.9 response.
+                version = ""
+        if not version.startswith('HTTP/'):
+            if self.strict:
+                self.close()
+                raise BadStatusLine(line)
+            else:
+                # assume it's a Simple-Response from an 0.9 server
+                self.fp = LineAndFileWrapper(line, self.fp)
+                return "HTTP/0.9", 200, ""
+
+        # The status code is a three-digit number
+        try:
+            status = int(status)
+            if status < 100 or status > 999:
+                raise BadStatusLine(line)
+        except ValueError:
+            raise BadStatusLine(line)
+        return version, status, reason
+
+    def begin(self):
+        if self.msg is not None:
+            # we've already started reading the response
+            return
+
+        # read until we get a non-100 response
+        while True:
+            version, status, reason = self._read_status()
+            if status != CONTINUE:
+                break
+            # skip the header from the 100 response
+            while True:
+                skip = self.fp.readline().strip()
+                if not skip:
+                    break
+                if self.debuglevel > 0:
+                    print "header:", skip
+
+        self.status = status
+        self.reason = reason.strip()
+        if version == 'HTTP/1.0':
+            self.version = 10
+        elif version.startswith('HTTP/1.'):
+            self.version = 11   # use HTTP/1.1 code for HTTP/1.x where x>=1
+        elif version == 'HTTP/0.9':
+            self.version = 9
+        else:
+            raise UnknownProtocol(version)
+
+        if self.version == 9:
+            self.length = None
+            self.chunked = 0
+            self.will_close = 1
+            self.msg = HTTPMessage(StringIO())
+            return
+
+        self.msg = HTTPMessage(self.fp, 0)
+        if self.debuglevel > 0:
+            for hdr in self.msg.headers:
+                print "header:", hdr,
+
+        # don't let the msg keep an fp
+        self.msg.fp = None
+
+        # are we using the chunked-style of transfer encoding?
+        tr_enc = self.msg.getheader('transfer-encoding')
+        if tr_enc and tr_enc.lower() == "chunked":
+            self.chunked = 1
+            self.chunk_left = None
+        else:
+            self.chunked = 0
+
+        # will the connection close at the end of the response?
+        self.will_close = self._check_close()
+
+        # do we have a Content-Length?
+        # NOTE: RFC 2616, S4.4, #3 says we ignore this if tr_enc is "chunked"
+        length = self.msg.getheader('content-length')
+        if length and not self.chunked:
+            try:
+                self.length = int(length)
+            except ValueError:
+                self.length = None
+            else:
+                if self.length < 0:  # ignore nonsensical negative lengths
+                    self.length = None
+        else:
+            self.length = None
+
+        # does the body have a fixed length? (of zero)
+        if (status == NO_CONTENT or status == NOT_MODIFIED or
+            100 <= status < 200 or      # 1xx codes
+            self._method == 'HEAD'):
+            self.length = 0
+
+        # if the connection remains open, and we aren't using chunked, and
+        # a content-length was not provided, then assume that the connection
+        # WILL close.
+        if not self.will_close and \
+           not self.chunked and \
+           self.length is None:
+            self.will_close = 1
+
+    def _check_close(self):
+        conn = self.msg.getheader('connection')
+        if self.version == 11:
+            # An HTTP/1.1 proxy is assumed to stay open unless
+            # explicitly closed.
+            conn = self.msg.getheader('connection')
+            if conn and "close" in conn.lower():
+                return True
+            return False
+
+        # Some HTTP/1.0 implementations have support for persistent
+        # connections, using rules different than HTTP/1.1.
+
+        # For older HTTP, Keep-Alive indicates persistent connection.
+        if self.msg.getheader('keep-alive'):
+            return False
+
+        # At least Akamai returns a "Connection: Keep-Alive" header,
+        # which was supposed to be sent by the client.
+        if conn and "keep-alive" in conn.lower():
+            return False
+
+        # Proxy-Connection is a netscape hack.
+        pconn = self.msg.getheader('proxy-connection')
+        if pconn and "keep-alive" in pconn.lower():
+            return False
+
+        # otherwise, assume it will close
+        return True
+
+    def close(self):
+        if self.fp:
+            self.fp.close()
+            self.fp = None
+
+    def isclosed(self):
+        # NOTE: it is possible that we will not ever call self.close(). This
+        #       case occurs when will_close is TRUE, length is None, and we
+        #       read up to the last byte, but NOT past it.
+        #
+        # IMPLIES: if will_close is FALSE, then self.close() will ALWAYS be
+        #          called, meaning self.isclosed() is meaningful.
+        return self.fp is None
+
+    # XXX It would be nice to have readline and __iter__ for this, too.
+
+    def read(self, amt=None):
+        if self.fp is None:
+            return ''
+
+        if self._method == 'HEAD':
+            self.close()
+            return ''
+
+        if self.chunked:
+            return self._read_chunked(amt)
+
+        if amt is None:
+            # unbounded read
+            if self.length is None:
+                s = self.fp.read()
+            else:
+                s = self._safe_read(self.length)
+                self.length = 0
+            self.close()        # we read everything
+            return s
+
+        if self.length is not None:
+            if amt > self.length:
+                # clip the read to the "end of response"
+                amt = self.length
+
+        # we do not use _safe_read() here because this may be a .will_close
+        # connection, and the user is reading more bytes than will be provided
+        # (for example, reading in 1k chunks)
+        s = self.fp.read(amt)
+        if self.length is not None:
+            self.length -= len(s)
+            if not self.length:
+                self.close()
+        return s
+
+    def _read_chunked(self, amt):
+        assert self.chunked != _UNKNOWN
+        chunk_left = self.chunk_left
+        value = []
+        while True:
+            if chunk_left is None:
+                line = self.fp.readline()
+                i = line.find(';')
+                if i >= 0:
+                    line = line[:i] # strip chunk-extensions
+                try:
+                    chunk_left = int(line, 16)
+                except ValueError:
+                    # close the connection as protocol synchronisation is
+                    # probably lost
+                    self.close()
+                    raise IncompleteRead(''.join(value))
+                if chunk_left == 0:
+                    break
+            if amt is None:
+                value.append(self._safe_read(chunk_left))
+            elif amt < chunk_left:
+                value.append(self._safe_read(amt))
+                self.chunk_left = chunk_left - amt
+                return ''.join(value)
+            elif amt == chunk_left:
+                value.append(self._safe_read(amt))
+                self._safe_read(2)  # toss the CRLF at the end of the chunk
+                self.chunk_left = None
+                return ''.join(value)
+            else:
+                value.append(self._safe_read(chunk_left))
+                amt -= chunk_left
+
+            # we read the whole chunk, get another
+            self._safe_read(2)      # toss the CRLF at the end of the chunk
+            chunk_left = None
+
+        # read and discard trailer up to the CRLF terminator
+        ### note: we shouldn't have any trailers!
+        while True:
+            line = self.fp.readline()
+            if not line:
+                # a vanishingly small number of sites EOF without
+                # sending the trailer
+                break
+            if line == '\r\n':
+                break
+
+        # we read everything; close the "file"
+        self.close()
+
+        return ''.join(value)
+
+    def _safe_read(self, amt):
+        """Read the number of bytes requested, compensating for partial reads.
+
+        Normally, we have a blocking socket, but a read() can be interrupted
+        by a signal (resulting in a partial read).
+
+        Note that we cannot distinguish between EOF and an interrupt when zero
+        bytes have been read. IncompleteRead() will be raised in this
+        situation.
+
+        This function should be used when <amt> bytes "should" be present for
+        reading. If the bytes are truly not available (due to EOF), then the
+        IncompleteRead exception can be used to detect the problem.
+        """
+        # NOTE(gps): As of svn r74426 socket._fileobject.read(x) will never
+        # return less than x bytes unless EOF is encountered.  It now handles
+        # signal interruptions (socket.error EINTR) internally.  This code
+        # never caught that exception anyways.  It seems largely pointless.
+        # self.fp.read(amt) will work fine.
+        s = []
+        while amt > 0:
+            chunk = self.fp.read(min(amt, MAXAMOUNT))
+            if not chunk:
+                raise IncompleteRead(''.join(s), amt)
+            s.append(chunk)
+            amt -= len(chunk)
+        return ''.join(s)
+
+    def fileno(self):
+        return self.fp.fileno()
+
+    def getheader(self, name, default=None):
+        if self.msg is None:
+            raise ResponseNotReady()
+        return self.msg.getheader(name, default)
+
+    def getheaders(self):
+        """Return list of (header, value) tuples."""
+        if self.msg is None:
+            raise ResponseNotReady()
+        return self.msg.items()
+
+
+class HTTPConnection:
+
+    _http_vsn = 11
+    _http_vsn_str = 'HTTP/1.1'
+
+    response_class = HTTPResponse
+    default_port = HTTP_PORT
+    auto_open = 1
+    debuglevel = 0
+    strict = 0
+
+    def __init__(self, host, port=None, strict=None,
+                 timeout=socket._GLOBAL_DEFAULT_TIMEOUT, source_address=None):
+        self.timeout = timeout
+        self.source_address = source_address
+        self.sock = None
+        self._buffer = []
+        self.__response = None
+        self.__state = _CS_IDLE
+        self._method = None
+        self._tunnel_host = None
+        self._tunnel_port = None
+        self._tunnel_headers = {}
+
+        self._set_hostport(host, port)
+        if strict is not None:
+            self.strict = strict
+
+    def set_tunnel(self, host, port=None, headers=None):
+        """ Sets up the host and the port for the HTTP CONNECT Tunnelling.
+
+        The headers argument should be a mapping of extra HTTP headers
+        to send with the CONNECT request.
+        """
+        self._tunnel_host = host
+        self._tunnel_port = port
+        if headers:
+            self._tunnel_headers = headers
+        else:
+            self._tunnel_headers.clear()
+
+    def _set_hostport(self, host, port):
+        if port is None:
+            i = host.rfind(':')
+            j = host.rfind(']')         # ipv6 addresses have [...]
+            if i > j:
+                try:
+                    port = int(host[i+1:])
+                except ValueError:
+                    raise InvalidURL("nonnumeric port: '%s'" % host[i+1:])
+                host = host[:i]
+            else:
+                port = self.default_port
+            if host and host[0] == '[' and host[-1] == ']':
+                host = host[1:-1]
+        self.host = host
+        self.port = port
+
+    def set_debuglevel(self, level):
+        self.debuglevel = level
+
+    def _tunnel(self):
+        self._set_hostport(self._tunnel_host, self._tunnel_port)
+        self.send("CONNECT %s:%d HTTP/1.0\r\n" % (self.host, self.port))
+        for header, value in self._tunnel_headers.iteritems():
+            self.send("%s: %s\r\n" % (header, value))
+        self.send("\r\n")
+        response = self.response_class(self.sock, strict = self.strict,
+                                       method = self._method)
+        (version, code, message) = response._read_status()
+
+        if code != 200:
+            self.close()
+            raise socket.error("Tunnel connection failed: %d %s" % (code,
+                                                                    message.strip()))
+        while True:
+            line = response.fp.readline()
+            if line == '\r\n': break
+
+
+    def connect(self):
+        """Connect to the host and port specified in __init__."""
+        self.sock = socket.create_connection((self.host,self.port),
+                                             self.timeout, self.source_address)
+
+        if self._tunnel_host:
+            self._tunnel()
+
+    def close(self):
+        """Close the connection to the HTTP server."""
+        if self.sock:
+            self.sock.close()   # close it manually... there may be other refs
+            self.sock = None
+        if self.__response:
+            self.__response.close()
+            self.__response = None
+        self.__state = _CS_IDLE
+
+    def send(self, data):
+        """Send `data' to the server."""
+        if self.sock is None:
+            if self.auto_open:
+                self.connect()
+            else:
+                raise NotConnected()
+
+        if self.debuglevel > 0:
+            print "send:", repr(data)
+        blocksize = 8192
+        if hasattr(data,'read') and not isinstance(data, array):
+            if self.debuglevel > 0: print "sendIng a read()able"
+            datablock = data.read(blocksize)
+            while datablock:
+                self.sock.sendall(datablock)
+                datablock = data.read(blocksize)
+        else:
+            self.sock.sendall(data)
+
+    def _output(self, s):
+        """Add a line of output to the current request buffer.
+
+        Assumes that the line does *not* end with \\r\\n.
+        """
+        self._buffer.append(s)
+
+    def _send_output(self, message_body=None):
+        """Send the currently buffered request and clear the buffer.
+
+        Appends an extra \\r\\n to the buffer.
+        A message_body may be specified, to be appended to the request.
+        """
+        self._buffer.extend(("", ""))
+        msg = "\r\n".join(self._buffer)
+        del self._buffer[:]
+        # If msg and message_body are sent in a single send() call,
+        # it will avoid performance problems caused by the interaction
+        # between delayed ack and the Nagle algorithim.
+        if isinstance(message_body, str):
+            msg += message_body
+            message_body = None
+        self.send(msg)
+        if message_body is not None:
+            #message_body was not a string (i.e. it is a file) and
+            #we must run the risk of Nagle
+            self.send(message_body)
+
+    def putrequest(self, method, url, skip_host=0, skip_accept_encoding=0):
+        """Send a request to the server.
+
+        `method' specifies an HTTP request method, e.g. 'GET'.
+        `url' specifies the object being requested, e.g. '/index.html'.
+        `skip_host' if True does not add automatically a 'Host:' header
+        `skip_accept_encoding' if True does not add automatically an
+           'Accept-Encoding:' header
+        """
+
+        # if a prior response has been completed, then forget about it.
+        if self.__response and self.__response.isclosed():
+            self.__response = None
+
+
+        # in certain cases, we cannot issue another request on this connection.
+        # this occurs when:
+        #   1) we are in the process of sending a request.   (_CS_REQ_STARTED)
+        #   2) a response to a previous request has signalled that it is going
+        #      to close the connection upon completion.
+        #   3) the headers for the previous response have not been read, thus
+        #      we cannot determine whether point (2) is true.   (_CS_REQ_SENT)
+        #
+        # if there is no prior response, then we can request at will.
+        #
+        # if point (2) is true, then we will have passed the socket to the
+        # response (effectively meaning, "there is no prior response"), and
+        # will open a new one when a new request is made.
+        #
+        # Note: if a prior response exists, then we *can* start a new request.
+        #       We are not allowed to begin fetching the response to this new
+        #       request, however, until that prior response is complete.
+        #
+        if self.__state == _CS_IDLE:
+            self.__state = _CS_REQ_STARTED
+        else:
+            raise CannotSendRequest()
+
+        # Save the method we use, we need it later in the response phase
+        self._method = method
+        if not url:
+            url = '/'
+        hdr = '%s %s %s' % (method, url, self._http_vsn_str)
+
+        self._output(hdr)
+
+        if self._http_vsn == 11:
+            # Issue some standard headers for better HTTP/1.1 compliance
+
+            if not skip_host:
+                # this header is issued *only* for HTTP/1.1
+                # connections. more specifically, this means it is
+                # only issued when the client uses the new
+                # HTTPConnection() class. backwards-compat clients
+                # will be using HTTP/1.0 and those clients may be
+                # issuing this header themselves. we should NOT issue
+                # it twice; some web servers (such as Apache) barf
+                # when they see two Host: headers
+
+                # If we need a non-standard port,include it in the
+                # header.  If the request is going through a proxy,
+                # but the host of the actual URL, not the host of the
+                # proxy.
+
+                netloc = ''
+                if url.startswith('http'):
+                    nil, netloc, nil, nil, nil = urlsplit(url)
+
+                if netloc:
+                    try:
+                        netloc_enc = netloc.encode("ascii")
+                    except UnicodeEncodeError:
+                        netloc_enc = netloc.encode("idna")
+                    self.putheader('Host', netloc_enc)
+                else:
+                    try:
+                        host_enc = self.host.encode("ascii")
+                    except UnicodeEncodeError:
+                        host_enc = self.host.encode("idna")
+                    # Wrap the IPv6 Host Header with [] (RFC 2732)
+                    if host_enc.find(':') >= 0:
+                        host_enc = "[" + host_enc + "]"
+                    if self.port == self.default_port:
+                        self.putheader('Host', host_enc)
+                    else:
+                        self.putheader('Host', "%s:%s" % (host_enc, self.port))
+
+            # note: we are assuming that clients will not attempt to set these
+            #       headers since *this* library must deal with the
+            #       consequences. this also means that when the supporting
+            #       libraries are updated to recognize other forms, then this
+            #       code should be changed (removed or updated).
+
+            # we only want a Content-Encoding of "identity" since we don't
+            # support encodings such as x-gzip or x-deflate.
+            if not skip_accept_encoding:
+                self.putheader('Accept-Encoding', 'identity')
+
+            # we can accept "chunked" Transfer-Encodings, but no others
+            # NOTE: no TE header implies *only* "chunked"
+            #self.putheader('TE', 'chunked')
+
+            # if TE is supplied in the header, then it must appear in a
+            # Connection header.
+            #self.putheader('Connection', 'TE')
+
+        else:
+            # For HTTP/1.0, the server will assume "not chunked"
+            pass
+
+    def putheader(self, header, *values):
+        """Send a request header line to the server.
+
+        For example: h.putheader('Accept', 'text/html')
+        """
+        if self.__state != _CS_REQ_STARTED:
+            raise CannotSendHeader()
+
+        hdr = '%s: %s' % (header, '\r\n\t'.join([str(v) for v in values]))
+        self._output(hdr)
+
+    def endheaders(self, message_body=None):
+        """Indicate that the last header line has been sent to the server.
+
+        This method sends the request to the server.  The optional
+        message_body argument can be used to pass message body
+        associated with the request.  The message body will be sent in
+        the same packet as the message headers if possible.  The
+        message_body should be a string.
+        """
+        if self.__state == _CS_REQ_STARTED:
+            self.__state = _CS_REQ_SENT
+        else:
+            raise CannotSendHeader()
+        self._send_output(message_body)
+
+    def request(self, method, url, body=None, headers={}):
+        """Send a complete request to the server."""
+        self._send_request(method, url, body, headers)
+
+    def _set_content_length(self, body):
+        # Set the content-length based on the body.
+        thelen = None
+        try:
+            thelen = str(len(body))
+        except TypeError, te:
+            # If this is a file-like object, try to
+            # fstat its file descriptor
+            try:
+                thelen = str(os.fstat(body.fileno()).st_size)
+            except (AttributeError, OSError):
+                # Don't send a length if this failed
+                if self.debuglevel > 0: print "Cannot stat!!"
+
+        if thelen is not None:
+            self.putheader('Content-Length', thelen)
+
+    def _send_request(self, method, url, body, headers):
+        # Honor explicitly requested Host: and Accept-Encoding: headers.
+        header_names = dict.fromkeys([k.lower() for k in headers])
+        skips = {}
+        if 'host' in header_names:
+            skips['skip_host'] = 1
+        if 'accept-encoding' in header_names:
+            skips['skip_accept_encoding'] = 1
+
+        self.putrequest(method, url, **skips)
+
+        if body and ('content-length' not in header_names):
+            self._set_content_length(body)
+        for hdr, value in headers.iteritems():
+            self.putheader(hdr, value)
+        self.endheaders(body)
+
+    def getresponse(self, buffering=False):
+        "Get the response from the server."
+
+        # if a prior response has been completed, then forget about it.
+        if self.__response and self.__response.isclosed():
+            self.__response = None
+
+        #
+        # if a prior response exists, then it must be completed (otherwise, we
+        # cannot read this response's header to determine the connection-close
+        # behavior)
+        #
+        # note: if a prior response existed, but was connection-close, then the
+        # socket and response were made independent of this HTTPConnection
+        # object since a new request requires that we open a whole new
+        # connection
+        #
+        # this means the prior response had one of two states:
+        #   1) will_close: this connection was reset and the prior socket and
+        #                  response operate independently
+        #   2) persistent: the response was retained and we await its
+        #                  isclosed() status to become true.
+        #
+        if self.__state != _CS_REQ_SENT or self.__response:
+            raise ResponseNotReady()
+
+        args = (self.sock,)
+        kwds = {"strict":self.strict, "method":self._method}
+        if self.debuglevel > 0:
+            args += (self.debuglevel,)
+        if buffering:
+            #only add this keyword if non-default, for compatibility with
+            #other response_classes.
+            kwds["buffering"] = True;
+        response = self.response_class(*args, **kwds)
+
+        try:
+            response.begin()
+        except:
+            response.close()
+            raise
+        assert response.will_close != _UNKNOWN
+        self.__state = _CS_IDLE
+
+        if response.will_close:
+            # this effectively passes the connection to the response
+            self.close()
+        else:
+            # remember this, so we can tell when it is complete
+            self.__response = response
+
+        return response
+
+
+class HTTP:
+    "Compatibility class with httplib.py from 1.5."
+
+    _http_vsn = 10
+    _http_vsn_str = 'HTTP/1.0'
+
+    debuglevel = 0
+
+    _connection_class = HTTPConnection
+
+    def __init__(self, host='', port=None, strict=None):
+        "Provide a default host, since the superclass requires one."
+
+        # some joker passed 0 explicitly, meaning default port
+        if port == 0:
+            port = None
+
+        # Note that we may pass an empty string as the host; this will throw
+        # an error when we attempt to connect. Presumably, the client code
+        # will call connect before then, with a proper host.
+        self._setup(self._connection_class(host, port, strict))
+
+    def _setup(self, conn):
+        self._conn = conn
+
+        # set up delegation to flesh out interface
+        self.send = conn.send
+        self.putrequest = conn.putrequest
+        self.putheader = conn.putheader
+        self.endheaders = conn.endheaders
+        self.set_debuglevel = conn.set_debuglevel
+
+        conn._http_vsn = self._http_vsn
+        conn._http_vsn_str = self._http_vsn_str
+
+        self.file = None
+
+    def connect(self, host=None, port=None):
+        "Accept arguments to set the host/port, since the superclass doesn't."
+
+        if host is not None:
+            self._conn._set_hostport(host, port)
+        self._conn.connect()
+
+    def getfile(self):
+        "Provide a getfile, since the superclass' does not use this concept."
+        return self.file
+
+    def getreply(self, buffering=False):
+        """Compat definition since superclass does not define it.
+
+        Returns a tuple consisting of:
+        - server status code (e.g. '200' if all goes well)
+        - server "reason" corresponding to status code
+        - any RFC822 headers in the response from the server
+        """
+        try:
+            if not buffering:
+                response = self._conn.getresponse()
+            else:
+                #only add this keyword if non-default for compatibility
+                #with other connection classes
+                response = self._conn.getresponse(buffering)
+        except BadStatusLine, e:
+            ### hmm. if getresponse() ever closes the socket on a bad request,
+            ### then we are going to have problems with self.sock
+
+            ### should we keep this behavior? do people use it?
+            # keep the socket open (as a file), and return it
+            self.file = self._conn.sock.makefile('rb', 0)
+
+            # close our socket -- we want to restart after any protocol error
+            self.close()
+
+            self.headers = None
+            return -1, e.line, None
+
+        self.headers = response.msg
+        self.file = response.fp
+        return response.status, response.reason, response.msg
+
+    def close(self):
+        self._conn.close()
+
+        # note that self.file == response.fp, which gets closed by the
+        # superclass. just clear the object ref here.
+        ### hmm. messy. if status==-1, then self.file is owned by us.
+        ### well... we aren't explicitly closing, but losing this ref will
+        ### do it
+        self.file = None
+
+try:
+    import ssl
+except ImportError:
+    pass
+else:
+    class HTTPSConnection(HTTPConnection):
+        "This class allows communication via SSL."
+
+        default_port = HTTPS_PORT
+
+        def __init__(self, host, port=None, key_file=None, cert_file=None,
+                     strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
+                     source_address=None):
+            HTTPConnection.__init__(self, host, port, strict, timeout,
+                                    source_address)
+            self.key_file = key_file
+            self.cert_file = cert_file
+
+        def connect(self):
+            "Connect to a host on a given (SSL) port."
+
+            sock = socket.create_connection((self.host, self.port),
+                                            self.timeout, self.source_address)
+            if self._tunnel_host:
+                self.sock = sock
+                self._tunnel()
+            self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file)
+
+    __all__.append("HTTPSConnection")
+
+    class HTTPS(HTTP):
+        """Compatibility with 1.5 httplib interface
+
+        Python 1.5.2 did not have an HTTPS class, but it defined an
+        interface for sending http requests that is also useful for
+        https.
+        """
+
+        _connection_class = HTTPSConnection
+
+        def __init__(self, host='', port=None, key_file=None, cert_file=None,
+                     strict=None):
+            # provide a default host, pass the X509 cert info
+
+            # urf. compensate for bad input.
+            if port == 0:
+                port = None
+            self._setup(self._connection_class(host, port, key_file,
+                                               cert_file, strict))
+
+            # we never actually use these for anything, but we keep them
+            # here for compatibility with post-1.5.2 CVS.
+            self.key_file = key_file
+            self.cert_file = cert_file
+
+
+    def FakeSocket (sock, sslobj):
+        warnings.warn("FakeSocket is deprecated, and won't be in 3.x.  " +
+                      "Use the result of ssl.wrap_socket() directly instead.",
+                      DeprecationWarning, stacklevel=2)
+        return sslobj
+
+
+class HTTPException(Exception):
+    # Subclasses that define an __init__ must call Exception.__init__
+    # or define self.args.  Otherwise, str() will fail.
+    pass
+
+class NotConnected(HTTPException):
+    pass
+
+class InvalidURL(HTTPException):
+    pass
+
+class UnknownProtocol(HTTPException):
+    def __init__(self, version):
+        self.args = version,
+        self.version = version
+
+class UnknownTransferEncoding(HTTPException):
+    pass
+
+class UnimplementedFileMode(HTTPException):
+    pass
+
+class IncompleteRead(HTTPException):
+    def __init__(self, partial, expected=None):
+        self.args = partial,
+        self.partial = partial
+        self.expected = expected
+    def __repr__(self):
+        if self.expected is not None:
+            e = ', %i more expected' % self.expected
+        else:
+            e = ''
+        return 'IncompleteRead(%i bytes read%s)' % (len(self.partial), e)
+    def __str__(self):
+        return repr(self)
+
+class ImproperConnectionState(HTTPException):
+    pass
+
+class CannotSendRequest(ImproperConnectionState):
+    pass
+
+class CannotSendHeader(ImproperConnectionState):
+    pass
+
+class ResponseNotReady(ImproperConnectionState):
+    pass
+
+class BadStatusLine(HTTPException):
+    def __init__(self, line):
+        if not line:
+            line = repr(line)
+        self.args = line,
+        self.line = line
+
+# for backwards compatibility
+error = HTTPException
+
+class LineAndFileWrapper:
+    """A limited file-like object for HTTP/0.9 responses."""
+
+    # The status-line parsing code calls readline(), which normally
+    # get the HTTP status line.  For a 0.9 response, however, this is
+    # actually the first line of the body!  Clients need to get a
+    # readable file object that contains that line.
+
+    def __init__(self, line, file):
+        self._line = line
+        self._file = file
+        self._line_consumed = 0
+        self._line_offset = 0
+        self._line_left = len(line)
+
+    def __getattr__(self, attr):
+        return getattr(self._file, attr)
+
+    def _done(self):
+        # called when the last byte is read from the line.  After the
+        # call, all read methods are delegated to the underlying file
+        # object.
+        self._line_consumed = 1
+        self.read = self._file.read
+        self.readline = self._file.readline
+        self.readlines = self._file.readlines
+
+    def read(self, amt=None):
+        if self._line_consumed:
+            return self._file.read(amt)
+        assert self._line_left
+        if amt is None or amt > self._line_left:
+            s = self._line[self._line_offset:]
+            self._done()
+            if amt is None:
+                return s + self._file.read()
+            else:
+                return s + self._file.read(amt - len(s))
+        else:
+            assert amt <= self._line_left
+            i = self._line_offset
+            j = i + amt
+            s = self._line[i:j]
+            self._line_offset = j
+            self._line_left -= amt
+            if self._line_left == 0:
+                self._done()
+            return s
+
+    def readline(self):
+        if self._line_consumed:
+            return self._file.readline()
+        assert self._line_left
+        s = self._line[self._line_offset:]
+        self._done()
+        return s
+
+    def readlines(self, size=None):
+        if self._line_consumed:
+            return self._file.readlines(size)
+        assert self._line_left
+        L = [self._line[self._line_offset:]]
+        self._done()
+        if size is None:
+            return L + self._file.readlines()
+        else:
+            return L + self._file.readlines(size)
+
+def test():
+    """Test this module.
+
+    A hodge podge of tests collected here, because they have too many
+    external dependencies for the regular test suite.
+    """
+
+    import sys
+    import getopt
+    opts, args = getopt.getopt(sys.argv[1:], 'd')
+    dl = 0
+    for o, a in opts:
+        if o == '-d': dl = dl + 1
+    host = 'www.python.org'
+    selector = '/'
+    if args[0:]: host = args[0]
+    if args[1:]: selector = args[1]
+    h = HTTP()
+    h.set_debuglevel(dl)
+    h.connect(host)
+    h.putrequest('GET', selector)
+    h.endheaders()
+    status, reason, headers = h.getreply()
+    print 'status =', status
+    print 'reason =', reason
+    print "read", len(h.getfile().read())
+    print
+    if headers:
+        for header in headers.headers: print header.strip()
+    print
+
+    # minimal test that code to extract host from url works
+    class HTTP11(HTTP):
+        _http_vsn = 11
+        _http_vsn_str = 'HTTP/1.1'
+
+    h = HTTP11('www.python.org')
+    h.putrequest('GET', 'http://www.python.org/~jeremy/')
+    h.endheaders()
+    h.getreply()
+    h.close()
+
+    try:
+        import ssl
+    except ImportError:
+        pass
+    else:
+
+        for host, selector in (('sourceforge.net', '/projects/python'),
+                               ):
+            print "https://%s%s" % (host, selector)
+            hs = HTTPS()
+            hs.set_debuglevel(dl)
+            hs.connect(host)
+            hs.putrequest('GET', selector)
+            hs.endheaders()
+            status, reason, headers = hs.getreply()
+            print 'status =', status
+            print 'reason =', reason
+            print "read", len(hs.getfile().read())
+            print
+            if headers:
+                for header in headers.headers: print header.strip()
+            print
+
+if __name__ == '__main__':
+    test()
diff --git a/lib-python/modified-2.7/test/test_array.py b/lib-python/modified-2.7/test/test_array.py
--- a/lib-python/modified-2.7/test/test_array.py
+++ b/lib-python/modified-2.7/test/test_array.py
@@ -295,9 +295,10 @@
         )
 
         b = array.array(self.badtypecode())
-        self.assertRaises(TypeError, "a + b")
-
-        self.assertRaises(TypeError, "a + 'bad'")
+        with self.assertRaises(TypeError):
+            a + b
+        with self.assertRaises(TypeError):
+            a + 'bad'
 
     def test_iadd(self):
         a = array.array(self.typecode, self.example[::-1])
@@ -316,9 +317,10 @@
         )
 
         b = array.array(self.badtypecode())
-        self.assertRaises(TypeError, "a += b")
-
-        self.assertRaises(TypeError, "a += 'bad'")
+        with self.assertRaises(TypeError):
+            a += b
+        with self.assertRaises(TypeError):
+            a += 'bad'
 
     def test_mul(self):
         a = 5*array.array(self.typecode, self.example)
@@ -345,7 +347,8 @@
             array.array(self.typecode)
         )
 
-        self.assertRaises(TypeError, "a * 'bad'")
+        with self.assertRaises(TypeError):
+            a * 'bad'
 
     def test_imul(self):
         a = array.array(self.typecode, self.example)
@@ -374,7 +377,8 @@
         a *= -1
         self.assertEqual(a, array.array(self.typecode))
 
-        self.assertRaises(TypeError, "a *= 'bad'")
+        with self.assertRaises(TypeError):
+            a *= 'bad'
 
     def test_getitem(self):
         a = array.array(self.typecode, self.example)
diff --git a/lib-python/modified-2.7/test/test_sys_settrace.py b/lib-python/modified-2.7/test/test_sys_settrace.py
--- a/lib-python/modified-2.7/test/test_sys_settrace.py
+++ b/lib-python/modified-2.7/test/test_sys_settrace.py
@@ -286,11 +286,11 @@
         self.compare_events(func.func_code.co_firstlineno,
                             tracer.events, func.events)
 
-    def set_and_retrieve_none(self):
+    def test_set_and_retrieve_none(self):
         sys.settrace(None)
         assert sys.gettrace() is None
 
-    def set_and_retrieve_func(self):
+    def test_set_and_retrieve_func(self):
         def fn(*args):
             pass
 
diff --git a/lib-python/modified-2.7/test/test_urllib2.py b/lib-python/modified-2.7/test/test_urllib2.py
--- a/lib-python/modified-2.7/test/test_urllib2.py
+++ b/lib-python/modified-2.7/test/test_urllib2.py
@@ -307,6 +307,9 @@
     def getresponse(self):
         return MockHTTPResponse(MockFile(), {}, 200, "OK")
 
+    def close(self):
+        pass
+
 class MockHandler:
     # useful for testing handler machinery
     # see add_ordered_mock_handlers() docstring
diff --git a/lib-python/modified-2.7/urllib2.py b/lib-python/modified-2.7/urllib2.py
new file mode 100644
--- /dev/null
+++ b/lib-python/modified-2.7/urllib2.py
@@ -0,0 +1,1440 @@
+"""An extensible library for opening URLs using a variety of protocols
+
+The simplest way to use this module is to call the urlopen function,
+which accepts a string containing a URL or a Request object (described
+below).  It opens the URL and returns the results as file-like
+object; the returned object has some extra methods described below.
+
+The OpenerDirector manages a collection of Handler objects that do
+all the actual work.  Each Handler implements a particular protocol or
+option.  The OpenerDirector is a composite object that invokes the
+Handlers needed to open the requested URL.  For example, the
+HTTPHandler performs HTTP GET and POST requests and deals with
+non-error returns.  The HTTPRedirectHandler automatically deals with
+HTTP 301, 302, 303 and 307 redirect errors, and the HTTPDigestAuthHandler
+deals with digest authentication.
+
+urlopen(url, data=None) -- Basic usage is the same as original
+urllib.  pass the url and optionally data to post to an HTTP URL, and
+get a file-like object back.  One difference is that you can also pass
+a Request instance instead of URL.  Raises a URLError (subclass of
+IOError); for HTTP errors, raises an HTTPError, which can also be
+treated as a valid response.
+
+build_opener -- Function that creates a new OpenerDirector instance.
+Will install the default handlers.  Accepts one or more Handlers as
+arguments, either instances or Handler classes that it will
+instantiate.  If one of the argument is a subclass of the default
+handler, the argument will be installed instead of the default.
+
+install_opener -- Installs a new opener as the default opener.
+
+objects of interest:
+
+OpenerDirector -- Sets up the User Agent as the Python-urllib client and manages
+the Handler classes, while dealing with requests and responses.
+
+Request -- An object that encapsulates the state of a request.  The
+state can be as simple as the URL.  It can also include extra HTTP
+headers, e.g. a User-Agent.
+
+BaseHandler --
+
+exceptions:
+URLError -- A subclass of IOError, individual protocols have their own
+specific subclass.
+
+HTTPError -- Also a valid HTTP response, so you can treat an HTTP error
+as an exceptional event or valid response.
+
+internals:
+BaseHandler and parent
+_call_chain conventions
+
+Example usage:
+
+import urllib2
+
+# set up authentication info
+authinfo = urllib2.HTTPBasicAuthHandler()
+authinfo.add_password(realm='PDQ Application',
+                      uri='https://mahler:8092/site-updates.py',
+                      user='klem',
+                      passwd='geheim$parole')
+
+proxy_support = urllib2.ProxyHandler({"http" : "http://ahad-haam:3128"})
+
+# build a new opener that adds authentication and caching FTP handlers
+opener = urllib2.build_opener(proxy_support, authinfo, urllib2.CacheFTPHandler)
+
+# install it
+urllib2.install_opener(opener)
+
+f = urllib2.urlopen('http://www.python.org/')
+
+
+"""
+
+# XXX issues:
+# If an authentication error handler that tries to perform
+# authentication for some reason but fails, how should the error be
+# signalled?  The client needs to know the HTTP error code.  But if
+# the handler knows that the problem was, e.g., that it didn't know
+# that hash algo that requested in the challenge, it would be good to
+# pass that information along to the client, too.
+# ftp errors aren't handled cleanly
+# check digest against correct (i.e. non-apache) implementation
+
+# Possible extensions:
+# complex proxies  XXX not sure what exactly was meant by this
+# abstract factory for opener
+
+import base64
+import hashlib
+import httplib
+import mimetools
+import os
+import posixpath
+import random
+import re
+import socket
+import sys
+import time
+import urlparse
+import bisect
+
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from StringIO import StringIO
+
+from urllib import (unwrap, unquote, splittype, splithost, quote,
+     addinfourl, splitport, splittag,
+     splitattr, ftpwrapper, splituser, splitpasswd, splitvalue)
+
+# support for FileHandler, proxies via environment variables
+from urllib import localhost, url2pathname, getproxies, proxy_bypass
+
+# used in User-Agent header sent
+__version__ = sys.version[:3]
+
+_opener = None
+def urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
+    global _opener
+    if _opener is None:
+        _opener = build_opener()
+    return _opener.open(url, data, timeout)
+
+def install_opener(opener):
+    global _opener
+    _opener = opener
+
+# do these error classes make sense?
+# make sure all of the IOError stuff is overridden.  we just want to be
+# subtypes.
+
+class URLError(IOError):
+    # URLError is a sub-type of IOError, but it doesn't share any of
+    # the implementation.  need to override __init__ and __str__.
+    # It sets self.args for compatibility with other EnvironmentError
+    # subclasses, but args doesn't have the typical format with errno in
+    # slot 0 and strerror in slot 1.  This may be better than nothing.
+    def __init__(self, reason):
+        self.args = reason,
+        self.reason = reason
+
+    def __str__(self):
+        return '<urlopen error %s>' % self.reason
+
+class HTTPError(URLError, addinfourl):
+    """Raised when HTTP error occurs, but also acts like non-error return"""
+    __super_init = addinfourl.__init__
+
+    def __init__(self, url, code, msg, hdrs, fp):
+        self.code = code
+        self.msg = msg
+        self.hdrs = hdrs
+        self.fp = fp
+        self.filename = url
+        # The addinfourl classes depend on fp being a valid file
+        # object.  In some cases, the HTTPError may not have a valid
+        # file object.  If this happens, the simplest workaround is to
+        # not initialize the base classes.
+        if fp is not None:
+            self.__super_init(fp, hdrs, url, code)
+
+    def __str__(self):
+        return 'HTTP Error %s: %s' % (self.code, self.msg)
+
+# copied from cookielib.py
+_cut_port_re = re.compile(r":\d+$")
+def request_host(request):
+    """Return request-host, as defined by RFC 2965.
+
+    Variation from RFC: returned value is lowercased, for convenient
+    comparison.
+
+    """
+    url = request.get_full_url()
+    host = urlparse.urlparse(url)[1]
+    if host == "":
+        host = request.get_header("Host", "")
+
+    # remove port, if present
+    host = _cut_port_re.sub("", host, 1)
+    return host.lower()
+
+class Request:
+
+    def __init__(self, url, data=None, headers={},
+                 origin_req_host=None, unverifiable=False):
+        # unwrap('<URL:type://host/path>') --> 'type://host/path'
+        self.__original = unwrap(url)
+        self.__original, fragment = splittag(self.__original)
+        self.type = None
+        # self.__r_type is what's left after doing the splittype
+        self.host = None
+        self.port = None
+        self._tunnel_host = None
+        self.data = data
+        self.headers = {}
+        for key, value in headers.items():
+            self.add_header(key, value)
+        self.unredirected_hdrs = {}
+        if origin_req_host is None:
+            origin_req_host = request_host(self)
+        self.origin_req_host = origin_req_host
+        self.unverifiable = unverifiable
+
+    def __getattr__(self, attr):
+        # XXX this is a fallback mechanism to guard against these
+        # methods getting called in a non-standard order.  this may be
+        # too complicated and/or unnecessary.
+        # XXX should the __r_XXX attributes be public?
+        if attr[:12] == '_Request__r_':
+            name = attr[12:]
+            if hasattr(Request, 'get_' + name):
+                getattr(self, 'get_' + name)()
+                return getattr(self, attr)
+        raise AttributeError, attr
+
+    def get_method(self):
+        if self.has_data():
+            return "POST"
+        else:
+            return "GET"
+
+    # XXX these helper methods are lame
+
+    def add_data(self, data):
+        self.data = data
+
+    def has_data(self):
+        return self.data is not None
+
+    def get_data(self):
+        return self.data
+
+    def get_full_url(self):
+        return self.__original
+
+    def get_type(self):
+        if self.type is None:
+            self.type, self.__r_type = splittype(self.__original)
+            if self.type is None:
+                raise ValueError, "unknown url type: %s" % self.__original
+        return self.type
+
+    def get_host(self):
+        if self.host is None:
+            self.host, self.__r_host = splithost(self.__r_type)
+            if self.host:
+                self.host = unquote(self.host)
+        return self.host
+
+    def get_selector(self):
+        return self.__r_host
+
+    def set_proxy(self, host, type):
+        if self.type == 'https' and not self._tunnel_host:
+            self._tunnel_host = self.host
+        else:
+            self.type = type
+            self.__r_host = self.__original
+
+        self.host = host
+
+    def has_proxy(self):
+        return self.__r_host == self.__original
+
+    def get_origin_req_host(self):
+        return self.origin_req_host
+
+    def is_unverifiable(self):
+        return self.unverifiable
+
+    def add_header(self, key, val):
+        # useful for something like authentication
+        self.headers[key.capitalize()] = val
+
+    def add_unredirected_header(self, key, val):
+        # will not be added to a redirected request
+        self.unredirected_hdrs[key.capitalize()] = val
+
+    def has_header(self, header_name):
+        return (header_name in self.headers or
+                header_name in self.unredirected_hdrs)
+
+    def get_header(self, header_name, default=None):
+        return self.headers.get(
+            header_name,
+            self.unredirected_hdrs.get(header_name, default))
+
+    def header_items(self):
+        hdrs = self.unredirected_hdrs.copy()
+        hdrs.update(self.headers)
+        return hdrs.items()
+
+class OpenerDirector:
+    def __init__(self):
+        client_version = "Python-urllib/%s" % __version__
+        self.addheaders = [('User-agent', client_version)]
+        # manage the individual handlers
+        self.handlers = []
+        self.handle_open = {}
+        self.handle_error = {}
+        self.process_response = {}
+        self.process_request = {}
+
+    def add_handler(self, handler):
+        if not hasattr(handler, "add_parent"):
+            raise TypeError("expected BaseHandler instance, got %r" %
+                            type(handler))
+
+        added = False
+        for meth in dir(handler):
+            if meth in ["redirect_request", "do_open", "proxy_open"]:
+                # oops, coincidental match
+                continue
+
+            i = meth.find("_")
+            protocol = meth[:i]
+            condition = meth[i+1:]
+
+            if condition.startswith("error"):
+                j = condition.find("_") + i + 1
+                kind = meth[j+1:]
+                try:
+                    kind = int(kind)
+                except ValueError:
+                    pass
+                lookup = self.handle_error.get(protocol, {})
+                self.handle_error[protocol] = lookup
+            elif condition == "open":
+                kind = protocol
+                lookup = self.handle_open
+            elif condition == "response":
+                kind = protocol
+                lookup = self.process_response
+            elif condition == "request":
+                kind = protocol
+                lookup = self.process_request
+            else:
+                continue
+
+            handlers = lookup.setdefault(kind, [])
+            if handlers:
+                bisect.insort(handlers, handler)
+            else:
+                handlers.append(handler)
+            added = True
+
+        if added:
+            # the handlers must work in an specific order, the order
+            # is specified in a Handler attribute
+            bisect.insort(self.handlers, handler)
+            handler.add_parent(self)
+
+    def close(self):
+        # Only exists for backwards compatibility.
+        pass
+
+    def _call_chain(self, chain, kind, meth_name, *args):
+        # Handlers raise an exception if no one else should try to handle
+        # the request, or return None if they can't but another handler
+        # could.  Otherwise, they return the response.
+        handlers = chain.get(kind, ())
+        for handler in handlers:
+            func = getattr(handler, meth_name)
+
+            result = func(*args)
+            if result is not None:
+                return result
+
+    def open(self, fullurl, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
+        # accept a URL or a Request object
+        if isinstance(fullurl, basestring):
+            req = Request(fullurl, data)
+        else:
+            req = fullurl
+            if data is not None:
+                req.add_data(data)
+
+        req.timeout = timeout
+        protocol = req.get_type()
+
+        # pre-process request
+        meth_name = protocol+"_request"
+        for processor in self.process_request.get(protocol, []):
+            meth = getattr(processor, meth_name)
+            req = meth(req)
+
+        response = self._open(req, data)
+
+        # post-process response
+        meth_name = protocol+"_response"
+        for processor in self.process_response.get(protocol, []):
+            meth = getattr(processor, meth_name)
+            try:
+                response = meth(req, response)
+            except:
+                response.close()
+                raise
+
+        return response
+
+    def _open(self, req, data=None):
+        result = self._call_chain(self.handle_open, 'default',
+                                  'default_open', req)
+        if result:
+            return result
+
+        protocol = req.get_type()
+        result = self._call_chain(self.handle_open, protocol, protocol +
+                                  '_open', req)
+        if result:
+            return result
+
+        return self._call_chain(self.handle_open, 'unknown',
+                                'unknown_open', req)
+
+    def error(self, proto, *args):
+        if proto in ('http', 'https'):
+            # XXX http[s] protocols are special-cased
+            dict = self.handle_error['http'] # https is not different than http
+            proto = args[2]  # YUCK!
+            meth_name = 'http_error_%s' % proto
+            http_err = 1
+            orig_args = args
+        else:
+            dict = self.handle_error
+            meth_name = proto + '_error'
+            http_err = 0
+        args = (dict, proto, meth_name) + args
+        result = self._call_chain(*args)
+        if result:
+            return result
+
+        if http_err:
+            args = (dict, 'default', 'http_error_default') + orig_args
+            return self._call_chain(*args)
+
+# XXX probably also want an abstract factory that knows when it makes
+# sense to skip a superclass in favor of a subclass and when it might
+# make sense to include both
+
+def build_opener(*handlers):
+    """Create an opener object from a list of handlers.
+
+    The opener will use several default handlers, including support
+    for HTTP, FTP and when applicable, HTTPS.
+
+    If any of the handlers passed as arguments are subclasses of the
+    default handlers, the default handlers will not be used.
+    """
+    import types
+    def isclass(obj):
+        return isinstance(obj, (types.ClassType, type))
+
+    opener = OpenerDirector()
+    default_classes = [ProxyHandler, UnknownHandler, HTTPHandler,
+                       HTTPDefaultErrorHandler, HTTPRedirectHandler,
+                       FTPHandler, FileHandler, HTTPErrorProcessor]
+    if hasattr(httplib, 'HTTPS'):
+        default_classes.append(HTTPSHandler)
+    skip = set()
+    for klass in default_classes:
+        for check in handlers:
+            if isclass(check):
+                if issubclass(check, klass):
+                    skip.add(klass)
+            elif isinstance(check, klass):
+                skip.add(klass)
+    for klass in skip:
+        default_classes.remove(klass)
+
+    for klass in default_classes:
+        opener.add_handler(klass())
+
+    for h in handlers:
+        if isclass(h):
+            h = h()
+        opener.add_handler(h)
+    return opener
+
+class BaseHandler:
+    handler_order = 500
+
+    def add_parent(self, parent):
+        self.parent = parent
+
+    def close(self):
+        # Only exists for backwards compatibility
+        pass
+
+    def __lt__(self, other):
+        if not hasattr(other, "handler_order"):
+            # Try to preserve the old behavior of having custom classes
+            # inserted after default ones (works only for custom user
+            # classes which are not aware of handler_order).
+            return True
+        return self.handler_order < other.handler_order
+
+
+class HTTPErrorProcessor(BaseHandler):
+    """Process HTTP error responses."""
+    handler_order = 1000  # after all other processing
+
+    def http_response(self, request, response):
+        code, msg, hdrs = response.code, response.msg, response.info()
+
+        # According to RFC 2616, "2xx" code indicates that the client's
+        # request was successfully received, understood, and accepted.
+        if not (200 <= code < 300):
+            response = self.parent.error(
+                'http', request, response, code, msg, hdrs)
+
+        return response
+
+    https_response = http_response
+
+class HTTPDefaultErrorHandler(BaseHandler):
+    def http_error_default(self, req, fp, code, msg, hdrs):
+        raise HTTPError(req.get_full_url(), code, msg, hdrs, fp)
+
+class HTTPRedirectHandler(BaseHandler):
+    # maximum number of redirections to any single URL
+    # this is needed because of the state that cookies introduce
+    max_repeats = 4
+    # maximum total number of redirections (regardless of URL) before
+    # assuming we're in a loop
+    max_redirections = 10
+
+    def redirect_request(self, req, fp, code, msg, headers, newurl):
+        """Return a Request or None in response to a redirect.
+
+        This is called by the http_error_30x methods when a
+        redirection response is received.  If a redirection should
+        take place, return a new Request to allow http_error_30x to
+        perform the redirect.  Otherwise, raise HTTPError if no-one
+        else should try to handle this url.  Return None if you can't
+        but another Handler might.
+        """
+        m = req.get_method()
+        if (code in (301, 302, 303, 307) and m in ("GET", "HEAD")
+            or code in (301, 302, 303) and m == "POST"):
+            # Strictly (according to RFC 2616), 301 or 302 in response
+            # to a POST MUST NOT cause a redirection without confirmation
+            # from the user (of urllib2, in this case).  In practice,
+            # essentially all clients do redirect in this case, so we
+            # do the same.
+            # be conciliant with URIs containing a space
+            newurl = newurl.replace(' ', '%20')
+            newheaders = dict((k,v) for k,v in req.headers.items()
+                              if k.lower() not in ("content-length", "content-type")
+                             )
+            return Request(newurl,
+                           headers=newheaders,
+                           origin_req_host=req.get_origin_req_host(),
+                           unverifiable=True)
+        else:
+            raise HTTPError(req.get_full_url(), code, msg, headers, fp)
+
+    # Implementation note: To avoid the server sending us into an
+    # infinite loop, the request object needs to track what URLs we
+    # have already seen.  Do this by adding a handler-specific
+    # attribute to the Request object.
+    def http_error_302(self, req, fp, code, msg, headers):
+        # Some servers (incorrectly) return multiple Location headers
+        # (so probably same goes for URI).  Use first header.
+        if 'location' in headers:
+            newurl = headers.getheaders('location')[0]
+        elif 'uri' in headers:
+            newurl = headers.getheaders('uri')[0]
+        else:
+            return
+
+        # fix a possible malformed URL
+        urlparts = urlparse.urlparse(newurl)
+        if not urlparts.path:
+            urlparts = list(urlparts)
+            urlparts[2] = "/"
+        newurl = urlparse.urlunparse(urlparts)
+
+        newurl = urlparse.urljoin(req.get_full_url(), newurl)
+
+        # XXX Probably want to forget about the state of the current
+        # request, although that might interact poorly with other
+        # handlers that also use handler-specific request attributes
+        new = self.redirect_request(req, fp, code, msg, headers, newurl)
+        if new is None:
+            return
+
+        # loop detection
+        # .redirect_dict has a key url if url was previously visited.
+        if hasattr(req, 'redirect_dict'):
+            visited = new.redirect_dict = req.redirect_dict
+            if (visited.get(newurl, 0) >= self.max_repeats or
+                len(visited) >= self.max_redirections):
+                raise HTTPError(req.get_full_url(), code,
+                                self.inf_msg + msg, headers, fp)
+        else:
+            visited = new.redirect_dict = req.redirect_dict = {}
+        visited[newurl] = visited.get(newurl, 0) + 1
+
+        # Don't close the fp until we are sure that we won't use it
+        # with HTTPError.
+        fp.read()
+        fp.close()
+
+        return self.parent.open(new, timeout=req.timeout)
+
+    http_error_301 = http_error_303 = http_error_307 = http_error_302
+
+    inf_msg = "The HTTP server returned a redirect error that would " \
+              "lead to an infinite loop.\n" \
+              "The last 30x error message was:\n"
+
+
+def _parse_proxy(proxy):
+    """Return (scheme, user, password, host/port) given a URL or an authority.
+
+    If a URL is supplied, it must have an authority (host:port) component.
+    According to RFC 3986, having an authority component means the URL must
+    have two slashes after the scheme:
+
+    >>> _parse_proxy('file:/ftp.example.com/')
+    Traceback (most recent call last):
+    ValueError: proxy URL with no authority: 'file:/ftp.example.com/'
+
+    The first three items of the returned tuple may be None.
+
+    Examples of authority parsing:
+
+    >>> _parse_proxy('proxy.example.com')
+    (None, None, None, 'proxy.example.com')
+    >>> _parse_proxy('proxy.example.com:3128')
+    (None, None, None, 'proxy.example.com:3128')
+
+    The authority component may optionally include userinfo (assumed to be
+    username:password):
+
+    >>> _parse_proxy('joe:password at proxy.example.com')
+    (None, 'joe', 'password', 'proxy.example.com')
+    >>> _parse_proxy('joe:password at proxy.example.com:3128')
+    (None, 'joe', 'password', 'proxy.example.com:3128')
+
+    Same examples, but with URLs instead:
+
+    >>> _parse_proxy('http://proxy.example.com/')
+    ('http', None, None, 'proxy.example.com')
+    >>> _parse_proxy('http://proxy.example.com:3128/')
+    ('http', None, None, 'proxy.example.com:3128')
+    >>> _parse_proxy('http://joe:password at proxy.example.com/')
+    ('http', 'joe', 'password', 'proxy.example.com')
+    >>> _parse_proxy('http://joe:password at proxy.example.com:3128')
+    ('http', 'joe', 'password', 'proxy.example.com:3128')
+
+    Everything after the authority is ignored:
+
+    >>> _parse_proxy('ftp://joe:password at proxy.example.com/rubbish:3128')
+    ('ftp', 'joe', 'password', 'proxy.example.com')
+
+    Test for no trailing '/' case:
+
+    >>> _parse_proxy('http://joe:password at proxy.example.com')
+    ('http', 'joe', 'password', 'proxy.example.com')
+
+    """
+    scheme, r_scheme = splittype(proxy)
+    if not r_scheme.startswith("/"):
+        # authority
+        scheme = None
+        authority = proxy
+    else:
+        # URL
+        if not r_scheme.startswith("//"):
+            raise ValueError("proxy URL with no authority: %r" % proxy)
+        # We have an authority, so for RFC 3986-compliant URLs (by ss 3.
+        # and 3.3.), path is empty or starts with '/'
+        end = r_scheme.find("/", 2)
+        if end == -1:
+            end = None
+        authority = r_scheme[2:end]
+    userinfo, hostport = splituser(authority)
+    if userinfo is not None:
+        user, password = splitpasswd(userinfo)
+    else:
+        user = password = None
+    return scheme, user, password, hostport
+
+class ProxyHandler(BaseHandler):
+    # Proxies must be in front
+    handler_order = 100
+
+    def __init__(self, proxies=None):
+        if proxies is None:
+            proxies = getproxies()
+        assert hasattr(proxies, 'has_key'), "proxies must be a mapping"
+        self.proxies = proxies
+        for type, url in proxies.items():
+            setattr(self, '%s_open' % type,
+                    lambda r, proxy=url, type=type, meth=self.proxy_open: \
+                    meth(r, proxy, type))
+
+    def proxy_open(self, req, proxy, type):
+        orig_type = req.get_type()
+        proxy_type, user, password, hostport = _parse_proxy(proxy)
+
+        if proxy_type is None:
+            proxy_type = orig_type
+
+        if req.host and proxy_bypass(req.host):
+            return None
+
+        if user and password:
+            user_pass = '%s:%s' % (unquote(user), unquote(password))
+            creds = base64.b64encode(user_pass).strip()
+            req.add_header('Proxy-authorization', 'Basic ' + creds)
+        hostport = unquote(hostport)
+        req.set_proxy(hostport, proxy_type)
+
+        if orig_type == proxy_type or orig_type == 'https':
+            # let other handlers take care of it
+            return None
+        else:
+            # need to start over, because the other handlers don't
+            # grok the proxy's URL type
+            # e.g. if we have a constructor arg proxies like so:
+            # {'http': 'ftp://proxy.example.com'}, we may end up turning
+            # a request for http://acme.example.com/a into one for
+            # ftp://proxy.example.com/a
+            return self.parent.open(req, timeout=req.timeout)
+
+class HTTPPasswordMgr:
+
+    def __init__(self):
+        self.passwd = {}
+
+    def add_password(self, realm, uri, user, passwd):
+        # uri could be a single URI or a sequence
+        if isinstance(uri, basestring):
+            uri = [uri]
+        if not realm in self.passwd:
+            self.passwd[realm] = {}
+        for default_port in True, False:
+            reduced_uri = tuple(
+                [self.reduce_uri(u, default_port) for u in uri])
+            self.passwd[realm][reduced_uri] = (user, passwd)
+
+    def find_user_password(self, realm, authuri):
+        domains = self.passwd.get(realm, {})
+        for default_port in True, False:
+            reduced_authuri = self.reduce_uri(authuri, default_port)
+            for uris, authinfo in domains.iteritems():
+                for uri in uris:
+                    if self.is_suburi(uri, reduced_authuri):
+                        return authinfo
+        return None, None
+
+    def reduce_uri(self, uri, default_port=True):
+        """Accept authority or URI and extract only the authority and path."""
+        # note HTTP URLs do not have a userinfo component
+        parts = urlparse.urlsplit(uri)
+        if parts[1]:
+            # URI
+            scheme = parts[0]
+            authority = parts[1]
+            path = parts[2] or '/'
+        else:
+            # host or host:port
+            scheme = None
+            authority = uri
+            path = '/'
+        host, port = splitport(authority)
+        if default_port and port is None and scheme is not None:
+            dport = {"http": 80,
+                     "https": 443,
+                     }.get(scheme)
+            if dport is not None:
+                authority = "%s:%d" % (host, dport)
+        return authority, path
+
+    def is_suburi(self, base, test):
+        """Check if test is below base in a URI tree
+
+        Both args must be URIs in reduced form.
+        """
+        if base == test:
+            return True
+        if base[0] != test[0]:
+            return False
+        common = posixpath.commonprefix((base[1], test[1]))
+        if len(common) == len(base[1]):
+            return True
+        return False
+
+
+class HTTPPasswordMgrWithDefaultRealm(HTTPPasswordMgr):
+
+    def find_user_password(self, realm, authuri):
+        user, password = HTTPPasswordMgr.find_user_password(self, realm,
+                                                            authuri)
+        if user is not None:
+            return user, password
+        return HTTPPasswordMgr.find_user_password(self, None, authuri)
+
+
+class AbstractBasicAuthHandler:
+
+    # XXX this allows for multiple auth-schemes, but will stupidly pick
+    # the last one with a realm specified.
+
+    # allow for double- and single-quoted realm values
+    # (single quotes are a violation of the RFC, but appear in the wild)
+    rx = re.compile('(?:.*,)*[ \t]*([^ \t]+)[ \t]+'
+                    'realm=(["\'])(.*?)\\2', re.I)
+
+    # XXX could pre-emptively send auth info already accepted (RFC 2617,
+    # end of section 2, and section 1.2 immediately after "credentials"
+    # production).
+
+    def __init__(self, password_mgr=None):
+        if password_mgr is None:
+            password_mgr = HTTPPasswordMgr()
+        self.passwd = password_mgr
+        self.add_password = self.passwd.add_password
+        self.retried = 0
+
+    def reset_retry_count(self):
+        self.retried = 0
+
+    def http_error_auth_reqed(self, authreq, host, req, headers):
+        # host may be an authority (without userinfo) or a URL with an
+        # authority
+        # XXX could be multiple headers
+        authreq = headers.get(authreq, None)
+
+        if self.retried > 5:
+            # retry sending the username:password 5 times before failing.
+            raise HTTPError(req.get_full_url(), 401, "basic auth failed",
+                            headers, None)
+        else:
+            self.retried += 1
+
+        if authreq:
+            mo = AbstractBasicAuthHandler.rx.search(authreq)
+            if mo:
+                scheme, quote, realm = mo.groups()
+                if scheme.lower() == 'basic':
+                    response = self.retry_http_basic_auth(host, req, realm)
+                    if response and response.code != 401:
+                        self.retried = 0
+                    return response
+
+    def retry_http_basic_auth(self, host, req, realm):
+        user, pw = self.passwd.find_user_password(realm, host)
+        if pw is not None:
+            raw = "%s:%s" % (user, pw)
+            auth = 'Basic %s' % base64.b64encode(raw).strip()
+            if req.headers.get(self.auth_header, None) == auth:
+                return None
+            req.add_unredirected_header(self.auth_header, auth)
+            return self.parent.open(req, timeout=req.timeout)
+        else:
+            return None
+
+
+class HTTPBasicAuthHandler(AbstractBasicAuthHandler, BaseHandler):
+
+    auth_header = 'Authorization'
+
+    def http_error_401(self, req, fp, code, msg, headers):
+        url = req.get_full_url()
+        response = self.http_error_auth_reqed('www-authenticate',
+                                              url, req, headers)
+        self.reset_retry_count()
+        return response
+
+
+class ProxyBasicAuthHandler(AbstractBasicAuthHandler, BaseHandler):
+
+    auth_header = 'Proxy-authorization'
+
+    def http_error_407(self, req, fp, code, msg, headers):
+        # http_error_auth_reqed requires that there is no userinfo component in
+        # authority.  Assume there isn't one, since urllib2 does not (and
+        # should not, RFC 3986 s. 3.2.1) support requests for URLs containing
+        # userinfo.
+        authority = req.get_host()
+        response = self.http_error_auth_reqed('proxy-authenticate',
+                                          authority, req, headers)
+        self.reset_retry_count()
+        return response
+
+
+def randombytes(n):
+    """Return n random bytes."""
+    # Use /dev/urandom if it is available.  Fall back to random module
+    # if not.  It might be worthwhile to extend this function to use
+    # other platform-specific mechanisms for getting random bytes.
+    if os.path.exists("/dev/urandom"):
+        f = open("/dev/urandom")
+        s = f.read(n)
+        f.close()
+        return s
+    else:
+        L = [chr(random.randrange(0, 256)) for i in range(n)]
+        return "".join(L)
+
+class AbstractDigestAuthHandler:
+    # Digest authentication is specified in RFC 2617.
+
+    # XXX The client does not inspect the Authentication-Info header
+    # in a successful response.
+
+    # XXX It should be possible to test this implementation against
+    # a mock server that just generates a static set of challenges.
+
+    # XXX qop="auth-int" supports is shaky
+
+    def __init__(self, passwd=None):
+        if passwd is None:
+            passwd = HTTPPasswordMgr()
+        self.passwd = passwd
+        self.add_password = self.passwd.add_password
+        self.retried = 0
+        self.nonce_count = 0
+        self.last_nonce = None
+
+    def reset_retry_count(self):
+        self.retried = 0
+
+    def http_error_auth_reqed(self, auth_header, host, req, headers):
+        authreq = headers.get(auth_header, None)
+        if self.retried > 5:
+            # Don't fail endlessly - if we failed once, we'll probably
+            # fail a second time. Hm. Unless the Password Manager is
+            # prompting for the information. Crap. This isn't great
+            # but it's better than the current 'repeat until recursion
+            # depth exceeded' approach <wink>
+            raise HTTPError(req.get_full_url(), 401, "digest auth failed",
+                            headers, None)
+        else:
+            self.retried += 1
+        if authreq:
+            scheme = authreq.split()[0]
+            if scheme.lower() == 'digest':
+                return self.retry_http_digest_auth(req, authreq)
+
+    def retry_http_digest_auth(self, req, auth):
+        token, challenge = auth.split(' ', 1)
+        chal = parse_keqv_list(parse_http_list(challenge))
+        auth = self.get_authorization(req, chal)
+        if auth:
+            auth_val = 'Digest %s' % auth
+            if req.headers.get(self.auth_header, None) == auth_val:
+                return None
+            req.add_unredirected_header(self.auth_header, auth_val)
+            resp = self.parent.open(req, timeout=req.timeout)
+            return resp
+
+    def get_cnonce(self, nonce):
+        # The cnonce-value is an opaque
+        # quoted string value provided by the client and used by both client
+        # and server to avoid chosen plaintext attacks, to provide mutual
+        # authentication, and to provide some message integrity protection.
+        # This isn't a fabulous effort, but it's probably Good Enough.
+        dig = hashlib.sha1("%s:%s:%s:%s" % (self.nonce_count, nonce, time.ctime(),
+                                            randombytes(8))).hexdigest()
+        return dig[:16]
+
+    def get_authorization(self, req, chal):
+        try:
+            realm = chal['realm']
+            nonce = chal['nonce']
+            qop = chal.get('qop')
+            algorithm = chal.get('algorithm', 'MD5')
+            # mod_digest doesn't send an opaque, even though it isn't
+            # supposed to be optional
+            opaque = chal.get('opaque', None)
+        except KeyError:
+            return None
+
+        H, KD = self.get_algorithm_impls(algorithm)
+        if H is None:
+            return None
+
+        user, pw = self.passwd.find_user_password(realm, req.get_full_url())
+        if user is None:
+            return None
+
+        # XXX not implemented yet
+        if req.has_data():
+            entdig = self.get_entity_digest(req.get_data(), chal)
+        else:
+            entdig = None
+
+        A1 = "%s:%s:%s" % (user, realm, pw)
+        A2 = "%s:%s" % (req.get_method(),
+                        # XXX selector: what about proxies and full urls
+                        req.get_selector())
+        if qop == 'auth':
+            if nonce == self.last_nonce:
+                self.nonce_count += 1
+            else:
+                self.nonce_count = 1
+                self.last_nonce = nonce
+
+            ncvalue = '%08x' % self.nonce_count
+            cnonce = self.get_cnonce(nonce)
+            noncebit = "%s:%s:%s:%s:%s" % (nonce, ncvalue, cnonce, qop, H(A2))
+            respdig = KD(H(A1), noncebit)
+        elif qop is None:
+            respdig = KD(H(A1), "%s:%s" % (nonce, H(A2)))
+        else:
+            # XXX handle auth-int.
+            raise URLError("qop '%s' is not supported." % qop)
+
+        # XXX should the partial digests be encoded too?
+
+        base = 'username="%s", realm="%s", nonce="%s", uri="%s", ' \
+               'response="%s"' % (user, realm, nonce, req.get_selector(),
+                                  respdig)
+        if opaque:
+            base += ', opaque="%s"' % opaque
+        if entdig:
+            base += ', digest="%s"' % entdig
+        base += ', algorithm="%s"' % algorithm
+        if qop:
+            base += ', qop=auth, nc=%s, cnonce="%s"' % (ncvalue, cnonce)
+        return base
+
+    def get_algorithm_impls(self, algorithm):
+        # algorithm should be case-insensitive according to RFC2617
+        algorithm = algorithm.upper()
+        # lambdas assume digest modules are imported at the top level
+        if algorithm == 'MD5':
+            H = lambda x: hashlib.md5(x).hexdigest()
+        elif algorithm == 'SHA':
+            H = lambda x: hashlib.sha1(x).hexdigest()
+        # XXX MD5-sess
+        KD = lambda s, d: H("%s:%s" % (s, d))
+        return H, KD
+
+    def get_entity_digest(self, data, chal):
+        # XXX not implemented yet
+        return None
+
+
+class HTTPDigestAuthHandler(BaseHandler, AbstractDigestAuthHandler):
+    """An authentication protocol defined by RFC 2069
+
+    Digest authentication improves on basic authentication because it
+    does not transmit passwords in the clear.
+    """
+
+    auth_header = 'Authorization'
+    handler_order = 490  # before Basic auth
+
+    def http_error_401(self, req, fp, code, msg, headers):
+        host = urlparse.urlparse(req.get_full_url())[1]
+        retry = self.http_error_auth_reqed('www-authenticate',
+                                           host, req, headers)
+        self.reset_retry_count()
+        return retry
+
+
+class ProxyDigestAuthHandler(BaseHandler, AbstractDigestAuthHandler):
+
+    auth_header = 'Proxy-Authorization'
+    handler_order = 490  # before Basic auth
+
+    def http_error_407(self, req, fp, code, msg, headers):
+        host = req.get_host()
+        retry = self.http_error_auth_reqed('proxy-authenticate',
+                                           host, req, headers)
+        self.reset_retry_count()
+        return retry
+
+class AbstractHTTPHandler(BaseHandler):
+
+    def __init__(self, debuglevel=0):
+        self._debuglevel = debuglevel
+
+    def set_http_debuglevel(self, level):
+        self._debuglevel = level
+
+    def do_request_(self, request):
+        host = request.get_host()
+        if not host:
+            raise URLError('no host given')
+
+        if request.has_data():  # POST
+            data = request.get_data()
+            if not request.has_header('Content-type'):
+                request.add_unredirected_header(
+                    'Content-type',
+                    'application/x-www-form-urlencoded')
+            if not request.has_header('Content-length'):
+                request.add_unredirected_header(
+                    'Content-length', '%d' % len(data))
+
+        sel_host = host
+        if request.has_proxy():
+            scheme, sel = splittype(request.get_selector())
+            sel_host, sel_path = splithost(sel)
+
+        if not request.has_header('Host'):
+            request.add_unredirected_header('Host', sel_host)
+        for name, value in self.parent.addheaders:
+            name = name.capitalize()
+            if not request.has_header(name):
+                request.add_unredirected_header(name, value)
+
+        return request
+
+    def do_open(self, http_class, req):
+        """Return an addinfourl object for the request, using http_class.
+
+        http_class must implement the HTTPConnection API from httplib.
+        The addinfourl return value is a file-like object.  It also
+        has methods and attributes including:
+            - info(): return a mimetools.Message object for the headers
+            - geturl(): return the original request URL
+            - code: HTTP status code
+        """
+        host = req.get_host()
+        if not host:
+            raise URLError('no host given')
+
+        h = http_class(host, timeout=req.timeout) # will parse host:port
+        h.set_debuglevel(self._debuglevel)
+
+        headers = dict(req.unredirected_hdrs)
+        headers.update(dict((k, v) for k, v in req.headers.items()
+                            if k not in headers))
+
+        # We want to make an HTTP/1.1 request, but the addinfourl
+        # class isn't prepared to deal with a persistent connection.
+        # It will try to read all remaining data from the socket,
+        # which will block while the server waits for the next request.
+        # So make sure the connection gets closed after the (only)
+        # request.
+        headers["Connection"] = "close"
+        headers = dict(
+            (name.title(), val) for name, val in headers.items())
+
+        if req._tunnel_host:
+            tunnel_headers = {}
+            proxy_auth_hdr = "Proxy-Authorization"
+            if proxy_auth_hdr in headers:
+                tunnel_headers[proxy_auth_hdr] = headers[proxy_auth_hdr]
+                # Proxy-Authorization should not be sent to origin
+                # server.
+                del headers[proxy_auth_hdr]
+            h.set_tunnel(req._tunnel_host, headers=tunnel_headers)
+
+        try:
+            h.request(req.get_method(), req.get_selector(), req.data, headers)
+            try:
+                r = h.getresponse(buffering=True)
+            except TypeError: #buffering kw not supported
+                r = h.getresponse()
+        except socket.error, err: # XXX what error?
+            h.close()
+            raise URLError(err)
+
+        # Pick apart the HTTPResponse object to get the addinfourl
+        # object initialized properly.
+
+        # Wrap the HTTPResponse object in socket's file object adapter
+        # for Windows.  That adapter calls recv(), so delegate recv()
+        # to read().  This weird wrapping allows the returned object to
+        # have readline() and readlines() methods.
+
+        # XXX It might be better to extract the read buffering code
+        # out of socket._fileobject() and into a base class.
+
+        r.recv = r.read
+        fp = socket._fileobject(r, close=True)
+
+        resp = addinfourl(fp, r.msg, req.get_full_url())
+        resp.code = r.status
+        resp.msg = r.reason
+        return resp
+
+
+class HTTPHandler(AbstractHTTPHandler):
+
+    def http_open(self, req):
+        return self.do_open(httplib.HTTPConnection, req)
+
+    http_request = AbstractHTTPHandler.do_request_
+
+if hasattr(httplib, 'HTTPS'):
+    class HTTPSHandler(AbstractHTTPHandler):
+
+        def https_open(self, req):
+            return self.do_open(httplib.HTTPSConnection, req)
+
+        https_request = AbstractHTTPHandler.do_request_
+
+class HTTPCookieProcessor(BaseHandler):
+    def __init__(self, cookiejar=None):
+        import cookielib
+        if cookiejar is None:
+            cookiejar = cookielib.CookieJar()
+        self.cookiejar = cookiejar
+
+    def http_request(self, request):
+        self.cookiejar.add_cookie_header(request)
+        return request
+
+    def http_response(self, request, response):
+        self.cookiejar.extract_cookies(response, request)
+        return response
+
+    https_request = http_request
+    https_response = http_response
+
+class UnknownHandler(BaseHandler):
+    def unknown_open(self, req):
+        type = req.get_type()
+        raise URLError('unknown url type: %s' % type)
+
+def parse_keqv_list(l):
+    """Parse list of key=value strings where keys are not duplicated."""
+    parsed = {}
+    for elt in l:
+        k, v = elt.split('=', 1)
+        if v[0] == '"' and v[-1] == '"':
+            v = v[1:-1]
+        parsed[k] = v
+    return parsed
+
+def parse_http_list(s):
+    """Parse lists as described by RFC 2068 Section 2.
+
+    In particular, parse comma-separated lists where the elements of
+    the list may include quoted-strings.  A quoted-string could
+    contain a comma.  A non-quoted string could have quotes in the
+    middle.  Neither commas nor quotes count if they are escaped.
+    Only double-quotes count, not single-quotes.
+    """
+    res = []
+    part = ''
+
+    escape = quote = False
+    for cur in s:
+        if escape:
+            part += cur
+            escape = False
+            continue
+        if quote:
+            if cur == '\\':
+                escape = True
+                continue
+            elif cur == '"':
+                quote = False
+            part += cur
+            continue
+
+        if cur == ',':
+            res.append(part)
+            part = ''
+            continue
+
+        if cur == '"':
+            quote = True
+
+        part += cur
+
+    # append last part
+    if part:
+        res.append(part)
+
+    return [part.strip() for part in res]
+
+def _safe_gethostbyname(host):
+    try:
+        return socket.gethostbyname(host)
+    except socket.gaierror:
+        return None
+
+class FileHandler(BaseHandler):
+    # Use local file or FTP depending on form of URL
+    def file_open(self, req):
+        url = req.get_selector()
+        if url[:2] == '//' and url[2:3] != '/' and (req.host and
+                req.host != 'localhost'):
+            req.type = 'ftp'
+            return self.parent.open(req)
+        else:
+            return self.open_local_file(req)
+
+    # names for the localhost
+    names = None
+    def get_names(self):
+        if FileHandler.names is None:
+            try:
+                FileHandler.names = tuple(
+                    socket.gethostbyname_ex('localhost')[2] +
+                    socket.gethostbyname_ex(socket.gethostname())[2])
+            except socket.gaierror:
+                FileHandler.names = (socket.gethostbyname('localhost'),)
+        return FileHandler.names
+
+    # not entirely sure what the rules are here
+    def open_local_file(self, req):
+        import email.utils
+        import mimetypes
+        host = req.get_host()
+        filename = req.get_selector()
+        localfile = url2pathname(filename)
+        try:
+            stats = os.stat(localfile)
+            size = stats.st_size
+            modified = email.utils.formatdate(stats.st_mtime, usegmt=True)
+            mtype = mimetypes.guess_type(filename)[0]
+            headers = mimetools.Message(StringIO(
+                'Content-type: %s\nContent-length: %d\nLast-modified: %s\n' %
+                (mtype or 'text/plain', size, modified)))
+            if host:
+                host, port = splitport(host)
+            if not host or \
+                (not port and _safe_gethostbyname(host) in self.get_names()):
+                if host:
+                    origurl = 'file://' + host + filename
+                else:
+                    origurl = 'file://' + filename
+                return addinfourl(open(localfile, 'rb'), headers, origurl)
+        except OSError, msg:
+            # urllib2 users shouldn't expect OSErrors coming from urlopen()
+            raise URLError(msg)
+        raise URLError('file not on local host')
+
+class FTPHandler(BaseHandler):
+    def ftp_open(self, req):
+        import ftplib
+        import mimetypes
+        host = req.get_host()
+        if not host:
+            raise URLError('ftp error: no host given')
+        host, port = splitport(host)
+        if port is None:
+            port = ftplib.FTP_PORT
+        else:
+            port = int(port)
+
+        # username/password handling
+        user, host = splituser(host)
+        if user:
+            user, passwd = splitpasswd(user)
+        else:
+            passwd = None
+        host = unquote(host)
+        user = user or ''
+        passwd = passwd or ''
+
+        try:
+            host = socket.gethostbyname(host)
+        except socket.error, msg:
+            raise URLError(msg)
+        path, attrs = splitattr(req.get_selector())
+        dirs = path.split('/')
+        dirs = map(unquote, dirs)
+        dirs, file = dirs[:-1], dirs[-1]
+        if dirs and not dirs[0]:
+            dirs = dirs[1:]
+        try:
+            fw = self.connect_ftp(user, passwd, host, port, dirs, req.timeout)
+            type = file and 'I' or 'D'
+            for attr in attrs:
+                attr, value = splitvalue(attr)
+                if attr.lower() == 'type' and \
+                   value in ('a', 'A', 'i', 'I', 'd', 'D'):
+                    type = value.upper()
+            fp, retrlen = fw.retrfile(file, type)
+            headers = ""
+            mtype = mimetypes.guess_type(req.get_full_url())[0]
+            if mtype:
+                headers += "Content-type: %s\n" % mtype
+            if retrlen is not None and retrlen >= 0:
+                headers += "Content-length: %d\n" % retrlen
+            sf = StringIO(headers)
+            headers = mimetools.Message(sf)
+            return addinfourl(fp, headers, req.get_full_url())
+        except ftplib.all_errors, msg:
+            raise URLError, ('ftp error: %s' % msg), sys.exc_info()[2]
+
+    def connect_ftp(self, user, passwd, host, port, dirs, timeout):
+        fw = ftpwrapper(user, passwd, host, port, dirs, timeout)
+##        fw.ftp.set_debuglevel(1)
+        return fw
+
+class CacheFTPHandler(FTPHandler):
+    # XXX would be nice to have pluggable cache strategies
+    # XXX this stuff is definitely not thread safe
+    def __init__(self):
+        self.cache = {}
+        self.timeout = {}
+        self.soonest = 0
+        self.delay = 60
+        self.max_conns = 16
+
+    def setTimeout(self, t):
+        self.delay = t
+
+    def setMaxConns(self, m):
+        self.max_conns = m
+
+    def connect_ftp(self, user, passwd, host, port, dirs, timeout):
+        key = user, host, port, '/'.join(dirs), timeout
+        if key in self.cache:
+            self.timeout[key] = time.time() + self.delay
+        else:
+            self.cache[key] = ftpwrapper(user, passwd, host, port, dirs, timeout)
+            self.timeout[key] = time.time() + self.delay
+        self.check_cache()
+        return self.cache[key]
+
+    def check_cache(self):
+        # first check for old ones
+        t = time.time()
+        if self.soonest <= t:
+            for k, v in self.timeout.items():
+                if v < t:
+                    self.cache[k].close()
+                    del self.cache[k]
+                    del self.timeout[k]
+        self.soonest = min(self.timeout.values())
+
+        # then check the size
+        if len(self.cache) == self.max_conns:
+            for k, v in self.timeout.items():
+                if v == self.soonest:
+                    del self.cache[k]
+                    del self.timeout[k]
+                    break
+            self.soonest = min(self.timeout.values())
diff --git a/lib_pypy/resource.py b/lib_pypy/resource.py
--- a/lib_pypy/resource.py
+++ b/lib_pypy/resource.py
@@ -7,7 +7,7 @@
 
 from ctypes_support import standard_c_lib as libc
 from ctypes_support import get_errno
-from ctypes import Structure, c_int, c_long, byref, sizeof, POINTER
+from ctypes import Structure, c_int, c_long, byref, POINTER
 from errno import EINVAL, EPERM
 import _structseq
 
@@ -165,7 +165,6 @@
 
 @builtinify
 def getpagesize():
-    pagesize = 0
     if _getpagesize:
         return _getpagesize()
     else:
diff --git a/pypy/annotation/classdef.py b/pypy/annotation/classdef.py
--- a/pypy/annotation/classdef.py
+++ b/pypy/annotation/classdef.py
@@ -276,8 +276,8 @@
         # create the Attribute and do the generalization asked for
         newattr = Attribute(attr, self.bookkeeper)
         if s_value:
-            if newattr.name == 'intval' and getattr(s_value, 'unsigned', False):
-                import pdb; pdb.set_trace()
+            #if newattr.name == 'intval' and getattr(s_value, 'unsigned', False):
+            #    import pdb; pdb.set_trace()
             newattr.s_value = s_value
 
         # keep all subattributes' values
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -127,7 +127,7 @@
 
 pypy_optiondescription = OptionDescription("objspace", "Object Space Options", [
     ChoiceOption("name", "Object Space name",
-                 ["std", "flow", "thunk", "dump", "taint"],
+                 ["std", "flow", "thunk", "dump"],
                  "std",
                  cmdline='--objspace -o'),
 
diff --git a/pypy/doc/__pypy__-module.rst b/pypy/doc/__pypy__-module.rst
--- a/pypy/doc/__pypy__-module.rst
+++ b/pypy/doc/__pypy__-module.rst
@@ -37,29 +37,6 @@
 .. _`thunk object space docs`: objspace-proxies.html#thunk
 .. _`interface section of the thunk object space docs`: objspace-proxies.html#thunk-interface
 
-.. broken:
-
-    Taint Object Space Functionality
-    ================================
-
-    When the taint object space is used (choose with :config:`objspace.name`),
-    the following names are put into ``__pypy__``:
-
-     - ``taint``
-     - ``is_tainted``
-     - ``untaint``
-     - ``taint_atomic``
-     - ``_taint_debug``
-     - ``_taint_look``
-     - ``TaintError``
-
-    Those are all described in the `interface section of the taint object space
-    docs`_.
-
-    For more detailed explanations and examples see the `taint object space docs`_.
-
-    .. _`taint object space docs`: objspace-proxies.html#taint
-    .. _`interface section of the taint object space docs`: objspace-proxies.html#taint-interface
 
 Transparent Proxy Functionality
 ===============================
diff --git a/pypy/doc/config/objspace.name.txt b/pypy/doc/config/objspace.name.txt
--- a/pypy/doc/config/objspace.name.txt
+++ b/pypy/doc/config/objspace.name.txt
@@ -4,7 +4,6 @@
 for normal usage):
 
   * thunk_: The thunk object space adds lazy evaluation to PyPy.
-  * taint_: The taint object space adds soft security features.
   * dump_:  Using this object spaces results in the dumpimp of all operations
     to a log.
 
@@ -12,5 +11,4 @@
 .. _`Object Space Proxies`: ../objspace-proxies.html
 .. _`Standard Object Space`: ../objspace.html#standard-object-space
 .. _thunk: ../objspace-proxies.html#thunk
-.. _taint: ../objspace-proxies.html#taint
 .. _dump: ../objspace-proxies.html#dump
diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst
--- a/pypy/doc/index.rst
+++ b/pypy/doc/index.rst
@@ -309,7 +309,6 @@
 .. _`object space`: objspace.html
 .. _FlowObjSpace: objspace.html#the-flow-object-space 
 .. _`trace object space`: objspace.html#the-trace-object-space 
-.. _`taint object space`: objspace-proxies.html#taint
 .. _`thunk object space`: objspace-proxies.html#thunk
 .. _`transparent proxies`: objspace-proxies.html#tproxy
 .. _`Differences between PyPy and CPython`: cpython_differences.html
diff --git a/pypy/doc/objspace-proxies.rst b/pypy/doc/objspace-proxies.rst
--- a/pypy/doc/objspace-proxies.rst
+++ b/pypy/doc/objspace-proxies.rst
@@ -129,297 +129,6 @@
    function behaves lazily: all calls to it return a thunk object.
 
 
-.. broken right now:
-
-    .. _taint:
-
-    The Taint Object Space
-    ======================
-
-    Motivation
-    ----------
-
-    The Taint Object Space provides a form of security: "tainted objects",
-    inspired by various sources, see [D12.1]_ for a more detailed discussion. 
-
-    The basic idea of this kind of security is not to protect against
-    malicious code but to help with handling and boxing sensitive data. 
-    It covers two kinds of sensitive data: secret data which should not leak, 
-    and untrusted data coming from an external source and that must be 
-    validated before it is used.
-
-    The idea is that, considering a large application that handles these
-    kinds of sensitive data, there are typically only a small number of
-    places that need to explicitly manipulate that sensitive data; all the
-    other places merely pass it around, or do entirely unrelated things.
-
-    Nevertheless, if a large application needs to be reviewed for security,
-    it must be entirely carefully checked, because it is possible that a
-    bug at some apparently unrelated place could lead to a leak of sensitive
-    information in a way that an external attacker could exploit.  For
-    example, if any part of the application provides web services, an
-    attacker might be able to issue unexpected requests with a regular web
-    browser and deduce secret information from the details of the answers he
-    gets.  Another example is the common CGI attack where an attacker sends
-    malformed inputs and causes the CGI script to do unintended things.
-
-    An approach like that of the Taint Object Space allows the small parts
-    of the program that manipulate sensitive data to be explicitly marked.
-    The effect of this is that although these small parts still need a
-    careful security review, the rest of the application no longer does,
-    because even a bug would be unable to leak the information.
-
-    We have implemented a simple two-level model: objects are either
-    regular (untainted), or sensitive (tainted).  Objects are marked as
-    sensitive if they are secret or untrusted, and only declassified at
-    carefully-checked positions (e.g. where the secret data is needed, or
-    after the untrusted data has been fully validated).
-
-    It would be simple to extend the code for more fine-grained scales of
-    secrecy.  For example it is typical in the literature to consider
-    user-specified lattices of secrecy levels, corresponding to multiple
-    "owners" that cannot access data belonging to another "owner" unless
-    explicitly authorized to do so.
-
-    Tainting and untainting
-    -----------------------
-
-    Start a py.py with the Taint Object Space and try the following example::
-
-        $ py.py -o taint
-        >>>> from __pypy__ import taint
-        >>>> x = taint(6)
-
-        # x is hidden from now on.  We can pass it around and
-        # even operate on it, but not inspect it.  Taintness
-        # is propagated to operation results.
-
-        >>>> x
-        TaintError
-
-        >>>> if x > 5: y = 2   # see below
-        TaintError
-
-        >>>> y = x + 5         # ok
-        >>>> lst = [x, y]
-        >>>> z = lst.pop()
-        >>>> t = type(z)       # type() works too, tainted answer
-        >>>> t
-        TaintError
-        >>>> u = t is int      # even 'is' works
-        >>>> u
-        TaintError
-
-    Notice that using a tainted boolean like ``x > 5`` in an ``if``
-    statement is forbidden.  This is because knowing which path is followed
-    would give away a hint about ``x``; in the example above, if the
-    statement ``if x > 5: y = 2`` was allowed to run, we would know
-    something about the value of ``x`` by looking at the (untainted) value
-    in the variable ``y``.
-
-    Of course, there is a way to inspect tainted objects.  The basic way is
-    to explicitly "declassify" it with the ``untaint()`` function.  In an
-    application, the places that use ``untaint()`` are the places that need
-    careful security review.  To avoid unexpected objects showing up, the
-    ``untaint()`` function must be called with the exact type of the object
-    to declassify.  It will raise ``TaintError`` if the type doesn't match::
-
-        >>>> from __pypy__ import taint
-        >>>> untaint(int, x)
-        6
-        >>>> untaint(int, z)
-        11
-        >>>> untaint(bool, x > 5)
-        True
-        >>>> untaint(int, x > 5)
-        TaintError
-
-
-    Taint Bombs
-    -----------
-
-    In this area, a common problem is what to do about failing operations.
-    If an operation raises an exception when manipulating a tainted object,
-    then the very presence of the exception can leak information about the
-    tainted object itself.  Consider::
-
-        >>>> 5 / (x-6)
-
-    By checking if this raises ``ZeroDivisionError`` or not, we would know
-    if ``x`` was equal to 6 or not.  The solution to this problem in the
-    Taint Object Space is to introduce *Taint Bombs*.  They are a kind of
-    tainted object that doesn't contain a real object, but a pending
-    exception.  Taint Bombs are indistinguishable from normal tainted
-    objects to unprivileged code. See::
-
-        >>>> x = taint(6)
-        >>>> i = 5 / (x-6)     # no exception here
-        >>>> j = i + 1         # nor here
-        >>>> k = j + 5         # nor here
-        >>>> untaint(int, k)
-        TaintError
-
-    In the above example, all of ``i``, ``j`` and ``k`` contain a Taint
-    Bomb.  Trying to untaint it raises an exception - a generic
-    ``TaintError``.  What we win is that the exception gives little away,
-    and most importantly it occurs at the point where ``untaint()`` is
-    called, not where the operation failed.  This means that all calls to
-    ``untaint()`` - but not the rest of the code - must be carefully
-    reviewed for what occurs if they receive a Taint Bomb; they might catch
-    the ``TaintError`` and give the user a generic message that something
-    went wrong, if we are reasonably careful that the message or even its
-    presence doesn't give information away.  This might be a
-    problem by itself, but there is no satisfying general solution here:
-    it must be considered on a case-by-case basis.  Again, what the
-    Taint Object Space approach achieves is not solving these problems, but
-    localizing them to well-defined small parts of the application - namely,
-    around calls to ``untaint()``.
-
-    The ``TaintError`` exception deliberately does not include any
-    useful error messages, because they might give information away.
-    Of course, this makes debugging quite a bit harder; a difficult
-    problem to solve properly.  So far we have implemented a way to peek in a Taint
-    Box or Bomb, ``__pypy__._taint_look(x)``, and a "debug mode" that
-    prints the exception as soon as a Bomb is created - both write
-    information to the low-level stderr of the application, where we hope
-    that it is unlikely to be seen by anyone but the application
-    developer.
-
-
-    Taint Atomic functions
-    ----------------------
-
-    Occasionally, a more complicated computation must be performed on a
-    tainted object.  This requires first untainting the object, performing the
-    computations, and then carefully tainting the result again (including
-    hiding all exceptions into Bombs).
-
-    There is a built-in decorator that does this for you::
-
-        >>>> @__pypy__.taint_atomic
-        >>>> def myop(x, y):
-        ....     while x > 0:
-        ....         x -= y
-        ....     return x
-        ....
-        >>>> myop(42, 10)
-        -8
-        >>>> z = myop(taint(42), 10)
-        >>>> z
-        TaintError
-        >>>> untaint(int, z)
-        -8
-
-    The decorator makes a whole function behave like a built-in operation.
-    If no tainted argument is passed in, the function behaves normally.  But
-    if any of the arguments is tainted, it is automatically untainted - so
-    the function body always sees untainted arguments - and the eventual
-    result is tainted again (possibly in a Taint Bomb).
-
-    It is important for the function marked as ``taint_atomic`` to have no
-    visible side effects, as these could cause information leakage.
-    This is currently not enforced, which means that all ``taint_atomic``
-    functions have to be carefully reviewed for security (but not the
-    callers of ``taint_atomic`` functions).
-
-    A possible future extension would be to forbid side-effects on
-    non-tainted objects from all ``taint_atomic`` functions.
-
-    An example of usage: given a tainted object ``passwords_db`` that
-    references a database of passwords, we can write a function
-    that checks if a password is valid as follows::
-
-        @taint_atomic
-        def validate(passwords_db, username, password):
-            assert type(passwords_db) is PasswordDatabase
-            assert type(username) is str
-            assert type(password) is str
-            ...load username entry from passwords_db...
-            return expected_password == password
-
-    It returns a tainted boolean answer, or a Taint Bomb if something
-    went wrong.  A caller can do::
-
-        ok = validate(passwords_db, 'john', '1234')
-        ok = untaint(bool, ok)
-
-    This can give three outcomes: ``True``, ``False``, or a ``TaintError``
-    exception (with no information on it) if anything went wrong.  If even
-    this is considered giving too much information away, the ``False`` case
-    can be made indistinguishable from the ``TaintError`` case (simply by
-    raising an exception in ``validate()`` if the password is wrong).
-
-    In the above example, the security results achieved are the following:
-    as long as ``validate()`` does not leak information, no other part of
-    the code can obtain more information about a passwords database than a
-    Yes/No answer to a precise query.
-
-    A possible extension of the ``taint_atomic`` decorator would be to check
-    the argument types, as ``untaint()`` does, for the same reason: to
-    prevent bugs where a function like ``validate()`` above is accidentally
-    called with the wrong kind of tainted object, which would make it
-    misbehave.  For now, all ``taint_atomic`` functions should be
-    conservative and carefully check all assumptions on their input
-    arguments.
-
-
-    .. _`taint-interface`:
-
-    Interface
-    ---------
-
-    .. _`like a built-in operation`:
-
-    The basic rule of the Tainted Object Space is that it introduces two new
-    kinds of objects, Tainted Boxes and Tainted Bombs (which are not types
-    in the Python sense).  Each box internally contains a regular object;
-    each bomb internally contains an exception object.  An operation
-    involving Tainted Boxes is performed on the objects contained in the
-    boxes, and gives a Tainted Box or a Tainted Bomb as a result (such an
-    operation does not let an exception be raised).  An operation called
-    with a Tainted Bomb argument immediately returns the same Tainted Bomb.
-
-    In a PyPy running with (or translated with) the Taint Object Space,
-    the ``__pypy__`` module exposes the following interface:
-
-    * ``taint(obj)``
-
-        Return a new Tainted Box wrapping ``obj``.  Return ``obj`` itself
-        if it is already tainted (a Box or a Bomb).
-
-    * ``is_tainted(obj)``
-
-        Check if ``obj`` is tainted (a Box or a Bomb).
-
-    * ``untaint(type, obj)``
-
-        Untaints ``obj`` if it is tainted.  Raise ``TaintError`` if the type
-        of the untainted object is not exactly ``type``, or if ``obj`` is a
-        Bomb.
-
-    * ``taint_atomic(func)``
-
-        Return a wrapper function around the callable ``func``.  The wrapper
-        behaves `like a built-in operation`_ with respect to untainting the
-        arguments, tainting the result, and returning a Bomb.
-
-    * ``TaintError``
-
-        Exception.  On purpose, it provides no attribute or error message.
-
-    * ``_taint_debug(level)``
-
-        Set the debugging level to ``level`` (0=off).  At level 1 or above,
-        all Taint Bombs print a diagnostic message to stderr when they are
-        created.
-
-    * ``_taint_look(obj)``
-
-        For debugging purposes: prints (to stderr) the type and address of
-        the object in a Tainted Box, or prints the exception if ``obj`` is
-        a Taint Bomb.
-
-
 .. _dump:
 
 The Dump Object Space
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -175,6 +175,9 @@
             self.w_tracefunc = w_func
             self.space.frame_trace_action.fire()
 
+    def gettrace(self):
+        return self.w_tracefunc
+
     def setprofile(self, w_func):
         """Set the global trace function."""
         if self.space.is_w(w_func, self.space.w_None):
@@ -388,8 +391,11 @@
     def decrement_ticker(self, by):
         value = self._ticker
         if self.has_bytecode_counter:    # this 'if' is constant-folded
-            value -= by
-            self._ticker = value
+            if jit.isconstant(by) and by == 0:
+                pass     # normally constant-folded too
+            else:
+                value -= by
+                self._ticker = value
         return value
 
 
diff --git a/pypy/interpreter/pyparser/pytokenizer.py b/pypy/interpreter/pyparser/pytokenizer.py
--- a/pypy/interpreter/pyparser/pytokenizer.py
+++ b/pypy/interpreter/pyparser/pytokenizer.py
@@ -226,7 +226,7 @@
                         parenlev = parenlev - 1
                         if parenlev < 0:
                             raise TokenError("unmatched '%s'" % initial, line,
-                                             lnum-1, 0, token_list)
+                                             lnum, start + 1, token_list)
                     if token in python_opmap:
                         punct = python_opmap[token]
                     else:
diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py
--- a/pypy/interpreter/pyparser/test/test_pyparse.py
+++ b/pypy/interpreter/pyparser/test/test_pyparse.py
@@ -87,6 +87,10 @@
         assert exc.lineno == 1
         assert exc.offset == 5
         assert exc.lastlineno == 5
+        exc = py.test.raises(SyntaxError, parse, "abc)").value
+        assert exc.msg == "unmatched ')'"
+        assert exc.lineno == 1
+        assert exc.offset == 4
 
     def test_is(self):
         self.parse("x is y")
diff --git a/pypy/jit/backend/llgraph/llimpl.py b/pypy/jit/backend/llgraph/llimpl.py
--- a/pypy/jit/backend/llgraph/llimpl.py
+++ b/pypy/jit/backend/llgraph/llimpl.py
@@ -165,6 +165,7 @@
     'unicodegetitem'  : (('ref', 'int'), 'int'),
     'unicodesetitem'  : (('ref', 'int', 'int'), 'int'),
     'cast_ptr_to_int' : (('ref',), 'int'),
+    'cast_int_to_ptr' : (('int',), 'ref'),
     'debug_merge_point': (('ref', 'int'), None),
     'force_token'     : ((), 'int'),
     'call_may_force'  : (('int', 'varargs'), 'intorptr'),
@@ -875,9 +876,6 @@
     def op_new_array(self, arraydescr, count):
         return do_new_array(arraydescr.ofs, count)
 
-    def op_cast_ptr_to_int(self, descr, ptr):
-        return cast_to_int(ptr)
-
     def op_force_token(self, descr):
         opaque_frame = _to_opaque(self)
         return llmemory.cast_ptr_to_adr(opaque_frame)
diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py
--- a/pypy/jit/backend/llsupport/gc.py
+++ b/pypy/jit/backend/llsupport/gc.py
@@ -45,6 +45,14 @@
     def freeing_block(self, start, stop):
         pass
 
+    def record_constptrs(self, op, gcrefs_output_list):
+        for i in range(op.numargs()):
+            v = op.getarg(i)
+            if isinstance(v, ConstPtr) and bool(v.value):
+                p = v.value
+                rgc._make_sure_does_not_move(p)
+                gcrefs_output_list.append(p)
+
 # ____________________________________________________________
 
 class GcLLDescr_boehm(GcLLDescription):
@@ -141,6 +149,14 @@
     get_funcptr_for_newstr = None
     get_funcptr_for_newunicode = None
 
+    def rewrite_assembler(self, cpu, operations, gcrefs_output_list):
+        # record all GCREFs too, because Boehm cannot see them and keep them
+        # alive if they end up as constants in the assembler
+        for op in operations:
+            self.record_constptrs(op, gcrefs_output_list)
+        return GcLLDescription.rewrite_assembler(self, cpu, operations,
+                                                 gcrefs_output_list)
+
 
 # ____________________________________________________________
 # All code below is for the hybrid or minimark GC
@@ -757,14 +773,6 @@
             funcptr(llmemory.cast_ptr_to_adr(gcref_struct),
                     llmemory.cast_ptr_to_adr(gcref_newptr))
 
-    def record_constptrs(self, op, gcrefs_output_list):
-        for i in range(op.numargs()):
-            v = op.getarg(i)
-            if isinstance(v, ConstPtr) and bool(v.value):
-                p = v.value
-                rgc._make_sure_does_not_move(p)
-                gcrefs_output_list.append(p)
-
     def rewrite_assembler(self, cpu, operations, gcrefs_output_list):
         # Perform two kinds of rewrites in parallel:
         #
diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -1468,20 +1468,16 @@
         return u''.join(u.chars)
 
 
-    def test_casts(self):
-        py.test.skip("xxx fix or kill")
-        from pypy.rpython.lltypesystem import lltype, llmemory
-        TP = lltype.GcStruct('x')
-        x = lltype.malloc(TP)        
-        x = lltype.cast_opaque_ptr(llmemory.GCREF, x)
+    def test_cast_int_to_ptr(self):
+        res = self.execute_operation(rop.CAST_INT_TO_PTR,
+                                     [BoxInt(-17)],  'ref').value
+        assert lltype.cast_ptr_to_int(res) == -17
+
+    def test_cast_ptr_to_int(self):
+        x = lltype.cast_int_to_ptr(llmemory.GCREF, -19)
         res = self.execute_operation(rop.CAST_PTR_TO_INT,
-                                     [BoxPtr(x)],  'int').value
-        expected = self.cpu.cast_adr_to_int(llmemory.cast_ptr_to_adr(x))
-        assert rffi.get_real_int(res) == rffi.get_real_int(expected)
-        res = self.execute_operation(rop.CAST_PTR_TO_INT,
-                                     [ConstPtr(x)],  'int').value
-        expected = self.cpu.cast_adr_to_int(llmemory.cast_ptr_to_adr(x))
-        assert rffi.get_real_int(res) == rffi.get_real_int(expected)
+                                     [BoxPtr(x)], 'int').value
+        assert res == -19
 
     def test_ooops_non_gc(self):
         x = lltype.malloc(lltype.Struct('x'), flavor='raw')
@@ -2299,13 +2295,6 @@
         #
         cpu.bh_strsetitem(x, 4, ord('/'))
         assert str.chars[4] == '/'
-        #
-##        x = cpu.bh_newstr(5)
-##        y = cpu.bh_cast_ptr_to_int(x)
-##        z = cpu.bh_cast_ptr_to_int(x)
-##        y = rffi.get_real_int(y)
-##        z = rffi.get_real_int(z)
-##        assert type(y) == type(z) == int and y == z
 
     def test_sorting_of_fields(self):
         S = self.S
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1387,7 +1387,8 @@
 
     def genop_same_as(self, op, arglocs, resloc):
         self.mov(arglocs[0], resloc)
-    #genop_cast_ptr_to_int = genop_same_as
+    genop_cast_ptr_to_int = genop_same_as
+    genop_cast_int_to_ptr = genop_same_as
 
     def genop_int_mod(self, op, arglocs, resloc):
         if IS_X86_32:
diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py
--- a/pypy/jit/backend/x86/regalloc.py
+++ b/pypy/jit/backend/x86/regalloc.py
@@ -1152,7 +1152,8 @@
         self.possibly_free_var(op.getarg(0))
         resloc = self.force_allocate_reg(op.result)
         self.Perform(op, [argloc], resloc)
-    #consider_cast_ptr_to_int = consider_same_as
+    consider_cast_ptr_to_int = consider_same_as
+    consider_cast_int_to_ptr = consider_same_as
 
     def consider_strlen(self, op):
         args = op.getarglist()
diff --git a/pypy/jit/backend/x86/tool/viewcode.py b/pypy/jit/backend/x86/tool/viewcode.py
--- a/pypy/jit/backend/x86/tool/viewcode.py
+++ b/pypy/jit/backend/x86/tool/viewcode.py
@@ -9,7 +9,12 @@
 """
 
 import autopath
-import operator, sys, os, re, py, new
+import new
+import operator
+import py
+import re
+import sys
+import subprocess
 from bisect import bisect_left
 
 # don't use pypy.tool.udir here to avoid removing old usessions which
@@ -44,14 +49,16 @@
     f = open(tmpfile, 'wb')
     f.write(data)
     f.close()
-    g = os.popen(objdump % {
+    p = subprocess.Popen(objdump % {
         'file': tmpfile,
         'origin': originaddr,
         'backend': objdump_backend_option[backend_name],
-    }, 'r')
-    result = g.readlines()
-    g.close()
-    lines = result[6:]   # drop some objdump cruft
+    }, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    stdout, stderr = p.communicate()
+    assert not p.returncode, ('Encountered an error running objdump: %s' %
+                              stderr)
+    # drop some objdump cruft
+    lines = stdout.splitlines()[6:]
     return format_code_dump_with_labels(originaddr, lines, label_list)
 
 def format_code_dump_with_labels(originaddr, lines, label_list):
@@ -85,8 +92,12 @@
     #
     print 'loading symbols from %s...' % (filename,)
     symbols = {}
-    g = os.popen(symbollister % filename, "r")
-    for line in g:
+    p = subprocess.Popen(symbollister % filename, shell=True,
+                         stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    stdout, stderr = p.communicate()
+    assert not p.returncode, ('Encountered an error running nm: %s' %
+                              stderr)
+    for line in stdout.splitlines():
         match = re_symbolentry.match(line)
         if match:
             addr = long(match.group(1), 16)
@@ -94,7 +105,6 @@
             if name.startswith('pypy_g_'):
                 name = '\xb7' + name[7:]
             symbols[addr] = name
-    g.close()
     print '%d symbols found' % (len(symbols),)
     return symbols
 
diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -455,6 +455,23 @@
             # the special return value None forces op.result to be considered
             # equal to op.args[0]
             return [op0, op1, None]
+        if (hints.get('promote_string') and
+            op.args[0].concretetype is not lltype.Void):
+            S = lltype.Ptr(rstr.STR)
+            assert op.args[0].concretetype == S
+            self._register_extra_helper(EffectInfo.OS_STREQ_NONNULL,
+                                        "str.eq_nonnull",
+                                        [S, S],
+                                        lltype.Signed,
+                                        EffectInfo.EF_ELIDABLE_CANNOT_RAISE)
+            descr, p = self.callcontrol.callinfocollection.callinfo_for_oopspec(
+                EffectInfo.OS_STREQ_NONNULL)
+            # XXX this is fairly ugly way of creating a constant,
+            #     however, callinfocollection has no better interface
+            c = Constant(p.adr.ptr, lltype.typeOf(p.adr.ptr))
+            op1 = SpaceOperation('str_guard_value', [op.args[0], c, descr],
+                                 op.result)
+            return [SpaceOperation('-live-', [], None), op1, None]
         else:
             log.WARNING('ignoring hint %r at %r' % (hints, self.graph))
 
@@ -783,8 +800,7 @@
 
     def rewrite_op_cast_ptr_to_int(self, op):
         if self._is_gc(op.args[0]):
-            #return op
-            raise NotImplementedError("cast_ptr_to_int")
+            return op
 
     def rewrite_op_force_cast(self, op):
         v_arg = op.args[0]
@@ -1526,6 +1542,10 @@
     def rewrite_op_jit_force_virtual(self, op):
         return self._do_builtin_call(op)
 
+    def rewrite_op_jit_is_virtual(self, op):
+        raise Exception, (
+            "'vref.virtual' should not be used from jit-visible code")
+
     def rewrite_op_jit_force_virtualizable(self, op):
         # this one is for virtualizables
         vinfo = self.get_vinfo(op.args[0])
diff --git a/pypy/jit/codewriter/test/test_jtransform.py b/pypy/jit/codewriter/test/test_jtransform.py
--- a/pypy/jit/codewriter/test/test_jtransform.py
+++ b/pypy/jit/codewriter/test/test_jtransform.py
@@ -1,5 +1,19 @@
 import py
 import random
+try:
+    from itertools import product
+except ImportError:
+    # Python 2.5, this is taken from the CPython docs, but simplified.
+    def product(*args):
+        # product('ABCD', 'xy') --> Ax Ay Bx By Cx Cy Dx Dy
+        # product(range(2), repeat=3) --> 000 001 010 011 100 101 110 111
+        pools = map(tuple, args)
+        result = [[]]
+        for pool in pools:
+            result = [x+[y] for x in result for y in pool]
+        for prod in result:
+            yield tuple(prod)
+
 from pypy.objspace.flow.model import FunctionGraph, Block, Link
 from pypy.objspace.flow.model import SpaceOperation, Variable, Constant
 from pypy.jit.codewriter.jtransform import Transformer
@@ -85,6 +99,12 @@
             if i == oopspecindex:
                 return True
         return False
+    def callinfo_for_oopspec(self, oopspecindex):
+        assert oopspecindex == effectinfo.EffectInfo.OS_STREQ_NONNULL
+        class c:
+            class adr:
+                ptr = 1
+        return ('calldescr', c)
 
 class FakeBuiltinCallControl:
     def __init__(self):
@@ -105,6 +125,7 @@
              EI.OS_STR2UNICODE:([PSTR], PUNICODE),
              EI.OS_STR_CONCAT: ([PSTR, PSTR], PSTR),
              EI.OS_STR_SLICE:  ([PSTR, INT, INT], PSTR),
+             EI.OS_STREQ_NONNULL:  ([PSTR, PSTR], INT),
              EI.OS_UNI_CONCAT: ([PUNICODE, PUNICODE], PUNICODE),
              EI.OS_UNI_SLICE:  ([PUNICODE, INT, INT], PUNICODE),
              EI.OS_UNI_EQUAL:  ([PUNICODE, PUNICODE], lltype.Bool),
@@ -254,26 +275,35 @@
             assert op1.result is None
 
 def test_calls():
-    for RESTYPE in [lltype.Signed, rclass.OBJECTPTR,
-                    lltype.Float, lltype.Void]:
-      for with_void in [False, True]:
-        for with_i in [False, True]:
-          for with_r in [False, True]:
-            for with_f in [False, True]:
-              ARGS = []
-              if with_void: ARGS += [lltype.Void, lltype.Void]
-              if with_i: ARGS += [lltype.Signed, lltype.Char]
-              if with_r: ARGS += [rclass.OBJECTPTR, lltype.Ptr(rstr.STR)]
-              if with_f: ARGS += [lltype.Float, lltype.Float]
-              random.shuffle(ARGS)
-              if RESTYPE == lltype.Float: with_f = True
-              if with_f: expectedkind = 'irf'   # all kinds
-              elif with_i: expectedkind = 'ir'  # integers and references
-              else: expectedkind = 'r'          # only references
-              yield residual_call_test, ARGS, RESTYPE, expectedkind
-              yield direct_call_test, ARGS, RESTYPE, expectedkind
-              yield indirect_residual_call_test, ARGS, RESTYPE, expectedkind
-              yield indirect_regular_call_test, ARGS, RESTYPE, expectedkind
+    for RESTYPE, with_void, with_i, with_r, with_f in product(
+        [lltype.Signed, rclass.OBJECTPTR, lltype.Float, lltype.Void],
+        [False, True],
+        [False, True],
+        [False, True],
+        [False, True],
+    ):
+        ARGS = []
+        if with_void:
+            ARGS += [lltype.Void, lltype.Void]
+        if with_i:
+            ARGS += [lltype.Signed, lltype.Char]
+        if with_r:
+            ARGS += [rclass.OBJECTPTR, lltype.Ptr(rstr.STR)]
+        if with_f:
+            ARGS += [lltype.Float, lltype.Float]
+        random.shuffle(ARGS)
+        if RESTYPE == lltype.Float:
+            with_f = True
+        if with_f:
+            expectedkind = 'irf'   # all kinds
+        elif with_i:
+            expectedkind = 'ir'  # integers and references
+        else:
+            expectedkind = 'r'          # only references
+        yield residual_call_test, ARGS, RESTYPE, expectedkind
+        yield direct_call_test, ARGS, RESTYPE, expectedkind
+        yield indirect_residual_call_test, ARGS, RESTYPE, expectedkind
+        yield indirect_regular_call_test, ARGS, RESTYPE, expectedkind
 
 def get_direct_call_op(argtypes, restype):
     FUNC = lltype.FuncType(argtypes, restype)
@@ -821,6 +851,21 @@
     assert op1.args[2] == ListOfKind('ref', [v1, v2])
     assert op1.result == v3
 
+def test_str_promote():
+    PSTR = lltype.Ptr(rstr.STR)
+    v1 = varoftype(PSTR)
+    v2 = varoftype(PSTR)
+    op = SpaceOperation('hint',
+                        [v1, Constant({'promote_string': True}, lltype.Void)],
+                        v2)
+    tr = Transformer(FakeCPU(), FakeBuiltinCallControl())
+    op0, op1, _ = tr.rewrite_operation(op)
+    assert op1.opname == 'str_guard_value'
+    assert op1.args[0] == v1
+    assert op1.args[2] == 'calldescr'
+    assert op1.result == v2
+    assert op0.opname == '-live-'
+
 def test_unicode_concat():
     # test that the oopspec is present and correctly transformed
     PSTR = lltype.Ptr(rstr.UNICODE)
diff --git a/pypy/jit/metainterp/blackhole.py b/pypy/jit/metainterp/blackhole.py
--- a/pypy/jit/metainterp/blackhole.py
+++ b/pypy/jit/metainterp/blackhole.py
@@ -2,7 +2,7 @@
 from pypy.rlib.rtimer import read_timestamp
 from pypy.rlib.rarithmetic import intmask, LONG_BIT, r_uint, ovfcheck
 from pypy.rlib.objectmodel import we_are_translated
-from pypy.rlib.debug import debug_start, debug_stop
+from pypy.rlib.debug import debug_start, debug_stop, ll_assert
 from pypy.rlib.debug import make_sure_not_resized
 from pypy.rpython.lltypesystem import lltype, llmemory, rclass
 from pypy.rpython.lltypesystem.lloperation import llop
@@ -503,6 +503,15 @@
     @arguments("r", returns="r")
     def bhimpl_cast_opaque_ptr(a):
         return a
+    @arguments("r", returns="i")
+    def bhimpl_cast_ptr_to_int(a):
+        i = lltype.cast_ptr_to_int(a)
+        ll_assert((i & 1) == 1, "bhimpl_cast_ptr_to_int: not an odd int")
+        return i
+    @arguments("i", returns="r")
+    def bhimpl_cast_int_to_ptr(i):
+        ll_assert((i & 1) == 1, "bhimpl_cast_int_to_ptr: not an odd int")
+        return lltype.cast_int_to_ptr(llmemory.GCREF, i)
 
     @arguments("i", returns="i")
     def bhimpl_int_copy(a):
@@ -523,6 +532,9 @@
     @arguments("f")
     def bhimpl_float_guard_value(a):
         pass
+    @arguments("r", "i", "d", returns="r")
+    def bhimpl_str_guard_value(a, i, d):
+        return a
 
     @arguments("self", "i")
     def bhimpl_int_push(self, a):
diff --git a/pypy/jit/metainterp/heapcache.py b/pypy/jit/metainterp/heapcache.py
--- a/pypy/jit/metainterp/heapcache.py
+++ b/pypy/jit/metainterp/heapcache.py
@@ -54,9 +54,10 @@
         if box in self.new_boxes:
             self.new_boxes[box] = False
         if box in self.dependencies:
-            for dep in self.dependencies[box]:
+            deps = self.dependencies[box]
+            del self.dependencies[box]
+            for dep in deps:
                 self._escape(dep)
-            del self.dependencies[box]
 
     def clear_caches(self, opnum, descr, argboxes):
         if opnum == rop.SETFIELD_GC:
diff --git a/pypy/jit/metainterp/history.py b/pypy/jit/metainterp/history.py
--- a/pypy/jit/metainterp/history.py
+++ b/pypy/jit/metainterp/history.py
@@ -9,6 +9,7 @@
 
 from pypy.jit.metainterp.resoperation import ResOperation, rop
 from pypy.jit.codewriter import heaptracker, longlong
+from pypy.rlib.objectmodel import compute_identity_hash
 
 # ____________________________________________________________
 
@@ -104,7 +105,7 @@
     getref._annspecialcase_ = 'specialize:arg(1)'
 
     def _get_hash_(self):
-        raise NotImplementedError
+        return compute_identity_hash(self)
 
     def clonebox(self):
         raise NotImplementedError
@@ -133,6 +134,9 @@
     def _get_str(self):
         raise NotImplementedError
 
+    def same_box(self, other):
+        return self is other
+
 class AbstractDescr(AbstractValue):
     __slots__ = ()
 
@@ -241,32 +245,15 @@
     def constbox(self):
         return self
 
+    def same_box(self, other):
+        return self.same_constant(other)
+
     def same_constant(self, other):
         raise NotImplementedError
 
     def __repr__(self):
         return 'Const(%s)' % self._getrepr_()
 
-    def __eq__(self, other):
-        "NOT_RPYTHON"
-        # Remember that you should not compare Consts with '==' in RPython.
-        # Consts have no special __hash__, in order to force different Consts
-        # from being considered as different keys when stored in dicts
-        # (as they always are after translation).  Use a dict_equal_consts()
-        # to get the other behavior (i.e. using this __eq__).
-        if self.__class__ is not other.__class__:
-            return False
-        try:
-            return self.value == other.value
-        except TypeError:
-            if (isinstance(self.value, Symbolic) and
-                isinstance(other.value, Symbolic)):
-                return self.value is other.value
-            raise
-
-    def __ne__(self, other):
-        return not (self == other)
-
 
 class ConstInt(Const):
     type = INT
@@ -688,33 +675,6 @@
 
 # ____________________________________________________________
 
-def dict_equal_consts():
-    "NOT_RPYTHON"
-    # Returns a dict in which Consts that compare as equal
-    # are identified when used as keys.
-    return r_dict(dc_eq, dc_hash)
-
-def dc_eq(c1, c2):
-    return c1 == c2
-
-def dc_hash(c):
-    "NOT_RPYTHON"
-    # This is called during translation only.  Avoid using identityhash(),
-    # to avoid forcing a hash, at least on lltype objects.
-    if not isinstance(c, Const):
-        return hash(c)
-    if isinstance(c.value, Symbolic):
-        return id(c.value)
-    try:
-        if isinstance(c, ConstPtr):
-            p = lltype.normalizeptr(c.value)
-            if p is not None:
-                return hash(p._obj)
-            else:
-                return 0
-        return c._get_hash_()
-    except lltype.DelayedPointer:
-        return -2      # xxx risk of changing hash...
 
 def make_hashable_int(i):
     from pypy.rpython.lltypesystem.ll2ctypes import NotCtypesAllocatedStructure
diff --git a/pypy/jit/metainterp/optimizeopt/rewrite.py b/pypy/jit/metainterp/optimizeopt/rewrite.py
--- a/pypy/jit/metainterp/optimizeopt/rewrite.py
+++ b/pypy/jit/metainterp/optimizeopt/rewrite.py
@@ -49,32 +49,35 @@
     def find_rewritable_bool(self, op, args):
         try:
             oldopnum = opboolinvers[op.getopnum()]
+        except KeyError:
+            pass
+        else:
             targs = self.optimizer.make_args_key(ResOperation(oldopnum, [args[0], args[1]],
                                                               None))
             if self.try_boolinvers(op, targs):
                 return True
-        except KeyError:
-            pass
 
         try:
             oldopnum = opboolreflex[op.getopnum()] # FIXME: add INT_ADD, INT_MUL
+        except KeyError:
+            pass
+        else:
             targs = self.optimizer.make_args_key(ResOperation(oldopnum, [args[1], args[0]],
                                                               None))
             oldop = self.get_pure_result(targs)
             if oldop is not None and oldop.getdescr() is op.getdescr():
                 self.make_equal_to(op.result, self.getvalue(oldop.result))
                 return True
-        except KeyError:
-            pass
 
         try:
             oldopnum = opboolinvers[opboolreflex[op.getopnum()]]
+        except KeyError:
+            pass
+        else:
             targs = self.optimizer.make_args_key(ResOperation(oldopnum, [args[1], args[0]],
                                                               None))
             if self.try_boolinvers(op, targs):
                 return True
-        except KeyError:
-            pass
 
         return False
 
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -2329,7 +2329,7 @@
         def _variables_equal(box, varname, strict):
             if varname not in virtuals:
                 if strict:
-                    assert box == oparse.getvar(varname)
+                    assert box.same_box(oparse.getvar(varname))
                 else:
                     assert box.value == oparse.getvar(varname).value
             else:
diff --git a/pypy/jit/metainterp/optimizeopt/util.py b/pypy/jit/metainterp/optimizeopt/util.py
--- a/pypy/jit/metainterp/optimizeopt/util.py
+++ b/pypy/jit/metainterp/optimizeopt/util.py
@@ -90,14 +90,11 @@
     for i in range(len(args1)):
         arg1 = args1[i]
         arg2 = args2[i]
-        if isinstance(arg1, history.Const):
-            if arg1.__class__ is not arg2.__class__:
+        if arg1 is None:
+            if arg2 is not None:
                 return False
-            if not arg1.same_constant(arg2):
-                return False
-        else:
-            if not arg1 is arg2:
-                return False
+        elif not arg1.same_box(arg2):
+            return False
     return True
 
 def args_hash(args):
@@ -106,10 +103,8 @@
     for arg in args:
         if arg is None:
             y = 17
-        elif isinstance(arg, history.Const):
+        else:
             y = arg._get_hash_()
-        else:
-            y = compute_identity_hash(arg)
         res = intmask((1000003 * res) ^ y)
     return res
 
@@ -145,9 +140,12 @@
         for i in range(op1.numargs()):
             x = op1.getarg(i)
             y = op2.getarg(i)
-            assert x == remap.get(y, y)
+            assert x.same_box(remap.get(y, y))
         if op2.result in remap:
-            assert op1.result == remap[op2.result]
+            if op2.result is None:
+                assert op1.result == remap[op2.result]
+            else:
+                assert op1.result.same_box(remap[op2.result])
         else:
             remap[op2.result] = op1.result
         if op1.getopnum() != rop.JUMP:      # xxx obscure
@@ -156,11 +154,20 @@
             assert len(op1.getfailargs()) == len(op2.getfailargs())
             if strict_fail_args:
                 for x, y in zip(op1.getfailargs(), op2.getfailargs()):
-                    assert x == remap.get(y, y)
+                    if x is None:
+                        assert remap.get(y, y) is None
+                    else:
+                        assert x.same_box(remap.get(y, y))
             else:
                 fail_args1 = set(op1.getfailargs())
                 fail_args2 = set([remap.get(y, y) for y in op2.getfailargs()])
-                assert fail_args1 == fail_args2
+                for x in fail_args1:
+                    for y in fail_args2:
+                        if x.same_box(y):
+                            fail_args2.remove(y)
+                            break
+                    else:
+                        assert False
     assert len(oplist1) == len(oplist2)
     print '-'*totwidth
     return True
diff --git a/pypy/jit/metainterp/optimizeopt/vstring.py b/pypy/jit/metainterp/optimizeopt/vstring.py
--- a/pypy/jit/metainterp/optimizeopt/vstring.py
+++ b/pypy/jit/metainterp/optimizeopt/vstring.py
@@ -587,10 +587,7 @@
             return True
         #
         if v1.is_nonnull() and v2.is_nonnull():
-            if l1box is not None and l2box is not None and (
-                l1box == l2box or (isinstance(l1box, ConstInt) and
-                                   isinstance(l2box, ConstInt) and
-                                   l1box.value == l2box.value)):
+            if l1box is not None and l2box is not None and l1box.same_box(l2box):
                 do = EffectInfo.OS_STREQ_LENGTHOK
             else:
                 do = EffectInfo.OS_STREQ_NONNULL
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -162,15 +162,18 @@
             if registers[i] is oldbox:
                 registers[i] = newbox
         if not we_are_translated():
-            assert oldbox not in registers[count:]
+            for b in registers[count:]:
+                assert not oldbox.same_box(b)
+                
 
     def make_result_of_lastop(self, resultbox):
         got_type = resultbox.type
-        if not we_are_translated():
-            typeof = {'i': history.INT,
-                      'r': history.REF,
-                      'f': history.FLOAT}
-            assert typeof[self.jitcode._resulttypes[self.pc]] == got_type
+        # XXX disabled for now, conflicts with str_guard_value
+        #if not we_are_translated():
+        #    typeof = {'i': history.INT,
+        #              'r': history.REF,
+        #              'f': history.FLOAT}
+        #    assert typeof[self.jitcode._resulttypes[self.pc]] == got_type
         target_index = ord(self.bytecode[self.pc-1])
         if got_type == history.INT:
             self.registers_i[target_index] = resultbox
@@ -219,6 +222,7 @@
                     'cast_float_to_int', 'cast_int_to_float',
                     'cast_float_to_singlefloat', 'cast_singlefloat_to_float',
                     'float_neg', 'float_abs',
+                    'cast_ptr_to_int', 'cast_int_to_ptr',
                     ]:
         exec py.code.Source('''
             @arguments("box")
@@ -895,6 +899,21 @@
     def _opimpl_guard_value(self, orgpc, box):
         self.implement_guard_value(orgpc, box)
 
+    @arguments("orgpc", "box", "box", "descr")
+    def opimpl_str_guard_value(self, orgpc, box, funcbox, descr):
+        if isinstance(box, Const):
+            return box     # no promotion needed, already a Const
+        else:
+            constbox = box.constbox()
+            resbox = self.do_residual_call(funcbox, descr, [box, constbox])
+            promoted_box = resbox.constbox()
+            # This is GUARD_VALUE because GUARD_TRUE assumes the existance
+            # of a label when computing resumepc
+            self.generate_guard(rop.GUARD_VALUE, resbox, [promoted_box],
+                                resumepc=orgpc)
+            self.metainterp.replace_box(box, constbox)
+            return constbox
+
     opimpl_int_guard_value = _opimpl_guard_value
     opimpl_ref_guard_value = _opimpl_guard_value
     opimpl_float_guard_value = _opimpl_guard_value
diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py
--- a/pypy/jit/metainterp/resoperation.py
+++ b/pypy/jit/metainterp/resoperation.py
@@ -433,6 +433,8 @@
     'INT_INVERT/1',
     #
     'SAME_AS/1',      # gets a Const or a Box, turns it into another Box
+    'CAST_PTR_TO_INT/1',
+    'CAST_INT_TO_PTR/1',
     #
     'PTR_EQ/2b',
     'PTR_NE/2b',
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -13,7 +13,7 @@
 from pypy.rlib.jit import (JitDriver, we_are_jitted, hint, dont_look_inside,
     loop_invariant, elidable, promote, jit_debug, assert_green,
     AssertGreenFailed, unroll_safe, current_trace_length, look_inside_iff,
-    isconstant, isvirtual)
+    isconstant, isvirtual, promote_string)
 from pypy.rlib.rarithmetic import ovfcheck
 from pypy.rpython.lltypesystem import lltype, llmemory, rffi
 from pypy.rpython.ootypesystem import ootype
@@ -3440,7 +3440,6 @@
 
 class TestLLtype(BaseLLtypeTests, LLJitMixin):
     def test_tagged(self):
-        py.test.skip("implement me")
         from pypy.rlib.objectmodel import UnboxedValue
         class Base(object):
             __slots__ = ()
@@ -3492,3 +3491,35 @@
                 pc += 1
             return pc
         res = self.meta_interp(main, [False, 100, True], taggedpointers=True)
+
+    def test_rerased(self):
+        from pypy.rlib.rerased import erase_int, unerase_int, new_erasing_pair
+        eraseX, uneraseX = new_erasing_pair("X")
+        #
+        class X:
+            def __init__(self, a, b):
+                self.a = a
+                self.b = b
+        #
+        def f(i, j):
+            # 'j' should be 0 or 1, not other values
+            if j > 0:
+                e = eraseX(X(i, j))
+            else:
+                try:
+                    e = erase_int(i)
+                except OverflowError:
+                    return -42
+            if j & 1:
+                x = uneraseX(e)
+                return x.a - x.b
+            else:
+                return unerase_int(e)
+        #
+        x = self.interp_operations(f, [-128, 0], taggedpointers=True)
+        assert x == -128
+        bigint = sys.maxint//2 + 1
+        x = self.interp_operations(f, [bigint, 0], taggedpointers=True)
+        assert x == -42
+        x = self.interp_operations(f, [1000, 1], taggedpointers=True)
+        assert x == 999
diff --git a/pypy/jit/metainterp/test/test_heapcache.py b/pypy/jit/metainterp/test/test_heapcache.py
--- a/pypy/jit/metainterp/test/test_heapcache.py
+++ b/pypy/jit/metainterp/test/test_heapcache.py
@@ -355,6 +355,14 @@
         assert not h.is_unescaped(box1)
         assert not h.is_unescaped(box2)
 
+    def test_circular_virtuals(self):
+        h = HeapCache()
+        h.new(box1)
+        h.new(box2)
+        h.invalidate_caches(rop.SETFIELD_GC, None, [box1, box2])
+        h.invalidate_caches(rop.SETFIELD_GC, None, [box2, box1])
+        h.invalidate_caches(rop.SETFIELD_GC, None, [box3, box1]) # does not crash
+
     def test_unescaped_array(self):
         h = HeapCache()
         h.new_array(box1, lengthbox1)
@@ -362,4 +370,4 @@
         h.invalidate_caches(rop.SETARRAYITEM_GC, None, [box1, index1, box2])
         assert h.is_unescaped(box1)
         h.invalidate_caches(rop.SETARRAYITEM_GC, None, [box2, index1, box1])
-        assert not h.is_unescaped(box1)
\ No newline at end of file
+        assert not h.is_unescaped(box1)
diff --git a/pypy/jit/metainterp/test/test_resume.py b/pypy/jit/metainterp/test/test_resume.py
--- a/pypy/jit/metainterp/test/test_resume.py
+++ b/pypy/jit/metainterp/test/test_resume.py
@@ -1,3 +1,4 @@
+from __future__ import with_statement
 import py
 from pypy.rpython.lltypesystem import lltype, llmemory, rffi
 from pypy.jit.metainterp.optimizeopt.optimizer import OptValue
@@ -180,22 +181,27 @@
     reader.consume_boxes(info, bi, br, bf)
     b1s = reader.liveboxes[0]
     b2s = reader.liveboxes[1]
-    assert bi == [b1s, ConstInt(111), b1s]
-    assert br == [ConstPtr(gcrefnull), b2s]
+    assert_same(bi, [b1s, ConstInt(111), b1s])
+    assert_same(br, [ConstPtr(gcrefnull), b2s])
     bi, br, bf = [None]*2, [None]*0, [None]*0
     info = MyBlackholeInterp([lltype.Signed,
                               lltype.Signed]).get_current_position_info()
     reader.consume_boxes(info, bi, br, bf)
-    assert bi == [ConstInt(222), ConstInt(333)]
+    assert_same(bi, [ConstInt(222), ConstInt(333)])
     bi, br, bf = [None]*2, [None]*1, [None]*0
     info = MyBlackholeInterp([lltype.Signed, llmemory.GCREF,
                               lltype.Signed]).get_current_position_info()
     reader.consume_boxes(info, bi, br, bf)
     b3s = reader.liveboxes[2]
-    assert bi == [b1s, b3s]
-    assert br == [b2s]
+    assert_same(bi, [b1s, b3s])
+    assert_same(br, [b2s])
     #
 
+def assert_same(list1, list2):
+    assert len(list1) == len(list2)
+    for b1, b2 in zip(list1, list2):
+        assert b1.same_box(b2)
+
 def test_simple_read_tagged_ints():
     storage = Storage()
     storage.rd_consts = []
@@ -1023,11 +1029,11 @@
     metainterp = MyMetaInterp()
     reader = ResumeDataFakeReader(storage, newboxes, metainterp)
     lst = reader.consume_boxes()
-    assert lst == [b1t, b1t, b3t]
+    assert_same(lst, [b1t, b1t, b3t])
     lst = reader.consume_boxes()
-    assert lst == [ConstInt(2), ConstInt(3)]
+    assert_same(lst, [ConstInt(2), ConstInt(3)])
     lst = reader.consume_boxes()
-    assert lst == [b1t, ConstInt(1), b1t, b1t]
+    assert_same(lst, [b1t, ConstInt(1), b1t, b1t])
     assert metainterp.trace == []    
 
 
@@ -1044,11 +1050,11 @@
     reader = ResumeDataFakeReader(storage, newboxes, metainterp)
     lst = reader.consume_boxes()
     c1t = ConstInt(111)
-    assert lst == [c1t, b2t, b3t]
+    assert_same(lst, [c1t, b2t, b3t])
     lst = reader.consume_boxes()
-    assert lst == [ConstInt(2), ConstInt(3)]
+    assert_same(lst, [ConstInt(2), ConstInt(3)])
     lst = reader.consume_boxes()
-    assert lst == [c1t, ConstInt(1), c1t, b2t]
+    assert_same(lst, [c1t, ConstInt(1), c1t, b2t])
     assert metainterp.trace == []
 
 
@@ -1114,9 +1120,11 @@
     # check that we get the operations in 'expected', in a possibly different
     # order.
     assert len(trace) == len(expected)
-    for x in trace:
-        assert x in expected
-        expected.remove(x)
+    with CompareableConsts():
+        for x in trace:
+            assert x in expected
+            expected.remove(x)
+
     ptr = b2t.value._obj.container._as_ptr()
     assert lltype.typeOf(ptr) == lltype.Ptr(LLtypeMixin.NODE)
     assert ptr.value == 111
@@ -1126,6 +1134,18 @@
     assert ptr2.parent.value == 33
     assert ptr2.parent.next == ptr
 
+class CompareableConsts(object):
+    def __init__(self):
+        self.oldeq = None
+        
+    def __enter__(self):
+        assert self.oldeq is None
+        self.oldeq = Const.__eq__
+        Const.__eq__ = Const.same_box
+        
+    def __exit__(self, type, value, traceback):
+        Const.__eq__ = self.oldeq
+
 def test_virtual_adder_make_varray():
     b2s, b4s = [BoxPtr(), BoxInt(4)]
     c1s = ConstInt(111)
@@ -1163,8 +1183,9 @@
         (rop.SETARRAYITEM_GC, [b2t,ConstInt(1), c1s], None,
                               LLtypeMixin.arraydescr),
         ]
-    for x, y in zip(expected, trace):
-        assert x == y
+    with CompareableConsts():
+        for x, y in zip(expected, trace):
+            assert x == y
     #
     ptr = b2t.value._obj.container._as_ptr()
     assert lltype.typeOf(ptr) == lltype.Ptr(lltype.GcArray(lltype.Signed))
@@ -1207,8 +1228,9 @@
         (rop.SETFIELD_GC, [b2t, c1s],  None, LLtypeMixin.adescr),
         (rop.SETFIELD_GC, [b2t, b4t], None, LLtypeMixin.bdescr),
         ]
-    for x, y in zip(expected, trace):
-        assert x == y
+    with CompareableConsts():
+        for x, y in zip(expected, trace):
+            assert x == y
     #
     ptr = b2t.value._obj.container._as_ptr()
     assert lltype.typeOf(ptr) == lltype.Ptr(LLtypeMixin.S)
diff --git a/pypy/jit/metainterp/test/test_string.py b/pypy/jit/metainterp/test/test_string.py
--- a/pypy/jit/metainterp/test/test_string.py
+++ b/pypy/jit/metainterp/test/test_string.py
@@ -3,7 +3,8 @@
 from pypy.jit.codewriter.policy import StopAtXPolicy
 from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin
 from pypy.rlib.debug import debug_print
-from pypy.rlib.jit import JitDriver, dont_look_inside, we_are_jitted
+from pypy.rlib.jit import JitDriver, dont_look_inside, we_are_jitted,\
+     promote_string
 from pypy.rlib.rstring import StringBuilder
 from pypy.rpython.ootypesystem import ootype
 
@@ -507,6 +508,18 @@
                           'jump': 1, 'int_is_true': 1,
                           'guard_not_invalidated': 1})
 
+    def test_promote_string(self):
+        driver = JitDriver(greens = [], reds = ['n'])
+        
+        def f(n):
+            while n < 21:
+                driver.jit_merge_point(n=n)
+                promote_string(str(n % 3))
+                n += 1
+            return 0
+
+        self.meta_interp(f, [0])
+        self.check_loops(call=3 + 1) # one for int2str
 
 #class TestOOtype(StringTests, OOJitMixin):
 #    CALL = "oosend"
diff --git a/pypy/jit/metainterp/test/test_virtualref.py b/pypy/jit/metainterp/test/test_virtualref.py
--- a/pypy/jit/metainterp/test/test_virtualref.py
+++ b/pypy/jit/metainterp/test/test_virtualref.py
@@ -3,6 +3,7 @@
 from pypy.rpython.llinterp import LLException
 from pypy.rlib.jit import JitDriver, dont_look_inside, vref_None
 from pypy.rlib.jit import virtual_ref, virtual_ref_finish, InvalidVirtualRef
+from pypy.rlib.jit import non_virtual_ref
 from pypy.rlib.objectmodel import compute_unique_id
 from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin, _get_jitcodes
 from pypy.jit.metainterp.resoperation import rop
@@ -595,6 +596,65 @@
         res = self.meta_interp(fn, [10])
         assert res == 6
 
+    def test_is_virtual(self):
+        myjitdriver = JitDriver(greens=[], reds=['n', 'res1'])
+        class X:
+            pass
+        @dont_look_inside
+        def residual(vref):
+            return vref.virtual
+        #
+        def f(n):
+            res1 = -42
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, res1=res1)
+                x = X()
+                vref = virtual_ref(x)
+                res1 = residual(vref)
+                virtual_ref_finish(vref, x)
+                n -= 1
+            return res1
+        #
+        res = self.meta_interp(f, [10])
+        assert res == 1
+
+    def test_is_not_virtual_none(self):
+        myjitdriver = JitDriver(greens=[], reds=['n', 'res1'])
+        @dont_look_inside
+        def residual(vref):
+            return vref.virtual
+        #
+        def f(n):
+            res1 = -42
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, res1=res1)
+                res1 = residual(vref_None)
+                n -= 1
+            return res1
+        #
+        res = self.meta_interp(f, [10])
+        assert res == 0
+
+    def test_is_not_virtual_non_none(self):
+        myjitdriver = JitDriver(greens=[], reds=['n', 'res1'])
+        class X:
+            pass
+        @dont_look_inside
+        def residual(vref):
+            return vref.virtual
+        #
+        def f(n):
+            res1 = -42
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, res1=res1)
+                x = X()
+                res1 = residual(non_virtual_ref(x))
+                n -= 1
+            return res1
+        #
+        res = self.meta_interp(f, [10])
+        assert res == 0
+
 
 class TestLLtype(VRefTests, LLJitMixin):
     pass
diff --git a/pypy/jit/metainterp/virtualref.py b/pypy/jit/metainterp/virtualref.py
--- a/pypy/jit/metainterp/virtualref.py
+++ b/pypy/jit/metainterp/virtualref.py
@@ -39,6 +39,7 @@
     def replace_force_virtual_with_call(self, graphs):
         # similar to rvirtualizable2.replace_force_virtualizable_with_call().
         c_force_virtual_ptr = None
+        c_is_virtual_ptr = None
         force_virtual_count = 0
         for graph in graphs:
             for block in graph.iterblocks():
@@ -52,6 +53,13 @@
                         op.opname = 'direct_call'
                         op.args = [c_force_virtual_ptr, op.args[0]]
                         force_virtual_count += 1
+                    #
+                    if op.opname == 'jit_is_virtual':
+                        if c_is_virtual_ptr is None:
+                            c_is_virtual_ptr = self.get_is_virtual_fnptr()
+                        #
+                        op.opname = 'direct_call'
+                        op.args = [c_is_virtual_ptr, op.args[0]]
         #
         if c_force_virtual_ptr is not None:
             log("replaced %d 'jit_force_virtual' with %r" % (force_virtual_count,
@@ -129,6 +137,17 @@
             force_virtual_if_necessary)
         return inputconst(lltype.typeOf(funcptr), funcptr)
 
+    def get_is_virtual_fnptr(self):
+        #
+        def is_virtual(inst):
+            if not inst:
+                return False
+            return inst.typeptr == self.jit_virtual_ref_vtable
+        #
+        FUNC = lltype.FuncType([rclass.OBJECTPTR], lltype.Bool)
+        funcptr = self.warmrunnerdesc.helper_func(lltype.Ptr(FUNC), is_virtual)
+        return inputconst(lltype.typeOf(funcptr), funcptr)
+
     def force_virtual(self, inst):
         vref = lltype.cast_pointer(lltype.Ptr(self.JIT_VIRTUAL_REF), inst)
         token = vref.virtual_token
diff --git a/pypy/module/__builtin__/__init__.py b/pypy/module/__builtin__/__init__.py
--- a/pypy/module/__builtin__/__init__.py
+++ b/pypy/module/__builtin__/__init__.py
@@ -20,6 +20,9 @@
         'any'           : 'app_functional.any',
         'all'           : 'app_functional.all',
         'sum'           : 'app_functional.sum',
+        'map'           : 'app_functional.map',
+        'reduce'        : 'app_functional.reduce',
+        'filter'        : 'app_functional.filter',
         'vars'          : 'app_inspect.vars',
         'dir'           : 'app_inspect.dir',
 
@@ -86,11 +89,8 @@
         'enumerate'     : 'functional.W_Enumerate',
         'min'           : 'functional.min',
         'max'           : 'functional.max',
-        'map'           : 'functional.map',
         'zip'           : 'functional.zip',
-        'reduce'        : 'functional.reduce',
         'reversed'      : 'functional.reversed',
-        'filter'        : 'functional.filter',
         'super'         : 'descriptor.W_Super',
         'staticmethod'  : 'descriptor.StaticMethod',
         'classmethod'   : 'descriptor.ClassMethod',
@@ -119,7 +119,7 @@
            builtin = space.interpclass_w(w_builtin)
            if isinstance(builtin, module.Module):
                return builtin
-       # no builtin! make a default one.  Given them None, at least.
+       # no builtin! make a default one.  Give them None, at least.
        builtin = module.Module(space, None)
        space.setitem(builtin.w_dict, space.wrap('None'), space.w_None)
        return builtin
diff --git a/pypy/module/__builtin__/app_functional.py b/pypy/module/__builtin__/app_functional.py
--- a/pypy/module/__builtin__/app_functional.py
+++ b/pypy/module/__builtin__/app_functional.py
@@ -48,4 +48,118 @@
         # Very intentionally *not* +=, that would have different semantics if
         # start was a mutable type, such as a list
         last = last + x
-    return last
\ No newline at end of file
+    return last
+
+def map(func, *collections):
+    """map(function, sequence[, sequence, ...]) -> list
+
+Return a list of the results of applying the function to the items of
+the argument sequence(s).  If more than one sequence is given, the
+function is called with an argument list consisting of the corresponding
+item of each sequence, substituting None for missing values when not all
+sequences have the same length.  If the function is None, return a list of
+the items of the sequence (or a list of tuples if more than one sequence)."""
+    if not collections:
+        raise TypeError("map() requires at least two arguments")
+    num_collections = len(collections)
+    none_func = func is None
+    if num_collections == 1:
+        if none_func:
+            return list(collections[0])
+        else:
+            # Special case for the really common case of a single collection,
+            # this can be eliminated if we could unroll that loop that creates
+            # `args` based on whether or not len(collections) was constant
+            result = []
+            for item in collections[0]:
+                result.append(func(item))
+            return result
+    result = []
+    # Pair of (iterator, has_finished)
+    iterators = [(iter(seq), False) for seq in collections]
+    while True:
+        cont = False
+        args = []
+        for idx, (iterator, has_finished) in enumerate(iterators):
+            val = None
+            if not has_finished:
+                try:
+                    val = next(iterator)
+                except StopIteration:
+                    iterators[idx] = (None, True)
+                else:
+                    cont = True
+            args.append(val)
+        args = tuple(args)
+        if cont:
+            if none_func:
+                result.append(args)
+            else:
+                result.append(func(*args))
+        else:
+            return result
+
+sentinel = object()
+
+def reduce(func, sequence, initial=sentinel):
+    """reduce(function, sequence[, initial]) -> value
+
+Apply a function of two arguments cumulatively to the items of a sequence,
+from left to right, so as to reduce the sequence to a single value.
+For example, reduce(lambda x, y: x+y, [1, 2, 3, 4, 5]) calculates
+((((1+2)+3)+4)+5).  If initial is present, it is placed before the items
+of the sequence in the calculation, and serves as a default when the
+sequence is empty."""
+    iterator = iter(sequence)
+    if initial is sentinel:
+        try:
+            initial = next(iterator)
+        except StopIteration:
+            raise TypeError("reduce() of empty sequence with no initial value")
+    result = initial
+    for item in iterator:
+        result = func(result, item)
+    return result
+
+def filter(func, seq):
+    """filter(function or None, sequence) -> list, tuple, or string
+
+Return those items of sequence for which function(item) is true.  If
+function is None, return the items that are true.  If sequence is a tuple
+or string, return the same type, else return a list."""
+    if func is None:
+        func = bool
+    if isinstance(seq, str):
+        return _filter_string(func, seq, str)
+    elif isinstance(seq, unicode):
+        return _filter_string(func, seq, unicode)
+    elif isinstance(seq, tuple):
+        return _filter_tuple(func, seq)
+    result = []
+    for item in seq:
+        if func(item):
+            result.append(item)
+    return result
+
+def _filter_string(func, string, str_type):
+    if func is bool and type(string) is str_type:
+        return string
+    result = []
+    for i in range(len(string)):
+        # You must call __getitem__ on the strings, simply iterating doesn't
+        # work :/
+        item = string[i]
+        if func(item):
+            if not isinstance(item, str_type):
+                raise TypeError("__getitem__ returned a non-string type")
+            result.append(item)
+    return str_type().join(result)
+
+def _filter_tuple(func, seq):
+    result = []
+    for i in range(len(seq)):
+        # Again, must call __getitem__, at least there are tests.
+        item = seq[i]
+        if func(item):
+            result.append(item)
+    return tuple(result)
\ No newline at end of file
diff --git a/pypy/module/__builtin__/compiling.py b/pypy/module/__builtin__/compiling.py
--- a/pypy/module/__builtin__/compiling.py
+++ b/pypy/module/__builtin__/compiling.py
@@ -84,8 +84,8 @@
         raise OperationError(space.w_TypeError,
               w('eval() arg 1 must be a string or code object'))
 
-    caller = space.getexecutioncontext().gettopframe_nohidden()
     if space.is_w(w_globals, space.w_None):
+        caller = space.getexecutioncontext().gettopframe_nohidden()
         if caller is None:
             w_globals = space.newdict()
             if space.is_w(w_locals, space.w_None):
@@ -97,13 +97,9 @@
     elif space.is_w(w_locals, space.w_None):
         w_locals = w_globals
 
-    try:
-        space.getitem(w_globals, space.wrap('__builtins__'))
-    except OperationError, e:
-        if not e.match(space, space.w_KeyError):
-            raise
-        if caller is not None:
-            w_builtin = space.builtin.pick_builtin(caller.w_globals)
-            space.setitem(w_globals, space.wrap('__builtins__'), w_builtin)
+    # xxx removed: adding '__builtins__' to the w_globals dict, if there
+    # is none.  This logic was removed as costly (it requires to get at
+    # the gettopframe_nohidden()).  I bet no test fails, and it's a really
+    # obscure case.
 
     return codeobj.exec_code(space, w_globals, w_locals)
diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py
--- a/pypy/module/__builtin__/functional.py
+++ b/pypy/module/__builtin__/functional.py
@@ -193,124 +193,14 @@
     return min_max(space, __args__, "max")
 
 def min(space, __args__):
-    """Return the smallest item in a sequence.
+    """min(iterable[, key=func]) -> value
+    min(a, b, c, ...[, key=func]) -> value
 
-    If more than one argument is passed, return the minimum of them.
+    With a single iterable argument, return its smallest item.
+    With two or more arguments, return the smallest argument.
     """
     return min_max(space, __args__, "min")
 
- at unwrap_spec(collections_w="args_w")
-def map(space, w_func, collections_w):
-    """does 3 separate things, hence this enormous docstring.
-       1.  if function is None, return a list of tuples, each with one
-           item from each collection.  If the collections have different
-           lengths,  shorter ones are padded with None.
-
-       2.  if function is not None, and there is only one collection,
-           apply function to every item in the collection and return a
-           list of the results.
-
-       3.  if function is not None, and there are several collections,
-           repeatedly call the function with one argument from each
-           collection.  If the collections have different lengths,
-           shorter ones are padded with None
-    """
-    if not collections_w:
-        msg = "map() requires at least two arguments"
-        raise OperationError(space.w_TypeError, space.wrap(msg))
-    none_func = space.is_w(w_func, space.w_None)
-    if len(collections_w) == 1:
-        w_collection = collections_w[0]
-        if none_func:
-            result_w = space.unpackiterable(w_collection)
-        else:
-            result_w = map_single_collection(space, w_func, w_collection)
-    else:
-        result_w = map_multiple_collections(space, w_func, collections_w,
-                                            none_func)
-    return space.newlist(result_w)
-
-def map_single_collection(space, w_func, w_collection):
-    """Special case for 'map(func, coll)', where 'func' is not None and there
-    is only one 'coll' argument."""
-    w_iter = space.iter(w_collection)
-    # xxx special hacks for speed
-    from pypy.interpreter import function, pycode
-    if isinstance(w_func, function.Function):
-        # xxx compatibility issue: what if func_code is modified in the
-        # middle of running map()??  That's far too obscure for me to care...
-        code = w_func.getcode()
-        fast_natural_arity = code.fast_natural_arity
-        if fast_natural_arity == (1|pycode.PyCode.FLATPYCALL):
-            assert isinstance(code, pycode.PyCode)
-            return map_single_user_function(code, w_func, w_iter)
-    # /xxx end of special hacks
-    return map_single_other_callable(space, w_func, w_iter)
-
-def map_single_other_callable(space, w_func, w_iter):
-    result_w = []
-    while True:
-        try:
-            w_item = space.next(w_iter)
-        except OperationError, e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            break
-        result_w.append(space.call_function(w_func, w_item))
-    return result_w
-map_single_other_callable._dont_inline_ = True
-
-from pypy.rlib.jit import JitDriver
-mapjitdriver = JitDriver(greens = ['code'],
-                         reds = ['w_func', 'w_iter', 'result_w'])
-def map_single_user_function(code, w_func, w_iter):
-    result_w = []
-    while True:
-        mapjitdriver.can_enter_jit(code=code, w_func=w_func,
-                                   w_iter=w_iter, result_w=result_w)
-        mapjitdriver.jit_merge_point(code=code, w_func=w_func,
-                                     w_iter=w_iter, result_w=result_w)
-        space = w_func.space
-        try:
-            w_item = space.next(w_iter)
-        except OperationError, e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            break
-        new_frame = space.createframe(code, w_func.w_func_globals,
-                                      w_func)
-        new_frame.locals_stack_w[0] = w_item
-        w_res = new_frame.run()
-        result_w.append(w_res)
-    return result_w
-
-def map_multiple_collections(space, w_func, collections_w, none_func):
-    result_w = []
-    iterators_w = [space.iter(w_seq) for w_seq in collections_w]
-    num_iterators = len(iterators_w)
-    while True:
-        cont = False
-        args_w = [space.w_None] * num_iterators
-        for i in range(num_iterators):
-            if iterators_w[i] is not None:
-                try:
-                    args_w[i] = space.next(iterators_w[i])
-                except OperationError, e:
-                    if not e.match(space, space.w_StopIteration):
-                        raise
-                    iterators_w[i] = None
-                else:
-                    cont = True
-        if not cont:
-            break
-        w_args = space.newtuple(args_w)
-        if none_func:
-            w_res = w_args
-        else:
-            w_res = space.call(w_func, w_args)
-        result_w.append(w_res)
-    return result_w
-
 @unwrap_spec(sequences_w="args_w")
 def zip(space, sequences_w):
     """Return a list of tuples, where the nth tuple contains every nth item of
@@ -332,90 +222,6 @@
             return space.newlist(result_w)
         result_w.append(space.newtuple(items_w))
 
-def reduce(space, w_func, w_sequence, w_initial=NoneNotWrapped):
-    """ Apply function of two arguments cumulatively to the items of sequence,
-        from left to right, so as to reduce the sequence to a single value.
-        Optionally begin with an initial value.
-    """
-    w_iter = space.iter(w_sequence)
-    if w_initial is None:
-        try:
-            w_initial = space.next(w_iter)
-        except OperationError, e:
-            if e.match(space, space.w_StopIteration):
-                msg = "reduce() of empty sequence with no initial value"
-                raise OperationError(space.w_TypeError, space.wrap(msg))
-            raise
-    w_result = w_initial
-    while True:
-        try:
-            w_next = space.next(w_iter)
-        except OperationError, e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            break
-        w_result = space.call_function(w_func, w_result, w_next)
-    return w_result
-
-def filter(space, w_func, w_seq):
-    """construct a list of those elements of collection for which function
-       is True.  If function is None, then return the items in the sequence
-       which are True.
-    """
-    if space.is_true(space.isinstance(w_seq, space.w_str)):
-        return _filter_string(space, w_func, w_seq, space.w_str)
-    if space.is_true(space.isinstance(w_seq, space.w_unicode)):
-        return _filter_string(space, w_func, w_seq, space.w_unicode)
-    if space.is_true(space.isinstance(w_seq, space.w_tuple)):
-        return _filter_tuple(space, w_func, w_seq)
-    w_iter = space.iter(w_seq)
-    result_w = []
-    none_func = space.is_w(w_func, space.w_None)
-    while True:
-        try:
-            w_next = space.next(w_iter)
-        except OperationError, e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            break
-        if none_func:
-            w_keep = w_next
-        else:
-            w_keep = space.call_function(w_func, w_next)
-        if space.is_true(w_keep):
-            result_w.append(w_next)
-    return space.newlist(result_w)
-
-def _filter_tuple(space, w_func, w_tuple):
-    none_func = space.is_w(w_func, space.w_None)
-    length = space.len_w(w_tuple)
-    result_w = []
-    for i in range(length):
-        w_item = space.getitem(w_tuple, space.wrap(i))
-        if none_func:
-            w_keep = w_item
-        else:
-            w_keep = space.call_function(w_func, w_item)
-        if space.is_true(w_keep):
-            result_w.append(w_item)
-    return space.newtuple(result_w)
-
-def _filter_string(space, w_func, w_string, w_str_type):
-    none_func = space.is_w(w_func, space.w_None)
-    if none_func and space.is_w(space.type(w_string), w_str_type):
-        return w_string
-    length = space.len_w(w_string)
-    result_w = []
-    for i in range(length):
-        w_item = space.getitem(w_string, space.wrap(i))
-        if none_func or space.is_true(space.call_function(w_func, w_item)):
-            if not space.is_true(space.isinstance(w_item, w_str_type)):
-                msg = "__getitem__ returned a non-string type"
-                raise OperationError(space.w_TypeError, space.wrap(msg))
-            result_w.append(w_item)
-    w_empty = space.call_function(w_str_type)
-    return space.call_method(w_empty, "join", space.newlist(result_w))
-
 class W_Enumerate(Wrappable):
 
     def __init__(self, w_iter, w_start):
diff --git a/pypy/module/__builtin__/test/test_functional.py b/pypy/module/__builtin__/test/test_functional.py
--- a/pypy/module/__builtin__/test/test_functional.py
+++ b/pypy/module/__builtin__/test/test_functional.py
@@ -147,7 +147,7 @@
        assert list(xrange(A())) == [0, 1, 2, 3, 4]
        assert list(xrange(0, A())) == [0, 1, 2, 3, 4]
        assert list(xrange(0, 10, A())) == [0, 5]
- 
+
    def test_xrange_float(self):
       assert list(xrange(0.1, 2.0, 1.1)) == [0, 1]
 
diff --git a/pypy/module/__pypy__/interp_builders.py b/pypy/module/__pypy__/interp_builders.py
--- a/pypy/module/__pypy__/interp_builders.py
+++ b/pypy/module/__pypy__/interp_builders.py
@@ -16,7 +16,8 @@
 
         def _check_done(self, space):
             if self.builder is None:
-                raise OperationError(space.w_ValueError, space.wrap("Can't operate on a done builder"))
+                raise OperationError(space.w_ValueError, space.wrap(
+                        "Can't operate on a built builder"))
 
         @unwrap_spec(size=int)
         def descr__new__(space, w_subtype, size=-1):
@@ -31,7 +32,8 @@
         def descr_append_slice(self, space, s, start, end):
             self._check_done(space)
             if not 0 <= start <= end <= len(s):
-                raise OperationError(space.w_ValueError, space.wrap("bad start/stop"))
+                raise OperationError(space.w_ValueError, space.wrap(
+                        "bad start/stop"))
             self.builder.append_slice(s, start, end)
 
         def descr_build(self, space):
@@ -40,6 +42,12 @@
             self.builder = None
             return w_s
 
+        def descr_len(self, space):
+            if self.builder is None:
+                raise OperationError(space.w_ValueError, space.wrap(
+                        "no length of built builder"))
+            return space.wrap(self.builder.getlength())
+
     W_Builder.__name__ = "W_%s" % name
     W_Builder.typedef = TypeDef(name,
         __new__ = interp2app(func_with_new_name(
@@ -48,6 +56,7 @@
         append = interp2app(W_Builder.descr_append),
         append_slice = interp2app(W_Builder.descr_append_slice),
         build = interp2app(W_Builder.descr_build),
+        __len__ = interp2app(W_Builder.descr_len),
     )
     W_Builder.typedef.acceptable_as_base_class = False
     return W_Builder
diff --git a/pypy/module/__pypy__/test/test_builders.py b/pypy/module/__pypy__/test/test_builders.py
--- a/pypy/module/__pypy__/test/test_builders.py
+++ b/pypy/module/__pypy__/test/test_builders.py
@@ -38,7 +38,9 @@
         b = StringBuilder()
         b.append("abc")
         b.append("123")
+        assert len(b) == 6
         b.append("you and me")
         s = b.build()
+        raises(ValueError, len, b)
         assert s == "abc123you and me"
-        raises(ValueError, b.build)
\ No newline at end of file
+        raises(ValueError, b.build)
diff --git a/pypy/module/_continuation/interp_continuation.py b/pypy/module/_continuation/interp_continuation.py
--- a/pypy/module/_continuation/interp_continuation.py
+++ b/pypy/module/_continuation/interp_continuation.py
@@ -19,6 +19,11 @@
         #  - normal:      self.sthread != None, not is_empty_handle(self.h)
         #  - finished:    self.sthread != None, is_empty_handle(self.h)
 
+    def __del__(self):
+        sthread = self.sthread
+        if sthread is not None and not sthread.is_empty_handle(self.h):
+            sthread.destroy(self.h)
+
     def check_sthread(self):
         ec = self.space.getexecutioncontext()
         if ec.stacklet_thread is not self.sthread:
@@ -28,6 +33,8 @@
     def descr_init(self, w_callable, __args__):
         if self.sthread is not None:
             raise geterror(self.space, "continulet already __init__ialized")
+        sthread = build_sthread(self.space)
+        workaround_disable_jit(sthread)
         #
         # hackish: build the frame "by hand", passing it the correct arguments
         space = self.space
@@ -41,7 +48,6 @@
         self.bottomframe = bottomframe
         #
         global_state.origin = self
-        sthread = build_sthread(self.space)
         self.sthread = sthread
         h = sthread.new(new_stacklet_callback)
         post_switch(sthread, h)
@@ -71,6 +77,7 @@
                 global_state.clear()
                 raise geterror(self.space, "continulet already finished")
         self.check_sthread()
+        workaround_disable_jit(self.sthread)
         #
         global_state.origin = self
         if to is None:
@@ -259,6 +266,16 @@
         sthread = ec.stacklet_thread = SThread(space, ec)
     return sthread
 
+def workaround_disable_jit(sthread):
+    # A bad workaround to kill the JIT anywhere in this thread.
+    # This forces all the frames.  It's a bad workaround because
+    # it takes O(depth) time, and it will cause some "abort:
+    # vable escape" in the JIT.  The goal is to prevent any frame
+    # from being still virtuals, because the JIT generates code
+    # to un-virtualizable them "on demand" by loading values based
+    # on FORCE_TOKEN, which is an address in the stack.
+    sthread.ec.force_all_frames()
+
 # ____________________________________________________________
 
 def permute(space, args_w):
diff --git a/pypy/module/_file/test/test_file.py b/pypy/module/_file/test/test_file.py
--- a/pypy/module/_file/test/test_file.py
+++ b/pypy/module/_file/test/test_file.py
@@ -1,5 +1,5 @@
 from __future__ import with_statement
-import py
+import py, os, errno
 
 from pypy.conftest import gettestobjspace, option
 
@@ -257,6 +257,39 @@
             assert self.temppath in g.getvalue()
 
 
+class AppTestNonblocking(object):
+    def setup_class(cls):
+        from pypy.module._file.interp_file import W_File
+
+        cls.old_read = os.read
+
+        if option.runappdirect:
+            py.test.skip("works with internals of _file impl on py.py")
+
+        state = [0]
+        def read(fd, n=None):
+            if fd != 42:
+                return cls.old_read(fd, n)
+            if state[0] == 0:
+                state[0] += 1
+                return "xyz"
+            if state[0] < 3:
+                state[0] += 1
+                raise OSError(errno.EAGAIN, "xyz")
+            return ''
+        os.read = read
+        stdin = W_File(cls.space)
+        stdin.file_fdopen(42, "r", 1)
+        stdin.name = '<stdin>'
+        cls.w_stream = stdin
+
+    def teardown_class(cls):
+        os.read = cls.old_read
+
+    def test_nonblocking_file(self):
+        res = self.stream.read()
+        assert res == 'xyz'
+
 class AppTestConcurrency(object):
     # these tests only really make sense on top of a translated pypy-c,
     # because on top of py.py the inner calls to os.write() don't
@@ -374,24 +407,24 @@
         with self.file(self.temppath, 'w') as f:
             f.write('foo')
         assert f.closed
-        
+
         with self.file(self.temppath, 'r') as f:
             s = f.readline()
 
         assert s == "foo"
         assert f.closed
-    
+
     def test_subclass_with(self):
         file = self.file
         class C(file):
             def __init__(self, *args, **kwargs):
                 self.subclass_closed = False
                 file.__init__(self, *args, **kwargs)
-            
+
             def close(self):
                 self.subclass_closed = True
                 file.close(self)
-        
+
         with C(self.temppath, 'w') as f:
             pass
         assert f.subclass_closed
diff --git a/pypy/module/_numpy/interp_dtype.py b/pypy/module/_numpy/interp_dtype.py
--- a/pypy/module/_numpy/interp_dtype.py
+++ b/pypy/module/_numpy/interp_dtype.py
@@ -161,9 +161,6 @@
     @binop
     def mul(self, v1, v2):
         return v1 * v2
-    @binop
-    def div(self, v1, v2):
-        return v1 / v2
 
     @unaryop
     def pos(self, v):
@@ -217,6 +214,14 @@
         return float2string(self.for_computation(self.unbox(item)), 'g', rfloat.DTSF_STR_PRECISION)
 
     @binop
+    def div(self, v1, v2):
+        try:
+            return v1 / v2
+        except ZeroDivisionError:
+            if v1 == v2 == 0.0:
+                return rfloat.NAN
+            return rfloat.copysign(rfloat.INFINITY, v1 * v2)
+    @binop
     def mod(self, v1, v2):
         return math.fmod(v1, v2)
     @binop
@@ -297,9 +302,15 @@
         return str(widen(self.unbox(item)))
 
     @binop
+    def div(self, v1, v2):
+        if v2 == 0:
+            return 0
+        return v1 / v2
+    @binop
     def mod(self, v1, v2):
         return v1 % v2
 
+
 class SignedIntegerArithmeticDtype(IntegerArithmeticDtype):
     _mixin_ = True
 
diff --git a/pypy/module/_numpy/test/test_numarray.py b/pypy/module/_numpy/test/test_numarray.py
--- a/pypy/module/_numpy/test/test_numarray.py
+++ b/pypy/module/_numpy/test/test_numarray.py
@@ -294,7 +294,10 @@
             assert b[i] == i * 5
 
     def test_div(self):
+        from math import isnan
         from _numpy import array, dtype
+        from numpy import inf
+
         a = array(range(1, 6))
         b = a / a
         for i in range(5):
@@ -306,6 +309,24 @@
         for i in range(5):
             assert b[i] == 1
 
+        a = array([-1, 0, 1])
+        b = array([0, 0, 0])
+        c = a / b
+        assert (c == [0, 0, 0]).all()
+
+        a = array([-1.0, 0.0, 1.0])
+        b = array([0.0, 0.0, 0.0])
+        c = a / b
+        assert c[0] == -inf
+        assert isnan(c[1])
+        assert c[2] == inf
+
+        b = array([-0.0, -0.0, -0.0])
+        c = a / b
+        assert c[0] == inf
+        assert isnan(c[1])
+        assert c[2] == -inf
+
     def test_div_other(self):
         from _numpy import array
         a = array(range(5))
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -10,6 +10,7 @@
 from pypy.rpython.lltypesystem import rffi, lltype
 from pypy.rpython.tool import rffi_platform
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
+from pypy.module.sys.interp_encoding import getfilesystemencoding
 
 import os, sys
 _WIN = sys.platform == 'win32'
@@ -32,17 +33,19 @@
             raise OperationError(space.w_OverflowError,
                                  space.wrap("integer out of range"))
 
+def fsencode_w(space, w_obj):
+    if space.isinstance_w(w_obj, space.w_unicode):
+        w_obj = space.call_method(w_obj, 'encode',
+                                  getfilesystemencoding(space))
+    return space.str_w(w_obj)
+
 class FileEncoder(object):
     def __init__(self, space, w_obj):
         self.space = space
         self.w_obj = w_obj
 
     def as_bytes(self):
-        from pypy.module.sys.interp_encoding import getfilesystemencoding
-        space = self.space
-        w_bytes = space.call_method(self.w_obj, 'encode',
-                                    getfilesystemencoding(space))
-        return space.str_w(w_bytes)
+        return fsencode_w(self.space, self.w_obj)
 
     def as_unicode(self):
         return self.space.unicode_w(self.w_obj)
@@ -56,7 +59,6 @@
         return self.space.str_w(self.w_obj)
 
     def as_unicode(self):
-        from pypy.module.sys.interp_encoding import getfilesystemencoding
         space = self.space
         w_unicode = space.call_method(self.w_obj, 'decode',
                                       getfilesystemencoding(space))
@@ -536,7 +538,6 @@
 
 The list is in arbitrary order.  It does not include the special
 entries '.' and '..' even if they are present in the directory."""
-    from pypy.module.sys.interp_encoding import getfilesystemencoding
     try:
         if space.isinstance_w(w_dirname, space.w_unicode):
             dirname = FileEncoder(space, w_dirname)
@@ -734,8 +735,7 @@
 def _exit(space, status):
     os._exit(status)
 
- at unwrap_spec(command=str)
-def execv(space, command, w_args):
+def execv(space, w_command, w_args):
     """ execv(path, args)
 
 Execute an executable path with arguments, replacing current process.
@@ -743,12 +743,13 @@
         path: path of executable file
         args: iterable of strings
     """
+    command = fsencode_w(space, w_command)
     try:
         args_w = space.unpackiterable(w_args)
         if len(args_w) < 1:
             w_msg = space.wrap("execv() must have at least one argument")
             raise OperationError(space.w_ValueError, w_msg)
-        args = [space.str_w(w_arg) for w_arg in args_w]
+        args = [fsencode_w(space, w_arg) for w_arg in args_w]
     except OperationError, e:
         if not e.match(space, space.w_TypeError):
             raise
@@ -759,8 +760,7 @@
     except OSError, e:
         raise wrap_oserror(space, e)
 
- at unwrap_spec(command=str)
-def execve(space, command, w_args, w_env):
+def execve(space, w_command, w_args, w_env):
     """ execve(path, args, env)
 
 Execute a path with arguments and environment, replacing current process.
@@ -769,7 +769,8 @@
         args: iterable of arguments
         env: dictionary of strings mapping to strings
     """
-    args = [space.str_w(w_arg) for w_arg in space.unpackiterable(w_args)]
+    command = fsencode_w(space, w_command)
+    args = [fsencode_w(space, w_arg) for w_arg in space.unpackiterable(w_args)]
     env = {}
     w_keys = space.call_method(w_env, 'keys')
     for w_key in space.unpackiterable(w_keys):
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -415,6 +415,23 @@
                 else:
                     py.test.fail("didn't raise")
 
+        def test_execv_unicode(self):
+            os = self.posix
+            import sys
+            if not hasattr(os, "fork"):
+                skip("Need fork() to test execv()")
+            try:
+                output = u"caf\xe9 \u1234\n".encode(sys.getfilesystemencoding())
+            except UnicodeEncodeError:
+                skip("encoding not good enough")
+            pid = os.fork()
+            if pid == 0:
+                os.execv(u"/bin/sh", ["sh", "-c",
+                                      u"echo caf\xe9 \u1234 > onefile"])
+            os.waitpid(pid, 0)
+            assert open("onefile").read() == output
+            os.unlink("onefile")
+
         def test_execve(self):
             os = self.posix
             if not hasattr(os, "fork"):
@@ -425,6 +442,24 @@
             os.waitpid(pid, 0)
             assert open("onefile").read() == "xxx"
             os.unlink("onefile")
+
+        def test_execve_unicode(self):
+            os = self.posix
+            import sys
+            if not hasattr(os, "fork"):
+                skip("Need fork() to test execve()")
+            try:
+                output = u"caf\xe9 \u1234\n".encode(sys.getfilesystemencoding())
+            except UnicodeEncodeError:
+                skip("encoding not good enough")
+            pid = os.fork()
+            if pid == 0:
+                os.execve(u"/bin/sh", ["sh", "-c",
+                                      u"echo caf\xe9 \u1234 > onefile"],
+                          {'ddd': 'xxx'})
+            os.waitpid(pid, 0)
+            assert open("onefile").read() == output
+            os.unlink("onefile")
         pass # <- please, inspect.getsource(), don't crash
 
     if hasattr(__import__(os.name), "spawnv"):
diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py
--- a/pypy/module/pypyjit/interp_jit.py
+++ b/pypy/module/pypyjit/interp_jit.py
@@ -137,20 +137,15 @@
 
     def jump_absolute(self, jumpto, _, ec=None):
         if we_are_jitted():
-            # Normally, the tick counter is decremented by 100 for every
-            # Python opcode.  Here, to better support JIT compilation of
-            # small loops, we decrement it by a possibly smaller constant.
-            # We get the maximum 100 when the (unoptimized) trace length
-            # is at least 3200 (a bit randomly).
-            trace_length = r_uint(current_trace_length())
-            decr_by = trace_length // 32
-            if decr_by < 1:
-                decr_by = 1
-            elif decr_by > 100:    # also if current_trace_length() returned -1
-                decr_by = 100
+            #
+            # assume that only threads are using the bytecode counter
+            decr_by = 0
+            if self.space.actionflag.has_bytecode_counter:   # constant-folded
+                if self.space.threadlocals.gil_ready:   # quasi-immutable field
+                    decr_by = _get_adapted_tick_counter()
             #
             self.last_instr = intmask(jumpto)
-            ec.bytecode_trace(self, intmask(decr_by))
+            ec.bytecode_trace(self, decr_by)
             jumpto = r_uint(self.last_instr)
         #
         pypyjitdriver.can_enter_jit(frame=self, ec=ec, next_instr=jumpto,
@@ -158,6 +153,20 @@
                                     is_being_profiled=self.is_being_profiled)
         return jumpto
 
+def _get_adapted_tick_counter():
+    # Normally, the tick counter is decremented by 100 for every
+    # Python opcode.  Here, to better support JIT compilation of
+    # small loops, we decrement it by a possibly smaller constant.
+    # We get the maximum 100 when the (unoptimized) trace length
+    # is at least 3200 (a bit randomly).
+    trace_length = r_uint(current_trace_length())
+    decr_by = trace_length // 32
+    if decr_by < 1:
+        decr_by = 1
+    elif decr_by > 100:    # also if current_trace_length() returned -1
+        decr_by = 100
+    return intmask(decr_by)
+
 
 PyCode__initialize = PyCode._initialize
 
diff --git a/pypy/module/pypyjit/test_pypy_c/model.py b/pypy/module/pypyjit/test_pypy_c/model.py
--- a/pypy/module/pypyjit/test_pypy_c/model.py
+++ b/pypy/module/pypyjit/test_pypy_c/model.py
@@ -225,6 +225,8 @@
         # strip comment
         if '#' in line:
             line = line[:line.index('#')]
+        if line.strip() == 'guard_not_invalidated?':
+            return 'guard_not_invalidated', None, [], '...', False
         # find the resvar, if any
         if ' = ' in line:
             resvar, _, line = line.partition(' = ')
@@ -249,7 +251,7 @@
             descr = descr[len('descr='):]
         else:
             descr = None
-        return opname, resvar, args, descr
+        return opname, resvar, args, descr, True
 
     @classmethod
     def preprocess_expected_src(cls, src):
@@ -258,13 +260,23 @@
         # replaced with the corresponding operations, so that tests don't have
         # to repeat it every time
         ticker_check = """
+            guard_not_invalidated?
+            ticker0 = getfield_raw(ticker_address, descr=<SignedFieldDescr pypysig_long_struct.c_value .*>)
+            ticker_cond0 = int_lt(ticker0, 0)
+            guard_false(ticker_cond0, descr=...)
+        """
+        src = src.replace('--TICK--', ticker_check)
+        #
+        # this is the ticker check generated if we have threads
+        thread_ticker_check = """
+            guard_not_invalidated?
             ticker0 = getfield_raw(ticker_address, descr=<SignedFieldDescr pypysig_long_struct.c_value .*>)
             ticker1 = int_sub(ticker0, 1)
             setfield_raw(ticker_address, ticker1, descr=<SignedFieldDescr pypysig_long_struct.c_value .*>)
             ticker_cond0 = int_lt(ticker1, 0)
             guard_false(ticker_cond0, descr=...)
         """
-        src = src.replace('--TICK--', ticker_check)
+        src = src.replace('--THREAD-TICK--', thread_ticker_check)
         #
         # this is the ticker check generated in PyFrame.handle_operation_error
         exc_ticker_check = """
@@ -298,7 +310,7 @@
         if not cond:
             raise InvalidMatch(message, frame=sys._getframe(1))
 
-    def match_op(self, op, (exp_opname, exp_res, exp_args, exp_descr)):
+    def match_op(self, op, (exp_opname, exp_res, exp_args, exp_descr, _)):
         self._assert(op.name == exp_opname, "operation mismatch")
         self.match_var(op.res, exp_res)
         if exp_args != ['...']:
@@ -341,7 +353,7 @@
         what is after the '...'
         """
         iter_exp_ops = iter(expected_ops)
-        iter_ops = iter(self.ops)
+        iter_ops = RevertableIterator(self.ops)
         for opindex, exp_op in enumerate(iter_exp_ops):
             try:
                 if exp_op == '...':
@@ -360,6 +372,9 @@
                             break
                 self.match_op(op, exp_op)
             except InvalidMatch, e:
+                if exp_op[4] is False:    # optional operation
+                    iter_ops.revert_one()
+                    continue       # try to match with the next exp_op
                 e.opindex = opindex
                 raise
         #
@@ -398,3 +413,18 @@
         else:
             return True
 
+
+class RevertableIterator(object):
+    def __init__(self, sequence):
+        self.sequence = sequence
+        self.index = 0
+    def __iter__(self):
+        return self
+    def next(self):
+        index = self.index
+        if index == len(self.sequence):
+            raise StopIteration
+        self.index = index + 1
+        return self.sequence[index]
+    def revert_one(self):
+        self.index -= 1
diff --git a/pypy/module/pypyjit/test_pypy_c/test_00_model.py b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
--- a/pypy/module/pypyjit/test_pypy_c/test_00_model.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
@@ -145,15 +145,17 @@
 
     def test_parse_op(self):
         res = OpMatcher.parse_op("  a =   int_add(  b,  3 ) # foo")
-        assert res == ("int_add", "a", ["b", "3"], None)
+        assert res == ("int_add", "a", ["b", "3"], None, True)
         res = OpMatcher.parse_op("guard_true(a)")
-        assert res == ("guard_true", None, ["a"], None)
+        assert res == ("guard_true", None, ["a"], None, True)
         res = OpMatcher.parse_op("setfield_gc(p0, i0, descr=<foobar>)")
-        assert res == ("setfield_gc", None, ["p0", "i0"], "<foobar>")
+        assert res == ("setfield_gc", None, ["p0", "i0"], "<foobar>", True)
         res = OpMatcher.parse_op("i1 = getfield_gc(p0, descr=<foobar>)")
-        assert res == ("getfield_gc", "i1", ["p0"], "<foobar>")
+        assert res == ("getfield_gc", "i1", ["p0"], "<foobar>", True)
         res = OpMatcher.parse_op("p0 = force_token()")
-        assert res == ("force_token", "p0", [], None)
+        assert res == ("force_token", "p0", [], None, True)
+        res = OpMatcher.parse_op("guard_not_invalidated?")
+        assert res == ("guard_not_invalidated", None, [], '...', False)
 
     def test_exact_match(self):
         loop = """
@@ -341,7 +343,7 @@
             # this is the actual loop
             'int_lt', 'guard_true', 'int_add',
             # this is the signal checking stuff
-            'getfield_raw', 'int_sub', 'setfield_raw', 'int_lt', 'guard_false',
+            'guard_not_invalidated', 'getfield_raw', 'int_lt', 'guard_false',
             'jump'
             ]
 
@@ -407,7 +409,7 @@
             # this is the actual loop
             'int_lt', 'guard_true', 'force_token', 'int_add',
             # this is the signal checking stuff
-            'getfield_raw', 'int_sub', 'setfield_raw', 'int_lt', 'guard_false',
+            'guard_not_invalidated', 'getfield_raw', 'int_lt', 'guard_false',
             'jump'
             ]
 
@@ -425,10 +427,9 @@
             guard_true(i6, descr=...)
             i8 = int_add(i4, 1)
             # signal checking stuff
+            guard_not_invalidated(descr=...)
             i10 = getfield_raw(37212896, descr=<.* pypysig_long_struct.c_value .*>)
-            i12 = int_sub(i10, 1)
-            setfield_raw(37212896, i12, descr=<.* pypysig_long_struct.c_value .*>)
-            i14 = int_lt(i12, 0)
+            i14 = int_lt(i10, 0)
             guard_false(i14, descr=...)
             jump(p0, p1, p2, p3, i8, descr=...)
         """)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_misc.py b/pypy/module/pypyjit/test_pypy_c/test_misc.py
--- a/pypy/module/pypyjit/test_pypy_c/test_misc.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_misc.py
@@ -329,4 +329,20 @@
             guard_false(i28, descr=...)
             i30 = int_lshift(i20, 24)
             i31 = int_or(i26, i30)
-        """ % {"32_bit_only": extra})
\ No newline at end of file
+        """ % {"32_bit_only": extra})
+
+    def test_eval(self):
+        def main():
+            i = 1
+            a = compile('x+x+x+x+x+x', 'eval', 'eval')
+            b = {'x': 7}
+            while i < 1000:
+                y = eval(a,b,b)  # ID: eval
+                i += 1
+            return y
+
+        log = self.run(main)
+        assert log.result == 42
+        # the following assertion fails if the loop was cancelled due
+        # to "abort: vable escape"
+        assert len(log.loops_by_id("eval")) == 1
diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -156,4 +156,41 @@
             i40 = int_sub(i4, 1)
             --TICK--
             jump(p0, p1, p2, p3, i40, i38, descr=<Loop0>)
-        """)
\ No newline at end of file
+        """)
+
+    def test_getattr_promote(self):
+        def main(n):
+            class A(object):
+                def meth_a(self):
+                    return 1
+                def meth_b(self):
+                    return 2
+            a = A()
+
+            l = ['a', 'b']
+            s = 0
+            for i in range(n):
+                name = 'meth_' + l[i & 1]
+                meth = getattr(a, name) # ID: getattr
+                s += meth()
+            return s
+
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loops = log.loops_by_filename(self.filepath)
+        assert len(loops) == 2
+        for loop in loops:
+            loop.match_by_id('getattr','''
+            guard_not_invalidated(descr=...)
+            i32 = strlen(p31)
+            i34 = int_add(5, i32)
+            p35 = newstr(i34)
+            strsetitem(p35, 0, 109)
+            strsetitem(p35, 1, 101)
+            strsetitem(p35, 2, 116)
+            strsetitem(p35, 3, 104)
+            strsetitem(p35, 4, 95)
+            copystrcontent(p31, p35, 0, 5, i32)
+            i49 = call(ConstClass(_ll_2_str_eq_nonnull__rpy_stringPtr_rpy_stringPtr), p35, ConstPtr(ptr48), descr=<SignedCallDescr>)
+            guard_value(i49, 1, descr=<Guard8>)
+            ''')
diff --git a/pypy/module/pypyjit/test_pypy_c/test_thread.py b/pypy/module/pypyjit/test_pypy_c/test_thread.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/pypyjit/test_pypy_c/test_thread.py
@@ -0,0 +1,28 @@
+from pypy.module.pypyjit.test_pypy_c.test_00_model import BaseTestPyPyC
+
+
+class TestThread(BaseTestPyPyC):
+    def test_simple(self):
+        def main(n):
+            import thread
+            def f():
+                i = 0
+                while i < n:
+                    i += 1
+                done.release()
+
+            done = thread.allocate_lock()
+            done.acquire()
+            thread.start_new_thread(f, ())
+            done.acquire()
+            return 0
+        log = self.run(main, [500])
+        assert round(log.result, 6) == round(main(500), 6)
+        loop, = log.loops_by_filename(self.filepath)
+        assert loop.match("""
+            i2 = int_lt(i0, i1)
+            guard_true(i2, descr=...)
+            i3 = int_add(i0, 1)
+            --THREAD-TICK--
+            jump(..., descr=<Loop0>)
+        """)
diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py
--- a/pypy/module/signal/interp_signal.py
+++ b/pypy/module/signal/interp_signal.py
@@ -109,8 +109,11 @@
         p = pypysig_getaddr_occurred()
         value = p.c_value
         if self.has_bytecode_counter:    # this 'if' is constant-folded
-            value -= by
-            p.c_value = value
+            if jit.isconstant(by) and by == 0:
+                pass     # normally constant-folded too
+            else:
+                value -= by
+                p.c_value = value
         return value
 
 
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -47,7 +47,7 @@
         'pypy_initial_path'     : 'state.pypy_initial_path',
 
         '_getframe'             : 'vm._getframe', 
-        '_current_frames'       : 'vm._current_frames', 
+        '_current_frames'       : 'currentframes._current_frames', 
         'setrecursionlimit'     : 'vm.setrecursionlimit', 
         'getrecursionlimit'     : 'vm.getrecursionlimit', 
         'setcheckinterval'      : 'vm.setcheckinterval', 
@@ -55,6 +55,7 @@
         'exc_info'              : 'vm.exc_info', 
         'exc_clear'             : 'vm.exc_clear', 
         'settrace'              : 'vm.settrace',
+        'gettrace'              : 'vm.gettrace',
         'setprofile'            : 'vm.setprofile',
         'getprofile'            : 'vm.getprofile',
         'call_tracing'          : 'vm.call_tracing',
diff --git a/pypy/module/sys/currentframes.py b/pypy/module/sys/currentframes.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/sys/currentframes.py
@@ -0,0 +1,78 @@
+"""
+Implementation of the 'sys._current_frames()' routine.
+"""
+from pypy.interpreter import gateway
+
+app = gateway.applevel('''
+"NOT_RPYTHON"
+import __builtin__
+
+class fake_code(object):
+    co_name = "?"
+    co_filename = "?"
+    co_firstlineno = 0
+
+class fake_frame(object):
+    f_back = None
+    f_builtins = __builtin__.__dict__
+    f_code = fake_code()
+    f_exc_traceback = None
+    f_exc_type = None
+    f_exc_value = None
+    f_globals = {}
+    f_lasti = -1
+    f_lineno = 0
+    f_locals = {}
+    f_restricted = False
+    f_trace = None
+
+    def __init__(self, f):
+        if f is not None:
+            for name in ["f_builtins", "f_code", "f_globals", "f_lasti",
+                         "f_lineno"]:
+                setattr(self, name, getattr(f, name))
+''')
+
+def _current_frames(space):
+    """_current_frames() -> dictionary
+
+    Return a dictionary mapping each current thread T's thread id to T's
+    current stack "frame".  Functions in the traceback module can build the
+    call stack given such a frame.
+
+    Note that in PyPy this returns fake frame objects, to avoid a runtime
+    penalty everywhere with the JIT.  (So far these fake frames can be
+    completely uninformative depending on the JIT state; we could return
+    more with more efforts.)
+
+    This function should be used for specialized purposes only."""
+    w_result = space.newdict()
+    w_fake_frame = app.wget(space, "fake_frame")
+    w_fake_code  = app.wget(space, "fake_code")
+    ecs = space.threadlocals.getallvalues()
+    for thread_ident, ec in ecs.items():
+        vref = ec.topframeref
+        frames = []
+        while not vref.virtual:
+            f = vref()
+            if f is None:
+                break
+            frames.append(f)
+            vref = f.f_backref
+        else:
+            frames.append(None)
+        #
+        w_topframe = space.wrap(None)
+        w_prevframe = None
+        for f in frames:
+            w_nextframe = space.call_function(w_fake_frame, space.wrap(f))
+            if w_prevframe is None:
+                w_topframe = w_nextframe
+            else:
+                space.setattr(w_prevframe, space.wrap('f_back'), w_nextframe)
+            w_prevframe = w_nextframe
+        #
+        space.setitem(w_result,
+                      space.wrap(thread_ident),
+                      w_topframe)
+    return w_result
diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -478,6 +478,7 @@
         sys.settrace(trace)
         try:
             x()
+            assert sys.gettrace() is trace
         finally:
             sys.settrace(None)
         assert len(counts) == 1
@@ -555,7 +556,7 @@
             return sys._current_frames()
         frames = f()
         assert frames.keys() == [0]
-        assert frames[0].f_code.co_name == 'f'
+        assert frames[0].f_code.co_name in ('f', '?')
 
 class AppTestCurrentFramesWithThread(AppTestCurrentFrames):
     def setup_class(cls):
@@ -567,23 +568,25 @@
         import thread
 
         thread_id = thread.get_ident()
-        self.ready = False
         def other_thread():
-            self.ready = True
             print "thread started"
-            time.sleep(5)
+            lock2.release()
+            lock1.acquire()
+        lock1 = thread.allocate_lock()
+        lock2 = thread.allocate_lock()
+        lock1.acquire()
+        lock2.acquire()
         thread.start_new_thread(other_thread, ())
 
         def f():
-            for i in range(100):
-                if self.ready: break
-                time.sleep(0.1)
+            lock2.acquire()
             return sys._current_frames()
 
         frames = f()
+        lock1.release()
         thisframe = frames.pop(thread_id)
-        assert thisframe.f_code.co_name == 'f'
+        assert thisframe.f_code.co_name in ('f', '?')
 
         assert len(frames) == 1
         _, other_frame = frames.popitem()
-        assert other_frame.f_code.co_name == 'other_thread'
+        assert other_frame.f_code.co_name in ('other_thread', '?')
diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -45,25 +45,6 @@
     f.mark_as_escaped()
     return space.wrap(f)
 
-def _current_frames(space):
-    """_current_frames() -> dictionary
-
-    Return a dictionary mapping each current thread T's thread id to T's
-    current stack frame.
-
-    This function should be used for specialized purposes only."""
-    raise OperationError(space.w_NotImplementedError,
-        space.wrap("XXX sys._current_frames() incompatible with the JIT"))
-    w_result = space.newdict()
-    ecs = space.threadlocals.getallvalues()
-    for thread_ident, ec in ecs.items():
-        f = ec.gettopframe_nohidden()
-        f.mark_as_escaped()
-        space.setitem(w_result,
-                      space.wrap(thread_ident),
-                      space.wrap(f))
-    return w_result
-
 def setrecursionlimit(space, w_new_limit):
     """setrecursionlimit() sets the maximum number of nested calls that
 can occur before a RuntimeError is raised.  On PyPy the limit is
@@ -129,14 +110,19 @@
 function call.  See the debugger chapter in the library manual."""
     space.getexecutioncontext().settrace(w_func)
 
+def gettrace(space):
+    """Return the global debug tracing function set with sys.settrace.
+See the debugger chapter in the library manual."""
+    return space.getexecutioncontext().gettrace()
+
 def setprofile(space, w_func):
     """Set the profiling function.  It will be called on each function call
 and return.  See the profiler chapter in the library manual."""
     space.getexecutioncontext().setprofile(w_func)
 
 def getprofile(space):
-    """Set the profiling function.  It will be called on each function call
-and return.  See the profiler chapter in the library manual."""
+    """Return the profiling function set with sys.setprofile.
+See the profiler chapter in the library manual."""
     w_func = space.getexecutioncontext().getprofile()
     if w_func is not None:
         return w_func
diff --git a/pypy/module/thread/gil.py b/pypy/module/thread/gil.py
--- a/pypy/module/thread/gil.py
+++ b/pypy/module/thread/gil.py
@@ -17,6 +17,7 @@
 class GILThreadLocals(OSThreadLocals):
     """A version of OSThreadLocals that enforces a GIL."""
     gil_ready = False
+    _immutable_fields_ = ['gil_ready?']
 
     def initialize(self, space):
         # add the GIL-releasing callback as an action on the space
diff --git a/pypy/module/thread/threadlocals.py b/pypy/module/thread/threadlocals.py
--- a/pypy/module/thread/threadlocals.py
+++ b/pypy/module/thread/threadlocals.py
@@ -8,9 +8,14 @@
 
     def __init__(self):
         self._valuedict = {}   # {thread_ident: ExecutionContext()}
+        self._freeze_()
+
+    def _freeze_(self):
+        self._valuedict.clear()
         self._mainthreadident = 0
         self._mostrecentkey = 0        # fast minicaching for the common case
         self._mostrecentvalue = None   # fast minicaching for the common case
+        return False
 
     def getvalue(self):
         ident = thread.get_ident()
diff --git a/pypy/objspace/std/celldict.py b/pypy/objspace/std/celldict.py
--- a/pypy/objspace/std/celldict.py
+++ b/pypy/objspace/std/celldict.py
@@ -37,10 +37,10 @@
         self.version = VersionTag()
 
     def get_empty_storage(self):
-       return self.erase({})
+        return self.erase({})
 
     def mutated(self):
-       self.version = VersionTag()
+        self.version = VersionTag()
 
     def getdictvalue_no_unwrapping(self, w_dict, key):
         # NB: it's important to promote self here, so that self.version is a
diff --git a/pypy/objspace/std/objecttype.py b/pypy/objspace/std/objecttype.py
--- a/pypy/objspace/std/objecttype.py
+++ b/pypy/objspace/std/objecttype.py
@@ -44,7 +44,6 @@
         raise OperationError(space.w_TypeError,
                              space.wrap("__class__ assignment: only for heap types"))
     w_oldcls = space.type(w_obj)
-    # XXX taint space should raise a TaintError here if w_oldcls is tainted
     assert isinstance(w_oldcls, W_TypeObject)
     if w_oldcls.get_full_instance_layout() == w_newcls.get_full_instance_layout():
         w_obj.setclass(space, w_newcls)
diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py
--- a/pypy/objspace/std/stringobject.py
+++ b/pypy/objspace/std/stringobject.py
@@ -161,57 +161,59 @@
 
 def str_swapcase__String(space, w_self):
     self = w_self._value
-    res = [' '] * len(self)
+    builder = StringBuilder(len(self))
     for i in range(len(self)):
         ch = self[i]
         if ch.isupper():
             o = ord(ch) + 32
-            res[i] = chr(o)
+            builder.append(chr(o))
         elif ch.islower():
             o = ord(ch) - 32
-            res[i] = chr(o)
+            builder.append(chr(o))
         else:
-            res[i] = ch
+            builder.append(ch)
 
-    return space.wrap("".join(res))
+    return space.wrap(builder.build())
 
 
 def str_capitalize__String(space, w_self):
     input = w_self._value
-    buffer = [' '] * len(input)
+    builder = StringBuilder(len(input))
     if len(input) > 0:
         ch = input[0]
         if ch.islower():
             o = ord(ch) - 32
-            buffer[0] = chr(o)
+            builder.append(chr(o))
         else:
-            buffer[0] = ch
+            builder.append(ch)
 
         for i in range(1, len(input)):
             ch = input[i]
             if ch.isupper():
                 o = ord(ch) + 32
-                buffer[i] = chr(o)
+                builder.append(chr(o))
             else:
-                buffer[i] = ch
+                builder.append(ch)
 
-    return space.wrap("".join(buffer))
+    return space.wrap(builder.build())
 
 def str_title__String(space, w_self):
     input = w_self._value
-    buffer = [' '] * len(input)
+    builder = StringBuilder(len(input))
     prev_letter=' '
 
-    for pos in range(0, len(input)):
+    for pos in range(len(input)):
         ch = input[pos]
         if not prev_letter.isalpha():
-            buffer[pos] = _upper(ch)
+            ch = _upper(ch)
+            builder.append(ch)
         else:
-            buffer[pos] = _lower(ch)
+            ch = _lower(ch)
+            builder.append(ch)
 
-        prev_letter = buffer[pos]
+        prev_letter = ch
 
-    return space.wrap("".join(buffer))
+    return space.wrap(builder.build())
 
 def str_split__String_None_ANY(space, w_self, w_none, w_maxsplit=-1):
     maxsplit = space.int_w(w_maxsplit)
@@ -754,27 +756,21 @@
     input = w_self._value
     width = space.int_w(w_width)
 
-    if len(input) >= width:
+    num_zeros = width - len(input)
+    if num_zeros <= 0:
         # cannot return w_self, in case it is a subclass of str
         return space.wrap(input)
 
-    buf = [' '] * width
+    builder = StringBuilder(width)
     if len(input) > 0 and (input[0] == '+' or input[0] == '-'):
-        buf[0] = input[0]
+        builder.append(input[0])
         start = 1
-        middle = width - len(input) + 1
     else:
         start = 0
-        middle = width - len(input)
 
-    for i in range(start, middle):
-        buf[i] = '0'
-
-    for i in range(middle, width):
-        buf[i] = input[start]
-        start = start + 1
-
-    return space.wrap("".join(buf))
+    builder.append_multiple_char('0', num_zeros)
+    builder.append_slice(input, start, len(input))
+    return space.wrap(builder.build())
 
 
 def hash__String(space, w_str):
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -10,7 +10,8 @@
 from pypy.objspace.std import identitydict
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.rlib.objectmodel import current_object_addr_as_int, compute_hash
-from pypy.rlib.jit import promote, elidable_promote, we_are_jitted
+from pypy.rlib.jit import promote, elidable_promote, we_are_jitted,\
+     promote_string
 from pypy.rlib.jit import elidable, dont_look_inside, unroll_safe
 from pypy.rlib.rarithmetic import intmask, r_uint
 
@@ -101,6 +102,7 @@
                           'instancetypedef',
                           'terminator',
                           '_version_tag?',
+                          'interplevel_cls',
                           ]
 
     # for config.objspace.std.getattributeshortcut
@@ -399,6 +401,7 @@
         if version_tag is None:
             tup = w_self._lookup_where(name)
             return tup
+        name = promote_string(name)
         w_class, w_value = w_self._pure_lookup_where_with_method_cache(name, version_tag)
         return w_class, unwrap_cell(space, w_value)
 
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -417,54 +417,54 @@
     input = w_self._value
     if len(input) == 0:
         return W_UnicodeObject.EMPTY
-    result = [u'\0'] * len(input)
-    result[0] = unichr(unicodedb.toupper(ord(input[0])))
+    builder = UnicodeBuilder(len(input))
+    builder.append(unichr(unicodedb.toupper(ord(input[0]))))
     for i in range(1, len(input)):
-        result[i] = unichr(unicodedb.tolower(ord(input[i])))
-    return W_UnicodeObject(u''.join(result))
+        builder.append(unichr(unicodedb.tolower(ord(input[i]))))
+    return W_UnicodeObject(builder.build())
 
 def unicode_title__Unicode(space, w_self):
     input = w_self._value
     if len(input) == 0:
         return w_self
-    result = [u'\0'] * len(input)
+    builder = UnicodeBuilder(len(input))
 
     previous_is_cased = False
     for i in range(len(input)):
         unichar = ord(input[i])
         if previous_is_cased:
-            result[i] = unichr(unicodedb.tolower(unichar))
+            builder.append(unichr(unicodedb.tolower(unichar)))
         else:
-            result[i] = unichr(unicodedb.totitle(unichar))
+            builder.append(unichr(unicodedb.totitle(unichar)))
         previous_is_cased = unicodedb.iscased(unichar)
-    return W_UnicodeObject(u''.join(result))
+    return W_UnicodeObject(builder.build())
 
 def unicode_lower__Unicode(space, w_self):
     input = w_self._value
-    result = [u'\0'] * len(input)
+    builder = UnicodeBuilder(len(input))
     for i in range(len(input)):
-        result[i] = unichr(unicodedb.tolower(ord(input[i])))
-    return W_UnicodeObject(u''.join(result))
+        builder.append(unichr(unicodedb.tolower(ord(input[i]))))
+    return W_UnicodeObject(builder.build())
 
 def unicode_upper__Unicode(space, w_self):
     input = w_self._value
-    result = [u'\0'] * len(input)
+    builder = UnicodeBuilder(len(input))
     for i in range(len(input)):
-        result[i] = unichr(unicodedb.toupper(ord(input[i])))
-    return W_UnicodeObject(u''.join(result))
+        builder.append(unichr(unicodedb.toupper(ord(input[i]))))
+    return W_UnicodeObject(builder.build())
 
 def unicode_swapcase__Unicode(space, w_self):
     input = w_self._value
-    result = [u'\0'] * len(input)
+    builder = UnicodeBuilder(len(input))
     for i in range(len(input)):
         unichar = ord(input[i])
         if unicodedb.islower(unichar):
-            result[i] = unichr(unicodedb.toupper(unichar))
+            builder.append(unichr(unicodedb.toupper(unichar)))
         elif unicodedb.isupper(unichar):
-            result[i] = unichr(unicodedb.tolower(unichar))
+            builder.append(unichr(unicodedb.tolower(unichar)))
         else:
-            result[i] = input[i]
-    return W_UnicodeObject(u''.join(result))
+            builder.append(input[i])
+    return W_UnicodeObject(builder.build())
 
 def _normalize_index(length, index):
     if index < 0:
diff --git a/pypy/objspace/taint.py b/pypy/objspace/taint.py
deleted file mode 100644
--- a/pypy/objspace/taint.py
+++ /dev/null
@@ -1,294 +0,0 @@
-"""
-Just an experiment.
-"""
-import os
-from pypy.objspace.std.objspace import StdObjSpace
-from pypy.objspace.proxy import patch_space_in_place
-from pypy.objspace.thunk import nb_forcing_args
-from pypy.interpreter.error import OperationError
-from pypy.interpreter import baseobjspace, gateway, executioncontext
-from pypy.interpreter.function import Method
-from pypy.interpreter.pyframe import PyFrame
-from pypy.tool.sourcetools import func_with_new_name
-from pypy.rlib.unroll import unrolling_iterable
-
-
-class W_Tainted(baseobjspace.W_Root):
-    def __init__(self, w_obj):
-        self.w_obj = w_obj
-
-##    def getdict(self, space):
-##        return taint(self.w_obj.getdict(space))
-
-##    def getdictvalue(self, space, attr):
-##        return taint(self.w_obj.getdictvalue(space, attr))
-
-##    def setdictvalue(self, space, attr, w_value):
-##        return self.w_obj.setdictvalue(space, attr, w_value)
-
-##    ...
-
-class W_TaintBomb(baseobjspace.W_Root):
-    filename = '?'
-    codename = '?'
-    codeline = 0
-
-    def __init__(self, space, operr):
-        self.space = space
-        self.operr = operr
-        self.record_debug_info()
-
-    def record_debug_info(self):
-        ec = self.space.getexecutioncontext()
-        frame = ec.gettopframe_nohidden()
-        if isinstance(frame, PyFrame):     # and, in particular, frame != None
-            self.filename = frame.pycode.co_filename
-            self.codename = frame.pycode.co_name
-            self.codeline = frame.get_last_lineno()
-        if get_debug_level(self.space) > 0:
-            self.debug_dump()
-
-    def debug_dump(self):
-        os.write(2, 'Taint Bomb from file "%s", line %d, in %s\n    %s\n' % (
-            self.filename, self.codeline, self.codename,
-            self.operr.errorstr(self.space)))
-
-    def explode(self):
-        #msg = self.operr.errorstr(space)
-        raise OperationError(self.space.w_TaintError, self.space.w_None)
-
-
-def taint(w_obj):
-    """Return a tainted version of the argument."""
-    if w_obj is None or isinstance(w_obj, W_Tainted):
-        return w_obj
-    else:
-        return W_Tainted(w_obj)
-app_taint = gateway.interp2app(taint)
-
-def is_tainted(space, w_obj):
-    """Return whether the argument is tainted."""
-    res = isinstance(w_obj, W_Tainted) or isinstance(w_obj, W_TaintBomb)
-    return space.wrap(res)
-app_is_tainted = gateway.interp2app(is_tainted)
-
-def untaint(space, w_expectedtype, w_obj):
-    """untaint(expectedtype, tainted_obj) -> obj
-Untaint untainted_obj and return it. If the result is not of expectedtype,
-raise a type error."""
-    if (isinstance(w_expectedtype, W_Tainted) or
-        isinstance(w_expectedtype, W_TaintBomb)):
-        raise OperationError(space.w_TypeError,
-                  space.wrap("untaint() arg 1 must be an untainted type"))
-    if not space.is_true(space.isinstance(w_expectedtype, space.w_type)):
-        raise OperationError(space.w_TypeError,
-                             space.wrap("untaint() arg 1 must be a type"))
-    if isinstance(w_obj, W_Tainted):
-        w_obj = w_obj.w_obj
-    elif isinstance(w_obj, W_TaintBomb):
-        w_obj.explode()
-    #if isinstance(w_expectedtype, W_Tainted):
-    #    w_expectedtype = w_expectedtype.w_obj
-    w_realtype = space.type(w_obj)
-    if not space.is_w(w_realtype, w_expectedtype):
-        #msg = "expected an object of type '%s'" % (
-        #    w_expectedtype.getname(space),)
-        #    #w_realtype.getname(space))
-        raise OperationError(space.w_TaintError, space.w_None)
-    return w_obj
-app_untaint = gateway.interp2app(untaint)
-
-# ____________________________________________________________
-
- at gateway.unwrap_spec(args_w='args_w')
-def taint_atomic_function(space, w_func, args_w):
-    newargs_w = []
-    tainted = False
-    for w_arg in args_w:
-        if isinstance(w_arg, W_Tainted):
-            tainted = True
-            w_arg = w_arg.w_obj
-        elif isinstance(w_arg, W_TaintBomb):
-            return w_arg
-        newargs_w.append(w_arg)
-    w_newargs = space.newtuple(newargs_w)
-    try:
-        w_res = space.call(w_func, w_newargs)
-    except OperationError, operr:
-        if not tainted:
-            raise
-        return W_TaintBomb(space, operr)
-    if tainted:
-        w_res = taint(w_res)
-    return w_res
-
-app_taint_atomic_function = gateway.interp2app(taint_atomic_function)
-
-def taint_atomic(space, w_callable):
-    """decorator to make a callable "taint-atomic": if the function is called
-with tainted arguments, those are untainted. The result of the function is
-tainted again.  All exceptions that the callable raises are turned into
-taint bombs."""
-    meth = Method(space, space.w_fn_taint_atomic_function,
-                  w_callable, space.type(w_callable))
-    return space.wrap(meth)
-app_taint_atomic = gateway.interp2app(taint_atomic)
-
-# ____________________________________________________________
-
-executioncontext.ExecutionContext.taint_debug = 0
-
- at gateway.unwrap_spec(level=int)
-def taint_debug(space, level):
-    """Set the debug level. If the debug level is greater than 0, the creation
-of taint bombs will print debug information. For debugging purposes
-only!"""
-    space.getexecutioncontext().taint_debug = level
-app_taint_debug = gateway.interp2app(taint_debug)
-
-def taint_look(space, w_obj):
-    """Print some info about the taintedness of an object. For debugging
-purposes only!"""
-    if isinstance(w_obj, W_Tainted):
-        info = space.type(w_obj.w_obj).getname(space)
-        msg = space.str_w(w_obj.w_obj.getrepr(space, info))
-        msg = 'Taint Box %s\n' % msg
-        os.write(2, msg)
-    elif isinstance(w_obj, W_TaintBomb):
-        w_obj.debug_dump()
-    else:
-        os.write(2, 'not tainted\n')
-app_taint_look = gateway.interp2app(taint_look)
-
-def get_debug_level(space):
-    return space.getexecutioncontext().taint_debug
-
-def debug_bomb(space, operr):
-    ec = space.getexecutioncontext()
-    filename = '?'
-    codename = '?'
-    codeline = 0
-    frame = ec.gettopframe_nohidden()
-    if isinstance(frame, PyFrame):     # and, in particular, frame != None
-        filename = frame.pycode.co_filename
-        codename = frame.pycode.co_name
-        codeline = frame.get_last_lineno()
-    os.write(2, 'Taint Bomb in file "%s", line %d, in %s\n    %s\n' % (
-        filename, codeline, codename, operr.errorstr(space)))
-
-# ____________________________________________________________
-
-
-class TaintSpace(StdObjSpace):
-
-    def __init__(self, *args, **kwds):
-        StdObjSpace.__init__(self, *args, **kwds)
-        w_dict = self.newdict()
-        self.setitem(w_dict, self.wrap("__doc__"), self.wrap("""\
-Exception that is raised when an operation revealing information on a tainted
-object is performed."""))
-        self.w_TaintError = self.call_function(
-            self.w_type,
-            self.wrap("TaintError"),
-            self.newtuple([self.w_Exception]),
-            w_dict
-            )
-        w___pypy__ = self.getbuiltinmodule("__pypy__")
-        self.setattr(w___pypy__, self.wrap('taint'),
-                     self.wrap(app_taint))
-        self.setattr(w___pypy__, self.wrap('is_tainted'),
-                     self.wrap(app_is_tainted))
-        self.setattr(w___pypy__, self.wrap('untaint'),
-                     self.wrap(app_untaint))
-        self.w_fn_taint_atomic_function = self.wrap(app_taint_atomic_function)
-        self.setattr(w___pypy__, self.wrap('taint_atomic'),
-                     self.wrap(app_taint_atomic))
-        self.setattr(w___pypy__, self.wrap('TaintError'),
-                     self.w_TaintError)
-        self.setattr(w___pypy__, self.wrap('_taint_debug'),
-                     self.wrap(app_taint_debug))
-        self.setattr(w___pypy__, self.wrap('_taint_look'),
-                     self.wrap(app_taint_look))
-        patch_space_in_place(self, 'taint', proxymaker)
-
-        # XXX may leak info, perfomance hit, what about taint bombs?
-        from pypy.objspace.std.typeobject import W_TypeObject
-
-        def taint_lookup(w_obj, name):
-            if isinstance(w_obj, W_Tainted):
-                w_obj = w_obj.w_obj
-            w_type = self.type(w_obj)
-            assert isinstance(w_type, W_TypeObject)
-            return w_type.lookup(name)
-
-        def taint_lookup_in_type_where(w_obj, name):
-            if isinstance(w_obj, W_Tainted):
-                w_type = w_obj.w_obj
-            else:
-                w_type = w_obj
-            assert isinstance(w_type, W_TypeObject)
-            return w_type.lookup_where(name)
-
-        self.lookup = taint_lookup
-        self.lookup_in_type_where = taint_lookup_in_type_where
-
-
-Space = TaintSpace
-
-
-def tainted_error(space, name):
-    #msg = "operation '%s' forbidden on tainted object" % (name,)
-    raise OperationError(space.w_TaintError, space.w_None)# space.wrap(msg))
-
-
-RegularMethods = dict.fromkeys(
-    [name for name, _, _, _ in baseobjspace.ObjSpace.MethodTable])
-
-TaintResultIrregularMethods = dict.fromkeys(
-    ['wrap', 'call_args'] +
-    [name for name in baseobjspace.ObjSpace.IrregularOpTable
-          if name.startswith('new')])
-
-def proxymaker(space, name, parentfn):
-    arity = nb_forcing_args[name]
-    indices = unrolling_iterable(range(arity))
-    if name in RegularMethods:
-
-        def proxy(*args_w):
-            newargs_w = ()
-            tainted = False
-            for i in indices:
-                w_arg = args_w[i]
-                if isinstance(w_arg, W_Tainted):
-                    tainted = True
-                    w_arg = w_arg.w_obj
-                elif isinstance(w_arg, W_TaintBomb):
-                    return w_arg
-                newargs_w += (w_arg,)
-            newargs_w += args_w[arity:]
-            try:
-                w_res = parentfn(*newargs_w)
-            except OperationError, operr:
-                if not tainted:
-                    raise
-                return W_TaintBomb(space, operr)
-            if tainted:
-                w_res = taint(w_res)
-            return w_res
-
-    elif arity == 0:
-        return None
-
-    else:
-
-        def proxy(*args_w):
-            for i in indices:
-                w_arg = args_w[i]
-                if isinstance(w_arg, W_Tainted):
-                    tainted_error(space, name)
-                elif isinstance(w_arg, W_TaintBomb):
-                    w_arg.explode()
-            return parentfn(*args_w)
-
-    proxy = func_with_new_name(proxy, '%s_proxy' % name)
-    return proxy
diff --git a/pypy/objspace/test/test_taintobjspace.py b/pypy/objspace/test/test_taintobjspace.py
deleted file mode 100644
--- a/pypy/objspace/test/test_taintobjspace.py
+++ /dev/null
@@ -1,77 +0,0 @@
-from pypy.conftest import gettestobjspace
-
-class AppTest_Taint:
-
-    def setup_class(cls):
-        cls.space = gettestobjspace('taint')
-
-    def test_simple(self):
-        from __pypy__ import taint, untaint, TaintError
-        x = taint(6)
-        x = x * 7
-        raises(TaintError, "if x: y = 1")
-        t = type(x)
-        raises(TaintError, "if t is int: y = 1")
-        assert untaint(int, x) == 42
-        raises(TaintError, "untaint(float, x)")
-
-    def test_bomb(self):
-        from __pypy__ import taint, untaint, TaintError
-        x = taint(6)
-        x = x / 0
-        raises(TaintError, "if x: y = 1")
-        t = type(x)
-        raises(TaintError, "if t is int: y = 1")
-        raises(TaintError, "untaint(int, x)")
-        raises(TaintError, "untaint(float, x)")
-
-    def test_taint_atomic(self):
-        from __pypy__ import taint, untaint, TaintError, taint_atomic
-        x = taint(6)
-        x *= 7
-
-        def dummy(x):
-            if x > 40:
-                return 5
-            else:
-                return 3
-        dummy = taint_atomic(dummy)
-
-        y = dummy(x)
-        raises(TaintError, "if y == 3: z = 1")
-        assert untaint(int, y) == 5
-
-    def test_taint_atomic_exception(self):
-        from __pypy__ import taint, untaint, TaintError, taint_atomic
-        x = taint(6)
-        x *= 7
-
-        def dummy(x):
-            if x + "world" == "hello world":
-                return 5
-            else:
-                return 3
-        dummy = taint_atomic(dummy)
-
-        y = dummy(x)
-        raises(TaintError, "if y == 3: z = 1")
-        raises(TaintError, "untaint(int, y)")
-
-    def test_taint_atomic_incoming_bomb(self):
-        from __pypy__ import taint, untaint, TaintError, taint_atomic
-        x = taint(6)
-        x /= 0
-        lst = []
-
-        def dummy(x):
-            lst.append("running!")
-            if x > 40:
-                return 5
-            else:
-                return 3
-        dummy = taint_atomic(dummy)
-
-        y = dummy(x)
-        raises(TaintError, "if y == 3: z = 1")
-        assert lst == []
-        raises(TaintError, "untaint(int, y)")
diff --git a/pypy/rlib/_jit_vref.py b/pypy/rlib/_jit_vref.py
--- a/pypy/rlib/_jit_vref.py
+++ b/pypy/rlib/_jit_vref.py
@@ -25,6 +25,10 @@
     def simple_call(self):
         return self.s_instance
 
+    def getattr(self, s_attr):
+        assert s_attr.const == 'virtual'
+        return annmodel.s_Bool
+
     def rtyper_makerepr(self, rtyper):
         if rtyper.type_system.name == 'lltypesystem':
             return vrefrepr
@@ -61,6 +65,13 @@
                              " prebuilt virtual_ref")
         return lltype.nullptr(OBJECTPTR.TO)
 
+    def rtype_getattr(self, hop):
+        s_attr = hop.args_s[1]
+        assert s_attr.const == 'virtual'
+        v = hop.inputarg(self, arg=0)
+        hop.exception_cannot_occur()
+        return hop.genop('jit_is_virtual', [v], resulttype = lltype.Bool)
+
 from pypy.rpython.ootypesystem.rclass import OBJECT
 
 class OOVRefRepr(VRefRepr):
diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -38,6 +38,7 @@
     possible arguments are:
 
     * promote - promote the argument from a variable into a constant
+    * promote_string - same, but promote string by *value*
     * access_directly - directly access a virtualizable, as a structure
                         and don't treat it as a virtualizable
     * fresh_virtualizable - means that virtualizable was just allocated.
@@ -51,6 +52,9 @@
 def promote(x):
     return hint(x, promote=True)
 
+def promote_string(x):
+    return hint(x, promote_string=True)
+
 def dont_look_inside(func):
     """ Make sure the JIT does not trace inside decorated function
     (it becomes a call instead)
@@ -313,6 +317,12 @@
             raise InvalidVirtualRef
         return self._x
 
+    @property
+    def virtual(self):
+        """A property that is True if the vref contains a virtual that would
+        be forced by the '()' operator."""
+        return self._state == 'non-forced'
+
     def _finish(self):
         if self._state == 'non-forced':
             self._state = 'invalid'
diff --git a/pypy/rlib/rerased.py b/pypy/rlib/rerased.py
--- a/pypy/rlib/rerased.py
+++ b/pypy/rlib/rerased.py
@@ -218,15 +218,13 @@
         [v_value] = hop.inputargs(lltype.Signed)
         c_one = hop.inputconst(lltype.Signed, 1)
         hop.exception_is_here()
-        v2 = hop.genop('int_lshift_ovf', [v_value, c_one],
+        v2 = hop.genop('int_add_ovf', [v_value, v_value],
                        resulttype = lltype.Signed)
         v2p1 = hop.genop('int_add', [v2, c_one],
                          resulttype = lltype.Signed)
         v_instance = hop.genop('cast_int_to_ptr', [v2p1],
                                resulttype=self.lowleveltype)
-        v = hop.genop('cast_opaque_ptr', [v_instance],
-                      resulttype=self.lowleveltype)
-        return v
+        return v_instance
 
     def convert_const(self, value):
         if value._identity is _identity_for_ints:
@@ -266,10 +264,10 @@
         return hop.genop('int_rshift', [v2, c_one], resulttype=lltype.Signed)
 
     def rtype_erase_int(self, hop):
-        hop.exception_is_here()
         [v_value] = hop.inputargs(lltype.Signed)
         c_one = hop.inputconst(lltype.Signed, 1)
-        v2 = hop.genop('int_lshift_ovf', [v_value, c_one],
+        hop.exception_is_here()
+        v2 = hop.genop('int_add_ovf', [v_value, v_value],
                        resulttype = lltype.Signed)
         v2p1 = hop.genop('int_add', [v2, c_one],
                          resulttype = lltype.Signed)
diff --git a/pypy/rlib/streamio.py b/pypy/rlib/streamio.py
--- a/pypy/rlib/streamio.py
+++ b/pypy/rlib/streamio.py
@@ -37,7 +37,7 @@
 # return value of tell(), but not as argument to read().
 #
 
-import os, sys
+import os, sys, errno
 from pypy.rlib.objectmodel import specialize, we_are_translated
 from pypy.rlib.rarithmetic import r_longlong, intmask
 from pypy.rlib import rposix
@@ -587,12 +587,22 @@
     def readall(self):
         pos = self.pos
         assert pos >= 0
-        chunks = [self.buf[pos:]]
+        if self.buf:
+            chunks = [self.buf[pos:]]
+        else:
+            chunks = []
         self.buf = ""
         self.pos = 0
         bufsize = self.bufsize
         while 1:
-            data = self.do_read(bufsize)
+            try:
+                data = self.do_read(bufsize)
+            except OSError, o:
+                if o.errno != errno.EAGAIN:
+                    raise
+                if not chunks:
+                    raise
+                break
             if not data:
                 break
             chunks.append(data)
diff --git a/pypy/rlib/test/test__jit_vref.py b/pypy/rlib/test/test__jit_vref.py
--- a/pypy/rlib/test/test__jit_vref.py
+++ b/pypy/rlib/test/test__jit_vref.py
@@ -27,10 +27,13 @@
     x1 = X()
     vref = virtual_ref(x1)
     assert vref._state == 'non-forced'
+    assert vref.virtual is True
     assert vref() is x1
     assert vref._state == 'forced'
+    assert vref.virtual is False
     virtual_ref_finish(vref, x1)
     assert vref._state == 'forced'
+    assert vref.virtual is False
     assert vref() is x1
 
 def test_direct_invalid():
@@ -135,6 +138,13 @@
         x = self.interpret(f, [])
         assert x == 42
 
+    def test_rtype_virtualattr(self):
+        def f():
+            vref = virtual_ref(X())
+            return vref.virtual
+        x = self.interpret(f, [])
+        assert x is False
+
 
 class TestLLtype(BaseTestVRef, LLRtypeMixin):
     OBJECTTYPE = OBJECTPTR
diff --git a/pypy/rlib/test/test_jit.py b/pypy/rlib/test/test_jit.py
--- a/pypy/rlib/test/test_jit.py
+++ b/pypy/rlib/test/test_jit.py
@@ -139,12 +139,11 @@
 
     def test_isconstant(self):
         def f(n):
-            assert n >= 0
             assert isconstant(n) is False
             l = []
             l.append(n)
             return len(l)
-        res = self.interpret(f, [234])
+        res = self.interpret(f, [-234])
         assert res == 1
 
 
diff --git a/pypy/rpython/lltypesystem/ll2ctypes.py b/pypy/rpython/lltypesystem/ll2ctypes.py
--- a/pypy/rpython/lltypesystem/ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/ll2ctypes.py
@@ -658,6 +658,8 @@
         if T == llmemory.GCREF:
             if isinstance(llobj._obj, _llgcopaque):
                 return ctypes.c_void_p(llobj._obj.intval)
+            if isinstance(llobj._obj, int):    # tagged pointer
+                return ctypes.c_void_p(llobj._obj)
             container = llobj._obj.container
             T = lltype.Ptr(lltype.typeOf(container))
             # otherwise it came from integer and we want a c_void_p with
@@ -1268,6 +1270,7 @@
 class _llgcopaque(lltype._container):
     _TYPE = llmemory.GCREF.TO
     _name = "_llgcopaque"
+    _read_directly_intval = True     # for _ptr._cast_to_int()
 
     def __init__(self, void_p):
         if isinstance(void_p, (int, long)):
@@ -1288,6 +1291,9 @@
             return False
         return force_cast(lltype.Signed, other._as_ptr()) == self.intval
 
+    def __hash__(self):
+        return self.intval
+
     def __ne__(self, other):
         return not self == other
 
@@ -1296,11 +1302,6 @@
             return _opaque_objs[self.intval // 2]
         return force_cast(PTRTYPE, self.intval)
 
-##     def _cast_to_int(self):
-##         return self.intval
-
-##     def _cast_to_adr(self):
-##         return _lladdress(self.intval)
 
 def cast_adr_to_int(addr):
     if isinstance(addr, llmemory.fakeaddress):
diff --git a/pypy/rpython/lltypesystem/llmemory.py b/pypy/rpython/lltypesystem/llmemory.py
--- a/pypy/rpython/lltypesystem/llmemory.py
+++ b/pypy/rpython/lltypesystem/llmemory.py
@@ -498,6 +498,8 @@
 
     def _cast_to_int(self, symbolic=False):
         if self:
+            if isinstance(self.ptr._obj0, int):    # tagged integer
+                return self.ptr._obj0
             if symbolic:
                 return AddressAsInt(self)
             else:
diff --git a/pypy/rpython/lltypesystem/lloperation.py b/pypy/rpython/lltypesystem/lloperation.py
--- a/pypy/rpython/lltypesystem/lloperation.py
+++ b/pypy/rpython/lltypesystem/lloperation.py
@@ -428,6 +428,7 @@
     'jit_marker':           LLOp(),
     'jit_force_virtualizable':LLOp(canrun=True),
     'jit_force_virtual':    LLOp(canrun=True),
+    'jit_is_virtual':       LLOp(canrun=True),
     'jit_force_quasi_immutable': LLOp(canrun=True),
     'get_exception_addr':   LLOp(),
     'get_exc_value_addr':   LLOp(),
diff --git a/pypy/rpython/lltypesystem/lltype.py b/pypy/rpython/lltypesystem/lltype.py
--- a/pypy/rpython/lltypesystem/lltype.py
+++ b/pypy/rpython/lltypesystem/lltype.py
@@ -1360,6 +1360,8 @@
         obj = normalizeptr(self, check)._getobj(check)
         if isinstance(obj, int):
             return obj     # special case for cast_int_to_ptr() results put into opaques
+        if getattr(obj, '_read_directly_intval', False):
+            return obj.intval   # special case for _llgcopaque
         result = intmask(obj._getid())
         # assume that id() returns an addressish value which is
         # not zero and aligned to at least a multiple of 4
diff --git a/pypy/rpython/lltypesystem/opimpl.py b/pypy/rpython/lltypesystem/opimpl.py
--- a/pypy/rpython/lltypesystem/opimpl.py
+++ b/pypy/rpython/lltypesystem/opimpl.py
@@ -538,6 +538,9 @@
 def op_jit_force_virtual(x):
     return x
 
+def op_jit_is_virtual(x):
+    return False
+
 def op_jit_force_quasi_immutable(*args):
     pass
 
diff --git a/pypy/rpython/lltypesystem/rstr.py b/pypy/rpython/lltypesystem/rstr.py
--- a/pypy/rpython/lltypesystem/rstr.py
+++ b/pypy/rpython/lltypesystem/rstr.py
@@ -694,8 +694,8 @@
             return -1
         return count
 
+    @enforceargs(int, None)
     @jit.look_inside_iff(lambda length, items: jit.isconstant(length) and length <= 2)
-    @enforceargs(int, None)
     def ll_join_strs(length, items):
         # Special case for length 1 items, helps both the JIT and other code
         if length == 1:
diff --git a/pypy/rpython/lltypesystem/rtagged.py b/pypy/rpython/lltypesystem/rtagged.py
--- a/pypy/rpython/lltypesystem/rtagged.py
+++ b/pypy/rpython/lltypesystem/rtagged.py
@@ -43,7 +43,7 @@
         v_value = hop.inputarg(lltype.Signed, arg=1)
         c_one = hop.inputconst(lltype.Signed, 1)
         hop.exception_is_here()
-        v2 = hop.genop('int_lshift_ovf', [v_value, c_one],
+        v2 = hop.genop('int_add_ovf', [v_value, v_value],
                        resulttype = lltype.Signed)
         v2p1 = hop.genop('int_add', [v2, c_one],
                          resulttype = lltype.Signed)
diff --git a/pypy/rpython/lltypesystem/test/test_ll2ctypes.py b/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
--- a/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
@@ -1123,6 +1123,9 @@
 
         #assert lltype.cast_ptr_to_int(ref1) == intval
 
+        x = rffi.cast(llmemory.GCREF, -17)
+        assert lltype.cast_ptr_to_int(x) == -17
+
     def test_ptr_truth(self):
         abc = rffi.cast(lltype.Ptr(lltype.FuncType([], lltype.Void)), 0)
         assert not abc
diff --git a/pypy/rpython/rclass.py b/pypy/rpython/rclass.py
--- a/pypy/rpython/rclass.py
+++ b/pypy/rpython/rclass.py
@@ -191,6 +191,10 @@
                     "class %r inherits from its parent _immutable_=True, "
                     "so it should also declare _immutable_=True" % (
                     self.classdef,))
+            if loc.classdict.get('_immutable_').value is not True:
+                raise TyperError(
+                    "class %r: _immutable_ = something else than True" % (
+                    self.classdef,))
             hints = hints.copy()
             hints['immutable'] = True
         self.immutable_field_set = set()  # unless overwritten below
diff --git a/pypy/tool/logparser.py b/pypy/tool/logparser.py
--- a/pypy/tool/logparser.py
+++ b/pypy/tool/logparser.py
@@ -298,6 +298,8 @@
         image.paste(textpercent, (t1x, 5), textpercent)
         image.paste(textlabel,   (t2x, 5), textlabel)
         images.append(image)
+    if not images:
+        return None
     return combine(images, spacing=0, border=1, horizontal=False)
 
 def get_timesummary_single_image(totaltimes, totaltime0, componentdict,
@@ -333,6 +335,8 @@
         del totaltimes[None]
     img2 = render_histogram(totaltimes, totaltime0, {},
                             width, summarybarheight)
+    if img2 is None:
+        return img1
     return combine([img1, img2], spacing=spacing, horizontal=True)
 
 # ----------
diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -43,34 +43,29 @@
 def package(basedir, name='pypy-nightly', rename_pypy_c='pypy',
             copy_to_dir = None, override_pypy_c = None):
     basedir = py.path.local(basedir)
+    if override_pypy_c is None:
+        basename = 'pypy-c'
+        if sys.platform == 'win32':
+            basename += '.exe'
+        pypy_c = basedir.join('pypy', 'translator', 'goal', basename)
+    else:
+        pypy_c = py.path.local(override_pypy_c)
+    if not pypy_c.check():
+        print pypy_c
+        raise PyPyCNotFound('Please compile pypy first, using translate.py')
+    binaries = [(pypy_c, rename_pypy_c)]
+    #
     if sys.platform == 'win32':
-        # Can't rename a DLL
-        if override_pypy_c is not None:
-            rename_pypy_c = py.path.local(override_pypy_c).purebasename
-            pypy_c_dir = py.path.local(override_pypy_c).dirname
-        else:
-            pypy_c_dir = basedir.join('pypy', 'translator', 'goal')
-        pypy_c = pypy_c_dir.join(rename_pypy_c + '.exe')
-        libpypy_c = pypy_c_dir.join('lib' + rename_pypy_c + '.dll')
-        binaries = [(pypy_c, pypy_c.basename),
-                    (libpypy_c, libpypy_c.basename)]
-        for extra in ['libexpat.dll', 'sqlite3.dll', 'msvcr90.dll']:
-            p = pypy_c_dir.join(extra)
+        # Can't rename a DLL: it is always called 'libpypy_c.dll'
+        for extra in ['libpypy_c.dll',
+                      'libexpat.dll', 'sqlite3.dll', 'msvcr90.dll']:
+            p = pypy_c.dirpath().join(extra)
             if not p.check():
                 p = py.path.local.sysfind(extra)
                 assert p, "%s not found" % (extra,)
             print "Picking %s" % p
             binaries.append((p, p.basename))
-    else:
-        basename = 'pypy-c'
-        if override_pypy_c is None:
-            pypy_c = basedir.join('pypy', 'translator', 'goal', basename)
-        else:
-            pypy_c = py.path.local(override_pypy_c)
-        binaries = [(pypy_c, rename_pypy_c)]
-    if not pypy_c.check():
-        print pypy_c
-        raise PyPyCNotFound('Please compile pypy first, using translate.py')
+    #
     builddir = udir.ensure("build", dir=True)
     pypydir = builddir.ensure(name, dir=True)
     # Careful: to copy lib_pypy, copying just the svn-tracked files
diff --git a/pypy/translator/c/funcgen.py b/pypy/translator/c/funcgen.py
--- a/pypy/translator/c/funcgen.py
+++ b/pypy/translator/c/funcgen.py
@@ -833,7 +833,7 @@
         return 'INSTRUMENT_COUNT(%s);' % counter_label
             
     def OP_IS_EARLY_CONSTANT(self, op):
-        return self.expr(op.result)  + ' = 0;' # Allways false
+        return '%s = 0; /* IS_EARLY_CONSTANT */' % (self.expr(op.result),)
 
     def OP_JIT_MARKER(self, op):
         return '/* JIT_MARKER %s */' % op
@@ -845,6 +845,9 @@
         return '%s = %s; /* JIT_FORCE_VIRTUAL */' % (self.expr(op.result),
                                                      self.expr(op.args[0]))
 
+    def OP_JIT_IS_VIRTUAL(self, op):
+        return '%s = 0; /* JIT_IS_VIRTUAL */' % (self.expr(op.result),)
+
     def OP_JIT_FORCE_QUASI_IMMUTABLE(self, op):
         return '/* JIT_FORCE_QUASI_IMMUTABLE %s */' % op
 
diff --git a/pypy/translator/c/genc.py b/pypy/translator/c/genc.py
--- a/pypy/translator/c/genc.py
+++ b/pypy/translator/c/genc.py
@@ -8,7 +8,7 @@
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
 from pypy.rpython.lltypesystem import lltype
 from pypy.tool.udir import udir
-from pypy.tool import isolate
+from pypy.tool import isolate, runsubprocess
 from pypy.translator.c.support import log, c_string_constant
 from pypy.rpython.typesystem import getfunctionptr
 from pypy.translator.c import gc
@@ -563,14 +563,19 @@
             else:
                 mk.definition('PYPY_MAIN_FUNCTION', "main")
 
-            if (py.path.local.sysfind('python') or
-                py.path.local.sysfind('python.exe')):
-                python = 'python '
-            elif sys.platform == 'win32':
+            if sys.platform == 'win32':
                 python = sys.executable.replace('\\', '/') + ' '
             else:
                 python = sys.executable + ' '
 
+            # Is there a command 'python' that runs python 2.5-2.7?
+            # If there is, then we can use it instead of sys.executable
+            returncode, stdout, stderr = runsubprocess.run_subprocess(
+                "python", "-V")
+            if (stdout.startswith('Python 2.') or
+                stderr.startswith('Python 2.')):
+                python = 'python '
+
             if self.translator.platform.name == 'msvc':
                 lblofiles = []
                 for cfile in mk.cfiles:
diff --git a/pypy/translator/c/src/thread_nt.h b/pypy/translator/c/src/thread_nt.h
--- a/pypy/translator/c/src/thread_nt.h
+++ b/pypy/translator/c/src/thread_nt.h
@@ -245,7 +245,7 @@
     if (pending_acquires <= 0)
         return 0;
     InterlockedIncrement(&pending_acquires);
-    PulseEvent(&cond_gil);
+    PulseEvent(cond_gil);
 
     /* hack: the three following lines do a pthread_cond_wait(), and
        normally specifying a timeout of INFINITE would be fine.  But the
@@ -253,7 +253,7 @@
        (small) risk that PulseEvent misses the WaitForSingleObject().
        In this case the process will just sleep a few milliseconds. */
     LeaveCriticalSection(&mutex_gil);
-    WaitForSingleObject(&cond_gil, 15);
+    WaitForSingleObject(cond_gil, 15);
     EnterCriticalSection(&mutex_gil);
 
     InterlockedDecrement(&pending_acquires);
@@ -263,7 +263,7 @@
 void RPyGilRelease(void)
 {
     LeaveCriticalSection(&mutex_gil);
-    PulseEvent(&cond_gil);
+    PulseEvent(cond_gil);
 }
 
 void RPyGilAcquire(void)
diff --git a/pypy/translator/goal/app_main.py b/pypy/translator/goal/app_main.py
--- a/pypy/translator/goal/app_main.py
+++ b/pypy/translator/goal/app_main.py
@@ -144,7 +144,7 @@
     print '  --jit off                  turn off the JIT'
 
 def print_version(*args):
-    print "Python", sys.version
+    print >> sys.stderr, "Python", sys.version
     raise SystemExit
 
 def set_jit_option(options, jitparam, *args):
diff --git a/pypy/translator/platform/linux.py b/pypy/translator/platform/linux.py
--- a/pypy/translator/platform/linux.py
+++ b/pypy/translator/platform/linux.py
@@ -24,15 +24,14 @@
         return self._pkg_config("libffi", "--libs-only-L",
                                 ['/usr/lib/libffi'])
 
+    def library_dirs_for_libffi_a(self):
+        # places where we need to look for libffi.a
+        return self.library_dirs_for_libffi() + ['/usr/lib']
+
 
 class Linux(BaseLinux):
     shared_only = ()    # it seems that on 32-bit linux, compiling with -fPIC
                         # gives assembler that asmgcc is not happy about.
 
-    def library_dirs_for_libffi_a(self):
-        # places where we need to look for libffi.a
-        return self.library_dirs_for_libffi() + ['/usr/lib']
-
-
 class Linux64(BaseLinux):
     pass

From noreply at buildbot.pypy.org  Mon Oct 17 11:49:56 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Oct 2011 11:49:56 +0200 (CEST)
Subject: [pypy-commit] pypy default: Add a failing test.
Message-ID: <20111017094956.7C311820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48116:0b99a4b937e0
Date: 2011-10-17 11:21 +0200
http://bitbucket.org/pypy/pypy/changeset/0b99a4b937e0/

Log:	Add a failing test.

diff --git a/pypy/jit/metainterp/test/test_quasiimmut.py b/pypy/jit/metainterp/test/test_quasiimmut.py
--- a/pypy/jit/metainterp/test/test_quasiimmut.py
+++ b/pypy/jit/metainterp/test/test_quasiimmut.py
@@ -457,6 +457,27 @@
                          getarrayitem_gc=0, getarrayitem_gc_pure=0,
                          call_may_force=0, guard_not_forced=0)
 
+    def test_invalidated_loop_is_not_used_any_more_as_target(self):
+        py.test.skip("in-progress")
+        myjitdriver = JitDriver(greens=['foo'], reds=['x'])
+        class Foo:
+            _immutable_fields_ = ['step?']
+        @dont_look_inside
+        def residual(x, foo):
+            if x == 20:
+                foo.step = 1
+        def f(x):
+            foo = Foo()
+            foo.step = 2
+            while x > 0:
+                myjitdriver.jit_merge_point(foo=foo, x=x)
+                residual(x, foo)
+                x -= foo.step
+            return foo.step
+        res = self.meta_interp(f, [60])
+        assert res == 1
+        self.check_tree_loop_count(3)   # maybe --- at least not 2 like now
+
 
 class TestLLtypeGreenFieldsTests(QuasiImmutTests, LLJitMixin):
     pass

From noreply at buildbot.pypy.org  Mon Oct 17 11:49:57 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Oct 2011 11:49:57 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix the test. Add in the graphpage
 viewer shading in gray the
Message-ID: <20111017094957.ADC9D820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48117:849075adda3f
Date: 2011-10-17 11:43 +0200
http://bitbucket.org/pypy/pypy/changeset/849075adda3f/

Log:	Fix the test. Add in the graphpage viewer shading in gray the
	invalidated loops.

diff --git a/pypy/jit/metainterp/graphpage.py b/pypy/jit/metainterp/graphpage.py
--- a/pypy/jit/metainterp/graphpage.py
+++ b/pypy/jit/metainterp/graphpage.py
@@ -12,8 +12,8 @@
     def get_display_text(self):
         return None
 
-def display_loops(loops, errmsg=None, highlight_loops=()):
-    graphs = [(loop, loop in highlight_loops) for loop in loops]    
+def display_loops(loops, errmsg=None, highlight_loops={}):
+    graphs = [(loop, highlight_loops.get(loop, 0)) for loop in loops]    
     for graph, highlight in graphs:
         for op in graph.get_operations():
             if is_interesting_guard(op):
@@ -65,8 +65,7 @@
     def add_graph(self, graph, highlight=False):
         graphindex = len(self.graphs)
         self.graphs.append(graph)
-        if highlight:
-            self.highlight_graphs[graph] = True
+        self.highlight_graphs[graph] = highlight
         for i, op in enumerate(graph.get_operations()):
             self.all_operations[op] = graphindex, i
 
@@ -126,10 +125,13 @@
             self.dotgen.emit('subgraph cluster%d {' % graphindex)
         label = graph.get_display_text()
         if label is not None:
-            if self.highlight_graphs.get(graph):
-                fillcolor = '#f084c2'
+            colorindex = self.highlight_graphs.get(graph, 0)
+            if colorindex == 1:
+                fillcolor = '#f084c2'    # highlighted graph
+            elif colorindex == 2:
+                fillcolor = '#808080'    # invalidated graph
             else:
-                fillcolor = '#84f0c2'
+                fillcolor = '#84f0c2'    # normal color
             self.dotgen.emit_node(graphname, shape="octagon",
                                   label=label, fillcolor=fillcolor)
             self.pendingedges.append((graphname,
diff --git a/pypy/jit/metainterp/history.py b/pypy/jit/metainterp/history.py
--- a/pypy/jit/metainterp/history.py
+++ b/pypy/jit/metainterp/history.py
@@ -732,6 +732,7 @@
     failed_states = None
     retraced_count = 0
     terminating = False # see TerminatingLoopToken in compile.py
+    invalidated = False
     outermost_jitdriver_sd = None
     # and more data specified by the backend when the loop is compiled
     number = -1
@@ -934,6 +935,7 @@
         self.loops = []
         self.locations = []
         self.aborted_keys = []
+        self.invalidated_token_numbers = set()
 
     def set_history(self, history):
         self.operations = history.operations
@@ -1012,7 +1014,12 @@
             if loop in loops:
                 loops.remove(loop)
             loops.append(loop)
-        display_loops(loops, errmsg, extraloops)
+        highlight_loops = dict.fromkeys(extraloops, 1)
+        for loop in loops:
+            if hasattr(loop, '_looptoken_number') and (
+                    loop._looptoken_number in self.invalidated_token_numbers):
+                highlight_loops.setdefault(loop, 2)
+        display_loops(loops, errmsg, highlight_loops)
 
 # ----------------------------------------------------------------
 
diff --git a/pypy/jit/metainterp/quasiimmut.py b/pypy/jit/metainterp/quasiimmut.py
--- a/pypy/jit/metainterp/quasiimmut.py
+++ b/pypy/jit/metainterp/quasiimmut.py
@@ -2,6 +2,7 @@
 from pypy.rpython.lltypesystem import lltype, rclass
 from pypy.rpython.annlowlevel import cast_base_ptr_to_instance
 from pypy.jit.metainterp.history import AbstractDescr
+from pypy.rlib.objectmodel import we_are_translated
 
 
 def get_mutate_field_name(fieldname):
@@ -73,8 +74,12 @@
         self.looptokens_wrefs.append(wref_looptoken)
 
     def compress_looptokens_list(self):
-        self.looptokens_wrefs = [wref for wref in self.looptokens_wrefs
-                                      if wref() is not None]
+        newlist = []
+        for wref in self.looptokens_wrefs:
+            looptoken = wref()
+            if looptoken is not None and not looptoken.invalidated:
+                newlist.append(wref)
+        self.looptokens_wrefs = wref
         self.compress_limit = (len(self.looptokens_wrefs) + 15) * 2
 
     def invalidate(self):
@@ -85,8 +90,12 @@
         self.looptokens_wrefs = []
         for wref in wrefs:
             looptoken = wref()
-            if looptoken is not None:
+            if looptoken is not None and not looptoken.invalidated:
+                looptoken.invalidated = True
                 self.cpu.invalidate_loop(looptoken)
+                if not we_are_translated():
+                    self.cpu.stats.invalidated_token_numbers.add(
+                        looptoken.number)
 
 
 class QuasiImmutDescr(AbstractDescr):
diff --git a/pypy/jit/metainterp/test/test_quasiimmut.py b/pypy/jit/metainterp/test/test_quasiimmut.py
--- a/pypy/jit/metainterp/test/test_quasiimmut.py
+++ b/pypy/jit/metainterp/test/test_quasiimmut.py
@@ -289,7 +289,7 @@
             return total
 
         res = self.meta_interp(main, [])
-        self.check_loop_count(9)
+        self.check_tree_loop_count(6)
         assert res == main()
 
     def test_change_during_running(self):
@@ -317,7 +317,7 @@
         assert f(100, 15) == 3009
         res = self.meta_interp(f, [100, 15])
         assert res == 3009
-        self.check_loops(guard_not_invalidated=2, getfield_gc=0,
+        self.check_loops(guard_not_invalidated=4, getfield_gc=0,
                          call_may_force=0, guard_not_forced=0)
 
     def test_list_simple_1(self):
@@ -458,7 +458,6 @@
                          call_may_force=0, guard_not_forced=0)
 
     def test_invalidated_loop_is_not_used_any_more_as_target(self):
-        py.test.skip("in-progress")
         myjitdriver = JitDriver(greens=['foo'], reds=['x'])
         class Foo:
             _immutable_fields_ = ['step?']
@@ -476,7 +475,7 @@
             return foo.step
         res = self.meta_interp(f, [60])
         assert res == 1
-        self.check_tree_loop_count(3)   # maybe --- at least not 2 like now
+        self.check_tree_loop_count(4)   # at least not 2 like before
 
 
 class TestLLtypeGreenFieldsTests(QuasiImmutTests, LLJitMixin):
diff --git a/pypy/jit/metainterp/warmstate.py b/pypy/jit/metainterp/warmstate.py
--- a/pypy/jit/metainterp/warmstate.py
+++ b/pypy/jit/metainterp/warmstate.py
@@ -178,7 +178,7 @@
         if self.compiled_merge_points_wref is not None:
             for wref in self.compiled_merge_points_wref:
                 looptoken = wref()
-                if looptoken is not None:
+                if looptoken is not None and not looptoken.invalidated:
                     result.append(looptoken)
         return result
 

From noreply at buildbot.pypy.org  Mon Oct 17 12:24:27 2011
From: noreply at buildbot.pypy.org (hager)
Date: Mon, 17 Oct 2011 12:24:27 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: Added fail_boxes_ptr to class
 AssemblerPPC class to make test_get_latest_value_count pass.
Message-ID: <20111017102427.A3B6F820D7@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r48118:af1bd77832d4
Date: 2011-10-17 12:24 +0200
http://bitbucket.org/pypy/pypy/changeset/af1bd77832d4/

Log:	Added fail_boxes_ptr to class AssemblerPPC class to make
	test_get_latest_value_count pass.

diff --git a/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py b/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py
--- a/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py
@@ -76,9 +76,11 @@
     def __init__(self, cpu, failargs_limit=1000):
         self.cpu = cpu
         self.fail_boxes_int = values_array(lltype.Signed, failargs_limit)
+        self.fail_boxes_ptr = values_array(llmemory.GCREF, failargs_limit)
         self.mc = None
         self.datablockwrapper = None
         self.memcpy_addr = 0
+        self.fail_boxes_count = 0
 
     def load_imm(self, rD, word):
         if word <= 32767 and word >= -32768:
diff --git a/pypy/jit/backend/ppc/runner.py b/pypy/jit/backend/ppc/runner.py
--- a/pypy/jit/backend/ppc/runner.py
+++ b/pypy/jit/backend/ppc/runner.py
@@ -94,8 +94,9 @@
         self.fail_boxes_int.setitem(index, sign_ptr)
 
     def clear_latest_values(self, count):
+        null = lltype.nullptr(llmemory.GCREF.TO)
         for index in range(count):
-            self.fail_boxes_int.setitem(index, 0)
+            self.asm.fail_boxes_ptr.setitem(index, null)
 
     # executes the stored machine code in the token
     def execute_token(self, looptoken):   
@@ -124,7 +125,7 @@
 
     # return the number of values that can be returned
     def get_latest_value_count(self):
-        return self.fail_box_count
+        return self.asm.fail_boxes_count
 
     # fetch the result of the computation and return it
     def get_latest_value_int(self, index):

From noreply at buildbot.pypy.org  Mon Oct 17 13:54:05 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Oct 2011 13:54:05 +0200 (CEST)
Subject: [pypy-commit] pypy default: Make the marker more obvious.
Message-ID: <20111017115405.BB2B5820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48119:2d6c7a33ca50
Date: 2011-10-17 13:53 +0200
http://bitbucket.org/pypy/pypy/changeset/2d6c7a33ca50/

Log:	Make the marker more obvious.

diff --git a/pypy/module/pypyjit/test_pypy_c/model.py b/pypy/module/pypyjit/test_pypy_c/model.py
--- a/pypy/module/pypyjit/test_pypy_c/model.py
+++ b/pypy/module/pypyjit/test_pypy_c/model.py
@@ -387,8 +387,8 @@
                 return ''
             text = str(py.code.Source(src).deindent().indent())
             lines = text.splitlines(True)
-            if opindex is not None and 0 <= opindex < len(lines):
-                lines[opindex] = lines[opindex].rstrip() + '\t<=====\n'
+            if opindex is not None and 0 <= opindex <= len(lines):
+                lines.insert(opindex, '\n\t===== HERE =====\n')
             return ''.join(lines)
         #
         expected_src = self.preprocess_expected_src(expected_src)

From noreply at buildbot.pypy.org  Mon Oct 17 13:54:06 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Oct 2011 13:54:06 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix the tests.
Message-ID: <20111017115406.E8284820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48120:0bd82cbe735b
Date: 2011-10-17 13:53 +0200
http://bitbucket.org/pypy/pypy/changeset/0bd82cbe735b/

Log:	Fix the tests.

diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -41,7 +41,7 @@
             guard_true(i32, descr=...)
             i34 = int_add(i6, 1)
             --TICK--
-            jump(p0, p1, p2, p3, p4, p5, i34, p7, p8, i9, i10, p11, i12, p13, descr=<Loop4>)
+            jump(p0, p1, p2, p3, p4, p5, i34, p7, p8, i9, i10, p11, i12, p13, descr=...)
         """)
 
     def test_long(self):
@@ -106,7 +106,7 @@
             i58 = int_add_ovf(i6, i57)
             guard_no_overflow(descr=...)
             --TICK--
-            jump(p0, p1, p2, p3, p4, p5, i58, i7, descr=<Loop4>)
+            jump(p0, p1, p2, p3, p4, p5, i58, i7, descr=...)
         """)
 
     def test_str_mod(self):

From noreply at buildbot.pypy.org  Mon Oct 17 14:05:02 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Oct 2011 14:05:02 +0200 (CEST)
Subject: [pypy-commit] pypy default: - fix a typo in untested code
Message-ID: <20111017120502.50890820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48121:854bc5e4b87f
Date: 2011-10-17 14:04 +0200
http://bitbucket.org/pypy/pypy/changeset/854bc5e4b87f/

Log:	- fix a typo in untested code
	- change the tests so that this code is tested too
	- independently, fix a test

diff --git a/pypy/jit/metainterp/quasiimmut.py b/pypy/jit/metainterp/quasiimmut.py
--- a/pypy/jit/metainterp/quasiimmut.py
+++ b/pypy/jit/metainterp/quasiimmut.py
@@ -51,13 +51,13 @@
 
 class QuasiImmut(object):
     llopaque = True
+    compress_limit = 30
     
     def __init__(self, cpu):
         self.cpu = cpu
         # list of weakrefs to the LoopTokens that must be invalidated if
         # this value ever changes
         self.looptokens_wrefs = []
-        self.compress_limit = 30
 
     def hide(self):
         qmut_ptr = self.cpu.ts.cast_instance_to_base_ref(self)
@@ -79,7 +79,7 @@
             looptoken = wref()
             if looptoken is not None and not looptoken.invalidated:
                 newlist.append(wref)
-        self.looptokens_wrefs = wref
+        self.looptokens_wrefs = newlist
         self.compress_limit = (len(self.looptokens_wrefs) + 15) * 2
 
     def invalidate(self):
diff --git a/pypy/jit/metainterp/test/test_quasiimmut.py b/pypy/jit/metainterp/test/test_quasiimmut.py
--- a/pypy/jit/metainterp/test/test_quasiimmut.py
+++ b/pypy/jit/metainterp/test/test_quasiimmut.py
@@ -48,6 +48,13 @@
 
 class QuasiImmutTests(object):
 
+    def setup_method(self, meth):
+        self.prev_compress_limit = QuasiImmut.compress_limit
+        QuasiImmut.compress_limit = 1
+
+    def teardown_method(self, meth):
+        QuasiImmut.compress_limit = self.prev_compress_limit
+
     def test_simple_1(self):
         myjitdriver = JitDriver(greens=['foo'], reds=['x', 'total'])
         class Foo:
@@ -453,7 +460,7 @@
         assert f(100, 15) == 3009
         res = self.meta_interp(f, [100, 15])
         assert res == 3009
-        self.check_loops(guard_not_invalidated=2, getfield_gc=0,
+        self.check_loops(guard_not_invalidated=4, getfield_gc=0,
                          getarrayitem_gc=0, getarrayitem_gc_pure=0,
                          call_may_force=0, guard_not_forced=0)
 

From noreply at buildbot.pypy.org  Mon Oct 17 15:17:16 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Oct 2011 15:17:16 +0200 (CEST)
Subject: [pypy-commit] pypy default: Obscure, obscure,
	obscure workaround for the fact
Message-ID: <20111017131716.8EA99820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48122:4b39d3acfd86
Date: 2011-10-17 15:16 +0200
http://bitbucket.org/pypy/pypy/changeset/4b39d3acfd86/

Log:	Obscure, obscure, obscure workaround for the fact that libffi.a
	cannot always be gcc'ed into .so files.

diff --git a/pypy/translator/platform/linux.py b/pypy/translator/platform/linux.py
--- a/pypy/translator/platform/linux.py
+++ b/pypy/translator/platform/linux.py
@@ -1,5 +1,6 @@
 """Support for Linux."""
 
+import sys
 from pypy.translator.platform.posix import BasePosix
 
 class BaseLinux(BasePosix):
@@ -26,7 +27,11 @@
 
     def library_dirs_for_libffi_a(self):
         # places where we need to look for libffi.a
-        return self.library_dirs_for_libffi() + ['/usr/lib']
+        # XXX obscuuure!  only look for libffi.a if run with translate.py
+        if 'translate' in sys.modules:
+            return self.library_dirs_for_libffi() + ['/usr/lib']
+        else:
+            return []
 
 
 class Linux(BaseLinux):

From noreply at buildbot.pypy.org  Mon Oct 17 15:41:32 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Oct 2011 15:41:32 +0200 (CEST)
Subject: [pypy-commit] pypy default: In the memmgr,
 free loops that have been marked as 'invalidated' too.
Message-ID: <20111017134132.B40F2820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48123:1d42dc6c3971
Date: 2011-10-17 15:41 +0200
http://bitbucket.org/pypy/pypy/changeset/1d42dc6c3971/

Log:	In the memmgr, free loops that have been marked as 'invalidated'
	too.

diff --git a/pypy/jit/metainterp/memmgr.py b/pypy/jit/metainterp/memmgr.py
--- a/pypy/jit/metainterp/memmgr.py
+++ b/pypy/jit/metainterp/memmgr.py
@@ -68,7 +68,8 @@
         debug_print("Loop tokens before:", oldtotal)
         max_generation = self.current_generation - (self.max_age-1)
         for looptoken in self.alive_loops.keys():
-            if 0 <= looptoken.generation < max_generation:
+            if (0 <= looptoken.generation < max_generation or
+                looptoken.invalidated):
                 del self.alive_loops[looptoken]
         newtotal = len(self.alive_loops)
         debug_print("Loop tokens freed: ", oldtotal - newtotal)
diff --git a/pypy/jit/metainterp/test/test_memmgr.py b/pypy/jit/metainterp/test/test_memmgr.py
--- a/pypy/jit/metainterp/test/test_memmgr.py
+++ b/pypy/jit/metainterp/test/test_memmgr.py
@@ -18,6 +18,7 @@
 
 class FakeLoopToken:
     generation = 0
+    invalidated = False
 
 
 class _TestMemoryManager:

From noreply at buildbot.pypy.org  Mon Oct 17 17:05:09 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Oct 2011 17:05:09 +0200 (CEST)
Subject: [pypy-commit] pypy gil-improvement: Close branch: (minimally)
 tested 759db9b281eb and found it
Message-ID: <20111017150509.666DE820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: gil-improvement
Changeset: r48124:49d7fea9a1a2
Date: 2011-10-17 16:56 +0200
http://bitbucket.org/pypy/pypy/changeset/49d7fea9a1a2/

Log:	Close branch: (minimally) tested 759db9b281eb and found it working
	correctly on Windows.


From noreply at buildbot.pypy.org  Mon Oct 17 17:08:45 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Oct 2011 17:08:45 +0200 (CEST)
Subject: [pypy-commit] pypy closed-branches: Merge closed head aa2f47508529
 on branch effectinfo-cleanup
Message-ID: <20111017150845.45A54820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: closed-branches
Changeset: r48125:613757a7239f
Date: 2011-10-17 17:06 +0200
http://bitbucket.org/pypy/pypy/changeset/613757a7239f/

Log:	Merge closed head aa2f47508529 on branch effectinfo-cleanup


From noreply at buildbot.pypy.org  Mon Oct 17 17:08:46 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Oct 2011 17:08:46 +0200 (CEST)
Subject: [pypy-commit] pypy closed-branches: Merge closed head 3eba0bde5e95
 on branch faster-sum
Message-ID: <20111017150846.5AE7F820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: closed-branches
Changeset: r48126:8fdb510ebdc6
Date: 2011-10-17 17:06 +0200
http://bitbucket.org/pypy/pypy/changeset/8fdb510ebdc6/

Log:	Merge closed head 3eba0bde5e95 on branch faster-sum


From noreply at buildbot.pypy.org  Mon Oct 17 17:08:47 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Oct 2011 17:08:47 +0200 (CEST)
Subject: [pypy-commit] pypy closed-branches: Merge closed head 5a08d2cfb491
 on branch numpy-comparison
Message-ID: <20111017150847.6F699820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: closed-branches
Changeset: r48127:74004a240f77
Date: 2011-10-17 17:06 +0200
http://bitbucket.org/pypy/pypy/changeset/74004a240f77/

Log:	Merge closed head 5a08d2cfb491 on branch numpy-comparison


From noreply at buildbot.pypy.org  Mon Oct 17 17:08:48 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Oct 2011 17:08:48 +0200 (CEST)
Subject: [pypy-commit] pypy closed-branches: Merge closed head 4723df6a9296
 on branch jit-str_in_preamble
Message-ID: <20111017150848.83B2B820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: closed-branches
Changeset: r48128:0b00ecd2031b
Date: 2011-10-17 17:06 +0200
http://bitbucket.org/pypy/pypy/changeset/0b00ecd2031b/

Log:	Merge closed head 4723df6a9296 on branch jit-str_in_preamble


From noreply at buildbot.pypy.org  Mon Oct 17 17:08:49 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Oct 2011 17:08:49 +0200 (CEST)
Subject: [pypy-commit] pypy closed-branches: Merge closed head df69255f489f
 on branch unroll-if-const
Message-ID: <20111017150849.96471820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: closed-branches
Changeset: r48129:909e228180a0
Date: 2011-10-17 17:06 +0200
http://bitbucket.org/pypy/pypy/changeset/909e228180a0/

Log:	Merge closed head df69255f489f on branch unroll-if-const


From noreply at buildbot.pypy.org  Mon Oct 17 17:08:50 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Oct 2011 17:08:50 +0200 (CEST)
Subject: [pypy-commit] pypy closed-branches: Merge closed head ec0e6a5ca3db
 on branch numpy-singledim
Message-ID: <20111017150850.A88B3820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: closed-branches
Changeset: r48130:f9d5dc481e6e
Date: 2011-10-17 17:06 +0200
http://bitbucket.org/pypy/pypy/changeset/f9d5dc481e6e/

Log:	Merge closed head ec0e6a5ca3db on branch numpy-singledim


From noreply at buildbot.pypy.org  Mon Oct 17 17:08:51 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Oct 2011 17:08:51 +0200 (CEST)
Subject: [pypy-commit] pypy closed-branches: Merge closed head 3453088df88a
 on branch unpack-ints-fast
Message-ID: <20111017150851.BA0B8820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: closed-branches
Changeset: r48131:277298c96126
Date: 2011-10-17 17:06 +0200
http://bitbucket.org/pypy/pypy/changeset/277298c96126/

Log:	Merge closed head 3453088df88a on branch unpack-ints-fast


From noreply at buildbot.pypy.org  Mon Oct 17 17:08:52 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Oct 2011 17:08:52 +0200 (CEST)
Subject: [pypy-commit] pypy closed-branches: Merge closed head 6d556a687a82
 on branch unroll-if-alt
Message-ID: <20111017150852.CB52A820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: closed-branches
Changeset: r48132:d2b474d2e543
Date: 2011-10-17 17:06 +0200
http://bitbucket.org/pypy/pypy/changeset/d2b474d2e543/

Log:	Merge closed head 6d556a687a82 on branch unroll-if-alt


From noreply at buildbot.pypy.org  Mon Oct 17 17:08:53 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Oct 2011 17:08:53 +0200 (CEST)
Subject: [pypy-commit] pypy closed-branches: Merge closed head 89ff5a00266c
 on branch jit-frontend-unescaped
Message-ID: <20111017150853.DDCB2820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: closed-branches
Changeset: r48133:879cb414f7f9
Date: 2011-10-17 17:07 +0200
http://bitbucket.org/pypy/pypy/changeset/879cb414f7f9/

Log:	Merge closed head 89ff5a00266c on branch jit-frontend-unescaped


From noreply at buildbot.pypy.org  Mon Oct 17 17:08:54 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Oct 2011 17:08:54 +0200 (CEST)
Subject: [pypy-commit] pypy closed-branches: Merge closed head 6ab48f8a69a9
 on branch faster-isinstance
Message-ID: <20111017150854.F0122820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: closed-branches
Changeset: r48134:8d2a832064c9
Date: 2011-10-17 17:07 +0200
http://bitbucket.org/pypy/pypy/changeset/8d2a832064c9/

Log:	Merge closed head 6ab48f8a69a9 on branch faster-isinstance


From noreply at buildbot.pypy.org  Mon Oct 17 17:08:56 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Oct 2011 17:08:56 +0200 (CEST)
Subject: [pypy-commit] pypy closed-branches: Merge closed head b8ec4711d8ac
 on branch kill-unary-multimethods
Message-ID: <20111017150856.0F1D7820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: closed-branches
Changeset: r48135:50bc6a8f9a8d
Date: 2011-10-17 17:07 +0200
http://bitbucket.org/pypy/pypy/changeset/50bc6a8f9a8d/

Log:	Merge closed head b8ec4711d8ac on branch kill-unary-multimethods


From noreply at buildbot.pypy.org  Mon Oct 17 17:08:57 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Oct 2011 17:08:57 +0200 (CEST)
Subject: [pypy-commit] pypy closed-branches: Merge closed head 73b24cb9f319
 on branch space-iterator-improvements
Message-ID: <20111017150857.214AB820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: closed-branches
Changeset: r48136:120c0b2a17f5
Date: 2011-10-17 17:07 +0200
http://bitbucket.org/pypy/pypy/changeset/120c0b2a17f5/

Log:	Merge closed head 73b24cb9f319 on branch space-iterator-improvements


From noreply at buildbot.pypy.org  Mon Oct 17 17:08:58 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Oct 2011 17:08:58 +0200 (CEST)
Subject: [pypy-commit] pypy closed-branches: Merge closed head 08713d1ad7d8
 on branch rmmap-nohint
Message-ID: <20111017150858.33D79820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: closed-branches
Changeset: r48137:5eb09f46bbf1
Date: 2011-10-17 17:07 +0200
http://bitbucket.org/pypy/pypy/changeset/5eb09f46bbf1/

Log:	Merge closed head 08713d1ad7d8 on branch rmmap-nohint


From noreply at buildbot.pypy.org  Mon Oct 17 17:08:59 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Oct 2011 17:08:59 +0200 (CEST)
Subject: [pypy-commit] pypy closed-branches: Merge closed head 39fb2ab2290b
 on branch unsigned-dtypes
Message-ID: <20111017150859.44EF3820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: closed-branches
Changeset: r48138:efeade6bcd67
Date: 2011-10-17 17:07 +0200
http://bitbucket.org/pypy/pypy/changeset/efeade6bcd67/

Log:	Merge closed head 39fb2ab2290b on branch unsigned-dtypes


From noreply at buildbot.pypy.org  Mon Oct 17 17:09:00 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Oct 2011 17:09:00 +0200 (CEST)
Subject: [pypy-commit] pypy closed-branches: Merge closed head f3b267864355
 on branch string-promote-2
Message-ID: <20111017150900.5A09A820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: closed-branches
Changeset: r48139:67c6efb4e802
Date: 2011-10-17 17:07 +0200
http://bitbucket.org/pypy/pypy/changeset/67c6efb4e802/

Log:	Merge closed head f3b267864355 on branch string-promote-2


From noreply at buildbot.pypy.org  Mon Oct 17 17:09:01 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Oct 2011 17:09:01 +0200 (CEST)
Subject: [pypy-commit] pypy closed-branches: Merge closed head dfa764e18840
 on branch malloc-value
Message-ID: <20111017150901.6C727820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: closed-branches
Changeset: r48140:5b929443701c
Date: 2011-10-17 17:07 +0200
http://bitbucket.org/pypy/pypy/changeset/5b929443701c/

Log:	Merge closed head dfa764e18840 on branch malloc-value


From noreply at buildbot.pypy.org  Mon Oct 17 17:09:02 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Oct 2011 17:09:02 +0200 (CEST)
Subject: [pypy-commit] pypy closed-branches: Merge closed head 49d7fea9a1a2
 on branch gil-improvement
Message-ID: <20111017150902.7DEC2820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: closed-branches
Changeset: r48141:6d09a049be8b
Date: 2011-10-17 17:07 +0200
http://bitbucket.org/pypy/pypy/changeset/6d09a049be8b/

Log:	Merge closed head 49d7fea9a1a2 on branch gil-improvement


From noreply at buildbot.pypy.org  Mon Oct 17 17:09:03 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Oct 2011 17:09:03 +0200 (CEST)
Subject: [pypy-commit] pypy closed-branches: re-close this branch
Message-ID: <20111017150903.8D8E1820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: closed-branches
Changeset: r48142:4d205329819f
Date: 2011-10-17 17:07 +0200
http://bitbucket.org/pypy/pypy/changeset/4d205329819f/

Log:	re-close this branch


From noreply at buildbot.pypy.org  Mon Oct 17 17:19:41 2011
From: noreply at buildbot.pypy.org (justinpeel)
Date: Mon, 17 Oct 2011 17:19:41 +0200 (CEST)
Subject: [pypy-commit] pypy rgc-mem-pressure: another fix
Message-ID: <20111017151941.31AA1820D7@wyvern.cs.uni-duesseldorf.de>

Author: Justin Peel <notmuchtotell at gmail.com>
Branch: rgc-mem-pressure
Changeset: r48143:3cb6f09da80a
Date: 2011-10-17 09:14 -0600
http://bitbucket.org/pypy/pypy/changeset/3cb6f09da80a/

Log:	another fix

diff --git a/pypy/rpython/memory/gctransform/transform.py b/pypy/rpython/memory/gctransform/transform.py
--- a/pypy/rpython/memory/gctransform/transform.py
+++ b/pypy/rpython/memory/gctransform/transform.py
@@ -560,7 +560,7 @@
         return self.gct_malloc_varsize(*args, **kwds)
 
     def gct_add_memory_pressure(self, hop):
-        if hasattr(self, 'add_memory_pressure_ptr'):
+        if hasattr(self, 'raw_malloc_memory_pressure_ptr'):
             op = hop.spaceop
             size = op.args[0]
             hop.genop("direct_call",
diff --git a/pypy/rpython/memory/test/test_transformed_gc.py b/pypy/rpython/memory/test/test_transformed_gc.py
--- a/pypy/rpython/memory/test/test_transformed_gc.py
+++ b/pypy/rpython/memory/test/test_transformed_gc.py
@@ -1024,7 +1024,7 @@
             llop.gc__collect(lltype.Void)
             return static.p.x + i
         def cleanup():
-            static.p = lltype.nullptr(T1)        
+            static.p = lltype.nullptr(T1)
         return f, cleanup, None
 
     def test_nongc_static_root_minor_collect(self):

From noreply at buildbot.pypy.org  Mon Oct 17 17:19:42 2011
From: noreply at buildbot.pypy.org (justinpeel)
Date: Mon, 17 Oct 2011 17:19:42 +0200 (CEST)
Subject: [pypy-commit] pypy rgc-mem-pressure: more fixes
Message-ID: <20111017151942.5ACE6820D7@wyvern.cs.uni-duesseldorf.de>

Author: Justin Peel <notmuchtotell at gmail.com>
Branch: rgc-mem-pressure
Changeset: r48144:c7d1e2b807a3
Date: 2011-10-17 09:19 -0600
http://bitbucket.org/pypy/pypy/changeset/c7d1e2b807a3/

Log:	more fixes

diff --git a/pypy/rpython/memory/gcwrapper.py b/pypy/rpython/memory/gcwrapper.py
--- a/pypy/rpython/memory/gcwrapper.py
+++ b/pypy/rpython/memory/gcwrapper.py
@@ -66,7 +66,8 @@
         return result
 
     def add_memory_pressure(self, size):
-        self.gc.add_memory_pressure(self)
+        if hasattr(self.gc, 'raw_malloc_memory_pressure'):
+            self.gc.raw_malloc_memory_pressure(size)
 
     def shrink_array(self, p, smallersize):
         if hasattr(self.gc, 'shrink_array'):

From noreply at buildbot.pypy.org  Mon Oct 17 17:24:10 2011
From: noreply at buildbot.pypy.org (justinpeel)
Date: Mon, 17 Oct 2011 17:24:10 +0200 (CEST)
Subject: [pypy-commit] pypy rgc-mem-pressure: merge in default
Message-ID: <20111017152410.9FAEA820D7@wyvern.cs.uni-duesseldorf.de>

Author: Justin Peel <notmuchtotell at gmail.com>
Branch: rgc-mem-pressure
Changeset: r48145:a2265c47e5dc
Date: 2011-10-17 09:23 -0600
http://bitbucket.org/pypy/pypy/changeset/a2265c47e5dc/

Log:	merge in default

diff --git a/pypy/jit/metainterp/graphpage.py b/pypy/jit/metainterp/graphpage.py
--- a/pypy/jit/metainterp/graphpage.py
+++ b/pypy/jit/metainterp/graphpage.py
@@ -12,8 +12,8 @@
     def get_display_text(self):
         return None
 
-def display_loops(loops, errmsg=None, highlight_loops=()):
-    graphs = [(loop, loop in highlight_loops) for loop in loops]    
+def display_loops(loops, errmsg=None, highlight_loops={}):
+    graphs = [(loop, highlight_loops.get(loop, 0)) for loop in loops]    
     for graph, highlight in graphs:
         for op in graph.get_operations():
             if is_interesting_guard(op):
@@ -65,8 +65,7 @@
     def add_graph(self, graph, highlight=False):
         graphindex = len(self.graphs)
         self.graphs.append(graph)
-        if highlight:
-            self.highlight_graphs[graph] = True
+        self.highlight_graphs[graph] = highlight
         for i, op in enumerate(graph.get_operations()):
             self.all_operations[op] = graphindex, i
 
@@ -126,10 +125,13 @@
             self.dotgen.emit('subgraph cluster%d {' % graphindex)
         label = graph.get_display_text()
         if label is not None:
-            if self.highlight_graphs.get(graph):
-                fillcolor = '#f084c2'
+            colorindex = self.highlight_graphs.get(graph, 0)
+            if colorindex == 1:
+                fillcolor = '#f084c2'    # highlighted graph
+            elif colorindex == 2:
+                fillcolor = '#808080'    # invalidated graph
             else:
-                fillcolor = '#84f0c2'
+                fillcolor = '#84f0c2'    # normal color
             self.dotgen.emit_node(graphname, shape="octagon",
                                   label=label, fillcolor=fillcolor)
             self.pendingedges.append((graphname,
diff --git a/pypy/jit/metainterp/history.py b/pypy/jit/metainterp/history.py
--- a/pypy/jit/metainterp/history.py
+++ b/pypy/jit/metainterp/history.py
@@ -732,6 +732,7 @@
     failed_states = None
     retraced_count = 0
     terminating = False # see TerminatingLoopToken in compile.py
+    invalidated = False
     outermost_jitdriver_sd = None
     # and more data specified by the backend when the loop is compiled
     number = -1
@@ -934,6 +935,7 @@
         self.loops = []
         self.locations = []
         self.aborted_keys = []
+        self.invalidated_token_numbers = set()
 
     def set_history(self, history):
         self.operations = history.operations
@@ -1012,7 +1014,12 @@
             if loop in loops:
                 loops.remove(loop)
             loops.append(loop)
-        display_loops(loops, errmsg, extraloops)
+        highlight_loops = dict.fromkeys(extraloops, 1)
+        for loop in loops:
+            if hasattr(loop, '_looptoken_number') and (
+                    loop._looptoken_number in self.invalidated_token_numbers):
+                highlight_loops.setdefault(loop, 2)
+        display_loops(loops, errmsg, highlight_loops)
 
 # ----------------------------------------------------------------
 
diff --git a/pypy/jit/metainterp/memmgr.py b/pypy/jit/metainterp/memmgr.py
--- a/pypy/jit/metainterp/memmgr.py
+++ b/pypy/jit/metainterp/memmgr.py
@@ -68,7 +68,8 @@
         debug_print("Loop tokens before:", oldtotal)
         max_generation = self.current_generation - (self.max_age-1)
         for looptoken in self.alive_loops.keys():
-            if 0 <= looptoken.generation < max_generation:
+            if (0 <= looptoken.generation < max_generation or
+                looptoken.invalidated):
                 del self.alive_loops[looptoken]
         newtotal = len(self.alive_loops)
         debug_print("Loop tokens freed: ", oldtotal - newtotal)
diff --git a/pypy/jit/metainterp/quasiimmut.py b/pypy/jit/metainterp/quasiimmut.py
--- a/pypy/jit/metainterp/quasiimmut.py
+++ b/pypy/jit/metainterp/quasiimmut.py
@@ -2,6 +2,7 @@
 from pypy.rpython.lltypesystem import lltype, rclass
 from pypy.rpython.annlowlevel import cast_base_ptr_to_instance
 from pypy.jit.metainterp.history import AbstractDescr
+from pypy.rlib.objectmodel import we_are_translated
 
 
 def get_mutate_field_name(fieldname):
@@ -50,13 +51,13 @@
 
 class QuasiImmut(object):
     llopaque = True
+    compress_limit = 30
     
     def __init__(self, cpu):
         self.cpu = cpu
         # list of weakrefs to the LoopTokens that must be invalidated if
         # this value ever changes
         self.looptokens_wrefs = []
-        self.compress_limit = 30
 
     def hide(self):
         qmut_ptr = self.cpu.ts.cast_instance_to_base_ref(self)
@@ -73,8 +74,12 @@
         self.looptokens_wrefs.append(wref_looptoken)
 
     def compress_looptokens_list(self):
-        self.looptokens_wrefs = [wref for wref in self.looptokens_wrefs
-                                      if wref() is not None]
+        newlist = []
+        for wref in self.looptokens_wrefs:
+            looptoken = wref()
+            if looptoken is not None and not looptoken.invalidated:
+                newlist.append(wref)
+        self.looptokens_wrefs = newlist
         self.compress_limit = (len(self.looptokens_wrefs) + 15) * 2
 
     def invalidate(self):
@@ -85,8 +90,12 @@
         self.looptokens_wrefs = []
         for wref in wrefs:
             looptoken = wref()
-            if looptoken is not None:
+            if looptoken is not None and not looptoken.invalidated:
+                looptoken.invalidated = True
                 self.cpu.invalidate_loop(looptoken)
+                if not we_are_translated():
+                    self.cpu.stats.invalidated_token_numbers.add(
+                        looptoken.number)
 
 
 class QuasiImmutDescr(AbstractDescr):
diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py
--- a/pypy/jit/metainterp/resoperation.py
+++ b/pypy/jit/metainterp/resoperation.py
@@ -431,11 +431,11 @@
     'INT_IS_TRUE/1b',
     'INT_NEG/1',
     'INT_INVERT/1',
+    #
+    'SAME_AS/1',      # gets a Const or a Box, turns it into another Box
     'CAST_PTR_TO_INT/1',
     'CAST_INT_TO_PTR/1',
     #
-    'SAME_AS/1',      # gets a Const or a Box, turns it into another Box
-    #
     'PTR_EQ/2b',
     'PTR_NE/2b',
     'CAST_OPAQUE_PTR/1b',
diff --git a/pypy/jit/metainterp/test/test_memmgr.py b/pypy/jit/metainterp/test/test_memmgr.py
--- a/pypy/jit/metainterp/test/test_memmgr.py
+++ b/pypy/jit/metainterp/test/test_memmgr.py
@@ -18,6 +18,7 @@
 
 class FakeLoopToken:
     generation = 0
+    invalidated = False
 
 
 class _TestMemoryManager:
diff --git a/pypy/jit/metainterp/test/test_quasiimmut.py b/pypy/jit/metainterp/test/test_quasiimmut.py
--- a/pypy/jit/metainterp/test/test_quasiimmut.py
+++ b/pypy/jit/metainterp/test/test_quasiimmut.py
@@ -48,6 +48,13 @@
 
 class QuasiImmutTests(object):
 
+    def setup_method(self, meth):
+        self.prev_compress_limit = QuasiImmut.compress_limit
+        QuasiImmut.compress_limit = 1
+
+    def teardown_method(self, meth):
+        QuasiImmut.compress_limit = self.prev_compress_limit
+
     def test_simple_1(self):
         myjitdriver = JitDriver(greens=['foo'], reds=['x', 'total'])
         class Foo:
@@ -289,7 +296,7 @@
             return total
 
         res = self.meta_interp(main, [])
-        self.check_loop_count(9)
+        self.check_tree_loop_count(6)
         assert res == main()
 
     def test_change_during_running(self):
@@ -317,7 +324,7 @@
         assert f(100, 15) == 3009
         res = self.meta_interp(f, [100, 15])
         assert res == 3009
-        self.check_loops(guard_not_invalidated=2, getfield_gc=0,
+        self.check_loops(guard_not_invalidated=4, getfield_gc=0,
                          call_may_force=0, guard_not_forced=0)
 
     def test_list_simple_1(self):
@@ -453,10 +460,30 @@
         assert f(100, 15) == 3009
         res = self.meta_interp(f, [100, 15])
         assert res == 3009
-        self.check_loops(guard_not_invalidated=2, getfield_gc=0,
+        self.check_loops(guard_not_invalidated=4, getfield_gc=0,
                          getarrayitem_gc=0, getarrayitem_gc_pure=0,
                          call_may_force=0, guard_not_forced=0)
 
+    def test_invalidated_loop_is_not_used_any_more_as_target(self):
+        myjitdriver = JitDriver(greens=['foo'], reds=['x'])
+        class Foo:
+            _immutable_fields_ = ['step?']
+        @dont_look_inside
+        def residual(x, foo):
+            if x == 20:
+                foo.step = 1
+        def f(x):
+            foo = Foo()
+            foo.step = 2
+            while x > 0:
+                myjitdriver.jit_merge_point(foo=foo, x=x)
+                residual(x, foo)
+                x -= foo.step
+            return foo.step
+        res = self.meta_interp(f, [60])
+        assert res == 1
+        self.check_tree_loop_count(4)   # at least not 2 like before
+
 
 class TestLLtypeGreenFieldsTests(QuasiImmutTests, LLJitMixin):
     pass
diff --git a/pypy/jit/metainterp/test/test_resume.py b/pypy/jit/metainterp/test/test_resume.py
--- a/pypy/jit/metainterp/test/test_resume.py
+++ b/pypy/jit/metainterp/test/test_resume.py
@@ -1,3 +1,4 @@
+from __future__ import with_statement
 import py
 from pypy.rpython.lltypesystem import lltype, llmemory, rffi
 from pypy.jit.metainterp.optimizeopt.optimizer import OptValue
diff --git a/pypy/jit/metainterp/warmstate.py b/pypy/jit/metainterp/warmstate.py
--- a/pypy/jit/metainterp/warmstate.py
+++ b/pypy/jit/metainterp/warmstate.py
@@ -178,7 +178,7 @@
         if self.compiled_merge_points_wref is not None:
             for wref in self.compiled_merge_points_wref:
                 looptoken = wref()
-                if looptoken is not None:
+                if looptoken is not None and not looptoken.invalidated:
                     result.append(looptoken)
         return result
 
diff --git a/pypy/module/__builtin__/__init__.py b/pypy/module/__builtin__/__init__.py
--- a/pypy/module/__builtin__/__init__.py
+++ b/pypy/module/__builtin__/__init__.py
@@ -119,7 +119,7 @@
            builtin = space.interpclass_w(w_builtin)
            if isinstance(builtin, module.Module):
                return builtin
-       # no builtin! make a default one.  Given them None, at least.
+       # no builtin! make a default one.  Give them None, at least.
        builtin = module.Module(space, None)
        space.setitem(builtin.w_dict, space.wrap('None'), space.w_None)
        return builtin
diff --git a/pypy/module/__builtin__/compiling.py b/pypy/module/__builtin__/compiling.py
--- a/pypy/module/__builtin__/compiling.py
+++ b/pypy/module/__builtin__/compiling.py
@@ -97,6 +97,9 @@
     elif space.is_w(w_locals, space.w_None):
         w_locals = w_globals
 
-    space.builtin.pick_builtin(w_globals)
+    # xxx removed: adding '__builtins__' to the w_globals dict, if there
+    # is none.  This logic was removed as costly (it requires to get at
+    # the gettopframe_nohidden()).  I bet no test fails, and it's a really
+    # obscure case.
 
     return codeobj.exec_code(space, w_globals, w_locals)
diff --git a/pypy/module/pypyjit/test_pypy_c/model.py b/pypy/module/pypyjit/test_pypy_c/model.py
--- a/pypy/module/pypyjit/test_pypy_c/model.py
+++ b/pypy/module/pypyjit/test_pypy_c/model.py
@@ -387,8 +387,8 @@
                 return ''
             text = str(py.code.Source(src).deindent().indent())
             lines = text.splitlines(True)
-            if opindex is not None and 0 <= opindex < len(lines):
-                lines[opindex] = lines[opindex].rstrip() + '\t<=====\n'
+            if opindex is not None and 0 <= opindex <= len(lines):
+                lines.insert(opindex, '\n\t===== HERE =====\n')
             return ''.join(lines)
         #
         expected_src = self.preprocess_expected_src(expected_src)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -41,7 +41,7 @@
             guard_true(i32, descr=...)
             i34 = int_add(i6, 1)
             --TICK--
-            jump(p0, p1, p2, p3, p4, p5, i34, p7, p8, i9, i10, p11, i12, p13, descr=<Loop4>)
+            jump(p0, p1, p2, p3, p4, p5, i34, p7, p8, i9, i10, p11, i12, p13, descr=...)
         """)
 
     def test_long(self):
@@ -106,7 +106,7 @@
             i58 = int_add_ovf(i6, i57)
             guard_no_overflow(descr=...)
             --TICK--
-            jump(p0, p1, p2, p3, p4, p5, i58, i7, descr=<Loop4>)
+            jump(p0, p1, p2, p3, p4, p5, i58, i7, descr=...)
         """)
 
     def test_str_mod(self):
diff --git a/pypy/translator/platform/linux.py b/pypy/translator/platform/linux.py
--- a/pypy/translator/platform/linux.py
+++ b/pypy/translator/platform/linux.py
@@ -1,5 +1,6 @@
 """Support for Linux."""
 
+import sys
 from pypy.translator.platform.posix import BasePosix
 
 class BaseLinux(BasePosix):
@@ -26,7 +27,11 @@
 
     def library_dirs_for_libffi_a(self):
         # places where we need to look for libffi.a
-        return self.library_dirs_for_libffi() + ['/usr/lib']
+        # XXX obscuuure!  only look for libffi.a if run with translate.py
+        if 'translate' in sys.modules:
+            return self.library_dirs_for_libffi() + ['/usr/lib']
+        else:
+            return []
 
 
 class Linux(BaseLinux):

From noreply at buildbot.pypy.org  Mon Oct 17 17:42:50 2011
From: noreply at buildbot.pypy.org (justinpeel)
Date: Mon, 17 Oct 2011 17:42:50 +0200 (CEST)
Subject: [pypy-commit] pypy rgc-mem-pressure: fix fix - getting closer I hope
Message-ID: <20111017154250.4177A820D7@wyvern.cs.uni-duesseldorf.de>

Author: Justin Peel <notmuchtotell at gmail.com>
Branch: rgc-mem-pressure
Changeset: r48146:172e741aa26f
Date: 2011-10-17 09:42 -0600
http://bitbucket.org/pypy/pypy/changeset/172e741aa26f/

Log:	fix fix - getting closer I hope

diff --git a/pypy/rpython/memory/gctransform/transform.py b/pypy/rpython/memory/gctransform/transform.py
--- a/pypy/rpython/memory/gctransform/transform.py
+++ b/pypy/rpython/memory/gctransform/transform.py
@@ -559,11 +559,11 @@
     def gct_malloc_nonmovable_varsize(self, *args, **kwds):
         return self.gct_malloc_varsize(*args, **kwds)
 
-    def gct_add_memory_pressure(self, hop):
+    def gct_gc_add_memory_pressure(self, hop):
         if hasattr(self, 'raw_malloc_memory_pressure_ptr'):
             op = hop.spaceop
             size = op.args[0]
-            hop.genop("direct_call",
+            return hop.genop("direct_call",
                           [self.raw_malloc_memory_pressure_ptr,
                            size])
 

From noreply at buildbot.pypy.org  Mon Oct 17 19:07:14 2011
From: noreply at buildbot.pypy.org (hager)
Date: Mon, 17 Oct 2011 19:07:14 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: Ooops, forgot to remove print.
Message-ID: <20111017170714.1558782A8A@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r48148:db91d415ffd5
Date: 2011-10-17 18:47 +0200
http://bitbucket.org/pypy/pypy/changeset/db91d415ffd5/

Log:	Ooops, forgot to remove print.

diff --git a/pypy/jit/backend/ppc/ppcgen/opassembler.py b/pypy/jit/backend/ppc/ppcgen/opassembler.py
--- a/pypy/jit/backend/ppc/ppcgen/opassembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/opassembler.py
@@ -35,7 +35,6 @@
         elif l1.is_imm():
             self.mc.subi(res.value, l0.value, l1.value)
         else:
-            print "unten"
             self.mc.sub(res.value, l0.value, l1.value)
    
     emit_int_le = gen_emit_cmp_op(c.LE)   

From noreply at buildbot.pypy.org  Mon Oct 17 19:07:12 2011
From: noreply at buildbot.pypy.org (hager)
Date: Mon, 17 Oct 2011 19:07:12 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: Implemented INT_SUB.
Message-ID: <20111017170712.E353D820D7@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r48147:ef026b253cbb
Date: 2011-10-17 18:43 +0200
http://bitbucket.org/pypy/pypy/changeset/ef026b253cbb/

Log:	Implemented INT_SUB.

diff --git a/pypy/jit/backend/ppc/ppcgen/opassembler.py b/pypy/jit/backend/ppc/ppcgen/opassembler.py
--- a/pypy/jit/backend/ppc/ppcgen/opassembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/opassembler.py
@@ -26,6 +26,17 @@
             self.mc.addi(res.value, l0.value, l1.value)
         else:
             self.mc.add(res.value, l0.value, l1.value)
+
+    def emit_int_sub(self, op, arglocs, regalloc):
+        l0, l1, res = arglocs
+        if l0.is_imm():
+            self.mc.load_imm(r.r0, l0.value)
+            self.mc.sub(res.value, r.r0.value, l1.value)
+        elif l1.is_imm():
+            self.mc.subi(res.value, l0.value, l1.value)
+        else:
+            print "unten"
+            self.mc.sub(res.value, l0.value, l1.value)
    
     emit_int_le = gen_emit_cmp_op(c.LE)   
 
diff --git a/pypy/jit/backend/ppc/ppcgen/regalloc.py b/pypy/jit/backend/ppc/ppcgen/regalloc.py
--- a/pypy/jit/backend/ppc/ppcgen/regalloc.py
+++ b/pypy/jit/backend/ppc/ppcgen/regalloc.py
@@ -198,6 +198,29 @@
         res = self.force_allocate_reg(op.result)
         return locs + [res]
 
+    def prepare_int_sub(self, op):
+        boxes = list(op.getarglist())
+        b0, b1 = boxes
+        imm_b0 = _check_imm_arg(b0)
+        imm_b1 = _check_imm_arg(b1)
+        if not imm_b0 and imm_b1:
+            l0, box = self._ensure_value_is_boxed(b0, boxes)
+            l1 = self.make_sure_var_in_reg(b1, [b0])
+            boxes.append(box)
+        elif imm_b0 and not imm_b1:
+            l0 = self.make_sure_var_in_reg(b0)
+            l1, box = self._ensure_value_is_boxed(b1, boxes)
+            boxes.append(box)
+        else:
+            l0, box = self._ensure_value_is_boxed(b0, boxes)
+            boxes.append(box)
+            l1, box = self._ensure_value_is_boxed(b1, boxes)
+            boxes.append(box)
+        locs = [l0, l1]
+        self.possibly_free_vars(boxes)
+        res = self.force_allocate_reg(op.result)
+        return locs + [res]
+
     def prepare_finish(self, op):
         args = [locations.imm(self.frame_manager.frame_depth)]
         for i in range(op.numargs()):

From noreply at buildbot.pypy.org  Mon Oct 17 19:07:15 2011
From: noreply at buildbot.pypy.org (hager)
Date: Mon, 17 Oct 2011 19:07:15 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: Reimplemented INT_MUL.
Message-ID: <20111017170715.4225B820D7@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r48149:f853fd60f4d0
Date: 2011-10-17 19:06 +0200
http://bitbucket.org/pypy/pypy/changeset/f853fd60f4d0/

Log:	Reimplemented INT_MUL.

diff --git a/pypy/jit/backend/ppc/ppcgen/opassembler.py b/pypy/jit/backend/ppc/ppcgen/opassembler.py
--- a/pypy/jit/backend/ppc/ppcgen/opassembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/opassembler.py
@@ -18,6 +18,10 @@
 
 class OpAssembler(object):
         
+    # ********************************************************
+    # *               I N T    O P E R A T I O N S           *
+    # ********************************************************
+
     def emit_int_add(self, op, arglocs, regalloc):
         l0, l1, res = arglocs
         if l0.is_imm():
@@ -36,7 +40,11 @@
             self.mc.subi(res.value, l0.value, l1.value)
         else:
             self.mc.sub(res.value, l0.value, l1.value)
-   
+ 
+    def emit_int_mul(self, op, arglocs, regalloc):
+        reg1, reg2, res = arglocs
+        self.mc.mullw(res.value, reg1.value, reg2.value)
+
     emit_int_le = gen_emit_cmp_op(c.LE)   
 
     def _emit_guard(self, op, arglocs, fcond, save_exc=False,
diff --git a/pypy/jit/backend/ppc/ppcgen/regalloc.py b/pypy/jit/backend/ppc/ppcgen/regalloc.py
--- a/pypy/jit/backend/ppc/ppcgen/regalloc.py
+++ b/pypy/jit/backend/ppc/ppcgen/regalloc.py
@@ -159,7 +159,7 @@
                 box = TempInt()
             loc = self.force_allocate_reg(box, forbidden_vars=forbidden_vars)
             imm = self.rm.convert_to_imm(thing)
-            self.assembler.load_word(loc, imm)
+            self.assembler.load_imm(loc.value, imm.value)
         else:
             loc = self.make_sure_var_in_reg(thing,
                     forbidden_vars=forbidden_vars)
@@ -221,6 +221,21 @@
         res = self.force_allocate_reg(op.result)
         return locs + [res]
 
+    def prepare_int_mul(self, op):
+        boxes = list(op.getarglist())
+        b0, b1 = boxes
+
+        reg1, box = self._ensure_value_is_boxed(b0, forbidden_vars=boxes)
+        boxes.append(box)
+        reg2, box = self._ensure_value_is_boxed(b1, forbidden_vars=boxes)
+        boxes.append(box)
+
+        self.possibly_free_vars(boxes)
+        self.possibly_free_vars_for_op(op)
+        res = self.force_allocate_reg(op.result)
+        self.possibly_free_var(op.result)
+        return [reg1, reg2, res]
+
     def prepare_finish(self, op):
         args = [locations.imm(self.frame_manager.frame_depth)]
         for i in range(op.numargs()):

From noreply at buildbot.pypy.org  Mon Oct 17 19:38:54 2011
From: noreply at buildbot.pypy.org (justinpeel)
Date: Mon, 17 Oct 2011 19:38:54 +0200 (CEST)
Subject: [pypy-commit] pypy rgc-mem-pressure: put in hash size calculation.
 Not sure why I can't use rffi_platform
Message-ID: <20111017173854.5C4A5820D7@wyvern.cs.uni-duesseldorf.de>

Author: Justin Peel <notmuchtotell at gmail.com>
Branch: rgc-mem-pressure
Changeset: r48150:c790df25f234
Date: 2011-10-17 11:38 -0600
http://bitbucket.org/pypy/pypy/changeset/c790df25f234/

Log:	put in hash size calculation. Not sure why I can't use rffi_platform

diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py
--- a/pypy/module/_hashlib/interp_hashlib.py
+++ b/pypy/module/_hashlib/interp_hashlib.py
@@ -29,7 +29,7 @@
                                  space.wrap("unknown hash function"))
         ctx = lltype.malloc(ropenssl.EVP_MD_CTX.TO, flavor='raw')
         ropenssl.EVP_DigestInit(ctx, digest)
-        rgc.add_memory_pressure(184 + self._digest_size())
+        rgc.add_memory_pressure(ropenssl.HASH_MALLOC_SIZE + self._digest_size())
         self.ctx = ctx
 
     def __del__(self):
diff --git a/pypy/rlib/ropenssl.py b/pypy/rlib/ropenssl.py
--- a/pypy/rlib/ropenssl.py
+++ b/pypy/rlib/ropenssl.py
@@ -277,6 +277,19 @@
 EVP_MD_CTX_cleanup = external(
     'EVP_MD_CTX_cleanup', [EVP_MD_CTX], rffi.INT, threadsafe=False)
 
+# size of EVP_MD, EVP_MD_CTX plus their points. Used for adding 
+# memory pressure
+
+# not sure why this code does not work
+#HASH_MALLOC_SIZE = rffi_platform.sizeof(EVP_MD.TO, eci) \
+#    + rffi_platform.sizeof(EVP_MD_CTX.TO, eci) \
+#    + rffi.sizeof(EVP_MD) + rffi.sizeof(EVP_MD_CTX)
+
+# but this code does
+HASH_MALLOC_SIZE = EVP_MD.TO.hints['getsize']() \
+     + EVP_MD_CTX.TO.hints['getsize']() \
+     + rffi.sizeof(EVP_MD) + rffi.sizeof(EVP_MD_CTX)
+
 def init_ssl():
     libssl_SSL_load_error_strings()
     libssl_SSL_library_init()

From noreply at buildbot.pypy.org  Mon Oct 17 19:48:59 2011
From: noreply at buildbot.pypy.org (lac)
Date: Mon, 17 Oct 2011 19:48:59 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: get this thing ready to mail
Message-ID: <20111017174859.7D73F820D7@wyvern.cs.uni-duesseldorf.de>

Author: Laura Creighton <lac at openend.se>
Branch: extradoc
Changeset: r3929:2492c770a394
Date: 2011-10-17 19:48 +0200
http://bitbucket.org/pypy/extradoc/changeset/2492c770a394/

Log:	get this thing ready to mail

diff --git a/sprintinfo/gothenburg-2011-2/announce.txt b/sprintinfo/gothenburg-2011-2/announce.txt
--- a/sprintinfo/gothenburg-2011-2/announce.txt
+++ b/sprintinfo/gothenburg-2011-2/announce.txt
@@ -2,20 +2,21 @@
 =========================================================
 
 The next PyPy sprint will be in Gothenburg, Sweden. It is a public sprint, 
-suitable for newcomers.  We'll focus on making a public kick off for
-both the numpy/pypy integration project and the py3k interpreter project,
-as well as whatever interests the Sprint attendees.
+suitable for newcomers.  We'll focus on making a public kickoff for
+both the `numpy/pypy integration project`_
+and the `Py3k support project`_,
+as well as whatever interests the Sprint attendees.  Since both of these
+projects are very new, there will be plenty of work suitable for newcomers
+to PyPy.
+ 
+.. _`numpy/pypy integration project`: http://pypy.org/numpydonate.html
+.. _`Py3k interpreter project`: http://pypy.org/py3donate.html
 
-Topics and goals
-----------------
-
-The main goal is to XXX
-
-Other topics:
+Other topics might include:
 
 - Helping people get their code running with PyPy
 
-- maybe work on FSCon talk
+- work on a FSCons talk?
 
 - state of the STM Vinnova project (We most likely, but not for certain will
   know whether or not we are approved by this date.)
@@ -33,9 +34,6 @@
 Location
 --------
 
-XXX I am fine with having the sprint here.  Or we could have it at OE.
-    It's the 'sleeping at my house' part that isn't working.
-
 The sprint will be held in the apartment of Laura Creighton and Jacob Hall&#65533;n
 which is at G&#65533;tabergsgatan 22 in Gothenburg, Sweden.  Here is a map_.  This is
 in central Gothenburg.  It is between the tram_ stops of Vasaplatsen and 
@@ -61,7 +59,8 @@
 from 10:00 until people have had enough.  It's a good idea to arrive a
 day before the sprint starts and leave a day later.  In the middle of
 the sprint there usually is a break day and it's usually ok to take
-half-days off if you feel like it.
+half-days off if you feel like it.  Of course, many of you may be interested
+in sticking around for FSCons, held the weekend after the sprint.
 
 
 Good to Know

From noreply at buildbot.pypy.org  Mon Oct 17 19:57:39 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Mon, 17 Oct 2011 19:57:39 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: This translation option was removed
Message-ID: <20111017175739.73618820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48151:028879834329
Date: 2011-10-17 19:50 +0200
http://bitbucket.org/pypy/pypy/changeset/028879834329/

Log:	This translation option was removed

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -348,7 +348,6 @@
     # all the good optimizations for PyPy should be listed here
     if level in ['2', '3', 'jit']:
         config.objspace.opcodes.suggest(CALL_METHOD=True)
-        config.objspace.std.suggest(withrangelist=True)
         config.objspace.std.suggest(withmethodcache=True)
         config.objspace.std.suggest(withprebuiltchar=True)
         config.objspace.std.suggest(builtinshortcut=True)
@@ -367,7 +366,6 @@
     # memory-saving optimizations
     if level == 'mem':
         config.objspace.std.suggest(withprebuiltint=True)
-        config.objspace.std.suggest(withrangelist=True)
         config.objspace.std.suggest(withprebuiltchar=True)
         config.objspace.std.suggest(withmapdict=True)
         config.objspace.std.suggest(withstrslice=True)

From noreply at buildbot.pypy.org  Mon Oct 17 19:57:40 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Mon, 17 Oct 2011 19:57:40 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: In flow object space,
 restore opcodes and operations removed in the base
Message-ID: <20111017175740.A5517820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48152:211a8e36ea40
Date: 2011-10-17 19:50 +0200
http://bitbucket.org/pypy/pypy/changeset/211a8e36ea40/

Log:	In flow object space, restore opcodes and operations removed in the
	base objectspace

diff --git a/pypy/objspace/flow/flowcontext.py b/pypy/objspace/flow/flowcontext.py
--- a/pypy/objspace/flow/flowcontext.py
+++ b/pypy/objspace/flow/flowcontext.py
@@ -425,6 +425,100 @@
     def MAP_ADD(self, oparg, next_instr):
         raise NotImplementedError("MAP_ADD")
 
+    # opcodes removed or heavily changed in python 3
+
+    @jit.unroll_safe
+    def RAISE_VARARGS(self, nbargs, next_instr):
+        space = self.space
+        if nbargs == 0:
+            frame = self
+            ec = self.space.getexecutioncontext()
+            while frame:
+                if frame.last_exception is not None:
+                    operror = ec._convert_exc(frame.last_exception)
+                    break
+                frame = frame.f_backref()
+            else:
+                raise OperationError(space.w_TypeError,
+                    space.wrap("raise: no active exception to re-raise"))
+            # re-raise, no new traceback obj will be attached
+            self.last_exception = operror
+            from pypy.interpreter.pyopcode import Reraise
+            raise Reraise
+
+        w_value = w_traceback = space.w_None
+        if nbargs >= 3:
+            w_traceback = self.popvalue()
+        if nbargs >= 2:
+            w_value = self.popvalue()
+        if 1:
+            w_type = self.popvalue()
+        operror = OperationError(w_type, w_value)
+        operror.normalize_exception(space)
+        raise operror
+
+    def slice(self, w_start, w_end):
+        w_obj = self.popvalue()
+        w_result = self.space.getslice(w_obj, w_start, w_end)
+        self.pushvalue(w_result)
+
+    def SLICE_0(self, oparg, next_instr):
+        self.slice(self.space.w_None, self.space.w_None)
+
+    def SLICE_1(self, oparg, next_instr):
+        w_start = self.popvalue()
+        self.slice(w_start, self.space.w_None)
+
+    def SLICE_2(self, oparg, next_instr):
+        w_end = self.popvalue()
+        self.slice(self.space.w_None, w_end)
+
+    def SLICE_3(self, oparg, next_instr):
+        w_end = self.popvalue()
+        w_start = self.popvalue()
+        self.slice(w_start, w_end)
+
+    def storeslice(self, w_start, w_end):
+        w_obj = self.popvalue()
+        w_newvalue = self.popvalue()
+        self.space.setslice(w_obj, w_start, w_end, w_newvalue)
+
+    def STORE_SLICE_0(self, oparg, next_instr):
+        self.storeslice(self.space.w_None, self.space.w_None)
+
+    def STORE_SLICE_1(self, oparg, next_instr):
+        w_start = self.popvalue()
+        self.storeslice(w_start, self.space.w_None)
+
+    def STORE_SLICE_2(self, oparg, next_instr):
+        w_end = self.popvalue()
+        self.storeslice(self.space.w_None, w_end)
+
+    def STORE_SLICE_3(self, oparg, next_instr):
+        w_end = self.popvalue()
+        w_start = self.popvalue()
+        self.storeslice(w_start, w_end)
+
+    def deleteslice(self, w_start, w_end):
+        w_obj = self.popvalue()
+        self.space.delslice(w_obj, w_start, w_end)
+
+    def DELETE_SLICE_0(self, oparg, next_instr):
+        self.deleteslice(self.space.w_None, self.space.w_None)
+
+    def DELETE_SLICE_1(self, oparg, next_instr):
+        w_start = self.popvalue()
+        self.deleteslice(w_start, self.space.w_None)
+
+    def DELETE_SLICE_2(self, oparg, next_instr):
+        w_end = self.popvalue()
+        self.deleteslice(self.space.w_None, w_end)
+
+    def DELETE_SLICE_3(self, oparg, next_instr):
+        w_end = self.popvalue()
+        w_start = self.popvalue()
+        self.deleteslice(w_start, w_end)
+
     def make_arguments(self, nargs):
         return ArgumentsForTranslation(self.space, self.peekvalues(nargs))
     def argument_factory(self, *args):
diff --git a/pypy/objspace/flow/operation.py b/pypy/objspace/flow/operation.py
--- a/pypy/objspace/flow/operation.py
+++ b/pypy/objspace/flow/operation.py
@@ -14,6 +14,13 @@
 from pypy.rlib.rarithmetic import ovfcheck, ovfcheck_lshift
 from pypy.objspace.flow import model
 
+MethodTable = ObjSpace.MethodTable[:]
+MethodTable.extend([
+    ('getslice',        'getslice',  3, ['__getslice__']),
+    ('setslice',        'setslice',  4, ['__setslice__']),
+    ('delslice',        'delslice',  3, ['__delslice__']),
+])
+
 
 class OperationThatShouldNotBePropagatedError(OperationError):
     pass
@@ -235,7 +242,7 @@
 
 def setup():
     # insert all operators
-    for line in ObjSpace.MethodTable:
+    for line in MethodTable:
         name = line[0]
         if hasattr(operator, name):
             Table.append((name, getattr(operator, name)))
@@ -246,7 +253,7 @@
         if func not in OperationName:
             OperationName[func] = name
     # check that the result is complete
-    for line in ObjSpace.MethodTable:
+    for line in MethodTable:
         name = line[0]
         Arity[name] = line[2]
         assert name in FunctionByName
@@ -428,6 +435,6 @@
 
 def add_operations(fs):
     """Add function operations to the flow space."""
-    for line in ObjSpace.MethodTable:
+    for line in MethodTable:
         make_op(fs, *line)
     special_overrides(fs)

From noreply at buildbot.pypy.org  Mon Oct 17 19:57:41 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Mon, 17 Oct 2011 19:57:41 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Convert more applevel code to py3k syntax
Message-ID: <20111017175741.D5DAD820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48153:bb3821775a33
Date: 2011-10-17 19:50 +0200
http://bitbucket.org/pypy/pypy/changeset/bb3821775a33/

Log:	Convert more applevel code to py3k syntax

diff --git a/pypy/module/__builtin__/operation.py b/pypy/module/__builtin__/operation.py
--- a/pypy/module/__builtin__/operation.py
+++ b/pypy/module/__builtin__/operation.py
@@ -161,7 +161,7 @@
 
     def iter_sentinel(callable_, sentinel):
         if not callable(callable_):
-            raise TypeError, 'iter(v, w): v must be callable'
+            raise TypeError('iter(v, w): v must be callable')
         return iter_generator(callable_, sentinel)
 
 ''', filename=__file__).interphook("iter_sentinel")
diff --git a/pypy/translator/goal/app_main.py b/pypy/translator/goal/app_main.py
--- a/pypy/translator/goal/app_main.py
+++ b/pypy/translator/goal/app_main.py
@@ -122,16 +122,16 @@
         optitems = options.items()
         optitems.sort()
         for name, value in optitems:
-            print ' %51s: %s' % (name, value)
+            print(' %51s: %s' % (name, value))
     raise SystemExit
 
 def print_help(*args):
-    print 'usage: %s [options] [-c cmd|-m mod|file.py|-] [arg...]' % (
-        sys.executable,)
-    print __doc__.rstrip()
+    print('usage: %s [options] [-c cmd|-m mod|file.py|-] [arg...]' % (
+        sys.executable,))
+    print(__doc__.rstrip())
     if 'pypyjit' in sys.builtin_module_names:
         _print_jit_help()
-    print
+    print()
     raise SystemExit
 
 def _print_jit_help():
@@ -139,18 +139,18 @@
     items = pypyjit.defaults.items()
     items.sort()
     for key, value in items:
-        print '  --jit %s=N %slow-level JIT parameter (default %s)' % (
-            key, ' '*(18-len(key)), value)
-    print '  --jit off                  turn off the JIT'
+        print('  --jit %s=N %slow-level JIT parameter (default %s)' % (
+            key, ' '*(18-len(key)), value))
+    print('  --jit off                  turn off the JIT')
 
 def print_version(*args):
-    print >> sys.stderr, "Python", sys.version
+    print ("Python", sys.version, file=sys.stderr)
     raise SystemExit
 
 def set_jit_option(options, jitparam, *args):
     if 'pypyjit' not in sys.builtin_module_names:
-        print >> sys.stderr, ("Warning: No jit support in %s" %
-                              (sys.executable,))
+        print("Warning: No jit support in %s" % (sys.executable,),
+              file=sys.stderr)
     else:
         import pypyjit
         pypyjit.set_param(jitparam)
@@ -159,9 +159,9 @@
     pass
 
 def print_error(msg):
-    print >> sys.stderr, msg
-    print >> sys.stderr, 'usage: %s [options]' % (sys.executable,)
-    print >> sys.stderr, 'Try `%s -h` for more information.' % (sys.executable,)
+    print(msg, file=sys.stderr)
+    print('usage: %s [options]' % (sys.executable,), file=sys.stderr)
+    print('Try `%s -h` for more information.' % (sys.executable,), file=sys.stderr)
 
 def fdopen(fd, mode, bufsize=-1):
     try:
@@ -203,11 +203,12 @@
         dirname = resolvedirof(search)
         if dirname == search:
             # not found!  let's hope that the compiled-in path is ok
-            print >> sys.stderr, """\
+            print("""\
 debug: WARNING: Library path not found, using compiled-in sys.path.
 debug: WARNING: 'sys.prefix' will not be set.
 debug: WARNING: Make sure the pypy binary is kept inside its tree of files.
-debug: WARNING: It is ok to create a symlink to it from somewhere else."""
+debug: WARNING: It is ok to create a symlink to it from somewhere else.""",
+                  file=sys.stderr)
             newpath = sys.path[:]
             break
         newpath = sys.pypy_initial_path(dirname)
@@ -441,8 +442,8 @@
         sys.dont_write_bytecode = bool(sys.flags.dont_write_bytecode)
 
         if sys.py3kwarning:
-            print >> sys.stderr, (
-                "Warning: pypy does not implement py3k warnings")
+            print("Warning: pypy does not implement py3k warnings",
+                  file=sys.stderr)
 
 ##    if not we_are_translated():
 ##        for key in sorted(options):
@@ -478,7 +479,7 @@
         try:
             import site
         except:
-            print >> sys.stderr, "'import site' failed"
+            print("'import site' failed", file=sys.stderr)
 
     readenv = not ignore_environment
     io_encoding = ((readenv and os.getenv("PYTHONIOENCODING"))
@@ -531,7 +532,7 @@
             sys.path.insert(0, '')
 
             def run_it():
-                exec run_command in mainmodule.__dict__
+                exec(run_command, mainmodule.__dict__)
             success = run_toplevel(run_it)
         elif run_module:
             # handle the "-m" command
@@ -560,14 +561,14 @@
                         startup = f.read()
                         f.close()
                     except IOError, e:
-                        print >> sys.stderr, "Could not open PYTHONSTARTUP"
-                        print >> sys.stderr, "IOError:", e
+                        print("Could not open PYTHONSTARTUP", file=sys.stderr)
+                        print("IOError:", e, file=sys.stderr)
                     else:
                         def run_it():
                             co_python_startup = compile(startup,
                                                         python_startup,
                                                         'exec')
-                            exec co_python_startup in mainmodule.__dict__
+                            exec(co_python_startup, mainmodule.__dict__)
                         run_toplevel(run_it)
                 # Then we need a prompt.
                 inspect = True
@@ -575,7 +576,7 @@
                 # If not interactive, just read and execute stdin normally.
                 def run_it():
                     co_stdin = compile(sys.stdin.read(), '<stdin>', 'exec')
-                    exec co_stdin in mainmodule.__dict__
+                    exec(co_stdin, mainmodule.__dict__)
                 mainmodule.__file__ = '<stdin>'
                 success = run_toplevel(run_it)
         else:
@@ -644,9 +645,9 @@
     return dirname
 
 def print_banner():
-    print 'Python %s on %s' % (sys.version, sys.platform)
-    print ('Type "help", "copyright", "credits" or '
-           '"license" for more information.')
+    print('Python %s on %s' % (sys.version, sys.platform))
+    print('Type "help", "copyright", "credits" or '
+          '"license" for more information.')
 
 def entry_point(executable, argv, nanos):
     setup_sys_executable(executable, nanos)
@@ -686,7 +687,7 @@
         assert filename.endswith('.pyc')
         assert file is None
         assert module.__name__ == '__main__'
-        print 'in _run_compiled_module'
+        print('in _run_compiled_module')
     def _getimporter(path):
         import os, imp
         if os.path.isdir(path):

From noreply at buildbot.pypy.org  Mon Oct 17 19:57:43 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Mon, 17 Oct 2011 19:57:43 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Try to fix cpyext. Does not work yet,
 importing it segfaults CPython :-/
Message-ID: <20111017175743.10D10820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48154:d1414c1ba250
Date: 2011-10-17 19:50 +0200
http://bitbucket.org/pypy/pypy/changeset/d1414c1ba250/

Log:	Try to fix cpyext. Does not work yet, importing it segfaults CPython
	:-/

diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -335,13 +335,13 @@
     track_reference(space, lltype.nullptr(PyObject.TO), space.w_type)
     track_reference(space, lltype.nullptr(PyObject.TO), space.w_object)
     track_reference(space, lltype.nullptr(PyObject.TO), space.w_tuple)
-    track_reference(space, lltype.nullptr(PyObject.TO), space.w_str)
+    track_reference(space, lltype.nullptr(PyObject.TO), space.w_unicode)
 
     # create the objects
     py_type = create_ref(space, space.w_type)
     py_object = create_ref(space, space.w_object)
     py_tuple = create_ref(space, space.w_tuple)
-    py_str = create_ref(space, space.w_str)
+    py_str = create_ref(space, space.w_unicode)
 
     # form cycles
     pto_type = rffi.cast(PyTypeObjectPtr, py_type)
@@ -366,7 +366,7 @@
     track_reference(space, py_type, space.w_type, replace=True)
     track_reference(space, py_object, space.w_object, replace=True)
     track_reference(space, py_tuple, space.w_tuple, replace=True)
-    track_reference(space, py_str, space.w_str, replace=True)
+    track_reference(space, py_str, space.w_unicode, replace=True)
 
 
 @cpython_api([PyObject], lltype.Void, external=False)
@@ -691,7 +691,7 @@
     w_type = from_ref(space, rffi.cast(PyObject, type))
     assert isinstance(w_type, W_TypeObject)
 
-    if not space.isinstance_w(w_name, space.w_str):
+    if not space.isinstance_w(w_name, space.w_unicode):
         return None
     name = space.str_w(w_name)
     w_obj = w_type.lookup(name)

From noreply at buildbot.pypy.org  Mon Oct 17 19:57:44 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Mon, 17 Oct 2011 19:57:44 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Merge default
Message-ID: <20111017175744.E5D6F820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48155:a0e3eb8d4121
Date: 2011-10-17 19:56 +0200
http://bitbucket.org/pypy/pypy/changeset/a0e3eb8d4121/

Log:	Merge default

diff --git a/lib-python/modified-2.7/test/test_array.py b/lib-python/modified-2.7/test/test_array.py
--- a/lib-python/modified-2.7/test/test_array.py
+++ b/lib-python/modified-2.7/test/test_array.py
@@ -295,9 +295,10 @@
         )
 
         b = array.array(self.badtypecode())
-        self.assertRaises(TypeError, "a + b")
-
-        self.assertRaises(TypeError, "a + 'bad'")
+        with self.assertRaises(TypeError):
+            a + b
+        with self.assertRaises(TypeError):
+            a + 'bad'
 
     def test_iadd(self):
         a = array.array(self.typecode, self.example[::-1])
@@ -316,9 +317,10 @@
         )
 
         b = array.array(self.badtypecode())
-        self.assertRaises(TypeError, "a += b")
-
-        self.assertRaises(TypeError, "a += 'bad'")
+        with self.assertRaises(TypeError):
+            a += b
+        with self.assertRaises(TypeError):
+            a += 'bad'
 
     def test_mul(self):
         a = 5*array.array(self.typecode, self.example)
@@ -345,7 +347,8 @@
             array.array(self.typecode)
         )
 
-        self.assertRaises(TypeError, "a * 'bad'")
+        with self.assertRaises(TypeError):
+            a * 'bad'
 
     def test_imul(self):
         a = array.array(self.typecode, self.example)
@@ -374,7 +377,8 @@
         a *= -1
         self.assertEqual(a, array.array(self.typecode))
 
-        self.assertRaises(TypeError, "a *= 'bad'")
+        with self.assertRaises(TypeError):
+            a *= 'bad'
 
     def test_getitem(self):
         a = array.array(self.typecode, self.example)
diff --git a/lib_pypy/resource.py b/lib_pypy/resource.py
--- a/lib_pypy/resource.py
+++ b/lib_pypy/resource.py
@@ -7,7 +7,7 @@
 
 from ctypes_support import standard_c_lib as libc
 from ctypes_support import get_errno
-from ctypes import Structure, c_int, c_long, byref, sizeof, POINTER
+from ctypes import Structure, c_int, c_long, byref, POINTER
 from errno import EINVAL, EPERM
 import _structseq
 
@@ -165,7 +165,6 @@
 
 @builtinify
 def getpagesize():
-    pagesize = 0
     if _getpagesize:
         return _getpagesize()
     else:
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -127,7 +127,7 @@
 
 pypy_optiondescription = OptionDescription("objspace", "Object Space Options", [
     ChoiceOption("name", "Object Space name",
-                 ["std", "flow", "thunk", "dump", "taint"],
+                 ["std", "flow", "thunk", "dump"],
                  "std",
                  cmdline='--objspace -o'),
 
diff --git a/pypy/doc/__pypy__-module.rst b/pypy/doc/__pypy__-module.rst
--- a/pypy/doc/__pypy__-module.rst
+++ b/pypy/doc/__pypy__-module.rst
@@ -37,29 +37,6 @@
 .. _`thunk object space docs`: objspace-proxies.html#thunk
 .. _`interface section of the thunk object space docs`: objspace-proxies.html#thunk-interface
 
-.. broken:
-
-    Taint Object Space Functionality
-    ================================
-
-    When the taint object space is used (choose with :config:`objspace.name`),
-    the following names are put into ``__pypy__``:
-
-     - ``taint``
-     - ``is_tainted``
-     - ``untaint``
-     - ``taint_atomic``
-     - ``_taint_debug``
-     - ``_taint_look``
-     - ``TaintError``
-
-    Those are all described in the `interface section of the taint object space
-    docs`_.
-
-    For more detailed explanations and examples see the `taint object space docs`_.
-
-    .. _`taint object space docs`: objspace-proxies.html#taint
-    .. _`interface section of the taint object space docs`: objspace-proxies.html#taint-interface
 
 Transparent Proxy Functionality
 ===============================
diff --git a/pypy/doc/config/objspace.name.txt b/pypy/doc/config/objspace.name.txt
--- a/pypy/doc/config/objspace.name.txt
+++ b/pypy/doc/config/objspace.name.txt
@@ -4,7 +4,6 @@
 for normal usage):
 
   * thunk_: The thunk object space adds lazy evaluation to PyPy.
-  * taint_: The taint object space adds soft security features.
   * dump_:  Using this object spaces results in the dumpimp of all operations
     to a log.
 
@@ -12,5 +11,4 @@
 .. _`Object Space Proxies`: ../objspace-proxies.html
 .. _`Standard Object Space`: ../objspace.html#standard-object-space
 .. _thunk: ../objspace-proxies.html#thunk
-.. _taint: ../objspace-proxies.html#taint
 .. _dump: ../objspace-proxies.html#dump
diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst
--- a/pypy/doc/index.rst
+++ b/pypy/doc/index.rst
@@ -309,7 +309,6 @@
 .. _`object space`: objspace.html
 .. _FlowObjSpace: objspace.html#the-flow-object-space 
 .. _`trace object space`: objspace.html#the-trace-object-space 
-.. _`taint object space`: objspace-proxies.html#taint
 .. _`thunk object space`: objspace-proxies.html#thunk
 .. _`transparent proxies`: objspace-proxies.html#tproxy
 .. _`Differences between PyPy and CPython`: cpython_differences.html
diff --git a/pypy/doc/objspace-proxies.rst b/pypy/doc/objspace-proxies.rst
--- a/pypy/doc/objspace-proxies.rst
+++ b/pypy/doc/objspace-proxies.rst
@@ -129,297 +129,6 @@
    function behaves lazily: all calls to it return a thunk object.
 
 
-.. broken right now:
-
-    .. _taint:
-
-    The Taint Object Space
-    ======================
-
-    Motivation
-    ----------
-
-    The Taint Object Space provides a form of security: "tainted objects",
-    inspired by various sources, see [D12.1]_ for a more detailed discussion. 
-
-    The basic idea of this kind of security is not to protect against
-    malicious code but to help with handling and boxing sensitive data. 
-    It covers two kinds of sensitive data: secret data which should not leak, 
-    and untrusted data coming from an external source and that must be 
-    validated before it is used.
-
-    The idea is that, considering a large application that handles these
-    kinds of sensitive data, there are typically only a small number of
-    places that need to explicitly manipulate that sensitive data; all the
-    other places merely pass it around, or do entirely unrelated things.
-
-    Nevertheless, if a large application needs to be reviewed for security,
-    it must be entirely carefully checked, because it is possible that a
-    bug at some apparently unrelated place could lead to a leak of sensitive
-    information in a way that an external attacker could exploit.  For
-    example, if any part of the application provides web services, an
-    attacker might be able to issue unexpected requests with a regular web
-    browser and deduce secret information from the details of the answers he
-    gets.  Another example is the common CGI attack where an attacker sends
-    malformed inputs and causes the CGI script to do unintended things.
-
-    An approach like that of the Taint Object Space allows the small parts
-    of the program that manipulate sensitive data to be explicitly marked.
-    The effect of this is that although these small parts still need a
-    careful security review, the rest of the application no longer does,
-    because even a bug would be unable to leak the information.
-
-    We have implemented a simple two-level model: objects are either
-    regular (untainted), or sensitive (tainted).  Objects are marked as
-    sensitive if they are secret or untrusted, and only declassified at
-    carefully-checked positions (e.g. where the secret data is needed, or
-    after the untrusted data has been fully validated).
-
-    It would be simple to extend the code for more fine-grained scales of
-    secrecy.  For example it is typical in the literature to consider
-    user-specified lattices of secrecy levels, corresponding to multiple
-    "owners" that cannot access data belonging to another "owner" unless
-    explicitly authorized to do so.
-
-    Tainting and untainting
-    -----------------------
-
-    Start a py.py with the Taint Object Space and try the following example::
-
-        $ py.py -o taint
-        >>>> from __pypy__ import taint
-        >>>> x = taint(6)
-
-        # x is hidden from now on.  We can pass it around and
-        # even operate on it, but not inspect it.  Taintness
-        # is propagated to operation results.
-
-        >>>> x
-        TaintError
-
-        >>>> if x > 5: y = 2   # see below
-        TaintError
-
-        >>>> y = x + 5         # ok
-        >>>> lst = [x, y]
-        >>>> z = lst.pop()
-        >>>> t = type(z)       # type() works too, tainted answer
-        >>>> t
-        TaintError
-        >>>> u = t is int      # even 'is' works
-        >>>> u
-        TaintError
-
-    Notice that using a tainted boolean like ``x > 5`` in an ``if``
-    statement is forbidden.  This is because knowing which path is followed
-    would give away a hint about ``x``; in the example above, if the
-    statement ``if x > 5: y = 2`` was allowed to run, we would know
-    something about the value of ``x`` by looking at the (untainted) value
-    in the variable ``y``.
-
-    Of course, there is a way to inspect tainted objects.  The basic way is
-    to explicitly "declassify" it with the ``untaint()`` function.  In an
-    application, the places that use ``untaint()`` are the places that need
-    careful security review.  To avoid unexpected objects showing up, the
-    ``untaint()`` function must be called with the exact type of the object
-    to declassify.  It will raise ``TaintError`` if the type doesn't match::
-
-        >>>> from __pypy__ import taint
-        >>>> untaint(int, x)
-        6
-        >>>> untaint(int, z)
-        11
-        >>>> untaint(bool, x > 5)
-        True
-        >>>> untaint(int, x > 5)
-        TaintError
-
-
-    Taint Bombs
-    -----------
-
-    In this area, a common problem is what to do about failing operations.
-    If an operation raises an exception when manipulating a tainted object,
-    then the very presence of the exception can leak information about the
-    tainted object itself.  Consider::
-
-        >>>> 5 / (x-6)
-
-    By checking if this raises ``ZeroDivisionError`` or not, we would know
-    if ``x`` was equal to 6 or not.  The solution to this problem in the
-    Taint Object Space is to introduce *Taint Bombs*.  They are a kind of
-    tainted object that doesn't contain a real object, but a pending
-    exception.  Taint Bombs are indistinguishable from normal tainted
-    objects to unprivileged code. See::
-
-        >>>> x = taint(6)
-        >>>> i = 5 / (x-6)     # no exception here
-        >>>> j = i + 1         # nor here
-        >>>> k = j + 5         # nor here
-        >>>> untaint(int, k)
-        TaintError
-
-    In the above example, all of ``i``, ``j`` and ``k`` contain a Taint
-    Bomb.  Trying to untaint it raises an exception - a generic
-    ``TaintError``.  What we win is that the exception gives little away,
-    and most importantly it occurs at the point where ``untaint()`` is
-    called, not where the operation failed.  This means that all calls to
-    ``untaint()`` - but not the rest of the code - must be carefully
-    reviewed for what occurs if they receive a Taint Bomb; they might catch
-    the ``TaintError`` and give the user a generic message that something
-    went wrong, if we are reasonably careful that the message or even its
-    presence doesn't give information away.  This might be a
-    problem by itself, but there is no satisfying general solution here:
-    it must be considered on a case-by-case basis.  Again, what the
-    Taint Object Space approach achieves is not solving these problems, but
-    localizing them to well-defined small parts of the application - namely,
-    around calls to ``untaint()``.
-
-    The ``TaintError`` exception deliberately does not include any
-    useful error messages, because they might give information away.
-    Of course, this makes debugging quite a bit harder; a difficult
-    problem to solve properly.  So far we have implemented a way to peek in a Taint
-    Box or Bomb, ``__pypy__._taint_look(x)``, and a "debug mode" that
-    prints the exception as soon as a Bomb is created - both write
-    information to the low-level stderr of the application, where we hope
-    that it is unlikely to be seen by anyone but the application
-    developer.
-
-
-    Taint Atomic functions
-    ----------------------
-
-    Occasionally, a more complicated computation must be performed on a
-    tainted object.  This requires first untainting the object, performing the
-    computations, and then carefully tainting the result again (including
-    hiding all exceptions into Bombs).
-
-    There is a built-in decorator that does this for you::
-
-        >>>> @__pypy__.taint_atomic
-        >>>> def myop(x, y):
-        ....     while x > 0:
-        ....         x -= y
-        ....     return x
-        ....
-        >>>> myop(42, 10)
-        -8
-        >>>> z = myop(taint(42), 10)
-        >>>> z
-        TaintError
-        >>>> untaint(int, z)
-        -8
-
-    The decorator makes a whole function behave like a built-in operation.
-    If no tainted argument is passed in, the function behaves normally.  But
-    if any of the arguments is tainted, it is automatically untainted - so
-    the function body always sees untainted arguments - and the eventual
-    result is tainted again (possibly in a Taint Bomb).
-
-    It is important for the function marked as ``taint_atomic`` to have no
-    visible side effects, as these could cause information leakage.
-    This is currently not enforced, which means that all ``taint_atomic``
-    functions have to be carefully reviewed for security (but not the
-    callers of ``taint_atomic`` functions).
-
-    A possible future extension would be to forbid side-effects on
-    non-tainted objects from all ``taint_atomic`` functions.
-
-    An example of usage: given a tainted object ``passwords_db`` that
-    references a database of passwords, we can write a function
-    that checks if a password is valid as follows::
-
-        @taint_atomic
-        def validate(passwords_db, username, password):
-            assert type(passwords_db) is PasswordDatabase
-            assert type(username) is str
-            assert type(password) is str
-            ...load username entry from passwords_db...
-            return expected_password == password
-
-    It returns a tainted boolean answer, or a Taint Bomb if something
-    went wrong.  A caller can do::
-
-        ok = validate(passwords_db, 'john', '1234')
-        ok = untaint(bool, ok)
-
-    This can give three outcomes: ``True``, ``False``, or a ``TaintError``
-    exception (with no information on it) if anything went wrong.  If even
-    this is considered giving too much information away, the ``False`` case
-    can be made indistinguishable from the ``TaintError`` case (simply by
-    raising an exception in ``validate()`` if the password is wrong).
-
-    In the above example, the security results achieved are the following:
-    as long as ``validate()`` does not leak information, no other part of
-    the code can obtain more information about a passwords database than a
-    Yes/No answer to a precise query.
-
-    A possible extension of the ``taint_atomic`` decorator would be to check
-    the argument types, as ``untaint()`` does, for the same reason: to
-    prevent bugs where a function like ``validate()`` above is accidentally
-    called with the wrong kind of tainted object, which would make it
-    misbehave.  For now, all ``taint_atomic`` functions should be
-    conservative and carefully check all assumptions on their input
-    arguments.
-
-
-    .. _`taint-interface`:
-
-    Interface
-    ---------
-
-    .. _`like a built-in operation`:
-
-    The basic rule of the Tainted Object Space is that it introduces two new
-    kinds of objects, Tainted Boxes and Tainted Bombs (which are not types
-    in the Python sense).  Each box internally contains a regular object;
-    each bomb internally contains an exception object.  An operation
-    involving Tainted Boxes is performed on the objects contained in the
-    boxes, and gives a Tainted Box or a Tainted Bomb as a result (such an
-    operation does not let an exception be raised).  An operation called
-    with a Tainted Bomb argument immediately returns the same Tainted Bomb.
-
-    In a PyPy running with (or translated with) the Taint Object Space,
-    the ``__pypy__`` module exposes the following interface:
-
-    * ``taint(obj)``
-
-        Return a new Tainted Box wrapping ``obj``.  Return ``obj`` itself
-        if it is already tainted (a Box or a Bomb).
-
-    * ``is_tainted(obj)``
-
-        Check if ``obj`` is tainted (a Box or a Bomb).
-
-    * ``untaint(type, obj)``
-
-        Untaints ``obj`` if it is tainted.  Raise ``TaintError`` if the type
-        of the untainted object is not exactly ``type``, or if ``obj`` is a
-        Bomb.
-
-    * ``taint_atomic(func)``
-
-        Return a wrapper function around the callable ``func``.  The wrapper
-        behaves `like a built-in operation`_ with respect to untainting the
-        arguments, tainting the result, and returning a Bomb.
-
-    * ``TaintError``
-
-        Exception.  On purpose, it provides no attribute or error message.
-
-    * ``_taint_debug(level)``
-
-        Set the debugging level to ``level`` (0=off).  At level 1 or above,
-        all Taint Bombs print a diagnostic message to stderr when they are
-        created.
-
-    * ``_taint_look(obj)``
-
-        For debugging purposes: prints (to stderr) the type and address of
-        the object in a Tainted Box, or prints the exception if ``obj`` is
-        a Taint Bomb.
-
-
 .. _dump:
 
 The Dump Object Space
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -391,8 +391,11 @@
     def decrement_ticker(self, by):
         value = self._ticker
         if self.has_bytecode_counter:    # this 'if' is constant-folded
-            value -= by
-            self._ticker = value
+            if jit.isconstant(by) and by == 0:
+                pass     # normally constant-folded too
+            else:
+                value -= by
+                self._ticker = value
         return value
 
 
diff --git a/pypy/interpreter/pyparser/pytokenizer.py b/pypy/interpreter/pyparser/pytokenizer.py
--- a/pypy/interpreter/pyparser/pytokenizer.py
+++ b/pypy/interpreter/pyparser/pytokenizer.py
@@ -226,7 +226,7 @@
                         parenlev = parenlev - 1
                         if parenlev < 0:
                             raise TokenError("unmatched '%s'" % initial, line,
-                                             lnum-1, 0, token_list)
+                                             lnum, start + 1, token_list)
                     if token in python_opmap:
                         punct = python_opmap[token]
                     else:
diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py
--- a/pypy/interpreter/pyparser/test/test_pyparse.py
+++ b/pypy/interpreter/pyparser/test/test_pyparse.py
@@ -87,6 +87,10 @@
         assert exc.lineno == 1
         assert exc.offset == 5
         assert exc.lastlineno == 5
+        exc = py.test.raises(SyntaxError, parse, "abc)").value
+        assert exc.msg == "unmatched ')'"
+        assert exc.lineno == 1
+        assert exc.offset == 4
 
     def test_is(self):
         self.parse("x is y")
diff --git a/pypy/jit/backend/llgraph/llimpl.py b/pypy/jit/backend/llgraph/llimpl.py
--- a/pypy/jit/backend/llgraph/llimpl.py
+++ b/pypy/jit/backend/llgraph/llimpl.py
@@ -165,6 +165,7 @@
     'unicodegetitem'  : (('ref', 'int'), 'int'),
     'unicodesetitem'  : (('ref', 'int', 'int'), 'int'),
     'cast_ptr_to_int' : (('ref',), 'int'),
+    'cast_int_to_ptr' : (('int',), 'ref'),
     'debug_merge_point': (('ref', 'int'), None),
     'force_token'     : ((), 'int'),
     'call_may_force'  : (('int', 'varargs'), 'intorptr'),
@@ -875,9 +876,6 @@
     def op_new_array(self, arraydescr, count):
         return do_new_array(arraydescr.ofs, count)
 
-    def op_cast_ptr_to_int(self, descr, ptr):
-        return cast_to_int(ptr)
-
     def op_force_token(self, descr):
         opaque_frame = _to_opaque(self)
         return llmemory.cast_ptr_to_adr(opaque_frame)
diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py
--- a/pypy/jit/backend/llsupport/gc.py
+++ b/pypy/jit/backend/llsupport/gc.py
@@ -45,6 +45,14 @@
     def freeing_block(self, start, stop):
         pass
 
+    def record_constptrs(self, op, gcrefs_output_list):
+        for i in range(op.numargs()):
+            v = op.getarg(i)
+            if isinstance(v, ConstPtr) and bool(v.value):
+                p = v.value
+                rgc._make_sure_does_not_move(p)
+                gcrefs_output_list.append(p)
+
 # ____________________________________________________________
 
 class GcLLDescr_boehm(GcLLDescription):
@@ -141,6 +149,14 @@
     get_funcptr_for_newstr = None
     get_funcptr_for_newunicode = None
 
+    def rewrite_assembler(self, cpu, operations, gcrefs_output_list):
+        # record all GCREFs too, because Boehm cannot see them and keep them
+        # alive if they end up as constants in the assembler
+        for op in operations:
+            self.record_constptrs(op, gcrefs_output_list)
+        return GcLLDescription.rewrite_assembler(self, cpu, operations,
+                                                 gcrefs_output_list)
+
 
 # ____________________________________________________________
 # All code below is for the hybrid or minimark GC
@@ -757,14 +773,6 @@
             funcptr(llmemory.cast_ptr_to_adr(gcref_struct),
                     llmemory.cast_ptr_to_adr(gcref_newptr))
 
-    def record_constptrs(self, op, gcrefs_output_list):
-        for i in range(op.numargs()):
-            v = op.getarg(i)
-            if isinstance(v, ConstPtr) and bool(v.value):
-                p = v.value
-                rgc._make_sure_does_not_move(p)
-                gcrefs_output_list.append(p)
-
     def rewrite_assembler(self, cpu, operations, gcrefs_output_list):
         # Perform two kinds of rewrites in parallel:
         #
diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -1468,20 +1468,16 @@
         return u''.join(u.chars)
 
 
-    def test_casts(self):
-        py.test.skip("xxx fix or kill")
-        from pypy.rpython.lltypesystem import lltype, llmemory
-        TP = lltype.GcStruct('x')
-        x = lltype.malloc(TP)        
-        x = lltype.cast_opaque_ptr(llmemory.GCREF, x)
+    def test_cast_int_to_ptr(self):
+        res = self.execute_operation(rop.CAST_INT_TO_PTR,
+                                     [BoxInt(-17)],  'ref').value
+        assert lltype.cast_ptr_to_int(res) == -17
+
+    def test_cast_ptr_to_int(self):
+        x = lltype.cast_int_to_ptr(llmemory.GCREF, -19)
         res = self.execute_operation(rop.CAST_PTR_TO_INT,
-                                     [BoxPtr(x)],  'int').value
-        expected = self.cpu.cast_adr_to_int(llmemory.cast_ptr_to_adr(x))
-        assert rffi.get_real_int(res) == rffi.get_real_int(expected)
-        res = self.execute_operation(rop.CAST_PTR_TO_INT,
-                                     [ConstPtr(x)],  'int').value
-        expected = self.cpu.cast_adr_to_int(llmemory.cast_ptr_to_adr(x))
-        assert rffi.get_real_int(res) == rffi.get_real_int(expected)
+                                     [BoxPtr(x)], 'int').value
+        assert res == -19
 
     def test_ooops_non_gc(self):
         x = lltype.malloc(lltype.Struct('x'), flavor='raw')
@@ -2299,13 +2295,6 @@
         #
         cpu.bh_strsetitem(x, 4, ord('/'))
         assert str.chars[4] == '/'
-        #
-##        x = cpu.bh_newstr(5)
-##        y = cpu.bh_cast_ptr_to_int(x)
-##        z = cpu.bh_cast_ptr_to_int(x)
-##        y = rffi.get_real_int(y)
-##        z = rffi.get_real_int(z)
-##        assert type(y) == type(z) == int and y == z
 
     def test_sorting_of_fields(self):
         S = self.S
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1387,7 +1387,8 @@
 
     def genop_same_as(self, op, arglocs, resloc):
         self.mov(arglocs[0], resloc)
-    #genop_cast_ptr_to_int = genop_same_as
+    genop_cast_ptr_to_int = genop_same_as
+    genop_cast_int_to_ptr = genop_same_as
 
     def genop_int_mod(self, op, arglocs, resloc):
         if IS_X86_32:
diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py
--- a/pypy/jit/backend/x86/regalloc.py
+++ b/pypy/jit/backend/x86/regalloc.py
@@ -1152,7 +1152,8 @@
         self.possibly_free_var(op.getarg(0))
         resloc = self.force_allocate_reg(op.result)
         self.Perform(op, [argloc], resloc)
-    #consider_cast_ptr_to_int = consider_same_as
+    consider_cast_ptr_to_int = consider_same_as
+    consider_cast_int_to_ptr = consider_same_as
 
     def consider_strlen(self, op):
         args = op.getarglist()
diff --git a/pypy/jit/backend/x86/tool/viewcode.py b/pypy/jit/backend/x86/tool/viewcode.py
--- a/pypy/jit/backend/x86/tool/viewcode.py
+++ b/pypy/jit/backend/x86/tool/viewcode.py
@@ -9,7 +9,12 @@
 """
 
 import autopath
-import operator, sys, os, re, py, new
+import new
+import operator
+import py
+import re
+import sys
+import subprocess
 from bisect import bisect_left
 
 # don't use pypy.tool.udir here to avoid removing old usessions which
@@ -44,14 +49,16 @@
     f = open(tmpfile, 'wb')
     f.write(data)
     f.close()
-    g = os.popen(objdump % {
+    p = subprocess.Popen(objdump % {
         'file': tmpfile,
         'origin': originaddr,
         'backend': objdump_backend_option[backend_name],
-    }, 'r')
-    result = g.readlines()
-    g.close()
-    lines = result[6:]   # drop some objdump cruft
+    }, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    stdout, stderr = p.communicate()
+    assert not p.returncode, ('Encountered an error running objdump: %s' %
+                              stderr)
+    # drop some objdump cruft
+    lines = stdout.splitlines()[6:]
     return format_code_dump_with_labels(originaddr, lines, label_list)
 
 def format_code_dump_with_labels(originaddr, lines, label_list):
@@ -85,8 +92,12 @@
     #
     print 'loading symbols from %s...' % (filename,)
     symbols = {}
-    g = os.popen(symbollister % filename, "r")
-    for line in g:
+    p = subprocess.Popen(symbollister % filename, shell=True,
+                         stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    stdout, stderr = p.communicate()
+    assert not p.returncode, ('Encountered an error running nm: %s' %
+                              stderr)
+    for line in stdout.splitlines():
         match = re_symbolentry.match(line)
         if match:
             addr = long(match.group(1), 16)
@@ -94,7 +105,6 @@
             if name.startswith('pypy_g_'):
                 name = '\xb7' + name[7:]
             symbols[addr] = name
-    g.close()
     print '%d symbols found' % (len(symbols),)
     return symbols
 
diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -455,6 +455,23 @@
             # the special return value None forces op.result to be considered
             # equal to op.args[0]
             return [op0, op1, None]
+        if (hints.get('promote_string') and
+            op.args[0].concretetype is not lltype.Void):
+            S = lltype.Ptr(rstr.STR)
+            assert op.args[0].concretetype == S
+            self._register_extra_helper(EffectInfo.OS_STREQ_NONNULL,
+                                        "str.eq_nonnull",
+                                        [S, S],
+                                        lltype.Signed,
+                                        EffectInfo.EF_ELIDABLE_CANNOT_RAISE)
+            descr, p = self.callcontrol.callinfocollection.callinfo_for_oopspec(
+                EffectInfo.OS_STREQ_NONNULL)
+            # XXX this is fairly ugly way of creating a constant,
+            #     however, callinfocollection has no better interface
+            c = Constant(p.adr.ptr, lltype.typeOf(p.adr.ptr))
+            op1 = SpaceOperation('str_guard_value', [op.args[0], c, descr],
+                                 op.result)
+            return [SpaceOperation('-live-', [], None), op1, None]
         else:
             log.WARNING('ignoring hint %r at %r' % (hints, self.graph))
 
@@ -783,8 +800,7 @@
 
     def rewrite_op_cast_ptr_to_int(self, op):
         if self._is_gc(op.args[0]):
-            #return op
-            raise NotImplementedError("cast_ptr_to_int")
+            return op
 
     def rewrite_op_force_cast(self, op):
         v_arg = op.args[0]
@@ -1526,6 +1542,10 @@
     def rewrite_op_jit_force_virtual(self, op):
         return self._do_builtin_call(op)
 
+    def rewrite_op_jit_is_virtual(self, op):
+        raise Exception, (
+            "'vref.virtual' should not be used from jit-visible code")
+
     def rewrite_op_jit_force_virtualizable(self, op):
         # this one is for virtualizables
         vinfo = self.get_vinfo(op.args[0])
diff --git a/pypy/jit/codewriter/test/test_jtransform.py b/pypy/jit/codewriter/test/test_jtransform.py
--- a/pypy/jit/codewriter/test/test_jtransform.py
+++ b/pypy/jit/codewriter/test/test_jtransform.py
@@ -99,6 +99,12 @@
             if i == oopspecindex:
                 return True
         return False
+    def callinfo_for_oopspec(self, oopspecindex):
+        assert oopspecindex == effectinfo.EffectInfo.OS_STREQ_NONNULL
+        class c:
+            class adr:
+                ptr = 1
+        return ('calldescr', c)
 
 class FakeBuiltinCallControl:
     def __init__(self):
@@ -119,6 +125,7 @@
              EI.OS_STR2UNICODE:([PSTR], PUNICODE),
              EI.OS_STR_CONCAT: ([PSTR, PSTR], PSTR),
              EI.OS_STR_SLICE:  ([PSTR, INT, INT], PSTR),
+             EI.OS_STREQ_NONNULL:  ([PSTR, PSTR], INT),
              EI.OS_UNI_CONCAT: ([PUNICODE, PUNICODE], PUNICODE),
              EI.OS_UNI_SLICE:  ([PUNICODE, INT, INT], PUNICODE),
              EI.OS_UNI_EQUAL:  ([PUNICODE, PUNICODE], lltype.Bool),
@@ -844,6 +851,21 @@
     assert op1.args[2] == ListOfKind('ref', [v1, v2])
     assert op1.result == v3
 
+def test_str_promote():
+    PSTR = lltype.Ptr(rstr.STR)
+    v1 = varoftype(PSTR)
+    v2 = varoftype(PSTR)
+    op = SpaceOperation('hint',
+                        [v1, Constant({'promote_string': True}, lltype.Void)],
+                        v2)
+    tr = Transformer(FakeCPU(), FakeBuiltinCallControl())
+    op0, op1, _ = tr.rewrite_operation(op)
+    assert op1.opname == 'str_guard_value'
+    assert op1.args[0] == v1
+    assert op1.args[2] == 'calldescr'
+    assert op1.result == v2
+    assert op0.opname == '-live-'
+
 def test_unicode_concat():
     # test that the oopspec is present and correctly transformed
     PSTR = lltype.Ptr(rstr.UNICODE)
diff --git a/pypy/jit/metainterp/blackhole.py b/pypy/jit/metainterp/blackhole.py
--- a/pypy/jit/metainterp/blackhole.py
+++ b/pypy/jit/metainterp/blackhole.py
@@ -2,7 +2,7 @@
 from pypy.rlib.rtimer import read_timestamp
 from pypy.rlib.rarithmetic import intmask, LONG_BIT, r_uint, ovfcheck
 from pypy.rlib.objectmodel import we_are_translated
-from pypy.rlib.debug import debug_start, debug_stop
+from pypy.rlib.debug import debug_start, debug_stop, ll_assert
 from pypy.rlib.debug import make_sure_not_resized
 from pypy.rpython.lltypesystem import lltype, llmemory, rclass
 from pypy.rpython.lltypesystem.lloperation import llop
@@ -503,6 +503,15 @@
     @arguments("r", returns="r")
     def bhimpl_cast_opaque_ptr(a):
         return a
+    @arguments("r", returns="i")
+    def bhimpl_cast_ptr_to_int(a):
+        i = lltype.cast_ptr_to_int(a)
+        ll_assert((i & 1) == 1, "bhimpl_cast_ptr_to_int: not an odd int")
+        return i
+    @arguments("i", returns="r")
+    def bhimpl_cast_int_to_ptr(i):
+        ll_assert((i & 1) == 1, "bhimpl_cast_int_to_ptr: not an odd int")
+        return lltype.cast_int_to_ptr(llmemory.GCREF, i)
 
     @arguments("i", returns="i")
     def bhimpl_int_copy(a):
@@ -523,6 +532,9 @@
     @arguments("f")
     def bhimpl_float_guard_value(a):
         pass
+    @arguments("r", "i", "d", returns="r")
+    def bhimpl_str_guard_value(a, i, d):
+        return a
 
     @arguments("self", "i")
     def bhimpl_int_push(self, a):
diff --git a/pypy/jit/metainterp/graphpage.py b/pypy/jit/metainterp/graphpage.py
--- a/pypy/jit/metainterp/graphpage.py
+++ b/pypy/jit/metainterp/graphpage.py
@@ -12,8 +12,8 @@
     def get_display_text(self):
         return None
 
-def display_loops(loops, errmsg=None, highlight_loops=()):
-    graphs = [(loop, loop in highlight_loops) for loop in loops]    
+def display_loops(loops, errmsg=None, highlight_loops={}):
+    graphs = [(loop, highlight_loops.get(loop, 0)) for loop in loops]    
     for graph, highlight in graphs:
         for op in graph.get_operations():
             if is_interesting_guard(op):
@@ -65,8 +65,7 @@
     def add_graph(self, graph, highlight=False):
         graphindex = len(self.graphs)
         self.graphs.append(graph)
-        if highlight:
-            self.highlight_graphs[graph] = True
+        self.highlight_graphs[graph] = highlight
         for i, op in enumerate(graph.get_operations()):
             self.all_operations[op] = graphindex, i
 
@@ -126,10 +125,13 @@
             self.dotgen.emit('subgraph cluster%d {' % graphindex)
         label = graph.get_display_text()
         if label is not None:
-            if self.highlight_graphs.get(graph):
-                fillcolor = '#f084c2'
+            colorindex = self.highlight_graphs.get(graph, 0)
+            if colorindex == 1:
+                fillcolor = '#f084c2'    # highlighted graph
+            elif colorindex == 2:
+                fillcolor = '#808080'    # invalidated graph
             else:
-                fillcolor = '#84f0c2'
+                fillcolor = '#84f0c2'    # normal color
             self.dotgen.emit_node(graphname, shape="octagon",
                                   label=label, fillcolor=fillcolor)
             self.pendingedges.append((graphname,
diff --git a/pypy/jit/metainterp/history.py b/pypy/jit/metainterp/history.py
--- a/pypy/jit/metainterp/history.py
+++ b/pypy/jit/metainterp/history.py
@@ -9,6 +9,7 @@
 
 from pypy.jit.metainterp.resoperation import ResOperation, rop
 from pypy.jit.codewriter import heaptracker, longlong
+from pypy.rlib.objectmodel import compute_identity_hash
 
 # ____________________________________________________________
 
@@ -104,7 +105,7 @@
     getref._annspecialcase_ = 'specialize:arg(1)'
 
     def _get_hash_(self):
-        raise NotImplementedError
+        return compute_identity_hash(self)
 
     def clonebox(self):
         raise NotImplementedError
@@ -133,6 +134,9 @@
     def _get_str(self):
         raise NotImplementedError
 
+    def same_box(self, other):
+        return self is other
+
 class AbstractDescr(AbstractValue):
     __slots__ = ()
 
@@ -241,32 +245,15 @@
     def constbox(self):
         return self
 
+    def same_box(self, other):
+        return self.same_constant(other)
+
     def same_constant(self, other):
         raise NotImplementedError
 
     def __repr__(self):
         return 'Const(%s)' % self._getrepr_()
 
-    def __eq__(self, other):
-        "NOT_RPYTHON"
-        # Remember that you should not compare Consts with '==' in RPython.
-        # Consts have no special __hash__, in order to force different Consts
-        # from being considered as different keys when stored in dicts
-        # (as they always are after translation).  Use a dict_equal_consts()
-        # to get the other behavior (i.e. using this __eq__).
-        if self.__class__ is not other.__class__:
-            return False
-        try:
-            return self.value == other.value
-        except TypeError:
-            if (isinstance(self.value, Symbolic) and
-                isinstance(other.value, Symbolic)):
-                return self.value is other.value
-            raise
-
-    def __ne__(self, other):
-        return not (self == other)
-
 
 class ConstInt(Const):
     type = INT
@@ -688,33 +675,6 @@
 
 # ____________________________________________________________
 
-def dict_equal_consts():
-    "NOT_RPYTHON"
-    # Returns a dict in which Consts that compare as equal
-    # are identified when used as keys.
-    return r_dict(dc_eq, dc_hash)
-
-def dc_eq(c1, c2):
-    return c1 == c2
-
-def dc_hash(c):
-    "NOT_RPYTHON"
-    # This is called during translation only.  Avoid using identityhash(),
-    # to avoid forcing a hash, at least on lltype objects.
-    if not isinstance(c, Const):
-        return hash(c)
-    if isinstance(c.value, Symbolic):
-        return id(c.value)
-    try:
-        if isinstance(c, ConstPtr):
-            p = lltype.normalizeptr(c.value)
-            if p is not None:
-                return hash(p._obj)
-            else:
-                return 0
-        return c._get_hash_()
-    except lltype.DelayedPointer:
-        return -2      # xxx risk of changing hash...
 
 def make_hashable_int(i):
     from pypy.rpython.lltypesystem.ll2ctypes import NotCtypesAllocatedStructure
@@ -772,6 +732,7 @@
     failed_states = None
     retraced_count = 0
     terminating = False # see TerminatingLoopToken in compile.py
+    invalidated = False
     outermost_jitdriver_sd = None
     # and more data specified by the backend when the loop is compiled
     number = -1
@@ -974,6 +935,7 @@
         self.loops = []
         self.locations = []
         self.aborted_keys = []
+        self.invalidated_token_numbers = set()
 
     def set_history(self, history):
         self.operations = history.operations
@@ -1052,7 +1014,12 @@
             if loop in loops:
                 loops.remove(loop)
             loops.append(loop)
-        display_loops(loops, errmsg, extraloops)
+        highlight_loops = dict.fromkeys(extraloops, 1)
+        for loop in loops:
+            if hasattr(loop, '_looptoken_number') and (
+                    loop._looptoken_number in self.invalidated_token_numbers):
+                highlight_loops.setdefault(loop, 2)
+        display_loops(loops, errmsg, highlight_loops)
 
 # ----------------------------------------------------------------
 
diff --git a/pypy/jit/metainterp/memmgr.py b/pypy/jit/metainterp/memmgr.py
--- a/pypy/jit/metainterp/memmgr.py
+++ b/pypy/jit/metainterp/memmgr.py
@@ -68,7 +68,8 @@
         debug_print("Loop tokens before:", oldtotal)
         max_generation = self.current_generation - (self.max_age-1)
         for looptoken in self.alive_loops.keys():
-            if 0 <= looptoken.generation < max_generation:
+            if (0 <= looptoken.generation < max_generation or
+                looptoken.invalidated):
                 del self.alive_loops[looptoken]
         newtotal = len(self.alive_loops)
         debug_print("Loop tokens freed: ", oldtotal - newtotal)
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -2329,7 +2329,7 @@
         def _variables_equal(box, varname, strict):
             if varname not in virtuals:
                 if strict:
-                    assert box == oparse.getvar(varname)
+                    assert box.same_box(oparse.getvar(varname))
                 else:
                     assert box.value == oparse.getvar(varname).value
             else:
diff --git a/pypy/jit/metainterp/optimizeopt/util.py b/pypy/jit/metainterp/optimizeopt/util.py
--- a/pypy/jit/metainterp/optimizeopt/util.py
+++ b/pypy/jit/metainterp/optimizeopt/util.py
@@ -90,14 +90,11 @@
     for i in range(len(args1)):
         arg1 = args1[i]
         arg2 = args2[i]
-        if isinstance(arg1, history.Const):
-            if arg1.__class__ is not arg2.__class__:
+        if arg1 is None:
+            if arg2 is not None:
                 return False
-            if not arg1.same_constant(arg2):
-                return False
-        else:
-            if not arg1 is arg2:
-                return False
+        elif not arg1.same_box(arg2):
+            return False
     return True
 
 def args_hash(args):
@@ -106,10 +103,8 @@
     for arg in args:
         if arg is None:
             y = 17
-        elif isinstance(arg, history.Const):
+        else:
             y = arg._get_hash_()
-        else:
-            y = compute_identity_hash(arg)
         res = intmask((1000003 * res) ^ y)
     return res
 
@@ -145,9 +140,12 @@
         for i in range(op1.numargs()):
             x = op1.getarg(i)
             y = op2.getarg(i)
-            assert x == remap.get(y, y)
+            assert x.same_box(remap.get(y, y))
         if op2.result in remap:
-            assert op1.result == remap[op2.result]
+            if op2.result is None:
+                assert op1.result == remap[op2.result]
+            else:
+                assert op1.result.same_box(remap[op2.result])
         else:
             remap[op2.result] = op1.result
         if op1.getopnum() != rop.JUMP:      # xxx obscure
@@ -156,11 +154,20 @@
             assert len(op1.getfailargs()) == len(op2.getfailargs())
             if strict_fail_args:
                 for x, y in zip(op1.getfailargs(), op2.getfailargs()):
-                    assert x == remap.get(y, y)
+                    if x is None:
+                        assert remap.get(y, y) is None
+                    else:
+                        assert x.same_box(remap.get(y, y))
             else:
                 fail_args1 = set(op1.getfailargs())
                 fail_args2 = set([remap.get(y, y) for y in op2.getfailargs()])
-                assert fail_args1 == fail_args2
+                for x in fail_args1:
+                    for y in fail_args2:
+                        if x.same_box(y):
+                            fail_args2.remove(y)
+                            break
+                    else:
+                        assert False
     assert len(oplist1) == len(oplist2)
     print '-'*totwidth
     return True
diff --git a/pypy/jit/metainterp/optimizeopt/vstring.py b/pypy/jit/metainterp/optimizeopt/vstring.py
--- a/pypy/jit/metainterp/optimizeopt/vstring.py
+++ b/pypy/jit/metainterp/optimizeopt/vstring.py
@@ -587,10 +587,7 @@
             return True
         #
         if v1.is_nonnull() and v2.is_nonnull():
-            if l1box is not None and l2box is not None and (
-                l1box == l2box or (isinstance(l1box, ConstInt) and
-                                   isinstance(l2box, ConstInt) and
-                                   l1box.value == l2box.value)):
+            if l1box is not None and l2box is not None and l1box.same_box(l2box):
                 do = EffectInfo.OS_STREQ_LENGTHOK
             else:
                 do = EffectInfo.OS_STREQ_NONNULL
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -162,15 +162,18 @@
             if registers[i] is oldbox:
                 registers[i] = newbox
         if not we_are_translated():
-            assert oldbox not in registers[count:]
+            for b in registers[count:]:
+                assert not oldbox.same_box(b)
+                
 
     def make_result_of_lastop(self, resultbox):
         got_type = resultbox.type
-        if not we_are_translated():
-            typeof = {'i': history.INT,
-                      'r': history.REF,
-                      'f': history.FLOAT}
-            assert typeof[self.jitcode._resulttypes[self.pc]] == got_type
+        # XXX disabled for now, conflicts with str_guard_value
+        #if not we_are_translated():
+        #    typeof = {'i': history.INT,
+        #              'r': history.REF,
+        #              'f': history.FLOAT}
+        #    assert typeof[self.jitcode._resulttypes[self.pc]] == got_type
         target_index = ord(self.bytecode[self.pc-1])
         if got_type == history.INT:
             self.registers_i[target_index] = resultbox
@@ -219,6 +222,7 @@
                     'cast_float_to_int', 'cast_int_to_float',
                     'cast_float_to_singlefloat', 'cast_singlefloat_to_float',
                     'float_neg', 'float_abs',
+                    'cast_ptr_to_int', 'cast_int_to_ptr',
                     ]:
         exec py.code.Source('''
             @arguments("box")
@@ -895,6 +899,21 @@
     def _opimpl_guard_value(self, orgpc, box):
         self.implement_guard_value(orgpc, box)
 
+    @arguments("orgpc", "box", "box", "descr")
+    def opimpl_str_guard_value(self, orgpc, box, funcbox, descr):
+        if isinstance(box, Const):
+            return box     # no promotion needed, already a Const
+        else:
+            constbox = box.constbox()
+            resbox = self.do_residual_call(funcbox, descr, [box, constbox])
+            promoted_box = resbox.constbox()
+            # This is GUARD_VALUE because GUARD_TRUE assumes the existance
+            # of a label when computing resumepc
+            self.generate_guard(rop.GUARD_VALUE, resbox, [promoted_box],
+                                resumepc=orgpc)
+            self.metainterp.replace_box(box, constbox)
+            return constbox
+
     opimpl_int_guard_value = _opimpl_guard_value
     opimpl_ref_guard_value = _opimpl_guard_value
     opimpl_float_guard_value = _opimpl_guard_value
diff --git a/pypy/jit/metainterp/quasiimmut.py b/pypy/jit/metainterp/quasiimmut.py
--- a/pypy/jit/metainterp/quasiimmut.py
+++ b/pypy/jit/metainterp/quasiimmut.py
@@ -2,6 +2,7 @@
 from pypy.rpython.lltypesystem import lltype, rclass
 from pypy.rpython.annlowlevel import cast_base_ptr_to_instance
 from pypy.jit.metainterp.history import AbstractDescr
+from pypy.rlib.objectmodel import we_are_translated
 
 
 def get_mutate_field_name(fieldname):
@@ -50,13 +51,13 @@
 
 class QuasiImmut(object):
     llopaque = True
+    compress_limit = 30
     
     def __init__(self, cpu):
         self.cpu = cpu
         # list of weakrefs to the LoopTokens that must be invalidated if
         # this value ever changes
         self.looptokens_wrefs = []
-        self.compress_limit = 30
 
     def hide(self):
         qmut_ptr = self.cpu.ts.cast_instance_to_base_ref(self)
@@ -73,8 +74,12 @@
         self.looptokens_wrefs.append(wref_looptoken)
 
     def compress_looptokens_list(self):
-        self.looptokens_wrefs = [wref for wref in self.looptokens_wrefs
-                                      if wref() is not None]
+        newlist = []
+        for wref in self.looptokens_wrefs:
+            looptoken = wref()
+            if looptoken is not None and not looptoken.invalidated:
+                newlist.append(wref)
+        self.looptokens_wrefs = newlist
         self.compress_limit = (len(self.looptokens_wrefs) + 15) * 2
 
     def invalidate(self):
@@ -85,8 +90,12 @@
         self.looptokens_wrefs = []
         for wref in wrefs:
             looptoken = wref()
-            if looptoken is not None:
+            if looptoken is not None and not looptoken.invalidated:
+                looptoken.invalidated = True
                 self.cpu.invalidate_loop(looptoken)
+                if not we_are_translated():
+                    self.cpu.stats.invalidated_token_numbers.add(
+                        looptoken.number)
 
 
 class QuasiImmutDescr(AbstractDescr):
diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py
--- a/pypy/jit/metainterp/resoperation.py
+++ b/pypy/jit/metainterp/resoperation.py
@@ -433,6 +433,8 @@
     'INT_INVERT/1',
     #
     'SAME_AS/1',      # gets a Const or a Box, turns it into another Box
+    'CAST_PTR_TO_INT/1',
+    'CAST_INT_TO_PTR/1',
     #
     'PTR_EQ/2b',
     'PTR_NE/2b',
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -13,7 +13,7 @@
 from pypy.rlib.jit import (JitDriver, we_are_jitted, hint, dont_look_inside,
     loop_invariant, elidable, promote, jit_debug, assert_green,
     AssertGreenFailed, unroll_safe, current_trace_length, look_inside_iff,
-    isconstant, isvirtual)
+    isconstant, isvirtual, promote_string)
 from pypy.rlib.rarithmetic import ovfcheck
 from pypy.rpython.lltypesystem import lltype, llmemory, rffi
 from pypy.rpython.ootypesystem import ootype
@@ -3440,7 +3440,6 @@
 
 class TestLLtype(BaseLLtypeTests, LLJitMixin):
     def test_tagged(self):
-        py.test.skip("implement me")
         from pypy.rlib.objectmodel import UnboxedValue
         class Base(object):
             __slots__ = ()
@@ -3492,3 +3491,35 @@
                 pc += 1
             return pc
         res = self.meta_interp(main, [False, 100, True], taggedpointers=True)
+
+    def test_rerased(self):
+        from pypy.rlib.rerased import erase_int, unerase_int, new_erasing_pair
+        eraseX, uneraseX = new_erasing_pair("X")
+        #
+        class X:
+            def __init__(self, a, b):
+                self.a = a
+                self.b = b
+        #
+        def f(i, j):
+            # 'j' should be 0 or 1, not other values
+            if j > 0:
+                e = eraseX(X(i, j))
+            else:
+                try:
+                    e = erase_int(i)
+                except OverflowError:
+                    return -42
+            if j & 1:
+                x = uneraseX(e)
+                return x.a - x.b
+            else:
+                return unerase_int(e)
+        #
+        x = self.interp_operations(f, [-128, 0], taggedpointers=True)
+        assert x == -128
+        bigint = sys.maxint//2 + 1
+        x = self.interp_operations(f, [bigint, 0], taggedpointers=True)
+        assert x == -42
+        x = self.interp_operations(f, [1000, 1], taggedpointers=True)
+        assert x == 999
diff --git a/pypy/jit/metainterp/test/test_memmgr.py b/pypy/jit/metainterp/test/test_memmgr.py
--- a/pypy/jit/metainterp/test/test_memmgr.py
+++ b/pypy/jit/metainterp/test/test_memmgr.py
@@ -18,6 +18,7 @@
 
 class FakeLoopToken:
     generation = 0
+    invalidated = False
 
 
 class _TestMemoryManager:
diff --git a/pypy/jit/metainterp/test/test_quasiimmut.py b/pypy/jit/metainterp/test/test_quasiimmut.py
--- a/pypy/jit/metainterp/test/test_quasiimmut.py
+++ b/pypy/jit/metainterp/test/test_quasiimmut.py
@@ -48,6 +48,13 @@
 
 class QuasiImmutTests(object):
 
+    def setup_method(self, meth):
+        self.prev_compress_limit = QuasiImmut.compress_limit
+        QuasiImmut.compress_limit = 1
+
+    def teardown_method(self, meth):
+        QuasiImmut.compress_limit = self.prev_compress_limit
+
     def test_simple_1(self):
         myjitdriver = JitDriver(greens=['foo'], reds=['x', 'total'])
         class Foo:
@@ -289,7 +296,7 @@
             return total
 
         res = self.meta_interp(main, [])
-        self.check_loop_count(9)
+        self.check_tree_loop_count(6)
         assert res == main()
 
     def test_change_during_running(self):
@@ -317,7 +324,7 @@
         assert f(100, 15) == 3009
         res = self.meta_interp(f, [100, 15])
         assert res == 3009
-        self.check_loops(guard_not_invalidated=2, getfield_gc=0,
+        self.check_loops(guard_not_invalidated=4, getfield_gc=0,
                          call_may_force=0, guard_not_forced=0)
 
     def test_list_simple_1(self):
@@ -453,10 +460,30 @@
         assert f(100, 15) == 3009
         res = self.meta_interp(f, [100, 15])
         assert res == 3009
-        self.check_loops(guard_not_invalidated=2, getfield_gc=0,
+        self.check_loops(guard_not_invalidated=4, getfield_gc=0,
                          getarrayitem_gc=0, getarrayitem_gc_pure=0,
                          call_may_force=0, guard_not_forced=0)
 
+    def test_invalidated_loop_is_not_used_any_more_as_target(self):
+        myjitdriver = JitDriver(greens=['foo'], reds=['x'])
+        class Foo:
+            _immutable_fields_ = ['step?']
+        @dont_look_inside
+        def residual(x, foo):
+            if x == 20:
+                foo.step = 1
+        def f(x):
+            foo = Foo()
+            foo.step = 2
+            while x > 0:
+                myjitdriver.jit_merge_point(foo=foo, x=x)
+                residual(x, foo)
+                x -= foo.step
+            return foo.step
+        res = self.meta_interp(f, [60])
+        assert res == 1
+        self.check_tree_loop_count(4)   # at least not 2 like before
+
 
 class TestLLtypeGreenFieldsTests(QuasiImmutTests, LLJitMixin):
     pass
diff --git a/pypy/jit/metainterp/test/test_resume.py b/pypy/jit/metainterp/test/test_resume.py
--- a/pypy/jit/metainterp/test/test_resume.py
+++ b/pypy/jit/metainterp/test/test_resume.py
@@ -1,3 +1,4 @@
+from __future__ import with_statement
 import py
 from pypy.rpython.lltypesystem import lltype, llmemory, rffi
 from pypy.jit.metainterp.optimizeopt.optimizer import OptValue
@@ -180,22 +181,27 @@
     reader.consume_boxes(info, bi, br, bf)
     b1s = reader.liveboxes[0]
     b2s = reader.liveboxes[1]
-    assert bi == [b1s, ConstInt(111), b1s]
-    assert br == [ConstPtr(gcrefnull), b2s]
+    assert_same(bi, [b1s, ConstInt(111), b1s])
+    assert_same(br, [ConstPtr(gcrefnull), b2s])
     bi, br, bf = [None]*2, [None]*0, [None]*0
     info = MyBlackholeInterp([lltype.Signed,
                               lltype.Signed]).get_current_position_info()
     reader.consume_boxes(info, bi, br, bf)
-    assert bi == [ConstInt(222), ConstInt(333)]
+    assert_same(bi, [ConstInt(222), ConstInt(333)])
     bi, br, bf = [None]*2, [None]*1, [None]*0
     info = MyBlackholeInterp([lltype.Signed, llmemory.GCREF,
                               lltype.Signed]).get_current_position_info()
     reader.consume_boxes(info, bi, br, bf)
     b3s = reader.liveboxes[2]
-    assert bi == [b1s, b3s]
-    assert br == [b2s]
+    assert_same(bi, [b1s, b3s])
+    assert_same(br, [b2s])
     #
 
+def assert_same(list1, list2):
+    assert len(list1) == len(list2)
+    for b1, b2 in zip(list1, list2):
+        assert b1.same_box(b2)
+
 def test_simple_read_tagged_ints():
     storage = Storage()
     storage.rd_consts = []
@@ -1023,11 +1029,11 @@
     metainterp = MyMetaInterp()
     reader = ResumeDataFakeReader(storage, newboxes, metainterp)
     lst = reader.consume_boxes()
-    assert lst == [b1t, b1t, b3t]
+    assert_same(lst, [b1t, b1t, b3t])
     lst = reader.consume_boxes()
-    assert lst == [ConstInt(2), ConstInt(3)]
+    assert_same(lst, [ConstInt(2), ConstInt(3)])
     lst = reader.consume_boxes()
-    assert lst == [b1t, ConstInt(1), b1t, b1t]
+    assert_same(lst, [b1t, ConstInt(1), b1t, b1t])
     assert metainterp.trace == []    
 
 
@@ -1044,11 +1050,11 @@
     reader = ResumeDataFakeReader(storage, newboxes, metainterp)
     lst = reader.consume_boxes()
     c1t = ConstInt(111)
-    assert lst == [c1t, b2t, b3t]
+    assert_same(lst, [c1t, b2t, b3t])
     lst = reader.consume_boxes()
-    assert lst == [ConstInt(2), ConstInt(3)]
+    assert_same(lst, [ConstInt(2), ConstInt(3)])
     lst = reader.consume_boxes()
-    assert lst == [c1t, ConstInt(1), c1t, b2t]
+    assert_same(lst, [c1t, ConstInt(1), c1t, b2t])
     assert metainterp.trace == []
 
 
@@ -1114,9 +1120,11 @@
     # check that we get the operations in 'expected', in a possibly different
     # order.
     assert len(trace) == len(expected)
-    for x in trace:
-        assert x in expected
-        expected.remove(x)
+    with CompareableConsts():
+        for x in trace:
+            assert x in expected
+            expected.remove(x)
+
     ptr = b2t.value._obj.container._as_ptr()
     assert lltype.typeOf(ptr) == lltype.Ptr(LLtypeMixin.NODE)
     assert ptr.value == 111
@@ -1126,6 +1134,18 @@
     assert ptr2.parent.value == 33
     assert ptr2.parent.next == ptr
 
+class CompareableConsts(object):
+    def __init__(self):
+        self.oldeq = None
+        
+    def __enter__(self):
+        assert self.oldeq is None
+        self.oldeq = Const.__eq__
+        Const.__eq__ = Const.same_box
+        
+    def __exit__(self, type, value, traceback):
+        Const.__eq__ = self.oldeq
+
 def test_virtual_adder_make_varray():
     b2s, b4s = [BoxPtr(), BoxInt(4)]
     c1s = ConstInt(111)
@@ -1163,8 +1183,9 @@
         (rop.SETARRAYITEM_GC, [b2t,ConstInt(1), c1s], None,
                               LLtypeMixin.arraydescr),
         ]
-    for x, y in zip(expected, trace):
-        assert x == y
+    with CompareableConsts():
+        for x, y in zip(expected, trace):
+            assert x == y
     #
     ptr = b2t.value._obj.container._as_ptr()
     assert lltype.typeOf(ptr) == lltype.Ptr(lltype.GcArray(lltype.Signed))
@@ -1207,8 +1228,9 @@
         (rop.SETFIELD_GC, [b2t, c1s],  None, LLtypeMixin.adescr),
         (rop.SETFIELD_GC, [b2t, b4t], None, LLtypeMixin.bdescr),
         ]
-    for x, y in zip(expected, trace):
-        assert x == y
+    with CompareableConsts():
+        for x, y in zip(expected, trace):
+            assert x == y
     #
     ptr = b2t.value._obj.container._as_ptr()
     assert lltype.typeOf(ptr) == lltype.Ptr(LLtypeMixin.S)
diff --git a/pypy/jit/metainterp/test/test_string.py b/pypy/jit/metainterp/test/test_string.py
--- a/pypy/jit/metainterp/test/test_string.py
+++ b/pypy/jit/metainterp/test/test_string.py
@@ -3,7 +3,8 @@
 from pypy.jit.codewriter.policy import StopAtXPolicy
 from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin
 from pypy.rlib.debug import debug_print
-from pypy.rlib.jit import JitDriver, dont_look_inside, we_are_jitted
+from pypy.rlib.jit import JitDriver, dont_look_inside, we_are_jitted,\
+     promote_string
 from pypy.rlib.rstring import StringBuilder
 from pypy.rpython.ootypesystem import ootype
 
@@ -507,6 +508,18 @@
                           'jump': 1, 'int_is_true': 1,
                           'guard_not_invalidated': 1})
 
+    def test_promote_string(self):
+        driver = JitDriver(greens = [], reds = ['n'])
+        
+        def f(n):
+            while n < 21:
+                driver.jit_merge_point(n=n)
+                promote_string(str(n % 3))
+                n += 1
+            return 0
+
+        self.meta_interp(f, [0])
+        self.check_loops(call=3 + 1) # one for int2str
 
 #class TestOOtype(StringTests, OOJitMixin):
 #    CALL = "oosend"
diff --git a/pypy/jit/metainterp/test/test_virtualref.py b/pypy/jit/metainterp/test/test_virtualref.py
--- a/pypy/jit/metainterp/test/test_virtualref.py
+++ b/pypy/jit/metainterp/test/test_virtualref.py
@@ -3,6 +3,7 @@
 from pypy.rpython.llinterp import LLException
 from pypy.rlib.jit import JitDriver, dont_look_inside, vref_None
 from pypy.rlib.jit import virtual_ref, virtual_ref_finish, InvalidVirtualRef
+from pypy.rlib.jit import non_virtual_ref
 from pypy.rlib.objectmodel import compute_unique_id
 from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin, _get_jitcodes
 from pypy.jit.metainterp.resoperation import rop
@@ -595,6 +596,65 @@
         res = self.meta_interp(fn, [10])
         assert res == 6
 
+    def test_is_virtual(self):
+        myjitdriver = JitDriver(greens=[], reds=['n', 'res1'])
+        class X:
+            pass
+        @dont_look_inside
+        def residual(vref):
+            return vref.virtual
+        #
+        def f(n):
+            res1 = -42
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, res1=res1)
+                x = X()
+                vref = virtual_ref(x)
+                res1 = residual(vref)
+                virtual_ref_finish(vref, x)
+                n -= 1
+            return res1
+        #
+        res = self.meta_interp(f, [10])
+        assert res == 1
+
+    def test_is_not_virtual_none(self):
+        myjitdriver = JitDriver(greens=[], reds=['n', 'res1'])
+        @dont_look_inside
+        def residual(vref):
+            return vref.virtual
+        #
+        def f(n):
+            res1 = -42
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, res1=res1)
+                res1 = residual(vref_None)
+                n -= 1
+            return res1
+        #
+        res = self.meta_interp(f, [10])
+        assert res == 0
+
+    def test_is_not_virtual_non_none(self):
+        myjitdriver = JitDriver(greens=[], reds=['n', 'res1'])
+        class X:
+            pass
+        @dont_look_inside
+        def residual(vref):
+            return vref.virtual
+        #
+        def f(n):
+            res1 = -42
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, res1=res1)
+                x = X()
+                res1 = residual(non_virtual_ref(x))
+                n -= 1
+            return res1
+        #
+        res = self.meta_interp(f, [10])
+        assert res == 0
+
 
 class TestLLtype(VRefTests, LLJitMixin):
     pass
diff --git a/pypy/jit/metainterp/virtualref.py b/pypy/jit/metainterp/virtualref.py
--- a/pypy/jit/metainterp/virtualref.py
+++ b/pypy/jit/metainterp/virtualref.py
@@ -39,6 +39,7 @@
     def replace_force_virtual_with_call(self, graphs):
         # similar to rvirtualizable2.replace_force_virtualizable_with_call().
         c_force_virtual_ptr = None
+        c_is_virtual_ptr = None
         force_virtual_count = 0
         for graph in graphs:
             for block in graph.iterblocks():
@@ -52,6 +53,13 @@
                         op.opname = 'direct_call'
                         op.args = [c_force_virtual_ptr, op.args[0]]
                         force_virtual_count += 1
+                    #
+                    if op.opname == 'jit_is_virtual':
+                        if c_is_virtual_ptr is None:
+                            c_is_virtual_ptr = self.get_is_virtual_fnptr()
+                        #
+                        op.opname = 'direct_call'
+                        op.args = [c_is_virtual_ptr, op.args[0]]
         #
         if c_force_virtual_ptr is not None:
             log("replaced %d 'jit_force_virtual' with %r" % (force_virtual_count,
@@ -129,6 +137,17 @@
             force_virtual_if_necessary)
         return inputconst(lltype.typeOf(funcptr), funcptr)
 
+    def get_is_virtual_fnptr(self):
+        #
+        def is_virtual(inst):
+            if not inst:
+                return False
+            return inst.typeptr == self.jit_virtual_ref_vtable
+        #
+        FUNC = lltype.FuncType([rclass.OBJECTPTR], lltype.Bool)
+        funcptr = self.warmrunnerdesc.helper_func(lltype.Ptr(FUNC), is_virtual)
+        return inputconst(lltype.typeOf(funcptr), funcptr)
+
     def force_virtual(self, inst):
         vref = lltype.cast_pointer(lltype.Ptr(self.JIT_VIRTUAL_REF), inst)
         token = vref.virtual_token
diff --git a/pypy/jit/metainterp/warmstate.py b/pypy/jit/metainterp/warmstate.py
--- a/pypy/jit/metainterp/warmstate.py
+++ b/pypy/jit/metainterp/warmstate.py
@@ -178,7 +178,7 @@
         if self.compiled_merge_points_wref is not None:
             for wref in self.compiled_merge_points_wref:
                 looptoken = wref()
-                if looptoken is not None:
+                if looptoken is not None and not looptoken.invalidated:
                     result.append(looptoken)
         return result
 
diff --git a/pypy/module/__builtin__/__init__.py b/pypy/module/__builtin__/__init__.py
--- a/pypy/module/__builtin__/__init__.py
+++ b/pypy/module/__builtin__/__init__.py
@@ -22,6 +22,9 @@
         'any'           : 'app_functional.any',
         'all'           : 'app_functional.all',
         'sum'           : 'app_functional.sum',
+        'map'           : 'app_functional.map',
+        'reduce'        : 'app_functional.reduce',
+        'filter'        : 'app_functional.filter',
         'vars'          : 'app_inspect.vars',
         'dir'           : 'app_inspect.dir',
 
@@ -85,11 +88,8 @@
         'enumerate'     : 'functional.W_Enumerate',
         'min'           : 'functional.min',
         'max'           : 'functional.max',
-        'map'           : 'functional.map',
         'zip'           : 'functional.zip',
-        'reduce'        : 'functional.reduce',
         'reversed'      : 'functional.reversed',
-        'filter'        : 'functional.filter',
         'super'         : 'descriptor.W_Super',
         'staticmethod'  : 'descriptor.StaticMethod',
         'classmethod'   : 'descriptor.ClassMethod',
@@ -118,7 +118,7 @@
            builtin = space.interpclass_w(w_builtin)
            if isinstance(builtin, module.Module):
                return builtin
-       # no builtin! make a default one.  Given them None, at least.
+       # no builtin! make a default one.  Give them None, at least.
        builtin = module.Module(space, None)
        space.setitem(builtin.w_dict, space.wrap('None'), space.w_None)
        return builtin
diff --git a/pypy/module/__builtin__/app_functional.py b/pypy/module/__builtin__/app_functional.py
--- a/pypy/module/__builtin__/app_functional.py
+++ b/pypy/module/__builtin__/app_functional.py
@@ -48,4 +48,118 @@
         # Very intentionally *not* +=, that would have different semantics if
         # start was a mutable type, such as a list
         last = last + x
-    return last
\ No newline at end of file
+    return last
+
+def map(func, *collections):
+    """map(function, sequence[, sequence, ...]) -> list
+
+Return a list of the results of applying the function to the items of
+the argument sequence(s).  If more than one sequence is given, the
+function is called with an argument list consisting of the corresponding
+item of each sequence, substituting None for missing values when not all
+sequences have the same length.  If the function is None, return a list of
+the items of the sequence (or a list of tuples if more than one sequence)."""
+    if not collections:
+        raise TypeError("map() requires at least two arguments")
+    num_collections = len(collections)
+    none_func = func is None
+    if num_collections == 1:
+        if none_func:
+            return list(collections[0])
+        else:
+            # Special case for the really common case of a single collection,
+            # this can be eliminated if we could unroll that loop that creates
+            # `args` based on whether or not len(collections) was constant
+            result = []
+            for item in collections[0]:
+                result.append(func(item))
+            return result
+    result = []
+    # Pair of (iterator, has_finished)
+    iterators = [(iter(seq), False) for seq in collections]
+    while True:
+        cont = False
+        args = []
+        for idx, (iterator, has_finished) in enumerate(iterators):
+            val = None
+            if not has_finished:
+                try:
+                    val = next(iterator)
+                except StopIteration:
+                    iterators[idx] = (None, True)
+                else:
+                    cont = True
+            args.append(val)
+        args = tuple(args)
+        if cont:
+            if none_func:
+                result.append(args)
+            else:
+                result.append(func(*args))
+        else:
+            return result
+
+sentinel = object()
+
+def reduce(func, sequence, initial=sentinel):
+    """reduce(function, sequence[, initial]) -> value
+
+Apply a function of two arguments cumulatively to the items of a sequence,
+from left to right, so as to reduce the sequence to a single value.
+For example, reduce(lambda x, y: x+y, [1, 2, 3, 4, 5]) calculates
+((((1+2)+3)+4)+5).  If initial is present, it is placed before the items
+of the sequence in the calculation, and serves as a default when the
+sequence is empty."""
+    iterator = iter(sequence)
+    if initial is sentinel:
+        try:
+            initial = next(iterator)
+        except StopIteration:
+            raise TypeError("reduce() of empty sequence with no initial value")
+    result = initial
+    for item in iterator:
+        result = func(result, item)
+    return result
+
+def filter(func, seq):
+    """filter(function or None, sequence) -> list, tuple, or string
+
+Return those items of sequence for which function(item) is true.  If
+function is None, return the items that are true.  If sequence is a tuple
+or string, return the same type, else return a list."""
+    if func is None:
+        func = bool
+    if isinstance(seq, str):
+        return _filter_string(func, seq, str)
+    elif isinstance(seq, unicode):
+        return _filter_string(func, seq, unicode)
+    elif isinstance(seq, tuple):
+        return _filter_tuple(func, seq)
+    result = []
+    for item in seq:
+        if func(item):
+            result.append(item)
+    return result
+
+def _filter_string(func, string, str_type):
+    if func is bool and type(string) is str_type:
+        return string
+    result = []
+    for i in range(len(string)):
+        # You must call __getitem__ on the strings, simply iterating doesn't
+        # work :/
+        item = string[i]
+        if func(item):
+            if not isinstance(item, str_type):
+                raise TypeError("__getitem__ returned a non-string type")
+            result.append(item)
+    return str_type().join(result)
+
+def _filter_tuple(func, seq):
+    result = []
+    for i in range(len(seq)):
+        # Again, must call __getitem__, at least there are tests.
+        item = seq[i]
+        if func(item):
+            result.append(item)
+    return tuple(result)
\ No newline at end of file
diff --git a/pypy/module/__builtin__/compiling.py b/pypy/module/__builtin__/compiling.py
--- a/pypy/module/__builtin__/compiling.py
+++ b/pypy/module/__builtin__/compiling.py
@@ -82,8 +82,8 @@
         raise OperationError(space.w_TypeError,
               w('eval() arg 1 must be a string or code object'))
 
-    caller = space.getexecutioncontext().gettopframe_nohidden()
     if space.is_w(w_globals, space.w_None):
+        caller = space.getexecutioncontext().gettopframe_nohidden()
         if caller is None:
             w_globals = space.newdict()
             if space.is_w(w_locals, space.w_None):
@@ -95,14 +95,10 @@
     elif space.is_w(w_locals, space.w_None):
         w_locals = w_globals
 
-    try:
-        space.getitem(w_globals, space.wrap('__builtins__'))
-    except OperationError, e:
-        if not e.match(space, space.w_KeyError):
-            raise
-        if caller is not None:
-            w_builtin = space.builtin.pick_builtin(caller.w_globals)
-            space.setitem(w_globals, space.wrap('__builtins__'), w_builtin)
+    # xxx removed: adding '__builtins__' to the w_globals dict, if there
+    # is none.  This logic was removed as costly (it requires to get at
+    # the gettopframe_nohidden()).  I bet no test fails, and it's a really
+    # obscure case.
 
     return codeobj.exec_code(space, w_globals, w_locals)
 
diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py
--- a/pypy/module/__builtin__/functional.py
+++ b/pypy/module/__builtin__/functional.py
@@ -116,118 +116,6 @@
     """
     return min_max(space, __args__, "min")
 
- at unwrap_spec(collections_w="args_w")
-def map(space, w_func, collections_w):
-    """does 3 separate things, hence this enormous docstring.
-       1.  if function is None, return a list of tuples, each with one
-           item from each collection.  If the collections have different
-           lengths,  shorter ones are padded with None.
-
-       2.  if function is not None, and there is only one collection,
-           apply function to every item in the collection and return a
-           list of the results.
-
-       3.  if function is not None, and there are several collections,
-           repeatedly call the function with one argument from each
-           collection.  If the collections have different lengths,
-           shorter ones are padded with None
-    """
-    if not collections_w:
-        msg = "map() requires at least two arguments"
-        raise OperationError(space.w_TypeError, space.wrap(msg))
-    none_func = space.is_w(w_func, space.w_None)
-    if len(collections_w) == 1:
-        w_collection = collections_w[0]
-        if none_func:
-            result_w = space.unpackiterable(w_collection)
-        else:
-            result_w = map_single_collection(space, w_func, w_collection)
-    else:
-        result_w = map_multiple_collections(space, w_func, collections_w,
-                                            none_func)
-    return space.newlist(result_w)
-
-def map_single_collection(space, w_func, w_collection):
-    """Special case for 'map(func, coll)', where 'func' is not None and there
-    is only one 'coll' argument."""
-    w_iter = space.iter(w_collection)
-    # xxx special hacks for speed
-    from pypy.interpreter import function, pycode
-    if isinstance(w_func, function.Function):
-        # xxx compatibility issue: what if func_code is modified in the
-        # middle of running map()??  That's far too obscure for me to care...
-        code = w_func.getcode()
-        fast_natural_arity = code.fast_natural_arity
-        if fast_natural_arity == (1|pycode.PyCode.FLATPYCALL):
-            assert isinstance(code, pycode.PyCode)
-            return map_single_user_function(code, w_func, w_iter)
-    # /xxx end of special hacks
-    return map_single_other_callable(space, w_func, w_iter)
-
-def map_single_other_callable(space, w_func, w_iter):
-    result_w = []
-    while True:
-        try:
-            w_item = space.next(w_iter)
-        except OperationError, e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            break
-        result_w.append(space.call_function(w_func, w_item))
-    return result_w
-map_single_other_callable._dont_inline_ = True
-
-from pypy.rlib.jit import JitDriver
-mapjitdriver = JitDriver(greens = ['code'],
-                         reds = ['w_func', 'w_iter', 'result_w'])
-def map_single_user_function(code, w_func, w_iter):
-    result_w = []
-    while True:
-        mapjitdriver.can_enter_jit(code=code, w_func=w_func,
-                                   w_iter=w_iter, result_w=result_w)
-        mapjitdriver.jit_merge_point(code=code, w_func=w_func,
-                                     w_iter=w_iter, result_w=result_w)
-        space = w_func.space
-        try:
-            w_item = space.next(w_iter)
-        except OperationError, e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            break
-        new_frame = space.createframe(code, w_func.w_func_globals,
-                                      w_func)
-        new_frame.locals_stack_w[0] = w_item
-        w_res = new_frame.run()
-        result_w.append(w_res)
-    return result_w
-
-def map_multiple_collections(space, w_func, collections_w, none_func):
-    result_w = []
-    iterators_w = [space.iter(w_seq) for w_seq in collections_w]
-    num_iterators = len(iterators_w)
-    while True:
-        cont = False
-        args_w = [space.w_None] * num_iterators
-        for i in range(num_iterators):
-            if iterators_w[i] is not None:
-                try:
-                    args_w[i] = space.next(iterators_w[i])
-                except OperationError, e:
-                    if not e.match(space, space.w_StopIteration):
-                        raise
-                    iterators_w[i] = None
-                else:
-                    cont = True
-        if not cont:
-            break
-        w_args = space.newtuple(args_w)
-        if none_func:
-            w_res = w_args
-        else:
-            w_res = space.call(w_func, w_args)
-        result_w.append(w_res)
-    return result_w
-
 @unwrap_spec(sequences_w="args_w")
 def zip(space, sequences_w):
     """Return a list of tuples, where the nth tuple contains every nth item of
@@ -249,90 +137,6 @@
             return space.newlist(result_w)
         result_w.append(space.newtuple(items_w))
 
-def reduce(space, w_func, w_sequence, w_initial=NoneNotWrapped):
-    """ Apply function of two arguments cumulatively to the items of sequence,
-        from left to right, so as to reduce the sequence to a single value.
-        Optionally begin with an initial value.
-    """
-    w_iter = space.iter(w_sequence)
-    if w_initial is None:
-        try:
-            w_initial = space.next(w_iter)
-        except OperationError, e:
-            if e.match(space, space.w_StopIteration):
-                msg = "reduce() of empty sequence with no initial value"
-                raise OperationError(space.w_TypeError, space.wrap(msg))
-            raise
-    w_result = w_initial
-    while True:
-        try:
-            w_next = space.next(w_iter)
-        except OperationError, e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            break
-        w_result = space.call_function(w_func, w_result, w_next)
-    return w_result
-
-def filter(space, w_func, w_seq):
-    """construct a list of those elements of collection for which function
-       is True.  If function is None, then return the items in the sequence
-       which are True.
-    """
-    if space.is_true(space.isinstance(w_seq, space.w_str)):
-        return _filter_string(space, w_func, w_seq, space.w_str)
-    if space.is_true(space.isinstance(w_seq, space.w_unicode)):
-        return _filter_string(space, w_func, w_seq, space.w_unicode)
-    if space.is_true(space.isinstance(w_seq, space.w_tuple)):
-        return _filter_tuple(space, w_func, w_seq)
-    w_iter = space.iter(w_seq)
-    result_w = []
-    none_func = space.is_w(w_func, space.w_None)
-    while True:
-        try:
-            w_next = space.next(w_iter)
-        except OperationError, e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            break
-        if none_func:
-            w_keep = w_next
-        else:
-            w_keep = space.call_function(w_func, w_next)
-        if space.is_true(w_keep):
-            result_w.append(w_next)
-    return space.newlist(result_w)
-
-def _filter_tuple(space, w_func, w_tuple):
-    none_func = space.is_w(w_func, space.w_None)
-    length = space.len_w(w_tuple)
-    result_w = []
-    for i in range(length):
-        w_item = space.getitem(w_tuple, space.wrap(i))
-        if none_func:
-            w_keep = w_item
-        else:
-            w_keep = space.call_function(w_func, w_item)
-        if space.is_true(w_keep):
-            result_w.append(w_item)
-    return space.newtuple(result_w)
-
-def _filter_string(space, w_func, w_string, w_str_type):
-    none_func = space.is_w(w_func, space.w_None)
-    if none_func and space.is_w(space.type(w_string), w_str_type):
-        return w_string
-    length = space.len_w(w_string)
-    result_w = []
-    for i in range(length):
-        w_item = space.getitem(w_string, space.wrap(i))
-        if none_func or space.is_true(space.call_function(w_func, w_item)):
-            if not space.is_true(space.isinstance(w_item, w_str_type)):
-                msg = "__getitem__ returned a non-string type"
-                raise OperationError(space.w_TypeError, space.wrap(msg))
-            result_w.append(w_item)
-    w_empty = space.call_function(w_str_type)
-    return space.call_method(w_empty, "join", space.newlist(result_w))
-
 class W_Enumerate(Wrappable):
 
     def __init__(self, w_iter, w_start):
diff --git a/pypy/module/__builtin__/test/test_functional.py b/pypy/module/__builtin__/test/test_functional.py
--- a/pypy/module/__builtin__/test/test_functional.py
+++ b/pypy/module/__builtin__/test/test_functional.py
@@ -147,7 +147,7 @@
        assert list(range(A())) == [0, 1, 2, 3, 4]
        assert list(range(0, A())) == [0, 1, 2, 3, 4]
        assert list(range(0, 10, A())) == [0, 5]
- 
+
    def test_range_float(self):
       assert list(range(0.1, 2.0, 1.1)) == [0, 1]
 
diff --git a/pypy/module/_continuation/interp_continuation.py b/pypy/module/_continuation/interp_continuation.py
--- a/pypy/module/_continuation/interp_continuation.py
+++ b/pypy/module/_continuation/interp_continuation.py
@@ -19,6 +19,11 @@
         #  - normal:      self.sthread != None, not is_empty_handle(self.h)
         #  - finished:    self.sthread != None, is_empty_handle(self.h)
 
+    def __del__(self):
+        sthread = self.sthread
+        if sthread is not None and not sthread.is_empty_handle(self.h):
+            sthread.destroy(self.h)
+
     def check_sthread(self):
         ec = self.space.getexecutioncontext()
         if ec.stacklet_thread is not self.sthread:
@@ -28,6 +33,8 @@
     def descr_init(self, w_callable, __args__):
         if self.sthread is not None:
             raise geterror(self.space, "continulet already __init__ialized")
+        sthread = build_sthread(self.space)
+        workaround_disable_jit(sthread)
         #
         # hackish: build the frame "by hand", passing it the correct arguments
         space = self.space
@@ -41,7 +48,6 @@
         self.bottomframe = bottomframe
         #
         global_state.origin = self
-        sthread = build_sthread(self.space)
         self.sthread = sthread
         h = sthread.new(new_stacklet_callback)
         post_switch(sthread, h)
@@ -71,6 +77,7 @@
                 global_state.clear()
                 raise geterror(self.space, "continulet already finished")
         self.check_sthread()
+        workaround_disable_jit(self.sthread)
         #
         global_state.origin = self
         if to is None:
@@ -259,6 +266,16 @@
         sthread = ec.stacklet_thread = SThread(space, ec)
     return sthread
 
+def workaround_disable_jit(sthread):
+    # A bad workaround to kill the JIT anywhere in this thread.
+    # This forces all the frames.  It's a bad workaround because
+    # it takes O(depth) time, and it will cause some "abort:
+    # vable escape" in the JIT.  The goal is to prevent any frame
+    # from being still virtuals, because the JIT generates code
+    # to un-virtualizable them "on demand" by loading values based
+    # on FORCE_TOKEN, which is an address in the stack.
+    sthread.ec.force_all_frames()
+
 # ____________________________________________________________
 
 def permute(space, args_w):
diff --git a/pypy/module/_file/test/test_file.py b/pypy/module/_file/test/test_file.py
--- a/pypy/module/_file/test/test_file.py
+++ b/pypy/module/_file/test/test_file.py
@@ -260,8 +260,12 @@
 class AppTestNonblocking(object):
     def setup_class(cls):
         from pypy.module._file.interp_file import W_File
-        
+
         cls.old_read = os.read
+
+        if option.runappdirect:
+            py.test.skip("works with internals of _file impl on py.py")
+
         state = [0]
         def read(fd, n=None):
             if fd != 42:
@@ -281,7 +285,7 @@
 
     def teardown_class(cls):
         os.read = cls.old_read
-        
+
     def test_nonblocking_file(self):
         res = self.stream.read()
         assert res == 'xyz'
@@ -403,24 +407,24 @@
         with self.file(self.temppath, 'w') as f:
             f.write('foo')
         assert f.closed
-        
+
         with self.file(self.temppath, 'r') as f:
             s = f.readline()
 
         assert s == "foo"
         assert f.closed
-    
+
     def test_subclass_with(self):
         file = self.file
         class C(file):
             def __init__(self, *args, **kwargs):
                 self.subclass_closed = False
                 file.__init__(self, *args, **kwargs)
-            
+
             def close(self):
                 self.subclass_closed = True
                 file.close(self)
-        
+
         with C(self.temppath, 'w') as f:
             pass
         assert f.subclass_closed
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -161,9 +161,6 @@
     @binop
     def mul(self, v1, v2):
         return v1 * v2
-    @binop
-    def div(self, v1, v2):
-        return v1 / v2
 
     @unaryop
     def pos(self, v):
@@ -217,6 +214,14 @@
         return float2string(self.for_computation(self.unbox(item)), 'g', rfloat.DTSF_STR_PRECISION)
 
     @binop
+    def div(self, v1, v2):
+        try:
+            return v1 / v2
+        except ZeroDivisionError:
+            if v1 == v2 == 0.0:
+                return rfloat.NAN
+            return rfloat.copysign(rfloat.INFINITY, v1 * v2)
+    @binop
     def mod(self, v1, v2):
         return math.fmod(v1, v2)
     @binop
@@ -295,6 +300,11 @@
         return str(widen(self.unbox(item)))
 
     @binop
+    def div(self, v1, v2):
+        if v2 == 0:
+            return 0
+        return v1 / v2
+    @binop
     def mod(self, v1, v2):
         return v1 % v2
 
@@ -426,23 +436,22 @@
     pass
 
 if LONG_BIT == 32:
-    class W_LongDtype(W_Int32Dtype):
-        pass
+    long_dtype = W_Int32Dtype
+    ulong_dtype = W_UInt32Dtype
+elif LONG_BIT == 64:
+    long_dtype = W_Int64Dtype
+    ulong_dtype = W_UInt64Dtype
+else:
+    assert False
 
-    class W_ULongDtype(W_UInt32Dtype):
-        pass
-else:
-    class W_LongDtype(W_Int64Dtype):
-        pass
+class W_LongDtype(long_dtype):
+    num = 7
+    aliases = ["l"]
+    applevel_types = ["int"]
 
-    class W_ULongDtype(W_UInt64Dtype):
-        pass
-
-W_LongDtype.num = 7
-W_LongDtype.aliases = ["l"]
-W_LongDtype.applevel_types = ["int"]
-W_ULongDtype.num = 8
-W_ULongDtype.aliases = ["L"]
+class W_ULongDtype(ulong_dtype):
+    num = 8
+    aliases = ["L"]
 
 W_Float32Dtype = create_low_level_dtype(
     num = 11, kind = FLOATINGLTR, name = "float32",
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -285,7 +285,9 @@
             assert b[i] == i * 5
 
     def test_div(self):
-        from numpy import array, dtype
+        from math import isnan
+        from numpy import array, dtype, inf
+
         a = array(range(1, 6))
         b = a / a
         for i in range(5):
@@ -297,6 +299,24 @@
         for i in range(5):
             assert b[i] == 1
 
+        a = array([-1, 0, 1])
+        b = array([0, 0, 0])
+        c = a / b
+        assert (c == [0, 0, 0]).all()
+
+        a = array([-1.0, 0.0, 1.0])
+        b = array([0.0, 0.0, 0.0])
+        c = a / b
+        assert c[0] == -inf
+        assert isnan(c[1])
+        assert c[2] == inf
+
+        b = array([-0.0, -0.0, -0.0])
+        c = a / b
+        assert c[0] == inf
+        assert isnan(c[1])
+        assert c[2] == -inf
+
     def test_div_other(self):
         from numpy import array
         a = array(range(5))
diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py
--- a/pypy/module/pypyjit/interp_jit.py
+++ b/pypy/module/pypyjit/interp_jit.py
@@ -137,20 +137,15 @@
 
     def jump_absolute(self, jumpto, _, ec=None):
         if we_are_jitted():
-            # Normally, the tick counter is decremented by 100 for every
-            # Python opcode.  Here, to better support JIT compilation of
-            # small loops, we decrement it by a possibly smaller constant.
-            # We get the maximum 100 when the (unoptimized) trace length
-            # is at least 3200 (a bit randomly).
-            trace_length = r_uint(current_trace_length())
-            decr_by = trace_length // 32
-            if decr_by < 1:
-                decr_by = 1
-            elif decr_by > 100:    # also if current_trace_length() returned -1
-                decr_by = 100
+            #
+            # assume that only threads are using the bytecode counter
+            decr_by = 0
+            if self.space.actionflag.has_bytecode_counter:   # constant-folded
+                if self.space.threadlocals.gil_ready:   # quasi-immutable field
+                    decr_by = _get_adapted_tick_counter()
             #
             self.last_instr = intmask(jumpto)
-            ec.bytecode_trace(self, intmask(decr_by))
+            ec.bytecode_trace(self, decr_by)
             jumpto = r_uint(self.last_instr)
         #
         pypyjitdriver.can_enter_jit(frame=self, ec=ec, next_instr=jumpto,
@@ -158,6 +153,20 @@
                                     is_being_profiled=self.is_being_profiled)
         return jumpto
 
+def _get_adapted_tick_counter():
+    # Normally, the tick counter is decremented by 100 for every
+    # Python opcode.  Here, to better support JIT compilation of
+    # small loops, we decrement it by a possibly smaller constant.
+    # We get the maximum 100 when the (unoptimized) trace length
+    # is at least 3200 (a bit randomly).
+    trace_length = r_uint(current_trace_length())
+    decr_by = trace_length // 32
+    if decr_by < 1:
+        decr_by = 1
+    elif decr_by > 100:    # also if current_trace_length() returned -1
+        decr_by = 100
+    return intmask(decr_by)
+
 
 PyCode__initialize = PyCode._initialize
 
diff --git a/pypy/module/pypyjit/test_pypy_c/model.py b/pypy/module/pypyjit/test_pypy_c/model.py
--- a/pypy/module/pypyjit/test_pypy_c/model.py
+++ b/pypy/module/pypyjit/test_pypy_c/model.py
@@ -225,6 +225,8 @@
         # strip comment
         if '#' in line:
             line = line[:line.index('#')]
+        if line.strip() == 'guard_not_invalidated?':
+            return 'guard_not_invalidated', None, [], '...', False
         # find the resvar, if any
         if ' = ' in line:
             resvar, _, line = line.partition(' = ')
@@ -249,7 +251,7 @@
             descr = descr[len('descr='):]
         else:
             descr = None
-        return opname, resvar, args, descr
+        return opname, resvar, args, descr, True
 
     @classmethod
     def preprocess_expected_src(cls, src):
@@ -258,13 +260,23 @@
         # replaced with the corresponding operations, so that tests don't have
         # to repeat it every time
         ticker_check = """
+            guard_not_invalidated?
+            ticker0 = getfield_raw(ticker_address, descr=<SignedFieldDescr pypysig_long_struct.c_value .*>)
+            ticker_cond0 = int_lt(ticker0, 0)
+            guard_false(ticker_cond0, descr=...)
+        """
+        src = src.replace('--TICK--', ticker_check)
+        #
+        # this is the ticker check generated if we have threads
+        thread_ticker_check = """
+            guard_not_invalidated?
             ticker0 = getfield_raw(ticker_address, descr=<SignedFieldDescr pypysig_long_struct.c_value .*>)
             ticker1 = int_sub(ticker0, 1)
             setfield_raw(ticker_address, ticker1, descr=<SignedFieldDescr pypysig_long_struct.c_value .*>)
             ticker_cond0 = int_lt(ticker1, 0)
             guard_false(ticker_cond0, descr=...)
         """
-        src = src.replace('--TICK--', ticker_check)
+        src = src.replace('--THREAD-TICK--', thread_ticker_check)
         #
         # this is the ticker check generated in PyFrame.handle_operation_error
         exc_ticker_check = """
@@ -298,7 +310,7 @@
         if not cond:
             raise InvalidMatch(message, frame=sys._getframe(1))
 
-    def match_op(self, op, (exp_opname, exp_res, exp_args, exp_descr)):
+    def match_op(self, op, (exp_opname, exp_res, exp_args, exp_descr, _)):
         self._assert(op.name == exp_opname, "operation mismatch")
         self.match_var(op.res, exp_res)
         if exp_args != ['...']:
@@ -341,7 +353,7 @@
         what is after the '...'
         """
         iter_exp_ops = iter(expected_ops)
-        iter_ops = iter(self.ops)
+        iter_ops = RevertableIterator(self.ops)
         for opindex, exp_op in enumerate(iter_exp_ops):
             try:
                 if exp_op == '...':
@@ -360,6 +372,9 @@
                             break
                 self.match_op(op, exp_op)
             except InvalidMatch, e:
+                if exp_op[4] is False:    # optional operation
+                    iter_ops.revert_one()
+                    continue       # try to match with the next exp_op
                 e.opindex = opindex
                 raise
         #
@@ -372,8 +387,8 @@
                 return ''
             text = str(py.code.Source(src).deindent().indent())
             lines = text.splitlines(True)
-            if opindex is not None and 0 <= opindex < len(lines):
-                lines[opindex] = lines[opindex].rstrip() + '\t<=====\n'
+            if opindex is not None and 0 <= opindex <= len(lines):
+                lines.insert(opindex, '\n\t===== HERE =====\n')
             return ''.join(lines)
         #
         expected_src = self.preprocess_expected_src(expected_src)
@@ -398,3 +413,18 @@
         else:
             return True
 
+
+class RevertableIterator(object):
+    def __init__(self, sequence):
+        self.sequence = sequence
+        self.index = 0
+    def __iter__(self):
+        return self
+    def next(self):
+        index = self.index
+        if index == len(self.sequence):
+            raise StopIteration
+        self.index = index + 1
+        return self.sequence[index]
+    def revert_one(self):
+        self.index -= 1
diff --git a/pypy/module/pypyjit/test_pypy_c/test_00_model.py b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
--- a/pypy/module/pypyjit/test_pypy_c/test_00_model.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
@@ -145,15 +145,17 @@
 
     def test_parse_op(self):
         res = OpMatcher.parse_op("  a =   int_add(  b,  3 ) # foo")
-        assert res == ("int_add", "a", ["b", "3"], None)
+        assert res == ("int_add", "a", ["b", "3"], None, True)
         res = OpMatcher.parse_op("guard_true(a)")
-        assert res == ("guard_true", None, ["a"], None)
+        assert res == ("guard_true", None, ["a"], None, True)
         res = OpMatcher.parse_op("setfield_gc(p0, i0, descr=<foobar>)")
-        assert res == ("setfield_gc", None, ["p0", "i0"], "<foobar>")
+        assert res == ("setfield_gc", None, ["p0", "i0"], "<foobar>", True)
         res = OpMatcher.parse_op("i1 = getfield_gc(p0, descr=<foobar>)")
-        assert res == ("getfield_gc", "i1", ["p0"], "<foobar>")
+        assert res == ("getfield_gc", "i1", ["p0"], "<foobar>", True)
         res = OpMatcher.parse_op("p0 = force_token()")
-        assert res == ("force_token", "p0", [], None)
+        assert res == ("force_token", "p0", [], None, True)
+        res = OpMatcher.parse_op("guard_not_invalidated?")
+        assert res == ("guard_not_invalidated", None, [], '...', False)
 
     def test_exact_match(self):
         loop = """
@@ -341,7 +343,7 @@
             # this is the actual loop
             'int_lt', 'guard_true', 'int_add',
             # this is the signal checking stuff
-            'getfield_raw', 'int_sub', 'setfield_raw', 'int_lt', 'guard_false',
+            'guard_not_invalidated', 'getfield_raw', 'int_lt', 'guard_false',
             'jump'
             ]
 
@@ -407,7 +409,7 @@
             # this is the actual loop
             'int_lt', 'guard_true', 'force_token', 'int_add',
             # this is the signal checking stuff
-            'getfield_raw', 'int_sub', 'setfield_raw', 'int_lt', 'guard_false',
+            'guard_not_invalidated', 'getfield_raw', 'int_lt', 'guard_false',
             'jump'
             ]
 
@@ -425,10 +427,9 @@
             guard_true(i6, descr=...)
             i8 = int_add(i4, 1)
             # signal checking stuff
+            guard_not_invalidated(descr=...)
             i10 = getfield_raw(37212896, descr=<.* pypysig_long_struct.c_value .*>)
-            i12 = int_sub(i10, 1)
-            setfield_raw(37212896, i12, descr=<.* pypysig_long_struct.c_value .*>)
-            i14 = int_lt(i12, 0)
+            i14 = int_lt(i10, 0)
             guard_false(i14, descr=...)
             jump(p0, p1, p2, p3, i8, descr=...)
         """)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_misc.py b/pypy/module/pypyjit/test_pypy_c/test_misc.py
--- a/pypy/module/pypyjit/test_pypy_c/test_misc.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_misc.py
@@ -329,4 +329,20 @@
             guard_false(i28, descr=...)
             i30 = int_lshift(i20, 24)
             i31 = int_or(i26, i30)
-        """ % {"32_bit_only": extra})
\ No newline at end of file
+        """ % {"32_bit_only": extra})
+
+    def test_eval(self):
+        def main():
+            i = 1
+            a = compile('x+x+x+x+x+x', 'eval', 'eval')
+            b = {'x': 7}
+            while i < 1000:
+                y = eval(a,b,b)  # ID: eval
+                i += 1
+            return y
+
+        log = self.run(main)
+        assert log.result == 42
+        # the following assertion fails if the loop was cancelled due
+        # to "abort: vable escape"
+        assert len(log.loops_by_id("eval")) == 1
diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -41,7 +41,7 @@
             guard_true(i32, descr=...)
             i34 = int_add(i6, 1)
             --TICK--
-            jump(p0, p1, p2, p3, p4, p5, i34, p7, p8, i9, i10, p11, i12, p13, descr=<Loop4>)
+            jump(p0, p1, p2, p3, p4, p5, i34, p7, p8, i9, i10, p11, i12, p13, descr=...)
         """)
 
     def test_long(self):
@@ -106,7 +106,7 @@
             i58 = int_add_ovf(i6, i57)
             guard_no_overflow(descr=...)
             --TICK--
-            jump(p0, p1, p2, p3, p4, p5, i58, i7, descr=<Loop4>)
+            jump(p0, p1, p2, p3, p4, p5, i58, i7, descr=...)
         """)
 
     def test_str_mod(self):
@@ -156,4 +156,41 @@
             i40 = int_sub(i4, 1)
             --TICK--
             jump(p0, p1, p2, p3, i40, i38, descr=<Loop0>)
-        """)
\ No newline at end of file
+        """)
+
+    def test_getattr_promote(self):
+        def main(n):
+            class A(object):
+                def meth_a(self):
+                    return 1
+                def meth_b(self):
+                    return 2
+            a = A()
+
+            l = ['a', 'b']
+            s = 0
+            for i in range(n):
+                name = 'meth_' + l[i & 1]
+                meth = getattr(a, name) # ID: getattr
+                s += meth()
+            return s
+
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loops = log.loops_by_filename(self.filepath)
+        assert len(loops) == 2
+        for loop in loops:
+            loop.match_by_id('getattr','''
+            guard_not_invalidated(descr=...)
+            i32 = strlen(p31)
+            i34 = int_add(5, i32)
+            p35 = newstr(i34)
+            strsetitem(p35, 0, 109)
+            strsetitem(p35, 1, 101)
+            strsetitem(p35, 2, 116)
+            strsetitem(p35, 3, 104)
+            strsetitem(p35, 4, 95)
+            copystrcontent(p31, p35, 0, 5, i32)
+            i49 = call(ConstClass(_ll_2_str_eq_nonnull__rpy_stringPtr_rpy_stringPtr), p35, ConstPtr(ptr48), descr=<SignedCallDescr>)
+            guard_value(i49, 1, descr=<Guard8>)
+            ''')
diff --git a/pypy/module/pypyjit/test_pypy_c/test_thread.py b/pypy/module/pypyjit/test_pypy_c/test_thread.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/pypyjit/test_pypy_c/test_thread.py
@@ -0,0 +1,28 @@
+from pypy.module.pypyjit.test_pypy_c.test_00_model import BaseTestPyPyC
+
+
+class TestThread(BaseTestPyPyC):
+    def test_simple(self):
+        def main(n):
+            import thread
+            def f():
+                i = 0
+                while i < n:
+                    i += 1
+                done.release()
+
+            done = thread.allocate_lock()
+            done.acquire()
+            thread.start_new_thread(f, ())
+            done.acquire()
+            return 0
+        log = self.run(main, [500])
+        assert round(log.result, 6) == round(main(500), 6)
+        loop, = log.loops_by_filename(self.filepath)
+        assert loop.match("""
+            i2 = int_lt(i0, i1)
+            guard_true(i2, descr=...)
+            i3 = int_add(i0, 1)
+            --THREAD-TICK--
+            jump(..., descr=<Loop0>)
+        """)
diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py
--- a/pypy/module/signal/interp_signal.py
+++ b/pypy/module/signal/interp_signal.py
@@ -109,8 +109,11 @@
         p = pypysig_getaddr_occurred()
         value = p.c_value
         if self.has_bytecode_counter:    # this 'if' is constant-folded
-            value -= by
-            p.c_value = value
+            if jit.isconstant(by) and by == 0:
+                pass     # normally constant-folded too
+            else:
+                value -= by
+                p.c_value = value
         return value
 
 
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -47,7 +47,7 @@
         'pypy_initial_path'     : 'state.pypy_initial_path',
 
         '_getframe'             : 'vm._getframe', 
-        '_current_frames'       : 'vm._current_frames', 
+        '_current_frames'       : 'currentframes._current_frames', 
         'setrecursionlimit'     : 'vm.setrecursionlimit', 
         'getrecursionlimit'     : 'vm.getrecursionlimit', 
         'setcheckinterval'      : 'vm.setcheckinterval', 
diff --git a/pypy/module/sys/currentframes.py b/pypy/module/sys/currentframes.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/sys/currentframes.py
@@ -0,0 +1,78 @@
+"""
+Implementation of the 'sys._current_frames()' routine.
+"""
+from pypy.interpreter import gateway
+
+app = gateway.applevel('''
+"NOT_RPYTHON"
+import __builtin__
+
+class fake_code(object):
+    co_name = "?"
+    co_filename = "?"
+    co_firstlineno = 0
+
+class fake_frame(object):
+    f_back = None
+    f_builtins = __builtin__.__dict__
+    f_code = fake_code()
+    f_exc_traceback = None
+    f_exc_type = None
+    f_exc_value = None
+    f_globals = {}
+    f_lasti = -1
+    f_lineno = 0
+    f_locals = {}
+    f_restricted = False
+    f_trace = None
+
+    def __init__(self, f):
+        if f is not None:
+            for name in ["f_builtins", "f_code", "f_globals", "f_lasti",
+                         "f_lineno"]:
+                setattr(self, name, getattr(f, name))
+''')
+
+def _current_frames(space):
+    """_current_frames() -> dictionary
+
+    Return a dictionary mapping each current thread T's thread id to T's
+    current stack "frame".  Functions in the traceback module can build the
+    call stack given such a frame.
+
+    Note that in PyPy this returns fake frame objects, to avoid a runtime
+    penalty everywhere with the JIT.  (So far these fake frames can be
+    completely uninformative depending on the JIT state; we could return
+    more with more efforts.)
+
+    This function should be used for specialized purposes only."""
+    w_result = space.newdict()
+    w_fake_frame = app.wget(space, "fake_frame")
+    w_fake_code  = app.wget(space, "fake_code")
+    ecs = space.threadlocals.getallvalues()
+    for thread_ident, ec in ecs.items():
+        vref = ec.topframeref
+        frames = []
+        while not vref.virtual:
+            f = vref()
+            if f is None:
+                break
+            frames.append(f)
+            vref = f.f_backref
+        else:
+            frames.append(None)
+        #
+        w_topframe = space.wrap(None)
+        w_prevframe = None
+        for f in frames:
+            w_nextframe = space.call_function(w_fake_frame, space.wrap(f))
+            if w_prevframe is None:
+                w_topframe = w_nextframe
+            else:
+                space.setattr(w_prevframe, space.wrap('f_back'), w_nextframe)
+            w_prevframe = w_nextframe
+        #
+        space.setitem(w_result,
+                      space.wrap(thread_ident),
+                      w_topframe)
+    return w_result
diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -556,7 +556,7 @@
             return sys._current_frames()
         frames = f()
         assert frames.keys() == [0]
-        assert frames[0].f_code.co_name == 'f'
+        assert frames[0].f_code.co_name in ('f', '?')
 
 class AppTestCurrentFramesWithThread(AppTestCurrentFrames):
     def setup_class(cls):
@@ -568,23 +568,25 @@
         import thread
 
         thread_id = thread.get_ident()
-        self.ready = False
         def other_thread():
-            self.ready = True
             print "thread started"
-            time.sleep(5)
+            lock2.release()
+            lock1.acquire()
+        lock1 = thread.allocate_lock()
+        lock2 = thread.allocate_lock()
+        lock1.acquire()
+        lock2.acquire()
         thread.start_new_thread(other_thread, ())
 
         def f():
-            for i in range(100):
-                if self.ready: break
-                time.sleep(0.1)
+            lock2.acquire()
             return sys._current_frames()
 
         frames = f()
+        lock1.release()
         thisframe = frames.pop(thread_id)
-        assert thisframe.f_code.co_name == 'f'
+        assert thisframe.f_code.co_name in ('f', '?')
 
         assert len(frames) == 1
         _, other_frame = frames.popitem()
-        assert other_frame.f_code.co_name == 'other_thread'
+        assert other_frame.f_code.co_name in ('other_thread', '?')
diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -45,25 +45,6 @@
     f.mark_as_escaped()
     return space.wrap(f)
 
-def _current_frames(space):
-    """_current_frames() -> dictionary
-
-    Return a dictionary mapping each current thread T's thread id to T's
-    current stack frame.
-
-    This function should be used for specialized purposes only."""
-    raise OperationError(space.w_NotImplementedError,
-        space.wrap("XXX sys._current_frames() incompatible with the JIT"))
-    w_result = space.newdict()
-    ecs = space.threadlocals.getallvalues()
-    for thread_ident, ec in ecs.items():
-        f = ec.gettopframe_nohidden()
-        f.mark_as_escaped()
-        space.setitem(w_result,
-                      space.wrap(thread_ident),
-                      space.wrap(f))
-    return w_result
-
 def setrecursionlimit(space, w_new_limit):
     """setrecursionlimit() sets the maximum number of nested calls that
 can occur before a RuntimeError is raised.  On PyPy the limit is
diff --git a/pypy/module/thread/gil.py b/pypy/module/thread/gil.py
--- a/pypy/module/thread/gil.py
+++ b/pypy/module/thread/gil.py
@@ -17,6 +17,7 @@
 class GILThreadLocals(OSThreadLocals):
     """A version of OSThreadLocals that enforces a GIL."""
     gil_ready = False
+    _immutable_fields_ = ['gil_ready?']
 
     def initialize(self, space):
         # add the GIL-releasing callback as an action on the space
diff --git a/pypy/module/thread/threadlocals.py b/pypy/module/thread/threadlocals.py
--- a/pypy/module/thread/threadlocals.py
+++ b/pypy/module/thread/threadlocals.py
@@ -8,9 +8,14 @@
 
     def __init__(self):
         self._valuedict = {}   # {thread_ident: ExecutionContext()}
+        self._freeze_()
+
+    def _freeze_(self):
+        self._valuedict.clear()
         self._mainthreadident = 0
         self._mostrecentkey = 0        # fast minicaching for the common case
         self._mostrecentvalue = None   # fast minicaching for the common case
+        return False
 
     def getvalue(self):
         ident = thread.get_ident()
diff --git a/pypy/objspace/std/objecttype.py b/pypy/objspace/std/objecttype.py
--- a/pypy/objspace/std/objecttype.py
+++ b/pypy/objspace/std/objecttype.py
@@ -44,7 +44,6 @@
         raise OperationError(space.w_TypeError,
                              space.wrap("__class__ assignment: only for heap types"))
     w_oldcls = space.type(w_obj)
-    # XXX taint space should raise a TaintError here if w_oldcls is tainted
     assert isinstance(w_oldcls, W_TypeObject)
     if w_oldcls.get_full_instance_layout() == w_newcls.get_full_instance_layout():
         w_obj.setclass(space, w_newcls)
diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py
--- a/pypy/objspace/std/stringobject.py
+++ b/pypy/objspace/std/stringobject.py
@@ -161,19 +161,19 @@
 
 def str_swapcase__String(space, w_self):
     self = w_self._value
-    res = [' '] * len(self)
+    builder = StringBuilder(len(self))
     for i in range(len(self)):
         ch = self[i]
         if ch.isupper():
             o = ord(ch) + 32
-            res[i] = chr(o)
+            builder.append(chr(o))
         elif ch.islower():
             o = ord(ch) - 32
-            res[i] = chr(o)
+            builder.append(chr(o))
         else:
-            res[i] = ch
+            builder.append(ch)
 
-    return space.wrap("".join(res))
+    return space.wrap(builder.build())
 
 
 def str_capitalize__String(space, w_self):
@@ -199,19 +199,21 @@
 
 def str_title__String(space, w_self):
     input = w_self._value
-    buffer = [' '] * len(input)
+    builder = StringBuilder(len(input))
     prev_letter=' '
 
-    for pos in range(0, len(input)):
+    for pos in range(len(input)):
         ch = input[pos]
         if not prev_letter.isalpha():
-            buffer[pos] = _upper(ch)
+            ch = _upper(ch)
+            builder.append(ch)
         else:
-            buffer[pos] = _lower(ch)
+            ch = _lower(ch)
+            builder.append(ch)
 
-        prev_letter = buffer[pos]
+        prev_letter = ch
 
-    return space.wrap("".join(buffer))
+    return space.wrap(builder.build())
 
 def str_split__String_None_ANY(space, w_self, w_none, w_maxsplit=-1):
     maxsplit = space.int_w(w_maxsplit)
@@ -754,27 +756,21 @@
     input = w_self._value
     width = space.int_w(w_width)
 
-    if len(input) >= width:
+    num_zeros = width - len(input)
+    if num_zeros <= 0:
         # cannot return w_self, in case it is a subclass of str
         return space.wrap(input)
 
-    buf = [' '] * width
+    builder = StringBuilder(width)
     if len(input) > 0 and (input[0] == '+' or input[0] == '-'):
-        buf[0] = input[0]
+        builder.append(input[0])
         start = 1
-        middle = width - len(input) + 1
     else:
         start = 0
-        middle = width - len(input)
 
-    for i in range(start, middle):
-        buf[i] = '0'
-
-    for i in range(middle, width):
-        buf[i] = input[start]
-        start = start + 1
-
-    return space.wrap("".join(buf))
+    builder.append_multiple_char('0', num_zeros)
+    builder.append_slice(input, start, len(input))
+    return space.wrap(builder.build())
 
 
 def hash__String(space, w_str):
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -10,7 +10,8 @@
 from pypy.objspace.std import identitydict
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.rlib.objectmodel import current_object_addr_as_int, compute_hash
-from pypy.rlib.jit import promote, elidable_promote, we_are_jitted
+from pypy.rlib.jit import promote, elidable_promote, we_are_jitted,\
+     promote_string
 from pypy.rlib.jit import elidable, dont_look_inside, unroll_safe
 from pypy.rlib.rarithmetic import intmask, r_uint
 
@@ -101,6 +102,7 @@
                           'instancetypedef',
                           'terminator',
                           '_version_tag?',
+                          'interplevel_cls',
                           ]
 
     # for config.objspace.std.getattributeshortcut
@@ -399,6 +401,7 @@
         if version_tag is None:
             tup = w_self._lookup_where(name)
             return tup
+        name = promote_string(name)
         w_class, w_value = w_self._pure_lookup_where_with_method_cache(name, version_tag)
         return w_class, unwrap_cell(space, w_value)
 
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -317,54 +317,54 @@
     input = w_self._value
     if len(input) == 0:
         return W_UnicodeObject.EMPTY
-    result = [u'\0'] * len(input)
-    result[0] = unichr(unicodedb.toupper(ord(input[0])))
+    builder = UnicodeBuilder(len(input))
+    builder.append(unichr(unicodedb.toupper(ord(input[0]))))
     for i in range(1, len(input)):
-        result[i] = unichr(unicodedb.tolower(ord(input[i])))
-    return W_UnicodeObject(u''.join(result))
+        builder.append(unichr(unicodedb.tolower(ord(input[i]))))
+    return W_UnicodeObject(builder.build())
 
 def unicode_title__Unicode(space, w_self):
     input = w_self._value
     if len(input) == 0:
         return w_self
-    result = [u'\0'] * len(input)
+    builder = UnicodeBuilder(len(input))
 
     previous_is_cased = False
     for i in range(len(input)):
         unichar = ord(input[i])
         if previous_is_cased:
-            result[i] = unichr(unicodedb.tolower(unichar))
+            builder.append(unichr(unicodedb.tolower(unichar)))
         else:
-            result[i] = unichr(unicodedb.totitle(unichar))
+            builder.append(unichr(unicodedb.totitle(unichar)))
         previous_is_cased = unicodedb.iscased(unichar)
-    return W_UnicodeObject(u''.join(result))
+    return W_UnicodeObject(builder.build())
 
 def unicode_lower__Unicode(space, w_self):
     input = w_self._value
-    result = [u'\0'] * len(input)
+    builder = UnicodeBuilder(len(input))
     for i in range(len(input)):
-        result[i] = unichr(unicodedb.tolower(ord(input[i])))
-    return W_UnicodeObject(u''.join(result))
+        builder.append(unichr(unicodedb.tolower(ord(input[i]))))
+    return W_UnicodeObject(builder.build())
 
 def unicode_upper__Unicode(space, w_self):
     input = w_self._value
-    result = [u'\0'] * len(input)
+    builder = UnicodeBuilder(len(input))
     for i in range(len(input)):
-        result[i] = unichr(unicodedb.toupper(ord(input[i])))
-    return W_UnicodeObject(u''.join(result))
+        builder.append(unichr(unicodedb.toupper(ord(input[i]))))
+    return W_UnicodeObject(builder.build())
 
 def unicode_swapcase__Unicode(space, w_self):
     input = w_self._value
-    result = [u'\0'] * len(input)
+    builder = UnicodeBuilder(len(input))
     for i in range(len(input)):
         unichar = ord(input[i])
         if unicodedb.islower(unichar):
-            result[i] = unichr(unicodedb.toupper(unichar))
+            builder.append(unichr(unicodedb.toupper(unichar)))
         elif unicodedb.isupper(unichar):
-            result[i] = unichr(unicodedb.tolower(unichar))
+            builder.append(unichr(unicodedb.tolower(unichar)))
         else:
-            result[i] = input[i]
-    return W_UnicodeObject(u''.join(result))
+            builder.append(input[i])
+    return W_UnicodeObject(builder.build())
 
 def _normalize_index(length, index):
     if index < 0:
diff --git a/pypy/objspace/taint.py b/pypy/objspace/taint.py
deleted file mode 100644
--- a/pypy/objspace/taint.py
+++ /dev/null
@@ -1,294 +0,0 @@
-"""
-Just an experiment.
-"""
-import os
-from pypy.objspace.std.objspace import StdObjSpace
-from pypy.objspace.proxy import patch_space_in_place
-from pypy.objspace.thunk import nb_forcing_args
-from pypy.interpreter.error import OperationError
-from pypy.interpreter import baseobjspace, gateway, executioncontext
-from pypy.interpreter.function import Method
-from pypy.interpreter.pyframe import PyFrame
-from pypy.tool.sourcetools import func_with_new_name
-from pypy.rlib.unroll import unrolling_iterable
-
-
-class W_Tainted(baseobjspace.W_Root):
-    def __init__(self, w_obj):
-        self.w_obj = w_obj
-
-##    def getdict(self, space):
-##        return taint(self.w_obj.getdict(space))
-
-##    def getdictvalue(self, space, attr):
-##        return taint(self.w_obj.getdictvalue(space, attr))
-
-##    def setdictvalue(self, space, attr, w_value):
-##        return self.w_obj.setdictvalue(space, attr, w_value)
-
-##    ...
-
-class W_TaintBomb(baseobjspace.W_Root):
-    filename = '?'
-    codename = '?'
-    codeline = 0
-
-    def __init__(self, space, operr):
-        self.space = space
-        self.operr = operr
-        self.record_debug_info()
-
-    def record_debug_info(self):
-        ec = self.space.getexecutioncontext()
-        frame = ec.gettopframe_nohidden()
-        if isinstance(frame, PyFrame):     # and, in particular, frame != None
-            self.filename = frame.pycode.co_filename
-            self.codename = frame.pycode.co_name
-            self.codeline = frame.get_last_lineno()
-        if get_debug_level(self.space) > 0:
-            self.debug_dump()
-
-    def debug_dump(self):
-        os.write(2, 'Taint Bomb from file "%s", line %d, in %s\n    %s\n' % (
-            self.filename, self.codeline, self.codename,
-            self.operr.errorstr(self.space)))
-
-    def explode(self):
-        #msg = self.operr.errorstr(space)
-        raise OperationError(self.space.w_TaintError, self.space.w_None)
-
-
-def taint(w_obj):
-    """Return a tainted version of the argument."""
-    if w_obj is None or isinstance(w_obj, W_Tainted):
-        return w_obj
-    else:
-        return W_Tainted(w_obj)
-app_taint = gateway.interp2app(taint)
-
-def is_tainted(space, w_obj):
-    """Return whether the argument is tainted."""
-    res = isinstance(w_obj, W_Tainted) or isinstance(w_obj, W_TaintBomb)
-    return space.wrap(res)
-app_is_tainted = gateway.interp2app(is_tainted)
-
-def untaint(space, w_expectedtype, w_obj):
-    """untaint(expectedtype, tainted_obj) -> obj
-Untaint untainted_obj and return it. If the result is not of expectedtype,
-raise a type error."""
-    if (isinstance(w_expectedtype, W_Tainted) or
-        isinstance(w_expectedtype, W_TaintBomb)):
-        raise OperationError(space.w_TypeError,
-                  space.wrap("untaint() arg 1 must be an untainted type"))
-    if not space.is_true(space.isinstance(w_expectedtype, space.w_type)):
-        raise OperationError(space.w_TypeError,
-                             space.wrap("untaint() arg 1 must be a type"))
-    if isinstance(w_obj, W_Tainted):
-        w_obj = w_obj.w_obj
-    elif isinstance(w_obj, W_TaintBomb):
-        w_obj.explode()
-    #if isinstance(w_expectedtype, W_Tainted):
-    #    w_expectedtype = w_expectedtype.w_obj
-    w_realtype = space.type(w_obj)
-    if not space.is_w(w_realtype, w_expectedtype):
-        #msg = "expected an object of type '%s'" % (
-        #    w_expectedtype.getname(space),)
-        #    #w_realtype.getname(space))
-        raise OperationError(space.w_TaintError, space.w_None)
-    return w_obj
-app_untaint = gateway.interp2app(untaint)
-
-# ____________________________________________________________
-
- at gateway.unwrap_spec(args_w='args_w')
-def taint_atomic_function(space, w_func, args_w):
-    newargs_w = []
-    tainted = False
-    for w_arg in args_w:
-        if isinstance(w_arg, W_Tainted):
-            tainted = True
-            w_arg = w_arg.w_obj
-        elif isinstance(w_arg, W_TaintBomb):
-            return w_arg
-        newargs_w.append(w_arg)
-    w_newargs = space.newtuple(newargs_w)
-    try:
-        w_res = space.call(w_func, w_newargs)
-    except OperationError, operr:
-        if not tainted:
-            raise
-        return W_TaintBomb(space, operr)
-    if tainted:
-        w_res = taint(w_res)
-    return w_res
-
-app_taint_atomic_function = gateway.interp2app(taint_atomic_function)
-
-def taint_atomic(space, w_callable):
-    """decorator to make a callable "taint-atomic": if the function is called
-with tainted arguments, those are untainted. The result of the function is
-tainted again.  All exceptions that the callable raises are turned into
-taint bombs."""
-    meth = Method(space, space.w_fn_taint_atomic_function,
-                  w_callable, space.type(w_callable))
-    return space.wrap(meth)
-app_taint_atomic = gateway.interp2app(taint_atomic)
-
-# ____________________________________________________________
-
-executioncontext.ExecutionContext.taint_debug = 0
-
- at gateway.unwrap_spec(level=int)
-def taint_debug(space, level):
-    """Set the debug level. If the debug level is greater than 0, the creation
-of taint bombs will print debug information. For debugging purposes
-only!"""
-    space.getexecutioncontext().taint_debug = level
-app_taint_debug = gateway.interp2app(taint_debug)
-
-def taint_look(space, w_obj):
-    """Print some info about the taintedness of an object. For debugging
-purposes only!"""
-    if isinstance(w_obj, W_Tainted):
-        info = space.type(w_obj.w_obj).getname(space)
-        msg = space.str_w(w_obj.w_obj.getrepr(space, info))
-        msg = 'Taint Box %s\n' % msg
-        os.write(2, msg)
-    elif isinstance(w_obj, W_TaintBomb):
-        w_obj.debug_dump()
-    else:
-        os.write(2, 'not tainted\n')
-app_taint_look = gateway.interp2app(taint_look)
-
-def get_debug_level(space):
-    return space.getexecutioncontext().taint_debug
-
-def debug_bomb(space, operr):
-    ec = space.getexecutioncontext()
-    filename = '?'
-    codename = '?'
-    codeline = 0
-    frame = ec.gettopframe_nohidden()
-    if isinstance(frame, PyFrame):     # and, in particular, frame != None
-        filename = frame.pycode.co_filename
-        codename = frame.pycode.co_name
-        codeline = frame.get_last_lineno()
-    os.write(2, 'Taint Bomb in file "%s", line %d, in %s\n    %s\n' % (
-        filename, codeline, codename, operr.errorstr(space)))
-
-# ____________________________________________________________
-
-
-class TaintSpace(StdObjSpace):
-
-    def __init__(self, *args, **kwds):
-        StdObjSpace.__init__(self, *args, **kwds)
-        w_dict = self.newdict()
-        self.setitem(w_dict, self.wrap("__doc__"), self.wrap("""\
-Exception that is raised when an operation revealing information on a tainted
-object is performed."""))
-        self.w_TaintError = self.call_function(
-            self.w_type,
-            self.wrap("TaintError"),
-            self.newtuple([self.w_Exception]),
-            w_dict
-            )
-        w___pypy__ = self.getbuiltinmodule("__pypy__")
-        self.setattr(w___pypy__, self.wrap('taint'),
-                     self.wrap(app_taint))
-        self.setattr(w___pypy__, self.wrap('is_tainted'),
-                     self.wrap(app_is_tainted))
-        self.setattr(w___pypy__, self.wrap('untaint'),
-                     self.wrap(app_untaint))
-        self.w_fn_taint_atomic_function = self.wrap(app_taint_atomic_function)
-        self.setattr(w___pypy__, self.wrap('taint_atomic'),
-                     self.wrap(app_taint_atomic))
-        self.setattr(w___pypy__, self.wrap('TaintError'),
-                     self.w_TaintError)
-        self.setattr(w___pypy__, self.wrap('_taint_debug'),
-                     self.wrap(app_taint_debug))
-        self.setattr(w___pypy__, self.wrap('_taint_look'),
-                     self.wrap(app_taint_look))
-        patch_space_in_place(self, 'taint', proxymaker)
-
-        # XXX may leak info, perfomance hit, what about taint bombs?
-        from pypy.objspace.std.typeobject import W_TypeObject
-
-        def taint_lookup(w_obj, name):
-            if isinstance(w_obj, W_Tainted):
-                w_obj = w_obj.w_obj
-            w_type = self.type(w_obj)
-            assert isinstance(w_type, W_TypeObject)
-            return w_type.lookup(name)
-
-        def taint_lookup_in_type_where(w_obj, name):
-            if isinstance(w_obj, W_Tainted):
-                w_type = w_obj.w_obj
-            else:
-                w_type = w_obj
-            assert isinstance(w_type, W_TypeObject)
-            return w_type.lookup_where(name)
-
-        self.lookup = taint_lookup
-        self.lookup_in_type_where = taint_lookup_in_type_where
-
-
-Space = TaintSpace
-
-
-def tainted_error(space, name):
-    #msg = "operation '%s' forbidden on tainted object" % (name,)
-    raise OperationError(space.w_TaintError, space.w_None)# space.wrap(msg))
-
-
-RegularMethods = dict.fromkeys(
-    [name for name, _, _, _ in baseobjspace.ObjSpace.MethodTable])
-
-TaintResultIrregularMethods = dict.fromkeys(
-    ['wrap', 'call_args'] +
-    [name for name in baseobjspace.ObjSpace.IrregularOpTable
-          if name.startswith('new')])
-
-def proxymaker(space, name, parentfn):
-    arity = nb_forcing_args[name]
-    indices = unrolling_iterable(range(arity))
-    if name in RegularMethods:
-
-        def proxy(*args_w):
-            newargs_w = ()
-            tainted = False
-            for i in indices:
-                w_arg = args_w[i]
-                if isinstance(w_arg, W_Tainted):
-                    tainted = True
-                    w_arg = w_arg.w_obj
-                elif isinstance(w_arg, W_TaintBomb):
-                    return w_arg
-                newargs_w += (w_arg,)
-            newargs_w += args_w[arity:]
-            try:
-                w_res = parentfn(*newargs_w)
-            except OperationError, operr:
-                if not tainted:
-                    raise
-                return W_TaintBomb(space, operr)
-            if tainted:
-                w_res = taint(w_res)
-            return w_res
-
-    elif arity == 0:
-        return None
-
-    else:
-
-        def proxy(*args_w):
-            for i in indices:
-                w_arg = args_w[i]
-                if isinstance(w_arg, W_Tainted):
-                    tainted_error(space, name)
-                elif isinstance(w_arg, W_TaintBomb):
-                    w_arg.explode()
-            return parentfn(*args_w)
-
-    proxy = func_with_new_name(proxy, '%s_proxy' % name)
-    return proxy
diff --git a/pypy/objspace/test/test_taintobjspace.py b/pypy/objspace/test/test_taintobjspace.py
deleted file mode 100644
--- a/pypy/objspace/test/test_taintobjspace.py
+++ /dev/null
@@ -1,77 +0,0 @@
-from pypy.conftest import gettestobjspace
-
-class AppTest_Taint:
-
-    def setup_class(cls):
-        cls.space = gettestobjspace('taint')
-
-    def test_simple(self):
-        from __pypy__ import taint, untaint, TaintError
-        x = taint(6)
-        x = x * 7
-        raises(TaintError, "if x: y = 1")
-        t = type(x)
-        raises(TaintError, "if t is int: y = 1")
-        assert untaint(int, x) == 42
-        raises(TaintError, "untaint(float, x)")
-
-    def test_bomb(self):
-        from __pypy__ import taint, untaint, TaintError
-        x = taint(6)
-        x = x / 0
-        raises(TaintError, "if x: y = 1")
-        t = type(x)
-        raises(TaintError, "if t is int: y = 1")
-        raises(TaintError, "untaint(int, x)")
-        raises(TaintError, "untaint(float, x)")
-
-    def test_taint_atomic(self):
-        from __pypy__ import taint, untaint, TaintError, taint_atomic
-        x = taint(6)
-        x *= 7
-
-        def dummy(x):
-            if x > 40:
-                return 5
-            else:
-                return 3
-        dummy = taint_atomic(dummy)
-
-        y = dummy(x)
-        raises(TaintError, "if y == 3: z = 1")
-        assert untaint(int, y) == 5
-
-    def test_taint_atomic_exception(self):
-        from __pypy__ import taint, untaint, TaintError, taint_atomic
-        x = taint(6)
-        x *= 7
-
-        def dummy(x):
-            if x + "world" == "hello world":
-                return 5
-            else:
-                return 3
-        dummy = taint_atomic(dummy)
-
-        y = dummy(x)
-        raises(TaintError, "if y == 3: z = 1")
-        raises(TaintError, "untaint(int, y)")
-
-    def test_taint_atomic_incoming_bomb(self):
-        from __pypy__ import taint, untaint, TaintError, taint_atomic
-        x = taint(6)
-        x /= 0
-        lst = []
-
-        def dummy(x):
-            lst.append("running!")
-            if x > 40:
-                return 5
-            else:
-                return 3
-        dummy = taint_atomic(dummy)
-
-        y = dummy(x)
-        raises(TaintError, "if y == 3: z = 1")
-        assert lst == []
-        raises(TaintError, "untaint(int, y)")
diff --git a/pypy/rlib/_jit_vref.py b/pypy/rlib/_jit_vref.py
--- a/pypy/rlib/_jit_vref.py
+++ b/pypy/rlib/_jit_vref.py
@@ -25,6 +25,10 @@
     def simple_call(self):
         return self.s_instance
 
+    def getattr(self, s_attr):
+        assert s_attr.const == 'virtual'
+        return annmodel.s_Bool
+
     def rtyper_makerepr(self, rtyper):
         if rtyper.type_system.name == 'lltypesystem':
             return vrefrepr
@@ -61,6 +65,13 @@
                              " prebuilt virtual_ref")
         return lltype.nullptr(OBJECTPTR.TO)
 
+    def rtype_getattr(self, hop):
+        s_attr = hop.args_s[1]
+        assert s_attr.const == 'virtual'
+        v = hop.inputarg(self, arg=0)
+        hop.exception_cannot_occur()
+        return hop.genop('jit_is_virtual', [v], resulttype = lltype.Bool)
+
 from pypy.rpython.ootypesystem.rclass import OBJECT
 
 class OOVRefRepr(VRefRepr):
diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -38,6 +38,7 @@
     possible arguments are:
 
     * promote - promote the argument from a variable into a constant
+    * promote_string - same, but promote string by *value*
     * access_directly - directly access a virtualizable, as a structure
                         and don't treat it as a virtualizable
     * fresh_virtualizable - means that virtualizable was just allocated.
@@ -51,6 +52,9 @@
 def promote(x):
     return hint(x, promote=True)
 
+def promote_string(x):
+    return hint(x, promote_string=True)
+
 def dont_look_inside(func):
     """ Make sure the JIT does not trace inside decorated function
     (it becomes a call instead)
@@ -313,6 +317,12 @@
             raise InvalidVirtualRef
         return self._x
 
+    @property
+    def virtual(self):
+        """A property that is True if the vref contains a virtual that would
+        be forced by the '()' operator."""
+        return self._state == 'non-forced'
+
     def _finish(self):
         if self._state == 'non-forced':
             self._state = 'invalid'
diff --git a/pypy/rlib/rerased.py b/pypy/rlib/rerased.py
--- a/pypy/rlib/rerased.py
+++ b/pypy/rlib/rerased.py
@@ -218,15 +218,13 @@
         [v_value] = hop.inputargs(lltype.Signed)
         c_one = hop.inputconst(lltype.Signed, 1)
         hop.exception_is_here()
-        v2 = hop.genop('int_lshift_ovf', [v_value, c_one],
+        v2 = hop.genop('int_add_ovf', [v_value, v_value],
                        resulttype = lltype.Signed)
         v2p1 = hop.genop('int_add', [v2, c_one],
                          resulttype = lltype.Signed)
         v_instance = hop.genop('cast_int_to_ptr', [v2p1],
                                resulttype=self.lowleveltype)
-        v = hop.genop('cast_opaque_ptr', [v_instance],
-                      resulttype=self.lowleveltype)
-        return v
+        return v_instance
 
     def convert_const(self, value):
         if value._identity is _identity_for_ints:
@@ -266,10 +264,10 @@
         return hop.genop('int_rshift', [v2, c_one], resulttype=lltype.Signed)
 
     def rtype_erase_int(self, hop):
-        hop.exception_is_here()
         [v_value] = hop.inputargs(lltype.Signed)
         c_one = hop.inputconst(lltype.Signed, 1)
-        v2 = hop.genop('int_lshift_ovf', [v_value, c_one],
+        hop.exception_is_here()
+        v2 = hop.genop('int_add_ovf', [v_value, v_value],
                        resulttype = lltype.Signed)
         v2p1 = hop.genop('int_add', [v2, c_one],
                          resulttype = lltype.Signed)
diff --git a/pypy/rlib/test/test__jit_vref.py b/pypy/rlib/test/test__jit_vref.py
--- a/pypy/rlib/test/test__jit_vref.py
+++ b/pypy/rlib/test/test__jit_vref.py
@@ -27,10 +27,13 @@
     x1 = X()
     vref = virtual_ref(x1)
     assert vref._state == 'non-forced'
+    assert vref.virtual is True
     assert vref() is x1
     assert vref._state == 'forced'
+    assert vref.virtual is False
     virtual_ref_finish(vref, x1)
     assert vref._state == 'forced'
+    assert vref.virtual is False
     assert vref() is x1
 
 def test_direct_invalid():
@@ -135,6 +138,13 @@
         x = self.interpret(f, [])
         assert x == 42
 
+    def test_rtype_virtualattr(self):
+        def f():
+            vref = virtual_ref(X())
+            return vref.virtual
+        x = self.interpret(f, [])
+        assert x is False
+
 
 class TestLLtype(BaseTestVRef, LLRtypeMixin):
     OBJECTTYPE = OBJECTPTR
diff --git a/pypy/rlib/test/test_jit.py b/pypy/rlib/test/test_jit.py
--- a/pypy/rlib/test/test_jit.py
+++ b/pypy/rlib/test/test_jit.py
@@ -139,12 +139,11 @@
 
     def test_isconstant(self):
         def f(n):
-            assert n >= 0
             assert isconstant(n) is False
             l = []
             l.append(n)
             return len(l)
-        res = self.interpret(f, [234])
+        res = self.interpret(f, [-234])
         assert res == 1
 
 
diff --git a/pypy/rpython/lltypesystem/ll2ctypes.py b/pypy/rpython/lltypesystem/ll2ctypes.py
--- a/pypy/rpython/lltypesystem/ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/ll2ctypes.py
@@ -658,6 +658,8 @@
         if T == llmemory.GCREF:
             if isinstance(llobj._obj, _llgcopaque):
                 return ctypes.c_void_p(llobj._obj.intval)
+            if isinstance(llobj._obj, int):    # tagged pointer
+                return ctypes.c_void_p(llobj._obj)
             container = llobj._obj.container
             T = lltype.Ptr(lltype.typeOf(container))
             # otherwise it came from integer and we want a c_void_p with
@@ -1268,6 +1270,7 @@
 class _llgcopaque(lltype._container):
     _TYPE = llmemory.GCREF.TO
     _name = "_llgcopaque"
+    _read_directly_intval = True     # for _ptr._cast_to_int()
 
     def __init__(self, void_p):
         if isinstance(void_p, (int, long)):
@@ -1299,11 +1302,6 @@
             return _opaque_objs[self.intval // 2]
         return force_cast(PTRTYPE, self.intval)
 
-##     def _cast_to_int(self):
-##         return self.intval
-
-##     def _cast_to_adr(self):
-##         return _lladdress(self.intval)
 
 def cast_adr_to_int(addr):
     if isinstance(addr, llmemory.fakeaddress):
diff --git a/pypy/rpython/lltypesystem/llmemory.py b/pypy/rpython/lltypesystem/llmemory.py
--- a/pypy/rpython/lltypesystem/llmemory.py
+++ b/pypy/rpython/lltypesystem/llmemory.py
@@ -498,6 +498,8 @@
 
     def _cast_to_int(self, symbolic=False):
         if self:
+            if isinstance(self.ptr._obj0, int):    # tagged integer
+                return self.ptr._obj0
             if symbolic:
                 return AddressAsInt(self)
             else:
diff --git a/pypy/rpython/lltypesystem/lloperation.py b/pypy/rpython/lltypesystem/lloperation.py
--- a/pypy/rpython/lltypesystem/lloperation.py
+++ b/pypy/rpython/lltypesystem/lloperation.py
@@ -428,6 +428,7 @@
     'jit_marker':           LLOp(),
     'jit_force_virtualizable':LLOp(canrun=True),
     'jit_force_virtual':    LLOp(canrun=True),
+    'jit_is_virtual':       LLOp(canrun=True),
     'jit_force_quasi_immutable': LLOp(canrun=True),
     'get_exception_addr':   LLOp(),
     'get_exc_value_addr':   LLOp(),
diff --git a/pypy/rpython/lltypesystem/lltype.py b/pypy/rpython/lltypesystem/lltype.py
--- a/pypy/rpython/lltypesystem/lltype.py
+++ b/pypy/rpython/lltypesystem/lltype.py
@@ -1360,6 +1360,8 @@
         obj = normalizeptr(self, check)._getobj(check)
         if isinstance(obj, int):
             return obj     # special case for cast_int_to_ptr() results put into opaques
+        if getattr(obj, '_read_directly_intval', False):
+            return obj.intval   # special case for _llgcopaque
         result = intmask(obj._getid())
         # assume that id() returns an addressish value which is
         # not zero and aligned to at least a multiple of 4
diff --git a/pypy/rpython/lltypesystem/opimpl.py b/pypy/rpython/lltypesystem/opimpl.py
--- a/pypy/rpython/lltypesystem/opimpl.py
+++ b/pypy/rpython/lltypesystem/opimpl.py
@@ -538,6 +538,9 @@
 def op_jit_force_virtual(x):
     return x
 
+def op_jit_is_virtual(x):
+    return False
+
 def op_jit_force_quasi_immutable(*args):
     pass
 
diff --git a/pypy/rpython/lltypesystem/rtagged.py b/pypy/rpython/lltypesystem/rtagged.py
--- a/pypy/rpython/lltypesystem/rtagged.py
+++ b/pypy/rpython/lltypesystem/rtagged.py
@@ -43,7 +43,7 @@
         v_value = hop.inputarg(lltype.Signed, arg=1)
         c_one = hop.inputconst(lltype.Signed, 1)
         hop.exception_is_here()
-        v2 = hop.genop('int_lshift_ovf', [v_value, c_one],
+        v2 = hop.genop('int_add_ovf', [v_value, v_value],
                        resulttype = lltype.Signed)
         v2p1 = hop.genop('int_add', [v2, c_one],
                          resulttype = lltype.Signed)
diff --git a/pypy/rpython/lltypesystem/test/test_ll2ctypes.py b/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
--- a/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
@@ -1123,6 +1123,9 @@
 
         #assert lltype.cast_ptr_to_int(ref1) == intval
 
+        x = rffi.cast(llmemory.GCREF, -17)
+        assert lltype.cast_ptr_to_int(x) == -17
+
     def test_ptr_truth(self):
         abc = rffi.cast(lltype.Ptr(lltype.FuncType([], lltype.Void)), 0)
         assert not abc
diff --git a/pypy/rpython/rclass.py b/pypy/rpython/rclass.py
--- a/pypy/rpython/rclass.py
+++ b/pypy/rpython/rclass.py
@@ -191,6 +191,10 @@
                     "class %r inherits from its parent _immutable_=True, "
                     "so it should also declare _immutable_=True" % (
                     self.classdef,))
+            if loc.classdict.get('_immutable_').value is not True:
+                raise TyperError(
+                    "class %r: _immutable_ = something else than True" % (
+                    self.classdef,))
             hints = hints.copy()
             hints['immutable'] = True
         self.immutable_field_set = set()  # unless overwritten below
diff --git a/pypy/tool/logparser.py b/pypy/tool/logparser.py
--- a/pypy/tool/logparser.py
+++ b/pypy/tool/logparser.py
@@ -298,6 +298,8 @@
         image.paste(textpercent, (t1x, 5), textpercent)
         image.paste(textlabel,   (t2x, 5), textlabel)
         images.append(image)
+    if not images:
+        return None
     return combine(images, spacing=0, border=1, horizontal=False)
 
 def get_timesummary_single_image(totaltimes, totaltime0, componentdict,
@@ -333,6 +335,8 @@
         del totaltimes[None]
     img2 = render_histogram(totaltimes, totaltime0, {},
                             width, summarybarheight)
+    if img2 is None:
+        return img1
     return combine([img1, img2], spacing=spacing, horizontal=True)
 
 # ----------
diff --git a/pypy/translator/c/funcgen.py b/pypy/translator/c/funcgen.py
--- a/pypy/translator/c/funcgen.py
+++ b/pypy/translator/c/funcgen.py
@@ -833,7 +833,7 @@
         return 'INSTRUMENT_COUNT(%s);' % counter_label
             
     def OP_IS_EARLY_CONSTANT(self, op):
-        return self.expr(op.result)  + ' = 0;' # Allways false
+        return '%s = 0; /* IS_EARLY_CONSTANT */' % (self.expr(op.result),)
 
     def OP_JIT_MARKER(self, op):
         return '/* JIT_MARKER %s */' % op
@@ -845,6 +845,9 @@
         return '%s = %s; /* JIT_FORCE_VIRTUAL */' % (self.expr(op.result),
                                                      self.expr(op.args[0]))
 
+    def OP_JIT_IS_VIRTUAL(self, op):
+        return '%s = 0; /* JIT_IS_VIRTUAL */' % (self.expr(op.result),)
+
     def OP_JIT_FORCE_QUASI_IMMUTABLE(self, op):
         return '/* JIT_FORCE_QUASI_IMMUTABLE %s */' % op
 
diff --git a/pypy/translator/c/src/thread_nt.h b/pypy/translator/c/src/thread_nt.h
--- a/pypy/translator/c/src/thread_nt.h
+++ b/pypy/translator/c/src/thread_nt.h
@@ -245,7 +245,7 @@
     if (pending_acquires <= 0)
         return 0;
     InterlockedIncrement(&pending_acquires);
-    PulseEvent(&cond_gil);
+    PulseEvent(cond_gil);
 
     /* hack: the three following lines do a pthread_cond_wait(), and
        normally specifying a timeout of INFINITE would be fine.  But the
@@ -253,7 +253,7 @@
        (small) risk that PulseEvent misses the WaitForSingleObject().
        In this case the process will just sleep a few milliseconds. */
     LeaveCriticalSection(&mutex_gil);
-    WaitForSingleObject(&cond_gil, 15);
+    WaitForSingleObject(cond_gil, 15);
     EnterCriticalSection(&mutex_gil);
 
     InterlockedDecrement(&pending_acquires);
@@ -263,7 +263,7 @@
 void RPyGilRelease(void)
 {
     LeaveCriticalSection(&mutex_gil);
-    PulseEvent(&cond_gil);
+    PulseEvent(cond_gil);
 }
 
 void RPyGilAcquire(void)
diff --git a/pypy/translator/platform/linux.py b/pypy/translator/platform/linux.py
--- a/pypy/translator/platform/linux.py
+++ b/pypy/translator/platform/linux.py
@@ -1,5 +1,6 @@
 """Support for Linux."""
 
+import sys
 from pypy.translator.platform.posix import BasePosix
 
 class BaseLinux(BasePosix):
@@ -24,15 +25,18 @@
         return self._pkg_config("libffi", "--libs-only-L",
                                 ['/usr/lib/libffi'])
 
+    def library_dirs_for_libffi_a(self):
+        # places where we need to look for libffi.a
+        # XXX obscuuure!  only look for libffi.a if run with translate.py
+        if 'translate' in sys.modules:
+            return self.library_dirs_for_libffi() + ['/usr/lib']
+        else:
+            return []
+
 
 class Linux(BaseLinux):
     shared_only = ()    # it seems that on 32-bit linux, compiling with -fPIC
                         # gives assembler that asmgcc is not happy about.
 
-    def library_dirs_for_libffi_a(self):
-        # places where we need to look for libffi.a
-        return self.library_dirs_for_libffi() + ['/usr/lib']
-
-
 class Linux64(BaseLinux):
     pass

From noreply at buildbot.pypy.org  Mon Oct 17 20:27:14 2011
From: noreply at buildbot.pypy.org (edelsohn)
Date: Mon, 17 Oct 2011 20:27:14 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: Start PPC64 support in prologue.
Message-ID: <20111017182714.B9905820D7@wyvern.cs.uni-duesseldorf.de>

Author: edelsohn
Branch: ppc-jit-backend
Changeset: r48156:d0b4864a9813
Date: 2011-10-17 14:19 -0400
http://bitbucket.org/pypy/pypy/changeset/d0b4864a9813/

Log:	Start PPC64 support in prologue.

diff --git a/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py b/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py
--- a/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py
@@ -109,7 +109,7 @@
         if IS_PPC_32:
             self.mc.stwx(source_reg.value, 0, 0)
         else:
-            self.mc.std(source_reg.value, 0, 0)
+            self.mc.stdx(source_reg.value, 0, 0)
 
     def _save_nonvolatiles(self):
         for i, reg in enumerate(NONVOLATILES):
@@ -161,17 +161,21 @@
             # save r31 at the bottom of the stack frame
             self.mc.stw(r.SPP.value, r.SP.value, WORD)
         else:
-            self.mc.stdu(1, 1, -frame_depth)
-            self.mc.mflr(0)
-            self.mc.std(0, 1, frame_depth + 4)
+            self.mc.stdu(r.SP.value, r.SP.value, -frame_depth)
+            self.mc.mflr(r.r0.value)
+            self.mc.std(r.r0.value, r.SP.value, frame_depth + 2 * WORD)
         offset = GPR_SAVE_AREA + WORD
         # compute spilling pointer (SPP)
         self.mc.addi(r.SPP.value, r.SP.value, frame_depth - offset)
         self._save_nonvolatiles()
         # save r31, use r30 as scratch register
         # this is safe because r30 has been saved already
-        self.mc.lwz(r.r30.value, r.SP.value, WORD)
-        self.mc.stw(r.r30.value, r.SPP.value, WORD * len(NONVOLATILES))
+        if IS_PPC_32:
+            self.mc.lwz(r.r30.value, r.SP.value, WORD)
+            self.mc.stw(r.r30.value, r.SPP.value, WORD * len(NONVOLATILES))
+        else:
+            self.mc.ld(r.r30.value, r.SP.value, WORD)
+            self.mc.std(r.r30.value, r.SPP.value, WORD * len(NONVOLATILES))
         # branch to loop code
         curpos = self.mc.currpos()
         offset = target_pos - curpos

From noreply at buildbot.pypy.org  Mon Oct 17 21:15:42 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Mon, 17 Oct 2011 21:15:42 +0200 (CEST)
Subject: [pypy-commit] pypy no-force-guard-lazy-set: a branch to benchmark
 not forcing all lazy setfield/setarrayitem at guards
Message-ID: <20111017191542.7D3D1820D7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: no-force-guard-lazy-set
Changeset: r48157:c11cb030a542
Date: 2011-10-17 15:15 -0400
http://bitbucket.org/pypy/pypy/changeset/c11cb030a542/

Log:	a branch to benchmark not forcing all lazy setfield/setarrayitem at
	guards

diff --git a/pypy/jit/metainterp/optimizeopt/heap.py b/pypy/jit/metainterp/optimizeopt/heap.py
--- a/pypy/jit/metainterp/optimizeopt/heap.py
+++ b/pypy/jit/metainterp/optimizeopt/heap.py
@@ -39,7 +39,7 @@
         cached_fieldvalue = self._cached_fields.get(structvalue, None)
 
         # Hack to ensure constants are imported from the preamble
-        if cached_fieldvalue and fieldvalue.is_constant(): 
+        if cached_fieldvalue and fieldvalue.is_constant():
             optheap.optimizer.ensure_imported(cached_fieldvalue)
             cached_fieldvalue = self._cached_fields.get(structvalue, None)
 
@@ -50,7 +50,7 @@
             if not self._lazy_setfield_registered:
                 optheap._lazy_setfields_and_arrayitems.append(self)
                 self._lazy_setfield_registered = True
-            
+
         else:
             # this is the case where the pending setfield ends up
             # storing precisely the value that is already there,
@@ -336,7 +336,7 @@
             value = self.getvalue(op.getarg(0))
             assert not value.is_virtual()      # it must be a non-virtual
             fieldvalue = self.getvalue(op.getarglist()[-1])
-            if fieldvalue.is_virtual():
+            if True: #fieldvalue.is_virtual():
                 # this is the case that we leave to resume.py
                 opnum = op.getopnum()
                 if opnum == rop.SETFIELD_GC:
@@ -388,7 +388,7 @@
         #
         cf = self.field_cache(op.getdescr())
         cf.do_setfield(self, op)
-        
+
 
     def optimize_GETARRAYITEM_GC(self, op):
         arrayvalue = self.getvalue(op.getarg(0))

From noreply at buildbot.pypy.org  Mon Oct 17 21:24:51 2011
From: noreply at buildbot.pypy.org (justinpeel)
Date: Mon, 17 Oct 2011 21:24:51 +0200 (CEST)
Subject: [pypy-commit] pypy rgc-mem-pressure: fix to use rffi_platform
 correctly. thanks fijal
Message-ID: <20111017192451.19038820D7@wyvern.cs.uni-duesseldorf.de>

Author: Justin Peel <notmuchtotell at gmail.com>
Branch: rgc-mem-pressure
Changeset: r48158:17db19c683dd
Date: 2011-10-17 13:24 -0600
http://bitbucket.org/pypy/pypy/changeset/17db19c683dd/

Log:	fix to use rffi_platform correctly. thanks fijal

diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py
--- a/pypy/module/_hashlib/interp_hashlib.py
+++ b/pypy/module/_hashlib/interp_hashlib.py
@@ -12,6 +12,11 @@
 
 algorithms = ('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512')
 
+# HASH_MALLOC_SIZE is the size of EVP_MD, EVP_MD_CTX plus their points.
+# Used for adding memory pressure.
+HASH_MALLOC_SIZE = ropenssl.EVP_MD_SIZE + ropenssl.EVP_MD_CTX_SIZE \
+        + rffi.sizeof(ropenssl.EVP_MD) * 2
+
 class W_Hash(Wrappable):
     ctx = lltype.nullptr(ropenssl.EVP_MD_CTX.TO)
 
@@ -29,7 +34,7 @@
                                  space.wrap("unknown hash function"))
         ctx = lltype.malloc(ropenssl.EVP_MD_CTX.TO, flavor='raw')
         ropenssl.EVP_DigestInit(ctx, digest)
-        rgc.add_memory_pressure(ropenssl.HASH_MALLOC_SIZE + self._digest_size())
+        rgc.add_memory_pressure(HASH_MALLOC_SIZE + self._digest_size())
         self.ctx = ctx
 
     def __del__(self):
diff --git a/pypy/rlib/ropenssl.py b/pypy/rlib/ropenssl.py
--- a/pypy/rlib/ropenssl.py
+++ b/pypy/rlib/ropenssl.py
@@ -109,7 +109,9 @@
     GENERAL_NAME_st = rffi_platform.Struct(
         'struct GENERAL_NAME_st',
         [('type', rffi.INT),
-         ]) 
+         ])
+    EVP_MD_SIZE = rffi_platform.SizeOf('EVP_MD')
+    EVP_MD_CTX_SIZE = rffi_platform.SizeOf('EVP_MD_CTX')
 
 
 for k, v in rffi_platform.configure(CConfig).items():
@@ -277,19 +279,6 @@
 EVP_MD_CTX_cleanup = external(
     'EVP_MD_CTX_cleanup', [EVP_MD_CTX], rffi.INT, threadsafe=False)
 
-# size of EVP_MD, EVP_MD_CTX plus their points. Used for adding 
-# memory pressure
-
-# not sure why this code does not work
-#HASH_MALLOC_SIZE = rffi_platform.sizeof(EVP_MD.TO, eci) \
-#    + rffi_platform.sizeof(EVP_MD_CTX.TO, eci) \
-#    + rffi.sizeof(EVP_MD) + rffi.sizeof(EVP_MD_CTX)
-
-# but this code does
-HASH_MALLOC_SIZE = EVP_MD.TO.hints['getsize']() \
-     + EVP_MD_CTX.TO.hints['getsize']() \
-     + rffi.sizeof(EVP_MD) + rffi.sizeof(EVP_MD_CTX)
-
 def init_ssl():
     libssl_SSL_load_error_strings()
     libssl_SSL_library_init()

From noreply at buildbot.pypy.org  Mon Oct 17 21:40:32 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Oct 2011 21:40:32 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: Add a people.txt.
Message-ID: <20111017194032.6E638820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r3930:b557f4aadb0e
Date: 2011-10-17 21:40 +0200
http://bitbucket.org/pypy/extradoc/changeset/b557f4aadb0e/

Log:	Add a people.txt.

diff --git a/sprintinfo/gothenburg-2011-2/people.txt b/sprintinfo/gothenburg-2011-2/people.txt
new file mode 100644
--- /dev/null
+++ b/sprintinfo/gothenburg-2011-2/people.txt
@@ -0,0 +1,14 @@
+People coming to the Gothenburg sprint Nov 2nd - 9th, 2011
+============================================================
+
+People who have a ``?`` in their arrive/depart or accomodation
+column are known to be coming but there are no details 
+available yet from them.
+
+==================== ============== ===================== ==================
+    Name              Arrive/Depart     Accomodation       Food
+==================== ============== ===================== ==================
+Jacob Hallen         lives there
+Laura Creighton      lives there
+Armin Rigo           ?              ?
+==================== ============== ===================== ==================

From noreply at buildbot.pypy.org  Mon Oct 17 22:17:03 2011
From: noreply at buildbot.pypy.org (justinpeel)
Date: Mon, 17 Oct 2011 22:17:03 +0200 (CEST)
Subject: [pypy-commit] pypy rgc-mem-pressure: better estimate for size of
	W_Hash's mallocs
Message-ID: <20111017201703.09EF4820D7@wyvern.cs.uni-duesseldorf.de>

Author: Justin Peel <notmuchtotell at gmail.com>
Branch: rgc-mem-pressure
Changeset: r48159:46f5717326d9
Date: 2011-10-17 14:16 -0600
http://bitbucket.org/pypy/pypy/changeset/46f5717326d9/

Log:	better estimate for size of W_Hash's mallocs

diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py
--- a/pypy/module/_hashlib/interp_hashlib.py
+++ b/pypy/module/_hashlib/interp_hashlib.py
@@ -12,10 +12,12 @@
 
 algorithms = ('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512')
 
-# HASH_MALLOC_SIZE is the size of EVP_MD, EVP_MD_CTX plus their points.
-# Used for adding memory pressure.
+# HASH_MALLOC_SIZE is the size of EVP_MD, EVP_MD_CTX plus their points
+# Used for adding memory pressure. Last number is an (under?)estimate of
+# EVP_PKEY_CTX's size.
+# XXX: Make a better estimate here
 HASH_MALLOC_SIZE = ropenssl.EVP_MD_SIZE + ropenssl.EVP_MD_CTX_SIZE \
-        + rffi.sizeof(ropenssl.EVP_MD) * 2
+        + rffi.sizeof(ropenssl.EVP_MD) * 2 + 208
 
 class W_Hash(Wrappable):
     ctx = lltype.nullptr(ropenssl.EVP_MD_CTX.TO)

From noreply at buildbot.pypy.org  Mon Oct 17 22:44:15 2011
From: noreply at buildbot.pypy.org (justinpeel)
Date: Mon, 17 Oct 2011 22:44:15 +0200 (CEST)
Subject: [pypy-commit] pypy rgc-mem-pressure: Add memory pressure to locks
Message-ID: <20111017204415.965BD820D7@wyvern.cs.uni-duesseldorf.de>

Author: Justin Peel <notmuchtotell at gmail.com>
Branch: rgc-mem-pressure
Changeset: r48160:1fda06647621
Date: 2011-10-17 14:44 -0600
http://bitbucket.org/pypy/pypy/changeset/1fda06647621/

Log:	Add memory pressure to locks

diff --git a/pypy/module/thread/ll_thread.py b/pypy/module/thread/ll_thread.py
--- a/pypy/module/thread/ll_thread.py
+++ b/pypy/module/thread/ll_thread.py
@@ -2,10 +2,11 @@
 from pypy.rpython.lltypesystem import rffi, lltype, llmemory
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
 import py
-from pypy.rlib import jit
+from pypy.rlib import jit, rgc
 from pypy.rlib.debug import ll_assert
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.rpython.lltypesystem.lloperation import llop
+from pypy.rpython.tool import rffi_platform
 from pypy.tool import autopath
 
 class error(Exception):
@@ -49,7 +50,7 @@
 
 TLOCKP = rffi.COpaquePtr('struct RPyOpaque_ThreadLock',
                           compilation_info=eci)
-
+TLOCKP_SIZE = rffi_platform.sizeof('struct RPyOpaque_ThreadLock', eci)
 c_thread_lock_init = llexternal('RPyThreadLockInit', [TLOCKP], rffi.INT,
                                 threadsafe=False)   # may add in a global list
 c_thread_lock_dealloc_NOAUTO = llexternal('RPyOpaqueDealloc_ThreadLock',
@@ -164,6 +165,9 @@
     if rffi.cast(lltype.Signed, res) <= 0:
         lltype.free(ll_lock, flavor='raw', track_allocation=False)
         raise error("out of resources")
+    # Add some memory pressure for the size of the lock because it is an
+    # Opaque object
+    rgc.add_memory_pressure(TLOCKP_SIZE)
     return ll_lock
 
 def free_ll_lock(ll_lock):

From noreply at buildbot.pypy.org  Mon Oct 17 23:15:37 2011
From: noreply at buildbot.pypy.org (justinpeel)
Date: Mon, 17 Oct 2011 23:15:37 +0200 (CEST)
Subject: [pypy-commit] pypy rgc-mem-pressure: Add some memory pressure to
	pyexpat
Message-ID: <20111017211537.C9E0E820D7@wyvern.cs.uni-duesseldorf.de>

Author: Justin Peel <notmuchtotell at gmail.com>
Branch: rgc-mem-pressure
Changeset: r48161:f210fd304537
Date: 2011-10-17 15:15 -0600
http://bitbucket.org/pypy/pypy/changeset/f210fd304537/

Log:	Add some memory pressure to pyexpat

diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py
--- a/pypy/module/pyexpat/interp_pyexpat.py
+++ b/pypy/module/pyexpat/interp_pyexpat.py
@@ -4,9 +4,9 @@
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.error import OperationError
 from pypy.objspace.descroperation import object_setattr
+from pypy.rlib import rgc
+from pypy.rlib.unroll import unrolling_iterable
 from pypy.rpython.lltypesystem import rffi, lltype
-from pypy.rlib.unroll import unrolling_iterable
-
 from pypy.rpython.tool import rffi_platform
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
 from pypy.translator.platform import platform
@@ -104,6 +104,7 @@
 
     for name in xml_error_list:
         locals()[name] = rffi_platform.ConstantInteger(name)
+    XML_Parser_SIZE = rffi_platform.SizeOf("XML_Parser")
 
 for k, v in rffi_platform.configure(CConfigure).items():
     globals()[k] = v
@@ -779,7 +780,10 @@
             rffi.cast(rffi.CHAR, namespace_separator))
     else:
         xmlparser = XML_ParserCreate(encoding)
-
+    # Currently this is just the size of the pointer and some estimated bytes.
+    # The struct isn't actually defined in expat.h - it is in xmlparse.c
+    # XXX: find a good estimate of the XML_ParserStruct
+    rgc.add_memory_pressure(XML_Parser_SIZE + 300)
     if not xmlparser:
         raise OperationError(space.w_RuntimeError,
                              space.wrap('XML_ParserCreate failed'))

From noreply at buildbot.pypy.org  Tue Oct 18 00:37:53 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 18 Oct 2011 00:37:53 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Fix utf-8 encoding;
	all test_runicode passes.
Message-ID: <20111017223753.618A982A8A@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48163:8942f2c46162
Date: 2011-10-17 20:18 +0200
http://bitbucket.org/pypy/pypy/changeset/8942f2c46162/

Log:	Fix utf-8 encoding; all test_runicode passes.

diff --git a/pypy/rlib/runicode.py b/pypy/rlib/runicode.py
--- a/pypy/rlib/runicode.py
+++ b/pypy/rlib/runicode.py
@@ -253,6 +253,8 @@
     result.append((chr((0x80 | (ch & 0x3f)))))
 
 def unicode_encode_utf_8(s, size, errors, errorhandler=None):
+    if errorhandler is None:
+        errorhandler = raise_unicode_exception_encode
     assert(size >= 0)
     result = StringBuilder(size)
     pos = 0
@@ -279,11 +281,14 @@
                         pos += 1
                         _encodeUCS4(result, ch3)
                         continue
-                r, pos = errorhandler(errors, 'utf-8',
-                                      'surrogates not allowed',
-                                      s, pos-1, pos)
-                result.append(r)
-                continue
+                    r, pos = errorhandler(errors, 'utf-8',
+                                          'surrogates not allowed',
+                                          s, pos-1, pos)
+                    result.append(r)
+                    continue
+                result.append((chr((0xe0 | (ch >> 12)))))
+                result.append((chr((0x80 | ((ch >> 6) & 0x3f)))))
+                result.append((chr((0x80 | (ch & 0x3f)))))
             else:
                 _encodeUCS4(result, ch)
     return result.build()
diff --git a/pypy/rlib/test/test_runicode.py b/pypy/rlib/test/test_runicode.py
--- a/pypy/rlib/test/test_runicode.py
+++ b/pypy/rlib/test/test_runicode.py
@@ -118,6 +118,9 @@
         for i in range(10000):
             for encoding in ("utf-7 utf-8 utf-16 utf-16-be utf-16-le "
                              "utf-32 utf-32-be utf-32-le").split():
+                if encoding == 'utf-8' and 0xd800 <= i <= 0xdfff:
+                    # Don't try to encode lone surrogates
+                    continue
                 self.checkdecode(unichr(i), encoding)
 
     def test_random(self):

From noreply at buildbot.pypy.org  Tue Oct 18 00:37:52 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 18 Oct 2011 00:37:52 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Implement rdict.pop(). Always with default.
Message-ID: <20111017223752.2FBDF820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48162:5364600194fb
Date: 2011-10-17 20:10 +0200
http://bitbucket.org/pypy/pypy/changeset/5364600194fb/

Log:	Implement rdict.pop(). Always with default.

diff --git a/pypy/annotation/unaryop.py b/pypy/annotation/unaryop.py
--- a/pypy/annotation/unaryop.py
+++ b/pypy/annotation/unaryop.py
@@ -407,6 +407,7 @@
         return dct.dictdef.read_value()
 
     method_setdefault = method_get
+    method_pop = method_get
 
     def method_copy(dct):
         return SomeDict(dct.dictdef)
diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py
--- a/pypy/rpython/lltypesystem/rdict.py
+++ b/pypy/rpython/lltypesystem/rdict.py
@@ -275,6 +275,13 @@
         v_res = hop.gendirectcall(ll_setdefault, v_dict, v_key, v_default)
         return self.recast_value(hop.llops, v_res)
 
+    def rtype_method_pop(self, hop):
+        v_dict, v_key, v_default = hop.inputargs(self, self.key_repr,
+                                                 self.value_repr)
+        hop.exception_cannot_occur()
+        v_res = hop.gendirectcall(ll_dict_pop, v_dict, v_key, v_default)
+        return self.recast_value(hop.llops, v_res)
+
     def rtype_method_copy(self, hop):
         v_dict, = hop.inputargs(self)
         hop.exception_cannot_occur()
@@ -492,6 +499,15 @@
         raise KeyError
     _ll_dict_del(d, i)
 
+def ll_dict_pop(d, key, default):
+    i = ll_dict_lookup(d, key, d.keyhash(key))
+    if not i & HIGHEST_BIT:
+        value = ll_get_value(d, i)
+        _ll_dict_del(d, i)
+        return value
+    else:
+        return default
+
 @jit.dont_look_inside
 def _ll_dict_del(d, i):
     d.entries.mark_deleted(i)
diff --git a/pypy/rpython/ootypesystem/rdict.py b/pypy/rpython/ootypesystem/rdict.py
--- a/pypy/rpython/ootypesystem/rdict.py
+++ b/pypy/rpython/ootypesystem/rdict.py
@@ -105,6 +105,13 @@
         v_res = hop.gendirectcall(ll_dict_setdefault, v_dict, v_key, v_default)
         return self.recast_value(hop.llops, v_res)
 
+    def rtype_method_pop(self, hop):
+        v_dict, v_key, v_default = hop.inputargs(self, self.key_repr,
+                                                 self.value_repr)
+        hop.exception_cannot_occur()
+        v_res = hop.gendirectcall(ll_dict_pop, v_dict, v_key, v_default)
+        return self.recast_value(hop.llops, v_res)
+
     def rtype_method_copy(self, hop):
         v_dict, = hop.inputargs(self)
         cDICT = hop.inputconst(ootype.Void, self.lowleveltype)
@@ -334,6 +341,14 @@
         d.ll_set(key, default)
         return default
 
+def ll_dict_pop(d, key, default):
+    if d.ll_contains(key):
+        value = d.ll_get(key)
+        d.ll_remove(key)
+        return value
+    else:
+        return default
+
 def _make_ll_keys_values_items(kind):
     def ll_dict_kvi(LIST, d):
         length = d.ll_length()
diff --git a/pypy/rpython/test/test_rdict.py b/pypy/rpython/test/test_rdict.py
--- a/pypy/rpython/test/test_rdict.py
+++ b/pypy/rpython/test/test_rdict.py
@@ -171,6 +171,16 @@
         res = self.interpret(func, ())
         assert res == 422
 
+    def test_dict_pop(self):
+        def func():
+            dic = {}
+            x1 = dic.pop('hi', 42)
+            dic['blah'] = 1
+            x2 = dic.pop('blah', 2)
+            return len(dic) * 100 + x1 * 10 + x2
+        res = self.interpret(func, ())
+        assert res == 421
+
     def test_dict_setdefault(self):
         def f():
             d = {}

From noreply at buildbot.pypy.org  Tue Oct 18 00:37:54 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 18 Oct 2011 00:37:54 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Fix test_bytes.py, test_buffer.py
Message-ID: <20111017223754.99C19820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48164:913552c73f58
Date: 2011-10-17 21:13 +0200
http://bitbucket.org/pypy/pypy/changeset/913552c73f58/

Log:	Fix test_bytes.py, test_buffer.py

diff --git a/pypy/interpreter/buffer.py b/pypy/interpreter/buffer.py
--- a/pypy/interpreter/buffer.py
+++ b/pypy/interpreter/buffer.py
@@ -51,9 +51,9 @@
     def descr_getitem(self, space, w_index):
         start, stop, step, size = space.decode_index4(w_index, self.getlength())
         if step == 0:  # index only
-            return space.wrap(self.getitem(start))
+            return space.wrapbytes(self.getitem(start))
         res = self.getslice(start, stop, step, size)
-        return space.wrap(res)
+        return space.wrapbytes(res)
 
     @unwrap_spec(newstring='bufferstr')
     def descr_setitem(self, space, w_index, newstring):
@@ -86,7 +86,7 @@
 
     @unwrap_spec(other='bufferstr')
     def descr_add(self, space, other):
-        return space.wrap(self.as_str() + other)
+        return space.wrapbytes(self.as_str() + other)
 
     def _make_descr__cmp(name):
         def descr__cmp(self, space, w_other):
@@ -112,7 +112,7 @@
 
     def descr_mul(self, space, w_times):
         # xxx not the most efficient implementation
-        w_string = space.wrap(self.as_str())
+        w_string = space.wrapbytes(self.as_str())
         # use the __mul__ method instead of space.mul() so that we
         # return NotImplemented instead of raising a TypeError
         return space.call_method(w_string, '__mul__', w_times)
diff --git a/pypy/interpreter/test/test_buffer.py b/pypy/interpreter/test/test_buffer.py
--- a/pypy/interpreter/test/test_buffer.py
+++ b/pypy/interpreter/test/test_buffer.py
@@ -9,7 +9,7 @@
 
     def test_buffer_w(self):
         space = self.space
-        w_hello = space.wrap('hello world')
+        w_hello = space.wrapbytes('hello world')
         buf = space.buffer_w(w_hello)
         assert isinstance(buf, Buffer)
         assert buf.getlength() == 11
@@ -23,7 +23,7 @@
 
     def test_file_write(self):
         space = self.space
-        w_buffer = space.buffer(space.wrap('hello world'))
+        w_buffer = space.buffer(space.wrapbytes('hello world'))
         filename = str(testdir.join('test_file_write'))
         space.appexec([w_buffer, space.wrap(filename)], """(buffer, filename):
             f = open(filename, 'wb')
@@ -35,13 +35,4 @@
         f.close()
         assert data == 'hello world'
 
-    def test_unicode(self):
-        space = self.space
-        s = space.bufferstr_w(space.wrap(u'hello'))
-        assert type(s) is str
-        assert s == 'hello'
-        space.raises_w(space.w_UnicodeEncodeError,
-                       space.bufferstr_w, space.wrap(u'\xe9'))
-
-
 # Note: some app-level tests for buffer are in module/__builtin__/test/.
diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -127,13 +127,13 @@
 
 def contains__Bytearray_String(space, w_bytearray, w_str):
     # XXX slow - copies, needs rewriting
-    w_str2 = str__Bytearray(space, w_bytearray)
+    w_str2 = _to_bytes(space, w_bytearray)
     return stringobject.contains__String_String(space, w_str2, w_str)
 
 def contains__Bytearray_ANY(space, w_bytearray, w_sub):
     # XXX slow - copies, needs rewriting
-    w_str = space.wrap(space.bufferstr_new_w(w_sub))
-    w_str2 = str__Bytearray(space, w_bytearray)
+    w_str = space.wrapbytes(space.bufferstr_new_w(w_sub))
+    w_str2 = _to_bytes(space, w_bytearray)
     return stringobject.contains__String_String(space, w_str2, w_str)
 
 def add__Bytearray_Bytearray(space, w_bytearray1, w_bytearray2):
@@ -148,7 +148,7 @@
 
 def add__String_Bytearray(space, w_str, w_bytearray):
     data2 = w_bytearray.data
-    data1 = [c for c in space.str_w(w_str)]
+    data1 = [c for c in space.bytes_w(w_str)]
     return W_BytearrayObject(data1 + data2)
 
 def mul_bytearray_times(space, w_bytearray, w_times):
@@ -188,14 +188,14 @@
     return space.w_True
 
 def String2Bytearray(space, w_str):
-    data = [c for c in space.str_w(w_str)]
+    data = [c for c in space.bytes_w(w_str)]
     return W_BytearrayObject(data)
 
 def eq__Bytearray_String(space, w_bytearray, w_other):
-    return space.eq(str__Bytearray(space, w_bytearray), w_other)
+    return space.eq(_to_bytes(space, w_bytearray), w_other)
 
 def ne__Bytearray_String(space, w_bytearray, w_other):
-    return space.ne(str__Bytearray(space, w_bytearray), w_other)
+    return space.ne(_to_bytes(space, w_bytearray), w_other)
 
 def _min(a, b):
     if a < b:
@@ -226,7 +226,7 @@
 
 def str_translate__Bytearray_ANY_ANY(space, w_bytearray1, w_table, w_deletechars):
     # XXX slow, copies *twice* needs proper implementation
-    w_str_copy = str__Bytearray(space, w_bytearray1)
+    w_str_copy = _to_bytes(space, w_bytearray1)
     w_res = stringobject.str_translate__String_ANY_ANY(space, w_str_copy,
                                                        w_table, w_deletechars)
     return String2Bytearray(space, w_res)
@@ -265,8 +265,8 @@
 
     return space.wrap(buf.build())
 
-def str__Bytearray(space, w_bytearray):
-    return space.wrap(''.join(w_bytearray.data))
+def _to_bytes(space, w_bytearray):
+    return space.wrapbytes(''.join(w_bytearray.data))
 
 def _convert_idx_params(space, w_self, w_start, w_stop):
     start = slicetype.eval_slice_index(space, w_start)
@@ -294,42 +294,42 @@
 
 def str_count__Bytearray_ANY_ANY_ANY(space, w_bytearray, w_char, w_start, w_stop):
     w_char = space.wrap(space.bufferstr_new_w(w_char))
-    w_str = str__Bytearray(space, w_bytearray)
+    w_str = _to_bytes(space, w_bytearray)
     return stringobject.str_count__String_String_ANY_ANY(space, w_str, w_char,
                                                          w_start, w_stop)
 
 def str_index__Bytearray_ANY_ANY_ANY(space, w_bytearray, w_char, w_start, w_stop):
     w_char = space.wrap(space.bufferstr_new_w(w_char))
-    w_str = str__Bytearray(space, w_bytearray)
+    w_str = _to_bytes(space, w_bytearray)
     return stringobject.str_index__String_String_ANY_ANY(space, w_str, w_char,
                                                          w_start, w_stop)
 
 def str_rindex__Bytearray_ANY_ANY_ANY(space, w_bytearray, w_char, w_start, w_stop):
     w_char = space.wrap(space.bufferstr_new_w(w_char))
-    w_str = str__Bytearray(space, w_bytearray)
+    w_str = _to_bytes(space, w_bytearray)
     return stringobject.str_rindex__String_String_ANY_ANY(space, w_str, w_char,
                                                          w_start, w_stop)
 
 def str_find__Bytearray_ANY_ANY_ANY(space, w_bytearray, w_char, w_start, w_stop):
     w_char = space.wrap(space.bufferstr_new_w(w_char))
-    w_str = str__Bytearray(space, w_bytearray)
+    w_str = _to_bytes(space, w_bytearray)
     return stringobject.str_find__String_String_ANY_ANY(space, w_str, w_char,
                                                          w_start, w_stop)
 
 def str_rfind__Bytearray_ANY_ANY_ANY(space, w_bytearray, w_char, w_start, w_stop):
     w_char = space.wrap(space.bufferstr_new_w(w_char))
-    w_str = str__Bytearray(space, w_bytearray)
+    w_str = _to_bytes(space, w_bytearray)
     return stringobject.str_rfind__String_String_ANY_ANY(space, w_str, w_char,
                                                          w_start, w_stop)
 
 def str_startswith__Bytearray_ANY_ANY_ANY(space, w_bytearray, w_prefix, w_start, w_stop):
     w_prefix = space.wrap(space.bufferstr_new_w(w_prefix))
-    w_str = str__Bytearray(space, w_bytearray)
+    w_str = _to_bytes(space, w_bytearray)
     return stringobject.str_startswith__String_String_ANY_ANY(space, w_str, w_prefix,
                                                               w_start, w_stop)
 
 def str_startswith__Bytearray_Tuple_ANY_ANY(space, w_bytearray, w_prefix, w_start, w_stop):
-    w_str = str__Bytearray(space, w_bytearray)
+    w_str = _to_bytes(space, w_bytearray)
     w_prefix = space.newtuple([space.wrap(space.bufferstr_new_w(w_entry)) for w_entry in
                                space.unpackiterable(w_prefix)])
     return stringobject.str_startswith__String_Tuple_ANY_ANY(space, w_str, w_prefix,
@@ -337,12 +337,12 @@
 
 def str_endswith__Bytearray_ANY_ANY_ANY(space, w_bytearray, w_suffix, w_start, w_stop):
     w_suffix = space.wrap(space.bufferstr_new_w(w_suffix))
-    w_str = str__Bytearray(space, w_bytearray)
+    w_str = _to_bytes(space, w_bytearray)
     return stringobject.str_endswith__String_String_ANY_ANY(space, w_str, w_suffix,
                                                               w_start, w_stop)
 
 def str_endswith__Bytearray_Tuple_ANY_ANY(space, w_bytearray, w_suffix, w_start, w_stop):
-    w_str = str__Bytearray(space, w_bytearray)
+    w_str = _to_bytes(space, w_bytearray)
     w_suffix = space.newtuple([space.wrap(space.bufferstr_new_w(w_entry)) for w_entry in
                                space.unpackiterable(w_suffix)])
     return stringobject.str_endswith__String_Tuple_ANY_ANY(space, w_str, w_suffix,
@@ -369,35 +369,35 @@
     return W_BytearrayObject(newdata)
 
 def str_decode__Bytearray_ANY_ANY(space, w_bytearray, w_encoding, w_errors):
-    w_str = str__Bytearray(space, w_bytearray)
+    w_str = _to_bytes(space, w_bytearray)
     return stringobject.str_decode__String_ANY_ANY(space, w_str, w_encoding, w_errors)
 
 def str_islower__Bytearray(space, w_bytearray):
-    w_str = str__Bytearray(space, w_bytearray)
+    w_str = _to_bytes(space, w_bytearray)
     return stringobject.str_islower__String(space, w_str)
 
 def str_isupper__Bytearray(space, w_bytearray):
-    w_str = str__Bytearray(space, w_bytearray)
+    w_str = _to_bytes(space, w_bytearray)
     return stringobject.str_isupper__String(space, w_str)
 
 def str_isalpha__Bytearray(space, w_bytearray):
-    w_str = str__Bytearray(space, w_bytearray)
+    w_str = _to_bytes(space, w_bytearray)
     return stringobject.str_isalpha__String(space, w_str)
 
 def str_isalnum__Bytearray(space, w_bytearray):
-    w_str = str__Bytearray(space, w_bytearray)
+    w_str = _to_bytes(space, w_bytearray)
     return stringobject.str_isalnum__String(space, w_str)
 
 def str_isdigit__Bytearray(space, w_bytearray):
-    w_str = str__Bytearray(space, w_bytearray)
+    w_str = _to_bytes(space, w_bytearray)
     return stringobject.str_isdigit__String(space, w_str)
 
 def str_istitle__Bytearray(space, w_bytearray):
-    w_str = str__Bytearray(space, w_bytearray)
+    w_str = _to_bytes(space, w_bytearray)
     return stringobject.str_istitle__String(space, w_str)
 
 def str_isspace__Bytearray(space, w_bytearray):
-    w_str = str__Bytearray(space, w_bytearray)
+    w_str = _to_bytes(space, w_bytearray)
     return stringobject.str_isspace__String(space, w_str)
 
 def bytearray_insert__Bytearray_Int_ANY(space, w_bytearray, w_idx, w_other):
@@ -455,66 +455,67 @@
 # These methods could just delegate to the string implementation,
 # but they have to return a bytearray.
 def str_replace__Bytearray_ANY_ANY_ANY(space, w_bytearray, w_str1, w_str2, w_max):
-    w_str = str__Bytearray(space, w_bytearray)
+    w_str = _to_bytes(space, w_bytearray)
+                                                         w_str2, w_max)
     w_res = stringobject.str_replace__String_ANY_ANY_ANY(space, w_str, w_str1,
                                                          w_str2, w_max)
     return String2Bytearray(space, w_res)
 
 def str_upper__Bytearray(space, w_bytearray):
-    w_str = str__Bytearray(space, w_bytearray)
+    w_str = _to_bytes(space, w_bytearray)
     w_res = stringobject.str_upper__String(space, w_str)
     return String2Bytearray(space, w_res)
 
 def str_lower__Bytearray(space, w_bytearray):
-    w_str = str__Bytearray(space, w_bytearray)
+    w_str = _to_bytes(space, w_bytearray)
     w_res = stringobject.str_lower__String(space, w_str)
     return String2Bytearray(space, w_res)
 
 def str_title__Bytearray(space, w_bytearray):
-    w_str = str__Bytearray(space, w_bytearray)
+    w_str = _to_bytes(space, w_bytearray)
     w_res = stringobject.str_title__String(space, w_str)
     return String2Bytearray(space, w_res)
 
 def str_swapcase__Bytearray(space, w_bytearray):
-    w_str = str__Bytearray(space, w_bytearray)
+    w_str = _to_bytes(space, w_bytearray)
     w_res = stringobject.str_swapcase__String(space, w_str)
     return String2Bytearray(space, w_res)
 
 def str_capitalize__Bytearray(space, w_bytearray):
-    w_str = str__Bytearray(space, w_bytearray)
+    w_str = _to_bytes(space, w_bytearray)
     w_res = stringobject.str_capitalize__String(space, w_str)
     return String2Bytearray(space, w_res)
 
 def str_ljust__Bytearray_ANY_ANY(space, w_bytearray, w_width, w_fillchar):
-    w_str = str__Bytearray(space, w_bytearray)
+    w_str = _to_bytes(space, w_bytearray)
     w_res = stringobject.str_ljust__String_ANY_ANY(space, w_str, w_width,
                                                    w_fillchar)
     return String2Bytearray(space, w_res)
 
 def str_rjust__Bytearray_ANY_ANY(space, w_bytearray, w_width, w_fillchar):
-    w_str = str__Bytearray(space, w_bytearray)
+    w_str = _to_bytes(space, w_bytearray)
     w_res = stringobject.str_rjust__String_ANY_ANY(space, w_str, w_width,
                                                    w_fillchar)
     return String2Bytearray(space, w_res)
 
 def str_center__Bytearray_ANY_ANY(space, w_bytearray, w_width, w_fillchar):
-    w_str = str__Bytearray(space, w_bytearray)
+    w_str = _to_bytes(space, w_bytearray)
     w_res = stringobject.str_center__String_ANY_ANY(space, w_str, w_width,
                                                     w_fillchar)
     return String2Bytearray(space, w_res)
 
 def str_zfill__Bytearray_ANY(space, w_bytearray, w_width):
-    w_str = str__Bytearray(space, w_bytearray)
+    w_str = _to_bytes(space, w_bytearray)
     w_res = stringobject.str_zfill__String_ANY(space, w_str, w_width)
     return String2Bytearray(space, w_res)
 
 def str_expandtabs__Bytearray_ANY(space, w_bytearray, w_tabsize):
-    w_str = str__Bytearray(space, w_bytearray)
+    w_str = _to_bytes(space, w_bytearray)
     w_res = stringobject.str_expandtabs__String_ANY(space, w_str, w_tabsize)
     return String2Bytearray(space, w_res)
 
 def str_splitlines__Bytearray_ANY(space, w_bytearray, w_keepends):
-    w_str = str__Bytearray(space, w_bytearray)
+    w_str = _to_bytes(space, w_bytearray)
     w_result = stringobject.str_splitlines__String_ANY(space, w_str, w_keepends)
     return space.newlist([
         new_bytearray(space, space.w_bytearray, makebytesdata_w(space, w_entry))
@@ -522,9 +523,9 @@
     ])
 
 def str_split__Bytearray_ANY_ANY(space, w_bytearray, w_by, w_maxsplit=-1):
-    w_str = str__Bytearray(space, w_bytearray)
+    w_str = _to_bytes(space, w_bytearray)
     if not space.is_w(w_by, space.w_None):
-        w_by = space.wrap(space.bufferstr_new_w(w_by))
+        w_by = space.wrapbytes(space.bufferstr_new_w(w_by))
     w_list = space.call_method(w_str, "split", w_by, w_maxsplit)
     length = space.int_w(space.len(w_list))
     for i in range(length):
@@ -533,9 +534,9 @@
     return w_list
 
 def str_rsplit__Bytearray_ANY_ANY(space, w_bytearray, w_by, w_maxsplit=-1):
-    w_str = str__Bytearray(space, w_bytearray)
+    w_str = _to_bytes(space, w_bytearray)
     if not space.is_w(w_by, space.w_None):
-        w_by = space.wrap(space.bufferstr_new_w(w_by))
+        w_by = space.wrapbytes(space.bufferstr_new_w(w_by))
     w_list = space.call_method(w_str, "rsplit", w_by, w_maxsplit)
     length = space.int_w(space.len(w_list))
     for i in range(length):
@@ -544,8 +545,8 @@
     return w_list
 
 def str_partition__Bytearray_ANY(space, w_bytearray, w_sub):
-    w_str = str__Bytearray(space, w_bytearray)
-    w_sub = space.wrap(space.bufferstr_new_w(w_sub))
+    w_str = _to_bytes(space, w_bytearray)
+    w_sub = space.wrapbytes(space.bufferstr_new_w(w_sub))
     w_tuple = stringobject.str_partition__String_String(space, w_str, w_sub)
     w_a, w_b, w_c = space.fixedview(w_tuple, 3)
     return space.newtuple([
@@ -554,8 +555,8 @@
         String2Bytearray(space, w_c)])
 
 def str_rpartition__Bytearray_ANY(space, w_bytearray, w_sub):
-    w_str = str__Bytearray(space, w_bytearray)
-    w_sub = space.wrap(space.bufferstr_new_w(w_sub))
+    w_str = _to_bytes(space, w_bytearray)
+    w_sub = space.wrapbytes(space.bufferstr_new_w(w_sub))
     w_tuple = stringobject.str_rpartition__String_String(space, w_str, w_sub)
     w_a, w_b, w_c = space.fixedview(w_tuple, 3)
     return space.newtuple([
@@ -567,7 +568,6 @@
 # Mutability methods
 
 def list_append__Bytearray_ANY(space, w_bytearray, w_item):
-    from pypy.objspace.std.bytearraytype import getbytevalue
     w_bytearray.data.append(getbytevalue(space, w_item))
 
 def list_extend__Bytearray_Bytearray(space, w_bytearray, w_other):
@@ -585,7 +585,6 @@
     return w_bytearray1
 
 def setitem__Bytearray_ANY_ANY(space, w_bytearray, w_index, w_item):
-    from pypy.objspace.std.bytearraytype import getbytevalue
     idx = space.getindex_w(w_index, space.w_IndexError, "bytearray index")
     try:
         w_bytearray.data[idx] = getbytevalue(space, w_item)
diff --git a/pypy/objspace/std/bytearraytype.py b/pypy/objspace/std/bytearraytype.py
--- a/pypy/objspace/std/bytearraytype.py
+++ b/pypy/objspace/std/bytearraytype.py
@@ -87,7 +87,7 @@
     "from a string of hexadecimal numbers.\nSpaces between two numbers are "
     "accepted.\nExample: bytearray.fromhex('B9 01EF') -> "
     "bytearray(b'\\xb9\\x01\\xef')."
-    hexstring = space.str_w(w_hexstring)
+    hexstring = space.unicode_w(w_hexstring)
     hexstring = hexstring.lower()
     data = []
     length = len(hexstring)
diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py
--- a/pypy/objspace/std/stringobject.py
+++ b/pypy/objspace/std/stringobject.py
@@ -153,11 +153,11 @@
 
 def str_upper__String(space, w_self):
     self = w_self._value
-    return space.wrap(self.upper())
+    return space.wrapbytes(self.upper())
 
 def str_lower__String(space, w_self):
     self = w_self._value
-    return space.wrap(self.lower())
+    return space.wrapbytes(self.lower())
 
 def str_swapcase__String(space, w_self):
     self = w_self._value
@@ -173,7 +173,7 @@
         else:
             builder.append(ch)
 
-    return space.wrap(builder.build())
+    return space.wrapbytes(builder.build())
 
 
 def str_capitalize__String(space, w_self):
@@ -195,7 +195,7 @@
             else:
                 builder.append(ch)
 
-    return space.wrap(builder.build())
+    return space.wrapbytes(builder.build())
 
 def str_title__String(space, w_self):
     input = w_self._value
@@ -213,7 +213,7 @@
 
         prev_letter = ch
 
-    return space.wrap(builder.build())
+    return space.wrapbytes(builder.build())
 
 def str_split__String_None_ANY(space, w_self, w_none, w_maxsplit=-1):
     maxsplit = space.int_w(w_maxsplit)
@@ -402,7 +402,7 @@
         fillchar = fillchar[0]    # annotator hint: it's a single character
         u_self = d * fillchar + u_self
 
-    return space.wrap(u_self)
+    return space.wrapbytes(u_self)
 
 
 def str_ljust__String_ANY_ANY(space, w_self, w_arg, w_fillchar):
@@ -418,7 +418,7 @@
         fillchar = fillchar[0]    # annotator hint: it's a single character
         u_self += d * fillchar
 
-    return space.wrap(u_self)
+    return space.wrapbytes(u_self)
 
 def _convert_idx_params(space, w_self, w_sub, w_start, w_end, upper_bound=False):
     self = w_self._value
@@ -503,7 +503,7 @@
 
 def _string_replace(space, input, sub, by, maxsplit):
     if maxsplit == 0:
-        return space.wrap(input)
+        return space.wrapbytes(input)
 
     #print "from replace, input: %s, sub: %s, by: %s" % (input, sub, by)
 
@@ -542,7 +542,7 @@
             space.w_OverflowError,
             space.wrap("replace string is too long"))
 
-    return space.wrap(by.join(substrings_w))
+    return space.wrapbytes(by.join(substrings_w))
 
 
 def str_replace__String_ANY_ANY_ANY(space, w_self, w_sub, w_by, w_maxsplit):
@@ -759,7 +759,7 @@
     num_zeros = width - len(input)
     if num_zeros <= 0:
         # cannot return w_self, in case it is a subclass of str
-        return space.wrap(input)
+        return space.wrapbytes(input)
 
     builder = StringBuilder(width)
     if len(input) > 0 and (input[0] == '+' or input[0] == '-'):
@@ -770,7 +770,7 @@
 
     builder.append_multiple_char('0', num_zeros)
     builder.append_slice(input, start, len(input))
-    return space.wrap(builder.build())
+    return space.wrapbytes(builder.build())
 
 
 def hash__String(space, w_str):
diff --git a/pypy/objspace/std/test/test_bytes.py b/pypy/objspace/std/test/test_bytes.py
--- a/pypy/objspace/std/test/test_bytes.py
+++ b/pypy/objspace/std/test/test_bytes.py
@@ -6,12 +6,11 @@
         assert b.__class__ is bytearray
 
     def test_constructor(self):
-        assert bytearray() == ""
-        assert bytearray('abc') == "abc"
-        assert bytearray(['a', 'b', 'c']) == "abc"
-        assert bytearray([65, 66, 67]) == "ABC"
-        assert bytearray(5) == '\0' * 5
-        raises(ValueError, bytearray, ['a', 'bc'])
+        assert bytearray() == b""
+        assert bytearray(b'abc') == b"abc"
+        assert bytearray([65, 66, 67]) == b"ABC"
+        assert bytearray(5) == b'\0' * 5
+        raises(TypeError, bytearray, ['a', 'bc'])
         raises(ValueError, bytearray, [65, -3])
         raises(TypeError, bytearray, [65.0])
         raises(ValueError, bytearray, -1)
@@ -20,8 +19,8 @@
         class subclass(bytearray):
             def __init__(self, newarg=1, *args, **kwargs):
                 bytearray.__init__(self, *args, **kwargs)
-        x = subclass(4, source="abcd")
-        assert x == "abcd"
+        x = subclass(4, source=b"abcd")
+        assert x == b"abcd"
 
     def test_encoding(self):
         data = u"Hello world\n\u1234\u5678\u9abc\def0\def0"
@@ -33,10 +32,10 @@
     def test_encoding_with_ignore_errors(self):
         data = u"H\u1234"
         b = bytearray(data, "latin1", errors="ignore")
-        assert b == "H"
+        assert b == b"H"
 
     def test_len(self):
-        b = bytearray('test')
+        b = bytearray(b'test')
         assert len(b) == 4
 
     def test_nohash(self):
@@ -44,146 +43,142 @@
 
     def test_repr(self):
         assert repr(bytearray()) == "bytearray(b'')"
-        assert repr(bytearray('test')) == "bytearray(b'test')"
-        assert repr(bytearray("d'oh")) == r"bytearray(b'd\'oh')"
+        assert repr(bytearray(b'test')) == "bytearray(b'test')"
+        assert repr(bytearray(b"d'oh")) == r"bytearray(b'd\'oh')"
 
     def test_str(self):
-        assert str(bytearray()) == ""
-        assert str(bytearray('test')) == "test"
-        assert str(bytearray("d'oh")) == "d'oh"
+        assert str(bytearray()) == "bytearray(b'')"
+        assert str(bytearray(b'test')) == "bytearray(b'test')"
+        assert str(bytearray(b"d'oh")) == r"bytearray(b'd\'oh')"
 
     def test_getitem(self):
-        b = bytearray('test')
+        b = bytearray(b'test')
         assert b[0] == ord('t')
         assert b[2] == ord('s')
         raises(IndexError, b.__getitem__, 4)
-        assert b[1:5] == bytearray('est')
-        assert b[slice(1,5)] == bytearray('est')
+        assert b[1:5] == bytearray(b'est')
+        assert b[slice(1,5)] == bytearray(b'est')
 
     def test_arithmetic(self):
-        b1 = bytearray('hello ')
-        b2 = bytearray('world')
-        assert b1 + b2 == bytearray('hello world')
-        assert b1 * 2 == bytearray('hello hello ')
+        b1 = bytearray(b'hello ')
+        b2 = bytearray(b'world')
+        assert b1 + b2 == bytearray(b'hello world')
+        assert b1 * 2 == bytearray(b'hello hello ')
         assert b1 * 1 is not b1
 
         b3 = b1
         b3 *= 3
-        assert b3 == 'hello hello hello '
+        assert b3 == b'hello hello hello '
         assert type(b3) == bytearray
         assert b3 is b1
 
     def test_contains(self):
-        assert ord('l') in bytearray('hello')
-        assert 'l' in bytearray('hello')
-        assert bytearray('ll') in bytearray('hello')
-        assert memoryview('ll') in bytearray('hello')
+        assert ord('l') in bytearray(b'hello')
+        assert b'l' in bytearray(b'hello')
+        assert bytearray(b'll') in bytearray(b'hello')
+        assert memoryview(b'll') in bytearray(b'hello')
 
-        raises(TypeError, lambda: u'foo' in bytearray('foobar'))
+        raises(TypeError, lambda: u'foo' in bytearray(b'foobar'))
 
     def test_splitlines(self):
-        b = bytearray('1234')
+        b = bytearray(b'1234')
         assert b.splitlines()[0] == b
         assert b.splitlines()[0] is not b
 
-        assert len(bytearray('foo\nbar').splitlines()) == 2
-        for item in bytearray('foo\nbar').splitlines():
+        assert len(bytearray(b'foo\nbar').splitlines()) == 2
+        for item in bytearray(b'foo\nbar').splitlines():
             assert isinstance(item, bytearray)
 
     def test_ord(self):
-        b = bytearray('\0A\x7f\x80\xff')
+        b = bytearray(b'\0A\x7f\x80\xff')
         assert ([ord(b[i:i+1]) for i in range(len(b))] ==
                          [0, 65, 127, 128, 255])
-        raises(TypeError, ord, bytearray('ll'))
+        raises(TypeError, ord, bytearray(b'll'))
         raises(TypeError, ord, bytearray())
 
     def test_translate(self):
-        b = 'hello'
+        b = b'hello'
         ba = bytearray(b)
         rosetta = bytearray(range(0, 256))
         rosetta[ord('o')] = ord('e')
 
-        for table in rosetta, str(rosetta):
-            c = ba.translate(table)
-            assert ba == bytearray('hello')
-            assert c == bytearray('helle')
-
-            c = ba.translate(rosetta, 'l')
-            assert c == bytearray('hee')
-            assert isinstance(c, bytearray)
+        c = ba.translate(rosetta)
+        assert ba == bytearray(b'hello')
+        assert c == bytearray(b'helle')
+        assert isinstance(c, bytearray)
 
     def test_strip(self):
-        b = bytearray('mississippi ')
+        b = bytearray(b'mississippi ')
 
-        assert b.strip() == 'mississippi'
-        assert b.strip(None) == 'mississippi'
+        assert b.strip() == b'mississippi'
+        assert b.strip(None) == b'mississippi'
 
-        b = bytearray('mississippi')
+        b = bytearray(b'mississippi')
 
-        for strip_type in str, memoryview:
-            assert b.strip(strip_type('i')) == 'mississipp'
-            assert b.strip(strip_type('m')) == 'ississippi'
-            assert b.strip(strip_type('pi')) == 'mississ'
-            assert b.strip(strip_type('im')) == 'ssissipp'
-            assert b.strip(strip_type('pim')) == 'ssiss'
-            assert b.strip(strip_type(b)) == ''
+        for strip_type in bytes, memoryview:
+            assert b.strip(strip_type(b'i')) == b'mississipp'
+            assert b.strip(strip_type(b'm')) == b'ississippi'
+            assert b.strip(strip_type(b'pi')) == b'mississ'
+            assert b.strip(strip_type(b'im')) == b'ssissipp'
+            assert b.strip(strip_type(b'pim')) == b'ssiss'
+            assert b.strip(strip_type(b)) == b''
 
     def test_iter(self):
-        assert list(bytearray('hello')) == [104, 101, 108, 108, 111]
+        assert list(bytearray(b'hello')) == [104, 101, 108, 108, 111]
 
     def test_compare(self):
-        assert bytearray('hello') == bytearray('hello')
-        assert bytearray('hello') < bytearray('world')
-        assert bytearray('world') > bytearray('hello')
+        assert bytearray(b'hello') == bytearray(b'hello')
+        assert bytearray(b'hello') < bytearray(b'world')
+        assert bytearray(b'world') > bytearray(b'hello')
 
     def test_compare_str(self):
-        assert bytearray('hello1') == 'hello1'
-        assert not (bytearray('hello1') != 'hello1')
-        assert 'hello2' == bytearray('hello2')
-        assert not ('hello1' != bytearray('hello1'))
+        assert bytearray(b'hello1') == b'hello1'
+        assert not (bytearray(b'hello1') != b'hello1')
+        assert b'hello2' == bytearray(b'hello2')
+        assert not (b'hello1' != bytearray(b'hello1'))
         # unicode is always different
-        assert not (bytearray('hello3') == unicode('world'))
-        assert bytearray('hello3') != unicode('hello3')
-        assert unicode('hello3') != bytearray('world')
-        assert unicode('hello4') != bytearray('hello4')
-        assert not (bytearray('') == u'')
-        assert not (u'' == bytearray(''))
-        assert bytearray('') != u''
-        assert u'' != bytearray('')
+        assert not (bytearray(b'hello3') == 'world')
+        assert bytearray(b'hello3') != 'hello3'
+        assert 'hello3' != bytearray(b'world')
+        assert 'hello4' != bytearray(b'hello4')
+        assert not (bytearray(b'') == u'')
+        assert not (u'' == bytearray(b''))
+        assert bytearray(b'') != u''
+        assert u'' != bytearray(b'')
 
     def test_stringlike_operations(self):
-        assert bytearray('hello').islower()
-        assert bytearray('HELLO').isupper()
-        assert bytearray('hello').isalpha()
-        assert not bytearray('hello2').isalpha()
-        assert bytearray('hello2').isalnum()
-        assert bytearray('1234').isdigit()
-        assert bytearray('   ').isspace()
-        assert bytearray('Abc').istitle()
+        assert bytearray(b'hello').islower()
+        assert bytearray(b'HELLO').isupper()
+        assert bytearray(b'hello').isalpha()
+        assert not bytearray(b'hello2').isalpha()
+        assert bytearray(b'hello2').isalnum()
+        assert bytearray(b'1234').isdigit()
+        assert bytearray(b'   ').isspace()
+        assert bytearray(b'Abc').istitle()
 
-        assert bytearray('hello').count('l') == 2
-        assert bytearray('hello').count(bytearray('l')) == 2
-        assert bytearray('hello').count(memoryview('l')) == 2
-        assert bytearray('hello').count(ord('l')) == 2
+        assert bytearray(b'hello').count(b'l') == 2
+        assert bytearray(b'hello').count(bytearray(b'l')) == 2
+        assert bytearray(b'hello').count(memoryview(b'l')) == 2
+        assert bytearray(b'hello').count(ord('l')) == 2
 
-        assert bytearray('hello').index('e') == 1
-        assert bytearray('hello').rindex('l') == 3
-        assert bytearray('hello').index(bytearray('e')) == 1
-        assert bytearray('hello').find('l') == 2
-        assert bytearray('hello').rfind('l') == 3
+        assert bytearray(b'hello').index(b'e') == 1
+        assert bytearray(b'hello').rindex(b'l') == 3
+        assert bytearray(b'hello').index(bytearray(b'e')) == 1
+        assert bytearray(b'hello').find(b'l') == 2
+        assert bytearray(b'hello').rfind(b'l') == 3
 
         # these checks used to not raise in pypy but they should
-        raises(TypeError, bytearray('hello').index, ord('e'))
-        raises(TypeError, bytearray('hello').rindex, ord('e'))
-        raises(TypeError, bytearray('hello').find, ord('e'))
-        raises(TypeError, bytearray('hello').rfind, ord('e'))
+        raises(TypeError, bytearray(b'hello').index, ord('e'))
+        raises(TypeError, bytearray(b'hello').rindex, ord('e'))
+        raises(TypeError, bytearray(b'hello').find, ord('e'))
+        raises(TypeError, bytearray(b'hello').rfind, ord('e'))
 
-        assert bytearray('hello').startswith('he')
-        assert bytearray('hello').startswith(bytearray('he'))
-        assert bytearray('hello').startswith(('lo', bytearray('he')))
-        assert bytearray('hello').endswith('lo')
-        assert bytearray('hello').endswith(bytearray('lo'))
-        assert bytearray('hello').endswith((bytearray('lo'), 'he'))
+        assert bytearray(b'hello').startswith(b'he')
+        assert bytearray(b'hello').startswith(bytearray(b'he'))
+        assert bytearray(b'hello').startswith((b'lo', bytearray(b'he')))
+        assert bytearray(b'hello').endswith(b'lo')
+        assert bytearray(b'hello').endswith(bytearray(b'lo'))
+        assert bytearray(b'hello').endswith((bytearray(b'lo'), b'he'))
 
     def test_stringlike_conversions(self):
         # methods that should return bytearray (and not str)
@@ -191,27 +186,27 @@
             assert result == expected
             assert type(result) is bytearray
 
-        check(bytearray('abc').replace('b', bytearray('d')), 'adc')
-        check(bytearray('abc').replace('b', 'd'), 'adc')
+        check(bytearray(b'abc').replace(b'b', bytearray(b'd')), b'adc')
+        check(bytearray(b'abc').replace(b'b', b'd'), b'adc')
 
-        check(bytearray('abc').upper(), 'ABC')
-        check(bytearray('ABC').lower(), 'abc')
-        check(bytearray('abc').title(), 'Abc')
-        check(bytearray('AbC').swapcase(), 'aBc')
-        check(bytearray('abC').capitalize(), 'Abc')
+        check(bytearray(b'abc').upper(), b'ABC')
+        check(bytearray(b'ABC').lower(), b'abc')
+        check(bytearray(b'abc').title(), b'Abc')
+        check(bytearray(b'AbC').swapcase(), b'aBc')
+        check(bytearray(b'abC').capitalize(), b'Abc')
 
-        check(bytearray('abc').ljust(5),  'abc  ')
-        check(bytearray('abc').rjust(5),  '  abc')
-        check(bytearray('abc').center(5), ' abc ')
-        check(bytearray('1').zfill(5), '00001')
-        check(bytearray('1\t2').expandtabs(5), '1    2')
+        check(bytearray(b'abc').ljust(5),  b'abc  ')
+        check(bytearray(b'abc').rjust(5),  b'  abc')
+        check(bytearray(b'abc').center(5), b' abc ')
+        check(bytearray(b'1').zfill(5), b'00001')
+        check(bytearray(b'1\t2').expandtabs(5), b'1    2')
 
-        check(bytearray(',').join(['a', bytearray('b')]), 'a,b')
-        check(bytearray('abca').lstrip('a'), 'bca')
-        check(bytearray('cabc').rstrip('c'), 'cab')
-        check(bytearray('abc').lstrip(memoryview('a')), 'bc')
-        check(bytearray('abc').rstrip(memoryview('c')), 'ab')
-        check(bytearray('aba').strip('a'), 'b')
+        check(bytearray(b',').join([b'a', bytearray(b'b')]), b'a,b')
+        check(bytearray(b'abca').lstrip(b'a'), b'bca')
+        check(bytearray(b'cabc').rstrip(b'c'), b'cab')
+        check(bytearray(b'abc').lstrip(memoryview(b'a')), b'bc')
+        check(bytearray(b'abc').rstrip(memoryview(b'c')), b'ab')
+        check(bytearray(b'aba').strip(b'a'), b'b')
 
     def test_split(self):
         # methods that should return a sequence of bytearrays
@@ -219,66 +214,67 @@
             assert result == expected
             assert set(type(x) for x in result) == set([bytearray])
 
-        b = bytearray('mississippi')
-        check(b.split('i'), ['m', 'ss', 'ss', 'pp', ''])
-        check(b.split(memoryview('i')), ['m', 'ss', 'ss', 'pp', ''])
-        check(b.rsplit('i'), ['m', 'ss', 'ss', 'pp', ''])
-        check(b.rsplit(memoryview('i')), ['m', 'ss', 'ss', 'pp', ''])
-        check(b.rsplit('i', 2), ['mississ', 'pp', ''])
+        b = bytearray(b'mississippi')
+        check(b.split(b'i'), [b'm', b'ss', b'ss', b'pp', b''])
+        check(b.split(memoryview(b'i')), [b'm', b'ss', b'ss', b'pp', b''])
+        check(b.rsplit(b'i'), [b'm', b'ss', b'ss', b'pp', b''])
+        check(b.rsplit(memoryview(b'i')), [b'm', b'ss', b'ss', b'pp', b''])
+        check(b.rsplit(b'i', 2), [b'mississ', b'pp', b''])
 
-        check(bytearray('foo bar').split(), ['foo', 'bar'])
-        check(bytearray('foo bar').split(None), ['foo', 'bar'])
+        check(bytearray(b'foo bar').split(), [b'foo', b'bar'])
+        check(bytearray(b'foo bar').split(None), [b'foo', b'bar'])
 
-        check(b.partition('ss'), ('mi', 'ss', 'issippi'))
-        check(b.partition(memoryview('ss')), ('mi', 'ss', 'issippi'))
-        check(b.rpartition('ss'), ('missi', 'ss', 'ippi'))
-        check(b.rpartition(memoryview('ss')), ('missi', 'ss', 'ippi'))
+        check(b.partition(b'ss'), (b'mi', b'ss', b'issippi'))
+        check(b.partition(memoryview(b'ss')), (b'mi', b'ss', b'issippi'))
+        check(b.rpartition(b'ss'), (b'missi', b'ss', b'ippi'))
+        check(b.rpartition(memoryview(b'ss')), (b'missi', b'ss', b'ippi'))
 
     def test_append(self):
-        b = bytearray('abc')
-        b.append('d')
+        b = bytearray(b'abc')
+        b.append(ord('d'))
         b.append(ord('e'))
-        assert b == 'abcde'
+        assert b == b'abcde'
 
     def test_insert(self):
-        b = bytearray('abc')
-        b.insert(0, 'd')
-        assert b == bytearray('dabc')
+        b = bytearray(b'abc')
+        b.insert(0, ord('d'))
+        assert b == bytearray(b'dabc')
 
         b.insert(-1, ord('e'))
-        assert b == bytearray('dabec')
+        assert b == bytearray(b'dabec')
 
-        b.insert(6, 'f')
-        assert b == bytearray('dabecf')
+        b.insert(6, ord('f'))
+        assert b == bytearray(b'dabecf')
 
-        b.insert(1, 'g')
-        assert b == bytearray('dgabecf')
+        b.insert(1, ord('g'))
+        assert b == bytearray(b'dgabecf')
 
-        b.insert(-12, 'h')
-        assert b == bytearray('hdgabecf')
+        b.insert(-12, ord('h'))
+        assert b == bytearray(b'hdgabecf')
 
-        raises(ValueError, b.insert, 1, 'go')
-        raises(TypeError, b.insert, 'g', 'o')
+        raises(TypeError, b.insert, 1, 'g')
+        raises(TypeError, b.insert, 1, b'g')
+        raises(TypeError, b.insert, b'g', b'o')
 
     def test_pop(self):
-        b = bytearray('world')
+        b = bytearray(b'world')
         assert b.pop() == ord('d')
         assert b.pop(0) == ord('w')
         assert b.pop(-2) == ord('r')
         raises(IndexError, b.pop, 10)
         raises(OverflowError, bytearray().pop)
-        assert bytearray('\xff').pop() == 0xff
+        assert bytearray(b'\xff').pop() == 0xff
 
     def test_remove(self):
         class Indexable:
             def __index__(self):
                 return ord('e')
 
-        b = bytearray('hello')
+        b = bytearray(b'hello')
         b.remove(ord('l'))
-        assert b == 'helo'
+        assert b == b'helo'
         b.remove(ord('l'))
-        assert b == 'heo'
+        assert b == b'heo'
         raises(ValueError, b.remove, ord('l'))
         raises(ValueError, b.remove, 400)
         raises(TypeError, b.remove, u'e')
@@ -286,49 +282,49 @@
         # remove first and last
         b.remove(ord('o'))
         b.remove(ord('h'))
-        assert b == 'e'
+        assert b == b'e'
         raises(TypeError, b.remove, u'e')
         b.remove(Indexable())
-        assert b == ''
+        assert b == b''
 
     def test_reverse(self):
-        b = bytearray('hello')
+        b = bytearray(b'hello')
         b.reverse()
-        assert b == bytearray('olleh')
+        assert b == bytearray(b'olleh')
 
     def test_delitem(self):
-        b = bytearray('abc')
+        b = bytearray(b'abc')
         del b[1]
-        assert b == bytearray('ac')
+        assert b == bytearray(b'ac')
         del b[1:1]
-        assert b == bytearray('ac')
+        assert b == bytearray(b'ac')
         del b[:]
         assert b == bytearray()
 
-        b = bytearray('fooble')
+        b = bytearray(b'fooble')
         del b[::2]
-        assert b == bytearray('obe')
+        assert b == bytearray(b'obe')
 
     def test_iadd(self):
-        b = bytearray('abc')
-        b += 'def'
-        assert b == 'abcdef'
+        b = bytearray(b'abc')
+        b += b'def'
+        assert b == b'abcdef'
         assert isinstance(b, bytearray)
         raises(TypeError, b.__iadd__, u"")
 
     def test_add(self):
-        b1 = bytearray("abc")
-        b2 = bytearray("def")
+        b1 = bytearray(b"abc")
+        b2 = bytearray(b"def")
 
         def check(a, b, expected):
             result = a + b
             assert result == expected
             assert isinstance(result, bytearray)
 
-        check(b1, b2, "abcdef")
-        check(b1, "def", "abcdef")
-        check("def", b1, "defabc")
-        check(b1, memoryview("def"), "abcdef")
+        check(b1, b2, b"abcdef")
+        check(b1, b"def", b"abcdef")
+        check(b"def", b1, b"defabc")
+        check(b1, memoryview(b"def"), b"abcdef")
         raises(TypeError, lambda: b1 + u"def")
         raises(TypeError, lambda: u"abc" + b2)
 
@@ -340,9 +336,8 @@
 
         b = bytearray([0x1a, 0x2b, 0x30])
         assert bytearray.fromhex('1a2B30') == b
-        assert bytearray.fromhex(u'1a2B30') == b
-        assert bytearray.fromhex(u'  1A 2B  30   ') == b
-        assert bytearray.fromhex(u'0000') == '\0\0'
+        assert bytearray.fromhex('  1A 2B  30   ') == b
+        assert bytearray.fromhex('0000') == b'\0\0'
 
         raises(ValueError, bytearray.fromhex, u'a')
         raises(ValueError, bytearray.fromhex, u'A')
@@ -350,92 +345,92 @@
         raises(ValueError, bytearray.fromhex, u'1a b cd')
         raises(ValueError, bytearray.fromhex, u'\x00')
         raises(ValueError, bytearray.fromhex, u'12   \x00   34')
-        raises(UnicodeEncodeError, bytearray.fromhex, u'\u1234')
+        raises(ValueError, bytearray.fromhex, u'\u1234')
 
     def test_extend(self):
-        b = bytearray('abc')
-        b.extend(bytearray('def'))
-        b.extend('ghi')
-        assert b == 'abcdefghi'
-        b.extend(buffer('jkl'))
-        assert b == 'abcdefghijkl'
+        b = bytearray(b'abc')
+        b.extend(bytearray(b'def'))
+        b.extend(b'ghi')
+        assert b == b'abcdefghi'
+        b.extend(buffer(b'jkl'))
+        assert b == b'abcdefghijkl'
 
-        b = bytearray('world')
+        b = bytearray(b'world')
         b.extend([ord(c) for c in 'hello'])
-        assert b == bytearray('worldhello')
+        assert b == bytearray(b'worldhello')
 
-        b = bytearray('world')
-        b.extend(list('hello'))
-        assert b == bytearray('worldhello')
+        b = bytearray(b'world')
+        b.extend(list(b'hello'))
+        assert b == bytearray(b'worldhello')
 
-        b = bytearray('world')
-        b.extend(c for c in 'hello')
-        assert b == bytearray('worldhello')
+        b = bytearray(b'world')
+        b.extend(c for c in b'hello')
+        assert b == bytearray(b'worldhello')
 
-        raises(ValueError, b.extend, ['fish'])
+        raises(TypeError, b.extend, [b'fish'])
         raises(ValueError, b.extend, [256])
         raises(TypeError, b.extend, object())
         raises(TypeError, b.extend, [object()])
-        raises(TypeError, b.extend, u"unicode")
+        raises(TypeError, b.extend, "unicode")
 
     def test_setslice(self):
-        b = bytearray('hello')
+        b = bytearray(b'hello')
         b[:] = [ord(c) for c in 'world']
-        assert b == bytearray('world')
+        assert b == bytearray(b'world')
 
-        b = bytearray('hello world')
-        b[::2] = 'bogoff'
-        assert b == bytearray('beolg ooflf')
+        b = bytearray(b'hello world')
+        b[::2] = b'bogoff'
+        assert b == bytearray(b'beolg ooflf')
 
         def set_wrong_size():
-            b[::2] = 'foo'
+            b[::2] = b'foo'
         raises(ValueError, set_wrong_size)
 
     def test_delitem_slice(self):
-        b = bytearray('abcdefghi')
+        b = bytearray(b'abcdefghi')
         del b[5:8]
-        assert b == 'abcdei'
+        assert b == b'abcdei'
         del b[:3]
-        assert b == 'dei'
+        assert b == b'dei'
 
-        b = bytearray('hello world')
+        b = bytearray(b'hello world')
         del b[::2]
-        assert b == bytearray('el ol')
+        assert b == bytearray(b'el ol')
 
     def test_setitem(self):
-        b = bytearray('abcdefghi')
-        b[1] = 'B'
-        assert b == 'aBcdefghi'
+        b = bytearray(b'abcdefghi')
+        b[1] = ord('B')
+        assert b == b'aBcdefghi'
 
     def test_setitem_slice(self):
-        b = bytearray('abcdefghi')
-        b[0:3] = 'ABC'
-        assert b == 'ABCdefghi'
-        b[3:3] = '...'
-        assert b == 'ABC...defghi'
-        b[3:6] = '()'
-        assert b == 'ABC()defghi'
-        b[6:6] = '<<'
-        assert b == 'ABC()d<<efghi'
+        b = bytearray(b'abcdefghi')
+        b[0:3] = b'ABC'
+        assert b == b'ABCdefghi'
+        b[3:3] = b'...'
+        assert b == b'ABC...defghi'
+        b[3:6] = b'()'
+        assert b == b'ABC()defghi'
+        b[6:6] = b'<<'
+        assert b == b'ABC()d<<efghi'
 
     def test_buffer(self):
-        b = bytearray('abcdefghi')
+        b = bytearray(b'abcdefghi')
         buf = buffer(b)
-        assert buf[2] == 'c'
-        buf[3] = 'D'
-        assert b == 'abcDefghi'
-        buf[4:6] = 'EF'
-        assert b == 'abcDEFghi'
+        assert buf[2] == b'c'
+        buf[3] = b'D'
+        assert b == b'abcDefghi'
+        buf[4:6] = b'EF'
+        assert b == b'abcDEFghi'
 
     def test_decode(self):
-        b = bytearray('abcdefghi')
+        b = bytearray(b'abcdefghi')
         u = b.decode('utf-8')
-        assert isinstance(u, unicode)
+        assert isinstance(u, str)
         assert u == u'abcdefghi'
 
     def test_int(self):
-        assert int(bytearray('-1234')) == -1234
+        assert int(bytearray(b'-1234')) == -1234
 
     def test_reduce(self):
-        assert bytearray('caf\xe9').__reduce__() == (
+        assert bytearray(b'caf\xe9').__reduce__() == (
             bytearray, (u'caf\xe9', 'latin-1'), None)

From noreply at buildbot.pypy.org  Tue Oct 18 00:37:55 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 18 Oct 2011 00:37:55 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: oops
Message-ID: <20111017223755.C818B820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48165:d7c06c912cdd
Date: 2011-10-17 21:19 +0200
http://bitbucket.org/pypy/pypy/changeset/d7c06c912cdd/

Log:	oops

diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -456,7 +456,6 @@
 # but they have to return a bytearray.
 def str_replace__Bytearray_ANY_ANY_ANY(space, w_bytearray, w_str1, w_str2, w_max):
     w_str = _to_bytes(space, w_bytearray)
-                                                         w_str2, w_max)
     w_res = stringobject.str_replace__String_ANY_ANY_ANY(space, w_str, w_str1,
                                                          w_str2, w_max)
     return String2Bytearray(space, w_res)

From noreply at buildbot.pypy.org  Tue Oct 18 00:37:57 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 18 Oct 2011 00:37:57 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Remove bytes.__mod__ and bytes.format
Message-ID: <20111017223757.05EA7820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48166:e6e61c35b8f3
Date: 2011-10-17 21:30 +0200
http://bitbucket.org/pypy/pypy/changeset/e6e61c35b8f3/

Log:	Remove bytes.__mod__ and bytes.format

diff --git a/pypy/objspace/std/ropeobject.py b/pypy/objspace/std/ropeobject.py
--- a/pypy/objspace/std/ropeobject.py
+++ b/pypy/objspace/std/ropeobject.py
@@ -15,8 +15,6 @@
 
 from pypy.rlib import rope
 from pypy.objspace.std.stringobject import (
-    mod__String_ANY as mod__Rope_ANY,
-    str_format__String as str_format__Rope,
     _upper, _lower, DEFAULT_NOOP_TABLE)
 
 class W_RopeObject(W_Object):
diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py
--- a/pypy/objspace/std/stringobject.py
+++ b/pypy/objspace/std/stringobject.py
@@ -17,8 +17,6 @@
 from pypy.objspace.std.stringtype import sliced, wrapstr, wrapchar, \
      stringendswith, stringstartswith, joined2
 
-from pypy.objspace.std.formatting import mod_format
-
 class W_StringObject(W_Object):
     from pypy.objspace.std.stringtype import str_typedef as typedef
     _immutable_fields_ = ['_value']
@@ -996,15 +994,6 @@
     encoding, errors = _get_encoding_and_errors(space, w_encoding, w_errors)
     return decode_object(space, w_string, encoding, errors)
 
-# CPython's logic for deciding if  ""%values  is
-# an error (1 value, 0 %-formatters) or not
-# (values is of a mapping type)
-def mod__String_ANY(space, w_format, w_values):
-    return mod_format(space, w_format, w_values, do_unicode=False)
-
-def str_format__String(space, w_string, __args__):
-    return newformat.format_method(space, w_string, __args__, False)
-
 def format__String_ANY(space, w_string, w_format_spec):
     if not space.isinstance_w(w_format_spec, space.w_str):
         w_format_spec = space.str(w_format_spec)
diff --git a/pypy/objspace/std/stringtype.py b/pypy/objspace/std/stringtype.py
--- a/pypy/objspace/std/stringtype.py
+++ b/pypy/objspace/std/stringtype.py
@@ -85,7 +85,6 @@
                       ' maxsplit is given, at most maxsplit splits are\ndone.'
                       ' If sep is not specified or is None, any whitespace'
                       ' string\nis a separator.')
-str_format     = SMM('format', 1, general__args__=True)
 str_isdigit    = SMM('isdigit', 1,
                      doc='S.isdigit() -> bool\n\nReturn True if all characters'
                          ' in S are digits\nand there is at least one'

From noreply at buildbot.pypy.org  Tue Oct 18 00:37:58 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 18 Oct 2011 00:37:58 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Fix test_stringobject.py
Message-ID: <20111017223758.3CF92820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48167:a7d563ccdf7d
Date: 2011-10-17 22:26 +0200
http://bitbucket.org/pypy/pypy/changeset/a7d563ccdf7d/

Log:	Fix test_stringobject.py

diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py
--- a/pypy/objspace/std/stringobject.py
+++ b/pypy/objspace/std/stringobject.py
@@ -367,24 +367,18 @@
     reslen = len(self) * (size - 1)
     for i in range(size):
         w_s = list_w[i]
-        if not space.isinstance_w(w_s, space.w_str):
-            if space.isinstance_w(w_s, space.w_unicode):
-                # we need to rebuild w_list here, because the original
-                # w_list might be an iterable which we already consumed
-                w_list = space.newlist(list_w)
-                w_u = space.call_function(space.w_unicode, w_self)
-                return space.call_method(w_u, "join", w_list)
+        if not space.isinstance_w(w_s, space.w_bytes):
             raise operationerrfmt(
                 space.w_TypeError,
-                "sequence item %d: expected string, %s "
+                "sequence item %d: expected bytes, %s "
                 "found", i, space.type(w_s).getname(space))
-        reslen += len(space.str_w(w_s))
+        reslen += len(space.bytes_w(w_s))
 
     sb = StringBuilder(reslen)
     for i in range(size):
         if self and i != 0:
             sb.append(self)
-        sb.append(space.str_w(list_w[i]))
+        sb.append(space.bytes_w(list_w[i]))
     return space.wrapbytes(sb.build())
 
 def str_rjust__String_ANY_ANY(space, w_self, w_arg, w_fillchar):
@@ -458,7 +452,7 @@
                              space.wrap("empty separator"))
     pos = self.find(sub)
     if pos == -1:
-        return space.newtuple([w_self, space.wrap(''), space.wrap('')])
+        return space.newtuple([w_self, space.wrapbytes(''), space.wrapbytes('')])
     else:
         return space.newtuple([sliced(space, self, 0, pos, w_self),
                                w_sub,
@@ -473,7 +467,7 @@
                              space.wrap("empty separator"))
     pos = self.rfind(sub)
     if pos == -1:
-        return space.newtuple([space.wrap(''), space.wrap(''), w_self])
+        return space.newtuple([space.wrapbytes(''), space.wrapbytes(''), w_self])
     else:
         return space.newtuple([sliced(space, self, 0, pos, w_self),
                                w_sub,
@@ -653,7 +647,7 @@
             w_u = space.call_function(space.w_unicode, w_self)
             return space.call_method(w_u, "endswith", w_suffixes, w_start,
                                      w_end)
-        suffix = space.str_w(w_suffix)
+        suffix = space.bytes_w(w_suffix)
         if stringendswith(u_self, suffix, start, end):
             return space.w_True
     return space.w_False
@@ -672,7 +666,7 @@
             w_u = space.call_function(space.w_unicode, w_self)
             return space.call_method(w_u, "startswith", w_prefixes, w_start,
                                      w_end)
-        prefix = space.str_w(w_prefix)
+        prefix = space.bytes_w(w_prefix)
         if stringstartswith(u_self, prefix, start, end):
             return space.w_True
     return space.w_False
diff --git a/pypy/objspace/std/test/test_stringobject.py b/pypy/objspace/std/test/test_stringobject.py
--- a/pypy/objspace/std/test/test_stringobject.py
+++ b/pypy/objspace/std/test/test_stringobject.py
@@ -8,33 +8,33 @@
     def teardown_method(self, method):
         pass
 
-    def test_str_w(self):
-        assert self.space.str_w(self.space.wrap("foo")) == "foo"
+    def test_bytes_w(self):
+        assert self.space.bytes_w(self.space.wrapbytes("foo")) == "foo"
 
     def test_equality(self):
-        w = self.space.wrap
+        w = self.space.wrapbytes
         assert self.space.eq_w(w('abc'), w('abc'))
         assert not self.space.eq_w(w('abc'), w('def'))
 
     def test_order_cmp(self):
         space = self.space
-        w = space.wrap
+        w = space.wrapbytes
         assert self.space.is_true(space.lt(w('a'), w('b')))
         assert self.space.is_true(space.lt(w('a'), w('ab')))
         assert self.space.is_true(space.le(w('a'), w('a')))
         assert self.space.is_true(space.gt(w('a'), w('')))
 
     def test_truth(self):
-        w = self.space.wrap
+        w = self.space.wrapbytes
         assert self.space.is_true(w('non-empty'))
         assert not self.space.is_true(w(''))
 
     def test_getitem(self):
         space = self.space
         w = space.wrap
-        w_str = w('abc')
-        assert self.space.eq_w(space.getitem(w_str, w(0)), w('a'))
-        assert self.space.eq_w(space.getitem(w_str, w(-1)), w('c'))
+        w_str = space.wrapbytes('abc')
+        assert space.eq_w(space.getitem(w_str, w(0)), w(ord('a')))
+        assert space.eq_w(space.getitem(w_str, w(-1)), w(ord('c')))
         self.space.raises_w(space.w_IndexError,
                             space.getitem,
                             w_str,
@@ -43,25 +43,26 @@
     def test_slice(self):
         space = self.space
         w = space.wrap
-        w_str = w('abc')
+        wb = space.wrapbytes
+        w_str = wb('abc')
 
         w_slice = space.newslice(w(0), w(0), space.w_None)
-        assert self.space.eq_w(space.getitem(w_str, w_slice), w(''))
+        assert self.space.eq_w(space.getitem(w_str, w_slice), wb(''))
 
         w_slice = space.newslice(w(0), w(1), space.w_None)
-        assert self.space.eq_w(space.getitem(w_str, w_slice), w('a'))
+        assert self.space.eq_w(space.getitem(w_str, w_slice), wb('a'))
 
         w_slice = space.newslice(w(0), w(10), space.w_None)
-        assert self.space.eq_w(space.getitem(w_str, w_slice), w('abc'))
+        assert self.space.eq_w(space.getitem(w_str, w_slice), wb('abc'))
 
         w_slice = space.newslice(space.w_None, space.w_None, space.w_None)
-        assert self.space.eq_w(space.getitem(w_str, w_slice), w('abc'))
+        assert self.space.eq_w(space.getitem(w_str, w_slice), wb('abc'))
 
         w_slice = space.newslice(space.w_None, w(-1), space.w_None)
-        assert self.space.eq_w(space.getitem(w_str, w_slice), w('ab'))
+        assert self.space.eq_w(space.getitem(w_str, w_slice), wb('ab'))
 
         w_slice = space.newslice(w(-1), space.w_None, space.w_None)
-        assert self.space.eq_w(space.getitem(w_str, w_slice), w('c'))
+        assert self.space.eq_w(space.getitem(w_str, w_slice), wb('c'))
 
     def test_extended_slice(self):
         space = self.space
@@ -71,192 +72,182 @@
                 return
         w_None = space.w_None
         w = space.wrap
-        w_str = w('hello')
+        wb = space.wrapbytes
+        w_str = wb('hello')
 
         w_slice = space.newslice(w_None, w_None, w(1))
-        assert self.space.eq_w(space.getitem(w_str, w_slice), w('hello'))
+        assert self.space.eq_w(space.getitem(w_str, w_slice), wb('hello'))
 
         w_slice = space.newslice(w_None, w_None, w(-1))
-        assert self.space.eq_w(space.getitem(w_str, w_slice), w('olleh'))
+        assert self.space.eq_w(space.getitem(w_str, w_slice), wb('olleh'))
 
         w_slice = space.newslice(w_None, w_None, w(2))
-        assert self.space.eq_w(space.getitem(w_str, w_slice), w('hlo'))
+        assert self.space.eq_w(space.getitem(w_str, w_slice), wb('hlo'))
 
         w_slice = space.newslice(w(1), w_None, w(2))
-        assert self.space.eq_w(space.getitem(w_str, w_slice), w('el'))
+        assert self.space.eq_w(space.getitem(w_str, w_slice), wb('el'))
 
 class AppTestStringObject:
 
-    def test_format_wrongchar(self):
-        raises(ValueError, 'a%Zb'.__mod__, ((23,),))
-
     def test_format(self):
-        raises(TypeError, "foo".__mod__, "bar")
-        raises(TypeError, u"foo".__mod__, "bar")
-        raises(TypeError, "foo".__mod__, u"bar")
-
-        for format, arg, cls in [("a %s b", "foo", str),
-                                 (u"a %s b", "foo", unicode),
-                                 ("a %s b", u"foo", unicode),
-                                 (u"a %s b", u"foo", unicode)]:
-            raises(TypeError, format[:2].__mod__, arg)
-            result = format % arg
-            assert result == "a foo b"
-            assert isinstance(result, cls)
+        import operator
+        raises(TypeError, operator.mod, b"%s", (1,))
 
     def test_split(self):
-        assert "".split() == []
-        assert "".split('x') == ['']
-        assert " ".split() == []
-        assert "a".split() == ['a']
-        assert "a".split("a", 1) == ['', '']
-        assert " ".split(" ", 1) == ['', '']
-        assert "aa".split("a", 2) == ['', '', '']
-        assert " a ".split() == ['a']
-        assert "a b c".split() == ['a','b','c']
-        assert 'this is the split function'.split() == ['this', 'is', 'the', 'split', 'function']
-        assert 'a|b|c|d'.split('|') == ['a', 'b', 'c', 'd']
-        assert 'a|b|c|d'.split('|', 2) == ['a', 'b', 'c|d']
-        assert 'a b c d'.split(None, 1) == ['a', 'b c d']
-        assert 'a b c d'.split(None, 2) == ['a', 'b', 'c d']
-        assert 'a b c d'.split(None, 3) == ['a', 'b', 'c', 'd']
-        assert 'a b c d'.split(None, 4) == ['a', 'b', 'c', 'd']
-        assert 'a b c d'.split(None, 0) == ['a b c d']
-        assert 'a  b  c  d'.split(None, 2) == ['a', 'b', 'c  d']
-        assert 'a b c d '.split() == ['a', 'b', 'c', 'd']
-        assert 'a//b//c//d'.split('//') == ['a', 'b', 'c', 'd']
-        assert 'endcase test'.split('test') == ['endcase ', '']
-        raises(ValueError, 'abc'.split, '')
+        assert b"".split() == []
+        assert b"".split(b'x') == [b'']
+        assert b" ".split() == []
+        assert b"a".split() == [b'a']
+        assert b"a".split(b"a", 1) == [b'', b'']
+        assert b" ".split(b" ", 1) == [b'', b'']
+        assert b"aa".split(b"a", 2) == [b'', b'', b'']
+        assert b" a ".split() == [b'a']
+        assert b"a b c".split() == [b'a',b'b',b'c']
+        assert b'this is the split function'.split() == [
+            b'this', b'is', b'the', b'split', b'function']
+        assert b'a|b|c|d'.split(b'|') == [b'a', b'b', b'c', b'd']
+        assert b'a|b|c|d'.split(b'|', 2) == [b'a', b'b', b'c|d']
+        assert b'a b c d'.split(None, 1) == [b'a', b'b c d']
+        assert b'a b c d'.split(None, 2) == [b'a', b'b', b'c d']
+        assert b'a b c d'.split(None, 3) == [b'a', b'b', b'c', b'd']
+        assert b'a b c d'.split(None, 4) == [b'a', b'b', b'c', b'd']
+        assert b'a b c d'.split(None, 0) == [b'a b c d']
+        assert b'a  b  c  d'.split(None, 2) == [b'a', b'b', b'c  d']
+        assert b'a b c d '.split() == [b'a', b'b', b'c', b'd']
+        assert b'a//b//c//d'.split(b'//') == [b'a', b'b', b'c', b'd']
+        assert b'endcase test'.split(b'test') == [b'endcase ', b'']
+        raises(ValueError, b'abc'.split, b'')
 
     def test_rsplit(self):
-        assert "".rsplit() == []
-        assert " ".rsplit() == []
-        assert "a".rsplit() == ['a']
-        assert "a".rsplit("a", 1) == ['', '']
-        assert " ".rsplit(" ", 1) == ['', '']
-        assert "aa".rsplit("a", 2) == ['', '', '']
-        assert " a ".rsplit() == ['a']
-        assert "a b c".rsplit() == ['a','b','c']
-        assert 'this is the rsplit function'.rsplit() == ['this', 'is', 'the', 'rsplit', 'function']
-        assert 'a|b|c|d'.rsplit('|') == ['a', 'b', 'c', 'd']
-        assert 'a|b|c|d'.rsplit('|', 2) == ['a|b', 'c', 'd']
-        assert 'a b c d'.rsplit(None, 1) == ['a b c', 'd']
-        assert 'a b c d'.rsplit(None, 2) == ['a b', 'c', 'd']
-        assert 'a b c d'.rsplit(None, 3) == ['a', 'b', 'c', 'd']
-        assert 'a b c d'.rsplit(None, 4) == ['a', 'b', 'c', 'd']
-        assert 'a b c d'.rsplit(None, 0) == ['a b c d']
-        assert 'a  b  c  d'.rsplit(None, 2) == ['a  b', 'c', 'd']
-        assert 'a b c d '.rsplit() == ['a', 'b', 'c', 'd']
-        assert 'a//b//c//d'.rsplit('//') == ['a', 'b', 'c', 'd']
-        assert 'endcase test'.rsplit('test') == ['endcase ', '']
-        raises(ValueError, 'abc'.rsplit, '')
+        assert b"".rsplit() == []
+        assert b" ".rsplit() == []
+        assert b"a".rsplit() == [b'a']
+        assert b"a".rsplit(b"a", 1) == [b'', b'']
+        assert b" ".rsplit(b" ", 1) == [b'', b'']
+        assert b"aa".rsplit(b"a", 2) == [b'', b'', b'']
+        assert b" a ".rsplit() == [b'a']
+        assert b"a b c".rsplit() == [b'a',b'b',b'c']
+        assert b'this is the rsplit function'.rsplit() == [
+            b'this', b'is', b'the', b'rsplit', b'function']
+        assert b'a|b|c|d'.rsplit(b'|') == [b'a', b'b', b'c', b'd']
+        assert b'a|b|c|d'.rsplit(b'|', 2) == [b'a|b', b'c', b'd']
+        assert b'a b c d'.rsplit(None, 1) == [b'a b c', b'd']
+        assert b'a b c d'.rsplit(None, 2) == [b'a b', b'c', b'd']
+        assert b'a b c d'.rsplit(None, 3) == [b'a', b'b', b'c', b'd']
+        assert b'a b c d'.rsplit(None, 4) == [b'a', b'b', b'c', b'd']
+        assert b'a b c d'.rsplit(None, 0) == [b'a b c d']
+        assert b'a  b  c  d'.rsplit(None, 2) == [b'a  b', b'c', b'd']
+        assert b'a b c d '.rsplit() == [b'a', b'b', b'c', b'd']
+        assert b'a//b//c//d'.rsplit(b'//') == [b'a', b'b', b'c', b'd']
+        assert b'endcase test'.rsplit(b'test') == [b'endcase ', b'']
+        raises(ValueError, b'abc'.rsplit, b'')
 
     def test_split_splitchar(self):
-        assert "/a/b/c".split('/') == ['','a','b','c']
+        assert b"/a/b/c".split(b'/') == [b'',b'a',b'b',b'c']
 
     def test_title(self):
-        assert "brown fox".title() == "Brown Fox"
-        assert "!brown fox".title() == "!Brown Fox"
-        assert "bROWN fOX".title() == "Brown Fox"
-        assert "Brown Fox".title() == "Brown Fox"
-        assert "bro!wn fox".title() == "Bro!Wn Fox"
+        assert b"brown fox".title() == b"Brown Fox"
+        assert b"!brown fox".title() == b"!Brown Fox"
+        assert b"bROWN fOX".title() == b"Brown Fox"
+        assert b"Brown Fox".title() == b"Brown Fox"
+        assert b"bro!wn fox".title() == b"Bro!Wn Fox"
 
     def test_istitle(self):
-        assert "".istitle() == False
-        assert "!".istitle() == False
-        assert "!!".istitle() == False
-        assert "brown fox".istitle() == False
-        assert "!brown fox".istitle() == False
-        assert "bROWN fOX".istitle() == False
-        assert "Brown Fox".istitle() == True
-        assert "bro!wn fox".istitle() == False
-        assert "Bro!wn fox".istitle() == False
-        assert "!brown Fox".istitle() == False
-        assert "!Brown Fox".istitle() == True
-        assert "Brow&&&&N Fox".istitle() == True
-        assert "!Brow&&&&n Fox".istitle() == False
+        assert b"".istitle() == False
+        assert b"!".istitle() == False
+        assert b"!!".istitle() == False
+        assert b"brown fox".istitle() == False
+        assert b"!brown fox".istitle() == False
+        assert b"bROWN fOX".istitle() == False
+        assert b"Brown Fox".istitle() == True
+        assert b"bro!wn fox".istitle() == False
+        assert b"Bro!wn fox".istitle() == False
+        assert b"!brown Fox".istitle() == False
+        assert b"!Brown Fox".istitle() == True
+        assert b"Brow&&&&N Fox".istitle() == True
+        assert b"!Brow&&&&n Fox".istitle() == False
 
     def test_capitalize(self):
-        assert "brown fox".capitalize() == "Brown fox"
-        assert ' hello '.capitalize() == ' hello '
-        assert 'Hello '.capitalize() == 'Hello '
-        assert 'hello '.capitalize() == 'Hello '
-        assert 'aaaa'.capitalize() == 'Aaaa'
-        assert 'AaAa'.capitalize() == 'Aaaa'
+        assert b"brown fox".capitalize() == b"Brown fox"
+        assert b' hello '.capitalize() == b' hello '
+        assert b'Hello '.capitalize() == b'Hello '
+        assert b'hello '.capitalize() == b'Hello '
+        assert b'aaaa'.capitalize() == b'Aaaa'
+        assert b'AaAa'.capitalize() == b'Aaaa'
 
     def test_rjust(self):
-        s = "abc"
+        s = b"abc"
         assert s.rjust(2) == s
         assert s.rjust(3) == s
-        assert s.rjust(4) == " " + s
-        assert s.rjust(5) == "  " + s
-        assert 'abc'.rjust(10) == '       abc'
-        assert 'abc'.rjust(6) == '   abc'
-        assert 'abc'.rjust(3) == 'abc'
-        assert 'abc'.rjust(2) == 'abc'
-        assert 'abc'.rjust(5, '*') == '**abc'     # Python 2.4
-        raises(TypeError, 'abc'.rjust, 5, 'xx')
+        assert s.rjust(4) == b" " + s
+        assert s.rjust(5) == b"  " + s
+        assert b'abc'.rjust(10) == b'       abc'
+        assert b'abc'.rjust(6) == b'   abc'
+        assert b'abc'.rjust(3) == b'abc'
+        assert b'abc'.rjust(2) == b'abc'
+        assert b'abc'.rjust(5, '*') == b'**abc'     # Python 2.4
+        raises(TypeError, b'abc'.rjust, 5, 'xx')
 
     def test_ljust(self):
-        s = "abc"
+        s = b"abc"
         assert s.ljust(2) == s
         assert s.ljust(3) == s
-        assert s.ljust(4) == s + " "
-        assert s.ljust(5) == s + "  "
-        assert 'abc'.ljust(10) == 'abc       '
-        assert 'abc'.ljust(6) == 'abc   '
-        assert 'abc'.ljust(3) == 'abc'
-        assert 'abc'.ljust(2) == 'abc'
-        assert 'abc'.ljust(5, '*') == 'abc**'     # Python 2.4
-        raises(TypeError, 'abc'.ljust, 6, '')
+        assert s.ljust(4) == s + b" "
+        assert s.ljust(5) == s + b"  "
+        assert b'abc'.ljust(10) == b'abc       '
+        assert b'abc'.ljust(6) == b'abc   '
+        assert b'abc'.ljust(3) == b'abc'
+        assert b'abc'.ljust(2) == b'abc'
+        assert b'abc'.ljust(5, '*') == b'abc**'     # Python 2.4
+        raises(TypeError, b'abc'.ljust, 6, '')
 
     def test_replace(self):
-        assert 'one!two!three!'.replace('!', '@', 1) == 'one at two!three!'
-        assert 'one!two!three!'.replace('!', '') == 'onetwothree'
-        assert 'one!two!three!'.replace('!', '@', 2) == 'one at two@three!'
-        assert 'one!two!three!'.replace('!', '@', 3) == 'one at two@three@'
-        assert 'one!two!three!'.replace('!', '@', 4) == 'one at two@three@'
-        assert 'one!two!three!'.replace('!', '@', 0) == 'one!two!three!'
-        assert 'one!two!three!'.replace('!', '@') == 'one at two@three@'
-        assert 'one!two!three!'.replace('x', '@') == 'one!two!three!'
-        assert 'one!two!three!'.replace('x', '@', 2) == 'one!two!three!'
-        assert 'abc'.replace('', '-') == '-a-b-c-'
-        assert 'abc'.replace('', '-', 3) == '-a-b-c'
-        assert 'abc'.replace('', '-', 0) == 'abc'
-        assert ''.replace('', '') == ''
-        assert ''.replace('', 'a') == 'a'
-        assert 'abc'.replace('ab', '--', 0) == 'abc'
-        assert 'abc'.replace('xy', '--') == 'abc'
-        assert '123'.replace('123', '') == ''
-        assert '123123'.replace('123', '') == ''
-        assert '123x123'.replace('123', '') == 'x'
+        assert b'one!two!three!'.replace(b'!', b'@', 1) == b'one at two!three!'
+        assert b'one!two!three!'.replace(b'!', b'') == b'onetwothree'
+        assert b'one!two!three!'.replace(b'!', b'@', 2) == b'one at two@three!'
+        assert b'one!two!three!'.replace(b'!', b'@', 3) == b'one at two@three@'
+        assert b'one!two!three!'.replace(b'!', b'@', 4) == b'one at two@three@'
+        assert b'one!two!three!'.replace(b'!', b'@', 0) == b'one!two!three!'
+        assert b'one!two!three!'.replace(b'!', b'@') == b'one at two@three@'
+        assert b'one!two!three!'.replace(b'x', b'@') == b'one!two!three!'
+        assert b'one!two!three!'.replace(b'x', b'@', 2) == b'one!two!three!'
+        assert b'abc'.replace(b'', b'-') == b'-a-b-c-'
+        assert b'abc'.replace(b'', b'-', 3) == b'-a-b-c'
+        assert b'abc'.replace(b'', b'-', 0) == b'abc'
+        assert b''.replace(b'', b'') == b''
+        assert b''.replace(b'', b'a') == b'a'
+        assert b'abc'.replace(b'ab', b'--', 0) == b'abc'
+        assert b'abc'.replace(b'xy', b'--') == b'abc'
+        assert b'123'.replace(b'123', b'') == b''
+        assert b'123123'.replace(b'123', b'') == b''
+        assert b'123x123'.replace(b'123', b'') == b'x'
 
     def test_replace_buffer(self):
-        assert 'one'.replace(buffer('o'), buffer('n'), 1) == 'nne'
-        assert 'one'.replace(buffer('o'), buffer('n')) == 'nne'
+        assert b'one'.replace(buffer(b'o'), buffer(b'n'), 1) == b'nne'
+        assert b'one'.replace(buffer(b'o'), buffer(b'n')) == b'nne'
 
     def test_strip(self):
         s = " a b "
         assert s.strip() == "a b"
         assert s.rstrip() == " a b"
         assert s.lstrip() == "a b "
-        assert 'xyzzyhelloxyzzy'.strip('xyz') == 'hello'
-        assert 'xyzzyhelloxyzzy'.lstrip('xyz') == 'helloxyzzy'
-        assert 'xyzzyhelloxyzzy'.rstrip('xyz') == 'xyzzyhello'
+        assert b'xyzzyhelloxyzzy'.strip(b'xyz') == b'hello'
+        assert b'xyzzyhelloxyzzy'.lstrip(b'xyz') == b'helloxyzzy'
+        assert b'xyzzyhelloxyzzy'.rstrip(b'xyz') == b'xyzzyhello'
 
     def test_zfill(self):
-        assert '123'.zfill(2) == '123'
-        assert '123'.zfill(3) == '123'
-        assert '123'.zfill(4) == '0123'
-        assert '+123'.zfill(3) == '+123'
-        assert '+123'.zfill(4) == '+123'
-        assert '+123'.zfill(5) == '+0123'
-        assert '-123'.zfill(3) == '-123'
-        assert '-123'.zfill(4) == '-123'
-        assert '-123'.zfill(5) == '-0123'
-        assert ''.zfill(3) == '000'
-        assert '34'.zfill(1) == '34'
-        assert '34'.zfill(4) == '0034'
+        assert b'123'.zfill(2) == b'123'
+        assert b'123'.zfill(3) == b'123'
+        assert b'123'.zfill(4) == b'0123'
+        assert b'+123'.zfill(3) == b'+123'
+        assert b'+123'.zfill(4) == b'+123'
+        assert b'+123'.zfill(5) == b'+0123'
+        assert b'-123'.zfill(3) == b'-123'
+        assert b'-123'.zfill(4) == b'-123'
+        assert b'-123'.zfill(5) == b'-0123'
+        assert b''.zfill(3) == b'000'
+        assert b'34'.zfill(1) == b'34'
+        assert b'34'.zfill(4) == b'0034'
 
     def test_center(self):
         s="a b"
@@ -270,235 +261,235 @@
         assert s.center(7) == "  a b  "
         assert s.center(8) == "  a b   "
         assert s.center(9) == "   a b   "
-        assert 'abc'.center(10) == '   abc    '
-        assert 'abc'.center(6) == ' abc  '
-        assert 'abc'.center(3) == 'abc'
-        assert 'abc'.center(2) == 'abc'
-        assert 'abc'.center(5, '*') == '*abc*'     # Python 2.4
-        raises(TypeError, 'abc'.center, 4, 'cba')
-        assert ' abc'.center(7) == '   abc '
+        assert b'abc'.center(10) == b'   abc    '
+        assert b'abc'.center(6) == b' abc  '
+        assert b'abc'.center(3) == b'abc'
+        assert b'abc'.center(2) == b'abc'
+        assert b'abc'.center(5, '*') == b'*abc*'     # Python 2.4
+        raises(TypeError, b'abc'.center, 4, 'cba')
+        assert b' abc'.center(7) == b'   abc '
 
     def test_count(self):
-        assert "".count("x") ==0
-        assert "".count("") ==1
-        assert "Python".count("") ==7
-        assert "ab aaba".count("ab") ==2
-        assert 'aaa'.count('a') == 3
-        assert 'aaa'.count('b') == 0
-        assert 'aaa'.count('a', -1) == 1
-        assert 'aaa'.count('a', -10) == 3
-        assert 'aaa'.count('a', 0, -1) == 2
-        assert 'aaa'.count('a', 0, -10) == 0
-        assert 'ababa'.count('aba') == 1
+        assert b"".count(b"x") ==0
+        assert b"".count(b"") ==1
+        assert b"Python".count(b"") ==7
+        assert b"ab aaba".count(b"ab") ==2
+        assert b'aaa'.count(b'a') == 3
+        assert b'aaa'.count(b'b') == 0
+        assert b'aaa'.count(b'a', -1) == 1
+        assert b'aaa'.count(b'a', -10) == 3
+        assert b'aaa'.count(b'a', 0, -1) == 2
+        assert b'aaa'.count(b'a', 0, -10) == 0
+        assert b'ababa'.count(b'aba') == 1
 
     def test_startswith(self):
-        assert 'ab'.startswith('ab') is True
-        assert 'ab'.startswith('a') is True
-        assert 'ab'.startswith('') is True
-        assert 'x'.startswith('a') is False
-        assert 'x'.startswith('x') is True
-        assert ''.startswith('') is True
-        assert ''.startswith('a') is False
-        assert 'x'.startswith('xx') is False
-        assert 'y'.startswith('xx') is False
+        assert b'ab'.startswith(b'ab') is True
+        assert b'ab'.startswith(b'a') is True
+        assert b'ab'.startswith(b'') is True
+        assert b'x'.startswith(b'a') is False
+        assert b'x'.startswith(b'x') is True
+        assert b''.startswith(b'') is True
+        assert b''.startswith(b'a') is False
+        assert b'x'.startswith(b'xx') is False
+        assert b'y'.startswith(b'xx') is False
 
     def test_startswith_more(self):
-        assert 'ab'.startswith('a', 0) is True
-        assert 'ab'.startswith('a', 1) is False
-        assert 'ab'.startswith('b', 1) is True
-        assert 'abc'.startswith('bc', 1, 2) is False
-        assert 'abc'.startswith('c', -1, 4) is True
+        assert b'ab'.startswith(b'a', 0) is True
+        assert b'ab'.startswith(b'a', 1) is False
+        assert b'ab'.startswith(b'b', 1) is True
+        assert b'abc'.startswith(b'bc', 1, 2) is False
+        assert b'abc'.startswith(b'c', -1, 4) is True
 
     def test_startswith_tuples(self):
-        assert 'hello'.startswith(('he', 'ha'))
-        assert not 'hello'.startswith(('lo', 'llo'))
-        assert 'hello'.startswith(('hellox', 'hello'))
-        assert not 'hello'.startswith(())
-        assert 'helloworld'.startswith(('hellowo', 'rld', 'lowo'), 3)
-        assert not 'helloworld'.startswith(('hellowo', 'ello', 'rld'), 3)
-        assert 'hello'.startswith(('lo', 'he'), 0, -1)
-        assert not 'hello'.startswith(('he', 'hel'), 0, 1)
-        assert 'hello'.startswith(('he', 'hel'), 0, 2)
-        raises(TypeError, 'hello'.startswith, (42,))
+        assert b'hello'.startswith((b'he', b'ha'))
+        assert not b'hello'.startswith((b'lo', b'llo'))
+        assert b'hello'.startswith((b'hellox', b'hello'))
+        assert not b'hello'.startswith(())
+        assert b'helloworld'.startswith((b'hellowo', b'rld', b'lowo'), 3)
+        assert not b'helloworld'.startswith((b'hellowo', b'ello', b'rld'), 3)
+        assert b'hello'.startswith((b'lo', b'he'), 0, -1)
+        assert not b'hello'.startswith((b'he', b'hel'), 0, 1)
+        assert b'hello'.startswith((b'he', b'hel'), 0, 2)
+        raises(TypeError, b'hello'.startswith, (42,))
 
     def test_endswith(self):
-        assert 'ab'.endswith('ab') is True
-        assert 'ab'.endswith('b') is True
-        assert 'ab'.endswith('') is True
-        assert 'x'.endswith('a') is False
-        assert 'x'.endswith('x') is True
-        assert ''.endswith('') is True
-        assert ''.endswith('a') is False
-        assert 'x'.endswith('xx') is False
-        assert 'y'.endswith('xx') is False
+        assert b'ab'.endswith(b'ab') is True
+        assert b'ab'.endswith(b'b') is True
+        assert b'ab'.endswith(b'') is True
+        assert b'x'.endswith(b'a') is False
+        assert b'x'.endswith(b'x') is True
+        assert b''.endswith(b'') is True
+        assert b''.endswith(b'a') is False
+        assert b'x'.endswith(b'xx') is False
+        assert b'y'.endswith(b'xx') is False
 
     def test_endswith_more(self):
-        assert 'abc'.endswith('ab', 0, 2) is True
-        assert 'abc'.endswith('bc', 1) is True
-        assert 'abc'.endswith('bc', 2) is False
-        assert 'abc'.endswith('b', -3, -1) is True
+        assert b'abc'.endswith(b'ab', 0, 2) is True
+        assert b'abc'.endswith(b'bc', 1) is True
+        assert b'abc'.endswith(b'bc', 2) is False
+        assert b'abc'.endswith(b'b', -3, -1) is True
 
     def test_endswith_tuple(self):
-        assert not 'hello'.endswith(('he', 'ha'))
-        assert 'hello'.endswith(('lo', 'llo'))
-        assert 'hello'.endswith(('hellox', 'hello'))
-        assert not 'hello'.endswith(())
-        assert 'helloworld'.endswith(('hellowo', 'rld', 'lowo'), 3)
-        assert not 'helloworld'.endswith(('hellowo', 'ello', 'rld'), 3, -1)
-        assert 'hello'.endswith(('hell', 'ell'), 0, -1)
-        assert not 'hello'.endswith(('he', 'hel'), 0, 1)
-        assert 'hello'.endswith(('he', 'hell'), 0, 4)
-        raises(TypeError, 'hello'.endswith, (42,))
+        assert not b'hello'.endswith((b'he', b'ha'))
+        assert b'hello'.endswith((b'lo', b'llo'))
+        assert b'hello'.endswith((b'hellox', b'hello'))
+        assert not b'hello'.endswith(())
+        assert b'helloworld'.endswith((b'hellowo', b'rld', b'lowo'), 3)
+        assert not b'helloworld'.endswith((b'hellowo', b'ello', b'rld'), 3, -1)
+        assert b'hello'.endswith((b'hell', b'ell'), 0, -1)
+        assert not b'hello'.endswith((b'he', b'hel'), 0, 1)
+        assert b'hello'.endswith((b'he', b'hell'), 0, 4)
+        raises(TypeError, b'hello'.endswith, (42,))
 
     def test_expandtabs(self):
         import sys
 
-        assert 'abc\rab\tdef\ng\thi'.expandtabs() ==    'abc\rab      def\ng       hi'
-        assert 'abc\rab\tdef\ng\thi'.expandtabs(8) ==   'abc\rab      def\ng       hi'
-        assert 'abc\rab\tdef\ng\thi'.expandtabs(4) ==   'abc\rab  def\ng   hi'
-        assert 'abc\r\nab\tdef\ng\thi'.expandtabs(4) == 'abc\r\nab  def\ng   hi'
-        assert 'abc\rab\tdef\ng\thi'.expandtabs() ==    'abc\rab      def\ng       hi'
-        assert 'abc\rab\tdef\ng\thi'.expandtabs(8) ==   'abc\rab      def\ng       hi'
-        assert 'abc\r\nab\r\ndef\ng\r\nhi'.expandtabs(4) == 'abc\r\nab\r\ndef\ng\r\nhi'
+        assert b'abc\rab\tdef\ng\thi'.expandtabs() ==    b'abc\rab      def\ng       hi'
+        assert b'abc\rab\tdef\ng\thi'.expandtabs(8) ==   b'abc\rab      def\ng       hi'
+        assert b'abc\rab\tdef\ng\thi'.expandtabs(4) ==   b'abc\rab  def\ng   hi'
+        assert b'abc\r\nab\tdef\ng\thi'.expandtabs(4) == b'abc\r\nab  def\ng   hi'
+        assert b'abc\rab\tdef\ng\thi'.expandtabs() ==    b'abc\rab      def\ng       hi'
+        assert b'abc\rab\tdef\ng\thi'.expandtabs(8) ==   b'abc\rab      def\ng       hi'
+        assert b'abc\r\nab\r\ndef\ng\r\nhi'.expandtabs(4) == b'abc\r\nab\r\ndef\ng\r\nhi'
 
-        s = 'xy\t'
-        assert s.expandtabs() == 'xy      '
+        s = b'xy\t'
+        assert s.expandtabs() == b'xy      '
 
-        s = '\txy\t'
-        assert s.expandtabs() == '        xy      '
-        assert s.expandtabs(1) == ' xy '
-        assert s.expandtabs(2) == '  xy  '
-        assert s.expandtabs(3) == '   xy '
+        s = b'\txy\t'
+        assert s.expandtabs() == b'        xy      '
+        assert s.expandtabs(1) == b' xy '
+        assert s.expandtabs(2) == b'  xy  '
+        assert s.expandtabs(3) == b'   xy '
 
-        assert 'xy'.expandtabs() == 'xy'
-        assert ''.expandtabs() == ''
+        assert b'xy'.expandtabs() == b'xy'
+        assert b''.expandtabs() == b''
 
-        raises(OverflowError, "t\tt\t".expandtabs, sys.maxint)
+        raises(OverflowError, b"t\tt\t".expandtabs, sys.maxint)
 
     def test_expandtabs_overflows_gracefully(self):
         import sys
         if sys.maxint > (1 << 32):
             skip("Wrong platform")
-        raises((MemoryError, OverflowError), 't\tt\t'.expandtabs, sys.maxint)
+        raises((MemoryError, OverflowError), b't\tt\t'.expandtabs, sys.maxint)
 
     def test_splitlines(self):
-        s = ""
+        s = b""
         assert s.splitlines() == []
         assert s.splitlines() == s.splitlines(1)
-        s = "a + 4"
-        assert s.splitlines() == ['a + 4']
+        s = b"a + 4"
+        assert s.splitlines() == [b'a + 4']
         # The following is true if no newline in string.
         assert s.splitlines() == s.splitlines(1)
-        s = "a + 4\nb + 2"
-        assert s.splitlines() == ['a + 4', 'b + 2']
-        assert s.splitlines(1) == ['a + 4\n', 'b + 2']
-        s="ab\nab\n \n  x\n\n\n"
-        assert s.splitlines() ==['ab',    'ab',  ' ',   '  x',   '',    '']
+        s = b"a + 4\nb + 2"
+        assert s.splitlines() == [b'a + 4', b'b + 2']
+        assert s.splitlines(1) == [b'a + 4\n', b'b + 2']
+        s = b"ab\nab\n \n  x\n\n\n"
+        assert s.splitlines() ==[b'ab',    b'ab',  b' ',   b'  x',   b'',    b'']
         assert s.splitlines() ==s.splitlines(0)
-        assert s.splitlines(1) ==['ab\n', 'ab\n', ' \n', '  x\n', '\n', '\n']
-        s="\none\n\two\nthree\n\n"
-        assert s.splitlines() ==['', 'one', '\two', 'three', '']
-        assert s.splitlines(1) ==['\n', 'one\n', '\two\n', 'three\n', '\n']
+        assert s.splitlines(1) ==[b'ab\n', b'ab\n', b' \n', b'  x\n', b'\n', b'\n']
+        s = b"\none\n\two\nthree\n\n"
+        assert s.splitlines() ==[b'', b'one', b'\two', b'three', b'']
+        assert s.splitlines(1) ==[b'\n', b'one\n', b'\two\n', b'three\n', b'\n']
         # Split on \r and \r\n too
-        assert '12\r34\r\n56'.splitlines() == ['12', '34', '56']
-        assert '12\r34\r\n56'.splitlines(1) == ['12\r', '34\r\n', '56']
+        assert b'12\r34\r\n56'.splitlines() == [b'12', b'34', b'56']
+        assert b'12\r34\r\n56'.splitlines(1) == [b'12\r', b'34\r\n', b'56']
 
     def test_find(self):
-        assert 'abcdefghiabc'.find('abc') == 0
-        assert 'abcdefghiabc'.find('abc', 1) == 9
-        assert 'abcdefghiabc'.find('def', 4) == -1
-        assert 'abcdef'.find('', 13) == -1
-        assert 'abcdefg'.find('def', 5, None) == -1
+        assert b'abcdefghiabc'.find(b'abc') == 0
+        assert b'abcdefghiabc'.find(b'abc', 1) == 9
+        assert b'abcdefghiabc'.find(b'def', 4) == -1
+        assert b'abcdef'.find(b'', 13) == -1
+        assert b'abcdefg'.find(b'def', 5, None) == -1
 
     def test_index(self):
         from sys import maxint
-        assert 'abcdefghiabc'.index('') == 0
-        assert 'abcdefghiabc'.index('def') == 3
-        assert 'abcdefghiabc'.index('abc') == 0
-        assert 'abcdefghiabc'.index('abc', 1) == 9
-        assert 'abcdefghiabc'.index('def', -4*maxint, 4*maxint) == 3
-        assert 'abcdefgh'.index('def', 2, None) == 3
-        assert 'abcdefgh'.index('def', None, None) == 3
-        raises(ValueError, 'abcdefghiabc'.index, 'hib')
-        raises(ValueError, 'abcdefghiab'.index, 'abc', 1)
-        raises(ValueError, 'abcdefghi'.index, 'ghi', 8)
-        raises(ValueError, 'abcdefghi'.index, 'ghi', -1)
-        raises(TypeError, 'abcdefghijklmn'.index, 'abc', 0, 0.0)
-        raises(TypeError, 'abcdefghijklmn'.index, 'abc', -10.0, 30)
+        assert b'abcdefghiabc'.index(b'') == 0
+        assert b'abcdefghiabc'.index(b'def') == 3
+        assert b'abcdefghiabc'.index(b'abc') == 0
+        assert b'abcdefghiabc'.index(b'abc', 1) == 9
+        assert b'abcdefghiabc'.index(b'def', -4*maxint, 4*maxint) == 3
+        assert b'abcdefgh'.index(b'def', 2, None) == 3
+        assert b'abcdefgh'.index(b'def', None, None) == 3
+        raises(ValueError, b'abcdefghiabc'.index, b'hib')
+        raises(ValueError, b'abcdefghiab'.index, b'abc', 1)
+        raises(ValueError, b'abcdefghi'.index, b'ghi', 8)
+        raises(ValueError, b'abcdefghi'.index, b'ghi', -1)
+        raises(TypeError, b'abcdefghijklmn'.index, b'abc', 0, 0.0)
+        raises(TypeError, b'abcdefghijklmn'.index, b'abc', -10.0, 30)
 
     def test_rfind(self):
-        assert 'abc'.rfind('', 4) == -1
-        assert 'abcdefghiabc'.rfind('abc') == 9
-        assert 'abcdefghiabc'.rfind('') == 12
-        assert 'abcdefghiabc'.rfind('abcd') == 0
-        assert 'abcdefghiabc'.rfind('abcz') == -1
-        assert 'abc'.rfind('', 0) == 3
-        assert 'abc'.rfind('', 3) == 3
-        assert 'abcdefgh'.rfind('def', 2, None) == 3
+        assert b'abc'.rfind(b'', 4) == -1
+        assert b'abcdefghiabc'.rfind(b'abc') == 9
+        assert b'abcdefghiabc'.rfind(b'') == 12
+        assert b'abcdefghiabc'.rfind(b'abcd') == 0
+        assert b'abcdefghiabc'.rfind(b'abcz') == -1
+        assert b'abc'.rfind(b'', 0) == 3
+        assert b'abc'.rfind(b'', 3) == 3
+        assert b'abcdefgh'.rfind(b'def', 2, None) == 3
 
     def test_rindex(self):
         from sys import maxint
-        assert 'abcdefghiabc'.rindex('') == 12
-        assert 'abcdefghiabc'.rindex('def') == 3
-        assert 'abcdefghiabc'.rindex('abc') == 9
-        assert 'abcdefghiabc'.rindex('abc', 0, -1) == 0
-        assert 'abcdefghiabc'.rindex('abc', -4*maxint, 4*maxint) == 9
-        raises(ValueError, 'abcdefghiabc'.rindex, 'hib')
-        raises(ValueError, 'defghiabc'.rindex, 'def', 1)
-        raises(ValueError, 'defghiabc'.rindex, 'abc', 0, -1)
-        raises(ValueError, 'abcdefghi'.rindex, 'ghi', 0, 8)
-        raises(ValueError, 'abcdefghi'.rindex, 'ghi', 0, -1)
-        raises(TypeError, 'abcdefghijklmn'.rindex, 'abc', 0, 0.0)
-        raises(TypeError, 'abcdefghijklmn'.rindex, 'abc', -10.0, 30)
+        assert b'abcdefghiabc'.rindex(b'') == 12
+        assert b'abcdefghiabc'.rindex(b'def') == 3
+        assert b'abcdefghiabc'.rindex(b'abc') == 9
+        assert b'abcdefghiabc'.rindex(b'abc', 0, -1) == 0
+        assert b'abcdefghiabc'.rindex(b'abc', -4*maxint, 4*maxint) == 9
+        raises(ValueError, b'abcdefghiabc'.rindex, b'hib')
+        raises(ValueError, b'defghiabc'.rindex, b'def', 1)
+        raises(ValueError, b'defghiabc'.rindex, b'abc', 0, -1)
+        raises(ValueError, b'abcdefghi'.rindex, b'ghi', 0, 8)
+        raises(ValueError, b'abcdefghi'.rindex, b'ghi', 0, -1)
+        raises(TypeError, b'abcdefghijklmn'.rindex, b'abc', 0, 0.0)
+        raises(TypeError, b'abcdefghijklmn'.rindex, b'abc', -10.0, 30)
 
 
     def test_partition(self):
 
-        assert ('this is the par', 'ti', 'tion method') == \
-            'this is the partition method'.partition('ti')
+        assert (b'this is the par', b'ti', b'tion method') == \
+            b'this is the partition method'.partition(b'ti')
 
         # from raymond's original specification
-        S = 'http://www.python.org'
-        assert ('http', '://', 'www.python.org') == S.partition('://')
-        assert ('http://www.python.org', '', '') == S.partition('?')
-        assert ('', 'http://', 'www.python.org') == S.partition('http://')
-        assert ('http://www.python.', 'org', '') == S.partition('org')
+        S = b'http://www.python.org'
+        assert (b'http', b'://', b'www.python.org') == S.partition(b'://')
+        assert (b'http://www.python.org', b'', b'') == S.partition(b'?')
+        assert (b'', b'http://', b'www.python.org') == S.partition(b'http://')
+        assert (b'http://www.python.', b'org', b'') == S.partition(b'org')
 
-        raises(ValueError, S.partition, '')
+        raises(ValueError, S.partition, b'')
         raises(TypeError, S.partition, None)
 
     def test_rpartition(self):
 
-        assert ('this is the rparti', 'ti', 'on method') == \
-            'this is the rpartition method'.rpartition('ti')
+        assert (b'this is the rparti', b'ti', b'on method') == \
+            b'this is the rpartition method'.rpartition(b'ti')
 
         # from raymond's original specification
-        S = 'http://www.python.org'
-        assert ('http', '://', 'www.python.org') == S.rpartition('://')
-        assert ('', '', 'http://www.python.org') == S.rpartition('?')
-        assert ('', 'http://', 'www.python.org') == S.rpartition('http://')
-        assert ('http://www.python.', 'org', '') == S.rpartition('org')
+        S = b'http://www.python.org'
+        assert (b'http', b'://', b'www.python.org') == S.rpartition(b'://')
+        assert (b'', b'', b'http://www.python.org') == S.rpartition(b'?')
+        assert (b'', b'http://', b'www.python.org') == S.rpartition(b'http://')
+        assert (b'http://www.python.', b'org', b'') == S.rpartition(b'org')
 
-        raises(ValueError, S.rpartition, '')
+        raises(ValueError, S.rpartition, b'')
         raises(TypeError, S.rpartition, None)
 
     def test_split_maxsplit(self):
-        assert "/a/b/c".split('/', 2) == ['','a','b/c']
-        assert "a/b/c".split("/") == ['a', 'b', 'c']
-        assert " a ".split(None, 0) == ['a ']
-        assert " a ".split(None, 1) == ['a']
-        assert " a a ".split(" ", 0) == [' a a ']
-        assert " a a ".split(" ", 1) == ['', 'a a ']
+        assert b"/a/b/c".split(b'/', 2) == [b'',b'a',b'b/c']
+        assert b"a/b/c".split(b"/") == [b'a', b'b', b'c']
+        assert b" a ".split(None, 0) == [b'a ']
+        assert b" a ".split(None, 1) == [b'a']
+        assert b" a a ".split(b" ", 0) == [b' a a ']
+        assert b" a a ".split(b" ", 1) == [b'', b'a a ']
 
     def test_join(self):
-        assert ", ".join(['a', 'b', 'c']) == "a, b, c"
-        assert "".join([]) == ""
-        assert "-".join(['a', 'b']) == 'a-b'
-        text = 'text'
-        assert "".join([text]) is text
-        raises(TypeError, ''.join, 1)
-        raises(TypeError, ''.join, [1])
-        raises(TypeError, ''.join, [[1]])
+        assert b", ".join([b'a', b'b', b'c']) == b"a, b, c"
+        assert b"".join([]) == b""
+        assert b"-".join([b'a', b'b']) == b'a-b'
+        text = b'text'
+        assert b"".join([text]) is text
+        raises(TypeError, b''.join, 1)
+        raises(TypeError, b''.join, [1])
+        raises(TypeError, b''.join, [[1]])
 
     def test_unicode_join_endcase(self):
         # This class inserts a Unicode object into its argument's natural
@@ -511,142 +502,139 @@
             def __iter__(self):
                 return self
 
-            def next(self):
+            def __next__(self):
                 i = self.i
                 self.i = i+1
                 if i == 2:
-                    return unicode("fooled you!")
-                return self.it.next()
+                    return "fooled you!"
+                return next(self.it)
 
-        f = ('a\n', 'b\n', 'c\n')
-        got = " - ".join(OhPhooey(f))
-        assert got == unicode("a\n - b\n - fooled you! - c\n")
+        f = (b'a\n', b'b\n', b'c\n')
+        raises(TypeError, b" - ".join, OhPhooey(f))
 
     def test_lower(self):
-        assert "aaa AAA".lower() == "aaa aaa"
-        assert "".lower() == ""
+        assert b"aaa AAA".lower() == b"aaa aaa"
+        assert b"".lower() == b""
 
     def test_upper(self):
-        assert "aaa AAA".upper() == "AAA AAA"
-        assert "".upper() == ""
+        assert b"aaa AAA".upper() == b"AAA AAA"
+        assert b"".upper() == b""
 
     def test_isalnum(self):
-        assert "".isalnum() == False
-        assert "!Bro12345w&&&&n Fox".isalnum() == False
-        assert "125 Brown Foxes".isalnum() == False
-        assert "125BrownFoxes".isalnum() == True
+        assert b"".isalnum() == False
+        assert b"!Bro12345w&&&&n Fox".isalnum() == False
+        assert b"125 Brown Foxes".isalnum() == False
+        assert b"125BrownFoxes".isalnum() == True
 
     def test_isalpha(self):
-        assert "".isalpha() == False
-        assert "!Bro12345w&&&&nFox".isalpha() == False
-        assert "Brown Foxes".isalpha() == False
-        assert "125".isalpha() == False
+        assert b"".isalpha() == False
+        assert b"!Bro12345w&&&&nFox".isalpha() == False
+        assert b"Brown Foxes".isalpha() == False
+        assert b"125".isalpha() == False
 
     def test_isdigit(self):
-        assert "".isdigit() == False
-        assert "!Bro12345w&&&&nFox".isdigit() == False
-        assert "Brown Foxes".isdigit() == False
-        assert "125".isdigit() == True
+        assert b"".isdigit() == False
+        assert b"!Bro12345w&&&&nFox".isdigit() == False
+        assert b"Brown Foxes".isdigit() == False
+        assert b"125".isdigit() == True
 
     def test_isspace(self):
-        assert "".isspace() == False
-        assert "!Bro12345w&&&&nFox".isspace() == False
-        assert " ".isspace() ==  True
-        assert "\t\t\b\b\n".isspace() == False
-        assert "\t\t".isspace() == True
-        assert "\t\t\r\r\n".isspace() == True
+        assert b"".isspace() == False
+        assert b"!Bro12345w&&&&nFox".isspace() == False
+        assert b" ".isspace() ==  True
+        assert b"\t\t\b\b\n".isspace() == False
+        assert b"\t\t".isspace() == True
+        assert b"\t\t\r\r\n".isspace() == True
 
     def test_islower(self):
-        assert "".islower() == False
-        assert " ".islower() ==  False
-        assert "\t\t\b\b\n".islower() == False
-        assert "b".islower() == True
-        assert "bbb".islower() == True
-        assert "!bbb".islower() == True
-        assert "BBB".islower() == False
-        assert "bbbBBB".islower() == False
+        assert b"".islower() == False
+        assert b" ".islower() ==  False
+        assert b"\t\t\b\b\n".islower() == False
+        assert b"b".islower() == True
+        assert b"bbb".islower() == True
+        assert b"!bbb".islower() == True
+        assert b"BBB".islower() == False
+        assert b"bbbBBB".islower() == False
 
     def test_isupper(self):
-        assert "".isupper() == False
-        assert " ".isupper() ==  False
-        assert "\t\t\b\b\n".isupper() == False
-        assert "B".isupper() == True
-        assert "BBB".isupper() == True
-        assert "!BBB".isupper() == True
-        assert "bbb".isupper() == False
-        assert "BBBbbb".isupper() == False
+        assert b"".isupper() == False
+        assert b" ".isupper() ==  False
+        assert b"\t\t\b\b\n".isupper() == False
+        assert b"B".isupper() == True
+        assert b"BBB".isupper() == True
+        assert b"!BBB".isupper() == True
+        assert b"bbb".isupper() == False
+        assert b"BBBbbb".isupper() == False
 
 
     def test_swapcase(self):
-        assert "aaa AAA 111".swapcase() == "AAA aaa 111"
-        assert "".swapcase() == ""
+        assert b"aaa AAA 111".swapcase() == b"AAA aaa 111"
+        assert b"".swapcase() == b""
 
     def test_translate(self):
         def maketrans(origin, image):
             if len(origin) != len(image):
                 raise ValueError("maketrans arguments must have same length")
-            L = [chr(i) for i in range(256)]
+            L = [i for i in range(256)]
             for i in range(len(origin)):
-                L[ord(origin[i])] = image[i]
+                L[origin[i]] = image[i]
 
-            tbl = ''.join(L)
+            tbl = bytes(L)
             return tbl
 
-        table = maketrans('abc', 'xyz')
-        assert 'xyzxyz' == 'xyzabcdef'.translate(table, 'def')
-        assert 'xyzxyz' == 'xyzabcdef'.translate(memoryview(table), 'def')
+        table = maketrans(b'abc', b'xyz')
+        assert b'xyzxyz' == b'xyzabcdef'.translate(table, 'def')
+        assert b'xyzxyz' == b'xyzabcdef'.translate(memoryview(table), 'def')
 
-        table = maketrans('a', 'A')
-        assert 'Abc' == 'abc'.translate(table)
-        assert 'xyz' == 'xyz'.translate(table)
-        assert 'yz' ==  'xyz'.translate(table, 'x')
+        table = maketrans(b'a', b'A')
+        assert b'Abc' == b'abc'.translate(table)
+        assert b'xyz' == b'xyz'.translate(table)
+        assert b'yz' ==  b'xyz'.translate(table, 'x')
 
-        raises(ValueError, 'xyz'.translate, 'too short', 'strip')
-        raises(ValueError, 'xyz'.translate, 'too short')
-        raises(ValueError, 'xyz'.translate, 'too long'*33)
+        raises(ValueError, b'xyz'.translate, b'too short', 'strip')
+        raises(ValueError, b'xyz'.translate, b'too short')
+        raises(ValueError, b'xyz'.translate, b'too long'*33)
 
-        assert 'yz' == 'xyz'.translate(None, 'x')     # 2.6
+        assert b'yz' == b'xyz'.translate(None, 'x')     # 2.6
 
     def test_iter(self):
         l=[]
-        for i in iter("42"):
+        for i in iter(b"42"):
             l.append(i)
-        assert l == ['4','2']
+        assert l == [52, 50]
 
     def test_repr(self):
-        assert repr("")       =="''"
-        assert repr("a")      =="'a'"
-        assert repr("'")      =='"\'"'
-        assert repr("\'")     =="\"\'\""
-        assert repr("\"")     =='\'"\''
-        assert repr("\t")     =="'\\t'"
-        assert repr("\\")     =="'\\\\'"
-        assert repr('')       =="''"
-        assert repr('a')      =="'a'"
-        assert repr('"')      =="'\"'"
-        assert repr('\'')     =='"\'"'
-        assert repr('\"')     =="'\"'"
-        assert repr('\t')     =="'\\t'"
-        assert repr('\\')     =="'\\\\'"
-        assert repr("'''\"")  =='\'\\\'\\\'\\\'"\''
-        assert repr(chr(19))  =="'\\x13'"
-        assert repr(chr(2))   =="'\\x02'"
+        assert repr(b"")       =="''"
+        assert repr(b"a")      =="'a'"
+        assert repr(b"'")      =='"\'"'
+        assert repr(b"\'")     =="\"\'\""
+        assert repr(b"\"")     =='\'"\''
+        assert repr(b"\t")     =="'\\t'"
+        assert repr(b"\\")     =="'\\\\'"
+        assert repr(b'')       =="''"
+        assert repr(b'a')      =="'a'"
+        assert repr(b'"')      =="'\"'"
+        assert repr(b'\'')     =='"\'"'
+        assert repr(b'\"')     =="'\"'"
+        assert repr(b'\t')     =="'\\t'"
+        assert repr(b'\\')     =="'\\\\'"
+        assert repr(b"'''\"")  =='\'\\\'\\\'\\\'"\''
+        assert repr(b"\x13")   =="'\\x13'"
+        assert repr(b"\x02")   =="'\\x02'"
 
     def test_contains(self):
-        assert '' in 'abc'
-        assert 'a' in 'abc'
-        assert 'ab' in 'abc'
-        assert not 'd' in 'abc'
-        raises(TypeError, 'a'.__contains__, 1)
+        assert b'' in b'abc'
+        assert b'a' in b'abc'
+        assert b'ab' in b'abc'
+        assert not b'd' in b'abc'
+        raises(TypeError, b'a'.__contains__, 1)
 
     def test_decode(self):
-        assert 'hello'.decode('rot-13') == 'uryyb'
-        assert 'hello'.decode('string-escape') == 'hello'
-        assert u'hello'.decode('rot-13') == 'uryyb'
+        assert b'hello'.decode('ascii') == 'hello'
 
     def test_encode(self):
-        assert 'hello'.encode() == 'hello'
-        assert type('hello'.encode()) is str
+        assert 'hello'.encode() == b'hello'
+        assert type('hello'.encode()) is bytes
 
     def test_hash(self):
         # check that we have the same hash as CPython for at least 31 bits
@@ -656,92 +644,70 @@
         assert hash('hello world!') & 0x7fffffff == 0x2f0bb411
 
     def test_buffer(self):
-        x = "he"
-        x += "llo"
+        x = b"he"
+        x += b"llo"
         b = buffer(x)
         assert len(b) == 5
-        assert b[-1] == "o"
-        assert b[:] == "hello"
-        assert b[1:0] == ""
+        assert b[-1] == b"o"
+        assert b[:] == b"hello"
+        assert b[1:0] == b""
         raises(TypeError, "b[3] = 'x'")
 
     def test_getnewargs(self):
-        assert  "foo".__getnewargs__() == ("foo",)
+        assert  b"foo".__getnewargs__() == (b"foo",)
 
     def test_subclass(self):
-        class S(str):
+        class S(bytes):
             pass
-        s = S('abc')
-        assert type(''.join([s])) is str
-        assert type(s.join([])) is str
-        assert type(s.split('x')[0]) is str
-        assert type(s.ljust(3)) is str
-        assert type(s.rjust(3)) is str
-        assert type(S('A').upper()) is str
-        assert type(S('a').lower()) is str
-        assert type(S('A').capitalize()) is str
-        assert type(S('A').title()) is str
-        assert type(s.replace(s, s)) is str
-        assert type(s.replace('x', 'y')) is str
-        assert type(s.replace('x', 'y', 0)) is str
-        assert type(s.zfill(3)) is str
-        assert type(s.strip()) is str
-        assert type(s.rstrip()) is str
-        assert type(s.lstrip()) is str
-        assert type(s.center(3)) is str
-        assert type(s.splitlines()[0]) is str
-
-    def test_str_unicode_interchangeable(self):
-        stuff = ['xxxxx', u'xxxxx']
-        for x in stuff:
-            for y in stuff:
-                assert x.startswith(y)
-                assert x.endswith(y)
-                assert x.count(y) == 1
-                assert x.find(y) != -1
-                assert x.index(y) == 0
-                d = ["x", u"x"]
-                for a in d:
-                    for b in d:
-                        assert x.replace(a, b) == y
-                assert x.rfind(y) != -1
-                assert x.rindex(y) == 0
-                assert x.split(y) == ['', '']
-                assert x.rsplit(y) == ['', '']
-                assert x.strip(y) == ''
-                assert x.rstrip(y) == ''
-                assert x.lstrip(y) == ''
+        s = S(b'abc')
+        assert type(b''.join([s])) is bytes
+        assert type(s.join([])) is bytes
+        assert type(s.split(b'x')[0]) is bytes
+        assert type(s.ljust(3)) is bytes
+        assert type(s.rjust(3)) is bytes
+        assert type(S(b'A').upper()) is bytes
+        assert type(S(b'a').lower()) is bytes
+        assert type(S(b'A').capitalize()) is bytes
+        assert type(S(b'A').title()) is bytes
+        assert type(s.replace(s, s)) is bytes
+        assert type(s.replace(b'x', b'y')) is bytes
+        assert type(s.replace(b'x', b'y', 0)) is bytes
+        assert type(s.zfill(3)) is bytes
+        assert type(s.strip()) is bytes
+        assert type(s.rstrip()) is bytes
+        assert type(s.lstrip()) is bytes
+        assert type(s.center(3)) is bytes
+        assert type(s.splitlines()[0]) is bytes
 
     def test_replace_overflow(self):
         import sys
         if sys.maxint > 2**31-1:
             skip("Wrong platform")
-        s = "a" * (2**16)
+        s = b"a" * (2**16)
         raises(OverflowError, s.replace, "", s)
 
     def test_getslice(self):
-        assert "foobar".__getslice__(4, 4321) == "ar"
-        s = "abc"
-        assert s[:] == "abc"
-        assert s[1:] == "bc"
-        assert s[:2] == "ab"
-        assert s[1:2] == "b"
-        assert s[-2:] == "bc"
-        assert s[:-1] == "ab"
-        assert s[-2:2] == "b"
-        assert s[1:-1] == "b"
-        assert s[-2:-1] == "b"
+        s = b"abc"
+        assert s[:] == b"abc"
+        assert s[1:] == b"bc"
+        assert s[:2] == b"ab"
+        assert s[1:2] == b"b"
+        assert s[-2:] == b"bc"
+        assert s[:-1] == b"ab"
+        assert s[-2:2] == b"b"
+        assert s[1:-1] == b"b"
+        assert s[-2:-1] == b"b"
 
     def test_no_len_on_str_iter(self):
-        iterable = "hello"
+        iterable = b"hello"
         raises(TypeError, len, iter(iterable))
 
     def test_overflow_replace(self):
         import sys
         if sys.maxint > 2**31-1:
             skip("Wrong platform")
-        x = "A" * (2**16)
-        raises(OverflowError, x.replace, '', x)
+        x = b"A" * (2**16)
+        raises(OverflowError, x.replace, b'', x)
 
 class AppTestPrebuilt(AppTestStringObject):
     def setup_class(cls):

From noreply at buildbot.pypy.org  Tue Oct 18 00:37:59 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 18 Oct 2011 00:37:59 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Fix test_newformat
Message-ID: <20111017223759.71FBB820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48168:f12a2184cae5
Date: 2011-10-17 22:50 +0200
http://bitbucket.org/pypy/pypy/changeset/f12a2184cae5/

Log:	Fix test_newformat

diff --git a/pypy/objspace/std/newformat.py b/pypy/objspace/std/newformat.py
--- a/pypy/objspace/std/newformat.py
+++ b/pypy/objspace/std/newformat.py
@@ -324,7 +324,7 @@
             if recursive:
                 spec = self._build_string(spec_start, end, level)
             w_rendered = self.space.format(w_obj, self.space.wrap(spec))
-            unwrapper = "unicode_w" if self.is_unicode else "str_w"
+            unwrapper = "unicode_w" if self.is_unicode else "bytes_w"
             to_interp = getattr(self.space, unwrapper)
             return to_interp(w_rendered)
 
@@ -360,7 +360,7 @@
                                               space.unicode_w(w_string))
         return space.wrap(template.build(args))
     else:
-        template = str_template_formatter(space, space.str_w(w_string))
+        template = str_template_formatter(space, space.bytes_w(w_string))
         return space.wrap(template.build(args))
 
 
@@ -534,8 +534,11 @@
             w_msg = self.space.wrap(msg  % (tp, self._type))
             raise OperationError(self.space.w_ValueError, w_msg)
 
-        def format_string(self, string):
+        def format_string(self, w_string):
             space = self.space
+            if not space.isinstance_w(w_string, space.w_unicode):
+                w_string = space.str(w_string)
+            string = space.unicode_w(w_string)
             if self._parse_spec("s", "<"):
                 return space.wrap(string)
             if self._type != "s":
@@ -1133,21 +1136,12 @@
     return Formatter
 
 StrFormatter = make_formatting_class()
-UnicodeFormatter = make_formatting_class()
 
 
 def unicode_formatter(space, spec):
     return StrFormatter(space, True, spec)
 
-def str_formatter(space, spec):
-    return UnicodeFormatter(space, False, spec)
-
-
 @specialize.arg(2)
 def run_formatter(space, w_format_spec, meth, *args):
-    if space.isinstance_w(w_format_spec, space.w_unicode):
-        formatter = unicode_formatter(space, space.unicode_w(w_format_spec))
-        return getattr(formatter, meth)(*args)
-    else:
-        formatter = str_formatter(space, space.str_w(w_format_spec))
-        return getattr(formatter, meth)(*args)
+    formatter = unicode_formatter(space, space.unicode_w(w_format_spec))
+    return getattr(formatter, meth)(*args)
diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py
--- a/pypy/objspace/std/stringobject.py
+++ b/pypy/objspace/std/stringobject.py
@@ -988,13 +988,6 @@
     encoding, errors = _get_encoding_and_errors(space, w_encoding, w_errors)
     return decode_object(space, w_string, encoding, errors)
 
-def format__String_ANY(space, w_string, w_format_spec):
-    if not space.isinstance_w(w_format_spec, space.w_str):
-        w_format_spec = space.str(w_format_spec)
-    spec = space.str_w(w_format_spec)
-    formatter = newformat.str_formatter(space, spec)
-    return formatter.format_string(w_string._value)
-
 def buffer__String(space, w_string):
     return space.wrap(StringBuffer(w_string._value))
 
diff --git a/pypy/objspace/std/test/test_newformat.py b/pypy/objspace/std/test/test_newformat.py
--- a/pypy/objspace/std/test/test_newformat.py
+++ b/pypy/objspace/std/test/test_newformat.py
@@ -179,9 +179,7 @@
                 return "32"
             def __str__(self):
                 return "18"
-            def __unicode__(self):
-                return "42"
-        assert self.s("{!s}").format(x()) == "42"
+        assert self.s("{!s}").format(x()) == "18"
         assert self.s("{!r}").format(x()) == "32"
 
     def test_non_latin1_key(self):
@@ -189,28 +187,11 @@
 
 
 
-class AppTestStringFormat(BaseStringFormatTests):
-
-    def setup_class(cls):
-        cls.w_s = cls.space.w_str
-
-    def test_string_conversion(self):
-        class x(object):
-            def __repr__(self):
-                return "32"
-            def __str__(self):
-                return "18"
-            def __unicode__(self):
-                return "42"
-        assert self.s("{!s}").format(x()) == "18"
-        assert self.s("{!r}").format(x()) == "32"
-
-
 class BaseIntegralFormattingTest:
 
     def test_simple(self):
         assert format(self.i(2)) == "2"
-        assert isinstance(format(self.i(2), u""), unicode)
+        assert isinstance(format(self.i(2), u""), str)
 
     def test_invalid(self):
         raises(ValueError, format, self.i(8), "s")
@@ -221,7 +202,7 @@
         assert format(a, "c") == "a"
         as_uni = format(a, u"c")
         assert as_uni == u"a"
-        assert isinstance(as_uni, unicode)
+        assert isinstance(as_uni, str)
         raises(ValueError, format, a, "-c")
         raises(ValueError, format, a, ",c")
         assert format(a, "3c") == "  a"
@@ -353,70 +334,74 @@
     # undocumented API on string and unicode object, but used by string.py
 
     def test_formatter_parser(self):
-        l = list('abcd'._formatter_parser())
+        import _string
+        l = list(_string.formatter_parser('abcd'))
         assert l == [('abcd', None, None, None)]
         #
-        l = list('ab{0}cd'._formatter_parser())
+        l = list(_string.formatter_parser('ab{0}cd'))
         assert l == [('ab', '0', '', None), ('cd', None, None, None)]
         #
-        l = list('{0}cd'._formatter_parser())
+        l = list(_string.formatter_parser('{0}cd'))
         assert l == [('', '0', '', None), ('cd', None, None, None)]
         #
-        l = list('ab{0}'._formatter_parser())
+        l = list(_string.formatter_parser('ab{0}'))
         assert l == [('ab', '0', '', None)]
         #
-        l = list(''._formatter_parser())
+        l = list(_string.formatter_parser(''))
         assert l == []
         #
-        l = list('{0:123}'._formatter_parser())
+        l = list(_string.formatter_parser('{0:123}'))
         assert l == [('', '0', '123', None)]
         #
-        l = list('{0!x:123}'._formatter_parser())
+        l = list(_string.formatter_parser('{0!x:123}'))
         assert l == [('', '0', '123', 'x')]
         #
-        l = list('{0!x:12{sdd}3}'._formatter_parser())
+        l = list(_string.formatter_parser('{0!x:12{sdd}3}'))
         assert l == [('', '0', '12{sdd}3', 'x')]
 
     def test_u_formatter_parser(self):
-        l = list(u'{0!x:12{sdd}3}'._formatter_parser())
+        import _string
+        l = list(_string.formatter_parser('{0!x:12{sdd}3}'))
         assert l == [(u'', u'0', u'12{sdd}3', u'x')]
         for x in l[0]:
-            assert isinstance(x, unicode)
+            assert isinstance(x, str)
 
     def test_formatter_field_name_split(self):
-        first, rest = ''._formatter_field_name_split()
+        import _string
+        first, rest = _string.formatter_field_name_split('')
         assert first == ''
         assert list(rest) == []
         #
-        first, rest = '31'._formatter_field_name_split()
+        first, rest = _string.formatter_field_name_split('31')
         assert first == 31
         assert list(rest) == []
         #
-        first, rest = 'foo'._formatter_field_name_split()
+        first, rest = _string.formatter_field_name_split('foo')
         assert first == 'foo'
         assert list(rest) == []
         #
-        first, rest = 'foo.bar'._formatter_field_name_split()
+        first, rest = _string.formatter_field_name_split('foo.bar')
         assert first == 'foo'
         assert list(rest) == [(True, 'bar')]
         #
-        first, rest = 'foo[123]'._formatter_field_name_split()
+        first, rest = _string.formatter_field_name_split('foo[123]')
         assert first == 'foo'
         assert list(rest) == [(False, 123)]
         #
-        first, rest = 'foo.baz[123].bok'._formatter_field_name_split()
+        first, rest = _string.formatter_field_name_split('foo.baz[123].bok')
         assert first == 'foo'
         assert list(rest) == [(True, 'baz'), (False, 123), (True, 'bok')]
         #
-        first, rest = 'foo.baz[hi].bok'._formatter_field_name_split()
+        first, rest = _string.formatter_field_name_split('foo.baz[hi].bok')
         assert first == 'foo'
         assert list(rest) == [(True, 'baz'), (False, 'hi'), (True, 'bok')]
 
     def test_u_formatter_field_name_split(self):
-        first, rest = u'foo.baz[hi].bok'._formatter_field_name_split()
+        import _string
+        first, rest = _string.formatter_field_name_split('foo.baz[hi].bok')
         l = list(rest)
         assert first == u'foo'
         assert l == [(True, u'baz'), (False, u'hi'), (True, u'bok')]
-        assert isinstance(first, unicode)
+        assert isinstance(first, str)
         for x, y in l:
-            assert isinstance(y, unicode)
+            assert isinstance(y, str)
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -827,14 +827,8 @@
 def unicode_format__Unicode(space, w_unicode, __args__):
     return newformat.format_method(space, w_unicode, __args__, True)
 
-def format__Unicode_ANY(space, w_unicode, w_format_spec):
-    if not space.isinstance_w(w_format_spec, space.w_unicode):
-        w_format_spec = space.call_function(space.w_unicode, w_format_spec)
-    from pypy.objspace.std.unicodetype import unicode_from_object
-    w_unicode = unicode_from_object(space, w_unicode)
-    spec = space.unicode_w(w_format_spec)
-    formatter = newformat.unicode_formatter(space, spec)
-    return formatter.format_string(space.unicode_w(w_unicode))
+def format__Unicode_ANY(space, w_unicode, w_spec):
+    return newformat.run_formatter(space, w_spec, "format_string", w_unicode)
 
 
 from pypy.objspace.std import unicodetype

From noreply at buildbot.pypy.org  Tue Oct 18 00:38:00 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 18 Oct 2011 00:38:00 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Adapt some applevel code to py3k
Message-ID: <20111017223800.9F1C3820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48169:d3e2d6ec5903
Date: 2011-10-17 23:33 +0200
http://bitbucket.org/pypy/pypy/changeset/d3e2d6ec5903/

Log:	Adapt some applevel code to py3k

diff --git a/pypy/module/__builtin__/app_io.py b/pypy/module/__builtin__/app_io.py
--- a/pypy/module/__builtin__/app_io.py
+++ b/pypy/module/__builtin__/app_io.py
@@ -18,7 +18,7 @@
             loc = locals()
     elif loc is None:
         loc = glob
-    f = file(filename, 'rU')
+    f = open(filename, 'rU')
     try:
         source = f.read()
     finally:
diff --git a/pypy/module/sys/currentframes.py b/pypy/module/sys/currentframes.py
--- a/pypy/module/sys/currentframes.py
+++ b/pypy/module/sys/currentframes.py
@@ -5,7 +5,7 @@
 
 app = gateway.applevel('''
 "NOT_RPYTHON"
-import __builtin__
+import builtins
 
 class fake_code(object):
     co_name = "?"
@@ -14,7 +14,7 @@
 
 class fake_frame(object):
     f_back = None
-    f_builtins = __builtin__.__dict__
+    f_builtins = builtins.__dict__
     f_code = fake_code()
     f_exc_traceback = None
     f_exc_type = None

From noreply at buildbot.pypy.org  Tue Oct 18 00:38:01 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 18 Oct 2011 00:38:01 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Fix interp_level helper functions in
	AppTest
Message-ID: <20111017223801.CC88C820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48170:d3b60f18ce64
Date: 2011-10-17 23:34 +0200
http://bitbucket.org/pypy/pypy/changeset/d3b60f18ce64/

Log:	Fix interp_level helper functions in AppTest

diff --git a/pypy/conftest.py b/pypy/conftest.py
--- a/pypy/conftest.py
+++ b/pypy/conftest.py
@@ -460,7 +460,7 @@
                             %s
                             return %s
                         """) % (source, name))
-                        w_obj = Method(space, w_func, w_instance, space.w_None)
+                        w_obj = Method(space, w_func, w_instance)
                     else:
                         w_obj = obj
                     space.setattr(w_instance, space.wrap(name[2:]), w_obj)

From noreply at buildbot.pypy.org  Tue Oct 18 00:38:03 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 18 Oct 2011 00:38:03 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Fix syntax of applevel code
Message-ID: <20111017223803.0A223820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48171:ce1ccc552d38
Date: 2011-10-17 23:46 +0200
http://bitbucket.org/pypy/pypy/changeset/ce1ccc552d38/

Log:	Fix syntax of applevel code

diff --git a/pypy/objspace/std/objecttype.py b/pypy/objspace/std/objecttype.py
--- a/pypy/objspace/std/objecttype.py
+++ b/pypy/objspace/std/objecttype.py
@@ -152,7 +152,7 @@
     else:
         args = getnewargs()
         if not isinstance(args, tuple):
-            raise TypeError, "__getnewargs__ should return a tuple"
+            raise TypeError("__getnewargs__ should return a tuple")
 
     try:
         getstate = obj.__getstate__
@@ -201,7 +201,7 @@
     import copy_reg
     slotnames = copy_reg._slotnames(cls)
     if not isinstance(slotnames, list) and slotnames is not None:
-        raise TypeError, "copy_reg._slotnames didn't return a list or None"
+        raise TypeError("copy_reg._slotnames didn't return a list or None")
     return slotnames
 ''', filename=__file__)
 
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -924,7 +924,7 @@
             if klass not in mro:
                 mro.append(klass)
                 if not isinstance(klass.__bases__, tuple):
-                    raise TypeError, '__bases__ must be a tuple'
+                    raise TypeError('__bases__ must be a tuple')
                 stack += klass.__bases__[::-1]
         return mro
 """, filename=__file__).interphook("abstract_mro")
diff --git a/pypy/translator/goal/app_main.py b/pypy/translator/goal/app_main.py
--- a/pypy/translator/goal/app_main.py
+++ b/pypy/translator/goal/app_main.py
@@ -61,7 +61,7 @@
             if softspace:
                 stdout.write('\n')
 
-    except SystemExit, e:
+    except SystemExit as e:
         handle_sys_exit(e)
     except:
         display_exception()
@@ -377,7 +377,7 @@
             funcarg = ''.join(remaining)
         else:
             try:
-                funcarg = iterargv.next()
+                funcarg = next(iterargv)
             except StopIteration:
                 if len(c) == 1:
                     c = '-' + c
@@ -409,7 +409,7 @@
         # Else interpret the rest of the argument character by character
         else:
             iterarg = iter(arg)
-            iterarg.next()     # skip the '-'
+            next(iterarg)      # skip the '-'
             for c in iterarg:
                 if c not in cmdline_options:
                     raise CommandLineError('Unknown option: -%s' % (c,))
@@ -560,7 +560,7 @@
                         f = open(python_startup)
                         startup = f.read()
                         f.close()
-                    except IOError, e:
+                    except IOError as e:
                         print("Could not open PYTHONSTARTUP", file=sys.stderr)
                         print("IOError:", e, file=sys.stderr)
                     else:
@@ -609,7 +609,7 @@
                     args = (execfile, filename, mainmodule.__dict__)
             success = run_toplevel(*args)
 
-    except SystemExit, e:
+    except SystemExit as e:
         status = e.code
         if inspect_requested():
             display_exception()
@@ -622,7 +622,7 @@
         try:
             from _pypy_interact import interactive_console
             success = run_toplevel(interactive_console, mainmodule)
-        except SystemExit, e:
+        except SystemExit as e:
             status = e.code
         else:
             status = not success

From noreply at buildbot.pypy.org  Tue Oct 18 00:38:04 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 18 Oct 2011 00:38:04 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: unicode.lower() is not rpython it seems.
Message-ID: <20111017223804.39391820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48172:8b97595e7f42
Date: 2011-10-18 00:06 +0200
http://bitbucket.org/pypy/pypy/changeset/8b97595e7f42/

Log:	unicode.lower() is not rpython it seems.

diff --git a/pypy/objspace/std/bytearraytype.py b/pypy/objspace/std/bytearraytype.py
--- a/pypy/objspace/std/bytearraytype.py
+++ b/pypy/objspace/std/bytearraytype.py
@@ -78,6 +78,8 @@
     val = ord(d)
     if 47 < val < 58:
         return val - 48
+    if 64 < val < 71:
+        return val - 55
     if 96 < val < 103:
         return val - 87
     return -1
@@ -88,7 +90,6 @@
     "accepted.\nExample: bytearray.fromhex('B9 01EF') -> "
     "bytearray(b'\\xb9\\x01\\xef')."
     hexstring = space.unicode_w(w_hexstring)
-    hexstring = hexstring.lower()
     data = []
     length = len(hexstring)
     i = -2

From noreply at buildbot.pypy.org  Tue Oct 18 00:38:05 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 18 Oct 2011 00:38:05 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Translation fixes
Message-ID: <20111017223805.6BEB5820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48173:3693b7e7f95e
Date: 2011-10-18 00:35 +0200
http://bitbucket.org/pypy/pypy/changeset/3693b7e7f95e/

Log:	Translation fixes

diff --git a/pypy/interpreter/module.py b/pypy/interpreter/module.py
--- a/pypy/interpreter/module.py
+++ b/pypy/interpreter/module.py
@@ -31,7 +31,7 @@
     def install(self):
         """NOT_RPYTHON: installs this module into space.builtin_modules"""
         w_mod = self.space.wrap(self)
-        self.space.builtin_modules[self.space.unwrap(self.w_name)] = w_mod
+        self.space.builtin_modules[self.space.str_w(self.w_name)] = w_mod
 
     def setup_after_space_initialization(self):
         """NOT_RPYTHON: to allow built-in modules to do some more setup
diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -293,58 +293,58 @@
     return space.wrap(count)
 
 def str_count__Bytearray_ANY_ANY_ANY(space, w_bytearray, w_char, w_start, w_stop):
-    w_char = space.wrap(space.bufferstr_new_w(w_char))
+    w_char = space.wrapbytes(space.bufferstr_new_w(w_char))
     w_str = _to_bytes(space, w_bytearray)
     return stringobject.str_count__String_String_ANY_ANY(space, w_str, w_char,
                                                          w_start, w_stop)
 
 def str_index__Bytearray_ANY_ANY_ANY(space, w_bytearray, w_char, w_start, w_stop):
-    w_char = space.wrap(space.bufferstr_new_w(w_char))
+    w_char = space.wrapbytes(space.bufferstr_new_w(w_char))
     w_str = _to_bytes(space, w_bytearray)
     return stringobject.str_index__String_String_ANY_ANY(space, w_str, w_char,
                                                          w_start, w_stop)
 
 def str_rindex__Bytearray_ANY_ANY_ANY(space, w_bytearray, w_char, w_start, w_stop):
-    w_char = space.wrap(space.bufferstr_new_w(w_char))
+    w_char = space.wrapbytes(space.bufferstr_new_w(w_char))
     w_str = _to_bytes(space, w_bytearray)
     return stringobject.str_rindex__String_String_ANY_ANY(space, w_str, w_char,
                                                          w_start, w_stop)
 
 def str_find__Bytearray_ANY_ANY_ANY(space, w_bytearray, w_char, w_start, w_stop):
-    w_char = space.wrap(space.bufferstr_new_w(w_char))
+    w_char = space.wrapbytes(space.bufferstr_new_w(w_char))
     w_str = _to_bytes(space, w_bytearray)
     return stringobject.str_find__String_String_ANY_ANY(space, w_str, w_char,
                                                          w_start, w_stop)
 
 def str_rfind__Bytearray_ANY_ANY_ANY(space, w_bytearray, w_char, w_start, w_stop):
-    w_char = space.wrap(space.bufferstr_new_w(w_char))
+    w_char = space.wrapbytes(space.bufferstr_new_w(w_char))
     w_str = _to_bytes(space, w_bytearray)
     return stringobject.str_rfind__String_String_ANY_ANY(space, w_str, w_char,
                                                          w_start, w_stop)
 
 def str_startswith__Bytearray_ANY_ANY_ANY(space, w_bytearray, w_prefix, w_start, w_stop):
-    w_prefix = space.wrap(space.bufferstr_new_w(w_prefix))
+    w_prefix = space.wrapbytes(space.bufferstr_new_w(w_prefix))
     w_str = _to_bytes(space, w_bytearray)
     return stringobject.str_startswith__String_String_ANY_ANY(space, w_str, w_prefix,
                                                               w_start, w_stop)
 
 def str_startswith__Bytearray_Tuple_ANY_ANY(space, w_bytearray, w_prefix, w_start, w_stop):
     w_str = _to_bytes(space, w_bytearray)
-    w_prefix = space.newtuple([space.wrap(space.bufferstr_new_w(w_entry)) for w_entry in
-                               space.unpackiterable(w_prefix)])
+    w_prefix = space.newtuple([space.wrapbytes(space.bufferstr_new_w(w_entry))
+                               for w_entry in space.unpackiterable(w_prefix)])
     return stringobject.str_startswith__String_Tuple_ANY_ANY(space, w_str, w_prefix,
                                                               w_start, w_stop)
 
 def str_endswith__Bytearray_ANY_ANY_ANY(space, w_bytearray, w_suffix, w_start, w_stop):
-    w_suffix = space.wrap(space.bufferstr_new_w(w_suffix))
+    w_suffix = space.wrapbytes(space.bufferstr_new_w(w_suffix))
     w_str = _to_bytes(space, w_bytearray)
     return stringobject.str_endswith__String_String_ANY_ANY(space, w_str, w_suffix,
                                                               w_start, w_stop)
 
 def str_endswith__Bytearray_Tuple_ANY_ANY(space, w_bytearray, w_suffix, w_start, w_stop):
     w_str = _to_bytes(space, w_bytearray)
-    w_suffix = space.newtuple([space.wrap(space.bufferstr_new_w(w_entry)) for w_entry in
-                               space.unpackiterable(w_suffix)])
+    w_suffix = space.newtuple([space.wrapbytes(space.bufferstr_new_w(w_entry))
+                               for w_entry in space.unpackiterable(w_suffix)])
     return stringobject.str_endswith__String_Tuple_ANY_ANY(space, w_str, w_suffix,
                                                               w_start, w_stop)
 
diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py
--- a/pypy/objspace/std/stringobject.py
+++ b/pypy/objspace/std/stringobject.py
@@ -640,8 +640,8 @@
 
 def str_endswith__String_Tuple_ANY_ANY(space, w_self, w_suffixes, w_start, w_end):
     (u_self, _, start, end) = _convert_idx_params(space, w_self,
-                                                  space.wrap(''), w_start,
-                                                  w_end, True)
+                                                  space.wrapbytes(''),
+                                                  w_start, w_end, True)
     for w_suffix in space.fixedview(w_suffixes):
         if space.isinstance_w(w_suffix, space.w_unicode):
             w_u = space.call_function(space.w_unicode, w_self)
@@ -659,7 +659,8 @@
     return space.newbool(stringstartswith(u_self, prefix, start, end))
 
 def str_startswith__String_Tuple_ANY_ANY(space, w_self, w_prefixes, w_start, w_end):
-    (u_self, _, start, end) = _convert_idx_params(space, w_self, space.wrap(''),
+    (u_self, _, start, end) = _convert_idx_params(space, w_self,
+                                                  space.wrapbytes(''),
                                                   w_start, w_end, True)
     for w_prefix in space.fixedview(w_prefixes):
         if space.isinstance_w(w_prefix, space.w_unicode):
diff --git a/pypy/rlib/rbigint.py b/pypy/rlib/rbigint.py
--- a/pypy/rlib/rbigint.py
+++ b/pypy/rlib/rbigint.py
@@ -231,7 +231,7 @@
                 accumbits -= SHIFT
         if accumbits:
             digits.append(_store_digit(accum))
-        return rbigint(digits, 1)
+        return rbigint(digits[:], 1)
 
     @jit.elidable
     def toint(self):

From noreply at buildbot.pypy.org  Tue Oct 18 01:04:24 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 18 Oct 2011 01:04:24 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Partially revert ef504fcb59bb,
	fixes a translation issue.
Message-ID: <20111017230424.43DCC820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48174:e301a12848ad
Date: 2011-10-18 00:42 +0200
http://bitbucket.org/pypy/pypy/changeset/e301a12848ad/

Log:	Partially revert ef504fcb59bb, fixes a translation issue.

diff --git a/pypy/interpreter/pyparser/pyparse.py b/pypy/interpreter/pyparser/pyparse.py
--- a/pypy/interpreter/pyparser/pyparse.py
+++ b/pypy/interpreter/pyparser/pyparse.py
@@ -4,15 +4,15 @@
 from pypy.interpreter.astcompiler import consts
 
 
-def decode_source(space, bytes, encoding=None):
+def recode_to_utf8(space, bytes, encoding=None):
     if encoding is None:
         encoding = 'utf-8'
     if encoding == 'utf-8':
         return bytes
-    text = space.unicode_w(space.call_function(space.w_unicode,
-                                               space.wrapbytes(bytes),
-                                               space.wrap(encoding)))
-    return text.encode('utf-8')
+    w_text = space.call_method(space.wrapbytes(bytes), "decode",
+                               space.wrap(encoding))
+    w_recoded = space.call_method(w_text, "encode", space.wrap("utf-8"))
+    return space.bytes_w(w_recoded)
 
 def _normalize_encoding(encoding):
     """returns normalized name for <encoding>
@@ -112,17 +112,17 @@
             if decl_enc and decl_enc != "utf-8":
                 raise error.SyntaxError("UTF-8 BOM with non-utf8 coding cookie",
                                         filename=compile_info.filename)
-            textsrc = decode_source(self.space, bytessrc, enc)
+            textsrc = bytessrc
         elif compile_info.flags & consts.PyCF_SOURCE_IS_UTF8:
             enc = 'utf-8'
             if _check_for_encoding(bytessrc) is not None:
                 raise error.SyntaxError("coding declaration in unicode string",
                                         filename=compile_info.filename)
-            textsrc = decode_source(self.space, bytessrc, enc)
+            textsrc = bytessrc
         else:
             enc = _normalize_encoding(_check_for_encoding(bytessrc))
             try:
-                textsrc = decode_source(self.space, bytessrc, enc)
+                textsrc = recode_to_utf8(self.space, bytessrc, enc)
             except OperationError, e:
                 # if the codec is not found, LookupError is raised.  we
                 # check using 'is_w' not to mask potential IndexError or

From noreply at buildbot.pypy.org  Tue Oct 18 01:04:25 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 18 Oct 2011 01:04:25 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Remove code generation for the Print
	statement.
Message-ID: <20111017230425.735A4820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48175:9b8d2f9b420c
Date: 2011-10-18 01:01 +0200
http://bitbucket.org/pypy/pypy/changeset/9b8d2f9b420c/

Log:	Remove code generation for the Print statement.

diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -393,29 +393,6 @@
             self.load_const(self.space.w_None)
         self.emit_op(ops.RETURN_VALUE)
 
-    def visit_Print(self, pr):
-        self.update_position(pr.lineno, True)
-        have_dest = bool(pr.dest)
-        if have_dest:
-            pr.dest.walkabout(self)
-        if pr.values:
-            for value in pr.values:
-                if have_dest:
-                    self.emit_op(ops.DUP_TOP)
-                    value.walkabout(self)
-                    self.emit_op(ops.ROT_TWO)
-                    self.emit_op(ops.PRINT_ITEM_TO)
-                else:
-                    value.walkabout(self)
-                    self.emit_op(ops.PRINT_ITEM)
-        if pr.nl:
-            if have_dest:
-                self.emit_op(ops.PRINT_NEWLINE_TO)
-            else:
-                self.emit_op(ops.PRINT_NEWLINE)
-        elif have_dest:
-            self.emit_op(ops.POP_TOP)
-
     def visit_Delete(self, delete):
         self.update_position(delete.lineno, True)
         self.visit_sequence(delete.targets)

From noreply at buildbot.pypy.org  Tue Oct 18 01:04:26 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 18 Oct 2011 01:04:26 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Add stack effect for new opcodes
Message-ID: <20111017230426.A0CFD820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48176:b14ef64468e2
Date: 2011-10-18 01:03 +0200
http://bitbucket.org/pypy/pypy/changeset/b14ef64468e2/

Log:	Add stack effect for new opcodes

diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -544,6 +544,7 @@
     ops.LOAD_BUILD_CLASS : 1,
     ops.STORE_LOCALS : -1,
     ops.POP_BLOCK : 0,
+    ops.POP_EXCEPT : 0,
     ops.END_FINALLY : -3,
     ops.SETUP_WITH : 1,
     ops.SETUP_FINALLY : 0,
@@ -572,6 +573,7 @@
     ops.LOAD_GLOBAL : 1,
     ops.STORE_GLOBAL : -1,
     ops.DELETE_GLOBAL : 0,
+    ops.DELETE_DEREF : 0,
 
     ops.LOAD_CLOSURE : 1,
     ops.LOAD_DEREF : 1,
@@ -595,6 +597,9 @@
 def _compute_UNPACK_SEQUENCE(arg):
     return arg + 1
 
+def _compute_UNPACK_EX(arg):
+    return (arg % 256) + (arg // 256)
+
 def _compute_BUILD_TUPLE(arg):
     return 1 - arg
 

From noreply at buildbot.pypy.org  Tue Oct 18 06:09:05 2011
From: noreply at buildbot.pypy.org (justinpeel)
Date: Tue, 18 Oct 2011 06:09:05 +0200 (CEST)
Subject: [pypy-commit] pypy rgc-mem-pressure: add a __del__ to semaphores
	(untested)
Message-ID: <20111018040905.5E634820D7@wyvern.cs.uni-duesseldorf.de>

Author: Justin Peel <notmuchtotell at gmail.com>
Branch: rgc-mem-pressure
Changeset: r48177:caf2ca4afb8d
Date: 2011-10-17 22:08 -0600
http://bitbucket.org/pypy/pypy/changeset/caf2ca4afb8d/

Log:	add a __del__ to semaphores (untested)

diff --git a/pypy/module/_multiprocessing/interp_semaphore.py b/pypy/module/_multiprocessing/interp_semaphore.py
--- a/pypy/module/_multiprocessing/interp_semaphore.py
+++ b/pypy/module/_multiprocessing/interp_semaphore.py
@@ -23,6 +23,8 @@
     _CreateSemaphore = rwin32.winexternal(
         'CreateSemaphoreA', [rffi.VOIDP, rffi.LONG, rffi.LONG, rwin32.LPCSTR],
         rwin32.HANDLE)
+    _CloseHandle = rwin32.winexternal('CloseHandle', [rwin32.HANDLE],
+        rwin32.BOOL)
     _ReleaseSemaphore = rwin32.winexternal(
         'ReleaseSemaphore', [rwin32.HANDLE, rffi.LONG, rffi.LONGP],
         rwin32.BOOL)
@@ -73,6 +75,7 @@
     _sem_open = external('sem_open',
                          [rffi.CCHARP, rffi.INT, rffi.INT, rffi.UINT],
                          SEM_T)
+    _sem_close = external('sem_close', [SEM_T], rffi.INT)
     _sem_unlink = external('sem_unlink', [rffi.CCHARP], rffi.INT)
     _sem_wait = external('sem_wait', [SEM_T], rffi.INT)
     _sem_trywait = external('sem_trywait', [SEM_T], rffi.INT)
@@ -90,6 +93,11 @@
             raise OSError(rposix.get_errno(), "sem_open failed")
         return res
 
+    def sem_close(handle):
+        res = _sem_close(handle)
+        if res < 0:
+            raise OSError(rposix.get_errno(), "sem_close failed")
+
     def sem_unlink(name):
         res = _sem_unlink(name)
         if res < 0:
@@ -205,6 +213,11 @@
             raise WindowsError(err, "CreateSemaphore")
         return handle
 
+    def delete_semaphore(handle):
+        if not _CloseHandle(handle):
+            err = rwin32.GetLastError()
+            raise WindowsError(err, "CloseHandle")
+
     def semlock_acquire(self, space, block, w_timeout):
         if not block:
             full_msecs = 0
@@ -293,6 +306,9 @@
             pass
         return sem
 
+    def delete_semaphore(space, handle):
+        sem_close(handle)
+
     def semlock_acquire(self, space, block, w_timeout):
         if not block:
             deadline = lltype.nullptr(TIMESPECP.TO)
@@ -483,6 +499,9 @@
     def exit(self, space, __args__):
         self.release(space)
 
+    def __del__(self):
+        delete_semaphore(self.handle)
+
 @unwrap_spec(kind=int, value=int, maxvalue=int)
 def descr_new(space, w_subtype, kind, value, maxvalue):
     if kind != RECURSIVE_MUTEX and kind != SEMAPHORE:

From noreply at buildbot.pypy.org  Tue Oct 18 06:15:30 2011
From: noreply at buildbot.pypy.org (justinpeel)
Date: Tue, 18 Oct 2011 06:15:30 +0200 (CEST)
Subject: [pypy-commit] pypy rgc-mem-pressure: add memory pressure to
	semaphores for linux
Message-ID: <20111018041530.2EBD6820D7@wyvern.cs.uni-duesseldorf.de>

Author: Justin Peel <notmuchtotell at gmail.com>
Branch: rgc-mem-pressure
Changeset: r48178:f6cf2a0021b8
Date: 2011-10-17 22:15 -0600
http://bitbucket.org/pypy/pypy/changeset/f6cf2a0021b8/

Log:	add memory pressure to semaphores for linux

diff --git a/pypy/module/_multiprocessing/interp_semaphore.py b/pypy/module/_multiprocessing/interp_semaphore.py
--- a/pypy/module/_multiprocessing/interp_semaphore.py
+++ b/pypy/module/_multiprocessing/interp_semaphore.py
@@ -4,6 +4,7 @@
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.error import wrap_oserror, OperationError
 from pypy.rpython.lltypesystem import rffi, lltype
+from pypy.rlib import rgc
 from pypy.rlib.rarithmetic import r_uint
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
 from pypy.rpython.tool import rffi_platform as platform
@@ -53,6 +54,7 @@
         SEM_FAILED = platform.ConstantInteger('SEM_FAILED')
         SEM_VALUE_MAX = platform.ConstantInteger('SEM_VALUE_MAX')
         SEM_TIMED_WAIT = platform.Has('sem_timedwait')
+        SEM_T_SIZE = platform.SizeOf('sem_t')
 
     config = platform.configure(CConfig)
     TIMEVAL        = config['TIMEVAL']
@@ -63,6 +65,7 @@
     SEM_FAILED     = config['SEM_FAILED'] # rffi.cast(SEM_T, config['SEM_FAILED'])
     SEM_VALUE_MAX  = config['SEM_VALUE_MAX']
     SEM_TIMED_WAIT = config['SEM_TIMED_WAIT']
+    SEM_T_SIZE = config['SEM_T_SIZE']
     if sys.platform == 'darwin':
         HAVE_BROKEN_SEM_GETVALUE = True
     else:
@@ -302,6 +305,7 @@
         sem = sem_open(name, os.O_CREAT | os.O_EXCL, 0600, val)
         try:
             sem_unlink(name)
+            rgc.add_memory_pressure(SEM_T_SIZE)
         except OSError:
             pass
         return sem

From noreply at buildbot.pypy.org  Tue Oct 18 06:18:18 2011
From: noreply at buildbot.pypy.org (justinpeel)
Date: Tue, 18 Oct 2011 06:18:18 +0200 (CEST)
Subject: [pypy-commit] pypy rgc-mem-pressure: small change to mem pressure
	for semaphores
Message-ID: <20111018041818.E2410820D7@wyvern.cs.uni-duesseldorf.de>

Author: Justin Peel <notmuchtotell at gmail.com>
Branch: rgc-mem-pressure
Changeset: r48179:e177050f0c14
Date: 2011-10-17 22:18 -0600
http://bitbucket.org/pypy/pypy/changeset/e177050f0c14/

Log:	small change to mem pressure for semaphores

diff --git a/pypy/module/_multiprocessing/interp_semaphore.py b/pypy/module/_multiprocessing/interp_semaphore.py
--- a/pypy/module/_multiprocessing/interp_semaphore.py
+++ b/pypy/module/_multiprocessing/interp_semaphore.py
@@ -305,9 +305,10 @@
         sem = sem_open(name, os.O_CREAT | os.O_EXCL, 0600, val)
         try:
             sem_unlink(name)
-            rgc.add_memory_pressure(SEM_T_SIZE)
         except OSError:
             pass
+        else:
+            rgc.add_memory_pressure(SEM_T_SIZE)
         return sem
 
     def delete_semaphore(space, handle):

From noreply at buildbot.pypy.org  Tue Oct 18 07:24:02 2011
From: noreply at buildbot.pypy.org (justinpeel)
Date: Tue, 18 Oct 2011 07:24:02 +0200 (CEST)
Subject: [pypy-commit] pypy rgc-mem-pressure: small fix for semaphores
Message-ID: <20111018052402.550D8820D7@wyvern.cs.uni-duesseldorf.de>

Author: Justin Peel <notmuchtotell at gmail.com>
Branch: rgc-mem-pressure
Changeset: r48180:3d953befd58a
Date: 2011-10-17 23:23 -0600
http://bitbucket.org/pypy/pypy/changeset/3d953befd58a/

Log:	small fix for semaphores

diff --git a/pypy/module/_multiprocessing/interp_semaphore.py b/pypy/module/_multiprocessing/interp_semaphore.py
--- a/pypy/module/_multiprocessing/interp_semaphore.py
+++ b/pypy/module/_multiprocessing/interp_semaphore.py
@@ -311,7 +311,7 @@
             rgc.add_memory_pressure(SEM_T_SIZE)
         return sem
 
-    def delete_semaphore(space, handle):
+    def delete_semaphore(handle):
         sem_close(handle)
 
     def semlock_acquire(self, space, block, w_timeout):

From noreply at buildbot.pypy.org  Tue Oct 18 08:28:23 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 18 Oct 2011 08:28:23 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Store exception __cause__ differently:
 first in the OperationError, 
Message-ID: <20111018062823.6A017820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48181:a736f7e040d3
Date: 2011-10-18 01:13 +0200
http://bitbucket.org/pypy/pypy/changeset/a736f7e040d3/

Log:	Store exception __cause__ differently: first in the OperationError,
	and move it to exception object when it is normalized.

diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -19,14 +19,16 @@
 
     _w_value = None
     _application_traceback = None
+    w_cause = None
 
-    def __init__(self, w_type, w_value, tb=None):
+    def __init__(self, w_type, w_value, tb=None, w_cause=None):
         if not we_are_translated() and w_type is None:
             from pypy.tool.error import FlowingError
             raise FlowingError(w_value)
         self.setup(w_type)
         self._w_value = w_value
         self._application_traceback = tb
+        self.w_cause = w_cause
 
     def setup(self, w_type):
         self.w_type = w_type
@@ -205,6 +207,8 @@
                         # raise Type, X: assume X is the constructor argument
                         w_value = space.call_function(w_type, w_value)
                     w_type = self._exception_getclass(space, w_value)
+            if self.w_cause:
+                space.setattr(w_value, space.wrap("__cause__"), self.w_cause)
 
         else:
             # the only case left here is (inst, None), from a 'raise inst'.
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -499,8 +499,7 @@
             w_value = space.call_function(w_type)
         else:
             w_type = space.type(w_value)
-        w_value.w_cause = w_cause
-        operror = OperationError(w_type, w_value)
+        operror = OperationError(w_type, w_value, w_cause=w_cause)
         operror.normalize_exception(space)
         w_traceback = space.w_None # XXX with_traceback?
         if not space.full_exceptions or space.is_w(w_traceback, space.w_None):

From noreply at buildbot.pypy.org  Tue Oct 18 08:28:24 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 18 Oct 2011 08:28:24 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Add sys.abiflags. The empty string is good
	enough for now.
Message-ID: <20111018062824.989B8820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48182:0ea3cb4c5772
Date: 2011-10-18 01:21 +0200
http://bitbucket.org/pypy/pypy/changeset/0ea3cb4c5772/

Log:	Add sys.abiflags. The empty string is good enough for now.

diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -42,6 +42,7 @@
         'argv'                  : 'state.get(space).w_argv',
         'py3kwarning'           : 'space.w_False',
         'warnoptions'           : 'state.get(space).w_warnoptions', 
+        'abiflags'              : 'space.wrap("")',
         'builtin_module_names'  : 'space.w_None',
         'pypy_getudir'          : 'state.pypy_getudir',    # not translated
         'pypy_initial_path'     : 'state.pypy_initial_path',

From noreply at buildbot.pypy.org  Tue Oct 18 08:28:25 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 18 Oct 2011 08:28:25 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Unbound methods are gone now :-)
Message-ID: <20111018062825.D229C820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48183:63ea3ff98f1e
Date: 2011-10-18 01:26 +0200
http://bitbucket.org/pypy/pypy/changeset/63ea3ff98f1e/

Log:	Unbound methods are gone now :-)

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -871,15 +871,10 @@
             # start of hack for performance
             from pypy.interpreter.function import Function, Method
             if isinstance(w_func, Method):
-                w_inst = w_func.w_instance
-                if w_inst is not None:
-                    if nargs < 4:
-                        func = w_func.w_function
-                        if isinstance(func, Function):
-                            return func.funccall(w_inst, *args_w)
-                elif args_w and (
-                        self.abstract_isinstance_w(args_w[0], w_func.w_class)):
-                    w_func = w_func.w_function
+                if nargs < 4:
+                    func = w_func.w_function
+                    if isinstance(func, Function):
+                        return func.funccall(w_func.w_instance, *args_w)
 
             if isinstance(w_func, Function):
                 return w_func.funccall(*args_w)
@@ -899,16 +894,10 @@
         if not self.config.objspace.disable_call_speedhacks:
             # start of hack for performance
             if isinstance(w_func, Method):
-                w_inst = w_func.w_instance
-                if w_inst is not None:
-                    w_func = w_func.w_function
-                    # reuse callable stack place for w_inst
-                    frame.settopvalue(w_inst, nargs)
-                    nargs += 1
-                elif nargs > 0 and (
-                    self.abstract_isinstance_w(frame.peekvalue(nargs-1),   #    :-(
-                                               w_func.w_class)):
-                    w_func = w_func.w_function
+                # reuse callable stack place for w_inst
+                frame.settopvalue(w_func.w_instance, nargs)
+                nargs += 1
+                w_func = w_func.w_function
 
             if isinstance(w_func, Function):
                 return w_func.funccall_valuestack(nargs, frame)
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -783,7 +783,6 @@
     __func__ = interp_attrproperty_w('w_function', cls=Method),
     im_self  = interp_attrproperty_w('w_instance', cls=Method),
     __self__ = interp_attrproperty_w('w_instance', cls=Method),
-    im_class = interp_attrproperty_w('w_class', cls=Method),
     __getattribute__ = interp2app(Method.descr_method_getattribute),
     __eq__ = interp2app(Method.descr_method_eq),
     __ne__ = descr_generic_ne,

From noreply at buildbot.pypy.org  Tue Oct 18 08:28:27 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 18 Oct 2011 08:28:27 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Fix the Print part of the read-eval-print
	loop.
Message-ID: <20111018062827.1DCB5820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48184:c7013541688e
Date: 2011-10-18 01:28 +0200
http://bitbucket.org/pypy/pypy/changeset/c7013541688e/

Log:	Fix the Print part of the read-eval-print loop.

diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1345,7 +1345,7 @@
 
         # give to write() an argument which is either a string or a unicode
         # (and let it deals itself with unicode handling)
-        if not isinstance(x, unicode):
+        if not isinstance(x, str):
             x = str(x)
         stream.write(x)
 

From noreply at buildbot.pypy.org  Tue Oct 18 08:28:28 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 18 Oct 2011 08:28:28 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Translation fix
Message-ID: <20111018062828.6E51F820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48185:d0b350aec8bc
Date: 2011-10-18 08:16 +0200
http://bitbucket.org/pypy/pypy/changeset/d0b350aec8bc/

Log:	Translation fix

diff --git a/pypy/module/__builtin__/operation.py b/pypy/module/__builtin__/operation.py
--- a/pypy/module/__builtin__/operation.py
+++ b/pypy/module/__builtin__/operation.py
@@ -38,7 +38,7 @@
     # Note that if w_name is already a string (or a subclass of str),
     # it must be returned unmodified (and not e.g. unwrapped-rewrapped).
     if not space.is_w(space.type(w_name), space.w_text):
-        name = space.text_w(w_name)   # typecheck
+        name = space.str_w(w_name)    # typecheck
         w_name = space.wrap(name)     # rewrap as a real string
     return w_name
 

From noreply at buildbot.pypy.org  Tue Oct 18 08:28:29 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 18 Oct 2011 08:28:29 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: It appears that all opcodes must be
	present in PyFrame, 
Message-ID: <20111018062829.9ED71820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48186:4f2915663f8e
Date: 2011-10-18 08:24 +0200
http://bitbucket.org/pypy/pypy/changeset/4f2915663f8e/

Log:	It appears that all opcodes must be present in PyFrame, even if the
	compiler does not emit them.

diff --git a/pypy/interpreter/nestedscope.py b/pypy/interpreter/nestedscope.py
--- a/pypy/interpreter/nestedscope.py
+++ b/pypy/interpreter/nestedscope.py
@@ -213,6 +213,9 @@
         cell = self.cells[varindex]
         cell.set(w_newvalue)
 
+    def DELETE_DEREF(self, varindex, next_instr):
+        raise NotImplementedError
+
     @jit.unroll_safe
     def MAKE_CLOSURE(self, numdefaults, next_instr):
         w_codeobj = self.popvalue()
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -362,20 +362,13 @@
         self.pushvalue(w_3)
         self.pushvalue(w_2)
 
-    def ROT_FOUR(self, oparg, next_instr):
-        w_1 = self.popvalue()
-        w_2 = self.popvalue()
-        w_3 = self.popvalue()
-        w_4 = self.popvalue()
-        self.pushvalue(w_1)
-        self.pushvalue(w_4)
-        self.pushvalue(w_3)
-        self.pushvalue(w_2)
-
     def DUP_TOP(self, oparg, next_instr):
         w_1 = self.peekvalue()
         self.pushvalue(w_1)
 
+    def DUP_TOP_TWO(self, oparg, next_instr):
+        self.dupvalues(2)
+
     def DUP_TOPX(self, itemcount, next_instr):
         assert 1 <= itemcount <= 5, "limitation of the current interpreter"
         self.dupvalues(itemcount)
@@ -539,6 +532,9 @@
         if plain:
             self.setdictscope(w_locals)
 
+    def POP_EXCEPT(self, oparg, next_instr):
+        raise NotImplementedError
+
     def POP_BLOCK(self, oparg, next_instr):
         block = self.pop_block()
         block.cleanup(self)  # the block knows how to clean up the value stack
@@ -585,6 +581,9 @@
         items = self.space.fixedview_unroll(w_iterable, itemcount)
         self.pushrevvalues(itemcount, items)
 
+    def UNPACK_EX(self, oparg, next_instr):
+        raise NotImplementedError
+
     def STORE_ATTR(self, nameindex, next_instr):
         "obj.attributename = newvalue"
         w_attributename = self.getname_w(nameindex)

From noreply at buildbot.pypy.org  Tue Oct 18 08:28:30 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 18 Oct 2011 08:28:30 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: I don't know why this assert is necessary, 
Message-ID: <20111018062830.CD73F820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48187:c1d89830a817
Date: 2011-10-18 08:23 +0200
http://bitbucket.org/pypy/pypy/changeset/c1d89830a817/

Log:	I don't know why this assert is necessary, there must be something
	wrong in some unrelated location.

diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py
--- a/pypy/objspace/std/stringobject.py
+++ b/pypy/objspace/std/stringobject.py
@@ -22,6 +22,7 @@
     _immutable_fields_ = ['_value']
 
     def __init__(w_self, str):
+        assert str is not None
         w_self._value = str
 
     def __repr__(w_self):

From noreply at buildbot.pypy.org  Tue Oct 18 08:28:32 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Tue, 18 Oct 2011 08:28:32 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: This is not necessary and breaks
	translation :-(
Message-ID: <20111018062832.07595820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48188:813961afa8c7
Date: 2011-10-18 08:27 +0200
http://bitbucket.org/pypy/pypy/changeset/813961afa8c7/

Log:	This is not necessary and breaks translation :-(

diff --git a/pypy/interpreter/pyparser/parsestring.py b/pypy/interpreter/pyparser/parsestring.py
--- a/pypy/interpreter/pyparser/parsestring.py
+++ b/pypy/interpreter/pyparser/parsestring.py
@@ -13,8 +13,6 @@
     rawmode = False
     unicode = True
 
-    assert isinstance(s, str)
-
     # string decoration handling
     o = ord(quote)
     isalpha = (o>=97 and o<=122) or (o>=65 and o<=90)

From noreply at buildbot.pypy.org  Tue Oct 18 09:11:02 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Oct 2011 09:11:02 +0200 (CEST)
Subject: [pypy-commit] pypy default: Baaah.
Message-ID: <20111018071102.D77A3820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48189:6e9cbb110fe5
Date: 2011-10-18 09:10 +0200
http://bitbucket.org/pypy/pypy/changeset/6e9cbb110fe5/

Log:	Baaah.

diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -56,8 +56,8 @@
     binaries = [(pypy_c, rename_pypy_c)]
     #
     if sys.platform == 'win32':
-        # Can't rename a DLL: it is always called 'libpypy_c.dll'
-        for extra in ['libpypy_c.dll',
+        # Can't rename a DLL: it is always called 'libpypy-c.dll'
+        for extra in ['libpypy-c.dll',
                       'libexpat.dll', 'sqlite3.dll', 'msvcr90.dll']:
             p = pypy_c.dirpath().join(extra)
             if not p.check():

From noreply at buildbot.pypy.org  Tue Oct 18 11:15:51 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 18 Oct 2011 11:15:51 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: list of release blockers for 1.7
Message-ID: <20111018091551.AB828820D7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: extradoc
Changeset: r3931:ec5f21f3cc50
Date: 2011-10-18 11:15 +0200
http://bitbucket.org/pypy/extradoc/changeset/ec5f21f3cc50/

Log:	list of release blockers for 1.7

diff --git a/planning/release-blockers-1.7.rst b/planning/release-blockers-1.7.rst
new file mode 100644
--- /dev/null
+++ b/planning/release-blockers-1.7.rst
@@ -0,0 +1,11 @@
+
+* [fijal] I might want finish the json work. I'll look into it this week
+
+* sobel demo, https://bugs.pypy.org/issue911
+
+* django tests (?) https://bugs.pypy.org/issue908
+
+* twisted threading tests (?) https://bugs.pypy.org/issue645
+
+* https://bugs.pypy.org/issue889
+

From noreply at buildbot.pypy.org  Tue Oct 18 11:17:46 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Oct 2011 11:17:46 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: Add what it is
Message-ID: <20111018091746.75FA8820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r3932:7ead70d33954
Date: 2011-10-18 11:17 +0200
http://bitbucket.org/pypy/extradoc/changeset/7ead70d33954/

Log:	Add what it is

diff --git a/planning/release-blockers-1.7.rst b/planning/release-blockers-1.7.rst
--- a/planning/release-blockers-1.7.rst
+++ b/planning/release-blockers-1.7.rst
@@ -7,5 +7,5 @@
 
 * twisted threading tests (?) https://bugs.pypy.org/issue645
 
-* https://bugs.pypy.org/issue889
+* cpyext header files: https://bugs.pypy.org/issue889
 

From noreply at buildbot.pypy.org  Tue Oct 18 12:07:40 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Oct 2011 12:07:40 +0200 (CEST)
Subject: [pypy-commit] pypy default: Add debugging support for @elidable
	decorators.
Message-ID: <20111018100740.58D40820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48190:81e9199c50b9
Date: 2011-10-18 10:21 +0200
http://bitbucket.org/pypy/pypy/changeset/81e9199c50b9/

Log:	Add debugging support for @elidable decorators. Disabled by default
	because it keeps all arguments ever passed alive.

diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -8,6 +8,8 @@
 from pypy.rpython.extregistry import ExtRegistryEntry
 from pypy.tool.sourcetools import func_with_new_name
 
+DEBUG_ELIDABLE_FUNCTIONS = False
+
 
 def elidable(func):
     """ Decorate a function as "trace-elidable". This means precisely that:
@@ -24,6 +26,18 @@
     If a particular call to this function ends up raising an exception, then it
     is handled like a normal function call (this decorator is ignored).
     """
+    if DEBUG_ELIDABLE_FUNCTIONS:
+        cache = {}
+        oldfunc = func
+        def func(*args):
+            result = oldfunc(*args)    # if it raises, no caching
+            try:
+                oldresult = cache.setdefault(args, result)
+            except TypeError:
+                pass           # unhashable args
+            else:
+                assert oldresult == result
+            return result
     func._elidable_function_ = True
     return func
 

From noreply at buildbot.pypy.org  Tue Oct 18 12:07:41 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Oct 2011 12:07:41 +0200 (CEST)
Subject: [pypy-commit] pypy default: Uh,
 this seems to fix pypy.objspace.test.test_binop_overriding.
Message-ID: <20111018100741.81C97820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48191:8ba326c260b0
Date: 2011-10-18 12:05 +0200
http://bitbucket.org/pypy/pypy/changeset/8ba326c260b0/

Log:	Uh, this seems to fix pypy.objspace.test.test_binop_overriding. I
	will try to write a unit test and comments for it.

diff --git a/pypy/jit/metainterp/quasiimmut.py b/pypy/jit/metainterp/quasiimmut.py
--- a/pypy/jit/metainterp/quasiimmut.py
+++ b/pypy/jit/metainterp/quasiimmut.py
@@ -74,12 +74,8 @@
         self.looptokens_wrefs.append(wref_looptoken)
 
     def compress_looptokens_list(self):
-        newlist = []
-        for wref in self.looptokens_wrefs:
-            looptoken = wref()
-            if looptoken is not None and not looptoken.invalidated:
-                newlist.append(wref)
-        self.looptokens_wrefs = newlist
+        self.looptokens_wref = [wref for wref in self.looptokens_wrefs
+                                     if wref() is not None]
         self.compress_limit = (len(self.looptokens_wrefs) + 15) * 2
 
     def invalidate(self):
@@ -90,7 +86,7 @@
         self.looptokens_wrefs = []
         for wref in wrefs:
             looptoken = wref()
-            if looptoken is not None and not looptoken.invalidated:
+            if looptoken is not None:
                 looptoken.invalidated = True
                 self.cpu.invalidate_loop(looptoken)
                 if not we_are_translated():

From noreply at buildbot.pypy.org  Tue Oct 18 12:54:01 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Oct 2011 12:54:01 +0200 (CEST)
Subject: [pypy-commit] pypy default: Add the comments already. The test is
	harder :-(
Message-ID: <20111018105401.4A1D7820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48192:1ae56b2043d4
Date: 2011-10-18 12:53 +0200
http://bitbucket.org/pypy/pypy/changeset/1ae56b2043d4/

Log:	Add the comments already. The test is harder :-(

diff --git a/pypy/jit/metainterp/quasiimmut.py b/pypy/jit/metainterp/quasiimmut.py
--- a/pypy/jit/metainterp/quasiimmut.py
+++ b/pypy/jit/metainterp/quasiimmut.py
@@ -76,6 +76,8 @@
     def compress_looptokens_list(self):
         self.looptokens_wref = [wref for wref in self.looptokens_wrefs
                                      if wref() is not None]
+        # NB. we must keep around the looptoken_wrefs that are
+        # already invalidated; see below
         self.compress_limit = (len(self.looptokens_wrefs) + 15) * 2
 
     def invalidate(self):
@@ -89,6 +91,11 @@
             if looptoken is not None:
                 looptoken.invalidated = True
                 self.cpu.invalidate_loop(looptoken)
+                # NB. we must call cpu.invalidate_loop() even if
+                # looptoken.invalidated was already set to True.
+                # It's possible to invalidate several times the
+                # same looptoken; see comments in jit.backend.model
+                # in invalidate_loop().
                 if not we_are_translated():
                     self.cpu.stats.invalidated_token_numbers.add(
                         looptoken.number)

From noreply at buildbot.pypy.org  Tue Oct 18 14:02:14 2011
From: noreply at buildbot.pypy.org (cfbolz)
Date: Tue, 18 Oct 2011 14:02:14 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix failing tests
Message-ID: <20111018120214.48C07820D7@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r48193:31c39356de7a
Date: 2011-10-18 09:54 +0200
http://bitbucket.org/pypy/pypy/changeset/31c39356de7a/

Log:	fix failing tests

diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -223,6 +223,8 @@
                 continue
             if 'FLOAT' in op:
                 continue
+            if 'CAST' in op:
+                continue
             argtypes, restype = TYPES[op.lower()]
             args = []
             for argtype in argtypes:
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -212,6 +212,8 @@
                 continue
             if 'FLOAT' in op:
                 continue
+            if 'CAST' in op:
+                continue
             argtypes, restype = TYPES[op.lower()]
             args = []
             for argtype in argtypes:

From noreply at buildbot.pypy.org  Tue Oct 18 14:02:15 2011
From: noreply at buildbot.pypy.org (cfbolz)
Date: Tue, 18 Oct 2011 14:02:15 +0200 (CEST)
Subject: [pypy-commit] pypy default: makes no sense to synthesize the
 reverse ops for operations that are removed anyway.
Message-ID: <20111018120215.73459820D7@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r48194:25b6ee1b8082
Date: 2011-10-18 09:55 +0200
http://bitbucket.org/pypy/pypy/changeset/25b6ee1b8082/

Log:	makes no sense to synthesize the reverse ops for operations that are
	removed anyway.

diff --git a/pypy/jit/metainterp/optimizeopt/rewrite.py b/pypy/jit/metainterp/optimizeopt/rewrite.py
--- a/pypy/jit/metainterp/optimizeopt/rewrite.py
+++ b/pypy/jit/metainterp/optimizeopt/rewrite.py
@@ -106,10 +106,9 @@
             self.make_equal_to(op.result, v1)
         else:
             self.emit_operation(op)
-
-        # Synthesize the reverse ops for optimize_default to reuse
-        self.pure(rop.INT_ADD, [op.result, op.getarg(1)], op.getarg(0))
-        self.pure(rop.INT_SUB, [op.getarg(0), op.result], op.getarg(1))
+            # Synthesize the reverse ops for optimize_default to reuse
+            self.pure(rop.INT_ADD, [op.result, op.getarg(1)], op.getarg(0))
+            self.pure(rop.INT_SUB, [op.getarg(0), op.result], op.getarg(1))
 
     def optimize_INT_ADD(self, op):
         v1 = self.getvalue(op.getarg(0))
@@ -122,10 +121,9 @@
             self.make_equal_to(op.result, v1)
         else:
             self.emit_operation(op)
-
-        # Synthesize the reverse op for optimize_default to reuse
-        self.pure(rop.INT_SUB, [op.result, op.getarg(1)], op.getarg(0))
-        self.pure(rop.INT_SUB, [op.result, op.getarg(0)], op.getarg(1))
+            # Synthesize the reverse op for optimize_default to reuse
+            self.pure(rop.INT_SUB, [op.result, op.getarg(1)], op.getarg(0))
+            self.pure(rop.INT_SUB, [op.result, op.getarg(0)], op.getarg(1))
 
     def optimize_INT_MUL(self, op):
         v1 = self.getvalue(op.getarg(0))

From noreply at buildbot.pypy.org  Tue Oct 18 14:02:16 2011
From: noreply at buildbot.pypy.org (cfbolz)
Date: Tue, 18 Oct 2011 14:02:16 +0200 (CEST)
Subject: [pypy-commit] pypy default: add reverse operations for cast from
	and to int pointer
Message-ID: <20111018120216.A1BD9820D7@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r48195:98e1c25e7b93
Date: 2011-10-18 10:01 +0200
http://bitbucket.org/pypy/pypy/changeset/98e1c25e7b93/

Log:	add reverse operations for cast from and to int pointer

diff --git a/pypy/jit/metainterp/optimizeopt/rewrite.py b/pypy/jit/metainterp/optimizeopt/rewrite.py
--- a/pypy/jit/metainterp/optimizeopt/rewrite.py
+++ b/pypy/jit/metainterp/optimizeopt/rewrite.py
@@ -460,6 +460,14 @@
         self.optimizer.opaque_pointers[value] = True
         self.make_equal_to(op.result, value)
 
+    def optimize_CAST_PTR_TO_INT(self, op):
+        self.pure(rop.CAST_INT_TO_PTR, [op.result], op.getarg(0))
+        self.emit_operation(op)
+
+    def optimize_CAST_INT_TO_PTR(self, op):
+        self.pure(rop.CAST_PTR_TO_INT, [op.result], op.getarg(0))
+        self.emit_operation(op)
+
 dispatch_opt = make_dispatcher_method(OptRewrite, 'optimize_',
         default=OptRewrite.emit_operation)
 optimize_guards = _findall(OptRewrite, 'optimize_', 'GUARD')
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -236,6 +236,30 @@
             """ % expected_value
             self.optimize_loop(ops, expected)
 
+    def test_reverse_of_cast(self):
+        ops = """
+        [i0]
+        p0 = cast_int_to_ptr(i0)
+        i1 = cast_ptr_to_int(p0)
+        jump(i1)
+        """
+        expected = """
+        [i0]
+        jump(i0)
+        """
+        self.optimize_loop(ops, expected)
+        ops = """
+        [p0]
+        i1 = cast_ptr_to_int(p0)
+        p1 = cast_int_to_ptr(i1)
+        jump(p1)
+        """
+        expected = """
+        [p0]
+        jump(p0)
+        """
+        self.optimize_loop(ops, expected)
+
     # ----------
 
     def test_remove_guard_class_1(self):

From noreply at buildbot.pypy.org  Tue Oct 18 14:02:17 2011
From: noreply at buildbot.pypy.org (cfbolz)
Date: Tue, 18 Oct 2011 14:02:17 +0200 (CEST)
Subject: [pypy-commit] pypy default: slightly pointless micro-optimization,
 but I found the adjective "quick" in
Message-ID: <20111018120217.CA498820D7@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r48196:ba7afb49a544
Date: 2011-10-18 10:48 +0200
http://bitbucket.org/pypy/pypy/changeset/ba7afb49a544/

Log:	slightly pointless micro-optimization, but I found the adjective
	"quick" in this comment too ironic

diff --git a/pypy/jit/metainterp/optimizeopt/rewrite.py b/pypy/jit/metainterp/optimizeopt/rewrite.py
--- a/pypy/jit/metainterp/optimizeopt/rewrite.py
+++ b/pypy/jit/metainterp/optimizeopt/rewrite.py
@@ -139,13 +139,13 @@
             self.make_constant_int(op.result, 0)
         else:
             for lhs, rhs in [(v1, v2), (v2, v1)]:
-                # x & (x -1) == 0 is a quick test for power of 2
-                if (lhs.is_constant() and
-                    (lhs.box.getint() & (lhs.box.getint() - 1)) == 0):
-                    new_rhs = ConstInt(highest_bit(lhs.box.getint()))
-                    op = op.copy_and_change(rop.INT_LSHIFT, args=[rhs.box, new_rhs])
-                    break
-
+                if lhs.is_constant():
+                    x = lhs.box.getint()
+                    # x & (x - 1) == 0 is a quick test for power of 2
+                    if x & (x - 1) == 0:
+                        new_rhs = ConstInt(highest_bit(lhs.box.getint()))
+                        op = op.copy_and_change(rop.INT_LSHIFT, args=[rhs.box, new_rhs])
+                        break
             self.emit_operation(op)
 
     def optimize_UINT_FLOORDIV(self, op):

From noreply at buildbot.pypy.org  Tue Oct 18 14:10:20 2011
From: noreply at buildbot.pypy.org (cfbolz)
Date: Tue, 18 Oct 2011 14:10:20 +0200 (CEST)
Subject: [pypy-commit] pypy default: revert 31c39356de7a. armin had fixed it
	in the meantime.
Message-ID: <20111018121020.14AA2820D7@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r48197:69cffe91b5f4
Date: 2011-10-18 14:09 +0200
http://bitbucket.org/pypy/pypy/changeset/69cffe91b5f4/

Log:	revert 31c39356de7a. armin had fixed it in the meantime.

diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -223,8 +223,6 @@
                 continue
             if 'FLOAT' in op:
                 continue
-            if 'CAST' in op:
-                continue
             argtypes, restype = TYPES[op.lower()]
             args = []
             for argtype in argtypes:
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -212,8 +212,6 @@
                 continue
             if 'FLOAT' in op:
                 continue
-            if 'CAST' in op:
-                continue
             argtypes, restype = TYPES[op.lower()]
             args = []
             for argtype in argtypes:

From noreply at buildbot.pypy.org  Tue Oct 18 14:17:03 2011
From: noreply at buildbot.pypy.org (hager)
Date: Tue, 18 Oct 2011 14:17:03 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: Implemented INT_FLOORDIV.
Message-ID: <20111018121703.6F29E820D7@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r48198:14fd9b6e54bb
Date: 2011-10-17 20:16 +0200
http://bitbucket.org/pypy/pypy/changeset/14fd9b6e54bb/

Log:	Implemented INT_FLOORDIV.

diff --git a/pypy/jit/backend/ppc/ppcgen/opassembler.py b/pypy/jit/backend/ppc/ppcgen/opassembler.py
--- a/pypy/jit/backend/ppc/ppcgen/opassembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/opassembler.py
@@ -45,6 +45,17 @@
         reg1, reg2, res = arglocs
         self.mc.mullw(res.value, reg1.value, reg2.value)
 
+    def emit_int_floordiv(self, op, arglocs, regalloc):
+        l0, l1, res = arglocs
+        if l0.is_imm():
+            self.mc.load_imm(r.r0, l0.value)
+            self.mc.divw(res.value, r.r0.value, l1.value)
+        elif l1.is_imm():
+            self.mc.load_imm(r.r0, l1.value)
+            self.mc.divw(res.value, l0.value, r.r0.value)
+        else:
+            self.mc.divw(res.value, l0.value, l1.value)
+
     emit_int_le = gen_emit_cmp_op(c.LE)   
 
     def _emit_guard(self, op, arglocs, fcond, save_exc=False,
diff --git a/pypy/jit/backend/ppc/ppcgen/regalloc.py b/pypy/jit/backend/ppc/ppcgen/regalloc.py
--- a/pypy/jit/backend/ppc/ppcgen/regalloc.py
+++ b/pypy/jit/backend/ppc/ppcgen/regalloc.py
@@ -5,7 +5,8 @@
 from pypy.jit.backend.ppc.ppcgen.jump import remap_frame_layout_mixed
 from pypy.jit.backend.ppc.ppcgen.locations import imm
 from pypy.jit.backend.ppc.ppcgen.helper.regalloc import (_check_imm_arg, 
-                                                         prepare_cmp_op)
+                                                         prepare_cmp_op,
+                                                         prepare_op_by_helper_call)
 from pypy.jit.metainterp.history import (INT, REF, FLOAT, Const, ConstInt, 
                                          ConstPtr, LoopToken)
 from pypy.jit.metainterp.resoperation import rop
@@ -221,6 +222,29 @@
         res = self.force_allocate_reg(op.result)
         return locs + [res]
 
+    def prepare_int_floordiv(self, op):
+        boxes = list(op.getarglist())
+        b0, b1 = boxes
+        imm_b0 = _check_imm_arg(b0)
+        imm_b1 = _check_imm_arg(b1)
+        if not imm_b0 and imm_b1:
+            l0, box = self._ensure_value_is_boxed(b0, boxes)
+            l1 = self.make_sure_var_in_reg(b1, [b0])
+            boxes.append(box)
+        elif imm_b0 and not imm_b1:
+            l0 = self.make_sure_var_in_reg(b0)
+            l1, box = self._ensure_value_is_boxed(b1, boxes)
+            boxes.append(box)
+        else:
+            l0, box = self._ensure_value_is_boxed(b0, boxes)
+            boxes.append(box)
+            l1, box = self._ensure_value_is_boxed(b1, boxes)
+            boxes.append(box)
+        locs = [l0, l1]
+        self.possibly_free_vars(boxes)
+        res = self.force_allocate_reg(op.result)
+        return locs + [res]
+
     def prepare_int_mul(self, op):
         boxes = list(op.getarglist())
         b0, b1 = boxes

From noreply at buildbot.pypy.org  Tue Oct 18 14:17:04 2011
From: noreply at buildbot.pypy.org (hager)
Date: Tue, 18 Oct 2011 14:17:04 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: Beautified code for
 prepare_int_sub and prepare_int_floordiv.
Message-ID: <20111018121704.97EE6820D7@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r48199:ee9b75f555bb
Date: 2011-10-18 12:28 +0200
http://bitbucket.org/pypy/pypy/changeset/ee9b75f555bb/

Log:	Beautified code for prepare_int_sub and prepare_int_floordiv.

diff --git a/pypy/jit/backend/ppc/ppcgen/regalloc.py b/pypy/jit/backend/ppc/ppcgen/regalloc.py
--- a/pypy/jit/backend/ppc/ppcgen/regalloc.py
+++ b/pypy/jit/backend/ppc/ppcgen/regalloc.py
@@ -5,8 +5,7 @@
 from pypy.jit.backend.ppc.ppcgen.jump import remap_frame_layout_mixed
 from pypy.jit.backend.ppc.ppcgen.locations import imm
 from pypy.jit.backend.ppc.ppcgen.helper.regalloc import (_check_imm_arg, 
-                                                         prepare_cmp_op,
-                                                         prepare_op_by_helper_call)
+                                                         prepare_cmp_op)
 from pypy.jit.metainterp.history import (INT, REF, FLOAT, Const, ConstInt, 
                                          ConstPtr, LoopToken)
 from pypy.jit.metainterp.resoperation import rop
@@ -200,50 +199,10 @@
         return locs + [res]
 
     def prepare_int_sub(self, op):
-        boxes = list(op.getarglist())
-        b0, b1 = boxes
-        imm_b0 = _check_imm_arg(b0)
-        imm_b1 = _check_imm_arg(b1)
-        if not imm_b0 and imm_b1:
-            l0, box = self._ensure_value_is_boxed(b0, boxes)
-            l1 = self.make_sure_var_in_reg(b1, [b0])
-            boxes.append(box)
-        elif imm_b0 and not imm_b1:
-            l0 = self.make_sure_var_in_reg(b0)
-            l1, box = self._ensure_value_is_boxed(b1, boxes)
-            boxes.append(box)
-        else:
-            l0, box = self._ensure_value_is_boxed(b0, boxes)
-            boxes.append(box)
-            l1, box = self._ensure_value_is_boxed(b1, boxes)
-            boxes.append(box)
-        locs = [l0, l1]
-        self.possibly_free_vars(boxes)
-        res = self.force_allocate_reg(op.result)
-        return locs + [res]
+        return self.prepare_int_add(op)
 
     def prepare_int_floordiv(self, op):
-        boxes = list(op.getarglist())
-        b0, b1 = boxes
-        imm_b0 = _check_imm_arg(b0)
-        imm_b1 = _check_imm_arg(b1)
-        if not imm_b0 and imm_b1:
-            l0, box = self._ensure_value_is_boxed(b0, boxes)
-            l1 = self.make_sure_var_in_reg(b1, [b0])
-            boxes.append(box)
-        elif imm_b0 and not imm_b1:
-            l0 = self.make_sure_var_in_reg(b0)
-            l1, box = self._ensure_value_is_boxed(b1, boxes)
-            boxes.append(box)
-        else:
-            l0, box = self._ensure_value_is_boxed(b0, boxes)
-            boxes.append(box)
-            l1, box = self._ensure_value_is_boxed(b1, boxes)
-            boxes.append(box)
-        locs = [l0, l1]
-        self.possibly_free_vars(boxes)
-        res = self.force_allocate_reg(op.result)
-        return locs + [res]
+        return self.prepare_int_add(op)
 
     def prepare_int_mul(self, op):
         boxes = list(op.getarglist())

From noreply at buildbot.pypy.org  Tue Oct 18 14:17:05 2011
From: noreply at buildbot.pypy.org (hager)
Date: Tue, 18 Oct 2011 14:17:05 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: Implement emit_int_floordiv for
	PPC64.
Message-ID: <20111018121705.C07BB820D7@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r48200:3f5bfca0c466
Date: 2011-10-18 12:31 +0200
http://bitbucket.org/pypy/pypy/changeset/3f5bfca0c466/

Log:	Implement emit_int_floordiv for PPC64.

diff --git a/pypy/jit/backend/ppc/ppcgen/opassembler.py b/pypy/jit/backend/ppc/ppcgen/opassembler.py
--- a/pypy/jit/backend/ppc/ppcgen/opassembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/opassembler.py
@@ -47,12 +47,17 @@
 
     def emit_int_floordiv(self, op, arglocs, regalloc):
         l0, l1, res = arglocs
+        if IS_PPC_32:
+            div = self.mc.divw
+        else:
+            div = self.mc.divd
+
         if l0.is_imm():
             self.mc.load_imm(r.r0, l0.value)
-            self.mc.divw(res.value, r.r0.value, l1.value)
+            div(res.value, r.r0.value, l1.value)
         elif l1.is_imm():
             self.mc.load_imm(r.r0, l1.value)
-            self.mc.divw(res.value, l0.value, r.r0.value)
+            div(res.value, l0.value, r.r0.value)
         else:
             self.mc.divw(res.value, l0.value, l1.value)
 

From noreply at buildbot.pypy.org  Tue Oct 18 14:17:06 2011
From: noreply at buildbot.pypy.org (hager)
Date: Tue, 18 Oct 2011 14:17:06 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: Implemented INT_MOD.
Message-ID: <20111018121706.EAEC0820D7@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r48201:62d9032ce3fa
Date: 2011-10-18 12:32 +0200
http://bitbucket.org/pypy/pypy/changeset/62d9032ce3fa/

Log:	Implemented INT_MOD.

diff --git a/pypy/jit/backend/ppc/ppcgen/opassembler.py b/pypy/jit/backend/ppc/ppcgen/opassembler.py
--- a/pypy/jit/backend/ppc/ppcgen/opassembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/opassembler.py
@@ -59,7 +59,18 @@
             self.mc.load_imm(r.r0, l1.value)
             div(res.value, l0.value, r.r0.value)
         else:
-            self.mc.divw(res.value, l0.value, l1.value)
+            div(res.value, l0.value, l1.value)
+
+    def emit_int_mod(self, op, arglocs, regalloc):
+        l0, l1, res = arglocs
+        if IS_PPC_32:
+            self.mc.divw(r.r0.value, l0.value, l1.value)
+            self.mc.mullw(r.r0.value, r.r0.value, l1.value)
+        else:
+            self.mc.divd(r.r0.value, l0.value, l1.value)
+            self.mc.mulld(r.r0.value, r.r0.value, l1.value)
+        self.mc.subf(r.r0.value, r.r0.value, l0.value)
+        self.mc.mr(res.value, r.r0.value)
 
     emit_int_le = gen_emit_cmp_op(c.LE)   
 
diff --git a/pypy/jit/backend/ppc/ppcgen/regalloc.py b/pypy/jit/backend/ppc/ppcgen/regalloc.py
--- a/pypy/jit/backend/ppc/ppcgen/regalloc.py
+++ b/pypy/jit/backend/ppc/ppcgen/regalloc.py
@@ -219,6 +219,9 @@
         self.possibly_free_var(op.result)
         return [reg1, reg2, res]
 
+    def prepare_int_mod(self, op):
+        return self.prepare_int_mul(op)
+
     def prepare_finish(self, op):
         args = [locations.imm(self.frame_manager.frame_depth)]
         for i in range(op.numargs()):

From noreply at buildbot.pypy.org  Tue Oct 18 14:17:08 2011
From: noreply at buildbot.pypy.org (hager)
Date: Tue, 18 Oct 2011 14:17:08 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: merge with PPC64 code
Message-ID: <20111018121708.29895820D7@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r48202:2bf69d152817
Date: 2011-10-18 14:16 +0200
http://bitbucket.org/pypy/pypy/changeset/2bf69d152817/

Log:	merge with PPC64 code

diff --git a/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py b/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py
--- a/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py
@@ -109,7 +109,7 @@
         if IS_PPC_32:
             self.mc.stwx(source_reg.value, 0, 0)
         else:
-            self.mc.std(source_reg.value, 0, 0)
+            self.mc.stdx(source_reg.value, 0, 0)
 
     def _save_nonvolatiles(self):
         for i, reg in enumerate(NONVOLATILES):
@@ -161,17 +161,21 @@
             # save r31 at the bottom of the stack frame
             self.mc.stw(r.SPP.value, r.SP.value, WORD)
         else:
-            self.mc.stdu(1, 1, -frame_depth)
-            self.mc.mflr(0)
-            self.mc.std(0, 1, frame_depth + 4)
+            self.mc.stdu(r.SP.value, r.SP.value, -frame_depth)
+            self.mc.mflr(r.r0.value)
+            self.mc.std(r.r0.value, r.SP.value, frame_depth + 2 * WORD)
         offset = GPR_SAVE_AREA + WORD
         # compute spilling pointer (SPP)
         self.mc.addi(r.SPP.value, r.SP.value, frame_depth - offset)
         self._save_nonvolatiles()
         # save r31, use r30 as scratch register
         # this is safe because r30 has been saved already
-        self.mc.lwz(r.r30.value, r.SP.value, WORD)
-        self.mc.stw(r.r30.value, r.SPP.value, WORD * len(NONVOLATILES))
+        if IS_PPC_32:
+            self.mc.lwz(r.r30.value, r.SP.value, WORD)
+            self.mc.stw(r.r30.value, r.SPP.value, WORD * len(NONVOLATILES))
+        else:
+            self.mc.ld(r.r30.value, r.SP.value, WORD)
+            self.mc.std(r.r30.value, r.SPP.value, WORD * len(NONVOLATILES))
         # branch to loop code
         curpos = self.mc.currpos()
         offset = target_pos - curpos

From noreply at buildbot.pypy.org  Tue Oct 18 14:43:59 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Oct 2011 14:43:59 +0200 (CEST)
Subject: [pypy-commit] pypy default: Waaaaaaa. Thanks gedd.
Message-ID: <20111018124359.DF454820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48203:668adc4d1bc6
Date: 2011-10-18 14:43 +0200
http://bitbucket.org/pypy/pypy/changeset/668adc4d1bc6/

Log:	Waaaaaaa. Thanks gedd.

diff --git a/pypy/jit/metainterp/quasiimmut.py b/pypy/jit/metainterp/quasiimmut.py
--- a/pypy/jit/metainterp/quasiimmut.py
+++ b/pypy/jit/metainterp/quasiimmut.py
@@ -74,8 +74,8 @@
         self.looptokens_wrefs.append(wref_looptoken)
 
     def compress_looptokens_list(self):
-        self.looptokens_wref = [wref for wref in self.looptokens_wrefs
-                                     if wref() is not None]
+        self.looptokens_wrefs = [wref for wref in self.looptokens_wrefs
+                                      if wref() is not None]
         # NB. we must keep around the looptoken_wrefs that are
         # already invalidated; see below
         self.compress_limit = (len(self.looptokens_wrefs) + 15) * 2

From noreply at buildbot.pypy.org  Tue Oct 18 14:46:26 2011
From: noreply at buildbot.pypy.org (cfbolz)
Date: Tue, 18 Oct 2011 14:46:26 +0200 (CEST)
Subject: [pypy-commit] pypy default: quite obscure change: add the linked
 libraries at the end of the command line.
Message-ID: <20111018124626.DDD04820D7@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r48204:491120610ab3
Date: 2011-10-18 14:43 +0200
http://bitbucket.org/pypy/pypy/changeset/491120610ab3/

Log:	quite obscure change: add the linked libraries at the end of the
	command line. this seems required on Ubuntu 11.10 to run "make"
	successfully.

diff --git a/pypy/translator/platform/posix.py b/pypy/translator/platform/posix.py
--- a/pypy/translator/platform/posix.py
+++ b/pypy/translator/platform/posix.py
@@ -157,7 +157,7 @@
 
         rules = [
             ('all', '$(DEFAULT_TARGET)', []),
-            ('$(TARGET)', '$(OBJECTS)', '$(CC_LINK) $(LDFLAGS) $(LDFLAGSEXTRA) -o $@ $(OBJECTS) $(LIBDIRS) $(LIBS) $(LINKFILES)'),
+            ('$(TARGET)', '$(OBJECTS)', '$(CC_LINK) $(LDFLAGSEXTRA) -o $@ $(OBJECTS) $(LIBDIRS) $(LIBS) $(LINKFILES) $(LDFLAGS)'),
             ('%.o', '%.c', '$(CC) $(CFLAGS) $(CFLAGSEXTRA) -o $@ -c $< $(INCLUDEDIRS)'),
             ]
 

From noreply at buildbot.pypy.org  Tue Oct 18 14:46:28 2011
From: noreply at buildbot.pypy.org (cfbolz)
Date: Tue, 18 Oct 2011 14:46:28 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20111018124628.10F92820D7@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r48205:a6f3750521c0
Date: 2011-10-18 14:46 +0200
http://bitbucket.org/pypy/pypy/changeset/a6f3750521c0/

Log:	merge

diff --git a/pypy/jit/metainterp/quasiimmut.py b/pypy/jit/metainterp/quasiimmut.py
--- a/pypy/jit/metainterp/quasiimmut.py
+++ b/pypy/jit/metainterp/quasiimmut.py
@@ -74,8 +74,8 @@
         self.looptokens_wrefs.append(wref_looptoken)
 
     def compress_looptokens_list(self):
-        self.looptokens_wref = [wref for wref in self.looptokens_wrefs
-                                     if wref() is not None]
+        self.looptokens_wrefs = [wref for wref in self.looptokens_wrefs
+                                      if wref() is not None]
         # NB. we must keep around the looptoken_wrefs that are
         # already invalidated; see below
         self.compress_limit = (len(self.looptokens_wrefs) + 15) * 2

From noreply at buildbot.pypy.org  Tue Oct 18 15:24:10 2011
From: noreply at buildbot.pypy.org (hager)
Date: Tue, 18 Oct 2011 15:24:10 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: Implemented INT_AND, INT_OR,
	INT_XOR.
Message-ID: <20111018132410.BB04B820D7@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r48206:b7a4abd18923
Date: 2011-10-18 15:23 +0200
http://bitbucket.org/pypy/pypy/changeset/b7a4abd18923/

Log:	Implemented INT_AND, INT_OR, INT_XOR.

diff --git a/pypy/jit/backend/ppc/ppcgen/opassembler.py b/pypy/jit/backend/ppc/ppcgen/opassembler.py
--- a/pypy/jit/backend/ppc/ppcgen/opassembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/opassembler.py
@@ -72,6 +72,18 @@
         self.mc.subf(r.r0.value, r.r0.value, l0.value)
         self.mc.mr(res.value, r.r0.value)
 
+    def emit_int_and(self, op, arglocs, regalloc):
+        l0, l1, res = arglocs
+        self.mc.and_(res.value, l0.value, l1.value)
+
+    def emit_int_or(self, op, arglocs, regalloc):
+        l0, l1, res = arglocs
+        self.mc.or_(res.value, l0.value, l1.value)
+
+    def emit_int_xor(self, op, arglocs, regalloc):
+        l0, l1, res = arglocs
+        self.mc.xor(res.value, l0.value, l1.value)
+
     emit_int_le = gen_emit_cmp_op(c.LE)   
 
     def _emit_guard(self, op, arglocs, fcond, save_exc=False,
diff --git a/pypy/jit/backend/ppc/ppcgen/regalloc.py b/pypy/jit/backend/ppc/ppcgen/regalloc.py
--- a/pypy/jit/backend/ppc/ppcgen/regalloc.py
+++ b/pypy/jit/backend/ppc/ppcgen/regalloc.py
@@ -222,6 +222,15 @@
     def prepare_int_mod(self, op):
         return self.prepare_int_mul(op)
 
+    def prepare_int_and(self, op):
+        return self.prepare_int_mul(op)
+
+    def prepare_int_or(self, op):
+        return self.prepare_int_mul(op)
+
+    def prepare_int_xor(self, op):
+        return self.prepare_int_mul(op)
+
     def prepare_finish(self, op):
         args = [locations.imm(self.frame_manager.frame_depth)]
         for i in range(op.numargs()):

From noreply at buildbot.pypy.org  Tue Oct 18 15:36:10 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue, 18 Oct 2011 15:36:10 +0200 (CEST)
Subject: [pypy-commit] pypy no-force-guard-lazy-set: close branch,
 it wasn't a one liner, I don't care that much
Message-ID: <20111018133610.F3A2D820D7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: no-force-guard-lazy-set
Changeset: r48207:6d7644a6fd38
Date: 2011-10-18 09:35 -0400
http://bitbucket.org/pypy/pypy/changeset/6d7644a6fd38/

Log:	close branch, it wasn't a one liner, I don't care that much


From noreply at buildbot.pypy.org  Tue Oct 18 15:49:53 2011
From: noreply at buildbot.pypy.org (hager)
Date: Tue, 18 Oct 2011 15:49:53 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: Implemented further binary int
	operations.
Message-ID: <20111018134953.8BCB2820D7@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r48208:2e644469f894
Date: 2011-10-18 15:49 +0200
http://bitbucket.org/pypy/pypy/changeset/2e644469f894/

Log:	Implemented further binary int operations.

diff --git a/pypy/jit/backend/ppc/ppcgen/opassembler.py b/pypy/jit/backend/ppc/ppcgen/opassembler.py
--- a/pypy/jit/backend/ppc/ppcgen/opassembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/opassembler.py
@@ -83,6 +83,34 @@
     def emit_int_xor(self, op, arglocs, regalloc):
         l0, l1, res = arglocs
         self.mc.xor(res.value, l0.value, l1.value)
+        
+    def emit_int_lshift(self, op, arglocs, regalloc):
+        l0, l1, res = arglocs
+        if IS_PPC_32:
+            self.mc.slw(res.value, l0.value, l1.value)
+        else:
+            self.mc.sld(res.value, l0.value, l1.value)
+
+    def emit_int_rshift(self, op, arglocs, regalloc):
+        l0, l1, res = arglocs
+        if IS_PPC_32:
+            self.mc.sraw(res.value, l0.value, l1.value)
+        else:
+            self.mc.srad(res.value, l0.value, l1.value)
+
+    def emit_uint_rshift(self, op, arglocs, regalloc):
+        l0, l1, res = arglocs
+        if IS_PPC_32:
+            self.mc.srw(res.value, l0.value, l1.value)
+        else:
+            self.mc.srd(res.value, l0.value, l1.value)
+    
+    def emit_uint_floordiv(self, op, arglocs, regalloc):
+        l0, l1, res = arglocs
+        if IS_PPC_32:
+            self.mc.divwu(res.value, l0.value, l1.value)
+        else:
+            self.mc.divdu(res.value, l0.value, l1.value)
 
     emit_int_le = gen_emit_cmp_op(c.LE)   
 
diff --git a/pypy/jit/backend/ppc/ppcgen/regalloc.py b/pypy/jit/backend/ppc/ppcgen/regalloc.py
--- a/pypy/jit/backend/ppc/ppcgen/regalloc.py
+++ b/pypy/jit/backend/ppc/ppcgen/regalloc.py
@@ -231,6 +231,18 @@
     def prepare_int_xor(self, op):
         return self.prepare_int_mul(op)
 
+    def prepare_int_lshift(self, op):
+        return self.prepare_int_mul(op)
+
+    def prepare_int_rshift(self, op):
+        return self.prepare_int_mul(op)
+
+    def prepare_uint_rshift(self, op):
+        return self.prepare_int_mul(op)
+
+    def prepare_uint_floordiv(self, op):
+        return self.prepare_int_mul(op)
+
     def prepare_finish(self, op):
         args = [locations.imm(self.frame_manager.frame_depth)]
         for i in range(op.numargs()):

From noreply at buildbot.pypy.org  Tue Oct 18 16:24:26 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Oct 2011 16:24:26 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: Kill the ones that are either
 resolved or 'some non-standard binary distribution of pypy'.
Message-ID: <20111018142426.44950820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r3933:4ee2466e4095
Date: 2011-10-18 16:24 +0200
http://bitbucket.org/pypy/extradoc/changeset/4ee2466e4095/

Log:	Kill the ones that are either resolved or 'some non-standard binary
	distribution of pypy'.

diff --git a/planning/release-blockers-1.7.rst b/planning/release-blockers-1.7.rst
--- a/planning/release-blockers-1.7.rst
+++ b/planning/release-blockers-1.7.rst
@@ -1,11 +1,5 @@
 
 * [fijal] I might want finish the json work. I'll look into it this week
 
-* sobel demo, https://bugs.pypy.org/issue911
-
 * django tests (?) https://bugs.pypy.org/issue908
 
-* twisted threading tests (?) https://bugs.pypy.org/issue645
-
-* cpyext header files: https://bugs.pypy.org/issue889
-

From noreply at buildbot.pypy.org  Tue Oct 18 16:51:23 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Oct 2011 16:51:23 +0200 (CEST)
Subject: [pypy-commit] pypy default: Add the 'pyexpat.model' module.
Message-ID: <20111018145123.64427820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48209:595bfa0049db
Date: 2011-10-18 16:50 +0200
http://bitbucket.org/pypy/pypy/changeset/595bfa0049db/

Log:	Add the 'pyexpat.model' module.

diff --git a/pypy/module/pyexpat/__init__.py b/pypy/module/pyexpat/__init__.py
--- a/pypy/module/pyexpat/__init__.py
+++ b/pypy/module/pyexpat/__init__.py
@@ -4,12 +4,8 @@
 
 class ErrorsModule(MixedModule):
     "Definition of pyexpat.errors module."
-
-    appleveldefs = {
-        }
-
-    interpleveldefs = {
-        }
+    appleveldefs = {}
+    interpleveldefs = {}
 
     def setup_after_space_initialization(self):
         from pypy.module.pyexpat import interp_pyexpat
@@ -18,6 +14,18 @@
                     interp_pyexpat.ErrorString(self.space,
                     getattr(interp_pyexpat, name)))
 
+class ModelModule(MixedModule):
+    "Definition of pyexpat.model module."
+    appleveldefs = {}
+    interpleveldefs = {}
+
+    def setup_after_space_initialization(self):
+        from pypy.module.pyexpat import interp_pyexpat
+        space = self.space
+        for name in interp_pyexpat.xml_model_list:
+            value = getattr(interp_pyexpat, name)
+            space.setattr(self, space.wrap(name), space.wrap(value))
+
 class Module(MixedModule):
     "Python wrapper for Expat parser."
 
@@ -39,6 +47,7 @@
 
     submodules = {
         'errors': ErrorsModule,
+        'model':  ModelModule,
     }
 
     for name in ['XML_PARAM_ENTITY_PARSING_NEVER',
diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py
--- a/pypy/module/pyexpat/interp_pyexpat.py
+++ b/pypy/module/pyexpat/interp_pyexpat.py
@@ -76,6 +76,18 @@
     "XML_ERROR_FINISHED",
     "XML_ERROR_SUSPEND_PE",
     ]
+xml_model_list = [
+    "XML_CTYPE_EMPTY",
+    "XML_CTYPE_ANY",
+    "XML_CTYPE_MIXED",
+    "XML_CTYPE_NAME",
+    "XML_CTYPE_CHOICE",
+    "XML_CTYPE_SEQ",
+    "XML_CQUANT_NONE",
+    "XML_CQUANT_OPT",
+    "XML_CQUANT_REP",
+    "XML_CQUANT_PLUS",
+    ]
 
 class CConfigure:
     _compilation_info_ = eci
@@ -104,6 +116,8 @@
 
     for name in xml_error_list:
         locals()[name] = rffi_platform.ConstantInteger(name)
+    for name in xml_model_list:
+        locals()[name] = rffi_platform.ConstantInteger(name)
 
 for k, v in rffi_platform.configure(CConfigure).items():
     globals()[k] = v
diff --git a/pypy/module/pyexpat/test/test_parser.py b/pypy/module/pyexpat/test/test_parser.py
--- a/pypy/module/pyexpat/test/test_parser.py
+++ b/pypy/module/pyexpat/test/test_parser.py
@@ -131,3 +131,7 @@
                'encoding specified in XML declaration is incorrect')
         assert (pyexpat.errors.XML_ERROR_XML_DECL ==
                 'XML declaration not well-formed')
+
+    def test_model(self):
+        import pyexpat
+        assert isinstance(pyexpat.model.XML_CTYPE_EMPTY, int)

From noreply at buildbot.pypy.org  Tue Oct 18 16:51:24 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Oct 2011 16:51:24 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20111018145124.8CE4C820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48210:9f869e457c00
Date: 2011-10-18 16:51 +0200
http://bitbucket.org/pypy/pypy/changeset/9f869e457c00/

Log:	merge heads

diff --git a/pypy/translator/platform/posix.py b/pypy/translator/platform/posix.py
--- a/pypy/translator/platform/posix.py
+++ b/pypy/translator/platform/posix.py
@@ -157,7 +157,7 @@
 
         rules = [
             ('all', '$(DEFAULT_TARGET)', []),
-            ('$(TARGET)', '$(OBJECTS)', '$(CC_LINK) $(LDFLAGS) $(LDFLAGSEXTRA) -o $@ $(OBJECTS) $(LIBDIRS) $(LIBS) $(LINKFILES)'),
+            ('$(TARGET)', '$(OBJECTS)', '$(CC_LINK) $(LDFLAGSEXTRA) -o $@ $(OBJECTS) $(LIBDIRS) $(LIBS) $(LINKFILES) $(LDFLAGS)'),
             ('%.o', '%.c', '$(CC) $(CFLAGS) $(CFLAGSEXTRA) -o $@ -c $< $(INCLUDEDIRS)'),
             ]
 

From noreply at buildbot.pypy.org  Tue Oct 18 18:06:46 2011
From: noreply at buildbot.pypy.org (hager)
Date: Tue, 18 Oct 2011 18:06:46 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: Implemented comparison
 operations and refactored existing code regarding comparison operations.
Message-ID: <20111018160646.9CDDF820D7@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r48211:eabb2cbd47c9
Date: 2011-10-18 18:06 +0200
http://bitbucket.org/pypy/pypy/changeset/eabb2cbd47c9/

Log:	Implemented comparison operations and refactored existing code
	regarding comparison operations.

diff --git a/pypy/jit/backend/ppc/ppcgen/condition.py b/pypy/jit/backend/ppc/ppcgen/condition.py
--- a/pypy/jit/backend/ppc/ppcgen/condition.py
+++ b/pypy/jit/backend/ppc/ppcgen/condition.py
@@ -5,4 +5,9 @@
 EQ = 12
 GE = 33
 
+U_LT = 50
+U_LE = 60
+U_GT = 70
+U_GE = 80
+
 opposites = {LE: GT, NE: EQ, LT: GE, GE: LT, EQ: NE, GT: LE}
diff --git a/pypy/jit/backend/ppc/ppcgen/helper/assembler.py b/pypy/jit/backend/ppc/ppcgen/helper/assembler.py
--- a/pypy/jit/backend/ppc/ppcgen/helper/assembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/helper/assembler.py
@@ -1,27 +1,49 @@
 import pypy.jit.backend.ppc.ppcgen.condition as c
 from pypy.rlib.rarithmetic import r_uint, r_longlong, intmask
 
-def gen_emit_cmp_op(condition):
+def gen_emit_cmp_op(condition, signed=True):
     def f(self, op, arglocs, regalloc):
         l0, l1, res = arglocs
         # do the comparison
-        if l1.is_imm():
-            self.mc.cmpwi(0, l0.value, l1.value)
+        if signed:
+            if l1.is_imm():
+                self.mc.cmpwi(0, l0.value, l1.value)
+            else:
+                self.mc.cmpw(0, l0.value, l1.value)
+
+            # After the comparison, place the result
+            # in the first bit of the CR
+            if condition == c.LT:
+                self.mc.cror(0, 0, 0)
+            elif condition == c.LE:
+                self.mc.cror(0, 0, 2)
+            elif condition == c.EQ:
+                self.mc.cror(0, 2, 2)
+            elif condition == c.GE:
+                self.mc.cror(0, 1, 2)
+            elif condition == c.GT:
+                self.mc.cror(0, 1, 1)
+            elif condition == c.NE:
+                self.mc.cror(0, 0, 1)
+            else:
+                assert 0, "condition not known"
+
         else:
-            self.mc.cmpw(0, l0.value, l1.value)
+            if l1.is_imm():
+                self.mc.cmpli(0, l0.value, l1.value)
+            else:
+                self.mc.cmpl(0, l0.value, l1.value)
 
-        # After the comparison, place the result
-        # in the first bit of the CR
-        if condition == c.LT:
-            self.mc.cror(0, 0, 0)
-        elif condition == c.LE:
-            self.mc.cror(0, 0, 2)
-        elif condition == c.EQ:
-            self.mc.cror(0, 2, 2)
-        elif condition == c.GE:
-            self.mc.cror(0, 1, 2)
-        elif condition == c.GT:
-            self.mc.cror(0, 1, 1)
+            if condition == c.U_LT:
+                self.mc.cror(0, 0, 0)
+            elif condition == c.U_LE:
+                self.mc.cror(0, 0, 2)
+            elif condition == c.U_GT:
+                self.mc.cror(0, 1, 1)
+            elif condition == c.U_GE:
+                self.mc.cror(0, 1, 2)
+            else:
+                assert 0, "condition not known"
 
         resval = res.value 
         # move the content of the CR to resval
diff --git a/pypy/jit/backend/ppc/ppcgen/helper/regalloc.py b/pypy/jit/backend/ppc/ppcgen/helper/regalloc.py
--- a/pypy/jit/backend/ppc/ppcgen/helper/regalloc.py
+++ b/pypy/jit/backend/ppc/ppcgen/helper/regalloc.py
@@ -1,7 +1,14 @@
 from pypy.jit.metainterp.history import ConstInt
 
-def _check_imm_arg(arg):
-    return isinstance(arg, ConstInt)
+def _check_imm_arg(arg, size=0xFF, allow_zero=True):
+    if isinstance(arg, ConstInt):
+        i = arg.getint()
+        if allow_zero:
+            lower_bound = i >= 0
+        else:
+            lower_bound = i > 0
+        return i <= size and lower_bound
+    return False
 
 def prepare_cmp_op():
     def f(self, op):
@@ -21,3 +28,45 @@
         self.possibly_free_var(op.result)
         return [l0, l1, res]
     return f
+
+def prepare_binary_int_op_with_imm():
+    def f(self, op):
+        boxes = op.getarglist()
+        b0, b1 = boxes
+        imm_b0 = _check_imm_arg(b0)
+        imm_b1 = _check_imm_arg(b1)
+        if not imm_b0 and imm_b1:
+            l0, box = self._ensure_value_is_boxed(b0)
+            l1 = self.make_sure_var_in_reg(b1, [b0])
+            boxes.append(box)
+        elif imm_b0 and not imm_b1:
+            l0 = self.make_sure_var_in_reg(b0)
+            l1, box = self._ensure_value_is_boxed(b1, [b0])
+            boxes.append(box)
+        else:
+            l0, box = self._ensure_value_is_boxed(b0)
+            boxes.append(box)
+            l1, box = self._ensure_value_is_boxed(b1, [box])
+            boxes.append(box)
+        locs = [l0, l1]
+        self.possibly_free_vars(boxes)
+        res = self.force_allocate_reg(op.result)
+        return locs + [res]
+    return f
+
+def prepare_binary_int_op():
+    def f(self, op):
+        boxes = list(op.getarglist())
+        b0, b1 = boxes
+
+        reg1, box = self._ensure_value_is_boxed(b0, forbidden_vars=boxes)
+        boxes.append(box)
+        reg2, box = self._ensure_value_is_boxed(b1, forbidden_vars=boxes)
+        boxes.append(box)
+
+        self.possibly_free_vars(boxes)
+        self.possibly_free_vars_for_op(op)
+        res = self.force_allocate_reg(op.result)
+        self.possibly_free_var(op.result)
+        return [reg1, reg2, res]
+    return f
diff --git a/pypy/jit/backend/ppc/ppcgen/opassembler.py b/pypy/jit/backend/ppc/ppcgen/opassembler.py
--- a/pypy/jit/backend/ppc/ppcgen/opassembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/opassembler.py
@@ -112,7 +112,17 @@
         else:
             self.mc.divdu(res.value, l0.value, l1.value)
 
-    emit_int_le = gen_emit_cmp_op(c.LE)   
+    emit_int_le = gen_emit_cmp_op(c.LE)
+    emit_int_lt = gen_emit_cmp_op(c.LT)
+    emit_int_gt = gen_emit_cmp_op(c.GT)
+    emit_int_ge = gen_emit_cmp_op(c.GE)
+    emit_int_eq = gen_emit_cmp_op(c.EQ)
+    emit_int_ne = gen_emit_cmp_op(c.NE)
+
+    emit_uint_lt = gen_emit_cmp_op(c.U_LT, signed=False)
+    emit_uint_le = gen_emit_cmp_op(c.U_LE, signed=False)
+    emit_uint_gt = gen_emit_cmp_op(c.U_GT, signed=False)
+    emit_uint_ge = gen_emit_cmp_op(c.U_GE, signed=False)
 
     def _emit_guard(self, op, arglocs, fcond, save_exc=False,
             is_guard_not_invalidated=False):
diff --git a/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py b/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py
--- a/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py
@@ -399,7 +399,7 @@
         self._make_prologue(regalloc_head, frame_depth)
      
         self.write_pending_failure_recoveries()
-        loop_start = self.materialize_loop(looptoken, True)
+        loop_start = self.materialize_loop(looptoken, False)
         looptoken.ppc_code = loop_start + start_pos
         self.process_pending_guards(loop_start)
         self._teardown()
diff --git a/pypy/jit/backend/ppc/ppcgen/regalloc.py b/pypy/jit/backend/ppc/ppcgen/regalloc.py
--- a/pypy/jit/backend/ppc/ppcgen/regalloc.py
+++ b/pypy/jit/backend/ppc/ppcgen/regalloc.py
@@ -5,7 +5,9 @@
 from pypy.jit.backend.ppc.ppcgen.jump import remap_frame_layout_mixed
 from pypy.jit.backend.ppc.ppcgen.locations import imm
 from pypy.jit.backend.ppc.ppcgen.helper.regalloc import (_check_imm_arg, 
-                                                         prepare_cmp_op)
+                                                         prepare_cmp_op,
+                                                         prepare_binary_int_op,
+                                                         prepare_binary_int_op_with_imm)
 from pypy.jit.metainterp.history import (INT, REF, FLOAT, Const, ConstInt, 
                                          ConstPtr, LoopToken)
 from pypy.jit.metainterp.resoperation import rop
@@ -175,73 +177,31 @@
     # *         P R E P A R E  O P E R A T I O N S         * 
     # ******************************************************
 
-    def prepare_int_add(self, op):
-        boxes = op.getarglist()
-        b0, b1 = boxes
-        imm_b0 = _check_imm_arg(b0)
-        imm_b1 = _check_imm_arg(b1)
-        if not imm_b0 and imm_b1:
-            l0, box = self._ensure_value_is_boxed(b0)
-            l1 = self.make_sure_var_in_reg(b1, [b0])
-            boxes.append(box)
-        elif imm_b0 and not imm_b1:
-            l0 = self.make_sure_var_in_reg(b0)
-            l1, box = self._ensure_value_is_boxed(b1, [b0])
-            boxes.append(box)
-        else:
-            l0, box = self._ensure_value_is_boxed(b0)
-            boxes.append(box)
-            l1, box = self._ensure_value_is_boxed(b1, [box])
-            boxes.append(box)
-        locs = [l0, l1]
-        self.possibly_free_vars(boxes)
-        res = self.force_allocate_reg(op.result)
-        return locs + [res]
+    prepare_int_add = prepare_binary_int_op_with_imm()
+    prepare_int_sub = prepare_binary_int_op_with_imm()
+    prepare_int_floordiv = prepare_binary_int_op_with_imm()
 
-    def prepare_int_sub(self, op):
-        return self.prepare_int_add(op)
+    prepare_int_mul = prepare_binary_int_op()
+    prepare_int_mod = prepare_binary_int_op()
+    prepare_int_and = prepare_binary_int_op()
+    prepare_int_or = prepare_binary_int_op()
+    prepare_int_xor = prepare_binary_int_op()
+    prepare_int_lshift = prepare_binary_int_op()
+    prepare_int_rshift = prepare_binary_int_op()
+    prepare_uint_rshift = prepare_binary_int_op()
+    prepare_uint_floordiv = prepare_binary_int_op()
 
-    def prepare_int_floordiv(self, op):
-        return self.prepare_int_add(op)
+    prepare_int_le = prepare_cmp_op()
+    prepare_int_lt = prepare_cmp_op()
+    prepare_int_ge = prepare_cmp_op()
+    prepare_int_gt = prepare_cmp_op()
+    prepare_int_eq = prepare_cmp_op()
+    prepare_int_ne = prepare_cmp_op()
 
-    def prepare_int_mul(self, op):
-        boxes = list(op.getarglist())
-        b0, b1 = boxes
-
-        reg1, box = self._ensure_value_is_boxed(b0, forbidden_vars=boxes)
-        boxes.append(box)
-        reg2, box = self._ensure_value_is_boxed(b1, forbidden_vars=boxes)
-        boxes.append(box)
-
-        self.possibly_free_vars(boxes)
-        self.possibly_free_vars_for_op(op)
-        res = self.force_allocate_reg(op.result)
-        self.possibly_free_var(op.result)
-        return [reg1, reg2, res]
-
-    def prepare_int_mod(self, op):
-        return self.prepare_int_mul(op)
-
-    def prepare_int_and(self, op):
-        return self.prepare_int_mul(op)
-
-    def prepare_int_or(self, op):
-        return self.prepare_int_mul(op)
-
-    def prepare_int_xor(self, op):
-        return self.prepare_int_mul(op)
-
-    def prepare_int_lshift(self, op):
-        return self.prepare_int_mul(op)
-
-    def prepare_int_rshift(self, op):
-        return self.prepare_int_mul(op)
-
-    def prepare_uint_rshift(self, op):
-        return self.prepare_int_mul(op)
-
-    def prepare_uint_floordiv(self, op):
-        return self.prepare_int_mul(op)
+    prepare_uint_lt = prepare_cmp_op()
+    prepare_uint_le = prepare_cmp_op()
+    prepare_uint_gt = prepare_cmp_op()
+    prepare_uint_ge = prepare_cmp_op()
 
     def prepare_finish(self, op):
         args = [locations.imm(self.frame_manager.frame_depth)]
@@ -287,8 +247,6 @@
                                  [], [], None)
         return []
 
-    prepare_int_le = prepare_cmp_op()
-
 def make_operation_list():
     def not_implemented(self, op, *args):
         raise NotImplementedError, op

From noreply at buildbot.pypy.org  Tue Oct 18 19:05:00 2011
From: noreply at buildbot.pypy.org (hager)
Date: Tue, 18 Oct 2011 19:05:00 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: Implemented remaining INT_*
	operations.
Message-ID: <20111018170500.921C7820D7@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r48212:089618b63dd9
Date: 2011-10-18 19:04 +0200
http://bitbucket.org/pypy/pypy/changeset/089618b63dd9/

Log:	Implemented remaining INT_* operations.

diff --git a/pypy/jit/backend/ppc/ppcgen/condition.py b/pypy/jit/backend/ppc/ppcgen/condition.py
--- a/pypy/jit/backend/ppc/ppcgen/condition.py
+++ b/pypy/jit/backend/ppc/ppcgen/condition.py
@@ -10,4 +10,7 @@
 U_GT = 70
 U_GE = 80
 
+IS_TRUE = 90
+IS_ZERO = 100
+
 opposites = {LE: GT, NE: EQ, LT: GE, GE: LT, EQ: NE, GT: LE}
diff --git a/pypy/jit/backend/ppc/ppcgen/helper/assembler.py b/pypy/jit/backend/ppc/ppcgen/helper/assembler.py
--- a/pypy/jit/backend/ppc/ppcgen/helper/assembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/helper/assembler.py
@@ -52,6 +52,22 @@
         self.mc.rlwinm(resval, resval, 1, 31, 31)
     return f
 
+def gen_emit_unary_cmp_op(condition):
+    def f(self, op, arglocs, regalloc):
+        reg, res = arglocs
+
+        self.mc.cmpwi(0, reg.value, 0)
+        if condition == c.IS_ZERO:
+            self.mc.cror(0, 2, 2)
+        elif condition == c.IS_TRUE:
+            self.mc.cror(0, 0, 1)
+        else:
+            assert 0, "condition not known"
+
+        self.mc.mfcr(res.value)
+        self.mc.rlwinm(res.value, res.value, 1, 31, 31)
+    return f
+
 def encode32(mem, i, n):
     mem[i+3] = chr(n & 0xFF)
     mem[i+2] = chr((n >> 8) & 0xFF)
diff --git a/pypy/jit/backend/ppc/ppcgen/helper/regalloc.py b/pypy/jit/backend/ppc/ppcgen/helper/regalloc.py
--- a/pypy/jit/backend/ppc/ppcgen/helper/regalloc.py
+++ b/pypy/jit/backend/ppc/ppcgen/helper/regalloc.py
@@ -29,6 +29,24 @@
         return [l0, l1, res]
     return f
 
+def prepare_unary_cmp():
+    def f(self, op):
+        a0 = op.getarg(0)
+        reg, box = self._ensure_value_is_boxed(a0)
+        res = self.force_allocate_reg(op.result, [box])
+        self.possibly_free_vars([a0, box, op.result])
+        return [reg, res]
+    return f
+
+def prepare_unary_int_op():
+    def f(self, op):
+        l0, box = self._ensure_value_is_boxed(op.getarg(0))
+        self.possibly_free_var(box)
+        res = self.force_allocate_reg(op.result)
+        self.possibly_free_var(op.result)
+        return [l0, res]
+    return f
+
 def prepare_binary_int_op_with_imm():
     def f(self, op):
         boxes = op.getarglist()
diff --git a/pypy/jit/backend/ppc/ppcgen/opassembler.py b/pypy/jit/backend/ppc/ppcgen/opassembler.py
--- a/pypy/jit/backend/ppc/ppcgen/opassembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/opassembler.py
@@ -1,4 +1,5 @@
-from pypy.jit.backend.ppc.ppcgen.helper.assembler import gen_emit_cmp_op
+from pypy.jit.backend.ppc.ppcgen.helper.assembler import (gen_emit_cmp_op, 
+                                                          gen_emit_unary_cmp_op)
 import pypy.jit.backend.ppc.ppcgen.condition as c
 import pypy.jit.backend.ppc.ppcgen.register as r
 from pypy.jit.backend.ppc.ppcgen.arch import GPR_SAVE_AREA, IS_PPC_32, WORD
@@ -124,6 +125,17 @@
     emit_uint_gt = gen_emit_cmp_op(c.U_GT, signed=False)
     emit_uint_ge = gen_emit_cmp_op(c.U_GE, signed=False)
 
+    emit_int_is_zero = gen_emit_unary_cmp_op(c.IS_ZERO)
+    emit_int_is_true = gen_emit_unary_cmp_op(c.IS_TRUE)
+
+    def emit_int_neg(self, op, arglocs, regalloc):
+        l0, res = arglocs
+        self.mc.neg(res.value, l0.value)
+
+    def emit_int_invert(self, op, arglocs, regalloc):
+        l0, res = arglocs
+        self.mc.not_(res.value, l0.value)
+
     def _emit_guard(self, op, arglocs, fcond, save_exc=False,
             is_guard_not_invalidated=False):
         descr = op.getdescr()
diff --git a/pypy/jit/backend/ppc/ppcgen/regalloc.py b/pypy/jit/backend/ppc/ppcgen/regalloc.py
--- a/pypy/jit/backend/ppc/ppcgen/regalloc.py
+++ b/pypy/jit/backend/ppc/ppcgen/regalloc.py
@@ -6,8 +6,10 @@
 from pypy.jit.backend.ppc.ppcgen.locations import imm
 from pypy.jit.backend.ppc.ppcgen.helper.regalloc import (_check_imm_arg, 
                                                          prepare_cmp_op,
+                                                         prepare_unary_int_op,
                                                          prepare_binary_int_op,
-                                                         prepare_binary_int_op_with_imm)
+                                                         prepare_binary_int_op_with_imm,
+                                                         prepare_unary_cmp)
 from pypy.jit.metainterp.history import (INT, REF, FLOAT, Const, ConstInt, 
                                          ConstPtr, LoopToken)
 from pypy.jit.metainterp.resoperation import rop
@@ -191,6 +193,9 @@
     prepare_uint_rshift = prepare_binary_int_op()
     prepare_uint_floordiv = prepare_binary_int_op()
 
+    prepare_int_neg = prepare_unary_int_op()
+    prepare_int_invert = prepare_unary_int_op()
+
     prepare_int_le = prepare_cmp_op()
     prepare_int_lt = prepare_cmp_op()
     prepare_int_ge = prepare_cmp_op()
@@ -203,6 +208,9 @@
     prepare_uint_gt = prepare_cmp_op()
     prepare_uint_ge = prepare_cmp_op()
 
+    prepare_int_is_true = prepare_unary_cmp()
+    prepare_int_is_zero = prepare_unary_cmp()
+
     def prepare_finish(self, op):
         args = [locations.imm(self.frame_manager.frame_depth)]
         for i in range(op.numargs()):

From noreply at buildbot.pypy.org  Tue Oct 18 20:04:09 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Oct 2011 20:04:09 +0200 (CEST)
Subject: [pypy-commit] pypy default: Give the walk_roots operation during a
 nursery collection its own
Message-ID: <20111018180409.7E9AC820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48213:5fcf5ac416dc
Date: 2011-10-18 20:03 +0200
http://bitbucket.org/pypy/pypy/changeset/5fcf5ac416dc/

Log:	Give the walk_roots operation during a nursery collection its own
	starting/ending subtime. When running "translate.py --annotate", it
	is found to take 15% of the time of nursery collections (not really
	enough to care). For reference, the times are:

	 12.4% gc-minor including 1.9% gc-minor-walkroots 6.5%
	gc-collect (i.e. major collections)

diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -1292,10 +1292,12 @@
         # if a prebuilt GcStruct contains a pointer to a young object,
         # then the write_barrier must have ensured that the prebuilt
         # GcStruct is in the list self.old_objects_pointing_to_young.
+        debug_start("gc-minor-walkroots")
         self.root_walker.walk_roots(
             MiniMarkGC._trace_drag_out1,  # stack roots
             MiniMarkGC._trace_drag_out1,  # static in prebuilt non-gc
             None)                         # static in prebuilt gc
+        debug_stop("gc-minor-walkroots")
 
     def collect_cardrefs_to_nursery(self):
         size_gc_header = self.gcheaderbuilder.size_gc_header

From noreply at buildbot.pypy.org  Tue Oct 18 23:49:57 2011
From: noreply at buildbot.pypy.org (mattip)
Date: Tue, 18 Oct 2011 23:49:57 +0200 (CEST)
Subject: [pypy-commit] pypy numpy NDimArray: Add shape checking,
 implement NDim binary ops (mult and div are wrong)
Message-ID: <20111018214957.0DF33820D7@wyvern.cs.uni-duesseldorf.de>

Author: mattip
Branch: numpy NDimArray
Changeset: r48214:1be18bf996dd
Date: 2011-10-18 23:48 +0200
http://bitbucket.org/pypy/pypy/changeset/1be18bf996dd/

Log:	Add shape checking, implement NDim binary ops (mult and div are
	wrong)

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -107,6 +107,12 @@
 
     def _binop_impl(ufunc_name):
         def impl(self, space, w_other):
+            if not space.eq_w(self.descr_shape(space),w_other.descr_shape(space)):
+                raise OperationError(space.w_ValueError,
+                    space.wrap("shape mismatch: objects cannot be broadcast to a single shape %s <> %s" \
+                            % (self.descr_shape(space).unwrap(space), 
+                              w_other.descr_shape(space).unwrap(space),
+                            )))
             return getattr(interp_ufuncs.get(space), ufunc_name).call(space, [self, w_other])
         return func_with_new_name(impl, "binop_%s_impl" % ufunc_name)
 
@@ -297,7 +303,7 @@
             length = space.len_w(w_idx)
             if length > 1: # only one dimension for now.
                 raise OperationError(space.w_IndexError,
-                                     space.wrap("invalid index"))
+                                     space.wrap("invalid index: cannot handle tuple indices yet"))
             if length == 0:
                 w_idx = space.newslice(space.wrap(0),
                                       space.wrap(self.find_size()),
@@ -392,8 +398,12 @@
     def compute(self):
         i = 0
         signature = self.signature
+        result_shape = self.find_shape()
         result_size = self.find_size()
-        result = SingleDimArray(result_size, self.find_dtype())
+        if len(result_shape)>1:
+            result = NDimArray(result_shape, self.find_dtype())
+        else:
+            result = SingleDimArray(result_size, self.find_dtype())
         while i < result_size:
             numpy_driver.jit_merge_point(signature=signature,
                                          result_size=result_size, i=i,
@@ -419,6 +429,12 @@
     def setitem(self, item, value):
         return self.get_concrete().setitem(item, value)
 
+    def find_shape(self):
+        if self.forced_result is not None:
+            # The result has been computed and sources may be unavailable
+            return self.forced_result.find_shape()
+        return self._find_shape()
+
     def find_size(self):
         if self.forced_result is not None:
             # The result has been computed and sources may be unavailable
@@ -466,6 +482,13 @@
         self.left = None
         self.right = None
 
+    def _find_shape(self):
+        try:
+            return self.left.find_shape()
+        except ValueError:
+            pass
+        return self.right.find_shape()
+
     def _find_size(self):
         try:
             return self.left.find_size()
@@ -538,6 +561,9 @@
     def get_root_storage(self):
         return self.parent.get_concrete().get_root_storage()
 
+    def find_shape(self):
+        return (self.size,)
+
     def find_size(self):
         return self.size
 
@@ -582,6 +608,9 @@
     def get_root_storage(self):
         return self.storage
 
+    def find_shape(self):
+        return (self.size,)
+
     def find_size(self):
         return self.size
 
@@ -639,6 +668,9 @@
     def get_root_storage(self):
         return self.storage
 
+    def find_shape(self):
+        return self.shape
+
     def find_size(self):
         return self.size
 
@@ -649,7 +681,7 @@
         return self.dtype.getitem(self.storage, i)
 
     def descr_shape(self, space):
-        return self.shape
+        return space.wrap(self.shape)
 
     def descr_len(self, space):
         return space.wrap(self.size)
@@ -673,7 +705,6 @@
                 self.setitem_recurse_w(space,i+index*self.shape[-depth], depth+1, w_item)
                 i+=1
         else:
-            print 'setting',index,'to',w_value
             self.setitem_w(space,index,w_value) 
     def setitem(self, item, value):
         self.invalidated()

From noreply at buildbot.pypy.org  Wed Oct 19 01:43:17 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 19 Oct 2011 01:43:17 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Fixes for applevel interactive code
Message-ID: <20111018234318.01251820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48215:6b5837b87cdf
Date: 2011-10-18 12:01 +0200
http://bitbucket.org/pypy/pypy/changeset/6b5837b87cdf/

Log:	Fixes for applevel interactive code

diff --git a/lib_pypy/_pypy_interact.py b/lib_pypy/_pypy_interact.py
--- a/lib_pypy/_pypy_interact.py
+++ b/lib_pypy/_pypy_interact.py
@@ -18,9 +18,9 @@
             some_topic(),)
         while len(text) >= 80:
             i = text[:80].rfind(' ')
-            print text[:i]
+            print(text[:i])
             text = text[i+1:]
-        print text
+        print(text)
     except ImportError:
         pass
     #
diff --git a/lib_pypy/_pypy_irc_topic.py b/lib_pypy/_pypy_irc_topic.py
--- a/lib_pypy/_pypy_irc_topic.py
+++ b/lib_pypy/_pypy_irc_topic.py
@@ -181,7 +181,25 @@
 clcl vf n enpr orgjrra gur vaqhfgel gelvat gb ohvyq znpuvarf jvgu zber naq zber erfbheprf, naq gur clcl qrirybcref gelvat gb rng nyy bs gurz. Fb sne, gur jvaare vf fgvyy hapyrne
 """
 
+from string import ascii_uppercase, ascii_lowercase
+ 
+def rot13(data):
+    """ A simple rot-13 encoder since `str.encode('rot13')` was removed from
+        Python as of version 3.0.  It rotates both uppercase and lowercase letters individually.
+    """
+    total = []
+    for char in data:
+        if char in ascii_uppercase:
+            index = (ascii_uppercase.find(char) + 13) % 26
+            total.append(ascii_uppercase[index])
+        elif char in ascii_lowercase:
+            index = (ascii_lowercase.find(char) + 13) % 26
+            total.append(ascii_lowercase[index])
+        else:
+            total.append(char)
+    return "".join(total)
+ 
 def some_topic():
     import time
     lines = __doc__.splitlines()
-    return lines[int(time.time()) % len(lines)].decode('rot13')
+    return rot13(lines[int(time.time()) % len(lines)])

From noreply at buildbot.pypy.org  Wed Oct 19 01:43:19 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 19 Oct 2011 01:43:19 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Fix two test files.
Message-ID: <20111018234319.3855C820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48216:72a97784aaf3
Date: 2011-10-18 20:35 +0200
http://bitbucket.org/pypy/pypy/changeset/72a97784aaf3/

Log:	Fix two test files.

diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -456,16 +456,6 @@
         yield self.st, decl, 'A,A1,A2,B2,C,C1,C2,D1,E,G,G1,G2,N1', \
                              (6,6 ,4 ,1 ,5,5 ,5 ,3 ,8,2,2 ,2 ,7 )
 
-        decl = py.code.Source("""
-            def f((a, b)):
-                def g((c, d)):
-                    return (a, b, c, d)
-                return g
-            x = f((1, 2))((3, 4))
-        """)
-        decl = str(decl) + "\n"
-        yield self.st, decl, 'x', (1, 2, 3, 4)
-
         source = """if 1:
         def f(a):
             del a
@@ -673,7 +663,7 @@
                 if not d: self.fail("Full mapping must compare to True")
                 # keys(), items(), iterkeys() ...
                 def check_iterandlist(iter, lst, ref):
-                    self.assert_(hasattr(iter, 'next'))
+                    self.assert_(hasattr(iter, '__next__'))
                     self.assert_(hasattr(iter, '__iter__'))
                     x = list(iter)
                     self.assert_(set(x)==set(lst)==set(ref))
@@ -682,8 +672,8 @@
                 check_iterandlist(d.itervalues(), d.values(), self.reference.values())
                 check_iterandlist(d.iteritems(), d.items(), self.reference.items())
                 #get
-                key, value = d.iteritems().next()
-                knownkey, knownvalue = self.other.iteritems().next()
+                key, value = next(d.iteritems())
+                knownkey, knownvalue = next(self.other.iteritems())
                 self.assertEqual(d.get(key, knownvalue), value)
                 self.assertEqual(d.get(knownkey, knownvalue), knownvalue)
                 self.failIf(knownkey in d)
@@ -817,7 +807,7 @@
         exec("def f():\n    'hi'", ns)
         f = ns["f"]
         save = sys.stdout
-        sys.stdout = output = io.BytesIO()
+        sys.stdout = output = io.StringIO()
         try:
             dis.dis(f)
         finally:
diff --git a/pypy/interpreter/test/test_code.py b/pypy/interpreter/test/test_code.py
--- a/pypy/interpreter/test/test_code.py
+++ b/pypy/interpreter/test/test_code.py
@@ -42,7 +42,7 @@
                                  'co_flags': 0,
                                  'co_consts': ("abs(number) -> number\n\nReturn the absolute value of the argument.",),
                                  }),
-                (object.__init__.im_func.func_code,
+                (object.__init__.func_code,
                                 {#'co_name': '__init__',   XXX getting descr__init__
                                  'co_varnames': ('obj', 'args', 'keywords'),
                                  'co_argcount': 1,
@@ -70,7 +70,7 @@
             foo(g)
 '''
         d = {}
-        exec src in d
+        exec(src, d)
 
         assert list(sorted(d['f'].func_code.co_names)) == ['foo', 'g']
 
@@ -97,7 +97,7 @@
                       ccode.co_freevars,
                       ccode.co_cellvars)
         d = {}
-        exec co in d
+        exec(co, d)
         assert d['c'] == 3
         # test backwards-compatibility version with no freevars or cellvars
         co = new.code(ccode.co_argcount,
@@ -113,7 +113,7 @@
                       ccode.co_firstlineno,
                       ccode.co_lnotab)
         d = {}
-        exec co in d
+        exec(co, d)
         assert d['c'] == 3
         def f(x):
             y = 1
@@ -147,9 +147,9 @@
 
     def test_hash(self):
         d1 = {}
-        exec "def f(): pass" in d1
+        exec("def f(): pass", d1)
         d2 = {}
-        exec "def f(): pass" in d2
+        exec("def f(): pass", d2)
         assert d1['f'].func_code == d2['f'].func_code
         assert hash(d1['f'].func_code) == hash(d2['f'].func_code)
 
@@ -164,47 +164,48 @@
             assert i in res
 
     def test_code_extra(self):
-        exec """if 1:
+        d = {}
+        exec("""if 1:
         def f():
             "docstring"
             'stuff'
             56
-"""
+""", d)
 
         # check for new flag, CO_NOFREE
-        assert f.func_code.co_flags & 0x40
+        assert d['f'].func_code.co_flags & 0x40
 
-        exec """if 1:
+        exec("""if 1:
         def f(x):
             def g(y):
                 return x+y
             return g
-"""
+""", d)
 
         # CO_NESTED
-        assert f(4).func_code.co_flags & 0x10
-        assert f.func_code.co_flags & 0x10 == 0
+        assert d['f'](4).func_code.co_flags & 0x10
+        assert d['f'].func_code.co_flags & 0x10 == 0
         # check for CO_CONTAINSGLOBALS
-        assert not f.func_code.co_flags & 0x0800
+        assert not d['f'].func_code.co_flags & 0x0800
 
 
-        exec """if 1:
+        exec("""if 1:
         r = range
         def f():
             return [l for l in r(100)]
         def g():
             return [l for l in [1, 2, 3, 4]]
-"""
+""", d)
 
         # check for CO_CONTAINSGLOBALS
-        assert f.func_code.co_flags & 0x0800
-        assert not g.func_code.co_flags & 0x0800
+        assert d['f'].func_code.co_flags & 0x0800
+        assert not d['g'].func_code.co_flags & 0x0800
 
-        exec """if 1:
+        exec("""if 1:
         b = 2
         def f(x):
-            exec "a = 1";
+            exec("a = 1")
             return a + b + x
-"""
+""", d)
         # check for CO_CONTAINSGLOBALS
-        assert f.func_code.co_flags & 0x0800
+        assert d['f'].func_code.co_flags & 0x0800
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -472,6 +472,12 @@
         return space.wrap(getattr(obj, name))
     return GetSetProperty(fget, cls=cls, doc=doc)
 
+def interp_attrproperty_bytes(name, cls, doc=None):
+    "NOT_RPYTHON: initialization-time only"
+    def fget(space, obj):
+        return space.wrapbytes(getattr(obj, name))
+    return GetSetProperty(fget, cls=cls, doc=doc)
+
 def interp_attrproperty_w(name, cls, doc=None):
     "NOT_RPYTHON: initialization-time only"
     def fget(space, obj):
@@ -686,7 +692,7 @@
     co_nlocals = interp_attrproperty('co_nlocals', cls=PyCode),
     co_stacksize = interp_attrproperty('co_stacksize', cls=PyCode),
     co_flags = interp_attrproperty('co_flags', cls=PyCode),
-    co_code = interp_attrproperty('co_code', cls=PyCode),
+    co_code = interp_attrproperty_bytes('co_code', cls=PyCode),
     co_consts = GetSetProperty(PyCode.fget_co_consts),
     co_names = GetSetProperty(PyCode.fget_co_names),
     co_varnames =  GetSetProperty(PyCode.fget_co_varnames),
@@ -695,7 +701,7 @@
     co_filename = interp_attrproperty('co_filename', cls=PyCode),
     co_name = interp_attrproperty('co_name', cls=PyCode),
     co_firstlineno = interp_attrproperty('co_firstlineno', cls=PyCode),
-    co_lnotab = interp_attrproperty('co_lnotab', cls=PyCode),
+    co_lnotab = interp_attrproperty_bytes('co_lnotab', cls=PyCode),
     __weakref__ = make_weakref_descr(PyCode),
     )
 PyCode.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/__builtin__/app_functional.py b/pypy/module/__builtin__/app_functional.py
--- a/pypy/module/__builtin__/app_functional.py
+++ b/pypy/module/__builtin__/app_functional.py
@@ -41,8 +41,8 @@
 Returns the sum of a sequence of numbers (NOT strings) plus the value
 of parameter 'start' (which defaults to 0).  When the sequence is
 empty, returns start."""
-    if isinstance(start, basestring):
-        raise TypeError("sum() can't sum strings")
+    if isinstance(start, str):
+        raise TypeError("sum() can't sum strings [use ''.join(seq) instead]")
     last = start
     for x in sequence:
         # Very intentionally *not* +=, that would have different semantics if
@@ -162,4 +162,4 @@
         item = seq[i]
         if func(item):
             result.append(item)
-    return tuple(result)
\ No newline at end of file
+    return tuple(result)
diff --git a/pypy/module/__builtin__/app_io.py b/pypy/module/__builtin__/app_io.py
--- a/pypy/module/__builtin__/app_io.py
+++ b/pypy/module/__builtin__/app_io.py
@@ -78,6 +78,8 @@
     def write(data):
         if not isinstance(data, str):
             data = str(data)
+        if getattr(fp, 'encoding', None):
+            data = data.encode(fp.encoding)
         fp.write(data)
     sep = kwargs.pop("sep", None)
     if sep is not None:

From noreply at buildbot.pypy.org  Wed Oct 19 01:43:20 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 19 Oct 2011 01:43:20 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Fix a nasty crash with the "global"
	keyword.
Message-ID: <20111018234320.6A2DA820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48217:c53430cd1712
Date: 2011-10-18 21:44 +0200
http://bitbucket.org/pypy/pypy/changeset/c53430cd1712/

Log:	Fix a nasty crash with the "global" keyword. Unfortunately I could
	not find any failing test for this...

diff --git a/pypy/interpreter/astcompiler/symtable.py b/pypy/interpreter/astcompiler/symtable.py
--- a/pypy/interpreter/astcompiler/symtable.py
+++ b/pypy/interpreter/astcompiler/symtable.py
@@ -236,7 +236,7 @@
         # Special-case super: it counts as a use of __class__
         if role == SYM_USED and identifier == 'super':
             self.note_symbol('@__class__', SYM_USED)
-        Scope.note_symbol(self, identifier, role)
+        return Scope.note_symbol(self, identifier, role)
 
     def note_yield(self, yield_node):
         if self.return_with_value:

From noreply at buildbot.pypy.org  Wed Oct 19 01:43:21 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 19 Oct 2011 01:43:21 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Fix test_astbuilder
Message-ID: <20111018234321.9C402820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48218:75c3ac958a43
Date: 2011-10-18 22:16 +0200
http://bitbucket.org/pypy/pypy/changeset/75c3ac958a43/

Log:	Fix test_astbuilder

diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py b/pypy/interpreter/astcompiler/test/test_astbuilder.py
--- a/pypy/interpreter/astcompiler/test/test_astbuilder.py
+++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py
@@ -70,33 +70,6 @@
         for stmt in mod.body:
             assert isinstance(stmt, ast.Assign)
 
-    def test_print(self):
-        pri = self.get_first_stmt("print x")
-        assert isinstance(pri, ast.Print)
-        assert pri.dest is None
-        assert pri.nl
-        assert len(pri.values) == 1
-        assert isinstance(pri.values[0], ast.Name)
-        pri = self.get_first_stmt("print x, 34")
-        assert len(pri.values) == 2
-        assert isinstance(pri.values[0], ast.Name)
-        assert isinstance(pri.values[1], ast.Num)
-        pri = self.get_first_stmt("print")
-        assert pri.nl
-        assert pri.values is None
-        pri = self.get_first_stmt("print x,")
-        assert len(pri.values) == 1
-        assert not pri.nl
-        pri = self.get_first_stmt("print >> y, 4")
-        assert isinstance(pri.dest, ast.Name)
-        assert len(pri.values) == 1
-        assert isinstance(pri.values[0], ast.Num)
-        assert pri.nl
-        pri = self.get_first_stmt("print >> y")
-        assert isinstance(pri.dest, ast.Name)
-        assert pri.values is None
-        assert pri.nl
-
     def test_del(self):
         d = self.get_first_stmt("del x")
         assert isinstance(d, ast.Delete)
@@ -135,21 +108,14 @@
 
     def test_raise(self):
         ra = self.get_first_stmt("raise")
-        assert ra.type is None
-        assert ra.inst is None
-        assert ra.tback is None
+        assert ra.exc is None
+        assert ra.cause is None
         ra = self.get_first_stmt("raise x")
-        assert isinstance(ra.type, ast.Name)
-        assert ra.inst is None
-        assert ra.tback is None
-        ra = self.get_first_stmt("raise x, 3")
-        assert isinstance(ra.type, ast.Name)
-        assert isinstance(ra.inst, ast.Num)
-        assert ra.tback is None
-        ra = self.get_first_stmt("raise x, 4, 'hi'")
-        assert isinstance(ra.type, ast.Name)
-        assert isinstance(ra.inst, ast.Num)
-        assert isinstance(ra.tback, ast.Str)
+        assert isinstance(ra.exc, ast.Name)
+        assert ra.cause is None
+        ra = self.get_first_stmt("raise x from 3")
+        assert isinstance(ra.exc, ast.Name)
+        assert isinstance(ra.cause, ast.Num)
 
     def test_import(self):
         im = self.get_first_stmt("import x")
@@ -235,20 +201,12 @@
         glob = self.get_first_stmt("global x, y")
         assert glob.names == ["x", "y"]
 
-    def test_exec(self):
-        exc = self.get_first_stmt("exec x")
-        assert isinstance(exc, ast.Exec)
-        assert isinstance(exc.body, ast.Name)
-        assert exc.globals is None
-        assert exc.locals is None
-        exc = self.get_first_stmt("exec 'hi' in x")
-        assert isinstance(exc.body, ast.Str)
-        assert isinstance(exc.globals, ast.Name)
-        assert exc.locals is None
-        exc = self.get_first_stmt("exec 'hi' in x, 2")
-        assert isinstance(exc.body, ast.Str)
-        assert isinstance(exc.globals, ast.Name)
-        assert isinstance(exc.locals, ast.Num)
+    def test_nonlocal(self):
+        nonloc = self.get_first_stmt("nonlocal x")
+        assert isinstance(nonloc, ast.Nonlocal)
+        assert nonloc.names == ["x"]
+        nonloc = self.get_first_stmt("nonlocal x, y")
+        assert nonloc.names == ["x", "y"]
 
     def test_assert(self):
         asrt = self.get_first_stmt("assert x")
@@ -538,35 +496,6 @@
         assert args.defaults is None
         assert args.vararg is None
         assert args.kwarg == "a"
-        args = self.get_first_stmt("def f((a, b)): pass").args
-        assert args.defaults is None
-        assert args.kwarg is None
-        assert args.vararg is None
-        assert len(args.args) == 1
-        tup = args.args[0]
-        assert isinstance(tup, ast.Tuple)
-        assert tup.ctx == ast.Store
-        assert len(tup.elts) == 2
-        e1, e2 = tup.elts
-        assert isinstance(e1, ast.Name)
-        assert e1.ctx == ast.Store
-        assert e1.id == "a"
-        assert isinstance(e2, ast.Name)
-        assert e2.ctx == ast.Store
-        assert e2.id == "b"
-        args = self.get_first_stmt("def f((a, (b, c))): pass").args
-        assert len(args.args) == 1
-        tup = args.args[0]
-        assert isinstance(tup, ast.Tuple)
-        assert len(tup.elts) == 2
-        tup2 = tup.elts[1]
-        assert isinstance(tup2, ast.Tuple)
-        assert tup2.ctx == ast.Store
-        for elt in tup2.elts:
-            assert isinstance(elt, ast.Name)
-            assert elt.ctx == ast.Store
-        assert tup2.elts[0].id == "b"
-        assert tup2.elts[1].id == "c"
         args = self.get_first_stmt("def f(a, b, c=d, *e, **f): pass").args
         assert len(args.args) == 3
         for arg in args.args:
@@ -580,9 +509,6 @@
         input = "def f(a=b, c): pass"
         exc = py.test.raises(SyntaxError, self.get_ast, input).value
         assert exc.msg == "non-default argument follows default argument"
-        input = "def f((x)=23): pass"
-        exc = py.test.raises(SyntaxError, self.get_ast, input).value
-        assert exc.msg == "parenthesized arg with default"
 
     def test_decorators(self):
         to_examine = (("def f(): pass", ast.FunctionDef),
@@ -762,7 +688,6 @@
             ("{1, 2, 3}", "literal"),
             ("(x > 4)", "comparison"),
             ("(x if y else a)", "conditional expression"),
-            ("`x`", "repr")
         )
         test_contexts = (
             ("assign to", "%s = 23"),
@@ -1093,11 +1018,6 @@
         assert isinstance(complex_slc.upper, ast.Num)
         assert complex_slc.step is None
 
-    def test_repr(self):
-        rep = self.get_first_expr("`x`")
-        assert isinstance(rep, ast.Repr)
-        assert isinstance(rep.value, ast.Name)
-
     def test_string(self):
         space = self.space
         s = self.get_first_expr("'hi'")
diff --git a/pypy/interpreter/pyparser/parsestring.py b/pypy/interpreter/pyparser/parsestring.py
--- a/pypy/interpreter/pyparser/parsestring.py
+++ b/pypy/interpreter/pyparser/parsestring.py
@@ -208,7 +208,7 @@
         ps += 1
     w_u = space.wrap(unicodehelper.PyUnicode_DecodeUTF8(space, s[pt:ps]))
     w_v = unicodehelper.PyUnicode_AsEncodedString(space, w_u, space.wrap(encoding))
-    v = space.str_w(w_v)
+    v = space.bytes_w(w_v)
     return v, ps
 
 def raise_app_valueerror(space, msg):

From noreply at buildbot.pypy.org  Wed Oct 19 01:43:22 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 19 Oct 2011 01:43:22 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: _socket.socket is not weakrefable in
 CPython, socket.socket() is.
Message-ID: <20111018234322.CCD7C820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48219:16e3fa80fbcf
Date: 2011-10-18 22:25 +0200
http://bitbucket.org/pypy/pypy/changeset/16e3fa80fbcf/

Log:	_socket.socket is not weakrefable in CPython, socket.socket() is.
	("import socket" failed because the latter now inherits from the
	former, and the two __weakref__ slots conflicts)

diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -1,6 +1,5 @@
 from pypy.interpreter.baseobjspace import Wrappable
-from pypy.interpreter.typedef import TypeDef, make_weakref_descr,\
-     interp_attrproperty
+from pypy.interpreter.typedef import TypeDef, interp_attrproperty
 from pypy.interpreter.gateway import NoneNotWrapped, interp2app, unwrap_spec
 from pypy.rlib.rarithmetic import intmask
 from pypy.rlib import rsocket
@@ -18,7 +17,6 @@
 
 class W_RSocket(Wrappable, RSocket):
     def __del__(self):
-        self.clear_all_weakrefs()
         self.close()
 
     def accept_w(self, space):
@@ -524,7 +522,6 @@
 
  [*] not available on all platforms!""",
     __new__ = descr_socket_new,
-    __weakref__ = make_weakref_descr(W_RSocket),
     type = interp_attrproperty('type', W_RSocket),
     proto = interp_attrproperty('proto', W_RSocket),
     family = interp_attrproperty('family', W_RSocket),

From noreply at buildbot.pypy.org  Wed Oct 19 01:43:24 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 19 Oct 2011 01:43:24 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Make sockets really subclassable
Message-ID: <20111018234324.0A57F820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48220:4c11bf0deece
Date: 2011-10-18 22:45 +0200
http://bitbucket.org/pypy/pypy/changeset/4c11bf0deece/

Log:	Make sockets really subclassable

diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -415,13 +415,9 @@
 @unwrap_spec(family=int, type=int, proto=int)
 def newsocket(space, w_subtype, family=AF_INET,
               type=SOCK_STREAM, proto=0):
-    # XXX If we want to support subclassing the socket type we will need
-    # something along these lines. But allocate_instance is only defined
-    # on the standard object space, so this is not really correct.
-    #sock = space.allocate_instance(W_RSocket, w_subtype)
-    #Socket.__init__(sock, space, fd, family, type, proto)
+    sock = space.allocate_instance(W_RSocket, w_subtype)
     try:
-        sock = W_RSocket(family, type, proto)
+        W_RSocket.__init__(sock, family, type, proto)
     except SocketError, e:
         raise converted_error(space, e)
     return space.wrap(sock)

From noreply at buildbot.pypy.org  Wed Oct 19 01:43:25 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 19 Oct 2011 01:43:25 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Fixes for the struct module
Message-ID: <20111018234325.3EC2B820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48221:3b27b16f499d
Date: 2011-10-18 23:58 +0200
http://bitbucket.org/pypy/pypy/changeset/3b27b16f499d/

Log:	Fixes for the struct module

diff --git a/lib_pypy/struct.py b/lib_pypy/struct.py
--- a/lib_pypy/struct.py
+++ b/lib_pypy/struct.py
@@ -48,7 +48,7 @@
     pass
 error = StructError
 def unpack_int(data,index,size,le):
-    bytes = [ord(b) for b in data[index:index+size]]
+    bytes = [b for b in data[index:index+size]]
     if le == 'little':
         bytes.reverse()
     number = 0L
@@ -73,11 +73,11 @@
     x=number
     res=[]
     for i in range(size):
-        res.append(chr(x&0xff))
+        res.append(x&0xff)
         x >>= 8
     if le == 'big':
         res.reverse()
-    return ''.join(res)
+    return bytes(res)
 
 def pack_signed_int(number,size,le):
     if not isinstance(number, (int,long)):
@@ -96,7 +96,7 @@
     return pack_int(number,size,le)
 
 def pack_char(char,size,le):
-    return str(char)
+    return bytes(char)
 
 def isinf(x):
     return x != 0.0 and x / 2 == x
@@ -107,10 +107,10 @@
     unsigned = float_pack(x, size)
     result = []
     for i in range(8):
-        result.append(chr((unsigned >> (i * 8)) & 0xFF))
+        result.append((unsigned >> (i * 8)) & 0xFF)
     if le == "big":
         result.reverse()
-    return ''.join(result)
+    return bytes(result)
 
 def unpack_float(data, index, size, le):
     binary = [data[i] for i in range(index, index + 8)]
@@ -266,7 +266,7 @@
     try:
         formatdef,endianness = formatmode[fmt[0]]
         index = 1
-    except KeyError:
+    except (IndexError, KeyError):
         formatdef,endianness = formatmode['@']
         index = 0
     return formatdef,endianness,index
@@ -327,25 +327,25 @@
             num_s = num
 
         if cur == 'x':
-            result += ['\0'*num]
+            result += [b'\0'*num]
         elif cur == 's':
-            if isinstance(args[0], str):
+            if isinstance(args[0], bytes):
                 padding = num - len(args[0])
-                result += [args[0][:num] + '\0'*padding]
+                result += [args[0][:num] + b'\0'*padding]
                 args.pop(0)
             else:
                 raise StructError("arg for string format not a string")
         elif cur == 'p':
-            if isinstance(args[0], str):
+            if isinstance(args[0], bytes):
                 padding = num - len(args[0]) - 1
 
                 if padding > 0:
-                    result += [chr(len(args[0])) + args[0][:num-1] + '\0'*padding]
+                    result += [bytes([len(args[0])]) + args[0][:num-1] + b'\0'*padding]
                 else:
                     if num<255:
-                        result += [chr(num-1) + args[0][:num-1]]
+                        result += [bytes([num-1]) + args[0][:num-1]]
                     else:
-                        result += [chr(255) + args[0][:num-1]]
+                        result += [bytes([255]) + args[0][:num-1]]
                 args.pop(0)
             else:
                 raise StructError("arg for string format not a string")
@@ -360,7 +360,7 @@
         i += 1
     if len(args) != 0:
         raise StructError("too many arguments for pack format")
-    return ''.join(result)
+    return b''.join(result)
 
 def unpack(fmt,data):
     """unpack(fmt, string) -> (v1, v2, ...)
@@ -392,7 +392,7 @@
             result.append(data[j:j+num])
             j += num
         elif cur == 'p':
-            n=ord(data[j])
+            n=data[j]
             if n >= num:
                 n = num-1
             result.append(data[j+1:j+n+1])
diff --git a/pypy/module/struct/formatiterator.py b/pypy/module/struct/formatiterator.py
--- a/pypy/module/struct/formatiterator.py
+++ b/pypy/module/struct/formatiterator.py
@@ -116,7 +116,7 @@
 
     def accept_str_arg(self):
         w_obj = self.accept_obj_arg()
-        return self.space.str_w(w_obj)
+        return self.space.bytes_w(w_obj)
 
     def accept_unicode_arg(self):
         w_obj = self.accept_obj_arg()
@@ -163,4 +163,7 @@
 
     @specialize.argtype(1)
     def appendobj(self, value):
-        self.result_w.append(self.space.wrap(value))
+        if isinstance(value, str):
+            self.result_w.append(self.space.wrapbytes(value))
+        else:
+            self.result_w.append(self.space.wrap(value))
diff --git a/pypy/module/struct/interp_struct.py b/pypy/module/struct/interp_struct.py
--- a/pypy/module/struct/interp_struct.py
+++ b/pypy/module/struct/interp_struct.py
@@ -22,7 +22,7 @@
     except StructError, e:
         raise e.at_applevel(space)
     result = ''.join(fmtiter.result)
-    return space.wrap(result)
+    return space.wrapbytes(result)
 
 
 @unwrap_spec(format=str, input='bufferstr')
diff --git a/pypy/module/struct/test/test_struct.py b/pypy/module/struct/test/test_struct.py
--- a/pypy/module/struct/test/test_struct.py
+++ b/pypy/module/struct/test/test_struct.py
@@ -78,13 +78,13 @@
         Check packing with the '<' format specifier.
         """
         pack = self.struct.pack
-        assert pack("<i", 0x41424344) == 'DCBA'
-        assert pack("<i", -3) == '\xfd\xff\xff\xff'
-        assert pack("<i", -2147483648) == '\x00\x00\x00\x80'
-        assert pack("<I", 0x81424344) == 'DCB\x81'
-        assert pack("<q", 0x4142434445464748) == 'HGFEDCBA'
-        assert pack("<q", -0x41B2B3B4B5B6B7B8) == 'HHIJKLM\xbe'
-        assert pack("<Q", 0x8142434445464748) == 'HGFEDCB\x81'
+        assert pack("<i", 0x41424344) == b'DCBA'
+        assert pack("<i", -3) == b'\xfd\xff\xff\xff'
+        assert pack("<i", -2147483648) == b'\x00\x00\x00\x80'
+        assert pack("<I", 0x81424344) == b'DCB\x81'
+        assert pack("<q", 0x4142434445464748) == b'HGFEDCBA'
+        assert pack("<q", -0x41B2B3B4B5B6B7B8) == b'HHIJKLM\xbe'
+        assert pack("<Q", 0x8142434445464748) == b'HGFEDCB\x81'
 
 
     def test_unpack_standard_little(self):
@@ -92,13 +92,13 @@
         Check unpacking with the '<' format specifier.
         """
         unpack = self.struct.unpack
-        assert unpack("<i", 'DCBA') == (0x41424344,)
-        assert unpack("<i", '\xfd\xff\xff\xff') == (-3,)
-        assert unpack("<i", '\x00\x00\x00\x80') == (-2147483648,)
-        assert unpack("<I", 'DCB\x81') == (0x81424344,)
-        assert unpack("<q", 'HGFEDCBA') == (0x4142434445464748,)
-        assert unpack("<q", 'HHIJKLM\xbe') == (-0x41B2B3B4B5B6B7B8,)
-        assert unpack("<Q", 'HGFEDCB\x81') == (0x8142434445464748,)
+        assert unpack("<i", b'DCBA') == (0x41424344,)
+        assert unpack("<i", b'\xfd\xff\xff\xff') == (-3,)
+        assert unpack("<i", b'\x00\x00\x00\x80') == (-2147483648,)
+        assert unpack("<I", b'DCB\x81') == (0x81424344,)
+        assert unpack("<q", b'HGFEDCBA') == (0x4142434445464748,)
+        assert unpack("<q", b'HHIJKLM\xbe') == (-0x41B2B3B4B5B6B7B8,)
+        assert unpack("<Q", b'HGFEDCB\x81') == (0x8142434445464748,)
 
 
     def test_pack_standard_big(self):
@@ -106,13 +106,13 @@
         Check packing with the '>' format specifier.
         """
         pack = self.struct.pack
-        assert pack(">i", 0x41424344) == 'ABCD'
-        assert pack(">i", -3) == '\xff\xff\xff\xfd'
-        assert pack(">i", -2147483648) == '\x80\x00\x00\x00'
-        assert pack(">I", 0x81424344) == '\x81BCD'
-        assert pack(">q", 0x4142434445464748) == 'ABCDEFGH'
-        assert pack(">q", -0x41B2B3B4B5B6B7B8) == '\xbeMLKJIHH'
-        assert pack(">Q", 0x8142434445464748) == '\x81BCDEFGH'
+        assert pack(">i", 0x41424344) == b'ABCD'
+        assert pack(">i", -3) == b'\xff\xff\xff\xfd'
+        assert pack(">i", -2147483648) == b'\x80\x00\x00\x00'
+        assert pack(">I", 0x81424344) == b'\x81BCD'
+        assert pack(">q", 0x4142434445464748) == b'ABCDEFGH'
+        assert pack(">q", -0x41B2B3B4B5B6B7B8) == b'\xbeMLKJIHH'
+        assert pack(">Q", 0x8142434445464748) == b'\x81BCDEFGH'
 
 
     def test_unpack_standard_big(self):
@@ -120,13 +120,13 @@
         Check unpacking with the '>' format specifier.
         """
         unpack = self.struct.unpack
-        assert unpack(">i", 'ABCD') == (0x41424344,)
-        assert unpack(">i", '\xff\xff\xff\xfd') == (-3,)
-        assert unpack(">i", '\x80\x00\x00\x00') == (-2147483648,)
-        assert unpack(">I", '\x81BCD') == (0x81424344,)
-        assert unpack(">q", 'ABCDEFGH') == (0x4142434445464748,)
-        assert unpack(">q", '\xbeMLKJIHH') == (-0x41B2B3B4B5B6B7B8,)
-        assert unpack(">Q", '\x81BCDEFGH') == (0x8142434445464748,)
+        assert unpack(">i", b'ABCD') == (0x41424344,)
+        assert unpack(">i", b'\xff\xff\xff\xfd') == (-3,)
+        assert unpack(">i", b'\x80\x00\x00\x00') == (-2147483648,)
+        assert unpack(">I", b'\x81BCD') == (0x81424344,)
+        assert unpack(">q", b'ABCDEFGH') == (0x4142434445464748,)
+        assert unpack(">q", b'\xbeMLKJIHH') == (-0x41B2B3B4B5B6B7B8,)
+        assert unpack(">Q", b'\x81BCDEFGH') == (0x8142434445464748,)
 
 
     def test_calcsize_native(self):
@@ -168,13 +168,13 @@
         sizeofi = calcsize("i")
         res = pack("bi", -2, 5)
         assert len(res) == 2 * sizeofi
-        assert res[0] == '\xfe'
-        assert res[1:sizeofi] == '\x00' * (sizeofi-1)    # padding
+        assert res[0] == 0xfe
+        assert res[1:sizeofi] == b'\x00' * (sizeofi-1)    # padding
         if self.native_is_bigendian:
-            assert res[sizeofi:] == '\x00' * (sizeofi-1) + '\x05'
+            assert res[sizeofi:] == b'\x00' * (sizeofi-1) + b'\x05'
         else:
-            assert res[sizeofi:] == '\x05' + '\x00' * (sizeofi-1)
-        assert pack("q", -1) == '\xff' * calcsize("q")
+            assert res[sizeofi:] == b'\x05' + b'\x00' * (sizeofi-1)
+        assert pack("q", -1) == b'\xff' * calcsize("q")
 
 
     def test_unpack_native(self):
@@ -185,7 +185,7 @@
         pack = self.struct.pack
         unpack = self.struct.unpack
         assert unpack("bi", pack("bi", -2, 5)) == (-2, 5)
-        assert unpack("q", '\xff' * calcsize("q")) == (-1,)
+        assert unpack("q", b'\xff' * calcsize("q")) == (-1,)
 
 
     def test_string_format(self):
@@ -194,13 +194,13 @@
         """
         pack = self.struct.pack
         unpack = self.struct.unpack
-        assert pack("7s", "hello") == "hello\x00\x00"
-        assert pack("5s", "world") == "world"
-        assert pack("3s", "spam") == "spa"
-        assert pack("0s", "foo") == ""
-        assert unpack("7s", "hello\x00\x00") == ("hello\x00\x00",)
-        assert unpack("5s3s", "worldspa") == ("world", "spa")
-        assert unpack("0s", "") == ("",)
+        assert pack("7s", b"hello") == b"hello\x00\x00"
+        assert pack("5s", b"world") == b"world"
+        assert pack("3s", b"spam") == b"spa"
+        assert pack("0s", b"foo") == b""
+        assert unpack("7s", b"hello\x00\x00") == (b"hello\x00\x00",)
+        assert unpack("5s3s", b"worldspa") == (b"world", b"spa")
+        assert unpack("0s", b"") == (b"",)
 
 
     def test_pascal_format(self):
@@ -209,17 +209,17 @@
         """
         pack = self.struct.pack
         unpack = self.struct.unpack
-        longstring = str(range(70))     # this has 270 chars
-        longpacked300 = "\xff" + longstring + "\x00" * (299-len(longstring))
-        assert pack("8p", "hello") == "\x05hello\x00\x00"
-        assert pack("6p", "world") == "\x05world"
-        assert pack("4p", "spam") == "\x03spa"
-        assert pack("1p", "foo") == "\x00"
-        assert pack("10p", longstring) == "\x09" + longstring[:9]
+        longstring = bytes(range(135)) * 2    # this has 270 chars
+        longpacked300 = b"\xff" + longstring + b"\x00" * (299-len(longstring))
+        assert pack("8p", b"hello") == b"\x05hello\x00\x00"
+        assert pack("6p", b"world") == b"\x05world"
+        assert pack("4p", b"spam") == b"\x03spa"
+        assert pack("1p", b"foo") == b"\x00"
+        assert pack("10p", longstring) == b"\x09" + longstring[:9]
         assert pack("300p", longstring) == longpacked300
-        assert unpack("8p", "\x05helloxx") == ("hello",)
-        assert unpack("5p", "\x80abcd") == ("abcd",)
-        assert unpack("1p", "\x03") == ("",)
+        assert unpack("8p", b"\x05helloxx") == (b"hello",)
+        assert unpack("5p", b"\x80abcd") == (b"abcd",)
+        assert unpack("1p", b"\x03") == (b"",)
         assert unpack("300p", longpacked300) == (longstring[:255],)
 
 
@@ -229,10 +229,10 @@
         """
         pack = self.struct.pack
         unpack = self.struct.unpack
-        assert pack("c", "?") == "?"
-        assert pack("5c", "a", "\xc0", "\x00", "\n", "-") == "a\xc0\x00\n-"
-        assert unpack("c", "?") == ("?",)
-        assert unpack("5c", "a\xc0\x00\n-") == ("a", "\xc0", "\x00", "\n", "-")
+        assert pack("c", b"?") == b"?"
+        assert pack("5c", b"a", b"\xc0", b"\x00", b"\n", b"-") == b"a\xc0\x00\n-"
+        assert unpack("c", b"?") == (b"?",)
+        assert unpack("5c", b"a\xc0\x00\n-") == (b"a", b"\xc0", b"\x00", b"\n", b"-")
 
 
     def test_pad_format(self):
@@ -241,10 +241,10 @@
         """
         pack = self.struct.pack
         unpack = self.struct.unpack
-        assert pack("x") == "\x00"
-        assert pack("5x") == "\x00" * 5
-        assert unpack("x", "?") == ()
-        assert unpack("5x", "hello") == ()
+        assert pack("x") == b"\x00"
+        assert pack("5x") == b"\x00" * 5
+        assert unpack("x", b"?") == ()
+        assert unpack("5x", b"hello") == ()
 
 
     def test_native_floats(self):
@@ -270,28 +270,28 @@
         """
         pack = self.struct.pack
         unpack = self.struct.unpack
-        assert pack("!d", 12.5) == '@)\x00\x00\x00\x00\x00\x00'
-        assert pack("<d", -12.5) == '\x00\x00\x00\x00\x00\x00)\xc0'
-        assert unpack("!d", '\xc0)\x00\x00\x00\x00\x00\x00') == (-12.5,)
-        assert unpack("<d", '\x00\x00\x00\x00\x00\x00)@') == (12.5,)
-        assert pack("!f", -12.5) == '\xc1H\x00\x00'
-        assert pack("<f", 12.5) == '\x00\x00HA'
-        assert unpack("!f", 'AH\x00\x00') == (12.5,)
-        assert unpack("<f", '\x00\x00H\xc1') == (-12.5,)
+        assert pack("!d", 12.5) == b'@)\x00\x00\x00\x00\x00\x00'
+        assert pack("<d", -12.5) == b'\x00\x00\x00\x00\x00\x00)\xc0'
+        assert unpack("!d", b'\xc0)\x00\x00\x00\x00\x00\x00') == (-12.5,)
+        assert unpack("<d", b'\x00\x00\x00\x00\x00\x00)@') == (12.5,)
+        assert pack("!f", -12.5) == b'\xc1H\x00\x00'
+        assert pack("<f", 12.5) == b'\x00\x00HA'
+        assert unpack("!f", b'AH\x00\x00') == (12.5,)
+        assert unpack("<f", b'\x00\x00H\xc1') == (-12.5,)
         raises(OverflowError, pack, "<f", 10e100)
 
     def test_bool(self):
         pack = self.struct.pack
         unpack = self.struct.unpack
-        assert pack("!?", True) == '\x01'
-        assert pack(">?", True) == '\x01'
-        assert pack("!?", False) == '\x00'
-        assert pack(">?", False) == '\x00'
-        assert pack("@?", True) == '\x01'
-        assert pack("@?", False) == '\x00'
+        assert pack("!?", True) == b'\x01'
+        assert pack(">?", True) == b'\x01'
+        assert pack("!?", False) == b'\x00'
+        assert pack(">?", False) == b'\x00'
+        assert pack("@?", True) == b'\x01'
+        assert pack("@?", False) == b'\x00'
 
     def test_transitiveness(self):
-        c = 'a'
+        c = b'a'
         b = 1
         h = 255
         i = 65535
@@ -333,15 +333,15 @@
         raises(error, calcsize, "!P")   # bad char in struct format
         raises(error, pack, "ii", 15)   # struct format requires more arguments
         raises(error, pack, "i", 3, 4)  # too many arguments for struct format
-        raises(error, unpack, "ii", "?")# unpack str size too short for format
-        raises(error, unpack, "b", "??")# unpack str size too long for format
-        raises(error, pack, "c", "foo") # expected a string of length 1
+        raises(error, unpack, "ii", b"?")# unpack str size too short for format
+        raises(error, unpack, "b", b"??")# unpack str size too long for format
+        raises(error, pack, "c", b"foo") # expected a string of length 1
         try:
             pack("0p")                  # bad '0p' in struct format
         except error:                   # (but ignored on CPython)
             pass
         if '__pypy__' in sys.builtin_module_names:
-            raises(error, unpack, "0p", "")   # segfaults on CPython 2.5.2!
+            raises(error, unpack, "0p", b"")   # segfaults on CPython 2.5.2!
         raises(error, pack, "b", 150)   # argument out of range
         # XXX the accepted ranges still differs between PyPy and CPython
 
@@ -373,7 +373,7 @@
         if '__pypy__' not in sys.builtin_module_names:
             skip("PyPy extension")
         data = self.struct.pack("uuu", u'X', u'Y', u'Z')
-        assert data == str(buffer(u'XYZ'))
+        assert data == bytes(buffer(u'XYZ'))
         assert self.struct.unpack("uuu", data) == (u'X', u'Y', u'Z')
 
 
@@ -406,9 +406,9 @@
         b = self.bytebuffer(19)
         sz = self.struct.calcsize("ii")
         self.struct.pack_into("ii", b, 2, 17, 42)
-        assert b[:] == ('\x00' * 2 +
+        assert b[:] == (b'\x00' * 2 +
                         self.struct.pack("ii", 17, 42) +
-                        '\x00' * (19-sz-2))
+                        b'\x00' * (19-sz-2))
 
     def test_unpack_from(self):
         b = self.bytebuffer(19)

From noreply at buildbot.pypy.org  Wed Oct 19 01:43:26 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 19 Oct 2011 01:43:26 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Various fixes for the array module
Message-ID: <20111018234326.79CD1820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48222:1c7672de6deb
Date: 2011-10-19 00:57 +0200
http://bitbucket.org/pypy/pypy/changeset/1c7672de6deb/

Log:	Various fixes for the array module

diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -487,7 +487,7 @@
         s = rffi.charp2str(charp_addr)
     else:
         s = rffi.charp2strn(charp_addr, maxlength)
-    return space.wrap(s)
+    return space.wrapbytes(s)
 
 @unwrap_spec(address=r_uint, maxlength=int)
 def wcharp2unicode(space, address, maxlength=-1):
@@ -505,7 +505,7 @@
     if maxlength == -1:
         return charp2string(space, address)
     s = rffi.charpsize2str(rffi.cast(rffi.CCHARP, address), maxlength)
-    return space.wrap(s)
+    return space.wrapbytes(s)
 
 @unwrap_spec(address=r_uint, maxlength=int)
 def wcharp2rawunicode(space, address, maxlength=-1):
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -36,15 +36,15 @@
 
             if len(__args__.arguments_w) > 0:
                 w_initializer = __args__.arguments_w[0]
-                if space.type(w_initializer) is space.w_str:
-                    a.fromstring(space.str_w(w_initializer))
+                if space.type(w_initializer) is space.w_bytes:
+                    a.fromstring(space.bytes_w(w_initializer))
                 elif space.type(w_initializer) is space.w_list:
                     a.fromlist(w_initializer)
                 else:
                     a.extend(w_initializer, True)
             break
     else:
-        msg = 'bad typecode (must be c, b, B, u, h, H, i, I, l, L, f or d)'
+        msg = 'bad typecode (must be b, B, u, h, H, i, I, l, L, f or d)'
         raise OperationError(space.w_ValueError, space.wrap(msg))
 
     return a
@@ -63,6 +63,7 @@
 array_tolist = SMM('tolist', 1)
 array_fromlist = SMM('fromlist', 2)
 array_tostring = SMM('tostring', 1)
+array_tobytes = SMM('tobytes', 1)
 array_fromstring = SMM('fromstring', 2)
 array_tounicode = SMM('tounicode', 1)
 array_fromunicode = SMM('fromunicode', 2)
@@ -123,7 +124,6 @@
         return True
 
 types = {
-    'c': TypeCode(lltype.Char,        'str_w'),
     'u': TypeCode(lltype.UniChar,     'unicode_w'),
     'b': TypeCode(rffi.SIGNEDCHAR,    'int_w', True, True),
     'B': TypeCode(rffi.UCHAR,         'int_w', True),
@@ -513,16 +513,18 @@
         self.fromlist(w_lst)
 
     def array_fromstring__Array_ANY(space, self, w_s):
-        self.fromstring(space.str_w(w_s))
+        self.fromstring(space.bytes_w(w_s))
+
+    def array_tobytes__Array(space, self):
+        cbuf = self.charbuf()
+        return self.space.wrapbytes(rffi.charpsize2str(cbuf, self.len * mytype.bytes))
 
     def array_tostring__Array(space, self):
-        cbuf = self.charbuf()
-        return self.space.wrap(rffi.charpsize2str(cbuf, self.len * mytype.bytes))
+        space.warn("tostring() is deprecated. Use tobytes() instead.",
+                   space.w_DeprecationWarning)
+        return array_tobytes__Array(space, self)
 
     def array_fromfile__Array_ANY_ANY(space, self, w_f, w_n):
-        if not isinstance(w_f, W_File):
-            msg = "arg1 must be open file"
-            raise OperationError(space.w_TypeError, space.wrap(msg))
         n = space.int_w(w_n)
 
         try:
@@ -530,23 +532,20 @@
         except OverflowError:
             raise MemoryError
         w_item = space.call_method(w_f, 'read', space.wrap(size))
-        item = space.str_w(w_item)
+        item = space.bytes_w(w_item)
         if len(item) < size:
             n = len(item) % self.itemsize
             elems = max(0, len(item) - (len(item) % self.itemsize))
             if n != 0:
                 item = item[0:elems]
-            w_item = space.wrap(item)
+            w_item = space.wrapbytes(item)
             array_fromstring__Array_ANY(space, self, w_item)
             msg = "not enough items in file"
             raise OperationError(space.w_EOFError, space.wrap(msg))
         array_fromstring__Array_ANY(space, self, w_item)
 
     def array_tofile__Array_ANY(space, self, w_f):
-        if not isinstance(w_f, W_File):
-            msg = "arg1 must be open file"
-            raise OperationError(space.w_TypeError, space.wrap(msg))
-        w_s = array_tostring__Array(space, self)
+        w_s = array_tobytes__Array(space, self)
         space.call_method(w_f, 'write', w_s)
 
     if mytype.typecode == 'u':
@@ -593,7 +592,7 @@
 
     def array_reduce__Array(space, self):
         if self.len > 0:
-            w_s = array_tostring__Array(space, self)
+            w_s = array_tobytes__Array(space, self)
             args = [space.wrap(mytype.typecode), w_s]
         else:
             args = [space.wrap(mytype.typecode)]
@@ -631,10 +630,6 @@
     def repr__Array(space, self):
         if self.len == 0:
             return space.wrap("array('%s')" % self.typecode)
-        elif self.typecode == "c":
-            r = space.repr(array_tostring__Array(space, self))
-            s = "array('%s', %s)" % (self.typecode, space.str_w(r))
-            return space.wrap(s)
         elif self.typecode == "u":
             r = space.repr(array_tounicode__Array(space, self))
             s = "array('%s', %s)" % (self.typecode, space.str_w(r))
diff --git a/pypy/module/array/test/test_array.py b/pypy/module/array/test/test_array.py
--- a/pypy/module/array/test/test_array.py
+++ b/pypy/module/array/test/test_array.py
@@ -22,40 +22,20 @@
 
     
     def test_ctor(self):
-        assert len(self.array('c')) == 0
         assert len(self.array('i')) == 0
 
         raises(TypeError, self.array, 'hi')
         raises(TypeError, self.array, 1)
         raises(ValueError, self.array, 'q')
 
-        a = self.array('c')
+        a = self.array('u')
         raises(TypeError, a.append, 7)
-        raises(TypeError, a.append, 'hi')
+        raises(TypeError, a.append, u'hi')
         a.append('h')
         assert a[0] == 'h'
         assert type(a[0]) is str
         assert len(a) == 1
 
-        a = self.array('u')
-        raises(TypeError, a.append, 7)
-        raises(TypeError, a.append, u'hi')
-        a.append(unicode('h'))
-        assert a[0] == unicode('h')
-        assert type(a[0]) is unicode
-        assert len(a) == 1
-
-        a = self.array('c', ('a', 'b', 'c'))
-        assert a[0] == 'a'
-        assert a[1] == 'b'
-        assert a[2] == 'c'
-        assert len(a) == 3
-
-        b = self.array('c', a)
-        assert len(b) == 3
-        assert a == b
-        raises(TypeError, self.array, 'i', a)
-
         a = self.array('i', (1, 2, 3))
         b = self.array('h', (1, 2, 3))
         assert a == b
@@ -85,7 +65,7 @@
                            ('H', (     0, 56783, 65535),  int),
                            ('i', (-32768, 30535, 32767),  int),
                            ('I', (     0, 56783, 65535), long),
-                           ('l', (-2 ** 32 / 2, 34, 2 ** 32 / 2 - 1),  int),
+                           ('l', (-2 ** 32 // 2, 34, 2 ** 32 // 2 - 1),  int),
                            ('L', (0, 3523532, 2 ** 32 - 1), long),
                            ):
             a = self.array(tc, ok)
@@ -122,7 +102,7 @@
             assert a.tolist() == vals
 
             a = self.array(tc.lower())
-            vals = [-1 * (2 ** a.itemsize) / 2,  (2 ** a.itemsize) / 2 - 1]
+            vals = [-1 * (2 ** a.itemsize) // 2,  (2 ** a.itemsize) // 2 - 1]
             a.fromlist(vals)
             assert a.tolist() == vals
 
@@ -141,7 +121,7 @@
             assert len(a) == len(values)
 
     def test_itemsize(self):
-        for t in 'cbB':
+        for t in 'bB':
             assert(self.array(t).itemsize >= 1)
         for t in 'uhHiI':
             assert(self.array(t).itemsize >= 2)
@@ -154,11 +134,11 @@
         for t in inttypes:
             a = self.array(t, [1, 2, 3])
             b = a.itemsize
-            for v in (-2 ** (8 * b) / 2, 2 ** (8 * b) / 2 - 1):
+            for v in (-2 ** (8 * b) // 2, 2 ** (8 * b) // 2 - 1):
                 a[1] = v
                 assert a[0] == 1 and a[1] == v and a[2] == 3
-            raises(OverflowError, a.append, -2 ** (8 * b) / 2 - 1)
-            raises(OverflowError, a.append, 2 ** (8 * b) / 2)
+            raises(OverflowError, a.append, -2 ** (8 * b) // 2 - 1)
+            raises(OverflowError, a.append, 2 ** (8 * b) // 2)
 
             a = self.array(t.upper(), [1, 2, 3])
             b = a.itemsize
@@ -169,20 +149,16 @@
             raises(OverflowError, a.append, 2 ** (8 * b))
 
     def test_fromstring(self):
-        a = self.array('c')
-        a.fromstring('Hi!')
-        assert a[0] == 'H' and a[1] == 'i' and a[2] == '!' and len(a) == 3
-
         for t in 'bBhHiIlLfd':
             a = self.array(t)
-            a.fromstring('\x00' * a.itemsize * 2)
+            a.fromstring(b'\x00' * a.itemsize * 2)
             assert len(a) == 2 and a[0] == 0 and a[1] == 0
             if a.itemsize > 1:
-                raises(ValueError, a.fromstring, '\x00' * (a.itemsize - 1))
-                raises(ValueError, a.fromstring, '\x00' * (a.itemsize + 1))
-                raises(ValueError, a.fromstring, '\x00' * (2 * a.itemsize - 1))
-                raises(ValueError, a.fromstring, '\x00' * (2 * a.itemsize + 1))
-            b = self.array(t, '\x00' * a.itemsize * 2)
+                raises(ValueError, a.fromstring, b'\x00' * (a.itemsize - 1))
+                raises(ValueError, a.fromstring, b'\x00' * (a.itemsize + 1))
+                raises(ValueError, a.fromstring, b'\x00' * (2 * a.itemsize - 1))
+                raises(ValueError, a.fromstring, b'\x00' * (2 * a.itemsize + 1))
+            b = self.array(t, b'\x00' * a.itemsize * 2)
             assert len(b) == 2 and b[0] == 0 and b[1] == 0
 
     def test_fromfile(self):
@@ -194,28 +170,28 @@
         ##     def read(self,n):
         ##         return self.c*min(n,self.s)
         def myfile(c, s):
-            f = open(self.tempfile, 'w')
+            f = open(self.tempfile, 'wb')
             f.write(c * s)
             f.close()
-            return open(self.tempfile, 'r')
+            return open(self.tempfile, 'rb')
 
-        f = myfile('\x00', 100)
+        f = myfile(b'\x00', 100)
         for t in 'bBhHiIlLfd':
             a = self.array(t)
             a.fromfile(f, 2)
             assert len(a) == 2 and a[0] == 0 and a[1] == 0
 
         a = self.array('b')
-        a.fromfile(myfile('\x01', 20), 2)
+        a.fromfile(myfile(b'\x01', 20), 2)
         assert len(a) == 2 and a[0] == 1 and a[1] == 1
 
         a = self.array('h')
-        a.fromfile(myfile('\x01', 20), 2)
+        a.fromfile(myfile(b'\x01', 20), 2)
         assert len(a) == 2 and a[0] == 257 and a[1] == 257
 
         for i in (0, 1):
             a = self.array('h')
-            raises(EOFError, a.fromfile, myfile('\x01', 2 + i), 2)
+            raises(EOFError, a.fromfile, myfile(b'\x01', 2 + i), 2)
             assert len(a) == 1 and a[0] == 257
 
     def test_fromlist(self):
@@ -255,12 +231,12 @@
         assert repr(a) == "array('b', [1, 2, 1, 2])"
 
     def test_fromunicode(self):
-        raises(ValueError, self.array('i').fromunicode, unicode('hi'))
+        raises(ValueError, self.array('i').fromunicode, 'hi')
         a = self.array('u')
-        a.fromunicode(unicode('hi'))
+        a.fromunicode('hi')
         assert len(a) == 2 and a[0] == 'h' and a[1] == 'i'
 
-        b = self.array('u', unicode('hi'))
+        b = self.array('u', 'hi')
         assert len(b) == 2 and b[0] == 'h' and b[1] == 'i'
 
     def test_sequence(self):
@@ -380,24 +356,23 @@
         assert type(l) is list and len(l) == 3
         assert a[0] == 1 and a[1] == 2 and a[2] == 3
 
-        b = self.array('i', a.tostring())
+        b = self.array('i', a.tobytes())
         assert len(b) == 3 and b[0] == 1 and b[1] == 2 and b[2] == 3
 
-        assert self.array('c', ('h', 'i')).tostring() == 'hi'
         a = self.array('i', [0, 0, 0])
-        assert a.tostring() == '\x00' * 3 * a.itemsize
+        assert a.tobytes() == b'\x00' * 3 * a.itemsize
 
-        s = self.array('i', [1, 2, 3]).tostring()
-        assert '\x00' in s
-        assert '\x01' in s
-        assert '\x02' in s
-        assert '\x03' in s
+        s = self.array('i', [1, 2, 3]).tobytes()
+        assert 0x00 in s
+        assert 0x01 in s
+        assert 0x02 in s
+        assert 0x03 in s
         a = self.array('i', s)
         assert a[0] == 1 and a[1] == 2 and a[2] == 3
 
         from struct import unpack
         values = (-129, 128, -128, 127, 0, 255, -1, 256, -32760, 32760)
-        s = self.array('i', values).tostring()
+        s = self.array('i', values).tobytes()
         fmt = 'i' * len(values)
         a = unpack(fmt, s)
         assert a == values
@@ -406,28 +381,27 @@
                                ('BHILfd', (127, 0, 1, 7, 255, 169)),
                                ('hilHILfd', (32760, 30123, 3422, 23244))):
             for tc in tcodes:
-                values += ((2 ** self.array(tc).itemsize) / 2 - 1, )
-                s = self.array(tc, values).tostring()
+                values += ((2 ** self.array(tc).itemsize) // 2 - 1, )
+                s = self.array(tc, values).tobytes()
                 a = unpack(tc * len(values), s)
                 assert a == values
 
-        f = open(self.tempfile, 'w')
-        self.array('c', ('h', 'i')).tofile(f)
+        f = open(self.tempfile, 'wb')
+        self.array('b', (ord('h'), ord('i'))).tofile(f)
         f.close()
-        assert open(self.tempfile, 'r').readline() == 'hi'
+        assert open(self.tempfile, 'rb').readline() == b'hi'
 
-        a = self.array('c')
-        a.fromfile(open(self.tempfile, 'r'), 2)
-        assert repr(a) == "array('c', 'hi')"
+        a = self.array('b')
+        a.fromfile(open(self.tempfile, 'rb'), 2)
+        assert repr(a) == "array('b', [104, 105])"
 
         raises(ValueError, self.array('i').tounicode)
-        assert self.array('u', unicode('hello')).tounicode() == \
-               unicode('hello')
+        assert self.array('u', 'hello').tounicode() == 'hello'
 
     def test_buffer(self):
-        a = self.array('h', 'Hi')
+        a = self.array('h', b'Hi')
         buf = buffer(a)
-        assert buf[1] == 'i'
+        assert buf[1] == b'i'
         #raises(TypeError, buf.__setitem__, 1, 'o')
 
     def test_list_methods(self):
@@ -482,8 +456,7 @@
             pass
         
         for v1, v2, tt in (([1, 2, 3], [1, 3, 2], 'bhilBHIL'),
-                         ('abc', 'acb', 'c'),
-                         (unicode('abc'), unicode('acb'), 'u')):
+                         ('abc', 'acb', 'u')):
             for t in tt:
                 a = self.array(t, v1)
                 b = self.array(t, v1)
@@ -677,7 +650,7 @@
             def __len__(self):
                 return 3
 
-            def next(self):
+            def __next__(self):
                 self.n -= 1
                 if self.n < 0:
                     raise StopIteration
@@ -697,7 +670,7 @@
         assert repr(a) == "array('i', [4, 3, 2, 1, 0])"
 
     def test_type(self):
-        for t in 'bBhHiIlLfdcu':
+        for t in 'bBhHiIlLfdu':
             assert type(self.array(t)) is self.array
             assert isinstance(self.array(t), self.array)
 
@@ -727,7 +700,8 @@
                 self.height = height
                 return self
 
-            def _index(self, (x,y)):
+            def _index(self, xy):
+                x, y = xy
                 x = min(max(x, 0), self.width-1)
                 y = min(max(y, 0), self.height-1)
                 return y * self.width + x
@@ -755,7 +729,7 @@
                 self.append(7)
 
             def fromstring(self, lst):
-                self.append('8')
+                self.append(8)
 
             def fromunicode(self, lst):
                 self.append(u'9')
@@ -763,8 +737,7 @@
             def extend(self, lst):
                 self.append(10)
 
-        assert repr(mya('c', 'hi')) == "array('c', 'hi')"
-        assert repr(mya('u', u'hi')) == "array('u', u'hi')"
+        assert repr(mya('u', u'hi')) == "array('u', 'hi')"
         assert repr(mya('i', [1, 2, 3])) == "array('i', [1, 2, 3])"
         assert repr(mya('i', (1, 2, 3))) == "array('i', [1, 2, 3])"
 
@@ -772,13 +745,13 @@
         a.fromlist([1, 2, 3])
         assert repr(a) == "array('i', [7])"
 
-        a = mya('c')
-        a.fromstring('hi')
-        assert repr(a) == "array('c', '8')"
+        a = mya('b')
+        a.fromstring(b'hi')
+        assert repr(a) == "array('b', [8])"
 
         a = mya('u')
         a.fromunicode(u'hi')
-        assert repr(a) == "array('u', u'9')"
+        assert repr(a) == "array('u', '9')"
 
         a = mya('i')
         a.extend([1, 2, 3])
@@ -789,34 +762,33 @@
             def tolist(self):
                 return 'list'
 
-            def tostring(self):
+            def tobytes(self):
                 return 'str'
 
             def tounicode(self):
                 return 'unicode'
 
         assert mya('i', [1, 2, 3]).tolist() == 'list'
-        assert mya('c', 'hi').tostring() == 'str'
-        assert mya('u', u'hi').tounicode() == 'unicode'
+        assert mya('u', 'hi').tobytes() == 'str'
+        assert mya('u', 'hi').tounicode() == 'unicode'
 
-        assert repr(mya('c', 'hi')) == "array('c', 'hi')"
-        assert repr(mya('u', u'hi')) == "array('u', u'hi')"
+        assert repr(mya('u', u'hi')) == "array('u', 'hi')"
         assert repr(mya('i', [1, 2, 3])) == "array('i', [1, 2, 3])"
         assert repr(mya('i', (1, 2, 3))) == "array('i', [1, 2, 3])"
 
     def test_unicode_outofrange(self):
-        a = self.array('u', unicode(r'\x01\u263a\x00\ufeff', 'unicode-escape'))
-        b = self.array('u', unicode(r'\x01\u263a\x00\ufeff', 'unicode-escape'))
+        a = self.array('u', '\x01\u263a\x00\ufeff')
+        b = self.array('u', '\x01\u263a\x00\ufeff')
         b.byteswap()
         assert a != b
 
     def test_weakref(self):
         import weakref
-        a = self.array('c', 'Hi!')
+        a = self.array('u', 'Hi!')
         r = weakref.ref(a)
         assert r() is a
 
-class TestCPythonsOwnArray(BaseArrayTests):
+class DontTestCPythonsOwnArray(BaseArrayTests):
 
     def setup_class(cls):
         import array
@@ -842,13 +814,13 @@
 
 
     def test_buffer_info(self):
-        a = self.array('c', 'Hi!')
+        a = self.array('b', b'Hi!')
         bi = a.buffer_info()
         assert bi[0] != 0
         assert bi[1] == 3
         import _rawffi
         data = _rawffi.charp2string(bi[0])
-        assert data[0:3] == 'Hi!'
+        assert data[0:3] == b'Hi!'
 
     def test_array_reverse_slice_assign_self(self):
         a = self.array('b', range(4))
diff --git a/pypy/module/array/test/test_array_old.py b/pypy/module/array/test/test_array_old.py
--- a/pypy/module/array/test/test_array_old.py
+++ b/pypy/module/array/test/test_array_old.py
@@ -12,9 +12,9 @@
     native_sizes = {'l': struct.calcsize('l')}
 
     def test_attributes(self):
-        a = self.array.array('c')
-        assert a.typecode == 'c'
-        assert a.itemsize == 1
+        a = self.array.array('u')
+        assert a.typecode == 'u'
+        assert a.itemsize == 4
         a = self.array.array('l')
         assert a.typecode == 'l'
         assert a.itemsize == self.native_sizes['l']
@@ -26,9 +26,9 @@
 
     def test_unicode(self):
         a = self.array.array('u')
-        a.fromunicode(unichr(9999))
+        a.fromunicode(chr(9999))
         assert len(a) == 1
-        assert a.tolist() == [unichr(9999)]
+        assert a.tolist() == [chr(9999)]
 
     def test_pickle(self):
         import sys
@@ -51,11 +51,11 @@
                 self.args = args
                 self.kwds = kwds
 
-        a = A('c', foo='bar')
-        assert a.args == ('c',)
+        a = A('u', foo='bar')
+        assert a.args == ('u',)
         assert a.kwds == {'foo': 'bar'}
-        a = A('i', range(10), some=42)
-        assert a.args == ('i', range(10))
+        a = A('i', list(range(10)), some=42)
+        assert a.args == ('i', list(range(10)))
         assert a.kwds == {'some': 42}
         raises(TypeError, A)
         raises(TypeError, A, 42)
@@ -63,7 +63,7 @@
         raises(TypeError, self.array.array, 'i', [], foo='bar')
 
 
-class TestCPythonsOwnArray(BaseArrayTests):
+class DontTestCPythonsOwnArray(BaseArrayTests):
 
     def setup_class(cls):
         import array
diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py
--- a/pypy/objspace/std/stringobject.py
+++ b/pypy/objspace/std/stringobject.py
@@ -8,6 +8,7 @@
 from pypy.objspace.std.inttype import wrapint
 from pypy.objspace.std.sliceobject import W_SliceObject, normalize_simple_slice
 from pypy.objspace.std import slicetype, newformat
+from pypy.objspace.std.intobject import W_IntObject
 from pypy.objspace.std.listobject import W_ListObject
 from pypy.objspace.std.noneobject import W_NoneObject
 from pypy.objspace.std.tupleobject import W_TupleObject
@@ -435,6 +436,11 @@
     sub = w_sub._value
     return space.newbool(self.find(sub) >= 0)
 
+def contains__String_Int(space, w_self, w_char):
+    self = w_self._value
+    char = w_char.intval
+    return space.newbool(self.find(chr(char)) >= 0)
+
 def str_find__String_String_ANY_ANY(space, w_self, w_sub, w_start, w_end):
     (self, sub, start, end) =  _convert_idx_params(space, w_self, w_sub, w_start, w_end)
     res = self.find(sub, start, end)

From noreply at buildbot.pypy.org  Wed Oct 19 01:43:27 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 19 Oct 2011 01:43:27 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Fixes for the socket module.
Message-ID: <20111018234327.AEE83820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48223:3ca3586ad5f2
Date: 2011-10-19 01:25 +0200
http://bitbucket.org/pypy/pypy/changeset/3ca3586ad5f2/

Log:	Fixes for the socket module.

diff --git a/lib_pypy/array.py b/lib_pypy/array.py
--- a/lib_pypy/array.py
+++ b/lib_pypy/array.py
@@ -106,9 +106,7 @@
         self.itemsize = calcsize(typecode)
         if isinstance(initializer, list):
             self.fromlist(initializer)
-        elif isinstance(initializer, str):
-            self.fromstring(initializer)
-        elif isinstance(initializer, unicode) and self.typecode == "u":
+        elif isinstance(initializer, str) and self.typecode == "u":
             self.fromunicode(initializer)
         else:
             self.extend(initializer)
@@ -188,6 +186,7 @@
 
     def tostring(self):
         return self._data[:]
+    tobytes = tostring
 
     def __buffer__(self):
         return buffer(self._data)
diff --git a/pypy/module/_socket/interp_func.py b/pypy/module/_socket/interp_func.py
--- a/pypy/module/_socket/interp_func.py
+++ b/pypy/module/_socket/interp_func.py
@@ -227,9 +227,9 @@
         buf = rsocket.inet_pton(family, ip)
     except SocketError, e:
         raise converted_error(space, e)
-    return space.wrap(buf)
+    return space.wrapbytes(buf)
 
- at unwrap_spec(family=int, packed=str)
+ at unwrap_spec(family=int, packed="bufferstr")
 def inet_ntop(space, family, packed):
     """inet_ntop(family, packed_ip) -> string formatted IP address
 
@@ -255,11 +255,11 @@
     # host can be None, string or unicode
     if space.is_w(w_host, space.w_None):
         host = None
-    elif space.is_true(space.isinstance(w_host, space.w_str)):
-        host = space.str_w(w_host)
+    elif space.is_true(space.isinstance(w_host, space.w_bytes)):
+        host = space.bytes_w(w_host)
     elif space.is_true(space.isinstance(w_host, space.w_unicode)):
         w_shost = space.call_method(w_host, "encode", space.wrap("idna"))
-        host = space.str_w(w_shost)
+        host = space.bytes_w(w_shost)
     else:
         raise OperationError(space.w_TypeError,
                              space.wrap(
diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -19,8 +19,8 @@
     def __del__(self):
         self.close()
 
-    def accept_w(self, space):
-        """accept() -> (socket object, address info)
+    def _accept_w(self, space):
+        """_accept() -> (socket object, address info)
 
         Wait for an incoming connection.  Return a new socket representing the
         connection, and the address of the client.  For IP sockets, the address
@@ -133,7 +133,7 @@
             except SocketError, e:
                 raise converted_error(space, e)
         buflen = space.int_w(w_buflen)
-        return space.wrap(self.getsockopt(level, optname, buflen))
+        return space.wrapbytes(self.getsockopt(level, optname, buflen))
 
     def gettimeout_w(self, space):
         """gettimeout() -> timeout
@@ -180,7 +180,7 @@
             data = self.recv(buffersize, flags)
         except SocketError, e:
             raise converted_error(space, e)
-        return space.wrap(data)
+        return space.wrapbytes(data)
 
     @unwrap_spec(buffersize='nonnegint', flags=int)
     def recvfrom_w(self, space, buffersize, flags=0):
@@ -194,7 +194,7 @@
                 w_addr = addr.as_object(self.fd, space)
             else:
                 w_addr = space.w_None
-            return space.newtuple([space.wrap(data), w_addr])
+            return space.newtuple([space.wrapbytes(data), w_addr])
         except SocketError, e:
             raise converted_error(space, e)
 
@@ -268,7 +268,7 @@
         try:
             optval = space.int_w(w_optval)
         except:
-            optval = space.str_w(w_optval)
+            optval = space.bytes_w(w_optval)
             try:
                 self.setsockopt(level, optname, optval)
             except SocketError, e:
@@ -462,7 +462,7 @@
 # ____________________________________________________________
 
 socketmethodnames = """
-accept bind close connect connect_ex dup fileno
+_accept bind close connect connect_ex dup fileno
 getpeername getsockname getsockopt gettimeout listen makefile
 recv recvfrom send sendall sendto setblocking
 setsockopt settimeout shutdown _reuse _drop recv_into recvfrom_into
@@ -493,7 +493,7 @@
 
 Methods of socket objects (keyword arguments not allowed):
 
-accept() -- accept a connection, returning new socket and client address
+_accept() -- accept a connection, returning new socket and client address
 bind(addr) -- bind the socket to a local address
 close() -- close the socket
 connect(addr) -- connect the socket to a remote address
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -181,7 +181,7 @@
         ("\x00" * 10 + "\xff\xff\x01\x02\x03\x04", "::ffff:1.2.3.4"),
     ]
     for packed, ip in tests:
-        w_ip = space.appexec([w_socket, space.wrap(packed)],
+        w_ip = space.appexec([w_socket, space.wrapbytes(packed)],
             "(_socket, packed): return _socket.inet_ntop(_socket.AF_INET6, packed)")
         if ip is not None:   # else don't check for the precise representation
             assert space.unwrap(w_ip) == ip
@@ -217,14 +217,14 @@
     assert space.unwrap(res) == socket.has_ipv6
 
 def test_getaddrinfo():
-    host = "localhost"
+    host = b"localhost"
     port = 25
     info = socket.getaddrinfo(host, port)
-    w_l = space.appexec([w_socket, space.wrap(host), space.wrap(port)],
+    w_l = space.appexec([w_socket, space.wrapbytes(host), space.wrap(port)],
                         "(_socket, host, port): return _socket.getaddrinfo(host, port)")
     assert space.unwrap(w_l) == info
     py.test.skip("Unicode conversion is too slow")
-    w_l = space.appexec([w_socket, space.wrap(unicode(host)), space.wrap(port)],
+    w_l = space.appexec([w_socket, space.wrap(host), space.wrap(port)],
                         "(_socket, host, port): return _socket.getaddrinfo(host, port)")
     assert space.unwrap(w_l) == info
 
@@ -314,10 +314,10 @@
         if not hasattr(_socket, 'inet_ntop'):
             skip('No socket.inet_pton on this platform')
         for family, packed, exception in \
-                    [(_socket.AF_INET + _socket.AF_INET6, "", _socket.error),
-                     (_socket.AF_INET, "a", ValueError),
-                     (_socket.AF_INET6, "a", ValueError),
-                     (_socket.AF_INET, u"aa\u2222a", UnicodeEncodeError)]:
+                    [(_socket.AF_INET + _socket.AF_INET6, b"", _socket.error),
+                     (_socket.AF_INET, b"a", ValueError),
+                     (_socket.AF_INET6, b"a", ValueError),
+                     (_socket.AF_INET, u"aa\u2222a", TypeError)]:
             raises(exception, _socket.inet_ntop, family, packed)
 
     def test_pton_exceptions(self):
@@ -503,8 +503,8 @@
         assert s.fileno() != s2.fileno()
         assert s.getsockname() == s2.getsockname()
 
-    def test_buffer_or_unicode(self):
-        # Test that send/sendall/sendto accept a buffer or a unicode as arg
+    def test_buffer(self):
+        # Test that send/sendall/sendto accept a buffer as arg
         import _socket, os
         s = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM, 0)
         # XXX temporarily we use python.org to test, will have more robust tests
@@ -515,14 +515,13 @@
             s.connect(("www.python.org", 80))
         except _socket.gaierror, ex:
             skip("GAIError - probably no connection: %s" % str(ex.args))
-        s.send(buffer(''))
-        s.sendall(buffer(''))
-        s.send(u'')
-        s.sendall(u'')
-        raises(UnicodeEncodeError, s.send, u'\xe9')
+        s.send(buffer(b''))
+        s.sendall(buffer(b''))
+        raises(TypeError, s.send, u'')
+        raises(TypeError, s.sendall, u'')
         s.close()
         s = _socket.socket(_socket.AF_INET, _socket.SOCK_DGRAM, 0)
-        s.sendto(buffer(''), ('localhost', 9)) # Send to discard port.
+        s.sendto(buffer(b''), ('localhost', 9)) # Send to discard port.
         s.close()
 
     def test_unix_socket_connect(self):
@@ -537,15 +536,15 @@
 
         clientsock = _socket.socket(_socket.AF_UNIX)
         clientsock.connect(sockpath)
-        s, addr = serversock.accept()
+        s, addr = serversock._accept()
         assert not addr
 
-        s.send('X')
+        s.send(b'X')
         data = clientsock.recv(100)
-        assert data == 'X'
-        clientsock.send('Y')
+        assert data == b'X'
+        clientsock.send(b'Y')
         data = s.recv(100)
-        assert data == 'Y'
+        assert data == b'Y'
 
         clientsock.close()
         s.close()
@@ -575,45 +574,45 @@
         from _socket import timeout
         def raise_timeout():
             self.serv.settimeout(1.0)
-            self.serv.accept()
+            self.serv._accept()
         raises(timeout, raise_timeout)
 
     def test_timeout_zero(self):
         from _socket import error
         def raise_error():
             self.serv.settimeout(0.0)
-            foo = self.serv.accept()
+            foo = self.serv._accept()
         raises(error, raise_error)
 
     def test_recv_send_timeout(self):
         from _socket import socket, timeout
         cli = socket()
         cli.connect(self.serv.getsockname())
-        t, addr = self.serv.accept()
+        t, addr = self.serv._accept()
         cli.settimeout(1.0)
         # test recv() timeout
-        t.send('*')
+        t.send(b'*')
         buf = cli.recv(100)
-        assert buf == '*'
+        assert buf == b'*'
         raises(timeout, cli.recv, 100)
         # test that send() works
-        count = cli.send('!')
+        count = cli.send(b'!')
         assert count == 1
         buf = t.recv(1)
-        assert buf == '!'
+        assert buf == b'!'
         # test that sendall() works
-        cli.sendall('?')
+        cli.sendall(b'?')
         assert count == 1
         buf = t.recv(1)
-        assert buf == '?'
+        assert buf == b'?'
         # test send() timeout
         try:
             while 1:
-                cli.send('foobar' * 70)
+                cli.send(b'foobar' * 70)
         except timeout:
             pass
         # test sendall() timeout
-        raises(timeout, cli.sendall, 'foobar' * 70)
+        raises(timeout, cli.sendall, b'foobar' * 70)
         # done
         cli.close()
         t.close()
@@ -621,31 +620,31 @@
     def test_recv_into(self):
         import socket
         import array
-        MSG = 'dupa was here\n'
+        MSG = b'dupa was here\n'
         cli = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         cli.connect(self.serv.getsockname())
-        conn, addr = self.serv.accept()
+        conn, addr = self.serv._accept()
         buf = buffer(MSG)
         conn.send(buf)
-        buf = array.array('c', ' '*1024)
+        buf = array.array('b', b' '*1024)
         nbytes = cli.recv_into(buf)
         assert nbytes == len(MSG)
-        msg = buf.tostring()[:len(MSG)]
+        msg = buf.tobytes()[:len(MSG)]
         assert msg == MSG
 
     def test_recvfrom_into(self):
         import socket
         import array
-        MSG = 'dupa was here\n'
+        MSG = b'dupa was here\n'
         cli = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         cli.connect(self.serv.getsockname())
-        conn, addr = self.serv.accept()
+        conn, addr = self.serv._accept()
         buf = buffer(MSG)
         conn.send(buf)
-        buf = array.array('c', ' '*1024)
+        buf = array.array('b', b' '*1024)
         nbytes, addr = cli.recvfrom_into(buf)
         assert nbytes == len(MSG)
-        msg = buf.tostring()[:len(MSG)]
+        msg = buf.tobytes()[:len(MSG)]
         assert msg == MSG
 
     def test_family(self):

From noreply at buildbot.pypy.org  Wed Oct 19 07:34:36 2011
From: noreply at buildbot.pypy.org (justinpeel)
Date: Wed, 19 Oct 2011 07:34:36 +0200 (CEST)
Subject: [pypy-commit] pypy rgc-mem-pressure: make sem_close non-threadsafe
 for now to get __del__ to be acceptable
Message-ID: <20111019053436.274FF820D7@wyvern.cs.uni-duesseldorf.de>

Author: Justin Peel <notmuchtotell at gmail.com>
Branch: rgc-mem-pressure
Changeset: r48224:557f718a62e5
Date: 2011-10-19 07:32 +0200
http://bitbucket.org/pypy/pypy/changeset/557f718a62e5/

Log:	make sem_close non-threadsafe for now to get __del__ to be
	acceptable

diff --git a/pypy/module/_multiprocessing/interp_semaphore.py b/pypy/module/_multiprocessing/interp_semaphore.py
--- a/pypy/module/_multiprocessing/interp_semaphore.py
+++ b/pypy/module/_multiprocessing/interp_semaphore.py
@@ -71,14 +71,15 @@
     else:
         HAVE_BROKEN_SEM_GETVALUE = False
 
-    def external(name, args, result):
+    def external(name, args, result, **kwargs):
         return rffi.llexternal(name, args, result,
-                               compilation_info=eci)
+                               compilation_info=eci, **kwargs)
 
     _sem_open = external('sem_open',
                          [rffi.CCHARP, rffi.INT, rffi.INT, rffi.UINT],
                          SEM_T)
-    _sem_close = external('sem_close', [SEM_T], rffi.INT)
+    # tread sem_close as not threadsafe for now to be able to use the __del__
+    _sem_close = external('sem_close', [SEM_T], rffi.INT, threadsafe=False)
     _sem_unlink = external('sem_unlink', [rffi.CCHARP], rffi.INT)
     _sem_wait = external('sem_wait', [SEM_T], rffi.INT)
     _sem_trywait = external('sem_trywait', [SEM_T], rffi.INT)

From noreply at buildbot.pypy.org  Wed Oct 19 10:44:41 2011
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Wed, 19 Oct 2011 10:44:41 +0200 (CEST)
Subject: [pypy-commit] pyrepl py3ksupport: make the tests discoverable by
	pytest
Message-ID: <20111019084441.95810820D7@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: py3ksupport
Changeset: r129:3901976ecb47
Date: 2011-10-18 10:58 +0200
http://bitbucket.org/pypy/pyrepl/changeset/3901976ecb47/

Log:	make the tests discoverable by pytest

diff --git a/pyrepl/tests/basic.py b/pyrepl/tests/test_basic.py
rename from pyrepl/tests/basic.py
rename to pyrepl/tests/test_basic.py
diff --git a/pyrepl/tests/bugs.py b/pyrepl/tests/test_bugs.py
rename from pyrepl/tests/bugs.py
rename to pyrepl/tests/test_bugs.py
diff --git a/pyrepl/tests/wishes.py b/pyrepl/tests/test_wishes.py
rename from pyrepl/tests/wishes.py
rename to pyrepl/tests/test_wishes.py

From noreply at buildbot.pypy.org  Wed Oct 19 10:44:42 2011
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Wed, 19 Oct 2011 10:44:42 +0200 (CEST)
Subject: [pypy-commit] pyrepl py3ksupport: move the functional tests to new
	dir
Message-ID: <20111019084442.D4EFE820D7@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: py3ksupport
Changeset: r130:ab5192bb7c88
Date: 2011-10-18 10:59 +0200
http://bitbucket.org/pypy/pyrepl/changeset/ab5192bb7c88/

Log:	move the functional tests to new dir

diff --git a/testing/__init__.py b/testing/__init__.py
new file mode 100644
diff --git a/pyrepl/test/test_functional.py b/testing/test_functional.py
rename from pyrepl/test/test_functional.py
rename to testing/test_functional.py

From noreply at buildbot.pypy.org  Wed Oct 19 10:44:43 2011
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Wed, 19 Oct 2011 10:44:43 +0200 (CEST)
Subject: [pypy-commit] pyrepl py3ksupport: test events need to be a 2 tuple
Message-ID: <20111019084443.ED000820D7@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: py3ksupport
Changeset: r131:70c2f802c6c2
Date: 2011-10-18 11:13 +0200
http://bitbucket.org/pypy/pyrepl/changeset/70c2f802c6c2/

Log:	test events need to be a 2 tuple

diff --git a/pyrepl/tests/infrastructure.py b/pyrepl/tests/infrastructure.py
--- a/pyrepl/tests/infrastructure.py
+++ b/pyrepl/tests/infrastructure.py
@@ -49,7 +49,7 @@
         ev, sc = self.events.pop(0)
         self.next_screen = sc
         if not isinstance(ev, tuple):
-            ev = (ev,)
+            ev = (ev, None)
         self.last_event_name = ev[0]
         if self.verbose:
             print "event", ev

From noreply at buildbot.pypy.org  Wed Oct 19 10:44:45 2011
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Wed, 19 Oct 2011 10:44:45 +0200 (CEST)
Subject: [pypy-commit] pyrepl py3ksupport: move rest of the tests to testing
Message-ID: <20111019084445.0AF93820D7@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: py3ksupport
Changeset: r132:deb881a5a096
Date: 2011-10-18 11:21 +0200
http://bitbucket.org/pypy/pyrepl/changeset/deb881a5a096/

Log:	move rest of the tests to testing

diff --git a/pyrepl/tests/__init__.py b/pyrepl/tests/__init__.py
deleted file mode 100644
--- a/pyrepl/tests/__init__.py
+++ /dev/null
@@ -1,20 +0,0 @@
-#   Copyright 2000-2004 Michael Hudson-Doyle <micahel at gmail.com>
-#
-#                        All Rights Reserved
-#
-#
-# Permission to use, copy, modify, and distribute this software and
-# its documentation for any purpose is hereby granted without fee,
-# provided that the above copyright notice appear in all copies and
-# that both that copyright notice and this permission notice appear in
-# supporting documentation.
-#
-# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
-# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
-# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
-# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
-# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
-# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
-# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
-
-# moo
diff --git a/pyrepl/tests/infrastructure.py b/testing/infrastructure.py
rename from pyrepl/tests/infrastructure.py
rename to testing/infrastructure.py
diff --git a/pyrepl/tests/test_basic.py b/testing/test_basic.py
rename from pyrepl/tests/test_basic.py
rename to testing/test_basic.py
--- a/pyrepl/tests/test_basic.py
+++ b/testing/test_basic.py
@@ -18,7 +18,7 @@
 # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 
 from pyrepl.console import Event
-from pyrepl.tests.infrastructure import ReaderTestCase, EA, run_testcase
+from .infrastructure import ReaderTestCase, EA, run_testcase
 
 class SimpleTestCase(ReaderTestCase):
 
diff --git a/pyrepl/tests/test_bugs.py b/testing/test_bugs.py
rename from pyrepl/tests/test_bugs.py
rename to testing/test_bugs.py
--- a/pyrepl/tests/test_bugs.py
+++ b/testing/test_bugs.py
@@ -18,7 +18,7 @@
 # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 
 from pyrepl.console import Event
-from pyrepl.tests.infrastructure import ReaderTestCase, EA, run_testcase
+from .infrastructure import ReaderTestCase, EA, run_testcase
 
 # this test case should contain as-verbatim-as-possible versions of
 # (applicable) bug reports
diff --git a/testing/test_functional.py b/testing/test_functional.py
--- a/testing/test_functional.py
+++ b/testing/test_functional.py
@@ -47,4 +47,4 @@
         child.sendline('a = 3')
         child.sendline('a')
         child.expect('3')
-        
+
diff --git a/pyrepl/tests/test_wishes.py b/testing/test_wishes.py
rename from pyrepl/tests/test_wishes.py
rename to testing/test_wishes.py
--- a/pyrepl/tests/test_wishes.py
+++ b/testing/test_wishes.py
@@ -18,7 +18,7 @@
 # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 
 from pyrepl.console import Event
-from pyrepl.tests.infrastructure import ReaderTestCase, EA, run_testcase
+from .infrastructure import ReaderTestCase, EA, run_testcase
 
 # this test case should contain as-verbatim-as-possible versions of
 # (applicable) feature requests

From noreply at buildbot.pypy.org  Wed Oct 19 10:44:46 2011
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Wed, 19 Oct 2011 10:44:46 +0200 (CEST)
Subject: [pypy-commit] pyrepl py3ksupport: pytest-ize test functions
Message-ID: <20111019084446.1871B820D7@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: py3ksupport
Changeset: r133:c3b33f80a61d
Date: 2011-10-18 12:12 +0200
http://bitbucket.org/pypy/pyrepl/changeset/c3b33f80a61d/

Log:	pytest-ize test functions

diff --git a/testing/infrastructure.py b/testing/infrastructure.py
--- a/testing/infrastructure.py
+++ b/testing/infrastructure.py
@@ -32,18 +32,15 @@
     width = 80
     encoding = 'utf-8'
 
-    def __init__(self, events, testcase, verbose=False):
+    def __init__(self, events, verbose=False):
         self.events = events
         self.next_screen = None
         self.verbose = verbose
-        self.testcase = testcase
 
     def refresh(self, screen, xy):
         if self.next_screen is not None:
-            self.testcase.assertEqual(
-                screen, self.next_screen,
-                "[ %s != %s after %r ]"%(screen, self.next_screen,
-                                         self.last_event_name))
+                assert screen == self.next_screen, "[ %s != %s after %r ]"%(
+                    screen, self.next_screen, self.last_event_name)
 
     def get_event(self, block=1):
         ev, sc = self.events.pop(0)
@@ -62,21 +59,9 @@
         Reader.refresh(self)
         self.dirty = True
 
-class ReaderTestCase(unittest.TestCase):
-    def run_test(self, test_spec, reader_class=TestReader):
-        # remember to finish your test_spec with 'accept' or similar!
-        con = TestConsole(test_spec, self)
-        reader = reader_class(con)
-        reader.readline()
+def read_spec(test_spec, reader_class=TestReader):
+    # remember to finish your test_spec with 'accept' or similar!
+    con = TestConsole(test_spec)
+    reader = reader_class(con)
+    reader.readline()
 
-class BasicTestRunner:
-    def run(self, test):
-        result = unittest.TestResult()
-        test(result)
-        return result
-
-def run_testcase(testclass):
-    suite = unittest.makeSuite(testclass)
-    runner = unittest.TextTestRunner(sys.stdout, verbosity=1)
-    result = runner.run(suite)
-    
diff --git a/testing/test_basic.py b/testing/test_basic.py
--- a/testing/test_basic.py
+++ b/testing/test_basic.py
@@ -18,98 +18,88 @@
 # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 
 from pyrepl.console import Event
-from .infrastructure import ReaderTestCase, EA, run_testcase
+from .infrastructure import read_spec, EA
 
-class SimpleTestCase(ReaderTestCase):
 
-    def test_basic(self):
-        self.run_test([(('self-insert', 'a'), ['a']),
-                       ( 'accept',            ['a'])])
+def test_basic():
+    read_spec([(('self-insert', 'a'), ['a']),
+               ( 'accept',            ['a'])])
 
-    def test_repeat(self):
-        self.run_test([(('digit-arg', '3'),   ['']),
-                       (('self-insert', 'a'), ['aaa']),
-                       ( 'accept',            ['aaa'])])
+def test_repeat():
+    read_spec([(('digit-arg', '3'),   ['']),
+               (('self-insert', 'a'), ['aaa']),
+               ( 'accept',            ['aaa'])])
 
-    def test_kill_line(self):
-        self.run_test([(('self-insert', 'abc'), ['abc']),
-                       ( 'left',                None),
-                       ( 'kill-line',           ['ab']),
-                       ( 'accept',              ['ab'])])
+def test_kill_line():
+    read_spec([(('self-insert', 'abc'), ['abc']),
+               ( 'left',                None),
+               ( 'kill-line',           ['ab']),
+               ( 'accept',              ['ab'])])
 
-    def test_unix_line_discard(self):
-        self.run_test([(('self-insert', 'abc'), ['abc']),
-                       ( 'left',                None),
-                       ( 'unix-word-rubout',    ['c']),
-                       ( 'accept',              ['c'])])
+def test_unix_line_discard():
+    read_spec([(('self-insert', 'abc'), ['abc']),
+               ( 'left',                None),
+               ( 'unix-word-rubout',    ['c']),
+               ( 'accept',              ['c'])])
 
-    def test_kill_word(self):
-        self.run_test([(('self-insert', 'ab cd'), ['ab cd']),
-                       ( 'beginning-of-line',     ['ab cd']),
-                       ( 'kill-word',             [' cd']),
-                       ( 'accept',                [' cd'])])
+def test_kill_word():
+    read_spec([(('self-insert', 'ab cd'), ['ab cd']),
+               ( 'beginning-of-line',     ['ab cd']),
+               ( 'kill-word',             [' cd']),
+               ( 'accept',                [' cd'])])
 
-    def test_backward_kill_word(self):
-        self.run_test([(('self-insert', 'ab cd'), ['ab cd']),
-                       ( 'backward-kill-word',    ['ab ']),
-                       ( 'accept',                ['ab '])])
+def test_backward_kill_word():
+    read_spec([(('self-insert', 'ab cd'), ['ab cd']),
+               ( 'backward-kill-word',    ['ab ']),
+               ( 'accept',                ['ab '])])
 
-    def test_yank(self):
-        self.run_test([(('self-insert', 'ab cd'), ['ab cd']),
-                       ( 'backward-kill-word',    ['ab ']),
-                       ( 'beginning-of-line',     ['ab ']),
-                       ( 'yank',                  ['cdab ']),
-                       ( 'accept',                ['cdab '])])
-        
-    def test_yank_pop(self):
-        self.run_test([(('self-insert', 'ab cd'), ['ab cd']),
-                       ( 'backward-kill-word',    ['ab ']),
-                       ( 'left',                  ['ab ']),
-                       ( 'backward-kill-word',    [' ']),
-                       ( 'yank',                  ['ab ']),
-                       ( 'yank-pop',              ['cd ']),
-                       ( 'accept',                ['cd '])])
+def test_yank():
+    read_spec([(('self-insert', 'ab cd'), ['ab cd']),
+               ( 'backward-kill-word',    ['ab ']),
+               ( 'beginning-of-line',     ['ab ']),
+               ( 'yank',                  ['cdab ']),
+               ( 'accept',                ['cdab '])])
+    
+def test_yank_pop():
+    read_spec([(('self-insert', 'ab cd'), ['ab cd']),
+               ( 'backward-kill-word',    ['ab ']),
+               ( 'left',                  ['ab ']),
+               ( 'backward-kill-word',    [' ']),
+               ( 'yank',                  ['ab ']),
+               ( 'yank-pop',              ['cd ']),
+               ( 'accept',                ['cd '])])
 
-    def test_interrupt(self):
-        try:
-            self.run_test([( 'interrupt',  [''])])
-        except KeyboardInterrupt:
-            pass
-        else:
-            self.fail('KeyboardInterrupt got lost')
+def test_interrupt():
+    with pytest.raises(KeyboardInterrupt):
+        read_spec([( 'interrupt',  [''])])
 
-    # test_suspend -- hah
+# test_suspend -- hah
 
-    def test_up(self):
-        self.run_test([(('self-insert', 'ab\ncd'), ['ab', 'cd']),
-                       ( 'up',                     ['ab', 'cd']),
-                       (('self-insert', 'e'),      ['abe', 'cd']),
-                       ( 'accept',                 ['abe', 'cd'])])
+def test_up():
+    read_spec([(('self-insert', 'ab\ncd'), ['ab', 'cd']),
+               ( 'up',                     ['ab', 'cd']),
+               (('self-insert', 'e'),      ['abe', 'cd']),
+               ( 'accept',                 ['abe', 'cd'])])
 
-    def test_down(self):
-        self.run_test([(('self-insert', 'ab\ncd'), ['ab', 'cd']),
-                       ( 'up',                     ['ab', 'cd']),
-                       (('self-insert', 'e'),      ['abe', 'cd']),
-                       ( 'down',                   ['abe', 'cd']),
-                       (('self-insert', 'f'),      ['abe', 'cdf']),
-                       ( 'accept',                 ['abe', 'cdf'])])
+def test_down():
+    read_spec([(('self-insert', 'ab\ncd'), ['ab', 'cd']),
+               ( 'up',                     ['ab', 'cd']),
+               (('self-insert', 'e'),      ['abe', 'cd']),
+               ( 'down',                   ['abe', 'cd']),
+               (('self-insert', 'f'),      ['abe', 'cdf']),
+               ( 'accept',                 ['abe', 'cdf'])])
 
-    def test_left(self):
-        self.run_test([(('self-insert', 'ab'), ['ab']),
-                       ( 'left',               ['ab']),
-                       (('self-insert', 'c'),  ['acb']),
-                       ( 'accept',             ['acb'])])
+def test_left():
+    read_spec([(('self-insert', 'ab'), ['ab']),
+               ( 'left',               ['ab']),
+               (('self-insert', 'c'),  ['acb']),
+               ( 'accept',             ['acb'])])
 
-    def test_right(self):
-        self.run_test([(('self-insert', 'ab'), ['ab']),
-                       ( 'left',               ['ab']),
-                       (('self-insert', 'c'),  ['acb']),
-                       ( 'right',              ['acb']),
-                       (('self-insert', 'd'),  ['acbd']),
-                       ( 'accept',             ['acbd'])])
-        
-def test():
-    run_testcase(SimpleTestCase)
+def test_right():
+    read_spec([(('self-insert', 'ab'), ['ab']),
+               ( 'left',               ['ab']),
+               (('self-insert', 'c'),  ['acb']),
+               ( 'right',              ['acb']),
+               (('self-insert', 'd'),  ['acbd']),
+               ( 'accept',             ['acbd'])])
 
-if __name__ == '__main__':
-    test()
diff --git a/testing/test_bugs.py b/testing/test_bugs.py
--- a/testing/test_bugs.py
+++ b/testing/test_bugs.py
@@ -17,20 +17,13 @@
 # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
 # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 
-from pyrepl.console import Event
-from .infrastructure import ReaderTestCase, EA, run_testcase
+from .infrastructure import EA, read_spec
 
 # this test case should contain as-verbatim-as-possible versions of
 # (applicable) bug reports
 
-class BugsTestCase(ReaderTestCase):
 
-    def test_transpose_at_start(self):
-        self.run_test([( 'transpose', [EA, '']),
-                       ( 'accept',    [''])])
+def test_transpose_at_start():
+    read_spec([( 'transpose', [EA, '']),
+               ( 'accept',    [''])])
 
-def test():
-    run_testcase(BugsTestCase)
-
-if __name__ == '__main__':
-    test()
diff --git a/testing/test_wishes.py b/testing/test_wishes.py
--- a/testing/test_wishes.py
+++ b/testing/test_wishes.py
@@ -17,22 +17,15 @@
 # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
 # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 
-from pyrepl.console import Event
-from .infrastructure import ReaderTestCase, EA, run_testcase
+from .infrastructure import EA, read_spec
 
 # this test case should contain as-verbatim-as-possible versions of
 # (applicable) feature requests
 
-class WishesTestCase(ReaderTestCase):
 
-    def test_quoted_insert_repeat(self):
-        self.run_test([(('digit-arg', '3'),      ['']),
-                       ( 'quoted-insert',        ['']),
-                       (('self-insert', '\033'), ['^[^[^[']),
-                       ( 'accept',               None)])
+def test_quoted_insert_repeat():
+   read_spec([(('digit-arg', '3'),      ['']),
+              ( 'quoted-insert',        ['']),
+              (('self-insert', '\033'), ['^[^[^[']),
+              ( 'accept',               None)])
 
-def test():
-    run_testcase(WishesTestCase)
-
-if __name__ == '__main__':
-    test()

From noreply at buildbot.pypy.org  Wed Oct 19 10:44:47 2011
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Wed, 19 Oct 2011 10:44:47 +0200 (CEST)
Subject: [pypy-commit] pyrepl py3ksupport: pytest-ize functonal tests
Message-ID: <20111019084447.2ACC8820D7@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: py3ksupport
Changeset: r134:828ecb76b616
Date: 2011-10-18 12:24 +0200
http://bitbucket.org/pypy/pyrepl/changeset/828ecb76b616/

Log:	pytest-ize functonal tests

diff --git a/testing/test_functional.py b/testing/test_functional.py
--- a/testing/test_functional.py
+++ b/testing/test_functional.py
@@ -20,31 +20,19 @@
 
 # some functional tests, to see if this is really working
 
-import py
+import pytest
 import sys
 
-class TestTerminal(object):
-    def _spawn(self, *args, **kwds):
-        try:
-            import pexpect
-        except ImportError, e:
-            py.test.skip(str(e))
-        kwds.setdefault('timeout', 10)
-        child = pexpect.spawn(*args, **kwds)
-        child.logfile = sys.stdout
-        return child
+def pytest_funcarg__child(request):
+    pexpect = pytest.importorskip('pexpect')
+    child = pexpect.spawn(sys.executable, ['-S'], timeout=10)
+    child.logfile = sys.stdout
+    child.sendline('from pyrepl.python_reader import main')
+    child.sendline('main()')
+    return child
 
-    def spawn(self, argv=[]):
-        # avoid running start.py, cause it might contain
-        # things like readline or rlcompleter(2) included
-        child = self._spawn(sys.executable, ['-S'] + argv)
-        child.sendline('from pyrepl.python_reader import main')
-        child.sendline('main()')
-        return child
+def test_basic(child):
+    child.sendline('a = 3')
+    child.sendline('a')
+    child.expect('3')
 
-    def test_basic(self):
-        child = self.spawn()
-        child.sendline('a = 3')
-        child.sendline('a')
-        child.expect('3')
-

From noreply at buildbot.pypy.org  Wed Oct 19 10:44:48 2011
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Wed, 19 Oct 2011 10:44:48 +0200 (CEST)
Subject: [pypy-commit] pyrepl py3ksupport: verbose test events by default
Message-ID: <20111019084448.371B2820D7@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: py3ksupport
Changeset: r135:f589811464d5
Date: 2011-10-18 12:26 +0200
http://bitbucket.org/pypy/pyrepl/changeset/f589811464d5/

Log:	verbose test events by default

diff --git a/testing/infrastructure.py b/testing/infrastructure.py
--- a/testing/infrastructure.py
+++ b/testing/infrastructure.py
@@ -61,7 +61,7 @@
 
 def read_spec(test_spec, reader_class=TestReader):
     # remember to finish your test_spec with 'accept' or similar!
-    con = TestConsole(test_spec)
+    con = TestConsole(test_spec, verbose=True)
     reader = reader_class(con)
     reader.readline()
 

From noreply at buildbot.pypy.org  Wed Oct 19 10:44:49 2011
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Wed, 19 Oct 2011 10:44:49 +0200 (CEST)
Subject: [pypy-commit] pyrepl py3ksupport: objectify console.py
Message-ID: <20111019084449.43032820D7@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: py3ksupport
Changeset: r136:5acd791e2f76
Date: 2011-10-19 09:20 +0200
http://bitbucket.org/pypy/pyrepl/changeset/5acd791e2f76/

Log:	objectify console.py

diff --git a/pyrepl/console.py b/pyrepl/console.py
--- a/pyrepl/console.py
+++ b/pyrepl/console.py
@@ -17,8 +17,9 @@
 # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
 # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 
-class Event:
+class Event(object):
     """An Event.  `evt' is 'key' or somesuch."""
+    __slots__ = 'evt', 'data', 'raw'
 
     def __init__(self, evt, data, raw=''):
         self.evt = evt
@@ -28,7 +29,7 @@
     def __repr__(self):
         return 'Event(%r, %r)'%(self.evt, self.data)
 
-class Console:
+class Console(object):
     """Attributes:
 
     screen,

From noreply at buildbot.pypy.org  Wed Oct 19 10:44:50 2011
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Wed, 19 Oct 2011 10:44:50 +0200 (CEST)
Subject: [pypy-commit] pyrepl py3ksupport: use deque + O(1) popleft in
 inputtranslator instead of list + O(N) pop(0)
Message-ID: <20111019084450.4F8EA820D7@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: py3ksupport
Changeset: r137:1d4238db5d7e
Date: 2011-10-19 09:21 +0200
http://bitbucket.org/pypy/pyrepl/changeset/1d4238db5d7e/

Log:	use deque + O(1) popleft in inputtranslator instead of list + O(N)
	pop(0)

diff --git a/pyrepl/input.py b/pyrepl/input.py
--- a/pyrepl/input.py
+++ b/pyrepl/input.py
@@ -34,6 +34,8 @@
 # class does quite a lot towards emulating a unix terminal.
 
 from pyrepl import unicodedata_
+from collections import deque
+
 
 class InputTranslator(object):
     def push(self, evt):
@@ -43,7 +45,9 @@
     def empty(self):
         pass
 
+
 class KeymapTranslator(InputTranslator):
+
     def __init__(self, keymap, verbose=0,
                  invalid_cls=None, character_cls=None):
         self.verbose = verbose
@@ -58,8 +62,9 @@
         if self.verbose:
             print d
         self.k = self.ck = compile_keymap(d, ())
-        self.results = []
+        self.results = deque()
         self.stack = []
+
     def push(self, evt):
         if self.verbose:
             print "pushed", evt.data,
@@ -88,10 +93,12 @@
                 self.results.append((d, self.stack + [key]))
             self.stack = []
             self.k = self.ck
+
     def get(self):
         if self.results:
-            return self.results.pop(0)
+            return self.results.popleft()
         else:
             return None
+
     def empty(self):
         return not self.results

From noreply at buildbot.pypy.org  Wed Oct 19 10:44:51 2011
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Wed, 19 Oct 2011 10:44:51 +0200 (CEST)
Subject: [pypy-commit] pyrepl py3ksupport: import/whitespace cleanup in
 testing infrastructure
Message-ID: <20111019084451.5C90D820D7@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: py3ksupport
Changeset: r138:a4b000ce4256
Date: 2011-10-19 09:21 +0200
http://bitbucket.org/pypy/pyrepl/changeset/a4b000ce4256/

Log:	import/whitespace cleanup in testing infrastructure

diff --git a/testing/infrastructure.py b/testing/infrastructure.py
--- a/testing/infrastructure.py
+++ b/testing/infrastructure.py
@@ -19,8 +19,6 @@
 
 from pyrepl.reader import Reader
 from pyrepl.console import Console, Event
-import unittest
-import sys
 
 class EqualsAnything(object):
     def __eq__(self, other):
@@ -53,8 +51,10 @@
         return Event(*ev)
 
 class TestReader(Reader):
+
     def get_prompt(self, lineno, cursor_on_line):
         return ''
+    
     def refresh(self):
         Reader.refresh(self)
         self.dirty = True

From noreply at buildbot.pypy.org  Wed Oct 19 10:44:52 2011
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Wed, 19 Oct 2011 10:44:52 +0200 (CEST)
Subject: [pypy-commit] pyrepl py3ksupport: allow consoles to stream in
 events that dont need translation
Message-ID: <20111019084452.69560820D7@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: py3ksupport
Changeset: r139:af08afe96096
Date: 2011-10-19 09:25 +0200
http://bitbucket.org/pypy/pyrepl/changeset/af08afe96096/

Log:	allow consoles to stream in events that dont need translation

diff --git a/pyrepl/reader.py b/pyrepl/reader.py
--- a/pyrepl/reader.py
+++ b/pyrepl/reader.py
@@ -552,6 +552,8 @@
             if not event: # can only happen if we're not blocking
                 return None
 
+            translate = True
+
             if event.evt == 'key':
                 self.input_trans.push(event)
             elif event.evt == 'scroll':
@@ -559,9 +561,12 @@
             elif event.evt == 'resize':
                 self.refresh()
             else:
-                pass
+                translate = False
 
-            cmd = self.input_trans.get()
+            if translate:
+                cmd = self.input_trans.get()
+            else:
+                cmd = event.evt, event.data
 
             if cmd is None:
                 if block:

From noreply at buildbot.pypy.org  Wed Oct 19 10:44:53 2011
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Wed, 19 Oct 2011 10:44:53 +0200 (CEST)
Subject: [pypy-commit] pyrepl py3ksupport: fix rest of tests
Message-ID: <20111019084453.73873820D7@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: py3ksupport
Changeset: r140:720b6383d757
Date: 2011-10-19 09:27 +0200
http://bitbucket.org/pypy/pyrepl/changeset/720b6383d757/

Log:	fix rest of tests

diff --git a/testing/test_basic.py b/testing/test_basic.py
--- a/testing/test_basic.py
+++ b/testing/test_basic.py
@@ -16,8 +16,7 @@
 # RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
 # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
 # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
-
-from pyrepl.console import Event
+import pytest
 from .infrastructure import read_spec, EA
 
 
diff --git a/testing/test_bugs.py b/testing/test_bugs.py
--- a/testing/test_bugs.py
+++ b/testing/test_bugs.py
@@ -22,7 +22,9 @@
 # this test case should contain as-verbatim-as-possible versions of
 # (applicable) bug reports
 
+import pytest
 
+ at pytest.mark.xfail(reason='event missing', run=False)
 def test_transpose_at_start():
     read_spec([( 'transpose', [EA, '']),
                ( 'accept',    [''])])

From noreply at buildbot.pypy.org  Wed Oct 19 10:44:54 2011
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Wed, 19 Oct 2011 10:44:54 +0200 (CEST)
Subject: [pypy-commit] pyrepl py3ksupport: fix up stuff for tox usage
Message-ID: <20111019084454.7FD96820D7@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: py3ksupport
Changeset: r141:f1c267d79023
Date: 2011-10-19 09:34 +0200
http://bitbucket.org/pypy/pyrepl/changeset/f1c267d79023/

Log:	fix up stuff for tox usage

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -1,3 +1,4 @@
 dist/
 build/
+\.tox/
 .*\.egg-info
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -39,7 +39,7 @@
     license = "MIT X11 style",
     description = "A library for building flexible command line interfaces",
     platforms = ["unix", "linux"],
-    packages = ["pyrepl", "pyrepl.tests"],
+    packages = ["pyrepl" ],
     #ext_modules = [Extension("_pyrepl_utils", ["pyrepl_utilsmodule.c"])],
     scripts = ["pythoni", "pythoni1"],
     long_description = long_desc,
diff --git a/tox.ini b/tox.ini
new file mode 100644
--- /dev/null
+++ b/tox.ini
@@ -0,0 +1,7 @@
+[tox]
+envlist= py27, py32
+
+[testenv]
+deps=pytest
+commands=
+    py.test --junitxml={envdir}/junit.xml []

From noreply at buildbot.pypy.org  Wed Oct 19 10:44:55 2011
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Wed, 19 Oct 2011 10:44:55 +0200 (CEST)
Subject: [pypy-commit] pyrepl py3ksupport: py3 save input
Message-ID: <20111019084455.8ADF8820D7@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: py3ksupport
Changeset: r142:f95ff4b955a4
Date: 2011-10-19 10:27 +0200
http://bitbucket.org/pypy/pyrepl/changeset/f95ff4b955a4/

Log:	py3 save input

diff --git a/pyrepl/input.py b/pyrepl/input.py
--- a/pyrepl/input.py
+++ b/pyrepl/input.py
@@ -32,7 +32,7 @@
 # executive, temporary decision: [tab] and [C-i] are distinct, but
 # [meta-key] is identified with [esc key].  We demand that any console
 # class does quite a lot towards emulating a unix terminal.
-
+from __future__ import print_function
 from pyrepl import unicodedata_
 from collections import deque
 
@@ -60,25 +60,25 @@
             keyseq = tuple(parse_keys(keyspec))
             d[keyseq] = command
         if self.verbose:
-            print d
+            print(d)
         self.k = self.ck = compile_keymap(d, ())
         self.results = deque()
         self.stack = []
 
     def push(self, evt):
         if self.verbose:
-            print "pushed", evt.data,
+            print("pushed", evt.data, end='')
         key = evt.data
         d = self.k.get(key)
         if isinstance(d, dict):
             if self.verbose:
-                print "transition"
+                print("transition")
             self.stack.append(key)
             self.k = d
         else:
             if d is None:
                 if self.verbose:
-                    print "invalid"
+                    print("invalid")
                 if self.stack or len(key) > 1 or unicodedata_.category(key) == 'C':
                     self.results.append(
                         (self.invalid_cls, self.stack + [key]))
@@ -89,7 +89,7 @@
                         (self.character_cls, [key]))
             else:
                 if self.verbose:
-                    print "matched", d
+                    print("matched", d)
                 self.results.append((d, self.stack + [key]))
             self.stack = []
             self.k = self.ck

From noreply at buildbot.pypy.org  Wed Oct 19 10:44:56 2011
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Wed, 19 Oct 2011 10:44:56 +0200 (CEST)
Subject: [pypy-commit] pyrepl py3ksupport: py3 save testing infrastructure
Message-ID: <20111019084456.9EC3B820D7@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: py3ksupport
Changeset: r143:b135d7b5a777
Date: 2011-10-19 10:27 +0200
http://bitbucket.org/pypy/pyrepl/changeset/b135d7b5a777/

Log:	py3 save testing infrastructure

diff --git a/testing/infrastructure.py b/testing/infrastructure.py
--- a/testing/infrastructure.py
+++ b/testing/infrastructure.py
@@ -17,6 +17,7 @@
 # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
 # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 
+from __future__ import print_function
 from pyrepl.reader import Reader
 from pyrepl.console import Console, Event
 
@@ -47,7 +48,7 @@
             ev = (ev, None)
         self.last_event_name = ev[0]
         if self.verbose:
-            print "event", ev
+            print("event", ev)
         return Event(*ev)
 
 class TestReader(Reader):

From noreply at buildbot.pypy.org  Wed Oct 19 10:44:57 2011
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Wed, 19 Oct 2011 10:44:57 +0200 (CEST)
Subject: [pypy-commit] pyrepl py3ksupport: py3 safe keymap
Message-ID: <20111019084457.AF4C7820D7@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: py3ksupport
Changeset: r144:e9e3e95c2a67
Date: 2011-10-19 10:29 +0200
http://bitbucket.org/pypy/pyrepl/changeset/e9e3e95c2a67/

Log:	py3 safe keymap

diff --git a/pyrepl/keymap.py b/pyrepl/keymap.py
--- a/pyrepl/keymap.py
+++ b/pyrepl/keymap.py
@@ -101,27 +101,27 @@
     while not ret and s < len(key):
         if key[s] == '\\':
             c = key[s+1].lower()
-            if _escapes.has_key(c):
+            if c in _escapes:
                 ret = _escapes[c]
                 s += 2
             elif c == "c":
                 if key[s + 2] != '-':
-                    raise KeySpecError, \
+                    raise KeySpecError(
                               "\\C must be followed by `-' (char %d of %s)"%(
-                        s + 2, repr(key))
+                        s + 2, repr(key)))
                 if ctrl:
-                    raise KeySpecError, "doubled \\C- (char %d of %s)"%(
-                        s + 1, repr(key))
+                    raise KeySpecError("doubled \\C- (char %d of %s)"%(
+                        s + 1, repr(key)))
                 ctrl = 1
                 s += 3
             elif c == "m":
                 if key[s + 2] != '-':
-                    raise KeySpecError, \
+                    raise KeySpecError(
                               "\\M must be followed by `-' (char %d of %s)"%(
-                        s + 2, repr(key))
+                        s + 2, repr(key)))
                 if meta:
-                    raise KeySpecError, "doubled \\M- (char %d of %s)"%(
-                        s + 1, repr(key))
+                    raise KeySpecError("doubled \\M- (char %d of %s)"%(
+                        s + 1, repr(key)))
                 meta = 1
                 s += 3
             elif c.isdigit():
@@ -135,26 +135,26 @@
             elif c == '<':
                 t = key.find('>', s)
                 if t == -1:
-                    raise KeySpecError, \
+                    raise KeySpecError(
                               "unterminated \\< starting at char %d of %s"%(
-                        s + 1, repr(key))                        
+                        s + 1, repr(key)))
                 ret = key[s+2:t].lower()
                 if ret not in _keynames:
-                    raise KeySpecError, \
+                    raise KeySpecError(
                               "unrecognised keyname `%s' at char %d of %s"%(
-                        ret, s + 2, repr(key))
+                        ret, s + 2, repr(key)))
                 ret = _keynames[ret]
                 s = t + 1
             else:
-                raise KeySpecError, \
+                raise KeySpecError(
                           "unknown backslash escape %s at char %d of %s"%(
-                    `c`, s + 2, repr(key))
+                    repr(c), s + 2, repr(key)))
         else:
             ret = key[s]
             s += 1
     if ctrl:
         if len(ret) > 1:
-            raise KeySpecError, "\\C- must be followed by a character"
+            raise KeySpecError("\\C- must be followed by a character")
         ret = chr(ord(ret) & 0x1f)   # curses.ascii.ctrl()
     if meta:
         ret = ['\033', ret]
@@ -176,9 +176,9 @@
         r.setdefault(key[0], {})[key[1:]] = value
     for key, value in r.items():
         if empty in value:
-            if len(value) <> 1:
-                raise KeySpecError, \
-                      "key definitions for %s clash"%(value.values(),)
+            if len(value) != 1:
+                raise KeySpecError(
+                      "key definitions for %s clash"%(value.values(),))
             else:
                 r[key] = value[empty]
         else:

From noreply at buildbot.pypy.org  Wed Oct 19 10:44:58 2011
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Wed, 19 Oct 2011 10:44:58 +0200 (CEST)
Subject: [pypy-commit] pyrepl py3ksupport: use unnamed argument expansion
 for passing event data to a command
Message-ID: <20111019084458.B9D23820D7@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: py3ksupport
Changeset: r145:007debcdb09f
Date: 2011-10-19 10:34 +0200
http://bitbucket.org/pypy/pyrepl/changeset/007debcdb09f/

Log:	use unnamed argument expansion for passing event data to a command

diff --git a/pyrepl/commands.py b/pyrepl/commands.py
--- a/pyrepl/commands.py
+++ b/pyrepl/commands.py
@@ -33,10 +33,12 @@
 class Command(object):
     finish = 0
     kills_digit_arg = 1
-    def __init__(self, reader, (event_name, event)):
+
+    def __init__(self, reader, event_name, event):
         self.reader = reader
         self.event = event
         self.event_name = event_name
+
     def do(self):
         pass
 
diff --git a/pyrepl/reader.py b/pyrepl/reader.py
--- a/pyrepl/reader.py
+++ b/pyrepl/reader.py
@@ -517,7 +517,7 @@
         #print cmd
         if isinstance(cmd[0], str):
             cmd = self.commands.get(cmd[0],
-                                    commands.invalid_command)(self, cmd)
+                                    commands.invalid_command)(self, *cmd)
         elif isinstance(cmd[0], type):
             cmd = cmd[0](self, cmd)
 

From noreply at buildbot.pypy.org  Wed Oct 19 10:44:59 2011
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Wed, 19 Oct 2011 10:44:59 +0200 (CEST)
Subject: [pypy-commit] pyrepl py3ksupport: partial py3 compat of reader
Message-ID: <20111019084459.CB313820D7@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: py3ksupport
Changeset: r146:d75fa967d0c3
Date: 2011-10-19 10:43 +0200
http://bitbucket.org/pypy/pyrepl/changeset/d75fa967d0c3/

Log:	partial py3 compat of reader

diff --git a/pyrepl/reader.py b/pyrepl/reader.py
--- a/pyrepl/reader.py
+++ b/pyrepl/reader.py
@@ -19,25 +19,30 @@
 # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
 # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 
-import types
+from __future__ import unicode_literals
 from pyrepl import unicodedata_
 from pyrepl import commands
 from pyrepl import input
+try:
+    unicode
+except NameError:
+    unicode = str
+    unichr = chr
 
 def _make_unctrl_map():
     uc_map = {}
     for c in map(unichr, range(256)):
-        if unicodedata_.category(c)[0] <> 'C':
+        if unicodedata_.category(c)[0] != 'C':
             uc_map[c] = c
     for i in range(32):
         c = unichr(i)
-        uc_map[c] = u'^' + unichr(ord('A') + i - 1)
-    uc_map['\t'] = '    ' # display TABs as 4 characters
-    uc_map['\177'] = u'^?'
+        uc_map[c] = '^' + unichr(ord('A') + i - 1)
+    uc_map[b'\t'] = '    ' # display TABs as 4 characters
+    uc_map[b'\177'] = unicode('^?')
     for i in range(256):
         c = unichr(i)
-        if not uc_map.has_key(c):
-            uc_map[c] = u'\\%03o'%i 
+        if c not in uc_map:
+            uc_map[c] = unicode('\\%03o')%i 
     return uc_map
 
 # disp_str proved to be a bottleneck for large inputs, so it's been
@@ -56,7 +61,7 @@
             return u[c]
         else:
             if unicodedata_.category(c).startswith('C'):
-                return '\u%04x'%(ord(c),)
+                return b'\u%04x'%(ord(c))
             else:
                 return c
 
@@ -72,9 +77,12 @@
 
         the list always contains 0s or 1s at present; it could conceivably
         go higher as and when unicode support happens."""
-        s = map(uc, buffer)
-        return (join(s),
-                map(ord, join(map(lambda x:'\001'+(len(x)-1)*'\000', s))))
+        s = [uc(x) for x in buffer]
+        b = [] #XXX: bytearray
+        for x in s:
+            b.append(1)
+            b.extend([0]*(len(x)-1))
+        return join(s), b
 
     del _my_unctrl
 
@@ -93,7 +101,7 @@
         st[c] = SYNTAX_SYMBOL
     for c in [a for a in map(unichr, range(256)) if a.isalpha()]:
         st[c] = SYNTAX_WORD
-    st[u'\n'] = st[u' '] = SYNTAX_WHITESPACE
+    st[unicode('\n')] = st[unicode(' ')] = SYNTAX_WHITESPACE
     return st
 
 default_keymap = tuple(
@@ -141,7 +149,7 @@
      #(r'\M-\n', 'insert-nl'),
      ('\\\\', 'self-insert')] + \
     [(c, 'self-insert')
-     for c in map(chr, range(32, 127)) if c <> '\\'] + \
+     for c in map(chr, range(32, 127)) if c != '\\'] + \
     [(c, 'self-insert')
      for c in map(chr, range(128, 256)) if c.isalpha()] + \
     [(r'\<up>', 'up'),
@@ -160,7 +168,8 @@
                         # workaround
      ])
 
-del c # from the listcomps
+if 'c' in globals():  # only on python 2.x
+    del c # from the listcomps
 
 class Reader(object):
     """The Reader class implements the bare bones of a command reader,
@@ -337,7 +346,7 @@
         st = self.syntax_table
         b = self.buffer
         p -= 1
-        while p >= 0 and st.get(b[p], SYNTAX_WORD) <> SYNTAX_WORD:
+        while p >= 0 and st.get(b[p], SYNTAX_WORD) != SYNTAX_WORD:
             p -= 1
         while p >= 0 and st.get(b[p], SYNTAX_WORD) == SYNTAX_WORD:
             p -= 1
@@ -353,7 +362,7 @@
             p = self.pos
         st = self.syntax_table
         b = self.buffer
-        while p < len(b) and st.get(b[p], SYNTAX_WORD) <> SYNTAX_WORD:
+        while p < len(b) and st.get(b[p], SYNTAX_WORD) != SYNTAX_WORD:
             p += 1
         while p < len(b) and st.get(b[p], SYNTAX_WORD) == SYNTAX_WORD:
             p += 1
@@ -369,7 +378,7 @@
             p = self.pos
         b = self.buffer
         p -= 1
-        while p >= 0 and b[p] <> '\n':
+        while p >= 0 and b[p] != '\n':
             p -= 1
         return p + 1
     
@@ -381,7 +390,7 @@
         if p is None:
             p = self.pos
         b = self.buffer
-        while p < len(b) and b[p] <> '\n':
+        while p < len(b) and b[p] != '\n':
             p += 1
         return p
 
@@ -606,11 +615,11 @@
     def get_buffer(self, encoding=None):
         if encoding is None:
             encoding = self.console.encoding
-        return u''.join(self.buffer).encode(self.console.encoding)
+        return unicode('').join(self.buffer).encode(self.console.encoding)
 
     def get_unicode(self):
         """Return the current buffer as a unicode string."""
-        return u''.join(self.buffer)
+        return unicode('').join(self.buffer)
 
 def test():
     from pyrepl.unix_console import UnixConsole

From noreply at buildbot.pypy.org  Wed Oct 19 10:53:58 2011
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Wed, 19 Oct 2011 10:53:58 +0200 (CEST)
Subject: [pypy-commit] pyrepl py3ksupport: use integer division for the
 wrapcount, testsuite passes now
Message-ID: <20111019085358.0F886820D7@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: py3ksupport
Changeset: r147:25ba09cae1bb
Date: 2011-10-19 10:53 +0200
http://bitbucket.org/pypy/pyrepl/changeset/25ba09cae1bb/

Log:	use integer division for the wrapcount, testsuite passes now

diff --git a/pyrepl/reader.py b/pyrepl/reader.py
--- a/pyrepl/reader.py
+++ b/pyrepl/reader.py
@@ -290,7 +290,7 @@
             p -= ll + 1
             prompt, lp = self.process_prompt(prompt)
             l, l2 = disp_str(line)
-            wrapcount = (len(l) + lp) / w
+            wrapcount = (len(l) + lp) // w
             if wrapcount == 0:
                 screen.append(prompt + l)
                 screeninfo.append((lp, l2+[1]))

From noreply at buildbot.pypy.org  Wed Oct 19 13:16:04 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 19 Oct 2011 13:16:04 +0200 (CEST)
Subject: [pypy-commit] pypy virtualizable-experiments: Avoid ping-pong when
 doing return/yield, instead store the value on the
Message-ID: <20111019111604.9E3EB820D7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: virtualizable-experiments
Changeset: r48225:6768b5b353a1
Date: 2011-10-14 12:38 +0200
http://bitbucket.org/pypy/pypy/changeset/6768b5b353a1/

Log:	Avoid ping-pong when doing return/yield, instead store the value on
	the exception itself

diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -83,8 +83,8 @@
         try:
             while True:
                 next_instr = self.handle_bytecode(co_code, next_instr, ec)
-        except ExitFrame:
-            return self.popvalue()
+        except ExitFrame, e:
+            return e.w_value
 
     def handle_bytecode(self, co_code, next_instr, ec):
         try:
@@ -205,8 +205,7 @@
                 w_returnvalue = self.popvalue()
                 block = self.unrollstack(SReturnValue.kind)
                 if block is None:
-                    self.pushvalue(w_returnvalue)   # XXX ping pong
-                    raise Return
+                    raise Return(w_returnvalue)
                 else:
                     unroller = SReturnValue(w_returnvalue)
                     next_instr = block.handle(self, unroller)
@@ -219,8 +218,7 @@
                     block = self.unrollstack(unroller.kind)
                     if block is None:
                         w_result = unroller.nomoreblocks()
-                        self.pushvalue(w_result)
-                        raise Return
+                        raise Return(w_result)
                     else:
                         next_instr = block.handle(self, unroller)
                 return next_instr
@@ -834,7 +832,7 @@
         self.pushvalue(w_obj)
 
     def YIELD_VALUE(self, oparg, next_instr):
-        raise Yield
+        raise Yield(self.popvalue())
 
     def jump_absolute(self, jumpto, next_instr, ec):
         return jumpto
@@ -1137,7 +1135,8 @@
 ### ____________________________________________________________ ###
 
 class ExitFrame(Exception):
-    pass
+    def __init__(self, w_value):
+        self.w_value = w_value
 
 class Return(ExitFrame):
     """Raised when exiting a frame via a 'return' statement."""

From noreply at buildbot.pypy.org  Wed Oct 19 13:16:05 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 19 Oct 2011 13:16:05 +0200 (CEST)
Subject: [pypy-commit] pypy faster-json: Step one - remove irrelevant
 clutter, mostly CPython optimizations.
Message-ID: <20111019111605.D0E9182A8A@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: faster-json
Changeset: r48226:4c970cf4ece9
Date: 2011-10-19 13:11 +0200
http://bitbucket.org/pypy/pypy/changeset/4c970cf4ece9/

Log:	Step one - remove irrelevant clutter, mostly CPython optimizations.

diff --git a/lib-python/modified-2.7/json/encoder.py b/lib-python/modified-2.7/json/encoder.py
--- a/lib-python/modified-2.7/json/encoder.py
+++ b/lib-python/modified-2.7/json/encoder.py
@@ -2,15 +2,6 @@
 """
 import re
 
-try:
-    from _json import encode_basestring_ascii as c_encode_basestring_ascii
-except ImportError:
-    c_encode_basestring_ascii = None
-try:
-    from _json import make_encoder as c_make_encoder
-except ImportError:
-    c_make_encoder = None
-
 ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]')
 ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])')
 HAS_UTF8 = re.compile(r'[\x80-\xff]')
@@ -24,8 +15,8 @@
     '\t': '\\t',
 }
 for i in range(0x20):
-    ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i))
-    #ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,))
+    #ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i))
+    ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,))
 
 # Assume this produces an infinity on all machines (probably not guaranteed)
 INFINITY = float('1e66666')
@@ -39,8 +30,7 @@
         return ESCAPE_DCT[match.group(0)]
     return '"' + ESCAPE.sub(replace, s) + '"'
 
-
-def py_encode_basestring_ascii(s):
+def encode_basestring_ascii(s):
     """Return an ASCII-only JSON representation of a Python string
 
     """
@@ -63,10 +53,8 @@
                 return '\\u{0:04x}\\u{1:04x}'.format(s1, s2)
                 #return '\\u%04x\\u%04x' % (s1, s2)
     return '"' + str(ESCAPE_ASCII.sub(replace, s)) + '"'
-
-
-encode_basestring_ascii = (
-    c_encode_basestring_ascii or py_encode_basestring_ascii)
+py_encode_basestring_ascii = encode_basestring_ascii
+c_encode_basestring_ascii = None
 
 class JSONEncoder(object):
     """Extensible JSON <http://json.org> encoder for Python data structures.
@@ -147,6 +135,10 @@
 
         self.skipkeys = skipkeys
         self.ensure_ascii = ensure_ascii
+        if ensure_ascii:
+            self.encoder = encode_basestring_ascii
+        else:
+            self.encoder = encode_basestring
         self.check_circular = check_circular
         self.allow_nan = allow_nan
         self.sort_keys = sort_keys
@@ -197,7 +189,7 @@
                 return encode_basestring(o)
         # This doesn't pass the iterator directly to ''.join() because the
         # exceptions aren't as detailed.  The list call should be roughly
-        # equivalent to the PySequence_Fast that ''.join() would do.
+        # equivalent to the PySequence_Fast that ''.join() would do.        
         chunks = self.iterencode(o, _one_shot=True)
         if not isinstance(chunks, (list, tuple)):
             chunks = list(chunks)
@@ -227,59 +219,30 @@
                     o = o.decode(_encoding)
                 return _orig_encoder(o)
 
-        def floatstr(o, allow_nan=self.allow_nan,
-                _repr=FLOAT_REPR, _inf=INFINITY, _neginf=-INFINITY):
-            # Check for specials.  Note that this type of test is processor
-            # and/or platform-specific, so do tests which don't depend on the
-            # internals.
+        return self._iterencode(o, markers, 0)
 
-            if o != o:
-                text = 'NaN'
-            elif o == _inf:
-                text = 'Infinity'
-            elif o == _neginf:
-                text = '-Infinity'
-            else:
-                return _repr(o)
+    def _floatstr(self, o):
+        # Check for specials.  Note that this type of test is processor
+        # and/or platform-specific, so do tests which don't depend on the
+        # internals.
 
-            if not allow_nan:
-                raise ValueError(
-                    "Out of range float values are not JSON compliant: " +
-                    repr(o))
+        if o != o:
+            text = 'NaN'
+        elif o == INFINITY:
+            text = 'Infinity'
+        elif o == -INFINITY:
+            text = '-Infinity'
+        else:
+            return FLOAT_REPR(o)
 
-            return text
+        if not self.allow_nan:
+            raise ValueError(
+                "Out of range float values are not JSON compliant: " +
+                repr(o))
 
+        return text
 
-        if (_one_shot and c_make_encoder is not None
-                and not self.indent and not self.sort_keys):
-            _iterencode = c_make_encoder(
-                markers, self.default, _encoder, self.indent,
-                self.key_separator, self.item_separator, self.sort_keys,
-                self.skipkeys, self.allow_nan)
-        else:
-            _iterencode = _make_iterencode(
-                markers, self.default, _encoder, self.indent, floatstr,
-                self.key_separator, self.item_separator, self.sort_keys,
-                self.skipkeys, _one_shot)
-        return _iterencode(o, 0)
-
-def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,
-        _key_separator, _item_separator, _sort_keys, _skipkeys, _one_shot,
-        ## HACK: hand-optimized bytecode; turn globals into locals
-        ValueError=ValueError,
-        basestring=basestring,
-        dict=dict,
-        float=float,
-        id=id,
-        int=int,
-        isinstance=isinstance,
-        list=list,
-        long=long,
-        str=str,
-        tuple=tuple,
-    ):
-
-    def _iterencode_list(lst, _current_indent_level):
+    def _iterencode_list(self, lst, markers, _current_indent_level):
         if not lst:
             yield '[]'
             return
@@ -289,14 +252,15 @@
                 raise ValueError("Circular reference detected")
             markers[markerid] = lst
         buf = '['
-        if _indent is not None:
+        if self.indent is not None:
             _current_indent_level += 1
-            newline_indent = '\n' + (' ' * (_indent * _current_indent_level))
-            separator = _item_separator + newline_indent
+            newline_indent = '\n' + (' ' * (self.indent *
+                                            _current_indent_level))
+            separator = self.item_separator + newline_indent
             buf += newline_indent
         else:
             newline_indent = None
-            separator = _item_separator
+            separator = self.item_separator
         first = True
         for value in lst:
             if first:
@@ -304,7 +268,7 @@
             else:
                 buf = separator
             if isinstance(value, basestring):
-                yield buf + _encoder(value)
+                yield buf + self.encoder(value)
             elif value is None:
                 yield buf + 'null'
             elif value is True:
@@ -314,25 +278,28 @@
             elif isinstance(value, (int, long)):
                 yield buf + str(value)
             elif isinstance(value, float):
-                yield buf + _floatstr(value)
+                yield buf + self._floatstr(value)
             else:
                 yield buf
                 if isinstance(value, (list, tuple)):
-                    chunks = _iterencode_list(value, _current_indent_level)
+                    chunks = self._iterencode_list(value, markers,
+                                                   _current_indent_level)
                 elif isinstance(value, dict):
-                    chunks = _iterencode_dict(value, _current_indent_level)
+                    chunks = self._iterencode_dict(value, markers,
+                                                   _current_indent_level)
                 else:
-                    chunks = _iterencode(value, _current_indent_level)
+                    chunks = self._iterencode(value, markers,
+                                              _current_indent_level)
                 for chunk in chunks:
                     yield chunk
         if newline_indent is not None:
             _current_indent_level -= 1
-            yield '\n' + (' ' * (_indent * _current_indent_level))
+            yield '\n' + (' ' * (self.indent * _current_indent_level))
         yield ']'
         if markers is not None:
             del markers[markerid]
 
-    def _iterencode_dict(dct, _current_indent_level):
+    def _iterencode_dict(self, dct, markers, _current_indent_level):
         if not dct:
             yield '{}'
             return
@@ -342,16 +309,17 @@
                 raise ValueError("Circular reference detected")
             markers[markerid] = dct
         yield '{'
-        if _indent is not None:
+        if self.indent is not None:
             _current_indent_level += 1
-            newline_indent = '\n' + (' ' * (_indent * _current_indent_level))
-            item_separator = _item_separator + newline_indent
+            newline_indent = '\n' + (' ' * (self.indent *
+                                            _current_indent_level))
+            item_separator = self.item_separator + newline_indent
             yield newline_indent
         else:
             newline_indent = None
-            item_separator = _item_separator
+            item_separator = self.item_separator
         first = True
-        if _sort_keys:
+        if self.sort_keys:
             items = sorted(dct.items(), key=lambda kv: kv[0])
         else:
             items = dct.iteritems()
@@ -361,7 +329,7 @@
             # JavaScript is weakly typed for these, so it makes sense to
             # also allow them.  Many encoders seem to do something like this.
             elif isinstance(key, float):
-                key = _floatstr(key)
+                key = self._floatstr(key)
             elif key is True:
                 key = 'true'
             elif key is False:
@@ -370,7 +338,7 @@
                 key = 'null'
             elif isinstance(key, (int, long)):
                 key = str(key)
-            elif _skipkeys:
+            elif self.skipkeys:
                 continue
             else:
                 raise TypeError("key " + repr(key) + " is not a string")
@@ -378,10 +346,10 @@
                 first = False
             else:
                 yield item_separator
-            yield _encoder(key)
-            yield _key_separator
+            yield self.encoder(key)
+            yield self.key_separator
             if isinstance(value, basestring):
-                yield _encoder(value)
+                yield self.encoder(value)
             elif value is None:
                 yield 'null'
             elif value is True:
@@ -391,26 +359,29 @@
             elif isinstance(value, (int, long)):
                 yield str(value)
             elif isinstance(value, float):
-                yield _floatstr(value)
+                yield self._floatstr(value)
             else:
                 if isinstance(value, (list, tuple)):
-                    chunks = _iterencode_list(value, _current_indent_level)
+                    chunks = self._iterencode_list(value, markers,
+                                                   _current_indent_level)
                 elif isinstance(value, dict):
-                    chunks = _iterencode_dict(value, _current_indent_level)
+                    chunks = self._iterencode_dict(value, markers,
+                                                   _current_indent_level)
                 else:
-                    chunks = _iterencode(value, _current_indent_level)
+                    chunks = self._iterencode(value, markers,
+                                              _current_indent_level)
                 for chunk in chunks:
                     yield chunk
         if newline_indent is not None:
             _current_indent_level -= 1
-            yield '\n' + (' ' * (_indent * _current_indent_level))
+            yield '\n' + (' ' * (self.indent * _current_indent_level))
         yield '}'
         if markers is not None:
             del markers[markerid]
 
-    def _iterencode(o, _current_indent_level):
+    def _iterencode(self, o, markers, _current_indent_level):
         if isinstance(o, basestring):
-            yield _encoder(o)
+            yield self.encoder(o)
         elif o is None:
             yield 'null'
         elif o is True:
@@ -420,12 +391,14 @@
         elif isinstance(o, (int, long)):
             yield str(o)
         elif isinstance(o, float):
-            yield _floatstr(o)
+            yield self._floatstr(o)
         elif isinstance(o, (list, tuple)):
-            for chunk in _iterencode_list(o, _current_indent_level):
+            for chunk in self._iterencode_list(o, markers,
+                                               _current_indent_level):
                 yield chunk
         elif isinstance(o, dict):
-            for chunk in _iterencode_dict(o, _current_indent_level):
+            for chunk in self._iterencode_dict(o, markers,
+                                               _current_indent_level):
                 yield chunk
         else:
             if markers is not None:
@@ -433,10 +406,10 @@
                 if markerid in markers:
                     raise ValueError("Circular reference detected")
                 markers[markerid] = o
-            o = _default(o)
-            for chunk in _iterencode(o, _current_indent_level):
+            o = self.default(o)
+            for chunk in self._iterencode(o, markers,
+                                          _current_indent_level):
                 yield chunk
             if markers is not None:
                 del markers[markerid]
 
-    return _iterencode

From noreply at buildbot.pypy.org  Wed Oct 19 13:26:10 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 19 Oct 2011 13:26:10 +0200 (CEST)
Subject: [pypy-commit] pypy default: Issue916: test and fix by dhx.
Message-ID: <20111019112610.B8A54820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48227:ff2cfafa5e63
Date: 2011-10-19 13:25 +0200
http://bitbucket.org/pypy/pypy/changeset/ff2cfafa5e63/

Log:	Issue916: test and fix by dhx.

diff --git a/pypy/objspace/std/strutil.py b/pypy/objspace/std/strutil.py
--- a/pypy/objspace/std/strutil.py
+++ b/pypy/objspace/std/strutil.py
@@ -35,7 +35,7 @@
 
     def error(self):
         raise ParseStringError("invalid literal for %s() with base %d: '%s'" %
-                               (self.fname, self.base, self.literal))
+                               (self.fname, self.original_base, self.literal))
 
     def __init__(self, s, literal, base, fname):
         self.literal = literal
@@ -47,7 +47,8 @@
         elif s.startswith('+'):
             s = strip_spaces(s[1:])
         self.sign = sign
-        
+        self.original_base = base
+
         if base == 0:
             if s.startswith('0x') or s.startswith('0X'):
                 base = 16
diff --git a/pypy/objspace/std/test/test_strutil.py b/pypy/objspace/std/test/test_strutil.py
--- a/pypy/objspace/std/test/test_strutil.py
+++ b/pypy/objspace/std/test/test_strutil.py
@@ -89,6 +89,8 @@
 
         exc = raises(ParseStringError, string_to_int, '')
         assert exc.value.msg == "invalid literal for int() with base 10: ''"
+        exc = raises(ParseStringError, string_to_int, '', 0)
+        assert exc.value.msg == "invalid literal for int() with base 0: ''"
 
     def test_string_to_int_overflow(self):
         import sys

From noreply at buildbot.pypy.org  Wed Oct 19 13:29:24 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 19 Oct 2011 13:29:24 +0200 (CEST)
Subject: [pypy-commit] pypy faster-json: use identity_dict and factor out
 common infrastructure
Message-ID: <20111019112924.C2335820D7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: faster-json
Changeset: r48228:e562bb829fbf
Date: 2011-10-19 13:29 +0200
http://bitbucket.org/pypy/pypy/changeset/e562bb829fbf/

Log:	use identity_dict and factor out common infrastructure

diff --git a/lib-python/modified-2.7/json/encoder.py b/lib-python/modified-2.7/json/encoder.py
--- a/lib-python/modified-2.7/json/encoder.py
+++ b/lib-python/modified-2.7/json/encoder.py
@@ -2,6 +2,8 @@
 """
 import re
 
+from __pypy__ import identity_dict
+
 ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]')
 ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])')
 HAS_UTF8 = re.compile(r'[\x80-\xff]')
@@ -206,7 +208,7 @@
 
         """
         if self.check_circular:
-            markers = {}
+            markers = identity_dict()
         else:
             markers = None
         if self.ensure_ascii:
@@ -242,15 +244,21 @@
 
         return text
 
+    def _mark_markers(self, markers, o):
+        if markers is not None:
+            if o in markers:
+                raise ValueError("Circular reference detected")
+            markers[o] = None
+
+    def _remove_markers(self, markers, o):
+        if markers is not None:
+            del markers[o]
+
     def _iterencode_list(self, lst, markers, _current_indent_level):
         if not lst:
             yield '[]'
             return
-        if markers is not None:
-            markerid = id(lst)
-            if markerid in markers:
-                raise ValueError("Circular reference detected")
-            markers[markerid] = lst
+        self._mark_markers(markers, lst)
         buf = '['
         if self.indent is not None:
             _current_indent_level += 1
@@ -296,18 +304,13 @@
             _current_indent_level -= 1
             yield '\n' + (' ' * (self.indent * _current_indent_level))
         yield ']'
-        if markers is not None:
-            del markers[markerid]
+        self._remove_markers(markers, lst)
 
     def _iterencode_dict(self, dct, markers, _current_indent_level):
         if not dct:
             yield '{}'
             return
-        if markers is not None:
-            markerid = id(dct)
-            if markerid in markers:
-                raise ValueError("Circular reference detected")
-            markers[markerid] = dct
+        self._mark_markers(markers, dct)
         yield '{'
         if self.indent is not None:
             _current_indent_level += 1
@@ -376,8 +379,7 @@
             _current_indent_level -= 1
             yield '\n' + (' ' * (self.indent * _current_indent_level))
         yield '}'
-        if markers is not None:
-            del markers[markerid]
+        self._remove_markers(markers, dct)
 
     def _iterencode(self, o, markers, _current_indent_level):
         if isinstance(o, basestring):
@@ -401,15 +403,9 @@
                                                _current_indent_level):
                 yield chunk
         else:
-            if markers is not None:
-                markerid = id(o)
-                if markerid in markers:
-                    raise ValueError("Circular reference detected")
-                markers[markerid] = o
-            o = self.default(o)
-            for chunk in self._iterencode(o, markers,
+            self._mark_markers(markers, o)
+            obj = self.default(o)
+            for chunk in self._iterencode(obj, markers,
                                           _current_indent_level):
                 yield chunk
-            if markers is not None:
-                del markers[markerid]
-
+            self._remove_markers(markers, o)

From noreply at buildbot.pypy.org  Wed Oct 19 14:05:19 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 19 Oct 2011 14:05:19 +0200 (CEST)
Subject: [pypy-commit] pypy faster-json: a test and an obscure fix
Message-ID: <20111019120519.44038820D7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: faster-json
Changeset: r48229:53aa34f815e0
Date: 2011-10-19 14:05 +0200
http://bitbucket.org/pypy/pypy/changeset/53aa34f815e0/

Log:	a test and an obscure fix

diff --git a/lib-python/modified-2.7/json/encoder.py b/lib-python/modified-2.7/json/encoder.py
--- a/lib-python/modified-2.7/json/encoder.py
+++ b/lib-python/modified-2.7/json/encoder.py
@@ -141,6 +141,13 @@
             self.encoder = encode_basestring_ascii
         else:
             self.encoder = encode_basestring
+        if encoding != 'utf-8':
+            orig_encoder = self.encoder
+            def encoder(o):
+                if isinstance(o, str):
+                    o = o.decode(encoding)
+                return orig_encoder(o)
+            self.encoder = encoder
         self.check_circular = check_circular
         self.allow_nan = allow_nan
         self.sort_keys = sort_keys
@@ -211,16 +218,6 @@
             markers = identity_dict()
         else:
             markers = None
-        if self.ensure_ascii:
-            _encoder = encode_basestring_ascii
-        else:
-            _encoder = encode_basestring
-        if self.encoding != 'utf-8':
-            def _encoder(o, _orig_encoder=_encoder, _encoding=self.encoding):
-                if isinstance(o, str):
-                    o = o.decode(_encoding)
-                return _orig_encoder(o)
-
         return self._iterencode(o, markers, 0)
 
     def _floatstr(self, o):
diff --git a/lib-python/modified-2.7/json/tests/test_unicode.py b/lib-python/modified-2.7/json/tests/test_unicode.py
--- a/lib-python/modified-2.7/json/tests/test_unicode.py
+++ b/lib-python/modified-2.7/json/tests/test_unicode.py
@@ -80,3 +80,9 @@
         self.assertEqual(type(json.loads(u'["a"]')[0]), unicode)
         # Issue 10038.
         self.assertEqual(type(json.loads('"foo"')), unicode)
+
+    def test_encode_not_utf_8(self):
+        self.assertEqual(json.dumps('\xb1\xe6', encoding='iso8859-2'),
+                         '"\\u0105\\u0107"')
+        self.assertEqual(json.dumps(['\xb1\xe6'], encoding='iso8859-2'),
+                         '["\\u0105\\u0107"]')

From noreply at buildbot.pypy.org  Wed Oct 19 17:37:13 2011
From: noreply at buildbot.pypy.org (justinpeel)
Date: Wed, 19 Oct 2011 17:37:13 +0200 (CEST)
Subject: [pypy-commit] pypy rgc-mem-pressure: improve _digest method in
	hashlib
Message-ID: <20111019153713.5E468820D7@wyvern.cs.uni-duesseldorf.de>

Author: Justin Peel <notmuchtotell at gmail.com>
Branch: rgc-mem-pressure
Changeset: r48230:063291428036
Date: 2011-10-19 09:36 -0600
http://bitbucket.org/pypy/pypy/changeset/063291428036/

Log:	improve _digest method in hashlib

diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py
--- a/pypy/module/_hashlib/interp_hashlib.py
+++ b/pypy/module/_hashlib/interp_hashlib.py
@@ -86,18 +86,14 @@
         return space.wrap(self._block_size())
 
     def _digest(self, space):
-        copy = self.copy(space)
-        ctx = copy.ctx
-        digest_size = self._digest_size()
-        digest = lltype.malloc(rffi.CCHARP.TO, digest_size, flavor='raw')
-
-        try:
-            ropenssl.EVP_DigestFinal(ctx, digest, None)
-            return rffi.charpsize2str(digest, digest_size)
-        finally:
-            keepalive_until_here(copy)
-            lltype.free(digest, flavor='raw')
-
+        with lltype.scoped_alloc(ropenssl.EVP_MD_CTX.TO) as ctx:
+            with self.lock:
+                ropenssl.EVP_MD_CTX_copy(ctx, self.ctx)
+            digest_size = self._digest_size()
+            with lltype.scoped_alloc(rffi.CCHARP.TO, digest_size) as digest:
+                ropenssl.EVP_DigestFinal(ctx, digest, None)
+                ropenssl.EVP_MD_CTX_cleanup(ctx)
+                return rffi.charpsize2str(digest, digest_size)
 
     def _digest_size(self):
         # XXX This isn't the nicest way, but the EVP_MD_size OpenSSL

From noreply at buildbot.pypy.org  Wed Oct 19 19:19:01 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 19 Oct 2011 19:19:01 +0200 (CEST)
Subject: [pypy-commit] pypy.org extradoc: update numbers
Message-ID: <20111019171901.0F691820D7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: extradoc
Changeset: r287:054f7a00b551
Date: 2011-10-19 19:19 +0200
http://bitbucket.org/pypy/pypy.org/changeset/054f7a00b551/

Log:	update numbers

diff --git a/don1.html b/don1.html
--- a/don1.html
+++ b/don1.html
@@ -8,12 +8,12 @@
 <script>
   $(function() {
     $("#progressbar").progressbar({
-      value: 3.1
+      value: 4.1
    });
   });
 </script>
 
-   $3286 of $105000 (3.1%)
+   $4321 of $105000 (4.1%)
    <div id="progressbar">
    </div>
 
diff --git a/source/_layouts/site.genshi b/source/_layouts/site.genshi
--- a/source/_layouts/site.genshi
+++ b/source/_layouts/site.genshi
@@ -46,7 +46,7 @@
 	<script type="text/javascript" src="http://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script>
   <script type="text/javascript" src="js/jquery-ui-1.8.14.custom.min.js"/>
   <script type="text/javascript" src="js/detect.js"></script>
-  <script type="text/javascript" src="js/script2.js"></script>
+  <script type="text/javascript" src="js/script2.js?bust=1"></script>
 </head>
 <body>
 <script type="text/javascript">

From noreply at buildbot.pypy.org  Wed Oct 19 19:22:16 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 19 Oct 2011 19:22:16 +0200 (CEST)
Subject: [pypy-commit] pypy.org extradoc: add numbers to numpy as well
Message-ID: <20111019172216.B8908820D7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: extradoc
Changeset: r288:6aecc698aa56
Date: 2011-10-19 19:22 +0200
http://bitbucket.org/pypy/pypy.org/changeset/6aecc698aa56/

Log:	add numbers to numpy as well

diff --git a/don3.html b/don3.html
--- a/don3.html
+++ b/don3.html
@@ -5,6 +5,20 @@
     <a href="#" onclick="numpy_donate(); return false;"><b>Donate towards NumPy in pypy</b></a><br/>
   </li>
   <li>
+<script>
+  $(function() {
+    $("#progressbar").progressbar({
+      value: 2.9
+   });
+  });
+</script>
+
+   $1750 of $60000 (2.9%)
+   <div id="progressbar">
+   </div>
+
+  </li>
+  <li>
     <span id="donate_text">This donation will go towards supporting NumPy in PyPy <a href="numpydonate.html">Read proposal</a>
   </li>
   <li>

From noreply at buildbot.pypy.org  Wed Oct 19 20:32:47 2011
From: noreply at buildbot.pypy.org (justinpeel)
Date: Wed, 19 Oct 2011 20:32:47 +0200 (CEST)
Subject: [pypy-commit] pypy rgc-mem-pressure: make CloseHandle (win32) not
 thread safe to satisfy __del__ conditions
Message-ID: <20111019183247.D958B820D7@wyvern.cs.uni-duesseldorf.de>

Author: Justin Peel <notmuchtotell at gmail.com>
Branch: rgc-mem-pressure
Changeset: r48231:20a9ccb0d85d
Date: 2011-10-19 12:32 -0600
http://bitbucket.org/pypy/pypy/changeset/20a9ccb0d85d/

Log:	make CloseHandle (win32) not thread safe to satisfy __del__
	conditions

diff --git a/pypy/module/_multiprocessing/interp_semaphore.py b/pypy/module/_multiprocessing/interp_semaphore.py
--- a/pypy/module/_multiprocessing/interp_semaphore.py
+++ b/pypy/module/_multiprocessing/interp_semaphore.py
@@ -25,7 +25,7 @@
         'CreateSemaphoreA', [rffi.VOIDP, rffi.LONG, rffi.LONG, rwin32.LPCSTR],
         rwin32.HANDLE)
     _CloseHandle = rwin32.winexternal('CloseHandle', [rwin32.HANDLE],
-        rwin32.BOOL)
+        rwin32.BOOL, threadsafe=False)
     _ReleaseSemaphore = rwin32.winexternal(
         'ReleaseSemaphore', [rwin32.HANDLE, rffi.LONG, rffi.LONGP],
         rwin32.BOOL)

From noreply at buildbot.pypy.org  Wed Oct 19 21:17:06 2011
From: noreply at buildbot.pypy.org (justinpeel)
Date: Wed, 19 Oct 2011 21:17:06 +0200 (CEST)
Subject: [pypy-commit] pypy rgc-mem-pressure: merge in default
Message-ID: <20111019191706.F330A820D7@wyvern.cs.uni-duesseldorf.de>

Author: Justin Peel <notmuchtotell at gmail.com>
Branch: rgc-mem-pressure
Changeset: r48232:ae454f6ace0a
Date: 2011-10-19 13:13 -0600
http://bitbucket.org/pypy/pypy/changeset/ae454f6ace0a/

Log:	merge in default

diff --git a/pypy/jit/metainterp/optimizeopt/rewrite.py b/pypy/jit/metainterp/optimizeopt/rewrite.py
--- a/pypy/jit/metainterp/optimizeopt/rewrite.py
+++ b/pypy/jit/metainterp/optimizeopt/rewrite.py
@@ -106,10 +106,9 @@
             self.make_equal_to(op.result, v1)
         else:
             self.emit_operation(op)
-
-        # Synthesize the reverse ops for optimize_default to reuse
-        self.pure(rop.INT_ADD, [op.result, op.getarg(1)], op.getarg(0))
-        self.pure(rop.INT_SUB, [op.getarg(0), op.result], op.getarg(1))
+            # Synthesize the reverse ops for optimize_default to reuse
+            self.pure(rop.INT_ADD, [op.result, op.getarg(1)], op.getarg(0))
+            self.pure(rop.INT_SUB, [op.getarg(0), op.result], op.getarg(1))
 
     def optimize_INT_ADD(self, op):
         v1 = self.getvalue(op.getarg(0))
@@ -122,10 +121,9 @@
             self.make_equal_to(op.result, v1)
         else:
             self.emit_operation(op)
-
-        # Synthesize the reverse op for optimize_default to reuse
-        self.pure(rop.INT_SUB, [op.result, op.getarg(1)], op.getarg(0))
-        self.pure(rop.INT_SUB, [op.result, op.getarg(0)], op.getarg(1))
+            # Synthesize the reverse op for optimize_default to reuse
+            self.pure(rop.INT_SUB, [op.result, op.getarg(1)], op.getarg(0))
+            self.pure(rop.INT_SUB, [op.result, op.getarg(0)], op.getarg(1))
 
     def optimize_INT_MUL(self, op):
         v1 = self.getvalue(op.getarg(0))
@@ -141,13 +139,13 @@
             self.make_constant_int(op.result, 0)
         else:
             for lhs, rhs in [(v1, v2), (v2, v1)]:
-                # x & (x -1) == 0 is a quick test for power of 2
-                if (lhs.is_constant() and
-                    (lhs.box.getint() & (lhs.box.getint() - 1)) == 0):
-                    new_rhs = ConstInt(highest_bit(lhs.box.getint()))
-                    op = op.copy_and_change(rop.INT_LSHIFT, args=[rhs.box, new_rhs])
-                    break
-
+                if lhs.is_constant():
+                    x = lhs.box.getint()
+                    # x & (x - 1) == 0 is a quick test for power of 2
+                    if x & (x - 1) == 0:
+                        new_rhs = ConstInt(highest_bit(lhs.box.getint()))
+                        op = op.copy_and_change(rop.INT_LSHIFT, args=[rhs.box, new_rhs])
+                        break
             self.emit_operation(op)
 
     def optimize_UINT_FLOORDIV(self, op):
@@ -462,6 +460,14 @@
         self.optimizer.opaque_pointers[value] = True
         self.make_equal_to(op.result, value)
 
+    def optimize_CAST_PTR_TO_INT(self, op):
+        self.pure(rop.CAST_INT_TO_PTR, [op.result], op.getarg(0))
+        self.emit_operation(op)
+
+    def optimize_CAST_INT_TO_PTR(self, op):
+        self.pure(rop.CAST_PTR_TO_INT, [op.result], op.getarg(0))
+        self.emit_operation(op)
+
 dispatch_opt = make_dispatcher_method(OptRewrite, 'optimize_',
         default=OptRewrite.emit_operation)
 optimize_guards = _findall(OptRewrite, 'optimize_', 'GUARD')
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -234,6 +234,30 @@
             """ % expected_value
             self.optimize_loop(ops, expected)
 
+    def test_reverse_of_cast(self):
+        ops = """
+        [i0]
+        p0 = cast_int_to_ptr(i0)
+        i1 = cast_ptr_to_int(p0)
+        jump(i1)
+        """
+        expected = """
+        [i0]
+        jump(i0)
+        """
+        self.optimize_loop(ops, expected)
+        ops = """
+        [p0]
+        i1 = cast_ptr_to_int(p0)
+        p1 = cast_int_to_ptr(i1)
+        jump(p1)
+        """
+        expected = """
+        [p0]
+        jump(p0)
+        """
+        self.optimize_loop(ops, expected)
+
     # ----------
 
     def test_remove_guard_class_1(self):
diff --git a/pypy/jit/metainterp/quasiimmut.py b/pypy/jit/metainterp/quasiimmut.py
--- a/pypy/jit/metainterp/quasiimmut.py
+++ b/pypy/jit/metainterp/quasiimmut.py
@@ -74,12 +74,10 @@
         self.looptokens_wrefs.append(wref_looptoken)
 
     def compress_looptokens_list(self):
-        newlist = []
-        for wref in self.looptokens_wrefs:
-            looptoken = wref()
-            if looptoken is not None and not looptoken.invalidated:
-                newlist.append(wref)
-        self.looptokens_wrefs = newlist
+        self.looptokens_wrefs = [wref for wref in self.looptokens_wrefs
+                                      if wref() is not None]
+        # NB. we must keep around the looptoken_wrefs that are
+        # already invalidated; see below
         self.compress_limit = (len(self.looptokens_wrefs) + 15) * 2
 
     def invalidate(self):
@@ -90,9 +88,14 @@
         self.looptokens_wrefs = []
         for wref in wrefs:
             looptoken = wref()
-            if looptoken is not None and not looptoken.invalidated:
+            if looptoken is not None:
                 looptoken.invalidated = True
                 self.cpu.invalidate_loop(looptoken)
+                # NB. we must call cpu.invalidate_loop() even if
+                # looptoken.invalidated was already set to True.
+                # It's possible to invalidate several times the
+                # same looptoken; see comments in jit.backend.model
+                # in invalidate_loop().
                 if not we_are_translated():
                     self.cpu.stats.invalidated_token_numbers.add(
                         looptoken.number)
diff --git a/pypy/module/pyexpat/__init__.py b/pypy/module/pyexpat/__init__.py
--- a/pypy/module/pyexpat/__init__.py
+++ b/pypy/module/pyexpat/__init__.py
@@ -4,12 +4,8 @@
 
 class ErrorsModule(MixedModule):
     "Definition of pyexpat.errors module."
-
-    appleveldefs = {
-        }
-
-    interpleveldefs = {
-        }
+    appleveldefs = {}
+    interpleveldefs = {}
 
     def setup_after_space_initialization(self):
         from pypy.module.pyexpat import interp_pyexpat
@@ -18,6 +14,18 @@
                     interp_pyexpat.ErrorString(self.space,
                     getattr(interp_pyexpat, name)))
 
+class ModelModule(MixedModule):
+    "Definition of pyexpat.model module."
+    appleveldefs = {}
+    interpleveldefs = {}
+
+    def setup_after_space_initialization(self):
+        from pypy.module.pyexpat import interp_pyexpat
+        space = self.space
+        for name in interp_pyexpat.xml_model_list:
+            value = getattr(interp_pyexpat, name)
+            space.setattr(self, space.wrap(name), space.wrap(value))
+
 class Module(MixedModule):
     "Python wrapper for Expat parser."
 
@@ -39,6 +47,7 @@
 
     submodules = {
         'errors': ErrorsModule,
+        'model':  ModelModule,
     }
 
     for name in ['XML_PARAM_ENTITY_PARSING_NEVER',
diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py
--- a/pypy/module/pyexpat/interp_pyexpat.py
+++ b/pypy/module/pyexpat/interp_pyexpat.py
@@ -76,6 +76,18 @@
     "XML_ERROR_FINISHED",
     "XML_ERROR_SUSPEND_PE",
     ]
+xml_model_list = [
+    "XML_CTYPE_EMPTY",
+    "XML_CTYPE_ANY",
+    "XML_CTYPE_MIXED",
+    "XML_CTYPE_NAME",
+    "XML_CTYPE_CHOICE",
+    "XML_CTYPE_SEQ",
+    "XML_CQUANT_NONE",
+    "XML_CQUANT_OPT",
+    "XML_CQUANT_REP",
+    "XML_CQUANT_PLUS",
+    ]
 
 class CConfigure:
     _compilation_info_ = eci
@@ -104,6 +116,20 @@
 
     for name in xml_error_list:
         locals()[name] = rffi_platform.ConstantInteger(name)
+    for name in xml_model_list:
+        locals()[name] = rffi_platform.ConstantInteger(name)
+    for name in xml_model_list:
+        locals()[name] = rffi_platform.ConstantInteger(name)
+    for name in xml_model_list:
+        locals()[name] = rffi_platform.ConstantInteger(name)
+    for name in xml_model_list:
+        locals()[name] = rffi_platform.ConstantInteger(name)
+    for name in xml_model_list:
+        locals()[name] = rffi_platform.ConstantInteger(name)
+    for name in xml_model_list:
+        locals()[name] = rffi_platform.ConstantInteger(name)
+    for name in xml_model_list:
+        locals()[name] = rffi_platform.ConstantInteger(name)
     XML_Parser_SIZE = rffi_platform.SizeOf("XML_Parser")
 
 for k, v in rffi_platform.configure(CConfigure).items():
diff --git a/pypy/module/pyexpat/test/test_parser.py b/pypy/module/pyexpat/test/test_parser.py
--- a/pypy/module/pyexpat/test/test_parser.py
+++ b/pypy/module/pyexpat/test/test_parser.py
@@ -131,3 +131,7 @@
                'encoding specified in XML declaration is incorrect')
         assert (pyexpat.errors.XML_ERROR_XML_DECL ==
                 'XML declaration not well-formed')
+
+    def test_model(self):
+        import pyexpat
+        assert isinstance(pyexpat.model.XML_CTYPE_EMPTY, int)
diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -8,6 +8,8 @@
 from pypy.rpython.extregistry import ExtRegistryEntry
 from pypy.tool.sourcetools import func_with_new_name
 
+DEBUG_ELIDABLE_FUNCTIONS = False
+
 
 def elidable(func):
     """ Decorate a function as "trace-elidable". This means precisely that:
@@ -24,6 +26,18 @@
     If a particular call to this function ends up raising an exception, then it
     is handled like a normal function call (this decorator is ignored).
     """
+    if DEBUG_ELIDABLE_FUNCTIONS:
+        cache = {}
+        oldfunc = func
+        def func(*args):
+            result = oldfunc(*args)    # if it raises, no caching
+            try:
+                oldresult = cache.setdefault(args, result)
+            except TypeError:
+                pass           # unhashable args
+            else:
+                assert oldresult == result
+            return result
     func._elidable_function_ = True
     return func
 
diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -1292,10 +1292,12 @@
         # if a prebuilt GcStruct contains a pointer to a young object,
         # then the write_barrier must have ensured that the prebuilt
         # GcStruct is in the list self.old_objects_pointing_to_young.
+        debug_start("gc-minor-walkroots")
         self.root_walker.walk_roots(
             MiniMarkGC._trace_drag_out1,  # stack roots
             MiniMarkGC._trace_drag_out1,  # static in prebuilt non-gc
             None)                         # static in prebuilt gc
+        debug_stop("gc-minor-walkroots")
 
     def collect_cardrefs_to_nursery(self):
         size_gc_header = self.gcheaderbuilder.size_gc_header
diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -56,8 +56,8 @@
     binaries = [(pypy_c, rename_pypy_c)]
     #
     if sys.platform == 'win32':
-        # Can't rename a DLL: it is always called 'libpypy_c.dll'
-        for extra in ['libpypy_c.dll',
+        # Can't rename a DLL: it is always called 'libpypy-c.dll'
+        for extra in ['libpypy-c.dll',
                       'libexpat.dll', 'sqlite3.dll', 'msvcr90.dll']:
             p = pypy_c.dirpath().join(extra)
             if not p.check():
diff --git a/pypy/translator/platform/posix.py b/pypy/translator/platform/posix.py
--- a/pypy/translator/platform/posix.py
+++ b/pypy/translator/platform/posix.py
@@ -157,7 +157,7 @@
 
         rules = [
             ('all', '$(DEFAULT_TARGET)', []),
-            ('$(TARGET)', '$(OBJECTS)', '$(CC_LINK) $(LDFLAGS) $(LDFLAGSEXTRA) -o $@ $(OBJECTS) $(LIBDIRS) $(LIBS) $(LINKFILES)'),
+            ('$(TARGET)', '$(OBJECTS)', '$(CC_LINK) $(LDFLAGSEXTRA) -o $@ $(OBJECTS) $(LIBDIRS) $(LIBS) $(LINKFILES) $(LDFLAGS)'),
             ('%.o', '%.c', '$(CC) $(CFLAGS) $(CFLAGSEXTRA) -o $@ -c $< $(INCLUDEDIRS)'),
             ]
 

From noreply at buildbot.pypy.org  Wed Oct 19 22:30:23 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Wed, 19 Oct 2011 22:30:23 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: another blocker
Message-ID: <20111019203023.EFB3B820D7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: extradoc
Changeset: r3934:f35ad4f8d263
Date: 2011-10-19 16:30 -0400
http://bitbucket.org/pypy/extradoc/changeset/f35ad4f8d263/

Log:	another blocker

diff --git a/planning/release-blockers-1.7.rst b/planning/release-blockers-1.7.rst
--- a/planning/release-blockers-1.7.rst
+++ b/planning/release-blockers-1.7.rst
@@ -3,3 +3,4 @@
 
 * django tests (?) https://bugs.pypy.org/issue908
 
+* urllib2_localnet test failure

From noreply at buildbot.pypy.org  Wed Oct 19 22:38:15 2011
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Wed, 19 Oct 2011 22:38:15 +0200 (CEST)
Subject: [pypy-commit] pyrepl py3ksupport: tonns of python3 fixes in various
 files, normal usage is still fine
Message-ID: <20111019203815.805B2820D7@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: py3ksupport
Changeset: r148:9fd2a470fd2a
Date: 2011-10-19 22:37 +0200
http://bitbucket.org/pypy/pyrepl/changeset/9fd2a470fd2a/

Log:	tonns of python3 fixes in various files, normal usage is still fine

diff --git a/pyrepl/cmdrepl.py b/pyrepl/cmdrepl.py
--- a/pyrepl/cmdrepl.py
+++ b/pyrepl/cmdrepl.py
@@ -33,7 +33,7 @@
 which is in fact done by the `pythoni' script that comes with
 pyrepl."""
 
-from __future__ import nested_scopes
+from __future__ import print_function
 
 from pyrepl import completing_reader as cr, reader, completer
 from pyrepl.completing_reader import CompletingReader as CR
@@ -96,7 +96,7 @@
             if intro is not None:
                 self.intro = intro
             if self.intro:
-                print self.intro
+                print(self.intro)
             stop = None
             while not stop:
                 if self.cmdqueue:
diff --git a/pyrepl/completer.py b/pyrepl/completer.py
--- a/pyrepl/completer.py
+++ b/pyrepl/completer.py
@@ -17,7 +17,10 @@
 # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
 # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 
-import __builtin__
+try:
+    import __builtin__ as builtins
+except ImportError:
+    import builtins
 
 class Completer:
     def __init__(self, ns):
@@ -38,12 +41,11 @@
         """
         import keyword
         matches = []
-        n = len(text)
         for list in [keyword.kwlist,
-                     __builtin__.__dict__.keys(),
+                     builtins.__dict__.keys(),
                      self.ns.keys()]:
             for word in list:
-                if word[:n] == text and word != "__builtins__":
+                if word.startswith(text) and word != "__builtins__":
                     matches.append(word)
         return matches
 
diff --git a/pyrepl/completing_reader.py b/pyrepl/completing_reader.py
--- a/pyrepl/completing_reader.py
+++ b/pyrepl/completing_reader.py
@@ -168,7 +168,7 @@
             r.insert(completions[0][len(stem):])
         else:
             p = prefix(completions, len(stem))
-            if p <> '':
+            if p:
                 r.insert(p)
             if r.last_command_is(self.__class__):
                 if not r.cmpltn_menu_vis:
@@ -259,7 +259,7 @@
         p = self.pos - 1
         while p >= 0 and st.get(b[p], SW) == SW:
             p -= 1
-        return u''.join(b[p+1:self.pos])
+        return ''.join(b[p+1:self.pos])
 
     def get_completions(self, stem):
         return []
diff --git a/pyrepl/copy_code.py b/pyrepl/copy_code.py
--- a/pyrepl/copy_code.py
+++ b/pyrepl/copy_code.py
@@ -17,7 +17,7 @@
 # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
 # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 
-import new
+from types import CodeType
 
 def copy_code_with_changes(codeobject,
                            argcount=None,
@@ -44,7 +44,7 @@
     if name        is None: name        = codeobject.co_name
     if firstlineno is None: firstlineno = codeobject.co_firstlineno
     if lnotab      is None: lnotab      = codeobject.co_lnotab
-    return new.code(argcount,
+    return CodeType(argcount,
                     nlocals,
                     stacksize,
                     flags,
diff --git a/pyrepl/historical_reader.py b/pyrepl/historical_reader.py
--- a/pyrepl/historical_reader.py
+++ b/pyrepl/historical_reader.py
@@ -33,7 +33,8 @@
      (r'\C-g', 'isearch-cancel'),
      (r'\<backspace>', 'isearch-backspace')])
 
-del c
+if 'c' in globals():
+    del c
 
 ISEARCH_DIRECTION_NONE = ''
 ISEARCH_DIRECTION_BACKWARDS = 'r'
@@ -230,7 +231,7 @@
         self.dirty = 1
 
     def get_item(self, i):
-        if i <> len(self.history):
+        if i != len(self.history):
             return self.transient_history.get(i, self.history[i])
         else:
             return self.transient_history.get(i, self.get_unicode())
@@ -253,7 +254,7 @@
             raise
 
     def get_prompt(self, lineno, cursor_on_line):
-        if cursor_on_line and self.isearch_direction <> ISEARCH_DIRECTION_NONE:
+        if cursor_on_line and self.isearch_direction != ISEARCH_DIRECTION_NONE:
             d = 'rf'[self.isearch_direction == ISEARCH_DIRECTION_FORWARDS]
             return "(%s-search `%s') "%(d, self.isearch_term)
         else:
diff --git a/pyrepl/module_lister.py b/pyrepl/module_lister.py
--- a/pyrepl/module_lister.py
+++ b/pyrepl/module_lister.py
@@ -66,5 +66,5 @@
     try:
         mods = _packages[pack]
     except KeyError:
-        raise ImportError, "can't find \"%s\" package"%pack
+        raise ImportError("can't find \"%s\" package" % pack)
     return [mod for mod in mods if mod.startswith(stem)]
diff --git a/pyrepl/python_reader.py b/pyrepl/python_reader.py
--- a/pyrepl/python_reader.py
+++ b/pyrepl/python_reader.py
@@ -20,12 +20,13 @@
 # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 
 # one impressive collections of imports:
+from __future__ import print_function
 from pyrepl.completing_reader import CompletingReader
 from pyrepl.historical_reader import HistoricalReader
 from pyrepl import completing_reader, reader
 from pyrepl import copy_code, commands, completer
 from pyrepl import module_lister
-import new, sys, os, re, code, traceback
+import imp, sys, os, re, code, traceback
 import atexit, warnings
 try:
     import cPickle as pickle
@@ -47,6 +48,21 @@
     """this function eats warnings, if you were wondering"""
     pass
 
+if sys.version_info >= (3,0):
+    def _reraise(cls, val, tb):
+        __tracebackhide__ = True
+        assert hasattr(val, '__traceback__')
+        raise val
+else:
+    exec ("""
+def _reraise(cls, val, tb):
+    __tracebackhide__ = True
+    raise cls, val, tb
+""")
+
+
+
+
 class maybe_accept(commands.Command):
     def do(self):
         r = self.reader
@@ -192,7 +208,7 @@
                     finally:
                         warnings.showwarning = sv
                 except KeyboardInterrupt:
-                    print "KeyboardInterrupt"
+                    print("KeyboardInterrupt")
                 else:
                     if l:
                         self.execute(l)
@@ -217,7 +233,7 @@
             r = self.reader.handle1(block)
         except KeyboardInterrupt:
             self.restore()
-            print "KeyboardInterrupt"
+            print("KeyboardInterrupt")
             self.prepare()
         else:
             if self.reader.finished:
@@ -253,7 +269,7 @@
             if self.exc_info:
                 type, value, tb = self.exc_info
                 self.exc_info = None
-                raise type, value, tb
+                _reraise(type, value, tb)
         
     def tkinteract(self):
         """Run a Tk-aware Python interactive session.
@@ -370,13 +386,13 @@
                     encoding = None # so you get ASCII...
             con = UnixConsole(0, 1, None, encoding)
         if print_banner:
-            print "Python", sys.version, "on", sys.platform
-            print 'Type "help", "copyright", "credits" or "license" '\
-                  'for more information.'
+            print("Python", sys.version, "on", sys.platform)
+            print('Type "help", "copyright", "credits" or "license" '\
+                  'for more information.')
         sys.path.insert(0, os.getcwd())
 
         if clear_main and __name__ != '__main__':
-            mainmod = new.module('__main__')
+            mainmod = imp.new_module('__main__')
             sys.modules['__main__'] = mainmod
         else:
             mainmod = sys.modules['__main__']
diff --git a/pyrepl/reader.py b/pyrepl/reader.py
--- a/pyrepl/reader.py
+++ b/pyrepl/reader.py
@@ -529,6 +529,8 @@
                                     commands.invalid_command)(self, *cmd)
         elif isinstance(cmd[0], type):
             cmd = cmd[0](self, cmd)
+        else:
+            return # nothing to do
 
         cmd.do()
 
diff --git a/pyrepl/unix_console.py b/pyrepl/unix_console.py
--- a/pyrepl/unix_console.py
+++ b/pyrepl/unix_console.py
@@ -41,8 +41,8 @@
 def _my_getstr(cap, optional=0):
     r = curses.tigetstr(cap)
     if not optional and r is None:
-        raise InvalidTerminal, \
-              "terminal doesn't have the required '%s' capability"%cap
+        raise InvalidTerminal(
+              "terminal doesn't have the required '%s' capability"%cap)
     return r
 
 # at this point, can we say: AAAAAAAAAAAAAAAAAAAAAARGH!
@@ -131,14 +131,14 @@
         elif self._cub1 and self._cuf1:
             self.__move_x = self.__move_x_cub1_cuf1
         else:
-            raise RuntimeError, "insufficient terminal (horizontal)"
+            raise RuntimeError("insufficient terminal (horizontal)")
 
         if self._cuu and self._cud:
             self.__move_y = self.__move_y_cuu_cud
         elif self._cuu1 and self._cud1:
             self.__move_y = self.__move_y_cuu1_cud1
         else:
-            raise RuntimeError, "insufficient terminal (vertical)"
+            raise RuntimeError("insufficient terminal (vertical)")
 
         if self._dch1:
             self.dch1 = self._dch1
@@ -163,9 +163,9 @@
     def change_encoding(self, encoding):
         self.encoding = encoding
     
-    def refresh(self, screen, (cx, cy)):
+    def refresh(self, screen, c_xy):
         # this function is still too long (over 90 lines)
-
+        cx, cy = c_xy
         if not self.__gone_tall:
             while len(self.screen) < min(len(screen), self.height):
                 self.__hide_cursor()
@@ -406,7 +406,7 @@
         self.partial_char += char
         try:
             c = unicode(self.partial_char, self.encoding)
-        except UnicodeError, e:
+        except UnicodeError as e:
             if len(e.args) > 4 and \
                    e.args[4] == 'unexpected end of data':
                 pass
@@ -421,7 +421,7 @@
             while 1: # All hail Unix!
                 try:
                     self.push_char(os.read(self.input_fd, 1))
-                except (IOError, OSError), err:
+                except (IOError, OSError) as err:
                     if err.errno == errno.EINTR:
                         if not self.event_queue.empty():
                             return self.event_queue.get()
diff --git a/tox.ini b/tox.ini
--- a/tox.ini
+++ b/tox.ini
@@ -2,6 +2,8 @@
 envlist= py27, py32
 
 [testenv]
-deps=pytest
+deps=
+    pytest
+    pexpect
 commands=
     py.test --junitxml={envdir}/junit.xml []

From noreply at buildbot.pypy.org  Wed Oct 19 23:11:06 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 19 Oct 2011 23:11:06 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Fix unicodedata tests.
Message-ID: <20111019211106.0EAF7820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48233:0c4dd4a4f750
Date: 2011-10-19 02:02 +0200
http://bitbucket.org/pypy/pypy/changeset/0c4dd4a4f750/

Log:	Fix unicodedata tests.

diff --git a/pypy/module/unicodedata/test/test_unicodedata.py b/pypy/module/unicodedata/test/test_unicodedata.py
--- a/pypy/module/unicodedata/test/test_unicodedata.py
+++ b/pypy/module/unicodedata/test/test_unicodedata.py
@@ -42,11 +42,11 @@
                            (0xBCBA, 'HANGUL SYLLABLE BEP'),
                            (0xBF23, 'HANGUL SYLLABLE BBYEOH'),
                            (0xD7A3, 'HANGUL SYLLABLE HIH')):
-            assert unicodedata.name(unichr(code)) == name
-            assert unicodedata.lookup(name) == unichr(code)
+            assert unicodedata.name(chr(code)) == name
+            assert unicodedata.lookup(name) == chr(code)
         # Test outside the range
-        raises(ValueError, unicodedata.name, unichr(0xAC00 - 1))
-        raises(ValueError, unicodedata.name, unichr(0xD7A3 + 1))
+        raises(ValueError, unicodedata.name, chr(0xAC00 - 1))
+        raises(ValueError, unicodedata.name, chr(0xD7A3 + 1))
 
     def test_cjk(self):
         import sys
@@ -61,13 +61,13 @@
             # Test at and inside the boundary
             for i in (first, first + 1, last - 1, last):
                 charname = 'CJK UNIFIED IDEOGRAPH-%X'%i
-                char = ('\\U%08X' % i).decode('unicode-escape')
+                char = chr(i)
                 assert unicodedata.name(char) == charname
                 assert unicodedata.lookup(charname) == char
             # Test outside the boundary
             for i in first - 1, last + 1:
                 charname = 'CJK UNIFIED IDEOGRAPH-%X'%i
-                char = ('\\U%08X' % i).decode('unicode-escape')
+                char = chr(i)
                 try:
                     unicodedata.name(char)
                 except ValueError, e:
@@ -93,7 +93,7 @@
                       0x1c, 0x1d, 0x1e, 0x2028, 0x2029)
         for i in linebreaks:
             for j in range(-2, 3):
-                lines = (unichr(i + j) + u'A').splitlines()
+                lines = (chr(i + j) + u'A').splitlines()
                 if i + j in linebreaks:
                     assert len(lines) == 2
                 else:

From noreply at buildbot.pypy.org  Wed Oct 19 23:11:07 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 19 Oct 2011 23:11:07 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Fixes for the select module.
Message-ID: <20111019211107.4876F820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48234:e2fc88ce5b9c
Date: 2011-10-19 22:31 +0200
http://bitbucket.org/pypy/pypy/changeset/e2fc88ce5b9c/

Log:	Fixes for the select module. _socket._accept() must return the raw
	fd.

diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -22,14 +22,14 @@
     def _accept_w(self, space):
         """_accept() -> (socket object, address info)
 
-        Wait for an incoming connection.  Return a new socket representing the
-        connection, and the address of the client.  For IP sockets, the address
-        info is a pair (hostaddr, port).
+        Wait for an incoming connection.  Return a new socket file descriptor
+        representing the connection, and the address of the client.
+        For IP sockets, the address info is a pair (hostaddr, port).
         """
         try:
-            sock, addr = self.accept(W_RSocket)
-            return space.newtuple([space.wrap(sock),
-                                   addr.as_object(sock.fd, space)])
+            fd, addr = self.accept()
+            return space.newtuple([space.wrap(fd),
+                                   addr.as_object(fd, space)])
         except SocketError, e:
             raise converted_error(space, e)
 
@@ -414,10 +414,14 @@
 
 @unwrap_spec(family=int, type=int, proto=int)
 def newsocket(space, w_subtype, family=AF_INET,
-              type=SOCK_STREAM, proto=0):
+              type=SOCK_STREAM, proto=0, w_fileno=NoneNotWrapped):
     sock = space.allocate_instance(W_RSocket, w_subtype)
     try:
-        W_RSocket.__init__(sock, family, type, proto)
+        if w_fileno:
+            W_RSocket.__init__(sock, family, type, proto,
+                               fd=space.c_filedescriptor_w(w_fileno))
+        else:
+            W_RSocket.__init__(sock, family, type, proto)
     except SocketError, e:
         raise converted_error(space, e)
     return space.wrap(sock)
@@ -493,7 +497,7 @@
 
 Methods of socket objects (keyword arguments not allowed):
 
-_accept() -- accept a connection, returning new socket and client address
+_accept() -- accept a connection, returning new socket fd and client address
 bind(addr) -- bind the socket to a local address
 close() -- close the socket
 connect(addr) -- connect the socket to a remote address
diff --git a/pypy/module/_warnings/test/test_warnings.py b/pypy/module/_warnings/test/test_warnings.py
--- a/pypy/module/_warnings/test/test_warnings.py
+++ b/pypy/module/_warnings/test/test_warnings.py
@@ -45,14 +45,14 @@
 
     def test_show_source_line(self):
         import warnings
-        import sys, StringIO
+        import sys, io
         from test.warning_tests import inner
         # With showarning() missing, make sure that output is okay.
         del warnings.showwarning
 
         stderr = sys.stderr
         try:
-            sys.stderr = StringIO.StringIO()
+            sys.stderr = io.StringIO()
             inner('test message')
             result = sys.stderr.getvalue()
         finally:
diff --git a/pypy/module/select/test/test_epoll.py b/pypy/module/select/test/test_epoll.py
--- a/pypy/module/select/test/test_epoll.py
+++ b/pypy/module/select/test/test_epoll.py
@@ -145,8 +145,8 @@
         then = time.time()
         assert not events
 
-        client.send("Hello!")
-        server.send("world!!!")
+        client.send(b"Hello!")
+        server.send(b"world!!!")
 
         now = time.time()
         events = ep.poll(1, 4)
diff --git a/pypy/module/select/test/test_select.py b/pypy/module/select/test/test_select.py
--- a/pypy/module/select/test/test_select.py
+++ b/pypy/module/select/test/test_select.py
@@ -39,7 +39,7 @@
         try:
             iwtd, owtd, ewtd = select.select([readend], [], [], 0)
             assert iwtd == owtd == ewtd == []
-            writeend.send('X')
+            writeend.send(b'X')
             iwtd, owtd, ewtd = select.select([readend], [], [])
             assert iwtd == [readend]
             assert owtd == ewtd == []
@@ -80,7 +80,7 @@
                 if owtd == []:
                     break
                 assert owtd == [writeend]
-                total_out += writeend.send('x' * 512)
+                total_out += writeend.send(b'x' * 512)
             total_in = 0
             while True:
                 iwtd, owtd, ewtd = select.select([readend], [], [], 0)
@@ -106,7 +106,7 @@
         readend, writeend = self.getpair()
         try:
             try:
-                total_out = writeend.send('x' * 512)
+                total_out = writeend.send(b'x' * 512)
             finally:
                 # win32 sends the 'closed' event immediately, even when
                 # more data is available
@@ -167,7 +167,7 @@
 
             for i in range(50):
                 n = (i*3) % 10
-                writeends[n].send('X')
+                writeends[n].send(b'X')
                 iwtd, owtd, ewtd = select.select(readends, [], [])
                 assert iwtd == [readends[n]]
                 assert owtd == ewtd == []
@@ -239,7 +239,7 @@
             def send(self, data):
                 return os.write(self.fd, data)
             def recv(self, length):
-                return os.read(self.fd, length)
+                return os.read(self.fd, length).decode()
             def close(self):
                 return os.close(self.fd)
         s1, s2 = os.pipe()
diff --git a/pypy/rlib/rsocket.py b/pypy/rlib/rsocket.py
--- a/pypy/rlib/rsocket.py
+++ b/pypy/rlib/rsocket.py
@@ -599,9 +599,11 @@
     """
     _mixin_ = True        # for interp_socket.py
     fd = _c.INVALID_SOCKET
-    def __init__(self, family=AF_INET, type=SOCK_STREAM, proto=0):
+    def __init__(self, family=AF_INET, type=SOCK_STREAM, proto=0,
+                 fd=_c.INVALID_SOCKET):
         """Create a new socket."""
-        fd = _c.socket(family, type, proto)
+        if _c.invalid_socket(fd):
+            fd = _c.socket(family, type, proto)
         if _c.invalid_socket(fd):
             raise self.error_handler()
         # PLAT RISCOS
@@ -704,11 +706,9 @@
         addrlen_p[0] = rffi.cast(_c.socklen_t, maxlen)
         return addr, addr.addr_p, addrlen_p
 
-    def accept(self, SocketClass=None):
+    def accept(self):
         """Wait for an incoming connection.
-        Return (new socket object, client address)."""
-        if SocketClass is None:
-            SocketClass = RSocket
+        Return (new socket fd, client address)."""
         if self._select(False) == 1:
             raise SocketTimeout
         address, addr_p, addrlen_p = self._addrbuf()
@@ -721,9 +721,7 @@
         if _c.invalid_socket(newfd):
             raise self.error_handler()
         address.addrlen = rffi.cast(lltype.Signed, addrlen)
-        sock = make_socket(newfd, self.family, self.type, self.proto,
-                           SocketClass)
-        return (sock, address)
+        return (newfd, address)
 
     def bind(self, address):
         """Bind the socket to a local address."""
diff --git a/pypy/rlib/test/test_rsocket.py b/pypy/rlib/test/test_rsocket.py
--- a/pypy/rlib/test/test_rsocket.py
+++ b/pypy/rlib/test/test_rsocket.py
@@ -166,7 +166,8 @@
     lock.acquire()
     thread.start_new_thread(connecting, ())
     print 'waiting for connection'
-    s1, addr2 = sock.accept()
+    fd1, addr2 = sock.accept()
+    s1 = RSocket(fd=fd1)
     print 'connection accepted'
     lock.acquire()
     print 'connecting side knows that the connection was accepted too'
@@ -253,7 +254,8 @@
     if errcodesok:
         assert err.value.errno in (errno.EINPROGRESS, errno.EWOULDBLOCK)
 
-    s1, addr2 = sock.accept()
+    fd1, addr2 = sock.accept()
+    s1 = RSocket(fd=fd1)
     s1.setblocking(False)
     assert addr.eq(s2.getpeername())
     assert addr2.get_port() == s2.getsockname().get_port()
@@ -400,7 +402,8 @@
 
     clientsock = RSocket(AF_UNIX)
     clientsock.connect(a)
-    s, addr = serversock.accept()
+    fd, addr = serversock.accept()
+    s = RSocket(AF_UNIX, fd=fd)
 
     s.send('X')
     data = clientsock.recv(100)

From noreply at buildbot.pypy.org  Wed Oct 19 23:11:08 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 19 Oct 2011 23:11:08 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Fix fcntl module
Message-ID: <20111019211108.7A580820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48235:059c4d141093
Date: 2011-10-19 22:31 +0200
http://bitbucket.org/pypy/pypy/changeset/059c4d141093/

Log:	Fix fcntl module

diff --git a/pypy/module/fcntl/interp_fcntl.py b/pypy/module/fcntl/interp_fcntl.py
--- a/pypy/module/fcntl/interp_fcntl.py
+++ b/pypy/module/fcntl/interp_fcntl.py
@@ -123,17 +123,22 @@
     except OperationError, e:
         if not e.match(space, space.w_TypeError):
             raise
-    else:
-        ll_arg = rffi.str2charp(arg)
-        rv = fcntl_str(fd, op, ll_arg)
-        arg = rffi.charpsize2str(ll_arg, len(arg))
-        lltype.free(ll_arg, flavor='raw')
-        if rv < 0:
-            raise _get_error(space, "fcntl")
-        return space.wrap(arg)
+        try:
+            arg = space.str_w(w_arg)
+        except OperationError, e:
+            if not e.match(space, space.w_TypeError):
+                raise
+            raise OperationError(space.w_TypeError, space.wrap(
+                    "int or string or buffer required"))
 
-    raise OperationError(space.w_TypeError,
-                         space.wrap("int or string or buffer required"))
+    ll_arg = rffi.str2charp(arg)
+    rv = fcntl_str(fd, op, ll_arg)
+    arg = rffi.charpsize2str(ll_arg, len(arg))
+    lltype.free(ll_arg, flavor='raw')
+    if rv < 0:
+        raise _get_error(space, "fcntl")
+    return space.wrapbytes(arg)
+
 
 @unwrap_spec(op=int)
 def flock(space, w_fd, op):
diff --git a/pypy/module/fcntl/test/test_fcntl.py b/pypy/module/fcntl/test/test_fcntl.py
--- a/pypy/module/fcntl/test/test_fcntl.py
+++ b/pypy/module/fcntl/test/test_fcntl.py
@@ -32,8 +32,8 @@
         raises(TypeError, fcntl.fcntl, f, "foo")
         raises((IOError, ValueError), fcntl.fcntl, -1, 1, 0)
         assert fcntl.fcntl(f, 1, 0) == 0
-        assert fcntl.fcntl(f, 2, "foo") == "foo"
-        assert fcntl.fcntl(f, 2, buffer("foo")) == "foo"
+        assert fcntl.fcntl(f, 2, "foo") == b"foo"
+        assert fcntl.fcntl(f, 2, buffer(b"foo")) == b"foo"
 
         try:
             os.O_LARGEFILE

From noreply at buildbot.pypy.org  Wed Oct 19 23:11:09 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 19 Oct 2011 23:11:09 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: filter() should return an iterable object,
 not a list or string or whatever
Message-ID: <20111019211109.B8954820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48236:bde154c88f96
Date: 2011-10-19 22:31 +0200
http://bitbucket.org/pypy/pypy/changeset/bde154c88f96/

Log:	filter() should return an iterable object, not a list or string or
	whatever

diff --git a/pypy/module/__builtin__/app_functional.py b/pypy/module/__builtin__/app_functional.py
--- a/pypy/module/__builtin__/app_functional.py
+++ b/pypy/module/__builtin__/app_functional.py
@@ -129,37 +129,6 @@
 or string, return the same type, else return a list."""
     if func is None:
         func = bool
-    if isinstance(seq, str):
-        return _filter_string(func, seq, str)
-    elif isinstance(seq, unicode):
-        return _filter_string(func, seq, unicode)
-    elif isinstance(seq, tuple):
-        return _filter_tuple(func, seq)
-    result = []
     for item in seq:
         if func(item):
-            result.append(item)
-    return result
-
-def _filter_string(func, string, str_type):
-    if func is bool and type(string) is str_type:
-        return string
-    result = []
-    for i in range(len(string)):
-        # You must call __getitem__ on the strings, simply iterating doesn't
-        # work :/
-        item = string[i]
-        if func(item):
-            if not isinstance(item, str_type):
-                raise TypeError("__getitem__ returned a non-string type")
-            result.append(item)
-    return str_type().join(result)
-
-def _filter_tuple(func, seq):
-    result = []
-    for i in range(len(seq)):
-        # Again, must call __getitem__, at least there are tests.
-        item = seq[i]
-        if func(item):
-            result.append(item)
-    return tuple(result)
+            yield item
diff --git a/pypy/module/__builtin__/test/test_functional.py b/pypy/module/__builtin__/test/test_functional.py
--- a/pypy/module/__builtin__/test/test_functional.py
+++ b/pypy/module/__builtin__/test/test_functional.py
@@ -101,23 +101,17 @@
 
 class AppTestFilter:
    def test_None(self):
-       assert filter(None, ['a', 'b', 1, 0, None]) == ['a', 'b', 1]
+       assert list(filter(None, ['a', 'b', 1, 0, None])) == ['a', 'b', 1]
 
    def test_return_type(self):
        txt = "This is a test text"
-       assert filter(None, txt) == txt
+       assert list(filter(None, txt)) == list(txt)
        tup = ("a", None, 0, [], 1)
-       assert filter(None, tup) == ("a", 1)
+       assert list(filter(None, tup)) == ["a", 1]
 
    def test_function(self):
-       assert filter(lambda x: x != "a", "a small text") == " smll text"
-       assert filter(lambda x: x < 20, [3, 33, 5, 55]) == [3, 5]
-
-   def test_filter_tuple_calls_getitem(self):
-       class T(tuple):
-           def __getitem__(self, i):
-               return i * 10
-       assert filter(lambda x: x != 20, T("abcd")) == (0, 10, 30)
+       assert list(filter(lambda x: x != "a", "a small text")) == list(" smll text")
+       assert list(filter(lambda x: x < 20, [3, 33, 5, 55])) == [3, 5]
 
 class AppTestRange:
    def test_range(self):

From noreply at buildbot.pypy.org  Wed Oct 19 23:11:10 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 19 Oct 2011 23:11:10 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Add traceback to exception objects.
Message-ID: <20111019211110.EAF6D820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48237:e279f4abab78
Date: 2011-10-19 22:31 +0200
http://bitbucket.org/pypy/pypy/changeset/e279f4abab78/

Log:	Add traceback to exception objects. Not filled by the interpreter
	yet, at least with_traceback() won't fail.

diff --git a/pypy/module/exceptions/interp_exceptions.py b/pypy/module/exceptions/interp_exceptions.py
--- a/pypy/module/exceptions/interp_exceptions.py
+++ b/pypy/module/exceptions/interp_exceptions.py
@@ -78,6 +78,7 @@
     descr_set_dict, descr_del_dict)
 from pypy.interpreter.gateway import interp2app
 from pypy.interpreter.error import OperationError
+from pypy.interpreter.pytraceback import PyTraceback
 from pypy.rlib import rwin32
 
 def readwrite_attrproperty_w(name, cls):
@@ -97,6 +98,7 @@
     args_w = []
     w_cause = None
     w_context = None
+    w_traceback = None
 
     def __init__(self, space):
         self.w_message = space.w_None
@@ -169,6 +171,14 @@
                     "derive from BaseException"))
         self.w_context = w_newcontext
 
+    def descr_gettraceback(self, space):
+        return self.w_traceback
+
+    def descr_settraceback(self, space, w_newtraceback):
+        # Check argument
+        space.interp_w(PyTraceback, w_newtraceback, can_be_None=True)
+        self.w_traceback = w_newtraceback
+
     def descr_getitem(self, space, w_index):
         return space.getitem(space.newtuple(self.args_w), w_index)
 
@@ -192,6 +202,10 @@
         w_olddict = self.getdict(space)
         space.call_method(w_olddict, 'update', w_dict)
 
+    def descr_with_traceback(self, space, w_traceback):
+        self.descr_settraceback(space, w_traceback)
+        return space.wrap(self)
+
     def descr_message_get(self, space):
         w_dict = self.w_dict
         if w_dict is not None:
@@ -242,6 +256,7 @@
     __getitem__ = interp2app(W_BaseException.descr_getitem),
     __reduce__ = interp2app(W_BaseException.descr_reduce),
     __setstate__ = interp2app(W_BaseException.descr_setstate),
+    with_traceback = interp2app(W_BaseException.descr_with_traceback),
     message = GetSetProperty(W_BaseException.descr_message_get,
                             W_BaseException.descr_message_set,
                             W_BaseException.descr_message_del),
@@ -251,6 +266,8 @@
                                W_BaseException.descr_setcause),
     __context__ = GetSetProperty(W_BaseException.descr_getcontext,
                                  W_BaseException.descr_setcontext),
+    __traceback__ = GetSetProperty(W_BaseException.descr_gettraceback,
+                                   W_BaseException.descr_settraceback),
 )
 
 def _new_exception(name, base, docstring, **kwargs):
diff --git a/pypy/module/exceptions/test/test_exc.py b/pypy/module/exceptions/test/test_exc.py
--- a/pypy/module/exceptions/test/test_exc.py
+++ b/pypy/module/exceptions/test/test_exc.py
@@ -261,3 +261,13 @@
         raises(TypeError, setattr, e1, '__context__', 1)
         raises(AttributeError, delattr, e1, '__context__')
 
+    def test_traceback(self):
+        assert ValueError().with_traceback(None).__traceback__ is None
+        raises(TypeError, ValueError().with_traceback, 3)
+        try:
+            XXX
+        except NameError as e:
+            import sys
+            tb = sys.exc_info()[2]
+            assert e.with_traceback(tb) is e
+            assert e.__traceback__ is tb

From noreply at buildbot.pypy.org  Wed Oct 19 23:11:12 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 19 Oct 2011 23:11:12 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Fix the binascii module
Message-ID: <20111019211112.2DF1E820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48238:e2959749a482
Date: 2011-10-19 22:31 +0200
http://bitbucket.org/pypy/pypy/changeset/e2959749a482/

Log:	Fix the binascii module

diff --git a/pypy/module/binascii/interp_base64.py b/pypy/module/binascii/interp_base64.py
--- a/pypy/module/binascii/interp_base64.py
+++ b/pypy/module/binascii/interp_base64.py
@@ -71,7 +71,7 @@
         if leftbits != 0:
             raise_Error(space, "Incorrect padding")
 
-    return space.wrap(res.build())
+    return space.wrapbytes(res.build())
 
 # ____________________________________________________________
 
@@ -110,4 +110,4 @@
         res.append(table_b2a_base64[(leftchar & 0xf) << 2])
         res.append(PAD)
     res.append('\n')
-    return space.wrap(res.build())
+    return space.wrapbytes(res.build())
diff --git a/pypy/module/binascii/interp_hexlify.py b/pypy/module/binascii/interp_hexlify.py
--- a/pypy/module/binascii/interp_hexlify.py
+++ b/pypy/module/binascii/interp_hexlify.py
@@ -24,7 +24,7 @@
     for c in data:
         res.append(_value2char(ord(c) >> 4))
         res.append(_value2char(ord(c) & 0xf))
-    return space.wrap(res.build())
+    return space.wrapbytes(res.build())
 
 # ____________________________________________________________
 
@@ -55,4 +55,4 @@
         a = _char2value(space, hexstr[i])
         b = _char2value(space, hexstr[i+1])
         res.append(chr((a << 4) | b))
-    return space.wrap(res.build())
+    return space.wrapbytes(res.build())
diff --git a/pypy/module/binascii/interp_hqx.py b/pypy/module/binascii/interp_hqx.py
--- a/pypy/module/binascii/interp_hqx.py
+++ b/pypy/module/binascii/interp_hqx.py
@@ -97,7 +97,7 @@
     else:
         if pending_bits > 0:
             raise_Incomplete(space, 'String has incomplete number of bytes')
-    return space.newtuple([space.wrap(res.build()), space.wrap(done)])
+    return space.newtuple([space.wrapbytes(res.build()), space.wrap(done)])
 
 # ____________________________________________________________
 
@@ -128,7 +128,7 @@
     if leftbits > 0:
         leftchar <<= (6 - leftbits)
         res.append(hqx_encoding[leftchar & 0x3f])
-    return space.wrap(res.build())
+    return space.wrapbytes(res.build())
 
 # ____________________________________________________________
 
@@ -150,7 +150,7 @@
             lastpushed = ord(c)
         else:
             if i == end:
-                raise_Incomplete(space, 'String ends with the RLE code \x90')
+                raise_Incomplete(space, 'String ends with the RLE code \\x90')
             count = ord(hexbin[i]) - 1
             i += 1
             if count < 0:
@@ -158,9 +158,9 @@
                 lastpushed = 0x90
             else:
                 if lastpushed < 0:
-                    raise_Error(space, 'String starts with the RLE code \x90')
+                    raise_Error(space, 'String starts with the RLE code \\x90')
                 res.append_multiple_char(chr(lastpushed), count)
-    return space.wrap(res.build())
+    return space.wrapbytes(res.build())
 
 # ____________________________________________________________
 
@@ -197,7 +197,7 @@
     # string that rledecode_hqx() would expand back to 'data', there are
     # some programs somewhere that would start failing obscurely in rare
     # cases.
-    return space.wrap(res.build())
+    return space.wrapbytes(res.build())
 
 # ____________________________________________________________
 
diff --git a/pypy/module/binascii/interp_qp.py b/pypy/module/binascii/interp_qp.py
--- a/pypy/module/binascii/interp_qp.py
+++ b/pypy/module/binascii/interp_qp.py
@@ -56,7 +56,7 @@
             if header and c == '_':
                 c = ' '
             odata.append(c)
-    return space.wrap(odata.build())
+    return space.wrapbytes(odata.build())
 
 # ____________________________________________________________
 
@@ -159,4 +159,4 @@
                 odata.append(c)
                 inp += 1
 
-    return space.wrap(odata.build())
+    return space.wrapbytes(odata.build())
diff --git a/pypy/module/binascii/interp_uu.py b/pypy/module/binascii/interp_uu.py
--- a/pypy/module/binascii/interp_uu.py
+++ b/pypy/module/binascii/interp_uu.py
@@ -52,7 +52,7 @@
     remaining = length - res.getlength()
     if remaining > 0:
         res.append_multiple_char('\x00', remaining)
-    return space.wrap(res.build())
+    return space.wrapbytes(res.build())
 
 # ____________________________________________________________
 
@@ -84,4 +84,4 @@
         res.append(chr(0x20 +  (C & 0x3F)))
 
     res.append('\n')
-    return space.wrap(res.build())
+    return space.wrapbytes(res.build())
diff --git a/pypy/module/binascii/test/test_binascii.py b/pypy/module/binascii/test/test_binascii.py
--- a/pypy/module/binascii/test/test_binascii.py
+++ b/pypy/module/binascii/test/test_binascii.py
@@ -16,405 +16,405 @@
 
     def test_a2b_uu(self):
         # obscure case, for compability with CPython
-        assert self.binascii.a2b_uu("") == "\x00" * 0x20
+        assert self.binascii.a2b_uu(b"") == b"\x00" * 0x20
         #
         for input, expected in [
-            (" ", ""),
-            ("!", "\x00"),
-            ("!6", "X"),
-            ('"6', "X\x00"),
-            ('"W', "\xdc\x00"),
-            ('"WA', "\xde\x10"),
-            ('"WAX', "\xde\x1e"),
-            ('#WAX', "\xde\x1e\x00"),
-            ('#WAXR', "\xde\x1e2"),
-            ('$WAXR', "\xde\x1e2\x00"),
-            ('$WAXR6', "\xde\x1e2X"),
-            ('%WAXR6U', "\xde\x1e2[P"),
-            ('&WAXR6UB', "\xde\x1e2[X\x80"),
-            ("'WAXR6UBA3", "\xde\x1e2[X\xa1L"),
-            ('(WAXR6UBA3#', "\xde\x1e2[X\xa1L0"),
-            (')WAXR6UBA3#Q', "\xde\x1e2[X\xa1L<@"),
-            ('*WAXR6UBA3#Q!5', "\xde\x1e2[X\xa1L<AT"),
+            (b" ", b""),
+            (b"!", b"\x00"),
+            (b"!6", b"X"),
+            (b'"6', b"X\x00"),
+            (b'"W', b"\xdc\x00"),
+            (b'"WA', b"\xde\x10"),
+            (b'"WAX', b"\xde\x1e"),
+            (b'#WAX', b"\xde\x1e\x00"),
+            (b'#WAXR', b"\xde\x1e2"),
+            (b'$WAXR', b"\xde\x1e2\x00"),
+            (b'$WAXR6', b"\xde\x1e2X"),
+            (b'%WAXR6U', b"\xde\x1e2[P"),
+            (b'&WAXR6UB', b"\xde\x1e2[X\x80"),
+            (b"'WAXR6UBA3", b"\xde\x1e2[X\xa1L"),
+            (b'(WAXR6UBA3#', b"\xde\x1e2[X\xa1L0"),
+            (b')WAXR6UBA3#Q', b"\xde\x1e2[X\xa1L<@"),
+            (b'*WAXR6UBA3#Q!5', b"\xde\x1e2[X\xa1L<AT"),
             ]:
             assert self.binascii.a2b_uu(input) == expected
-            assert self.binascii.a2b_uu(input + ' ') == expected
-            assert self.binascii.a2b_uu(input + '  ') == expected
-            assert self.binascii.a2b_uu(input + '   ') == expected
-            assert self.binascii.a2b_uu(input + '    ') == expected
-            assert self.binascii.a2b_uu(input + '\n') == expected
-            assert self.binascii.a2b_uu(input + '\r\n') == expected
-            assert self.binascii.a2b_uu(input + '  \r\n') == expected
-            assert self.binascii.a2b_uu(input + '    \r\n') == expected
+            assert self.binascii.a2b_uu(input + b' ') == expected
+            assert self.binascii.a2b_uu(input + b'  ') == expected
+            assert self.binascii.a2b_uu(input + b'   ') == expected
+            assert self.binascii.a2b_uu(input + b'    ') == expected
+            assert self.binascii.a2b_uu(input + b'\n') == expected
+            assert self.binascii.a2b_uu(input + b'\r\n') == expected
+            assert self.binascii.a2b_uu(input + b'  \r\n') == expected
+            assert self.binascii.a2b_uu(input + b'    \r\n') == expected
         #
         for bogus in [
-            "!w",
-            "! w",
-            "!  w",
-            "!   w",
-            "!    w",
-            "!     w",
-            "!      w",
-            "#a",
-            '"WAXR',
+            b"!w",
+            b"! w",
+            b"!  w",
+            b"!   w",
+            b"!    w",
+            b"!     w",
+            b"!      w",
+            b"#a",
+            b'"WAXR',
             ]:
             raises(self.binascii.Error, self.binascii.a2b_uu, bogus)
-            raises(self.binascii.Error, self.binascii.a2b_uu, bogus + ' ')
-            raises(self.binascii.Error, self.binascii.a2b_uu, bogus + '  ')
-            raises(self.binascii.Error, self.binascii.a2b_uu, bogus + '   ')
-            raises(self.binascii.Error, self.binascii.a2b_uu, bogus + '    ')
-            raises(self.binascii.Error, self.binascii.a2b_uu, bogus + '\n')
-            raises(self.binascii.Error, self.binascii.a2b_uu, bogus + '\r\n')
-            raises(self.binascii.Error, self.binascii.a2b_uu, bogus + '  \r\n')
+            raises(self.binascii.Error, self.binascii.a2b_uu, bogus + b' ')
+            raises(self.binascii.Error, self.binascii.a2b_uu, bogus + b'  ')
+            raises(self.binascii.Error, self.binascii.a2b_uu, bogus + b'   ')
+            raises(self.binascii.Error, self.binascii.a2b_uu, bogus + b'    ')
+            raises(self.binascii.Error, self.binascii.a2b_uu, bogus + b'\n')
+            raises(self.binascii.Error, self.binascii.a2b_uu, bogus + b'\r\n')
+            raises(self.binascii.Error, self.binascii.a2b_uu, bogus + b'  \r\n')
 
     def test_b2a_uu(self):
         for input, expected in [
-            ("", " "),
-            ("\x00", "!    "),
-            ("X", "!6   "),
-            ("X\x00", '"6   '),
-            ("\xdc\x00", '"W   '),
-            ("\xde\x10", '"WA  '),
-            ("\xde\x1e", '"WAX '),
-            ("\xde\x1e\x00", '#WAX '),
-            ("\xde\x1e2", '#WAXR'),
-            ("\xde\x1e2\x00", '$WAXR    '),
-            ("\xde\x1e2X", '$WAXR6   '),
-            ("\xde\x1e2[P", '%WAXR6U  '),
-            ("\xde\x1e2[X\x80", '&WAXR6UB '),
-            ("\xde\x1e2[X\xa1L", "'WAXR6UBA3   "),
-            ("\xde\x1e2[X\xa1L0", '(WAXR6UBA3#  '),
-            ("\xde\x1e2[X\xa1L<@", ')WAXR6UBA3#Q '),
-            ("\xde\x1e2[X\xa1L<AT", '*WAXR6UBA3#Q!5   '),
+            (b"", b" "),
+            (b"\x00", b"!    "),
+            (b"X", b"!6   "),
+            (b"X\x00", b'"6   '),
+            (b"\xdc\x00", b'"W   '),
+            (b"\xde\x10", b'"WA  '),
+            (b"\xde\x1e", b'"WAX '),
+            (b"\xde\x1e\x00", b'#WAX '),
+            (b"\xde\x1e2", b'#WAXR'),
+            (b"\xde\x1e2\x00", b'$WAXR    '),
+            (b"\xde\x1e2X", b'$WAXR6   '),
+            (b"\xde\x1e2[P", b'%WAXR6U  '),
+            (b"\xde\x1e2[X\x80", b'&WAXR6UB '),
+            (b"\xde\x1e2[X\xa1L", b"'WAXR6UBA3   "),
+            (b"\xde\x1e2[X\xa1L0", b'(WAXR6UBA3#  '),
+            (b"\xde\x1e2[X\xa1L<@", b')WAXR6UBA3#Q '),
+            (b"\xde\x1e2[X\xa1L<AT", b'*WAXR6UBA3#Q!5   '),
             ]:
-            assert self.binascii.b2a_uu(input) == expected + '\n'
+            assert self.binascii.b2a_uu(input) == expected + b'\n'
 
     def test_a2b_base64(self):
         for input, expected in [
-            ("", ""),
-            ("\n", ""),
-            ("Yg==\n", "b"),
-            ("Y g = \n = \r", "b"),     # random spaces
-            ("Y\x80g\xff=\xc4=", "b"),  # random junk chars, >= 0x80
-            ("abcd", "i\xb7\x1d"),
-            ("abcdef==", "i\xb7\x1dy"),
-            ("abcdefg=", "i\xb7\x1dy\xf8"),
-            ("abcdefgh", "i\xb7\x1dy\xf8!"),
-            ("abcdef==FINISHED", "i\xb7\x1dy"),
-            ("abcdef=   \n   =FINISHED", "i\xb7\x1dy"),
-            ("abcdefg=FINISHED", "i\xb7\x1dy\xf8"),
-            ("abcd=efgh", "i\xb7\x1dy\xf8!"),
-            ("abcde=fgh", "i\xb7\x1dy\xf8!"),
-            ("abcdef=gh", "i\xb7\x1dy\xf8!"),
+            (b"", b""),
+            (b"\n", b""),
+            (b"Yg==\n", b"b"),
+            (b"Y g = \n = \r", b"b"),     # random spaces
+            (b"Y\x80g\xff=\xc4=", b"b"),  # random junk chars, >= 0x80
+            (b"abcd", b"i\xb7\x1d"),
+            (b"abcdef==", b"i\xb7\x1dy"),
+            (b"abcdefg=", b"i\xb7\x1dy\xf8"),
+            (b"abcdefgh", b"i\xb7\x1dy\xf8!"),
+            (b"abcdef==FINISHED", b"i\xb7\x1dy"),
+            (b"abcdef=   \n   =FINISHED", b"i\xb7\x1dy"),
+            (b"abcdefg=FINISHED", b"i\xb7\x1dy\xf8"),
+            (b"abcd=efgh", b"i\xb7\x1dy\xf8!"),
+            (b"abcde=fgh", b"i\xb7\x1dy\xf8!"),
+            (b"abcdef=gh", b"i\xb7\x1dy\xf8!"),
             ]:
             assert self.binascii.a2b_base64(input) == expected
         #
         for bogus in [
-            "abcde",
-            "abcde=",
-            "abcde==",
-            "abcde===",
-            "abcdef",
-            "abcdef=",
-            "abcdefg",
+            b"abcde",
+            b"abcde=",
+            b"abcde==",
+            b"abcde===",
+            b"abcdef",
+            b"abcdef=",
+            b"abcdefg",
             ]:
             raises(self.binascii.Error, self.binascii.a2b_base64, bogus)
 
     def test_b2a_base64(self):
         for input, expected in [
-            ("", ""),
-            ("b", "Yg=="),
-            ("i\xb7\x1d", "abcd"),
-            ("i\xb7\x1dy", "abcdeQ=="),
-            ("i\xb7\x1dy\xf8", "abcdefg="),
-            ("i\xb7\x1dy\xf8!", "abcdefgh"),
-            ("i\xb7\x1d" * 345, "abcd" * 345),
+            (b"", b""),
+            (b"b", b"Yg=="),
+            (b"i\xb7\x1d", b"abcd"),
+            (b"i\xb7\x1dy", b"abcdeQ=="),
+            (b"i\xb7\x1dy\xf8", b"abcdefg="),
+            (b"i\xb7\x1dy\xf8!", b"abcdefgh"),
+            (b"i\xb7\x1d" * 345, b"abcd" * 345),
             ]:
-            assert self.binascii.b2a_base64(input) == expected + '\n'
+            assert self.binascii.b2a_base64(input) == expected + b'\n'
 
     def test_a2b_qp(self):
         for input, expected in [
             # these are the tests from CPython 2.7
-            ("= ", "= "),
-            ("==", "="),
-            ("=AX", "=AX"),
-            ("=00\r\n=00", "\x00\r\n\x00"),
+            (b"= ", b"= "),
+            (b"==", b"="),
+            (b"=AX", b"=AX"),
+            (b"=00\r\n=00", b"\x00\r\n\x00"),
             # more tests follow
-            ("=", ""),
-            ("abc=", "abc"),
-            ("ab=\ncd", "abcd"),
-            ("ab=\r\ncd", "abcd"),
-            (''.join(["=%02x" % n for n in range(256)]),
-                          ''.join(map(chr, range(256)))),
-            (''.join(["=%02X" % n for n in range(256)]),
-                          ''.join(map(chr, range(256)))),
+            (b"=", b""),
+            (b"abc=", b"abc"),
+            (b"ab=\ncd", b"abcd"),
+            (b"ab=\r\ncd", b"abcd"),
+            (''.join(["=%02x" % n for n in range(256)]).encode(),
+                          bytes(range(256))),
+            (''.join(["=%02X" % n for n in range(256)]).encode(),
+                          bytes(range(256))),
             ]:
             assert self.binascii.a2b_qp(input) == expected
         #
         for input, expected in [
-            ("xyz", "xyz"),
-            ("__", "  "),
-            ("a_b", "a b"),
+            (b"xyz", b"xyz"),
+            (b"__", b"  "),
+            (b"a_b", b"a b"),
             ]:
             assert self.binascii.a2b_qp(input, header=True) == expected
 
     def test_b2a_qp(self):
         for input, flags, expected in [
             # these are the tests from CPython 2.7
-            ("\xff\r\n\xff\n\xff", {}, "=FF\r\n=FF\r\n=FF"),
-            ("0"*75+"\xff\r\n\xff\r\n\xff",{},"0"*75+"=\r\n=FF\r\n=FF\r\n=FF"),
-            ('\0\n', {}, '=00\n'),
-            ('\0\n', {'quotetabs': True}, '=00\n'),
-            ('foo\tbar\t\n', {}, 'foo\tbar=09\n'),
-            ('foo\tbar\t\n', {'quotetabs': True}, 'foo=09bar=09\n'),
-            ('.', {}, '=2E'),
-            ('.\n', {}, '=2E\n'),
-            ('a.\n', {}, 'a.\n'),
+            (b"\xff\r\n\xff\n\xff", {}, b"=FF\r\n=FF\r\n=FF"),
+            (b"0"*75+b"\xff\r\n\xff\r\n\xff",{},b"0"*75+b"=\r\n=FF\r\n=FF\r\n=FF"),
+            (b'\0\n', {}, b'=00\n'),
+            (b'\0\n', {'quotetabs': True}, b'=00\n'),
+            (b'foo\tbar\t\n', {}, b'foo\tbar=09\n'),
+            (b'foo\tbar\t\n', {'quotetabs': True}, b'foo=09bar=09\n'),
+            (b'.', {}, b'=2E'),
+            (b'.\n', {}, b'=2E\n'),
+            (b'a.\n', {}, b'a.\n'),
             # more tests follow
-            ('_', {}, '_'),
-            ('_', {'header': True}, '=5F'),
-            ('.x', {}, '.x'),
-            ('.\r\nn', {}, '=2E\r\nn'),
-            ('\nn', {}, '\nn'),
-            ('\r\nn', {}, '\r\nn'),
-            ('\nn', {'istext': False}, '=0An'),
-            ('\r\nn', {'istext': False}, '=0D=0An'),
-            (' ', {}, '=20'),
-            ('\t', {}, '=09'),
-            (' x', {}, ' x'),
-            ('\tx', {}, '\tx'),
-            ('\x16x', {}, '=16x'),
-            (' x', {'quotetabs': True}, '=20x'),
-            ('\tx', {'quotetabs': True}, '=09x'),
-            (' \nn', {}, '=20\nn'),
-            ('\t\nn', {}, '=09\nn'),
-            ('x\nn', {}, 'x\nn'),
-            (' \r\nn', {}, '=20\r\nn'),
-            ('\t\r\nn', {}, '=09\r\nn'),
-            ('x\r\nn', {}, 'x\r\nn'),
-            ('x\nn', {'istext': False}, 'x=0An'),
-            ('   ', {}, '  =20'),
-            ('   ', {'header': True}, '__=20'),
-            ('   \nn', {}, '  =20\nn'),
-            ('   \nn', {'header': True}, '___\nn'),
-            ('   ', {}, '  =20'),
-            ('\t\t\t', {'header': True}, '\t\t=09'),
-            ('\t\t\t\nn', {}, '\t\t=09\nn'),
-            ('\t\t\t\nn', {'header': True}, '\t\t=09\nn'),
+            (b'_', {}, b'_'),
+            (b'_', {'header': True}, b'=5F'),
+            (b'.x', {}, b'.x'),
+            (b'.\r\nn', {}, b'=2E\r\nn'),
+            (b'\nn', {}, b'\nn'),
+            (b'\r\nn', {}, b'\r\nn'),
+            (b'\nn', {'istext': False}, b'=0An'),
+            (b'\r\nn', {'istext': False}, b'=0D=0An'),
+            (b' ', {}, b'=20'),
+            (b'\t', {}, b'=09'),
+            (b' x', {}, b' x'),
+            (b'\tx', {}, b'\tx'),
+            (b'\x16x', {}, b'=16x'),
+            (b' x', {'quotetabs': True}, b'=20x'),
+            (b'\tx', {'quotetabs': True}, b'=09x'),
+            (b' \nn', {}, b'=20\nn'),
+            (b'\t\nn', {}, b'=09\nn'),
+            (b'x\nn', {}, b'x\nn'),
+            (b' \r\nn', {}, b'=20\r\nn'),
+            (b'\t\r\nn', {}, b'=09\r\nn'),
+            (b'x\r\nn', {}, b'x\r\nn'),
+            (b'x\nn', {'istext': False}, b'x=0An'),
+            (b'   ', {}, b'  =20'),
+            (b'   ', {'header': True}, b'__=20'),
+            (b'   \nn', {}, b'  =20\nn'),
+            (b'   \nn', {'header': True}, b'___\nn'),
+            (b'   ', {}, b'  =20'),
+            (b'\t\t\t', {'header': True}, b'\t\t=09'),
+            (b'\t\t\t\nn', {}, b'\t\t=09\nn'),
+            (b'\t\t\t\nn', {'header': True}, b'\t\t=09\nn'),
             ]:
             assert self.binascii.b2a_qp(input, **flags) == expected
 
     def test_a2b_hqx(self):
         for input, expected, done in [
-            ("", "", 0),
-            ("AAAA", "]u\xd7", 0),
-            ("A\nA\rAA", "]u\xd7", 0),
-            (":", "", 1),
-            ("A:", "", 1),
-            ("AA:", "]", 1),
-            ("AAA:", "]u", 1),
-            ("AAAA:", "]u\xd7", 1),
-            ("AAAA:foobarbaz", "]u\xd7", 1),
-            ("41-CZ:", "D\xe3\x19", 1),
-            ("41-CZl:", "D\xe3\x19\xbb", 1),
-            ("41-CZlm:", "D\xe3\x19\xbb\xbf", 1),
-            ("41-CZlm@:", "D\xe3\x19\xbb\xbf\x16", 1),
+            (b"", b"", 0),
+            (b"AAAA", b"]u\xd7", 0),
+            (b"A\nA\rAA", b"]u\xd7", 0),
+            (b":", b"", 1),
+            (b"A:", b"", 1),
+            (b"AA:", b"]", 1),
+            (b"AAA:", b"]u", 1),
+            (b"AAAA:", b"]u\xd7", 1),
+            (b"AAAA:foobarbaz", b"]u\xd7", 1),
+            (b"41-CZ:", b"D\xe3\x19", 1),
+            (b"41-CZl:", b"D\xe3\x19\xbb", 1),
+            (b"41-CZlm:", b"D\xe3\x19\xbb\xbf", 1),
+            (b"41-CZlm@:", b"D\xe3\x19\xbb\xbf\x16", 1),
             ]:
             assert self.binascii.a2b_hqx(input) == (expected, done)
         #
         for incomplete in [
-            "A",
-            "AA",
-            "AAA",
-            "12345",
-            "123456",
-            "1234560",
+            b"A",
+            b"AA",
+            b"AAA",
+            b"12345",
+            b"123456",
+            b"1234560",
             ]:
             raises(self.binascii.Incomplete, self.binascii.a2b_hqx, incomplete)
         #
         for bogus in [
-            "\x00",
-            ".",
-            "AAA AAAAAA:",
+            b"\x00",
+            b".",
+            b"AAA AAAAAA:",
             ]:
             raises(self.binascii.Error, self.binascii.a2b_hqx, bogus)
 
     def test_b2a_hqx(self):
         for input, expected in [
-            ("", ""),
-            ("A", "33"),
-            ("AB", "38)"),
-            ("ABC", "38*$"),
-            ("ABCD", "38*$4!"),
-            ("ABCDE", "38*$4%8"),
-            ("ABCDEF", "38*$4%9'"),
-            ("ABCDEFG", "38*$4%9'4`"),
-            ("]u\xd7", "AAAA"),
+            (b"", b""),
+            (b"A", b"33"),
+            (b"AB", b"38)"),
+            (b"ABC", b"38*$"),
+            (b"ABCD", b"38*$4!"),
+            (b"ABCDE", b"38*$4%8"),
+            (b"ABCDEF", b"38*$4%9'"),
+            (b"ABCDEFG", b"38*$4%9'4`"),
+            (b"]u\xd7", b"AAAA"),
             ]:
             assert self.binascii.b2a_hqx(input) == expected
 
     def test_rledecode_hqx(self):
         for input, expected in [
-            ("", ""),
-            ("hello world", "hello world"),
-            ("\x90\x00", "\x90"),
-            ("a\x90\x05", "a" * 5),
-            ("a\x90\xff", "a" * 0xFF),
-            ("abc\x90\x01def", "abcdef"),
-            ("abc\x90\x02def", "abccdef"),
-            ("abc\x90\x03def", "abcccdef"),
-            ("abc\x90\xa1def", "ab" + "c" * 0xA1 + "def"),
-            ("abc\x90\x03\x90\x02def", "abccccdef"),
-            ("abc\x90\x00\x90\x03def", "abc\x90\x90\x90def"),
-            ("abc\x90\x03\x90\x00def", "abccc\x90def"),
+            (b"", b""),
+            (b"hello world", b"hello world"),
+            (b"\x90\x00", b"\x90"),
+            (b"a\x90\x05", b"a" * 5),
+            (b"a\x90\xff", b"a" * 0xFF),
+            (b"abc\x90\x01def", b"abcdef"),
+            (b"abc\x90\x02def", b"abccdef"),
+            (b"abc\x90\x03def", b"abcccdef"),
+            (b"abc\x90\xa1def", b"ab" + b"c" * 0xA1 + b"def"),
+            (b"abc\x90\x03\x90\x02def", b"abccccdef"),
+            (b"abc\x90\x00\x90\x03def", b"abc\x90\x90\x90def"),
+            (b"abc\x90\x03\x90\x00def", b"abccc\x90def"),
             ]:
             assert self.binascii.rledecode_hqx(input) == expected
         #
         for input in [
-            "\x90",
-            "a\x90",
-            "hello world\x90",
+            b"\x90",
+            b"a\x90",
+            b"hello world\x90",
             ]:
             raises(self.binascii.Incomplete, self.binascii.rledecode_hqx,
                    input)
         #
-        raises(self.binascii.Error, self.binascii.rledecode_hqx, "\x90\x01")
-        raises(self.binascii.Error, self.binascii.rledecode_hqx, "\x90\x02")
-        raises(self.binascii.Error, self.binascii.rledecode_hqx, "\x90\xff")
+        raises(self.binascii.Error, self.binascii.rledecode_hqx, b"\x90\x01")
+        raises(self.binascii.Error, self.binascii.rledecode_hqx, b"\x90\x02")
+        raises(self.binascii.Error, self.binascii.rledecode_hqx, b"\x90\xff")
 
     def test_rlecode_hqx(self):
         for input, expected in [
-            ("", ""),
-            ("hello world", "hello world"),
-            ("helllo world", "helllo world"),
-            ("hellllo world", "hel\x90\x04o world"),
-            ("helllllo world", "hel\x90\x05o world"),
-            ("aaa", "aaa"),
-            ("aaaa", "a\x90\x04"),
-            ("a" * 0xff, "a\x90\xff"),
-            ("a" * 0x100, "a\x90\xffa"),
-            ("a" * 0x101, "a\x90\xffaa"),
-            ("a" * 0x102, "a\x90\xffaaa"),      # see comments in the source
-            ("a" * 0x103, "a\x90\xffa\x90\x04"),
-            ("a" * 0x1fe, "a\x90\xffa\x90\xff"),
-            ("a" * 0x1ff, "a\x90\xffa\x90\xffa"),
-            ("\x90", "\x90\x00"),
-            ("\x90" * 2, "\x90\x00" * 2),
-            ("\x90" * 3, "\x90\x00" * 3),       # see comments in the source
-            ("\x90" * 345, "\x90\x00" * 345),
+            (b"", b""),
+            (b"hello world", b"hello world"),
+            (b"helllo world", b"helllo world"),
+            (b"hellllo world", b"hel\x90\x04o world"),
+            (b"helllllo world", b"hel\x90\x05o world"),
+            (b"aaa", b"aaa"),
+            (b"aaaa", b"a\x90\x04"),
+            (b"a" * 0xff, b"a\x90\xff"),
+            (b"a" * 0x100, b"a\x90\xffa"),
+            (b"a" * 0x101, b"a\x90\xffaa"),
+            (b"a" * 0x102, b"a\x90\xffaaa"),      # see comments in the source
+            (b"a" * 0x103, b"a\x90\xffa\x90\x04"),
+            (b"a" * 0x1fe, b"a\x90\xffa\x90\xff"),
+            (b"a" * 0x1ff, b"a\x90\xffa\x90\xffa"),
+            (b"\x90", b"\x90\x00"),
+            (b"\x90" * 2, b"\x90\x00" * 2),
+            (b"\x90" * 3, b"\x90\x00" * 3),       # see comments in the source
+            (b"\x90" * 345, b"\x90\x00" * 345),
             ]:
             assert self.binascii.rlecode_hqx(input) == expected
 
     def test_crc_hqx(self):
         for input, initial, expected in [
-            ("", 0, 0),
-            ("", 123, 123),
-            ("hello", 321, 28955),
-            ("world", 65535, 12911),
-            ("uh", 40102, 37544),
-            ('a', 10000, 14338),
-            ('b', 10000, 2145),
-            ('c', 10000, 6208),
-            ('d', 10000, 26791),
-            ('e', 10000, 30854),
-            ('f', 10000, 18661),
-            ('g', 10000, 22724),
-            ('h', 10000, 43307),
-            ('i', 10000, 47370),
-            ('j', 10000, 35177),
-            ('k', 10000, 39240),
-            ('l', 10000, 59823),
-            ('m', 10000, 63886),
-            ('n', 10000, 51693),
-            ('o', 10000, 55756),
-            ('p', 10000, 14866),
-            ('q', 10000, 10803),
-            ('r', 10000, 6736),
-            ('s', 10000, 2673),
-            ('t', 10000, 31382),
-            ('u', 10000, 27319),
-            ('v', 10000, 23252),
-            ('w', 10000, 19189),
-            ('x', 10000, 47898),
-            ('y', 10000, 43835),
-            ('z', 10000, 39768),
+            (b"", 0, 0),
+            (b"", 123, 123),
+            (b"hello", 321, 28955),
+            (b"world", 65535, 12911),
+            (b"uh", 40102, 37544),
+            (b'a', 10000, 14338),
+            (b'b', 10000, 2145),
+            (b'c', 10000, 6208),
+            (b'd', 10000, 26791),
+            (b'e', 10000, 30854),
+            (b'f', 10000, 18661),
+            (b'g', 10000, 22724),
+            (b'h', 10000, 43307),
+            (b'i', 10000, 47370),
+            (b'j', 10000, 35177),
+            (b'k', 10000, 39240),
+            (b'l', 10000, 59823),
+            (b'm', 10000, 63886),
+            (b'n', 10000, 51693),
+            (b'o', 10000, 55756),
+            (b'p', 10000, 14866),
+            (b'q', 10000, 10803),
+            (b'r', 10000, 6736),
+            (b's', 10000, 2673),
+            (b't', 10000, 31382),
+            (b'u', 10000, 27319),
+            (b'v', 10000, 23252),
+            (b'w', 10000, 19189),
+            (b'x', 10000, 47898),
+            (b'y', 10000, 43835),
+            (b'z', 10000, 39768),
             ]:
             assert self.binascii.crc_hqx(input, initial) == expected
 
     def test_crc32(self):
         for input, initial, expected in [
-            ("", 0, 0),
-            ("", 123, 123),
-            ("hello", 321, -348147686),
-            ("world", -2147483648, 32803080),
-            ("world", 2147483647, 942244330),
-            ('a', 10000, -184504832),
-            ('b', 10000, 1812594618),
-            ('c', 10000, 453955372),
-            ('d', 10000, -2056627569),
-            ('e', 10000, -227710439),
-            ('f', 10000, 1801730979),
-            ('g', 10000, 476252981),
-            ('h', 10000, -1931733340),
-            ('i', 10000, -69523918),
-            ('j', 10000, 1657960328),
-            ('k', 10000, 366298910),
-            ('l', 10000, -1951280451),
-            ('m', 10000, -55123413),
-            ('n', 10000, 1707062161),
-            ('o', 10000, 314082055),
-            ('p', 10000, -1615819022),
-            ('q', 10000, -390611356),
-            ('r', 10000, 1908338654),
-            ('s', 10000, 112844616),
-            ('t', 10000, -1730327829),
-            ('u', 10000, -270894467),
-            ('v', 10000, 1993550791),
-            ('w', 10000, 30677841),
-            ('x', 10000, -1855256896),
-            ('y', 10000, -429115818),
-            ('z', 10000, 2137352172),
-            ('foo', 99999999999999999999999999, -1932704816),
-            ('bar', -99999999999999999999999999, 2000545409),
+            (b"", 0, 0),
+            (b"", 123, 123),
+            (b"hello", 321, -348147686),
+            (b"world", -2147483648, 32803080),
+            (b"world", 2147483647, 942244330),
+            (b'a', 10000, -184504832),
+            (b'b', 10000, 1812594618),
+            (b'c', 10000, 453955372),
+            (b'd', 10000, -2056627569),
+            (b'e', 10000, -227710439),
+            (b'f', 10000, 1801730979),
+            (b'g', 10000, 476252981),
+            (b'h', 10000, -1931733340),
+            (b'i', 10000, -69523918),
+            (b'j', 10000, 1657960328),
+            (b'k', 10000, 366298910),
+            (b'l', 10000, -1951280451),
+            (b'm', 10000, -55123413),
+            (b'n', 10000, 1707062161),
+            (b'o', 10000, 314082055),
+            (b'p', 10000, -1615819022),
+            (b'q', 10000, -390611356),
+            (b'r', 10000, 1908338654),
+            (b's', 10000, 112844616),
+            (b't', 10000, -1730327829),
+            (b'u', 10000, -270894467),
+            (b'v', 10000, 1993550791),
+            (b'w', 10000, 30677841),
+            (b'x', 10000, -1855256896),
+            (b'y', 10000, -429115818),
+            (b'z', 10000, 2137352172),
+            (b'foo', 99999999999999999999999999, -1932704816),
+            (b'bar', -99999999999999999999999999, 2000545409),
             ]:
             assert self.binascii.crc32(input, initial) == expected
 
     def test_hexlify(self):
         for input, expected in [
-            ("", ""),
-            ("0", "30"),
-            ("1", "31"),
-            ("2", "32"),
-            ("8", "38"),
-            ("9", "39"),
-            ("A", "41"),
-            ("O", "4f"),
-            ("\xde", "de"),
-            ("ABC", "414243"),
-            ("\x00\x00\x00\xff\x00\x00", "000000ff0000"),
-            ("\x28\x9c\xc8\xc0\x3d\x8e", "289cc8c03d8e"),
+            (b"", b""),
+            (b"0", b"30"),
+            (b"1", b"31"),
+            (b"2", b"32"),
+            (b"8", b"38"),
+            (b"9", b"39"),
+            (b"A", b"41"),
+            (b"O", b"4f"),
+            (b"\xde", b"de"),
+            (b"ABC", b"414243"),
+            (b"\x00\x00\x00\xff\x00\x00", b"000000ff0000"),
+            (b"\x28\x9c\xc8\xc0\x3d\x8e", b"289cc8c03d8e"),
             ]:
             assert self.binascii.hexlify(input) == expected
             assert self.binascii.b2a_hex(input) == expected
 
     def test_unhexlify(self):
         for input, expected in [
-            ("", ""),
-            ("30", "0"),
-            ("31", "1"),
-            ("32", "2"),
-            ("38", "8"),
-            ("39", "9"),
-            ("41", "A"),
-            ("4F", "O"),
-            ("4f", "O"),
-            ("DE", "\xde"),
-            ("De", "\xde"),
-            ("dE", "\xde"),
-            ("de", "\xde"),
-            ("414243", "ABC"),
-            ("000000FF0000", "\x00\x00\x00\xff\x00\x00"),
-            ("289cc8C03d8e", "\x28\x9c\xc8\xc0\x3d\x8e"),
+            (b"", b""),
+            (b"30", b"0"),
+            (b"31", b"1"),
+            (b"32", b"2"),
+            (b"38", b"8"),
+            (b"39", b"9"),
+            (b"41", b"A"),
+            (b"4F", b"O"),
+            (b"4f", b"O"),
+            (b"DE", b"\xde"),
+            (b"De", b"\xde"),
+            (b"dE", b"\xde"),
+            (b"de", b"\xde"),
+            (b"414243", b"ABC"),
+            (b"000000FF0000", b"\x00\x00\x00\xff\x00\x00"),
+            (b"289cc8C03d8e", b"\x28\x9c\xc8\xc0\x3d\x8e"),
             ]:
             assert self.binascii.unhexlify(input) == expected
             assert self.binascii.a2b_hex(input) == expected

From noreply at buildbot.pypy.org  Wed Oct 19 23:11:13 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 19 Oct 2011 23:11:13 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Remove many modules from "working_modules".
Message-ID: <20111019211113.6238F820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48239:f24a5ad97e35
Date: 2011-10-19 22:31 +0200
http://bitbucket.org/pypy/pypy/changeset/f24a5ad97e35/

Log:	Remove many modules from "working_modules". There are still enough
	builtin modules to run most regression tests

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -37,6 +37,18 @@
      "_continuation"]
 ))
 
+# XXX Here is the list of modules not known to work yet
+for name in [
+    "mmap", "_locale", "pwd",
+    "zipimport", "_lsprof",
+    "crypt", "_rawffi", "termios", "zlib", "bz2",
+    "_hashlib", "_md5", "_sha", "_minimal_curses", "cStringIO",
+    "thread", "itertools", "pyexpat", "_ssl", "cpyext",
+    "_bisect", "_multiprocessing",
+    "_collections", "_multibytecodec", "micronumpy", "_ffi",
+    "_continuation"]:
+    del working_modules[name]
+
 translation_modules = default_modules.copy()
 translation_modules.update(dict.fromkeys(
     ["fcntl", "rctime", "select", "signal", "_rawffi", "zlib",

From noreply at buildbot.pypy.org  Wed Oct 19 23:11:14 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 19 Oct 2011 23:11:14 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Fix all tests in module/_locale
Message-ID: <20111019211114.94746820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48240:149eb1fd7f2a
Date: 2011-10-19 22:31 +0200
http://bitbucket.org/pypy/pypy/changeset/149eb1fd7f2a/

Log:	Fix all tests in module/_locale

diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -73,8 +73,7 @@
         if self.unicodedata_handler:
             return self.unicodedata_handler
         try:
-            w_builtin = space.getbuiltinmodule('__builtin__')
-            w_import = space.getattr(w_builtin, space.wrap("__import__"))
+            w_import = space.getattr(space.builtin, space.wrap("__import__"))
             w_unicodedata = space.call_function(w_import,
                                                 space.wrap("unicodedata"))
             w_getcode = space.getattr(w_unicodedata, space.wrap("_get_code"))
diff --git a/pypy/module/_locale/interp_locale.py b/pypy/module/_locale/interp_locale.py
--- a/pypy/module/_locale/interp_locale.py
+++ b/pypy/module/_locale/interp_locale.py
@@ -21,7 +21,7 @@
 
 def _fixup_ulcase(space):
     stringmod = space.call_function(
-        space.getattr(space.getbuiltinmodule('__builtin__'),
+        space.getattr(space.getbuiltinmodule('builtins'),
                       space.wrap('__import__')), space.wrap('string'))
     # create uppercase map string
     ul = []
@@ -69,6 +69,12 @@
         groups.append(space.wrap(0))
     return space.newlist(groups)
 
+def charp2uni(space, s):
+    "Convert a char* pointer to unicode according to the current locale"
+    w_bytes = space.wrapbytes(rffi.charp2str(s))
+    # XXX use mbstowcs()
+    return space.call_method(w_bytes, "decode", space.wrap("utf-8"))
+
 def localeconv(space):
     "() -> dict. Returns numeric and monetary locale-specific parameters."
     lp = rlocale.localeconv()
@@ -77,25 +83,25 @@
     w_result = space.newdict()
     w = space.wrap
     space.setitem(w_result, w("decimal_point"),
-                  w(rffi.charp2str(lp.c_decimal_point)))
+                  charp2uni(space, lp.c_decimal_point))
     space.setitem(w_result, w("thousands_sep"),
-                  w(rffi.charp2str(lp.c_thousands_sep)))
+                  charp2uni(space, lp.c_thousands_sep))
     space.setitem(w_result, w("grouping"),
                   _w_copy_grouping(space, rffi.charp2str(lp.c_grouping)))
     space.setitem(w_result, w("int_curr_symbol"),
-                  w(rffi.charp2str(lp.c_int_curr_symbol)))
+                  charp2uni(space, lp.c_int_curr_symbol))
     space.setitem(w_result, w("currency_symbol"),
-                  w(rffi.charp2str(lp.c_currency_symbol)))
+                  charp2uni(space, lp.c_currency_symbol))
     space.setitem(w_result, w("mon_decimal_point"),
-                  w(rffi.charp2str(lp.c_mon_decimal_point)))
+                  charp2uni(space, lp.c_mon_decimal_point))
     space.setitem(w_result, w("mon_thousands_sep"),
-                  w(rffi.charp2str(lp.c_mon_thousands_sep)))
+                  charp2uni(space, lp.c_mon_thousands_sep))
     space.setitem(w_result, w("mon_grouping"),
                   _w_copy_grouping(space, rffi.charp2str(lp.c_mon_grouping)))
     space.setitem(w_result, w("positive_sign"),
-                  w(rffi.charp2str(lp.c_positive_sign)))
+                  charp2uni(space, lp.c_positive_sign))
     space.setitem(w_result, w("negative_sign"),
-                  w(rffi.charp2str(lp.c_negative_sign)))
+                  charp2uni(space, lp.c_negative_sign))
     space.setitem(w_result, w("int_frac_digits"),
                   w(lp.c_int_frac_digits))
     space.setitem(w_result, w("frac_digits"),
@@ -115,29 +121,11 @@
 
     return w_result
 
-_strcoll = rlocale.external('strcoll', [rffi.CCHARP, rffi.CCHARP], rffi.INT)
 _wcscoll = rlocale.external('wcscoll', [rffi.CWCHARP, rffi.CWCHARP], rffi.INT)
 
 def strcoll(space, w_s1, w_s2):
     "string,string -> int. Compares two strings according to the locale."
 
-    if space.is_true(space.isinstance(w_s1, space.w_str)) and \
-       space.is_true(space.isinstance(w_s2, space.w_str)):
-
-        s1, s2 = space.str_w(w_s1), space.str_w(w_s2)
-        s1_c = rffi.str2charp(s1)
-        s2_c = rffi.str2charp(s2)
-        try:
-            return space.wrap(_strcoll(s1_c, s2_c))
-        finally:
-            rffi.free_charp(s1_c)
-            rffi.free_charp(s2_c)
-
-    #if not space.is_true(space.isinstance(w_s1, space.w_unicode)) and \
-    #   not space.is_true(space.isinstance(w_s2, space.w_unicode)):
-    #    raise OperationError(space.w_ValueError,
-    #                         space.wrap("strcoll arguments must be strings"))
-
     s1, s2 = space.unicode_w(w_s1), space.unicode_w(w_s2)
 
     s1_c = rffi.unicode2wcharp(s1)
diff --git a/pypy/module/_locale/test/test_locale.py b/pypy/module/_locale/test/test_locale.py
--- a/pypy/module/_locale/test/test_locale.py
+++ b/pypy/module/_locale/test/test_locale.py
@@ -35,6 +35,7 @@
                 _locale.setlocale(_locale.LC_ALL,
                                   space.str_w(cls.w_language_pl))
             except _locale.Error:
+                raise
                 py.test.skip("necessary locales not installed")
 
             # Windows forbids the UTF-8 character set since Windows XP.
@@ -160,11 +161,11 @@
         _locale.setlocale(_locale.LC_ALL, self.language_pl)
         assert _locale.strcoll("a", "b") < 0
         assert _locale.strcoll(
-            u"\N{LATIN SMALL LETTER A WITH OGONEK}".encode(self.encoding_pl),
+            u"\N{LATIN SMALL LETTER A WITH OGONEK}",
             "b") < 0
 
         assert _locale.strcoll(
-            u"\N{LATIN SMALL LETTER C WITH ACUTE}".encode(self.encoding_pl),
+            u"\N{LATIN SMALL LETTER C WITH ACUTE}",
             "b") > 0
         assert _locale.strcoll("c", "b") > 0
 
@@ -173,17 +174,6 @@
         raises(TypeError, _locale.strcoll, 1, "b")
         raises(TypeError, _locale.strcoll, "b", 1)
 
-    def test_strcoll_unicode(self):
-        import _locale
-
-        _locale.setlocale(_locale.LC_ALL, self.language_pl)
-        assert _locale.strcoll(u"b", u"b") == 0
-        assert _locale.strcoll(u"a", u"b") < 0
-        assert _locale.strcoll(u"b", u"a") > 0
-
-        raises(TypeError, _locale.strcoll, 1, u"b")
-        raises(TypeError, _locale.strcoll, u"b", 1)
-
     def test_strxfrm(self):
         # TODO more tests would be nice
         import _locale

From noreply at buildbot.pypy.org  Wed Oct 19 23:11:15 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 19 Oct 2011 23:11:15 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Lot of fixes in the _codecs module
Message-ID: <20111019211115.D20B0820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48241:e0cf3d8b87a2
Date: 2011-10-19 22:31 +0200
http://bitbucket.org/pypy/pypy/changeset/e0cf3d8b87a2/

Log:	Lot of fixes in the _codecs module

diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -49,7 +49,7 @@
                            "(unicode, int) tuple, not %s")
                 raise operationerrfmt(
                     space.w_TypeError, msg,
-                    space.str_w(space.repr(w_res)))
+                    space.unicode_w(space.repr(w_res)))
             w_replace, w_newpos = space.fixedview(w_res, 2)
             newpos = space.int_w(w_newpos)
             if newpos < 0:
@@ -487,7 +487,7 @@
     make_encoder_wrapper('mbcs_encode')
     make_decoder_wrapper('mbcs_decode')
 
- at unwrap_spec(data=str, errors='str_or_None', byteorder=int)
+ at unwrap_spec(data="bufferstr", errors='str_or_None', byteorder=int)
 def utf_16_ex_decode(space, data, errors='strict', byteorder=0, w_final=False):
     if errors is None:
         errors = 'strict'
@@ -507,7 +507,7 @@
     return space.newtuple([space.wrap(res), space.wrap(consumed),
                            space.wrap(byteorder)])
 
- at unwrap_spec(data=str, errors='str_or_None', byteorder=int)
+ at unwrap_spec(data="bufferstr", errors='str_or_None', byteorder=int)
 def utf_32_ex_decode(space, data, errors='strict', byteorder=0, w_final=False):
     final = space.is_true(w_final)
     state = space.fromcache(CodecState)
@@ -599,7 +599,7 @@
 
         # Charmap may return a string
         try:
-            x = space.realstr_w(w_ch)
+            x = space.bytes_w(w_ch)
         except OperationError, e:
             if not e.match(space, space.w_TypeError):
                 raise
@@ -626,7 +626,7 @@
         raise OperationError(space.w_TypeError, space.wrap("invalid mapping"))
 
 
- at unwrap_spec(string=str, errors='str_or_None')
+ at unwrap_spec(string="bufferstr", errors='str_or_None')
 def charmap_decode(space, string, errors="strict", w_mapping=None):
     if errors is None:
         errors = 'strict'
@@ -658,7 +658,7 @@
     result = runicode.unicode_encode_charmap(
         uni, len(uni), errors,
         state.encode_error_handler, mapping)
-    return space.newtuple([space.wrap(result), space.wrap(len(uni))])
+    return space.newtuple([space.wrapbytes(result), space.wrap(len(uni))])
 
 
 @unwrap_spec(chars=unicode)
@@ -716,7 +716,7 @@
     if space.isinstance_w(w_string, space.w_unicode):
         return space.newtuple([w_string, space.len(w_string)])
 
-    string = space.str_w(w_string)
+    string = space.bytes_w(w_string)
 
     if len(string) == 0:
         return space.newtuple([space.wrap(u''), space.wrap(0)])
@@ -729,21 +729,21 @@
     return space.newtuple([space.wrap(result), space.wrap(consumed)])
 
 # ____________________________________________________________
-# support for the "string escape" codec
+# support for the "string escape" translation
 # This is a bytes-to bytes transformation
 
- at unwrap_spec(data=str, errors='str_or_None')
+ at unwrap_spec(data="bufferstr", errors='str_or_None')
 def escape_encode(space, data, errors='strict'):
     from pypy.objspace.std.stringobject import string_escape_encode
     result = string_escape_encode(data, quote="'")
     start = 1
     end = len(result) - 1
     assert end >= 0
-    w_result = space.wrap(result[start:end])
+    w_result = space.wrapbytes(result[start:end])
     return space.newtuple([w_result, space.wrap(len(data))])
 
- at unwrap_spec(data=str, errors='str_or_None')
+ at unwrap_spec(data="bufferstr", errors='str_or_None')
 def escape_decode(space, data, errors='strict'):
     from pypy.interpreter.pyparser.parsestring import PyString_DecodeEscape
     result = PyString_DecodeEscape(space, data, None)
-    return space.newtuple([space.wrap(result), space.wrap(len(data))])
+    return space.newtuple([space.wrapbytes(result), space.wrap(len(data))])
diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py
--- a/pypy/module/_codecs/test/test_codecs.py
+++ b/pypy/module/_codecs/test/test_codecs.py
@@ -17,7 +17,7 @@
                          'utf-32', 'utf-32-le', 'utf-32-be',
                          'raw_unicode_escape',
                          'unicode_escape', 'unicode_internal'):
-            assert unicode(u.encode(encoding),encoding) == u
+            assert str(u.encode(encoding),encoding) == u
 
     def test_ucs4(self):
         x = u'\U00100000'
@@ -25,14 +25,14 @@
         assert x == y 
 
     def test_named_unicode(self):
-        assert unicode('\\N{SPACE}','unicode-escape') == u" "
-        raises( UnicodeDecodeError, unicode,'\\N{SPACE','unicode-escape')
-        raises( UnicodeDecodeError, unicode,'\\NSPACE}','unicode-escape')
-        raises( UnicodeDecodeError, unicode,'\\NSPACE','unicode-escape')
-        raises( UnicodeDecodeError, unicode,'\\N','unicode-escape')
-        assert  unicode('\\N{SPACE}\\N{SPACE}','unicode-escape') == u"  " 
-        assert  unicode('\\N{SPACE}a\\N{SPACE}','unicode-escape') == u" a " 
-        assert "\\N{foo}xx".decode("unicode-escape", "ignore") == u"xx"
+        assert str(b'\\N{SPACE}','unicode-escape') == u" "
+        raises( UnicodeDecodeError, str,b'\\N{SPACE','unicode-escape')
+        raises( UnicodeDecodeError, str,b'\\NSPACE}','unicode-escape')
+        raises( UnicodeDecodeError, str,b'\\NSPACE','unicode-escape')
+        raises( UnicodeDecodeError, str,b'\\N','unicode-escape')
+        assert  str(b'\\N{SPACE}\\N{SPACE}','unicode-escape') == u"  " 
+        assert  str(b'\\N{SPACE}a\\N{SPACE}','unicode-escape') == u" a " 
+        assert b"\\N{foo}xx".decode("unicode-escape", "ignore") == u"xx"
         assert 1 <= len(u"\N{CJK UNIFIED IDEOGRAPH-20000}") <= 2
 
     def test_literals(self):
@@ -40,26 +40,26 @@
 
     def test_insecure_pickle(self):
         import pickle
-        insecure = ["abc", "2 + 2", # not quoted
+        insecure = [b"abc", b"2 + 2", # not quoted
                     #"'abc' + 'def'", # not a single quoted string
-                    "'abc", # quote is not closed
-                    "'abc\"", # open quote and close quote don't match
-                    "'abc'   ?", # junk after close quote
-                    "'\\'", # trailing backslash
+                    b"'abc", # quote is not closed
+                    b"'abc\"", # open quote and close quote don't match
+                    b"'abc'   ?", # junk after close quote
+                    b"'\\'", # trailing backslash
                     # some tests of the quoting rules
                     #"'abc\"\''",
                     #"'\\\\a\'\'\'\\\'\\\\\''",
                     ]
         for s in insecure:
-            buf = "S" + s + "\012p0\012."
+            buf = b"S" + s + b"\012p0\012."
             raises (ValueError, pickle.loads, buf)
 
     def test_unicodedecodeerror(self):
         assert str(UnicodeDecodeError(
-            "ascii", "g\xfcrk", 1, 2, "ouch")) == "'ascii' codec can't decode byte 0xfc in position 1: ouch"
+            "ascii", b"g\xfcrk", 1, 2, "ouch")) == "'ascii' codec can't decode byte 0xfc in position 1: ouch"
         
         assert str(UnicodeDecodeError(
-            "ascii", "g\xfcrk", 1, 3, "ouch")) == "'ascii' codec can't decode bytes in position 1-2: ouch"
+            "ascii", b"g\xfcrk", 1, 3, "ouch")) == "'ascii' codec can't decode bytes in position 1-2: ouch"
         
 
     def test_unicodetranslateerror(self):
@@ -73,7 +73,7 @@
         assert str(UnicodeTranslateError(
             u"g\uffffrk", 1, 2, "ouch"))== "can't translate character u'\\uffff' in position 1: ouch"
         
-        if sys.maxunicode > 0xffff and len(unichr(0x10000)) == 1:
+        if sys.maxunicode > 0xffff and len(chr(0x10000)) == 1:
             assert str(UnicodeTranslateError(
                 u"g\U00010000rk", 1, 2, "ouch"))== "can't translate character u'\\U00010000' in position 1: ouch"
             
@@ -96,30 +96,31 @@
        
         assert str(UnicodeEncodeError(
             "ascii", u"\uffffx", 0, 1, "ouch"))=="'ascii' codec can't encode character u'\\uffff' in position 0: ouch"
-        if sys.maxunicode > 0xffff and len(unichr(0x10000)) == 1:
+        if sys.maxunicode > 0xffff and len(chr(0x10000)) == 1:
             assert str(UnicodeEncodeError(
                 "ascii", u"\U00010000x", 0, 1, "ouch")) =="'ascii' codec can't encode character u'\\U00010000' in position 0: ouch"
     
     def test_indexerror(self):
-        test =   "\\"     # trailing backslash
-        raises (ValueError, test.decode,'string-escape')
+        import _codecs
+        test =   b"\\"     # trailing backslash
+        raises (ValueError, _codecs.escape_decode, test)
 
     def test_charmap_decode(self):
         from _codecs import charmap_decode
         import sys
-        assert charmap_decode('', 'strict', 'blablabla') == ('', 0)
-        assert charmap_decode('xxx') == ('xxx', 3)
-        assert charmap_decode('xxx', 'strict', {ord('x'): u'XX'}) == ('XXXXXX', 3)
-        map = tuple([unichr(i) for i in range(256)])
-        assert charmap_decode('xxx\xff', 'strict', map) == (u'xxx\xff', 4)
+        assert charmap_decode(b'', 'strict', 'blablabla') == ('', 0)
+        assert charmap_decode(b'xxx') == ('xxx', 3)
+        assert charmap_decode(b'xxx', 'strict', {ord('x'): u'XX'}) == ('XXXXXX', 3)
+        map = tuple([chr(i) for i in range(256)])
+        assert charmap_decode(b'xxx\xff', 'strict', map) == (u'xxx\xff', 4)
 
         raises(TypeError, charmap_decode, '\xff', "replace",  {0xff: 0x10001})
 
     def test_unicode_escape(self):
         from _codecs import unicode_escape_encode, unicode_escape_decode
         assert unicode_escape_encode(u'abc') == (u'abc'.encode('unicode_escape'), 3)
-        assert unicode_escape_decode('abc') == (u'abc'.decode('unicode_escape'), 3)
-        assert unicode_escape_decode('\\x61\\x62\\x63') == (u'abc', 12)
+        assert unicode_escape_decode(b'abc') == (b'abc'.decode('unicode_escape'), 3)
+        assert unicode_escape_decode(b'\\x61\\x62\\x63') == (u'abc', 12)
 
 class AppTestPartialEvaluation:
 
@@ -144,13 +145,13 @@
                 u"\x00\xff\u07ff\u0800\uffff",
             ]
             
-        buffer = ''
+        buffer = b''
         result = u""
         for (c, partialresult) in zip(u"\x00\xff\u07ff\u0800\uffff".encode(encoding), check_partial):
-            buffer += c
+            buffer += bytes([c])
             res = _codecs.utf_8_decode(buffer,'strict',False)
             if res[1] >0 :
-                buffer = ''
+                buffer = b''
             result += res[0]
             assert result == partialresult
 
@@ -169,26 +170,26 @@
                     u"\x00\xff\u0100",
                     u"\x00\xff\u0100\uffff",
                 ]
-        buffer = ''
+        buffer = b''
         result = u""
         for (c, partialresult) in zip(u"\x00\xff\u0100\uffff".encode(encoding), check_partial):
-            buffer += c
+            buffer += bytes([c])
             res = _codecs.utf_16_decode(buffer,'strict',False)
             if res[1] >0 :
-                buffer = ''
+                buffer = b''
             result += res[0]
             assert result == partialresult
 
     def test_bug1098990_a(self):
 
-        import codecs, StringIO
+        import codecs, io
         self.encoding = 'utf-8'
         s1 = u"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy\r\n"
         s2 = u"offending line: ladfj askldfj klasdj fskla dfzaskdj fasklfj laskd fjasklfzzzzaa%whereisthis!!!\r\n"
         s3 = u"next line.\r\n"
        
         s = (s1+s2+s3).encode(self.encoding)
-        stream = StringIO.StringIO(s)
+        stream = io.BytesIO(s)
         reader = codecs.getreader(self.encoding)(stream)
         assert reader.readline() == s1
         assert reader.readline() == s2
@@ -196,7 +197,7 @@
         assert reader.readline() == u""
 
     def test_bug1098990_b(self):
-        import codecs, StringIO
+        import codecs, io
         self.encoding = 'utf-8'
         s1 = u"aaaaaaaaaaaaaaaaaaaaaaaa\r\n"
         s2 = u"bbbbbbbbbbbbbbbbbbbbbbbb\r\n"
@@ -205,7 +206,7 @@
         s5 = u"againokay.\r\n"
 
         s = (s1+s2+s3+s4+s5).encode(self.encoding)
-        stream = StringIO.StringIO(s)
+        stream = io.BytesIO(s)
         reader = codecs.getreader(self.encoding)(stream)
         assert reader.readline() == s1
         assert reader.readline() == s2
@@ -216,11 +217,11 @@
     
     def test_seek_utf16le(self):
         # all codecs should be able to encode these
-        import codecs, StringIO
+        import codecs, io
         encoding = 'utf-16-le'
         s = u"%s\n%s\n" % (10*u"abc123", 10*u"def456")
-        reader = codecs.getreader(encoding)(StringIO.StringIO(s.encode(encoding)))
-        for t in xrange(5):
+        reader = codecs.getreader(encoding)(io.BytesIO(s.encode(encoding)))
+        for t in range(5):
             # Test that calling seek resets the internal codec state and buffers
             reader.seek(0, 0)
             line = reader.readline()
@@ -229,71 +230,75 @@
 
     def test_unicode_internal_encode(self):
         import sys
-        class U(unicode):
+        class U(str):
             pass
         enc = U(u"a").encode("unicode_internal")
         if sys.maxunicode == 65535: # UCS2 build
             if sys.byteorder == "big":
-                assert enc == "\x00a"
+                assert enc == b"\x00a"
             else:
-                assert enc == "a\x00"
+                assert enc == b"a\x00"
         elif len(u"\U00010098") == 1:
             # UCS4 build on a UCS4 CPython
             enc2 = u"\U00010098".encode("unicode_internal")
             if sys.byteorder == "big":
-                assert enc == "\x00\x00\x00a"
-                assert enc2 == "\x00\x01\x00\x98"
+                assert enc == b"\x00\x00\x00a"
+                assert enc2 == b"\x00\x01\x00\x98"
             else:
-                assert enc == "a\x00\x00\x00"
-                assert enc2 == "\x98\x00\x01\x00"
+                assert enc == b"a\x00\x00\x00"
+                assert enc2 == b"\x98\x00\x01\x00"
         else:
             # UCS4 build on a UCS2 CPython
             if sys.byteorder == "big":
-                assert enc == "\x00\x00\x00a"
+                assert enc == b"\x00\x00\x00a"
             else:
-                assert enc == "a\x00\x00\x00"
+                assert enc == b"a\x00\x00\x00"
 
     def test_unicode_internal_decode(self):
         import sys
         if sys.maxunicode == 65535: # UCS2 build
             if sys.byteorder == "big":
-                bytes = "\x00a"
+                bytes = b"\x00a"
             else:
-                bytes = "a\x00"
+                bytes = b"a\x00"
         else: # UCS4 build
             if sys.byteorder == "big":
-                bytes = "\x00\x00\x00a"
-                bytes2 = "\x00\x01\x00\x98"
+                bytes = b"\x00\x00\x00a"
+                bytes2 = b"\x00\x01\x00\x98"
             else:
-                bytes = "a\x00\x00\x00"
-                bytes2 = "\x98\x00\x01\x00"
+                bytes = b"a\x00\x00\x00"
+                bytes2 = b"\x98\x00\x01\x00"
             assert bytes2.decode("unicode_internal") == u"\U00010098"
         assert bytes.decode("unicode_internal") == u"a"
 
     def test_raw_unicode_escape(self):
-        assert unicode("\u0663", "raw-unicode-escape") == u"\u0663"
-        assert u"\u0663".encode("raw-unicode-escape") == "\u0663"
+        assert str(b"\u0663", "raw-unicode-escape") == u"\u0663"
+        assert u"\u0663".encode("raw-unicode-escape") == b"\u0663"
 
     def test_escape_decode(self):
-        test = 'a\n\\b\x00c\td\u2045'.encode('string_escape')
-        assert test.decode('string_escape') =='a\n\\b\x00c\td\u2045'
-        assert '\\077'.decode('string_escape') == '?'
-        assert '\\100'.decode('string_escape') == '@'
-        assert '\\253'.decode('string_escape') == chr(0253)
-        assert '\\312'.decode('string_escape') == chr(0312)
+        import _codecs
+        test = _codecs.escape_encode(b'a\n\\b\x00c\td\u2045')[0]
+        assert _codecs.escape_decode(test)[0] == b'a\n\\b\x00c\td\u2045'
+        assert _codecs.escape_decode(b'\\077')[0] == b'?'
+        assert _codecs.escape_decode(b'\\100')[0] == b'@'
+        assert _codecs.escape_decode(b'\\253')[0] == bytes([0253])
+        assert _codecs.escape_decode(b'\\312')[0] == bytes([0312])
 
     def test_escape_decode_wrap_around(self):
-        assert '\\400'.decode('string_escape') == chr(0)
+        import _codecs
+        assert _codecs.escape_decode(b'\\400')[0] == b'\0'
 
     def test_escape_decode_ignore_invalid(self):
-        assert '\\9'.decode('string_escape') == '\\9'
-        assert '\\01'.decode('string_escape') == chr(01)
-        assert '\\0f'.decode('string_escape') == chr(0) + 'f'
-        assert '\\08'.decode('string_escape') == chr(0) + '8'
+        import _codecs
+        assert _codecs.escape_decode(b'\\9')[0] == b'\\9'
+        assert _codecs.escape_decode(b'\\01')[0] == b'\x01'
+        assert _codecs.escape_decode(b'\\0f')[0] == b'\0' + b'f'
+        assert _codecs.escape_decode(b'\\08')[0] == b'\0' + b'8'
 
     def test_escape_encode(self):
-        assert '"'.encode('string_escape') == '"'
-        assert "'".encode('string_escape') == "\\'"
+        import _codecs
+        assert _codecs.escape_encode(b'"')[0] == b'"'
+        assert _codecs.escape_encode(b"'")[0] == b"\\'"
 
     def test_decode_utf8_different_case(self):
         constant = u"a"
@@ -304,35 +309,35 @@
         def search_function(encoding):
             def f(input, errors="strict"):
                 return 42
-            print encoding
+            print(encoding)
             if encoding == 'test.mytestenc':
                 return (f, f, None, None)
             return None
         _codecs.register(search_function)
-        raises(TypeError, "hello".decode, "test.mytestenc")
+        raises(TypeError, b"hello".decode, "test.mytestenc")
         raises(TypeError, u"hello".encode, "test.mytestenc")
 
     def test_cpytest_decode(self):
         import codecs
-        assert codecs.decode('\xe4\xf6\xfc', 'latin-1') == u'\xe4\xf6\xfc'
+        assert codecs.decode(b'\xe4\xf6\xfc', 'latin-1') == u'\xe4\xf6\xfc'
         raises(TypeError, codecs.decode)
-        assert codecs.decode('abc') == u'abc'
-        raises(UnicodeDecodeError, codecs.decode, '\xff', 'ascii')
+        assert codecs.decode(b'abc') == u'abc'
+        raises(UnicodeDecodeError, codecs.decode, b'\xff', 'ascii')
 
     def test_bad_errorhandler_return(self):
         import codecs
         def baddecodereturn1(exc):
             return 42
         codecs.register_error("test.baddecodereturn1", baddecodereturn1)
-        raises(TypeError, "\xff".decode, "ascii", "test.baddecodereturn1")
-        raises(TypeError, "\\".decode, "unicode-escape", "test.baddecodereturn1")
-        raises(TypeError, "\\x0".decode, "unicode-escape", "test.baddecodereturn1")
-        raises(TypeError, "\\x0y".decode, "unicode-escape", "test.baddecodereturn1")
-        raises(TypeError, "\\Uffffeeee".decode, "unicode-escape", "test.baddecodereturn1")
-        raises(TypeError, "\\uyyyy".decode, "raw-unicode-escape", "test.baddecodereturn1")
+        raises(TypeError, b"\xff".decode, "ascii", "test.baddecodereturn1")
+        raises(TypeError, b"\\".decode, "unicode-escape", "test.baddecodereturn1")
+        raises(TypeError, b"\\x0".decode, "unicode-escape", "test.baddecodereturn1")
+        raises(TypeError, b"\\x0y".decode, "unicode-escape", "test.baddecodereturn1")
+        raises(TypeError, b"\\Uffffeeee".decode, "unicode-escape", "test.baddecodereturn1")
+        raises(TypeError, b"\\uyyyy".decode, "raw-unicode-escape", "test.baddecodereturn1")
 
     def test_cpy_bug1175396(self):
-        import codecs, StringIO
+        import codecs, io
         s = [
             '<%!--===================================================\r\n',
             '    BLOG index page: show recent articles,\r\n',
@@ -364,15 +369,15 @@
             '        log.error("Error loading articles: "+str(x))\r\n',
             '        self.abort("cannot load articles")\r\n',
         ]
-        stream = StringIO.StringIO("".join(s).encode("utf7"))
-        assert "aborrt" not in stream.getvalue()
+        stream = io.BytesIO("".join(s).encode("utf7"))
+        assert b"aborrt" not in stream.getvalue()
         reader = codecs.getreader("utf7")(stream)
         for (i, line) in enumerate(reader):
             assert line == s[i]
 
     def test_array(self):
         import _codecs, array
-        _codecs.readbuffer_encode(array.array('c', 'spam')) == ('spam', 4)
+        _codecs.readbuffer_encode(array.array('b', b'spam')) == ('spam', 4)
 
     def test_utf8sig(self):
         import codecs
@@ -382,28 +387,28 @@
 
     def test_escape_decode_escaped_newline(self):
         import _codecs
-        s = '\\\n'
+        s = b'\\\n'
         decoded = _codecs.unicode_escape_decode(s)[0]
         assert decoded == ''
 
     def test_charmap_decode_1(self):
         import codecs
-        assert codecs.charmap_encode(u'xxx') == ('xxx', 3)
-        assert codecs.charmap_encode(u'xxx', 'strict', {ord('x'): 'XX'}) == ('XXXXXX', 3)
+        assert codecs.charmap_encode(u'xxx') == (b'xxx', 3)
+        assert codecs.charmap_encode(u'xxx', 'strict', {ord('x'): b'XX'}) == (b'XXXXXX', 3)
 
-        res = codecs.charmap_decode("\x00\x01\x02", "replace", u"ab")
+        res = codecs.charmap_decode(b"\x00\x01\x02", "replace", u"ab")
         assert res == (u"ab\ufffd", 3)
-        res = codecs.charmap_decode("\x00\x01\x02", "replace", u"ab\ufffe")
+        res = codecs.charmap_decode(b"\x00\x01\x02", "replace", u"ab\ufffe")
         assert res == (u'ab\ufffd', 3)
 
     def test_decode_errors(self):
         import sys
         if sys.maxunicode > 0xffff:
             try:
-                "\x00\x00\x00\x00\x00\x11\x11\x00".decode("unicode_internal")
+                b"\x00\x00\x00\x00\x00\x11\x11\x00".decode("unicode_internal")
             except UnicodeDecodeError, ex:
                 assert "unicode_internal" == ex.encoding
-                assert "\x00\x00\x00\x00\x00\x11\x11\x00" == ex.object
+                assert b"\x00\x00\x00\x00\x00\x11\x11\x00" == ex.object
                 assert ex.start == 4
                 assert ex.end == 8
             else:
@@ -414,14 +419,14 @@
         assert codecs.replace_errors(UnicodeEncodeError(
             "ascii", u"\u3042", 0, 1, "ouch")) == (u"?", 1)
         assert codecs.replace_errors(UnicodeDecodeError(
-            "ascii", "\xff", 0, 1, "ouch")) == (u"\ufffd", 1)
+            "ascii", b"\xff", 0, 1, "ouch")) == (u"\ufffd", 1)
         assert codecs.replace_errors(UnicodeTranslateError(
             u"\u3042", 0, 1, "ouch")) == (u"\ufffd", 1)
 
         assert codecs.replace_errors(UnicodeEncodeError(
             "ascii", u"\u3042\u3042", 0, 2, "ouch")) == (u"??", 2)
         assert codecs.replace_errors(UnicodeDecodeError(
-            "ascii", "\xff\xff", 0, 2, "ouch")) == (u"\ufffd", 2)
+            "ascii", b"\xff\xff", 0, 2, "ouch")) == (u"\ufffd", 2)
         assert codecs.replace_errors(UnicodeTranslateError(
             u"\u3042\u3042", 0, 2, "ouch")) == (u"\ufffd\ufffd", 2)
 
@@ -439,13 +444,13 @@
         # A UnicodeDecodeError object without an end attribute
         class NoEndUnicodeDecodeError(UnicodeDecodeError):
             def __init__(self):
-                UnicodeDecodeError.__init__(self, "ascii", "", 0, 1, "bad")
+                UnicodeDecodeError.__init__(self, "ascii", b"", 0, 1, "bad")
                 del self.end
 
         # A UnicodeDecodeError object with a bad object attribute
         class BadObjectUnicodeDecodeError(UnicodeDecodeError):
             def __init__(self):
-                UnicodeDecodeError.__init__(self, "ascii", "", 0, 1, "bad")
+                UnicodeDecodeError.__init__(self, "ascii", b"", 0, 1, "bad")
                 self.object = []
 
         # A UnicodeTranslateError object without a start attribute
@@ -477,11 +482,11 @@
         # With the correct exception, "replace" returns an "?" or u"\ufffd" replacement
 
     def test_decode_ignore(self):
-        assert '\xff'.decode('utf-7', 'ignore') == ''
-        assert '\x00'.decode('unicode-internal', 'ignore') == ''
+        assert b'\xff'.decode('utf-7', 'ignore') == ''
+        assert b'\x00'.decode('unicode-internal', 'ignore') == ''
 
     def test_backslahreplace(self):
-        assert u'a\xac\u1234\u20ac\u8000'.encode('ascii', 'backslashreplace') == 'a\\xac\u1234\u20ac\u8000'
+        assert u'a\xac\u1234\u20ac\u8000'.encode('ascii', 'backslashreplace') == b'a\\xac\u1234\u20ac\u8000'
 
     def test_surrogateescape(self):
         assert b'a\x80b'.decode('utf-8', 'surrogateescape') == 'a\udc80b'
@@ -502,10 +507,10 @@
                     "test.badhandler"
                 )
             for (enc, bytes) in (
-                ("utf-8", "\xff"),
-                ("ascii", "\xff"),
-                ("utf-7", "+x-"),
-                ("unicode-internal", "\x00"),
+                ("utf-8", b"\xff"),
+                ("ascii", b"\xff"),
+                ("utf-7", b"+x-"),
+                ("unicode-internal", b"\x00"),
             ):
                 raises(
                     TypeError,
@@ -518,19 +523,19 @@
         import codecs
         import sys
         try:
-            '\x00'.decode('unicode-internal')
+            b'\x00'.decode('unicode-internal')
         except UnicodeDecodeError:
             pass
         else:
             raise Exception("DID NOT RAISE")
 
-        res = "\x00\x00\x00\x00\x00".decode("unicode-internal", "replace")
+        res = b"\x00\x00\x00\x00\x00".decode("unicode-internal", "replace")
         if sys.maxunicode > 65535:
             assert res == u"\u0000\ufffd"    # UCS4 build
         else:
             assert res == u"\x00\x00\ufffd"  # UCS2 build
 
-        res = "\x00\x00\x00\x00\x00".decode("unicode-internal", "ignore")
+        res = b"\x00\x00\x00\x00\x00".decode("unicode-internal", "ignore")
         if sys.maxunicode > 65535:
             assert res == u"\u0000"   # UCS4 build
         else:
@@ -541,7 +546,7 @@
                 raise TypeError("don't know how to handle %r" % exc)
             return (u"\x01", 1)
         codecs.register_error("test.hui", handler_unicodeinternal)
-        res = "\x00\x00\x00\x00\x00".decode("unicode-internal", "test.hui")
+        res = b"\x00\x00\x00\x00\x00".decode("unicode-internal", "test.hui")
         if sys.maxunicode > 65535:
             assert res == u"\u0000\u0001\u0000"   # UCS4 build
         else:
@@ -550,31 +555,31 @@
     def test_encode_error_bad_handler(self):
         import codecs
         codecs.register_error("test.bad_handler", lambda e: (repl, 1))
-        assert u"xyz".encode("latin-1", "test.bad_handler") == "xyz"
+        assert u"xyz".encode("latin-1", "test.bad_handler") == b"xyz"
         repl = u"\u1234"
         raises(UnicodeEncodeError, u"\u5678".encode, "latin-1",
                "test.bad_handler")
         repl = u"\u00E9"
         s = u"\u5678".encode("latin-1", "test.bad_handler")
-        assert s == '\xe9'
+        assert s == b'\xe9'
 
     def test_charmap_encode(self):
-        assert 'xxx'.encode('charmap') == 'xxx'
+        assert 'xxx'.encode('charmap') == b'xxx'
 
         import codecs
         raises(TypeError, codecs.charmap_encode, u'\xff', "replace",  {0xff: 300})
         raises(UnicodeError, codecs.charmap_encode, u"\xff", "replace", {0xff: None})
 
     def test_charmap_encode_replace(self):
-        charmap = dict([ (ord(c), 2*c.upper()) for c in "abcdefgh"])
-        charmap[ord("?")] = "XYZ"
+        charmap = dict([(c, bytes([c, c]).upper()) for c in b"abcdefgh"])
+        charmap[ord("?")] = b"XYZ"
         import codecs
         sin = u"abcDEF"
         sout = codecs.charmap_encode(sin, "replace", charmap)[0]
-        assert sout == "AABBCCXYZXYZXYZ"
+        assert sout == b"AABBCCXYZXYZXYZ"
 
     def test_charmap_decode_2(self):
-        assert 'foo'.decode('charmap') == 'foo'
+        assert b'foo'.decode('charmap') == 'foo'
 
     def test_charmap_build(self):
         import codecs
@@ -583,25 +588,25 @@
 
     def test_utf7_start_end_in_exception(self):
         try:
-            '+IC'.decode('utf-7')
+            b'+IC'.decode('utf-7')
         except UnicodeDecodeError, exc:
             assert exc.start == 0
             assert exc.end == 3
 
     def test_utf7_surrogate(self):
-        raises(UnicodeDecodeError, '+3ADYAA-'.decode, 'utf-7')
+        raises(UnicodeDecodeError, b'+3ADYAA-'.decode, 'utf-7')
 
     def test_utf_16_encode_decode(self):
         import codecs
         x = u'123abc'
-        assert codecs.getencoder('utf-16')(x) == ('\xff\xfe1\x002\x003\x00a\x00b\x00c\x00', 6)
-        assert codecs.getdecoder('utf-16')('\xff\xfe1\x002\x003\x00a\x00b\x00c\x00') == (x, 14)
+        assert codecs.getencoder('utf-16')(x) == (b'\xff\xfe1\x002\x003\x00a\x00b\x00c\x00', 6)
+        assert codecs.getdecoder('utf-16')(b'\xff\xfe1\x002\x003\x00a\x00b\x00c\x00') == (x, 14)
 
     def test_unicode_escape(self):        
-        assert u'\\'.encode('unicode-escape') == '\\\\'
-        assert '\\\\'.decode('unicode-escape') == u'\\'
-        assert u'\ud801'.encode('unicode-escape') == '\\ud801'
-        assert u'\u0013'.encode('unicode-escape') == '\\x13'
+        assert u'\\'.encode('unicode-escape') == b'\\\\'
+        assert b'\\\\'.decode('unicode-escape') == u'\\'
+        assert u'\ud801'.encode('unicode-escape') == b'\\ud801'
+        assert u'\u0013'.encode('unicode-escape') == b'\\x13'
 
     def test_mbcs(self):
         import sys
@@ -611,11 +616,3 @@
         assert u'caf\xe9'.encode('mbcs') == 'caf\xe9'
         assert u'\u040a'.encode('mbcs') == '?' # some cyrillic letter
         assert 'cafx\e9'.decode('mbcs') == u'cafx\e9'
-
-    def test_bad_handler_string_result(self):
-        import _codecs
-        def f(exc):
-            return ('foo', exc.end)
-        _codecs.register_error("test.test_codecs_not_a_string", f)
-        raises(TypeError, u'\u1234'.encode, 'ascii',
-               'test.test_codecs_not_a_string')
diff --git a/pypy/rlib/runicode.py b/pypy/rlib/runicode.py
--- a/pypy/rlib/runicode.py
+++ b/pypy/rlib/runicode.py
@@ -272,11 +272,11 @@
             # Encode UCS2 Unicode ordinals
             if ch < 0x10000:
                 # Special case: check for high surrogate
-                if 0xD800 <= ch <= 0xDBFF and pos != size:
+                if 0xD800 <= ch <= 0xDFFF and pos != size:
                     ch2 = ord(s[pos])
                     # Check for low surrogate and combine the two to
                     # form a UCS4 value
-                    if 0xDC00 <= ch2 <= 0xDFFF:
+                    if ch <= 0xDBFF and 0xDC00 <= ch2 <= 0xDFFF:
                         ch3 = ((ch - 0xD800) << 10 | (ch2 - 0xDC00)) + 0x10000
                         pos += 1
                         _encodeUCS4(result, ch3)

From noreply at buildbot.pypy.org  Wed Oct 19 23:11:17 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 19 Oct 2011 23:11:17 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Fix tests in the _io module
Message-ID: <20111019211117.30625820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48242:74214095bb4a
Date: 2011-10-19 22:31 +0200
http://bitbucket.org/pypy/pypy/changeset/74214095bb4a/

Log:	Fix tests in the _io module

diff --git a/pypy/module/_io/interp_bufferedio.py b/pypy/module/_io/interp_bufferedio.py
--- a/pypy/module/_io/interp_bufferedio.py
+++ b/pypy/module/_io/interp_bufferedio.py
@@ -50,7 +50,7 @@
         if not space.isinstance_w(w_data, space.w_str):
             raise OperationError(space.w_TypeError, space.wrap(
                 "read() should return bytes"))
-        data = space.str_w(w_data)
+        data = space.bytes_w(w_data)
         rwbuffer.setslice(0, data)
         return space.wrap(len(data))
 
@@ -468,7 +468,7 @@
                 if current_size == 0:
                     return w_data
                 break
-            data = space.str_w(w_data)
+            data = space.bytes_w(w_data)
             size = len(data)
             if size == 0:
                 break
diff --git a/pypy/module/_io/interp_bytesio.py b/pypy/module/_io/interp_bytesio.py
--- a/pypy/module/_io/interp_bytesio.py
+++ b/pypy/module/_io/interp_bytesio.py
@@ -50,7 +50,7 @@
 
         output = buffer2string(self.buf, self.pos, self.pos + size)
         self.pos += size
-        return space.wrap(output)
+        return space.wrapbytes(output)
 
     def read1_w(self, space, w_size):
         return self.read_w(space, w_size)
@@ -125,7 +125,7 @@
 
     def getvalue_w(self, space):
         self._check_closed(space)
-        return space.wrap(buffer2string(self.buf, 0, self.string_size))
+        return space.wrapbytes(buffer2string(self.buf, 0, self.string_size))
 
     def tell_w(self, space):
         self._check_closed(space)
@@ -176,7 +176,7 @@
 
     def getstate_w(self, space):
         self._check_closed(space)
-        w_content = space.wrap(buffer2string(self.buf, 0, self.string_size))
+        w_content = space.wrapbytes(buffer2string(self.buf, 0, self.string_size))
         return space.newtuple([
             w_content,
             space.wrap(self.pos),
diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py
--- a/pypy/module/_io/interp_fileio.py
+++ b/pypy/module/_io/interp_fileio.py
@@ -391,7 +391,7 @@
                 break
             builder.append(chunk)
             total += len(chunk)
-        return space.wrap(builder.build())
+        return space.wrapbytes(builder.build())
 
     if sys.platform == "win32":
         def _truncate(self, size):
diff --git a/pypy/module/_io/interp_iobase.py b/pypy/module/_io/interp_iobase.py
--- a/pypy/module/_io/interp_iobase.py
+++ b/pypy/module/_io/interp_iobase.py
@@ -275,7 +275,7 @@
         if space.is_w(w_length, space.w_None):
             return w_length
         space.delslice(w_buffer, w_length, space.len(w_buffer))
-        return space.str(w_buffer)
+        return space.call_function(space.w_bytes, w_buffer)
 
     def readall_w(self, space):
         builder = StringBuilder()
@@ -283,7 +283,7 @@
             w_data = space.call_method(self, "read",
                                        space.wrap(DEFAULT_BUFFER_SIZE))
 
-            if not space.isinstance_w(w_data, space.w_str):
+            if not space.isinstance_w(w_data, space.w_bytes):
                 raise OperationError(space.w_TypeError, space.wrap(
                     "read() should return bytes"))
             data = space.bytes_w(w_data)
diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py
--- a/pypy/module/_io/interp_textio.py
+++ b/pypy/module/_io/interp_textio.py
@@ -923,7 +923,7 @@
             i = 0
             while i < len(input):
                 w_decoded = space.call_method(self.w_decoder, "decode",
-                                              space.wrap(input[i]))
+                                              space.wrapbytes(input[i]))
                 chars_decoded += len(space.unicode_w(w_decoded))
 
                 cookie.bytes_to_feed += 1
diff --git a/pypy/module/_io/test/test_bufferedio.py b/pypy/module/_io/test/test_bufferedio.py
--- a/pypy/module/_io/test/test_bufferedio.py
+++ b/pypy/module/_io/test/test_bufferedio.py
@@ -14,24 +14,24 @@
         import _io
         raw = _io.FileIO(self.tmpfile)
         f = _io.BufferedReader(raw)
-        assert f.read() == "a\nb\nc"
+        assert f.read() == b"a\nb\nc"
         raises(ValueError, f.read, -2)
         f.close()
         #
         raw = _io.FileIO(self.tmpfile)
         f = _io.BufferedReader(raw)
         r = f.read(4)
-        assert r == "a\nb\n"
+        assert r == b"a\nb\n"
         f.close()
 
     def test_read_pieces(self):
         import _io
         raw = _io.FileIO(self.tmpfile)
         f = _io.BufferedReader(raw)
-        assert f.read(3) == "a\nb"
-        assert f.read(3) == "\nc"
-        assert f.read(3) == ""
-        assert f.read(3) == ""
+        assert f.read(3) == b"a\nb"
+        assert f.read(3) == b"\nc"
+        assert f.read(3) == b""
+        assert f.read(3) == b""
         f.close()
 
     def test_slow_provider(self):
@@ -40,16 +40,16 @@
             def readable(self):
                 return True
             def readinto(self, buf):
-                buf[:3] = "abc"
+                buf[:3] = b"abc"
                 return 3
         bufio = _io.BufferedReader(MockIO())
         r = bufio.read(5)
-        assert r == "abcab"
+        assert r == b"abcab"
 
     def test_read_past_eof(self):
         import _io
         class MockIO(_io._IOBase):
-            stack = ["abc", "d", "efg"]
+            stack = [b"abc", b"d", b"efg"]
             def readable(self):
                 return True
             def readinto(self, buf):
@@ -60,7 +60,7 @@
                 else:
                     return 0
         bufio = _io.BufferedReader(MockIO())
-        assert bufio.read(9000) == "abcdefg"
+        assert bufio.read(9000) == b"abcdefg"
 
     def test_buffering(self):
         import _io
@@ -102,10 +102,10 @@
         import _io
         raw = _io.FileIO(self.tmpfile)
         f = _io.BufferedReader(raw)
-        assert f.read(2) == 'a\n'
-        assert f.peek().startswith('b\nc')
-        assert f.read(3) == 'b\nc'
-        assert f.peek() == ''
+        assert f.read(2) == b'a\n'
+        assert f.peek().startswith(b'b\nc')
+        assert f.read(3) == b'b\nc'
+        assert f.peek() == b''
 
     def test_read1(self):
         import _io
@@ -119,42 +119,42 @@
         raw = RecordingFileIO(self.tmpfile)
         raw.nbreads = 0
         f = _io.BufferedReader(raw, buffer_size=3)
-        assert f.read(1) == 'a'
-        assert f.read1(1) == '\n'
+        assert f.read(1) == b'a'
+        assert f.read1(1) == b'\n'
         assert raw.nbreads == 1
-        assert f.read1(100) == 'b'
+        assert f.read1(100) == b'b'
         assert raw.nbreads == 1
-        assert f.read1(100) == '\nc'
+        assert f.read1(100) == b'\nc'
         assert raw.nbreads == 2
-        assert f.read1(100) == ''
+        assert f.read1(100) == b''
         assert raw.nbreads == 3
         f.close()
 
     def test_readinto(self):
         import _io
-        a = bytearray('x' * 10)
+        a = bytearray(b'x' * 10)
         raw = _io.FileIO(self.tmpfile)
         f = _io.BufferedReader(raw)
         assert f.readinto(a) == 5
         f.close()
-        assert a == 'a\nb\ncxxxxx'
+        assert a == b'a\nb\ncxxxxx'
 
     def test_seek(self):
         import _io
         raw = _io.FileIO(self.tmpfile)
         f = _io.BufferedReader(raw)
-        assert f.read() == "a\nb\nc"
+        assert f.read() == b"a\nb\nc"
         f.seek(0)
-        assert f.read() == "a\nb\nc"
+        assert f.read() == b"a\nb\nc"
         f.seek(-2, 2)
-        assert f.read() == "\nc"
+        assert f.read() == b"\nc"
         f.close()
 
     def test_readlines(self):
         import _io
         raw = _io.FileIO(self.tmpfile)
         f = _io.BufferedReader(raw)
-        assert f.readlines() == ['a\n', 'b\n', 'c']
+        assert f.readlines() == [b'a\n', b'b\n', b'c']
 
     def test_detach(self):
         import _io
@@ -204,24 +204,24 @@
             cls.w_readfile = tmpfile.read
         else:
             def readfile(space):
-                return space.wrap(tmpfile.read())
+                return space.wrapbytes(tmpfile.read())
             cls.w_readfile = cls.space.wrap(interp2app(readfile))
 
     def test_write(self):
         import _io
         raw = _io.FileIO(self.tmpfile, 'w')
         f = _io.BufferedWriter(raw)
-        f.write("abcd")
+        f.write(b"abcd")
         f.close()
-        assert self.readfile() == "abcd"
+        assert self.readfile() == b"abcd"
 
     def test_largewrite(self):
         import _io
         raw = _io.FileIO(self.tmpfile, 'w')
         f = _io.BufferedWriter(raw)
-        f.write("abcd" * 5000)
+        f.write(b"abcd" * 5000)
         f.close()
-        assert self.readfile() == "abcd" * 5000
+        assert self.readfile() == b"abcd" * 5000
 
     def test_incomplete(self):
         import _io
@@ -249,9 +249,9 @@
         b = _io.BufferedWriter(raw, 13)
 
         for i in range(4):
-            assert b.write('x' * 10) == 10
+            assert b.write(b'x' * 10) == 10
         b.flush()
-        assert self.readfile() == 'x' * 40
+        assert self.readfile() == b'x' * 40
 
     def test_destructor(self):
         import _io
@@ -275,7 +275,7 @@
     def test_truncate(self):
         import _io
         raw = _io.FileIO(self.tmpfile, 'w+')
-        raw.write('x' * 20)
+        raw.write(b'x' * 20)
         b = _io.BufferedReader(raw)
         assert b.seek(8) == 8
         assert b.truncate() == 8
@@ -293,7 +293,7 @@
             closed = False
 
             def pop_written(self):
-                s = ''.join(self._write_stack)
+                s = b''.join(self._write_stack)
                 self._write_stack[:] = []
                 return s
 
@@ -322,11 +322,11 @@
         raw = MockNonBlockWriterIO()
         bufio = _io.BufferedWriter(raw, 8)
 
-        assert bufio.write("abcd") == 4
-        assert bufio.write("efghi") == 5
+        assert bufio.write(b"abcd") == 4
+        assert bufio.write(b"efghi") == 5
         # 1 byte will be written, the rest will be buffered
         raw.block_on(b"k")
-        assert bufio.write("jklmn") == 5
+        assert bufio.write(b"jklmn") == 5
 
         # 8 bytes will be written, 8 will be buffered and the rest will be lost
         raw.block_on(b"0")
@@ -337,12 +337,12 @@
         else:
             self.fail("BlockingIOError should have been raised")
         assert written == 16
-        assert raw.pop_written() == "abcdefghijklmnopqrwxyz"
+        assert raw.pop_written() == b"abcdefghijklmnopqrwxyz"
 
-        assert bufio.write("ABCDEFGHI") == 9
+        assert bufio.write(b"ABCDEFGHI") == 9
         s = raw.pop_written()
         # Previously buffered bytes were flushed
-        assert s.startswith("01234567A")
+        assert s.startswith(b"01234567A")
 
     def test_read_non_blocking(self):
         import _io
@@ -374,28 +374,28 @@
                 try:
                     return self._read_stack.pop(0)
                 except IndexError:
-                    return ""
+                    return b""
         # Inject some None's in there to simulate EWOULDBLOCK
         rawio = MockRawIO((b"abc", b"d", None, b"efg", None, None, None))
         bufio = _io.BufferedReader(rawio)
 
-        assert bufio.read(6) == "abcd"
-        assert bufio.read(1) == "e"
-        assert bufio.read() == "fg"
-        assert bufio.peek(1) == ""
+        assert bufio.read(6) == b"abcd"
+        assert bufio.read(1) == b"e"
+        assert bufio.read() == b"fg"
+        assert bufio.peek(1) == b""
         assert bufio.read() is None
-        assert bufio.read() == ""
+        assert bufio.read() == b""
 
 class AppTestBufferedRWPair:
     def test_pair(self):
         import _io
-        pair = _io.BufferedRWPair(_io.BytesIO("abc"), _io.BytesIO())
+        pair = _io.BufferedRWPair(_io.BytesIO(b"abc"), _io.BytesIO())
         assert not pair.closed
         assert pair.readable()
         assert pair.writable()
         assert not pair.isatty()
-        assert pair.read() == "abc"
-        assert pair.write("abc") == 3
+        assert pair.read() == b"abc"
+        assert pair.write(b"abc") == 3
 
     def test_constructor_with_not_readable(self):
         import _io
@@ -417,14 +417,14 @@
     def setup_class(cls):
         cls.space = gettestobjspace(usemodules=['_io'])
         tmpfile = udir.join('tmpfile')
-        tmpfile.write("a\nb\nc", mode='wb')
+        tmpfile.write(b"a\nb\nc", mode='wb')
         cls.w_tmpfile = cls.space.wrap(str(tmpfile))
 
     def test_simple_read(self):
         import _io
         raw = _io.FileIO(self.tmpfile, 'rb+')
         f = _io.BufferedRandom(raw)
-        assert f.read(3) == 'a\nb'
-        f.write('xxxx')
+        assert f.read(3) == b'a\nb'
+        f.write(b'xxxx')
         f.seek(0)
-        assert f.read() == 'a\nbxxxx'
+        assert f.read() == b'a\nbxxxx'
diff --git a/pypy/module/_io/test/test_bytesio.py b/pypy/module/_io/test/test_bytesio.py
--- a/pypy/module/_io/test/test_bytesio.py
+++ b/pypy/module/_io/test/test_bytesio.py
@@ -11,7 +11,7 @@
     def test_init_kwargs(self):
         import _io
 
-        buf = "1234567890"
+        buf = b"1234567890"
         b = _io.BytesIO(initial_bytes=buf)
         assert b.read() == buf
         raises(TypeError, _io.BytesIO, buf, foo=None)
@@ -27,22 +27,22 @@
     def test_write(self):
         import _io
         f = _io.BytesIO()
-        assert f.write("hello") == 5
+        assert f.write(b"hello") == 5
         import gc; gc.collect()
-        assert f.getvalue() == "hello"
+        assert f.getvalue() == b"hello"
         f.close()
 
     def test_read(self):
         import _io
-        f = _io.BytesIO("hello")
-        assert f.read() == "hello"
+        f = _io.BytesIO(b"hello")
+        assert f.read() == b"hello"
         import gc; gc.collect()
-        assert f.read(8192) == ""
+        assert f.read(8192) == b""
         f.close()
 
     def test_seek(self):
         import _io
-        f = _io.BytesIO("hello")
+        f = _io.BytesIO(b"hello")
         assert f.tell() == 0
         assert f.seek(-1, 2) == 4
         assert f.tell() == 4
@@ -50,24 +50,24 @@
 
     def test_truncate(self):
         import _io
-        f = _io.BytesIO("hello")
+        f = _io.BytesIO(b"hello")
         f.seek(3)
         assert f.truncate() == 3
-        assert f.getvalue() == "hel"
+        assert f.getvalue() == b"hel"
 
     def test_setstate(self):
         # state is (content, position, __dict__)
         import _io
-        f = _io.BytesIO("hello")
+        f = _io.BytesIO(b"hello")
         content, pos, __dict__ = f.__getstate__()
-        assert (content, pos) == ("hello", 0)
+        assert (content, pos) == (b"hello", 0)
         assert __dict__ is None or __dict__ == {}
-        f.__setstate__(("world", 3, {"a": 1}))
-        assert f.getvalue() == "world"
-        assert f.read() == "ld"
+        f.__setstate__((b"world", 3, {"a": 1}))
+        assert f.getvalue() == b"world"
+        assert f.read() == b"ld"
         assert f.a == 1
-        assert f.__getstate__() == ("world", 5, {"a": 1})
-        raises(TypeError, f.__setstate__, ("", 0))
+        assert f.__getstate__() == (b"world", 5, {"a": 1})
+        raises(TypeError, f.__setstate__, (b"", 0))
         f.close()
         raises(ValueError, f.__getstate__)
         raises(ValueError, f.__setstate__, ("world", 3, {"a": 1}))
@@ -75,6 +75,6 @@
     def test_readinto(self):
         import _io
 
-        b = _io.BytesIO("hello")
+        b = _io.BytesIO(b"hello")
         b.close()
-        raises(ValueError, b.readinto, bytearray("hello"))
+        raises(ValueError, b.readinto, bytearray(b"hello"))
diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py
--- a/pypy/module/_io/test/test_fileio.py
+++ b/pypy/module/_io/test/test_fileio.py
@@ -46,34 +46,34 @@
     def test_readline(self):
         import _io
         f = _io.FileIO(self.tmpfile, 'rb')
-        assert f.readline() == 'a\n'
-        assert f.readline() == 'b\n'
-        assert f.readline() == 'c'
-        assert f.readline() == ''
+        assert f.readline() == b'a\n'
+        assert f.readline() == b'b\n'
+        assert f.readline() == b'c'
+        assert f.readline() == b''
         f.close()
 
     def test_readlines(self):
         import _io
         f = _io.FileIO(self.tmpfile, 'rb')
-        assert f.readlines() == ["a\n", "b\n", "c"]
+        assert f.readlines() == [b"a\n", b"b\n", b"c"]
         f.seek(0)
-        assert f.readlines(3) == ["a\n", "b\n"]
+        assert f.readlines(3) == [b"a\n", b"b\n"]
         f.close()
 
     def test_readall(self):
         import _io
         f = _io.FileIO(self.tmpfile, 'rb')
-        assert f.readall() == "a\nb\nc"
+        assert f.readall() == b"a\nb\nc"
         f.close()
 
     def test_write(self):
         import _io
         filename = self.tmpfile + '_w'
         f = _io.FileIO(filename, 'wb')
-        f.write("test")
+        f.write(b"test")
         # try without flushing
         f2 = _io.FileIO(filename, 'rb')
-        assert f2.read() == "test"
+        assert f2.read() == b"test"
         f.close()
         f2.close()
 
@@ -81,9 +81,9 @@
         import _io
         filename = self.tmpfile + '_w'
         f = _io.FileIO(filename, 'wb')
-        f.writelines(["line1\n", "line2", "line3"])
+        f.writelines([b"line1\n", b"line2", b"line3"])
         f2 = _io.FileIO(filename, 'rb')
-        assert f2.read() == "line1\nline2line3"
+        assert f2.read() == b"line1\nline2line3"
         f.close()
         f2.close()
 
@@ -120,18 +120,18 @@
 
     def test_readinto(self):
         import _io
-        a = bytearray('x' * 10)
+        a = bytearray(b'x' * 10)
         f = _io.FileIO(self.tmpfile, 'r+')
         assert f.readinto(a) == 10
         f.close()
-        assert a == 'a\nb\nc\0\0\0\0\0'
+        assert a == b'a\nb\nc\0\0\0\0\0'
         #
-        a = bytearray('x' * 10)
+        a = bytearray(b'x' * 10)
         f = _io.FileIO(self.tmpfile, 'r+')
         f.truncate(3)
         assert f.readinto(a) == 3
         f.close()
-        assert a == 'a\nbxxxxxxx'
+        assert a == b'a\nbxxxxxxx'
 
     def test_repr(self):
         import _io
diff --git a/pypy/module/_io/test/test_io.py b/pypy/module/_io/test/test_io.py
--- a/pypy/module/_io/test/test_io.py
+++ b/pypy/module/_io/test/test_io.py
@@ -69,9 +69,9 @@
             return _io.BytesIO.write(f, data)
         f.write = write
         bufio = _io.BufferedWriter(f)
-        bufio.write("abc")
+        bufio.write(b"abc")
         bufio.flush()
-        assert f.getvalue() == "ABC"
+        assert f.getvalue() == b"ABC"
 
     def test_destructor(self):
         import io
@@ -109,17 +109,17 @@
     def test_rawio_read(self):
         import _io
         class MockRawIO(_io._RawIOBase):
-            stack = ['abc', 'de', '']
+            stack = [b'abc', b'de', b'']
             def readinto(self, buf):
                 data = self.stack.pop(0)
                 buf[:len(data)] = data
                 return len(data)
-        assert MockRawIO().read() == 'abcde'
+        assert MockRawIO().read() == b'abcde'
 
     def test_rawio_read_pieces(self):
         import _io
         class MockRawIO(_io._RawIOBase):
-            stack = ['abc', 'de', None, 'fg', '']
+            stack = [b'abc', b'de', None, b'fg', b'']
             def readinto(self, buf):
                 data = self.stack.pop(0)
                 if data is None:
@@ -132,12 +132,12 @@
                     self.stack.insert(0, data[len(buf):])
                     return len(buf)
         r = MockRawIO()
-        assert r.read(2) == 'ab'
-        assert r.read(2) == 'c'
-        assert r.read(2) == 'de'
+        assert r.read(2) == b'ab'
+        assert r.read(2) == b'c'
+        assert r.read(2) == b'de'
         assert r.read(2) == None
-        assert r.read(2) == 'fg'
-        assert r.read(2) == ''
+        assert r.read(2) == b'fg'
+        assert r.read(2) == b''
 
 class AppTestOpen:
     def setup_class(cls):
@@ -169,7 +169,7 @@
 
     def test_array_write(self):
         import _io, array
-        a = array.array(b'i', range(10))
+        a = array.array('i', range(10))
         n = len(a.tostring())
         with _io.open(self.tmpfile, "wb", 0) as f:
             res = f.write(a)
@@ -208,13 +208,13 @@
         import _io
 
         with _io.open(self.tmpfile, "wb") as f:
-            f.write("abcd")
+            f.write(b"abcd")
 
         with _io.open(self.tmpfile) as f:
             decoded = f.read()
 
         # seek positions
-        for i in xrange(len(decoded) + 1):
+        for i in range(len(decoded) + 1):
             # read lenghts
             for j in [1, 5, len(decoded) - i]:
                 with _io.open(self.tmpfile) as f:
@@ -306,7 +306,7 @@
             assert f.newlines is None
 
         with _io.open(self.tmpfile, "wb") as f:
-            f.write("hello\nworld\n")
+            f.write(b"hello\nworld\n")
 
         with _io.open(self.tmpfile, "r") as f:
             res = f.readline()
@@ -314,4 +314,4 @@
             res = f.readline()
             assert res == "world\n"
             assert f.newlines == "\n"
-            assert type(f.newlines) is unicode
+            assert type(f.newlines) is str
diff --git a/pypy/module/_io/test/test_stringio.py b/pypy/module/_io/test/test_stringio.py
--- a/pypy/module/_io/test/test_stringio.py
+++ b/pypy/module/_io/test/test_stringio.py
@@ -248,7 +248,7 @@
 
         assert iter(sio) is sio
         assert hasattr(sio, "__iter__")
-        assert hasattr(sio, "next")
+        assert hasattr(sio, "__next__")
 
         i = 0
         for line in sio:
@@ -271,7 +271,7 @@
         sio = io.StringIO()
         state = sio.__getstate__()
         assert len(state) == 4
-        assert isinstance(state[0], unicode)
+        assert isinstance(state[0], str)
         assert isinstance(state[1], str)
         assert isinstance(state[2], int)
         assert isinstance(state[3], dict)
@@ -286,7 +286,7 @@
         sio.__setstate__((u"no error", u"", 0, {"spam": 3}))
         raises(ValueError, sio.__setstate__, (u"", u"f", 0, None))
         raises(ValueError, sio.__setstate__, (u"", u"", -1, None))
-        raises(TypeError, sio.__setstate__, ("", u"", 0, None))
+        raises(TypeError, sio.__setstate__, (b"", u"", 0, None))
         raises(TypeError, sio.__setstate__, (u"", u"", 0.0, None))
         raises(TypeError, sio.__setstate__, (u"", u"", 0, 0))
         raises(TypeError, sio.__setstate__, (u"len-test", 0))
diff --git a/pypy/module/_io/test/test_textio.py b/pypy/module/_io/test/test_textio.py
--- a/pypy/module/_io/test/test_textio.py
+++ b/pypy/module/_io/test/test_textio.py
@@ -71,7 +71,7 @@
 
     def test_read_some_then_all(self):
         import _io
-        r = _io.BytesIO("abc\ndef\n")
+        r = _io.BytesIO(b"abc\ndef\n")
         t = _io.TextIOWrapper(r)
         reads = t.read(4)
         reads += t.read()
@@ -79,7 +79,7 @@
 
     def test_read_some_then_readline(self):
         import _io
-        r = _io.BytesIO("abc\ndef\n")
+        r = _io.BytesIO(b"abc\ndef\n")
         t = _io.TextIOWrapper(r)
         reads = t.read(4)
         reads += t.readline()
@@ -108,7 +108,7 @@
 
     def test_tell(self):
         import _io
-        r = _io.BytesIO("abc\ndef\n")
+        r = _io.BytesIO(b"abc\ndef\n")
         t = _io.TextIOWrapper(r)
         assert t.tell() == 0
         t.read(4)
@@ -126,7 +126,7 @@
         t.write(u"abc")
         del t
         import gc; gc.collect()
-        assert l == ["abc"]
+        assert l == [b"abc"]
 
     def test_newlines(self):
         import _io
@@ -169,8 +169,8 @@
     def test_readline(self):
         import _io
 
-        s = "AAA\r\nBBB\rCCC\r\nDDD\nEEE\r\n"
-        r = "AAA\nBBB\nCCC\nDDD\nEEE\n".decode("ascii")
+        s = b"AAA\r\nBBB\rCCC\r\nDDD\nEEE\r\n"
+        r = "AAA\nBBB\nCCC\nDDD\nEEE\n"
         txt = _io.TextIOWrapper(_io.BytesIO(s), encoding="ascii")
         txt._CHUNK_SIZE = 4
 
@@ -184,20 +184,20 @@
     def test_name(self):
         import _io
 
-        t = _io.TextIOWrapper(_io.BytesIO(""))
+        t = _io.TextIOWrapper(_io.BytesIO(b""))
         # CPython raises an AttributeError, we raise a TypeError.
         raises((AttributeError, TypeError), setattr, t, "name", "anything")
 
     def test_repr(self):
         import _io
 
-        t = _io.TextIOWrapper(_io.BytesIO(""), encoding="utf-8")
+        t = _io.TextIOWrapper(_io.BytesIO(b""), encoding="utf-8")
         assert repr(t) == "<_io.TextIOWrapper encoding='utf-8'>"
-        t = _io.TextIOWrapper(_io.BytesIO(""), encoding="ascii")
+        t = _io.TextIOWrapper(_io.BytesIO(b""), encoding="ascii")
         assert repr(t) == "<_io.TextIOWrapper encoding='ascii'>"
-        t = _io.TextIOWrapper(_io.BytesIO(""), encoding=u"utf-8")
+        t = _io.TextIOWrapper(_io.BytesIO(b""), encoding=u"utf-8")
         assert repr(t) == "<_io.TextIOWrapper encoding='utf-8'>"
-        b = _io.BytesIO("")
+        b = _io.BytesIO(b"")
         t = _io.TextIOWrapper(b, encoding="utf-8")
         b.name = "dummy"
         assert repr(t) == "<_io.TextIOWrapper name='dummy' encoding='utf-8'>"
@@ -256,7 +256,7 @@
                 def _decode_bytewise(s):
                     # Decode one byte at a time
                     for b in encoder.encode(s):
-                        result.append(decoder.decode(b))
+                        result.append(decoder.decode(bytes([b])))
             else:
                 encoder = None
                 def _decode_bytewise(s):

From noreply at buildbot.pypy.org  Wed Oct 19 23:11:18 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Wed, 19 Oct 2011 23:11:18 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Merge default
Message-ID: <20111019211118.78353820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48243:e6393bbc25cf
Date: 2011-10-19 22:50 +0200
http://bitbucket.org/pypy/pypy/changeset/e6393bbc25cf/

Log:	Merge default

diff --git a/pypy/jit/metainterp/optimizeopt/rewrite.py b/pypy/jit/metainterp/optimizeopt/rewrite.py
--- a/pypy/jit/metainterp/optimizeopt/rewrite.py
+++ b/pypy/jit/metainterp/optimizeopt/rewrite.py
@@ -106,10 +106,9 @@
             self.make_equal_to(op.result, v1)
         else:
             self.emit_operation(op)
-
-        # Synthesize the reverse ops for optimize_default to reuse
-        self.pure(rop.INT_ADD, [op.result, op.getarg(1)], op.getarg(0))
-        self.pure(rop.INT_SUB, [op.getarg(0), op.result], op.getarg(1))
+            # Synthesize the reverse ops for optimize_default to reuse
+            self.pure(rop.INT_ADD, [op.result, op.getarg(1)], op.getarg(0))
+            self.pure(rop.INT_SUB, [op.getarg(0), op.result], op.getarg(1))
 
     def optimize_INT_ADD(self, op):
         v1 = self.getvalue(op.getarg(0))
@@ -122,10 +121,9 @@
             self.make_equal_to(op.result, v1)
         else:
             self.emit_operation(op)
-
-        # Synthesize the reverse op for optimize_default to reuse
-        self.pure(rop.INT_SUB, [op.result, op.getarg(1)], op.getarg(0))
-        self.pure(rop.INT_SUB, [op.result, op.getarg(0)], op.getarg(1))
+            # Synthesize the reverse op for optimize_default to reuse
+            self.pure(rop.INT_SUB, [op.result, op.getarg(1)], op.getarg(0))
+            self.pure(rop.INT_SUB, [op.result, op.getarg(0)], op.getarg(1))
 
     def optimize_INT_MUL(self, op):
         v1 = self.getvalue(op.getarg(0))
@@ -141,13 +139,13 @@
             self.make_constant_int(op.result, 0)
         else:
             for lhs, rhs in [(v1, v2), (v2, v1)]:
-                # x & (x -1) == 0 is a quick test for power of 2
-                if (lhs.is_constant() and
-                    (lhs.box.getint() & (lhs.box.getint() - 1)) == 0):
-                    new_rhs = ConstInt(highest_bit(lhs.box.getint()))
-                    op = op.copy_and_change(rop.INT_LSHIFT, args=[rhs.box, new_rhs])
-                    break
-
+                if lhs.is_constant():
+                    x = lhs.box.getint()
+                    # x & (x - 1) == 0 is a quick test for power of 2
+                    if x & (x - 1) == 0:
+                        new_rhs = ConstInt(highest_bit(lhs.box.getint()))
+                        op = op.copy_and_change(rop.INT_LSHIFT, args=[rhs.box, new_rhs])
+                        break
             self.emit_operation(op)
 
     def optimize_UINT_FLOORDIV(self, op):
@@ -462,6 +460,14 @@
         self.optimizer.opaque_pointers[value] = True
         self.make_equal_to(op.result, value)
 
+    def optimize_CAST_PTR_TO_INT(self, op):
+        self.pure(rop.CAST_INT_TO_PTR, [op.result], op.getarg(0))
+        self.emit_operation(op)
+
+    def optimize_CAST_INT_TO_PTR(self, op):
+        self.pure(rop.CAST_PTR_TO_INT, [op.result], op.getarg(0))
+        self.emit_operation(op)
+
 dispatch_opt = make_dispatcher_method(OptRewrite, 'optimize_',
         default=OptRewrite.emit_operation)
 optimize_guards = _findall(OptRewrite, 'optimize_', 'GUARD')
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -234,6 +234,30 @@
             """ % expected_value
             self.optimize_loop(ops, expected)
 
+    def test_reverse_of_cast(self):
+        ops = """
+        [i0]
+        p0 = cast_int_to_ptr(i0)
+        i1 = cast_ptr_to_int(p0)
+        jump(i1)
+        """
+        expected = """
+        [i0]
+        jump(i0)
+        """
+        self.optimize_loop(ops, expected)
+        ops = """
+        [p0]
+        i1 = cast_ptr_to_int(p0)
+        p1 = cast_int_to_ptr(i1)
+        jump(p1)
+        """
+        expected = """
+        [p0]
+        jump(p0)
+        """
+        self.optimize_loop(ops, expected)
+
     # ----------
 
     def test_remove_guard_class_1(self):
diff --git a/pypy/jit/metainterp/quasiimmut.py b/pypy/jit/metainterp/quasiimmut.py
--- a/pypy/jit/metainterp/quasiimmut.py
+++ b/pypy/jit/metainterp/quasiimmut.py
@@ -74,12 +74,10 @@
         self.looptokens_wrefs.append(wref_looptoken)
 
     def compress_looptokens_list(self):
-        newlist = []
-        for wref in self.looptokens_wrefs:
-            looptoken = wref()
-            if looptoken is not None and not looptoken.invalidated:
-                newlist.append(wref)
-        self.looptokens_wrefs = newlist
+        self.looptokens_wrefs = [wref for wref in self.looptokens_wrefs
+                                      if wref() is not None]
+        # NB. we must keep around the looptoken_wrefs that are
+        # already invalidated; see below
         self.compress_limit = (len(self.looptokens_wrefs) + 15) * 2
 
     def invalidate(self):
@@ -90,9 +88,14 @@
         self.looptokens_wrefs = []
         for wref in wrefs:
             looptoken = wref()
-            if looptoken is not None and not looptoken.invalidated:
+            if looptoken is not None:
                 looptoken.invalidated = True
                 self.cpu.invalidate_loop(looptoken)
+                # NB. we must call cpu.invalidate_loop() even if
+                # looptoken.invalidated was already set to True.
+                # It's possible to invalidate several times the
+                # same looptoken; see comments in jit.backend.model
+                # in invalidate_loop().
                 if not we_are_translated():
                     self.cpu.stats.invalidated_token_numbers.add(
                         looptoken.number)
diff --git a/pypy/module/pyexpat/__init__.py b/pypy/module/pyexpat/__init__.py
--- a/pypy/module/pyexpat/__init__.py
+++ b/pypy/module/pyexpat/__init__.py
@@ -4,12 +4,8 @@
 
 class ErrorsModule(MixedModule):
     "Definition of pyexpat.errors module."
-
-    appleveldefs = {
-        }
-
-    interpleveldefs = {
-        }
+    appleveldefs = {}
+    interpleveldefs = {}
 
     def setup_after_space_initialization(self):
         from pypy.module.pyexpat import interp_pyexpat
@@ -18,6 +14,18 @@
                     interp_pyexpat.ErrorString(self.space,
                     getattr(interp_pyexpat, name)))
 
+class ModelModule(MixedModule):
+    "Definition of pyexpat.model module."
+    appleveldefs = {}
+    interpleveldefs = {}
+
+    def setup_after_space_initialization(self):
+        from pypy.module.pyexpat import interp_pyexpat
+        space = self.space
+        for name in interp_pyexpat.xml_model_list:
+            value = getattr(interp_pyexpat, name)
+            space.setattr(self, space.wrap(name), space.wrap(value))
+
 class Module(MixedModule):
     "Python wrapper for Expat parser."
 
@@ -39,6 +47,7 @@
 
     submodules = {
         'errors': ErrorsModule,
+        'model':  ModelModule,
     }
 
     for name in ['XML_PARAM_ENTITY_PARSING_NEVER',
diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py
--- a/pypy/module/pyexpat/interp_pyexpat.py
+++ b/pypy/module/pyexpat/interp_pyexpat.py
@@ -76,6 +76,18 @@
     "XML_ERROR_FINISHED",
     "XML_ERROR_SUSPEND_PE",
     ]
+xml_model_list = [
+    "XML_CTYPE_EMPTY",
+    "XML_CTYPE_ANY",
+    "XML_CTYPE_MIXED",
+    "XML_CTYPE_NAME",
+    "XML_CTYPE_CHOICE",
+    "XML_CTYPE_SEQ",
+    "XML_CQUANT_NONE",
+    "XML_CQUANT_OPT",
+    "XML_CQUANT_REP",
+    "XML_CQUANT_PLUS",
+    ]
 
 class CConfigure:
     _compilation_info_ = eci
@@ -104,6 +116,8 @@
 
     for name in xml_error_list:
         locals()[name] = rffi_platform.ConstantInteger(name)
+    for name in xml_model_list:
+        locals()[name] = rffi_platform.ConstantInteger(name)
 
 for k, v in rffi_platform.configure(CConfigure).items():
     globals()[k] = v
diff --git a/pypy/module/pyexpat/test/test_parser.py b/pypy/module/pyexpat/test/test_parser.py
--- a/pypy/module/pyexpat/test/test_parser.py
+++ b/pypy/module/pyexpat/test/test_parser.py
@@ -131,3 +131,7 @@
                'encoding specified in XML declaration is incorrect')
         assert (pyexpat.errors.XML_ERROR_XML_DECL ==
                 'XML declaration not well-formed')
+
+    def test_model(self):
+        import pyexpat
+        assert isinstance(pyexpat.model.XML_CTYPE_EMPTY, int)
diff --git a/pypy/objspace/std/strutil.py b/pypy/objspace/std/strutil.py
--- a/pypy/objspace/std/strutil.py
+++ b/pypy/objspace/std/strutil.py
@@ -35,7 +35,7 @@
 
     def error(self):
         raise ParseStringError("invalid literal for %s() with base %d: '%s'" %
-                               (self.fname, self.base, self.literal))
+                               (self.fname, self.original_base, self.literal))
 
     def __init__(self, s, literal, base, fname):
         self.literal = literal
@@ -47,7 +47,8 @@
         elif s.startswith('+'):
             s = strip_spaces(s[1:])
         self.sign = sign
-        
+        self.original_base = base
+
         if base == 0:
             if s.startswith('0x') or s.startswith('0X'):
                 base = 16
diff --git a/pypy/objspace/std/test/test_strutil.py b/pypy/objspace/std/test/test_strutil.py
--- a/pypy/objspace/std/test/test_strutil.py
+++ b/pypy/objspace/std/test/test_strutil.py
@@ -89,6 +89,8 @@
 
         exc = raises(ParseStringError, string_to_int, '')
         assert exc.value.msg == "invalid literal for int() with base 10: ''"
+        exc = raises(ParseStringError, string_to_int, '', 0)
+        assert exc.value.msg == "invalid literal for int() with base 0: ''"
 
     def test_string_to_int_overflow(self):
         import sys
diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -8,6 +8,8 @@
 from pypy.rpython.extregistry import ExtRegistryEntry
 from pypy.tool.sourcetools import func_with_new_name
 
+DEBUG_ELIDABLE_FUNCTIONS = False
+
 
 def elidable(func):
     """ Decorate a function as "trace-elidable". This means precisely that:
@@ -24,6 +26,18 @@
     If a particular call to this function ends up raising an exception, then it
     is handled like a normal function call (this decorator is ignored).
     """
+    if DEBUG_ELIDABLE_FUNCTIONS:
+        cache = {}
+        oldfunc = func
+        def func(*args):
+            result = oldfunc(*args)    # if it raises, no caching
+            try:
+                oldresult = cache.setdefault(args, result)
+            except TypeError:
+                pass           # unhashable args
+            else:
+                assert oldresult == result
+            return result
     func._elidable_function_ = True
     return func
 
diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -1292,10 +1292,12 @@
         # if a prebuilt GcStruct contains a pointer to a young object,
         # then the write_barrier must have ensured that the prebuilt
         # GcStruct is in the list self.old_objects_pointing_to_young.
+        debug_start("gc-minor-walkroots")
         self.root_walker.walk_roots(
             MiniMarkGC._trace_drag_out1,  # stack roots
             MiniMarkGC._trace_drag_out1,  # static in prebuilt non-gc
             None)                         # static in prebuilt gc
+        debug_stop("gc-minor-walkroots")
 
     def collect_cardrefs_to_nursery(self):
         size_gc_header = self.gcheaderbuilder.size_gc_header
diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -56,8 +56,8 @@
     binaries = [(pypy_c, rename_pypy_c)]
     #
     if sys.platform == 'win32':
-        # Can't rename a DLL: it is always called 'libpypy_c.dll'
-        for extra in ['libpypy_c.dll',
+        # Can't rename a DLL: it is always called 'libpypy-c.dll'
+        for extra in ['libpypy-c.dll',
                       'libexpat.dll', 'sqlite3.dll', 'msvcr90.dll']:
             p = pypy_c.dirpath().join(extra)
             if not p.check():
diff --git a/pypy/translator/platform/posix.py b/pypy/translator/platform/posix.py
--- a/pypy/translator/platform/posix.py
+++ b/pypy/translator/platform/posix.py
@@ -157,7 +157,7 @@
 
         rules = [
             ('all', '$(DEFAULT_TARGET)', []),
-            ('$(TARGET)', '$(OBJECTS)', '$(CC_LINK) $(LDFLAGS) $(LDFLAGSEXTRA) -o $@ $(OBJECTS) $(LIBDIRS) $(LIBS) $(LINKFILES)'),
+            ('$(TARGET)', '$(OBJECTS)', '$(CC_LINK) $(LDFLAGSEXTRA) -o $@ $(OBJECTS) $(LIBDIRS) $(LIBS) $(LINKFILES) $(LDFLAGS)'),
             ('%.o', '%.c', '$(CC) $(CFLAGS) $(CFLAGSEXTRA) -o $@ -c $< $(INCLUDEDIRS)'),
             ]
 

From noreply at buildbot.pypy.org  Wed Oct 19 23:42:46 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 19 Oct 2011 23:42:46 +0200 (CEST)
Subject: [pypy-commit] pypy default: Test and fix for issue917.
Message-ID: <20111019214246.3952A820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48244:2a1eb9241e77
Date: 2011-10-19 23:26 +0200
http://bitbucket.org/pypy/pypy/changeset/2a1eb9241e77/

Log:	Test and fix for issue917.

diff --git a/pypy/module/__builtin__/app_io.py b/pypy/module/__builtin__/app_io.py
--- a/pypy/module/__builtin__/app_io.py
+++ b/pypy/module/__builtin__/app_io.py
@@ -27,7 +27,7 @@
     co = compile(source.rstrip()+"\n", filename, 'exec')
     exec co in glob, loc
 
-def raw_input(prompt=None):
+def raw_input(prompt=''):
     """raw_input([prompt]) -> string
 
 Read a string from standard input.  The trailing newline is stripped.
@@ -51,14 +51,17 @@
             prompt = ''
         return sys.__raw_input__(prompt)
 
-    if prompt is not None:
-        stdout.write(prompt)
-        try:
-            flush = stdout.flush
-        except AttributeError:
-            pass
-        else:
-            flush()
+    print >> stdout, prompt,
+    try:
+        flush = stdout.flush
+    except AttributeError:
+        pass
+    else:
+        flush()
+    try:
+        stdout.softspace = 0
+    except (AttributeError, TypeError):
+        pass
     line = stdin.readline()
     if not line:    # inputting an empty line gives line == '\n'
         raise EOFError
diff --git a/pypy/module/__builtin__/test/test_builtin.py b/pypy/module/__builtin__/test/test_builtin.py
--- a/pypy/module/__builtin__/test/test_builtin.py
+++ b/pypy/module/__builtin__/test/test_builtin.py
@@ -631,6 +631,30 @@
         raises(TypeError, pr, end=3)
         raises(TypeError, pr, sep=42)
 
+    def test_raw_input(self):
+        import sys, StringIO
+        for prompt, expected in [("def:", "abc/ def:/ghi\n"),
+                                 ("", "abc/ /ghi\n"),
+                                 (42, "abc/ 42/ghi\n"),
+                                 (None, "abc/ None/ghi\n"),
+                                 (Ellipsis, "abc/ /ghi\n")]:
+            save = sys.stdin, sys.stdout
+            try:
+                sys.stdin = StringIO.StringIO("foo\nbar\n")
+                out = sys.stdout = StringIO.StringIO()
+                print "abc",    # softspace = 1
+                out.write('/')
+                if prompt is Ellipsis:
+                    got = raw_input()
+                else:
+                    got = raw_input(prompt)
+                out.write('/')
+                print "ghi"
+            finally:
+                sys.stdin, sys.stdout = save
+            assert out.getvalue() == expected
+            assert got == "foo"
+
     def test_round(self):
         assert round(11.234) == 11.0
         assert round(11.234, -1) == 10.0

From noreply at buildbot.pypy.org  Wed Oct 19 23:49:13 2011
From: noreply at buildbot.pypy.org (snus_mumrik)
Date: Wed, 19 Oct 2011 23:49:13 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-comparison: merge default,
	making all equal to default
Message-ID: <20111019214913.34346820D7@wyvern.cs.uni-duesseldorf.de>

Author: Ilya Osadchiy <osadchiy.ilya at gmail.com>
Branch: numpy-comparison
Changeset: r48245:8e5dc0cfdf63
Date: 2011-10-01 15:31 +0300
http://bitbucket.org/pypy/pypy/changeset/8e5dc0cfdf63/

Log:	merge default, making all equal to default

diff too long, truncating to 10000 out of 22893 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -1,2 +1,3 @@
 b590cf6de4190623aad9aa698694c22e614d67b9 release-1.5
 b48df0bf4e75b81d98f19ce89d4a7dc3e1dab5e5 benchmarked
+d8ac7d23d3ec5f9a0fa1264972f74a010dbfd07f release-1.6
diff --git a/dotviewer/graphparse.py b/dotviewer/graphparse.py
--- a/dotviewer/graphparse.py
+++ b/dotviewer/graphparse.py
@@ -36,48 +36,45 @@
     print >> sys.stderr, "Warning: could not guess file type, using 'dot'"
     return 'unknown'
 
-def dot2plain(content, contenttype, use_codespeak=False):
-    if contenttype == 'plain':
-        # already a .plain file
-        return content
+def dot2plain_graphviz(content, contenttype, use_codespeak=False):
+    if contenttype != 'neato':
+        cmdline = 'dot -Tplain'
+    else:
+        cmdline = 'neato -Tplain'
+    #print >> sys.stderr, '* running:', cmdline
+    close_fds = sys.platform != 'win32'
+    p = subprocess.Popen(cmdline, shell=True, close_fds=close_fds,
+                         stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+    (child_in, child_out) = (p.stdin, p.stdout)
+    try:
+        import thread
+    except ImportError:
+        bkgndwrite(child_in, content)
+    else:
+        thread.start_new_thread(bkgndwrite, (child_in, content))
+    plaincontent = child_out.read()
+    child_out.close()
+    if not plaincontent:    # 'dot' is likely not installed
+        raise PlainParseError("no result from running 'dot'")
+    return plaincontent
 
-    if not use_codespeak:
-        if contenttype != 'neato':
-            cmdline = 'dot -Tplain'
-        else:
-            cmdline = 'neato -Tplain'
-        #print >> sys.stderr, '* running:', cmdline
-        close_fds = sys.platform != 'win32'
-        p = subprocess.Popen(cmdline, shell=True, close_fds=close_fds,
-                             stdin=subprocess.PIPE, stdout=subprocess.PIPE)
-        (child_in, child_out) = (p.stdin, p.stdout)
-        try:
-            import thread
-        except ImportError:
-            bkgndwrite(child_in, content)
-        else:
-            thread.start_new_thread(bkgndwrite, (child_in, content))
-        plaincontent = child_out.read()
-        child_out.close()
-        if not plaincontent:    # 'dot' is likely not installed
-            raise PlainParseError("no result from running 'dot'")
-    else:
-        import urllib
-        request = urllib.urlencode({'dot': content})
-        url = 'http://codespeak.net/pypy/convertdot.cgi'
-        print >> sys.stderr, '* posting:', url
-        g = urllib.urlopen(url, data=request)
-        result = []
-        while True:
-            data = g.read(16384)
-            if not data:
-                break
-            result.append(data)
-        g.close()
-        plaincontent = ''.join(result)
-        # very simple-minded way to give a somewhat better error message
-        if plaincontent.startswith('<body'):
-            raise Exception("the dot on codespeak has very likely crashed")
+def dot2plain_codespeak(content, contenttype):
+    import urllib
+    request = urllib.urlencode({'dot': content})
+    url = 'http://codespeak.net/pypy/convertdot.cgi'
+    print >> sys.stderr, '* posting:', url
+    g = urllib.urlopen(url, data=request)
+    result = []
+    while True:
+        data = g.read(16384)
+        if not data:
+            break
+        result.append(data)
+    g.close()
+    plaincontent = ''.join(result)
+    # very simple-minded way to give a somewhat better error message
+    if plaincontent.startswith('<body'):
+        raise Exception("the dot on codespeak has very likely crashed")
     return plaincontent
 
 def bkgndwrite(f, data):
@@ -148,10 +145,13 @@
 
 def parse_dot(graph_id, content, links={}, fixedfont=False):
     contenttype = guess_type(content)
-    try:
-        plaincontent = dot2plain(content, contenttype, use_codespeak=False)
-        return list(parse_plain(graph_id, plaincontent, links, fixedfont))
-    except PlainParseError:
-        # failed, retry via codespeak
-        plaincontent = dot2plain(content, contenttype, use_codespeak=True)
-        return list(parse_plain(graph_id, plaincontent, links, fixedfont))
+    if contenttype == 'plain':
+        plaincontent = content
+    else:
+        try:
+            plaincontent = dot2plain_graphviz(content, contenttype)
+        except PlainParseError, e:
+            print e
+            # failed, retry via codespeak
+            plaincontent = dot2plain_codespeak(content, contenttype)
+    return list(parse_plain(graph_id, plaincontent, links, fixedfont))
diff --git a/lib-python/2.7/ssl.py b/lib-python/2.7/ssl.py
--- a/lib-python/2.7/ssl.py
+++ b/lib-python/2.7/ssl.py
@@ -62,7 +62,6 @@
 from _ssl import OPENSSL_VERSION_NUMBER, OPENSSL_VERSION_INFO, OPENSSL_VERSION
 from _ssl import SSLError
 from _ssl import CERT_NONE, CERT_OPTIONAL, CERT_REQUIRED
-from _ssl import PROTOCOL_SSLv2, PROTOCOL_SSLv3, PROTOCOL_SSLv23, PROTOCOL_TLSv1
 from _ssl import RAND_status, RAND_egd, RAND_add
 from _ssl import \
      SSL_ERROR_ZERO_RETURN, \
@@ -74,6 +73,18 @@
      SSL_ERROR_WANT_CONNECT, \
      SSL_ERROR_EOF, \
      SSL_ERROR_INVALID_ERROR_CODE
+from _ssl import PROTOCOL_SSLv3, PROTOCOL_SSLv23, PROTOCOL_TLSv1
+_PROTOCOL_NAMES = {
+    PROTOCOL_TLSv1: "TLSv1",
+    PROTOCOL_SSLv23: "SSLv23",
+    PROTOCOL_SSLv3: "SSLv3",
+}
+try:
+    from _ssl import PROTOCOL_SSLv2
+except ImportError:
+    pass
+else:
+    _PROTOCOL_NAMES[PROTOCOL_SSLv2] = "SSLv2"
 
 from socket import socket, _fileobject, _delegate_methods, error as socket_error
 from socket import getnameinfo as _getnameinfo
@@ -408,16 +419,7 @@
     return DER_cert_to_PEM_cert(dercert)
 
 def get_protocol_name(protocol_code):
-    if protocol_code == PROTOCOL_TLSv1:
-        return "TLSv1"
-    elif protocol_code == PROTOCOL_SSLv23:
-        return "SSLv23"
-    elif protocol_code == PROTOCOL_SSLv2:
-        return "SSLv2"
-    elif protocol_code == PROTOCOL_SSLv3:
-        return "SSLv3"
-    else:
-        return "<unknown>"
+    return _PROTOCOL_NAMES.get(protocol_code, '<unknown>')
 
 
 # a replacement for the old socket.ssl function
diff --git a/lib-python/2.7/test/test_ssl.py b/lib-python/2.7/test/test_ssl.py
--- a/lib-python/2.7/test/test_ssl.py
+++ b/lib-python/2.7/test/test_ssl.py
@@ -58,32 +58,35 @@
 
 # Issue #9415: Ubuntu hijacks their OpenSSL and forcefully disables SSLv2
 def skip_if_broken_ubuntu_ssl(func):
-    # We need to access the lower-level wrapper in order to create an
-    # implicit SSL context without trying to connect or listen.
-    try:
-        import _ssl
-    except ImportError:
-        # The returned function won't get executed, just ignore the error
-        pass
-    @functools.wraps(func)
-    def f(*args, **kwargs):
+    if hasattr(ssl, 'PROTOCOL_SSLv2'):
+        # We need to access the lower-level wrapper in order to create an
+        # implicit SSL context without trying to connect or listen.
         try:
-            s = socket.socket(socket.AF_INET)
-            _ssl.sslwrap(s._sock, 0, None, None,
-                         ssl.CERT_NONE, ssl.PROTOCOL_SSLv2, None, None)
-        except ssl.SSLError as e:
-            if (ssl.OPENSSL_VERSION_INFO == (0, 9, 8, 15, 15) and
-                platform.linux_distribution() == ('debian', 'squeeze/sid', '')
-                and 'Invalid SSL protocol variant specified' in str(e)):
-                raise unittest.SkipTest("Patched Ubuntu OpenSSL breaks behaviour")
-        return func(*args, **kwargs)
-    return f
+            import _ssl
+        except ImportError:
+            # The returned function won't get executed, just ignore the error
+            pass
+        @functools.wraps(func)
+        def f(*args, **kwargs):
+            try:
+                s = socket.socket(socket.AF_INET)
+                _ssl.sslwrap(s._sock, 0, None, None,
+                             ssl.CERT_NONE, ssl.PROTOCOL_SSLv2, None, None)
+            except ssl.SSLError as e:
+                if (ssl.OPENSSL_VERSION_INFO == (0, 9, 8, 15, 15) and
+                    platform.linux_distribution() == ('debian', 'squeeze/sid', '')
+                    and 'Invalid SSL protocol variant specified' in str(e)):
+                    raise unittest.SkipTest("Patched Ubuntu OpenSSL breaks behaviour")
+            return func(*args, **kwargs)
+        return f
+    else:
+        return func
 
 
 class BasicSocketTests(unittest.TestCase):
 
     def test_constants(self):
-        ssl.PROTOCOL_SSLv2
+        #ssl.PROTOCOL_SSLv2
         ssl.PROTOCOL_SSLv23
         ssl.PROTOCOL_SSLv3
         ssl.PROTOCOL_TLSv1
@@ -964,7 +967,8 @@
             try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, True)
             try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, True, ssl.CERT_OPTIONAL)
             try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, True, ssl.CERT_REQUIRED)
-            try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv2, False)
+            if hasattr(ssl, 'PROTOCOL_SSLv2'):
+                try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv2, False)
             try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv23, False)
             try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_TLSv1, False)
 
@@ -976,7 +980,8 @@
             try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, True)
             try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, True, ssl.CERT_OPTIONAL)
             try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, True, ssl.CERT_REQUIRED)
-            try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv2, False)
+            if hasattr(ssl, 'PROTOCOL_SSLv2'):
+                try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv2, False)
             try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv3, False)
             try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv23, False)
 
diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -317,7 +317,7 @@
     RegrTest('test_multibytecodec.py', usemodules='_multibytecodec'),
     RegrTest('test_multibytecodec_support.py', skip="not a test"),
     RegrTest('test_multifile.py'),
-    RegrTest('test_multiprocessing.py', skip='FIXME leaves subprocesses'),
+    RegrTest('test_multiprocessing.py', skip="FIXME leaves subprocesses"),
     RegrTest('test_mutants.py', core="possibly"),
     RegrTest('test_mutex.py'),
     RegrTest('test_netrc.py'),
@@ -359,7 +359,7 @@
     RegrTest('test_property.py', core=True),
     RegrTest('test_pstats.py'),
     RegrTest('test_pty.py', skip="unsupported extension module"),
-    RegrTest('test_pwd.py', skip=skip_win32),
+    RegrTest('test_pwd.py', usemodules="pwd", skip=skip_win32),
     RegrTest('test_py3kwarn.py'),
     RegrTest('test_pyclbr.py'),
     RegrTest('test_pydoc.py'),
diff --git a/lib-python/modified-2.7/gzip.py b/lib-python/modified-2.7/gzip.py
deleted file mode 100644
--- a/lib-python/modified-2.7/gzip.py
+++ /dev/null
@@ -1,514 +0,0 @@
-"""Functions that read and write gzipped files.
-
-The user of the file doesn't have to worry about the compression,
-but random access is not allowed."""
-
-# based on Andrew Kuchling's minigzip.py distributed with the zlib module
-
-import struct, sys, time, os
-import zlib
-import io
-import __builtin__
-
-__all__ = ["GzipFile","open"]
-
-FTEXT, FHCRC, FEXTRA, FNAME, FCOMMENT = 1, 2, 4, 8, 16
-
-READ, WRITE = 1, 2
-
-def write32u(output, value):
-    # The L format writes the bit pattern correctly whether signed
-    # or unsigned.
-    output.write(struct.pack("<L", value))
-
-def read32(input):
-    return struct.unpack("<I", input.read(4))[0]
-
-def open(filename, mode="rb", compresslevel=9):
-    """Shorthand for GzipFile(filename, mode, compresslevel).
-
-    The filename argument is required; mode defaults to 'rb'
-    and compresslevel defaults to 9.
-
-    """
-    return GzipFile(filename, mode, compresslevel)
-
-class GzipFile(io.BufferedIOBase):
-    """The GzipFile class simulates most of the methods of a file object with
-    the exception of the readinto() and truncate() methods.
-
-    """
-
-    myfileobj = None
-    max_read_chunk = 10 * 1024 * 1024   # 10Mb
-
-    def __init__(self, filename=None, mode=None,
-                 compresslevel=9, fileobj=None, mtime=None):
-        """Constructor for the GzipFile class.
-
-        At least one of fileobj and filename must be given a
-        non-trivial value.
-
-        The new class instance is based on fileobj, which can be a regular
-        file, a StringIO object, or any other object which simulates a file.
-        It defaults to None, in which case filename is opened to provide
-        a file object.
-
-        When fileobj is not None, the filename argument is only used to be
-        included in the gzip file header, which may includes the original
-        filename of the uncompressed file.  It defaults to the filename of
-        fileobj, if discernible; otherwise, it defaults to the empty string,
-        and in this case the original filename is not included in the header.
-
-        The mode argument can be any of 'r', 'rb', 'a', 'ab', 'w', or 'wb',
-        depending on whether the file will be read or written.  The default
-        is the mode of fileobj if discernible; otherwise, the default is 'rb'.
-        Be aware that only the 'rb', 'ab', and 'wb' values should be used
-        for cross-platform portability.
-
-        The compresslevel argument is an integer from 1 to 9 controlling the
-        level of compression; 1 is fastest and produces the least compression,
-        and 9 is slowest and produces the most compression.  The default is 9.
-
-        The mtime argument is an optional numeric timestamp to be written
-        to the stream when compressing.  All gzip compressed streams
-        are required to contain a timestamp.  If omitted or None, the
-        current time is used.  This module ignores the timestamp when
-        decompressing; however, some programs, such as gunzip, make use
-        of it.  The format of the timestamp is the same as that of the
-        return value of time.time() and of the st_mtime member of the
-        object returned by os.stat().
-
-        """
-
-        # guarantee the file is opened in binary mode on platforms
-        # that care about that sort of thing
-        if mode and 'b' not in mode:
-            mode += 'b'
-        if fileobj is None:
-            fileobj = self.myfileobj = __builtin__.open(filename, mode or 'rb')
-        if filename is None:
-            if hasattr(fileobj, 'name'): filename = fileobj.name
-            else: filename = ''
-        if mode is None:
-            if hasattr(fileobj, 'mode'): mode = fileobj.mode
-            else: mode = 'rb'
-
-        if mode[0:1] == 'r':
-            self.mode = READ
-            # Set flag indicating start of a new member
-            self._new_member = True
-            # Buffer data read from gzip file. extrastart is offset in
-            # stream where buffer starts. extrasize is number of
-            # bytes remaining in buffer from current stream position.
-            self.extrabuf = ""
-            self.extrasize = 0
-            self.extrastart = 0
-            self.name = filename
-            # Starts small, scales exponentially
-            self.min_readsize = 100
-
-        elif mode[0:1] == 'w' or mode[0:1] == 'a':
-            self.mode = WRITE
-            self._init_write(filename)
-            self.compress = zlib.compressobj(compresslevel,
-                                             zlib.DEFLATED,
-                                             -zlib.MAX_WBITS,
-                                             zlib.DEF_MEM_LEVEL,
-                                             0)
-        else:
-            raise IOError, "Mode " + mode + " not supported"
-
-        self.fileobj = fileobj
-        self.offset = 0
-        self.mtime = mtime
-
-        if self.mode == WRITE:
-            self._write_gzip_header()
-
-    @property
-    def filename(self):
-        import warnings
-        warnings.warn("use the name attribute", DeprecationWarning, 2)
-        if self.mode == WRITE and self.name[-3:] != ".gz":
-            return self.name + ".gz"
-        return self.name
-
-    def __repr__(self):
-        s = repr(self.fileobj)
-        return '<gzip ' + s[1:-1] + ' ' + hex(id(self)) + '>'
-
-    def _check_closed(self):
-        """Raises a ValueError if the underlying file object has been closed.
-
-        """
-        if self.closed:
-            raise ValueError('I/O operation on closed file.')
-
-    def _init_write(self, filename):
-        self.name = filename
-        self.crc = zlib.crc32("") & 0xffffffffL
-        self.size = 0
-        self.writebuf = []
-        self.bufsize = 0
-
-    def _write_gzip_header(self):
-        self.fileobj.write('\037\213')             # magic header
-        self.fileobj.write('\010')                 # compression method
-        fname = os.path.basename(self.name)
-        if fname.endswith(".gz"):
-            fname = fname[:-3]
-        flags = 0
-        if fname:
-            flags = FNAME
-        self.fileobj.write(chr(flags))
-        mtime = self.mtime
-        if mtime is None:
-            mtime = time.time()
-        write32u(self.fileobj, long(mtime))
-        self.fileobj.write('\002')
-        self.fileobj.write('\377')
-        if fname:
-            self.fileobj.write(fname + '\000')
-
-    def _init_read(self):
-        self.crc = zlib.crc32("") & 0xffffffffL
-        self.size = 0
-
-    def _read_gzip_header(self):
-        magic = self.fileobj.read(2)
-        if magic != '\037\213':
-            raise IOError, 'Not a gzipped file'
-        method = ord( self.fileobj.read(1) )
-        if method != 8:
-            raise IOError, 'Unknown compression method'
-        flag = ord( self.fileobj.read(1) )
-        self.mtime = read32(self.fileobj)
-        # extraflag = self.fileobj.read(1)
-        # os = self.fileobj.read(1)
-        self.fileobj.read(2)
-
-        if flag & FEXTRA:
-            # Read & discard the extra field, if present
-            xlen = ord(self.fileobj.read(1))
-            xlen = xlen + 256*ord(self.fileobj.read(1))
-            self.fileobj.read(xlen)
-        if flag & FNAME:
-            # Read and discard a null-terminated string containing the filename
-            while True:
-                s = self.fileobj.read(1)
-                if not s or s=='\000':
-                    break
-        if flag & FCOMMENT:
-            # Read and discard a null-terminated string containing a comment
-            while True:
-                s = self.fileobj.read(1)
-                if not s or s=='\000':
-                    break
-        if flag & FHCRC:
-            self.fileobj.read(2)     # Read & discard the 16-bit header CRC
-
-    def write(self,data):
-        self._check_closed()
-        if self.mode != WRITE:
-            import errno
-            raise IOError(errno.EBADF, "write() on read-only GzipFile object")
-
-        if self.fileobj is None:
-            raise ValueError, "write() on closed GzipFile object"
-
-        # Convert data type if called by io.BufferedWriter.
-        if isinstance(data, memoryview):
-            data = data.tobytes()
-
-        if len(data) > 0:
-            self.size = self.size + len(data)
-            self.crc = zlib.crc32(data, self.crc) & 0xffffffffL
-            self.fileobj.write( self.compress.compress(data) )
-            self.offset += len(data)
-
-        return len(data)
-
-    def read(self, size=-1):
-        self._check_closed()
-        if self.mode != READ:
-            import errno
-            raise IOError(errno.EBADF, "read() on write-only GzipFile object")
-
-        if self.extrasize <= 0 and self.fileobj is None:
-            return ''
-
-        readsize = 1024
-        if size < 0:        # get the whole thing
-            try:
-                while True:
-                    self._read(readsize)
-                    readsize = min(self.max_read_chunk, readsize * 2)
-            except EOFError:
-                size = self.extrasize
-        elif size == 0:
-            return ""
-        else:               # just get some more of it
-            try:
-                while size > self.extrasize:
-                    self._read(readsize)
-                    readsize = min(self.max_read_chunk, readsize * 2)
-            except EOFError:
-                if size > self.extrasize:
-                    size = self.extrasize
-
-        offset = self.offset - self.extrastart
-        chunk = self.extrabuf[offset: offset + size]
-        self.extrasize = self.extrasize - size
-
-        self.offset += size
-        return chunk
-
-    def _unread(self, buf):
-        self.extrasize = len(buf) + self.extrasize
-        self.offset -= len(buf)
-
-    def _read(self, size=1024):
-        if self.fileobj is None:
-            raise EOFError, "Reached EOF"
-
-        if self._new_member:
-            # If the _new_member flag is set, we have to
-            # jump to the next member, if there is one.
-            #
-            # First, check if we're at the end of the file;
-            # if so, it's time to stop; no more members to read.
-            pos = self.fileobj.tell()   # Save current position
-            self.fileobj.seek(0, 2)     # Seek to end of file
-            if pos == self.fileobj.tell():
-                raise EOFError, "Reached EOF"
-            else:
-                self.fileobj.seek( pos ) # Return to original position
-
-            self._init_read()
-            self._read_gzip_header()
-            self.decompress = zlib.decompressobj(-zlib.MAX_WBITS)
-            self._new_member = False
-
-        # Read a chunk of data from the file
-        buf = self.fileobj.read(size)
-
-        # If the EOF has been reached, flush the decompression object
-        # and mark this object as finished.
-
-        if buf == "":
-            uncompress = self.decompress.flush()
-            self._read_eof()
-            self._add_read_data( uncompress )
-            raise EOFError, 'Reached EOF'
-
-        uncompress = self.decompress.decompress(buf)
-        self._add_read_data( uncompress )
-
-        if self.decompress.unused_data != "":
-            # Ending case: we've come to the end of a member in the file,
-            # so seek back to the start of the unused data, finish up
-            # this member, and read a new gzip header.
-            # (The number of bytes to seek back is the length of the unused
-            # data, minus 8 because _read_eof() will rewind a further 8 bytes)
-            self.fileobj.seek( -len(self.decompress.unused_data)+8, 1)
-
-            # Check the CRC and file size, and set the flag so we read
-            # a new member on the next call
-            self._read_eof()
-            self._new_member = True
-
-    def _add_read_data(self, data):
-        self.crc = zlib.crc32(data, self.crc) & 0xffffffffL
-        offset = self.offset - self.extrastart
-        self.extrabuf = self.extrabuf[offset:] + data
-        self.extrasize = self.extrasize + len(data)
-        self.extrastart = self.offset
-        self.size = self.size + len(data)
-
-    def _read_eof(self):
-        # We've read to the end of the file, so we have to rewind in order
-        # to reread the 8 bytes containing the CRC and the file size.
-        # We check the that the computed CRC and size of the
-        # uncompressed data matches the stored values.  Note that the size
-        # stored is the true file size mod 2**32.
-        self.fileobj.seek(-8, 1)
-        crc32 = read32(self.fileobj)
-        isize = read32(self.fileobj)  # may exceed 2GB
-        if crc32 != self.crc:
-            raise IOError("CRC check failed %s != %s" % (hex(crc32),
-                                                         hex(self.crc)))
-        elif isize != (self.size & 0xffffffffL):
-            raise IOError, "Incorrect length of data produced"
-
-        # Gzip files can be padded with zeroes and still have archives.
-        # Consume all zero bytes and set the file position to the first
-        # non-zero byte. See http://www.gzip.org/#faq8
-        c = "\x00"
-        while c == "\x00":
-            c = self.fileobj.read(1)
-        if c:
-            self.fileobj.seek(-1, 1)
-
-    @property
-    def closed(self):
-        return self.fileobj is None
-
-    def close(self):
-        if self.fileobj is None:
-            return
-        if self.mode == WRITE:
-            self.fileobj.write(self.compress.flush())
-            write32u(self.fileobj, self.crc)
-            # self.size may exceed 2GB, or even 4GB
-            write32u(self.fileobj, self.size & 0xffffffffL)
-            self.fileobj = None
-        elif self.mode == READ:
-            self.fileobj = None
-        if self.myfileobj:
-            self.myfileobj.close()
-            self.myfileobj = None
-
-    def flush(self,zlib_mode=zlib.Z_SYNC_FLUSH):
-        self._check_closed()
-        if self.mode == WRITE:
-            # Ensure the compressor's buffer is flushed
-            self.fileobj.write(self.compress.flush(zlib_mode))
-            self.fileobj.flush()
-
-    def fileno(self):
-        """Invoke the underlying file object's fileno() method.
-
-        This will raise AttributeError if the underlying file object
-        doesn't support fileno().
-        """
-        return self.fileobj.fileno()
-
-    def rewind(self):
-        '''Return the uncompressed stream file position indicator to the
-        beginning of the file'''
-        if self.mode != READ:
-            raise IOError("Can't rewind in write mode")
-        self.fileobj.seek(0)
-        self._new_member = True
-        self.extrabuf = ""
-        self.extrasize = 0
-        self.extrastart = 0
-        self.offset = 0
-
-    def readable(self):
-        return self.mode == READ
-
-    def writable(self):
-        return self.mode == WRITE
-
-    def seekable(self):
-        return True
-
-    def seek(self, offset, whence=0):
-        if whence:
-            if whence == 1:
-                offset = self.offset + offset
-            else:
-                raise ValueError('Seek from end not supported')
-        if self.mode == WRITE:
-            if offset < self.offset:
-                raise IOError('Negative seek in write mode')
-            count = offset - self.offset
-            for i in range(count // 1024):
-                self.write(1024 * '\0')
-            self.write((count % 1024) * '\0')
-        elif self.mode == READ:
-            if offset == self.offset:
-                self.read(0) # to make sure that this file is open
-                return self.offset
-            if offset < self.offset:
-                # for negative seek, rewind and do positive seek
-                self.rewind()
-            count = offset - self.offset
-            for i in range(count // 1024):
-                self.read(1024)
-            self.read(count % 1024)
-
-        return self.offset
-
-    def readline(self, size=-1):
-        if size < 0:
-            # Shortcut common case - newline found in buffer.
-            offset = self.offset - self.extrastart
-            i = self.extrabuf.find('\n', offset) + 1
-            if i > 0:
-                self.extrasize -= i - offset
-                self.offset += i - offset
-                return self.extrabuf[offset: i]
-
-            size = sys.maxint
-            readsize = self.min_readsize
-        else:
-            readsize = size
-        bufs = []
-        while size != 0:
-            c = self.read(readsize)
-            i = c.find('\n')
-
-            # We set i=size to break out of the loop under two
-            # conditions: 1) there's no newline, and the chunk is
-            # larger than size, or 2) there is a newline, but the
-            # resulting line would be longer than 'size'.
-            if (size <= i) or (i == -1 and len(c) > size):
-                i = size - 1
-
-            if i >= 0 or c == '':
-                bufs.append(c[:i + 1])    # Add portion of last chunk
-                self._unread(c[i + 1:])   # Push back rest of chunk
-                break
-
-            # Append chunk to list, decrease 'size',
-            bufs.append(c)
-            size = size - len(c)
-            readsize = min(size, readsize * 2)
-        if readsize > self.min_readsize:
-            self.min_readsize = min(readsize, self.min_readsize * 2, 512)
-        return ''.join(bufs) # Return resulting line
-
-
-def _test():
-    # Act like gzip; with -d, act like gunzip.
-    # The input file is not deleted, however, nor are any other gzip
-    # options or features supported.
-    args = sys.argv[1:]
-    decompress = args and args[0] == "-d"
-    if decompress:
-        args = args[1:]
-    if not args:
-        args = ["-"]
-    for arg in args:
-        if decompress:
-            if arg == "-":
-                f = GzipFile(filename="", mode="rb", fileobj=sys.stdin)
-                g = sys.stdout
-            else:
-                if arg[-3:] != ".gz":
-                    print "filename doesn't end in .gz:", repr(arg)
-                    continue
-                f = open(arg, "rb")
-                g = __builtin__.open(arg[:-3], "wb")
-        else:
-            if arg == "-":
-                f = sys.stdin
-                g = GzipFile(filename="", mode="wb", fileobj=sys.stdout)
-            else:
-                f = __builtin__.open(arg, "rb")
-                g = open(arg + ".gz", "wb")
-        while True:
-            chunk = f.read(1024)
-            if not chunk:
-                break
-            g.write(chunk)
-        if g is not sys.stdout:
-            g.close()
-        if f is not sys.stdin:
-            f.close()
-
-if __name__ == '__main__':
-    _test()
diff --git a/lib-python/modified-2.7/sqlite3/test/regression.py b/lib-python/modified-2.7/sqlite3/test/regression.py
--- a/lib-python/modified-2.7/sqlite3/test/regression.py
+++ b/lib-python/modified-2.7/sqlite3/test/regression.py
@@ -274,6 +274,18 @@
         cur.execute("UPDATE foo SET id = 3 WHERE id = 1")
         self.assertEqual(cur.description, None)
 
+    def CheckStatementCache(self):
+        cur = self.con.cursor()
+        cur.execute("CREATE TABLE foo (id INTEGER)")
+        values = [(i,) for i in xrange(5)]
+        cur.executemany("INSERT INTO foo (id) VALUES (?)", values)
+
+        cur.execute("SELECT id FROM foo")
+        self.assertEqual(list(cur), values)
+        self.con.commit()
+        cur.execute("SELECT id FROM foo")
+        self.assertEqual(list(cur), values)
+
 def suite():
     regression_suite = unittest.makeSuite(RegressionTests, "Check")
     return unittest.TestSuite((regression_suite,))
diff --git a/lib-python/modified-2.7/ssl.py b/lib-python/modified-2.7/ssl.py
--- a/lib-python/modified-2.7/ssl.py
+++ b/lib-python/modified-2.7/ssl.py
@@ -62,7 +62,6 @@
 from _ssl import OPENSSL_VERSION_NUMBER, OPENSSL_VERSION_INFO, OPENSSL_VERSION
 from _ssl import SSLError
 from _ssl import CERT_NONE, CERT_OPTIONAL, CERT_REQUIRED
-from _ssl import PROTOCOL_SSLv2, PROTOCOL_SSLv3, PROTOCOL_SSLv23, PROTOCOL_TLSv1
 from _ssl import RAND_status, RAND_egd, RAND_add
 from _ssl import \
      SSL_ERROR_ZERO_RETURN, \
@@ -74,6 +73,18 @@
      SSL_ERROR_WANT_CONNECT, \
      SSL_ERROR_EOF, \
      SSL_ERROR_INVALID_ERROR_CODE
+from _ssl import PROTOCOL_SSLv3, PROTOCOL_SSLv23, PROTOCOL_TLSv1
+_PROTOCOL_NAMES = {
+    PROTOCOL_TLSv1: "TLSv1",
+    PROTOCOL_SSLv23: "SSLv23",
+    PROTOCOL_SSLv3: "SSLv3",
+}
+try:
+    from _ssl import PROTOCOL_SSLv2
+except ImportError:
+    pass
+else:
+    _PROTOCOL_NAMES[PROTOCOL_SSLv2] = "SSLv2"
 
 from socket import socket, _fileobject, error as socket_error
 from socket import getnameinfo as _getnameinfo
@@ -400,16 +411,7 @@
     return DER_cert_to_PEM_cert(dercert)
 
 def get_protocol_name(protocol_code):
-    if protocol_code == PROTOCOL_TLSv1:
-        return "TLSv1"
-    elif protocol_code == PROTOCOL_SSLv23:
-        return "SSLv23"
-    elif protocol_code == PROTOCOL_SSLv2:
-        return "SSLv2"
-    elif protocol_code == PROTOCOL_SSLv3:
-        return "SSLv3"
-    else:
-        return "<unknown>"
+    return _PROTOCOL_NAMES.get(protocol_code, '<unknown>')
 
 
 # a replacement for the old socket.ssl function
diff --git a/lib-python/modified-2.7/tarfile.py b/lib-python/modified-2.7/tarfile.py
--- a/lib-python/modified-2.7/tarfile.py
+++ b/lib-python/modified-2.7/tarfile.py
@@ -252,8 +252,8 @@
        the high bit set. So we calculate two checksums, unsigned and
        signed.
     """
-    unsigned_chksum = 256 + sum(struct.unpack("148B8x356B", buf[:512]))
-    signed_chksum = 256 + sum(struct.unpack("148b8x356b", buf[:512]))
+    unsigned_chksum = 256 + sum(struct.unpack("148B", buf[:148]) + struct.unpack("356B", buf[156:512]))
+    signed_chksum = 256 + sum(struct.unpack("148b", buf[:148]) + struct.unpack("356b", buf[156:512]))
     return unsigned_chksum, signed_chksum
 
 def copyfileobj(src, dst, length=None):
@@ -265,6 +265,7 @@
     if length is None:
         shutil.copyfileobj(src, dst)
         return
+
     BUFSIZE = 16 * 1024
     blocks, remainder = divmod(length, BUFSIZE)
     for b in xrange(blocks):
@@ -801,19 +802,19 @@
         if self.closed:
             raise ValueError("I/O operation on closed file")
 
+        buf = ""
         if self.buffer:
             if size is None:
-                buf = self.buffer + self.fileobj.read()
+                buf = self.buffer
                 self.buffer = ""
             else:
                 buf = self.buffer[:size]
                 self.buffer = self.buffer[size:]
-                buf += self.fileobj.read(size - len(buf))
+
+        if size is None:
+            buf += self.fileobj.read()
         else:
-            if size is None:
-                buf = self.fileobj.read()
-            else:
-                buf = self.fileobj.read(size)
+            buf += self.fileobj.read(size - len(buf))
 
         self.position += len(buf)
         return buf
diff --git a/lib-python/modified-2.7/test/test_multiprocessing.py b/lib-python/modified-2.7/test/test_multiprocessing.py
--- a/lib-python/modified-2.7/test/test_multiprocessing.py
+++ b/lib-python/modified-2.7/test/test_multiprocessing.py
@@ -510,7 +510,6 @@
 
         p.join()
 
-    @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
     def test_qsize(self):
         q = self.Queue()
         try:
@@ -532,7 +531,6 @@
             time.sleep(DELTA)
             q.task_done()
 
-    @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
     def test_task_done(self):
         queue = self.JoinableQueue()
 
@@ -1091,7 +1089,6 @@
 class _TestPoolWorkerLifetime(BaseTestCase):
 
     ALLOWED_TYPES = ('processes', )
-    @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
     def test_pool_worker_lifetime(self):
         p = multiprocessing.Pool(3, maxtasksperchild=10)
         self.assertEqual(3, len(p._pool))
@@ -1280,7 +1277,6 @@
         queue = manager.get_queue()
         queue.put('hello world')
 
-    @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
     def test_rapid_restart(self):
         authkey = os.urandom(32)
         manager = QueueManager(
@@ -1297,6 +1293,7 @@
         queue = manager.get_queue()
         self.assertEqual(queue.get(), 'hello world')
         del queue
+        test_support.gc_collect()
         manager.shutdown()
         manager = QueueManager(
             address=addr, authkey=authkey, serializer=SERIALIZER)
@@ -1573,7 +1570,6 @@
 
     ALLOWED_TYPES = ('processes',)
 
-    @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
     def test_heap(self):
         iterations = 5000
         maxblocks = 50
diff --git a/lib-python/modified-2.7/test/test_ssl.py b/lib-python/modified-2.7/test/test_ssl.py
--- a/lib-python/modified-2.7/test/test_ssl.py
+++ b/lib-python/modified-2.7/test/test_ssl.py
@@ -58,32 +58,35 @@
 
 # Issue #9415: Ubuntu hijacks their OpenSSL and forcefully disables SSLv2
 def skip_if_broken_ubuntu_ssl(func):
-    # We need to access the lower-level wrapper in order to create an
-    # implicit SSL context without trying to connect or listen.
-    try:
-        import _ssl
-    except ImportError:
-        # The returned function won't get executed, just ignore the error
-        pass
-    @functools.wraps(func)
-    def f(*args, **kwargs):
+    if hasattr(ssl, 'PROTOCOL_SSLv2'):
+        # We need to access the lower-level wrapper in order to create an
+        # implicit SSL context without trying to connect or listen.
         try:
-            s = socket.socket(socket.AF_INET)
-            _ssl.sslwrap(s._sock, 0, None, None,
-                         ssl.CERT_NONE, ssl.PROTOCOL_SSLv2, None, None)
-        except ssl.SSLError as e:
-            if (ssl.OPENSSL_VERSION_INFO == (0, 9, 8, 15, 15) and
-                platform.linux_distribution() == ('debian', 'squeeze/sid', '')
-                and 'Invalid SSL protocol variant specified' in str(e)):
-                raise unittest.SkipTest("Patched Ubuntu OpenSSL breaks behaviour")
-        return func(*args, **kwargs)
-    return f
+            import _ssl
+        except ImportError:
+            # The returned function won't get executed, just ignore the error
+            pass
+        @functools.wraps(func)
+        def f(*args, **kwargs):
+            try:
+                s = socket.socket(socket.AF_INET)
+                _ssl.sslwrap(s._sock, 0, None, None,
+                             ssl.CERT_NONE, ssl.PROTOCOL_SSLv2, None, None)
+            except ssl.SSLError as e:
+                if (ssl.OPENSSL_VERSION_INFO == (0, 9, 8, 15, 15) and
+                    platform.linux_distribution() == ('debian', 'squeeze/sid', '')
+                    and 'Invalid SSL protocol variant specified' in str(e)):
+                    raise unittest.SkipTest("Patched Ubuntu OpenSSL breaks behaviour")
+            return func(*args, **kwargs)
+        return f
+    else:
+        return func
 
 
 class BasicSocketTests(unittest.TestCase):
 
     def test_constants(self):
-        ssl.PROTOCOL_SSLv2
+        #ssl.PROTOCOL_SSLv2
         ssl.PROTOCOL_SSLv23
         ssl.PROTOCOL_SSLv3
         ssl.PROTOCOL_TLSv1
@@ -966,7 +969,8 @@
             try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, True)
             try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, True, ssl.CERT_OPTIONAL)
             try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, True, ssl.CERT_REQUIRED)
-            try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv2, False)
+            if hasattr(ssl, 'PROTOCOL_SSLv2'):
+                try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv2, False)
             try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv23, False)
             try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_TLSv1, False)
 
@@ -978,7 +982,8 @@
             try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, True)
             try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, True, ssl.CERT_OPTIONAL)
             try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, True, ssl.CERT_REQUIRED)
-            try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv2, False)
+            if hasattr(ssl, 'PROTOCOL_SSLv2'):
+                try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv2, False)
             try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv3, False)
             try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv23, False)
 
diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -54,7 +54,8 @@
     def get_ffi_argtype(self):
         if self._ffiargtype:
             return self._ffiargtype
-        return _shape_to_ffi_type(self._ffiargshape)
+        self._ffiargtype = _shape_to_ffi_type(self._ffiargshape)
+        return self._ffiargtype
 
     def _CData_output(self, resbuffer, base=None, index=-1):
         #assert isinstance(resbuffer, _rawffi.ArrayInstance)
@@ -225,6 +226,7 @@
     'Z' : _ffi.types.void_p,
     'X' : _ffi.types.void_p,
     'v' : _ffi.types.sshort,
+    '?' : _ffi.types.ubyte,
     }
 
 
diff --git a/lib_pypy/_elementtree.py b/lib_pypy/_elementtree.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_elementtree.py
@@ -0,0 +1,6 @@
+# Just use ElementTree.
+
+from xml.etree import ElementTree
+
+globals().update(ElementTree.__dict__)
+del __all__
diff --git a/lib_pypy/_functools.py b/lib_pypy/_functools.py
--- a/lib_pypy/_functools.py
+++ b/lib_pypy/_functools.py
@@ -14,10 +14,9 @@
             raise TypeError("the first argument must be callable")
         self.func = func
         self.args = args
-        self.keywords = keywords
+        self.keywords = keywords or None
 
     def __call__(self, *fargs, **fkeywords):
-        newkeywords = self.keywords.copy()
-        newkeywords.update(fkeywords)
-        return self.func(*(self.args + fargs), **newkeywords)
-
+        if self.keywords is not None:
+            fkeywords = dict(self.keywords, **fkeywords)
+        return self.func(*(self.args + fargs), **fkeywords)
diff --git a/lib_pypy/_pypy_interact.py b/lib_pypy/_pypy_interact.py
--- a/lib_pypy/_pypy_interact.py
+++ b/lib_pypy/_pypy_interact.py
@@ -56,6 +56,10 @@
                 prompt = getattr(sys, 'ps1', '>>> ')
             try:
                 line = raw_input(prompt)
+                # Can be None if sys.stdin was redefined
+                encoding = getattr(sys.stdin, 'encoding', None)
+                if encoding and not isinstance(line, unicode):
+                    line = line.decode(encoding)
             except EOFError:
                 console.write("\n")
                 break
diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py
--- a/lib_pypy/_sqlite3.py
+++ b/lib_pypy/_sqlite3.py
@@ -293,7 +293,7 @@
         #
         if stat.in_use:
             stat = Statement(self.connection, sql)
-        stat.set_cursor_and_factory(cursor, row_factory)
+        stat.set_row_factory(row_factory)
         return stat
 
 
@@ -705,6 +705,8 @@
         from sqlite3.dump import _iterdump
         return _iterdump(self)
 
+DML, DQL, DDL = range(3)
+
 class Cursor(object):
     def __init__(self, con):
         if not isinstance(con, Connection):
@@ -735,9 +737,9 @@
         self.statement = self.connection.statement_cache.get(sql, self, self.row_factory)
 
         if self.connection._isolation_level is not None:
-            if self.statement.kind == "DDL":
+            if self.statement.kind == DDL:
                 self.connection.commit()
-            elif self.statement.kind == "DML":
+            elif self.statement.kind == DML:
                 self.connection._begin()
 
         self.statement.set_params(params)
@@ -748,18 +750,18 @@
             self.statement.reset()
             raise self.connection._get_exception(ret)
 
-        if self.statement.kind == "DQL"and ret == SQLITE_ROW:
+        if self.statement.kind == DQL and ret == SQLITE_ROW:
             self.statement._build_row_cast_map()
-            self.statement._readahead()
+            self.statement._readahead(self)
         else:
             self.statement.item = None
             self.statement.exhausted = True
 
-        if self.statement.kind in ("DML", "DDL"):
+        if self.statement.kind == DML or self.statement.kind == DDL:
             self.statement.reset()
 
         self.rowcount = -1
-        if self.statement.kind == "DML":
+        if self.statement.kind == DML:
             self.rowcount = sqlite.sqlite3_changes(self.connection.db)
 
         return self
@@ -771,8 +773,8 @@
             sql = sql.encode("utf-8")
         self._check_closed()
         self.statement = self.connection.statement_cache.get(sql, self, self.row_factory)
-        
-        if self.statement.kind == "DML":
+
+        if self.statement.kind == DML:
             self.connection._begin()
         else:
             raise ProgrammingError, "executemany is only for DML statements"
@@ -824,7 +826,7 @@
         return self
 
     def __iter__(self):
-        return self.statement
+        return iter(self.fetchone, None)
 
     def _check_reset(self):
         if self.reset:
@@ -841,7 +843,7 @@
             return None
 
         try:
-            return self.statement.next()
+            return self.statement.next(self)
         except StopIteration:
             return None
 
@@ -855,7 +857,7 @@
         if size is None:
             size = self.arraysize
         lst = []
-        for row in self.statement:
+        for row in self:
             lst.append(row)
             if len(lst) == size:
                 break
@@ -866,7 +868,7 @@
         self._check_reset()
         if self.statement is None:
             return []
-        return list(self.statement)
+        return list(self)
 
     def _getdescription(self):
         if self._description is None:
@@ -904,16 +906,15 @@
         self.sql = sql # DEBUG ONLY
         first_word = self._statement_kind = sql.lstrip().split(" ")[0].upper()
         if first_word in ("INSERT", "UPDATE", "DELETE", "REPLACE"):
-            self.kind = "DML"
+            self.kind = DML
         elif first_word in ("SELECT", "PRAGMA"):
-            self.kind = "DQL"
+            self.kind = DQL
         else:
-            self.kind = "DDL"
+            self.kind = DDL
         self.exhausted = False
         self.in_use = False
         #
-        # set by set_cursor_and_factory
-        self.cur = None
+        # set by set_row_factory
         self.row_factory = None
 
         self.statement = c_void_p()
@@ -923,7 +924,7 @@
         if ret == SQLITE_OK and self.statement.value is None:
             # an empty statement, we work around that, as it's the least trouble
             ret = sqlite.sqlite3_prepare_v2(self.con.db, "select 42", -1, byref(self.statement), byref(next_char))
-            self.kind = "DQL"
+            self.kind = DQL
 
         if ret != SQLITE_OK:
             raise self.con._get_exception(ret)
@@ -935,8 +936,7 @@
 
         self._build_row_cast_map()
 
-    def set_cursor_and_factory(self, cur, row_factory):
-        self.cur = weakref.ref(cur)
+    def set_row_factory(self, row_factory):
         self.row_factory = row_factory
 
     def _build_row_cast_map(self):
@@ -1039,10 +1039,7 @@
                     raise ProgrammingError("missing parameter '%s'" %param)
                 self.set_param(idx, param)
 
-    def __iter__(self):
-        return self
-
-    def next(self):
+    def next(self, cursor):
         self.con._check_closed()
         self.con._check_thread()
         if self.exhausted:
@@ -1058,10 +1055,10 @@
             sqlite.sqlite3_reset(self.statement)
             raise exc
 
-        self._readahead()
+        self._readahead(cursor)
         return item
 
-    def _readahead(self):
+    def _readahead(self, cursor):
         self.column_count = sqlite.sqlite3_column_count(self.statement)
         row = []
         for i in xrange(self.column_count):
@@ -1096,13 +1093,14 @@
 
         row = tuple(row)
         if self.row_factory is not None:
-            row = self.row_factory(self.cur(), row)
+            row = self.row_factory(cursor, row)
         self.item = row
 
     def reset(self):
         self.row_cast_map = None
         ret = sqlite.sqlite3_reset(self.statement)
         self.in_use = False
+        self.exhausted = False
         return ret
 
     def finalize(self):
@@ -1118,7 +1116,7 @@
         self.statement = None
 
     def _get_description(self):
-        if self.kind == "DML":
+        if self.kind == DML:
             return None
         desc = []
         for i in xrange(sqlite.sqlite3_column_count(self.statement)):
diff --git a/lib_pypy/distributed/test/test_distributed.py b/lib_pypy/distributed/test/test_distributed.py
--- a/lib_pypy/distributed/test/test_distributed.py
+++ b/lib_pypy/distributed/test/test_distributed.py
@@ -9,7 +9,7 @@
 class AppTestDistributed(object):
     def setup_class(cls):
         cls.space = gettestobjspace(**{"objspace.std.withtproxy": True,
-            "usemodules":("_stackless",)})
+            "usemodules":("_continuation",)})
 
     def test_init(self):
         import distributed
@@ -91,10 +91,8 @@
 
 class AppTestDistributedTasklets(object):
     spaceconfig = {"objspace.std.withtproxy": True,
-                   "objspace.usemodules._stackless": True}
+                   "objspace.usemodules._continuation": True}
     def setup_class(cls):
-        #cls.space = gettestobjspace(**{"objspace.std.withtproxy": True,
-        #    "usemodules":("_stackless",)})
         cls.w_test_env = cls.space.appexec([], """():
         from distributed import test_env
         return test_env
diff --git a/lib_pypy/distributed/test/test_greensock.py b/lib_pypy/distributed/test/test_greensock.py
--- a/lib_pypy/distributed/test/test_greensock.py
+++ b/lib_pypy/distributed/test/test_greensock.py
@@ -10,7 +10,7 @@
         if not option.runappdirect:
             py.test.skip("Cannot run this on top of py.py because of PopenGateway")
         cls.space = gettestobjspace(**{"objspace.std.withtproxy": True,
-                                       "usemodules":("_stackless",)})
+                                       "usemodules":("_continuation",)})
         cls.w_remote_side_code = cls.space.appexec([], """():
         import sys
         sys.path.insert(0, '%s')
diff --git a/lib_pypy/distributed/test/test_socklayer.py b/lib_pypy/distributed/test/test_socklayer.py
--- a/lib_pypy/distributed/test/test_socklayer.py
+++ b/lib_pypy/distributed/test/test_socklayer.py
@@ -9,7 +9,8 @@
 class AppTestSocklayer:
     def setup_class(cls):
         cls.space = gettestobjspace(**{"objspace.std.withtproxy": True,
-                                       "usemodules":("_stackless","_socket", "select")})
+                                       "usemodules":("_continuation",
+                                                     "_socket", "select")})
     
     def test_socklayer(self):
         class X(object):
diff --git a/lib_pypy/greenlet.py b/lib_pypy/greenlet.py
--- a/lib_pypy/greenlet.py
+++ b/lib_pypy/greenlet.py
@@ -48,18 +48,23 @@
     def switch(self, *args):
         "Switch execution to this greenlet, optionally passing the values "
         "given as argument(s).  Returns the value passed when switching back."
-        return self.__switch(_continulet.switch, args)
+        return self.__switch('switch', args)
 
     def throw(self, typ=GreenletExit, val=None, tb=None):
         "raise exception in greenlet, return value passed when switching back"
-        return self.__switch(_continulet.throw, typ, val, tb)
+        return self.__switch('throw', typ, val, tb)
 
-    def __switch(target, unbound_method, *args):
+    def __switch(target, methodname, *args):
         current = getcurrent()
         #
         while not target:
             if not target.__started:
-                _continulet.__init__(target, _greenlet_start, *args)
+                if methodname == 'switch':
+                    greenlet_func = _greenlet_start
+                else:
+                    greenlet_func = _greenlet_throw
+                _continulet.__init__(target, greenlet_func, *args)
+                methodname = 'switch'
                 args = ()
                 target.__started = True
                 break
@@ -70,22 +75,8 @@
             target = target.parent
         #
         try:
-            if current.__main:
-                if target.__main:
-                    # switch from main to main
-                    if unbound_method == _continulet.throw:
-                        raise args[0], args[1], args[2]
-                    (args,) = args
-                else:
-                    # enter from main to target
-                    args = unbound_method(target, *args)
-            else:
-                if target.__main:
-                    # leave to go to target=main
-                    args = unbound_method(current, *args)
-                else:
-                    # switch from non-main to non-main
-                    args = unbound_method(current, *args, to=target)
+            unbound_method = getattr(_continulet, methodname)
+            args = unbound_method(current, *args, to=target)
         except GreenletExit, e:
             args = (e,)
         finally:
@@ -105,7 +96,16 @@
 
     @property
     def gr_frame(self):
-        raise NotImplementedError("attribute 'gr_frame' of greenlet objects")
+        # xxx this doesn't work when called on either the current or
+        # the main greenlet of another thread
+        if self is getcurrent():
+            return None
+        if self.__main:
+            self = getcurrent()
+        f = _continulet.__reduce__(self)[2][0]
+        if not f:
+            return None
+        return f.f_back.f_back.f_back   # go past start(), __switch(), switch()
 
 # ____________________________________________________________
 # Internal stuff
@@ -133,6 +133,12 @@
     try:
         res = greenlet.run(*args)
     finally:
-        if greenlet.parent is not _tls.main:
-            _continuation.permute(greenlet, greenlet.parent)
+        _continuation.permute(greenlet, greenlet.parent)
     return (res,)
+
+def _greenlet_throw(greenlet, exc, value, tb):
+    _tls.current = greenlet
+    try:
+        raise exc, value, tb
+    finally:
+        _continuation.permute(greenlet, greenlet.parent)
diff --git a/lib_pypy/pypy_test/test_coroutine.py b/lib_pypy/pypy_test/test_coroutine.py
--- a/lib_pypy/pypy_test/test_coroutine.py
+++ b/lib_pypy/pypy_test/test_coroutine.py
@@ -2,7 +2,7 @@
 from py.test import skip, raises
 
 try:
-    from lib_pypy.stackless import coroutine, CoroutineExit
+    from stackless import coroutine, CoroutineExit
 except ImportError, e:
     skip('cannot import stackless: %s' % (e,))
 
@@ -20,10 +20,6 @@
         assert not co.is_zombie
 
     def test_is_zombie_del_without_frame(self):
-        try:
-            import _stackless # are we on pypy with a stackless build?
-        except ImportError:
-            skip("only works on pypy-c-stackless")
         import gc
         res = []
         class MyCoroutine(coroutine):
@@ -45,10 +41,6 @@
         assert res[0], "is_zombie was False in __del__"
 
     def test_is_zombie_del_with_frame(self):
-        try:
-            import _stackless # are we on pypy with a stackless build?
-        except ImportError:
-            skip("only works on pypy-c-stackless")
         import gc
         res = []
         class MyCoroutine(coroutine):
diff --git a/lib_pypy/pypy_test/test_stackless_pickling.py b/lib_pypy/pypy_test/test_stackless_pickling.py
--- a/lib_pypy/pypy_test/test_stackless_pickling.py
+++ b/lib_pypy/pypy_test/test_stackless_pickling.py
@@ -1,7 +1,3 @@
-"""
-this test should probably not run from CPython or py.py.
-I'm not entirely sure, how to do that.
-"""
 from __future__ import absolute_import
 from py.test import skip
 try:
@@ -16,11 +12,15 @@
 
 class Test_StacklessPickling:
 
+    def test_pickle_main_coroutine(self):
+        import stackless, pickle
+        s = pickle.dumps(stackless.coroutine.getcurrent())
+        print s
+        c = pickle.loads(s)
+        assert c is stackless.coroutine.getcurrent()
+
     def test_basic_tasklet_pickling(self):
-        try:
-            import stackless
-        except ImportError:
-            skip("can't load stackless and don't know why!!!")
+        import stackless
         from stackless import run, schedule, tasklet
         import pickle
 
diff --git a/lib_pypy/pyrepl/completing_reader.py b/lib_pypy/pyrepl/completing_reader.py
--- a/lib_pypy/pyrepl/completing_reader.py
+++ b/lib_pypy/pyrepl/completing_reader.py
@@ -229,7 +229,8 @@
 
     def after_command(self, cmd):
         super(CompletingReader, self).after_command(cmd)
-        if not isinstance(cmd, complete) and not isinstance(cmd, self_insert):
+        if not isinstance(cmd, self.commands['complete']) \
+           and not isinstance(cmd, self.commands['self_insert']):
             self.cmpltn_reset()
 
     def calc_screen(self):
diff --git a/lib_pypy/pyrepl/reader.py b/lib_pypy/pyrepl/reader.py
--- a/lib_pypy/pyrepl/reader.py
+++ b/lib_pypy/pyrepl/reader.py
@@ -576,7 +576,7 @@
         self.console.push_char(char)
         self.handle1(0)
     
-    def readline(self):
+    def readline(self, returns_unicode=False):
         """Read a line.  The implementation of this method also shows
         how to drive Reader if you want more control over the event
         loop."""
@@ -585,6 +585,8 @@
             self.refresh()
             while not self.finished:
                 self.handle1()
+            if returns_unicode:
+                return self.get_unicode()
             return self.get_buffer()
         finally:
             self.restore()
diff --git a/lib_pypy/pyrepl/readline.py b/lib_pypy/pyrepl/readline.py
--- a/lib_pypy/pyrepl/readline.py
+++ b/lib_pypy/pyrepl/readline.py
@@ -198,7 +198,7 @@
         reader.ps1 = prompt
         return reader.readline()
 
-    def multiline_input(self, more_lines, ps1, ps2):
+    def multiline_input(self, more_lines, ps1, ps2, returns_unicode=False):
         """Read an input on possibly multiple lines, asking for more
         lines as long as 'more_lines(unicodetext)' returns an object whose
         boolean value is true.
@@ -209,7 +209,7 @@
             reader.more_lines = more_lines
             reader.ps1 = reader.ps2 = ps1
             reader.ps3 = reader.ps4 = ps2
-            return reader.readline()
+            return reader.readline(returns_unicode=returns_unicode)
         finally:
             reader.more_lines = saved
 
diff --git a/lib_pypy/pyrepl/simple_interact.py b/lib_pypy/pyrepl/simple_interact.py
--- a/lib_pypy/pyrepl/simple_interact.py
+++ b/lib_pypy/pyrepl/simple_interact.py
@@ -54,7 +54,8 @@
             ps1 = getattr(sys, 'ps1', '>>> ')
             ps2 = getattr(sys, 'ps2', '... ')
             try:
-                statement = multiline_input(more_lines, ps1, ps2)
+                statement = multiline_input(more_lines, ps1, ps2,
+                                            returns_unicode=True)
             except EOFError:
                 break
             more = console.push(statement)
diff --git a/lib_pypy/stackless.py b/lib_pypy/stackless.py
--- a/lib_pypy/stackless.py
+++ b/lib_pypy/stackless.py
@@ -4,121 +4,110 @@
 Please refer to their documentation.
 """
 
-DEBUG = True
 
-def dprint(*args):
-    for arg in args:
-        print arg,
-    print
+import _continuation
 
-import traceback
-import sys
+class TaskletExit(Exception):
+    pass
+
+CoroutineExit = TaskletExit
+
+
+def _coroutine_getcurrent():
+    "Returns the current coroutine (i.e. the one which called this function)."
+    try:
+        return _tls.current_coroutine
+    except AttributeError:
+        # first call in this thread: current == main
+        return _coroutine_getmain()
+
+def _coroutine_getmain():
+    try:
+        return _tls.main_coroutine
+    except AttributeError:
+        # create the main coroutine for this thread
+        continulet = _continuation.continulet
+        main = coroutine()
+        main._frame = continulet.__new__(continulet)
+        main._is_started = -1
+        _tls.current_coroutine = _tls.main_coroutine = main
+        return _tls.main_coroutine
+
+
+class coroutine(object):
+    _is_started = 0      # 0=no, 1=yes, -1=main
+
+    def __init__(self):
+        self._frame = None
+
+    def bind(self, func, *argl, **argd):
+        """coro.bind(f, *argl, **argd) -> None.
+           binds function f to coro. f will be called with
+           arguments *argl, **argd
+        """
+        if self.is_alive:
+            raise ValueError("cannot bind a bound coroutine")
+        def run(c):
+            _tls.current_coroutine = self
+            self._is_started = 1
+            return func(*argl, **argd)
+        self._is_started = 0
+        self._frame = _continuation.continulet(run)
+
+    def switch(self):
+        """coro.switch() -> returnvalue
+           switches to coroutine coro. If the bound function
+           f finishes, the returnvalue is that of f, otherwise
+           None is returned
+        """
+        current = _coroutine_getcurrent()
+        try:
+            current._frame.switch(to=self._frame)
+        finally:
+            _tls.current_coroutine = current
+
+    def kill(self):
+        """coro.kill() : kill coroutine coro"""
+        current = _coroutine_getcurrent()
+        try:
+            current._frame.throw(CoroutineExit, to=self._frame)
+        finally:
+            _tls.current_coroutine = current
+
+    @property
+    def is_alive(self):
+        return self._is_started < 0 or (
+            self._frame is not None and self._frame.is_pending())
+
+    @property
+    def is_zombie(self):
+        return self._is_started > 0 and not self._frame.is_pending()
+
+    getcurrent = staticmethod(_coroutine_getcurrent)
+
+    def __reduce__(self):
+        if self._is_started < 0:
+            return _coroutine_getmain, ()
+        else:
+            return type(self), (), self.__dict__
+
+
 try:
-    # If _stackless can be imported then TaskletExit and CoroutineExit are 
-    # automatically added to the builtins.
-    from _stackless import coroutine, greenlet
-except ImportError: # we are running from CPython
-    from greenlet import greenlet, GreenletExit
-    TaskletExit = CoroutineExit = GreenletExit
-    del GreenletExit
-    try:
-        from functools import partial
-    except ImportError: # we are not running python 2.5
-        class partial(object):
-            # just enough of 'partial' to be usefull
-            def __init__(self, func, *argl, **argd):
-                self.func = func
-                self.argl = argl
-                self.argd = argd
+    from thread import _local
+except ImportError:
+    class _local(object):    # assume no threads
+        pass
 
-            def __call__(self):
-                return self.func(*self.argl, **self.argd)
+_tls = _local()
 
-    class GWrap(greenlet):
-        """This is just a wrapper around greenlets to allow
-           to stick additional attributes to a greenlet.
-           To be more concrete, we need a backreference to
-           the coroutine object"""
 
-    class MWrap(object):
-        def __init__(self,something):
-            self.something = something
+# ____________________________________________________________
 
-        def __getattr__(self, attr):
-            return getattr(self.something, attr)
-
-    class coroutine(object):
-        "we can't have greenlet as a base, because greenlets can't be rebound"
-
-        def __init__(self):
-            self._frame = None
-            self.is_zombie = False
-
-        def __getattr__(self, attr):
-            return getattr(self._frame, attr)
-
-        def __del__(self):
-            self.is_zombie = True
-            del self._frame
-            self._frame = None
-
-        def bind(self, func, *argl, **argd):
-            """coro.bind(f, *argl, **argd) -> None.
-               binds function f to coro. f will be called with
-               arguments *argl, **argd
-            """
-            if self._frame is None or self._frame.dead:
-                self._frame = frame = GWrap()
-                frame.coro = self
-            if hasattr(self._frame, 'run') and self._frame.run:
-                raise ValueError("cannot bind a bound coroutine")
-            self._frame.run = partial(func, *argl, **argd)
-
-        def switch(self):
-            """coro.switch() -> returnvalue
-               switches to coroutine coro. If the bound function
-               f finishes, the returnvalue is that of f, otherwise
-               None is returned
-            """
-            try:
-                return greenlet.switch(self._frame)
-            except TypeError, exp: # self._frame is the main coroutine
-                return greenlet.switch(self._frame.something)
-
-        def kill(self):
-            """coro.kill() : kill coroutine coro"""
-            self._frame.throw()
-
-        def _is_alive(self):
-            if self._frame is None:
-                return False
-            return not self._frame.dead
-        is_alive = property(_is_alive)
-        del _is_alive
-
-        def getcurrent():
-            """coroutine.getcurrent() -> the currently running coroutine"""
-            try:
-                return greenlet.getcurrent().coro
-            except AttributeError:
-                return _maincoro
-        getcurrent = staticmethod(getcurrent)
-
-        def __reduce__(self):
-            raise TypeError, 'pickling is not possible based upon greenlets'
-
-    _maincoro = coroutine()
-    maingreenlet = greenlet.getcurrent()
-    _maincoro._frame = frame = MWrap(maingreenlet)
-    frame.coro = _maincoro
-    del frame
-    del maingreenlet
 
 from collections import deque
 
 import operator
-__all__ = 'run getcurrent getmain schedule tasklet channel coroutine \
-                greenlet'.split()
+__all__ = 'run getcurrent getmain schedule tasklet channel coroutine'.split()
 
 _global_task_id = 0
 _squeue = None
@@ -131,7 +120,8 @@
 def _scheduler_remove(value):
     try:
         del _squeue[operator.indexOf(_squeue, value)]
-    except ValueError:pass
+    except ValueError:
+        pass
 
 def _scheduler_append(value, normal=True):
     if normal:
@@ -157,10 +147,7 @@
     _last_task = next
     assert not next.blocked
     if next is not current:
-        try:
-            next.switch()
-        except CoroutineExit:
-            raise TaskletExit
+        next.switch()
     return current
 
 def set_schedule_callback(callback):
@@ -184,34 +171,6 @@
         raise self.type, self.value, self.traceback
 
 #
-# helpers for pickling
-#
-
-_stackless_primitive_registry = {}
-
-def register_stackless_primitive(thang, retval_expr='None'):
-    import types
-    func = thang
-    if isinstance(thang, types.MethodType):
-        func = thang.im_func
-    code = func.func_code
-    _stackless_primitive_registry[code] = retval_expr
-    # It is not too nice to attach info via the code object, but
-    # I can't think of a better solution without a real transform.
-
-def rewrite_stackless_primitive(coro_state, alive, tempval):
-    flags, frame, thunk, parent = coro_state
-    while frame is not None:
-        retval_expr = _stackless_primitive_registry.get(frame.f_code)
-        if retval_expr:
-            # this tasklet needs to stop pickling here and return its value.
-            tempval = eval(retval_expr, globals(), frame.f_locals)
-            coro_state = flags, frame, thunk, parent
-            break
-        frame = frame.f_back
-    return coro_state, alive, tempval
-
-#
 #
 
 class channel(object):
@@ -363,8 +322,6 @@
         """
         return self._channel_action(None, -1)
 
-    register_stackless_primitive(receive, retval_expr='receiver.tempval')
-
     def send_exception(self, exp_type, msg):
         self.send(bomb(exp_type, exp_type(msg)))
 
@@ -381,9 +338,8 @@
         the runnables list.
         """
         return self._channel_action(msg, 1)
-            
-    register_stackless_primitive(send)
-            
+
+
 class tasklet(coroutine):
     """
     A tasklet object represents a tiny task in a Python thread.
@@ -455,6 +411,7 @@
         def _func():
             try:
                 try:
+                    coroutine.switch(back)
                     func(*argl, **argd)
                 except TaskletExit:
                     pass
@@ -464,6 +421,8 @@
 
         self.func = None
         coroutine.bind(self, _func)
+        back = _coroutine_getcurrent()
+        coroutine.switch(self)
         self.alive = True
         _scheduler_append(self)
         return self
@@ -486,39 +445,6 @@
             raise RuntimeError, "The current tasklet cannot be removed."
             # not sure if I will revive this  " Use t=tasklet().capture()"
         _scheduler_remove(self)
-        
-    def __reduce__(self):
-        one, two, coro_state = coroutine.__reduce__(self)
-        assert one is coroutine
-        assert two == ()
-        # we want to get rid of the parent thing.
-        # for now, we just drop it
-        a, frame, c, d = coro_state
-
-        # Removing all frames related to stackless.py.
-        # They point to stuff we don't want to be pickled.
-
-        pickleframe = frame
-        while frame is not None:
-            if frame.f_code == schedule.func_code:
-                # Removing everything including and after the
-                # call to stackless.schedule()
-                pickleframe = frame.f_back
-                break
-            frame = frame.f_back
-        if d:
-            assert isinstance(d, coroutine)
-        coro_state = a, pickleframe, c, None
-        coro_state, alive, tempval = rewrite_stackless_primitive(coro_state, self.alive, self.tempval)
-        inst_dict = self.__dict__.copy()
-        inst_dict.pop('tempval', None)
-        return self.__class__, (), (coro_state, alive, tempval, inst_dict)
-
-    def __setstate__(self, (coro_state, alive, tempval, inst_dict)):
-        coroutine.__setstate__(self, coro_state)
-        self.__dict__.update(inst_dict)
-        self.alive = alive
-        self.tempval = tempval
 
 def getmain():
     """
@@ -607,30 +533,7 @@
     global _last_task
     _global_task_id = 0
     _main_tasklet = coroutine.getcurrent()
-    try:
-        _main_tasklet.__class__ = tasklet
-    except TypeError: # we are running pypy-c
-        class TaskletProxy(object):
-            """TaskletProxy is needed to give the _main_coroutine tasklet behaviour"""
-            def __init__(self, coro):
-                self._coro = coro
-
-            def __getattr__(self,attr):
-                return getattr(self._coro,attr)
-
-            def __str__(self):
-                return '<tasklet %s a:%s>' % (self._task_id, self.is_alive)
-
-            def __reduce__(self):
-                return getmain, ()
-
-            __repr__ = __str__
-
-
-        global _main_coroutine
-        _main_coroutine = _main_tasklet
-        _main_tasklet = TaskletProxy(_main_tasklet)
-        assert _main_tasklet.is_alive and not _main_tasklet.is_zombie
+    _main_tasklet.__class__ = tasklet         # XXX HAAAAAAAAAAAAAAAAAAAAACK
     _last_task = _main_tasklet
     tasklet._init.im_func(_main_tasklet, label='main')
     _squeue = deque()
diff --git a/py/_code/source.py b/py/_code/source.py
--- a/py/_code/source.py
+++ b/py/_code/source.py
@@ -139,7 +139,7 @@
                 trysource = self[start:end]
                 if trysource.isparseable():
                     return start, end
-        return start, end
+        return start, len(self)
 
     def getblockend(self, lineno):
         # XXX
diff --git a/pypy/annotation/annrpython.py b/pypy/annotation/annrpython.py
--- a/pypy/annotation/annrpython.py
+++ b/pypy/annotation/annrpython.py
@@ -149,7 +149,7 @@
         desc = olddesc.bind_self(classdef)
         args = self.bookkeeper.build_args("simple_call", args_s[:])
         desc.consider_call_site(self.bookkeeper, desc.getcallfamily(), [desc],
-            args, annmodel.s_ImpossibleValue)
+            args, annmodel.s_ImpossibleValue, None)
         result = []
         def schedule(graph, inputcells):
             result.append((graph, inputcells))
diff --git a/pypy/annotation/bookkeeper.py b/pypy/annotation/bookkeeper.py
--- a/pypy/annotation/bookkeeper.py
+++ b/pypy/annotation/bookkeeper.py
@@ -209,8 +209,8 @@
                 self.consider_call_site(call_op)
 
             for pbc, args_s in self.emulated_pbc_calls.itervalues():
-                self.consider_call_site_for_pbc(pbc, 'simple_call', 
-                                                args_s, s_ImpossibleValue)
+                self.consider_call_site_for_pbc(pbc, 'simple_call',
+                                                args_s, s_ImpossibleValue, None)
             self.emulated_pbc_calls = {}
         finally:
             self.leave()
@@ -257,18 +257,18 @@
             args_s = [lltype_to_annotation(adtmeth.ll_ptrtype)] + args_s
         if isinstance(s_callable, SomePBC):
             s_result = binding(call_op.result, s_ImpossibleValue)
-            self.consider_call_site_for_pbc(s_callable,
-                                            call_op.opname,
-                                            args_s, s_result)
+            self.consider_call_site_for_pbc(s_callable, call_op.opname, args_s,
+                                            s_result, call_op)
 
-    def consider_call_site_for_pbc(self, s_callable, opname, args_s, s_result):
+    def consider_call_site_for_pbc(self, s_callable, opname, args_s, s_result,
+                                   call_op):
         descs = list(s_callable.descriptions)
         if not descs:
             return
         family = descs[0].getcallfamily()
         args = self.build_args(opname, args_s)
         s_callable.getKind().consider_call_site(self, family, descs, args,
-                                                s_result)
+                                                s_result, call_op)
 
     def getuniqueclassdef(self, cls):
         """Get the ClassDef associated with the given user cls.
@@ -656,6 +656,7 @@
                 whence = None
             else:
                 whence = emulated # callback case
+            op = None
             s_previous_result = s_ImpossibleValue
 
         def schedule(graph, inputcells):
@@ -663,7 +664,7 @@
 
         results = []
         for desc in descs:
-            results.append(desc.pycall(schedule, args, s_previous_result))
+            results.append(desc.pycall(schedule, args, s_previous_result, op))
         s_result = unionof(*results)
         return s_result
 
diff --git a/pypy/annotation/description.py b/pypy/annotation/description.py
--- a/pypy/annotation/description.py
+++ b/pypy/annotation/description.py
@@ -255,7 +255,11 @@
             raise TypeError, "signature mismatch: %s" % e.getmsg(self.name)
         return inputcells
 
-    def specialize(self, inputcells):
+    def specialize(self, inputcells, op=None):
+        if (op is None and
+            getattr(self.bookkeeper, "position_key", None) is not None):
+            _, block, i = self.bookkeeper.position_key
+            op = block.operations[i]
         if self.specializer is None:
             # get the specializer based on the tag of the 'pyobj'
             # (if any), according to the current policy
@@ -269,11 +273,14 @@
                 enforceargs = Sig(*enforceargs)
                 self.pyobj._annenforceargs_ = enforceargs
             enforceargs(self, inputcells) # can modify inputcells in-place
-        return self.specializer(self, inputcells)
+        if getattr(self.pyobj, '_annspecialcase_', '').endswith("call_location"):
+            return self.specializer(self, inputcells, op)
+        else:
+            return self.specializer(self, inputcells)
 
-    def pycall(self, schedule, args, s_previous_result):
+    def pycall(self, schedule, args, s_previous_result, op=None):
         inputcells = self.parse_arguments(args)
-        result = self.specialize(inputcells)
+        result = self.specialize(inputcells, op)
         if isinstance(result, FunctionGraph):
             graph = result         # common case
             # if that graph has a different signature, we need to re-parse
@@ -296,17 +303,17 @@
                                              None,       # selfclassdef
                                              name)
 
-    def consider_call_site(bookkeeper, family, descs, args, s_result):
+    def consider_call_site(bookkeeper, family, descs, args, s_result, op):
         shape = rawshape(args)
-        row = FunctionDesc.row_to_consider(descs, args)
+        row = FunctionDesc.row_to_consider(descs, args, op)
         family.calltable_add_row(shape, row)
     consider_call_site = staticmethod(consider_call_site)
 
-    def variant_for_call_site(bookkeeper, family, descs, args):
+    def variant_for_call_site(bookkeeper, family, descs, args, op):
         shape = rawshape(args)
         bookkeeper.enter(None)
         try:
-            row = FunctionDesc.row_to_consider(descs, args)
+            row = FunctionDesc.row_to_consider(descs, args, op)
         finally:
             bookkeeper.leave()
         index = family.calltable_lookup_row(shape, row)
@@ -316,7 +323,7 @@
     def rowkey(self):
         return self
 
-    def row_to_consider(descs, args):
+    def row_to_consider(descs, args, op):
         # see comments in CallFamily
         from pypy.annotation.model import s_ImpossibleValue
         row = {}
@@ -324,7 +331,7 @@
             def enlist(graph, ignore):
                 row[desc.rowkey()] = graph
                 return s_ImpossibleValue   # meaningless
-            desc.pycall(enlist, args, s_ImpossibleValue)
+            desc.pycall(enlist, args, s_ImpossibleValue, op)
         return row
     row_to_consider = staticmethod(row_to_consider)
 
@@ -521,7 +528,7 @@
                             "specialization" % (self.name,))
         return self.getclassdef(None)
 
-    def pycall(self, schedule, args, s_previous_result):
+    def pycall(self, schedule, args, s_previous_result, op=None):
         from pypy.annotation.model import SomeInstance, SomeImpossibleValue
         if self.specialize:
             if self.specialize == 'specialize:ctr_location':
@@ -664,7 +671,7 @@
             cdesc = cdesc.basedesc
         return s_result     # common case
 
-    def consider_call_site(bookkeeper, family, descs, args, s_result):
+    def consider_call_site(bookkeeper, family, descs, args, s_result, op):
         from pypy.annotation.model import SomeInstance, SomePBC, s_None
         if len(descs) == 1:
             # call to a single class, look at the result annotation
@@ -709,7 +716,7 @@
             initdescs[0].mergecallfamilies(*initdescs[1:])
             initfamily = initdescs[0].getcallfamily()
             MethodDesc.consider_call_site(bookkeeper, initfamily, initdescs,
-                                          args, s_None)
+                                          args, s_None, op)
     consider_call_site = staticmethod(consider_call_site)
 
     def getallbases(self):
@@ -782,13 +789,13 @@
     def getuniquegraph(self):
         return self.funcdesc.getuniquegraph()
 
-    def pycall(self, schedule, args, s_previous_result):
+    def pycall(self, schedule, args, s_previous_result, op=None):
         from pypy.annotation.model import SomeInstance
         if self.selfclassdef is None:
             raise Exception("calling %r" % (self,))
         s_instance = SomeInstance(self.selfclassdef, flags = self.flags)
         args = args.prepend(s_instance)
-        return self.funcdesc.pycall(schedule, args, s_previous_result)
+        return self.funcdesc.pycall(schedule, args, s_previous_result, op)
 
     def bind_under(self, classdef, name):
         self.bookkeeper.warning("rebinding an already bound %r" % (self,))
@@ -801,10 +808,10 @@
                                              self.name,
                                              flags)
 
-    def consider_call_site(bookkeeper, family, descs, args, s_result):
+    def consider_call_site(bookkeeper, family, descs, args, s_result, op):
         shape = rawshape(args, nextra=1)     # account for the extra 'self'
         funcdescs = [methoddesc.funcdesc for methoddesc in descs]
-        row = FunctionDesc.row_to_consider(descs, args)
+        row = FunctionDesc.row_to_consider(descs, args, op)
         family.calltable_add_row(shape, row)
     consider_call_site = staticmethod(consider_call_site)
 
@@ -956,16 +963,16 @@
         return '<MethodOfFrozenDesc %r of %r>' % (self.funcdesc,
                                                   self.frozendesc)
 
-    def pycall(self, schedule, args, s_previous_result):
+    def pycall(self, schedule, args, s_previous_result, op=None):
         from pypy.annotation.model import SomePBC
         s_self = SomePBC([self.frozendesc])
         args = args.prepend(s_self)
-        return self.funcdesc.pycall(schedule, args, s_previous_result)
+        return self.funcdesc.pycall(schedule, args, s_previous_result, op)
 
-    def consider_call_site(bookkeeper, family, descs, args, s_result):
+    def consider_call_site(bookkeeper, family, descs, args, s_result, op):
         shape = rawshape(args, nextra=1)    # account for the extra 'self'
         funcdescs = [mofdesc.funcdesc for mofdesc in descs]
-        row = FunctionDesc.row_to_consider(descs, args)
+        row = FunctionDesc.row_to_consider(descs, args, op)
         family.calltable_add_row(shape, row)
     consider_call_site = staticmethod(consider_call_site)
 
diff --git a/pypy/annotation/policy.py b/pypy/annotation/policy.py
--- a/pypy/annotation/policy.py
+++ b/pypy/annotation/policy.py
@@ -1,7 +1,7 @@
 # base annotation policy for specialization
 from pypy.annotation.specialize import default_specialize as default
-from pypy.annotation.specialize import specialize_argvalue, specialize_argtype, specialize_arglistitemtype
-from pypy.annotation.specialize import memo
+from pypy.annotation.specialize import specialize_argvalue, specialize_argtype, specialize_arglistitemtype, specialize_arg_or_var
+from pypy.annotation.specialize import memo, specialize_call_location
 # for some reason, model must be imported first,
 # or we create a cycle.
 from pypy.annotation import model as annmodel
@@ -73,8 +73,10 @@
     default_specialize = staticmethod(default)
     specialize__memo = staticmethod(memo)
     specialize__arg = staticmethod(specialize_argvalue) # specialize:arg(N)
+    specialize__arg_or_var = staticmethod(specialize_arg_or_var)
     specialize__argtype = staticmethod(specialize_argtype) # specialize:argtype(N)
     specialize__arglistitemtype = staticmethod(specialize_arglistitemtype)
+    specialize__call_location = staticmethod(specialize_call_location)
 
     def specialize__ll(pol, *args):
         from pypy.rpython.annlowlevel import LowLevelAnnotatorPolicy
diff --git a/pypy/annotation/specialize.py b/pypy/annotation/specialize.py
--- a/pypy/annotation/specialize.py
+++ b/pypy/annotation/specialize.py
@@ -353,6 +353,16 @@
     key = tuple(key)
     return maybe_star_args(funcdesc, key, args_s)
 
+def specialize_arg_or_var(funcdesc, args_s, *argindices):
+    for argno in argindices:
+        if not args_s[argno].is_constant():
+            break
+    else:
+        # all constant
+        return specialize_argvalue(funcdesc, args_s, *argindices)
+    # some not constant
+    return maybe_star_args(funcdesc, None, args_s)
+
 def specialize_argtype(funcdesc, args_s, *argindices):
     key = tuple([args_s[i].knowntype for i in argindices])
     for cls in key:
@@ -370,3 +380,7 @@
     else:
         key = s.listdef.listitem.s_value.knowntype
     return maybe_star_args(funcdesc, key, args_s)
+
+def specialize_call_location(funcdesc, args_s, op):
+    assert op is not None
+    return maybe_star_args(funcdesc, op, args_s)
diff --git a/pypy/annotation/test/test_annrpython.py b/pypy/annotation/test/test_annrpython.py
--- a/pypy/annotation/test/test_annrpython.py
+++ b/pypy/annotation/test/test_annrpython.py
@@ -1099,8 +1099,8 @@
         allocdesc = a.bookkeeper.getdesc(alloc)
         s_C1 = a.bookkeeper.immutablevalue(C1)
         s_C2 = a.bookkeeper.immutablevalue(C2)
-        graph1 = allocdesc.specialize([s_C1])
-        graph2 = allocdesc.specialize([s_C2])
+        graph1 = allocdesc.specialize([s_C1], None)
+        graph2 = allocdesc.specialize([s_C2], None)
         assert a.binding(graph1.getreturnvar()).classdef == C1df
         assert a.binding(graph2.getreturnvar()).classdef == C2df
         assert graph1 in a.translator.graphs
@@ -1135,8 +1135,8 @@
         allocdesc = a.bookkeeper.getdesc(alloc)
         s_C1 = a.bookkeeper.immutablevalue(C1)
         s_C2 = a.bookkeeper.immutablevalue(C2)
-        graph1 = allocdesc.specialize([s_C1, s_C2])
-        graph2 = allocdesc.specialize([s_C2, s_C2])
+        graph1 = allocdesc.specialize([s_C1, s_C2], None)
+        graph2 = allocdesc.specialize([s_C2, s_C2], None)
         assert a.binding(graph1.getreturnvar()).classdef == C1df
         assert a.binding(graph2.getreturnvar()).classdef == C2df
         assert graph1 in a.translator.graphs
@@ -1194,6 +1194,33 @@
         assert len(executedesc._cache[(0, 'star', 2)].startblock.inputargs) == 4
         assert len(executedesc._cache[(1, 'star', 3)].startblock.inputargs) == 5
 
+    def test_specialize_arg_or_var(self):
+        def f(a):
+            return 1
+        f._annspecialcase_ = 'specialize:arg_or_var(0)'
+
+        def fn(a):
+            return f(3) + f(a)
+
+        a = self.RPythonAnnotator()
+        a.build_types(fn, [int])
+        executedesc = a.bookkeeper.getdesc(f)
+        assert sorted(executedesc._cache.keys()) == [None, (3,)]
+        # we got two different special
+
+    def test_specialize_call_location(self):
+        def g(a):
+            return a
+        g._annspecialcase_ = "specialize:call_location"
+        def f(x):
+            return g(x)
+        f._annspecialcase_ = "specialize:argtype(0)"
+        def h(y):
+            w = f(y)
+            return int(f(str(y))) + w
+        a = self.RPythonAnnotator()
+        assert a.build_types(h, [int]) == annmodel.SomeInteger()
+
     def test_assert_list_doesnt_lose_info(self):
         class T(object):
             pass
@@ -3177,6 +3204,8 @@
         s = a.build_types(f, [])
         assert isinstance(s, annmodel.SomeList)
         assert not s.listdef.listitem.resized
+        assert not s.listdef.listitem.immutable
+        assert s.listdef.listitem.mutated
 
     def test_delslice(self):
         def f():
diff --git a/pypy/annotation/unaryop.py b/pypy/annotation/unaryop.py
--- a/pypy/annotation/unaryop.py
+++ b/pypy/annotation/unaryop.py
@@ -352,6 +352,7 @@
         check_negative_slice(s_start, s_stop)
         if not isinstance(s_iterable, SomeList):
             raise Exception("list[start:stop] = x: x must be a list")
+        lst.listdef.mutate()
         lst.listdef.agree(s_iterable.listdef)
         # note that setslice is not allowed to resize a list in RPython
 
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -27,7 +27,7 @@
 # --allworkingmodules
 working_modules = default_modules.copy()
 working_modules.update(dict.fromkeys(
-    ["_socket", "unicodedata", "mmap", "fcntl", "_locale",
+    ["_socket", "unicodedata", "mmap", "fcntl", "_locale", "pwd",
      "rctime" , "select", "zipimport", "_lsprof",
      "crypt", "signal", "_rawffi", "termios", "zlib", "bz2",
      "struct", "_hashlib", "_md5", "_sha", "_minimal_curses", "cStringIO",
@@ -58,6 +58,7 @@
     # unix only modules
     del working_modules["crypt"]
     del working_modules["fcntl"]
+    del working_modules["pwd"]
     del working_modules["termios"]
     del working_modules["_minimal_curses"]
 
diff --git a/pypy/config/test/test_config.py b/pypy/config/test/test_config.py
--- a/pypy/config/test/test_config.py
+++ b/pypy/config/test/test_config.py
@@ -281,11 +281,11 @@
 
 def test_underscore_in_option_name():
     descr = OptionDescription("opt", "", [
-        BoolOption("_stackless", "", default=False),
+        BoolOption("_foobar", "", default=False),
     ])
     config = Config(descr)
     parser = to_optparse(config)
-    assert parser.has_option("--_stackless")
+    assert parser.has_option("--_foobar")
 
 def test_none():
     dummy1 = BoolOption('dummy1', 'doc dummy', default=False, cmdline=None)
diff --git a/pypy/doc/config/objspace.usemodules._stackless.txt b/pypy/doc/config/objspace.usemodules._stackless.txt
deleted file mode 100644
--- a/pypy/doc/config/objspace.usemodules._stackless.txt
+++ /dev/null
@@ -1,1 +0,0 @@
-Deprecated.
diff --git a/pypy/doc/config/objspace.usemodules.pwd.txt b/pypy/doc/config/objspace.usemodules.pwd.txt
new file mode 100644
--- /dev/null
+++ b/pypy/doc/config/objspace.usemodules.pwd.txt
@@ -0,0 +1,2 @@
+Use the 'pwd' module. 
+This module is expected to be fully working.
diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst
--- a/pypy/doc/index.rst
+++ b/pypy/doc/index.rst
@@ -21,8 +21,6 @@
 
 * `Papers`_: Academic papers, talks, and related projects
 
-* `Videos`_: Videos of PyPy talks and presentations
-
 * `speed.pypy.org`_: Daily benchmarks of how fast PyPy is
 
 * `potential project ideas`_: In case you want to get your feet wet...
diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -53,6 +53,18 @@
 this is an ideal task to get started, because it does not require any deep
 knowledge of the internals.
 
+Optimized Unicode Representation
+--------------------------------
+
+CPython 3.3 will use an `optimized unicode representation`_ which switches between
+different ways to represent a unicode string, depending on whether the string
+fits into ASCII, has only two-byte characters or needs four-byte characters.
+
+The actual details would be rather differen in PyPy, but we would like to have
+the same optimization implemented.
+
+.. _`optimized unicode representation`: http://www.python.org/dev/peps/pep-0393/
+
 Translation Toolchain
 ---------------------
 
diff --git a/pypy/doc/stackless.rst b/pypy/doc/stackless.rst
--- a/pypy/doc/stackless.rst
+++ b/pypy/doc/stackless.rst
@@ -66,7 +66,7 @@
 In practice, in PyPy, you cannot change the ``f_back`` of an abitrary
 frame, but only of frames stored in ``continulets``.
 
-Continulets are internally implemented using stacklets.  Stacklets are a
+Continulets are internally implemented using stacklets_.  Stacklets are a
 bit more primitive (they are really one-shot continuations), but that
 idea only works in C, not in Python.  The basic idea of continulets is
 to have at any point in time a complete valid stack; this is important
@@ -215,11 +215,6 @@
 
 * Support for other CPUs than x86 and x86-64
 
-* The app-level ``f_back`` field of frames crossing continulet boundaries
-  is None for now, unlike what I explain in the theoretical overview
-  above.  It mostly means that in a ``pdb.set_trace()`` you cannot go
-  ``up`` past countinulet boundaries.  This could be fixed.
-
 .. __: `recursion depth limit`_
 
 (*) Pickling, as well as changing threads, could be implemented by using
@@ -285,6 +280,24 @@
 to use other interfaces like genlets and greenlets.)
 
 
+Stacklets
++++++++++
+
+Continulets are internally implemented using stacklets, which is the
+generic RPython-level building block for "one-shot continuations".  For
+more information about them please see the documentation in the C source
+at `pypy/translator/c/src/stacklet/stacklet.h`_.
+
+The module ``pypy.rlib.rstacklet`` is a thin wrapper around the above
+functions.  The key point is that new() and switch() always return a
+fresh stacklet handle (or an empty one), and switch() additionally
+consumes one.  It makes no sense to have code in which the returned
+handle is ignored, or used more than once.  Note that ``stacklet.c`` is
+written assuming that the user knows that, and so no additional checking
+occurs; this can easily lead to obscure crashes if you don't use a
+wrapper like PyPy's '_continuation' module.
+
+
 Theory of composability
 +++++++++++++++++++++++
 
diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -125,6 +125,7 @@
 
     ###  Manipulation  ###
 
+    @jit.look_inside_iff(lambda self: not self._dont_jit)
     def unpack(self): # slowish
         "Return a ([w1,w2...], {'kw':w3...}) pair."
         kwds_w = {}
@@ -245,6 +246,8 @@
 
     ###  Parsing for function calls  ###
 
+    # XXX: this should be @jit.look_inside_iff, but we need key word arguments,
+    # and it doesn't support them for now.
     def _match_signature(self, w_firstarg, scope_w, signature, defaults_w=None,
                          blindargs=0):
         """Parse args and kwargs according to the signature of a code object,
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -3,18 +3,18 @@
 from pypy.interpreter.executioncontext import ExecutionContext, ActionFlag
 from pypy.interpreter.executioncontext import UserDelAction, FrameTraceAction
 from pypy.interpreter.error import OperationError, operationerrfmt
-from pypy.interpreter.error import new_exception_class
+from pypy.interpreter.error import new_exception_class, typed_unwrap_error_msg
 from pypy.interpreter.argument import Arguments
 from pypy.interpreter.miscutils import ThreadLocals
 from pypy.tool.cache import Cache
 from pypy.tool.uid import HUGEVAL_BYTES
-from pypy.rlib.objectmodel import we_are_translated
+from pypy.rlib.objectmodel import we_are_translated, newlist, compute_unique_id
 from pypy.rlib.debug import make_sure_not_resized
 from pypy.rlib.timer import DummyTimer, Timer
 from pypy.rlib.rarithmetic import r_uint
 from pypy.rlib import jit
 from pypy.tool.sourcetools import func_with_new_name
-import os, sys, py
+import os, sys
 
 __all__ = ['ObjSpace', 'OperationError', 'Wrappable', 'W_Root']
 
@@ -186,6 +186,28 @@
     def _set_mapdict_storage_and_map(self, storage, map):
         raise NotImplementedError
 
+    # -------------------------------------------------------------------
+
+    def str_w(self, space):
+        w_msg = typed_unwrap_error_msg(space, "string", self)
+        raise OperationError(space.w_TypeError, w_msg)
+
+    def unicode_w(self, space):
+        raise OperationError(space.w_TypeError,
+                             typed_unwrap_error_msg(space, "unicode", self))
+
+    def int_w(self, space):
+        raise OperationError(space.w_TypeError,
+                             typed_unwrap_error_msg(space, "integer", self))
+    
+    def uint_w(self, space):
+        raise OperationError(space.w_TypeError,
+                             typed_unwrap_error_msg(space, "integer", self))
+    
+    def bigint_w(self, space):
+        raise OperationError(space.w_TypeError,
+                             typed_unwrap_error_msg(space, "integer", self))
+
 
 class Wrappable(W_Root):
     """A subclass of Wrappable is an internal, interpreter-level class
@@ -757,7 +779,18 @@
         w_iterator = self.iter(w_iterable)
         # If we know the expected length we can preallocate.
         if expected_length == -1:
-            items = []
+            try:
+                lgt_estimate = self.len_w(w_iterable)
+            except OperationError, o:
+                if (not o.match(self, self.w_AttributeError) and
+                    not o.match(self, self.w_TypeError)):
+                    raise
+                items = []
+            else:
+                try:
+                    items = newlist(lgt_estimate)
+                except MemoryError:
+                    items = [] # it might have lied
         else:
             items = [None] * expected_length
         idx = 0
@@ -890,7 +923,7 @@
         ec.c_call_trace(frame, w_func, args)
         try:
             w_res = self.call_args(w_func, args)
-        except OperationError, e:
+        except OperationError:
             ec.c_exception_trace(frame, w_func)
             raise
         ec.c_return_trace(frame, w_func, args)
@@ -936,6 +969,9 @@
     def isinstance_w(self, w_obj, w_type):
         return self.is_true(self.isinstance(w_obj, w_type))
 
+    def id(self, w_obj):
+        return self.wrap(compute_unique_id(w_obj))
+
     # The code below only works
     # for the simple case (new-style instance).
     # These methods are patched with the full logic by the __builtin__
@@ -988,8 +1024,6 @@
 
     def eval(self, expression, w_globals, w_locals, hidden_applevel=False):
         "NOT_RPYTHON: For internal debugging."
-        import types
-        from pypy.interpreter.pycode import PyCode
         if isinstance(expression, str):
             compiler = self.createcompiler()
             expression = compiler.compile(expression, '?', 'eval', 0,
@@ -1001,7 +1035,6 @@
     def exec_(self, statement, w_globals, w_locals, hidden_applevel=False,
               filename=None):
         "NOT_RPYTHON: For internal debugging."
-        import types
         if filename is None:
             filename = '?'
         from pypy.interpreter.pycode import PyCode
@@ -1199,6 +1232,18 @@
             return None
         return self.str_w(w_obj)
 
+    def str_w(self, w_obj):
+        return w_obj.str_w(self)
+
+    def int_w(self, w_obj):
+        return w_obj.int_w(self)
+
+    def uint_w(self, w_obj):
+        return w_obj.uint_w(self)
+
+    def bigint_w(self, w_obj):
+        return w_obj.bigint_w(self)
+
     def realstr_w(self, w_obj):
         # Like str_w, but only works if w_obj is really of type 'str'.
         if not self.is_true(self.isinstance(w_obj, self.w_str)):
@@ -1206,6 +1251,9 @@
                                  self.wrap('argument must be a string'))
         return self.str_w(w_obj)
 
+    def unicode_w(self, w_obj):
+        return w_obj.unicode_w(self)
+
     def realunicode_w(self, w_obj):
         # Like unicode_w, but only works if w_obj is really of type
         # 'unicode'.
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -458,3 +458,7 @@
     if module:
         space.setattr(w_exc, space.wrap("__module__"), space.wrap(module))
     return w_exc
+
+def typed_unwrap_error_msg(space, expected, w_obj):
+    type_name = space.type(w_obj).getname(space)
+    return space.wrap("expected %s, got %s object" % (expected, type_name))
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -1,5 +1,4 @@
 import sys
-from pypy.interpreter.miscutils import Stack
 from pypy.interpreter.error import OperationError
 from pypy.rlib.rarithmetic import LONG_BIT
 from pypy.rlib.unroll import unrolling_iterable
@@ -48,6 +47,7 @@
         return frame
 
     @staticmethod
+    @jit.unroll_safe  # should usually loop 0 times, very rarely more than once
     def getnextframe_nohidden(frame):
         frame = frame.f_backref()
         while frame and frame.hide():
@@ -81,58 +81,6 @@
 
     # ________________________________________________________________
 
-
-    class Subcontext(object):
-        # coroutine: subcontext support
-
-        def __init__(self):
-            self.topframe = None
-            self.w_tracefunc = None
-            self.profilefunc = None
-            self.w_profilefuncarg = None
-            self.is_tracing = 0
-
-        def enter(self, ec):
-            ec.topframeref = jit.non_virtual_ref(self.topframe)
-            ec.w_tracefunc = self.w_tracefunc
-            ec.profilefunc = self.profilefunc
-            ec.w_profilefuncarg = self.w_profilefuncarg
-            ec.is_tracing = self.is_tracing
-            ec.space.frame_trace_action.fire()
-
-        def leave(self, ec):
-            self.topframe = ec.gettopframe()
-            self.w_tracefunc = ec.w_tracefunc
-            self.profilefunc = ec.profilefunc
-            self.w_profilefuncarg = ec.w_profilefuncarg
-            self.is_tracing = ec.is_tracing
-
-        def clear_framestack(self):
-            self.topframe = None
-
-        # the following interface is for pickling and unpickling
-        def getstate(self, space):
-            if self.topframe is None:
-                return space.w_None
-            return self.topframe
-
-        def setstate(self, space, w_state):
-            from pypy.interpreter.pyframe import PyFrame
-            if space.is_w(w_state, space.w_None):
-                self.topframe = None
-            else:
-                self.topframe = space.interp_w(PyFrame, w_state)
-
-        def getframestack(self):
-            lst = []
-            f = self.topframe
-            while f is not None:
-                lst.append(f)
-                f = f.f_backref()
-            lst.reverse()
-            return lst
-        # coroutine: I think this is all, folks!
-
     def c_call_trace(self, frame, w_func, args=None):
         "Profile the call of a builtin function"
         self._c_call_return_trace(frame, w_func, args, 'c_call')
diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py
--- a/pypy/interpreter/function.py
+++ b/pypy/interpreter/function.py
@@ -242,8 +242,10 @@
             # we have been seen by other means so rtyping should not choke
             # on us
             identifier = self.code.identifier
-            assert Function._all.get(identifier, self) is self, ("duplicate "
-                                                                 "function ids")
+            previous = Function._all.get(identifier, self)
+            assert previous is self, (
+                "duplicate function ids with identifier=%r: %r and %r" % (
+                identifier, previous, self))
             self.add_to_table()
         return False
 
diff --git a/pypy/interpreter/miscutils.py b/pypy/interpreter/miscutils.py
--- a/pypy/interpreter/miscutils.py
+++ b/pypy/interpreter/miscutils.py
@@ -2,154 +2,6 @@
 Miscellaneous utilities.
 """
 
-import types
-
-from pypy.rlib.rarithmetic import r_uint
-
-class RootStack:
-    pass
-
-class Stack(RootStack):
-    """Utility class implementing a stack."""
-
-    _annspecialcase_ = "specialize:ctr_location" # polymorphic
-
-    def __init__(self):
-        self.items = []
-
-    def clone(self):
-        s = self.__class__()
-        for item in self.items:
-            try:
-                item = item.clone()
-            except AttributeError:
-                pass
-            s.push(item)
-        return s
-
-    def push(self, item):
-        self.items.append(item)
-
-    def pop(self):
-        return self.items.pop()
-
-    def drop(self, n):
-        if n > 0:
-            del self.items[-n:]
-
-    def top(self, position=0):
-        """'position' is 0 for the top of the stack, 1 for the item below,
-        and so on.  It must not be negative."""
-        if position < 0:
-            raise ValueError, 'negative stack position'
-        if position >= len(self.items):
-            raise IndexError, 'not enough entries in stack'
-        return self.items[~position]
-
-    def set_top(self, value, position=0):
-        """'position' is 0 for the top of the stack, 1 for the item below,
-        and so on.  It must not be negative."""
-        if position < 0:
-            raise ValueError, 'negative stack position'
-        if position >= len(self.items):
-            raise IndexError, 'not enough entries in stack'
-        self.items[~position] = value
-
-    def depth(self):
-        return len(self.items)
-
-    def empty(self):
-        return len(self.items) == 0
-
-
-class FixedStack(RootStack):
-    _annspecialcase_ = "specialize:ctr_location" # polymorphic
-
-    # unfortunately, we have to re-do everything
-    def __init__(self):
-        pass
-
-    def setup(self, stacksize):
-        self.ptr = r_uint(0) # we point after the last element
-        self.items = [None] * stacksize
-
-    def clone(self):
-        # this is only needed if we support flow space
-        s = self.__class__()
-        s.setup(len(self.items))
-        for item in self.items[:self.ptr]:
-            try:
-                item = item.clone()
-            except AttributeError:
-                pass
-            s.push(item)
-        return s
-
-    def push(self, item):
-        ptr = self.ptr
-        self.items[ptr] = item
-        self.ptr = ptr + 1
-
-    def pop(self):
-        ptr = self.ptr - 1
-        ret = self.items[ptr]   # you get OverflowError if the stack is empty
-        self.items[ptr] = None
-        self.ptr = ptr
-        return ret
-
-    def drop(self, n):
-        while n > 0:
-            n -= 1
-            self.ptr -= 1
-            self.items[self.ptr] = None
-
-    def top(self, position=0):
-        # for a fixed stack, we assume correct indices
-        return self.items[self.ptr + ~position]
-
-    def set_top(self, value, position=0):
-        # for a fixed stack, we assume correct indices
-        self.items[self.ptr + ~position] = value
-
-    def depth(self):
-        return self.ptr
-
-    def empty(self):
-        return not self.ptr
-
-
-class InitializedClass(type):
-    """NOT_RPYTHON.  A meta-class that allows a class to initialize itself (or
-    its subclasses) by calling __initclass__() as a class method."""
-    def __init__(self, name, bases, dict):
-        super(InitializedClass, self).__init__(name, bases, dict)
-        for basecls in self.__mro__:
-            raw = basecls.__dict__.get('__initclass__')
-            if isinstance(raw, types.FunctionType):
-                raw(self)   # call it as a class method
-
-
-class RwDictProxy(object):
-    """NOT_RPYTHON.  A dict-like class standing for 'cls.__dict__', to work
-    around the fact that the latter is a read-only proxy for new-style
-    classes."""
-    
-    def __init__(self, cls):
-        self.cls = cls
-
-    def __getitem__(self, attr):
-        return self.cls.__dict__[attr]
-
-    def __setitem__(self, attr, value):
-        setattr(self.cls, attr, value)
-
-    def __contains__(self, value):
-        return value in self.cls.__dict__
-
-    def items(self):
-        return self.cls.__dict__.items()
-
-
 class ThreadLocals:
     """Pseudo thread-local storage, for 'space.threadlocals'.
     This is not really thread-local at all; the intention is that the PyPy
@@ -167,3 +19,7 @@
 
     def getmainthreadvalue(self):
         return self._value
+
+    def getallvalues(self):
+        return {0: self._value}
+
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -10,7 +10,7 @@
 from pypy.interpreter.argument import Signature
 from pypy.interpreter.error import OperationError
 from pypy.interpreter.gateway import NoneNotWrapped, unwrap_spec
-from pypy.interpreter.astcompiler.consts import (CO_OPTIMIZED,
+from pypy.interpreter.astcompiler.consts import (
     CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, CO_VARKEYWORDS, CO_NESTED,
     CO_GENERATOR, CO_CONTAINSGLOBALS)
 from pypy.rlib.rarithmetic import intmask
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -614,7 +614,8 @@
         return self.get_builtin().getdict(space)
 
     def fget_f_back(self, space):
-        return self.space.wrap(self.f_backref())
+        f_back = ExecutionContext.getnextframe_nohidden(self)
+        return self.space.wrap(f_back)
 
     def fget_f_lasti(self, space):
         return self.space.wrap(self.last_instr)
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1523,10 +1523,8 @@
 
         if not isinstance(prog, codetype):
             filename = '<string>'
-            if not isinstance(prog, str):
-                if isinstance(prog, basestring):
-                    prog = str(prog)
-                elif isinstance(prog, file):
+            if not isinstance(prog, basestring):
+                if isinstance(prog, file):
                     filename = prog.name
                     prog = prog.read()
                 else:
diff --git a/pypy/interpreter/pyparser/future.py b/pypy/interpreter/pyparser/future.py
--- a/pypy/interpreter/pyparser/future.py
+++ b/pypy/interpreter/pyparser/future.py
@@ -109,25 +109,19 @@
             self.getc() == self.getc(+2)):
             self.pos += 3
             while 1: # Deal with a triple quoted docstring
-                if self.getc() == '\\':
-                    self.pos += 2
+                c = self.getc()
+                if c == '\\':
+                    self.pos += 1
+                    self._skip_next_char_from_docstring()
+                elif c != endchar:
+                    self._skip_next_char_from_docstring()
                 else:
-                    c = self.getc()
-                    if c != endchar:
-                        self.pos += 1
-                        if c == '\n':
-                            self.atbol()
-                        elif c == '\r':
-                            if self.getc() == '\n':
-                                self.pos += 1
-                                self.atbol()
-                    else:
-                        self.pos += 1
-                        if (self.getc() == endchar and
-                            self.getc(+1) == endchar):
-                            self.pos += 2
-                            self.consume_empty_line()
-                            break
+                    self.pos += 1
+                    if (self.getc() == endchar and
+                        self.getc(+1) == endchar):
+                        self.pos += 2
+                        self.consume_empty_line()
+                        break
 
         else: # Deal with a single quoted docstring
             self.pos += 1
@@ -138,17 +132,21 @@
                     self.consume_empty_line()
                     return
                 elif c == '\\':
-                    # Deal with linefeeds
-                    if self.getc() != '\r':
-                        self.pos += 1
-                    else:
-                        self.pos += 1
-                        if self.getc() == '\n':
-                            self.pos += 1
+                    self._skip_next_char_from_docstring()
                 elif c in '\r\n':
                     # Syntax error
                     return
 
+    def _skip_next_char_from_docstring(self):
+        c = self.getc()
+        self.pos += 1
+        if c == '\n':
+            self.atbol()
+        elif c == '\r':
+            if self.getc() == '\n':
+                self.pos += 1
+            self.atbol()
+
     def consume_continuation(self):
         c = self.getc()
         if c in '\n\r':
@@ -227,14 +225,16 @@
             raise DoneException
         self.consume_whitespace()
 
-    def consume_whitespace(self):
+    def consume_whitespace(self, newline_ok=False):
         while 1:
             c = self.getc()
             if c in whitespace:
                 self.pos += 1
                 continue
-            elif c == '\\':
-                self.pos += 1
+            elif c == '\\' or newline_ok:
+                slash = c == '\\'
+                if slash:
+                    self.pos += 1
                 c = self.getc()
                 if c == '\n':
                     self.pos += 1
@@ -245,8 +245,10 @@
                     if self.getc() == '\n':
                         self.pos += 1
                         self.atbol()
+                elif slash:
+                    raise DoneException
                 else:
-                    raise DoneException
+                    return
             else:
                 return
 
@@ -283,7 +285,7 @@
             return
         else:
             self.pos += 1
-            self.consume_whitespace()
+            self.consume_whitespace(paren_list)
             if paren_list and self.getc() == ')':
                 self.pos += 1
                 return # Handles trailing comma inside parenthesis
diff --git a/pypy/interpreter/pyparser/test/test_futureautomaton.py b/pypy/interpreter/pyparser/test/test_futureautomaton.py
--- a/pypy/interpreter/pyparser/test/test_futureautomaton.py
+++ b/pypy/interpreter/pyparser/test/test_futureautomaton.py
@@ -3,7 +3,7 @@
 from pypy.tool import stdlib___future__ as fut
 
 def run(s):
-    f = future.FutureAutomaton(future.futureFlags_2_5, s)
+    f = future.FutureAutomaton(future.futureFlags_2_7, s)
     try:
         f.start()
     except future.DoneException:
@@ -113,6 +113,14 @@
     assert f.lineno == 1
     assert f.col_offset == 0
 
+def test_paren_with_newline():
+    s = 'from __future__ import (division,\nabsolute_import)\n'
+    f = run(s)
+    assert f.pos == len(s)
+    assert f.flags == (fut.CO_FUTURE_DIVISION | fut.CO_FUTURE_ABSOLUTE_IMPORT)
+    assert f.lineno == 1
+    assert f.col_offset == 0
+
 def test_multiline():
     s = '"abc" #def\n  #ghi\nfrom  __future__ import (division as b, generators,)\nfrom __future__ import with_statement\n'
     f = run(s)
@@ -221,6 +229,14 @@
     assert f.lineno == 3
     assert f.col_offset == 0
 
+def test_lots_of_continuation_lines():
+    s = "\\\n\\\n\\\n\\\n\\\n\\\n\nfrom __future__ import with_statement\n"
+    f = run(s)
+    assert f.pos == len(s)
+    assert f.flags == fut.CO_FUTURE_WITH_STATEMENT
+    assert f.lineno == 8
+    assert f.col_offset == 0
+
 # This looks like a bug in cpython parser
 # and would require extensive modifications
 # to future.py in order to emulate the same behaviour
@@ -239,3 +255,19 @@
         raise AssertionError('IndentationError not raised')
     assert f.lineno == 2
     assert f.col_offset == 0
+
+def test_continuation_lines_in_docstring_single_quoted():
+    s = '"\\\n\\\n\\\n\\\n\\\n\\\n"\nfrom  __future__ import division\n'
+    f = run(s)
+    assert f.pos == len(s)
+    assert f.flags == fut.CO_FUTURE_DIVISION
+    assert f.lineno == 8
+    assert f.col_offset == 0
+
+def test_continuation_lines_in_docstring_triple_quoted():
+    s = '"""\\\n\\\n\\\n\\\n\\\n\\\n"""\nfrom  __future__ import division\n'
+    f = run(s)
+    assert f.pos == len(s)
+    assert f.flags == fut.CO_FUTURE_DIVISION
+    assert f.lineno == 8
+    assert f.col_offset == 0
diff --git a/pypy/interpreter/test/test_exec.py b/pypy/interpreter/test/test_exec.py
--- a/pypy/interpreter/test/test_exec.py
+++ b/pypy/interpreter/test/test_exec.py
@@ -219,3 +219,30 @@
             raise e
 
         assert res == 1
+
+    def test_exec_unicode(self):
+        # 's' is a string
+        s = "x = u'\xd0\xb9\xd1\x86\xd1\x83\xd0\xba\xd0\xb5\xd0\xbd'"
+        # 'u' is a unicode
+        u = s.decode('utf-8')
+        exec u
+        assert len(x) == 6
+        assert ord(x[0]) == 0x0439
+        assert ord(x[1]) == 0x0446
+        assert ord(x[2]) == 0x0443
+        assert ord(x[3]) == 0x043a
+        assert ord(x[4]) == 0x0435
+        assert ord(x[5]) == 0x043d
+
+    def test_eval_unicode(self):
+        u = "u'%s'" % unichr(0x1234)
+        v = eval(u)
+        assert v == unichr(0x1234)
+
+    def test_compile_unicode(self):
+        s = "x = u'\xd0\xb9\xd1\x86\xd1\x83\xd0\xba\xd0\xb5\xd0\xbd'"
+        u = s.decode('utf-8')
+        c = compile(u, '<input>', 'exec')
+        exec c
+        assert len(x) == 6
+        assert ord(x[0]) == 0x0439
diff --git a/pypy/interpreter/test/test_objspace.py b/pypy/interpreter/test/test_objspace.py
--- a/pypy/interpreter/test/test_objspace.py
+++ b/pypy/interpreter/test/test_objspace.py
@@ -71,6 +71,23 @@
         assert err.value.match(space, space.w_ValueError)
         err = raises(OperationError, space.unpackiterable, w_l, 5)
         assert err.value.match(space, space.w_ValueError)
+        w_a = space.appexec((), """():
+        class A(object):
+            def __iter__(self):
+                return self
+            def next(self):
+                raise StopIteration
+            def __len__(self):
+                1/0
+        return A()
+        """)
+        try:
+            space.unpackiterable(w_a)
+        except OperationError, o:
+            if not o.match(space, space.w_ZeroDivisionError):
+                raise Exception("DID NOT RAISE")
+        else:
+            raise Exception("DID NOT RAISE")
 
     def test_fixedview(self):
         space = self.space
diff --git a/pypy/interpreter/test/test_pyframe.py b/pypy/interpreter/test/test_pyframe.py
--- a/pypy/interpreter/test/test_pyframe.py
+++ b/pypy/interpreter/test/test_pyframe.py
@@ -1,4 +1,5 @@
 from pypy.tool import udir
+from pypy.conftest import option
 
 
 class AppTestPyFrame:
@@ -6,6 +7,15 @@
     def setup_class(cls):
         cls.w_udir = cls.space.wrap(str(udir.udir))
         cls.w_tempfile1 = cls.space.wrap(str(udir.udir.join('tempfile1')))
+        if not option.runappdirect:
+            w_call_further = cls.space.appexec([], """():
+                def call_further(f):
+                    return f()
+                return call_further
+            """)
+            assert not w_call_further.code.hidden_applevel
+            w_call_further.code.hidden_applevel = True       # hack
+            cls.w_call_further = w_call_further
 
     # test for the presence of the attributes, not functionality
 
@@ -107,6 +117,22 @@
         frame = f()
         assert frame.f_back.f_code.co_name == 'f'
 
+    def test_f_back_hidden(self):
+        if not hasattr(self, 'call_further'):
+            skip("not for runappdirect testing")
+        import sys
+        def f():
+            return (sys._getframe(0),
+                    sys._getframe(1),
+                    sys._getframe(0).f_back)
+        def main():
+            return self.call_further(f)
+        f0, f1, f1bis = main()
+        assert f0.f_code.co_name == 'f'
+        assert f1.f_code.co_name == 'main'
+        assert f1bis is f1
+        assert f0.f_back is f1
+
     def test_f_exc_xxx(self):
         import sys
 
diff --git a/pypy/jit/backend/llsupport/llmodel.py b/pypy/jit/backend/llsupport/llmodel.py
--- a/pypy/jit/backend/llsupport/llmodel.py
+++ b/pypy/jit/backend/llsupport/llmodel.py
@@ -496,6 +496,16 @@
         u = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), string)
         u.chars[index] = unichr(newvalue)
 
+    def bh_copystrcontent(self, src, dst, srcstart, dststart, length):
+        src = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), src)
+        dst = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), dst)
+        rstr.copy_string_contents(src, dst, srcstart, dststart, length)
+
+    def bh_copyunicodecontent(self, src, dst, srcstart, dststart, length):
+        src = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), src)
+        dst = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), dst)
+        rstr.copy_unicode_contents(src, dst, srcstart, dststart, length)
+
     def bh_call_i(self, func, calldescr, args_i, args_r, args_f):
         assert isinstance(calldescr, BaseIntCallDescr)
         if not we_are_translated():
diff --git a/pypy/jit/backend/llsupport/regalloc.py b/pypy/jit/backend/llsupport/regalloc.py
--- a/pypy/jit/backend/llsupport/regalloc.py
+++ b/pypy/jit/backend/llsupport/regalloc.py
@@ -57,11 +57,13 @@
     all_regs              = []
     no_lower_byte_regs    = []
     save_around_call_regs = []
-    
+    frame_reg             = None
+
     def __init__(self, longevity, frame_manager=None, assembler=None):
         self.free_regs = self.all_regs[:]
         self.longevity = longevity
         self.reg_bindings = {}
+        self.bindings_to_frame_reg = {}
         self.position = -1
         self.frame_manager = frame_manager
         self.assembler = assembler
@@ -218,6 +220,10 @@
         self.reg_bindings[v] = loc
         return loc
 
+    def force_allocate_frame_reg(self, v):
+        """ Allocate the new variable v in the frame register."""
+        self.bindings_to_frame_reg[v] = None
+
     def force_spill_var(self, var):
         self._sync_var(var)
         try:
@@ -236,6 +242,8 @@
         try:
             return self.reg_bindings[box]
         except KeyError:
+            if box in self.bindings_to_frame_reg:
+                return self.frame_reg
             return self.frame_manager.loc(box)
 
     def return_constant(self, v, forbidden_vars=[], selected_reg=None):
@@ -264,8 +272,9 @@
         self._check_type(v)
         if isinstance(v, Const):
             return self.return_constant(v, forbidden_vars, selected_reg)
-        
         prev_loc = self.loc(v)
+        if prev_loc is self.frame_reg and selected_reg is None:
+            return prev_loc
         loc = self.force_allocate_reg(v, forbidden_vars, selected_reg,
                                       need_lower_byte=need_lower_byte)
         if prev_loc is not loc:
diff --git a/pypy/jit/backend/model.py b/pypy/jit/backend/model.py
--- a/pypy/jit/backend/model.py
+++ b/pypy/jit/backend/model.py
@@ -78,7 +78,7 @@
         Optionally, return a ``ops_offset`` dictionary.  See the docstring of
         ``compiled_loop`` for more informations about it.
         """
-        raise NotImplementedError    
+        raise NotImplementedError
 
     def dump_loop_token(self, looptoken):
         """Print a disassembled version of looptoken to stdout"""
@@ -298,6 +298,10 @@
         raise NotImplementedError
     def bh_unicodesetitem(self, string, index, newvalue):
         raise NotImplementedError
+    def bh_copystrcontent(self, src, dst, srcstart, dststart, length):
+        raise NotImplementedError
+    def bh_copyunicodecontent(self, src, dst, srcstart, dststart, length):
+        raise NotImplementedError
 
     def force(self, force_token):
         raise NotImplementedError
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -957,6 +957,7 @@
         if (isinstance(from_loc, RegLoc) and from_loc.is_xmm) or (isinstance(to_loc, RegLoc) and to_loc.is_xmm):
             self.mc.MOVSD(to_loc, from_loc)
         else:
+            assert to_loc is not ebp
             self.mc.MOV(to_loc, from_loc)
 
     regalloc_mov = mov # legacy interface
@@ -2510,11 +2511,6 @@
 
     genop_discard_cond_call_gc_wb_array = genop_discard_cond_call_gc_wb
 
-    def genop_force_token(self, op, arglocs, resloc):
-        # RegAlloc.consider_force_token ensures this:
-        assert isinstance(resloc, RegLoc)
-        self.mc.LEA_rb(resloc.value, FORCE_INDEX_OFS)
-
     def not_implemented_op_discard(self, op, arglocs):
         not_implemented("not implemented operation: %s" % op.getopname())
 
diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py
--- a/pypy/jit/backend/x86/regalloc.py
+++ b/pypy/jit/backend/x86/regalloc.py
@@ -29,6 +29,7 @@
     all_regs = [eax, ecx, edx, ebx, esi, edi]
     no_lower_byte_regs = [esi, edi]
     save_around_call_regs = [eax, edx, ecx]
+    frame_reg = ebp
 
     REGLOC_TO_GCROOTMAP_REG_INDEX = {
         ebx: 1,
@@ -312,8 +313,11 @@
                     self.fm.frame_bindings[arg] = loc
             else:
                 if isinstance(loc, RegLoc):
-                    self.rm.reg_bindings[arg] = loc
-                    used[loc] = None
+                    if loc is ebp:
+                        self.rm.bindings_to_frame_reg[arg] = None
+                    else:
+                        self.rm.reg_bindings[arg] = loc
+                        used[loc] = None
                 else:
                     self.fm.frame_bindings[arg] = loc
         self.rm.free_regs = []
@@ -1358,8 +1362,8 @@
                                             self.assembler.datablockwrapper)
 
     def consider_force_token(self, op):
-        loc = self.rm.force_allocate_reg(op.result)
-        self.Perform(op, [], loc)
+        # the FORCE_TOKEN operation returns directly 'ebp'
+        self.rm.force_allocate_frame_reg(op.result)
 
     def not_implemented_op(self, op):
         not_implemented("not implemented operation: %s" % op.getopname())
diff --git a/pypy/jit/backend/x86/runner.py b/pypy/jit/backend/x86/runner.py
--- a/pypy/jit/backend/x86/runner.py
+++ b/pypy/jit/backend/x86/runner.py
@@ -119,7 +119,8 @@
             setitem(index, null)
 
     def get_latest_force_token(self):
-        return self.assembler.fail_ebp + FORCE_INDEX_OFS
+        # the FORCE_TOKEN operation and this helper both return 'ebp'.
+        return self.assembler.fail_ebp
 
     def execute_token(self, executable_token):
         addr = executable_token._x86_bootstrap_code
@@ -153,8 +154,9 @@
                                        flavor='raw', zero=True,
                                        immortal=True)
 
-    def force(self, addr_of_force_index):
+    def force(self, addr_of_force_token):
         TP = rffi.CArrayPtr(lltype.Signed)
+        addr_of_force_index = addr_of_force_token + FORCE_INDEX_OFS
         fail_index = rffi.cast(TP, addr_of_force_index)[0]
         assert fail_index >= 0, "already forced!"
         faildescr = self.get_fail_descr_from_number(fail_index)
@@ -164,7 +166,7 @@
         # start of "no gc operation!" block
         fail_index_2 = self.assembler.grab_frame_values(
             bytecode,
-            addr_of_force_index - FORCE_INDEX_OFS,
+            addr_of_force_token,
             self.all_null_registers)
         self.assembler.leave_jitted_hook()
         # end of "no gc operation!" block
diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -1158,6 +1158,12 @@
             return SpaceOperation('%s_assert_green' % kind, args, None)
         elif oopspec_name == 'jit.current_trace_length':
             return SpaceOperation('current_trace_length', [], op.result)
+        elif oopspec_name == 'jit.isconstant':
+            kind = getkind(args[0].concretetype)
+            return SpaceOperation('%s_isconstant' % kind, args, op.result)
+        elif oopspec_name == 'jit.isvirtual':
+            kind = getkind(args[0].concretetype)
+            return SpaceOperation('%s_isvirtual' % kind, args, op.result)
         else:
             raise AssertionError("missing support for %r" % oopspec_name)
 
@@ -1415,6 +1421,14 @@
         else:
             assert 0, "args[0].concretetype must be STR or UNICODE"
         #
+        if oopspec_name == 'stroruni.copy_contents':
+            if SoU.TO == rstr.STR:
+                new_op = 'copystrcontent'
+            elif SoU.TO == rstr.UNICODE:
+                new_op = 'copyunicodecontent'
+            else:
+                assert 0
+            return SpaceOperation(new_op, args, op.result)
         if oopspec_name == "stroruni.equal":
             for otherindex, othername, argtypes, resulttype in [
                 (EffectInfo.OS_STREQ_SLICE_CHECKNULL,
diff --git a/pypy/jit/metainterp/blackhole.py b/pypy/jit/metainterp/blackhole.py
--- a/pypy/jit/metainterp/blackhole.py
+++ b/pypy/jit/metainterp/blackhole.py
@@ -835,6 +835,18 @@
     def bhimpl_current_trace_length():
         return -1
 
+    @arguments("i", returns="i")
+    def bhimpl_int_isconstant(x):
+        return False
+
+    @arguments("r", returns="i")
+    def bhimpl_ref_isconstant(x):
+        return False
+
+    @arguments("r", returns="i")
+    def bhimpl_ref_isvirtual(x):
+        return False
+
     # ----------
     # the main hints and recursive calls
 
@@ -1224,6 +1236,9 @@
     @arguments("cpu", "r", "i", "i")
     def bhimpl_strsetitem(cpu, string, index, newchr):
         cpu.bh_strsetitem(string, index, newchr)
+    @arguments("cpu", "r", "r", "i", "i", "i")
+    def bhimpl_copystrcontent(cpu, src, dst, srcstart, dststart, length):
+        cpu.bh_copystrcontent(src, dst, srcstart, dststart, length)
 
     @arguments("cpu", "i", returns="r")
     def bhimpl_newunicode(cpu, length):
@@ -1237,6 +1252,9 @@
     @arguments("cpu", "r", "i", "i")
     def bhimpl_unicodesetitem(cpu, unicode, index, newchr):
         cpu.bh_unicodesetitem(unicode, index, newchr)
+    @arguments("cpu", "r", "r", "i", "i", "i")
+    def bhimpl_copyunicodecontent(cpu, src, dst, srcstart, dststart, length):
+        cpu.bh_copyunicodecontent(src, dst, srcstart, dststart, length)
 
     @arguments(returns=(longlong.is_64_bit and "i" or "f"))
     def bhimpl_ll_read_timestamp():
@@ -1441,7 +1459,7 @@
 def resume_in_blackhole(metainterp_sd, jitdriver_sd, resumedescr,
                         all_virtuals=None):
     from pypy.jit.metainterp.resume import blackhole_from_resumedata
-    debug_start('jit-blackhole')
+    #debug_start('jit-blackhole')
     metainterp_sd.profiler.start_blackhole()
     blackholeinterp = blackhole_from_resumedata(
         metainterp_sd.blackholeinterpbuilder,
@@ -1460,12 +1478,12 @@
         _run_forever(blackholeinterp, current_exc)
     finally:
         metainterp_sd.profiler.end_blackhole()
-        debug_stop('jit-blackhole')
+        #debug_stop('jit-blackhole')
 
 def convert_and_run_from_pyjitpl(metainterp, raising_exception=False):
     # Get a chain of blackhole interpreters and fill them by copying
     # 'metainterp.framestack'.
-    debug_start('jit-blackhole')
+    #debug_start('jit-blackhole')
     metainterp_sd = metainterp.staticdata
     metainterp_sd.profiler.start_blackhole()
     nextbh = None
@@ -1488,4 +1506,4 @@
         _run_forever(firstbh, current_exc)
     finally:
         metainterp_sd.profiler.end_blackhole()
-        debug_stop('jit-blackhole')
+        #debug_stop('jit-blackhole')
diff --git a/pypy/jit/metainterp/heapcache.py b/pypy/jit/metainterp/heapcache.py
new file mode 100644
--- /dev/null
+++ b/pypy/jit/metainterp/heapcache.py
@@ -0,0 +1,210 @@
+from pypy.jit.metainterp.history import ConstInt
+from pypy.jit.metainterp.resoperation import rop
+
+
+class HeapCache(object):
+    def __init__(self):
+        self.reset()
+
+    def reset(self):
+        # contains boxes where the class is already known
+        self.known_class_boxes = {}
+        # store the boxes that contain newly allocated objects, this maps the
+        # boxes to a bool, the bool indicates whether or not the object has
+        # escaped the trace or not (True means the box never escaped, False
+        # means it did escape), its presences in the mapping shows that it was
+        # allocated inside the trace
+        self.new_boxes = {}
+        # Tracks which boxes should be marked as escaped when the key box
+        # escapes.
+        self.dependencies = {}
+        # contains frame boxes that are not virtualizables
+        self.nonstandard_virtualizables = {}
+        # heap cache
+        # maps descrs to {from_box, to_box} dicts
+        self.heap_cache = {}
+        # heap array cache
+        # maps descrs to {index: {from_box: to_box}} dicts
+        self.heap_array_cache = {}
+        # cache the length of arrays
+        self.length_cache = {}
+
+    def invalidate_caches(self, opnum, descr, argboxes):
+        self.mark_escaped(opnum, argboxes)
+        self.clear_caches(opnum, descr, argboxes)
+
+    def mark_escaped(self, opnum, argboxes):
+        idx = 0
+        if opnum == rop.SETFIELD_GC:
+            assert len(argboxes) == 2
+            box, valuebox = argboxes
+            if self.is_unescaped(box) and self.is_unescaped(valuebox):
+                self.dependencies.setdefault(box, []).append(valuebox)
+            else:
+                self._escape(valuebox)
+        # GETFIELD_GC doesn't escape it's argument
+        elif opnum != rop.GETFIELD_GC:
+            for box in argboxes:
+                # setarrayitem_gc don't escape its first argument
+                if not (idx == 0 and opnum in [rop.SETARRAYITEM_GC]):
+                    self._escape(box)
+                idx += 1
+
+    def _escape(self, box):
+        if box in self.new_boxes:
+            self.new_boxes[box] = False
+        if box in self.dependencies:
+            for dep in self.dependencies[box]:
+                self._escape(dep)
+            del self.dependencies[box]
+
+    def clear_caches(self, opnum, descr, argboxes):
+        if opnum == rop.SETFIELD_GC:
+            return
+        if opnum == rop.SETARRAYITEM_GC:
+            return
+        if opnum == rop.SETFIELD_RAW:
+            return
+        if opnum == rop.SETARRAYITEM_RAW:
+            return
+        if rop._OVF_FIRST <= opnum <= rop._OVF_LAST:
+            return
+        if rop._NOSIDEEFFECT_FIRST <= opnum <= rop._NOSIDEEFFECT_LAST:
+            return
+        if opnum == rop.CALL or opnum == rop.CALL_LOOPINVARIANT:
+            effectinfo = descr.get_extra_info()
+            ef = effectinfo.extraeffect
+            if ef == effectinfo.EF_LOOPINVARIANT or \
+               ef == effectinfo.EF_ELIDABLE_CANNOT_RAISE or \
+               ef == effectinfo.EF_ELIDABLE_CAN_RAISE:
+                return
+            # A special case for ll_arraycopy, because it is so common, and its
+            # effects are so well defined.
+            elif effectinfo.oopspecindex == effectinfo.OS_ARRAYCOPY:
+                # The destination box
+                if argboxes[2] in self.new_boxes:
+                    # XXX: no descr here so we invalidate any of them, not just
+                    # of the correct type
+                    # XXX: in theory the indices of the copy could be looked at
+                    # as well
+                    for descr, cache in self.heap_array_cache.iteritems():
+                        for idx, cache in cache.iteritems():
+                            for frombox in cache.keys():
+                                if frombox not in self.new_boxes:
+                                    del cache[frombox]
+                    return
+
+        self.heap_cache.clear()
+        self.heap_array_cache.clear()
+
+    def is_class_known(self, box):
+        return box in self.known_class_boxes
+
+    def class_now_known(self, box):
+        self.known_class_boxes[box] = None
+
+    def is_nonstandard_virtualizable(self, box):
+        return box in self.nonstandard_virtualizables
+
+    def nonstandard_virtualizables_now_known(self, box):
+        self.nonstandard_virtualizables[box] = None
+
+    def is_unescaped(self, box):
+        return self.new_boxes.get(box, False)
+
+    def new(self, box):
+        self.new_boxes[box] = True
+
+    def new_array(self, box, lengthbox):
+        self.new(box)
+        self.arraylen_now_known(box, lengthbox)
+
+    def getfield(self, box, descr):
+        d = self.heap_cache.get(descr, None)
+        if d:
+            tobox = d.get(box, None)
+            if tobox:
+                return tobox
+        return None
+
+    def getfield_now_known(self, box, descr, fieldbox):
+        self.heap_cache.setdefault(descr, {})[box] = fieldbox
+
+    def setfield(self, box, descr, fieldbox):
+        d = self.heap_cache.get(descr, None)
+        new_d = self._do_write_with_aliasing(d, box, fieldbox)
+        self.heap_cache[descr] = new_d
+
+    def _do_write_with_aliasing(self, d, box, fieldbox):
+        # slightly subtle logic here
+        # a write to an arbitrary box, all other boxes can alias this one
+        if not d or box not in self.new_boxes:
+            # therefore we throw away the cache
+            return {box: fieldbox}
+        # the object we are writing to is freshly allocated
+        # only remove some boxes from the cache
+        new_d = {}
+        for frombox, tobox in d.iteritems():
+            # the other box is *also* freshly allocated
+            # therefore frombox and box *must* contain different objects
+            # thus we can keep it in the cache
+            if frombox in self.new_boxes:
+                new_d[frombox] = tobox
+        new_d[box] = fieldbox
+        return new_d
+
+    def getarrayitem(self, box, descr, indexbox):
+        if not isinstance(indexbox, ConstInt):
+            return
+        index = indexbox.getint()
+        cache = self.heap_array_cache.get(descr, None)
+        if cache:
+            indexcache = cache.get(index, None)
+            if indexcache is not None:
+                return indexcache.get(box, None)
+
+    def getarrayitem_now_known(self, box, descr, indexbox, valuebox):
+        if not isinstance(indexbox, ConstInt):
+            return
+        index = indexbox.getint()
+        cache = self.heap_array_cache.setdefault(descr, {})
+        indexcache = cache.get(index, None)
+        if indexcache is not None:
+            indexcache[box] = valuebox
+        else:
+            cache[index] = {box: valuebox}
+
+    def setarrayitem(self, box, descr, indexbox, valuebox):
+        if not isinstance(indexbox, ConstInt):
+            cache = self.heap_array_cache.get(descr, None)
+            if cache is not None:
+                cache.clear()
+            return
+        index = indexbox.getint()
+        cache = self.heap_array_cache.setdefault(descr, {})
+        indexcache = cache.get(index, None)
+        cache[index] = self._do_write_with_aliasing(indexcache, box, valuebox)
+
+    def arraylen(self, box):
+        return self.length_cache.get(box, None)
+
+    def arraylen_now_known(self, box, lengthbox):
+        self.length_cache[box] = lengthbox
+
+    def _replace_box(self, d, oldbox, newbox):
+        new_d = {}
+        for frombox, tobox in d.iteritems():
+            if frombox is oldbox:
+                frombox = newbox
+            if tobox is oldbox:
+                tobox = newbox
+            new_d[frombox] = tobox
+        return new_d
+
+    def replace_box(self, oldbox, newbox):
+        for descr, d in self.heap_cache.iteritems():
+            self.heap_cache[descr] = self._replace_box(d, oldbox, newbox)
+        for descr, d in self.heap_array_cache.iteritems():
+            for index, cache in d.iteritems():
+                d[index] = self._replace_box(cache, oldbox, newbox)
+        self.length_cache = self._replace_box(self.length_cache, oldbox, newbox)
diff --git a/pypy/jit/metainterp/optimizeopt/heap.py b/pypy/jit/metainterp/optimizeopt/heap.py
--- a/pypy/jit/metainterp/optimizeopt/heap.py
+++ b/pypy/jit/metainterp/optimizeopt/heap.py
@@ -25,7 +25,7 @@
         #      'cached_fields'.
         #
         self._cached_fields = {}
-        self._cached_fields_getfield_op = {}        
+        self._cached_fields_getfield_op = {}
         self._lazy_setfield = None
         self._lazy_setfield_registered = False
 
@@ -37,6 +37,12 @@
             self.force_lazy_setfield(optheap)
             assert not self.possible_aliasing(optheap, structvalue)
         cached_fieldvalue = self._cached_fields.get(structvalue, None)
+
+        # Hack to ensure constants are imported from the preamble
+        if cached_fieldvalue and fieldvalue.is_constant(): 
+            optheap.optimizer.ensure_imported(cached_fieldvalue)
+            cached_fieldvalue = self._cached_fields.get(structvalue, None)
+
         if cached_fieldvalue is not fieldvalue:
             # common case: store the 'op' as lazy_setfield, and register
             # myself in the optheap's _lazy_setfields_and_arrayitems list
@@ -75,7 +81,7 @@
     def remember_field_value(self, structvalue, fieldvalue, getfield_op=None):
         assert self._lazy_setfield is None
         self._cached_fields[structvalue] = fieldvalue
-        self._cached_fields_getfield_op[structvalue] = getfield_op        
+        self._cached_fields_getfield_op[structvalue] = getfield_op
 
     def force_lazy_setfield(self, optheap, can_cache=True):
         op = self._lazy_setfield
@@ -132,9 +138,7 @@
                         result = newresult
                     getop = ResOperation(rop.GETFIELD_GC, [op.getarg(0)],
                                          result, op.getdescr())
-                    getop = shortboxes.add_potential(getop)
-                    self._cached_fields_getfield_op[structvalue] = getop
-                    self._cached_fields[structvalue] = optimizer.getvalue(result)
+                    shortboxes.add_potential(getop, synthetic=True)
                 elif op.result is not None:
                     shortboxes.add_potential(op)
 
@@ -163,7 +167,7 @@
 
     def new(self):
         return OptHeap()
-        
+
     def produce_potential_short_preamble_ops(self, sb):
         descrkeys = self.cached_fields.keys()
         if not we_are_translated():
diff --git a/pypy/jit/metainterp/optimizeopt/optimizer.py b/pypy/jit/metainterp/optimizeopt/optimizer.py
--- a/pypy/jit/metainterp/optimizeopt/optimizer.py
+++ b/pypy/jit/metainterp/optimizeopt/optimizer.py
@@ -10,6 +10,7 @@
 from pypy.jit.metainterp.typesystem import llhelper, oohelper
 from pypy.tool.pairtype import extendabletype
 from pypy.rlib.debug import debug_start, debug_stop, debug_print
+from pypy.rlib.objectmodel import specialize
 
 LEVEL_UNKNOWN    = '\x00'
 LEVEL_NONNULL    = '\x01'
@@ -25,6 +26,9 @@
         self.descr = descr
         self.bound = bound
 
+    def clone(self):
+        return LenBound(self.mode, self.descr, self.bound.clone())
+
 class OptValue(object):
     __metaclass__ = extendabletype
     _attrs_ = ('box', 'known_class', 'last_guard_index', 'level', 'intbound', 'lenbound')
@@ -67,7 +71,7 @@
             guards.append(op)
         elif self.level == LEVEL_KNOWNCLASS:
             op = ResOperation(rop.GUARD_NONNULL, [box], None)
-            guards.append(op)            
+            guards.append(op)
             op = ResOperation(rop.GUARD_CLASS, [box, self.known_class], None)
             guards.append(op)
         else:
@@ -88,8 +92,27 @@
                     assert False
                 guards.append(op)
                 self.lenbound.bound.make_guards(lenbox, guards)
+        return guards
 
-        return guards
+    def import_from(self, other, optimizer):
+        assert self.level <= LEVEL_NONNULL
+        if other.level == LEVEL_CONSTANT:
+            self.make_constant(other.get_key_box())
+            optimizer.turned_constant(self)
+        elif other.level == LEVEL_KNOWNCLASS:
+            self.make_constant_class(other.known_class, -1)
+        else:
+            if other.level == LEVEL_NONNULL:
+                self.ensure_nonnull()
+            self.intbound.intersect(other.intbound)
+            if other.lenbound:
+                if self.lenbound:
+                    assert other.lenbound.mode == self.lenbound.mode
+                    assert other.lenbound.descr == self.lenbound.descr
+                    self.lenbound.bound.intersect(other.lenbound.bound)
+                else:
+                    self.lenbound = other.lenbound.clone()
+
 
     def force_box(self):
         return self.box
@@ -123,7 +146,7 @@
         assert isinstance(constbox, Const)
         self.box = constbox
         self.level = LEVEL_CONSTANT
-        
+
         if isinstance(constbox, ConstInt):
             val = constbox.getint()
             self.intbound = IntBound(val, val)
@@ -200,6 +223,9 @@
     def __init__(self, box):
         self.make_constant(box)
 
+    def __repr__(self):
+        return 'Constant(%r)' % (self.box,)
+
 CONST_0      = ConstInt(0)
 CONST_1      = ConstInt(1)
 CVAL_ZERO    = ConstantValue(CONST_0)
@@ -308,7 +334,6 @@
         self.resumedata_memo = resume.ResumeDataLoopMemo(metainterp_sd)
         self.bool_boxes = {}
         self.pure_operations = args_dict()
-        self.emitted_pure_operations = {}
         self.producer = {}
         self.pendingfields = []
         self.posponedop = None
@@ -316,12 +341,11 @@
         self.quasi_immutable_deps = None
         self.opaque_pointers = {}
         self.newoperations = []
-        self.emitting_dissabled = False
-        self.emitted_guards = 0        
         if loop is not None:
             self.call_pure_results = loop.call_pure_results
 
         self.set_optimizations(optimizations)
+        self.setup()
 
     def set_optimizations(self, optimizations):
         if optimizations:
@@ -348,23 +372,18 @@
         assert self.posponedop is None
 
     def new(self):
+        new = Optimizer(self.metainterp_sd, self.loop)
+        return self._new(new)
+
+    def _new(self, new):
         assert self.posponedop is None
-        new = Optimizer(self.metainterp_sd, self.loop)
         optimizations = [o.new() for o in self.optimizations]
         new.set_optimizations(optimizations)
         new.quasi_immutable_deps = self.quasi_immutable_deps
         return new
-        
+
     def produce_potential_short_preamble_ops(self, sb):
-        for op in self.emitted_pure_operations:
-            if op.getopnum() == rop.GETARRAYITEM_GC_PURE or \
-               op.getopnum() == rop.STRGETITEM or \
-               op.getopnum() == rop.UNICODEGETITEM:
-                if not self.getvalue(op.getarg(1)).is_constant():
-                    continue
-            sb.add_potential(op)
-        for opt in self.optimizations:
-            opt.produce_potential_short_preamble_ops(sb)
+        raise NotImplementedError('This is implemented in unroll.UnrollableOptimizer')
 
     def turned_constant(self, value):
         for o in self.optimizations:
@@ -386,19 +405,26 @@
         else:
             return box
 
+    @specialize.argtype(0)
     def getvalue(self, box):
         box = self.getinterned(box)
         try:
             value = self.values[box]
         except KeyError:
             value = self.values[box] = OptValue(box)
+        self.ensure_imported(value)
         return value
 
+    def ensure_imported(self, value):
+        pass
+
+    @specialize.argtype(0)
     def get_constant_box(self, box):
         if isinstance(box, Const):
             return box
         try:
             value = self.values[box]
+            self.ensure_imported(value)
         except KeyError:
             return None
         if value.is_constant():
@@ -481,18 +507,22 @@
     def emit_operation(self, op):
         if op.returns_bool_result():
             self.bool_boxes[self.getvalue(op.result)] = None
-        if self.emitting_dissabled:
-            return
-        
+        self._emit_operation(op)
+
+    @specialize.argtype(0)
+    def _emit_operation(self, op):
         for i in range(op.numargs()):
             arg = op.getarg(i)
-            if arg in self.values:
-                box = self.values[arg].force_box()
-                op.setarg(i, box)
+            try:
+                value = self.values[arg]
+            except KeyError:
+                pass
+            else:
+                self.ensure_imported(value)
+                op.setarg(i, value.force_box())
         self.metainterp_sd.profiler.count(jitprof.OPT_OPS)
         if op.is_guard():
             self.metainterp_sd.profiler.count(jitprof.OPT_GUARDS)
-            self.emitted_guards += 1 # FIXME: can we reuse above counter?
             op = self.store_final_boxes_in_guard(op)
         elif op.can_raise():
             self.exception_might_have_happened = True
@@ -541,9 +571,10 @@
                 arg = value.get_key_box()
             args[i] = arg
         args[n] = ConstInt(op.getopnum())
-        args[n+1] = op.getdescr()
+        args[n + 1] = op.getdescr()
         return args
 
+    @specialize.argtype(0)
     def optimize_default(self, op):
         canfold = op.is_always_pure()
         if op.is_ovf():
@@ -579,13 +610,16 @@
                 return
             else:
                 self.pure_operations[args] = op
-                self.emitted_pure_operations[op] = True
+                self.remember_emitting_pure(op)
 
         # otherwise, the operation remains
         self.emit_operation(op)
         if nextop:
             self.emit_operation(nextop)
 
+    def remember_emitting_pure(self, op):
+        pass
+
     def constant_fold(self, op):
         argboxes = [self.get_constant_box(op.getarg(i))
                     for i in range(op.numargs())]
@@ -627,9 +661,9 @@
             arrayvalue = self.getvalue(op.getarg(0))
             arrayvalue.make_len_gt(MODE_UNICODE, op.getdescr(), indexvalue.box.getint())
         self.optimize_default(op)
-        
 
-    
+
+
 
 dispatch_opt = make_dispatcher_method(Optimizer, 'optimize_',
         default=Optimizer.optimize_default)
diff --git a/pypy/jit/metainterp/optimizeopt/rewrite.py b/pypy/jit/metainterp/optimizeopt/rewrite.py
--- a/pypy/jit/metainterp/optimizeopt/rewrite.py
+++ b/pypy/jit/metainterp/optimizeopt/rewrite.py
@@ -19,7 +19,7 @@
 
     def new(self):
         return OptRewrite()
-        
+
     def produce_potential_short_preamble_ops(self, sb):
         for op in self.loop_invariant_producer.values():
             sb.add_potential(op)
@@ -231,6 +231,17 @@
             else:
                 self.make_constant(op.result, result)
                 return
+
+        args = self.optimizer.make_args_key(op)
+        oldop = self.optimizer.pure_operations.get(args, None)
+        if oldop is not None and oldop.getdescr() is op.getdescr():
+            assert oldop.getopnum() == op.getopnum()
+            self.make_equal_to(op.result, self.getvalue(oldop.result))
+            return
+        else:
+            self.optimizer.pure_operations[args] = op
+            self.optimizer.remember_emitting_pure(op)
+
         # replace CALL_PURE with just CALL
         args = op.getarglist()
         self.emit_operation(ResOperation(rop.CALL, args, op.result,
@@ -351,7 +362,7 @@
         # expects a compile-time constant
         assert isinstance(arg, Const)
         key = make_hashable_int(arg.getint())
-        
+
         resvalue = self.loop_invariant_results.get(key, None)
         if resvalue is not None:
             self.make_equal_to(op.result, resvalue)
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -4711,6 +4711,83 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_empty_copystrunicontent(self):
+        ops = """
+        [p0, p1, i0, i2, i3]
+        i4 = int_eq(i3, 0)
+        guard_true(i4) []
+        copystrcontent(p0, p1, i0, i2, i3)
+        jump(p0, p1, i0, i2, i3)
+        """
+        expected = """
+        [p0, p1, i0, i2, i3]
+        i4 = int_eq(i3, 0)
+        guard_true(i4) []
+        jump(p0, p1, i0, i2, 0)
+        """
+        self.optimize_strunicode_loop(ops, expected)
+
+    def test_empty_copystrunicontent_virtual(self):
+        ops = """
+        [p0]
+        p1 = newstr(23)
+        copystrcontent(p0, p1, 0, 0, 0)
+        jump(p0)
+        """
+        expected = """
+        [p0]
+        jump(p0)
+        """
+        self.optimize_strunicode_loop(ops, expected)
+
+    def test_forced_virtuals_aliasing(self):
+        ops = """
+        [i0, i1]
+        p0 = new(descr=ssize)
+        p1 = new(descr=ssize)
+        escape(p0)
+        escape(p1)
+        setfield_gc(p0, i0, descr=adescr)
+        setfield_gc(p1, i1, descr=adescr)
+        i2 = getfield_gc(p0, descr=adescr)
+        jump(i2, i2)
+        """
+        expected = """
+        [i0, i1]
+        p0 = new(descr=ssize)
+        escape(p0)
+        p1 = new(descr=ssize)
+        escape(p1)
+        setfield_gc(p0, i0, descr=adescr)
+        setfield_gc(p1, i1, descr=adescr)
+        jump(i0, i0)
+        """
+        py.test.skip("not implemented")
+        # setfields on things that used to be virtual still can't alias each
+        # other
+        self.optimize_loop(ops, expected)
+
+    def test_plain_virtual_string_copy_content(self):
+        ops = """
+        []
+        p0 = newstr(6)
+        copystrcontent(s"hello!", p0, 0, 0, 6)
+        p1 = call(0, p0, s"abc123", descr=strconcatdescr)
+        i0 = strgetitem(p1, 0)
+        finish(i0)
+        """
+        expected = """
+        []
+        p0 = newstr(6)
+        copystrcontent(s"hello!", p0, 0, 0, 6)
+        p1 = newstr(12)
+        copystrcontent(p0, p1, 0, 0, 6)
+        copystrcontent(s"abc123", p1, 0, 6, 6)
+        i0 = strgetitem(p1, 0)
+        finish(i0)
+        """
+        self.optimize_strunicode_loop(ops, expected)
+
 
 class TestLLtype(BaseTestOptimizeBasic, LLtypeMixin):
     pass
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -102,9 +102,9 @@
             print "Short Preamble:"
             short = loop.preamble.token.short_preamble[0]
             print short.inputargs
-            print '\n'.join([str(o) for o in short.operations])        
+            print '\n'.join([str(o) for o in short.operations])
             print
-        
+
         assert expected != "crash!", "should have raised an exception"
         self.assert_equal(loop, expected)
         if expected_preamble:
@@ -113,7 +113,7 @@
         if expected_short:
             self.assert_equal(short, expected_short,
                               text_right='expected short preamble')
-            
+
         return loop
 
 class OptimizeOptTest(BaseTestWithUnroll):
@@ -472,7 +472,13 @@
         [i0]
         jump(i0)
         """
-        self.optimize_loop(ops, expected, preamble)
+        short = """
+        [i0]
+        i1 = int_is_true(i0)
+        guard_value(i1, 1) []
+        jump(i0)
+        """
+        self.optimize_loop(ops, expected, preamble, expected_short=short)
 
     def test_bound_int_is_true(self):
         ops = """
@@ -860,10 +866,10 @@
         setfield_gc(p3sub, i1, descr=valuedescr)
         setfield_gc(p1, p3sub, descr=nextdescr)
         # XXX: We get two extra operations here because the setfield
-        #      above is the result of forcing p1 and thus not 
+        #      above is the result of forcing p1 and thus not
         #      registered with the heap optimizer. I've makred tests
         #      below with VIRTUALHEAP if they suffer from this issue
-        p3sub2 = getfield_gc(p1, descr=nextdescr) 
+        p3sub2 = getfield_gc(p1, descr=nextdescr)
         guard_nonnull_class(p3sub2, ConstClass(node_vtable2)) []
         jump(i1, p1, p3sub2)
         """
@@ -1405,7 +1411,7 @@
         guard_isnull(p18) [p0, p8]
         p31 = new(descr=ssize)
         p35 = new_with_vtable(ConstClass(node_vtable))
-        setfield_gc(p35, p31, descr=valuedescr)        
+        setfield_gc(p35, p31, descr=valuedescr)
         jump(p0, p35)
         """
         expected = """
@@ -1420,7 +1426,7 @@
         guard_isnull(p18) [p0, p8]
         p31 = new(descr=ssize)
         p35 = new_with_vtable(ConstClass(node_vtable))
-        setfield_gc(p35, p31, descr=valuedescr)        
+        setfield_gc(p35, p31, descr=valuedescr)
         jump(p0, p35, p19, p18)
         """
         expected = """
@@ -1429,7 +1435,7 @@
         jump(p0, NULL)
         """
         self.optimize_loop(ops, expected)
-        
+
     def test_varray_1(self):
         ops = """
         [i1]
@@ -2175,7 +2181,7 @@
         jump(p1)
         """
         self.optimize_loop(ops, expected)
-        
+
     def test_duplicate_getarrayitem_2(self):
         ops = """
         [p1, i0]
@@ -2193,7 +2199,7 @@
         jump(p1, i7, i6)
         """
         self.optimize_loop(ops, expected)
-        
+
     def test_duplicate_getarrayitem_after_setarrayitem_1(self):
         ops = """
         [p1, p2]
@@ -2806,14 +2812,14 @@
         guard_no_overflow() []
         i3b = int_is_true(i3)
         guard_true(i3b) []
-        setfield_gc(p1, i1, descr=valuedescr)        
+        setfield_gc(p1, i1, descr=valuedescr)
         escape(i3)
         escape(i3)
         jump(i1, p1, i3)
         """
         expected = """
         [i1, p1, i3]
-        setfield_gc(p1, i1, descr=valuedescr)        
+        setfield_gc(p1, i1, descr=valuedescr)
         escape(i3)
         escape(i3)
         jump(i1, p1, i3)
@@ -2824,7 +2830,7 @@
         ops = """
         [p8, p11, i24]
         p26 = new_with_vtable(ConstClass(node_vtable))
-        setfield_gc(p26, i24, descr=adescr)        
+        setfield_gc(p26, i24, descr=adescr)
         i34 = getfield_gc_pure(p11, descr=valuedescr)
         i35 = getfield_gc_pure(p26, descr=adescr)
         i36 = int_add_ovf(i34, i35)
@@ -2833,10 +2839,10 @@
         """
         expected = """
         [p8, p11, i26]
-        jump(p8, p11, i26)        
-        """
-        self.optimize_loop(ops, expected)
-        
+        jump(p8, p11, i26)
+        """
+        self.optimize_loop(ops, expected)
+
     def test_ovf_guard_in_short_preamble2(self):
         ops = """
         [p8, p11, p12]
@@ -3185,13 +3191,18 @@
         jump(p1, i4, i3)
         '''
         expected = '''
+        [p1, i4, i3, i5]
+        setfield_gc(p1, i5, descr=valuedescr)
+        jump(p1, i3, i5, i5)
+        '''
+        preamble = '''
         [p1, i1, i4]
         setfield_gc(p1, i1, descr=valuedescr)
         i3 = call(p1, descr=plaincalldescr)
         setfield_gc(p1, i3, descr=valuedescr)
-        jump(p1, i4, i3)
+        jump(p1, i4, i3, i3)
         '''
-        self.optimize_loop(ops, expected, expected)
+        self.optimize_loop(ops, expected, preamble)
 
     def test_call_pure_invalidates_heap_knowledge(self):
         # CALL_PURE should still force the setfield_gc() to occur before it
@@ -3203,21 +3214,20 @@
         jump(p1, i4, i3)
         '''
         expected = '''
+        [p1, i4, i3, i5]
+        setfield_gc(p1, i4, descr=valuedescr)
+        jump(p1, i3, i5, i5)
+        '''
+        preamble = '''
         [p1, i1, i4]
         setfield_gc(p1, i1, descr=valuedescr)
         i3 = call(p1, descr=plaincalldescr)
         setfield_gc(p1, i1, descr=valuedescr)
-        jump(p1, i4, i3)
+        jump(p1, i4, i3, i3)
         '''
-        self.optimize_loop(ops, expected, expected)
+        self.optimize_loop(ops, expected, preamble)
 
     def test_call_pure_constant_folding(self):
-        # CALL_PURE is not marked as is_always_pure(), because it is wrong
-        # to call the function arbitrary many times at arbitrary points in
-        # time.  Check that it is either constant-folded (and replaced by
-        # the result of the call, recorded as the first arg), or turned into
-        # a regular CALL.
-        # XXX can this test be improved with unrolling?
         arg_consts = [ConstInt(i) for i in (123456, 4, 5, 6)]
         call_pure_results = {tuple(arg_consts): ConstInt(42)}
         ops = '''
@@ -3233,14 +3243,13 @@
         escape(i1)
         escape(i2)
         i4 = call(123456, 4, i0, 6, descr=plaincalldescr)
-        jump(i0, i4)
+        jump(i0, i4, i4)
         '''
         expected = '''
-        [i0, i2]
+        [i0, i4, i5]
         escape(42)
-        escape(i2)
-        i4 = call(123456, 4, i0, 6, descr=plaincalldescr)
-        jump(i0, i4)
+        escape(i4)
+        jump(i0, i5, i5)
         '''
         self.optimize_loop(ops, expected, preamble, call_pure_results)
 
@@ -3264,18 +3273,43 @@
         escape(i2)
         i4 = call(123456, 4, i0, 6, descr=plaincalldescr)
         guard_no_exception() []
-        jump(i0, i4)
+        jump(i0, i4, i4)
         '''
         expected = '''
-        [i0, i2]
+        [i0, i2, i3]
         escape(42)
         escape(i2)
-        i4 = call(123456, 4, i0, 6, descr=plaincalldescr)
-        guard_no_exception() []
-        jump(i0, i4)
+        jump(i0, i3, i3)
         '''
         self.optimize_loop(ops, expected, preamble, call_pure_results)
 
+    def test_call_pure_returning_virtual(self):
+        # XXX: This kind of loop invaraint call_pure will be forced
+        #      both in the preamble and in the peeled loop
+        ops = '''
+        [p1, i1, i2]
+        p2 = call_pure(0, p1, i1, i2, descr=strslicedescr)
+        escape(p2)
+        jump(p1, i1, i2)
+        '''
+        preamble = '''
+        [p1, i1, i2]
+        i6 = int_sub(i2, i1)
+        p2 = newstr(i6)
+        copystrcontent(p1, p2, i1, 0, i6)
+        escape(p2)
+        jump(p1, i1, i2, i6)
+        '''
+        expected = '''
+        [p1, i1, i2, i6]
+        p2 = newstr(i6)
+        copystrcontent(p1, p2, i1, 0, i6)
+        escape(p2)
+        jump(p1, i1, i2, i6)
+        '''
+        self.optimize_loop(ops, expected, preamble)
+        
+
     # ----------
 
     def test_vref_nonvirtual_nonescape(self):
@@ -5144,14 +5178,14 @@
         [i0, i1, i10, i11, i2, i3, i4]
         escape(i2)
         escape(i3)
-        escape(i4)        
+        escape(i4)
         i24 = int_mul_ovf(i10, i11)
         guard_no_overflow() []
         i23 = int_sub_ovf(i10, i11)
         guard_no_overflow() []
         i22 = int_add_ovf(i10, i11)
         guard_no_overflow() []
-        jump(i0, i1, i10, i11, i2, i3, i4) 
+        jump(i0, i1, i10, i11, i2, i3, i4)
         """
         self.optimize_loop(ops, expected)
 
@@ -5360,6 +5394,8 @@
         """
         self.optimize_strunicode_loop(ops, expected, expected)
 
+    # XXX Should some of the call's below now be call_pure?
+
     def test_str_concat_1(self):
         ops = """
         [p1, p2]
@@ -5693,14 +5729,14 @@
         ops = """
         [p0, i0]
         i1 = unicodegetitem(p0, i0)
-        i10 = unicodegetitem(p0, i0)        
+        i10 = unicodegetitem(p0, i0)
         i2 = int_lt(i1, 0)
         guard_false(i2) []
         jump(p0, i0)
         """
         expected = """
         [p0, i0]
-        i1 = unicodegetitem(p0, i0)        
+        i1 = unicodegetitem(p0, i0)
         jump(p0, i0)
         """
         self.optimize_loop(ops, expected)
@@ -5859,7 +5895,7 @@
         """
         preamble = """
         [p1, i1, i2, p3]
-        guard_nonnull(p3) []        
+        guard_nonnull(p3) []
         i4 = int_sub(i2, i1)
         i0 = call(0, p1, i1, i4, p3, descr=streq_slice_nonnull_descr)
         escape(i0)
@@ -6468,7 +6504,7 @@
         setfield_gc(p3, i1, descr=adescr)
         setfield_gc(p3, i2, descr=bdescr)
         i5 = int_gt(ii, 42)
-        guard_true(i5) []        
+        guard_true(i5) []
         jump(p0, p1, p3, ii2, ii, i1, i2)
         """
         self.optimize_loop(ops, expected)
@@ -6494,7 +6530,7 @@
         p1 = getfield_gc(p0, descr=nextdescr)
         guard_nonnull_class(p1, ConstClass(node_vtable)) []
         p2 = getfield_gc(p1, descr=nextdescr)
-        guard_nonnull_class(p2, ConstClass(node_vtable)) []        
+        guard_nonnull_class(p2, ConstClass(node_vtable)) []
         jump(p0)
         """
         expected = """
@@ -6508,11 +6544,11 @@
         guard_class(p1, ConstClass(node_vtable)) []
         p2 = getfield_gc(p1, descr=nextdescr)
         guard_nonnull(p2) []
-        guard_class(p2, ConstClass(node_vtable)) []        
+        guard_class(p2, ConstClass(node_vtable)) []
         jump(p0)
         """
         self.optimize_loop(ops, expected, expected_short=short)
-        
+
     def test_forced_virtual_pure_getfield(self):
         ops = """
         [p0]
@@ -6576,7 +6612,7 @@
         jump(p1, i2)
         """
         self.optimize_loop(ops, expected)
-        
+
     def test_loopinvariant_strlen(self):
         ops = """
         [p9]
@@ -6709,7 +6745,7 @@
         [p0, p1]
         p2 = new_with_vtable(ConstClass(node_vtable))
         p3 = new_with_vtable(ConstClass(node_vtable))
-        setfield_gc(p2, p3, descr=nextdescr) 
+        setfield_gc(p2, p3, descr=nextdescr)
         jump(p2, p3)
         """
         expected = """
@@ -6728,7 +6764,7 @@
         jump(p2, i2)
         """
         expected = """
-        [p1]        
+        [p1]
         p2 = getarrayitem_gc(p1, 7, descr=<GcPtrArrayDescr>)
         i1 = arraylen_gc(p1)
         jump(p2)
@@ -6769,8 +6805,8 @@
         jump(p0, p2, p1)
         """
         self.optimize_loop(ops, expected, expected_short=short)
-        
-        
+
+
     def test_loopinvariant_constant_strgetitem(self):
         ops = """
         [p0]
@@ -6824,11 +6860,11 @@
         expected = """
         [p0, i22, p1]
         call(i22, descr=nonwritedescr)
-        i3 = unicodelen(p1) # Should be killed by backend        
+        i3 = unicodelen(p1) # Should be killed by backend
         jump(p0, i22, p1)
         """
         self.optimize_loop(ops, expected, expected_short=short)
-        
+
     def test_propagate_virtual_arryalen(self):
         ops = """
         [p0]
@@ -6897,7 +6933,7 @@
         [p0, p1, p10, p11]
         i1 = arraylen_gc(p10, descr=arraydescr)
         getarrayitem_gc(p11, 1, descr=arraydescr)
-        call(i1, descr=nonwritedescr)        
+        call(i1, descr=nonwritedescr)
         jump(p1, p0, p11, p10)
         """
         self.optimize_loop(ops, expected)
@@ -6906,20 +6942,20 @@
         ops = """
         [p5]
         i10 = getfield_gc(p5, descr=valuedescr)
-        call(i10, descr=nonwritedescr) 
+        call(i10, descr=nonwritedescr)
         setfield_gc(p5, 1, descr=valuedescr)
         jump(p5)
         """
         preamble = """
         [p5]
         i10 = getfield_gc(p5, descr=valuedescr)
-        call(i10, descr=nonwritedescr) 
+        call(i10, descr=nonwritedescr)
         setfield_gc(p5, 1, descr=valuedescr)
         jump(p5)
         """
         expected = """
         [p5]
-        call(1, descr=nonwritedescr) 
+        call(1, descr=nonwritedescr)
         jump(p5)
         """
         self.optimize_loop(ops, expected, preamble)
@@ -6957,7 +6993,7 @@
         [p9]
         call_assembler(0, descr=asmdescr)
         i18 = getfield_gc(p9, descr=valuedescr)
-        guard_value(i18, 0) []        
+        guard_value(i18, 0) []
         jump(p9)
         """
         self.optimize_loop(ops, expected)
@@ -6986,17 +7022,37 @@
         i10 = getfield_gc(p5, descr=valuedescr)
         i11 = getfield_gc(p6, descr=nextdescr)
         call(i10, i11, descr=nonwritedescr)
-        setfield_gc(p6, i10, descr=nextdescr)        
+        setfield_gc(p6, i10, descr=nextdescr)
         jump(p5, p6)
         """
         expected = """
         [p5, p6, i10, i11]
         call(i10, i11, descr=nonwritedescr)
-        setfield_gc(p6, i10, descr=nextdescr)        
+        setfield_gc(p6, i10, descr=nextdescr)
         jump(p5, p6, i10, i10)
         """
         self.optimize_loop(ops, expected)
-        
+
+    def test_cached_pure_func_of_equal_fields(self):
+        ops = """
+        [p5, p6]
+        i10 = getfield_gc(p5, descr=valuedescr)
+        i11 = getfield_gc(p6, descr=nextdescr)
+        i12 = int_add(i10, 7)
+        i13 = int_add(i11, 7)
+        call(i12, i13, descr=nonwritedescr)
+        setfield_gc(p6, i10, descr=nextdescr)
+        jump(p5, p6)
+        """
+        expected = """
+        [p5, p6, i14, i12, i10]
+        i13 = int_add(i14, 7)
+        call(i12, i13, descr=nonwritedescr)
+        setfield_gc(p6, i10, descr=nextdescr)
+        jump(p5, p6, i10, i12, i10)
+        """
+        self.optimize_loop(ops, expected)
+
     def test_forced_counter(self):
         # XXX: VIRTUALHEAP (see above)
         py.test.skip("would be fixed by make heap optimizer aware of virtual setfields")
@@ -7086,8 +7142,94 @@
         """
         self.optimize_loop(ops, expected)
 
-        
+    def test_import_constants_when_folding_pure_operations(self):
+        ops = """
+        [p0]
+        f1 = getfield_gc(p0, descr=valuedescr)
+        f2 = float_abs(f1)
+        call(7.0, descr=nonwritedescr)
+        setfield_gc(p0, -7.0, descr=valuedescr)
+        jump(p0)
+        """
+        expected = """
+        [p0]
+        call(7.0, descr=nonwritedescr)
+        jump(p0)
+        """
+        self.optimize_loop(ops, expected)
+
+    def test_exploding_duplicatipon(self):
+        ops = """
+        [i1, i2]
+        i3 = int_add(i1, i1)
+        i4 = int_add(i3, i3)
+        i5 = int_add(i4, i4)
+        i6 = int_add(i5, i5)
+        call(i6, descr=nonwritedescr)
+        jump(i1, i3)
+        """
+        expected = """
+        [i1, i2, i6, i3]
+        call(i6, descr=nonwritedescr)
+        jump(i1, i3, i6, i3)
+        """
+        short = """
+        [i1, i2]
+        i3 = int_add(i1, i1)
+        i4 = int_add(i3, i3)
+        i5 = int_add(i4, i4)
+        i6 = int_add(i5, i5)
+        jump(i1, i2, i6, i3)
+        """
+        self.optimize_loop(ops, expected, expected_short=short)
+
+    def test_prioritize_getfield1(self):
+        ops = """
+        [p1, p2]
+        i1 = getfield_gc(p1, descr=valuedescr)
+        setfield_gc(p2, i1, descr=nextdescr)
+        i2 = int_neg(i1)
+        call(i2, descr=nonwritedescr)
+        jump(p1, p2)
+        """
+        expected = """
+        [p1, p2, i2, i1]
+        call(i2, descr=nonwritedescr)
+        setfield_gc(p2, i1, descr=nextdescr)
+        jump(p1, p2, i2, i1)
+        """
+        self.optimize_loop(ops, expected)
+
+    def test_prioritize_getfield2(self):
+        # Same as previous, but with descrs intercahnged which means
+        # that the getfield is discovered first when looking for
+        # potential short boxes during tests
+        ops = """
+        [p1, p2]
+        i1 = getfield_gc(p1, descr=nextdescr)
+        setfield_gc(p2, i1, descr=valuedescr)
+        i2 = int_neg(i1)
+        call(i2, descr=nonwritedescr)
+        jump(p1, p2)
+        """
+        expected = """
+        [p1, p2, i2, i1]
+        call(i2, descr=nonwritedescr)
+        setfield_gc(p2, i1, descr=valuedescr)
+        jump(p1, p2, i2, i1)
+        """
+        self.optimize_loop(ops, expected)
+
+    def test_setarrayitem_followed_by_arraycopy(self):
+        ops = """
+        [p1, p2]
+        setarrayitem_gc(p1, 2, 10, descr=arraydescr)
+        setarrayitem_gc(p2, 3, 13, descr=arraydescr)        
+        call(0, p1, p2, 0, 0, 10, descr=arraycopydescr)
+        jump(p1, p2)
+        """
+        self.optimize_loop(ops, ops)
 
 class TestLLtype(OptimizeOptTest, LLtypeMixin):
     pass
-        
+
diff --git a/pypy/jit/metainterp/optimizeopt/unroll.py b/pypy/jit/metainterp/optimizeopt/unroll.py
--- a/pypy/jit/metainterp/optimizeopt/unroll.py
+++ b/pypy/jit/metainterp/optimizeopt/unroll.py
@@ -70,6 +70,47 @@
         self.snapshot_map[snapshot] = new_snapshot
         return new_snapshot
 
+class UnrollableOptimizer(Optimizer):
+    def setup(self):
+        self.importable_values = {}
+        self.emitting_dissabled = False
+        self.emitted_guards = 0
+        self.emitted_pure_operations = {}
+
+    def ensure_imported(self, value):
+        if not self.emitting_dissabled and value in self.importable_values:
+            imp = self.importable_values[value]
+            del self.importable_values[value]
+            imp.import_value(value)
+
+    def emit_operation(self, op):
+        if op.returns_bool_result():
+            self.bool_boxes[self.getvalue(op.result)] = None
+        if self.emitting_dissabled:
+            return
+        if op.is_guard():
+            self.emitted_guards += 1 # FIXME: can we use counter in self._emit_operation?
+        self._emit_operation(op)
+
+    def new(self):
+        new = UnrollableOptimizer(self.metainterp_sd, self.loop)
+        return self._new(new)
+
+    def remember_emitting_pure(self, op):
+        self.emitted_pure_operations[op] = True
+
+    def produce_potential_short_preamble_ops(self, sb):
+        for op in self.emitted_pure_operations:
+            if op.getopnum() == rop.GETARRAYITEM_GC_PURE or \
+               op.getopnum() == rop.STRGETITEM or \
+               op.getopnum() == rop.UNICODEGETITEM:
+                if not self.getvalue(op.getarg(1)).is_constant():
+                    continue
+            sb.add_potential(op)
+        for opt in self.optimizations:
+            opt.produce_potential_short_preamble_ops(sb)
+
+
 
 class UnrollOptimizer(Optimization):
     """Unroll the loop into two iterations. The first one will
@@ -77,7 +118,7 @@
     distinction anymore)"""
 
     def __init__(self, metainterp_sd, loop, optimizations):
-        self.optimizer = Optimizer(metainterp_sd, loop, optimizations)
+        self.optimizer = UnrollableOptimizer(metainterp_sd, loop, optimizations)
         self.cloned_operations = []
         for op in self.optimizer.loop.operations:
             newop = op.clone()
@@ -150,6 +191,7 @@
                 args = ", ".join([logops.repr_of_arg(arg) for arg in short_inputargs])
                 debug_print('short inputargs: ' + args)
                 self.short_boxes.debug_print(logops)
+                
 
             # Force virtuals amoung the jump_args of the preamble to get the
             # operations needed to setup the proper state of those virtuals
@@ -161,8 +203,9 @@
                 if box in seen:
                     continue
                 seen[box] = True
-                value = preamble_optimizer.getvalue(box)
-                inputarg_setup_ops.extend(value.make_guards(box))
+                preamble_value = preamble_optimizer.getvalue(box)
+                value = self.optimizer.getvalue(box)
+                value.import_from(preamble_value, self.optimizer)
             for box in short_inputargs:
                 if box in seen:
                     continue
@@ -181,23 +224,17 @@
             for op in self.short_boxes.operations():
                 self.ensure_short_op_emitted(op, self.optimizer, seen)
                 if op and op.result:
-                    # The order of these guards is not important as 
-                    # self.optimizer.emitting_dissabled is False
-                    value = preamble_optimizer.getvalue(op.result)
-                    for guard in value.make_guards(op.result):
-                        self.optimizer.send_extra_operation(guard)
+                    preamble_value = preamble_optimizer.getvalue(op.result)
+                    value = self.optimizer.getvalue(op.result)
+                    if not value.is_virtual():
+                        imp = ValueImporter(self, preamble_value, op)
+                        self.optimizer.importable_values[value] = imp
                     newresult = self.optimizer.getvalue(op.result).get_key_box()
                     if newresult is not op.result:
                         self.short_boxes.alias(newresult, op.result)
             self.optimizer.flush()
             self.optimizer.emitting_dissabled = False
 
-            # XXX Hack to prevent the arraylen/strlen/unicodelen ops generated
-            #     by value.make_guards() from ending up in pure_operations
-            for key, op in self.optimizer.pure_operations.items():
-                if not self.short_boxes.has_producer(op.result):
-                    del self.optimizer.pure_operations[key]
-
             initial_inputargs_len = len(inputargs)
             self.inliner = Inliner(loop.inputargs, jump_args)
 
@@ -276,16 +313,11 @@
 
         short_jumpargs = inputargs[:]
 
-        short = []
-        short_seen = {}
+        short = self.short = []
+        short_seen = self.short_seen = {}
         for box, const in self.constant_inputargs.items():
             short_seen[box] = True
 
-        for op in self.short_boxes.operations():
-            if op is not None:
-                if len(self.getvalue(op.result).make_guards(op.result)) > 0:
-                    self.add_op_to_short(op, short, short_seen, False, True)
-
         # This loop is equivalent to the main optimization loop in
         # Optimizer.propagate_all_forward
         jumpop = None
@@ -380,7 +412,7 @@
         if op.is_ovf():
             guard = ResOperation(rop.GUARD_NO_OVERFLOW, [], None)
             optimizer.send_extra_operation(guard)
-        
+
     def add_op_to_short(self, op, short, short_seen, emit=True, guards_needed=False):
         if op is None:
             return None
@@ -536,6 +568,13 @@
                         loop_token.failed_states.append(virtual_state)
         self.emit_operation(op)
 
+class ValueImporter(object):
+    def __init__(self, unroll, value, op):
+        self.unroll = unroll
+        self.preamble_value = value
+        self.op = op
 
-
-
+    def import_value(self, value):
+        value.import_from(self.preamble_value, self.unroll.optimizer)
+        self.unroll.add_op_to_short(self.op, self.unroll.short, self.unroll.short_seen, False, True)        
+        
diff --git a/pypy/jit/metainterp/optimizeopt/virtualize.py b/pypy/jit/metainterp/optimizeopt/virtualize.py
--- a/pypy/jit/metainterp/optimizeopt/virtualize.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualize.py
@@ -58,6 +58,9 @@
     def _really_force(self):
         raise NotImplementedError("abstract base")
 
+    def import_from(self, other, optimizer):
+        raise NotImplementedError("should not be called at this level")
+    
 def get_fielddescrlist_cache(cpu):
     if not hasattr(cpu, '_optimizeopt_fielddescrlist_cache'):
         result = descrlist_dict()
diff --git a/pypy/jit/metainterp/optimizeopt/virtualstate.py b/pypy/jit/metainterp/optimizeopt/virtualstate.py
--- a/pypy/jit/metainterp/optimizeopt/virtualstate.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualstate.py
@@ -12,6 +12,7 @@
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.rlib.debug import debug_start, debug_stop, debug_print
 from pypy.rlib.objectmodel import we_are_translated
+import os
 
 class AbstractVirtualStateInfo(resume.AbstractVirtualInfo):
     position = -1
@@ -461,8 +462,10 @@
 class ShortBoxes(object):
     def __init__(self, optimizer, surviving_boxes):
         self.potential_ops = {}
-        self.duplicates = {}
+        self.alternatives = {}
+        self.synthetic = {}
         self.aliases = {}
+        self.rename = {}
         self.optimizer = optimizer
         for box in surviving_boxes:
             self.potential_ops[box] = None
@@ -476,33 +479,81 @@
             except BoxNotProducable:
                 pass
 
+    def prioritized_alternatives(self, box):
+        if box not in self.alternatives:
+            return [self.potential_ops[box]]
+        alts = self.alternatives[box]
+        hi, lo = 0, len(alts) - 1
+        while hi < lo:
+            if alts[lo] is None: # Inputarg, lowest priority
+                alts[lo], alts[-1] = alts[-1], alts[lo]
+                lo -= 1
+            elif alts[lo] not in self.synthetic: # Hi priority
+                alts[hi], alts[lo] = alts[lo], alts[hi]
+                hi += 1
+            else: # Low priority
+                lo -= 1
+        return alts
+            
+    def renamed(self, box):
+        if box in self.rename:
+            return self.rename[box]
+        return box
+    
+    def add_to_short(self, box, op):
+        if op:
+            op = op.clone()
+            for i in range(op.numargs()):
+                op.setarg(i, self.renamed(op.getarg(i)))
+        if box in self.short_boxes:
+            if op is None:
+                oldop = self.short_boxes[box].clone()
+                oldres = oldop.result
+                newbox = oldop.result = oldres.clonebox()
+                self.rename[box] = newbox
+                self.short_boxes[box] = None
+                self.short_boxes[newbox] = oldop
+            else:
+                newop = op.clone()
+                newbox = newop.result = op.result.clonebox()
+                self.short_boxes[newop.result] = newop
+            value = self.optimizer.getvalue(box)
+            self.optimizer.make_equal_to(newbox, value)
+        else:
+            self.short_boxes[box] = op
+        
     def produce_short_preamble_box(self, box):
         if box in self.short_boxes:
             return 
         if isinstance(box, Const):
             return 
         if box in self.potential_ops:
-            op = self.potential_ops[box]
-            if op:
-                for arg in op.getarglist():
-                    self.produce_short_preamble_box(arg)
-            self.short_boxes[box] = op
+            ops = self.prioritized_alternatives(box)
+            produced_one = False
+            for op in ops:
+                try:
+                    if op:
+                        for arg in op.getarglist():
+                            self.produce_short_preamble_box(arg)
+                except BoxNotProducable:
+                    pass
+                else:
+                    produced_one = True
+                    self.add_to_short(box, op)
+            if not produced_one:
+                raise BoxNotProducable
         else:
             raise BoxNotProducable
 
-    def add_potential(self, op):
+    def add_potential(self, op, synthetic=False):
         if op.result not in self.potential_ops:
             self.potential_ops[op.result] = op
-            return op
-        newop = op.clone()
-        newop.result = op.result.clonebox()
-        self.potential_ops[newop.result] = newop
-        if op.result in self.duplicates:
-            self.duplicates[op.result].append(newop.result)
         else:
-            self.duplicates[op.result] = [newop.result]
-        self.optimizer.make_equal_to(newop.result, self.optimizer.getvalue(op.result))
-        return newop
+            if op.result not in self.alternatives:
+                self.alternatives[op.result] = [self.potential_ops[op.result]]
+            self.alternatives[op.result].append(op)
+        if synthetic:
+            self.synthetic[op] = True
 
     def debug_print(self, logops):
         debug_start('jit-short-boxes')
diff --git a/pypy/jit/metainterp/optimizeopt/vstring.py b/pypy/jit/metainterp/optimizeopt/vstring.py
--- a/pypy/jit/metainterp/optimizeopt/vstring.py
+++ b/pypy/jit/metainterp/optimizeopt/vstring.py
@@ -141,6 +141,11 @@
                                    for c in self._chars])
 
     def string_copy_parts(self, optimizer, targetbox, offsetbox, mode):
+        if not self.is_virtual() and targetbox is not self.box:
+            lengthbox = self.getstrlen(optimizer, mode)
+            srcbox = self.force_box()
+            return copy_str_content(optimizer, srcbox, targetbox,
+                                CONST_0, offsetbox, lengthbox, mode)
         for i in range(len(self._chars)):
             charbox = self._chars[i].force_box()
             if not (isinstance(charbox, Const) and charbox.same_constant(CONST_0)):
@@ -296,7 +301,7 @@
 
 
 def copy_str_content(optimizer, srcbox, targetbox,
-                     srcoffsetbox, offsetbox, lengthbox, mode):
+                     srcoffsetbox, offsetbox, lengthbox, mode, need_next_offset=True):
     if isinstance(srcbox, ConstPtr) and isinstance(srcoffsetbox, Const):
         M = 5
     else:
@@ -313,7 +318,10 @@
                                               None))
             offsetbox = _int_add(optimizer, offsetbox, CONST_1)
     else:
-        nextoffsetbox = _int_add(optimizer, offsetbox, lengthbox)
+        if need_next_offset:
+            nextoffsetbox = _int_add(optimizer, offsetbox, lengthbox)
+        else:
+            nextoffsetbox = None
         op = ResOperation(mode.COPYSTRCONTENT, [srcbox, targetbox,
                                                 srcoffsetbox, offsetbox,
                                                 lengthbox], None)
@@ -365,7 +373,7 @@
 
     def new(self):
         return OptString()
-    
+
     def make_vstring_plain(self, box, source_op, mode):
         vvalue = VStringPlainValue(self.optimizer, box, source_op, mode)
         self.make_equal_to(box, vvalue)
@@ -435,7 +443,11 @@
         #
         if isinstance(value, VStringPlainValue):  # even if no longer virtual
             if vindex.is_constant():
-                return value.getitem(vindex.box.getint())
+                res = value.getitem(vindex.box.getint())
+                # If it is uninitialized we can't return it, it was set by a
+                # COPYSTRCONTENT, not a STRSETITEM
+                if res is not optimizer.CVAL_UNINITIALIZED_ZERO:
+                    return res
         #
         resbox = _strgetitem(self.optimizer, value.force_box(), vindex.force_box(), mode)
         return self.getvalue(resbox)
@@ -450,6 +462,30 @@
         lengthbox = value.getstrlen(self.optimizer, mode)
         self.make_equal_to(op.result, self.getvalue(lengthbox))
 
+    def optimize_COPYSTRCONTENT(self, op):
+        self._optimize_COPYSTRCONTENT(op, mode_string)
+    def optimize_COPYUNICODECONTENT(self, op):
+        self._optimize_COPYSTRCONTENT(op, mode_unicode)
+
+    def _optimize_COPYSTRCONTENT(self, op, mode):
+        # args: src dst srcstart dststart length
+        src = self.getvalue(op.getarg(0))
+        dst = self.getvalue(op.getarg(1))
+        srcstart = self.getvalue(op.getarg(2))
+        dststart = self.getvalue(op.getarg(3))
+        length = self.getvalue(op.getarg(4))
+
+        if length.is_constant() and length.box.getint() == 0:
+            return
+        copy_str_content(self.optimizer,
+            src.force_box(),
+            dst.force_box(),
+            srcstart.force_box(),
+            dststart.force_box(),
+            length.force_box(),
+            mode, need_next_offset=False
+        )
+
     def optimize_CALL(self, op):
         # dispatch based on 'oopspecindex' to a method that handles
         # specifically the given oopspec call.  For non-oopspec calls,
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -17,6 +17,7 @@
 from pypy.jit.metainterp.jitprof import ABORT_TOO_LONG, ABORT_BRIDGE, \
                                         ABORT_FORCE_QUASIIMMUT, ABORT_BAD_LOOP
 from pypy.jit.metainterp.jitexc import JitException, get_llexception
+from pypy.jit.metainterp.heapcache import HeapCache
 from pypy.rlib.objectmodel import specialize
 from pypy.jit.codewriter.jitcode import JitCode, SwitchDictDescr
 from pypy.jit.codewriter import heaptracker
@@ -209,7 +210,8 @@
                 self.metainterp.clear_exception()
                 resbox = self.execute(rop.%s, b1, b2)
                 self.make_result_of_lastop(resbox)  # same as execute_varargs()
-                self.metainterp.handle_possible_overflow_error()
+                if not isinstance(resbox, Const):
+                    self.metainterp.handle_possible_overflow_error()
                 return resbox
         ''' % (_opimpl, _opimpl.upper())).compile()
 
@@ -321,7 +323,7 @@
     def _establish_nullity(self, box, orgpc):
         value = box.nonnull()
         if value:
-            if box not in self.metainterp.known_class_boxes:
+            if not self.metainterp.heapcache.is_class_known(box):
                 self.generate_guard(rop.GUARD_NONNULL, box, resumepc=orgpc)
         else:
             if not isinstance(box, Const):
@@ -366,14 +368,17 @@
 
     @arguments("descr")
     def opimpl_new(self, sizedescr):
-        return self.execute_with_descr(rop.NEW, sizedescr)
+        resbox = self.execute_with_descr(rop.NEW, sizedescr)
+        self.metainterp.heapcache.new(resbox)
+        return resbox
 
     @arguments("descr")
     def opimpl_new_with_vtable(self, sizedescr):
         cpu = self.metainterp.cpu
         cls = heaptracker.descr2vtable(cpu, sizedescr)
         resbox = self.execute(rop.NEW_WITH_VTABLE, ConstInt(cls))
-        self.metainterp.known_class_boxes[resbox] = None
+        self.metainterp.heapcache.new(resbox)
+        self.metainterp.heapcache.class_now_known(resbox)
         return resbox
 
 ##    @FixME  #arguments("box")
@@ -392,26 +397,30 @@
 ##        self.execute(rop.SUBCLASSOF, box1, box2)
 
     @arguments("descr", "box")
-    def opimpl_new_array(self, itemsizedescr, countbox):
-        return self.execute_with_descr(rop.NEW_ARRAY, itemsizedescr, countbox)
+    def opimpl_new_array(self, itemsizedescr, lengthbox):
+        resbox = self.execute_with_descr(rop.NEW_ARRAY, itemsizedescr, lengthbox)
+        self.metainterp.heapcache.new_array(resbox, lengthbox)
+        return resbox
+
+    @specialize.arg(1)
+    def _do_getarrayitem_gc_any(self, op, arraybox, arraydescr, indexbox):
+        tobox = self.metainterp.heapcache.getarrayitem(
+                arraybox, arraydescr, indexbox)
+        if tobox:
+            # sanity check: see whether the current array value
+            # corresponds to what the cache thinks the value is
+            resbox = executor.execute(self.metainterp.cpu, self.metainterp, op,
+                                      arraydescr, arraybox, indexbox)
+            assert resbox.constbox().same_constant(tobox.constbox())
+            return tobox
+        resbox = self.execute_with_descr(op, arraydescr, arraybox, indexbox)
+        self.metainterp.heapcache.getarrayitem_now_known(
+                arraybox, arraydescr, indexbox, resbox)
+        return resbox
 
     @arguments("box", "descr", "box")
     def _opimpl_getarrayitem_gc_any(self, arraybox, arraydescr, indexbox):
-        cache = self.metainterp.heap_array_cache.get(arraydescr, None)
-        if cache and isinstance(indexbox, ConstInt):
-            index = indexbox.getint()
-            frombox, tobox = cache.get(index, (None, None))
-            if frombox is arraybox:
-                return tobox
-        resbox = self.execute_with_descr(rop.GETARRAYITEM_GC,
-                                         arraydescr, arraybox, indexbox)
-        if isinstance(indexbox, ConstInt):
-            if not cache:
-                cache = self.metainterp.heap_array_cache[arraydescr] = {}
-            index = indexbox.getint()
-            cache[index] = arraybox, resbox
-        return resbox
-
+        return self._do_getarrayitem_gc_any(rop.GETARRAYITEM_GC, arraybox, arraydescr, indexbox)
 
     opimpl_getarrayitem_gc_i = _opimpl_getarrayitem_gc_any
     opimpl_getarrayitem_gc_r = _opimpl_getarrayitem_gc_any
@@ -427,8 +436,7 @@
 
     @arguments("box", "descr", "box")
     def _opimpl_getarrayitem_gc_pure_any(self, arraybox, arraydescr, indexbox):
-        return self.execute_with_descr(rop.GETARRAYITEM_GC_PURE,
-                                       arraydescr, arraybox, indexbox)
+        return self._do_getarrayitem_gc_any(rop.GETARRAYITEM_GC_PURE, arraybox, arraydescr, indexbox)
 
     opimpl_getarrayitem_gc_pure_i = _opimpl_getarrayitem_gc_pure_any
     opimpl_getarrayitem_gc_pure_r = _opimpl_getarrayitem_gc_pure_any
@@ -439,13 +447,8 @@
                                     indexbox, itembox):
         self.execute_with_descr(rop.SETARRAYITEM_GC, arraydescr, arraybox,
                                 indexbox, itembox)
-        if isinstance(indexbox, ConstInt):
-            cache = self.metainterp.heap_array_cache.setdefault(arraydescr, {})
-            cache[indexbox.getint()] = arraybox, itembox
-        else:
-            cache = self.metainterp.heap_array_cache.get(arraydescr, None)
-            if cache:
-                cache.clear()
+        self.metainterp.heapcache.setarrayitem(
+                arraybox, arraydescr, indexbox, itembox)
 
     opimpl_setarrayitem_gc_i = _opimpl_setarrayitem_gc_any
     opimpl_setarrayitem_gc_r = _opimpl_setarrayitem_gc_any
@@ -462,7 +465,12 @@
 
     @arguments("box", "descr")
     def opimpl_arraylen_gc(self, arraybox, arraydescr):
-        return self.execute_with_descr(rop.ARRAYLEN_GC, arraydescr, arraybox)
+        lengthbox = self.metainterp.heapcache.arraylen(arraybox)
+        if lengthbox is None:
+            lengthbox = self.execute_with_descr(
+                    rop.ARRAYLEN_GC, arraydescr, arraybox)
+            self.metainterp.heapcache.arraylen_now_known(arraybox, lengthbox)
+        return lengthbox
 
     @arguments("orgpc", "box", "descr", "box")
     def opimpl_check_neg_index(self, orgpc, arraybox, arraydescr, indexbox):
@@ -471,19 +479,17 @@
         negbox = self.implement_guard_value(orgpc, negbox)
         if negbox.getint():
             # the index is < 0; add the array length to it
-            lenbox = self.metainterp.execute_and_record(
-                rop.ARRAYLEN_GC, arraydescr, arraybox)
+            lengthbox = self.opimpl_arraylen_gc(arraybox, arraydescr)
             indexbox = self.metainterp.execute_and_record(
-                rop.INT_ADD, None, indexbox, lenbox)
+                rop.INT_ADD, None, indexbox, lengthbox)
         return indexbox
 
     @arguments("descr", "descr", "descr", "descr", "box")
     def opimpl_newlist(self, structdescr, lengthdescr, itemsdescr, arraydescr,
                        sizebox):
-        sbox = self.metainterp.execute_and_record(rop.NEW, structdescr)
+        sbox = self.opimpl_new(structdescr)
         self._opimpl_setfield_gc_any(sbox, lengthdescr, sizebox)
-        abox = self.metainterp.execute_and_record(rop.NEW_ARRAY, arraydescr,
-                                                  sizebox)
+        abox = self.opimpl_new_array(arraydescr, sizebox)
         self._opimpl_setfield_gc_any(sbox, itemsdescr, abox)
         return sbox
 
@@ -540,11 +546,15 @@
 
     @specialize.arg(1)
     def _opimpl_getfield_gc_any_pureornot(self, opnum, box, fielddescr):
-        frombox, tobox = self.metainterp.heap_cache.get(fielddescr, (None, None))
-        if frombox is box:
+        tobox = self.metainterp.heapcache.getfield(box, fielddescr)
+        if tobox is not None:
+            # sanity check: see whether the current struct value
+            # corresponds to what the cache thinks the value is
+            resbox = executor.execute(self.metainterp.cpu, self.metainterp,
+                                      rop.GETFIELD_GC, fielddescr, box)
             return tobox
         resbox = self.execute_with_descr(opnum, fielddescr, box)
-        self.metainterp.heap_cache[fielddescr] = (box, resbox)
+        self.metainterp.heapcache.getfield_now_known(box, fielddescr, resbox)
         return resbox
 
     @arguments("orgpc", "box", "descr")
@@ -565,11 +575,11 @@
 
     @arguments("box", "descr", "box")
     def _opimpl_setfield_gc_any(self, box, fielddescr, valuebox):
-        frombox, tobox = self.metainterp.heap_cache.get(fielddescr, (None, None))
-        if frombox is box and tobox is valuebox:
+        tobox = self.metainterp.heapcache.getfield(box, fielddescr)
+        if tobox is valuebox:
             return
         self.execute_with_descr(rop.SETFIELD_GC, fielddescr, box, valuebox)
-        self.metainterp.heap_cache[fielddescr] = (box, valuebox)
+        self.metainterp.heapcache.setfield(box, fielddescr, valuebox)
     opimpl_setfield_gc_i = _opimpl_setfield_gc_any
     opimpl_setfield_gc_r = _opimpl_setfield_gc_any
     opimpl_setfield_gc_f = _opimpl_setfield_gc_any
@@ -633,7 +643,7 @@
         standard_box = self.metainterp.virtualizable_boxes[-1]
         if standard_box is box:
             return False
-        if box in self.metainterp.nonstandard_virtualizables:
+        if self.metainterp.heapcache.is_nonstandard_virtualizable(box):
             return True
         eqbox = self.metainterp.execute_and_record(rop.PTR_EQ, None,
                                                    box, standard_box)
@@ -642,7 +652,7 @@
         if isstandard:
             self.metainterp.replace_box(box, standard_box)
         else:
-            self.metainterp.nonstandard_virtualizables[box] = None
+            self.metainterp.heapcache.nonstandard_virtualizables_now_known(box)
         return not isstandard
 
     def _get_virtualizable_field_index(self, fielddescr):
@@ -727,7 +737,7 @@
     def opimpl_arraylen_vable(self, pc, box, fdescr, adescr):
         if self._nonstandard_virtualizable(pc, box):
             arraybox = self._opimpl_getfield_gc_any(box, fdescr)
-            return self.execute_with_descr(rop.ARRAYLEN_GC, adescr, arraybox)
+            return self.opimpl_arraylen_gc(arraybox, adescr)
         vinfo = self.metainterp.jitdriver_sd.virtualizable_info
         virtualizable_box = self.metainterp.virtualizable_boxes[-1]
         virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box)
@@ -858,6 +868,14 @@
     def opimpl_newunicode(self, lengthbox):
         return self.execute(rop.NEWUNICODE, lengthbox)
 
+    @arguments("box", "box", "box", "box", "box")
+    def opimpl_copystrcontent(self, srcbox, dstbox, srcstartbox, dststartbox, lengthbox):
+        return self.execute(rop.COPYSTRCONTENT, srcbox, dstbox, srcstartbox, dststartbox, lengthbox)
+
+    @arguments("box", "box", "box", "box", "box")
+    def opimpl_copyunicodecontent(self, srcbox, dstbox, srcstartbox, dststartbox, lengthbox):
+        return self.execute(rop.COPYUNICODECONTENT, srcbox, dstbox, srcstartbox, dststartbox, lengthbox)
+
 ##    @FixME  #arguments("descr", "varargs")
 ##    def opimpl_residual_oosend_canraise(self, methdescr, varargs):
 ##        return self.execute_varargs(rop.OOSEND, varargs, descr=methdescr,
@@ -884,9 +902,9 @@
     @arguments("orgpc", "box")
     def opimpl_guard_class(self, orgpc, box):
         clsbox = self.cls_of_box(box)
-        if box not in self.metainterp.known_class_boxes:
+        if not self.metainterp.heapcache.is_class_known(box):
             self.generate_guard(rop.GUARD_CLASS, box, [clsbox], resumepc=orgpc)
-            self.metainterp.known_class_boxes[box] = None
+            self.metainterp.heapcache.class_now_known(box)
         return clsbox
 
     @arguments("int", "orgpc")
@@ -1052,6 +1070,18 @@
         return ConstInt(trace_length)
 
     @arguments("box")
+    def _opimpl_isconstant(self, box):
+        return ConstInt(isinstance(box, Const))
+
+    opimpl_int_isconstant = opimpl_ref_isconstant = _opimpl_isconstant
+
+    @arguments("box")
+    def _opimpl_isvirtual(self, box):
+        return ConstInt(self.metainterp.heapcache.is_unescaped(box))
+
+    opimpl_ref_isvirtual = _opimpl_isvirtual
+
+    @arguments("box")
     def opimpl_virtual_ref(self, box):
         # Details on the content of metainterp.virtualref_boxes:
         #
@@ -1492,16 +1522,7 @@
         self.last_exc_value_box = None
         self.retracing_loop_from = None
         self.call_pure_results = args_dict_box()
-        # contains boxes where the class is already known
-        self.known_class_boxes = {}
-        # contains frame boxes that are not virtualizables
-        self.nonstandard_virtualizables = {}
-        # heap cache
-        # maps descrs to (from_box, to_box) tuples
-        self.heap_cache = {}
-        # heap array cache
-        # maps descrs to {index: (from_box, to_box)} dicts
-        self.heap_array_cache = {}
+        self.heapcache = HeapCache()
 
     def perform_call(self, jitcode, boxes, greenkey=None):
         # causes the metainterp to enter the given subfunction
@@ -1674,32 +1695,18 @@
 
     def _record_helper_nonpure_varargs(self, opnum, resbox, descr, argboxes):
         assert resbox is None or isinstance(resbox, Box)
+        if (rop._OVF_FIRST <= opnum <= rop._OVF_LAST and
+            self.last_exc_value_box is None and
+            self._all_constants_varargs(argboxes)):
+            return resbox.constbox()
         # record the operation
         profiler = self.staticdata.profiler
         profiler.count_ops(opnum, RECORDED_OPS)
-        self._invalidate_caches(opnum, descr)
+        self.heapcache.invalidate_caches(opnum, descr, argboxes)
         op = self.history.record(opnum, argboxes, resbox, descr)
         self.attach_debug_info(op)
         return resbox
 
-    def _invalidate_caches(self, opnum, descr):
-        if opnum == rop.SETFIELD_GC:
-            return
-        if opnum == rop.SETARRAYITEM_GC:
-            return
-        if rop._NOSIDEEFFECT_FIRST <= opnum <= rop._NOSIDEEFFECT_LAST:
-            return
-        if opnum == rop.CALL:
-            effectinfo = descr.get_extra_info()
-            ef = effectinfo.extraeffect
-            if ef == effectinfo.EF_LOOPINVARIANT or \
-               ef == effectinfo.EF_ELIDABLE_CANNOT_RAISE or \
-               ef == effectinfo.EF_ELIDABLE_CAN_RAISE:
-                return
-        if self.heap_cache:
-            self.heap_cache.clear()
-        if self.heap_array_cache:
-            self.heap_array_cache.clear()
 
     def attach_debug_info(self, op):
         if (not we_are_translated() and op is not None
@@ -1862,10 +1869,7 @@
                 duplicates[box] = None
 
     def reached_loop_header(self, greenboxes, redboxes, resumedescr):
-        self.known_class_boxes = {}
-        self.nonstandard_virtualizables = {} # XXX maybe not needed?
-        self.heap_cache = {}
-        self.heap_array_cache = {}
+        self.heapcache.reset()
 
         duplicates = {}
         self.remove_consts_and_duplicates(redboxes, len(redboxes),
@@ -2373,17 +2377,7 @@
             for i in range(len(boxes)):
                 if boxes[i] is oldbox:
                     boxes[i] = newbox
-        for descr, (frombox, tobox) in self.heap_cache.iteritems():
-            change = False
-            if frombox is oldbox:
-                change = True
-                frombox = newbox
-            if tobox is oldbox:
-                change = True
-                tobox = newbox
-            if change:
-                self.heap_cache[descr] = frombox, tobox
-        # XXX what about self.heap_array_cache?
+        self.heapcache.replace_box(oldbox, newbox)
 
     def find_biggest_function(self):
         start_stack = []
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -1,23 +1,25 @@
+import sys
+
 import py
-import sys
-from pypy.rlib.jit import JitDriver, we_are_jitted, hint, dont_look_inside
-from pypy.rlib.jit import loop_invariant, elidable, promote
-from pypy.rlib.jit import jit_debug, assert_green, AssertGreenFailed
-from pypy.rlib.jit import unroll_safe, current_trace_length
+
+from pypy import conftest
+from pypy.jit.codewriter.policy import JitPolicy, StopAtXPolicy
 from pypy.jit.metainterp import pyjitpl, history
+from pypy.jit.metainterp.optimizeopt import ALL_OPTS_DICT
+from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin, noConst
+from pypy.jit.metainterp.typesystem import LLTypeHelper, OOTypeHelper
+from pypy.jit.metainterp.warmspot import get_stats
 from pypy.jit.metainterp.warmstate import set_future_value
-from pypy.jit.metainterp.warmspot import get_stats
-from pypy.jit.codewriter.policy import JitPolicy, StopAtXPolicy
-from pypy import conftest
+from pypy.rlib.jit import (JitDriver, we_are_jitted, hint, dont_look_inside,
+    loop_invariant, elidable, promote, jit_debug, assert_green,
+    AssertGreenFailed, unroll_safe, current_trace_length, look_inside_iff,
+    isconstant, isvirtual)
 from pypy.rlib.rarithmetic import ovfcheck
-from pypy.jit.metainterp.typesystem import LLTypeHelper, OOTypeHelper
 from pypy.rpython.lltypesystem import lltype, llmemory, rffi
 from pypy.rpython.ootypesystem import ootype
-from pypy.jit.metainterp.optimizeopt import ALL_OPTS_DICT
-from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin, noConst
+
 
 class BasicTests:
-
     def test_basic(self):
         def f(x, y):
             return x + y
@@ -99,14 +101,14 @@
                 myjitdriver.jit_merge_point(x=x, y=y, res=res)
                 res += x * x
                 x += 1
-                res += x * x                
+                res += x * x
                 y -= 1
             return res
         res = self.meta_interp(f, [6, 7])
         assert res == 1323
         self.check_loop_count(1)
         self.check_loops(int_mul=1)
-        
+
     def test_loop_variant_mul_ovf(self):
         myjitdriver = JitDriver(greens = [], reds = ['y', 'res', 'x'])
         def f(x, y):
@@ -1372,7 +1374,7 @@
             return x
         res = self.meta_interp(f, [299], listops=True)
         assert res == f(299)
-        self.check_loops(guard_class=0, guard_value=3)        
+        self.check_loops(guard_class=0, guard_value=3)
         self.check_loops(guard_class=0, guard_value=6, everywhere=True)
 
     def test_merge_guardnonnull_guardclass(self):
@@ -2118,7 +2120,7 @@
             return sa
         res = self.meta_interp(f, [32, 7])
         assert res == f(32, 7)
-        
+
     def test_caching_setarrayitem_fixed(self):
         myjitdriver = JitDriver(greens = [], reds = ['sa', 'i', 'n', 'a', 'node'])
         def f(n, a):
@@ -2138,7 +2140,7 @@
             return sa
         res = self.meta_interp(f, [32, 7])
         assert res == f(32, 7)
-        
+
     def test_caching_setarrayitem_var(self):
         myjitdriver = JitDriver(greens = [], reds = ['sa', 'i', 'n', 'a', 'b', 'node'])
         def f(n, a, b):
@@ -2668,7 +2670,7 @@
             myjitdriver.set_param('threshold', 3)
             myjitdriver.set_param('trace_eagerness', 1)
             myjitdriver.set_param('retrace_limit', 5)
-            myjitdriver.set_param('function_threshold', -1)            
+            myjitdriver.set_param('function_threshold', -1)
             pc = sa = i = 0
             while pc < len(bytecode):
                 myjitdriver.jit_merge_point(pc=pc, n=n, sa=sa, i=i)
@@ -2693,12 +2695,12 @@
         def g(n1, n2):
             for i in range(10):
                 f(n1)
-            for i in range(10):                
+            for i in range(10):
                 f(n2)
 
         nn = [10, 3]
         assert self.meta_interp(g, nn) == g(*nn)
-        
+
         # The attempts of retracing first loop will end up retracing the
         # second and thus fail 5 times, saturating the retrace_count. Instead a
         # bridge back to the preamble of the first loop is produced. A guard in
@@ -2709,7 +2711,7 @@
         self.check_tree_loop_count(2 + 3)
 
         # FIXME: Add a gloabl retrace counter and test that we are not trying more than 5 times.
-        
+
         def g(n):
             for i in range(n):
                 for j in range(10):
@@ -2945,15 +2947,15 @@
             a = [0, 1, 2, 3, 4]
             while i < n:
                 myjitdriver.jit_merge_point(sa=sa, n=n, a=a, i=i)
-                if i < n/2:
+                if i < n / 2:
                     sa += a[4]
-                elif i == n/2:
+                elif i == n / 2:
                     a.pop()
                 i += 1
         res = self.meta_interp(f, [32])
         assert res == f(32)
         self.check_loops(arraylen_gc=2)
-        
+
 class TestOOtype(BasicTests, OOJitMixin):
 
     def test_oohash(self):
@@ -3173,7 +3175,7 @@
         res = self.meta_interp(f, [32])
         assert res == f(32)
         self.check_tree_loop_count(3)
-        
+
     def test_two_loopinvariant_arrays3(self):
         from pypy.rpython.lltypesystem import lltype, llmemory, rffi
         myjitdriver = JitDriver(greens = [], reds = ['sa', 'n', 'i', 'a'])
@@ -3197,7 +3199,7 @@
         res = self.meta_interp(f, [32])
         assert res == f(32)
         self.check_tree_loop_count(2)
-        
+
     def test_two_loopinvariant_arrays_boxed(self):
         class A(object):
             def __init__(self, a):
@@ -3222,7 +3224,7 @@
         res = self.meta_interp(f, [32])
         assert res == f(32)
         self.check_loops(arraylen_gc=2, everywhere=True)
-        
+
     def test_release_gil_flush_heap_cache(self):
         if sys.platform == "win32":
             py.test.skip("needs 'time'")
@@ -3276,7 +3278,205 @@
             return n
 
         self.meta_interp(f, [10], repeat=3)
+
+    def test_jit_merge_point_with_pbc(self):
+        driver = JitDriver(greens = [], reds = ['x'])
+
+        class A(object):
+            def __init__(self, x):
+                self.x = x
+            def _freeze_(self):
+                return True
+        pbc = A(1)
+
+        def main(x):
+            return f(x, pbc)
+
+        def f(x, pbc):
+            while x > 0:
+                driver.jit_merge_point(x = x)
+                x -= pbc.x
+            return x
+
+        self.meta_interp(main, [10])
+
+    def test_look_inside_iff_const(self):
+        @look_inside_iff(lambda arg: isconstant(arg))
+        def f(arg):
+            s = 0
+            while arg > 0:
+                s += arg
+                arg -= 1
+            return s
+
+        driver = JitDriver(greens = ['code'], reds = ['n', 'arg', 's'])
+
+        def main(code, n, arg):
+            s = 0
+            while n > 0:
+                driver.jit_merge_point(code=code, n=n, arg=arg, s=s)
+                if code == 0:
+                    s += f(arg)
+                else:
+                    s += f(1)
+                n -= 1
+            return s
+
+        res = self.meta_interp(main, [0, 10, 2], enable_opts='')
+        assert res == main(0, 10, 2)
+        self.check_loops(call=1)
+        res = self.meta_interp(main, [1, 10, 2], enable_opts='')
+        assert res == main(1, 10, 2)
+        self.check_loops(call=0)
+
+    def test_look_inside_iff_virtual(self):
+        # There's no good reason for this to be look_inside_iff, but it's a test!
+        @look_inside_iff(lambda arg, n: isvirtual(arg))
+        def f(arg, n):
+            if n == 100:
+                for i in xrange(n):
+                    n += i
+            return arg.x
+        class A(object):
+            def __init__(self, x):
+                self.x = x
+        driver = JitDriver(greens=['n'], reds=['i', 'a'])
+        def main(n):
+            i = 0
+            a = A(3)
+            while i < 20:
+                driver.jit_merge_point(i=i, n=n, a=a)
+                if n == 0:
+                    i += f(a, n)
+                else:
+                    i += f(A(2), n)
+        res = self.meta_interp(main, [0], enable_opts='')
+        assert res == main(0)
+        self.check_loops(call=1, getfield_gc=0)
+        res = self.meta_interp(main, [1], enable_opts='')
+        assert res == main(1)
+        self.check_loops(call=0, getfield_gc=0)
+
+    def test_reuse_elidable_result(self):
+        driver = JitDriver(reds=['n', 's'], greens = [])
+        def main(n):
+            s = 0
+            while n > 0:
+                driver.jit_merge_point(s=s, n=n)
+                s += len(str(n)) + len(str(n))
+                n -= 1
+            return s
+        res = self.meta_interp(main, [10])
+        assert res == main(10)
+        self.check_loops({
+            'call': 1, 'guard_no_exception': 1, 'guard_true': 1, 'int_add': 2,
+            'int_gt': 1, 'int_sub': 1, 'strlen': 1, 'jump': 1,
+        })
+
+    def test_look_inside_iff_const_getarrayitem_gc_pure(self):
+        driver = JitDriver(greens=['unroll'], reds=['s', 'n'])
+
+        class A(object):
+            _immutable_fields_ = ["x[*]"]
+            def __init__(self, x):
+                self.x = [x]
+
+        @look_inside_iff(lambda x: isconstant(x))
+        def f(x):
+            i = 0
+            for c in x:
+                i += 1
+            return i
+
+        def main(unroll, n):
+            s = 0
+            while n > 0:
+                driver.jit_merge_point(s=s, n=n, unroll=unroll)
+                if unroll:
+                    x = A("xx")
+                else:
+                    x = A("x" * n)
+                s += f(x.x[0])
+                n -= 1
+            return s
+
+        res = self.meta_interp(main, [0, 10])
+        assert res == main(0, 10)
+        # 2 calls, one for f() and one for char_mul
+        self.check_loops(call=2)
+        res = self.meta_interp(main, [1, 10])
+        assert res == main(1, 10)
+        self.check_loops(call=0)
+
+    def test_setarrayitem_followed_by_arraycopy(self):
+        myjitdriver = JitDriver(greens = [], reds = ['n', 'sa', 'x', 'y'])
+        def f(n):
+            sa = 0
+            x = [1,2,n]
+            y = [1,2,3]
+            while n > 0:
+                myjitdriver.jit_merge_point(sa=sa, n=n, x=x, y=y)
+                y[0] = n
+                x[0:3] = y
+                sa += x[0]
+                n -= 1
+            return sa
+        res = self.meta_interp(f, [16])
+        assert res == f(16)
         
 
+
 class TestLLtype(BaseLLtypeTests, LLJitMixin):
-    pass
+    def test_tagged(self):
+        py.test.skip("implement me")
+        from pypy.rlib.objectmodel import UnboxedValue
+        class Base(object):
+            __slots__ = ()
+
+        class Int(UnboxedValue, Base):
+            __slots__ = ["a"]
+
+            def is_pos(self):
+                return self.a > 0
+
+            def dec(self):
+                return Int(self.a - 1)
+
+
+        class Float(Base):
+            def __init__(self, a):
+                self.a = a
+
+            def is_pos(self):
+                return self.a > 0
+
+            def dec(self):
+                return Float(self.a - 1)
+
+        driver = JitDriver(greens=['pc', 's'], reds=['o'])
+
+        def main(fl, n, s):
+            if s:
+                s = "--j"
+            else:
+                s = "---j"
+            if fl:
+                o = Float(float(n))
+            else:
+                o = Int(n)
+            pc = 0
+            while True:
+                driver.jit_merge_point(s=s, pc=pc, o=o)
+                c = s[pc]
+                if c == "j":
+                    driver.can_enter_jit(s=s, pc=pc, o=o)
+                    if o.is_pos():
+                        pc = 0
+                        continue
+                    else:
+                        break
+                elif c == "-":
+                    o = o.dec()
+                pc += 1
+            return pc
+        res = self.meta_interp(main, [False, 100, True], taggedpointers=True)
diff --git a/pypy/jit/metainterp/test/test_dict.py b/pypy/jit/metainterp/test/test_dict.py
--- a/pypy/jit/metainterp/test/test_dict.py
+++ b/pypy/jit/metainterp/test/test_dict.py
@@ -153,11 +153,7 @@
 
         res = self.meta_interp(f, [100], listops=True)
         assert res == f(50)
-        # XXX: ideally there would be 7 calls here, but repeated CALL_PURE with
-        # the same arguments are not folded, because we have conflicting
-        # definitions of pure, once strhash can be appropriately folded
-        # this should be decreased to seven.
-        self.check_loops({"call": 8, "guard_false": 1, "guard_no_exception": 6,
+        self.check_loops({"call": 7, "guard_false": 1, "guard_no_exception": 6,
                           "guard_true": 1, "int_and": 1, "int_gt": 1,
                           "int_is_true": 1, "int_sub": 1, "jump": 1,
                           "new_with_vtable": 1, "setfield_gc": 1})
diff --git a/pypy/jit/metainterp/test/test_heapcache.py b/pypy/jit/metainterp/test/test_heapcache.py
new file mode 100644
--- /dev/null
+++ b/pypy/jit/metainterp/test/test_heapcache.py
@@ -0,0 +1,365 @@
+from pypy.jit.metainterp.heapcache import HeapCache
+from pypy.jit.metainterp.resoperation import rop
+from pypy.jit.metainterp.history import ConstInt
+
+box1 = object()
+box2 = object()
+box3 = object()
+box4 = object()
+lengthbox1 = object()
+lengthbox2 = object()
+descr1 = object()
+descr2 = object()
+descr3 = object()
+
+index1 = ConstInt(0)
+index2 = ConstInt(1)
+
+
+class FakeEffektinfo(object):
+    EF_ELIDABLE_CANNOT_RAISE           = 0 #elidable function (and cannot raise)
+    EF_LOOPINVARIANT                   = 1 #special: call it only once per loop
+    EF_CANNOT_RAISE                    = 2 #a function which cannot raise
+    EF_ELIDABLE_CAN_RAISE              = 3 #elidable function (but can raise)
+    EF_CAN_RAISE                       = 4 #normal function (can raise)
+    EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE = 5 #can raise and force virtualizables
+    EF_RANDOM_EFFECTS                  = 6 #can do whatever
+
+    OS_ARRAYCOPY = 0
+
+    def __init__(self, extraeffect, oopspecindex):
+        self.extraeffect = extraeffect
+        self.oopspecindex = oopspecindex
+
+class FakeCallDescr(object):
+    def __init__(self, extraeffect, oopspecindex=None):
+        self.extraeffect = extraeffect
+        self.oopspecindex = oopspecindex
+
+    def get_extra_info(self):
+        return FakeEffektinfo(self.extraeffect, self.oopspecindex)
+
+class TestHeapCache(object):
+    def test_known_class_box(self):
+        h = HeapCache()
+        assert not h.is_class_known(1)
+        assert not h.is_class_known(2)
+        h.class_now_known(1)
+        assert h.is_class_known(1)
+        assert not h.is_class_known(2)
+
+        h.reset()
+        assert not h.is_class_known(1)
+        assert not h.is_class_known(2)
+
+    def test_nonstandard_virtualizable(self):
+        h = HeapCache()
+        assert not h.is_nonstandard_virtualizable(1)
+        assert not h.is_nonstandard_virtualizable(2)
+        h.nonstandard_virtualizables_now_known(1)
+        assert h.is_nonstandard_virtualizable(1)
+        assert not h.is_nonstandard_virtualizable(2)
+
+        h.reset()
+        assert not h.is_nonstandard_virtualizable(1)
+        assert not h.is_nonstandard_virtualizable(2)
+
+
+    def test_heapcache_fields(self):
+        h = HeapCache()
+        assert h.getfield(box1, descr1) is None
+        assert h.getfield(box1, descr2) is None
+        h.setfield(box1, descr1, box2)
+        assert h.getfield(box1, descr1) is box2
+        assert h.getfield(box1, descr2) is None
+        h.setfield(box1, descr2, box3)
+        assert h.getfield(box1, descr1) is box2
+        assert h.getfield(box1, descr2) is box3
+        h.setfield(box1, descr1, box3)
+        assert h.getfield(box1, descr1) is box3
+        assert h.getfield(box1, descr2) is box3
+        h.setfield(box3, descr1, box1)
+        assert h.getfield(box3, descr1) is box1
+        assert h.getfield(box1, descr1) is None
+        assert h.getfield(box1, descr2) is box3
+
+        h.reset()
+        assert h.getfield(box1, descr1) is None
+        assert h.getfield(box1, descr2) is None
+        assert h.getfield(box3, descr1) is None
+
+    def test_heapcache_read_fields_multiple(self):
+        h = HeapCache()
+        h.getfield_now_known(box1, descr1, box2)
+        h.getfield_now_known(box3, descr1, box4)
+        assert h.getfield(box1, descr1) is box2
+        assert h.getfield(box1, descr2) is None
+        assert h.getfield(box3, descr1) is box4
+        assert h.getfield(box3, descr2) is None
+
+        h.reset()
+        assert h.getfield(box1, descr1) is None
+        assert h.getfield(box1, descr2) is None
+        assert h.getfield(box3, descr1) is None
+        assert h.getfield(box3, descr2) is None
+
+    def test_heapcache_write_fields_multiple(self):
+        h = HeapCache()
+        h.setfield(box1, descr1, box2)
+        assert h.getfield(box1, descr1) is box2
+        h.setfield(box3, descr1, box4)
+        assert h.getfield(box3, descr1) is box4
+        assert h.getfield(box1, descr1) is None # box1 and box3 can alias
+
+        h = HeapCache()
+        h.new(box1)
+        h.setfield(box1, descr1, box2)
+        assert h.getfield(box1, descr1) is box2
+        h.setfield(box3, descr1, box4)
+        assert h.getfield(box3, descr1) is box4
+        assert h.getfield(box1, descr1) is None # box1 and box3 can alias
+
+        h = HeapCache()
+        h.new(box1)
+        h.new(box3)
+        h.setfield(box1, descr1, box2)
+        assert h.getfield(box1, descr1) is box2
+        h.setfield(box3, descr1, box4)
+        assert h.getfield(box3, descr1) is box4
+        assert h.getfield(box1, descr1) is box2 # box1 and box3 cannot alias
+        h.setfield(box1, descr1, box3)
+        assert h.getfield(box1, descr1) is box3
+
+
+    def test_heapcache_arrays(self):
+        h = HeapCache()
+        assert h.getarrayitem(box1, descr1, index1) is None
+        assert h.getarrayitem(box1, descr2, index1) is None
+        assert h.getarrayitem(box1, descr1, index2) is None
+        assert h.getarrayitem(box1, descr2, index2) is None
+
+        h.setarrayitem(box1, descr1, index1, box2)
+        assert h.getarrayitem(box1, descr1, index1) is box2
+        assert h.getarrayitem(box1, descr2, index1) is None
+        assert h.getarrayitem(box1, descr1, index2) is None
+        assert h.getarrayitem(box1, descr2, index2) is None
+        h.setarrayitem(box1, descr1, index2, box4)
+        assert h.getarrayitem(box1, descr1, index1) is box2
+        assert h.getarrayitem(box1, descr2, index1) is None
+        assert h.getarrayitem(box1, descr1, index2) is box4
+        assert h.getarrayitem(box1, descr2, index2) is None
+
+        h.setarrayitem(box1, descr2, index1, box3)
+        assert h.getarrayitem(box1, descr1, index1) is box2
+        assert h.getarrayitem(box1, descr2, index1) is box3
+        assert h.getarrayitem(box1, descr1, index2) is box4
+        assert h.getarrayitem(box1, descr2, index2) is None
+
+        h.setarrayitem(box1, descr1, index1, box3)
+        assert h.getarrayitem(box1, descr1, index1) is box3
+        assert h.getarrayitem(box1, descr2, index1) is box3
+        assert h.getarrayitem(box1, descr1, index2) is box4
+        assert h.getarrayitem(box1, descr2, index2) is None
+
+        h.setarrayitem(box3, descr1, index1, box1)
+        assert h.getarrayitem(box3, descr1, index1) is box1
+        assert h.getarrayitem(box1, descr1, index1) is None
+        assert h.getarrayitem(box1, descr2, index1) is box3
+        assert h.getarrayitem(box1, descr1, index2) is box4
+        assert h.getarrayitem(box1, descr2, index2) is None
+
+        h.reset()
+        assert h.getarrayitem(box1, descr1, index1) is None
+        assert h.getarrayitem(box1, descr2, index1) is None
+        assert h.getarrayitem(box3, descr1, index1) is None
+
+    def test_heapcache_array_nonconst_index(self):
+        h = HeapCache()
+        h.setarrayitem(box1, descr1, index1, box2)
+        h.setarrayitem(box1, descr1, index2, box4)
+        assert h.getarrayitem(box1, descr1, index1) is box2
+        assert h.getarrayitem(box1, descr1, index2) is box4
+        h.setarrayitem(box1, descr1, box2, box3)
+        assert h.getarrayitem(box1, descr1, index1) is None
+        assert h.getarrayitem(box1, descr1, index2) is None
+
+    def test_heapcache_read_fields_multiple_array(self):
+        h = HeapCache()
+        h.getarrayitem_now_known(box1, descr1, index1, box2)
+        h.getarrayitem_now_known(box3, descr1, index1, box4)
+        assert h.getarrayitem(box1, descr1, index1) is box2
+        assert h.getarrayitem(box1, descr2, index1) is None
+        assert h.getarrayitem(box3, descr1, index1) is box4
+        assert h.getarrayitem(box3, descr2, index1) is None
+
+        h.reset()
+        assert h.getarrayitem(box1, descr1, index1) is None
+        assert h.getarrayitem(box1, descr2, index1) is None
+        assert h.getarrayitem(box3, descr1, index1) is None
+        assert h.getarrayitem(box3, descr2, index1) is None
+
+    def test_heapcache_write_fields_multiple_array(self):
+        h = HeapCache()
+        h.setarrayitem(box1, descr1, index1, box2)
+        assert h.getarrayitem(box1, descr1, index1) is box2
+        h.setarrayitem(box3, descr1, index1, box4)
+        assert h.getarrayitem(box3, descr1, index1) is box4
+        assert h.getarrayitem(box1, descr1, index1) is None # box1 and box3 can alias
+
+        h = HeapCache()
+        h.new(box1)
+        h.setarrayitem(box1, descr1, index1, box2)
+        assert h.getarrayitem(box1, descr1, index1) is box2
+        h.setarrayitem(box3, descr1, index1, box4)
+        assert h.getarrayitem(box3, descr1, index1) is box4
+        assert h.getarrayitem(box1, descr1, index1) is None # box1 and box3 can alias
+
+        h = HeapCache()
+        h.new(box1)
+        h.new(box3)
+        h.setarrayitem(box1, descr1, index1, box2)
+        assert h.getarrayitem(box1, descr1, index1) is box2
+        h.setarrayitem(box3, descr1, index1, box4)
+        assert h.getarrayitem(box3, descr1, index1) is box4
+        assert h.getarrayitem(box1, descr1, index1) is box2 # box1 and box3 cannot alias
+        h.setarrayitem(box1, descr1, index1, box3)
+        assert h.getarrayitem(box3, descr1, index1) is box4
+        assert h.getarrayitem(box1, descr1, index1) is box3 # box1 and box3 cannot alias
+
+    def test_length_cache(self):
+        h = HeapCache()
+        h.new_array(box1, lengthbox1)
+        assert h.arraylen(box1) is lengthbox1
+
+        assert h.arraylen(box2) is None
+        h.arraylen_now_known(box2, lengthbox2)
+        assert h.arraylen(box2) is lengthbox2
+
+
+    def test_invalidate_cache(self):
+        h = HeapCache()
+        h.setfield(box1, descr1, box2)
+        h.setarrayitem(box1, descr1, index1, box2)
+        h.setarrayitem(box1, descr1, index2, box4)
+        h.invalidate_caches(rop.INT_ADD, None, [])
+        h.invalidate_caches(rop.INT_ADD_OVF, None, [])
+        h.invalidate_caches(rop.SETFIELD_RAW, None, [])
+        h.invalidate_caches(rop.SETARRAYITEM_RAW, None, [])
+        assert h.getfield(box1, descr1) is box2
+        assert h.getarrayitem(box1, descr1, index1) is box2
+        assert h.getarrayitem(box1, descr1, index2) is box4
+
+        h.invalidate_caches(
+            rop.CALL, FakeCallDescr(FakeEffektinfo.EF_ELIDABLE_CANNOT_RAISE), [])
+        assert h.getfield(box1, descr1) is box2
+        assert h.getarrayitem(box1, descr1, index1) is box2
+        assert h.getarrayitem(box1, descr1, index2) is box4
+
+        h.invalidate_caches(
+            rop.CALL_LOOPINVARIANT, FakeCallDescr(FakeEffektinfo.EF_LOOPINVARIANT), [])
+
+        h.invalidate_caches(
+            rop.CALL, FakeCallDescr(FakeEffektinfo.EF_RANDOM_EFFECTS), [])
+        assert h.getfield(box1, descr1) is None
+        assert h.getarrayitem(box1, descr1, index1) is None
+        assert h.getarrayitem(box1, descr1, index2) is None
+
+
+    def test_replace_box(self):
+        h = HeapCache()
+        h.setfield(box1, descr1, box2)
+        h.setfield(box1, descr2, box3)
+        h.setfield(box2, descr3, box3)
+        h.replace_box(box1, box4)
+        assert h.getfield(box1, descr1) is None
+        assert h.getfield(box1, descr2) is None
+        assert h.getfield(box4, descr1) is box2
+        assert h.getfield(box4, descr2) is box3
+        assert h.getfield(box2, descr3) is box3
+
+    def test_replace_box_array(self):
+        h = HeapCache()
+        h.setarrayitem(box1, descr1, index1, box2)
+        h.setarrayitem(box1, descr2, index1, box3)
+        h.arraylen_now_known(box1, lengthbox1)
+        h.setarrayitem(box2, descr1, index2, box1)
+        h.setarrayitem(box3, descr2, index2, box1)
+        h.setarrayitem(box2, descr3, index2, box3)
+        h.replace_box(box1, box4)
+        assert h.getarrayitem(box1, descr1, index1) is None
+        assert h.getarrayitem(box1, descr2, index1) is None
+        assert h.arraylen(box1) is None
+        assert h.arraylen(box4) is lengthbox1
+        assert h.getarrayitem(box4, descr1, index1) is box2
+        assert h.getarrayitem(box4, descr2, index1) is box3
+        assert h.getarrayitem(box2, descr1, index2) is box4
+        assert h.getarrayitem(box3, descr2, index2) is box4
+        assert h.getarrayitem(box2, descr3, index2) is box3
+
+        h.replace_box(lengthbox1, lengthbox2)
+        assert h.arraylen(box4) is lengthbox2
+
+    def test_ll_arraycopy(self):
+        h = HeapCache()
+        h.new_array(box1, lengthbox1)
+        h.setarrayitem(box1, descr1, index1, box2)
+        h.new_array(box2, lengthbox1)
+        # Just need the destination box for this call
+        h.invalidate_caches(
+            rop.CALL,
+            FakeCallDescr(FakeEffektinfo.EF_CANNOT_RAISE, FakeEffektinfo.OS_ARRAYCOPY),
+            [None, None, box2, None, None]
+        )
+        assert h.getarrayitem(box1, descr1, index1) is box2
+        h.invalidate_caches(
+            rop.CALL,
+            FakeCallDescr(FakeEffektinfo.EF_CANNOT_RAISE, FakeEffektinfo.OS_ARRAYCOPY),
+            [None, None, box3, None, None]
+        )
+        assert h.getarrayitem(box1, descr1, index1) is None
+
+        h.setarrayitem(box4, descr1, index1, box2)
+        assert h.getarrayitem(box4, descr1, index1) is box2
+        h.invalidate_caches(
+            rop.CALL,
+            FakeCallDescr(FakeEffektinfo.EF_CANNOT_RAISE, FakeEffektinfo.OS_ARRAYCOPY),
+            [None, None, box2, None, None]
+        )
+        assert h.getarrayitem(box4, descr1, index1) is None
+
+    def test_unescaped(self):
+        h = HeapCache()
+        assert not h.is_unescaped(box1)
+        h.new(box2)
+        assert h.is_unescaped(box2)
+        h.invalidate_caches(rop.SETFIELD_GC, None, [box2, box1])
+        assert h.is_unescaped(box2)
+        h.invalidate_caches(rop.SETFIELD_GC, None, [box1, box2])
+        assert not h.is_unescaped(box2)
+
+    def test_unescaped_testing(self):
+        h = HeapCache()
+        h.new(box1)
+        h.new(box2)
+        assert h.is_unescaped(box1)
+        assert h.is_unescaped(box2)
+        # Putting a virtual inside of another virtual doesn't escape it.
+        h.invalidate_caches(rop.SETFIELD_GC, None, [box1, box2])
+        assert h.is_unescaped(box2)
+        # Reading a field from a virtual doesn't escape it.
+        h.invalidate_caches(rop.GETFIELD_GC, None, [box1])
+        assert h.is_unescaped(box1)
+        # Escaping a virtual transitively escapes anything inside of it.
+        assert not h.is_unescaped(box3)
+        h.invalidate_caches(rop.SETFIELD_GC, None, [box3, box1])
+        assert not h.is_unescaped(box1)
+        assert not h.is_unescaped(box2)
+
+    def test_unescaped_array(self):
+        h = HeapCache()
+        h.new_array(box1, lengthbox1)
+        assert h.is_unescaped(box1)
+        h.invalidate_caches(rop.SETARRAYITEM_GC, None, [box1, index1, box2])
+        assert h.is_unescaped(box1)
+        h.invalidate_caches(rop.SETARRAYITEM_GC, None, [box2, index1, box1])
+        assert not h.is_unescaped(box1)
\ No newline at end of file
diff --git a/pypy/jit/metainterp/test/test_list.py b/pypy/jit/metainterp/test/test_list.py
--- a/pypy/jit/metainterp/test/test_list.py
+++ b/pypy/jit/metainterp/test/test_list.py
@@ -34,7 +34,7 @@
                 l = [x + 1]
                 n -= 1
             return l[0]
-        
+
         res = self.meta_interp(f, [10], listops=True)
         assert res == f(10)
         self.check_all_virtualized()
@@ -60,7 +60,7 @@
 
     def test_ll_fixed_setitem_fast(self):
         jitdriver = JitDriver(greens = [], reds = ['n', 'l'])
-        
+
         def f(n):
             l = [1, 2, 3]
 
@@ -116,7 +116,7 @@
         assert res == f(10)
         py.test.skip("'[non-null] * n' gives a residual call so far")
         self.check_loops(setarrayitem_gc=0, getarrayitem_gc=0, call=0)
-    
+
     def test_arraycopy_simpleoptimize(self):
         def f():
             l = [1, 2, 3, 4]
@@ -208,6 +208,26 @@
         assert res == f(15)
         self.check_loops(guard_exception=0)
 
+    def test_virtual_resize(self):
+        jitdriver = JitDriver(greens = [], reds = ['n', 's'])
+        def f(n):
+            s = 0
+            while n > 0:
+                jitdriver.jit_merge_point(n=n, s=s)
+                lst = []
+                lst += [1]
+                n -= len(lst)
+                s += lst[0]
+                lst.pop()
+                lst.append(1)
+                s /= lst.pop()
+            return s
+        res = self.meta_interp(f, [15], listops=True)
+        assert res == f(15)
+        self.check_loops({"int_add": 1, "int_sub": 1, "int_gt": 1,
+                          "guard_true": 1, "jump": 1})
+
+
 class TestOOtype(ListTests, OOJitMixin):
     pass
 
@@ -236,8 +256,6 @@
             return a * b
         res = self.meta_interp(f, [37])
         assert res == f(37)
-        # There is the one actual field on a, plus 2 getfield's from the list
-        # itself, 1 to get the length (which is then incremented and passed to
-        # the resize func), and then a read of the items field to actually
-        # perform the setarrayitem on
-        self.check_loops(getfield_gc=5, everywhere=True)
+        # There is the one actual field on a, plus several fields on the list
+        # itself
+        self.check_loops(getfield_gc=10, everywhere=True)
diff --git a/pypy/jit/metainterp/test/test_slist.py b/pypy/jit/metainterp/test/test_slist.py
--- a/pypy/jit/metainterp/test/test_slist.py
+++ b/pypy/jit/metainterp/test/test_slist.py
@@ -5,7 +5,6 @@
 class ListTests(object):
 
     def test_basic_list(self):
-        py.test.skip("not yet")
         myjitdriver = JitDriver(greens = [], reds = ['n', 'lst'])
         def f(n):
             lst = []
@@ -34,7 +33,7 @@
             return m
         res = self.interp_operations(f, [11], listops=True)
         assert res == 49
-        self.check_operations_history(call=5)
+        self.check_operations_history(call=3)
 
     def test_list_of_voids(self):
         myjitdriver = JitDriver(greens = [], reds = ['n', 'lst'])
@@ -93,7 +92,7 @@
             return x
         res = self.meta_interp(f, [-2], listops=True)
         assert res == 41
-        self.check_loops(call=1, guard_value=0)
+        self.check_loops(call=0, guard_value=0)
 
 # we don't support resizable lists on ootype
 #class TestOOtype(ListTests, OOJitMixin):
diff --git a/pypy/jit/metainterp/test/test_string.py b/pypy/jit/metainterp/test/test_string.py
--- a/pypy/jit/metainterp/test/test_string.py
+++ b/pypy/jit/metainterp/test/test_string.py
@@ -1,9 +1,11 @@
 import py
+
+from pypy.jit.codewriter.policy import StopAtXPolicy
+from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin
+from pypy.rlib.debug import debug_print
 from pypy.rlib.jit import JitDriver, dont_look_inside, we_are_jitted
-from pypy.rlib.debug import debug_print
-from pypy.jit.codewriter.policy import StopAtXPolicy
+from pypy.rlib.rstring import StringBuilder
 from pypy.rpython.ootypesystem import ootype
-from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin
 
 
 class StringTests:
@@ -27,7 +29,7 @@
             return i
         res = self.meta_interp(f, [10, True, _str('h')], listops=True)
         assert res == 5
-        self.check_loops(**{self.CALL: 1, self.CALL_PURE: 0})
+        self.check_loops(**{self.CALL: 1, self.CALL_PURE: 0, 'everywhere': True})
 
     def test_eq_folded(self):
         _str = self._str
@@ -327,7 +329,7 @@
     def test_str_slice_len_surviving(self):
         _str = self._str
         longstring = _str("Unrolling Trouble")
-        mydriver = JitDriver(reds = ['i', 'a', 'sa'], greens = []) 
+        mydriver = JitDriver(reds = ['i', 'a', 'sa'], greens = [])
         def f(a):
             i = sa = a
             while i < len(longstring):
@@ -343,7 +345,7 @@
         fillers = _str("abcdefghijklmnopqrstuvwxyz")
         data = _str("ABCDEFGHIJKLMNOPQRSTUVWXYZ")
 
-        mydriver = JitDriver(reds = ['line', 'noise', 'res'], greens = []) 
+        mydriver = JitDriver(reds = ['line', 'noise', 'res'], greens = [])
         def f():
             line = data
             noise = fillers
@@ -370,7 +372,7 @@
             def __init__(self, value):
                 self.value = value
         mydriver = JitDriver(reds = ['ratio', 'line', 'noise', 'res'],
-                             greens = []) 
+                             greens = [])
         def f():
             line = Str(data)
             noise = Str(fillers)
@@ -408,7 +410,7 @@
             return len(sa)
         assert self.meta_interp(f, [16]) == f(16)
 
-    def test_loop_invariant_string_slize(self):
+    def test_loop_invariant_string_slice(self):
         _str = self._str
         mydriver = JitDriver(reds = ['i', 'n', 'sa', 's', 's1'], greens = [])
         def f(n, c):
@@ -425,7 +427,7 @@
             return sa
         assert self.meta_interp(f, [16, 'a']) == f(16, 'a')
 
-    def test_loop_invariant_string_slize_boxed(self):
+    def test_loop_invariant_string_slice_boxed(self):
         class Str(object):
             def __init__(self, value):
                 self.value = value
@@ -445,7 +447,7 @@
             return sa
         assert self.meta_interp(f, [16, 'a']) == f(16, 'a')
 
-    def test_loop_invariant_string_slize_in_array(self):
+    def test_loop_invariant_string_slice_in_array(self):
         _str = self._str
         mydriver = JitDriver(reds = ['i', 'n', 'sa', 's', 's1'], greens = [])
         def f(n, c):
@@ -513,7 +515,7 @@
                 m -= 1
             return 42
         self.meta_interp(f, [6, 7])
-        self.check_loops(call=3,    # str(), _str(), escape()
+        self.check_loops(call=1,    # escape()
                          newunicode=1, unicodegetitem=0,
                          unicodesetitem=1, copyunicodecontent=1)
 
@@ -536,3 +538,55 @@
         self.check_loops(call_pure=0, call=1,
                          newunicode=0, unicodegetitem=0,
                          unicodesetitem=0, copyunicodecontent=0)
+
+    def test_join_chars(self):
+        jitdriver = JitDriver(reds=['a', 'b', 'c', 'i'], greens=[])
+        def f(a, b, c):
+            i = 0
+            while i < 10:
+                jitdriver.jit_merge_point(a=a, b=b, c=c, i=i)
+                x = []
+                if a:
+                    x.append("a")
+                if b:
+                    x.append("b")
+                if c:
+                    x.append("c")
+                i += len("".join(x))
+            return i
+        res = self.meta_interp(f, [1, 1, 1])
+        assert res == f(True, True, True)
+        # The "".join should be unrolled, since the length of x is known since
+        # it is virtual, ensure there are no calls to ll_join_chars, or
+        # allocations.
+        self.check_loops({
+            "guard_true": 5, "int_is_true": 3, "int_lt": 2, "int_add": 2, "jump": 2,
+        }, everywhere=True)
+
+    def test_virtual_copystringcontent(self):
+        jitdriver = JitDriver(reds=['n', 'result'], greens=[])
+        def main(n):
+            result = 0
+            while n >= 0:
+                jitdriver.jit_merge_point(n=n, result=result)
+                b = StringBuilder(6)
+                b.append("Hello!")
+                result += ord(b.build()[0])
+                n -= 1
+            return result
+        res = self.meta_interp(main, [9])
+        assert res == main(9)
+
+    def test_virtual_copystringcontent2(self):
+        jitdriver = JitDriver(reds=['n', 'result'], greens=[])
+        def main(n):
+            result = 0
+            while n >= 0:
+                jitdriver.jit_merge_point(n=n, result=result)
+                b = StringBuilder(6)
+                b.append("Hello!")
+                result += ord((b.build() + "xyz")[0])
+                n -= 1
+            return result
+        res = self.meta_interp(main, [9])
+        assert res == main(9)
diff --git a/pypy/jit/metainterp/test/test_tracingopts.py b/pypy/jit/metainterp/test/test_tracingopts.py
--- a/pypy/jit/metainterp/test/test_tracingopts.py
+++ b/pypy/jit/metainterp/test/test_tracingopts.py
@@ -1,7 +1,10 @@
+import sys
+
+from pypy.jit.metainterp.test.support import LLJitMixin
+from pypy.rlib import jit
+from pypy.rlib.rarithmetic import ovfcheck
+
 import py
-import sys
-from pypy.rlib import jit
-from pypy.jit.metainterp.test.support import LLJitMixin
 
 
 class TestLLtype(LLJitMixin):
@@ -257,6 +260,28 @@
         self.check_operations_history(setarrayitem_gc=2, setfield_gc=2,
                                       getarrayitem_gc=0, getfield_gc=2)
 
+    def test_promote_changes_array_cache(self):
+        a1 = [0, 0]
+        a2 = [0, 0]
+        def fn(n):
+            if n > 0:
+                a = a1
+            else:
+                a = a2
+            a[0] = n
+            jit.hint(n, promote=True)
+            x1 = a[0]
+            jit.hint(x1, promote=True)
+            a[n - n] = n + 1
+            return a[0] + x1
+        res = self.interp_operations(fn, [7])
+        assert res == 7 + 7 + 1
+        self.check_operations_history(getarrayitem_gc=0, guard_value=1)
+        res = self.interp_operations(fn, [-7])
+        assert res == -7 - 7 + 1
+        self.check_operations_history(getarrayitem_gc=0, guard_value=1)
+
+
     def test_list_caching(self):
         a1 = [0, 0]
         a2 = [0, 0]
@@ -357,7 +382,7 @@
         assert res == f(10, 1, 1)
         self.check_history(getarrayitem_gc=0, getfield_gc=0)
 
-    def test_heap_caching_pure(self):
+    def test_heap_caching_array_pure(self):
         class A(object):
             pass
         p1 = A()
@@ -405,3 +430,164 @@
         assert res == -7 + 7
         self.check_operations_history(getfield_gc=0)
         return
+
+    def test_heap_caching_multiple_objects(self):
+        class Gbl(object):
+            pass
+        g = Gbl()
+        class A(object):
+            pass
+        a1 = A()
+        g.a1 = a1
+        a1.x = 7
+        a2 = A()
+        g.a2 = a2
+        a2.x = 7
+        def gn(a1, a2):
+            return a1.x + a2.x
+        def fn(n):
+            if n < 0:
+                a1 = A()
+                g.a1 = a1
+                a1.x = n
+                a2 = A()
+                g.a2 = a2
+                a2.x = n - 1
+            else:
+                a1 = g.a1
+                a2 = g.a2
+            return a1.x + a2.x + gn(a1, a2)
+        res = self.interp_operations(fn, [-7])
+        assert res == 2 * -7 + 2 * -8
+        self.check_operations_history(setfield_gc=4, getfield_gc=0)
+        res = self.interp_operations(fn, [7])
+        assert res == 4 * 7
+        self.check_operations_history(getfield_gc=4)
+
+    def test_heap_caching_multiple_tuples(self):
+        class Gbl(object):
+            pass
+        g = Gbl()
+        def gn(a1, a2):
+            return a1[0] + a2[0]
+        def fn(n):
+            a1 = (n, )
+            g.a = a1
+            a2 = (n - 1, )
+            g.a = a2
+            jit.promote(n)
+            return a1[0] + a2[0] + gn(a1, a2)
+        res = self.interp_operations(fn, [7])
+        assert res == 2 * 7 + 2 * 6
+        self.check_operations_history(getfield_gc_pure=0)
+        res = self.interp_operations(fn, [-7])
+        assert res == 2 * -7 + 2 * -8
+        self.check_operations_history(getfield_gc_pure=0)
+
+    def test_heap_caching_multiple_arrays(self):
+        class Gbl(object):
+            pass
+        g = Gbl()
+        def fn(n):
+            a1 = [n, n, n]
+            g.a = a1
+            a1[0] = n
+            a2 = [n, n, n]
+            g.a = a2
+            a2[0] = n - 1
+            return a1[0] + a2[0] + a1[0] + a2[0]
+        res = self.interp_operations(fn, [7])
+        assert res == 2 * 7 + 2 * 6
+        self.check_operations_history(getarrayitem_gc=0)
+        res = self.interp_operations(fn, [-7])
+        assert res == 2 * -7 + 2 * -8
+        self.check_operations_history(getarrayitem_gc=0)
+
+    def test_heap_caching_multiple_arrays_getarrayitem(self):
+        class Gbl(object):
+            pass
+        g = Gbl()
+        g.a1 = [7, 8, 9]
+        g.a2 = [8, 9, 10, 11]
+
+        def fn(i):
+            if i < 0:
+                g.a1 = [7, 8, 9]
+                g.a2 = [7, 8, 9, 10]
+            jit.promote(i)
+            a1 = g.a1
+            a1[i + 1] = 15 # make lists mutable
+            a2 = g.a2
+            a2[i + 1] = 19
+            return a1[i] + a2[i] + a1[i] + a2[i]
+        res = self.interp_operations(fn, [0])
+        assert res == 2 * 7 + 2 * 8
+        self.check_operations_history(getarrayitem_gc=2)
+
+
+    def test_heap_caching_multiple_lists(self):
+        class Gbl(object):
+            pass
+        g = Gbl()
+        g.l = []
+        def fn(n):
+            if n < -100:
+                g.l.append(1)
+            a1 = [n, n, n]
+            g.l = a1
+            a1[0] = n
+            a2 = [n, n, n]
+            g.l = a2
+            a2[0] = n - 1
+            return a1[0] + a2[0] + a1[0] + a2[0]
+        res = self.interp_operations(fn, [7])
+        assert res == 2 * 7 + 2 * 6
+        self.check_operations_history(getarrayitem_gc=0, getfield_gc=0)
+        res = self.interp_operations(fn, [-7])
+        assert res == 2 * -7 + 2 * -8
+        self.check_operations_history(getarrayitem_gc=0, getfield_gc=0)
+
+    def test_length_caching(self):
+        class Gbl(object):
+            pass
+        g = Gbl()
+        g.a = [0] * 7
+        def fn(n):
+            a = g.a
+            res = len(a) + len(a)
+            a1 = [0] * n
+            g.a = a1
+            return len(a1) + res
+        res = self.interp_operations(fn, [7])
+        assert res == 7 * 3
+        self.check_operations_history(arraylen_gc=1)
+
+    def test_arraycopy(self):
+        class Gbl(object):
+            pass
+        g = Gbl()
+        g.a = [0] * 7
+        def fn(n):
+            assert n >= 0
+            a = g.a
+            x = [0] * n
+            x[2] = 21
+            return len(a[:n]) + x[2]
+        res = self.interp_operations(fn, [3])
+        assert res == 24
+        self.check_operations_history(getarrayitem_gc=0)
+
+    def test_fold_int_add_ovf(self):
+        def fn(n):
+            jit.promote(n)
+            try:
+                n = ovfcheck(n + 1)
+            except OverflowError:
+                return 12
+            else:
+                return n
+        res = self.interp_operations(fn, [3])
+        assert res == 4
+        self.check_operations_history(int_add_ovf=0)
+        res = self.interp_operations(fn, [sys.maxint])
+        assert res == 12
\ No newline at end of file
diff --git a/pypy/jit/metainterp/test/test_virtualstate.py b/pypy/jit/metainterp/test/test_virtualstate.py
--- a/pypy/jit/metainterp/test/test_virtualstate.py
+++ b/pypy/jit/metainterp/test/test_virtualstate.py
@@ -2,7 +2,7 @@
 import py
 from pypy.jit.metainterp.optimize import InvalidLoop
 from pypy.jit.metainterp.optimizeopt.virtualstate import VirtualStateInfo, VStructStateInfo, \
-     VArrayStateInfo, NotVirtualStateInfo, VirtualState
+     VArrayStateInfo, NotVirtualStateInfo, VirtualState, ShortBoxes
 from pypy.jit.metainterp.optimizeopt.optimizer import OptValue
 from pypy.jit.metainterp.history import BoxInt, BoxFloat, BoxPtr, ConstInt, ConstPtr
 from pypy.rpython.lltypesystem import lltype
@@ -11,6 +11,7 @@
 from pypy.jit.metainterp.history import TreeLoop, LoopToken
 from pypy.jit.metainterp.optimizeopt.test.test_optimizeopt import FakeDescr, FakeMetaInterpStaticData
 from pypy.jit.metainterp.optimize import RetraceLoop
+from pypy.jit.metainterp.resoperation import ResOperation, rop
 
 class TestBasic:
     someptr1 = LLtypeMixin.myptr
@@ -129,6 +130,7 @@
             info.fieldstate = [info]
             assert info.generalization_of(info, {}, {})
 
+
 class BaseTestGenerateGuards(BaseTest):
     def guards(self, info1, info2, box, expected):
         info1.position = info2.position = 0
@@ -910,3 +912,111 @@
 class TestLLtypeBridges(BaseTestBridges, LLtypeMixin):
     pass
 
+class FakeOptimizer:
+    def make_equal_to(*args):
+        pass
+    def getvalue(*args):
+        pass
+
+class TestShortBoxes:
+    p1 = BoxPtr()
+    p2 = BoxPtr()
+    p3 = BoxPtr()
+    p4 = BoxPtr()
+    i1 = BoxInt()
+    i2 = BoxInt()
+    i3 = BoxInt()
+    i4 = BoxInt()
+    
+    def test_short_box_duplication_direct(self):
+        class Optimizer(FakeOptimizer):
+            def produce_potential_short_preamble_ops(_self, sb):
+                sb.add_potential(ResOperation(rop.GETFIELD_GC, [self.p1], self.i1))
+                sb.add_potential(ResOperation(rop.GETFIELD_GC, [self.p2], self.i1))
+        sb = ShortBoxes(Optimizer(), [self.p1, self.p2])
+        assert len(sb.short_boxes) == 4
+        assert self.i1 in sb.short_boxes
+        assert sum([op.result is self.i1 for op in sb.short_boxes.values() if op]) == 1
+
+    def test_dont_duplicate_potential_boxes(self):
+        class Optimizer(FakeOptimizer):
+            def produce_potential_short_preamble_ops(_self, sb):
+                sb.add_potential(ResOperation(rop.GETFIELD_GC, [self.p1], self.i1))
+                sb.add_potential(ResOperation(rop.GETFIELD_GC, [BoxPtr()], self.i1))
+                sb.add_potential(ResOperation(rop.INT_NEG, [self.i1], self.i2))
+                sb.add_potential(ResOperation(rop.INT_ADD, [ConstInt(7), self.i2],
+                                              self.i3))
+        sb = ShortBoxes(Optimizer(), [self.p1, self.p2])
+        assert len(sb.short_boxes) == 5
+
+    def test_prioritize1(self):
+        class Optimizer(FakeOptimizer):
+            def produce_potential_short_preamble_ops(_self, sb):
+                sb.add_potential(ResOperation(rop.GETFIELD_GC, [self.p1], self.i1))
+                sb.add_potential(ResOperation(rop.GETFIELD_GC, [self.p2], self.i1))
+                sb.add_potential(ResOperation(rop.INT_NEG, [self.i1], self.i2))
+        sb = ShortBoxes(Optimizer(), [self.p1, self.p2])
+        assert len(sb.short_boxes.values()) == 5
+        int_neg = [op for op in sb.short_boxes.values()
+                   if op and op.getopnum() == rop.INT_NEG]
+        assert len(int_neg) == 1
+        int_neg = int_neg[0]
+        getfield = [op for op in sb.short_boxes.values()
+                    if op and op.result == int_neg.getarg(0)]
+        assert len(getfield) == 1
+        assert getfield[0].getarg(0) in [self.p1, self.p2]
+
+    def test_prioritize1bis(self):
+        class Optimizer(FakeOptimizer):
+            def produce_potential_short_preamble_ops(_self, sb):
+                sb.add_potential(ResOperation(rop.GETFIELD_GC, [self.p1], self.i1),
+                                 synthetic=True)
+                sb.add_potential(ResOperation(rop.GETFIELD_GC, [self.p2], self.i1),
+                                 synthetic=True)
+                sb.add_potential(ResOperation(rop.INT_NEG, [self.i1], self.i2))
+        sb = ShortBoxes(Optimizer(), [self.p1, self.p2])
+        assert len(sb.short_boxes.values()) == 5
+        int_neg = [op for op in sb.short_boxes.values()
+                   if op and op.getopnum() == rop.INT_NEG]
+        assert len(int_neg) == 1
+        int_neg = int_neg[0]
+        getfield = [op for op in sb.short_boxes.values()
+                    if op and op.result == int_neg.getarg(0)]
+        assert len(getfield) == 1
+        assert getfield[0].getarg(0) in [self.p1, self.p2]
+        
+    def test_prioritize2(self):
+        class Optimizer(FakeOptimizer):
+            def produce_potential_short_preamble_ops(_self, sb):
+                sb.add_potential(ResOperation(rop.GETFIELD_GC, [self.p1], self.i1),
+                                 synthetic=True)
+                sb.add_potential(ResOperation(rop.GETFIELD_GC, [self.p2], self.i1))
+                sb.add_potential(ResOperation(rop.INT_NEG, [self.i1], self.i2))
+        sb = ShortBoxes(Optimizer(), [self.p1, self.p2])
+        assert len(sb.short_boxes.values()) == 5
+        int_neg = [op for op in sb.short_boxes.values()
+                   if op and op.getopnum() == rop.INT_NEG]
+        assert len(int_neg) == 1
+        int_neg = int_neg[0]
+        getfield = [op for op in sb.short_boxes.values()
+                    if op and op.result == int_neg.getarg(0)]
+        assert len(getfield) == 1
+        assert getfield[0].getarg(0) == self.p2
+        
+    def test_prioritize3(self):
+        class Optimizer(FakeOptimizer):
+            def produce_potential_short_preamble_ops(_self, sb):
+                sb.add_potential(ResOperation(rop.GETFIELD_GC, [self.p1], self.i1))
+                sb.add_potential(ResOperation(rop.GETFIELD_GC, [self.p2], self.i1),
+                                 synthetic=True)
+                sb.add_potential(ResOperation(rop.INT_NEG, [self.i1], self.i2))
+        sb = ShortBoxes(Optimizer(), [self.p1, self.p2])
+        assert len(sb.short_boxes.values()) == 5
+        int_neg = [op for op in sb.short_boxes.values()
+                   if op and op.getopnum() == rop.INT_NEG]
+        assert len(int_neg) == 1
+        int_neg = int_neg[0]
+        getfield = [op for op in sb.short_boxes.values()
+                    if op and op.result == int_neg.getarg(0)]
+        assert len(getfield) == 1
+        assert getfield[0].getarg(0) == self.p1
diff --git a/pypy/jit/metainterp/warmspot.py b/pypy/jit/metainterp/warmspot.py
--- a/pypy/jit/metainterp/warmspot.py
+++ b/pypy/jit/metainterp/warmspot.py
@@ -53,6 +53,8 @@
         extraconfigopts = {'translation.list_comprehension_operations': True}
     else:
         extraconfigopts = {}
+    if kwds.pop("taggedpointers", False):
+        extraconfigopts["translation.taggedpointers"] = True
     interp, graph = get_interpreter(function, args,
                                     backendopt=False,  # will be done below
                                     type_system=type_system,
@@ -130,8 +132,15 @@
     results = _find_jit_marker(graphs, 'jit_merge_point')
     if not results:
         raise Exception("no jit_merge_point found!")
+    seen = set([graph for graph, block, pos in results])
+    assert len(seen) == len(results), (
+        "found several jit_merge_points in the same graph")
     return results
 
+def locate_jit_merge_point(graph):
+    [(graph, block, pos)] = find_jit_merge_points([graph])
+    return block, pos, block.operations[pos]
+
 def find_set_param(graphs):
     return _find_jit_marker(graphs, 'set_param')
 
@@ -235,7 +244,7 @@
     def split_graph_and_record_jitdriver(self, graph, block, pos):
         op = block.operations[pos]
         jd = JitDriverStaticData()
-        jd._jit_merge_point_pos = (graph, op)
+        jd._jit_merge_point_in = graph
         args = op.args[2:]
         s_binding = self.translator.annotator.binding
         jd._portal_args_s = [s_binding(v) for v in args]
@@ -504,7 +513,8 @@
             self.make_args_specification(jd)
 
     def make_args_specification(self, jd):
-        graph, op = jd._jit_merge_point_pos
+        graph = jd._jit_merge_point_in
+        _, _, op = locate_jit_merge_point(graph)
         greens_v, reds_v = support.decode_hp_hint_args(op)
         ALLARGS = [v.concretetype for v in (greens_v + reds_v)]
         jd._green_args_spec = [v.concretetype for v in greens_v]
@@ -552,7 +562,7 @@
             assert jitdriver in sublists, \
                    "can_enter_jit with no matching jit_merge_point"
             jd, sublist = sublists[jitdriver]
-            origportalgraph = jd._jit_merge_point_pos[0]
+            origportalgraph = jd._jit_merge_point_in
             if graph is not origportalgraph:
                 sublist.append((graph, block, index))
                 jd.no_loop_header = False
@@ -582,7 +592,7 @@
             can_enter_jits = [(jd.portal_graph, jd.portal_graph.startblock, 0)]
 
         for graph, block, index in can_enter_jits:
-            if graph is jd._jit_merge_point_pos[0]:
+            if graph is jd._jit_merge_point_in:
                 continue
 
             op = block.operations[index]
@@ -640,7 +650,7 @@
         #           while 1:
         #               more stuff
         #
-        origportalgraph = jd._jit_merge_point_pos[0]
+        origportalgraph = jd._jit_merge_point_in
         portalgraph = jd.portal_graph
         PORTALFUNC = jd._PORTAL_FUNCTYPE
 
@@ -794,14 +804,7 @@
         # ____________________________________________________________
         # Now mutate origportalgraph to end with a call to portal_runner_ptr
         #
-        _, op = jd._jit_merge_point_pos
-        for origblock in origportalgraph.iterblocks():
-            if op in origblock.operations:
-                break
-        else:
-            assert False, "lost the operation %r in the graph %r" % (
-                op, origportalgraph)
-        origindex = origblock.operations.index(op)
+        origblock, origindex, op = locate_jit_merge_point(origportalgraph)
         assert op.opname == 'jit_marker'
         assert op.args[0].value == 'jit_merge_point'
         greens_v, reds_v = support.decode_hp_hint_args(op)
diff --git a/pypy/jit/metainterp/warmstate.py b/pypy/jit/metainterp/warmstate.py
--- a/pypy/jit/metainterp/warmstate.py
+++ b/pypy/jit/metainterp/warmstate.py
@@ -367,9 +367,9 @@
             # ---------- execute assembler ----------
             while True:     # until interrupted by an exception
                 metainterp_sd.profiler.start_running()
-                debug_start("jit-running")
+                #debug_start("jit-running")
                 fail_descr = warmrunnerdesc.execute_token(loop_token)
-                debug_stop("jit-running")
+                #debug_stop("jit-running")
                 metainterp_sd.profiler.end_running()
                 loop_token = None     # for test_memmgr
                 if vinfo is not None:
diff --git a/pypy/jit/tl/pypyjit.py b/pypy/jit/tl/pypyjit.py
--- a/pypy/jit/tl/pypyjit.py
+++ b/pypy/jit/tl/pypyjit.py
@@ -40,7 +40,7 @@
 config.objspace.usemodules.array = False
 config.objspace.usemodules._weakref = True
 config.objspace.usemodules._sre = False
-config.objspace.usemodules._lsprof = True
+config.objspace.usemodules._lsprof = False
 #
 config.objspace.usemodules._ffi = True
 config.objspace.usemodules.micronumpy = False
@@ -77,7 +77,7 @@
 
 def read_code():
     from pypy.module.marshal.interp_marshal import dumps
-    
+
     filename = 'pypyjit_demo.py'
     source = readfile(filename)
     ec = space.getexecutioncontext()
diff --git a/pypy/jit/tl/pypyjit_demo.py b/pypy/jit/tl/pypyjit_demo.py
--- a/pypy/jit/tl/pypyjit_demo.py
+++ b/pypy/jit/tl/pypyjit_demo.py
@@ -2,22 +2,16 @@
 pypyjit.set_param(threshold=200)
 
 
-def main(a, b):
-    i = sa = 0
-    while i < 300:
-        if a > 0: # Specialises the loop
-            pass
-        if b < 2 and b > 0:
-            pass
-        if (a >> b) >= 0:
-            sa += 1
-        if (a << b) > 2:
-            sa += 10000
-        i += 1
-    return sa
+def f(n):
+    pairs = [(0.0, 1.0), (2.0, 3.0)] * n
+    mag = 0
+    for (x1, x2) in pairs:
+        dx = x1 - x2
+        mag += ((dx * dx ) ** (-1.5))            
+    return n
 
 try:
-    print main(2, 1)
+    print f(301)
 
 except Exception, e:
     print "Exception: ", type(e)
diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py
--- a/pypy/module/__builtin__/functional.py
+++ b/pypy/module/__builtin__/functional.py
@@ -3,13 +3,13 @@
 
 """
 
+from pypy.interpreter.baseobjspace import Wrappable
 from pypy.interpreter.error import OperationError
-from pypy.interpreter.gateway import NoneNotWrapped
-from pypy.interpreter.gateway import interp2app, unwrap_spec
+from pypy.interpreter.gateway import NoneNotWrapped, interp2app, unwrap_spec
 from pypy.interpreter.typedef import TypeDef
-from pypy.interpreter.baseobjspace import Wrappable
+from pypy.rlib import jit
+from pypy.rlib.objectmodel import specialize
 from pypy.rlib.rarithmetic import r_uint, intmask
-from pypy.rlib.objectmodel import specialize
 from pypy.rlib.rbigint import rbigint
 
 
@@ -134,29 +134,15 @@
 
 
 @specialize.arg(2)
+ at jit.look_inside_iff(lambda space, args, implementation_of:
+    jit.isconstant(len(args.arguments_w)) and
+    len(args.arguments_w) == 2
+)
 def min_max(space, args, implementation_of):
     if implementation_of == "max":
         compare = space.gt
     else:
         compare = space.lt
-
-    args_w = args.arguments_w
-    if len(args_w) == 2 and not args.keywords:
-        # simple case, suitable for the JIT
-        w_arg0, w_arg1 = args_w
-        if space.is_true(compare(w_arg0, w_arg1)):
-            return w_arg0
-        else:
-            return w_arg1
-    else:
-        return min_max_loop(space, args, implementation_of)
-
- at specialize.arg(2)
-def min_max_loop(space, args, implementation_of):
-    if implementation_of == "max":
-        compare = space.gt
-    else:
-        compare = space.lt
     args_w = args.arguments_w
     if len(args_w) > 1:
         w_sequence = space.newtuple(args_w)
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -8,6 +8,7 @@
     appleveldefs = {}
 
     interpleveldefs = {
+        "StringBuilder": "interp_builders.W_StringBuilder",
         "UnicodeBuilder": "interp_builders.W_UnicodeBuilder",
     }
 
diff --git a/pypy/module/__pypy__/interp_builders.py b/pypy/module/__pypy__/interp_builders.py
--- a/pypy/module/__pypy__/interp_builders.py
+++ b/pypy/module/__pypy__/interp_builders.py
@@ -2,49 +2,55 @@
 from pypy.interpreter.error import OperationError
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.typedef import TypeDef
-from pypy.rlib.rstring import UnicodeBuilder
+from pypy.rlib.rstring import UnicodeBuilder, StringBuilder
+from pypy.tool.sourcetools import func_with_new_name
 
 
-class W_UnicodeBuilder(Wrappable):
-    def __init__(self, space, size):
-        if size < 0:
-            self.builder = UnicodeBuilder()
-        else:
-            self.builder = UnicodeBuilder(size)
-        self.done = False
+def create_builder(name, strtype, builder_cls):
+    class W_Builder(Wrappable):
+        def __init__(self, space, size):
+            if size < 0:
+                self.builder = builder_cls()
+            else:
+                self.builder = builder_cls(size)
 
-    def _check_done(self, space):
-        if self.done:
-            raise OperationError(space.w_ValueError, space.wrap("Can't operate on a done builder"))
+        def _check_done(self, space):
+            if self.builder is None:
+                raise OperationError(space.w_ValueError, space.wrap("Can't operate on a done builder"))
 
-    @unwrap_spec(size=int)
-    def descr__new__(space, w_subtype, size=-1):
-        return W_UnicodeBuilder(space, size)
+        @unwrap_spec(size=int)
+        def descr__new__(space, w_subtype, size=-1):
+            return W_Builder(space, size)
 
-    @unwrap_spec(s=unicode)
-    def descr_append(self, space, s):
-        self._check_done(space)
-        self.builder.append(s)
+        @unwrap_spec(s=strtype)
+        def descr_append(self, space, s):
+            self._check_done(space)
+            self.builder.append(s)
 
-    @unwrap_spec(s=unicode, start=int, end=int)
-    def descr_append_slice(self, space, s, start, end):
-        self._check_done(space)
-        if not 0 <= start <= end <= len(s):
-            raise OperationError(space.w_ValueError, space.wrap("bad start/stop"))
-        self.builder.append_slice(s, start, end)
+        @unwrap_spec(s=strtype, start=int, end=int)
+        def descr_append_slice(self, space, s, start, end):
+            self._check_done(space)
+            if not 0 <= start <= end <= len(s):
+                raise OperationError(space.w_ValueError, space.wrap("bad start/stop"))
+            self.builder.append_slice(s, start, end)
 
-    def descr_build(self, space):
-        self._check_done(space)
-        w_s = space.wrap(self.builder.build())
-        self.done = True
-        return w_s
+        def descr_build(self, space):
+            self._check_done(space)
+            w_s = space.wrap(self.builder.build())
+            self.builder = None
+            return w_s
 
+    W_Builder.__name__ = "W_%s" % name
+    W_Builder.typedef = TypeDef(name,
+        __new__ = interp2app(func_with_new_name(
+                                    W_Builder.descr__new__.im_func,
+                                    '%s_new' % (name,))),
+        append = interp2app(W_Builder.descr_append),
+        append_slice = interp2app(W_Builder.descr_append_slice),
+        build = interp2app(W_Builder.descr_build),
+    )
+    W_Builder.typedef.acceptable_as_base_class = False
+    return W_Builder
 
-W_UnicodeBuilder.typedef = TypeDef("UnicodeBuilder",
-    __new__ = interp2app(W_UnicodeBuilder.descr__new__.im_func),
-
-    append = interp2app(W_UnicodeBuilder.descr_append),
-    append_slice = interp2app(W_UnicodeBuilder.descr_append_slice),
-    build = interp2app(W_UnicodeBuilder.descr_build),
-)
-W_UnicodeBuilder.typedef.acceptable_as_base_class = False
+W_StringBuilder = create_builder("StringBuilder", str, StringBuilder)
+W_UnicodeBuilder = create_builder("UnicodeBuilder", unicode, UnicodeBuilder)
diff --git a/pypy/module/__pypy__/test/test_builders.py b/pypy/module/__pypy__/test/test_builders.py
--- a/pypy/module/__pypy__/test/test_builders.py
+++ b/pypy/module/__pypy__/test/test_builders.py
@@ -31,4 +31,14 @@
         raises(ValueError, b.append_slice, u"1", 2, 1)
         s = b.build()
         assert s == "cde"
-        raises(ValueError, b.append_slice, u"abc", 1, 2)
\ No newline at end of file
+        raises(ValueError, b.append_slice, u"abc", 1, 2)
+
+    def test_stringbuilder(self):
+        from __pypy__.builders import StringBuilder
+        b = StringBuilder()
+        b.append("abc")
+        b.append("123")
+        b.append("you and me")
+        s = b.build()
+        assert s == "abc123you and me"
+        raises(ValueError, b.build)
\ No newline at end of file
diff --git a/pypy/module/_continuation/__init__.py b/pypy/module/_continuation/__init__.py
--- a/pypy/module/_continuation/__init__.py
+++ b/pypy/module/_continuation/__init__.py
@@ -37,4 +37,5 @@
     interpleveldefs = {
         'continulet': 'interp_continuation.W_Continulet',
         'permute': 'interp_continuation.permute',
+        '_p': 'interp_continuation.unpickle',      # pickle support
     }
diff --git a/pypy/module/_continuation/interp_continuation.py b/pypy/module/_continuation/interp_continuation.py
--- a/pypy/module/_continuation/interp_continuation.py
+++ b/pypy/module/_continuation/interp_continuation.py
@@ -5,6 +5,8 @@
 from pypy.interpreter.baseobjspace import Wrappable
 from pypy.interpreter.typedef import TypeDef
 from pypy.interpreter.gateway import interp2app
+from pypy.interpreter.pycode import PyCode
+from pypy.interpreter.pyframe import PyFrame
 
 
 class W_Continulet(Wrappable):
@@ -20,66 +22,69 @@
     def check_sthread(self):
         ec = self.space.getexecutioncontext()
         if ec.stacklet_thread is not self.sthread:
-            start_state.clear()
+            global_state.clear()
             raise geterror(self.space, "inter-thread support is missing")
-        return ec
 
     def descr_init(self, w_callable, __args__):
         if self.sthread is not None:
             raise geterror(self.space, "continulet already __init__ialized")
-        start_state.origin = self
-        start_state.w_callable = w_callable
-        start_state.args = __args__
-        self.sthread = build_sthread(self.space)
-        try:
-            self.h = self.sthread.new(new_stacklet_callback)
-            if self.sthread.is_empty_handle(self.h):    # early return
-                raise MemoryError
-        except MemoryError:
-            self.sthread = None
-            start_state.clear()
-            raise getmemoryerror(self.space)
+        #
+        # hackish: build the frame "by hand", passing it the correct arguments
+        space = self.space
+        w_args, w_kwds = __args__.topacked()
+        bottomframe = space.createframe(get_entrypoint_pycode(space),
+                                        get_w_module_dict(space), None)
+        bottomframe.locals_stack_w[0] = space.wrap(self)
+        bottomframe.locals_stack_w[1] = w_callable
+        bottomframe.locals_stack_w[2] = w_args
+        bottomframe.locals_stack_w[3] = w_kwds
+        self.bottomframe = bottomframe
+        #
+        global_state.origin = self
+        sthread = build_sthread(self.space)
+        self.sthread = sthread
+        h = sthread.new(new_stacklet_callback)
+        post_switch(sthread, h)
 
     def switch(self, w_to):
+        sthread = self.sthread
+        if sthread is not None and sthread.is_empty_handle(self.h):
+            global_state.clear()
+            raise geterror(self.space, "continulet already finished")
         to = self.space.interp_w(W_Continulet, w_to, can_be_None=True)
+        if to is not None and to.sthread is None:
+            to = None
+        if sthread is None:      # if self is non-initialized:
+            if to is not None:   #     if we are given a 'to'
+                self = to        #         then just use it and ignore 'self'
+                sthread = self.sthread
+                to = None
+            else:
+                return get_result()  # else: no-op
         if to is not None:
+            if to.sthread is not sthread:
+                global_state.clear()
+                raise geterror(self.space, "cross-thread double switch")
             if self is to:    # double-switch to myself: no-op
                 return get_result()
-            if to.sthread is None:
-                start_state.clear()
-                raise geterror(self.space, "continulet not initialized yet")
-        if self.sthread is None:
-            start_state.clear()
-            raise geterror(self.space, "continulet not initialized yet")
-        ec = self.check_sthread()
-        saved_topframeref = ec.topframeref
+            if sthread.is_empty_handle(to.h):
+                global_state.clear()
+                raise geterror(self.space, "continulet already finished")
+        self.check_sthread()
         #
-        start_state.origin = self
+        global_state.origin = self
         if to is None:
             # simple switch: going to self.h
-            start_state.destination = self
+            global_state.destination = self
         else:
             # double switch: the final destination is to.h
-            start_state.destination = to
+            global_state.destination = to
         #
-        h = start_state.destination.h
-        sthread = self.sthread
-        if sthread.is_empty_handle(h):
-            start_state.clear()
-            raise geterror(self.space, "continulet already finished")
-        #
-        try:
-            do_switch(sthread, h)
-        except MemoryError:
-            start_state.clear()
-            raise getmemoryerror(self.space)
-        #
-        ec = sthread.ec
-        ec.topframeref = saved_topframeref
-        return get_result()
+        h = sthread.switch(global_state.destination.h)
+        return post_switch(sthread, h)
 
     def descr_switch(self, w_value=None, w_to=None):
-        start_state.w_value = w_value
+        global_state.w_value = w_value
         return self.switch(w_to)
 
     def descr_throw(self, w_type, w_val=None, w_tb=None, w_to=None):
@@ -94,8 +99,8 @@
         #
         operr = OperationError(w_type, w_val, tb)
         operr.normalize_exception(space)
-        start_state.w_value = None
-        start_state.propagate_exception = operr
+        global_state.w_value = None
+        global_state.propagate_exception = operr
         return self.switch(w_to)
 
     def descr_is_pending(self):
@@ -103,12 +108,26 @@
                  and not self.sthread.is_empty_handle(self.h))
         return self.space.newbool(valid)
 
+    def descr__reduce__(self):
+        from pypy.module._continuation import interp_pickle
+        return interp_pickle.reduce(self)
+
+    def descr__setstate__(self, w_args):
+        from pypy.module._continuation import interp_pickle
+        interp_pickle.setstate(self, w_args)
+
 
 def W_Continulet___new__(space, w_subtype, __args__):
     r = space.allocate_instance(W_Continulet, w_subtype)
     r.__init__(space)
     return space.wrap(r)
 
+def unpickle(space, w_subtype):
+    """Pickle support."""
+    r = space.allocate_instance(W_Continulet, w_subtype)
+    r.__init__(space)
+    return space.wrap(r)
+
 
 W_Continulet.typedef = TypeDef(
     'continulet',
@@ -118,26 +137,52 @@
     switch      = interp2app(W_Continulet.descr_switch),
     throw       = interp2app(W_Continulet.descr_throw),
     is_pending  = interp2app(W_Continulet.descr_is_pending),
+    __reduce__  = interp2app(W_Continulet.descr__reduce__),
+    __setstate__= interp2app(W_Continulet.descr__setstate__),
     )
 
-
 # ____________________________________________________________
 
+# Continulet objects maintain a dummy frame object in order to ensure
+# that the 'f_back' chain is consistent.  We hide this dummy frame
+# object by giving it a dummy code object with hidden_applevel=True.
 
 class State:
     def __init__(self, space):
-        self.space = space 
+        self.space = space
         w_module = space.getbuiltinmodule('_continuation')
         self.w_error = space.getattr(w_module, space.wrap('error'))
-        self.w_memoryerror = OperationError(space.w_MemoryError, space.w_None)
+        # the following function switches away immediately, so that
+        # continulet.__init__() doesn't immediately run func(), but it
+        # also has the hidden purpose of making sure we have a single
+        # bottomframe for the whole duration of the continulet's run.
+        # Hackish: only the func_code is used, and used in the context
+        # of w_globals == this module, so we can access the name
+        # 'continulet' directly.
+        w_code = space.appexec([], '''():
+            def start(c, func, args, kwds):
+                if continulet.switch(c) is not None:
+                    raise TypeError(
+                     "can\'t send non-None value to a just-started continulet")
+                return func(c, *args, **kwds)
+            return start.func_code
+        ''')
+        self.entrypoint_pycode = space.interp_w(PyCode, w_code)
+        self.entrypoint_pycode.hidden_applevel = True
+        self.w_unpickle = w_module.get('_p')
+        self.w_module_dict = w_module.getdict(space)
 
 def geterror(space, message):
     cs = space.fromcache(State)
     return OperationError(cs.w_error, space.wrap(message))
 
-def getmemoryerror(space):
+def get_entrypoint_pycode(space):
     cs = space.fromcache(State)
-    return cs.w_memoryerror
+    return cs.entrypoint_pycode
+
+def get_w_module_dict(space):
+    cs = space.fromcache(State)
+    return cs.w_module_dict
 
 # ____________________________________________________________
 
@@ -148,71 +193,63 @@
         StackletThread.__init__(self, space.config)
         self.space = space
         self.ec = ec
+        # for unpickling
+        from pypy.rlib.rweakref import RWeakKeyDictionary
+        self.frame2continulet = RWeakKeyDictionary(PyFrame, W_Continulet)
 
 ExecutionContext.stacklet_thread = None
 
 # ____________________________________________________________
 
 
-class StartState:   # xxx a single global to pass around the function to start
+class GlobalState:
     def clear(self):
         self.origin = None
         self.destination = None
-        self.w_callable = None
-        self.args = None
         self.w_value = None
         self.propagate_exception = None
-start_state = StartState()
-start_state.clear()
+global_state = GlobalState()
+global_state.clear()
 
 
 def new_stacklet_callback(h, arg):
-    self       = start_state.origin
-    w_callable = start_state.w_callable
-    args       = start_state.args
-    start_state.clear()
-    try:
-        do_switch(self.sthread, h)
-    except MemoryError:
-        return h       # oups!  do an early return in this case
-    #
+    self = global_state.origin
+    self.h = h
+    global_state.clear()
     space = self.space
     try:
-        ec = self.sthread.ec
-        ec.topframeref = jit.vref_None
-
-        if start_state.propagate_exception is not None:
-            raise start_state.propagate_exception   # just propagate it further
-        if start_state.w_value is not space.w_None:
-            raise OperationError(space.w_TypeError, space.wrap(
-                "can't send non-None value to a just-started continulet"))
-
-        args = args.prepend(self.space.wrap(self))
-        w_result = space.call_args(w_callable, args)
+        frame = self.bottomframe
+        w_result = frame.execute_frame()
     except Exception, e:
-        start_state.propagate_exception = e
+        global_state.propagate_exception = e
     else:
-        start_state.w_value = w_result
-    start_state.origin = self
-    start_state.destination = self
+        global_state.w_value = w_result
+    self.sthread.ec.topframeref = jit.vref_None
+    global_state.origin = self
+    global_state.destination = self
     return self.h
 
-
-def do_switch(sthread, h):
-    h = sthread.switch(h)
-    origin = start_state.origin
-    self = start_state.destination
-    start_state.origin = None
-    start_state.destination = None
+def post_switch(sthread, h):
+    origin = global_state.origin
+    self = global_state.destination
+    global_state.origin = None
+    global_state.destination = None
     self.h, origin.h = origin.h, h
+    #
+    current = sthread.ec.topframeref
+    sthread.ec.topframeref = self.bottomframe.f_backref
+    self.bottomframe.f_backref = origin.bottomframe.f_backref
+    origin.bottomframe.f_backref = current
+    #
+    return get_result()
 
 def get_result():
-    if start_state.propagate_exception:
-        e = start_state.propagate_exception
-        start_state.propagate_exception = None
+    if global_state.propagate_exception:
+        e = global_state.propagate_exception
+        global_state.propagate_exception = None
         raise e
-    w_value = start_state.w_value
-    start_state.w_value = None
+    w_value = global_state.w_value
+    global_state.w_value = None
     return w_value
 
 def build_sthread(space):
@@ -232,7 +269,7 @@
         cont = space.interp_w(W_Continulet, w_cont)
         if cont.sthread is not sthread:
             if cont.sthread is None:
-                raise geterror(space, "got a non-initialized continulet")
+                continue   # ignore non-initialized continulets
             else:
                 raise geterror(space, "inter-thread support is missing")
         elif sthread.is_empty_handle(cont.h):
@@ -240,6 +277,9 @@
         contlist.append(cont)
     #
     if len(contlist) > 1:
-        other = contlist[-1].h
+        otherh = contlist[-1].h
+        otherb = contlist[-1].bottomframe.f_backref
         for cont in contlist:
-            other, cont.h = cont.h, other
+            otherh, cont.h = cont.h, otherh
+            b = cont.bottomframe
+            otherb, b.f_backref = b.f_backref, otherb
diff --git a/pypy/module/_continuation/interp_pickle.py b/pypy/module/_continuation/interp_pickle.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_continuation/interp_pickle.py
@@ -0,0 +1,128 @@
+from pypy.tool import stdlib_opcode as pythonopcode
+from pypy.rlib import jit
+from pypy.interpreter.error import OperationError
+from pypy.interpreter.pyframe import PyFrame
+from pypy.module._continuation.interp_continuation import State, global_state
+from pypy.module._continuation.interp_continuation import build_sthread
+from pypy.module._continuation.interp_continuation import post_switch
+from pypy.module._continuation.interp_continuation import get_result, geterror
+
+
+def getunpickle(space):
+    cs = space.fromcache(State)
+    return cs.w_unpickle
+
+
+def reduce(self):
+    # xxx this is known to be not completely correct with respect
+    # to subclasses, e.g. no __slots__ support, no looking for a
+    # __getnewargs__ or __getstate__ defined in the subclass, etc.
+    # Doing the right thing looks involved, though...
+    space = self.space
+    if self.sthread is None:
+        w_frame = space.w_False
+    elif self.sthread.is_empty_handle(self.h):
+        w_frame = space.w_None
+    else:
+        w_frame = space.wrap(self.bottomframe)
+    w_continulet_type = space.type(space.wrap(self))
+    w_dict = self.getdict(space) or space.w_None
+    args = [getunpickle(space),
+            space.newtuple([w_continulet_type]),
+            space.newtuple([w_frame, w_dict]),
+            ]
+    return space.newtuple(args)
+
+def setstate(self, w_args):
+    space = self.space
+    if self.sthread is not None:
+        raise geterror(space, "continulet.__setstate__() on an already-"
+                              "initialized continulet")
+    w_frame, w_dict = space.fixedview(w_args, expected_length=2)
+    if not space.is_w(w_dict, space.w_None):
+        self.setdict(space, w_dict)
+    if space.is_w(w_frame, space.w_False):
+        return    # not initialized
+    sthread = build_sthread(self.space)
+    self.sthread = sthread
+    self.bottomframe = space.interp_w(PyFrame, w_frame, can_be_None=True)
+    #
+    global_state.origin = self
+    if self.bottomframe is not None:
+        sthread.frame2continulet.set(self.bottomframe, self)
+    self.h = sthread.new(resume_trampoline_callback)
+    get_result()    # propagate the eventual MemoryError
+
+# ____________________________________________________________
+
+def resume_trampoline_callback(h, arg):
+    self = global_state.origin
+    self.h = h
+    space = self.space
+    sthread = self.sthread
+    try:
+        global_state.clear()
+        if self.bottomframe is None:
+            w_result = space.w_None
+        else:
+            h = sthread.switch(self.h)
+            try:
+                w_result = post_switch(sthread, h)
+                operr = None
+            except OperationError, e:
+                w_result = None
+                operr = e
+            #
+            while True:
+                ec = sthread.ec
+                frame = ec.topframeref()
+                assert frame is not None     # XXX better error message
+                exit_continulet = sthread.frame2continulet.get(frame)
+                #
+                continue_after_call(frame)
+                #
+                # small hack: unlink frame out of the execution context,
+                # because execute_frame will add it there again
+                ec.topframeref = frame.f_backref
+                #
+                try:
+                    w_result = frame.execute_frame(w_result, operr)
+                    operr = None
+                except OperationError, e:
+                    w_result = None
+                    operr = e
+                if exit_continulet is not None:
+                    self = exit_continulet
+                    break
+            sthread.ec.topframeref = jit.vref_None
+            if operr:
+                raise operr
+    except Exception, e:
+        global_state.propagate_exception = e
+    else:
+        global_state.w_value = w_result
+    global_state.origin = self
+    global_state.destination = self
+    return self.h
+
+def continue_after_call(frame):
+    code = frame.pycode.co_code
+    instr = frame.last_instr
+    opcode = ord(code[instr])
+    map = pythonopcode.opmap
+    call_ops = [map['CALL_FUNCTION'], map['CALL_FUNCTION_KW'],
+                map['CALL_FUNCTION_VAR'], map['CALL_FUNCTION_VAR_KW'],
+                map['CALL_METHOD']]
+    assert opcode in call_ops   # XXX check better, and complain better
+    instr += 1
+    oparg = ord(code[instr]) | ord(code[instr + 1]) << 8
+    nargs = oparg & 0xff
+    nkwds = (oparg >> 8) & 0xff
+    if nkwds == 0:     # only positional arguments
+        # fast paths leaves things on the stack, pop them
+        if (frame.space.config.objspace.opcodes.CALL_METHOD and
+            opcode == map['CALL_METHOD']):
+            frame.dropvalues(nargs + 2)
+        elif opcode == map['CALL_FUNCTION']:
+            frame.dropvalues(nargs + 1)
+    frame.last_instr = instr + 1    # continue after the call
diff --git a/pypy/module/_continuation/test/support.py b/pypy/module/_continuation/test/support.py
--- a/pypy/module/_continuation/test/support.py
+++ b/pypy/module/_continuation/test/support.py
@@ -9,4 +9,4 @@
             import pypy.rlib.rstacklet
         except CompilationError, e:
             py.test.skip("cannot import rstacklet: %s" % e)
-        cls.space = gettestobjspace(usemodules=['_continuation'])
+        cls.space = gettestobjspace(usemodules=['_continuation'], continuation=True)
diff --git a/pypy/module/_continuation/test/test_stacklet.py b/pypy/module/_continuation/test/test_stacklet.py
--- a/pypy/module/_continuation/test/test_stacklet.py
+++ b/pypy/module/_continuation/test/test_stacklet.py
@@ -13,7 +13,7 @@
         from _continuation import continulet
         #
         def empty_callback(c):
-            pass
+            never_called
         #
         c = continulet(empty_callback)
         assert type(c) is continulet
@@ -36,7 +36,7 @@
         from _continuation import continulet, error
         #
         def empty_callback(c1):
-            pass
+            never_called
         #
         c = continulet(empty_callback)
         raises(error, c.__init__, empty_callback)
@@ -135,12 +135,6 @@
         e = raises(error, c.switch)
         assert str(e.value) == "continulet already finished"
 
-    def test_not_initialized_yet(self):
-        from _continuation import continulet, error
-        c = continulet.__new__(continulet)
-        e = raises(error, c.switch)
-        assert str(e.value) == "continulet not initialized yet"
-
     def test_go_depth2(self):
         from _continuation import continulet
         #
@@ -254,6 +248,15 @@
         res = c_upper.switch('D')
         assert res == 'E'
 
+    def test_switch_not_initialized(self):
+        from _continuation import continulet
+        c0 = continulet.__new__(continulet)
+        res = c0.switch()
+        assert res is None
+        res = c0.switch(123)
+        assert res == 123
+        raises(ValueError, c0.throw, ValueError)
+
     def test_exception_with_switch_depth2(self):
         from _continuation import continulet
         #
@@ -312,7 +315,7 @@
         res = f()
         assert res == 2002
 
-    def test_f_back_is_None_for_now(self):
+    def test_f_back(self):
         import sys
         from _continuation import continulet
         #
@@ -321,6 +324,7 @@
             c.switch(sys._getframe(0).f_back)
             c.switch(sys._getframe(1))
             c.switch(sys._getframe(1).f_back)
+            assert sys._getframe(2) is f3.f_back
             c.switch(sys._getframe(2))
         def f(c):
             g(c)
@@ -331,10 +335,21 @@
         f2 = c.switch()
         assert f2.f_code.co_name == 'f'
         f3 = c.switch()
-        assert f3.f_code.co_name == 'f'
-        f4 = c.switch()
-        assert f4 is None
-        raises(ValueError, c.switch)    # "call stack is not deep enough"
+        assert f3 is f2
+        assert f1.f_back is f3
+        def main():
+            f4 = c.switch()
+            assert f4.f_code.co_name == 'main', repr(f4.f_code.co_name)
+            assert f3.f_back is f1    # not running, so a loop
+        def main2():
+            f5 = c.switch()
+            assert f5.f_code.co_name == 'main2', repr(f5.f_code.co_name)
+            assert f3.f_back is f1    # not running, so a loop
+        main()
+        main2()
+        res = c.switch()
+        assert res is None
+        assert f3.f_back is None
 
     def test_traceback_is_complete(self):
         import sys
@@ -487,16 +502,31 @@
         assert res == 'z'
         raises(TypeError, c1.switch, to=c2)  # "can't send non-None value"
 
-    def test_switch2_not_initialized_yet(self):
-        from _continuation import continulet, error
+    def test_switch2_not_initialized(self):
+        from _continuation import continulet
+        c0 = continulet.__new__(continulet)
+        c0bis = continulet.__new__(continulet)
+        res = c0.switch(123, to=c0)
+        assert res == 123
+        res = c0.switch(123, to=c0bis)
+        assert res == 123
+        raises(ValueError, c0.throw, ValueError, to=c0)
+        raises(ValueError, c0.throw, ValueError, to=c0bis)
         #
         def f1(c1):
-            not_reachable
-        #
+            c1.switch('a')
+            raises(ValueError, c1.switch, 'b')
+            raises(KeyError, c1.switch, 'c')
+            return 'd'
         c1 = continulet(f1)
-        c2 = continulet.__new__(continulet)
-        e = raises(error, c1.switch, to=c2)
-        assert str(e.value) == "continulet not initialized yet"
+        res = c0.switch(to=c1)
+        assert res == 'a'
+        res = c1.switch(to=c0)
+        assert res == 'b'
+        res = c1.throw(ValueError, to=c0)
+        assert res == 'c'
+        res = c0.throw(KeyError, to=c1)
+        assert res == 'd'
 
     def test_switch2_already_finished(self):
         from _continuation import continulet, error
@@ -609,6 +639,7 @@
         assert res == "ok"
 
     def test_permute(self):
+        import sys
         from _continuation import continulet, permute
         #
         def f1(c1):
@@ -617,14 +648,34 @@
             return "done"
         #
         def f2(c2):
+            assert sys._getframe(1).f_code.co_name == 'main'
             permute(c1, c2)
+            assert sys._getframe(1).f_code.co_name == 'f1'
             return "ok"
         #
         c1 = continulet(f1)
         c2 = continulet(f2)
+        def main():
+            c1.switch()
+            res = c2.switch()
+            assert res == "done"
+        main()
+
+    def test_permute_noninitialized(self):
+        from _continuation import continulet, permute
+        permute(continulet.__new__(continulet))    # ignored
+        permute(continulet.__new__(continulet),    # ignored
+                continulet.__new__(continulet))
+
+    def test_bug_finish_with_already_finished_stacklet(self):
+        from _continuation import continulet, error
+        # make an already-finished continulet
+        c1 = continulet(lambda x: x)
         c1.switch()
-        res = c2.switch()
-        assert res == "done"
+        # make another continulet
+        c2 = continulet(lambda x: x)
+        # this switch is forbidden, because it causes a crash when c2 finishes
+        raises(error, c1.switch, to=c2)
 
     def test_various_depths(self):
         skip("may fail on top of CPython")
diff --git a/pypy/module/_continuation/test/test_zpickle.py b/pypy/module/_continuation/test/test_zpickle.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_continuation/test/test_zpickle.py
@@ -0,0 +1,262 @@
+from pypy.conftest import gettestobjspace
+
+
+class AppTestCopy:
+    def setup_class(cls):
+        cls.space = gettestobjspace(usemodules=('_continuation',),
+                                    CALL_METHOD=True)
+        cls.space.config.translation.continuation = True
+
+    def test_basic_setup(self):
+        from _continuation import continulet
+        lst = [4]
+        co = continulet(lst.append)
+        assert lst == [4]
+        res = co.switch()
+        assert res is None
+        assert lst == [4, co]
+
+    def test_copy_continulet_not_started(self):
+        from _continuation import continulet, error
+        import copy
+        lst = []
+        co = continulet(lst.append)
+        co2, lst2 = copy.deepcopy((co, lst))
+        #
+        assert lst == []
+        co.switch()
+        assert lst == [co]
+        #
+        assert lst2 == []
+        co2.switch()
+        assert lst2 == [co2]
+
+    def test_copy_continulet_not_started_multiple(self):
+        from _continuation import continulet, error
+        import copy
+        lst = []
+        co = continulet(lst.append)
+        co2, lst2 = copy.deepcopy((co, lst))
+        co3, lst3 = copy.deepcopy((co, lst))
+        co4, lst4 = copy.deepcopy((co, lst))
+        #
+        assert lst == []
+        co.switch()
+        assert lst == [co]
+        #
+        assert lst2 == []
+        co2.switch()
+        assert lst2 == [co2]
+        #
+        assert lst3 == []
+        co3.switch()
+        assert lst3 == [co3]
+        #
+        assert lst4 == []
+        co4.switch()
+        assert lst4 == [co4]
+
+    def test_copy_continulet_real(self):
+        import new, sys
+        mod = new.module('test_copy_continulet_real')
+        sys.modules['test_copy_continulet_real'] = mod
+        exec '''if 1:
+            from _continuation import continulet
+            import copy
+            def f(co, x):
+                co.switch(x + 1)
+                co.switch(x + 2)
+                return x + 3
+            co = continulet(f, 40)
+            res = co.switch()
+            assert res == 41
+            co2 = copy.deepcopy(co)
+            #
+            res = co2.switch()
+            assert res == 42
+            assert co2.is_pending()
+            res = co2.switch()
+            assert res == 43
+            assert not co2.is_pending()
+            #
+            res = co.switch()
+            assert res == 42
+            assert co.is_pending()
+            res = co.switch()
+            assert res == 43
+            assert not co.is_pending()
+        ''' in mod.__dict__
+
+    def test_copy_continulet_already_finished(self):
+        from _continuation import continulet, error
+        import copy
+        lst = []
+        co = continulet(lst.append)
+        co.switch()
+        co2 = copy.deepcopy(co)
+        assert not co.is_pending()
+        assert not co2.is_pending()
+        raises(error, co.__init__, lst.append)
+        raises(error, co2.__init__, lst.append)
+        raises(error, co.switch)
+        raises(error, co2.switch)
+
+
+class AppTestPickle:
+    version = 0
+
+    def setup_class(cls):
+        cls.space = gettestobjspace(usemodules=('_continuation',),
+                                    CALL_METHOD=True)
+        cls.space.appexec([], """():
+            global continulet, A, __name__
+
+            import sys
+            __name__ = 'test_pickle_continulet'
+            thismodule = type(sys)(__name__)
+            sys.modules[__name__] = thismodule
+
+            from _continuation import continulet
+            class A(continulet):
+                pass
+
+            thismodule.__dict__.update(globals())
+        """)
+        cls.w_version = cls.space.wrap(cls.version)
+
+    def test_pickle_continulet_empty(self):
+        from _continuation import continulet
+        lst = [4]
+        co = continulet.__new__(continulet)
+        import pickle
+        pckl = pickle.dumps(co, self.version)
+        print repr(pckl)
+        co2 = pickle.loads(pckl)
+        assert co2 is not co
+        assert not co.is_pending()
+        assert not co2.is_pending()
+        # the empty unpickled coroutine can still be used:
+        result = [5]
+        co2.__init__(result.append)
+        res = co2.switch()
+        assert res is None
+        assert result == [5, co2]
+
+    def test_pickle_continulet_empty_subclass(self):
+        from test_pickle_continulet import continulet, A
+        lst = [4]
+        co = continulet.__new__(A)
+        co.foo = 'bar'
+        co.bar = 'baz'
+        import pickle
+        pckl = pickle.dumps(co, self.version)
+        print repr(pckl)
+        co2 = pickle.loads(pckl)
+        assert co2 is not co
+        assert not co.is_pending()
+        assert not co2.is_pending()
+        assert type(co) is type(co2) is A
+        assert co.foo == co2.foo == 'bar'
+        assert co.bar == co2.bar == 'baz'
+        # the empty unpickled coroutine can still be used:
+        result = [5]
+        co2.__init__(result.append)
+        res = co2.switch()
+        assert res is None
+        assert result == [5, co2]
+
+    def test_pickle_continulet_not_started(self):
+        from _continuation import continulet, error
+        import pickle
+        lst = []
+        co = continulet(lst.append)
+        pckl = pickle.dumps((co, lst))
+        print pckl
+        del co, lst
+        for i in range(2):
+            print 'resume...'
+            co2, lst2 = pickle.loads(pckl)
+            assert lst2 == []
+            co2.switch()
+            assert lst2 == [co2]
+
+    def test_pickle_continulet_real(self):
+        import new, sys
+        mod = new.module('test_pickle_continulet_real')
+        sys.modules['test_pickle_continulet_real'] = mod
+        mod.version = self.version
+        exec '''if 1:
+            from _continuation import continulet
+            import pickle
+            def f(co, x):
+                co.switch(x + 1)
+                co.switch(x + 2)
+                return x + 3
+            co = continulet(f, 40)
+            res = co.switch()
+            assert res == 41
+            pckl = pickle.dumps(co, version)
+            print repr(pckl)
+            co2 = pickle.loads(pckl)
+            #
+            res = co2.switch()
+            assert res == 42
+            assert co2.is_pending()
+            res = co2.switch()
+            assert res == 43
+            assert not co2.is_pending()
+            #
+            res = co.switch()
+            assert res == 42
+            assert co.is_pending()
+            res = co.switch()
+            assert res == 43
+            assert not co.is_pending()
+        ''' in mod.__dict__
+
+    def test_pickle_continulet_real_subclass(self):
+        import new, sys
+        mod = new.module('test_pickle_continulet_real_subclass')
+        sys.modules['test_pickle_continulet_real_subclass'] = mod
+        mod.version = self.version
+        exec '''if 1:
+            from _continuation import continulet
+            import pickle
+            class A(continulet):
+                def __init__(self):
+                    crash
+            def f(co):
+                co.switch(co.x + 1)
+                co.switch(co.x + 2)
+                return co.x + 3
+            co = A.__new__(A)
+            continulet.__init__(co, f)
+            co.x = 40
+            res = co.switch()
+            assert res == 41
+            pckl = pickle.dumps(co, version)
+            print repr(pckl)
+            co2 = pickle.loads(pckl)
+            #
+            assert type(co2) is A
+            res = co2.switch()
+            assert res == 42
+            assert co2.is_pending()
+            res = co2.switch()
+            assert res == 43
+            assert not co2.is_pending()
+            #
+            res = co.switch()
+            assert res == 42
+            assert co.is_pending()
+            res = co.switch()
+            assert res == 43
+            assert not co.is_pending()
+        ''' in mod.__dict__
+
+
+class AppTestPickle_v1(AppTestPickle):
+    version = 1
+
+class AppTestPickle_v2(AppTestPickle):
+    version = 2
diff --git a/pypy/module/_multiprocessing/interp_connection.py b/pypy/module/_multiprocessing/interp_connection.py
--- a/pypy/module/_multiprocessing/interp_connection.py
+++ b/pypy/module/_multiprocessing/interp_connection.py
@@ -225,7 +225,9 @@
             except OSError:
                 pass
 
-    def __init__(self, fd, flags):
+    def __init__(self, space, fd, flags):
+        if fd == self.INVALID_HANDLE_VALUE or fd < 0:
+            raise OperationError(space.w_IOError, space.wrap("invalid handle %d" % fd))
         W_BaseConnection.__init__(self, flags)
         self.fd = fd
 
@@ -234,7 +236,7 @@
         flags = (readable and READABLE) | (writable and WRITABLE)
 
         self = space.allocate_instance(W_FileConnection, w_subtype)
-        W_FileConnection.__init__(self, fd, flags)
+        W_FileConnection.__init__(self, space, fd, flags)
         return space.wrap(self)
 
     def fileno(self, space):
diff --git a/pypy/module/_multiprocessing/interp_semaphore.py b/pypy/module/_multiprocessing/interp_semaphore.py
--- a/pypy/module/_multiprocessing/interp_semaphore.py
+++ b/pypy/module/_multiprocessing/interp_semaphore.py
@@ -468,6 +468,9 @@
 
         self.count -= 1
 
+    def after_fork(self):
+        self.count = 0
+
     @unwrap_spec(kind=int, maxvalue=int)
     def rebuild(space, w_cls, w_handle, kind, maxvalue):
         self = space.allocate_instance(W_SemLock, w_cls)
@@ -512,6 +515,7 @@
     acquire = interp2app(W_SemLock.acquire),
     release = interp2app(W_SemLock.release),
     _rebuild = interp2app(W_SemLock.rebuild.im_func, as_classmethod=True),
+    _after_fork = interp2app(W_SemLock.after_fork),
     __enter__=interp2app(W_SemLock.enter),
     __exit__=interp2app(W_SemLock.exit),
     SEM_VALUE_MAX=SEM_VALUE_MAX,
diff --git a/pypy/module/_multiprocessing/test/test_connection.py b/pypy/module/_multiprocessing/test/test_connection.py
--- a/pypy/module/_multiprocessing/test/test_connection.py
+++ b/pypy/module/_multiprocessing/test/test_connection.py
@@ -145,3 +145,9 @@
             else:
                 c.close()
         space.delslice(w_connections, space.wrap(0), space.wrap(100))
+
+    def test_bad_fd(self):
+        import _multiprocessing
+
+        raises(IOError, _multiprocessing.Connection, -1)
+        raises(IOError, _multiprocessing.Connection, -15)
\ No newline at end of file
diff --git a/pypy/module/_multiprocessing/test/test_semaphore.py b/pypy/module/_multiprocessing/test/test_semaphore.py
--- a/pypy/module/_multiprocessing/test/test_semaphore.py
+++ b/pypy/module/_multiprocessing/test/test_semaphore.py
@@ -39,6 +39,10 @@
         sem.release()
         assert sem._count() == 0
 
+        sem.acquire()
+        sem._after_fork()
+        assert sem._count() == 0
+
     def test_recursive(self):
         from _multiprocessing import SemLock
         kind = self.RECURSIVE
diff --git a/pypy/module/_rawffi/test/test__rawffi.py b/pypy/module/_rawffi/test/test__rawffi.py
--- a/pypy/module/_rawffi/test/test__rawffi.py
+++ b/pypy/module/_rawffi/test/test__rawffi.py
@@ -639,33 +639,6 @@
         a1.free()
         cb.free()
 
-    def test_another_callback_in_stackless(self):
-        try:
-            import _stackless
-        except ImportError:
-            skip("only valid in a stackless pypy-c")
-
-        import _rawffi
-        lib = _rawffi.CDLL(self.lib_name)
-        runcallback = lib.ptr('runcallback', ['P'], 'q')
-        def callback():
-            co = _stackless.coroutine()
-            def f():
-                pass
-            try:
-                co.bind(f)
-                co.switch()
-            except RuntimeError:
-                return 1<<42
-            return -5
-
-        cb = _rawffi.CallbackPtr(callback, [], 'q')
-        a1 = cb.byptr()
-        res = runcallback(a1)
-        assert res[0] == 1<<42
-        a1.free()
-        cb.free()
-
     def test_raising_callback(self):
         import _rawffi, sys
         import StringIO
diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py
--- a/pypy/module/_sre/interp_sre.py
+++ b/pypy/module/_sre/interp_sre.py
@@ -99,6 +99,7 @@
 # SRE_Pattern class
 
 class W_SRE_Pattern(Wrappable):
+    _immutable_fields_ = ["code", "flags"]
 
     def cannot_copy_w(self):
         space = self.space
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -52,7 +52,8 @@
 constants["CERT_OPTIONAL"] = PY_SSL_CERT_OPTIONAL
 constants["CERT_REQUIRED"] = PY_SSL_CERT_REQUIRED
 
-constants["PROTOCOL_SSLv2"]  = PY_SSL_VERSION_SSL2
+if not OPENSSL_NO_SSL2:
+    constants["PROTOCOL_SSLv2"]  = PY_SSL_VERSION_SSL2
 constants["PROTOCOL_SSLv3"]  = PY_SSL_VERSION_SSL3
 constants["PROTOCOL_SSLv23"] = PY_SSL_VERSION_SSL23
 constants["PROTOCOL_TLSv1"]  = PY_SSL_VERSION_TLS1
@@ -673,7 +674,7 @@
         method = libssl_TLSv1_method()
     elif protocol == PY_SSL_VERSION_SSL3:
         method = libssl_SSLv3_method()
-    elif protocol == PY_SSL_VERSION_SSL2:
+    elif protocol == PY_SSL_VERSION_SSL2 and not OPENSSL_NO_SSL2:
         method = libssl_SSLv2_method()
     elif protocol == PY_SSL_VERSION_SSL23:
         method = libssl_SSLv23_method()
diff --git a/pypy/module/_stackless/__init__.py b/pypy/module/_stackless/__init__.py
deleted file mode 100644
--- a/pypy/module/_stackless/__init__.py
+++ /dev/null
@@ -1,36 +0,0 @@
-# Package initialisation
-from pypy.interpreter.mixedmodule import MixedModule
-
-class Module(MixedModule):
-    """
-    This module implements Stackless for applications.
-    """
-
-    appleveldefs = {
-        'GreenletExit' : 'app_greenlet.GreenletExit',
-        'GreenletError' : 'app_greenlet.GreenletError',
-    }
-
-    interpleveldefs = {
-        'tasklet'    : 'interp_stackless.tasklet',
-        'coroutine'  : 'interp_coroutine.AppCoroutine',
-        'greenlet'   : 'interp_greenlet.AppGreenlet',
-        'usercostate': 'interp_composable_coroutine.W_UserCoState',
-        '_return_main' : 'interp_coroutine.return_main',
-        'get_stack_depth_limit': 'interp_coroutine.get_stack_depth_limit',
-        'set_stack_depth_limit': 'interp_coroutine.set_stack_depth_limit',
-    }
-
-    def setup_after_space_initialization(self):
-        # post-installing classmethods/staticmethods which
-        # are not yet directly supported
-        from pypy.module._stackless.interp_coroutine import post_install as post_install_coro
-        post_install_coro(self)
-        from pypy.module._stackless.interp_greenlet import post_install as post_install_greenlet
-        post_install_greenlet(self)
-
-        if self.space.config.translation.gc == 'marksweep':
-            from pypy.module._stackless.interp_clonable import post_install as post_install_clonable
-            self.extra_interpdef('clonable', 'interp_clonable.AppClonableCoroutine')
-            self.extra_interpdef('fork',     'interp_clonable.fork')
-            post_install_clonable(self)
diff --git a/pypy/module/_stackless/app_greenlet.py b/pypy/module/_stackless/app_greenlet.py
deleted file mode 100644
--- a/pypy/module/_stackless/app_greenlet.py
+++ /dev/null
@@ -1,5 +0,0 @@
-class GreenletExit(Exception):
-    pass
-
-class GreenletError(Exception):
-    pass
diff --git a/pypy/module/_stackless/interp_clonable.py b/pypy/module/_stackless/interp_clonable.py
deleted file mode 100644
--- a/pypy/module/_stackless/interp_clonable.py
+++ /dev/null
@@ -1,106 +0,0 @@
-from pypy.interpreter.error import OperationError
-from pypy.interpreter.typedef import TypeDef
-from pypy.interpreter.gateway import interp2app
-from pypy.module._stackless.interp_coroutine import AppCoroutine, AppCoState
-from pypy.module._stackless.interp_coroutine import makeStaticMethod
-from pypy.module._stackless.rcoroutine import AbstractThunk
-from pypy.module._stackless.rclonable import InterpClonableMixin
-
-
-class AppClonableCoroutine(AppCoroutine, InterpClonableMixin):
-
-    def newsubctx(self):
-        self.hello_local_pool()
-        AppCoroutine.newsubctx(self)
-        self.goodbye_local_pool()
-
-    def hello(self):
-        self.hello_local_pool()
-        AppCoroutine.hello(self)
-
-    def goodbye(self):
-        AppCoroutine.goodbye(self)
-        self.goodbye_local_pool()
-
-    def descr_method__new__(space, w_subtype):
-        co = space.allocate_instance(AppClonableCoroutine, w_subtype)
-        costate = AppClonableCoroutine._get_state(space)
-        AppClonableCoroutine.__init__(co, space, state=costate)
-        return space.wrap(co)
-
-    def _get_state(space):
-        return space.fromcache(AppClonableCoState)
-    _get_state = staticmethod(_get_state)
-
-    def w_getcurrent(space):
-        return space.wrap(AppClonableCoroutine._get_state(space).current)
-    w_getcurrent = staticmethod(w_getcurrent)
-
-    def w_clone(self):
-        space = self.space
-        costate = self.costate
-        if costate.current is self:
-            raise OperationError(space.w_RuntimeError,
-                                 space.wrap("clone() cannot clone the "
-                                            "current coroutine"
-                                            "; use fork() instead"))
-        copy = AppClonableCoroutine(space, state=costate)
-        copy.subctx = self.clone_into(copy, self.subctx)
-        return space.wrap(copy)
-
-    def descr__reduce__(self, space):
-        raise OperationError(space.w_TypeError,
-                             space.wrap("_stackless.clonable instances are "
-                                        "not picklable"))
-
-
-AppClonableCoroutine.typedef = TypeDef("clonable", AppCoroutine.typedef,
-    __new__    = interp2app(AppClonableCoroutine.descr_method__new__.im_func),
-    getcurrent = interp2app(AppClonableCoroutine.w_getcurrent),
-    clone      = interp2app(AppClonableCoroutine.w_clone),
-    __reduce__ = interp2app(AppClonableCoroutine.descr__reduce__),
-)
-
-class AppClonableCoState(AppCoState):
-    def post_install(self):
-        self.current = self.main = AppClonableCoroutine(self.space, state=self)
-        self.main.subctx.clear_framestack()      # wack
-
-def post_install(module):
-    makeStaticMethod(module, 'clonable', 'getcurrent')
-    space = module.space
-    AppClonableCoroutine._get_state(space).post_install()
-
-# ____________________________________________________________
-
-class ForkThunk(AbstractThunk):
-    def __init__(self, coroutine):
-        self.coroutine = coroutine
-        self.newcoroutine = None
-    def call(self):
-        oldcoro = self.coroutine
-        self.coroutine = None
-        newcoro = AppClonableCoroutine(oldcoro.space, state=oldcoro.costate)
-        newcoro.subctx = oldcoro.clone_into(newcoro, oldcoro.subctx)
-        newcoro.parent = oldcoro
-        self.newcoroutine = newcoro
-
-def fork(space):
-    """Fork, as in the Unix fork(): the call returns twice, and the return
-    value of the call is either the new 'child' coroutine object (if returning
-    into the parent), or None (if returning into the child).  This returns
-    into the parent first, which can switch to the child later.
-    """
-    costate = AppClonableCoroutine._get_state(space)
-    current = costate.current
-    if current is costate.main:
-        raise OperationError(space.w_RuntimeError,
-                             space.wrap("cannot fork() in the main "
-                                        "clonable coroutine"))
-    thunk = ForkThunk(current)
-    coro_fork = AppClonableCoroutine(space, state=costate)
-    coro_fork.bind(thunk)
-    coro_fork.switch()
-    # we resume here twice.  The following would need explanations about
-    # why it returns the correct thing in both the parent and the child...
-    return space.wrap(thunk.newcoroutine)
diff --git a/pypy/module/_stackless/interp_composable_coroutine b/pypy/module/_stackless/interp_composable_coroutine
deleted file mode 100644
--- a/pypy/module/_stackless/interp_composable_coroutine
+++ /dev/null
@@ -1,33 +0,0 @@
-from pypy.interpreter.baseobjspace import Wrappable
-from pypy.interpreter.typedef import TypeDef, interp2app
-from pypy.module._stackless.coroutine import AppCoState, AppCoroutine
-
-
-class W_UserCoState(Wrappable):
-    def __init__(self, space):
-        self.costate = AppCoState(space)
-        self.costate.post_install()
-
-    def descr_method__new__(space, w_subtype):
-        costate = space.allocate_instance(W_UserCoState, w_subtype)
-        W_UserCoState.__init__(costate, space)
-        return space.wrap(costate)
-
-    def w_getcurrent(self):
-        space = self.costate.space
-        return space.wrap(self.costate.current)
-
-    def w_spawn(self, w_subtype=None):
-        space = self.costate.space
-        if space.is_w(w_subtype, space.w_None):
-            w_subtype = space.gettypeobject(AppCoroutine.typedef)
-        co = space.allocate_instance(AppCoroutine, w_subtype)
-        AppCoroutine.__init__(co, space, state=self.costate)
-        return space.wrap(co)
-
-W_UserCoState.typedef = TypeDef("usercostate",
-    __new__ = interp2app(W_UserCoState.descr_method__new__.im_func),
-    __module__ = '_stackless',
-    getcurrent = interp2app(W_UserCoState.w_getcurrent),
-    spawn      = interp2app(W_UserCoState.w_spawn),
-)
diff --git a/pypy/module/_stackless/interp_composable_coroutine.py b/pypy/module/_stackless/interp_composable_coroutine.py
deleted file mode 100644
--- a/pypy/module/_stackless/interp_composable_coroutine.py
+++ /dev/null
@@ -1,34 +0,0 @@
-from pypy.interpreter.baseobjspace import Wrappable
-from pypy.interpreter.typedef import TypeDef, interp2app
-from pypy.module._stackless.interp_coroutine import AppCoState, AppCoroutine
-
-
-class W_UserCoState(Wrappable):
-    def __init__(self, space):
-        self.costate = AppCoState(space)
-        self.costate.post_install()
-
-    def descr_method__new__(space, w_subtype):
-        costate = space.allocate_instance(W_UserCoState, w_subtype)
-        W_UserCoState.__init__(costate, space)
-        return space.wrap(costate)
-
-    def w_getcurrent(self):
-        space = self.costate.space
-        return space.wrap(self.costate.current)
-
-    def w_spawn(self, w_subtype=None):
-        space = self.costate.space
-        if space.is_w(w_subtype, space.w_None):
-            w_subtype = space.gettypeobject(AppCoroutine.typedef)
-        co = space.allocate_instance(AppCoroutine, w_subtype)
-        AppCoroutine.__init__(co, space, state=self.costate)
-        return space.wrap(co)
-
-W_UserCoState.typedef = TypeDef("usercostate",
-    __new__ = interp2app(W_UserCoState.descr_method__new__.im_func),
-    __module__ = '_stackless',
-    getcurrent = interp2app(W_UserCoState.w_getcurrent),
-    spawn      = interp2app(W_UserCoState.w_spawn),
-)
-W_UserCoState.acceptable_as_base_class = False
diff --git a/pypy/module/_stackless/interp_coroutine.py b/pypy/module/_stackless/interp_coroutine.py
deleted file mode 100644
--- a/pypy/module/_stackless/interp_coroutine.py
+++ /dev/null
@@ -1,403 +0,0 @@
-"""
-Coroutine implementation for application level on top
-of the internal coroutines.
-This is an extensible concept. Multiple implementations
-of concurrency can exist together, if they follow the
-basic concept of maintaining their own costate.
-
-There is also some diversification possible by using
-multiple costates for the same type. This leads to
-disjoint switchable sets within the same type.
-
-I'm not so sure to what extent the opposite is possible, too.
-I.e., merging the costate of tasklets and greenlets would
-allow them to be parents of each other. Needs a bit more
-experience to decide where to set the limits.
-"""
-
-from pypy.interpreter.argument import Arguments
-from pypy.interpreter.typedef import GetSetProperty, TypeDef
-from pypy.interpreter.gateway import interp2app, unwrap_spec
-from pypy.interpreter.error import OperationError, operationerrfmt
-
-from pypy.module._stackless.stackless_flags import StacklessFlags
-from pypy.module._stackless.rcoroutine import Coroutine, BaseCoState, AbstractThunk, CoroutineExit
-
-from pypy.module.exceptions.interp_exceptions import W_SystemExit, _new_exception
-
-from pypy.rlib import rstack, jit # for resume points
-from pypy.tool import stdlib_opcode as pythonopcode
-
-class _AppThunk(AbstractThunk):
-
-    def __init__(self, space, costate, w_obj, args):
-        self.space = space
-        self.costate = costate
-        if not space.is_true(space.callable(w_obj)):
-            raise operationerrfmt(
-                space.w_TypeError,
-                "'%s' object is not callable",
-                space.type(w_obj).getname(space))
-        self.w_func = w_obj
-        self.args = args
-
-    def call(self):
-        costate = self.costate
-        w_result = self.space.call_args(self.w_func, self.args)
-        costate.w_tempval = w_result
-
-class _ResumeThunk(AbstractThunk):
-    def __init__(self, space, costate, w_frame):
-        self.space = space
-        self.costate = costate
-        self.w_frame = w_frame
-
-    def call(self):
-        w_result = resume_frame(self.space, self.w_frame)
-        # costate.w_tempval = w_result #XXX?
-
-
-W_CoroutineExit = _new_exception('CoroutineExit', W_SystemExit,
-                        """Coroutine killed manually.""")
-
-# Should be moved to interp_stackless.py if it's ever implemented... Currently
-# used by pypy/lib/stackless.py.
-W_TaskletExit = _new_exception('TaskletExit', W_SystemExit,
-            """Tasklet killed manually.""")
-
-class AppCoroutine(Coroutine): # XXX, StacklessFlags):
-
-    def __init__(self, space, state=None):
-        self.space = space
-        if state is None:
-            state = AppCoroutine._get_state(space)
-        Coroutine.__init__(self, state)
-        self.flags = 0
-        self.newsubctx()
-
-    def newsubctx(self):
-        ec = self.space.getexecutioncontext()
-        self.subctx = ec.Subcontext()
-
-    def descr_method__new__(space, w_subtype):
-        co = space.allocate_instance(AppCoroutine, w_subtype)
-        AppCoroutine.__init__(co, space)
-        return space.wrap(co)
-
-    def _get_state(space):
-        return space.fromcache(AppCoState)
-    _get_state = staticmethod(_get_state)
-
-    def w_bind(self, w_func, __args__):
-        space = self.space
-        if self.frame is not None:
-            raise OperationError(space.w_ValueError, space.wrap(
-                "cannot bind a bound Coroutine"))
-        state = self.costate
-        thunk = _AppThunk(space, state, w_func, __args__)
-        self.bind(thunk)
-
-    def w_switch(self):
-        space = self.space
-        if self.frame is None:
-            raise OperationError(space.w_ValueError, space.wrap(
-                "cannot switch to an unbound Coroutine"))
-        state = self.costate
-        self.switch()
-        w_ret, state.w_tempval = state.w_tempval, space.w_None
-        return w_ret
-
-    def switch(self):
-        space = self.space
-        try:
-            Coroutine.switch(self)
-        except CoroutineExit:
-            raise OperationError(self.costate.w_CoroutineExit, space.w_None)
-
-    def w_finished(self, w_excinfo):
-        pass
-
-    def finish(self, operror=None):
-        space = self.space
-        if isinstance(operror, OperationError):
-            w_exctype = operror.w_type
-            w_excvalue = operror.get_w_value(space)
-            w_exctraceback = operror.get_traceback()
-            w_excinfo = space.newtuple([w_exctype, w_excvalue, w_exctraceback])
-
-            if w_exctype is self.costate.w_CoroutineExit:
-                self.coroutine_exit = True
-        else:
-            w_N = space.w_None
-            w_excinfo = space.newtuple([w_N, w_N, w_N])
-
-        return space.call_method(space.wrap(self),'finished', w_excinfo)
-
-    def hello(self):
-        ec = self.space.getexecutioncontext()
-        self.subctx.enter(ec)
-
-    def goodbye(self):
-        ec = self.space.getexecutioncontext()
-        self.subctx.leave(ec)
-
-    def w_kill(self):
-        self.kill()
-
-    def w_throw(self, w_type, w_value=None, w_traceback=None):
-        space = self.space
-
-        operror = OperationError(w_type, w_value)
-        operror.normalize_exception(space)
-
-        if not space.is_w(w_traceback, space.w_None):
-            from pypy.interpreter import pytraceback
-            tb = space.interpclass_w(w_traceback)
-            if tb is None or not space.is_true(space.isinstance(tb,
-                space.gettypeobject(pytraceback.PyTraceback.typedef))):
-                raise OperationError(space.w_TypeError,
-                      space.wrap("throw: arg 3 must be a traceback or None"))
-            operror.set_traceback(tb)
-
-        self._kill(operror)
-
-    def _userdel(self):
-        if self.get_is_zombie():
-            return
-        self.set_is_zombie(True)
-        self.space.userdel(self.space.wrap(self))
-
-    def w_getcurrent(space):
-        return space.wrap(AppCoroutine._get_state(space).current)
-    w_getcurrent = staticmethod(w_getcurrent)
-
-    def w_getmain(space):
-        return space.wrap(AppCoroutine._get_state(space).main)
-    w_getmain = staticmethod(w_getmain)
-
-    # pickling interface
-    def descr__reduce__(self, space):
-        # this is trying to be simplistic at the moment.
-        # we neither allow to pickle main (which can become a mess
-        # since it has some deep anchestor frames)
-        # nor we allow to pickle the current coroutine.
-        # rule: switch before pickling.
-        # you cannot construct the tree that you are climbing.
-        from pypy.interpreter.mixedmodule import MixedModule
-        w_mod    = space.getbuiltinmodule('_stackless')
-        mod      = space.interp_w(MixedModule, w_mod)
-        w_mod2    = space.getbuiltinmodule('_pickle_support')
-        mod2      = space.interp_w(MixedModule, w_mod2)
-        w_new_inst = mod.get('coroutine')
-        w        = space.wrap
-        nt = space.newtuple
-        ec = self.space.getexecutioncontext()
-
-        if self is self.costate.main:
-            return nt([mod.get('_return_main'), nt([])])
-
-        thunk = self.thunk
-        if isinstance(thunk, _AppThunk):
-            w_args, w_kwds = thunk.args.topacked()
-            w_thunk = nt([thunk.w_func, w_args, w_kwds])
-        else:
-            w_thunk = space.w_None
-
-        tup_base = [
-            ]
-        tup_state = [
-            w(self.flags),
-            self.subctx.getstate(space),
-            w_thunk,
-            w(self.parent),
-            ]
-
-        return nt([w_new_inst, nt(tup_base), nt(tup_state)])
-
-    def descr__setstate__(self, space, w_args):
-        w_flags, w_state, w_thunk, w_parent = space.unpackiterable(w_args,
-                                                        expected_length=4)
-        self.flags = space.int_w(w_flags)
-        if space.is_w(w_parent, space.w_None):
-            w_parent = self.w_getmain(space)
-        self.parent = space.interp_w(AppCoroutine, w_parent)
-        ec = self.space.getexecutioncontext()
-        self.subctx.setstate(space, w_state)
-        if space.is_w(w_thunk, space.w_None):
-            if space.is_w(w_state, space.w_None):
-                self.thunk = None
-            else:
-                self.bind(_ResumeThunk(space, self.costate, self.subctx.topframe))
-        else:
-            w_func, w_args, w_kwds = space.unpackiterable(w_thunk,
-                                                          expected_length=3)
-            args = Arguments.frompacked(space, w_args, w_kwds)
-            self.bind(_AppThunk(space, self.costate, w_func, args))
-
-
-# _mixin_ did not work
-for methname in StacklessFlags.__dict__:
-    meth = getattr(StacklessFlags, methname)
-    if hasattr(meth, 'im_func'):
-        setattr(AppCoroutine, meth.__name__, meth.im_func)
-del meth, methname
-
-def w_get_is_zombie(self, space):
-    return space.wrap(self.get_is_zombie())
-AppCoroutine.w_get_is_zombie = w_get_is_zombie
-
-def w_get_is_alive(self, space):
-    return space.wrap(self.is_alive())
-AppCoroutine.w_get_is_alive = w_get_is_alive
-
-def w_descr__framestack(self, space):
-    assert isinstance(self, AppCoroutine)
-    counter = 0
-    f = self.subctx.topframe
-    while f is not None:
-        counter += 1
-        f = f.f_backref()
-    items = [None] * counter
-    f = self.subctx.topframe
-    while f is not None:
-        counter -= 1
-        assert counter >= 0
-        items[counter] = space.wrap(f)
-        f = f.f_backref()
-    assert counter == 0
-    return space.newtuple(items)
-
-def makeStaticMethod(module, classname, funcname):
-    "NOT_RPYTHON"
-    space = module.space
-    w_klass = space.getattr(space.wrap(module), space.wrap(classname))
-    # HACK HACK HACK
-    # make the typeobject mutable for a while
-    from pypy.objspace.std.typeobject import W_TypeObject
-    assert isinstance(w_klass, W_TypeObject)
-    old_flag = w_klass.flag_heaptype
-    w_klass.flag_heaptype = True
-
-    space.appexec([w_klass, space.wrap(funcname)], """
-        (klass, funcname):
-            func = getattr(klass, funcname)
-            setattr(klass, funcname, staticmethod(func.im_func))
-    """)
-    w_klass.flag_heaptype = old_flag
-
-def post_install(module):
-    makeStaticMethod(module, 'coroutine', 'getcurrent')
-    makeStaticMethod(module, 'coroutine', 'getmain')
-    space = module.space
-    AppCoroutine._get_state(space).post_install()
-
-# space.appexec("""() :
-
-# maybe use __spacebind__ for postprocessing
-
-AppCoroutine.typedef = TypeDef("coroutine",
-    __new__ = interp2app(AppCoroutine.descr_method__new__.im_func),
-    bind = interp2app(AppCoroutine.w_bind),
-    switch = interp2app(AppCoroutine.w_switch),
-    kill = interp2app(AppCoroutine.w_kill),
-    throw = interp2app(AppCoroutine.w_throw),
-    finished = interp2app(AppCoroutine.w_finished),
-    is_alive = GetSetProperty(AppCoroutine.w_get_is_alive),
-    is_zombie = GetSetProperty(AppCoroutine.w_get_is_zombie,
-      doc=AppCoroutine.get_is_zombie.__doc__), #--- this flag is a bit obscure
-      # and not useful (it's totally different from Coroutine.is_zombie(), too)
-      # but lib/stackless.py uses it
-    _framestack = GetSetProperty(w_descr__framestack),
-    getcurrent = interp2app(AppCoroutine.w_getcurrent),
-    getmain = interp2app(AppCoroutine.w_getmain),
-    __reduce__   = interp2app(AppCoroutine.descr__reduce__),
-    __setstate__ = interp2app(AppCoroutine.descr__setstate__),
-    __module__ = '_stackless',
-)
-
-class AppCoState(BaseCoState):
-    def __init__(self, space):
-        BaseCoState.__init__(self)
-        self.w_tempval = space.w_None
-        self.space = space
-
-        # XXX Workaround: for now we need to instantiate these classes
-        # explicitly for translation to work
-        W_CoroutineExit(space)
-        W_TaskletExit(space)
-
-        # Exporting new exception to space
-        self.w_CoroutineExit = space.gettypefor(W_CoroutineExit)
-        space.setitem(
-                      space.exceptions_module.w_dict,
-                      space.new_interned_str('CoroutineExit'),
-                      self.w_CoroutineExit)
-        space.setitem(space.builtin.w_dict,
-                      space.new_interned_str('CoroutineExit'),
-                      self.w_CoroutineExit)
-
-        # Should be moved to interp_stackless.py if it's ever implemented...
-        self.w_TaskletExit = space.gettypefor(W_TaskletExit)
-        space.setitem(
-                      space.exceptions_module.w_dict,
-                      space.new_interned_str('TaskletExit'),
-                      self.w_TaskletExit)
-        space.setitem(space.builtin.w_dict,
-                      space.new_interned_str('TaskletExit'),
-                      self.w_TaskletExit)
-
-    def post_install(self):
-        self.current = self.main = AppCoroutine(self.space, state=self)
-        self.main.subctx.clear_framestack()      # wack
-
-def return_main(space):
-    return AppCoroutine._get_state(space).main
-
-def get_stack_depth_limit(space):
-    return space.wrap(rstack.get_stack_depth_limit())
-
- at unwrap_spec(limit=int)
-def set_stack_depth_limit(space, limit):
-    rstack.set_stack_depth_limit(limit)
-
-
-# ___________________________________________________________________
-# unpickling trampoline
-
-def resume_frame(space, w_frame):
-    from pypy.interpreter.pyframe import PyFrame
-    frame = space.interp_w(PyFrame, w_frame, can_be_None=True)
-    w_result = space.w_None
-    operr = None
-    executioncontext = frame.space.getexecutioncontext()
-    while frame is not None:
-        code = frame.pycode.co_code
-        instr = frame.last_instr
-        opcode = ord(code[instr])
-        map = pythonopcode.opmap
-        call_ops = [map['CALL_FUNCTION'], map['CALL_FUNCTION_KW'], map['CALL_FUNCTION_VAR'],
-                    map['CALL_FUNCTION_VAR_KW'], map['CALL_METHOD']]
-        assert opcode in call_ops
-        instr += 1
-        oparg = ord(code[instr]) | ord(code[instr + 1]) << 8
-        nargs = oparg & 0xff
-        nkwds = (oparg >> 8) & 0xff
-        if nkwds == 0:     # only positional arguments
-            # fast paths leaves things on the stack, pop them
-            if space.config.objspace.opcodes.CALL_METHOD and opcode == map['CALL_METHOD']:
-                frame.dropvalues(nargs + 2)
-            elif opcode == map['CALL_FUNCTION']:
-                frame.dropvalues(nargs + 1)
-
-        # small hack: unlink frame out of the execution context, because
-        # execute_frame will add it there again
-        executioncontext.topframeref = jit.non_virtual_ref(frame.f_backref())
-        frame.last_instr = instr + 1 # continue after the call
-        try:
-            w_result = frame.execute_frame(w_result, operr)
-        except OperationError, operr:
-            pass
-        frame = frame.f_backref()
-    if operr:
-        raise operr
-    return w_result
diff --git a/pypy/module/_stackless/interp_greenlet.py b/pypy/module/_stackless/interp_greenlet.py
deleted file mode 100644
--- a/pypy/module/_stackless/interp_greenlet.py
+++ /dev/null
@@ -1,238 +0,0 @@
-from pypy.interpreter.argument import Arguments
-from pypy.interpreter.typedef import GetSetProperty, TypeDef
-from pypy.interpreter.gateway import interp2app
-from pypy.interpreter.gateway import NoneNotWrapped
-from pypy.interpreter.error import OperationError
-
-from pypy.module._stackless.rcoroutine import Coroutine, BaseCoState
-from pypy.module._stackless.rcoroutine import AbstractThunk, syncstate
-from pypy.module._stackless.interp_coroutine import makeStaticMethod
-
-
-class GreenletThunk(AbstractThunk):
-
-    def __init__(self, greenlet):
-        self.greenlet = greenlet
-
-    def call(self):
-        greenlet = self.greenlet
-        greenlet.active = True
-        try:
-            space = greenlet.space
-            args_w = greenlet.costate.args_w
-            __args__ = Arguments(space, args_w)
-            try:
-                w_run = space.getattr(space.wrap(greenlet), space.wrap('run'))
-                greenlet.w_callable = None
-                w_result = space.call_args(w_run, __args__)
-            except OperationError, operror:
-                if not operror.match(space, greenlet.costate.w_GreenletExit):
-                    raise
-                w_result = operror.get_w_value(space)
-        finally:
-            greenlet.active = False
-        greenlet.costate.args_w = [w_result]
-
-class AppGreenletCoState(BaseCoState):
-    def __init__(self, space):
-        BaseCoState.__init__(self)
-        self.args_w = None
-        self.space = space
-        self.w_GreenletExit  = get(space, "GreenletExit")
-        self.w_GreenletError = get(space, "GreenletError")
-
-    def post_install(self):
-        self.current = self.main = AppGreenlet(self.space, is_main=True)
-
-class AppGreenlet(Coroutine):
-    def __init__(self, space, w_callable=None, is_main=False):
-        Coroutine.__init__(self, self._get_state(space))
-        self.space = space
-        self.w_callable = w_callable
-        self.active = is_main
-        self.subctx = space.getexecutioncontext().Subcontext()
-        if is_main:
-            self.subctx.clear_framestack()      # wack
-        else:
-            self.bind(GreenletThunk(self))
-
-    def descr_method__new__(space, w_subtype, __args__):
-        co = space.allocate_instance(AppGreenlet, w_subtype)
-        AppGreenlet.__init__(co, space)
-        return space.wrap(co)
-
-    def descr_method__init__(self, w_run=NoneNotWrapped,
-                                   w_parent=NoneNotWrapped):
-        if w_run is not None:
-            self.set_run(w_run)
-        if w_parent is not None:
-            self.set_parent(w_parent)
-
-    def _get_state(space):
-        return space.fromcache(AppGreenletCoState)
-    _get_state = staticmethod(_get_state)
-
-    def hello(self):
-        ec = self.space.getexecutioncontext()
-        self.subctx.enter(ec)
-
-    def goodbye(self):
-        ec = self.space.getexecutioncontext()
-        self.subctx.leave(ec)
-
-    def w_getcurrent(space):
-        return space.wrap(AppGreenlet._get_state(space).current)
-    w_getcurrent = staticmethod(w_getcurrent)
-
-    def w_switch(self, args_w):
-        # Find the switch target - it might be a parent greenlet
-        space = self.space
-        costate = self.costate
-        target = self
-        while target.isdead():
-            target = target.parent
-            assert isinstance(target, AppGreenlet)
-        # Switch to it
-        costate.args_w = args_w
-        if target is not costate.current:
-            target.switch()
-        else:
-            # case not handled in Coroutine.switch()
-            syncstate._do_things_to_do()
-        result_w = costate.args_w
-        costate.args_w = None
-        # costate.args_w can be set to None above for throw(), but then
-        # switch() should have raised.  At this point cosstate.args_w != None.
-        assert result_w is not None
-        # Return the result of a switch, packaging it in a tuple if
-        # there is more than one value.
-        if len(result_w) == 1:
-            return result_w[0]
-        return space.newtuple(result_w)
-
-    def w_throw(self, w_type=None, w_value=None, w_traceback=None):
-        space = self.space
-        if space.is_w(w_type, space.w_None):
-            w_type = self.costate.w_GreenletExit
-        # Code copied from RAISE_VARARGS but slightly modified.  Not too nice.
-        operror = OperationError(w_type, w_value)
-        operror.normalize_exception(space)
-        if not space.is_w(w_traceback, space.w_None):
-            from pypy.interpreter import pytraceback
-            tb = space.interpclass_w(w_traceback)
-            if tb is None or not space.is_true(space.isinstance(tb, 
-                space.gettypeobject(pytraceback.PyTraceback.typedef))):
-                raise OperationError(space.w_TypeError,
-                      space.wrap("throw: arg 3 must be a traceback or None"))
-            operror.set_traceback(tb)
-        # Dead greenlet: turn GreenletExit into a regular return
-        if self.isdead() and operror.match(space, self.costate.w_GreenletExit):
-            args_w = [operror.get_w_value(space)]
-        else:
-            syncstate.push_exception(operror)
-            args_w = None
-        return self.w_switch(args_w)
-
-    def _userdel(self):
-        self.space.userdel(self.space.wrap(self))
-
-    def isdead(self):
-        return self.thunk is None and not self.active
-
-    def w_get_is_dead(self, space):
-        return space.newbool(self.isdead())
-
-    def descr__nonzero__(self):
-        return self.space.newbool(self.active)
-
-    def w_get_run(self, space):
-        w_run = self.w_callable
-        if w_run is None:
-            raise OperationError(space.w_AttributeError, space.wrap("run"))
-        return w_run
-
-    def set_run(self, w_run):
-        space = self.space
-        if self.thunk is None:
-            raise OperationError(space.w_AttributeError,
-                                 space.wrap("run cannot be set "
-                                            "after the start of the greenlet"))
-        self.w_callable = w_run
-
-    def w_set_run(self, space, w_run):
-        self.set_run(w_run)
-
-    def w_del_run(self, space):
-        if self.w_callable is None:
-            raise OperationError(space.w_AttributeError, space.wrap("run"))
-        self.w_callable = None
-
-    def w_get_parent(self, space):
-        return space.wrap(self.parent)
-
-    def set_parent(self, w_parent):
-        space = self.space
-        newparent = space.interp_w(AppGreenlet, w_parent)
-        if newparent.costate is not self.costate:
-            raise OperationError(self.costate.w_GreenletError,
-                                 space.wrap("invalid foreign parent"))
-        curr = newparent
-        while curr:
-            if curr is self:
-                raise OperationError(space.w_ValueError,
-                                     space.wrap("cyclic parent chain"))
-            curr = curr.parent
-        self.parent = newparent
-
-    def w_set_parent(self, space, w_parent):
-        self.set_parent(w_parent)
-
-    def w_get_frame(self, space):
-        if not self.active or self.costate.current is self:
-            f = None
-        else:
-            f = self.subctx.topframe
-        return space.wrap(f)
-
-def get(space, name):
-    w_module = space.getbuiltinmodule('_stackless')
-    return space.getattr(w_module, space.wrap(name))
-
-def post_install(module):
-    "NOT_RPYTHON"
-    makeStaticMethod(module, 'greenlet', 'getcurrent')
-    space = module.space
-    state = AppGreenlet._get_state(space)
-    state.post_install()
-    w_greenlet = get(space, 'greenlet')
-    # HACK HACK HACK
-    # make the typeobject mutable for a while
-    from pypy.objspace.std.typeobject import W_TypeObject
-    assert isinstance(w_greenlet, W_TypeObject)
-    old_flag = w_greenlet.flag_heaptype
-    w_greenlet.flag_heaptype = True
-    space.appexec([w_greenlet,
-                   state.w_GreenletExit,
-                   state.w_GreenletError], """
-    (greenlet, exit, error):
-        greenlet.GreenletExit = exit
-        greenlet.error = error
-    """)
-    w_greenlet.flag_heaptype = old_flag
-
-AppGreenlet.typedef = TypeDef("greenlet",
-    __new__ = interp2app(AppGreenlet.descr_method__new__.im_func),
-    __init__ = interp2app(AppGreenlet.descr_method__init__),
-    switch = interp2app(AppGreenlet.w_switch),
-    dead = GetSetProperty(AppGreenlet.w_get_is_dead),
-    run = GetSetProperty(AppGreenlet.w_get_run,
-                         AppGreenlet.w_set_run,
-                         AppGreenlet.w_del_run),
-    parent = GetSetProperty(AppGreenlet.w_get_parent,
-                            AppGreenlet.w_set_parent),
-    getcurrent = interp2app(AppGreenlet.w_getcurrent),
-    throw = interp2app(AppGreenlet.w_throw),
-    gr_frame = GetSetProperty(AppGreenlet.w_get_frame),
-    __nonzero__ = interp2app(AppGreenlet.descr__nonzero__),
-    __module__ = '_stackless',
-)
diff --git a/pypy/module/_stackless/interp_stackless.py b/pypy/module/_stackless/interp_stackless.py
deleted file mode 100644
--- a/pypy/module/_stackless/interp_stackless.py
+++ /dev/null
@@ -1,28 +0,0 @@
-from pypy.interpreter.baseobjspace import Wrappable
-from pypy.interpreter.typedef import TypeDef
-from pypy.interpreter.gateway import interp2app
-import os
-
-
-class tasklet(Wrappable):
-
-    def __init__(self, space):
-        self.space = space
-        self.flags = 0
-        self.state = None
-
-    def descr_method__new__(space, w_subtype):
-        t = space.allocate_instance(tasklet, w_subtype)
-        tasklet.__init__(t, space)
-        return space.wrap(t)
-
-    def w_demo(self):
-        output("42")
-
-tasklet.typedef = TypeDef("tasklet",
-    __new__ = interp2app(tasklet.descr_method__new__.im_func),
-    demo = interp2app(tasklet.w_demo),
-)
-
-def output(stuff):
-    os.write(2, stuff + '\n')
diff --git a/pypy/module/_stackless/rclonable.py b/pypy/module/_stackless/rclonable.py
deleted file mode 100644
--- a/pypy/module/_stackless/rclonable.py
+++ /dev/null
@@ -1,87 +0,0 @@
-from pypy.module._stackless.interp_coroutine import AbstractThunk, Coroutine
-from pypy.rlib.rgc import gc_swap_pool, gc_clone
-from pypy.rlib.objectmodel import we_are_translated
-
-
-class InterpClonableMixin:
-    local_pool = None
-    _mixin_ = True
-
-    def hello_local_pool(self):
-        if we_are_translated():
-            self.saved_pool = gc_swap_pool(self.local_pool)
-
-    def goodbye_local_pool(self):
-        if we_are_translated():
-            self.local_pool = gc_swap_pool(self.saved_pool)
-            self.saved_pool = None
-
-    def clone_into(self, copy, extradata=None):
-        if not we_are_translated():
-            raise NotImplementedError
-        # cannot gc_clone() directly self, because it is not in its own
-        # local_pool.  Moreover, it has a __del__, which cloning doesn't
-        # support properly at the moment.
-        copy.parent = self.parent
-        # the hello/goodbye pair has two purposes: it forces
-        # self.local_pool to be computed even if it was None up to now,
-        # and it puts the 'data' tuple in the correct pool to be cloned.
-        self.hello_local_pool()
-        data = (self.frame, extradata)
-        self.goodbye_local_pool()
-        # clone!
-        data, copy.local_pool = gc_clone(data, self.local_pool)
-        copy.frame, extradata = data
-        copy.thunk = self.thunk # in case we haven't switched to self yet
-        return extradata
-
-
-class InterpClonableCoroutine(Coroutine, InterpClonableMixin):
-
-    def hello(self):
-        self.hello_local_pool()
-
-    def goodbye(self):
-        self.goodbye_local_pool()
-
-    def clone(self):
-        # hack, this is overridden in AppClonableCoroutine
-        if self.getcurrent() is self:
-            raise RuntimeError("clone() cannot clone the current coroutine; "
-                               "use fork() instead")
-        copy = InterpClonableCoroutine(self.costate)
-        self.clone_into(copy)
-        return copy
-
-
-class ForkThunk(AbstractThunk):
-    def __init__(self, coroutine):
-        self.coroutine = coroutine
-        self.newcoroutine = None
-    def call(self):
-        oldcoro = self.coroutine
-        self.coroutine = None
-        newcoro = oldcoro.clone()
-        newcoro.parent = oldcoro
-        self.newcoroutine = newcoro
-
-def fork():
-    """Fork, as in the Unix fork(): the call returns twice, and the return
-    value of the call is either the new 'child' coroutine object (if returning
-    into the parent), or None (if returning into the child).  This returns
-    into the parent first, which can switch to the child later.
-    """
-    current = InterpClonableCoroutine.getcurrent()
-    if not isinstance(current, InterpClonableCoroutine):
-        raise RuntimeError("fork() in a non-clonable coroutine")
-    thunk = ForkThunk(current)
-    coro_fork = InterpClonableCoroutine()
-    coro_fork.bind(thunk)
-    coro_fork.switch()
-    # we resume here twice.  The following would need explanations about
-    # why it returns the correct thing in both the parent and the child...
-    return thunk.newcoroutine
-
-##    from pypy.rpython.lltypesystem import lltype, lloperation
-##    lloperation.llop.debug_view(lltype.Void, current, thunk,
-##        lloperation.llop.gc_x_size_header(lltype.Signed))
diff --git a/pypy/module/_stackless/rcoroutine.py b/pypy/module/_stackless/rcoroutine.py
deleted file mode 100644
--- a/pypy/module/_stackless/rcoroutine.py
+++ /dev/null
@@ -1,10 +0,0 @@
-from pypy.rlib.rcoroutine import make_coroutine_classes
-from pypy.interpreter.baseobjspace import Wrappable
-
-d = make_coroutine_classes(Wrappable)
-
-Coroutine = d['Coroutine']
-BaseCoState = d['BaseCoState']
-AbstractThunk = d['AbstractThunk']
-syncstate = d['syncstate']
-CoroutineExit = d['CoroutineExit']
diff --git a/pypy/module/_stackless/stackless_flags.py b/pypy/module/_stackless/stackless_flags.py
deleted file mode 100644
--- a/pypy/module/_stackless/stackless_flags.py
+++ /dev/null
@@ -1,201 +0,0 @@
-"""
-basic definitions for tasklet flags.
-For simplicity and compatibility,
-they are defined the same for coroutines,
-even if they are not used.
-
-taken from tasklet_structs.h
-----------------------------
-
-/***************************************************************************
-
-    Tasklet Flag Definition
-    -----------------------
-
-    blocked:        The tasklet is either waiting in a channel for
-                    writing (1) or reading (-1) or not blocked (0).
-                    Maintained by the channel logic. Do not change.
-
-    atomic:         If true, schedulers will never switch. Driven by
-                    the code object or dynamically, see below.
-
-    ignore_nesting: Allows auto-scheduling, even if nesting_level
-                    is not zero.
-
-    autoschedule:   The tasklet likes to be auto-scheduled. User driven.
-
-    block_trap:     Debugging aid. Whenever the tasklet would be
-                    blocked by a channel, an exception is raised.
-
-    is_zombie:      This tasklet is almost dead, its deallocation has
-                    started. The tasklet *must* die at some time, or the
-                    process can never end.
-
-    pending_irq:    If set, an interrupt was issued during an atomic
-                    operation, and should be handled when possible.
-
-
-    Policy for atomic/autoschedule and switching:
-    ---------------------------------------------
-    A tasklet switch can always be done explicitly by calling schedule().
-    Atomic and schedule are concerned with automatic features.
-
-    atomic  autoschedule
-
-        1       any     Neither a scheduler nor a watchdog will
-                        try to switch this tasklet.
-
-        0       0       The tasklet can be stopped on desire, or it
-                        can be killed by an exception.
-
-        0       1       Like above, plus auto-scheduling is enabled.
-
-    Default settings:
-    -----------------
-    All flags are zero by default.
-
- ***************************************************************************/
-
-typedef struct _tasklet_flags {
-        int blocked: 2;
-        unsigned int atomic: 1;
-        unsigned int ignore_nesting: 1;
-        unsigned int autoschedule: 1;
-        unsigned int block_trap: 1;
-        unsigned int is_zombie: 1;
-        unsigned int pending_irq: 1;
-} PyTaskletFlagStruc;
-"""
-
-from pypy.rlib.rarithmetic import LONG_BIT, intmask
-
-class BitSetDef(object):
-    __slots__ = "_names __dict__ _attrname".split()
-
-    def __init__(self, _attrname):
-        self._names = []
-        self._attrname = _attrname
-        
-    def __setattr__(self, key, value):
-        if key not in self.__slots__:
-            assert key not in self.__dict__
-            self._names.append(key)
-        object.__setattr__(self, key, value)
-
-    def __iter__(self):
-        return self._enum_objects()
-    
-    def _enum_objects(self):
-        for name in self._names:
-            yield name, getattr(self, name)
-
-# negative values are user-writable
-flags = BitSetDef("flags")
-flags.blocked           =   2, """writing (1) or reading (-1) or not blocked (0)"""
-flags.atomic            =  -1, """If true, schedulers will never switch"""
-flags.ignore_nesting    =  -1, """allow auto-scheduling in nested interpreters"""
-flags.autoschedule      =  -1, """enable auto-scheduling"""
-flags.block_trap        =  -1, """raise an exception instead of blocking"""
-flags.is_zombie         =   1, """__del__ is in progress"""
-flags.pending_irq       =   1, """an interrupt occured while being atomic"""
-
-def make_get_bits(name, bits, shift):
-    """ return a bool for single bits, signed int otherwise """
-    signmask = 1 << (bits - 1 + shift)
-    lshift = bits + shift
-    rshift = bits
-    if bits == 1:
-        return "bool(%s & 0x%x)" % (name, signmask)
-    else:
-        return "intmask(%s << (LONG_BIT-%d)) >> (LONG_BIT-%d)" % (name, lshift, rshift)
-
-def make_set_bits(name, bits, shift):
-    datamask = int('1' * bits, 2)
-    clearmask = datamask << shift
-    return "%s & ~0x%x | (value & 0x%x) << %d" % (name, clearmask, datamask, shift)
-
-def gen_code():
-    from cStringIO import StringIO
-    f = StringIO()
-    print >> f, "class StacklessFlags(object):"
-    print >> f, "    _mixin_ = True"
-    shift = 0
-    field = "self.%s" % flags._attrname
-    for name, (bits, doc) in flags:
-        write, bits = bits < 0, abs(bits)
-        print >> f
-        print >> f, '    def get_%s(self):' % name
-        print >> f, '        """%s"""' % doc
-        print >> f, '        return %s' % make_get_bits(field, bits, shift)
-        print >> f, '    def set_%s(self, value):' % name
-        print >> f, '        """%s"""' % doc
-        print >> f, '        %s = %s' % (field, make_set_bits(field, bits, shift))
-        print >> f, '    set_%s._public = %s' % (name, write)
-        shift += bits
-    return f.getvalue()
-
-# BEGIN generated code
-class StacklessFlags(object):
-    _mixin_ = True
-
-    def get_blocked(self):
-        """writing (1) or reading (-1) or not blocked (0)"""
-        return intmask(self.flags << (LONG_BIT-2)) >> (LONG_BIT-2)
-    def set_blocked(self, value):
-        """writing (1) or reading (-1) or not blocked (0)"""
-        self.flags = self.flags & ~0x3 | (value & 0x3) << 0
-    set_blocked._public = False
-
-    def get_atomic(self):
-        """If true, schedulers will never switch"""
-        return bool(self.flags & 0x4)
-    def set_atomic(self, value):
-        """If true, schedulers will never switch"""
-        self.flags = self.flags & ~0x4 | (value & 0x1) << 2
-    set_atomic._public = True
-
-    def get_ignore_nesting(self):
-        """allow auto-scheduling in nested interpreters"""
-        return bool(self.flags & 0x8)
-    def set_ignore_nesting(self, value):
-        """allow auto-scheduling in nested interpreters"""
-        self.flags = self.flags & ~0x8 | (value & 0x1) << 3
-    set_ignore_nesting._public = True
-
-    def get_autoschedule(self):
-        """enable auto-scheduling"""
-        return bool(self.flags & 0x10)
-    def set_autoschedule(self, value):
-        """enable auto-scheduling"""
-        self.flags = self.flags & ~0x10 | (value & 0x1) << 4
-    set_autoschedule._public = True
-
-    def get_block_trap(self):
-        """raise an exception instead of blocking"""
-        return bool(self.flags & 0x20)
-    def set_block_trap(self, value):
-        """raise an exception instead of blocking"""
-        self.flags = self.flags & ~0x20 | (value & 0x1) << 5
-    set_block_trap._public = True
-
-    def get_is_zombie(self):
-        """__del__ is in progress"""
-        return bool(self.flags & 0x40)
-    def set_is_zombie(self, value):
-        """__del__ is in progress"""
-        self.flags = self.flags & ~0x40 | (value & 0x1) << 6
-    set_is_zombie._public = False
-
-    def get_pending_irq(self):
-        """an interrupt occured while being atomic"""
-        return bool(self.flags & 0x80)
-    def set_pending_irq(self, value):
-        """an interrupt occured while being atomic"""
-        self.flags = self.flags & ~0x80 | (value & 0x1) << 7
-    set_pending_irq._public = False
-
-# END generated code
-
-if __name__ == '__main__':
-    # paste this into the file
-    print gen_code()
diff --git a/pypy/module/_stackless/test/__init__.py b/pypy/module/_stackless/test/__init__.py
deleted file mode 100644
--- a/pypy/module/_stackless/test/__init__.py
+++ /dev/null
@@ -1,1 +0,0 @@
-#
\ No newline at end of file
diff --git a/pypy/module/_stackless/test/conftest.py b/pypy/module/_stackless/test/conftest.py
deleted file mode 100644
--- a/pypy/module/_stackless/test/conftest.py
+++ /dev/null
@@ -1,8 +0,0 @@
-import sys
-import py.test
-
-def pytest_runtest_setup(item):
-    py.test.importorskip('greenlet')
-    if sys.platform == 'win32':
-        py.test.skip("stackless tests segfault on Windows")
-
diff --git a/pypy/module/_stackless/test/slp_test_pickle.py b/pypy/module/_stackless/test/slp_test_pickle.py
deleted file mode 100644
--- a/pypy/module/_stackless/test/slp_test_pickle.py
+++ /dev/null
@@ -1,35 +0,0 @@
-from pypy.conftest import gettestobjspace
-
-# app-level testing of coroutine pickling
-
-class AppTest_Pickle:
-
-    def setup_class(cls):
-        space = gettestobjspace(usemodules=('_stackless',))
-        cls.space = space
-
-    def test_simple_ish(self):
-
-        output = []
-        import _stackless
-        def f(coro, n, x):
-            if n == 0:
-                coro.switch()
-                return
-            f(coro, n-1, 2*x)
-            output.append(x)
-
-        def example():
-            main_coro = _stackless.coroutine.getcurrent()
-            sub_coro = _stackless.coroutine()
-            sub_coro.bind(f, main_coro, 5, 1)
-            sub_coro.switch()
-
-            import pickle
-            pckl = pickle.dumps(sub_coro)
-            new_coro = pickle.loads(pckl)
-
-            new_coro.switch()
-
-        example()
-        assert output == [16, 8, 4, 2, 1]
diff --git a/pypy/module/_stackless/test/test_choicepoint.py b/pypy/module/_stackless/test/test_choicepoint.py
deleted file mode 100644
--- a/pypy/module/_stackless/test/test_choicepoint.py
+++ /dev/null
@@ -1,85 +0,0 @@
-import py; py.test.skip("clonable coroutines not really maintained any more")
-
-from pypy.rlib.rcoroutine import AbstractThunk
-from pypy.module._stackless.rclonable import InterpClonableCoroutine as ClonableCoroutine
-
-class ChoicePointHolder(object):
-    def __init__(self):
-        self.choicepoints = []
-        self.clone_me = False
-        self.answer = 0
-        self.solutions_count = 0
-
-    def next_choice(self):
-        return self.choicepoints.pop()
-
-    def add(self, choice, answer=0):
-        self.choicepoints.append((choice, answer))
-
-    def more_choices(self):
-        return bool(self.choicepoints)
-
-    def choice(self):
-        #os.write(1, "choice\n")
-        self.clone_me = True
-        self.g_main.switch()
-        #os.write(1, "answer: %d\n" % (self.answer,))
-        return self.answer
-
-    def fail(self):
-        self.g_main.switch()
-        assert False
-
-choicepoints = ChoicePointHolder()
-
-# ____________________________________________________________
-
-class SearchTask(AbstractThunk):
-    def call(self):
-        path = []
-        for i in range(10):
-            res = choicepoints.choice()
-            assert len(path) == i
-            path.append(res)
-            #os.write(1, "{%x} trying: %s\n" % (id(path), path))
-            if i == 3:
-                import gc; gc.collect()
-        #os.write(1, "{%x} found a solution: %s\n" % (id(path), path))
-        choicepoints.solutions_count += 1
-
-# ____________________________________________________________
-
-
-class SearchAllTask(AbstractThunk):
-    def call(self):
-        search_coro = ClonableCoroutine()
-        search_coro.bind(SearchTask())
-        choicepoints.add(search_coro)
-
-        #os.write(1, "starting\n")
-        while choicepoints.more_choices():
-            searcher, nextvalue = choicepoints.next_choice()
-            choicepoints.clone_me = False
-            choicepoints.answer = nextvalue
-            #os.write(1, '<<< {%x} %d\n' % (id(searcher), nextvalue))
-            searcher.switch()
-            #os.write(1, '>>> %d\n' % (choicepoints.clone_me,))
-            if choicepoints.clone_me:
-                searcher2 = searcher.clone()
-                #os.write(1, 'searcher = {%x}, searcher2 = {%x}\n' % (
-                #    id(searcher), id(searcher2)))
-                choicepoints.add(searcher, 5)
-                choicepoints.add(searcher2, 4)
-
-def entry_point():
-    choicepoints.g_main = ClonableCoroutine()
-    choicepoints.g_main.bind(SearchAllTask())
-    choicepoints.g_main.switch()
-    return choicepoints.solutions_count
-
-def test_choicepoint():
-    from pypy.translator.c.test import test_newgc
-    tester = test_newgc.TestUsingStacklessFramework()
-    fn = tester.getcompiled(entry_point)
-    res = fn()
-    assert res == 2 ** 10
diff --git a/pypy/module/_stackless/test/test_clonable.py b/pypy/module/_stackless/test/test_clonable.py
deleted file mode 100644
--- a/pypy/module/_stackless/test/test_clonable.py
+++ /dev/null
@@ -1,187 +0,0 @@
-import py; py.test.skip("clonable coroutines not really maintained any more")
-
-from pypy.conftest import gettestobjspace, option
-import py, sys
-
-# app-level testing of coroutine cloning
-
-class AppTestClonable:
-
-    def setup_class(cls):
-        if not option.runappdirect:
-            py.test.skip('pure appdirect test (run with -A)')
-        cls.space = space = gettestobjspace(usemodules=('_stackless',))
-        if not space.is_true(space.appexec([], """():
-            import _stackless
-            return hasattr(_stackless, 'clonable')
-        """)):
-            py.test.skip('no _stackless.clonable')
-
-
-    def test_solver(self):
-        import _stackless
-
-        class Fail(Exception):
-            pass
-
-        class Success(Exception):
-            pass
-
-        def first_solution(func):
-            global next_answer
-            co = _stackless.clonable()
-            co.bind(func)
-            pending = [(co, None)]
-            while pending:
-                co, next_answer = pending.pop()
-                try:
-                    co.switch()
-                except Fail:
-                    pass
-                except Success, e:
-                    return e.args[0]
-                else:
-                    # zero_or_one() called, clone the coroutine
-                    co2 = co.clone()
-                    pending.append((co2, 1))
-                    pending.append((co, 0))
-            raise Fail("no solution")
-
-        pending = []
-        main = _stackless.clonable.getcurrent()
-
-        def zero_or_one():
-            main.switch()
-            return next_answer
-
-        # ____________________________________________________________
-
-        invalid_prefixes = {
-            (0, 0): True,
-            (0, 1, 0): True,
-            (0, 1, 1): True,
-            (1, 0): True,
-            (1, 1, 0, 0): True,
-            }
-
-        def example():
-            test = []
-            for n in range(5):
-                test.append(zero_or_one())
-                if tuple(test) in invalid_prefixes:
-                    raise Fail
-            raise Success(test)
-
-        res = first_solution(example)
-        assert res == [1, 1, 0, 1, 0]
-
-
-    def test_myself_may_not_be_me_any_more(self):
-        import gc
-        from _stackless import clonable
-
-        counter = [0]
-
-        def runner():
-            while 1:
-                assert clonable.getcurrent() is coro
-                counter[0] += 1
-                main.switch()
-
-        main = clonable.getcurrent()
-        coro = clonable()
-        coro.bind(runner)
-
-        coro.switch()
-        assert counter == [1]
-
-        assert clonable.getcurrent() is main
-        coro1 = coro.clone()
-        assert counter == [1]
-        assert clonable.getcurrent() is main
-        coro.switch()
-        assert counter == [2]
-        coro.switch()
-        assert counter == [3]
-        assert clonable.getcurrent() is main
-        del coro1
-        gc.collect()
-        #print "collected!"
-        assert clonable.getcurrent() is main
-        assert counter == [3]
-        coro.switch()
-        assert clonable.getcurrent() is main
-        assert counter == [4]
-
-
-    def test_fork(self):
-        import _stackless
-
-        class Fail(Exception):
-            pass
-
-        class Success(Exception):
-            pass
-
-        def first_solution(func):
-            global next_answer
-            co = _stackless.clonable()
-            co.bind(func)
-            try:
-                co.switch()
-            except Success, e:
-                return e.args[0]
-
-        def zero_or_one():
-            sub = _stackless.fork()
-            if sub is not None:
-                # in the parent: run the child first
-                try:
-                    sub.switch()
-                except Fail:
-                    pass
-                # then proceed with answer '1'
-                return 1
-            else:
-                # in the child: answer '0'
-                return 0
-
-        # ____________________________________________________________
-
-        invalid_prefixes = {
-            (0, 0): True,
-            (0, 1, 0): True,
-            (0, 1, 1): True,
-            (1, 0): True,
-            (1, 1, 0, 0): True,
-            }
-
-        def example():
-            test = []
-            for n in range(5):
-                test.append(zero_or_one())
-                if tuple(test) in invalid_prefixes:
-                    raise Fail
-            raise Success(test)
-
-        res = first_solution(example)
-        assert res == [1, 1, 0, 1, 0]
-
-    def test_clone_before_start(self):
-        """Tests that a clonable coroutine can be
-        cloned before it is started
-        (this used to fail with a segmentation fault)
-        """
-        import _stackless
-
-        counter = [0]
-        def simple_coro():
-            print "hello"
-            counter[0] += 1
-
-        s = _stackless.clonable()
-        s.bind(simple_coro)
-        t = s.clone()
-        s.switch()
-        t.switch()
-        assert counter[0] == 2
diff --git a/pypy/module/_stackless/test/test_composable_coroutine.py b/pypy/module/_stackless/test/test_composable_coroutine.py
deleted file mode 100644
--- a/pypy/module/_stackless/test/test_composable_coroutine.py
+++ /dev/null
@@ -1,133 +0,0 @@
-""" a faith is the connection between past and future that divides the
-    application into switch-compatible chunks.
-    -- stakkars
-"""
-from pypy.conftest import gettestobjspace
-from py.test import skip
-
-class AppTest_ComposableCoroutine:
-
-    def setup_class(cls):
-        space = gettestobjspace(usemodules=('_stackless',))
-        cls.space = space
-
-        cls.w_generator_ = space.appexec([], """():
-            import _stackless
-
-            generators_costate = _stackless.usercostate()
-            main = generators_costate.getcurrent()
-
-            class generator_iterator(_stackless.coroutine):
-
-                def __iter__(self):
-                    return self
-
-                def next(self):
-                    if self.gi_answer is not None:
-                        raise ValueError('stackless-generator'
-                                         ' already executing')
-                    self.gi_answer = []
-                    self.gi_caller = generators_costate.getcurrent()
-                    self.switch()
-                    answer = self.gi_answer
-                    self.gi_answer = None
-                    if answer:
-                        return answer[0]
-                    else:
-                        raise StopIteration
-
-            def generator(f):
-                def myfunc(*args, **kwds):
-                    g = generators_costate.spawn(generator_iterator)
-                    g.gi_answer = None
-                    g.bind(f, *args, **kwds)
-                    return g
-                return myfunc
-
-            def Yield(value):
-                g = generators_costate.getcurrent()
-                if g is main:
-                    raise ValueError('Yield() outside any stackless-generator')
-                assert isinstance(g, generator_iterator)
-                assert g.gi_answer == []
-                g.gi_answer.append(value)
-                g.gi_caller.switch()
-
-            generator.Yield = Yield
-            generator._costate = generators_costate
-            return (generator,)
-        """)
-
-    def test_simple_costate(self):
-        import _stackless
-        costate = _stackless.usercostate()
-        main = costate.getcurrent()
-
-        result = []
-        def f():
-            result.append(costate.getcurrent())
-        co = costate.spawn()
-        co.bind(f)
-        co.switch()
-        assert result == [co]
-
-    def test_generator(self):
-        generator, = self.generator_
-
-        def squares(n):
-            for i in range(n):
-                generator.Yield(i*i)
-        squares = generator(squares)
-
-        lst1 = [i*i for i in range(10)]
-        for got in squares(10):
-            expected = lst1.pop(0)
-            assert got == expected
-        assert lst1 == []
-
-    def test_multiple_costates(self):
-        """Test that two independent costates mix transparently:
-
-        - compute_costate, used for a coroutine that fills a list with
-                           some more items each time it is switched to
-
-        - generators_costate, used interally by self.generator (see above)
-        """
-
-        import _stackless
-        generator, = self.generator_
-
-        # you can see how it fails if we don't have two different costates
-        # by setting compute_costate to generator._costate instead
-        compute_costate = _stackless.usercostate()
-        compute_main = compute_costate.getcurrent()
-        lst = []
-
-        def filler():     # -> 0, 1, 2, 100, 101, 102, 200, 201, 202, 300 ...
-            for k in range(5):
-                for j in range(3):
-                    lst.append(100 * k + j)
-                compute_main.switch()
-
-        filler_co = compute_costate.spawn()
-        filler_co.bind(filler)
-
-        def grab_next_value():
-            while not lst:
-                #print 'filling more...'
-                filler_co.switch()
-                #print 'now lst =', lst
-            #print 'grabbing', lst[0]
-            return lst.pop(0)
-
-        def squares(n):
-            for i in range(n):
-                #print 'square:', i
-                generator.Yield(i*grab_next_value())
-        squares = generator(squares)
-
-        lst1 = [0, 1, 4,  300, 404, 510,  1200, 1407, 1616,  2700]
-        for got in squares(10):
-            expected = lst1.pop(0)
-            assert got == expected
-        assert lst1 == []
diff --git a/pypy/module/_stackless/test/test_coroutine.py b/pypy/module/_stackless/test/test_coroutine.py
deleted file mode 100644
--- a/pypy/module/_stackless/test/test_coroutine.py
+++ /dev/null
@@ -1,168 +0,0 @@
-from pypy.conftest import gettestobjspace, option
-from py.test import skip
-
-
-class AppTest_Coroutine:
-
-    def setup_class(cls):
-        space = gettestobjspace(usemodules=('_stackless',))
-        cls.space = space
-
-    def test_raise_propagate(self):
-        import _stackless as stackless
-        co = stackless.coroutine()
-        def f():
-            return 1/0
-        co.bind(f)
-        try:
-            co.switch()
-        except ZeroDivisionError:
-            pass
-        else:
-            raise AssertionError("exception not propagated")
-
-    def test_strange_test(self):
-        from _stackless import coroutine
-        def f():
-            print "in new coro"
-            return 42
-        def create():
-            b = coroutine()
-            b.bind(f)
-            print "bound"
-            b.switch()
-            print "switched"
-            return b
-        a = coroutine()
-        a.bind(create)
-        b = a.switch()
-        # now b.parent = a
-        def nothing():
-            pass
-        a.bind(nothing)
-        def kill():
-            # this sets a.parent = b
-            a.kill()
-        b.bind(kill)
-        b.switch()
-
-    def test_kill(self):
-        import _stackless as stackless
-        co = stackless.coroutine()
-        def f():
-            pass
-        co.bind(f)
-        assert co.is_alive
-        co.kill()
-        assert not co.is_alive
-
-    def test_kill_running(self):
-        coroutineexit = []
-        import _stackless as stackless
-        main = stackless.coroutine.getcurrent()
-        result = []
-        co = stackless.coroutine()
-        def f():
-            x = 2
-            try:
-                result.append(1)
-                main.switch()
-                x = 3
-            except CoroutineExit:
-                coroutineexit.append(True)
-                raise
-            finally:
-                result.append(x)
-            result.append(4)
-        co.bind(f)
-        assert co.is_alive
-        co.switch()
-        assert co.is_alive
-        assert result == [1]
-        co.kill()
-        assert not co.is_alive
-        assert result == [1, 2]
-        assert coroutineexit == [True]
-

From noreply at buildbot.pypy.org  Wed Oct 19 23:49:14 2011
From: noreply at buildbot.pypy.org (snus_mumrik)
Date: Wed, 19 Oct 2011 23:49:14 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-comparison: Returning False for "arr ==
 None". Preparations for "zeors(3) == zeros(4)" to
Message-ID: <20111019214914.72AD7820D7@wyvern.cs.uni-duesseldorf.de>

Author: Ilya Osadchiy <osadchiy.ilya at gmail.com>
Branch: numpy-comparison
Changeset: r48246:58762c30991d
Date: 2011-10-01 21:26 +0300
http://bitbucket.org/pypy/pypy/changeset/58762c30991d/

Log:	Returning False for "arr == None". Preparations for "zeors(3) ==
	zeros(4)" to return False

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -74,8 +74,24 @@
     descr_pow = _binop_impl("power")
     descr_mod = _binop_impl("mod")
 
-    descr_eq = _binop_impl("equal")
-    descr_ne = _binop_impl("not_equal")
+    def _eq_ne_impl(ufunc_name, fallback_res):
+        assert isinstance(fallback_res, bool)
+        def impl(self, space, w_other):
+            # Unlike ufunc, operator may return simple bool
+            if space.is_w(w_other, space.w_None):
+                # Special case
+                return space.wrap(fallback_res)
+            try:
+                return getattr(interp_ufuncs.get(space), ufunc_name).call(space, [self, w_other])
+            except OperationError, e:
+                if e.match(space, space.w_ValueError):
+                    # For the case when arrays of incompatible size are compared
+                    return space.wrap(fallback_res)
+                raise
+        return func_with_new_name(impl, "binop_%s_impl" % ufunc_name)
+
+    descr_eq = _eq_ne_impl("equal", False)
+    descr_ne = _eq_ne_impl("not_equal", True)
     descr_lt = _binop_impl("less")
     descr_le = _binop_impl("less_equal")
     descr_gt = _binop_impl("greater")
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -577,6 +577,15 @@
             for i in xrange(5):
                 assert c[i] == func(b[i], 3)
 
+    def test_eq_ne(self):
+        from numpy import array
+        a = array(range(5))
+        assert (a == None) is False
+        assert (a != None) is True
+        # TODO: uncomment after size check is implemented
+        # b = array(range(2))
+        # assert (a == b) is False
+        # assert (a != b) is True
 
 class AppTestSupport(object):
     def setup_class(cls):

From noreply at buildbot.pypy.org  Wed Oct 19 23:49:17 2011
From: noreply at buildbot.pypy.org (snus_mumrik)
Date: Wed, 19 Oct 2011 23:49:17 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-comparison: merge default
Message-ID: <20111019214917.B8803820D7@wyvern.cs.uni-duesseldorf.de>

Author: Ilya Osadchiy <osadchiy.ilya at gmail.com>
Branch: numpy-comparison
Changeset: r48247:20ce8b08cdd5
Date: 2011-10-19 20:19 +0200
http://bitbucket.org/pypy/pypy/changeset/20ce8b08cdd5/

Log:	merge default

diff too long, truncating to 10000 out of 11563 lines

diff --git a/lib-python/modified-2.7/httplib.py b/lib-python/modified-2.7/httplib.py
new file mode 100644
--- /dev/null
+++ b/lib-python/modified-2.7/httplib.py
@@ -0,0 +1,1377 @@
+"""HTTP/1.1 client library
+
+<intro stuff goes here>
+<other stuff, too>
+
+HTTPConnection goes through a number of "states", which define when a client
+may legally make another request or fetch the response for a particular
+request. This diagram details these state transitions:
+
+    (null)
+      |
+      | HTTPConnection()
+      v
+    Idle
+      |
+      | putrequest()
+      v
+    Request-started
+      |
+      | ( putheader() )*  endheaders()
+      v
+    Request-sent
+      |
+      | response = getresponse()
+      v
+    Unread-response   [Response-headers-read]
+      |\____________________
+      |                     |
+      | response.read()     | putrequest()
+      v                     v
+    Idle                  Req-started-unread-response
+                     ______/|
+                   /        |
+   response.read() |        | ( putheader() )*  endheaders()
+                   v        v
+       Request-started    Req-sent-unread-response
+                            |
+                            | response.read()
+                            v
+                          Request-sent
+
+This diagram presents the following rules:
+  -- a second request may not be started until {response-headers-read}
+  -- a response [object] cannot be retrieved until {request-sent}
+  -- there is no differentiation between an unread response body and a
+     partially read response body
+
+Note: this enforcement is applied by the HTTPConnection class. The
+      HTTPResponse class does not enforce this state machine, which
+      implies sophisticated clients may accelerate the request/response
+      pipeline. Caution should be taken, though: accelerating the states
+      beyond the above pattern may imply knowledge of the server's
+      connection-close behavior for certain requests. For example, it
+      is impossible to tell whether the server will close the connection
+      UNTIL the response headers have been read; this means that further
+      requests cannot be placed into the pipeline until it is known that
+      the server will NOT be closing the connection.
+
+Logical State                  __state            __response
+-------------                  -------            ----------
+Idle                           _CS_IDLE           None
+Request-started                _CS_REQ_STARTED    None
+Request-sent                   _CS_REQ_SENT       None
+Unread-response                _CS_IDLE           <response_class>
+Req-started-unread-response    _CS_REQ_STARTED    <response_class>
+Req-sent-unread-response       _CS_REQ_SENT       <response_class>
+"""
+
+from array import array
+import os
+import socket
+from sys import py3kwarning
+from urlparse import urlsplit
+import warnings
+with warnings.catch_warnings():
+    if py3kwarning:
+        warnings.filterwarnings("ignore", ".*mimetools has been removed",
+                                DeprecationWarning)
+    import mimetools
+
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from StringIO import StringIO
+
+__all__ = ["HTTP", "HTTPResponse", "HTTPConnection",
+           "HTTPException", "NotConnected", "UnknownProtocol",
+           "UnknownTransferEncoding", "UnimplementedFileMode",
+           "IncompleteRead", "InvalidURL", "ImproperConnectionState",
+           "CannotSendRequest", "CannotSendHeader", "ResponseNotReady",
+           "BadStatusLine", "error", "responses"]
+
+HTTP_PORT = 80
+HTTPS_PORT = 443
+
+_UNKNOWN = 'UNKNOWN'
+
+# connection states
+_CS_IDLE = 'Idle'
+_CS_REQ_STARTED = 'Request-started'
+_CS_REQ_SENT = 'Request-sent'
+
+# status codes
+# informational
+CONTINUE = 100
+SWITCHING_PROTOCOLS = 101
+PROCESSING = 102
+
+# successful
+OK = 200
+CREATED = 201
+ACCEPTED = 202
+NON_AUTHORITATIVE_INFORMATION = 203
+NO_CONTENT = 204
+RESET_CONTENT = 205
+PARTIAL_CONTENT = 206
+MULTI_STATUS = 207
+IM_USED = 226
+
+# redirection
+MULTIPLE_CHOICES = 300
+MOVED_PERMANENTLY = 301
+FOUND = 302
+SEE_OTHER = 303
+NOT_MODIFIED = 304
+USE_PROXY = 305
+TEMPORARY_REDIRECT = 307
+
+# client error
+BAD_REQUEST = 400
+UNAUTHORIZED = 401
+PAYMENT_REQUIRED = 402
+FORBIDDEN = 403
+NOT_FOUND = 404
+METHOD_NOT_ALLOWED = 405
+NOT_ACCEPTABLE = 406
+PROXY_AUTHENTICATION_REQUIRED = 407
+REQUEST_TIMEOUT = 408
+CONFLICT = 409
+GONE = 410
+LENGTH_REQUIRED = 411
+PRECONDITION_FAILED = 412
+REQUEST_ENTITY_TOO_LARGE = 413
+REQUEST_URI_TOO_LONG = 414
+UNSUPPORTED_MEDIA_TYPE = 415
+REQUESTED_RANGE_NOT_SATISFIABLE = 416
+EXPECTATION_FAILED = 417
+UNPROCESSABLE_ENTITY = 422
+LOCKED = 423
+FAILED_DEPENDENCY = 424
+UPGRADE_REQUIRED = 426
+
+# server error
+INTERNAL_SERVER_ERROR = 500
+NOT_IMPLEMENTED = 501
+BAD_GATEWAY = 502
+SERVICE_UNAVAILABLE = 503
+GATEWAY_TIMEOUT = 504
+HTTP_VERSION_NOT_SUPPORTED = 505
+INSUFFICIENT_STORAGE = 507
+NOT_EXTENDED = 510
+
+# Mapping status codes to official W3C names
+responses = {
+    100: 'Continue',
+    101: 'Switching Protocols',
+
+    200: 'OK',
+    201: 'Created',
+    202: 'Accepted',
+    203: 'Non-Authoritative Information',
+    204: 'No Content',
+    205: 'Reset Content',
+    206: 'Partial Content',
+
+    300: 'Multiple Choices',
+    301: 'Moved Permanently',
+    302: 'Found',
+    303: 'See Other',
+    304: 'Not Modified',
+    305: 'Use Proxy',
+    306: '(Unused)',
+    307: 'Temporary Redirect',
+
+    400: 'Bad Request',
+    401: 'Unauthorized',
+    402: 'Payment Required',
+    403: 'Forbidden',
+    404: 'Not Found',
+    405: 'Method Not Allowed',
+    406: 'Not Acceptable',
+    407: 'Proxy Authentication Required',
+    408: 'Request Timeout',
+    409: 'Conflict',
+    410: 'Gone',
+    411: 'Length Required',
+    412: 'Precondition Failed',
+    413: 'Request Entity Too Large',
+    414: 'Request-URI Too Long',
+    415: 'Unsupported Media Type',
+    416: 'Requested Range Not Satisfiable',
+    417: 'Expectation Failed',
+
+    500: 'Internal Server Error',
+    501: 'Not Implemented',
+    502: 'Bad Gateway',
+    503: 'Service Unavailable',
+    504: 'Gateway Timeout',
+    505: 'HTTP Version Not Supported',
+}
+
+# maximal amount of data to read at one time in _safe_read
+MAXAMOUNT = 1048576
+
+class HTTPMessage(mimetools.Message):
+
+    def addheader(self, key, value):
+        """Add header for field key handling repeats."""
+        prev = self.dict.get(key)
+        if prev is None:
+            self.dict[key] = value
+        else:
+            combined = ", ".join((prev, value))
+            self.dict[key] = combined
+
+    def addcontinue(self, key, more):
+        """Add more field data from a continuation line."""
+        prev = self.dict[key]
+        self.dict[key] = prev + "\n " + more
+
+    def readheaders(self):
+        """Read header lines.
+
+        Read header lines up to the entirely blank line that terminates them.
+        The (normally blank) line that ends the headers is skipped, but not
+        included in the returned list.  If a non-header line ends the headers,
+        (which is an error), an attempt is made to backspace over it; it is
+        never included in the returned list.
+
+        The variable self.status is set to the empty string if all went well,
+        otherwise it is an error message.  The variable self.headers is a
+        completely uninterpreted list of lines contained in the header (so
+        printing them will reproduce the header exactly as it appears in the
+        file).
+
+        If multiple header fields with the same name occur, they are combined
+        according to the rules in RFC 2616 sec 4.2:
+
+        Appending each subsequent field-value to the first, each separated
+        by a comma. The order in which header fields with the same field-name
+        are received is significant to the interpretation of the combined
+        field value.
+        """
+        # XXX The implementation overrides the readheaders() method of
+        # rfc822.Message.  The base class design isn't amenable to
+        # customized behavior here so the method here is a copy of the
+        # base class code with a few small changes.
+
+        self.dict = {}
+        self.unixfrom = ''
+        self.headers = hlist = []
+        self.status = ''
+        headerseen = ""
+        firstline = 1
+        startofline = unread = tell = None
+        if hasattr(self.fp, 'unread'):
+            unread = self.fp.unread
+        elif self.seekable:
+            tell = self.fp.tell
+        while True:
+            if tell:
+                try:
+                    startofline = tell()
+                except IOError:
+                    startofline = tell = None
+                    self.seekable = 0
+            line = self.fp.readline()
+            if not line:
+                self.status = 'EOF in headers'
+                break
+            # Skip unix From name time lines
+            if firstline and line.startswith('From '):
+                self.unixfrom = self.unixfrom + line
+                continue
+            firstline = 0
+            if headerseen and line[0] in ' \t':
+                # XXX Not sure if continuation lines are handled properly
+                # for http and/or for repeating headers
+                # It's a continuation line.
+                hlist.append(line)
+                self.addcontinue(headerseen, line.strip())
+                continue
+            elif self.iscomment(line):
+                # It's a comment.  Ignore it.
+                continue
+            elif self.islast(line):
+                # Note! No pushback here!  The delimiter line gets eaten.
+                break
+            headerseen = self.isheader(line)
+            if headerseen:
+                # It's a legal header line, save it.
+                hlist.append(line)
+                self.addheader(headerseen, line[len(headerseen)+1:].strip())
+                continue
+            else:
+                # It's not a header line; throw it back and stop here.
+                if not self.dict:
+                    self.status = 'No headers'
+                else:
+                    self.status = 'Non-header line where header expected'
+                # Try to undo the read.
+                if unread:
+                    unread(line)
+                elif tell:
+                    self.fp.seek(startofline)
+                else:
+                    self.status = self.status + '; bad seek'
+                break
+
+class HTTPResponse:
+
+    # strict: If true, raise BadStatusLine if the status line can't be
+    # parsed as a valid HTTP/1.0 or 1.1 status line.  By default it is
+    # false because it prevents clients from talking to HTTP/0.9
+    # servers.  Note that a response with a sufficiently corrupted
+    # status line will look like an HTTP/0.9 response.
+
+    # See RFC 2616 sec 19.6 and RFC 1945 sec 6 for details.
+
+    def __init__(self, sock, debuglevel=0, strict=0, method=None, buffering=False):
+        if buffering:
+            # The caller won't be using any sock.recv() calls, so buffering
+            # is fine and recommended for performance.
+            self.fp = sock.makefile('rb')
+        else:
+            # The buffer size is specified as zero, because the headers of
+            # the response are read with readline().  If the reads were
+            # buffered the readline() calls could consume some of the
+            # response, which make be read via a recv() on the underlying
+            # socket.
+            self.fp = sock.makefile('rb', 0)
+        self.debuglevel = debuglevel
+        self.strict = strict
+        self._method = method
+
+        self.msg = None
+
+        # from the Status-Line of the response
+        self.version = _UNKNOWN # HTTP-Version
+        self.status = _UNKNOWN  # Status-Code
+        self.reason = _UNKNOWN  # Reason-Phrase
+
+        self.chunked = _UNKNOWN         # is "chunked" being used?
+        self.chunk_left = _UNKNOWN      # bytes left to read in current chunk
+        self.length = _UNKNOWN          # number of bytes left in response
+        self.will_close = _UNKNOWN      # conn will close at end of response
+
+    def _read_status(self):
+        # Initialize with Simple-Response defaults
+        line = self.fp.readline()
+        if self.debuglevel > 0:
+            print "reply:", repr(line)
+        if not line:
+            # Presumably, the server closed the connection before
+            # sending a valid response.
+            raise BadStatusLine(line)
+        try:
+            [version, status, reason] = line.split(None, 2)
+        except ValueError:
+            try:
+                [version, status] = line.split(None, 1)
+                reason = ""
+            except ValueError:
+                # empty version will cause next test to fail and status
+                # will be treated as 0.9 response.
+                version = ""
+        if not version.startswith('HTTP/'):
+            if self.strict:
+                self.close()
+                raise BadStatusLine(line)
+            else:
+                # assume it's a Simple-Response from an 0.9 server
+                self.fp = LineAndFileWrapper(line, self.fp)
+                return "HTTP/0.9", 200, ""
+
+        # The status code is a three-digit number
+        try:
+            status = int(status)
+            if status < 100 or status > 999:
+                raise BadStatusLine(line)
+        except ValueError:
+            raise BadStatusLine(line)
+        return version, status, reason
+
+    def begin(self):
+        if self.msg is not None:
+            # we've already started reading the response
+            return
+
+        # read until we get a non-100 response
+        while True:
+            version, status, reason = self._read_status()
+            if status != CONTINUE:
+                break
+            # skip the header from the 100 response
+            while True:
+                skip = self.fp.readline().strip()
+                if not skip:
+                    break
+                if self.debuglevel > 0:
+                    print "header:", skip
+
+        self.status = status
+        self.reason = reason.strip()
+        if version == 'HTTP/1.0':
+            self.version = 10
+        elif version.startswith('HTTP/1.'):
+            self.version = 11   # use HTTP/1.1 code for HTTP/1.x where x>=1
+        elif version == 'HTTP/0.9':
+            self.version = 9
+        else:
+            raise UnknownProtocol(version)
+
+        if self.version == 9:
+            self.length = None
+            self.chunked = 0
+            self.will_close = 1
+            self.msg = HTTPMessage(StringIO())
+            return
+
+        self.msg = HTTPMessage(self.fp, 0)
+        if self.debuglevel > 0:
+            for hdr in self.msg.headers:
+                print "header:", hdr,
+
+        # don't let the msg keep an fp
+        self.msg.fp = None
+
+        # are we using the chunked-style of transfer encoding?
+        tr_enc = self.msg.getheader('transfer-encoding')
+        if tr_enc and tr_enc.lower() == "chunked":
+            self.chunked = 1
+            self.chunk_left = None
+        else:
+            self.chunked = 0
+
+        # will the connection close at the end of the response?
+        self.will_close = self._check_close()
+
+        # do we have a Content-Length?
+        # NOTE: RFC 2616, S4.4, #3 says we ignore this if tr_enc is "chunked"
+        length = self.msg.getheader('content-length')
+        if length and not self.chunked:
+            try:
+                self.length = int(length)
+            except ValueError:
+                self.length = None
+            else:
+                if self.length < 0:  # ignore nonsensical negative lengths
+                    self.length = None
+        else:
+            self.length = None
+
+        # does the body have a fixed length? (of zero)
+        if (status == NO_CONTENT or status == NOT_MODIFIED or
+            100 <= status < 200 or      # 1xx codes
+            self._method == 'HEAD'):
+            self.length = 0
+
+        # if the connection remains open, and we aren't using chunked, and
+        # a content-length was not provided, then assume that the connection
+        # WILL close.
+        if not self.will_close and \
+           not self.chunked and \
+           self.length is None:
+            self.will_close = 1
+
+    def _check_close(self):
+        conn = self.msg.getheader('connection')
+        if self.version == 11:
+            # An HTTP/1.1 proxy is assumed to stay open unless
+            # explicitly closed.
+            conn = self.msg.getheader('connection')
+            if conn and "close" in conn.lower():
+                return True
+            return False
+
+        # Some HTTP/1.0 implementations have support for persistent
+        # connections, using rules different than HTTP/1.1.
+
+        # For older HTTP, Keep-Alive indicates persistent connection.
+        if self.msg.getheader('keep-alive'):
+            return False
+
+        # At least Akamai returns a "Connection: Keep-Alive" header,
+        # which was supposed to be sent by the client.
+        if conn and "keep-alive" in conn.lower():
+            return False
+
+        # Proxy-Connection is a netscape hack.
+        pconn = self.msg.getheader('proxy-connection')
+        if pconn and "keep-alive" in pconn.lower():
+            return False
+
+        # otherwise, assume it will close
+        return True
+
+    def close(self):
+        if self.fp:
+            self.fp.close()
+            self.fp = None
+
+    def isclosed(self):
+        # NOTE: it is possible that we will not ever call self.close(). This
+        #       case occurs when will_close is TRUE, length is None, and we
+        #       read up to the last byte, but NOT past it.
+        #
+        # IMPLIES: if will_close is FALSE, then self.close() will ALWAYS be
+        #          called, meaning self.isclosed() is meaningful.
+        return self.fp is None
+
+    # XXX It would be nice to have readline and __iter__ for this, too.
+
+    def read(self, amt=None):
+        if self.fp is None:
+            return ''
+
+        if self._method == 'HEAD':
+            self.close()
+            return ''
+
+        if self.chunked:
+            return self._read_chunked(amt)
+
+        if amt is None:
+            # unbounded read
+            if self.length is None:
+                s = self.fp.read()
+            else:
+                s = self._safe_read(self.length)
+                self.length = 0
+            self.close()        # we read everything
+            return s
+
+        if self.length is not None:
+            if amt > self.length:
+                # clip the read to the "end of response"
+                amt = self.length
+
+        # we do not use _safe_read() here because this may be a .will_close
+        # connection, and the user is reading more bytes than will be provided
+        # (for example, reading in 1k chunks)
+        s = self.fp.read(amt)
+        if self.length is not None:
+            self.length -= len(s)
+            if not self.length:
+                self.close()
+        return s
+
+    def _read_chunked(self, amt):
+        assert self.chunked != _UNKNOWN
+        chunk_left = self.chunk_left
+        value = []
+        while True:
+            if chunk_left is None:
+                line = self.fp.readline()
+                i = line.find(';')
+                if i >= 0:
+                    line = line[:i] # strip chunk-extensions
+                try:
+                    chunk_left = int(line, 16)
+                except ValueError:
+                    # close the connection as protocol synchronisation is
+                    # probably lost
+                    self.close()
+                    raise IncompleteRead(''.join(value))
+                if chunk_left == 0:
+                    break
+            if amt is None:
+                value.append(self._safe_read(chunk_left))
+            elif amt < chunk_left:
+                value.append(self._safe_read(amt))
+                self.chunk_left = chunk_left - amt
+                return ''.join(value)
+            elif amt == chunk_left:
+                value.append(self._safe_read(amt))
+                self._safe_read(2)  # toss the CRLF at the end of the chunk
+                self.chunk_left = None
+                return ''.join(value)
+            else:
+                value.append(self._safe_read(chunk_left))
+                amt -= chunk_left
+
+            # we read the whole chunk, get another
+            self._safe_read(2)      # toss the CRLF at the end of the chunk
+            chunk_left = None
+
+        # read and discard trailer up to the CRLF terminator
+        ### note: we shouldn't have any trailers!
+        while True:
+            line = self.fp.readline()
+            if not line:
+                # a vanishingly small number of sites EOF without
+                # sending the trailer
+                break
+            if line == '\r\n':
+                break
+
+        # we read everything; close the "file"
+        self.close()
+
+        return ''.join(value)
+
+    def _safe_read(self, amt):
+        """Read the number of bytes requested, compensating for partial reads.
+
+        Normally, we have a blocking socket, but a read() can be interrupted
+        by a signal (resulting in a partial read).
+
+        Note that we cannot distinguish between EOF and an interrupt when zero
+        bytes have been read. IncompleteRead() will be raised in this
+        situation.
+
+        This function should be used when <amt> bytes "should" be present for
+        reading. If the bytes are truly not available (due to EOF), then the
+        IncompleteRead exception can be used to detect the problem.
+        """
+        # NOTE(gps): As of svn r74426 socket._fileobject.read(x) will never
+        # return less than x bytes unless EOF is encountered.  It now handles
+        # signal interruptions (socket.error EINTR) internally.  This code
+        # never caught that exception anyways.  It seems largely pointless.
+        # self.fp.read(amt) will work fine.
+        s = []
+        while amt > 0:
+            chunk = self.fp.read(min(amt, MAXAMOUNT))
+            if not chunk:
+                raise IncompleteRead(''.join(s), amt)
+            s.append(chunk)
+            amt -= len(chunk)
+        return ''.join(s)
+
+    def fileno(self):
+        return self.fp.fileno()
+
+    def getheader(self, name, default=None):
+        if self.msg is None:
+            raise ResponseNotReady()
+        return self.msg.getheader(name, default)
+
+    def getheaders(self):
+        """Return list of (header, value) tuples."""
+        if self.msg is None:
+            raise ResponseNotReady()
+        return self.msg.items()
+
+
+class HTTPConnection:
+
+    _http_vsn = 11
+    _http_vsn_str = 'HTTP/1.1'
+
+    response_class = HTTPResponse
+    default_port = HTTP_PORT
+    auto_open = 1
+    debuglevel = 0
+    strict = 0
+
+    def __init__(self, host, port=None, strict=None,
+                 timeout=socket._GLOBAL_DEFAULT_TIMEOUT, source_address=None):
+        self.timeout = timeout
+        self.source_address = source_address
+        self.sock = None
+        self._buffer = []
+        self.__response = None
+        self.__state = _CS_IDLE
+        self._method = None
+        self._tunnel_host = None
+        self._tunnel_port = None
+        self._tunnel_headers = {}
+
+        self._set_hostport(host, port)
+        if strict is not None:
+            self.strict = strict
+
+    def set_tunnel(self, host, port=None, headers=None):
+        """ Sets up the host and the port for the HTTP CONNECT Tunnelling.
+
+        The headers argument should be a mapping of extra HTTP headers
+        to send with the CONNECT request.
+        """
+        self._tunnel_host = host
+        self._tunnel_port = port
+        if headers:
+            self._tunnel_headers = headers
+        else:
+            self._tunnel_headers.clear()
+
+    def _set_hostport(self, host, port):
+        if port is None:
+            i = host.rfind(':')
+            j = host.rfind(']')         # ipv6 addresses have [...]
+            if i > j:
+                try:
+                    port = int(host[i+1:])
+                except ValueError:
+                    raise InvalidURL("nonnumeric port: '%s'" % host[i+1:])
+                host = host[:i]
+            else:
+                port = self.default_port
+            if host and host[0] == '[' and host[-1] == ']':
+                host = host[1:-1]
+        self.host = host
+        self.port = port
+
+    def set_debuglevel(self, level):
+        self.debuglevel = level
+
+    def _tunnel(self):
+        self._set_hostport(self._tunnel_host, self._tunnel_port)
+        self.send("CONNECT %s:%d HTTP/1.0\r\n" % (self.host, self.port))
+        for header, value in self._tunnel_headers.iteritems():
+            self.send("%s: %s\r\n" % (header, value))
+        self.send("\r\n")
+        response = self.response_class(self.sock, strict = self.strict,
+                                       method = self._method)
+        (version, code, message) = response._read_status()
+
+        if code != 200:
+            self.close()
+            raise socket.error("Tunnel connection failed: %d %s" % (code,
+                                                                    message.strip()))
+        while True:
+            line = response.fp.readline()
+            if line == '\r\n': break
+
+
+    def connect(self):
+        """Connect to the host and port specified in __init__."""
+        self.sock = socket.create_connection((self.host,self.port),
+                                             self.timeout, self.source_address)
+
+        if self._tunnel_host:
+            self._tunnel()
+
+    def close(self):
+        """Close the connection to the HTTP server."""
+        if self.sock:
+            self.sock.close()   # close it manually... there may be other refs
+            self.sock = None
+        if self.__response:
+            self.__response.close()
+            self.__response = None
+        self.__state = _CS_IDLE
+
+    def send(self, data):
+        """Send `data' to the server."""
+        if self.sock is None:
+            if self.auto_open:
+                self.connect()
+            else:
+                raise NotConnected()
+
+        if self.debuglevel > 0:
+            print "send:", repr(data)
+        blocksize = 8192
+        if hasattr(data,'read') and not isinstance(data, array):
+            if self.debuglevel > 0: print "sendIng a read()able"
+            datablock = data.read(blocksize)
+            while datablock:
+                self.sock.sendall(datablock)
+                datablock = data.read(blocksize)
+        else:
+            self.sock.sendall(data)
+
+    def _output(self, s):
+        """Add a line of output to the current request buffer.
+
+        Assumes that the line does *not* end with \\r\\n.
+        """
+        self._buffer.append(s)
+
+    def _send_output(self, message_body=None):
+        """Send the currently buffered request and clear the buffer.
+
+        Appends an extra \\r\\n to the buffer.
+        A message_body may be specified, to be appended to the request.
+        """
+        self._buffer.extend(("", ""))
+        msg = "\r\n".join(self._buffer)
+        del self._buffer[:]
+        # If msg and message_body are sent in a single send() call,
+        # it will avoid performance problems caused by the interaction
+        # between delayed ack and the Nagle algorithim.
+        if isinstance(message_body, str):
+            msg += message_body
+            message_body = None
+        self.send(msg)
+        if message_body is not None:
+            #message_body was not a string (i.e. it is a file) and
+            #we must run the risk of Nagle
+            self.send(message_body)
+
+    def putrequest(self, method, url, skip_host=0, skip_accept_encoding=0):
+        """Send a request to the server.
+
+        `method' specifies an HTTP request method, e.g. 'GET'.
+        `url' specifies the object being requested, e.g. '/index.html'.
+        `skip_host' if True does not add automatically a 'Host:' header
+        `skip_accept_encoding' if True does not add automatically an
+           'Accept-Encoding:' header
+        """
+
+        # if a prior response has been completed, then forget about it.
+        if self.__response and self.__response.isclosed():
+            self.__response = None
+
+
+        # in certain cases, we cannot issue another request on this connection.
+        # this occurs when:
+        #   1) we are in the process of sending a request.   (_CS_REQ_STARTED)
+        #   2) a response to a previous request has signalled that it is going
+        #      to close the connection upon completion.
+        #   3) the headers for the previous response have not been read, thus
+        #      we cannot determine whether point (2) is true.   (_CS_REQ_SENT)
+        #
+        # if there is no prior response, then we can request at will.
+        #
+        # if point (2) is true, then we will have passed the socket to the
+        # response (effectively meaning, "there is no prior response"), and
+        # will open a new one when a new request is made.
+        #
+        # Note: if a prior response exists, then we *can* start a new request.
+        #       We are not allowed to begin fetching the response to this new
+        #       request, however, until that prior response is complete.
+        #
+        if self.__state == _CS_IDLE:
+            self.__state = _CS_REQ_STARTED
+        else:
+            raise CannotSendRequest()
+
+        # Save the method we use, we need it later in the response phase
+        self._method = method
+        if not url:
+            url = '/'
+        hdr = '%s %s %s' % (method, url, self._http_vsn_str)
+
+        self._output(hdr)
+
+        if self._http_vsn == 11:
+            # Issue some standard headers for better HTTP/1.1 compliance
+
+            if not skip_host:
+                # this header is issued *only* for HTTP/1.1
+                # connections. more specifically, this means it is
+                # only issued when the client uses the new
+                # HTTPConnection() class. backwards-compat clients
+                # will be using HTTP/1.0 and those clients may be
+                # issuing this header themselves. we should NOT issue
+                # it twice; some web servers (such as Apache) barf
+                # when they see two Host: headers
+
+                # If we need a non-standard port,include it in the
+                # header.  If the request is going through a proxy,
+                # but the host of the actual URL, not the host of the
+                # proxy.
+
+                netloc = ''
+                if url.startswith('http'):
+                    nil, netloc, nil, nil, nil = urlsplit(url)
+
+                if netloc:
+                    try:
+                        netloc_enc = netloc.encode("ascii")
+                    except UnicodeEncodeError:
+                        netloc_enc = netloc.encode("idna")
+                    self.putheader('Host', netloc_enc)
+                else:
+                    try:
+                        host_enc = self.host.encode("ascii")
+                    except UnicodeEncodeError:
+                        host_enc = self.host.encode("idna")
+                    # Wrap the IPv6 Host Header with [] (RFC 2732)
+                    if host_enc.find(':') >= 0:
+                        host_enc = "[" + host_enc + "]"
+                    if self.port == self.default_port:
+                        self.putheader('Host', host_enc)
+                    else:
+                        self.putheader('Host', "%s:%s" % (host_enc, self.port))
+
+            # note: we are assuming that clients will not attempt to set these
+            #       headers since *this* library must deal with the
+            #       consequences. this also means that when the supporting
+            #       libraries are updated to recognize other forms, then this
+            #       code should be changed (removed or updated).
+
+            # we only want a Content-Encoding of "identity" since we don't
+            # support encodings such as x-gzip or x-deflate.
+            if not skip_accept_encoding:
+                self.putheader('Accept-Encoding', 'identity')
+
+            # we can accept "chunked" Transfer-Encodings, but no others
+            # NOTE: no TE header implies *only* "chunked"
+            #self.putheader('TE', 'chunked')
+
+            # if TE is supplied in the header, then it must appear in a
+            # Connection header.
+            #self.putheader('Connection', 'TE')
+
+        else:
+            # For HTTP/1.0, the server will assume "not chunked"
+            pass
+
+    def putheader(self, header, *values):
+        """Send a request header line to the server.
+
+        For example: h.putheader('Accept', 'text/html')
+        """
+        if self.__state != _CS_REQ_STARTED:
+            raise CannotSendHeader()
+
+        hdr = '%s: %s' % (header, '\r\n\t'.join([str(v) for v in values]))
+        self._output(hdr)
+
+    def endheaders(self, message_body=None):
+        """Indicate that the last header line has been sent to the server.
+
+        This method sends the request to the server.  The optional
+        message_body argument can be used to pass message body
+        associated with the request.  The message body will be sent in
+        the same packet as the message headers if possible.  The
+        message_body should be a string.
+        """
+        if self.__state == _CS_REQ_STARTED:
+            self.__state = _CS_REQ_SENT
+        else:
+            raise CannotSendHeader()
+        self._send_output(message_body)
+
+    def request(self, method, url, body=None, headers={}):
+        """Send a complete request to the server."""
+        self._send_request(method, url, body, headers)
+
+    def _set_content_length(self, body):
+        # Set the content-length based on the body.
+        thelen = None
+        try:
+            thelen = str(len(body))
+        except TypeError, te:
+            # If this is a file-like object, try to
+            # fstat its file descriptor
+            try:
+                thelen = str(os.fstat(body.fileno()).st_size)
+            except (AttributeError, OSError):
+                # Don't send a length if this failed
+                if self.debuglevel > 0: print "Cannot stat!!"
+
+        if thelen is not None:
+            self.putheader('Content-Length', thelen)
+
+    def _send_request(self, method, url, body, headers):
+        # Honor explicitly requested Host: and Accept-Encoding: headers.
+        header_names = dict.fromkeys([k.lower() for k in headers])
+        skips = {}
+        if 'host' in header_names:
+            skips['skip_host'] = 1
+        if 'accept-encoding' in header_names:
+            skips['skip_accept_encoding'] = 1
+
+        self.putrequest(method, url, **skips)
+
+        if body and ('content-length' not in header_names):
+            self._set_content_length(body)
+        for hdr, value in headers.iteritems():
+            self.putheader(hdr, value)
+        self.endheaders(body)
+
+    def getresponse(self, buffering=False):
+        "Get the response from the server."
+
+        # if a prior response has been completed, then forget about it.
+        if self.__response and self.__response.isclosed():
+            self.__response = None
+
+        #
+        # if a prior response exists, then it must be completed (otherwise, we
+        # cannot read this response's header to determine the connection-close
+        # behavior)
+        #
+        # note: if a prior response existed, but was connection-close, then the
+        # socket and response were made independent of this HTTPConnection
+        # object since a new request requires that we open a whole new
+        # connection
+        #
+        # this means the prior response had one of two states:
+        #   1) will_close: this connection was reset and the prior socket and
+        #                  response operate independently
+        #   2) persistent: the response was retained and we await its
+        #                  isclosed() status to become true.
+        #
+        if self.__state != _CS_REQ_SENT or self.__response:
+            raise ResponseNotReady()
+
+        args = (self.sock,)
+        kwds = {"strict":self.strict, "method":self._method}
+        if self.debuglevel > 0:
+            args += (self.debuglevel,)
+        if buffering:
+            #only add this keyword if non-default, for compatibility with
+            #other response_classes.
+            kwds["buffering"] = True;
+        response = self.response_class(*args, **kwds)
+
+        try:
+            response.begin()
+        except:
+            response.close()
+            raise
+        assert response.will_close != _UNKNOWN
+        self.__state = _CS_IDLE
+
+        if response.will_close:
+            # this effectively passes the connection to the response
+            self.close()
+        else:
+            # remember this, so we can tell when it is complete
+            self.__response = response
+
+        return response
+
+
+class HTTP:
+    "Compatibility class with httplib.py from 1.5."
+
+    _http_vsn = 10
+    _http_vsn_str = 'HTTP/1.0'
+
+    debuglevel = 0
+
+    _connection_class = HTTPConnection
+
+    def __init__(self, host='', port=None, strict=None):
+        "Provide a default host, since the superclass requires one."
+
+        # some joker passed 0 explicitly, meaning default port
+        if port == 0:
+            port = None
+
+        # Note that we may pass an empty string as the host; this will throw
+        # an error when we attempt to connect. Presumably, the client code
+        # will call connect before then, with a proper host.
+        self._setup(self._connection_class(host, port, strict))
+
+    def _setup(self, conn):
+        self._conn = conn
+
+        # set up delegation to flesh out interface
+        self.send = conn.send
+        self.putrequest = conn.putrequest
+        self.putheader = conn.putheader
+        self.endheaders = conn.endheaders
+        self.set_debuglevel = conn.set_debuglevel
+
+        conn._http_vsn = self._http_vsn
+        conn._http_vsn_str = self._http_vsn_str
+
+        self.file = None
+
+    def connect(self, host=None, port=None):
+        "Accept arguments to set the host/port, since the superclass doesn't."
+
+        if host is not None:
+            self._conn._set_hostport(host, port)
+        self._conn.connect()
+
+    def getfile(self):
+        "Provide a getfile, since the superclass' does not use this concept."
+        return self.file
+
+    def getreply(self, buffering=False):
+        """Compat definition since superclass does not define it.
+
+        Returns a tuple consisting of:
+        - server status code (e.g. '200' if all goes well)
+        - server "reason" corresponding to status code
+        - any RFC822 headers in the response from the server
+        """
+        try:
+            if not buffering:
+                response = self._conn.getresponse()
+            else:
+                #only add this keyword if non-default for compatibility
+                #with other connection classes
+                response = self._conn.getresponse(buffering)
+        except BadStatusLine, e:
+            ### hmm. if getresponse() ever closes the socket on a bad request,
+            ### then we are going to have problems with self.sock
+
+            ### should we keep this behavior? do people use it?
+            # keep the socket open (as a file), and return it
+            self.file = self._conn.sock.makefile('rb', 0)
+
+            # close our socket -- we want to restart after any protocol error
+            self.close()
+
+            self.headers = None
+            return -1, e.line, None
+
+        self.headers = response.msg
+        self.file = response.fp
+        return response.status, response.reason, response.msg
+
+    def close(self):
+        self._conn.close()
+
+        # note that self.file == response.fp, which gets closed by the
+        # superclass. just clear the object ref here.
+        ### hmm. messy. if status==-1, then self.file is owned by us.
+        ### well... we aren't explicitly closing, but losing this ref will
+        ### do it
+        self.file = None
+
+try:
+    import ssl
+except ImportError:
+    pass
+else:
+    class HTTPSConnection(HTTPConnection):
+        "This class allows communication via SSL."
+
+        default_port = HTTPS_PORT
+
+        def __init__(self, host, port=None, key_file=None, cert_file=None,
+                     strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
+                     source_address=None):
+            HTTPConnection.__init__(self, host, port, strict, timeout,
+                                    source_address)
+            self.key_file = key_file
+            self.cert_file = cert_file
+
+        def connect(self):
+            "Connect to a host on a given (SSL) port."
+
+            sock = socket.create_connection((self.host, self.port),
+                                            self.timeout, self.source_address)
+            if self._tunnel_host:
+                self.sock = sock
+                self._tunnel()
+            self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file)
+
+    __all__.append("HTTPSConnection")
+
+    class HTTPS(HTTP):
+        """Compatibility with 1.5 httplib interface
+
+        Python 1.5.2 did not have an HTTPS class, but it defined an
+        interface for sending http requests that is also useful for
+        https.
+        """
+
+        _connection_class = HTTPSConnection
+
+        def __init__(self, host='', port=None, key_file=None, cert_file=None,
+                     strict=None):
+            # provide a default host, pass the X509 cert info
+
+            # urf. compensate for bad input.
+            if port == 0:
+                port = None
+            self._setup(self._connection_class(host, port, key_file,
+                                               cert_file, strict))
+
+            # we never actually use these for anything, but we keep them
+            # here for compatibility with post-1.5.2 CVS.
+            self.key_file = key_file
+            self.cert_file = cert_file
+
+
+    def FakeSocket (sock, sslobj):
+        warnings.warn("FakeSocket is deprecated, and won't be in 3.x.  " +
+                      "Use the result of ssl.wrap_socket() directly instead.",
+                      DeprecationWarning, stacklevel=2)
+        return sslobj
+
+
+class HTTPException(Exception):
+    # Subclasses that define an __init__ must call Exception.__init__
+    # or define self.args.  Otherwise, str() will fail.
+    pass
+
+class NotConnected(HTTPException):
+    pass
+
+class InvalidURL(HTTPException):
+    pass
+
+class UnknownProtocol(HTTPException):
+    def __init__(self, version):
+        self.args = version,
+        self.version = version
+
+class UnknownTransferEncoding(HTTPException):
+    pass
+
+class UnimplementedFileMode(HTTPException):
+    pass
+
+class IncompleteRead(HTTPException):
+    def __init__(self, partial, expected=None):
+        self.args = partial,
+        self.partial = partial
+        self.expected = expected
+    def __repr__(self):
+        if self.expected is not None:
+            e = ', %i more expected' % self.expected
+        else:
+            e = ''
+        return 'IncompleteRead(%i bytes read%s)' % (len(self.partial), e)
+    def __str__(self):
+        return repr(self)
+
+class ImproperConnectionState(HTTPException):
+    pass
+
+class CannotSendRequest(ImproperConnectionState):
+    pass
+
+class CannotSendHeader(ImproperConnectionState):
+    pass
+
+class ResponseNotReady(ImproperConnectionState):
+    pass
+
+class BadStatusLine(HTTPException):
+    def __init__(self, line):
+        if not line:
+            line = repr(line)
+        self.args = line,
+        self.line = line
+
+# for backwards compatibility
+error = HTTPException
+
+class LineAndFileWrapper:
+    """A limited file-like object for HTTP/0.9 responses."""
+
+    # The status-line parsing code calls readline(), which normally
+    # get the HTTP status line.  For a 0.9 response, however, this is
+    # actually the first line of the body!  Clients need to get a
+    # readable file object that contains that line.
+
+    def __init__(self, line, file):
+        self._line = line
+        self._file = file
+        self._line_consumed = 0
+        self._line_offset = 0
+        self._line_left = len(line)
+
+    def __getattr__(self, attr):
+        return getattr(self._file, attr)
+
+    def _done(self):
+        # called when the last byte is read from the line.  After the
+        # call, all read methods are delegated to the underlying file
+        # object.
+        self._line_consumed = 1
+        self.read = self._file.read
+        self.readline = self._file.readline
+        self.readlines = self._file.readlines
+
+    def read(self, amt=None):
+        if self._line_consumed:
+            return self._file.read(amt)
+        assert self._line_left
+        if amt is None or amt > self._line_left:
+            s = self._line[self._line_offset:]
+            self._done()
+            if amt is None:
+                return s + self._file.read()
+            else:
+                return s + self._file.read(amt - len(s))
+        else:
+            assert amt <= self._line_left
+            i = self._line_offset
+            j = i + amt
+            s = self._line[i:j]
+            self._line_offset = j
+            self._line_left -= amt
+            if self._line_left == 0:
+                self._done()
+            return s
+
+    def readline(self):
+        if self._line_consumed:
+            return self._file.readline()
+        assert self._line_left
+        s = self._line[self._line_offset:]
+        self._done()
+        return s
+
+    def readlines(self, size=None):
+        if self._line_consumed:
+            return self._file.readlines(size)
+        assert self._line_left
+        L = [self._line[self._line_offset:]]
+        self._done()
+        if size is None:
+            return L + self._file.readlines()
+        else:
+            return L + self._file.readlines(size)
+
+def test():
+    """Test this module.
+
+    A hodge podge of tests collected here, because they have too many
+    external dependencies for the regular test suite.
+    """
+
+    import sys
+    import getopt
+    opts, args = getopt.getopt(sys.argv[1:], 'd')
+    dl = 0
+    for o, a in opts:
+        if o == '-d': dl = dl + 1
+    host = 'www.python.org'
+    selector = '/'
+    if args[0:]: host = args[0]
+    if args[1:]: selector = args[1]
+    h = HTTP()
+    h.set_debuglevel(dl)
+    h.connect(host)
+    h.putrequest('GET', selector)
+    h.endheaders()
+    status, reason, headers = h.getreply()
+    print 'status =', status
+    print 'reason =', reason
+    print "read", len(h.getfile().read())
+    print
+    if headers:
+        for header in headers.headers: print header.strip()
+    print
+
+    # minimal test that code to extract host from url works
+    class HTTP11(HTTP):
+        _http_vsn = 11
+        _http_vsn_str = 'HTTP/1.1'
+
+    h = HTTP11('www.python.org')
+    h.putrequest('GET', 'http://www.python.org/~jeremy/')
+    h.endheaders()
+    h.getreply()
+    h.close()
+
+    try:
+        import ssl
+    except ImportError:
+        pass
+    else:
+
+        for host, selector in (('sourceforge.net', '/projects/python'),
+                               ):
+            print "https://%s%s" % (host, selector)
+            hs = HTTPS()
+            hs.set_debuglevel(dl)
+            hs.connect(host)
+            hs.putrequest('GET', selector)
+            hs.endheaders()
+            status, reason, headers = hs.getreply()
+            print 'status =', status
+            print 'reason =', reason
+            print "read", len(hs.getfile().read())
+            print
+            if headers:
+                for header in headers.headers: print header.strip()
+            print
+
+if __name__ == '__main__':
+    test()
diff --git a/lib-python/modified-2.7/test/test_array.py b/lib-python/modified-2.7/test/test_array.py
--- a/lib-python/modified-2.7/test/test_array.py
+++ b/lib-python/modified-2.7/test/test_array.py
@@ -295,9 +295,10 @@
         )
 
         b = array.array(self.badtypecode())
-        self.assertRaises(TypeError, "a + b")
-
-        self.assertRaises(TypeError, "a + 'bad'")
+        with self.assertRaises(TypeError):
+            a + b
+        with self.assertRaises(TypeError):
+            a + 'bad'
 
     def test_iadd(self):
         a = array.array(self.typecode, self.example[::-1])
@@ -316,9 +317,10 @@
         )
 
         b = array.array(self.badtypecode())
-        self.assertRaises(TypeError, "a += b")
-
-        self.assertRaises(TypeError, "a += 'bad'")
+        with self.assertRaises(TypeError):
+            a += b
+        with self.assertRaises(TypeError):
+            a += 'bad'
 
     def test_mul(self):
         a = 5*array.array(self.typecode, self.example)
@@ -345,7 +347,8 @@
             array.array(self.typecode)
         )
 
-        self.assertRaises(TypeError, "a * 'bad'")
+        with self.assertRaises(TypeError):
+            a * 'bad'
 
     def test_imul(self):
         a = array.array(self.typecode, self.example)
@@ -374,7 +377,8 @@
         a *= -1
         self.assertEqual(a, array.array(self.typecode))
 
-        self.assertRaises(TypeError, "a *= 'bad'")
+        with self.assertRaises(TypeError):
+            a *= 'bad'
 
     def test_getitem(self):
         a = array.array(self.typecode, self.example)
diff --git a/lib-python/modified-2.7/test/test_sys_settrace.py b/lib-python/modified-2.7/test/test_sys_settrace.py
--- a/lib-python/modified-2.7/test/test_sys_settrace.py
+++ b/lib-python/modified-2.7/test/test_sys_settrace.py
@@ -286,11 +286,11 @@
         self.compare_events(func.func_code.co_firstlineno,
                             tracer.events, func.events)
 
-    def set_and_retrieve_none(self):
+    def test_set_and_retrieve_none(self):
         sys.settrace(None)
         assert sys.gettrace() is None
 
-    def set_and_retrieve_func(self):
+    def test_set_and_retrieve_func(self):
         def fn(*args):
             pass
 
diff --git a/lib-python/modified-2.7/test/test_urllib2.py b/lib-python/modified-2.7/test/test_urllib2.py
--- a/lib-python/modified-2.7/test/test_urllib2.py
+++ b/lib-python/modified-2.7/test/test_urllib2.py
@@ -307,6 +307,9 @@
     def getresponse(self):
         return MockHTTPResponse(MockFile(), {}, 200, "OK")
 
+    def close(self):
+        pass
+
 class MockHandler:
     # useful for testing handler machinery
     # see add_ordered_mock_handlers() docstring
diff --git a/lib-python/modified-2.7/urllib2.py b/lib-python/modified-2.7/urllib2.py
new file mode 100644
--- /dev/null
+++ b/lib-python/modified-2.7/urllib2.py
@@ -0,0 +1,1440 @@
+"""An extensible library for opening URLs using a variety of protocols
+
+The simplest way to use this module is to call the urlopen function,
+which accepts a string containing a URL or a Request object (described
+below).  It opens the URL and returns the results as file-like
+object; the returned object has some extra methods described below.
+
+The OpenerDirector manages a collection of Handler objects that do
+all the actual work.  Each Handler implements a particular protocol or
+option.  The OpenerDirector is a composite object that invokes the
+Handlers needed to open the requested URL.  For example, the
+HTTPHandler performs HTTP GET and POST requests and deals with
+non-error returns.  The HTTPRedirectHandler automatically deals with
+HTTP 301, 302, 303 and 307 redirect errors, and the HTTPDigestAuthHandler
+deals with digest authentication.
+
+urlopen(url, data=None) -- Basic usage is the same as original
+urllib.  pass the url and optionally data to post to an HTTP URL, and
+get a file-like object back.  One difference is that you can also pass
+a Request instance instead of URL.  Raises a URLError (subclass of
+IOError); for HTTP errors, raises an HTTPError, which can also be
+treated as a valid response.
+
+build_opener -- Function that creates a new OpenerDirector instance.
+Will install the default handlers.  Accepts one or more Handlers as
+arguments, either instances or Handler classes that it will
+instantiate.  If one of the argument is a subclass of the default
+handler, the argument will be installed instead of the default.
+
+install_opener -- Installs a new opener as the default opener.
+
+objects of interest:
+
+OpenerDirector -- Sets up the User Agent as the Python-urllib client and manages
+the Handler classes, while dealing with requests and responses.
+
+Request -- An object that encapsulates the state of a request.  The
+state can be as simple as the URL.  It can also include extra HTTP
+headers, e.g. a User-Agent.
+
+BaseHandler --
+
+exceptions:
+URLError -- A subclass of IOError, individual protocols have their own
+specific subclass.
+
+HTTPError -- Also a valid HTTP response, so you can treat an HTTP error
+as an exceptional event or valid response.
+
+internals:
+BaseHandler and parent
+_call_chain conventions
+
+Example usage:
+
+import urllib2
+
+# set up authentication info
+authinfo = urllib2.HTTPBasicAuthHandler()
+authinfo.add_password(realm='PDQ Application',
+                      uri='https://mahler:8092/site-updates.py',
+                      user='klem',
+                      passwd='geheim$parole')
+
+proxy_support = urllib2.ProxyHandler({"http" : "http://ahad-haam:3128"})
+
+# build a new opener that adds authentication and caching FTP handlers
+opener = urllib2.build_opener(proxy_support, authinfo, urllib2.CacheFTPHandler)
+
+# install it
+urllib2.install_opener(opener)
+
+f = urllib2.urlopen('http://www.python.org/')
+
+
+"""
+
+# XXX issues:
+# If an authentication error handler that tries to perform
+# authentication for some reason but fails, how should the error be
+# signalled?  The client needs to know the HTTP error code.  But if
+# the handler knows that the problem was, e.g., that it didn't know
+# that hash algo that requested in the challenge, it would be good to
+# pass that information along to the client, too.
+# ftp errors aren't handled cleanly
+# check digest against correct (i.e. non-apache) implementation
+
+# Possible extensions:
+# complex proxies  XXX not sure what exactly was meant by this
+# abstract factory for opener
+
+import base64
+import hashlib
+import httplib
+import mimetools
+import os
+import posixpath
+import random
+import re
+import socket
+import sys
+import time
+import urlparse
+import bisect
+
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from StringIO import StringIO
+
+from urllib import (unwrap, unquote, splittype, splithost, quote,
+     addinfourl, splitport, splittag,
+     splitattr, ftpwrapper, splituser, splitpasswd, splitvalue)
+
+# support for FileHandler, proxies via environment variables
+from urllib import localhost, url2pathname, getproxies, proxy_bypass
+
+# used in User-Agent header sent
+__version__ = sys.version[:3]
+
+_opener = None
+def urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
+    global _opener
+    if _opener is None:
+        _opener = build_opener()
+    return _opener.open(url, data, timeout)
+
+def install_opener(opener):
+    global _opener
+    _opener = opener
+
+# do these error classes make sense?
+# make sure all of the IOError stuff is overridden.  we just want to be
+# subtypes.
+
+class URLError(IOError):
+    # URLError is a sub-type of IOError, but it doesn't share any of
+    # the implementation.  need to override __init__ and __str__.
+    # It sets self.args for compatibility with other EnvironmentError
+    # subclasses, but args doesn't have the typical format with errno in
+    # slot 0 and strerror in slot 1.  This may be better than nothing.
+    def __init__(self, reason):
+        self.args = reason,
+        self.reason = reason
+
+    def __str__(self):
+        return '<urlopen error %s>' % self.reason
+
+class HTTPError(URLError, addinfourl):
+    """Raised when HTTP error occurs, but also acts like non-error return"""
+    __super_init = addinfourl.__init__
+
+    def __init__(self, url, code, msg, hdrs, fp):
+        self.code = code
+        self.msg = msg
+        self.hdrs = hdrs
+        self.fp = fp
+        self.filename = url
+        # The addinfourl classes depend on fp being a valid file
+        # object.  In some cases, the HTTPError may not have a valid
+        # file object.  If this happens, the simplest workaround is to
+        # not initialize the base classes.
+        if fp is not None:
+            self.__super_init(fp, hdrs, url, code)
+
+    def __str__(self):
+        return 'HTTP Error %s: %s' % (self.code, self.msg)
+
+# copied from cookielib.py
+_cut_port_re = re.compile(r":\d+$")
+def request_host(request):
+    """Return request-host, as defined by RFC 2965.
+
+    Variation from RFC: returned value is lowercased, for convenient
+    comparison.
+
+    """
+    url = request.get_full_url()
+    host = urlparse.urlparse(url)[1]
+    if host == "":
+        host = request.get_header("Host", "")
+
+    # remove port, if present
+    host = _cut_port_re.sub("", host, 1)
+    return host.lower()
+
+class Request:
+
+    def __init__(self, url, data=None, headers={},
+                 origin_req_host=None, unverifiable=False):
+        # unwrap('<URL:type://host/path>') --> 'type://host/path'
+        self.__original = unwrap(url)
+        self.__original, fragment = splittag(self.__original)
+        self.type = None
+        # self.__r_type is what's left after doing the splittype
+        self.host = None
+        self.port = None
+        self._tunnel_host = None
+        self.data = data
+        self.headers = {}
+        for key, value in headers.items():
+            self.add_header(key, value)
+        self.unredirected_hdrs = {}
+        if origin_req_host is None:
+            origin_req_host = request_host(self)
+        self.origin_req_host = origin_req_host
+        self.unverifiable = unverifiable
+
+    def __getattr__(self, attr):
+        # XXX this is a fallback mechanism to guard against these
+        # methods getting called in a non-standard order.  this may be
+        # too complicated and/or unnecessary.
+        # XXX should the __r_XXX attributes be public?
+        if attr[:12] == '_Request__r_':
+            name = attr[12:]
+            if hasattr(Request, 'get_' + name):
+                getattr(self, 'get_' + name)()
+                return getattr(self, attr)
+        raise AttributeError, attr
+
+    def get_method(self):
+        if self.has_data():
+            return "POST"
+        else:
+            return "GET"
+
+    # XXX these helper methods are lame
+
+    def add_data(self, data):
+        self.data = data
+
+    def has_data(self):
+        return self.data is not None
+
+    def get_data(self):
+        return self.data
+
+    def get_full_url(self):
+        return self.__original
+
+    def get_type(self):
+        if self.type is None:
+            self.type, self.__r_type = splittype(self.__original)
+            if self.type is None:
+                raise ValueError, "unknown url type: %s" % self.__original
+        return self.type
+
+    def get_host(self):
+        if self.host is None:
+            self.host, self.__r_host = splithost(self.__r_type)
+            if self.host:
+                self.host = unquote(self.host)
+        return self.host
+
+    def get_selector(self):
+        return self.__r_host
+
+    def set_proxy(self, host, type):
+        if self.type == 'https' and not self._tunnel_host:
+            self._tunnel_host = self.host
+        else:
+            self.type = type
+            self.__r_host = self.__original
+
+        self.host = host
+
+    def has_proxy(self):
+        return self.__r_host == self.__original
+
+    def get_origin_req_host(self):
+        return self.origin_req_host
+
+    def is_unverifiable(self):
+        return self.unverifiable
+
+    def add_header(self, key, val):
+        # useful for something like authentication
+        self.headers[key.capitalize()] = val
+
+    def add_unredirected_header(self, key, val):
+        # will not be added to a redirected request
+        self.unredirected_hdrs[key.capitalize()] = val
+
+    def has_header(self, header_name):
+        return (header_name in self.headers or
+                header_name in self.unredirected_hdrs)
+
+    def get_header(self, header_name, default=None):
+        return self.headers.get(
+            header_name,
+            self.unredirected_hdrs.get(header_name, default))
+
+    def header_items(self):
+        hdrs = self.unredirected_hdrs.copy()
+        hdrs.update(self.headers)
+        return hdrs.items()
+
+class OpenerDirector:
+    def __init__(self):
+        client_version = "Python-urllib/%s" % __version__
+        self.addheaders = [('User-agent', client_version)]
+        # manage the individual handlers
+        self.handlers = []
+        self.handle_open = {}
+        self.handle_error = {}
+        self.process_response = {}
+        self.process_request = {}
+
+    def add_handler(self, handler):
+        if not hasattr(handler, "add_parent"):
+            raise TypeError("expected BaseHandler instance, got %r" %
+                            type(handler))
+
+        added = False
+        for meth in dir(handler):
+            if meth in ["redirect_request", "do_open", "proxy_open"]:
+                # oops, coincidental match
+                continue
+
+            i = meth.find("_")
+            protocol = meth[:i]
+            condition = meth[i+1:]
+
+            if condition.startswith("error"):
+                j = condition.find("_") + i + 1
+                kind = meth[j+1:]
+                try:
+                    kind = int(kind)
+                except ValueError:
+                    pass
+                lookup = self.handle_error.get(protocol, {})
+                self.handle_error[protocol] = lookup
+            elif condition == "open":
+                kind = protocol
+                lookup = self.handle_open
+            elif condition == "response":
+                kind = protocol
+                lookup = self.process_response
+            elif condition == "request":
+                kind = protocol
+                lookup = self.process_request
+            else:
+                continue
+
+            handlers = lookup.setdefault(kind, [])
+            if handlers:
+                bisect.insort(handlers, handler)
+            else:
+                handlers.append(handler)
+            added = True
+
+        if added:
+            # the handlers must work in an specific order, the order
+            # is specified in a Handler attribute
+            bisect.insort(self.handlers, handler)
+            handler.add_parent(self)
+
+    def close(self):
+        # Only exists for backwards compatibility.
+        pass
+
+    def _call_chain(self, chain, kind, meth_name, *args):
+        # Handlers raise an exception if no one else should try to handle
+        # the request, or return None if they can't but another handler
+        # could.  Otherwise, they return the response.
+        handlers = chain.get(kind, ())
+        for handler in handlers:
+            func = getattr(handler, meth_name)
+
+            result = func(*args)
+            if result is not None:
+                return result
+
+    def open(self, fullurl, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
+        # accept a URL or a Request object
+        if isinstance(fullurl, basestring):
+            req = Request(fullurl, data)
+        else:
+            req = fullurl
+            if data is not None:
+                req.add_data(data)
+
+        req.timeout = timeout
+        protocol = req.get_type()
+
+        # pre-process request
+        meth_name = protocol+"_request"
+        for processor in self.process_request.get(protocol, []):
+            meth = getattr(processor, meth_name)
+            req = meth(req)
+
+        response = self._open(req, data)
+
+        # post-process response
+        meth_name = protocol+"_response"
+        for processor in self.process_response.get(protocol, []):
+            meth = getattr(processor, meth_name)
+            try:
+                response = meth(req, response)
+            except:
+                response.close()
+                raise
+
+        return response
+
+    def _open(self, req, data=None):
+        result = self._call_chain(self.handle_open, 'default',
+                                  'default_open', req)
+        if result:
+            return result
+
+        protocol = req.get_type()
+        result = self._call_chain(self.handle_open, protocol, protocol +
+                                  '_open', req)
+        if result:
+            return result
+
+        return self._call_chain(self.handle_open, 'unknown',
+                                'unknown_open', req)
+
+    def error(self, proto, *args):
+        if proto in ('http', 'https'):
+            # XXX http[s] protocols are special-cased
+            dict = self.handle_error['http'] # https is not different than http
+            proto = args[2]  # YUCK!
+            meth_name = 'http_error_%s' % proto
+            http_err = 1
+            orig_args = args
+        else:
+            dict = self.handle_error
+            meth_name = proto + '_error'
+            http_err = 0
+        args = (dict, proto, meth_name) + args
+        result = self._call_chain(*args)
+        if result:
+            return result
+
+        if http_err:
+            args = (dict, 'default', 'http_error_default') + orig_args
+            return self._call_chain(*args)
+
+# XXX probably also want an abstract factory that knows when it makes
+# sense to skip a superclass in favor of a subclass and when it might
+# make sense to include both
+
+def build_opener(*handlers):
+    """Create an opener object from a list of handlers.
+
+    The opener will use several default handlers, including support
+    for HTTP, FTP and when applicable, HTTPS.
+
+    If any of the handlers passed as arguments are subclasses of the
+    default handlers, the default handlers will not be used.
+    """
+    import types
+    def isclass(obj):
+        return isinstance(obj, (types.ClassType, type))
+
+    opener = OpenerDirector()
+    default_classes = [ProxyHandler, UnknownHandler, HTTPHandler,
+                       HTTPDefaultErrorHandler, HTTPRedirectHandler,
+                       FTPHandler, FileHandler, HTTPErrorProcessor]
+    if hasattr(httplib, 'HTTPS'):
+        default_classes.append(HTTPSHandler)
+    skip = set()
+    for klass in default_classes:
+        for check in handlers:
+            if isclass(check):
+                if issubclass(check, klass):
+                    skip.add(klass)
+            elif isinstance(check, klass):
+                skip.add(klass)
+    for klass in skip:
+        default_classes.remove(klass)
+
+    for klass in default_classes:
+        opener.add_handler(klass())
+
+    for h in handlers:
+        if isclass(h):
+            h = h()
+        opener.add_handler(h)
+    return opener
+
+class BaseHandler:
+    handler_order = 500
+
+    def add_parent(self, parent):
+        self.parent = parent
+
+    def close(self):
+        # Only exists for backwards compatibility
+        pass
+
+    def __lt__(self, other):
+        if not hasattr(other, "handler_order"):
+            # Try to preserve the old behavior of having custom classes
+            # inserted after default ones (works only for custom user
+            # classes which are not aware of handler_order).
+            return True
+        return self.handler_order < other.handler_order
+
+
+class HTTPErrorProcessor(BaseHandler):
+    """Process HTTP error responses."""
+    handler_order = 1000  # after all other processing
+
+    def http_response(self, request, response):
+        code, msg, hdrs = response.code, response.msg, response.info()
+
+        # According to RFC 2616, "2xx" code indicates that the client's
+        # request was successfully received, understood, and accepted.
+        if not (200 <= code < 300):
+            response = self.parent.error(
+                'http', request, response, code, msg, hdrs)
+
+        return response
+
+    https_response = http_response
+
+class HTTPDefaultErrorHandler(BaseHandler):
+    def http_error_default(self, req, fp, code, msg, hdrs):
+        raise HTTPError(req.get_full_url(), code, msg, hdrs, fp)
+
+class HTTPRedirectHandler(BaseHandler):
+    # maximum number of redirections to any single URL
+    # this is needed because of the state that cookies introduce
+    max_repeats = 4
+    # maximum total number of redirections (regardless of URL) before
+    # assuming we're in a loop
+    max_redirections = 10
+
+    def redirect_request(self, req, fp, code, msg, headers, newurl):
+        """Return a Request or None in response to a redirect.
+
+        This is called by the http_error_30x methods when a
+        redirection response is received.  If a redirection should
+        take place, return a new Request to allow http_error_30x to
+        perform the redirect.  Otherwise, raise HTTPError if no-one
+        else should try to handle this url.  Return None if you can't
+        but another Handler might.
+        """
+        m = req.get_method()
+        if (code in (301, 302, 303, 307) and m in ("GET", "HEAD")
+            or code in (301, 302, 303) and m == "POST"):
+            # Strictly (according to RFC 2616), 301 or 302 in response
+            # to a POST MUST NOT cause a redirection without confirmation
+            # from the user (of urllib2, in this case).  In practice,
+            # essentially all clients do redirect in this case, so we
+            # do the same.
+            # be conciliant with URIs containing a space
+            newurl = newurl.replace(' ', '%20')
+            newheaders = dict((k,v) for k,v in req.headers.items()
+                              if k.lower() not in ("content-length", "content-type")
+                             )
+            return Request(newurl,
+                           headers=newheaders,
+                           origin_req_host=req.get_origin_req_host(),
+                           unverifiable=True)
+        else:
+            raise HTTPError(req.get_full_url(), code, msg, headers, fp)
+
+    # Implementation note: To avoid the server sending us into an
+    # infinite loop, the request object needs to track what URLs we
+    # have already seen.  Do this by adding a handler-specific
+    # attribute to the Request object.
+    def http_error_302(self, req, fp, code, msg, headers):
+        # Some servers (incorrectly) return multiple Location headers
+        # (so probably same goes for URI).  Use first header.
+        if 'location' in headers:
+            newurl = headers.getheaders('location')[0]
+        elif 'uri' in headers:
+            newurl = headers.getheaders('uri')[0]
+        else:
+            return
+
+        # fix a possible malformed URL
+        urlparts = urlparse.urlparse(newurl)
+        if not urlparts.path:
+            urlparts = list(urlparts)
+            urlparts[2] = "/"
+        newurl = urlparse.urlunparse(urlparts)
+
+        newurl = urlparse.urljoin(req.get_full_url(), newurl)
+
+        # XXX Probably want to forget about the state of the current
+        # request, although that might interact poorly with other
+        # handlers that also use handler-specific request attributes
+        new = self.redirect_request(req, fp, code, msg, headers, newurl)
+        if new is None:
+            return
+
+        # loop detection
+        # .redirect_dict has a key url if url was previously visited.
+        if hasattr(req, 'redirect_dict'):
+            visited = new.redirect_dict = req.redirect_dict
+            if (visited.get(newurl, 0) >= self.max_repeats or
+                len(visited) >= self.max_redirections):
+                raise HTTPError(req.get_full_url(), code,
+                                self.inf_msg + msg, headers, fp)
+        else:
+            visited = new.redirect_dict = req.redirect_dict = {}
+        visited[newurl] = visited.get(newurl, 0) + 1
+
+        # Don't close the fp until we are sure that we won't use it
+        # with HTTPError.
+        fp.read()
+        fp.close()
+
+        return self.parent.open(new, timeout=req.timeout)
+
+    http_error_301 = http_error_303 = http_error_307 = http_error_302
+
+    inf_msg = "The HTTP server returned a redirect error that would " \
+              "lead to an infinite loop.\n" \
+              "The last 30x error message was:\n"
+
+
+def _parse_proxy(proxy):
+    """Return (scheme, user, password, host/port) given a URL or an authority.
+
+    If a URL is supplied, it must have an authority (host:port) component.
+    According to RFC 3986, having an authority component means the URL must
+    have two slashes after the scheme:
+
+    >>> _parse_proxy('file:/ftp.example.com/')
+    Traceback (most recent call last):
+    ValueError: proxy URL with no authority: 'file:/ftp.example.com/'
+
+    The first three items of the returned tuple may be None.
+
+    Examples of authority parsing:
+
+    >>> _parse_proxy('proxy.example.com')
+    (None, None, None, 'proxy.example.com')
+    >>> _parse_proxy('proxy.example.com:3128')
+    (None, None, None, 'proxy.example.com:3128')
+
+    The authority component may optionally include userinfo (assumed to be
+    username:password):
+
+    >>> _parse_proxy('joe:password at proxy.example.com')
+    (None, 'joe', 'password', 'proxy.example.com')
+    >>> _parse_proxy('joe:password at proxy.example.com:3128')
+    (None, 'joe', 'password', 'proxy.example.com:3128')
+
+    Same examples, but with URLs instead:
+
+    >>> _parse_proxy('http://proxy.example.com/')
+    ('http', None, None, 'proxy.example.com')
+    >>> _parse_proxy('http://proxy.example.com:3128/')
+    ('http', None, None, 'proxy.example.com:3128')
+    >>> _parse_proxy('http://joe:password at proxy.example.com/')
+    ('http', 'joe', 'password', 'proxy.example.com')
+    >>> _parse_proxy('http://joe:password at proxy.example.com:3128')
+    ('http', 'joe', 'password', 'proxy.example.com:3128')
+
+    Everything after the authority is ignored:
+
+    >>> _parse_proxy('ftp://joe:password at proxy.example.com/rubbish:3128')
+    ('ftp', 'joe', 'password', 'proxy.example.com')
+
+    Test for no trailing '/' case:
+
+    >>> _parse_proxy('http://joe:password at proxy.example.com')
+    ('http', 'joe', 'password', 'proxy.example.com')
+
+    """
+    scheme, r_scheme = splittype(proxy)
+    if not r_scheme.startswith("/"):
+        # authority
+        scheme = None
+        authority = proxy
+    else:
+        # URL
+        if not r_scheme.startswith("//"):
+            raise ValueError("proxy URL with no authority: %r" % proxy)
+        # We have an authority, so for RFC 3986-compliant URLs (by ss 3.
+        # and 3.3.), path is empty or starts with '/'
+        end = r_scheme.find("/", 2)
+        if end == -1:
+            end = None
+        authority = r_scheme[2:end]
+    userinfo, hostport = splituser(authority)
+    if userinfo is not None:
+        user, password = splitpasswd(userinfo)
+    else:
+        user = password = None
+    return scheme, user, password, hostport
+
+class ProxyHandler(BaseHandler):
+    # Proxies must be in front
+    handler_order = 100
+
+    def __init__(self, proxies=None):
+        if proxies is None:
+            proxies = getproxies()
+        assert hasattr(proxies, 'has_key'), "proxies must be a mapping"
+        self.proxies = proxies
+        for type, url in proxies.items():
+            setattr(self, '%s_open' % type,
+                    lambda r, proxy=url, type=type, meth=self.proxy_open: \
+                    meth(r, proxy, type))
+
+    def proxy_open(self, req, proxy, type):
+        orig_type = req.get_type()
+        proxy_type, user, password, hostport = _parse_proxy(proxy)
+
+        if proxy_type is None:
+            proxy_type = orig_type
+
+        if req.host and proxy_bypass(req.host):
+            return None
+
+        if user and password:
+            user_pass = '%s:%s' % (unquote(user), unquote(password))
+            creds = base64.b64encode(user_pass).strip()
+            req.add_header('Proxy-authorization', 'Basic ' + creds)
+        hostport = unquote(hostport)
+        req.set_proxy(hostport, proxy_type)
+
+        if orig_type == proxy_type or orig_type == 'https':
+            # let other handlers take care of it
+            return None
+        else:
+            # need to start over, because the other handlers don't
+            # grok the proxy's URL type
+            # e.g. if we have a constructor arg proxies like so:
+            # {'http': 'ftp://proxy.example.com'}, we may end up turning
+            # a request for http://acme.example.com/a into one for
+            # ftp://proxy.example.com/a
+            return self.parent.open(req, timeout=req.timeout)
+
+class HTTPPasswordMgr:
+
+    def __init__(self):
+        self.passwd = {}
+
+    def add_password(self, realm, uri, user, passwd):
+        # uri could be a single URI or a sequence
+        if isinstance(uri, basestring):
+            uri = [uri]
+        if not realm in self.passwd:
+            self.passwd[realm] = {}
+        for default_port in True, False:
+            reduced_uri = tuple(
+                [self.reduce_uri(u, default_port) for u in uri])
+            self.passwd[realm][reduced_uri] = (user, passwd)
+
+    def find_user_password(self, realm, authuri):
+        domains = self.passwd.get(realm, {})
+        for default_port in True, False:
+            reduced_authuri = self.reduce_uri(authuri, default_port)
+            for uris, authinfo in domains.iteritems():
+                for uri in uris:
+                    if self.is_suburi(uri, reduced_authuri):
+                        return authinfo
+        return None, None
+
+    def reduce_uri(self, uri, default_port=True):
+        """Accept authority or URI and extract only the authority and path."""
+        # note HTTP URLs do not have a userinfo component
+        parts = urlparse.urlsplit(uri)
+        if parts[1]:
+            # URI
+            scheme = parts[0]
+            authority = parts[1]
+            path = parts[2] or '/'
+        else:
+            # host or host:port
+            scheme = None
+            authority = uri
+            path = '/'
+        host, port = splitport(authority)
+        if default_port and port is None and scheme is not None:
+            dport = {"http": 80,
+                     "https": 443,
+                     }.get(scheme)
+            if dport is not None:
+                authority = "%s:%d" % (host, dport)
+        return authority, path
+
+    def is_suburi(self, base, test):
+        """Check if test is below base in a URI tree
+
+        Both args must be URIs in reduced form.
+        """
+        if base == test:
+            return True
+        if base[0] != test[0]:
+            return False
+        common = posixpath.commonprefix((base[1], test[1]))
+        if len(common) == len(base[1]):
+            return True
+        return False
+
+
+class HTTPPasswordMgrWithDefaultRealm(HTTPPasswordMgr):
+
+    def find_user_password(self, realm, authuri):
+        user, password = HTTPPasswordMgr.find_user_password(self, realm,
+                                                            authuri)
+        if user is not None:
+            return user, password
+        return HTTPPasswordMgr.find_user_password(self, None, authuri)
+
+
+class AbstractBasicAuthHandler:
+
+    # XXX this allows for multiple auth-schemes, but will stupidly pick
+    # the last one with a realm specified.
+
+    # allow for double- and single-quoted realm values
+    # (single quotes are a violation of the RFC, but appear in the wild)
+    rx = re.compile('(?:.*,)*[ \t]*([^ \t]+)[ \t]+'
+                    'realm=(["\'])(.*?)\\2', re.I)
+
+    # XXX could pre-emptively send auth info already accepted (RFC 2617,
+    # end of section 2, and section 1.2 immediately after "credentials"
+    # production).
+
+    def __init__(self, password_mgr=None):
+        if password_mgr is None:
+            password_mgr = HTTPPasswordMgr()
+        self.passwd = password_mgr
+        self.add_password = self.passwd.add_password
+        self.retried = 0
+
+    def reset_retry_count(self):
+        self.retried = 0
+
+    def http_error_auth_reqed(self, authreq, host, req, headers):
+        # host may be an authority (without userinfo) or a URL with an
+        # authority
+        # XXX could be multiple headers
+        authreq = headers.get(authreq, None)
+
+        if self.retried > 5:
+            # retry sending the username:password 5 times before failing.
+            raise HTTPError(req.get_full_url(), 401, "basic auth failed",
+                            headers, None)
+        else:
+            self.retried += 1
+
+        if authreq:
+            mo = AbstractBasicAuthHandler.rx.search(authreq)
+            if mo:
+                scheme, quote, realm = mo.groups()
+                if scheme.lower() == 'basic':
+                    response = self.retry_http_basic_auth(host, req, realm)
+                    if response and response.code != 401:
+                        self.retried = 0
+                    return response
+
+    def retry_http_basic_auth(self, host, req, realm):
+        user, pw = self.passwd.find_user_password(realm, host)
+        if pw is not None:
+            raw = "%s:%s" % (user, pw)
+            auth = 'Basic %s' % base64.b64encode(raw).strip()
+            if req.headers.get(self.auth_header, None) == auth:
+                return None
+            req.add_unredirected_header(self.auth_header, auth)
+            return self.parent.open(req, timeout=req.timeout)
+        else:
+            return None
+
+
+class HTTPBasicAuthHandler(AbstractBasicAuthHandler, BaseHandler):
+
+    auth_header = 'Authorization'
+
+    def http_error_401(self, req, fp, code, msg, headers):
+        url = req.get_full_url()
+        response = self.http_error_auth_reqed('www-authenticate',
+                                              url, req, headers)
+        self.reset_retry_count()
+        return response
+
+
+class ProxyBasicAuthHandler(AbstractBasicAuthHandler, BaseHandler):
+
+    auth_header = 'Proxy-authorization'
+
+    def http_error_407(self, req, fp, code, msg, headers):
+        # http_error_auth_reqed requires that there is no userinfo component in
+        # authority.  Assume there isn't one, since urllib2 does not (and
+        # should not, RFC 3986 s. 3.2.1) support requests for URLs containing
+        # userinfo.
+        authority = req.get_host()
+        response = self.http_error_auth_reqed('proxy-authenticate',
+                                          authority, req, headers)
+        self.reset_retry_count()
+        return response
+
+
+def randombytes(n):
+    """Return n random bytes."""
+    # Use /dev/urandom if it is available.  Fall back to random module
+    # if not.  It might be worthwhile to extend this function to use
+    # other platform-specific mechanisms for getting random bytes.
+    if os.path.exists("/dev/urandom"):
+        f = open("/dev/urandom")
+        s = f.read(n)
+        f.close()
+        return s
+    else:
+        L = [chr(random.randrange(0, 256)) for i in range(n)]
+        return "".join(L)
+
+class AbstractDigestAuthHandler:
+    # Digest authentication is specified in RFC 2617.
+
+    # XXX The client does not inspect the Authentication-Info header
+    # in a successful response.
+
+    # XXX It should be possible to test this implementation against
+    # a mock server that just generates a static set of challenges.
+
+    # XXX qop="auth-int" supports is shaky
+
+    def __init__(self, passwd=None):
+        if passwd is None:
+            passwd = HTTPPasswordMgr()
+        self.passwd = passwd
+        self.add_password = self.passwd.add_password
+        self.retried = 0
+        self.nonce_count = 0
+        self.last_nonce = None
+
+    def reset_retry_count(self):
+        self.retried = 0
+
+    def http_error_auth_reqed(self, auth_header, host, req, headers):
+        authreq = headers.get(auth_header, None)
+        if self.retried > 5:
+            # Don't fail endlessly - if we failed once, we'll probably
+            # fail a second time. Hm. Unless the Password Manager is
+            # prompting for the information. Crap. This isn't great
+            # but it's better than the current 'repeat until recursion
+            # depth exceeded' approach <wink>
+            raise HTTPError(req.get_full_url(), 401, "digest auth failed",
+                            headers, None)
+        else:
+            self.retried += 1
+        if authreq:
+            scheme = authreq.split()[0]
+            if scheme.lower() == 'digest':
+                return self.retry_http_digest_auth(req, authreq)
+
+    def retry_http_digest_auth(self, req, auth):
+        token, challenge = auth.split(' ', 1)
+        chal = parse_keqv_list(parse_http_list(challenge))
+        auth = self.get_authorization(req, chal)
+        if auth:
+            auth_val = 'Digest %s' % auth
+            if req.headers.get(self.auth_header, None) == auth_val:
+                return None
+            req.add_unredirected_header(self.auth_header, auth_val)
+            resp = self.parent.open(req, timeout=req.timeout)
+            return resp
+
+    def get_cnonce(self, nonce):
+        # The cnonce-value is an opaque
+        # quoted string value provided by the client and used by both client
+        # and server to avoid chosen plaintext attacks, to provide mutual
+        # authentication, and to provide some message integrity protection.
+        # This isn't a fabulous effort, but it's probably Good Enough.
+        dig = hashlib.sha1("%s:%s:%s:%s" % (self.nonce_count, nonce, time.ctime(),
+                                            randombytes(8))).hexdigest()
+        return dig[:16]
+
+    def get_authorization(self, req, chal):
+        try:
+            realm = chal['realm']
+            nonce = chal['nonce']
+            qop = chal.get('qop')
+            algorithm = chal.get('algorithm', 'MD5')
+            # mod_digest doesn't send an opaque, even though it isn't
+            # supposed to be optional
+            opaque = chal.get('opaque', None)
+        except KeyError:
+            return None
+
+        H, KD = self.get_algorithm_impls(algorithm)
+        if H is None:
+            return None
+
+        user, pw = self.passwd.find_user_password(realm, req.get_full_url())
+        if user is None:
+            return None
+
+        # XXX not implemented yet
+        if req.has_data():
+            entdig = self.get_entity_digest(req.get_data(), chal)
+        else:
+            entdig = None
+
+        A1 = "%s:%s:%s" % (user, realm, pw)
+        A2 = "%s:%s" % (req.get_method(),
+                        # XXX selector: what about proxies and full urls
+                        req.get_selector())
+        if qop == 'auth':
+            if nonce == self.last_nonce:
+                self.nonce_count += 1
+            else:
+                self.nonce_count = 1
+                self.last_nonce = nonce
+
+            ncvalue = '%08x' % self.nonce_count
+            cnonce = self.get_cnonce(nonce)
+            noncebit = "%s:%s:%s:%s:%s" % (nonce, ncvalue, cnonce, qop, H(A2))
+            respdig = KD(H(A1), noncebit)
+        elif qop is None:
+            respdig = KD(H(A1), "%s:%s" % (nonce, H(A2)))
+        else:
+            # XXX handle auth-int.
+            raise URLError("qop '%s' is not supported." % qop)
+
+        # XXX should the partial digests be encoded too?
+
+        base = 'username="%s", realm="%s", nonce="%s", uri="%s", ' \
+               'response="%s"' % (user, realm, nonce, req.get_selector(),
+                                  respdig)
+        if opaque:
+            base += ', opaque="%s"' % opaque
+        if entdig:
+            base += ', digest="%s"' % entdig
+        base += ', algorithm="%s"' % algorithm
+        if qop:
+            base += ', qop=auth, nc=%s, cnonce="%s"' % (ncvalue, cnonce)
+        return base
+
+    def get_algorithm_impls(self, algorithm):
+        # algorithm should be case-insensitive according to RFC2617
+        algorithm = algorithm.upper()
+        # lambdas assume digest modules are imported at the top level
+        if algorithm == 'MD5':
+            H = lambda x: hashlib.md5(x).hexdigest()
+        elif algorithm == 'SHA':
+            H = lambda x: hashlib.sha1(x).hexdigest()
+        # XXX MD5-sess
+        KD = lambda s, d: H("%s:%s" % (s, d))
+        return H, KD
+
+    def get_entity_digest(self, data, chal):
+        # XXX not implemented yet
+        return None
+
+
+class HTTPDigestAuthHandler(BaseHandler, AbstractDigestAuthHandler):
+    """An authentication protocol defined by RFC 2069
+
+    Digest authentication improves on basic authentication because it
+    does not transmit passwords in the clear.
+    """
+
+    auth_header = 'Authorization'
+    handler_order = 490  # before Basic auth
+
+    def http_error_401(self, req, fp, code, msg, headers):
+        host = urlparse.urlparse(req.get_full_url())[1]
+        retry = self.http_error_auth_reqed('www-authenticate',
+                                           host, req, headers)
+        self.reset_retry_count()
+        return retry
+
+
+class ProxyDigestAuthHandler(BaseHandler, AbstractDigestAuthHandler):
+
+    auth_header = 'Proxy-Authorization'
+    handler_order = 490  # before Basic auth
+
+    def http_error_407(self, req, fp, code, msg, headers):
+        host = req.get_host()
+        retry = self.http_error_auth_reqed('proxy-authenticate',
+                                           host, req, headers)
+        self.reset_retry_count()
+        return retry
+
+class AbstractHTTPHandler(BaseHandler):
+
+    def __init__(self, debuglevel=0):
+        self._debuglevel = debuglevel
+
+    def set_http_debuglevel(self, level):
+        self._debuglevel = level
+
+    def do_request_(self, request):
+        host = request.get_host()
+        if not host:
+            raise URLError('no host given')
+
+        if request.has_data():  # POST
+            data = request.get_data()
+            if not request.has_header('Content-type'):
+                request.add_unredirected_header(
+                    'Content-type',
+                    'application/x-www-form-urlencoded')
+            if not request.has_header('Content-length'):
+                request.add_unredirected_header(
+                    'Content-length', '%d' % len(data))
+
+        sel_host = host
+        if request.has_proxy():
+            scheme, sel = splittype(request.get_selector())
+            sel_host, sel_path = splithost(sel)
+
+        if not request.has_header('Host'):
+            request.add_unredirected_header('Host', sel_host)
+        for name, value in self.parent.addheaders:
+            name = name.capitalize()
+            if not request.has_header(name):
+                request.add_unredirected_header(name, value)
+
+        return request
+
+    def do_open(self, http_class, req):
+        """Return an addinfourl object for the request, using http_class.
+
+        http_class must implement the HTTPConnection API from httplib.
+        The addinfourl return value is a file-like object.  It also
+        has methods and attributes including:
+            - info(): return a mimetools.Message object for the headers
+            - geturl(): return the original request URL
+            - code: HTTP status code
+        """
+        host = req.get_host()
+        if not host:
+            raise URLError('no host given')
+
+        h = http_class(host, timeout=req.timeout) # will parse host:port
+        h.set_debuglevel(self._debuglevel)
+
+        headers = dict(req.unredirected_hdrs)
+        headers.update(dict((k, v) for k, v in req.headers.items()
+                            if k not in headers))
+
+        # We want to make an HTTP/1.1 request, but the addinfourl
+        # class isn't prepared to deal with a persistent connection.
+        # It will try to read all remaining data from the socket,
+        # which will block while the server waits for the next request.
+        # So make sure the connection gets closed after the (only)
+        # request.
+        headers["Connection"] = "close"
+        headers = dict(
+            (name.title(), val) for name, val in headers.items())
+
+        if req._tunnel_host:
+            tunnel_headers = {}
+            proxy_auth_hdr = "Proxy-Authorization"
+            if proxy_auth_hdr in headers:
+                tunnel_headers[proxy_auth_hdr] = headers[proxy_auth_hdr]
+                # Proxy-Authorization should not be sent to origin
+                # server.
+                del headers[proxy_auth_hdr]
+            h.set_tunnel(req._tunnel_host, headers=tunnel_headers)
+
+        try:
+            h.request(req.get_method(), req.get_selector(), req.data, headers)
+            try:
+                r = h.getresponse(buffering=True)
+            except TypeError: #buffering kw not supported
+                r = h.getresponse()
+        except socket.error, err: # XXX what error?
+            h.close()
+            raise URLError(err)
+
+        # Pick apart the HTTPResponse object to get the addinfourl
+        # object initialized properly.
+
+        # Wrap the HTTPResponse object in socket's file object adapter
+        # for Windows.  That adapter calls recv(), so delegate recv()
+        # to read().  This weird wrapping allows the returned object to
+        # have readline() and readlines() methods.
+
+        # XXX It might be better to extract the read buffering code
+        # out of socket._fileobject() and into a base class.
+
+        r.recv = r.read
+        fp = socket._fileobject(r, close=True)
+
+        resp = addinfourl(fp, r.msg, req.get_full_url())
+        resp.code = r.status
+        resp.msg = r.reason
+        return resp
+
+
+class HTTPHandler(AbstractHTTPHandler):
+
+    def http_open(self, req):
+        return self.do_open(httplib.HTTPConnection, req)
+
+    http_request = AbstractHTTPHandler.do_request_
+
+if hasattr(httplib, 'HTTPS'):
+    class HTTPSHandler(AbstractHTTPHandler):
+
+        def https_open(self, req):
+            return self.do_open(httplib.HTTPSConnection, req)
+
+        https_request = AbstractHTTPHandler.do_request_
+
+class HTTPCookieProcessor(BaseHandler):
+    def __init__(self, cookiejar=None):
+        import cookielib
+        if cookiejar is None:
+            cookiejar = cookielib.CookieJar()
+        self.cookiejar = cookiejar
+
+    def http_request(self, request):
+        self.cookiejar.add_cookie_header(request)
+        return request
+
+    def http_response(self, request, response):
+        self.cookiejar.extract_cookies(response, request)
+        return response
+
+    https_request = http_request
+    https_response = http_response
+
+class UnknownHandler(BaseHandler):
+    def unknown_open(self, req):
+        type = req.get_type()
+        raise URLError('unknown url type: %s' % type)
+
+def parse_keqv_list(l):
+    """Parse list of key=value strings where keys are not duplicated."""
+    parsed = {}
+    for elt in l:
+        k, v = elt.split('=', 1)
+        if v[0] == '"' and v[-1] == '"':
+            v = v[1:-1]
+        parsed[k] = v
+    return parsed
+
+def parse_http_list(s):
+    """Parse lists as described by RFC 2068 Section 2.
+
+    In particular, parse comma-separated lists where the elements of
+    the list may include quoted-strings.  A quoted-string could
+    contain a comma.  A non-quoted string could have quotes in the
+    middle.  Neither commas nor quotes count if they are escaped.
+    Only double-quotes count, not single-quotes.
+    """
+    res = []
+    part = ''
+
+    escape = quote = False
+    for cur in s:
+        if escape:
+            part += cur
+            escape = False
+            continue
+        if quote:
+            if cur == '\\':
+                escape = True
+                continue
+            elif cur == '"':
+                quote = False
+            part += cur
+            continue
+
+        if cur == ',':
+            res.append(part)
+            part = ''
+            continue
+
+        if cur == '"':
+            quote = True
+
+        part += cur
+
+    # append last part
+    if part:
+        res.append(part)
+
+    return [part.strip() for part in res]
+
+def _safe_gethostbyname(host):
+    try:
+        return socket.gethostbyname(host)
+    except socket.gaierror:
+        return None
+
+class FileHandler(BaseHandler):
+    # Use local file or FTP depending on form of URL
+    def file_open(self, req):
+        url = req.get_selector()
+        if url[:2] == '//' and url[2:3] != '/' and (req.host and
+                req.host != 'localhost'):
+            req.type = 'ftp'
+            return self.parent.open(req)
+        else:
+            return self.open_local_file(req)
+
+    # names for the localhost
+    names = None
+    def get_names(self):
+        if FileHandler.names is None:
+            try:
+                FileHandler.names = tuple(
+                    socket.gethostbyname_ex('localhost')[2] +
+                    socket.gethostbyname_ex(socket.gethostname())[2])
+            except socket.gaierror:
+                FileHandler.names = (socket.gethostbyname('localhost'),)
+        return FileHandler.names
+
+    # not entirely sure what the rules are here
+    def open_local_file(self, req):
+        import email.utils
+        import mimetypes
+        host = req.get_host()
+        filename = req.get_selector()
+        localfile = url2pathname(filename)
+        try:
+            stats = os.stat(localfile)
+            size = stats.st_size
+            modified = email.utils.formatdate(stats.st_mtime, usegmt=True)
+            mtype = mimetypes.guess_type(filename)[0]
+            headers = mimetools.Message(StringIO(
+                'Content-type: %s\nContent-length: %d\nLast-modified: %s\n' %
+                (mtype or 'text/plain', size, modified)))
+            if host:
+                host, port = splitport(host)
+            if not host or \
+                (not port and _safe_gethostbyname(host) in self.get_names()):
+                if host:
+                    origurl = 'file://' + host + filename
+                else:
+                    origurl = 'file://' + filename
+                return addinfourl(open(localfile, 'rb'), headers, origurl)
+        except OSError, msg:
+            # urllib2 users shouldn't expect OSErrors coming from urlopen()
+            raise URLError(msg)
+        raise URLError('file not on local host')
+
+class FTPHandler(BaseHandler):
+    def ftp_open(self, req):
+        import ftplib
+        import mimetypes
+        host = req.get_host()
+        if not host:
+            raise URLError('ftp error: no host given')
+        host, port = splitport(host)
+        if port is None:
+            port = ftplib.FTP_PORT
+        else:
+            port = int(port)
+
+        # username/password handling
+        user, host = splituser(host)
+        if user:
+            user, passwd = splitpasswd(user)
+        else:
+            passwd = None
+        host = unquote(host)
+        user = user or ''
+        passwd = passwd or ''
+
+        try:
+            host = socket.gethostbyname(host)
+        except socket.error, msg:
+            raise URLError(msg)
+        path, attrs = splitattr(req.get_selector())
+        dirs = path.split('/')
+        dirs = map(unquote, dirs)
+        dirs, file = dirs[:-1], dirs[-1]
+        if dirs and not dirs[0]:
+            dirs = dirs[1:]
+        try:
+            fw = self.connect_ftp(user, passwd, host, port, dirs, req.timeout)
+            type = file and 'I' or 'D'
+            for attr in attrs:
+                attr, value = splitvalue(attr)
+                if attr.lower() == 'type' and \
+                   value in ('a', 'A', 'i', 'I', 'd', 'D'):
+                    type = value.upper()
+            fp, retrlen = fw.retrfile(file, type)
+            headers = ""
+            mtype = mimetypes.guess_type(req.get_full_url())[0]
+            if mtype:
+                headers += "Content-type: %s\n" % mtype
+            if retrlen is not None and retrlen >= 0:
+                headers += "Content-length: %d\n" % retrlen
+            sf = StringIO(headers)
+            headers = mimetools.Message(sf)
+            return addinfourl(fp, headers, req.get_full_url())
+        except ftplib.all_errors, msg:
+            raise URLError, ('ftp error: %s' % msg), sys.exc_info()[2]
+
+    def connect_ftp(self, user, passwd, host, port, dirs, timeout):
+        fw = ftpwrapper(user, passwd, host, port, dirs, timeout)
+##        fw.ftp.set_debuglevel(1)
+        return fw
+
+class CacheFTPHandler(FTPHandler):
+    # XXX would be nice to have pluggable cache strategies
+    # XXX this stuff is definitely not thread safe
+    def __init__(self):
+        self.cache = {}
+        self.timeout = {}
+        self.soonest = 0
+        self.delay = 60
+        self.max_conns = 16
+
+    def setTimeout(self, t):
+        self.delay = t
+
+    def setMaxConns(self, m):
+        self.max_conns = m
+
+    def connect_ftp(self, user, passwd, host, port, dirs, timeout):
+        key = user, host, port, '/'.join(dirs), timeout
+        if key in self.cache:
+            self.timeout[key] = time.time() + self.delay
+        else:
+            self.cache[key] = ftpwrapper(user, passwd, host, port, dirs, timeout)
+            self.timeout[key] = time.time() + self.delay
+        self.check_cache()
+        return self.cache[key]
+
+    def check_cache(self):
+        # first check for old ones
+        t = time.time()
+        if self.soonest <= t:
+            for k, v in self.timeout.items():
+                if v < t:
+                    self.cache[k].close()
+                    del self.cache[k]
+                    del self.timeout[k]
+        self.soonest = min(self.timeout.values())
+
+        # then check the size
+        if len(self.cache) == self.max_conns:
+            for k, v in self.timeout.items():
+                if v == self.soonest:
+                    del self.cache[k]
+                    del self.timeout[k]
+                    break
+            self.soonest = min(self.timeout.values())
diff --git a/lib_pypy/resource.py b/lib_pypy/resource.py
--- a/lib_pypy/resource.py
+++ b/lib_pypy/resource.py
@@ -7,7 +7,7 @@
 
 from ctypes_support import standard_c_lib as libc
 from ctypes_support import get_errno
-from ctypes import Structure, c_int, c_long, byref, sizeof, POINTER
+from ctypes import Structure, c_int, c_long, byref, POINTER
 from errno import EINVAL, EPERM
 import _structseq
 
@@ -165,7 +165,6 @@
 
 @builtinify
 def getpagesize():
-    pagesize = 0
     if _getpagesize:
         return _getpagesize()
     else:
diff --git a/pypy/annotation/classdef.py b/pypy/annotation/classdef.py
--- a/pypy/annotation/classdef.py
+++ b/pypy/annotation/classdef.py
@@ -276,8 +276,8 @@
         # create the Attribute and do the generalization asked for
         newattr = Attribute(attr, self.bookkeeper)
         if s_value:
-            if newattr.name == 'intval' and getattr(s_value, 'unsigned', False):
-                import pdb; pdb.set_trace()
+            #if newattr.name == 'intval' and getattr(s_value, 'unsigned', False):
+            #    import pdb; pdb.set_trace()
             newattr.s_value = s_value
 
         # keep all subattributes' values
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -127,7 +127,7 @@
 
 pypy_optiondescription = OptionDescription("objspace", "Object Space Options", [
     ChoiceOption("name", "Object Space name",
-                 ["std", "flow", "thunk", "dump", "taint"],
+                 ["std", "flow", "thunk", "dump"],
                  "std",
                  cmdline='--objspace -o'),
 
diff --git a/pypy/doc/__pypy__-module.rst b/pypy/doc/__pypy__-module.rst
--- a/pypy/doc/__pypy__-module.rst
+++ b/pypy/doc/__pypy__-module.rst
@@ -37,29 +37,6 @@
 .. _`thunk object space docs`: objspace-proxies.html#thunk
 .. _`interface section of the thunk object space docs`: objspace-proxies.html#thunk-interface
 
-.. broken:
-
-    Taint Object Space Functionality
-    ================================
-
-    When the taint object space is used (choose with :config:`objspace.name`),
-    the following names are put into ``__pypy__``:
-
-     - ``taint``
-     - ``is_tainted``
-     - ``untaint``
-     - ``taint_atomic``
-     - ``_taint_debug``
-     - ``_taint_look``
-     - ``TaintError``
-
-    Those are all described in the `interface section of the taint object space
-    docs`_.
-
-    For more detailed explanations and examples see the `taint object space docs`_.
-
-    .. _`taint object space docs`: objspace-proxies.html#taint
-    .. _`interface section of the taint object space docs`: objspace-proxies.html#taint-interface
 
 Transparent Proxy Functionality
 ===============================
diff --git a/pypy/doc/config/objspace.name.txt b/pypy/doc/config/objspace.name.txt
--- a/pypy/doc/config/objspace.name.txt
+++ b/pypy/doc/config/objspace.name.txt
@@ -4,7 +4,6 @@
 for normal usage):
 
   * thunk_: The thunk object space adds lazy evaluation to PyPy.
-  * taint_: The taint object space adds soft security features.
   * dump_:  Using this object spaces results in the dumpimp of all operations
     to a log.
 
@@ -12,5 +11,4 @@
 .. _`Object Space Proxies`: ../objspace-proxies.html
 .. _`Standard Object Space`: ../objspace.html#standard-object-space
 .. _thunk: ../objspace-proxies.html#thunk
-.. _taint: ../objspace-proxies.html#taint
 .. _dump: ../objspace-proxies.html#dump
diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst
--- a/pypy/doc/index.rst
+++ b/pypy/doc/index.rst
@@ -309,7 +309,6 @@
 .. _`object space`: objspace.html
 .. _FlowObjSpace: objspace.html#the-flow-object-space 
 .. _`trace object space`: objspace.html#the-trace-object-space 
-.. _`taint object space`: objspace-proxies.html#taint
 .. _`thunk object space`: objspace-proxies.html#thunk
 .. _`transparent proxies`: objspace-proxies.html#tproxy
 .. _`Differences between PyPy and CPython`: cpython_differences.html
diff --git a/pypy/doc/objspace-proxies.rst b/pypy/doc/objspace-proxies.rst
--- a/pypy/doc/objspace-proxies.rst
+++ b/pypy/doc/objspace-proxies.rst
@@ -129,297 +129,6 @@
    function behaves lazily: all calls to it return a thunk object.
 
 
-.. broken right now:
-
-    .. _taint:
-
-    The Taint Object Space
-    ======================
-
-    Motivation
-    ----------
-
-    The Taint Object Space provides a form of security: "tainted objects",
-    inspired by various sources, see [D12.1]_ for a more detailed discussion. 
-
-    The basic idea of this kind of security is not to protect against
-    malicious code but to help with handling and boxing sensitive data. 
-    It covers two kinds of sensitive data: secret data which should not leak, 
-    and untrusted data coming from an external source and that must be 
-    validated before it is used.
-
-    The idea is that, considering a large application that handles these
-    kinds of sensitive data, there are typically only a small number of
-    places that need to explicitly manipulate that sensitive data; all the
-    other places merely pass it around, or do entirely unrelated things.
-
-    Nevertheless, if a large application needs to be reviewed for security,
-    it must be entirely carefully checked, because it is possible that a
-    bug at some apparently unrelated place could lead to a leak of sensitive
-    information in a way that an external attacker could exploit.  For
-    example, if any part of the application provides web services, an
-    attacker might be able to issue unexpected requests with a regular web
-    browser and deduce secret information from the details of the answers he
-    gets.  Another example is the common CGI attack where an attacker sends
-    malformed inputs and causes the CGI script to do unintended things.
-
-    An approach like that of the Taint Object Space allows the small parts
-    of the program that manipulate sensitive data to be explicitly marked.
-    The effect of this is that although these small parts still need a
-    careful security review, the rest of the application no longer does,
-    because even a bug would be unable to leak the information.
-
-    We have implemented a simple two-level model: objects are either
-    regular (untainted), or sensitive (tainted).  Objects are marked as
-    sensitive if they are secret or untrusted, and only declassified at
-    carefully-checked positions (e.g. where the secret data is needed, or
-    after the untrusted data has been fully validated).
-
-    It would be simple to extend the code for more fine-grained scales of
-    secrecy.  For example it is typical in the literature to consider
-    user-specified lattices of secrecy levels, corresponding to multiple
-    "owners" that cannot access data belonging to another "owner" unless
-    explicitly authorized to do so.
-
-    Tainting and untainting
-    -----------------------
-
-    Start a py.py with the Taint Object Space and try the following example::
-
-        $ py.py -o taint
-        >>>> from __pypy__ import taint
-        >>>> x = taint(6)
-
-        # x is hidden from now on.  We can pass it around and
-        # even operate on it, but not inspect it.  Taintness
-        # is propagated to operation results.
-
-        >>>> x
-        TaintError
-
-        >>>> if x > 5: y = 2   # see below
-        TaintError
-
-        >>>> y = x + 5         # ok
-        >>>> lst = [x, y]
-        >>>> z = lst.pop()
-        >>>> t = type(z)       # type() works too, tainted answer
-        >>>> t
-        TaintError
-        >>>> u = t is int      # even 'is' works
-        >>>> u
-        TaintError
-
-    Notice that using a tainted boolean like ``x > 5`` in an ``if``
-    statement is forbidden.  This is because knowing which path is followed
-    would give away a hint about ``x``; in the example above, if the
-    statement ``if x > 5: y = 2`` was allowed to run, we would know
-    something about the value of ``x`` by looking at the (untainted) value
-    in the variable ``y``.
-
-    Of course, there is a way to inspect tainted objects.  The basic way is
-    to explicitly "declassify" it with the ``untaint()`` function.  In an
-    application, the places that use ``untaint()`` are the places that need
-    careful security review.  To avoid unexpected objects showing up, the
-    ``untaint()`` function must be called with the exact type of the object
-    to declassify.  It will raise ``TaintError`` if the type doesn't match::
-
-        >>>> from __pypy__ import taint
-        >>>> untaint(int, x)
-        6
-        >>>> untaint(int, z)
-        11
-        >>>> untaint(bool, x > 5)
-        True
-        >>>> untaint(int, x > 5)
-        TaintError
-
-
-    Taint Bombs
-    -----------
-
-    In this area, a common problem is what to do about failing operations.
-    If an operation raises an exception when manipulating a tainted object,
-    then the very presence of the exception can leak information about the
-    tainted object itself.  Consider::
-
-        >>>> 5 / (x-6)
-
-    By checking if this raises ``ZeroDivisionError`` or not, we would know
-    if ``x`` was equal to 6 or not.  The solution to this problem in the
-    Taint Object Space is to introduce *Taint Bombs*.  They are a kind of
-    tainted object that doesn't contain a real object, but a pending
-    exception.  Taint Bombs are indistinguishable from normal tainted
-    objects to unprivileged code. See::
-
-        >>>> x = taint(6)
-        >>>> i = 5 / (x-6)     # no exception here
-        >>>> j = i + 1         # nor here
-        >>>> k = j + 5         # nor here
-        >>>> untaint(int, k)
-        TaintError
-
-    In the above example, all of ``i``, ``j`` and ``k`` contain a Taint
-    Bomb.  Trying to untaint it raises an exception - a generic
-    ``TaintError``.  What we win is that the exception gives little away,
-    and most importantly it occurs at the point where ``untaint()`` is
-    called, not where the operation failed.  This means that all calls to
-    ``untaint()`` - but not the rest of the code - must be carefully
-    reviewed for what occurs if they receive a Taint Bomb; they might catch
-    the ``TaintError`` and give the user a generic message that something
-    went wrong, if we are reasonably careful that the message or even its
-    presence doesn't give information away.  This might be a
-    problem by itself, but there is no satisfying general solution here:
-    it must be considered on a case-by-case basis.  Again, what the
-    Taint Object Space approach achieves is not solving these problems, but
-    localizing them to well-defined small parts of the application - namely,
-    around calls to ``untaint()``.
-
-    The ``TaintError`` exception deliberately does not include any
-    useful error messages, because they might give information away.
-    Of course, this makes debugging quite a bit harder; a difficult
-    problem to solve properly.  So far we have implemented a way to peek in a Taint
-    Box or Bomb, ``__pypy__._taint_look(x)``, and a "debug mode" that
-    prints the exception as soon as a Bomb is created - both write
-    information to the low-level stderr of the application, where we hope
-    that it is unlikely to be seen by anyone but the application
-    developer.
-
-
-    Taint Atomic functions
-    ----------------------
-
-    Occasionally, a more complicated computation must be performed on a
-    tainted object.  This requires first untainting the object, performing the
-    computations, and then carefully tainting the result again (including
-    hiding all exceptions into Bombs).
-
-    There is a built-in decorator that does this for you::
-
-        >>>> @__pypy__.taint_atomic
-        >>>> def myop(x, y):
-        ....     while x > 0:
-        ....         x -= y
-        ....     return x
-        ....
-        >>>> myop(42, 10)
-        -8
-        >>>> z = myop(taint(42), 10)
-        >>>> z
-        TaintError
-        >>>> untaint(int, z)
-        -8
-
-    The decorator makes a whole function behave like a built-in operation.
-    If no tainted argument is passed in, the function behaves normally.  But
-    if any of the arguments is tainted, it is automatically untainted - so
-    the function body always sees untainted arguments - and the eventual
-    result is tainted again (possibly in a Taint Bomb).
-
-    It is important for the function marked as ``taint_atomic`` to have no
-    visible side effects, as these could cause information leakage.
-    This is currently not enforced, which means that all ``taint_atomic``
-    functions have to be carefully reviewed for security (but not the
-    callers of ``taint_atomic`` functions).
-
-    A possible future extension would be to forbid side-effects on
-    non-tainted objects from all ``taint_atomic`` functions.
-
-    An example of usage: given a tainted object ``passwords_db`` that
-    references a database of passwords, we can write a function
-    that checks if a password is valid as follows::
-
-        @taint_atomic
-        def validate(passwords_db, username, password):
-            assert type(passwords_db) is PasswordDatabase
-            assert type(username) is str
-            assert type(password) is str
-            ...load username entry from passwords_db...
-            return expected_password == password
-
-    It returns a tainted boolean answer, or a Taint Bomb if something
-    went wrong.  A caller can do::
-
-        ok = validate(passwords_db, 'john', '1234')
-        ok = untaint(bool, ok)
-
-    This can give three outcomes: ``True``, ``False``, or a ``TaintError``
-    exception (with no information on it) if anything went wrong.  If even
-    this is considered giving too much information away, the ``False`` case
-    can be made indistinguishable from the ``TaintError`` case (simply by
-    raising an exception in ``validate()`` if the password is wrong).
-
-    In the above example, the security results achieved are the following:
-    as long as ``validate()`` does not leak information, no other part of
-    the code can obtain more information about a passwords database than a
-    Yes/No answer to a precise query.
-
-    A possible extension of the ``taint_atomic`` decorator would be to check
-    the argument types, as ``untaint()`` does, for the same reason: to
-    prevent bugs where a function like ``validate()`` above is accidentally
-    called with the wrong kind of tainted object, which would make it
-    misbehave.  For now, all ``taint_atomic`` functions should be
-    conservative and carefully check all assumptions on their input
-    arguments.
-
-
-    .. _`taint-interface`:
-
-    Interface
-    ---------
-
-    .. _`like a built-in operation`:
-
-    The basic rule of the Tainted Object Space is that it introduces two new
-    kinds of objects, Tainted Boxes and Tainted Bombs (which are not types
-    in the Python sense).  Each box internally contains a regular object;
-    each bomb internally contains an exception object.  An operation
-    involving Tainted Boxes is performed on the objects contained in the
-    boxes, and gives a Tainted Box or a Tainted Bomb as a result (such an
-    operation does not let an exception be raised).  An operation called
-    with a Tainted Bomb argument immediately returns the same Tainted Bomb.
-
-    In a PyPy running with (or translated with) the Taint Object Space,
-    the ``__pypy__`` module exposes the following interface:
-
-    * ``taint(obj)``
-
-        Return a new Tainted Box wrapping ``obj``.  Return ``obj`` itself
-        if it is already tainted (a Box or a Bomb).
-
-    * ``is_tainted(obj)``
-
-        Check if ``obj`` is tainted (a Box or a Bomb).
-
-    * ``untaint(type, obj)``
-
-        Untaints ``obj`` if it is tainted.  Raise ``TaintError`` if the type
-        of the untainted object is not exactly ``type``, or if ``obj`` is a
-        Bomb.
-
-    * ``taint_atomic(func)``
-
-        Return a wrapper function around the callable ``func``.  The wrapper
-        behaves `like a built-in operation`_ with respect to untainting the
-        arguments, tainting the result, and returning a Bomb.
-
-    * ``TaintError``
-
-        Exception.  On purpose, it provides no attribute or error message.
-
-    * ``_taint_debug(level)``
-
-        Set the debugging level to ``level`` (0=off).  At level 1 or above,
-        all Taint Bombs print a diagnostic message to stderr when they are
-        created.
-
-    * ``_taint_look(obj)``
-
-        For debugging purposes: prints (to stderr) the type and address of
-        the object in a Tainted Box, or prints the exception if ``obj`` is
-        a Taint Bomb.
-
-
 .. _dump:
 
 The Dump Object Space
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -175,6 +175,9 @@
             self.w_tracefunc = w_func
             self.space.frame_trace_action.fire()
 
+    def gettrace(self):
+        return self.w_tracefunc
+
     def setprofile(self, w_func):
         """Set the global trace function."""
         if self.space.is_w(w_func, self.space.w_None):
@@ -307,7 +310,11 @@
         self._nonperiodic_actions = []
         self.has_bytecode_counter = False
         self.fired_actions = None
-        self.checkinterval_scaled = 100 * TICK_COUNTER_STEP
+        # the default value is not 100, unlike CPython 2.7, but a much
+        # larger value, because we use a technique that not only allows
+        # but actually *forces* another thread to run whenever the counter
+        # reaches zero.
+        self.checkinterval_scaled = 10000 * TICK_COUNTER_STEP
         self._rebuild_action_dispatcher()
 
     def fire(self, action):
@@ -346,6 +353,7 @@
         elif interval > MAX:
             interval = MAX
         self.checkinterval_scaled = interval * TICK_COUNTER_STEP
+        self.reset_ticker(-1)
 
     def _rebuild_action_dispatcher(self):
         periodic_actions = unrolling_iterable(self._periodic_actions)
@@ -383,8 +391,11 @@
     def decrement_ticker(self, by):
         value = self._ticker
         if self.has_bytecode_counter:    # this 'if' is constant-folded
-            value -= by
-            self._ticker = value
+            if jit.isconstant(by) and by == 0:
+                pass     # normally constant-folded too
+            else:
+                value -= by
+                self._ticker = value
         return value
 
 
diff --git a/pypy/interpreter/pyparser/pytokenizer.py b/pypy/interpreter/pyparser/pytokenizer.py
--- a/pypy/interpreter/pyparser/pytokenizer.py
+++ b/pypy/interpreter/pyparser/pytokenizer.py
@@ -226,7 +226,7 @@
                         parenlev = parenlev - 1
                         if parenlev < 0:
                             raise TokenError("unmatched '%s'" % initial, line,
-                                             lnum-1, 0, token_list)
+                                             lnum, start + 1, token_list)
                     if token in python_opmap:
                         punct = python_opmap[token]
                     else:
diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py
--- a/pypy/interpreter/pyparser/test/test_pyparse.py
+++ b/pypy/interpreter/pyparser/test/test_pyparse.py
@@ -87,6 +87,10 @@
         assert exc.lineno == 1
         assert exc.offset == 5
         assert exc.lastlineno == 5
+        exc = py.test.raises(SyntaxError, parse, "abc)").value
+        assert exc.msg == "unmatched ')'"
+        assert exc.lineno == 1
+        assert exc.offset == 4
 
     def test_is(self):
         self.parse("x is y")
diff --git a/pypy/interpreter/test/test_executioncontext.py b/pypy/interpreter/test/test_executioncontext.py
--- a/pypy/interpreter/test/test_executioncontext.py
+++ b/pypy/interpreter/test/test_executioncontext.py
@@ -42,6 +42,7 @@
         assert i == 9
 
     def test_periodic_action(self):
+        from pypy.interpreter.executioncontext import ActionFlag
 
         class DemoAction(executioncontext.PeriodicAsyncAction):
             counter = 0
@@ -53,17 +54,20 @@
 
         space = self.space
         a2 = DemoAction(space)
-        space.actionflag.register_periodic_action(a2, True)
         try:
-            for i in range(500):
-                space.appexec([], """():
-                    n = 5
-                    return n + 2
-                """)
-        except Finished:
-            pass
-        checkinterval = space.actionflag.getcheckinterval()
-        assert checkinterval / 10 < i < checkinterval * 1.1
+            space.actionflag.setcheckinterval(100)
+            space.actionflag.register_periodic_action(a2, True)
+            try:
+                for i in range(500):
+                    space.appexec([], """():
+                        n = 5
+                        return n + 2
+                    """)
+            except Finished:
+                pass
+        finally:
+            space.actionflag = ActionFlag()   # reset to default
+        assert 10 < i < 110
 
     def test_llprofile(self):
         l = []
diff --git a/pypy/jit/backend/llgraph/llimpl.py b/pypy/jit/backend/llgraph/llimpl.py
--- a/pypy/jit/backend/llgraph/llimpl.py
+++ b/pypy/jit/backend/llgraph/llimpl.py
@@ -165,6 +165,7 @@
     'unicodegetitem'  : (('ref', 'int'), 'int'),
     'unicodesetitem'  : (('ref', 'int', 'int'), 'int'),
     'cast_ptr_to_int' : (('ref',), 'int'),
+    'cast_int_to_ptr' : (('int',), 'ref'),
     'debug_merge_point': (('ref', 'int'), None),
     'force_token'     : ((), 'int'),
     'call_may_force'  : (('int', 'varargs'), 'intorptr'),
@@ -875,9 +876,6 @@
     def op_new_array(self, arraydescr, count):
         return do_new_array(arraydescr.ofs, count)
 
-    def op_cast_ptr_to_int(self, descr, ptr):
-        return cast_to_int(ptr)
-
     def op_force_token(self, descr):
         opaque_frame = _to_opaque(self)
         return llmemory.cast_ptr_to_adr(opaque_frame)
diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py
--- a/pypy/jit/backend/llsupport/gc.py
+++ b/pypy/jit/backend/llsupport/gc.py
@@ -45,6 +45,14 @@
     def freeing_block(self, start, stop):
         pass
 
+    def record_constptrs(self, op, gcrefs_output_list):
+        for i in range(op.numargs()):
+            v = op.getarg(i)
+            if isinstance(v, ConstPtr) and bool(v.value):
+                p = v.value
+                rgc._make_sure_does_not_move(p)
+                gcrefs_output_list.append(p)
+
 # ____________________________________________________________
 
 class GcLLDescr_boehm(GcLLDescription):
@@ -141,6 +149,14 @@
     get_funcptr_for_newstr = None
     get_funcptr_for_newunicode = None
 
+    def rewrite_assembler(self, cpu, operations, gcrefs_output_list):
+        # record all GCREFs too, because Boehm cannot see them and keep them
+        # alive if they end up as constants in the assembler
+        for op in operations:
+            self.record_constptrs(op, gcrefs_output_list)
+        return GcLLDescription.rewrite_assembler(self, cpu, operations,
+                                                 gcrefs_output_list)
+
 
 # ____________________________________________________________
 # All code below is for the hybrid or minimark GC
@@ -757,14 +773,6 @@
             funcptr(llmemory.cast_ptr_to_adr(gcref_struct),
                     llmemory.cast_ptr_to_adr(gcref_newptr))
 
-    def record_constptrs(self, op, gcrefs_output_list):
-        for i in range(op.numargs()):
-            v = op.getarg(i)
-            if isinstance(v, ConstPtr) and bool(v.value):
-                p = v.value
-                rgc._make_sure_does_not_move(p)
-                gcrefs_output_list.append(p)
-
     def rewrite_assembler(self, cpu, operations, gcrefs_output_list):
         # Perform two kinds of rewrites in parallel:
         #
diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -1468,20 +1468,16 @@
         return u''.join(u.chars)
 
 
-    def test_casts(self):
-        py.test.skip("xxx fix or kill")
-        from pypy.rpython.lltypesystem import lltype, llmemory
-        TP = lltype.GcStruct('x')
-        x = lltype.malloc(TP)        
-        x = lltype.cast_opaque_ptr(llmemory.GCREF, x)
+    def test_cast_int_to_ptr(self):
+        res = self.execute_operation(rop.CAST_INT_TO_PTR,
+                                     [BoxInt(-17)],  'ref').value
+        assert lltype.cast_ptr_to_int(res) == -17
+
+    def test_cast_ptr_to_int(self):
+        x = lltype.cast_int_to_ptr(llmemory.GCREF, -19)
         res = self.execute_operation(rop.CAST_PTR_TO_INT,
-                                     [BoxPtr(x)],  'int').value
-        expected = self.cpu.cast_adr_to_int(llmemory.cast_ptr_to_adr(x))
-        assert rffi.get_real_int(res) == rffi.get_real_int(expected)
-        res = self.execute_operation(rop.CAST_PTR_TO_INT,
-                                     [ConstPtr(x)],  'int').value
-        expected = self.cpu.cast_adr_to_int(llmemory.cast_ptr_to_adr(x))
-        assert rffi.get_real_int(res) == rffi.get_real_int(expected)
+                                     [BoxPtr(x)], 'int').value
+        assert res == -19
 
     def test_ooops_non_gc(self):
         x = lltype.malloc(lltype.Struct('x'), flavor='raw')
@@ -2299,13 +2295,6 @@
         #
         cpu.bh_strsetitem(x, 4, ord('/'))
         assert str.chars[4] == '/'
-        #
-##        x = cpu.bh_newstr(5)
-##        y = cpu.bh_cast_ptr_to_int(x)
-##        z = cpu.bh_cast_ptr_to_int(x)
-##        y = rffi.get_real_int(y)
-##        z = rffi.get_real_int(z)
-##        assert type(y) == type(z) == int and y == z
 
     def test_sorting_of_fields(self):
         S = self.S
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1387,7 +1387,8 @@
 
     def genop_same_as(self, op, arglocs, resloc):
         self.mov(arglocs[0], resloc)
-    #genop_cast_ptr_to_int = genop_same_as
+    genop_cast_ptr_to_int = genop_same_as
+    genop_cast_int_to_ptr = genop_same_as
 
     def genop_int_mod(self, op, arglocs, resloc):
         if IS_X86_32:
diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py
--- a/pypy/jit/backend/x86/regalloc.py
+++ b/pypy/jit/backend/x86/regalloc.py
@@ -1152,7 +1152,8 @@
         self.possibly_free_var(op.getarg(0))
         resloc = self.force_allocate_reg(op.result)
         self.Perform(op, [argloc], resloc)
-    #consider_cast_ptr_to_int = consider_same_as
+    consider_cast_ptr_to_int = consider_same_as
+    consider_cast_int_to_ptr = consider_same_as
 
     def consider_strlen(self, op):
         args = op.getarglist()
diff --git a/pypy/jit/backend/x86/tool/viewcode.py b/pypy/jit/backend/x86/tool/viewcode.py
--- a/pypy/jit/backend/x86/tool/viewcode.py
+++ b/pypy/jit/backend/x86/tool/viewcode.py
@@ -9,7 +9,12 @@
 """
 
 import autopath
-import operator, sys, os, re, py, new
+import new
+import operator
+import py
+import re
+import sys
+import subprocess
 from bisect import bisect_left
 
 # don't use pypy.tool.udir here to avoid removing old usessions which
@@ -44,14 +49,16 @@
     f = open(tmpfile, 'wb')
     f.write(data)
     f.close()
-    g = os.popen(objdump % {
+    p = subprocess.Popen(objdump % {
         'file': tmpfile,
         'origin': originaddr,
         'backend': objdump_backend_option[backend_name],
-    }, 'r')
-    result = g.readlines()
-    g.close()
-    lines = result[6:]   # drop some objdump cruft
+    }, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    stdout, stderr = p.communicate()
+    assert not p.returncode, ('Encountered an error running objdump: %s' %
+                              stderr)
+    # drop some objdump cruft
+    lines = stdout.splitlines()[6:]
     return format_code_dump_with_labels(originaddr, lines, label_list)
 
 def format_code_dump_with_labels(originaddr, lines, label_list):
@@ -85,8 +92,12 @@
     #
     print 'loading symbols from %s...' % (filename,)
     symbols = {}
-    g = os.popen(symbollister % filename, "r")
-    for line in g:
+    p = subprocess.Popen(symbollister % filename, shell=True,
+                         stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    stdout, stderr = p.communicate()
+    assert not p.returncode, ('Encountered an error running nm: %s' %
+                              stderr)
+    for line in stdout.splitlines():
         match = re_symbolentry.match(line)
         if match:
             addr = long(match.group(1), 16)
@@ -94,7 +105,6 @@
             if name.startswith('pypy_g_'):
                 name = '\xb7' + name[7:]
             symbols[addr] = name
-    g.close()
     print '%d symbols found' % (len(symbols),)
     return symbols
 
diff --git a/pypy/jit/codewriter/effectinfo.py b/pypy/jit/codewriter/effectinfo.py
--- a/pypy/jit/codewriter/effectinfo.py
+++ b/pypy/jit/codewriter/effectinfo.py
@@ -74,6 +74,7 @@
     OS_LLONG_UGE                = 91
     OS_LLONG_URSHIFT            = 92
     OS_LLONG_FROM_UINT          = 93
+    OS_LLONG_U_TO_FLOAT         = 94
     #
     OS_MATH_SQRT                = 100
 
diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -440,6 +440,7 @@
     rewrite_op_ullong_mod_zer      = _do_builtin_call
     rewrite_op_gc_identityhash = _do_builtin_call
     rewrite_op_gc_id           = _do_builtin_call
+    rewrite_op_uint_mod        = _do_builtin_call
 
     # ----------
     # getfield/setfield/mallocs etc.
@@ -454,6 +455,23 @@
             # the special return value None forces op.result to be considered
             # equal to op.args[0]
             return [op0, op1, None]
+        if (hints.get('promote_string') and
+            op.args[0].concretetype is not lltype.Void):
+            S = lltype.Ptr(rstr.STR)
+            assert op.args[0].concretetype == S
+            self._register_extra_helper(EffectInfo.OS_STREQ_NONNULL,
+                                        "str.eq_nonnull",
+                                        [S, S],
+                                        lltype.Signed,
+                                        EffectInfo.EF_ELIDABLE_CANNOT_RAISE)
+            descr, p = self.callcontrol.callinfocollection.callinfo_for_oopspec(
+                EffectInfo.OS_STREQ_NONNULL)
+            # XXX this is fairly ugly way of creating a constant,
+            #     however, callinfocollection has no better interface
+            c = Constant(p.adr.ptr, lltype.typeOf(p.adr.ptr))
+            op1 = SpaceOperation('str_guard_value', [op.args[0], c, descr],
+                                 op.result)
+            return [SpaceOperation('-live-', [], None), op1, None]
         else:
             log.WARNING('ignoring hint %r at %r' % (hints, self.graph))
 
@@ -782,8 +800,7 @@
 
     def rewrite_op_cast_ptr_to_int(self, op):
         if self._is_gc(op.args[0]):
-            #return op
-            raise NotImplementedError("cast_ptr_to_int")
+            return op
 
     def rewrite_op_force_cast(self, op):
         v_arg = op.args[0]
@@ -883,9 +900,15 @@
                 v = v_arg
                 oplist = []
             if unsigned1:
-                opname = 'cast_uint_to_longlong'
+                if unsigned2:
+                    opname = 'cast_uint_to_ulonglong'
+                else:
+                    opname = 'cast_uint_to_longlong'
             else:
-                opname = 'cast_int_to_longlong'
+                if unsigned2:
+                    opname = 'cast_int_to_ulonglong'
+                else:
+                    opname = 'cast_int_to_longlong'
             op2 = self.rewrite_operation(
                 SpaceOperation(opname, [v], v_result)
             )
@@ -995,6 +1018,21 @@
                 return op2
         ''' % (_op, _oopspec.lower(), _oopspec, _oopspec)).compile()
 
+    for _op, _oopspec in [('cast_int_to_ulonglong',     'FROM_INT'),
+                          ('cast_uint_to_ulonglong',    'FROM_UINT'),
+                          ('cast_float_to_ulonglong',   'FROM_FLOAT'),
+                          ('cast_ulonglong_to_float',   'U_TO_FLOAT'),
+                         ]:
+        exec py.code.Source('''
+            def rewrite_op_%s(self, op):
+                args = op.args
+                op1 = self.prepare_builtin_call(op, "ullong_%s", args)
+                op2 = self._handle_oopspec_call(op1, args,
+                                                EffectInfo.OS_LLONG_%s,
+                                           EffectInfo.EF_ELIDABLE_CANNOT_RAISE)
+                return op2
+        ''' % (_op, _oopspec.lower(), _oopspec)).compile()
+
     def _normalize(self, oplist):
         if isinstance(oplist, SpaceOperation):
             return [oplist]
@@ -1504,6 +1542,10 @@
     def rewrite_op_jit_force_virtual(self, op):
         return self._do_builtin_call(op)
 
+    def rewrite_op_jit_is_virtual(self, op):
+        raise Exception, (
+            "'vref.virtual' should not be used from jit-visible code")
+
     def rewrite_op_jit_force_virtualizable(self, op):
         # this one is for virtualizables
         vinfo = self.get_vinfo(op.args[0])
diff --git a/pypy/jit/codewriter/support.py b/pypy/jit/codewriter/support.py
--- a/pypy/jit/codewriter/support.py
+++ b/pypy/jit/codewriter/support.py
@@ -315,18 +315,30 @@
 def _ll_1_llong_from_int(x):
     return r_longlong(intmask(x))
 
+def _ll_1_ullong_from_int(x):
+    return r_ulonglong(intmask(x))
+
 def _ll_1_llong_from_uint(x):
     return r_longlong(r_uint(x))
 
+def _ll_1_ullong_from_uint(x):
+    return r_ulonglong(r_uint(x))
+
 def _ll_1_llong_to_int(xll):
     return intmask(xll)
 
 def _ll_1_llong_from_float(xf):
     return r_longlong(xf)
 
+def _ll_1_ullong_from_float(xf):
+    return r_ulonglong(xf)
+
 def _ll_1_llong_to_float(xll):
     return float(rffi.cast(lltype.SignedLongLong, xll))
 
+def _ll_1_ullong_u_to_float(xull):
+    return float(rffi.cast(lltype.UnsignedLongLong, xull))
+
 
 def _ll_1_llong_abs(xll):
     if xll < 0:
@@ -351,20 +363,23 @@
     return llop.llong_mod(lltype.SignedLongLong, xll, yll)
 
 def _ll_2_ullong_floordiv(xll, yll):
-    return llop.ullong_floordiv(lltype.SignedLongLong, xll, yll)
+    return llop.ullong_floordiv(lltype.UnsignedLongLong, xll, yll)
 
 def _ll_2_ullong_floordiv_zer(xll, yll):
     if yll == 0:
         raise ZeroDivisionError
-    return llop.ullong_floordiv(lltype.SignedLongLong, xll, yll)
+    return llop.ullong_floordiv(lltype.UnsignedLongLong, xll, yll)
 
 def _ll_2_ullong_mod(xll, yll):
-    return llop.ullong_mod(lltype.SignedLongLong, xll, yll)
+    return llop.ullong_mod(lltype.UnsignedLongLong, xll, yll)
 
 def _ll_2_ullong_mod_zer(xll, yll):
     if yll == 0:
         raise ZeroDivisionError
-    return llop.ullong_mod(lltype.SignedLongLong, xll, yll)
+    return llop.ullong_mod(lltype.UnsignedLongLong, xll, yll)
+
+def _ll_2_uint_mod(xll, yll):
+    return llop.uint_mod(lltype.Unsigned, xll, yll)
 
 
 # libffi support
diff --git a/pypy/jit/codewriter/test/test_flatten.py b/pypy/jit/codewriter/test/test_flatten.py
--- a/pypy/jit/codewriter/test/test_flatten.py
+++ b/pypy/jit/codewriter/test/test_flatten.py
@@ -829,14 +829,15 @@
                     self.encoding_test(f, [rffi.cast(FROM, 42)], expectedstr,
                                        transform=True)
                 elif TO in (rffi.LONG, rffi.ULONG):
+                    if rffi.cast(FROM, -1) < 0:
+                        fnname = "llong_from_int"
+                    else:
+                        fnname = "llong_from_uint"
                     if TO == rffi.LONG:
                         TO = rffi.LONGLONG
                     else:
                         TO = rffi.ULONGLONG
-                    if rffi.cast(FROM, -1) < 0:
-                        fnname = "llong_from_int"
-                    else:
-                        fnname = "llong_from_uint"
+                        fnname = "u" + fnname
                     expected.pop()   # remove int_return
                     expected.append(
                         "residual_call_irf_f $<* fn %s>, <Descr>, I[%s], R[], F[] -> %%f0"
diff --git a/pypy/jit/codewriter/test/test_jtransform.py b/pypy/jit/codewriter/test/test_jtransform.py
--- a/pypy/jit/codewriter/test/test_jtransform.py
+++ b/pypy/jit/codewriter/test/test_jtransform.py
@@ -1,5 +1,19 @@
 import py
 import random
+try:
+    from itertools import product
+except ImportError:
+    # Python 2.5, this is taken from the CPython docs, but simplified.
+    def product(*args):
+        # product('ABCD', 'xy') --> Ax Ay Bx By Cx Cy Dx Dy
+        # product(range(2), repeat=3) --> 000 001 010 011 100 101 110 111
+        pools = map(tuple, args)
+        result = [[]]
+        for pool in pools:
+            result = [x+[y] for x in result for y in pool]
+        for prod in result:
+            yield tuple(prod)
+
 from pypy.objspace.flow.model import FunctionGraph, Block, Link
 from pypy.objspace.flow.model import SpaceOperation, Variable, Constant
 from pypy.jit.codewriter.jtransform import Transformer
@@ -85,6 +99,12 @@
             if i == oopspecindex:
                 return True
         return False
+    def callinfo_for_oopspec(self, oopspecindex):
+        assert oopspecindex == effectinfo.EffectInfo.OS_STREQ_NONNULL
+        class c:
+            class adr:
+                ptr = 1
+        return ('calldescr', c)
 
 class FakeBuiltinCallControl:
     def __init__(self):
@@ -105,6 +125,7 @@
              EI.OS_STR2UNICODE:([PSTR], PUNICODE),
              EI.OS_STR_CONCAT: ([PSTR, PSTR], PSTR),
              EI.OS_STR_SLICE:  ([PSTR, INT, INT], PSTR),
+             EI.OS_STREQ_NONNULL:  ([PSTR, PSTR], INT),
              EI.OS_UNI_CONCAT: ([PUNICODE, PUNICODE], PUNICODE),
              EI.OS_UNI_SLICE:  ([PUNICODE, INT, INT], PUNICODE),
              EI.OS_UNI_EQUAL:  ([PUNICODE, PUNICODE], lltype.Bool),
@@ -254,26 +275,35 @@
             assert op1.result is None
 
 def test_calls():
-    for RESTYPE in [lltype.Signed, rclass.OBJECTPTR,
-                    lltype.Float, lltype.Void]:
-      for with_void in [False, True]:
-        for with_i in [False, True]:
-          for with_r in [False, True]:
-            for with_f in [False, True]:
-              ARGS = []
-              if with_void: ARGS += [lltype.Void, lltype.Void]
-              if with_i: ARGS += [lltype.Signed, lltype.Char]
-              if with_r: ARGS += [rclass.OBJECTPTR, lltype.Ptr(rstr.STR)]
-              if with_f: ARGS += [lltype.Float, lltype.Float]
-              random.shuffle(ARGS)
-              if RESTYPE == lltype.Float: with_f = True
-              if with_f: expectedkind = 'irf'   # all kinds
-              elif with_i: expectedkind = 'ir'  # integers and references
-              else: expectedkind = 'r'          # only references
-              yield residual_call_test, ARGS, RESTYPE, expectedkind
-              yield direct_call_test, ARGS, RESTYPE, expectedkind
-              yield indirect_residual_call_test, ARGS, RESTYPE, expectedkind
-              yield indirect_regular_call_test, ARGS, RESTYPE, expectedkind
+    for RESTYPE, with_void, with_i, with_r, with_f in product(
+        [lltype.Signed, rclass.OBJECTPTR, lltype.Float, lltype.Void],
+        [False, True],
+        [False, True],
+        [False, True],
+        [False, True],
+    ):
+        ARGS = []
+        if with_void:
+            ARGS += [lltype.Void, lltype.Void]
+        if with_i:
+            ARGS += [lltype.Signed, lltype.Char]
+        if with_r:
+            ARGS += [rclass.OBJECTPTR, lltype.Ptr(rstr.STR)]
+        if with_f:
+            ARGS += [lltype.Float, lltype.Float]
+        random.shuffle(ARGS)
+        if RESTYPE == lltype.Float:
+            with_f = True
+        if with_f:
+            expectedkind = 'irf'   # all kinds
+        elif with_i:
+            expectedkind = 'ir'  # integers and references
+        else:
+            expectedkind = 'r'          # only references
+        yield residual_call_test, ARGS, RESTYPE, expectedkind
+        yield direct_call_test, ARGS, RESTYPE, expectedkind
+        yield indirect_residual_call_test, ARGS, RESTYPE, expectedkind
+        yield indirect_regular_call_test, ARGS, RESTYPE, expectedkind
 
 def get_direct_call_op(argtypes, restype):
     FUNC = lltype.FuncType(argtypes, restype)
@@ -821,6 +851,21 @@
     assert op1.args[2] == ListOfKind('ref', [v1, v2])
     assert op1.result == v3
 
+def test_str_promote():
+    PSTR = lltype.Ptr(rstr.STR)
+    v1 = varoftype(PSTR)
+    v2 = varoftype(PSTR)
+    op = SpaceOperation('hint',
+                        [v1, Constant({'promote_string': True}, lltype.Void)],
+                        v2)
+    tr = Transformer(FakeCPU(), FakeBuiltinCallControl())
+    op0, op1, _ = tr.rewrite_operation(op)
+    assert op1.opname == 'str_guard_value'
+    assert op1.args[0] == v1
+    assert op1.args[2] == 'calldescr'
+    assert op1.result == v2
+    assert op0.opname == '-live-'
+
 def test_unicode_concat():
     # test that the oopspec is present and correctly transformed
     PSTR = lltype.Ptr(rstr.UNICODE)
diff --git a/pypy/jit/codewriter/test/test_longlong.py b/pypy/jit/codewriter/test/test_longlong.py
--- a/pypy/jit/codewriter/test/test_longlong.py
+++ b/pypy/jit/codewriter/test/test_longlong.py
@@ -57,7 +57,8 @@
             assert op1.opname == 'residual_call_irf_f'
         else:
             assert op1.opname == 'residual_call_irf_i'
-        gotindex = getattr(EffectInfo, 'OS_' + op1.args[0].value.upper())
+        gotindex = getattr(EffectInfo,
+                           'OS_' + op1.args[0].value.upper().lstrip('U'))
         assert gotindex == oopspecindex
         assert op1.args[1] == 'calldescr-%d' % oopspecindex
         assert list(op1.args[2]) == [v for v in vlist
@@ -192,8 +193,12 @@
                       [lltype.SignedLongLong], lltype.Signed)
         self.do_check('cast_float_to_longlong', EffectInfo.OS_LLONG_FROM_FLOAT,
                       [lltype.Float], lltype.SignedLongLong)
+        self.do_check('cast_float_to_ulonglong', EffectInfo.OS_LLONG_FROM_FLOAT,
+                      [lltype.Float], lltype.UnsignedLongLong)
         self.do_check('cast_longlong_to_float', EffectInfo.OS_LLONG_TO_FLOAT,
                       [lltype.SignedLongLong], lltype.Float)
+        self.do_check('cast_ulonglong_to_float', EffectInfo.OS_LLONG_U_TO_FLOAT,
+                      [lltype.UnsignedLongLong], lltype.Float)
         for T1 in [lltype.SignedLongLong, lltype.UnsignedLongLong]:
             for T2 in [lltype.Signed, lltype.Unsigned]:
                 self.do_check('cast_primitive', EffectInfo.OS_LLONG_TO_INT,
diff --git a/pypy/jit/metainterp/blackhole.py b/pypy/jit/metainterp/blackhole.py
--- a/pypy/jit/metainterp/blackhole.py
+++ b/pypy/jit/metainterp/blackhole.py
@@ -2,7 +2,7 @@
 from pypy.rlib.rtimer import read_timestamp
 from pypy.rlib.rarithmetic import intmask, LONG_BIT, r_uint, ovfcheck
 from pypy.rlib.objectmodel import we_are_translated
-from pypy.rlib.debug import debug_start, debug_stop
+from pypy.rlib.debug import debug_start, debug_stop, ll_assert
 from pypy.rlib.debug import make_sure_not_resized
 from pypy.rpython.lltypesystem import lltype, llmemory, rclass
 from pypy.rpython.lltypesystem.lloperation import llop
@@ -503,6 +503,15 @@
     @arguments("r", returns="r")
     def bhimpl_cast_opaque_ptr(a):
         return a
+    @arguments("r", returns="i")
+    def bhimpl_cast_ptr_to_int(a):
+        i = lltype.cast_ptr_to_int(a)
+        ll_assert((i & 1) == 1, "bhimpl_cast_ptr_to_int: not an odd int")
+        return i
+    @arguments("i", returns="r")
+    def bhimpl_cast_int_to_ptr(i):
+        ll_assert((i & 1) == 1, "bhimpl_cast_int_to_ptr: not an odd int")
+        return lltype.cast_int_to_ptr(llmemory.GCREF, i)
 
     @arguments("i", returns="i")
     def bhimpl_int_copy(a):
@@ -523,6 +532,9 @@
     @arguments("f")
     def bhimpl_float_guard_value(a):
         pass
+    @arguments("r", "i", "d", returns="r")
+    def bhimpl_str_guard_value(a, i, d):
+        return a
 
     @arguments("self", "i")
     def bhimpl_int_push(self, a):
diff --git a/pypy/jit/metainterp/graphpage.py b/pypy/jit/metainterp/graphpage.py
--- a/pypy/jit/metainterp/graphpage.py
+++ b/pypy/jit/metainterp/graphpage.py
@@ -12,8 +12,8 @@
     def get_display_text(self):
         return None
 
-def display_loops(loops, errmsg=None, highlight_loops=()):
-    graphs = [(loop, loop in highlight_loops) for loop in loops]    
+def display_loops(loops, errmsg=None, highlight_loops={}):
+    graphs = [(loop, highlight_loops.get(loop, 0)) for loop in loops]    
     for graph, highlight in graphs:
         for op in graph.get_operations():
             if is_interesting_guard(op):
@@ -65,8 +65,7 @@
     def add_graph(self, graph, highlight=False):
         graphindex = len(self.graphs)
         self.graphs.append(graph)
-        if highlight:
-            self.highlight_graphs[graph] = True
+        self.highlight_graphs[graph] = highlight
         for i, op in enumerate(graph.get_operations()):
             self.all_operations[op] = graphindex, i
 
@@ -126,10 +125,13 @@
             self.dotgen.emit('subgraph cluster%d {' % graphindex)
         label = graph.get_display_text()
         if label is not None:
-            if self.highlight_graphs.get(graph):
-                fillcolor = '#f084c2'
+            colorindex = self.highlight_graphs.get(graph, 0)
+            if colorindex == 1:
+                fillcolor = '#f084c2'    # highlighted graph
+            elif colorindex == 2:
+                fillcolor = '#808080'    # invalidated graph
             else:
-                fillcolor = '#84f0c2'
+                fillcolor = '#84f0c2'    # normal color
             self.dotgen.emit_node(graphname, shape="octagon",
                                   label=label, fillcolor=fillcolor)
             self.pendingedges.append((graphname,
diff --git a/pypy/jit/metainterp/heapcache.py b/pypy/jit/metainterp/heapcache.py
--- a/pypy/jit/metainterp/heapcache.py
+++ b/pypy/jit/metainterp/heapcache.py
@@ -54,9 +54,10 @@
         if box in self.new_boxes:
             self.new_boxes[box] = False
         if box in self.dependencies:
-            for dep in self.dependencies[box]:
+            deps = self.dependencies[box]
+            del self.dependencies[box]
+            for dep in deps:
                 self._escape(dep)
-            del self.dependencies[box]
 
     def clear_caches(self, opnum, descr, argboxes):
         if opnum == rop.SETFIELD_GC:
diff --git a/pypy/jit/metainterp/history.py b/pypy/jit/metainterp/history.py
--- a/pypy/jit/metainterp/history.py
+++ b/pypy/jit/metainterp/history.py
@@ -9,6 +9,7 @@
 
 from pypy.jit.metainterp.resoperation import ResOperation, rop
 from pypy.jit.codewriter import heaptracker, longlong
+from pypy.rlib.objectmodel import compute_identity_hash
 
 # ____________________________________________________________
 
@@ -104,7 +105,7 @@
     getref._annspecialcase_ = 'specialize:arg(1)'
 
     def _get_hash_(self):
-        raise NotImplementedError
+        return compute_identity_hash(self)
 
     def clonebox(self):
         raise NotImplementedError
@@ -133,6 +134,9 @@
     def _get_str(self):
         raise NotImplementedError
 
+    def same_box(self, other):
+        return self is other
+
 class AbstractDescr(AbstractValue):
     __slots__ = ()
 
@@ -241,32 +245,15 @@
     def constbox(self):
         return self
 
+    def same_box(self, other):
+        return self.same_constant(other)
+
     def same_constant(self, other):
         raise NotImplementedError
 
     def __repr__(self):
         return 'Const(%s)' % self._getrepr_()
 
-    def __eq__(self, other):
-        "NOT_RPYTHON"
-        # Remember that you should not compare Consts with '==' in RPython.
-        # Consts have no special __hash__, in order to force different Consts
-        # from being considered as different keys when stored in dicts
-        # (as they always are after translation).  Use a dict_equal_consts()
-        # to get the other behavior (i.e. using this __eq__).
-        if self.__class__ is not other.__class__:
-            return False
-        try:
-            return self.value == other.value
-        except TypeError:
-            if (isinstance(self.value, Symbolic) and
-                isinstance(other.value, Symbolic)):
-                return self.value is other.value
-            raise
-
-    def __ne__(self, other):
-        return not (self == other)
-
 
 class ConstInt(Const):
     type = INT
@@ -688,33 +675,6 @@
 
 # ____________________________________________________________
 
-def dict_equal_consts():
-    "NOT_RPYTHON"
-    # Returns a dict in which Consts that compare as equal
-    # are identified when used as keys.
-    return r_dict(dc_eq, dc_hash)
-
-def dc_eq(c1, c2):
-    return c1 == c2
-
-def dc_hash(c):
-    "NOT_RPYTHON"
-    # This is called during translation only.  Avoid using identityhash(),
-    # to avoid forcing a hash, at least on lltype objects.
-    if not isinstance(c, Const):
-        return hash(c)
-    if isinstance(c.value, Symbolic):
-        return id(c.value)
-    try:
-        if isinstance(c, ConstPtr):
-            p = lltype.normalizeptr(c.value)
-            if p is not None:
-                return hash(p._obj)
-            else:
-                return 0
-        return c._get_hash_()
-    except lltype.DelayedPointer:
-        return -2      # xxx risk of changing hash...
 
 def make_hashable_int(i):
     from pypy.rpython.lltypesystem.ll2ctypes import NotCtypesAllocatedStructure
@@ -772,6 +732,7 @@
     failed_states = None
     retraced_count = 0
     terminating = False # see TerminatingLoopToken in compile.py
+    invalidated = False
     outermost_jitdriver_sd = None
     # and more data specified by the backend when the loop is compiled
     number = -1
@@ -974,6 +935,7 @@
         self.loops = []
         self.locations = []
         self.aborted_keys = []
+        self.invalidated_token_numbers = set()
 
     def set_history(self, history):
         self.operations = history.operations
@@ -1052,7 +1014,12 @@
             if loop in loops:
                 loops.remove(loop)
             loops.append(loop)
-        display_loops(loops, errmsg, extraloops)
+        highlight_loops = dict.fromkeys(extraloops, 1)
+        for loop in loops:
+            if hasattr(loop, '_looptoken_number') and (
+                    loop._looptoken_number in self.invalidated_token_numbers):
+                highlight_loops.setdefault(loop, 2)
+        display_loops(loops, errmsg, highlight_loops)
 
 # ----------------------------------------------------------------
 
diff --git a/pypy/jit/metainterp/memmgr.py b/pypy/jit/metainterp/memmgr.py
--- a/pypy/jit/metainterp/memmgr.py
+++ b/pypy/jit/metainterp/memmgr.py
@@ -68,7 +68,8 @@
         debug_print("Loop tokens before:", oldtotal)
         max_generation = self.current_generation - (self.max_age-1)
         for looptoken in self.alive_loops.keys():
-            if 0 <= looptoken.generation < max_generation:
+            if (0 <= looptoken.generation < max_generation or
+                looptoken.invalidated):
                 del self.alive_loops[looptoken]
         newtotal = len(self.alive_loops)
         debug_print("Loop tokens freed: ", oldtotal - newtotal)
diff --git a/pypy/jit/metainterp/optimizeopt/__init__.py b/pypy/jit/metainterp/optimizeopt/__init__.py
--- a/pypy/jit/metainterp/optimizeopt/__init__.py
+++ b/pypy/jit/metainterp/optimizeopt/__init__.py
@@ -7,6 +7,8 @@
 from pypy.jit.metainterp.optimizeopt.unroll import optimize_unroll, OptInlineShortPreamble
 from pypy.jit.metainterp.optimizeopt.fficall import OptFfiCall
 from pypy.jit.metainterp.optimizeopt.simplify import OptSimplify
+from pypy.jit.metainterp.optimizeopt.pure import OptPure
+from pypy.jit.metainterp.optimizeopt.earlyforce import OptEarlyForce
 from pypy.rlib.jit import PARAMETERS
 from pypy.rlib.unroll import unrolling_iterable
 
@@ -14,6 +16,8 @@
             ('rewrite', OptRewrite),
             ('virtualize', OptVirtualize),
             ('string', OptString),
+            ('earlyforce', OptEarlyForce),
+            ('pure', OptPure),
             ('heap', OptHeap),
             ('ffi', None),
             ('unroll', None)]
diff --git a/pypy/jit/metainterp/optimizeopt/earlyforce.py b/pypy/jit/metainterp/optimizeopt/earlyforce.py
new file mode 100644
--- /dev/null
+++ b/pypy/jit/metainterp/optimizeopt/earlyforce.py
@@ -0,0 +1,24 @@
+from pypy.jit.metainterp.optimizeopt.optimizer import Optimization
+from pypy.jit.metainterp.optimizeopt.vstring import VAbstractStringValue
+from pypy.jit.metainterp.resoperation import rop, ResOperation
+
+class OptEarlyForce(Optimization):
+    def propagate_forward(self, op):
+        opnum = op.getopnum()
+        if (opnum != rop.SETFIELD_GC and 
+            opnum != rop.SETARRAYITEM_GC and
+            opnum != rop.QUASIIMMUT_FIELD):
+               
+            for arg in op.getarglist():
+                if arg in self.optimizer.values:
+                    value = self.getvalue(arg)
+                    value.force_box(self)
+        self.emit_operation(op)
+
+    def new(self):
+        return OptEarlyForce()
+
+    def setup(self):
+        self.optimizer.optearlyforce = self
+
+    
diff --git a/pypy/jit/metainterp/optimizeopt/fficall.py b/pypy/jit/metainterp/optimizeopt/fficall.py
--- a/pypy/jit/metainterp/optimizeopt/fficall.py
+++ b/pypy/jit/metainterp/optimizeopt/fficall.py
@@ -177,10 +177,10 @@
             funcinfo.descr is None):
             return [op] # cannot optimize
         funcsymval = self.getvalue(op.getarg(2))
-        arglist = [funcsymval.force_box()]
+        arglist = [funcsymval.get_key_box()]
         for push_op in funcinfo.opargs:
             argval = self.getvalue(push_op.getarg(2))
-            arglist.append(argval.force_box())
+            arglist.append(argval.get_key_box())
         newop = ResOperation(rop.CALL_RELEASE_GIL, arglist, op.result,
                              descr=funcinfo.descr)
         self.commit_optimization()
diff --git a/pypy/jit/metainterp/optimizeopt/heap.py b/pypy/jit/metainterp/optimizeopt/heap.py
--- a/pypy/jit/metainterp/optimizeopt/heap.py
+++ b/pypy/jit/metainterp/optimizeopt/heap.py
@@ -50,6 +50,7 @@
             if not self._lazy_setfield_registered:
                 optheap._lazy_setfields_and_arrayitems.append(self)
                 self._lazy_setfield_registered = True
+            
         else:
             # this is the case where the pending setfield ends up
             # storing precisely the value that is already there,
@@ -158,12 +159,17 @@
         self._lazy_setfields_and_arrayitems = []
         self._remove_guard_not_invalidated = False
         self._seen_guard_not_invalidated = False
+        self.posponedop = None
 
     def force_at_end_of_preamble(self):
         self.force_all_lazy_setfields_and_arrayitems()
 
     def flush(self):
         self.force_all_lazy_setfields_and_arrayitems()
+        if self.posponedop:
+            posponedop = self.posponedop
+            self.posponedop = None
+            self.next_optimization.propagate_forward(posponedop)
 
     def new(self):
         return OptHeap()
@@ -211,7 +217,15 @@
 
     def emit_operation(self, op):
         self.emitting_operation(op)
-        self.next_optimization.propagate_forward(op)
+        if self.posponedop:
+            posponedop = self.posponedop
+            self.posponedop = None
+            self.next_optimization.propagate_forward(posponedop)
+        if (op.is_comparison() or op.getopnum() == rop.CALL_MAY_FORCE
+            or op.is_ovf()):
+            self.posponedop = op
+        else:
+            self.next_optimization.propagate_forward(op)
 
     def emitting_operation(self, op):
         if op.has_no_side_effect():
@@ -293,30 +307,6 @@
             if indexvalue is None or indexvalue.intbound.contains(idx):
                 cf.force_lazy_setfield(self, can_cache)
 
-    def fixup_guard_situation(self):
-        # hackish: reverse the order of the last two operations if it makes
-        # sense to avoid a situation like "int_eq/setfield_gc/guard_true",
-        # which the backend (at least the x86 backend) does not handle well.
-        newoperations = self.optimizer.newoperations
-        if len(newoperations) < 2:
-            return
-        lastop = newoperations[-1]
-        if (lastop.getopnum() != rop.SETFIELD_GC and
-            lastop.getopnum() != rop.SETARRAYITEM_GC):
-            return
-        # - is_comparison() for cases like "int_eq/setfield_gc/guard_true"
-        # - CALL_MAY_FORCE: "call_may_force/setfield_gc/guard_not_forced"
-        # - is_ovf(): "int_add_ovf/setfield_gc/guard_no_overflow"
-        prevop = newoperations[-2]
-        opnum = prevop.getopnum()
-        if not (prevop.is_comparison() or opnum == rop.CALL_MAY_FORCE
-                or prevop.is_ovf()):
-            return
-        if prevop.result in lastop.getarglist():
-            return
-        newoperations[-2] = lastop
-        newoperations[-1] = prevop
-
     def _assert_valid_cf(self, cf):
         # check that 'cf' is in cached_fields or cached_arrayitems
         if not we_are_translated():
@@ -362,7 +352,6 @@
                                       fieldvalue.get_key_box(), itemindex))
             else:
                 cf.force_lazy_setfield(self)
-                self.fixup_guard_situation()
         return pendingfields
 
     def optimize_GETFIELD_GC(self, op):
@@ -374,12 +363,22 @@
             return
         # default case: produce the operation
         structvalue.ensure_nonnull()
-        ###self.optimizer.optimize_default(op)
         self.emit_operation(op)
         # then remember the result of reading the field
         fieldvalue = self.getvalue(op.result)
         cf.remember_field_value(structvalue, fieldvalue, op)
 
+    def optimize_GETFIELD_GC_PURE(self, op):
+        structvalue = self.getvalue(op.getarg(0))
+        cf = self.field_cache(op.getdescr())
+        fieldvalue = cf.getfield_from_cache(self, structvalue)
+        if fieldvalue is not None:
+            self.make_equal_to(op.result, fieldvalue)
+            return
+        # default case: produce the operation
+        structvalue.ensure_nonnull()
+        self.emit_operation(op)
+
     def optimize_SETFIELD_GC(self, op):
         if self.has_pure_result(rop.GETFIELD_GC_PURE, [op.getarg(0)],
                                 op.getdescr()):
@@ -389,6 +388,7 @@
         #
         cf = self.field_cache(op.getdescr())
         cf.do_setfield(self, op)
+        
 
     def optimize_GETARRAYITEM_GC(self, op):
         arrayvalue = self.getvalue(op.getarg(0))
@@ -413,6 +413,25 @@
             fieldvalue = self.getvalue(op.result)
             cf.remember_field_value(arrayvalue, fieldvalue, op)
 
+    def optimize_GETARRAYITEM_GC_PURE(self, op):
+        arrayvalue = self.getvalue(op.getarg(0))
+        indexvalue = self.getvalue(op.getarg(1))
+        cf = None
+        if indexvalue.is_constant():
+            arrayvalue.make_len_gt(MODE_ARRAY, op.getdescr(), indexvalue.box.getint())
+            # use the cache on (arraydescr, index), which is a constant
+            cf = self.arrayitem_cache(op.getdescr(), indexvalue.box.getint())
+            fieldvalue = cf.getfield_from_cache(self, arrayvalue)
+            if fieldvalue is not None:
+                self.make_equal_to(op.result, fieldvalue)
+                return
+        else:
+            # variable index, so make sure the lazy setarrayitems are done
+            self.force_lazy_setarrayitem(op.getdescr(), indexvalue=indexvalue)
+        # default case: produce the operation
+        arrayvalue.ensure_nonnull()
+        self.emit_operation(op)
+
     def optimize_SETARRAYITEM_GC(self, op):
         if self.has_pure_result(rop.GETARRAYITEM_GC_PURE, [op.getarg(0),
                                                            op.getarg(1)],
diff --git a/pypy/jit/metainterp/optimizeopt/optimizer.py b/pypy/jit/metainterp/optimizeopt/optimizer.py
--- a/pypy/jit/metainterp/optimizeopt/optimizer.py
+++ b/pypy/jit/metainterp/optimizeopt/optimizer.py
@@ -31,8 +31,8 @@
 
 class OptValue(object):
     __metaclass__ = extendabletype
-    _attrs_ = ('box', 'known_class', 'last_guard_index', 'level', 'intbound', 'lenbound')
-    last_guard_index = -1
+    _attrs_ = ('box', 'known_class', 'last_guard', 'level', 'intbound', 'lenbound')
+    last_guard = None
 
     level = LEVEL_UNKNOWN
     known_class = None
@@ -100,7 +100,7 @@
             self.make_constant(other.get_key_box())
             optimizer.turned_constant(self)
         elif other.level == LEVEL_KNOWNCLASS:
-            self.make_constant_class(other.known_class, -1)
+            self.make_constant_class(other.known_class, None)
         else:
             if other.level == LEVEL_NONNULL:
                 self.ensure_nonnull()
@@ -114,13 +114,13 @@
                     self.lenbound = other.lenbound.clone()
 
 
-    def force_box(self):
+    def force_box(self, optforce):
         return self.box
 
     def get_key_box(self):
         return self.box
 
-    def force_at_end_of_preamble(self, already_forced):
+    def force_at_end_of_preamble(self, already_forced, optforce):
         return self
 
     def get_args_for_fail(self, modifier):
@@ -162,16 +162,16 @@
         else:
             return None
 
-    def make_constant_class(self, classbox, opindex):
+    def make_constant_class(self, classbox, guardop):
         assert self.level < LEVEL_KNOWNCLASS
         self.known_class = classbox
         self.level = LEVEL_KNOWNCLASS
-        self.last_guard_index = opindex
+        self.last_guard = guardop
 
-    def make_nonnull(self, opindex):
+    def make_nonnull(self, guardop):
         assert self.level < LEVEL_NONNULL
         self.level = LEVEL_NONNULL
-        self.last_guard_index = opindex
+        self.last_guard = guardop
 
     def is_nonnull(self):
         level = self.level
@@ -240,26 +240,12 @@
     def __init__(self):
         pass # make rpython happy
 
-    def propagate_begin_forward(self):
-        if self.next_optimization:
-            self.next_optimization.propagate_begin_forward()
-
-    def propagate_end_forward(self):
-        if self.next_optimization:
-            self.next_optimization.propagate_end_forward()
-
     def propagate_forward(self, op):
         raise NotImplementedError
 
     def emit_operation(self, op):
         self.next_optimization.propagate_forward(op)
 
-    def test_emittable(self, op):
-        return self.is_emittable(op)
-
-    def is_emittable(self, op):
-        return self.next_optimization.test_emittable(op)
-
     # FIXME: Move some of these here?
     def getvalue(self, box):
         return self.optimizer.getvalue(box)
@@ -289,19 +275,19 @@
         return self.optimizer.new_const_item(arraydescr)
 
     def pure(self, opnum, args, result):
-        op = ResOperation(opnum, args, result)
-        key = self.optimizer.make_args_key(op)
-        if key not in self.optimizer.pure_operations:
-            self.optimizer.pure_operations[key] = op
+        if self.optimizer.optpure:
+            self.optimizer.optpure.pure(opnum, args, result)
 
     def has_pure_result(self, opnum, args, descr):
-        op = ResOperation(opnum, args, None, descr)
-        key = self.optimizer.make_args_key(op)
-        op = self.optimizer.pure_operations.get(key, None)
-        if op is None:
-            return False
-        return op.getdescr() is descr
+        if self.optimizer.optpure:
+            return self.optimizer.optpure.has_pure_result(opnum, args, descr)
+        return False
 
+    def get_pure_result(self, key):    
+        if self.optimizer.optpure:
+            return self.optimizer.optpure.get_pure_result(key)
+        return None
+    
     def setup(self):
         pass
 
@@ -322,6 +308,9 @@
     def produce_potential_short_preamble_ops(self, potential_ops):
         pass
 
+    def forget_numberings(self, box):
+        self.optimizer.forget_numberings(box)
+
 class Optimizer(Optimization):
 
     def __init__(self, metainterp_sd, loop, optimizations=None, bridge=False):
@@ -333,14 +322,16 @@
         self.interned_refs = self.cpu.ts.new_ref_dict()
         self.resumedata_memo = resume.ResumeDataLoopMemo(metainterp_sd)
         self.bool_boxes = {}
-        self.pure_operations = args_dict()
         self.producer = {}
         self.pendingfields = []
-        self.posponedop = None
         self.exception_might_have_happened = False
         self.quasi_immutable_deps = None
         self.opaque_pointers = {}
-        self.newoperations = []
+        self.replaces_guard = {}
+        self._newoperations = []
+        self.optimizer = self
+        self.optpure = None
+        self.optearlyforce = None
         if loop is not None:
             self.call_pure_results = loop.call_pure_results
 
@@ -369,21 +360,20 @@
     def flush(self):
         for o in self.optimizations:
             o.flush()
-        assert self.posponedop is None
 
     def new(self):
         new = Optimizer(self.metainterp_sd, self.loop)
         return self._new(new)
 
     def _new(self, new):
-        assert self.posponedop is None
         optimizations = [o.new() for o in self.optimizations]
         new.set_optimizations(optimizations)
         new.quasi_immutable_deps = self.quasi_immutable_deps
         return new
 
     def produce_potential_short_preamble_ops(self, sb):
-        raise NotImplementedError('This is implemented in unroll.UnrollableOptimizer')
+        for opt in self.optimizations:
+            opt.produce_potential_short_preamble_ops(sb)
 
     def turned_constant(self, value):
         for o in self.optimizations:
@@ -433,6 +423,13 @@
             return constbox
         return None
 
+    def get_newoperations(self):
+        self.flush()
+        return self._newoperations
+
+    def clear_newoperations(self):
+        self._newoperations = []
+
     def make_equal_to(self, box, value, replace=False):
         assert isinstance(value, OptValue)
         assert replace or box not in self.values
@@ -481,15 +478,10 @@
 
     def propagate_all_forward(self):
         self.exception_might_have_happened = self.bridge
-        self.newoperations = []
-        self.first_optimization.propagate_begin_forward()
-        self.i = 0
-        while self.i < len(self.loop.operations):
-            op = self.loop.operations[self.i]
+        self.clear_newoperations()
+        for op in self.loop.operations:
             self.first_optimization.propagate_forward(op)
-            self.i += 1
-        self.first_optimization.propagate_end_forward()
-        self.loop.operations = self.newoperations
+        self.loop.operations = self.get_newoperations()
         self.loop.quasi_immutable_deps = self.quasi_immutable_deps
         # accumulate counters
         self.resumedata_memo.update_counters(self.metainterp_sd.profiler)
@@ -501,9 +493,6 @@
         self.producer[op.result] = op
         dispatch_opt(self, op)
 
-    def test_emittable(self, op):
-        return True
-
     def emit_operation(self, op):
         if op.returns_bool_result():
             self.bool_boxes[self.getvalue(op.result)] = None
@@ -519,14 +508,30 @@
                 pass
             else:
                 self.ensure_imported(value)
-                op.setarg(i, value.force_box())
+                op.setarg(i, value.force_box(self))
         self.metainterp_sd.profiler.count(jitprof.OPT_OPS)
         if op.is_guard():
             self.metainterp_sd.profiler.count(jitprof.OPT_GUARDS)
-            op = self.store_final_boxes_in_guard(op)
+            if self.replaces_guard and op in self.replaces_guard:
+                self.replace_op(self.replaces_guard[op], op)
+                del self.replaces_guard[op]
+                return
+            else:
+                op = self.store_final_boxes_in_guard(op)
         elif op.can_raise():
             self.exception_might_have_happened = True
-        self.newoperations.append(op)
+        self._newoperations.append(op)
+
+    def replace_op(self, old_op, new_op):
+        # XXX: Do we want to cache indexes to prevent search?
+        i = len(self._newoperations) 
+        while i > 0:
+            i -= 1
+            if self._newoperations[i] is old_op:
+                self._newoperations[i] = new_op
+                break
+        else:
+            assert False
 
     def store_final_boxes_in_guard(self, op):
         descr = op.getdescr()
@@ -574,51 +579,8 @@
         args[n + 1] = op.getdescr()
         return args
 
-    @specialize.argtype(0)
     def optimize_default(self, op):
-        canfold = op.is_always_pure()
-        if op.is_ovf():
-            self.posponedop = op
-            return
-        if self.posponedop:
-            nextop = op
-            op = self.posponedop
-            self.posponedop = None
-            canfold = nextop.getopnum() == rop.GUARD_NO_OVERFLOW
-        else:
-            nextop = None
-
-        if canfold:
-            for i in range(op.numargs()):
-                if self.get_constant_box(op.getarg(i)) is None:
-                    break
-            else:
-                # all constant arguments: constant-fold away
-                resbox = self.constant_fold(op)
-                # note that INT_xxx_OVF is not done from here, and the
-                # overflows in the INT_xxx operations are ignored
-                self.make_constant(op.result, resbox)
-                return
-
-            # did we do the exact same operation already?
-            args = self.make_args_key(op)
-            oldop = self.pure_operations.get(args, None)
-            if oldop is not None and oldop.getdescr() is op.getdescr():
-                assert oldop.getopnum() == op.getopnum()
-                self.make_equal_to(op.result, self.getvalue(oldop.result),
-                                   True)
-                return
-            else:
-                self.pure_operations[args] = op
-                self.remember_emitting_pure(op)
-
-        # otherwise, the operation remains
         self.emit_operation(op)
-        if nextop:
-            self.emit_operation(nextop)
-
-    def remember_emitting_pure(self, op):
-        pass
 
     def constant_fold(self, op):
         argboxes = [self.get_constant_box(op.getarg(i))
@@ -636,11 +598,6 @@
     def optimize_DEBUG_MERGE_POINT(self, op):
         self.emit_operation(op)
 
-    def optimize_CAST_OPAQUE_PTR(self, op):
-        value = self.getvalue(op.getarg(0))
-        self.opaque_pointers[value] = True
-        self.make_equal_to(op.result, value)
-
     def optimize_GETARRAYITEM_GC_PURE(self, op):
         indexvalue = self.getvalue(op.getarg(1))
         if indexvalue.is_constant():
diff --git a/pypy/jit/metainterp/optimizeopt/pure.py b/pypy/jit/metainterp/optimizeopt/pure.py
new file mode 100644
--- /dev/null
+++ b/pypy/jit/metainterp/optimizeopt/pure.py
@@ -0,0 +1,115 @@
+from pypy.jit.metainterp.optimizeopt.optimizer import Optimization
+from pypy.jit.metainterp.resoperation import rop, ResOperation
+from pypy.jit.metainterp.optimizeopt.util import (make_dispatcher_method,
+    args_dict)
+
+class OptPure(Optimization):
+    def __init__(self):
+        self.posponedop = None
+        self.pure_operations = args_dict()
+        self.emitted_pure_operations = {}
+
+    def propagate_forward(self, op):
+        dispatch_opt(self, op)
+
+    def optimize_default(self, op):
+        canfold = op.is_always_pure()
+        if op.is_ovf():
+            self.posponedop = op
+            return
+        if self.posponedop:
+            nextop = op
+            op = self.posponedop
+            self.posponedop = None
+            canfold = nextop.getopnum() == rop.GUARD_NO_OVERFLOW
+        else:
+            nextop = None
+
+        if canfold:
+            for i in range(op.numargs()):
+                if self.get_constant_box(op.getarg(i)) is None:
+                    break
+            else:
+                # all constant arguments: constant-fold away
+                resbox = self.optimizer.constant_fold(op)
+                # note that INT_xxx_OVF is not done from here, and the
+                # overflows in the INT_xxx operations are ignored
+                self.optimizer.make_constant(op.result, resbox)
+                return
+
+            # did we do the exact same operation already?
+            args = self.optimizer.make_args_key(op)
+            oldop = self.pure_operations.get(args, None)
+            if oldop is not None and oldop.getdescr() is op.getdescr():
+                assert oldop.getopnum() == op.getopnum()
+                self.optimizer.make_equal_to(op.result, self.getvalue(oldop.result),
+                                   True)
+                return
+            else:
+                self.pure_operations[args] = op
+                self.remember_emitting_pure(op)
+
+        # otherwise, the operation remains
+        self.emit_operation(op)
+        if op.returns_bool_result():
+            self.optimizer.bool_boxes[self.getvalue(op.result)] = None        
+        if nextop:
+            self.emit_operation(nextop)
+
+    def optimize_CALL_PURE(self, op):
+        args = self.optimizer.make_args_key(op)
+        oldop = self.pure_operations.get(args, None)
+        if oldop is not None and oldop.getdescr() is op.getdescr():
+            assert oldop.getopnum() == op.getopnum()
+            self.make_equal_to(op.result, self.getvalue(oldop.result))
+            return
+        else:
+            self.pure_operations[args] = op
+            self.remember_emitting_pure(op)
+
+        # replace CALL_PURE with just CALL
+        args = op.getarglist()
+        self.emit_operation(ResOperation(rop.CALL, args, op.result,
+                                         op.getdescr()))
+
+    def flush(self):
+        assert self.posponedop is None
+
+    def new(self):
+        assert self.posponedop is None
+        return OptPure()
+
+    def setup(self):
+        self.optimizer.optpure = self
+
+    def pure(self, opnum, args, result):
+        op = ResOperation(opnum, args, result)
+        key = self.optimizer.make_args_key(op)
+        if key not in self.pure_operations:
+            self.pure_operations[key] = op
+
+    def has_pure_result(self, opnum, args, descr):
+        op = ResOperation(opnum, args, None, descr)
+        key = self.optimizer.make_args_key(op)
+        op = self.pure_operations.get(key, None)
+        if op is None:
+            return False
+        return op.getdescr() is descr
+
+    def get_pure_result(self, key):
+        return self.pure_operations.get(key, None)
+
+    def remember_emitting_pure(self, op):
+        self.emitted_pure_operations[op] = True
+
+    def produce_potential_short_preamble_ops(self, sb):
+        for op in self.emitted_pure_operations:
+            if op.getopnum() == rop.GETARRAYITEM_GC_PURE or \
+               op.getopnum() == rop.STRGETITEM or \
+               op.getopnum() == rop.UNICODEGETITEM:
+                if not self.getvalue(op.getarg(1)).is_constant():
+                    continue
+            sb.add_potential(op)
+
+dispatch_opt = make_dispatcher_method(OptPure, 'optimize_',
+                                      default=OptPure.optimize_default)
diff --git a/pypy/jit/metainterp/optimizeopt/rewrite.py b/pypy/jit/metainterp/optimizeopt/rewrite.py
--- a/pypy/jit/metainterp/optimizeopt/rewrite.py
+++ b/pypy/jit/metainterp/optimizeopt/rewrite.py
@@ -31,21 +31,8 @@
 
         dispatch_opt(self, op)
 
-    def test_emittable(self, op):
-        opnum = op.getopnum()
-        for value, cls, func in optimize_guards:
-            if opnum == value:
-                assert isinstance(op, cls)
-                try:
-                    func(self, op, dryrun=True)
-                    return self.is_emittable(op)
-                except InvalidLoop:
-                    return False
-        return self.is_emittable(op)
-
-
     def try_boolinvers(self, op, targs):
-        oldop = self.optimizer.pure_operations.get(targs, None)
+        oldop = self.get_pure_result(targs)
         if oldop is not None and oldop.getdescr() is op.getdescr():
             value = self.getvalue(oldop.result)
             if value.is_constant():
@@ -62,32 +49,35 @@
     def find_rewritable_bool(self, op, args):
         try:
             oldopnum = opboolinvers[op.getopnum()]
+        except KeyError:
+            pass
+        else:
             targs = self.optimizer.make_args_key(ResOperation(oldopnum, [args[0], args[1]],
                                                               None))
             if self.try_boolinvers(op, targs):
                 return True
-        except KeyError:
-            pass
 
         try:
             oldopnum = opboolreflex[op.getopnum()] # FIXME: add INT_ADD, INT_MUL
+        except KeyError:
+            pass
+        else:
             targs = self.optimizer.make_args_key(ResOperation(oldopnum, [args[1], args[0]],
                                                               None))
-            oldop = self.optimizer.pure_operations.get(targs, None)
+            oldop = self.get_pure_result(targs)
             if oldop is not None and oldop.getdescr() is op.getdescr():
                 self.make_equal_to(op.result, self.getvalue(oldop.result))
                 return True
-        except KeyError:
-            pass
 
         try:
             oldopnum = opboolinvers[opboolreflex[op.getopnum()]]
+        except KeyError:
+            pass
+        else:
             targs = self.optimizer.make_args_key(ResOperation(oldopnum, [args[1], args[0]],
                                                               None))
             if self.try_boolinvers(op, targs):
                 return True
-        except KeyError:
-            pass
 
         return False
 
@@ -116,10 +106,9 @@
             self.make_equal_to(op.result, v1)
         else:
             self.emit_operation(op)
-
-        # Synthesize the reverse ops for optimize_default to reuse
-        self.pure(rop.INT_ADD, [op.result, op.getarg(1)], op.getarg(0))
-        self.pure(rop.INT_SUB, [op.getarg(0), op.result], op.getarg(1))
+            # Synthesize the reverse ops for optimize_default to reuse
+            self.pure(rop.INT_ADD, [op.result, op.getarg(1)], op.getarg(0))
+            self.pure(rop.INT_SUB, [op.getarg(0), op.result], op.getarg(1))
 
     def optimize_INT_ADD(self, op):
         v1 = self.getvalue(op.getarg(0))
@@ -132,10 +121,9 @@
             self.make_equal_to(op.result, v1)
         else:
             self.emit_operation(op)
-
-        # Synthesize the reverse op for optimize_default to reuse
-        self.pure(rop.INT_SUB, [op.result, op.getarg(1)], op.getarg(0))
-        self.pure(rop.INT_SUB, [op.result, op.getarg(0)], op.getarg(1))
+            # Synthesize the reverse op for optimize_default to reuse
+            self.pure(rop.INT_SUB, [op.result, op.getarg(1)], op.getarg(0))
+            self.pure(rop.INT_SUB, [op.result, op.getarg(0)], op.getarg(1))
 
     def optimize_INT_MUL(self, op):
         v1 = self.getvalue(op.getarg(0))
@@ -151,13 +139,13 @@
             self.make_constant_int(op.result, 0)
         else:
             for lhs, rhs in [(v1, v2), (v2, v1)]:
-                # x & (x -1) == 0 is a quick test for power of 2
-                if (lhs.is_constant() and
-                    (lhs.box.getint() & (lhs.box.getint() - 1)) == 0):
-                    new_rhs = ConstInt(highest_bit(lhs.box.getint()))
-                    op = op.copy_and_change(rop.INT_LSHIFT, args=[rhs.box, new_rhs])
-                    break
-
+                if lhs.is_constant():
+                    x = lhs.box.getint()
+                    # x & (x - 1) == 0 is a quick test for power of 2
+                    if x & (x - 1) == 0:
+                        new_rhs = ConstInt(highest_bit(lhs.box.getint()))
+                        op = op.copy_and_change(rop.INT_LSHIFT, args=[rhs.box, new_rhs])
+                        break
             self.emit_operation(op)
 
     def optimize_UINT_FLOORDIV(self, op):
@@ -214,40 +202,7 @@
         self.emit_operation(op)
         self.pure(rop.FLOAT_NEG, [op.result], v1)
 
-    def optimize_CALL_PURE(self, op):
-        arg_consts = []
-        for i in range(op.numargs()):
-            arg = op.getarg(i)
-            const = self.get_constant_box(arg)
-            if const is None:
-                break
-            arg_consts.append(const)
-        else:
-            # all constant arguments: check if we already know the result
-            try:
-                result = self.optimizer.call_pure_results[arg_consts]
-            except KeyError:
-                pass
-            else:
-                self.make_constant(op.result, result)
-                return
-
-        args = self.optimizer.make_args_key(op)
-        oldop = self.optimizer.pure_operations.get(args, None)
-        if oldop is not None and oldop.getdescr() is op.getdescr():
-            assert oldop.getopnum() == op.getopnum()
-            self.make_equal_to(op.result, self.getvalue(oldop.result))
-            return
-        else:
-            self.optimizer.pure_operations[args] = op
-            self.optimizer.remember_emitting_pure(op)
-
-        # replace CALL_PURE with just CALL
-        args = op.getarglist()
-        self.emit_operation(ResOperation(rop.CALL, args, op.result,
-                                         op.getdescr()))
-
-    def optimize_guard(self, op, constbox, emit_operation=True, dryrun=False):
+    def optimize_guard(self, op, constbox, emit_operation=True):
         value = self.getvalue(op.getarg(0))
         if value.is_constant():
             box = value.box
@@ -255,62 +210,57 @@
             if not box.same_constant(constbox):
                 raise InvalidLoop
             return
-        if dryrun: return
         if emit_operation:
             self.emit_operation(op)
         value.make_constant(constbox)
         self.optimizer.turned_constant(value)
 
-    def optimize_GUARD_ISNULL(self, op, dryrun=False):
+    def optimize_GUARD_ISNULL(self, op):
         value = self.getvalue(op.getarg(0))
         if value.is_null():
             return
         elif value.is_nonnull():
             raise InvalidLoop
-        if dryrun: return
         self.emit_operation(op)
         value.make_constant(self.optimizer.cpu.ts.CONST_NULL)
 
-    def optimize_GUARD_NONNULL(self, op, dryrun=False):
+    def optimize_GUARD_NONNULL(self, op):
         value = self.getvalue(op.getarg(0))
         if value.is_nonnull():
             return
         elif value.is_null():
             raise InvalidLoop
-        if dryrun: return
         self.emit_operation(op)
-        value.make_nonnull(len(self.optimizer.newoperations) - 1)
+        value.make_nonnull(op)
 
-    def optimize_GUARD_VALUE(self, op, dryrun=False):
+    def optimize_GUARD_VALUE(self, op):
         value = self.getvalue(op.getarg(0))
-        emit_operation = True
-        if not dryrun and value.last_guard_index != -1:
+        if value.last_guard:
             # there already has been a guard_nonnull or guard_class or
             # guard_nonnull_class on this value, which is rather silly.
             # replace the original guard with a guard_value
-            old_guard_op = self.optimizer.newoperations[value.last_guard_index]
-            new_guard_op = old_guard_op.copy_and_change(rop.GUARD_VALUE,
-                                             args = [old_guard_op.getarg(0), op.getarg(1)])
-            self.optimizer.newoperations[value.last_guard_index] = new_guard_op
+            old_guard_op = value.last_guard
+            op = old_guard_op.copy_and_change(rop.GUARD_VALUE,
+                                      args = [old_guard_op.getarg(0), op.getarg(1)])
+            self.optimizer.replaces_guard[op] = old_guard_op
             # hack hack hack.  Change the guard_opnum on
             # new_guard_op.getdescr() so that when resuming,
             # the operation is not skipped by pyjitpl.py.
-            descr = new_guard_op.getdescr()
+            descr = op.getdescr()
             assert isinstance(descr, compile.ResumeGuardDescr)
             descr.guard_opnum = rop.GUARD_VALUE
-            descr.make_a_counter_per_value(new_guard_op)
-            emit_operation = False
+            descr.make_a_counter_per_value(op)
         constbox = op.getarg(1)
         assert isinstance(constbox, Const)
-        self.optimize_guard(op, constbox, emit_operation, dryrun)
+        self.optimize_guard(op, constbox)
 
-    def optimize_GUARD_TRUE(self, op, dryrun=False):
-        self.optimize_guard(op, CONST_1, dryrun=dryrun)
+    def optimize_GUARD_TRUE(self, op):
+        self.optimize_guard(op, CONST_1)
 
-    def optimize_GUARD_FALSE(self, op, dryrun=False):
-        self.optimize_guard(op, CONST_0, dryrun=dryrun)
+    def optimize_GUARD_FALSE(self, op):
+        self.optimize_guard(op, CONST_0)
 
-    def optimize_GUARD_CLASS(self, op, dryrun=False):
+    def optimize_GUARD_CLASS(self, op):
         value = self.getvalue(op.getarg(0))
         expectedclassbox = op.getarg(1)
         assert isinstance(expectedclassbox, Const)
@@ -319,38 +269,32 @@
             if realclassbox.same_constant(expectedclassbox):
                 return
             raise InvalidLoop
-        if dryrun: return
-        emit_operation = True
-        if value.last_guard_index != -1:
+        if value.last_guard:
             # there already has been a guard_nonnull or guard_class or
             # guard_nonnull_class on this value.
-            old_guard_op = self.optimizer.newoperations[value.last_guard_index]
+            old_guard_op = value.last_guard
             if old_guard_op.getopnum() == rop.GUARD_NONNULL:
                 # it was a guard_nonnull, which we replace with a
                 # guard_nonnull_class.
-                new_guard_op = old_guard_op.copy_and_change (rop.GUARD_NONNULL_CLASS,
+                op = old_guard_op.copy_and_change (rop.GUARD_NONNULL_CLASS,
                                          args = [old_guard_op.getarg(0), op.getarg(1)])
-                self.optimizer.newoperations[value.last_guard_index] = new_guard_op
+                self.optimizer.replaces_guard[op] = old_guard_op
                 # hack hack hack.  Change the guard_opnum on
                 # new_guard_op.getdescr() so that when resuming,
                 # the operation is not skipped by pyjitpl.py.
-                descr = new_guard_op.getdescr()
+                descr = op.getdescr()
                 assert isinstance(descr, compile.ResumeGuardDescr)
                 descr.guard_opnum = rop.GUARD_NONNULL_CLASS
-                emit_operation = False
-        if emit_operation:
-            self.emit_operation(op)
-            last_guard_index = len(self.optimizer.newoperations) - 1
-        else:
-            last_guard_index = value.last_guard_index
-        value.make_constant_class(expectedclassbox, last_guard_index)
+        self.emit_operation(op)
+        value.make_constant_class(expectedclassbox, op)
 
-    def optimize_GUARD_NONNULL_CLASS(self, op, dryrun=False):
-        self.optimize_GUARD_NONNULL(op, True)
-        self.optimize_GUARD_CLASS(op, dryrun)
+    def optimize_GUARD_NONNULL_CLASS(self, op):
+        value = self.getvalue(op.getarg(0))
+        if value.is_null():
+            raise InvalidLoop
+        self.optimize_GUARD_CLASS(op)
 
-    def optimize_GUARD_NO_EXCEPTION(self, op, dryrun=False):
-        if dryrun: return
+    def optimize_GUARD_NO_EXCEPTION(self, op):
         if not self.optimizer.exception_might_have_happened:
             return
         self.emit_operation(op)
@@ -470,7 +414,7 @@
                     newop = ResOperation(rop.SETARRAYITEM_GC,
                                          [op.getarg(2),
                                           ConstInt(index + dest_start),
-                                          val.force_box()], None,
+                                          val.get_key_box()], None,
                                          descr=source_value.arraydescr)
                     self.emit_operation(newop)
             return True
@@ -478,6 +422,25 @@
             return True # 0-length arraycopy
         return False
 
+    def optimize_CALL_PURE(self, op):
+        arg_consts = []
+        for i in range(op.numargs()):
+            arg = op.getarg(i)
+            const = self.get_constant_box(arg)
+            if const is None:
+                break
+            arg_consts.append(const)
+        else:
+            # all constant arguments: check if we already know the result
+            try:
+                result = self.optimizer.call_pure_results[arg_consts]
+            except KeyError:
+                pass
+            else:
+                self.make_constant(op.result, result)
+                return
+        self.emit_operation(op)
+
     def optimize_INT_FLOORDIV(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
@@ -492,6 +455,18 @@
                                         args = [op.getarg(0), ConstInt(highest_bit(val))])
         self.emit_operation(op)
 
+    def optimize_CAST_OPAQUE_PTR(self, op):
+        value = self.getvalue(op.getarg(0))
+        self.optimizer.opaque_pointers[value] = True
+        self.make_equal_to(op.result, value)
+
+    def optimize_CAST_PTR_TO_INT(self, op):
+        self.pure(rop.CAST_INT_TO_PTR, [op.result], op.getarg(0))
+        self.emit_operation(op)
+
+    def optimize_CAST_INT_TO_PTR(self, op):
+        self.pure(rop.CAST_PTR_TO_INT, [op.result], op.getarg(0))
+        self.emit_operation(op)
 
 dispatch_opt = make_dispatcher_method(OptRewrite, 'optimize_',
         default=OptRewrite.emit_operation)
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -39,8 +39,9 @@
 
 def test_sharing_field_lists_of_virtual():
     class FakeOptimizer(object):
-        class cpu(object):
-            pass
+        class optimizer(object):
+            class cpu(object):
+                pass
     opt = FakeOptimizer()
     virt1 = virtualize.AbstractVirtualStructValue(opt, None)
     lst1 = virt1._get_field_descr_list()
@@ -69,7 +70,7 @@
     class FakeVirtualValue(virtualize.AbstractVirtualValue):
         def _make_virtual(self, *args):
             return FakeVInfo()
-    v1 = FakeVirtualValue(None, None, None)
+    v1 = FakeVirtualValue(None, None)
     vinfo1 = v1.make_virtual_info(None, [1, 2, 4])
     vinfo2 = v1.make_virtual_info(None, [1, 2, 4])
     assert vinfo1 is vinfo2
@@ -111,7 +112,7 @@
 
 class BaseTestBasic(BaseTest):
 
-    enable_opts = "intbounds:rewrite:virtualize:string:heap"
+    enable_opts = "intbounds:rewrite:virtualize:string:earlyforce:pure:heap"
 
     def optimize_loop(self, ops, optops, call_pure_results=None):
 
@@ -651,8 +652,8 @@
         i3 = getfield_gc(p3, descr=valuedescr)
         escape(i3)
         p1 = new_with_vtable(ConstClass(node_vtable))
+        p1sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p1, i1, descr=valuedescr)
-        p1sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p1sub, i1, descr=valuedescr)
         setfield_gc(p1, p1sub, descr=nextdescr)
         jump(i1, p1, p2)
@@ -667,10 +668,10 @@
         p3sub = getfield_gc(p3, descr=nextdescr)
         i3 = getfield_gc(p3sub, descr=valuedescr)
         escape(i3)
+        p1 = new_with_vtable(ConstClass(node_vtable))
         p2sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p2sub, i1, descr=valuedescr)
         setfield_gc(p2, p2sub, descr=nextdescr)
-        p1 = new_with_vtable(ConstClass(node_vtable))
         jump(i1, p1, p2)
         """
         # The same as test_p123_simple, but in the end the "old" p2 contains
@@ -2328,7 +2329,7 @@
         def _variables_equal(box, varname, strict):
             if varname not in virtuals:
                 if strict:
-                    assert box == oparse.getvar(varname)
+                    assert box.same_box(oparse.getvar(varname))
                 else:
                     assert box.value == oparse.getvar(varname).value
             else:
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizefficall.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizefficall.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizefficall.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizefficall.py
@@ -36,7 +36,7 @@
 
 class TestFfiCall(BaseTestBasic, LLtypeMixin):
 
-    enable_opts = "intbounds:rewrite:virtualize:string:heap:ffi"
+    enable_opts = "intbounds:rewrite:virtualize:string:pure:earlyforce:heap:ffi"
 
     class namespace:
         cpu = LLtypeMixin.cpu
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -68,7 +68,7 @@
 
 class BaseTestWithUnroll(BaseTest):
 
-    enable_opts = "intbounds:rewrite:virtualize:string:heap:unroll"
+    enable_opts = "intbounds:rewrite:virtualize:string:earlyforce:pure:heap:unroll"
 
     def optimize_loop(self, ops, expected, expected_preamble=None,
                       call_pure_results=None, expected_short=None):
@@ -234,6 +234,30 @@
             """ % expected_value
             self.optimize_loop(ops, expected)
 
+    def test_reverse_of_cast(self):
+        ops = """
+        [i0]
+        p0 = cast_int_to_ptr(i0)
+        i1 = cast_ptr_to_int(p0)
+        jump(i1)
+        """
+        expected = """
+        [i0]
+        jump(i0)
+        """
+        self.optimize_loop(ops, expected)
+        ops = """
+        [p0]
+        i1 = cast_ptr_to_int(p0)
+        p1 = cast_int_to_ptr(i1)
+        jump(p1)
+        """
+        expected = """
+        [p0]
+        jump(p0)
+        """
+        self.optimize_loop(ops, expected)
+
     # ----------
 
     def test_remove_guard_class_1(self):
@@ -825,8 +849,8 @@
         i3 = getfield_gc(p2, descr=valuedescr)
         escape(i3)
         p4 = new_with_vtable(ConstClass(node_vtable))
+        p1sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p4, i1, descr=valuedescr)
-        p1sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p1sub, i1, descr=valuedescr)
         setfield_gc(p4, p1sub, descr=nextdescr)
         jump(i1, p4)
@@ -865,13 +889,7 @@
         p3sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p3sub, i1, descr=valuedescr)
         setfield_gc(p1, p3sub, descr=nextdescr)
-        # XXX: We get two extra operations here because the setfield
-        #      above is the result of forcing p1 and thus not
-        #      registered with the heap optimizer. I've makred tests
-        #      below with VIRTUALHEAP if they suffer from this issue
-        p3sub2 = getfield_gc(p1, descr=nextdescr)
-        guard_nonnull_class(p3sub2, ConstClass(node_vtable2)) []
-        jump(i1, p1, p3sub2)
+        jump(i1, p1, p3sub)
         """
         self.optimize_loop(ops, expected, preamble)
 
@@ -902,9 +920,7 @@
         guard_true(i2b) []
         p3 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3, i2, descr=nextdescr)
-        # XXX: VIRTUALHEAP (see above)
-        i3 = getfield_gc(p3, descr=nextdescr)
-        jump(p3, i3)
+        jump(p3, i2)
         """
         self.optimize_loop(ops, expected, preamble)
 
@@ -1219,7 +1235,15 @@
         setfield_gc(p3, p30, descr=valuedescr)
         jump(i29, p30, p3)
         """
-        expected = preamble
+        expected = """
+        [i0, p1, p3]
+        i28 = int_add(i0, 1)
+        i29 = int_add(i28, 1)
+        p30 = new_with_vtable(ConstClass(node_vtable))
+        setfield_gc(p3, p30, descr=valuedescr)
+        setfield_gc(p30, i28, descr=nextdescr)
+        jump(i29, p30, p3)
+        """
         self.optimize_loop(ops, expected, preamble)
 
     def test_nonvirtual_1(self):
@@ -2408,8 +2432,8 @@
         guard_class(p2, ConstClass(node_vtable)) []
         p3 = getfield_gc(p1, descr=otherdescr)
         guard_class(p3, ConstClass(node_vtable)) []
+        p3a = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3, p2, descr=otherdescr)
-        p3a = new_with_vtable(ConstClass(node_vtable))
         escape(p3a)
         jump(p3a)
         """
@@ -2421,9 +2445,9 @@
         # setfield_gc(p3, p2, descr=otherdescr) # p3a.other = p2a
         # p1a = new_with_vtable(ConstClass(node_vtable2))
         # p2a = new_with_vtable(ConstClass(node_vtable))
+        p3anew = new_with_vtable(ConstClass(node_vtable))
         p2 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3a, p2, descr=otherdescr) # p3a.other = p2a
-        p3anew = new_with_vtable(ConstClass(node_vtable))
         escape(p3anew)
         jump(p3anew)
         """
@@ -2458,9 +2482,9 @@
         p3 = getfield_gc(p1, descr=otherdescr)
         guard_class(p3, ConstClass(node_vtable)) []
         # p1a = new_with_vtable(ConstClass(node_vtable2))
+        p3a = new_with_vtable(ConstClass(node_vtable))
         p2a = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3, p2a, descr=otherdescr)
-        p3a = new_with_vtable(ConstClass(node_vtable))
         escape(p3a)
         # setfield_gc(p1a, p2a, descr=nextdescr)
         # setfield_gc(p1a, p3a, descr=otherdescr)
@@ -2468,9 +2492,9 @@
         """
         expected = """
         [p2, p3]
+        p3a = new_with_vtable(ConstClass(node_vtable))
         p2a = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3, p2a, descr=otherdescr)
-        p3a = new_with_vtable(ConstClass(node_vtable))
         escape(p3a)
         jump(p2a, p3a)
         """
@@ -2790,7 +2814,6 @@
         self.optimize_loop(ops, expected, preamble)
 
     def test_remove_duplicate_pure_op_ovf_with_lazy_setfield(self):
-        py.test.skip('this optimization is not yet supprted')
         ops = """
         [i1, p1]
         i3 = int_add_ovf(i1, 1)
@@ -5961,13 +5984,18 @@
         escape(i0)
         jump(p1)
         """
-        expected = """
+        preamble = """
         [p1]
         i0 = ptr_eq(p1, NULL)
         escape(i0)
-        jump(p1)
-        """
-        self.optimize_strunicode_loop_extradescrs(ops, expected, expected)
+        jump(p1, i0)
+        """
+        expected = """
+        [p1, i0]
+        escape(i0)
+        jump(p1, i0)
+        """
+        self.optimize_strunicode_loop_extradescrs(ops, expected, preamble)
 
     def test_str_equal_none2(self):
         ops = """
@@ -5976,13 +6004,18 @@
         escape(i0)
         jump(p1)
         """
-        expected = """
+        preamble = """
         [p1]
         i0 = ptr_eq(p1, NULL)
         escape(i0)
-        jump(p1)
-        """
-        self.optimize_strunicode_loop_extradescrs(ops, expected, expected)
+        jump(p1, i0)
+        """
+        expected = """
+        [p1, i0]
+        escape(i0)
+        jump(p1, i0)
+        """
+        self.optimize_strunicode_loop_extradescrs(ops, expected, preamble)
 
     def test_str_equal_nonnull1(self):
         ops = """
@@ -7220,6 +7253,29 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_heap_cache_forced_virtuals(self):
+        ops = """
+        [i1, i2, p0]
+        p1 = new(descr=ssize)
+        setfield_gc(p1, i1, descr=adescr)
+        setfield_gc(p1, i2, descr=bdescr)
+        call(p0, p1, descr=writeadescr)
+        i3 = getfield_gc(p1, descr=adescr)
+        i4 = getfield_gc(p1, descr=bdescr)
+        jump(i3, i4, p0)
+        """
+        expected = """
+        [i1, i2, p0]
+        p1 = new(descr=ssize)
+        setfield_gc(p1, i1, descr=adescr)
+        call(p0, p1, descr=writeadescr)
+        i3 = getfield_gc(p1, descr=adescr)
+        setfield_gc(p1, i2, descr=bdescr)
+        jump(i3, i2, p0)
+        """
+        self.optimize_loop(ops, expected)
+
+
     def test_setarrayitem_followed_by_arraycopy(self):
         ops = """
         [p1, p2]
@@ -7230,6 +7286,27 @@
         """
         self.optimize_loop(ops, ops)
 
+    def test_heap_cache_virtuals_forced_by_delayed_setfield(self):
+        py.test.skip('not yet supoprted')
+        ops = """
+        [i1, p0]
+        p1 = new(descr=ssize)
+        setfield_gc(p1, i1, descr=valuedescr)
+        setfield_gc(p0, p1, descr=adescr)
+        call(p0, descr=writeadescr)
+        i2 = getfield_gc(p1, descr=valuedescr)
+        jump(i2, p0)
+        """
+        expected = """
+        [i1, p0]
+        p1 = new(descr=ssize)
+        setfield_gc(p1, i1, descr=valuedescr)
+        setfield_gc(p0, p1, descr=adescr)
+        call(p0, descr=writeadescr)
+        jump(i1, p0)
+        """
+        self.optimize_loop(ops, expected)
+
 class TestLLtype(OptimizeOptTest, LLtypeMixin):
     pass
 
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_util.py b/pypy/jit/metainterp/optimizeopt/test/test_util.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_util.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_util.py
@@ -182,7 +182,8 @@
                             EffectInfo.EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE,
                             can_invalidate=True))
     arraycopydescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT,
-             EffectInfo([], [], [], [], oopspecindex=EffectInfo.OS_ARRAYCOPY))
+             EffectInfo([], [arraydescr], [], [arraydescr],
+                        oopspecindex=EffectInfo.OS_ARRAYCOPY))
 
     for _name, _os in [
         ('strconcatdescr',               'OS_STR_CONCAT'),
diff --git a/pypy/jit/metainterp/optimizeopt/unroll.py b/pypy/jit/metainterp/optimizeopt/unroll.py
--- a/pypy/jit/metainterp/optimizeopt/unroll.py
+++ b/pypy/jit/metainterp/optimizeopt/unroll.py
@@ -75,7 +75,6 @@
         self.importable_values = {}
         self.emitting_dissabled = False
         self.emitted_guards = 0
-        self.emitted_pure_operations = {}
 
     def ensure_imported(self, value):
         if not self.emitting_dissabled and value in self.importable_values:
@@ -96,21 +95,6 @@
         new = UnrollableOptimizer(self.metainterp_sd, self.loop)
         return self._new(new)
 
-    def remember_emitting_pure(self, op):
-        self.emitted_pure_operations[op] = True
-
-    def produce_potential_short_preamble_ops(self, sb):
-        for op in self.emitted_pure_operations:
-            if op.getopnum() == rop.GETARRAYITEM_GC_PURE or \
-               op.getopnum() == rop.STRGETITEM or \
-               op.getopnum() == rop.UNICODEGETITEM:
-                if not self.getvalue(op.getarg(1)).is_constant():
-                    continue
-            sb.add_potential(op)
-        for opt in self.optimizations:
-            opt.produce_potential_short_preamble_ops(sb)
-
-
 
 class UnrollOptimizer(Optimization):
     """Unroll the loop into two iterations. The first one will
@@ -154,7 +138,7 @@
 
             KillHugeIntBounds(self.optimizer).apply()
             
-            loop.preamble.operations = self.optimizer.newoperations
+            loop.preamble.operations = self.optimizer.get_newoperations()
             jump_args = [self.getvalue(a).get_key_box() for a in jump_args]
 
             start_resumedescr = loop.preamble.start_resumedescr.clone_if_mutable()
@@ -167,8 +151,9 @@
             virtual_state = modifier.get_virtual_state(jump_args)
             
             values = [self.getvalue(arg) for arg in jump_args]
-            inputargs = virtual_state.make_inputargs(values)
-            short_inputargs = virtual_state.make_inputargs(values, keyboxes=True)
+            inputargs = virtual_state.make_inputargs(values, self.optimizer)
+            short_inputargs = virtual_state.make_inputargs(values, self.optimizer,
+                                                           keyboxes=True)
 
             self.constant_inputargs = {}
             for box in jump_args: 
@@ -197,7 +182,7 @@
             # operations needed to setup the proper state of those virtuals
             # in the peeled loop
             inputarg_setup_ops = []
-            preamble_optimizer.newoperations = []
+            preamble_optimizer.clear_newoperations()
             seen = {}
             for box in inputargs:
                 if box in seen:
@@ -211,9 +196,8 @@
                     continue
                 seen[box] = True
                 value = preamble_optimizer.getvalue(box)
-                value.force_box()
-            preamble_optimizer.flush()
-            inputarg_setup_ops += preamble_optimizer.newoperations
+                value.force_box(preamble_optimizer)
+            inputarg_setup_ops += preamble_optimizer.get_newoperations()
 
             # Setup the state of the new optimizer by emiting the
             # short preamble operations and discarding the result
@@ -244,13 +228,13 @@
                                 virtual_state)
             
             loop.inputargs = inputargs
-            args = [preamble_optimizer.getvalue(self.short_boxes.original(a)).force_box()\
+            args = [preamble_optimizer.getvalue(self.short_boxes.original(a)).force_box(preamble_optimizer)\
                     for a in inputargs]
             jmp = ResOperation(rop.JUMP, args, None)
             jmp.setdescr(loop.token)
             loop.preamble.operations.append(jmp)
 
-            loop.operations = self.optimizer.newoperations
+            loop.operations = self.optimizer.get_newoperations()
             maxguards = self.optimizer.metainterp_sd.warmrunnerdesc.memory_manager.max_retrace_guards
             
             if self.optimizer.emitted_guards > maxguards:
@@ -335,9 +319,10 @@
         assert jumpop
         original_jumpargs = jumpop.getarglist()[:]
         values = [self.getvalue(arg) for arg in jumpop.getarglist()]
-        jumpargs = virtual_state.make_inputargs(values)
+        jumpargs = virtual_state.make_inputargs(values, self.optimizer)
         jumpop.initarglist(jumpargs)
-        jmp_to_short_args = virtual_state.make_inputargs(values, keyboxes=True)
+        jmp_to_short_args = virtual_state.make_inputargs(values, self.optimizer,
+                                                         keyboxes=True)
         self.short_inliner = Inliner(short_inputargs, jmp_to_short_args)
         
         for box, const in self.constant_inputargs.items():
@@ -347,11 +332,11 @@
             newop = self.short_inliner.inline_op(op)
             self.optimizer.send_extra_operation(newop)
         
-        self.optimizer.flush()
+        newoperations = self.optimizer.get_newoperations()
 
         i = j = 0
-        while i < len(self.optimizer.newoperations) or j < len(jumpargs):
-            if i == len(self.optimizer.newoperations):
+        while i < len(newoperations) or j < len(jumpargs):
+            if i == len(newoperations):
                 while j < len(jumpargs):
                     a = jumpargs[j]
                     if self.optimizer.loop.logops:
@@ -360,7 +345,7 @@
                                     jumpargs, short_seen)
                     j += 1
             else:
-                op = self.optimizer.newoperations[i]
+                op = newoperations[i]
 
                 self.boxes_created_this_iteration[op.result] = True
                 args = op.getarglist()
@@ -375,6 +360,7 @@
                     self.import_box(a, inputargs, short, short_jumpargs,
                                     jumpargs, short_seen)
                 i += 1
+            newoperations = self.optimizer.get_newoperations()
 
         jumpop.initarglist(jumpargs)
         self.optimizer.send_extra_operation(jumpop)
@@ -468,7 +454,7 @@
         inputargs.append(box)
         box = newresult
         if box in self.optimizer.values:
-            box = self.optimizer.values[box].force_box()
+            box = self.optimizer.values[box].force_box(self.optimizer)
         jumpargs.append(box)
         
 
@@ -483,11 +469,6 @@
         if op.getopnum() == rop.JUMP:
             loop_token = op.getdescr()
             assert isinstance(loop_token, LoopToken)
-            # FIXME: Use a tree, similar to the tree formed by the full
-            # preamble and it's bridges, instead of a list to save time and
-            # memory. This should also allow better behaviour in
-            # situations that the is_emittable() chain currently cant
-            # handle and the inlining fails unexpectedly belwo.
             short = loop_token.short_preamble
             if short:
                 args = op.getarglist()
@@ -523,7 +504,7 @@
 
                         values = [self.getvalue(arg)
                                   for arg in op.getarglist()]
-                        args = sh.virtual_state.make_inputargs(values,
+                        args = sh.virtual_state.make_inputargs(values, self.optimizer,
                                                                keyboxes=True)
                         inliner = Inliner(sh.inputargs, args)
                         
diff --git a/pypy/jit/metainterp/optimizeopt/util.py b/pypy/jit/metainterp/optimizeopt/util.py
--- a/pypy/jit/metainterp/optimizeopt/util.py
+++ b/pypy/jit/metainterp/optimizeopt/util.py
@@ -90,14 +90,11 @@
     for i in range(len(args1)):
         arg1 = args1[i]
         arg2 = args2[i]
-        if isinstance(arg1, history.Const):
-            if arg1.__class__ is not arg2.__class__:
+        if arg1 is None:
+            if arg2 is not None:
                 return False
-            if not arg1.same_constant(arg2):
-                return False
-        else:
-            if not arg1 is arg2:
-                return False
+        elif not arg1.same_box(arg2):
+            return False
     return True
 
 def args_hash(args):
@@ -106,10 +103,8 @@
     for arg in args:
         if arg is None:
             y = 17
-        elif isinstance(arg, history.Const):
+        else:
             y = arg._get_hash_()
-        else:
-            y = compute_identity_hash(arg)
         res = intmask((1000003 * res) ^ y)
     return res
 
@@ -145,9 +140,12 @@
         for i in range(op1.numargs()):
             x = op1.getarg(i)
             y = op2.getarg(i)
-            assert x == remap.get(y, y)
+            assert x.same_box(remap.get(y, y))
         if op2.result in remap:
-            assert op1.result == remap[op2.result]
+            if op2.result is None:
+                assert op1.result == remap[op2.result]
+            else:
+                assert op1.result.same_box(remap[op2.result])
         else:
             remap[op2.result] = op1.result
         if op1.getopnum() != rop.JUMP:      # xxx obscure
@@ -156,11 +154,20 @@
             assert len(op1.getfailargs()) == len(op2.getfailargs())
             if strict_fail_args:
                 for x, y in zip(op1.getfailargs(), op2.getfailargs()):
-                    assert x == remap.get(y, y)
+                    if x is None:
+                        assert remap.get(y, y) is None
+                    else:
+                        assert x.same_box(remap.get(y, y))
             else:
                 fail_args1 = set(op1.getfailargs())
                 fail_args2 = set([remap.get(y, y) for y in op2.getfailargs()])
-                assert fail_args1 == fail_args2
+                for x in fail_args1:
+                    for y in fail_args2:
+                        if x.same_box(y):
+                            fail_args2.remove(y)
+                            break
+                    else:
+                        assert False
     assert len(oplist1) == len(oplist2)
     print '-'*totwidth
     return True
diff --git a/pypy/jit/metainterp/optimizeopt/virtualize.py b/pypy/jit/metainterp/optimizeopt/virtualize.py
--- a/pypy/jit/metainterp/optimizeopt/virtualize.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualize.py
@@ -10,13 +10,12 @@
 
 
 class AbstractVirtualValue(optimizer.OptValue):
-    _attrs_ = ('optimizer', 'keybox', 'source_op', '_cached_vinfo')
+    _attrs_ = ('keybox', 'source_op', '_cached_vinfo')
     box = None
     level = optimizer.LEVEL_NONNULL
     _cached_vinfo = None
 
-    def __init__(self, optimizer, keybox, source_op=None):
-        self.optimizer = optimizer
+    def __init__(self, keybox, source_op=None):
         self.keybox = keybox   # only used as a key in dictionaries
         self.source_op = source_op  # the NEW_WITH_VTABLE/NEW_ARRAY operation
                                     # that builds this box
@@ -29,17 +28,17 @@
             return self.keybox
         return self.box
 
-    def force_box(self):
+    def force_box(self, optforce):
         if self.box is None:
-            self.optimizer.forget_numberings(self.keybox)
-            self._really_force()
+            optforce.forget_numberings(self.keybox)
+            self._really_force(optforce)
         return self.box
 
-    def force_at_end_of_preamble(self, already_forced):
+    def force_at_end_of_preamble(self, already_forced, optforce):
         value = already_forced.get(self, None)
         if value:
             return value
-        return OptValue(self.force_box())
+        return OptValue(self.force_box(optforce))
 
     def make_virtual_info(self, modifier, fieldnums):
         if fieldnums is None:
@@ -55,7 +54,7 @@
     def _make_virtual(self, modifier):
         raise NotImplementedError("abstract base")
 
-    def _really_force(self):
+    def _really_force(self, optforce):
         raise NotImplementedError("abstract base")
 
     def import_from(self, other, optimizer):
@@ -70,10 +69,11 @@
 get_fielddescrlist_cache._annspecialcase_ = "specialize:memo"
 
 class AbstractVirtualStructValue(AbstractVirtualValue):
-    _attrs_ = ('_fields', '_cached_sorted_fields')
+    _attrs_ = ('_fields', 'cpu', '_cached_sorted_fields')
 
-    def __init__(self, optimizer, keybox, source_op=None):
-        AbstractVirtualValue.__init__(self, optimizer, keybox, source_op)
+    def __init__(self, cpu, keybox, source_op=None):
+        AbstractVirtualValue.__init__(self, keybox, source_op)
+        self.cpu = cpu
         self._fields = {}
         self._cached_sorted_fields = None
 
@@ -87,45 +87,44 @@
     def _get_descr(self):
         raise NotImplementedError
 
-    def _is_immutable_and_filled_with_constants(self):
+    def _is_immutable_and_filled_with_constants(self, optforce):
         count = self._get_descr().count_fields_if_immutable()
         if count != len(self._fields):    # always the case if count == -1
             return False
         for value in self._fields.itervalues():
-            subbox = value.force_box()
+            subbox = value.force_box(optforce)
             if not isinstance(subbox, Const):
                 return False
         return True
 
-    def force_at_end_of_preamble(self, already_forced):
+    def force_at_end_of_preamble(self, already_forced, optforce):
         if self in already_forced:
             return self
         already_forced[self] = self
         if self._fields:
             for ofs in self._fields.keys():
-                self._fields[ofs] = self._fields[ofs].force_at_end_of_preamble(already_forced)
+                self._fields[ofs] = self._fields[ofs].force_at_end_of_preamble(already_forced, optforce)
         return self
 
-    def _really_force(self):
+    def _really_force(self, optforce):
         op = self.source_op
         assert op is not None
         # ^^^ This case should not occur any more (see test_bug_3).
         #
         if not we_are_translated():
             op.name = 'FORCE ' + self.source_op.name
-
-        if self._is_immutable_and_filled_with_constants():
-            box = self.optimizer.constant_fold(op)
+            
+        if self._is_immutable_and_filled_with_constants(optforce):
+            box = optforce.optimizer.constant_fold(op)
             self.make_constant(box)
             for ofs, value in self._fields.iteritems():
-                subbox = value.force_box()
+                subbox = value.force_box(optforce)
                 assert isinstance(subbox, Const)
-                execute(self.optimizer.cpu, None, rop.SETFIELD_GC,
+                execute(optforce.optimizer.cpu, None, rop.SETFIELD_GC,
                         ofs, box, subbox)
             # keep self._fields, because it's all immutable anyway
         else:
-            newoperations = self.optimizer.newoperations
-            newoperations.append(op)
+            optforce.emit_operation(op)
             self.box = box = op.result
             #
             iteritems = self._fields.iteritems()
@@ -135,10 +134,11 @@
             for ofs, value in iteritems:
                 if value.is_null():
                     continue
-                subbox = value.force_box()
+                subbox = value.force_box(optforce)
                 op = ResOperation(rop.SETFIELD_GC, [box, subbox], None,
                                   descr=ofs)
-                newoperations.append(op)
+
+                optforce.emit_operation(op)
 
     def _get_field_descr_list(self):
         _cached_sorted_fields = self._cached_sorted_fields
@@ -155,7 +155,7 @@
             else:
                 lst = self._fields.keys()
             sort_descrs(lst)
-            cache = get_fielddescrlist_cache(self.optimizer.cpu)
+            cache = get_fielddescrlist_cache(self.cpu)
             result = cache.get(lst, None)
             if result is None:
                 cache[lst] = lst
@@ -180,8 +180,8 @@
 class VirtualValue(AbstractVirtualStructValue):
     level = optimizer.LEVEL_KNOWNCLASS
 
-    def __init__(self, optimizer, known_class, keybox, source_op=None):
-        AbstractVirtualStructValue.__init__(self, optimizer, keybox, source_op)
+    def __init__(self, cpu, known_class, keybox, source_op=None):
+        AbstractVirtualStructValue.__init__(self, cpu, keybox, source_op)
         assert isinstance(known_class, Const)
         self.known_class = known_class
 
@@ -190,7 +190,7 @@
         return modifier.make_virtual(self.known_class, fielddescrs)
 
     def _get_descr(self):
-        return vtable2descr(self.optimizer.cpu, self.known_class.getint())
+        return vtable2descr(self.cpu, self.known_class.getint())
 
     def __repr__(self):
         cls_name = self.known_class.value.adr.ptr._obj._TYPE._name
@@ -201,8 +201,8 @@
 
 class VStructValue(AbstractVirtualStructValue):
 
-    def __init__(self, optimizer, structdescr, keybox, source_op=None):
-        AbstractVirtualStructValue.__init__(self, optimizer, keybox, source_op)
+    def __init__(self, cpu, structdescr, keybox, source_op=None):
+        AbstractVirtualStructValue.__init__(self, cpu, keybox, source_op)
         self.structdescr = structdescr
 
     def _make_virtual(self, modifier):
@@ -215,10 +215,10 @@
 
 class VArrayValue(AbstractVirtualValue):
 
-    def __init__(self, optimizer, arraydescr, size, keybox, source_op=None):
-        AbstractVirtualValue.__init__(self, optimizer, keybox, source_op)
+    def __init__(self, arraydescr, constvalue, size, keybox, source_op=None):
+        AbstractVirtualValue.__init__(self, keybox, source_op)
         self.arraydescr = arraydescr
-        self.constvalue = optimizer.new_const_item(arraydescr)
+        self.constvalue = constvalue
         self._items = [self.constvalue] * size
 
     def getlength(self):
@@ -232,31 +232,30 @@
         assert isinstance(itemvalue, optimizer.OptValue)
         self._items[index] = itemvalue
 
-    def force_at_end_of_preamble(self, already_forced):
+    def force_at_end_of_preamble(self, already_forced, optforce):
         if self in already_forced:
             return self
         already_forced[self] = self
         for index in range(len(self._items)):
-            self._items[index] = self._items[index].force_at_end_of_preamble(already_forced)
+            self._items[index] = self._items[index].force_at_end_of_preamble(already_forced, optforce)
         return self
     
-    def _really_force(self):
+    def _really_force(self, optforce):
         assert self.source_op is not None
         if not we_are_translated():
             self.source_op.name = 'FORCE ' + self.source_op.name
-        newoperations = self.optimizer.newoperations
-        newoperations.append(self.source_op)
+        optforce.emit_operation(self.source_op)        
         self.box = box = self.source_op.result
         for index in range(len(self._items)):
             subvalue = self._items[index]
             if subvalue is not self.constvalue:
                 if subvalue.is_null():
                     continue
-                subbox = subvalue.force_box()
+                subbox = subvalue.force_box(optforce)
                 op = ResOperation(rop.SETARRAYITEM_GC,
                                   [box, ConstInt(index), subbox], None,
                                   descr=self.arraydescr)
-                newoperations.append(op)
+                optforce.emit_operation(op)
 
     def get_args_for_fail(self, modifier):
         if self.box is None and not modifier.already_seen_virtual(self.keybox):
@@ -279,17 +278,18 @@
         return OptVirtualize()
         
     def make_virtual(self, known_class, box, source_op=None):
-        vvalue = VirtualValue(self.optimizer, known_class, box, source_op)
+        vvalue = VirtualValue(self.optimizer.cpu, known_class, box, source_op)
         self.make_equal_to(box, vvalue)
         return vvalue
 
     def make_varray(self, arraydescr, size, box, source_op=None):
-        vvalue = VArrayValue(self.optimizer, arraydescr, size, box, source_op)
+        constvalue = self.new_const_item(arraydescr)
+        vvalue = VArrayValue(arraydescr, constvalue, size, box, source_op)
         self.make_equal_to(box, vvalue)
         return vvalue
 
     def make_vstruct(self, structdescr, box, source_op=None):
-        vvalue = VStructValue(self.optimizer, structdescr, box, source_op)
+        vvalue = VStructValue(self.optimizer.cpu, structdescr, box, source_op)
         self.make_equal_to(box, vvalue)
         return vvalue
 
@@ -362,7 +362,6 @@
             self.make_equal_to(op.result, fieldvalue)
         else:
             value.ensure_nonnull()
-            ###self.heap_op_optimizer.optimize_GETFIELD_GC(op, value)
             self.emit_operation(op)
 
     # note: the following line does not mean that the two operations are
@@ -377,7 +376,6 @@
             value.setfield(op.getdescr(), fieldvalue)
         else:
             value.ensure_nonnull()
-            ###self.heap_op_optimizer.optimize_SETFIELD_GC(op, value, fieldvalue)
             self.emit_operation(op)
 
     def optimize_NEW_WITH_VTABLE(self, op):
@@ -417,7 +415,6 @@
                 self.make_equal_to(op.result, itemvalue)
                 return
         value.ensure_nonnull()
-        ###self.heap_op_optimizer.optimize_GETARRAYITEM_GC(op, value)
         self.emit_operation(op)
 
     # note: the following line does not mean that the two operations are
@@ -432,7 +429,6 @@
                 value.setitem(indexbox.getint(), self.getvalue(op.getarg(2)))
                 return
         value.ensure_nonnull()
-        ###self.heap_op_optimizer.optimize_SETARRAYITEM_GC(op, value, fieldvalue)
         self.emit_operation(op)
 
 
diff --git a/pypy/jit/metainterp/optimizeopt/virtualstate.py b/pypy/jit/metainterp/optimizeopt/virtualstate.py
--- a/pypy/jit/metainterp/optimizeopt/virtualstate.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualstate.py
@@ -30,7 +30,7 @@
     def _generate_guards(self, other, box, cpu, extra_guards):
         raise InvalidLoop
 
-    def enum_forced_boxes(self, boxes, value):
+    def enum_forced_boxes(self, boxes, value, optimizer):
         raise NotImplementedError
 
     def enum(self, virtual_state):
@@ -100,14 +100,14 @@
     def _generalization_of(self, other):
         raise NotImplementedError
 
-    def enum_forced_boxes(self, boxes, value):
+    def enum_forced_boxes(self, boxes, value, optimizer):
         assert isinstance(value, virtualize.AbstractVirtualStructValue)
         assert value.is_virtual()
         for i in range(len(self.fielddescrs)):
             v = value._fields[self.fielddescrs[i]]
             s = self.fieldstate[i]
             if s.position > self.position:
-                s.enum_forced_boxes(boxes, v)
+                s.enum_forced_boxes(boxes, v, optimizer)
 
     def _enum(self, virtual_state):
         for s in self.fieldstate:
@@ -177,14 +177,14 @@
                 return False
         return True
 
-    def enum_forced_boxes(self, boxes, value):
+    def enum_forced_boxes(self, boxes, value, optimizer):
         assert isinstance(value, virtualize.VArrayValue)
         assert value.is_virtual()
         for i in range(len(self.fieldstate)):
             v = value._items[i]
             s = self.fieldstate[i]
             if s.position > self.position:
-                s.enum_forced_boxes(boxes, v)
+                s.enum_forced_boxes(boxes, v, optimizer)
 
     def _enum(self, virtual_state):
         for s in self.fieldstate:
@@ -316,11 +316,11 @@
             import pdb; pdb.set_trace()
             raise NotImplementedError
 
-    def enum_forced_boxes(self, boxes, value):
+    def enum_forced_boxes(self, boxes, value, optimizer):
         if self.level == LEVEL_CONSTANT:
             return
         assert 0 <= self.position_in_notvirtuals 
-        boxes[self.position_in_notvirtuals] = value.force_box()
+        boxes[self.position_in_notvirtuals] = value.force_box(optimizer)
 
     def _enum(self, virtual_state):
         if self.level == LEVEL_CONSTANT:
@@ -377,11 +377,13 @@
             self.state[i].generate_guards(other.state[i], args[i],
                                           cpu, extra_guards, renum)
 
-    def make_inputargs(self, values, keyboxes=False):
+    def make_inputargs(self, values, optimizer, keyboxes=False):
+        if optimizer.optearlyforce:
+            optimizer = optimizer.optearlyforce
         assert len(values) == len(self.state)
         inputargs = [None] * len(self.notvirtuals)
         for i in range(len(values)):
-            self.state[i].enum_forced_boxes(inputargs, values[i])
+            self.state[i].enum_forced_boxes(inputargs, values[i], optimizer)
 
         if keyboxes:
             for i in range(len(values)):
@@ -434,7 +436,12 @@
     def get_virtual_state(self, jump_args):
         self.optimizer.force_at_end_of_preamble()
         already_forced = {}
-        values = [self.getvalue(box).force_at_end_of_preamble(already_forced)
+        if self.optimizer.optearlyforce:
+            opt = self.optimizer.optearlyforce
+        else:
+            opt = self.optimizer
+        values = [self.getvalue(box).force_at_end_of_preamble(already_forced,
+                                                              opt)
                   for box in jump_args]
 
         for value in values:
diff --git a/pypy/jit/metainterp/optimizeopt/vstring.py b/pypy/jit/metainterp/optimizeopt/vstring.py
--- a/pypy/jit/metainterp/optimizeopt/vstring.py
+++ b/pypy/jit/metainterp/optimizeopt/vstring.py
@@ -43,7 +43,7 @@
 class __extend__(optimizer.OptValue):
     """New methods added to the base class OptValue for this file."""
 
-    def getstrlen(self, optimization, mode):
+    def getstrlen(self, string_optimizer, mode):
         if mode is mode_string:
             s = self.get_constant_string_spec(mode_string)
             if s is not None:
@@ -52,12 +52,12 @@
             s = self.get_constant_string_spec(mode_unicode)
             if s is not None:
                 return ConstInt(len(s))
-        if optimization is None:
+        if string_optimizer is None:
             return None
         self.ensure_nonnull()
-        box = self.force_box()
+        box = self.force_box(string_optimizer)
         lengthbox = BoxInt()
-        optimization.propagate_forward(ResOperation(mode.STRLEN, [box], lengthbox))
+        string_optimizer.emit_operation(ResOperation(mode.STRLEN, [box], lengthbox))
         return lengthbox
 
     @specialize.arg(1)
@@ -68,25 +68,25 @@
         else:
             return None
 
-    def string_copy_parts(self, optimizer, targetbox, offsetbox, mode):
+    def string_copy_parts(self, string_optimizer, targetbox, offsetbox, mode):
         # Copies the pointer-to-string 'self' into the target string
         # given by 'targetbox', at the specified offset.  Returns the offset
         # at the end of the copy.
-        lengthbox = self.getstrlen(optimizer, mode)
-        srcbox = self.force_box()
-        return copy_str_content(optimizer, srcbox, targetbox,
+        lengthbox = self.getstrlen(string_optimizer, mode)
+        srcbox = self.force_box(string_optimizer)
+        return copy_str_content(string_optimizer, srcbox, targetbox,
                                 CONST_0, offsetbox, lengthbox, mode)
 
 
 class VAbstractStringValue(virtualize.AbstractVirtualValue):
     _attrs_ = ('mode',)
 
-    def __init__(self, optimizer, keybox, source_op, mode):
-        virtualize.AbstractVirtualValue.__init__(self, optimizer, keybox,
+    def __init__(self, keybox, source_op, mode):
+        virtualize.AbstractVirtualValue.__init__(self, keybox,
                                                  source_op)
         self.mode = mode
 
-    def _really_force(self):
+    def _really_force(self, optforce):
         if self.mode is mode_string:
             s = self.get_constant_string_spec(mode_string)
             if s is not None:
@@ -101,12 +101,12 @@
                 return
         assert self.source_op is not None
         self.box = box = self.source_op.result
-        lengthbox = self.getstrlen(self.optimizer, self.mode)
+        lengthbox = self.getstrlen(optforce, self.mode)
         op = ResOperation(self.mode.NEWSTR, [lengthbox], box)
         if not we_are_translated():
             op.name = 'FORCE'
-        self.optimizer.emit_operation(op)
-        self.string_copy_parts(self.optimizer, box, CONST_0, self.mode)
+        optforce.emit_operation(op)
+        self.string_copy_parts(optforce, box, CONST_0, self.mode)
 
 
 class VStringPlainValue(VAbstractStringValue):
@@ -140,20 +140,20 @@
         return mode.emptystr.join([mode.chr(c.box.getint())
                                    for c in self._chars])
 
-    def string_copy_parts(self, optimizer, targetbox, offsetbox, mode):
+    def string_copy_parts(self, string_optimizer, targetbox, offsetbox, mode):
         if not self.is_virtual() and targetbox is not self.box:
-            lengthbox = self.getstrlen(optimizer, mode)
-            srcbox = self.force_box()
-            return copy_str_content(optimizer, srcbox, targetbox,
+            lengthbox = self.getstrlen(string_optimizer, mode)
+            srcbox = self.force_box(string_optimizer)
+            return copy_str_content(string_optimizer, srcbox, targetbox,
                                 CONST_0, offsetbox, lengthbox, mode)
         for i in range(len(self._chars)):
-            charbox = self._chars[i].force_box()
+            charbox = self._chars[i].force_box(string_optimizer)
             if not (isinstance(charbox, Const) and charbox.same_constant(CONST_0)):
-                optimizer.emit_operation(ResOperation(mode.STRSETITEM, [targetbox,
-                                                                    offsetbox,
-                                                                    charbox],
+                string_optimizer.emit_operation(ResOperation(mode.STRSETITEM, [targetbox,
+                                                                               offsetbox,
+                                                                               charbox],
                                                   None))
-            offsetbox = _int_add(optimizer, offsetbox, CONST_1)
+            offsetbox = _int_add(string_optimizer, offsetbox, CONST_1)
         return offsetbox
 
     def get_args_for_fail(self, modifier):
@@ -187,16 +187,16 @@
         self.left = left
         self.right = right
 
-    def getstrlen(self, optimizer, mode):
+    def getstrlen(self, string_optimizer, mode):
         if self.lengthbox is None:
-            len1box = self.left.getstrlen(optimizer, mode)
+            len1box = self.left.getstrlen(string_optimizer, mode)
             if len1box is None:
                 return None
-            len2box = self.right.getstrlen(optimizer, mode)
+            len2box = self.right.getstrlen(string_optimizer, mode)
             if len2box is None:
                 return None
-            self.lengthbox = _int_add(optimizer, len1box, len2box)
-            # ^^^ may still be None, if optimizer is None
+            self.lengthbox = _int_add(string_optimizer, len1box, len2box)
+            # ^^^ may still be None, if string_optimizer is None
         return self.lengthbox
 
     @specialize.arg(1)
@@ -209,10 +209,10 @@
             return None
         return s1 + s2
 
-    def string_copy_parts(self, optimizer, targetbox, offsetbox, mode):
-        offsetbox = self.left.string_copy_parts(optimizer, targetbox,
+    def string_copy_parts(self, string_optimizer, targetbox, offsetbox, mode):
+        offsetbox = self.left.string_copy_parts(string_optimizer, targetbox,
                                                 offsetbox, mode)
-        offsetbox = self.right.string_copy_parts(optimizer, targetbox,
+        offsetbox = self.right.string_copy_parts(string_optimizer, targetbox,
                                                  offsetbox, mode)
         return offsetbox
 
@@ -251,8 +251,8 @@
         self.vstart = vstart
         self.vlength = vlength
 
-    def getstrlen(self, _, mode):
-        return self.vlength.force_box()
+    def getstrlen(self, optforce, mode):
+        return self.vlength.force_box(optforce)
 
     @specialize.arg(1)
     def get_constant_string_spec(self, mode):
@@ -267,11 +267,11 @@
             return s1[start : start + length]
         return None
 
-    def string_copy_parts(self, optimizer, targetbox, offsetbox, mode):
-        lengthbox = self.getstrlen(optimizer, mode)
-        return copy_str_content(optimizer,
-                                self.vstr.force_box(), targetbox,
-                                self.vstart.force_box(), offsetbox,
+    def string_copy_parts(self, string_optimizer, targetbox, offsetbox, mode):
+        lengthbox = self.getstrlen(string_optimizer, mode)
+        return copy_str_content(string_optimizer,
+                                self.vstr.force_box(string_optimizer), targetbox,
+                                self.vstart.force_box(string_optimizer), offsetbox,
                                 lengthbox, mode)
 
     def get_args_for_fail(self, modifier):
@@ -300,7 +300,7 @@
         return modifier.make_vstrslice(self.mode is mode_unicode)
 
 
-def copy_str_content(optimizer, srcbox, targetbox,
+def copy_str_content(string_optimizer, srcbox, targetbox,
                      srcoffsetbox, offsetbox, lengthbox, mode, need_next_offset=True):
     if isinstance(srcbox, ConstPtr) and isinstance(srcoffsetbox, Const):
         M = 5
@@ -310,26 +310,26 @@
         # up to M characters are done "inline", i.e. with STRGETITEM/STRSETITEM
         # instead of just a COPYSTRCONTENT.
         for i in range(lengthbox.value):
-            charbox = _strgetitem(optimizer, srcbox, srcoffsetbox, mode)
-            srcoffsetbox = _int_add(optimizer, srcoffsetbox, CONST_1)
-            optimizer.emit_operation(ResOperation(mode.STRSETITEM, [targetbox,
-                                                                offsetbox,
-                                                                charbox],
+            charbox = _strgetitem(string_optimizer, srcbox, srcoffsetbox, mode)
+            srcoffsetbox = _int_add(string_optimizer, srcoffsetbox, CONST_1)
+            string_optimizer.emit_operation(ResOperation(mode.STRSETITEM, [targetbox,
+                                                                           offsetbox,
+                                                                           charbox],
                                               None))
-            offsetbox = _int_add(optimizer, offsetbox, CONST_1)
+            offsetbox = _int_add(string_optimizer, offsetbox, CONST_1)
     else:
         if need_next_offset:
-            nextoffsetbox = _int_add(optimizer, offsetbox, lengthbox)
+            nextoffsetbox = _int_add(string_optimizer, offsetbox, lengthbox)
         else:
             nextoffsetbox = None
         op = ResOperation(mode.COPYSTRCONTENT, [srcbox, targetbox,
                                                 srcoffsetbox, offsetbox,
                                                 lengthbox], None)
-        optimizer.emit_operation(op)
+        string_optimizer.emit_operation(op)
         offsetbox = nextoffsetbox
     return offsetbox
 
-def _int_add(optimizer, box1, box2):
+def _int_add(string_optimizer, box1, box2):
     if isinstance(box1, ConstInt):
         if box1.value == 0:
             return box2
@@ -337,23 +337,23 @@
             return ConstInt(box1.value + box2.value)
     elif isinstance(box2, ConstInt) and box2.value == 0:
         return box1
-    if optimizer is None:
+    if string_optimizer is None:
         return None
     resbox = BoxInt()
-    optimizer.propagate_forward(ResOperation(rop.INT_ADD, [box1, box2], resbox))
+    string_optimizer.emit_operation(ResOperation(rop.INT_ADD, [box1, box2], resbox))
     return resbox
 
-def _int_sub(optimizer, box1, box2):
+def _int_sub(string_optimizer, box1, box2):
     if isinstance(box2, ConstInt):
         if box2.value == 0:
             return box1
         if isinstance(box1, ConstInt):
             return ConstInt(box1.value - box2.value)
     resbox = BoxInt()
-    optimizer.propagate_forward(ResOperation(rop.INT_SUB, [box1, box2], resbox))
+    string_optimizer.emit_operation(ResOperation(rop.INT_SUB, [box1, box2], resbox))
     return resbox
 
-def _strgetitem(optimizer, strbox, indexbox, mode):
+def _strgetitem(string_optimizer, strbox, indexbox, mode):
     if isinstance(strbox, ConstPtr) and isinstance(indexbox, ConstInt):
         if mode is mode_string:
             s = strbox.getref(lltype.Ptr(rstr.STR))
@@ -362,30 +362,28 @@
             s = strbox.getref(lltype.Ptr(rstr.UNICODE))
             return ConstInt(ord(s.chars[indexbox.getint()]))
     resbox = BoxInt()
-    optimizer.propagate_forward(ResOperation(mode.STRGETITEM, [strbox, indexbox],
-                                      resbox))
+    string_optimizer.emit_operation(ResOperation(mode.STRGETITEM, [strbox, indexbox],
+                                                 resbox))
     return resbox
 
 
 class OptString(optimizer.Optimization):
     "Handling of strings and unicodes."
-    enabled = True
-
     def new(self):
         return OptString()
 
     def make_vstring_plain(self, box, source_op, mode):
-        vvalue = VStringPlainValue(self.optimizer, box, source_op, mode)
+        vvalue = VStringPlainValue(box, source_op, mode)
         self.make_equal_to(box, vvalue)
         return vvalue
 
     def make_vstring_concat(self, box, source_op, mode):
-        vvalue = VStringConcatValue(self.optimizer, box, source_op, mode)
+        vvalue = VStringConcatValue(box, source_op, mode)
         self.make_equal_to(box, vvalue)
         return vvalue
 
     def make_vstring_slice(self, box, source_op, mode):
-        vvalue = VStringSliceValue(self.optimizer, box, source_op, mode)
+        vvalue = VStringSliceValue(box, source_op, mode)
         self.make_equal_to(box, vvalue)
         return vvalue
 
@@ -435,9 +433,9 @@
         value.ensure_nonnull()
         #
         if value.is_virtual() and isinstance(value, VStringSliceValue):
-            fullindexbox = _int_add(self.optimizer,
-                                    value.vstart.force_box(),
-                                    vindex.force_box())
+            fullindexbox = _int_add(self,
+                                    value.vstart.force_box(self),
+                                    vindex.force_box(self))
             value = value.vstr
             vindex = self.getvalue(fullindexbox)
         #
@@ -449,7 +447,7 @@
                 if res is not optimizer.CVAL_UNINITIALIZED_ZERO:
                     return res
         #
-        resbox = _strgetitem(self.optimizer, value.force_box(), vindex.force_box(), mode)
+        resbox = _strgetitem(self, value.force_box(self), vindex.force_box(self), mode)
         return self.getvalue(resbox)
 
     def optimize_STRLEN(self, op):
@@ -459,7 +457,7 @@
 
     def _optimize_STRLEN(self, op, mode):
         value = self.getvalue(op.getarg(0))
-        lengthbox = value.getstrlen(self.optimizer, mode)
+        lengthbox = value.getstrlen(self, mode)
         self.make_equal_to(op.result, self.getvalue(lengthbox))
 
     def optimize_COPYSTRCONTENT(self, op):
@@ -477,12 +475,12 @@
 
         if length.is_constant() and length.box.getint() == 0:
             return
-        copy_str_content(self.optimizer,
-            src.force_box(),
-            dst.force_box(),
-            srcstart.force_box(),
-            dststart.force_box(),
-            length.force_box(),
+        copy_str_content(self,
+            src.force_box(self),
+            dst.force_box(self),
+            srcstart.force_box(self),
+            dststart.force_box(self),
+            length.force_box(self),
             mode, need_next_offset=False
         )
 
@@ -508,6 +506,8 @@
                     return
         self.emit_operation(op)
 
+    optimize_CALL_PURE = optimize_CALL
+
     def opt_call_str_STR2UNICODE(self, op):
         # Constant-fold unicode("constant string").
         # More generally, supporting non-constant but virtual cases is
@@ -547,16 +547,16 @@
             return True
         #
         vstr.ensure_nonnull()
-        lengthbox = _int_sub(self.optimizer, vstop.force_box(),
-                                            vstart.force_box())
+        lengthbox = _int_sub(self, vstop.force_box(self),
+                                   vstart.force_box(self))
         #
         if isinstance(vstr, VStringSliceValue):
             # double slicing  s[i:j][k:l]
             vintermediate = vstr
             vstr = vintermediate.vstr
-            startbox = _int_add(self.optimizer,
-                                vintermediate.vstart.force_box(),
-                                vstart.force_box())
+            startbox = _int_add(self,
+                                vintermediate.vstart.force_box(self),
+                                vstart.force_box(self))
             vstart = self.getvalue(startbox)
         #
         value = self.make_vstring_slice(op.result, op, mode)
@@ -587,15 +587,12 @@
             return True
         #
         if v1.is_nonnull() and v2.is_nonnull():
-            if l1box is not None and l2box is not None and (
-                l1box == l2box or (isinstance(l1box, ConstInt) and
-                                   isinstance(l2box, ConstInt) and
-                                   l1box.value == l2box.value)):
+            if l1box is not None and l2box is not None and l1box.same_box(l2box):
                 do = EffectInfo.OS_STREQ_LENGTHOK
             else:
                 do = EffectInfo.OS_STREQ_NONNULL
-            self.generate_modified_call(do, [v1.force_box(),
-                                             v2.force_box()], op.result, mode)
+            self.generate_modified_call(do, [v1.force_box(self),
+                                             v2.force_box(self)], op.result, mode)
             return True
         return False
 
@@ -603,7 +600,7 @@
         l2box = v2.getstrlen(None, mode)
         if isinstance(l2box, ConstInt):
             if l2box.value == 0:
-                lengthbox = v1.getstrlen(self.optimizer, mode)
+                lengthbox = v1.getstrlen(self, mode)
                 seo = self.optimizer.send_extra_operation
                 seo(ResOperation(rop.INT_EQ, [lengthbox, CONST_0], resultbox))
                 return True
@@ -614,17 +611,17 @@
                     vchar1 = self.strgetitem(v1, optimizer.CVAL_ZERO, mode)
                     vchar2 = self.strgetitem(v2, optimizer.CVAL_ZERO, mode)
                     seo = self.optimizer.send_extra_operation
-                    seo(ResOperation(rop.INT_EQ, [vchar1.force_box(),
-                                                  vchar2.force_box()],
+                    seo(ResOperation(rop.INT_EQ, [vchar1.force_box(self),
+                                                  vchar2.force_box(self)],
                                      resultbox))
                     return True
                 if isinstance(v1, VStringSliceValue):
                     vchar = self.strgetitem(v2, optimizer.CVAL_ZERO, mode)
                     do = EffectInfo.OS_STREQ_SLICE_CHAR
-                    self.generate_modified_call(do, [v1.vstr.force_box(),
-                                                     v1.vstart.force_box(),
-                                                     v1.vlength.force_box(),
-                                                     vchar.force_box()],
+                    self.generate_modified_call(do, [v1.vstr.force_box(self),
+                                                     v1.vstart.force_box(self),
+                                                     v1.vlength.force_box(self),
+                                                     vchar.force_box(self)],
                                                 resultbox, mode)
                     return True
         #
@@ -635,10 +632,10 @@
             if v1.is_null():
                 self.make_constant(resultbox, CONST_1)
                 return True
-            op = ResOperation(rop.PTR_EQ, [v1.force_box(),
+            op = ResOperation(rop.PTR_EQ, [v1.force_box(self),
                                            llhelper.CONST_NULL],
                               resultbox)
-            self.optimizer.emit_operation(op)
+            self.emit_operation(op)
             return True
         #
         return False
@@ -652,8 +649,8 @@
                     do = EffectInfo.OS_STREQ_NONNULL_CHAR
                 else:
                     do = EffectInfo.OS_STREQ_CHECKNULL_CHAR
-                self.generate_modified_call(do, [v1.force_box(),
-                                                 vchar.force_box()], resultbox,
+                self.generate_modified_call(do, [v1.force_box(self),
+                                                 vchar.force_box(self)], resultbox,
                                             mode)
                 return True
         #
@@ -662,10 +659,10 @@
                 do = EffectInfo.OS_STREQ_SLICE_NONNULL
             else:
                 do = EffectInfo.OS_STREQ_SLICE_CHECKNULL
-            self.generate_modified_call(do, [v1.vstr.force_box(),
-                                             v1.vstart.force_box(),
-                                             v1.vlength.force_box(),
-                                             v2.force_box()], resultbox, mode)
+            self.generate_modified_call(do, [v1.vstr.force_box(self),
+                                             v1.vstart.force_box(self),
+                                             v1.vlength.force_box(self),
+                                             v2.force_box(self)], resultbox, mode)
             return True
         return False
 
@@ -675,16 +672,11 @@
         calldescr, func = cic.callinfo_for_oopspec(oopspecindex)
         op = ResOperation(rop.CALL, [ConstInt(func)] + args, result,
                           descr=calldescr)
-        self.optimizer.emit_operation(op)
+        self.emit_operation(op)
 
     def propagate_forward(self, op):
-        if not self.enabled:
-            self.emit_operation(op)
-            return
-
         dispatch_opt(self, op)
 
-
 dispatch_opt = make_dispatcher_method(OptString, 'optimize_',
         default=OptString.emit_operation)
 
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -162,15 +162,18 @@
             if registers[i] is oldbox:
                 registers[i] = newbox
         if not we_are_translated():
-            assert oldbox not in registers[count:]
+            for b in registers[count:]:
+                assert not oldbox.same_box(b)
+                
 
     def make_result_of_lastop(self, resultbox):
         got_type = resultbox.type
-        if not we_are_translated():
-            typeof = {'i': history.INT,
-                      'r': history.REF,
-                      'f': history.FLOAT}
-            assert typeof[self.jitcode._resulttypes[self.pc]] == got_type
+        # XXX disabled for now, conflicts with str_guard_value
+        #if not we_are_translated():
+        #    typeof = {'i': history.INT,
+        #              'r': history.REF,
+        #              'f': history.FLOAT}
+        #    assert typeof[self.jitcode._resulttypes[self.pc]] == got_type
         target_index = ord(self.bytecode[self.pc-1])
         if got_type == history.INT:
             self.registers_i[target_index] = resultbox
@@ -219,6 +222,7 @@
                     'cast_float_to_int', 'cast_int_to_float',
                     'cast_float_to_singlefloat', 'cast_singlefloat_to_float',
                     'float_neg', 'float_abs',
+                    'cast_ptr_to_int', 'cast_int_to_ptr',
                     ]:
         exec py.code.Source('''
             @arguments("box")
@@ -895,6 +899,21 @@
     def _opimpl_guard_value(self, orgpc, box):
         self.implement_guard_value(orgpc, box)
 
+    @arguments("orgpc", "box", "box", "descr")
+    def opimpl_str_guard_value(self, orgpc, box, funcbox, descr):
+        if isinstance(box, Const):
+            return box     # no promotion needed, already a Const
+        else:
+            constbox = box.constbox()
+            resbox = self.do_residual_call(funcbox, descr, [box, constbox])
+            promoted_box = resbox.constbox()
+            # This is GUARD_VALUE because GUARD_TRUE assumes the existance
+            # of a label when computing resumepc
+            self.generate_guard(rop.GUARD_VALUE, resbox, [promoted_box],
+                                resumepc=orgpc)
+            self.metainterp.replace_box(box, constbox)
+            return constbox
+
     opimpl_int_guard_value = _opimpl_guard_value
     opimpl_ref_guard_value = _opimpl_guard_value
     opimpl_float_guard_value = _opimpl_guard_value
diff --git a/pypy/jit/metainterp/quasiimmut.py b/pypy/jit/metainterp/quasiimmut.py
--- a/pypy/jit/metainterp/quasiimmut.py
+++ b/pypy/jit/metainterp/quasiimmut.py
@@ -2,6 +2,7 @@
 from pypy.rpython.lltypesystem import lltype, rclass
 from pypy.rpython.annlowlevel import cast_base_ptr_to_instance
 from pypy.jit.metainterp.history import AbstractDescr
+from pypy.rlib.objectmodel import we_are_translated
 
 
 def get_mutate_field_name(fieldname):
@@ -50,13 +51,13 @@
 
 class QuasiImmut(object):
     llopaque = True
+    compress_limit = 30
     
     def __init__(self, cpu):
         self.cpu = cpu
         # list of weakrefs to the LoopTokens that must be invalidated if
         # this value ever changes
         self.looptokens_wrefs = []
-        self.compress_limit = 30
 
     def hide(self):
         qmut_ptr = self.cpu.ts.cast_instance_to_base_ref(self)
@@ -75,6 +76,8 @@
     def compress_looptokens_list(self):
         self.looptokens_wrefs = [wref for wref in self.looptokens_wrefs
                                       if wref() is not None]
+        # NB. we must keep around the looptoken_wrefs that are
+        # already invalidated; see below
         self.compress_limit = (len(self.looptokens_wrefs) + 15) * 2
 
     def invalidate(self):
@@ -86,7 +89,16 @@
         for wref in wrefs:
             looptoken = wref()
             if looptoken is not None:
+                looptoken.invalidated = True
                 self.cpu.invalidate_loop(looptoken)
+                # NB. we must call cpu.invalidate_loop() even if
+                # looptoken.invalidated was already set to True.
+                # It's possible to invalidate several times the
+                # same looptoken; see comments in jit.backend.model
+                # in invalidate_loop().
+                if not we_are_translated():
+                    self.cpu.stats.invalidated_token_numbers.add(
+                        looptoken.number)
 
 
 class QuasiImmutDescr(AbstractDescr):
diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py
--- a/pypy/jit/metainterp/resoperation.py
+++ b/pypy/jit/metainterp/resoperation.py
@@ -221,7 +221,6 @@
         newop.setfailargs(self.getfailargs())
         return newop
 
-
 # ============
 # arity mixins
 # ============
@@ -434,6 +433,8 @@
     'INT_INVERT/1',
     #
     'SAME_AS/1',      # gets a Const or a Box, turns it into another Box
+    'CAST_PTR_TO_INT/1',
+    'CAST_INT_TO_PTR/1',
     #
     'PTR_EQ/2b',
     'PTR_NE/2b',
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -13,7 +13,7 @@
 from pypy.rlib.jit import (JitDriver, we_are_jitted, hint, dont_look_inside,
     loop_invariant, elidable, promote, jit_debug, assert_green,
     AssertGreenFailed, unroll_safe, current_trace_length, look_inside_iff,
-    isconstant, isvirtual)
+    isconstant, isvirtual, promote_string)
 from pypy.rlib.rarithmetic import ovfcheck
 from pypy.rpython.lltypesystem import lltype, llmemory, rffi
 from pypy.rpython.ootypesystem import ootype
@@ -2956,6 +2956,18 @@
         assert res == f(32)
         self.check_loops(arraylen_gc=2)
 
+    def test_ulonglong_mod(self):
+        myjitdriver = JitDriver(greens = [], reds = ['n', 'sa', 'i'])
+        def f(n):
+            sa = i = rffi.cast(rffi.ULONGLONG, 1)
+            while i < rffi.cast(rffi.ULONGLONG, n):
+                myjitdriver.jit_merge_point(sa=sa, n=n, i=i)
+                sa += sa % i
+                i += 1
+        res = self.meta_interp(f, [32])
+        assert res == f(32)
+
+
 class TestOOtype(BasicTests, OOJitMixin):
 
     def test_oohash(self):
@@ -3428,7 +3440,6 @@
 
 class TestLLtype(BaseLLtypeTests, LLJitMixin):
     def test_tagged(self):
-        py.test.skip("implement me")
         from pypy.rlib.objectmodel import UnboxedValue
         class Base(object):
             __slots__ = ()
@@ -3480,3 +3491,35 @@
                 pc += 1
             return pc
         res = self.meta_interp(main, [False, 100, True], taggedpointers=True)
+
+    def test_rerased(self):
+        from pypy.rlib.rerased import erase_int, unerase_int, new_erasing_pair
+        eraseX, uneraseX = new_erasing_pair("X")
+        #
+        class X:
+            def __init__(self, a, b):
+                self.a = a
+                self.b = b
+        #
+        def f(i, j):
+            # 'j' should be 0 or 1, not other values
+            if j > 0:
+                e = eraseX(X(i, j))
+            else:
+                try:
+                    e = erase_int(i)
+                except OverflowError:
+                    return -42
+            if j & 1:
+                x = uneraseX(e)
+                return x.a - x.b
+            else:
+                return unerase_int(e)
+        #
+        x = self.interp_operations(f, [-128, 0], taggedpointers=True)
+        assert x == -128
+        bigint = sys.maxint//2 + 1
+        x = self.interp_operations(f, [bigint, 0], taggedpointers=True)
+        assert x == -42
+        x = self.interp_operations(f, [1000, 1], taggedpointers=True)
+        assert x == 999
diff --git a/pypy/jit/metainterp/test/test_heapcache.py b/pypy/jit/metainterp/test/test_heapcache.py
--- a/pypy/jit/metainterp/test/test_heapcache.py
+++ b/pypy/jit/metainterp/test/test_heapcache.py
@@ -355,6 +355,14 @@
         assert not h.is_unescaped(box1)
         assert not h.is_unescaped(box2)
 
+    def test_circular_virtuals(self):
+        h = HeapCache()
+        h.new(box1)
+        h.new(box2)
+        h.invalidate_caches(rop.SETFIELD_GC, None, [box1, box2])
+        h.invalidate_caches(rop.SETFIELD_GC, None, [box2, box1])
+        h.invalidate_caches(rop.SETFIELD_GC, None, [box3, box1]) # does not crash
+
     def test_unescaped_array(self):
         h = HeapCache()
         h.new_array(box1, lengthbox1)
@@ -362,4 +370,4 @@
         h.invalidate_caches(rop.SETARRAYITEM_GC, None, [box1, index1, box2])
         assert h.is_unescaped(box1)
         h.invalidate_caches(rop.SETARRAYITEM_GC, None, [box2, index1, box1])
-        assert not h.is_unescaped(box1)
\ No newline at end of file
+        assert not h.is_unescaped(box1)
diff --git a/pypy/jit/metainterp/test/test_longlong.py b/pypy/jit/metainterp/test/test_longlong.py
--- a/pypy/jit/metainterp/test/test_longlong.py
+++ b/pypy/jit/metainterp/test/test_longlong.py
@@ -118,6 +118,26 @@
         res = self.interp_operations(f, [1000000000])
         assert res == 123500000000.0
 
+    def test_floats_negative(self):
+        def f(i):
+            # i == 1000000000
+            f = i * -123.5
+            n = r_longlong(f)
+            compare(n, -29, 1054051584)
+            return float(n)
+        res = self.interp_operations(f, [1000000000])
+        assert res == -123500000000.0
+
+    def test_floats_ulonglong(self):
+        def f(i):
+            # i == 1000000000
+            f = i * 12350000000.0
+            n = r_ulonglong(f)
+            compare(n, -1419508847, 538116096)
+            return float(n)
+        res = self.interp_operations(f, [1000000000])
+        assert res == 12350000000000000000.0
+
     def test_unsigned_compare_ops(self):
         def f(n1, n2):
             # n == 30002000000000
diff --git a/pypy/jit/metainterp/test/test_memmgr.py b/pypy/jit/metainterp/test/test_memmgr.py
--- a/pypy/jit/metainterp/test/test_memmgr.py
+++ b/pypy/jit/metainterp/test/test_memmgr.py
@@ -18,6 +18,7 @@
 
 class FakeLoopToken:
     generation = 0
+    invalidated = False
 
 
 class _TestMemoryManager:
diff --git a/pypy/jit/metainterp/test/test_quasiimmut.py b/pypy/jit/metainterp/test/test_quasiimmut.py
--- a/pypy/jit/metainterp/test/test_quasiimmut.py
+++ b/pypy/jit/metainterp/test/test_quasiimmut.py
@@ -48,6 +48,13 @@
 
 class QuasiImmutTests(object):
 
+    def setup_method(self, meth):
+        self.prev_compress_limit = QuasiImmut.compress_limit
+        QuasiImmut.compress_limit = 1
+
+    def teardown_method(self, meth):
+        QuasiImmut.compress_limit = self.prev_compress_limit
+
     def test_simple_1(self):
         myjitdriver = JitDriver(greens=['foo'], reds=['x', 'total'])
         class Foo:
@@ -289,7 +296,7 @@
             return total
 
         res = self.meta_interp(main, [])
-        self.check_loop_count(9)
+        self.check_tree_loop_count(6)
         assert res == main()
 
     def test_change_during_running(self):
@@ -317,7 +324,7 @@
         assert f(100, 15) == 3009
         res = self.meta_interp(f, [100, 15])
         assert res == 3009
-        self.check_loops(guard_not_invalidated=2, getfield_gc=0,
+        self.check_loops(guard_not_invalidated=4, getfield_gc=0,
                          call_may_force=0, guard_not_forced=0)
 
     def test_list_simple_1(self):
@@ -453,10 +460,30 @@
         assert f(100, 15) == 3009
         res = self.meta_interp(f, [100, 15])
         assert res == 3009
-        self.check_loops(guard_not_invalidated=2, getfield_gc=0,
+        self.check_loops(guard_not_invalidated=4, getfield_gc=0,
                          getarrayitem_gc=0, getarrayitem_gc_pure=0,
                          call_may_force=0, guard_not_forced=0)
 
+    def test_invalidated_loop_is_not_used_any_more_as_target(self):
+        myjitdriver = JitDriver(greens=['foo'], reds=['x'])
+        class Foo:
+            _immutable_fields_ = ['step?']
+        @dont_look_inside
+        def residual(x, foo):
+            if x == 20:
+                foo.step = 1
+        def f(x):
+            foo = Foo()
+            foo.step = 2
+            while x > 0:
+                myjitdriver.jit_merge_point(foo=foo, x=x)
+                residual(x, foo)
+                x -= foo.step
+            return foo.step
+        res = self.meta_interp(f, [60])
+        assert res == 1
+        self.check_tree_loop_count(4)   # at least not 2 like before
+
 
 class TestLLtypeGreenFieldsTests(QuasiImmutTests, LLJitMixin):
     pass
diff --git a/pypy/jit/metainterp/test/test_resume.py b/pypy/jit/metainterp/test/test_resume.py
--- a/pypy/jit/metainterp/test/test_resume.py
+++ b/pypy/jit/metainterp/test/test_resume.py
@@ -1,3 +1,4 @@
+from __future__ import with_statement
 import py
 from pypy.rpython.lltypesystem import lltype, llmemory, rffi
 from pypy.jit.metainterp.optimizeopt.optimizer import OptValue
@@ -180,22 +181,27 @@
     reader.consume_boxes(info, bi, br, bf)
     b1s = reader.liveboxes[0]
     b2s = reader.liveboxes[1]
-    assert bi == [b1s, ConstInt(111), b1s]
-    assert br == [ConstPtr(gcrefnull), b2s]
+    assert_same(bi, [b1s, ConstInt(111), b1s])
+    assert_same(br, [ConstPtr(gcrefnull), b2s])
     bi, br, bf = [None]*2, [None]*0, [None]*0
     info = MyBlackholeInterp([lltype.Signed,
                               lltype.Signed]).get_current_position_info()
     reader.consume_boxes(info, bi, br, bf)
-    assert bi == [ConstInt(222), ConstInt(333)]
+    assert_same(bi, [ConstInt(222), ConstInt(333)])
     bi, br, bf = [None]*2, [None]*1, [None]*0
     info = MyBlackholeInterp([lltype.Signed, llmemory.GCREF,
                               lltype.Signed]).get_current_position_info()
     reader.consume_boxes(info, bi, br, bf)
     b3s = reader.liveboxes[2]
-    assert bi == [b1s, b3s]
-    assert br == [b2s]
+    assert_same(bi, [b1s, b3s])
+    assert_same(br, [b2s])
     #
 
+def assert_same(list1, list2):
+    assert len(list1) == len(list2)
+    for b1, b2 in zip(list1, list2):
+        assert b1.same_box(b2)
+
 def test_simple_read_tagged_ints():
     storage = Storage()
     storage.rd_consts = []
@@ -576,8 +582,9 @@
 
 
 class FakeOptimizer_VirtualValue(object):
-    class cpu:
-        pass
+    class optimizer:
+        class cpu:
+            pass
 fakeoptimizer = FakeOptimizer_VirtualValue()
 
 def ConstAddr(addr, cpu):   # compatibility
@@ -1022,11 +1029,11 @@
     metainterp = MyMetaInterp()
     reader = ResumeDataFakeReader(storage, newboxes, metainterp)
     lst = reader.consume_boxes()
-    assert lst == [b1t, b1t, b3t]
+    assert_same(lst, [b1t, b1t, b3t])
     lst = reader.consume_boxes()
-    assert lst == [ConstInt(2), ConstInt(3)]
+    assert_same(lst, [ConstInt(2), ConstInt(3)])
     lst = reader.consume_boxes()
-    assert lst == [b1t, ConstInt(1), b1t, b1t]
+    assert_same(lst, [b1t, ConstInt(1), b1t, b1t])
     assert metainterp.trace == []    
 
 
@@ -1043,11 +1050,11 @@
     reader = ResumeDataFakeReader(storage, newboxes, metainterp)
     lst = reader.consume_boxes()
     c1t = ConstInt(111)
-    assert lst == [c1t, b2t, b3t]
+    assert_same(lst, [c1t, b2t, b3t])
     lst = reader.consume_boxes()
-    assert lst == [ConstInt(2), ConstInt(3)]
+    assert_same(lst, [ConstInt(2), ConstInt(3)])
     lst = reader.consume_boxes()
-    assert lst == [c1t, ConstInt(1), c1t, b2t]
+    assert_same(lst, [c1t, ConstInt(1), c1t, b2t])
     assert metainterp.trace == []
 
 
@@ -1113,9 +1120,11 @@
     # check that we get the operations in 'expected', in a possibly different
     # order.
     assert len(trace) == len(expected)
-    for x in trace:
-        assert x in expected
-        expected.remove(x)
+    with CompareableConsts():
+        for x in trace:
+            assert x in expected
+            expected.remove(x)
+
     ptr = b2t.value._obj.container._as_ptr()
     assert lltype.typeOf(ptr) == lltype.Ptr(LLtypeMixin.NODE)
     assert ptr.value == 111
@@ -1125,6 +1134,18 @@
     assert ptr2.parent.value == 33
     assert ptr2.parent.next == ptr
 
+class CompareableConsts(object):
+    def __init__(self):
+        self.oldeq = None
+        
+    def __enter__(self):
+        assert self.oldeq is None
+        self.oldeq = Const.__eq__
+        Const.__eq__ = Const.same_box
+        
+    def __exit__(self, type, value, traceback):
+        Const.__eq__ = self.oldeq
+
 def test_virtual_adder_make_varray():
     b2s, b4s = [BoxPtr(), BoxInt(4)]
     c1s = ConstInt(111)
@@ -1135,12 +1156,7 @@
     modifier.liveboxes = {}
     modifier.vfieldboxes = {}
 
-    class FakeOptimizer(object):
-        class cpu:
-            pass
-        def new_const_item(self, descr):
-            return None
-    v2 = VArrayValue(FakeOptimizer(), LLtypeMixin.arraydescr, 2, b2s)
+    v2 = VArrayValue(LLtypeMixin.arraydescr, None, 2, b2s)
     v2._items = [b4s, c1s]
     modifier.register_virtual_fields(b2s, [b4s, c1s])
     liveboxes = []
@@ -1167,8 +1183,9 @@
         (rop.SETARRAYITEM_GC, [b2t,ConstInt(1), c1s], None,
                               LLtypeMixin.arraydescr),
         ]
-    for x, y in zip(expected, trace):
-        assert x == y
+    with CompareableConsts():
+        for x, y in zip(expected, trace):
+            assert x == y
     #
     ptr = b2t.value._obj.container._as_ptr()
     assert lltype.typeOf(ptr) == lltype.Ptr(lltype.GcArray(lltype.Signed))
@@ -1211,8 +1228,9 @@
         (rop.SETFIELD_GC, [b2t, c1s],  None, LLtypeMixin.adescr),
         (rop.SETFIELD_GC, [b2t, b4t], None, LLtypeMixin.bdescr),
         ]
-    for x, y in zip(expected, trace):
-        assert x == y
+    with CompareableConsts():
+        for x, y in zip(expected, trace):
+            assert x == y
     #
     ptr = b2t.value._obj.container._as_ptr()
     assert lltype.typeOf(ptr) == lltype.Ptr(LLtypeMixin.S)
diff --git a/pypy/jit/metainterp/test/test_string.py b/pypy/jit/metainterp/test/test_string.py
--- a/pypy/jit/metainterp/test/test_string.py
+++ b/pypy/jit/metainterp/test/test_string.py
@@ -3,7 +3,8 @@
 from pypy.jit.codewriter.policy import StopAtXPolicy
 from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin
 from pypy.rlib.debug import debug_print
-from pypy.rlib.jit import JitDriver, dont_look_inside, we_are_jitted
+from pypy.rlib.jit import JitDriver, dont_look_inside, we_are_jitted,\
+     promote_string
 from pypy.rlib.rstring import StringBuilder
 from pypy.rpython.ootypesystem import ootype
 
@@ -484,6 +485,42 @@
             return len(sa.val)
         assert self.meta_interp(f, ['a']) == f('a')
 
+    def test_string_comepare_quasiimmutable(self):
+        class Sys(object):
+            _immutable_fields_ = ["defaultencoding?"]
+            def __init__(self, s):
+                self.defaultencoding = s
+        _str = self._str
+        sys = Sys(_str('ascii'))        
+        mydriver = JitDriver(reds = ['n', 'sa'], greens = [])
+        def f(n):
+            sa = 0
+            sys.defaultencoding = _str('ascii')
+            while n:
+                mydriver.jit_merge_point(n=n, sa=sa)
+                if sys.defaultencoding == _str('ascii'):
+                    sa += 1
+                n -= 1
+            sys.defaultencoding = _str('utf-8')
+            return sa
+        assert self.meta_interp(f, [8]) == f(8)
+        self.check_loops({'int_add': 1, 'guard_true': 1, 'int_sub': 1,
+                          'jump': 1, 'int_is_true': 1,
+                          'guard_not_invalidated': 1})
+
+    def test_promote_string(self):
+        driver = JitDriver(greens = [], reds = ['n'])
+        
+        def f(n):
+            while n < 21:
+                driver.jit_merge_point(n=n)
+                promote_string(str(n % 3))
+                n += 1
+            return 0
+
+        self.meta_interp(f, [0])
+        self.check_loops(call=3 + 1) # one for int2str
+
 #class TestOOtype(StringTests, OOJitMixin):
 #    CALL = "oosend"
 #    CALL_PURE = "oosend_pure"
diff --git a/pypy/jit/metainterp/test/test_virtualref.py b/pypy/jit/metainterp/test/test_virtualref.py
--- a/pypy/jit/metainterp/test/test_virtualref.py
+++ b/pypy/jit/metainterp/test/test_virtualref.py
@@ -3,6 +3,7 @@
 from pypy.rpython.llinterp import LLException
 from pypy.rlib.jit import JitDriver, dont_look_inside, vref_None
 from pypy.rlib.jit import virtual_ref, virtual_ref_finish, InvalidVirtualRef
+from pypy.rlib.jit import non_virtual_ref
 from pypy.rlib.objectmodel import compute_unique_id
 from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin, _get_jitcodes
 from pypy.jit.metainterp.resoperation import rop
@@ -595,6 +596,65 @@
         res = self.meta_interp(fn, [10])
         assert res == 6
 
+    def test_is_virtual(self):
+        myjitdriver = JitDriver(greens=[], reds=['n', 'res1'])
+        class X:
+            pass
+        @dont_look_inside
+        def residual(vref):
+            return vref.virtual
+        #
+        def f(n):
+            res1 = -42
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, res1=res1)
+                x = X()
+                vref = virtual_ref(x)
+                res1 = residual(vref)
+                virtual_ref_finish(vref, x)
+                n -= 1
+            return res1
+        #
+        res = self.meta_interp(f, [10])
+        assert res == 1
+
+    def test_is_not_virtual_none(self):
+        myjitdriver = JitDriver(greens=[], reds=['n', 'res1'])
+        @dont_look_inside
+        def residual(vref):
+            return vref.virtual
+        #
+        def f(n):
+            res1 = -42
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, res1=res1)
+                res1 = residual(vref_None)
+                n -= 1
+            return res1
+        #
+        res = self.meta_interp(f, [10])
+        assert res == 0
+
+    def test_is_not_virtual_non_none(self):
+        myjitdriver = JitDriver(greens=[], reds=['n', 'res1'])
+        class X:
+            pass
+        @dont_look_inside
+        def residual(vref):
+            return vref.virtual
+        #
+        def f(n):
+            res1 = -42
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, res1=res1)
+                x = X()
+                res1 = residual(non_virtual_ref(x))
+                n -= 1
+            return res1
+        #
+        res = self.meta_interp(f, [10])
+        assert res == 0
+
 
 class TestLLtype(VRefTests, LLJitMixin):
     pass
diff --git a/pypy/jit/metainterp/test/test_virtualstate.py b/pypy/jit/metainterp/test/test_virtualstate.py
--- a/pypy/jit/metainterp/test/test_virtualstate.py
+++ b/pypy/jit/metainterp/test/test_virtualstate.py
@@ -431,7 +431,7 @@
         
 
 class BaseTestBridges(BaseTest):
-    enable_opts = "intbounds:rewrite:virtualize:string:heap:unroll"
+    enable_opts = "intbounds:rewrite:virtualize:string:pure:heap:unroll"
 
     def _do_optimize_bridge(self, bridge, call_pure_results):
         from pypy.jit.metainterp.optimizeopt import optimize_bridge_1, build_opt_chain
diff --git a/pypy/jit/metainterp/virtualref.py b/pypy/jit/metainterp/virtualref.py
--- a/pypy/jit/metainterp/virtualref.py
+++ b/pypy/jit/metainterp/virtualref.py
@@ -39,6 +39,7 @@
     def replace_force_virtual_with_call(self, graphs):
         # similar to rvirtualizable2.replace_force_virtualizable_with_call().
         c_force_virtual_ptr = None
+        c_is_virtual_ptr = None
         force_virtual_count = 0
         for graph in graphs:
             for block in graph.iterblocks():
@@ -52,6 +53,13 @@
                         op.opname = 'direct_call'
                         op.args = [c_force_virtual_ptr, op.args[0]]
                         force_virtual_count += 1
+                    #
+                    if op.opname == 'jit_is_virtual':
+                        if c_is_virtual_ptr is None:
+                            c_is_virtual_ptr = self.get_is_virtual_fnptr()
+                        #
+                        op.opname = 'direct_call'
+                        op.args = [c_is_virtual_ptr, op.args[0]]
         #
         if c_force_virtual_ptr is not None:
             log("replaced %d 'jit_force_virtual' with %r" % (force_virtual_count,
@@ -129,6 +137,17 @@
             force_virtual_if_necessary)
         return inputconst(lltype.typeOf(funcptr), funcptr)
 
+    def get_is_virtual_fnptr(self):
+        #
+        def is_virtual(inst):
+            if not inst:
+                return False
+            return inst.typeptr == self.jit_virtual_ref_vtable
+        #
+        FUNC = lltype.FuncType([rclass.OBJECTPTR], lltype.Bool)
+        funcptr = self.warmrunnerdesc.helper_func(lltype.Ptr(FUNC), is_virtual)
+        return inputconst(lltype.typeOf(funcptr), funcptr)
+
     def force_virtual(self, inst):
         vref = lltype.cast_pointer(lltype.Ptr(self.JIT_VIRTUAL_REF), inst)
         token = vref.virtual_token
diff --git a/pypy/jit/metainterp/warmstate.py b/pypy/jit/metainterp/warmstate.py
--- a/pypy/jit/metainterp/warmstate.py
+++ b/pypy/jit/metainterp/warmstate.py
@@ -178,7 +178,7 @@
         if self.compiled_merge_points_wref is not None:
             for wref in self.compiled_merge_points_wref:
                 looptoken = wref()
-                if looptoken is not None:
+                if looptoken is not None and not looptoken.invalidated:
                     result.append(looptoken)
         return result
 
diff --git a/pypy/module/__builtin__/__init__.py b/pypy/module/__builtin__/__init__.py
--- a/pypy/module/__builtin__/__init__.py
+++ b/pypy/module/__builtin__/__init__.py
@@ -20,6 +20,9 @@
         'any'           : 'app_functional.any',
         'all'           : 'app_functional.all',
         'sum'           : 'app_functional.sum',
+        'map'           : 'app_functional.map',
+        'reduce'        : 'app_functional.reduce',
+        'filter'        : 'app_functional.filter',
         'vars'          : 'app_inspect.vars',
         'dir'           : 'app_inspect.dir',
 
@@ -86,11 +89,8 @@
         'enumerate'     : 'functional.W_Enumerate',
         'min'           : 'functional.min',
         'max'           : 'functional.max',
-        'map'           : 'functional.map',
         'zip'           : 'functional.zip',
-        'reduce'        : 'functional.reduce',
         'reversed'      : 'functional.reversed',
-        'filter'        : 'functional.filter',
         'super'         : 'descriptor.W_Super',
         'staticmethod'  : 'descriptor.StaticMethod',
         'classmethod'   : 'descriptor.ClassMethod',
@@ -119,7 +119,7 @@
            builtin = space.interpclass_w(w_builtin)
            if isinstance(builtin, module.Module):
                return builtin
-       # no builtin! make a default one.  Given them None, at least.
+       # no builtin! make a default one.  Give them None, at least.
        builtin = module.Module(space, None)
        space.setitem(builtin.w_dict, space.wrap('None'), space.w_None)
        return builtin
diff --git a/pypy/module/__builtin__/app_functional.py b/pypy/module/__builtin__/app_functional.py
--- a/pypy/module/__builtin__/app_functional.py
+++ b/pypy/module/__builtin__/app_functional.py
@@ -48,4 +48,118 @@
         # Very intentionally *not* +=, that would have different semantics if
         # start was a mutable type, such as a list
         last = last + x
-    return last
\ No newline at end of file
+    return last
+
+def map(func, *collections):
+    """map(function, sequence[, sequence, ...]) -> list
+
+Return a list of the results of applying the function to the items of
+the argument sequence(s).  If more than one sequence is given, the
+function is called with an argument list consisting of the corresponding
+item of each sequence, substituting None for missing values when not all
+sequences have the same length.  If the function is None, return a list of
+the items of the sequence (or a list of tuples if more than one sequence)."""
+    if not collections:
+        raise TypeError("map() requires at least two arguments")
+    num_collections = len(collections)
+    none_func = func is None
+    if num_collections == 1:
+        if none_func:
+            return list(collections[0])
+        else:
+            # Special case for the really common case of a single collection,
+            # this can be eliminated if we could unroll that loop that creates
+            # `args` based on whether or not len(collections) was constant
+            result = []
+            for item in collections[0]:
+                result.append(func(item))
+            return result
+    result = []
+    # Pair of (iterator, has_finished)
+    iterators = [(iter(seq), False) for seq in collections]
+    while True:
+        cont = False
+        args = []
+        for idx, (iterator, has_finished) in enumerate(iterators):
+            val = None
+            if not has_finished:
+                try:
+                    val = next(iterator)
+                except StopIteration:
+                    iterators[idx] = (None, True)
+                else:
+                    cont = True
+            args.append(val)
+        args = tuple(args)
+        if cont:
+            if none_func:
+                result.append(args)
+            else:
+                result.append(func(*args))
+        else:
+            return result
+
+sentinel = object()
+
+def reduce(func, sequence, initial=sentinel):
+    """reduce(function, sequence[, initial]) -> value
+
+Apply a function of two arguments cumulatively to the items of a sequence,
+from left to right, so as to reduce the sequence to a single value.
+For example, reduce(lambda x, y: x+y, [1, 2, 3, 4, 5]) calculates
+((((1+2)+3)+4)+5).  If initial is present, it is placed before the items
+of the sequence in the calculation, and serves as a default when the
+sequence is empty."""
+    iterator = iter(sequence)
+    if initial is sentinel:
+        try:
+            initial = next(iterator)
+        except StopIteration:
+            raise TypeError("reduce() of empty sequence with no initial value")
+    result = initial
+    for item in iterator:
+        result = func(result, item)
+    return result
+
+def filter(func, seq):
+    """filter(function or None, sequence) -> list, tuple, or string
+
+Return those items of sequence for which function(item) is true.  If
+function is None, return the items that are true.  If sequence is a tuple
+or string, return the same type, else return a list."""
+    if func is None:
+        func = bool
+    if isinstance(seq, str):
+        return _filter_string(func, seq, str)
+    elif isinstance(seq, unicode):
+        return _filter_string(func, seq, unicode)
+    elif isinstance(seq, tuple):
+        return _filter_tuple(func, seq)
+    result = []
+    for item in seq:
+        if func(item):
+            result.append(item)
+    return result
+
+def _filter_string(func, string, str_type):
+    if func is bool and type(string) is str_type:
+        return string
+    result = []
+    for i in range(len(string)):
+        # You must call __getitem__ on the strings, simply iterating doesn't
+        # work :/
+        item = string[i]
+        if func(item):
+            if not isinstance(item, str_type):
+                raise TypeError("__getitem__ returned a non-string type")
+            result.append(item)
+    return str_type().join(result)
+
+def _filter_tuple(func, seq):
+    result = []
+    for i in range(len(seq)):
+        # Again, must call __getitem__, at least there are tests.
+        item = seq[i]
+        if func(item):
+            result.append(item)
+    return tuple(result)
\ No newline at end of file
diff --git a/pypy/module/__builtin__/compiling.py b/pypy/module/__builtin__/compiling.py
--- a/pypy/module/__builtin__/compiling.py
+++ b/pypy/module/__builtin__/compiling.py
@@ -84,8 +84,8 @@
         raise OperationError(space.w_TypeError,
               w('eval() arg 1 must be a string or code object'))
 
-    caller = space.getexecutioncontext().gettopframe_nohidden()
     if space.is_w(w_globals, space.w_None):
+        caller = space.getexecutioncontext().gettopframe_nohidden()
         if caller is None:
             w_globals = space.newdict()
             if space.is_w(w_locals, space.w_None):
@@ -97,13 +97,9 @@
     elif space.is_w(w_locals, space.w_None):
         w_locals = w_globals
 
-    try:
-        space.getitem(w_globals, space.wrap('__builtins__'))
-    except OperationError, e:
-        if not e.match(space, space.w_KeyError):
-            raise
-        if caller is not None:
-            w_builtin = space.builtin.pick_builtin(caller.w_globals)
-            space.setitem(w_globals, space.wrap('__builtins__'), w_builtin)
+    # xxx removed: adding '__builtins__' to the w_globals dict, if there
+    # is none.  This logic was removed as costly (it requires to get at
+    # the gettopframe_nohidden()).  I bet no test fails, and it's a really
+    # obscure case.
 
     return codeobj.exec_code(space, w_globals, w_locals)
diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py
--- a/pypy/module/__builtin__/functional.py
+++ b/pypy/module/__builtin__/functional.py
@@ -193,124 +193,14 @@
     return min_max(space, __args__, "max")
 
 def min(space, __args__):
-    """Return the smallest item in a sequence.
+    """min(iterable[, key=func]) -> value
+    min(a, b, c, ...[, key=func]) -> value
 
-    If more than one argument is passed, return the minimum of them.
+    With a single iterable argument, return its smallest item.
+    With two or more arguments, return the smallest argument.
     """
     return min_max(space, __args__, "min")
 
- at unwrap_spec(collections_w="args_w")
-def map(space, w_func, collections_w):
-    """does 3 separate things, hence this enormous docstring.
-       1.  if function is None, return a list of tuples, each with one
-           item from each collection.  If the collections have different
-           lengths,  shorter ones are padded with None.
-
-       2.  if function is not None, and there is only one collection,
-           apply function to every item in the collection and return a
-           list of the results.
-
-       3.  if function is not None, and there are several collections,
-           repeatedly call the function with one argument from each
-           collection.  If the collections have different lengths,
-           shorter ones are padded with None
-    """
-    if not collections_w:
-        msg = "map() requires at least two arguments"
-        raise OperationError(space.w_TypeError, space.wrap(msg))
-    none_func = space.is_w(w_func, space.w_None)
-    if len(collections_w) == 1:
-        w_collection = collections_w[0]
-        if none_func:
-            result_w = space.unpackiterable(w_collection)
-        else:
-            result_w = map_single_collection(space, w_func, w_collection)
-    else:
-        result_w = map_multiple_collections(space, w_func, collections_w,
-                                            none_func)
-    return space.newlist(result_w)
-
-def map_single_collection(space, w_func, w_collection):
-    """Special case for 'map(func, coll)', where 'func' is not None and there
-    is only one 'coll' argument."""
-    w_iter = space.iter(w_collection)
-    # xxx special hacks for speed
-    from pypy.interpreter import function, pycode
-    if isinstance(w_func, function.Function):
-        # xxx compatibility issue: what if func_code is modified in the
-        # middle of running map()??  That's far too obscure for me to care...
-        code = w_func.getcode()
-        fast_natural_arity = code.fast_natural_arity
-        if fast_natural_arity == (1|pycode.PyCode.FLATPYCALL):
-            assert isinstance(code, pycode.PyCode)
-            return map_single_user_function(code, w_func, w_iter)
-    # /xxx end of special hacks
-    return map_single_other_callable(space, w_func, w_iter)
-
-def map_single_other_callable(space, w_func, w_iter):
-    result_w = []
-    while True:
-        try:
-            w_item = space.next(w_iter)
-        except OperationError, e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            break
-        result_w.append(space.call_function(w_func, w_item))
-    return result_w
-map_single_other_callable._dont_inline_ = True
-
-from pypy.rlib.jit import JitDriver
-mapjitdriver = JitDriver(greens = ['code'],
-                         reds = ['w_func', 'w_iter', 'result_w'])
-def map_single_user_function(code, w_func, w_iter):
-    result_w = []
-    while True:
-        mapjitdriver.can_enter_jit(code=code, w_func=w_func,
-                                   w_iter=w_iter, result_w=result_w)
-        mapjitdriver.jit_merge_point(code=code, w_func=w_func,
-                                     w_iter=w_iter, result_w=result_w)
-        space = w_func.space
-        try:
-            w_item = space.next(w_iter)
-        except OperationError, e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            break
-        new_frame = space.createframe(code, w_func.w_func_globals,
-                                      w_func)
-        new_frame.locals_stack_w[0] = w_item
-        w_res = new_frame.run()
-        result_w.append(w_res)
-    return result_w
-
-def map_multiple_collections(space, w_func, collections_w, none_func):
-    result_w = []
-    iterators_w = [space.iter(w_seq) for w_seq in collections_w]
-    num_iterators = len(iterators_w)
-    while True:
-        cont = False
-        args_w = [space.w_None] * num_iterators
-        for i in range(num_iterators):
-            if iterators_w[i] is not None:
-                try:
-                    args_w[i] = space.next(iterators_w[i])
-                except OperationError, e:
-                    if not e.match(space, space.w_StopIteration):
-                        raise
-                    iterators_w[i] = None
-                else:
-                    cont = True
-        if not cont:
-            break
-        w_args = space.newtuple(args_w)
-        if none_func:
-            w_res = w_args
-        else:
-            w_res = space.call(w_func, w_args)
-        result_w.append(w_res)
-    return result_w
-
 @unwrap_spec(sequences_w="args_w")
 def zip(space, sequences_w):
     """Return a list of tuples, where the nth tuple contains every nth item of
@@ -332,90 +222,6 @@
             return space.newlist(result_w)
         result_w.append(space.newtuple(items_w))
 
-def reduce(space, w_func, w_sequence, w_initial=NoneNotWrapped):
-    """ Apply function of two arguments cumulatively to the items of sequence,
-        from left to right, so as to reduce the sequence to a single value.
-        Optionally begin with an initial value.
-    """
-    w_iter = space.iter(w_sequence)
-    if w_initial is None:
-        try:
-            w_initial = space.next(w_iter)
-        except OperationError, e:
-            if e.match(space, space.w_StopIteration):
-                msg = "reduce() of empty sequence with no initial value"
-                raise OperationError(space.w_TypeError, space.wrap(msg))
-            raise
-    w_result = w_initial
-    while True:
-        try:
-            w_next = space.next(w_iter)
-        except OperationError, e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            break
-        w_result = space.call_function(w_func, w_result, w_next)
-    return w_result
-
-def filter(space, w_func, w_seq):
-    """construct a list of those elements of collection for which function
-       is True.  If function is None, then return the items in the sequence
-       which are True.
-    """
-    if space.is_true(space.isinstance(w_seq, space.w_str)):
-        return _filter_string(space, w_func, w_seq, space.w_str)
-    if space.is_true(space.isinstance(w_seq, space.w_unicode)):
-        return _filter_string(space, w_func, w_seq, space.w_unicode)
-    if space.is_true(space.isinstance(w_seq, space.w_tuple)):
-        return _filter_tuple(space, w_func, w_seq)
-    w_iter = space.iter(w_seq)
-    result_w = []
-    none_func = space.is_w(w_func, space.w_None)
-    while True:
-        try:
-            w_next = space.next(w_iter)
-        except OperationError, e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            break
-        if none_func:
-            w_keep = w_next
-        else:
-            w_keep = space.call_function(w_func, w_next)
-        if space.is_true(w_keep):
-            result_w.append(w_next)
-    return space.newlist(result_w)
-
-def _filter_tuple(space, w_func, w_tuple):
-    none_func = space.is_w(w_func, space.w_None)
-    length = space.len_w(w_tuple)
-    result_w = []
-    for i in range(length):
-        w_item = space.getitem(w_tuple, space.wrap(i))
-        if none_func:
-            w_keep = w_item
-        else:
-            w_keep = space.call_function(w_func, w_item)
-        if space.is_true(w_keep):
-            result_w.append(w_item)
-    return space.newtuple(result_w)
-
-def _filter_string(space, w_func, w_string, w_str_type):
-    none_func = space.is_w(w_func, space.w_None)
-    if none_func and space.is_w(space.type(w_string), w_str_type):
-        return w_string
-    length = space.len_w(w_string)
-    result_w = []
-    for i in range(length):
-        w_item = space.getitem(w_string, space.wrap(i))
-        if none_func or space.is_true(space.call_function(w_func, w_item)):
-            if not space.is_true(space.isinstance(w_item, w_str_type)):
-                msg = "__getitem__ returned a non-string type"
-                raise OperationError(space.w_TypeError, space.wrap(msg))
-            result_w.append(w_item)
-    w_empty = space.call_function(w_str_type)
-    return space.call_method(w_empty, "join", space.newlist(result_w))
-
 class W_Enumerate(Wrappable):
 
     def __init__(self, w_iter, w_start):
diff --git a/pypy/module/__builtin__/test/test_functional.py b/pypy/module/__builtin__/test/test_functional.py
--- a/pypy/module/__builtin__/test/test_functional.py
+++ b/pypy/module/__builtin__/test/test_functional.py
@@ -147,7 +147,7 @@
        assert list(xrange(A())) == [0, 1, 2, 3, 4]
        assert list(xrange(0, A())) == [0, 1, 2, 3, 4]
        assert list(xrange(0, 10, A())) == [0, 5]
- 
+
    def test_xrange_float(self):
       assert list(xrange(0.1, 2.0, 1.1)) == [0, 1]
 
diff --git a/pypy/module/__pypy__/interp_builders.py b/pypy/module/__pypy__/interp_builders.py
--- a/pypy/module/__pypy__/interp_builders.py
+++ b/pypy/module/__pypy__/interp_builders.py
@@ -16,7 +16,8 @@
 
         def _check_done(self, space):
             if self.builder is None:
-                raise OperationError(space.w_ValueError, space.wrap("Can't operate on a done builder"))
+                raise OperationError(space.w_ValueError, space.wrap(
+                        "Can't operate on a built builder"))
 
         @unwrap_spec(size=int)
         def descr__new__(space, w_subtype, size=-1):
@@ -31,7 +32,8 @@
         def descr_append_slice(self, space, s, start, end):
             self._check_done(space)
             if not 0 <= start <= end <= len(s):
-                raise OperationError(space.w_ValueError, space.wrap("bad start/stop"))
+                raise OperationError(space.w_ValueError, space.wrap(
+                        "bad start/stop"))
             self.builder.append_slice(s, start, end)
 
         def descr_build(self, space):
@@ -40,6 +42,12 @@
             self.builder = None
             return w_s
 
+        def descr_len(self, space):
+            if self.builder is None:
+                raise OperationError(space.w_ValueError, space.wrap(
+                        "no length of built builder"))
+            return space.wrap(self.builder.getlength())
+
     W_Builder.__name__ = "W_%s" % name
     W_Builder.typedef = TypeDef(name,
         __new__ = interp2app(func_with_new_name(
@@ -48,6 +56,7 @@
         append = interp2app(W_Builder.descr_append),
         append_slice = interp2app(W_Builder.descr_append_slice),
         build = interp2app(W_Builder.descr_build),
+        __len__ = interp2app(W_Builder.descr_len),
     )
     W_Builder.typedef.acceptable_as_base_class = False
     return W_Builder
diff --git a/pypy/module/__pypy__/test/test_builders.py b/pypy/module/__pypy__/test/test_builders.py
--- a/pypy/module/__pypy__/test/test_builders.py
+++ b/pypy/module/__pypy__/test/test_builders.py
@@ -38,7 +38,9 @@
         b = StringBuilder()
         b.append("abc")
         b.append("123")
+        assert len(b) == 6
         b.append("you and me")
         s = b.build()
+        raises(ValueError, len, b)
         assert s == "abc123you and me"
-        raises(ValueError, b.build)
\ No newline at end of file
+        raises(ValueError, b.build)
diff --git a/pypy/module/_continuation/interp_continuation.py b/pypy/module/_continuation/interp_continuation.py
--- a/pypy/module/_continuation/interp_continuation.py
+++ b/pypy/module/_continuation/interp_continuation.py
@@ -19,6 +19,11 @@
         #  - normal:      self.sthread != None, not is_empty_handle(self.h)
         #  - finished:    self.sthread != None, is_empty_handle(self.h)
 
+    def __del__(self):
+        sthread = self.sthread
+        if sthread is not None and not sthread.is_empty_handle(self.h):
+            sthread.destroy(self.h)
+
     def check_sthread(self):
         ec = self.space.getexecutioncontext()
         if ec.stacklet_thread is not self.sthread:
@@ -28,6 +33,8 @@
     def descr_init(self, w_callable, __args__):
         if self.sthread is not None:
             raise geterror(self.space, "continulet already __init__ialized")
+        sthread = build_sthread(self.space)
+        workaround_disable_jit(sthread)
         #
         # hackish: build the frame "by hand", passing it the correct arguments
         space = self.space
@@ -41,7 +48,6 @@
         self.bottomframe = bottomframe
         #
         global_state.origin = self
-        sthread = build_sthread(self.space)
         self.sthread = sthread
         h = sthread.new(new_stacklet_callback)
         post_switch(sthread, h)
@@ -71,6 +77,7 @@
                 global_state.clear()
                 raise geterror(self.space, "continulet already finished")
         self.check_sthread()
+        workaround_disable_jit(self.sthread)
         #
         global_state.origin = self
         if to is None:
@@ -259,6 +266,16 @@
         sthread = ec.stacklet_thread = SThread(space, ec)
     return sthread
 
+def workaround_disable_jit(sthread):
+    # A bad workaround to kill the JIT anywhere in this thread.
+    # This forces all the frames.  It's a bad workaround because
+    # it takes O(depth) time, and it will cause some "abort:
+    # vable escape" in the JIT.  The goal is to prevent any frame
+    # from being still virtuals, because the JIT generates code
+    # to un-virtualizable them "on demand" by loading values based
+    # on FORCE_TOKEN, which is an address in the stack.
+    sthread.ec.force_all_frames()
+
 # ____________________________________________________________
 
 def permute(space, args_w):
diff --git a/pypy/module/_file/test/test_file.py b/pypy/module/_file/test/test_file.py
--- a/pypy/module/_file/test/test_file.py
+++ b/pypy/module/_file/test/test_file.py
@@ -1,5 +1,5 @@
 from __future__ import with_statement
-import py
+import py, os, errno
 
 from pypy.conftest import gettestobjspace, option
 
@@ -257,6 +257,39 @@
             assert self.temppath in g.getvalue()
 
 
+class AppTestNonblocking(object):
+    def setup_class(cls):
+        from pypy.module._file.interp_file import W_File
+
+        cls.old_read = os.read
+
+        if option.runappdirect:
+            py.test.skip("works with internals of _file impl on py.py")
+
+        state = [0]
+        def read(fd, n=None):
+            if fd != 42:
+                return cls.old_read(fd, n)
+            if state[0] == 0:
+                state[0] += 1
+                return "xyz"
+            if state[0] < 3:
+                state[0] += 1
+                raise OSError(errno.EAGAIN, "xyz")
+            return ''
+        os.read = read
+        stdin = W_File(cls.space)
+        stdin.file_fdopen(42, "r", 1)
+        stdin.name = '<stdin>'
+        cls.w_stream = stdin
+
+    def teardown_class(cls):
+        os.read = cls.old_read
+
+    def test_nonblocking_file(self):
+        res = self.stream.read()
+        assert res == 'xyz'
+
 class AppTestConcurrency(object):
     # these tests only really make sense on top of a translated pypy-c,
     # because on top of py.py the inner calls to os.write() don't
@@ -374,24 +407,24 @@
         with self.file(self.temppath, 'w') as f:
             f.write('foo')
         assert f.closed
-        
+
         with self.file(self.temppath, 'r') as f:
             s = f.readline()
 
         assert s == "foo"
         assert f.closed
-    
+
     def test_subclass_with(self):
         file = self.file
         class C(file):
             def __init__(self, *args, **kwargs):
                 self.subclass_closed = False
                 file.__init__(self, *args, **kwargs)
-            
+
             def close(self):
                 self.subclass_closed = True
                 file.close(self)
-        
+
         with C(self.temppath, 'w') as f:
             pass
         assert f.subclass_closed
diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -23,6 +23,8 @@
         ("arccos", "arccos"),
         ("arcsin", "arcsin"),
         ("arctan", "arctan"),
+        ("arcsinh", "arcsinh"),
+        ("arctanh", "arctanh"),
         ("copysign", "copysign"),
         ("cos", "cos"),
         ("divide", "divide"),
@@ -50,4 +52,6 @@
     appleveldefs = {
         'average': 'app_numpy.average',
         'mean': 'app_numpy.mean',
+        'inf': 'app_numpy.inf',
+        'e': 'app_numpy.e',
     }
diff --git a/pypy/module/micronumpy/app_numpy.py b/pypy/module/micronumpy/app_numpy.py
--- a/pypy/module/micronumpy/app_numpy.py
+++ b/pypy/module/micronumpy/app_numpy.py
@@ -1,5 +1,11 @@
+import math
+
 import numpy
 
+
+inf = float("inf")
+e = math.e
+
 def average(a):
     # This implements a weighted average, for now we don't implement the
     # weighting, just the average part!
@@ -8,4 +14,4 @@
 def mean(a):
     if not hasattr(a, "mean"):
         a = numpy.array(a)
-    return a.mean()
\ No newline at end of file
+    return a.mean()
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -7,13 +7,14 @@
 from pypy.interpreter.typedef import TypeDef, interp_attrproperty, GetSetProperty
 from pypy.module.micronumpy import signature
 from pypy.objspace.std.floatobject import float2string
-from pypy.rlib import rfloat
-from pypy.rlib.rarithmetic import widen
+from pypy.rlib import rarithmetic, rfloat
+from pypy.rlib.rarithmetic import LONG_BIT, widen
 from pypy.rlib.objectmodel import specialize, enforceargs
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.rpython.lltypesystem import lltype, rffi
 
 
+UNSIGNEDLTR = "u"
 SIGNEDLTR = "i"
 BOOLLTR = "b"
 FLOATINGLTR = "f"
@@ -61,7 +62,10 @@
             self.val = val
 
         def wrap(self, space):
-            return space.wrap(self.val)
+            val = self.val
+            if valtype is rarithmetic.r_singlefloat:
+                val = float(val)
+            return space.wrap(val)
 
         def convert_to(self, dtype):
             return dtype.adapt_val(self.val)
@@ -145,7 +149,7 @@
         return self.adapt_val(func(self, self.for_computation(self.unbox(v))))
     return impl
 
-class ArithmaticTypeMixin(object):
+class ArithmeticTypeMixin(object):
     _mixin_ = True
 
     @binop
@@ -157,9 +161,6 @@
     @binop
     def mul(self, v1, v2):
         return v1 * v2
-    @binop
-    def div(self, v1, v2):
-        return v1 / v2
 
     @unaryop
     def pos(self, v):
@@ -200,12 +201,26 @@
         return v1 >= v2
 
 
-class FloatArithmeticDtype(ArithmaticTypeMixin):
+class FloatArithmeticDtype(ArithmeticTypeMixin):
     _mixin_ = True
 
+    def unwrap(self, space, w_item):
+        return self.adapt_val(space.float_w(space.float(w_item)))
+
     def for_computation(self, v):
-        return v
+        return float(v)
 
+    def str_format(self, item):
+        return float2string(self.for_computation(self.unbox(item)), 'g', rfloat.DTSF_STR_PRECISION)
+
+    @binop
+    def div(self, v1, v2):
+        try:
+            return v1 / v2
+        except ZeroDivisionError:
+            if v1 == v2 == 0.0:
+                return rfloat.NAN
+            return rfloat.copysign(rfloat.INFINITY, v1 * v2)
     @binop
     def mod(self, v1, v2):
         return math.fmod(v1, v2)
@@ -250,19 +265,29 @@
         return math.tan(v)
     @unaryop
     def arcsin(self, v):
-        if v < -1.0 or v > 1.0:
+        if not -1.0 <= v <= 1.0:
             return rfloat.NAN
         return math.asin(v)
     @unaryop
     def arccos(self, v):
-        if v < -1.0 or v > 1.0:
+        if not -1.0 <= v <= 1.0:
             return rfloat.NAN
         return math.acos(v)
     @unaryop
     def arctan(self, v):
         return math.atan(v)
+    @unaryop
+    def arcsinh(self, v):
+        return math.asinh(v)
+    @unaryop
+    def arctanh(self, v):
+        if v == 1.0 or v == -1.0:
+            return math.copysign(rfloat.INFINITY, v)
+        if not -1.0 < v < 1.0:
+            return rfloat.NAN
+        return math.atanh(v)
 
-class IntegerArithmeticDtype(ArithmaticTypeMixin):
+class IntegerArithmeticDtype(ArithmeticTypeMixin):
     _mixin_ = True
 
     def unwrap(self, space, w_item):
@@ -271,10 +296,21 @@
     def for_computation(self, v):
         return widen(v)
 
+    def str_format(self, item):
+        return str(widen(self.unbox(item)))
+
+    @binop
+    def div(self, v1, v2):
+        if v2 == 0:
+            return 0
+        return v1 / v2
     @binop
     def mod(self, v1, v2):
         return v1 % v2
 
+class SignedIntegerArithmeticDtype(IntegerArithmeticDtype):
+    _mixin_ = True
+
     @unaryop
     def sign(self, v):
         if v > 0:
@@ -285,17 +321,22 @@
             assert v == 0
             return 0
 
-    def str_format(self, item):
-        return str(widen(self.unbox(item)))
+class UnsignedIntegerArithmeticDtype(IntegerArithmeticDtype):
+    _mixin_ = True
+
+    @unaryop
+    def sign(self, v):
+        return int(v != 0)
+
 
 W_BoolDtype = create_low_level_dtype(
     num = 0, kind = BOOLLTR, name = "bool",
-    aliases = ["?"],
+    aliases = ["?", "bool", "bool8"],
     applevel_types = ["bool"],
     T = lltype.Bool,
     valtype = bool,
 )
-class W_BoolDtype(IntegerArithmeticDtype, W_BoolDtype):
+class W_BoolDtype(SignedIntegerArithmeticDtype, W_BoolDtype):
     def unwrap(self, space, w_item):
         return self.adapt_val(space.is_true(w_item))
 
@@ -308,67 +349,138 @@
 
 W_Int8Dtype = create_low_level_dtype(
     num = 1, kind = SIGNEDLTR, name = "int8",
-    aliases = ["int8"],
+    aliases = ["b", "int8", "i1"],
     applevel_types = [],
     T = rffi.SIGNEDCHAR,
     valtype = rffi.SIGNEDCHAR._type,
     expected_size = 1,
 )
-class W_Int8Dtype(IntegerArithmeticDtype, W_Int8Dtype):
+class W_Int8Dtype(SignedIntegerArithmeticDtype, W_Int8Dtype):
+    pass
+
+W_UInt8Dtype = create_low_level_dtype(
+    num = 2, kind = UNSIGNEDLTR, name = "uint8",
+    aliases = ["B", "uint8", "I1"],
+    applevel_types = [],
+    T = rffi.UCHAR,
+    valtype = rffi.UCHAR._type,
+    expected_size = 1,
+)
+class W_UInt8Dtype(UnsignedIntegerArithmeticDtype, W_UInt8Dtype):
     pass
 
 W_Int16Dtype = create_low_level_dtype(
     num = 3, kind = SIGNEDLTR, name = "int16",
-    aliases = ["int16"],
+    aliases = ["h", "int16", "i2"],
     applevel_types = [],
     T = rffi.SHORT,
     valtype = rffi.SHORT._type,
     expected_size = 2,
 )
-class W_Int16Dtype(IntegerArithmeticDtype, W_Int16Dtype):
+class W_Int16Dtype(SignedIntegerArithmeticDtype, W_Int16Dtype):
+    pass
+
+W_UInt16Dtype = create_low_level_dtype(
+    num = 4, kind = UNSIGNEDLTR, name = "uint16",
+    aliases = ["H", "uint16", "I2"],
+    applevel_types = [],
+    T = rffi.USHORT,
+    valtype = rffi.USHORT._type,
+    expected_size = 2,
+)
+class W_UInt16Dtype(UnsignedIntegerArithmeticDtype, W_UInt16Dtype):
     pass
 
 W_Int32Dtype = create_low_level_dtype(
     num = 5, kind = SIGNEDLTR, name = "int32",
-    aliases = ["i"],
+    aliases = ["i", "int32", "i4"],
     applevel_types = [],
     T = rffi.INT,
     valtype = rffi.INT._type,
     expected_size = 4,
 )
-class W_Int32Dtype(IntegerArithmeticDtype, W_Int32Dtype):
+class W_Int32Dtype(SignedIntegerArithmeticDtype, W_Int32Dtype):
+    pass
+
+W_UInt32Dtype = create_low_level_dtype(
+    num = 6, kind = UNSIGNEDLTR, name = "uint32",
+    aliases = ["I", "uint32", "I4"],
+    applevel_types = [],
+    T = rffi.UINT,
+    valtype = rffi.UINT._type,
+    expected_size = 4,
+)
+class W_UInt32Dtype(UnsignedIntegerArithmeticDtype, W_UInt32Dtype):
     pass
 
 W_Int64Dtype = create_low_level_dtype(
     num = 9, kind = SIGNEDLTR, name = "int64",
-    aliases = [],
+    aliases = ["q", "int64", "i8"],
     applevel_types = ["long"],
     T = rffi.LONGLONG,
     valtype = rffi.LONGLONG._type,
     expected_size = 8,
 )
-class W_Int64Dtype(IntegerArithmeticDtype, W_Int64Dtype):
+class W_Int64Dtype(SignedIntegerArithmeticDtype, W_Int64Dtype):
+    pass
+
+W_UInt64Dtype = create_low_level_dtype(
+    num = 10, kind = UNSIGNEDLTR, name = "uint64",
+    aliases = ["Q", "uint64", "I8"],
+    applevel_types = [],
+    T = rffi.ULONGLONG,
+    valtype = rffi.ULONGLONG._type,
+    expected_size = 8,
+)
+class W_UInt64Dtype(UnsignedIntegerArithmeticDtype, W_UInt64Dtype):
+    pass
+
+if LONG_BIT == 32:
+    long_dtype = W_Int32Dtype
+    ulong_dtype = W_UInt32Dtype
+elif LONG_BIT == 64:
+    long_dtype = W_Int64Dtype
+    ulong_dtype = W_UInt64Dtype
+else:
+    assert False
+
+class W_LongDtype(long_dtype):
+    num = 7
+    aliases = ["l"]
+    applevel_types = ["int"]
+
+class W_ULongDtype(ulong_dtype):
+    num = 8
+    aliases = ["L"]
+
+W_Float32Dtype = create_low_level_dtype(
+    num = 11, kind = FLOATINGLTR, name = "float32",
+    aliases = ["f", "float32", "f4"],
+    applevel_types = [],
+    T = lltype.SingleFloat,
+    valtype = rarithmetic.r_singlefloat,
+    expected_size = 4,
+)
+class W_Float32Dtype(FloatArithmeticDtype, W_Float32Dtype):
     pass
 
 W_Float64Dtype = create_low_level_dtype(
     num = 12, kind = FLOATINGLTR, name = "float64",
-    aliases = [],
+    aliases = ["d", "float64", "f8"],
     applevel_types = ["float"],
     T = lltype.Float,
     valtype = float,
     expected_size = 8,
 )
 class W_Float64Dtype(FloatArithmeticDtype, W_Float64Dtype):
-    def unwrap(self, space, w_item):
-        return self.adapt_val(space.float_w(space.float(w_item)))
-
-    def str_format(self, item):
-        return float2string(self.unbox(item), 'g', rfloat.DTSF_STR_PRECISION)
+    pass
 
 ALL_DTYPES = [
     W_BoolDtype,
-    W_Int8Dtype, W_Int16Dtype, W_Int32Dtype, W_Int64Dtype,
-    W_Float64Dtype
+    W_Int8Dtype, W_UInt8Dtype, W_Int16Dtype, W_UInt16Dtype,
+    W_Int32Dtype, W_UInt32Dtype, W_LongDtype, W_ULongDtype,
+    W_Int64Dtype, W_UInt64Dtype,
+    W_Float32Dtype, W_Float64Dtype,
 ]
 
 dtypes_by_alias = unrolling_iterable([
@@ -395,6 +507,7 @@
 
     num = interp_attrproperty("num", cls=W_Dtype),
     kind = interp_attrproperty("kind", cls=W_Dtype),
+    itemsize = interp_attrproperty("num_bytes", cls=W_Dtype),
     shape = GetSetProperty(W_Dtype.descr_get_shape),
 )
 W_Dtype.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -4,6 +4,7 @@
 from pypy.interpreter.typedef import TypeDef, GetSetProperty, interp_attrproperty
 from pypy.module.micronumpy import interp_dtype, signature
 from pypy.rlib import jit
+from pypy.rlib.rarithmetic import LONG_BIT
 from pypy.tool.sourcetools import func_with_new_name
 
 
@@ -180,23 +181,56 @@
 
     # Everything promotes to float, and bool promotes to everything.
     if dt2.kind == interp_dtype.FLOATINGLTR or dt1.kind == interp_dtype.BOOLLTR:
+        # Float32 + 8-bit int = Float64
+        if dt2.num == 11 and dt1.num_bytes >= 4:
+            return space.fromcache(interp_dtype.W_Float64Dtype)
         return dt2
 
-    assert False
+    # for now this means mixing signed and unsigned
+    if dt2.kind == interp_dtype.SIGNEDLTR:
+        # if dt2 has a greater number of bytes, then just go with it
+        if dt1.num_bytes < dt2.num_bytes:
+            return dt2
+        # we need to promote both dtypes
+        dtypenum = dt2.num + 2
+    else:
+        # increase to the next signed type (or to float)
+        dtypenum = dt2.num + 1
+        # UInt64 + signed = Float64
+        if dt2.num == 10:
+            dtypenum += 1
+    newdtype = interp_dtype.ALL_DTYPES[dtypenum]
+
+    if newdtype.num_bytes > dt2.num_bytes or newdtype.kind == interp_dtype.FLOATINGLTR:
+        return space.fromcache(newdtype)
+    else:
+        # we only promoted to long on 32-bit or to longlong on 64-bit
+        # this is really for dealing with the Long and Ulong dtypes
+        if LONG_BIT == 32:
+            dtypenum += 2
+        else:
+            dtypenum += 3
+        return space.fromcache(interp_dtype.ALL_DTYPES[dtypenum])
 
 def find_unaryop_result_dtype(space, dt, promote_to_float=False,
     promote_bools=False, promote_to_largest=False):
     if promote_bools and (dt.kind == interp_dtype.BOOLLTR):
         return space.fromcache(interp_dtype.W_Int8Dtype)
     if promote_to_float:
+        if dt.kind == interp_dtype.FLOATINGLTR:
+            return dt
+        if dt.num >= 5:
+            return space.fromcache(interp_dtype.W_Float64Dtype)
         for bytes, dtype in interp_dtype.dtypes_by_num_bytes:
-            if dtype.kind == interp_dtype.FLOATINGLTR and dtype.num_bytes >= dt.num_bytes:
+            if dtype.kind == interp_dtype.FLOATINGLTR and dtype.num_bytes > dt.num_bytes:
                 return space.fromcache(dtype)
     if promote_to_largest:
         if dt.kind == interp_dtype.BOOLLTR or dt.kind == interp_dtype.SIGNEDLTR:
             return space.fromcache(interp_dtype.W_Int64Dtype)
         elif dt.kind == interp_dtype.FLOATINGLTR:
             return space.fromcache(interp_dtype.W_Float64Dtype)
+        elif dt.kind == interp_dtype.UNSIGNEDLTR:
+            return space.fromcache(interp_dtype.W_UInt64Dtype)
         else:
             assert False
     return dt
@@ -205,15 +239,23 @@
     w_type = space.type(w_obj)
 
     bool_dtype = space.fromcache(interp_dtype.W_BoolDtype)
+    long_dtype = space.fromcache(interp_dtype.W_LongDtype)
     int64_dtype = space.fromcache(interp_dtype.W_Int64Dtype)
 
     if space.is_w(w_type, space.w_bool):
-        if current_guess is None:
+        if current_guess is None or current_guess is bool_dtype:
             return bool_dtype
+        return current_guess
     elif space.is_w(w_type, space.w_int):
         if (current_guess is None or current_guess is bool_dtype or
-            current_guess is int64_dtype):
+            current_guess is long_dtype):
+            return long_dtype
+        return current_guess
+    elif space.is_w(w_type, space.w_long):
+        if (current_guess is None or current_guess is bool_dtype or
+            current_guess is long_dtype or current_guess is int64_dtype):
             return int64_dtype
+        return current_guess
     return space.fromcache(interp_dtype.W_Float64Dtype)
 
 
@@ -225,7 +267,9 @@
         def impl(res_dtype, lvalue, rvalue):
             res = getattr(res_dtype, op_name)(lvalue, rvalue)
             if comparison_func:
-                res = space.fromcache(interp_dtype.W_BoolDtype).box(res)
+                booldtype = space.fromcache(interp_dtype.W_BoolDtype)
+                assert isinstance(booldtype, interp_dtype.W_BoolDtype)
+                res = booldtype.box(res)
             return res
     return func_with_new_name(impl, ufunc_name)
 
@@ -268,6 +312,8 @@
             ("arcsin", "arcsin", 1, {"promote_to_float": True}),
             ("arccos", "arccos", 1, {"promote_to_float": True}),
             ("arctan", "arctan", 1, {"promote_to_float": True}),
+            ("arcsinh", "arcsinh", 1, {"promote_to_float": True}),
+            ("arctanh", "arctanh", 1, {"promote_to_float": True}),
         ]:
             self.add_ufunc(space, *ufunc_def)
 
@@ -277,7 +323,7 @@
 
         identity = extra_kwargs.get("identity")
         if identity is not None:
-            identity = space.fromcache(interp_dtype.W_Int64Dtype).adapt_val(identity)
+            identity = space.fromcache(interp_dtype.W_LongDtype).adapt_val(identity)
         extra_kwargs["identity"] = identity
 
         func = ufunc_dtype_caller(space, ufunc_name, op_name, argcount,
@@ -290,4 +336,4 @@
         setattr(self, ufunc_name, ufunc)
 
 def get(space):
-    return space.fromcache(UfuncState)
\ No newline at end of file
+    return space.fromcache(UfuncState)
diff --git a/pypy/module/micronumpy/test/test_base.py b/pypy/module/micronumpy/test/test_base.py
--- a/pypy/module/micronumpy/test/test_base.py
+++ b/pypy/module/micronumpy/test/test_base.py
@@ -64,18 +64,46 @@
     def test_unaryops(self, space):
         bool_dtype = space.fromcache(interp_dtype.W_BoolDtype)
         int8_dtype = space.fromcache(interp_dtype.W_Int8Dtype)
+        uint8_dtype = space.fromcache(interp_dtype.W_UInt8Dtype)
+        int16_dtype = space.fromcache(interp_dtype.W_Int16Dtype)
+        uint16_dtype = space.fromcache(interp_dtype.W_UInt16Dtype)
         int32_dtype = space.fromcache(interp_dtype.W_Int32Dtype)
+        uint32_dtype = space.fromcache(interp_dtype.W_UInt32Dtype)
+        long_dtype = space.fromcache(interp_dtype.W_LongDtype)
+        ulong_dtype = space.fromcache(interp_dtype.W_ULongDtype)
+        int64_dtype = space.fromcache(interp_dtype.W_Int64Dtype)
+        uint64_dtype = space.fromcache(interp_dtype.W_UInt64Dtype)
+        float32_dtype = space.fromcache(interp_dtype.W_Float32Dtype)
         float64_dtype = space.fromcache(interp_dtype.W_Float64Dtype)
 
-        # Normal rules, everythign returns itself
+        # Normal rules, everything returns itself
         assert find_unaryop_result_dtype(space, bool_dtype) is bool_dtype
         assert find_unaryop_result_dtype(space, int8_dtype) is int8_dtype
+        assert find_unaryop_result_dtype(space, uint8_dtype) is uint8_dtype
+        assert find_unaryop_result_dtype(space, int16_dtype) is int16_dtype
+        assert find_unaryop_result_dtype(space, uint16_dtype) is uint16_dtype
         assert find_unaryop_result_dtype(space, int32_dtype) is int32_dtype
+        assert find_unaryop_result_dtype(space, uint32_dtype) is uint32_dtype
+        assert find_unaryop_result_dtype(space, long_dtype) is long_dtype
+        assert find_unaryop_result_dtype(space, ulong_dtype) is ulong_dtype
+        assert find_unaryop_result_dtype(space, int64_dtype) is int64_dtype
+        assert find_unaryop_result_dtype(space, uint64_dtype) is uint64_dtype
+        assert find_unaryop_result_dtype(space, float32_dtype) is float32_dtype
         assert find_unaryop_result_dtype(space, float64_dtype) is float64_dtype
 
         # Coerce to floats, some of these will eventually be float16, or
         # whatever our smallest float type is.
-        assert find_unaryop_result_dtype(space, bool_dtype, promote_to_float=True) is float64_dtype
-        assert find_unaryop_result_dtype(space, int8_dtype, promote_to_float=True) is float64_dtype
+        assert find_unaryop_result_dtype(space, bool_dtype, promote_to_float=True) is float32_dtype # will be float16 if we ever put that in
+        assert find_unaryop_result_dtype(space, int8_dtype, promote_to_float=True) is float32_dtype # will be float16 if we ever put that in
+        assert find_unaryop_result_dtype(space, uint8_dtype, promote_to_float=True) is float32_dtype # will be float16 if we ever put that in
+        assert find_unaryop_result_dtype(space, int16_dtype, promote_to_float=True) is float32_dtype
+        assert find_unaryop_result_dtype(space, uint16_dtype, promote_to_float=True) is float32_dtype
         assert find_unaryop_result_dtype(space, int32_dtype, promote_to_float=True) is float64_dtype
-        assert find_unaryop_result_dtype(space, float64_dtype, promote_to_float=True) is float64_dtype
\ No newline at end of file
+        assert find_unaryop_result_dtype(space, uint32_dtype, promote_to_float=True) is float64_dtype
+        assert find_unaryop_result_dtype(space, int64_dtype, promote_to_float=True) is float64_dtype
+        assert find_unaryop_result_dtype(space, uint64_dtype, promote_to_float=True) is float64_dtype
+        assert find_unaryop_result_dtype(space, float32_dtype, promote_to_float=True) is float32_dtype
+        assert find_unaryop_result_dtype(space, float64_dtype, promote_to_float=True) is float64_dtype
+
+        # promote bools, happens with sign ufunc
+        assert find_unaryop_result_dtype(space, bool_dtype, promote_bools=True) is int8_dtype
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -17,6 +17,7 @@
         from numpy import dtype
 
         assert dtype(bool).num == 0
+        assert dtype(int).num == 7
         assert dtype(long).num == 9
         assert dtype(float).num == 12
 
@@ -81,6 +82,48 @@
             assert isinstance(a[i], (int, long))
             assert a[1] == 1
 
+    def test_overflow(self):
+        from numpy import array, dtype
+        assert array([128], 'b')[0] == -128
+        assert array([256], 'B')[0] == 0
+        assert array([32768], 'h')[0] == -32768
+        assert array([65536], 'H')[0] == 0
+        if dtype('l').itemsize == 4: # 32-bit
+            raises(OverflowError, "array([2**32/2], 'i')")
+            raises(OverflowError, "array([2**32], 'I')")
+        raises(OverflowError, "array([2**64/2], 'q')")
+        raises(OverflowError, "array([2**64], 'Q')")
+
+    def test_bool_binop_types(self):
+        from numpy import array, dtype
+        types = ('?','b','B','h','H','i','I','l','L','q','Q','f','d')
+        N = len(types)
+        a = array([True], '?')
+        for t in types:
+            assert (a + array([0], t)).dtype is dtype(t)
+
+    def test_binop_types(self):
+        from numpy import array, dtype
+        tests = [('b','B','h'), ('b','h','h'), ('b','H','i'), ('b','i','i'),
+                 ('b','l','l'), ('b','q','q'), ('b','Q','d'), ('B','h','h'),
+                 ('B','H','H'), ('B','i','i'), ('B','I','I'), ('B','l','l'),
+                 ('B','L','L'), ('B','q','q'), ('B','Q','Q'), ('h','H','i'),
+                 ('h','i','i'), ('h','l','l'), ('h','q','q'), ('h','Q','d'),
+                 ('H','i','i'), ('H','I','I'), ('H','l','l'), ('H','L','L'),
+                 ('H','q','q'), ('H','Q','Q'), ('i','l','l'), ('i','q','q'),
+                 ('i','Q','d'), ('I','L','L'), ('I','q','q'), ('I','Q','Q'),
+                 ('q','Q','d'), ('b','f','f'), ('B','f','f'), ('h','f','f'),
+                 ('H','f','f'), ('i','f','d'), ('I','f','d'), ('l','f','d'),
+                 ('L','f','d'), ('q','f','d'), ('Q','f','d'), ('q','d','d')]
+        if dtype('i').itemsize == dtype('l').itemsize: # 32-bit
+            tests.extend([('b','I','q'), ('b','L','q'), ('h','I','q'),
+                          ('h','L','q'), ('i','I','q'), ('i','L','q')])
+        else:
+            tests.extend([('b','I','l'), ('b','L','d'), ('h','I','l'),
+                          ('h','L','d'), ('i','I','l'), ('i','L','d')])
+        for d1, d2, dout in tests:
+            assert (array([1], d1) + array([1], d2)).dtype is dtype(dout)
+
     def test_add_int8(self):
         from numpy import array, dtype
 
@@ -99,6 +142,15 @@
         for i in range(5):
             assert b[i] == i * 2
 
+    def test_add_uint32(self):
+        from numpy import array, dtype
+
+        a = array(range(5), dtype="I")
+        b = a + a
+        assert b.dtype is dtype("I")
+        for i in range(5):
+            assert b[i] == i * 2
+
     def test_shape(self):
         from numpy import dtype
 
diff --git a/pypy/module/micronumpy/test/test_module.py b/pypy/module/micronumpy/test/test_module.py
--- a/pypy/module/micronumpy/test/test_module.py
+++ b/pypy/module/micronumpy/test/test_module.py
@@ -10,4 +10,12 @@
     def test_average(self):
         from numpy import array, average
         assert average(range(10)) == 4.5
-        assert average(array(range(10))) == 4.5
\ No newline at end of file
+        assert average(array(range(10))) == 4.5
+
+    def test_constants(self):
+        import math
+        from numpy import inf, e
+        assert type(inf) is float
+        assert inf == float("inf")
+        assert e == math.e
+        assert type(e) is float
\ No newline at end of file
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -285,7 +285,9 @@
             assert b[i] == i * 5
 
     def test_div(self):
-        from numpy import array, dtype
+        from math import isnan
+        from numpy import array, dtype, inf
+
         a = array(range(1, 6))
         b = a / a
         for i in range(5):
@@ -297,6 +299,24 @@
         for i in range(5):
             assert b[i] == 1
 
+        a = array([-1, 0, 1])
+        b = array([0, 0, 0])
+        c = a / b
+        assert (c == [0, 0, 0]).all()
+
+        a = array([-1.0, 0.0, 1.0])
+        b = array([0.0, 0.0, 0.0])
+        c = a / b
+        assert c[0] == -inf
+        assert isnan(c[1])
+        assert c[2] == inf
+
+        b = array([-0.0, -0.0, -0.0])
+        c = a / b
+        assert c[0] == inf
+        assert isnan(c[1])
+        assert c[2] == -inf
+
     def test_div_other(self):
         from numpy import array
         a = array(range(5))
@@ -551,8 +571,10 @@
         from numpy import array, dtype
 
         assert array([True]).dtype is dtype(bool)
-        assert array([True, 1]).dtype is dtype(long)
-        assert array([1, 2, 3]).dtype is dtype(long)
+        assert array([True, False]).dtype is dtype(bool)
+        assert array([True, 1]).dtype is dtype(int)
+        assert array([1, 2, 3]).dtype is dtype(int)
+        assert array([1L, 2, 3]).dtype is dtype(long)
         assert array([1.2, True]).dtype is dtype(float)
         assert array([1.2, 5]).dtype is dtype(float)
         assert array([]).dtype is dtype(float)
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -234,7 +234,7 @@
             assert b[i] == math.sin(a[i])
 
         a = sin(array([True, False], dtype=bool))
-        assert a[0] == sin(1)
+        assert abs(a[0] - sin(1)) < 1e-7 # a[0] will be less precise
         assert a[1] == 0.0
 
     def test_cos(self):
@@ -298,6 +298,25 @@
         b = arctan(a)
         assert math.isnan(b[0])
 
+    def test_arcsinh(self):
+        import math
+        from numpy import arcsinh, inf
+
+        for v in [inf, -inf, 1.0, math.e]:
+            assert math.asinh(v) == arcsinh(v)
+        assert math.isnan(arcsinh(float("nan")))
+
+    def test_arctanh(self):
+        import math
+        from numpy import arctanh
+
+        for v in [.99, .5, 0, -.5, -.99]:
+            assert math.atanh(v) == arctanh(v)
+        for v in [2.0, -2.0]:
+            assert math.isnan(arctanh(v))
+        for v in [1.0, -1.0]:
+            assert arctanh(v) == math.copysign(float("inf"), v)
+
     def test_reduce_errors(self):
         from numpy import sin, add
 
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -1,20 +1,24 @@
 from pypy.jit.metainterp.test.support import LLJitMixin
 from pypy.module.micronumpy import interp_ufuncs, signature
 from pypy.module.micronumpy.compile import (numpy_compile, FakeSpace,
-    FloatObject)
-from pypy.module.micronumpy.interp_dtype import W_Float64Dtype, W_Int64Dtype
+    FloatObject, IntObject)
+from pypy.module.micronumpy.interp_dtype import W_Int32Dtype, W_Float64Dtype, W_Int64Dtype, W_UInt64Dtype
 from pypy.module.micronumpy.interp_numarray import (BaseArray, SingleDimArray,
     SingleDimSlice, scalar_w)
 from pypy.rlib.nonconst import NonConstant
 from pypy.rpython.annlowlevel import llstr
 from pypy.rpython.test.test_llinterp import interpret
 
+import py
+
 
 class TestNumpyJIt(LLJitMixin):
     def setup_class(cls):
         cls.space = FakeSpace()
         cls.float64_dtype = cls.space.fromcache(W_Float64Dtype)
         cls.int64_dtype = cls.space.fromcache(W_Int64Dtype)
+        cls.uint64_dtype = cls.space.fromcache(W_UInt64Dtype)
+        cls.int32_dtype = cls.space.fromcache(W_Int32Dtype)
 
     def test_add(self):
         def f(i):
@@ -303,6 +307,31 @@
                           'int_lt': 1, 'guard_true': 1, 'jump': 1})
         assert result == 11.0
 
+    def test_int32_sum(self):
+        py.test.skip("pypy/jit/backend/llimpl.py needs to be changed to "
+                     "deal correctly with int dtypes for this test to "
+                     "work. skip for now until someone feels up to the task")
+        space = self.space
+        float64_dtype = self.float64_dtype
+        int32_dtype = self.int32_dtype
+
+        def f(n):
+            if NonConstant(False):
+                dtype = float64_dtype
+            else:
+                dtype = int32_dtype
+            ar = SingleDimArray(n, dtype=dtype)
+            i = 0
+            while i < n:
+                ar.get_concrete().setitem(i, int32_dtype.box(7))
+                i += 1
+            v = ar.descr_add(space, ar).descr_sum(space)
+            assert isinstance(v, IntObject)
+            return v.intval
+
+        result = self.meta_interp(f, [5], listops=True, backendopt=True)
+        assert result == f(5)
+
 class TestTranslation(object):
     def test_compile(self):
         x = numpy_compile('aa+f*f/a-', 10)
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -10,6 +10,7 @@
 from pypy.rpython.lltypesystem import rffi, lltype
 from pypy.rpython.tool import rffi_platform
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
+from pypy.module.sys.interp_encoding import getfilesystemencoding
 
 import os, sys
 _WIN = sys.platform == 'win32'
@@ -32,17 +33,19 @@
             raise OperationError(space.w_OverflowError,
                                  space.wrap("integer out of range"))
 
+def fsencode_w(space, w_obj):
+    if space.isinstance_w(w_obj, space.w_unicode):
+        w_obj = space.call_method(w_obj, 'encode',
+                                  getfilesystemencoding(space))
+    return space.str_w(w_obj)
+
 class FileEncoder(object):
     def __init__(self, space, w_obj):
         self.space = space
         self.w_obj = w_obj
 
     def as_bytes(self):
-        from pypy.module.sys.interp_encoding import getfilesystemencoding
-        space = self.space
-        w_bytes = space.call_method(self.w_obj, 'encode',
-                                    getfilesystemencoding(space))
-        return space.str_w(w_bytes)
+        return fsencode_w(self.space, self.w_obj)
 
     def as_unicode(self):
         return self.space.unicode_w(self.w_obj)
@@ -56,7 +59,6 @@
         return self.space.str_w(self.w_obj)
 
     def as_unicode(self):
-        from pypy.module.sys.interp_encoding import getfilesystemencoding
         space = self.space
         w_unicode = space.call_method(self.w_obj, 'decode',
                                       getfilesystemencoding(space))
@@ -536,7 +538,6 @@
 
 The list is in arbitrary order.  It does not include the special
 entries '.' and '..' even if they are present in the directory."""
-    from pypy.module.sys.interp_encoding import getfilesystemencoding
     try:
         if space.isinstance_w(w_dirname, space.w_unicode):
             dirname = FileEncoder(space, w_dirname)
@@ -734,8 +735,7 @@
 def _exit(space, status):
     os._exit(status)
 
- at unwrap_spec(command=str)
-def execv(space, command, w_args):
+def execv(space, w_command, w_args):
     """ execv(path, args)
 
 Execute an executable path with arguments, replacing current process.
@@ -743,12 +743,13 @@
         path: path of executable file
         args: iterable of strings
     """
+    command = fsencode_w(space, w_command)
     try:
         args_w = space.unpackiterable(w_args)
         if len(args_w) < 1:
             w_msg = space.wrap("execv() must have at least one argument")
             raise OperationError(space.w_ValueError, w_msg)
-        args = [space.str_w(w_arg) for w_arg in args_w]
+        args = [fsencode_w(space, w_arg) for w_arg in args_w]
     except OperationError, e:
         if not e.match(space, space.w_TypeError):
             raise
@@ -759,8 +760,7 @@
     except OSError, e:
         raise wrap_oserror(space, e)
 
- at unwrap_spec(command=str)
-def execve(space, command, w_args, w_env):
+def execve(space, w_command, w_args, w_env):
     """ execve(path, args, env)
 
 Execute a path with arguments and environment, replacing current process.
@@ -769,7 +769,8 @@
         args: iterable of arguments
         env: dictionary of strings mapping to strings
     """
-    args = [space.str_w(w_arg) for w_arg in space.unpackiterable(w_args)]
+    command = fsencode_w(space, w_command)
+    args = [fsencode_w(space, w_arg) for w_arg in space.unpackiterable(w_args)]
     env = {}
     w_keys = space.call_method(w_env, 'keys')
     for w_key in space.unpackiterable(w_keys):
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -415,6 +415,23 @@
                 else:
                     py.test.fail("didn't raise")
 
+        def test_execv_unicode(self):
+            os = self.posix
+            import sys
+            if not hasattr(os, "fork"):
+                skip("Need fork() to test execv()")
+            try:
+                output = u"caf\xe9 \u1234\n".encode(sys.getfilesystemencoding())
+            except UnicodeEncodeError:
+                skip("encoding not good enough")
+            pid = os.fork()
+            if pid == 0:
+                os.execv(u"/bin/sh", ["sh", "-c",
+                                      u"echo caf\xe9 \u1234 > onefile"])
+            os.waitpid(pid, 0)
+            assert open("onefile").read() == output
+            os.unlink("onefile")
+
         def test_execve(self):
             os = self.posix
             if not hasattr(os, "fork"):
@@ -425,6 +442,24 @@
             os.waitpid(pid, 0)
             assert open("onefile").read() == "xxx"
             os.unlink("onefile")
+
+        def test_execve_unicode(self):
+            os = self.posix
+            import sys
+            if not hasattr(os, "fork"):
+                skip("Need fork() to test execve()")
+            try:
+                output = u"caf\xe9 \u1234\n".encode(sys.getfilesystemencoding())
+            except UnicodeEncodeError:
+                skip("encoding not good enough")
+            pid = os.fork()
+            if pid == 0:
+                os.execve(u"/bin/sh", ["sh", "-c",
+                                      u"echo caf\xe9 \u1234 > onefile"],
+                          {'ddd': 'xxx'})
+            os.waitpid(pid, 0)
+            assert open("onefile").read() == output
+            os.unlink("onefile")
         pass # <- please, inspect.getsource(), don't crash
 
     if hasattr(__import__(os.name), "spawnv"):
diff --git a/pypy/module/pyexpat/__init__.py b/pypy/module/pyexpat/__init__.py
--- a/pypy/module/pyexpat/__init__.py
+++ b/pypy/module/pyexpat/__init__.py
@@ -4,12 +4,8 @@
 
 class ErrorsModule(MixedModule):
     "Definition of pyexpat.errors module."
-
-    appleveldefs = {
-        }
-
-    interpleveldefs = {
-        }
+    appleveldefs = {}
+    interpleveldefs = {}
 
     def setup_after_space_initialization(self):
         from pypy.module.pyexpat import interp_pyexpat
@@ -18,6 +14,18 @@
                     interp_pyexpat.ErrorString(self.space,
                     getattr(interp_pyexpat, name)))
 
+class ModelModule(MixedModule):
+    "Definition of pyexpat.model module."
+    appleveldefs = {}
+    interpleveldefs = {}
+
+    def setup_after_space_initialization(self):
+        from pypy.module.pyexpat import interp_pyexpat
+        space = self.space
+        for name in interp_pyexpat.xml_model_list:
+            value = getattr(interp_pyexpat, name)
+            space.setattr(self, space.wrap(name), space.wrap(value))
+
 class Module(MixedModule):
     "Python wrapper for Expat parser."
 
@@ -39,6 +47,7 @@
 
     submodules = {
         'errors': ErrorsModule,
+        'model':  ModelModule,
     }
 
     for name in ['XML_PARAM_ENTITY_PARSING_NEVER',
diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py
--- a/pypy/module/pyexpat/interp_pyexpat.py
+++ b/pypy/module/pyexpat/interp_pyexpat.py
@@ -76,6 +76,18 @@
     "XML_ERROR_FINISHED",
     "XML_ERROR_SUSPEND_PE",
     ]
+xml_model_list = [
+    "XML_CTYPE_EMPTY",
+    "XML_CTYPE_ANY",
+    "XML_CTYPE_MIXED",
+    "XML_CTYPE_NAME",
+    "XML_CTYPE_CHOICE",
+    "XML_CTYPE_SEQ",
+    "XML_CQUANT_NONE",
+    "XML_CQUANT_OPT",
+    "XML_CQUANT_REP",
+    "XML_CQUANT_PLUS",
+    ]
 
 class CConfigure:
     _compilation_info_ = eci
@@ -104,6 +116,8 @@
 
     for name in xml_error_list:
         locals()[name] = rffi_platform.ConstantInteger(name)
+    for name in xml_model_list:
+        locals()[name] = rffi_platform.ConstantInteger(name)
 
 for k, v in rffi_platform.configure(CConfigure).items():
     globals()[k] = v
diff --git a/pypy/module/pyexpat/test/test_parser.py b/pypy/module/pyexpat/test/test_parser.py
--- a/pypy/module/pyexpat/test/test_parser.py
+++ b/pypy/module/pyexpat/test/test_parser.py
@@ -131,3 +131,7 @@
                'encoding specified in XML declaration is incorrect')
         assert (pyexpat.errors.XML_ERROR_XML_DECL ==
                 'XML declaration not well-formed')
+
+    def test_model(self):
+        import pyexpat
+        assert isinstance(pyexpat.model.XML_CTYPE_EMPTY, int)
diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py
--- a/pypy/module/pypyjit/interp_jit.py
+++ b/pypy/module/pypyjit/interp_jit.py
@@ -137,20 +137,15 @@
 
     def jump_absolute(self, jumpto, _, ec=None):
         if we_are_jitted():
-            # Normally, the tick counter is decremented by 100 for every
-            # Python opcode.  Here, to better support JIT compilation of
-            # small loops, we decrement it by a possibly smaller constant.
-            # We get the maximum 100 when the (unoptimized) trace length
-            # is at least 3200 (a bit randomly).
-            trace_length = r_uint(current_trace_length())
-            decr_by = trace_length // 32
-            if decr_by < 1:
-                decr_by = 1
-            elif decr_by > 100:    # also if current_trace_length() returned -1
-                decr_by = 100
+            #
+            # assume that only threads are using the bytecode counter
+            decr_by = 0
+            if self.space.actionflag.has_bytecode_counter:   # constant-folded
+                if self.space.threadlocals.gil_ready:   # quasi-immutable field
+                    decr_by = _get_adapted_tick_counter()
             #
             self.last_instr = intmask(jumpto)
-            ec.bytecode_trace(self, intmask(decr_by))
+            ec.bytecode_trace(self, decr_by)
             jumpto = r_uint(self.last_instr)
         #
         pypyjitdriver.can_enter_jit(frame=self, ec=ec, next_instr=jumpto,
@@ -158,6 +153,20 @@
                                     is_being_profiled=self.is_being_profiled)
         return jumpto
 
+def _get_adapted_tick_counter():
+    # Normally, the tick counter is decremented by 100 for every
+    # Python opcode.  Here, to better support JIT compilation of
+    # small loops, we decrement it by a possibly smaller constant.
+    # We get the maximum 100 when the (unoptimized) trace length
+    # is at least 3200 (a bit randomly).
+    trace_length = r_uint(current_trace_length())
+    decr_by = trace_length // 32
+    if decr_by < 1:
+        decr_by = 1
+    elif decr_by > 100:    # also if current_trace_length() returned -1
+        decr_by = 100
+    return intmask(decr_by)
+
 
 PyCode__initialize = PyCode._initialize
 
diff --git a/pypy/module/pypyjit/test_pypy_c/model.py b/pypy/module/pypyjit/test_pypy_c/model.py
--- a/pypy/module/pypyjit/test_pypy_c/model.py
+++ b/pypy/module/pypyjit/test_pypy_c/model.py
@@ -225,6 +225,8 @@
         # strip comment
         if '#' in line:
             line = line[:line.index('#')]
+        if line.strip() == 'guard_not_invalidated?':
+            return 'guard_not_invalidated', None, [], '...', False
         # find the resvar, if any
         if ' = ' in line:
             resvar, _, line = line.partition(' = ')
@@ -249,7 +251,7 @@
             descr = descr[len('descr='):]
         else:
             descr = None
-        return opname, resvar, args, descr
+        return opname, resvar, args, descr, True
 
     @classmethod
     def preprocess_expected_src(cls, src):
@@ -258,13 +260,23 @@
         # replaced with the corresponding operations, so that tests don't have
         # to repeat it every time
         ticker_check = """
+            guard_not_invalidated?
+            ticker0 = getfield_raw(ticker_address, descr=<SignedFieldDescr pypysig_long_struct.c_value .*>)
+            ticker_cond0 = int_lt(ticker0, 0)
+            guard_false(ticker_cond0, descr=...)
+        """
+        src = src.replace('--TICK--', ticker_check)
+        #
+        # this is the ticker check generated if we have threads
+        thread_ticker_check = """
+            guard_not_invalidated?
             ticker0 = getfield_raw(ticker_address, descr=<SignedFieldDescr pypysig_long_struct.c_value .*>)
             ticker1 = int_sub(ticker0, 1)
             setfield_raw(ticker_address, ticker1, descr=<SignedFieldDescr pypysig_long_struct.c_value .*>)
             ticker_cond0 = int_lt(ticker1, 0)
             guard_false(ticker_cond0, descr=...)
         """
-        src = src.replace('--TICK--', ticker_check)
+        src = src.replace('--THREAD-TICK--', thread_ticker_check)
         #
         # this is the ticker check generated in PyFrame.handle_operation_error
         exc_ticker_check = """
@@ -298,7 +310,7 @@
         if not cond:
             raise InvalidMatch(message, frame=sys._getframe(1))
 
-    def match_op(self, op, (exp_opname, exp_res, exp_args, exp_descr)):
+    def match_op(self, op, (exp_opname, exp_res, exp_args, exp_descr, _)):
         self._assert(op.name == exp_opname, "operation mismatch")
         self.match_var(op.res, exp_res)
         if exp_args != ['...']:
@@ -341,7 +353,7 @@
         what is after the '...'
         """
         iter_exp_ops = iter(expected_ops)
-        iter_ops = iter(self.ops)
+        iter_ops = RevertableIterator(self.ops)
         for opindex, exp_op in enumerate(iter_exp_ops):
             try:
                 if exp_op == '...':
@@ -360,6 +372,9 @@
                             break
                 self.match_op(op, exp_op)
             except InvalidMatch, e:
+                if exp_op[4] is False:    # optional operation
+                    iter_ops.revert_one()
+                    continue       # try to match with the next exp_op
                 e.opindex = opindex
                 raise
         #
@@ -372,8 +387,8 @@
                 return ''
             text = str(py.code.Source(src).deindent().indent())
             lines = text.splitlines(True)
-            if opindex is not None and 0 <= opindex < len(lines):
-                lines[opindex] = lines[opindex].rstrip() + '\t<=====\n'
+            if opindex is not None and 0 <= opindex <= len(lines):
+                lines.insert(opindex, '\n\t===== HERE =====\n')
             return ''.join(lines)
         #
         expected_src = self.preprocess_expected_src(expected_src)
@@ -398,3 +413,18 @@
         else:
             return True
 
+
+class RevertableIterator(object):
+    def __init__(self, sequence):
+        self.sequence = sequence
+        self.index = 0
+    def __iter__(self):
+        return self
+    def next(self):
+        index = self.index
+        if index == len(self.sequence):
+            raise StopIteration
+        self.index = index + 1
+        return self.sequence[index]
+    def revert_one(self):
+        self.index -= 1
diff --git a/pypy/module/pypyjit/test_pypy_c/test_00_model.py b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
--- a/pypy/module/pypyjit/test_pypy_c/test_00_model.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
@@ -50,8 +50,7 @@
         cmdline.append(str(self.filepath))
         #
         print cmdline, logfile
-        env={'PYPYLOG': 'jit-log-opt,jit-summary:' + str(logfile)}
-        #env={'PYPYLOG': ':' + str(logfile)}
+        env={'PYPYLOG': 'jit-log-opt,jit-log-noopt,jit-summary:' + str(logfile)}
         pipe = subprocess.Popen(cmdline,
                                 env=env,
                                 stdout=subprocess.PIPE,
@@ -146,15 +145,17 @@
 
     def test_parse_op(self):
         res = OpMatcher.parse_op("  a =   int_add(  b,  3 ) # foo")
-        assert res == ("int_add", "a", ["b", "3"], None)
+        assert res == ("int_add", "a", ["b", "3"], None, True)
         res = OpMatcher.parse_op("guard_true(a)")
-        assert res == ("guard_true", None, ["a"], None)
+        assert res == ("guard_true", None, ["a"], None, True)
         res = OpMatcher.parse_op("setfield_gc(p0, i0, descr=<foobar>)")
-        assert res == ("setfield_gc", None, ["p0", "i0"], "<foobar>")
+        assert res == ("setfield_gc", None, ["p0", "i0"], "<foobar>", True)
         res = OpMatcher.parse_op("i1 = getfield_gc(p0, descr=<foobar>)")
-        assert res == ("getfield_gc", "i1", ["p0"], "<foobar>")
+        assert res == ("getfield_gc", "i1", ["p0"], "<foobar>", True)
         res = OpMatcher.parse_op("p0 = force_token()")
-        assert res == ("force_token", "p0", [], None)
+        assert res == ("force_token", "p0", [], None, True)
+        res = OpMatcher.parse_op("guard_not_invalidated?")
+        assert res == ("guard_not_invalidated", None, [], '...', False)
 
     def test_exact_match(self):
         loop = """
@@ -342,7 +343,7 @@
             # this is the actual loop
             'int_lt', 'guard_true', 'int_add',
             # this is the signal checking stuff
-            'getfield_raw', 'int_sub', 'setfield_raw', 'int_lt', 'guard_false',
+            'guard_not_invalidated', 'getfield_raw', 'int_lt', 'guard_false',
             'jump'
             ]
 
@@ -408,7 +409,7 @@
             # this is the actual loop
             'int_lt', 'guard_true', 'force_token', 'int_add',
             # this is the signal checking stuff
-            'getfield_raw', 'int_sub', 'setfield_raw', 'int_lt', 'guard_false',
+            'guard_not_invalidated', 'getfield_raw', 'int_lt', 'guard_false',
             'jump'
             ]
 
@@ -426,10 +427,9 @@
             guard_true(i6, descr=...)
             i8 = int_add(i4, 1)
             # signal checking stuff
+            guard_not_invalidated(descr=...)
             i10 = getfield_raw(37212896, descr=<.* pypysig_long_struct.c_value .*>)
-            i12 = int_sub(i10, 1)
-            setfield_raw(37212896, i12, descr=<.* pypysig_long_struct.c_value .*>)
-            i14 = int_lt(i12, 0)
+            i14 = int_lt(i10, 0)
             guard_false(i14, descr=...)
             jump(p0, p1, p2, p3, i8, descr=...)
         """)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py
--- a/pypy/module/pypyjit/test_pypy_c/test_call.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_call.py
@@ -366,12 +366,12 @@
             # make sure that the "block" is not allocated
             ...
             i20 = force_token()
-            setfield_gc(p0, i20, descr=<SignedFieldDescr .*PyFrame.vable_token .*>)
             p22 = new_with_vtable(19511408)
             p24 = new_array(1, descr=<GcPtrArrayDescr>)
             p26 = new_with_vtable(ConstClass(W_ListObject))
             p27 = new(descr=<SizeDescr .*>)
             p29 = new_array(0, descr=<GcPtrArrayDescr>)
+            setfield_gc(p0, i20, descr=<SignedFieldDescr .*PyFrame.vable_token .*>)
             setfield_gc(p27, p29, descr=<GcPtrFieldDescr list.items .*>)
             setfield_gc(p26, p27, descr=<.* .*W_ListObject.inst_wrappeditems .*>)
             setarrayitem_gc(p24, 0, p26, descr=<GcPtrArrayDescr>)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_generators.py b/pypy/module/pypyjit/test_pypy_c/test_generators.py
--- a/pypy/module/pypyjit/test_pypy_c/test_generators.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_generators.py
@@ -19,8 +19,8 @@
         assert loop.match_by_id("generator", """
             i16 = force_token()
             p45 = new_with_vtable(ConstClass(W_IntObject))
-            setfield_gc(p45, i29, descr=<SignedFieldDescr .*>)
             i47 = arraylen_gc(p8, descr=<GcPtrArrayDescr>) # Should be removed by backend
             setarrayitem_gc(p8, 0, p45, descr=<GcPtrArrayDescr>)
+            setfield_gc(p45, i29, descr=<SignedFieldDescr .*>)
             jump(..., descr=...)
             """)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_instance.py b/pypy/module/pypyjit/test_pypy_c/test_instance.py
--- a/pypy/module/pypyjit/test_pypy_c/test_instance.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_instance.py
@@ -125,8 +125,8 @@
             i12 = force_token()
             --TICK--
             p20 = new_with_vtable(ConstClass(W_IntObject))
+            setfield_gc(ConstPtr(ptr21), p20, descr=<GcPtrFieldDescr .*TypeCell.inst_w_value .*>)
             setfield_gc(p20, i11, descr=<SignedFieldDescr.*W_IntObject.inst_intval .*>)
-            setfield_gc(ConstPtr(ptr21), p20, descr=<GcPtrFieldDescr .*TypeCell.inst_w_value .*>)
             jump(p0, p1, p2, p3, p4, p20, p6, i7, p20, descr=<Loop.>)
         """)
 
diff --git a/pypy/module/pypyjit/test_pypy_c/test_misc.py b/pypy/module/pypyjit/test_pypy_c/test_misc.py
--- a/pypy/module/pypyjit/test_pypy_c/test_misc.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_misc.py
@@ -329,4 +329,20 @@
             guard_false(i28, descr=...)
             i30 = int_lshift(i20, 24)
             i31 = int_or(i26, i30)
-        """ % {"32_bit_only": extra})
\ No newline at end of file
+        """ % {"32_bit_only": extra})
+
+    def test_eval(self):
+        def main():
+            i = 1
+            a = compile('x+x+x+x+x+x', 'eval', 'eval')
+            b = {'x': 7}
+            while i < 1000:
+                y = eval(a,b,b)  # ID: eval
+                i += 1
+            return y
+
+        log = self.run(main)
+        assert log.result == 42
+        # the following assertion fails if the loop was cancelled due
+        # to "abort: vable escape"
+        assert len(log.loops_by_id("eval")) == 1
diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -41,7 +41,7 @@
             guard_true(i32, descr=...)
             i34 = int_add(i6, 1)
             --TICK--
-            jump(p0, p1, p2, p3, p4, p5, i34, p7, p8, i9, i10, p11, i12, p13, descr=<Loop4>)
+            jump(p0, p1, p2, p3, p4, p5, i34, p7, p8, i9, i10, p11, i12, p13, descr=...)
         """)
 
     def test_long(self):
@@ -106,7 +106,7 @@
             i58 = int_add_ovf(i6, i57)
             guard_no_overflow(descr=...)
             --TICK--
-            jump(p0, p1, p2, p3, p4, p5, i58, i7, descr=<Loop4>)
+            jump(p0, p1, p2, p3, p4, p5, i58, i7, descr=...)
         """)
 
     def test_str_mod(self):
@@ -156,4 +156,41 @@
             i40 = int_sub(i4, 1)
             --TICK--
             jump(p0, p1, p2, p3, i40, i38, descr=<Loop0>)
-        """)
\ No newline at end of file
+        """)
+
+    def test_getattr_promote(self):
+        def main(n):
+            class A(object):
+                def meth_a(self):
+                    return 1
+                def meth_b(self):
+                    return 2
+            a = A()
+
+            l = ['a', 'b']
+            s = 0
+            for i in range(n):
+                name = 'meth_' + l[i & 1]
+                meth = getattr(a, name) # ID: getattr
+                s += meth()
+            return s
+
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loops = log.loops_by_filename(self.filepath)
+        assert len(loops) == 2
+        for loop in loops:
+            loop.match_by_id('getattr','''
+            guard_not_invalidated(descr=...)
+            i32 = strlen(p31)
+            i34 = int_add(5, i32)
+            p35 = newstr(i34)
+            strsetitem(p35, 0, 109)
+            strsetitem(p35, 1, 101)
+            strsetitem(p35, 2, 116)
+            strsetitem(p35, 3, 104)
+            strsetitem(p35, 4, 95)
+            copystrcontent(p31, p35, 0, 5, i32)
+            i49 = call(ConstClass(_ll_2_str_eq_nonnull__rpy_stringPtr_rpy_stringPtr), p35, ConstPtr(ptr48), descr=<SignedCallDescr>)
+            guard_value(i49, 1, descr=<Guard8>)
+            ''')
diff --git a/pypy/module/pypyjit/test_pypy_c/test_thread.py b/pypy/module/pypyjit/test_pypy_c/test_thread.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/pypyjit/test_pypy_c/test_thread.py
@@ -0,0 +1,28 @@
+from pypy.module.pypyjit.test_pypy_c.test_00_model import BaseTestPyPyC
+
+
+class TestThread(BaseTestPyPyC):
+    def test_simple(self):
+        def main(n):
+            import thread
+            def f():
+                i = 0
+                while i < n:
+                    i += 1
+                done.release()
+
+            done = thread.allocate_lock()
+            done.acquire()
+            thread.start_new_thread(f, ())
+            done.acquire()
+            return 0
+        log = self.run(main, [500])
+        assert round(log.result, 6) == round(main(500), 6)
+        loop, = log.loops_by_filename(self.filepath)
+        assert loop.match("""
+            i2 = int_lt(i0, i1)
+            guard_true(i2, descr=...)
+            i3 = int_add(i0, 1)
+            --THREAD-TICK--
+            jump(..., descr=<Loop0>)
+        """)
diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py
--- a/pypy/module/signal/interp_signal.py
+++ b/pypy/module/signal/interp_signal.py
@@ -109,8 +109,11 @@
         p = pypysig_getaddr_occurred()
         value = p.c_value
         if self.has_bytecode_counter:    # this 'if' is constant-folded
-            value -= by
-            p.c_value = value
+            if jit.isconstant(by) and by == 0:
+                pass     # normally constant-folded too
+            else:
+                value -= by
+                p.c_value = value
         return value
 
 
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -47,7 +47,7 @@
         'pypy_initial_path'     : 'state.pypy_initial_path',
 
         '_getframe'             : 'vm._getframe', 
-        '_current_frames'       : 'vm._current_frames', 
+        '_current_frames'       : 'currentframes._current_frames', 
         'setrecursionlimit'     : 'vm.setrecursionlimit', 
         'getrecursionlimit'     : 'vm.getrecursionlimit', 
         'setcheckinterval'      : 'vm.setcheckinterval', 
@@ -55,6 +55,7 @@
         'exc_info'              : 'vm.exc_info', 
         'exc_clear'             : 'vm.exc_clear', 
         'settrace'              : 'vm.settrace',
+        'gettrace'              : 'vm.gettrace',
         'setprofile'            : 'vm.setprofile',
         'getprofile'            : 'vm.getprofile',
         'call_tracing'          : 'vm.call_tracing',
diff --git a/pypy/module/sys/currentframes.py b/pypy/module/sys/currentframes.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/sys/currentframes.py
@@ -0,0 +1,78 @@
+"""
+Implementation of the 'sys._current_frames()' routine.
+"""
+from pypy.interpreter import gateway
+
+app = gateway.applevel('''
+"NOT_RPYTHON"
+import __builtin__
+
+class fake_code(object):
+    co_name = "?"
+    co_filename = "?"
+    co_firstlineno = 0
+
+class fake_frame(object):
+    f_back = None
+    f_builtins = __builtin__.__dict__
+    f_code = fake_code()
+    f_exc_traceback = None
+    f_exc_type = None
+    f_exc_value = None
+    f_globals = {}
+    f_lasti = -1
+    f_lineno = 0
+    f_locals = {}
+    f_restricted = False
+    f_trace = None
+
+    def __init__(self, f):
+        if f is not None:
+            for name in ["f_builtins", "f_code", "f_globals", "f_lasti",
+                         "f_lineno"]:
+                setattr(self, name, getattr(f, name))
+''')
+
+def _current_frames(space):
+    """_current_frames() -> dictionary
+
+    Return a dictionary mapping each current thread T's thread id to T's
+    current stack "frame".  Functions in the traceback module can build the
+    call stack given such a frame.
+
+    Note that in PyPy this returns fake frame objects, to avoid a runtime
+    penalty everywhere with the JIT.  (So far these fake frames can be
+    completely uninformative depending on the JIT state; we could return
+    more with more efforts.)
+
+    This function should be used for specialized purposes only."""
+    w_result = space.newdict()
+    w_fake_frame = app.wget(space, "fake_frame")
+    w_fake_code  = app.wget(space, "fake_code")
+    ecs = space.threadlocals.getallvalues()
+    for thread_ident, ec in ecs.items():
+        vref = ec.topframeref
+        frames = []
+        while not vref.virtual:
+            f = vref()
+            if f is None:
+                break
+            frames.append(f)
+            vref = f.f_backref
+        else:
+            frames.append(None)
+        #
+        w_topframe = space.wrap(None)
+        w_prevframe = None
+        for f in frames:
+            w_nextframe = space.call_function(w_fake_frame, space.wrap(f))
+            if w_prevframe is None:
+                w_topframe = w_nextframe
+            else:
+                space.setattr(w_prevframe, space.wrap('f_back'), w_nextframe)
+            w_prevframe = w_nextframe
+        #
+        space.setitem(w_result,
+                      space.wrap(thread_ident),
+                      w_topframe)
+    return w_result
diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -478,6 +478,7 @@
         sys.settrace(trace)
         try:
             x()
+            assert sys.gettrace() is trace
         finally:
             sys.settrace(None)
         assert len(counts) == 1
@@ -555,7 +556,7 @@
             return sys._current_frames()
         frames = f()
         assert frames.keys() == [0]
-        assert frames[0].f_code.co_name == 'f'
+        assert frames[0].f_code.co_name in ('f', '?')
 
 class AppTestCurrentFramesWithThread(AppTestCurrentFrames):
     def setup_class(cls):
@@ -567,23 +568,25 @@
         import thread
 
         thread_id = thread.get_ident()
-        self.ready = False
         def other_thread():
-            self.ready = True
             print "thread started"
-            time.sleep(5)
+            lock2.release()
+            lock1.acquire()
+        lock1 = thread.allocate_lock()
+        lock2 = thread.allocate_lock()
+        lock1.acquire()
+        lock2.acquire()
         thread.start_new_thread(other_thread, ())
 
         def f():
-            for i in range(100):
-                if self.ready: break
-                time.sleep(0.1)
+            lock2.acquire()
             return sys._current_frames()
 
         frames = f()
+        lock1.release()
         thisframe = frames.pop(thread_id)
-        assert thisframe.f_code.co_name == 'f'
+        assert thisframe.f_code.co_name in ('f', '?')
 
         assert len(frames) == 1
         _, other_frame = frames.popitem()
-        assert other_frame.f_code.co_name == 'other_thread'
+        assert other_frame.f_code.co_name in ('other_thread', '?')
diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -45,25 +45,6 @@
     f.mark_as_escaped()
     return space.wrap(f)
 
-def _current_frames(space):
-    """_current_frames() -> dictionary
-
-    Return a dictionary mapping each current thread T's thread id to T's
-    current stack frame.
-
-    This function should be used for specialized purposes only."""
-    raise OperationError(space.w_NotImplementedError,
-        space.wrap("XXX sys._current_frames() incompatible with the JIT"))
-    w_result = space.newdict()
-    ecs = space.threadlocals.getallvalues()
-    for thread_ident, ec in ecs.items():
-        f = ec.gettopframe_nohidden()
-        f.mark_as_escaped()
-        space.setitem(w_result,
-                      space.wrap(thread_ident),
-                      space.wrap(f))
-    return w_result
-
 def setrecursionlimit(space, w_new_limit):
     """setrecursionlimit() sets the maximum number of nested calls that
 can occur before a RuntimeError is raised.  On PyPy the limit is
@@ -129,14 +110,19 @@
 function call.  See the debugger chapter in the library manual."""
     space.getexecutioncontext().settrace(w_func)
 
+def gettrace(space):
+    """Return the global debug tracing function set with sys.settrace.
+See the debugger chapter in the library manual."""
+    return space.getexecutioncontext().gettrace()
+
 def setprofile(space, w_func):
     """Set the profiling function.  It will be called on each function call
 and return.  See the profiler chapter in the library manual."""
     space.getexecutioncontext().setprofile(w_func)
 
 def getprofile(space):
-    """Set the profiling function.  It will be called on each function call
-and return.  See the profiler chapter in the library manual."""
+    """Return the profiling function set with sys.setprofile.
+See the profiler chapter in the library manual."""
     w_func = space.getexecutioncontext().getprofile()
     if w_func is not None:
         return w_func
diff --git a/pypy/module/thread/gil.py b/pypy/module/thread/gil.py
--- a/pypy/module/thread/gil.py
+++ b/pypy/module/thread/gil.py
@@ -16,7 +16,8 @@
 
 class GILThreadLocals(OSThreadLocals):
     """A version of OSThreadLocals that enforces a GIL."""
-    ll_GIL = thread.null_ll_lock
+    gil_ready = False
+    _immutable_fields_ = ['gil_ready?']
 
     def initialize(self, space):
         # add the GIL-releasing callback as an action on the space
@@ -25,12 +26,10 @@
 
     def setup_threads(self, space):
         """Enable threads in the object space, if they haven't already been."""
-        if not self.ll_GIL:
-            try:
-                self.ll_GIL = thread.allocate_ll_lock()
-            except thread.error:
+        if not self.gil_ready:
+            if not thread.gil_allocate():
                 raise wrap_thread_error(space, "can't allocate GIL")
-            thread.acquire_NOAUTO(self.ll_GIL, True)
+            self.gil_ready = True
             self.enter_thread(space)   # setup the main thread
             result = True
         else:
@@ -44,19 +43,16 @@
         # test_lock_again after the global state was cleared by
         # test_compile_lock.  As a workaround, we repatch these global
         # fields systematically.
-        spacestate.ll_GIL = self.ll_GIL
         invoke_around_extcall(before_external_call, after_external_call)
         return result
 
     def reinit_threads(self, space):
-        if self.ll_GIL:
-            self.ll_GIL = thread.allocate_ll_lock()
-            thread.acquire_NOAUTO(self.ll_GIL, True)
-            self.enter_thread(space)
+        if self.gil_ready:
+            self.gil_ready = False
+            self.setup_threads(space)
 
     def yield_thread(self):
-        thread.yield_thread()  # explicitly release the gil (used by test_gil)
-
+        do_yield_thread()
 
 class GILReleaseAction(PeriodicAsyncAction):
     """An action called every sys.checkinterval bytecodes.  It releases
@@ -64,16 +60,12 @@
     """
 
     def perform(self, executioncontext, frame):
-        # Other threads can run between the release() and the acquire()
-        # implicit in the following external function call (which has
-        # otherwise no effect).
-        thread.yield_thread()
+        do_yield_thread()
 
 
 class SpaceState:
 
     def _freeze_(self):
-        self.ll_GIL = thread.null_ll_lock
         self.action_after_thread_switch = None
         # ^^^ set by AsyncAction.fire_after_thread_switch()
         return False
@@ -95,14 +87,14 @@
     # this function must not raise, in such a way that the exception
     # transformer knows that it cannot raise!
     e = get_errno()
-    thread.release_NOAUTO(spacestate.ll_GIL)
+    thread.gil_release()
     set_errno(e)
 before_external_call._gctransformer_hint_cannot_collect_ = True
 before_external_call._dont_reach_me_in_del_ = True
 
 def after_external_call():
     e = get_errno()
-    thread.acquire_NOAUTO(spacestate.ll_GIL, True)
+    thread.gil_acquire()
     thread.gc_thread_run()
     spacestate.after_thread_switch()
     set_errno(e)
@@ -115,3 +107,18 @@
 # pointers in the shadow stack.  This is necessary because the GIL is
 # not held after the call to before_external_call() or before the call
 # to after_external_call().
+
+def do_yield_thread():
+    # explicitly release the gil, in a way that tries to give more
+    # priority to other threads (as opposed to continuing to run in
+    # the same thread).
+    if thread.gil_yield_thread():
+        thread.gc_thread_run()
+        spacestate.after_thread_switch()
+do_yield_thread._gctransformer_hint_close_stack_ = True
+do_yield_thread._dont_reach_me_in_del_ = True
+do_yield_thread._dont_inline_ = True
+
+# do_yield_thread() needs a different hint: _gctransformer_hint_close_stack_.
+# The *_external_call() functions are themselves called only from the rffi
+# module from a helper function that also has this hint.
diff --git a/pypy/module/thread/ll_thread.py b/pypy/module/thread/ll_thread.py
--- a/pypy/module/thread/ll_thread.py
+++ b/pypy/module/thread/ll_thread.py
@@ -17,7 +17,8 @@
     include_dirs = [str(py.path.local(autopath.pypydir).join('translator', 'c'))],
     export_symbols = ['RPyThreadGetIdent', 'RPyThreadLockInit',
                       'RPyThreadAcquireLock', 'RPyThreadReleaseLock',
-                      'RPyThreadYield',
+                      'RPyGilAllocate', 'RPyGilYieldThread',
+                      'RPyGilRelease', 'RPyGilAcquire',
                       'RPyThreadGetStackSize', 'RPyThreadSetStackSize',
                       'RPyOpaqueDealloc_ThreadLock',
                       'RPyThreadAfterFork']
@@ -69,8 +70,16 @@
                                          [TLOCKP], lltype.Void,
                                          _nowrapper=True)
 
-# this function does nothing apart from releasing the GIL temporarily.
-yield_thread = llexternal('RPyThreadYield', [], lltype.Void, threadsafe=True)
+# these functions manipulate directly the GIL, whose definition does not
+# escape the C code itself
+gil_allocate     = llexternal('RPyGilAllocate', [], lltype.Signed,
+                              _nowrapper=True)
+gil_yield_thread = llexternal('RPyGilYieldThread', [], lltype.Signed,
+                              _nowrapper=True)
+gil_release      = llexternal('RPyGilRelease', [], lltype.Void,
+                              _nowrapper=True)
+gil_acquire      = llexternal('RPyGilAcquire', [], lltype.Void,
+                              _nowrapper=True)
 
 def allocate_lock():
     return Lock(allocate_ll_lock())
diff --git a/pypy/module/thread/test/test_gil.py b/pypy/module/thread/test/test_gil.py
--- a/pypy/module/thread/test/test_gil.py
+++ b/pypy/module/thread/test/test_gil.py
@@ -30,19 +30,34 @@
     use_threads = True
     bigtest = False
 
-    def test_one_thread(self):
+    def test_one_thread(self, skew=+1):
+        from pypy.rlib.debug import debug_print
         if self.bigtest:
-            N = 1000000
+            N = 100000
+            skew *= 25000
         else:
             N = 100
+            skew *= 25
         space = FakeSpace()
         class State:
             pass
         state = State()
-        def runme():
-            for i in range(N):
+        def runme(main=False):
+            j = 0
+            for i in range(N + [-skew, skew][main]):
+                state.datalen1 += 1   # try to crash if the GIL is not
+                state.datalen2 += 1   # correctly acquired
                 state.data.append((thread.get_ident(), i))
+                state.datalen3 += 1
+                state.datalen4 += 1
+                assert state.datalen1 == len(state.data)
+                assert state.datalen2 == len(state.data)
+                assert state.datalen3 == len(state.data)
+                assert state.datalen4 == len(state.data)
+                debug_print(main, i, state.datalen4)
                 state.threadlocals.yield_thread()
+                assert i == j
+                j += 1
         def bootstrap():
             try:
                 runme()
@@ -50,20 +65,26 @@
                 thread.gc_thread_die()
         def f():
             state.data = []
+            state.datalen1 = 0
+            state.datalen2 = 0
+            state.datalen3 = 0
+            state.datalen4 = 0
             state.threadlocals = gil.GILThreadLocals()
             state.threadlocals.setup_threads(space)
             thread.gc_thread_prepare()
             subident = thread.start_new_thread(bootstrap, ())
             mainident = thread.get_ident()
-            runme()
+            runme(True)
             still_waiting = 3000
             while len(state.data) < 2*N:
+                debug_print(len(state.data))
                 if not still_waiting:
                     raise ValueError("time out")
                 still_waiting -= 1
                 if not we_are_translated(): gil.before_external_call()
                 time.sleep(0.01)
                 if not we_are_translated(): gil.after_external_call()
+            debug_print("leaving!")
             i1 = i2 = 0
             for tid, i in state.data:
                 if tid == mainident:
@@ -72,14 +93,17 @@
                     assert i == i2; i2 += 1
                 else:
                     assert 0
-            assert i1 == N
-            assert i2 == N
+            assert i1 == N + skew
+            assert i2 == N - skew
             return len(state.data)
 
         fn = self.getcompiled(f, [])
         res = fn()
         assert res == 2*N
 
+    def test_one_thread_rev(self):
+        self.test_one_thread(skew=-1)
+
 
 class TestRunDirectly(GILTests):
     def getcompiled(self, f, argtypes):
diff --git a/pypy/module/thread/test/test_thread.py b/pypy/module/thread/test/test_thread.py
--- a/pypy/module/thread/test/test_thread.py
+++ b/pypy/module/thread/test/test_thread.py
@@ -225,7 +225,8 @@
 
         def busy_wait():
             for x in range(1000):
-                time.sleep(0.01)
+                print 'tick...', x  # <-force the GIL to be released, as
+                time.sleep(0.01)    #   time.sleep doesn't do non-translated
 
         # This is normally called by app_main.py
         signal.signal(signal.SIGINT, signal.default_int_handler)
diff --git a/pypy/module/thread/threadlocals.py b/pypy/module/thread/threadlocals.py
--- a/pypy/module/thread/threadlocals.py
+++ b/pypy/module/thread/threadlocals.py
@@ -8,9 +8,14 @@
 
     def __init__(self):
         self._valuedict = {}   # {thread_ident: ExecutionContext()}
+        self._freeze_()
+
+    def _freeze_(self):
+        self._valuedict.clear()
         self._mainthreadident = 0
         self._mostrecentkey = 0        # fast minicaching for the common case
         self._mostrecentvalue = None   # fast minicaching for the common case
+        return False
 
     def getvalue(self):
         ident = thread.get_ident()
diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py
--- a/pypy/objspace/descroperation.py
+++ b/pypy/objspace/descroperation.py
@@ -258,15 +258,15 @@
             msg = "'%s' has no length" % (name,)
             raise OperationError(space.w_TypeError, space.wrap(msg))
         w_res = space.get_and_call_function(w_descr, w_obj)
-        space._check_len_result(w_res)
-        return w_res
+        return space.wrap(space._check_len_result(w_res))
 
     def _check_len_result(space, w_obj):
         # Will complain if result is too big.
-        result = space.int_w(w_obj)
+        result = space.int_w(space.int(w_obj))
         if result < 0:
             raise OperationError(space.w_ValueError,
                                  space.wrap("__len__() should return >= 0"))
+        return result
 
     def iter(space, w_obj):
         w_descr = space.lookup(w_obj, '__iter__')
diff --git a/pypy/objspace/std/celldict.py b/pypy/objspace/std/celldict.py
--- a/pypy/objspace/std/celldict.py
+++ b/pypy/objspace/std/celldict.py
@@ -37,10 +37,10 @@
         self.version = VersionTag()
 
     def get_empty_storage(self):
-       return self.erase({})
+        return self.erase({})
 
     def mutated(self):
-       self.version = VersionTag()
+        self.version = VersionTag()
 
     def getdictvalue_no_unwrapping(self, w_dict, key):
         # NB: it's important to promote self here, so that self.version is a
diff --git a/pypy/objspace/std/objecttype.py b/pypy/objspace/std/objecttype.py
--- a/pypy/objspace/std/objecttype.py
+++ b/pypy/objspace/std/objecttype.py
@@ -44,7 +44,6 @@
         raise OperationError(space.w_TypeError,
                              space.wrap("__class__ assignment: only for heap types"))
     w_oldcls = space.type(w_obj)
-    # XXX taint space should raise a TaintError here if w_oldcls is tainted
     assert isinstance(w_oldcls, W_TypeObject)
     if w_oldcls.get_full_instance_layout() == w_newcls.get_full_instance_layout():
         w_obj.setclass(space, w_newcls)
diff --git a/pypy/objspace/std/rangeobject.py b/pypy/objspace/std/rangeobject.py
--- a/pypy/objspace/std/rangeobject.py
+++ b/pypy/objspace/std/rangeobject.py
@@ -23,7 +23,7 @@
 
 class W_RangeListObject(W_Object):
     typedef = listtype.list_typedef
-    
+
     def __init__(w_self, start, step, length):
         assert step != 0
         w_self.start = start
@@ -40,7 +40,7 @@
         if not length:
             w_self.w_list = space.newlist([])
             return w_self.w_list
-        
+
         arr = [None] * length  # this is to avoid using append.
 
         i = start
@@ -146,7 +146,11 @@
     if length == 0:
         raise OperationError(space.w_IndexError,
                              space.wrap("pop from empty list"))
-    idx = space.int_w(w_idx)
+    if space.isinstance_w(w_idx, space.w_float):
+        raise OperationError(space.w_TypeError,
+            space.wrap("integer argument expected, got float")
+        )
+    idx = space.int_w(space.int(w_idx))
     if idx == 0:
         result = w_rangelist.start
         w_rangelist.start += w_rangelist.step
diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py
--- a/pypy/objspace/std/stringobject.py
+++ b/pypy/objspace/std/stringobject.py
@@ -161,57 +161,59 @@
 
 def str_swapcase__String(space, w_self):
     self = w_self._value
-    res = [' '] * len(self)
+    builder = StringBuilder(len(self))
     for i in range(len(self)):
         ch = self[i]
         if ch.isupper():
             o = ord(ch) + 32
-            res[i] = chr(o)
+            builder.append(chr(o))
         elif ch.islower():
             o = ord(ch) - 32
-            res[i] = chr(o)
+            builder.append(chr(o))
         else:
-            res[i] = ch
+            builder.append(ch)
 
-    return space.wrap("".join(res))
+    return space.wrap(builder.build())
 
 
 def str_capitalize__String(space, w_self):
     input = w_self._value
-    buffer = [' '] * len(input)
+    builder = StringBuilder(len(input))
     if len(input) > 0:
         ch = input[0]
         if ch.islower():
             o = ord(ch) - 32
-            buffer[0] = chr(o)
+            builder.append(chr(o))
         else:
-            buffer[0] = ch
+            builder.append(ch)
 
         for i in range(1, len(input)):
             ch = input[i]
             if ch.isupper():
                 o = ord(ch) + 32
-                buffer[i] = chr(o)
+                builder.append(chr(o))
             else:
-                buffer[i] = ch
+                builder.append(ch)
 
-    return space.wrap("".join(buffer))
+    return space.wrap(builder.build())
 
 def str_title__String(space, w_self):
     input = w_self._value
-    buffer = [' '] * len(input)
+    builder = StringBuilder(len(input))
     prev_letter=' '
 
-    for pos in range(0, len(input)):
+    for pos in range(len(input)):
         ch = input[pos]
         if not prev_letter.isalpha():
-            buffer[pos] = _upper(ch)
+            ch = _upper(ch)
+            builder.append(ch)
         else:
-            buffer[pos] = _lower(ch)
+            ch = _lower(ch)

From noreply at buildbot.pypy.org  Wed Oct 19 23:49:18 2011
From: noreply at buildbot.pypy.org (snus_mumrik)
Date: Wed, 19 Oct 2011 23:49:18 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-comparison: Check size of arrays in binary
 operations (needed for comparison)
Message-ID: <20111019214918.ECB81820D7@wyvern.cs.uni-duesseldorf.de>

Author: Ilya Osadchiy <osadchiy.ilya at gmail.com>
Branch: numpy-comparison
Changeset: r48248:949a5026cd03
Date: 2011-10-19 22:51 +0200
http://bitbucket.org/pypy/pypy/changeset/949a5026cd03/

Log:	Check size of arrays in binary operations (needed for comparison)

diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -144,6 +144,7 @@
                 w_rhs.value.convert_to(calc_dtype)
             ).wrap(space)
 
+        w_lhs, w_rhs = _broadcast_arrays(space, w_lhs, w_rhs)
         new_sig = signature.Signature.find_sig([
             self.signature, w_lhs.signature, w_rhs.signature
         ])
@@ -165,6 +166,21 @@
     reduce = interp2app(W_Ufunc.descr_reduce),
 )
 
+def _broadcast_arrays(space, a1, a2):
+    from pypy.module.micronumpy.interp_numarray import Scalar
+    '''
+    Broadcast arrays to common size
+    '''
+    # For now just check sizes of two 1D arrays
+    if isinstance(a1, Scalar) or isinstance(a2, Scalar):
+        return a1, a2
+    s1 = a1.find_size()
+    s2 = a2.find_size()
+    if s1 != s2:
+        raise operationerrfmt(space.w_ValueError, "operands could not "
+            "be broadcast together with shapes (%d) (%d)", s1, s2)
+    return a1, a2
+
 def find_binop_result_dtype(space, dt1, dt2, promote_to_float=False,
     promote_bools=False):
     # dt1.num should be <= dt2.num
diff --git a/pypy/module/micronumpy/test/test_base.py b/pypy/module/micronumpy/test/test_base.py
--- a/pypy/module/micronumpy/test/test_base.py
+++ b/pypy/module/micronumpy/test/test_base.py
@@ -31,13 +31,14 @@
 
     def test_slice_signature(self, space):
         ar = SingleDimArray(10, dtype=space.fromcache(interp_dtype.W_Float64Dtype))
-        v1 = ar.descr_getitem(space, space.wrap(slice(1, 5, 1)))
+        v1 = ar.descr_getitem(space, space.wrap(slice(1, 3, 1)))
         v2 = ar.descr_getitem(space, space.wrap(slice(4, 6, 1)))
+        v3 = ar.descr_getitem(space, space.wrap(slice(3, 5, 1)))
         assert v1.signature is v2.signature
 
-        v3 = ar.descr_add(space, v1)
-        v4 = ar.descr_add(space, v2)
-        assert v3.signature is v4.signature
+        v4 = v1.descr_add(space, v2)
+        v5 = v1.descr_add(space, v3)
+        assert v4.signature is v5.signature
 
 class TestUfuncCoerscion(object):
     def test_binops(self, space):
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -604,10 +604,9 @@
         a = array(range(5))
         assert (a == None) is False
         assert (a != None) is True
-        # TODO: uncomment after size check is implemented
-        # b = array(range(2))
-        # assert (a == b) is False
-        # assert (a != b) is True
+        b = array(range(2))
+        assert (a == b) is False
+        assert (a != b) is True
 
 class AppTestSupport(object):
     def setup_class(cls):

From noreply at buildbot.pypy.org  Thu Oct 20 00:00:36 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 20 Oct 2011 00:00:36 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix also the case of '__raw_input__'.
	Harder to test :-/
Message-ID: <20111019220036.1805E820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48249:4519d98952d9
Date: 2011-10-19 23:57 +0200
http://bitbucket.org/pypy/pypy/changeset/4519d98952d9/

Log:	Fix also the case of '__raw_input__'. Harder to test :-/

diff --git a/pypy/module/__builtin__/app_io.py b/pypy/module/__builtin__/app_io.py
--- a/pypy/module/__builtin__/app_io.py
+++ b/pypy/module/__builtin__/app_io.py
@@ -27,6 +27,19 @@
     co = compile(source.rstrip()+"\n", filename, 'exec')
     exec co in glob, loc
 
+def _write_prompt(stdout, prompt):
+    print >> stdout, prompt,
+    try:
+        flush = stdout.flush
+    except AttributeError:
+        pass
+    else:
+        flush()
+    try:
+        stdout.softspace = 0
+    except (AttributeError, TypeError):
+        pass
+
 def raw_input(prompt=''):
     """raw_input([prompt]) -> string
 
@@ -47,21 +60,10 @@
     if (hasattr(sys, '__raw_input__') and
         isinstance(stdin, file)  and stdin.fileno() == 0 and stdin.isatty() and
         isinstance(stdout, file) and stdout.fileno() == 1):
-        if prompt is None:
-            prompt = ''
-        return sys.__raw_input__(prompt)
+        _write_prompt(stdout, '')
+        return sys.__raw_input__(str(prompt))
 
-    print >> stdout, prompt,
-    try:
-        flush = stdout.flush
-    except AttributeError:
-        pass
-    else:
-        flush()
-    try:
-        stdout.softspace = 0
-    except (AttributeError, TypeError):
-        pass
+    _write_prompt(stdout, prompt)
     line = stdin.readline()
     if not line:    # inputting an empty line gives line == '\n'
         raise EOFError

From noreply at buildbot.pypy.org  Thu Oct 20 00:06:48 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 20 Oct 2011 00:06:48 +0200 (CEST)
Subject: [pypy-commit] pypy default: Also add the tests provided on issue917.
Message-ID: <20111019220648.81167820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48250:0d404d07e902
Date: 2011-10-20 00:05 +0200
http://bitbucket.org/pypy/pypy/changeset/0d404d07e902/

Log:	Also add the tests provided on issue917.

diff --git a/pypy/module/__builtin__/test/test_builtin.py b/pypy/module/__builtin__/test/test_builtin.py
--- a/pypy/module/__builtin__/test/test_builtin.py
+++ b/pypy/module/__builtin__/test/test_builtin.py
@@ -631,30 +631,6 @@
         raises(TypeError, pr, end=3)
         raises(TypeError, pr, sep=42)
 
-    def test_raw_input(self):
-        import sys, StringIO
-        for prompt, expected in [("def:", "abc/ def:/ghi\n"),
-                                 ("", "abc/ /ghi\n"),
-                                 (42, "abc/ 42/ghi\n"),
-                                 (None, "abc/ None/ghi\n"),
-                                 (Ellipsis, "abc/ /ghi\n")]:
-            save = sys.stdin, sys.stdout
-            try:
-                sys.stdin = StringIO.StringIO("foo\nbar\n")
-                out = sys.stdout = StringIO.StringIO()
-                print "abc",    # softspace = 1
-                out.write('/')
-                if prompt is Ellipsis:
-                    got = raw_input()
-                else:
-                    got = raw_input(prompt)
-                out.write('/')
-                print "ghi"
-            finally:
-                sys.stdin, sys.stdout = save
-            assert out.getvalue() == expected
-            assert got == "foo"
-
     def test_round(self):
         assert round(11.234) == 11.0
         assert round(11.234, -1) == 10.0
diff --git a/pypy/module/__builtin__/test/test_rawinput.py b/pypy/module/__builtin__/test/test_rawinput.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/__builtin__/test/test_rawinput.py
@@ -0,0 +1,77 @@
+import autopath
+
+
+class AppTestRawInput():
+
+    def test_raw_input(self):
+        import sys, StringIO
+        for prompt, expected in [("def:", "abc/ def:/ghi\n"),
+                                 ("", "abc/ /ghi\n"),
+                                 (42, "abc/ 42/ghi\n"),
+                                 (None, "abc/ None/ghi\n"),
+                                 (Ellipsis, "abc/ /ghi\n")]:
+            save = sys.stdin, sys.stdout
+            try:
+                sys.stdin = StringIO.StringIO("foo\nbar\n")
+                out = sys.stdout = StringIO.StringIO()
+                print "abc",    # softspace = 1
+                out.write('/')
+                if prompt is Ellipsis:
+                    got = raw_input()
+                else:
+                    got = raw_input(prompt)
+                out.write('/')
+                print "ghi"
+            finally:
+                sys.stdin, sys.stdout = save
+            assert out.getvalue() == expected
+            assert got == "foo"
+
+    def test_softspace(self):
+        import sys
+        import StringIO
+        fin = StringIO.StringIO()
+        fout = StringIO.StringIO()
+
+        fin.write("Coconuts\n")
+        fin.seek(0)
+
+        sys_stdin_orig = sys.stdin
+        sys_stdout_orig = sys.stdout
+
+        sys.stdin = fin
+        sys.stdout = fout
+
+        print "test",
+        raw_input("test")
+
+        sys.stdin = sys_stdin_orig
+        sys.stdout = sys_stdout_orig
+
+        fout.seek(0)
+        assert fout.read() == "test test"
+
+    def test_softspace_carryover(self):
+        import sys
+        import StringIO
+        fin = StringIO.StringIO()
+        fout = StringIO.StringIO()
+
+        fin.write("Coconuts\n")
+        fin.seek(0)
+
+        sys_stdin_orig = sys.stdin
+        sys_stdout_orig = sys.stdout
+
+        sys.stdin = fin
+        sys.stdout = fout
+
+        print "test",
+        raw_input("test")
+        print "test",
+
+        sys.stdin = sys_stdin_orig
+        sys.stdout = sys_stdout_orig
+
+        fout.seek(0)
+        assert fout.read() == "test testtest"

From noreply at buildbot.pypy.org  Thu Oct 20 03:20:29 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Thu, 20 Oct 2011 03:20:29 +0200 (CEST)
Subject: [pypy-commit] pypy default: copy a tad less in rbuilder,
 only copy the amount of data we have, not the amount we allocated
Message-ID: <20111020012029.EFB18820D7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r48251:48ae76147f6f
Date: 2011-10-19 21:18 -0400
http://bitbucket.org/pypy/pypy/changeset/48ae76147f6f/

Log:	copy a tad less in rbuilder, only copy the amount of data we have,
	not the amount we allocated

diff --git a/pypy/rpython/lltypesystem/rbuilder.py b/pypy/rpython/lltypesystem/rbuilder.py
--- a/pypy/rpython/lltypesystem/rbuilder.py
+++ b/pypy/rpython/lltypesystem/rbuilder.py
@@ -29,7 +29,7 @@
         except OverflowError:
             raise MemoryError
         newbuf = mallocfn(new_allocated)
-        copycontentsfn(ll_builder.buf, newbuf, 0, 0, ll_builder.allocated)
+        copycontentsfn(ll_builder.buf, newbuf, 0, 0, ll_builder.used)
         ll_builder.buf = newbuf
         ll_builder.allocated = new_allocated
     return func_with_new_name(stringbuilder_grow, name)
@@ -56,7 +56,7 @@
 class BaseStringBuilderRepr(AbstractStringBuilderRepr):
     def empty(self):
         return nullptr(self.lowleveltype.TO)
-    
+
     @classmethod
     def ll_new(cls, init_size):
         if init_size < 0 or init_size > MAX:

From noreply at buildbot.pypy.org  Thu Oct 20 03:20:31 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Thu, 20 Oct 2011 03:20:31 +0200 (CEST)
Subject: [pypy-commit] pypy default: merged upstream
Message-ID: <20111020012031.2277582A8A@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r48252:63d990952c7a
Date: 2011-10-19 21:20 -0400
http://bitbucket.org/pypy/pypy/changeset/63d990952c7a/

Log:	merged upstream

diff --git a/pypy/module/__builtin__/app_io.py b/pypy/module/__builtin__/app_io.py
--- a/pypy/module/__builtin__/app_io.py
+++ b/pypy/module/__builtin__/app_io.py
@@ -27,7 +27,20 @@
     co = compile(source.rstrip()+"\n", filename, 'exec')
     exec co in glob, loc
 
-def raw_input(prompt=None):
+def _write_prompt(stdout, prompt):
+    print >> stdout, prompt,
+    try:
+        flush = stdout.flush
+    except AttributeError:
+        pass
+    else:
+        flush()
+    try:
+        stdout.softspace = 0
+    except (AttributeError, TypeError):
+        pass
+
+def raw_input(prompt=''):
     """raw_input([prompt]) -> string
 
 Read a string from standard input.  The trailing newline is stripped.
@@ -47,18 +60,10 @@
     if (hasattr(sys, '__raw_input__') and
         isinstance(stdin, file)  and stdin.fileno() == 0 and stdin.isatty() and
         isinstance(stdout, file) and stdout.fileno() == 1):
-        if prompt is None:
-            prompt = ''
-        return sys.__raw_input__(prompt)
+        _write_prompt(stdout, '')
+        return sys.__raw_input__(str(prompt))
 
-    if prompt is not None:
-        stdout.write(prompt)
-        try:
-            flush = stdout.flush
-        except AttributeError:
-            pass
-        else:
-            flush()
+    _write_prompt(stdout, prompt)
     line = stdin.readline()
     if not line:    # inputting an empty line gives line == '\n'
         raise EOFError
diff --git a/pypy/module/__builtin__/test/test_rawinput.py b/pypy/module/__builtin__/test/test_rawinput.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/__builtin__/test/test_rawinput.py
@@ -0,0 +1,77 @@
+import autopath
+
+
+class AppTestRawInput():
+
+    def test_raw_input(self):
+        import sys, StringIO
+        for prompt, expected in [("def:", "abc/ def:/ghi\n"),
+                                 ("", "abc/ /ghi\n"),
+                                 (42, "abc/ 42/ghi\n"),
+                                 (None, "abc/ None/ghi\n"),
+                                 (Ellipsis, "abc/ /ghi\n")]:
+            save = sys.stdin, sys.stdout
+            try:
+                sys.stdin = StringIO.StringIO("foo\nbar\n")
+                out = sys.stdout = StringIO.StringIO()
+                print "abc",    # softspace = 1
+                out.write('/')
+                if prompt is Ellipsis:
+                    got = raw_input()
+                else:
+                    got = raw_input(prompt)
+                out.write('/')
+                print "ghi"
+            finally:
+                sys.stdin, sys.stdout = save
+            assert out.getvalue() == expected
+            assert got == "foo"
+
+    def test_softspace(self):
+        import sys
+        import StringIO
+        fin = StringIO.StringIO()
+        fout = StringIO.StringIO()
+
+        fin.write("Coconuts\n")
+        fin.seek(0)
+
+        sys_stdin_orig = sys.stdin
+        sys_stdout_orig = sys.stdout
+
+        sys.stdin = fin
+        sys.stdout = fout
+
+        print "test",
+        raw_input("test")
+
+        sys.stdin = sys_stdin_orig
+        sys.stdout = sys_stdout_orig
+
+        fout.seek(0)
+        assert fout.read() == "test test"
+
+    def test_softspace_carryover(self):
+        import sys
+        import StringIO
+        fin = StringIO.StringIO()
+        fout = StringIO.StringIO()
+
+        fin.write("Coconuts\n")
+        fin.seek(0)
+
+        sys_stdin_orig = sys.stdin
+        sys_stdout_orig = sys.stdout
+
+        sys.stdin = fin
+        sys.stdout = fout
+
+        print "test",
+        raw_input("test")
+        print "test",
+
+        sys.stdin = sys_stdin_orig
+        sys.stdout = sys_stdout_orig
+
+        fout.seek(0)
+        assert fout.read() == "test testtest"

From noreply at buildbot.pypy.org  Thu Oct 20 06:04:34 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Thu, 20 Oct 2011 06:04:34 +0200 (CEST)
Subject: [pypy-commit] pypy default: convert zip to app-level,
 it's a tad faster this way.  also, more app-level code makes me happy
Message-ID: <20111020040434.1D31B820D7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r48253:868ae3832ed8
Date: 2011-10-20 00:04 -0400
http://bitbucket.org/pypy/pypy/changeset/868ae3832ed8/

Log:	convert zip to app-level, it's a tad faster this way. also, more
	app-level code makes me happy

diff --git a/pypy/module/__builtin__/__init__.py b/pypy/module/__builtin__/__init__.py
--- a/pypy/module/__builtin__/__init__.py
+++ b/pypy/module/__builtin__/__init__.py
@@ -23,6 +23,7 @@
         'map'           : 'app_functional.map',
         'reduce'        : 'app_functional.reduce',
         'filter'        : 'app_functional.filter',
+        'zip'           : 'app_functional.zip',
         'vars'          : 'app_inspect.vars',
         'dir'           : 'app_inspect.dir',
 
@@ -89,7 +90,6 @@
         'enumerate'     : 'functional.W_Enumerate',
         'min'           : 'functional.min',
         'max'           : 'functional.max',
-        'zip'           : 'functional.zip',
         'reversed'      : 'functional.reversed',
         'super'         : 'descriptor.W_Super',
         'staticmethod'  : 'descriptor.StaticMethod',
diff --git a/pypy/module/__builtin__/app_functional.py b/pypy/module/__builtin__/app_functional.py
--- a/pypy/module/__builtin__/app_functional.py
+++ b/pypy/module/__builtin__/app_functional.py
@@ -162,4 +162,21 @@
         item = seq[i]
         if func(item):
             result.append(item)
-    return tuple(result)
\ No newline at end of file
+    return tuple(result)
+
+def zip(*sequences):
+    """zip(seq1 [, seq2 [...]]) -> [(seq1[0], seq2[0] ...), (...)]
+
+Return a list of tuples, where each tuple contains the i-th element
+from each of the argument sequences.  The returned list is truncated
+in length to the length of the shortest argument sequence."""
+    if not sequences:
+        return []
+    result = []
+    iterators = [iter(seq) for seq in sequences]
+    while True:
+        try:
+            items = [next(it) for it in iterators]
+        except StopIteration:
+            return result
+        result.append(tuple(items))
diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py
--- a/pypy/module/__builtin__/functional.py
+++ b/pypy/module/__builtin__/functional.py
@@ -201,27 +201,6 @@
     """
     return min_max(space, __args__, "min")
 
- at unwrap_spec(sequences_w="args_w")
-def zip(space, sequences_w):
-    """Return a list of tuples, where the nth tuple contains every nth item of
-    each collection.
-
-    If the collections have different lengths, zip returns a list as long as the
-    shortest collection, ignoring the trailing items in the other collections.
-    """
-    if not sequences_w:
-        return space.newlist([])
-    result_w = []
-    iterators_w = [space.iter(w_seq) for w_seq in sequences_w]
-    while True:
-        try:
-            items_w = [space.next(w_it) for w_it in iterators_w]
-        except OperationError, e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            return space.newlist(result_w)
-        result_w.append(space.newtuple(items_w))
-
 class W_Enumerate(Wrappable):
 
     def __init__(self, w_iter, w_start):

From noreply at buildbot.pypy.org  Thu Oct 20 11:00:14 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 20 Oct 2011 11:00:14 +0200 (CEST)
Subject: [pypy-commit] pypy default: Change the detection of 'AF_PACKET':
 instead of detecting it
Message-ID: <20111020090014.3E8C9820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48254:82601a1c1f1d
Date: 2011-10-20 10:58 +0200
http://bitbucket.org/pypy/pypy/changeset/82601a1c1f1d/

Log:	Change the detection of 'AF_PACKET': instead of detecting it
	automatically, force it to be True on Linux only. According to
	http://bugs.python.org/issue8852:

	 The AF_PACKET support was original meant for Linux. When
	Solaris now also supports that socket family, and with a similar
	interface, it might be interesting to port that support to
	Solaris.

	Notably, "similar" means "but not identical".

diff --git a/pypy/rlib/_rsocket_rffi.py b/pypy/rlib/_rsocket_rffi.py
--- a/pypy/rlib/_rsocket_rffi.py
+++ b/pypy/rlib/_rsocket_rffi.py
@@ -16,6 +16,7 @@
 _MINGW = target_platform.name == "mingw32"
 _SOLARIS = sys.platform == "sunos5"
 _MACOSX = sys.platform == "darwin"
+_HAS_AF_PACKET = sys.platform.startswith('linux')   # only Linux for now
 
 if _POSIX:
     includes = ('sys/types.h',
@@ -34,11 +35,12 @@
                 'stdint.h', 
                 'errno.h',
                 )
+    if _HAS_AF_PACKET:
+        includes += ('netpacket/packet.h',
+                     'sys/ioctl.h',
+                     'net/if.h')
 
-    cond_includes = [('AF_NETLINK', 'linux/netlink.h'),
-                     ('AF_PACKET', 'netpacket/packet.h'),
-                     ('AF_PACKET', 'sys/ioctl.h'),
-                     ('AF_PACKET', 'net/if.h')]
+    cond_includes = [('AF_NETLINK', 'linux/netlink.h')]
     
     libraries = ()
     calling_conv = 'c'
@@ -320,18 +322,18 @@
                                              ('events', rffi.SHORT),
                                              ('revents', rffi.SHORT)])
 
-    CConfig.sockaddr_ll = platform.Struct('struct sockaddr_ll',
+    if _HAS_AF_PACKET:
+        CConfig.sockaddr_ll = platform.Struct('struct sockaddr_ll',
                               [('sll_ifindex', rffi.INT),
                                ('sll_protocol', rffi.INT),
                                ('sll_pkttype', rffi.INT),
                                ('sll_hatype', rffi.INT),
                                ('sll_addr', rffi.CFixedArray(rffi.CHAR, 8)),
-                               ('sll_halen', rffi.INT)],
-                              ifdef='AF_PACKET')
+                               ('sll_halen', rffi.INT)])
 
-    CConfig.ifreq = platform.Struct('struct ifreq', [('ifr_ifindex', rffi.INT),
-                                 ('ifr_name', rffi.CFixedArray(rffi.CHAR, 8))],
-                                    ifdef='AF_PACKET')
+        CConfig.ifreq = platform.Struct('struct ifreq',
+                                [('ifr_ifindex', rffi.INT),
+                                 ('ifr_name', rffi.CFixedArray(rffi.CHAR, 8))])
 
 if _WIN32:
     CConfig.WSAEVENT = platform.SimpleType('WSAEVENT', rffi.VOIDP)
@@ -386,6 +388,8 @@
         constants[name] = value
     else:
         constants[name] = default
+if not _HAS_AF_PACKET and 'AF_PACKET' in constants:
+    del constants['AF_PACKET']
 
 constants['has_ipv6'] = True # This is a configuration option in CPython
 for name, value in constants.items():
@@ -439,21 +443,14 @@
 if _POSIX:
     nfds_t = cConfig.nfds_t
     pollfd = cConfig.pollfd
-    if cConfig.sockaddr_ll is not None:
+    if _HAS_AF_PACKET:
         sockaddr_ll = cConfig.sockaddr_ll
-    ifreq = cConfig.ifreq
+        ifreq = cConfig.ifreq
 if WIN32:
     WSAEVENT = cConfig.WSAEVENT
     WSANETWORKEVENTS = cConfig.WSANETWORKEVENTS
 timeval = cConfig.timeval
 
-#if _POSIX:
-#    includes = list(includes)
-#    for _name, _header in cond_includes:
-#        if getattr(cConfig, _name) is not None:
-#            includes.append(_header)
-#    eci = ExternalCompilationInfo(includes=includes, libraries=libraries,
-#                                  separate_module_sources=sources)
 
 def external(name, args, result, **kwds):
     return rffi.llexternal(name, args, result, compilation_info=eci,
@@ -544,7 +541,7 @@
     socketpair_t = rffi.CArray(socketfd_type)
     socketpair = external('socketpair', [rffi.INT, rffi.INT, rffi.INT,
                           lltype.Ptr(socketpair_t)], rffi.INT)
-    if ifreq is not None:
+    if _HAS_AF_PACKET:
         ioctl = external('ioctl', [socketfd_type, rffi.INT, lltype.Ptr(ifreq)],
                          rffi.INT)
 
diff --git a/pypy/rlib/rsocket.py b/pypy/rlib/rsocket.py
--- a/pypy/rlib/rsocket.py
+++ b/pypy/rlib/rsocket.py
@@ -8,6 +8,7 @@
 # Known missing features:
 #
 #   - address families other than AF_INET, AF_INET6, AF_UNIX, AF_PACKET
+#   - AF_PACKET is only supported on Linux
 #   - methods makefile(),
 #   - SSL
 #

From noreply at buildbot.pypy.org  Thu Oct 20 11:00:15 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 20 Oct 2011 11:00:15 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20111020090015.82D0F820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48255:8fe3564dc575
Date: 2011-10-20 10:59 +0200
http://bitbucket.org/pypy/pypy/changeset/8fe3564dc575/

Log:	merge heads

diff --git a/pypy/module/__builtin__/__init__.py b/pypy/module/__builtin__/__init__.py
--- a/pypy/module/__builtin__/__init__.py
+++ b/pypy/module/__builtin__/__init__.py
@@ -23,6 +23,7 @@
         'map'           : 'app_functional.map',
         'reduce'        : 'app_functional.reduce',
         'filter'        : 'app_functional.filter',
+        'zip'           : 'app_functional.zip',
         'vars'          : 'app_inspect.vars',
         'dir'           : 'app_inspect.dir',
 
@@ -89,7 +90,6 @@
         'enumerate'     : 'functional.W_Enumerate',
         'min'           : 'functional.min',
         'max'           : 'functional.max',
-        'zip'           : 'functional.zip',
         'reversed'      : 'functional.reversed',
         'super'         : 'descriptor.W_Super',
         'staticmethod'  : 'descriptor.StaticMethod',
diff --git a/pypy/module/__builtin__/app_functional.py b/pypy/module/__builtin__/app_functional.py
--- a/pypy/module/__builtin__/app_functional.py
+++ b/pypy/module/__builtin__/app_functional.py
@@ -162,4 +162,21 @@
         item = seq[i]
         if func(item):
             result.append(item)
-    return tuple(result)
\ No newline at end of file
+    return tuple(result)
+
+def zip(*sequences):
+    """zip(seq1 [, seq2 [...]]) -> [(seq1[0], seq2[0] ...), (...)]
+
+Return a list of tuples, where each tuple contains the i-th element
+from each of the argument sequences.  The returned list is truncated
+in length to the length of the shortest argument sequence."""
+    if not sequences:
+        return []
+    result = []
+    iterators = [iter(seq) for seq in sequences]
+    while True:
+        try:
+            items = [next(it) for it in iterators]
+        except StopIteration:
+            return result
+        result.append(tuple(items))
diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py
--- a/pypy/module/__builtin__/functional.py
+++ b/pypy/module/__builtin__/functional.py
@@ -201,27 +201,6 @@
     """
     return min_max(space, __args__, "min")
 
- at unwrap_spec(sequences_w="args_w")
-def zip(space, sequences_w):
-    """Return a list of tuples, where the nth tuple contains every nth item of
-    each collection.
-
-    If the collections have different lengths, zip returns a list as long as the
-    shortest collection, ignoring the trailing items in the other collections.
-    """
-    if not sequences_w:
-        return space.newlist([])
-    result_w = []
-    iterators_w = [space.iter(w_seq) for w_seq in sequences_w]
-    while True:
-        try:
-            items_w = [space.next(w_it) for w_it in iterators_w]
-        except OperationError, e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            return space.newlist(result_w)
-        result_w.append(space.newtuple(items_w))
-
 class W_Enumerate(Wrappable):
 
     def __init__(self, w_iter, w_start):
diff --git a/pypy/rpython/lltypesystem/rbuilder.py b/pypy/rpython/lltypesystem/rbuilder.py
--- a/pypy/rpython/lltypesystem/rbuilder.py
+++ b/pypy/rpython/lltypesystem/rbuilder.py
@@ -29,7 +29,7 @@
         except OverflowError:
             raise MemoryError
         newbuf = mallocfn(new_allocated)
-        copycontentsfn(ll_builder.buf, newbuf, 0, 0, ll_builder.allocated)
+        copycontentsfn(ll_builder.buf, newbuf, 0, 0, ll_builder.used)
         ll_builder.buf = newbuf
         ll_builder.allocated = new_allocated
     return func_with_new_name(stringbuilder_grow, name)
@@ -56,7 +56,7 @@
 class BaseStringBuilderRepr(AbstractStringBuilderRepr):
     def empty(self):
         return nullptr(self.lowleveltype.TO)
-    
+
     @classmethod
     def ll_new(cls, init_size):
         if init_size < 0 or init_size > MAX:

From noreply at buildbot.pypy.org  Thu Oct 20 11:02:48 2011
From: noreply at buildbot.pypy.org (hager)
Date: Thu, 20 Oct 2011 11:02:48 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: little beautification
Message-ID: <20111020090248.7EF21820D7@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r48256:5ef0586e29c2
Date: 2011-10-20 11:00 +0200
http://bitbucket.org/pypy/pypy/changeset/5ef0586e29c2/

Log:	little beautification

diff --git a/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py b/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py
--- a/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py
@@ -534,7 +534,8 @@
 
             if not tok.is_invalidate:
                 mc = PPCBuilder()
-                mc.b_cond_offset(descr._ppc_guard_pos - tok.offset, tok.fcond)
+                offset = descr._ppc_guard_pos - tok.offset
+                mc.b_cond_offset(offset, tok.fcond)
                 mc.prepare_insts_blocks(True)
                 mc.copy_to_raw_memory(block_start + tok.offset)
             else:

From noreply at buildbot.pypy.org  Thu Oct 20 11:02:49 2011
From: noreply at buildbot.pypy.org (hager)
Date: Thu, 20 Oct 2011 11:02:49 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: Implemented overflow operations.
Message-ID: <20111020090249.ABE9B820D7@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r48257:d2527483c5c3
Date: 2011-10-20 11:02 +0200
http://bitbucket.org/pypy/pypy/changeset/d2527483c5c3/

Log:	Implemented overflow operations.

diff --git a/pypy/jit/backend/ppc/ppcgen/opassembler.py b/pypy/jit/backend/ppc/ppcgen/opassembler.py
--- a/pypy/jit/backend/ppc/ppcgen/opassembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/opassembler.py
@@ -32,6 +32,10 @@
         else:
             self.mc.add(res.value, l0.value, l1.value)
 
+    def emit_int_add_ovf(self, op, arglocs, regalloc):
+        l0, l1, res = arglocs
+        self.mc.addo(res.value, l0.value, l1.value)
+
     def emit_int_sub(self, op, arglocs, regalloc):
         l0, l1, res = arglocs
         if l0.is_imm():
@@ -42,10 +46,22 @@
         else:
             self.mc.sub(res.value, l0.value, l1.value)
  
+    def emit_int_sub_ovf(self, op, arglocs, regalloc):
+        l0, l1, res = arglocs
+        self.mc.subfo(res.value, l1.value, l0.value)
+
     def emit_int_mul(self, op, arglocs, regalloc):
         reg1, reg2, res = arglocs
         self.mc.mullw(res.value, reg1.value, reg2.value)
 
+    def emit_int_mul_ovf(self, op, arglocs, regalloc):
+        reg1, reg2, res = arglocs
+        self.mc.mullwo(res.value, reg1.value, reg2.value)
+
+    def emit_int_mul_ovf(self, op, arglocs, regalloc):
+        l0, l1, res = arglocs
+        self.mc.mullwo(res.value, l0.value, l1.value)
+
     def emit_int_floordiv(self, op, arglocs, regalloc):
         l0, l1, res = arglocs
         if IS_PPC_32:
@@ -158,6 +174,23 @@
         #                        #      ^^^^ If this condition is met,
         #                        #           then the guard fails.
 
+    # TODO - Evaluate whether this can be done with 
+    #        SO bit instead of OV bit => usage of CR
+    #        instead of XER could be more efficient
+    def _emit_ovf_guard(self, op, arglocs, cond):
+        # move content of XER to GPR
+        self.mc.mfspr(r.r0.value, 1)
+        # shift and mask to get comparison result
+        self.mc.rlwinm(r.r0.value, r.r0.value, 1, 0, 0)
+        self.mc.cmpi(r.r0.value, 0)
+        self._emit_guard(op, arglocs, cond)
+
+    def emit_guard_no_overflow(self, op, arglocs, regalloc):
+        self._emit_ovf_guard(op, arglocs, c.NE)
+
+    def emit_guard_overflow(self, op, arglocs, regalloc):
+        self._emit_ovf_guard(op, arglocs, c.EQ)
+
     def emit_finish(self, op, arglocs, regalloc):
         self.gen_exit_stub(op.getdescr(), op.getarglist(), arglocs)
 
diff --git a/pypy/jit/backend/ppc/ppcgen/regalloc.py b/pypy/jit/backend/ppc/ppcgen/regalloc.py
--- a/pypy/jit/backend/ppc/ppcgen/regalloc.py
+++ b/pypy/jit/backend/ppc/ppcgen/regalloc.py
@@ -193,6 +193,10 @@
     prepare_uint_rshift = prepare_binary_int_op()
     prepare_uint_floordiv = prepare_binary_int_op()
 
+    prepare_int_add_ovf = prepare_binary_int_op()
+    prepare_int_sub_ovf = prepare_binary_int_op()
+    prepare_int_mul_ovf = prepare_binary_int_op()
+
     prepare_int_neg = prepare_unary_int_op()
     prepare_int_invert = prepare_unary_int_op()
 
@@ -240,6 +244,13 @@
         self.possibly_free_vars(op.getfailargs())
         return args
 
+    def prepare_guard_no_overflow(self, op):
+        locs = self._prepare_guard(op)
+        self.possibly_free_vars(op.getfailargs())
+        return locs
+
+    prepare_guard_overflow = prepare_guard_no_overflow
+
     def prepare_jump(self, op):
         descr = op.getdescr()
         assert isinstance(descr, LoopToken)

From noreply at buildbot.pypy.org  Thu Oct 20 11:10:28 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 20 Oct 2011 11:10:28 +0200 (CEST)
Subject: [pypy-commit] pypy default: Maybe fixes solaris for now.
Message-ID: <20111020091028.BCDF3820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48258:5b15ee11ff63
Date: 2011-10-20 11:09 +0200
http://bitbucket.org/pypy/pypy/changeset/5b15ee11ff63/

Log:	Maybe fixes solaris for now.

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -72,6 +72,7 @@
     del working_modules['fcntl']  # LOCK_NB not defined
     del working_modules["_minimal_curses"]
     del working_modules["termios"]
+    del working_modules["_multiprocessing"]   # depends on rctime
 
 
 

From noreply at buildbot.pypy.org  Thu Oct 20 12:43:43 2011
From: noreply at buildbot.pypy.org (hager)
Date: Thu, 20 Oct 2011 12:43:43 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: Fixed bug in load_from_addr. It
 did not do the intended stuff when called with r0.
Message-ID: <20111020104343.B067D820D7@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r48259:440b9131acbf
Date: 2011-10-20 12:38 +0200
http://bitbucket.org/pypy/pypy/changeset/440b9131acbf/

Log:	Fixed bug in load_from_addr. It did not do the intended stuff when
	called with r0.

diff --git a/pypy/jit/backend/ppc/ppcgen/codebuilder.py b/pypy/jit/backend/ppc/ppcgen/codebuilder.py
--- a/pypy/jit/backend/ppc/ppcgen/codebuilder.py
+++ b/pypy/jit/backend/ppc/ppcgen/codebuilder.py
@@ -951,8 +951,8 @@
 
     def load_from_addr(self, rD, addr):
         if IS_PPC_32:
-            self.addis(rD, 0, ha(addr))
-            self.lwz(rD, rD, la(addr))
+            self.load_imm(rD, addr)
+            self.lwzx(rD.value, 0, rD.value)
         else:
             self.load_word(rD, addr)
             self.ld(rD, rD, 0)

From noreply at buildbot.pypy.org  Thu Oct 20 12:43:44 2011
From: noreply at buildbot.pypy.org (hager)
Date: Thu, 20 Oct 2011 12:43:44 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: Fixed test to work with new
	load_from_addr.
Message-ID: <20111020104344.D6A3E82A8A@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r48260:b96e59515c28
Date: 2011-10-20 12:40 +0200
http://bitbucket.org/pypy/pypy/changeset/b96e59515c28/

Log:	Fixed test to work with new load_from_addr.

diff --git a/pypy/jit/backend/ppc/ppcgen/test/test_ppc.py b/pypy/jit/backend/ppc/ppcgen/test/test_ppc.py
--- a/pypy/jit/backend/ppc/ppcgen/test/test_ppc.py
+++ b/pypy/jit/backend/ppc/ppcgen/test/test_ppc.py
@@ -294,7 +294,7 @@
         addr = rffi.cast(lltype.Signed, p)
         p[0] = rffi.cast(rffi.LONG, 200)
 
-        a.load_from_addr(3, addr)
+        a.load_from_addr(r3, addr)
         a.blr()
         f = a.assemble()
         assert f() == 200

From noreply at buildbot.pypy.org  Thu Oct 20 12:43:46 2011
From: noreply at buildbot.pypy.org (hager)
Date: Thu, 20 Oct 2011 12:43:46 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: Implemented passing of pointer
	values.
Message-ID: <20111020104346.0C75A82A8B@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r48261:c32669c9d45d
Date: 2011-10-20 12:43 +0200
http://bitbucket.org/pypy/pypy/changeset/c32669c9d45d/

Log:	Implemented passing of pointer values.

diff --git a/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py b/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py
--- a/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py
@@ -96,14 +96,6 @@
             self.mc.oris(rD, rD, high(word))
             self.mc.ori(rD, rD, lo(word))
 
-    def load_from_addr(self, rD, addr):
-        if IS_PPC_32:
-            self.mc.addis(rD, 0, ha(addr))
-            self.mc.lwz(rD, rD, la(addr))
-        else:
-            self.load_word(rD, addr)
-            self.mc.ld(rD, rD, 0)
-
     def store_reg(self, source_reg, addr):
         self.load_imm(r.r0.value, addr)
         if IS_PPC_32:
@@ -342,15 +334,17 @@
             assert arg.type != FLOAT
             if arg.type == INT:
                 addr = self.fail_boxes_int.get_addr_for_num(i)
-            elif args.type == REF:
+            elif arg.type == REF:
                 addr = self.fail_boxes_ptr.get_addr_for_num(i)
             else:
                 assert 0, "%s not supported" % arg.type
             if loc.is_reg():
                 reg = loc
             else:
-                assert 0, "FIX LATER"
-            self.load_from_addr(reg.value, addr)
+                reg = r.r0
+            self.mc.load_from_addr(reg, addr)
+            if loc.is_stack():
+                self.regalloc_mov(r.r0, loc)
 
     def setup(self, looptoken, operations):
         operations = self.cpu.gc_ll_descr.rewrite_assembler(self.cpu, 
diff --git a/pypy/jit/backend/ppc/runner.py b/pypy/jit/backend/ppc/runner.py
--- a/pypy/jit/backend/ppc/runner.py
+++ b/pypy/jit/backend/ppc/runner.py
@@ -90,8 +90,7 @@
         self.asm.fail_boxes_int.setitem(index, value_int)
 
     def set_future_value_ref(self, index, pointer):
-        sign_ptr = rffi.cast(lltype.Signed, pointer)
-        self.fail_boxes_int.setitem(index, sign_ptr)
+        self.asm.fail_boxes_ptr.setitem(index, pointer)
 
     def clear_latest_values(self, count):
         null = lltype.nullptr(llmemory.GCREF.TO)
@@ -133,8 +132,7 @@
         return value
 
     def get_latest_value_ref(self, index):
-        value = self.fail_boxes_int.getitem(index)
-        return rffi.cast(llmemory.GCREF, value)
+        return self.asm.fail_boxes_ptr.getitem(index)
     
     # walk through the given trace and generate machine code
     def _walk_trace_ops(self, codebuilder, operations):

From noreply at buildbot.pypy.org  Thu Oct 20 13:37:37 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 20 Oct 2011 13:37:37 +0200 (CEST)
Subject: [pypy-commit] pypy faster-json: implement encode enough so it
	passes tests
Message-ID: <20111020113737.3E03A820D7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: faster-json
Changeset: r48262:1139520345e7
Date: 2011-10-20 13:37 +0200
http://bitbucket.org/pypy/pypy/changeset/1139520345e7/

Log:	implement encode enough so it passes tests

diff --git a/lib-python/modified-2.7/json/encoder.py b/lib-python/modified-2.7/json/encoder.py
--- a/lib-python/modified-2.7/json/encoder.py
+++ b/lib-python/modified-2.7/json/encoder.py
@@ -3,6 +3,7 @@
 import re
 
 from __pypy__ import identity_dict
+from __pypy__.builders import StringBuilder, UnicodeBuilder
 
 ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]')
 ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])')
@@ -30,7 +31,7 @@
     """
     def replace(match):
         return ESCAPE_DCT[match.group(0)]
-    return '"' + ESCAPE.sub(replace, s) + '"'
+    return ESCAPE.sub(replace, s)
 
 def encode_basestring_ascii(s):
     """Return an ASCII-only JSON representation of a Python string
@@ -45,17 +46,17 @@
         except KeyError:
             n = ord(s)
             if n < 0x10000:
-                return '\\u{0:04x}'.format(n)
-                #return '\\u%04x' % (n,)
+                #return '\\u{0:04x}'.format(n)
+                return '\\u%04x' % (n,)
             else:
                 # surrogate pair
                 n -= 0x10000
                 s1 = 0xd800 | ((n >> 10) & 0x3ff)
                 s2 = 0xdc00 | (n & 0x3ff)
-                return '\\u{0:04x}\\u{1:04x}'.format(s1, s2)
-                #return '\\u%04x\\u%04x' % (s1, s2)
-    return '"' + str(ESCAPE_ASCII.sub(replace, s)) + '"'
-py_encode_basestring_ascii = encode_basestring_ascii
+                #return '\\u{0:04x}\\u{1:04x}'.format(s1, s2)
+                return '\\u%04x\\u%04x' % (s1, s2)
+    return str(ESCAPE_ASCII.sub(replace, s))
+py_encode_basestring_ascii = lambda s: '"' + encode_basestring_ascii(s) + '"'
 c_encode_basestring_ascii = None
 
 class JSONEncoder(object):
@@ -185,24 +186,126 @@
         '{"foo": ["bar", "baz"]}'
 
         """
-        # This is for extremely simple cases and benchmarks.
+        if self.check_circular:
+            markers = identity_dict()
+        else:
+            markers = None
+        if self.ensure_ascii:
+            builder = StringBuilder()
+        else:
+            builder = UnicodeBuilder()
+        self._encode(o, markers, builder, 0)
+        return builder.build()
+
+    def _emit_indent(self, builder, _current_indent_level):        
+        if self.indent is not None:
+            _current_indent_level += 1
+            newline_indent = '\n' + (' ' * (self.indent *
+                                            _current_indent_level))
+            separator = self.item_separator + newline_indent
+            builder.append(newline_indent)
+        else:
+            separator = self.item_separator
+        return separator, _current_indent_level
+
+    def _emit_unindent(self, builder, _current_indent_level):
+        if self.indent is not None:
+            builder.append('\n')
+            builder.append(' ' * (self.indent * (_current_indent_level - 1)))
+
+    def _encode(self, o, markers, builder, _current_indent_level):
         if isinstance(o, basestring):
-            if isinstance(o, str):
-                _encoding = self.encoding
-                if (_encoding is not None
-                        and not (_encoding == 'utf-8')):
-                    o = o.decode(_encoding)
-            if self.ensure_ascii:
-                return encode_basestring_ascii(o)
+            builder.append('"')
+            builder.append(self.encoder(o))
+            builder.append('"')
+        elif o is None:
+            builder.append('null')
+        elif o is True:
+            builder.append('true')
+        elif o is False:
+            builder.append('false')
+        elif isinstance(o, (int, long)):
+            builder.append(str(o))
+        elif isinstance(o, float):
+            builder.append(self._floatstr(o))
+        elif isinstance(o, (list, tuple)):
+            if not o:
+                builder.append('[]')
+                return
+            self._encode_list(o, markers, builder, _current_indent_level)
+        elif isinstance(o, dict):
+            if not o:
+                builder.append('{}')
+                return
+            self._encode_dict(o, markers, builder, _current_indent_level)
+        else:
+            self._mark_markers(markers, o)
+            res = self.default(o)
+            self._encode(res, markers, builder, _current_indent_level)
+            self._remove_markers(markers, o)
+            return res
+
+    def _encode_list(self, l, markers, builder, _current_indent_level):
+        self._mark_markers(markers, l)
+        builder.append('[')
+        first = True
+        separator, _current_indent_level = self._emit_indent(builder,
+                                                      _current_indent_level)
+        for elem in l:
+            if first:
+                first = False
             else:
-                return encode_basestring(o)
-        # This doesn't pass the iterator directly to ''.join() because the
-        # exceptions aren't as detailed.  The list call should be roughly
-        # equivalent to the PySequence_Fast that ''.join() would do.        
-        chunks = self.iterencode(o, _one_shot=True)
-        if not isinstance(chunks, (list, tuple)):
-            chunks = list(chunks)
-        return ''.join(chunks)
+                builder.append(separator)
+            self._encode(elem, markers, builder, _current_indent_level)
+            del elem # XXX grumble
+        self._emit_unindent(builder, _current_indent_level)
+        builder.append(']')
+        self._remove_markers(markers, l)
+
+    def _encode_dict(self, d, markers, builder, _current_indent_level):
+        self._mark_markers(markers, d)
+        first = True
+        builder.append('{')
+        separator, _current_indent_level = self._emit_indent(builder,
+                                                         _current_indent_level)
+        if self.sort_keys:
+            items = sorted(d.items(), key=lambda kv: kv[0])
+        else:
+            items = d.iteritems()
+
+        for key, v in items:
+            if first:
+                first = False
+            else:
+                builder.append(separator)
+            if isinstance(key, basestring):
+                pass
+            # JavaScript is weakly typed for these, so it makes sense to
+            # also allow them.  Many encoders seem to do something like this.
+            elif isinstance(key, float):
+                key = self._floatstr(key)
+            elif key is True:
+                key = 'true'
+            elif key is False:
+                key = 'false'
+            elif key is None:
+                key = 'null'
+            elif isinstance(key, (int, long)):
+                key = str(key)
+            elif self.skipkeys:
+                continue
+            else:
+                raise TypeError("key " + repr(key) + " is not a string")
+            builder.append('"')
+            builder.append(self.encoder(key))
+            builder.append('"')
+            builder.append(self.key_separator)
+            self._encode(v, markers, builder, _current_indent_level)
+            del key
+            del v # XXX grumble
+        self._emit_unindent(builder, _current_indent_level)
+        builder.append('}')
+        self._remove_markers(markers, d)
 
     def iterencode(self, o, _one_shot=False):
         """Encode the given object and yield each string
@@ -273,7 +376,7 @@
             else:
                 buf = separator
             if isinstance(value, basestring):
-                yield buf + self.encoder(value)
+                yield buf + '"' + self.encoder(value) + '"'
             elif value is None:
                 yield buf + 'null'
             elif value is True:
@@ -346,10 +449,10 @@
                 first = False
             else:
                 yield item_separator
-            yield self.encoder(key)
+            yield '"' + self.encoder(key) + '"'
             yield self.key_separator
             if isinstance(value, basestring):
-                yield self.encoder(value)
+                yield '"' + self.encoder(value) + '"'
             elif value is None:
                 yield 'null'
             elif value is True:
@@ -380,7 +483,7 @@
 
     def _iterencode(self, o, markers, _current_indent_level):
         if isinstance(o, basestring):
-            yield self.encoder(o)
+            yield '"' + self.encoder(o) + '"'
         elif o is None:
             yield 'null'
         elif o is True:

From noreply at buildbot.pypy.org  Thu Oct 20 13:38:40 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 20 Oct 2011 13:38:40 +0200 (CEST)
Subject: [pypy-commit] pypy faster-json: a little speedup
Message-ID: <20111020113840.31F17820D7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: faster-json
Changeset: r48263:947621c9169c
Date: 2011-10-20 13:38 +0200
http://bitbucket.org/pypy/pypy/changeset/947621c9169c/

Log:	a little speedup

diff --git a/lib-python/modified-2.7/json/encoder.py b/lib-python/modified-2.7/json/encoder.py
--- a/lib-python/modified-2.7/json/encoder.py
+++ b/lib-python/modified-2.7/json/encoder.py
@@ -55,7 +55,9 @@
                 s2 = 0xdc00 | (n & 0x3ff)
                 #return '\\u{0:04x}\\u{1:04x}'.format(s1, s2)
                 return '\\u%04x\\u%04x' % (s1, s2)
-    return str(ESCAPE_ASCII.sub(replace, s))
+    if ESCAPE_ASCII.search(s):
+        return str(ESCAPE_ASCII.sub(replace, s))
+    return s
 py_encode_basestring_ascii = lambda s: '"' + encode_basestring_ascii(s) + '"'
 c_encode_basestring_ascii = None
 

From noreply at buildbot.pypy.org  Thu Oct 20 15:14:08 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 20 Oct 2011 15:14:08 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix: the case marked as "#PyPy" was
 never actually followed any more.
Message-ID: <20111020131408.D1464820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48264:0d7e8867e6c0
Date: 2011-10-20 15:13 +0200
http://bitbucket.org/pypy/pypy/changeset/0d7e8867e6c0/

Log:	Fix: the case marked as "#PyPy" was never actually followed any
	more. It is doing something better than the other case, so use it
	again. It also fixes issue917 on the interactive prompt.

diff --git a/lib_pypy/pyrepl/readline.py b/lib_pypy/pyrepl/readline.py
--- a/lib_pypy/pyrepl/readline.py
+++ b/lib_pypy/pyrepl/readline.py
@@ -395,9 +395,21 @@
     _wrapper.f_in = f_in
     _wrapper.f_out = f_out
 
-    if hasattr(sys, '__raw_input__'):    # PyPy
-        _old_raw_input = sys.__raw_input__
+    if '__pypy__' in sys.builtin_module_names:    # PyPy
+
+        def _old_raw_input(prompt=''):
+            # sys.__raw_input__() is only called when stdin and stdout are
+            # as expected and are ttys.  If it is the case, then get_reader()
+            # should not really fail in _wrapper.raw_input().  If it still
+            # does, then we will just cancel the redirection and call again
+            # the built-in raw_input().
+            try:
+                del sys.__raw_input__
+            except AttributeError:
+                pass
+            return raw_input(prompt)
         sys.__raw_input__ = _wrapper.raw_input
+
     else:
         # this is not really what readline.c does.  Better than nothing I guess
         import __builtin__

From noreply at buildbot.pypy.org  Thu Oct 20 15:15:58 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 20 Oct 2011 15:15:58 +0200 (CEST)
Subject: [pypy-commit] pyrepl default: Patch from pypy's 0d7e8867e6c0
Message-ID: <20111020131558.0A7E5820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r149:f4d54136a74c
Date: 2011-10-20 15:15 +0200
http://bitbucket.org/pypy/pyrepl/changeset/f4d54136a74c/

Log:	Patch from pypy's 0d7e8867e6c0

diff --git a/pyrepl/readline.py b/pyrepl/readline.py
--- a/pyrepl/readline.py
+++ b/pyrepl/readline.py
@@ -395,9 +395,21 @@
     _wrapper.f_in = f_in
     _wrapper.f_out = f_out
 
-    if hasattr(sys, '__raw_input__'):    # PyPy
-        _old_raw_input = sys.__raw_input__
+    if '__pypy__' in sys.builtin_module_names:    # PyPy
+
+        def _old_raw_input(prompt=''):
+            # sys.__raw_input__() is only called when stdin and stdout are
+            # as expected and are ttys.  If it is the case, then get_reader()
+            # should not really fail in _wrapper.raw_input().  If it still
+            # does, then we will just cancel the redirection and call again
+            # the built-in raw_input().
+            try:
+                del sys.__raw_input__
+            except AttributeError:
+                pass
+            return raw_input(prompt)
         sys.__raw_input__ = _wrapper.raw_input
+
     else:
         # this is not really what readline.c does.  Better than nothing I guess
         import __builtin__

From noreply at buildbot.pypy.org  Thu Oct 20 16:12:11 2011
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Thu, 20 Oct 2011 16:12:11 +0200 (CEST)
Subject: [pypy-commit] pyrepl py3ksupport: fix type check error
Message-ID: <20111020141211.A1397820D7@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: py3ksupport
Changeset: r150:20637596ceea
Date: 2011-10-20 15:39 +0200
http://bitbucket.org/pypy/pyrepl/changeset/20637596ceea/

Log:	fix type check error

diff --git a/pyrepl/reader.py b/pyrepl/reader.py
--- a/pyrepl/reader.py
+++ b/pyrepl/reader.py
@@ -524,7 +524,7 @@
 
     def do_cmd(self, cmd):
         #print cmd
-        if isinstance(cmd[0], str):
+        if isinstance(cmd[0], basestring): #XXX: unify to text
             cmd = self.commands.get(cmd[0],
                                     commands.invalid_command)(self, *cmd)
         elif isinstance(cmd[0], type):

From noreply at buildbot.pypy.org  Thu Oct 20 16:37:30 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 20 Oct 2011 16:37:30 +0200 (CEST)
Subject: [pypy-commit] pypy gmp: A branch in which to reimplement rbigint
 using the gmp library.
Message-ID: <20111020143730.987F3820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: gmp
Changeset: r48265:df022cedef34
Date: 2011-10-20 15:31 +0200
http://bitbucket.org/pypy/pypy/changeset/df022cedef34/

Log:	A branch in which to reimplement rbigint using the gmp library.


From noreply at buildbot.pypy.org  Thu Oct 20 16:37:31 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 20 Oct 2011 16:37:31 +0200 (CEST)
Subject: [pypy-commit] pypy gmp: Move away rbigint.py to its own
 "one-possible-implementation" module.
Message-ID: <20111020143731.D3A5B820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: gmp
Changeset: r48266:0bfa78d1c3c5
Date: 2011-10-20 15:41 +0200
http://bitbucket.org/pypy/pypy/changeset/0bfa78d1c3c5/

Log:	Move away rbigint.py to its own "one-possible-implementation"
	module.

diff --git a/pypy/objspace/std/longobject.py b/pypy/objspace/std/longobject.py
--- a/pypy/objspace/std/longobject.py
+++ b/pypy/objspace/std/longobject.py
@@ -6,7 +6,7 @@
 from pypy.objspace.std.multimethod import FailedToImplementArgs
 from pypy.objspace.std.intobject import W_IntObject
 from pypy.objspace.std.noneobject import W_NoneObject
-from pypy.rlib.rbigint import rbigint, SHIFT
+from pypy.rlib.rbigint import rbigint
 
 class W_LongObject(W_Object):
     """This is a wrapper of rbigint."""
diff --git a/pypy/rlib/rbigint.py b/pypy/rlib/_rbigint_native.py
copy from pypy/rlib/rbigint.py
copy to pypy/rlib/_rbigint_native.py
diff --git a/pypy/rlib/rbigint.py b/pypy/rlib/rbigint.py
--- a/pypy/rlib/rbigint.py
+++ b/pypy/rlib/rbigint.py
@@ -1,1904 +1,9 @@
-from pypy.rlib.rarithmetic import LONG_BIT, intmask, r_uint, r_ulonglong
-from pypy.rlib.rarithmetic import ovfcheck, r_longlong, widen
-from pypy.rlib.rarithmetic import most_neg_value_of_same_type
-from pypy.rlib.rfloat import isfinite
-from pypy.rlib.debug import make_sure_not_resized, check_regular_int
-from pypy.rlib.objectmodel import we_are_translated, specialize
-from pypy.rlib import jit
-from pypy.rpython.lltypesystem import lltype, rffi
-from pypy.rpython import extregistry
 
-import math, sys
 
-# note about digit sizes:
-# In division, the native integer type must be able to hold
-# a sign bit plus two digits plus 1 overflow bit.
+USE_GMP = False
 
-#SHIFT = (LONG_BIT // 2) - 1
-SHIFT = 31
 
-MASK = int((1 << SHIFT) - 1)
-FLOAT_MULTIPLIER = float(1 << SHIFT)
-
-
-# Debugging digit array access.
-#
-# False == no checking at all
-# True == check 0 <= value <= MASK
-
-
-# For long multiplication, use the O(N**2) school algorithm unless
-# both operands contain more than KARATSUBA_CUTOFF digits (this
-# being an internal Python long digit, in base BASE).
-
-USE_KARATSUBA = True # set to False for comparison
-KARATSUBA_CUTOFF = 70
-KARATSUBA_SQUARE_CUTOFF = 2 * KARATSUBA_CUTOFF
-
-# For exponentiation, use the binary left-to-right algorithm
-# unless the exponent contains more than FIVEARY_CUTOFF digits.
-# In that case, do 5 bits at a time.  The potential drawback is that
-# a table of 2**5 intermediate results is computed.
-
-## FIVEARY_CUTOFF = 8   disabled for now
-
-
-def _mask_digit(x):
-    if not we_are_translated():
-        assert type(x) is not long, "overflow occurred!"
-    return intmask(x & MASK)
-_mask_digit._annspecialcase_ = 'specialize:argtype(0)'
-
-def _widen_digit(x):
-    if not we_are_translated():
-        assert type(x) is int, "widen_digit() takes an int, got a %r" % type(x)
-    if SHIFT <= 15:
-        return int(x)
-    return r_longlong(x)
-
-def _store_digit(x):
-    if not we_are_translated():
-        assert type(x) is int, "store_digit() takes an int, got a %r" % type(x)
-    if SHIFT <= 15:
-        return rffi.cast(rffi.SHORT, x)
-    elif SHIFT <= 31:
-        return rffi.cast(rffi.INT, x)
-    else:
-        raise ValueError("SHIFT too large!")
-
-def _load_digit(x):
-    return rffi.cast(lltype.Signed, x)
-
-def _load_unsigned_digit(x):
-    return rffi.cast(lltype.Unsigned, x)
-
-NULLDIGIT = _store_digit(0)
-ONEDIGIT  = _store_digit(1)
-
-def _check_digits(l):
-    for x in l:
-        assert type(x) is type(NULLDIGIT)
-        assert intmask(x) & MASK == intmask(x)
-class Entry(extregistry.ExtRegistryEntry):
-    _about_ = _check_digits
-    def compute_result_annotation(self, s_list):
-        from pypy.annotation import model as annmodel
-        assert isinstance(s_list, annmodel.SomeList)
-        s_DIGIT = self.bookkeeper.valueoftype(type(NULLDIGIT))
-        assert s_DIGIT.contains(s_list.listdef.listitem.s_value)
-    def specialize_call(self, hop):
-        pass
-
-
-class rbigint(object):
-    """This is a reimplementation of longs using a list of digits."""
-
-    def __init__(self, digits=[], sign=0):
-        if len(digits) == 0:
-            digits = [NULLDIGIT]
-        _check_digits(digits)
-        make_sure_not_resized(digits)
-        self._digits = digits
-        self.sign = sign
-
-    def digit(self, x):
-        """Return the x'th digit, as an int."""
-        return _load_digit(self._digits[x])
-
-    def widedigit(self, x):
-        """Return the x'th digit, as a long long int if needed
-        to have enough room to contain two digits."""
-        return _widen_digit(_load_digit(self._digits[x]))
-
-    def udigit(self, x):
-        """Return the x'th digit, as an unsigned int."""
-        return _load_unsigned_digit(self._digits[x])
-
-    def setdigit(self, x, val):
-        val = _mask_digit(val)
-        assert val >= 0
-        self._digits[x] = _store_digit(val)
-    setdigit._annspecialcase_ = 'specialize:argtype(2)'
-
-    def numdigits(self):
-        return len(self._digits)
-
-    @staticmethod
-    @jit.elidable
-    def fromint(intval):
-        # This function is marked as pure, so you must not call it and
-        # then modify the result.
-        check_regular_int(intval)
-        if intval < 0:
-            sign = -1
-            ival = r_uint(-intval)
-        elif intval > 0:
-            sign = 1
-            ival = r_uint(intval)
-        else:
-            return rbigint()
-        # Count the number of Python digits.
-        # We used to pick 5 ("big enough for anything"), but that's a
-        # waste of time and space given that 5*15 = 75 bits are rarely
-        # needed.
-        t = ival
-        ndigits = 0
-        while t:
-            ndigits += 1
-            t >>= SHIFT
-        v = rbigint([NULLDIGIT] * ndigits, sign)
-        t = ival
-        p = 0
-        while t:
-            v.setdigit(p, t)
-            t >>= SHIFT
-            p += 1
-        return v
-
-    @staticmethod
-    @jit.elidable
-    def frombool(b):
-        # This function is marked as pure, so you must not call it and
-        # then modify the result.
-        if b:
-            return rbigint([ONEDIGIT], 1)
-        return rbigint()
-
-    @staticmethod
-    def fromlong(l):
-        "NOT_RPYTHON"
-        return rbigint(*args_from_long(l))
-
-    @staticmethod
-    def fromfloat(dval):
-        """ Create a new bigint object from a float """
-        # This function is not marked as pure because it can raise
-        if isfinite(dval):
-            return rbigint._fromfloat_finite(dval)
-        else:
-            raise OverflowError
-
-    @staticmethod
-    @jit.elidable
-    def _fromfloat_finite(dval):
-        sign = 1
-        if dval < 0.0:
-            sign = -1
-            dval = -dval
-        frac, expo = math.frexp(dval) # dval = frac*2**expo; 0.0 <= frac < 1.0
-        if expo <= 0:
-            return rbigint()
-        ndig = (expo-1) // SHIFT + 1 # Number of 'digits' in result
-        v = rbigint([NULLDIGIT] * ndig, sign)
-        frac = math.ldexp(frac, (expo-1) % SHIFT + 1)
-        for i in range(ndig-1, -1, -1):
-            # use int(int(frac)) as a workaround for a CPython bug:
-            # with frac == 2147483647.0, int(frac) == 2147483647L
-            bits = int(int(frac))
-            v.setdigit(i, bits)
-            frac -= float(bits)
-            frac = math.ldexp(frac, SHIFT)
-        return v
-
-    @staticmethod
-    @jit.elidable
-    @specialize.argtype(0)
-    def fromrarith_int(i):
-        # This function is marked as pure, so you must not call it and
-        # then modify the result.
-        return rbigint(*args_from_rarith_int(i))
-
-    @staticmethod
-    @jit.elidable
-    def fromdecimalstr(s):
-        # This function is marked as pure, so you must not call it and
-        # then modify the result.
-        return _decimalstr_to_bigint(s)
-
-    def toint(self):
-        """
-        Get an integer from a bigint object.
-        Raises OverflowError if overflow occurs.
-        """
-        x = self._touint_helper()
-        # Haven't lost any bits, but if the sign bit is set we're in
-        # trouble *unless* this is the min negative number.  So,
-        # trouble iff sign bit set && (positive || some bit set other
-        # than the sign bit).
-        sign = self.sign
-        if intmask(x) < 0 and (sign > 0 or (x << 1) != 0):
-            raise OverflowError
-        return intmask(x * sign)
-
-    def tolonglong(self):
-        return _AsLongLong(self)
-
-    def tobool(self):
-        return self.sign != 0
-
-    def touint(self):
-        if self.sign == -1:
-            raise ValueError("cannot convert negative integer to unsigned int")
-        return self._touint_helper()
-
-    def _touint_helper(self):
-        x = r_uint(0)
-        i = self.numdigits() - 1
-        while i >= 0:
-            prev = x
-            x = (x << SHIFT) + self.udigit(i)
-            if (x >> SHIFT) != prev:
-                raise OverflowError(
-                        "long int too large to convert to unsigned int")
-            i -= 1
-        return x
-
-    def toulonglong(self):
-        if self.sign == -1:
-            raise ValueError("cannot convert negative integer to unsigned int")
-        return _AsULonglong_ignore_sign(self)
-
-    def uintmask(self):
-        return _AsUInt_mask(self)
-
-    def ulonglongmask(self):
-        """Return r_ulonglong(self), truncating."""
-        return _AsULonglong_mask(self)
-
-    def tofloat(self):
-        return _AsDouble(self)
-
-    def format(self, digits, prefix='', suffix=''):
-        # 'digits' is a string whose length is the base to use,
-        # and where each character is the corresponding digit.
-        return _format(self, digits, prefix, suffix)
-
-    def repr(self):
-        return _format(self, BASE10, '', 'L')
-
-    def str(self):
-        return _format(self, BASE10)
-
-    def eq(self, other):
-        if (self.sign != other.sign or
-            self.numdigits() != other.numdigits()):
-            return False
-        i = 0
-        ld = self.numdigits()
-        while i < ld:
-            if self.digit(i) != other.digit(i):
-                return False
-            i += 1
-        return True
-
-    def ne(self, other):
-        return not self.eq(other)
-
-    def lt(self, other):
-        if self.sign > other.sign:
-            return False
-        if self.sign < other.sign:
-            return True
-        ld1 = self.numdigits()
-        ld2 = other.numdigits()
-        if ld1 > ld2:
-            if other.sign > 0:
-                return False
-            else:
-                return True
-        elif ld1 < ld2:
-            if other.sign > 0:
-                return True
-            else:
-                return False
-        i = ld1 - 1
-        while i >= 0:
-            d1 = self.digit(i)
-            d2 = other.digit(i)
-            if d1 < d2:
-                if other.sign > 0:
-                    return True
-                else:
-                    return False
-            elif d1 > d2:
-                if other.sign > 0:
-                    return False
-                else:
-                    return True
-            i -= 1
-        return False
-
-    def le(self, other):
-        return not other.lt(self)
-
-    def gt(self, other):
-        return other.lt(self)
-
-    def ge(self, other):
-        return not self.lt(other)
-
-    def hash(self):
-        return _hash(self)
-
-    def add(self, other):
-        if self.sign == 0:
-            return other
-        if other.sign == 0:
-            return self
-        if self.sign == other.sign:
-            result = _x_add(self, other)
-        else:
-            result = _x_sub(other, self)
-        result.sign *= other.sign
-        return result
-
-    def sub(self, other):
-        if other.sign == 0:
-            return self
-        if self.sign == 0:
-            return rbigint(other._digits[:], -other.sign)
-        if self.sign == other.sign:
-            result = _x_sub(self, other)
-        else:
-            result = _x_add(self, other)
-        result.sign *= self.sign
-        result._normalize()
-        return result
-
-    def mul(self, other):
-        if USE_KARATSUBA:
-            result = _k_mul(self, other)
-        else:
-            result = _x_mul(self, other)
-        result.sign = self.sign * other.sign
-        return result
-
-    def truediv(self, other):
-        div = _bigint_true_divide(self, other)
-        return div
-
-    def floordiv(self, other):
-        div, mod = self.divmod(other)
-        return div
-
-    def div(self, other):
-        return self.floordiv(other)
-
-    def mod(self, other):
-        div, mod = self.divmod(other)
-        return mod
-
-    def divmod(v, w):
-        """
-        The / and % operators are now defined in terms of divmod().
-        The expression a mod b has the value a - b*floor(a/b).
-        The _divrem function gives the remainder after division of
-        |a| by |b|, with the sign of a.  This is also expressed
-        as a - b*trunc(a/b), if trunc truncates towards zero.
-        Some examples:
-          a   b   a rem b     a mod b
-          13  10   3           3
-         -13  10  -3           7
-          13 -10   3          -7
-         -13 -10  -3          -3
-        So, to get from rem to mod, we have to add b if a and b
-        have different signs.  We then subtract one from the 'div'
-        part of the outcome to keep the invariant intact.
-        """
-        div, mod = _divrem(v, w)
-        if mod.sign * w.sign == -1:
-            mod = mod.add(w)
-            div = div.sub(rbigint([_store_digit(1)], 1))
-        return div, mod
-
-    def pow(a, b, c=None):
-        negativeOutput = False  # if x<0 return negative output
-
-        # 5-ary values.  If the exponent is large enough, table is
-        # precomputed so that table[i] == a**i % c for i in range(32).
-        # python translation: the table is computed when needed.
-
-        if b.sign < 0:  # if exponent is negative
-            if c is not None:
-                raise TypeError(
-                    "pow() 2nd argument "
-                    "cannot be negative when 3rd argument specified")
-            # XXX failed to implement
-            raise ValueError("bigint pow() too negative")
-
-        if c is not None:
-            if c.sign == 0:
-                raise ValueError("pow() 3rd argument cannot be 0")
-
-            # if modulus < 0:
-            #     negativeOutput = True
-            #     modulus = -modulus
-            if c.sign < 0:
-                negativeOutput = True
-                c = c.neg()
-
-            # if modulus == 1:
-            #     return 0
-            if c.numdigits() == 1 and c.digit(0) == 1:
-                return rbigint()
-
-            # if base < 0:
-            #     base = base % modulus
-            # Having the base positive just makes things easier.
-            if a.sign < 0:
-                a, temp = a.divmod(c)
-                a = temp
-
-        # At this point a, b, and c are guaranteed non-negative UNLESS
-        # c is NULL, in which case a may be negative. */
-
-        z = rbigint([_store_digit(1)], 1)
-
-        # python adaptation: moved macros REDUCE(X) and MULT(X, Y, result)
-        # into helper function result = _help_mult(x, y, c)
-        if 1:   ## b.numdigits() <= FIVEARY_CUTOFF:
-            # Left-to-right binary exponentiation (HAC Algorithm 14.79)
-            # http://www.cacr.math.uwaterloo.ca/hac/about/chap14.pdf
-            i = b.numdigits() - 1
-            while i >= 0:
-                bi = b.digit(i)
-                j = 1 << (SHIFT-1)
-                while j != 0:
-                    z = _help_mult(z, z, c)
-                    if bi & j:
-                        z = _help_mult(z, a, c)
-                    j >>= 1
-                i -= 1
-##        else:
-##            This code is disabled for now, because it assumes that
-##            SHIFT is a multiple of 5.  It could be fixed but it looks
-##            like it's more troubles than benefits...
-##
-##            # Left-to-right 5-ary exponentiation (HAC Algorithm 14.82)
-##            # This is only useful in the case where c != None.
-##            # z still holds 1L
-##            table = [z] * 32
-##            table[0] = z
-##            for i in range(1, 32):
-##                table[i] = _help_mult(table[i-1], a, c)
-##            i = b.numdigits() - 1
-##            while i >= 0:
-##                bi = b.digit(i)
-##                j = SHIFT - 5
-##                while j >= 0:
-##                    index = (bi >> j) & 0x1f
-##                    for k in range(5):
-##                        z = _help_mult(z, z, c)
-##                    if index:
-##                        z = _help_mult(z, table[index], c)
-##                    j -= 5
-##                i -= 1
-
-        if negativeOutput and z.sign != 0:
-            z = z.sub(c)
-        return z
-
-    def neg(self):
-        return rbigint(self._digits, -self.sign)
-
-    def abs(self):
-        return rbigint(self._digits, abs(self.sign))
-
-    def invert(self): #Implement ~x as -(x + 1)
-        return self.add(rbigint([_store_digit(1)], 1)).neg()
-
-    def lshift(self, int_other):
-        if int_other < 0:
-            raise ValueError("negative shift count")
-        elif int_other == 0:
-            return self
-
-        # wordshift, remshift = divmod(int_other, SHIFT)
-        wordshift = int_other // SHIFT
-        remshift  = int_other - wordshift * SHIFT
-
-        oldsize = self.numdigits()
-        newsize = oldsize + wordshift
-        if remshift:
-            newsize += 1
-        z = rbigint([NULLDIGIT] * newsize, self.sign)
-        accum = _widen_digit(0)
-        i = wordshift
-        j = 0
-        while j < oldsize:
-            accum |= self.widedigit(j) << remshift
-            z.setdigit(i, accum)
-            accum >>= SHIFT
-            i += 1
-            j += 1
-        if remshift:
-            z.setdigit(newsize - 1, accum)
-        else:
-            assert not accum
-        z._normalize()
-        return z
-
-    def rshift(self, int_other, dont_invert=False):
-        if int_other < 0:
-            raise ValueError("negative shift count")
-        elif int_other == 0:
-            return self
-        if self.sign == -1 and not dont_invert:
-            a1 = self.invert()
-            a2 = a1.rshift(int_other)
-            return a2.invert()
-
-        wordshift = int_other // SHIFT
-        newsize = self.numdigits() - wordshift
-        if newsize <= 0:
-            return rbigint()
-
-        loshift = int_other % SHIFT
-        hishift = SHIFT - loshift
-        lomask = intmask((r_uint(1) << hishift) - 1)
-        himask = MASK ^ lomask
-        z = rbigint([NULLDIGIT] * newsize, self.sign)
-        i = 0
-        j = wordshift
-        while i < newsize:
-            newdigit = (self.digit(j) >> loshift) & lomask
-            if i+1 < newsize:
-                newdigit |= intmask(self.digit(j+1) << hishift) & himask
-            z.setdigit(i, newdigit)
-            i += 1
-            j += 1
-        z._normalize()
-        return z
-
-    def and_(self, other):
-        return _bitwise(self, '&', other)
-
-    def xor(self, other):
-        return _bitwise(self, '^', other)
-
-    def or_(self, other):
-        return _bitwise(self, '|', other)
-
-    def oct(self):
-        if self.sign == 0:
-            return '0L'
-        else:
-            return _format(self, BASE8, '0', 'L')
-
-    def hex(self):
-        return _format(self, BASE16, '0x', 'L')
-
-    def log(self, base):
-        # base is supposed to be positive or 0.0, which means we use e
-        if base == 10.0:
-            return _loghelper(math.log10, self)
-        ret = _loghelper(math.log, self)
-        if base != 0.0:
-            ret /= math.log(base)
-        return ret
-
-    def tolong(self):
-        "NOT_RPYTHON"
-        l = 0L
-        digits = list(self._digits)
-        digits.reverse()
-        for d in digits:
-            l = l << SHIFT
-            l += intmask(d)
-        return l * self.sign
-
-    def _normalize(self):
-        if self.numdigits() == 0:
-            self.sign = 0
-            self._digits = [NULLDIGIT]
-            return
-        i = self.numdigits()
-        while i > 1 and self.digit(i - 1) == 0:
-            i -= 1
-        assert i >= 1
-        if i != self.numdigits():
-            self._digits = self._digits[:i]
-        if self.numdigits() == 1 and self.digit(0) == 0:
-            self.sign = 0
-
-    def bit_length(self):
-        i = self.numdigits()
-        if i == 1 and self.digit(0) == 0:
-            return 0
-        msd = self.digit(i - 1)
-        msd_bits = 0
-        while msd >= 32:
-            msd_bits += 6
-            msd >>= 6
-        msd_bits += [
-            0, 1, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4,
-            5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5
-            ][msd]
-        # yes, this can overflow: a huge number which fits 3 gigabytes of
-        # memory has around 24 gigabits!
-        bits = ovfcheck((i-1) * SHIFT) + msd_bits
-        return bits
-
-    def __repr__(self):
-        return "<rbigint digits=%s, sign=%s, %s>" % (self._digits,
-                                                     self.sign, self.str())
-
-#_________________________________________________________________
-
-# Helper Functions
-
-
-def _help_mult(x, y, c):
-    """
-    Multiply two values, then reduce the result:
-    result = X*Y % c.  If c is None, skip the mod.
-    """
-    res = x.mul(y)
-    # Perform a modular reduction, X = X % c, but leave X alone if c
-    # is NULL.
-    if c is not None:
-        res, temp = res.divmod(c)
-        res = temp
-    return res
-
-
-
-def digits_from_nonneg_long(l):
-    digits = []
-    while True:
-        digits.append(_store_digit(intmask(l & MASK)))
-        l = l >> SHIFT
-        if not l:
-            return digits[:] # to make it non-resizable
-digits_from_nonneg_long._annspecialcase_ = "specialize:argtype(0)"
-
-def digits_for_most_neg_long(l):
-    # This helper only works if 'l' is the most negative integer of its
-    # type, which in base 2 looks like: 1000000..0000
-    digits = []
-    while _mask_digit(l) == 0:
-        digits.append(NULLDIGIT)
-        l = l >> SHIFT
-    # now 'l' looks like: ...111100000
-    # turn it into:       ...000100000
-    # to drop the extra unwanted 1's introduced by the signed right shift
-    l = -intmask(l)
-    assert l & MASK == l
-    digits.append(_store_digit(l))
-    return digits[:] # to make it non-resizable
-digits_for_most_neg_long._annspecialcase_ = "specialize:argtype(0)"
-
-def args_from_rarith_int1(x):
-    if x > 0:
-        return digits_from_nonneg_long(x), 1
-    elif x == 0:
-        return [NULLDIGIT], 0
-    elif x != most_neg_value_of_same_type(x):
-        # normal case
-        return digits_from_nonneg_long(-x), -1
-    else:
-        # the most negative integer! hacks needed...
-        return digits_for_most_neg_long(x), -1
-args_from_rarith_int1._annspecialcase_ = "specialize:argtype(0)"
-
-def args_from_rarith_int(x):
-    return args_from_rarith_int1(widen(x))
-args_from_rarith_int._annspecialcase_ = "specialize:argtype(0)"
-# ^^^ specialized by the precise type of 'x', which is typically a r_xxx
-#     instance from rlib.rarithmetic
-
-def args_from_long(x):
-    "NOT_RPYTHON"
-    if x >= 0:
-        if x == 0:
-            return [NULLDIGIT], 0
-        else:
-            return digits_from_nonneg_long(x), 1
-    else:
-        return digits_from_nonneg_long(-x), -1
-
-def _x_add(a, b):
-    """ Add the absolute values of two bigint integers. """
-    size_a = a.numdigits()
-    size_b = b.numdigits()
-
-    # Ensure a is the larger of the two:
-    if size_a < size_b:
-        a, b = b, a
-        size_a, size_b = size_b, size_a
-    z = rbigint([NULLDIGIT] * (a.numdigits() + 1), 1)
-    i = 0
-    carry = r_uint(0)
-    while i < size_b:
-        carry += a.udigit(i) + b.udigit(i)
-        z.setdigit(i, carry)
-        carry >>= SHIFT
-        i += 1
-    while i < size_a:
-        carry += a.udigit(i)
-        z.setdigit(i, carry)
-        carry >>= SHIFT
-        i += 1
-    z.setdigit(i, carry)
-    z._normalize()
-    return z
-
-def _x_sub(a, b):
-    """ Subtract the absolute values of two integers. """
-    size_a = a.numdigits()
-    size_b = b.numdigits()
-    sign = 1
-
-    # Ensure a is the larger of the two:
-    if size_a < size_b:
-        sign = -1
-        a, b = b, a
-        size_a, size_b = size_b, size_a
-    elif size_a == size_b:
-        # Find highest digit where a and b differ:
-        i = size_a - 1
-        while i >= 0 and a.digit(i) == b.digit(i):
-            i -= 1
-        if i < 0:
-            return rbigint()
-        if a.digit(i) < b.digit(i):
-            sign = -1
-            a, b = b, a
-        size_a = size_b = i+1
-    z = rbigint([NULLDIGIT] * size_a, sign)
-    borrow = r_uint(0)
-    i = 0
-    while i < size_b:
-        # The following assumes unsigned arithmetic
-        # works modulo 2**N for some N>SHIFT.
-        borrow = a.udigit(i) - b.udigit(i) - borrow
-        z.setdigit(i, borrow)
-        borrow >>= SHIFT
-        borrow &= 1 # Keep only one sign bit
-        i += 1
-    while i < size_a:
-        borrow = a.udigit(i) - borrow
-        z.setdigit(i, borrow)
-        borrow >>= SHIFT
-        borrow &= 1 # Keep only one sign bit
-        i += 1
-    assert borrow == 0
-    z._normalize()
-    return z
-
-
-def _x_mul(a, b):
-    """
-    Grade school multiplication, ignoring the signs.
-    Returns the absolute value of the product, or None if error.
-    """
-
-    size_a = a.numdigits()
-    size_b = b.numdigits()
-    z = rbigint([NULLDIGIT] * (size_a + size_b), 1)
-    if a is b:
-        # Efficient squaring per HAC, Algorithm 14.16:
-        # http://www.cacr.math.uwaterloo.ca/hac/about/chap14.pdf
-        # Gives slightly less than a 2x speedup when a == b,
-        # via exploiting that each entry in the multiplication
-        # pyramid appears twice (except for the size_a squares).
-        i = 0
-        while i < size_a:
-            f = a.widedigit(i)
-            pz = i << 1
-            pa = i + 1
-            paend = size_a
-
-            carry = z.widedigit(pz) + f * f
-            z.setdigit(pz, carry)
-            pz += 1
-            carry >>= SHIFT
-            assert carry <= MASK
-
-            # Now f is added in twice in each column of the
-            # pyramid it appears.  Same as adding f<<1 once.
-            f <<= 1
-            while pa < paend:
-                carry += z.widedigit(pz) + a.widedigit(pa) * f
-                pa += 1
-                z.setdigit(pz, carry)
-                pz += 1
-                carry >>= SHIFT
-                assert carry <= (_widen_digit(MASK) << 1)
-            if carry:
-                carry += z.widedigit(pz)
-                z.setdigit(pz, carry)
-                pz += 1
-                carry >>= SHIFT
-            if carry:
-                z.setdigit(pz, z.widedigit(pz) + carry)
-            assert (carry >> SHIFT) == 0
-            i += 1
-    else:
-        # a is not the same as b -- gradeschool long mult
-        i = 0
-        while i < size_a:
-            carry = 0
-            f = a.widedigit(i)
-            pz = i
-            pb = 0
-            pbend = size_b
-            while pb < pbend:
-                carry += z.widedigit(pz) + b.widedigit(pb) * f
-                pb += 1
-                z.setdigit(pz, carry)
-                pz += 1
-                carry >>= SHIFT
-                assert carry <= MASK
-            if carry:
-                z.setdigit(pz, z.widedigit(pz) + carry)
-            assert (carry >> SHIFT) == 0
-            i += 1
-    z._normalize()
-    return z
-
-
-def _kmul_split(n, size):
-    """
-    A helper for Karatsuba multiplication (k_mul).
-    Takes a bigint "n" and an integer "size" representing the place to
-    split, and sets low and high such that abs(n) == (high << size) + low,
-    viewing the shift as being by digits.  The sign bit is ignored, and
-    the return values are >= 0.
-    """
-    size_n = n.numdigits()
-    size_lo = min(size_n, size)
-
-    lo = rbigint(n._digits[:size_lo], 1)
-    hi = rbigint(n._digits[size_lo:], 1)
-    lo._normalize()
-    hi._normalize()
-    return hi, lo
-
-def _k_mul(a, b):
-    """
-    Karatsuba multiplication.  Ignores the input signs, and returns the
-    absolute value of the product (or raises if error).
-    See Knuth Vol. 2 Chapter 4.3.3 (Pp. 294-295).
-    """
-    asize = a.numdigits()
-    bsize = b.numdigits()
-    # (ah*X+al)(bh*X+bl) = ah*bh*X*X + (ah*bl + al*bh)*X + al*bl
-    # Let k = (ah+al)*(bh+bl) = ah*bl + al*bh  + ah*bh + al*bl
-    # Then the original product is
-    #     ah*bh*X*X + (k - ah*bh - al*bl)*X + al*bl
-    # By picking X to be a power of 2, "*X" is just shifting, and it's
-    # been reduced to 3 multiplies on numbers half the size.
-
-    # We want to split based on the larger number; fiddle so that b
-    # is largest.
-    if asize > bsize:
-        a, b, asize, bsize = b, a, bsize, asize
-
-    # Use gradeschool math when either number is too small.
-    if a is b:
-        i = KARATSUBA_SQUARE_CUTOFF
-    else:
-        i = KARATSUBA_CUTOFF
-    if asize <= i:
-        if a.sign == 0:
-            return rbigint()     # zero
-        else:
-            return _x_mul(a, b)
-
-    # If a is small compared to b, splitting on b gives a degenerate
-    # case with ah==0, and Karatsuba may be (even much) less efficient
-    # than "grade school" then.  However, we can still win, by viewing
-    # b as a string of "big digits", each of width a->ob_size.  That
-    # leads to a sequence of balanced calls to k_mul.
-    if 2 * asize <= bsize:
-        return _k_lopsided_mul(a, b)
-
-    # Split a & b into hi & lo pieces.
-    shift = bsize >> 1
-    ah, al = _kmul_split(a, shift)
-    assert ah.sign == 1    # the split isn't degenerate
-
-    if a == b:
-        bh = ah
-        bl = al
-    else:
-        bh, bl = _kmul_split(b, shift)
-
-    # The plan:
-    # 1. Allocate result space (asize + bsize digits:  that's always
-    #    enough).
-    # 2. Compute ah*bh, and copy into result at 2*shift.
-    # 3. Compute al*bl, and copy into result at 0.  Note that this
-    #    can't overlap with #2.
-    # 4. Subtract al*bl from the result, starting at shift.  This may
-    #    underflow (borrow out of the high digit), but we don't care:
-    #    we're effectively doing unsigned arithmetic mod
-    #    BASE**(sizea + sizeb), and so long as the *final* result fits,
-    #    borrows and carries out of the high digit can be ignored.
-    # 5. Subtract ah*bh from the result, starting at shift.
-    # 6. Compute (ah+al)*(bh+bl), and add it into the result starting
-    #    at shift.
-
-    # 1. Allocate result space.
-    ret = rbigint([NULLDIGIT] * (asize + bsize), 1)
-
-    # 2. t1 <- ah*bh, and copy into high digits of result.
-    t1 = _k_mul(ah, bh)
-    assert t1.sign >= 0
-    assert 2*shift + t1.numdigits() <= ret.numdigits()
-    ret._digits[2*shift : 2*shift + t1.numdigits()] = t1._digits
-
-    # Zero-out the digits higher than the ah*bh copy. */
-    ## ignored, assuming that we initialize to zero
-    ##i = ret->ob_size - 2*shift - t1->ob_size;
-    ##if (i)
-    ##    memset(ret->ob_digit + 2*shift + t1->ob_size, 0,
-    ##           i * sizeof(digit));
-
-    # 3. t2 <- al*bl, and copy into the low digits.
-    t2 = _k_mul(al, bl)
-    assert t2.sign >= 0
-    assert t2.numdigits() <= 2*shift # no overlap with high digits
-    ret._digits[:t2.numdigits()] = t2._digits
-
-    # Zero out remaining digits.
-    ## ignored, assuming that we initialize to zero
-    ##i = 2*shift - t2->ob_size;  /* number of uninitialized digits */
-    ##if (i)
-    ##    memset(ret->ob_digit + t2->ob_size, 0, i * sizeof(digit));
-
-    # 4 & 5. Subtract ah*bh (t1) and al*bl (t2).  We do al*bl first
-    # because it's fresher in cache.
-    i = ret.numdigits() - shift  # # digits after shift
-    _v_isub(ret, shift, i, t2, t2.numdigits())
-    _v_isub(ret, shift, i, t1, t1.numdigits())
-    del t1, t2
-
-    # 6. t3 <- (ah+al)(bh+bl), and add into result.
-    t1 = _x_add(ah, al)
-    del ah, al
-
-    if a == b:
-        t2 = t1
-    else:
-        t2 = _x_add(bh, bl)
-    del bh, bl
-
-    t3 = _k_mul(t1, t2)
-    del t1, t2
-    assert t3.sign >=0
-
-    # Add t3.  It's not obvious why we can't run out of room here.
-    # See the (*) comment after this function.
-    _v_iadd(ret, shift, i, t3, t3.numdigits())
-    del t3
-
-    ret._normalize()
-    return ret
-
-""" (*) Why adding t3 can't "run out of room" above.
-
-Let f(x) mean the floor of x and c(x) mean the ceiling of x.  Some facts
-to start with:
-
-1. For any integer i, i = c(i/2) + f(i/2).  In particular,
-   bsize = c(bsize/2) + f(bsize/2).
-2. shift = f(bsize/2)
-3. asize <= bsize
-4. Since we call k_lopsided_mul if asize*2 <= bsize, asize*2 > bsize in this
-   routine, so asize > bsize/2 >= f(bsize/2) in this routine.
-
-We allocated asize + bsize result digits, and add t3 into them at an offset
-of shift.  This leaves asize+bsize-shift allocated digit positions for t3
-to fit into, = (by #1 and #2) asize + f(bsize/2) + c(bsize/2) - f(bsize/2) =
-asize + c(bsize/2) available digit positions.
-
-bh has c(bsize/2) digits, and bl at most f(size/2) digits.  So bh+hl has
-at most c(bsize/2) digits + 1 bit.
-
-If asize == bsize, ah has c(bsize/2) digits, else ah has at most f(bsize/2)
-digits, and al has at most f(bsize/2) digits in any case.  So ah+al has at
-most (asize == bsize ? c(bsize/2) : f(bsize/2)) digits + 1 bit.
-
-The product (ah+al)*(bh+bl) therefore has at most
-
-    c(bsize/2) + (asize == bsize ? c(bsize/2) : f(bsize/2)) digits + 2 bits
-
-and we have asize + c(bsize/2) available digit positions.  We need to show
-this is always enough.  An instance of c(bsize/2) cancels out in both, so
-the question reduces to whether asize digits is enough to hold
-(asize == bsize ? c(bsize/2) : f(bsize/2)) digits + 2 bits.  If asize < bsize,
-then we're asking whether asize digits >= f(bsize/2) digits + 2 bits.  By #4,
-asize is at least f(bsize/2)+1 digits, so this in turn reduces to whether 1
-digit is enough to hold 2 bits.  This is so since SHIFT=15 >= 2.  If
-asize == bsize, then we're asking whether bsize digits is enough to hold
-c(bsize/2) digits + 2 bits, or equivalently (by #1) whether f(bsize/2) digits
-is enough to hold 2 bits.  This is so if bsize >= 2, which holds because
-bsize >= KARATSUBA_CUTOFF >= 2.
-
-Note that since there's always enough room for (ah+al)*(bh+bl), and that's
-clearly >= each of ah*bh and al*bl, there's always enough room to subtract
-ah*bh and al*bl too.
-"""
-
-def _k_lopsided_mul(a, b):
-    """
-    b has at least twice the digits of a, and a is big enough that Karatsuba
-    would pay off *if* the inputs had balanced sizes.  View b as a sequence
-    of slices, each with a->ob_size digits, and multiply the slices by a,
-    one at a time.  This gives k_mul balanced inputs to work with, and is
-    also cache-friendly (we compute one double-width slice of the result
-    at a time, then move on, never bactracking except for the helpful
-    single-width slice overlap between successive partial sums).
-    """
-    asize = a.numdigits()
-    bsize = b.numdigits()
-    # nbdone is # of b digits already multiplied
-
-    assert asize > KARATSUBA_CUTOFF
-    assert 2 * asize <= bsize
-
-    # Allocate result space, and zero it out.
-    ret = rbigint([NULLDIGIT] * (asize + bsize), 1)
-
-    # Successive slices of b are copied into bslice.
-    #bslice = rbigint([0] * asize, 1)
-    # XXX we cannot pre-allocate, see comments below!
-    bslice = rbigint([NULLDIGIT], 1)
-
-    nbdone = 0;
-    while bsize > 0:
-        nbtouse = min(bsize, asize)
-
-        # Multiply the next slice of b by a.
-
-        #bslice.digits[:nbtouse] = b.digits[nbdone : nbdone + nbtouse]
-        # XXX: this would be more efficient if we adopted CPython's
-        # way to store the size, instead of resizing the list!
-        # XXX change the implementation, encoding length via the sign.
-        bslice._digits = b._digits[nbdone : nbdone + nbtouse]
-        product = _k_mul(a, bslice)
-
-        # Add into result.
-        _v_iadd(ret, nbdone, ret.numdigits() - nbdone,
-                 product, product.numdigits())
-        del product
-
-        bsize -= nbtouse
-        nbdone += nbtouse
-
-    ret._normalize()
-    return ret
-
-
-def _inplace_divrem1(pout, pin, n, size=0):
-    """
-    Divide bigint pin by non-zero digit n, storing quotient
-    in pout, and returning the remainder. It's OK for pin == pout on entry.
-    """
-    rem = _widen_digit(0)
-    assert n > 0 and n <= MASK
-    if not size:
-        size = pin.numdigits()
-    size -= 1
-    while size >= 0:
-        rem = (rem << SHIFT) + pin.widedigit(size)
-        hi = rem // n
-        pout.setdigit(size, hi)
-        rem -= hi * n
-        size -= 1
-    return _mask_digit(rem)
-
-def _divrem1(a, n):
-    """
-    Divide a bigint integer by a digit, returning both the quotient
-    and the remainder as a tuple.
-    The sign of a is ignored; n should not be zero.
-    """
-    assert n > 0 and n <= MASK
-    size = a.numdigits()
-    z = rbigint([NULLDIGIT] * size, 1)
-    rem = _inplace_divrem1(z, a, n)
-    z._normalize()
-    return z, rem
-
-def _v_iadd(x, xofs, m, y, n):
-    """
-    x and y are rbigints, m >= n required.  x.digits[0:n] is modified in place,
-    by adding y.digits[0:m] to it.  Carries are propagated as far as
-    x[m-1], and the remaining carry (0 or 1) is returned.
-    Python adaptation: x is addressed relative to xofs!
-    """
-    carry = r_uint(0)
-
-    assert m >= n
-    i = xofs
-    iend = xofs + n
-    while i < iend:
-        carry += x.udigit(i) + y.udigit(i-xofs)
-        x.setdigit(i, carry)
-        carry >>= SHIFT
-        assert (carry & 1) == carry
-        i += 1
-    iend = xofs + m
-    while carry and i < iend:
-        carry += x.udigit(i)
-        x.setdigit(i, carry)
-        carry >>= SHIFT
-        assert (carry & 1) == carry
-        i += 1
-    return carry
-
-def _v_isub(x, xofs, m, y, n):
-    """
-    x and y are rbigints, m >= n required.  x.digits[0:n] is modified in place,
-    by substracting y.digits[0:m] to it. Borrows are propagated as
-    far as x[m-1], and the remaining borrow (0 or 1) is returned.
-    Python adaptation: x is addressed relative to xofs!
-    """
-    borrow = r_uint(0)
-
-    assert m >= n
-    i = xofs
-    iend = xofs + n
-    while i < iend:
-        borrow = x.udigit(i) - y.udigit(i-xofs) - borrow
-        x.setdigit(i, borrow)
-        borrow >>= SHIFT
-        borrow &= 1    # keep only 1 sign bit
-        i += 1
-    iend = xofs + m
-    while borrow and i < iend:
-        borrow = x.udigit(i) - borrow
-        x.setdigit(i, borrow)
-        borrow >>= SHIFT
-        borrow &= 1
-        i += 1
-    return borrow
-
-
-def _muladd1(a, n, extra=0):
-    """Multiply by a single digit and add a single digit, ignoring the sign.
-    """
-    size_a = a.numdigits()
-    z = rbigint([NULLDIGIT] * (size_a+1), 1)
-    assert extra & MASK == extra
-    carry = _widen_digit(extra)
-    i = 0
-    while i < size_a:
-        carry += a.widedigit(i) * n
-        z.setdigit(i, carry)
-        carry >>= SHIFT
-        i += 1
-    z.setdigit(i, carry)
-    z._normalize()
-    return z
-
-
-def _x_divrem(v1, w1):
-    """ Unsigned bigint division with remainder -- the algorithm """
-    size_w = w1.numdigits()
-    d = (r_uint(MASK)+1) // (w1.udigit(size_w-1) + 1)
-    assert d <= MASK    # because the first digit of w1 is not zero
-    d = intmask(d)
-    v = _muladd1(v1, d)
-    w = _muladd1(w1, d)
-    size_v = v.numdigits()
-    size_w = w.numdigits()
-    assert size_v >= size_w and size_w > 1 # Assert checks by div()
-
-    size_a = size_v - size_w + 1
-    a = rbigint([NULLDIGIT] * size_a, 1)
-
-    j = size_v
-    k = size_a - 1
-    while k >= 0:
-        if j >= size_v:
-            vj = 0
-        else:
-            vj = v.widedigit(j)
-        carry = 0
-
-        if vj == w.widedigit(size_w-1):
-            q = MASK
-        else:
-            q = ((vj << SHIFT) + v.widedigit(j-1)) // w.widedigit(size_w-1)
-
-        while (w.widedigit(size_w-2) * q >
-                ((
-                    (vj << SHIFT)
-                    + v.widedigit(j-1)
-                    - q * w.widedigit(size_w-1)
-                                ) << SHIFT)
-                + v.widedigit(j-2)):
-            q -= 1
-        i = 0
-        while i < size_w and i+k < size_v:
-            z = w.widedigit(i) * q
-            zz = z >> SHIFT
-            carry += v.widedigit(i+k) - z + (zz << SHIFT)
-            v.setdigit(i+k, carry)
-            carry >>= SHIFT
-            carry -= zz
-            i += 1
-
-        if i+k < size_v:
-            carry += v.widedigit(i+k)
-            v.setdigit(i+k, 0)
-
-        if carry == 0:
-            a.setdigit(k, q)
-            assert not q >> SHIFT
-        else:
-            assert carry == -1
-            q -= 1
-            a.setdigit(k, q)
-            assert not q >> SHIFT
-
-            carry = 0
-            i = 0
-            while i < size_w and i+k < size_v:
-                carry += v.udigit(i+k) + w.udigit(i)
-                v.setdigit(i+k, carry)
-                carry >>= SHIFT
-                i += 1
-        j -= 1
-        k -= 1
-
-    a._normalize()
-    rem, _ = _divrem1(v, d)
-    return a, rem
-
-
-def _divrem(a, b):
-    """ Long division with remainder, top-level routine """
-    size_a = a.numdigits()
-    size_b = b.numdigits()
-
-    if b.sign == 0:
-        raise ZeroDivisionError("long division or modulo by zero")
-
-    if (size_a < size_b or
-        (size_a == size_b and
-         a.digit(size_a-1) < b.digit(size_b-1))):
-        # |a| < |b|
-        z = rbigint()   # result is 0
-        rem = a
-        return z, rem
-    if size_b == 1:
-        z, urem = _divrem1(a, b.digit(0))
-        rem = rbigint([_store_digit(urem)], int(urem != 0))
-    else:
-        z, rem = _x_divrem(a, b)
-    # Set the signs.
-    # The quotient z has the sign of a*b;
-    # the remainder r has the sign of a,
-    # so a = b*z + r.
-    if a.sign != b.sign:
-        z.sign = - z.sign
-    if a.sign < 0 and rem.sign != 0:
-        rem.sign = - rem.sign
-    return z, rem
-
-# ______________ conversions to double _______________
-
-def _AsScaledDouble(v):
-    """
-    NBITS_WANTED should be > the number of bits in a double's precision,
-    but small enough so that 2**NBITS_WANTED is within the normal double
-    range.  nbitsneeded is set to 1 less than that because the most-significant
-    Python digit contains at least 1 significant bit, but we don't want to
-    bother counting them (catering to the worst case cheaply).
-
-    57 is one more than VAX-D double precision; I (Tim) don't know of a double
-    format with more precision than that; it's 1 larger so that we add in at
-    least one round bit to stand in for the ignored least-significant bits.
-    """
-    NBITS_WANTED = 57
-    if v.sign == 0:
-        return 0.0, 0
-    i = v.numdigits() - 1
-    sign = v.sign
-    x = float(v.digit(i))
-    nbitsneeded = NBITS_WANTED - 1
-    # Invariant:  i Python digits remain unaccounted for.
-    while i > 0 and nbitsneeded > 0:
-        i -= 1
-        x = x * FLOAT_MULTIPLIER + float(v.digit(i))
-        nbitsneeded -= SHIFT
-    # There are i digits we didn't shift in.  Pretending they're all
-    # zeroes, the true value is x * 2**(i*SHIFT).
-    exponent = i
-    assert x > 0.0
-    return x * sign, exponent
-
-##def ldexp(x, exp):
-##    assert type(x) is float
-##    lb1 = LONG_BIT - 1
-##    multiplier = float(1 << lb1)
-##    while exp >= lb1:
-##        x *= multiplier
-##        exp -= lb1
-##    if exp:
-##        x *= float(1 << exp)
-##    return x
-
-# note that math.ldexp checks for overflows,
-# while the C ldexp is not guaranteed to do.
-# XXX make sure that we don't ignore this!
-# YYY no, we decided to do ignore this!
-
- at jit.dont_look_inside
-def _AsDouble(n):
-    """ Get a C double from a bigint object. """
-    # This is a "correctly-rounded" version from Python 2.7.
-    #
-    from pypy.rlib import rfloat
-    DBL_MANT_DIG = rfloat.DBL_MANT_DIG  # 53 for IEEE 754 binary64
-    DBL_MAX_EXP = rfloat.DBL_MAX_EXP    # 1024 for IEEE 754 binary64
-    assert DBL_MANT_DIG < r_ulonglong.BITS
-
-    # Reduce to case n positive.
-    sign = n.sign
-    if sign == 0:
-        return 0.0
-    elif sign < 0:
-        n = n.neg()
-
-    # Find exponent: 2**(exp - 1) <= n < 2**exp
-    exp = n.bit_length()
-
-    # Get top DBL_MANT_DIG + 2 significant bits of n, with a 'sticky'
-    # last bit: that is, the least significant bit of the result is 1
-    # iff any of the shifted-out bits is set.
-    shift = DBL_MANT_DIG + 2 - exp
-    if shift >= 0:
-        q = _AsULonglong_mask(n) << shift
-        if not we_are_translated():
-            assert q == n.tolong() << shift   # no masking actually done
-    else:
-        shift = -shift
-        n2 = n.rshift(shift)
-        q = _AsULonglong_mask(n2)
-        if not we_are_translated():
-            assert q == n2.tolong()           # no masking actually done
-        if not n.eq(n2.lshift(shift)):
-            q |= 1
-
-    # Now remove the excess 2 bits, rounding to nearest integer (with
-    # ties rounded to even).
-    q = (q >> 2) + (bool(q & 2) and bool(q & 5))
-
-    if exp > DBL_MAX_EXP or (exp == DBL_MAX_EXP and
-                             q == r_ulonglong(1) << DBL_MANT_DIG):
-        raise OverflowError("integer too large to convert to float")
-
-    ad = math.ldexp(float(q), exp - DBL_MANT_DIG)
-    if sign < 0:
-        ad = -ad
-    return ad
-
-def _loghelper(func, arg):
-    """
-    A decent logarithm is easy to compute even for huge bigints, but libm can't
-    do that by itself -- loghelper can.  func is log or log10.
-    Note that overflow isn't possible:  a bigint can contain
-    no more than INT_MAX * SHIFT bits, so has value certainly less than
-    2**(2**64 * 2**16) == 2**2**80, and log2 of that is 2**80, which is
-    small enough to fit in an IEEE single.  log and log10 are even smaller.
-    """
-    x, e = _AsScaledDouble(arg)
-    if x <= 0.0:
-        raise ValueError
-    # Value is ~= x * 2**(e*SHIFT), so the log ~=
-    # log(x) + log(2) * e * SHIFT.
-    # CAUTION:  e*SHIFT may overflow using int arithmetic,
-    # so force use of double. */
-    return func(x) + (e * float(SHIFT) * func(2.0))
-_loghelper._annspecialcase_ = 'specialize:arg(0)'
-
-# ____________________________________________________________
-
-BASE_AS_FLOAT = float(1 << SHIFT)     # note that it may not fit an int
-
-BitLengthTable = ''.join(map(chr, [
-    0, 1, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4,
-    5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5]))
-
-def bits_in_digit(d):
-    # returns the unique integer k such that 2**(k-1) <= d <
-    # 2**k if d is nonzero, else 0.
-    d_bits = 0
-    while d >= 32:
-        d_bits += 6
-        d >>= 6
-    d_bits += ord(BitLengthTable[d])
-    return d_bits
-
-def _truediv_result(result, negate):
-    if negate:
-        result = -result
-    return result
-
-def _truediv_overflow():
-    raise OverflowError("integer division result too large for a float")
-
-def _bigint_true_divide(a, b):
-    # A longish method to obtain the floating-point result with as much
-    # precision as theoretically possible.  The code is almost directly
-    # copied from CPython.  See there (Objects/longobject.c,
-    # long_true_divide) for detailled comments.  Method in a nutshell:
-    #
-    #    0. reduce to case a, b > 0; filter out obvious underflow/overflow
-    #    1. choose a suitable integer 'shift'
-    #    2. use integer arithmetic to compute x = floor(2**-shift*a/b)
-    #    3. adjust x for correct rounding
-    #    4. convert x to a double dx with the same value
-    #    5. return ldexp(dx, shift).
-
-    from pypy.rlib import rfloat
-    DBL_MANT_DIG = rfloat.DBL_MANT_DIG  # 53 for IEEE 754 binary64
-    DBL_MAX_EXP = rfloat.DBL_MAX_EXP    # 1024 for IEEE 754 binary64
-    DBL_MIN_EXP = rfloat.DBL_MIN_EXP
-    MANT_DIG_DIGITS = DBL_MANT_DIG // SHIFT
-    MANT_DIG_BITS = DBL_MANT_DIG % SHIFT
-
-    # Reduce to case where a and b are both positive.
-    negate = (a.sign < 0) ^ (b.sign < 0)
-    if not b.tobool():
-        raise ZeroDivisionError("long division or modulo by zero")
-    if not a.tobool():
-        return _truediv_result(0.0, negate)
-
-    a_size = a.numdigits()
-    b_size = b.numdigits()
-
-    # Fast path for a and b small (exactly representable in a double).
-    # Relies on floating-point division being correctly rounded; results
-    # may be subject to double rounding on x86 machines that operate with
-    # the x87 FPU set to 64-bit precision.
-    a_is_small = (a_size <= MANT_DIG_DIGITS or
-                  (a_size == MANT_DIG_DIGITS+1 and
-                   a.digit(MANT_DIG_DIGITS) >> MANT_DIG_BITS == 0))
-    b_is_small = (b_size <= MANT_DIG_DIGITS or
-                  (b_size == MANT_DIG_DIGITS+1 and
-                   b.digit(MANT_DIG_DIGITS) >> MANT_DIG_BITS == 0))
-    if a_is_small and b_is_small:
-        a_size -= 1
-        da = float(a.digit(a_size))
-        while True:
-            a_size -= 1
-            if a_size < 0: break
-            da = da * BASE_AS_FLOAT + a.digit(a_size)
-
-        b_size -= 1
-        db = float(b.digit(b_size))
-        while True:
-            b_size -= 1
-            if b_size < 0: break
-            db = db * BASE_AS_FLOAT + b.digit(b_size)
-
-        return _truediv_result(da / db, negate)
-
-    # Catch obvious cases of underflow and overflow
-    diff = a_size - b_size
-    if diff > sys.maxint/SHIFT - 1:
-        return _truediv_overflow()           # Extreme overflow
-    elif diff < 1 - sys.maxint/SHIFT:
-        return _truediv_result(0.0, negate)  # Extreme underflow
-    # Next line is now safe from overflowing integers
-    diff = (diff * SHIFT + bits_in_digit(a.digit(a_size - 1)) -
-                           bits_in_digit(b.digit(b_size - 1)))
-    # Now diff = a_bits - b_bits.
-    if diff > DBL_MAX_EXP:
-        return _truediv_overflow()
-    elif diff < DBL_MIN_EXP - DBL_MANT_DIG - 1:
-        return _truediv_result(0.0, negate)
-
-    # Choose value for shift; see comments for step 1 in CPython.
-    shift = max(diff, DBL_MIN_EXP) - DBL_MANT_DIG - 2
-
-    inexact = False
-
-    # x = abs(a * 2**-shift)
-    if shift <= 0:
-        x = a.lshift(-shift)
-    else:
-        x = a.rshift(shift, dont_invert=True)
-        # set inexact if any of the bits shifted out is nonzero
-        if not a.eq(x.lshift(shift)):
-            inexact = True
-
-    # x //= b. If the remainder is nonzero, set inexact.
-    x, rem = _divrem(x, b)
-    if rem.tobool():
-        inexact = True
-
-    assert x.tobool()    # result of division is never zero
-    x_size = x.numdigits()
-    x_bits = (x_size-1)*SHIFT + bits_in_digit(x.digit(x_size-1))
-
-    # The number of extra bits that have to be rounded away.
-    extra_bits = max(x_bits, DBL_MIN_EXP - shift) - DBL_MANT_DIG
-    assert extra_bits == 2 or extra_bits == 3
-
-    # Round by remembering a modified copy of the low digit of x
-    mask = 1 << (extra_bits - 1)
-    low = x.udigit(0) | inexact
-    if (low & mask) != 0 and (low & (3*mask-1)) != 0:
-        low += mask
-    x_digit_0 = low & ~(mask-1)
-
-    # Convert x to a double dx; the conversion is exact.
-    x_size -= 1
-    dx = 0.0
-    while x_size > 0:
-        dx += x.digit(x_size)
-        dx *= BASE_AS_FLOAT
-        x_size -= 1
-    dx += x_digit_0
-
-    # Check whether ldexp result will overflow a double.
-    if (shift + x_bits >= DBL_MAX_EXP and
-        (shift + x_bits > DBL_MAX_EXP or dx == math.ldexp(1.0, x_bits))):
-        return _truediv_overflow()
-
-    return _truediv_result(math.ldexp(dx, shift), negate)
-
-# ____________________________________________________________
-
-BASE8  = '01234567'
-BASE10 = '0123456789'
-BASE16 = '0123456789abcdef'
-
-def _format(a, digits, prefix='', suffix=''):
-    """
-    Convert a bigint object to a string, using a given conversion base.
-    Return a string object.
-    """
-    size_a = a.numdigits()
-
-    base = len(digits)
-    assert base >= 2 and base <= 36
-
-    # Compute a rough upper bound for the length of the string
-    i = base
-    bits = 0
-    while i > 1:
-        bits += 1
-        i >>= 1
-    i = 5 + len(prefix) + len(suffix) + (size_a*SHIFT + bits-1) // bits
-    s = [chr(0)] * i
-    p = i
-    j = len(suffix)
-    while j > 0:
-        p -= 1
-        j -= 1
-        s[p] = suffix[j]
-
-    if a.sign == 0:
-        p -= 1
-        s[p] = '0'
-    elif (base & (base - 1)) == 0:
-        # JRH: special case for power-of-2 bases
-        accum = 0
-        accumbits = 0  # # of bits in accum 
-        basebits = 1   # # of bits in base-1
-        i = base
-        while 1:
-            i >>= 1
-            if i <= 1:
-                break
-            basebits += 1
-
-        for i in range(size_a):
-            accum |= a.widedigit(i) << accumbits
-            accumbits += SHIFT
-            assert accumbits >= basebits
-            while 1:
-                cdigit = intmask(accum & (base - 1))
-                p -= 1
-                assert p >= 0
-                s[p] = digits[cdigit]
-                accumbits -= basebits
-                accum >>= basebits
-                if i < size_a - 1:
-                    if accumbits < basebits:
-                        break
-                else:
-                    if accum <= 0:
-                        break
-    else:
-        # Not 0, and base not a power of 2.  Divide repeatedly by
-        # base, but for speed use the highest power of base that
-        # fits in a digit.
-        size = size_a
-        pin = a # just for similarity to C source which uses the array
-        # powbase <- largest power of base that fits in a digit.
-        powbase = _widen_digit(base)  # powbase == base ** power
-        power = 1
-        while 1:
-            newpow = powbase * base
-            if newpow >> SHIFT:  # doesn't fit in a digit
-                break
-            powbase = newpow
-            power += 1
-
-        # Get a scratch area for repeated division.
-        scratch = rbigint([NULLDIGIT] * size, 1)
-
-        # Repeatedly divide by powbase.
-        while 1:
-            ntostore = power
-            rem = _inplace_divrem1(scratch, pin, powbase, size)
-            pin = scratch  # no need to use a again
-            if pin.digit(size - 1) == 0:
-                size -= 1
-
-            # Break rem into digits.
-            assert ntostore > 0
-            while 1:
-                nextrem = rem // base
-                c = rem - nextrem * base
-                p -= 1
-                assert p >= 0
-                s[p] = digits[c]
-                rem = nextrem
-                ntostore -= 1
-                # Termination is a bit delicate:  must not
-                # store leading zeroes, so must get out if
-                # remaining quotient and rem are both 0.
-                if not (ntostore and (size or rem)):
-                    break
-            if size == 0:
-                break
-
-    j = len(prefix)
-    while j > 0:
-        p -= 1
-        j -= 1
-        s[p] = prefix[j]
-
-    if a.sign < 0:
-        p -= 1
-        s[p] = '-'
-
-    assert p >= 0    # otherwise, buffer overflow (this is also a
-                     # hint for the annotator for the slice below)
-    return ''.join(s[p:])
-
-
-def _bitwise(a, op, b): # '&', '|', '^'
-    """ Bitwise and/or/xor operations """
-
-    if a.sign < 0:
-        a = a.invert()
-        maska = MASK
-    else:
-        maska = 0
-    if b.sign < 0:
-        b = b.invert()
-        maskb = MASK
-    else:
-        maskb = 0
-
-    negz = 0
-    if op == '^':
-        if maska != maskb:
-            maska ^= MASK
-            negz = -1
-    elif op == '&':
-        if maska and maskb:
-            op = '|'
-            maska ^= MASK
-            maskb ^= MASK
-            negz = -1
-    elif op == '|':
-        if maska or maskb:
-            op = '&'
-            maska ^= MASK
-            maskb ^= MASK
-            negz = -1
-
-    # JRH: The original logic here was to allocate the result value (z)
-    # as the longer of the two operands.  However, there are some cases
-    # where the result is guaranteed to be shorter than that: AND of two
-    # positives, OR of two negatives: use the shorter number.  AND with
-    # mixed signs: use the positive number.  OR with mixed signs: use the
-    # negative number.  After the transformations above, op will be '&'
-    # iff one of these cases applies, and mask will be non-0 for operands
-    # whose length should be ignored.
-
-    size_a = a.numdigits()
-    size_b = b.numdigits()
-    if op == '&':
-        if maska:
-            size_z = size_b
-        else:
-            if maskb:
-                size_z = size_a
-            else:
-                size_z = min(size_a, size_b)
-    else:
-        size_z = max(size_a, size_b)
-
-    z = rbigint([NULLDIGIT] * size_z, 1)
-
-    for i in range(size_z):
-        if i < size_a:
-            diga = a.digit(i) ^ maska
-        else:
-            diga = maska
-        if i < size_b:
-            digb = b.digit(i) ^ maskb
-        else:
-            digb = maskb
-        if op == '&':
-            z.setdigit(i, diga & digb)
-        elif op == '|':
-            z.setdigit(i, diga | digb)
-        elif op == '^':
-            z.setdigit(i, diga ^ digb)
-
-    z._normalize()
-    if negz == 0:
-        return z
-    return z.invert()
-_bitwise._annspecialcase_ = "specialize:arg(1)"
-
-
-ULONGLONG_BOUND = r_ulonglong(1L << (r_longlong.BITS-1))
-LONGLONG_MIN = r_longlong(-(1L << (r_longlong.BITS-1)))
-
-def _AsLongLong(v):
-    """
-    Get a r_longlong integer from a bigint object.
-    Raises OverflowError if overflow occurs.
-    """
-    x = _AsULonglong_ignore_sign(v)
-    # grr grr grr
-    if x >= ULONGLONG_BOUND:
-        if x == ULONGLONG_BOUND and v.sign < 0:
-            x = LONGLONG_MIN
-        else:
-            raise OverflowError
-    else:
-        x = r_longlong(x)
-        if v.sign < 0:
-            x = -x
-    return x
-
-def _AsULonglong_ignore_sign(v):
-    x = r_ulonglong(0)
-    i = v.numdigits() - 1
-    while i >= 0:
-        prev = x
-        x = (x << SHIFT) + v.widedigit(i)
-        if (x >> SHIFT) != prev:
-                raise OverflowError(
-                    "long int too large to convert to unsigned long long int")
-        i -= 1
-    return x
-
-def make_unsigned_mask_conversion(T):
-    def _As_unsigned_mask(v):
-        x = T(0)
-        i = v.numdigits() - 1
-        while i >= 0:
-            x = (x << SHIFT) + T(v.digit(i))
-            i -= 1
-        if v.sign < 0:
-            x = -x
-        return x
-    return _As_unsigned_mask
-
-_AsULonglong_mask = make_unsigned_mask_conversion(r_ulonglong)
-_AsUInt_mask = make_unsigned_mask_conversion(r_uint)
-
-def _hash(v):
-    # This is designed so that Python ints and longs with the
-    # same value hash to the same value, otherwise comparisons
-    # of mapping keys will turn out weird.  Moreover, purely
-    # to please decimal.py, we return a hash that satisfies
-    # hash(x) == hash(x % ULONG_MAX).  In particular, this
-    # implies that hash(x) == hash(x % (2**64-1)).
-    i = v.numdigits() - 1
-    sign = v.sign
-    x = r_uint(0)
-    LONG_BIT_SHIFT = LONG_BIT - SHIFT
-    while i >= 0:
-        # Force a native long #-bits (32 or 64) circular shift
-        x = (x << SHIFT) | (x >> LONG_BIT_SHIFT)
-        x += v.udigit(i)
-        # If the addition above overflowed we compensate by
-        # incrementing.  This preserves the value modulo
-        # ULONG_MAX.
-        if x < v.udigit(i):
-            x += 1
-        i -= 1
-    x = intmask(x * sign)
-    return x
-
-#_________________________________________________________________
-
-# a few internal helpers
-
-def digits_max_for_base(base):
-    dec_per_digit = 1
-    while base ** dec_per_digit < MASK:
-        dec_per_digit += 1
-    dec_per_digit -= 1
-    return base ** dec_per_digit
-
-BASE_MAX = [0, 0] + [digits_max_for_base(_base) for _base in range(2, 37)]
-DEC_MAX = digits_max_for_base(10)
-assert DEC_MAX == BASE_MAX[10]
-
-def _decimalstr_to_bigint(s):
-    # a string that has been already parsed to be decimal and valid,
-    # is turned into a bigint
-    p = 0
-    lim = len(s)
-    sign = False
-    if s[p] == '-':
-        sign = True
-        p += 1
-    elif s[p] == '+':
-        p += 1
-
-    a = rbigint()
-    tens = 1
-    dig = 0
-    ord0 = ord('0')
-    while p < lim:
-        dig = dig * 10 + ord(s[p]) - ord0
-        p += 1
-        tens *= 10
-        if tens == DEC_MAX or p == lim:
-            a = _muladd1(a, tens, dig)
-            tens = 1
-            dig = 0
-    if sign and a.sign == 1:
-        a.sign = -1
-    return a
-
-def parse_digit_string(parser):
-    # helper for objspace.std.strutil
-    a = rbigint()
-    base = parser.base
-    digitmax = BASE_MAX[base]
-    tens, dig = 1, 0
-    while True:
-        digit = parser.next_digit()
-        if tens == digitmax or digit < 0:
-            a = _muladd1(a, tens, dig)
-            if digit < 0:
-                break
-            dig = digit
-            tens = base
-        else:
-            dig = dig * base + digit
-            tens *= base
-    a.sign *= parser.sign
-    return a
+if USE_GMP:
+    xxx
+else:
+    from pypy.rlib._rbigint_native import rbigint, parse_digit_string
diff --git a/pypy/rlib/test/test_rbigint.py b/pypy/rlib/test/test_rbigint_native.py
rename from pypy/rlib/test/test_rbigint.py
rename to pypy/rlib/test/test_rbigint_native.py
--- a/pypy/rlib/test/test_rbigint.py
+++ b/pypy/rlib/test/test_rbigint_native.py
@@ -2,9 +2,9 @@
 import py
 import operator, sys
 from random import random, randint, sample
-from pypy.rlib.rbigint import rbigint, SHIFT, MASK, KARATSUBA_CUTOFF
-from pypy.rlib.rbigint import _store_digit
-from pypy.rlib import rbigint as lobj
+from pypy.rlib._rbigint_native import rbigint, SHIFT, MASK, KARATSUBA_CUTOFF
+from pypy.rlib._rbigint_native import _store_digit, parse_digit_string
+from pypy.rlib import _rbigint_native as lobj
 from pypy.rlib.rarithmetic import r_uint, r_longlong, r_ulonglong, intmask
 from pypy.rpython.test.test_llinterp import interpret
 
@@ -599,7 +599,6 @@
                 r_ulonglong(-9**50))
 
     def test_parse_digit_string(self):
-        from pypy.rlib.rbigint import parse_digit_string
         class Parser:
             def __init__(self, base, sign, digits):
                 self.base = base

From noreply at buildbot.pypy.org  Thu Oct 20 16:37:33 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 20 Oct 2011 16:37:33 +0200 (CEST)
Subject: [pypy-commit] pypy gmp: First test passes.
Message-ID: <20111020143733.08412820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: gmp
Changeset: r48267:b7758f1003d0
Date: 2011-10-20 16:13 +0200
http://bitbucket.org/pypy/pypy/changeset/b7758f1003d0/

Log:	First test passes.

diff --git a/pypy/rlib/_rbigint_gmp.py b/pypy/rlib/_rbigint_gmp.py
new file mode 100644
--- /dev/null
+++ b/pypy/rlib/_rbigint_gmp.py
@@ -0,0 +1,58 @@
+from pypy.rpython.lltypesystem import lltype, rffi
+from pypy.translator.tool.cbuild import ExternalCompilationInfo
+
+eci = ExternalCompilationInfo(includes=["gmp.h"],
+                              libraries=["gmp"])
+
+mpz_t = rffi.COpaque("mpz_t", ptr_typedef="mpz_ptr", compilation_info=eci)
+mpz_ptr = lltype.Ptr(mpz_t)
+
+def external(name, args, result=lltype.Void):
+    name = "__g" + name    # temporary hack?
+    return rffi.llexternal(name, args, result, compilation_info=eci)
+
+mpz_init        = external("mpz_init", [mpz_ptr])
+mpz_init_set_si = external("mpz_init_set_si", [mpz_ptr, rffi.LONG])
+mpz_get_si      = external("mpz_get_si", [mpz_ptr], rffi.LONG)
+mpz_add         = external("mpz_add", [mpz_ptr, mpz_ptr, mpz_ptr])
+mpz_sub         = external("mpz_sub", [mpz_ptr, mpz_ptr, mpz_ptr])
+mpz_mul         = external("mpz_mul", [mpz_ptr, mpz_ptr, mpz_ptr])
+
+# ____________________________________________________________
+
+class _adtmeths:
+
+    @lltype.typeMethod
+    def fromint(RBIGINT, value):
+        r = lltype.malloc(RBIGINT)
+        mpz_init_set_si(r.mpz, value)
+        return r
+
+    def tolong(r):
+        return mpz_get_si(r.mpz)
+
+    def add(r1, r2):
+        r = lltype.malloc(RBIGINT)
+        mpz_init(r.mpz)
+        mpz_add(r.mpz, r1.mpz, r2.mpz)
+        return r
+
+    def sub(r1, r2):
+        r = lltype.malloc(RBIGINT)
+        mpz_init(r.mpz)
+        mpz_sub(r.mpz, r1.mpz, r2.mpz)
+        return r
+
+    def mul(r1, r2):
+        r = lltype.malloc(RBIGINT)
+        mpz_init(r.mpz)
+        mpz_mul(r.mpz, r1.mpz, r2.mpz)
+        return r
+
+_adtmeths = dict([(key, value) for (key, value) in _adtmeths.__dict__.items()
+                               if not key.startswith('_')])
+
+RBIGINT = lltype.GcStruct("RBIGINT_GMP",
+                          ('mpz', mpz_t),
+                          adtmeths = _adtmeths)
+# XXX call mpz_clear() in a lightweight finalizer
diff --git a/pypy/rlib/test/test_rbigint_native.py b/pypy/rlib/test/test_rbigint_gmp.py
copy from pypy/rlib/test/test_rbigint_native.py
copy to pypy/rlib/test/test_rbigint_gmp.py
--- a/pypy/rlib/test/test_rbigint_native.py
+++ b/pypy/rlib/test/test_rbigint_gmp.py
@@ -2,9 +2,8 @@
 import py
 import operator, sys
 from random import random, randint, sample
-from pypy.rlib._rbigint_native import rbigint, SHIFT, MASK, KARATSUBA_CUTOFF
-from pypy.rlib._rbigint_native import _store_digit, parse_digit_string
-from pypy.rlib import _rbigint_native as lobj
+from pypy.rlib._rbigint_gmp import RBIGINT as rbigint
+from pypy.rlib import _rbigint_gmp as lobj
 from pypy.rlib.rarithmetic import r_uint, r_longlong, r_ulonglong, intmask
 from pypy.rpython.test.test_llinterp import interpret
 
@@ -118,6 +117,7 @@
             yield -s
 
 def bigint(lst, sign):
+    XXX
     for digit in lst:
         assert digit & MASK == digit    # wrongly written test!
     return rbigint(map(_store_digit, lst), sign)
@@ -626,8 +626,6 @@
         assert x.tobool() is False
 
 
-BASE = 2 ** SHIFT
-
 class TestTranslatable(object):
     def test_square(self):
         def test():
diff --git a/pypy/rlib/test/test_rbigint_native.py b/pypy/rlib/test/test_rbigint_native.py
--- a/pypy/rlib/test/test_rbigint_native.py
+++ b/pypy/rlib/test/test_rbigint_native.py
@@ -626,8 +626,6 @@
         assert x.tobool() is False
 
 
-BASE = 2 ** SHIFT
-
 class TestTranslatable(object):
     def test_square(self):
         def test():

From noreply at buildbot.pypy.org  Thu Oct 20 16:37:34 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 20 Oct 2011 16:37:34 +0200 (CEST)
Subject: [pypy-commit] pypy gmp: More tests pass.
Message-ID: <20111020143734.2DBDA820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: gmp
Changeset: r48268:c40c0e765925
Date: 2011-10-20 16:24 +0200
http://bitbucket.org/pypy/pypy/changeset/c40c0e765925/

Log:	More tests pass.

diff --git a/pypy/rlib/_rbigint_gmp.py b/pypy/rlib/_rbigint_gmp.py
--- a/pypy/rlib/_rbigint_gmp.py
+++ b/pypy/rlib/_rbigint_gmp.py
@@ -1,5 +1,7 @@
 from pypy.rpython.lltypesystem import lltype, rffi
+from pypy.rpython.lltypesystem.lltype import typeMethod
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
+from pypy.rlib import jit
 
 eci = ExternalCompilationInfo(includes=["gmp.h"],
                               libraries=["gmp"])
@@ -8,24 +10,45 @@
 mpz_ptr = lltype.Ptr(mpz_t)
 
 def external(name, args, result=lltype.Void):
-    name = "__g" + name    # temporary hack?
+    if name.startswith('mp'):
+        name = "__g" + name    # temporary hack?
     return rffi.llexternal(name, args, result, compilation_info=eci)
 
 mpz_init        = external("mpz_init", [mpz_ptr])
 mpz_init_set_si = external("mpz_init_set_si", [mpz_ptr, rffi.LONG])
+mpz_init_set_str= external("mpz_init_set_str", [mpz_ptr, rffi.CCHARP,rffi.INT])
 mpz_get_si      = external("mpz_get_si", [mpz_ptr], rffi.LONG)
+mpz_get_str     = external("mpz_get_str", [rffi.CCHARP, rffi.INT, mpz_ptr],
+                           rffi.CCHARP)
 mpz_add         = external("mpz_add", [mpz_ptr, mpz_ptr, mpz_ptr])
 mpz_sub         = external("mpz_sub", [mpz_ptr, mpz_ptr, mpz_ptr])
 mpz_mul         = external("mpz_mul", [mpz_ptr, mpz_ptr, mpz_ptr])
+_free           = external("free", [rffi.CCHARP])
 
 # ____________________________________________________________
 
+def _fromint(value):
+    r = lltype.malloc(RBIGINT)
+    mpz_init_set_si(r.mpz, value)
+    return r
+
+
 class _adtmeths:
 
-    @lltype.typeMethod
+    @typeMethod
+    @jit.elidable
     def fromint(RBIGINT, value):
+        return _fromint(value)
+
+    @typeMethod
+    def frombool(RBIGINT, b):
+        return _fromint(int(b))    # maybe do some caching?
+
+    @typeMethod
+    def fromlong(RBIGINT, l):
+        "NOT_RPYTHON"
         r = lltype.malloc(RBIGINT)
-        mpz_init_set_si(r.mpz, value)
+        mpz_init_set_str(r.mpz, str(l), 10)
         return r
 
     def tolong(r):
@@ -49,9 +72,17 @@
         mpz_mul(r.mpz, r1.mpz, r2.mpz)
         return r
 
+    def str(r):
+        p = mpz_get_str(lltype.nullptr(rffi.CCHARP.TO), 10, r.mpz)
+        result = rffi.charp2str(p)
+        _free(p)
+        return result
+
 _adtmeths = dict([(key, value) for (key, value) in _adtmeths.__dict__.items()
                                if not key.startswith('_')])
 
+# ____________________________________________________________
+
 RBIGINT = lltype.GcStruct("RBIGINT_GMP",
                           ('mpz', mpz_t),
                           adtmeths = _adtmeths)

From noreply at buildbot.pypy.org  Thu Oct 20 17:59:14 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 20 Oct 2011 17:59:14 +0200 (CEST)
Subject: [pypy-commit] pypy gmp: Some more operations.
Message-ID: <20111020155914.9D82D820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: gmp
Changeset: r48269:5308f097341b
Date: 2011-10-20 16:44 +0200
http://bitbucket.org/pypy/pypy/changeset/5308f097341b/

Log:	Some more operations.

diff --git a/pypy/rlib/_rbigint_gmp.py b/pypy/rlib/_rbigint_gmp.py
--- a/pypy/rlib/_rbigint_gmp.py
+++ b/pypy/rlib/_rbigint_gmp.py
@@ -23,10 +23,13 @@
 mpz_add         = external("mpz_add", [mpz_ptr, mpz_ptr, mpz_ptr])
 mpz_sub         = external("mpz_sub", [mpz_ptr, mpz_ptr, mpz_ptr])
 mpz_mul         = external("mpz_mul", [mpz_ptr, mpz_ptr, mpz_ptr])
+mpz_fdiv_q      = external("mpz_fdiv_q", [mpz_ptr, mpz_ptr, mpz_ptr])
+mpz_fdiv_r      = external("mpz_fdiv_r", [mpz_ptr, mpz_ptr, mpz_ptr])
 _free           = external("free", [rffi.CCHARP])
 
 # ____________________________________________________________
 
+
 def _fromint(value):
     r = lltype.malloc(RBIGINT)
     mpz_init_set_si(r.mpz, value)
@@ -51,33 +54,35 @@
         mpz_init_set_str(r.mpz, str(l), 10)
         return r
 
-    def tolong(r):
-        return mpz_get_si(r.mpz)
-
-    def add(r1, r2):
-        r = lltype.malloc(RBIGINT)
-        mpz_init(r.mpz)
-        mpz_add(r.mpz, r1.mpz, r2.mpz)
-        return r
-
-    def sub(r1, r2):
-        r = lltype.malloc(RBIGINT)
-        mpz_init(r.mpz)
-        mpz_sub(r.mpz, r1.mpz, r2.mpz)
-        return r
-
-    def mul(r1, r2):
-        r = lltype.malloc(RBIGINT)
-        mpz_init(r.mpz)
-        mpz_mul(r.mpz, r1.mpz, r2.mpz)
-        return r
-
     def str(r):
         p = mpz_get_str(lltype.nullptr(rffi.CCHARP.TO), 10, r.mpz)
         result = rffi.charp2str(p)
         _free(p)
         return result
 
+    def tolong(r):
+        return mpz_get_si(r.mpz)
+
+    def _binary(opname):
+        mpz_op = globals()['mpz_' + opname]
+        def operation(r1, r2):
+            r = lltype.malloc(RBIGINT)
+            mpz_init(r.mpz)
+            mpz_op(r.mpz, r1.mpz, r2.mpz)
+            return r
+        operation.__name__ = opname
+        return operation
+
+    add = _binary('add')
+    sub = _binary('sub')
+    mul = _binary('mul')
+    div = _binary('fdiv_q')
+    floordiv = div
+    mod = _binary('fdiv_r')
+
+    def truediv(r1, r2):
+        import py; py.test.skip("XXX")
+
 _adtmeths = dict([(key, value) for (key, value) in _adtmeths.__dict__.items()
                                if not key.startswith('_')])
 

From noreply at buildbot.pypy.org  Thu Oct 20 17:59:15 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 20 Oct 2011 17:59:15 +0200 (CEST)
Subject: [pypy-commit] pypy gmp: In-progress.
Message-ID: <20111020155915.C4DCE820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: gmp
Changeset: r48270:118e6ccfb4c1
Date: 2011-10-20 17:26 +0200
http://bitbucket.org/pypy/pypy/changeset/118e6ccfb4c1/

Log:	In-progress.

diff --git a/pypy/rlib/_rbigint_gmp.py b/pypy/rlib/_rbigint_gmp.py
--- a/pypy/rlib/_rbigint_gmp.py
+++ b/pypy/rlib/_rbigint_gmp.py
@@ -2,6 +2,7 @@
 from pypy.rpython.lltypesystem.lltype import typeMethod
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
 from pypy.rlib import jit
+from pypy.rlib.objectmodel import specialize
 
 eci = ExternalCompilationInfo(includes=["gmp.h"],
                               libraries=["gmp"])
@@ -18,6 +19,7 @@
 mpz_init_set_si = external("mpz_init_set_si", [mpz_ptr, rffi.LONG])
 mpz_init_set_str= external("mpz_init_set_str", [mpz_ptr, rffi.CCHARP,rffi.INT])
 mpz_get_si      = external("mpz_get_si", [mpz_ptr], rffi.LONG)
+mpz_get_ui      = external("mpz_get_ui", [mpz_ptr], rffi.ULONG)
 mpz_get_str     = external("mpz_get_str", [rffi.CCHARP, rffi.INT, mpz_ptr],
                            rffi.CCHARP)
 mpz_add         = external("mpz_add", [mpz_ptr, mpz_ptr, mpz_ptr])
@@ -25,6 +27,10 @@
 mpz_mul         = external("mpz_mul", [mpz_ptr, mpz_ptr, mpz_ptr])
 mpz_fdiv_q      = external("mpz_fdiv_q", [mpz_ptr, mpz_ptr, mpz_ptr])
 mpz_fdiv_r      = external("mpz_fdiv_r", [mpz_ptr, mpz_ptr, mpz_ptr])
+mpz_pow_ui      = external("mpz_pow_ui", [mpz_ptr, mpz_ptr, rffi.ULONG])
+
+mpz_fits_ulong_p = external("mpz_fits_ulong_p", [mpz_ptr], rffi.INT)
+
 _free           = external("free", [rffi.CCHARP])
 
 # ____________________________________________________________
@@ -35,6 +41,12 @@
     mpz_init_set_si(r.mpz, value)
     return r
 
+def _str_base_10(r):
+    p = mpz_get_str(lltype.nullptr(rffi.CCHARP.TO), 10, r.mpz)
+    result = rffi.charp2str(p)
+    _free(p)
+    return result
+
 
 class _adtmeths:
 
@@ -54,14 +66,11 @@
         mpz_init_set_str(r.mpz, str(l), 10)
         return r
 
-    def str(r):
-        p = mpz_get_str(lltype.nullptr(rffi.CCHARP.TO), 10, r.mpz)
-        result = rffi.charp2str(p)
-        _free(p)
-        return result
+    str = _str_base_10
 
     def tolong(r):
-        return mpz_get_si(r.mpz)
+        "NOT_RPYTHON"
+        return int(_str_base_10(r))
 
     def _binary(opname):
         mpz_op = globals()['mpz_' + opname]
@@ -83,6 +92,19 @@
     def truediv(r1, r2):
         import py; py.test.skip("XXX")
 
+    @specialize.argtype(2)
+    def pow(a, b, c=None):
+        if c is None:
+            fits = rffi.cast(lltype.Signed, mpz_fits_ulong_p(b.mpz))
+            assert fits, "XXX"
+            b_ulong = mpz_get_ui(b.mpz)
+            r = lltype.malloc(RBIGINT)
+            mpz_init(r.mpz)
+            mpz_pow_ui(r.mpz, a.mpz, b_ulong)
+            return r
+        else:
+            yyy
+
 _adtmeths = dict([(key, value) for (key, value) in _adtmeths.__dict__.items()
                                if not key.startswith('_')])
 

From noreply at buildbot.pypy.org  Thu Oct 20 17:59:16 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 20 Oct 2011 17:59:16 +0200 (CEST)
Subject: [pypy-commit] pypy gmp: Next function.
Message-ID: <20111020155916.E99E6820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: gmp
Changeset: r48271:b396542050a4
Date: 2011-10-20 17:31 +0200
http://bitbucket.org/pypy/pypy/changeset/b396542050a4/

Log:	Next function.

diff --git a/pypy/rlib/_rbigint_gmp.py b/pypy/rlib/_rbigint_gmp.py
--- a/pypy/rlib/_rbigint_gmp.py
+++ b/pypy/rlib/_rbigint_gmp.py
@@ -72,6 +72,15 @@
         "NOT_RPYTHON"
         return int(_str_base_10(r))
 
+    def touint(r):
+        if mpz_fits_ulong_p(r.mpz):
+            return mpz_get_ui(r.mpz)
+        elif XXX + mpz_sgn(r.mpz) < 0:
+            raise ValueError("cannot convert negative integer to unsigned int")
+        else:
+            raise OverflowError(
+                "long int too large to convert to unsigned int")
+
     def _binary(opname):
         mpz_op = globals()['mpz_' + opname]
         def operation(r1, r2):

From noreply at buildbot.pypy.org  Thu Oct 20 17:59:18 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 20 Oct 2011 17:59:18 +0200 (CEST)
Subject: [pypy-commit] pypy gmp: in-progress.
Message-ID: <20111020155918.21445820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: gmp
Changeset: r48272:d81a2c7c6ca1
Date: 2011-10-20 17:58 +0200
http://bitbucket.org/pypy/pypy/changeset/d81a2c7c6ca1/

Log:	in-progress.

diff --git a/pypy/rlib/_rbigint_gmp.py b/pypy/rlib/_rbigint_gmp.py
--- a/pypy/rlib/_rbigint_gmp.py
+++ b/pypy/rlib/_rbigint_gmp.py
@@ -36,9 +36,15 @@
 # ____________________________________________________________
 
 
+ at specialize.argtype(0)
 def _fromint(value):
     r = lltype.malloc(RBIGINT)
-    mpz_init_set_si(r.mpz, value)
+    #
+    T = type(value)
+    if T is int or T is r_int:
+        mpz_init_set_si(r.mpz, value)
+    elif ...
+    
     return r
 
 def _str_base_10(r):
@@ -60,6 +66,11 @@
         return _fromint(int(b))    # maybe do some caching?
 
     @typeMethod
+    @specialize.argtype(1)
+    def fromrarith_int(RBIGINT, i):
+        return _fromint(widen(i))
+
+    @typeMethod
     def fromlong(RBIGINT, l):
         "NOT_RPYTHON"
         r = lltype.malloc(RBIGINT)
diff --git a/pypy/rlib/test/test_rbigint_gmp.py b/pypy/rlib/test/test_rbigint_gmp.py
--- a/pypy/rlib/test/test_rbigint_gmp.py
+++ b/pypy/rlib/test/test_rbigint_gmp.py
@@ -117,7 +117,6 @@
             yield -s
 
 def bigint(lst, sign):
-    XXX
     for digit in lst:
         assert digit & MASK == digit    # wrongly written test!
     return rbigint(map(_store_digit, lst), sign)
@@ -126,6 +125,7 @@
 class Test_rbigint(object):
 
     def test_args_from_long(self):
+        py.test.skip("not valid")
         BASE = 1 << SHIFT
         assert rbigint.fromlong(0).eq(bigint([0], 0))
         assert rbigint.fromlong(17).eq(bigint([17], 1))
@@ -139,7 +139,17 @@
 #        assert rbigint.fromlong(-sys.maxint-1).eq(
 #            rbigint.digits_for_most_neg_long(-sys.maxint-1), -1)
 
+    def test_args_from_int_simple(self):
+        IMAX = sys.maxint
+        LLMAX = r_longlong(2**63 - 1)
+        assert rbigint.fromrarith_int(-17).eq(rbigint.fromlong(-17))
+        assert rbigint.fromrarith_int(IMAX).eq(rbigint.fromlong(IMAX))
+        assert rbigint.fromrarith_int(LLMAX).eq(rbigint.fromlong(LLMAX))
+        assert rbigint.fromrarith_int(-IMAX-1).eq(rbigint.fromlong(-IMAX))
+        assert rbigint.fromrarith_int(-LLMAX-1).eq(rbigint.fromlong(-LLMAX-1))
+
     def test_args_from_int(self):
+        py.test.skip("not valid")
         BASE = 1 << SHIFT
         MAX = int(BASE-1)
         assert rbigint.fromrarith_int(0).eq(bigint([0], 0))

From noreply at buildbot.pypy.org  Thu Oct 20 17:59:19 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 20 Oct 2011 17:59:19 +0200 (CEST)
Subject: [pypy-commit] pypy gmp: Close the "gmp" branch already and give up, 
Message-ID: <20111020155919.46BE5820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: gmp
Changeset: r48273:105fe17b27f7
Date: 2011-10-20 17:58 +0200
http://bitbucket.org/pypy/pypy/changeset/105fe17b27f7/

Log:	Close the "gmp" branch already and give up, as described in
	https://bugs.pypy.org/issue892.


From noreply at buildbot.pypy.org  Thu Oct 20 18:25:46 2011
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Thu, 20 Oct 2011 18:25:46 +0200 (CEST)
Subject: [pypy-commit] pyrepl py3ksupport: add for forgotten basestring
	tuple for python3
Message-ID: <20111020162546.11EB5820D7@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: py3ksupport
Changeset: r151:69c3300fb54e
Date: 2011-10-20 18:25 +0200
http://bitbucket.org/pypy/pyrepl/changeset/69c3300fb54e/

Log:	add for forgotten basestring tuple for python3

diff --git a/pyrepl/reader.py b/pyrepl/reader.py
--- a/pyrepl/reader.py
+++ b/pyrepl/reader.py
@@ -28,6 +28,7 @@
 except NameError:
     unicode = str
     unichr = chr
+    basestring = bytes, str
 
 def _make_unctrl_map():
     uc_map = {}

From noreply at buildbot.pypy.org  Thu Oct 20 18:37:13 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 20 Oct 2011 18:37:13 +0200 (CEST)
Subject: [pypy-commit] benchmarks default: add a simple json benchmark
Message-ID: <20111020163713.8EFD0820D7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r147:a561cae55462
Date: 2011-10-20 18:34 +0200
http://bitbucket.org/pypy/benchmarks/changeset/a561cae55462/

Log:	add a simple json benchmark

diff --git a/benchmarks.py b/benchmarks.py
--- a/benchmarks.py
+++ b/benchmarks.py
@@ -54,7 +54,8 @@
 
 for name in ['float', 'nbody_modified', 'meteor-contest', 'fannkuch',
              'spectral-norm', 'chaos', 'telco', 'go', 'pyflate-fast',
-             'raytrace-simple', 'crypto_pyaes', 'bm_mako', 'bm_chameleon']:
+             'raytrace-simple', 'crypto_pyaes', 'bm_mako', 'bm_chameleon',
+             'json_bench']:
     _register_new_bm(name, name, globals(), **opts.get(name, {}))
 for name in ['names', 'iteration', 'tcp', 'pb']:#, 'web', 'accepts']:
     iteration_scaling = 1.0
diff --git a/own/json_bench.py b/own/json_bench.py
new file mode 100644
--- /dev/null
+++ b/own/json_bench.py
@@ -0,0 +1,33 @@
+import time
+import json
+
+# execution runtime per test case
+TARGET_RUNTIME = 10
+
+EMPTY = ({}, 200000)
+SIMPLE = ({'key1': 0, 'key2': True, 'key3': 'value', 'key4': 'foo', 'key5': 'string'}, 100000)
+NESTED = ({'key1': 0, 'key2': SIMPLE[0], 'key3': 'value', 'key4': SIMPLE[0], 'key5': SIMPLE[0], u'key': u'\u0105\u0107\u017c'}, 100000)
+HUGE = ([NESTED[0]] * 1000, 100)
+
+cases = ['EMPTY', 'SIMPLE', 'NESTED', 'HUGE']
+
+def main(n):
+    l = []
+    for i in range(n):
+        t0 = time.time()
+        for case in cases:
+            data, count = globals()[case]
+            for i in range(count):
+                json.dumps(data)
+        l.append(time.time() - t0)
+    return l
+
+if __name__ == '__main__':
+    import util, optparse
+    parser = optparse.OptionParser(
+        usage="%prog [options]",
+        description="Test the performance of the Go benchmark")
+    util.add_standard_options_to(parser)
+    options, args = parser.parse_args()
+
+    util.run_benchmark(options, options.num_runs, main)

From notifications-noreply at bitbucket.org  Thu Oct 20 19:14:23 2011
From: notifications-noreply at bitbucket.org (Bitbucket)
Date: Thu, 20 Oct 2011 17:14:23 -0000
Subject: [pypy-commit] Notification: pypy
Message-ID: <20111020171423.13391.93792@bitbucket12.managed.contegix.com>

You have received a notification from soymo.

Hi, I forked pypy. My fork is at https://bitbucket.org/soymo/pypy.

--
Disable notifications at https://bitbucket.org/account/notifications/

From noreply at buildbot.pypy.org  Thu Oct 20 19:28:04 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 20 Oct 2011 19:28:04 +0200 (CEST)
Subject: [pypy-commit] pypy faster-json: don't use identity_dict and remove
 the commented out stuff
Message-ID: <20111020172804.09A84820D7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: faster-json
Changeset: r48274:abcebe394ced
Date: 2011-10-20 19:27 +0200
http://bitbucket.org/pypy/pypy/changeset/abcebe394ced/

Log:	don't use identity_dict and remove the commented out stuff

diff --git a/lib-python/modified-2.7/json/encoder.py b/lib-python/modified-2.7/json/encoder.py
--- a/lib-python/modified-2.7/json/encoder.py
+++ b/lib-python/modified-2.7/json/encoder.py
@@ -2,7 +2,6 @@
 """
 import re
 
-from __pypy__ import identity_dict
 from __pypy__.builders import StringBuilder, UnicodeBuilder
 
 ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]')
@@ -18,7 +17,6 @@
     '\t': '\\t',
 }
 for i in range(0x20):
-    #ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i))
     ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,))
 
 # Assume this produces an infinity on all machines (probably not guaranteed)
@@ -46,14 +44,12 @@
         except KeyError:
             n = ord(s)
             if n < 0x10000:
-                #return '\\u{0:04x}'.format(n)
                 return '\\u%04x' % (n,)
             else:
                 # surrogate pair
                 n -= 0x10000
                 s1 = 0xd800 | ((n >> 10) & 0x3ff)
                 s2 = 0xdc00 | (n & 0x3ff)
-                #return '\\u{0:04x}\\u{1:04x}'.format(s1, s2)
                 return '\\u%04x\\u%04x' % (s1, s2)
     if ESCAPE_ASCII.search(s):
         return str(ESCAPE_ASCII.sub(replace, s))
@@ -189,7 +185,7 @@
 
         """
         if self.check_circular:
-            markers = identity_dict()
+            markers = {}
         else:
             markers = None
         if self.ensure_ascii:
@@ -320,7 +316,7 @@
 
         """
         if self.check_circular:
-            markers = identity_dict()
+            markers = {}
         else:
             markers = None
         return self._iterencode(o, markers, 0)
@@ -348,13 +344,13 @@
 
     def _mark_markers(self, markers, o):
         if markers is not None:
-            if o in markers:
+            if id(o) in markers:
                 raise ValueError("Circular reference detected")
-            markers[o] = None
+            markers[id(o)] = None
 
     def _remove_markers(self, markers, o):
         if markers is not None:
-            del markers[o]
+            del markers[id(o)]
 
     def _iterencode_list(self, lst, markers, _current_indent_level):
         if not lst:

From noreply at buildbot.pypy.org  Thu Oct 20 19:30:56 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 20 Oct 2011 19:30:56 +0200 (CEST)
Subject: [pypy-commit] pypy faster-json: make alex happier
Message-ID: <20111020173056.1C7C5820D7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: faster-json
Changeset: r48275:6c562364a3e2
Date: 2011-10-20 19:30 +0200
http://bitbucket.org/pypy/pypy/changeset/6c562364a3e2/

Log:	make alex happier

diff --git a/lib-python/modified-2.7/json/encoder.py b/lib-python/modified-2.7/json/encoder.py
--- a/lib-python/modified-2.7/json/encoder.py
+++ b/lib-python/modified-2.7/json/encoder.py
@@ -195,7 +195,7 @@
         self._encode(o, markers, builder, 0)
         return builder.build()
 
-    def _emit_indent(self, builder, _current_indent_level):        
+    def _emit_indent(self, builder, _current_indent_level):
         if self.indent is not None:
             _current_indent_level += 1
             newline_indent = '\n' + (' ' * (self.indent *

From noreply at buildbot.pypy.org  Thu Oct 20 19:50:34 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 20 Oct 2011 19:50:34 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge faster-json branch. It provides
 encode() optimization for json module, 
Message-ID: <20111020175034.AAA6B820D7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r48276:51aebf5ad032
Date: 2011-10-20 19:49 +0200
http://bitbucket.org/pypy/pypy/changeset/51aebf5ad032/

Log:	merge faster-json branch. It provides encode() optimization for json
	module, that runs faster than the C extension on CPython (for our
	benchmark)

diff --git a/lib-python/modified-2.7/json/encoder.py b/lib-python/modified-2.7/json/encoder.py
--- a/lib-python/modified-2.7/json/encoder.py
+++ b/lib-python/modified-2.7/json/encoder.py
@@ -2,14 +2,7 @@
 """
 import re
 
-try:
-    from _json import encode_basestring_ascii as c_encode_basestring_ascii
-except ImportError:
-    c_encode_basestring_ascii = None
-try:
-    from _json import make_encoder as c_make_encoder
-except ImportError:
-    c_make_encoder = None
+from __pypy__.builders import StringBuilder, UnicodeBuilder
 
 ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]')
 ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])')
@@ -24,8 +17,7 @@
     '\t': '\\t',
 }
 for i in range(0x20):
-    ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i))
-    #ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,))
+    ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,))
 
 # Assume this produces an infinity on all machines (probably not guaranteed)
 INFINITY = float('1e66666')
@@ -37,10 +29,9 @@
     """
     def replace(match):
         return ESCAPE_DCT[match.group(0)]
-    return '"' + ESCAPE.sub(replace, s) + '"'
+    return ESCAPE.sub(replace, s)
 
-
-def py_encode_basestring_ascii(s):
+def encode_basestring_ascii(s):
     """Return an ASCII-only JSON representation of a Python string
 
     """
@@ -53,20 +44,18 @@
         except KeyError:
             n = ord(s)
             if n < 0x10000:
-                return '\\u{0:04x}'.format(n)
-                #return '\\u%04x' % (n,)
+                return '\\u%04x' % (n,)
             else:
                 # surrogate pair
                 n -= 0x10000
                 s1 = 0xd800 | ((n >> 10) & 0x3ff)
                 s2 = 0xdc00 | (n & 0x3ff)
-                return '\\u{0:04x}\\u{1:04x}'.format(s1, s2)
-                #return '\\u%04x\\u%04x' % (s1, s2)
-    return '"' + str(ESCAPE_ASCII.sub(replace, s)) + '"'
-
-
-encode_basestring_ascii = (
-    c_encode_basestring_ascii or py_encode_basestring_ascii)
+                return '\\u%04x\\u%04x' % (s1, s2)
+    if ESCAPE_ASCII.search(s):
+        return str(ESCAPE_ASCII.sub(replace, s))
+    return s
+py_encode_basestring_ascii = lambda s: '"' + encode_basestring_ascii(s) + '"'
+c_encode_basestring_ascii = None
 
 class JSONEncoder(object):
     """Extensible JSON <http://json.org> encoder for Python data structures.
@@ -147,6 +136,17 @@
 
         self.skipkeys = skipkeys
         self.ensure_ascii = ensure_ascii
+        if ensure_ascii:
+            self.encoder = encode_basestring_ascii
+        else:
+            self.encoder = encode_basestring
+        if encoding != 'utf-8':
+            orig_encoder = self.encoder
+            def encoder(o):
+                if isinstance(o, str):
+                    o = o.decode(encoding)
+                return orig_encoder(o)
+            self.encoder = encoder
         self.check_circular = check_circular
         self.allow_nan = allow_nan
         self.sort_keys = sort_keys
@@ -184,24 +184,126 @@
         '{"foo": ["bar", "baz"]}'
 
         """
-        # This is for extremely simple cases and benchmarks.
+        if self.check_circular:
+            markers = {}
+        else:
+            markers = None
+        if self.ensure_ascii:
+            builder = StringBuilder()
+        else:
+            builder = UnicodeBuilder()
+        self._encode(o, markers, builder, 0)
+        return builder.build()
+
+    def _emit_indent(self, builder, _current_indent_level):
+        if self.indent is not None:
+            _current_indent_level += 1
+            newline_indent = '\n' + (' ' * (self.indent *
+                                            _current_indent_level))
+            separator = self.item_separator + newline_indent
+            builder.append(newline_indent)
+        else:
+            separator = self.item_separator
+        return separator, _current_indent_level
+
+    def _emit_unindent(self, builder, _current_indent_level):
+        if self.indent is not None:
+            builder.append('\n')
+            builder.append(' ' * (self.indent * (_current_indent_level - 1)))
+
+    def _encode(self, o, markers, builder, _current_indent_level):
         if isinstance(o, basestring):
-            if isinstance(o, str):
-                _encoding = self.encoding
-                if (_encoding is not None
-                        and not (_encoding == 'utf-8')):
-                    o = o.decode(_encoding)
-            if self.ensure_ascii:
-                return encode_basestring_ascii(o)
+            builder.append('"')
+            builder.append(self.encoder(o))
+            builder.append('"')
+        elif o is None:
+            builder.append('null')
+        elif o is True:
+            builder.append('true')
+        elif o is False:
+            builder.append('false')
+        elif isinstance(o, (int, long)):
+            builder.append(str(o))
+        elif isinstance(o, float):
+            builder.append(self._floatstr(o))
+        elif isinstance(o, (list, tuple)):
+            if not o:
+                builder.append('[]')
+                return
+            self._encode_list(o, markers, builder, _current_indent_level)
+        elif isinstance(o, dict):
+            if not o:
+                builder.append('{}')
+                return
+            self._encode_dict(o, markers, builder, _current_indent_level)
+        else:
+            self._mark_markers(markers, o)
+            res = self.default(o)
+            self._encode(res, markers, builder, _current_indent_level)
+            self._remove_markers(markers, o)
+            return res
+
+    def _encode_list(self, l, markers, builder, _current_indent_level):
+        self._mark_markers(markers, l)
+        builder.append('[')
+        first = True
+        separator, _current_indent_level = self._emit_indent(builder,
+                                                      _current_indent_level)
+        for elem in l:
+            if first:
+                first = False
             else:
-                return encode_basestring(o)
-        # This doesn't pass the iterator directly to ''.join() because the
-        # exceptions aren't as detailed.  The list call should be roughly
-        # equivalent to the PySequence_Fast that ''.join() would do.
-        chunks = self.iterencode(o, _one_shot=True)
-        if not isinstance(chunks, (list, tuple)):
-            chunks = list(chunks)
-        return ''.join(chunks)
+                builder.append(separator)
+            self._encode(elem, markers, builder, _current_indent_level)
+            del elem # XXX grumble
+        self._emit_unindent(builder, _current_indent_level)
+        builder.append(']')
+        self._remove_markers(markers, l)
+
+    def _encode_dict(self, d, markers, builder, _current_indent_level):
+        self._mark_markers(markers, d)
+        first = True
+        builder.append('{')
+        separator, _current_indent_level = self._emit_indent(builder,
+                                                         _current_indent_level)
+        if self.sort_keys:
+            items = sorted(d.items(), key=lambda kv: kv[0])
+        else:
+            items = d.iteritems()
+
+        for key, v in items:
+            if first:
+                first = False
+            else:
+                builder.append(separator)
+            if isinstance(key, basestring):
+                pass
+            # JavaScript is weakly typed for these, so it makes sense to
+            # also allow them.  Many encoders seem to do something like this.
+            elif isinstance(key, float):
+                key = self._floatstr(key)
+            elif key is True:
+                key = 'true'
+            elif key is False:
+                key = 'false'
+            elif key is None:
+                key = 'null'
+            elif isinstance(key, (int, long)):
+                key = str(key)
+            elif self.skipkeys:
+                continue
+            else:
+                raise TypeError("key " + repr(key) + " is not a string")
+            builder.append('"')
+            builder.append(self.encoder(key))
+            builder.append('"')
+            builder.append(self.key_separator)
+            self._encode(v, markers, builder, _current_indent_level)
+            del key
+            del v # XXX grumble
+        self._emit_unindent(builder, _current_indent_level)
+        builder.append('}')
+        self._remove_markers(markers, d)
 
     def iterencode(self, o, _one_shot=False):
         """Encode the given object and yield each string
@@ -217,86 +319,54 @@
             markers = {}
         else:
             markers = None
-        if self.ensure_ascii:
-            _encoder = encode_basestring_ascii
+        return self._iterencode(o, markers, 0)
+
+    def _floatstr(self, o):
+        # Check for specials.  Note that this type of test is processor
+        # and/or platform-specific, so do tests which don't depend on the
+        # internals.
+
+        if o != o:
+            text = 'NaN'
+        elif o == INFINITY:
+            text = 'Infinity'
+        elif o == -INFINITY:
+            text = '-Infinity'
         else:
-            _encoder = encode_basestring
-        if self.encoding != 'utf-8':
-            def _encoder(o, _orig_encoder=_encoder, _encoding=self.encoding):
-                if isinstance(o, str):
-                    o = o.decode(_encoding)
-                return _orig_encoder(o)
+            return FLOAT_REPR(o)
 
-        def floatstr(o, allow_nan=self.allow_nan,
-                _repr=FLOAT_REPR, _inf=INFINITY, _neginf=-INFINITY):
-            # Check for specials.  Note that this type of test is processor
-            # and/or platform-specific, so do tests which don't depend on the
-            # internals.
+        if not self.allow_nan:
+            raise ValueError(
+                "Out of range float values are not JSON compliant: " +
+                repr(o))
 
-            if o != o:
-                text = 'NaN'
-            elif o == _inf:
-                text = 'Infinity'
-            elif o == _neginf:
-                text = '-Infinity'
-            else:
-                return _repr(o)
+        return text
 
-            if not allow_nan:
-                raise ValueError(
-                    "Out of range float values are not JSON compliant: " +
-                    repr(o))
+    def _mark_markers(self, markers, o):
+        if markers is not None:
+            if id(o) in markers:
+                raise ValueError("Circular reference detected")
+            markers[id(o)] = None
 
-            return text
+    def _remove_markers(self, markers, o):
+        if markers is not None:
+            del markers[id(o)]
 
-
-        if (_one_shot and c_make_encoder is not None
-                and not self.indent and not self.sort_keys):
-            _iterencode = c_make_encoder(
-                markers, self.default, _encoder, self.indent,
-                self.key_separator, self.item_separator, self.sort_keys,
-                self.skipkeys, self.allow_nan)
-        else:
-            _iterencode = _make_iterencode(
-                markers, self.default, _encoder, self.indent, floatstr,
-                self.key_separator, self.item_separator, self.sort_keys,
-                self.skipkeys, _one_shot)
-        return _iterencode(o, 0)
-
-def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,
-        _key_separator, _item_separator, _sort_keys, _skipkeys, _one_shot,
-        ## HACK: hand-optimized bytecode; turn globals into locals
-        ValueError=ValueError,
-        basestring=basestring,
-        dict=dict,
-        float=float,
-        id=id,
-        int=int,
-        isinstance=isinstance,
-        list=list,
-        long=long,
-        str=str,
-        tuple=tuple,
-    ):
-
-    def _iterencode_list(lst, _current_indent_level):
+    def _iterencode_list(self, lst, markers, _current_indent_level):
         if not lst:
             yield '[]'
             return
-        if markers is not None:
-            markerid = id(lst)
-            if markerid in markers:
-                raise ValueError("Circular reference detected")
-            markers[markerid] = lst
+        self._mark_markers(markers, lst)
         buf = '['
-        if _indent is not None:
+        if self.indent is not None:
             _current_indent_level += 1
-            newline_indent = '\n' + (' ' * (_indent * _current_indent_level))
-            separator = _item_separator + newline_indent
+            newline_indent = '\n' + (' ' * (self.indent *
+                                            _current_indent_level))
+            separator = self.item_separator + newline_indent
             buf += newline_indent
         else:
             newline_indent = None
-            separator = _item_separator
+            separator = self.item_separator
         first = True
         for value in lst:
             if first:
@@ -304,7 +374,7 @@
             else:
                 buf = separator
             if isinstance(value, basestring):
-                yield buf + _encoder(value)
+                yield buf + '"' + self.encoder(value) + '"'
             elif value is None:
                 yield buf + 'null'
             elif value is True:
@@ -314,44 +384,43 @@
             elif isinstance(value, (int, long)):
                 yield buf + str(value)
             elif isinstance(value, float):
-                yield buf + _floatstr(value)
+                yield buf + self._floatstr(value)
             else:
                 yield buf
                 if isinstance(value, (list, tuple)):
-                    chunks = _iterencode_list(value, _current_indent_level)
+                    chunks = self._iterencode_list(value, markers,
+                                                   _current_indent_level)
                 elif isinstance(value, dict):
-                    chunks = _iterencode_dict(value, _current_indent_level)
+                    chunks = self._iterencode_dict(value, markers,
+                                                   _current_indent_level)
                 else:
-                    chunks = _iterencode(value, _current_indent_level)
+                    chunks = self._iterencode(value, markers,
+                                              _current_indent_level)
                 for chunk in chunks:
                     yield chunk
         if newline_indent is not None:
             _current_indent_level -= 1
-            yield '\n' + (' ' * (_indent * _current_indent_level))
+            yield '\n' + (' ' * (self.indent * _current_indent_level))
         yield ']'
-        if markers is not None:
-            del markers[markerid]
+        self._remove_markers(markers, lst)
 
-    def _iterencode_dict(dct, _current_indent_level):
+    def _iterencode_dict(self, dct, markers, _current_indent_level):
         if not dct:
             yield '{}'
             return
-        if markers is not None:
-            markerid = id(dct)
-            if markerid in markers:
-                raise ValueError("Circular reference detected")
-            markers[markerid] = dct
+        self._mark_markers(markers, dct)
         yield '{'
-        if _indent is not None:
+        if self.indent is not None:
             _current_indent_level += 1
-            newline_indent = '\n' + (' ' * (_indent * _current_indent_level))
-            item_separator = _item_separator + newline_indent
+            newline_indent = '\n' + (' ' * (self.indent *
+                                            _current_indent_level))
+            item_separator = self.item_separator + newline_indent
             yield newline_indent
         else:
             newline_indent = None
-            item_separator = _item_separator
+            item_separator = self.item_separator
         first = True
-        if _sort_keys:
+        if self.sort_keys:
             items = sorted(dct.items(), key=lambda kv: kv[0])
         else:
             items = dct.iteritems()
@@ -361,7 +430,7 @@
             # JavaScript is weakly typed for these, so it makes sense to
             # also allow them.  Many encoders seem to do something like this.
             elif isinstance(key, float):
-                key = _floatstr(key)
+                key = self._floatstr(key)
             elif key is True:
                 key = 'true'
             elif key is False:
@@ -370,7 +439,7 @@
                 key = 'null'
             elif isinstance(key, (int, long)):
                 key = str(key)
-            elif _skipkeys:
+            elif self.skipkeys:
                 continue
             else:
                 raise TypeError("key " + repr(key) + " is not a string")
@@ -378,10 +447,10 @@
                 first = False
             else:
                 yield item_separator
-            yield _encoder(key)
-            yield _key_separator
+            yield '"' + self.encoder(key) + '"'
+            yield self.key_separator
             if isinstance(value, basestring):
-                yield _encoder(value)
+                yield '"' + self.encoder(value) + '"'
             elif value is None:
                 yield 'null'
             elif value is True:
@@ -391,26 +460,28 @@
             elif isinstance(value, (int, long)):
                 yield str(value)
             elif isinstance(value, float):
-                yield _floatstr(value)
+                yield self._floatstr(value)
             else:
                 if isinstance(value, (list, tuple)):
-                    chunks = _iterencode_list(value, _current_indent_level)
+                    chunks = self._iterencode_list(value, markers,
+                                                   _current_indent_level)
                 elif isinstance(value, dict):
-                    chunks = _iterencode_dict(value, _current_indent_level)
+                    chunks = self._iterencode_dict(value, markers,
+                                                   _current_indent_level)
                 else:
-                    chunks = _iterencode(value, _current_indent_level)
+                    chunks = self._iterencode(value, markers,
+                                              _current_indent_level)
                 for chunk in chunks:
                     yield chunk
         if newline_indent is not None:
             _current_indent_level -= 1
-            yield '\n' + (' ' * (_indent * _current_indent_level))
+            yield '\n' + (' ' * (self.indent * _current_indent_level))
         yield '}'
-        if markers is not None:
-            del markers[markerid]
+        self._remove_markers(markers, dct)
 
-    def _iterencode(o, _current_indent_level):
+    def _iterencode(self, o, markers, _current_indent_level):
         if isinstance(o, basestring):
-            yield _encoder(o)
+            yield '"' + self.encoder(o) + '"'
         elif o is None:
             yield 'null'
         elif o is True:
@@ -420,23 +491,19 @@
         elif isinstance(o, (int, long)):
             yield str(o)
         elif isinstance(o, float):
-            yield _floatstr(o)
+            yield self._floatstr(o)
         elif isinstance(o, (list, tuple)):
-            for chunk in _iterencode_list(o, _current_indent_level):
+            for chunk in self._iterencode_list(o, markers,
+                                               _current_indent_level):
                 yield chunk
         elif isinstance(o, dict):
-            for chunk in _iterencode_dict(o, _current_indent_level):
+            for chunk in self._iterencode_dict(o, markers,
+                                               _current_indent_level):
                 yield chunk
         else:
-            if markers is not None:
-                markerid = id(o)
-                if markerid in markers:
-                    raise ValueError("Circular reference detected")
-                markers[markerid] = o
-            o = _default(o)
-            for chunk in _iterencode(o, _current_indent_level):
+            self._mark_markers(markers, o)
+            obj = self.default(o)
+            for chunk in self._iterencode(obj, markers,
+                                          _current_indent_level):
                 yield chunk
-            if markers is not None:
-                del markers[markerid]
-
-    return _iterencode
+            self._remove_markers(markers, o)
diff --git a/lib-python/modified-2.7/json/tests/test_unicode.py b/lib-python/modified-2.7/json/tests/test_unicode.py
--- a/lib-python/modified-2.7/json/tests/test_unicode.py
+++ b/lib-python/modified-2.7/json/tests/test_unicode.py
@@ -80,3 +80,9 @@
         self.assertEqual(type(json.loads(u'["a"]')[0]), unicode)
         # Issue 10038.
         self.assertEqual(type(json.loads('"foo"')), unicode)
+
+    def test_encode_not_utf_8(self):
+        self.assertEqual(json.dumps('\xb1\xe6', encoding='iso8859-2'),
+                         '"\\u0105\\u0107"')
+        self.assertEqual(json.dumps(['\xb1\xe6'], encoding='iso8859-2'),
+                         '["\\u0105\\u0107"]')

From noreply at buildbot.pypy.org  Thu Oct 20 19:50:36 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 20 Oct 2011 19:50:36 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge default
Message-ID: <20111020175036.3FA1E820D7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r48277:39882f1dfd15
Date: 2011-10-20 19:50 +0200
http://bitbucket.org/pypy/pypy/changeset/39882f1dfd15/

Log:	merge default

diff --git a/lib_pypy/pyrepl/readline.py b/lib_pypy/pyrepl/readline.py
--- a/lib_pypy/pyrepl/readline.py
+++ b/lib_pypy/pyrepl/readline.py
@@ -395,9 +395,21 @@
     _wrapper.f_in = f_in
     _wrapper.f_out = f_out
 
-    if hasattr(sys, '__raw_input__'):    # PyPy
-        _old_raw_input = sys.__raw_input__
+    if '__pypy__' in sys.builtin_module_names:    # PyPy
+
+        def _old_raw_input(prompt=''):
+            # sys.__raw_input__() is only called when stdin and stdout are
+            # as expected and are ttys.  If it is the case, then get_reader()
+            # should not really fail in _wrapper.raw_input().  If it still
+            # does, then we will just cancel the redirection and call again
+            # the built-in raw_input().
+            try:
+                del sys.__raw_input__
+            except AttributeError:
+                pass
+            return raw_input(prompt)
         sys.__raw_input__ = _wrapper.raw_input
+
     else:
         # this is not really what readline.c does.  Better than nothing I guess
         import __builtin__
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -72,6 +72,7 @@
     del working_modules['fcntl']  # LOCK_NB not defined
     del working_modules["_minimal_curses"]
     del working_modules["termios"]
+    del working_modules["_multiprocessing"]   # depends on rctime
 
 
 
diff --git a/pypy/jit/metainterp/optimizeopt/rewrite.py b/pypy/jit/metainterp/optimizeopt/rewrite.py
--- a/pypy/jit/metainterp/optimizeopt/rewrite.py
+++ b/pypy/jit/metainterp/optimizeopt/rewrite.py
@@ -106,10 +106,9 @@
             self.make_equal_to(op.result, v1)
         else:
             self.emit_operation(op)
-
-        # Synthesize the reverse ops for optimize_default to reuse
-        self.pure(rop.INT_ADD, [op.result, op.getarg(1)], op.getarg(0))
-        self.pure(rop.INT_SUB, [op.getarg(0), op.result], op.getarg(1))
+            # Synthesize the reverse ops for optimize_default to reuse
+            self.pure(rop.INT_ADD, [op.result, op.getarg(1)], op.getarg(0))
+            self.pure(rop.INT_SUB, [op.getarg(0), op.result], op.getarg(1))
 
     def optimize_INT_ADD(self, op):
         v1 = self.getvalue(op.getarg(0))
@@ -122,10 +121,9 @@
             self.make_equal_to(op.result, v1)
         else:
             self.emit_operation(op)
-
-        # Synthesize the reverse op for optimize_default to reuse
-        self.pure(rop.INT_SUB, [op.result, op.getarg(1)], op.getarg(0))
-        self.pure(rop.INT_SUB, [op.result, op.getarg(0)], op.getarg(1))
+            # Synthesize the reverse op for optimize_default to reuse
+            self.pure(rop.INT_SUB, [op.result, op.getarg(1)], op.getarg(0))
+            self.pure(rop.INT_SUB, [op.result, op.getarg(0)], op.getarg(1))
 
     def optimize_INT_MUL(self, op):
         v1 = self.getvalue(op.getarg(0))
@@ -141,13 +139,13 @@
             self.make_constant_int(op.result, 0)
         else:
             for lhs, rhs in [(v1, v2), (v2, v1)]:
-                # x & (x -1) == 0 is a quick test for power of 2
-                if (lhs.is_constant() and
-                    (lhs.box.getint() & (lhs.box.getint() - 1)) == 0):
-                    new_rhs = ConstInt(highest_bit(lhs.box.getint()))
-                    op = op.copy_and_change(rop.INT_LSHIFT, args=[rhs.box, new_rhs])
-                    break
-
+                if lhs.is_constant():
+                    x = lhs.box.getint()
+                    # x & (x - 1) == 0 is a quick test for power of 2
+                    if x & (x - 1) == 0:
+                        new_rhs = ConstInt(highest_bit(lhs.box.getint()))
+                        op = op.copy_and_change(rop.INT_LSHIFT, args=[rhs.box, new_rhs])
+                        break
             self.emit_operation(op)
 
     def optimize_UINT_FLOORDIV(self, op):
@@ -462,6 +460,14 @@
         self.optimizer.opaque_pointers[value] = True
         self.make_equal_to(op.result, value)
 
+    def optimize_CAST_PTR_TO_INT(self, op):
+        self.pure(rop.CAST_INT_TO_PTR, [op.result], op.getarg(0))
+        self.emit_operation(op)
+
+    def optimize_CAST_INT_TO_PTR(self, op):
+        self.pure(rop.CAST_PTR_TO_INT, [op.result], op.getarg(0))
+        self.emit_operation(op)
+
 dispatch_opt = make_dispatcher_method(OptRewrite, 'optimize_',
         default=OptRewrite.emit_operation)
 optimize_guards = _findall(OptRewrite, 'optimize_', 'GUARD')
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -234,6 +234,30 @@
             """ % expected_value
             self.optimize_loop(ops, expected)
 
+    def test_reverse_of_cast(self):
+        ops = """
+        [i0]
+        p0 = cast_int_to_ptr(i0)
+        i1 = cast_ptr_to_int(p0)
+        jump(i1)
+        """
+        expected = """
+        [i0]
+        jump(i0)
+        """
+        self.optimize_loop(ops, expected)
+        ops = """
+        [p0]
+        i1 = cast_ptr_to_int(p0)
+        p1 = cast_int_to_ptr(i1)
+        jump(p1)
+        """
+        expected = """
+        [p0]
+        jump(p0)
+        """
+        self.optimize_loop(ops, expected)
+
     # ----------
 
     def test_remove_guard_class_1(self):
diff --git a/pypy/jit/metainterp/quasiimmut.py b/pypy/jit/metainterp/quasiimmut.py
--- a/pypy/jit/metainterp/quasiimmut.py
+++ b/pypy/jit/metainterp/quasiimmut.py
@@ -74,8 +74,10 @@
         self.looptokens_wrefs.append(wref_looptoken)
 
     def compress_looptokens_list(self):
-        self.looptokens_wref = [wref for wref in self.looptokens_wrefs
-                                     if wref() is not None]
+        self.looptokens_wrefs = [wref for wref in self.looptokens_wrefs
+                                      if wref() is not None]
+        # NB. we must keep around the looptoken_wrefs that are
+        # already invalidated; see below
         self.compress_limit = (len(self.looptokens_wrefs) + 15) * 2
 
     def invalidate(self):
@@ -89,6 +91,11 @@
             if looptoken is not None:
                 looptoken.invalidated = True
                 self.cpu.invalidate_loop(looptoken)
+                # NB. we must call cpu.invalidate_loop() even if
+                # looptoken.invalidated was already set to True.
+                # It's possible to invalidate several times the
+                # same looptoken; see comments in jit.backend.model
+                # in invalidate_loop().
                 if not we_are_translated():
                     self.cpu.stats.invalidated_token_numbers.add(
                         looptoken.number)
diff --git a/pypy/module/__builtin__/__init__.py b/pypy/module/__builtin__/__init__.py
--- a/pypy/module/__builtin__/__init__.py
+++ b/pypy/module/__builtin__/__init__.py
@@ -23,6 +23,7 @@
         'map'           : 'app_functional.map',
         'reduce'        : 'app_functional.reduce',
         'filter'        : 'app_functional.filter',
+        'zip'           : 'app_functional.zip',
         'vars'          : 'app_inspect.vars',
         'dir'           : 'app_inspect.dir',
 
@@ -89,7 +90,6 @@
         'enumerate'     : 'functional.W_Enumerate',
         'min'           : 'functional.min',
         'max'           : 'functional.max',
-        'zip'           : 'functional.zip',
         'reversed'      : 'functional.reversed',
         'super'         : 'descriptor.W_Super',
         'staticmethod'  : 'descriptor.StaticMethod',
diff --git a/pypy/module/__builtin__/app_functional.py b/pypy/module/__builtin__/app_functional.py
--- a/pypy/module/__builtin__/app_functional.py
+++ b/pypy/module/__builtin__/app_functional.py
@@ -162,4 +162,21 @@
         item = seq[i]
         if func(item):
             result.append(item)
-    return tuple(result)
\ No newline at end of file
+    return tuple(result)
+
+def zip(*sequences):
+    """zip(seq1 [, seq2 [...]]) -> [(seq1[0], seq2[0] ...), (...)]
+
+Return a list of tuples, where each tuple contains the i-th element
+from each of the argument sequences.  The returned list is truncated
+in length to the length of the shortest argument sequence."""
+    if not sequences:
+        return []
+    result = []
+    iterators = [iter(seq) for seq in sequences]
+    while True:
+        try:
+            items = [next(it) for it in iterators]
+        except StopIteration:
+            return result
+        result.append(tuple(items))
diff --git a/pypy/module/__builtin__/app_io.py b/pypy/module/__builtin__/app_io.py
--- a/pypy/module/__builtin__/app_io.py
+++ b/pypy/module/__builtin__/app_io.py
@@ -27,7 +27,20 @@
     co = compile(source.rstrip()+"\n", filename, 'exec')
     exec co in glob, loc
 
-def raw_input(prompt=None):
+def _write_prompt(stdout, prompt):
+    print >> stdout, prompt,
+    try:
+        flush = stdout.flush
+    except AttributeError:
+        pass
+    else:
+        flush()
+    try:
+        stdout.softspace = 0
+    except (AttributeError, TypeError):
+        pass
+
+def raw_input(prompt=''):
     """raw_input([prompt]) -> string
 
 Read a string from standard input.  The trailing newline is stripped.
@@ -47,18 +60,10 @@
     if (hasattr(sys, '__raw_input__') and
         isinstance(stdin, file)  and stdin.fileno() == 0 and stdin.isatty() and
         isinstance(stdout, file) and stdout.fileno() == 1):
-        if prompt is None:
-            prompt = ''
-        return sys.__raw_input__(prompt)
+        _write_prompt(stdout, '')
+        return sys.__raw_input__(str(prompt))
 
-    if prompt is not None:
-        stdout.write(prompt)
-        try:
-            flush = stdout.flush
-        except AttributeError:
-            pass
-        else:
-            flush()
+    _write_prompt(stdout, prompt)
     line = stdin.readline()
     if not line:    # inputting an empty line gives line == '\n'
         raise EOFError
diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py
--- a/pypy/module/__builtin__/functional.py
+++ b/pypy/module/__builtin__/functional.py
@@ -201,27 +201,6 @@
     """
     return min_max(space, __args__, "min")
 
- at unwrap_spec(sequences_w="args_w")
-def zip(space, sequences_w):
-    """Return a list of tuples, where the nth tuple contains every nth item of
-    each collection.
-
-    If the collections have different lengths, zip returns a list as long as the
-    shortest collection, ignoring the trailing items in the other collections.
-    """
-    if not sequences_w:
-        return space.newlist([])
-    result_w = []
-    iterators_w = [space.iter(w_seq) for w_seq in sequences_w]
-    while True:
-        try:
-            items_w = [space.next(w_it) for w_it in iterators_w]
-        except OperationError, e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            return space.newlist(result_w)
-        result_w.append(space.newtuple(items_w))
-
 class W_Enumerate(Wrappable):
 
     def __init__(self, w_iter, w_start):
diff --git a/pypy/module/__builtin__/test/test_rawinput.py b/pypy/module/__builtin__/test/test_rawinput.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/__builtin__/test/test_rawinput.py
@@ -0,0 +1,77 @@
+import autopath
+
+
+class AppTestRawInput():
+
+    def test_raw_input(self):
+        import sys, StringIO
+        for prompt, expected in [("def:", "abc/ def:/ghi\n"),
+                                 ("", "abc/ /ghi\n"),
+                                 (42, "abc/ 42/ghi\n"),
+                                 (None, "abc/ None/ghi\n"),
+                                 (Ellipsis, "abc/ /ghi\n")]:
+            save = sys.stdin, sys.stdout
+            try:
+                sys.stdin = StringIO.StringIO("foo\nbar\n")
+                out = sys.stdout = StringIO.StringIO()
+                print "abc",    # softspace = 1
+                out.write('/')
+                if prompt is Ellipsis:
+                    got = raw_input()
+                else:
+                    got = raw_input(prompt)
+                out.write('/')
+                print "ghi"
+            finally:
+                sys.stdin, sys.stdout = save
+            assert out.getvalue() == expected
+            assert got == "foo"
+
+    def test_softspace(self):
+        import sys
+        import StringIO
+        fin = StringIO.StringIO()
+        fout = StringIO.StringIO()
+
+        fin.write("Coconuts\n")
+        fin.seek(0)
+
+        sys_stdin_orig = sys.stdin
+        sys_stdout_orig = sys.stdout
+
+        sys.stdin = fin
+        sys.stdout = fout
+
+        print "test",
+        raw_input("test")
+
+        sys.stdin = sys_stdin_orig
+        sys.stdout = sys_stdout_orig
+
+        fout.seek(0)
+        assert fout.read() == "test test"
+
+    def test_softspace_carryover(self):
+        import sys
+        import StringIO
+        fin = StringIO.StringIO()
+        fout = StringIO.StringIO()
+
+        fin.write("Coconuts\n")
+        fin.seek(0)
+
+        sys_stdin_orig = sys.stdin
+        sys_stdout_orig = sys.stdout
+
+        sys.stdin = fin
+        sys.stdout = fout
+
+        print "test",
+        raw_input("test")
+        print "test",
+
+        sys.stdin = sys_stdin_orig
+        sys.stdout = sys_stdout_orig
+
+        fout.seek(0)
+        assert fout.read() == "test testtest"
diff --git a/pypy/module/pyexpat/__init__.py b/pypy/module/pyexpat/__init__.py
--- a/pypy/module/pyexpat/__init__.py
+++ b/pypy/module/pyexpat/__init__.py
@@ -4,12 +4,8 @@
 
 class ErrorsModule(MixedModule):
     "Definition of pyexpat.errors module."
-
-    appleveldefs = {
-        }
-
-    interpleveldefs = {
-        }
+    appleveldefs = {}
+    interpleveldefs = {}
 
     def setup_after_space_initialization(self):
         from pypy.module.pyexpat import interp_pyexpat
@@ -18,6 +14,18 @@
                     interp_pyexpat.ErrorString(self.space,
                     getattr(interp_pyexpat, name)))
 
+class ModelModule(MixedModule):
+    "Definition of pyexpat.model module."
+    appleveldefs = {}
+    interpleveldefs = {}
+
+    def setup_after_space_initialization(self):
+        from pypy.module.pyexpat import interp_pyexpat
+        space = self.space
+        for name in interp_pyexpat.xml_model_list:
+            value = getattr(interp_pyexpat, name)
+            space.setattr(self, space.wrap(name), space.wrap(value))
+
 class Module(MixedModule):
     "Python wrapper for Expat parser."
 
@@ -39,6 +47,7 @@
 
     submodules = {
         'errors': ErrorsModule,
+        'model':  ModelModule,
     }
 
     for name in ['XML_PARAM_ENTITY_PARSING_NEVER',
diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py
--- a/pypy/module/pyexpat/interp_pyexpat.py
+++ b/pypy/module/pyexpat/interp_pyexpat.py
@@ -76,6 +76,18 @@
     "XML_ERROR_FINISHED",
     "XML_ERROR_SUSPEND_PE",
     ]
+xml_model_list = [
+    "XML_CTYPE_EMPTY",
+    "XML_CTYPE_ANY",
+    "XML_CTYPE_MIXED",
+    "XML_CTYPE_NAME",
+    "XML_CTYPE_CHOICE",
+    "XML_CTYPE_SEQ",
+    "XML_CQUANT_NONE",
+    "XML_CQUANT_OPT",
+    "XML_CQUANT_REP",
+    "XML_CQUANT_PLUS",
+    ]
 
 class CConfigure:
     _compilation_info_ = eci
@@ -104,6 +116,8 @@
 
     for name in xml_error_list:
         locals()[name] = rffi_platform.ConstantInteger(name)
+    for name in xml_model_list:
+        locals()[name] = rffi_platform.ConstantInteger(name)
 
 for k, v in rffi_platform.configure(CConfigure).items():
     globals()[k] = v
diff --git a/pypy/module/pyexpat/test/test_parser.py b/pypy/module/pyexpat/test/test_parser.py
--- a/pypy/module/pyexpat/test/test_parser.py
+++ b/pypy/module/pyexpat/test/test_parser.py
@@ -131,3 +131,7 @@
                'encoding specified in XML declaration is incorrect')
         assert (pyexpat.errors.XML_ERROR_XML_DECL ==
                 'XML declaration not well-formed')
+
+    def test_model(self):
+        import pyexpat
+        assert isinstance(pyexpat.model.XML_CTYPE_EMPTY, int)
diff --git a/pypy/objspace/std/strutil.py b/pypy/objspace/std/strutil.py
--- a/pypy/objspace/std/strutil.py
+++ b/pypy/objspace/std/strutil.py
@@ -35,7 +35,7 @@
 
     def error(self):
         raise ParseStringError("invalid literal for %s() with base %d: '%s'" %
-                               (self.fname, self.base, self.literal))
+                               (self.fname, self.original_base, self.literal))
 
     def __init__(self, s, literal, base, fname):
         self.literal = literal
@@ -47,7 +47,8 @@
         elif s.startswith('+'):
             s = strip_spaces(s[1:])
         self.sign = sign
-        
+        self.original_base = base
+
         if base == 0:
             if s.startswith('0x') or s.startswith('0X'):
                 base = 16
diff --git a/pypy/objspace/std/test/test_strutil.py b/pypy/objspace/std/test/test_strutil.py
--- a/pypy/objspace/std/test/test_strutil.py
+++ b/pypy/objspace/std/test/test_strutil.py
@@ -89,6 +89,8 @@
 
         exc = raises(ParseStringError, string_to_int, '')
         assert exc.value.msg == "invalid literal for int() with base 10: ''"
+        exc = raises(ParseStringError, string_to_int, '', 0)
+        assert exc.value.msg == "invalid literal for int() with base 0: ''"
 
     def test_string_to_int_overflow(self):
         import sys
diff --git a/pypy/rlib/_rsocket_rffi.py b/pypy/rlib/_rsocket_rffi.py
--- a/pypy/rlib/_rsocket_rffi.py
+++ b/pypy/rlib/_rsocket_rffi.py
@@ -16,6 +16,7 @@
 _MINGW = target_platform.name == "mingw32"
 _SOLARIS = sys.platform == "sunos5"
 _MACOSX = sys.platform == "darwin"
+_HAS_AF_PACKET = sys.platform.startswith('linux')   # only Linux for now
 
 if _POSIX:
     includes = ('sys/types.h',
@@ -34,11 +35,12 @@
                 'stdint.h', 
                 'errno.h',
                 )
+    if _HAS_AF_PACKET:
+        includes += ('netpacket/packet.h',
+                     'sys/ioctl.h',
+                     'net/if.h')
 
-    cond_includes = [('AF_NETLINK', 'linux/netlink.h'),
-                     ('AF_PACKET', 'netpacket/packet.h'),
-                     ('AF_PACKET', 'sys/ioctl.h'),
-                     ('AF_PACKET', 'net/if.h')]
+    cond_includes = [('AF_NETLINK', 'linux/netlink.h')]
     
     libraries = ()
     calling_conv = 'c'
@@ -320,18 +322,18 @@
                                              ('events', rffi.SHORT),
                                              ('revents', rffi.SHORT)])
 
-    CConfig.sockaddr_ll = platform.Struct('struct sockaddr_ll',
+    if _HAS_AF_PACKET:
+        CConfig.sockaddr_ll = platform.Struct('struct sockaddr_ll',
                               [('sll_ifindex', rffi.INT),
                                ('sll_protocol', rffi.INT),
                                ('sll_pkttype', rffi.INT),
                                ('sll_hatype', rffi.INT),
                                ('sll_addr', rffi.CFixedArray(rffi.CHAR, 8)),
-                               ('sll_halen', rffi.INT)],
-                              ifdef='AF_PACKET')
+                               ('sll_halen', rffi.INT)])
 
-    CConfig.ifreq = platform.Struct('struct ifreq', [('ifr_ifindex', rffi.INT),
-                                 ('ifr_name', rffi.CFixedArray(rffi.CHAR, 8))],
-                                    ifdef='AF_PACKET')
+        CConfig.ifreq = platform.Struct('struct ifreq',
+                                [('ifr_ifindex', rffi.INT),
+                                 ('ifr_name', rffi.CFixedArray(rffi.CHAR, 8))])
 
 if _WIN32:
     CConfig.WSAEVENT = platform.SimpleType('WSAEVENT', rffi.VOIDP)
@@ -386,6 +388,8 @@
         constants[name] = value
     else:
         constants[name] = default
+if not _HAS_AF_PACKET and 'AF_PACKET' in constants:
+    del constants['AF_PACKET']
 
 constants['has_ipv6'] = True # This is a configuration option in CPython
 for name, value in constants.items():
@@ -439,21 +443,14 @@
 if _POSIX:
     nfds_t = cConfig.nfds_t
     pollfd = cConfig.pollfd
-    if cConfig.sockaddr_ll is not None:
+    if _HAS_AF_PACKET:
         sockaddr_ll = cConfig.sockaddr_ll
-    ifreq = cConfig.ifreq
+        ifreq = cConfig.ifreq
 if WIN32:
     WSAEVENT = cConfig.WSAEVENT
     WSANETWORKEVENTS = cConfig.WSANETWORKEVENTS
 timeval = cConfig.timeval
 
-#if _POSIX:
-#    includes = list(includes)
-#    for _name, _header in cond_includes:
-#        if getattr(cConfig, _name) is not None:
-#            includes.append(_header)
-#    eci = ExternalCompilationInfo(includes=includes, libraries=libraries,
-#                                  separate_module_sources=sources)
 
 def external(name, args, result, **kwds):
     return rffi.llexternal(name, args, result, compilation_info=eci,
@@ -544,7 +541,7 @@
     socketpair_t = rffi.CArray(socketfd_type)
     socketpair = external('socketpair', [rffi.INT, rffi.INT, rffi.INT,
                           lltype.Ptr(socketpair_t)], rffi.INT)
-    if ifreq is not None:
+    if _HAS_AF_PACKET:
         ioctl = external('ioctl', [socketfd_type, rffi.INT, lltype.Ptr(ifreq)],
                          rffi.INT)
 
diff --git a/pypy/rlib/rsocket.py b/pypy/rlib/rsocket.py
--- a/pypy/rlib/rsocket.py
+++ b/pypy/rlib/rsocket.py
@@ -8,6 +8,7 @@
 # Known missing features:
 #
 #   - address families other than AF_INET, AF_INET6, AF_UNIX, AF_PACKET
+#   - AF_PACKET is only supported on Linux
 #   - methods makefile(),
 #   - SSL
 #
diff --git a/pypy/rpython/lltypesystem/rbuilder.py b/pypy/rpython/lltypesystem/rbuilder.py
--- a/pypy/rpython/lltypesystem/rbuilder.py
+++ b/pypy/rpython/lltypesystem/rbuilder.py
@@ -29,7 +29,7 @@
         except OverflowError:
             raise MemoryError
         newbuf = mallocfn(new_allocated)
-        copycontentsfn(ll_builder.buf, newbuf, 0, 0, ll_builder.allocated)
+        copycontentsfn(ll_builder.buf, newbuf, 0, 0, ll_builder.used)
         ll_builder.buf = newbuf
         ll_builder.allocated = new_allocated
     return func_with_new_name(stringbuilder_grow, name)
@@ -56,7 +56,7 @@
 class BaseStringBuilderRepr(AbstractStringBuilderRepr):
     def empty(self):
         return nullptr(self.lowleveltype.TO)
-    
+
     @classmethod
     def ll_new(cls, init_size):
         if init_size < 0 or init_size > MAX:
diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -1292,10 +1292,12 @@
         # if a prebuilt GcStruct contains a pointer to a young object,
         # then the write_barrier must have ensured that the prebuilt
         # GcStruct is in the list self.old_objects_pointing_to_young.
+        debug_start("gc-minor-walkroots")
         self.root_walker.walk_roots(
             MiniMarkGC._trace_drag_out1,  # stack roots
             MiniMarkGC._trace_drag_out1,  # static in prebuilt non-gc
             None)                         # static in prebuilt gc
+        debug_stop("gc-minor-walkroots")
 
     def collect_cardrefs_to_nursery(self):
         size_gc_header = self.gcheaderbuilder.size_gc_header
diff --git a/pypy/translator/platform/posix.py b/pypy/translator/platform/posix.py
--- a/pypy/translator/platform/posix.py
+++ b/pypy/translator/platform/posix.py
@@ -157,7 +157,7 @@
 
         rules = [
             ('all', '$(DEFAULT_TARGET)', []),
-            ('$(TARGET)', '$(OBJECTS)', '$(CC_LINK) $(LDFLAGS) $(LDFLAGSEXTRA) -o $@ $(OBJECTS) $(LIBDIRS) $(LIBS) $(LINKFILES)'),
+            ('$(TARGET)', '$(OBJECTS)', '$(CC_LINK) $(LDFLAGSEXTRA) -o $@ $(OBJECTS) $(LIBDIRS) $(LIBS) $(LINKFILES) $(LDFLAGS)'),
             ('%.o', '%.c', '$(CC) $(CFLAGS) $(CFLAGSEXTRA) -o $@ -c $< $(INCLUDEDIRS)'),
             ]
 

From noreply at buildbot.pypy.org  Thu Oct 20 20:00:09 2011
From: noreply at buildbot.pypy.org (justinpeel)
Date: Thu, 20 Oct 2011 20:00:09 +0200 (CEST)
Subject: [pypy-commit] pypy rgc-mem-pressure: fix hashlib's copy() leak
Message-ID: <20111020180009.45801820D7@wyvern.cs.uni-duesseldorf.de>

Author: Justin Peel <notmuchtotell at gmail.com>
Branch: rgc-mem-pressure
Changeset: r48278:38375b2c1ddb
Date: 2011-10-20 11:57 -0600
http://bitbucket.org/pypy/pypy/changeset/38375b2c1ddb/

Log:	fix hashlib's copy() leak

diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py
--- a/pypy/module/_hashlib/interp_hashlib.py
+++ b/pypy/module/_hashlib/interp_hashlib.py
@@ -30,15 +30,17 @@
         # and use a custom lock only when needed.
         self.lock = Lock(space)
 
-        digest = ropenssl.EVP_get_digestbyname(name)
-        if not digest:
-            raise OperationError(space.w_ValueError,
-                                 space.wrap("unknown hash function"))
         ctx = lltype.malloc(ropenssl.EVP_MD_CTX.TO, flavor='raw')
-        ropenssl.EVP_DigestInit(ctx, digest)
         rgc.add_memory_pressure(HASH_MALLOC_SIZE + self._digest_size())
         self.ctx = ctx
 
+    def initdigest(self):
+        digest = ropenssl.EVP_get_digestbyname(self.name)
+        if not digest:
+            raise OperationError(space.w_Value,
+                                 space.wrap("unknown hash function"))
+        ropenssl.EVP_DigestInit(self.ctx, digest)
+
     def __del__(self):
         # self.lock.free()
         if self.ctx:
@@ -139,6 +141,7 @@
 @unwrap_spec(name=str, string='bufferstr')
 def new(space, name, string=''):
     w_hash = W_Hash(space, name)
+    w_hash.initdigest()
     w_hash.update(space, string)
     return space.wrap(w_hash)
 

From noreply at buildbot.pypy.org  Thu Oct 20 20:38:55 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 20 Oct 2011 20:38:55 +0200 (CEST)
Subject: [pypy-commit] pypy inline-dict-ops: just always keepalive parent.
 Passes tests and seems that the original reason
Message-ID: <20111020183855.C3C30820D7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: inline-dict-ops
Changeset: r48279:7e7d1b1bd768
Date: 2011-10-20 20:38 +0200
http://bitbucket.org/pypy/pypy/changeset/7e7d1b1bd768/

Log:	just always keepalive parent. Passes tests and seems that the
	original reason for a convoluted if disappeared (as per skipped
	tests)

diff --git a/pypy/rpython/lltypesystem/lltype.py b/pypy/rpython/lltypesystem/lltype.py
--- a/pypy/rpython/lltypesystem/lltype.py
+++ b/pypy/rpython/lltypesystem/lltype.py
@@ -1517,12 +1517,7 @@
         self._wrparent = weakref.ref(parent)
         self._parent_type = typeOf(parent)
         self._parent_index = parentindex
-        if (isinstance(self._parent_type, Struct)
-            and self._parent_type._names
-            and parentindex in (self._parent_type._names[0], 0)
-            and self._TYPE._gckind == typeOf(parent)._gckind):
-            # keep strong reference to parent, we share the same allocation
-            self._keepparent = parent 
+        self._keepparent = parent
 
     def _parentstructure(self, check=True):
         if self._wrparent is not None:

From noreply at buildbot.pypy.org  Thu Oct 20 20:40:49 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Thu, 20 Oct 2011 20:40:49 +0200 (CEST)
Subject: [pypy-commit] pypy inline-dict-ops: merged default in.
Message-ID: <20111020184049.04AE4820D7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: inline-dict-ops
Changeset: r48280:129cc57adb40
Date: 2011-10-20 14:40 -0400
http://bitbucket.org/pypy/pypy/changeset/129cc57adb40/

Log:	merged default in.

diff --git a/lib-python/modified-2.7/json/encoder.py b/lib-python/modified-2.7/json/encoder.py
--- a/lib-python/modified-2.7/json/encoder.py
+++ b/lib-python/modified-2.7/json/encoder.py
@@ -2,14 +2,7 @@
 """
 import re
 
-try:
-    from _json import encode_basestring_ascii as c_encode_basestring_ascii
-except ImportError:
-    c_encode_basestring_ascii = None
-try:
-    from _json import make_encoder as c_make_encoder
-except ImportError:
-    c_make_encoder = None
+from __pypy__.builders import StringBuilder, UnicodeBuilder
 
 ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]')
 ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])')
@@ -24,8 +17,7 @@
     '\t': '\\t',
 }
 for i in range(0x20):
-    ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i))
-    #ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,))
+    ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,))
 
 # Assume this produces an infinity on all machines (probably not guaranteed)
 INFINITY = float('1e66666')
@@ -37,10 +29,9 @@
     """
     def replace(match):
         return ESCAPE_DCT[match.group(0)]
-    return '"' + ESCAPE.sub(replace, s) + '"'
+    return ESCAPE.sub(replace, s)
 
-
-def py_encode_basestring_ascii(s):
+def encode_basestring_ascii(s):
     """Return an ASCII-only JSON representation of a Python string
 
     """
@@ -53,20 +44,18 @@
         except KeyError:
             n = ord(s)
             if n < 0x10000:
-                return '\\u{0:04x}'.format(n)
-                #return '\\u%04x' % (n,)
+                return '\\u%04x' % (n,)
             else:
                 # surrogate pair
                 n -= 0x10000
                 s1 = 0xd800 | ((n >> 10) & 0x3ff)
                 s2 = 0xdc00 | (n & 0x3ff)
-                return '\\u{0:04x}\\u{1:04x}'.format(s1, s2)
-                #return '\\u%04x\\u%04x' % (s1, s2)
-    return '"' + str(ESCAPE_ASCII.sub(replace, s)) + '"'
-
-
-encode_basestring_ascii = (
-    c_encode_basestring_ascii or py_encode_basestring_ascii)
+                return '\\u%04x\\u%04x' % (s1, s2)
+    if ESCAPE_ASCII.search(s):
+        return str(ESCAPE_ASCII.sub(replace, s))
+    return s
+py_encode_basestring_ascii = lambda s: '"' + encode_basestring_ascii(s) + '"'
+c_encode_basestring_ascii = None
 
 class JSONEncoder(object):
     """Extensible JSON <http://json.org> encoder for Python data structures.
@@ -147,6 +136,17 @@
 
         self.skipkeys = skipkeys
         self.ensure_ascii = ensure_ascii
+        if ensure_ascii:
+            self.encoder = encode_basestring_ascii
+        else:
+            self.encoder = encode_basestring
+        if encoding != 'utf-8':
+            orig_encoder = self.encoder
+            def encoder(o):
+                if isinstance(o, str):
+                    o = o.decode(encoding)
+                return orig_encoder(o)
+            self.encoder = encoder
         self.check_circular = check_circular
         self.allow_nan = allow_nan
         self.sort_keys = sort_keys
@@ -184,24 +184,126 @@
         '{"foo": ["bar", "baz"]}'
 
         """
-        # This is for extremely simple cases and benchmarks.
+        if self.check_circular:
+            markers = {}
+        else:
+            markers = None
+        if self.ensure_ascii:
+            builder = StringBuilder()
+        else:
+            builder = UnicodeBuilder()
+        self._encode(o, markers, builder, 0)
+        return builder.build()
+
+    def _emit_indent(self, builder, _current_indent_level):
+        if self.indent is not None:
+            _current_indent_level += 1
+            newline_indent = '\n' + (' ' * (self.indent *
+                                            _current_indent_level))
+            separator = self.item_separator + newline_indent
+            builder.append(newline_indent)
+        else:
+            separator = self.item_separator
+        return separator, _current_indent_level
+
+    def _emit_unindent(self, builder, _current_indent_level):
+        if self.indent is not None:
+            builder.append('\n')
+            builder.append(' ' * (self.indent * (_current_indent_level - 1)))
+
+    def _encode(self, o, markers, builder, _current_indent_level):
         if isinstance(o, basestring):
-            if isinstance(o, str):
-                _encoding = self.encoding
-                if (_encoding is not None
-                        and not (_encoding == 'utf-8')):
-                    o = o.decode(_encoding)
-            if self.ensure_ascii:
-                return encode_basestring_ascii(o)
+            builder.append('"')
+            builder.append(self.encoder(o))
+            builder.append('"')
+        elif o is None:
+            builder.append('null')
+        elif o is True:
+            builder.append('true')
+        elif o is False:
+            builder.append('false')
+        elif isinstance(o, (int, long)):
+            builder.append(str(o))
+        elif isinstance(o, float):
+            builder.append(self._floatstr(o))
+        elif isinstance(o, (list, tuple)):
+            if not o:
+                builder.append('[]')
+                return
+            self._encode_list(o, markers, builder, _current_indent_level)
+        elif isinstance(o, dict):
+            if not o:
+                builder.append('{}')
+                return
+            self._encode_dict(o, markers, builder, _current_indent_level)
+        else:
+            self._mark_markers(markers, o)
+            res = self.default(o)
+            self._encode(res, markers, builder, _current_indent_level)
+            self._remove_markers(markers, o)
+            return res
+
+    def _encode_list(self, l, markers, builder, _current_indent_level):
+        self._mark_markers(markers, l)
+        builder.append('[')
+        first = True
+        separator, _current_indent_level = self._emit_indent(builder,
+                                                      _current_indent_level)
+        for elem in l:
+            if first:
+                first = False
             else:
-                return encode_basestring(o)
-        # This doesn't pass the iterator directly to ''.join() because the
-        # exceptions aren't as detailed.  The list call should be roughly
-        # equivalent to the PySequence_Fast that ''.join() would do.
-        chunks = self.iterencode(o, _one_shot=True)
-        if not isinstance(chunks, (list, tuple)):
-            chunks = list(chunks)
-        return ''.join(chunks)
+                builder.append(separator)
+            self._encode(elem, markers, builder, _current_indent_level)
+            del elem # XXX grumble
+        self._emit_unindent(builder, _current_indent_level)
+        builder.append(']')
+        self._remove_markers(markers, l)
+
+    def _encode_dict(self, d, markers, builder, _current_indent_level):
+        self._mark_markers(markers, d)
+        first = True
+        builder.append('{')
+        separator, _current_indent_level = self._emit_indent(builder,
+                                                         _current_indent_level)
+        if self.sort_keys:
+            items = sorted(d.items(), key=lambda kv: kv[0])
+        else:
+            items = d.iteritems()
+
+        for key, v in items:
+            if first:
+                first = False
+            else:
+                builder.append(separator)
+            if isinstance(key, basestring):
+                pass
+            # JavaScript is weakly typed for these, so it makes sense to
+            # also allow them.  Many encoders seem to do something like this.
+            elif isinstance(key, float):
+                key = self._floatstr(key)
+            elif key is True:
+                key = 'true'
+            elif key is False:
+                key = 'false'
+            elif key is None:
+                key = 'null'
+            elif isinstance(key, (int, long)):
+                key = str(key)
+            elif self.skipkeys:
+                continue
+            else:
+                raise TypeError("key " + repr(key) + " is not a string")
+            builder.append('"')
+            builder.append(self.encoder(key))
+            builder.append('"')
+            builder.append(self.key_separator)
+            self._encode(v, markers, builder, _current_indent_level)
+            del key
+            del v # XXX grumble
+        self._emit_unindent(builder, _current_indent_level)
+        builder.append('}')
+        self._remove_markers(markers, d)
 
     def iterencode(self, o, _one_shot=False):
         """Encode the given object and yield each string
@@ -217,86 +319,54 @@
             markers = {}
         else:
             markers = None
-        if self.ensure_ascii:
-            _encoder = encode_basestring_ascii
+        return self._iterencode(o, markers, 0)
+
+    def _floatstr(self, o):
+        # Check for specials.  Note that this type of test is processor
+        # and/or platform-specific, so do tests which don't depend on the
+        # internals.
+
+        if o != o:
+            text = 'NaN'
+        elif o == INFINITY:
+            text = 'Infinity'
+        elif o == -INFINITY:
+            text = '-Infinity'
         else:
-            _encoder = encode_basestring
-        if self.encoding != 'utf-8':
-            def _encoder(o, _orig_encoder=_encoder, _encoding=self.encoding):
-                if isinstance(o, str):
-                    o = o.decode(_encoding)
-                return _orig_encoder(o)
+            return FLOAT_REPR(o)
 
-        def floatstr(o, allow_nan=self.allow_nan,
-                _repr=FLOAT_REPR, _inf=INFINITY, _neginf=-INFINITY):
-            # Check for specials.  Note that this type of test is processor
-            # and/or platform-specific, so do tests which don't depend on the
-            # internals.
+        if not self.allow_nan:
+            raise ValueError(
+                "Out of range float values are not JSON compliant: " +
+                repr(o))
 
-            if o != o:
-                text = 'NaN'
-            elif o == _inf:
-                text = 'Infinity'
-            elif o == _neginf:
-                text = '-Infinity'
-            else:
-                return _repr(o)
+        return text
 
-            if not allow_nan:
-                raise ValueError(
-                    "Out of range float values are not JSON compliant: " +
-                    repr(o))
+    def _mark_markers(self, markers, o):
+        if markers is not None:
+            if id(o) in markers:
+                raise ValueError("Circular reference detected")
+            markers[id(o)] = None
 
-            return text
+    def _remove_markers(self, markers, o):
+        if markers is not None:
+            del markers[id(o)]
 
-
-        if (_one_shot and c_make_encoder is not None
-                and not self.indent and not self.sort_keys):
-            _iterencode = c_make_encoder(
-                markers, self.default, _encoder, self.indent,
-                self.key_separator, self.item_separator, self.sort_keys,
-                self.skipkeys, self.allow_nan)
-        else:
-            _iterencode = _make_iterencode(
-                markers, self.default, _encoder, self.indent, floatstr,
-                self.key_separator, self.item_separator, self.sort_keys,
-                self.skipkeys, _one_shot)
-        return _iterencode(o, 0)
-
-def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,
-        _key_separator, _item_separator, _sort_keys, _skipkeys, _one_shot,
-        ## HACK: hand-optimized bytecode; turn globals into locals
-        ValueError=ValueError,
-        basestring=basestring,
-        dict=dict,
-        float=float,
-        id=id,
-        int=int,
-        isinstance=isinstance,
-        list=list,
-        long=long,
-        str=str,
-        tuple=tuple,
-    ):
-
-    def _iterencode_list(lst, _current_indent_level):
+    def _iterencode_list(self, lst, markers, _current_indent_level):
         if not lst:
             yield '[]'
             return
-        if markers is not None:
-            markerid = id(lst)
-            if markerid in markers:
-                raise ValueError("Circular reference detected")
-            markers[markerid] = lst
+        self._mark_markers(markers, lst)
         buf = '['
-        if _indent is not None:
+        if self.indent is not None:
             _current_indent_level += 1
-            newline_indent = '\n' + (' ' * (_indent * _current_indent_level))
-            separator = _item_separator + newline_indent
+            newline_indent = '\n' + (' ' * (self.indent *
+                                            _current_indent_level))
+            separator = self.item_separator + newline_indent
             buf += newline_indent
         else:
             newline_indent = None
-            separator = _item_separator
+            separator = self.item_separator
         first = True
         for value in lst:
             if first:
@@ -304,7 +374,7 @@
             else:
                 buf = separator
             if isinstance(value, basestring):
-                yield buf + _encoder(value)
+                yield buf + '"' + self.encoder(value) + '"'
             elif value is None:
                 yield buf + 'null'
             elif value is True:
@@ -314,44 +384,43 @@
             elif isinstance(value, (int, long)):
                 yield buf + str(value)
             elif isinstance(value, float):
-                yield buf + _floatstr(value)
+                yield buf + self._floatstr(value)
             else:
                 yield buf
                 if isinstance(value, (list, tuple)):
-                    chunks = _iterencode_list(value, _current_indent_level)
+                    chunks = self._iterencode_list(value, markers,
+                                                   _current_indent_level)
                 elif isinstance(value, dict):
-                    chunks = _iterencode_dict(value, _current_indent_level)
+                    chunks = self._iterencode_dict(value, markers,
+                                                   _current_indent_level)
                 else:
-                    chunks = _iterencode(value, _current_indent_level)
+                    chunks = self._iterencode(value, markers,
+                                              _current_indent_level)
                 for chunk in chunks:
                     yield chunk
         if newline_indent is not None:
             _current_indent_level -= 1
-            yield '\n' + (' ' * (_indent * _current_indent_level))
+            yield '\n' + (' ' * (self.indent * _current_indent_level))
         yield ']'
-        if markers is not None:
-            del markers[markerid]
+        self._remove_markers(markers, lst)
 
-    def _iterencode_dict(dct, _current_indent_level):
+    def _iterencode_dict(self, dct, markers, _current_indent_level):
         if not dct:
             yield '{}'
             return
-        if markers is not None:
-            markerid = id(dct)
-            if markerid in markers:
-                raise ValueError("Circular reference detected")
-            markers[markerid] = dct
+        self._mark_markers(markers, dct)
         yield '{'
-        if _indent is not None:
+        if self.indent is not None:
             _current_indent_level += 1
-            newline_indent = '\n' + (' ' * (_indent * _current_indent_level))
-            item_separator = _item_separator + newline_indent
+            newline_indent = '\n' + (' ' * (self.indent *
+                                            _current_indent_level))
+            item_separator = self.item_separator + newline_indent
             yield newline_indent
         else:
             newline_indent = None
-            item_separator = _item_separator
+            item_separator = self.item_separator
         first = True
-        if _sort_keys:
+        if self.sort_keys:
             items = sorted(dct.items(), key=lambda kv: kv[0])
         else:
             items = dct.iteritems()
@@ -361,7 +430,7 @@
             # JavaScript is weakly typed for these, so it makes sense to
             # also allow them.  Many encoders seem to do something like this.
             elif isinstance(key, float):
-                key = _floatstr(key)
+                key = self._floatstr(key)
             elif key is True:
                 key = 'true'
             elif key is False:
@@ -370,7 +439,7 @@
                 key = 'null'
             elif isinstance(key, (int, long)):
                 key = str(key)
-            elif _skipkeys:
+            elif self.skipkeys:
                 continue
             else:
                 raise TypeError("key " + repr(key) + " is not a string")
@@ -378,10 +447,10 @@
                 first = False
             else:
                 yield item_separator
-            yield _encoder(key)
-            yield _key_separator
+            yield '"' + self.encoder(key) + '"'
+            yield self.key_separator
             if isinstance(value, basestring):
-                yield _encoder(value)
+                yield '"' + self.encoder(value) + '"'
             elif value is None:
                 yield 'null'
             elif value is True:
@@ -391,26 +460,28 @@
             elif isinstance(value, (int, long)):
                 yield str(value)
             elif isinstance(value, float):
-                yield _floatstr(value)
+                yield self._floatstr(value)
             else:
                 if isinstance(value, (list, tuple)):
-                    chunks = _iterencode_list(value, _current_indent_level)
+                    chunks = self._iterencode_list(value, markers,
+                                                   _current_indent_level)
                 elif isinstance(value, dict):
-                    chunks = _iterencode_dict(value, _current_indent_level)
+                    chunks = self._iterencode_dict(value, markers,
+                                                   _current_indent_level)
                 else:
-                    chunks = _iterencode(value, _current_indent_level)
+                    chunks = self._iterencode(value, markers,
+                                              _current_indent_level)
                 for chunk in chunks:
                     yield chunk
         if newline_indent is not None:
             _current_indent_level -= 1
-            yield '\n' + (' ' * (_indent * _current_indent_level))
+            yield '\n' + (' ' * (self.indent * _current_indent_level))
         yield '}'
-        if markers is not None:
-            del markers[markerid]
+        self._remove_markers(markers, dct)
 
-    def _iterencode(o, _current_indent_level):
+    def _iterencode(self, o, markers, _current_indent_level):
         if isinstance(o, basestring):
-            yield _encoder(o)
+            yield '"' + self.encoder(o) + '"'
         elif o is None:
             yield 'null'
         elif o is True:
@@ -420,23 +491,19 @@
         elif isinstance(o, (int, long)):
             yield str(o)
         elif isinstance(o, float):
-            yield _floatstr(o)
+            yield self._floatstr(o)
         elif isinstance(o, (list, tuple)):
-            for chunk in _iterencode_list(o, _current_indent_level):
+            for chunk in self._iterencode_list(o, markers,
+                                               _current_indent_level):
                 yield chunk
         elif isinstance(o, dict):
-            for chunk in _iterencode_dict(o, _current_indent_level):
+            for chunk in self._iterencode_dict(o, markers,
+                                               _current_indent_level):
                 yield chunk
         else:
-            if markers is not None:
-                markerid = id(o)
-                if markerid in markers:
-                    raise ValueError("Circular reference detected")
-                markers[markerid] = o
-            o = _default(o)
-            for chunk in _iterencode(o, _current_indent_level):
+            self._mark_markers(markers, o)
+            obj = self.default(o)
+            for chunk in self._iterencode(obj, markers,
+                                          _current_indent_level):
                 yield chunk
-            if markers is not None:
-                del markers[markerid]
-
-    return _iterencode
+            self._remove_markers(markers, o)
diff --git a/lib-python/modified-2.7/json/tests/test_unicode.py b/lib-python/modified-2.7/json/tests/test_unicode.py
--- a/lib-python/modified-2.7/json/tests/test_unicode.py
+++ b/lib-python/modified-2.7/json/tests/test_unicode.py
@@ -80,3 +80,9 @@
         self.assertEqual(type(json.loads(u'["a"]')[0]), unicode)
         # Issue 10038.
         self.assertEqual(type(json.loads('"foo"')), unicode)
+
+    def test_encode_not_utf_8(self):
+        self.assertEqual(json.dumps('\xb1\xe6', encoding='iso8859-2'),
+                         '"\\u0105\\u0107"')
+        self.assertEqual(json.dumps(['\xb1\xe6'], encoding='iso8859-2'),
+                         '["\\u0105\\u0107"]')
diff --git a/lib_pypy/pyrepl/readline.py b/lib_pypy/pyrepl/readline.py
--- a/lib_pypy/pyrepl/readline.py
+++ b/lib_pypy/pyrepl/readline.py
@@ -395,9 +395,21 @@
     _wrapper.f_in = f_in
     _wrapper.f_out = f_out
 
-    if hasattr(sys, '__raw_input__'):    # PyPy
-        _old_raw_input = sys.__raw_input__
+    if '__pypy__' in sys.builtin_module_names:    # PyPy
+
+        def _old_raw_input(prompt=''):
+            # sys.__raw_input__() is only called when stdin and stdout are
+            # as expected and are ttys.  If it is the case, then get_reader()
+            # should not really fail in _wrapper.raw_input().  If it still
+            # does, then we will just cancel the redirection and call again
+            # the built-in raw_input().
+            try:
+                del sys.__raw_input__
+            except AttributeError:
+                pass
+            return raw_input(prompt)
         sys.__raw_input__ = _wrapper.raw_input
+
     else:
         # this is not really what readline.c does.  Better than nothing I guess
         import __builtin__
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -72,6 +72,7 @@
     del working_modules['fcntl']  # LOCK_NB not defined
     del working_modules["_minimal_curses"]
     del working_modules["termios"]
+    del working_modules["_multiprocessing"]   # depends on rctime
 
 
 
diff --git a/pypy/interpreter/pyparser/pytokenizer.py b/pypy/interpreter/pyparser/pytokenizer.py
--- a/pypy/interpreter/pyparser/pytokenizer.py
+++ b/pypy/interpreter/pyparser/pytokenizer.py
@@ -226,7 +226,7 @@
                         parenlev = parenlev - 1
                         if parenlev < 0:
                             raise TokenError("unmatched '%s'" % initial, line,
-                                             lnum-1, 0, token_list)
+                                             lnum, start + 1, token_list)
                     if token in python_opmap:
                         punct = python_opmap[token]
                     else:
diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py
--- a/pypy/interpreter/pyparser/test/test_pyparse.py
+++ b/pypy/interpreter/pyparser/test/test_pyparse.py
@@ -87,6 +87,10 @@
         assert exc.lineno == 1
         assert exc.offset == 5
         assert exc.lastlineno == 5
+        exc = py.test.raises(SyntaxError, parse, "abc)").value
+        assert exc.msg == "unmatched ')'"
+        assert exc.lineno == 1
+        assert exc.offset == 4
 
     def test_is(self):
         self.parse("x is y")
diff --git a/pypy/jit/metainterp/graphpage.py b/pypy/jit/metainterp/graphpage.py
--- a/pypy/jit/metainterp/graphpage.py
+++ b/pypy/jit/metainterp/graphpage.py
@@ -12,8 +12,8 @@
     def get_display_text(self):
         return None
 
-def display_loops(loops, errmsg=None, highlight_loops=()):
-    graphs = [(loop, loop in highlight_loops) for loop in loops]    
+def display_loops(loops, errmsg=None, highlight_loops={}):
+    graphs = [(loop, highlight_loops.get(loop, 0)) for loop in loops]    
     for graph, highlight in graphs:
         for op in graph.get_operations():
             if is_interesting_guard(op):
@@ -65,8 +65,7 @@
     def add_graph(self, graph, highlight=False):
         graphindex = len(self.graphs)
         self.graphs.append(graph)
-        if highlight:
-            self.highlight_graphs[graph] = True
+        self.highlight_graphs[graph] = highlight
         for i, op in enumerate(graph.get_operations()):
             self.all_operations[op] = graphindex, i
 
@@ -126,10 +125,13 @@
             self.dotgen.emit('subgraph cluster%d {' % graphindex)
         label = graph.get_display_text()
         if label is not None:
-            if self.highlight_graphs.get(graph):
-                fillcolor = '#f084c2'
+            colorindex = self.highlight_graphs.get(graph, 0)
+            if colorindex == 1:
+                fillcolor = '#f084c2'    # highlighted graph
+            elif colorindex == 2:
+                fillcolor = '#808080'    # invalidated graph
             else:
-                fillcolor = '#84f0c2'
+                fillcolor = '#84f0c2'    # normal color
             self.dotgen.emit_node(graphname, shape="octagon",
                                   label=label, fillcolor=fillcolor)
             self.pendingedges.append((graphname,
diff --git a/pypy/jit/metainterp/history.py b/pypy/jit/metainterp/history.py
--- a/pypy/jit/metainterp/history.py
+++ b/pypy/jit/metainterp/history.py
@@ -732,6 +732,7 @@
     failed_states = None
     retraced_count = 0
     terminating = False # see TerminatingLoopToken in compile.py
+    invalidated = False
     outermost_jitdriver_sd = None
     # and more data specified by the backend when the loop is compiled
     number = -1
@@ -934,6 +935,7 @@
         self.loops = []
         self.locations = []
         self.aborted_keys = []
+        self.invalidated_token_numbers = set()
 
     def set_history(self, history):
         self.operations = history.operations
@@ -1012,7 +1014,12 @@
             if loop in loops:
                 loops.remove(loop)
             loops.append(loop)
-        display_loops(loops, errmsg, extraloops)
+        highlight_loops = dict.fromkeys(extraloops, 1)
+        for loop in loops:
+            if hasattr(loop, '_looptoken_number') and (
+                    loop._looptoken_number in self.invalidated_token_numbers):
+                highlight_loops.setdefault(loop, 2)
+        display_loops(loops, errmsg, highlight_loops)
 
 # ----------------------------------------------------------------
 
diff --git a/pypy/jit/metainterp/memmgr.py b/pypy/jit/metainterp/memmgr.py
--- a/pypy/jit/metainterp/memmgr.py
+++ b/pypy/jit/metainterp/memmgr.py
@@ -68,7 +68,8 @@
         debug_print("Loop tokens before:", oldtotal)
         max_generation = self.current_generation - (self.max_age-1)
         for looptoken in self.alive_loops.keys():
-            if 0 <= looptoken.generation < max_generation:
+            if (0 <= looptoken.generation < max_generation or
+                looptoken.invalidated):
                 del self.alive_loops[looptoken]
         newtotal = len(self.alive_loops)
         debug_print("Loop tokens freed: ", oldtotal - newtotal)
diff --git a/pypy/jit/metainterp/optimizeopt/rewrite.py b/pypy/jit/metainterp/optimizeopt/rewrite.py
--- a/pypy/jit/metainterp/optimizeopt/rewrite.py
+++ b/pypy/jit/metainterp/optimizeopt/rewrite.py
@@ -106,10 +106,9 @@
             self.make_equal_to(op.result, v1)
         else:
             self.emit_operation(op)
-
-        # Synthesize the reverse ops for optimize_default to reuse
-        self.pure(rop.INT_ADD, [op.result, op.getarg(1)], op.getarg(0))
-        self.pure(rop.INT_SUB, [op.getarg(0), op.result], op.getarg(1))
+            # Synthesize the reverse ops for optimize_default to reuse
+            self.pure(rop.INT_ADD, [op.result, op.getarg(1)], op.getarg(0))
+            self.pure(rop.INT_SUB, [op.getarg(0), op.result], op.getarg(1))
 
     def optimize_INT_ADD(self, op):
         v1 = self.getvalue(op.getarg(0))
@@ -122,10 +121,9 @@
             self.make_equal_to(op.result, v1)
         else:
             self.emit_operation(op)
-
-        # Synthesize the reverse op for optimize_default to reuse
-        self.pure(rop.INT_SUB, [op.result, op.getarg(1)], op.getarg(0))
-        self.pure(rop.INT_SUB, [op.result, op.getarg(0)], op.getarg(1))
+            # Synthesize the reverse op for optimize_default to reuse
+            self.pure(rop.INT_SUB, [op.result, op.getarg(1)], op.getarg(0))
+            self.pure(rop.INT_SUB, [op.result, op.getarg(0)], op.getarg(1))
 
     def optimize_INT_MUL(self, op):
         v1 = self.getvalue(op.getarg(0))
@@ -141,13 +139,13 @@
             self.make_constant_int(op.result, 0)
         else:
             for lhs, rhs in [(v1, v2), (v2, v1)]:
-                # x & (x -1) == 0 is a quick test for power of 2
-                if (lhs.is_constant() and
-                    (lhs.box.getint() & (lhs.box.getint() - 1)) == 0):
-                    new_rhs = ConstInt(highest_bit(lhs.box.getint()))
-                    op = op.copy_and_change(rop.INT_LSHIFT, args=[rhs.box, new_rhs])
-                    break
-
+                if lhs.is_constant():
+                    x = lhs.box.getint()
+                    # x & (x - 1) == 0 is a quick test for power of 2
+                    if x & (x - 1) == 0:
+                        new_rhs = ConstInt(highest_bit(lhs.box.getint()))
+                        op = op.copy_and_change(rop.INT_LSHIFT, args=[rhs.box, new_rhs])
+                        break
             self.emit_operation(op)
 
     def optimize_UINT_FLOORDIV(self, op):
@@ -462,6 +460,14 @@
         self.optimizer.opaque_pointers[value] = True
         self.make_equal_to(op.result, value)
 
+    def optimize_CAST_PTR_TO_INT(self, op):
+        self.pure(rop.CAST_INT_TO_PTR, [op.result], op.getarg(0))
+        self.emit_operation(op)
+
+    def optimize_CAST_INT_TO_PTR(self, op):
+        self.pure(rop.CAST_PTR_TO_INT, [op.result], op.getarg(0))
+        self.emit_operation(op)
+
 dispatch_opt = make_dispatcher_method(OptRewrite, 'optimize_',
         default=OptRewrite.emit_operation)
 optimize_guards = _findall(OptRewrite, 'optimize_', 'GUARD')
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -234,6 +234,30 @@
             """ % expected_value
             self.optimize_loop(ops, expected)
 
+    def test_reverse_of_cast(self):
+        ops = """
+        [i0]
+        p0 = cast_int_to_ptr(i0)
+        i1 = cast_ptr_to_int(p0)
+        jump(i1)
+        """
+        expected = """
+        [i0]
+        jump(i0)
+        """
+        self.optimize_loop(ops, expected)
+        ops = """
+        [p0]
+        i1 = cast_ptr_to_int(p0)
+        p1 = cast_int_to_ptr(i1)
+        jump(p1)
+        """
+        expected = """
+        [p0]
+        jump(p0)
+        """
+        self.optimize_loop(ops, expected)
+
     # ----------
 
     def test_remove_guard_class_1(self):
diff --git a/pypy/jit/metainterp/quasiimmut.py b/pypy/jit/metainterp/quasiimmut.py
--- a/pypy/jit/metainterp/quasiimmut.py
+++ b/pypy/jit/metainterp/quasiimmut.py
@@ -2,6 +2,7 @@
 from pypy.rpython.lltypesystem import lltype, rclass
 from pypy.rpython.annlowlevel import cast_base_ptr_to_instance
 from pypy.jit.metainterp.history import AbstractDescr
+from pypy.rlib.objectmodel import we_are_translated
 
 
 def get_mutate_field_name(fieldname):
@@ -50,13 +51,13 @@
 
 class QuasiImmut(object):
     llopaque = True
+    compress_limit = 30
     
     def __init__(self, cpu):
         self.cpu = cpu
         # list of weakrefs to the LoopTokens that must be invalidated if
         # this value ever changes
         self.looptokens_wrefs = []
-        self.compress_limit = 30
 
     def hide(self):
         qmut_ptr = self.cpu.ts.cast_instance_to_base_ref(self)
@@ -75,6 +76,8 @@
     def compress_looptokens_list(self):
         self.looptokens_wrefs = [wref for wref in self.looptokens_wrefs
                                       if wref() is not None]
+        # NB. we must keep around the looptoken_wrefs that are
+        # already invalidated; see below
         self.compress_limit = (len(self.looptokens_wrefs) + 15) * 2
 
     def invalidate(self):
@@ -86,7 +89,16 @@
         for wref in wrefs:
             looptoken = wref()
             if looptoken is not None:
+                looptoken.invalidated = True
                 self.cpu.invalidate_loop(looptoken)
+                # NB. we must call cpu.invalidate_loop() even if
+                # looptoken.invalidated was already set to True.
+                # It's possible to invalidate several times the
+                # same looptoken; see comments in jit.backend.model
+                # in invalidate_loop().
+                if not we_are_translated():
+                    self.cpu.stats.invalidated_token_numbers.add(
+                        looptoken.number)
 
 
 class QuasiImmutDescr(AbstractDescr):
diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py
--- a/pypy/jit/metainterp/resoperation.py
+++ b/pypy/jit/metainterp/resoperation.py
@@ -430,11 +430,11 @@
     'INT_IS_TRUE/1b',
     'INT_NEG/1',
     'INT_INVERT/1',
+    #
+    'SAME_AS/1',      # gets a Const or a Box, turns it into another Box
     'CAST_PTR_TO_INT/1',
     'CAST_INT_TO_PTR/1',
     #
-    'SAME_AS/1',      # gets a Const or a Box, turns it into another Box
-    #
     'PTR_EQ/2b',
     'PTR_NE/2b',
     'CAST_OPAQUE_PTR/1b',
diff --git a/pypy/jit/metainterp/test/test_memmgr.py b/pypy/jit/metainterp/test/test_memmgr.py
--- a/pypy/jit/metainterp/test/test_memmgr.py
+++ b/pypy/jit/metainterp/test/test_memmgr.py
@@ -18,6 +18,7 @@
 
 class FakeLoopToken:
     generation = 0
+    invalidated = False
 
 
 class _TestMemoryManager:
diff --git a/pypy/jit/metainterp/test/test_quasiimmut.py b/pypy/jit/metainterp/test/test_quasiimmut.py
--- a/pypy/jit/metainterp/test/test_quasiimmut.py
+++ b/pypy/jit/metainterp/test/test_quasiimmut.py
@@ -48,6 +48,13 @@
 
 class QuasiImmutTests(object):
 
+    def setup_method(self, meth):
+        self.prev_compress_limit = QuasiImmut.compress_limit
+        QuasiImmut.compress_limit = 1
+
+    def teardown_method(self, meth):
+        QuasiImmut.compress_limit = self.prev_compress_limit
+
     def test_simple_1(self):
         myjitdriver = JitDriver(greens=['foo'], reds=['x', 'total'])
         class Foo:
@@ -289,7 +296,7 @@
             return total
 
         res = self.meta_interp(main, [])
-        self.check_loop_count(9)
+        self.check_tree_loop_count(6)
         assert res == main()
 
     def test_change_during_running(self):
@@ -317,7 +324,7 @@
         assert f(100, 15) == 3009
         res = self.meta_interp(f, [100, 15])
         assert res == 3009
-        self.check_loops(guard_not_invalidated=2, getfield_gc=0,
+        self.check_loops(guard_not_invalidated=4, getfield_gc=0,
                          call_may_force=0, guard_not_forced=0)
 
     def test_list_simple_1(self):
@@ -453,10 +460,30 @@
         assert f(100, 15) == 3009
         res = self.meta_interp(f, [100, 15])
         assert res == 3009
-        self.check_loops(guard_not_invalidated=2, getfield_gc=0,
+        self.check_loops(guard_not_invalidated=4, getfield_gc=0,
                          getarrayitem_gc=0, getarrayitem_gc_pure=0,
                          call_may_force=0, guard_not_forced=0)
 
+    def test_invalidated_loop_is_not_used_any_more_as_target(self):
+        myjitdriver = JitDriver(greens=['foo'], reds=['x'])
+        class Foo:
+            _immutable_fields_ = ['step?']
+        @dont_look_inside
+        def residual(x, foo):
+            if x == 20:
+                foo.step = 1
+        def f(x):
+            foo = Foo()
+            foo.step = 2
+            while x > 0:
+                myjitdriver.jit_merge_point(foo=foo, x=x)
+                residual(x, foo)
+                x -= foo.step
+            return foo.step
+        res = self.meta_interp(f, [60])
+        assert res == 1
+        self.check_tree_loop_count(4)   # at least not 2 like before
+
 
 class TestLLtypeGreenFieldsTests(QuasiImmutTests, LLJitMixin):
     pass
diff --git a/pypy/jit/metainterp/test/test_resume.py b/pypy/jit/metainterp/test/test_resume.py
--- a/pypy/jit/metainterp/test/test_resume.py
+++ b/pypy/jit/metainterp/test/test_resume.py
@@ -1,3 +1,4 @@
+from __future__ import with_statement
 import py
 from pypy.rpython.lltypesystem import lltype, llmemory, rffi
 from pypy.jit.metainterp.optimizeopt.optimizer import OptValue
diff --git a/pypy/jit/metainterp/warmstate.py b/pypy/jit/metainterp/warmstate.py
--- a/pypy/jit/metainterp/warmstate.py
+++ b/pypy/jit/metainterp/warmstate.py
@@ -178,7 +178,7 @@
         if self.compiled_merge_points_wref is not None:
             for wref in self.compiled_merge_points_wref:
                 looptoken = wref()
-                if looptoken is not None:
+                if looptoken is not None and not looptoken.invalidated:
                     result.append(looptoken)
         return result
 
diff --git a/pypy/module/__builtin__/__init__.py b/pypy/module/__builtin__/__init__.py
--- a/pypy/module/__builtin__/__init__.py
+++ b/pypy/module/__builtin__/__init__.py
@@ -23,6 +23,7 @@
         'map'           : 'app_functional.map',
         'reduce'        : 'app_functional.reduce',
         'filter'        : 'app_functional.filter',
+        'zip'           : 'app_functional.zip',
         'vars'          : 'app_inspect.vars',
         'dir'           : 'app_inspect.dir',
 
@@ -89,7 +90,6 @@
         'enumerate'     : 'functional.W_Enumerate',
         'min'           : 'functional.min',
         'max'           : 'functional.max',
-        'zip'           : 'functional.zip',
         'reversed'      : 'functional.reversed',
         'super'         : 'descriptor.W_Super',
         'staticmethod'  : 'descriptor.StaticMethod',
@@ -119,7 +119,7 @@
            builtin = space.interpclass_w(w_builtin)
            if isinstance(builtin, module.Module):
                return builtin
-       # no builtin! make a default one.  Given them None, at least.
+       # no builtin! make a default one.  Give them None, at least.
        builtin = module.Module(space, None)
        space.setitem(builtin.w_dict, space.wrap('None'), space.w_None)
        return builtin
diff --git a/pypy/module/__builtin__/app_functional.py b/pypy/module/__builtin__/app_functional.py
--- a/pypy/module/__builtin__/app_functional.py
+++ b/pypy/module/__builtin__/app_functional.py
@@ -162,4 +162,21 @@
         item = seq[i]
         if func(item):
             result.append(item)
-    return tuple(result)
\ No newline at end of file
+    return tuple(result)
+
+def zip(*sequences):
+    """zip(seq1 [, seq2 [...]]) -> [(seq1[0], seq2[0] ...), (...)]
+
+Return a list of tuples, where each tuple contains the i-th element
+from each of the argument sequences.  The returned list is truncated
+in length to the length of the shortest argument sequence."""
+    if not sequences:
+        return []
+    result = []
+    iterators = [iter(seq) for seq in sequences]
+    while True:
+        try:
+            items = [next(it) for it in iterators]
+        except StopIteration:
+            return result
+        result.append(tuple(items))
diff --git a/pypy/module/__builtin__/app_io.py b/pypy/module/__builtin__/app_io.py
--- a/pypy/module/__builtin__/app_io.py
+++ b/pypy/module/__builtin__/app_io.py
@@ -27,7 +27,20 @@
     co = compile(source.rstrip()+"\n", filename, 'exec')
     exec co in glob, loc
 
-def raw_input(prompt=None):
+def _write_prompt(stdout, prompt):
+    print >> stdout, prompt,
+    try:
+        flush = stdout.flush
+    except AttributeError:
+        pass
+    else:
+        flush()
+    try:
+        stdout.softspace = 0
+    except (AttributeError, TypeError):
+        pass
+
+def raw_input(prompt=''):
     """raw_input([prompt]) -> string
 
 Read a string from standard input.  The trailing newline is stripped.
@@ -47,18 +60,10 @@
     if (hasattr(sys, '__raw_input__') and
         isinstance(stdin, file)  and stdin.fileno() == 0 and stdin.isatty() and
         isinstance(stdout, file) and stdout.fileno() == 1):
-        if prompt is None:
-            prompt = ''
-        return sys.__raw_input__(prompt)
+        _write_prompt(stdout, '')
+        return sys.__raw_input__(str(prompt))
 
-    if prompt is not None:
-        stdout.write(prompt)
-        try:
-            flush = stdout.flush
-        except AttributeError:
-            pass
-        else:
-            flush()
+    _write_prompt(stdout, prompt)
     line = stdin.readline()
     if not line:    # inputting an empty line gives line == '\n'
         raise EOFError
diff --git a/pypy/module/__builtin__/compiling.py b/pypy/module/__builtin__/compiling.py
--- a/pypy/module/__builtin__/compiling.py
+++ b/pypy/module/__builtin__/compiling.py
@@ -84,8 +84,8 @@
         raise OperationError(space.w_TypeError,
               w('eval() arg 1 must be a string or code object'))
 
-    caller = space.getexecutioncontext().gettopframe_nohidden()
     if space.is_w(w_globals, space.w_None):
+        caller = space.getexecutioncontext().gettopframe_nohidden()
         if caller is None:
             w_globals = space.newdict()
             if space.is_w(w_locals, space.w_None):
@@ -97,13 +97,9 @@
     elif space.is_w(w_locals, space.w_None):
         w_locals = w_globals
 
-    try:
-        space.getitem(w_globals, space.wrap('__builtins__'))
-    except OperationError, e:
-        if not e.match(space, space.w_KeyError):
-            raise
-        if caller is not None:
-            w_builtin = space.builtin.pick_builtin(caller.w_globals)
-            space.setitem(w_globals, space.wrap('__builtins__'), w_builtin)
+    # xxx removed: adding '__builtins__' to the w_globals dict, if there
+    # is none.  This logic was removed as costly (it requires to get at
+    # the gettopframe_nohidden()).  I bet no test fails, and it's a really
+    # obscure case.
 
     return codeobj.exec_code(space, w_globals, w_locals)
diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py
--- a/pypy/module/__builtin__/functional.py
+++ b/pypy/module/__builtin__/functional.py
@@ -201,27 +201,6 @@
     """
     return min_max(space, __args__, "min")
 
- at unwrap_spec(sequences_w="args_w")
-def zip(space, sequences_w):
-    """Return a list of tuples, where the nth tuple contains every nth item of
-    each collection.
-
-    If the collections have different lengths, zip returns a list as long as the
-    shortest collection, ignoring the trailing items in the other collections.
-    """
-    if not sequences_w:
-        return space.newlist([])
-    result_w = []
-    iterators_w = [space.iter(w_seq) for w_seq in sequences_w]
-    while True:
-        try:
-            items_w = [space.next(w_it) for w_it in iterators_w]
-        except OperationError, e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            return space.newlist(result_w)
-        result_w.append(space.newtuple(items_w))
-
 class W_Enumerate(Wrappable):
 
     def __init__(self, w_iter, w_start):
diff --git a/pypy/module/__builtin__/test/test_rawinput.py b/pypy/module/__builtin__/test/test_rawinput.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/__builtin__/test/test_rawinput.py
@@ -0,0 +1,77 @@
+import autopath
+
+
+class AppTestRawInput():
+
+    def test_raw_input(self):
+        import sys, StringIO
+        for prompt, expected in [("def:", "abc/ def:/ghi\n"),
+                                 ("", "abc/ /ghi\n"),
+                                 (42, "abc/ 42/ghi\n"),
+                                 (None, "abc/ None/ghi\n"),
+                                 (Ellipsis, "abc/ /ghi\n")]:
+            save = sys.stdin, sys.stdout
+            try:
+                sys.stdin = StringIO.StringIO("foo\nbar\n")
+                out = sys.stdout = StringIO.StringIO()
+                print "abc",    # softspace = 1
+                out.write('/')
+                if prompt is Ellipsis:
+                    got = raw_input()
+                else:
+                    got = raw_input(prompt)
+                out.write('/')
+                print "ghi"
+            finally:
+                sys.stdin, sys.stdout = save
+            assert out.getvalue() == expected
+            assert got == "foo"
+
+    def test_softspace(self):
+        import sys
+        import StringIO
+        fin = StringIO.StringIO()
+        fout = StringIO.StringIO()
+
+        fin.write("Coconuts\n")
+        fin.seek(0)
+
+        sys_stdin_orig = sys.stdin
+        sys_stdout_orig = sys.stdout
+
+        sys.stdin = fin
+        sys.stdout = fout
+
+        print "test",
+        raw_input("test")
+
+        sys.stdin = sys_stdin_orig
+        sys.stdout = sys_stdout_orig
+
+        fout.seek(0)
+        assert fout.read() == "test test"
+
+    def test_softspace_carryover(self):
+        import sys
+        import StringIO
+        fin = StringIO.StringIO()
+        fout = StringIO.StringIO()
+
+        fin.write("Coconuts\n")
+        fin.seek(0)
+
+        sys_stdin_orig = sys.stdin
+        sys_stdout_orig = sys.stdout
+
+        sys.stdin = fin
+        sys.stdout = fout
+
+        print "test",
+        raw_input("test")
+        print "test",
+
+        sys.stdin = sys_stdin_orig
+        sys.stdout = sys_stdout_orig
+
+        fout.seek(0)
+        assert fout.read() == "test testtest"
diff --git a/pypy/module/_continuation/interp_continuation.py b/pypy/module/_continuation/interp_continuation.py
--- a/pypy/module/_continuation/interp_continuation.py
+++ b/pypy/module/_continuation/interp_continuation.py
@@ -19,6 +19,11 @@
         #  - normal:      self.sthread != None, not is_empty_handle(self.h)
         #  - finished:    self.sthread != None, is_empty_handle(self.h)
 
+    def __del__(self):
+        sthread = self.sthread
+        if sthread is not None and not sthread.is_empty_handle(self.h):
+            sthread.destroy(self.h)
+
     def check_sthread(self):
         ec = self.space.getexecutioncontext()
         if ec.stacklet_thread is not self.sthread:
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -161,9 +161,6 @@
     @binop
     def mul(self, v1, v2):
         return v1 * v2
-    @binop
-    def div(self, v1, v2):
-        return v1 / v2
 
     @unaryop
     def pos(self, v):
@@ -217,6 +214,14 @@
         return float2string(self.for_computation(self.unbox(item)), 'g', rfloat.DTSF_STR_PRECISION)
 
     @binop
+    def div(self, v1, v2):
+        try:
+            return v1 / v2
+        except ZeroDivisionError:
+            if v1 == v2 == 0.0:
+                return rfloat.NAN
+            return rfloat.copysign(rfloat.INFINITY, v1 * v2)
+    @binop
     def mod(self, v1, v2):
         return math.fmod(v1, v2)
     @binop
@@ -295,6 +300,11 @@
         return str(widen(self.unbox(item)))
 
     @binop
+    def div(self, v1, v2):
+        if v2 == 0:
+            return 0
+        return v1 / v2
+    @binop
     def mod(self, v1, v2):
         return v1 % v2
 
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -285,7 +285,9 @@
             assert b[i] == i * 5
 
     def test_div(self):
-        from numpy import array, dtype
+        from math import isnan
+        from numpy import array, dtype, inf
+
         a = array(range(1, 6))
         b = a / a
         for i in range(5):
@@ -297,6 +299,24 @@
         for i in range(5):
             assert b[i] == 1
 
+        a = array([-1, 0, 1])
+        b = array([0, 0, 0])
+        c = a / b
+        assert (c == [0, 0, 0]).all()
+
+        a = array([-1.0, 0.0, 1.0])
+        b = array([0.0, 0.0, 0.0])
+        c = a / b
+        assert c[0] == -inf
+        assert isnan(c[1])
+        assert c[2] == inf
+
+        b = array([-0.0, -0.0, -0.0])
+        c = a / b
+        assert c[0] == inf
+        assert isnan(c[1])
+        assert c[2] == -inf
+
     def test_div_other(self):
         from numpy import array
         a = array(range(5))
diff --git a/pypy/module/pyexpat/__init__.py b/pypy/module/pyexpat/__init__.py
--- a/pypy/module/pyexpat/__init__.py
+++ b/pypy/module/pyexpat/__init__.py
@@ -4,12 +4,8 @@
 
 class ErrorsModule(MixedModule):
     "Definition of pyexpat.errors module."
-
-    appleveldefs = {
-        }
-
-    interpleveldefs = {
-        }
+    appleveldefs = {}
+    interpleveldefs = {}
 
     def setup_after_space_initialization(self):
         from pypy.module.pyexpat import interp_pyexpat
@@ -18,6 +14,18 @@
                     interp_pyexpat.ErrorString(self.space,
                     getattr(interp_pyexpat, name)))
 
+class ModelModule(MixedModule):
+    "Definition of pyexpat.model module."
+    appleveldefs = {}
+    interpleveldefs = {}
+
+    def setup_after_space_initialization(self):
+        from pypy.module.pyexpat import interp_pyexpat
+        space = self.space
+        for name in interp_pyexpat.xml_model_list:
+            value = getattr(interp_pyexpat, name)
+            space.setattr(self, space.wrap(name), space.wrap(value))
+
 class Module(MixedModule):
     "Python wrapper for Expat parser."
 
@@ -39,6 +47,7 @@
 
     submodules = {
         'errors': ErrorsModule,
+        'model':  ModelModule,
     }
 
     for name in ['XML_PARAM_ENTITY_PARSING_NEVER',
diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py
--- a/pypy/module/pyexpat/interp_pyexpat.py
+++ b/pypy/module/pyexpat/interp_pyexpat.py
@@ -76,6 +76,18 @@
     "XML_ERROR_FINISHED",
     "XML_ERROR_SUSPEND_PE",
     ]
+xml_model_list = [
+    "XML_CTYPE_EMPTY",
+    "XML_CTYPE_ANY",
+    "XML_CTYPE_MIXED",
+    "XML_CTYPE_NAME",
+    "XML_CTYPE_CHOICE",
+    "XML_CTYPE_SEQ",
+    "XML_CQUANT_NONE",
+    "XML_CQUANT_OPT",
+    "XML_CQUANT_REP",
+    "XML_CQUANT_PLUS",
+    ]
 
 class CConfigure:
     _compilation_info_ = eci
@@ -104,6 +116,8 @@
 
     for name in xml_error_list:
         locals()[name] = rffi_platform.ConstantInteger(name)
+    for name in xml_model_list:
+        locals()[name] = rffi_platform.ConstantInteger(name)
 
 for k, v in rffi_platform.configure(CConfigure).items():
     globals()[k] = v
diff --git a/pypy/module/pyexpat/test/test_parser.py b/pypy/module/pyexpat/test/test_parser.py
--- a/pypy/module/pyexpat/test/test_parser.py
+++ b/pypy/module/pyexpat/test/test_parser.py
@@ -131,3 +131,7 @@
                'encoding specified in XML declaration is incorrect')
         assert (pyexpat.errors.XML_ERROR_XML_DECL ==
                 'XML declaration not well-formed')
+
+    def test_model(self):
+        import pyexpat
+        assert isinstance(pyexpat.model.XML_CTYPE_EMPTY, int)
diff --git a/pypy/module/pypyjit/test_pypy_c/model.py b/pypy/module/pypyjit/test_pypy_c/model.py
--- a/pypy/module/pypyjit/test_pypy_c/model.py
+++ b/pypy/module/pypyjit/test_pypy_c/model.py
@@ -387,8 +387,8 @@
                 return ''
             text = str(py.code.Source(src).deindent().indent())
             lines = text.splitlines(True)
-            if opindex is not None and 0 <= opindex < len(lines):
-                lines[opindex] = lines[opindex].rstrip() + '\t<=====\n'
+            if opindex is not None and 0 <= opindex <= len(lines):
+                lines.insert(opindex, '\n\t===== HERE =====\n')
             return ''.join(lines)
         #
         expected_src = self.preprocess_expected_src(expected_src)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_misc.py b/pypy/module/pypyjit/test_pypy_c/test_misc.py
--- a/pypy/module/pypyjit/test_pypy_c/test_misc.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_misc.py
@@ -329,4 +329,20 @@
             guard_false(i28, descr=...)
             i30 = int_lshift(i20, 24)
             i31 = int_or(i26, i30)
-        """ % {"32_bit_only": extra})
\ No newline at end of file
+        """ % {"32_bit_only": extra})
+
+    def test_eval(self):
+        def main():
+            i = 1
+            a = compile('x+x+x+x+x+x', 'eval', 'eval')
+            b = {'x': 7}
+            while i < 1000:
+                y = eval(a,b,b)  # ID: eval
+                i += 1
+            return y
+
+        log = self.run(main)
+        assert log.result == 42
+        # the following assertion fails if the loop was cancelled due
+        # to "abort: vable escape"
+        assert len(log.loops_by_id("eval")) == 1
diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -41,7 +41,7 @@
             guard_true(i32, descr=...)
             i34 = int_add(i6, 1)
             --TICK--
-            jump(p0, p1, p2, p3, p4, p5, i34, p7, p8, i9, i10, p11, i12, p13, descr=<Loop4>)
+            jump(p0, p1, p2, p3, p4, p5, i34, p7, p8, i9, i10, p11, i12, p13, descr=...)
         """)
 
     def test_long(self):
@@ -106,7 +106,7 @@
             i58 = int_add_ovf(i6, i57)
             guard_no_overflow(descr=...)
             --TICK--
-            jump(p0, p1, p2, p3, p4, p5, i58, i7, descr=<Loop4>)
+            jump(p0, p1, p2, p3, p4, p5, i58, i7, descr=...)
         """)
 
     def test_str_mod(self):
diff --git a/pypy/objspace/std/strutil.py b/pypy/objspace/std/strutil.py
--- a/pypy/objspace/std/strutil.py
+++ b/pypy/objspace/std/strutil.py
@@ -35,7 +35,7 @@
 
     def error(self):
         raise ParseStringError("invalid literal for %s() with base %d: '%s'" %
-                               (self.fname, self.base, self.literal))
+                               (self.fname, self.original_base, self.literal))
 
     def __init__(self, s, literal, base, fname):
         self.literal = literal
@@ -47,7 +47,8 @@
         elif s.startswith('+'):
             s = strip_spaces(s[1:])
         self.sign = sign
-        
+        self.original_base = base
+
         if base == 0:
             if s.startswith('0x') or s.startswith('0X'):
                 base = 16
diff --git a/pypy/objspace/std/test/test_strutil.py b/pypy/objspace/std/test/test_strutil.py
--- a/pypy/objspace/std/test/test_strutil.py
+++ b/pypy/objspace/std/test/test_strutil.py
@@ -89,6 +89,8 @@
 
         exc = raises(ParseStringError, string_to_int, '')
         assert exc.value.msg == "invalid literal for int() with base 10: ''"
+        exc = raises(ParseStringError, string_to_int, '', 0)
+        assert exc.value.msg == "invalid literal for int() with base 0: ''"
 
     def test_string_to_int_overflow(self):
         import sys
diff --git a/pypy/rlib/_rsocket_rffi.py b/pypy/rlib/_rsocket_rffi.py
--- a/pypy/rlib/_rsocket_rffi.py
+++ b/pypy/rlib/_rsocket_rffi.py
@@ -16,6 +16,7 @@
 _MINGW = target_platform.name == "mingw32"
 _SOLARIS = sys.platform == "sunos5"
 _MACOSX = sys.platform == "darwin"
+_HAS_AF_PACKET = sys.platform.startswith('linux')   # only Linux for now
 
 if _POSIX:
     includes = ('sys/types.h',
@@ -34,11 +35,12 @@
                 'stdint.h', 
                 'errno.h',
                 )
+    if _HAS_AF_PACKET:
+        includes += ('netpacket/packet.h',
+                     'sys/ioctl.h',
+                     'net/if.h')
 
-    cond_includes = [('AF_NETLINK', 'linux/netlink.h'),
-                     ('AF_PACKET', 'netpacket/packet.h'),
-                     ('AF_PACKET', 'sys/ioctl.h'),
-                     ('AF_PACKET', 'net/if.h')]
+    cond_includes = [('AF_NETLINK', 'linux/netlink.h')]
     
     libraries = ()
     calling_conv = 'c'
@@ -320,18 +322,18 @@
                                              ('events', rffi.SHORT),
                                              ('revents', rffi.SHORT)])
 
-    CConfig.sockaddr_ll = platform.Struct('struct sockaddr_ll',
+    if _HAS_AF_PACKET:
+        CConfig.sockaddr_ll = platform.Struct('struct sockaddr_ll',
                               [('sll_ifindex', rffi.INT),
                                ('sll_protocol', rffi.INT),
                                ('sll_pkttype', rffi.INT),
                                ('sll_hatype', rffi.INT),
                                ('sll_addr', rffi.CFixedArray(rffi.CHAR, 8)),
-                               ('sll_halen', rffi.INT)],
-                              ifdef='AF_PACKET')
+                               ('sll_halen', rffi.INT)])
 
-    CConfig.ifreq = platform.Struct('struct ifreq', [('ifr_ifindex', rffi.INT),
-                                 ('ifr_name', rffi.CFixedArray(rffi.CHAR, 8))],
-                                    ifdef='AF_PACKET')
+        CConfig.ifreq = platform.Struct('struct ifreq',
+                                [('ifr_ifindex', rffi.INT),
+                                 ('ifr_name', rffi.CFixedArray(rffi.CHAR, 8))])
 
 if _WIN32:
     CConfig.WSAEVENT = platform.SimpleType('WSAEVENT', rffi.VOIDP)
@@ -386,6 +388,8 @@
         constants[name] = value
     else:
         constants[name] = default
+if not _HAS_AF_PACKET and 'AF_PACKET' in constants:
+    del constants['AF_PACKET']
 
 constants['has_ipv6'] = True # This is a configuration option in CPython
 for name, value in constants.items():
@@ -439,21 +443,14 @@
 if _POSIX:
     nfds_t = cConfig.nfds_t
     pollfd = cConfig.pollfd
-    if cConfig.sockaddr_ll is not None:
+    if _HAS_AF_PACKET:
         sockaddr_ll = cConfig.sockaddr_ll
-    ifreq = cConfig.ifreq
+        ifreq = cConfig.ifreq
 if WIN32:
     WSAEVENT = cConfig.WSAEVENT
     WSANETWORKEVENTS = cConfig.WSANETWORKEVENTS
 timeval = cConfig.timeval
 
-#if _POSIX:
-#    includes = list(includes)
-#    for _name, _header in cond_includes:
-#        if getattr(cConfig, _name) is not None:
-#            includes.append(_header)
-#    eci = ExternalCompilationInfo(includes=includes, libraries=libraries,
-#                                  separate_module_sources=sources)
 
 def external(name, args, result, **kwds):
     return rffi.llexternal(name, args, result, compilation_info=eci,
@@ -544,7 +541,7 @@
     socketpair_t = rffi.CArray(socketfd_type)
     socketpair = external('socketpair', [rffi.INT, rffi.INT, rffi.INT,
                           lltype.Ptr(socketpair_t)], rffi.INT)
-    if ifreq is not None:
+    if _HAS_AF_PACKET:
         ioctl = external('ioctl', [socketfd_type, rffi.INT, lltype.Ptr(ifreq)],
                          rffi.INT)
 
diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -8,6 +8,8 @@
 from pypy.rpython.extregistry import ExtRegistryEntry
 from pypy.tool.sourcetools import func_with_new_name
 
+DEBUG_ELIDABLE_FUNCTIONS = False
+
 
 def elidable(func):
     """ Decorate a function as "trace-elidable". This means precisely that:
@@ -24,6 +26,18 @@
     If a particular call to this function ends up raising an exception, then it
     is handled like a normal function call (this decorator is ignored).
     """
+    if DEBUG_ELIDABLE_FUNCTIONS:
+        cache = {}
+        oldfunc = func
+        def func(*args):
+            result = oldfunc(*args)    # if it raises, no caching
+            try:
+                oldresult = cache.setdefault(args, result)
+            except TypeError:
+                pass           # unhashable args
+            else:
+                assert oldresult == result
+            return result
     func._elidable_function_ = True
     return func
 
diff --git a/pypy/rlib/rsocket.py b/pypy/rlib/rsocket.py
--- a/pypy/rlib/rsocket.py
+++ b/pypy/rlib/rsocket.py
@@ -8,6 +8,7 @@
 # Known missing features:
 #
 #   - address families other than AF_INET, AF_INET6, AF_UNIX, AF_PACKET
+#   - AF_PACKET is only supported on Linux
 #   - methods makefile(),
 #   - SSL
 #
diff --git a/pypy/rpython/lltypesystem/rbuilder.py b/pypy/rpython/lltypesystem/rbuilder.py
--- a/pypy/rpython/lltypesystem/rbuilder.py
+++ b/pypy/rpython/lltypesystem/rbuilder.py
@@ -29,7 +29,7 @@
         except OverflowError:
             raise MemoryError
         newbuf = mallocfn(new_allocated)
-        copycontentsfn(ll_builder.buf, newbuf, 0, 0, ll_builder.allocated)
+        copycontentsfn(ll_builder.buf, newbuf, 0, 0, ll_builder.used)
         ll_builder.buf = newbuf
         ll_builder.allocated = new_allocated
     return func_with_new_name(stringbuilder_grow, name)
@@ -56,7 +56,7 @@
 class BaseStringBuilderRepr(AbstractStringBuilderRepr):
     def empty(self):
         return nullptr(self.lowleveltype.TO)
-    
+
     @classmethod
     def ll_new(cls, init_size):
         if init_size < 0 or init_size > MAX:
diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -1292,10 +1292,12 @@
         # if a prebuilt GcStruct contains a pointer to a young object,
         # then the write_barrier must have ensured that the prebuilt
         # GcStruct is in the list self.old_objects_pointing_to_young.
+        debug_start("gc-minor-walkroots")
         self.root_walker.walk_roots(
             MiniMarkGC._trace_drag_out1,  # stack roots
             MiniMarkGC._trace_drag_out1,  # static in prebuilt non-gc
             None)                         # static in prebuilt gc
+        debug_stop("gc-minor-walkroots")
 
     def collect_cardrefs_to_nursery(self):
         size_gc_header = self.gcheaderbuilder.size_gc_header
diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -56,8 +56,8 @@
     binaries = [(pypy_c, rename_pypy_c)]
     #
     if sys.platform == 'win32':
-        # Can't rename a DLL: it is always called 'libpypy_c.dll'
-        for extra in ['libpypy_c.dll',
+        # Can't rename a DLL: it is always called 'libpypy-c.dll'
+        for extra in ['libpypy-c.dll',
                       'libexpat.dll', 'sqlite3.dll', 'msvcr90.dll']:
             p = pypy_c.dirpath().join(extra)
             if not p.check():
diff --git a/pypy/translator/platform/linux.py b/pypy/translator/platform/linux.py
--- a/pypy/translator/platform/linux.py
+++ b/pypy/translator/platform/linux.py
@@ -1,5 +1,6 @@
 """Support for Linux."""
 
+import sys
 from pypy.translator.platform.posix import BasePosix
 
 class BaseLinux(BasePosix):
@@ -24,15 +25,18 @@
         return self._pkg_config("libffi", "--libs-only-L",
                                 ['/usr/lib/libffi'])
 
+    def library_dirs_for_libffi_a(self):
+        # places where we need to look for libffi.a
+        # XXX obscuuure!  only look for libffi.a if run with translate.py
+        if 'translate' in sys.modules:
+            return self.library_dirs_for_libffi() + ['/usr/lib']
+        else:
+            return []
+
 
 class Linux(BaseLinux):
     shared_only = ()    # it seems that on 32-bit linux, compiling with -fPIC
                         # gives assembler that asmgcc is not happy about.
 
-    def library_dirs_for_libffi_a(self):
-        # places where we need to look for libffi.a
-        return self.library_dirs_for_libffi() + ['/usr/lib']
-
-
 class Linux64(BaseLinux):
     pass
diff --git a/pypy/translator/platform/posix.py b/pypy/translator/platform/posix.py
--- a/pypy/translator/platform/posix.py
+++ b/pypy/translator/platform/posix.py
@@ -157,7 +157,7 @@
 
         rules = [
             ('all', '$(DEFAULT_TARGET)', []),
-            ('$(TARGET)', '$(OBJECTS)', '$(CC_LINK) $(LDFLAGS) $(LDFLAGSEXTRA) -o $@ $(OBJECTS) $(LIBDIRS) $(LIBS) $(LINKFILES)'),
+            ('$(TARGET)', '$(OBJECTS)', '$(CC_LINK) $(LDFLAGSEXTRA) -o $@ $(OBJECTS) $(LIBDIRS) $(LIBS) $(LINKFILES) $(LDFLAGS)'),
             ('%.o', '%.c', '$(CC) $(CFLAGS) $(CFLAGSEXTRA) -o $@ -c $< $(INCLUDEDIRS)'),
             ]
 

From noreply at buildbot.pypy.org  Thu Oct 20 20:42:34 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Thu, 20 Oct 2011 20:42:34 +0200 (CEST)
Subject: [pypy-commit] pypy inline-dict-ops: allow newdict to be inlined
Message-ID: <20111020184234.57475820D7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: inline-dict-ops
Changeset: r48281:5815c00f560b
Date: 2011-10-20 14:42 -0400
http://bitbucket.org/pypy/pypy/changeset/5815c00f560b/

Log:	allow newdict to be inlined

diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py
--- a/pypy/rpython/lltypesystem/rdict.py
+++ b/pypy/rpython/lltypesystem/rdict.py
@@ -623,7 +623,6 @@
     d.num_items = 0
     d.resize_counter = DICT_INITSIZE * 2
     return d
-ll_newdict.oopspec = 'newdict()'
 
 def ll_newdict_size(DICT, length_estimate):
     length_estimate = (length_estimate // 2) * 3
@@ -635,7 +634,6 @@
     d.num_items = 0
     d.resize_counter = n * 2
     return d
-ll_newdict_size.oopspec = 'newdict()'
 
 # pypy.rpython.memory.lldict uses a dict based on Struct and Array
 # instead of GcStruct and GcArray, which is done by using different

From noreply at buildbot.pypy.org  Thu Oct 20 21:04:51 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Thu, 20 Oct 2011 21:04:51 +0200 (CEST)
Subject: [pypy-commit] pypy inline-dict-ops: fix tests for new inlining
Message-ID: <20111020190451.A1146820D7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: inline-dict-ops
Changeset: r48282:f566dee94e85
Date: 2011-10-20 15:04 -0400
http://bitbucket.org/pypy/pypy/changeset/f566dee94e85/

Log:	fix tests for new inlining

diff --git a/pypy/jit/metainterp/test/test_dict.py b/pypy/jit/metainterp/test/test_dict.py
--- a/pypy/jit/metainterp/test/test_dict.py
+++ b/pypy/jit/metainterp/test/test_dict.py
@@ -91,7 +91,8 @@
         res1 = f(100)
         res2 = self.meta_interp(f, [100], listops=True)
         assert res1 == res2
-        self.check_loops(int_mod=3) # the hash was traced and eq
+        # the hash was traced
+        self.check_loops(int_mod=1)
 
     def test_dict_setdefault(self):
         myjitdriver = JitDriver(greens = [], reds = ['total', 'dct'])
@@ -128,7 +129,7 @@
         assert f(100) == 50
         res = self.meta_interp(f, [100], listops=True)
         assert res == 50
-        self.check_loops(int_mod=3) # key + eq
+        self.check_loops(int_mod=1) # key
 
     def test_repeated_lookup(self):
         myjitdriver = JitDriver(greens = [], reds = ['n', 'd'])
@@ -153,10 +154,12 @@
 
         res = self.meta_interp(f, [100], listops=True)
         assert res == f(50)
-        self.check_loops({"call": 7, "guard_false": 1, "guard_no_exception": 6,
+        self.check_loops({"call": 5, "getfield_gc": 1, "getinteriorfield_gc": 1,
+                          "guard_false": 1, "guard_no_exception": 4,
                           "guard_true": 1, "int_and": 1, "int_gt": 1,
                           "int_is_true": 1, "int_sub": 1, "jump": 1,
-                          "new_with_vtable": 1, "setfield_gc": 1})
+                          "new_with_vtable": 1, "new": 1, "new_array": 1,
+                          "setfield_gc": 3, })
 
 
 class TestOOtype(DictTests, OOJitMixin):

From noreply at buildbot.pypy.org  Thu Oct 20 21:39:31 2011
From: noreply at buildbot.pypy.org (antocuni)
Date: Thu, 20 Oct 2011 21:39:31 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: my dates
Message-ID: <20111020193931.B15CE820D7@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: extradoc
Changeset: r3935:16335a1394f1
Date: 2011-10-20 21:39 +0200
http://bitbucket.org/pypy/extradoc/changeset/16335a1394f1/

Log:	my dates

diff --git a/sprintinfo/gothenburg-2011-2/people.txt b/sprintinfo/gothenburg-2011-2/people.txt
--- a/sprintinfo/gothenburg-2011-2/people.txt
+++ b/sprintinfo/gothenburg-2011-2/people.txt
@@ -11,4 +11,5 @@
 Jacob Hallen         lives there
 Laura Creighton      lives there
 Armin Rigo           ?              ?
+Antonio Cuni         4-10 november  Laura's and Jacob's
 ==================== ============== ===================== ==================

From noreply at buildbot.pypy.org  Thu Oct 20 22:39:55 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Thu, 20 Oct 2011 22:39:55 +0200 (CEST)
Subject: [pypy-commit] pypy inline-dict-ops: make boehm use the same code
 path as framework gcs for allocating arrays
Message-ID: <20111020203955.A00F0820D7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: inline-dict-ops
Changeset: r48283:c24558868b27
Date: 2011-10-20 16:39 -0400
http://bitbucket.org/pypy/pypy/changeset/c24558868b27/

Log:	make boehm use the same code path as framework gcs for allocating
	arrays

diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py
--- a/pypy/jit/backend/llsupport/gc.py
+++ b/pypy/jit/backend/llsupport/gc.py
@@ -96,6 +96,15 @@
         malloc_fn_ptr = self.configure_boehm_once()
         self.funcptr_for_new = malloc_fn_ptr
 
+        def malloc_array(basesize, itemsize, ofs_length, num_elem):
+            size = basesize + itemsize * num_elem
+            res = self.funcptr_for_new(size)
+            rffi.cast(rffi.CArrayPtr(lltype.Signed), res)[ofs_length/WORD] = num_elem
+            return res
+        self.malloc_array = malloc_array
+        self.GC_MALLOC_ARRAY = lltype.Ptr(lltype.FuncType(
+            [lltype.Signed] * 4, llmemory.GCREF))
+
         # on some platform GC_init is required before any other
         # GC_* functions, call it here for the benefit of tests
         # XXX move this to tests
@@ -116,10 +125,7 @@
         ofs_length = arraydescr.get_ofs_length(self.translate_support_code)
         basesize = arraydescr.get_base_size(self.translate_support_code)
         itemsize = arraydescr.get_item_size(self.translate_support_code)
-        size = basesize + itemsize * num_elem
-        res = self.funcptr_for_new(size)
-        rffi.cast(rffi.CArrayPtr(lltype.Signed), res)[ofs_length/WORD] = num_elem
-        return res
+        return self.malloc_array(basesize, itemsize, ofs_length, num_elem)
 
     def gc_malloc_str(self, num_elem):
         basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR,
@@ -142,10 +148,18 @@
         assert isinstance(sizedescr, BaseSizeDescr)
         return [sizedescr.size]
 
+    def args_for_new_array(self, arraydescr):
+        ofs_length = arraydescr.get_ofs_length(self.translate_support_code)
+        basesize = arraydescr.get_base_size(self.translate_support_code)
+        itemsize = arraydescr.get_item_size(self.translate_support_code)
+        return [basesize, itemsize, ofs_length]
+
     def get_funcptr_for_new(self):
         return self.funcptr_for_new
 
-    get_funcptr_for_newarray = None
+    def get_funcptr_for_newarray(self):
+        return llhelper(self.GC_MALLOC_ARRAY, self.malloc_array)
+
     get_funcptr_for_newstr = None
     get_funcptr_for_newunicode = None
 
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -993,7 +993,7 @@
         effectinfo = op.getdescr().get_extra_info()
         oopspecindex = effectinfo.oopspecindex
         genop_llong_list[oopspecindex](self, op, arglocs, resloc)
-        
+
     def regalloc_perform_math(self, op, arglocs, resloc):
         effectinfo = op.getdescr().get_extra_info()
         oopspecindex = effectinfo.oopspecindex
@@ -1310,7 +1310,7 @@
     genop_guard_float_eq = _cmpop_guard_float("E", "E", "NE","NE")
     genop_guard_float_gt = _cmpop_guard_float("A", "B", "BE","AE")
     genop_guard_float_ge = _cmpop_guard_float("AE","BE", "B", "A")
-    
+
     def genop_math_sqrt(self, op, arglocs, resloc):
         self.mc.SQRTSD(arglocs[0], resloc)
 
diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py
--- a/pypy/jit/backend/x86/regalloc.py
+++ b/pypy/jit/backend/x86/regalloc.py
@@ -434,7 +434,7 @@
             if self.can_merge_with_next_guard(op, i, operations):
                 oplist_with_guard[op.getopnum()](self, op, operations[i + 1])
                 i += 1
-            elif not we_are_translated() and op.getopnum() == -124: 
+            elif not we_are_translated() and op.getopnum() == -124:
                 self._consider_force_spill(op)
             else:
                 oplist[op.getopnum()](self, op)
@@ -1020,27 +1020,18 @@
 
     def consider_new_array(self, op):
         gc_ll_descr = self.assembler.cpu.gc_ll_descr
-        if gc_ll_descr.get_funcptr_for_newarray is not None:
-            # framework GC
-            box_num_elem = op.getarg(0)
-            if isinstance(box_num_elem, ConstInt):
-                num_elem = box_num_elem.value
-                if gc_ll_descr.can_inline_malloc_varsize(op.getdescr(),
-                                                         num_elem):
-                    self.fastpath_malloc_varsize(op, op.getdescr(), num_elem)
-                    return
-            args = self.assembler.cpu.gc_ll_descr.args_for_new_array(
-                op.getdescr())
-            arglocs = [imm(x) for x in args]
-            arglocs.append(self.loc(box_num_elem))
-            self._call(op, arglocs)
-            return
-        # boehm GC (XXX kill the following code at some point)
-        itemsize, basesize, ofs_length, _, _ = (
-            self._unpack_arraydescr(op.getdescr()))
-        scale_of_field = _get_scale(itemsize)
-        self._malloc_varsize(basesize, ofs_length, scale_of_field,
-                             op.getarg(0), op.result)
+        box_num_elem = op.getarg(0)
+        if isinstance(box_num_elem, ConstInt):
+            num_elem = box_num_elem.value
+            if gc_ll_descr.can_inline_malloc_varsize(op.getdescr(),
+                                                     num_elem):
+                self.fastpath_malloc_varsize(op, op.getdescr(), num_elem)
+                return
+        args = self.assembler.cpu.gc_ll_descr.args_for_new_array(
+            op.getdescr())
+        arglocs = [imm(x) for x in args]
+        arglocs.append(self.loc(box_num_elem))
+        self._call(op, arglocs)
 
     def _unpack_arraydescr(self, arraydescr):
         assert isinstance(arraydescr, BaseArrayDescr)
@@ -1098,7 +1089,7 @@
         self.possibly_free_vars(args)
         self.rm.possibly_free_var(tmpvar)
         self.PerformDiscard(op, [base_loc, ofs, itemsize, fieldsize,
-                                 index_loc, value_loc])        
+                                 index_loc, value_loc])
 
     def consider_strsetitem(self, op):
         args = op.getarglist()

From noreply at buildbot.pypy.org  Fri Oct 21 00:09:19 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Fri, 21 Oct 2011 00:09:19 +0200 (CEST)
Subject: [pypy-commit] pypy inline-dict-ops: add descr to interior field
 descr, fix pypyjit test
Message-ID: <20111020220919.6C38D820D7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: inline-dict-ops
Changeset: r48284:12999013a845
Date: 2011-10-20 18:09 -0400
http://bitbucket.org/pypy/pypy/changeset/12999013a845/

Log:	add descr to interior field descr, fix pypyjit test

diff --git a/pypy/jit/backend/llsupport/descr.py b/pypy/jit/backend/llsupport/descr.py
--- a/pypy/jit/backend/llsupport/descr.py
+++ b/pypy/jit/backend/llsupport/descr.py
@@ -153,6 +153,9 @@
     def is_float_field(self):
         return self.fielddescr.is_float_field()
 
+    def repr_of_descr(self):
+        return '<InteriorFieldDescr %s>' % self.fielddescr.repr_of_descr()
+
 # ____________________________________________________________
 # ArrayDescrs
 
diff --git a/pypy/jit/backend/x86/test/test_runner.py b/pypy/jit/backend/x86/test/test_runner.py
--- a/pypy/jit/backend/x86/test/test_runner.py
+++ b/pypy/jit/backend/x86/test/test_runner.py
@@ -31,7 +31,7 @@
 
     # for the individual tests see
     # ====> ../../test/runner_test.py
-    
+
     def setup_method(self, meth):
         self.cpu = CPU(rtyper=None, stats=FakeStats())
         self.cpu.setup_once()
@@ -69,7 +69,7 @@
 
     def test_allocations(self):
         from pypy.rpython.lltypesystem import rstr
-        
+
         allocs = [None]
         all = []
         def f(size):
@@ -82,7 +82,7 @@
         # ctypes produces an unsigned value. We need it to be signed for, eg,
         # relative addressing to work properly.
         addr = rffi.cast(lltype.Signed, addr)
-        
+
         self.cpu.assembler.setup_once()
         self.cpu.assembler.malloc_func_addr = addr
         ofs = symbolic.get_field_token(rstr.STR, 'chars', False)[0]
@@ -108,7 +108,7 @@
         res = self.execute_operation(rop.NEW_ARRAY, [ConstInt(10)],
                                          'ref', descr)
         assert allocs[0] == 10*WORD + ofs + WORD
-        resbuf = self._resbuf(res)            
+        resbuf = self._resbuf(res)
         assert resbuf[ofs/WORD] == 10
 
         # ------------------------------------------------------------
@@ -116,7 +116,7 @@
         res = self.execute_operation(rop.NEW_ARRAY, [BoxInt(10)],
                                          'ref', descr)
         assert allocs[0] == 10*WORD + ofs + WORD
-        resbuf = self._resbuf(res)                        
+        resbuf = self._resbuf(res)
         assert resbuf[ofs/WORD] == 10
 
     def test_stringitems(self):
@@ -146,7 +146,7 @@
                                                      ConstInt(2), BoxInt(38)],
                                'void', descr)
         assert resbuf[itemsofs/WORD + 2] == 38
-        
+
         self.execute_operation(rop.SETARRAYITEM_GC, [res,
                                                      BoxInt(3), BoxInt(42)],
                                'void', descr)
@@ -167,7 +167,7 @@
                                                          BoxInt(2)],
                                    'int', descr)
         assert r.value == 38
-        
+
         r = self.execute_operation(rop.GETARRAYITEM_GC, [res, BoxInt(3)],
                                    'int', descr)
         assert r.value == 42
@@ -226,7 +226,7 @@
         self.execute_operation(rop.SETFIELD_GC, [res, BoxInt(1234)], 'void', ofs_i)
         i = self.execute_operation(rop.GETFIELD_GC, [res], 'int', ofs_i)
         assert i.value == 1234
-        
+
         #u = self.execute_operation(rop.GETFIELD_GC, [res, ofs_u], 'int')
         #assert u.value == 5
         self.execute_operation(rop.SETFIELD_GC, [res, ConstInt(1)], 'void',
@@ -299,7 +299,7 @@
                     else:
                         assert result != execute(self.cpu, None,
                                                  op, None, b).value
-                    
+
 
     def test_stuff_followed_by_guard(self):
         boxes = [(BoxInt(1), BoxInt(0)),
@@ -523,7 +523,7 @@
     def test_debugger_on(self):
         from pypy.tool.logparser import parse_log_file, extract_category
         from pypy.rlib import debug
-        
+
         loop = """
         [i0]
         debug_merge_point('xyz', 0)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_containers.py b/pypy/module/pypyjit/test_pypy_c/test_containers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_containers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_containers.py
@@ -46,7 +46,7 @@
         assert loop.match_by_id("getitem", """
             i28 = call(ConstClass(ll_dict_lookup__dicttablePtr_objectPtr_Signed), p18, p6, i25, descr=...)
             ...
-            p33 = call(ConstClass(ll_get_value__dicttablePtr_Signed), p18, i28, descr=...)
+            p33 = getinteriorfield_gc(p18, i28, <InteriorFieldDescr <>>)
             ...
         """)
 

From noreply at buildbot.pypy.org  Fri Oct 21 00:21:48 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Fri, 21 Oct 2011 00:21:48 +0200 (CEST)
Subject: [pypy-commit] pypy inline-dict-ops: kill boehm special cases in the
	JIT
Message-ID: <20111020222148.BA954820D7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: inline-dict-ops
Changeset: r48285:9fbb0836e52d
Date: 2011-10-20 18:21 -0400
http://bitbucket.org/pypy/pypy/changeset/9fbb0836e52d/

Log:	kill boehm special cases in the JIT

diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py
--- a/pypy/jit/backend/llsupport/gc.py
+++ b/pypy/jit/backend/llsupport/gc.py
@@ -45,6 +45,14 @@
     def freeing_block(self, start, stop):
         pass
 
+    def get_funcptr_for_newarray(self):
+        return llhelper(self.GC_MALLOC_ARRAY, self.malloc_array)
+    def get_funcptr_for_newstr(self):
+        return llhelper(self.GC_MALLOC_STR_UNICODE, self.malloc_str)
+    def get_funcptr_for_newunicode(self):
+        return llhelper(self.GC_MALLOC_STR_UNICODE, self.malloc_unicode)
+
+
     def record_constptrs(self, op, gcrefs_output_list):
         for i in range(op.numargs()):
             v = op.getarg(i)
@@ -105,6 +113,25 @@
         self.GC_MALLOC_ARRAY = lltype.Ptr(lltype.FuncType(
             [lltype.Signed] * 4, llmemory.GCREF))
 
+
+        (str_basesize, str_itemsize, str_ofs_length
+         ) = symbolic.get_array_token(rstr.STR, self.translate_support_code)
+        (unicode_basesize, unicode_itemsize, unicode_ofs_length
+         ) = symbolic.get_array_token(rstr.UNICODE, self.translate_support_code)
+        def malloc_str(length):
+            return self.malloc_array(
+                str_basesize, str_itemsize, str_ofs_length, length
+            )
+        def malloc_unicode(length):
+            return self.malloc_array(
+                unicode_basesize, unicode_itemsize, unicode_ofs_length, length
+            )
+        self.malloc_str = malloc_str
+        self.malloc_unicode = malloc_unicode
+        self.GC_MALLOC_STR_UNICODE = lltype.Ptr(lltype.FuncType(
+            [lltype.Signed], llmemory.GCREF))
+
+
         # on some platform GC_init is required before any other
         # GC_* functions, call it here for the benefit of tests
         # XXX move this to tests
@@ -128,21 +155,10 @@
         return self.malloc_array(basesize, itemsize, ofs_length, num_elem)
 
     def gc_malloc_str(self, num_elem):
-        basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR,
-                                                   self.translate_support_code)
-        assert itemsize == 1
-        size = basesize + num_elem
-        res = self.funcptr_for_new(size)
-        rffi.cast(rffi.CArrayPtr(lltype.Signed), res)[ofs_length/WORD] = num_elem
-        return res
+        return self.malloc_str(num_elem)
 
     def gc_malloc_unicode(self, num_elem):
-        basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE,
-                                                   self.translate_support_code)
-        size = basesize + num_elem * itemsize
-        res = self.funcptr_for_new(size)
-        rffi.cast(rffi.CArrayPtr(lltype.Signed), res)[ofs_length/WORD] = num_elem
-        return res
+        return self.malloc_unicode(num_elem)
 
     def args_for_new(self, sizedescr):
         assert isinstance(sizedescr, BaseSizeDescr)
@@ -157,12 +173,6 @@
     def get_funcptr_for_new(self):
         return self.funcptr_for_new
 
-    def get_funcptr_for_newarray(self):
-        return llhelper(self.GC_MALLOC_ARRAY, self.malloc_array)
-
-    get_funcptr_for_newstr = None
-    get_funcptr_for_newunicode = None
-
     def rewrite_assembler(self, cpu, operations, gcrefs_output_list):
         # record all GCREFs too, because Boehm cannot see them and keep them
         # alive if they end up as constants in the assembler
@@ -766,15 +776,6 @@
     def get_funcptr_for_new(self):
         return llhelper(self.GC_MALLOC_BASIC, self.malloc_basic)
 
-    def get_funcptr_for_newarray(self):
-        return llhelper(self.GC_MALLOC_ARRAY, self.malloc_array)
-
-    def get_funcptr_for_newstr(self):
-        return llhelper(self.GC_MALLOC_STR_UNICODE, self.malloc_str)
-
-    def get_funcptr_for_newunicode(self):
-        return llhelper(self.GC_MALLOC_STR_UNICODE, self.malloc_unicode)
-
     def do_write_barrier(self, gcref_struct, gcref_newptr):
         hdr_addr = llmemory.cast_ptr_to_adr(gcref_struct)
         hdr_addr -= self.gcheaderbuilder.size_gc_header
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1506,6 +1506,7 @@
 
     def genop_newstr(self, op, arglocs, result_loc):
         assert result_loc is eax
+        assert self.malloc_str_func_addr
         self.call(self.malloc_str_func_addr, arglocs, eax)
         self.propagate_memoryerror_if_eax_is_null()
 
diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py
--- a/pypy/jit/backend/x86/regalloc.py
+++ b/pypy/jit/backend/x86/regalloc.py
@@ -973,50 +973,12 @@
             return self._call(op, arglocs)
 
     def consider_newstr(self, op):
-        gc_ll_descr = self.assembler.cpu.gc_ll_descr
-        if gc_ll_descr.get_funcptr_for_newstr is not None:
-            # framework GC
-            loc = self.loc(op.getarg(0))
-            return self._call(op, [loc])
-        # boehm GC (XXX kill the following code at some point)
-        ofs_items, itemsize, ofs = symbolic.get_array_token(rstr.STR, self.translate_support_code)
-        assert itemsize == 1
-        return self._malloc_varsize(ofs_items, ofs, 0, op.getarg(0),
-                                    op.result)
+        loc = self.loc(op.getarg(0))
+        return self._call(op, [loc])
 
     def consider_newunicode(self, op):
-        gc_ll_descr = self.assembler.cpu.gc_ll_descr
-        if gc_ll_descr.get_funcptr_for_newunicode is not None:
-            # framework GC
-            loc = self.loc(op.getarg(0))
-            return self._call(op, [loc])
-        # boehm GC (XXX kill the following code at some point)
-        ofs_items, _, ofs = symbolic.get_array_token(rstr.UNICODE,
-                                                   self.translate_support_code)
-        scale = self._get_unicode_item_scale()
-        return self._malloc_varsize(ofs_items, ofs, scale, op.getarg(0),
-                                    op.result)
-
-    def _malloc_varsize(self, ofs_items, ofs_length, scale, v, res_v):
-        # XXX kill this function at some point
-        if isinstance(v, Box):
-            loc = self.rm.make_sure_var_in_reg(v, [v])
-            tempbox = TempBox()
-            other_loc = self.rm.force_allocate_reg(tempbox, [v])
-            self.assembler.load_effective_addr(loc, ofs_items,scale, other_loc)
-        else:
-            tempbox = None
-            other_loc = imm(ofs_items + (v.getint() << scale))
-        self._call(ResOperation(rop.NEW, [], res_v),
-                   [other_loc], [v])
-        loc = self.rm.make_sure_var_in_reg(v, [res_v])
-        assert self.loc(res_v) == eax
-        # now we have to reload length to some reasonable place
-        self.rm.possibly_free_var(v)
-        if tempbox is not None:
-            self.rm.possibly_free_var(tempbox)
-        self.PerformDiscard(ResOperation(rop.SETFIELD_GC, [None, None], None),
-                            [eax, imm(ofs_length), imm(WORD), loc])
+        loc = self.loc(op.getarg(0))
+        return self._call(op, [loc])
 
     def consider_new_array(self, op):
         gc_ll_descr = self.assembler.cpu.gc_ll_descr

From noreply at buildbot.pypy.org  Fri Oct 21 01:14:04 2011
From: noreply at buildbot.pypy.org (justinpeel)
Date: Fri, 21 Oct 2011 01:14:04 +0200 (CEST)
Subject: [pypy-commit] pypy rgc-mem-pressure: small fixes. memory leak is
 now completely stopped.
Message-ID: <20111020231404.184D7820D7@wyvern.cs.uni-duesseldorf.de>

Author: Justin Peel <notmuchtotell at gmail.com>
Branch: rgc-mem-pressure
Changeset: r48286:1376c2803ce7
Date: 2011-10-20 17:13 -0600
http://bitbucket.org/pypy/pypy/changeset/1376c2803ce7/

Log:	small fixes. memory leak is now completely stopped.

diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py
--- a/pypy/module/_hashlib/interp_hashlib.py
+++ b/pypy/module/_hashlib/interp_hashlib.py
@@ -34,10 +34,10 @@
         rgc.add_memory_pressure(HASH_MALLOC_SIZE + self._digest_size())
         self.ctx = ctx
 
-    def initdigest(self):
-        digest = ropenssl.EVP_get_digestbyname(self.name)
+    def initdigest(self, space, name):
+        digest = ropenssl.EVP_get_digestbyname(name)
         if not digest:
-            raise OperationError(space.w_Value,
+            raise OperationError(space.w_ValueError,
                                  space.wrap("unknown hash function"))
         ropenssl.EVP_DigestInit(self.ctx, digest)
 
@@ -141,7 +141,7 @@
 @unwrap_spec(name=str, string='bufferstr')
 def new(space, name, string=''):
     w_hash = W_Hash(space, name)
-    w_hash.initdigest()
+    w_hash.initdigest(space, name)
     w_hash.update(space, string)
     return space.wrap(w_hash)
 

From noreply at buildbot.pypy.org  Fri Oct 21 01:28:30 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Fri, 21 Oct 2011 01:28:30 +0200 (CEST)
Subject: [pypy-commit] pypy inline-dict-ops: fix various tests
Message-ID: <20111020232830.44459820D7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: inline-dict-ops
Changeset: r48287:43a10843f902
Date: 2011-10-20 19:28 -0400
http://bitbucket.org/pypy/pypy/changeset/43a10843f902/

Log:	fix various tests

diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py
--- a/pypy/jit/backend/llsupport/gc.py
+++ b/pypy/jit/backend/llsupport/gc.py
@@ -107,6 +107,8 @@
         def malloc_array(basesize, itemsize, ofs_length, num_elem):
             size = basesize + itemsize * num_elem
             res = self.funcptr_for_new(size)
+            if not res:
+                return res
             rffi.cast(rffi.CArrayPtr(lltype.Signed), res)[ofs_length/WORD] = num_elem
             return res
         self.malloc_array = malloc_array
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1506,7 +1506,6 @@
 
     def genop_newstr(self, op, arglocs, result_loc):
         assert result_loc is eax
-        assert self.malloc_str_func_addr
         self.call(self.malloc_str_func_addr, arglocs, eax)
         self.propagate_memoryerror_if_eax_is_null()
 
diff --git a/pypy/jit/backend/x86/test/test_runner.py b/pypy/jit/backend/x86/test/test_runner.py
--- a/pypy/jit/backend/x86/test/test_runner.py
+++ b/pypy/jit/backend/x86/test/test_runner.py
@@ -72,19 +72,13 @@
 
         allocs = [None]
         all = []
+        orig_new = self.cpu.gc_ll_descr.funcptr_for_new
         def f(size):
             allocs.insert(0, size)
-            buf = ctypes.create_string_buffer(size)
-            all.append(buf)
-            return ctypes.cast(buf, ctypes.c_void_p).value
-        func = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int)(f)
-        addr = ctypes.cast(func, ctypes.c_void_p).value
-        # ctypes produces an unsigned value. We need it to be signed for, eg,
-        # relative addressing to work properly.
-        addr = rffi.cast(lltype.Signed, addr)
+            return orig_new(size)
 
         self.cpu.assembler.setup_once()
-        self.cpu.assembler.malloc_func_addr = addr
+        self.cpu.gc_ll_descr.funcptr_for_new = f
         ofs = symbolic.get_field_token(rstr.STR, 'chars', False)[0]
 
         res = self.execute_operation(rop.NEWSTR, [ConstInt(7)], 'ref')

From noreply at buildbot.pypy.org  Fri Oct 21 01:32:27 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Fri, 21 Oct 2011 01:32:27 +0200 (CEST)
Subject: [pypy-commit] pypy inline-dict-ops: revert changes to debug, they
 break tons of tests. fijal: feel free to revert if you fix tests ; )
Message-ID: <20111020233227.21410820D7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: inline-dict-ops
Changeset: r48288:99cce3fdfcfd
Date: 2011-10-20 19:32 -0400
http://bitbucket.org/pypy/pypy/changeset/99cce3fdfcfd/

Log:	revert changes to debug, they break tons of tests. fijal: feel free
	to revert if you fix tests ;)

diff --git a/pypy/rlib/debug.py b/pypy/rlib/debug.py
--- a/pypy/rlib/debug.py
+++ b/pypy/rlib/debug.py
@@ -1,26 +1,6 @@
-import sys, time, os
+import sys, time
 from pypy.rpython.extregistry import ExtRegistryEntry
 
-class DebugState(object):
-    def __init__(self):
-        self.prefixes = os.environ.get('PYPYDEFAULTLOG', 'all').split(',')
-        if self.prefixes == ['']:
-            self.prefixes = []
-        self.categories = []
-
-    def should_print(self, category=None):
-        if category is None:
-            category = self.categories[-1]
-        if self.prefixes != ['all']:
-            for prefix in self.prefixes:
-                if category.startswith(prefix):
-                    break
-            else:
-                return False
-        return True
-
-debug_state = DebugState() # a global state object
-
 def ll_assert(x, msg):
     """After translation to C, this becomes an RPyAssert."""
     assert x, msg
@@ -76,8 +56,6 @@
                   # or compatible
 
 def debug_print(*args):
-    if not debug_state.should_print():
-        return
     for arg in args:
         print >> sys.stderr, arg,
     print >> sys.stderr
@@ -108,25 +86,18 @@
     _stop_colors = ""
 
 def debug_start(category):
-    debug_state.categories.append(category)
-    if _log is not None:
-        _log.debug_start(category)
-    if not debug_state.should_print(category):
-        return
     c = int(time.clock() * 100)
     print >> sys.stderr, '%s[%x] {%s%s' % (_start_colors_1, c,
                                            category, _stop_colors)
-    
+    if _log is not None:
+        _log.debug_start(category)
+
 def debug_stop(category):
-    if _log is not None:
-        _log.debug_stop(category)
-    last = debug_state.categories.pop()
-    assert category == last
-    if not debug_state.should_print(category):
-        return
     c = int(time.clock() * 100)
     print >> sys.stderr, '%s[%x] %s}%s' % (_start_colors_2, c,
                                            category, _stop_colors)
+    if _log is not None:
+        _log.debug_stop(category)
 
 class Entry(ExtRegistryEntry):
     _about_ = debug_start, debug_stop

From noreply at buildbot.pypy.org  Fri Oct 21 02:44:02 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Fri, 21 Oct 2011 02:44:02 +0200 (CEST)
Subject: [pypy-commit] pypy inline-dict-ops: fix this test in full
Message-ID: <20111021004402.0EDAC820D7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: inline-dict-ops
Changeset: r48289:99db09a93a34
Date: 2011-10-20 20:43 -0400
http://bitbucket.org/pypy/pypy/changeset/99db09a93a34/

Log:	fix this test in full

diff --git a/pypy/module/pypyjit/test_pypy_c/test_containers.py b/pypy/module/pypyjit/test_pypy_c/test_containers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_containers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_containers.py
@@ -46,7 +46,7 @@
         assert loop.match_by_id("getitem", """
             i28 = call(ConstClass(ll_dict_lookup__dicttablePtr_objectPtr_Signed), p18, p6, i25, descr=...)
             ...
-            p33 = getinteriorfield_gc(p18, i28, <InteriorFieldDescr <>>)
+            p33 = getinteriorfield_gc(p18, i28, <InteriorFieldDescr <GcPtrFieldDescr dictentry.value .*>>)
             ...
         """)
 

From noreply at buildbot.pypy.org  Fri Oct 21 03:49:24 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Fri, 21 Oct 2011 03:49:24 +0200 (CEST)
Subject: [pypy-commit] pypy inline-dict-ops: fix tests on 32-bit
Message-ID: <20111021014924.DC3D0820D7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: inline-dict-ops
Changeset: r48290:3e352268b119
Date: 2011-10-20 21:49 -0400
http://bitbucket.org/pypy/pypy/changeset/3e352268b119/

Log:	fix tests on 32-bit

diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -111,7 +111,7 @@
         self.cpu.set_future_value_int(0, 2)
         fail = self.cpu.execute_token(looptoken)
         res = self.cpu.get_latest_value_int(0)
-        assert res == 3        
+        assert res == 3
         assert fail.identifier == 1
 
     def test_compile_loop(self):
@@ -127,7 +127,7 @@
             ]
         inputargs = [i0]
         operations[2].setfailargs([i1])
-        
+
         self.cpu.compile_loop(inputargs, operations, looptoken)
         self.cpu.set_future_value_int(0, 2)
         fail = self.cpu.execute_token(looptoken)
@@ -148,7 +148,7 @@
             ]
         inputargs = [i0]
         operations[2].setfailargs([None, None, i1, None])
-        
+
         self.cpu.compile_loop(inputargs, operations, looptoken)
         self.cpu.set_future_value_int(0, 2)
         fail = self.cpu.execute_token(looptoken)
@@ -372,7 +372,7 @@
         for opnum, boxargs, retvalue in get_int_tests():
             res = self.execute_operation(opnum, boxargs, 'int')
             assert res.value == retvalue
-        
+
     def test_float_operations(self):
         from pypy.jit.metainterp.test.test_executor import get_float_tests
         for opnum, boxargs, rettype, retvalue in get_float_tests(self.cpu):
@@ -438,7 +438,7 @@
 
     def test_ovf_operations_reversed(self):
         self.test_ovf_operations(reversed=True)
-        
+
     def test_bh_call(self):
         cpu = self.cpu
         #
@@ -503,7 +503,7 @@
                                          [funcbox, BoxInt(num), BoxInt(num)],
                                          'int', descr=dyn_calldescr)
             assert res.value == 2 * num
-            
+
 
         if cpu.supports_floats:
             def func(f0, f1, f2, f3, f4, f5, f6, i0, i1, f7, f8, f9):
@@ -543,7 +543,7 @@
         funcbox = self.get_funcbox(self.cpu, func_ptr)
         res = self.execute_operation(rop.CALL, [funcbox] + map(BoxInt, args), 'int', descr=calldescr)
         assert res.value == func(*args)
-        
+
     def test_call_stack_alignment(self):
         # test stack alignment issues, notably for Mac OS/X.
         # also test the ordering of the arguments.
@@ -615,7 +615,7 @@
         res = self.execute_operation(rop.GETFIELD_GC, [t_box],
                                      'int', descr=shortdescr)
         assert res.value == 1331
-        
+
         #
         u_box, U_box = self.alloc_instance(self.U)
         fielddescr2 = self.cpu.fielddescrof(self.S, 'next')
@@ -695,7 +695,7 @@
 
     def test_failing_guard_class(self):
         t_box, T_box = self.alloc_instance(self.T)
-        u_box, U_box = self.alloc_instance(self.U)        
+        u_box, U_box = self.alloc_instance(self.U)
         null_box = self.null_instance()
         for opname, args in [(rop.GUARD_CLASS, [t_box, U_box]),
                              (rop.GUARD_CLASS, [u_box, T_box]),
@@ -787,7 +787,7 @@
         r = self.execute_operation(rop.GETARRAYITEM_GC, [a_box, BoxInt(3)],
                                    'int', descr=arraydescr)
         assert r.value == 160
-        
+
         #
         if isinstance(A, lltype.GcArray):
             A = lltype.Ptr(A)
@@ -891,7 +891,7 @@
         vdescr = self.cpu.interiorfielddescrof(A, 'v')
         pdescr = self.cpu.interiorfielddescrof(A, 'p')
         self.execute_operation(rop.SETINTERIORFIELD_GC, [a_box, BoxInt(3),
-                                                         BoxFloat(1.5)],
+                                                         boxfloat(1.5)],
                                'void', descr=kdescr)
         f = self.cpu.bh_getinteriorfield_gc_f(a_box.getref_base(), 3, kdescr)
         assert f == 1.5
@@ -1441,7 +1441,7 @@
         addr = llmemory.cast_ptr_to_adr(func_ptr)
         return ConstInt(heaptracker.adr2int(addr))
 
-    
+
     MY_VTABLE = rclass.OBJECT_VTABLE    # for tests only
 
     S = lltype.GcForwardReference()
@@ -2356,7 +2356,7 @@
         for opname, arg, res in ops:
             self.execute_operation(opname, [arg], 'void')
             assert self.guard_failed == res
-        
+
         lltype.free(x, flavor='raw')
 
     def test_assembler_call(self):
@@ -2436,7 +2436,7 @@
         FakeJitDriverSD.portal_calldescr = self.cpu.calldescrof(
             lltype.Ptr(lltype.FuncType(ARGS, RES)), ARGS, RES,
             EffectInfo.MOST_GENERAL)
-        
+
         ops = '''
         [f0, f1]
         f2 = float_add(f0, f1)
@@ -2527,7 +2527,7 @@
         FakeJitDriverSD.portal_calldescr = self.cpu.calldescrof(
             lltype.Ptr(lltype.FuncType(ARGS, RES)), ARGS, RES,
             EffectInfo.MOST_GENERAL)
-        
+
         ops = '''
         [f0, f1]
         f2 = float_add(f0, f1)
@@ -2978,4 +2978,4 @@
 
     def alloc_unicode(self, unicode):
         py.test.skip("implement me")
-    
+

From noreply at buildbot.pypy.org  Fri Oct 21 03:55:17 2011
From: noreply at buildbot.pypy.org (justinpeel)
Date: Fri, 21 Oct 2011 03:55:17 +0200 (CEST)
Subject: [pypy-commit] pypy rgc-mem-pressure: improve a test for
 rgc.add_memory_pressure. not sure that it is really doing what we want it
 to do.
Message-ID: <20111021015517.39D68820D7@wyvern.cs.uni-duesseldorf.de>

Author: Justin Peel <notmuchtotell at gmail.com>
Branch: rgc-mem-pressure
Changeset: r48291:eac625745920
Date: 2011-10-20 19:54 -0600
http://bitbucket.org/pypy/pypy/changeset/eac625745920/

Log:	improve a test for rgc.add_memory_pressure. not sure that it is
	really doing what we want it to do.

diff --git a/pypy/translator/c/test/test_newgc.py b/pypy/translator/c/test/test_newgc.py
--- a/pypy/translator/c/test/test_newgc.py
+++ b/pypy/translator/c/test/test_newgc.py
@@ -1459,14 +1459,18 @@
         assert res == -99997
 
     def define_nongc_opaque_attached_to_gc(cls):
+        from pypy.module._hashlib.interp_hashlib import HASH_MALLOC_SIZE
+        from pypy.rlib import rgc, ropenssl
         from pypy.rpython.lltypesystem import rffi
-        from pypy.rlib import ropenssl
+
         class A:
             def __init__(self):
                 self.ctx = lltype.malloc(ropenssl.EVP_MD_CTX.TO,
                     flavor='raw')
                 digest = ropenssl.EVP_get_digestbyname('sha1')
                 ropenssl.EVP_DigestInit(self.ctx, digest)
+                rgc.add_memory_pressure(HASH_MALLOC_SIZE + 64)
+
             def __del__(self):
                 ropenssl.EVP_MD_CTX_cleanup(self.ctx)
                 lltype.free(self.ctx, flavor='raw')

From noreply at buildbot.pypy.org  Fri Oct 21 04:02:17 2011
From: noreply at buildbot.pypy.org (justinpeel)
Date: Fri, 21 Oct 2011 04:02:17 +0200 (CEST)
Subject: [pypy-commit] pypy rgc-mem-pressure: merge in default
Message-ID: <20111021020217.1F1A3820D7@wyvern.cs.uni-duesseldorf.de>

Author: Justin Peel <notmuchtotell at gmail.com>
Branch: rgc-mem-pressure
Changeset: r48292:170dff2f43d2
Date: 2011-10-20 20:01 -0600
http://bitbucket.org/pypy/pypy/changeset/170dff2f43d2/

Log:	merge in default

diff --git a/lib-python/modified-2.7/json/encoder.py b/lib-python/modified-2.7/json/encoder.py
--- a/lib-python/modified-2.7/json/encoder.py
+++ b/lib-python/modified-2.7/json/encoder.py
@@ -2,14 +2,7 @@
 """
 import re
 
-try:
-    from _json import encode_basestring_ascii as c_encode_basestring_ascii
-except ImportError:
-    c_encode_basestring_ascii = None
-try:
-    from _json import make_encoder as c_make_encoder
-except ImportError:
-    c_make_encoder = None
+from __pypy__.builders import StringBuilder, UnicodeBuilder
 
 ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]')
 ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])')
@@ -24,8 +17,7 @@
     '\t': '\\t',
 }
 for i in range(0x20):
-    ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i))
-    #ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,))
+    ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,))
 
 # Assume this produces an infinity on all machines (probably not guaranteed)
 INFINITY = float('1e66666')
@@ -37,10 +29,9 @@
     """
     def replace(match):
         return ESCAPE_DCT[match.group(0)]
-    return '"' + ESCAPE.sub(replace, s) + '"'
+    return ESCAPE.sub(replace, s)
 
-
-def py_encode_basestring_ascii(s):
+def encode_basestring_ascii(s):
     """Return an ASCII-only JSON representation of a Python string
 
     """
@@ -53,20 +44,18 @@
         except KeyError:
             n = ord(s)
             if n < 0x10000:
-                return '\\u{0:04x}'.format(n)
-                #return '\\u%04x' % (n,)
+                return '\\u%04x' % (n,)
             else:
                 # surrogate pair
                 n -= 0x10000
                 s1 = 0xd800 | ((n >> 10) & 0x3ff)
                 s2 = 0xdc00 | (n & 0x3ff)
-                return '\\u{0:04x}\\u{1:04x}'.format(s1, s2)
-                #return '\\u%04x\\u%04x' % (s1, s2)
-    return '"' + str(ESCAPE_ASCII.sub(replace, s)) + '"'
-
-
-encode_basestring_ascii = (
-    c_encode_basestring_ascii or py_encode_basestring_ascii)
+                return '\\u%04x\\u%04x' % (s1, s2)
+    if ESCAPE_ASCII.search(s):
+        return str(ESCAPE_ASCII.sub(replace, s))
+    return s
+py_encode_basestring_ascii = lambda s: '"' + encode_basestring_ascii(s) + '"'
+c_encode_basestring_ascii = None
 
 class JSONEncoder(object):
     """Extensible JSON <http://json.org> encoder for Python data structures.
@@ -147,6 +136,17 @@
 
         self.skipkeys = skipkeys
         self.ensure_ascii = ensure_ascii
+        if ensure_ascii:
+            self.encoder = encode_basestring_ascii
+        else:
+            self.encoder = encode_basestring
+        if encoding != 'utf-8':
+            orig_encoder = self.encoder
+            def encoder(o):
+                if isinstance(o, str):
+                    o = o.decode(encoding)
+                return orig_encoder(o)
+            self.encoder = encoder
         self.check_circular = check_circular
         self.allow_nan = allow_nan
         self.sort_keys = sort_keys
@@ -184,24 +184,126 @@
         '{"foo": ["bar", "baz"]}'
 
         """
-        # This is for extremely simple cases and benchmarks.
+        if self.check_circular:
+            markers = {}
+        else:
+            markers = None
+        if self.ensure_ascii:
+            builder = StringBuilder()
+        else:
+            builder = UnicodeBuilder()
+        self._encode(o, markers, builder, 0)
+        return builder.build()
+
+    def _emit_indent(self, builder, _current_indent_level):
+        if self.indent is not None:
+            _current_indent_level += 1
+            newline_indent = '\n' + (' ' * (self.indent *
+                                            _current_indent_level))
+            separator = self.item_separator + newline_indent
+            builder.append(newline_indent)
+        else:
+            separator = self.item_separator
+        return separator, _current_indent_level
+
+    def _emit_unindent(self, builder, _current_indent_level):
+        if self.indent is not None:
+            builder.append('\n')
+            builder.append(' ' * (self.indent * (_current_indent_level - 1)))
+
+    def _encode(self, o, markers, builder, _current_indent_level):
         if isinstance(o, basestring):
-            if isinstance(o, str):
-                _encoding = self.encoding
-                if (_encoding is not None
-                        and not (_encoding == 'utf-8')):
-                    o = o.decode(_encoding)
-            if self.ensure_ascii:
-                return encode_basestring_ascii(o)
+            builder.append('"')
+            builder.append(self.encoder(o))
+            builder.append('"')
+        elif o is None:
+            builder.append('null')
+        elif o is True:
+            builder.append('true')
+        elif o is False:
+            builder.append('false')
+        elif isinstance(o, (int, long)):
+            builder.append(str(o))
+        elif isinstance(o, float):
+            builder.append(self._floatstr(o))
+        elif isinstance(o, (list, tuple)):
+            if not o:
+                builder.append('[]')
+                return
+            self._encode_list(o, markers, builder, _current_indent_level)
+        elif isinstance(o, dict):
+            if not o:
+                builder.append('{}')
+                return
+            self._encode_dict(o, markers, builder, _current_indent_level)
+        else:
+            self._mark_markers(markers, o)
+            res = self.default(o)
+            self._encode(res, markers, builder, _current_indent_level)
+            self._remove_markers(markers, o)
+            return res
+
+    def _encode_list(self, l, markers, builder, _current_indent_level):
+        self._mark_markers(markers, l)
+        builder.append('[')
+        first = True
+        separator, _current_indent_level = self._emit_indent(builder,
+                                                      _current_indent_level)
+        for elem in l:
+            if first:
+                first = False
             else:
-                return encode_basestring(o)
-        # This doesn't pass the iterator directly to ''.join() because the
-        # exceptions aren't as detailed.  The list call should be roughly
-        # equivalent to the PySequence_Fast that ''.join() would do.
-        chunks = self.iterencode(o, _one_shot=True)
-        if not isinstance(chunks, (list, tuple)):
-            chunks = list(chunks)
-        return ''.join(chunks)
+                builder.append(separator)
+            self._encode(elem, markers, builder, _current_indent_level)
+            del elem # XXX grumble
+        self._emit_unindent(builder, _current_indent_level)
+        builder.append(']')
+        self._remove_markers(markers, l)
+
+    def _encode_dict(self, d, markers, builder, _current_indent_level):
+        self._mark_markers(markers, d)
+        first = True
+        builder.append('{')
+        separator, _current_indent_level = self._emit_indent(builder,
+                                                         _current_indent_level)
+        if self.sort_keys:
+            items = sorted(d.items(), key=lambda kv: kv[0])
+        else:
+            items = d.iteritems()
+
+        for key, v in items:
+            if first:
+                first = False
+            else:
+                builder.append(separator)
+            if isinstance(key, basestring):
+                pass
+            # JavaScript is weakly typed for these, so it makes sense to
+            # also allow them.  Many encoders seem to do something like this.
+            elif isinstance(key, float):
+                key = self._floatstr(key)
+            elif key is True:
+                key = 'true'
+            elif key is False:
+                key = 'false'
+            elif key is None:
+                key = 'null'
+            elif isinstance(key, (int, long)):
+                key = str(key)
+            elif self.skipkeys:
+                continue
+            else:
+                raise TypeError("key " + repr(key) + " is not a string")
+            builder.append('"')
+            builder.append(self.encoder(key))
+            builder.append('"')
+            builder.append(self.key_separator)
+            self._encode(v, markers, builder, _current_indent_level)
+            del key
+            del v # XXX grumble
+        self._emit_unindent(builder, _current_indent_level)
+        builder.append('}')
+        self._remove_markers(markers, d)
 
     def iterencode(self, o, _one_shot=False):
         """Encode the given object and yield each string
@@ -217,86 +319,54 @@
             markers = {}
         else:
             markers = None
-        if self.ensure_ascii:
-            _encoder = encode_basestring_ascii
+        return self._iterencode(o, markers, 0)
+
+    def _floatstr(self, o):
+        # Check for specials.  Note that this type of test is processor
+        # and/or platform-specific, so do tests which don't depend on the
+        # internals.
+
+        if o != o:
+            text = 'NaN'
+        elif o == INFINITY:
+            text = 'Infinity'
+        elif o == -INFINITY:
+            text = '-Infinity'
         else:
-            _encoder = encode_basestring
-        if self.encoding != 'utf-8':
-            def _encoder(o, _orig_encoder=_encoder, _encoding=self.encoding):
-                if isinstance(o, str):
-                    o = o.decode(_encoding)
-                return _orig_encoder(o)
+            return FLOAT_REPR(o)
 
-        def floatstr(o, allow_nan=self.allow_nan,
-                _repr=FLOAT_REPR, _inf=INFINITY, _neginf=-INFINITY):
-            # Check for specials.  Note that this type of test is processor
-            # and/or platform-specific, so do tests which don't depend on the
-            # internals.
+        if not self.allow_nan:
+            raise ValueError(
+                "Out of range float values are not JSON compliant: " +
+                repr(o))
 
-            if o != o:
-                text = 'NaN'
-            elif o == _inf:
-                text = 'Infinity'
-            elif o == _neginf:
-                text = '-Infinity'
-            else:
-                return _repr(o)
+        return text
 
-            if not allow_nan:
-                raise ValueError(
-                    "Out of range float values are not JSON compliant: " +
-                    repr(o))
+    def _mark_markers(self, markers, o):
+        if markers is not None:
+            if id(o) in markers:
+                raise ValueError("Circular reference detected")
+            markers[id(o)] = None
 
-            return text
+    def _remove_markers(self, markers, o):
+        if markers is not None:
+            del markers[id(o)]
 
-
-        if (_one_shot and c_make_encoder is not None
-                and not self.indent and not self.sort_keys):
-            _iterencode = c_make_encoder(
-                markers, self.default, _encoder, self.indent,
-                self.key_separator, self.item_separator, self.sort_keys,
-                self.skipkeys, self.allow_nan)
-        else:
-            _iterencode = _make_iterencode(
-                markers, self.default, _encoder, self.indent, floatstr,
-                self.key_separator, self.item_separator, self.sort_keys,
-                self.skipkeys, _one_shot)
-        return _iterencode(o, 0)
-
-def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,
-        _key_separator, _item_separator, _sort_keys, _skipkeys, _one_shot,
-        ## HACK: hand-optimized bytecode; turn globals into locals
-        ValueError=ValueError,
-        basestring=basestring,
-        dict=dict,
-        float=float,
-        id=id,
-        int=int,
-        isinstance=isinstance,
-        list=list,
-        long=long,
-        str=str,
-        tuple=tuple,
-    ):
-
-    def _iterencode_list(lst, _current_indent_level):
+    def _iterencode_list(self, lst, markers, _current_indent_level):
         if not lst:
             yield '[]'
             return
-        if markers is not None:
-            markerid = id(lst)
-            if markerid in markers:
-                raise ValueError("Circular reference detected")
-            markers[markerid] = lst
+        self._mark_markers(markers, lst)
         buf = '['
-        if _indent is not None:
+        if self.indent is not None:
             _current_indent_level += 1
-            newline_indent = '\n' + (' ' * (_indent * _current_indent_level))
-            separator = _item_separator + newline_indent
+            newline_indent = '\n' + (' ' * (self.indent *
+                                            _current_indent_level))
+            separator = self.item_separator + newline_indent
             buf += newline_indent
         else:
             newline_indent = None
-            separator = _item_separator
+            separator = self.item_separator
         first = True
         for value in lst:
             if first:
@@ -304,7 +374,7 @@
             else:
                 buf = separator
             if isinstance(value, basestring):
-                yield buf + _encoder(value)
+                yield buf + '"' + self.encoder(value) + '"'
             elif value is None:
                 yield buf + 'null'
             elif value is True:
@@ -314,44 +384,43 @@
             elif isinstance(value, (int, long)):
                 yield buf + str(value)
             elif isinstance(value, float):
-                yield buf + _floatstr(value)
+                yield buf + self._floatstr(value)
             else:
                 yield buf
                 if isinstance(value, (list, tuple)):
-                    chunks = _iterencode_list(value, _current_indent_level)
+                    chunks = self._iterencode_list(value, markers,
+                                                   _current_indent_level)
                 elif isinstance(value, dict):
-                    chunks = _iterencode_dict(value, _current_indent_level)
+                    chunks = self._iterencode_dict(value, markers,
+                                                   _current_indent_level)
                 else:
-                    chunks = _iterencode(value, _current_indent_level)
+                    chunks = self._iterencode(value, markers,
+                                              _current_indent_level)
                 for chunk in chunks:
                     yield chunk
         if newline_indent is not None:
             _current_indent_level -= 1
-            yield '\n' + (' ' * (_indent * _current_indent_level))
+            yield '\n' + (' ' * (self.indent * _current_indent_level))
         yield ']'
-        if markers is not None:
-            del markers[markerid]
+        self._remove_markers(markers, lst)
 
-    def _iterencode_dict(dct, _current_indent_level):
+    def _iterencode_dict(self, dct, markers, _current_indent_level):
         if not dct:
             yield '{}'
             return
-        if markers is not None:
-            markerid = id(dct)
-            if markerid in markers:
-                raise ValueError("Circular reference detected")
-            markers[markerid] = dct
+        self._mark_markers(markers, dct)
         yield '{'
-        if _indent is not None:
+        if self.indent is not None:
             _current_indent_level += 1
-            newline_indent = '\n' + (' ' * (_indent * _current_indent_level))
-            item_separator = _item_separator + newline_indent
+            newline_indent = '\n' + (' ' * (self.indent *
+                                            _current_indent_level))
+            item_separator = self.item_separator + newline_indent
             yield newline_indent
         else:
             newline_indent = None
-            item_separator = _item_separator
+            item_separator = self.item_separator
         first = True
-        if _sort_keys:
+        if self.sort_keys:
             items = sorted(dct.items(), key=lambda kv: kv[0])
         else:
             items = dct.iteritems()
@@ -361,7 +430,7 @@
             # JavaScript is weakly typed for these, so it makes sense to
             # also allow them.  Many encoders seem to do something like this.
             elif isinstance(key, float):
-                key = _floatstr(key)
+                key = self._floatstr(key)
             elif key is True:
                 key = 'true'
             elif key is False:
@@ -370,7 +439,7 @@
                 key = 'null'
             elif isinstance(key, (int, long)):
                 key = str(key)
-            elif _skipkeys:
+            elif self.skipkeys:
                 continue
             else:
                 raise TypeError("key " + repr(key) + " is not a string")
@@ -378,10 +447,10 @@
                 first = False
             else:
                 yield item_separator
-            yield _encoder(key)
-            yield _key_separator
+            yield '"' + self.encoder(key) + '"'
+            yield self.key_separator
             if isinstance(value, basestring):
-                yield _encoder(value)
+                yield '"' + self.encoder(value) + '"'
             elif value is None:
                 yield 'null'
             elif value is True:
@@ -391,26 +460,28 @@
             elif isinstance(value, (int, long)):
                 yield str(value)
             elif isinstance(value, float):
-                yield _floatstr(value)
+                yield self._floatstr(value)
             else:
                 if isinstance(value, (list, tuple)):
-                    chunks = _iterencode_list(value, _current_indent_level)
+                    chunks = self._iterencode_list(value, markers,
+                                                   _current_indent_level)
                 elif isinstance(value, dict):
-                    chunks = _iterencode_dict(value, _current_indent_level)
+                    chunks = self._iterencode_dict(value, markers,
+                                                   _current_indent_level)
                 else:
-                    chunks = _iterencode(value, _current_indent_level)
+                    chunks = self._iterencode(value, markers,
+                                              _current_indent_level)
                 for chunk in chunks:
                     yield chunk
         if newline_indent is not None:
             _current_indent_level -= 1
-            yield '\n' + (' ' * (_indent * _current_indent_level))
+            yield '\n' + (' ' * (self.indent * _current_indent_level))
         yield '}'
-        if markers is not None:
-            del markers[markerid]
+        self._remove_markers(markers, dct)
 
-    def _iterencode(o, _current_indent_level):
+    def _iterencode(self, o, markers, _current_indent_level):
         if isinstance(o, basestring):
-            yield _encoder(o)
+            yield '"' + self.encoder(o) + '"'
         elif o is None:
             yield 'null'
         elif o is True:
@@ -420,23 +491,19 @@
         elif isinstance(o, (int, long)):
             yield str(o)
         elif isinstance(o, float):
-            yield _floatstr(o)
+            yield self._floatstr(o)
         elif isinstance(o, (list, tuple)):
-            for chunk in _iterencode_list(o, _current_indent_level):
+            for chunk in self._iterencode_list(o, markers,
+                                               _current_indent_level):
                 yield chunk
         elif isinstance(o, dict):
-            for chunk in _iterencode_dict(o, _current_indent_level):
+            for chunk in self._iterencode_dict(o, markers,
+                                               _current_indent_level):
                 yield chunk
         else:
-            if markers is not None:
-                markerid = id(o)
-                if markerid in markers:
-                    raise ValueError("Circular reference detected")
-                markers[markerid] = o
-            o = _default(o)
-            for chunk in _iterencode(o, _current_indent_level):
+            self._mark_markers(markers, o)
+            obj = self.default(o)
+            for chunk in self._iterencode(obj, markers,
+                                          _current_indent_level):
                 yield chunk
-            if markers is not None:
-                del markers[markerid]
-
-    return _iterencode
+            self._remove_markers(markers, o)
diff --git a/lib-python/modified-2.7/json/tests/test_unicode.py b/lib-python/modified-2.7/json/tests/test_unicode.py
--- a/lib-python/modified-2.7/json/tests/test_unicode.py
+++ b/lib-python/modified-2.7/json/tests/test_unicode.py
@@ -80,3 +80,9 @@
         self.assertEqual(type(json.loads(u'["a"]')[0]), unicode)
         # Issue 10038.
         self.assertEqual(type(json.loads('"foo"')), unicode)
+
+    def test_encode_not_utf_8(self):
+        self.assertEqual(json.dumps('\xb1\xe6', encoding='iso8859-2'),
+                         '"\\u0105\\u0107"')
+        self.assertEqual(json.dumps(['\xb1\xe6'], encoding='iso8859-2'),
+                         '["\\u0105\\u0107"]')
diff --git a/lib_pypy/pyrepl/readline.py b/lib_pypy/pyrepl/readline.py
--- a/lib_pypy/pyrepl/readline.py
+++ b/lib_pypy/pyrepl/readline.py
@@ -395,9 +395,21 @@
     _wrapper.f_in = f_in
     _wrapper.f_out = f_out
 
-    if hasattr(sys, '__raw_input__'):    # PyPy
-        _old_raw_input = sys.__raw_input__
+    if '__pypy__' in sys.builtin_module_names:    # PyPy
+
+        def _old_raw_input(prompt=''):
+            # sys.__raw_input__() is only called when stdin and stdout are
+            # as expected and are ttys.  If it is the case, then get_reader()
+            # should not really fail in _wrapper.raw_input().  If it still
+            # does, then we will just cancel the redirection and call again
+            # the built-in raw_input().
+            try:
+                del sys.__raw_input__
+            except AttributeError:
+                pass
+            return raw_input(prompt)
         sys.__raw_input__ = _wrapper.raw_input
+
     else:
         # this is not really what readline.c does.  Better than nothing I guess
         import __builtin__
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -72,6 +72,7 @@
     del working_modules['fcntl']  # LOCK_NB not defined
     del working_modules["_minimal_curses"]
     del working_modules["termios"]
+    del working_modules["_multiprocessing"]   # depends on rctime
 
 
 
diff --git a/pypy/module/__builtin__/__init__.py b/pypy/module/__builtin__/__init__.py
--- a/pypy/module/__builtin__/__init__.py
+++ b/pypy/module/__builtin__/__init__.py
@@ -23,6 +23,7 @@
         'map'           : 'app_functional.map',
         'reduce'        : 'app_functional.reduce',
         'filter'        : 'app_functional.filter',
+        'zip'           : 'app_functional.zip',
         'vars'          : 'app_inspect.vars',
         'dir'           : 'app_inspect.dir',
 
@@ -89,7 +90,6 @@
         'enumerate'     : 'functional.W_Enumerate',
         'min'           : 'functional.min',
         'max'           : 'functional.max',
-        'zip'           : 'functional.zip',
         'reversed'      : 'functional.reversed',
         'super'         : 'descriptor.W_Super',
         'staticmethod'  : 'descriptor.StaticMethod',
diff --git a/pypy/module/__builtin__/app_functional.py b/pypy/module/__builtin__/app_functional.py
--- a/pypy/module/__builtin__/app_functional.py
+++ b/pypy/module/__builtin__/app_functional.py
@@ -162,4 +162,21 @@
         item = seq[i]
         if func(item):
             result.append(item)
-    return tuple(result)
\ No newline at end of file
+    return tuple(result)
+
+def zip(*sequences):
+    """zip(seq1 [, seq2 [...]]) -> [(seq1[0], seq2[0] ...), (...)]
+
+Return a list of tuples, where each tuple contains the i-th element
+from each of the argument sequences.  The returned list is truncated
+in length to the length of the shortest argument sequence."""
+    if not sequences:
+        return []
+    result = []
+    iterators = [iter(seq) for seq in sequences]
+    while True:
+        try:
+            items = [next(it) for it in iterators]
+        except StopIteration:
+            return result
+        result.append(tuple(items))
diff --git a/pypy/module/__builtin__/app_io.py b/pypy/module/__builtin__/app_io.py
--- a/pypy/module/__builtin__/app_io.py
+++ b/pypy/module/__builtin__/app_io.py
@@ -27,7 +27,20 @@
     co = compile(source.rstrip()+"\n", filename, 'exec')
     exec co in glob, loc
 
-def raw_input(prompt=None):
+def _write_prompt(stdout, prompt):
+    print >> stdout, prompt,
+    try:
+        flush = stdout.flush
+    except AttributeError:
+        pass
+    else:
+        flush()
+    try:
+        stdout.softspace = 0
+    except (AttributeError, TypeError):
+        pass
+
+def raw_input(prompt=''):
     """raw_input([prompt]) -> string
 
 Read a string from standard input.  The trailing newline is stripped.
@@ -47,18 +60,10 @@
     if (hasattr(sys, '__raw_input__') and
         isinstance(stdin, file)  and stdin.fileno() == 0 and stdin.isatty() and
         isinstance(stdout, file) and stdout.fileno() == 1):
-        if prompt is None:
-            prompt = ''
-        return sys.__raw_input__(prompt)
+        _write_prompt(stdout, '')
+        return sys.__raw_input__(str(prompt))
 
-    if prompt is not None:
-        stdout.write(prompt)
-        try:
-            flush = stdout.flush
-        except AttributeError:
-            pass
-        else:
-            flush()
+    _write_prompt(stdout, prompt)
     line = stdin.readline()
     if not line:    # inputting an empty line gives line == '\n'
         raise EOFError
diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py
--- a/pypy/module/__builtin__/functional.py
+++ b/pypy/module/__builtin__/functional.py
@@ -201,27 +201,6 @@
     """
     return min_max(space, __args__, "min")
 
- at unwrap_spec(sequences_w="args_w")
-def zip(space, sequences_w):
-    """Return a list of tuples, where the nth tuple contains every nth item of
-    each collection.
-
-    If the collections have different lengths, zip returns a list as long as the
-    shortest collection, ignoring the trailing items in the other collections.
-    """
-    if not sequences_w:
-        return space.newlist([])
-    result_w = []
-    iterators_w = [space.iter(w_seq) for w_seq in sequences_w]
-    while True:
-        try:
-            items_w = [space.next(w_it) for w_it in iterators_w]
-        except OperationError, e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            return space.newlist(result_w)
-        result_w.append(space.newtuple(items_w))
-
 class W_Enumerate(Wrappable):
 
     def __init__(self, w_iter, w_start):
diff --git a/pypy/module/__builtin__/test/test_rawinput.py b/pypy/module/__builtin__/test/test_rawinput.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/__builtin__/test/test_rawinput.py
@@ -0,0 +1,77 @@
+import autopath
+
+
+class AppTestRawInput():
+
+    def test_raw_input(self):
+        import sys, StringIO
+        for prompt, expected in [("def:", "abc/ def:/ghi\n"),
+                                 ("", "abc/ /ghi\n"),
+                                 (42, "abc/ 42/ghi\n"),
+                                 (None, "abc/ None/ghi\n"),
+                                 (Ellipsis, "abc/ /ghi\n")]:
+            save = sys.stdin, sys.stdout
+            try:
+                sys.stdin = StringIO.StringIO("foo\nbar\n")
+                out = sys.stdout = StringIO.StringIO()
+                print "abc",    # softspace = 1
+                out.write('/')
+                if prompt is Ellipsis:
+                    got = raw_input()
+                else:
+                    got = raw_input(prompt)
+                out.write('/')
+                print "ghi"
+            finally:
+                sys.stdin, sys.stdout = save
+            assert out.getvalue() == expected
+            assert got == "foo"
+
+    def test_softspace(self):
+        import sys
+        import StringIO
+        fin = StringIO.StringIO()
+        fout = StringIO.StringIO()
+
+        fin.write("Coconuts\n")
+        fin.seek(0)
+
+        sys_stdin_orig = sys.stdin
+        sys_stdout_orig = sys.stdout
+
+        sys.stdin = fin
+        sys.stdout = fout
+
+        print "test",
+        raw_input("test")
+
+        sys.stdin = sys_stdin_orig
+        sys.stdout = sys_stdout_orig
+
+        fout.seek(0)
+        assert fout.read() == "test test"
+
+    def test_softspace_carryover(self):
+        import sys
+        import StringIO
+        fin = StringIO.StringIO()
+        fout = StringIO.StringIO()
+
+        fin.write("Coconuts\n")
+        fin.seek(0)
+
+        sys_stdin_orig = sys.stdin
+        sys_stdout_orig = sys.stdout
+
+        sys.stdin = fin
+        sys.stdout = fout
+
+        print "test",
+        raw_input("test")
+        print "test",
+
+        sys.stdin = sys_stdin_orig
+        sys.stdout = sys_stdout_orig
+
+        fout.seek(0)
+        assert fout.read() == "test testtest"
diff --git a/pypy/objspace/std/strutil.py b/pypy/objspace/std/strutil.py
--- a/pypy/objspace/std/strutil.py
+++ b/pypy/objspace/std/strutil.py
@@ -35,7 +35,7 @@
 
     def error(self):
         raise ParseStringError("invalid literal for %s() with base %d: '%s'" %
-                               (self.fname, self.base, self.literal))
+                               (self.fname, self.original_base, self.literal))
 
     def __init__(self, s, literal, base, fname):
         self.literal = literal
@@ -47,7 +47,8 @@
         elif s.startswith('+'):
             s = strip_spaces(s[1:])
         self.sign = sign
-        
+        self.original_base = base
+
         if base == 0:
             if s.startswith('0x') or s.startswith('0X'):
                 base = 16
diff --git a/pypy/objspace/std/test/test_strutil.py b/pypy/objspace/std/test/test_strutil.py
--- a/pypy/objspace/std/test/test_strutil.py
+++ b/pypy/objspace/std/test/test_strutil.py
@@ -89,6 +89,8 @@
 
         exc = raises(ParseStringError, string_to_int, '')
         assert exc.value.msg == "invalid literal for int() with base 10: ''"
+        exc = raises(ParseStringError, string_to_int, '', 0)
+        assert exc.value.msg == "invalid literal for int() with base 0: ''"
 
     def test_string_to_int_overflow(self):
         import sys
diff --git a/pypy/rlib/_rsocket_rffi.py b/pypy/rlib/_rsocket_rffi.py
--- a/pypy/rlib/_rsocket_rffi.py
+++ b/pypy/rlib/_rsocket_rffi.py
@@ -16,6 +16,7 @@
 _MINGW = target_platform.name == "mingw32"
 _SOLARIS = sys.platform == "sunos5"
 _MACOSX = sys.platform == "darwin"
+_HAS_AF_PACKET = sys.platform.startswith('linux')   # only Linux for now
 
 if _POSIX:
     includes = ('sys/types.h',
@@ -34,11 +35,12 @@
                 'stdint.h', 
                 'errno.h',
                 )
+    if _HAS_AF_PACKET:
+        includes += ('netpacket/packet.h',
+                     'sys/ioctl.h',
+                     'net/if.h')
 
-    cond_includes = [('AF_NETLINK', 'linux/netlink.h'),
-                     ('AF_PACKET', 'netpacket/packet.h'),
-                     ('AF_PACKET', 'sys/ioctl.h'),
-                     ('AF_PACKET', 'net/if.h')]
+    cond_includes = [('AF_NETLINK', 'linux/netlink.h')]
     
     libraries = ()
     calling_conv = 'c'
@@ -320,18 +322,18 @@
                                              ('events', rffi.SHORT),
                                              ('revents', rffi.SHORT)])
 
-    CConfig.sockaddr_ll = platform.Struct('struct sockaddr_ll',
+    if _HAS_AF_PACKET:
+        CConfig.sockaddr_ll = platform.Struct('struct sockaddr_ll',
                               [('sll_ifindex', rffi.INT),
                                ('sll_protocol', rffi.INT),
                                ('sll_pkttype', rffi.INT),
                                ('sll_hatype', rffi.INT),
                                ('sll_addr', rffi.CFixedArray(rffi.CHAR, 8)),
-                               ('sll_halen', rffi.INT)],
-                              ifdef='AF_PACKET')
+                               ('sll_halen', rffi.INT)])
 
-    CConfig.ifreq = platform.Struct('struct ifreq', [('ifr_ifindex', rffi.INT),
-                                 ('ifr_name', rffi.CFixedArray(rffi.CHAR, 8))],
-                                    ifdef='AF_PACKET')
+        CConfig.ifreq = platform.Struct('struct ifreq',
+                                [('ifr_ifindex', rffi.INT),
+                                 ('ifr_name', rffi.CFixedArray(rffi.CHAR, 8))])
 
 if _WIN32:
     CConfig.WSAEVENT = platform.SimpleType('WSAEVENT', rffi.VOIDP)
@@ -386,6 +388,8 @@
         constants[name] = value
     else:
         constants[name] = default
+if not _HAS_AF_PACKET and 'AF_PACKET' in constants:
+    del constants['AF_PACKET']
 
 constants['has_ipv6'] = True # This is a configuration option in CPython
 for name, value in constants.items():
@@ -439,21 +443,14 @@
 if _POSIX:
     nfds_t = cConfig.nfds_t
     pollfd = cConfig.pollfd
-    if cConfig.sockaddr_ll is not None:
+    if _HAS_AF_PACKET:
         sockaddr_ll = cConfig.sockaddr_ll
-    ifreq = cConfig.ifreq
+        ifreq = cConfig.ifreq
 if WIN32:
     WSAEVENT = cConfig.WSAEVENT
     WSANETWORKEVENTS = cConfig.WSANETWORKEVENTS
 timeval = cConfig.timeval
 
-#if _POSIX:
-#    includes = list(includes)
-#    for _name, _header in cond_includes:
-#        if getattr(cConfig, _name) is not None:
-#            includes.append(_header)
-#    eci = ExternalCompilationInfo(includes=includes, libraries=libraries,
-#                                  separate_module_sources=sources)
 
 def external(name, args, result, **kwds):
     return rffi.llexternal(name, args, result, compilation_info=eci,
@@ -544,7 +541,7 @@
     socketpair_t = rffi.CArray(socketfd_type)
     socketpair = external('socketpair', [rffi.INT, rffi.INT, rffi.INT,
                           lltype.Ptr(socketpair_t)], rffi.INT)
-    if ifreq is not None:
+    if _HAS_AF_PACKET:
         ioctl = external('ioctl', [socketfd_type, rffi.INT, lltype.Ptr(ifreq)],
                          rffi.INT)
 
diff --git a/pypy/rlib/rsocket.py b/pypy/rlib/rsocket.py
--- a/pypy/rlib/rsocket.py
+++ b/pypy/rlib/rsocket.py
@@ -8,6 +8,7 @@
 # Known missing features:
 #
 #   - address families other than AF_INET, AF_INET6, AF_UNIX, AF_PACKET
+#   - AF_PACKET is only supported on Linux
 #   - methods makefile(),
 #   - SSL
 #
diff --git a/pypy/rpython/lltypesystem/rbuilder.py b/pypy/rpython/lltypesystem/rbuilder.py
--- a/pypy/rpython/lltypesystem/rbuilder.py
+++ b/pypy/rpython/lltypesystem/rbuilder.py
@@ -29,7 +29,7 @@
         except OverflowError:
             raise MemoryError
         newbuf = mallocfn(new_allocated)
-        copycontentsfn(ll_builder.buf, newbuf, 0, 0, ll_builder.allocated)
+        copycontentsfn(ll_builder.buf, newbuf, 0, 0, ll_builder.used)
         ll_builder.buf = newbuf
         ll_builder.allocated = new_allocated
     return func_with_new_name(stringbuilder_grow, name)
@@ -56,7 +56,7 @@
 class BaseStringBuilderRepr(AbstractStringBuilderRepr):
     def empty(self):
         return nullptr(self.lowleveltype.TO)
-    
+
     @classmethod
     def ll_new(cls, init_size):
         if init_size < 0 or init_size > MAX:

From noreply at buildbot.pypy.org  Fri Oct 21 07:22:04 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 21 Oct 2011 07:22:04 +0200 (CEST)
Subject: [pypy-commit] pypy inline-dict-ops: improve comments
Message-ID: <20111021052204.472CF820D7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: inline-dict-ops
Changeset: r48293:48a8680ad5e0
Date: 2011-10-20 20:39 +0200
http://bitbucket.org/pypy/pypy/changeset/48a8680ad5e0/

Log:	improve comments

diff --git a/pypy/jit/metainterp/test/test_dict.py b/pypy/jit/metainterp/test/test_dict.py
--- a/pypy/jit/metainterp/test/test_dict.py
+++ b/pypy/jit/metainterp/test/test_dict.py
@@ -91,7 +91,7 @@
         res1 = f(100)
         res2 = self.meta_interp(f, [100], listops=True)
         assert res1 == res2
-        self.check_loops(int_mod=3) # the hash was traced and eq
+        self.check_loops(int_mod=1) # the hash was traced and eq, but cached
 
     def test_dict_setdefault(self):
         myjitdriver = JitDriver(greens = [], reds = ['total', 'dct'])
@@ -128,7 +128,7 @@
         assert f(100) == 50
         res = self.meta_interp(f, [100], listops=True)
         assert res == 50
-        self.check_loops(int_mod=3) # key + eq
+        self.check_loops(int_mod=1) # key + eq, but cached
 
     def test_repeated_lookup(self):
         myjitdriver = JitDriver(greens = [], reds = ['n', 'd'])

From noreply at buildbot.pypy.org  Fri Oct 21 07:22:05 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 21 Oct 2011 07:22:05 +0200 (CEST)
Subject: [pypy-commit] pypy inline-dict-ops: add caching to
	interiorfielddescr
Message-ID: <20111021052205.703C0820D7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: inline-dict-ops
Changeset: r48294:5c8426aeee4f
Date: 2011-10-21 07:20 +0200
http://bitbucket.org/pypy/pypy/changeset/5c8426aeee4f/

Log:	add caching to interiorfielddescr

diff --git a/pypy/jit/backend/llsupport/descr.py b/pypy/jit/backend/llsupport/descr.py
--- a/pypy/jit/backend/llsupport/descr.py
+++ b/pypy/jit/backend/llsupport/descr.py
@@ -20,6 +20,7 @@
         self._cache_field = {}
         self._cache_array = {}
         self._cache_call = {}
+        self._cache_interiorfield = {}
 
     def init_size_descr(self, STRUCT, sizedescr):
         assert isinstance(STRUCT, lltype.GcStruct)
@@ -240,6 +241,17 @@
                          NonGcPtrArrayNoLengthDescr, 'ArrayNoLength', 'get_item_size',
                          '_is_array_of_floats', '_is_item_signed')
 
+def get_interiorfield_descr(gc_ll_descr, ARRAY, FIELDTP, name):
+    cache = gc_ll_descr._cache_interiorfield
+    try:
+        return cache[(ARRAY, FIELDTP, name)]
+    except KeyError:
+        arraydescr = get_array_descr(gc_ll_descr, ARRAY)
+        fielddescr = get_field_descr(gc_ll_descr, FIELDTP, name)
+        descr = InteriorFieldDescr(arraydescr, fielddescr)
+        cache[(ARRAY, FIELDTP, name)] = descr
+        return descr
+
 def get_array_descr(gccache, ARRAY):
     cache = gccache._cache_array
     try:
diff --git a/pypy/jit/backend/llsupport/llmodel.py b/pypy/jit/backend/llsupport/llmodel.py
--- a/pypy/jit/backend/llsupport/llmodel.py
+++ b/pypy/jit/backend/llsupport/llmodel.py
@@ -11,7 +11,7 @@
 from pypy.jit.backend.llsupport.descr import (get_size_descr,
      get_field_descr, BaseFieldDescr, get_array_descr, BaseArrayDescr,
      get_call_descr, BaseIntCallDescr, GcPtrCallDescr, FloatCallDescr,
-     VoidCallDescr, InteriorFieldDescr)
+     VoidCallDescr, InteriorFieldDescr, get_interiorfield_descr)
 from pypy.jit.backend.llsupport.asmmemmgr import AsmMemoryManager
 
 
@@ -236,9 +236,7 @@
         return get_array_descr(self.gc_ll_descr, A)
 
     def interiorfielddescrof(self, A, fieldname):
-        arraydescr = get_array_descr(self.gc_ll_descr, A)
-        fielddescr = get_field_descr(self.gc_ll_descr, A.OF, fieldname)
-        return InteriorFieldDescr(arraydescr, fielddescr)
+        return get_interiorfield_descr(self.gc_ll_descr, A, A.OF, fieldname)
 
     def unpack_arraydescr(self, arraydescr):
         assert isinstance(arraydescr, BaseArrayDescr)

From noreply at buildbot.pypy.org  Fri Oct 21 07:22:06 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 21 Oct 2011 07:22:06 +0200 (CEST)
Subject: [pypy-commit] pypy inline-dict-ops: merge
Message-ID: <20111021052206.98086820D7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: inline-dict-ops
Changeset: r48295:ee82f2591c34
Date: 2011-10-21 07:21 +0200
http://bitbucket.org/pypy/pypy/changeset/ee82f2591c34/

Log:	merge

diff --git a/pypy/jit/metainterp/test/test_dict.py b/pypy/jit/metainterp/test/test_dict.py
--- a/pypy/jit/metainterp/test/test_dict.py
+++ b/pypy/jit/metainterp/test/test_dict.py
@@ -91,8 +91,7 @@
         res1 = f(100)
         res2 = self.meta_interp(f, [100], listops=True)
         assert res1 == res2
-        # the hash was traced
-        self.check_loops(int_mod=1)
+        self.check_loops(int_mod=1) # the hash was traced and eq, but cached
 
     def test_dict_setdefault(self):
         myjitdriver = JitDriver(greens = [], reds = ['total', 'dct'])
@@ -129,7 +128,7 @@
         assert f(100) == 50
         res = self.meta_interp(f, [100], listops=True)
         assert res == 50
-        self.check_loops(int_mod=1) # key
+        self.check_loops(int_mod=1) # key + eq, but cached
 
     def test_repeated_lookup(self):
         myjitdriver = JitDriver(greens = [], reds = ['n', 'd'])

From noreply at buildbot.pypy.org  Fri Oct 21 07:27:33 2011
From: noreply at buildbot.pypy.org (hakanardo)
Date: Fri, 21 Oct 2011 07:27:33 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: sign up
Message-ID: <20111021052733.6B5DC820D7@wyvern.cs.uni-duesseldorf.de>

Author: Hakan Ardo <hakan at debian.org>
Branch: extradoc
Changeset: r3936:4d40ab56fd27
Date: 2011-10-21 07:27 +0200
http://bitbucket.org/pypy/extradoc/changeset/4d40ab56fd27/

Log:	sign up

diff --git a/sprintinfo/gothenburg-2011-2/people.txt b/sprintinfo/gothenburg-2011-2/people.txt
--- a/sprintinfo/gothenburg-2011-2/people.txt
+++ b/sprintinfo/gothenburg-2011-2/people.txt
@@ -12,4 +12,5 @@
 Laura Creighton      lives there
 Armin Rigo           ?              ?
 Antonio Cuni         4-10 november  Laura's and Jacob's
+H&#229;kan Ard&#246;           3-8            ?
 ==================== ============== ===================== ==================

From noreply at buildbot.pypy.org  Fri Oct 21 07:38:15 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 21 Oct 2011 07:38:15 +0200 (CEST)
Subject: [pypy-commit] pypy inline-dict-ops: fix annotations for zrpy tests
Message-ID: <20111021053815.EBD9B820D7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: inline-dict-ops
Changeset: r48296:d0bcf756f012
Date: 2011-10-21 07:38 +0200
http://bitbucket.org/pypy/pypy/changeset/d0bcf756f012/

Log:	fix annotations for zrpy tests

diff --git a/pypy/jit/codewriter/assembler.py b/pypy/jit/codewriter/assembler.py
--- a/pypy/jit/codewriter/assembler.py
+++ b/pypy/jit/codewriter/assembler.py
@@ -246,9 +246,13 @@
             addr = llmemory.cast_ptr_to_adr(value)
             self.list_of_addr2name.append((addr, name))
 
-    def finished(self, callinfocollection):
+    def finished(self, callinfocollection, cpu):
         # Helper called at the end of assembling.  Registers the extra
         # functions shown in _callinfo_for_oopspec.
         for func in callinfocollection.all_function_addresses_as_int():
             func = heaptracker.int2adr(func)
             self.see_raw_object(func.ptr)
+        # register at least one interiorfielddescr
+        if hasattr(cpu, 'interiorfielddescrof'):
+            _T = lltype.GcArray(lltype.Struct('x', ('field', lltype.Signed)))
+            self.descrs.append(cpu.interiorfielddescrof(_T, 'field'))
diff --git a/pypy/jit/codewriter/codewriter.py b/pypy/jit/codewriter/codewriter.py
--- a/pypy/jit/codewriter/codewriter.py
+++ b/pypy/jit/codewriter/codewriter.py
@@ -73,7 +73,7 @@
             count += 1
             if not count % 500:
                 log.info("Produced %d jitcodes" % count)
-        self.assembler.finished(self.callcontrol.callinfocollection)
+        self.assembler.finished(self.callcontrol.callinfocollection, self.cpu)
         heaptracker.finish_registering(self.cpu)
         log.info("there are %d JitCode instances." % count)
 

From noreply at buildbot.pypy.org  Fri Oct 21 08:51:12 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Fri, 21 Oct 2011 08:51:12 +0200 (CEST)
Subject: [pypy-commit] pypy default: account for the new original_base field
Message-ID: <20111021065112.74059820D7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r48297:3c2fbef05b09
Date: 2011-10-21 02:50 -0400
http://bitbucket.org/pypy/pypy/changeset/3c2fbef05b09/

Log:	account for the new original_base field

diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -93,7 +93,8 @@
             i46 = call(ConstClass(ll_startswith__rpy_stringPtr_rpy_stringPtr), p28, ConstPtr(ptr45), descr=<BoolCallDescr>)
             guard_false(i46, descr=...)
             p51 = new_with_vtable(21136408)
-            setfield_gc(p51, _, descr=...)    # 6 setfields, but the order is dict-order-dependent
+            setfield_gc(p51, _, descr=...)    # 7 setfields, but the order is dict-order-dependent
+            setfield_gc(p51, _, descr=...)
             setfield_gc(p51, _, descr=...)
             setfield_gc(p51, _, descr=...)
             setfield_gc(p51, _, descr=...)

From noreply at buildbot.pypy.org  Fri Oct 21 09:56:07 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Fri, 21 Oct 2011 09:56:07 +0200 (CEST)
Subject: [pypy-commit] pypy inline-dict-ops: fix the numbers
Message-ID: <20111021075607.BA606820D7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: inline-dict-ops
Changeset: r48298:e9121bf561b1
Date: 2011-10-21 03:55 -0400
http://bitbucket.org/pypy/pypy/changeset/e9121bf561b1/

Log:	fix the numbers

diff --git a/pypy/module/pypyjit/test_pypy_c/test_containers.py b/pypy/module/pypyjit/test_pypy_c/test_containers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_containers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_containers.py
@@ -46,7 +46,7 @@
         assert loop.match_by_id("getitem", """
             i28 = call(ConstClass(ll_dict_lookup__dicttablePtr_objectPtr_Signed), p18, p6, i25, descr=...)
             ...
-            p33 = getinteriorfield_gc(p18, i28, <InteriorFieldDescr <GcPtrFieldDescr dictentry.value .*>>)
+            p33 = getinteriorfield_gc(p31, i26, <InteriorFieldDescr <GcPtrFieldDescr dictentry.value .*>>)
             ...
         """)
 

From noreply at buildbot.pypy.org  Fri Oct 21 11:07:58 2011
From: noreply at buildbot.pypy.org (l.diekmann)
Date: Fri, 21 Oct 2011 11:07:58 +0200 (CEST)
Subject: [pypy-commit] pypy list-strategies: more test coverage
Message-ID: <20111021090758.D5EB8820D7@wyvern.cs.uni-duesseldorf.de>

Author: Lukas Diekmann <lukas.diekmann at uni-duesseldorf.de>
Branch: list-strategies
Changeset: r48299:13c5be7af43d
Date: 2011-10-21 11:01 +0200
http://bitbucket.org/pypy/pypy/changeset/13c5be7af43d/

Log:	more test coverage

diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -336,6 +336,8 @@
         raise IndexError
 
     def getslice(self, w_list, start, stop, step, length):
+        # will never be called because the empty list case is already caught in
+        # getslice__List_ANY_ANY and getitem__List_Slice
         return W_ListObject(self.space, self.cached_emptylist_w)
 
     def getitems(self, w_list):
@@ -370,6 +372,8 @@
         pass
 
     def pop(self, w_list, index):
+        # will not be called becuase IndexError was already raised in
+        # list_pop__List_ANY
         raise IndexError
 
     def setitem(self, w_list, index, w_item):
diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py
--- a/pypy/objspace/std/test/test_listobject.py
+++ b/pypy/objspace/std/test/test_listobject.py
@@ -25,7 +25,7 @@
         assert self.space.eq_w(self.space.len(w_list), w(1))
         w_list = W_ListObject(self.space, [w(5), w(3), w(99)]*111)
         assert self.space.eq_w(self.space.len(w_list), w(333))
- 
+
     def test_getitem(self):
         w = self.space.wrap
         w_list = W_ListObject(self.space, [w(5), w(3)])
@@ -40,6 +40,15 @@
         self.space.raises_w(self.space.w_IndexError,
                             self.space.getitem, w_list, w(-3))
 
+    def test_getitems(self):
+        w = self.space.wrap
+        from pypy.objspace.std.listobject import make_range_list
+        r = make_range_list(self.space, 1,1,7)
+        l = [w(1),w(2),w(3),w(4),w(5),w(6),w(7)]
+        l2 = r.getitems()
+        for i in range(7):
+            assert self.space.eq_w(l[i], l2[i])
+
     def test_random_getitem(self):
         w = self.space.wrap
         s = list('qedx387tn3uixhvt 7fh387fymh3dh238 dwd-wq.dwq9')
@@ -353,6 +362,9 @@
         l2 = [1, "2", "a", "a"]
         assert sorted(l1) == sorted(l2)
 
+    def test_notequals(self):
+        assert [1,2,3,4] != [1,2,5,4]
+
     def test_contains(self):
         l = []
         assert not l.__contains__(2)
@@ -441,7 +453,6 @@
         assert m == [5,2,3]
         assert l == [1,2,3]
 
-
     def test_extend_tuple(self):
         l = l0 = [1]
         l.extend((2,))
@@ -475,6 +486,10 @@
         assert l is l0
         assert l == [1]
 
+        l = ["c", "a", "d", "b"]
+        l.sort(reverse=True)
+        assert l == ["d", "c", "b", "a"]
+
     def test_sort_cmp(self):
         def lencmp(a,b): return cmp(len(a), len(b))
         l = [ 'a', 'fiver', 'tre', '' ]
@@ -533,6 +548,21 @@
         assert l[-1] == 'c'
         assert l[-2] == 'b'
         raises(IndexError, "l[len(l)]")
+        l = []
+        raises(IndexError, "l[1]")
+
+    def test_setitem(self):
+
+        l = []
+        raises(IndexError, "l[1] = 2")
+
+        l = [5,3]
+        l[0] = 2
+        assert l == [2,3]
+
+        l = [5,3]
+        l[0] = "2"
+        assert l == ["2",3]
 
     def test_delitem(self):
         l = [1, 2, 3, 4, 5, 6, 9]
@@ -544,7 +574,7 @@
         assert l == [2, 3, 4, 6]
         raises(IndexError, "del l[len(l)]")
         raises(IndexError, "del l[-len(l)-1]")
-        
+
         l = l0 = ['a', 'b', 'c']
         del l[0]
         assert l == ['b', 'c']
@@ -698,7 +728,7 @@
         assert c.index(0) == 0
         raises(ValueError, c.index, 3)
 
-    def test_ass_slice(self):
+    def test_assign_slice(self):
         l = range(6)
         l[1:3] = 'abc'
         assert l == [0, 'a', 'b', 'c', 3, 4, 5]
@@ -718,7 +748,7 @@
         assert l == []
         assert l is l0
 
-    def test_ass_extended_slice(self):
+    def test_assign_extended_slice(self):
         l = l0 = ['a', 'b', 'c']
         l[::-1] = ['a', 'b', 'c']
         assert l == ['c', 'b', 'a']
@@ -734,6 +764,11 @@
         raises(ValueError, "l[0:2:2] = [1,2,3,4]")
         raises(ValueError, "l[::2] = []")
 
+        l = range(6)
+        l[::3] = ('a', 'b')
+        assert l == ['a', 1, 2, 'b', 4, 5]
+
+
     def test_recursive_repr(self):
         l = []
         assert repr(l) == '[]'
@@ -759,6 +794,10 @@
         l.append(4)
         assert l == range(5)
 
+        l = [1,2,3]
+        l.append("a")
+        assert l == [1,2,3,"a"]
+
     def test_count(self):
         c = list('hello')
         assert c.count('l') == 2
@@ -778,6 +817,14 @@
             ls.insert(0, i)
         assert len(ls) == 12
 
+        l = []
+        l.insert(4,2)
+        assert l == [2]
+
+        l = [1,2,3]
+        l.insert(0,"a")
+        assert l == ["a", 1, 2, 3]
+
     def test_pop(self):
         c = list('hello world')
         s = ''
@@ -791,6 +838,9 @@
         l.pop()
         assert l == range(9)
 
+        l = []
+        raises(IndexError, l.pop, 0)
+
     def test_pop_custom_int(self):
         class A(object):
             def __init__(self, x):
@@ -907,11 +957,18 @@
         res = l.__getslice__(0, 2)
         assert res == [1, 2]
 
+        l = []
+        assert l.__getslice__(0,2) == []
+
     def test___setslice__(self):
         l = [1,2,3,4]
         l.__setslice__(0, 2, [5, 6])
         assert l == [5, 6, 3, 4]
 
+        l = []
+        l.__setslice__(0,0,[3,4,5])
+        assert l == [3,4,5]
+
     def test___delslice__(self):
         l = [1,2,3,4]
         l.__delslice__(0, 2)
@@ -979,6 +1036,48 @@
         l.sort()
         assert l == [3, 6, 9]
 
+    def test_slice(self):
+        l = []
+        l2 = range(3)
+        l.__setslice__(0,3,l2)
+        assert l == [0,1,2]
+
+    def test_getitem(self):
+        l = range(5)
+        raises(IndexError, "l[-10]")
+
+    def test_append(self):
+        l = range(5)
+        l.append(26)
+        assert l == [0,1,2,3,4,26]
+
+        l = range(5)
+        l.append("a")
+        assert l == [0,1,2,3,4,"a"]
+
+        l = range(5)
+        l.append(5)
+        assert l == [0,1,2,3,4,5]
+
+    def test_pop(self):
+        l = range(3)
+        assert l.pop(0) == 0
+
+    def test_setitem(self):
+        l = range(3)
+        l[0] = 1
+        assert l == [1,1,2]
+
+    def test_inset(self):
+        l = range(3)
+        l.insert(1,5)
+        assert l == [0,5,1,2]
+
+    def test_reverse(self):
+        l = range(3)
+        l.reverse()
+        assert l == [2,1,0]
+
 class AppTestListFastSubscr:
 
     def setup_class(cls):

From noreply at buildbot.pypy.org  Fri Oct 21 11:08:00 2011
From: noreply at buildbot.pypy.org (l.diekmann)
Date: Fri, 21 Oct 2011 11:08:00 +0200 (CEST)
Subject: [pypy-commit] pypy list-strategies: quicksort turned out to be not
 that good. instead we use a special TimSort implementation for ints
Message-ID: <20111021090800.0B63D820D9@wyvern.cs.uni-duesseldorf.de>

Author: Lukas Diekmann <lukas.diekmann at uni-duesseldorf.de>
Branch: list-strategies
Changeset: r48300:05582d7696ee
Date: 2011-10-21 11:07 +0200
http://bitbucket.org/pypy/pypy/changeset/05582d7696ee/

Log:	quicksort turned out to be not that good. instead we use a special
	TimSort implementation for ints

diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -880,37 +880,6 @@
     def list_is_correct_type(self, w_list):
         return w_list.strategy is self.space.fromcache(IntegerListStrategy)
 
-    def custom_sort_for_ints(self, w_list):
-        l = self.unerase(w_list.lstorage)
-        self.quicksort(l, 0, len(l) - 1)
-
-    def partition(self, l, start, end):
-        left = start
-        right = end - 1
-        pivot = l[end]
-
-        while left < right:
-
-            while l[left] <= pivot and left < end:
-                left += 1
-
-            while l[right] >= pivot and right > start:
-                right -= 1
-
-            if left < right:
-                l[left], l[right] = l[right], l[left]
-
-        if l[left] > pivot:
-            l[left], l[end] = l[end], l[left]
-
-        return left
-
-    def quicksort(self, l, start, end):
-        if start < end:
-            p = self.partition(l, start, end)
-            self.quicksort(l, start, p - 1)
-            self.quicksort(l, p + 1, end)
-
     def sort(self, w_list, reverse):
         l = self.unerase(w_list.lstorage)
         sorter = IntSort(l, len(l))

From noreply at buildbot.pypy.org  Fri Oct 21 11:45:55 2011
From: noreply at buildbot.pypy.org (hager)
Date: Fri, 21 Oct 2011 11:45:55 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: (bivab,
	hager): Implemented support for bridges.
Message-ID: <20111021094555.805F5820D7@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r48301:e944a8524e86
Date: 2011-10-21 11:45 +0200
http://bitbucket.org/pypy/pypy/changeset/e944a8524e86/

Log:	(bivab, hager): Implemented support for bridges.

diff --git a/pypy/jit/backend/ppc/ppcgen/opassembler.py b/pypy/jit/backend/ppc/ppcgen/opassembler.py
--- a/pypy/jit/backend/ppc/ppcgen/opassembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/opassembler.py
@@ -201,12 +201,11 @@
             curpos = self.mc.get_rel_pos()
             self.mc.b(descr._ppc_loop_code - curpos)
         else:
-            assert 0, "case not implemented yet"
+            target = descr._ppc_bootstrap_code + descr._ppc_loop_code
+            self.mc.b_abs(target)
+            new_fd = max(regalloc.frame_manager.frame_depth,
+                         descr._ppc_frame_manager_depth)
+            regalloc.frame_manager.frame_depth = new_fd
 
     def nop(self):
         self.mc.ori(0, 0, 0)
-
-    def branch_abs(self, address):
-        self.load_imm(r.r0.value, address)
-        self.mc.mtctr(0)
-        self.mc.bctr()
diff --git a/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py b/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py
--- a/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py
@@ -261,6 +261,37 @@
         self.fail_force_index = spp_loc
         return descr
 
+    def decode_inputargs(self, enc, inputargs, regalloc):
+        locs = []
+        j = 0
+        for i in range(len(inputargs)):
+            res = enc[j]
+            if res == self.END_OF_LOCS:
+                assert 0, 'reached end of encoded area'
+            while res == self.EMPTY_LOC:
+                j += 1
+                res = enc[j]
+
+            assert res in [self.INT_TYPE, self.REF_TYPE],\
+                    'location type is not supported'
+            res_type = res
+            j += 1
+            res = enc[j]
+            if res == self.IMM_LOC:
+                # XXX decode imm if necessary
+                assert 0, 'Imm Locations are not supported'
+            elif res == self.STACK_LOC:
+                stack_loc = decode32(enc, j+1)
+                loc = regalloc.frame_manager.frame_pos(stack_loc, INT)
+                j += 4
+            else: # REG_LOC
+                #loc = r.all_regs[ord(res)]
+                #import pdb; pdb.set_trace()
+                loc = r.MANAGED_REGS[ord(res) - 2]
+            j += 1
+            locs.append(loc)
+        return locs
+
     def _gen_leave_jitted_hook_code(self, save_exc=False):
         mc = PPCBuilder()
 
@@ -390,14 +421,40 @@
 
         start_pos = self.mc.currpos()
         self.framesize = frame_depth = self.compute_frame_depth(regalloc)
+        looptoken._ppc_frame_manager_depth = regalloc.frame_manager.frame_depth
         self._make_prologue(regalloc_head, frame_depth)
      
         self.write_pending_failure_recoveries()
         loop_start = self.materialize_loop(looptoken, False)
+        looptoken._ppc_bootstrap_code = loop_start
         looptoken.ppc_code = loop_start + start_pos
         self.process_pending_guards(loop_start)
         self._teardown()
 
+    def assemble_bridge(self, faildescr, inputargs, operations, looptoken, log):
+        self.setup(looptoken, operations)
+        assert isinstance(faildescr, AbstractFailDescr)
+        code = faildescr._failure_recovery_code
+        enc = rffi.cast(rffi.CCHARP, code)
+        longevity = compute_vars_longevity(inputargs, operations)
+        regalloc = Regalloc(longevity, assembler=self, 
+                            frame_manager=PPCFrameManager())
+
+        #sp_patch_location = self._prepare_sp_patch_position()
+        frame_depth = faildescr._ppc_frame_depth
+        locs = self.decode_inputargs(enc, inputargs, regalloc)
+        regalloc.update_bindings(locs, frame_depth, inputargs)
+
+        self._walk_operations(operations, regalloc)
+
+        #self._patch_sp_offset(sp_patch_location, 
+        #                      regalloc.frame_manager.frame_depth)
+        self.write_pending_failure_recoveries()
+        bridge_start = self.materialize_loop(looptoken, False)
+        self.process_pending_guards(bridge_start)
+        self.patch_trace(faildescr, looptoken, bridge_start, regalloc)
+        self._teardown()
+
     # For an explanation of the encoding, see
     # backend/arm/assembler.py
     def gen_descr_encoding(self, descr, args, arglocs):
@@ -516,7 +573,7 @@
             path = self._leave_jitted_hook_save_exc
         else:
             path = self._leave_jitted_hook
-        self.branch_abs(path)
+        self.mc.b_abs(path)
         return memaddr
 
     def process_pending_guards(self, block_start):
@@ -535,6 +592,15 @@
             else:
                 assert 0, "not implemented yet"
 
+    def patch_trace(self, faildescr, looptoken, bridge_addr, regalloc):
+        # The first instruction (word) is not overwritten, because it is the
+        # one that actually checks the condition
+        mc = PPCBuilder()
+        patch_addr = faildescr._ppc_block_start + faildescr._ppc_guard_pos
+        mc.b_abs(bridge_addr)
+        mc.prepare_insts_blocks()
+        mc.copy_to_raw_memory(patch_addr)
+
     def get_asmmemmgr_blocks(self, looptoken):
         clt = looptoken.compiled_loop_token
         if clt.asmmemmgr_blocks is None:
diff --git a/pypy/jit/backend/ppc/ppcgen/regalloc.py b/pypy/jit/backend/ppc/ppcgen/regalloc.py
--- a/pypy/jit/backend/ppc/ppcgen/regalloc.py
+++ b/pypy/jit/backend/ppc/ppcgen/regalloc.py
@@ -120,6 +120,31 @@
         self.possibly_free_vars(inputargs)
         return nonfloatlocs
 
+    def update_bindings(self, locs, frame_depth, inputargs):
+        used = {}
+        i = 0
+        self.frame_manager.frame_depth = frame_depth
+        for loc in locs:
+            arg = inputargs[i]
+            i += 1
+            if loc.is_reg():
+                self.rm.reg_bindings[arg] = loc
+            elif loc.is_vfp_reg():
+                self.vfprm.reg_bindings[arg] = loc
+            else:
+                assert loc.is_stack()
+                self.frame_manager.frame_bindings[arg] = loc
+            used[loc] = None
+
+        # XXX combine with x86 code and move to llsupport
+        self.rm.free_regs = []
+        for reg in self.rm.all_regs:
+            if reg not in used:
+                self.rm.free_regs.append(reg)
+        # note: we need to make a copy of inputargs because possibly_free_vars
+        # is also used on op args, which is a non-resizable list
+        self.possibly_free_vars(list(inputargs))
+
     def possibly_free_var(self, var):
         self.rm.possibly_free_var(var)
 

From noreply at buildbot.pypy.org  Fri Oct 21 13:41:02 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Fri, 21 Oct 2011 13:41:02 +0200 (CEST)
Subject: [pypy-commit] pypy inline-dict-ops: fix for 32-bits
Message-ID: <20111021114102.5FDE6820D7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: inline-dict-ops
Changeset: r48302:c4e352fecd3c
Date: 2011-10-21 13:39 +0200
http://bitbucket.org/pypy/pypy/changeset/c4e352fecd3c/

Log:	fix for 32-bits

diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -894,8 +894,8 @@
                                                          boxfloat(1.5)],
                                'void', descr=kdescr)
         f = self.cpu.bh_getinteriorfield_gc_f(a_box.getref_base(), 3, kdescr)
-        assert f == 1.5
-        self.cpu.bh_setinteriorfield_gc_f(a_box.getref_base(), 3, kdescr, 2.5)
+        assert longlong.getrealfloat(f) == 1.5
+        self.cpu.bh_setinteriorfield_gc_f(a_box.getref_base(), 3, kdescr, longlong.getfloatstorage(2.5))
         r = self.execute_operation(rop.GETINTERIORFIELD_GC, [a_box, BoxInt(3)],
                                    'float', descr=kdescr)
         assert r.getfloat() == 2.5

From noreply at buildbot.pypy.org  Fri Oct 21 14:08:44 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 21 Oct 2011 14:08:44 +0200 (CEST)
Subject: [pypy-commit] pypy stm: Extra checks.
Message-ID: <20111021120844.28EDC820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm
Changeset: r48303:95787a032c12
Date: 2011-10-20 22:26 +0200
http://bitbucket.org/pypy/pypy/changeset/95787a032c12/

Log:	Extra checks.

diff --git a/pypy/translator/stm/test/test_rstm.py b/pypy/translator/stm/test/test_rstm.py
--- a/pypy/translator/stm/test/test_rstm.py
+++ b/pypy/translator/stm/test/test_rstm.py
@@ -70,6 +70,7 @@
     assert a.f == rf1
     assert float(a.sa) == float(rs1a)
     assert float(a.sb) == float(rs1b)
+    assert a.y == 10
     lltype.free(a, flavor='raw')
 
 def callback2(a):
@@ -142,6 +143,7 @@
     assert a.f == rf2
     assert float(a.sa) == float(rs2a)
     assert float(a.sb) == float(rs2b)
+    assert a.y == 10
     lltype.free(a, flavor='raw')
 
 # ____________________________________________________________

From noreply at buildbot.pypy.org  Fri Oct 21 14:08:48 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 21 Oct 2011 14:08:48 +0200 (CEST)
Subject: [pypy-commit] pypy stm: hg merge default
Message-ID: <20111021120848.CDB70820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm
Changeset: r48304:e56876588573
Date: 2011-10-20 22:26 +0200
http://bitbucket.org/pypy/pypy/changeset/e56876588573/

Log:	hg merge default

diff too long, truncating to 10000 out of 13877 lines

diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -317,7 +317,7 @@
     RegrTest('test_multibytecodec.py', usemodules='_multibytecodec'),
     RegrTest('test_multibytecodec_support.py', skip="not a test"),
     RegrTest('test_multifile.py'),
-    RegrTest('test_multiprocessing.py', skip='FIXME leaves subprocesses'),
+    RegrTest('test_multiprocessing.py', skip="FIXME leaves subprocesses"),
     RegrTest('test_mutants.py', core="possibly"),
     RegrTest('test_mutex.py'),
     RegrTest('test_netrc.py'),
diff --git a/lib-python/modified-2.7/httplib.py b/lib-python/modified-2.7/httplib.py
new file mode 100644
--- /dev/null
+++ b/lib-python/modified-2.7/httplib.py
@@ -0,0 +1,1377 @@
+"""HTTP/1.1 client library
+
+<intro stuff goes here>
+<other stuff, too>
+
+HTTPConnection goes through a number of "states", which define when a client
+may legally make another request or fetch the response for a particular
+request. This diagram details these state transitions:
+
+    (null)
+      |
+      | HTTPConnection()
+      v
+    Idle
+      |
+      | putrequest()
+      v
+    Request-started
+      |
+      | ( putheader() )*  endheaders()
+      v
+    Request-sent
+      |
+      | response = getresponse()
+      v
+    Unread-response   [Response-headers-read]
+      |\____________________
+      |                     |
+      | response.read()     | putrequest()
+      v                     v
+    Idle                  Req-started-unread-response
+                     ______/|
+                   /        |
+   response.read() |        | ( putheader() )*  endheaders()
+                   v        v
+       Request-started    Req-sent-unread-response
+                            |
+                            | response.read()
+                            v
+                          Request-sent
+
+This diagram presents the following rules:
+  -- a second request may not be started until {response-headers-read}
+  -- a response [object] cannot be retrieved until {request-sent}
+  -- there is no differentiation between an unread response body and a
+     partially read response body
+
+Note: this enforcement is applied by the HTTPConnection class. The
+      HTTPResponse class does not enforce this state machine, which
+      implies sophisticated clients may accelerate the request/response
+      pipeline. Caution should be taken, though: accelerating the states
+      beyond the above pattern may imply knowledge of the server's
+      connection-close behavior for certain requests. For example, it
+      is impossible to tell whether the server will close the connection
+      UNTIL the response headers have been read; this means that further
+      requests cannot be placed into the pipeline until it is known that
+      the server will NOT be closing the connection.
+
+Logical State                  __state            __response
+-------------                  -------            ----------
+Idle                           _CS_IDLE           None
+Request-started                _CS_REQ_STARTED    None
+Request-sent                   _CS_REQ_SENT       None
+Unread-response                _CS_IDLE           <response_class>
+Req-started-unread-response    _CS_REQ_STARTED    <response_class>
+Req-sent-unread-response       _CS_REQ_SENT       <response_class>
+"""
+
+from array import array
+import os
+import socket
+from sys import py3kwarning
+from urlparse import urlsplit
+import warnings
+with warnings.catch_warnings():
+    if py3kwarning:
+        warnings.filterwarnings("ignore", ".*mimetools has been removed",
+                                DeprecationWarning)
+    import mimetools
+
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from StringIO import StringIO
+
+__all__ = ["HTTP", "HTTPResponse", "HTTPConnection",
+           "HTTPException", "NotConnected", "UnknownProtocol",
+           "UnknownTransferEncoding", "UnimplementedFileMode",
+           "IncompleteRead", "InvalidURL", "ImproperConnectionState",
+           "CannotSendRequest", "CannotSendHeader", "ResponseNotReady",
+           "BadStatusLine", "error", "responses"]
+
+HTTP_PORT = 80
+HTTPS_PORT = 443
+
+_UNKNOWN = 'UNKNOWN'
+
+# connection states
+_CS_IDLE = 'Idle'
+_CS_REQ_STARTED = 'Request-started'
+_CS_REQ_SENT = 'Request-sent'
+
+# status codes
+# informational
+CONTINUE = 100
+SWITCHING_PROTOCOLS = 101
+PROCESSING = 102
+
+# successful
+OK = 200
+CREATED = 201
+ACCEPTED = 202
+NON_AUTHORITATIVE_INFORMATION = 203
+NO_CONTENT = 204
+RESET_CONTENT = 205
+PARTIAL_CONTENT = 206
+MULTI_STATUS = 207
+IM_USED = 226
+
+# redirection
+MULTIPLE_CHOICES = 300
+MOVED_PERMANENTLY = 301
+FOUND = 302
+SEE_OTHER = 303
+NOT_MODIFIED = 304
+USE_PROXY = 305
+TEMPORARY_REDIRECT = 307
+
+# client error
+BAD_REQUEST = 400
+UNAUTHORIZED = 401
+PAYMENT_REQUIRED = 402
+FORBIDDEN = 403
+NOT_FOUND = 404
+METHOD_NOT_ALLOWED = 405
+NOT_ACCEPTABLE = 406
+PROXY_AUTHENTICATION_REQUIRED = 407
+REQUEST_TIMEOUT = 408
+CONFLICT = 409
+GONE = 410
+LENGTH_REQUIRED = 411
+PRECONDITION_FAILED = 412
+REQUEST_ENTITY_TOO_LARGE = 413
+REQUEST_URI_TOO_LONG = 414
+UNSUPPORTED_MEDIA_TYPE = 415
+REQUESTED_RANGE_NOT_SATISFIABLE = 416
+EXPECTATION_FAILED = 417
+UNPROCESSABLE_ENTITY = 422
+LOCKED = 423
+FAILED_DEPENDENCY = 424
+UPGRADE_REQUIRED = 426
+
+# server error
+INTERNAL_SERVER_ERROR = 500
+NOT_IMPLEMENTED = 501
+BAD_GATEWAY = 502
+SERVICE_UNAVAILABLE = 503
+GATEWAY_TIMEOUT = 504
+HTTP_VERSION_NOT_SUPPORTED = 505
+INSUFFICIENT_STORAGE = 507
+NOT_EXTENDED = 510
+
+# Mapping status codes to official W3C names
+responses = {
+    100: 'Continue',
+    101: 'Switching Protocols',
+
+    200: 'OK',
+    201: 'Created',
+    202: 'Accepted',
+    203: 'Non-Authoritative Information',
+    204: 'No Content',
+    205: 'Reset Content',
+    206: 'Partial Content',
+
+    300: 'Multiple Choices',
+    301: 'Moved Permanently',
+    302: 'Found',
+    303: 'See Other',
+    304: 'Not Modified',
+    305: 'Use Proxy',
+    306: '(Unused)',
+    307: 'Temporary Redirect',
+
+    400: 'Bad Request',
+    401: 'Unauthorized',
+    402: 'Payment Required',
+    403: 'Forbidden',
+    404: 'Not Found',
+    405: 'Method Not Allowed',
+    406: 'Not Acceptable',
+    407: 'Proxy Authentication Required',
+    408: 'Request Timeout',
+    409: 'Conflict',
+    410: 'Gone',
+    411: 'Length Required',
+    412: 'Precondition Failed',
+    413: 'Request Entity Too Large',
+    414: 'Request-URI Too Long',
+    415: 'Unsupported Media Type',
+    416: 'Requested Range Not Satisfiable',
+    417: 'Expectation Failed',
+
+    500: 'Internal Server Error',
+    501: 'Not Implemented',
+    502: 'Bad Gateway',
+    503: 'Service Unavailable',
+    504: 'Gateway Timeout',
+    505: 'HTTP Version Not Supported',
+}
+
+# maximal amount of data to read at one time in _safe_read
+MAXAMOUNT = 1048576
+
+class HTTPMessage(mimetools.Message):
+
+    def addheader(self, key, value):
+        """Add header for field key handling repeats."""
+        prev = self.dict.get(key)
+        if prev is None:
+            self.dict[key] = value
+        else:
+            combined = ", ".join((prev, value))
+            self.dict[key] = combined
+
+    def addcontinue(self, key, more):
+        """Add more field data from a continuation line."""
+        prev = self.dict[key]
+        self.dict[key] = prev + "\n " + more
+
+    def readheaders(self):
+        """Read header lines.
+
+        Read header lines up to the entirely blank line that terminates them.
+        The (normally blank) line that ends the headers is skipped, but not
+        included in the returned list.  If a non-header line ends the headers,
+        (which is an error), an attempt is made to backspace over it; it is
+        never included in the returned list.
+
+        The variable self.status is set to the empty string if all went well,
+        otherwise it is an error message.  The variable self.headers is a
+        completely uninterpreted list of lines contained in the header (so
+        printing them will reproduce the header exactly as it appears in the
+        file).
+
+        If multiple header fields with the same name occur, they are combined
+        according to the rules in RFC 2616 sec 4.2:
+
+        Appending each subsequent field-value to the first, each separated
+        by a comma. The order in which header fields with the same field-name
+        are received is significant to the interpretation of the combined
+        field value.
+        """
+        # XXX The implementation overrides the readheaders() method of
+        # rfc822.Message.  The base class design isn't amenable to
+        # customized behavior here so the method here is a copy of the
+        # base class code with a few small changes.
+
+        self.dict = {}
+        self.unixfrom = ''
+        self.headers = hlist = []
+        self.status = ''
+        headerseen = ""
+        firstline = 1
+        startofline = unread = tell = None
+        if hasattr(self.fp, 'unread'):
+            unread = self.fp.unread
+        elif self.seekable:
+            tell = self.fp.tell
+        while True:
+            if tell:
+                try:
+                    startofline = tell()
+                except IOError:
+                    startofline = tell = None
+                    self.seekable = 0
+            line = self.fp.readline()
+            if not line:
+                self.status = 'EOF in headers'
+                break
+            # Skip unix From name time lines
+            if firstline and line.startswith('From '):
+                self.unixfrom = self.unixfrom + line
+                continue
+            firstline = 0
+            if headerseen and line[0] in ' \t':
+                # XXX Not sure if continuation lines are handled properly
+                # for http and/or for repeating headers
+                # It's a continuation line.
+                hlist.append(line)
+                self.addcontinue(headerseen, line.strip())
+                continue
+            elif self.iscomment(line):
+                # It's a comment.  Ignore it.
+                continue
+            elif self.islast(line):
+                # Note! No pushback here!  The delimiter line gets eaten.
+                break
+            headerseen = self.isheader(line)
+            if headerseen:
+                # It's a legal header line, save it.
+                hlist.append(line)
+                self.addheader(headerseen, line[len(headerseen)+1:].strip())
+                continue
+            else:
+                # It's not a header line; throw it back and stop here.
+                if not self.dict:
+                    self.status = 'No headers'
+                else:
+                    self.status = 'Non-header line where header expected'
+                # Try to undo the read.
+                if unread:
+                    unread(line)
+                elif tell:
+                    self.fp.seek(startofline)
+                else:
+                    self.status = self.status + '; bad seek'
+                break
+
+class HTTPResponse:
+
+    # strict: If true, raise BadStatusLine if the status line can't be
+    # parsed as a valid HTTP/1.0 or 1.1 status line.  By default it is
+    # false because it prevents clients from talking to HTTP/0.9
+    # servers.  Note that a response with a sufficiently corrupted
+    # status line will look like an HTTP/0.9 response.
+
+    # See RFC 2616 sec 19.6 and RFC 1945 sec 6 for details.
+
+    def __init__(self, sock, debuglevel=0, strict=0, method=None, buffering=False):
+        if buffering:
+            # The caller won't be using any sock.recv() calls, so buffering
+            # is fine and recommended for performance.
+            self.fp = sock.makefile('rb')
+        else:
+            # The buffer size is specified as zero, because the headers of
+            # the response are read with readline().  If the reads were
+            # buffered the readline() calls could consume some of the
+            # response, which make be read via a recv() on the underlying
+            # socket.
+            self.fp = sock.makefile('rb', 0)
+        self.debuglevel = debuglevel
+        self.strict = strict
+        self._method = method
+
+        self.msg = None
+
+        # from the Status-Line of the response
+        self.version = _UNKNOWN # HTTP-Version
+        self.status = _UNKNOWN  # Status-Code
+        self.reason = _UNKNOWN  # Reason-Phrase
+
+        self.chunked = _UNKNOWN         # is "chunked" being used?
+        self.chunk_left = _UNKNOWN      # bytes left to read in current chunk
+        self.length = _UNKNOWN          # number of bytes left in response
+        self.will_close = _UNKNOWN      # conn will close at end of response
+
+    def _read_status(self):
+        # Initialize with Simple-Response defaults
+        line = self.fp.readline()
+        if self.debuglevel > 0:
+            print "reply:", repr(line)
+        if not line:
+            # Presumably, the server closed the connection before
+            # sending a valid response.
+            raise BadStatusLine(line)
+        try:
+            [version, status, reason] = line.split(None, 2)
+        except ValueError:
+            try:
+                [version, status] = line.split(None, 1)
+                reason = ""
+            except ValueError:
+                # empty version will cause next test to fail and status
+                # will be treated as 0.9 response.
+                version = ""
+        if not version.startswith('HTTP/'):
+            if self.strict:
+                self.close()
+                raise BadStatusLine(line)
+            else:
+                # assume it's a Simple-Response from an 0.9 server
+                self.fp = LineAndFileWrapper(line, self.fp)
+                return "HTTP/0.9", 200, ""
+
+        # The status code is a three-digit number
+        try:
+            status = int(status)
+            if status < 100 or status > 999:
+                raise BadStatusLine(line)
+        except ValueError:
+            raise BadStatusLine(line)
+        return version, status, reason
+
+    def begin(self):
+        if self.msg is not None:
+            # we've already started reading the response
+            return
+
+        # read until we get a non-100 response
+        while True:
+            version, status, reason = self._read_status()
+            if status != CONTINUE:
+                break
+            # skip the header from the 100 response
+            while True:
+                skip = self.fp.readline().strip()
+                if not skip:
+                    break
+                if self.debuglevel > 0:
+                    print "header:", skip
+
+        self.status = status
+        self.reason = reason.strip()
+        if version == 'HTTP/1.0':
+            self.version = 10
+        elif version.startswith('HTTP/1.'):
+            self.version = 11   # use HTTP/1.1 code for HTTP/1.x where x>=1
+        elif version == 'HTTP/0.9':
+            self.version = 9
+        else:
+            raise UnknownProtocol(version)
+
+        if self.version == 9:
+            self.length = None
+            self.chunked = 0
+            self.will_close = 1
+            self.msg = HTTPMessage(StringIO())
+            return
+
+        self.msg = HTTPMessage(self.fp, 0)
+        if self.debuglevel > 0:
+            for hdr in self.msg.headers:
+                print "header:", hdr,
+
+        # don't let the msg keep an fp
+        self.msg.fp = None
+
+        # are we using the chunked-style of transfer encoding?
+        tr_enc = self.msg.getheader('transfer-encoding')
+        if tr_enc and tr_enc.lower() == "chunked":
+            self.chunked = 1
+            self.chunk_left = None
+        else:
+            self.chunked = 0
+
+        # will the connection close at the end of the response?
+        self.will_close = self._check_close()
+
+        # do we have a Content-Length?
+        # NOTE: RFC 2616, S4.4, #3 says we ignore this if tr_enc is "chunked"
+        length = self.msg.getheader('content-length')
+        if length and not self.chunked:
+            try:
+                self.length = int(length)
+            except ValueError:
+                self.length = None
+            else:
+                if self.length < 0:  # ignore nonsensical negative lengths
+                    self.length = None
+        else:
+            self.length = None
+
+        # does the body have a fixed length? (of zero)
+        if (status == NO_CONTENT or status == NOT_MODIFIED or
+            100 <= status < 200 or      # 1xx codes
+            self._method == 'HEAD'):
+            self.length = 0
+
+        # if the connection remains open, and we aren't using chunked, and
+        # a content-length was not provided, then assume that the connection
+        # WILL close.
+        if not self.will_close and \
+           not self.chunked and \
+           self.length is None:
+            self.will_close = 1
+
+    def _check_close(self):
+        conn = self.msg.getheader('connection')
+        if self.version == 11:
+            # An HTTP/1.1 proxy is assumed to stay open unless
+            # explicitly closed.
+            conn = self.msg.getheader('connection')
+            if conn and "close" in conn.lower():
+                return True
+            return False
+
+        # Some HTTP/1.0 implementations have support for persistent
+        # connections, using rules different than HTTP/1.1.
+
+        # For older HTTP, Keep-Alive indicates persistent connection.
+        if self.msg.getheader('keep-alive'):
+            return False
+
+        # At least Akamai returns a "Connection: Keep-Alive" header,
+        # which was supposed to be sent by the client.
+        if conn and "keep-alive" in conn.lower():
+            return False
+
+        # Proxy-Connection is a netscape hack.
+        pconn = self.msg.getheader('proxy-connection')
+        if pconn and "keep-alive" in pconn.lower():
+            return False
+
+        # otherwise, assume it will close
+        return True
+
+    def close(self):
+        if self.fp:
+            self.fp.close()
+            self.fp = None
+
+    def isclosed(self):
+        # NOTE: it is possible that we will not ever call self.close(). This
+        #       case occurs when will_close is TRUE, length is None, and we
+        #       read up to the last byte, but NOT past it.
+        #
+        # IMPLIES: if will_close is FALSE, then self.close() will ALWAYS be
+        #          called, meaning self.isclosed() is meaningful.
+        return self.fp is None
+
+    # XXX It would be nice to have readline and __iter__ for this, too.
+
+    def read(self, amt=None):
+        if self.fp is None:
+            return ''
+
+        if self._method == 'HEAD':
+            self.close()
+            return ''
+
+        if self.chunked:
+            return self._read_chunked(amt)
+
+        if amt is None:
+            # unbounded read
+            if self.length is None:
+                s = self.fp.read()
+            else:
+                s = self._safe_read(self.length)
+                self.length = 0
+            self.close()        # we read everything
+            return s
+
+        if self.length is not None:
+            if amt > self.length:
+                # clip the read to the "end of response"
+                amt = self.length
+
+        # we do not use _safe_read() here because this may be a .will_close
+        # connection, and the user is reading more bytes than will be provided
+        # (for example, reading in 1k chunks)
+        s = self.fp.read(amt)
+        if self.length is not None:
+            self.length -= len(s)
+            if not self.length:
+                self.close()
+        return s
+
+    def _read_chunked(self, amt):
+        assert self.chunked != _UNKNOWN
+        chunk_left = self.chunk_left
+        value = []
+        while True:
+            if chunk_left is None:
+                line = self.fp.readline()
+                i = line.find(';')
+                if i >= 0:
+                    line = line[:i] # strip chunk-extensions
+                try:
+                    chunk_left = int(line, 16)
+                except ValueError:
+                    # close the connection as protocol synchronisation is
+                    # probably lost
+                    self.close()
+                    raise IncompleteRead(''.join(value))
+                if chunk_left == 0:
+                    break
+            if amt is None:
+                value.append(self._safe_read(chunk_left))
+            elif amt < chunk_left:
+                value.append(self._safe_read(amt))
+                self.chunk_left = chunk_left - amt
+                return ''.join(value)
+            elif amt == chunk_left:
+                value.append(self._safe_read(amt))
+                self._safe_read(2)  # toss the CRLF at the end of the chunk
+                self.chunk_left = None
+                return ''.join(value)
+            else:
+                value.append(self._safe_read(chunk_left))
+                amt -= chunk_left
+
+            # we read the whole chunk, get another
+            self._safe_read(2)      # toss the CRLF at the end of the chunk
+            chunk_left = None
+
+        # read and discard trailer up to the CRLF terminator
+        ### note: we shouldn't have any trailers!
+        while True:
+            line = self.fp.readline()
+            if not line:
+                # a vanishingly small number of sites EOF without
+                # sending the trailer
+                break
+            if line == '\r\n':
+                break
+
+        # we read everything; close the "file"
+        self.close()
+
+        return ''.join(value)
+
+    def _safe_read(self, amt):
+        """Read the number of bytes requested, compensating for partial reads.
+
+        Normally, we have a blocking socket, but a read() can be interrupted
+        by a signal (resulting in a partial read).
+
+        Note that we cannot distinguish between EOF and an interrupt when zero
+        bytes have been read. IncompleteRead() will be raised in this
+        situation.
+
+        This function should be used when <amt> bytes "should" be present for
+        reading. If the bytes are truly not available (due to EOF), then the
+        IncompleteRead exception can be used to detect the problem.
+        """
+        # NOTE(gps): As of svn r74426 socket._fileobject.read(x) will never
+        # return less than x bytes unless EOF is encountered.  It now handles
+        # signal interruptions (socket.error EINTR) internally.  This code
+        # never caught that exception anyways.  It seems largely pointless.
+        # self.fp.read(amt) will work fine.
+        s = []
+        while amt > 0:
+            chunk = self.fp.read(min(amt, MAXAMOUNT))
+            if not chunk:
+                raise IncompleteRead(''.join(s), amt)
+            s.append(chunk)
+            amt -= len(chunk)
+        return ''.join(s)
+
+    def fileno(self):
+        return self.fp.fileno()
+
+    def getheader(self, name, default=None):
+        if self.msg is None:
+            raise ResponseNotReady()
+        return self.msg.getheader(name, default)
+
+    def getheaders(self):
+        """Return list of (header, value) tuples."""
+        if self.msg is None:
+            raise ResponseNotReady()
+        return self.msg.items()
+
+
+class HTTPConnection:
+
+    _http_vsn = 11
+    _http_vsn_str = 'HTTP/1.1'
+
+    response_class = HTTPResponse
+    default_port = HTTP_PORT
+    auto_open = 1
+    debuglevel = 0
+    strict = 0
+
+    def __init__(self, host, port=None, strict=None,
+                 timeout=socket._GLOBAL_DEFAULT_TIMEOUT, source_address=None):
+        self.timeout = timeout
+        self.source_address = source_address
+        self.sock = None
+        self._buffer = []
+        self.__response = None
+        self.__state = _CS_IDLE
+        self._method = None
+        self._tunnel_host = None
+        self._tunnel_port = None
+        self._tunnel_headers = {}
+
+        self._set_hostport(host, port)
+        if strict is not None:
+            self.strict = strict
+
+    def set_tunnel(self, host, port=None, headers=None):
+        """ Sets up the host and the port for the HTTP CONNECT Tunnelling.
+
+        The headers argument should be a mapping of extra HTTP headers
+        to send with the CONNECT request.
+        """
+        self._tunnel_host = host
+        self._tunnel_port = port
+        if headers:
+            self._tunnel_headers = headers
+        else:
+            self._tunnel_headers.clear()
+
+    def _set_hostport(self, host, port):
+        if port is None:
+            i = host.rfind(':')
+            j = host.rfind(']')         # ipv6 addresses have [...]
+            if i > j:
+                try:
+                    port = int(host[i+1:])
+                except ValueError:
+                    raise InvalidURL("nonnumeric port: '%s'" % host[i+1:])
+                host = host[:i]
+            else:
+                port = self.default_port
+            if host and host[0] == '[' and host[-1] == ']':
+                host = host[1:-1]
+        self.host = host
+        self.port = port
+
+    def set_debuglevel(self, level):
+        self.debuglevel = level
+
+    def _tunnel(self):
+        self._set_hostport(self._tunnel_host, self._tunnel_port)
+        self.send("CONNECT %s:%d HTTP/1.0\r\n" % (self.host, self.port))
+        for header, value in self._tunnel_headers.iteritems():
+            self.send("%s: %s\r\n" % (header, value))
+        self.send("\r\n")
+        response = self.response_class(self.sock, strict = self.strict,
+                                       method = self._method)
+        (version, code, message) = response._read_status()
+
+        if code != 200:
+            self.close()
+            raise socket.error("Tunnel connection failed: %d %s" % (code,
+                                                                    message.strip()))
+        while True:
+            line = response.fp.readline()
+            if line == '\r\n': break
+
+
+    def connect(self):
+        """Connect to the host and port specified in __init__."""
+        self.sock = socket.create_connection((self.host,self.port),
+                                             self.timeout, self.source_address)
+
+        if self._tunnel_host:
+            self._tunnel()
+
+    def close(self):
+        """Close the connection to the HTTP server."""
+        if self.sock:
+            self.sock.close()   # close it manually... there may be other refs
+            self.sock = None
+        if self.__response:
+            self.__response.close()
+            self.__response = None
+        self.__state = _CS_IDLE
+
+    def send(self, data):
+        """Send `data' to the server."""
+        if self.sock is None:
+            if self.auto_open:
+                self.connect()
+            else:
+                raise NotConnected()
+
+        if self.debuglevel > 0:
+            print "send:", repr(data)
+        blocksize = 8192
+        if hasattr(data,'read') and not isinstance(data, array):
+            if self.debuglevel > 0: print "sendIng a read()able"
+            datablock = data.read(blocksize)
+            while datablock:
+                self.sock.sendall(datablock)
+                datablock = data.read(blocksize)
+        else:
+            self.sock.sendall(data)
+
+    def _output(self, s):
+        """Add a line of output to the current request buffer.
+
+        Assumes that the line does *not* end with \\r\\n.
+        """
+        self._buffer.append(s)
+
+    def _send_output(self, message_body=None):
+        """Send the currently buffered request and clear the buffer.
+
+        Appends an extra \\r\\n to the buffer.
+        A message_body may be specified, to be appended to the request.
+        """
+        self._buffer.extend(("", ""))
+        msg = "\r\n".join(self._buffer)
+        del self._buffer[:]
+        # If msg and message_body are sent in a single send() call,
+        # it will avoid performance problems caused by the interaction
+        # between delayed ack and the Nagle algorithim.
+        if isinstance(message_body, str):
+            msg += message_body
+            message_body = None
+        self.send(msg)
+        if message_body is not None:
+            #message_body was not a string (i.e. it is a file) and
+            #we must run the risk of Nagle
+            self.send(message_body)
+
+    def putrequest(self, method, url, skip_host=0, skip_accept_encoding=0):
+        """Send a request to the server.
+
+        `method' specifies an HTTP request method, e.g. 'GET'.
+        `url' specifies the object being requested, e.g. '/index.html'.
+        `skip_host' if True does not add automatically a 'Host:' header
+        `skip_accept_encoding' if True does not add automatically an
+           'Accept-Encoding:' header
+        """
+
+        # if a prior response has been completed, then forget about it.
+        if self.__response and self.__response.isclosed():
+            self.__response = None
+
+
+        # in certain cases, we cannot issue another request on this connection.
+        # this occurs when:
+        #   1) we are in the process of sending a request.   (_CS_REQ_STARTED)
+        #   2) a response to a previous request has signalled that it is going
+        #      to close the connection upon completion.
+        #   3) the headers for the previous response have not been read, thus
+        #      we cannot determine whether point (2) is true.   (_CS_REQ_SENT)
+        #
+        # if there is no prior response, then we can request at will.
+        #
+        # if point (2) is true, then we will have passed the socket to the
+        # response (effectively meaning, "there is no prior response"), and
+        # will open a new one when a new request is made.
+        #
+        # Note: if a prior response exists, then we *can* start a new request.
+        #       We are not allowed to begin fetching the response to this new
+        #       request, however, until that prior response is complete.
+        #
+        if self.__state == _CS_IDLE:
+            self.__state = _CS_REQ_STARTED
+        else:
+            raise CannotSendRequest()
+
+        # Save the method we use, we need it later in the response phase
+        self._method = method
+        if not url:
+            url = '/'
+        hdr = '%s %s %s' % (method, url, self._http_vsn_str)
+
+        self._output(hdr)
+
+        if self._http_vsn == 11:
+            # Issue some standard headers for better HTTP/1.1 compliance
+
+            if not skip_host:
+                # this header is issued *only* for HTTP/1.1
+                # connections. more specifically, this means it is
+                # only issued when the client uses the new
+                # HTTPConnection() class. backwards-compat clients
+                # will be using HTTP/1.0 and those clients may be
+                # issuing this header themselves. we should NOT issue
+                # it twice; some web servers (such as Apache) barf
+                # when they see two Host: headers
+
+                # If we need a non-standard port,include it in the
+                # header.  If the request is going through a proxy,
+                # but the host of the actual URL, not the host of the
+                # proxy.
+
+                netloc = ''
+                if url.startswith('http'):
+                    nil, netloc, nil, nil, nil = urlsplit(url)
+
+                if netloc:
+                    try:
+                        netloc_enc = netloc.encode("ascii")
+                    except UnicodeEncodeError:
+                        netloc_enc = netloc.encode("idna")
+                    self.putheader('Host', netloc_enc)
+                else:
+                    try:
+                        host_enc = self.host.encode("ascii")
+                    except UnicodeEncodeError:
+                        host_enc = self.host.encode("idna")
+                    # Wrap the IPv6 Host Header with [] (RFC 2732)
+                    if host_enc.find(':') >= 0:
+                        host_enc = "[" + host_enc + "]"
+                    if self.port == self.default_port:
+                        self.putheader('Host', host_enc)
+                    else:
+                        self.putheader('Host', "%s:%s" % (host_enc, self.port))
+
+            # note: we are assuming that clients will not attempt to set these
+            #       headers since *this* library must deal with the
+            #       consequences. this also means that when the supporting
+            #       libraries are updated to recognize other forms, then this
+            #       code should be changed (removed or updated).
+
+            # we only want a Content-Encoding of "identity" since we don't
+            # support encodings such as x-gzip or x-deflate.
+            if not skip_accept_encoding:
+                self.putheader('Accept-Encoding', 'identity')
+
+            # we can accept "chunked" Transfer-Encodings, but no others
+            # NOTE: no TE header implies *only* "chunked"
+            #self.putheader('TE', 'chunked')
+
+            # if TE is supplied in the header, then it must appear in a
+            # Connection header.
+            #self.putheader('Connection', 'TE')
+
+        else:
+            # For HTTP/1.0, the server will assume "not chunked"
+            pass
+
+    def putheader(self, header, *values):
+        """Send a request header line to the server.
+
+        For example: h.putheader('Accept', 'text/html')
+        """
+        if self.__state != _CS_REQ_STARTED:
+            raise CannotSendHeader()
+
+        hdr = '%s: %s' % (header, '\r\n\t'.join([str(v) for v in values]))
+        self._output(hdr)
+
+    def endheaders(self, message_body=None):
+        """Indicate that the last header line has been sent to the server.
+
+        This method sends the request to the server.  The optional
+        message_body argument can be used to pass message body
+        associated with the request.  The message body will be sent in
+        the same packet as the message headers if possible.  The
+        message_body should be a string.
+        """
+        if self.__state == _CS_REQ_STARTED:
+            self.__state = _CS_REQ_SENT
+        else:
+            raise CannotSendHeader()
+        self._send_output(message_body)
+
+    def request(self, method, url, body=None, headers={}):
+        """Send a complete request to the server."""
+        self._send_request(method, url, body, headers)
+
+    def _set_content_length(self, body):
+        # Set the content-length based on the body.
+        thelen = None
+        try:
+            thelen = str(len(body))
+        except TypeError, te:
+            # If this is a file-like object, try to
+            # fstat its file descriptor
+            try:
+                thelen = str(os.fstat(body.fileno()).st_size)
+            except (AttributeError, OSError):
+                # Don't send a length if this failed
+                if self.debuglevel > 0: print "Cannot stat!!"
+
+        if thelen is not None:
+            self.putheader('Content-Length', thelen)
+
+    def _send_request(self, method, url, body, headers):
+        # Honor explicitly requested Host: and Accept-Encoding: headers.
+        header_names = dict.fromkeys([k.lower() for k in headers])
+        skips = {}
+        if 'host' in header_names:
+            skips['skip_host'] = 1
+        if 'accept-encoding' in header_names:
+            skips['skip_accept_encoding'] = 1
+
+        self.putrequest(method, url, **skips)
+
+        if body and ('content-length' not in header_names):
+            self._set_content_length(body)
+        for hdr, value in headers.iteritems():
+            self.putheader(hdr, value)
+        self.endheaders(body)
+
+    def getresponse(self, buffering=False):
+        "Get the response from the server."
+
+        # if a prior response has been completed, then forget about it.
+        if self.__response and self.__response.isclosed():
+            self.__response = None
+
+        #
+        # if a prior response exists, then it must be completed (otherwise, we
+        # cannot read this response's header to determine the connection-close
+        # behavior)
+        #
+        # note: if a prior response existed, but was connection-close, then the
+        # socket and response were made independent of this HTTPConnection
+        # object since a new request requires that we open a whole new
+        # connection
+        #
+        # this means the prior response had one of two states:
+        #   1) will_close: this connection was reset and the prior socket and
+        #                  response operate independently
+        #   2) persistent: the response was retained and we await its
+        #                  isclosed() status to become true.
+        #
+        if self.__state != _CS_REQ_SENT or self.__response:
+            raise ResponseNotReady()
+
+        args = (self.sock,)
+        kwds = {"strict":self.strict, "method":self._method}
+        if self.debuglevel > 0:
+            args += (self.debuglevel,)
+        if buffering:
+            #only add this keyword if non-default, for compatibility with
+            #other response_classes.
+            kwds["buffering"] = True;
+        response = self.response_class(*args, **kwds)
+
+        try:
+            response.begin()
+        except:
+            response.close()
+            raise
+        assert response.will_close != _UNKNOWN
+        self.__state = _CS_IDLE
+
+        if response.will_close:
+            # this effectively passes the connection to the response
+            self.close()
+        else:
+            # remember this, so we can tell when it is complete
+            self.__response = response
+
+        return response
+
+
+class HTTP:
+    "Compatibility class with httplib.py from 1.5."
+
+    _http_vsn = 10
+    _http_vsn_str = 'HTTP/1.0'
+
+    debuglevel = 0
+
+    _connection_class = HTTPConnection
+
+    def __init__(self, host='', port=None, strict=None):
+        "Provide a default host, since the superclass requires one."
+
+        # some joker passed 0 explicitly, meaning default port
+        if port == 0:
+            port = None
+
+        # Note that we may pass an empty string as the host; this will throw
+        # an error when we attempt to connect. Presumably, the client code
+        # will call connect before then, with a proper host.
+        self._setup(self._connection_class(host, port, strict))
+
+    def _setup(self, conn):
+        self._conn = conn
+
+        # set up delegation to flesh out interface
+        self.send = conn.send
+        self.putrequest = conn.putrequest
+        self.putheader = conn.putheader
+        self.endheaders = conn.endheaders
+        self.set_debuglevel = conn.set_debuglevel
+
+        conn._http_vsn = self._http_vsn
+        conn._http_vsn_str = self._http_vsn_str
+
+        self.file = None
+
+    def connect(self, host=None, port=None):
+        "Accept arguments to set the host/port, since the superclass doesn't."
+
+        if host is not None:
+            self._conn._set_hostport(host, port)
+        self._conn.connect()
+
+    def getfile(self):
+        "Provide a getfile, since the superclass' does not use this concept."
+        return self.file
+
+    def getreply(self, buffering=False):
+        """Compat definition since superclass does not define it.
+
+        Returns a tuple consisting of:
+        - server status code (e.g. '200' if all goes well)
+        - server "reason" corresponding to status code
+        - any RFC822 headers in the response from the server
+        """
+        try:
+            if not buffering:
+                response = self._conn.getresponse()
+            else:
+                #only add this keyword if non-default for compatibility
+                #with other connection classes
+                response = self._conn.getresponse(buffering)
+        except BadStatusLine, e:
+            ### hmm. if getresponse() ever closes the socket on a bad request,
+            ### then we are going to have problems with self.sock
+
+            ### should we keep this behavior? do people use it?
+            # keep the socket open (as a file), and return it
+            self.file = self._conn.sock.makefile('rb', 0)
+
+            # close our socket -- we want to restart after any protocol error
+            self.close()
+
+            self.headers = None
+            return -1, e.line, None
+
+        self.headers = response.msg
+        self.file = response.fp
+        return response.status, response.reason, response.msg
+
+    def close(self):
+        self._conn.close()
+
+        # note that self.file == response.fp, which gets closed by the
+        # superclass. just clear the object ref here.
+        ### hmm. messy. if status==-1, then self.file is owned by us.
+        ### well... we aren't explicitly closing, but losing this ref will
+        ### do it
+        self.file = None
+
+try:
+    import ssl
+except ImportError:
+    pass
+else:
+    class HTTPSConnection(HTTPConnection):
+        "This class allows communication via SSL."
+
+        default_port = HTTPS_PORT
+
+        def __init__(self, host, port=None, key_file=None, cert_file=None,
+                     strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
+                     source_address=None):
+            HTTPConnection.__init__(self, host, port, strict, timeout,
+                                    source_address)
+            self.key_file = key_file
+            self.cert_file = cert_file
+
+        def connect(self):
+            "Connect to a host on a given (SSL) port."
+
+            sock = socket.create_connection((self.host, self.port),
+                                            self.timeout, self.source_address)
+            if self._tunnel_host:
+                self.sock = sock
+                self._tunnel()
+            self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file)
+
+    __all__.append("HTTPSConnection")
+
+    class HTTPS(HTTP):
+        """Compatibility with 1.5 httplib interface
+
+        Python 1.5.2 did not have an HTTPS class, but it defined an
+        interface for sending http requests that is also useful for
+        https.
+        """
+
+        _connection_class = HTTPSConnection
+
+        def __init__(self, host='', port=None, key_file=None, cert_file=None,
+                     strict=None):
+            # provide a default host, pass the X509 cert info
+
+            # urf. compensate for bad input.
+            if port == 0:
+                port = None
+            self._setup(self._connection_class(host, port, key_file,
+                                               cert_file, strict))
+
+            # we never actually use these for anything, but we keep them
+            # here for compatibility with post-1.5.2 CVS.
+            self.key_file = key_file
+            self.cert_file = cert_file
+
+
+    def FakeSocket (sock, sslobj):
+        warnings.warn("FakeSocket is deprecated, and won't be in 3.x.  " +
+                      "Use the result of ssl.wrap_socket() directly instead.",
+                      DeprecationWarning, stacklevel=2)
+        return sslobj
+
+
+class HTTPException(Exception):
+    # Subclasses that define an __init__ must call Exception.__init__
+    # or define self.args.  Otherwise, str() will fail.
+    pass
+
+class NotConnected(HTTPException):
+    pass
+
+class InvalidURL(HTTPException):
+    pass
+
+class UnknownProtocol(HTTPException):
+    def __init__(self, version):
+        self.args = version,
+        self.version = version
+
+class UnknownTransferEncoding(HTTPException):
+    pass
+
+class UnimplementedFileMode(HTTPException):
+    pass
+
+class IncompleteRead(HTTPException):
+    def __init__(self, partial, expected=None):
+        self.args = partial,
+        self.partial = partial
+        self.expected = expected
+    def __repr__(self):
+        if self.expected is not None:
+            e = ', %i more expected' % self.expected
+        else:
+            e = ''
+        return 'IncompleteRead(%i bytes read%s)' % (len(self.partial), e)
+    def __str__(self):
+        return repr(self)
+
+class ImproperConnectionState(HTTPException):
+    pass
+
+class CannotSendRequest(ImproperConnectionState):
+    pass
+
+class CannotSendHeader(ImproperConnectionState):
+    pass
+
+class ResponseNotReady(ImproperConnectionState):
+    pass
+
+class BadStatusLine(HTTPException):
+    def __init__(self, line):
+        if not line:
+            line = repr(line)
+        self.args = line,
+        self.line = line
+
+# for backwards compatibility
+error = HTTPException
+
+class LineAndFileWrapper:
+    """A limited file-like object for HTTP/0.9 responses."""
+
+    # The status-line parsing code calls readline(), which normally
+    # get the HTTP status line.  For a 0.9 response, however, this is
+    # actually the first line of the body!  Clients need to get a
+    # readable file object that contains that line.
+
+    def __init__(self, line, file):
+        self._line = line
+        self._file = file
+        self._line_consumed = 0
+        self._line_offset = 0
+        self._line_left = len(line)
+
+    def __getattr__(self, attr):
+        return getattr(self._file, attr)
+
+    def _done(self):
+        # called when the last byte is read from the line.  After the
+        # call, all read methods are delegated to the underlying file
+        # object.
+        self._line_consumed = 1
+        self.read = self._file.read
+        self.readline = self._file.readline
+        self.readlines = self._file.readlines
+
+    def read(self, amt=None):
+        if self._line_consumed:
+            return self._file.read(amt)
+        assert self._line_left
+        if amt is None or amt > self._line_left:
+            s = self._line[self._line_offset:]
+            self._done()
+            if amt is None:
+                return s + self._file.read()
+            else:
+                return s + self._file.read(amt - len(s))
+        else:
+            assert amt <= self._line_left
+            i = self._line_offset
+            j = i + amt
+            s = self._line[i:j]
+            self._line_offset = j
+            self._line_left -= amt
+            if self._line_left == 0:
+                self._done()
+            return s
+
+    def readline(self):
+        if self._line_consumed:
+            return self._file.readline()
+        assert self._line_left
+        s = self._line[self._line_offset:]
+        self._done()
+        return s
+
+    def readlines(self, size=None):
+        if self._line_consumed:
+            return self._file.readlines(size)
+        assert self._line_left
+        L = [self._line[self._line_offset:]]
+        self._done()
+        if size is None:
+            return L + self._file.readlines()
+        else:
+            return L + self._file.readlines(size)
+
+def test():
+    """Test this module.
+
+    A hodge podge of tests collected here, because they have too many
+    external dependencies for the regular test suite.
+    """
+
+    import sys
+    import getopt
+    opts, args = getopt.getopt(sys.argv[1:], 'd')
+    dl = 0
+    for o, a in opts:
+        if o == '-d': dl = dl + 1
+    host = 'www.python.org'
+    selector = '/'
+    if args[0:]: host = args[0]
+    if args[1:]: selector = args[1]
+    h = HTTP()
+    h.set_debuglevel(dl)
+    h.connect(host)
+    h.putrequest('GET', selector)
+    h.endheaders()
+    status, reason, headers = h.getreply()
+    print 'status =', status
+    print 'reason =', reason
+    print "read", len(h.getfile().read())
+    print
+    if headers:
+        for header in headers.headers: print header.strip()
+    print
+
+    # minimal test that code to extract host from url works
+    class HTTP11(HTTP):
+        _http_vsn = 11
+        _http_vsn_str = 'HTTP/1.1'
+
+    h = HTTP11('www.python.org')
+    h.putrequest('GET', 'http://www.python.org/~jeremy/')
+    h.endheaders()
+    h.getreply()
+    h.close()
+
+    try:
+        import ssl
+    except ImportError:
+        pass
+    else:
+
+        for host, selector in (('sourceforge.net', '/projects/python'),
+                               ):
+            print "https://%s%s" % (host, selector)
+            hs = HTTPS()
+            hs.set_debuglevel(dl)
+            hs.connect(host)
+            hs.putrequest('GET', selector)
+            hs.endheaders()
+            status, reason, headers = hs.getreply()
+            print 'status =', status
+            print 'reason =', reason
+            print "read", len(hs.getfile().read())
+            print
+            if headers:
+                for header in headers.headers: print header.strip()
+            print
+
+if __name__ == '__main__':
+    test()
diff --git a/lib-python/modified-2.7/test/test_array.py b/lib-python/modified-2.7/test/test_array.py
--- a/lib-python/modified-2.7/test/test_array.py
+++ b/lib-python/modified-2.7/test/test_array.py
@@ -295,9 +295,10 @@
         )
 
         b = array.array(self.badtypecode())
-        self.assertRaises(TypeError, "a + b")
-
-        self.assertRaises(TypeError, "a + 'bad'")
+        with self.assertRaises(TypeError):
+            a + b
+        with self.assertRaises(TypeError):
+            a + 'bad'
 
     def test_iadd(self):
         a = array.array(self.typecode, self.example[::-1])
@@ -316,9 +317,10 @@
         )
 
         b = array.array(self.badtypecode())
-        self.assertRaises(TypeError, "a += b")
-
-        self.assertRaises(TypeError, "a += 'bad'")
+        with self.assertRaises(TypeError):
+            a += b
+        with self.assertRaises(TypeError):
+            a += 'bad'
 
     def test_mul(self):
         a = 5*array.array(self.typecode, self.example)
@@ -345,7 +347,8 @@
             array.array(self.typecode)
         )
 
-        self.assertRaises(TypeError, "a * 'bad'")
+        with self.assertRaises(TypeError):
+            a * 'bad'
 
     def test_imul(self):
         a = array.array(self.typecode, self.example)
@@ -374,7 +377,8 @@
         a *= -1
         self.assertEqual(a, array.array(self.typecode))
 
-        self.assertRaises(TypeError, "a *= 'bad'")
+        with self.assertRaises(TypeError):
+            a *= 'bad'
 
     def test_getitem(self):
         a = array.array(self.typecode, self.example)
diff --git a/lib-python/modified-2.7/test/test_multiprocessing.py b/lib-python/modified-2.7/test/test_multiprocessing.py
--- a/lib-python/modified-2.7/test/test_multiprocessing.py
+++ b/lib-python/modified-2.7/test/test_multiprocessing.py
@@ -510,7 +510,6 @@
 
         p.join()
 
-    @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
     def test_qsize(self):
         q = self.Queue()
         try:
@@ -532,7 +531,6 @@
             time.sleep(DELTA)
             q.task_done()
 
-    @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
     def test_task_done(self):
         queue = self.JoinableQueue()
 
@@ -1091,7 +1089,6 @@
 class _TestPoolWorkerLifetime(BaseTestCase):
 
     ALLOWED_TYPES = ('processes', )
-    @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
     def test_pool_worker_lifetime(self):
         p = multiprocessing.Pool(3, maxtasksperchild=10)
         self.assertEqual(3, len(p._pool))
@@ -1280,7 +1277,6 @@
         queue = manager.get_queue()
         queue.put('hello world')
 
-    @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
     def test_rapid_restart(self):
         authkey = os.urandom(32)
         manager = QueueManager(
@@ -1297,6 +1293,7 @@
         queue = manager.get_queue()
         self.assertEqual(queue.get(), 'hello world')
         del queue
+        test_support.gc_collect()
         manager.shutdown()
         manager = QueueManager(
             address=addr, authkey=authkey, serializer=SERIALIZER)
@@ -1573,7 +1570,6 @@
 
     ALLOWED_TYPES = ('processes',)
 
-    @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
     def test_heap(self):
         iterations = 5000
         maxblocks = 50
diff --git a/lib-python/modified-2.7/test/test_sys_settrace.py b/lib-python/modified-2.7/test/test_sys_settrace.py
--- a/lib-python/modified-2.7/test/test_sys_settrace.py
+++ b/lib-python/modified-2.7/test/test_sys_settrace.py
@@ -286,11 +286,11 @@
         self.compare_events(func.func_code.co_firstlineno,
                             tracer.events, func.events)
 
-    def set_and_retrieve_none(self):
+    def test_set_and_retrieve_none(self):
         sys.settrace(None)
         assert sys.gettrace() is None
 
-    def set_and_retrieve_func(self):
+    def test_set_and_retrieve_func(self):
         def fn(*args):
             pass
 
diff --git a/lib-python/modified-2.7/test/test_urllib2.py b/lib-python/modified-2.7/test/test_urllib2.py
--- a/lib-python/modified-2.7/test/test_urllib2.py
+++ b/lib-python/modified-2.7/test/test_urllib2.py
@@ -307,6 +307,9 @@
     def getresponse(self):
         return MockHTTPResponse(MockFile(), {}, 200, "OK")
 
+    def close(self):
+        pass
+
 class MockHandler:
     # useful for testing handler machinery
     # see add_ordered_mock_handlers() docstring
diff --git a/lib-python/modified-2.7/urllib2.py b/lib-python/modified-2.7/urllib2.py
new file mode 100644
--- /dev/null
+++ b/lib-python/modified-2.7/urllib2.py
@@ -0,0 +1,1440 @@
+"""An extensible library for opening URLs using a variety of protocols
+
+The simplest way to use this module is to call the urlopen function,
+which accepts a string containing a URL or a Request object (described
+below).  It opens the URL and returns the results as file-like
+object; the returned object has some extra methods described below.
+
+The OpenerDirector manages a collection of Handler objects that do
+all the actual work.  Each Handler implements a particular protocol or
+option.  The OpenerDirector is a composite object that invokes the
+Handlers needed to open the requested URL.  For example, the
+HTTPHandler performs HTTP GET and POST requests and deals with
+non-error returns.  The HTTPRedirectHandler automatically deals with
+HTTP 301, 302, 303 and 307 redirect errors, and the HTTPDigestAuthHandler
+deals with digest authentication.
+
+urlopen(url, data=None) -- Basic usage is the same as original
+urllib.  pass the url and optionally data to post to an HTTP URL, and
+get a file-like object back.  One difference is that you can also pass
+a Request instance instead of URL.  Raises a URLError (subclass of
+IOError); for HTTP errors, raises an HTTPError, which can also be
+treated as a valid response.
+
+build_opener -- Function that creates a new OpenerDirector instance.
+Will install the default handlers.  Accepts one or more Handlers as
+arguments, either instances or Handler classes that it will
+instantiate.  If one of the argument is a subclass of the default
+handler, the argument will be installed instead of the default.
+
+install_opener -- Installs a new opener as the default opener.
+
+objects of interest:
+
+OpenerDirector -- Sets up the User Agent as the Python-urllib client and manages
+the Handler classes, while dealing with requests and responses.
+
+Request -- An object that encapsulates the state of a request.  The
+state can be as simple as the URL.  It can also include extra HTTP
+headers, e.g. a User-Agent.
+
+BaseHandler --
+
+exceptions:
+URLError -- A subclass of IOError, individual protocols have their own
+specific subclass.
+
+HTTPError -- Also a valid HTTP response, so you can treat an HTTP error
+as an exceptional event or valid response.
+
+internals:
+BaseHandler and parent
+_call_chain conventions
+
+Example usage:
+
+import urllib2
+
+# set up authentication info
+authinfo = urllib2.HTTPBasicAuthHandler()
+authinfo.add_password(realm='PDQ Application',
+                      uri='https://mahler:8092/site-updates.py',
+                      user='klem',
+                      passwd='geheim$parole')
+
+proxy_support = urllib2.ProxyHandler({"http" : "http://ahad-haam:3128"})
+
+# build a new opener that adds authentication and caching FTP handlers
+opener = urllib2.build_opener(proxy_support, authinfo, urllib2.CacheFTPHandler)
+
+# install it
+urllib2.install_opener(opener)
+
+f = urllib2.urlopen('http://www.python.org/')
+
+
+"""
+
+# XXX issues:
+# If an authentication error handler that tries to perform
+# authentication for some reason but fails, how should the error be
+# signalled?  The client needs to know the HTTP error code.  But if
+# the handler knows that the problem was, e.g., that it didn't know
+# that hash algo that requested in the challenge, it would be good to
+# pass that information along to the client, too.
+# ftp errors aren't handled cleanly
+# check digest against correct (i.e. non-apache) implementation
+
+# Possible extensions:
+# complex proxies  XXX not sure what exactly was meant by this
+# abstract factory for opener
+
+import base64
+import hashlib
+import httplib
+import mimetools
+import os
+import posixpath
+import random
+import re
+import socket
+import sys
+import time
+import urlparse
+import bisect
+
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from StringIO import StringIO
+
+from urllib import (unwrap, unquote, splittype, splithost, quote,
+     addinfourl, splitport, splittag,
+     splitattr, ftpwrapper, splituser, splitpasswd, splitvalue)
+
+# support for FileHandler, proxies via environment variables
+from urllib import localhost, url2pathname, getproxies, proxy_bypass
+
+# used in User-Agent header sent
+__version__ = sys.version[:3]
+
+_opener = None
+def urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
+    global _opener
+    if _opener is None:
+        _opener = build_opener()
+    return _opener.open(url, data, timeout)
+
+def install_opener(opener):
+    global _opener
+    _opener = opener
+
+# do these error classes make sense?
+# make sure all of the IOError stuff is overridden.  we just want to be
+# subtypes.
+
+class URLError(IOError):
+    # URLError is a sub-type of IOError, but it doesn't share any of
+    # the implementation.  need to override __init__ and __str__.
+    # It sets self.args for compatibility with other EnvironmentError
+    # subclasses, but args doesn't have the typical format with errno in
+    # slot 0 and strerror in slot 1.  This may be better than nothing.
+    def __init__(self, reason):
+        self.args = reason,
+        self.reason = reason
+
+    def __str__(self):
+        return '<urlopen error %s>' % self.reason
+
+class HTTPError(URLError, addinfourl):
+    """Raised when HTTP error occurs, but also acts like non-error return"""
+    __super_init = addinfourl.__init__
+
+    def __init__(self, url, code, msg, hdrs, fp):
+        self.code = code
+        self.msg = msg
+        self.hdrs = hdrs
+        self.fp = fp
+        self.filename = url
+        # The addinfourl classes depend on fp being a valid file
+        # object.  In some cases, the HTTPError may not have a valid
+        # file object.  If this happens, the simplest workaround is to
+        # not initialize the base classes.
+        if fp is not None:
+            self.__super_init(fp, hdrs, url, code)
+
+    def __str__(self):
+        return 'HTTP Error %s: %s' % (self.code, self.msg)
+
+# copied from cookielib.py
+_cut_port_re = re.compile(r":\d+$")
+def request_host(request):
+    """Return request-host, as defined by RFC 2965.
+
+    Variation from RFC: returned value is lowercased, for convenient
+    comparison.
+
+    """
+    url = request.get_full_url()
+    host = urlparse.urlparse(url)[1]
+    if host == "":
+        host = request.get_header("Host", "")
+
+    # remove port, if present
+    host = _cut_port_re.sub("", host, 1)
+    return host.lower()
+
+class Request:
+
+    def __init__(self, url, data=None, headers={},
+                 origin_req_host=None, unverifiable=False):
+        # unwrap('<URL:type://host/path>') --> 'type://host/path'
+        self.__original = unwrap(url)
+        self.__original, fragment = splittag(self.__original)
+        self.type = None
+        # self.__r_type is what's left after doing the splittype
+        self.host = None
+        self.port = None
+        self._tunnel_host = None
+        self.data = data
+        self.headers = {}
+        for key, value in headers.items():
+            self.add_header(key, value)
+        self.unredirected_hdrs = {}
+        if origin_req_host is None:
+            origin_req_host = request_host(self)
+        self.origin_req_host = origin_req_host
+        self.unverifiable = unverifiable
+
+    def __getattr__(self, attr):
+        # XXX this is a fallback mechanism to guard against these
+        # methods getting called in a non-standard order.  this may be
+        # too complicated and/or unnecessary.
+        # XXX should the __r_XXX attributes be public?
+        if attr[:12] == '_Request__r_':
+            name = attr[12:]
+            if hasattr(Request, 'get_' + name):
+                getattr(self, 'get_' + name)()
+                return getattr(self, attr)
+        raise AttributeError, attr
+
+    def get_method(self):
+        if self.has_data():
+            return "POST"
+        else:
+            return "GET"
+
+    # XXX these helper methods are lame
+
+    def add_data(self, data):
+        self.data = data
+
+    def has_data(self):
+        return self.data is not None
+
+    def get_data(self):
+        return self.data
+
+    def get_full_url(self):
+        return self.__original
+
+    def get_type(self):
+        if self.type is None:
+            self.type, self.__r_type = splittype(self.__original)
+            if self.type is None:
+                raise ValueError, "unknown url type: %s" % self.__original
+        return self.type
+
+    def get_host(self):
+        if self.host is None:
+            self.host, self.__r_host = splithost(self.__r_type)
+            if self.host:
+                self.host = unquote(self.host)
+        return self.host
+
+    def get_selector(self):
+        return self.__r_host
+
+    def set_proxy(self, host, type):
+        if self.type == 'https' and not self._tunnel_host:
+            self._tunnel_host = self.host
+        else:
+            self.type = type
+            self.__r_host = self.__original
+
+        self.host = host
+
+    def has_proxy(self):
+        return self.__r_host == self.__original
+
+    def get_origin_req_host(self):
+        return self.origin_req_host
+
+    def is_unverifiable(self):
+        return self.unverifiable
+
+    def add_header(self, key, val):
+        # useful for something like authentication
+        self.headers[key.capitalize()] = val
+
+    def add_unredirected_header(self, key, val):
+        # will not be added to a redirected request
+        self.unredirected_hdrs[key.capitalize()] = val
+
+    def has_header(self, header_name):
+        return (header_name in self.headers or
+                header_name in self.unredirected_hdrs)
+
+    def get_header(self, header_name, default=None):
+        return self.headers.get(
+            header_name,
+            self.unredirected_hdrs.get(header_name, default))
+
+    def header_items(self):
+        hdrs = self.unredirected_hdrs.copy()
+        hdrs.update(self.headers)
+        return hdrs.items()
+
+class OpenerDirector:
+    def __init__(self):
+        client_version = "Python-urllib/%s" % __version__
+        self.addheaders = [('User-agent', client_version)]
+        # manage the individual handlers
+        self.handlers = []
+        self.handle_open = {}
+        self.handle_error = {}
+        self.process_response = {}
+        self.process_request = {}
+
+    def add_handler(self, handler):
+        if not hasattr(handler, "add_parent"):
+            raise TypeError("expected BaseHandler instance, got %r" %
+                            type(handler))
+
+        added = False
+        for meth in dir(handler):
+            if meth in ["redirect_request", "do_open", "proxy_open"]:
+                # oops, coincidental match
+                continue
+
+            i = meth.find("_")
+            protocol = meth[:i]
+            condition = meth[i+1:]
+
+            if condition.startswith("error"):
+                j = condition.find("_") + i + 1
+                kind = meth[j+1:]
+                try:
+                    kind = int(kind)
+                except ValueError:
+                    pass
+                lookup = self.handle_error.get(protocol, {})
+                self.handle_error[protocol] = lookup
+            elif condition == "open":
+                kind = protocol
+                lookup = self.handle_open
+            elif condition == "response":
+                kind = protocol
+                lookup = self.process_response
+            elif condition == "request":
+                kind = protocol
+                lookup = self.process_request
+            else:
+                continue
+
+            handlers = lookup.setdefault(kind, [])
+            if handlers:
+                bisect.insort(handlers, handler)
+            else:
+                handlers.append(handler)
+            added = True
+
+        if added:
+            # the handlers must work in an specific order, the order
+            # is specified in a Handler attribute
+            bisect.insort(self.handlers, handler)
+            handler.add_parent(self)
+
+    def close(self):
+        # Only exists for backwards compatibility.
+        pass
+
+    def _call_chain(self, chain, kind, meth_name, *args):
+        # Handlers raise an exception if no one else should try to handle
+        # the request, or return None if they can't but another handler
+        # could.  Otherwise, they return the response.
+        handlers = chain.get(kind, ())
+        for handler in handlers:
+            func = getattr(handler, meth_name)
+
+            result = func(*args)
+            if result is not None:
+                return result
+
+    def open(self, fullurl, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
+        # accept a URL or a Request object
+        if isinstance(fullurl, basestring):
+            req = Request(fullurl, data)
+        else:
+            req = fullurl
+            if data is not None:
+                req.add_data(data)
+
+        req.timeout = timeout
+        protocol = req.get_type()
+
+        # pre-process request
+        meth_name = protocol+"_request"
+        for processor in self.process_request.get(protocol, []):
+            meth = getattr(processor, meth_name)
+            req = meth(req)
+
+        response = self._open(req, data)
+
+        # post-process response
+        meth_name = protocol+"_response"
+        for processor in self.process_response.get(protocol, []):
+            meth = getattr(processor, meth_name)
+            try:
+                response = meth(req, response)
+            except:
+                response.close()
+                raise
+
+        return response
+
+    def _open(self, req, data=None):
+        result = self._call_chain(self.handle_open, 'default',
+                                  'default_open', req)
+        if result:
+            return result
+
+        protocol = req.get_type()
+        result = self._call_chain(self.handle_open, protocol, protocol +
+                                  '_open', req)
+        if result:
+            return result
+
+        return self._call_chain(self.handle_open, 'unknown',
+                                'unknown_open', req)
+
+    def error(self, proto, *args):
+        if proto in ('http', 'https'):
+            # XXX http[s] protocols are special-cased
+            dict = self.handle_error['http'] # https is not different than http
+            proto = args[2]  # YUCK!
+            meth_name = 'http_error_%s' % proto
+            http_err = 1
+            orig_args = args
+        else:
+            dict = self.handle_error
+            meth_name = proto + '_error'
+            http_err = 0
+        args = (dict, proto, meth_name) + args
+        result = self._call_chain(*args)
+        if result:
+            return result
+
+        if http_err:
+            args = (dict, 'default', 'http_error_default') + orig_args
+            return self._call_chain(*args)
+
+# XXX probably also want an abstract factory that knows when it makes
+# sense to skip a superclass in favor of a subclass and when it might
+# make sense to include both
+
+def build_opener(*handlers):
+    """Create an opener object from a list of handlers.
+
+    The opener will use several default handlers, including support
+    for HTTP, FTP and when applicable, HTTPS.
+
+    If any of the handlers passed as arguments are subclasses of the
+    default handlers, the default handlers will not be used.
+    """
+    import types
+    def isclass(obj):
+        return isinstance(obj, (types.ClassType, type))
+
+    opener = OpenerDirector()
+    default_classes = [ProxyHandler, UnknownHandler, HTTPHandler,
+                       HTTPDefaultErrorHandler, HTTPRedirectHandler,
+                       FTPHandler, FileHandler, HTTPErrorProcessor]
+    if hasattr(httplib, 'HTTPS'):
+        default_classes.append(HTTPSHandler)
+    skip = set()
+    for klass in default_classes:
+        for check in handlers:
+            if isclass(check):
+                if issubclass(check, klass):
+                    skip.add(klass)
+            elif isinstance(check, klass):
+                skip.add(klass)
+    for klass in skip:
+        default_classes.remove(klass)
+
+    for klass in default_classes:
+        opener.add_handler(klass())
+
+    for h in handlers:
+        if isclass(h):
+            h = h()
+        opener.add_handler(h)
+    return opener
+
+class BaseHandler:
+    handler_order = 500
+
+    def add_parent(self, parent):
+        self.parent = parent
+
+    def close(self):
+        # Only exists for backwards compatibility
+        pass
+
+    def __lt__(self, other):
+        if not hasattr(other, "handler_order"):
+            # Try to preserve the old behavior of having custom classes
+            # inserted after default ones (works only for custom user
+            # classes which are not aware of handler_order).
+            return True
+        return self.handler_order < other.handler_order
+
+
+class HTTPErrorProcessor(BaseHandler):
+    """Process HTTP error responses."""
+    handler_order = 1000  # after all other processing
+
+    def http_response(self, request, response):
+        code, msg, hdrs = response.code, response.msg, response.info()
+
+        # According to RFC 2616, "2xx" code indicates that the client's
+        # request was successfully received, understood, and accepted.
+        if not (200 <= code < 300):
+            response = self.parent.error(
+                'http', request, response, code, msg, hdrs)
+
+        return response
+
+    https_response = http_response
+
+class HTTPDefaultErrorHandler(BaseHandler):
+    def http_error_default(self, req, fp, code, msg, hdrs):
+        raise HTTPError(req.get_full_url(), code, msg, hdrs, fp)
+
+class HTTPRedirectHandler(BaseHandler):
+    # maximum number of redirections to any single URL
+    # this is needed because of the state that cookies introduce
+    max_repeats = 4
+    # maximum total number of redirections (regardless of URL) before
+    # assuming we're in a loop
+    max_redirections = 10
+
+    def redirect_request(self, req, fp, code, msg, headers, newurl):
+        """Return a Request or None in response to a redirect.
+
+        This is called by the http_error_30x methods when a
+        redirection response is received.  If a redirection should
+        take place, return a new Request to allow http_error_30x to
+        perform the redirect.  Otherwise, raise HTTPError if no-one
+        else should try to handle this url.  Return None if you can't
+        but another Handler might.
+        """
+        m = req.get_method()
+        if (code in (301, 302, 303, 307) and m in ("GET", "HEAD")
+            or code in (301, 302, 303) and m == "POST"):
+            # Strictly (according to RFC 2616), 301 or 302 in response
+            # to a POST MUST NOT cause a redirection without confirmation
+            # from the user (of urllib2, in this case).  In practice,
+            # essentially all clients do redirect in this case, so we
+            # do the same.
+            # be conciliant with URIs containing a space
+            newurl = newurl.replace(' ', '%20')
+            newheaders = dict((k,v) for k,v in req.headers.items()
+                              if k.lower() not in ("content-length", "content-type")
+                             )
+            return Request(newurl,
+                           headers=newheaders,
+                           origin_req_host=req.get_origin_req_host(),
+                           unverifiable=True)
+        else:
+            raise HTTPError(req.get_full_url(), code, msg, headers, fp)
+
+    # Implementation note: To avoid the server sending us into an
+    # infinite loop, the request object needs to track what URLs we
+    # have already seen.  Do this by adding a handler-specific
+    # attribute to the Request object.
+    def http_error_302(self, req, fp, code, msg, headers):
+        # Some servers (incorrectly) return multiple Location headers
+        # (so probably same goes for URI).  Use first header.
+        if 'location' in headers:
+            newurl = headers.getheaders('location')[0]
+        elif 'uri' in headers:
+            newurl = headers.getheaders('uri')[0]
+        else:
+            return
+
+        # fix a possible malformed URL
+        urlparts = urlparse.urlparse(newurl)
+        if not urlparts.path:
+            urlparts = list(urlparts)
+            urlparts[2] = "/"
+        newurl = urlparse.urlunparse(urlparts)
+
+        newurl = urlparse.urljoin(req.get_full_url(), newurl)
+
+        # XXX Probably want to forget about the state of the current
+        # request, although that might interact poorly with other
+        # handlers that also use handler-specific request attributes
+        new = self.redirect_request(req, fp, code, msg, headers, newurl)
+        if new is None:
+            return
+
+        # loop detection
+        # .redirect_dict has a key url if url was previously visited.
+        if hasattr(req, 'redirect_dict'):
+            visited = new.redirect_dict = req.redirect_dict
+            if (visited.get(newurl, 0) >= self.max_repeats or
+                len(visited) >= self.max_redirections):
+                raise HTTPError(req.get_full_url(), code,
+                                self.inf_msg + msg, headers, fp)
+        else:
+            visited = new.redirect_dict = req.redirect_dict = {}
+        visited[newurl] = visited.get(newurl, 0) + 1
+
+        # Don't close the fp until we are sure that we won't use it
+        # with HTTPError.
+        fp.read()
+        fp.close()
+
+        return self.parent.open(new, timeout=req.timeout)
+
+    http_error_301 = http_error_303 = http_error_307 = http_error_302
+
+    inf_msg = "The HTTP server returned a redirect error that would " \
+              "lead to an infinite loop.\n" \
+              "The last 30x error message was:\n"
+
+
+def _parse_proxy(proxy):
+    """Return (scheme, user, password, host/port) given a URL or an authority.
+
+    If a URL is supplied, it must have an authority (host:port) component.
+    According to RFC 3986, having an authority component means the URL must
+    have two slashes after the scheme:
+
+    >>> _parse_proxy('file:/ftp.example.com/')
+    Traceback (most recent call last):
+    ValueError: proxy URL with no authority: 'file:/ftp.example.com/'
+
+    The first three items of the returned tuple may be None.
+
+    Examples of authority parsing:
+
+    >>> _parse_proxy('proxy.example.com')
+    (None, None, None, 'proxy.example.com')
+    >>> _parse_proxy('proxy.example.com:3128')
+    (None, None, None, 'proxy.example.com:3128')
+
+    The authority component may optionally include userinfo (assumed to be
+    username:password):
+
+    >>> _parse_proxy('joe:password at proxy.example.com')
+    (None, 'joe', 'password', 'proxy.example.com')
+    >>> _parse_proxy('joe:password at proxy.example.com:3128')
+    (None, 'joe', 'password', 'proxy.example.com:3128')
+
+    Same examples, but with URLs instead:
+
+    >>> _parse_proxy('http://proxy.example.com/')
+    ('http', None, None, 'proxy.example.com')
+    >>> _parse_proxy('http://proxy.example.com:3128/')
+    ('http', None, None, 'proxy.example.com:3128')
+    >>> _parse_proxy('http://joe:password at proxy.example.com/')
+    ('http', 'joe', 'password', 'proxy.example.com')
+    >>> _parse_proxy('http://joe:password at proxy.example.com:3128')
+    ('http', 'joe', 'password', 'proxy.example.com:3128')
+
+    Everything after the authority is ignored:
+
+    >>> _parse_proxy('ftp://joe:password at proxy.example.com/rubbish:3128')
+    ('ftp', 'joe', 'password', 'proxy.example.com')
+
+    Test for no trailing '/' case:
+
+    >>> _parse_proxy('http://joe:password at proxy.example.com')
+    ('http', 'joe', 'password', 'proxy.example.com')
+
+    """
+    scheme, r_scheme = splittype(proxy)
+    if not r_scheme.startswith("/"):
+        # authority
+        scheme = None
+        authority = proxy
+    else:
+        # URL
+        if not r_scheme.startswith("//"):
+            raise ValueError("proxy URL with no authority: %r" % proxy)
+        # We have an authority, so for RFC 3986-compliant URLs (by ss 3.
+        # and 3.3.), path is empty or starts with '/'
+        end = r_scheme.find("/", 2)
+        if end == -1:
+            end = None
+        authority = r_scheme[2:end]
+    userinfo, hostport = splituser(authority)
+    if userinfo is not None:
+        user, password = splitpasswd(userinfo)
+    else:
+        user = password = None
+    return scheme, user, password, hostport
+
+class ProxyHandler(BaseHandler):
+    # Proxies must be in front
+    handler_order = 100
+
+    def __init__(self, proxies=None):
+        if proxies is None:
+            proxies = getproxies()
+        assert hasattr(proxies, 'has_key'), "proxies must be a mapping"
+        self.proxies = proxies
+        for type, url in proxies.items():
+            setattr(self, '%s_open' % type,
+                    lambda r, proxy=url, type=type, meth=self.proxy_open: \
+                    meth(r, proxy, type))
+
+    def proxy_open(self, req, proxy, type):
+        orig_type = req.get_type()
+        proxy_type, user, password, hostport = _parse_proxy(proxy)
+
+        if proxy_type is None:
+            proxy_type = orig_type
+
+        if req.host and proxy_bypass(req.host):
+            return None
+
+        if user and password:
+            user_pass = '%s:%s' % (unquote(user), unquote(password))
+            creds = base64.b64encode(user_pass).strip()
+            req.add_header('Proxy-authorization', 'Basic ' + creds)
+        hostport = unquote(hostport)
+        req.set_proxy(hostport, proxy_type)
+
+        if orig_type == proxy_type or orig_type == 'https':
+            # let other handlers take care of it
+            return None
+        else:
+            # need to start over, because the other handlers don't
+            # grok the proxy's URL type
+            # e.g. if we have a constructor arg proxies like so:
+            # {'http': 'ftp://proxy.example.com'}, we may end up turning
+            # a request for http://acme.example.com/a into one for
+            # ftp://proxy.example.com/a
+            return self.parent.open(req, timeout=req.timeout)
+
+class HTTPPasswordMgr:
+
+    def __init__(self):
+        self.passwd = {}
+
+    def add_password(self, realm, uri, user, passwd):
+        # uri could be a single URI or a sequence
+        if isinstance(uri, basestring):
+            uri = [uri]
+        if not realm in self.passwd:
+            self.passwd[realm] = {}
+        for default_port in True, False:
+            reduced_uri = tuple(
+                [self.reduce_uri(u, default_port) for u in uri])
+            self.passwd[realm][reduced_uri] = (user, passwd)
+
+    def find_user_password(self, realm, authuri):
+        domains = self.passwd.get(realm, {})
+        for default_port in True, False:
+            reduced_authuri = self.reduce_uri(authuri, default_port)
+            for uris, authinfo in domains.iteritems():
+                for uri in uris:
+                    if self.is_suburi(uri, reduced_authuri):
+                        return authinfo
+        return None, None
+
+    def reduce_uri(self, uri, default_port=True):
+        """Accept authority or URI and extract only the authority and path."""
+        # note HTTP URLs do not have a userinfo component
+        parts = urlparse.urlsplit(uri)
+        if parts[1]:
+            # URI
+            scheme = parts[0]
+            authority = parts[1]
+            path = parts[2] or '/'
+        else:
+            # host or host:port
+            scheme = None
+            authority = uri
+            path = '/'
+        host, port = splitport(authority)
+        if default_port and port is None and scheme is not None:
+            dport = {"http": 80,
+                     "https": 443,
+                     }.get(scheme)
+            if dport is not None:
+                authority = "%s:%d" % (host, dport)
+        return authority, path
+
+    def is_suburi(self, base, test):
+        """Check if test is below base in a URI tree
+
+        Both args must be URIs in reduced form.
+        """
+        if base == test:
+            return True
+        if base[0] != test[0]:
+            return False
+        common = posixpath.commonprefix((base[1], test[1]))
+        if len(common) == len(base[1]):
+            return True
+        return False
+
+
+class HTTPPasswordMgrWithDefaultRealm(HTTPPasswordMgr):
+
+    def find_user_password(self, realm, authuri):
+        user, password = HTTPPasswordMgr.find_user_password(self, realm,
+                                                            authuri)
+        if user is not None:
+            return user, password
+        return HTTPPasswordMgr.find_user_password(self, None, authuri)
+
+
+class AbstractBasicAuthHandler:
+
+    # XXX this allows for multiple auth-schemes, but will stupidly pick
+    # the last one with a realm specified.
+
+    # allow for double- and single-quoted realm values
+    # (single quotes are a violation of the RFC, but appear in the wild)
+    rx = re.compile('(?:.*,)*[ \t]*([^ \t]+)[ \t]+'
+                    'realm=(["\'])(.*?)\\2', re.I)
+
+    # XXX could pre-emptively send auth info already accepted (RFC 2617,
+    # end of section 2, and section 1.2 immediately after "credentials"
+    # production).
+
+    def __init__(self, password_mgr=None):
+        if password_mgr is None:
+            password_mgr = HTTPPasswordMgr()
+        self.passwd = password_mgr
+        self.add_password = self.passwd.add_password
+        self.retried = 0
+
+    def reset_retry_count(self):
+        self.retried = 0
+
+    def http_error_auth_reqed(self, authreq, host, req, headers):
+        # host may be an authority (without userinfo) or a URL with an
+        # authority
+        # XXX could be multiple headers
+        authreq = headers.get(authreq, None)
+
+        if self.retried > 5:
+            # retry sending the username:password 5 times before failing.
+            raise HTTPError(req.get_full_url(), 401, "basic auth failed",
+                            headers, None)
+        else:
+            self.retried += 1
+
+        if authreq:
+            mo = AbstractBasicAuthHandler.rx.search(authreq)
+            if mo:
+                scheme, quote, realm = mo.groups()
+                if scheme.lower() == 'basic':
+                    response = self.retry_http_basic_auth(host, req, realm)
+                    if response and response.code != 401:
+                        self.retried = 0
+                    return response
+
+    def retry_http_basic_auth(self, host, req, realm):
+        user, pw = self.passwd.find_user_password(realm, host)
+        if pw is not None:
+            raw = "%s:%s" % (user, pw)
+            auth = 'Basic %s' % base64.b64encode(raw).strip()
+            if req.headers.get(self.auth_header, None) == auth:
+                return None
+            req.add_unredirected_header(self.auth_header, auth)
+            return self.parent.open(req, timeout=req.timeout)
+        else:
+            return None
+
+
+class HTTPBasicAuthHandler(AbstractBasicAuthHandler, BaseHandler):
+
+    auth_header = 'Authorization'
+
+    def http_error_401(self, req, fp, code, msg, headers):
+        url = req.get_full_url()
+        response = self.http_error_auth_reqed('www-authenticate',
+                                              url, req, headers)
+        self.reset_retry_count()
+        return response
+
+
+class ProxyBasicAuthHandler(AbstractBasicAuthHandler, BaseHandler):
+
+    auth_header = 'Proxy-authorization'
+
+    def http_error_407(self, req, fp, code, msg, headers):
+        # http_error_auth_reqed requires that there is no userinfo component in
+        # authority.  Assume there isn't one, since urllib2 does not (and
+        # should not, RFC 3986 s. 3.2.1) support requests for URLs containing
+        # userinfo.
+        authority = req.get_host()
+        response = self.http_error_auth_reqed('proxy-authenticate',
+                                          authority, req, headers)
+        self.reset_retry_count()
+        return response
+
+
+def randombytes(n):
+    """Return n random bytes."""
+    # Use /dev/urandom if it is available.  Fall back to random module
+    # if not.  It might be worthwhile to extend this function to use
+    # other platform-specific mechanisms for getting random bytes.
+    if os.path.exists("/dev/urandom"):
+        f = open("/dev/urandom")
+        s = f.read(n)
+        f.close()
+        return s
+    else:
+        L = [chr(random.randrange(0, 256)) for i in range(n)]
+        return "".join(L)
+
+class AbstractDigestAuthHandler:
+    # Digest authentication is specified in RFC 2617.
+
+    # XXX The client does not inspect the Authentication-Info header
+    # in a successful response.
+
+    # XXX It should be possible to test this implementation against
+    # a mock server that just generates a static set of challenges.
+
+    # XXX qop="auth-int" supports is shaky
+
+    def __init__(self, passwd=None):
+        if passwd is None:
+            passwd = HTTPPasswordMgr()
+        self.passwd = passwd
+        self.add_password = self.passwd.add_password
+        self.retried = 0
+        self.nonce_count = 0
+        self.last_nonce = None
+
+    def reset_retry_count(self):
+        self.retried = 0
+
+    def http_error_auth_reqed(self, auth_header, host, req, headers):
+        authreq = headers.get(auth_header, None)
+        if self.retried > 5:
+            # Don't fail endlessly - if we failed once, we'll probably
+            # fail a second time. Hm. Unless the Password Manager is
+            # prompting for the information. Crap. This isn't great
+            # but it's better than the current 'repeat until recursion
+            # depth exceeded' approach <wink>
+            raise HTTPError(req.get_full_url(), 401, "digest auth failed",
+                            headers, None)
+        else:
+            self.retried += 1
+        if authreq:
+            scheme = authreq.split()[0]
+            if scheme.lower() == 'digest':
+                return self.retry_http_digest_auth(req, authreq)
+
+    def retry_http_digest_auth(self, req, auth):
+        token, challenge = auth.split(' ', 1)
+        chal = parse_keqv_list(parse_http_list(challenge))
+        auth = self.get_authorization(req, chal)
+        if auth:
+            auth_val = 'Digest %s' % auth
+            if req.headers.get(self.auth_header, None) == auth_val:
+                return None
+            req.add_unredirected_header(self.auth_header, auth_val)
+            resp = self.parent.open(req, timeout=req.timeout)
+            return resp
+
+    def get_cnonce(self, nonce):
+        # The cnonce-value is an opaque
+        # quoted string value provided by the client and used by both client
+        # and server to avoid chosen plaintext attacks, to provide mutual
+        # authentication, and to provide some message integrity protection.
+        # This isn't a fabulous effort, but it's probably Good Enough.
+        dig = hashlib.sha1("%s:%s:%s:%s" % (self.nonce_count, nonce, time.ctime(),
+                                            randombytes(8))).hexdigest()
+        return dig[:16]
+
+    def get_authorization(self, req, chal):
+        try:
+            realm = chal['realm']
+            nonce = chal['nonce']
+            qop = chal.get('qop')
+            algorithm = chal.get('algorithm', 'MD5')
+            # mod_digest doesn't send an opaque, even though it isn't
+            # supposed to be optional
+            opaque = chal.get('opaque', None)
+        except KeyError:
+            return None
+
+        H, KD = self.get_algorithm_impls(algorithm)
+        if H is None:
+            return None
+
+        user, pw = self.passwd.find_user_password(realm, req.get_full_url())
+        if user is None:
+            return None
+
+        # XXX not implemented yet
+        if req.has_data():
+            entdig = self.get_entity_digest(req.get_data(), chal)
+        else:
+            entdig = None
+
+        A1 = "%s:%s:%s" % (user, realm, pw)
+        A2 = "%s:%s" % (req.get_method(),
+                        # XXX selector: what about proxies and full urls
+                        req.get_selector())
+        if qop == 'auth':
+            if nonce == self.last_nonce:
+                self.nonce_count += 1
+            else:
+                self.nonce_count = 1
+                self.last_nonce = nonce
+
+            ncvalue = '%08x' % self.nonce_count
+            cnonce = self.get_cnonce(nonce)
+            noncebit = "%s:%s:%s:%s:%s" % (nonce, ncvalue, cnonce, qop, H(A2))
+            respdig = KD(H(A1), noncebit)
+        elif qop is None:
+            respdig = KD(H(A1), "%s:%s" % (nonce, H(A2)))
+        else:
+            # XXX handle auth-int.
+            raise URLError("qop '%s' is not supported." % qop)
+
+        # XXX should the partial digests be encoded too?
+
+        base = 'username="%s", realm="%s", nonce="%s", uri="%s", ' \
+               'response="%s"' % (user, realm, nonce, req.get_selector(),
+                                  respdig)
+        if opaque:
+            base += ', opaque="%s"' % opaque
+        if entdig:
+            base += ', digest="%s"' % entdig
+        base += ', algorithm="%s"' % algorithm
+        if qop:
+            base += ', qop=auth, nc=%s, cnonce="%s"' % (ncvalue, cnonce)
+        return base
+
+    def get_algorithm_impls(self, algorithm):
+        # algorithm should be case-insensitive according to RFC2617
+        algorithm = algorithm.upper()
+        # lambdas assume digest modules are imported at the top level
+        if algorithm == 'MD5':
+            H = lambda x: hashlib.md5(x).hexdigest()
+        elif algorithm == 'SHA':
+            H = lambda x: hashlib.sha1(x).hexdigest()
+        # XXX MD5-sess
+        KD = lambda s, d: H("%s:%s" % (s, d))
+        return H, KD
+
+    def get_entity_digest(self, data, chal):
+        # XXX not implemented yet
+        return None
+
+
+class HTTPDigestAuthHandler(BaseHandler, AbstractDigestAuthHandler):
+    """An authentication protocol defined by RFC 2069
+
+    Digest authentication improves on basic authentication because it
+    does not transmit passwords in the clear.
+    """
+
+    auth_header = 'Authorization'
+    handler_order = 490  # before Basic auth
+
+    def http_error_401(self, req, fp, code, msg, headers):
+        host = urlparse.urlparse(req.get_full_url())[1]
+        retry = self.http_error_auth_reqed('www-authenticate',
+                                           host, req, headers)
+        self.reset_retry_count()
+        return retry
+
+
+class ProxyDigestAuthHandler(BaseHandler, AbstractDigestAuthHandler):
+
+    auth_header = 'Proxy-Authorization'
+    handler_order = 490  # before Basic auth
+
+    def http_error_407(self, req, fp, code, msg, headers):
+        host = req.get_host()
+        retry = self.http_error_auth_reqed('proxy-authenticate',
+                                           host, req, headers)
+        self.reset_retry_count()
+        return retry
+
+class AbstractHTTPHandler(BaseHandler):
+
+    def __init__(self, debuglevel=0):
+        self._debuglevel = debuglevel
+
+    def set_http_debuglevel(self, level):
+        self._debuglevel = level
+
+    def do_request_(self, request):
+        host = request.get_host()
+        if not host:
+            raise URLError('no host given')
+
+        if request.has_data():  # POST
+            data = request.get_data()
+            if not request.has_header('Content-type'):
+                request.add_unredirected_header(
+                    'Content-type',
+                    'application/x-www-form-urlencoded')
+            if not request.has_header('Content-length'):
+                request.add_unredirected_header(
+                    'Content-length', '%d' % len(data))
+
+        sel_host = host
+        if request.has_proxy():
+            scheme, sel = splittype(request.get_selector())
+            sel_host, sel_path = splithost(sel)
+
+        if not request.has_header('Host'):
+            request.add_unredirected_header('Host', sel_host)
+        for name, value in self.parent.addheaders:
+            name = name.capitalize()
+            if not request.has_header(name):
+                request.add_unredirected_header(name, value)
+
+        return request
+
+    def do_open(self, http_class, req):
+        """Return an addinfourl object for the request, using http_class.
+
+        http_class must implement the HTTPConnection API from httplib.
+        The addinfourl return value is a file-like object.  It also
+        has methods and attributes including:
+            - info(): return a mimetools.Message object for the headers
+            - geturl(): return the original request URL
+            - code: HTTP status code
+        """
+        host = req.get_host()
+        if not host:
+            raise URLError('no host given')
+
+        h = http_class(host, timeout=req.timeout) # will parse host:port
+        h.set_debuglevel(self._debuglevel)
+
+        headers = dict(req.unredirected_hdrs)
+        headers.update(dict((k, v) for k, v in req.headers.items()
+                            if k not in headers))
+
+        # We want to make an HTTP/1.1 request, but the addinfourl
+        # class isn't prepared to deal with a persistent connection.
+        # It will try to read all remaining data from the socket,
+        # which will block while the server waits for the next request.
+        # So make sure the connection gets closed after the (only)
+        # request.
+        headers["Connection"] = "close"
+        headers = dict(
+            (name.title(), val) for name, val in headers.items())
+
+        if req._tunnel_host:
+            tunnel_headers = {}
+            proxy_auth_hdr = "Proxy-Authorization"
+            if proxy_auth_hdr in headers:
+                tunnel_headers[proxy_auth_hdr] = headers[proxy_auth_hdr]
+                # Proxy-Authorization should not be sent to origin
+                # server.
+                del headers[proxy_auth_hdr]
+            h.set_tunnel(req._tunnel_host, headers=tunnel_headers)
+
+        try:
+            h.request(req.get_method(), req.get_selector(), req.data, headers)
+            try:
+                r = h.getresponse(buffering=True)
+            except TypeError: #buffering kw not supported
+                r = h.getresponse()
+        except socket.error, err: # XXX what error?
+            h.close()
+            raise URLError(err)
+
+        # Pick apart the HTTPResponse object to get the addinfourl
+        # object initialized properly.
+
+        # Wrap the HTTPResponse object in socket's file object adapter
+        # for Windows.  That adapter calls recv(), so delegate recv()
+        # to read().  This weird wrapping allows the returned object to
+        # have readline() and readlines() methods.
+
+        # XXX It might be better to extract the read buffering code
+        # out of socket._fileobject() and into a base class.
+
+        r.recv = r.read
+        fp = socket._fileobject(r, close=True)
+
+        resp = addinfourl(fp, r.msg, req.get_full_url())
+        resp.code = r.status
+        resp.msg = r.reason
+        return resp
+
+
+class HTTPHandler(AbstractHTTPHandler):
+
+    def http_open(self, req):
+        return self.do_open(httplib.HTTPConnection, req)
+
+    http_request = AbstractHTTPHandler.do_request_
+
+if hasattr(httplib, 'HTTPS'):
+    class HTTPSHandler(AbstractHTTPHandler):
+
+        def https_open(self, req):
+            return self.do_open(httplib.HTTPSConnection, req)
+
+        https_request = AbstractHTTPHandler.do_request_
+
+class HTTPCookieProcessor(BaseHandler):
+    def __init__(self, cookiejar=None):
+        import cookielib
+        if cookiejar is None:
+            cookiejar = cookielib.CookieJar()
+        self.cookiejar = cookiejar
+
+    def http_request(self, request):
+        self.cookiejar.add_cookie_header(request)
+        return request
+
+    def http_response(self, request, response):
+        self.cookiejar.extract_cookies(response, request)
+        return response
+
+    https_request = http_request
+    https_response = http_response
+
+class UnknownHandler(BaseHandler):
+    def unknown_open(self, req):
+        type = req.get_type()
+        raise URLError('unknown url type: %s' % type)
+
+def parse_keqv_list(l):
+    """Parse list of key=value strings where keys are not duplicated."""
+    parsed = {}
+    for elt in l:
+        k, v = elt.split('=', 1)
+        if v[0] == '"' and v[-1] == '"':
+            v = v[1:-1]
+        parsed[k] = v
+    return parsed
+
+def parse_http_list(s):
+    """Parse lists as described by RFC 2068 Section 2.
+
+    In particular, parse comma-separated lists where the elements of
+    the list may include quoted-strings.  A quoted-string could
+    contain a comma.  A non-quoted string could have quotes in the
+    middle.  Neither commas nor quotes count if they are escaped.
+    Only double-quotes count, not single-quotes.
+    """
+    res = []
+    part = ''
+
+    escape = quote = False
+    for cur in s:
+        if escape:
+            part += cur
+            escape = False
+            continue
+        if quote:
+            if cur == '\\':
+                escape = True
+                continue
+            elif cur == '"':
+                quote = False
+            part += cur
+            continue
+
+        if cur == ',':
+            res.append(part)
+            part = ''
+            continue
+
+        if cur == '"':
+            quote = True
+
+        part += cur
+
+    # append last part
+    if part:
+        res.append(part)
+
+    return [part.strip() for part in res]
+
+def _safe_gethostbyname(host):
+    try:
+        return socket.gethostbyname(host)
+    except socket.gaierror:
+        return None
+
+class FileHandler(BaseHandler):
+    # Use local file or FTP depending on form of URL
+    def file_open(self, req):
+        url = req.get_selector()
+        if url[:2] == '//' and url[2:3] != '/' and (req.host and
+                req.host != 'localhost'):
+            req.type = 'ftp'
+            return self.parent.open(req)
+        else:
+            return self.open_local_file(req)
+
+    # names for the localhost
+    names = None
+    def get_names(self):
+        if FileHandler.names is None:
+            try:
+                FileHandler.names = tuple(
+                    socket.gethostbyname_ex('localhost')[2] +
+                    socket.gethostbyname_ex(socket.gethostname())[2])
+            except socket.gaierror:
+                FileHandler.names = (socket.gethostbyname('localhost'),)
+        return FileHandler.names
+
+    # not entirely sure what the rules are here
+    def open_local_file(self, req):
+        import email.utils
+        import mimetypes
+        host = req.get_host()
+        filename = req.get_selector()
+        localfile = url2pathname(filename)
+        try:
+            stats = os.stat(localfile)
+            size = stats.st_size
+            modified = email.utils.formatdate(stats.st_mtime, usegmt=True)
+            mtype = mimetypes.guess_type(filename)[0]
+            headers = mimetools.Message(StringIO(
+                'Content-type: %s\nContent-length: %d\nLast-modified: %s\n' %
+                (mtype or 'text/plain', size, modified)))
+            if host:
+                host, port = splitport(host)
+            if not host or \
+                (not port and _safe_gethostbyname(host) in self.get_names()):
+                if host:
+                    origurl = 'file://' + host + filename
+                else:
+                    origurl = 'file://' + filename
+                return addinfourl(open(localfile, 'rb'), headers, origurl)
+        except OSError, msg:
+            # urllib2 users shouldn't expect OSErrors coming from urlopen()
+            raise URLError(msg)
+        raise URLError('file not on local host')
+
+class FTPHandler(BaseHandler):
+    def ftp_open(self, req):
+        import ftplib
+        import mimetypes
+        host = req.get_host()
+        if not host:
+            raise URLError('ftp error: no host given')
+        host, port = splitport(host)
+        if port is None:
+            port = ftplib.FTP_PORT
+        else:
+            port = int(port)
+
+        # username/password handling
+        user, host = splituser(host)
+        if user:
+            user, passwd = splitpasswd(user)
+        else:
+            passwd = None
+        host = unquote(host)
+        user = user or ''
+        passwd = passwd or ''
+
+        try:
+            host = socket.gethostbyname(host)
+        except socket.error, msg:
+            raise URLError(msg)
+        path, attrs = splitattr(req.get_selector())
+        dirs = path.split('/')
+        dirs = map(unquote, dirs)
+        dirs, file = dirs[:-1], dirs[-1]
+        if dirs and not dirs[0]:
+            dirs = dirs[1:]
+        try:
+            fw = self.connect_ftp(user, passwd, host, port, dirs, req.timeout)
+            type = file and 'I' or 'D'
+            for attr in attrs:
+                attr, value = splitvalue(attr)
+                if attr.lower() == 'type' and \
+                   value in ('a', 'A', 'i', 'I', 'd', 'D'):
+                    type = value.upper()
+            fp, retrlen = fw.retrfile(file, type)
+            headers = ""
+            mtype = mimetypes.guess_type(req.get_full_url())[0]
+            if mtype:
+                headers += "Content-type: %s\n" % mtype
+            if retrlen is not None and retrlen >= 0:
+                headers += "Content-length: %d\n" % retrlen
+            sf = StringIO(headers)
+            headers = mimetools.Message(sf)
+            return addinfourl(fp, headers, req.get_full_url())
+        except ftplib.all_errors, msg:
+            raise URLError, ('ftp error: %s' % msg), sys.exc_info()[2]
+
+    def connect_ftp(self, user, passwd, host, port, dirs, timeout):
+        fw = ftpwrapper(user, passwd, host, port, dirs, timeout)
+##        fw.ftp.set_debuglevel(1)
+        return fw
+
+class CacheFTPHandler(FTPHandler):
+    # XXX would be nice to have pluggable cache strategies
+    # XXX this stuff is definitely not thread safe
+    def __init__(self):
+        self.cache = {}
+        self.timeout = {}
+        self.soonest = 0
+        self.delay = 60
+        self.max_conns = 16
+
+    def setTimeout(self, t):
+        self.delay = t
+
+    def setMaxConns(self, m):
+        self.max_conns = m
+
+    def connect_ftp(self, user, passwd, host, port, dirs, timeout):
+        key = user, host, port, '/'.join(dirs), timeout
+        if key in self.cache:
+            self.timeout[key] = time.time() + self.delay
+        else:
+            self.cache[key] = ftpwrapper(user, passwd, host, port, dirs, timeout)
+            self.timeout[key] = time.time() + self.delay
+        self.check_cache()
+        return self.cache[key]
+
+    def check_cache(self):
+        # first check for old ones
+        t = time.time()
+        if self.soonest <= t:
+            for k, v in self.timeout.items():
+                if v < t:
+                    self.cache[k].close()
+                    del self.cache[k]
+                    del self.timeout[k]
+        self.soonest = min(self.timeout.values())
+
+        # then check the size
+        if len(self.cache) == self.max_conns:
+            for k, v in self.timeout.items():
+                if v == self.soonest:
+                    del self.cache[k]
+                    del self.timeout[k]
+                    break
+            self.soonest = min(self.timeout.values())
diff --git a/lib_pypy/pyrepl/readline.py b/lib_pypy/pyrepl/readline.py
--- a/lib_pypy/pyrepl/readline.py
+++ b/lib_pypy/pyrepl/readline.py
@@ -395,9 +395,21 @@
     _wrapper.f_in = f_in
     _wrapper.f_out = f_out
 
-    if hasattr(sys, '__raw_input__'):    # PyPy
-        _old_raw_input = sys.__raw_input__
+    if '__pypy__' in sys.builtin_module_names:    # PyPy
+
+        def _old_raw_input(prompt=''):
+            # sys.__raw_input__() is only called when stdin and stdout are
+            # as expected and are ttys.  If it is the case, then get_reader()
+            # should not really fail in _wrapper.raw_input().  If it still
+            # does, then we will just cancel the redirection and call again
+            # the built-in raw_input().
+            try:
+                del sys.__raw_input__
+            except AttributeError:
+                pass
+            return raw_input(prompt)
         sys.__raw_input__ = _wrapper.raw_input
+
     else:
         # this is not really what readline.c does.  Better than nothing I guess
         import __builtin__
diff --git a/lib_pypy/resource.py b/lib_pypy/resource.py
--- a/lib_pypy/resource.py
+++ b/lib_pypy/resource.py
@@ -7,7 +7,7 @@
 
 from ctypes_support import standard_c_lib as libc
 from ctypes_support import get_errno
-from ctypes import Structure, c_int, c_long, byref, sizeof, POINTER
+from ctypes import Structure, c_int, c_long, byref, POINTER
 from errno import EINVAL, EPERM
 import _structseq
 
@@ -165,7 +165,6 @@
 
 @builtinify
 def getpagesize():
-    pagesize = 0
     if _getpagesize:
         return _getpagesize()
     else:
diff --git a/pypy/annotation/classdef.py b/pypy/annotation/classdef.py
--- a/pypy/annotation/classdef.py
+++ b/pypy/annotation/classdef.py
@@ -276,8 +276,8 @@
         # create the Attribute and do the generalization asked for
         newattr = Attribute(attr, self.bookkeeper)
         if s_value:
-            if newattr.name == 'intval' and getattr(s_value, 'unsigned', False):
-                import pdb; pdb.set_trace()
+            #if newattr.name == 'intval' and getattr(s_value, 'unsigned', False):
+            #    import pdb; pdb.set_trace()
             newattr.s_value = s_value
 
         # keep all subattributes' values
diff --git a/pypy/annotation/policy.py b/pypy/annotation/policy.py
--- a/pypy/annotation/policy.py
+++ b/pypy/annotation/policy.py
@@ -1,6 +1,6 @@
 # base annotation policy for specialization
 from pypy.annotation.specialize import default_specialize as default
-from pypy.annotation.specialize import specialize_argvalue, specialize_argtype, specialize_arglistitemtype
+from pypy.annotation.specialize import specialize_argvalue, specialize_argtype, specialize_arglistitemtype, specialize_arg_or_var
 from pypy.annotation.specialize import memo, specialize_call_location
 # for some reason, model must be imported first,
 # or we create a cycle.
@@ -73,6 +73,7 @@
     default_specialize = staticmethod(default)
     specialize__memo = staticmethod(memo)
     specialize__arg = staticmethod(specialize_argvalue) # specialize:arg(N)
+    specialize__arg_or_var = staticmethod(specialize_arg_or_var)
     specialize__argtype = staticmethod(specialize_argtype) # specialize:argtype(N)
     specialize__arglistitemtype = staticmethod(specialize_arglistitemtype)
     specialize__call_location = staticmethod(specialize_call_location)
diff --git a/pypy/annotation/specialize.py b/pypy/annotation/specialize.py
--- a/pypy/annotation/specialize.py
+++ b/pypy/annotation/specialize.py
@@ -353,6 +353,16 @@
     key = tuple(key)
     return maybe_star_args(funcdesc, key, args_s)
 
+def specialize_arg_or_var(funcdesc, args_s, *argindices):
+    for argno in argindices:
+        if not args_s[argno].is_constant():
+            break
+    else:
+        # all constant
+        return specialize_argvalue(funcdesc, args_s, *argindices)
+    # some not constant
+    return maybe_star_args(funcdesc, None, args_s)
+
 def specialize_argtype(funcdesc, args_s, *argindices):
     key = tuple([args_s[i].knowntype for i in argindices])
     for cls in key:
diff --git a/pypy/annotation/test/test_annrpython.py b/pypy/annotation/test/test_annrpython.py
--- a/pypy/annotation/test/test_annrpython.py
+++ b/pypy/annotation/test/test_annrpython.py
@@ -1194,6 +1194,20 @@
         assert len(executedesc._cache[(0, 'star', 2)].startblock.inputargs) == 4
         assert len(executedesc._cache[(1, 'star', 3)].startblock.inputargs) == 5
 
+    def test_specialize_arg_or_var(self):
+        def f(a):
+            return 1
+        f._annspecialcase_ = 'specialize:arg_or_var(0)'
+
+        def fn(a):
+            return f(3) + f(a)
+
+        a = self.RPythonAnnotator()
+        a.build_types(fn, [int])
+        executedesc = a.bookkeeper.getdesc(f)
+        assert sorted(executedesc._cache.keys()) == [None, (3,)]
+        # we got two different special
+
     def test_specialize_call_location(self):
         def g(a):
             return a
@@ -3190,6 +3204,8 @@
         s = a.build_types(f, [])
         assert isinstance(s, annmodel.SomeList)
         assert not s.listdef.listitem.resized
+        assert not s.listdef.listitem.immutable
+        assert s.listdef.listitem.mutated
 
     def test_delslice(self):
         def f():
diff --git a/pypy/annotation/unaryop.py b/pypy/annotation/unaryop.py
--- a/pypy/annotation/unaryop.py
+++ b/pypy/annotation/unaryop.py
@@ -352,6 +352,7 @@
         check_negative_slice(s_start, s_stop)
         if not isinstance(s_iterable, SomeList):
             raise Exception("list[start:stop] = x: x must be a list")
+        lst.listdef.mutate()
         lst.listdef.agree(s_iterable.listdef)
         # note that setslice is not allowed to resize a list in RPython
 
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -72,6 +72,7 @@
     del working_modules['fcntl']  # LOCK_NB not defined
     del working_modules["_minimal_curses"]
     del working_modules["termios"]
+    del working_modules["_multiprocessing"]   # depends on rctime
 
 
 
@@ -127,7 +128,7 @@
 
 pypy_optiondescription = OptionDescription("objspace", "Object Space Options", [
     ChoiceOption("name", "Object Space name",
-                 ["std", "flow", "thunk", "dump", "taint"],
+                 ["std", "flow", "thunk", "dump"],
                  "std",
                  cmdline='--objspace -o'),
 
diff --git a/pypy/doc/__pypy__-module.rst b/pypy/doc/__pypy__-module.rst
--- a/pypy/doc/__pypy__-module.rst
+++ b/pypy/doc/__pypy__-module.rst
@@ -37,29 +37,6 @@
 .. _`thunk object space docs`: objspace-proxies.html#thunk
 .. _`interface section of the thunk object space docs`: objspace-proxies.html#thunk-interface
 
-.. broken:
-
-    Taint Object Space Functionality
-    ================================
-
-    When the taint object space is used (choose with :config:`objspace.name`),
-    the following names are put into ``__pypy__``:
-
-     - ``taint``
-     - ``is_tainted``
-     - ``untaint``
-     - ``taint_atomic``
-     - ``_taint_debug``
-     - ``_taint_look``
-     - ``TaintError``
-
-    Those are all described in the `interface section of the taint object space
-    docs`_.
-
-    For more detailed explanations and examples see the `taint object space docs`_.
-
-    .. _`taint object space docs`: objspace-proxies.html#taint
-    .. _`interface section of the taint object space docs`: objspace-proxies.html#taint-interface
 
 Transparent Proxy Functionality
 ===============================
diff --git a/pypy/doc/config/objspace.name.txt b/pypy/doc/config/objspace.name.txt
--- a/pypy/doc/config/objspace.name.txt
+++ b/pypy/doc/config/objspace.name.txt
@@ -4,7 +4,6 @@
 for normal usage):
 
   * thunk_: The thunk object space adds lazy evaluation to PyPy.
-  * taint_: The taint object space adds soft security features.
   * dump_:  Using this object spaces results in the dumpimp of all operations
     to a log.
 
@@ -12,5 +11,4 @@
 .. _`Object Space Proxies`: ../objspace-proxies.html
 .. _`Standard Object Space`: ../objspace.html#standard-object-space
 .. _thunk: ../objspace-proxies.html#thunk
-.. _taint: ../objspace-proxies.html#taint
 .. _dump: ../objspace-proxies.html#dump
diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst
--- a/pypy/doc/index.rst
+++ b/pypy/doc/index.rst
@@ -309,7 +309,6 @@
 .. _`object space`: objspace.html
 .. _FlowObjSpace: objspace.html#the-flow-object-space 
 .. _`trace object space`: objspace.html#the-trace-object-space 
-.. _`taint object space`: objspace-proxies.html#taint
 .. _`thunk object space`: objspace-proxies.html#thunk
 .. _`transparent proxies`: objspace-proxies.html#tproxy
 .. _`Differences between PyPy and CPython`: cpython_differences.html
diff --git a/pypy/doc/objspace-proxies.rst b/pypy/doc/objspace-proxies.rst
--- a/pypy/doc/objspace-proxies.rst
+++ b/pypy/doc/objspace-proxies.rst
@@ -129,297 +129,6 @@
    function behaves lazily: all calls to it return a thunk object.
 
 
-.. broken right now:
-
-    .. _taint:
-
-    The Taint Object Space
-    ======================
-
-    Motivation
-    ----------
-
-    The Taint Object Space provides a form of security: "tainted objects",
-    inspired by various sources, see [D12.1]_ for a more detailed discussion. 
-
-    The basic idea of this kind of security is not to protect against
-    malicious code but to help with handling and boxing sensitive data. 
-    It covers two kinds of sensitive data: secret data which should not leak, 
-    and untrusted data coming from an external source and that must be 
-    validated before it is used.
-
-    The idea is that, considering a large application that handles these
-    kinds of sensitive data, there are typically only a small number of
-    places that need to explicitly manipulate that sensitive data; all the
-    other places merely pass it around, or do entirely unrelated things.
-
-    Nevertheless, if a large application needs to be reviewed for security,
-    it must be entirely carefully checked, because it is possible that a
-    bug at some apparently unrelated place could lead to a leak of sensitive
-    information in a way that an external attacker could exploit.  For
-    example, if any part of the application provides web services, an
-    attacker might be able to issue unexpected requests with a regular web
-    browser and deduce secret information from the details of the answers he
-    gets.  Another example is the common CGI attack where an attacker sends
-    malformed inputs and causes the CGI script to do unintended things.
-
-    An approach like that of the Taint Object Space allows the small parts
-    of the program that manipulate sensitive data to be explicitly marked.
-    The effect of this is that although these small parts still need a
-    careful security review, the rest of the application no longer does,
-    because even a bug would be unable to leak the information.
-
-    We have implemented a simple two-level model: objects are either
-    regular (untainted), or sensitive (tainted).  Objects are marked as
-    sensitive if they are secret or untrusted, and only declassified at
-    carefully-checked positions (e.g. where the secret data is needed, or
-    after the untrusted data has been fully validated).
-
-    It would be simple to extend the code for more fine-grained scales of
-    secrecy.  For example it is typical in the literature to consider
-    user-specified lattices of secrecy levels, corresponding to multiple
-    "owners" that cannot access data belonging to another "owner" unless
-    explicitly authorized to do so.
-
-    Tainting and untainting
-    -----------------------
-
-    Start a py.py with the Taint Object Space and try the following example::
-
-        $ py.py -o taint
-        >>>> from __pypy__ import taint
-        >>>> x = taint(6)
-
-        # x is hidden from now on.  We can pass it around and
-        # even operate on it, but not inspect it.  Taintness
-        # is propagated to operation results.
-
-        >>>> x
-        TaintError
-
-        >>>> if x > 5: y = 2   # see below
-        TaintError
-
-        >>>> y = x + 5         # ok
-        >>>> lst = [x, y]
-        >>>> z = lst.pop()
-        >>>> t = type(z)       # type() works too, tainted answer
-        >>>> t
-        TaintError
-        >>>> u = t is int      # even 'is' works
-        >>>> u
-        TaintError
-
-    Notice that using a tainted boolean like ``x > 5`` in an ``if``
-    statement is forbidden.  This is because knowing which path is followed
-    would give away a hint about ``x``; in the example above, if the
-    statement ``if x > 5: y = 2`` was allowed to run, we would know
-    something about the value of ``x`` by looking at the (untainted) value
-    in the variable ``y``.
-
-    Of course, there is a way to inspect tainted objects.  The basic way is
-    to explicitly "declassify" it with the ``untaint()`` function.  In an
-    application, the places that use ``untaint()`` are the places that need
-    careful security review.  To avoid unexpected objects showing up, the
-    ``untaint()`` function must be called with the exact type of the object
-    to declassify.  It will raise ``TaintError`` if the type doesn't match::
-
-        >>>> from __pypy__ import taint
-        >>>> untaint(int, x)
-        6
-        >>>> untaint(int, z)
-        11
-        >>>> untaint(bool, x > 5)
-        True
-        >>>> untaint(int, x > 5)
-        TaintError
-
-
-    Taint Bombs
-    -----------
-
-    In this area, a common problem is what to do about failing operations.
-    If an operation raises an exception when manipulating a tainted object,
-    then the very presence of the exception can leak information about the
-    tainted object itself.  Consider::
-
-        >>>> 5 / (x-6)
-
-    By checking if this raises ``ZeroDivisionError`` or not, we would know
-    if ``x`` was equal to 6 or not.  The solution to this problem in the
-    Taint Object Space is to introduce *Taint Bombs*.  They are a kind of
-    tainted object that doesn't contain a real object, but a pending
-    exception.  Taint Bombs are indistinguishable from normal tainted
-    objects to unprivileged code. See::
-
-        >>>> x = taint(6)
-        >>>> i = 5 / (x-6)     # no exception here
-        >>>> j = i + 1         # nor here
-        >>>> k = j + 5         # nor here
-        >>>> untaint(int, k)
-        TaintError
-
-    In the above example, all of ``i``, ``j`` and ``k`` contain a Taint
-    Bomb.  Trying to untaint it raises an exception - a generic
-    ``TaintError``.  What we win is that the exception gives little away,
-    and most importantly it occurs at the point where ``untaint()`` is
-    called, not where the operation failed.  This means that all calls to
-    ``untaint()`` - but not the rest of the code - must be carefully
-    reviewed for what occurs if they receive a Taint Bomb; they might catch
-    the ``TaintError`` and give the user a generic message that something
-    went wrong, if we are reasonably careful that the message or even its
-    presence doesn't give information away.  This might be a
-    problem by itself, but there is no satisfying general solution here:
-    it must be considered on a case-by-case basis.  Again, what the
-    Taint Object Space approach achieves is not solving these problems, but
-    localizing them to well-defined small parts of the application - namely,
-    around calls to ``untaint()``.
-
-    The ``TaintError`` exception deliberately does not include any
-    useful error messages, because they might give information away.
-    Of course, this makes debugging quite a bit harder; a difficult
-    problem to solve properly.  So far we have implemented a way to peek in a Taint
-    Box or Bomb, ``__pypy__._taint_look(x)``, and a "debug mode" that
-    prints the exception as soon as a Bomb is created - both write
-    information to the low-level stderr of the application, where we hope
-    that it is unlikely to be seen by anyone but the application
-    developer.
-
-
-    Taint Atomic functions
-    ----------------------
-
-    Occasionally, a more complicated computation must be performed on a
-    tainted object.  This requires first untainting the object, performing the
-    computations, and then carefully tainting the result again (including
-    hiding all exceptions into Bombs).
-
-    There is a built-in decorator that does this for you::
-
-        >>>> @__pypy__.taint_atomic
-        >>>> def myop(x, y):
-        ....     while x > 0:
-        ....         x -= y
-        ....     return x
-        ....
-        >>>> myop(42, 10)
-        -8
-        >>>> z = myop(taint(42), 10)
-        >>>> z
-        TaintError
-        >>>> untaint(int, z)
-        -8
-
-    The decorator makes a whole function behave like a built-in operation.
-    If no tainted argument is passed in, the function behaves normally.  But
-    if any of the arguments is tainted, it is automatically untainted - so
-    the function body always sees untainted arguments - and the eventual
-    result is tainted again (possibly in a Taint Bomb).
-
-    It is important for the function marked as ``taint_atomic`` to have no
-    visible side effects, as these could cause information leakage.
-    This is currently not enforced, which means that all ``taint_atomic``
-    functions have to be carefully reviewed for security (but not the
-    callers of ``taint_atomic`` functions).
-
-    A possible future extension would be to forbid side-effects on
-    non-tainted objects from all ``taint_atomic`` functions.
-
-    An example of usage: given a tainted object ``passwords_db`` that
-    references a database of passwords, we can write a function
-    that checks if a password is valid as follows::
-
-        @taint_atomic
-        def validate(passwords_db, username, password):
-            assert type(passwords_db) is PasswordDatabase
-            assert type(username) is str
-            assert type(password) is str
-            ...load username entry from passwords_db...
-            return expected_password == password
-
-    It returns a tainted boolean answer, or a Taint Bomb if something
-    went wrong.  A caller can do::
-
-        ok = validate(passwords_db, 'john', '1234')
-        ok = untaint(bool, ok)
-
-    This can give three outcomes: ``True``, ``False``, or a ``TaintError``
-    exception (with no information on it) if anything went wrong.  If even
-    this is considered giving too much information away, the ``False`` case
-    can be made indistinguishable from the ``TaintError`` case (simply by
-    raising an exception in ``validate()`` if the password is wrong).
-
-    In the above example, the security results achieved are the following:
-    as long as ``validate()`` does not leak information, no other part of
-    the code can obtain more information about a passwords database than a
-    Yes/No answer to a precise query.
-
-    A possible extension of the ``taint_atomic`` decorator would be to check
-    the argument types, as ``untaint()`` does, for the same reason: to
-    prevent bugs where a function like ``validate()`` above is accidentally
-    called with the wrong kind of tainted object, which would make it
-    misbehave.  For now, all ``taint_atomic`` functions should be
-    conservative and carefully check all assumptions on their input
-    arguments.
-
-
-    .. _`taint-interface`:
-
-    Interface
-    ---------
-
-    .. _`like a built-in operation`:
-
-    The basic rule of the Tainted Object Space is that it introduces two new
-    kinds of objects, Tainted Boxes and Tainted Bombs (which are not types
-    in the Python sense).  Each box internally contains a regular object;
-    each bomb internally contains an exception object.  An operation
-    involving Tainted Boxes is performed on the objects contained in the
-    boxes, and gives a Tainted Box or a Tainted Bomb as a result (such an
-    operation does not let an exception be raised).  An operation called
-    with a Tainted Bomb argument immediately returns the same Tainted Bomb.
-
-    In a PyPy running with (or translated with) the Taint Object Space,
-    the ``__pypy__`` module exposes the following interface:
-
-    * ``taint(obj)``
-
-        Return a new Tainted Box wrapping ``obj``.  Return ``obj`` itself
-        if it is already tainted (a Box or a Bomb).
-
-    * ``is_tainted(obj)``
-
-        Check if ``obj`` is tainted (a Box or a Bomb).
-
-    * ``untaint(type, obj)``
-
-        Untaints ``obj`` if it is tainted.  Raise ``TaintError`` if the type
-        of the untainted object is not exactly ``type``, or if ``obj`` is a
-        Bomb.
-
-    * ``taint_atomic(func)``
-
-        Return a wrapper function around the callable ``func``.  The wrapper
-        behaves `like a built-in operation`_ with respect to untainting the
-        arguments, tainting the result, and returning a Bomb.
-
-    * ``TaintError``
-
-        Exception.  On purpose, it provides no attribute or error message.
-
-    * ``_taint_debug(level)``
-
-        Set the debugging level to ``level`` (0=off).  At level 1 or above,
-        all Taint Bombs print a diagnostic message to stderr when they are
-        created.
-
-    * ``_taint_look(obj)``
-
-        For debugging purposes: prints (to stderr) the type and address of
-        the object in a Tainted Box, or prints the exception if ``obj`` is
-        a Taint Bomb.
-
-
 .. _dump:
 
 The Dump Object Space
diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -53,6 +53,18 @@
 this is an ideal task to get started, because it does not require any deep
 knowledge of the internals.
 
+Optimized Unicode Representation
+--------------------------------
+
+CPython 3.3 will use an `optimized unicode representation`_ which switches between
+different ways to represent a unicode string, depending on whether the string
+fits into ASCII, has only two-byte characters or needs four-byte characters.
+
+The actual details would be rather differen in PyPy, but we would like to have
+the same optimization implemented.
+
+.. _`optimized unicode representation`: http://www.python.org/dev/peps/pep-0393/
+
 Translation Toolchain
 ---------------------
 
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -3,12 +3,12 @@
 from pypy.interpreter.executioncontext import ExecutionContext, ActionFlag
 from pypy.interpreter.executioncontext import UserDelAction, FrameTraceAction
 from pypy.interpreter.error import OperationError, operationerrfmt
-from pypy.interpreter.error import new_exception_class
+from pypy.interpreter.error import new_exception_class, typed_unwrap_error_msg
 from pypy.interpreter.argument import Arguments
 from pypy.interpreter.miscutils import ThreadLocals
 from pypy.tool.cache import Cache
 from pypy.tool.uid import HUGEVAL_BYTES
-from pypy.rlib.objectmodel import we_are_translated, newlist
+from pypy.rlib.objectmodel import we_are_translated, newlist, compute_unique_id
 from pypy.rlib.debug import make_sure_not_resized
 from pypy.rlib.timer import DummyTimer, Timer
 from pypy.rlib.rarithmetic import r_uint
@@ -186,6 +186,28 @@
     def _set_mapdict_storage_and_map(self, storage, map):
         raise NotImplementedError
 
+    # -------------------------------------------------------------------
+
+    def str_w(self, space):
+        w_msg = typed_unwrap_error_msg(space, "string", self)
+        raise OperationError(space.w_TypeError, w_msg)
+
+    def unicode_w(self, space):
+        raise OperationError(space.w_TypeError,
+                             typed_unwrap_error_msg(space, "unicode", self))
+
+    def int_w(self, space):
+        raise OperationError(space.w_TypeError,
+                             typed_unwrap_error_msg(space, "integer", self))
+    
+    def uint_w(self, space):
+        raise OperationError(space.w_TypeError,
+                             typed_unwrap_error_msg(space, "integer", self))
+    
+    def bigint_w(self, space):
+        raise OperationError(space.w_TypeError,
+                             typed_unwrap_error_msg(space, "integer", self))
+
 
 class Wrappable(W_Root):
     """A subclass of Wrappable is an internal, interpreter-level class
@@ -901,7 +923,7 @@
         ec.c_call_trace(frame, w_func, args)
         try:
             w_res = self.call_args(w_func, args)
-        except OperationError, e:
+        except OperationError:
             ec.c_exception_trace(frame, w_func)
             raise
         ec.c_return_trace(frame, w_func, args)
@@ -947,6 +969,9 @@
     def isinstance_w(self, w_obj, w_type):
         return self.is_true(self.isinstance(w_obj, w_type))
 
+    def id(self, w_obj):
+        return self.wrap(compute_unique_id(w_obj))
+
     # The code below only works
     # for the simple case (new-style instance).
     # These methods are patched with the full logic by the __builtin__
@@ -999,8 +1024,6 @@
 
     def eval(self, expression, w_globals, w_locals, hidden_applevel=False):
         "NOT_RPYTHON: For internal debugging."
-        import types
-        from pypy.interpreter.pycode import PyCode
         if isinstance(expression, str):
             compiler = self.createcompiler()
             expression = compiler.compile(expression, '?', 'eval', 0,
@@ -1012,7 +1035,6 @@
     def exec_(self, statement, w_globals, w_locals, hidden_applevel=False,
               filename=None):
         "NOT_RPYTHON: For internal debugging."
-        import types
         if filename is None:
             filename = '?'
         from pypy.interpreter.pycode import PyCode
@@ -1210,6 +1232,18 @@
             return None
         return self.str_w(w_obj)
 
+    def str_w(self, w_obj):
+        return w_obj.str_w(self)
+
+    def int_w(self, w_obj):
+        return w_obj.int_w(self)
+
+    def uint_w(self, w_obj):
+        return w_obj.uint_w(self)
+
+    def bigint_w(self, w_obj):
+        return w_obj.bigint_w(self)
+
     def realstr_w(self, w_obj):
         # Like str_w, but only works if w_obj is really of type 'str'.
         if not self.is_true(self.isinstance(w_obj, self.w_str)):
@@ -1217,6 +1251,9 @@
                                  self.wrap('argument must be a string'))
         return self.str_w(w_obj)
 
+    def unicode_w(self, w_obj):
+        return w_obj.unicode_w(self)
+
     def realunicode_w(self, w_obj):
         # Like unicode_w, but only works if w_obj is really of type
         # 'unicode'.
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -458,3 +458,7 @@
     if module:
         space.setattr(w_exc, space.wrap("__module__"), space.wrap(module))
     return w_exc
+
+def typed_unwrap_error_msg(space, expected, w_obj):
+    type_name = space.type(w_obj).getname(space)
+    return space.wrap("expected %s, got %s object" % (expected, type_name))
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -175,6 +175,9 @@
             self.w_tracefunc = w_func
             self.space.frame_trace_action.fire()
 
+    def gettrace(self):
+        return self.w_tracefunc
+
     def setprofile(self, w_func):
         """Set the global trace function."""
         if self.space.is_w(w_func, self.space.w_None):
@@ -307,7 +310,11 @@
         self._nonperiodic_actions = []
         self.has_bytecode_counter = False
         self.fired_actions = None
-        self.checkinterval_scaled = 100 * TICK_COUNTER_STEP
+        # the default value is not 100, unlike CPython 2.7, but a much
+        # larger value, because we use a technique that not only allows
+        # but actually *forces* another thread to run whenever the counter
+        # reaches zero.
+        self.checkinterval_scaled = 10000 * TICK_COUNTER_STEP
         self._rebuild_action_dispatcher()
 
     def fire(self, action):
@@ -346,6 +353,7 @@
         elif interval > MAX:
             interval = MAX
         self.checkinterval_scaled = interval * TICK_COUNTER_STEP
+        self.reset_ticker(-1)
 
     def _rebuild_action_dispatcher(self):
         periodic_actions = unrolling_iterable(self._periodic_actions)
@@ -383,8 +391,11 @@
     def decrement_ticker(self, by):
         value = self._ticker
         if self.has_bytecode_counter:    # this 'if' is constant-folded
-            value -= by
-            self._ticker = value
+            if jit.isconstant(by) and by == 0:
+                pass     # normally constant-folded too
+            else:
+                value -= by
+                self._ticker = value
         return value
 
 
diff --git a/pypy/interpreter/pyparser/pytokenizer.py b/pypy/interpreter/pyparser/pytokenizer.py
--- a/pypy/interpreter/pyparser/pytokenizer.py
+++ b/pypy/interpreter/pyparser/pytokenizer.py
@@ -226,7 +226,7 @@
                         parenlev = parenlev - 1
                         if parenlev < 0:
                             raise TokenError("unmatched '%s'" % initial, line,
-                                             lnum-1, 0, token_list)
+                                             lnum, start + 1, token_list)
                     if token in python_opmap:
                         punct = python_opmap[token]
                     else:
diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py
--- a/pypy/interpreter/pyparser/test/test_pyparse.py
+++ b/pypy/interpreter/pyparser/test/test_pyparse.py
@@ -87,6 +87,10 @@
         assert exc.lineno == 1
         assert exc.offset == 5
         assert exc.lastlineno == 5
+        exc = py.test.raises(SyntaxError, parse, "abc)").value
+        assert exc.msg == "unmatched ')'"
+        assert exc.lineno == 1
+        assert exc.offset == 4
 
     def test_is(self):
         self.parse("x is y")
diff --git a/pypy/interpreter/test/test_executioncontext.py b/pypy/interpreter/test/test_executioncontext.py
--- a/pypy/interpreter/test/test_executioncontext.py
+++ b/pypy/interpreter/test/test_executioncontext.py
@@ -42,6 +42,7 @@
         assert i == 9
 
     def test_periodic_action(self):
+        from pypy.interpreter.executioncontext import ActionFlag
 
         class DemoAction(executioncontext.PeriodicAsyncAction):
             counter = 0
@@ -53,17 +54,20 @@
 
         space = self.space
         a2 = DemoAction(space)
-        space.actionflag.register_periodic_action(a2, True)
         try:
-            for i in range(500):
-                space.appexec([], """():
-                    n = 5
-                    return n + 2
-                """)
-        except Finished:
-            pass
-        checkinterval = space.actionflag.getcheckinterval()
-        assert checkinterval / 10 < i < checkinterval * 1.1
+            space.actionflag.setcheckinterval(100)
+            space.actionflag.register_periodic_action(a2, True)
+            try:
+                for i in range(500):
+                    space.appexec([], """():
+                        n = 5
+                        return n + 2
+                    """)
+            except Finished:
+                pass
+        finally:
+            space.actionflag = ActionFlag()   # reset to default
+        assert 10 < i < 110
 
     def test_llprofile(self):
         l = []
diff --git a/pypy/jit/backend/llgraph/llimpl.py b/pypy/jit/backend/llgraph/llimpl.py
--- a/pypy/jit/backend/llgraph/llimpl.py
+++ b/pypy/jit/backend/llgraph/llimpl.py
@@ -165,6 +165,7 @@
     'unicodegetitem'  : (('ref', 'int'), 'int'),
     'unicodesetitem'  : (('ref', 'int', 'int'), 'int'),
     'cast_ptr_to_int' : (('ref',), 'int'),
+    'cast_int_to_ptr' : (('int',), 'ref'),
     'debug_merge_point': (('ref', 'int'), None),
     'force_token'     : ((), 'int'),
     'call_may_force'  : (('int', 'varargs'), 'intorptr'),
@@ -875,9 +876,6 @@
     def op_new_array(self, arraydescr, count):
         return do_new_array(arraydescr.ofs, count)
 
-    def op_cast_ptr_to_int(self, descr, ptr):
-        return cast_to_int(ptr)
-
     def op_force_token(self, descr):
         opaque_frame = _to_opaque(self)
         return llmemory.cast_ptr_to_adr(opaque_frame)
diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py
--- a/pypy/jit/backend/llsupport/gc.py
+++ b/pypy/jit/backend/llsupport/gc.py
@@ -45,6 +45,14 @@
     def freeing_block(self, start, stop):
         pass
 
+    def record_constptrs(self, op, gcrefs_output_list):
+        for i in range(op.numargs()):
+            v = op.getarg(i)
+            if isinstance(v, ConstPtr) and bool(v.value):
+                p = v.value
+                rgc._make_sure_does_not_move(p)
+                gcrefs_output_list.append(p)
+
 # ____________________________________________________________
 
 class GcLLDescr_boehm(GcLLDescription):
@@ -141,6 +149,14 @@
     get_funcptr_for_newstr = None
     get_funcptr_for_newunicode = None
 
+    def rewrite_assembler(self, cpu, operations, gcrefs_output_list):
+        # record all GCREFs too, because Boehm cannot see them and keep them
+        # alive if they end up as constants in the assembler
+        for op in operations:
+            self.record_constptrs(op, gcrefs_output_list)
+        return GcLLDescription.rewrite_assembler(self, cpu, operations,
+                                                 gcrefs_output_list)
+
 
 # ____________________________________________________________
 # All code below is for the hybrid or minimark GC
@@ -757,14 +773,6 @@
             funcptr(llmemory.cast_ptr_to_adr(gcref_struct),
                     llmemory.cast_ptr_to_adr(gcref_newptr))
 
-    def record_constptrs(self, op, gcrefs_output_list):
-        for i in range(op.numargs()):
-            v = op.getarg(i)
-            if isinstance(v, ConstPtr) and bool(v.value):
-                p = v.value
-                rgc._make_sure_does_not_move(p)
-                gcrefs_output_list.append(p)
-
     def rewrite_assembler(self, cpu, operations, gcrefs_output_list):
         # Perform two kinds of rewrites in parallel:
         #
diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -1468,20 +1468,16 @@
         return u''.join(u.chars)
 
 
-    def test_casts(self):
-        py.test.skip("xxx fix or kill")
-        from pypy.rpython.lltypesystem import lltype, llmemory
-        TP = lltype.GcStruct('x')
-        x = lltype.malloc(TP)        
-        x = lltype.cast_opaque_ptr(llmemory.GCREF, x)
+    def test_cast_int_to_ptr(self):
+        res = self.execute_operation(rop.CAST_INT_TO_PTR,
+                                     [BoxInt(-17)],  'ref').value
+        assert lltype.cast_ptr_to_int(res) == -17
+
+    def test_cast_ptr_to_int(self):
+        x = lltype.cast_int_to_ptr(llmemory.GCREF, -19)
         res = self.execute_operation(rop.CAST_PTR_TO_INT,
-                                     [BoxPtr(x)],  'int').value
-        expected = self.cpu.cast_adr_to_int(llmemory.cast_ptr_to_adr(x))
-        assert rffi.get_real_int(res) == rffi.get_real_int(expected)
-        res = self.execute_operation(rop.CAST_PTR_TO_INT,
-                                     [ConstPtr(x)],  'int').value
-        expected = self.cpu.cast_adr_to_int(llmemory.cast_ptr_to_adr(x))
-        assert rffi.get_real_int(res) == rffi.get_real_int(expected)
+                                     [BoxPtr(x)], 'int').value
+        assert res == -19
 
     def test_ooops_non_gc(self):
         x = lltype.malloc(lltype.Struct('x'), flavor='raw')
@@ -2299,13 +2295,6 @@
         #
         cpu.bh_strsetitem(x, 4, ord('/'))
         assert str.chars[4] == '/'
-        #
-##        x = cpu.bh_newstr(5)
-##        y = cpu.bh_cast_ptr_to_int(x)
-##        z = cpu.bh_cast_ptr_to_int(x)
-##        y = rffi.get_real_int(y)
-##        z = rffi.get_real_int(z)
-##        assert type(y) == type(z) == int and y == z
 
     def test_sorting_of_fields(self):
         S = self.S
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1387,7 +1387,8 @@
 
     def genop_same_as(self, op, arglocs, resloc):
         self.mov(arglocs[0], resloc)
-    #genop_cast_ptr_to_int = genop_same_as
+    genop_cast_ptr_to_int = genop_same_as
+    genop_cast_int_to_ptr = genop_same_as
 
     def genop_int_mod(self, op, arglocs, resloc):
         if IS_X86_32:
diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py
--- a/pypy/jit/backend/x86/regalloc.py
+++ b/pypy/jit/backend/x86/regalloc.py
@@ -1152,7 +1152,8 @@
         self.possibly_free_var(op.getarg(0))
         resloc = self.force_allocate_reg(op.result)
         self.Perform(op, [argloc], resloc)
-    #consider_cast_ptr_to_int = consider_same_as
+    consider_cast_ptr_to_int = consider_same_as
+    consider_cast_int_to_ptr = consider_same_as
 
     def consider_strlen(self, op):
         args = op.getarglist()
diff --git a/pypy/jit/backend/x86/tool/viewcode.py b/pypy/jit/backend/x86/tool/viewcode.py
--- a/pypy/jit/backend/x86/tool/viewcode.py
+++ b/pypy/jit/backend/x86/tool/viewcode.py
@@ -9,7 +9,12 @@
 """
 
 import autopath
-import operator, sys, os, re, py, new
+import new
+import operator
+import py
+import re
+import sys
+import subprocess
 from bisect import bisect_left
 
 # don't use pypy.tool.udir here to avoid removing old usessions which
@@ -44,14 +49,16 @@
     f = open(tmpfile, 'wb')
     f.write(data)
     f.close()
-    g = os.popen(objdump % {
+    p = subprocess.Popen(objdump % {
         'file': tmpfile,
         'origin': originaddr,
         'backend': objdump_backend_option[backend_name],
-    }, 'r')
-    result = g.readlines()
-    g.close()
-    lines = result[6:]   # drop some objdump cruft
+    }, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    stdout, stderr = p.communicate()
+    assert not p.returncode, ('Encountered an error running objdump: %s' %
+                              stderr)
+    # drop some objdump cruft
+    lines = stdout.splitlines()[6:]
     return format_code_dump_with_labels(originaddr, lines, label_list)
 
 def format_code_dump_with_labels(originaddr, lines, label_list):
@@ -85,8 +92,12 @@
     #
     print 'loading symbols from %s...' % (filename,)
     symbols = {}
-    g = os.popen(symbollister % filename, "r")
-    for line in g:
+    p = subprocess.Popen(symbollister % filename, shell=True,
+                         stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    stdout, stderr = p.communicate()
+    assert not p.returncode, ('Encountered an error running nm: %s' %
+                              stderr)
+    for line in stdout.splitlines():
         match = re_symbolentry.match(line)
         if match:
             addr = long(match.group(1), 16)
@@ -94,7 +105,6 @@
             if name.startswith('pypy_g_'):
                 name = '\xb7' + name[7:]
             symbols[addr] = name
-    g.close()
     print '%d symbols found' % (len(symbols),)
     return symbols
 
diff --git a/pypy/jit/codewriter/effectinfo.py b/pypy/jit/codewriter/effectinfo.py
--- a/pypy/jit/codewriter/effectinfo.py
+++ b/pypy/jit/codewriter/effectinfo.py
@@ -74,6 +74,7 @@
     OS_LLONG_UGE                = 91
     OS_LLONG_URSHIFT            = 92
     OS_LLONG_FROM_UINT          = 93
+    OS_LLONG_U_TO_FLOAT         = 94
     #
     OS_MATH_SQRT                = 100
 
diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -440,6 +440,7 @@
     rewrite_op_ullong_mod_zer      = _do_builtin_call
     rewrite_op_gc_identityhash = _do_builtin_call
     rewrite_op_gc_id           = _do_builtin_call
+    rewrite_op_uint_mod        = _do_builtin_call
 
     # ----------
     # getfield/setfield/mallocs etc.
@@ -454,6 +455,23 @@
             # the special return value None forces op.result to be considered
             # equal to op.args[0]
             return [op0, op1, None]
+        if (hints.get('promote_string') and
+            op.args[0].concretetype is not lltype.Void):
+            S = lltype.Ptr(rstr.STR)
+            assert op.args[0].concretetype == S
+            self._register_extra_helper(EffectInfo.OS_STREQ_NONNULL,
+                                        "str.eq_nonnull",
+                                        [S, S],
+                                        lltype.Signed,
+                                        EffectInfo.EF_ELIDABLE_CANNOT_RAISE)
+            descr, p = self.callcontrol.callinfocollection.callinfo_for_oopspec(
+                EffectInfo.OS_STREQ_NONNULL)
+            # XXX this is fairly ugly way of creating a constant,
+            #     however, callinfocollection has no better interface
+            c = Constant(p.adr.ptr, lltype.typeOf(p.adr.ptr))
+            op1 = SpaceOperation('str_guard_value', [op.args[0], c, descr],
+                                 op.result)
+            return [SpaceOperation('-live-', [], None), op1, None]
         else:
             log.WARNING('ignoring hint %r at %r' % (hints, self.graph))
 
@@ -782,8 +800,7 @@
 
     def rewrite_op_cast_ptr_to_int(self, op):
         if self._is_gc(op.args[0]):
-            #return op
-            raise NotImplementedError("cast_ptr_to_int")
+            return op
 
     def rewrite_op_force_cast(self, op):
         v_arg = op.args[0]
@@ -883,9 +900,15 @@
                 v = v_arg
                 oplist = []
             if unsigned1:
-                opname = 'cast_uint_to_longlong'
+                if unsigned2:
+                    opname = 'cast_uint_to_ulonglong'
+                else:
+                    opname = 'cast_uint_to_longlong'
             else:
-                opname = 'cast_int_to_longlong'
+                if unsigned2:
+                    opname = 'cast_int_to_ulonglong'
+                else:
+                    opname = 'cast_int_to_longlong'
             op2 = self.rewrite_operation(
                 SpaceOperation(opname, [v], v_result)
             )
@@ -995,6 +1018,21 @@
                 return op2
         ''' % (_op, _oopspec.lower(), _oopspec, _oopspec)).compile()
 
+    for _op, _oopspec in [('cast_int_to_ulonglong',     'FROM_INT'),
+                          ('cast_uint_to_ulonglong',    'FROM_UINT'),
+                          ('cast_float_to_ulonglong',   'FROM_FLOAT'),
+                          ('cast_ulonglong_to_float',   'U_TO_FLOAT'),
+                         ]:
+        exec py.code.Source('''
+            def rewrite_op_%s(self, op):
+                args = op.args
+                op1 = self.prepare_builtin_call(op, "ullong_%s", args)
+                op2 = self._handle_oopspec_call(op1, args,
+                                                EffectInfo.OS_LLONG_%s,
+                                           EffectInfo.EF_ELIDABLE_CANNOT_RAISE)
+                return op2
+        ''' % (_op, _oopspec.lower(), _oopspec)).compile()
+
     def _normalize(self, oplist):
         if isinstance(oplist, SpaceOperation):
             return [oplist]
@@ -1504,6 +1542,10 @@
     def rewrite_op_jit_force_virtual(self, op):
         return self._do_builtin_call(op)
 
+    def rewrite_op_jit_is_virtual(self, op):
+        raise Exception, (
+            "'vref.virtual' should not be used from jit-visible code")
+
     def rewrite_op_jit_force_virtualizable(self, op):
         # this one is for virtualizables
         vinfo = self.get_vinfo(op.args[0])
diff --git a/pypy/jit/codewriter/support.py b/pypy/jit/codewriter/support.py
--- a/pypy/jit/codewriter/support.py
+++ b/pypy/jit/codewriter/support.py
@@ -315,18 +315,30 @@
 def _ll_1_llong_from_int(x):
     return r_longlong(intmask(x))
 
+def _ll_1_ullong_from_int(x):
+    return r_ulonglong(intmask(x))
+
 def _ll_1_llong_from_uint(x):
     return r_longlong(r_uint(x))
 
+def _ll_1_ullong_from_uint(x):
+    return r_ulonglong(r_uint(x))
+
 def _ll_1_llong_to_int(xll):
     return intmask(xll)
 
 def _ll_1_llong_from_float(xf):
     return r_longlong(xf)
 
+def _ll_1_ullong_from_float(xf):
+    return r_ulonglong(xf)
+
 def _ll_1_llong_to_float(xll):
     return float(rffi.cast(lltype.SignedLongLong, xll))
 
+def _ll_1_ullong_u_to_float(xull):
+    return float(rffi.cast(lltype.UnsignedLongLong, xull))
+
 
 def _ll_1_llong_abs(xll):
     if xll < 0:
@@ -351,20 +363,23 @@
     return llop.llong_mod(lltype.SignedLongLong, xll, yll)
 
 def _ll_2_ullong_floordiv(xll, yll):
-    return llop.ullong_floordiv(lltype.SignedLongLong, xll, yll)
+    return llop.ullong_floordiv(lltype.UnsignedLongLong, xll, yll)
 
 def _ll_2_ullong_floordiv_zer(xll, yll):
     if yll == 0:
         raise ZeroDivisionError
-    return llop.ullong_floordiv(lltype.SignedLongLong, xll, yll)
+    return llop.ullong_floordiv(lltype.UnsignedLongLong, xll, yll)
 
 def _ll_2_ullong_mod(xll, yll):
-    return llop.ullong_mod(lltype.SignedLongLong, xll, yll)
+    return llop.ullong_mod(lltype.UnsignedLongLong, xll, yll)
 
 def _ll_2_ullong_mod_zer(xll, yll):
     if yll == 0:
         raise ZeroDivisionError
-    return llop.ullong_mod(lltype.SignedLongLong, xll, yll)
+    return llop.ullong_mod(lltype.UnsignedLongLong, xll, yll)
+
+def _ll_2_uint_mod(xll, yll):
+    return llop.uint_mod(lltype.Unsigned, xll, yll)
 
 
 # libffi support
diff --git a/pypy/jit/codewriter/test/test_flatten.py b/pypy/jit/codewriter/test/test_flatten.py
--- a/pypy/jit/codewriter/test/test_flatten.py
+++ b/pypy/jit/codewriter/test/test_flatten.py
@@ -829,14 +829,15 @@
                     self.encoding_test(f, [rffi.cast(FROM, 42)], expectedstr,
                                        transform=True)
                 elif TO in (rffi.LONG, rffi.ULONG):
+                    if rffi.cast(FROM, -1) < 0:
+                        fnname = "llong_from_int"
+                    else:
+                        fnname = "llong_from_uint"
                     if TO == rffi.LONG:
                         TO = rffi.LONGLONG
                     else:
                         TO = rffi.ULONGLONG
-                    if rffi.cast(FROM, -1) < 0:
-                        fnname = "llong_from_int"
-                    else:
-                        fnname = "llong_from_uint"
+                        fnname = "u" + fnname
                     expected.pop()   # remove int_return
                     expected.append(
                         "residual_call_irf_f $<* fn %s>, <Descr>, I[%s], R[], F[] -> %%f0"
diff --git a/pypy/jit/codewriter/test/test_jtransform.py b/pypy/jit/codewriter/test/test_jtransform.py
--- a/pypy/jit/codewriter/test/test_jtransform.py
+++ b/pypy/jit/codewriter/test/test_jtransform.py
@@ -1,5 +1,19 @@
 import py
 import random
+try:
+    from itertools import product
+except ImportError:
+    # Python 2.5, this is taken from the CPython docs, but simplified.
+    def product(*args):
+        # product('ABCD', 'xy') --> Ax Ay Bx By Cx Cy Dx Dy
+        # product(range(2), repeat=3) --> 000 001 010 011 100 101 110 111
+        pools = map(tuple, args)
+        result = [[]]
+        for pool in pools:
+            result = [x+[y] for x in result for y in pool]
+        for prod in result:
+            yield tuple(prod)
+
 from pypy.objspace.flow.model import FunctionGraph, Block, Link
 from pypy.objspace.flow.model import SpaceOperation, Variable, Constant
 from pypy.jit.codewriter.jtransform import Transformer
@@ -85,6 +99,12 @@
             if i == oopspecindex:
                 return True
         return False
+    def callinfo_for_oopspec(self, oopspecindex):
+        assert oopspecindex == effectinfo.EffectInfo.OS_STREQ_NONNULL
+        class c:
+            class adr:
+                ptr = 1
+        return ('calldescr', c)
 
 class FakeBuiltinCallControl:
     def __init__(self):
@@ -105,6 +125,7 @@
              EI.OS_STR2UNICODE:([PSTR], PUNICODE),
              EI.OS_STR_CONCAT: ([PSTR, PSTR], PSTR),
              EI.OS_STR_SLICE:  ([PSTR, INT, INT], PSTR),
+             EI.OS_STREQ_NONNULL:  ([PSTR, PSTR], INT),
              EI.OS_UNI_CONCAT: ([PUNICODE, PUNICODE], PUNICODE),
              EI.OS_UNI_SLICE:  ([PUNICODE, INT, INT], PUNICODE),
              EI.OS_UNI_EQUAL:  ([PUNICODE, PUNICODE], lltype.Bool),
@@ -254,26 +275,35 @@
             assert op1.result is None
 
 def test_calls():
-    for RESTYPE in [lltype.Signed, rclass.OBJECTPTR,
-                    lltype.Float, lltype.Void]:
-      for with_void in [False, True]:
-        for with_i in [False, True]:
-          for with_r in [False, True]:
-            for with_f in [False, True]:
-              ARGS = []
-              if with_void: ARGS += [lltype.Void, lltype.Void]
-              if with_i: ARGS += [lltype.Signed, lltype.Char]
-              if with_r: ARGS += [rclass.OBJECTPTR, lltype.Ptr(rstr.STR)]
-              if with_f: ARGS += [lltype.Float, lltype.Float]
-              random.shuffle(ARGS)
-              if RESTYPE == lltype.Float: with_f = True
-              if with_f: expectedkind = 'irf'   # all kinds
-              elif with_i: expectedkind = 'ir'  # integers and references
-              else: expectedkind = 'r'          # only references
-              yield residual_call_test, ARGS, RESTYPE, expectedkind
-              yield direct_call_test, ARGS, RESTYPE, expectedkind
-              yield indirect_residual_call_test, ARGS, RESTYPE, expectedkind
-              yield indirect_regular_call_test, ARGS, RESTYPE, expectedkind
+    for RESTYPE, with_void, with_i, with_r, with_f in product(
+        [lltype.Signed, rclass.OBJECTPTR, lltype.Float, lltype.Void],
+        [False, True],
+        [False, True],
+        [False, True],
+        [False, True],
+    ):
+        ARGS = []
+        if with_void:
+            ARGS += [lltype.Void, lltype.Void]
+        if with_i:
+            ARGS += [lltype.Signed, lltype.Char]
+        if with_r:
+            ARGS += [rclass.OBJECTPTR, lltype.Ptr(rstr.STR)]
+        if with_f:
+            ARGS += [lltype.Float, lltype.Float]
+        random.shuffle(ARGS)
+        if RESTYPE == lltype.Float:
+            with_f = True
+        if with_f:
+            expectedkind = 'irf'   # all kinds
+        elif with_i:
+            expectedkind = 'ir'  # integers and references
+        else:
+            expectedkind = 'r'          # only references
+        yield residual_call_test, ARGS, RESTYPE, expectedkind
+        yield direct_call_test, ARGS, RESTYPE, expectedkind
+        yield indirect_residual_call_test, ARGS, RESTYPE, expectedkind
+        yield indirect_regular_call_test, ARGS, RESTYPE, expectedkind
 
 def get_direct_call_op(argtypes, restype):
     FUNC = lltype.FuncType(argtypes, restype)
@@ -821,6 +851,21 @@
     assert op1.args[2] == ListOfKind('ref', [v1, v2])
     assert op1.result == v3
 
+def test_str_promote():
+    PSTR = lltype.Ptr(rstr.STR)
+    v1 = varoftype(PSTR)
+    v2 = varoftype(PSTR)
+    op = SpaceOperation('hint',
+                        [v1, Constant({'promote_string': True}, lltype.Void)],
+                        v2)
+    tr = Transformer(FakeCPU(), FakeBuiltinCallControl())
+    op0, op1, _ = tr.rewrite_operation(op)
+    assert op1.opname == 'str_guard_value'
+    assert op1.args[0] == v1
+    assert op1.args[2] == 'calldescr'
+    assert op1.result == v2
+    assert op0.opname == '-live-'
+
 def test_unicode_concat():
     # test that the oopspec is present and correctly transformed
     PSTR = lltype.Ptr(rstr.UNICODE)
diff --git a/pypy/jit/codewriter/test/test_longlong.py b/pypy/jit/codewriter/test/test_longlong.py
--- a/pypy/jit/codewriter/test/test_longlong.py
+++ b/pypy/jit/codewriter/test/test_longlong.py
@@ -57,7 +57,8 @@
             assert op1.opname == 'residual_call_irf_f'
         else:
             assert op1.opname == 'residual_call_irf_i'
-        gotindex = getattr(EffectInfo, 'OS_' + op1.args[0].value.upper())
+        gotindex = getattr(EffectInfo,
+                           'OS_' + op1.args[0].value.upper().lstrip('U'))
         assert gotindex == oopspecindex
         assert op1.args[1] == 'calldescr-%d' % oopspecindex
         assert list(op1.args[2]) == [v for v in vlist
@@ -192,8 +193,12 @@
                       [lltype.SignedLongLong], lltype.Signed)
         self.do_check('cast_float_to_longlong', EffectInfo.OS_LLONG_FROM_FLOAT,
                       [lltype.Float], lltype.SignedLongLong)
+        self.do_check('cast_float_to_ulonglong', EffectInfo.OS_LLONG_FROM_FLOAT,
+                      [lltype.Float], lltype.UnsignedLongLong)
         self.do_check('cast_longlong_to_float', EffectInfo.OS_LLONG_TO_FLOAT,
                       [lltype.SignedLongLong], lltype.Float)
+        self.do_check('cast_ulonglong_to_float', EffectInfo.OS_LLONG_U_TO_FLOAT,
+                      [lltype.UnsignedLongLong], lltype.Float)
         for T1 in [lltype.SignedLongLong, lltype.UnsignedLongLong]:
             for T2 in [lltype.Signed, lltype.Unsigned]:
                 self.do_check('cast_primitive', EffectInfo.OS_LLONG_TO_INT,
diff --git a/pypy/jit/metainterp/blackhole.py b/pypy/jit/metainterp/blackhole.py
--- a/pypy/jit/metainterp/blackhole.py
+++ b/pypy/jit/metainterp/blackhole.py
@@ -2,7 +2,7 @@
 from pypy.rlib.rtimer import read_timestamp
 from pypy.rlib.rarithmetic import intmask, LONG_BIT, r_uint, ovfcheck
 from pypy.rlib.objectmodel import we_are_translated
-from pypy.rlib.debug import debug_start, debug_stop
+from pypy.rlib.debug import debug_start, debug_stop, ll_assert
 from pypy.rlib.debug import make_sure_not_resized
 from pypy.rpython.lltypesystem import lltype, llmemory, rclass
 from pypy.rpython.lltypesystem.lloperation import llop
@@ -503,6 +503,15 @@
     @arguments("r", returns="r")
     def bhimpl_cast_opaque_ptr(a):
         return a
+    @arguments("r", returns="i")
+    def bhimpl_cast_ptr_to_int(a):
+        i = lltype.cast_ptr_to_int(a)
+        ll_assert((i & 1) == 1, "bhimpl_cast_ptr_to_int: not an odd int")
+        return i
+    @arguments("i", returns="r")
+    def bhimpl_cast_int_to_ptr(i):
+        ll_assert((i & 1) == 1, "bhimpl_cast_int_to_ptr: not an odd int")
+        return lltype.cast_int_to_ptr(llmemory.GCREF, i)
 
     @arguments("i", returns="i")
     def bhimpl_int_copy(a):
@@ -523,6 +532,9 @@
     @arguments("f")
     def bhimpl_float_guard_value(a):
         pass
+    @arguments("r", "i", "d", returns="r")
+    def bhimpl_str_guard_value(a, i, d):
+        return a
 
     @arguments("self", "i")
     def bhimpl_int_push(self, a):
diff --git a/pypy/jit/metainterp/graphpage.py b/pypy/jit/metainterp/graphpage.py
--- a/pypy/jit/metainterp/graphpage.py
+++ b/pypy/jit/metainterp/graphpage.py
@@ -12,8 +12,8 @@
     def get_display_text(self):
         return None
 
-def display_loops(loops, errmsg=None, highlight_loops=()):
-    graphs = [(loop, loop in highlight_loops) for loop in loops]    
+def display_loops(loops, errmsg=None, highlight_loops={}):
+    graphs = [(loop, highlight_loops.get(loop, 0)) for loop in loops]    
     for graph, highlight in graphs:
         for op in graph.get_operations():
             if is_interesting_guard(op):
@@ -65,8 +65,7 @@
     def add_graph(self, graph, highlight=False):
         graphindex = len(self.graphs)
         self.graphs.append(graph)
-        if highlight:
-            self.highlight_graphs[graph] = True
+        self.highlight_graphs[graph] = highlight
         for i, op in enumerate(graph.get_operations()):
             self.all_operations[op] = graphindex, i
 
@@ -126,10 +125,13 @@
             self.dotgen.emit('subgraph cluster%d {' % graphindex)
         label = graph.get_display_text()
         if label is not None:
-            if self.highlight_graphs.get(graph):
-                fillcolor = '#f084c2'
+            colorindex = self.highlight_graphs.get(graph, 0)
+            if colorindex == 1:
+                fillcolor = '#f084c2'    # highlighted graph
+            elif colorindex == 2:
+                fillcolor = '#808080'    # invalidated graph
             else:
-                fillcolor = '#84f0c2'
+                fillcolor = '#84f0c2'    # normal color
             self.dotgen.emit_node(graphname, shape="octagon",
                                   label=label, fillcolor=fillcolor)
             self.pendingedges.append((graphname,
diff --git a/pypy/jit/metainterp/heapcache.py b/pypy/jit/metainterp/heapcache.py
--- a/pypy/jit/metainterp/heapcache.py
+++ b/pypy/jit/metainterp/heapcache.py
@@ -54,9 +54,10 @@
         if box in self.new_boxes:
             self.new_boxes[box] = False
         if box in self.dependencies:
-            for dep in self.dependencies[box]:
+            deps = self.dependencies[box]
+            del self.dependencies[box]
+            for dep in deps:
                 self._escape(dep)
-            del self.dependencies[box]
 
     def clear_caches(self, opnum, descr, argboxes):
         if opnum == rop.SETFIELD_GC:
diff --git a/pypy/jit/metainterp/history.py b/pypy/jit/metainterp/history.py
--- a/pypy/jit/metainterp/history.py
+++ b/pypy/jit/metainterp/history.py
@@ -9,6 +9,7 @@
 
 from pypy.jit.metainterp.resoperation import ResOperation, rop
 from pypy.jit.codewriter import heaptracker, longlong
+from pypy.rlib.objectmodel import compute_identity_hash
 
 # ____________________________________________________________
 
@@ -104,7 +105,7 @@
     getref._annspecialcase_ = 'specialize:arg(1)'
 
     def _get_hash_(self):
-        raise NotImplementedError
+        return compute_identity_hash(self)
 
     def clonebox(self):
         raise NotImplementedError
@@ -133,6 +134,9 @@
     def _get_str(self):
         raise NotImplementedError
 
+    def same_box(self, other):
+        return self is other
+
 class AbstractDescr(AbstractValue):
     __slots__ = ()
 
@@ -241,32 +245,15 @@
     def constbox(self):
         return self
 
+    def same_box(self, other):
+        return self.same_constant(other)
+
     def same_constant(self, other):
         raise NotImplementedError
 
     def __repr__(self):
         return 'Const(%s)' % self._getrepr_()
 
-    def __eq__(self, other):
-        "NOT_RPYTHON"
-        # Remember that you should not compare Consts with '==' in RPython.
-        # Consts have no special __hash__, in order to force different Consts
-        # from being considered as different keys when stored in dicts
-        # (as they always are after translation).  Use a dict_equal_consts()
-        # to get the other behavior (i.e. using this __eq__).
-        if self.__class__ is not other.__class__:
-            return False
-        try:
-            return self.value == other.value
-        except TypeError:
-            if (isinstance(self.value, Symbolic) and
-                isinstance(other.value, Symbolic)):
-                return self.value is other.value
-            raise
-
-    def __ne__(self, other):
-        return not (self == other)
-
 
 class ConstInt(Const):
     type = INT
@@ -688,33 +675,6 @@
 
 # ____________________________________________________________
 
-def dict_equal_consts():
-    "NOT_RPYTHON"
-    # Returns a dict in which Consts that compare as equal
-    # are identified when used as keys.
-    return r_dict(dc_eq, dc_hash)
-
-def dc_eq(c1, c2):
-    return c1 == c2
-
-def dc_hash(c):
-    "NOT_RPYTHON"
-    # This is called during translation only.  Avoid using identityhash(),
-    # to avoid forcing a hash, at least on lltype objects.
-    if not isinstance(c, Const):
-        return hash(c)
-    if isinstance(c.value, Symbolic):
-        return id(c.value)
-    try:
-        if isinstance(c, ConstPtr):
-            p = lltype.normalizeptr(c.value)
-            if p is not None:
-                return hash(p._obj)
-            else:
-                return 0
-        return c._get_hash_()
-    except lltype.DelayedPointer:
-        return -2      # xxx risk of changing hash...
 
 def make_hashable_int(i):
     from pypy.rpython.lltypesystem.ll2ctypes import NotCtypesAllocatedStructure
@@ -772,6 +732,7 @@
     failed_states = None
     retraced_count = 0
     terminating = False # see TerminatingLoopToken in compile.py
+    invalidated = False
     outermost_jitdriver_sd = None
     # and more data specified by the backend when the loop is compiled
     number = -1
@@ -974,6 +935,7 @@
         self.loops = []
         self.locations = []
         self.aborted_keys = []
+        self.invalidated_token_numbers = set()
 
     def set_history(self, history):
         self.operations = history.operations
@@ -1052,7 +1014,12 @@
             if loop in loops:
                 loops.remove(loop)
             loops.append(loop)
-        display_loops(loops, errmsg, extraloops)
+        highlight_loops = dict.fromkeys(extraloops, 1)
+        for loop in loops:
+            if hasattr(loop, '_looptoken_number') and (
+                    loop._looptoken_number in self.invalidated_token_numbers):
+                highlight_loops.setdefault(loop, 2)
+        display_loops(loops, errmsg, highlight_loops)
 
 # ----------------------------------------------------------------
 
diff --git a/pypy/jit/metainterp/memmgr.py b/pypy/jit/metainterp/memmgr.py
--- a/pypy/jit/metainterp/memmgr.py
+++ b/pypy/jit/metainterp/memmgr.py
@@ -68,7 +68,8 @@
         debug_print("Loop tokens before:", oldtotal)
         max_generation = self.current_generation - (self.max_age-1)
         for looptoken in self.alive_loops.keys():
-            if 0 <= looptoken.generation < max_generation:
+            if (0 <= looptoken.generation < max_generation or
+                looptoken.invalidated):
                 del self.alive_loops[looptoken]
         newtotal = len(self.alive_loops)
         debug_print("Loop tokens freed: ", oldtotal - newtotal)
diff --git a/pypy/jit/metainterp/optimizeopt/__init__.py b/pypy/jit/metainterp/optimizeopt/__init__.py
--- a/pypy/jit/metainterp/optimizeopt/__init__.py
+++ b/pypy/jit/metainterp/optimizeopt/__init__.py
@@ -7,6 +7,8 @@
 from pypy.jit.metainterp.optimizeopt.unroll import optimize_unroll, OptInlineShortPreamble
 from pypy.jit.metainterp.optimizeopt.fficall import OptFfiCall
 from pypy.jit.metainterp.optimizeopt.simplify import OptSimplify
+from pypy.jit.metainterp.optimizeopt.pure import OptPure
+from pypy.jit.metainterp.optimizeopt.earlyforce import OptEarlyForce
 from pypy.rlib.jit import PARAMETERS
 from pypy.rlib.unroll import unrolling_iterable
 
@@ -14,6 +16,8 @@
             ('rewrite', OptRewrite),
             ('virtualize', OptVirtualize),
             ('string', OptString),
+            ('earlyforce', OptEarlyForce),
+            ('pure', OptPure),
             ('heap', OptHeap),
             ('ffi', None),
             ('unroll', None)]
diff --git a/pypy/jit/metainterp/optimizeopt/earlyforce.py b/pypy/jit/metainterp/optimizeopt/earlyforce.py
new file mode 100644
--- /dev/null
+++ b/pypy/jit/metainterp/optimizeopt/earlyforce.py
@@ -0,0 +1,24 @@
+from pypy.jit.metainterp.optimizeopt.optimizer import Optimization
+from pypy.jit.metainterp.optimizeopt.vstring import VAbstractStringValue
+from pypy.jit.metainterp.resoperation import rop, ResOperation
+
+class OptEarlyForce(Optimization):
+    def propagate_forward(self, op):
+        opnum = op.getopnum()
+        if (opnum != rop.SETFIELD_GC and 
+            opnum != rop.SETARRAYITEM_GC and
+            opnum != rop.QUASIIMMUT_FIELD):
+               
+            for arg in op.getarglist():
+                if arg in self.optimizer.values:
+                    value = self.getvalue(arg)
+                    value.force_box(self)
+        self.emit_operation(op)
+
+    def new(self):
+        return OptEarlyForce()
+
+    def setup(self):
+        self.optimizer.optearlyforce = self
+
+    
diff --git a/pypy/jit/metainterp/optimizeopt/fficall.py b/pypy/jit/metainterp/optimizeopt/fficall.py
--- a/pypy/jit/metainterp/optimizeopt/fficall.py
+++ b/pypy/jit/metainterp/optimizeopt/fficall.py
@@ -177,10 +177,10 @@
             funcinfo.descr is None):
             return [op] # cannot optimize
         funcsymval = self.getvalue(op.getarg(2))
-        arglist = [funcsymval.force_box()]
+        arglist = [funcsymval.get_key_box()]
         for push_op in funcinfo.opargs:
             argval = self.getvalue(push_op.getarg(2))
-            arglist.append(argval.force_box())
+            arglist.append(argval.get_key_box())
         newop = ResOperation(rop.CALL_RELEASE_GIL, arglist, op.result,
                              descr=funcinfo.descr)
         self.commit_optimization()
diff --git a/pypy/jit/metainterp/optimizeopt/heap.py b/pypy/jit/metainterp/optimizeopt/heap.py
--- a/pypy/jit/metainterp/optimizeopt/heap.py
+++ b/pypy/jit/metainterp/optimizeopt/heap.py
@@ -50,6 +50,7 @@
             if not self._lazy_setfield_registered:
                 optheap._lazy_setfields_and_arrayitems.append(self)
                 self._lazy_setfield_registered = True
+            
         else:
             # this is the case where the pending setfield ends up
             # storing precisely the value that is already there,
@@ -158,12 +159,17 @@
         self._lazy_setfields_and_arrayitems = []
         self._remove_guard_not_invalidated = False
         self._seen_guard_not_invalidated = False
+        self.posponedop = None
 
     def force_at_end_of_preamble(self):
         self.force_all_lazy_setfields_and_arrayitems()
 
     def flush(self):
         self.force_all_lazy_setfields_and_arrayitems()
+        if self.posponedop:
+            posponedop = self.posponedop
+            self.posponedop = None
+            self.next_optimization.propagate_forward(posponedop)
 
     def new(self):
         return OptHeap()
@@ -211,7 +217,15 @@
 
     def emit_operation(self, op):
         self.emitting_operation(op)
-        self.next_optimization.propagate_forward(op)
+        if self.posponedop:
+            posponedop = self.posponedop
+            self.posponedop = None
+            self.next_optimization.propagate_forward(posponedop)
+        if (op.is_comparison() or op.getopnum() == rop.CALL_MAY_FORCE
+            or op.is_ovf()):
+            self.posponedop = op
+        else:
+            self.next_optimization.propagate_forward(op)
 
     def emitting_operation(self, op):
         if op.has_no_side_effect():
@@ -293,30 +307,6 @@
             if indexvalue is None or indexvalue.intbound.contains(idx):
                 cf.force_lazy_setfield(self, can_cache)
 
-    def fixup_guard_situation(self):
-        # hackish: reverse the order of the last two operations if it makes
-        # sense to avoid a situation like "int_eq/setfield_gc/guard_true",
-        # which the backend (at least the x86 backend) does not handle well.
-        newoperations = self.optimizer.newoperations
-        if len(newoperations) < 2:
-            return
-        lastop = newoperations[-1]
-        if (lastop.getopnum() != rop.SETFIELD_GC and
-            lastop.getopnum() != rop.SETARRAYITEM_GC):
-            return
-        # - is_comparison() for cases like "int_eq/setfield_gc/guard_true"
-        # - CALL_MAY_FORCE: "call_may_force/setfield_gc/guard_not_forced"
-        # - is_ovf(): "int_add_ovf/setfield_gc/guard_no_overflow"
-        prevop = newoperations[-2]
-        opnum = prevop.getopnum()
-        if not (prevop.is_comparison() or opnum == rop.CALL_MAY_FORCE
-                or prevop.is_ovf()):
-            return
-        if prevop.result in lastop.getarglist():
-            return
-        newoperations[-2] = lastop
-        newoperations[-1] = prevop
-
     def _assert_valid_cf(self, cf):
         # check that 'cf' is in cached_fields or cached_arrayitems
         if not we_are_translated():
@@ -362,7 +352,6 @@
                                       fieldvalue.get_key_box(), itemindex))
             else:
                 cf.force_lazy_setfield(self)
-                self.fixup_guard_situation()
         return pendingfields
 
     def optimize_GETFIELD_GC(self, op):
@@ -374,12 +363,22 @@
             return
         # default case: produce the operation
         structvalue.ensure_nonnull()
-        ###self.optimizer.optimize_default(op)
         self.emit_operation(op)
         # then remember the result of reading the field
         fieldvalue = self.getvalue(op.result)
         cf.remember_field_value(structvalue, fieldvalue, op)
 
+    def optimize_GETFIELD_GC_PURE(self, op):
+        structvalue = self.getvalue(op.getarg(0))
+        cf = self.field_cache(op.getdescr())
+        fieldvalue = cf.getfield_from_cache(self, structvalue)
+        if fieldvalue is not None:
+            self.make_equal_to(op.result, fieldvalue)
+            return
+        # default case: produce the operation
+        structvalue.ensure_nonnull()
+        self.emit_operation(op)
+
     def optimize_SETFIELD_GC(self, op):
         if self.has_pure_result(rop.GETFIELD_GC_PURE, [op.getarg(0)],
                                 op.getdescr()):
@@ -389,6 +388,7 @@
         #
         cf = self.field_cache(op.getdescr())
         cf.do_setfield(self, op)
+        
 
     def optimize_GETARRAYITEM_GC(self, op):
         arrayvalue = self.getvalue(op.getarg(0))
@@ -413,6 +413,25 @@
             fieldvalue = self.getvalue(op.result)
             cf.remember_field_value(arrayvalue, fieldvalue, op)
 
+    def optimize_GETARRAYITEM_GC_PURE(self, op):
+        arrayvalue = self.getvalue(op.getarg(0))
+        indexvalue = self.getvalue(op.getarg(1))
+        cf = None
+        if indexvalue.is_constant():
+            arrayvalue.make_len_gt(MODE_ARRAY, op.getdescr(), indexvalue.box.getint())
+            # use the cache on (arraydescr, index), which is a constant
+            cf = self.arrayitem_cache(op.getdescr(), indexvalue.box.getint())
+            fieldvalue = cf.getfield_from_cache(self, arrayvalue)
+            if fieldvalue is not None:
+                self.make_equal_to(op.result, fieldvalue)
+                return
+        else:
+            # variable index, so make sure the lazy setarrayitems are done
+            self.force_lazy_setarrayitem(op.getdescr(), indexvalue=indexvalue)
+        # default case: produce the operation
+        arrayvalue.ensure_nonnull()
+        self.emit_operation(op)
+
     def optimize_SETARRAYITEM_GC(self, op):
         if self.has_pure_result(rop.GETARRAYITEM_GC_PURE, [op.getarg(0),
                                                            op.getarg(1)],
diff --git a/pypy/jit/metainterp/optimizeopt/optimizer.py b/pypy/jit/metainterp/optimizeopt/optimizer.py
--- a/pypy/jit/metainterp/optimizeopt/optimizer.py
+++ b/pypy/jit/metainterp/optimizeopt/optimizer.py
@@ -31,8 +31,8 @@
 
 class OptValue(object):
     __metaclass__ = extendabletype
-    _attrs_ = ('box', 'known_class', 'last_guard_index', 'level', 'intbound', 'lenbound')
-    last_guard_index = -1
+    _attrs_ = ('box', 'known_class', 'last_guard', 'level', 'intbound', 'lenbound')
+    last_guard = None
 
     level = LEVEL_UNKNOWN
     known_class = None
@@ -100,7 +100,7 @@
             self.make_constant(other.get_key_box())
             optimizer.turned_constant(self)
         elif other.level == LEVEL_KNOWNCLASS:
-            self.make_constant_class(other.known_class, -1)
+            self.make_constant_class(other.known_class, None)
         else:
             if other.level == LEVEL_NONNULL:
                 self.ensure_nonnull()
@@ -114,13 +114,13 @@
                     self.lenbound = other.lenbound.clone()
 
 
-    def force_box(self):
+    def force_box(self, optforce):
         return self.box
 
     def get_key_box(self):
         return self.box
 
-    def force_at_end_of_preamble(self, already_forced):
+    def force_at_end_of_preamble(self, already_forced, optforce):
         return self
 
     def get_args_for_fail(self, modifier):
@@ -162,16 +162,16 @@
         else:
             return None
 
-    def make_constant_class(self, classbox, opindex):
+    def make_constant_class(self, classbox, guardop):
         assert self.level < LEVEL_KNOWNCLASS
         self.known_class = classbox
         self.level = LEVEL_KNOWNCLASS
-        self.last_guard_index = opindex
+        self.last_guard = guardop
 
-    def make_nonnull(self, opindex):
+    def make_nonnull(self, guardop):
         assert self.level < LEVEL_NONNULL
         self.level = LEVEL_NONNULL
-        self.last_guard_index = opindex
+        self.last_guard = guardop
 
     def is_nonnull(self):
         level = self.level
@@ -240,26 +240,12 @@
     def __init__(self):
         pass # make rpython happy
 
-    def propagate_begin_forward(self):
-        if self.next_optimization:
-            self.next_optimization.propagate_begin_forward()
-
-    def propagate_end_forward(self):
-        if self.next_optimization:
-            self.next_optimization.propagate_end_forward()
-
     def propagate_forward(self, op):
         raise NotImplementedError
 
     def emit_operation(self, op):
         self.next_optimization.propagate_forward(op)
 
-    def test_emittable(self, op):
-        return self.is_emittable(op)
-
-    def is_emittable(self, op):
-        return self.next_optimization.test_emittable(op)
-
     # FIXME: Move some of these here?
     def getvalue(self, box):
         return self.optimizer.getvalue(box)
@@ -289,19 +275,19 @@
         return self.optimizer.new_const_item(arraydescr)
 
     def pure(self, opnum, args, result):
-        op = ResOperation(opnum, args, result)
-        key = self.optimizer.make_args_key(op)
-        if key not in self.optimizer.pure_operations:
-            self.optimizer.pure_operations[key] = op
+        if self.optimizer.optpure:
+            self.optimizer.optpure.pure(opnum, args, result)
 
     def has_pure_result(self, opnum, args, descr):
-        op = ResOperation(opnum, args, None, descr)
-        key = self.optimizer.make_args_key(op)
-        op = self.optimizer.pure_operations.get(key, None)
-        if op is None:
-            return False
-        return op.getdescr() is descr
+        if self.optimizer.optpure:
+            return self.optimizer.optpure.has_pure_result(opnum, args, descr)
+        return False
 
+    def get_pure_result(self, key):    
+        if self.optimizer.optpure:
+            return self.optimizer.optpure.get_pure_result(key)
+        return None
+    
     def setup(self):
         pass
 
@@ -322,6 +308,9 @@
     def produce_potential_short_preamble_ops(self, potential_ops):
         pass
 
+    def forget_numberings(self, box):
+        self.optimizer.forget_numberings(box)
+
 class Optimizer(Optimization):
 
     def __init__(self, metainterp_sd, loop, optimizations=None, bridge=False):
@@ -333,14 +322,16 @@
         self.interned_refs = self.cpu.ts.new_ref_dict()
         self.resumedata_memo = resume.ResumeDataLoopMemo(metainterp_sd)
         self.bool_boxes = {}
-        self.pure_operations = args_dict()
         self.producer = {}
         self.pendingfields = []
-        self.posponedop = None
         self.exception_might_have_happened = False
         self.quasi_immutable_deps = None
         self.opaque_pointers = {}
-        self.newoperations = []
+        self.replaces_guard = {}
+        self._newoperations = []
+        self.optimizer = self
+        self.optpure = None
+        self.optearlyforce = None
         if loop is not None:
             self.call_pure_results = loop.call_pure_results
 
@@ -369,21 +360,20 @@
     def flush(self):
         for o in self.optimizations:
             o.flush()
-        assert self.posponedop is None
 
     def new(self):
         new = Optimizer(self.metainterp_sd, self.loop)
         return self._new(new)
 
     def _new(self, new):
-        assert self.posponedop is None
         optimizations = [o.new() for o in self.optimizations]
         new.set_optimizations(optimizations)
         new.quasi_immutable_deps = self.quasi_immutable_deps
         return new
 
     def produce_potential_short_preamble_ops(self, sb):
-        raise NotImplementedError('This is implemented in unroll.UnrollableOptimizer')
+        for opt in self.optimizations:
+            opt.produce_potential_short_preamble_ops(sb)
 
     def turned_constant(self, value):
         for o in self.optimizations:
@@ -433,6 +423,13 @@
             return constbox
         return None
 
+    def get_newoperations(self):
+        self.flush()
+        return self._newoperations
+
+    def clear_newoperations(self):
+        self._newoperations = []
+
     def make_equal_to(self, box, value, replace=False):
         assert isinstance(value, OptValue)
         assert replace or box not in self.values
@@ -481,15 +478,10 @@
 
     def propagate_all_forward(self):
         self.exception_might_have_happened = self.bridge
-        self.newoperations = []
-        self.first_optimization.propagate_begin_forward()
-        self.i = 0
-        while self.i < len(self.loop.operations):
-            op = self.loop.operations[self.i]
+        self.clear_newoperations()
+        for op in self.loop.operations:
             self.first_optimization.propagate_forward(op)
-            self.i += 1
-        self.first_optimization.propagate_end_forward()
-        self.loop.operations = self.newoperations
+        self.loop.operations = self.get_newoperations()
         self.loop.quasi_immutable_deps = self.quasi_immutable_deps
         # accumulate counters
         self.resumedata_memo.update_counters(self.metainterp_sd.profiler)
@@ -501,9 +493,6 @@
         self.producer[op.result] = op
         dispatch_opt(self, op)
 
-    def test_emittable(self, op):
-        return True
-
     def emit_operation(self, op):
         if op.returns_bool_result():
             self.bool_boxes[self.getvalue(op.result)] = None
@@ -519,14 +508,30 @@
                 pass
             else:
                 self.ensure_imported(value)
-                op.setarg(i, value.force_box())
+                op.setarg(i, value.force_box(self))
         self.metainterp_sd.profiler.count(jitprof.OPT_OPS)
         if op.is_guard():
             self.metainterp_sd.profiler.count(jitprof.OPT_GUARDS)
-            op = self.store_final_boxes_in_guard(op)
+            if self.replaces_guard and op in self.replaces_guard:
+                self.replace_op(self.replaces_guard[op], op)
+                del self.replaces_guard[op]
+                return
+            else:
+                op = self.store_final_boxes_in_guard(op)
         elif op.can_raise():
             self.exception_might_have_happened = True
-        self.newoperations.append(op)
+        self._newoperations.append(op)
+
+    def replace_op(self, old_op, new_op):
+        # XXX: Do we want to cache indexes to prevent search?
+        i = len(self._newoperations) 
+        while i > 0:
+            i -= 1
+            if self._newoperations[i] is old_op:
+                self._newoperations[i] = new_op
+                break
+        else:
+            assert False
 
     def store_final_boxes_in_guard(self, op):
         descr = op.getdescr()
@@ -574,51 +579,8 @@
         args[n + 1] = op.getdescr()
         return args
 
-    @specialize.argtype(0)
     def optimize_default(self, op):
-        canfold = op.is_always_pure()
-        if op.is_ovf():
-            self.posponedop = op
-            return
-        if self.posponedop:
-            nextop = op
-            op = self.posponedop
-            self.posponedop = None
-            canfold = nextop.getopnum() == rop.GUARD_NO_OVERFLOW
-        else:
-            nextop = None
-
-        if canfold:
-            for i in range(op.numargs()):
-                if self.get_constant_box(op.getarg(i)) is None:
-                    break
-            else:
-                # all constant arguments: constant-fold away
-                resbox = self.constant_fold(op)
-                # note that INT_xxx_OVF is not done from here, and the
-                # overflows in the INT_xxx operations are ignored
-                self.make_constant(op.result, resbox)
-                return
-
-            # did we do the exact same operation already?
-            args = self.make_args_key(op)
-            oldop = self.pure_operations.get(args, None)
-            if oldop is not None and oldop.getdescr() is op.getdescr():
-                assert oldop.getopnum() == op.getopnum()
-                self.make_equal_to(op.result, self.getvalue(oldop.result),
-                                   True)
-                return
-            else:
-                self.pure_operations[args] = op
-                self.remember_emitting_pure(op)
-
-        # otherwise, the operation remains
         self.emit_operation(op)
-        if nextop:
-            self.emit_operation(nextop)
-
-    def remember_emitting_pure(self, op):
-        pass
 
     def constant_fold(self, op):
         argboxes = [self.get_constant_box(op.getarg(i))
@@ -636,11 +598,6 @@
     def optimize_DEBUG_MERGE_POINT(self, op):
         self.emit_operation(op)
 
-    def optimize_CAST_OPAQUE_PTR(self, op):
-        value = self.getvalue(op.getarg(0))
-        self.opaque_pointers[value] = True
-        self.make_equal_to(op.result, value)
-
     def optimize_GETARRAYITEM_GC_PURE(self, op):
         indexvalue = self.getvalue(op.getarg(1))
         if indexvalue.is_constant():
diff --git a/pypy/jit/metainterp/optimizeopt/pure.py b/pypy/jit/metainterp/optimizeopt/pure.py
new file mode 100644
--- /dev/null
+++ b/pypy/jit/metainterp/optimizeopt/pure.py
@@ -0,0 +1,115 @@
+from pypy.jit.metainterp.optimizeopt.optimizer import Optimization
+from pypy.jit.metainterp.resoperation import rop, ResOperation
+from pypy.jit.metainterp.optimizeopt.util import (make_dispatcher_method,
+    args_dict)
+
+class OptPure(Optimization):
+    def __init__(self):
+        self.posponedop = None
+        self.pure_operations = args_dict()
+        self.emitted_pure_operations = {}
+
+    def propagate_forward(self, op):
+        dispatch_opt(self, op)
+
+    def optimize_default(self, op):
+        canfold = op.is_always_pure()
+        if op.is_ovf():
+            self.posponedop = op
+            return
+        if self.posponedop:
+            nextop = op
+            op = self.posponedop
+            self.posponedop = None
+            canfold = nextop.getopnum() == rop.GUARD_NO_OVERFLOW
+        else:
+            nextop = None
+
+        if canfold:
+            for i in range(op.numargs()):
+                if self.get_constant_box(op.getarg(i)) is None:
+                    break
+            else:
+                # all constant arguments: constant-fold away
+                resbox = self.optimizer.constant_fold(op)
+                # note that INT_xxx_OVF is not done from here, and the
+                # overflows in the INT_xxx operations are ignored
+                self.optimizer.make_constant(op.result, resbox)
+                return
+
+            # did we do the exact same operation already?
+            args = self.optimizer.make_args_key(op)
+            oldop = self.pure_operations.get(args, None)
+            if oldop is not None and oldop.getdescr() is op.getdescr():
+                assert oldop.getopnum() == op.getopnum()
+                self.optimizer.make_equal_to(op.result, self.getvalue(oldop.result),
+                                   True)
+                return
+            else:
+                self.pure_operations[args] = op
+                self.remember_emitting_pure(op)
+
+        # otherwise, the operation remains
+        self.emit_operation(op)
+        if op.returns_bool_result():
+            self.optimizer.bool_boxes[self.getvalue(op.result)] = None        
+        if nextop:
+            self.emit_operation(nextop)
+
+    def optimize_CALL_PURE(self, op):
+        args = self.optimizer.make_args_key(op)
+        oldop = self.pure_operations.get(args, None)
+        if oldop is not None and oldop.getdescr() is op.getdescr():
+            assert oldop.getopnum() == op.getopnum()
+            self.make_equal_to(op.result, self.getvalue(oldop.result))
+            return
+        else:
+            self.pure_operations[args] = op
+            self.remember_emitting_pure(op)
+
+        # replace CALL_PURE with just CALL
+        args = op.getarglist()
+        self.emit_operation(ResOperation(rop.CALL, args, op.result,
+                                         op.getdescr()))
+
+    def flush(self):
+        assert self.posponedop is None
+
+    def new(self):
+        assert self.posponedop is None
+        return OptPure()
+
+    def setup(self):
+        self.optimizer.optpure = self
+
+    def pure(self, opnum, args, result):
+        op = ResOperation(opnum, args, result)
+        key = self.optimizer.make_args_key(op)
+        if key not in self.pure_operations:
+            self.pure_operations[key] = op
+
+    def has_pure_result(self, opnum, args, descr):
+        op = ResOperation(opnum, args, None, descr)
+        key = self.optimizer.make_args_key(op)
+        op = self.pure_operations.get(key, None)
+        if op is None:
+            return False
+        return op.getdescr() is descr
+
+    def get_pure_result(self, key):
+        return self.pure_operations.get(key, None)
+
+    def remember_emitting_pure(self, op):
+        self.emitted_pure_operations[op] = True
+
+    def produce_potential_short_preamble_ops(self, sb):
+        for op in self.emitted_pure_operations:
+            if op.getopnum() == rop.GETARRAYITEM_GC_PURE or \
+               op.getopnum() == rop.STRGETITEM or \
+               op.getopnum() == rop.UNICODEGETITEM:
+                if not self.getvalue(op.getarg(1)).is_constant():
+                    continue
+            sb.add_potential(op)
+
+dispatch_opt = make_dispatcher_method(OptPure, 'optimize_',
+                                      default=OptPure.optimize_default)
diff --git a/pypy/jit/metainterp/optimizeopt/rewrite.py b/pypy/jit/metainterp/optimizeopt/rewrite.py
--- a/pypy/jit/metainterp/optimizeopt/rewrite.py
+++ b/pypy/jit/metainterp/optimizeopt/rewrite.py
@@ -31,21 +31,8 @@
 
         dispatch_opt(self, op)
 
-    def test_emittable(self, op):
-        opnum = op.getopnum()
-        for value, cls, func in optimize_guards:
-            if opnum == value:
-                assert isinstance(op, cls)
-                try:
-                    func(self, op, dryrun=True)
-                    return self.is_emittable(op)
-                except InvalidLoop:
-                    return False
-        return self.is_emittable(op)
-
-
     def try_boolinvers(self, op, targs):
-        oldop = self.optimizer.pure_operations.get(targs, None)
+        oldop = self.get_pure_result(targs)
         if oldop is not None and oldop.getdescr() is op.getdescr():
             value = self.getvalue(oldop.result)
             if value.is_constant():
@@ -62,32 +49,35 @@
     def find_rewritable_bool(self, op, args):
         try:
             oldopnum = opboolinvers[op.getopnum()]
+        except KeyError:
+            pass
+        else:
             targs = self.optimizer.make_args_key(ResOperation(oldopnum, [args[0], args[1]],
                                                               None))
             if self.try_boolinvers(op, targs):
                 return True
-        except KeyError:
-            pass
 
         try:
             oldopnum = opboolreflex[op.getopnum()] # FIXME: add INT_ADD, INT_MUL
+        except KeyError:
+            pass
+        else:
             targs = self.optimizer.make_args_key(ResOperation(oldopnum, [args[1], args[0]],
                                                               None))
-            oldop = self.optimizer.pure_operations.get(targs, None)
+            oldop = self.get_pure_result(targs)
             if oldop is not None and oldop.getdescr() is op.getdescr():
                 self.make_equal_to(op.result, self.getvalue(oldop.result))
                 return True
-        except KeyError:
-            pass
 
         try:
             oldopnum = opboolinvers[opboolreflex[op.getopnum()]]
+        except KeyError:
+            pass
+        else:
             targs = self.optimizer.make_args_key(ResOperation(oldopnum, [args[1], args[0]],
                                                               None))
             if self.try_boolinvers(op, targs):
                 return True
-        except KeyError:
-            pass
 
         return False
 
@@ -116,10 +106,9 @@
             self.make_equal_to(op.result, v1)
         else:
             self.emit_operation(op)
-
-        # Synthesize the reverse ops for optimize_default to reuse
-        self.pure(rop.INT_ADD, [op.result, op.getarg(1)], op.getarg(0))
-        self.pure(rop.INT_SUB, [op.getarg(0), op.result], op.getarg(1))
+            # Synthesize the reverse ops for optimize_default to reuse
+            self.pure(rop.INT_ADD, [op.result, op.getarg(1)], op.getarg(0))
+            self.pure(rop.INT_SUB, [op.getarg(0), op.result], op.getarg(1))
 
     def optimize_INT_ADD(self, op):
         v1 = self.getvalue(op.getarg(0))
@@ -132,10 +121,9 @@
             self.make_equal_to(op.result, v1)
         else:
             self.emit_operation(op)
-
-        # Synthesize the reverse op for optimize_default to reuse
-        self.pure(rop.INT_SUB, [op.result, op.getarg(1)], op.getarg(0))
-        self.pure(rop.INT_SUB, [op.result, op.getarg(0)], op.getarg(1))
+            # Synthesize the reverse op for optimize_default to reuse
+            self.pure(rop.INT_SUB, [op.result, op.getarg(1)], op.getarg(0))
+            self.pure(rop.INT_SUB, [op.result, op.getarg(0)], op.getarg(1))
 
     def optimize_INT_MUL(self, op):
         v1 = self.getvalue(op.getarg(0))
@@ -151,13 +139,13 @@
             self.make_constant_int(op.result, 0)
         else:
             for lhs, rhs in [(v1, v2), (v2, v1)]:
-                # x & (x -1) == 0 is a quick test for power of 2
-                if (lhs.is_constant() and
-                    (lhs.box.getint() & (lhs.box.getint() - 1)) == 0):
-                    new_rhs = ConstInt(highest_bit(lhs.box.getint()))
-                    op = op.copy_and_change(rop.INT_LSHIFT, args=[rhs.box, new_rhs])
-                    break
-
+                if lhs.is_constant():
+                    x = lhs.box.getint()
+                    # x & (x - 1) == 0 is a quick test for power of 2
+                    if x & (x - 1) == 0:
+                        new_rhs = ConstInt(highest_bit(lhs.box.getint()))
+                        op = op.copy_and_change(rop.INT_LSHIFT, args=[rhs.box, new_rhs])
+                        break
             self.emit_operation(op)
 
     def optimize_UINT_FLOORDIV(self, op):
@@ -214,40 +202,7 @@
         self.emit_operation(op)
         self.pure(rop.FLOAT_NEG, [op.result], v1)
 
-    def optimize_CALL_PURE(self, op):
-        arg_consts = []
-        for i in range(op.numargs()):
-            arg = op.getarg(i)
-            const = self.get_constant_box(arg)
-            if const is None:
-                break
-            arg_consts.append(const)
-        else:
-            # all constant arguments: check if we already know the result
-            try:
-                result = self.optimizer.call_pure_results[arg_consts]
-            except KeyError:
-                pass
-            else:
-                self.make_constant(op.result, result)
-                return
-
-        args = self.optimizer.make_args_key(op)
-        oldop = self.optimizer.pure_operations.get(args, None)
-        if oldop is not None and oldop.getdescr() is op.getdescr():
-            assert oldop.getopnum() == op.getopnum()
-            self.make_equal_to(op.result, self.getvalue(oldop.result))
-            return
-        else:
-            self.optimizer.pure_operations[args] = op
-            self.optimizer.remember_emitting_pure(op)
-
-        # replace CALL_PURE with just CALL
-        args = op.getarglist()
-        self.emit_operation(ResOperation(rop.CALL, args, op.result,
-                                         op.getdescr()))
-
-    def optimize_guard(self, op, constbox, emit_operation=True, dryrun=False):
+    def optimize_guard(self, op, constbox, emit_operation=True):
         value = self.getvalue(op.getarg(0))
         if value.is_constant():
             box = value.box
@@ -255,62 +210,57 @@
             if not box.same_constant(constbox):
                 raise InvalidLoop
             return
-        if dryrun: return
         if emit_operation:
             self.emit_operation(op)
         value.make_constant(constbox)
         self.optimizer.turned_constant(value)
 
-    def optimize_GUARD_ISNULL(self, op, dryrun=False):
+    def optimize_GUARD_ISNULL(self, op):
         value = self.getvalue(op.getarg(0))
         if value.is_null():
             return
         elif value.is_nonnull():
             raise InvalidLoop
-        if dryrun: return
         self.emit_operation(op)
         value.make_constant(self.optimizer.cpu.ts.CONST_NULL)
 
-    def optimize_GUARD_NONNULL(self, op, dryrun=False):
+    def optimize_GUARD_NONNULL(self, op):
         value = self.getvalue(op.getarg(0))
         if value.is_nonnull():
             return
         elif value.is_null():
             raise InvalidLoop
-        if dryrun: return
         self.emit_operation(op)
-        value.make_nonnull(len(self.optimizer.newoperations) - 1)
+        value.make_nonnull(op)
 
-    def optimize_GUARD_VALUE(self, op, dryrun=False):
+    def optimize_GUARD_VALUE(self, op):
         value = self.getvalue(op.getarg(0))
-        emit_operation = True
-        if not dryrun and value.last_guard_index != -1:
+        if value.last_guard:
             # there already has been a guard_nonnull or guard_class or
             # guard_nonnull_class on this value, which is rather silly.
             # replace the original guard with a guard_value
-            old_guard_op = self.optimizer.newoperations[value.last_guard_index]
-            new_guard_op = old_guard_op.copy_and_change(rop.GUARD_VALUE,
-                                             args = [old_guard_op.getarg(0), op.getarg(1)])
-            self.optimizer.newoperations[value.last_guard_index] = new_guard_op
+            old_guard_op = value.last_guard
+            op = old_guard_op.copy_and_change(rop.GUARD_VALUE,
+                                      args = [old_guard_op.getarg(0), op.getarg(1)])
+            self.optimizer.replaces_guard[op] = old_guard_op
             # hack hack hack.  Change the guard_opnum on
             # new_guard_op.getdescr() so that when resuming,
             # the operation is not skipped by pyjitpl.py.
-            descr = new_guard_op.getdescr()
+            descr = op.getdescr()
             assert isinstance(descr, compile.ResumeGuardDescr)
             descr.guard_opnum = rop.GUARD_VALUE
-            descr.make_a_counter_per_value(new_guard_op)
-            emit_operation = False
+            descr.make_a_counter_per_value(op)
         constbox = op.getarg(1)
         assert isinstance(constbox, Const)
-        self.optimize_guard(op, constbox, emit_operation, dryrun)
+        self.optimize_guard(op, constbox)
 
-    def optimize_GUARD_TRUE(self, op, dryrun=False):
-        self.optimize_guard(op, CONST_1, dryrun=dryrun)
+    def optimize_GUARD_TRUE(self, op):
+        self.optimize_guard(op, CONST_1)
 
-    def optimize_GUARD_FALSE(self, op, dryrun=False):
-        self.optimize_guard(op, CONST_0, dryrun=dryrun)
+    def optimize_GUARD_FALSE(self, op):
+        self.optimize_guard(op, CONST_0)
 
-    def optimize_GUARD_CLASS(self, op, dryrun=False):
+    def optimize_GUARD_CLASS(self, op):
         value = self.getvalue(op.getarg(0))
         expectedclassbox = op.getarg(1)
         assert isinstance(expectedclassbox, Const)
@@ -319,38 +269,32 @@
             if realclassbox.same_constant(expectedclassbox):
                 return
             raise InvalidLoop
-        if dryrun: return
-        emit_operation = True
-        if value.last_guard_index != -1:
+        if value.last_guard:
             # there already has been a guard_nonnull or guard_class or
             # guard_nonnull_class on this value.
-            old_guard_op = self.optimizer.newoperations[value.last_guard_index]
+            old_guard_op = value.last_guard
             if old_guard_op.getopnum() == rop.GUARD_NONNULL:
                 # it was a guard_nonnull, which we replace with a
                 # guard_nonnull_class.
-                new_guard_op = old_guard_op.copy_and_change (rop.GUARD_NONNULL_CLASS,
+                op = old_guard_op.copy_and_change (rop.GUARD_NONNULL_CLASS,
                                          args = [old_guard_op.getarg(0), op.getarg(1)])
-                self.optimizer.newoperations[value.last_guard_index] = new_guard_op
+                self.optimizer.replaces_guard[op] = old_guard_op
                 # hack hack hack.  Change the guard_opnum on
                 # new_guard_op.getdescr() so that when resuming,
                 # the operation is not skipped by pyjitpl.py.
-                descr = new_guard_op.getdescr()
+                descr = op.getdescr()
                 assert isinstance(descr, compile.ResumeGuardDescr)
                 descr.guard_opnum = rop.GUARD_NONNULL_CLASS
-                emit_operation = False
-        if emit_operation:
-            self.emit_operation(op)
-            last_guard_index = len(self.optimizer.newoperations) - 1
-        else:
-            last_guard_index = value.last_guard_index
-        value.make_constant_class(expectedclassbox, last_guard_index)
+        self.emit_operation(op)
+        value.make_constant_class(expectedclassbox, op)
 
-    def optimize_GUARD_NONNULL_CLASS(self, op, dryrun=False):
-        self.optimize_GUARD_NONNULL(op, True)
-        self.optimize_GUARD_CLASS(op, dryrun)
+    def optimize_GUARD_NONNULL_CLASS(self, op):
+        value = self.getvalue(op.getarg(0))
+        if value.is_null():
+            raise InvalidLoop
+        self.optimize_GUARD_CLASS(op)
 
-    def optimize_GUARD_NO_EXCEPTION(self, op, dryrun=False):
-        if dryrun: return
+    def optimize_GUARD_NO_EXCEPTION(self, op):
         if not self.optimizer.exception_might_have_happened:
             return
         self.emit_operation(op)
@@ -470,7 +414,7 @@
                     newop = ResOperation(rop.SETARRAYITEM_GC,
                                          [op.getarg(2),
                                           ConstInt(index + dest_start),
-                                          val.force_box()], None,
+                                          val.get_key_box()], None,
                                          descr=source_value.arraydescr)
                     self.emit_operation(newop)
             return True
@@ -478,6 +422,25 @@
             return True # 0-length arraycopy
         return False
 
+    def optimize_CALL_PURE(self, op):
+        arg_consts = []
+        for i in range(op.numargs()):
+            arg = op.getarg(i)
+            const = self.get_constant_box(arg)
+            if const is None:
+                break
+            arg_consts.append(const)
+        else:
+            # all constant arguments: check if we already know the result
+            try:
+                result = self.optimizer.call_pure_results[arg_consts]
+            except KeyError:
+                pass
+            else:
+                self.make_constant(op.result, result)
+                return
+        self.emit_operation(op)
+
     def optimize_INT_FLOORDIV(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
@@ -492,6 +455,18 @@
                                         args = [op.getarg(0), ConstInt(highest_bit(val))])
         self.emit_operation(op)
 
+    def optimize_CAST_OPAQUE_PTR(self, op):
+        value = self.getvalue(op.getarg(0))
+        self.optimizer.opaque_pointers[value] = True
+        self.make_equal_to(op.result, value)
+
+    def optimize_CAST_PTR_TO_INT(self, op):
+        self.pure(rop.CAST_INT_TO_PTR, [op.result], op.getarg(0))
+        self.emit_operation(op)
+
+    def optimize_CAST_INT_TO_PTR(self, op):
+        self.pure(rop.CAST_PTR_TO_INT, [op.result], op.getarg(0))
+        self.emit_operation(op)
 
 dispatch_opt = make_dispatcher_method(OptRewrite, 'optimize_',
         default=OptRewrite.emit_operation)
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -39,8 +39,9 @@
 
 def test_sharing_field_lists_of_virtual():
     class FakeOptimizer(object):
-        class cpu(object):
-            pass
+        class optimizer(object):
+            class cpu(object):
+                pass
     opt = FakeOptimizer()
     virt1 = virtualize.AbstractVirtualStructValue(opt, None)
     lst1 = virt1._get_field_descr_list()
@@ -69,7 +70,7 @@
     class FakeVirtualValue(virtualize.AbstractVirtualValue):
         def _make_virtual(self, *args):
             return FakeVInfo()
-    v1 = FakeVirtualValue(None, None, None)
+    v1 = FakeVirtualValue(None, None)
     vinfo1 = v1.make_virtual_info(None, [1, 2, 4])
     vinfo2 = v1.make_virtual_info(None, [1, 2, 4])
     assert vinfo1 is vinfo2
@@ -111,7 +112,7 @@
 
 class BaseTestBasic(BaseTest):
 
-    enable_opts = "intbounds:rewrite:virtualize:string:heap"
+    enable_opts = "intbounds:rewrite:virtualize:string:earlyforce:pure:heap"
 
     def optimize_loop(self, ops, optops, call_pure_results=None):
 
@@ -651,8 +652,8 @@
         i3 = getfield_gc(p3, descr=valuedescr)
         escape(i3)
         p1 = new_with_vtable(ConstClass(node_vtable))
+        p1sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p1, i1, descr=valuedescr)
-        p1sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p1sub, i1, descr=valuedescr)
         setfield_gc(p1, p1sub, descr=nextdescr)
         jump(i1, p1, p2)
@@ -667,10 +668,10 @@
         p3sub = getfield_gc(p3, descr=nextdescr)
         i3 = getfield_gc(p3sub, descr=valuedescr)
         escape(i3)
+        p1 = new_with_vtable(ConstClass(node_vtable))
         p2sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p2sub, i1, descr=valuedescr)
         setfield_gc(p2, p2sub, descr=nextdescr)
-        p1 = new_with_vtable(ConstClass(node_vtable))
         jump(i1, p1, p2)
         """
         # The same as test_p123_simple, but in the end the "old" p2 contains
@@ -2328,7 +2329,7 @@
         def _variables_equal(box, varname, strict):
             if varname not in virtuals:
                 if strict:
-                    assert box == oparse.getvar(varname)
+                    assert box.same_box(oparse.getvar(varname))
                 else:
                     assert box.value == oparse.getvar(varname).value
             else:
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizefficall.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizefficall.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizefficall.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizefficall.py
@@ -36,7 +36,7 @@
 
 class TestFfiCall(BaseTestBasic, LLtypeMixin):
 
-    enable_opts = "intbounds:rewrite:virtualize:string:heap:ffi"
+    enable_opts = "intbounds:rewrite:virtualize:string:pure:earlyforce:heap:ffi"
 
     class namespace:
         cpu = LLtypeMixin.cpu
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -68,7 +68,7 @@
 
 class BaseTestWithUnroll(BaseTest):
 
-    enable_opts = "intbounds:rewrite:virtualize:string:heap:unroll"
+    enable_opts = "intbounds:rewrite:virtualize:string:earlyforce:pure:heap:unroll"
 
     def optimize_loop(self, ops, expected, expected_preamble=None,
                       call_pure_results=None, expected_short=None):
@@ -234,6 +234,30 @@
             """ % expected_value
             self.optimize_loop(ops, expected)
 
+    def test_reverse_of_cast(self):
+        ops = """
+        [i0]
+        p0 = cast_int_to_ptr(i0)
+        i1 = cast_ptr_to_int(p0)
+        jump(i1)
+        """
+        expected = """
+        [i0]
+        jump(i0)
+        """
+        self.optimize_loop(ops, expected)
+        ops = """
+        [p0]
+        i1 = cast_ptr_to_int(p0)
+        p1 = cast_int_to_ptr(i1)
+        jump(p1)
+        """
+        expected = """
+        [p0]
+        jump(p0)
+        """
+        self.optimize_loop(ops, expected)
+
     # ----------
 
     def test_remove_guard_class_1(self):
@@ -825,8 +849,8 @@
         i3 = getfield_gc(p2, descr=valuedescr)
         escape(i3)
         p4 = new_with_vtable(ConstClass(node_vtable))
+        p1sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p4, i1, descr=valuedescr)
-        p1sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p1sub, i1, descr=valuedescr)
         setfield_gc(p4, p1sub, descr=nextdescr)
         jump(i1, p4)
@@ -865,13 +889,7 @@
         p3sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p3sub, i1, descr=valuedescr)
         setfield_gc(p1, p3sub, descr=nextdescr)
-        # XXX: We get two extra operations here because the setfield
-        #      above is the result of forcing p1 and thus not
-        #      registered with the heap optimizer. I've makred tests
-        #      below with VIRTUALHEAP if they suffer from this issue
-        p3sub2 = getfield_gc(p1, descr=nextdescr)
-        guard_nonnull_class(p3sub2, ConstClass(node_vtable2)) []
-        jump(i1, p1, p3sub2)
+        jump(i1, p1, p3sub)
         """
         self.optimize_loop(ops, expected, preamble)
 
@@ -902,9 +920,7 @@
         guard_true(i2b) []
         p3 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3, i2, descr=nextdescr)
-        # XXX: VIRTUALHEAP (see above)
-        i3 = getfield_gc(p3, descr=nextdescr)
-        jump(p3, i3)
+        jump(p3, i2)
         """
         self.optimize_loop(ops, expected, preamble)
 
@@ -1219,7 +1235,15 @@
         setfield_gc(p3, p30, descr=valuedescr)
         jump(i29, p30, p3)
         """
-        expected = preamble
+        expected = """
+        [i0, p1, p3]
+        i28 = int_add(i0, 1)
+        i29 = int_add(i28, 1)
+        p30 = new_with_vtable(ConstClass(node_vtable))
+        setfield_gc(p3, p30, descr=valuedescr)
+        setfield_gc(p30, i28, descr=nextdescr)
+        jump(i29, p30, p3)
+        """
         self.optimize_loop(ops, expected, preamble)
 
     def test_nonvirtual_1(self):
@@ -2408,8 +2432,8 @@
         guard_class(p2, ConstClass(node_vtable)) []
         p3 = getfield_gc(p1, descr=otherdescr)
         guard_class(p3, ConstClass(node_vtable)) []
+        p3a = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3, p2, descr=otherdescr)
-        p3a = new_with_vtable(ConstClass(node_vtable))
         escape(p3a)
         jump(p3a)
         """
@@ -2421,9 +2445,9 @@
         # setfield_gc(p3, p2, descr=otherdescr) # p3a.other = p2a
         # p1a = new_with_vtable(ConstClass(node_vtable2))
         # p2a = new_with_vtable(ConstClass(node_vtable))
+        p3anew = new_with_vtable(ConstClass(node_vtable))
         p2 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3a, p2, descr=otherdescr) # p3a.other = p2a
-        p3anew = new_with_vtable(ConstClass(node_vtable))
         escape(p3anew)
         jump(p3anew)
         """
@@ -2458,9 +2482,9 @@
         p3 = getfield_gc(p1, descr=otherdescr)
         guard_class(p3, ConstClass(node_vtable)) []
         # p1a = new_with_vtable(ConstClass(node_vtable2))
+        p3a = new_with_vtable(ConstClass(node_vtable))
         p2a = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3, p2a, descr=otherdescr)
-        p3a = new_with_vtable(ConstClass(node_vtable))
         escape(p3a)
         # setfield_gc(p1a, p2a, descr=nextdescr)
         # setfield_gc(p1a, p3a, descr=otherdescr)
@@ -2468,9 +2492,9 @@
         """
         expected = """
         [p2, p3]
+        p3a = new_with_vtable(ConstClass(node_vtable))
         p2a = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3, p2a, descr=otherdescr)
-        p3a = new_with_vtable(ConstClass(node_vtable))
         escape(p3a)
         jump(p2a, p3a)
         """
@@ -2790,7 +2814,6 @@
         self.optimize_loop(ops, expected, preamble)
 
     def test_remove_duplicate_pure_op_ovf_with_lazy_setfield(self):
-        py.test.skip('this optimization is not yet supprted')
         ops = """
         [i1, p1]
         i3 = int_add_ovf(i1, 1)
@@ -5961,13 +5984,18 @@
         escape(i0)
         jump(p1)
         """
-        expected = """
+        preamble = """
         [p1]
         i0 = ptr_eq(p1, NULL)
         escape(i0)
-        jump(p1)
-        """
-        self.optimize_strunicode_loop_extradescrs(ops, expected, expected)
+        jump(p1, i0)
+        """
+        expected = """
+        [p1, i0]
+        escape(i0)
+        jump(p1, i0)
+        """
+        self.optimize_strunicode_loop_extradescrs(ops, expected, preamble)
 
     def test_str_equal_none2(self):
         ops = """
@@ -5976,13 +6004,18 @@
         escape(i0)
         jump(p1)
         """
-        expected = """
+        preamble = """
         [p1]
         i0 = ptr_eq(p1, NULL)
         escape(i0)
-        jump(p1)
-        """
-        self.optimize_strunicode_loop_extradescrs(ops, expected, expected)
+        jump(p1, i0)
+        """
+        expected = """
+        [p1, i0]
+        escape(i0)
+        jump(p1, i0)
+        """
+        self.optimize_strunicode_loop_extradescrs(ops, expected, preamble)
 
     def test_str_equal_nonnull1(self):
         ops = """
@@ -7220,6 +7253,60 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_heap_cache_forced_virtuals(self):
+        ops = """
+        [i1, i2, p0]
+        p1 = new(descr=ssize)
+        setfield_gc(p1, i1, descr=adescr)
+        setfield_gc(p1, i2, descr=bdescr)
+        call(p0, p1, descr=writeadescr)
+        i3 = getfield_gc(p1, descr=adescr)
+        i4 = getfield_gc(p1, descr=bdescr)
+        jump(i3, i4, p0)
+        """
+        expected = """
+        [i1, i2, p0]
+        p1 = new(descr=ssize)
+        setfield_gc(p1, i1, descr=adescr)
+        call(p0, p1, descr=writeadescr)
+        i3 = getfield_gc(p1, descr=adescr)
+        setfield_gc(p1, i2, descr=bdescr)
+        jump(i3, i2, p0)
+        """
+        self.optimize_loop(ops, expected)
+
+
+    def test_setarrayitem_followed_by_arraycopy(self):
+        ops = """
+        [p1, p2]
+        setarrayitem_gc(p1, 2, 10, descr=arraydescr)
+        setarrayitem_gc(p2, 3, 13, descr=arraydescr)        
+        call(0, p1, p2, 0, 0, 10, descr=arraycopydescr)
+        jump(p1, p2)
+        """
+        self.optimize_loop(ops, ops)
+
+    def test_heap_cache_virtuals_forced_by_delayed_setfield(self):
+        py.test.skip('not yet supoprted')
+        ops = """
+        [i1, p0]
+        p1 = new(descr=ssize)
+        setfield_gc(p1, i1, descr=valuedescr)
+        setfield_gc(p0, p1, descr=adescr)
+        call(p0, descr=writeadescr)
+        i2 = getfield_gc(p1, descr=valuedescr)
+        jump(i2, p0)
+        """
+        expected = """
+        [i1, p0]
+        p1 = new(descr=ssize)
+        setfield_gc(p1, i1, descr=valuedescr)
+        setfield_gc(p0, p1, descr=adescr)
+        call(p0, descr=writeadescr)
+        jump(i1, p0)
+        """
+        self.optimize_loop(ops, expected)
+
 class TestLLtype(OptimizeOptTest, LLtypeMixin):
     pass
 
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_util.py b/pypy/jit/metainterp/optimizeopt/test/test_util.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_util.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_util.py
@@ -182,7 +182,8 @@
                             EffectInfo.EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE,
                             can_invalidate=True))
     arraycopydescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT,
-             EffectInfo([], [], [], [], oopspecindex=EffectInfo.OS_ARRAYCOPY))
+             EffectInfo([], [arraydescr], [], [arraydescr],
+                        oopspecindex=EffectInfo.OS_ARRAYCOPY))
 
     for _name, _os in [
         ('strconcatdescr',               'OS_STR_CONCAT'),
diff --git a/pypy/jit/metainterp/optimizeopt/unroll.py b/pypy/jit/metainterp/optimizeopt/unroll.py
--- a/pypy/jit/metainterp/optimizeopt/unroll.py
+++ b/pypy/jit/metainterp/optimizeopt/unroll.py
@@ -75,7 +75,6 @@
         self.importable_values = {}
         self.emitting_dissabled = False
         self.emitted_guards = 0
-        self.emitted_pure_operations = {}
 
     def ensure_imported(self, value):
         if not self.emitting_dissabled and value in self.importable_values:
@@ -96,21 +95,6 @@
         new = UnrollableOptimizer(self.metainterp_sd, self.loop)
         return self._new(new)
 
-    def remember_emitting_pure(self, op):
-        self.emitted_pure_operations[op] = True
-
-    def produce_potential_short_preamble_ops(self, sb):
-        for op in self.emitted_pure_operations:
-            if op.getopnum() == rop.GETARRAYITEM_GC_PURE or \
-               op.getopnum() == rop.STRGETITEM or \
-               op.getopnum() == rop.UNICODEGETITEM:
-                if not self.getvalue(op.getarg(1)).is_constant():
-                    continue
-            sb.add_potential(op)
-        for opt in self.optimizations:
-            opt.produce_potential_short_preamble_ops(sb)
-
-
 
 class UnrollOptimizer(Optimization):
     """Unroll the loop into two iterations. The first one will
@@ -154,7 +138,7 @@
 
             KillHugeIntBounds(self.optimizer).apply()
             
-            loop.preamble.operations = self.optimizer.newoperations
+            loop.preamble.operations = self.optimizer.get_newoperations()
             jump_args = [self.getvalue(a).get_key_box() for a in jump_args]
 
             start_resumedescr = loop.preamble.start_resumedescr.clone_if_mutable()
@@ -167,8 +151,9 @@
             virtual_state = modifier.get_virtual_state(jump_args)
             
             values = [self.getvalue(arg) for arg in jump_args]
-            inputargs = virtual_state.make_inputargs(values)
-            short_inputargs = virtual_state.make_inputargs(values, keyboxes=True)
+            inputargs = virtual_state.make_inputargs(values, self.optimizer)
+            short_inputargs = virtual_state.make_inputargs(values, self.optimizer,
+                                                           keyboxes=True)
 
             self.constant_inputargs = {}
             for box in jump_args: 
@@ -197,7 +182,7 @@
             # operations needed to setup the proper state of those virtuals
             # in the peeled loop
             inputarg_setup_ops = []
-            preamble_optimizer.newoperations = []
+            preamble_optimizer.clear_newoperations()
             seen = {}
             for box in inputargs:
                 if box in seen:
@@ -211,9 +196,8 @@
                     continue
                 seen[box] = True
                 value = preamble_optimizer.getvalue(box)
-                value.force_box()
-            preamble_optimizer.flush()
-            inputarg_setup_ops += preamble_optimizer.newoperations
+                value.force_box(preamble_optimizer)
+            inputarg_setup_ops += preamble_optimizer.get_newoperations()
 
             # Setup the state of the new optimizer by emiting the
             # short preamble operations and discarding the result
@@ -244,13 +228,13 @@
                                 virtual_state)
             
             loop.inputargs = inputargs
-            args = [preamble_optimizer.getvalue(self.short_boxes.original(a)).force_box()\
+            args = [preamble_optimizer.getvalue(self.short_boxes.original(a)).force_box(preamble_optimizer)\
                     for a in inputargs]
             jmp = ResOperation(rop.JUMP, args, None)
             jmp.setdescr(loop.token)
             loop.preamble.operations.append(jmp)
 
-            loop.operations = self.optimizer.newoperations
+            loop.operations = self.optimizer.get_newoperations()
             maxguards = self.optimizer.metainterp_sd.warmrunnerdesc.memory_manager.max_retrace_guards
             
             if self.optimizer.emitted_guards > maxguards:
@@ -335,9 +319,10 @@
         assert jumpop
         original_jumpargs = jumpop.getarglist()[:]
         values = [self.getvalue(arg) for arg in jumpop.getarglist()]
-        jumpargs = virtual_state.make_inputargs(values)
+        jumpargs = virtual_state.make_inputargs(values, self.optimizer)
         jumpop.initarglist(jumpargs)
-        jmp_to_short_args = virtual_state.make_inputargs(values, keyboxes=True)
+        jmp_to_short_args = virtual_state.make_inputargs(values, self.optimizer,
+                                                         keyboxes=True)
         self.short_inliner = Inliner(short_inputargs, jmp_to_short_args)
         
         for box, const in self.constant_inputargs.items():
@@ -347,11 +332,11 @@
             newop = self.short_inliner.inline_op(op)
             self.optimizer.send_extra_operation(newop)
         
-        self.optimizer.flush()
+        newoperations = self.optimizer.get_newoperations()
 
         i = j = 0
-        while i < len(self.optimizer.newoperations) or j < len(jumpargs):
-            if i == len(self.optimizer.newoperations):
+        while i < len(newoperations) or j < len(jumpargs):
+            if i == len(newoperations):
                 while j < len(jumpargs):
                     a = jumpargs[j]
                     if self.optimizer.loop.logops:
@@ -360,7 +345,7 @@
                                     jumpargs, short_seen)
                     j += 1
             else:
-                op = self.optimizer.newoperations[i]
+                op = newoperations[i]
 
                 self.boxes_created_this_iteration[op.result] = True
                 args = op.getarglist()
@@ -375,6 +360,7 @@
                     self.import_box(a, inputargs, short, short_jumpargs,
                                     jumpargs, short_seen)
                 i += 1
+            newoperations = self.optimizer.get_newoperations()
 
         jumpop.initarglist(jumpargs)
         self.optimizer.send_extra_operation(jumpop)
@@ -468,7 +454,7 @@
         inputargs.append(box)
         box = newresult
         if box in self.optimizer.values:
-            box = self.optimizer.values[box].force_box()
+            box = self.optimizer.values[box].force_box(self.optimizer)
         jumpargs.append(box)
         
 
@@ -483,11 +469,6 @@
         if op.getopnum() == rop.JUMP:
             loop_token = op.getdescr()
             assert isinstance(loop_token, LoopToken)
-            # FIXME: Use a tree, similar to the tree formed by the full
-            # preamble and it's bridges, instead of a list to save time and
-            # memory. This should also allow better behaviour in
-            # situations that the is_emittable() chain currently cant
-            # handle and the inlining fails unexpectedly belwo.
             short = loop_token.short_preamble
             if short:
                 args = op.getarglist()
@@ -523,7 +504,7 @@
 
                         values = [self.getvalue(arg)
                                   for arg in op.getarglist()]
-                        args = sh.virtual_state.make_inputargs(values,
+                        args = sh.virtual_state.make_inputargs(values, self.optimizer,
                                                                keyboxes=True)
                         inliner = Inliner(sh.inputargs, args)
                         
diff --git a/pypy/jit/metainterp/optimizeopt/util.py b/pypy/jit/metainterp/optimizeopt/util.py
--- a/pypy/jit/metainterp/optimizeopt/util.py
+++ b/pypy/jit/metainterp/optimizeopt/util.py
@@ -90,14 +90,11 @@
     for i in range(len(args1)):
         arg1 = args1[i]
         arg2 = args2[i]
-        if isinstance(arg1, history.Const):
-            if arg1.__class__ is not arg2.__class__:
+        if arg1 is None:
+            if arg2 is not None:
                 return False
-            if not arg1.same_constant(arg2):
-                return False
-        else:
-            if not arg1 is arg2:
-                return False
+        elif not arg1.same_box(arg2):
+            return False
     return True
 
 def args_hash(args):
@@ -106,10 +103,8 @@
     for arg in args:
         if arg is None:
             y = 17
-        elif isinstance(arg, history.Const):
+        else:
             y = arg._get_hash_()
-        else:
-            y = compute_identity_hash(arg)
         res = intmask((1000003 * res) ^ y)
     return res
 
@@ -145,9 +140,12 @@
         for i in range(op1.numargs()):
             x = op1.getarg(i)
             y = op2.getarg(i)
-            assert x == remap.get(y, y)
+            assert x.same_box(remap.get(y, y))
         if op2.result in remap:
-            assert op1.result == remap[op2.result]
+            if op2.result is None:
+                assert op1.result == remap[op2.result]
+            else:
+                assert op1.result.same_box(remap[op2.result])
         else:
             remap[op2.result] = op1.result
         if op1.getopnum() != rop.JUMP:      # xxx obscure
@@ -156,11 +154,20 @@
             assert len(op1.getfailargs()) == len(op2.getfailargs())
             if strict_fail_args:
                 for x, y in zip(op1.getfailargs(), op2.getfailargs()):
-                    assert x == remap.get(y, y)
+                    if x is None:
+                        assert remap.get(y, y) is None
+                    else:
+                        assert x.same_box(remap.get(y, y))
             else:
                 fail_args1 = set(op1.getfailargs())
                 fail_args2 = set([remap.get(y, y) for y in op2.getfailargs()])
-                assert fail_args1 == fail_args2
+                for x in fail_args1:
+                    for y in fail_args2:
+                        if x.same_box(y):
+                            fail_args2.remove(y)
+                            break
+                    else:
+                        assert False
     assert len(oplist1) == len(oplist2)
     print '-'*totwidth
     return True
diff --git a/pypy/jit/metainterp/optimizeopt/virtualize.py b/pypy/jit/metainterp/optimizeopt/virtualize.py
--- a/pypy/jit/metainterp/optimizeopt/virtualize.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualize.py
@@ -10,13 +10,12 @@
 
 
 class AbstractVirtualValue(optimizer.OptValue):
-    _attrs_ = ('optimizer', 'keybox', 'source_op', '_cached_vinfo')
+    _attrs_ = ('keybox', 'source_op', '_cached_vinfo')
     box = None
     level = optimizer.LEVEL_NONNULL
     _cached_vinfo = None
 
-    def __init__(self, optimizer, keybox, source_op=None):
-        self.optimizer = optimizer
+    def __init__(self, keybox, source_op=None):
         self.keybox = keybox   # only used as a key in dictionaries
         self.source_op = source_op  # the NEW_WITH_VTABLE/NEW_ARRAY operation
                                     # that builds this box
@@ -29,17 +28,17 @@
             return self.keybox
         return self.box
 
-    def force_box(self):
+    def force_box(self, optforce):
         if self.box is None:
-            self.optimizer.forget_numberings(self.keybox)
-            self._really_force()
+            optforce.forget_numberings(self.keybox)
+            self._really_force(optforce)
         return self.box
 
-    def force_at_end_of_preamble(self, already_forced):
+    def force_at_end_of_preamble(self, already_forced, optforce):
         value = already_forced.get(self, None)
         if value:
             return value
-        return OptValue(self.force_box())
+        return OptValue(self.force_box(optforce))
 
     def make_virtual_info(self, modifier, fieldnums):
         if fieldnums is None:
@@ -55,7 +54,7 @@
     def _make_virtual(self, modifier):
         raise NotImplementedError("abstract base")
 
-    def _really_force(self):
+    def _really_force(self, optforce):
         raise NotImplementedError("abstract base")
 
     def import_from(self, other, optimizer):
@@ -70,10 +69,11 @@
 get_fielddescrlist_cache._annspecialcase_ = "specialize:memo"
 
 class AbstractVirtualStructValue(AbstractVirtualValue):
-    _attrs_ = ('_fields', '_cached_sorted_fields')
+    _attrs_ = ('_fields', 'cpu', '_cached_sorted_fields')
 
-    def __init__(self, optimizer, keybox, source_op=None):
-        AbstractVirtualValue.__init__(self, optimizer, keybox, source_op)
+    def __init__(self, cpu, keybox, source_op=None):
+        AbstractVirtualValue.__init__(self, keybox, source_op)
+        self.cpu = cpu
         self._fields = {}
         self._cached_sorted_fields = None
 
@@ -87,45 +87,44 @@
     def _get_descr(self):
         raise NotImplementedError
 
-    def _is_immutable_and_filled_with_constants(self):
+    def _is_immutable_and_filled_with_constants(self, optforce):
         count = self._get_descr().count_fields_if_immutable()
         if count != len(self._fields):    # always the case if count == -1
             return False
         for value in self._fields.itervalues():
-            subbox = value.force_box()
+            subbox = value.force_box(optforce)
             if not isinstance(subbox, Const):
                 return False
         return True
 
-    def force_at_end_of_preamble(self, already_forced):
+    def force_at_end_of_preamble(self, already_forced, optforce):
         if self in already_forced:
             return self
         already_forced[self] = self
         if self._fields:
             for ofs in self._fields.keys():
-                self._fields[ofs] = self._fields[ofs].force_at_end_of_preamble(already_forced)
+                self._fields[ofs] = self._fields[ofs].force_at_end_of_preamble(already_forced, optforce)
         return self
 
-    def _really_force(self):
+    def _really_force(self, optforce):
         op = self.source_op
         assert op is not None
         # ^^^ This case should not occur any more (see test_bug_3).
         #
         if not we_are_translated():
             op.name = 'FORCE ' + self.source_op.name
-
-        if self._is_immutable_and_filled_with_constants():
-            box = self.optimizer.constant_fold(op)
+            
+        if self._is_immutable_and_filled_with_constants(optforce):
+            box = optforce.optimizer.constant_fold(op)
             self.make_constant(box)
             for ofs, value in self._fields.iteritems():
-                subbox = value.force_box()
+                subbox = value.force_box(optforce)
                 assert isinstance(subbox, Const)
-                execute(self.optimizer.cpu, None, rop.SETFIELD_GC,
+                execute(optforce.optimizer.cpu, None, rop.SETFIELD_GC,
                         ofs, box, subbox)
             # keep self._fields, because it's all immutable anyway
         else:
-            newoperations = self.optimizer.newoperations
-            newoperations.append(op)
+            optforce.emit_operation(op)
             self.box = box = op.result
             #
             iteritems = self._fields.iteritems()
@@ -135,10 +134,11 @@
             for ofs, value in iteritems:
                 if value.is_null():
                     continue
-                subbox = value.force_box()
+                subbox = value.force_box(optforce)
                 op = ResOperation(rop.SETFIELD_GC, [box, subbox], None,
                                   descr=ofs)
-                newoperations.append(op)
+
+                optforce.emit_operation(op)
 
     def _get_field_descr_list(self):
         _cached_sorted_fields = self._cached_sorted_fields
@@ -155,7 +155,7 @@
             else:
                 lst = self._fields.keys()
             sort_descrs(lst)
-            cache = get_fielddescrlist_cache(self.optimizer.cpu)
+            cache = get_fielddescrlist_cache(self.cpu)
             result = cache.get(lst, None)
             if result is None:
                 cache[lst] = lst
@@ -180,8 +180,8 @@
 class VirtualValue(AbstractVirtualStructValue):
     level = optimizer.LEVEL_KNOWNCLASS
 
-    def __init__(self, optimizer, known_class, keybox, source_op=None):
-        AbstractVirtualStructValue.__init__(self, optimizer, keybox, source_op)
+    def __init__(self, cpu, known_class, keybox, source_op=None):
+        AbstractVirtualStructValue.__init__(self, cpu, keybox, source_op)
         assert isinstance(known_class, Const)
         self.known_class = known_class
 
@@ -190,7 +190,7 @@
         return modifier.make_virtual(self.known_class, fielddescrs)
 
     def _get_descr(self):
-        return vtable2descr(self.optimizer.cpu, self.known_class.getint())
+        return vtable2descr(self.cpu, self.known_class.getint())
 
     def __repr__(self):
         cls_name = self.known_class.value.adr.ptr._obj._TYPE._name
@@ -201,8 +201,8 @@
 
 class VStructValue(AbstractVirtualStructValue):
 
-    def __init__(self, optimizer, structdescr, keybox, source_op=None):
-        AbstractVirtualStructValue.__init__(self, optimizer, keybox, source_op)
+    def __init__(self, cpu, structdescr, keybox, source_op=None):
+        AbstractVirtualStructValue.__init__(self, cpu, keybox, source_op)
         self.structdescr = structdescr
 
     def _make_virtual(self, modifier):
@@ -215,10 +215,10 @@
 
 class VArrayValue(AbstractVirtualValue):
 
-    def __init__(self, optimizer, arraydescr, size, keybox, source_op=None):
-        AbstractVirtualValue.__init__(self, optimizer, keybox, source_op)
+    def __init__(self, arraydescr, constvalue, size, keybox, source_op=None):
+        AbstractVirtualValue.__init__(self, keybox, source_op)
         self.arraydescr = arraydescr
-        self.constvalue = optimizer.new_const_item(arraydescr)
+        self.constvalue = constvalue
         self._items = [self.constvalue] * size
 
     def getlength(self):
@@ -232,31 +232,30 @@
         assert isinstance(itemvalue, optimizer.OptValue)
         self._items[index] = itemvalue
 
-    def force_at_end_of_preamble(self, already_forced):
+    def force_at_end_of_preamble(self, already_forced, optforce):
         if self in already_forced:
             return self
         already_forced[self] = self
         for index in range(len(self._items)):
-            self._items[index] = self._items[index].force_at_end_of_preamble(already_forced)
+            self._items[index] = self._items[index].force_at_end_of_preamble(already_forced, optforce)
         return self
     
-    def _really_force(self):
+    def _really_force(self, optforce):
         assert self.source_op is not None
         if not we_are_translated():
             self.source_op.name = 'FORCE ' + self.source_op.name
-        newoperations = self.optimizer.newoperations
-        newoperations.append(self.source_op)
+        optforce.emit_operation(self.source_op)        
         self.box = box = self.source_op.result
         for index in range(len(self._items)):
             subvalue = self._items[index]
             if subvalue is not self.constvalue:
                 if subvalue.is_null():
                     continue
-                subbox = subvalue.force_box()
+                subbox = subvalue.force_box(optforce)
                 op = ResOperation(rop.SETARRAYITEM_GC,
                                   [box, ConstInt(index), subbox], None,
                                   descr=self.arraydescr)
-                newoperations.append(op)
+                optforce.emit_operation(op)
 
     def get_args_for_fail(self, modifier):
         if self.box is None and not modifier.already_seen_virtual(self.keybox):
@@ -279,17 +278,18 @@
         return OptVirtualize()
         
     def make_virtual(self, known_class, box, source_op=None):
-        vvalue = VirtualValue(self.optimizer, known_class, box, source_op)
+        vvalue = VirtualValue(self.optimizer.cpu, known_class, box, source_op)
         self.make_equal_to(box, vvalue)
         return vvalue
 
     def make_varray(self, arraydescr, size, box, source_op=None):
-        vvalue = VArrayValue(self.optimizer, arraydescr, size, box, source_op)
+        constvalue = self.new_const_item(arraydescr)
+        vvalue = VArrayValue(arraydescr, constvalue, size, box, source_op)
         self.make_equal_to(box, vvalue)
         return vvalue
 
     def make_vstruct(self, structdescr, box, source_op=None):
-        vvalue = VStructValue(self.optimizer, structdescr, box, source_op)
+        vvalue = VStructValue(self.optimizer.cpu, structdescr, box, source_op)
         self.make_equal_to(box, vvalue)
         return vvalue
 
@@ -362,7 +362,6 @@
             self.make_equal_to(op.result, fieldvalue)
         else:
             value.ensure_nonnull()
-            ###self.heap_op_optimizer.optimize_GETFIELD_GC(op, value)
             self.emit_operation(op)
 
     # note: the following line does not mean that the two operations are
@@ -377,7 +376,6 @@
             value.setfield(op.getdescr(), fieldvalue)
         else:
             value.ensure_nonnull()
-            ###self.heap_op_optimizer.optimize_SETFIELD_GC(op, value, fieldvalue)
             self.emit_operation(op)
 
     def optimize_NEW_WITH_VTABLE(self, op):
@@ -417,7 +415,6 @@
                 self.make_equal_to(op.result, itemvalue)
                 return
         value.ensure_nonnull()
-        ###self.heap_op_optimizer.optimize_GETARRAYITEM_GC(op, value)
         self.emit_operation(op)
 
     # note: the following line does not mean that the two operations are
@@ -432,7 +429,6 @@
                 value.setitem(indexbox.getint(), self.getvalue(op.getarg(2)))
                 return
         value.ensure_nonnull()
-        ###self.heap_op_optimizer.optimize_SETARRAYITEM_GC(op, value, fieldvalue)
         self.emit_operation(op)
 
 
diff --git a/pypy/jit/metainterp/optimizeopt/virtualstate.py b/pypy/jit/metainterp/optimizeopt/virtualstate.py
--- a/pypy/jit/metainterp/optimizeopt/virtualstate.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualstate.py
@@ -30,7 +30,7 @@
     def _generate_guards(self, other, box, cpu, extra_guards):
         raise InvalidLoop
 
-    def enum_forced_boxes(self, boxes, value):
+    def enum_forced_boxes(self, boxes, value, optimizer):
         raise NotImplementedError
 
     def enum(self, virtual_state):
@@ -100,14 +100,14 @@
     def _generalization_of(self, other):
         raise NotImplementedError
 
-    def enum_forced_boxes(self, boxes, value):
+    def enum_forced_boxes(self, boxes, value, optimizer):
         assert isinstance(value, virtualize.AbstractVirtualStructValue)
         assert value.is_virtual()
         for i in range(len(self.fielddescrs)):
             v = value._fields[self.fielddescrs[i]]
             s = self.fieldstate[i]
             if s.position > self.position:
-                s.enum_forced_boxes(boxes, v)
+                s.enum_forced_boxes(boxes, v, optimizer)
 
     def _enum(self, virtual_state):
         for s in self.fieldstate:
@@ -177,14 +177,14 @@
                 return False
         return True
 
-    def enum_forced_boxes(self, boxes, value):
+    def enum_forced_boxes(self, boxes, value, optimizer):
         assert isinstance(value, virtualize.VArrayValue)
         assert value.is_virtual()
         for i in range(len(self.fieldstate)):
             v = value._items[i]
             s = self.fieldstate[i]
             if s.position > self.position:
-                s.enum_forced_boxes(boxes, v)
+                s.enum_forced_boxes(boxes, v, optimizer)
 
     def _enum(self, virtual_state):
         for s in self.fieldstate:
@@ -316,11 +316,11 @@
             import pdb; pdb.set_trace()
             raise NotImplementedError
 
-    def enum_forced_boxes(self, boxes, value):
+    def enum_forced_boxes(self, boxes, value, optimizer):
         if self.level == LEVEL_CONSTANT:
             return
         assert 0 <= self.position_in_notvirtuals 
-        boxes[self.position_in_notvirtuals] = value.force_box()
+        boxes[self.position_in_notvirtuals] = value.force_box(optimizer)
 
     def _enum(self, virtual_state):
         if self.level == LEVEL_CONSTANT:
@@ -377,11 +377,13 @@
             self.state[i].generate_guards(other.state[i], args[i],
                                           cpu, extra_guards, renum)
 
-    def make_inputargs(self, values, keyboxes=False):
+    def make_inputargs(self, values, optimizer, keyboxes=False):
+        if optimizer.optearlyforce:
+            optimizer = optimizer.optearlyforce
         assert len(values) == len(self.state)
         inputargs = [None] * len(self.notvirtuals)
         for i in range(len(values)):
-            self.state[i].enum_forced_boxes(inputargs, values[i])
+            self.state[i].enum_forced_boxes(inputargs, values[i], optimizer)
 
         if keyboxes:
             for i in range(len(values)):
@@ -434,7 +436,12 @@
     def get_virtual_state(self, jump_args):
         self.optimizer.force_at_end_of_preamble()
         already_forced = {}
-        values = [self.getvalue(box).force_at_end_of_preamble(already_forced)
+        if self.optimizer.optearlyforce:
+            opt = self.optimizer.optearlyforce
+        else:
+            opt = self.optimizer
+        values = [self.getvalue(box).force_at_end_of_preamble(already_forced,
+                                                              opt)
                   for box in jump_args]
 
         for value in values:
diff --git a/pypy/jit/metainterp/optimizeopt/vstring.py b/pypy/jit/metainterp/optimizeopt/vstring.py
--- a/pypy/jit/metainterp/optimizeopt/vstring.py
+++ b/pypy/jit/metainterp/optimizeopt/vstring.py
@@ -43,7 +43,7 @@
 class __extend__(optimizer.OptValue):
     """New methods added to the base class OptValue for this file."""
 
-    def getstrlen(self, optimization, mode):
+    def getstrlen(self, string_optimizer, mode):
         if mode is mode_string:
             s = self.get_constant_string_spec(mode_string)
             if s is not None:
@@ -52,12 +52,12 @@
             s = self.get_constant_string_spec(mode_unicode)
             if s is not None:
                 return ConstInt(len(s))
-        if optimization is None:
+        if string_optimizer is None:
             return None
         self.ensure_nonnull()
-        box = self.force_box()
+        box = self.force_box(string_optimizer)
         lengthbox = BoxInt()
-        optimization.propagate_forward(ResOperation(mode.STRLEN, [box], lengthbox))
+        string_optimizer.emit_operation(ResOperation(mode.STRLEN, [box], lengthbox))
         return lengthbox
 
     @specialize.arg(1)
@@ -68,25 +68,25 @@
         else:
             return None
 
-    def string_copy_parts(self, optimizer, targetbox, offsetbox, mode):
+    def string_copy_parts(self, string_optimizer, targetbox, offsetbox, mode):
         # Copies the pointer-to-string 'self' into the target string
         # given by 'targetbox', at the specified offset.  Returns the offset
         # at the end of the copy.
-        lengthbox = self.getstrlen(optimizer, mode)
-        srcbox = self.force_box()
-        return copy_str_content(optimizer, srcbox, targetbox,
+        lengthbox = self.getstrlen(string_optimizer, mode)
+        srcbox = self.force_box(string_optimizer)
+        return copy_str_content(string_optimizer, srcbox, targetbox,
                                 CONST_0, offsetbox, lengthbox, mode)
 
 
 class VAbstractStringValue(virtualize.AbstractVirtualValue):
     _attrs_ = ('mode',)
 
-    def __init__(self, optimizer, keybox, source_op, mode):
-        virtualize.AbstractVirtualValue.__init__(self, optimizer, keybox,
+    def __init__(self, keybox, source_op, mode):
+        virtualize.AbstractVirtualValue.__init__(self, keybox,
                                                  source_op)
         self.mode = mode
 
-    def _really_force(self):
+    def _really_force(self, optforce):
         if self.mode is mode_string:
             s = self.get_constant_string_spec(mode_string)
             if s is not None:
@@ -101,12 +101,12 @@
                 return
         assert self.source_op is not None
         self.box = box = self.source_op.result
-        lengthbox = self.getstrlen(self.optimizer, self.mode)
+        lengthbox = self.getstrlen(optforce, self.mode)
         op = ResOperation(self.mode.NEWSTR, [lengthbox], box)
         if not we_are_translated():
             op.name = 'FORCE'
-        self.optimizer.emit_operation(op)
-        self.string_copy_parts(self.optimizer, box, CONST_0, self.mode)
+        optforce.emit_operation(op)
+        self.string_copy_parts(optforce, box, CONST_0, self.mode)
 
 
 class VStringPlainValue(VAbstractStringValue):
@@ -140,20 +140,20 @@
         return mode.emptystr.join([mode.chr(c.box.getint())
                                    for c in self._chars])
 
-    def string_copy_parts(self, optimizer, targetbox, offsetbox, mode):
+    def string_copy_parts(self, string_optimizer, targetbox, offsetbox, mode):
         if not self.is_virtual() and targetbox is not self.box:
-            lengthbox = self.getstrlen(optimizer, mode)
-            srcbox = self.force_box()
-            return copy_str_content(optimizer, srcbox, targetbox,
+            lengthbox = self.getstrlen(string_optimizer, mode)
+            srcbox = self.force_box(string_optimizer)
+            return copy_str_content(string_optimizer, srcbox, targetbox,
                                 CONST_0, offsetbox, lengthbox, mode)
         for i in range(len(self._chars)):
-            charbox = self._chars[i].force_box()
+            charbox = self._chars[i].force_box(string_optimizer)
             if not (isinstance(charbox, Const) and charbox.same_constant(CONST_0)):
-                optimizer.emit_operation(ResOperation(mode.STRSETITEM, [targetbox,
-                                                                    offsetbox,
-                                                                    charbox],
+                string_optimizer.emit_operation(ResOperation(mode.STRSETITEM, [targetbox,
+                                                                               offsetbox,
+                                                                               charbox],
                                                   None))
-            offsetbox = _int_add(optimizer, offsetbox, CONST_1)
+            offsetbox = _int_add(string_optimizer, offsetbox, CONST_1)
         return offsetbox
 
     def get_args_for_fail(self, modifier):
@@ -187,16 +187,16 @@
         self.left = left
         self.right = right
 
-    def getstrlen(self, optimizer, mode):
+    def getstrlen(self, string_optimizer, mode):
         if self.lengthbox is None:
-            len1box = self.left.getstrlen(optimizer, mode)
+            len1box = self.left.getstrlen(string_optimizer, mode)
             if len1box is None:
                 return None
-            len2box = self.right.getstrlen(optimizer, mode)
+            len2box = self.right.getstrlen(string_optimizer, mode)
             if len2box is None:
                 return None
-            self.lengthbox = _int_add(optimizer, len1box, len2box)
-            # ^^^ may still be None, if optimizer is None
+            self.lengthbox = _int_add(string_optimizer, len1box, len2box)
+            # ^^^ may still be None, if string_optimizer is None
         return self.lengthbox
 
     @specialize.arg(1)
@@ -209,10 +209,10 @@
             return None
         return s1 + s2
 
-    def string_copy_parts(self, optimizer, targetbox, offsetbox, mode):
-        offsetbox = self.left.string_copy_parts(optimizer, targetbox,
+    def string_copy_parts(self, string_optimizer, targetbox, offsetbox, mode):
+        offsetbox = self.left.string_copy_parts(string_optimizer, targetbox,
                                                 offsetbox, mode)
-        offsetbox = self.right.string_copy_parts(optimizer, targetbox,
+        offsetbox = self.right.string_copy_parts(string_optimizer, targetbox,
                                                  offsetbox, mode)
         return offsetbox
 
@@ -251,8 +251,8 @@
         self.vstart = vstart
         self.vlength = vlength
 
-    def getstrlen(self, _, mode):
-        return self.vlength.force_box()
+    def getstrlen(self, optforce, mode):
+        return self.vlength.force_box(optforce)
 
     @specialize.arg(1)
     def get_constant_string_spec(self, mode):
@@ -267,11 +267,11 @@
             return s1[start : start + length]
         return None
 
-    def string_copy_parts(self, optimizer, targetbox, offsetbox, mode):
-        lengthbox = self.getstrlen(optimizer, mode)
-        return copy_str_content(optimizer,
-                                self.vstr.force_box(), targetbox,
-                                self.vstart.force_box(), offsetbox,
+    def string_copy_parts(self, string_optimizer, targetbox, offsetbox, mode):
+        lengthbox = self.getstrlen(string_optimizer, mode)
+        return copy_str_content(string_optimizer,
+                                self.vstr.force_box(string_optimizer), targetbox,
+                                self.vstart.force_box(string_optimizer), offsetbox,
                                 lengthbox, mode)
 
     def get_args_for_fail(self, modifier):
@@ -300,7 +300,7 @@
         return modifier.make_vstrslice(self.mode is mode_unicode)
 
 
-def copy_str_content(optimizer, srcbox, targetbox,
+def copy_str_content(string_optimizer, srcbox, targetbox,
                      srcoffsetbox, offsetbox, lengthbox, mode, need_next_offset=True):
     if isinstance(srcbox, ConstPtr) and isinstance(srcoffsetbox, Const):
         M = 5
@@ -310,26 +310,26 @@
         # up to M characters are done "inline", i.e. with STRGETITEM/STRSETITEM
         # instead of just a COPYSTRCONTENT.
         for i in range(lengthbox.value):
-            charbox = _strgetitem(optimizer, srcbox, srcoffsetbox, mode)
-            srcoffsetbox = _int_add(optimizer, srcoffsetbox, CONST_1)
-            optimizer.emit_operation(ResOperation(mode.STRSETITEM, [targetbox,
-                                                                offsetbox,
-                                                                charbox],
+            charbox = _strgetitem(string_optimizer, srcbox, srcoffsetbox, mode)
+            srcoffsetbox = _int_add(string_optimizer, srcoffsetbox, CONST_1)
+            string_optimizer.emit_operation(ResOperation(mode.STRSETITEM, [targetbox,
+                                                                           offsetbox,
+                                                                           charbox],
                                               None))
-            offsetbox = _int_add(optimizer, offsetbox, CONST_1)
+            offsetbox = _int_add(string_optimizer, offsetbox, CONST_1)
     else:
         if need_next_offset:
-            nextoffsetbox = _int_add(optimizer, offsetbox, lengthbox)
+            nextoffsetbox = _int_add(string_optimizer, offsetbox, lengthbox)
         else:
             nextoffsetbox = None
         op = ResOperation(mode.COPYSTRCONTENT, [srcbox, targetbox,
                                                 srcoffsetbox, offsetbox,
                                                 lengthbox], None)
-        optimizer.emit_operation(op)
+        string_optimizer.emit_operation(op)
         offsetbox = nextoffsetbox
     return offsetbox
 
-def _int_add(optimizer, box1, box2):
+def _int_add(string_optimizer, box1, box2):
     if isinstance(box1, ConstInt):
         if box1.value == 0:
             return box2
@@ -337,23 +337,23 @@
             return ConstInt(box1.value + box2.value)
     elif isinstance(box2, ConstInt) and box2.value == 0:
         return box1
-    if optimizer is None:
+    if string_optimizer is None:
         return None
     resbox = BoxInt()
-    optimizer.propagate_forward(ResOperation(rop.INT_ADD, [box1, box2], resbox))
+    string_optimizer.emit_operation(ResOperation(rop.INT_ADD, [box1, box2], resbox))
     return resbox
 
-def _int_sub(optimizer, box1, box2):
+def _int_sub(string_optimizer, box1, box2):
     if isinstance(box2, ConstInt):
         if box2.value == 0:
             return box1
         if isinstance(box1, ConstInt):
             return ConstInt(box1.value - box2.value)
     resbox = BoxInt()
-    optimizer.propagate_forward(ResOperation(rop.INT_SUB, [box1, box2], resbox))
+    string_optimizer.emit_operation(ResOperation(rop.INT_SUB, [box1, box2], resbox))
     return resbox
 
-def _strgetitem(optimizer, strbox, indexbox, mode):
+def _strgetitem(string_optimizer, strbox, indexbox, mode):
     if isinstance(strbox, ConstPtr) and isinstance(indexbox, ConstInt):
         if mode is mode_string:
             s = strbox.getref(lltype.Ptr(rstr.STR))
@@ -362,30 +362,28 @@
             s = strbox.getref(lltype.Ptr(rstr.UNICODE))
             return ConstInt(ord(s.chars[indexbox.getint()]))
     resbox = BoxInt()
-    optimizer.propagate_forward(ResOperation(mode.STRGETITEM, [strbox, indexbox],
-                                      resbox))
+    string_optimizer.emit_operation(ResOperation(mode.STRGETITEM, [strbox, indexbox],
+                                                 resbox))
     return resbox
 
 
 class OptString(optimizer.Optimization):
     "Handling of strings and unicodes."
-    enabled = True
-
     def new(self):
         return OptString()
 
     def make_vstring_plain(self, box, source_op, mode):
-        vvalue = VStringPlainValue(self.optimizer, box, source_op, mode)
+        vvalue = VStringPlainValue(box, source_op, mode)
         self.make_equal_to(box, vvalue)
         return vvalue
 
     def make_vstring_concat(self, box, source_op, mode):
-        vvalue = VStringConcatValue(self.optimizer, box, source_op, mode)
+        vvalue = VStringConcatValue(box, source_op, mode)
         self.make_equal_to(box, vvalue)
         return vvalue
 
     def make_vstring_slice(self, box, source_op, mode):
-        vvalue = VStringSliceValue(self.optimizer, box, source_op, mode)
+        vvalue = VStringSliceValue(box, source_op, mode)
         self.make_equal_to(box, vvalue)
         return vvalue
 
@@ -435,9 +433,9 @@
         value.ensure_nonnull()
         #
         if value.is_virtual() and isinstance(value, VStringSliceValue):
-            fullindexbox = _int_add(self.optimizer,
-                                    value.vstart.force_box(),
-                                    vindex.force_box())
+            fullindexbox = _int_add(self,
+                                    value.vstart.force_box(self),
+                                    vindex.force_box(self))
             value = value.vstr
             vindex = self.getvalue(fullindexbox)
         #
@@ -449,7 +447,7 @@
                 if res is not optimizer.CVAL_UNINITIALIZED_ZERO:
                     return res
         #
-        resbox = _strgetitem(self.optimizer, value.force_box(), vindex.force_box(), mode)
+        resbox = _strgetitem(self, value.force_box(self), vindex.force_box(self), mode)
         return self.getvalue(resbox)
 
     def optimize_STRLEN(self, op):
@@ -459,7 +457,7 @@
 
     def _optimize_STRLEN(self, op, mode):
         value = self.getvalue(op.getarg(0))
-        lengthbox = value.getstrlen(self.optimizer, mode)
+        lengthbox = value.getstrlen(self, mode)
         self.make_equal_to(op.result, self.getvalue(lengthbox))
 
     def optimize_COPYSTRCONTENT(self, op):
@@ -477,12 +475,12 @@
 
         if length.is_constant() and length.box.getint() == 0:
             return
-        copy_str_content(self.optimizer,
-            src.force_box(),
-            dst.force_box(),
-            srcstart.force_box(),
-            dststart.force_box(),
-            length.force_box(),
+        copy_str_content(self,
+            src.force_box(self),
+            dst.force_box(self),
+            srcstart.force_box(self),
+            dststart.force_box(self),
+            length.force_box(self),
             mode, need_next_offset=False
         )
 
@@ -508,6 +506,8 @@
                     return
         self.emit_operation(op)
 
+    optimize_CALL_PURE = optimize_CALL
+
     def opt_call_str_STR2UNICODE(self, op):
         # Constant-fold unicode("constant string").
         # More generally, supporting non-constant but virtual cases is
@@ -547,16 +547,16 @@
             return True
         #
         vstr.ensure_nonnull()
-        lengthbox = _int_sub(self.optimizer, vstop.force_box(),
-                                            vstart.force_box())
+        lengthbox = _int_sub(self, vstop.force_box(self),
+                                   vstart.force_box(self))
         #
         if isinstance(vstr, VStringSliceValue):
             # double slicing  s[i:j][k:l]
             vintermediate = vstr
             vstr = vintermediate.vstr
-            startbox = _int_add(self.optimizer,
-                                vintermediate.vstart.force_box(),
-                                vstart.force_box())
+            startbox = _int_add(self,
+                                vintermediate.vstart.force_box(self),
+                                vstart.force_box(self))
             vstart = self.getvalue(startbox)
         #
         value = self.make_vstring_slice(op.result, op, mode)
@@ -587,15 +587,12 @@
             return True
         #
         if v1.is_nonnull() and v2.is_nonnull():
-            if l1box is not None and l2box is not None and (
-                l1box == l2box or (isinstance(l1box, ConstInt) and
-                                   isinstance(l2box, ConstInt) and
-                                   l1box.value == l2box.value)):
+            if l1box is not None and l2box is not None and l1box.same_box(l2box):
                 do = EffectInfo.OS_STREQ_LENGTHOK
             else:
                 do = EffectInfo.OS_STREQ_NONNULL
-            self.generate_modified_call(do, [v1.force_box(),
-                                             v2.force_box()], op.result, mode)
+            self.generate_modified_call(do, [v1.force_box(self),
+                                             v2.force_box(self)], op.result, mode)
             return True
         return False
 
@@ -603,7 +600,7 @@
         l2box = v2.getstrlen(None, mode)
         if isinstance(l2box, ConstInt):
             if l2box.value == 0:
-                lengthbox = v1.getstrlen(self.optimizer, mode)
+                lengthbox = v1.getstrlen(self, mode)
                 seo = self.optimizer.send_extra_operation
                 seo(ResOperation(rop.INT_EQ, [lengthbox, CONST_0], resultbox))
                 return True
@@ -614,17 +611,17 @@
                     vchar1 = self.strgetitem(v1, optimizer.CVAL_ZERO, mode)
                     vchar2 = self.strgetitem(v2, optimizer.CVAL_ZERO, mode)
                     seo = self.optimizer.send_extra_operation
-                    seo(ResOperation(rop.INT_EQ, [vchar1.force_box(),
-                                                  vchar2.force_box()],
+                    seo(ResOperation(rop.INT_EQ, [vchar1.force_box(self),
+                                                  vchar2.force_box(self)],
                                      resultbox))
                     return True
                 if isinstance(v1, VStringSliceValue):
                     vchar = self.strgetitem(v2, optimizer.CVAL_ZERO, mode)
                     do = EffectInfo.OS_STREQ_SLICE_CHAR
-                    self.generate_modified_call(do, [v1.vstr.force_box(),
-                                                     v1.vstart.force_box(),
-                                                     v1.vlength.force_box(),
-                                                     vchar.force_box()],
+                    self.generate_modified_call(do, [v1.vstr.force_box(self),
+                                                     v1.vstart.force_box(self),
+                                                     v1.vlength.force_box(self),
+                                                     vchar.force_box(self)],
                                                 resultbox, mode)
                     return True
         #
@@ -635,10 +632,10 @@
             if v1.is_null():
                 self.make_constant(resultbox, CONST_1)
                 return True
-            op = ResOperation(rop.PTR_EQ, [v1.force_box(),
+            op = ResOperation(rop.PTR_EQ, [v1.force_box(self),
                                            llhelper.CONST_NULL],
                               resultbox)
-            self.optimizer.emit_operation(op)
+            self.emit_operation(op)
             return True
         #
         return False
@@ -652,8 +649,8 @@
                     do = EffectInfo.OS_STREQ_NONNULL_CHAR
                 else:
                     do = EffectInfo.OS_STREQ_CHECKNULL_CHAR
-                self.generate_modified_call(do, [v1.force_box(),
-                                                 vchar.force_box()], resultbox,
+                self.generate_modified_call(do, [v1.force_box(self),
+                                                 vchar.force_box(self)], resultbox,
                                             mode)
                 return True
         #
@@ -662,10 +659,10 @@
                 do = EffectInfo.OS_STREQ_SLICE_NONNULL
             else:
                 do = EffectInfo.OS_STREQ_SLICE_CHECKNULL
-            self.generate_modified_call(do, [v1.vstr.force_box(),
-                                             v1.vstart.force_box(),
-                                             v1.vlength.force_box(),
-                                             v2.force_box()], resultbox, mode)
+            self.generate_modified_call(do, [v1.vstr.force_box(self),
+                                             v1.vstart.force_box(self),
+                                             v1.vlength.force_box(self),
+                                             v2.force_box(self)], resultbox, mode)
             return True
         return False
 
@@ -675,16 +672,11 @@
         calldescr, func = cic.callinfo_for_oopspec(oopspecindex)
         op = ResOperation(rop.CALL, [ConstInt(func)] + args, result,
                           descr=calldescr)
-        self.optimizer.emit_operation(op)
+        self.emit_operation(op)
 
     def propagate_forward(self, op):
-        if not self.enabled:
-            self.emit_operation(op)
-            return
-
         dispatch_opt(self, op)
 
-
 dispatch_opt = make_dispatcher_method(OptString, 'optimize_',
         default=OptString.emit_operation)
 
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -162,15 +162,18 @@
             if registers[i] is oldbox:
                 registers[i] = newbox
         if not we_are_translated():
-            assert oldbox not in registers[count:]
+            for b in registers[count:]:
+                assert not oldbox.same_box(b)
+                
 
     def make_result_of_lastop(self, resultbox):
         got_type = resultbox.type
-        if not we_are_translated():
-            typeof = {'i': history.INT,
-                      'r': history.REF,
-                      'f': history.FLOAT}
-            assert typeof[self.jitcode._resulttypes[self.pc]] == got_type
+        # XXX disabled for now, conflicts with str_guard_value
+        #if not we_are_translated():
+        #    typeof = {'i': history.INT,
+        #              'r': history.REF,
+        #              'f': history.FLOAT}
+        #    assert typeof[self.jitcode._resulttypes[self.pc]] == got_type
         target_index = ord(self.bytecode[self.pc-1])
         if got_type == history.INT:
             self.registers_i[target_index] = resultbox
@@ -219,6 +222,7 @@
                     'cast_float_to_int', 'cast_int_to_float',
                     'cast_float_to_singlefloat', 'cast_singlefloat_to_float',
                     'float_neg', 'float_abs',
+                    'cast_ptr_to_int', 'cast_int_to_ptr',
                     ]:
         exec py.code.Source('''
             @arguments("box")
@@ -895,6 +899,21 @@
     def _opimpl_guard_value(self, orgpc, box):
         self.implement_guard_value(orgpc, box)
 
+    @arguments("orgpc", "box", "box", "descr")
+    def opimpl_str_guard_value(self, orgpc, box, funcbox, descr):
+        if isinstance(box, Const):
+            return box     # no promotion needed, already a Const
+        else:
+            constbox = box.constbox()
+            resbox = self.do_residual_call(funcbox, descr, [box, constbox])
+            promoted_box = resbox.constbox()
+            # This is GUARD_VALUE because GUARD_TRUE assumes the existance
+            # of a label when computing resumepc
+            self.generate_guard(rop.GUARD_VALUE, resbox, [promoted_box],
+                                resumepc=orgpc)
+            self.metainterp.replace_box(box, constbox)
+            return constbox
+
     opimpl_int_guard_value = _opimpl_guard_value
     opimpl_ref_guard_value = _opimpl_guard_value
     opimpl_float_guard_value = _opimpl_guard_value
diff --git a/pypy/jit/metainterp/quasiimmut.py b/pypy/jit/metainterp/quasiimmut.py
--- a/pypy/jit/metainterp/quasiimmut.py
+++ b/pypy/jit/metainterp/quasiimmut.py
@@ -2,6 +2,7 @@
 from pypy.rpython.lltypesystem import lltype, rclass
 from pypy.rpython.annlowlevel import cast_base_ptr_to_instance
 from pypy.jit.metainterp.history import AbstractDescr
+from pypy.rlib.objectmodel import we_are_translated
 
 
 def get_mutate_field_name(fieldname):
@@ -50,13 +51,13 @@
 
 class QuasiImmut(object):
     llopaque = True
+    compress_limit = 30
     
     def __init__(self, cpu):
         self.cpu = cpu
         # list of weakrefs to the LoopTokens that must be invalidated if
         # this value ever changes
         self.looptokens_wrefs = []
-        self.compress_limit = 30
 
     def hide(self):
         qmut_ptr = self.cpu.ts.cast_instance_to_base_ref(self)
@@ -75,6 +76,8 @@
     def compress_looptokens_list(self):
         self.looptokens_wrefs = [wref for wref in self.looptokens_wrefs
                                       if wref() is not None]
+        # NB. we must keep around the looptoken_wrefs that are
+        # already invalidated; see below
         self.compress_limit = (len(self.looptokens_wrefs) + 15) * 2
 
     def invalidate(self):
@@ -86,7 +89,16 @@
         for wref in wrefs:
             looptoken = wref()
             if looptoken is not None:
+                looptoken.invalidated = True
                 self.cpu.invalidate_loop(looptoken)
+                # NB. we must call cpu.invalidate_loop() even if
+                # looptoken.invalidated was already set to True.
+                # It's possible to invalidate several times the
+                # same looptoken; see comments in jit.backend.model
+                # in invalidate_loop().
+                if not we_are_translated():
+                    self.cpu.stats.invalidated_token_numbers.add(
+                        looptoken.number)
 
 
 class QuasiImmutDescr(AbstractDescr):
diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py
--- a/pypy/jit/metainterp/resoperation.py
+++ b/pypy/jit/metainterp/resoperation.py
@@ -221,7 +221,6 @@
         newop.setfailargs(self.getfailargs())
         return newop
 
-
 # ============
 # arity mixins
 # ============
@@ -434,6 +433,8 @@
     'INT_INVERT/1',
     #
     'SAME_AS/1',      # gets a Const or a Box, turns it into another Box
+    'CAST_PTR_TO_INT/1',
+    'CAST_INT_TO_PTR/1',
     #
     'PTR_EQ/2b',
     'PTR_NE/2b',
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -13,7 +13,7 @@
 from pypy.rlib.jit import (JitDriver, we_are_jitted, hint, dont_look_inside,
     loop_invariant, elidable, promote, jit_debug, assert_green,
     AssertGreenFailed, unroll_safe, current_trace_length, look_inside_iff,
-    isconstant, isvirtual)
+    isconstant, isvirtual, promote_string)
 from pypy.rlib.rarithmetic import ovfcheck
 from pypy.rpython.lltypesystem import lltype, llmemory, rffi
 from pypy.rpython.ootypesystem import ootype
@@ -2956,6 +2956,18 @@
         assert res == f(32)
         self.check_loops(arraylen_gc=2)
 
+    def test_ulonglong_mod(self):
+        myjitdriver = JitDriver(greens = [], reds = ['n', 'sa', 'i'])
+        def f(n):
+            sa = i = rffi.cast(rffi.ULONGLONG, 1)
+            while i < rffi.cast(rffi.ULONGLONG, n):
+                myjitdriver.jit_merge_point(sa=sa, n=n, i=i)
+                sa += sa % i
+                i += 1
+        res = self.meta_interp(f, [32])
+        assert res == f(32)
+
+
 class TestOOtype(BasicTests, OOJitMixin):
 
     def test_oohash(self):
@@ -3408,6 +3420,106 @@
         assert res == main(1, 10)
         self.check_loops(call=0)
 
+    def test_setarrayitem_followed_by_arraycopy(self):
+        myjitdriver = JitDriver(greens = [], reds = ['n', 'sa', 'x', 'y'])
+        def f(n):
+            sa = 0
+            x = [1,2,n]
+            y = [1,2,3]
+            while n > 0:
+                myjitdriver.jit_merge_point(sa=sa, n=n, x=x, y=y)
+                y[0] = n
+                x[0:3] = y
+                sa += x[0]
+                n -= 1
+            return sa
+        res = self.meta_interp(f, [16])
+        assert res == f(16)
+        
+
 
 class TestLLtype(BaseLLtypeTests, LLJitMixin):
-    pass
+    def test_tagged(self):
+        from pypy.rlib.objectmodel import UnboxedValue
+        class Base(object):
+            __slots__ = ()
+
+        class Int(UnboxedValue, Base):
+            __slots__ = ["a"]
+
+            def is_pos(self):
+                return self.a > 0
+
+            def dec(self):
+                return Int(self.a - 1)
+
+
+        class Float(Base):
+            def __init__(self, a):
+                self.a = a
+
+            def is_pos(self):
+                return self.a > 0
+
+            def dec(self):
+                return Float(self.a - 1)
+
+        driver = JitDriver(greens=['pc', 's'], reds=['o'])
+
+        def main(fl, n, s):
+            if s:
+                s = "--j"
+            else:
+                s = "---j"
+            if fl:
+                o = Float(float(n))
+            else:
+                o = Int(n)
+            pc = 0
+            while True:
+                driver.jit_merge_point(s=s, pc=pc, o=o)
+                c = s[pc]
+                if c == "j":
+                    driver.can_enter_jit(s=s, pc=pc, o=o)
+                    if o.is_pos():
+                        pc = 0
+                        continue
+                    else:
+                        break
+                elif c == "-":
+                    o = o.dec()
+                pc += 1
+            return pc
+        res = self.meta_interp(main, [False, 100, True], taggedpointers=True)
+
+    def test_rerased(self):
+        from pypy.rlib.rerased import erase_int, unerase_int, new_erasing_pair
+        eraseX, uneraseX = new_erasing_pair("X")
+        #
+        class X:
+            def __init__(self, a, b):
+                self.a = a
+                self.b = b
+        #
+        def f(i, j):
+            # 'j' should be 0 or 1, not other values
+            if j > 0:
+                e = eraseX(X(i, j))
+            else:
+                try:
+                    e = erase_int(i)
+                except OverflowError:
+                    return -42
+            if j & 1:
+                x = uneraseX(e)
+                return x.a - x.b
+            else:
+                return unerase_int(e)
+        #
+        x = self.interp_operations(f, [-128, 0], taggedpointers=True)
+        assert x == -128
+        bigint = sys.maxint//2 + 1
+        x = self.interp_operations(f, [bigint, 0], taggedpointers=True)
+        assert x == -42
+        x = self.interp_operations(f, [1000, 1], taggedpointers=True)
+        assert x == 999
diff --git a/pypy/jit/metainterp/test/test_heapcache.py b/pypy/jit/metainterp/test/test_heapcache.py
--- a/pypy/jit/metainterp/test/test_heapcache.py
+++ b/pypy/jit/metainterp/test/test_heapcache.py
@@ -355,6 +355,14 @@
         assert not h.is_unescaped(box1)
         assert not h.is_unescaped(box2)
 
+    def test_circular_virtuals(self):
+        h = HeapCache()
+        h.new(box1)
+        h.new(box2)
+        h.invalidate_caches(rop.SETFIELD_GC, None, [box1, box2])
+        h.invalidate_caches(rop.SETFIELD_GC, None, [box2, box1])
+        h.invalidate_caches(rop.SETFIELD_GC, None, [box3, box1]) # does not crash
+
     def test_unescaped_array(self):
         h = HeapCache()
         h.new_array(box1, lengthbox1)
@@ -362,4 +370,4 @@
         h.invalidate_caches(rop.SETARRAYITEM_GC, None, [box1, index1, box2])
         assert h.is_unescaped(box1)
         h.invalidate_caches(rop.SETARRAYITEM_GC, None, [box2, index1, box1])
-        assert not h.is_unescaped(box1)
\ No newline at end of file
+        assert not h.is_unescaped(box1)
diff --git a/pypy/jit/metainterp/test/test_longlong.py b/pypy/jit/metainterp/test/test_longlong.py
--- a/pypy/jit/metainterp/test/test_longlong.py
+++ b/pypy/jit/metainterp/test/test_longlong.py
@@ -118,6 +118,26 @@
         res = self.interp_operations(f, [1000000000])
         assert res == 123500000000.0
 
+    def test_floats_negative(self):
+        def f(i):
+            # i == 1000000000
+            f = i * -123.5
+            n = r_longlong(f)
+            compare(n, -29, 1054051584)
+            return float(n)
+        res = self.interp_operations(f, [1000000000])
+        assert res == -123500000000.0
+
+    def test_floats_ulonglong(self):
+        def f(i):
+            # i == 1000000000
+            f = i * 12350000000.0
+            n = r_ulonglong(f)
+            compare(n, -1419508847, 538116096)
+            return float(n)
+        res = self.interp_operations(f, [1000000000])
+        assert res == 12350000000000000000.0
+
     def test_unsigned_compare_ops(self):
         def f(n1, n2):
             # n == 30002000000000
diff --git a/pypy/jit/metainterp/test/test_memmgr.py b/pypy/jit/metainterp/test/test_memmgr.py
--- a/pypy/jit/metainterp/test/test_memmgr.py
+++ b/pypy/jit/metainterp/test/test_memmgr.py
@@ -18,6 +18,7 @@
 
 class FakeLoopToken:
     generation = 0
+    invalidated = False
 
 
 class _TestMemoryManager:
diff --git a/pypy/jit/metainterp/test/test_quasiimmut.py b/pypy/jit/metainterp/test/test_quasiimmut.py
--- a/pypy/jit/metainterp/test/test_quasiimmut.py
+++ b/pypy/jit/metainterp/test/test_quasiimmut.py
@@ -48,6 +48,13 @@
 
 class QuasiImmutTests(object):
 
+    def setup_method(self, meth):
+        self.prev_compress_limit = QuasiImmut.compress_limit
+        QuasiImmut.compress_limit = 1
+
+    def teardown_method(self, meth):
+        QuasiImmut.compress_limit = self.prev_compress_limit
+
     def test_simple_1(self):
         myjitdriver = JitDriver(greens=['foo'], reds=['x', 'total'])
         class Foo:
@@ -289,7 +296,7 @@
             return total
 
         res = self.meta_interp(main, [])
-        self.check_loop_count(9)
+        self.check_tree_loop_count(6)
         assert res == main()
 
     def test_change_during_running(self):
@@ -317,7 +324,7 @@
         assert f(100, 15) == 3009
         res = self.meta_interp(f, [100, 15])
         assert res == 3009
-        self.check_loops(guard_not_invalidated=2, getfield_gc=0,
+        self.check_loops(guard_not_invalidated=4, getfield_gc=0,
                          call_may_force=0, guard_not_forced=0)
 
     def test_list_simple_1(self):
@@ -453,10 +460,30 @@
         assert f(100, 15) == 3009
         res = self.meta_interp(f, [100, 15])
         assert res == 3009
-        self.check_loops(guard_not_invalidated=2, getfield_gc=0,
+        self.check_loops(guard_not_invalidated=4, getfield_gc=0,
                          getarrayitem_gc=0, getarrayitem_gc_pure=0,
                          call_may_force=0, guard_not_forced=0)
 
+    def test_invalidated_loop_is_not_used_any_more_as_target(self):
+        myjitdriver = JitDriver(greens=['foo'], reds=['x'])
+        class Foo:
+            _immutable_fields_ = ['step?']
+        @dont_look_inside
+        def residual(x, foo):
+            if x == 20:
+                foo.step = 1
+        def f(x):
+            foo = Foo()
+            foo.step = 2
+            while x > 0:
+                myjitdriver.jit_merge_point(foo=foo, x=x)
+                residual(x, foo)
+                x -= foo.step
+            return foo.step
+        res = self.meta_interp(f, [60])
+        assert res == 1
+        self.check_tree_loop_count(4)   # at least not 2 like before
+
 
 class TestLLtypeGreenFieldsTests(QuasiImmutTests, LLJitMixin):
     pass
diff --git a/pypy/jit/metainterp/test/test_resume.py b/pypy/jit/metainterp/test/test_resume.py
--- a/pypy/jit/metainterp/test/test_resume.py
+++ b/pypy/jit/metainterp/test/test_resume.py
@@ -1,3 +1,4 @@
+from __future__ import with_statement
 import py
 from pypy.rpython.lltypesystem import lltype, llmemory, rffi
 from pypy.jit.metainterp.optimizeopt.optimizer import OptValue
@@ -180,22 +181,27 @@
     reader.consume_boxes(info, bi, br, bf)
     b1s = reader.liveboxes[0]
     b2s = reader.liveboxes[1]
-    assert bi == [b1s, ConstInt(111), b1s]
-    assert br == [ConstPtr(gcrefnull), b2s]
+    assert_same(bi, [b1s, ConstInt(111), b1s])
+    assert_same(br, [ConstPtr(gcrefnull), b2s])
     bi, br, bf = [None]*2, [None]*0, [None]*0
     info = MyBlackholeInterp([lltype.Signed,
                               lltype.Signed]).get_current_position_info()
     reader.consume_boxes(info, bi, br, bf)
-    assert bi == [ConstInt(222), ConstInt(333)]
+    assert_same(bi, [ConstInt(222), ConstInt(333)])
     bi, br, bf = [None]*2, [None]*1, [None]*0
     info = MyBlackholeInterp([lltype.Signed, llmemory.GCREF,
                               lltype.Signed]).get_current_position_info()
     reader.consume_boxes(info, bi, br, bf)
     b3s = reader.liveboxes[2]
-    assert bi == [b1s, b3s]
-    assert br == [b2s]
+    assert_same(bi, [b1s, b3s])
+    assert_same(br, [b2s])
     #
 
+def assert_same(list1, list2):
+    assert len(list1) == len(list2)
+    for b1, b2 in zip(list1, list2):
+        assert b1.same_box(b2)
+
 def test_simple_read_tagged_ints():
     storage = Storage()
     storage.rd_consts = []
@@ -576,8 +582,9 @@
 
 
 class FakeOptimizer_VirtualValue(object):
-    class cpu:
-        pass
+    class optimizer:
+        class cpu:
+            pass
 fakeoptimizer = FakeOptimizer_VirtualValue()
 
 def ConstAddr(addr, cpu):   # compatibility
@@ -1022,11 +1029,11 @@
     metainterp = MyMetaInterp()
     reader = ResumeDataFakeReader(storage, newboxes, metainterp)
     lst = reader.consume_boxes()
-    assert lst == [b1t, b1t, b3t]
+    assert_same(lst, [b1t, b1t, b3t])
     lst = reader.consume_boxes()
-    assert lst == [ConstInt(2), ConstInt(3)]
+    assert_same(lst, [ConstInt(2), ConstInt(3)])
     lst = reader.consume_boxes()
-    assert lst == [b1t, ConstInt(1), b1t, b1t]
+    assert_same(lst, [b1t, ConstInt(1), b1t, b1t])
     assert metainterp.trace == []    
 
 
@@ -1043,11 +1050,11 @@
     reader = ResumeDataFakeReader(storage, newboxes, metainterp)
     lst = reader.consume_boxes()
     c1t = ConstInt(111)
-    assert lst == [c1t, b2t, b3t]
+    assert_same(lst, [c1t, b2t, b3t])
     lst = reader.consume_boxes()
-    assert lst == [ConstInt(2), ConstInt(3)]
+    assert_same(lst, [ConstInt(2), ConstInt(3)])
     lst = reader.consume_boxes()
-    assert lst == [c1t, ConstInt(1), c1t, b2t]
+    assert_same(lst, [c1t, ConstInt(1), c1t, b2t])
     assert metainterp.trace == []
 
 
@@ -1113,9 +1120,11 @@
     # check that we get the operations in 'expected', in a possibly different
     # order.
     assert len(trace) == len(expected)
-    for x in trace:
-        assert x in expected
-        expected.remove(x)
+    with CompareableConsts():
+        for x in trace:
+            assert x in expected
+            expected.remove(x)
+
     ptr = b2t.value._obj.container._as_ptr()
     assert lltype.typeOf(ptr) == lltype.Ptr(LLtypeMixin.NODE)
     assert ptr.value == 111
@@ -1125,6 +1134,18 @@
     assert ptr2.parent.value == 33
     assert ptr2.parent.next == ptr
 
+class CompareableConsts(object):
+    def __init__(self):
+        self.oldeq = None
+        
+    def __enter__(self):
+        assert self.oldeq is None
+        self.oldeq = Const.__eq__
+        Const.__eq__ = Const.same_box
+        
+    def __exit__(self, type, value, traceback):
+        Const.__eq__ = self.oldeq
+
 def test_virtual_adder_make_varray():
     b2s, b4s = [BoxPtr(), BoxInt(4)]
     c1s = ConstInt(111)
@@ -1135,12 +1156,7 @@
     modifier.liveboxes = {}
     modifier.vfieldboxes = {}
 
-    class FakeOptimizer(object):
-        class cpu:
-            pass
-        def new_const_item(self, descr):
-            return None
-    v2 = VArrayValue(FakeOptimizer(), LLtypeMixin.arraydescr, 2, b2s)
+    v2 = VArrayValue(LLtypeMixin.arraydescr, None, 2, b2s)
     v2._items = [b4s, c1s]
     modifier.register_virtual_fields(b2s, [b4s, c1s])
     liveboxes = []
@@ -1167,8 +1183,9 @@
         (rop.SETARRAYITEM_GC, [b2t,ConstInt(1), c1s], None,
                               LLtypeMixin.arraydescr),
         ]
-    for x, y in zip(expected, trace):
-        assert x == y
+    with CompareableConsts():
+        for x, y in zip(expected, trace):
+            assert x == y
     #
     ptr = b2t.value._obj.container._as_ptr()
     assert lltype.typeOf(ptr) == lltype.Ptr(lltype.GcArray(lltype.Signed))
@@ -1211,8 +1228,9 @@
         (rop.SETFIELD_GC, [b2t, c1s],  None, LLtypeMixin.adescr),
         (rop.SETFIELD_GC, [b2t, b4t], None, LLtypeMixin.bdescr),
         ]
-    for x, y in zip(expected, trace):
-        assert x == y
+    with CompareableConsts():
+        for x, y in zip(expected, trace):
+            assert x == y
     #
     ptr = b2t.value._obj.container._as_ptr()
     assert lltype.typeOf(ptr) == lltype.Ptr(LLtypeMixin.S)
diff --git a/pypy/jit/metainterp/test/test_string.py b/pypy/jit/metainterp/test/test_string.py
--- a/pypy/jit/metainterp/test/test_string.py
+++ b/pypy/jit/metainterp/test/test_string.py
@@ -3,7 +3,8 @@
 from pypy.jit.codewriter.policy import StopAtXPolicy
 from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin
 from pypy.rlib.debug import debug_print
-from pypy.rlib.jit import JitDriver, dont_look_inside, we_are_jitted
+from pypy.rlib.jit import JitDriver, dont_look_inside, we_are_jitted,\
+     promote_string
 from pypy.rlib.rstring import StringBuilder
 from pypy.rpython.ootypesystem import ootype
 
@@ -484,6 +485,42 @@
             return len(sa.val)
         assert self.meta_interp(f, ['a']) == f('a')
 
+    def test_string_comepare_quasiimmutable(self):
+        class Sys(object):
+            _immutable_fields_ = ["defaultencoding?"]
+            def __init__(self, s):
+                self.defaultencoding = s
+        _str = self._str
+        sys = Sys(_str('ascii'))        
+        mydriver = JitDriver(reds = ['n', 'sa'], greens = [])
+        def f(n):
+            sa = 0
+            sys.defaultencoding = _str('ascii')
+            while n:
+                mydriver.jit_merge_point(n=n, sa=sa)
+                if sys.defaultencoding == _str('ascii'):
+                    sa += 1
+                n -= 1
+            sys.defaultencoding = _str('utf-8')
+            return sa
+        assert self.meta_interp(f, [8]) == f(8)
+        self.check_loops({'int_add': 1, 'guard_true': 1, 'int_sub': 1,
+                          'jump': 1, 'int_is_true': 1,
+                          'guard_not_invalidated': 1})
+
+    def test_promote_string(self):
+        driver = JitDriver(greens = [], reds = ['n'])
+        
+        def f(n):
+            while n < 21:
+                driver.jit_merge_point(n=n)
+                promote_string(str(n % 3))
+                n += 1
+            return 0
+
+        self.meta_interp(f, [0])
+        self.check_loops(call=3 + 1) # one for int2str
+
 #class TestOOtype(StringTests, OOJitMixin):
 #    CALL = "oosend"
 #    CALL_PURE = "oosend_pure"
diff --git a/pypy/jit/metainterp/test/test_virtualref.py b/pypy/jit/metainterp/test/test_virtualref.py
--- a/pypy/jit/metainterp/test/test_virtualref.py
+++ b/pypy/jit/metainterp/test/test_virtualref.py
@@ -3,6 +3,7 @@
 from pypy.rpython.llinterp import LLException
 from pypy.rlib.jit import JitDriver, dont_look_inside, vref_None
 from pypy.rlib.jit import virtual_ref, virtual_ref_finish, InvalidVirtualRef
+from pypy.rlib.jit import non_virtual_ref
 from pypy.rlib.objectmodel import compute_unique_id
 from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin, _get_jitcodes
 from pypy.jit.metainterp.resoperation import rop
@@ -595,6 +596,65 @@
         res = self.meta_interp(fn, [10])
         assert res == 6
 
+    def test_is_virtual(self):
+        myjitdriver = JitDriver(greens=[], reds=['n', 'res1'])
+        class X:
+            pass
+        @dont_look_inside
+        def residual(vref):
+            return vref.virtual
+        #
+        def f(n):
+            res1 = -42
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, res1=res1)
+                x = X()
+                vref = virtual_ref(x)
+                res1 = residual(vref)
+                virtual_ref_finish(vref, x)
+                n -= 1
+            return res1
+        #
+        res = self.meta_interp(f, [10])
+        assert res == 1
+
+    def test_is_not_virtual_none(self):
+        myjitdriver = JitDriver(greens=[], reds=['n', 'res1'])
+        @dont_look_inside
+        def residual(vref):
+            return vref.virtual
+        #
+        def f(n):
+            res1 = -42
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, res1=res1)
+                res1 = residual(vref_None)
+                n -= 1
+            return res1
+        #
+        res = self.meta_interp(f, [10])
+        assert res == 0
+
+    def test_is_not_virtual_non_none(self):
+        myjitdriver = JitDriver(greens=[], reds=['n', 'res1'])
+        class X:
+            pass
+        @dont_look_inside
+        def residual(vref):
+            return vref.virtual
+        #
+        def f(n):
+            res1 = -42
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, res1=res1)
+                x = X()
+                res1 = residual(non_virtual_ref(x))
+                n -= 1
+            return res1
+        #
+        res = self.meta_interp(f, [10])
+        assert res == 0
+
 
 class TestLLtype(VRefTests, LLJitMixin):
     pass
diff --git a/pypy/jit/metainterp/test/test_virtualstate.py b/pypy/jit/metainterp/test/test_virtualstate.py
--- a/pypy/jit/metainterp/test/test_virtualstate.py
+++ b/pypy/jit/metainterp/test/test_virtualstate.py
@@ -431,7 +431,7 @@
         
 
 class BaseTestBridges(BaseTest):
-    enable_opts = "intbounds:rewrite:virtualize:string:heap:unroll"
+    enable_opts = "intbounds:rewrite:virtualize:string:pure:heap:unroll"
 
     def _do_optimize_bridge(self, bridge, call_pure_results):
         from pypy.jit.metainterp.optimizeopt import optimize_bridge_1, build_opt_chain
diff --git a/pypy/jit/metainterp/virtualref.py b/pypy/jit/metainterp/virtualref.py
--- a/pypy/jit/metainterp/virtualref.py
+++ b/pypy/jit/metainterp/virtualref.py
@@ -39,6 +39,7 @@
     def replace_force_virtual_with_call(self, graphs):
         # similar to rvirtualizable2.replace_force_virtualizable_with_call().
         c_force_virtual_ptr = None
+        c_is_virtual_ptr = None
         force_virtual_count = 0
         for graph in graphs:
             for block in graph.iterblocks():
@@ -52,6 +53,13 @@
                         op.opname = 'direct_call'
                         op.args = [c_force_virtual_ptr, op.args[0]]
                         force_virtual_count += 1
+                    #
+                    if op.opname == 'jit_is_virtual':
+                        if c_is_virtual_ptr is None:
+                            c_is_virtual_ptr = self.get_is_virtual_fnptr()
+                        #
+                        op.opname = 'direct_call'
+                        op.args = [c_is_virtual_ptr, op.args[0]]
         #
         if c_force_virtual_ptr is not None:
             log("replaced %d 'jit_force_virtual' with %r" % (force_virtual_count,
@@ -129,6 +137,17 @@
             force_virtual_if_necessary)
         return inputconst(lltype.typeOf(funcptr), funcptr)
 
+    def get_is_virtual_fnptr(self):
+        #
+        def is_virtual(inst):
+            if not inst:
+                return False
+            return inst.typeptr == self.jit_virtual_ref_vtable
+        #
+        FUNC = lltype.FuncType([rclass.OBJECTPTR], lltype.Bool)
+        funcptr = self.warmrunnerdesc.helper_func(lltype.Ptr(FUNC), is_virtual)
+        return inputconst(lltype.typeOf(funcptr), funcptr)
+
     def force_virtual(self, inst):
         vref = lltype.cast_pointer(lltype.Ptr(self.JIT_VIRTUAL_REF), inst)
         token = vref.virtual_token
diff --git a/pypy/jit/metainterp/warmspot.py b/pypy/jit/metainterp/warmspot.py
--- a/pypy/jit/metainterp/warmspot.py
+++ b/pypy/jit/metainterp/warmspot.py
@@ -53,6 +53,8 @@
         extraconfigopts = {'translation.list_comprehension_operations': True}
     else:
         extraconfigopts = {}
+    if kwds.pop("taggedpointers", False):
+        extraconfigopts["translation.taggedpointers"] = True
     interp, graph = get_interpreter(function, args,
                                     backendopt=False,  # will be done below
                                     type_system=type_system,
diff --git a/pypy/jit/metainterp/warmstate.py b/pypy/jit/metainterp/warmstate.py
--- a/pypy/jit/metainterp/warmstate.py
+++ b/pypy/jit/metainterp/warmstate.py
@@ -178,7 +178,7 @@
         if self.compiled_merge_points_wref is not None:
             for wref in self.compiled_merge_points_wref:
                 looptoken = wref()
-                if looptoken is not None:
+                if looptoken is not None and not looptoken.invalidated:
                     result.append(looptoken)
         return result
 
diff --git a/pypy/module/__builtin__/__init__.py b/pypy/module/__builtin__/__init__.py
--- a/pypy/module/__builtin__/__init__.py
+++ b/pypy/module/__builtin__/__init__.py
@@ -20,6 +20,10 @@
         'any'           : 'app_functional.any',
         'all'           : 'app_functional.all',
         'sum'           : 'app_functional.sum',
+        'map'           : 'app_functional.map',
+        'reduce'        : 'app_functional.reduce',
+        'filter'        : 'app_functional.filter',
+        'zip'           : 'app_functional.zip',
         'vars'          : 'app_inspect.vars',
         'dir'           : 'app_inspect.dir',
 
@@ -86,11 +90,7 @@
         'enumerate'     : 'functional.W_Enumerate',
         'min'           : 'functional.min',
         'max'           : 'functional.max',
-        'map'           : 'functional.map',
-        'zip'           : 'functional.zip',
-        'reduce'        : 'functional.reduce',
         'reversed'      : 'functional.reversed',
-        'filter'        : 'functional.filter',
         'super'         : 'descriptor.W_Super',
         'staticmethod'  : 'descriptor.StaticMethod',
         'classmethod'   : 'descriptor.ClassMethod',
@@ -119,7 +119,7 @@
            builtin = space.interpclass_w(w_builtin)
            if isinstance(builtin, module.Module):
                return builtin
-       # no builtin! make a default one.  Given them None, at least.
+       # no builtin! make a default one.  Give them None, at least.
        builtin = module.Module(space, None)
        space.setitem(builtin.w_dict, space.wrap('None'), space.w_None)
        return builtin
diff --git a/pypy/module/__builtin__/app_functional.py b/pypy/module/__builtin__/app_functional.py
--- a/pypy/module/__builtin__/app_functional.py
+++ b/pypy/module/__builtin__/app_functional.py
@@ -48,4 +48,135 @@
         # Very intentionally *not* +=, that would have different semantics if
         # start was a mutable type, such as a list
         last = last + x
-    return last
\ No newline at end of file
+    return last
+
+def map(func, *collections):
+    """map(function, sequence[, sequence, ...]) -> list
+
+Return a list of the results of applying the function to the items of
+the argument sequence(s).  If more than one sequence is given, the
+function is called with an argument list consisting of the corresponding
+item of each sequence, substituting None for missing values when not all
+sequences have the same length.  If the function is None, return a list of
+the items of the sequence (or a list of tuples if more than one sequence)."""
+    if not collections:
+        raise TypeError("map() requires at least two arguments")
+    num_collections = len(collections)
+    none_func = func is None
+    if num_collections == 1:
+        if none_func:
+            return list(collections[0])
+        else:
+            # Special case for the really common case of a single collection,
+            # this can be eliminated if we could unroll that loop that creates
+            # `args` based on whether or not len(collections) was constant
+            result = []
+            for item in collections[0]:
+                result.append(func(item))
+            return result
+    result = []
+    # Pair of (iterator, has_finished)
+    iterators = [(iter(seq), False) for seq in collections]
+    while True:
+        cont = False
+        args = []
+        for idx, (iterator, has_finished) in enumerate(iterators):
+            val = None
+            if not has_finished:
+                try:
+                    val = next(iterator)
+                except StopIteration:
+                    iterators[idx] = (None, True)
+                else:
+                    cont = True
+            args.append(val)
+        args = tuple(args)
+        if cont:
+            if none_func:
+                result.append(args)
+            else:
+                result.append(func(*args))
+        else:
+            return result
+
+sentinel = object()
+
+def reduce(func, sequence, initial=sentinel):
+    """reduce(function, sequence[, initial]) -> value
+
+Apply a function of two arguments cumulatively to the items of a sequence,
+from left to right, so as to reduce the sequence to a single value.
+For example, reduce(lambda x, y: x+y, [1, 2, 3, 4, 5]) calculates
+((((1+2)+3)+4)+5).  If initial is present, it is placed before the items
+of the sequence in the calculation, and serves as a default when the
+sequence is empty."""
+    iterator = iter(sequence)
+    if initial is sentinel:
+        try:
+            initial = next(iterator)
+        except StopIteration:
+            raise TypeError("reduce() of empty sequence with no initial value")
+    result = initial
+    for item in iterator:
+        result = func(result, item)
+    return result
+
+def filter(func, seq):
+    """filter(function or None, sequence) -> list, tuple, or string
+
+Return those items of sequence for which function(item) is true.  If
+function is None, return the items that are true.  If sequence is a tuple
+or string, return the same type, else return a list."""
+    if func is None:
+        func = bool
+    if isinstance(seq, str):
+        return _filter_string(func, seq, str)
+    elif isinstance(seq, unicode):
+        return _filter_string(func, seq, unicode)
+    elif isinstance(seq, tuple):
+        return _filter_tuple(func, seq)
+    result = []
+    for item in seq:
+        if func(item):
+            result.append(item)
+    return result
+
+def _filter_string(func, string, str_type):
+    if func is bool and type(string) is str_type:
+        return string
+    result = []
+    for i in range(len(string)):
+        # You must call __getitem__ on the strings, simply iterating doesn't
+        # work :/
+        item = string[i]
+        if func(item):
+            if not isinstance(item, str_type):
+                raise TypeError("__getitem__ returned a non-string type")
+            result.append(item)
+    return str_type().join(result)
+
+def _filter_tuple(func, seq):
+    result = []
+    for i in range(len(seq)):
+        # Again, must call __getitem__, at least there are tests.
+        item = seq[i]
+        if func(item):
+            result.append(item)
+    return tuple(result)
+
+def zip(*sequences):
+    """zip(seq1 [, seq2 [...]]) -> [(seq1[0], seq2[0] ...), (...)]
+
+Return a list of tuples, where each tuple contains the i-th element
+from each of the argument sequences.  The returned list is truncated
+in length to the length of the shortest argument sequence."""
+    if not sequences:
+        return []
+    result = []
+    iterators = [iter(seq) for seq in sequences]
+    while True:
+        try:
+            items = [next(it) for it in iterators]
+        except StopIteration:
+            return result
+        result.append(tuple(items))
diff --git a/pypy/module/__builtin__/app_io.py b/pypy/module/__builtin__/app_io.py
--- a/pypy/module/__builtin__/app_io.py
+++ b/pypy/module/__builtin__/app_io.py
@@ -27,7 +27,20 @@
     co = compile(source.rstrip()+"\n", filename, 'exec')
     exec co in glob, loc
 
-def raw_input(prompt=None):
+def _write_prompt(stdout, prompt):
+    print >> stdout, prompt,
+    try:
+        flush = stdout.flush
+    except AttributeError:
+        pass
+    else:
+        flush()
+    try:
+        stdout.softspace = 0
+    except (AttributeError, TypeError):
+        pass
+
+def raw_input(prompt=''):
     """raw_input([prompt]) -> string
 
 Read a string from standard input.  The trailing newline is stripped.
@@ -47,18 +60,10 @@
     if (hasattr(sys, '__raw_input__') and
         isinstance(stdin, file)  and stdin.fileno() == 0 and stdin.isatty() and
         isinstance(stdout, file) and stdout.fileno() == 1):
-        if prompt is None:
-            prompt = ''
-        return sys.__raw_input__(prompt)
+        _write_prompt(stdout, '')
+        return sys.__raw_input__(str(prompt))
 
-    if prompt is not None:
-        stdout.write(prompt)
-        try:
-            flush = stdout.flush
-        except AttributeError:
-            pass
-        else:
-            flush()
+    _write_prompt(stdout, prompt)
     line = stdin.readline()
     if not line:    # inputting an empty line gives line == '\n'
         raise EOFError
diff --git a/pypy/module/__builtin__/compiling.py b/pypy/module/__builtin__/compiling.py
--- a/pypy/module/__builtin__/compiling.py
+++ b/pypy/module/__builtin__/compiling.py
@@ -84,8 +84,8 @@
         raise OperationError(space.w_TypeError,
               w('eval() arg 1 must be a string or code object'))
 
-    caller = space.getexecutioncontext().gettopframe_nohidden()
     if space.is_w(w_globals, space.w_None):
+        caller = space.getexecutioncontext().gettopframe_nohidden()
         if caller is None:
             w_globals = space.newdict()
             if space.is_w(w_locals, space.w_None):
@@ -97,13 +97,9 @@
     elif space.is_w(w_locals, space.w_None):
         w_locals = w_globals
 
-    try:
-        space.getitem(w_globals, space.wrap('__builtins__'))
-    except OperationError, e:
-        if not e.match(space, space.w_KeyError):
-            raise
-        if caller is not None:
-            w_builtin = space.builtin.pick_builtin(caller.w_globals)
-            space.setitem(w_globals, space.wrap('__builtins__'), w_builtin)
+    # xxx removed: adding '__builtins__' to the w_globals dict, if there
+    # is none.  This logic was removed as costly (it requires to get at
+    # the gettopframe_nohidden()).  I bet no test fails, and it's a really
+    # obscure case.
 
     return codeobj.exec_code(space, w_globals, w_locals)
diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py
--- a/pypy/module/__builtin__/functional.py
+++ b/pypy/module/__builtin__/functional.py
@@ -193,229 +193,14 @@
     return min_max(space, __args__, "max")
 
 def min(space, __args__):
-    """Return the smallest item in a sequence.
+    """min(iterable[, key=func]) -> value
+    min(a, b, c, ...[, key=func]) -> value
 
-    If more than one argument is passed, return the minimum of them.
+    With a single iterable argument, return its smallest item.
+    With two or more arguments, return the smallest argument.
     """
     return min_max(space, __args__, "min")
 
- at unwrap_spec(collections_w="args_w")
-def map(space, w_func, collections_w):
-    """does 3 separate things, hence this enormous docstring.
-       1.  if function is None, return a list of tuples, each with one
-           item from each collection.  If the collections have different
-           lengths,  shorter ones are padded with None.
-
-       2.  if function is not None, and there is only one collection,
-           apply function to every item in the collection and return a
-           list of the results.
-
-       3.  if function is not None, and there are several collections,
-           repeatedly call the function with one argument from each
-           collection.  If the collections have different lengths,
-           shorter ones are padded with None
-    """
-    if not collections_w:
-        msg = "map() requires at least two arguments"
-        raise OperationError(space.w_TypeError, space.wrap(msg))
-    none_func = space.is_w(w_func, space.w_None)
-    if len(collections_w) == 1:
-        w_collection = collections_w[0]
-        if none_func:
-            result_w = space.unpackiterable(w_collection)
-        else:
-            result_w = map_single_collection(space, w_func, w_collection)
-    else:
-        result_w = map_multiple_collections(space, w_func, collections_w,
-                                            none_func)
-    return space.newlist(result_w)
-
-def map_single_collection(space, w_func, w_collection):
-    """Special case for 'map(func, coll)', where 'func' is not None and there
-    is only one 'coll' argument."""
-    w_iter = space.iter(w_collection)
-    # xxx special hacks for speed
-    from pypy.interpreter import function, pycode
-    if isinstance(w_func, function.Function):
-        # xxx compatibility issue: what if func_code is modified in the
-        # middle of running map()??  That's far too obscure for me to care...
-        code = w_func.getcode()
-        fast_natural_arity = code.fast_natural_arity
-        if fast_natural_arity == (1|pycode.PyCode.FLATPYCALL):
-            assert isinstance(code, pycode.PyCode)
-            return map_single_user_function(code, w_func, w_iter)
-    # /xxx end of special hacks
-    return map_single_other_callable(space, w_func, w_iter)
-
-def map_single_other_callable(space, w_func, w_iter):
-    result_w = []
-    while True:
-        try:
-            w_item = space.next(w_iter)
-        except OperationError, e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            break
-        result_w.append(space.call_function(w_func, w_item))
-    return result_w
-map_single_other_callable._dont_inline_ = True
-
-from pypy.rlib.jit import JitDriver
-mapjitdriver = JitDriver(greens = ['code'],
-                         reds = ['w_func', 'w_iter', 'result_w'])
-def map_single_user_function(code, w_func, w_iter):
-    result_w = []
-    while True:
-        mapjitdriver.can_enter_jit(code=code, w_func=w_func,
-                                   w_iter=w_iter, result_w=result_w)
-        mapjitdriver.jit_merge_point(code=code, w_func=w_func,
-                                     w_iter=w_iter, result_w=result_w)
-        space = w_func.space
-        try:
-            w_item = space.next(w_iter)
-        except OperationError, e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            break
-        new_frame = space.createframe(code, w_func.w_func_globals,
-                                      w_func)
-        new_frame.locals_stack_w[0] = w_item
-        w_res = new_frame.run()
-        result_w.append(w_res)
-    return result_w
-
-def map_multiple_collections(space, w_func, collections_w, none_func):
-    result_w = []
-    iterators_w = [space.iter(w_seq) for w_seq in collections_w]
-    num_iterators = len(iterators_w)
-    while True:
-        cont = False
-        args_w = [space.w_None] * num_iterators
-        for i in range(num_iterators):
-            if iterators_w[i] is not None:
-                try:
-                    args_w[i] = space.next(iterators_w[i])
-                except OperationError, e:
-                    if not e.match(space, space.w_StopIteration):
-                        raise
-                    iterators_w[i] = None
-                else:
-                    cont = True
-        if not cont:
-            break
-        w_args = space.newtuple(args_w)
-        if none_func:
-            w_res = w_args
-        else:
-            w_res = space.call(w_func, w_args)
-        result_w.append(w_res)
-    return result_w
-
- at unwrap_spec(sequences_w="args_w")
-def zip(space, sequences_w):
-    """Return a list of tuples, where the nth tuple contains every nth item of
-    each collection.
-
-    If the collections have different lengths, zip returns a list as long as the
-    shortest collection, ignoring the trailing items in the other collections.
-    """
-    if not sequences_w:
-        return space.newlist([])
-    result_w = []
-    iterators_w = [space.iter(w_seq) for w_seq in sequences_w]
-    while True:
-        try:
-            items_w = [space.next(w_it) for w_it in iterators_w]
-        except OperationError, e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            return space.newlist(result_w)
-        result_w.append(space.newtuple(items_w))
-
-def reduce(space, w_func, w_sequence, w_initial=NoneNotWrapped):
-    """ Apply function of two arguments cumulatively to the items of sequence,
-        from left to right, so as to reduce the sequence to a single value.
-        Optionally begin with an initial value.
-    """
-    w_iter = space.iter(w_sequence)
-    if w_initial is None:
-        try:
-            w_initial = space.next(w_iter)
-        except OperationError, e:
-            if e.match(space, space.w_StopIteration):
-                msg = "reduce() of empty sequence with no initial value"
-                raise OperationError(space.w_TypeError, space.wrap(msg))
-            raise
-    w_result = w_initial
-    while True:
-        try:
-            w_next = space.next(w_iter)
-        except OperationError, e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            break
-        w_result = space.call_function(w_func, w_result, w_next)
-    return w_result
-
-def filter(space, w_func, w_seq):
-    """construct a list of those elements of collection for which function
-       is True.  If function is None, then return the items in the sequence
-       which are True.
-    """
-    if space.is_true(space.isinstance(w_seq, space.w_str)):
-        return _filter_string(space, w_func, w_seq, space.w_str)
-    if space.is_true(space.isinstance(w_seq, space.w_unicode)):
-        return _filter_string(space, w_func, w_seq, space.w_unicode)
-    if space.is_true(space.isinstance(w_seq, space.w_tuple)):
-        return _filter_tuple(space, w_func, w_seq)
-    w_iter = space.iter(w_seq)
-    result_w = []
-    none_func = space.is_w(w_func, space.w_None)
-    while True:
-        try:
-            w_next = space.next(w_iter)
-        except OperationError, e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            break
-        if none_func:
-            w_keep = w_next
-        else:
-            w_keep = space.call_function(w_func, w_next)
-        if space.is_true(w_keep):
-            result_w.append(w_next)
-    return space.newlist(result_w)
-
-def _filter_tuple(space, w_func, w_tuple):
-    none_func = space.is_w(w_func, space.w_None)
-    length = space.len_w(w_tuple)
-    result_w = []
-    for i in range(length):
-        w_item = space.getitem(w_tuple, space.wrap(i))
-        if none_func:
-            w_keep = w_item
-        else:
-            w_keep = space.call_function(w_func, w_item)
-        if space.is_true(w_keep):
-            result_w.append(w_item)
-    return space.newtuple(result_w)
-
-def _filter_string(space, w_func, w_string, w_str_type):
-    none_func = space.is_w(w_func, space.w_None)
-    if none_func and space.is_w(space.type(w_string), w_str_type):
-        return w_string
-    length = space.len_w(w_string)
-    result_w = []
-    for i in range(length):
-        w_item = space.getitem(w_string, space.wrap(i))
-        if none_func or space.is_true(space.call_function(w_func, w_item)):
-            if not space.is_true(space.isinstance(w_item, w_str_type)):
-                msg = "__getitem__ returned a non-string type"
-                raise OperationError(space.w_TypeError, space.wrap(msg))
-            result_w.append(w_item)
-    w_empty = space.call_function(w_str_type)
-    return space.call_method(w_empty, "join", space.newlist(result_w))
-
 class W_Enumerate(Wrappable):
 
     def __init__(self, w_iter, w_start):
diff --git a/pypy/module/__builtin__/test/test_functional.py b/pypy/module/__builtin__/test/test_functional.py
--- a/pypy/module/__builtin__/test/test_functional.py
+++ b/pypy/module/__builtin__/test/test_functional.py
@@ -147,7 +147,7 @@
        assert list(xrange(A())) == [0, 1, 2, 3, 4]
        assert list(xrange(0, A())) == [0, 1, 2, 3, 4]
        assert list(xrange(0, 10, A())) == [0, 5]
- 
+
    def test_xrange_float(self):
       assert list(xrange(0.1, 2.0, 1.1)) == [0, 1]
 
diff --git a/pypy/module/__builtin__/test/test_rawinput.py b/pypy/module/__builtin__/test/test_rawinput.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/__builtin__/test/test_rawinput.py
@@ -0,0 +1,77 @@
+import autopath
+
+
+class AppTestRawInput():
+
+    def test_raw_input(self):
+        import sys, StringIO
+        for prompt, expected in [("def:", "abc/ def:/ghi\n"),
+                                 ("", "abc/ /ghi\n"),
+                                 (42, "abc/ 42/ghi\n"),
+                                 (None, "abc/ None/ghi\n"),
+                                 (Ellipsis, "abc/ /ghi\n")]:
+            save = sys.stdin, sys.stdout
+            try:
+                sys.stdin = StringIO.StringIO("foo\nbar\n")
+                out = sys.stdout = StringIO.StringIO()
+                print "abc",    # softspace = 1
+                out.write('/')
+                if prompt is Ellipsis:
+                    got = raw_input()
+                else:
+                    got = raw_input(prompt)
+                out.write('/')
+                print "ghi"
+            finally:
+                sys.stdin, sys.stdout = save
+            assert out.getvalue() == expected
+            assert got == "foo"
+
+    def test_softspace(self):
+        import sys
+        import StringIO
+        fin = StringIO.StringIO()
+        fout = StringIO.StringIO()
+
+        fin.write("Coconuts\n")
+        fin.seek(0)
+
+        sys_stdin_orig = sys.stdin
+        sys_stdout_orig = sys.stdout
+
+        sys.stdin = fin
+        sys.stdout = fout
+
+        print "test",
+        raw_input("test")
+
+        sys.stdin = sys_stdin_orig
+        sys.stdout = sys_stdout_orig
+
+        fout.seek(0)
+        assert fout.read() == "test test"
+
+    def test_softspace_carryover(self):
+        import sys
+        import StringIO
+        fin = StringIO.StringIO()
+        fout = StringIO.StringIO()
+
+        fin.write("Coconuts\n")
+        fin.seek(0)
+
+        sys_stdin_orig = sys.stdin
+        sys_stdout_orig = sys.stdout
+
+        sys.stdin = fin
+        sys.stdout = fout
+
+        print "test",
+        raw_input("test")
+        print "test",
+
+        sys.stdin = sys_stdin_orig
+        sys.stdout = sys_stdout_orig
+
+        fout.seek(0)
+        assert fout.read() == "test testtest"
diff --git a/pypy/module/__pypy__/interp_builders.py b/pypy/module/__pypy__/interp_builders.py
--- a/pypy/module/__pypy__/interp_builders.py
+++ b/pypy/module/__pypy__/interp_builders.py
@@ -16,7 +16,8 @@
 
         def _check_done(self, space):
             if self.builder is None:
-                raise OperationError(space.w_ValueError, space.wrap("Can't operate on a done builder"))
+                raise OperationError(space.w_ValueError, space.wrap(
+                        "Can't operate on a built builder"))
 
         @unwrap_spec(size=int)
         def descr__new__(space, w_subtype, size=-1):
@@ -31,7 +32,8 @@
         def descr_append_slice(self, space, s, start, end):
             self._check_done(space)
             if not 0 <= start <= end <= len(s):
-                raise OperationError(space.w_ValueError, space.wrap("bad start/stop"))
+                raise OperationError(space.w_ValueError, space.wrap(
+                        "bad start/stop"))
             self.builder.append_slice(s, start, end)
 
         def descr_build(self, space):
@@ -40,6 +42,12 @@
             self.builder = None
             return w_s
 
+        def descr_len(self, space):
+            if self.builder is None:
+                raise OperationError(space.w_ValueError, space.wrap(
+                        "no length of built builder"))
+            return space.wrap(self.builder.getlength())
+
     W_Builder.__name__ = "W_%s" % name
     W_Builder.typedef = TypeDef(name,
         __new__ = interp2app(func_with_new_name(
@@ -48,6 +56,7 @@
         append = interp2app(W_Builder.descr_append),
         append_slice = interp2app(W_Builder.descr_append_slice),
         build = interp2app(W_Builder.descr_build),
+        __len__ = interp2app(W_Builder.descr_len),
     )
     W_Builder.typedef.acceptable_as_base_class = False
     return W_Builder
diff --git a/pypy/module/__pypy__/test/test_builders.py b/pypy/module/__pypy__/test/test_builders.py
--- a/pypy/module/__pypy__/test/test_builders.py
+++ b/pypy/module/__pypy__/test/test_builders.py
@@ -38,7 +38,9 @@
         b = StringBuilder()
         b.append("abc")
         b.append("123")
+        assert len(b) == 6
         b.append("you and me")
         s = b.build()
+        raises(ValueError, len, b)
         assert s == "abc123you and me"
-        raises(ValueError, b.build)
\ No newline at end of file
+        raises(ValueError, b.build)
diff --git a/pypy/module/_continuation/interp_continuation.py b/pypy/module/_continuation/interp_continuation.py
--- a/pypy/module/_continuation/interp_continuation.py
+++ b/pypy/module/_continuation/interp_continuation.py
@@ -19,6 +19,11 @@
         #  - normal:      self.sthread != None, not is_empty_handle(self.h)
         #  - finished:    self.sthread != None, is_empty_handle(self.h)
 
+    def __del__(self):
+        sthread = self.sthread
+        if sthread is not None and not sthread.is_empty_handle(self.h):
+            sthread.destroy(self.h)
+
     def check_sthread(self):
         ec = self.space.getexecutioncontext()
         if ec.stacklet_thread is not self.sthread:
@@ -28,6 +33,8 @@
     def descr_init(self, w_callable, __args__):
         if self.sthread is not None:
             raise geterror(self.space, "continulet already __init__ialized")
+        sthread = build_sthread(self.space)
+        workaround_disable_jit(sthread)
         #
         # hackish: build the frame "by hand", passing it the correct arguments
         space = self.space
@@ -41,7 +48,6 @@
         self.bottomframe = bottomframe
         #
         global_state.origin = self
-        sthread = build_sthread(self.space)
         self.sthread = sthread
         h = sthread.new(new_stacklet_callback)
         post_switch(sthread, h)
@@ -71,6 +77,7 @@
                 global_state.clear()
                 raise geterror(self.space, "continulet already finished")
         self.check_sthread()
+        workaround_disable_jit(self.sthread)
         #
         global_state.origin = self
         if to is None:
@@ -259,6 +266,16 @@
         sthread = ec.stacklet_thread = SThread(space, ec)
     return sthread
 
+def workaround_disable_jit(sthread):
+    # A bad workaround to kill the JIT anywhere in this thread.
+    # This forces all the frames.  It's a bad workaround because
+    # it takes O(depth) time, and it will cause some "abort:
+    # vable escape" in the JIT.  The goal is to prevent any frame
+    # from being still virtuals, because the JIT generates code
+    # to un-virtualizable them "on demand" by loading values based
+    # on FORCE_TOKEN, which is an address in the stack.
+    sthread.ec.force_all_frames()
+
 # ____________________________________________________________
 
 def permute(space, args_w):
diff --git a/pypy/module/_file/test/test_file.py b/pypy/module/_file/test/test_file.py
--- a/pypy/module/_file/test/test_file.py
+++ b/pypy/module/_file/test/test_file.py
@@ -1,5 +1,5 @@
 from __future__ import with_statement
-import py
+import py, os, errno
 
 from pypy.conftest import gettestobjspace, option
 
@@ -257,6 +257,39 @@
             assert self.temppath in g.getvalue()
 
 
+class AppTestNonblocking(object):
+    def setup_class(cls):
+        from pypy.module._file.interp_file import W_File
+
+        cls.old_read = os.read
+
+        if option.runappdirect:
+            py.test.skip("works with internals of _file impl on py.py")
+
+        state = [0]
+        def read(fd, n=None):
+            if fd != 42:
+                return cls.old_read(fd, n)
+            if state[0] == 0:
+                state[0] += 1
+                return "xyz"
+            if state[0] < 3:
+                state[0] += 1
+                raise OSError(errno.EAGAIN, "xyz")
+            return ''
+        os.read = read
+        stdin = W_File(cls.space)
+        stdin.file_fdopen(42, "r", 1)
+        stdin.name = '<stdin>'
+        cls.w_stream = stdin
+
+    def teardown_class(cls):
+        os.read = cls.old_read
+
+    def test_nonblocking_file(self):
+        res = self.stream.read()
+        assert res == 'xyz'
+
 class AppTestConcurrency(object):
     # these tests only really make sense on top of a translated pypy-c,
     # because on top of py.py the inner calls to os.write() don't
@@ -374,24 +407,24 @@
         with self.file(self.temppath, 'w') as f:
             f.write('foo')
         assert f.closed
-        
+
         with self.file(self.temppath, 'r') as f:
             s = f.readline()
 
         assert s == "foo"
         assert f.closed
-    
+
     def test_subclass_with(self):
         file = self.file
         class C(file):
             def __init__(self, *args, **kwargs):
                 self.subclass_closed = False
                 file.__init__(self, *args, **kwargs)
-            
+
             def close(self):
                 self.subclass_closed = True
                 file.close(self)
-        
+
         with C(self.temppath, 'w') as f:
             pass
         assert f.subclass_closed
diff --git a/pypy/module/_multiprocessing/interp_connection.py b/pypy/module/_multiprocessing/interp_connection.py
--- a/pypy/module/_multiprocessing/interp_connection.py
+++ b/pypy/module/_multiprocessing/interp_connection.py
@@ -225,7 +225,9 @@
             except OSError:
                 pass
 
-    def __init__(self, fd, flags):
+    def __init__(self, space, fd, flags):
+        if fd == self.INVALID_HANDLE_VALUE or fd < 0:
+            raise OperationError(space.w_IOError, space.wrap("invalid handle %d" % fd))
         W_BaseConnection.__init__(self, flags)
         self.fd = fd
 
@@ -234,7 +236,7 @@
         flags = (readable and READABLE) | (writable and WRITABLE)
 
         self = space.allocate_instance(W_FileConnection, w_subtype)
-        W_FileConnection.__init__(self, fd, flags)
+        W_FileConnection.__init__(self, space, fd, flags)
         return space.wrap(self)
 
     def fileno(self, space):
diff --git a/pypy/module/_multiprocessing/interp_semaphore.py b/pypy/module/_multiprocessing/interp_semaphore.py
--- a/pypy/module/_multiprocessing/interp_semaphore.py
+++ b/pypy/module/_multiprocessing/interp_semaphore.py
@@ -468,6 +468,9 @@
 
         self.count -= 1
 
+    def after_fork(self):
+        self.count = 0
+
     @unwrap_spec(kind=int, maxvalue=int)
     def rebuild(space, w_cls, w_handle, kind, maxvalue):
         self = space.allocate_instance(W_SemLock, w_cls)
@@ -512,6 +515,7 @@
     acquire = interp2app(W_SemLock.acquire),
     release = interp2app(W_SemLock.release),
     _rebuild = interp2app(W_SemLock.rebuild.im_func, as_classmethod=True),
+    _after_fork = interp2app(W_SemLock.after_fork),
     __enter__=interp2app(W_SemLock.enter),
     __exit__=interp2app(W_SemLock.exit),
     SEM_VALUE_MAX=SEM_VALUE_MAX,
diff --git a/pypy/module/_multiprocessing/test/test_connection.py b/pypy/module/_multiprocessing/test/test_connection.py
--- a/pypy/module/_multiprocessing/test/test_connection.py
+++ b/pypy/module/_multiprocessing/test/test_connection.py
@@ -145,3 +145,9 @@
             else:
                 c.close()
         space.delslice(w_connections, space.wrap(0), space.wrap(100))
+
+    def test_bad_fd(self):
+        import _multiprocessing
+
+        raises(IOError, _multiprocessing.Connection, -1)
+        raises(IOError, _multiprocessing.Connection, -15)
\ No newline at end of file
diff --git a/pypy/module/_multiprocessing/test/test_semaphore.py b/pypy/module/_multiprocessing/test/test_semaphore.py
--- a/pypy/module/_multiprocessing/test/test_semaphore.py
+++ b/pypy/module/_multiprocessing/test/test_semaphore.py
@@ -39,6 +39,10 @@
         sem.release()
         assert sem._count() == 0
 
+        sem.acquire()
+        sem._after_fork()
+        assert sem._count() == 0
+
     def test_recursive(self):
         from _multiprocessing import SemLock
         kind = self.RECURSIVE
diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py
--- a/pypy/module/_sre/interp_sre.py
+++ b/pypy/module/_sre/interp_sre.py
@@ -99,6 +99,7 @@
 # SRE_Pattern class
 
 class W_SRE_Pattern(Wrappable):
+    _immutable_fields_ = ["code", "flags"]
 
     def cannot_copy_w(self):
         space = self.space
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -57,6 +57,9 @@
         compile_extra=['-DPy_BUILD_CORE'],
         )
 
+class CConfig2:
+    _compilation_info_ = CConfig._compilation_info_
+
 class CConfig_constants:
     _compilation_info_ = CConfig._compilation_info_
 
@@ -300,9 +303,13 @@
         return unwrapper_raise # used in 'normal' RPython code.
     return decorate
 
-def cpython_struct(name, fields, forward=None):
+def cpython_struct(name, fields, forward=None, level=1):
     configname = name.replace(' ', '__')
-    setattr(CConfig, configname, rffi_platform.Struct(name, fields))
+    if level == 1:
+        config = CConfig
+    else:
+        config = CConfig2
+    setattr(config, configname, rffi_platform.Struct(name, fields))
     if forward is None:
         forward = lltype.ForwardReference()
     TYPES[configname] = forward
@@ -445,9 +452,10 @@
 #              'int*': rffi.INTP}
 
 def configure_types():
-    for name, TYPE in rffi_platform.configure(CConfig).iteritems():
-        if name in TYPES:
-            TYPES[name].become(TYPE)
+    for config in (CConfig, CConfig2):
+        for name, TYPE in rffi_platform.configure(config).iteritems():
+            if name in TYPES:
+                TYPES[name].become(TYPE)
 
 def build_type_checkers(type_name, cls=None):
     """
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -321,6 +321,15 @@
 
 } PyTypeObject;
 
+typedef struct {
+    PyTypeObject ht_type;
+    PyNumberMethods as_number;
+    PyMappingMethods as_mapping;
+    PySequenceMethods as_sequence;
+    PyBufferProcs as_buffer;
+    PyObject *ht_name, *ht_slots;
+} PyHeapTypeObject;
+
 /* Flag bits for printing: */
 #define Py_PRINT_RAW	1	/* No string quotes etc. */
 
diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py
--- a/pypy/module/cpyext/pyobject.py
+++ b/pypy/module/cpyext/pyobject.py
@@ -19,13 +19,42 @@
     basestruct = PyObject.TO
 
     def get_dealloc(self, space):
-        raise NotImplementedError
+        from pypy.module.cpyext.typeobject import subtype_dealloc
+        return llhelper(
+            subtype_dealloc.api_func.functype,
+            subtype_dealloc.api_func.get_wrapper(space))
+
     def allocate(self, space, w_type, itemcount=0):
-        raise NotImplementedError
+        # similar to PyType_GenericAlloc?
+        # except that it's not related to any pypy object.
+
+        pytype = rffi.cast(PyTypeObjectPtr, make_ref(space, w_type))
+        # Don't increase refcount for non-heaptypes
+        if pytype:
+            flags = rffi.cast(lltype.Signed, pytype.c_tp_flags)
+            if not flags & Py_TPFLAGS_HEAPTYPE:
+                Py_DecRef(space, w_type)
+
+        if pytype:
+            size = pytype.c_tp_basicsize
+        else:
+            size = rffi.sizeof(self.basestruct)
+        if itemcount:
+            size += itemcount * pytype.c_tp_itemsize
+        buf = lltype.malloc(rffi.VOIDP.TO, size,
+                            flavor='raw', zero=True)
+        pyobj = rffi.cast(PyObject, buf)
+        pyobj.c_ob_refcnt = 1
+        pyobj.c_ob_type = pytype
+        return pyobj
+
     def attach(self, space, pyobj, w_obj):
-        raise NotImplementedError
+        pass
+
     def realize(self, space, ref):
-        raise NotImplementedError
+        # For most types, a reference cannot exist without
+        # a real interpreter object
+        raise InvalidPointerException(str(ref))
 
 typedescr_cache = {}
 
@@ -40,6 +69,7 @@
     """
 
     tp_basestruct = kw.pop('basestruct', PyObject.TO)
+    tp_alloc      = kw.pop('alloc', None)
     tp_attach     = kw.pop('attach', None)
     tp_realize    = kw.pop('realize', None)
     tp_dealloc    = kw.pop('dealloc', None)
@@ -49,58 +79,24 @@
 
     class CpyTypedescr(BaseCpyTypedescr):
         basestruct = tp_basestruct
-        realize = tp_realize
 
-        def get_dealloc(self, space):
-            if tp_dealloc:
+        if tp_alloc:
+            def allocate(self, space, w_type, itemcount=0):
+                return tp_alloc(space, w_type)
+
+        if tp_dealloc:
+            def get_dealloc(self, space):
                 return llhelper(
                     tp_dealloc.api_func.functype,
                     tp_dealloc.api_func.get_wrapper(space))
-            else:
-                from pypy.module.cpyext.typeobject import subtype_dealloc
-                return llhelper(
-                    subtype_dealloc.api_func.functype,
-                    subtype_dealloc.api_func.get_wrapper(space))
-
-        def allocate(self, space, w_type, itemcount=0):
-            # similar to PyType_GenericAlloc?
-            # except that it's not related to any pypy object.
-
-            pytype = rffi.cast(PyTypeObjectPtr, make_ref(space, w_type))
-            # Don't increase refcount for non-heaptypes
-            if pytype:
-                flags = rffi.cast(lltype.Signed, pytype.c_tp_flags)
-                if not flags & Py_TPFLAGS_HEAPTYPE:
-                    Py_DecRef(space, w_type)
-
-            if pytype:
-                size = pytype.c_tp_basicsize
-            else:
-                size = rffi.sizeof(tp_basestruct)
-            if itemcount:
-                size += itemcount * pytype.c_tp_itemsize
-            buf = lltype.malloc(rffi.VOIDP.TO, size,
-                                flavor='raw', zero=True)
-            pyobj = rffi.cast(PyObject, buf)
-            pyobj.c_ob_refcnt = 1
-            pyobj.c_ob_type = pytype
-            return pyobj
 
         if tp_attach:
             def attach(self, space, pyobj, w_obj):
                 tp_attach(space, pyobj, w_obj)
-        else:
-            def attach(self, space, pyobj, w_obj):
-                pass
 
         if tp_realize:
             def realize(self, space, ref):
                 return tp_realize(space, ref)
-        else:
-            def realize(self, space, ref):
-                # For most types, a reference cannot exist without
-                # a real interpreter object
-                raise InvalidPointerException(str(ref))
     if typedef:
         CpyTypedescr.__name__ = "CpyTypedescr_%s" % (typedef.name,)
 
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -268,6 +268,21 @@
         assert type(obj) is foo.Custom
         assert type(foo.Custom) is foo.MetaType
 
+    def test_heaptype(self):
+        module = self.import_extension('foo', [
+           ("name_by_heaptype", "METH_O",
+            '''
+                 PyHeapTypeObject *heaptype = (PyHeapTypeObject *)args;
+                 Py_INCREF(heaptype->ht_name);
+                 return heaptype->ht_name;
+             '''
+             )
+            ])
+        class C(object):
+            pass
+        assert module.name_by_heaptype(C) == "C"
+        
+
 class TestTypes(BaseApiTest):
     def test_type_attributes(self, space, api):
         w_class = space.appexec([], """():
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -11,7 +11,7 @@
     generic_cpy_call, Py_TPFLAGS_READY, Py_TPFLAGS_READYING,
     Py_TPFLAGS_HEAPTYPE, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL,
     Py_TPFLAGS_HAVE_GETCHARBUFFER,
-    build_type_checkers)
+    build_type_checkers, PyObjectFields)
 from pypy.module.cpyext.pyobject import (
     PyObject, make_ref, create_ref, from_ref, get_typedescr, make_typedescr,
     track_reference, RefcountState, borrow_from)
@@ -25,7 +25,7 @@
 from pypy.module.cpyext.structmember import PyMember_GetOne, PyMember_SetOne
 from pypy.module.cpyext.typeobjectdefs import (
     PyTypeObjectPtr, PyTypeObject, PyGetSetDef, PyMemberDef, newfunc,
-    PyNumberMethods, PySequenceMethods, PyBufferProcs)
+    PyNumberMethods, PyMappingMethods, PySequenceMethods, PyBufferProcs)
 from pypy.module.cpyext.slotdefs import (
     slotdefs_for_tp_slots, slotdefs_for_wrappers, get_slot_tp_function)
 from pypy.interpreter.error import OperationError
@@ -39,6 +39,19 @@
 
 PyType_Check, PyType_CheckExact = build_type_checkers("Type", "w_type")
 
+PyHeapTypeObjectStruct = lltype.ForwardReference()
+PyHeapTypeObject = lltype.Ptr(PyHeapTypeObjectStruct)
+PyHeapTypeObjectFields = (
+    ("ht_type", PyTypeObject),
+    ("ht_name", PyObject),
+    ("as_number", PyNumberMethods),
+    ("as_mapping", PyMappingMethods),
+    ("as_sequence", PySequenceMethods),
+    ("as_buffer", PyBufferProcs),
+    )
+cpython_struct("PyHeapTypeObject", PyHeapTypeObjectFields, PyHeapTypeObjectStruct,
+               level=2)
+
 class W_GetSetPropertyEx(GetSetProperty):
     def __init__(self, getset, w_type):
         self.getset = getset
@@ -136,6 +149,8 @@
             assert len(slot_names) == 2
             struct = getattr(pto, slot_names[0])
             if not struct:
+                assert not space.config.translating
+                assert not pto.c_tp_flags & Py_TPFLAGS_HEAPTYPE
                 if slot_names[0] == 'c_tp_as_number':
                     STRUCT_TYPE = PyNumberMethods
                 elif slot_names[0] == 'c_tp_as_sequence':
@@ -301,6 +316,7 @@
 
     make_typedescr(space.w_type.instancetypedef,
                    basestruct=PyTypeObject,
+                   alloc=type_alloc,
                    attach=type_attach,
                    realize=type_realize,
                    dealloc=type_dealloc)
@@ -319,11 +335,13 @@
     track_reference(space, lltype.nullptr(PyObject.TO), space.w_type)
     track_reference(space, lltype.nullptr(PyObject.TO), space.w_object)
     track_reference(space, lltype.nullptr(PyObject.TO), space.w_tuple)
+    track_reference(space, lltype.nullptr(PyObject.TO), space.w_str)
 
     # create the objects
     py_type = create_ref(space, space.w_type)
     py_object = create_ref(space, space.w_object)
     py_tuple = create_ref(space, space.w_tuple)
+    py_str = create_ref(space, space.w_str)
 
     # form cycles
     pto_type = rffi.cast(PyTypeObjectPtr, py_type)
@@ -340,10 +358,15 @@
     pto_object.c_tp_bases.c_ob_type = pto_tuple
     pto_tuple.c_tp_bases.c_ob_type = pto_tuple
 
+    for typ in (py_type, py_object, py_tuple, py_str):
+        heaptype = rffi.cast(PyHeapTypeObject, typ)
+        heaptype.c_ht_name.c_ob_type = pto_type
+
     # Restore the mapping
     track_reference(space, py_type, space.w_type, replace=True)
     track_reference(space, py_object, space.w_object, replace=True)
     track_reference(space, py_tuple, space.w_tuple, replace=True)
+    track_reference(space, py_str, space.w_str, replace=True)
 
 
 @cpython_api([PyObject], lltype.Void, external=False)
@@ -416,17 +439,34 @@
     Py_DecRef(space, obj_pto.c_tp_cache) # let's do it like cpython
     Py_DecRef(space, obj_pto.c_tp_dict)
     if obj_pto.c_tp_flags & Py_TPFLAGS_HEAPTYPE:
-        if obj_pto.c_tp_as_buffer:
-            lltype.free(obj_pto.c_tp_as_buffer, flavor='raw')
-        if obj_pto.c_tp_as_number:
-            lltype.free(obj_pto.c_tp_as_number, flavor='raw')
-        if obj_pto.c_tp_as_sequence:
-            lltype.free(obj_pto.c_tp_as_sequence, flavor='raw')
+        heaptype = rffi.cast(PyHeapTypeObject, obj)
+        Py_DecRef(space, heaptype.c_ht_name)
         Py_DecRef(space, base_pyo)
-        rffi.free_charp(obj_pto.c_tp_name)
         PyObject_dealloc(space, obj)
 
 
+def type_alloc(space, w_metatype):
+    size = rffi.sizeof(PyHeapTypeObject)
+    metatype = rffi.cast(PyTypeObjectPtr, make_ref(space, w_metatype))
+    # Don't increase refcount for non-heaptypes
+    if metatype:
+        flags = rffi.cast(lltype.Signed, metatype.c_tp_flags)
+        if not flags & Py_TPFLAGS_HEAPTYPE:
+            Py_DecRef(space, w_metatype)
+
+    heaptype = lltype.malloc(PyHeapTypeObject.TO,
+                             flavor='raw', zero=True)
+    pto = heaptype.c_ht_type
+    pto.c_ob_refcnt = 1
+    pto.c_ob_type = metatype
+    pto.c_tp_flags |= Py_TPFLAGS_HEAPTYPE
+    pto.c_tp_as_number = heaptype.c_as_number
+    pto.c_tp_as_sequence = heaptype.c_as_sequence
+    pto.c_tp_as_mapping = heaptype.c_as_mapping
+    pto.c_tp_as_buffer = heaptype.c_as_buffer
+    
+    return rffi.cast(PyObject, heaptype)
+
 def type_attach(space, py_obj, w_type):
     """
     Fills a newly allocated PyTypeObject from an existing type.
@@ -445,12 +485,18 @@
     if space.is_w(w_type, space.w_str):
         setup_string_buffer_procs(space, pto)
 
-    pto.c_tp_flags |= Py_TPFLAGS_HEAPTYPE
     pto.c_tp_free = llhelper(PyObject_Del.api_func.functype,
             PyObject_Del.api_func.get_wrapper(space))
     pto.c_tp_alloc = llhelper(PyType_GenericAlloc.api_func.functype,
             PyType_GenericAlloc.api_func.get_wrapper(space))
-    pto.c_tp_name = rffi.str2charp(w_type.getname(space))
+    if pto.c_tp_flags & Py_TPFLAGS_HEAPTYPE:
+        w_typename = space.getattr(w_type, space.wrap('__name__'))
+        heaptype = rffi.cast(PyHeapTypeObject, pto)
+        heaptype.c_ht_name = make_ref(space, w_typename)
+        from pypy.module.cpyext.stringobject import PyString_AsString
+        pto.c_tp_name = PyString_AsString(space, heaptype.c_ht_name)
+    else:
+        pto.c_tp_name = rffi.str2charp(w_type.getname(space))
     pto.c_tp_basicsize = -1 # hopefully this makes malloc bail out
     pto.c_tp_itemsize = 0
     # uninitialized fields:
diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -23,6 +23,8 @@
         ("arccos", "arccos"),
         ("arcsin", "arcsin"),
         ("arctan", "arctan"),
+        ("arcsinh", "arcsinh"),
+        ("arctanh", "arctanh"),
         ("copysign", "copysign"),
         ("cos", "cos"),
         ("divide", "divide"),
@@ -50,4 +52,6 @@
     appleveldefs = {
         'average': 'app_numpy.average',
         'mean': 'app_numpy.mean',
+        'inf': 'app_numpy.inf',
+        'e': 'app_numpy.e',
     }
diff --git a/pypy/module/micronumpy/app_numpy.py b/pypy/module/micronumpy/app_numpy.py
--- a/pypy/module/micronumpy/app_numpy.py
+++ b/pypy/module/micronumpy/app_numpy.py
@@ -1,5 +1,11 @@
+import math
+
 import numpy
 
+
+inf = float("inf")
+e = math.e
+
 def average(a):
     # This implements a weighted average, for now we don't implement the
     # weighting, just the average part!
@@ -8,4 +14,4 @@
 def mean(a):
     if not hasattr(a, "mean"):
         a = numpy.array(a)
-    return a.mean()
\ No newline at end of file
+    return a.mean()
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -7,13 +7,14 @@
 from pypy.interpreter.typedef import TypeDef, interp_attrproperty, GetSetProperty
 from pypy.module.micronumpy import signature
 from pypy.objspace.std.floatobject import float2string
-from pypy.rlib import rfloat
-from pypy.rlib.rarithmetic import widen
+from pypy.rlib import rarithmetic, rfloat
+from pypy.rlib.rarithmetic import LONG_BIT, widen
 from pypy.rlib.objectmodel import specialize, enforceargs
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.rpython.lltypesystem import lltype, rffi
 
 
+UNSIGNEDLTR = "u"
 SIGNEDLTR = "i"
 BOOLLTR = "b"
 FLOATINGLTR = "f"
@@ -61,7 +62,10 @@
             self.val = val
 
         def wrap(self, space):
-            return space.wrap(self.val)
+            val = self.val
+            if valtype is rarithmetic.r_singlefloat:
+                val = float(val)
+            return space.wrap(val)
 
         def convert_to(self, dtype):
             return dtype.adapt_val(self.val)
@@ -145,7 +149,7 @@
         return self.adapt_val(func(self, self.for_computation(self.unbox(v))))
     return impl
 
-class ArithmaticTypeMixin(object):
+class ArithmeticTypeMixin(object):
     _mixin_ = True
 
     @binop
@@ -157,9 +161,6 @@
     @binop
     def mul(self, v1, v2):
         return v1 * v2
-    @binop
-    def div(self, v1, v2):
-        return v1 / v2
 
     @unaryop
     def pos(self, v):
@@ -200,12 +201,26 @@
         return v1 >= v2
 
 
-class FloatArithmeticDtype(ArithmaticTypeMixin):
+class FloatArithmeticDtype(ArithmeticTypeMixin):
     _mixin_ = True
 
+    def unwrap(self, space, w_item):
+        return self.adapt_val(space.float_w(space.float(w_item)))
+
     def for_computation(self, v):
-        return v
+        return float(v)
 
+    def str_format(self, item):
+        return float2string(self.for_computation(self.unbox(item)), 'g', rfloat.DTSF_STR_PRECISION)
+
+    @binop
+    def div(self, v1, v2):
+        try:
+            return v1 / v2
+        except ZeroDivisionError:
+            if v1 == v2 == 0.0:
+                return rfloat.NAN
+            return rfloat.copysign(rfloat.INFINITY, v1 * v2)
     @binop
     def mod(self, v1, v2):
         return math.fmod(v1, v2)
@@ -250,19 +265,29 @@
         return math.tan(v)
     @unaryop
     def arcsin(self, v):
-        if v < -1.0 or v > 1.0:
+        if not -1.0 <= v <= 1.0:
             return rfloat.NAN
         return math.asin(v)
     @unaryop
     def arccos(self, v):
-        if v < -1.0 or v > 1.0:
+        if not -1.0 <= v <= 1.0:
             return rfloat.NAN
         return math.acos(v)
     @unaryop
     def arctan(self, v):
         return math.atan(v)
+    @unaryop
+    def arcsinh(self, v):
+        return math.asinh(v)
+    @unaryop
+    def arctanh(self, v):
+        if v == 1.0 or v == -1.0:
+            return math.copysign(rfloat.INFINITY, v)
+        if not -1.0 < v < 1.0:
+            return rfloat.NAN
+        return math.atanh(v)
 
-class IntegerArithmeticDtype(ArithmaticTypeMixin):
+class IntegerArithmeticDtype(ArithmeticTypeMixin):
     _mixin_ = True
 
     def unwrap(self, space, w_item):
@@ -271,10 +296,21 @@
     def for_computation(self, v):
         return widen(v)
 
+    def str_format(self, item):
+        return str(widen(self.unbox(item)))
+
+    @binop
+    def div(self, v1, v2):
+        if v2 == 0:
+            return 0
+        return v1 / v2
     @binop
     def mod(self, v1, v2):
         return v1 % v2
 
+class SignedIntegerArithmeticDtype(IntegerArithmeticDtype):
+    _mixin_ = True
+
     @unaryop
     def sign(self, v):
         if v > 0:
@@ -285,17 +321,22 @@
             assert v == 0
             return 0
 
-    def str_format(self, item):
-        return str(widen(self.unbox(item)))
+class UnsignedIntegerArithmeticDtype(IntegerArithmeticDtype):
+    _mixin_ = True
+
+    @unaryop
+    def sign(self, v):
+        return int(v != 0)
+
 
 W_BoolDtype = create_low_level_dtype(
     num = 0, kind = BOOLLTR, name = "bool",
-    aliases = ["?"],
+    aliases = ["?", "bool", "bool8"],
     applevel_types = ["bool"],
     T = lltype.Bool,
     valtype = bool,
 )
-class W_BoolDtype(IntegerArithmeticDtype, W_BoolDtype):
+class W_BoolDtype(SignedIntegerArithmeticDtype, W_BoolDtype):
     def unwrap(self, space, w_item):
         return self.adapt_val(space.is_true(w_item))
 
@@ -308,67 +349,138 @@
 
 W_Int8Dtype = create_low_level_dtype(
     num = 1, kind = SIGNEDLTR, name = "int8",
-    aliases = ["int8"],
+    aliases = ["b", "int8", "i1"],
     applevel_types = [],
     T = rffi.SIGNEDCHAR,
     valtype = rffi.SIGNEDCHAR._type,
     expected_size = 1,
 )
-class W_Int8Dtype(IntegerArithmeticDtype, W_Int8Dtype):
+class W_Int8Dtype(SignedIntegerArithmeticDtype, W_Int8Dtype):
+    pass
+
+W_UInt8Dtype = create_low_level_dtype(
+    num = 2, kind = UNSIGNEDLTR, name = "uint8",
+    aliases = ["B", "uint8", "I1"],
+    applevel_types = [],
+    T = rffi.UCHAR,
+    valtype = rffi.UCHAR._type,
+    expected_size = 1,
+)
+class W_UInt8Dtype(UnsignedIntegerArithmeticDtype, W_UInt8Dtype):
     pass
 
 W_Int16Dtype = create_low_level_dtype(
     num = 3, kind = SIGNEDLTR, name = "int16",
-    aliases = ["int16"],
+    aliases = ["h", "int16", "i2"],
     applevel_types = [],
     T = rffi.SHORT,
     valtype = rffi.SHORT._type,
     expected_size = 2,
 )
-class W_Int16Dtype(IntegerArithmeticDtype, W_Int16Dtype):
+class W_Int16Dtype(SignedIntegerArithmeticDtype, W_Int16Dtype):
+    pass
+
+W_UInt16Dtype = create_low_level_dtype(
+    num = 4, kind = UNSIGNEDLTR, name = "uint16",
+    aliases = ["H", "uint16", "I2"],
+    applevel_types = [],
+    T = rffi.USHORT,
+    valtype = rffi.USHORT._type,
+    expected_size = 2,
+)
+class W_UInt16Dtype(UnsignedIntegerArithmeticDtype, W_UInt16Dtype):
     pass
 
 W_Int32Dtype = create_low_level_dtype(
     num = 5, kind = SIGNEDLTR, name = "int32",
-    aliases = ["i"],
+    aliases = ["i", "int32", "i4"],
     applevel_types = [],
     T = rffi.INT,
     valtype = rffi.INT._type,
     expected_size = 4,
 )
-class W_Int32Dtype(IntegerArithmeticDtype, W_Int32Dtype):
+class W_Int32Dtype(SignedIntegerArithmeticDtype, W_Int32Dtype):
+    pass
+
+W_UInt32Dtype = create_low_level_dtype(
+    num = 6, kind = UNSIGNEDLTR, name = "uint32",
+    aliases = ["I", "uint32", "I4"],
+    applevel_types = [],
+    T = rffi.UINT,
+    valtype = rffi.UINT._type,
+    expected_size = 4,
+)
+class W_UInt32Dtype(UnsignedIntegerArithmeticDtype, W_UInt32Dtype):
     pass
 
 W_Int64Dtype = create_low_level_dtype(
     num = 9, kind = SIGNEDLTR, name = "int64",
-    aliases = [],
+    aliases = ["q", "int64", "i8"],
     applevel_types = ["long"],
     T = rffi.LONGLONG,
     valtype = rffi.LONGLONG._type,
     expected_size = 8,
 )
-class W_Int64Dtype(IntegerArithmeticDtype, W_Int64Dtype):
+class W_Int64Dtype(SignedIntegerArithmeticDtype, W_Int64Dtype):
+    pass
+
+W_UInt64Dtype = create_low_level_dtype(
+    num = 10, kind = UNSIGNEDLTR, name = "uint64",
+    aliases = ["Q", "uint64", "I8"],
+    applevel_types = [],
+    T = rffi.ULONGLONG,
+    valtype = rffi.ULONGLONG._type,
+    expected_size = 8,
+)
+class W_UInt64Dtype(UnsignedIntegerArithmeticDtype, W_UInt64Dtype):
+    pass
+
+if LONG_BIT == 32:
+    long_dtype = W_Int32Dtype
+    ulong_dtype = W_UInt32Dtype
+elif LONG_BIT == 64:
+    long_dtype = W_Int64Dtype
+    ulong_dtype = W_UInt64Dtype
+else:
+    assert False
+
+class W_LongDtype(long_dtype):
+    num = 7
+    aliases = ["l"]
+    applevel_types = ["int"]
+
+class W_ULongDtype(ulong_dtype):
+    num = 8
+    aliases = ["L"]
+
+W_Float32Dtype = create_low_level_dtype(
+    num = 11, kind = FLOATINGLTR, name = "float32",
+    aliases = ["f", "float32", "f4"],
+    applevel_types = [],
+    T = lltype.SingleFloat,
+    valtype = rarithmetic.r_singlefloat,
+    expected_size = 4,
+)
+class W_Float32Dtype(FloatArithmeticDtype, W_Float32Dtype):
     pass
 
 W_Float64Dtype = create_low_level_dtype(
     num = 12, kind = FLOATINGLTR, name = "float64",
-    aliases = [],
+    aliases = ["d", "float64", "f8"],
     applevel_types = ["float"],
     T = lltype.Float,
     valtype = float,
     expected_size = 8,
 )
 class W_Float64Dtype(FloatArithmeticDtype, W_Float64Dtype):
-    def unwrap(self, space, w_item):
-        return self.adapt_val(space.float_w(space.float(w_item)))
-
-    def str_format(self, item):
-        return float2string(self.unbox(item), 'g', rfloat.DTSF_STR_PRECISION)
+    pass
 
 ALL_DTYPES = [
     W_BoolDtype,
-    W_Int8Dtype, W_Int16Dtype, W_Int32Dtype, W_Int64Dtype,
-    W_Float64Dtype
+    W_Int8Dtype, W_UInt8Dtype, W_Int16Dtype, W_UInt16Dtype,
+    W_Int32Dtype, W_UInt32Dtype, W_LongDtype, W_ULongDtype,
+    W_Int64Dtype, W_UInt64Dtype,
+    W_Float32Dtype, W_Float64Dtype,
 ]
 
 dtypes_by_alias = unrolling_iterable([
@@ -395,6 +507,7 @@
 
     num = interp_attrproperty("num", cls=W_Dtype),
     kind = interp_attrproperty("kind", cls=W_Dtype),
+    itemsize = interp_attrproperty("num_bytes", cls=W_Dtype),
     shape = GetSetProperty(W_Dtype.descr_get_shape),
 )
 W_Dtype.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -4,6 +4,7 @@
 from pypy.interpreter.typedef import TypeDef, GetSetProperty, interp_attrproperty
 from pypy.module.micronumpy import interp_dtype, signature
 from pypy.rlib import jit
+from pypy.rlib.rarithmetic import LONG_BIT
 from pypy.tool.sourcetools import func_with_new_name
 
 
@@ -180,23 +181,56 @@
 
     # Everything promotes to float, and bool promotes to everything.
     if dt2.kind == interp_dtype.FLOATINGLTR or dt1.kind == interp_dtype.BOOLLTR:
+        # Float32 + 8-bit int = Float64
+        if dt2.num == 11 and dt1.num_bytes >= 4:
+            return space.fromcache(interp_dtype.W_Float64Dtype)
         return dt2
 
-    assert False
+    # for now this means mixing signed and unsigned
+    if dt2.kind == interp_dtype.SIGNEDLTR:
+        # if dt2 has a greater number of bytes, then just go with it
+        if dt1.num_bytes < dt2.num_bytes:
+            return dt2
+        # we need to promote both dtypes
+        dtypenum = dt2.num + 2
+    else:
+        # increase to the next signed type (or to float)
+        dtypenum = dt2.num + 1
+        # UInt64 + signed = Float64
+        if dt2.num == 10:
+            dtypenum += 1
+    newdtype = interp_dtype.ALL_DTYPES[dtypenum]
+
+    if newdtype.num_bytes > dt2.num_bytes or newdtype.kind == interp_dtype.FLOATINGLTR:
+        return space.fromcache(newdtype)
+    else:
+        # we only promoted to long on 32-bit or to longlong on 64-bit
+        # this is really for dealing with the Long and Ulong dtypes
+        if LONG_BIT == 32:
+            dtypenum += 2
+        else:
+            dtypenum += 3
+        return space.fromcache(interp_dtype.ALL_DTYPES[dtypenum])
 
 def find_unaryop_result_dtype(space, dt, promote_to_float=False,
     promote_bools=False, promote_to_largest=False):
     if promote_bools and (dt.kind == interp_dtype.BOOLLTR):
         return space.fromcache(interp_dtype.W_Int8Dtype)
     if promote_to_float:
+        if dt.kind == interp_dtype.FLOATINGLTR:
+            return dt
+        if dt.num >= 5:
+            return space.fromcache(interp_dtype.W_Float64Dtype)
         for bytes, dtype in interp_dtype.dtypes_by_num_bytes:
-            if dtype.kind == interp_dtype.FLOATINGLTR and dtype.num_bytes >= dt.num_bytes:
+            if dtype.kind == interp_dtype.FLOATINGLTR and dtype.num_bytes > dt.num_bytes:
                 return space.fromcache(dtype)
     if promote_to_largest:
         if dt.kind == interp_dtype.BOOLLTR or dt.kind == interp_dtype.SIGNEDLTR:
             return space.fromcache(interp_dtype.W_Int64Dtype)
         elif dt.kind == interp_dtype.FLOATINGLTR:
             return space.fromcache(interp_dtype.W_Float64Dtype)
+        elif dt.kind == interp_dtype.UNSIGNEDLTR:
+            return space.fromcache(interp_dtype.W_UInt64Dtype)
         else:
             assert False
     return dt
@@ -205,15 +239,23 @@
     w_type = space.type(w_obj)
 
     bool_dtype = space.fromcache(interp_dtype.W_BoolDtype)
+    long_dtype = space.fromcache(interp_dtype.W_LongDtype)
     int64_dtype = space.fromcache(interp_dtype.W_Int64Dtype)
 
     if space.is_w(w_type, space.w_bool):
-        if current_guess is None:
+        if current_guess is None or current_guess is bool_dtype:
             return bool_dtype
+        return current_guess
     elif space.is_w(w_type, space.w_int):
         if (current_guess is None or current_guess is bool_dtype or
-            current_guess is int64_dtype):
+            current_guess is long_dtype):
+            return long_dtype
+        return current_guess
+    elif space.is_w(w_type, space.w_long):
+        if (current_guess is None or current_guess is bool_dtype or
+            current_guess is long_dtype or current_guess is int64_dtype):
             return int64_dtype
+        return current_guess
     return space.fromcache(interp_dtype.W_Float64Dtype)
 
 
@@ -225,7 +267,9 @@
         def impl(res_dtype, lvalue, rvalue):
             res = getattr(res_dtype, op_name)(lvalue, rvalue)
             if comparison_func:
-                res = space.fromcache(interp_dtype.W_BoolDtype).box(res)
+                booldtype = space.fromcache(interp_dtype.W_BoolDtype)
+                assert isinstance(booldtype, interp_dtype.W_BoolDtype)
+                res = booldtype.box(res)
             return res
     return func_with_new_name(impl, ufunc_name)
 
@@ -268,6 +312,8 @@
             ("arcsin", "arcsin", 1, {"promote_to_float": True}),
             ("arccos", "arccos", 1, {"promote_to_float": True}),
             ("arctan", "arctan", 1, {"promote_to_float": True}),
+            ("arcsinh", "arcsinh", 1, {"promote_to_float": True}),
+            ("arctanh", "arctanh", 1, {"promote_to_float": True}),
         ]:
             self.add_ufunc(space, *ufunc_def)
 
@@ -277,7 +323,7 @@
 
         identity = extra_kwargs.get("identity")
         if identity is not None:
-            identity = space.fromcache(interp_dtype.W_Int64Dtype).adapt_val(identity)
+            identity = space.fromcache(interp_dtype.W_LongDtype).adapt_val(identity)
         extra_kwargs["identity"] = identity
 
         func = ufunc_dtype_caller(space, ufunc_name, op_name, argcount,
@@ -290,4 +336,4 @@
         setattr(self, ufunc_name, ufunc)
 
 def get(space):
-    return space.fromcache(UfuncState)
\ No newline at end of file
+    return space.fromcache(UfuncState)
diff --git a/pypy/module/micronumpy/test/test_base.py b/pypy/module/micronumpy/test/test_base.py
--- a/pypy/module/micronumpy/test/test_base.py
+++ b/pypy/module/micronumpy/test/test_base.py
@@ -64,18 +64,46 @@
     def test_unaryops(self, space):
         bool_dtype = space.fromcache(interp_dtype.W_BoolDtype)
         int8_dtype = space.fromcache(interp_dtype.W_Int8Dtype)
+        uint8_dtype = space.fromcache(interp_dtype.W_UInt8Dtype)
+        int16_dtype = space.fromcache(interp_dtype.W_Int16Dtype)
+        uint16_dtype = space.fromcache(interp_dtype.W_UInt16Dtype)
         int32_dtype = space.fromcache(interp_dtype.W_Int32Dtype)
+        uint32_dtype = space.fromcache(interp_dtype.W_UInt32Dtype)
+        long_dtype = space.fromcache(interp_dtype.W_LongDtype)
+        ulong_dtype = space.fromcache(interp_dtype.W_ULongDtype)
+        int64_dtype = space.fromcache(interp_dtype.W_Int64Dtype)
+        uint64_dtype = space.fromcache(interp_dtype.W_UInt64Dtype)
+        float32_dtype = space.fromcache(interp_dtype.W_Float32Dtype)
         float64_dtype = space.fromcache(interp_dtype.W_Float64Dtype)
 
-        # Normal rules, everythign returns itself
+        # Normal rules, everything returns itself
         assert find_unaryop_result_dtype(space, bool_dtype) is bool_dtype
         assert find_unaryop_result_dtype(space, int8_dtype) is int8_dtype
+        assert find_unaryop_result_dtype(space, uint8_dtype) is uint8_dtype
+        assert find_unaryop_result_dtype(space, int16_dtype) is int16_dtype
+        assert find_unaryop_result_dtype(space, uint16_dtype) is uint16_dtype
         assert find_unaryop_result_dtype(space, int32_dtype) is int32_dtype
+        assert find_unaryop_result_dtype(space, uint32_dtype) is uint32_dtype
+        assert find_unaryop_result_dtype(space, long_dtype) is long_dtype
+        assert find_unaryop_result_dtype(space, ulong_dtype) is ulong_dtype
+        assert find_unaryop_result_dtype(space, int64_dtype) is int64_dtype
+        assert find_unaryop_result_dtype(space, uint64_dtype) is uint64_dtype
+        assert find_unaryop_result_dtype(space, float32_dtype) is float32_dtype
         assert find_unaryop_result_dtype(space, float64_dtype) is float64_dtype
 
         # Coerce to floats, some of these will eventually be float16, or
         # whatever our smallest float type is.
-        assert find_unaryop_result_dtype(space, bool_dtype, promote_to_float=True) is float64_dtype
-        assert find_unaryop_result_dtype(space, int8_dtype, promote_to_float=True) is float64_dtype
+        assert find_unaryop_result_dtype(space, bool_dtype, promote_to_float=True) is float32_dtype # will be float16 if we ever put that in
+        assert find_unaryop_result_dtype(space, int8_dtype, promote_to_float=True) is float32_dtype # will be float16 if we ever put that in
+        assert find_unaryop_result_dtype(space, uint8_dtype, promote_to_float=True) is float32_dtype # will be float16 if we ever put that in
+        assert find_unaryop_result_dtype(space, int16_dtype, promote_to_float=True) is float32_dtype
+        assert find_unaryop_result_dtype(space, uint16_dtype, promote_to_float=True) is float32_dtype
         assert find_unaryop_result_dtype(space, int32_dtype, promote_to_float=True) is float64_dtype
-        assert find_unaryop_result_dtype(space, float64_dtype, promote_to_float=True) is float64_dtype
\ No newline at end of file
+        assert find_unaryop_result_dtype(space, uint32_dtype, promote_to_float=True) is float64_dtype
+        assert find_unaryop_result_dtype(space, int64_dtype, promote_to_float=True) is float64_dtype
+        assert find_unaryop_result_dtype(space, uint64_dtype, promote_to_float=True) is float64_dtype
+        assert find_unaryop_result_dtype(space, float32_dtype, promote_to_float=True) is float32_dtype
+        assert find_unaryop_result_dtype(space, float64_dtype, promote_to_float=True) is float64_dtype
+
+        # promote bools, happens with sign ufunc
+        assert find_unaryop_result_dtype(space, bool_dtype, promote_bools=True) is int8_dtype
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -17,6 +17,7 @@
         from numpy import dtype
 
         assert dtype(bool).num == 0
+        assert dtype(int).num == 7
         assert dtype(long).num == 9
         assert dtype(float).num == 12
 
@@ -81,6 +82,48 @@
             assert isinstance(a[i], (int, long))
             assert a[1] == 1
 
+    def test_overflow(self):
+        from numpy import array, dtype
+        assert array([128], 'b')[0] == -128
+        assert array([256], 'B')[0] == 0
+        assert array([32768], 'h')[0] == -32768
+        assert array([65536], 'H')[0] == 0
+        if dtype('l').itemsize == 4: # 32-bit
+            raises(OverflowError, "array([2**32/2], 'i')")
+            raises(OverflowError, "array([2**32], 'I')")
+        raises(OverflowError, "array([2**64/2], 'q')")
+        raises(OverflowError, "array([2**64], 'Q')")
+
+    def test_bool_binop_types(self):
+        from numpy import array, dtype
+        types = ('?','b','B','h','H','i','I','l','L','q','Q','f','d')
+        N = len(types)
+        a = array([True], '?')
+        for t in types:
+            assert (a + array([0], t)).dtype is dtype(t)
+
+    def test_binop_types(self):
+        from numpy import array, dtype
+        tests = [('b','B','h'), ('b','h','h'), ('b','H','i'), ('b','i','i'),
+                 ('b','l','l'), ('b','q','q'), ('b','Q','d'), ('B','h','h'),
+                 ('B','H','H'), ('B','i','i'), ('B','I','I'), ('B','l','l'),
+                 ('B','L','L'), ('B','q','q'), ('B','Q','Q'), ('h','H','i'),
+                 ('h','i','i'), ('h','l','l'), ('h','q','q'), ('h','Q','d'),
+                 ('H','i','i'), ('H','I','I'), ('H','l','l'), ('H','L','L'),
+                 ('H','q','q'), ('H','Q','Q'), ('i','l','l'), ('i','q','q'),
+                 ('i','Q','d'), ('I','L','L'), ('I','q','q'), ('I','Q','Q'),
+                 ('q','Q','d'), ('b','f','f'), ('B','f','f'), ('h','f','f'),
+                 ('H','f','f'), ('i','f','d'), ('I','f','d'), ('l','f','d'),
+                 ('L','f','d'), ('q','f','d'), ('Q','f','d'), ('q','d','d')]
+        if dtype('i').itemsize == dtype('l').itemsize: # 32-bit
+            tests.extend([('b','I','q'), ('b','L','q'), ('h','I','q'),
+                          ('h','L','q'), ('i','I','q'), ('i','L','q')])
+        else:
+            tests.extend([('b','I','l'), ('b','L','d'), ('h','I','l'),
+                          ('h','L','d'), ('i','I','l'), ('i','L','d')])
+        for d1, d2, dout in tests:
+            assert (array([1], d1) + array([1], d2)).dtype is dtype(dout)
+
     def test_add_int8(self):
         from numpy import array, dtype
 
@@ -99,6 +142,15 @@
         for i in range(5):
             assert b[i] == i * 2
 
+    def test_add_uint32(self):
+        from numpy import array, dtype
+
+        a = array(range(5), dtype="I")
+        b = a + a
+        assert b.dtype is dtype("I")
+        for i in range(5):
+            assert b[i] == i * 2
+
     def test_shape(self):
         from numpy import dtype
 
diff --git a/pypy/module/micronumpy/test/test_module.py b/pypy/module/micronumpy/test/test_module.py
--- a/pypy/module/micronumpy/test/test_module.py
+++ b/pypy/module/micronumpy/test/test_module.py
@@ -10,4 +10,12 @@
     def test_average(self):
         from numpy import array, average
         assert average(range(10)) == 4.5
-        assert average(array(range(10))) == 4.5
\ No newline at end of file
+        assert average(array(range(10))) == 4.5
+
+    def test_constants(self):
+        import math
+        from numpy import inf, e
+        assert type(inf) is float
+        assert inf == float("inf")
+        assert e == math.e
+        assert type(e) is float
\ No newline at end of file
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -285,7 +285,9 @@
             assert b[i] == i * 5
 
     def test_div(self):
-        from numpy import array, dtype
+        from math import isnan
+        from numpy import array, dtype, inf
+
         a = array(range(1, 6))
         b = a / a
         for i in range(5):
@@ -297,6 +299,24 @@
         for i in range(5):
             assert b[i] == 1
 
+        a = array([-1, 0, 1])
+        b = array([0, 0, 0])
+        c = a / b
+        assert (c == [0, 0, 0]).all()
+
+        a = array([-1.0, 0.0, 1.0])
+        b = array([0.0, 0.0, 0.0])
+        c = a / b
+        assert c[0] == -inf
+        assert isnan(c[1])
+        assert c[2] == inf
+
+        b = array([-0.0, -0.0, -0.0])
+        c = a / b
+        assert c[0] == inf
+        assert isnan(c[1])
+        assert c[2] == -inf
+
     def test_div_other(self):
         from numpy import array
         a = array(range(5))
@@ -551,8 +571,10 @@
         from numpy import array, dtype
 
         assert array([True]).dtype is dtype(bool)
-        assert array([True, 1]).dtype is dtype(long)
-        assert array([1, 2, 3]).dtype is dtype(long)
+        assert array([True, False]).dtype is dtype(bool)
+        assert array([True, 1]).dtype is dtype(int)
+        assert array([1, 2, 3]).dtype is dtype(int)
+        assert array([1L, 2, 3]).dtype is dtype(long)
         assert array([1.2, True]).dtype is dtype(float)
         assert array([1.2, 5]).dtype is dtype(float)
         assert array([]).dtype is dtype(float)
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -234,7 +234,7 @@
             assert b[i] == math.sin(a[i])
 
         a = sin(array([True, False], dtype=bool))
-        assert a[0] == sin(1)
+        assert abs(a[0] - sin(1)) < 1e-7 # a[0] will be less precise
         assert a[1] == 0.0
 
     def test_cos(self):
@@ -298,6 +298,25 @@
         b = arctan(a)
         assert math.isnan(b[0])
 
+    def test_arcsinh(self):
+        import math
+        from numpy import arcsinh, inf
+
+        for v in [inf, -inf, 1.0, math.e]:
+            assert math.asinh(v) == arcsinh(v)
+        assert math.isnan(arcsinh(float("nan")))
+
+    def test_arctanh(self):
+        import math
+        from numpy import arctanh
+
+        for v in [.99, .5, 0, -.5, -.99]:
+            assert math.atanh(v) == arctanh(v)
+        for v in [2.0, -2.0]:
+            assert math.isnan(arctanh(v))
+        for v in [1.0, -1.0]:
+            assert arctanh(v) == math.copysign(float("inf"), v)
+
     def test_reduce_errors(self):
         from numpy import sin, add
 
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -1,20 +1,24 @@
 from pypy.jit.metainterp.test.support import LLJitMixin
 from pypy.module.micronumpy import interp_ufuncs, signature
 from pypy.module.micronumpy.compile import (numpy_compile, FakeSpace,
-    FloatObject)
-from pypy.module.micronumpy.interp_dtype import W_Float64Dtype, W_Int64Dtype
+    FloatObject, IntObject)
+from pypy.module.micronumpy.interp_dtype import W_Int32Dtype, W_Float64Dtype, W_Int64Dtype, W_UInt64Dtype
 from pypy.module.micronumpy.interp_numarray import (BaseArray, SingleDimArray,
     SingleDimSlice, scalar_w)
 from pypy.rlib.nonconst import NonConstant
 from pypy.rpython.annlowlevel import llstr
 from pypy.rpython.test.test_llinterp import interpret
 
+import py
+
 
 class TestNumpyJIt(LLJitMixin):
     def setup_class(cls):
         cls.space = FakeSpace()
         cls.float64_dtype = cls.space.fromcache(W_Float64Dtype)
         cls.int64_dtype = cls.space.fromcache(W_Int64Dtype)
+        cls.uint64_dtype = cls.space.fromcache(W_UInt64Dtype)
+        cls.int32_dtype = cls.space.fromcache(W_Int32Dtype)
 
     def test_add(self):
         def f(i):
@@ -303,6 +307,31 @@
                           'int_lt': 1, 'guard_true': 1, 'jump': 1})
         assert result == 11.0
 
+    def test_int32_sum(self):
+        py.test.skip("pypy/jit/backend/llimpl.py needs to be changed to "
+                     "deal correctly with int dtypes for this test to "
+                     "work. skip for now until someone feels up to the task")
+        space = self.space
+        float64_dtype = self.float64_dtype
+        int32_dtype = self.int32_dtype
+
+        def f(n):
+            if NonConstant(False):
+                dtype = float64_dtype
+            else:
+                dtype = int32_dtype
+            ar = SingleDimArray(n, dtype=dtype)
+            i = 0
+            while i < n:
+                ar.get_concrete().setitem(i, int32_dtype.box(7))
+                i += 1
+            v = ar.descr_add(space, ar).descr_sum(space)
+            assert isinstance(v, IntObject)
+            return v.intval
+
+        result = self.meta_interp(f, [5], listops=True, backendopt=True)
+        assert result == f(5)
+
 class TestTranslation(object):
     def test_compile(self):
         x = numpy_compile('aa+f*f/a-', 10)
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -10,6 +10,7 @@
 from pypy.rpython.lltypesystem import rffi, lltype
 from pypy.rpython.tool import rffi_platform
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
+from pypy.module.sys.interp_encoding import getfilesystemencoding
 
 import os, sys
 _WIN = sys.platform == 'win32'
@@ -32,17 +33,19 @@
             raise OperationError(space.w_OverflowError,
                                  space.wrap("integer out of range"))
 
+def fsencode_w(space, w_obj):
+    if space.isinstance_w(w_obj, space.w_unicode):
+        w_obj = space.call_method(w_obj, 'encode',
+                                  getfilesystemencoding(space))
+    return space.str_w(w_obj)
+
 class FileEncoder(object):
     def __init__(self, space, w_obj):
         self.space = space
         self.w_obj = w_obj
 
     def as_bytes(self):
-        from pypy.module.sys.interp_encoding import getfilesystemencoding
-        space = self.space
-        w_bytes = space.call_method(self.w_obj, 'encode',
-                                    getfilesystemencoding(space))
-        return space.str_w(w_bytes)
+        return fsencode_w(self.space, self.w_obj)
 
     def as_unicode(self):
         return self.space.unicode_w(self.w_obj)
@@ -56,7 +59,6 @@
         return self.space.str_w(self.w_obj)
 
     def as_unicode(self):
-        from pypy.module.sys.interp_encoding import getfilesystemencoding
         space = self.space
         w_unicode = space.call_method(self.w_obj, 'decode',
                                       getfilesystemencoding(space))
@@ -536,7 +538,6 @@
 
 The list is in arbitrary order.  It does not include the special
 entries '.' and '..' even if they are present in the directory."""
-    from pypy.module.sys.interp_encoding import getfilesystemencoding
     try:
         if space.isinstance_w(w_dirname, space.w_unicode):
             dirname = FileEncoder(space, w_dirname)
@@ -734,8 +735,7 @@
 def _exit(space, status):
     os._exit(status)
 
- at unwrap_spec(command=str)
-def execv(space, command, w_args):
+def execv(space, w_command, w_args):
     """ execv(path, args)
 
 Execute an executable path with arguments, replacing current process.
@@ -743,12 +743,13 @@
         path: path of executable file
         args: iterable of strings
     """
+    command = fsencode_w(space, w_command)
     try:
         args_w = space.unpackiterable(w_args)
         if len(args_w) < 1:
             w_msg = space.wrap("execv() must have at least one argument")
             raise OperationError(space.w_ValueError, w_msg)
-        args = [space.str_w(w_arg) for w_arg in args_w]
+        args = [fsencode_w(space, w_arg) for w_arg in args_w]
     except OperationError, e:
         if not e.match(space, space.w_TypeError):
             raise
@@ -759,8 +760,7 @@
     except OSError, e:
         raise wrap_oserror(space, e)
 
- at unwrap_spec(command=str)
-def execve(space, command, w_args, w_env):
+def execve(space, w_command, w_args, w_env):
     """ execve(path, args, env)
 
 Execute a path with arguments and environment, replacing current process.
@@ -769,7 +769,8 @@
         args: iterable of arguments
         env: dictionary of strings mapping to strings
     """
-    args = [space.str_w(w_arg) for w_arg in space.unpackiterable(w_args)]
+    command = fsencode_w(space, w_command)
+    args = [fsencode_w(space, w_arg) for w_arg in space.unpackiterable(w_args)]
     env = {}
     w_keys = space.call_method(w_env, 'keys')
     for w_key in space.unpackiterable(w_keys):
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -415,6 +415,23 @@
                 else:
                     py.test.fail("didn't raise")
 
+        def test_execv_unicode(self):
+            os = self.posix
+            import sys
+            if not hasattr(os, "fork"):
+                skip("Need fork() to test execv()")
+            try:
+                output = u"caf\xe9 \u1234\n".encode(sys.getfilesystemencoding())
+            except UnicodeEncodeError:
+                skip("encoding not good enough")
+            pid = os.fork()
+            if pid == 0:
+                os.execv(u"/bin/sh", ["sh", "-c",
+                                      u"echo caf\xe9 \u1234 > onefile"])
+            os.waitpid(pid, 0)
+            assert open("onefile").read() == output
+            os.unlink("onefile")
+
         def test_execve(self):
             os = self.posix
             if not hasattr(os, "fork"):
@@ -425,6 +442,24 @@
             os.waitpid(pid, 0)
             assert open("onefile").read() == "xxx"
             os.unlink("onefile")
+
+        def test_execve_unicode(self):
+            os = self.posix
+            import sys
+            if not hasattr(os, "fork"):
+                skip("Need fork() to test execve()")
+            try:
+                output = u"caf\xe9 \u1234\n".encode(sys.getfilesystemencoding())
+            except UnicodeEncodeError:
+                skip("encoding not good enough")
+            pid = os.fork()
+            if pid == 0:
+                os.execve(u"/bin/sh", ["sh", "-c",
+                                      u"echo caf\xe9 \u1234 > onefile"],
+                          {'ddd': 'xxx'})
+            os.waitpid(pid, 0)
+            assert open("onefile").read() == output
+            os.unlink("onefile")
         pass # <- please, inspect.getsource(), don't crash
 
     if hasattr(__import__(os.name), "spawnv"):
diff --git a/pypy/module/pyexpat/__init__.py b/pypy/module/pyexpat/__init__.py
--- a/pypy/module/pyexpat/__init__.py
+++ b/pypy/module/pyexpat/__init__.py
@@ -4,12 +4,8 @@
 
 class ErrorsModule(MixedModule):
     "Definition of pyexpat.errors module."
-
-    appleveldefs = {
-        }
-
-    interpleveldefs = {
-        }
+    appleveldefs = {}
+    interpleveldefs = {}
 
     def setup_after_space_initialization(self):
         from pypy.module.pyexpat import interp_pyexpat
@@ -18,6 +14,18 @@
                     interp_pyexpat.ErrorString(self.space,
                     getattr(interp_pyexpat, name)))
 
+class ModelModule(MixedModule):
+    "Definition of pyexpat.model module."
+    appleveldefs = {}
+    interpleveldefs = {}
+
+    def setup_after_space_initialization(self):
+        from pypy.module.pyexpat import interp_pyexpat
+        space = self.space
+        for name in interp_pyexpat.xml_model_list:
+            value = getattr(interp_pyexpat, name)
+            space.setattr(self, space.wrap(name), space.wrap(value))
+
 class Module(MixedModule):
     "Python wrapper for Expat parser."
 
@@ -39,6 +47,7 @@
 
     submodules = {
         'errors': ErrorsModule,
+        'model':  ModelModule,
     }
 
     for name in ['XML_PARAM_ENTITY_PARSING_NEVER',
diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py
--- a/pypy/module/pyexpat/interp_pyexpat.py
+++ b/pypy/module/pyexpat/interp_pyexpat.py
@@ -76,6 +76,18 @@
     "XML_ERROR_FINISHED",
     "XML_ERROR_SUSPEND_PE",
     ]
+xml_model_list = [
+    "XML_CTYPE_EMPTY",
+    "XML_CTYPE_ANY",
+    "XML_CTYPE_MIXED",
+    "XML_CTYPE_NAME",
+    "XML_CTYPE_CHOICE",
+    "XML_CTYPE_SEQ",
+    "XML_CQUANT_NONE",
+    "XML_CQUANT_OPT",
+    "XML_CQUANT_REP",
+    "XML_CQUANT_PLUS",
+    ]
 
 class CConfigure:
     _compilation_info_ = eci
@@ -104,6 +116,8 @@
 
     for name in xml_error_list:
         locals()[name] = rffi_platform.ConstantInteger(name)
+    for name in xml_model_list:
+        locals()[name] = rffi_platform.ConstantInteger(name)
 
 for k, v in rffi_platform.configure(CConfigure).items():
     globals()[k] = v
diff --git a/pypy/module/pyexpat/test/test_parser.py b/pypy/module/pyexpat/test/test_parser.py
--- a/pypy/module/pyexpat/test/test_parser.py
+++ b/pypy/module/pyexpat/test/test_parser.py
@@ -131,3 +131,7 @@
                'encoding specified in XML declaration is incorrect')
         assert (pyexpat.errors.XML_ERROR_XML_DECL ==
                 'XML declaration not well-formed')
+
+    def test_model(self):
+        import pyexpat
+        assert isinstance(pyexpat.model.XML_CTYPE_EMPTY, int)
diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py
--- a/pypy/module/pypyjit/interp_jit.py
+++ b/pypy/module/pypyjit/interp_jit.py
@@ -137,20 +137,15 @@
 
     def jump_absolute(self, jumpto, _, ec=None):
         if we_are_jitted():
-            # Normally, the tick counter is decremented by 100 for every
-            # Python opcode.  Here, to better support JIT compilation of
-            # small loops, we decrement it by a possibly smaller constant.
-            # We get the maximum 100 when the (unoptimized) trace length
-            # is at least 3200 (a bit randomly).
-            trace_length = r_uint(current_trace_length())
-            decr_by = trace_length // 32
-            if decr_by < 1:
-                decr_by = 1
-            elif decr_by > 100:    # also if current_trace_length() returned -1
-                decr_by = 100
+            #

From noreply at buildbot.pypy.org  Fri Oct 21 14:08:50 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 21 Oct 2011 14:08:50 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: hg merge default
Message-ID: <20111021120850.E66F4820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r48305:8a71cedfbb59
Date: 2011-10-20 22:27 +0200
http://bitbucket.org/pypy/pypy/changeset/8a71cedfbb59/

Log:	hg merge default

diff --git a/lib-python/modified-2.7/httplib.py b/lib-python/modified-2.7/httplib.py
new file mode 100644
--- /dev/null
+++ b/lib-python/modified-2.7/httplib.py
@@ -0,0 +1,1377 @@
+"""HTTP/1.1 client library
+
+<intro stuff goes here>
+<other stuff, too>
+
+HTTPConnection goes through a number of "states", which define when a client
+may legally make another request or fetch the response for a particular
+request. This diagram details these state transitions:
+
+    (null)
+      |
+      | HTTPConnection()
+      v
+    Idle
+      |
+      | putrequest()
+      v
+    Request-started
+      |
+      | ( putheader() )*  endheaders()
+      v
+    Request-sent
+      |
+      | response = getresponse()
+      v
+    Unread-response   [Response-headers-read]
+      |\____________________
+      |                     |
+      | response.read()     | putrequest()
+      v                     v
+    Idle                  Req-started-unread-response
+                     ______/|
+                   /        |
+   response.read() |        | ( putheader() )*  endheaders()
+                   v        v
+       Request-started    Req-sent-unread-response
+                            |
+                            | response.read()
+                            v
+                          Request-sent
+
+This diagram presents the following rules:
+  -- a second request may not be started until {response-headers-read}
+  -- a response [object] cannot be retrieved until {request-sent}
+  -- there is no differentiation between an unread response body and a
+     partially read response body
+
+Note: this enforcement is applied by the HTTPConnection class. The
+      HTTPResponse class does not enforce this state machine, which
+      implies sophisticated clients may accelerate the request/response
+      pipeline. Caution should be taken, though: accelerating the states
+      beyond the above pattern may imply knowledge of the server's
+      connection-close behavior for certain requests. For example, it
+      is impossible to tell whether the server will close the connection
+      UNTIL the response headers have been read; this means that further
+      requests cannot be placed into the pipeline until it is known that
+      the server will NOT be closing the connection.
+
+Logical State                  __state            __response
+-------------                  -------            ----------
+Idle                           _CS_IDLE           None
+Request-started                _CS_REQ_STARTED    None
+Request-sent                   _CS_REQ_SENT       None
+Unread-response                _CS_IDLE           <response_class>
+Req-started-unread-response    _CS_REQ_STARTED    <response_class>
+Req-sent-unread-response       _CS_REQ_SENT       <response_class>
+"""
+
+from array import array
+import os
+import socket
+from sys import py3kwarning
+from urlparse import urlsplit
+import warnings
+with warnings.catch_warnings():
+    if py3kwarning:
+        warnings.filterwarnings("ignore", ".*mimetools has been removed",
+                                DeprecationWarning)
+    import mimetools
+
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from StringIO import StringIO
+
+__all__ = ["HTTP", "HTTPResponse", "HTTPConnection",
+           "HTTPException", "NotConnected", "UnknownProtocol",
+           "UnknownTransferEncoding", "UnimplementedFileMode",
+           "IncompleteRead", "InvalidURL", "ImproperConnectionState",
+           "CannotSendRequest", "CannotSendHeader", "ResponseNotReady",
+           "BadStatusLine", "error", "responses"]
+
+HTTP_PORT = 80
+HTTPS_PORT = 443
+
+_UNKNOWN = 'UNKNOWN'
+
+# connection states
+_CS_IDLE = 'Idle'
+_CS_REQ_STARTED = 'Request-started'
+_CS_REQ_SENT = 'Request-sent'
+
+# status codes
+# informational
+CONTINUE = 100
+SWITCHING_PROTOCOLS = 101
+PROCESSING = 102
+
+# successful
+OK = 200
+CREATED = 201
+ACCEPTED = 202
+NON_AUTHORITATIVE_INFORMATION = 203
+NO_CONTENT = 204
+RESET_CONTENT = 205
+PARTIAL_CONTENT = 206
+MULTI_STATUS = 207
+IM_USED = 226
+
+# redirection
+MULTIPLE_CHOICES = 300
+MOVED_PERMANENTLY = 301
+FOUND = 302
+SEE_OTHER = 303
+NOT_MODIFIED = 304
+USE_PROXY = 305
+TEMPORARY_REDIRECT = 307
+
+# client error
+BAD_REQUEST = 400
+UNAUTHORIZED = 401
+PAYMENT_REQUIRED = 402
+FORBIDDEN = 403
+NOT_FOUND = 404
+METHOD_NOT_ALLOWED = 405
+NOT_ACCEPTABLE = 406
+PROXY_AUTHENTICATION_REQUIRED = 407
+REQUEST_TIMEOUT = 408
+CONFLICT = 409
+GONE = 410
+LENGTH_REQUIRED = 411
+PRECONDITION_FAILED = 412
+REQUEST_ENTITY_TOO_LARGE = 413
+REQUEST_URI_TOO_LONG = 414
+UNSUPPORTED_MEDIA_TYPE = 415
+REQUESTED_RANGE_NOT_SATISFIABLE = 416
+EXPECTATION_FAILED = 417
+UNPROCESSABLE_ENTITY = 422
+LOCKED = 423
+FAILED_DEPENDENCY = 424
+UPGRADE_REQUIRED = 426
+
+# server error
+INTERNAL_SERVER_ERROR = 500
+NOT_IMPLEMENTED = 501
+BAD_GATEWAY = 502
+SERVICE_UNAVAILABLE = 503
+GATEWAY_TIMEOUT = 504
+HTTP_VERSION_NOT_SUPPORTED = 505
+INSUFFICIENT_STORAGE = 507
+NOT_EXTENDED = 510
+
+# Mapping status codes to official W3C names
+responses = {
+    100: 'Continue',
+    101: 'Switching Protocols',
+
+    200: 'OK',
+    201: 'Created',
+    202: 'Accepted',
+    203: 'Non-Authoritative Information',
+    204: 'No Content',
+    205: 'Reset Content',
+    206: 'Partial Content',
+
+    300: 'Multiple Choices',
+    301: 'Moved Permanently',
+    302: 'Found',
+    303: 'See Other',
+    304: 'Not Modified',
+    305: 'Use Proxy',
+    306: '(Unused)',
+    307: 'Temporary Redirect',
+
+    400: 'Bad Request',
+    401: 'Unauthorized',
+    402: 'Payment Required',
+    403: 'Forbidden',
+    404: 'Not Found',
+    405: 'Method Not Allowed',
+    406: 'Not Acceptable',
+    407: 'Proxy Authentication Required',
+    408: 'Request Timeout',
+    409: 'Conflict',
+    410: 'Gone',
+    411: 'Length Required',
+    412: 'Precondition Failed',
+    413: 'Request Entity Too Large',
+    414: 'Request-URI Too Long',
+    415: 'Unsupported Media Type',
+    416: 'Requested Range Not Satisfiable',
+    417: 'Expectation Failed',
+
+    500: 'Internal Server Error',
+    501: 'Not Implemented',
+    502: 'Bad Gateway',
+    503: 'Service Unavailable',
+    504: 'Gateway Timeout',
+    505: 'HTTP Version Not Supported',
+}
+
+# maximal amount of data to read at one time in _safe_read
+MAXAMOUNT = 1048576
+
+class HTTPMessage(mimetools.Message):
+
+    def addheader(self, key, value):
+        """Add header for field key handling repeats."""
+        prev = self.dict.get(key)
+        if prev is None:
+            self.dict[key] = value
+        else:
+            combined = ", ".join((prev, value))
+            self.dict[key] = combined
+
+    def addcontinue(self, key, more):
+        """Add more field data from a continuation line."""
+        prev = self.dict[key]
+        self.dict[key] = prev + "\n " + more
+
+    def readheaders(self):
+        """Read header lines.
+
+        Read header lines up to the entirely blank line that terminates them.
+        The (normally blank) line that ends the headers is skipped, but not
+        included in the returned list.  If a non-header line ends the headers,
+        (which is an error), an attempt is made to backspace over it; it is
+        never included in the returned list.
+
+        The variable self.status is set to the empty string if all went well,
+        otherwise it is an error message.  The variable self.headers is a
+        completely uninterpreted list of lines contained in the header (so
+        printing them will reproduce the header exactly as it appears in the
+        file).
+
+        If multiple header fields with the same name occur, they are combined
+        according to the rules in RFC 2616 sec 4.2:
+
+        Appending each subsequent field-value to the first, each separated
+        by a comma. The order in which header fields with the same field-name
+        are received is significant to the interpretation of the combined
+        field value.
+        """
+        # XXX The implementation overrides the readheaders() method of
+        # rfc822.Message.  The base class design isn't amenable to
+        # customized behavior here so the method here is a copy of the
+        # base class code with a few small changes.
+
+        self.dict = {}
+        self.unixfrom = ''
+        self.headers = hlist = []
+        self.status = ''
+        headerseen = ""
+        firstline = 1
+        startofline = unread = tell = None
+        if hasattr(self.fp, 'unread'):
+            unread = self.fp.unread
+        elif self.seekable:
+            tell = self.fp.tell
+        while True:
+            if tell:
+                try:
+                    startofline = tell()
+                except IOError:
+                    startofline = tell = None
+                    self.seekable = 0
+            line = self.fp.readline()
+            if not line:
+                self.status = 'EOF in headers'
+                break
+            # Skip unix From name time lines
+            if firstline and line.startswith('From '):
+                self.unixfrom = self.unixfrom + line
+                continue
+            firstline = 0
+            if headerseen and line[0] in ' \t':
+                # XXX Not sure if continuation lines are handled properly
+                # for http and/or for repeating headers
+                # It's a continuation line.
+                hlist.append(line)
+                self.addcontinue(headerseen, line.strip())
+                continue
+            elif self.iscomment(line):
+                # It's a comment.  Ignore it.
+                continue
+            elif self.islast(line):
+                # Note! No pushback here!  The delimiter line gets eaten.
+                break
+            headerseen = self.isheader(line)
+            if headerseen:
+                # It's a legal header line, save it.
+                hlist.append(line)
+                self.addheader(headerseen, line[len(headerseen)+1:].strip())
+                continue
+            else:
+                # It's not a header line; throw it back and stop here.
+                if not self.dict:
+                    self.status = 'No headers'
+                else:
+                    self.status = 'Non-header line where header expected'
+                # Try to undo the read.
+                if unread:
+                    unread(line)
+                elif tell:
+                    self.fp.seek(startofline)
+                else:
+                    self.status = self.status + '; bad seek'
+                break
+
+class HTTPResponse:
+
+    # strict: If true, raise BadStatusLine if the status line can't be
+    # parsed as a valid HTTP/1.0 or 1.1 status line.  By default it is
+    # false because it prevents clients from talking to HTTP/0.9
+    # servers.  Note that a response with a sufficiently corrupted
+    # status line will look like an HTTP/0.9 response.
+
+    # See RFC 2616 sec 19.6 and RFC 1945 sec 6 for details.
+
+    def __init__(self, sock, debuglevel=0, strict=0, method=None, buffering=False):
+        if buffering:
+            # The caller won't be using any sock.recv() calls, so buffering
+            # is fine and recommended for performance.
+            self.fp = sock.makefile('rb')
+        else:
+            # The buffer size is specified as zero, because the headers of
+            # the response are read with readline().  If the reads were
+            # buffered the readline() calls could consume some of the
+            # response, which make be read via a recv() on the underlying
+            # socket.
+            self.fp = sock.makefile('rb', 0)
+        self.debuglevel = debuglevel
+        self.strict = strict
+        self._method = method
+
+        self.msg = None
+
+        # from the Status-Line of the response
+        self.version = _UNKNOWN # HTTP-Version
+        self.status = _UNKNOWN  # Status-Code
+        self.reason = _UNKNOWN  # Reason-Phrase
+
+        self.chunked = _UNKNOWN         # is "chunked" being used?
+        self.chunk_left = _UNKNOWN      # bytes left to read in current chunk
+        self.length = _UNKNOWN          # number of bytes left in response
+        self.will_close = _UNKNOWN      # conn will close at end of response
+
+    def _read_status(self):
+        # Initialize with Simple-Response defaults
+        line = self.fp.readline()
+        if self.debuglevel > 0:
+            print "reply:", repr(line)
+        if not line:
+            # Presumably, the server closed the connection before
+            # sending a valid response.
+            raise BadStatusLine(line)
+        try:
+            [version, status, reason] = line.split(None, 2)
+        except ValueError:
+            try:
+                [version, status] = line.split(None, 1)
+                reason = ""
+            except ValueError:
+                # empty version will cause next test to fail and status
+                # will be treated as 0.9 response.
+                version = ""
+        if not version.startswith('HTTP/'):
+            if self.strict:
+                self.close()
+                raise BadStatusLine(line)
+            else:
+                # assume it's a Simple-Response from an 0.9 server
+                self.fp = LineAndFileWrapper(line, self.fp)
+                return "HTTP/0.9", 200, ""
+
+        # The status code is a three-digit number
+        try:
+            status = int(status)
+            if status < 100 or status > 999:
+                raise BadStatusLine(line)
+        except ValueError:
+            raise BadStatusLine(line)
+        return version, status, reason
+
+    def begin(self):
+        if self.msg is not None:
+            # we've already started reading the response
+            return
+
+        # read until we get a non-100 response
+        while True:
+            version, status, reason = self._read_status()
+            if status != CONTINUE:
+                break
+            # skip the header from the 100 response
+            while True:
+                skip = self.fp.readline().strip()
+                if not skip:
+                    break
+                if self.debuglevel > 0:
+                    print "header:", skip
+
+        self.status = status
+        self.reason = reason.strip()
+        if version == 'HTTP/1.0':
+            self.version = 10
+        elif version.startswith('HTTP/1.'):
+            self.version = 11   # use HTTP/1.1 code for HTTP/1.x where x>=1
+        elif version == 'HTTP/0.9':
+            self.version = 9
+        else:
+            raise UnknownProtocol(version)
+
+        if self.version == 9:
+            self.length = None
+            self.chunked = 0
+            self.will_close = 1
+            self.msg = HTTPMessage(StringIO())
+            return
+
+        self.msg = HTTPMessage(self.fp, 0)
+        if self.debuglevel > 0:
+            for hdr in self.msg.headers:
+                print "header:", hdr,
+
+        # don't let the msg keep an fp
+        self.msg.fp = None
+
+        # are we using the chunked-style of transfer encoding?
+        tr_enc = self.msg.getheader('transfer-encoding')
+        if tr_enc and tr_enc.lower() == "chunked":
+            self.chunked = 1
+            self.chunk_left = None
+        else:
+            self.chunked = 0
+
+        # will the connection close at the end of the response?
+        self.will_close = self._check_close()
+
+        # do we have a Content-Length?
+        # NOTE: RFC 2616, S4.4, #3 says we ignore this if tr_enc is "chunked"
+        length = self.msg.getheader('content-length')
+        if length and not self.chunked:
+            try:
+                self.length = int(length)
+            except ValueError:
+                self.length = None
+            else:
+                if self.length < 0:  # ignore nonsensical negative lengths
+                    self.length = None
+        else:
+            self.length = None
+
+        # does the body have a fixed length? (of zero)
+        if (status == NO_CONTENT or status == NOT_MODIFIED or
+            100 <= status < 200 or      # 1xx codes
+            self._method == 'HEAD'):
+            self.length = 0
+
+        # if the connection remains open, and we aren't using chunked, and
+        # a content-length was not provided, then assume that the connection
+        # WILL close.
+        if not self.will_close and \
+           not self.chunked and \
+           self.length is None:
+            self.will_close = 1
+
+    def _check_close(self):
+        conn = self.msg.getheader('connection')
+        if self.version == 11:
+            # An HTTP/1.1 proxy is assumed to stay open unless
+            # explicitly closed.
+            conn = self.msg.getheader('connection')
+            if conn and "close" in conn.lower():
+                return True
+            return False
+
+        # Some HTTP/1.0 implementations have support for persistent
+        # connections, using rules different than HTTP/1.1.
+
+        # For older HTTP, Keep-Alive indicates persistent connection.
+        if self.msg.getheader('keep-alive'):
+            return False
+
+        # At least Akamai returns a "Connection: Keep-Alive" header,
+        # which was supposed to be sent by the client.
+        if conn and "keep-alive" in conn.lower():
+            return False
+
+        # Proxy-Connection is a netscape hack.
+        pconn = self.msg.getheader('proxy-connection')
+        if pconn and "keep-alive" in pconn.lower():
+            return False
+
+        # otherwise, assume it will close
+        return True
+
+    def close(self):
+        if self.fp:
+            self.fp.close()
+            self.fp = None
+
+    def isclosed(self):
+        # NOTE: it is possible that we will not ever call self.close(). This
+        #       case occurs when will_close is TRUE, length is None, and we
+        #       read up to the last byte, but NOT past it.
+        #
+        # IMPLIES: if will_close is FALSE, then self.close() will ALWAYS be
+        #          called, meaning self.isclosed() is meaningful.
+        return self.fp is None
+
+    # XXX It would be nice to have readline and __iter__ for this, too.
+
+    def read(self, amt=None):
+        if self.fp is None:
+            return ''
+
+        if self._method == 'HEAD':
+            self.close()
+            return ''
+
+        if self.chunked:
+            return self._read_chunked(amt)
+
+        if amt is None:
+            # unbounded read
+            if self.length is None:
+                s = self.fp.read()
+            else:
+                s = self._safe_read(self.length)
+                self.length = 0
+            self.close()        # we read everything
+            return s
+
+        if self.length is not None:
+            if amt > self.length:
+                # clip the read to the "end of response"
+                amt = self.length
+
+        # we do not use _safe_read() here because this may be a .will_close
+        # connection, and the user is reading more bytes than will be provided
+        # (for example, reading in 1k chunks)
+        s = self.fp.read(amt)
+        if self.length is not None:
+            self.length -= len(s)
+            if not self.length:
+                self.close()
+        return s
+
+    def _read_chunked(self, amt):
+        assert self.chunked != _UNKNOWN
+        chunk_left = self.chunk_left
+        value = []
+        while True:
+            if chunk_left is None:
+                line = self.fp.readline()
+                i = line.find(';')
+                if i >= 0:
+                    line = line[:i] # strip chunk-extensions
+                try:
+                    chunk_left = int(line, 16)
+                except ValueError:
+                    # close the connection as protocol synchronisation is
+                    # probably lost
+                    self.close()
+                    raise IncompleteRead(''.join(value))
+                if chunk_left == 0:
+                    break
+            if amt is None:
+                value.append(self._safe_read(chunk_left))
+            elif amt < chunk_left:
+                value.append(self._safe_read(amt))
+                self.chunk_left = chunk_left - amt
+                return ''.join(value)
+            elif amt == chunk_left:
+                value.append(self._safe_read(amt))
+                self._safe_read(2)  # toss the CRLF at the end of the chunk
+                self.chunk_left = None
+                return ''.join(value)
+            else:
+                value.append(self._safe_read(chunk_left))
+                amt -= chunk_left
+
+            # we read the whole chunk, get another
+            self._safe_read(2)      # toss the CRLF at the end of the chunk
+            chunk_left = None
+
+        # read and discard trailer up to the CRLF terminator
+        ### note: we shouldn't have any trailers!
+        while True:
+            line = self.fp.readline()
+            if not line:
+                # a vanishingly small number of sites EOF without
+                # sending the trailer
+                break
+            if line == '\r\n':
+                break
+
+        # we read everything; close the "file"
+        self.close()
+
+        return ''.join(value)
+
+    def _safe_read(self, amt):
+        """Read the number of bytes requested, compensating for partial reads.
+
+        Normally, we have a blocking socket, but a read() can be interrupted
+        by a signal (resulting in a partial read).
+
+        Note that we cannot distinguish between EOF and an interrupt when zero
+        bytes have been read. IncompleteRead() will be raised in this
+        situation.
+
+        This function should be used when <amt> bytes "should" be present for
+        reading. If the bytes are truly not available (due to EOF), then the
+        IncompleteRead exception can be used to detect the problem.
+        """
+        # NOTE(gps): As of svn r74426 socket._fileobject.read(x) will never
+        # return less than x bytes unless EOF is encountered.  It now handles
+        # signal interruptions (socket.error EINTR) internally.  This code
+        # never caught that exception anyways.  It seems largely pointless.
+        # self.fp.read(amt) will work fine.
+        s = []
+        while amt > 0:
+            chunk = self.fp.read(min(amt, MAXAMOUNT))
+            if not chunk:
+                raise IncompleteRead(''.join(s), amt)
+            s.append(chunk)
+            amt -= len(chunk)
+        return ''.join(s)
+
+    def fileno(self):
+        return self.fp.fileno()
+
+    def getheader(self, name, default=None):
+        if self.msg is None:
+            raise ResponseNotReady()
+        return self.msg.getheader(name, default)
+
+    def getheaders(self):
+        """Return list of (header, value) tuples."""
+        if self.msg is None:
+            raise ResponseNotReady()
+        return self.msg.items()
+
+
+class HTTPConnection:
+
+    _http_vsn = 11
+    _http_vsn_str = 'HTTP/1.1'
+
+    response_class = HTTPResponse
+    default_port = HTTP_PORT
+    auto_open = 1
+    debuglevel = 0
+    strict = 0
+
+    def __init__(self, host, port=None, strict=None,
+                 timeout=socket._GLOBAL_DEFAULT_TIMEOUT, source_address=None):
+        self.timeout = timeout
+        self.source_address = source_address
+        self.sock = None
+        self._buffer = []
+        self.__response = None
+        self.__state = _CS_IDLE
+        self._method = None
+        self._tunnel_host = None
+        self._tunnel_port = None
+        self._tunnel_headers = {}
+
+        self._set_hostport(host, port)
+        if strict is not None:
+            self.strict = strict
+
+    def set_tunnel(self, host, port=None, headers=None):
+        """ Sets up the host and the port for the HTTP CONNECT Tunnelling.
+
+        The headers argument should be a mapping of extra HTTP headers
+        to send with the CONNECT request.
+        """
+        self._tunnel_host = host
+        self._tunnel_port = port
+        if headers:
+            self._tunnel_headers = headers
+        else:
+            self._tunnel_headers.clear()
+
+    def _set_hostport(self, host, port):
+        if port is None:
+            i = host.rfind(':')
+            j = host.rfind(']')         # ipv6 addresses have [...]
+            if i > j:
+                try:
+                    port = int(host[i+1:])
+                except ValueError:
+                    raise InvalidURL("nonnumeric port: '%s'" % host[i+1:])
+                host = host[:i]
+            else:
+                port = self.default_port
+            if host and host[0] == '[' and host[-1] == ']':
+                host = host[1:-1]
+        self.host = host
+        self.port = port
+
+    def set_debuglevel(self, level):
+        self.debuglevel = level
+
+    def _tunnel(self):
+        self._set_hostport(self._tunnel_host, self._tunnel_port)
+        self.send("CONNECT %s:%d HTTP/1.0\r\n" % (self.host, self.port))
+        for header, value in self._tunnel_headers.iteritems():
+            self.send("%s: %s\r\n" % (header, value))
+        self.send("\r\n")
+        response = self.response_class(self.sock, strict = self.strict,
+                                       method = self._method)
+        (version, code, message) = response._read_status()
+
+        if code != 200:
+            self.close()
+            raise socket.error("Tunnel connection failed: %d %s" % (code,
+                                                                    message.strip()))
+        while True:
+            line = response.fp.readline()
+            if line == '\r\n': break
+
+
+    def connect(self):
+        """Connect to the host and port specified in __init__."""
+        self.sock = socket.create_connection((self.host,self.port),
+                                             self.timeout, self.source_address)
+
+        if self._tunnel_host:
+            self._tunnel()
+
+    def close(self):
+        """Close the connection to the HTTP server."""
+        if self.sock:
+            self.sock.close()   # close it manually... there may be other refs
+            self.sock = None
+        if self.__response:
+            self.__response.close()
+            self.__response = None
+        self.__state = _CS_IDLE
+
+    def send(self, data):
+        """Send `data' to the server."""
+        if self.sock is None:
+            if self.auto_open:
+                self.connect()
+            else:
+                raise NotConnected()
+
+        if self.debuglevel > 0:
+            print "send:", repr(data)
+        blocksize = 8192
+        if hasattr(data,'read') and not isinstance(data, array):
+            if self.debuglevel > 0: print "sendIng a read()able"
+            datablock = data.read(blocksize)
+            while datablock:
+                self.sock.sendall(datablock)
+                datablock = data.read(blocksize)
+        else:
+            self.sock.sendall(data)
+
+    def _output(self, s):
+        """Add a line of output to the current request buffer.
+
+        Assumes that the line does *not* end with \\r\\n.
+        """
+        self._buffer.append(s)
+
+    def _send_output(self, message_body=None):
+        """Send the currently buffered request and clear the buffer.
+
+        Appends an extra \\r\\n to the buffer.
+        A message_body may be specified, to be appended to the request.
+        """
+        self._buffer.extend(("", ""))
+        msg = "\r\n".join(self._buffer)
+        del self._buffer[:]
+        # If msg and message_body are sent in a single send() call,
+        # it will avoid performance problems caused by the interaction
+        # between delayed ack and the Nagle algorithim.
+        if isinstance(message_body, str):
+            msg += message_body
+            message_body = None
+        self.send(msg)
+        if message_body is not None:
+            #message_body was not a string (i.e. it is a file) and
+            #we must run the risk of Nagle
+            self.send(message_body)
+
+    def putrequest(self, method, url, skip_host=0, skip_accept_encoding=0):
+        """Send a request to the server.
+
+        `method' specifies an HTTP request method, e.g. 'GET'.
+        `url' specifies the object being requested, e.g. '/index.html'.
+        `skip_host' if True does not add automatically a 'Host:' header
+        `skip_accept_encoding' if True does not add automatically an
+           'Accept-Encoding:' header
+        """
+
+        # if a prior response has been completed, then forget about it.
+        if self.__response and self.__response.isclosed():
+            self.__response = None
+
+
+        # in certain cases, we cannot issue another request on this connection.
+        # this occurs when:
+        #   1) we are in the process of sending a request.   (_CS_REQ_STARTED)
+        #   2) a response to a previous request has signalled that it is going
+        #      to close the connection upon completion.
+        #   3) the headers for the previous response have not been read, thus
+        #      we cannot determine whether point (2) is true.   (_CS_REQ_SENT)
+        #
+        # if there is no prior response, then we can request at will.
+        #
+        # if point (2) is true, then we will have passed the socket to the
+        # response (effectively meaning, "there is no prior response"), and
+        # will open a new one when a new request is made.
+        #
+        # Note: if a prior response exists, then we *can* start a new request.
+        #       We are not allowed to begin fetching the response to this new
+        #       request, however, until that prior response is complete.
+        #
+        if self.__state == _CS_IDLE:
+            self.__state = _CS_REQ_STARTED
+        else:
+            raise CannotSendRequest()
+
+        # Save the method we use, we need it later in the response phase
+        self._method = method
+        if not url:
+            url = '/'
+        hdr = '%s %s %s' % (method, url, self._http_vsn_str)
+
+        self._output(hdr)
+
+        if self._http_vsn == 11:
+            # Issue some standard headers for better HTTP/1.1 compliance
+
+            if not skip_host:
+                # this header is issued *only* for HTTP/1.1
+                # connections. more specifically, this means it is
+                # only issued when the client uses the new
+                # HTTPConnection() class. backwards-compat clients
+                # will be using HTTP/1.0 and those clients may be
+                # issuing this header themselves. we should NOT issue
+                # it twice; some web servers (such as Apache) barf
+                # when they see two Host: headers
+
+                # If we need a non-standard port,include it in the
+                # header.  If the request is going through a proxy,
+                # but the host of the actual URL, not the host of the
+                # proxy.
+
+                netloc = ''
+                if url.startswith('http'):
+                    nil, netloc, nil, nil, nil = urlsplit(url)
+
+                if netloc:
+                    try:
+                        netloc_enc = netloc.encode("ascii")
+                    except UnicodeEncodeError:
+                        netloc_enc = netloc.encode("idna")
+                    self.putheader('Host', netloc_enc)
+                else:
+                    try:
+                        host_enc = self.host.encode("ascii")
+                    except UnicodeEncodeError:
+                        host_enc = self.host.encode("idna")
+                    # Wrap the IPv6 Host Header with [] (RFC 2732)
+                    if host_enc.find(':') >= 0:
+                        host_enc = "[" + host_enc + "]"
+                    if self.port == self.default_port:
+                        self.putheader('Host', host_enc)
+                    else:
+                        self.putheader('Host', "%s:%s" % (host_enc, self.port))
+
+            # note: we are assuming that clients will not attempt to set these
+            #       headers since *this* library must deal with the
+            #       consequences. this also means that when the supporting
+            #       libraries are updated to recognize other forms, then this
+            #       code should be changed (removed or updated).
+
+            # we only want a Content-Encoding of "identity" since we don't
+            # support encodings such as x-gzip or x-deflate.
+            if not skip_accept_encoding:
+                self.putheader('Accept-Encoding', 'identity')
+
+            # we can accept "chunked" Transfer-Encodings, but no others
+            # NOTE: no TE header implies *only* "chunked"
+            #self.putheader('TE', 'chunked')
+
+            # if TE is supplied in the header, then it must appear in a
+            # Connection header.
+            #self.putheader('Connection', 'TE')
+
+        else:
+            # For HTTP/1.0, the server will assume "not chunked"
+            pass
+
+    def putheader(self, header, *values):
+        """Send a request header line to the server.
+
+        For example: h.putheader('Accept', 'text/html')
+        """
+        if self.__state != _CS_REQ_STARTED:
+            raise CannotSendHeader()
+
+        hdr = '%s: %s' % (header, '\r\n\t'.join([str(v) for v in values]))
+        self._output(hdr)
+
+    def endheaders(self, message_body=None):
+        """Indicate that the last header line has been sent to the server.
+
+        This method sends the request to the server.  The optional
+        message_body argument can be used to pass message body
+        associated with the request.  The message body will be sent in
+        the same packet as the message headers if possible.  The
+        message_body should be a string.
+        """
+        if self.__state == _CS_REQ_STARTED:
+            self.__state = _CS_REQ_SENT
+        else:
+            raise CannotSendHeader()
+        self._send_output(message_body)
+
+    def request(self, method, url, body=None, headers={}):
+        """Send a complete request to the server."""
+        self._send_request(method, url, body, headers)
+
+    def _set_content_length(self, body):
+        # Set the content-length based on the body.
+        thelen = None
+        try:
+            thelen = str(len(body))
+        except TypeError, te:
+            # If this is a file-like object, try to
+            # fstat its file descriptor
+            try:
+                thelen = str(os.fstat(body.fileno()).st_size)
+            except (AttributeError, OSError):
+                # Don't send a length if this failed
+                if self.debuglevel > 0: print "Cannot stat!!"
+
+        if thelen is not None:
+            self.putheader('Content-Length', thelen)
+
+    def _send_request(self, method, url, body, headers):
+        # Honor explicitly requested Host: and Accept-Encoding: headers.
+        header_names = dict.fromkeys([k.lower() for k in headers])
+        skips = {}
+        if 'host' in header_names:
+            skips['skip_host'] = 1
+        if 'accept-encoding' in header_names:
+            skips['skip_accept_encoding'] = 1
+
+        self.putrequest(method, url, **skips)
+
+        if body and ('content-length' not in header_names):
+            self._set_content_length(body)
+        for hdr, value in headers.iteritems():
+            self.putheader(hdr, value)
+        self.endheaders(body)
+
+    def getresponse(self, buffering=False):
+        "Get the response from the server."
+
+        # if a prior response has been completed, then forget about it.
+        if self.__response and self.__response.isclosed():
+            self.__response = None
+
+        #
+        # if a prior response exists, then it must be completed (otherwise, we
+        # cannot read this response's header to determine the connection-close
+        # behavior)
+        #
+        # note: if a prior response existed, but was connection-close, then the
+        # socket and response were made independent of this HTTPConnection
+        # object since a new request requires that we open a whole new
+        # connection
+        #
+        # this means the prior response had one of two states:
+        #   1) will_close: this connection was reset and the prior socket and
+        #                  response operate independently
+        #   2) persistent: the response was retained and we await its
+        #                  isclosed() status to become true.
+        #
+        if self.__state != _CS_REQ_SENT or self.__response:
+            raise ResponseNotReady()
+
+        args = (self.sock,)
+        kwds = {"strict":self.strict, "method":self._method}
+        if self.debuglevel > 0:
+            args += (self.debuglevel,)
+        if buffering:
+            #only add this keyword if non-default, for compatibility with
+            #other response_classes.
+            kwds["buffering"] = True;
+        response = self.response_class(*args, **kwds)
+
+        try:
+            response.begin()
+        except:
+            response.close()
+            raise
+        assert response.will_close != _UNKNOWN
+        self.__state = _CS_IDLE
+
+        if response.will_close:
+            # this effectively passes the connection to the response
+            self.close()
+        else:
+            # remember this, so we can tell when it is complete
+            self.__response = response
+
+        return response
+
+
+class HTTP:
+    "Compatibility class with httplib.py from 1.5."
+
+    _http_vsn = 10
+    _http_vsn_str = 'HTTP/1.0'
+
+    debuglevel = 0
+
+    _connection_class = HTTPConnection
+
+    def __init__(self, host='', port=None, strict=None):
+        "Provide a default host, since the superclass requires one."
+
+        # some joker passed 0 explicitly, meaning default port
+        if port == 0:
+            port = None
+
+        # Note that we may pass an empty string as the host; this will throw
+        # an error when we attempt to connect. Presumably, the client code
+        # will call connect before then, with a proper host.
+        self._setup(self._connection_class(host, port, strict))
+
+    def _setup(self, conn):
+        self._conn = conn
+
+        # set up delegation to flesh out interface
+        self.send = conn.send
+        self.putrequest = conn.putrequest
+        self.putheader = conn.putheader
+        self.endheaders = conn.endheaders
+        self.set_debuglevel = conn.set_debuglevel
+
+        conn._http_vsn = self._http_vsn
+        conn._http_vsn_str = self._http_vsn_str
+
+        self.file = None
+
+    def connect(self, host=None, port=None):
+        "Accept arguments to set the host/port, since the superclass doesn't."
+
+        if host is not None:
+            self._conn._set_hostport(host, port)
+        self._conn.connect()
+
+    def getfile(self):
+        "Provide a getfile, since the superclass' does not use this concept."
+        return self.file
+
+    def getreply(self, buffering=False):
+        """Compat definition since superclass does not define it.
+
+        Returns a tuple consisting of:
+        - server status code (e.g. '200' if all goes well)
+        - server "reason" corresponding to status code
+        - any RFC822 headers in the response from the server
+        """
+        try:
+            if not buffering:
+                response = self._conn.getresponse()
+            else:
+                #only add this keyword if non-default for compatibility
+                #with other connection classes
+                response = self._conn.getresponse(buffering)
+        except BadStatusLine, e:
+            ### hmm. if getresponse() ever closes the socket on a bad request,
+            ### then we are going to have problems with self.sock
+
+            ### should we keep this behavior? do people use it?
+            # keep the socket open (as a file), and return it
+            self.file = self._conn.sock.makefile('rb', 0)
+
+            # close our socket -- we want to restart after any protocol error
+            self.close()
+
+            self.headers = None
+            return -1, e.line, None
+
+        self.headers = response.msg
+        self.file = response.fp
+        return response.status, response.reason, response.msg
+
+    def close(self):
+        self._conn.close()
+
+        # note that self.file == response.fp, which gets closed by the
+        # superclass. just clear the object ref here.
+        ### hmm. messy. if status==-1, then self.file is owned by us.
+        ### well... we aren't explicitly closing, but losing this ref will
+        ### do it
+        self.file = None
+
+try:
+    import ssl
+except ImportError:
+    pass
+else:
+    class HTTPSConnection(HTTPConnection):
+        "This class allows communication via SSL."
+
+        default_port = HTTPS_PORT
+
+        def __init__(self, host, port=None, key_file=None, cert_file=None,
+                     strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
+                     source_address=None):
+            HTTPConnection.__init__(self, host, port, strict, timeout,
+                                    source_address)
+            self.key_file = key_file
+            self.cert_file = cert_file
+
+        def connect(self):
+            "Connect to a host on a given (SSL) port."
+
+            sock = socket.create_connection((self.host, self.port),
+                                            self.timeout, self.source_address)
+            if self._tunnel_host:
+                self.sock = sock
+                self._tunnel()
+            self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file)
+
+    __all__.append("HTTPSConnection")
+
+    class HTTPS(HTTP):
+        """Compatibility with 1.5 httplib interface
+
+        Python 1.5.2 did not have an HTTPS class, but it defined an
+        interface for sending http requests that is also useful for
+        https.
+        """
+
+        _connection_class = HTTPSConnection
+
+        def __init__(self, host='', port=None, key_file=None, cert_file=None,
+                     strict=None):
+            # provide a default host, pass the X509 cert info
+
+            # urf. compensate for bad input.
+            if port == 0:
+                port = None
+            self._setup(self._connection_class(host, port, key_file,
+                                               cert_file, strict))
+
+            # we never actually use these for anything, but we keep them
+            # here for compatibility with post-1.5.2 CVS.
+            self.key_file = key_file
+            self.cert_file = cert_file
+
+
+    def FakeSocket (sock, sslobj):
+        warnings.warn("FakeSocket is deprecated, and won't be in 3.x.  " +
+                      "Use the result of ssl.wrap_socket() directly instead.",
+                      DeprecationWarning, stacklevel=2)
+        return sslobj
+
+
+class HTTPException(Exception):
+    # Subclasses that define an __init__ must call Exception.__init__
+    # or define self.args.  Otherwise, str() will fail.
+    pass
+
+class NotConnected(HTTPException):
+    pass
+
+class InvalidURL(HTTPException):
+    pass
+
+class UnknownProtocol(HTTPException):
+    def __init__(self, version):
+        self.args = version,
+        self.version = version
+
+class UnknownTransferEncoding(HTTPException):
+    pass
+
+class UnimplementedFileMode(HTTPException):
+    pass
+
+class IncompleteRead(HTTPException):
+    def __init__(self, partial, expected=None):
+        self.args = partial,
+        self.partial = partial
+        self.expected = expected
+    def __repr__(self):
+        if self.expected is not None:
+            e = ', %i more expected' % self.expected
+        else:
+            e = ''
+        return 'IncompleteRead(%i bytes read%s)' % (len(self.partial), e)
+    def __str__(self):
+        return repr(self)
+
+class ImproperConnectionState(HTTPException):
+    pass
+
+class CannotSendRequest(ImproperConnectionState):
+    pass
+
+class CannotSendHeader(ImproperConnectionState):
+    pass
+
+class ResponseNotReady(ImproperConnectionState):
+    pass
+
+class BadStatusLine(HTTPException):
+    def __init__(self, line):
+        if not line:
+            line = repr(line)
+        self.args = line,
+        self.line = line
+
+# for backwards compatibility
+error = HTTPException
+
+class LineAndFileWrapper:
+    """A limited file-like object for HTTP/0.9 responses."""
+
+    # The status-line parsing code calls readline(), which normally
+    # get the HTTP status line.  For a 0.9 response, however, this is
+    # actually the first line of the body!  Clients need to get a
+    # readable file object that contains that line.
+
+    def __init__(self, line, file):
+        self._line = line
+        self._file = file
+        self._line_consumed = 0
+        self._line_offset = 0
+        self._line_left = len(line)
+
+    def __getattr__(self, attr):
+        return getattr(self._file, attr)
+
+    def _done(self):
+        # called when the last byte is read from the line.  After the
+        # call, all read methods are delegated to the underlying file
+        # object.
+        self._line_consumed = 1
+        self.read = self._file.read
+        self.readline = self._file.readline
+        self.readlines = self._file.readlines
+
+    def read(self, amt=None):
+        if self._line_consumed:
+            return self._file.read(amt)
+        assert self._line_left
+        if amt is None or amt > self._line_left:
+            s = self._line[self._line_offset:]
+            self._done()
+            if amt is None:
+                return s + self._file.read()
+            else:
+                return s + self._file.read(amt - len(s))
+        else:
+            assert amt <= self._line_left
+            i = self._line_offset
+            j = i + amt
+            s = self._line[i:j]
+            self._line_offset = j
+            self._line_left -= amt
+            if self._line_left == 0:
+                self._done()
+            return s
+
+    def readline(self):
+        if self._line_consumed:
+            return self._file.readline()
+        assert self._line_left
+        s = self._line[self._line_offset:]
+        self._done()
+        return s
+
+    def readlines(self, size=None):
+        if self._line_consumed:
+            return self._file.readlines(size)
+        assert self._line_left
+        L = [self._line[self._line_offset:]]
+        self._done()
+        if size is None:
+            return L + self._file.readlines()
+        else:
+            return L + self._file.readlines(size)
+
+def test():
+    """Test this module.
+
+    A hodge podge of tests collected here, because they have too many
+    external dependencies for the regular test suite.
+    """
+
+    import sys
+    import getopt
+    opts, args = getopt.getopt(sys.argv[1:], 'd')
+    dl = 0
+    for o, a in opts:
+        if o == '-d': dl = dl + 1
+    host = 'www.python.org'
+    selector = '/'
+    if args[0:]: host = args[0]
+    if args[1:]: selector = args[1]
+    h = HTTP()
+    h.set_debuglevel(dl)
+    h.connect(host)
+    h.putrequest('GET', selector)
+    h.endheaders()
+    status, reason, headers = h.getreply()
+    print 'status =', status
+    print 'reason =', reason
+    print "read", len(h.getfile().read())
+    print
+    if headers:
+        for header in headers.headers: print header.strip()
+    print
+
+    # minimal test that code to extract host from url works
+    class HTTP11(HTTP):
+        _http_vsn = 11
+        _http_vsn_str = 'HTTP/1.1'
+
+    h = HTTP11('www.python.org')
+    h.putrequest('GET', 'http://www.python.org/~jeremy/')
+    h.endheaders()
+    h.getreply()
+    h.close()
+
+    try:
+        import ssl
+    except ImportError:
+        pass
+    else:
+
+        for host, selector in (('sourceforge.net', '/projects/python'),
+                               ):
+            print "https://%s%s" % (host, selector)
+            hs = HTTPS()
+            hs.set_debuglevel(dl)
+            hs.connect(host)
+            hs.putrequest('GET', selector)
+            hs.endheaders()
+            status, reason, headers = hs.getreply()
+            print 'status =', status
+            print 'reason =', reason
+            print "read", len(hs.getfile().read())
+            print
+            if headers:
+                for header in headers.headers: print header.strip()
+            print
+
+if __name__ == '__main__':
+    test()
diff --git a/lib-python/modified-2.7/test/test_array.py b/lib-python/modified-2.7/test/test_array.py
--- a/lib-python/modified-2.7/test/test_array.py
+++ b/lib-python/modified-2.7/test/test_array.py
@@ -295,9 +295,10 @@
         )
 
         b = array.array(self.badtypecode())
-        self.assertRaises(TypeError, "a + b")
-
-        self.assertRaises(TypeError, "a + 'bad'")
+        with self.assertRaises(TypeError):
+            a + b
+        with self.assertRaises(TypeError):
+            a + 'bad'
 
     def test_iadd(self):
         a = array.array(self.typecode, self.example[::-1])
@@ -316,9 +317,10 @@
         )
 
         b = array.array(self.badtypecode())
-        self.assertRaises(TypeError, "a += b")
-
-        self.assertRaises(TypeError, "a += 'bad'")
+        with self.assertRaises(TypeError):
+            a += b
+        with self.assertRaises(TypeError):
+            a += 'bad'
 
     def test_mul(self):
         a = 5*array.array(self.typecode, self.example)
@@ -345,7 +347,8 @@
             array.array(self.typecode)
         )
 
-        self.assertRaises(TypeError, "a * 'bad'")
+        with self.assertRaises(TypeError):
+            a * 'bad'
 
     def test_imul(self):
         a = array.array(self.typecode, self.example)
@@ -374,7 +377,8 @@
         a *= -1
         self.assertEqual(a, array.array(self.typecode))
 
-        self.assertRaises(TypeError, "a *= 'bad'")
+        with self.assertRaises(TypeError):
+            a *= 'bad'
 
     def test_getitem(self):
         a = array.array(self.typecode, self.example)
diff --git a/lib-python/modified-2.7/test/test_urllib2.py b/lib-python/modified-2.7/test/test_urllib2.py
--- a/lib-python/modified-2.7/test/test_urllib2.py
+++ b/lib-python/modified-2.7/test/test_urllib2.py
@@ -307,6 +307,9 @@
     def getresponse(self):
         return MockHTTPResponse(MockFile(), {}, 200, "OK")
 
+    def close(self):
+        pass
+
 class MockHandler:
     # useful for testing handler machinery
     # see add_ordered_mock_handlers() docstring
diff --git a/lib-python/modified-2.7/urllib2.py b/lib-python/modified-2.7/urllib2.py
new file mode 100644
--- /dev/null
+++ b/lib-python/modified-2.7/urllib2.py
@@ -0,0 +1,1440 @@
+"""An extensible library for opening URLs using a variety of protocols
+
+The simplest way to use this module is to call the urlopen function,
+which accepts a string containing a URL or a Request object (described
+below).  It opens the URL and returns the results as file-like
+object; the returned object has some extra methods described below.
+
+The OpenerDirector manages a collection of Handler objects that do
+all the actual work.  Each Handler implements a particular protocol or
+option.  The OpenerDirector is a composite object that invokes the
+Handlers needed to open the requested URL.  For example, the
+HTTPHandler performs HTTP GET and POST requests and deals with
+non-error returns.  The HTTPRedirectHandler automatically deals with
+HTTP 301, 302, 303 and 307 redirect errors, and the HTTPDigestAuthHandler
+deals with digest authentication.
+
+urlopen(url, data=None) -- Basic usage is the same as original
+urllib.  pass the url and optionally data to post to an HTTP URL, and
+get a file-like object back.  One difference is that you can also pass
+a Request instance instead of URL.  Raises a URLError (subclass of
+IOError); for HTTP errors, raises an HTTPError, which can also be
+treated as a valid response.
+
+build_opener -- Function that creates a new OpenerDirector instance.
+Will install the default handlers.  Accepts one or more Handlers as
+arguments, either instances or Handler classes that it will
+instantiate.  If one of the argument is a subclass of the default
+handler, the argument will be installed instead of the default.
+
+install_opener -- Installs a new opener as the default opener.
+
+objects of interest:
+
+OpenerDirector -- Sets up the User Agent as the Python-urllib client and manages
+the Handler classes, while dealing with requests and responses.
+
+Request -- An object that encapsulates the state of a request.  The
+state can be as simple as the URL.  It can also include extra HTTP
+headers, e.g. a User-Agent.
+
+BaseHandler --
+
+exceptions:
+URLError -- A subclass of IOError, individual protocols have their own
+specific subclass.
+
+HTTPError -- Also a valid HTTP response, so you can treat an HTTP error
+as an exceptional event or valid response.
+
+internals:
+BaseHandler and parent
+_call_chain conventions
+
+Example usage:
+
+import urllib2
+
+# set up authentication info
+authinfo = urllib2.HTTPBasicAuthHandler()
+authinfo.add_password(realm='PDQ Application',
+                      uri='https://mahler:8092/site-updates.py',
+                      user='klem',
+                      passwd='geheim$parole')
+
+proxy_support = urllib2.ProxyHandler({"http" : "http://ahad-haam:3128"})
+
+# build a new opener that adds authentication and caching FTP handlers
+opener = urllib2.build_opener(proxy_support, authinfo, urllib2.CacheFTPHandler)
+
+# install it
+urllib2.install_opener(opener)
+
+f = urllib2.urlopen('http://www.python.org/')
+
+
+"""
+
+# XXX issues:
+# If an authentication error handler that tries to perform
+# authentication for some reason but fails, how should the error be
+# signalled?  The client needs to know the HTTP error code.  But if
+# the handler knows that the problem was, e.g., that it didn't know
+# that hash algo that requested in the challenge, it would be good to
+# pass that information along to the client, too.
+# ftp errors aren't handled cleanly
+# check digest against correct (i.e. non-apache) implementation
+
+# Possible extensions:
+# complex proxies  XXX not sure what exactly was meant by this
+# abstract factory for opener
+
+import base64
+import hashlib
+import httplib
+import mimetools
+import os
+import posixpath
+import random
+import re
+import socket
+import sys
+import time
+import urlparse
+import bisect
+
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from StringIO import StringIO
+
+from urllib import (unwrap, unquote, splittype, splithost, quote,
+     addinfourl, splitport, splittag,
+     splitattr, ftpwrapper, splituser, splitpasswd, splitvalue)
+
+# support for FileHandler, proxies via environment variables
+from urllib import localhost, url2pathname, getproxies, proxy_bypass
+
+# used in User-Agent header sent
+__version__ = sys.version[:3]
+
+_opener = None
+def urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
+    global _opener
+    if _opener is None:
+        _opener = build_opener()
+    return _opener.open(url, data, timeout)
+
+def install_opener(opener):
+    global _opener
+    _opener = opener
+
+# do these error classes make sense?
+# make sure all of the IOError stuff is overridden.  we just want to be
+# subtypes.
+
+class URLError(IOError):
+    # URLError is a sub-type of IOError, but it doesn't share any of
+    # the implementation.  need to override __init__ and __str__.
+    # It sets self.args for compatibility with other EnvironmentError
+    # subclasses, but args doesn't have the typical format with errno in
+    # slot 0 and strerror in slot 1.  This may be better than nothing.
+    def __init__(self, reason):
+        self.args = reason,
+        self.reason = reason
+
+    def __str__(self):
+        return '<urlopen error %s>' % self.reason
+
+class HTTPError(URLError, addinfourl):
+    """Raised when HTTP error occurs, but also acts like non-error return"""
+    __super_init = addinfourl.__init__
+
+    def __init__(self, url, code, msg, hdrs, fp):
+        self.code = code
+        self.msg = msg
+        self.hdrs = hdrs
+        self.fp = fp
+        self.filename = url
+        # The addinfourl classes depend on fp being a valid file
+        # object.  In some cases, the HTTPError may not have a valid
+        # file object.  If this happens, the simplest workaround is to
+        # not initialize the base classes.
+        if fp is not None:
+            self.__super_init(fp, hdrs, url, code)
+
+    def __str__(self):
+        return 'HTTP Error %s: %s' % (self.code, self.msg)
+
+# copied from cookielib.py
+_cut_port_re = re.compile(r":\d+$")
+def request_host(request):
+    """Return request-host, as defined by RFC 2965.
+
+    Variation from RFC: returned value is lowercased, for convenient
+    comparison.
+
+    """
+    url = request.get_full_url()
+    host = urlparse.urlparse(url)[1]
+    if host == "":
+        host = request.get_header("Host", "")
+
+    # remove port, if present
+    host = _cut_port_re.sub("", host, 1)
+    return host.lower()
+
+class Request:
+
+    def __init__(self, url, data=None, headers={},
+                 origin_req_host=None, unverifiable=False):
+        # unwrap('<URL:type://host/path>') --> 'type://host/path'
+        self.__original = unwrap(url)
+        self.__original, fragment = splittag(self.__original)
+        self.type = None
+        # self.__r_type is what's left after doing the splittype
+        self.host = None
+        self.port = None
+        self._tunnel_host = None
+        self.data = data
+        self.headers = {}
+        for key, value in headers.items():
+            self.add_header(key, value)
+        self.unredirected_hdrs = {}
+        if origin_req_host is None:
+            origin_req_host = request_host(self)
+        self.origin_req_host = origin_req_host
+        self.unverifiable = unverifiable
+
+    def __getattr__(self, attr):
+        # XXX this is a fallback mechanism to guard against these
+        # methods getting called in a non-standard order.  this may be
+        # too complicated and/or unnecessary.
+        # XXX should the __r_XXX attributes be public?
+        if attr[:12] == '_Request__r_':
+            name = attr[12:]
+            if hasattr(Request, 'get_' + name):
+                getattr(self, 'get_' + name)()
+                return getattr(self, attr)
+        raise AttributeError, attr
+
+    def get_method(self):
+        if self.has_data():
+            return "POST"
+        else:
+            return "GET"
+
+    # XXX these helper methods are lame
+
+    def add_data(self, data):
+        self.data = data
+
+    def has_data(self):
+        return self.data is not None
+
+    def get_data(self):
+        return self.data
+
+    def get_full_url(self):
+        return self.__original
+
+    def get_type(self):
+        if self.type is None:
+            self.type, self.__r_type = splittype(self.__original)
+            if self.type is None:
+                raise ValueError, "unknown url type: %s" % self.__original
+        return self.type
+
+    def get_host(self):
+        if self.host is None:
+            self.host, self.__r_host = splithost(self.__r_type)
+            if self.host:
+                self.host = unquote(self.host)
+        return self.host
+
+    def get_selector(self):
+        return self.__r_host
+
+    def set_proxy(self, host, type):
+        if self.type == 'https' and not self._tunnel_host:
+            self._tunnel_host = self.host
+        else:
+            self.type = type
+            self.__r_host = self.__original
+
+        self.host = host
+
+    def has_proxy(self):
+        return self.__r_host == self.__original
+
+    def get_origin_req_host(self):
+        return self.origin_req_host
+
+    def is_unverifiable(self):
+        return self.unverifiable
+
+    def add_header(self, key, val):
+        # useful for something like authentication
+        self.headers[key.capitalize()] = val
+
+    def add_unredirected_header(self, key, val):
+        # will not be added to a redirected request
+        self.unredirected_hdrs[key.capitalize()] = val
+
+    def has_header(self, header_name):
+        return (header_name in self.headers or
+                header_name in self.unredirected_hdrs)
+
+    def get_header(self, header_name, default=None):
+        return self.headers.get(
+            header_name,
+            self.unredirected_hdrs.get(header_name, default))
+
+    def header_items(self):
+        hdrs = self.unredirected_hdrs.copy()
+        hdrs.update(self.headers)
+        return hdrs.items()
+
+class OpenerDirector:
+    def __init__(self):
+        client_version = "Python-urllib/%s" % __version__
+        self.addheaders = [('User-agent', client_version)]
+        # manage the individual handlers
+        self.handlers = []
+        self.handle_open = {}
+        self.handle_error = {}
+        self.process_response = {}
+        self.process_request = {}
+
+    def add_handler(self, handler):
+        if not hasattr(handler, "add_parent"):
+            raise TypeError("expected BaseHandler instance, got %r" %
+                            type(handler))
+
+        added = False
+        for meth in dir(handler):
+            if meth in ["redirect_request", "do_open", "proxy_open"]:
+                # oops, coincidental match
+                continue
+
+            i = meth.find("_")
+            protocol = meth[:i]
+            condition = meth[i+1:]
+
+            if condition.startswith("error"):
+                j = condition.find("_") + i + 1
+                kind = meth[j+1:]
+                try:
+                    kind = int(kind)
+                except ValueError:
+                    pass
+                lookup = self.handle_error.get(protocol, {})
+                self.handle_error[protocol] = lookup
+            elif condition == "open":
+                kind = protocol
+                lookup = self.handle_open
+            elif condition == "response":
+                kind = protocol
+                lookup = self.process_response
+            elif condition == "request":
+                kind = protocol
+                lookup = self.process_request
+            else:
+                continue
+
+            handlers = lookup.setdefault(kind, [])
+            if handlers:
+                bisect.insort(handlers, handler)
+            else:
+                handlers.append(handler)
+            added = True
+
+        if added:
+            # the handlers must work in an specific order, the order
+            # is specified in a Handler attribute
+            bisect.insort(self.handlers, handler)
+            handler.add_parent(self)
+
+    def close(self):
+        # Only exists for backwards compatibility.
+        pass
+
+    def _call_chain(self, chain, kind, meth_name, *args):
+        # Handlers raise an exception if no one else should try to handle
+        # the request, or return None if they can't but another handler
+        # could.  Otherwise, they return the response.
+        handlers = chain.get(kind, ())
+        for handler in handlers:
+            func = getattr(handler, meth_name)
+
+            result = func(*args)
+            if result is not None:
+                return result
+
+    def open(self, fullurl, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
+        # accept a URL or a Request object
+        if isinstance(fullurl, basestring):
+            req = Request(fullurl, data)
+        else:
+            req = fullurl
+            if data is not None:
+                req.add_data(data)
+
+        req.timeout = timeout
+        protocol = req.get_type()
+
+        # pre-process request
+        meth_name = protocol+"_request"
+        for processor in self.process_request.get(protocol, []):
+            meth = getattr(processor, meth_name)
+            req = meth(req)
+
+        response = self._open(req, data)
+
+        # post-process response
+        meth_name = protocol+"_response"
+        for processor in self.process_response.get(protocol, []):
+            meth = getattr(processor, meth_name)
+            try:
+                response = meth(req, response)
+            except:
+                response.close()
+                raise
+
+        return response
+
+    def _open(self, req, data=None):
+        result = self._call_chain(self.handle_open, 'default',
+                                  'default_open', req)
+        if result:
+            return result
+
+        protocol = req.get_type()
+        result = self._call_chain(self.handle_open, protocol, protocol +
+                                  '_open', req)
+        if result:
+            return result
+
+        return self._call_chain(self.handle_open, 'unknown',
+                                'unknown_open', req)
+
+    def error(self, proto, *args):
+        if proto in ('http', 'https'):
+            # XXX http[s] protocols are special-cased
+            dict = self.handle_error['http'] # https is not different than http
+            proto = args[2]  # YUCK!
+            meth_name = 'http_error_%s' % proto
+            http_err = 1
+            orig_args = args
+        else:
+            dict = self.handle_error
+            meth_name = proto + '_error'
+            http_err = 0
+        args = (dict, proto, meth_name) + args
+        result = self._call_chain(*args)
+        if result:
+            return result
+
+        if http_err:
+            args = (dict, 'default', 'http_error_default') + orig_args
+            return self._call_chain(*args)
+
+# XXX probably also want an abstract factory that knows when it makes
+# sense to skip a superclass in favor of a subclass and when it might
+# make sense to include both
+
+def build_opener(*handlers):
+    """Create an opener object from a list of handlers.
+
+    The opener will use several default handlers, including support
+    for HTTP, FTP and when applicable, HTTPS.
+
+    If any of the handlers passed as arguments are subclasses of the
+    default handlers, the default handlers will not be used.
+    """
+    import types
+    def isclass(obj):
+        return isinstance(obj, (types.ClassType, type))
+
+    opener = OpenerDirector()
+    default_classes = [ProxyHandler, UnknownHandler, HTTPHandler,
+                       HTTPDefaultErrorHandler, HTTPRedirectHandler,
+                       FTPHandler, FileHandler, HTTPErrorProcessor]
+    if hasattr(httplib, 'HTTPS'):
+        default_classes.append(HTTPSHandler)
+    skip = set()
+    for klass in default_classes:
+        for check in handlers:
+            if isclass(check):
+                if issubclass(check, klass):
+                    skip.add(klass)
+            elif isinstance(check, klass):
+                skip.add(klass)
+    for klass in skip:
+        default_classes.remove(klass)
+
+    for klass in default_classes:
+        opener.add_handler(klass())
+
+    for h in handlers:
+        if isclass(h):
+            h = h()
+        opener.add_handler(h)
+    return opener
+
+class BaseHandler:
+    handler_order = 500
+
+    def add_parent(self, parent):
+        self.parent = parent
+
+    def close(self):
+        # Only exists for backwards compatibility
+        pass
+
+    def __lt__(self, other):
+        if not hasattr(other, "handler_order"):
+            # Try to preserve the old behavior of having custom classes
+            # inserted after default ones (works only for custom user
+            # classes which are not aware of handler_order).
+            return True
+        return self.handler_order < other.handler_order
+
+
+class HTTPErrorProcessor(BaseHandler):
+    """Process HTTP error responses."""
+    handler_order = 1000  # after all other processing
+
+    def http_response(self, request, response):
+        code, msg, hdrs = response.code, response.msg, response.info()
+
+        # According to RFC 2616, "2xx" code indicates that the client's
+        # request was successfully received, understood, and accepted.
+        if not (200 <= code < 300):
+            response = self.parent.error(
+                'http', request, response, code, msg, hdrs)
+
+        return response
+
+    https_response = http_response
+
+class HTTPDefaultErrorHandler(BaseHandler):
+    def http_error_default(self, req, fp, code, msg, hdrs):
+        raise HTTPError(req.get_full_url(), code, msg, hdrs, fp)
+
+class HTTPRedirectHandler(BaseHandler):
+    # maximum number of redirections to any single URL
+    # this is needed because of the state that cookies introduce
+    max_repeats = 4
+    # maximum total number of redirections (regardless of URL) before
+    # assuming we're in a loop
+    max_redirections = 10
+
+    def redirect_request(self, req, fp, code, msg, headers, newurl):
+        """Return a Request or None in response to a redirect.
+
+        This is called by the http_error_30x methods when a
+        redirection response is received.  If a redirection should
+        take place, return a new Request to allow http_error_30x to
+        perform the redirect.  Otherwise, raise HTTPError if no-one
+        else should try to handle this url.  Return None if you can't
+        but another Handler might.
+        """
+        m = req.get_method()
+        if (code in (301, 302, 303, 307) and m in ("GET", "HEAD")
+            or code in (301, 302, 303) and m == "POST"):
+            # Strictly (according to RFC 2616), 301 or 302 in response
+            # to a POST MUST NOT cause a redirection without confirmation
+            # from the user (of urllib2, in this case).  In practice,
+            # essentially all clients do redirect in this case, so we
+            # do the same.
+            # be conciliant with URIs containing a space
+            newurl = newurl.replace(' ', '%20')
+            newheaders = dict((k,v) for k,v in req.headers.items()
+                              if k.lower() not in ("content-length", "content-type")
+                             )
+            return Request(newurl,
+                           headers=newheaders,
+                           origin_req_host=req.get_origin_req_host(),
+                           unverifiable=True)
+        else:
+            raise HTTPError(req.get_full_url(), code, msg, headers, fp)
+
+    # Implementation note: To avoid the server sending us into an
+    # infinite loop, the request object needs to track what URLs we
+    # have already seen.  Do this by adding a handler-specific
+    # attribute to the Request object.
+    def http_error_302(self, req, fp, code, msg, headers):
+        # Some servers (incorrectly) return multiple Location headers
+        # (so probably same goes for URI).  Use first header.
+        if 'location' in headers:
+            newurl = headers.getheaders('location')[0]
+        elif 'uri' in headers:
+            newurl = headers.getheaders('uri')[0]
+        else:
+            return
+
+        # fix a possible malformed URL
+        urlparts = urlparse.urlparse(newurl)
+        if not urlparts.path:
+            urlparts = list(urlparts)
+            urlparts[2] = "/"
+        newurl = urlparse.urlunparse(urlparts)
+
+        newurl = urlparse.urljoin(req.get_full_url(), newurl)
+
+        # XXX Probably want to forget about the state of the current
+        # request, although that might interact poorly with other
+        # handlers that also use handler-specific request attributes
+        new = self.redirect_request(req, fp, code, msg, headers, newurl)
+        if new is None:
+            return
+
+        # loop detection
+        # .redirect_dict has a key url if url was previously visited.
+        if hasattr(req, 'redirect_dict'):
+            visited = new.redirect_dict = req.redirect_dict
+            if (visited.get(newurl, 0) >= self.max_repeats or
+                len(visited) >= self.max_redirections):
+                raise HTTPError(req.get_full_url(), code,
+                                self.inf_msg + msg, headers, fp)
+        else:
+            visited = new.redirect_dict = req.redirect_dict = {}
+        visited[newurl] = visited.get(newurl, 0) + 1
+
+        # Don't close the fp until we are sure that we won't use it
+        # with HTTPError.
+        fp.read()
+        fp.close()
+
+        return self.parent.open(new, timeout=req.timeout)
+
+    http_error_301 = http_error_303 = http_error_307 = http_error_302
+
+    inf_msg = "The HTTP server returned a redirect error that would " \
+              "lead to an infinite loop.\n" \
+              "The last 30x error message was:\n"
+
+
+def _parse_proxy(proxy):
+    """Return (scheme, user, password, host/port) given a URL or an authority.
+
+    If a URL is supplied, it must have an authority (host:port) component.
+    According to RFC 3986, having an authority component means the URL must
+    have two slashes after the scheme:
+
+    >>> _parse_proxy('file:/ftp.example.com/')
+    Traceback (most recent call last):
+    ValueError: proxy URL with no authority: 'file:/ftp.example.com/'
+
+    The first three items of the returned tuple may be None.
+
+    Examples of authority parsing:
+
+    >>> _parse_proxy('proxy.example.com')
+    (None, None, None, 'proxy.example.com')
+    >>> _parse_proxy('proxy.example.com:3128')
+    (None, None, None, 'proxy.example.com:3128')
+
+    The authority component may optionally include userinfo (assumed to be
+    username:password):
+
+    >>> _parse_proxy('joe:password at proxy.example.com')
+    (None, 'joe', 'password', 'proxy.example.com')
+    >>> _parse_proxy('joe:password at proxy.example.com:3128')
+    (None, 'joe', 'password', 'proxy.example.com:3128')
+
+    Same examples, but with URLs instead:
+
+    >>> _parse_proxy('http://proxy.example.com/')
+    ('http', None, None, 'proxy.example.com')
+    >>> _parse_proxy('http://proxy.example.com:3128/')
+    ('http', None, None, 'proxy.example.com:3128')
+    >>> _parse_proxy('http://joe:password at proxy.example.com/')
+    ('http', 'joe', 'password', 'proxy.example.com')
+    >>> _parse_proxy('http://joe:password at proxy.example.com:3128')
+    ('http', 'joe', 'password', 'proxy.example.com:3128')
+
+    Everything after the authority is ignored:
+
+    >>> _parse_proxy('ftp://joe:password at proxy.example.com/rubbish:3128')
+    ('ftp', 'joe', 'password', 'proxy.example.com')
+
+    Test for no trailing '/' case:
+
+    >>> _parse_proxy('http://joe:password at proxy.example.com')
+    ('http', 'joe', 'password', 'proxy.example.com')
+
+    """
+    scheme, r_scheme = splittype(proxy)
+    if not r_scheme.startswith("/"):
+        # authority
+        scheme = None
+        authority = proxy
+    else:
+        # URL
+        if not r_scheme.startswith("//"):
+            raise ValueError("proxy URL with no authority: %r" % proxy)
+        # We have an authority, so for RFC 3986-compliant URLs (by ss 3.
+        # and 3.3.), path is empty or starts with '/'
+        end = r_scheme.find("/", 2)
+        if end == -1:
+            end = None
+        authority = r_scheme[2:end]
+    userinfo, hostport = splituser(authority)
+    if userinfo is not None:
+        user, password = splitpasswd(userinfo)
+    else:
+        user = password = None
+    return scheme, user, password, hostport
+
+class ProxyHandler(BaseHandler):
+    # Proxies must be in front
+    handler_order = 100
+
+    def __init__(self, proxies=None):
+        if proxies is None:
+            proxies = getproxies()
+        assert hasattr(proxies, 'has_key'), "proxies must be a mapping"
+        self.proxies = proxies
+        for type, url in proxies.items():
+            setattr(self, '%s_open' % type,
+                    lambda r, proxy=url, type=type, meth=self.proxy_open: \
+                    meth(r, proxy, type))
+
+    def proxy_open(self, req, proxy, type):
+        orig_type = req.get_type()
+        proxy_type, user, password, hostport = _parse_proxy(proxy)
+
+        if proxy_type is None:
+            proxy_type = orig_type
+
+        if req.host and proxy_bypass(req.host):
+            return None
+
+        if user and password:
+            user_pass = '%s:%s' % (unquote(user), unquote(password))
+            creds = base64.b64encode(user_pass).strip()
+            req.add_header('Proxy-authorization', 'Basic ' + creds)
+        hostport = unquote(hostport)
+        req.set_proxy(hostport, proxy_type)
+
+        if orig_type == proxy_type or orig_type == 'https':
+            # let other handlers take care of it
+            return None
+        else:
+            # need to start over, because the other handlers don't
+            # grok the proxy's URL type
+            # e.g. if we have a constructor arg proxies like so:
+            # {'http': 'ftp://proxy.example.com'}, we may end up turning
+            # a request for http://acme.example.com/a into one for
+            # ftp://proxy.example.com/a
+            return self.parent.open(req, timeout=req.timeout)
+
+class HTTPPasswordMgr:
+
+    def __init__(self):
+        self.passwd = {}
+
+    def add_password(self, realm, uri, user, passwd):
+        # uri could be a single URI or a sequence
+        if isinstance(uri, basestring):
+            uri = [uri]
+        if not realm in self.passwd:
+            self.passwd[realm] = {}
+        for default_port in True, False:
+            reduced_uri = tuple(
+                [self.reduce_uri(u, default_port) for u in uri])
+            self.passwd[realm][reduced_uri] = (user, passwd)
+
+    def find_user_password(self, realm, authuri):
+        domains = self.passwd.get(realm, {})
+        for default_port in True, False:
+            reduced_authuri = self.reduce_uri(authuri, default_port)
+            for uris, authinfo in domains.iteritems():
+                for uri in uris:
+                    if self.is_suburi(uri, reduced_authuri):
+                        return authinfo
+        return None, None
+
+    def reduce_uri(self, uri, default_port=True):
+        """Accept authority or URI and extract only the authority and path."""
+        # note HTTP URLs do not have a userinfo component
+        parts = urlparse.urlsplit(uri)
+        if parts[1]:
+            # URI
+            scheme = parts[0]
+            authority = parts[1]
+            path = parts[2] or '/'
+        else:
+            # host or host:port
+            scheme = None
+            authority = uri
+            path = '/'
+        host, port = splitport(authority)
+        if default_port and port is None and scheme is not None:
+            dport = {"http": 80,
+                     "https": 443,
+                     }.get(scheme)
+            if dport is not None:
+                authority = "%s:%d" % (host, dport)
+        return authority, path
+
+    def is_suburi(self, base, test):
+        """Check if test is below base in a URI tree
+
+        Both args must be URIs in reduced form.
+        """
+        if base == test:
+            return True
+        if base[0] != test[0]:
+            return False
+        common = posixpath.commonprefix((base[1], test[1]))
+        if len(common) == len(base[1]):
+            return True
+        return False
+
+
+class HTTPPasswordMgrWithDefaultRealm(HTTPPasswordMgr):
+
+    def find_user_password(self, realm, authuri):
+        user, password = HTTPPasswordMgr.find_user_password(self, realm,
+                                                            authuri)
+        if user is not None:
+            return user, password
+        return HTTPPasswordMgr.find_user_password(self, None, authuri)
+
+
+class AbstractBasicAuthHandler:
+
+    # XXX this allows for multiple auth-schemes, but will stupidly pick
+    # the last one with a realm specified.
+
+    # allow for double- and single-quoted realm values
+    # (single quotes are a violation of the RFC, but appear in the wild)
+    rx = re.compile('(?:.*,)*[ \t]*([^ \t]+)[ \t]+'
+                    'realm=(["\'])(.*?)\\2', re.I)
+
+    # XXX could pre-emptively send auth info already accepted (RFC 2617,
+    # end of section 2, and section 1.2 immediately after "credentials"
+    # production).
+
+    def __init__(self, password_mgr=None):
+        if password_mgr is None:
+            password_mgr = HTTPPasswordMgr()
+        self.passwd = password_mgr
+        self.add_password = self.passwd.add_password
+        self.retried = 0
+
+    def reset_retry_count(self):
+        self.retried = 0
+
+    def http_error_auth_reqed(self, authreq, host, req, headers):
+        # host may be an authority (without userinfo) or a URL with an
+        # authority
+        # XXX could be multiple headers
+        authreq = headers.get(authreq, None)
+
+        if self.retried > 5:
+            # retry sending the username:password 5 times before failing.
+            raise HTTPError(req.get_full_url(), 401, "basic auth failed",
+                            headers, None)
+        else:
+            self.retried += 1
+
+        if authreq:
+            mo = AbstractBasicAuthHandler.rx.search(authreq)
+            if mo:
+                scheme, quote, realm = mo.groups()
+                if scheme.lower() == 'basic':
+                    response = self.retry_http_basic_auth(host, req, realm)
+                    if response and response.code != 401:
+                        self.retried = 0
+                    return response
+
+    def retry_http_basic_auth(self, host, req, realm):
+        user, pw = self.passwd.find_user_password(realm, host)
+        if pw is not None:
+            raw = "%s:%s" % (user, pw)
+            auth = 'Basic %s' % base64.b64encode(raw).strip()
+            if req.headers.get(self.auth_header, None) == auth:
+                return None
+            req.add_unredirected_header(self.auth_header, auth)
+            return self.parent.open(req, timeout=req.timeout)
+        else:
+            return None
+
+
+class HTTPBasicAuthHandler(AbstractBasicAuthHandler, BaseHandler):
+
+    auth_header = 'Authorization'
+
+    def http_error_401(self, req, fp, code, msg, headers):
+        url = req.get_full_url()
+        response = self.http_error_auth_reqed('www-authenticate',
+                                              url, req, headers)
+        self.reset_retry_count()
+        return response
+
+
+class ProxyBasicAuthHandler(AbstractBasicAuthHandler, BaseHandler):
+
+    auth_header = 'Proxy-authorization'
+
+    def http_error_407(self, req, fp, code, msg, headers):
+        # http_error_auth_reqed requires that there is no userinfo component in
+        # authority.  Assume there isn't one, since urllib2 does not (and
+        # should not, RFC 3986 s. 3.2.1) support requests for URLs containing
+        # userinfo.
+        authority = req.get_host()
+        response = self.http_error_auth_reqed('proxy-authenticate',
+                                          authority, req, headers)
+        self.reset_retry_count()
+        return response
+
+
+def randombytes(n):
+    """Return n random bytes."""
+    # Use /dev/urandom if it is available.  Fall back to random module
+    # if not.  It might be worthwhile to extend this function to use
+    # other platform-specific mechanisms for getting random bytes.
+    if os.path.exists("/dev/urandom"):
+        f = open("/dev/urandom")
+        s = f.read(n)
+        f.close()
+        return s
+    else:
+        L = [chr(random.randrange(0, 256)) for i in range(n)]
+        return "".join(L)
+
+class AbstractDigestAuthHandler:
+    # Digest authentication is specified in RFC 2617.
+
+    # XXX The client does not inspect the Authentication-Info header
+    # in a successful response.
+
+    # XXX It should be possible to test this implementation against
+    # a mock server that just generates a static set of challenges.
+
+    # XXX qop="auth-int" supports is shaky
+
+    def __init__(self, passwd=None):
+        if passwd is None:
+            passwd = HTTPPasswordMgr()
+        self.passwd = passwd
+        self.add_password = self.passwd.add_password
+        self.retried = 0
+        self.nonce_count = 0
+        self.last_nonce = None
+
+    def reset_retry_count(self):
+        self.retried = 0
+
+    def http_error_auth_reqed(self, auth_header, host, req, headers):
+        authreq = headers.get(auth_header, None)
+        if self.retried > 5:
+            # Don't fail endlessly - if we failed once, we'll probably
+            # fail a second time. Hm. Unless the Password Manager is
+            # prompting for the information. Crap. This isn't great
+            # but it's better than the current 'repeat until recursion
+            # depth exceeded' approach <wink>
+            raise HTTPError(req.get_full_url(), 401, "digest auth failed",
+                            headers, None)
+        else:
+            self.retried += 1
+        if authreq:
+            scheme = authreq.split()[0]
+            if scheme.lower() == 'digest':
+                return self.retry_http_digest_auth(req, authreq)
+
+    def retry_http_digest_auth(self, req, auth):
+        token, challenge = auth.split(' ', 1)
+        chal = parse_keqv_list(parse_http_list(challenge))
+        auth = self.get_authorization(req, chal)
+        if auth:
+            auth_val = 'Digest %s' % auth
+            if req.headers.get(self.auth_header, None) == auth_val:
+                return None
+            req.add_unredirected_header(self.auth_header, auth_val)
+            resp = self.parent.open(req, timeout=req.timeout)
+            return resp
+
+    def get_cnonce(self, nonce):
+        # The cnonce-value is an opaque
+        # quoted string value provided by the client and used by both client
+        # and server to avoid chosen plaintext attacks, to provide mutual
+        # authentication, and to provide some message integrity protection.
+        # This isn't a fabulous effort, but it's probably Good Enough.
+        dig = hashlib.sha1("%s:%s:%s:%s" % (self.nonce_count, nonce, time.ctime(),
+                                            randombytes(8))).hexdigest()
+        return dig[:16]
+
+    def get_authorization(self, req, chal):
+        try:
+            realm = chal['realm']
+            nonce = chal['nonce']
+            qop = chal.get('qop')
+            algorithm = chal.get('algorithm', 'MD5')
+            # mod_digest doesn't send an opaque, even though it isn't
+            # supposed to be optional
+            opaque = chal.get('opaque', None)
+        except KeyError:
+            return None
+
+        H, KD = self.get_algorithm_impls(algorithm)
+        if H is None:
+            return None
+
+        user, pw = self.passwd.find_user_password(realm, req.get_full_url())
+        if user is None:
+            return None
+
+        # XXX not implemented yet
+        if req.has_data():
+            entdig = self.get_entity_digest(req.get_data(), chal)
+        else:
+            entdig = None
+
+        A1 = "%s:%s:%s" % (user, realm, pw)
+        A2 = "%s:%s" % (req.get_method(),
+                        # XXX selector: what about proxies and full urls
+                        req.get_selector())
+        if qop == 'auth':
+            if nonce == self.last_nonce:
+                self.nonce_count += 1
+            else:
+                self.nonce_count = 1
+                self.last_nonce = nonce
+
+            ncvalue = '%08x' % self.nonce_count
+            cnonce = self.get_cnonce(nonce)
+            noncebit = "%s:%s:%s:%s:%s" % (nonce, ncvalue, cnonce, qop, H(A2))
+            respdig = KD(H(A1), noncebit)
+        elif qop is None:
+            respdig = KD(H(A1), "%s:%s" % (nonce, H(A2)))
+        else:
+            # XXX handle auth-int.
+            raise URLError("qop '%s' is not supported." % qop)
+
+        # XXX should the partial digests be encoded too?
+
+        base = 'username="%s", realm="%s", nonce="%s", uri="%s", ' \
+               'response="%s"' % (user, realm, nonce, req.get_selector(),
+                                  respdig)
+        if opaque:
+            base += ', opaque="%s"' % opaque
+        if entdig:
+            base += ', digest="%s"' % entdig
+        base += ', algorithm="%s"' % algorithm
+        if qop:
+            base += ', qop=auth, nc=%s, cnonce="%s"' % (ncvalue, cnonce)
+        return base
+
+    def get_algorithm_impls(self, algorithm):
+        # algorithm should be case-insensitive according to RFC2617
+        algorithm = algorithm.upper()
+        # lambdas assume digest modules are imported at the top level
+        if algorithm == 'MD5':
+            H = lambda x: hashlib.md5(x).hexdigest()
+        elif algorithm == 'SHA':
+            H = lambda x: hashlib.sha1(x).hexdigest()
+        # XXX MD5-sess
+        KD = lambda s, d: H("%s:%s" % (s, d))
+        return H, KD
+
+    def get_entity_digest(self, data, chal):
+        # XXX not implemented yet
+        return None
+
+
+class HTTPDigestAuthHandler(BaseHandler, AbstractDigestAuthHandler):
+    """An authentication protocol defined by RFC 2069
+
+    Digest authentication improves on basic authentication because it
+    does not transmit passwords in the clear.
+    """
+
+    auth_header = 'Authorization'
+    handler_order = 490  # before Basic auth
+
+    def http_error_401(self, req, fp, code, msg, headers):
+        host = urlparse.urlparse(req.get_full_url())[1]
+        retry = self.http_error_auth_reqed('www-authenticate',
+                                           host, req, headers)
+        self.reset_retry_count()
+        return retry
+
+
+class ProxyDigestAuthHandler(BaseHandler, AbstractDigestAuthHandler):
+
+    auth_header = 'Proxy-Authorization'
+    handler_order = 490  # before Basic auth
+
+    def http_error_407(self, req, fp, code, msg, headers):
+        host = req.get_host()
+        retry = self.http_error_auth_reqed('proxy-authenticate',
+                                           host, req, headers)
+        self.reset_retry_count()
+        return retry
+
+class AbstractHTTPHandler(BaseHandler):
+
+    def __init__(self, debuglevel=0):
+        self._debuglevel = debuglevel
+
+    def set_http_debuglevel(self, level):
+        self._debuglevel = level
+
+    def do_request_(self, request):
+        host = request.get_host()
+        if not host:
+            raise URLError('no host given')
+
+        if request.has_data():  # POST
+            data = request.get_data()
+            if not request.has_header('Content-type'):
+                request.add_unredirected_header(
+                    'Content-type',
+                    'application/x-www-form-urlencoded')
+            if not request.has_header('Content-length'):
+                request.add_unredirected_header(
+                    'Content-length', '%d' % len(data))
+
+        sel_host = host
+        if request.has_proxy():
+            scheme, sel = splittype(request.get_selector())
+            sel_host, sel_path = splithost(sel)
+
+        if not request.has_header('Host'):
+            request.add_unredirected_header('Host', sel_host)
+        for name, value in self.parent.addheaders:
+            name = name.capitalize()
+            if not request.has_header(name):
+                request.add_unredirected_header(name, value)
+
+        return request
+
+    def do_open(self, http_class, req):
+        """Return an addinfourl object for the request, using http_class.
+
+        http_class must implement the HTTPConnection API from httplib.
+        The addinfourl return value is a file-like object.  It also
+        has methods and attributes including:
+            - info(): return a mimetools.Message object for the headers
+            - geturl(): return the original request URL
+            - code: HTTP status code
+        """
+        host = req.get_host()
+        if not host:
+            raise URLError('no host given')
+
+        h = http_class(host, timeout=req.timeout) # will parse host:port
+        h.set_debuglevel(self._debuglevel)
+
+        headers = dict(req.unredirected_hdrs)
+        headers.update(dict((k, v) for k, v in req.headers.items()
+                            if k not in headers))
+
+        # We want to make an HTTP/1.1 request, but the addinfourl
+        # class isn't prepared to deal with a persistent connection.
+        # It will try to read all remaining data from the socket,
+        # which will block while the server waits for the next request.
+        # So make sure the connection gets closed after the (only)
+        # request.
+        headers["Connection"] = "close"
+        headers = dict(
+            (name.title(), val) for name, val in headers.items())
+
+        if req._tunnel_host:
+            tunnel_headers = {}
+            proxy_auth_hdr = "Proxy-Authorization"
+            if proxy_auth_hdr in headers:
+                tunnel_headers[proxy_auth_hdr] = headers[proxy_auth_hdr]
+                # Proxy-Authorization should not be sent to origin
+                # server.
+                del headers[proxy_auth_hdr]
+            h.set_tunnel(req._tunnel_host, headers=tunnel_headers)
+
+        try:
+            h.request(req.get_method(), req.get_selector(), req.data, headers)
+            try:
+                r = h.getresponse(buffering=True)
+            except TypeError: #buffering kw not supported
+                r = h.getresponse()
+        except socket.error, err: # XXX what error?
+            h.close()
+            raise URLError(err)
+
+        # Pick apart the HTTPResponse object to get the addinfourl
+        # object initialized properly.
+
+        # Wrap the HTTPResponse object in socket's file object adapter
+        # for Windows.  That adapter calls recv(), so delegate recv()
+        # to read().  This weird wrapping allows the returned object to
+        # have readline() and readlines() methods.
+
+        # XXX It might be better to extract the read buffering code
+        # out of socket._fileobject() and into a base class.
+
+        r.recv = r.read
+        fp = socket._fileobject(r, close=True)
+
+        resp = addinfourl(fp, r.msg, req.get_full_url())
+        resp.code = r.status
+        resp.msg = r.reason
+        return resp
+
+
+class HTTPHandler(AbstractHTTPHandler):
+
+    def http_open(self, req):
+        return self.do_open(httplib.HTTPConnection, req)
+
+    http_request = AbstractHTTPHandler.do_request_
+
+if hasattr(httplib, 'HTTPS'):
+    class HTTPSHandler(AbstractHTTPHandler):
+
+        def https_open(self, req):
+            return self.do_open(httplib.HTTPSConnection, req)
+
+        https_request = AbstractHTTPHandler.do_request_
+
+class HTTPCookieProcessor(BaseHandler):
+    def __init__(self, cookiejar=None):
+        import cookielib
+        if cookiejar is None:
+            cookiejar = cookielib.CookieJar()
+        self.cookiejar = cookiejar
+
+    def http_request(self, request):
+        self.cookiejar.add_cookie_header(request)
+        return request
+
+    def http_response(self, request, response):
+        self.cookiejar.extract_cookies(response, request)
+        return response
+
+    https_request = http_request
+    https_response = http_response
+
+class UnknownHandler(BaseHandler):
+    def unknown_open(self, req):
+        type = req.get_type()
+        raise URLError('unknown url type: %s' % type)
+
+def parse_keqv_list(l):
+    """Parse list of key=value strings where keys are not duplicated."""
+    parsed = {}
+    for elt in l:
+        k, v = elt.split('=', 1)
+        if v[0] == '"' and v[-1] == '"':
+            v = v[1:-1]
+        parsed[k] = v
+    return parsed
+
+def parse_http_list(s):
+    """Parse lists as described by RFC 2068 Section 2.
+
+    In particular, parse comma-separated lists where the elements of
+    the list may include quoted-strings.  A quoted-string could
+    contain a comma.  A non-quoted string could have quotes in the
+    middle.  Neither commas nor quotes count if they are escaped.
+    Only double-quotes count, not single-quotes.
+    """
+    res = []
+    part = ''
+
+    escape = quote = False
+    for cur in s:
+        if escape:
+            part += cur
+            escape = False
+            continue
+        if quote:
+            if cur == '\\':
+                escape = True
+                continue
+            elif cur == '"':
+                quote = False
+            part += cur
+            continue
+
+        if cur == ',':
+            res.append(part)
+            part = ''
+            continue
+
+        if cur == '"':
+            quote = True
+
+        part += cur
+
+    # append last part
+    if part:
+        res.append(part)
+
+    return [part.strip() for part in res]
+
+def _safe_gethostbyname(host):
+    try:
+        return socket.gethostbyname(host)
+    except socket.gaierror:
+        return None
+
+class FileHandler(BaseHandler):
+    # Use local file or FTP depending on form of URL
+    def file_open(self, req):
+        url = req.get_selector()
+        if url[:2] == '//' and url[2:3] != '/' and (req.host and
+                req.host != 'localhost'):
+            req.type = 'ftp'
+            return self.parent.open(req)
+        else:
+            return self.open_local_file(req)
+
+    # names for the localhost
+    names = None
+    def get_names(self):
+        if FileHandler.names is None:
+            try:
+                FileHandler.names = tuple(
+                    socket.gethostbyname_ex('localhost')[2] +
+                    socket.gethostbyname_ex(socket.gethostname())[2])
+            except socket.gaierror:
+                FileHandler.names = (socket.gethostbyname('localhost'),)
+        return FileHandler.names
+
+    # not entirely sure what the rules are here
+    def open_local_file(self, req):
+        import email.utils
+        import mimetypes
+        host = req.get_host()
+        filename = req.get_selector()
+        localfile = url2pathname(filename)
+        try:
+            stats = os.stat(localfile)
+            size = stats.st_size
+            modified = email.utils.formatdate(stats.st_mtime, usegmt=True)
+            mtype = mimetypes.guess_type(filename)[0]
+            headers = mimetools.Message(StringIO(
+                'Content-type: %s\nContent-length: %d\nLast-modified: %s\n' %
+                (mtype or 'text/plain', size, modified)))
+            if host:
+                host, port = splitport(host)
+            if not host or \
+                (not port and _safe_gethostbyname(host) in self.get_names()):
+                if host:
+                    origurl = 'file://' + host + filename
+                else:
+                    origurl = 'file://' + filename
+                return addinfourl(open(localfile, 'rb'), headers, origurl)
+        except OSError, msg:
+            # urllib2 users shouldn't expect OSErrors coming from urlopen()
+            raise URLError(msg)
+        raise URLError('file not on local host')
+
+class FTPHandler(BaseHandler):
+    def ftp_open(self, req):
+        import ftplib
+        import mimetypes
+        host = req.get_host()
+        if not host:
+            raise URLError('ftp error: no host given')
+        host, port = splitport(host)
+        if port is None:
+            port = ftplib.FTP_PORT
+        else:
+            port = int(port)
+
+        # username/password handling
+        user, host = splituser(host)
+        if user:
+            user, passwd = splitpasswd(user)
+        else:
+            passwd = None
+        host = unquote(host)
+        user = user or ''
+        passwd = passwd or ''
+
+        try:
+            host = socket.gethostbyname(host)
+        except socket.error, msg:
+            raise URLError(msg)
+        path, attrs = splitattr(req.get_selector())
+        dirs = path.split('/')
+        dirs = map(unquote, dirs)
+        dirs, file = dirs[:-1], dirs[-1]
+        if dirs and not dirs[0]:
+            dirs = dirs[1:]
+        try:
+            fw = self.connect_ftp(user, passwd, host, port, dirs, req.timeout)
+            type = file and 'I' or 'D'
+            for attr in attrs:
+                attr, value = splitvalue(attr)
+                if attr.lower() == 'type' and \
+                   value in ('a', 'A', 'i', 'I', 'd', 'D'):
+                    type = value.upper()
+            fp, retrlen = fw.retrfile(file, type)
+            headers = ""
+            mtype = mimetypes.guess_type(req.get_full_url())[0]
+            if mtype:
+                headers += "Content-type: %s\n" % mtype
+            if retrlen is not None and retrlen >= 0:
+                headers += "Content-length: %d\n" % retrlen
+            sf = StringIO(headers)
+            headers = mimetools.Message(sf)
+            return addinfourl(fp, headers, req.get_full_url())
+        except ftplib.all_errors, msg:
+            raise URLError, ('ftp error: %s' % msg), sys.exc_info()[2]
+
+    def connect_ftp(self, user, passwd, host, port, dirs, timeout):
+        fw = ftpwrapper(user, passwd, host, port, dirs, timeout)
+##        fw.ftp.set_debuglevel(1)
+        return fw
+
+class CacheFTPHandler(FTPHandler):
+    # XXX would be nice to have pluggable cache strategies
+    # XXX this stuff is definitely not thread safe
+    def __init__(self):
+        self.cache = {}
+        self.timeout = {}
+        self.soonest = 0
+        self.delay = 60
+        self.max_conns = 16
+
+    def setTimeout(self, t):
+        self.delay = t
+
+    def setMaxConns(self, m):
+        self.max_conns = m
+
+    def connect_ftp(self, user, passwd, host, port, dirs, timeout):
+        key = user, host, port, '/'.join(dirs), timeout
+        if key in self.cache:
+            self.timeout[key] = time.time() + self.delay
+        else:
+            self.cache[key] = ftpwrapper(user, passwd, host, port, dirs, timeout)
+            self.timeout[key] = time.time() + self.delay
+        self.check_cache()
+        return self.cache[key]
+
+    def check_cache(self):
+        # first check for old ones
+        t = time.time()
+        if self.soonest <= t:
+            for k, v in self.timeout.items():
+                if v < t:
+                    self.cache[k].close()
+                    del self.cache[k]
+                    del self.timeout[k]
+        self.soonest = min(self.timeout.values())
+
+        # then check the size
+        if len(self.cache) == self.max_conns:
+            for k, v in self.timeout.items():
+                if v == self.soonest:
+                    del self.cache[k]
+                    del self.timeout[k]
+                    break
+            self.soonest = min(self.timeout.values())
diff --git a/lib_pypy/pyrepl/readline.py b/lib_pypy/pyrepl/readline.py
--- a/lib_pypy/pyrepl/readline.py
+++ b/lib_pypy/pyrepl/readline.py
@@ -395,9 +395,21 @@
     _wrapper.f_in = f_in
     _wrapper.f_out = f_out
 
-    if hasattr(sys, '__raw_input__'):    # PyPy
-        _old_raw_input = sys.__raw_input__
+    if '__pypy__' in sys.builtin_module_names:    # PyPy
+
+        def _old_raw_input(prompt=''):
+            # sys.__raw_input__() is only called when stdin and stdout are
+            # as expected and are ttys.  If it is the case, then get_reader()
+            # should not really fail in _wrapper.raw_input().  If it still
+            # does, then we will just cancel the redirection and call again
+            # the built-in raw_input().
+            try:
+                del sys.__raw_input__
+            except AttributeError:
+                pass
+            return raw_input(prompt)
         sys.__raw_input__ = _wrapper.raw_input
+
     else:
         # this is not really what readline.c does.  Better than nothing I guess
         import __builtin__
diff --git a/lib_pypy/resource.py b/lib_pypy/resource.py
--- a/lib_pypy/resource.py
+++ b/lib_pypy/resource.py
@@ -7,7 +7,7 @@
 
 from ctypes_support import standard_c_lib as libc
 from ctypes_support import get_errno
-from ctypes import Structure, c_int, c_long, byref, sizeof, POINTER
+from ctypes import Structure, c_int, c_long, byref, POINTER
 from errno import EINVAL, EPERM
 import _structseq
 
@@ -165,7 +165,6 @@
 
 @builtinify
 def getpagesize():
-    pagesize = 0
     if _getpagesize:
         return _getpagesize()
     else:
diff --git a/pypy/annotation/classdef.py b/pypy/annotation/classdef.py
--- a/pypy/annotation/classdef.py
+++ b/pypy/annotation/classdef.py
@@ -276,8 +276,8 @@
         # create the Attribute and do the generalization asked for
         newattr = Attribute(attr, self.bookkeeper)
         if s_value:
-            if newattr.name == 'intval' and getattr(s_value, 'unsigned', False):
-                import pdb; pdb.set_trace()
+            #if newattr.name == 'intval' and getattr(s_value, 'unsigned', False):
+            #    import pdb; pdb.set_trace()
             newattr.s_value = s_value
 
         # keep all subattributes' values
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -72,6 +72,7 @@
     del working_modules['fcntl']  # LOCK_NB not defined
     del working_modules["_minimal_curses"]
     del working_modules["termios"]
+    del working_modules["_multiprocessing"]   # depends on rctime
 
 
 
@@ -127,7 +128,7 @@
 
 pypy_optiondescription = OptionDescription("objspace", "Object Space Options", [
     ChoiceOption("name", "Object Space name",
-                 ["std", "flow", "thunk", "dump", "taint"],
+                 ["std", "flow", "thunk", "dump"],
                  "std",
                  cmdline='--objspace -o'),
 
diff --git a/pypy/doc/__pypy__-module.rst b/pypy/doc/__pypy__-module.rst
--- a/pypy/doc/__pypy__-module.rst
+++ b/pypy/doc/__pypy__-module.rst
@@ -37,29 +37,6 @@
 .. _`thunk object space docs`: objspace-proxies.html#thunk
 .. _`interface section of the thunk object space docs`: objspace-proxies.html#thunk-interface
 
-.. broken:
-
-    Taint Object Space Functionality
-    ================================
-
-    When the taint object space is used (choose with :config:`objspace.name`),
-    the following names are put into ``__pypy__``:
-
-     - ``taint``
-     - ``is_tainted``
-     - ``untaint``
-     - ``taint_atomic``
-     - ``_taint_debug``
-     - ``_taint_look``
-     - ``TaintError``
-
-    Those are all described in the `interface section of the taint object space
-    docs`_.
-
-    For more detailed explanations and examples see the `taint object space docs`_.
-
-    .. _`taint object space docs`: objspace-proxies.html#taint
-    .. _`interface section of the taint object space docs`: objspace-proxies.html#taint-interface
 
 Transparent Proxy Functionality
 ===============================
diff --git a/pypy/doc/config/objspace.name.txt b/pypy/doc/config/objspace.name.txt
--- a/pypy/doc/config/objspace.name.txt
+++ b/pypy/doc/config/objspace.name.txt
@@ -4,7 +4,6 @@
 for normal usage):
 
   * thunk_: The thunk object space adds lazy evaluation to PyPy.
-  * taint_: The taint object space adds soft security features.
   * dump_:  Using this object spaces results in the dumpimp of all operations
     to a log.
 
@@ -12,5 +11,4 @@
 .. _`Object Space Proxies`: ../objspace-proxies.html
 .. _`Standard Object Space`: ../objspace.html#standard-object-space
 .. _thunk: ../objspace-proxies.html#thunk
-.. _taint: ../objspace-proxies.html#taint
 .. _dump: ../objspace-proxies.html#dump
diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst
--- a/pypy/doc/index.rst
+++ b/pypy/doc/index.rst
@@ -309,7 +309,6 @@
 .. _`object space`: objspace.html
 .. _FlowObjSpace: objspace.html#the-flow-object-space 
 .. _`trace object space`: objspace.html#the-trace-object-space 
-.. _`taint object space`: objspace-proxies.html#taint
 .. _`thunk object space`: objspace-proxies.html#thunk
 .. _`transparent proxies`: objspace-proxies.html#tproxy
 .. _`Differences between PyPy and CPython`: cpython_differences.html
diff --git a/pypy/doc/objspace-proxies.rst b/pypy/doc/objspace-proxies.rst
--- a/pypy/doc/objspace-proxies.rst
+++ b/pypy/doc/objspace-proxies.rst
@@ -129,297 +129,6 @@
    function behaves lazily: all calls to it return a thunk object.
 
 
-.. broken right now:
-
-    .. _taint:
-
-    The Taint Object Space
-    ======================
-
-    Motivation
-    ----------
-
-    The Taint Object Space provides a form of security: "tainted objects",
-    inspired by various sources, see [D12.1]_ for a more detailed discussion. 
-
-    The basic idea of this kind of security is not to protect against
-    malicious code but to help with handling and boxing sensitive data. 
-    It covers two kinds of sensitive data: secret data which should not leak, 
-    and untrusted data coming from an external source and that must be 
-    validated before it is used.
-
-    The idea is that, considering a large application that handles these
-    kinds of sensitive data, there are typically only a small number of
-    places that need to explicitly manipulate that sensitive data; all the
-    other places merely pass it around, or do entirely unrelated things.
-
-    Nevertheless, if a large application needs to be reviewed for security,
-    it must be entirely carefully checked, because it is possible that a
-    bug at some apparently unrelated place could lead to a leak of sensitive
-    information in a way that an external attacker could exploit.  For
-    example, if any part of the application provides web services, an
-    attacker might be able to issue unexpected requests with a regular web
-    browser and deduce secret information from the details of the answers he
-    gets.  Another example is the common CGI attack where an attacker sends
-    malformed inputs and causes the CGI script to do unintended things.
-
-    An approach like that of the Taint Object Space allows the small parts
-    of the program that manipulate sensitive data to be explicitly marked.
-    The effect of this is that although these small parts still need a
-    careful security review, the rest of the application no longer does,
-    because even a bug would be unable to leak the information.
-
-    We have implemented a simple two-level model: objects are either
-    regular (untainted), or sensitive (tainted).  Objects are marked as
-    sensitive if they are secret or untrusted, and only declassified at
-    carefully-checked positions (e.g. where the secret data is needed, or
-    after the untrusted data has been fully validated).
-
-    It would be simple to extend the code for more fine-grained scales of
-    secrecy.  For example it is typical in the literature to consider
-    user-specified lattices of secrecy levels, corresponding to multiple
-    "owners" that cannot access data belonging to another "owner" unless
-    explicitly authorized to do so.
-
-    Tainting and untainting
-    -----------------------
-
-    Start a py.py with the Taint Object Space and try the following example::
-
-        $ py.py -o taint
-        >>>> from __pypy__ import taint
-        >>>> x = taint(6)
-
-        # x is hidden from now on.  We can pass it around and
-        # even operate on it, but not inspect it.  Taintness
-        # is propagated to operation results.
-
-        >>>> x
-        TaintError
-
-        >>>> if x > 5: y = 2   # see below
-        TaintError
-
-        >>>> y = x + 5         # ok
-        >>>> lst = [x, y]
-        >>>> z = lst.pop()
-        >>>> t = type(z)       # type() works too, tainted answer
-        >>>> t
-        TaintError
-        >>>> u = t is int      # even 'is' works
-        >>>> u
-        TaintError
-
-    Notice that using a tainted boolean like ``x > 5`` in an ``if``
-    statement is forbidden.  This is because knowing which path is followed
-    would give away a hint about ``x``; in the example above, if the
-    statement ``if x > 5: y = 2`` was allowed to run, we would know
-    something about the value of ``x`` by looking at the (untainted) value
-    in the variable ``y``.
-
-    Of course, there is a way to inspect tainted objects.  The basic way is
-    to explicitly "declassify" it with the ``untaint()`` function.  In an
-    application, the places that use ``untaint()`` are the places that need
-    careful security review.  To avoid unexpected objects showing up, the
-    ``untaint()`` function must be called with the exact type of the object
-    to declassify.  It will raise ``TaintError`` if the type doesn't match::
-
-        >>>> from __pypy__ import taint
-        >>>> untaint(int, x)
-        6
-        >>>> untaint(int, z)
-        11
-        >>>> untaint(bool, x > 5)
-        True
-        >>>> untaint(int, x > 5)
-        TaintError
-
-
-    Taint Bombs
-    -----------
-
-    In this area, a common problem is what to do about failing operations.
-    If an operation raises an exception when manipulating a tainted object,
-    then the very presence of the exception can leak information about the
-    tainted object itself.  Consider::
-
-        >>>> 5 / (x-6)
-
-    By checking if this raises ``ZeroDivisionError`` or not, we would know
-    if ``x`` was equal to 6 or not.  The solution to this problem in the
-    Taint Object Space is to introduce *Taint Bombs*.  They are a kind of
-    tainted object that doesn't contain a real object, but a pending
-    exception.  Taint Bombs are indistinguishable from normal tainted
-    objects to unprivileged code. See::
-
-        >>>> x = taint(6)
-        >>>> i = 5 / (x-6)     # no exception here
-        >>>> j = i + 1         # nor here
-        >>>> k = j + 5         # nor here
-        >>>> untaint(int, k)
-        TaintError
-
-    In the above example, all of ``i``, ``j`` and ``k`` contain a Taint
-    Bomb.  Trying to untaint it raises an exception - a generic
-    ``TaintError``.  What we win is that the exception gives little away,
-    and most importantly it occurs at the point where ``untaint()`` is
-    called, not where the operation failed.  This means that all calls to
-    ``untaint()`` - but not the rest of the code - must be carefully
-    reviewed for what occurs if they receive a Taint Bomb; they might catch
-    the ``TaintError`` and give the user a generic message that something
-    went wrong, if we are reasonably careful that the message or even its
-    presence doesn't give information away.  This might be a
-    problem by itself, but there is no satisfying general solution here:
-    it must be considered on a case-by-case basis.  Again, what the
-    Taint Object Space approach achieves is not solving these problems, but
-    localizing them to well-defined small parts of the application - namely,
-    around calls to ``untaint()``.
-
-    The ``TaintError`` exception deliberately does not include any
-    useful error messages, because they might give information away.
-    Of course, this makes debugging quite a bit harder; a difficult
-    problem to solve properly.  So far we have implemented a way to peek in a Taint
-    Box or Bomb, ``__pypy__._taint_look(x)``, and a "debug mode" that
-    prints the exception as soon as a Bomb is created - both write
-    information to the low-level stderr of the application, where we hope
-    that it is unlikely to be seen by anyone but the application
-    developer.
-
-
-    Taint Atomic functions
-    ----------------------
-
-    Occasionally, a more complicated computation must be performed on a
-    tainted object.  This requires first untainting the object, performing the
-    computations, and then carefully tainting the result again (including
-    hiding all exceptions into Bombs).
-
-    There is a built-in decorator that does this for you::
-
-        >>>> @__pypy__.taint_atomic
-        >>>> def myop(x, y):
-        ....     while x > 0:
-        ....         x -= y
-        ....     return x
-        ....
-        >>>> myop(42, 10)
-        -8
-        >>>> z = myop(taint(42), 10)
-        >>>> z
-        TaintError
-        >>>> untaint(int, z)
-        -8
-
-    The decorator makes a whole function behave like a built-in operation.
-    If no tainted argument is passed in, the function behaves normally.  But
-    if any of the arguments is tainted, it is automatically untainted - so
-    the function body always sees untainted arguments - and the eventual
-    result is tainted again (possibly in a Taint Bomb).
-
-    It is important for the function marked as ``taint_atomic`` to have no
-    visible side effects, as these could cause information leakage.
-    This is currently not enforced, which means that all ``taint_atomic``
-    functions have to be carefully reviewed for security (but not the
-    callers of ``taint_atomic`` functions).
-
-    A possible future extension would be to forbid side-effects on
-    non-tainted objects from all ``taint_atomic`` functions.
-
-    An example of usage: given a tainted object ``passwords_db`` that
-    references a database of passwords, we can write a function
-    that checks if a password is valid as follows::
-
-        @taint_atomic
-        def validate(passwords_db, username, password):
-            assert type(passwords_db) is PasswordDatabase
-            assert type(username) is str
-            assert type(password) is str
-            ...load username entry from passwords_db...
-            return expected_password == password
-
-    It returns a tainted boolean answer, or a Taint Bomb if something
-    went wrong.  A caller can do::
-
-        ok = validate(passwords_db, 'john', '1234')
-        ok = untaint(bool, ok)
-
-    This can give three outcomes: ``True``, ``False``, or a ``TaintError``
-    exception (with no information on it) if anything went wrong.  If even
-    this is considered giving too much information away, the ``False`` case
-    can be made indistinguishable from the ``TaintError`` case (simply by
-    raising an exception in ``validate()`` if the password is wrong).
-
-    In the above example, the security results achieved are the following:
-    as long as ``validate()`` does not leak information, no other part of
-    the code can obtain more information about a passwords database than a
-    Yes/No answer to a precise query.
-
-    A possible extension of the ``taint_atomic`` decorator would be to check
-    the argument types, as ``untaint()`` does, for the same reason: to
-    prevent bugs where a function like ``validate()`` above is accidentally
-    called with the wrong kind of tainted object, which would make it
-    misbehave.  For now, all ``taint_atomic`` functions should be
-    conservative and carefully check all assumptions on their input
-    arguments.
-
-
-    .. _`taint-interface`:
-
-    Interface
-    ---------
-
-    .. _`like a built-in operation`:
-
-    The basic rule of the Tainted Object Space is that it introduces two new
-    kinds of objects, Tainted Boxes and Tainted Bombs (which are not types
-    in the Python sense).  Each box internally contains a regular object;
-    each bomb internally contains an exception object.  An operation
-    involving Tainted Boxes is performed on the objects contained in the
-    boxes, and gives a Tainted Box or a Tainted Bomb as a result (such an
-    operation does not let an exception be raised).  An operation called
-    with a Tainted Bomb argument immediately returns the same Tainted Bomb.
-
-    In a PyPy running with (or translated with) the Taint Object Space,
-    the ``__pypy__`` module exposes the following interface:
-
-    * ``taint(obj)``
-
-        Return a new Tainted Box wrapping ``obj``.  Return ``obj`` itself
-        if it is already tainted (a Box or a Bomb).
-
-    * ``is_tainted(obj)``
-
-        Check if ``obj`` is tainted (a Box or a Bomb).
-
-    * ``untaint(type, obj)``
-
-        Untaints ``obj`` if it is tainted.  Raise ``TaintError`` if the type
-        of the untainted object is not exactly ``type``, or if ``obj`` is a
-        Bomb.
-
-    * ``taint_atomic(func)``
-
-        Return a wrapper function around the callable ``func``.  The wrapper
-        behaves `like a built-in operation`_ with respect to untainting the
-        arguments, tainting the result, and returning a Bomb.
-
-    * ``TaintError``
-
-        Exception.  On purpose, it provides no attribute or error message.
-
-    * ``_taint_debug(level)``
-
-        Set the debugging level to ``level`` (0=off).  At level 1 or above,
-        all Taint Bombs print a diagnostic message to stderr when they are
-        created.
-
-    * ``_taint_look(obj)``
-
-        For debugging purposes: prints (to stderr) the type and address of
-        the object in a Tainted Box, or prints the exception if ``obj`` is
-        a Taint Bomb.
-
-
 .. _dump:
 
 The Dump Object Space
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -391,8 +391,11 @@
     def decrement_ticker(self, by):
         value = self._ticker
         if self.has_bytecode_counter:    # this 'if' is constant-folded
-            value -= by
-            self._ticker = value
+            if jit.isconstant(by) and by == 0:
+                pass     # normally constant-folded too
+            else:
+                value -= by
+                self._ticker = value
         return value
 
 
diff --git a/pypy/interpreter/pyparser/pytokenizer.py b/pypy/interpreter/pyparser/pytokenizer.py
--- a/pypy/interpreter/pyparser/pytokenizer.py
+++ b/pypy/interpreter/pyparser/pytokenizer.py
@@ -226,7 +226,7 @@
                         parenlev = parenlev - 1
                         if parenlev < 0:
                             raise TokenError("unmatched '%s'" % initial, line,
-                                             lnum-1, 0, token_list)
+                                             lnum, start + 1, token_list)
                     if token in python_opmap:
                         punct = python_opmap[token]
                     else:
diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py
--- a/pypy/interpreter/pyparser/test/test_pyparse.py
+++ b/pypy/interpreter/pyparser/test/test_pyparse.py
@@ -87,6 +87,10 @@
         assert exc.lineno == 1
         assert exc.offset == 5
         assert exc.lastlineno == 5
+        exc = py.test.raises(SyntaxError, parse, "abc)").value
+        assert exc.msg == "unmatched ')'"
+        assert exc.lineno == 1
+        assert exc.offset == 4
 
     def test_is(self):
         self.parse("x is y")
diff --git a/pypy/jit/backend/llgraph/llimpl.py b/pypy/jit/backend/llgraph/llimpl.py
--- a/pypy/jit/backend/llgraph/llimpl.py
+++ b/pypy/jit/backend/llgraph/llimpl.py
@@ -165,6 +165,7 @@
     'unicodegetitem'  : (('ref', 'int'), 'int'),
     'unicodesetitem'  : (('ref', 'int', 'int'), 'int'),
     'cast_ptr_to_int' : (('ref',), 'int'),
+    'cast_int_to_ptr' : (('int',), 'ref'),
     'debug_merge_point': (('ref', 'int'), None),
     'force_token'     : ((), 'int'),
     'call_may_force'  : (('int', 'varargs'), 'intorptr'),
@@ -875,9 +876,6 @@
     def op_new_array(self, arraydescr, count):
         return do_new_array(arraydescr.ofs, count)
 
-    def op_cast_ptr_to_int(self, descr, ptr):
-        return cast_to_int(ptr)
-
     def op_force_token(self, descr):
         opaque_frame = _to_opaque(self)
         return llmemory.cast_ptr_to_adr(opaque_frame)
diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py
--- a/pypy/jit/backend/llsupport/gc.py
+++ b/pypy/jit/backend/llsupport/gc.py
@@ -45,6 +45,14 @@
     def freeing_block(self, start, stop):
         pass
 
+    def record_constptrs(self, op, gcrefs_output_list):
+        for i in range(op.numargs()):
+            v = op.getarg(i)
+            if isinstance(v, ConstPtr) and bool(v.value):
+                p = v.value
+                rgc._make_sure_does_not_move(p)
+                gcrefs_output_list.append(p)
+
 # ____________________________________________________________
 
 class GcLLDescr_boehm(GcLLDescription):
@@ -141,6 +149,14 @@
     get_funcptr_for_newstr = None
     get_funcptr_for_newunicode = None
 
+    def rewrite_assembler(self, cpu, operations, gcrefs_output_list):
+        # record all GCREFs too, because Boehm cannot see them and keep them
+        # alive if they end up as constants in the assembler
+        for op in operations:
+            self.record_constptrs(op, gcrefs_output_list)
+        return GcLLDescription.rewrite_assembler(self, cpu, operations,
+                                                 gcrefs_output_list)
+
 
 # ____________________________________________________________
 # All code below is for the hybrid or minimark GC
@@ -757,14 +773,6 @@
             funcptr(llmemory.cast_ptr_to_adr(gcref_struct),
                     llmemory.cast_ptr_to_adr(gcref_newptr))
 
-    def record_constptrs(self, op, gcrefs_output_list):
-        for i in range(op.numargs()):
-            v = op.getarg(i)
-            if isinstance(v, ConstPtr) and bool(v.value):
-                p = v.value
-                rgc._make_sure_does_not_move(p)
-                gcrefs_output_list.append(p)
-
     def rewrite_assembler(self, cpu, operations, gcrefs_output_list):
         # Perform two kinds of rewrites in parallel:
         #
diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -1468,20 +1468,16 @@
         return u''.join(u.chars)
 
 
-    def test_casts(self):
-        py.test.skip("xxx fix or kill")
-        from pypy.rpython.lltypesystem import lltype, llmemory
-        TP = lltype.GcStruct('x')
-        x = lltype.malloc(TP)        
-        x = lltype.cast_opaque_ptr(llmemory.GCREF, x)
+    def test_cast_int_to_ptr(self):
+        res = self.execute_operation(rop.CAST_INT_TO_PTR,
+                                     [BoxInt(-17)],  'ref').value
+        assert lltype.cast_ptr_to_int(res) == -17
+
+    def test_cast_ptr_to_int(self):
+        x = lltype.cast_int_to_ptr(llmemory.GCREF, -19)
         res = self.execute_operation(rop.CAST_PTR_TO_INT,
-                                     [BoxPtr(x)],  'int').value
-        expected = self.cpu.cast_adr_to_int(llmemory.cast_ptr_to_adr(x))
-        assert rffi.get_real_int(res) == rffi.get_real_int(expected)
-        res = self.execute_operation(rop.CAST_PTR_TO_INT,
-                                     [ConstPtr(x)],  'int').value
-        expected = self.cpu.cast_adr_to_int(llmemory.cast_ptr_to_adr(x))
-        assert rffi.get_real_int(res) == rffi.get_real_int(expected)
+                                     [BoxPtr(x)], 'int').value
+        assert res == -19
 
     def test_ooops_non_gc(self):
         x = lltype.malloc(lltype.Struct('x'), flavor='raw')
@@ -2299,13 +2295,6 @@
         #
         cpu.bh_strsetitem(x, 4, ord('/'))
         assert str.chars[4] == '/'
-        #
-##        x = cpu.bh_newstr(5)
-##        y = cpu.bh_cast_ptr_to_int(x)
-##        z = cpu.bh_cast_ptr_to_int(x)
-##        y = rffi.get_real_int(y)
-##        z = rffi.get_real_int(z)
-##        assert type(y) == type(z) == int and y == z
 
     def test_sorting_of_fields(self):
         S = self.S
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1387,7 +1387,8 @@
 
     def genop_same_as(self, op, arglocs, resloc):
         self.mov(arglocs[0], resloc)
-    #genop_cast_ptr_to_int = genop_same_as
+    genop_cast_ptr_to_int = genop_same_as
+    genop_cast_int_to_ptr = genop_same_as
 
     def genop_int_mod(self, op, arglocs, resloc):
         if IS_X86_32:
diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py
--- a/pypy/jit/backend/x86/regalloc.py
+++ b/pypy/jit/backend/x86/regalloc.py
@@ -1152,7 +1152,8 @@
         self.possibly_free_var(op.getarg(0))
         resloc = self.force_allocate_reg(op.result)
         self.Perform(op, [argloc], resloc)
-    #consider_cast_ptr_to_int = consider_same_as
+    consider_cast_ptr_to_int = consider_same_as
+    consider_cast_int_to_ptr = consider_same_as
 
     def consider_strlen(self, op):
         args = op.getarglist()
diff --git a/pypy/jit/backend/x86/tool/viewcode.py b/pypy/jit/backend/x86/tool/viewcode.py
--- a/pypy/jit/backend/x86/tool/viewcode.py
+++ b/pypy/jit/backend/x86/tool/viewcode.py
@@ -9,7 +9,12 @@
 """
 
 import autopath
-import operator, sys, os, re, py, new
+import new
+import operator
+import py
+import re
+import sys
+import subprocess
 from bisect import bisect_left
 
 # don't use pypy.tool.udir here to avoid removing old usessions which
@@ -44,14 +49,16 @@
     f = open(tmpfile, 'wb')
     f.write(data)
     f.close()
-    g = os.popen(objdump % {
+    p = subprocess.Popen(objdump % {
         'file': tmpfile,
         'origin': originaddr,
         'backend': objdump_backend_option[backend_name],
-    }, 'r')
-    result = g.readlines()
-    g.close()
-    lines = result[6:]   # drop some objdump cruft
+    }, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    stdout, stderr = p.communicate()
+    assert not p.returncode, ('Encountered an error running objdump: %s' %
+                              stderr)
+    # drop some objdump cruft
+    lines = stdout.splitlines()[6:]
     return format_code_dump_with_labels(originaddr, lines, label_list)
 
 def format_code_dump_with_labels(originaddr, lines, label_list):
@@ -85,8 +92,12 @@
     #
     print 'loading symbols from %s...' % (filename,)
     symbols = {}
-    g = os.popen(symbollister % filename, "r")
-    for line in g:
+    p = subprocess.Popen(symbollister % filename, shell=True,
+                         stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    stdout, stderr = p.communicate()
+    assert not p.returncode, ('Encountered an error running nm: %s' %
+                              stderr)
+    for line in stdout.splitlines():
         match = re_symbolentry.match(line)
         if match:
             addr = long(match.group(1), 16)
@@ -94,7 +105,6 @@
             if name.startswith('pypy_g_'):
                 name = '\xb7' + name[7:]
             symbols[addr] = name
-    g.close()
     print '%d symbols found' % (len(symbols),)
     return symbols
 
diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -455,6 +455,23 @@
             # the special return value None forces op.result to be considered
             # equal to op.args[0]
             return [op0, op1, None]
+        if (hints.get('promote_string') and
+            op.args[0].concretetype is not lltype.Void):
+            S = lltype.Ptr(rstr.STR)
+            assert op.args[0].concretetype == S
+            self._register_extra_helper(EffectInfo.OS_STREQ_NONNULL,
+                                        "str.eq_nonnull",
+                                        [S, S],
+                                        lltype.Signed,
+                                        EffectInfo.EF_ELIDABLE_CANNOT_RAISE)
+            descr, p = self.callcontrol.callinfocollection.callinfo_for_oopspec(
+                EffectInfo.OS_STREQ_NONNULL)
+            # XXX this is fairly ugly way of creating a constant,
+            #     however, callinfocollection has no better interface
+            c = Constant(p.adr.ptr, lltype.typeOf(p.adr.ptr))
+            op1 = SpaceOperation('str_guard_value', [op.args[0], c, descr],
+                                 op.result)
+            return [SpaceOperation('-live-', [], None), op1, None]
         else:
             log.WARNING('ignoring hint %r at %r' % (hints, self.graph))
 
@@ -783,8 +800,7 @@
 
     def rewrite_op_cast_ptr_to_int(self, op):
         if self._is_gc(op.args[0]):
-            #return op
-            raise NotImplementedError("cast_ptr_to_int")
+            return op
 
     def rewrite_op_force_cast(self, op):
         v_arg = op.args[0]
@@ -1526,6 +1542,10 @@
     def rewrite_op_jit_force_virtual(self, op):
         return self._do_builtin_call(op)
 
+    def rewrite_op_jit_is_virtual(self, op):
+        raise Exception, (
+            "'vref.virtual' should not be used from jit-visible code")
+
     def rewrite_op_jit_force_virtualizable(self, op):
         # this one is for virtualizables
         vinfo = self.get_vinfo(op.args[0])
diff --git a/pypy/jit/codewriter/test/test_jtransform.py b/pypy/jit/codewriter/test/test_jtransform.py
--- a/pypy/jit/codewriter/test/test_jtransform.py
+++ b/pypy/jit/codewriter/test/test_jtransform.py
@@ -1,5 +1,19 @@
 import py
 import random
+try:
+    from itertools import product
+except ImportError:
+    # Python 2.5, this is taken from the CPython docs, but simplified.
+    def product(*args):
+        # product('ABCD', 'xy') --> Ax Ay Bx By Cx Cy Dx Dy
+        # product(range(2), repeat=3) --> 000 001 010 011 100 101 110 111
+        pools = map(tuple, args)
+        result = [[]]
+        for pool in pools:
+            result = [x+[y] for x in result for y in pool]
+        for prod in result:
+            yield tuple(prod)
+
 from pypy.objspace.flow.model import FunctionGraph, Block, Link
 from pypy.objspace.flow.model import SpaceOperation, Variable, Constant
 from pypy.jit.codewriter.jtransform import Transformer
@@ -85,6 +99,12 @@
             if i == oopspecindex:
                 return True
         return False
+    def callinfo_for_oopspec(self, oopspecindex):
+        assert oopspecindex == effectinfo.EffectInfo.OS_STREQ_NONNULL
+        class c:
+            class adr:
+                ptr = 1
+        return ('calldescr', c)
 
 class FakeBuiltinCallControl:
     def __init__(self):
@@ -105,6 +125,7 @@
              EI.OS_STR2UNICODE:([PSTR], PUNICODE),
              EI.OS_STR_CONCAT: ([PSTR, PSTR], PSTR),
              EI.OS_STR_SLICE:  ([PSTR, INT, INT], PSTR),
+             EI.OS_STREQ_NONNULL:  ([PSTR, PSTR], INT),
              EI.OS_UNI_CONCAT: ([PUNICODE, PUNICODE], PUNICODE),
              EI.OS_UNI_SLICE:  ([PUNICODE, INT, INT], PUNICODE),
              EI.OS_UNI_EQUAL:  ([PUNICODE, PUNICODE], lltype.Bool),
@@ -254,26 +275,35 @@
             assert op1.result is None
 
 def test_calls():
-    for RESTYPE in [lltype.Signed, rclass.OBJECTPTR,
-                    lltype.Float, lltype.Void]:
-      for with_void in [False, True]:
-        for with_i in [False, True]:
-          for with_r in [False, True]:
-            for with_f in [False, True]:
-              ARGS = []
-              if with_void: ARGS += [lltype.Void, lltype.Void]
-              if with_i: ARGS += [lltype.Signed, lltype.Char]
-              if with_r: ARGS += [rclass.OBJECTPTR, lltype.Ptr(rstr.STR)]
-              if with_f: ARGS += [lltype.Float, lltype.Float]
-              random.shuffle(ARGS)
-              if RESTYPE == lltype.Float: with_f = True
-              if with_f: expectedkind = 'irf'   # all kinds
-              elif with_i: expectedkind = 'ir'  # integers and references
-              else: expectedkind = 'r'          # only references
-              yield residual_call_test, ARGS, RESTYPE, expectedkind
-              yield direct_call_test, ARGS, RESTYPE, expectedkind
-              yield indirect_residual_call_test, ARGS, RESTYPE, expectedkind
-              yield indirect_regular_call_test, ARGS, RESTYPE, expectedkind
+    for RESTYPE, with_void, with_i, with_r, with_f in product(
+        [lltype.Signed, rclass.OBJECTPTR, lltype.Float, lltype.Void],
+        [False, True],
+        [False, True],
+        [False, True],
+        [False, True],
+    ):
+        ARGS = []
+        if with_void:
+            ARGS += [lltype.Void, lltype.Void]
+        if with_i:
+            ARGS += [lltype.Signed, lltype.Char]
+        if with_r:
+            ARGS += [rclass.OBJECTPTR, lltype.Ptr(rstr.STR)]
+        if with_f:
+            ARGS += [lltype.Float, lltype.Float]
+        random.shuffle(ARGS)
+        if RESTYPE == lltype.Float:
+            with_f = True
+        if with_f:
+            expectedkind = 'irf'   # all kinds
+        elif with_i:
+            expectedkind = 'ir'  # integers and references
+        else:
+            expectedkind = 'r'          # only references
+        yield residual_call_test, ARGS, RESTYPE, expectedkind
+        yield direct_call_test, ARGS, RESTYPE, expectedkind
+        yield indirect_residual_call_test, ARGS, RESTYPE, expectedkind
+        yield indirect_regular_call_test, ARGS, RESTYPE, expectedkind
 
 def get_direct_call_op(argtypes, restype):
     FUNC = lltype.FuncType(argtypes, restype)
@@ -821,6 +851,21 @@
     assert op1.args[2] == ListOfKind('ref', [v1, v2])
     assert op1.result == v3
 
+def test_str_promote():
+    PSTR = lltype.Ptr(rstr.STR)
+    v1 = varoftype(PSTR)
+    v2 = varoftype(PSTR)
+    op = SpaceOperation('hint',
+                        [v1, Constant({'promote_string': True}, lltype.Void)],
+                        v2)
+    tr = Transformer(FakeCPU(), FakeBuiltinCallControl())
+    op0, op1, _ = tr.rewrite_operation(op)
+    assert op1.opname == 'str_guard_value'
+    assert op1.args[0] == v1
+    assert op1.args[2] == 'calldescr'
+    assert op1.result == v2
+    assert op0.opname == '-live-'
+
 def test_unicode_concat():
     # test that the oopspec is present and correctly transformed
     PSTR = lltype.Ptr(rstr.UNICODE)
diff --git a/pypy/jit/metainterp/blackhole.py b/pypy/jit/metainterp/blackhole.py
--- a/pypy/jit/metainterp/blackhole.py
+++ b/pypy/jit/metainterp/blackhole.py
@@ -2,7 +2,7 @@
 from pypy.rlib.rtimer import read_timestamp
 from pypy.rlib.rarithmetic import intmask, LONG_BIT, r_uint, ovfcheck
 from pypy.rlib.objectmodel import we_are_translated
-from pypy.rlib.debug import debug_start, debug_stop
+from pypy.rlib.debug import debug_start, debug_stop, ll_assert
 from pypy.rlib.debug import make_sure_not_resized
 from pypy.rpython.lltypesystem import lltype, llmemory, rclass
 from pypy.rpython.lltypesystem.lloperation import llop
@@ -503,6 +503,15 @@
     @arguments("r", returns="r")
     def bhimpl_cast_opaque_ptr(a):
         return a
+    @arguments("r", returns="i")
+    def bhimpl_cast_ptr_to_int(a):
+        i = lltype.cast_ptr_to_int(a)
+        ll_assert((i & 1) == 1, "bhimpl_cast_ptr_to_int: not an odd int")
+        return i
+    @arguments("i", returns="r")
+    def bhimpl_cast_int_to_ptr(i):
+        ll_assert((i & 1) == 1, "bhimpl_cast_int_to_ptr: not an odd int")
+        return lltype.cast_int_to_ptr(llmemory.GCREF, i)
 
     @arguments("i", returns="i")
     def bhimpl_int_copy(a):
@@ -523,6 +532,9 @@
     @arguments("f")
     def bhimpl_float_guard_value(a):
         pass
+    @arguments("r", "i", "d", returns="r")
+    def bhimpl_str_guard_value(a, i, d):
+        return a
 
     @arguments("self", "i")
     def bhimpl_int_push(self, a):
diff --git a/pypy/jit/metainterp/graphpage.py b/pypy/jit/metainterp/graphpage.py
--- a/pypy/jit/metainterp/graphpage.py
+++ b/pypy/jit/metainterp/graphpage.py
@@ -12,8 +12,8 @@
     def get_display_text(self):
         return None
 
-def display_loops(loops, errmsg=None, highlight_loops=()):
-    graphs = [(loop, loop in highlight_loops) for loop in loops]    
+def display_loops(loops, errmsg=None, highlight_loops={}):
+    graphs = [(loop, highlight_loops.get(loop, 0)) for loop in loops]    
     for graph, highlight in graphs:
         for op in graph.get_operations():
             if is_interesting_guard(op):
@@ -65,8 +65,7 @@
     def add_graph(self, graph, highlight=False):
         graphindex = len(self.graphs)
         self.graphs.append(graph)
-        if highlight:
-            self.highlight_graphs[graph] = True
+        self.highlight_graphs[graph] = highlight
         for i, op in enumerate(graph.get_operations()):
             self.all_operations[op] = graphindex, i
 
@@ -126,10 +125,13 @@
             self.dotgen.emit('subgraph cluster%d {' % graphindex)
         label = graph.get_display_text()
         if label is not None:
-            if self.highlight_graphs.get(graph):
-                fillcolor = '#f084c2'
+            colorindex = self.highlight_graphs.get(graph, 0)
+            if colorindex == 1:
+                fillcolor = '#f084c2'    # highlighted graph
+            elif colorindex == 2:
+                fillcolor = '#808080'    # invalidated graph
             else:
-                fillcolor = '#84f0c2'
+                fillcolor = '#84f0c2'    # normal color
             self.dotgen.emit_node(graphname, shape="octagon",
                                   label=label, fillcolor=fillcolor)
             self.pendingedges.append((graphname,
diff --git a/pypy/jit/metainterp/heapcache.py b/pypy/jit/metainterp/heapcache.py
--- a/pypy/jit/metainterp/heapcache.py
+++ b/pypy/jit/metainterp/heapcache.py
@@ -54,9 +54,10 @@
         if box in self.new_boxes:
             self.new_boxes[box] = False
         if box in self.dependencies:
-            for dep in self.dependencies[box]:
+            deps = self.dependencies[box]
+            del self.dependencies[box]
+            for dep in deps:
                 self._escape(dep)
-            del self.dependencies[box]
 
     def clear_caches(self, opnum, descr, argboxes):
         if opnum == rop.SETFIELD_GC:
diff --git a/pypy/jit/metainterp/history.py b/pypy/jit/metainterp/history.py
--- a/pypy/jit/metainterp/history.py
+++ b/pypy/jit/metainterp/history.py
@@ -9,6 +9,7 @@
 
 from pypy.jit.metainterp.resoperation import ResOperation, rop
 from pypy.jit.codewriter import heaptracker, longlong
+from pypy.rlib.objectmodel import compute_identity_hash
 
 # ____________________________________________________________
 
@@ -104,7 +105,7 @@
     getref._annspecialcase_ = 'specialize:arg(1)'
 
     def _get_hash_(self):
-        raise NotImplementedError
+        return compute_identity_hash(self)
 
     def clonebox(self):
         raise NotImplementedError
@@ -133,6 +134,9 @@
     def _get_str(self):
         raise NotImplementedError
 
+    def same_box(self, other):
+        return self is other
+
 class AbstractDescr(AbstractValue):
     __slots__ = ()
 
@@ -241,32 +245,15 @@
     def constbox(self):
         return self
 
+    def same_box(self, other):
+        return self.same_constant(other)
+
     def same_constant(self, other):
         raise NotImplementedError
 
     def __repr__(self):
         return 'Const(%s)' % self._getrepr_()
 
-    def __eq__(self, other):
-        "NOT_RPYTHON"
-        # Remember that you should not compare Consts with '==' in RPython.
-        # Consts have no special __hash__, in order to force different Consts
-        # from being considered as different keys when stored in dicts
-        # (as they always are after translation).  Use a dict_equal_consts()
-        # to get the other behavior (i.e. using this __eq__).
-        if self.__class__ is not other.__class__:
-            return False
-        try:
-            return self.value == other.value
-        except TypeError:
-            if (isinstance(self.value, Symbolic) and
-                isinstance(other.value, Symbolic)):
-                return self.value is other.value
-            raise
-
-    def __ne__(self, other):
-        return not (self == other)
-
 
 class ConstInt(Const):
     type = INT
@@ -688,33 +675,6 @@
 
 # ____________________________________________________________
 
-def dict_equal_consts():
-    "NOT_RPYTHON"
-    # Returns a dict in which Consts that compare as equal
-    # are identified when used as keys.
-    return r_dict(dc_eq, dc_hash)
-
-def dc_eq(c1, c2):
-    return c1 == c2
-
-def dc_hash(c):
-    "NOT_RPYTHON"
-    # This is called during translation only.  Avoid using identityhash(),
-    # to avoid forcing a hash, at least on lltype objects.
-    if not isinstance(c, Const):
-        return hash(c)
-    if isinstance(c.value, Symbolic):
-        return id(c.value)
-    try:
-        if isinstance(c, ConstPtr):
-            p = lltype.normalizeptr(c.value)
-            if p is not None:
-                return hash(p._obj)
-            else:
-                return 0
-        return c._get_hash_()
-    except lltype.DelayedPointer:
-        return -2      # xxx risk of changing hash...
 
 def make_hashable_int(i):
     from pypy.rpython.lltypesystem.ll2ctypes import NotCtypesAllocatedStructure
@@ -772,6 +732,7 @@
     failed_states = None
     retraced_count = 0
     terminating = False # see TerminatingLoopToken in compile.py
+    invalidated = False
     outermost_jitdriver_sd = None
     # and more data specified by the backend when the loop is compiled
     number = -1
@@ -974,6 +935,7 @@
         self.loops = []
         self.locations = []
         self.aborted_keys = []
+        self.invalidated_token_numbers = set()
 
     def set_history(self, history):
         self.operations = history.operations
@@ -1052,7 +1014,12 @@
             if loop in loops:
                 loops.remove(loop)
             loops.append(loop)
-        display_loops(loops, errmsg, extraloops)
+        highlight_loops = dict.fromkeys(extraloops, 1)
+        for loop in loops:
+            if hasattr(loop, '_looptoken_number') and (
+                    loop._looptoken_number in self.invalidated_token_numbers):
+                highlight_loops.setdefault(loop, 2)
+        display_loops(loops, errmsg, highlight_loops)
 
 # ----------------------------------------------------------------
 
diff --git a/pypy/jit/metainterp/memmgr.py b/pypy/jit/metainterp/memmgr.py
--- a/pypy/jit/metainterp/memmgr.py
+++ b/pypy/jit/metainterp/memmgr.py
@@ -68,7 +68,8 @@
         debug_print("Loop tokens before:", oldtotal)
         max_generation = self.current_generation - (self.max_age-1)
         for looptoken in self.alive_loops.keys():
-            if 0 <= looptoken.generation < max_generation:
+            if (0 <= looptoken.generation < max_generation or
+                looptoken.invalidated):
                 del self.alive_loops[looptoken]
         newtotal = len(self.alive_loops)
         debug_print("Loop tokens freed: ", oldtotal - newtotal)
diff --git a/pypy/jit/metainterp/optimizeopt/rewrite.py b/pypy/jit/metainterp/optimizeopt/rewrite.py
--- a/pypy/jit/metainterp/optimizeopt/rewrite.py
+++ b/pypy/jit/metainterp/optimizeopt/rewrite.py
@@ -106,10 +106,9 @@
             self.make_equal_to(op.result, v1)
         else:
             self.emit_operation(op)
-
-        # Synthesize the reverse ops for optimize_default to reuse
-        self.pure(rop.INT_ADD, [op.result, op.getarg(1)], op.getarg(0))
-        self.pure(rop.INT_SUB, [op.getarg(0), op.result], op.getarg(1))
+            # Synthesize the reverse ops for optimize_default to reuse
+            self.pure(rop.INT_ADD, [op.result, op.getarg(1)], op.getarg(0))
+            self.pure(rop.INT_SUB, [op.getarg(0), op.result], op.getarg(1))
 
     def optimize_INT_ADD(self, op):
         v1 = self.getvalue(op.getarg(0))
@@ -122,10 +121,9 @@
             self.make_equal_to(op.result, v1)
         else:
             self.emit_operation(op)
-
-        # Synthesize the reverse op for optimize_default to reuse
-        self.pure(rop.INT_SUB, [op.result, op.getarg(1)], op.getarg(0))
-        self.pure(rop.INT_SUB, [op.result, op.getarg(0)], op.getarg(1))
+            # Synthesize the reverse op for optimize_default to reuse
+            self.pure(rop.INT_SUB, [op.result, op.getarg(1)], op.getarg(0))
+            self.pure(rop.INT_SUB, [op.result, op.getarg(0)], op.getarg(1))
 
     def optimize_INT_MUL(self, op):
         v1 = self.getvalue(op.getarg(0))
@@ -141,13 +139,13 @@
             self.make_constant_int(op.result, 0)
         else:
             for lhs, rhs in [(v1, v2), (v2, v1)]:
-                # x & (x -1) == 0 is a quick test for power of 2
-                if (lhs.is_constant() and
-                    (lhs.box.getint() & (lhs.box.getint() - 1)) == 0):
-                    new_rhs = ConstInt(highest_bit(lhs.box.getint()))
-                    op = op.copy_and_change(rop.INT_LSHIFT, args=[rhs.box, new_rhs])
-                    break
-
+                if lhs.is_constant():
+                    x = lhs.box.getint()
+                    # x & (x - 1) == 0 is a quick test for power of 2
+                    if x & (x - 1) == 0:
+                        new_rhs = ConstInt(highest_bit(lhs.box.getint()))
+                        op = op.copy_and_change(rop.INT_LSHIFT, args=[rhs.box, new_rhs])
+                        break
             self.emit_operation(op)
 
     def optimize_UINT_FLOORDIV(self, op):
@@ -462,6 +460,14 @@
         self.optimizer.opaque_pointers[value] = True
         self.make_equal_to(op.result, value)
 
+    def optimize_CAST_PTR_TO_INT(self, op):
+        self.pure(rop.CAST_INT_TO_PTR, [op.result], op.getarg(0))
+        self.emit_operation(op)
+
+    def optimize_CAST_INT_TO_PTR(self, op):
+        self.pure(rop.CAST_PTR_TO_INT, [op.result], op.getarg(0))
+        self.emit_operation(op)
+
 dispatch_opt = make_dispatcher_method(OptRewrite, 'optimize_',
         default=OptRewrite.emit_operation)
 optimize_guards = _findall(OptRewrite, 'optimize_', 'GUARD')
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -2329,7 +2329,7 @@
         def _variables_equal(box, varname, strict):
             if varname not in virtuals:
                 if strict:
-                    assert box == oparse.getvar(varname)
+                    assert box.same_box(oparse.getvar(varname))
                 else:
                     assert box.value == oparse.getvar(varname).value
             else:
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -234,6 +234,30 @@
             """ % expected_value
             self.optimize_loop(ops, expected)
 
+    def test_reverse_of_cast(self):
+        ops = """
+        [i0]
+        p0 = cast_int_to_ptr(i0)
+        i1 = cast_ptr_to_int(p0)
+        jump(i1)
+        """
+        expected = """
+        [i0]
+        jump(i0)
+        """
+        self.optimize_loop(ops, expected)
+        ops = """
+        [p0]
+        i1 = cast_ptr_to_int(p0)
+        p1 = cast_int_to_ptr(i1)
+        jump(p1)
+        """
+        expected = """
+        [p0]
+        jump(p0)
+        """
+        self.optimize_loop(ops, expected)
+
     # ----------
 
     def test_remove_guard_class_1(self):
diff --git a/pypy/jit/metainterp/optimizeopt/util.py b/pypy/jit/metainterp/optimizeopt/util.py
--- a/pypy/jit/metainterp/optimizeopt/util.py
+++ b/pypy/jit/metainterp/optimizeopt/util.py
@@ -90,14 +90,11 @@
     for i in range(len(args1)):
         arg1 = args1[i]
         arg2 = args2[i]
-        if isinstance(arg1, history.Const):
-            if arg1.__class__ is not arg2.__class__:
+        if arg1 is None:
+            if arg2 is not None:
                 return False
-            if not arg1.same_constant(arg2):
-                return False
-        else:
-            if not arg1 is arg2:
-                return False
+        elif not arg1.same_box(arg2):
+            return False
     return True
 
 def args_hash(args):
@@ -106,10 +103,8 @@
     for arg in args:
         if arg is None:
             y = 17
-        elif isinstance(arg, history.Const):
+        else:
             y = arg._get_hash_()
-        else:
-            y = compute_identity_hash(arg)
         res = intmask((1000003 * res) ^ y)
     return res
 
@@ -145,9 +140,12 @@
         for i in range(op1.numargs()):
             x = op1.getarg(i)
             y = op2.getarg(i)
-            assert x == remap.get(y, y)
+            assert x.same_box(remap.get(y, y))
         if op2.result in remap:
-            assert op1.result == remap[op2.result]
+            if op2.result is None:
+                assert op1.result == remap[op2.result]
+            else:
+                assert op1.result.same_box(remap[op2.result])
         else:
             remap[op2.result] = op1.result
         if op1.getopnum() != rop.JUMP:      # xxx obscure
@@ -156,11 +154,20 @@
             assert len(op1.getfailargs()) == len(op2.getfailargs())
             if strict_fail_args:
                 for x, y in zip(op1.getfailargs(), op2.getfailargs()):
-                    assert x == remap.get(y, y)
+                    if x is None:
+                        assert remap.get(y, y) is None
+                    else:
+                        assert x.same_box(remap.get(y, y))
             else:
                 fail_args1 = set(op1.getfailargs())
                 fail_args2 = set([remap.get(y, y) for y in op2.getfailargs()])
-                assert fail_args1 == fail_args2
+                for x in fail_args1:
+                    for y in fail_args2:
+                        if x.same_box(y):
+                            fail_args2.remove(y)
+                            break
+                    else:
+                        assert False
     assert len(oplist1) == len(oplist2)
     print '-'*totwidth
     return True
diff --git a/pypy/jit/metainterp/optimizeopt/vstring.py b/pypy/jit/metainterp/optimizeopt/vstring.py
--- a/pypy/jit/metainterp/optimizeopt/vstring.py
+++ b/pypy/jit/metainterp/optimizeopt/vstring.py
@@ -587,10 +587,7 @@
             return True
         #
         if v1.is_nonnull() and v2.is_nonnull():
-            if l1box is not None and l2box is not None and (
-                l1box == l2box or (isinstance(l1box, ConstInt) and
-                                   isinstance(l2box, ConstInt) and
-                                   l1box.value == l2box.value)):
+            if l1box is not None and l2box is not None and l1box.same_box(l2box):
                 do = EffectInfo.OS_STREQ_LENGTHOK
             else:
                 do = EffectInfo.OS_STREQ_NONNULL
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -162,15 +162,18 @@
             if registers[i] is oldbox:
                 registers[i] = newbox
         if not we_are_translated():
-            assert oldbox not in registers[count:]
+            for b in registers[count:]:
+                assert not oldbox.same_box(b)
+                
 
     def make_result_of_lastop(self, resultbox):
         got_type = resultbox.type
-        if not we_are_translated():
-            typeof = {'i': history.INT,
-                      'r': history.REF,
-                      'f': history.FLOAT}
-            assert typeof[self.jitcode._resulttypes[self.pc]] == got_type
+        # XXX disabled for now, conflicts with str_guard_value
+        #if not we_are_translated():
+        #    typeof = {'i': history.INT,
+        #              'r': history.REF,
+        #              'f': history.FLOAT}
+        #    assert typeof[self.jitcode._resulttypes[self.pc]] == got_type
         target_index = ord(self.bytecode[self.pc-1])
         if got_type == history.INT:
             self.registers_i[target_index] = resultbox
@@ -219,6 +222,7 @@
                     'cast_float_to_int', 'cast_int_to_float',
                     'cast_float_to_singlefloat', 'cast_singlefloat_to_float',
                     'float_neg', 'float_abs',
+                    'cast_ptr_to_int', 'cast_int_to_ptr',
                     ]:
         exec py.code.Source('''
             @arguments("box")
@@ -895,6 +899,21 @@
     def _opimpl_guard_value(self, orgpc, box):
         self.implement_guard_value(orgpc, box)
 
+    @arguments("orgpc", "box", "box", "descr")
+    def opimpl_str_guard_value(self, orgpc, box, funcbox, descr):
+        if isinstance(box, Const):
+            return box     # no promotion needed, already a Const
+        else:
+            constbox = box.constbox()
+            resbox = self.do_residual_call(funcbox, descr, [box, constbox])
+            promoted_box = resbox.constbox()
+            # This is GUARD_VALUE because GUARD_TRUE assumes the existance
+            # of a label when computing resumepc
+            self.generate_guard(rop.GUARD_VALUE, resbox, [promoted_box],
+                                resumepc=orgpc)
+            self.metainterp.replace_box(box, constbox)
+            return constbox
+
     opimpl_int_guard_value = _opimpl_guard_value
     opimpl_ref_guard_value = _opimpl_guard_value
     opimpl_float_guard_value = _opimpl_guard_value
diff --git a/pypy/jit/metainterp/quasiimmut.py b/pypy/jit/metainterp/quasiimmut.py
--- a/pypy/jit/metainterp/quasiimmut.py
+++ b/pypy/jit/metainterp/quasiimmut.py
@@ -2,6 +2,7 @@
 from pypy.rpython.lltypesystem import lltype, rclass
 from pypy.rpython.annlowlevel import cast_base_ptr_to_instance
 from pypy.jit.metainterp.history import AbstractDescr
+from pypy.rlib.objectmodel import we_are_translated
 
 
 def get_mutate_field_name(fieldname):
@@ -50,13 +51,13 @@
 
 class QuasiImmut(object):
     llopaque = True
+    compress_limit = 30
     
     def __init__(self, cpu):
         self.cpu = cpu
         # list of weakrefs to the LoopTokens that must be invalidated if
         # this value ever changes
         self.looptokens_wrefs = []
-        self.compress_limit = 30
 
     def hide(self):
         qmut_ptr = self.cpu.ts.cast_instance_to_base_ref(self)
@@ -75,6 +76,8 @@
     def compress_looptokens_list(self):
         self.looptokens_wrefs = [wref for wref in self.looptokens_wrefs
                                       if wref() is not None]
+        # NB. we must keep around the looptoken_wrefs that are
+        # already invalidated; see below
         self.compress_limit = (len(self.looptokens_wrefs) + 15) * 2
 
     def invalidate(self):
@@ -86,7 +89,16 @@
         for wref in wrefs:
             looptoken = wref()
             if looptoken is not None:
+                looptoken.invalidated = True
                 self.cpu.invalidate_loop(looptoken)
+                # NB. we must call cpu.invalidate_loop() even if
+                # looptoken.invalidated was already set to True.
+                # It's possible to invalidate several times the
+                # same looptoken; see comments in jit.backend.model
+                # in invalidate_loop().
+                if not we_are_translated():
+                    self.cpu.stats.invalidated_token_numbers.add(
+                        looptoken.number)
 
 
 class QuasiImmutDescr(AbstractDescr):
diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py
--- a/pypy/jit/metainterp/resoperation.py
+++ b/pypy/jit/metainterp/resoperation.py
@@ -433,6 +433,8 @@
     'INT_INVERT/1',
     #
     'SAME_AS/1',      # gets a Const or a Box, turns it into another Box
+    'CAST_PTR_TO_INT/1',
+    'CAST_INT_TO_PTR/1',
     #
     'PTR_EQ/2b',
     'PTR_NE/2b',
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -13,7 +13,7 @@
 from pypy.rlib.jit import (JitDriver, we_are_jitted, hint, dont_look_inside,
     loop_invariant, elidable, promote, jit_debug, assert_green,
     AssertGreenFailed, unroll_safe, current_trace_length, look_inside_iff,
-    isconstant, isvirtual)
+    isconstant, isvirtual, promote_string)
 from pypy.rlib.rarithmetic import ovfcheck
 from pypy.rpython.lltypesystem import lltype, llmemory, rffi
 from pypy.rpython.ootypesystem import ootype
@@ -3440,7 +3440,6 @@
 
 class TestLLtype(BaseLLtypeTests, LLJitMixin):
     def test_tagged(self):
-        py.test.skip("implement me")
         from pypy.rlib.objectmodel import UnboxedValue
         class Base(object):
             __slots__ = ()
@@ -3492,3 +3491,35 @@
                 pc += 1
             return pc
         res = self.meta_interp(main, [False, 100, True], taggedpointers=True)
+
+    def test_rerased(self):
+        from pypy.rlib.rerased import erase_int, unerase_int, new_erasing_pair
+        eraseX, uneraseX = new_erasing_pair("X")
+        #
+        class X:
+            def __init__(self, a, b):
+                self.a = a
+                self.b = b
+        #
+        def f(i, j):
+            # 'j' should be 0 or 1, not other values
+            if j > 0:
+                e = eraseX(X(i, j))
+            else:
+                try:
+                    e = erase_int(i)
+                except OverflowError:
+                    return -42
+            if j & 1:
+                x = uneraseX(e)
+                return x.a - x.b
+            else:
+                return unerase_int(e)
+        #
+        x = self.interp_operations(f, [-128, 0], taggedpointers=True)
+        assert x == -128
+        bigint = sys.maxint//2 + 1
+        x = self.interp_operations(f, [bigint, 0], taggedpointers=True)
+        assert x == -42
+        x = self.interp_operations(f, [1000, 1], taggedpointers=True)
+        assert x == 999
diff --git a/pypy/jit/metainterp/test/test_heapcache.py b/pypy/jit/metainterp/test/test_heapcache.py
--- a/pypy/jit/metainterp/test/test_heapcache.py
+++ b/pypy/jit/metainterp/test/test_heapcache.py
@@ -355,6 +355,14 @@
         assert not h.is_unescaped(box1)
         assert not h.is_unescaped(box2)
 
+    def test_circular_virtuals(self):
+        h = HeapCache()
+        h.new(box1)
+        h.new(box2)
+        h.invalidate_caches(rop.SETFIELD_GC, None, [box1, box2])
+        h.invalidate_caches(rop.SETFIELD_GC, None, [box2, box1])
+        h.invalidate_caches(rop.SETFIELD_GC, None, [box3, box1]) # does not crash
+
     def test_unescaped_array(self):
         h = HeapCache()
         h.new_array(box1, lengthbox1)
@@ -362,4 +370,4 @@
         h.invalidate_caches(rop.SETARRAYITEM_GC, None, [box1, index1, box2])
         assert h.is_unescaped(box1)
         h.invalidate_caches(rop.SETARRAYITEM_GC, None, [box2, index1, box1])
-        assert not h.is_unescaped(box1)
\ No newline at end of file
+        assert not h.is_unescaped(box1)
diff --git a/pypy/jit/metainterp/test/test_memmgr.py b/pypy/jit/metainterp/test/test_memmgr.py
--- a/pypy/jit/metainterp/test/test_memmgr.py
+++ b/pypy/jit/metainterp/test/test_memmgr.py
@@ -18,6 +18,7 @@
 
 class FakeLoopToken:
     generation = 0
+    invalidated = False
 
 
 class _TestMemoryManager:
diff --git a/pypy/jit/metainterp/test/test_quasiimmut.py b/pypy/jit/metainterp/test/test_quasiimmut.py
--- a/pypy/jit/metainterp/test/test_quasiimmut.py
+++ b/pypy/jit/metainterp/test/test_quasiimmut.py
@@ -48,6 +48,13 @@
 
 class QuasiImmutTests(object):
 
+    def setup_method(self, meth):
+        self.prev_compress_limit = QuasiImmut.compress_limit
+        QuasiImmut.compress_limit = 1
+
+    def teardown_method(self, meth):
+        QuasiImmut.compress_limit = self.prev_compress_limit
+
     def test_simple_1(self):
         myjitdriver = JitDriver(greens=['foo'], reds=['x', 'total'])
         class Foo:
@@ -289,7 +296,7 @@
             return total
 
         res = self.meta_interp(main, [])
-        self.check_loop_count(9)
+        self.check_tree_loop_count(6)
         assert res == main()
 
     def test_change_during_running(self):
@@ -317,7 +324,7 @@
         assert f(100, 15) == 3009
         res = self.meta_interp(f, [100, 15])
         assert res == 3009
-        self.check_loops(guard_not_invalidated=2, getfield_gc=0,
+        self.check_loops(guard_not_invalidated=4, getfield_gc=0,
                          call_may_force=0, guard_not_forced=0)
 
     def test_list_simple_1(self):
@@ -453,10 +460,30 @@
         assert f(100, 15) == 3009
         res = self.meta_interp(f, [100, 15])
         assert res == 3009
-        self.check_loops(guard_not_invalidated=2, getfield_gc=0,
+        self.check_loops(guard_not_invalidated=4, getfield_gc=0,
                          getarrayitem_gc=0, getarrayitem_gc_pure=0,
                          call_may_force=0, guard_not_forced=0)
 
+    def test_invalidated_loop_is_not_used_any_more_as_target(self):
+        myjitdriver = JitDriver(greens=['foo'], reds=['x'])
+        class Foo:
+            _immutable_fields_ = ['step?']
+        @dont_look_inside
+        def residual(x, foo):
+            if x == 20:
+                foo.step = 1
+        def f(x):
+            foo = Foo()
+            foo.step = 2
+            while x > 0:
+                myjitdriver.jit_merge_point(foo=foo, x=x)
+                residual(x, foo)
+                x -= foo.step
+            return foo.step
+        res = self.meta_interp(f, [60])
+        assert res == 1
+        self.check_tree_loop_count(4)   # at least not 2 like before
+
 
 class TestLLtypeGreenFieldsTests(QuasiImmutTests, LLJitMixin):
     pass
diff --git a/pypy/jit/metainterp/test/test_resume.py b/pypy/jit/metainterp/test/test_resume.py
--- a/pypy/jit/metainterp/test/test_resume.py
+++ b/pypy/jit/metainterp/test/test_resume.py
@@ -1,3 +1,4 @@
+from __future__ import with_statement
 import py
 from pypy.rpython.lltypesystem import lltype, llmemory, rffi
 from pypy.jit.metainterp.optimizeopt.optimizer import OptValue
@@ -180,22 +181,27 @@
     reader.consume_boxes(info, bi, br, bf)
     b1s = reader.liveboxes[0]
     b2s = reader.liveboxes[1]
-    assert bi == [b1s, ConstInt(111), b1s]
-    assert br == [ConstPtr(gcrefnull), b2s]
+    assert_same(bi, [b1s, ConstInt(111), b1s])
+    assert_same(br, [ConstPtr(gcrefnull), b2s])
     bi, br, bf = [None]*2, [None]*0, [None]*0
     info = MyBlackholeInterp([lltype.Signed,
                               lltype.Signed]).get_current_position_info()
     reader.consume_boxes(info, bi, br, bf)
-    assert bi == [ConstInt(222), ConstInt(333)]
+    assert_same(bi, [ConstInt(222), ConstInt(333)])
     bi, br, bf = [None]*2, [None]*1, [None]*0
     info = MyBlackholeInterp([lltype.Signed, llmemory.GCREF,
                               lltype.Signed]).get_current_position_info()
     reader.consume_boxes(info, bi, br, bf)
     b3s = reader.liveboxes[2]
-    assert bi == [b1s, b3s]
-    assert br == [b2s]
+    assert_same(bi, [b1s, b3s])
+    assert_same(br, [b2s])
     #
 
+def assert_same(list1, list2):
+    assert len(list1) == len(list2)
+    for b1, b2 in zip(list1, list2):
+        assert b1.same_box(b2)
+
 def test_simple_read_tagged_ints():
     storage = Storage()
     storage.rd_consts = []
@@ -1023,11 +1029,11 @@
     metainterp = MyMetaInterp()
     reader = ResumeDataFakeReader(storage, newboxes, metainterp)
     lst = reader.consume_boxes()
-    assert lst == [b1t, b1t, b3t]
+    assert_same(lst, [b1t, b1t, b3t])
     lst = reader.consume_boxes()
-    assert lst == [ConstInt(2), ConstInt(3)]
+    assert_same(lst, [ConstInt(2), ConstInt(3)])
     lst = reader.consume_boxes()
-    assert lst == [b1t, ConstInt(1), b1t, b1t]
+    assert_same(lst, [b1t, ConstInt(1), b1t, b1t])
     assert metainterp.trace == []    
 
 
@@ -1044,11 +1050,11 @@
     reader = ResumeDataFakeReader(storage, newboxes, metainterp)
     lst = reader.consume_boxes()
     c1t = ConstInt(111)
-    assert lst == [c1t, b2t, b3t]
+    assert_same(lst, [c1t, b2t, b3t])
     lst = reader.consume_boxes()
-    assert lst == [ConstInt(2), ConstInt(3)]
+    assert_same(lst, [ConstInt(2), ConstInt(3)])
     lst = reader.consume_boxes()
-    assert lst == [c1t, ConstInt(1), c1t, b2t]
+    assert_same(lst, [c1t, ConstInt(1), c1t, b2t])
     assert metainterp.trace == []
 
 
@@ -1114,9 +1120,11 @@
     # check that we get the operations in 'expected', in a possibly different
     # order.
     assert len(trace) == len(expected)
-    for x in trace:
-        assert x in expected
-        expected.remove(x)
+    with CompareableConsts():
+        for x in trace:
+            assert x in expected
+            expected.remove(x)
+
     ptr = b2t.value._obj.container._as_ptr()
     assert lltype.typeOf(ptr) == lltype.Ptr(LLtypeMixin.NODE)
     assert ptr.value == 111
@@ -1126,6 +1134,18 @@
     assert ptr2.parent.value == 33
     assert ptr2.parent.next == ptr
 
+class CompareableConsts(object):
+    def __init__(self):
+        self.oldeq = None
+        
+    def __enter__(self):
+        assert self.oldeq is None
+        self.oldeq = Const.__eq__
+        Const.__eq__ = Const.same_box
+        
+    def __exit__(self, type, value, traceback):
+        Const.__eq__ = self.oldeq
+
 def test_virtual_adder_make_varray():
     b2s, b4s = [BoxPtr(), BoxInt(4)]
     c1s = ConstInt(111)
@@ -1163,8 +1183,9 @@
         (rop.SETARRAYITEM_GC, [b2t,ConstInt(1), c1s], None,
                               LLtypeMixin.arraydescr),
         ]
-    for x, y in zip(expected, trace):
-        assert x == y
+    with CompareableConsts():
+        for x, y in zip(expected, trace):
+            assert x == y
     #
     ptr = b2t.value._obj.container._as_ptr()
     assert lltype.typeOf(ptr) == lltype.Ptr(lltype.GcArray(lltype.Signed))
@@ -1207,8 +1228,9 @@
         (rop.SETFIELD_GC, [b2t, c1s],  None, LLtypeMixin.adescr),
         (rop.SETFIELD_GC, [b2t, b4t], None, LLtypeMixin.bdescr),
         ]
-    for x, y in zip(expected, trace):
-        assert x == y
+    with CompareableConsts():
+        for x, y in zip(expected, trace):
+            assert x == y
     #
     ptr = b2t.value._obj.container._as_ptr()
     assert lltype.typeOf(ptr) == lltype.Ptr(LLtypeMixin.S)
diff --git a/pypy/jit/metainterp/test/test_string.py b/pypy/jit/metainterp/test/test_string.py
--- a/pypy/jit/metainterp/test/test_string.py
+++ b/pypy/jit/metainterp/test/test_string.py
@@ -3,7 +3,8 @@
 from pypy.jit.codewriter.policy import StopAtXPolicy
 from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin
 from pypy.rlib.debug import debug_print
-from pypy.rlib.jit import JitDriver, dont_look_inside, we_are_jitted
+from pypy.rlib.jit import JitDriver, dont_look_inside, we_are_jitted,\
+     promote_string
 from pypy.rlib.rstring import StringBuilder
 from pypy.rpython.ootypesystem import ootype
 
@@ -507,6 +508,18 @@
                           'jump': 1, 'int_is_true': 1,
                           'guard_not_invalidated': 1})
 
+    def test_promote_string(self):
+        driver = JitDriver(greens = [], reds = ['n'])
+        
+        def f(n):
+            while n < 21:
+                driver.jit_merge_point(n=n)
+                promote_string(str(n % 3))
+                n += 1
+            return 0
+
+        self.meta_interp(f, [0])
+        self.check_loops(call=3 + 1) # one for int2str
 
 #class TestOOtype(StringTests, OOJitMixin):
 #    CALL = "oosend"
diff --git a/pypy/jit/metainterp/test/test_virtualref.py b/pypy/jit/metainterp/test/test_virtualref.py
--- a/pypy/jit/metainterp/test/test_virtualref.py
+++ b/pypy/jit/metainterp/test/test_virtualref.py
@@ -3,6 +3,7 @@
 from pypy.rpython.llinterp import LLException
 from pypy.rlib.jit import JitDriver, dont_look_inside, vref_None
 from pypy.rlib.jit import virtual_ref, virtual_ref_finish, InvalidVirtualRef
+from pypy.rlib.jit import non_virtual_ref
 from pypy.rlib.objectmodel import compute_unique_id
 from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin, _get_jitcodes
 from pypy.jit.metainterp.resoperation import rop
@@ -595,6 +596,65 @@
         res = self.meta_interp(fn, [10])
         assert res == 6
 
+    def test_is_virtual(self):
+        myjitdriver = JitDriver(greens=[], reds=['n', 'res1'])
+        class X:
+            pass
+        @dont_look_inside
+        def residual(vref):
+            return vref.virtual
+        #
+        def f(n):
+            res1 = -42
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, res1=res1)
+                x = X()
+                vref = virtual_ref(x)
+                res1 = residual(vref)
+                virtual_ref_finish(vref, x)
+                n -= 1
+            return res1
+        #
+        res = self.meta_interp(f, [10])
+        assert res == 1
+
+    def test_is_not_virtual_none(self):
+        myjitdriver = JitDriver(greens=[], reds=['n', 'res1'])
+        @dont_look_inside
+        def residual(vref):
+            return vref.virtual
+        #
+        def f(n):
+            res1 = -42
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, res1=res1)
+                res1 = residual(vref_None)
+                n -= 1
+            return res1
+        #
+        res = self.meta_interp(f, [10])
+        assert res == 0
+
+    def test_is_not_virtual_non_none(self):
+        myjitdriver = JitDriver(greens=[], reds=['n', 'res1'])
+        class X:
+            pass
+        @dont_look_inside
+        def residual(vref):
+            return vref.virtual
+        #
+        def f(n):
+            res1 = -42
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, res1=res1)
+                x = X()
+                res1 = residual(non_virtual_ref(x))
+                n -= 1
+            return res1
+        #
+        res = self.meta_interp(f, [10])
+        assert res == 0
+
 
 class TestLLtype(VRefTests, LLJitMixin):
     pass
diff --git a/pypy/jit/metainterp/virtualref.py b/pypy/jit/metainterp/virtualref.py
--- a/pypy/jit/metainterp/virtualref.py
+++ b/pypy/jit/metainterp/virtualref.py
@@ -39,6 +39,7 @@
     def replace_force_virtual_with_call(self, graphs):
         # similar to rvirtualizable2.replace_force_virtualizable_with_call().
         c_force_virtual_ptr = None
+        c_is_virtual_ptr = None
         force_virtual_count = 0
         for graph in graphs:
             for block in graph.iterblocks():
@@ -52,6 +53,13 @@
                         op.opname = 'direct_call'
                         op.args = [c_force_virtual_ptr, op.args[0]]
                         force_virtual_count += 1
+                    #
+                    if op.opname == 'jit_is_virtual':
+                        if c_is_virtual_ptr is None:
+                            c_is_virtual_ptr = self.get_is_virtual_fnptr()
+                        #
+                        op.opname = 'direct_call'
+                        op.args = [c_is_virtual_ptr, op.args[0]]
         #
         if c_force_virtual_ptr is not None:
             log("replaced %d 'jit_force_virtual' with %r" % (force_virtual_count,
@@ -129,6 +137,17 @@
             force_virtual_if_necessary)
         return inputconst(lltype.typeOf(funcptr), funcptr)
 
+    def get_is_virtual_fnptr(self):
+        #
+        def is_virtual(inst):
+            if not inst:
+                return False
+            return inst.typeptr == self.jit_virtual_ref_vtable
+        #
+        FUNC = lltype.FuncType([rclass.OBJECTPTR], lltype.Bool)
+        funcptr = self.warmrunnerdesc.helper_func(lltype.Ptr(FUNC), is_virtual)
+        return inputconst(lltype.typeOf(funcptr), funcptr)
+
     def force_virtual(self, inst):
         vref = lltype.cast_pointer(lltype.Ptr(self.JIT_VIRTUAL_REF), inst)
         token = vref.virtual_token
diff --git a/pypy/jit/metainterp/warmstate.py b/pypy/jit/metainterp/warmstate.py
--- a/pypy/jit/metainterp/warmstate.py
+++ b/pypy/jit/metainterp/warmstate.py
@@ -178,7 +178,7 @@
         if self.compiled_merge_points_wref is not None:
             for wref in self.compiled_merge_points_wref:
                 looptoken = wref()
-                if looptoken is not None:
+                if looptoken is not None and not looptoken.invalidated:
                     result.append(looptoken)
         return result
 
diff --git a/pypy/module/__builtin__/__init__.py b/pypy/module/__builtin__/__init__.py
--- a/pypy/module/__builtin__/__init__.py
+++ b/pypy/module/__builtin__/__init__.py
@@ -20,6 +20,10 @@
         'any'           : 'app_functional.any',
         'all'           : 'app_functional.all',
         'sum'           : 'app_functional.sum',
+        'map'           : 'app_functional.map',
+        'reduce'        : 'app_functional.reduce',
+        'filter'        : 'app_functional.filter',
+        'zip'           : 'app_functional.zip',
         'vars'          : 'app_inspect.vars',
         'dir'           : 'app_inspect.dir',
 
@@ -86,11 +90,7 @@
         'enumerate'     : 'functional.W_Enumerate',
         'min'           : 'functional.min',
         'max'           : 'functional.max',
-        'map'           : 'functional.map',
-        'zip'           : 'functional.zip',
-        'reduce'        : 'functional.reduce',
         'reversed'      : 'functional.reversed',
-        'filter'        : 'functional.filter',
         'super'         : 'descriptor.W_Super',
         'staticmethod'  : 'descriptor.StaticMethod',
         'classmethod'   : 'descriptor.ClassMethod',
@@ -119,7 +119,7 @@
            builtin = space.interpclass_w(w_builtin)
            if isinstance(builtin, module.Module):
                return builtin
-       # no builtin! make a default one.  Given them None, at least.
+       # no builtin! make a default one.  Give them None, at least.
        builtin = module.Module(space, None)
        space.setitem(builtin.w_dict, space.wrap('None'), space.w_None)
        return builtin
diff --git a/pypy/module/__builtin__/app_functional.py b/pypy/module/__builtin__/app_functional.py
--- a/pypy/module/__builtin__/app_functional.py
+++ b/pypy/module/__builtin__/app_functional.py
@@ -48,4 +48,135 @@
         # Very intentionally *not* +=, that would have different semantics if
         # start was a mutable type, such as a list
         last = last + x
-    return last
\ No newline at end of file
+    return last
+
+def map(func, *collections):
+    """map(function, sequence[, sequence, ...]) -> list
+
+Return a list of the results of applying the function to the items of
+the argument sequence(s).  If more than one sequence is given, the
+function is called with an argument list consisting of the corresponding
+item of each sequence, substituting None for missing values when not all
+sequences have the same length.  If the function is None, return a list of
+the items of the sequence (or a list of tuples if more than one sequence)."""
+    if not collections:
+        raise TypeError("map() requires at least two arguments")
+    num_collections = len(collections)
+    none_func = func is None
+    if num_collections == 1:
+        if none_func:
+            return list(collections[0])
+        else:
+            # Special case for the really common case of a single collection,
+            # this can be eliminated if we could unroll that loop that creates
+            # `args` based on whether or not len(collections) was constant
+            result = []
+            for item in collections[0]:
+                result.append(func(item))
+            return result
+    result = []
+    # Pair of (iterator, has_finished)
+    iterators = [(iter(seq), False) for seq in collections]
+    while True:
+        cont = False
+        args = []
+        for idx, (iterator, has_finished) in enumerate(iterators):
+            val = None
+            if not has_finished:
+                try:
+                    val = next(iterator)
+                except StopIteration:
+                    iterators[idx] = (None, True)
+                else:
+                    cont = True
+            args.append(val)
+        args = tuple(args)
+        if cont:
+            if none_func:
+                result.append(args)
+            else:
+                result.append(func(*args))
+        else:
+            return result
+
+sentinel = object()
+
+def reduce(func, sequence, initial=sentinel):
+    """reduce(function, sequence[, initial]) -> value
+
+Apply a function of two arguments cumulatively to the items of a sequence,
+from left to right, so as to reduce the sequence to a single value.
+For example, reduce(lambda x, y: x+y, [1, 2, 3, 4, 5]) calculates
+((((1+2)+3)+4)+5).  If initial is present, it is placed before the items
+of the sequence in the calculation, and serves as a default when the
+sequence is empty."""
+    iterator = iter(sequence)
+    if initial is sentinel:
+        try:
+            initial = next(iterator)
+        except StopIteration:
+            raise TypeError("reduce() of empty sequence with no initial value")
+    result = initial
+    for item in iterator:
+        result = func(result, item)
+    return result
+
+def filter(func, seq):
+    """filter(function or None, sequence) -> list, tuple, or string
+
+Return those items of sequence for which function(item) is true.  If
+function is None, return the items that are true.  If sequence is a tuple
+or string, return the same type, else return a list."""
+    if func is None:
+        func = bool
+    if isinstance(seq, str):
+        return _filter_string(func, seq, str)
+    elif isinstance(seq, unicode):
+        return _filter_string(func, seq, unicode)
+    elif isinstance(seq, tuple):
+        return _filter_tuple(func, seq)
+    result = []
+    for item in seq:
+        if func(item):
+            result.append(item)
+    return result
+
+def _filter_string(func, string, str_type):
+    if func is bool and type(string) is str_type:
+        return string
+    result = []
+    for i in range(len(string)):
+        # You must call __getitem__ on the strings, simply iterating doesn't
+        # work :/
+        item = string[i]
+        if func(item):
+            if not isinstance(item, str_type):
+                raise TypeError("__getitem__ returned a non-string type")
+            result.append(item)
+    return str_type().join(result)
+
+def _filter_tuple(func, seq):
+    result = []
+    for i in range(len(seq)):
+        # Again, must call __getitem__, at least there are tests.
+        item = seq[i]
+        if func(item):
+            result.append(item)
+    return tuple(result)
+
+def zip(*sequences):
+    """zip(seq1 [, seq2 [...]]) -> [(seq1[0], seq2[0] ...), (...)]
+
+Return a list of tuples, where each tuple contains the i-th element
+from each of the argument sequences.  The returned list is truncated
+in length to the length of the shortest argument sequence."""
+    if not sequences:
+        return []
+    result = []
+    iterators = [iter(seq) for seq in sequences]
+    while True:
+        try:
+            items = [next(it) for it in iterators]
+        except StopIteration:
+            return result
+        result.append(tuple(items))
diff --git a/pypy/module/__builtin__/app_io.py b/pypy/module/__builtin__/app_io.py
--- a/pypy/module/__builtin__/app_io.py
+++ b/pypy/module/__builtin__/app_io.py
@@ -27,7 +27,20 @@
     co = compile(source.rstrip()+"\n", filename, 'exec')
     exec co in glob, loc
 
-def raw_input(prompt=None):
+def _write_prompt(stdout, prompt):
+    print >> stdout, prompt,
+    try:
+        flush = stdout.flush
+    except AttributeError:
+        pass
+    else:
+        flush()
+    try:
+        stdout.softspace = 0
+    except (AttributeError, TypeError):
+        pass
+
+def raw_input(prompt=''):
     """raw_input([prompt]) -> string
 
 Read a string from standard input.  The trailing newline is stripped.
@@ -47,18 +60,10 @@
     if (hasattr(sys, '__raw_input__') and
         isinstance(stdin, file)  and stdin.fileno() == 0 and stdin.isatty() and
         isinstance(stdout, file) and stdout.fileno() == 1):
-        if prompt is None:
-            prompt = ''
-        return sys.__raw_input__(prompt)
+        _write_prompt(stdout, '')
+        return sys.__raw_input__(str(prompt))
 
-    if prompt is not None:
-        stdout.write(prompt)
-        try:
-            flush = stdout.flush
-        except AttributeError:
-            pass
-        else:
-            flush()
+    _write_prompt(stdout, prompt)
     line = stdin.readline()
     if not line:    # inputting an empty line gives line == '\n'
         raise EOFError
diff --git a/pypy/module/__builtin__/compiling.py b/pypy/module/__builtin__/compiling.py
--- a/pypy/module/__builtin__/compiling.py
+++ b/pypy/module/__builtin__/compiling.py
@@ -84,8 +84,8 @@
         raise OperationError(space.w_TypeError,
               w('eval() arg 1 must be a string or code object'))
 
-    caller = space.getexecutioncontext().gettopframe_nohidden()
     if space.is_w(w_globals, space.w_None):
+        caller = space.getexecutioncontext().gettopframe_nohidden()
         if caller is None:
             w_globals = space.newdict()
             if space.is_w(w_locals, space.w_None):
@@ -97,13 +97,9 @@
     elif space.is_w(w_locals, space.w_None):
         w_locals = w_globals
 
-    try:
-        space.getitem(w_globals, space.wrap('__builtins__'))
-    except OperationError, e:
-        if not e.match(space, space.w_KeyError):
-            raise
-        if caller is not None:
-            w_builtin = space.builtin.pick_builtin(caller.w_globals)
-            space.setitem(w_globals, space.wrap('__builtins__'), w_builtin)
+    # xxx removed: adding '__builtins__' to the w_globals dict, if there
+    # is none.  This logic was removed as costly (it requires to get at
+    # the gettopframe_nohidden()).  I bet no test fails, and it's a really
+    # obscure case.
 
     return codeobj.exec_code(space, w_globals, w_locals)
diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py
--- a/pypy/module/__builtin__/functional.py
+++ b/pypy/module/__builtin__/functional.py
@@ -193,229 +193,14 @@
     return min_max(space, __args__, "max")
 
 def min(space, __args__):
-    """Return the smallest item in a sequence.
+    """min(iterable[, key=func]) -> value
+    min(a, b, c, ...[, key=func]) -> value
 
-    If more than one argument is passed, return the minimum of them.
+    With a single iterable argument, return its smallest item.
+    With two or more arguments, return the smallest argument.
     """
     return min_max(space, __args__, "min")
 
- at unwrap_spec(collections_w="args_w")
-def map(space, w_func, collections_w):
-    """does 3 separate things, hence this enormous docstring.
-       1.  if function is None, return a list of tuples, each with one
-           item from each collection.  If the collections have different
-           lengths,  shorter ones are padded with None.
-
-       2.  if function is not None, and there is only one collection,
-           apply function to every item in the collection and return a
-           list of the results.
-
-       3.  if function is not None, and there are several collections,
-           repeatedly call the function with one argument from each
-           collection.  If the collections have different lengths,
-           shorter ones are padded with None
-    """
-    if not collections_w:
-        msg = "map() requires at least two arguments"
-        raise OperationError(space.w_TypeError, space.wrap(msg))
-    none_func = space.is_w(w_func, space.w_None)
-    if len(collections_w) == 1:
-        w_collection = collections_w[0]
-        if none_func:
-            result_w = space.unpackiterable(w_collection)
-        else:
-            result_w = map_single_collection(space, w_func, w_collection)
-    else:
-        result_w = map_multiple_collections(space, w_func, collections_w,
-                                            none_func)
-    return space.newlist(result_w)
-
-def map_single_collection(space, w_func, w_collection):
-    """Special case for 'map(func, coll)', where 'func' is not None and there
-    is only one 'coll' argument."""
-    w_iter = space.iter(w_collection)
-    # xxx special hacks for speed
-    from pypy.interpreter import function, pycode
-    if isinstance(w_func, function.Function):
-        # xxx compatibility issue: what if func_code is modified in the
-        # middle of running map()??  That's far too obscure for me to care...
-        code = w_func.getcode()
-        fast_natural_arity = code.fast_natural_arity
-        if fast_natural_arity == (1|pycode.PyCode.FLATPYCALL):
-            assert isinstance(code, pycode.PyCode)
-            return map_single_user_function(code, w_func, w_iter)
-    # /xxx end of special hacks
-    return map_single_other_callable(space, w_func, w_iter)
-
-def map_single_other_callable(space, w_func, w_iter):
-    result_w = []
-    while True:
-        try:
-            w_item = space.next(w_iter)
-        except OperationError, e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            break
-        result_w.append(space.call_function(w_func, w_item))
-    return result_w
-map_single_other_callable._dont_inline_ = True
-
-from pypy.rlib.jit import JitDriver
-mapjitdriver = JitDriver(greens = ['code'],
-                         reds = ['w_func', 'w_iter', 'result_w'])
-def map_single_user_function(code, w_func, w_iter):
-    result_w = []
-    while True:
-        mapjitdriver.can_enter_jit(code=code, w_func=w_func,
-                                   w_iter=w_iter, result_w=result_w)
-        mapjitdriver.jit_merge_point(code=code, w_func=w_func,
-                                     w_iter=w_iter, result_w=result_w)
-        space = w_func.space
-        try:
-            w_item = space.next(w_iter)
-        except OperationError, e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            break
-        new_frame = space.createframe(code, w_func.w_func_globals,
-                                      w_func)
-        new_frame.locals_stack_w[0] = w_item
-        w_res = new_frame.run()
-        result_w.append(w_res)
-    return result_w
-
-def map_multiple_collections(space, w_func, collections_w, none_func):
-    result_w = []
-    iterators_w = [space.iter(w_seq) for w_seq in collections_w]
-    num_iterators = len(iterators_w)
-    while True:
-        cont = False
-        args_w = [space.w_None] * num_iterators
-        for i in range(num_iterators):
-            if iterators_w[i] is not None:
-                try:
-                    args_w[i] = space.next(iterators_w[i])
-                except OperationError, e:
-                    if not e.match(space, space.w_StopIteration):
-                        raise
-                    iterators_w[i] = None
-                else:
-                    cont = True
-        if not cont:
-            break
-        w_args = space.newtuple(args_w)
-        if none_func:
-            w_res = w_args
-        else:
-            w_res = space.call(w_func, w_args)
-        result_w.append(w_res)
-    return result_w
-
- at unwrap_spec(sequences_w="args_w")
-def zip(space, sequences_w):
-    """Return a list of tuples, where the nth tuple contains every nth item of
-    each collection.
-
-    If the collections have different lengths, zip returns a list as long as the
-    shortest collection, ignoring the trailing items in the other collections.
-    """
-    if not sequences_w:
-        return space.newlist([])
-    result_w = []
-    iterators_w = [space.iter(w_seq) for w_seq in sequences_w]
-    while True:
-        try:
-            items_w = [space.next(w_it) for w_it in iterators_w]
-        except OperationError, e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            return space.newlist(result_w)
-        result_w.append(space.newtuple(items_w))
-
-def reduce(space, w_func, w_sequence, w_initial=NoneNotWrapped):
-    """ Apply function of two arguments cumulatively to the items of sequence,
-        from left to right, so as to reduce the sequence to a single value.
-        Optionally begin with an initial value.
-    """
-    w_iter = space.iter(w_sequence)
-    if w_initial is None:
-        try:
-            w_initial = space.next(w_iter)
-        except OperationError, e:
-            if e.match(space, space.w_StopIteration):
-                msg = "reduce() of empty sequence with no initial value"
-                raise OperationError(space.w_TypeError, space.wrap(msg))
-            raise
-    w_result = w_initial
-    while True:
-        try:
-            w_next = space.next(w_iter)
-        except OperationError, e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            break
-        w_result = space.call_function(w_func, w_result, w_next)
-    return w_result
-
-def filter(space, w_func, w_seq):
-    """construct a list of those elements of collection for which function
-       is True.  If function is None, then return the items in the sequence
-       which are True.
-    """
-    if space.is_true(space.isinstance(w_seq, space.w_str)):
-        return _filter_string(space, w_func, w_seq, space.w_str)
-    if space.is_true(space.isinstance(w_seq, space.w_unicode)):
-        return _filter_string(space, w_func, w_seq, space.w_unicode)
-    if space.is_true(space.isinstance(w_seq, space.w_tuple)):
-        return _filter_tuple(space, w_func, w_seq)
-    w_iter = space.iter(w_seq)
-    result_w = []
-    none_func = space.is_w(w_func, space.w_None)
-    while True:
-        try:
-            w_next = space.next(w_iter)
-        except OperationError, e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            break
-        if none_func:
-            w_keep = w_next
-        else:
-            w_keep = space.call_function(w_func, w_next)
-        if space.is_true(w_keep):
-            result_w.append(w_next)
-    return space.newlist(result_w)
-
-def _filter_tuple(space, w_func, w_tuple):
-    none_func = space.is_w(w_func, space.w_None)
-    length = space.len_w(w_tuple)
-    result_w = []
-    for i in range(length):
-        w_item = space.getitem(w_tuple, space.wrap(i))
-        if none_func:
-            w_keep = w_item
-        else:
-            w_keep = space.call_function(w_func, w_item)
-        if space.is_true(w_keep):
-            result_w.append(w_item)
-    return space.newtuple(result_w)
-
-def _filter_string(space, w_func, w_string, w_str_type):
-    none_func = space.is_w(w_func, space.w_None)
-    if none_func and space.is_w(space.type(w_string), w_str_type):
-        return w_string
-    length = space.len_w(w_string)
-    result_w = []
-    for i in range(length):
-        w_item = space.getitem(w_string, space.wrap(i))
-        if none_func or space.is_true(space.call_function(w_func, w_item)):
-            if not space.is_true(space.isinstance(w_item, w_str_type)):
-                msg = "__getitem__ returned a non-string type"
-                raise OperationError(space.w_TypeError, space.wrap(msg))
-            result_w.append(w_item)
-    w_empty = space.call_function(w_str_type)
-    return space.call_method(w_empty, "join", space.newlist(result_w))
-
 class W_Enumerate(Wrappable):
 
     def __init__(self, w_iter, w_start):
diff --git a/pypy/module/__builtin__/test/test_functional.py b/pypy/module/__builtin__/test/test_functional.py
--- a/pypy/module/__builtin__/test/test_functional.py
+++ b/pypy/module/__builtin__/test/test_functional.py
@@ -147,7 +147,7 @@
        assert list(xrange(A())) == [0, 1, 2, 3, 4]
        assert list(xrange(0, A())) == [0, 1, 2, 3, 4]
        assert list(xrange(0, 10, A())) == [0, 5]
- 
+
    def test_xrange_float(self):
       assert list(xrange(0.1, 2.0, 1.1)) == [0, 1]
 
diff --git a/pypy/module/__builtin__/test/test_rawinput.py b/pypy/module/__builtin__/test/test_rawinput.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/__builtin__/test/test_rawinput.py
@@ -0,0 +1,77 @@
+import autopath
+
+
+class AppTestRawInput():
+
+    def test_raw_input(self):
+        import sys, StringIO
+        for prompt, expected in [("def:", "abc/ def:/ghi\n"),
+                                 ("", "abc/ /ghi\n"),
+                                 (42, "abc/ 42/ghi\n"),
+                                 (None, "abc/ None/ghi\n"),
+                                 (Ellipsis, "abc/ /ghi\n")]:
+            save = sys.stdin, sys.stdout
+            try:
+                sys.stdin = StringIO.StringIO("foo\nbar\n")
+                out = sys.stdout = StringIO.StringIO()
+                print "abc",    # softspace = 1
+                out.write('/')
+                if prompt is Ellipsis:
+                    got = raw_input()
+                else:
+                    got = raw_input(prompt)
+                out.write('/')
+                print "ghi"
+            finally:
+                sys.stdin, sys.stdout = save
+            assert out.getvalue() == expected
+            assert got == "foo"
+
+    def test_softspace(self):
+        import sys
+        import StringIO
+        fin = StringIO.StringIO()
+        fout = StringIO.StringIO()
+
+        fin.write("Coconuts\n")
+        fin.seek(0)
+
+        sys_stdin_orig = sys.stdin
+        sys_stdout_orig = sys.stdout
+
+        sys.stdin = fin
+        sys.stdout = fout
+
+        print "test",
+        raw_input("test")
+
+        sys.stdin = sys_stdin_orig
+        sys.stdout = sys_stdout_orig
+
+        fout.seek(0)
+        assert fout.read() == "test test"
+
+    def test_softspace_carryover(self):
+        import sys
+        import StringIO
+        fin = StringIO.StringIO()
+        fout = StringIO.StringIO()
+
+        fin.write("Coconuts\n")
+        fin.seek(0)
+
+        sys_stdin_orig = sys.stdin
+        sys_stdout_orig = sys.stdout
+
+        sys.stdin = fin
+        sys.stdout = fout
+
+        print "test",
+        raw_input("test")
+        print "test",
+
+        sys.stdin = sys_stdin_orig
+        sys.stdout = sys_stdout_orig
+
+        fout.seek(0)
+        assert fout.read() == "test testtest"
diff --git a/pypy/module/__pypy__/interp_builders.py b/pypy/module/__pypy__/interp_builders.py
--- a/pypy/module/__pypy__/interp_builders.py
+++ b/pypy/module/__pypy__/interp_builders.py
@@ -16,7 +16,8 @@
 
         def _check_done(self, space):
             if self.builder is None:
-                raise OperationError(space.w_ValueError, space.wrap("Can't operate on a done builder"))
+                raise OperationError(space.w_ValueError, space.wrap(
+                        "Can't operate on a built builder"))
 
         @unwrap_spec(size=int)
         def descr__new__(space, w_subtype, size=-1):
@@ -31,7 +32,8 @@
         def descr_append_slice(self, space, s, start, end):
             self._check_done(space)
             if not 0 <= start <= end <= len(s):
-                raise OperationError(space.w_ValueError, space.wrap("bad start/stop"))
+                raise OperationError(space.w_ValueError, space.wrap(
+                        "bad start/stop"))
             self.builder.append_slice(s, start, end)
 
         def descr_build(self, space):
@@ -42,8 +44,8 @@
 
         def descr_len(self, space):
             if self.builder is None:
-                raise OperationError(space.w_ValueError,
-                                     space.wrap('no lenght of built builder'))
+                raise OperationError(space.w_ValueError, space.wrap(
+                        "no length of built builder"))
             return space.wrap(self.builder.getlength())
 
     W_Builder.__name__ = "W_%s" % name
diff --git a/pypy/module/_continuation/interp_continuation.py b/pypy/module/_continuation/interp_continuation.py
--- a/pypy/module/_continuation/interp_continuation.py
+++ b/pypy/module/_continuation/interp_continuation.py
@@ -19,6 +19,11 @@
         #  - normal:      self.sthread != None, not is_empty_handle(self.h)
         #  - finished:    self.sthread != None, is_empty_handle(self.h)
 
+    def __del__(self):
+        sthread = self.sthread
+        if sthread is not None and not sthread.is_empty_handle(self.h):
+            sthread.destroy(self.h)
+
     def check_sthread(self):
         ec = self.space.getexecutioncontext()
         if ec.stacklet_thread is not self.sthread:
@@ -28,6 +33,8 @@
     def descr_init(self, w_callable, __args__):
         if self.sthread is not None:
             raise geterror(self.space, "continulet already __init__ialized")
+        sthread = build_sthread(self.space)
+        workaround_disable_jit(sthread)
         #
         # hackish: build the frame "by hand", passing it the correct arguments
         space = self.space
@@ -41,7 +48,6 @@
         self.bottomframe = bottomframe
         #
         global_state.origin = self
-        sthread = build_sthread(self.space)
         self.sthread = sthread
         h = sthread.new(new_stacklet_callback)
         post_switch(sthread, h)
@@ -71,6 +77,7 @@
                 global_state.clear()
                 raise geterror(self.space, "continulet already finished")
         self.check_sthread()
+        workaround_disable_jit(self.sthread)
         #
         global_state.origin = self
         if to is None:
@@ -259,6 +266,16 @@
         sthread = ec.stacklet_thread = SThread(space, ec)
     return sthread
 
+def workaround_disable_jit(sthread):
+    # A bad workaround to kill the JIT anywhere in this thread.
+    # This forces all the frames.  It's a bad workaround because
+    # it takes O(depth) time, and it will cause some "abort:
+    # vable escape" in the JIT.  The goal is to prevent any frame
+    # from being still virtuals, because the JIT generates code
+    # to un-virtualizable them "on demand" by loading values based
+    # on FORCE_TOKEN, which is an address in the stack.
+    sthread.ec.force_all_frames()
+
 # ____________________________________________________________
 
 def permute(space, args_w):
diff --git a/pypy/module/_file/test/test_file.py b/pypy/module/_file/test/test_file.py
--- a/pypy/module/_file/test/test_file.py
+++ b/pypy/module/_file/test/test_file.py
@@ -1,5 +1,5 @@
 from __future__ import with_statement
-import py
+import py, os, errno
 
 from pypy.conftest import gettestobjspace, option
 
@@ -257,6 +257,39 @@
             assert self.temppath in g.getvalue()
 
 
+class AppTestNonblocking(object):
+    def setup_class(cls):
+        from pypy.module._file.interp_file import W_File
+
+        cls.old_read = os.read
+
+        if option.runappdirect:
+            py.test.skip("works with internals of _file impl on py.py")
+
+        state = [0]
+        def read(fd, n=None):
+            if fd != 42:
+                return cls.old_read(fd, n)
+            if state[0] == 0:
+                state[0] += 1
+                return "xyz"
+            if state[0] < 3:
+                state[0] += 1
+                raise OSError(errno.EAGAIN, "xyz")
+            return ''
+        os.read = read
+        stdin = W_File(cls.space)
+        stdin.file_fdopen(42, "r", 1)
+        stdin.name = '<stdin>'
+        cls.w_stream = stdin
+
+    def teardown_class(cls):
+        os.read = cls.old_read
+
+    def test_nonblocking_file(self):
+        res = self.stream.read()
+        assert res == 'xyz'
+
 class AppTestConcurrency(object):
     # these tests only really make sense on top of a translated pypy-c,
     # because on top of py.py the inner calls to os.write() don't
@@ -374,24 +407,24 @@
         with self.file(self.temppath, 'w') as f:
             f.write('foo')
         assert f.closed
-        
+
         with self.file(self.temppath, 'r') as f:
             s = f.readline()
 
         assert s == "foo"
         assert f.closed
-    
+
     def test_subclass_with(self):
         file = self.file
         class C(file):
             def __init__(self, *args, **kwargs):
                 self.subclass_closed = False
                 file.__init__(self, *args, **kwargs)
-            
+
             def close(self):
                 self.subclass_closed = True
                 file.close(self)
-        
+
         with C(self.temppath, 'w') as f:
             pass
         assert f.subclass_closed
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -161,9 +161,6 @@
     @binop
     def mul(self, v1, v2):
         return v1 * v2
-    @binop
-    def div(self, v1, v2):
-        return v1 / v2
 
     @unaryop
     def pos(self, v):
@@ -217,6 +214,14 @@
         return float2string(self.for_computation(self.unbox(item)), 'g', rfloat.DTSF_STR_PRECISION)
 
     @binop
+    def div(self, v1, v2):
+        try:
+            return v1 / v2
+        except ZeroDivisionError:
+            if v1 == v2 == 0.0:
+                return rfloat.NAN
+            return rfloat.copysign(rfloat.INFINITY, v1 * v2)
+    @binop
     def mod(self, v1, v2):
         return math.fmod(v1, v2)
     @binop
@@ -295,6 +300,11 @@
         return str(widen(self.unbox(item)))
 
     @binop
+    def div(self, v1, v2):
+        if v2 == 0:
+            return 0
+        return v1 / v2
+    @binop
     def mod(self, v1, v2):
         return v1 % v2
 
@@ -426,23 +436,22 @@
     pass
 
 if LONG_BIT == 32:
-    class W_LongDtype(W_Int32Dtype):
-        pass
+    long_dtype = W_Int32Dtype
+    ulong_dtype = W_UInt32Dtype
+elif LONG_BIT == 64:
+    long_dtype = W_Int64Dtype
+    ulong_dtype = W_UInt64Dtype
+else:
+    assert False
 
-    class W_ULongDtype(W_UInt32Dtype):
-        pass
-else:
-    class W_LongDtype(W_Int64Dtype):
-        pass
+class W_LongDtype(long_dtype):
+    num = 7
+    aliases = ["l"]
+    applevel_types = ["int"]
 
-    class W_ULongDtype(W_UInt64Dtype):
-        pass
-
-W_LongDtype.num = 7
-W_LongDtype.aliases = ["l"]
-W_LongDtype.applevel_types = ["int"]
-W_ULongDtype.num = 8
-W_ULongDtype.aliases = ["L"]
+class W_ULongDtype(ulong_dtype):
+    num = 8
+    aliases = ["L"]
 
 W_Float32Dtype = create_low_level_dtype(
     num = 11, kind = FLOATINGLTR, name = "float32",
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -285,7 +285,9 @@
             assert b[i] == i * 5
 
     def test_div(self):
-        from numpy import array, dtype
+        from math import isnan
+        from numpy import array, dtype, inf
+
         a = array(range(1, 6))
         b = a / a
         for i in range(5):
@@ -297,6 +299,24 @@
         for i in range(5):
             assert b[i] == 1
 
+        a = array([-1, 0, 1])
+        b = array([0, 0, 0])
+        c = a / b
+        assert (c == [0, 0, 0]).all()
+
+        a = array([-1.0, 0.0, 1.0])
+        b = array([0.0, 0.0, 0.0])
+        c = a / b
+        assert c[0] == -inf
+        assert isnan(c[1])
+        assert c[2] == inf
+
+        b = array([-0.0, -0.0, -0.0])
+        c = a / b
+        assert c[0] == inf
+        assert isnan(c[1])
+        assert c[2] == -inf
+
     def test_div_other(self):
         from numpy import array
         a = array(range(5))
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -10,6 +10,7 @@
 from pypy.rpython.lltypesystem import rffi, lltype
 from pypy.rpython.tool import rffi_platform
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
+from pypy.module.sys.interp_encoding import getfilesystemencoding
 
 import os, sys
 _WIN = sys.platform == 'win32'
@@ -32,17 +33,19 @@
             raise OperationError(space.w_OverflowError,
                                  space.wrap("integer out of range"))
 
+def fsencode_w(space, w_obj):
+    if space.isinstance_w(w_obj, space.w_unicode):
+        w_obj = space.call_method(w_obj, 'encode',
+                                  getfilesystemencoding(space))
+    return space.str_w(w_obj)
+
 class FileEncoder(object):
     def __init__(self, space, w_obj):
         self.space = space
         self.w_obj = w_obj
 
     def as_bytes(self):
-        from pypy.module.sys.interp_encoding import getfilesystemencoding
-        space = self.space
-        w_bytes = space.call_method(self.w_obj, 'encode',
-                                    getfilesystemencoding(space))
-        return space.str_w(w_bytes)
+        return fsencode_w(self.space, self.w_obj)
 
     def as_unicode(self):
         return self.space.unicode_w(self.w_obj)
@@ -56,7 +59,6 @@
         return self.space.str_w(self.w_obj)
 
     def as_unicode(self):
-        from pypy.module.sys.interp_encoding import getfilesystemencoding
         space = self.space
         w_unicode = space.call_method(self.w_obj, 'decode',
                                       getfilesystemencoding(space))
@@ -536,7 +538,6 @@
 
 The list is in arbitrary order.  It does not include the special
 entries '.' and '..' even if they are present in the directory."""
-    from pypy.module.sys.interp_encoding import getfilesystemencoding
     try:
         if space.isinstance_w(w_dirname, space.w_unicode):
             dirname = FileEncoder(space, w_dirname)
@@ -734,8 +735,7 @@
 def _exit(space, status):
     os._exit(status)
 
- at unwrap_spec(command=str)
-def execv(space, command, w_args):
+def execv(space, w_command, w_args):
     """ execv(path, args)
 
 Execute an executable path with arguments, replacing current process.
@@ -743,12 +743,13 @@
         path: path of executable file
         args: iterable of strings
     """
+    command = fsencode_w(space, w_command)
     try:
         args_w = space.unpackiterable(w_args)
         if len(args_w) < 1:
             w_msg = space.wrap("execv() must have at least one argument")
             raise OperationError(space.w_ValueError, w_msg)
-        args = [space.str_w(w_arg) for w_arg in args_w]
+        args = [fsencode_w(space, w_arg) for w_arg in args_w]
     except OperationError, e:
         if not e.match(space, space.w_TypeError):
             raise
@@ -759,8 +760,7 @@
     except OSError, e:
         raise wrap_oserror(space, e)
 
- at unwrap_spec(command=str)
-def execve(space, command, w_args, w_env):
+def execve(space, w_command, w_args, w_env):
     """ execve(path, args, env)
 
 Execute a path with arguments and environment, replacing current process.
@@ -769,7 +769,8 @@
         args: iterable of arguments
         env: dictionary of strings mapping to strings
     """
-    args = [space.str_w(w_arg) for w_arg in space.unpackiterable(w_args)]
+    command = fsencode_w(space, w_command)
+    args = [fsencode_w(space, w_arg) for w_arg in space.unpackiterable(w_args)]
     env = {}
     w_keys = space.call_method(w_env, 'keys')
     for w_key in space.unpackiterable(w_keys):
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -415,6 +415,23 @@
                 else:
                     py.test.fail("didn't raise")
 
+        def test_execv_unicode(self):
+            os = self.posix
+            import sys
+            if not hasattr(os, "fork"):
+                skip("Need fork() to test execv()")
+            try:
+                output = u"caf\xe9 \u1234\n".encode(sys.getfilesystemencoding())
+            except UnicodeEncodeError:
+                skip("encoding not good enough")
+            pid = os.fork()
+            if pid == 0:
+                os.execv(u"/bin/sh", ["sh", "-c",
+                                      u"echo caf\xe9 \u1234 > onefile"])
+            os.waitpid(pid, 0)
+            assert open("onefile").read() == output
+            os.unlink("onefile")
+
         def test_execve(self):
             os = self.posix
             if not hasattr(os, "fork"):
@@ -425,6 +442,24 @@
             os.waitpid(pid, 0)
             assert open("onefile").read() == "xxx"
             os.unlink("onefile")
+
+        def test_execve_unicode(self):
+            os = self.posix
+            import sys
+            if not hasattr(os, "fork"):
+                skip("Need fork() to test execve()")
+            try:
+                output = u"caf\xe9 \u1234\n".encode(sys.getfilesystemencoding())
+            except UnicodeEncodeError:
+                skip("encoding not good enough")
+            pid = os.fork()
+            if pid == 0:
+                os.execve(u"/bin/sh", ["sh", "-c",
+                                      u"echo caf\xe9 \u1234 > onefile"],
+                          {'ddd': 'xxx'})
+            os.waitpid(pid, 0)
+            assert open("onefile").read() == output
+            os.unlink("onefile")
         pass # <- please, inspect.getsource(), don't crash
 
     if hasattr(__import__(os.name), "spawnv"):
diff --git a/pypy/module/pyexpat/__init__.py b/pypy/module/pyexpat/__init__.py
--- a/pypy/module/pyexpat/__init__.py
+++ b/pypy/module/pyexpat/__init__.py
@@ -4,12 +4,8 @@
 
 class ErrorsModule(MixedModule):
     "Definition of pyexpat.errors module."
-
-    appleveldefs = {
-        }
-
-    interpleveldefs = {
-        }
+    appleveldefs = {}
+    interpleveldefs = {}
 
     def setup_after_space_initialization(self):
         from pypy.module.pyexpat import interp_pyexpat
@@ -18,6 +14,18 @@
                     interp_pyexpat.ErrorString(self.space,
                     getattr(interp_pyexpat, name)))
 
+class ModelModule(MixedModule):
+    "Definition of pyexpat.model module."
+    appleveldefs = {}
+    interpleveldefs = {}
+
+    def setup_after_space_initialization(self):
+        from pypy.module.pyexpat import interp_pyexpat
+        space = self.space
+        for name in interp_pyexpat.xml_model_list:
+            value = getattr(interp_pyexpat, name)
+            space.setattr(self, space.wrap(name), space.wrap(value))
+
 class Module(MixedModule):
     "Python wrapper for Expat parser."
 
@@ -39,6 +47,7 @@
 
     submodules = {
         'errors': ErrorsModule,
+        'model':  ModelModule,
     }
 
     for name in ['XML_PARAM_ENTITY_PARSING_NEVER',
diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py
--- a/pypy/module/pyexpat/interp_pyexpat.py
+++ b/pypy/module/pyexpat/interp_pyexpat.py
@@ -76,6 +76,18 @@
     "XML_ERROR_FINISHED",
     "XML_ERROR_SUSPEND_PE",
     ]
+xml_model_list = [
+    "XML_CTYPE_EMPTY",
+    "XML_CTYPE_ANY",
+    "XML_CTYPE_MIXED",
+    "XML_CTYPE_NAME",
+    "XML_CTYPE_CHOICE",
+    "XML_CTYPE_SEQ",
+    "XML_CQUANT_NONE",
+    "XML_CQUANT_OPT",
+    "XML_CQUANT_REP",
+    "XML_CQUANT_PLUS",
+    ]
 
 class CConfigure:
     _compilation_info_ = eci
@@ -104,6 +116,8 @@
 
     for name in xml_error_list:
         locals()[name] = rffi_platform.ConstantInteger(name)
+    for name in xml_model_list:
+        locals()[name] = rffi_platform.ConstantInteger(name)
 
 for k, v in rffi_platform.configure(CConfigure).items():
     globals()[k] = v
diff --git a/pypy/module/pyexpat/test/test_parser.py b/pypy/module/pyexpat/test/test_parser.py
--- a/pypy/module/pyexpat/test/test_parser.py
+++ b/pypy/module/pyexpat/test/test_parser.py
@@ -131,3 +131,7 @@
                'encoding specified in XML declaration is incorrect')
         assert (pyexpat.errors.XML_ERROR_XML_DECL ==
                 'XML declaration not well-formed')
+
+    def test_model(self):
+        import pyexpat
+        assert isinstance(pyexpat.model.XML_CTYPE_EMPTY, int)
diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py
--- a/pypy/module/pypyjit/interp_jit.py
+++ b/pypy/module/pypyjit/interp_jit.py
@@ -137,20 +137,15 @@
 
     def jump_absolute(self, jumpto, _, ec=None):
         if we_are_jitted():
-            # Normally, the tick counter is decremented by 100 for every
-            # Python opcode.  Here, to better support JIT compilation of
-            # small loops, we decrement it by a possibly smaller constant.
-            # We get the maximum 100 when the (unoptimized) trace length
-            # is at least 3200 (a bit randomly).
-            trace_length = r_uint(current_trace_length())
-            decr_by = trace_length // 32
-            if decr_by < 1:
-                decr_by = 1
-            elif decr_by > 100:    # also if current_trace_length() returned -1
-                decr_by = 100
+            #
+            # assume that only threads are using the bytecode counter
+            decr_by = 0
+            if self.space.actionflag.has_bytecode_counter:   # constant-folded
+                if self.space.threadlocals.gil_ready:   # quasi-immutable field
+                    decr_by = _get_adapted_tick_counter()
             #
             self.last_instr = intmask(jumpto)
-            ec.bytecode_trace(self, intmask(decr_by))
+            ec.bytecode_trace(self, decr_by)
             jumpto = r_uint(self.last_instr)
         #
         pypyjitdriver.can_enter_jit(frame=self, ec=ec, next_instr=jumpto,
@@ -158,6 +153,20 @@
                                     is_being_profiled=self.is_being_profiled)
         return jumpto
 
+def _get_adapted_tick_counter():
+    # Normally, the tick counter is decremented by 100 for every
+    # Python opcode.  Here, to better support JIT compilation of
+    # small loops, we decrement it by a possibly smaller constant.
+    # We get the maximum 100 when the (unoptimized) trace length
+    # is at least 3200 (a bit randomly).
+    trace_length = r_uint(current_trace_length())
+    decr_by = trace_length // 32
+    if decr_by < 1:
+        decr_by = 1
+    elif decr_by > 100:    # also if current_trace_length() returned -1
+        decr_by = 100
+    return intmask(decr_by)
+
 
 PyCode__initialize = PyCode._initialize
 
diff --git a/pypy/module/pypyjit/test_pypy_c/model.py b/pypy/module/pypyjit/test_pypy_c/model.py
--- a/pypy/module/pypyjit/test_pypy_c/model.py
+++ b/pypy/module/pypyjit/test_pypy_c/model.py
@@ -225,6 +225,8 @@
         # strip comment
         if '#' in line:
             line = line[:line.index('#')]
+        if line.strip() == 'guard_not_invalidated?':
+            return 'guard_not_invalidated', None, [], '...', False
         # find the resvar, if any
         if ' = ' in line:
             resvar, _, line = line.partition(' = ')
@@ -249,7 +251,7 @@
             descr = descr[len('descr='):]
         else:
             descr = None
-        return opname, resvar, args, descr
+        return opname, resvar, args, descr, True
 
     @classmethod
     def preprocess_expected_src(cls, src):
@@ -258,13 +260,23 @@
         # replaced with the corresponding operations, so that tests don't have
         # to repeat it every time
         ticker_check = """
+            guard_not_invalidated?
+            ticker0 = getfield_raw(ticker_address, descr=<SignedFieldDescr pypysig_long_struct.c_value .*>)
+            ticker_cond0 = int_lt(ticker0, 0)
+            guard_false(ticker_cond0, descr=...)
+        """
+        src = src.replace('--TICK--', ticker_check)
+        #
+        # this is the ticker check generated if we have threads
+        thread_ticker_check = """
+            guard_not_invalidated?
             ticker0 = getfield_raw(ticker_address, descr=<SignedFieldDescr pypysig_long_struct.c_value .*>)
             ticker1 = int_sub(ticker0, 1)
             setfield_raw(ticker_address, ticker1, descr=<SignedFieldDescr pypysig_long_struct.c_value .*>)
             ticker_cond0 = int_lt(ticker1, 0)
             guard_false(ticker_cond0, descr=...)
         """
-        src = src.replace('--TICK--', ticker_check)
+        src = src.replace('--THREAD-TICK--', thread_ticker_check)
         #
         # this is the ticker check generated in PyFrame.handle_operation_error
         exc_ticker_check = """
@@ -298,7 +310,7 @@
         if not cond:
             raise InvalidMatch(message, frame=sys._getframe(1))
 
-    def match_op(self, op, (exp_opname, exp_res, exp_args, exp_descr)):
+    def match_op(self, op, (exp_opname, exp_res, exp_args, exp_descr, _)):
         self._assert(op.name == exp_opname, "operation mismatch")
         self.match_var(op.res, exp_res)
         if exp_args != ['...']:
@@ -341,7 +353,7 @@
         what is after the '...'
         """
         iter_exp_ops = iter(expected_ops)
-        iter_ops = iter(self.ops)
+        iter_ops = RevertableIterator(self.ops)
         for opindex, exp_op in enumerate(iter_exp_ops):
             try:
                 if exp_op == '...':
@@ -360,6 +372,9 @@
                             break
                 self.match_op(op, exp_op)
             except InvalidMatch, e:
+                if exp_op[4] is False:    # optional operation
+                    iter_ops.revert_one()
+                    continue       # try to match with the next exp_op
                 e.opindex = opindex
                 raise
         #
@@ -372,8 +387,8 @@
                 return ''
             text = str(py.code.Source(src).deindent().indent())
             lines = text.splitlines(True)
-            if opindex is not None and 0 <= opindex < len(lines):
-                lines[opindex] = lines[opindex].rstrip() + '\t<=====\n'
+            if opindex is not None and 0 <= opindex <= len(lines):
+                lines.insert(opindex, '\n\t===== HERE =====\n')
             return ''.join(lines)
         #
         expected_src = self.preprocess_expected_src(expected_src)
@@ -398,3 +413,18 @@
         else:
             return True
 
+
+class RevertableIterator(object):
+    def __init__(self, sequence):
+        self.sequence = sequence
+        self.index = 0
+    def __iter__(self):
+        return self
+    def next(self):
+        index = self.index
+        if index == len(self.sequence):
+            raise StopIteration
+        self.index = index + 1
+        return self.sequence[index]
+    def revert_one(self):
+        self.index -= 1
diff --git a/pypy/module/pypyjit/test_pypy_c/test_00_model.py b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
--- a/pypy/module/pypyjit/test_pypy_c/test_00_model.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
@@ -145,15 +145,17 @@
 
     def test_parse_op(self):
         res = OpMatcher.parse_op("  a =   int_add(  b,  3 ) # foo")
-        assert res == ("int_add", "a", ["b", "3"], None)
+        assert res == ("int_add", "a", ["b", "3"], None, True)
         res = OpMatcher.parse_op("guard_true(a)")
-        assert res == ("guard_true", None, ["a"], None)
+        assert res == ("guard_true", None, ["a"], None, True)
         res = OpMatcher.parse_op("setfield_gc(p0, i0, descr=<foobar>)")
-        assert res == ("setfield_gc", None, ["p0", "i0"], "<foobar>")
+        assert res == ("setfield_gc", None, ["p0", "i0"], "<foobar>", True)
         res = OpMatcher.parse_op("i1 = getfield_gc(p0, descr=<foobar>)")
-        assert res == ("getfield_gc", "i1", ["p0"], "<foobar>")
+        assert res == ("getfield_gc", "i1", ["p0"], "<foobar>", True)
         res = OpMatcher.parse_op("p0 = force_token()")
-        assert res == ("force_token", "p0", [], None)
+        assert res == ("force_token", "p0", [], None, True)
+        res = OpMatcher.parse_op("guard_not_invalidated?")
+        assert res == ("guard_not_invalidated", None, [], '...', False)
 
     def test_exact_match(self):
         loop = """
@@ -341,7 +343,7 @@
             # this is the actual loop
             'int_lt', 'guard_true', 'int_add',
             # this is the signal checking stuff
-            'getfield_raw', 'int_sub', 'setfield_raw', 'int_lt', 'guard_false',
+            'guard_not_invalidated', 'getfield_raw', 'int_lt', 'guard_false',
             'jump'
             ]
 
@@ -407,7 +409,7 @@
             # this is the actual loop
             'int_lt', 'guard_true', 'force_token', 'int_add',
             # this is the signal checking stuff
-            'getfield_raw', 'int_sub', 'setfield_raw', 'int_lt', 'guard_false',
+            'guard_not_invalidated', 'getfield_raw', 'int_lt', 'guard_false',
             'jump'
             ]
 
@@ -425,10 +427,9 @@
             guard_true(i6, descr=...)
             i8 = int_add(i4, 1)
             # signal checking stuff
+            guard_not_invalidated(descr=...)
             i10 = getfield_raw(37212896, descr=<.* pypysig_long_struct.c_value .*>)
-            i12 = int_sub(i10, 1)
-            setfield_raw(37212896, i12, descr=<.* pypysig_long_struct.c_value .*>)
-            i14 = int_lt(i12, 0)
+            i14 = int_lt(i10, 0)
             guard_false(i14, descr=...)
             jump(p0, p1, p2, p3, i8, descr=...)
         """)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_misc.py b/pypy/module/pypyjit/test_pypy_c/test_misc.py
--- a/pypy/module/pypyjit/test_pypy_c/test_misc.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_misc.py
@@ -329,4 +329,20 @@
             guard_false(i28, descr=...)
             i30 = int_lshift(i20, 24)
             i31 = int_or(i26, i30)
-        """ % {"32_bit_only": extra})
\ No newline at end of file
+        """ % {"32_bit_only": extra})
+
+    def test_eval(self):
+        def main():
+            i = 1
+            a = compile('x+x+x+x+x+x', 'eval', 'eval')
+            b = {'x': 7}
+            while i < 1000:
+                y = eval(a,b,b)  # ID: eval
+                i += 1
+            return y
+
+        log = self.run(main)
+        assert log.result == 42
+        # the following assertion fails if the loop was cancelled due
+        # to "abort: vable escape"
+        assert len(log.loops_by_id("eval")) == 1
diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -41,7 +41,7 @@
             guard_true(i32, descr=...)
             i34 = int_add(i6, 1)
             --TICK--
-            jump(p0, p1, p2, p3, p4, p5, i34, p7, p8, i9, i10, p11, i12, p13, descr=<Loop4>)
+            jump(p0, p1, p2, p3, p4, p5, i34, p7, p8, i9, i10, p11, i12, p13, descr=...)
         """)
 
     def test_long(self):
@@ -106,7 +106,7 @@
             i58 = int_add_ovf(i6, i57)
             guard_no_overflow(descr=...)
             --TICK--
-            jump(p0, p1, p2, p3, p4, p5, i58, i7, descr=<Loop4>)
+            jump(p0, p1, p2, p3, p4, p5, i58, i7, descr=...)
         """)
 
     def test_str_mod(self):
@@ -156,4 +156,41 @@
             i40 = int_sub(i4, 1)
             --TICK--
             jump(p0, p1, p2, p3, i40, i38, descr=<Loop0>)
-        """)
\ No newline at end of file
+        """)
+
+    def test_getattr_promote(self):
+        def main(n):
+            class A(object):
+                def meth_a(self):
+                    return 1
+                def meth_b(self):
+                    return 2
+            a = A()
+
+            l = ['a', 'b']
+            s = 0
+            for i in range(n):
+                name = 'meth_' + l[i & 1]
+                meth = getattr(a, name) # ID: getattr
+                s += meth()
+            return s
+
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loops = log.loops_by_filename(self.filepath)
+        assert len(loops) == 2
+        for loop in loops:
+            loop.match_by_id('getattr','''
+            guard_not_invalidated(descr=...)
+            i32 = strlen(p31)
+            i34 = int_add(5, i32)
+            p35 = newstr(i34)
+            strsetitem(p35, 0, 109)
+            strsetitem(p35, 1, 101)
+            strsetitem(p35, 2, 116)
+            strsetitem(p35, 3, 104)
+            strsetitem(p35, 4, 95)
+            copystrcontent(p31, p35, 0, 5, i32)
+            i49 = call(ConstClass(_ll_2_str_eq_nonnull__rpy_stringPtr_rpy_stringPtr), p35, ConstPtr(ptr48), descr=<SignedCallDescr>)
+            guard_value(i49, 1, descr=<Guard8>)
+            ''')
diff --git a/pypy/module/pypyjit/test_pypy_c/test_thread.py b/pypy/module/pypyjit/test_pypy_c/test_thread.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/pypyjit/test_pypy_c/test_thread.py
@@ -0,0 +1,28 @@
+from pypy.module.pypyjit.test_pypy_c.test_00_model import BaseTestPyPyC
+
+
+class TestThread(BaseTestPyPyC):
+    def test_simple(self):
+        def main(n):
+            import thread
+            def f():
+                i = 0
+                while i < n:
+                    i += 1
+                done.release()
+
+            done = thread.allocate_lock()
+            done.acquire()
+            thread.start_new_thread(f, ())
+            done.acquire()
+            return 0
+        log = self.run(main, [500])
+        assert round(log.result, 6) == round(main(500), 6)
+        loop, = log.loops_by_filename(self.filepath)
+        assert loop.match("""
+            i2 = int_lt(i0, i1)
+            guard_true(i2, descr=...)
+            i3 = int_add(i0, 1)
+            --THREAD-TICK--
+            jump(..., descr=<Loop0>)
+        """)
diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py
--- a/pypy/module/signal/interp_signal.py
+++ b/pypy/module/signal/interp_signal.py
@@ -109,8 +109,11 @@
         p = pypysig_getaddr_occurred()
         value = p.c_value
         if self.has_bytecode_counter:    # this 'if' is constant-folded
-            value -= by
-            p.c_value = value
+            if jit.isconstant(by) and by == 0:
+                pass     # normally constant-folded too
+            else:
+                value -= by
+                p.c_value = value
         return value
 
 
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -47,7 +47,7 @@
         'pypy_initial_path'     : 'state.pypy_initial_path',
 
         '_getframe'             : 'vm._getframe', 
-        '_current_frames'       : 'vm._current_frames', 
+        '_current_frames'       : 'currentframes._current_frames', 
         'setrecursionlimit'     : 'vm.setrecursionlimit', 
         'getrecursionlimit'     : 'vm.getrecursionlimit', 
         'setcheckinterval'      : 'vm.setcheckinterval', 
diff --git a/pypy/module/sys/currentframes.py b/pypy/module/sys/currentframes.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/sys/currentframes.py
@@ -0,0 +1,78 @@
+"""
+Implementation of the 'sys._current_frames()' routine.
+"""
+from pypy.interpreter import gateway
+
+app = gateway.applevel('''
+"NOT_RPYTHON"
+import __builtin__
+
+class fake_code(object):
+    co_name = "?"
+    co_filename = "?"
+    co_firstlineno = 0
+
+class fake_frame(object):
+    f_back = None
+    f_builtins = __builtin__.__dict__
+    f_code = fake_code()
+    f_exc_traceback = None
+    f_exc_type = None
+    f_exc_value = None
+    f_globals = {}
+    f_lasti = -1
+    f_lineno = 0
+    f_locals = {}
+    f_restricted = False
+    f_trace = None
+
+    def __init__(self, f):
+        if f is not None:
+            for name in ["f_builtins", "f_code", "f_globals", "f_lasti",
+                         "f_lineno"]:
+                setattr(self, name, getattr(f, name))
+''')
+
+def _current_frames(space):
+    """_current_frames() -> dictionary
+
+    Return a dictionary mapping each current thread T's thread id to T's
+    current stack "frame".  Functions in the traceback module can build the
+    call stack given such a frame.
+
+    Note that in PyPy this returns fake frame objects, to avoid a runtime
+    penalty everywhere with the JIT.  (So far these fake frames can be
+    completely uninformative depending on the JIT state; we could return
+    more with more efforts.)
+
+    This function should be used for specialized purposes only."""
+    w_result = space.newdict()
+    w_fake_frame = app.wget(space, "fake_frame")
+    w_fake_code  = app.wget(space, "fake_code")
+    ecs = space.threadlocals.getallvalues()
+    for thread_ident, ec in ecs.items():
+        vref = ec.topframeref
+        frames = []
+        while not vref.virtual:
+            f = vref()
+            if f is None:
+                break
+            frames.append(f)
+            vref = f.f_backref
+        else:
+            frames.append(None)
+        #
+        w_topframe = space.wrap(None)
+        w_prevframe = None
+        for f in frames:
+            w_nextframe = space.call_function(w_fake_frame, space.wrap(f))
+            if w_prevframe is None:
+                w_topframe = w_nextframe
+            else:
+                space.setattr(w_prevframe, space.wrap('f_back'), w_nextframe)
+            w_prevframe = w_nextframe
+        #
+        space.setitem(w_result,
+                      space.wrap(thread_ident),
+                      w_topframe)
+    return w_result
diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -556,7 +556,7 @@
             return sys._current_frames()
         frames = f()
         assert frames.keys() == [0]
-        assert frames[0].f_code.co_name == 'f'
+        assert frames[0].f_code.co_name in ('f', '?')
 
 class AppTestCurrentFramesWithThread(AppTestCurrentFrames):
     def setup_class(cls):
@@ -568,23 +568,25 @@
         import thread
 
         thread_id = thread.get_ident()
-        self.ready = False
         def other_thread():
-            self.ready = True
             print "thread started"
-            time.sleep(5)
+            lock2.release()
+            lock1.acquire()
+        lock1 = thread.allocate_lock()
+        lock2 = thread.allocate_lock()
+        lock1.acquire()
+        lock2.acquire()
         thread.start_new_thread(other_thread, ())
 
         def f():
-            for i in range(100):
-                if self.ready: break
-                time.sleep(0.1)
+            lock2.acquire()
             return sys._current_frames()
 
         frames = f()
+        lock1.release()
         thisframe = frames.pop(thread_id)
-        assert thisframe.f_code.co_name == 'f'
+        assert thisframe.f_code.co_name in ('f', '?')
 
         assert len(frames) == 1
         _, other_frame = frames.popitem()
-        assert other_frame.f_code.co_name == 'other_thread'
+        assert other_frame.f_code.co_name in ('other_thread', '?')
diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -45,25 +45,6 @@
     f.mark_as_escaped()
     return space.wrap(f)
 
-def _current_frames(space):
-    """_current_frames() -> dictionary
-
-    Return a dictionary mapping each current thread T's thread id to T's
-    current stack frame.
-
-    This function should be used for specialized purposes only."""
-    raise OperationError(space.w_NotImplementedError,
-        space.wrap("XXX sys._current_frames() incompatible with the JIT"))
-    w_result = space.newdict()
-    ecs = space.threadlocals.getallvalues()
-    for thread_ident, ec in ecs.items():
-        f = ec.gettopframe_nohidden()
-        f.mark_as_escaped()
-        space.setitem(w_result,
-                      space.wrap(thread_ident),
-                      space.wrap(f))
-    return w_result
-
 def setrecursionlimit(space, w_new_limit):
     """setrecursionlimit() sets the maximum number of nested calls that
 can occur before a RuntimeError is raised.  On PyPy the limit is
diff --git a/pypy/module/thread/gil.py b/pypy/module/thread/gil.py
--- a/pypy/module/thread/gil.py
+++ b/pypy/module/thread/gil.py
@@ -17,6 +17,7 @@
 class GILThreadLocals(OSThreadLocals):
     """A version of OSThreadLocals that enforces a GIL."""
     gil_ready = False
+    _immutable_fields_ = ['gil_ready?']
 
     def initialize(self, space):
         # add the GIL-releasing callback as an action on the space
diff --git a/pypy/module/thread/threadlocals.py b/pypy/module/thread/threadlocals.py
--- a/pypy/module/thread/threadlocals.py
+++ b/pypy/module/thread/threadlocals.py
@@ -8,9 +8,14 @@
 
     def __init__(self):
         self._valuedict = {}   # {thread_ident: ExecutionContext()}
+        self._freeze_()
+
+    def _freeze_(self):
+        self._valuedict.clear()
         self._mainthreadident = 0
         self._mostrecentkey = 0        # fast minicaching for the common case
         self._mostrecentvalue = None   # fast minicaching for the common case
+        return False
 
     def getvalue(self):
         ident = thread.get_ident()
diff --git a/pypy/objspace/std/celldict.py b/pypy/objspace/std/celldict.py
--- a/pypy/objspace/std/celldict.py
+++ b/pypy/objspace/std/celldict.py
@@ -37,10 +37,10 @@
         self.version = VersionTag()
 
     def get_empty_storage(self):
-       return self.erase({})
+        return self.erase({})
 
     def mutated(self):
-       self.version = VersionTag()
+        self.version = VersionTag()
 
     def getdictvalue_no_unwrapping(self, w_dict, key):
         # NB: it's important to promote self here, so that self.version is a
diff --git a/pypy/objspace/std/objecttype.py b/pypy/objspace/std/objecttype.py
--- a/pypy/objspace/std/objecttype.py
+++ b/pypy/objspace/std/objecttype.py
@@ -44,7 +44,6 @@
         raise OperationError(space.w_TypeError,
                              space.wrap("__class__ assignment: only for heap types"))
     w_oldcls = space.type(w_obj)
-    # XXX taint space should raise a TaintError here if w_oldcls is tainted
     assert isinstance(w_oldcls, W_TypeObject)
     if w_oldcls.get_full_instance_layout() == w_newcls.get_full_instance_layout():
         w_obj.setclass(space, w_newcls)
diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py
--- a/pypy/objspace/std/stringobject.py
+++ b/pypy/objspace/std/stringobject.py
@@ -161,57 +161,59 @@
 
 def str_swapcase__String(space, w_self):
     self = w_self._value
-    res = [' '] * len(self)
+    builder = StringBuilder(len(self))
     for i in range(len(self)):
         ch = self[i]
         if ch.isupper():
             o = ord(ch) + 32
-            res[i] = chr(o)
+            builder.append(chr(o))
         elif ch.islower():
             o = ord(ch) - 32
-            res[i] = chr(o)
+            builder.append(chr(o))
         else:
-            res[i] = ch
+            builder.append(ch)
 
-    return space.wrap("".join(res))
+    return space.wrap(builder.build())
 
 
 def str_capitalize__String(space, w_self):
     input = w_self._value
-    buffer = [' '] * len(input)
+    builder = StringBuilder(len(input))
     if len(input) > 0:
         ch = input[0]
         if ch.islower():
             o = ord(ch) - 32
-            buffer[0] = chr(o)
+            builder.append(chr(o))
         else:
-            buffer[0] = ch
+            builder.append(ch)
 
         for i in range(1, len(input)):
             ch = input[i]
             if ch.isupper():
                 o = ord(ch) + 32
-                buffer[i] = chr(o)
+                builder.append(chr(o))
             else:
-                buffer[i] = ch
+                builder.append(ch)
 
-    return space.wrap("".join(buffer))
+    return space.wrap(builder.build())
 
 def str_title__String(space, w_self):
     input = w_self._value
-    buffer = [' '] * len(input)
+    builder = StringBuilder(len(input))
     prev_letter=' '
 
-    for pos in range(0, len(input)):
+    for pos in range(len(input)):
         ch = input[pos]
         if not prev_letter.isalpha():
-            buffer[pos] = _upper(ch)
+            ch = _upper(ch)
+            builder.append(ch)
         else:
-            buffer[pos] = _lower(ch)
+            ch = _lower(ch)
+            builder.append(ch)
 
-        prev_letter = buffer[pos]
+        prev_letter = ch
 
-    return space.wrap("".join(buffer))
+    return space.wrap(builder.build())
 
 def str_split__String_None_ANY(space, w_self, w_none, w_maxsplit=-1):
     maxsplit = space.int_w(w_maxsplit)
@@ -754,27 +756,21 @@
     input = w_self._value
     width = space.int_w(w_width)
 
-    if len(input) >= width:
+    num_zeros = width - len(input)
+    if num_zeros <= 0:
         # cannot return w_self, in case it is a subclass of str
         return space.wrap(input)
 
-    buf = [' '] * width
+    builder = StringBuilder(width)
     if len(input) > 0 and (input[0] == '+' or input[0] == '-'):
-        buf[0] = input[0]
+        builder.append(input[0])
         start = 1
-        middle = width - len(input) + 1
     else:
         start = 0
-        middle = width - len(input)
 
-    for i in range(start, middle):
-        buf[i] = '0'
-
-    for i in range(middle, width):
-        buf[i] = input[start]
-        start = start + 1
-
-    return space.wrap("".join(buf))
+    builder.append_multiple_char('0', num_zeros)
+    builder.append_slice(input, start, len(input))
+    return space.wrap(builder.build())
 
 
 def hash__String(space, w_str):
diff --git a/pypy/objspace/std/strutil.py b/pypy/objspace/std/strutil.py
--- a/pypy/objspace/std/strutil.py
+++ b/pypy/objspace/std/strutil.py
@@ -35,7 +35,7 @@
 
     def error(self):
         raise ParseStringError("invalid literal for %s() with base %d: '%s'" %
-                               (self.fname, self.base, self.literal))
+                               (self.fname, self.original_base, self.literal))
 
     def __init__(self, s, literal, base, fname):
         self.literal = literal
@@ -47,7 +47,8 @@
         elif s.startswith('+'):
             s = strip_spaces(s[1:])
         self.sign = sign
-        
+        self.original_base = base
+
         if base == 0:
             if s.startswith('0x') or s.startswith('0X'):
                 base = 16
diff --git a/pypy/objspace/std/test/test_strutil.py b/pypy/objspace/std/test/test_strutil.py
--- a/pypy/objspace/std/test/test_strutil.py
+++ b/pypy/objspace/std/test/test_strutil.py
@@ -89,6 +89,8 @@
 
         exc = raises(ParseStringError, string_to_int, '')
         assert exc.value.msg == "invalid literal for int() with base 10: ''"
+        exc = raises(ParseStringError, string_to_int, '', 0)
+        assert exc.value.msg == "invalid literal for int() with base 0: ''"
 
     def test_string_to_int_overflow(self):
         import sys
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -10,7 +10,8 @@
 from pypy.objspace.std import identitydict
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.rlib.objectmodel import current_object_addr_as_int, compute_hash
-from pypy.rlib.jit import promote, elidable_promote, we_are_jitted
+from pypy.rlib.jit import promote, elidable_promote, we_are_jitted,\
+     promote_string
 from pypy.rlib.jit import elidable, dont_look_inside, unroll_safe
 from pypy.rlib.rarithmetic import intmask, r_uint
 
@@ -101,6 +102,7 @@
                           'instancetypedef',
                           'terminator',
                           '_version_tag?',
+                          'interplevel_cls',
                           ]
 
     # for config.objspace.std.getattributeshortcut
@@ -399,6 +401,7 @@
         if version_tag is None:
             tup = w_self._lookup_where(name)
             return tup
+        name = promote_string(name)
         w_class, w_value = w_self._pure_lookup_where_with_method_cache(name, version_tag)
         return w_class, unwrap_cell(space, w_value)
 
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -417,54 +417,54 @@
     input = w_self._value
     if len(input) == 0:
         return W_UnicodeObject.EMPTY
-    result = [u'\0'] * len(input)
-    result[0] = unichr(unicodedb.toupper(ord(input[0])))
+    builder = UnicodeBuilder(len(input))
+    builder.append(unichr(unicodedb.toupper(ord(input[0]))))
     for i in range(1, len(input)):
-        result[i] = unichr(unicodedb.tolower(ord(input[i])))
-    return W_UnicodeObject(u''.join(result))
+        builder.append(unichr(unicodedb.tolower(ord(input[i]))))
+    return W_UnicodeObject(builder.build())
 
 def unicode_title__Unicode(space, w_self):
     input = w_self._value
     if len(input) == 0:
         return w_self
-    result = [u'\0'] * len(input)
+    builder = UnicodeBuilder(len(input))
 
     previous_is_cased = False
     for i in range(len(input)):
         unichar = ord(input[i])
         if previous_is_cased:
-            result[i] = unichr(unicodedb.tolower(unichar))
+            builder.append(unichr(unicodedb.tolower(unichar)))
         else:
-            result[i] = unichr(unicodedb.totitle(unichar))
+            builder.append(unichr(unicodedb.totitle(unichar)))
         previous_is_cased = unicodedb.iscased(unichar)
-    return W_UnicodeObject(u''.join(result))
+    return W_UnicodeObject(builder.build())
 
 def unicode_lower__Unicode(space, w_self):
     input = w_self._value
-    result = [u'\0'] * len(input)
+    builder = UnicodeBuilder(len(input))
     for i in range(len(input)):
-        result[i] = unichr(unicodedb.tolower(ord(input[i])))
-    return W_UnicodeObject(u''.join(result))
+        builder.append(unichr(unicodedb.tolower(ord(input[i]))))
+    return W_UnicodeObject(builder.build())
 
 def unicode_upper__Unicode(space, w_self):
     input = w_self._value
-    result = [u'\0'] * len(input)
+    builder = UnicodeBuilder(len(input))
     for i in range(len(input)):
-        result[i] = unichr(unicodedb.toupper(ord(input[i])))
-    return W_UnicodeObject(u''.join(result))
+        builder.append(unichr(unicodedb.toupper(ord(input[i]))))
+    return W_UnicodeObject(builder.build())
 
 def unicode_swapcase__Unicode(space, w_self):
     input = w_self._value
-    result = [u'\0'] * len(input)
+    builder = UnicodeBuilder(len(input))
     for i in range(len(input)):
         unichar = ord(input[i])
         if unicodedb.islower(unichar):
-            result[i] = unichr(unicodedb.toupper(unichar))
+            builder.append(unichr(unicodedb.toupper(unichar)))
         elif unicodedb.isupper(unichar):
-            result[i] = unichr(unicodedb.tolower(unichar))
+            builder.append(unichr(unicodedb.tolower(unichar)))
         else:
-            result[i] = input[i]
-    return W_UnicodeObject(u''.join(result))
+            builder.append(input[i])
+    return W_UnicodeObject(builder.build())
 
 def _normalize_index(length, index):
     if index < 0:
diff --git a/pypy/objspace/taint.py b/pypy/objspace/taint.py
deleted file mode 100644
--- a/pypy/objspace/taint.py
+++ /dev/null
@@ -1,294 +0,0 @@
-"""
-Just an experiment.
-"""
-import os
-from pypy.objspace.std.objspace import StdObjSpace
-from pypy.objspace.proxy import patch_space_in_place
-from pypy.objspace.thunk import nb_forcing_args
-from pypy.interpreter.error import OperationError
-from pypy.interpreter import baseobjspace, gateway, executioncontext
-from pypy.interpreter.function import Method
-from pypy.interpreter.pyframe import PyFrame
-from pypy.tool.sourcetools import func_with_new_name
-from pypy.rlib.unroll import unrolling_iterable
-
-
-class W_Tainted(baseobjspace.W_Root):
-    def __init__(self, w_obj):
-        self.w_obj = w_obj
-
-##    def getdict(self, space):
-##        return taint(self.w_obj.getdict(space))
-
-##    def getdictvalue(self, space, attr):
-##        return taint(self.w_obj.getdictvalue(space, attr))
-
-##    def setdictvalue(self, space, attr, w_value):
-##        return self.w_obj.setdictvalue(space, attr, w_value)
-
-##    ...
-
-class W_TaintBomb(baseobjspace.W_Root):
-    filename = '?'
-    codename = '?'
-    codeline = 0
-
-    def __init__(self, space, operr):
-        self.space = space
-        self.operr = operr
-        self.record_debug_info()
-
-    def record_debug_info(self):
-        ec = self.space.getexecutioncontext()
-        frame = ec.gettopframe_nohidden()
-        if isinstance(frame, PyFrame):     # and, in particular, frame != None
-            self.filename = frame.pycode.co_filename
-            self.codename = frame.pycode.co_name
-            self.codeline = frame.get_last_lineno()
-        if get_debug_level(self.space) > 0:
-            self.debug_dump()
-
-    def debug_dump(self):
-        os.write(2, 'Taint Bomb from file "%s", line %d, in %s\n    %s\n' % (
-            self.filename, self.codeline, self.codename,
-            self.operr.errorstr(self.space)))
-
-    def explode(self):
-        #msg = self.operr.errorstr(space)
-        raise OperationError(self.space.w_TaintError, self.space.w_None)
-
-
-def taint(w_obj):
-    """Return a tainted version of the argument."""
-    if w_obj is None or isinstance(w_obj, W_Tainted):
-        return w_obj
-    else:
-        return W_Tainted(w_obj)
-app_taint = gateway.interp2app(taint)
-
-def is_tainted(space, w_obj):
-    """Return whether the argument is tainted."""
-    res = isinstance(w_obj, W_Tainted) or isinstance(w_obj, W_TaintBomb)
-    return space.wrap(res)
-app_is_tainted = gateway.interp2app(is_tainted)
-
-def untaint(space, w_expectedtype, w_obj):
-    """untaint(expectedtype, tainted_obj) -> obj
-Untaint untainted_obj and return it. If the result is not of expectedtype,
-raise a type error."""
-    if (isinstance(w_expectedtype, W_Tainted) or
-        isinstance(w_expectedtype, W_TaintBomb)):
-        raise OperationError(space.w_TypeError,
-                  space.wrap("untaint() arg 1 must be an untainted type"))
-    if not space.is_true(space.isinstance(w_expectedtype, space.w_type)):
-        raise OperationError(space.w_TypeError,
-                             space.wrap("untaint() arg 1 must be a type"))
-    if isinstance(w_obj, W_Tainted):
-        w_obj = w_obj.w_obj
-    elif isinstance(w_obj, W_TaintBomb):
-        w_obj.explode()
-    #if isinstance(w_expectedtype, W_Tainted):
-    #    w_expectedtype = w_expectedtype.w_obj
-    w_realtype = space.type(w_obj)
-    if not space.is_w(w_realtype, w_expectedtype):
-        #msg = "expected an object of type '%s'" % (
-        #    w_expectedtype.getname(space),)
-        #    #w_realtype.getname(space))
-        raise OperationError(space.w_TaintError, space.w_None)
-    return w_obj
-app_untaint = gateway.interp2app(untaint)
-
-# ____________________________________________________________
-
- at gateway.unwrap_spec(args_w='args_w')
-def taint_atomic_function(space, w_func, args_w):
-    newargs_w = []
-    tainted = False
-    for w_arg in args_w:
-        if isinstance(w_arg, W_Tainted):
-            tainted = True
-            w_arg = w_arg.w_obj
-        elif isinstance(w_arg, W_TaintBomb):
-            return w_arg
-        newargs_w.append(w_arg)
-    w_newargs = space.newtuple(newargs_w)
-    try:
-        w_res = space.call(w_func, w_newargs)
-    except OperationError, operr:
-        if not tainted:
-            raise
-        return W_TaintBomb(space, operr)
-    if tainted:
-        w_res = taint(w_res)
-    return w_res
-
-app_taint_atomic_function = gateway.interp2app(taint_atomic_function)
-
-def taint_atomic(space, w_callable):
-    """decorator to make a callable "taint-atomic": if the function is called
-with tainted arguments, those are untainted. The result of the function is
-tainted again.  All exceptions that the callable raises are turned into
-taint bombs."""
-    meth = Method(space, space.w_fn_taint_atomic_function,
-                  w_callable, space.type(w_callable))
-    return space.wrap(meth)
-app_taint_atomic = gateway.interp2app(taint_atomic)
-
-# ____________________________________________________________
-
-executioncontext.ExecutionContext.taint_debug = 0
-
- at gateway.unwrap_spec(level=int)
-def taint_debug(space, level):
-    """Set the debug level. If the debug level is greater than 0, the creation
-of taint bombs will print debug information. For debugging purposes
-only!"""
-    space.getexecutioncontext().taint_debug = level
-app_taint_debug = gateway.interp2app(taint_debug)
-
-def taint_look(space, w_obj):
-    """Print some info about the taintedness of an object. For debugging
-purposes only!"""
-    if isinstance(w_obj, W_Tainted):
-        info = space.type(w_obj.w_obj).getname(space)
-        msg = space.str_w(w_obj.w_obj.getrepr(space, info))
-        msg = 'Taint Box %s\n' % msg
-        os.write(2, msg)
-    elif isinstance(w_obj, W_TaintBomb):
-        w_obj.debug_dump()
-    else:
-        os.write(2, 'not tainted\n')
-app_taint_look = gateway.interp2app(taint_look)
-
-def get_debug_level(space):
-    return space.getexecutioncontext().taint_debug
-
-def debug_bomb(space, operr):
-    ec = space.getexecutioncontext()
-    filename = '?'
-    codename = '?'
-    codeline = 0
-    frame = ec.gettopframe_nohidden()
-    if isinstance(frame, PyFrame):     # and, in particular, frame != None
-        filename = frame.pycode.co_filename
-        codename = frame.pycode.co_name
-        codeline = frame.get_last_lineno()
-    os.write(2, 'Taint Bomb in file "%s", line %d, in %s\n    %s\n' % (
-        filename, codeline, codename, operr.errorstr(space)))
-
-# ____________________________________________________________
-
-
-class TaintSpace(StdObjSpace):
-
-    def __init__(self, *args, **kwds):
-        StdObjSpace.__init__(self, *args, **kwds)
-        w_dict = self.newdict()
-        self.setitem(w_dict, self.wrap("__doc__"), self.wrap("""\
-Exception that is raised when an operation revealing information on a tainted
-object is performed."""))
-        self.w_TaintError = self.call_function(
-            self.w_type,
-            self.wrap("TaintError"),
-            self.newtuple([self.w_Exception]),
-            w_dict
-            )
-        w___pypy__ = self.getbuiltinmodule("__pypy__")
-        self.setattr(w___pypy__, self.wrap('taint'),
-                     self.wrap(app_taint))
-        self.setattr(w___pypy__, self.wrap('is_tainted'),
-                     self.wrap(app_is_tainted))
-        self.setattr(w___pypy__, self.wrap('untaint'),
-                     self.wrap(app_untaint))
-        self.w_fn_taint_atomic_function = self.wrap(app_taint_atomic_function)
-        self.setattr(w___pypy__, self.wrap('taint_atomic'),
-                     self.wrap(app_taint_atomic))
-        self.setattr(w___pypy__, self.wrap('TaintError'),
-                     self.w_TaintError)
-        self.setattr(w___pypy__, self.wrap('_taint_debug'),
-                     self.wrap(app_taint_debug))
-        self.setattr(w___pypy__, self.wrap('_taint_look'),
-                     self.wrap(app_taint_look))
-        patch_space_in_place(self, 'taint', proxymaker)
-
-        # XXX may leak info, perfomance hit, what about taint bombs?
-        from pypy.objspace.std.typeobject import W_TypeObject
-
-        def taint_lookup(w_obj, name):
-            if isinstance(w_obj, W_Tainted):
-                w_obj = w_obj.w_obj
-            w_type = self.type(w_obj)
-            assert isinstance(w_type, W_TypeObject)
-            return w_type.lookup(name)
-
-        def taint_lookup_in_type_where(w_obj, name):
-            if isinstance(w_obj, W_Tainted):
-                w_type = w_obj.w_obj
-            else:
-                w_type = w_obj
-            assert isinstance(w_type, W_TypeObject)
-            return w_type.lookup_where(name)
-
-        self.lookup = taint_lookup
-        self.lookup_in_type_where = taint_lookup_in_type_where
-
-
-Space = TaintSpace
-
-
-def tainted_error(space, name):
-    #msg = "operation '%s' forbidden on tainted object" % (name,)
-    raise OperationError(space.w_TaintError, space.w_None)# space.wrap(msg))
-
-
-RegularMethods = dict.fromkeys(
-    [name for name, _, _, _ in baseobjspace.ObjSpace.MethodTable])
-
-TaintResultIrregularMethods = dict.fromkeys(
-    ['wrap', 'call_args'] +
-    [name for name in baseobjspace.ObjSpace.IrregularOpTable
-          if name.startswith('new')])
-
-def proxymaker(space, name, parentfn):
-    arity = nb_forcing_args[name]
-    indices = unrolling_iterable(range(arity))
-    if name in RegularMethods:
-
-        def proxy(*args_w):
-            newargs_w = ()
-            tainted = False
-            for i in indices:
-                w_arg = args_w[i]
-                if isinstance(w_arg, W_Tainted):
-                    tainted = True
-                    w_arg = w_arg.w_obj
-                elif isinstance(w_arg, W_TaintBomb):
-                    return w_arg
-                newargs_w += (w_arg,)
-            newargs_w += args_w[arity:]
-            try:
-                w_res = parentfn(*newargs_w)
-            except OperationError, operr:
-                if not tainted:
-                    raise
-                return W_TaintBomb(space, operr)
-            if tainted:
-                w_res = taint(w_res)
-            return w_res
-
-    elif arity == 0:
-        return None
-
-    else:
-
-        def proxy(*args_w):
-            for i in indices:
-                w_arg = args_w[i]
-                if isinstance(w_arg, W_Tainted):
-                    tainted_error(space, name)
-                elif isinstance(w_arg, W_TaintBomb):
-                    w_arg.explode()
-            return parentfn(*args_w)
-
-    proxy = func_with_new_name(proxy, '%s_proxy' % name)
-    return proxy
diff --git a/pypy/objspace/test/test_taintobjspace.py b/pypy/objspace/test/test_taintobjspace.py
deleted file mode 100644
--- a/pypy/objspace/test/test_taintobjspace.py
+++ /dev/null
@@ -1,77 +0,0 @@
-from pypy.conftest import gettestobjspace
-
-class AppTest_Taint:
-
-    def setup_class(cls):
-        cls.space = gettestobjspace('taint')
-
-    def test_simple(self):
-        from __pypy__ import taint, untaint, TaintError
-        x = taint(6)
-        x = x * 7
-        raises(TaintError, "if x: y = 1")
-        t = type(x)
-        raises(TaintError, "if t is int: y = 1")
-        assert untaint(int, x) == 42
-        raises(TaintError, "untaint(float, x)")
-
-    def test_bomb(self):
-        from __pypy__ import taint, untaint, TaintError
-        x = taint(6)
-        x = x / 0
-        raises(TaintError, "if x: y = 1")
-        t = type(x)
-        raises(TaintError, "if t is int: y = 1")
-        raises(TaintError, "untaint(int, x)")
-        raises(TaintError, "untaint(float, x)")
-
-    def test_taint_atomic(self):
-        from __pypy__ import taint, untaint, TaintError, taint_atomic
-        x = taint(6)
-        x *= 7
-
-        def dummy(x):
-            if x > 40:
-                return 5
-            else:
-                return 3
-        dummy = taint_atomic(dummy)
-
-        y = dummy(x)
-        raises(TaintError, "if y == 3: z = 1")
-        assert untaint(int, y) == 5
-
-    def test_taint_atomic_exception(self):
-        from __pypy__ import taint, untaint, TaintError, taint_atomic
-        x = taint(6)
-        x *= 7
-
-        def dummy(x):
-            if x + "world" == "hello world":
-                return 5
-            else:
-                return 3
-        dummy = taint_atomic(dummy)
-
-        y = dummy(x)
-        raises(TaintError, "if y == 3: z = 1")
-        raises(TaintError, "untaint(int, y)")
-
-    def test_taint_atomic_incoming_bomb(self):
-        from __pypy__ import taint, untaint, TaintError, taint_atomic
-        x = taint(6)
-        x /= 0
-        lst = []
-
-        def dummy(x):
-            lst.append("running!")
-            if x > 40:
-                return 5
-            else:
-                return 3
-        dummy = taint_atomic(dummy)
-
-        y = dummy(x)
-        raises(TaintError, "if y == 3: z = 1")
-        assert lst == []
-        raises(TaintError, "untaint(int, y)")
diff --git a/pypy/rlib/_jit_vref.py b/pypy/rlib/_jit_vref.py
--- a/pypy/rlib/_jit_vref.py
+++ b/pypy/rlib/_jit_vref.py
@@ -25,6 +25,10 @@
     def simple_call(self):
         return self.s_instance
 
+    def getattr(self, s_attr):
+        assert s_attr.const == 'virtual'
+        return annmodel.s_Bool
+
     def rtyper_makerepr(self, rtyper):
         if rtyper.type_system.name == 'lltypesystem':
             return vrefrepr
@@ -61,6 +65,13 @@
                              " prebuilt virtual_ref")
         return lltype.nullptr(OBJECTPTR.TO)
 
+    def rtype_getattr(self, hop):
+        s_attr = hop.args_s[1]
+        assert s_attr.const == 'virtual'
+        v = hop.inputarg(self, arg=0)
+        hop.exception_cannot_occur()
+        return hop.genop('jit_is_virtual', [v], resulttype = lltype.Bool)
+
 from pypy.rpython.ootypesystem.rclass import OBJECT
 
 class OOVRefRepr(VRefRepr):
diff --git a/pypy/rlib/_rsocket_rffi.py b/pypy/rlib/_rsocket_rffi.py
--- a/pypy/rlib/_rsocket_rffi.py
+++ b/pypy/rlib/_rsocket_rffi.py
@@ -16,6 +16,7 @@
 _MINGW = target_platform.name == "mingw32"
 _SOLARIS = sys.platform == "sunos5"
 _MACOSX = sys.platform == "darwin"
+_HAS_AF_PACKET = sys.platform.startswith('linux')   # only Linux for now
 
 if _POSIX:
     includes = ('sys/types.h',
@@ -34,11 +35,12 @@
                 'stdint.h', 
                 'errno.h',
                 )
+    if _HAS_AF_PACKET:
+        includes += ('netpacket/packet.h',
+                     'sys/ioctl.h',
+                     'net/if.h')
 
-    cond_includes = [('AF_NETLINK', 'linux/netlink.h'),
-                     ('AF_PACKET', 'netpacket/packet.h'),
-                     ('AF_PACKET', 'sys/ioctl.h'),
-                     ('AF_PACKET', 'net/if.h')]
+    cond_includes = [('AF_NETLINK', 'linux/netlink.h')]
     
     libraries = ()
     calling_conv = 'c'
@@ -320,18 +322,18 @@
                                              ('events', rffi.SHORT),
                                              ('revents', rffi.SHORT)])
 
-    CConfig.sockaddr_ll = platform.Struct('struct sockaddr_ll',
+    if _HAS_AF_PACKET:
+        CConfig.sockaddr_ll = platform.Struct('struct sockaddr_ll',
                               [('sll_ifindex', rffi.INT),
                                ('sll_protocol', rffi.INT),
                                ('sll_pkttype', rffi.INT),
                                ('sll_hatype', rffi.INT),
                                ('sll_addr', rffi.CFixedArray(rffi.CHAR, 8)),
-                               ('sll_halen', rffi.INT)],
-                              ifdef='AF_PACKET')
+                               ('sll_halen', rffi.INT)])
 
-    CConfig.ifreq = platform.Struct('struct ifreq', [('ifr_ifindex', rffi.INT),
-                                 ('ifr_name', rffi.CFixedArray(rffi.CHAR, 8))],
-                                    ifdef='AF_PACKET')
+        CConfig.ifreq = platform.Struct('struct ifreq',
+                                [('ifr_ifindex', rffi.INT),
+                                 ('ifr_name', rffi.CFixedArray(rffi.CHAR, 8))])
 
 if _WIN32:
     CConfig.WSAEVENT = platform.SimpleType('WSAEVENT', rffi.VOIDP)
@@ -386,6 +388,8 @@
         constants[name] = value
     else:
         constants[name] = default
+if not _HAS_AF_PACKET and 'AF_PACKET' in constants:
+    del constants['AF_PACKET']
 
 constants['has_ipv6'] = True # This is a configuration option in CPython
 for name, value in constants.items():
@@ -439,21 +443,14 @@
 if _POSIX:
     nfds_t = cConfig.nfds_t
     pollfd = cConfig.pollfd
-    if cConfig.sockaddr_ll is not None:
+    if _HAS_AF_PACKET:
         sockaddr_ll = cConfig.sockaddr_ll
-    ifreq = cConfig.ifreq
+        ifreq = cConfig.ifreq
 if WIN32:
     WSAEVENT = cConfig.WSAEVENT
     WSANETWORKEVENTS = cConfig.WSANETWORKEVENTS
 timeval = cConfig.timeval
 
-#if _POSIX:
-#    includes = list(includes)
-#    for _name, _header in cond_includes:
-#        if getattr(cConfig, _name) is not None:
-#            includes.append(_header)
-#    eci = ExternalCompilationInfo(includes=includes, libraries=libraries,
-#                                  separate_module_sources=sources)
 
 def external(name, args, result, **kwds):
     return rffi.llexternal(name, args, result, compilation_info=eci,
@@ -544,7 +541,7 @@
     socketpair_t = rffi.CArray(socketfd_type)
     socketpair = external('socketpair', [rffi.INT, rffi.INT, rffi.INT,
                           lltype.Ptr(socketpair_t)], rffi.INT)
-    if ifreq is not None:
+    if _HAS_AF_PACKET:
         ioctl = external('ioctl', [socketfd_type, rffi.INT, lltype.Ptr(ifreq)],
                          rffi.INT)
 
diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -8,6 +8,8 @@
 from pypy.rpython.extregistry import ExtRegistryEntry
 from pypy.tool.sourcetools import func_with_new_name
 
+DEBUG_ELIDABLE_FUNCTIONS = False
+
 
 def elidable(func):
     """ Decorate a function as "trace-elidable". This means precisely that:
@@ -24,6 +26,18 @@
     If a particular call to this function ends up raising an exception, then it
     is handled like a normal function call (this decorator is ignored).
     """
+    if DEBUG_ELIDABLE_FUNCTIONS:
+        cache = {}
+        oldfunc = func
+        def func(*args):
+            result = oldfunc(*args)    # if it raises, no caching
+            try:
+                oldresult = cache.setdefault(args, result)
+            except TypeError:
+                pass           # unhashable args
+            else:
+                assert oldresult == result
+            return result
     func._elidable_function_ = True
     return func
 
@@ -38,6 +52,7 @@
     possible arguments are:
 
     * promote - promote the argument from a variable into a constant
+    * promote_string - same, but promote string by *value*
     * access_directly - directly access a virtualizable, as a structure
                         and don't treat it as a virtualizable
     * fresh_virtualizable - means that virtualizable was just allocated.
@@ -51,6 +66,9 @@
 def promote(x):
     return hint(x, promote=True)
 
+def promote_string(x):
+    return hint(x, promote_string=True)
+
 def dont_look_inside(func):
     """ Make sure the JIT does not trace inside decorated function
     (it becomes a call instead)
@@ -313,6 +331,12 @@
             raise InvalidVirtualRef
         return self._x
 
+    @property
+    def virtual(self):
+        """A property that is True if the vref contains a virtual that would
+        be forced by the '()' operator."""
+        return self._state == 'non-forced'
+
     def _finish(self):
         if self._state == 'non-forced':
             self._state = 'invalid'
diff --git a/pypy/rlib/rerased.py b/pypy/rlib/rerased.py
--- a/pypy/rlib/rerased.py
+++ b/pypy/rlib/rerased.py
@@ -218,15 +218,13 @@
         [v_value] = hop.inputargs(lltype.Signed)
         c_one = hop.inputconst(lltype.Signed, 1)
         hop.exception_is_here()
-        v2 = hop.genop('int_lshift_ovf', [v_value, c_one],
+        v2 = hop.genop('int_add_ovf', [v_value, v_value],
                        resulttype = lltype.Signed)
         v2p1 = hop.genop('int_add', [v2, c_one],
                          resulttype = lltype.Signed)
         v_instance = hop.genop('cast_int_to_ptr', [v2p1],
                                resulttype=self.lowleveltype)
-        v = hop.genop('cast_opaque_ptr', [v_instance],
-                      resulttype=self.lowleveltype)
-        return v
+        return v_instance
 
     def convert_const(self, value):
         if value._identity is _identity_for_ints:
@@ -266,10 +264,10 @@
         return hop.genop('int_rshift', [v2, c_one], resulttype=lltype.Signed)
 
     def rtype_erase_int(self, hop):
-        hop.exception_is_here()
         [v_value] = hop.inputargs(lltype.Signed)
         c_one = hop.inputconst(lltype.Signed, 1)
-        v2 = hop.genop('int_lshift_ovf', [v_value, c_one],
+        hop.exception_is_here()
+        v2 = hop.genop('int_add_ovf', [v_value, v_value],
                        resulttype = lltype.Signed)
         v2p1 = hop.genop('int_add', [v2, c_one],
                          resulttype = lltype.Signed)
diff --git a/pypy/rlib/rsocket.py b/pypy/rlib/rsocket.py
--- a/pypy/rlib/rsocket.py
+++ b/pypy/rlib/rsocket.py
@@ -8,6 +8,7 @@
 # Known missing features:
 #
 #   - address families other than AF_INET, AF_INET6, AF_UNIX, AF_PACKET
+#   - AF_PACKET is only supported on Linux
 #   - methods makefile(),
 #   - SSL
 #
diff --git a/pypy/rlib/streamio.py b/pypy/rlib/streamio.py
--- a/pypy/rlib/streamio.py
+++ b/pypy/rlib/streamio.py
@@ -37,7 +37,7 @@
 # return value of tell(), but not as argument to read().
 #
 
-import os, sys
+import os, sys, errno
 from pypy.rlib.objectmodel import specialize, we_are_translated
 from pypy.rlib.rarithmetic import r_longlong, intmask
 from pypy.rlib import rposix
@@ -587,12 +587,22 @@
     def readall(self):
         pos = self.pos
         assert pos >= 0
-        chunks = [self.buf[pos:]]
+        if self.buf:
+            chunks = [self.buf[pos:]]
+        else:
+            chunks = []
         self.buf = ""
         self.pos = 0
         bufsize = self.bufsize
         while 1:
-            data = self.do_read(bufsize)
+            try:
+                data = self.do_read(bufsize)
+            except OSError, o:
+                if o.errno != errno.EAGAIN:
+                    raise
+                if not chunks:
+                    raise
+                break
             if not data:
                 break
             chunks.append(data)
diff --git a/pypy/rlib/test/test__jit_vref.py b/pypy/rlib/test/test__jit_vref.py
--- a/pypy/rlib/test/test__jit_vref.py
+++ b/pypy/rlib/test/test__jit_vref.py
@@ -27,10 +27,13 @@
     x1 = X()
     vref = virtual_ref(x1)
     assert vref._state == 'non-forced'
+    assert vref.virtual is True
     assert vref() is x1
     assert vref._state == 'forced'
+    assert vref.virtual is False
     virtual_ref_finish(vref, x1)
     assert vref._state == 'forced'
+    assert vref.virtual is False
     assert vref() is x1
 
 def test_direct_invalid():
@@ -135,6 +138,13 @@
         x = self.interpret(f, [])
         assert x == 42
 
+    def test_rtype_virtualattr(self):
+        def f():
+            vref = virtual_ref(X())
+            return vref.virtual
+        x = self.interpret(f, [])
+        assert x is False
+
 
 class TestLLtype(BaseTestVRef, LLRtypeMixin):
     OBJECTTYPE = OBJECTPTR
diff --git a/pypy/rlib/test/test_jit.py b/pypy/rlib/test/test_jit.py
--- a/pypy/rlib/test/test_jit.py
+++ b/pypy/rlib/test/test_jit.py
@@ -139,12 +139,11 @@
 
     def test_isconstant(self):
         def f(n):
-            assert n >= 0
             assert isconstant(n) is False
             l = []
             l.append(n)
             return len(l)
-        res = self.interpret(f, [234])
+        res = self.interpret(f, [-234])
         assert res == 1
 
 
diff --git a/pypy/rpython/lltypesystem/ll2ctypes.py b/pypy/rpython/lltypesystem/ll2ctypes.py
--- a/pypy/rpython/lltypesystem/ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/ll2ctypes.py
@@ -653,6 +653,8 @@
         if T == llmemory.GCREF:
             if isinstance(llobj._obj, _llgcopaque):
                 return ctypes.c_void_p(llobj._obj.intval)
+            if isinstance(llobj._obj, int):    # tagged pointer
+                return ctypes.c_void_p(llobj._obj)
             container = llobj._obj.container
             T = lltype.Ptr(lltype.typeOf(container))
             # otherwise it came from integer and we want a c_void_p with
@@ -1263,6 +1265,7 @@
 class _llgcopaque(lltype._container):
     _TYPE = llmemory.GCREF.TO
     _name = "_llgcopaque"
+    _read_directly_intval = True     # for _ptr._cast_to_int()
 
     def __init__(self, void_p):
         if isinstance(void_p, (int, long)):
@@ -1283,6 +1286,9 @@
             return False
         return force_cast(lltype.Signed, other._as_ptr()) == self.intval
 
+    def __hash__(self):
+        return self.intval
+
     def __ne__(self, other):
         return not self == other
 
@@ -1291,11 +1297,6 @@
             return _opaque_objs[self.intval // 2]
         return force_cast(PTRTYPE, self.intval)
 
-##     def _cast_to_int(self):
-##         return self.intval
-
-##     def _cast_to_adr(self):
-##         return _lladdress(self.intval)
 
 def cast_adr_to_int(addr):
     if isinstance(addr, llmemory.fakeaddress):
diff --git a/pypy/rpython/lltypesystem/llmemory.py b/pypy/rpython/lltypesystem/llmemory.py
--- a/pypy/rpython/lltypesystem/llmemory.py
+++ b/pypy/rpython/lltypesystem/llmemory.py
@@ -509,6 +509,8 @@
 
     def _cast_to_int(self, symbolic=False):
         if self:
+            if isinstance(self.ptr._obj0, int):    # tagged integer
+                return self.ptr._obj0
             if symbolic:
                 return AddressAsInt(self)
             else:
diff --git a/pypy/rpython/lltypesystem/lloperation.py b/pypy/rpython/lltypesystem/lloperation.py
--- a/pypy/rpython/lltypesystem/lloperation.py
+++ b/pypy/rpython/lltypesystem/lloperation.py
@@ -431,6 +431,7 @@
     'jit_marker':           LLOp(),
     'jit_force_virtualizable':LLOp(canrun=True),
     'jit_force_virtual':    LLOp(canrun=True),
+    'jit_is_virtual':       LLOp(canrun=True),
     'jit_force_quasi_immutable': LLOp(canrun=True),
     'get_exception_addr':   LLOp(),
     'get_exc_value_addr':   LLOp(),
diff --git a/pypy/rpython/lltypesystem/lltype.py b/pypy/rpython/lltypesystem/lltype.py
--- a/pypy/rpython/lltypesystem/lltype.py
+++ b/pypy/rpython/lltypesystem/lltype.py
@@ -1363,6 +1363,8 @@
         obj = normalizeptr(self, check)._getobj(check)
         if isinstance(obj, int):
             return obj     # special case for cast_int_to_ptr() results put into opaques
+        if getattr(obj, '_read_directly_intval', False):
+            return obj.intval   # special case for _llgcopaque
         result = intmask(obj._getid())
         # assume that id() returns an addressish value which is
         # not zero and aligned to at least a multiple of 4
diff --git a/pypy/rpython/lltypesystem/opimpl.py b/pypy/rpython/lltypesystem/opimpl.py
--- a/pypy/rpython/lltypesystem/opimpl.py
+++ b/pypy/rpython/lltypesystem/opimpl.py
@@ -549,6 +549,9 @@
 def op_jit_force_virtual(x):
     return x
 
+def op_jit_is_virtual(x):
+    return False
+
 def op_jit_force_quasi_immutable(*args):
     pass
 
diff --git a/pypy/rpython/lltypesystem/rbuilder.py b/pypy/rpython/lltypesystem/rbuilder.py
--- a/pypy/rpython/lltypesystem/rbuilder.py
+++ b/pypy/rpython/lltypesystem/rbuilder.py
@@ -29,7 +29,7 @@
         except OverflowError:
             raise MemoryError
         newbuf = mallocfn(new_allocated)
-        copycontentsfn(ll_builder.buf, newbuf, 0, 0, ll_builder.allocated)
+        copycontentsfn(ll_builder.buf, newbuf, 0, 0, ll_builder.used)
         ll_builder.buf = newbuf
         ll_builder.allocated = new_allocated
     return func_with_new_name(stringbuilder_grow, name)
@@ -56,7 +56,7 @@
 class BaseStringBuilderRepr(AbstractStringBuilderRepr):
     def empty(self):
         return nullptr(self.lowleveltype.TO)
-    
+
     @classmethod
     def ll_new(cls, init_size):
         if init_size < 0 or init_size > MAX:
diff --git a/pypy/rpython/lltypesystem/rtagged.py b/pypy/rpython/lltypesystem/rtagged.py
--- a/pypy/rpython/lltypesystem/rtagged.py
+++ b/pypy/rpython/lltypesystem/rtagged.py
@@ -43,7 +43,7 @@
         v_value = hop.inputarg(lltype.Signed, arg=1)
         c_one = hop.inputconst(lltype.Signed, 1)
         hop.exception_is_here()
-        v2 = hop.genop('int_lshift_ovf', [v_value, c_one],
+        v2 = hop.genop('int_add_ovf', [v_value, v_value],
                        resulttype = lltype.Signed)
         v2p1 = hop.genop('int_add', [v2, c_one],
                          resulttype = lltype.Signed)
diff --git a/pypy/rpython/lltypesystem/test/test_ll2ctypes.py b/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
--- a/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
@@ -1123,6 +1123,9 @@
 
         #assert lltype.cast_ptr_to_int(ref1) == intval
 
+        x = rffi.cast(llmemory.GCREF, -17)
+        assert lltype.cast_ptr_to_int(x) == -17
+
     def test_ptr_truth(self):
         abc = rffi.cast(lltype.Ptr(lltype.FuncType([], lltype.Void)), 0)
         assert not abc
diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -1292,10 +1292,12 @@
         # if a prebuilt GcStruct contains a pointer to a young object,
         # then the write_barrier must have ensured that the prebuilt
         # GcStruct is in the list self.old_objects_pointing_to_young.
+        debug_start("gc-minor-walkroots")
         self.root_walker.walk_roots(
             MiniMarkGC._trace_drag_out1,  # stack roots
             MiniMarkGC._trace_drag_out1,  # static in prebuilt non-gc
             None)                         # static in prebuilt gc
+        debug_stop("gc-minor-walkroots")
 
     def collect_cardrefs_to_nursery(self):
         size_gc_header = self.gcheaderbuilder.size_gc_header
diff --git a/pypy/rpython/rclass.py b/pypy/rpython/rclass.py
--- a/pypy/rpython/rclass.py
+++ b/pypy/rpython/rclass.py
@@ -191,6 +191,10 @@
                     "class %r inherits from its parent _immutable_=True, "
                     "so it should also declare _immutable_=True" % (
                     self.classdef,))
+            if loc.classdict.get('_immutable_').value is not True:
+                raise TyperError(
+                    "class %r: _immutable_ = something else than True" % (
+                    self.classdef,))
             hints = hints.copy()
             hints['immutable'] = True
         self.immutable_field_set = set()  # unless overwritten below
diff --git a/pypy/tool/logparser.py b/pypy/tool/logparser.py
--- a/pypy/tool/logparser.py
+++ b/pypy/tool/logparser.py
@@ -298,6 +298,8 @@
         image.paste(textpercent, (t1x, 5), textpercent)
         image.paste(textlabel,   (t2x, 5), textlabel)
         images.append(image)
+    if not images:
+        return None
     return combine(images, spacing=0, border=1, horizontal=False)
 
 def get_timesummary_single_image(totaltimes, totaltime0, componentdict,
@@ -333,6 +335,8 @@
         del totaltimes[None]
     img2 = render_histogram(totaltimes, totaltime0, {},
                             width, summarybarheight)
+    if img2 is None:
+        return img1
     return combine([img1, img2], spacing=spacing, horizontal=True)
 
 # ----------
diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -43,34 +43,29 @@
 def package(basedir, name='pypy-nightly', rename_pypy_c='pypy',
             copy_to_dir = None, override_pypy_c = None):
     basedir = py.path.local(basedir)
+    if override_pypy_c is None:
+        basename = 'pypy-c'
+        if sys.platform == 'win32':
+            basename += '.exe'
+        pypy_c = basedir.join('pypy', 'translator', 'goal', basename)
+    else:
+        pypy_c = py.path.local(override_pypy_c)
+    if not pypy_c.check():
+        print pypy_c
+        raise PyPyCNotFound('Please compile pypy first, using translate.py')
+    binaries = [(pypy_c, rename_pypy_c)]
+    #
     if sys.platform == 'win32':
-        # Can't rename a DLL
-        if override_pypy_c is not None:
-            rename_pypy_c = py.path.local(override_pypy_c).purebasename
-            pypy_c_dir = py.path.local(override_pypy_c).dirname
-        else:
-            pypy_c_dir = basedir.join('pypy', 'translator', 'goal')
-        pypy_c = pypy_c_dir.join(rename_pypy_c + '.exe')
-        libpypy_c = pypy_c_dir.join('lib' + rename_pypy_c + '.dll')
-        binaries = [(pypy_c, pypy_c.basename),
-                    (libpypy_c, libpypy_c.basename)]
-        for extra in ['libexpat.dll', 'sqlite3.dll', 'msvcr90.dll']:
-            p = pypy_c_dir.join(extra)
+        # Can't rename a DLL: it is always called 'libpypy-c.dll'
+        for extra in ['libpypy-c.dll',
+                      'libexpat.dll', 'sqlite3.dll', 'msvcr90.dll']:
+            p = pypy_c.dirpath().join(extra)
             if not p.check():
                 p = py.path.local.sysfind(extra)
                 assert p, "%s not found" % (extra,)
             print "Picking %s" % p
             binaries.append((p, p.basename))
-    else:
-        basename = 'pypy-c'
-        if override_pypy_c is None:
-            pypy_c = basedir.join('pypy', 'translator', 'goal', basename)
-        else:
-            pypy_c = py.path.local(override_pypy_c)
-        binaries = [(pypy_c, rename_pypy_c)]
-    if not pypy_c.check():
-        print pypy_c
-        raise PyPyCNotFound('Please compile pypy first, using translate.py')
+    #
     builddir = udir.ensure("build", dir=True)
     pypydir = builddir.ensure(name, dir=True)
     # Careful: to copy lib_pypy, copying just the svn-tracked files
diff --git a/pypy/translator/c/funcgen.py b/pypy/translator/c/funcgen.py
--- a/pypy/translator/c/funcgen.py
+++ b/pypy/translator/c/funcgen.py
@@ -836,7 +836,7 @@
         return 'INSTRUMENT_COUNT(%s);' % counter_label
             
     def OP_IS_EARLY_CONSTANT(self, op):
-        return self.expr(op.result)  + ' = 0;' # Allways false
+        return '%s = 0; /* IS_EARLY_CONSTANT */' % (self.expr(op.result),)
 
     def OP_JIT_MARKER(self, op):
         return '/* JIT_MARKER %s */' % op
@@ -848,6 +848,9 @@
         return '%s = %s; /* JIT_FORCE_VIRTUAL */' % (self.expr(op.result),
                                                      self.expr(op.args[0]))
 
+    def OP_JIT_IS_VIRTUAL(self, op):
+        return '%s = 0; /* JIT_IS_VIRTUAL */' % (self.expr(op.result),)
+
     def OP_JIT_FORCE_QUASI_IMMUTABLE(self, op):
         return '/* JIT_FORCE_QUASI_IMMUTABLE %s */' % op
 
diff --git a/pypy/translator/c/genc.py b/pypy/translator/c/genc.py
--- a/pypy/translator/c/genc.py
+++ b/pypy/translator/c/genc.py
@@ -8,7 +8,7 @@
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
 from pypy.rpython.lltypesystem import lltype
 from pypy.tool.udir import udir
-from pypy.tool import isolate
+from pypy.tool import isolate, runsubprocess
 from pypy.translator.c.support import log, c_string_constant
 from pypy.rpython.typesystem import getfunctionptr
 from pypy.translator.c import gc
@@ -588,14 +588,19 @@
             else:
                 mk.definition('PYPY_MAIN_FUNCTION', "main")
 
-            if (py.path.local.sysfind('python') or
-                py.path.local.sysfind('python.exe')):
-                python = 'python '
-            elif sys.platform == 'win32':
+            if sys.platform == 'win32':
                 python = sys.executable.replace('\\', '/') + ' '
             else:
                 python = sys.executable + ' '
 
+            # Is there a command 'python' that runs python 2.5-2.7?
+            # If there is, then we can use it instead of sys.executable
+            returncode, stdout, stderr = runsubprocess.run_subprocess(
+                "python", "-V")
+            if (stdout.startswith('Python 2.') or
+                stderr.startswith('Python 2.')):
+                python = 'python '
+
             if self.translator.platform.name == 'msvc':
                 lblofiles = []
                 for cfile in mk.cfiles:
diff --git a/pypy/translator/c/src/thread_nt.h b/pypy/translator/c/src/thread_nt.h
--- a/pypy/translator/c/src/thread_nt.h
+++ b/pypy/translator/c/src/thread_nt.h
@@ -245,7 +245,7 @@
     if (pending_acquires <= 0)
         return 0;
     InterlockedIncrement(&pending_acquires);
-    PulseEvent(&cond_gil);
+    PulseEvent(cond_gil);
 
     /* hack: the three following lines do a pthread_cond_wait(), and
        normally specifying a timeout of INFINITE would be fine.  But the
@@ -253,7 +253,7 @@
        (small) risk that PulseEvent misses the WaitForSingleObject().
        In this case the process will just sleep a few milliseconds. */
     LeaveCriticalSection(&mutex_gil);
-    WaitForSingleObject(&cond_gil, 15);
+    WaitForSingleObject(cond_gil, 15);
     EnterCriticalSection(&mutex_gil);
 
     InterlockedDecrement(&pending_acquires);
@@ -263,7 +263,7 @@
 void RPyGilRelease(void)
 {
     LeaveCriticalSection(&mutex_gil);
-    PulseEvent(&cond_gil);
+    PulseEvent(cond_gil);
 }
 
 void RPyGilAcquire(void)
diff --git a/pypy/translator/goal/app_main.py b/pypy/translator/goal/app_main.py
--- a/pypy/translator/goal/app_main.py
+++ b/pypy/translator/goal/app_main.py
@@ -144,7 +144,7 @@
     print '  --jit off                  turn off the JIT'
 
 def print_version(*args):
-    print "Python", sys.version
+    print >> sys.stderr, "Python", sys.version
     raise SystemExit
 
 def set_jit_option(options, jitparam, *args):
diff --git a/pypy/translator/platform/linux.py b/pypy/translator/platform/linux.py
--- a/pypy/translator/platform/linux.py
+++ b/pypy/translator/platform/linux.py
@@ -1,5 +1,6 @@
 """Support for Linux."""
 
+import sys
 from pypy.translator.platform.posix import BasePosix
 
 class BaseLinux(BasePosix):
@@ -24,15 +25,18 @@
         return self._pkg_config("libffi", "--libs-only-L",
                                 ['/usr/lib/libffi'])
 
+    def library_dirs_for_libffi_a(self):
+        # places where we need to look for libffi.a
+        # XXX obscuuure!  only look for libffi.a if run with translate.py
+        if 'translate' in sys.modules:
+            return self.library_dirs_for_libffi() + ['/usr/lib']
+        else:
+            return []
+
 
 class Linux(BaseLinux):
     shared_only = ()    # it seems that on 32-bit linux, compiling with -fPIC
                         # gives assembler that asmgcc is not happy about.
 
-    def library_dirs_for_libffi_a(self):
-        # places where we need to look for libffi.a
-        return self.library_dirs_for_libffi() + ['/usr/lib']
-
-
 class Linux64(BaseLinux):
     pass
diff --git a/pypy/translator/platform/posix.py b/pypy/translator/platform/posix.py
--- a/pypy/translator/platform/posix.py
+++ b/pypy/translator/platform/posix.py
@@ -157,7 +157,7 @@
 
         rules = [
             ('all', '$(DEFAULT_TARGET)', []),
-            ('$(TARGET)', '$(OBJECTS)', '$(CC_LINK) $(LDFLAGS) $(LDFLAGSEXTRA) -o $@ $(OBJECTS) $(LIBDIRS) $(LIBS) $(LINKFILES)'),
+            ('$(TARGET)', '$(OBJECTS)', '$(CC_LINK) $(LDFLAGSEXTRA) -o $@ $(OBJECTS) $(LIBDIRS) $(LIBS) $(LINKFILES) $(LDFLAGS)'),
             ('%.o', '%.c', '$(CC) $(CFLAGS) $(CFLAGSEXTRA) -o $@ -c $< $(INCLUDEDIRS)'),
             ]
 

From noreply at buildbot.pypy.org  Fri Oct 21 14:08:52 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 21 Oct 2011 14:08:52 +0200 (CEST)
Subject: [pypy-commit] pypy inline-dict-ops: Sign in getinteriorfield_gc,
 test and fix (at least for llsupport).
Message-ID: <20111021120852.254A8820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: inline-dict-ops
Changeset: r48306:c9296c708287
Date: 2011-10-21 14:08 +0200
http://bitbucket.org/pypy/pypy/changeset/c9296c708287/

Log:	Sign in getinteriorfield_gc, test and fix (at least for llsupport).

diff --git a/pypy/jit/backend/llsupport/llmodel.py b/pypy/jit/backend/llsupport/llmodel.py
--- a/pypy/jit/backend/llsupport/llmodel.py
+++ b/pypy/jit/backend/llsupport/llmodel.py
@@ -361,15 +361,20 @@
         ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
         ofs += descr.fielddescr.offset
         fieldsize = descr.fielddescr.get_field_size(self.translate_support_code)
+        sign = descr.fielddescr.is_field_signed()
         fullofs = itemindex * size + ofs
         # --- start of GC unsafe code (no GC operation!) ---
         items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), fullofs)
         for STYPE, UTYPE, itemsize in unroll_basic_sizes:
             if fieldsize == itemsize:
-                # XXX signedness
-                item = rffi.cast(rffi.CArrayPtr(STYPE), items)
-                val = item[0]
-                val = rffi.cast(lltype.Signed, val)
+                if sign:
+                    item = rffi.cast(rffi.CArrayPtr(STYPE), items)
+                    val = item[0]
+                    val = rffi.cast(lltype.Signed, val)
+                else:
+                    item = rffi.cast(rffi.CArrayPtr(UTYPE), items)
+                    val = item[0]
+                    val = rffi.cast(lltype.Signed, val)
                 # --- end of GC unsafe code ---
                 return val
         else:
diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -882,13 +882,20 @@
 
     def test_array_of_structs(self):
         TP = lltype.GcStruct('x')
-        ITEM = lltype.Struct('x', ('v', lltype.Signed),
+        ITEM = lltype.Struct('x',
+                             ('vs', lltype.Signed),
+                             ('vu', lltype.Unsigned),
+                             ('vsc', rffi.SIGNEDCHAR),
+                             ('vuc', rffi.UCHAR),
+                             ('vss', rffi.SHORT),
+                             ('vus', rffi.USHORT),
+                             ('vsi', rffi.INT),
+                             ('vui', rffi.UINT),
                              ('k', lltype.Float),
                              ('p', lltype.Ptr(TP)))
         a_box, A = self.alloc_array_of(ITEM, 15)
         s_box, S = self.alloc_instance(TP)
         kdescr = self.cpu.interiorfielddescrof(A, 'k')
-        vdescr = self.cpu.interiorfielddescrof(A, 'v')
         pdescr = self.cpu.interiorfielddescrof(A, 'p')
         self.execute_operation(rop.SETINTERIORFIELD_GC, [a_box, BoxInt(3),
                                                          boxfloat(1.5)],
@@ -899,15 +906,36 @@
         r = self.execute_operation(rop.GETINTERIORFIELD_GC, [a_box, BoxInt(3)],
                                    'float', descr=kdescr)
         assert r.getfloat() == 2.5
-        self.execute_operation(rop.SETINTERIORFIELD_GC, [a_box, BoxInt(3),
-                                                         BoxInt(15)],
-                               'void', descr=vdescr)
-        i = self.cpu.bh_getinteriorfield_gc_i(a_box.getref_base(), 3, vdescr)
-        assert i == 15
-        self.cpu.bh_setinteriorfield_gc_i(a_box.getref_base(), 3, vdescr, 25)
-        r = self.execute_operation(rop.GETINTERIORFIELD_GC, [a_box, BoxInt(3)],
-                                   'int', descr=vdescr)
-        assert r.getint() == 25
+        #
+        NUMBER_FIELDS = [('vs', lltype.Signed),
+                         ('vu', lltype.Unsigned),
+                         ('vsc', rffi.SIGNEDCHAR),
+                         ('vuc', rffi.UCHAR),
+                         ('vss', rffi.SHORT),
+                         ('vus', rffi.USHORT),
+                         ('vsi', rffi.INT),
+                         ('vui', rffi.UINT)]
+        for name, TYPE in NUMBER_FIELDS[::-1]:
+            vdescr = self.cpu.interiorfielddescrof(A, name)
+            self.execute_operation(rop.SETINTERIORFIELD_GC, [a_box, BoxInt(3),
+                                                             BoxInt(-15)],
+                                   'void', descr=vdescr)
+        for name, TYPE in NUMBER_FIELDS:
+            vdescr = self.cpu.interiorfielddescrof(A, name)
+            i = self.cpu.bh_getinteriorfield_gc_i(a_box.getref_base(), 3,
+                                                  vdescr)
+            assert i == rffi.cast(lltype.Signed, rffi.cast(TYPE, -15))
+        for name, TYPE in NUMBER_FIELDS[::-1]:
+            vdescr = self.cpu.interiorfielddescrof(A, name)
+            self.cpu.bh_setinteriorfield_gc_i(a_box.getref_base(), 3,
+                                              vdescr, -25)
+        for name, TYPE in NUMBER_FIELDS:
+            vdescr = self.cpu.interiorfielddescrof(A, name)
+            r = self.execute_operation(rop.GETINTERIORFIELD_GC,
+                                       [a_box, BoxInt(3)],
+                                       'int', descr=vdescr)
+            assert r.getint() == rffi.cast(lltype.Signed, rffi.cast(TYPE, -25))
+        #
         self.execute_operation(rop.SETINTERIORFIELD_GC, [a_box, BoxInt(4),
                                                          s_box],
                                'void', descr=pdescr)

From noreply at buildbot.pypy.org  Fri Oct 21 15:04:12 2011
From: noreply at buildbot.pypy.org (hager)
Date: Fri, 21 Oct 2011 15:04:12 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: (bivab,
	hager): Implemented GUARD_FALSE.
Message-ID: <20111021130412.D60E7820D7@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r48307:28ec327820ce
Date: 2011-10-21 11:52 +0200
http://bitbucket.org/pypy/pypy/changeset/28ec327820ce/

Log:	(bivab, hager): Implemented GUARD_FALSE.

diff --git a/pypy/jit/backend/ppc/ppcgen/opassembler.py b/pypy/jit/backend/ppc/ppcgen/opassembler.py
--- a/pypy/jit/backend/ppc/ppcgen/opassembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/opassembler.py
@@ -174,6 +174,12 @@
         #                        #      ^^^^ If this condition is met,
         #                        #           then the guard fails.
 
+    def emit_guard_false(self, op, arglocs, regalloc):
+            l0 = arglocs[0]
+            failargs = arglocs[1:]
+            self.mc.cmpi(l0.value, 0)
+            self._emit_guard(op, failargs, c.NE)
+
     # TODO - Evaluate whether this can be done with 
     #        SO bit instead of OV bit => usage of CR
     #        instead of XER could be more efficient
diff --git a/pypy/jit/backend/ppc/ppcgen/regalloc.py b/pypy/jit/backend/ppc/ppcgen/regalloc.py
--- a/pypy/jit/backend/ppc/ppcgen/regalloc.py
+++ b/pypy/jit/backend/ppc/ppcgen/regalloc.py
@@ -269,6 +269,8 @@
         self.possibly_free_vars(op.getfailargs())
         return args
 
+    prepare_guard_false = prepare_guard_true
+
     def prepare_guard_no_overflow(self, op):
         locs = self._prepare_guard(op)
         self.possibly_free_vars(op.getfailargs())

From noreply at buildbot.pypy.org  Fri Oct 21 15:04:14 2011
From: noreply at buildbot.pypy.org (hager)
Date: Fri, 21 Oct 2011 15:04:14 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: Implemented GETFIELD_GC and
	SETFIELD_GC.
Message-ID: <20111021130414.0D36E820D7@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r48308:528c82230eac
Date: 2011-10-21 15:03 +0200
http://bitbucket.org/pypy/pypy/changeset/528c82230eac/

Log:	Implemented GETFIELD_GC and SETFIELD_GC.

diff --git a/pypy/jit/backend/ppc/ppcgen/opassembler.py b/pypy/jit/backend/ppc/ppcgen/opassembler.py
--- a/pypy/jit/backend/ppc/ppcgen/opassembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/opassembler.py
@@ -213,5 +213,49 @@
                          descr._ppc_frame_manager_depth)
             regalloc.frame_manager.frame_depth = new_fd
 
+    def emit_setfield_gc(self, op, arglocs, regalloc):
+        value_loc, base_loc, ofs, size = arglocs
+        if size.value == 8:
+            assert 0, "not implemented yet"
+        elif size.value == 4:
+            if ofs.is_imm():
+                self.mc.stw(value_loc.value, base_loc.value, ofs.value)
+            else:
+                self.mc.stw(value_loc.value, base_loc.value, ofs.value)
+        elif size.value == 2:
+            if ofs.is_imm():
+                self.mc.sth(value_loc.value, base_loc.value, ofs.value)
+            else:
+                self.mc.sthx(value_loc.value, base_loc.value, ofs.value)
+        elif size.value == 1:
+            if ofs.is_imm():
+                self.mc.stb(value_loc.value, base_loc.value, ofs.value)
+            else:
+                self.mc.stbx(value_loc.value, base_loc.value, ofs.value)
+        else:
+            assert 0, "size not supported"
+
+    def emit_getfield_gc(self, op, arglocs, regalloc):
+        base_loc, ofs, res, size = arglocs
+        if size.value == 8:
+            assert 0, "not implemented yet"
+        elif size.value == 4:
+            if ofs.is_imm():
+                self.mc.lwz(res.value, base_loc.value, ofs.value)
+            else:
+                self.mc.lwzx(res.value, base_loc.value, ofs.value)
+        elif size.value == 2:
+            if ofs.is_imm():
+                self.mc.lhz(res.value, base_loc.value, ofs.value)
+            else:
+                self.mc.lhzx(res.value, base_loc.value, ofs.value)
+        elif size.value == 1:
+            if ofs.is_imm():
+                self.mc.lbz(res.value, base_loc.value, ofs.value)
+            else:
+                self.mc.lbzx(res.value, base_loc.value, ofs.value)
+        else:
+            assert 0, "size not supported"
+
     def nop(self):
         self.mc.ori(0, 0, 0)
diff --git a/pypy/jit/backend/ppc/ppcgen/regalloc.py b/pypy/jit/backend/ppc/ppcgen/regalloc.py
--- a/pypy/jit/backend/ppc/ppcgen/regalloc.py
+++ b/pypy/jit/backend/ppc/ppcgen/regalloc.py
@@ -12,6 +12,8 @@
                                                          prepare_unary_cmp)
 from pypy.jit.metainterp.history import (INT, REF, FLOAT, Const, ConstInt, 
                                          ConstPtr, LoopToken)
+from pypy.jit.backend.llsupport.descr import BaseFieldDescr, BaseArrayDescr, \
+                                             BaseCallDescr, BaseSizeDescr
 from pypy.jit.metainterp.resoperation import rop
 from pypy.jit.backend.ppc.ppcgen import locations
 from pypy.rpython.lltypesystem import rffi, lltype
@@ -293,6 +295,47 @@
                                  [], [], None)
         return []
 
+    def prepare_setfield_gc(self, op):
+        boxes = list(op.getarglist())
+        b0, b1 = boxes
+        ofs, size, ptr = self._unpack_fielddescr(op.getdescr())
+        base_loc, base_box = self._ensure_value_is_boxed(b0, boxes)
+        boxes.append(base_box)
+        value_loc, value_box = self._ensure_value_is_boxed(b1, boxes)
+        boxes.append(value_box)
+        c_ofs = ConstInt(ofs)
+        if _check_imm_arg(c_ofs):
+            ofs_loc = imm(ofs)
+        else:
+            ofs_loc, ofs_box = self._ensure_value_is_boxed(c_ofs, boxes)
+            boxes.append(ofs_box)
+        self.possibly_free_vars(boxes)
+        return [value_loc, base_loc, ofs_loc, imm(size)]
+
+    def prepare_getfield_gc(self, op):
+        a0 = op.getarg(0)
+        ofs, size, ptr = self._unpack_fielddescr(op.getdescr())
+        base_loc, base_box = self._ensure_value_is_boxed(a0)
+        c_ofs = ConstInt(ofs)
+        if _check_imm_arg(c_ofs):
+            ofs_loc = imm(ofs)
+        else:
+            ofs_loc, ofs_box = self._ensure_value_is_boxed(c_ofs, [base_box])
+            self.possibly_free_var(ofs_box)
+        self.possibly_free_var(a0)
+        self.possibly_free_var(base_box)
+        res = self.force_allocate_reg(op.result)
+        self.possibly_free_var(op.result)
+        return [base_loc, ofs_loc, res, imm(size)]
+
+    # from ../x86/regalloc.py:791
+    def _unpack_fielddescr(self, fielddescr):
+        assert isinstance(fielddescr, BaseFieldDescr)
+        ofs = fielddescr.offset
+        size = fielddescr.get_field_size(self.cpu.translate_support_code)
+        ptr = fielddescr.is_pointer_field()
+        return ofs, size, ptr
+
 def make_operation_list():
     def not_implemented(self, op, *args):
         raise NotImplementedError, op

From noreply at buildbot.pypy.org  Fri Oct 21 15:12:44 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 21 Oct 2011 15:12:44 +0200 (CEST)
Subject: [pypy-commit] pypy inline-dict-ops: Small fixes in preparation for
	the merge.
Message-ID: <20111021131244.BC3B2820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: inline-dict-ops
Changeset: r48309:bf8fa9caf308
Date: 2011-10-21 15:05 +0200
http://bitbucket.org/pypy/pypy/changeset/bf8fa9caf308/

Log:	Small fixes in preparation for the merge.

diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1598,6 +1598,7 @@
 
     def genop_getinteriorfield_gc(self, op, arglocs, resloc):
         base_loc, ofs_loc, itemsize_loc, fieldsize_loc, index_loc, sign_loc = arglocs
+        # XXX should not use IMUL in most cases
         self.mc.IMUL(index_loc, itemsize_loc)
         src_addr = AddressLoc(base_loc, index_loc, 0, ofs_loc.value)
         self.load_from_mem(resloc, src_addr, fieldsize_loc, sign_loc)
@@ -1611,6 +1612,7 @@
 
     def genop_discard_setinteriorfield_gc(self, op, arglocs):
         base_loc, ofs_loc, itemsize_loc, fieldsize_loc, index_loc, value_loc = arglocs
+        # XXX should not use IMUL in most cases
         self.mc.IMUL(index_loc, itemsize_loc)
         dest_addr = AddressLoc(base_loc, index_loc, 0, ofs_loc.value)
         self.save_into_mem(dest_addr, value_loc, fieldsize_loc)
diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -738,13 +738,12 @@
 
     def rewrite_op_getinteriorfield(self, op):
         assert len(op.args) == 3
-        if isinstance(op.args[1], Constant) and op.args[1].value == 'chars':
-            optype = op.args[0].concretetype
-            if optype == lltype.Ptr(rstr.STR):
-                opname = "strgetitem"
-            else:
-                assert optype == lltype.Ptr(rstr.UNICODE)
-                opname = "unicodegetitem"
+        optype = op.args[0].concretetype
+        if optype == lltype.Ptr(rstr.STR):
+            opname = "strgetitem"
+            return SpaceOperation(opname, [op.args[0], op.args[2]], op.result)
+        elif optype == lltype.Ptr(rstr.UNICODE):
+            opname = "unicodegetitem"
             return SpaceOperation(opname, [op.args[0], op.args[2]], op.result)
         else:
             v_inst, v_index, c_field = op.args
@@ -763,13 +762,13 @@
 
     def rewrite_op_setinteriorfield(self, op):
         assert len(op.args) == 4
-        if isinstance(op.args[1], Constant) and op.args[1].value == 'chars':
-            optype = op.args[0].concretetype
-            if optype == lltype.Ptr(rstr.STR):
-                opname = "strsetitem"
-            else:
-                assert optype == lltype.Ptr(rstr.UNICODE)
-                opname = "unicodesetitem"
+        optype = op.args[0].concretetype
+        if optype == lltype.Ptr(rstr.STR):
+            opname = "strsetitem"
+            return SpaceOperation(opname, [op.args[0], op.args[2], op.args[3]],
+                                  op.result)
+        elif optype == lltype.Ptr(rstr.UNICODE):
+            opname = "unicodesetitem"
             return SpaceOperation(opname, [op.args[0], op.args[2], op.args[3]],
                                   op.result)
         else:
diff --git a/pypy/jit/codewriter/support.py b/pypy/jit/codewriter/support.py
--- a/pypy/jit/codewriter/support.py
+++ b/pypy/jit/codewriter/support.py
@@ -47,6 +47,8 @@
     a.build_types(func, argtypes, main_entry_point=True)
     rtyper = t.buildrtyper(type_system = type_system)
     rtyper.specialize()
+    #if inline:
+    #    auto_inlining(t, threshold=inline)
     if backendoptimize:
         from pypy.translator.backendopt.all import backend_optimizations
         backend_optimizations(t, inline_threshold=inline or 0,

From noreply at buildbot.pypy.org  Fri Oct 21 15:12:45 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 21 Oct 2011 15:12:45 +0200 (CEST)
Subject: [pypy-commit] pypy inline-dict-ops: Move this hack away from
 lltype. This logic is needed to have
Message-ID: <20111021131245.EC72B820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: inline-dict-ops
Changeset: r48310:cebb862ad03f
Date: 2011-10-21 15:12 +0200
http://bitbucket.org/pypy/pypy/changeset/cebb862ad03f/

Log:	Move this hack away from lltype. This logic is needed to have
	lltype correctly detect accesses inside dead GcStruct.

diff --git a/pypy/rpython/lltypesystem/ll2ctypes.py b/pypy/rpython/lltypesystem/ll2ctypes.py
--- a/pypy/rpython/lltypesystem/ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/ll2ctypes.py
@@ -530,6 +530,10 @@
     def __str__(self):
         return repr(self)
 
+    def _setparentstructure(self, parent, parentindex):
+        super(_parentable_mixin, self)._setparentstructure(parent, parentindex)
+        self._keepparent = parent   # always keep a strong ref
+
 class _struct_mixin(_parentable_mixin):
     """Mixin added to _struct containers when they become ctypes-based."""
     __slots__ = ()
diff --git a/pypy/rpython/lltypesystem/lltype.py b/pypy/rpython/lltypesystem/lltype.py
--- a/pypy/rpython/lltypesystem/lltype.py
+++ b/pypy/rpython/lltypesystem/lltype.py
@@ -1517,7 +1517,12 @@
         self._wrparent = weakref.ref(parent)
         self._parent_type = typeOf(parent)
         self._parent_index = parentindex
-        self._keepparent = parent
+        if (isinstance(self._parent_type, Struct)
+            and self._parent_type._names
+            and parentindex in (self._parent_type._names[0], 0)
+            and self._TYPE._gckind == typeOf(parent)._gckind):
+            # keep strong reference to parent, we share the same allocation
+            self._keepparent = parent
 
     def _parentstructure(self, check=True):
         if self._wrparent is not None:
@@ -1726,7 +1731,7 @@
         # Don't do it if we are inside a GC object, though -- it's someone
         # else's job to keep the GC object alive
         if (typeOf(top_container(parent))._gckind == 'raw' or
-            hasattr(top_container(parent)._storage, 'contents')):
+            hasattr(top_container(parent)._storage, 'contents')):  # ll2ctypes
             self._keepparent = parent
 
     def __str__(self):

From noreply at buildbot.pypy.org  Fri Oct 21 16:06:01 2011
From: noreply at buildbot.pypy.org (hager)
Date: Fri, 21 Oct 2011 16:06:01 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: Implemented a few guard
	operations.
Message-ID: <20111021140601.5528E820D7@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r48311:63cb14dbd14c
Date: 2011-10-21 16:05 +0200
http://bitbucket.org/pypy/pypy/changeset/63cb14dbd14c/

Log:	Implemented a few guard operations.

diff --git a/pypy/jit/backend/ppc/ppcgen/opassembler.py b/pypy/jit/backend/ppc/ppcgen/opassembler.py
--- a/pypy/jit/backend/ppc/ppcgen/opassembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/opassembler.py
@@ -197,6 +197,23 @@
     def emit_guard_overflow(self, op, arglocs, regalloc):
         self._emit_ovf_guard(op, arglocs, c.EQ)
 
+    def emit_guard_value(self, op, arglocs, regalloc):
+        l0 = arglocs[0]
+        l1 = arglocs[1]
+        failargs = arglocs[2:]
+
+        if l0.is_reg():
+            if l1.is_imm():
+                self.mc.cmpi(l0.value, l1.getint())
+            else:
+                self.mc.cmp(l0.value, l1.value)
+        else:
+            assert 0, "not implemented yet"
+        self._emit_guard(op, failargs, c.NE)
+
+    emit_guard_nonnull = emit_guard_true
+    emit_guard_isnull = emit_guard_false
+
     def emit_finish(self, op, arglocs, regalloc):
         self.gen_exit_stub(op.getdescr(), op.getarglist(), arglocs)
 
@@ -213,6 +230,7 @@
                          descr._ppc_frame_manager_depth)
             regalloc.frame_manager.frame_depth = new_fd
 
+    # XXX adjust 64 bit
     def emit_setfield_gc(self, op, arglocs, regalloc):
         value_loc, base_loc, ofs, size = arglocs
         if size.value == 8:
@@ -235,6 +253,7 @@
         else:
             assert 0, "size not supported"
 
+    # XXX adjust 64 bit
     def emit_getfield_gc(self, op, arglocs, regalloc):
         base_loc, ofs, res, size = arglocs
         if size.value == 8:
diff --git a/pypy/jit/backend/ppc/ppcgen/regalloc.py b/pypy/jit/backend/ppc/ppcgen/regalloc.py
--- a/pypy/jit/backend/ppc/ppcgen/regalloc.py
+++ b/pypy/jit/backend/ppc/ppcgen/regalloc.py
@@ -272,6 +272,8 @@
         return args
 
     prepare_guard_false = prepare_guard_true
+    prepare_guard_nonnull = prepare_guard_true
+    prepare_guard_isnull = prepare_guard_true
 
     def prepare_guard_no_overflow(self, op):
         locs = self._prepare_guard(op)
@@ -280,6 +282,23 @@
 
     prepare_guard_overflow = prepare_guard_no_overflow
 
+    def prepare_guard_value(self, op):
+        boxes = list(op.getarglist())
+        b0, b1 = boxes
+        imm_b1 = _check_imm_arg(b1)
+        l0, box = self._ensure_value_is_boxed(b0, boxes)
+        boxes.append(box)
+        if not imm_b1:
+            l1, box = self._ensure_value_is_boxed(b1,boxes)
+            boxes.append(box)
+        else:
+            l1 = self.make_sure_var_in_reg(b1)
+        assert op.result is None
+        arglocs = self._prepare_guard(op, [l0, l1])
+        self.possibly_free_vars(boxes)
+        self.possibly_free_vars(op.getfailargs())
+        return arglocs
+
     def prepare_jump(self, op):
         descr = op.getdescr()
         assert isinstance(descr, LoopToken)

From noreply at buildbot.pypy.org  Fri Oct 21 16:23:51 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 21 Oct 2011 16:23:51 +0200 (CEST)
Subject: [pypy-commit] pypy inline-dict-ops: Add correct overflow checking
	for Boehm's malloc
Message-ID: <20111021142351.30596820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: inline-dict-ops
Changeset: r48312:ea30df33f75c
Date: 2011-10-21 16:23 +0200
http://bitbucket.org/pypy/pypy/changeset/ea30df33f75c/

Log:	Add correct overflow checking for Boehm's malloc

diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py
--- a/pypy/jit/backend/llsupport/gc.py
+++ b/pypy/jit/backend/llsupport/gc.py
@@ -105,7 +105,10 @@
         self.funcptr_for_new = malloc_fn_ptr
 
         def malloc_array(basesize, itemsize, ofs_length, num_elem):
-            size = basesize + itemsize * num_elem
+            try:
+                size = ovfcheck(basesize + ovfcheck(itemsize * num_elem))
+            except OverflowError:
+                return lltype.nullptr(llmemory.GCREF.TO)
             res = self.funcptr_for_new(size)
             if not res:
                 return res

From noreply at buildbot.pypy.org  Fri Oct 21 17:17:04 2011
From: noreply at buildbot.pypy.org (hager)
Date: Fri, 21 Oct 2011 17:17:04 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: Implemented GUARD_CLASS and
	GUARD_NONNULL_CLASS.
Message-ID: <20111021151704.B73F4820D7@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r48313:6c9c2b1b0146
Date: 2011-10-21 17:16 +0200
http://bitbucket.org/pypy/pypy/changeset/6c9c2b1b0146/

Log:	Implemented GUARD_CLASS and GUARD_NONNULL_CLASS.

diff --git a/pypy/jit/backend/ppc/ppcgen/opassembler.py b/pypy/jit/backend/ppc/ppcgen/opassembler.py
--- a/pypy/jit/backend/ppc/ppcgen/opassembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/opassembler.py
@@ -214,6 +214,30 @@
     emit_guard_nonnull = emit_guard_true
     emit_guard_isnull = emit_guard_false
 
+    def _cmp_guard_class(self, op, locs, regalloc):
+        offset = locs[2]
+        if offset is not None:
+            if offset.is_imm():
+                self.mc.lwz(r.r0.value, locs[0].value, offset.value)
+            else:
+                self.mc.lwzx(r.r0.value, locs[0].value, offset.value)
+            self.mc.cmp(r.r0.value, locs[1].value)
+        else:
+            assert 0, "not implemented yet"
+        self._emit_guard(op, locs[3:], c.NE)
+
+    def emit_guard_class(self, op, arglocs, regalloc):
+        self._cmp_guard_class(op, arglocs, regalloc)
+
+    def emit_guard_nonnull_class(self, op, arglocs, regalloc):
+        offset = self.cpu.vtable_offset
+        self.mc.cmpi(arglocs[0].value, 0)
+        if offset is not None:
+            self._emit_guard(op, arglocs[3:], c.EQ)
+        else:
+            raise NotImplementedError
+        self._cmp_guard_class(op, arglocs, regalloc)
+
     def emit_finish(self, op, arglocs, regalloc):
         self.gen_exit_stub(op.getdescr(), op.getarglist(), arglocs)
 
diff --git a/pypy/jit/backend/ppc/ppcgen/regalloc.py b/pypy/jit/backend/ppc/ppcgen/regalloc.py
--- a/pypy/jit/backend/ppc/ppcgen/regalloc.py
+++ b/pypy/jit/backend/ppc/ppcgen/regalloc.py
@@ -11,7 +11,7 @@
                                                          prepare_binary_int_op_with_imm,
                                                          prepare_unary_cmp)
 from pypy.jit.metainterp.history import (INT, REF, FLOAT, Const, ConstInt, 
-                                         ConstPtr, LoopToken)
+                                         ConstPtr, LoopToken, Box)
 from pypy.jit.backend.llsupport.descr import BaseFieldDescr, BaseArrayDescr, \
                                              BaseCallDescr, BaseSizeDescr
 from pypy.jit.metainterp.resoperation import rop
@@ -299,6 +299,30 @@
         self.possibly_free_vars(op.getfailargs())
         return arglocs
 
+    def prepare_guard_class(self, op):
+        assert isinstance(op.getarg(0), Box)
+        boxes = list(op.getarglist())
+
+        x, x_box = self._ensure_value_is_boxed(boxes[0], boxes)
+        boxes.append(x_box)
+
+        t = TempInt()
+        y = self.force_allocate_reg(t, boxes)
+        boxes.append(t)
+        y_val = rffi.cast(lltype.Signed, op.getarg(1).getint())
+        self.assembler.load_imm(y.value, y_val)
+
+        offset = self.cpu.vtable_offset
+        assert offset is not None
+        offset_loc, offset_box = self._ensure_value_is_boxed(ConstInt(offset), boxes)
+        boxes.append(offset_box)
+        arglocs = self._prepare_guard(op, [x, y, offset_loc])
+        self.possibly_free_vars(boxes)
+        self.possibly_free_vars(op.getfailargs())
+        return arglocs
+
+    prepare_guard_nonnull_class = prepare_guard_class
+
     def prepare_jump(self, op):
         descr = op.getdescr()
         assert isinstance(descr, LoopToken)

From noreply at buildbot.pypy.org  Fri Oct 21 17:36:23 2011
From: noreply at buildbot.pypy.org (hager)
Date: Fri, 21 Oct 2011 17:36:23 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: Implemented PTR_EQ and PTR_NE.
Message-ID: <20111021153623.A1604820D7@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r48314:51bb8d251b9f
Date: 2011-10-21 17:36 +0200
http://bitbucket.org/pypy/pypy/changeset/51bb8d251b9f/

Log:	Implemented PTR_EQ and PTR_NE.

diff --git a/pypy/jit/backend/ppc/ppcgen/opassembler.py b/pypy/jit/backend/ppc/ppcgen/opassembler.py
--- a/pypy/jit/backend/ppc/ppcgen/opassembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/opassembler.py
@@ -144,6 +144,9 @@
     emit_int_is_zero = gen_emit_unary_cmp_op(c.IS_ZERO)
     emit_int_is_true = gen_emit_unary_cmp_op(c.IS_TRUE)
 
+    emit_ptr_eq = emit_int_eq
+    emit_ptr_ne = emit_int_ne
+
     def emit_int_neg(self, op, arglocs, regalloc):
         l0, res = arglocs
         self.mc.neg(res.value, l0.value)
diff --git a/pypy/jit/backend/ppc/ppcgen/regalloc.py b/pypy/jit/backend/ppc/ppcgen/regalloc.py
--- a/pypy/jit/backend/ppc/ppcgen/regalloc.py
+++ b/pypy/jit/backend/ppc/ppcgen/regalloc.py
@@ -234,6 +234,9 @@
     prepare_int_eq = prepare_cmp_op()
     prepare_int_ne = prepare_cmp_op()
 
+    prepare_ptr_eq = prepare_int_eq
+    prepare_ptr_ne = prepare_int_ne
+
     prepare_uint_lt = prepare_cmp_op()
     prepare_uint_le = prepare_cmp_op()
     prepare_uint_gt = prepare_cmp_op()

From noreply at buildbot.pypy.org  Fri Oct 21 17:48:19 2011
From: noreply at buildbot.pypy.org (cfbolz)
Date: Fri, 21 Oct 2011 17:48:19 +0200 (CEST)
Subject: [pypy-commit] pypy default: a test that rerased works after
 translation to C, which it didn't, so a fix too.
Message-ID: <20111021154819.558E6820D7@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r48315:8494d35800e1
Date: 2011-10-21 17:23 +0200
http://bitbucket.org/pypy/pypy/changeset/8494d35800e1/

Log:	a test that rerased works after translation to C, which it didn't,
	so a fix too.

diff --git a/pypy/translator/c/gcc/test/test_asmgcroot.py b/pypy/translator/c/gcc/test/test_asmgcroot.py
--- a/pypy/translator/c/gcc/test/test_asmgcroot.py
+++ b/pypy/translator/c/gcc/test/test_asmgcroot.py
@@ -21,6 +21,7 @@
         config = get_pypy_config(translating=True)
         config.translation.gc = cls.gcpolicy
         config.translation.gcrootfinder = "asmgcc"
+        config.translation.taggedpointers = getattr(cls, "taggedpointers", False)
         return config
 
     @classmethod
diff --git a/pypy/translator/c/primitive.py b/pypy/translator/c/primitive.py
--- a/pypy/translator/c/primitive.py
+++ b/pypy/translator/c/primitive.py
@@ -141,7 +141,12 @@
 
 def name_gcref(value, db):
     if value:
-        realobj = value._obj.container
+        obj = value._obj
+        if isinstance(obj, int):
+            # a tagged pointer
+            assert obj & 1 == 1
+            return '((%s) %d)' % (cdecl("void*", ''), obj)
+        realobj = obj.container
         realvalue = cast_opaque_ptr(Ptr(typeOf(realobj)), value)
         return db.get(realvalue)
     else:
diff --git a/pypy/translator/c/test/test_newgc.py b/pypy/translator/c/test/test_newgc.py
--- a/pypy/translator/c/test/test_newgc.py
+++ b/pypy/translator/c/test/test_newgc.py
@@ -1487,6 +1487,43 @@
         res = self.run("tagged")
         assert res == expected
 
+    def define_erased(cls):
+        from pypy.rlib import rerased
+        erase, unerase = rerased.new_erasing_pair("test")
+        class Unrelated(object):
+            pass
+
+        u = Unrelated()
+        u.tagged = True
+        u.x = rerased.erase_int(41)
+        class A(object):
+            pass
+        def fn():
+            n = 1
+            while n >= 0:
+                if u.tagged:
+                    n = rerased.unerase_int(u.x)
+                    a = A()
+                    a.n = n - 1
+                    u.x = erase(a)
+                    u.tagged = False
+                else:
+                    n = unerase(u.x).n
+                    u.x = rerased.erase_int(n - 1)
+                    u.tagged = True
+        def func():
+            rgc.collect() # check that a prebuilt erased integer doesn't explode
+            u.x = rerased.erase_int(1000)
+            u.tagged = True
+            fn()
+            return 1
+        return func
+
+    def test_erased(self):
+        expected = self.run_orig("erased")
+        res = self.run("erased")
+        assert res == expected
+
 from pypy.rlib.objectmodel import UnboxedValue
 
 class TaggedBase(object):

From noreply at buildbot.pypy.org  Fri Oct 21 17:48:20 2011
From: noreply at buildbot.pypy.org (cfbolz)
Date: Fri, 21 Oct 2011 17:48:20 +0200 (CEST)
Subject: [pypy-commit] pypy default: (arigo,
 cfbolz): also split up the nonfuncnodes according to where they are
Message-ID: <20111021154820.8650C820D7@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r48316:9a6c4ff948c1
Date: 2011-10-21 17:37 +0200
http://bitbucket.org/pypy/pypy/changeset/9a6c4ff948c1/

Log:	(arigo, cfbolz): also split up the nonfuncnodes according to where
	they are used. slightly obscure (just as the rest of the C backend)

diff --git a/pypy/translator/c/database.py b/pypy/translator/c/database.py
--- a/pypy/translator/c/database.py
+++ b/pypy/translator/c/database.py
@@ -176,7 +176,7 @@
                      # introduced by the GC transformer, or the type_info_table
         return node
 
-    def get(self, obj):
+    def get(self, obj, funcgen=None):
         if isinstance(obj, CConstant):
             return obj.c_name  # without further checks
         T = typeOf(obj)
@@ -228,6 +228,8 @@
                     return '((%s) %d)' % (cdecl(self.gettype(T), ''),
                                           obj._obj)
                 node = self.getcontainernode(container)
+                if node._funccodegen_owner is None:
+                    node._funccodegen_owner = funcgen
                 return node.getptrname()
             else:
                 return '((%s) NULL)' % (cdecl(self.gettype(T), ''), )
@@ -284,14 +286,16 @@
             finish_callbacks.append(('GC transformer: finished tables',
                                      self.gctransformer.get_finish_tables()))
 
-        def add_dependencies(newdependencies):
+        def add_dependencies(newdependencies, parent=None):
             for value in newdependencies:
                 #if isinstance(value, _uninitialized):
                 #    continue
                 if isinstance(typeOf(value), ContainerType):
-                    self.getcontainernode(value)
+                    node = self.getcontainernode(value)
+                    if parent and node._funccodegen_owner is not None:
+                        node._funccodegen_owner = parent._funccodegen_owner
                 else:
-                    self.get(value)
+                    self.get(value, parent and parent._funccodegen_owner)
 
         while True:
             while True:
@@ -303,7 +307,7 @@
                 if i == len(self.containerlist):
                     break
                 node = self.containerlist[i]
-                add_dependencies(node.enum_dependencies())
+                add_dependencies(node.enum_dependencies(), node)
                 i += 1
                 self.completedcontainers = i
                 if i == show_i:
diff --git a/pypy/translator/c/gc.py b/pypy/translator/c/gc.py
--- a/pypy/translator/c/gc.py
+++ b/pypy/translator/c/gc.py
@@ -170,6 +170,7 @@
     nodekind = 'refcnt rtti'
     globalcontainer = True
     typename = 'void (@)(void *)'
+    _funccodegen_owner = None
 
     def __init__(self, db, T, obj):
         assert T == RuntimeTypeInfo
@@ -266,6 +267,7 @@
     nodekind = 'boehm rtti'
     globalcontainer = True
     typename = 'char @'
+    _funccodegen_owner = None
 
     def __init__(self, db, T, obj):
         assert T == RuntimeTypeInfo
diff --git a/pypy/translator/c/genc.py b/pypy/translator/c/genc.py
--- a/pypy/translator/c/genc.py
+++ b/pypy/translator/c/genc.py
@@ -682,8 +682,7 @@
     def getbasecfilefornode(self, node, basecname):
         # For FuncNode instances, use the python source filename (relative to
         # the top directory):
-        if hasattr(node.obj, 'graph'):
-            g = node.obj.graph
+        def invent_nice_name(g):
             # Lookup the filename from the function.
             # However, not all FunctionGraph objs actually have a "func":
             if hasattr(g, 'func'):
@@ -693,6 +692,15 @@
                     if pypkgpath:
                         relpypath =  localpath.relto(pypkgpath)
                         return relpypath.replace('.py', '.c')
+            return None
+        if hasattr(node.obj, 'graph'):
+            name = invent_nice_name(node.obj.graph)
+            if name is not None:
+                return name
+        elif node._funccodegen_owner is not None:
+            name = invent_nice_name(node._funccodegen_owner.graph)
+            if name is not None:
+                return "data_" + name
         return basecname
 
     def splitnodesimpl(self, basecname, nodes, nextra, nbetween,
diff --git a/pypy/translator/c/node.py b/pypy/translator/c/node.py
--- a/pypy/translator/c/node.py
+++ b/pypy/translator/c/node.py
@@ -485,6 +485,7 @@
         __slots__ = """db obj 
                        typename implementationtypename
                         name
+                        _funccodegen_owner
                         globalcontainer""".split()
     eci_name = '_compilation_info'
 
@@ -509,6 +510,7 @@
         if self.typename != self.implementationtypename:
             if db.gettypedefnode(T).extra_union_for_varlength:
                 self.name += '.b'
+        self._funccodegen_owner = None
 
     def getptrname(self):
         return '(&%s)' % self.name
@@ -842,6 +844,9 @@
         if self.funcgens:
             argnames = self.funcgens[0].argnames()  #Assume identical for all funcgens
             self.implementationtypename = self.db.gettype(self.T, argnames=argnames)
+            self._funccodegen_owner = self.funcgens[0]
+        else:
+            self._funccodegen_owner = None
 
     def basename(self):
         return self.obj._name
@@ -1005,6 +1010,7 @@
     globalcontainer = True
     typename = 'PyObject @'
     implementationtypename = 'PyObject *@'
+    _funccodegen_owner = None
 
     def __init__(self, db, T, obj):
         # obj is a _pyobject here; obj.value is the underlying CPython object

From noreply at buildbot.pypy.org  Fri Oct 21 17:48:21 2011
From: noreply at buildbot.pypy.org (cfbolz)
Date: Fri, 21 Oct 2011 17:48:21 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20111021154821.B82F0820D7@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r48317:9718181b5d75
Date: 2011-10-21 17:47 +0200
http://bitbucket.org/pypy/pypy/changeset/9718181b5d75/

Log:	merge

diff --git a/pypy/translator/c/database.py b/pypy/translator/c/database.py
--- a/pypy/translator/c/database.py
+++ b/pypy/translator/c/database.py
@@ -176,7 +176,7 @@
                      # introduced by the GC transformer, or the type_info_table
         return node
 
-    def get(self, obj):
+    def get(self, obj, funcgen=None):
         if isinstance(obj, CConstant):
             return obj.c_name  # without further checks
         T = typeOf(obj)
@@ -228,6 +228,8 @@
                     return '((%s) %d)' % (cdecl(self.gettype(T), ''),
                                           obj._obj)
                 node = self.getcontainernode(container)
+                if node._funccodegen_owner is None:
+                    node._funccodegen_owner = funcgen
                 return node.getptrname()
             else:
                 return '((%s) NULL)' % (cdecl(self.gettype(T), ''), )
@@ -284,14 +286,16 @@
             finish_callbacks.append(('GC transformer: finished tables',
                                      self.gctransformer.get_finish_tables()))
 
-        def add_dependencies(newdependencies):
+        def add_dependencies(newdependencies, parent=None):
             for value in newdependencies:
                 #if isinstance(value, _uninitialized):
                 #    continue
                 if isinstance(typeOf(value), ContainerType):
-                    self.getcontainernode(value)
+                    node = self.getcontainernode(value)
+                    if parent and node._funccodegen_owner is not None:
+                        node._funccodegen_owner = parent._funccodegen_owner
                 else:
-                    self.get(value)
+                    self.get(value, parent and parent._funccodegen_owner)
 
         while True:
             while True:
@@ -303,7 +307,7 @@
                 if i == len(self.containerlist):
                     break
                 node = self.containerlist[i]
-                add_dependencies(node.enum_dependencies())
+                add_dependencies(node.enum_dependencies(), node)
                 i += 1
                 self.completedcontainers = i
                 if i == show_i:
diff --git a/pypy/translator/c/gc.py b/pypy/translator/c/gc.py
--- a/pypy/translator/c/gc.py
+++ b/pypy/translator/c/gc.py
@@ -170,6 +170,7 @@
     nodekind = 'refcnt rtti'
     globalcontainer = True
     typename = 'void (@)(void *)'
+    _funccodegen_owner = None
 
     def __init__(self, db, T, obj):
         assert T == RuntimeTypeInfo
@@ -266,6 +267,7 @@
     nodekind = 'boehm rtti'
     globalcontainer = True
     typename = 'char @'
+    _funccodegen_owner = None
 
     def __init__(self, db, T, obj):
         assert T == RuntimeTypeInfo
diff --git a/pypy/translator/c/gcc/test/test_asmgcroot.py b/pypy/translator/c/gcc/test/test_asmgcroot.py
--- a/pypy/translator/c/gcc/test/test_asmgcroot.py
+++ b/pypy/translator/c/gcc/test/test_asmgcroot.py
@@ -21,6 +21,7 @@
         config = get_pypy_config(translating=True)
         config.translation.gc = cls.gcpolicy
         config.translation.gcrootfinder = "asmgcc"
+        config.translation.taggedpointers = getattr(cls, "taggedpointers", False)
         return config
 
     @classmethod
diff --git a/pypy/translator/c/genc.py b/pypy/translator/c/genc.py
--- a/pypy/translator/c/genc.py
+++ b/pypy/translator/c/genc.py
@@ -682,8 +682,7 @@
     def getbasecfilefornode(self, node, basecname):
         # For FuncNode instances, use the python source filename (relative to
         # the top directory):
-        if hasattr(node.obj, 'graph'):
-            g = node.obj.graph
+        def invent_nice_name(g):
             # Lookup the filename from the function.
             # However, not all FunctionGraph objs actually have a "func":
             if hasattr(g, 'func'):
@@ -693,6 +692,15 @@
                     if pypkgpath:
                         relpypath =  localpath.relto(pypkgpath)
                         return relpypath.replace('.py', '.c')
+            return None
+        if hasattr(node.obj, 'graph'):
+            name = invent_nice_name(node.obj.graph)
+            if name is not None:
+                return name
+        elif node._funccodegen_owner is not None:
+            name = invent_nice_name(node._funccodegen_owner.graph)
+            if name is not None:
+                return "data_" + name
         return basecname
 
     def splitnodesimpl(self, basecname, nodes, nextra, nbetween,
diff --git a/pypy/translator/c/node.py b/pypy/translator/c/node.py
--- a/pypy/translator/c/node.py
+++ b/pypy/translator/c/node.py
@@ -485,6 +485,7 @@
         __slots__ = """db obj 
                        typename implementationtypename
                         name
+                        _funccodegen_owner
                         globalcontainer""".split()
     eci_name = '_compilation_info'
 
@@ -509,6 +510,7 @@
         if self.typename != self.implementationtypename:
             if db.gettypedefnode(T).extra_union_for_varlength:
                 self.name += '.b'
+        self._funccodegen_owner = None
 
     def getptrname(self):
         return '(&%s)' % self.name
@@ -842,6 +844,9 @@
         if self.funcgens:
             argnames = self.funcgens[0].argnames()  #Assume identical for all funcgens
             self.implementationtypename = self.db.gettype(self.T, argnames=argnames)
+            self._funccodegen_owner = self.funcgens[0]
+        else:
+            self._funccodegen_owner = None
 
     def basename(self):
         return self.obj._name
@@ -1005,6 +1010,7 @@
     globalcontainer = True
     typename = 'PyObject @'
     implementationtypename = 'PyObject *@'
+    _funccodegen_owner = None
 
     def __init__(self, db, T, obj):
         # obj is a _pyobject here; obj.value is the underlying CPython object
diff --git a/pypy/translator/c/primitive.py b/pypy/translator/c/primitive.py
--- a/pypy/translator/c/primitive.py
+++ b/pypy/translator/c/primitive.py
@@ -141,7 +141,12 @@
 
 def name_gcref(value, db):
     if value:
-        realobj = value._obj.container
+        obj = value._obj
+        if isinstance(obj, int):
+            # a tagged pointer
+            assert obj & 1 == 1
+            return '((%s) %d)' % (cdecl("void*", ''), obj)
+        realobj = obj.container
         realvalue = cast_opaque_ptr(Ptr(typeOf(realobj)), value)
         return db.get(realvalue)
     else:
diff --git a/pypy/translator/c/test/test_newgc.py b/pypy/translator/c/test/test_newgc.py
--- a/pypy/translator/c/test/test_newgc.py
+++ b/pypy/translator/c/test/test_newgc.py
@@ -1487,6 +1487,43 @@
         res = self.run("tagged")
         assert res == expected
 
+    def define_erased(cls):
+        from pypy.rlib import rerased
+        erase, unerase = rerased.new_erasing_pair("test")
+        class Unrelated(object):
+            pass
+
+        u = Unrelated()
+        u.tagged = True
+        u.x = rerased.erase_int(41)
+        class A(object):
+            pass
+        def fn():
+            n = 1
+            while n >= 0:
+                if u.tagged:
+                    n = rerased.unerase_int(u.x)
+                    a = A()
+                    a.n = n - 1
+                    u.x = erase(a)
+                    u.tagged = False
+                else:
+                    n = unerase(u.x).n
+                    u.x = rerased.erase_int(n - 1)
+                    u.tagged = True
+        def func():
+            rgc.collect() # check that a prebuilt erased integer doesn't explode
+            u.x = rerased.erase_int(1000)
+            u.tagged = True
+            fn()
+            return 1
+        return func
+
+    def test_erased(self):
+        expected = self.run_orig("erased")
+        res = self.run("erased")
+        assert res == expected
+
 from pypy.rlib.objectmodel import UnboxedValue
 
 class TaggedBase(object):

From noreply at buildbot.pypy.org  Fri Oct 21 17:58:33 2011
From: noreply at buildbot.pypy.org (cfbolz)
Date: Fri, 21 Oct 2011 17:58:33 +0200 (CEST)
Subject: [pypy-commit] pypy default: (arigo,
 cfbolz): use operationerrfmt in two places (just because)
Message-ID: <20111021155833.3B9AD820D7@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r48318:f9df8a8ca762
Date: 2011-10-21 17:50 +0200
http://bitbucket.org/pypy/pypy/changeset/f9df8a8ca762/

Log:	(arigo, cfbolz): use operationerrfmt in two places (just because)

diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -2,7 +2,7 @@
 from pypy.interpreter.baseobjspace import Wrappable
 from pypy.interpreter import typedef
 from pypy.interpreter.gateway import interp2app
-from pypy.interpreter.error import OperationError
+from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.tool.pairtype import extendabletype
 from pypy.tool.sourcetools import func_with_new_name
@@ -2925,14 +2925,13 @@
 def Module_get_body(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -2968,14 +2967,13 @@
 def Interactive_get_body(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3015,8 +3013,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     return space.wrap(w_self.body)
 
 def Expression_set_body(space, w_self, w_new_value):
@@ -3057,14 +3054,13 @@
 def Suite_get_body(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3104,8 +3100,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._lineno_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'lineno'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lineno')
     return space.wrap(w_self.lineno)
 
 def stmt_set_lineno(space, w_self, w_new_value):
@@ -3126,8 +3121,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._col_offset_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'col_offset'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'col_offset')
     return space.wrap(w_self.col_offset)
 
 def stmt_set_col_offset(space, w_self, w_new_value):
@@ -3157,8 +3151,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'name'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name')
     return space.wrap(w_self.name)
 
 def FunctionDef_set_name(space, w_self, w_new_value):
@@ -3179,8 +3172,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'args'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args')
     return space.wrap(w_self.args)
 
 def FunctionDef_set_args(space, w_self, w_new_value):
@@ -3197,14 +3189,13 @@
 def FunctionDef_get_body(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3215,14 +3206,13 @@
 def FunctionDef_get_decorator_list(space, w_self):
     if not w_self.initialization_state & 8:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'decorator_list'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'decorator_list')
     if w_self.w_decorator_list is None:
         if w_self.decorator_list is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.decorator_list]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_decorator_list = w_list
     return w_self.w_decorator_list
 
@@ -3266,8 +3256,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'name'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name')
     return space.wrap(w_self.name)
 
 def ClassDef_set_name(space, w_self, w_new_value):
@@ -3284,14 +3273,13 @@
 def ClassDef_get_bases(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'bases'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'bases')
     if w_self.w_bases is None:
         if w_self.bases is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.bases]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_bases = w_list
     return w_self.w_bases
 
@@ -3302,14 +3290,13 @@
 def ClassDef_get_body(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3320,14 +3307,13 @@
 def ClassDef_get_decorator_list(space, w_self):
     if not w_self.initialization_state & 8:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'decorator_list'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'decorator_list')
     if w_self.w_decorator_list is None:
         if w_self.decorator_list is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.decorator_list]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_decorator_list = w_list
     return w_self.w_decorator_list
 
@@ -3372,8 +3358,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Return_set_value(space, w_self, w_new_value):
@@ -3414,14 +3399,13 @@
 def Delete_get_targets(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'targets'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'targets')
     if w_self.w_targets is None:
         if w_self.targets is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.targets]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_targets = w_list
     return w_self.w_targets
 
@@ -3457,14 +3441,13 @@
 def Assign_get_targets(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'targets'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'targets')
     if w_self.w_targets is None:
         if w_self.targets is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.targets]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_targets = w_list
     return w_self.w_targets
 
@@ -3479,8 +3462,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Assign_set_value(space, w_self, w_new_value):
@@ -3527,8 +3509,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'target'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'target')
     return space.wrap(w_self.target)
 
 def AugAssign_set_target(space, w_self, w_new_value):
@@ -3549,8 +3530,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'op'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op')
     return operator_to_class[w_self.op - 1]()
 
 def AugAssign_set_op(space, w_self, w_new_value):
@@ -3573,8 +3553,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def AugAssign_set_value(space, w_self, w_new_value):
@@ -3621,8 +3600,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'dest'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'dest')
     return space.wrap(w_self.dest)
 
 def Print_set_dest(space, w_self, w_new_value):
@@ -3639,14 +3617,13 @@
 def Print_get_values(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'values'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'values')
     if w_self.w_values is None:
         if w_self.values is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.values]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_values = w_list
     return w_self.w_values
 
@@ -3661,8 +3638,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'nl'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'nl')
     return space.wrap(w_self.nl)
 
 def Print_set_nl(space, w_self, w_new_value):
@@ -3710,8 +3686,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'target'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'target')
     return space.wrap(w_self.target)
 
 def For_set_target(space, w_self, w_new_value):
@@ -3732,8 +3707,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'iter'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'iter')
     return space.wrap(w_self.iter)
 
 def For_set_iter(space, w_self, w_new_value):
@@ -3750,14 +3724,13 @@
 def For_get_body(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3768,14 +3741,13 @@
 def For_get_orelse(space, w_self):
     if not w_self.initialization_state & 8:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'orelse'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
     if w_self.w_orelse is None:
         if w_self.orelse is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.orelse]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_orelse = w_list
     return w_self.w_orelse
 
@@ -3819,8 +3791,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'test'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test')
     return space.wrap(w_self.test)
 
 def While_set_test(space, w_self, w_new_value):
@@ -3837,14 +3808,13 @@
 def While_get_body(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3855,14 +3825,13 @@
 def While_get_orelse(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'orelse'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
     if w_self.w_orelse is None:
         if w_self.orelse is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.orelse]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_orelse = w_list
     return w_self.w_orelse
 
@@ -3905,8 +3874,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'test'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test')
     return space.wrap(w_self.test)
 
 def If_set_test(space, w_self, w_new_value):
@@ -3923,14 +3891,13 @@
 def If_get_body(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3941,14 +3908,13 @@
 def If_get_orelse(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'orelse'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
     if w_self.w_orelse is None:
         if w_self.orelse is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.orelse]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_orelse = w_list
     return w_self.w_orelse
 
@@ -3991,8 +3957,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'context_expr'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'context_expr')
     return space.wrap(w_self.context_expr)
 
 def With_set_context_expr(space, w_self, w_new_value):
@@ -4013,8 +3978,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'optional_vars'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'optional_vars')
     return space.wrap(w_self.optional_vars)
 
 def With_set_optional_vars(space, w_self, w_new_value):
@@ -4031,14 +3995,13 @@
 def With_get_body(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -4080,8 +4043,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'type'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'type')
     return space.wrap(w_self.type)
 
 def Raise_set_type(space, w_self, w_new_value):
@@ -4102,8 +4064,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'inst'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'inst')
     return space.wrap(w_self.inst)
 
 def Raise_set_inst(space, w_self, w_new_value):
@@ -4124,8 +4085,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'tback'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'tback')
     return space.wrap(w_self.tback)
 
 def Raise_set_tback(space, w_self, w_new_value):
@@ -4168,14 +4128,13 @@
 def TryExcept_get_body(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -4186,14 +4145,13 @@
 def TryExcept_get_handlers(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'handlers'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'handlers')
     if w_self.w_handlers is None:
         if w_self.handlers is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.handlers]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_handlers = w_list
     return w_self.w_handlers
 
@@ -4204,14 +4162,13 @@
 def TryExcept_get_orelse(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'orelse'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
     if w_self.w_orelse is None:
         if w_self.orelse is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.orelse]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_orelse = w_list
     return w_self.w_orelse
 
@@ -4251,14 +4208,13 @@
 def TryFinally_get_body(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -4269,14 +4225,13 @@
 def TryFinally_get_finalbody(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'finalbody'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'finalbody')
     if w_self.w_finalbody is None:
         if w_self.finalbody is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.finalbody]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_finalbody = w_list
     return w_self.w_finalbody
 
@@ -4318,8 +4273,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'test'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test')
     return space.wrap(w_self.test)
 
 def Assert_set_test(space, w_self, w_new_value):
@@ -4340,8 +4294,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'msg'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'msg')
     return space.wrap(w_self.msg)
 
 def Assert_set_msg(space, w_self, w_new_value):
@@ -4383,14 +4336,13 @@
 def Import_get_names(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'names'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'names')
     if w_self.w_names is None:
         if w_self.names is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.names]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_names = w_list
     return w_self.w_names
 
@@ -4430,8 +4382,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'module'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'module')
     return space.wrap(w_self.module)
 
 def ImportFrom_set_module(space, w_self, w_new_value):
@@ -4451,14 +4402,13 @@
 def ImportFrom_get_names(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'names'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'names')
     if w_self.w_names is None:
         if w_self.names is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.names]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_names = w_list
     return w_self.w_names
 
@@ -4473,8 +4423,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'level'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'level')
     return space.wrap(w_self.level)
 
 def ImportFrom_set_level(space, w_self, w_new_value):
@@ -4522,8 +4471,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     return space.wrap(w_self.body)
 
 def Exec_set_body(space, w_self, w_new_value):
@@ -4544,8 +4492,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'globals'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'globals')
     return space.wrap(w_self.globals)
 
 def Exec_set_globals(space, w_self, w_new_value):
@@ -4566,8 +4513,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'locals'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'locals')
     return space.wrap(w_self.locals)
 
 def Exec_set_locals(space, w_self, w_new_value):
@@ -4610,14 +4556,13 @@
 def Global_get_names(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'names'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'names')
     if w_self.w_names is None:
         if w_self.names is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.names]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_names = w_list
     return w_self.w_names
 
@@ -4657,8 +4602,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Expr_set_value(space, w_self, w_new_value):
@@ -4754,8 +4698,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._lineno_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'lineno'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lineno')
     return space.wrap(w_self.lineno)
 
 def expr_set_lineno(space, w_self, w_new_value):
@@ -4776,8 +4719,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._col_offset_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'col_offset'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'col_offset')
     return space.wrap(w_self.col_offset)
 
 def expr_set_col_offset(space, w_self, w_new_value):
@@ -4807,8 +4749,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'op'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op')
     return boolop_to_class[w_self.op - 1]()
 
 def BoolOp_set_op(space, w_self, w_new_value):
@@ -4827,14 +4768,13 @@
 def BoolOp_get_values(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'values'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'values')
     if w_self.w_values is None:
         if w_self.values is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.values]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_values = w_list
     return w_self.w_values
 
@@ -4875,8 +4815,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'left'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'left')
     return space.wrap(w_self.left)
 
 def BinOp_set_left(space, w_self, w_new_value):
@@ -4897,8 +4836,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'op'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op')
     return operator_to_class[w_self.op - 1]()
 
 def BinOp_set_op(space, w_self, w_new_value):
@@ -4921,8 +4859,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'right'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'right')
     return space.wrap(w_self.right)
 
 def BinOp_set_right(space, w_self, w_new_value):
@@ -4969,8 +4906,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'op'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op')
     return unaryop_to_class[w_self.op - 1]()
 
 def UnaryOp_set_op(space, w_self, w_new_value):
@@ -4993,8 +4929,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'operand'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'operand')
     return space.wrap(w_self.operand)
 
 def UnaryOp_set_operand(space, w_self, w_new_value):
@@ -5040,8 +4975,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'args'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args')
     return space.wrap(w_self.args)
 
 def Lambda_set_args(space, w_self, w_new_value):
@@ -5062,8 +4996,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     return space.wrap(w_self.body)
 
 def Lambda_set_body(space, w_self, w_new_value):
@@ -5109,8 +5042,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'test'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test')
     return space.wrap(w_self.test)
 
 def IfExp_set_test(space, w_self, w_new_value):
@@ -5131,8 +5063,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     return space.wrap(w_self.body)
 
 def IfExp_set_body(space, w_self, w_new_value):
@@ -5153,8 +5084,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'orelse'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
     return space.wrap(w_self.orelse)
 
 def IfExp_set_orelse(space, w_self, w_new_value):
@@ -5197,14 +5127,13 @@
 def Dict_get_keys(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'keys'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'keys')
     if w_self.w_keys is None:
         if w_self.keys is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.keys]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_keys = w_list
     return w_self.w_keys
 
@@ -5215,14 +5144,13 @@
 def Dict_get_values(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'values'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'values')
     if w_self.w_values is None:
         if w_self.values is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.values]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_values = w_list
     return w_self.w_values
 
@@ -5260,14 +5188,13 @@
 def Set_get_elts(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elts'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elts')
     if w_self.w_elts is None:
         if w_self.elts is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.elts]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_elts = w_list
     return w_self.w_elts
 
@@ -5307,8 +5234,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elt'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elt')
     return space.wrap(w_self.elt)
 
 def ListComp_set_elt(space, w_self, w_new_value):
@@ -5325,14 +5251,13 @@
 def ListComp_get_generators(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'generators'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators')
     if w_self.w_generators is None:
         if w_self.generators is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.generators]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_generators = w_list
     return w_self.w_generators
 
@@ -5373,8 +5298,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elt'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elt')
     return space.wrap(w_self.elt)
 
 def SetComp_set_elt(space, w_self, w_new_value):
@@ -5391,14 +5315,13 @@
 def SetComp_get_generators(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'generators'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators')
     if w_self.w_generators is None:
         if w_self.generators is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.generators]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_generators = w_list
     return w_self.w_generators
 
@@ -5439,8 +5362,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'key'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'key')
     return space.wrap(w_self.key)
 
 def DictComp_set_key(space, w_self, w_new_value):
@@ -5461,8 +5383,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def DictComp_set_value(space, w_self, w_new_value):
@@ -5479,14 +5400,13 @@
 def DictComp_get_generators(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'generators'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators')
     if w_self.w_generators is None:
         if w_self.generators is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.generators]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_generators = w_list
     return w_self.w_generators
 
@@ -5528,8 +5448,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elt'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elt')
     return space.wrap(w_self.elt)
 
 def GeneratorExp_set_elt(space, w_self, w_new_value):
@@ -5546,14 +5465,13 @@
 def GeneratorExp_get_generators(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'generators'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators')
     if w_self.w_generators is None:
         if w_self.generators is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.generators]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_generators = w_list
     return w_self.w_generators
 
@@ -5594,8 +5512,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Yield_set_value(space, w_self, w_new_value):
@@ -5640,8 +5557,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'left'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'left')
     return space.wrap(w_self.left)
 
 def Compare_set_left(space, w_self, w_new_value):
@@ -5658,14 +5574,13 @@
 def Compare_get_ops(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ops'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ops')
     if w_self.w_ops is None:
         if w_self.ops is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [cmpop_to_class[node - 1]() for node in w_self.ops]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_ops = w_list
     return w_self.w_ops
 
@@ -5676,14 +5591,13 @@
 def Compare_get_comparators(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'comparators'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'comparators')
     if w_self.w_comparators is None:
         if w_self.comparators is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.comparators]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_comparators = w_list
     return w_self.w_comparators
 
@@ -5726,8 +5640,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'func'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'func')
     return space.wrap(w_self.func)
 
 def Call_set_func(space, w_self, w_new_value):
@@ -5744,14 +5657,13 @@
 def Call_get_args(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'args'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args')
     if w_self.w_args is None:
         if w_self.args is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.args]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_args = w_list
     return w_self.w_args
 
@@ -5762,14 +5674,13 @@
 def Call_get_keywords(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'keywords'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'keywords')
     if w_self.w_keywords is None:
         if w_self.keywords is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.keywords]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_keywords = w_list
     return w_self.w_keywords
 
@@ -5784,8 +5695,7 @@
             return w_obj
     if not w_self.initialization_state & 8:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'starargs'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'starargs')
     return space.wrap(w_self.starargs)
 
 def Call_set_starargs(space, w_self, w_new_value):
@@ -5806,8 +5716,7 @@
             return w_obj
     if not w_self.initialization_state & 16:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'kwargs'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'kwargs')
     return space.wrap(w_self.kwargs)
 
 def Call_set_kwargs(space, w_self, w_new_value):
@@ -5858,8 +5767,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Repr_set_value(space, w_self, w_new_value):
@@ -5904,8 +5812,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'n'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'n')
     return w_self.n
 
 def Num_set_n(space, w_self, w_new_value):
@@ -5950,8 +5857,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 's'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 's')
     return w_self.s
 
 def Str_set_s(space, w_self, w_new_value):
@@ -5996,8 +5902,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Attribute_set_value(space, w_self, w_new_value):
@@ -6018,8 +5923,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'attr'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'attr')
     return space.wrap(w_self.attr)
 
 def Attribute_set_attr(space, w_self, w_new_value):
@@ -6040,8 +5944,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ctx'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def Attribute_set_ctx(space, w_self, w_new_value):
@@ -6090,8 +5993,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Subscript_set_value(space, w_self, w_new_value):
@@ -6112,8 +6014,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'slice'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'slice')
     return space.wrap(w_self.slice)
 
 def Subscript_set_slice(space, w_self, w_new_value):
@@ -6134,8 +6035,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ctx'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def Subscript_set_ctx(space, w_self, w_new_value):
@@ -6184,8 +6084,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'id'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'id')
     return space.wrap(w_self.id)
 
 def Name_set_id(space, w_self, w_new_value):
@@ -6206,8 +6105,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ctx'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def Name_set_ctx(space, w_self, w_new_value):
@@ -6251,14 +6149,13 @@
 def List_get_elts(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elts'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elts')
     if w_self.w_elts is None:
         if w_self.elts is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.elts]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_elts = w_list
     return w_self.w_elts
 
@@ -6273,8 +6170,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ctx'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def List_set_ctx(space, w_self, w_new_value):
@@ -6319,14 +6215,13 @@
 def Tuple_get_elts(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elts'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elts')
     if w_self.w_elts is None:
         if w_self.elts is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.elts]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_elts = w_list
     return w_self.w_elts
 
@@ -6341,8 +6236,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ctx'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def Tuple_set_ctx(space, w_self, w_new_value):
@@ -6391,8 +6285,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return w_self.value
 
 def Const_set_value(space, w_self, w_new_value):
@@ -6510,8 +6403,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'lower'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lower')
     return space.wrap(w_self.lower)
 
 def Slice_set_lower(space, w_self, w_new_value):
@@ -6532,8 +6424,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'upper'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'upper')
     return space.wrap(w_self.upper)
 
 def Slice_set_upper(space, w_self, w_new_value):
@@ -6554,8 +6445,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'step'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'step')
     return space.wrap(w_self.step)
 
 def Slice_set_step(space, w_self, w_new_value):
@@ -6598,14 +6488,13 @@
 def ExtSlice_get_dims(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'dims'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'dims')
     if w_self.w_dims is None:
         if w_self.dims is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.dims]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_dims = w_list
     return w_self.w_dims
 
@@ -6645,8 +6534,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Index_set_value(space, w_self, w_new_value):
@@ -6915,8 +6803,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'target'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'target')
     return space.wrap(w_self.target)
 
 def comprehension_set_target(space, w_self, w_new_value):
@@ -6937,8 +6824,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'iter'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'iter')
     return space.wrap(w_self.iter)
 
 def comprehension_set_iter(space, w_self, w_new_value):
@@ -6955,14 +6841,13 @@
 def comprehension_get_ifs(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ifs'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ifs')
     if w_self.w_ifs is None:
         if w_self.ifs is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.ifs]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_ifs = w_list
     return w_self.w_ifs
 
@@ -7004,8 +6889,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._lineno_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'lineno'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lineno')
     return space.wrap(w_self.lineno)
 
 def excepthandler_set_lineno(space, w_self, w_new_value):
@@ -7026,8 +6910,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._col_offset_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'col_offset'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'col_offset')
     return space.wrap(w_self.col_offset)
 
 def excepthandler_set_col_offset(space, w_self, w_new_value):
@@ -7057,8 +6940,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'type'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'type')
     return space.wrap(w_self.type)
 
 def ExceptHandler_set_type(space, w_self, w_new_value):
@@ -7079,8 +6961,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'name'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name')
     return space.wrap(w_self.name)
 
 def ExceptHandler_set_name(space, w_self, w_new_value):
@@ -7097,14 +6978,13 @@
 def ExceptHandler_get_body(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -7142,14 +7022,13 @@
 def arguments_get_args(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'args'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args')
     if w_self.w_args is None:
         if w_self.args is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.args]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_args = w_list
     return w_self.w_args
 
@@ -7164,8 +7043,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'vararg'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'vararg')
     return space.wrap(w_self.vararg)
 
 def arguments_set_vararg(space, w_self, w_new_value):
@@ -7189,8 +7067,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'kwarg'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'kwarg')
     return space.wrap(w_self.kwarg)
 
 def arguments_set_kwarg(space, w_self, w_new_value):
@@ -7210,14 +7087,13 @@
 def arguments_get_defaults(space, w_self):
     if not w_self.initialization_state & 8:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'defaults'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'defaults')
     if w_self.w_defaults is None:
         if w_self.defaults is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.defaults]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_defaults = w_list
     return w_self.w_defaults
 
@@ -7261,8 +7137,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'arg'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'arg')
     return space.wrap(w_self.arg)
 
 def keyword_set_arg(space, w_self, w_new_value):
@@ -7283,8 +7158,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def keyword_set_value(space, w_self, w_new_value):
@@ -7330,8 +7204,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'name'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name')
     return space.wrap(w_self.name)
 
 def alias_set_name(space, w_self, w_new_value):
@@ -7352,8 +7225,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'asname'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'asname')
     return space.wrap(w_self.asname)
 
 def alias_set_asname(space, w_self, w_new_value):
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -414,13 +414,12 @@
             self.emit("        return w_obj", 1)
         self.emit("if not w_self.initialization_state & %s:" % (flag,), 1)
         self.emit("typename = space.type(w_self).getname(space)", 2)
-        self.emit("w_err = space.wrap(\"'%%s' object has no attribute '%s'\" %% typename)" %
+        self.emit("raise operationerrfmt(space.w_AttributeError, \"'%%s' object has no attribute '%%s'\", typename, '%s')" %
                   (field.name,), 2)
-        self.emit("raise OperationError(space.w_AttributeError, w_err)", 2)
         if field.seq:
             self.emit("if w_self.w_%s is None:" % (field.name,), 1)
             self.emit("if w_self.%s is None:" % (field.name,), 2)
-            self.emit("w_list = space.newlist([])", 3)
+            self.emit("list_w = []", 3)
             self.emit("else:", 2)
             if field.type.value in self.data.simple_types:
                 wrapper = "%s_to_class[node - 1]()" % (field.type,)
@@ -428,7 +427,7 @@
                 wrapper = "space.wrap(node)"
             self.emit("list_w = [%s for node in w_self.%s]" %
                       (wrapper, field.name), 3)
-            self.emit("w_list = space.newlist(list_w)", 3)
+            self.emit("w_list = space.newlist(list_w)", 2)
             self.emit("w_self.w_%s = w_list" % (field.name,), 2)
             self.emit("return w_self.w_%s" % (field.name,), 1)
         elif field.type.value in self.data.simple_types:
@@ -540,7 +539,7 @@
 from pypy.interpreter.baseobjspace import Wrappable
 from pypy.interpreter import typedef
 from pypy.interpreter.gateway import interp2app
-from pypy.interpreter.error import OperationError
+from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.tool.pairtype import extendabletype
 from pypy.tool.sourcetools import func_with_new_name
@@ -639,9 +638,7 @@
             missing = required[i]
             if missing is not None:
                  err = "required field \\"%s\\" missing from %s"
-                 err = err % (missing, host)
-                 w_err = space.wrap(err)
-                 raise OperationError(space.w_TypeError, w_err)
+                 raise operationerrfmt(space.w_TypeError, err, missing, host)
     raise AssertionError("should not reach here")
 
 

From noreply at buildbot.pypy.org  Fri Oct 21 17:58:34 2011
From: noreply at buildbot.pypy.org (cfbolz)
Date: Fri, 21 Oct 2011 17:58:34 +0200 (CEST)
Subject: [pypy-commit] pypy default: (arigo,
 cfbolz): add sanity checks to make sure that tagged pointers are
Message-ID: <20111021155834.6773E820D7@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r48319:d33ecc0a1ac0
Date: 2011-10-21 17:58 +0200
http://bitbucket.org/pypy/pypy/changeset/d33ecc0a1ac0/

Log:	(arigo, cfbolz): add sanity checks to make sure that tagged pointers
	are enabled when erase_int is used.

diff --git a/pypy/rlib/rerased.py b/pypy/rlib/rerased.py
--- a/pypy/rlib/rerased.py
+++ b/pypy/rlib/rerased.py
@@ -135,6 +135,8 @@
     _about_ = erase_int
 
     def compute_result_annotation(self, s_obj):
+        config = self.bookkeeper.annotator.translator.config
+        assert config.translation.taggedpointers, "need to enable tagged pointers to use erase_int"
         assert annmodel.SomeInteger().contains(s_obj)
         return SomeErased()
 
@@ -228,6 +230,8 @@
 
     def convert_const(self, value):
         if value._identity is _identity_for_ints:
+            config = self.rtyper.annotator.translator.config
+            assert config.translation.taggedpointers, "need to enable tagged pointers to use erase_int"
             return lltype.cast_int_to_ptr(self.lowleveltype, value._x * 2 + 1)
         bk = self.rtyper.annotator.bookkeeper
         s_obj = value._identity.get_input_annotation(bk)
diff --git a/pypy/rlib/test/test_rerased.py b/pypy/rlib/test/test_rerased.py
--- a/pypy/rlib/test/test_rerased.py
+++ b/pypy/rlib/test/test_rerased.py
@@ -10,6 +10,13 @@
 
 from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin
 
+def make_annotator():
+    a = RPythonAnnotator()
+    a.translator.config.translation.taggedpointers = True
+    return a
+
+
+
 class X(object):
     pass
 
@@ -55,7 +62,7 @@
 def test_annotate_1():
     def f():
         return eraseX(X())
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [])
     assert isinstance(s, SomeErased)
 
@@ -66,7 +73,7 @@
         #assert not is_integer(e)
         x2 = uneraseX(e)
         return x2
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [])
     assert isinstance(s, annmodel.SomeInstance)
     assert s.classdef == a.bookkeeper.getuniqueclassdef(X)
@@ -77,7 +84,7 @@
         #assert is_integer(e)
         x2 = unerase_int(e)
         return x2
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [])
     assert isinstance(s, annmodel.SomeInteger)
 
@@ -105,7 +112,7 @@
         x = make(n)
         return check(x, n)
     #
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [int])
     assert isinstance(s, annmodel.SomeInteger)
 
@@ -135,7 +142,7 @@
             else:
                 return inst
     #
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [])
     assert isinstance(s, annmodel.SomeInstance)
     assert s.classdef == a.bookkeeper.getuniqueclassdef(A)
@@ -155,7 +162,7 @@
             e = e2
         return unerase(e)
     #
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [int])
     assert isinstance(s, annmodel.SomeInstance)
     assert s.classdef == a.bookkeeper.getuniqueclassdef(X)
@@ -165,11 +172,14 @@
     e1 = erase_int(42)
     def f(i):
         return unerase_int(e1)
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [int])
     assert isinstance(s, annmodel.SomeInteger)
 
 class BaseTestRErased(BaseRtypingTest):
+    def interpret(self, *args, **kwargs):
+        kwargs["taggedpointers"] = True
+        return BaseRtypingTest.interpret(self, *args, **kwargs)
     def test_rtype_1(self):
         def f():
             return eraseX(X())

From noreply at buildbot.pypy.org  Fri Oct 21 22:27:49 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 21 Oct 2011 22:27:49 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: Add Sam Lade.
Message-ID: <20111021202749.BDDD6820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r3937:e6950a6c5199
Date: 2011-10-21 22:27 +0200
http://bitbucket.org/pypy/extradoc/changeset/e6950a6c5199/

Log:	Add Sam Lade.

diff --git a/sprintinfo/gothenburg-2011-2/people.txt b/sprintinfo/gothenburg-2011-2/people.txt
--- a/sprintinfo/gothenburg-2011-2/people.txt
+++ b/sprintinfo/gothenburg-2011-2/people.txt
@@ -13,4 +13,5 @@
 Armin Rigo           ?              ?
 Antonio Cuni         4-10 november  Laura's and Jacob's
 H&#229;kan Ard&#246;           3-8            ?
+Sam Lade             1-9            Hotel Poseidon
 ==================== ============== ===================== ==================

From noreply at buildbot.pypy.org  Fri Oct 21 23:02:01 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 21 Oct 2011 23:02:01 +0200 (CEST)
Subject: [pypy-commit] pypy inline-dict-ops: Add missing setannotation(..).
Message-ID: <20111021210201.107CF820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: inline-dict-ops
Changeset: r48320:f5633dceb9bf
Date: 2011-10-21 23:01 +0200
http://bitbucket.org/pypy/pypy/changeset/f5633dceb9bf/

Log:	Add missing setannotation(..).

diff --git a/pypy/jit/backend/llgraph/llimpl.py b/pypy/jit/backend/llgraph/llimpl.py
--- a/pypy/jit/backend/llgraph/llimpl.py
+++ b/pypy/jit/backend/llgraph/llimpl.py
@@ -1791,6 +1791,9 @@
 setannotation(do_getfield_raw_int, annmodel.SomeInteger())
 setannotation(do_getfield_raw_ptr, annmodel.SomePtr(llmemory.GCREF))
 setannotation(do_getfield_raw_float, s_FloatStorage)
+setannotation(do_getinteriorfield_gc_int, annmodel.SomeInteger())
+setannotation(do_getinteriorfield_gc_ptr, annmodel.SomePtr(llmemory.GCREF))
+setannotation(do_getinteriorfield_gc_float, s_FloatStorage)
 setannotation(do_new, annmodel.SomePtr(llmemory.GCREF))
 setannotation(do_new_array, annmodel.SomePtr(llmemory.GCREF))
 setannotation(do_setarrayitem_gc_int, annmodel.s_None)
@@ -1804,6 +1807,9 @@
 setannotation(do_setfield_raw_int, annmodel.s_None)
 setannotation(do_setfield_raw_ptr, annmodel.s_None)
 setannotation(do_setfield_raw_float, annmodel.s_None)
+setannotation(do_setinteriorfield_gc_int, annmodel.s_None)
+setannotation(do_setinteriorfield_gc_ptr, annmodel.s_None)
+setannotation(do_setinteriorfield_gc_float, annmodel.s_None)
 setannotation(do_newstr, annmodel.SomePtr(llmemory.GCREF))
 setannotation(do_strsetitem, annmodel.s_None)
 setannotation(do_newunicode, annmodel.SomePtr(llmemory.GCREF))

From noreply at buildbot.pypy.org  Fri Oct 21 23:13:01 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 21 Oct 2011 23:13:01 +0200 (CEST)
Subject: [pypy-commit] pypy inline-dict-ops: Revert d0bcf756f012,
 which breaks test_ztranslation.  Will fix
Message-ID: <20111021211301.88A70820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: inline-dict-ops
Changeset: r48321:0c7d048ff30c
Date: 2011-10-21 23:12 +0200
http://bitbucket.org/pypy/pypy/changeset/0c7d048ff30c/

Log:	Revert d0bcf756f012, which breaks test_ztranslation. Will fix
	test_zrpy_gc in a way that is more consistent with the existing
	workarounds.

diff --git a/pypy/jit/codewriter/assembler.py b/pypy/jit/codewriter/assembler.py
--- a/pypy/jit/codewriter/assembler.py
+++ b/pypy/jit/codewriter/assembler.py
@@ -246,13 +246,9 @@
             addr = llmemory.cast_ptr_to_adr(value)
             self.list_of_addr2name.append((addr, name))
 
-    def finished(self, callinfocollection, cpu):
+    def finished(self, callinfocollection):
         # Helper called at the end of assembling.  Registers the extra
         # functions shown in _callinfo_for_oopspec.
         for func in callinfocollection.all_function_addresses_as_int():
             func = heaptracker.int2adr(func)
             self.see_raw_object(func.ptr)
-        # register at least one interiorfielddescr
-        if hasattr(cpu, 'interiorfielddescrof'):
-            _T = lltype.GcArray(lltype.Struct('x', ('field', lltype.Signed)))
-            self.descrs.append(cpu.interiorfielddescrof(_T, 'field'))
diff --git a/pypy/jit/codewriter/codewriter.py b/pypy/jit/codewriter/codewriter.py
--- a/pypy/jit/codewriter/codewriter.py
+++ b/pypy/jit/codewriter/codewriter.py
@@ -73,7 +73,7 @@
             count += 1
             if not count % 500:
                 log.info("Produced %d jitcodes" % count)
-        self.assembler.finished(self.callcontrol.callinfocollection, self.cpu)
+        self.assembler.finished(self.callcontrol.callinfocollection)
         heaptracker.finish_registering(self.cpu)
         log.info("there are %d JitCode instances." % count)
 

From noreply at buildbot.pypy.org  Fri Oct 21 23:15:19 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 21 Oct 2011 23:15:19 +0200 (CEST)
Subject: [pypy-commit] pypy inline-dict-ops: Workaround: add 'arraydescr'
 and 'fielddescr' on the class too, 
Message-ID: <20111021211519.752C1820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: inline-dict-ops
Changeset: r48322:60b28b27a311
Date: 2011-10-21 23:14 +0200
http://bitbucket.org/pypy/pypy/changeset/60b28b27a311/

Log:	Workaround: add 'arraydescr' and 'fielddescr' on the class too, like
	we do in other classes, for tests where the annotator doesn't see an
	actual instance.

diff --git a/pypy/jit/backend/llsupport/descr.py b/pypy/jit/backend/llsupport/descr.py
--- a/pypy/jit/backend/llsupport/descr.py
+++ b/pypy/jit/backend/llsupport/descr.py
@@ -141,23 +141,6 @@
         return fielddescr
 
 # ____________________________________________________________
-# InteriorFieldDescr
-
-class InteriorFieldDescr(AbstractDescr):
-    def __init__(self, arraydescr, fielddescr):
-        self.arraydescr = arraydescr
-        self.fielddescr = fielddescr
-
-    def is_pointer_field(self):
-        return self.fielddescr.is_pointer_field()
-
-    def is_float_field(self):
-        return self.fielddescr.is_float_field()
-
-    def repr_of_descr(self):
-        return '<InteriorFieldDescr %s>' % self.fielddescr.repr_of_descr()
-
-# ____________________________________________________________
 # ArrayDescrs
 
 _A = lltype.GcArray(lltype.Signed)     # a random gcarray
@@ -241,17 +224,6 @@
                          NonGcPtrArrayNoLengthDescr, 'ArrayNoLength', 'get_item_size',
                          '_is_array_of_floats', '_is_item_signed')
 
-def get_interiorfield_descr(gc_ll_descr, ARRAY, FIELDTP, name):
-    cache = gc_ll_descr._cache_interiorfield
-    try:
-        return cache[(ARRAY, FIELDTP, name)]
-    except KeyError:
-        arraydescr = get_array_descr(gc_ll_descr, ARRAY)
-        fielddescr = get_field_descr(gc_ll_descr, FIELDTP, name)
-        descr = InteriorFieldDescr(arraydescr, fielddescr)
-        cache[(ARRAY, FIELDTP, name)] = descr
-        return descr
-
 def get_array_descr(gccache, ARRAY):
     cache = gccache._cache_array
     try:
@@ -277,6 +249,36 @@
         cache[ARRAY] = arraydescr
         return arraydescr
 
+# ____________________________________________________________
+# InteriorFieldDescr
+
+class InteriorFieldDescr(AbstractDescr):
+    arraydescr = BaseArrayDescr()     # workaround for the annotator
+    fielddescr = BaseFieldDescr('', 0)
+
+    def __init__(self, arraydescr, fielddescr):
+        self.arraydescr = arraydescr
+        self.fielddescr = fielddescr
+
+    def is_pointer_field(self):
+        return self.fielddescr.is_pointer_field()
+
+    def is_float_field(self):
+        return self.fielddescr.is_float_field()
+
+    def repr_of_descr(self):
+        return '<InteriorFieldDescr %s>' % self.fielddescr.repr_of_descr()
+
+def get_interiorfield_descr(gc_ll_descr, ARRAY, FIELDTP, name):
+    cache = gc_ll_descr._cache_interiorfield
+    try:
+        return cache[(ARRAY, FIELDTP, name)]
+    except KeyError:
+        arraydescr = get_array_descr(gc_ll_descr, ARRAY)
+        fielddescr = get_field_descr(gc_ll_descr, FIELDTP, name)
+        descr = InteriorFieldDescr(arraydescr, fielddescr)
+        cache[(ARRAY, FIELDTP, name)] = descr
+        return descr
 
 # ____________________________________________________________
 # CallDescrs

From noreply at buildbot.pypy.org  Fri Oct 21 23:24:54 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Fri, 21 Oct 2011 23:24:54 +0200 (CEST)
Subject: [pypy-commit] pypy inline-dict-ops: merged in default
Message-ID: <20111021212454.73908820D7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: inline-dict-ops
Changeset: r48323:8ec8f9420262
Date: 2011-10-21 14:23 -0700
http://bitbucket.org/pypy/pypy/changeset/8ec8f9420262/

Log:	merged in default

diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -2,7 +2,7 @@
 from pypy.interpreter.baseobjspace import Wrappable
 from pypy.interpreter import typedef
 from pypy.interpreter.gateway import interp2app
-from pypy.interpreter.error import OperationError
+from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.tool.pairtype import extendabletype
 from pypy.tool.sourcetools import func_with_new_name
@@ -2925,14 +2925,13 @@
 def Module_get_body(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -2968,14 +2967,13 @@
 def Interactive_get_body(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3015,8 +3013,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     return space.wrap(w_self.body)
 
 def Expression_set_body(space, w_self, w_new_value):
@@ -3057,14 +3054,13 @@
 def Suite_get_body(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3104,8 +3100,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._lineno_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'lineno'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lineno')
     return space.wrap(w_self.lineno)
 
 def stmt_set_lineno(space, w_self, w_new_value):
@@ -3126,8 +3121,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._col_offset_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'col_offset'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'col_offset')
     return space.wrap(w_self.col_offset)
 
 def stmt_set_col_offset(space, w_self, w_new_value):
@@ -3157,8 +3151,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'name'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name')
     return space.wrap(w_self.name)
 
 def FunctionDef_set_name(space, w_self, w_new_value):
@@ -3179,8 +3172,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'args'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args')
     return space.wrap(w_self.args)
 
 def FunctionDef_set_args(space, w_self, w_new_value):
@@ -3197,14 +3189,13 @@
 def FunctionDef_get_body(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3215,14 +3206,13 @@
 def FunctionDef_get_decorator_list(space, w_self):
     if not w_self.initialization_state & 8:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'decorator_list'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'decorator_list')
     if w_self.w_decorator_list is None:
         if w_self.decorator_list is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.decorator_list]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_decorator_list = w_list
     return w_self.w_decorator_list
 
@@ -3266,8 +3256,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'name'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name')
     return space.wrap(w_self.name)
 
 def ClassDef_set_name(space, w_self, w_new_value):
@@ -3284,14 +3273,13 @@
 def ClassDef_get_bases(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'bases'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'bases')
     if w_self.w_bases is None:
         if w_self.bases is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.bases]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_bases = w_list
     return w_self.w_bases
 
@@ -3302,14 +3290,13 @@
 def ClassDef_get_body(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3320,14 +3307,13 @@
 def ClassDef_get_decorator_list(space, w_self):
     if not w_self.initialization_state & 8:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'decorator_list'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'decorator_list')
     if w_self.w_decorator_list is None:
         if w_self.decorator_list is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.decorator_list]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_decorator_list = w_list
     return w_self.w_decorator_list
 
@@ -3372,8 +3358,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Return_set_value(space, w_self, w_new_value):
@@ -3414,14 +3399,13 @@
 def Delete_get_targets(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'targets'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'targets')
     if w_self.w_targets is None:
         if w_self.targets is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.targets]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_targets = w_list
     return w_self.w_targets
 
@@ -3457,14 +3441,13 @@
 def Assign_get_targets(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'targets'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'targets')
     if w_self.w_targets is None:
         if w_self.targets is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.targets]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_targets = w_list
     return w_self.w_targets
 
@@ -3479,8 +3462,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Assign_set_value(space, w_self, w_new_value):
@@ -3527,8 +3509,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'target'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'target')
     return space.wrap(w_self.target)
 
 def AugAssign_set_target(space, w_self, w_new_value):
@@ -3549,8 +3530,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'op'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op')
     return operator_to_class[w_self.op - 1]()
 
 def AugAssign_set_op(space, w_self, w_new_value):
@@ -3573,8 +3553,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def AugAssign_set_value(space, w_self, w_new_value):
@@ -3621,8 +3600,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'dest'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'dest')
     return space.wrap(w_self.dest)
 
 def Print_set_dest(space, w_self, w_new_value):
@@ -3639,14 +3617,13 @@
 def Print_get_values(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'values'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'values')
     if w_self.w_values is None:
         if w_self.values is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.values]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_values = w_list
     return w_self.w_values
 
@@ -3661,8 +3638,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'nl'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'nl')
     return space.wrap(w_self.nl)
 
 def Print_set_nl(space, w_self, w_new_value):
@@ -3710,8 +3686,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'target'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'target')
     return space.wrap(w_self.target)
 
 def For_set_target(space, w_self, w_new_value):
@@ -3732,8 +3707,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'iter'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'iter')
     return space.wrap(w_self.iter)
 
 def For_set_iter(space, w_self, w_new_value):
@@ -3750,14 +3724,13 @@
 def For_get_body(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3768,14 +3741,13 @@
 def For_get_orelse(space, w_self):
     if not w_self.initialization_state & 8:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'orelse'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
     if w_self.w_orelse is None:
         if w_self.orelse is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.orelse]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_orelse = w_list
     return w_self.w_orelse
 
@@ -3819,8 +3791,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'test'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test')
     return space.wrap(w_self.test)
 
 def While_set_test(space, w_self, w_new_value):
@@ -3837,14 +3808,13 @@
 def While_get_body(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3855,14 +3825,13 @@
 def While_get_orelse(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'orelse'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
     if w_self.w_orelse is None:
         if w_self.orelse is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.orelse]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_orelse = w_list
     return w_self.w_orelse
 
@@ -3905,8 +3874,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'test'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test')
     return space.wrap(w_self.test)
 
 def If_set_test(space, w_self, w_new_value):
@@ -3923,14 +3891,13 @@
 def If_get_body(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3941,14 +3908,13 @@
 def If_get_orelse(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'orelse'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
     if w_self.w_orelse is None:
         if w_self.orelse is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.orelse]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_orelse = w_list
     return w_self.w_orelse
 
@@ -3991,8 +3957,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'context_expr'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'context_expr')
     return space.wrap(w_self.context_expr)
 
 def With_set_context_expr(space, w_self, w_new_value):
@@ -4013,8 +3978,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'optional_vars'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'optional_vars')
     return space.wrap(w_self.optional_vars)
 
 def With_set_optional_vars(space, w_self, w_new_value):
@@ -4031,14 +3995,13 @@
 def With_get_body(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -4080,8 +4043,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'type'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'type')
     return space.wrap(w_self.type)
 
 def Raise_set_type(space, w_self, w_new_value):
@@ -4102,8 +4064,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'inst'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'inst')
     return space.wrap(w_self.inst)
 
 def Raise_set_inst(space, w_self, w_new_value):
@@ -4124,8 +4085,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'tback'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'tback')
     return space.wrap(w_self.tback)
 
 def Raise_set_tback(space, w_self, w_new_value):
@@ -4168,14 +4128,13 @@
 def TryExcept_get_body(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -4186,14 +4145,13 @@
 def TryExcept_get_handlers(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'handlers'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'handlers')
     if w_self.w_handlers is None:
         if w_self.handlers is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.handlers]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_handlers = w_list
     return w_self.w_handlers
 
@@ -4204,14 +4162,13 @@
 def TryExcept_get_orelse(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'orelse'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
     if w_self.w_orelse is None:
         if w_self.orelse is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.orelse]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_orelse = w_list
     return w_self.w_orelse
 
@@ -4251,14 +4208,13 @@
 def TryFinally_get_body(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -4269,14 +4225,13 @@
 def TryFinally_get_finalbody(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'finalbody'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'finalbody')
     if w_self.w_finalbody is None:
         if w_self.finalbody is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.finalbody]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_finalbody = w_list
     return w_self.w_finalbody
 
@@ -4318,8 +4273,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'test'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test')
     return space.wrap(w_self.test)
 
 def Assert_set_test(space, w_self, w_new_value):
@@ -4340,8 +4294,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'msg'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'msg')
     return space.wrap(w_self.msg)
 
 def Assert_set_msg(space, w_self, w_new_value):
@@ -4383,14 +4336,13 @@
 def Import_get_names(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'names'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'names')
     if w_self.w_names is None:
         if w_self.names is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.names]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_names = w_list
     return w_self.w_names
 
@@ -4430,8 +4382,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'module'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'module')
     return space.wrap(w_self.module)
 
 def ImportFrom_set_module(space, w_self, w_new_value):
@@ -4451,14 +4402,13 @@
 def ImportFrom_get_names(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'names'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'names')
     if w_self.w_names is None:
         if w_self.names is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.names]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_names = w_list
     return w_self.w_names
 
@@ -4473,8 +4423,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'level'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'level')
     return space.wrap(w_self.level)
 
 def ImportFrom_set_level(space, w_self, w_new_value):
@@ -4522,8 +4471,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     return space.wrap(w_self.body)
 
 def Exec_set_body(space, w_self, w_new_value):
@@ -4544,8 +4492,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'globals'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'globals')
     return space.wrap(w_self.globals)
 
 def Exec_set_globals(space, w_self, w_new_value):
@@ -4566,8 +4513,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'locals'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'locals')
     return space.wrap(w_self.locals)
 
 def Exec_set_locals(space, w_self, w_new_value):
@@ -4610,14 +4556,13 @@
 def Global_get_names(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'names'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'names')
     if w_self.w_names is None:
         if w_self.names is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.names]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_names = w_list
     return w_self.w_names
 
@@ -4657,8 +4602,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Expr_set_value(space, w_self, w_new_value):
@@ -4754,8 +4698,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._lineno_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'lineno'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lineno')
     return space.wrap(w_self.lineno)
 
 def expr_set_lineno(space, w_self, w_new_value):
@@ -4776,8 +4719,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._col_offset_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'col_offset'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'col_offset')
     return space.wrap(w_self.col_offset)
 
 def expr_set_col_offset(space, w_self, w_new_value):
@@ -4807,8 +4749,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'op'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op')
     return boolop_to_class[w_self.op - 1]()
 
 def BoolOp_set_op(space, w_self, w_new_value):
@@ -4827,14 +4768,13 @@
 def BoolOp_get_values(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'values'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'values')
     if w_self.w_values is None:
         if w_self.values is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.values]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_values = w_list
     return w_self.w_values
 
@@ -4875,8 +4815,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'left'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'left')
     return space.wrap(w_self.left)
 
 def BinOp_set_left(space, w_self, w_new_value):
@@ -4897,8 +4836,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'op'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op')
     return operator_to_class[w_self.op - 1]()
 
 def BinOp_set_op(space, w_self, w_new_value):
@@ -4921,8 +4859,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'right'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'right')
     return space.wrap(w_self.right)
 
 def BinOp_set_right(space, w_self, w_new_value):
@@ -4969,8 +4906,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'op'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op')
     return unaryop_to_class[w_self.op - 1]()
 
 def UnaryOp_set_op(space, w_self, w_new_value):
@@ -4993,8 +4929,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'operand'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'operand')
     return space.wrap(w_self.operand)
 
 def UnaryOp_set_operand(space, w_self, w_new_value):
@@ -5040,8 +4975,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'args'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args')
     return space.wrap(w_self.args)
 
 def Lambda_set_args(space, w_self, w_new_value):
@@ -5062,8 +4996,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     return space.wrap(w_self.body)
 
 def Lambda_set_body(space, w_self, w_new_value):
@@ -5109,8 +5042,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'test'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test')
     return space.wrap(w_self.test)
 
 def IfExp_set_test(space, w_self, w_new_value):
@@ -5131,8 +5063,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     return space.wrap(w_self.body)
 
 def IfExp_set_body(space, w_self, w_new_value):
@@ -5153,8 +5084,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'orelse'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
     return space.wrap(w_self.orelse)
 
 def IfExp_set_orelse(space, w_self, w_new_value):
@@ -5197,14 +5127,13 @@
 def Dict_get_keys(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'keys'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'keys')
     if w_self.w_keys is None:
         if w_self.keys is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.keys]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_keys = w_list
     return w_self.w_keys
 
@@ -5215,14 +5144,13 @@
 def Dict_get_values(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'values'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'values')
     if w_self.w_values is None:
         if w_self.values is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.values]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_values = w_list
     return w_self.w_values
 
@@ -5260,14 +5188,13 @@
 def Set_get_elts(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elts'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elts')
     if w_self.w_elts is None:
         if w_self.elts is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.elts]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_elts = w_list
     return w_self.w_elts
 
@@ -5307,8 +5234,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elt'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elt')
     return space.wrap(w_self.elt)
 
 def ListComp_set_elt(space, w_self, w_new_value):
@@ -5325,14 +5251,13 @@
 def ListComp_get_generators(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'generators'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators')
     if w_self.w_generators is None:
         if w_self.generators is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.generators]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_generators = w_list
     return w_self.w_generators
 
@@ -5373,8 +5298,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elt'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elt')
     return space.wrap(w_self.elt)
 
 def SetComp_set_elt(space, w_self, w_new_value):
@@ -5391,14 +5315,13 @@
 def SetComp_get_generators(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'generators'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators')
     if w_self.w_generators is None:
         if w_self.generators is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.generators]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_generators = w_list
     return w_self.w_generators
 
@@ -5439,8 +5362,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'key'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'key')
     return space.wrap(w_self.key)
 
 def DictComp_set_key(space, w_self, w_new_value):
@@ -5461,8 +5383,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def DictComp_set_value(space, w_self, w_new_value):
@@ -5479,14 +5400,13 @@
 def DictComp_get_generators(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'generators'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators')
     if w_self.w_generators is None:
         if w_self.generators is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.generators]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_generators = w_list
     return w_self.w_generators
 
@@ -5528,8 +5448,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elt'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elt')
     return space.wrap(w_self.elt)
 
 def GeneratorExp_set_elt(space, w_self, w_new_value):
@@ -5546,14 +5465,13 @@
 def GeneratorExp_get_generators(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'generators'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators')
     if w_self.w_generators is None:
         if w_self.generators is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.generators]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_generators = w_list
     return w_self.w_generators
 
@@ -5594,8 +5512,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Yield_set_value(space, w_self, w_new_value):
@@ -5640,8 +5557,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'left'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'left')
     return space.wrap(w_self.left)
 
 def Compare_set_left(space, w_self, w_new_value):
@@ -5658,14 +5574,13 @@
 def Compare_get_ops(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ops'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ops')
     if w_self.w_ops is None:
         if w_self.ops is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [cmpop_to_class[node - 1]() for node in w_self.ops]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_ops = w_list
     return w_self.w_ops
 
@@ -5676,14 +5591,13 @@
 def Compare_get_comparators(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'comparators'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'comparators')
     if w_self.w_comparators is None:
         if w_self.comparators is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.comparators]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_comparators = w_list
     return w_self.w_comparators
 
@@ -5726,8 +5640,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'func'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'func')
     return space.wrap(w_self.func)
 
 def Call_set_func(space, w_self, w_new_value):
@@ -5744,14 +5657,13 @@
 def Call_get_args(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'args'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args')
     if w_self.w_args is None:
         if w_self.args is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.args]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_args = w_list
     return w_self.w_args
 
@@ -5762,14 +5674,13 @@
 def Call_get_keywords(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'keywords'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'keywords')
     if w_self.w_keywords is None:
         if w_self.keywords is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.keywords]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_keywords = w_list
     return w_self.w_keywords
 
@@ -5784,8 +5695,7 @@
             return w_obj
     if not w_self.initialization_state & 8:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'starargs'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'starargs')
     return space.wrap(w_self.starargs)
 
 def Call_set_starargs(space, w_self, w_new_value):
@@ -5806,8 +5716,7 @@
             return w_obj
     if not w_self.initialization_state & 16:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'kwargs'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'kwargs')
     return space.wrap(w_self.kwargs)
 
 def Call_set_kwargs(space, w_self, w_new_value):
@@ -5858,8 +5767,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Repr_set_value(space, w_self, w_new_value):
@@ -5904,8 +5812,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'n'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'n')
     return w_self.n
 
 def Num_set_n(space, w_self, w_new_value):
@@ -5950,8 +5857,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 's'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 's')
     return w_self.s
 
 def Str_set_s(space, w_self, w_new_value):
@@ -5996,8 +5902,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Attribute_set_value(space, w_self, w_new_value):
@@ -6018,8 +5923,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'attr'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'attr')
     return space.wrap(w_self.attr)
 
 def Attribute_set_attr(space, w_self, w_new_value):
@@ -6040,8 +5944,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ctx'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def Attribute_set_ctx(space, w_self, w_new_value):
@@ -6090,8 +5993,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Subscript_set_value(space, w_self, w_new_value):
@@ -6112,8 +6014,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'slice'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'slice')
     return space.wrap(w_self.slice)
 
 def Subscript_set_slice(space, w_self, w_new_value):
@@ -6134,8 +6035,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ctx'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def Subscript_set_ctx(space, w_self, w_new_value):
@@ -6184,8 +6084,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'id'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'id')
     return space.wrap(w_self.id)
 
 def Name_set_id(space, w_self, w_new_value):
@@ -6206,8 +6105,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ctx'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def Name_set_ctx(space, w_self, w_new_value):
@@ -6251,14 +6149,13 @@
 def List_get_elts(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elts'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elts')
     if w_self.w_elts is None:
         if w_self.elts is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.elts]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_elts = w_list
     return w_self.w_elts
 
@@ -6273,8 +6170,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ctx'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def List_set_ctx(space, w_self, w_new_value):
@@ -6319,14 +6215,13 @@
 def Tuple_get_elts(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elts'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elts')
     if w_self.w_elts is None:
         if w_self.elts is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.elts]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_elts = w_list
     return w_self.w_elts
 
@@ -6341,8 +6236,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ctx'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def Tuple_set_ctx(space, w_self, w_new_value):
@@ -6391,8 +6285,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return w_self.value
 
 def Const_set_value(space, w_self, w_new_value):
@@ -6510,8 +6403,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'lower'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lower')
     return space.wrap(w_self.lower)
 
 def Slice_set_lower(space, w_self, w_new_value):
@@ -6532,8 +6424,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'upper'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'upper')
     return space.wrap(w_self.upper)
 
 def Slice_set_upper(space, w_self, w_new_value):
@@ -6554,8 +6445,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'step'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'step')
     return space.wrap(w_self.step)
 
 def Slice_set_step(space, w_self, w_new_value):
@@ -6598,14 +6488,13 @@
 def ExtSlice_get_dims(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'dims'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'dims')
     if w_self.w_dims is None:
         if w_self.dims is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.dims]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_dims = w_list
     return w_self.w_dims
 
@@ -6645,8 +6534,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Index_set_value(space, w_self, w_new_value):
@@ -6915,8 +6803,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'target'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'target')
     return space.wrap(w_self.target)
 
 def comprehension_set_target(space, w_self, w_new_value):
@@ -6937,8 +6824,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'iter'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'iter')
     return space.wrap(w_self.iter)
 
 def comprehension_set_iter(space, w_self, w_new_value):
@@ -6955,14 +6841,13 @@
 def comprehension_get_ifs(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ifs'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ifs')
     if w_self.w_ifs is None:
         if w_self.ifs is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.ifs]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_ifs = w_list
     return w_self.w_ifs
 
@@ -7004,8 +6889,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._lineno_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'lineno'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lineno')
     return space.wrap(w_self.lineno)
 
 def excepthandler_set_lineno(space, w_self, w_new_value):
@@ -7026,8 +6910,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._col_offset_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'col_offset'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'col_offset')
     return space.wrap(w_self.col_offset)
 
 def excepthandler_set_col_offset(space, w_self, w_new_value):
@@ -7057,8 +6940,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'type'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'type')
     return space.wrap(w_self.type)
 
 def ExceptHandler_set_type(space, w_self, w_new_value):
@@ -7079,8 +6961,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'name'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name')
     return space.wrap(w_self.name)
 
 def ExceptHandler_set_name(space, w_self, w_new_value):
@@ -7097,14 +6978,13 @@
 def ExceptHandler_get_body(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -7142,14 +7022,13 @@
 def arguments_get_args(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'args'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args')
     if w_self.w_args is None:
         if w_self.args is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.args]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_args = w_list
     return w_self.w_args
 
@@ -7164,8 +7043,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'vararg'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'vararg')
     return space.wrap(w_self.vararg)
 
 def arguments_set_vararg(space, w_self, w_new_value):
@@ -7189,8 +7067,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'kwarg'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'kwarg')
     return space.wrap(w_self.kwarg)
 
 def arguments_set_kwarg(space, w_self, w_new_value):
@@ -7210,14 +7087,13 @@
 def arguments_get_defaults(space, w_self):
     if not w_self.initialization_state & 8:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'defaults'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'defaults')
     if w_self.w_defaults is None:
         if w_self.defaults is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.defaults]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_defaults = w_list
     return w_self.w_defaults
 
@@ -7261,8 +7137,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'arg'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'arg')
     return space.wrap(w_self.arg)
 
 def keyword_set_arg(space, w_self, w_new_value):
@@ -7283,8 +7158,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def keyword_set_value(space, w_self, w_new_value):
@@ -7330,8 +7204,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'name'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name')
     return space.wrap(w_self.name)
 
 def alias_set_name(space, w_self, w_new_value):
@@ -7352,8 +7225,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'asname'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'asname')
     return space.wrap(w_self.asname)
 
 def alias_set_asname(space, w_self, w_new_value):
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -414,13 +414,12 @@
             self.emit("        return w_obj", 1)
         self.emit("if not w_self.initialization_state & %s:" % (flag,), 1)
         self.emit("typename = space.type(w_self).getname(space)", 2)
-        self.emit("w_err = space.wrap(\"'%%s' object has no attribute '%s'\" %% typename)" %
+        self.emit("raise operationerrfmt(space.w_AttributeError, \"'%%s' object has no attribute '%%s'\", typename, '%s')" %
                   (field.name,), 2)
-        self.emit("raise OperationError(space.w_AttributeError, w_err)", 2)
         if field.seq:
             self.emit("if w_self.w_%s is None:" % (field.name,), 1)
             self.emit("if w_self.%s is None:" % (field.name,), 2)
-            self.emit("w_list = space.newlist([])", 3)
+            self.emit("list_w = []", 3)
             self.emit("else:", 2)
             if field.type.value in self.data.simple_types:
                 wrapper = "%s_to_class[node - 1]()" % (field.type,)
@@ -428,7 +427,7 @@
                 wrapper = "space.wrap(node)"
             self.emit("list_w = [%s for node in w_self.%s]" %
                       (wrapper, field.name), 3)
-            self.emit("w_list = space.newlist(list_w)", 3)
+            self.emit("w_list = space.newlist(list_w)", 2)
             self.emit("w_self.w_%s = w_list" % (field.name,), 2)
             self.emit("return w_self.w_%s" % (field.name,), 1)
         elif field.type.value in self.data.simple_types:
@@ -540,7 +539,7 @@
 from pypy.interpreter.baseobjspace import Wrappable
 from pypy.interpreter import typedef
 from pypy.interpreter.gateway import interp2app
-from pypy.interpreter.error import OperationError
+from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.tool.pairtype import extendabletype
 from pypy.tool.sourcetools import func_with_new_name
@@ -639,9 +638,7 @@
             missing = required[i]
             if missing is not None:
                  err = "required field \\"%s\\" missing from %s"
-                 err = err % (missing, host)
-                 w_err = space.wrap(err)
-                 raise OperationError(space.w_TypeError, w_err)
+                 raise operationerrfmt(space.w_TypeError, err, missing, host)
     raise AssertionError("should not reach here")
 
 
diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -93,7 +93,8 @@
             i46 = call(ConstClass(ll_startswith__rpy_stringPtr_rpy_stringPtr), p28, ConstPtr(ptr45), descr=<BoolCallDescr>)
             guard_false(i46, descr=...)
             p51 = new_with_vtable(21136408)
-            setfield_gc(p51, _, descr=...)    # 6 setfields, but the order is dict-order-dependent
+            setfield_gc(p51, _, descr=...)    # 7 setfields, but the order is dict-order-dependent
+            setfield_gc(p51, _, descr=...)
             setfield_gc(p51, _, descr=...)
             setfield_gc(p51, _, descr=...)
             setfield_gc(p51, _, descr=...)
diff --git a/pypy/rlib/rerased.py b/pypy/rlib/rerased.py
--- a/pypy/rlib/rerased.py
+++ b/pypy/rlib/rerased.py
@@ -135,6 +135,8 @@
     _about_ = erase_int
 
     def compute_result_annotation(self, s_obj):
+        config = self.bookkeeper.annotator.translator.config
+        assert config.translation.taggedpointers, "need to enable tagged pointers to use erase_int"
         assert annmodel.SomeInteger().contains(s_obj)
         return SomeErased()
 
@@ -228,6 +230,8 @@
 
     def convert_const(self, value):
         if value._identity is _identity_for_ints:
+            config = self.rtyper.annotator.translator.config
+            assert config.translation.taggedpointers, "need to enable tagged pointers to use erase_int"
             return lltype.cast_int_to_ptr(self.lowleveltype, value._x * 2 + 1)
         bk = self.rtyper.annotator.bookkeeper
         s_obj = value._identity.get_input_annotation(bk)
diff --git a/pypy/rlib/test/test_rerased.py b/pypy/rlib/test/test_rerased.py
--- a/pypy/rlib/test/test_rerased.py
+++ b/pypy/rlib/test/test_rerased.py
@@ -10,6 +10,13 @@
 
 from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin
 
+def make_annotator():
+    a = RPythonAnnotator()
+    a.translator.config.translation.taggedpointers = True
+    return a
+
+
+
 class X(object):
     pass
 
@@ -55,7 +62,7 @@
 def test_annotate_1():
     def f():
         return eraseX(X())
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [])
     assert isinstance(s, SomeErased)
 
@@ -66,7 +73,7 @@
         #assert not is_integer(e)
         x2 = uneraseX(e)
         return x2
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [])
     assert isinstance(s, annmodel.SomeInstance)
     assert s.classdef == a.bookkeeper.getuniqueclassdef(X)
@@ -77,7 +84,7 @@
         #assert is_integer(e)
         x2 = unerase_int(e)
         return x2
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [])
     assert isinstance(s, annmodel.SomeInteger)
 
@@ -105,7 +112,7 @@
         x = make(n)
         return check(x, n)
     #
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [int])
     assert isinstance(s, annmodel.SomeInteger)
 
@@ -135,7 +142,7 @@
             else:
                 return inst
     #
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [])
     assert isinstance(s, annmodel.SomeInstance)
     assert s.classdef == a.bookkeeper.getuniqueclassdef(A)
@@ -155,7 +162,7 @@
             e = e2
         return unerase(e)
     #
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [int])
     assert isinstance(s, annmodel.SomeInstance)
     assert s.classdef == a.bookkeeper.getuniqueclassdef(X)
@@ -165,11 +172,14 @@
     e1 = erase_int(42)
     def f(i):
         return unerase_int(e1)
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [int])
     assert isinstance(s, annmodel.SomeInteger)
 
 class BaseTestRErased(BaseRtypingTest):
+    def interpret(self, *args, **kwargs):
+        kwargs["taggedpointers"] = True
+        return BaseRtypingTest.interpret(self, *args, **kwargs)
     def test_rtype_1(self):
         def f():
             return eraseX(X())
diff --git a/pypy/translator/c/database.py b/pypy/translator/c/database.py
--- a/pypy/translator/c/database.py
+++ b/pypy/translator/c/database.py
@@ -176,7 +176,7 @@
                      # introduced by the GC transformer, or the type_info_table
         return node
 
-    def get(self, obj):
+    def get(self, obj, funcgen=None):
         if isinstance(obj, CConstant):
             return obj.c_name  # without further checks
         T = typeOf(obj)
@@ -228,6 +228,8 @@
                     return '((%s) %d)' % (cdecl(self.gettype(T), ''),
                                           obj._obj)
                 node = self.getcontainernode(container)
+                if node._funccodegen_owner is None:
+                    node._funccodegen_owner = funcgen
                 return node.getptrname()
             else:
                 return '((%s) NULL)' % (cdecl(self.gettype(T), ''), )
@@ -284,14 +286,16 @@
             finish_callbacks.append(('GC transformer: finished tables',
                                      self.gctransformer.get_finish_tables()))
 
-        def add_dependencies(newdependencies):
+        def add_dependencies(newdependencies, parent=None):
             for value in newdependencies:
                 #if isinstance(value, _uninitialized):
                 #    continue
                 if isinstance(typeOf(value), ContainerType):
-                    self.getcontainernode(value)
+                    node = self.getcontainernode(value)
+                    if parent and node._funccodegen_owner is not None:
+                        node._funccodegen_owner = parent._funccodegen_owner
                 else:
-                    self.get(value)
+                    self.get(value, parent and parent._funccodegen_owner)
 
         while True:
             while True:
@@ -303,7 +307,7 @@
                 if i == len(self.containerlist):
                     break
                 node = self.containerlist[i]
-                add_dependencies(node.enum_dependencies())
+                add_dependencies(node.enum_dependencies(), node)
                 i += 1
                 self.completedcontainers = i
                 if i == show_i:
diff --git a/pypy/translator/c/gc.py b/pypy/translator/c/gc.py
--- a/pypy/translator/c/gc.py
+++ b/pypy/translator/c/gc.py
@@ -170,6 +170,7 @@
     nodekind = 'refcnt rtti'
     globalcontainer = True
     typename = 'void (@)(void *)'
+    _funccodegen_owner = None
 
     def __init__(self, db, T, obj):
         assert T == RuntimeTypeInfo
@@ -266,6 +267,7 @@
     nodekind = 'boehm rtti'
     globalcontainer = True
     typename = 'char @'
+    _funccodegen_owner = None
 
     def __init__(self, db, T, obj):
         assert T == RuntimeTypeInfo
diff --git a/pypy/translator/c/gcc/test/test_asmgcroot.py b/pypy/translator/c/gcc/test/test_asmgcroot.py
--- a/pypy/translator/c/gcc/test/test_asmgcroot.py
+++ b/pypy/translator/c/gcc/test/test_asmgcroot.py
@@ -21,6 +21,7 @@
         config = get_pypy_config(translating=True)
         config.translation.gc = cls.gcpolicy
         config.translation.gcrootfinder = "asmgcc"
+        config.translation.taggedpointers = getattr(cls, "taggedpointers", False)
         return config
 
     @classmethod
diff --git a/pypy/translator/c/genc.py b/pypy/translator/c/genc.py
--- a/pypy/translator/c/genc.py
+++ b/pypy/translator/c/genc.py
@@ -682,8 +682,7 @@
     def getbasecfilefornode(self, node, basecname):
         # For FuncNode instances, use the python source filename (relative to
         # the top directory):
-        if hasattr(node.obj, 'graph'):
-            g = node.obj.graph
+        def invent_nice_name(g):
             # Lookup the filename from the function.
             # However, not all FunctionGraph objs actually have a "func":
             if hasattr(g, 'func'):
@@ -693,6 +692,15 @@
                     if pypkgpath:
                         relpypath =  localpath.relto(pypkgpath)
                         return relpypath.replace('.py', '.c')
+            return None
+        if hasattr(node.obj, 'graph'):
+            name = invent_nice_name(node.obj.graph)
+            if name is not None:
+                return name
+        elif node._funccodegen_owner is not None:
+            name = invent_nice_name(node._funccodegen_owner.graph)
+            if name is not None:
+                return "data_" + name
         return basecname
 
     def splitnodesimpl(self, basecname, nodes, nextra, nbetween,
diff --git a/pypy/translator/c/node.py b/pypy/translator/c/node.py
--- a/pypy/translator/c/node.py
+++ b/pypy/translator/c/node.py
@@ -485,6 +485,7 @@
         __slots__ = """db obj 
                        typename implementationtypename
                         name
+                        _funccodegen_owner
                         globalcontainer""".split()
     eci_name = '_compilation_info'
 
@@ -509,6 +510,7 @@
         if self.typename != self.implementationtypename:
             if db.gettypedefnode(T).extra_union_for_varlength:
                 self.name += '.b'
+        self._funccodegen_owner = None
 
     def getptrname(self):
         return '(&%s)' % self.name
@@ -842,6 +844,9 @@
         if self.funcgens:
             argnames = self.funcgens[0].argnames()  #Assume identical for all funcgens
             self.implementationtypename = self.db.gettype(self.T, argnames=argnames)
+            self._funccodegen_owner = self.funcgens[0]
+        else:
+            self._funccodegen_owner = None
 
     def basename(self):
         return self.obj._name
@@ -1005,6 +1010,7 @@
     globalcontainer = True
     typename = 'PyObject @'
     implementationtypename = 'PyObject *@'
+    _funccodegen_owner = None
 
     def __init__(self, db, T, obj):
         # obj is a _pyobject here; obj.value is the underlying CPython object
diff --git a/pypy/translator/c/primitive.py b/pypy/translator/c/primitive.py
--- a/pypy/translator/c/primitive.py
+++ b/pypy/translator/c/primitive.py
@@ -141,7 +141,12 @@
 
 def name_gcref(value, db):
     if value:
-        realobj = value._obj.container
+        obj = value._obj
+        if isinstance(obj, int):
+            # a tagged pointer
+            assert obj & 1 == 1
+            return '((%s) %d)' % (cdecl("void*", ''), obj)
+        realobj = obj.container
         realvalue = cast_opaque_ptr(Ptr(typeOf(realobj)), value)
         return db.get(realvalue)
     else:
diff --git a/pypy/translator/c/test/test_newgc.py b/pypy/translator/c/test/test_newgc.py
--- a/pypy/translator/c/test/test_newgc.py
+++ b/pypy/translator/c/test/test_newgc.py
@@ -1487,6 +1487,43 @@
         res = self.run("tagged")
         assert res == expected
 
+    def define_erased(cls):
+        from pypy.rlib import rerased
+        erase, unerase = rerased.new_erasing_pair("test")
+        class Unrelated(object):
+            pass
+
+        u = Unrelated()
+        u.tagged = True
+        u.x = rerased.erase_int(41)
+        class A(object):
+            pass
+        def fn():
+            n = 1
+            while n >= 0:
+                if u.tagged:
+                    n = rerased.unerase_int(u.x)
+                    a = A()
+                    a.n = n - 1
+                    u.x = erase(a)
+                    u.tagged = False
+                else:
+                    n = unerase(u.x).n
+                    u.x = rerased.erase_int(n - 1)
+                    u.tagged = True
+        def func():
+            rgc.collect() # check that a prebuilt erased integer doesn't explode
+            u.x = rerased.erase_int(1000)
+            u.tagged = True
+            fn()
+            return 1
+        return func
+
+    def test_erased(self):
+        expected = self.run_orig("erased")
+        res = self.run("erased")
+        assert res == expected
+
 from pypy.rlib.objectmodel import UnboxedValue
 
 class TaggedBase(object):

From noreply at buildbot.pypy.org  Fri Oct 21 23:24:55 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Fri, 21 Oct 2011 23:24:55 +0200 (CEST)
Subject: [pypy-commit] pypy inline-dict-ops: closed branch for merge
Message-ID: <20111021212455.9C1BC820D7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: inline-dict-ops
Changeset: r48324:f5cb6b7f9319
Date: 2011-10-21 14:24 -0700
http://bitbucket.org/pypy/pypy/changeset/f5cb6b7f9319/

Log:	closed branch for merge


From noreply at buildbot.pypy.org  Fri Oct 21 23:24:57 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Fri, 21 Oct 2011 23:24:57 +0200 (CEST)
Subject: [pypy-commit] pypy default: (fijal, alex,
 armin): Merged inline-dict-ops, this teaches the JIT about {get,
 set}iteriorfield_gc,
 which is needed to inline various dict functions such as ll_get_value.
Message-ID: <20111021212457.5B816820D7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r48325:2e38d92cac8e
Date: 2011-10-21 14:24 -0700
http://bitbucket.org/pypy/pypy/changeset/2e38d92cac8e/

Log:	(fijal, alex, armin): Merged inline-dict-ops, this teaches the JIT
	about {get,set}iteriorfield_gc, which is needed to inline various
	dict functions such as ll_get_value.

diff --git a/pypy/jit/backend/llgraph/llimpl.py b/pypy/jit/backend/llgraph/llimpl.py
--- a/pypy/jit/backend/llgraph/llimpl.py
+++ b/pypy/jit/backend/llgraph/llimpl.py
@@ -7,9 +7,7 @@
 import weakref
 from pypy.objspace.flow.model import Variable, Constant
 from pypy.annotation import model as annmodel
-from pypy.jit.metainterp.history import (ConstInt, ConstPtr,
-                                         BoxInt, BoxPtr, BoxObj, BoxFloat,
-                                         REF, INT, FLOAT)
+from pypy.jit.metainterp.history import REF, INT, FLOAT
 from pypy.jit.codewriter import heaptracker
 from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr, rffi
 from pypy.rpython.ootypesystem import ootype
@@ -17,7 +15,7 @@
 from pypy.rpython.llinterp import LLException
 from pypy.rpython.extregistry import ExtRegistryEntry
 
-from pypy.jit.metainterp import resoperation, executor
+from pypy.jit.metainterp import resoperation
 from pypy.jit.metainterp.resoperation import rop
 from pypy.jit.backend.llgraph import symbolic
 from pypy.jit.codewriter import longlong
@@ -334,6 +332,13 @@
     assert isinstance(type, str) and len(type) == 1
     op.descr = Descr(ofs, type, arg_types=arg_types)
 
+def compile_add_descr_arg(loop, ofs, type, arg_types):
+    from pypy.jit.backend.llgraph.runner import Descr
+    loop = _from_opaque(loop)
+    op = loop.operations[-1]
+    assert isinstance(type, str) and len(type) == 1
+    op.args.append(Descr(ofs, type, arg_types=arg_types))
+
 def compile_add_loop_token(loop, descr):
     if we_are_translated():
         raise ValueError("CALL_ASSEMBLER not supported")
@@ -438,8 +443,11 @@
         self._may_force = -1
 
     def getenv(self, v):
+        from pypy.jit.backend.llgraph.runner import Descr
         if isinstance(v, Constant):
             return v.value
+        elif isinstance(v, Descr):
+            return v
         else:
             return self.env[v]
 
@@ -807,6 +815,29 @@
         else:
             raise NotImplementedError
 
+    def op_getinteriorfield_gc(self, descr, array, index):
+        if descr.typeinfo == REF:
+            return do_getinteriorfield_gc_ptr(array, index, descr.ofs)
+        elif descr.typeinfo == INT:
+            return do_getinteriorfield_gc_int(array, index, descr.ofs)
+        elif descr.typeinfo == FLOAT:
+            return do_getinteriorfield_gc_float(array, index, descr.ofs)
+        else:
+            raise NotImplementedError
+
+    def op_setinteriorfield_gc(self, descr, array, index, newvalue):
+        if descr.typeinfo == REF:
+            return do_setinteriorfield_gc_ptr(array, index, descr.ofs,
+                                              newvalue)
+        elif descr.typeinfo == INT:
+            return do_setinteriorfield_gc_int(array, index, descr.ofs,
+                                              newvalue)
+        elif descr.typeinfo == FLOAT:
+            return do_setinteriorfield_gc_float(array, index, descr.ofs,
+                                                newvalue)
+        else:
+            raise NotImplementedError
+
     def op_setfield_gc(self, fielddescr, struct, newvalue):
         if fielddescr.typeinfo == REF:
             do_setfield_gc_ptr(struct, fielddescr.ofs, newvalue)
@@ -1354,6 +1385,22 @@
 def do_getfield_gc_ptr(struct, fieldnum):
     return cast_to_ptr(_getfield_gc(struct, fieldnum))
 
+def _getinteriorfield_gc(struct, fieldnum):
+    STRUCT, fieldname = symbolic.TokenToField[fieldnum]
+    return getattr(struct, fieldname)
+
+def do_getinteriorfield_gc_int(array, index, fieldnum):
+    struct = array._obj.container.getitem(index)
+    return cast_to_int(_getinteriorfield_gc(struct, fieldnum))
+
+def do_getinteriorfield_gc_float(array, index, fieldnum):
+    struct = array._obj.container.getitem(index)
+    return cast_to_floatstorage(_getinteriorfield_gc(struct, fieldnum))
+
+def do_getinteriorfield_gc_ptr(array, index, fieldnum):
+    struct = array._obj.container.getitem(index)
+    return cast_to_ptr(_getinteriorfield_gc(struct, fieldnum))
+
 def _getfield_raw(struct, fieldnum):
     STRUCT, fieldname = symbolic.TokenToField[fieldnum]
     ptr = cast_from_int(lltype.Ptr(STRUCT), struct)
@@ -1409,26 +1456,28 @@
     newvalue = cast_from_ptr(ITEMTYPE, newvalue)
     array.setitem(index, newvalue)
 
-def do_setfield_gc_int(struct, fieldnum, newvalue):
-    STRUCT, fieldname = symbolic.TokenToField[fieldnum]
-    ptr = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), struct)
-    FIELDTYPE = getattr(STRUCT, fieldname)
-    newvalue = cast_from_int(FIELDTYPE, newvalue)
-    setattr(ptr, fieldname, newvalue)
+def new_setfield_gc(cast_func):
+    def do_setfield_gc(struct, fieldnum, newvalue):
+        STRUCT, fieldname = symbolic.TokenToField[fieldnum]
+        ptr = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), struct)
+        FIELDTYPE = getattr(STRUCT, fieldname)
+        newvalue = cast_func(FIELDTYPE, newvalue)
+        setattr(ptr, fieldname, newvalue)
+    return do_setfield_gc
+do_setfield_gc_int = new_setfield_gc(cast_from_int)
+do_setfield_gc_float = new_setfield_gc(cast_from_floatstorage)
+do_setfield_gc_ptr = new_setfield_gc(cast_from_ptr)
 
-def do_setfield_gc_float(struct, fieldnum, newvalue):
-    STRUCT, fieldname = symbolic.TokenToField[fieldnum]
-    ptr = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), struct)
-    FIELDTYPE = getattr(STRUCT, fieldname)
-    newvalue = cast_from_floatstorage(FIELDTYPE, newvalue)
-    setattr(ptr, fieldname, newvalue)
-
-def do_setfield_gc_ptr(struct, fieldnum, newvalue):
-    STRUCT, fieldname = symbolic.TokenToField[fieldnum]
-    ptr = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), struct)
-    FIELDTYPE = getattr(STRUCT, fieldname)
-    newvalue = cast_from_ptr(FIELDTYPE, newvalue)
-    setattr(ptr, fieldname, newvalue)
+def new_setinteriorfield_gc(cast_func):
+    def do_setinteriorfield_gc(array, index, fieldnum, newvalue):
+        STRUCT, fieldname = symbolic.TokenToField[fieldnum]
+        struct = array._obj.container.getitem(index)
+        FIELDTYPE = getattr(STRUCT, fieldname)
+        setattr(struct, fieldname, cast_func(FIELDTYPE, newvalue))
+    return do_setinteriorfield_gc
+do_setinteriorfield_gc_int = new_setinteriorfield_gc(cast_from_int)
+do_setinteriorfield_gc_float = new_setinteriorfield_gc(cast_from_floatstorage)
+do_setinteriorfield_gc_ptr = new_setinteriorfield_gc(cast_from_ptr)        
 
 def do_setfield_raw_int(struct, fieldnum, newvalue):
     STRUCT, fieldname = symbolic.TokenToField[fieldnum]
@@ -1694,6 +1743,7 @@
 setannotation(compile_start_float_var, annmodel.SomeInteger())
 setannotation(compile_add, annmodel.s_None)
 setannotation(compile_add_descr, annmodel.s_None)
+setannotation(compile_add_descr_arg, annmodel.s_None)
 setannotation(compile_add_var, annmodel.s_None)
 setannotation(compile_add_int_const, annmodel.s_None)
 setannotation(compile_add_ref_const, annmodel.s_None)
@@ -1741,6 +1791,9 @@
 setannotation(do_getfield_raw_int, annmodel.SomeInteger())
 setannotation(do_getfield_raw_ptr, annmodel.SomePtr(llmemory.GCREF))
 setannotation(do_getfield_raw_float, s_FloatStorage)
+setannotation(do_getinteriorfield_gc_int, annmodel.SomeInteger())
+setannotation(do_getinteriorfield_gc_ptr, annmodel.SomePtr(llmemory.GCREF))
+setannotation(do_getinteriorfield_gc_float, s_FloatStorage)
 setannotation(do_new, annmodel.SomePtr(llmemory.GCREF))
 setannotation(do_new_array, annmodel.SomePtr(llmemory.GCREF))
 setannotation(do_setarrayitem_gc_int, annmodel.s_None)
@@ -1754,6 +1807,9 @@
 setannotation(do_setfield_raw_int, annmodel.s_None)
 setannotation(do_setfield_raw_ptr, annmodel.s_None)
 setannotation(do_setfield_raw_float, annmodel.s_None)
+setannotation(do_setinteriorfield_gc_int, annmodel.s_None)
+setannotation(do_setinteriorfield_gc_ptr, annmodel.s_None)
+setannotation(do_setinteriorfield_gc_float, annmodel.s_None)
 setannotation(do_newstr, annmodel.SomePtr(llmemory.GCREF))
 setannotation(do_strsetitem, annmodel.s_None)
 setannotation(do_newunicode, annmodel.SomePtr(llmemory.GCREF))
diff --git a/pypy/jit/backend/llgraph/runner.py b/pypy/jit/backend/llgraph/runner.py
--- a/pypy/jit/backend/llgraph/runner.py
+++ b/pypy/jit/backend/llgraph/runner.py
@@ -2,7 +2,6 @@
 Minimal-API wrapper around the llinterpreter to run operations.
 """
 
-import sys
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.rpython.lltypesystem import lltype, llmemory, rclass
@@ -11,12 +10,11 @@
 from pypy.jit.metainterp import history
 from pypy.jit.metainterp.history import REF, INT, FLOAT
 from pypy.jit.metainterp.warmstate import unwrap
-from pypy.jit.metainterp.resoperation import ResOperation, rop
+from pypy.jit.metainterp.resoperation import rop
 from pypy.jit.backend import model
 from pypy.jit.backend.llgraph import llimpl, symbolic
 from pypy.jit.metainterp.typesystem import llhelper, oohelper
 from pypy.jit.codewriter import heaptracker, longlong
-from pypy.rlib import rgc
 
 class MiniStats:
     pass
@@ -177,8 +175,10 @@
             llimpl.compile_add(c, op.getopnum())
             descr = op.getdescr()
             if isinstance(descr, Descr):
-                llimpl.compile_add_descr(c, descr.ofs, descr.typeinfo, descr.arg_types)
-            if isinstance(descr, history.LoopToken) and op.getopnum() != rop.JUMP:
+                llimpl.compile_add_descr(c, descr.ofs, descr.typeinfo,
+                                         descr.arg_types)
+            if (isinstance(descr, history.LoopToken) and
+                op.getopnum() != rop.JUMP):
                 llimpl.compile_add_loop_token(c, descr)
             if self.is_oo and isinstance(descr, (OODescr, MethDescr)):
                 # hack hack, not rpython
@@ -193,6 +193,9 @@
                     llimpl.compile_add_ref_const(c, x.value, self.ts.BASETYPE)
                 elif isinstance(x, history.ConstFloat):
                     llimpl.compile_add_float_const(c, x.value)
+                elif isinstance(x, Descr):
+                    llimpl.compile_add_descr_arg(c, x.ofs, x.typeinfo,
+                                                 x.arg_types)
                 else:
                     raise Exception("'%s' args contain: %r" % (op.getopname(),
                                                                x))
@@ -316,6 +319,13 @@
         token = history.getkind(getattr(S, fieldname))
         return self.getdescr(ofs, token[0], name=fieldname)
 
+    def interiorfielddescrof(self, A, fieldname):
+        S = A.OF
+        ofs2 = symbolic.get_size(A)
+        ofs, size = symbolic.get_field_token(S, fieldname)
+        token = history.getkind(getattr(S, fieldname))
+        return self.getdescr(ofs, token[0], name=fieldname, extrainfo=ofs2)
+
     def calldescrof(self, FUNC, ARGS, RESULT, extrainfo):
         arg_types = []
         for ARG in ARGS:
@@ -353,8 +363,11 @@
     def arraydescrof(self, A):
         assert A.OF != lltype.Void
         size = symbolic.get_size(A)
-        token = history.getkind(A.OF)
-        return self.getdescr(size, token[0])
+        if isinstance(A.OF, lltype.Ptr) or isinstance(A.OF, lltype.Primitive):
+            token = history.getkind(A.OF)[0]
+        else:
+            token = '?'
+        return self.getdescr(size, token)
 
     # ---------- the backend-dependent operations ----------
 
@@ -406,6 +419,29 @@
         assert isinstance(fielddescr, Descr)
         return llimpl.do_getfield_raw_float(struct, fielddescr.ofs)
 
+    def bh_getinteriorfield_gc_i(self, array, index, descr):
+        assert isinstance(descr, Descr)
+        return llimpl.do_getinteriorfield_gc_int(array, index, descr.ofs)
+    def bh_getinteriorfield_gc_r(self, array, index, descr):
+        assert isinstance(descr, Descr)
+        return llimpl.do_getinteriorfield_gc_ptr(array, index, descr.ofs)
+    def bh_getinteriorfield_gc_f(self, array, index, descr):
+        assert isinstance(descr, Descr)
+        return llimpl.do_getinteriorfield_gc_float(array, index, descr.ofs)
+
+    def bh_setinteriorfield_gc_i(self, array, index, descr, value):
+        assert isinstance(descr, Descr)
+        return llimpl.do_setinteriorfield_gc_int(array, index, descr.ofs,
+                                                 value)
+    def bh_setinteriorfield_gc_r(self, array, index, descr, value):
+        assert isinstance(descr, Descr)
+        return llimpl.do_setinteriorfield_gc_ptr(array, index, descr.ofs,
+                                                 value)
+    def bh_setinteriorfield_gc_f(self, array, index, descr, value):
+        assert isinstance(descr, Descr)
+        return llimpl.do_setinteriorfield_gc_float(array, index, descr.ofs,
+                                                   value)
+
     def bh_new(self, sizedescr):
         assert isinstance(sizedescr, Descr)
         return llimpl.do_new(sizedescr.ofs)
@@ -418,7 +454,6 @@
 
     def bh_classof(self, struct):
         struct = lltype.cast_opaque_ptr(rclass.OBJECTPTR, struct)
-        result = struct.typeptr
         result_adr = llmemory.cast_ptr_to_adr(struct.typeptr)
         return heaptracker.adr2int(result_adr)
 
diff --git a/pypy/jit/backend/llsupport/descr.py b/pypy/jit/backend/llsupport/descr.py
--- a/pypy/jit/backend/llsupport/descr.py
+++ b/pypy/jit/backend/llsupport/descr.py
@@ -1,13 +1,10 @@
 import py
-from pypy.rpython.lltypesystem import lltype, rffi, llmemory, rclass
+from pypy.rpython.lltypesystem import lltype, rffi, llmemory
 from pypy.rpython.lltypesystem.lloperation import llop
 from pypy.jit.backend.llsupport import symbolic, support
-from pypy.jit.metainterp.history import AbstractDescr, getkind, BoxInt, BoxPtr
-from pypy.jit.metainterp.history import BasicFailDescr, LoopToken, BoxFloat
+from pypy.jit.metainterp.history import AbstractDescr, getkind
 from pypy.jit.metainterp import history
-from pypy.jit.metainterp.resoperation import ResOperation, rop
 from pypy.jit.codewriter import heaptracker, longlong
-from pypy.rlib.rarithmetic import r_longlong, r_ulonglong
 
 # The point of the class organization in this file is to make instances
 # as compact as possible.  This is done by not storing the field size or
@@ -23,6 +20,7 @@
         self._cache_field = {}
         self._cache_array = {}
         self._cache_call = {}
+        self._cache_interiorfield = {}
 
     def init_size_descr(self, STRUCT, sizedescr):
         assert isinstance(STRUCT, lltype.GcStruct)
@@ -142,7 +140,6 @@
         cachedict[fieldname] = fielddescr
         return fielddescr
 
-
 # ____________________________________________________________
 # ArrayDescrs
 
@@ -252,6 +249,36 @@
         cache[ARRAY] = arraydescr
         return arraydescr
 
+# ____________________________________________________________
+# InteriorFieldDescr
+
+class InteriorFieldDescr(AbstractDescr):
+    arraydescr = BaseArrayDescr()     # workaround for the annotator
+    fielddescr = BaseFieldDescr('', 0)
+
+    def __init__(self, arraydescr, fielddescr):
+        self.arraydescr = arraydescr
+        self.fielddescr = fielddescr
+
+    def is_pointer_field(self):
+        return self.fielddescr.is_pointer_field()
+
+    def is_float_field(self):
+        return self.fielddescr.is_float_field()
+
+    def repr_of_descr(self):
+        return '<InteriorFieldDescr %s>' % self.fielddescr.repr_of_descr()
+
+def get_interiorfield_descr(gc_ll_descr, ARRAY, FIELDTP, name):
+    cache = gc_ll_descr._cache_interiorfield
+    try:
+        return cache[(ARRAY, FIELDTP, name)]
+    except KeyError:
+        arraydescr = get_array_descr(gc_ll_descr, ARRAY)
+        fielddescr = get_field_descr(gc_ll_descr, FIELDTP, name)
+        descr = InteriorFieldDescr(arraydescr, fielddescr)
+        cache[(ARRAY, FIELDTP, name)] = descr
+        return descr
 
 # ____________________________________________________________
 # CallDescrs
@@ -525,7 +552,8 @@
         #
         if TYPE is lltype.Float or is_longlong(TYPE):
             setattr(Descr, floatattrname, True)
-        elif TYPE is not lltype.Bool and rffi.cast(TYPE, -1) == -1:
+        elif (TYPE is not lltype.Bool and isinstance(TYPE, lltype.Number) and
+              rffi.cast(TYPE, -1) == -1):
             setattr(Descr, signedattrname, True)
         #
         _cache[nameprefix, TYPE] = Descr
diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py
--- a/pypy/jit/backend/llsupport/gc.py
+++ b/pypy/jit/backend/llsupport/gc.py
@@ -45,6 +45,14 @@
     def freeing_block(self, start, stop):
         pass
 
+    def get_funcptr_for_newarray(self):
+        return llhelper(self.GC_MALLOC_ARRAY, self.malloc_array)
+    def get_funcptr_for_newstr(self):
+        return llhelper(self.GC_MALLOC_STR_UNICODE, self.malloc_str)
+    def get_funcptr_for_newunicode(self):
+        return llhelper(self.GC_MALLOC_STR_UNICODE, self.malloc_unicode)
+
+
     def record_constptrs(self, op, gcrefs_output_list):
         for i in range(op.numargs()):
             v = op.getarg(i)
@@ -96,6 +104,39 @@
         malloc_fn_ptr = self.configure_boehm_once()
         self.funcptr_for_new = malloc_fn_ptr
 
+        def malloc_array(basesize, itemsize, ofs_length, num_elem):
+            try:
+                size = ovfcheck(basesize + ovfcheck(itemsize * num_elem))
+            except OverflowError:
+                return lltype.nullptr(llmemory.GCREF.TO)
+            res = self.funcptr_for_new(size)
+            if not res:
+                return res
+            rffi.cast(rffi.CArrayPtr(lltype.Signed), res)[ofs_length/WORD] = num_elem
+            return res
+        self.malloc_array = malloc_array
+        self.GC_MALLOC_ARRAY = lltype.Ptr(lltype.FuncType(
+            [lltype.Signed] * 4, llmemory.GCREF))
+
+
+        (str_basesize, str_itemsize, str_ofs_length
+         ) = symbolic.get_array_token(rstr.STR, self.translate_support_code)
+        (unicode_basesize, unicode_itemsize, unicode_ofs_length
+         ) = symbolic.get_array_token(rstr.UNICODE, self.translate_support_code)
+        def malloc_str(length):
+            return self.malloc_array(
+                str_basesize, str_itemsize, str_ofs_length, length
+            )
+        def malloc_unicode(length):
+            return self.malloc_array(
+                unicode_basesize, unicode_itemsize, unicode_ofs_length, length
+            )
+        self.malloc_str = malloc_str
+        self.malloc_unicode = malloc_unicode
+        self.GC_MALLOC_STR_UNICODE = lltype.Ptr(lltype.FuncType(
+            [lltype.Signed], llmemory.GCREF))
+
+
         # on some platform GC_init is required before any other
         # GC_* functions, call it here for the benefit of tests
         # XXX move this to tests
@@ -116,39 +157,27 @@
         ofs_length = arraydescr.get_ofs_length(self.translate_support_code)
         basesize = arraydescr.get_base_size(self.translate_support_code)
         itemsize = arraydescr.get_item_size(self.translate_support_code)
-        size = basesize + itemsize * num_elem
-        res = self.funcptr_for_new(size)
-        rffi.cast(rffi.CArrayPtr(lltype.Signed), res)[ofs_length/WORD] = num_elem
-        return res
+        return self.malloc_array(basesize, itemsize, ofs_length, num_elem)
 
     def gc_malloc_str(self, num_elem):
-        basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR,
-                                                   self.translate_support_code)
-        assert itemsize == 1
-        size = basesize + num_elem
-        res = self.funcptr_for_new(size)
-        rffi.cast(rffi.CArrayPtr(lltype.Signed), res)[ofs_length/WORD] = num_elem
-        return res
+        return self.malloc_str(num_elem)
 
     def gc_malloc_unicode(self, num_elem):
-        basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE,
-                                                   self.translate_support_code)
-        size = basesize + num_elem * itemsize
-        res = self.funcptr_for_new(size)
-        rffi.cast(rffi.CArrayPtr(lltype.Signed), res)[ofs_length/WORD] = num_elem
-        return res
+        return self.malloc_unicode(num_elem)
 
     def args_for_new(self, sizedescr):
         assert isinstance(sizedescr, BaseSizeDescr)
         return [sizedescr.size]
 
+    def args_for_new_array(self, arraydescr):
+        ofs_length = arraydescr.get_ofs_length(self.translate_support_code)
+        basesize = arraydescr.get_base_size(self.translate_support_code)
+        itemsize = arraydescr.get_item_size(self.translate_support_code)
+        return [basesize, itemsize, ofs_length]
+
     def get_funcptr_for_new(self):
         return self.funcptr_for_new
 
-    get_funcptr_for_newarray = None
-    get_funcptr_for_newstr = None
-    get_funcptr_for_newunicode = None
-
     def rewrite_assembler(self, cpu, operations, gcrefs_output_list):
         # record all GCREFs too, because Boehm cannot see them and keep them
         # alive if they end up as constants in the assembler
@@ -752,15 +781,6 @@
     def get_funcptr_for_new(self):
         return llhelper(self.GC_MALLOC_BASIC, self.malloc_basic)
 
-    def get_funcptr_for_newarray(self):
-        return llhelper(self.GC_MALLOC_ARRAY, self.malloc_array)
-
-    def get_funcptr_for_newstr(self):
-        return llhelper(self.GC_MALLOC_STR_UNICODE, self.malloc_str)
-
-    def get_funcptr_for_newunicode(self):
-        return llhelper(self.GC_MALLOC_STR_UNICODE, self.malloc_unicode)
-
     def do_write_barrier(self, gcref_struct, gcref_newptr):
         hdr_addr = llmemory.cast_ptr_to_adr(gcref_struct)
         hdr_addr -= self.gcheaderbuilder.size_gc_header
diff --git a/pypy/jit/backend/llsupport/llmodel.py b/pypy/jit/backend/llsupport/llmodel.py
--- a/pypy/jit/backend/llsupport/llmodel.py
+++ b/pypy/jit/backend/llsupport/llmodel.py
@@ -1,24 +1,18 @@
-import sys
 from pypy.rpython.lltypesystem import lltype, llmemory, rffi, rclass, rstr
 from pypy.rpython.lltypesystem.lloperation import llop
-from pypy.rpython.llinterp import LLInterpreter, LLException
+from pypy.rpython.llinterp import LLInterpreter
 from pypy.rpython.annlowlevel import llhelper
 from pypy.rlib.objectmodel import we_are_translated, specialize
-from pypy.jit.metainterp.history import BoxInt, BoxPtr, set_future_values,\
-     BoxFloat
 from pypy.jit.metainterp import history
 from pypy.jit.codewriter import heaptracker, longlong
 from pypy.jit.backend.model import AbstractCPU
 from pypy.jit.backend.llsupport import symbolic
 from pypy.jit.backend.llsupport.symbolic import WORD, unroll_basic_sizes
-from pypy.jit.backend.llsupport.descr import get_size_descr,  BaseSizeDescr
-from pypy.jit.backend.llsupport.descr import get_field_descr, BaseFieldDescr
-from pypy.jit.backend.llsupport.descr import get_array_descr, BaseArrayDescr
-from pypy.jit.backend.llsupport.descr import get_call_descr
-from pypy.jit.backend.llsupport.descr import BaseIntCallDescr, GcPtrCallDescr
-from pypy.jit.backend.llsupport.descr import FloatCallDescr, VoidCallDescr
+from pypy.jit.backend.llsupport.descr import (get_size_descr,
+     get_field_descr, BaseFieldDescr, get_array_descr, BaseArrayDescr,
+     get_call_descr, BaseIntCallDescr, GcPtrCallDescr, FloatCallDescr,
+     VoidCallDescr, InteriorFieldDescr, get_interiorfield_descr)
 from pypy.jit.backend.llsupport.asmmemmgr import AsmMemoryManager
-from pypy.rpython.annlowlevel import cast_instance_to_base_ptr
 
 
 class AbstractLLCPU(AbstractCPU):
@@ -241,6 +235,9 @@
     def arraydescrof(self, A):
         return get_array_descr(self.gc_ll_descr, A)
 
+    def interiorfielddescrof(self, A, fieldname):
+        return get_interiorfield_descr(self.gc_ll_descr, A, A.OF, fieldname)
+
     def unpack_arraydescr(self, arraydescr):
         assert isinstance(arraydescr, BaseArrayDescr)
         return arraydescr.get_base_size(self.translate_support_code)
@@ -358,6 +355,100 @@
     bh_getarrayitem_raw_i = bh_getarrayitem_gc_i
     bh_getarrayitem_raw_f = bh_getarrayitem_gc_f
 
+    def bh_getinteriorfield_gc_i(self, gcref, itemindex, descr):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        fieldsize = descr.fielddescr.get_field_size(self.translate_support_code)
+        sign = descr.fielddescr.is_field_signed()
+        fullofs = itemindex * size + ofs
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), fullofs)
+        for STYPE, UTYPE, itemsize in unroll_basic_sizes:
+            if fieldsize == itemsize:
+                if sign:
+                    item = rffi.cast(rffi.CArrayPtr(STYPE), items)
+                    val = item[0]
+                    val = rffi.cast(lltype.Signed, val)
+                else:
+                    item = rffi.cast(rffi.CArrayPtr(UTYPE), items)
+                    val = item[0]
+                    val = rffi.cast(lltype.Signed, val)
+                # --- end of GC unsafe code ---
+                return val
+        else:
+            raise NotImplementedError("size = %d" % fieldsize)
+
+    def bh_getinteriorfield_gc_r(self, gcref, itemindex, descr):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs +
+                            size * itemindex)
+        items = rffi.cast(rffi.CArrayPtr(lltype.Signed), items)
+        pval = self._cast_int_to_gcref(items[0])
+        # --- end of GC unsafe code ---
+        return pval
+
+    def bh_getinteriorfield_gc_f(self, gcref, itemindex, descr):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs +
+                            size * itemindex)
+        items = rffi.cast(rffi.CArrayPtr(longlong.FLOATSTORAGE), items)
+        fval = items[0]
+        # --- end of GC unsafe code ---
+        return fval
+
+    def bh_setinteriorfield_gc_i(self, gcref, itemindex, descr, value):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        fieldsize = descr.fielddescr.get_field_size(self.translate_support_code)
+        ofs = itemindex * size + ofs
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs)
+        for TYPE, _, itemsize in unroll_basic_sizes:
+            if fieldsize == itemsize:
+                items = rffi.cast(rffi.CArrayPtr(TYPE), items)
+                items[0] = rffi.cast(TYPE, value)
+                # --- end of GC unsafe code ---
+                return
+        else:
+            raise NotImplementedError("size = %d" % fieldsize)
+
+    def bh_setinteriorfield_gc_r(self, gcref, itemindex, descr, newvalue):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        self.gc_ll_descr.do_write_barrier(gcref, newvalue)
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref),
+                            ofs + size * itemindex)
+        items = rffi.cast(rffi.CArrayPtr(lltype.Signed), items)
+        items[0] = self.cast_gcref_to_int(newvalue)
+        # --- end of GC unsafe code ---
+
+    def bh_setinteriorfield_gc_f(self, gcref, itemindex, descr, newvalue):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref),
+                            ofs + size * itemindex)
+        items = rffi.cast(rffi.CArrayPtr(longlong.FLOATSTORAGE), items)
+        items[0] = newvalue
+        # --- end of GC unsafe code ---
+
     def bh_strlen(self, string):
         s = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), string)
         return len(s.chars)
@@ -475,7 +566,6 @@
 
     def bh_classof(self, struct):
         struct = lltype.cast_opaque_ptr(rclass.OBJECTPTR, struct)
-        result = struct.typeptr
         result_adr = llmemory.cast_ptr_to_adr(struct.typeptr)
         return heaptracker.adr2int(result_adr)
 
diff --git a/pypy/jit/backend/llsupport/regalloc.py b/pypy/jit/backend/llsupport/regalloc.py
--- a/pypy/jit/backend/llsupport/regalloc.py
+++ b/pypy/jit/backend/llsupport/regalloc.py
@@ -287,7 +287,6 @@
         self.reg_bindings[to_v] = reg
 
     def _move_variable_away(self, v, prev_loc):
-        reg = None
         if self.free_regs:
             loc = self.free_regs.pop()
             self.reg_bindings[v] = loc
diff --git a/pypy/jit/backend/llsupport/test/test_asmmemmgr.py b/pypy/jit/backend/llsupport/test/test_asmmemmgr.py
--- a/pypy/jit/backend/llsupport/test/test_asmmemmgr.py
+++ b/pypy/jit/backend/llsupport/test/test_asmmemmgr.py
@@ -211,14 +211,14 @@
     debug._log = debug.DebugLog()
     try:
         mc._dump(addr, 'test-logname-section')
-        log = list(debug._log)
+        log = list(debug._log) 
     finally:
         debug._log = None
     encoded = ''.join(writtencode).encode('hex').upper()
     ataddr = '@%x' % addr
     assert log == [('test-logname-section',
                     [('debug_print', 'CODE_DUMP', ataddr, '+0 ', encoded)])]
-    #
+    
     lltype.free(p, flavor='raw')
 
 def test_blockbuildermixin2():
diff --git a/pypy/jit/backend/llsupport/test/test_descr.py b/pypy/jit/backend/llsupport/test/test_descr.py
--- a/pypy/jit/backend/llsupport/test/test_descr.py
+++ b/pypy/jit/backend/llsupport/test/test_descr.py
@@ -3,7 +3,6 @@
 from pypy.jit.backend.llsupport import symbolic
 from pypy.rlib.objectmodel import Symbolic
 from pypy.rpython.annlowlevel import llhelper
-from pypy.jit.metainterp.history import BoxInt, BoxFloat, BoxPtr
 from pypy.jit.metainterp import history
 from pypy.jit.codewriter import longlong
 import sys, struct, py
@@ -149,7 +148,9 @@
     A2 = lltype.GcArray(lltype.Ptr(T))
     A3 = lltype.GcArray(lltype.Ptr(U))
     A4 = lltype.GcArray(lltype.Float)
-    A5 = lltype.GcArray(lltype.SingleFloat)
+    A5 = lltype.GcArray(lltype.Struct('x', ('v', lltype.Signed),
+                                      ('k', lltype.Signed)))
+    A6 = lltype.GcArray(lltype.SingleFloat)
     assert getArrayDescrClass(A2) is GcPtrArrayDescr
     assert getArrayDescrClass(A3) is NonGcPtrArrayDescr
     cls = getArrayDescrClass(A1)
@@ -158,7 +159,7 @@
     clsf = getArrayDescrClass(A4)
     assert clsf != cls
     assert clsf == getArrayDescrClass(lltype.GcArray(lltype.Float))
-    clss = getArrayDescrClass(A5)
+    clss = getArrayDescrClass(A6)
     assert clss not in (clsf, cls)
     assert clss == getArrayDescrClass(lltype.GcArray(rffi.UINT))
     #
@@ -168,11 +169,12 @@
     descr3 = get_array_descr(c0, A3)
     descr4 = get_array_descr(c0, A4)
     descr5 = get_array_descr(c0, A5)
+    descr6 = get_array_descr(c0, A6)
     assert descr1.__class__ is cls
     assert descr2.__class__ is GcPtrArrayDescr
     assert descr3.__class__ is NonGcPtrArrayDescr
     assert descr4.__class__ is clsf
-    assert descr5.__class__ is clss
+    assert descr6.__class__ is clss
     assert descr1 == get_array_descr(c0, lltype.GcArray(lltype.Char))
     assert not descr1.is_array_of_pointers()
     assert     descr2.is_array_of_pointers()
@@ -202,7 +204,8 @@
     assert descr2.get_item_size(False) == rffi.sizeof(lltype.Ptr(T))
     assert descr3.get_item_size(False) == rffi.sizeof(lltype.Ptr(U))
     assert descr4.get_item_size(False) == rffi.sizeof(lltype.Float)
-    assert descr5.get_item_size(False) == rffi.sizeof(lltype.SingleFloat)
+    assert descr5.get_item_size(False) == rffi.sizeof(lltype.Signed) * 2
+    assert descr6.get_item_size(False) == rffi.sizeof(lltype.SingleFloat)
     #
     assert isinstance(descr1.get_base_size(True), Symbolic)
     assert isinstance(descr2.get_base_size(True), Symbolic)
@@ -348,7 +351,6 @@
                             (rffi.SHORT,      True), (rffi.USHORT, False),
                             (rffi.INT,        True), (rffi.UINT,   False),
                             (rffi.LONG,       True), (rffi.ULONG,  False)]:
-        A = lltype.GcArray(RESTYPE)
         for tsc in [False, True]:
             c2 = GcCache(tsc)
             descr1 = get_call_descr(c2, [], RESTYPE)
@@ -379,7 +381,6 @@
     descr3i = get_array_descr(c0, lltype.GcArray(lltype.Char))
     assert descr3i.repr_of_descr() == '<CharArrayDescr>'
     #
-    cache = {}
     descr4 = get_call_descr(c0, [lltype.Char, lltype.Ptr(S)], lltype.Ptr(S))
     assert 'GcPtrCallDescr' in descr4.repr_of_descr()
     #
@@ -412,10 +413,10 @@
     ARGS = [lltype.Float, lltype.Ptr(ARRAY)]
     RES = lltype.Float
 
-    def f(a, b):
+    def f2(a, b):
         return float(b[0]) + a
 
-    fnptr = llhelper(lltype.Ptr(lltype.FuncType(ARGS, RES)), f)
+    fnptr = llhelper(lltype.Ptr(lltype.FuncType(ARGS, RES)), f2)
     descr2 = get_call_descr(c0, ARGS, RES)
     a = lltype.malloc(ARRAY, 3)
     opaquea = lltype.cast_opaque_ptr(llmemory.GCREF, a)
diff --git a/pypy/jit/backend/model.py b/pypy/jit/backend/model.py
--- a/pypy/jit/backend/model.py
+++ b/pypy/jit/backend/model.py
@@ -1,5 +1,5 @@
 from pypy.rlib.debug import debug_start, debug_print, debug_stop
-from pypy.jit.metainterp import history, compile
+from pypy.jit.metainterp import history
 
 
 class AbstractCPU(object):
@@ -213,6 +213,10 @@
     def typedescrof(TYPE):
         raise NotImplementedError
 
+    @staticmethod
+    def interiorfielddescrof(A, fieldname):
+        raise NotImplementedError
+
     # ---------- the backend-dependent operations ----------
 
     # lltype specific operations
diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -5,7 +5,7 @@
                                          BoxInt, Box, BoxPtr,
                                          LoopToken,
                                          ConstInt, ConstPtr,
-                                         BoxObj, Const,
+                                         BoxObj,
                                          ConstObj, BoxFloat, ConstFloat)
 from pypy.jit.metainterp.resoperation import ResOperation, rop
 from pypy.jit.metainterp.typesystem import deref
@@ -111,7 +111,7 @@
         self.cpu.set_future_value_int(0, 2)
         fail = self.cpu.execute_token(looptoken)
         res = self.cpu.get_latest_value_int(0)
-        assert res == 3        
+        assert res == 3
         assert fail.identifier == 1
 
     def test_compile_loop(self):
@@ -127,7 +127,7 @@
             ]
         inputargs = [i0]
         operations[2].setfailargs([i1])
-        
+
         self.cpu.compile_loop(inputargs, operations, looptoken)
         self.cpu.set_future_value_int(0, 2)
         fail = self.cpu.execute_token(looptoken)
@@ -148,7 +148,7 @@
             ]
         inputargs = [i0]
         operations[2].setfailargs([None, None, i1, None])
-        
+
         self.cpu.compile_loop(inputargs, operations, looptoken)
         self.cpu.set_future_value_int(0, 2)
         fail = self.cpu.execute_token(looptoken)
@@ -372,7 +372,7 @@
         for opnum, boxargs, retvalue in get_int_tests():
             res = self.execute_operation(opnum, boxargs, 'int')
             assert res.value == retvalue
-        
+
     def test_float_operations(self):
         from pypy.jit.metainterp.test.test_executor import get_float_tests
         for opnum, boxargs, rettype, retvalue in get_float_tests(self.cpu):
@@ -438,7 +438,7 @@
 
     def test_ovf_operations_reversed(self):
         self.test_ovf_operations(reversed=True)
-        
+
     def test_bh_call(self):
         cpu = self.cpu
         #
@@ -503,7 +503,7 @@
                                          [funcbox, BoxInt(num), BoxInt(num)],
                                          'int', descr=dyn_calldescr)
             assert res.value == 2 * num
-            
+
 
         if cpu.supports_floats:
             def func(f0, f1, f2, f3, f4, f5, f6, i0, i1, f7, f8, f9):
@@ -543,7 +543,7 @@
         funcbox = self.get_funcbox(self.cpu, func_ptr)
         res = self.execute_operation(rop.CALL, [funcbox] + map(BoxInt, args), 'int', descr=calldescr)
         assert res.value == func(*args)
-        
+
     def test_call_stack_alignment(self):
         # test stack alignment issues, notably for Mac OS/X.
         # also test the ordering of the arguments.
@@ -615,7 +615,7 @@
         res = self.execute_operation(rop.GETFIELD_GC, [t_box],
                                      'int', descr=shortdescr)
         assert res.value == 1331
-        
+
         #
         u_box, U_box = self.alloc_instance(self.U)
         fielddescr2 = self.cpu.fielddescrof(self.S, 'next')
@@ -695,7 +695,7 @@
 
     def test_failing_guard_class(self):
         t_box, T_box = self.alloc_instance(self.T)
-        u_box, U_box = self.alloc_instance(self.U)        
+        u_box, U_box = self.alloc_instance(self.U)
         null_box = self.null_instance()
         for opname, args in [(rop.GUARD_CLASS, [t_box, U_box]),
                              (rop.GUARD_CLASS, [u_box, T_box]),
@@ -787,7 +787,7 @@
         r = self.execute_operation(rop.GETARRAYITEM_GC, [a_box, BoxInt(3)],
                                    'int', descr=arraydescr)
         assert r.value == 160
-        
+
         #
         if isinstance(A, lltype.GcArray):
             A = lltype.Ptr(A)
@@ -880,6 +880,73 @@
                                    'int', descr=arraydescr)
         assert r.value == 7441
 
+    def test_array_of_structs(self):
+        TP = lltype.GcStruct('x')
+        ITEM = lltype.Struct('x',
+                             ('vs', lltype.Signed),
+                             ('vu', lltype.Unsigned),
+                             ('vsc', rffi.SIGNEDCHAR),
+                             ('vuc', rffi.UCHAR),
+                             ('vss', rffi.SHORT),
+                             ('vus', rffi.USHORT),
+                             ('vsi', rffi.INT),
+                             ('vui', rffi.UINT),
+                             ('k', lltype.Float),
+                             ('p', lltype.Ptr(TP)))
+        a_box, A = self.alloc_array_of(ITEM, 15)
+        s_box, S = self.alloc_instance(TP)
+        kdescr = self.cpu.interiorfielddescrof(A, 'k')
+        pdescr = self.cpu.interiorfielddescrof(A, 'p')
+        self.execute_operation(rop.SETINTERIORFIELD_GC, [a_box, BoxInt(3),
+                                                         boxfloat(1.5)],
+                               'void', descr=kdescr)
+        f = self.cpu.bh_getinteriorfield_gc_f(a_box.getref_base(), 3, kdescr)
+        assert longlong.getrealfloat(f) == 1.5
+        self.cpu.bh_setinteriorfield_gc_f(a_box.getref_base(), 3, kdescr, longlong.getfloatstorage(2.5))
+        r = self.execute_operation(rop.GETINTERIORFIELD_GC, [a_box, BoxInt(3)],
+                                   'float', descr=kdescr)
+        assert r.getfloat() == 2.5
+        #
+        NUMBER_FIELDS = [('vs', lltype.Signed),
+                         ('vu', lltype.Unsigned),
+                         ('vsc', rffi.SIGNEDCHAR),
+                         ('vuc', rffi.UCHAR),
+                         ('vss', rffi.SHORT),
+                         ('vus', rffi.USHORT),
+                         ('vsi', rffi.INT),
+                         ('vui', rffi.UINT)]
+        for name, TYPE in NUMBER_FIELDS[::-1]:
+            vdescr = self.cpu.interiorfielddescrof(A, name)
+            self.execute_operation(rop.SETINTERIORFIELD_GC, [a_box, BoxInt(3),
+                                                             BoxInt(-15)],
+                                   'void', descr=vdescr)
+        for name, TYPE in NUMBER_FIELDS:
+            vdescr = self.cpu.interiorfielddescrof(A, name)
+            i = self.cpu.bh_getinteriorfield_gc_i(a_box.getref_base(), 3,
+                                                  vdescr)
+            assert i == rffi.cast(lltype.Signed, rffi.cast(TYPE, -15))
+        for name, TYPE in NUMBER_FIELDS[::-1]:
+            vdescr = self.cpu.interiorfielddescrof(A, name)
+            self.cpu.bh_setinteriorfield_gc_i(a_box.getref_base(), 3,
+                                              vdescr, -25)
+        for name, TYPE in NUMBER_FIELDS:
+            vdescr = self.cpu.interiorfielddescrof(A, name)
+            r = self.execute_operation(rop.GETINTERIORFIELD_GC,
+                                       [a_box, BoxInt(3)],
+                                       'int', descr=vdescr)
+            assert r.getint() == rffi.cast(lltype.Signed, rffi.cast(TYPE, -25))
+        #
+        self.execute_operation(rop.SETINTERIORFIELD_GC, [a_box, BoxInt(4),
+                                                         s_box],
+                               'void', descr=pdescr)
+        r = self.cpu.bh_getinteriorfield_gc_r(a_box.getref_base(), 4, pdescr)
+        assert r == s_box.getref_base()
+        self.cpu.bh_setinteriorfield_gc_r(a_box.getref_base(), 3, pdescr,
+                                          s_box.getref_base())
+        r = self.execute_operation(rop.GETINTERIORFIELD_GC, [a_box, BoxInt(3)],
+                                   'ref', descr=pdescr)
+        assert r.getref_base() == s_box.getref_base()
+
     def test_string_basic(self):
         s_box = self.alloc_string("hello\xfe")
         r = self.execute_operation(rop.STRLEN, [s_box], 'int')
@@ -1402,7 +1469,7 @@
         addr = llmemory.cast_ptr_to_adr(func_ptr)
         return ConstInt(heaptracker.adr2int(addr))
 
-    
+
     MY_VTABLE = rclass.OBJECT_VTABLE    # for tests only
 
     S = lltype.GcForwardReference()
@@ -1439,7 +1506,6 @@
         return BoxPtr(lltype.nullptr(llmemory.GCREF.TO))
 
     def alloc_array_of(self, ITEM, length):
-        cpu = self.cpu
         A = lltype.GcArray(ITEM)
         a = lltype.malloc(A, length)
         a_box = BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, a))
@@ -2318,7 +2384,7 @@
         for opname, arg, res in ops:
             self.execute_operation(opname, [arg], 'void')
             assert self.guard_failed == res
-        
+
         lltype.free(x, flavor='raw')
 
     def test_assembler_call(self):
@@ -2398,7 +2464,7 @@
         FakeJitDriverSD.portal_calldescr = self.cpu.calldescrof(
             lltype.Ptr(lltype.FuncType(ARGS, RES)), ARGS, RES,
             EffectInfo.MOST_GENERAL)
-        
+
         ops = '''
         [f0, f1]
         f2 = float_add(f0, f1)
@@ -2489,7 +2555,7 @@
         FakeJitDriverSD.portal_calldescr = self.cpu.calldescrof(
             lltype.Ptr(lltype.FuncType(ARGS, RES)), ARGS, RES,
             EffectInfo.MOST_GENERAL)
-        
+
         ops = '''
         [f0, f1]
         f2 = float_add(f0, f1)
@@ -2940,4 +3006,4 @@
 
     def alloc_unicode(self, unicode):
         py.test.skip("implement me")
-    
+
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1,7 +1,7 @@
 import sys, os
 from pypy.jit.backend.llsupport import symbolic
 from pypy.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper
-from pypy.jit.metainterp.history import Const, Box, BoxInt, BoxPtr, BoxFloat
+from pypy.jit.metainterp.history import Const, Box, BoxInt, ConstInt
 from pypy.jit.metainterp.history import (AbstractFailDescr, INT, REF, FLOAT,
                                          LoopToken)
 from pypy.rpython.lltypesystem import lltype, rffi, rstr, llmemory
@@ -36,7 +36,6 @@
 from pypy.rlib import rgc
 from pypy.rlib.clibffi import FFI_DEFAULT_ABI
 from pypy.jit.backend.x86.jump import remap_frame_layout
-from pypy.jit.metainterp.history import ConstInt, BoxInt
 from pypy.jit.codewriter.effectinfo import EffectInfo
 from pypy.jit.codewriter import longlong
 
@@ -729,8 +728,8 @@
         # Also, make sure this is consistent with FRAME_FIXED_SIZE.
         self.mc.PUSH_r(ebp.value)
         self.mc.MOV_rr(ebp.value, esp.value)
-        for regloc in self.cpu.CALLEE_SAVE_REGISTERS:
-            self.mc.PUSH_r(regloc.value)
+        for loc in self.cpu.CALLEE_SAVE_REGISTERS:
+            self.mc.PUSH_r(loc.value)
 
         gcrootmap = self.cpu.gc_ll_descr.gcrootmap
         if gcrootmap and gcrootmap.is_shadow_stack:
@@ -994,7 +993,7 @@
         effectinfo = op.getdescr().get_extra_info()
         oopspecindex = effectinfo.oopspecindex
         genop_llong_list[oopspecindex](self, op, arglocs, resloc)
-        
+
     def regalloc_perform_math(self, op, arglocs, resloc):
         effectinfo = op.getdescr().get_extra_info()
         oopspecindex = effectinfo.oopspecindex
@@ -1311,7 +1310,7 @@
     genop_guard_float_eq = _cmpop_guard_float("E", "E", "NE","NE")
     genop_guard_float_gt = _cmpop_guard_float("A", "B", "BE","AE")
     genop_guard_float_ge = _cmpop_guard_float("AE","BE", "B", "A")
-    
+
     def genop_math_sqrt(self, op, arglocs, resloc):
         self.mc.SQRTSD(arglocs[0], resloc)
 
@@ -1597,12 +1596,27 @@
     genop_getarrayitem_gc_pure = genop_getarrayitem_gc
     genop_getarrayitem_raw = genop_getarrayitem_gc
 
+    def genop_getinteriorfield_gc(self, op, arglocs, resloc):
+        base_loc, ofs_loc, itemsize_loc, fieldsize_loc, index_loc, sign_loc = arglocs
+        # XXX should not use IMUL in most cases
+        self.mc.IMUL(index_loc, itemsize_loc)
+        src_addr = AddressLoc(base_loc, index_loc, 0, ofs_loc.value)
+        self.load_from_mem(resloc, src_addr, fieldsize_loc, sign_loc)
+
+
     def genop_discard_setfield_gc(self, op, arglocs):
         base_loc, ofs_loc, size_loc, value_loc = arglocs
         assert isinstance(size_loc, ImmedLoc)
         dest_addr = AddressLoc(base_loc, ofs_loc)
         self.save_into_mem(dest_addr, value_loc, size_loc)
 
+    def genop_discard_setinteriorfield_gc(self, op, arglocs):
+        base_loc, ofs_loc, itemsize_loc, fieldsize_loc, index_loc, value_loc = arglocs
+        # XXX should not use IMUL in most cases
+        self.mc.IMUL(index_loc, itemsize_loc)
+        dest_addr = AddressLoc(base_loc, index_loc, 0, ofs_loc.value)
+        self.save_into_mem(dest_addr, value_loc, fieldsize_loc)
+
     def genop_discard_setarrayitem_gc(self, op, arglocs):
         base_loc, ofs_loc, value_loc, size_loc, baseofs = arglocs
         assert isinstance(baseofs, ImmedLoc)
diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py
--- a/pypy/jit/backend/x86/regalloc.py
+++ b/pypy/jit/backend/x86/regalloc.py
@@ -7,7 +7,7 @@
                                          ResOperation, BoxPtr, ConstFloat,
                                          BoxFloat, LoopToken, INT, REF, FLOAT)
 from pypy.jit.backend.x86.regloc import *
-from pypy.rpython.lltypesystem import lltype, ll2ctypes, rffi, rstr
+from pypy.rpython.lltypesystem import lltype, rffi, rstr
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.rlib import rgc
 from pypy.jit.backend.llsupport import symbolic
@@ -17,11 +17,12 @@
 from pypy.jit.metainterp.resoperation import rop
 from pypy.jit.backend.llsupport.descr import BaseFieldDescr, BaseArrayDescr
 from pypy.jit.backend.llsupport.descr import BaseCallDescr, BaseSizeDescr
+from pypy.jit.backend.llsupport.descr import InteriorFieldDescr
 from pypy.jit.backend.llsupport.regalloc import FrameManager, RegisterManager,\
      TempBox
 from pypy.jit.backend.x86.arch import WORD, FRAME_FIXED_SIZE
 from pypy.jit.backend.x86.arch import IS_X86_32, IS_X86_64, MY_COPY_OF_REGS
-from pypy.rlib.rarithmetic import r_longlong, r_uint
+from pypy.rlib.rarithmetic import r_longlong
 
 class X86RegisterManager(RegisterManager):
 
@@ -433,7 +434,7 @@
             if self.can_merge_with_next_guard(op, i, operations):
                 oplist_with_guard[op.getopnum()](self, op, operations[i + 1])
                 i += 1
-            elif not we_are_translated() and op.getopnum() == -124: 
+            elif not we_are_translated() and op.getopnum() == -124:
                 self._consider_force_spill(op)
             else:
                 oplist[op.getopnum()](self, op)
@@ -815,7 +816,7 @@
         save_all_regs = guard_not_forced_op is not None
         self.xrm.before_call(force_store, save_all_regs=save_all_regs)
         if not save_all_regs:
-            gcrootmap = gc_ll_descr = self.assembler.cpu.gc_ll_descr.gcrootmap
+            gcrootmap = self.assembler.cpu.gc_ll_descr.gcrootmap
             if gcrootmap and gcrootmap.is_shadow_stack:
                 save_all_regs = 2
         self.rm.before_call(force_store, save_all_regs=save_all_regs)
@@ -972,74 +973,27 @@
             return self._call(op, arglocs)
 
     def consider_newstr(self, op):
-        gc_ll_descr = self.assembler.cpu.gc_ll_descr
-        if gc_ll_descr.get_funcptr_for_newstr is not None:
-            # framework GC
-            loc = self.loc(op.getarg(0))
-            return self._call(op, [loc])
-        # boehm GC (XXX kill the following code at some point)
-        ofs_items, itemsize, ofs = symbolic.get_array_token(rstr.STR, self.translate_support_code)
-        assert itemsize == 1
-        return self._malloc_varsize(ofs_items, ofs, 0, op.getarg(0),
-                                    op.result)
+        loc = self.loc(op.getarg(0))
+        return self._call(op, [loc])
 
     def consider_newunicode(self, op):
-        gc_ll_descr = self.assembler.cpu.gc_ll_descr
-        if gc_ll_descr.get_funcptr_for_newunicode is not None:
-            # framework GC
-            loc = self.loc(op.getarg(0))
-            return self._call(op, [loc])
-        # boehm GC (XXX kill the following code at some point)
-        ofs_items, _, ofs = symbolic.get_array_token(rstr.UNICODE,
-                                                   self.translate_support_code)
-        scale = self._get_unicode_item_scale()
-        return self._malloc_varsize(ofs_items, ofs, scale, op.getarg(0),
-                                    op.result)
-
-    def _malloc_varsize(self, ofs_items, ofs_length, scale, v, res_v):
-        # XXX kill this function at some point
-        if isinstance(v, Box):
-            loc = self.rm.make_sure_var_in_reg(v, [v])
-            tempbox = TempBox()
-            other_loc = self.rm.force_allocate_reg(tempbox, [v])
-            self.assembler.load_effective_addr(loc, ofs_items,scale, other_loc)
-        else:
-            tempbox = None
-            other_loc = imm(ofs_items + (v.getint() << scale))
-        self._call(ResOperation(rop.NEW, [], res_v),
-                   [other_loc], [v])
-        loc = self.rm.make_sure_var_in_reg(v, [res_v])
-        assert self.loc(res_v) == eax
-        # now we have to reload length to some reasonable place
-        self.rm.possibly_free_var(v)
-        if tempbox is not None:
-            self.rm.possibly_free_var(tempbox)
-        self.PerformDiscard(ResOperation(rop.SETFIELD_GC, [None, None], None),
-                            [eax, imm(ofs_length), imm(WORD), loc])
+        loc = self.loc(op.getarg(0))
+        return self._call(op, [loc])
 
     def consider_new_array(self, op):
         gc_ll_descr = self.assembler.cpu.gc_ll_descr
-        if gc_ll_descr.get_funcptr_for_newarray is not None:
-            # framework GC
-            box_num_elem = op.getarg(0)
-            if isinstance(box_num_elem, ConstInt):
-                num_elem = box_num_elem.value
-                if gc_ll_descr.can_inline_malloc_varsize(op.getdescr(),
-                                                         num_elem):
-                    self.fastpath_malloc_varsize(op, op.getdescr(), num_elem)
-                    return
-            args = self.assembler.cpu.gc_ll_descr.args_for_new_array(
-                op.getdescr())
-            arglocs = [imm(x) for x in args]
-            arglocs.append(self.loc(box_num_elem))
-            self._call(op, arglocs)
-            return
-        # boehm GC (XXX kill the following code at some point)
-        itemsize, basesize, ofs_length, _, _ = (
-            self._unpack_arraydescr(op.getdescr()))
-        scale_of_field = _get_scale(itemsize)
-        self._malloc_varsize(basesize, ofs_length, scale_of_field,
-                             op.getarg(0), op.result)
+        box_num_elem = op.getarg(0)
+        if isinstance(box_num_elem, ConstInt):
+            num_elem = box_num_elem.value
+            if gc_ll_descr.can_inline_malloc_varsize(op.getdescr(),
+                                                     num_elem):
+                self.fastpath_malloc_varsize(op, op.getdescr(), num_elem)
+                return
+        args = self.assembler.cpu.gc_ll_descr.args_for_new_array(
+            op.getdescr())
+        arglocs = [imm(x) for x in args]
+        arglocs.append(self.loc(box_num_elem))
+        self._call(op, arglocs)
 
     def _unpack_arraydescr(self, arraydescr):
         assert isinstance(arraydescr, BaseArrayDescr)
@@ -1058,6 +1012,16 @@
         sign = fielddescr.is_field_signed()
         return imm(ofs), imm(size), ptr, sign
 
+    def _unpack_interiorfielddescr(self, descr):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs = arraydescr.get_base_size(self.translate_support_code)
+        itemsize = arraydescr.get_item_size(self.translate_support_code)
+        fieldsize = descr.fielddescr.get_field_size(self.translate_support_code)
+        sign = descr.fielddescr.is_field_signed()
+        ofs += descr.fielddescr.offset
+        return imm(ofs), imm(itemsize), imm(fieldsize), sign
+
     def consider_setfield_gc(self, op):
         ofs_loc, size_loc, _, _ = self._unpack_fielddescr(op.getdescr())
         assert isinstance(size_loc, ImmedLoc)
@@ -1074,6 +1038,21 @@
 
     consider_setfield_raw = consider_setfield_gc
 
+    def consider_setinteriorfield_gc(self, op):
+        t = self._unpack_interiorfielddescr(op.getdescr())
+        ofs, itemsize, fieldsize, _ = t
+        args = op.getarglist()
+        tmpvar = TempBox()
+        base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
+        index_loc = self.rm.force_result_in_reg(tmpvar, op.getarg(1),
+                                                args)
+        # we're free to modify index now
+        value_loc = self.make_sure_var_in_reg(op.getarg(2), args)
+        self.possibly_free_vars(args)
+        self.rm.possibly_free_var(tmpvar)
+        self.PerformDiscard(op, [base_loc, ofs, itemsize, fieldsize,
+                                 index_loc, value_loc])
+
     def consider_strsetitem(self, op):
         args = op.getarglist()
         base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
@@ -1135,6 +1114,24 @@
     consider_getarrayitem_raw = consider_getarrayitem_gc
     consider_getarrayitem_gc_pure = consider_getarrayitem_gc
 
+    def consider_getinteriorfield_gc(self, op):
+        t = self._unpack_interiorfielddescr(op.getdescr())
+        ofs, itemsize, fieldsize, sign = t
+        if sign:
+            sign_loc = imm1
+        else:
+            sign_loc = imm0
+        args = op.getarglist()
+        tmpvar = TempBox()
+        base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
+        index_loc = self.rm.force_result_in_reg(tmpvar, op.getarg(1),
+                                                args)
+        self.rm.possibly_free_vars_for_op(op)
+        self.rm.possibly_free_var(tmpvar)
+        result_loc = self.force_allocate_reg(op.result)
+        self.Perform(op, [base_loc, ofs, itemsize, fieldsize,
+                          index_loc, sign_loc], result_loc)
+
     def consider_int_is_true(self, op, guard_op):
         # doesn't need arg to be in a register
         argloc = self.loc(op.getarg(0))
@@ -1241,7 +1238,6 @@
         self.rm.possibly_free_var(srcaddr_box)
 
     def _gen_address_inside_string(self, baseloc, ofsloc, resloc, is_unicode):
-        cpu = self.assembler.cpu
         if is_unicode:
             ofs_items, _, _ = symbolic.get_array_token(rstr.UNICODE,
                                                   self.translate_support_code)
@@ -1300,7 +1296,7 @@
         tmpreg = X86RegisterManager.all_regs[0]
         tmploc = self.rm.force_allocate_reg(box, selected_reg=tmpreg)
         xmmtmp = X86XMMRegisterManager.all_regs[0]
-        xmmtmploc = self.xrm.force_allocate_reg(box1, selected_reg=xmmtmp)
+        self.xrm.force_allocate_reg(box1, selected_reg=xmmtmp)
         # Part about non-floats
         # XXX we don't need a copy, we only just the original list
         src_locations1 = [self.loc(op.getarg(i)) for i in range(op.numargs())
@@ -1380,7 +1376,7 @@
     return lambda self, op: fn(self, op, None)
 
 def is_comparison_or_ovf_op(opnum):
-    from pypy.jit.metainterp.resoperation import opclasses, AbstractResOp
+    from pypy.jit.metainterp.resoperation import opclasses
     cls = opclasses[opnum]
     # hack hack: in theory they are instance method, but they don't use
     # any instance field, we can use a fake object
diff --git a/pypy/jit/backend/x86/test/test_del.py b/pypy/jit/backend/x86/test/test_del.py
--- a/pypy/jit/backend/x86/test/test_del.py
+++ b/pypy/jit/backend/x86/test/test_del.py
@@ -1,5 +1,4 @@
 
-import py
 from pypy.jit.backend.x86.test.test_basic import Jit386Mixin
 from pypy.jit.metainterp.test.test_del import DelTests
 
diff --git a/pypy/jit/backend/x86/test/test_dict.py b/pypy/jit/backend/x86/test/test_dict.py
new file mode 100644
--- /dev/null
+++ b/pypy/jit/backend/x86/test/test_dict.py
@@ -0,0 +1,9 @@
+
+from pypy.jit.backend.x86.test.test_basic import Jit386Mixin
+from pypy.jit.metainterp.test.test_dict import DictTests
+
+
+class TestDict(Jit386Mixin, DictTests):
+    # for the individual tests see
+    # ====> ../../../metainterp/test/test_dict.py
+    pass
diff --git a/pypy/jit/backend/x86/test/test_runner.py b/pypy/jit/backend/x86/test/test_runner.py
--- a/pypy/jit/backend/x86/test/test_runner.py
+++ b/pypy/jit/backend/x86/test/test_runner.py
@@ -31,7 +31,7 @@
 
     # for the individual tests see
     # ====> ../../test/runner_test.py
-    
+
     def setup_method(self, meth):
         self.cpu = CPU(rtyper=None, stats=FakeStats())
         self.cpu.setup_once()
@@ -69,22 +69,16 @@
 
     def test_allocations(self):
         from pypy.rpython.lltypesystem import rstr
-        
+
         allocs = [None]
         all = []
+        orig_new = self.cpu.gc_ll_descr.funcptr_for_new
         def f(size):
             allocs.insert(0, size)
-            buf = ctypes.create_string_buffer(size)
-            all.append(buf)
-            return ctypes.cast(buf, ctypes.c_void_p).value
-        func = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int)(f)
-        addr = ctypes.cast(func, ctypes.c_void_p).value
-        # ctypes produces an unsigned value. We need it to be signed for, eg,
-        # relative addressing to work properly.
-        addr = rffi.cast(lltype.Signed, addr)
-        
+            return orig_new(size)
+
         self.cpu.assembler.setup_once()
-        self.cpu.assembler.malloc_func_addr = addr
+        self.cpu.gc_ll_descr.funcptr_for_new = f
         ofs = symbolic.get_field_token(rstr.STR, 'chars', False)[0]
 
         res = self.execute_operation(rop.NEWSTR, [ConstInt(7)], 'ref')
@@ -108,7 +102,7 @@
         res = self.execute_operation(rop.NEW_ARRAY, [ConstInt(10)],
                                          'ref', descr)
         assert allocs[0] == 10*WORD + ofs + WORD
-        resbuf = self._resbuf(res)            
+        resbuf = self._resbuf(res)
         assert resbuf[ofs/WORD] == 10
 
         # ------------------------------------------------------------
@@ -116,7 +110,7 @@
         res = self.execute_operation(rop.NEW_ARRAY, [BoxInt(10)],
                                          'ref', descr)
         assert allocs[0] == 10*WORD + ofs + WORD
-        resbuf = self._resbuf(res)                        
+        resbuf = self._resbuf(res)
         assert resbuf[ofs/WORD] == 10
 
     def test_stringitems(self):
@@ -146,7 +140,7 @@
                                                      ConstInt(2), BoxInt(38)],
                                'void', descr)
         assert resbuf[itemsofs/WORD + 2] == 38
-        
+
         self.execute_operation(rop.SETARRAYITEM_GC, [res,
                                                      BoxInt(3), BoxInt(42)],
                                'void', descr)
@@ -167,7 +161,7 @@
                                                          BoxInt(2)],
                                    'int', descr)
         assert r.value == 38
-        
+
         r = self.execute_operation(rop.GETARRAYITEM_GC, [res, BoxInt(3)],
                                    'int', descr)
         assert r.value == 42
@@ -226,7 +220,7 @@
         self.execute_operation(rop.SETFIELD_GC, [res, BoxInt(1234)], 'void', ofs_i)
         i = self.execute_operation(rop.GETFIELD_GC, [res], 'int', ofs_i)
         assert i.value == 1234
-        
+
         #u = self.execute_operation(rop.GETFIELD_GC, [res, ofs_u], 'int')
         #assert u.value == 5
         self.execute_operation(rop.SETFIELD_GC, [res, ConstInt(1)], 'void',
@@ -299,7 +293,7 @@
                     else:
                         assert result != execute(self.cpu, None,
                                                  op, None, b).value
-                    
+
 
     def test_stuff_followed_by_guard(self):
         boxes = [(BoxInt(1), BoxInt(0)),
@@ -523,7 +517,7 @@
     def test_debugger_on(self):
         from pypy.tool.logparser import parse_log_file, extract_category
         from pypy.rlib import debug
-        
+
         loop = """
         [i0]
         debug_merge_point('xyz', 0)
diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -52,9 +52,11 @@
         newoperations = []
         #
         def do_rename(var, var_or_const):
+            if var.concretetype is lltype.Void:
+                renamings[var] = Constant(None, lltype.Void)
+                return
             renamings[var] = var_or_const
-            if (isinstance(var_or_const, Constant)
-                and var.concretetype != lltype.Void):
+            if isinstance(var_or_const, Constant):
                 value = var_or_const.value
                 value = lltype._cast_whatever(var.concretetype, value)
                 renamings_constants[var] = Constant(value, var.concretetype)
@@ -735,29 +737,54 @@
         return SpaceOperation(opname, [op.args[0]], op.result)
 
     def rewrite_op_getinteriorfield(self, op):
-        # only supports strings and unicodes
         assert len(op.args) == 3
-        assert op.args[1].value == 'chars'
         optype = op.args[0].concretetype
         if optype == lltype.Ptr(rstr.STR):
             opname = "strgetitem"
+            return SpaceOperation(opname, [op.args[0], op.args[2]], op.result)
+        elif optype == lltype.Ptr(rstr.UNICODE):
+            opname = "unicodegetitem"
+            return SpaceOperation(opname, [op.args[0], op.args[2]], op.result)
         else:
-            assert optype == lltype.Ptr(rstr.UNICODE)
-            opname = "unicodegetitem"
-        return SpaceOperation(opname, [op.args[0], op.args[2]], op.result)
+            v_inst, v_index, c_field = op.args
+            if op.result.concretetype is lltype.Void:
+                return
+            # only GcArray of Struct supported
+            assert isinstance(v_inst.concretetype.TO, lltype.GcArray)
+            STRUCT = v_inst.concretetype.TO.OF
+            assert isinstance(STRUCT, lltype.Struct)
+            descr = self.cpu.interiorfielddescrof(v_inst.concretetype.TO,
+                                                  c_field.value)
+            args = [v_inst, v_index, descr]
+            kind = getkind(op.result.concretetype)[0]
+            return SpaceOperation('getinteriorfield_gc_%s' % kind, args,
+                                  op.result)
 
     def rewrite_op_setinteriorfield(self, op):
-        # only supports strings and unicodes
         assert len(op.args) == 4
-        assert op.args[1].value == 'chars'
         optype = op.args[0].concretetype
         if optype == lltype.Ptr(rstr.STR):
             opname = "strsetitem"
+            return SpaceOperation(opname, [op.args[0], op.args[2], op.args[3]],
+                                  op.result)
+        elif optype == lltype.Ptr(rstr.UNICODE):
+            opname = "unicodesetitem"
+            return SpaceOperation(opname, [op.args[0], op.args[2], op.args[3]],
+                                  op.result)
         else:
-            assert optype == lltype.Ptr(rstr.UNICODE)
-            opname = "unicodesetitem"
-        return SpaceOperation(opname, [op.args[0], op.args[2], op.args[3]],
-                              op.result)
+            v_inst, v_index, c_field, v_value = op.args
+            if v_value.concretetype is lltype.Void:
+                return
+            # only GcArray of Struct supported
+            assert isinstance(v_inst.concretetype.TO, lltype.GcArray)
+            STRUCT = v_inst.concretetype.TO.OF
+            assert isinstance(STRUCT, lltype.Struct)
+            descr = self.cpu.interiorfielddescrof(v_inst.concretetype.TO,
+                                                  c_field.value)
+            kind = getkind(v_value.concretetype)[0]
+            args = [v_inst, v_index, v_value, descr]
+            return SpaceOperation('setinteriorfield_gc_%s' % kind, args,
+                                  op.result)
 
     def _rewrite_equality(self, op, opname):
         arg0, arg1 = op.args
diff --git a/pypy/jit/codewriter/support.py b/pypy/jit/codewriter/support.py
--- a/pypy/jit/codewriter/support.py
+++ b/pypy/jit/codewriter/support.py
@@ -13,7 +13,6 @@
 from pypy.translator.simplify import get_funcobj
 from pypy.translator.unsimplify import split_block
 from pypy.objspace.flow.model import Constant
-from pypy import conftest
 from pypy.translator.translator import TranslationContext
 from pypy.annotation.policy import AnnotatorPolicy
 from pypy.annotation import model as annmodel
@@ -48,15 +47,13 @@
     a.build_types(func, argtypes, main_entry_point=True)
     rtyper = t.buildrtyper(type_system = type_system)
     rtyper.specialize()
-    if inline:
-        auto_inlining(t, threshold=inline)
+    #if inline:
+    #    auto_inlining(t, threshold=inline)
     if backendoptimize:
         from pypy.translator.backendopt.all import backend_optimizations
         backend_optimizations(t, inline_threshold=inline or 0,
                 remove_asserts=True, really_remove_asserts=True)
 
-    #if conftest.option.view:
-    #    t.view()
     return rtyper
 
 def getgraph(func, values):
@@ -456,6 +453,8 @@
         return LLtypeHelpers._dictnext_items(lltype.Ptr(RES), iter)
     _ll_1_dictiter_nextitems.need_result_type = True
 
+    _ll_1_dict_resize = ll_rdict.ll_dict_resize
+
     # ---------- strings and unicode ----------
 
     _ll_1_str_str2unicode = ll_rstr.LLHelpers.ll_str2unicode
diff --git a/pypy/jit/codewriter/test/test_jtransform.py b/pypy/jit/codewriter/test/test_jtransform.py
--- a/pypy/jit/codewriter/test/test_jtransform.py
+++ b/pypy/jit/codewriter/test/test_jtransform.py
@@ -1,4 +1,3 @@
-import py
 import random
 try:
     from itertools import product
@@ -16,13 +15,13 @@
 
 from pypy.objspace.flow.model import FunctionGraph, Block, Link
 from pypy.objspace.flow.model import SpaceOperation, Variable, Constant
-from pypy.jit.codewriter.jtransform import Transformer
-from pypy.jit.metainterp.history import getkind
-from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr, rlist
+from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr
 from pypy.rpython.lltypesystem.module import ll_math
 from pypy.translator.unsimplify import varoftype
 from pypy.jit.codewriter import heaptracker, effectinfo
 from pypy.jit.codewriter.flatten import ListOfKind
+from pypy.jit.codewriter.jtransform import Transformer
+from pypy.jit.metainterp.history import getkind
 
 def const(x):
     return Constant(x, lltype.typeOf(x))
@@ -37,6 +36,8 @@
         return ('calldescr', FUNC, ARGS, RESULT)
     def fielddescrof(self, STRUCT, name):
         return ('fielddescr', STRUCT, name)
+    def interiorfielddescrof(self, ARRAY, name):
+        return ('interiorfielddescr', ARRAY, name)
     def arraydescrof(self, ARRAY):
         return FakeDescr(('arraydescr', ARRAY))
     def sizeof(self, STRUCT):
@@ -539,7 +540,7 @@
 
 def test_rename_on_links():
     v1 = Variable()
-    v2 = Variable()
+    v2 = Variable(); v2.concretetype = llmemory.Address
     v3 = Variable()
     block = Block([v1])
     block.operations = [SpaceOperation('cast_pointer', [v1], v2)]
@@ -676,6 +677,22 @@
     assert op1.args == [v, v_index]
     assert op1.result == v_result
 
+def test_dict_getinteriorfield():
+    DICT = lltype.GcArray(lltype.Struct('ENTRY', ('v', lltype.Signed),
+                                        ('k', lltype.Signed)))
+    v = varoftype(lltype.Ptr(DICT))
+    i = varoftype(lltype.Signed)
+    v_result = varoftype(lltype.Signed)
+    op = SpaceOperation('getinteriorfield', [v, i, Constant('v', lltype.Void)],
+                        v_result)
+    op1 = Transformer(FakeCPU()).rewrite_operation(op)
+    assert op1.opname == 'getinteriorfield_gc_i'
+    assert op1.args == [v, i, ('interiorfielddescr', DICT, 'v')]
+    op = SpaceOperation('getinteriorfield', [v, i, Constant('v', lltype.Void)],
+                        Constant(None, lltype.Void))
+    op1 = Transformer(FakeCPU()).rewrite_operation(op)
+    assert op1 is None
+
 def test_str_setinteriorfield():
     v = varoftype(lltype.Ptr(rstr.STR))
     v_index = varoftype(lltype.Signed)
@@ -702,6 +719,23 @@
     assert op1.args == [v, v_index, v_newchr]
     assert op1.result == v_void
 
+def test_dict_setinteriorfield():
+    DICT = lltype.GcArray(lltype.Struct('ENTRY', ('v', lltype.Signed),
+                                        ('k', lltype.Signed)))
+    v = varoftype(lltype.Ptr(DICT))
+    i = varoftype(lltype.Signed)
+    v_void = varoftype(lltype.Void)
+    op = SpaceOperation('setinteriorfield', [v, i, Constant('v', lltype.Void),
+                                             i],
+                        v_void)
+    op1 = Transformer(FakeCPU()).rewrite_operation(op)
+    assert op1.opname == 'setinteriorfield_gc_i'
+    assert op1.args == [v, i, i, ('interiorfielddescr', DICT, 'v')]
+    op = SpaceOperation('setinteriorfield', [v, i, Constant('v', lltype.Void),
+                                             v_void], v_void)
+    op1 = Transformer(FakeCPU()).rewrite_operation(op)
+    assert not op1
+
 def test_promote_1():
     v1 = varoftype(lltype.Signed)
     v2 = varoftype(lltype.Signed)
diff --git a/pypy/jit/metainterp/blackhole.py b/pypy/jit/metainterp/blackhole.py
--- a/pypy/jit/metainterp/blackhole.py
+++ b/pypy/jit/metainterp/blackhole.py
@@ -6,7 +6,6 @@
 from pypy.rlib.debug import make_sure_not_resized
 from pypy.rpython.lltypesystem import lltype, llmemory, rclass
 from pypy.rpython.lltypesystem.lloperation import llop
-from pypy.rpython.llinterp import LLException
 from pypy.jit.codewriter.jitcode import JitCode, SwitchDictDescr
 from pypy.jit.codewriter import heaptracker, longlong
 from pypy.jit.metainterp.jitexc import JitException, get_llexception, reraise
@@ -1154,6 +1153,26 @@
         array = cpu.bh_getfield_gc_r(vable, fdescr)
         return cpu.bh_arraylen_gc(adescr, array)
 
+    @arguments("cpu", "r", "i", "d", returns="i")
+    def bhimpl_getinteriorfield_gc_i(cpu, array, index, descr):
+        return cpu.bh_getinteriorfield_gc_i(array, index, descr)
+    @arguments("cpu", "r", "i", "d", returns="r")
+    def bhimpl_getinteriorfield_gc_r(cpu, array, index, descr):
+        return cpu.bh_getinteriorfield_gc_r(array, index, descr)
+    @arguments("cpu", "r", "i", "d", returns="f")
+    def bhimpl_getinteriorfield_gc_f(cpu, array, index, descr):
+        return cpu.bh_getinteriorfield_gc_f(array, index, descr)
+
+    @arguments("cpu", "r", "i", "d", "i")
+    def bhimpl_setinteriorfield_gc_i(cpu, array, index, descr, value):
+        cpu.bh_setinteriorfield_gc_i(array, index, descr, value)
+    @arguments("cpu", "r", "i", "d", "r")
+    def bhimpl_setinteriorfield_gc_r(cpu, array, index, descr, value):
+        cpu.bh_setinteriorfield_gc_r(array, index, descr, value)
+    @arguments("cpu", "r", "i", "d", "f")
+    def bhimpl_setinteriorfield_gc_f(cpu, array, index, descr, value):
+        cpu.bh_setinteriorfield_gc_f(array, index, descr, value)
+
     @arguments("cpu", "r", "d", returns="i")
     def bhimpl_getfield_gc_i(cpu, struct, fielddescr):
         return cpu.bh_getfield_gc_i(struct, fielddescr)
diff --git a/pypy/jit/metainterp/executor.py b/pypy/jit/metainterp/executor.py
--- a/pypy/jit/metainterp/executor.py
+++ b/pypy/jit/metainterp/executor.py
@@ -1,11 +1,8 @@
 """This implements pyjitpl's execution of operations.
 """
 
-import py
-from pypy.rpython.lltypesystem import lltype, llmemory, rstr
-from pypy.rpython.ootypesystem import ootype
-from pypy.rpython.lltypesystem.lloperation import llop
-from pypy.rlib.rarithmetic import ovfcheck, r_uint, intmask, r_longlong
+from pypy.rpython.lltypesystem import lltype, rstr
+from pypy.rlib.rarithmetic import ovfcheck, r_longlong
 from pypy.rlib.rtimer import read_timestamp
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.jit.metainterp.history import BoxInt, BoxPtr, BoxFloat, check_descr
@@ -123,6 +120,29 @@
     else:
         cpu.bh_setarrayitem_raw_i(arraydescr, array, index, itembox.getint())
 
+def do_getinteriorfield_gc(cpu, _, arraybox, indexbox, descr):
+    array = arraybox.getref_base()
+    index = indexbox.getint()
+    if descr.is_pointer_field():
+        return BoxPtr(cpu.bh_getinteriorfield_gc_r(array, index, descr))
+    elif descr.is_float_field():
+        return BoxFloat(cpu.bh_getinteriorfield_gc_f(array, index, descr))
+    else:
+        return BoxInt(cpu.bh_getinteriorfield_gc_i(array, index, descr))
+
+def do_setinteriorfield_gc(cpu, _, arraybox, indexbox, valuebox, descr):
+    array = arraybox.getref_base()
+    index = indexbox.getint()
+    if descr.is_pointer_field():
+        cpu.bh_setinteriorfield_gc_r(array, index, descr,
+                                     valuebox.getref_base())
+    elif descr.is_float_field():
+        cpu.bh_setinteriorfield_gc_f(array, index, descr,
+                                     valuebox.getfloatstorage())
+    else:
+        cpu.bh_setinteriorfield_gc_i(array, index, descr,
+                                     valuebox.getint())
+
 def do_getfield_gc(cpu, _, structbox, fielddescr):
     struct = structbox.getref_base()
     if fielddescr.is_pointer_field():
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -36,6 +36,7 @@
 
 
 class MIFrame(object):
+    debug = False
 
     def __init__(self, metainterp):
         self.metainterp = metainterp
@@ -548,6 +549,14 @@
     opimpl_getfield_gc_r_pure = _opimpl_getfield_gc_pure_any
     opimpl_getfield_gc_f_pure = _opimpl_getfield_gc_pure_any
 
+    @arguments("box", "box", "descr")
+    def _opimpl_getinteriorfield_gc_any(self, array, index, descr):
+        return self.execute_with_descr(rop.GETINTERIORFIELD_GC, descr,
+                                       array, index)
+    opimpl_getinteriorfield_gc_i = _opimpl_getinteriorfield_gc_any
+    opimpl_getinteriorfield_gc_f = _opimpl_getinteriorfield_gc_any
+    opimpl_getinteriorfield_gc_r = _opimpl_getinteriorfield_gc_any
+
     @specialize.arg(1)
     def _opimpl_getfield_gc_any_pureornot(self, opnum, box, fielddescr):
         tobox = self.metainterp.heapcache.getfield(box, fielddescr)
@@ -588,6 +597,15 @@
     opimpl_setfield_gc_r = _opimpl_setfield_gc_any
     opimpl_setfield_gc_f = _opimpl_setfield_gc_any
 
+    @arguments("box", "box", "box", "descr")
+    def _opimpl_setinteriorfield_gc_any(self, array, index, value, descr):
+        self.execute_with_descr(rop.SETINTERIORFIELD_GC, descr,
+                                array, index, value)
+    opimpl_setinteriorfield_gc_i = _opimpl_setinteriorfield_gc_any
+    opimpl_setinteriorfield_gc_f = _opimpl_setinteriorfield_gc_any
+    opimpl_setinteriorfield_gc_r = _opimpl_setinteriorfield_gc_any
+                                
+
     @arguments("box", "descr")
     def _opimpl_getfield_raw_any(self, box, fielddescr):
         return self.execute_with_descr(rop.GETFIELD_RAW, fielddescr, box)
@@ -2588,17 +2606,21 @@
         self.pc = position
         #
         if not we_are_translated():
-            print '\tpyjitpl: %s(%s)' % (name, ', '.join(map(repr, args))),
+            if self.debug:
+                print '\tpyjitpl: %s(%s)' % (name, ', '.join(map(repr, args))),
             try:
                 resultbox = unboundmethod(self, *args)
             except Exception, e:
-                print '-> %s!' % e.__class__.__name__
+                if self.debug:
+                    print '-> %s!' % e.__class__.__name__
                 raise
             if num_return_args == 0:
-                print
+                if self.debug:
+                    print
                 assert resultbox is None
             else:
-                print '-> %r' % (resultbox,)
+                if self.debug:
+                    print '-> %r' % (resultbox,)
                 assert argcodes[next_argcode] == '>'
                 result_argcode = argcodes[next_argcode + 1]
                 assert resultbox.type == {'i': history.INT,
diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py
--- a/pypy/jit/metainterp/resoperation.py
+++ b/pypy/jit/metainterp/resoperation.py
@@ -1,5 +1,4 @@
 from pypy.rlib.objectmodel import we_are_translated
-from pypy.rlib.debug import make_sure_not_resized
 
 def ResOperation(opnum, args, result, descr=None):
     cls = opclasses[opnum]
@@ -457,6 +456,7 @@
 
     'GETARRAYITEM_GC/2d',
     'GETARRAYITEM_RAW/2d',
+    'GETINTERIORFIELD_GC/2d',
     'GETFIELD_GC/1d',
     'GETFIELD_RAW/1d',
     '_MALLOC_FIRST',
@@ -473,6 +473,7 @@
 
     'SETARRAYITEM_GC/3d',
     'SETARRAYITEM_RAW/3d',
+    'SETINTERIORFIELD_GC/3d',
     'SETFIELD_GC/2d',
     'SETFIELD_RAW/2d',
     'STRSETITEM/3',
diff --git a/pypy/jit/metainterp/test/test_dict.py b/pypy/jit/metainterp/test/test_dict.py
--- a/pypy/jit/metainterp/test/test_dict.py
+++ b/pypy/jit/metainterp/test/test_dict.py
@@ -91,7 +91,7 @@
         res1 = f(100)
         res2 = self.meta_interp(f, [100], listops=True)
         assert res1 == res2
-        self.check_loops(int_mod=1) # the hash was traced
+        self.check_loops(int_mod=1) # the hash was traced and eq, but cached
 
     def test_dict_setdefault(self):
         myjitdriver = JitDriver(greens = [], reds = ['total', 'dct'])
@@ -128,7 +128,7 @@
         assert f(100) == 50
         res = self.meta_interp(f, [100], listops=True)
         assert res == 50
-        self.check_loops(int_mod=1)
+        self.check_loops(int_mod=1) # key + eq, but cached
 
     def test_repeated_lookup(self):
         myjitdriver = JitDriver(greens = [], reds = ['n', 'd'])
@@ -153,10 +153,12 @@
 
         res = self.meta_interp(f, [100], listops=True)
         assert res == f(50)
-        self.check_loops({"call": 7, "guard_false": 1, "guard_no_exception": 6,
+        self.check_loops({"call": 5, "getfield_gc": 1, "getinteriorfield_gc": 1,
+                          "guard_false": 1, "guard_no_exception": 4,
                           "guard_true": 1, "int_and": 1, "int_gt": 1,
                           "int_is_true": 1, "int_sub": 1, "jump": 1,
-                          "new_with_vtable": 1, "setfield_gc": 1})
+                          "new_with_vtable": 1, "new": 1, "new_array": 1,
+                          "setfield_gc": 3, })
 
 
 class TestOOtype(DictTests, OOJitMixin):
diff --git a/pypy/jit/metainterp/warmspot.py b/pypy/jit/metainterp/warmspot.py
--- a/pypy/jit/metainterp/warmspot.py
+++ b/pypy/jit/metainterp/warmspot.py
@@ -206,7 +206,7 @@
         self.make_enter_functions()
         self.rewrite_jit_merge_points(policy)
 
-        verbose = not self.cpu.translate_support_code
+        verbose = False # not self.cpu.translate_support_code
         self.codewriter.make_jitcodes(verbose=verbose)
         self.rewrite_can_enter_jits()
         self.rewrite_set_param()
diff --git a/pypy/module/pypyjit/test_pypy_c/test_containers.py b/pypy/module/pypyjit/test_pypy_c/test_containers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_containers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_containers.py
@@ -46,7 +46,7 @@
         assert loop.match_by_id("getitem", """
             i28 = call(ConstClass(ll_dict_lookup__dicttablePtr_objectPtr_Signed), p18, p6, i25, descr=...)
             ...
-            p33 = call(ConstClass(ll_get_value__dicttablePtr_Signed), p18, i28, descr=...)
+            p33 = getinteriorfield_gc(p31, i26, <InteriorFieldDescr <GcPtrFieldDescr dictentry.value .*>>)
             ...
         """)
 
diff --git a/pypy/rpython/lltypesystem/ll2ctypes.py b/pypy/rpython/lltypesystem/ll2ctypes.py
--- a/pypy/rpython/lltypesystem/ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/ll2ctypes.py
@@ -15,12 +15,11 @@
     load_library_kwargs = {}
 
 import os
-from pypy import conftest
 from pypy.rpython.lltypesystem import lltype, llmemory
 from pypy.rpython.extfunc import ExtRegistryEntry
 from pypy.rlib.objectmodel import Symbolic, ComputedIntSymbolic
 from pypy.tool.uid import fixid
-from pypy.rlib.rarithmetic import r_uint, r_singlefloat, r_longfloat, base_int, intmask
+from pypy.rlib.rarithmetic import r_singlefloat, r_longfloat, base_int, intmask
 from pypy.annotation import model as annmodel
 from pypy.rpython.llinterp import LLInterpreter, LLException
 from pypy.rpython.lltypesystem.rclass import OBJECT, OBJECT_VTABLE
@@ -531,6 +530,10 @@
     def __str__(self):
         return repr(self)
 
+    def _setparentstructure(self, parent, parentindex):
+        super(_parentable_mixin, self)._setparentstructure(parent, parentindex)
+        self._keepparent = parent   # always keep a strong ref
+
 class _struct_mixin(_parentable_mixin):
     """Mixin added to _struct containers when they become ctypes-based."""
     __slots__ = ()
@@ -566,7 +569,10 @@
         return 0, sys.maxint
 
     def getitem(self, index, uninitialized_ok=False):
-        return self._storage.contents._getitem(index, boundscheck=False)
+        res = self._storage.contents._getitem(index, boundscheck=False)
+        if isinstance(self._TYPE.OF, lltype.ContainerType):
+            res._obj._setparentstructure(self, index)
+        return res
 
     def setitem(self, index, value):
         self._storage.contents._setitem(index, value, boundscheck=False)
@@ -1279,6 +1285,8 @@
             self.intval = intmask(void_p.value)
 
     def __eq__(self, other):
+        if not other:
+            return self.intval == 0
         if isinstance(other, _llgcopaque):
             return self.intval == other.intval
         storage = object()
diff --git a/pypy/rpython/lltypesystem/lltype.py b/pypy/rpython/lltypesystem/lltype.py
--- a/pypy/rpython/lltypesystem/lltype.py
+++ b/pypy/rpython/lltypesystem/lltype.py
@@ -1522,7 +1522,7 @@
             and parentindex in (self._parent_type._names[0], 0)
             and self._TYPE._gckind == typeOf(parent)._gckind):
             # keep strong reference to parent, we share the same allocation
-            self._keepparent = parent 
+            self._keepparent = parent
 
     def _parentstructure(self, check=True):
         if self._wrparent is not None:
@@ -1730,7 +1730,8 @@
         # Keep the parent array alive, we share the same allocation.
         # Don't do it if we are inside a GC object, though -- it's someone
         # else's job to keep the GC object alive
-        if typeOf(top_container(parent))._gckind == 'raw':
+        if (typeOf(top_container(parent))._gckind == 'raw' or
+            hasattr(top_container(parent)._storage, 'contents')):  # ll2ctypes
             self._keepparent = parent
 
     def __str__(self):
diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py
--- a/pypy/rpython/lltypesystem/rdict.py
+++ b/pypy/rpython/lltypesystem/rdict.py
@@ -1,16 +1,14 @@
 from pypy.tool.pairtype import pairtype
-from pypy.annotation import model as annmodel
 from pypy.objspace.flow.model import Constant
-from pypy.rpython.rdict import AbstractDictRepr, AbstractDictIteratorRepr,\
-     rtype_newdict
+from pypy.rpython.rdict import (AbstractDictRepr, AbstractDictIteratorRepr,
+     rtype_newdict)
 from pypy.rpython.lltypesystem import lltype
+from pypy.rlib import objectmodel, jit
 from pypy.rlib.rarithmetic import r_uint, intmask, LONG_BIT
-from pypy.rlib.objectmodel import hlinvoke
-from pypy.rpython import robject
-from pypy.rlib import objectmodel, jit
 from pypy.rpython import rmodel
 from pypy.rpython.error import TyperError
 
+
 HIGHEST_BIT = intmask(1 << (LONG_BIT - 1))
 MASK = intmask(HIGHEST_BIT - 1)
 
@@ -417,17 +415,16 @@
     ENTRIES = lltype.typeOf(entries).TO
     return ENTRIES.fasthashfn(entries[i].key)
 
- at jit.dont_look_inside
 def ll_get_value(d, i):
     return d.entries[i].value
 
 def ll_keyhash_custom(d, key):
     DICT = lltype.typeOf(d).TO
-    return hlinvoke(DICT.r_rdict_hashfn, d.fnkeyhash, key)
+    return objectmodel.hlinvoke(DICT.r_rdict_hashfn, d.fnkeyhash, key)
 
 def ll_keyeq_custom(d, key1, key2):
     DICT = lltype.typeOf(d).TO
-    return hlinvoke(DICT.r_rdict_eqfn, d.fnkeyeq, key1, key2)
+    return objectmodel.hlinvoke(DICT.r_rdict_eqfn, d.fnkeyeq, key1, key2)
 
 def ll_dict_len(d):
     return d.num_items
@@ -448,6 +445,8 @@
     i = ll_dict_lookup(d, key, hash)
     return _ll_dict_setitem_lookup_done(d, key, value, hash, i)
 
+# Leaving as dont_look_inside ATM, it has a few branches which could lead to
+# many bridges if we don't consider their possible frequency.
 @jit.dont_look_inside
 def _ll_dict_setitem_lookup_done(d, key, value, hash, i):
     valid = (i & HIGHEST_BIT) == 0
@@ -492,6 +491,8 @@
         raise KeyError
     _ll_dict_del(d, i)
 
+# XXX: Move the size checking and resize into a single call which is opauqe to
+# the JIT to avoid extra branches.
 @jit.dont_look_inside
 def _ll_dict_del(d, i):
     d.entries.mark_deleted(i)
@@ -532,6 +533,7 @@
 # ------- a port of CPython's dictobject.c's lookdict implementation -------
 PERTURB_SHIFT = 5
 
+ at jit.dont_look_inside
 def ll_dict_lookup(d, key, hash):
     entries = d.entries
     ENTRIES = lltype.typeOf(entries).TO
@@ -621,7 +623,6 @@
     d.num_items = 0
     d.resize_counter = DICT_INITSIZE * 2
     return d
-ll_newdict.oopspec = 'newdict()'
 
 def ll_newdict_size(DICT, length_estimate):
     length_estimate = (length_estimate // 2) * 3
@@ -633,7 +634,6 @@
     d.num_items = 0
     d.resize_counter = n * 2
     return d
-ll_newdict_size.oopspec = 'newdict()'
 
 # pypy.rpython.memory.lldict uses a dict based on Struct and Array
 # instead of GcStruct and GcArray, which is done by using different
@@ -866,7 +866,6 @@
         global_popitem_index.nextindex = base + counter
     return i
 
- at jit.dont_look_inside
 def ll_popitem(ELEM, dic):
     i = _ll_getnextitem(dic)
     entry = dic.entries[i]
diff --git a/pypy/rpython/lltypesystem/test/test_ll2ctypes.py b/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
--- a/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
@@ -8,6 +8,7 @@
 from pypy.rpython.lltypesystem.ll2ctypes import uninitialized2ctypes
 from pypy.rpython.lltypesystem.ll2ctypes import ALLOCATED, force_cast
 from pypy.rpython.lltypesystem.ll2ctypes import cast_adr_to_int, get_ctypes_type
+from pypy.rpython.lltypesystem.ll2ctypes import _llgcopaque
 from pypy.rpython.annlowlevel import llhelper
 from pypy.rlib import rposix
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
@@ -1349,6 +1350,11 @@
         round = ctypes2lltype(llmemory.GCREF, lltype2ctypes(opaque.hide()))
         assert Opaque.show(round) is opaque
 
+    def test_array_of_structs(self):
+        A = lltype.GcArray(lltype.Struct('x', ('v', lltype.Signed)))
+        a = lltype.malloc(A, 5)
+        a2 = ctypes2lltype(lltype.Ptr(A), lltype2ctypes(a))
+        assert a2._obj.getitem(0)._obj._parentstructure() is a2._obj
 
 class TestPlatform(object):
     def test_lib_on_libpaths(self):
@@ -1390,3 +1396,7 @@
         f = rffi.llexternal('f', [rffi.INT, rffi.INT], rffi.INT,
                             compilation_info=eci)
         assert f(3, 4) == 7
+
+    def test_llgcopaque_eq(self):
+        assert _llgcopaque(1) != None
+        assert _llgcopaque(0) == None

From noreply at buildbot.pypy.org  Fri Oct 21 23:27:31 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 21 Oct 2011 23:27:31 +0200 (CEST)
Subject: [pypy-commit] pypy default: Test and trivial fix.
Message-ID: <20111021212731.814C6820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48326:a1b22fb6a092
Date: 2011-10-21 23:26 +0200
http://bitbucket.org/pypy/pypy/changeset/a1b22fb6a092/

Log:	Test and trivial fix.

diff --git a/pypy/module/__builtin__/app_io.py b/pypy/module/__builtin__/app_io.py
--- a/pypy/module/__builtin__/app_io.py
+++ b/pypy/module/__builtin__/app_io.py
@@ -71,7 +71,7 @@
         return line[:-1]
     return line
 
-def input(prompt=None):
+def input(prompt=''):
     """Equivalent to eval(raw_input(prompt))."""
     return eval(raw_input(prompt))
 
diff --git a/pypy/module/__builtin__/test/test_rawinput.py b/pypy/module/__builtin__/test/test_rawinput.py
--- a/pypy/module/__builtin__/test/test_rawinput.py
+++ b/pypy/module/__builtin__/test/test_rawinput.py
@@ -3,29 +3,32 @@
 
 class AppTestRawInput():
 
-    def test_raw_input(self):
+    def test_input_and_raw_input(self):
         import sys, StringIO
         for prompt, expected in [("def:", "abc/ def:/ghi\n"),
                                  ("", "abc/ /ghi\n"),
                                  (42, "abc/ 42/ghi\n"),
                                  (None, "abc/ None/ghi\n"),
                                  (Ellipsis, "abc/ /ghi\n")]:
-            save = sys.stdin, sys.stdout
-            try:
-                sys.stdin = StringIO.StringIO("foo\nbar\n")
-                out = sys.stdout = StringIO.StringIO()
-                print "abc",    # softspace = 1
-                out.write('/')
-                if prompt is Ellipsis:
-                    got = raw_input()
-                else:
-                    got = raw_input(prompt)
-                out.write('/')
-                print "ghi"
-            finally:
-                sys.stdin, sys.stdout = save
-            assert out.getvalue() == expected
-            assert got == "foo"
+            for inputfn, inputtext, gottext in [
+                    (raw_input, "foo\nbar\n", "foo"),
+                    (input, "40+2\n", 42)]:
+                save = sys.stdin, sys.stdout
+                try:
+                    sys.stdin = StringIO.StringIO(inputtext)
+                    out = sys.stdout = StringIO.StringIO()
+                    print "abc",    # softspace = 1
+                    out.write('/')
+                    if prompt is Ellipsis:
+                        got = inputfn()
+                    else:
+                        got = inputfn(prompt)
+                    out.write('/')
+                    print "ghi"
+                finally:
+                    sys.stdin, sys.stdout = save
+                assert out.getvalue() == expected
+                assert got == gottext
 
     def test_softspace(self):
         import sys

From noreply at buildbot.pypy.org  Fri Oct 21 23:27:33 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 21 Oct 2011 23:27:33 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20111021212733.12069820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48327:bef7ae83af19
Date: 2011-10-21 23:27 +0200
http://bitbucket.org/pypy/pypy/changeset/bef7ae83af19/

Log:	merge heads

diff --git a/pypy/jit/backend/llgraph/llimpl.py b/pypy/jit/backend/llgraph/llimpl.py
--- a/pypy/jit/backend/llgraph/llimpl.py
+++ b/pypy/jit/backend/llgraph/llimpl.py
@@ -7,9 +7,7 @@
 import weakref
 from pypy.objspace.flow.model import Variable, Constant
 from pypy.annotation import model as annmodel
-from pypy.jit.metainterp.history import (ConstInt, ConstPtr,
-                                         BoxInt, BoxPtr, BoxObj, BoxFloat,
-                                         REF, INT, FLOAT)
+from pypy.jit.metainterp.history import REF, INT, FLOAT
 from pypy.jit.codewriter import heaptracker
 from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr, rffi
 from pypy.rpython.ootypesystem import ootype
@@ -17,7 +15,7 @@
 from pypy.rpython.llinterp import LLException
 from pypy.rpython.extregistry import ExtRegistryEntry
 
-from pypy.jit.metainterp import resoperation, executor
+from pypy.jit.metainterp import resoperation
 from pypy.jit.metainterp.resoperation import rop
 from pypy.jit.backend.llgraph import symbolic
 from pypy.jit.codewriter import longlong
@@ -334,6 +332,13 @@
     assert isinstance(type, str) and len(type) == 1
     op.descr = Descr(ofs, type, arg_types=arg_types)
 
+def compile_add_descr_arg(loop, ofs, type, arg_types):
+    from pypy.jit.backend.llgraph.runner import Descr
+    loop = _from_opaque(loop)
+    op = loop.operations[-1]
+    assert isinstance(type, str) and len(type) == 1
+    op.args.append(Descr(ofs, type, arg_types=arg_types))
+
 def compile_add_loop_token(loop, descr):
     if we_are_translated():
         raise ValueError("CALL_ASSEMBLER not supported")
@@ -438,8 +443,11 @@
         self._may_force = -1
 
     def getenv(self, v):
+        from pypy.jit.backend.llgraph.runner import Descr
         if isinstance(v, Constant):
             return v.value
+        elif isinstance(v, Descr):
+            return v
         else:
             return self.env[v]
 
@@ -807,6 +815,29 @@
         else:
             raise NotImplementedError
 
+    def op_getinteriorfield_gc(self, descr, array, index):
+        if descr.typeinfo == REF:
+            return do_getinteriorfield_gc_ptr(array, index, descr.ofs)
+        elif descr.typeinfo == INT:
+            return do_getinteriorfield_gc_int(array, index, descr.ofs)
+        elif descr.typeinfo == FLOAT:
+            return do_getinteriorfield_gc_float(array, index, descr.ofs)
+        else:
+            raise NotImplementedError
+
+    def op_setinteriorfield_gc(self, descr, array, index, newvalue):
+        if descr.typeinfo == REF:
+            return do_setinteriorfield_gc_ptr(array, index, descr.ofs,
+                                              newvalue)
+        elif descr.typeinfo == INT:
+            return do_setinteriorfield_gc_int(array, index, descr.ofs,
+                                              newvalue)
+        elif descr.typeinfo == FLOAT:
+            return do_setinteriorfield_gc_float(array, index, descr.ofs,
+                                                newvalue)
+        else:
+            raise NotImplementedError
+
     def op_setfield_gc(self, fielddescr, struct, newvalue):
         if fielddescr.typeinfo == REF:
             do_setfield_gc_ptr(struct, fielddescr.ofs, newvalue)
@@ -1354,6 +1385,22 @@
 def do_getfield_gc_ptr(struct, fieldnum):
     return cast_to_ptr(_getfield_gc(struct, fieldnum))
 
+def _getinteriorfield_gc(struct, fieldnum):
+    STRUCT, fieldname = symbolic.TokenToField[fieldnum]
+    return getattr(struct, fieldname)
+
+def do_getinteriorfield_gc_int(array, index, fieldnum):
+    struct = array._obj.container.getitem(index)
+    return cast_to_int(_getinteriorfield_gc(struct, fieldnum))
+
+def do_getinteriorfield_gc_float(array, index, fieldnum):
+    struct = array._obj.container.getitem(index)
+    return cast_to_floatstorage(_getinteriorfield_gc(struct, fieldnum))
+
+def do_getinteriorfield_gc_ptr(array, index, fieldnum):
+    struct = array._obj.container.getitem(index)
+    return cast_to_ptr(_getinteriorfield_gc(struct, fieldnum))
+
 def _getfield_raw(struct, fieldnum):
     STRUCT, fieldname = symbolic.TokenToField[fieldnum]
     ptr = cast_from_int(lltype.Ptr(STRUCT), struct)
@@ -1409,26 +1456,28 @@
     newvalue = cast_from_ptr(ITEMTYPE, newvalue)
     array.setitem(index, newvalue)
 
-def do_setfield_gc_int(struct, fieldnum, newvalue):
-    STRUCT, fieldname = symbolic.TokenToField[fieldnum]
-    ptr = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), struct)
-    FIELDTYPE = getattr(STRUCT, fieldname)
-    newvalue = cast_from_int(FIELDTYPE, newvalue)
-    setattr(ptr, fieldname, newvalue)
+def new_setfield_gc(cast_func):
+    def do_setfield_gc(struct, fieldnum, newvalue):
+        STRUCT, fieldname = symbolic.TokenToField[fieldnum]
+        ptr = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), struct)
+        FIELDTYPE = getattr(STRUCT, fieldname)
+        newvalue = cast_func(FIELDTYPE, newvalue)
+        setattr(ptr, fieldname, newvalue)
+    return do_setfield_gc
+do_setfield_gc_int = new_setfield_gc(cast_from_int)
+do_setfield_gc_float = new_setfield_gc(cast_from_floatstorage)
+do_setfield_gc_ptr = new_setfield_gc(cast_from_ptr)
 
-def do_setfield_gc_float(struct, fieldnum, newvalue):
-    STRUCT, fieldname = symbolic.TokenToField[fieldnum]
-    ptr = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), struct)
-    FIELDTYPE = getattr(STRUCT, fieldname)
-    newvalue = cast_from_floatstorage(FIELDTYPE, newvalue)
-    setattr(ptr, fieldname, newvalue)
-
-def do_setfield_gc_ptr(struct, fieldnum, newvalue):
-    STRUCT, fieldname = symbolic.TokenToField[fieldnum]
-    ptr = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), struct)
-    FIELDTYPE = getattr(STRUCT, fieldname)
-    newvalue = cast_from_ptr(FIELDTYPE, newvalue)
-    setattr(ptr, fieldname, newvalue)
+def new_setinteriorfield_gc(cast_func):
+    def do_setinteriorfield_gc(array, index, fieldnum, newvalue):
+        STRUCT, fieldname = symbolic.TokenToField[fieldnum]
+        struct = array._obj.container.getitem(index)
+        FIELDTYPE = getattr(STRUCT, fieldname)
+        setattr(struct, fieldname, cast_func(FIELDTYPE, newvalue))
+    return do_setinteriorfield_gc
+do_setinteriorfield_gc_int = new_setinteriorfield_gc(cast_from_int)
+do_setinteriorfield_gc_float = new_setinteriorfield_gc(cast_from_floatstorage)
+do_setinteriorfield_gc_ptr = new_setinteriorfield_gc(cast_from_ptr)        
 
 def do_setfield_raw_int(struct, fieldnum, newvalue):
     STRUCT, fieldname = symbolic.TokenToField[fieldnum]
@@ -1694,6 +1743,7 @@
 setannotation(compile_start_float_var, annmodel.SomeInteger())
 setannotation(compile_add, annmodel.s_None)
 setannotation(compile_add_descr, annmodel.s_None)
+setannotation(compile_add_descr_arg, annmodel.s_None)
 setannotation(compile_add_var, annmodel.s_None)
 setannotation(compile_add_int_const, annmodel.s_None)
 setannotation(compile_add_ref_const, annmodel.s_None)
@@ -1741,6 +1791,9 @@
 setannotation(do_getfield_raw_int, annmodel.SomeInteger())
 setannotation(do_getfield_raw_ptr, annmodel.SomePtr(llmemory.GCREF))
 setannotation(do_getfield_raw_float, s_FloatStorage)
+setannotation(do_getinteriorfield_gc_int, annmodel.SomeInteger())
+setannotation(do_getinteriorfield_gc_ptr, annmodel.SomePtr(llmemory.GCREF))
+setannotation(do_getinteriorfield_gc_float, s_FloatStorage)
 setannotation(do_new, annmodel.SomePtr(llmemory.GCREF))
 setannotation(do_new_array, annmodel.SomePtr(llmemory.GCREF))
 setannotation(do_setarrayitem_gc_int, annmodel.s_None)
@@ -1754,6 +1807,9 @@
 setannotation(do_setfield_raw_int, annmodel.s_None)
 setannotation(do_setfield_raw_ptr, annmodel.s_None)
 setannotation(do_setfield_raw_float, annmodel.s_None)
+setannotation(do_setinteriorfield_gc_int, annmodel.s_None)
+setannotation(do_setinteriorfield_gc_ptr, annmodel.s_None)
+setannotation(do_setinteriorfield_gc_float, annmodel.s_None)
 setannotation(do_newstr, annmodel.SomePtr(llmemory.GCREF))
 setannotation(do_strsetitem, annmodel.s_None)
 setannotation(do_newunicode, annmodel.SomePtr(llmemory.GCREF))
diff --git a/pypy/jit/backend/llgraph/runner.py b/pypy/jit/backend/llgraph/runner.py
--- a/pypy/jit/backend/llgraph/runner.py
+++ b/pypy/jit/backend/llgraph/runner.py
@@ -2,7 +2,6 @@
 Minimal-API wrapper around the llinterpreter to run operations.
 """
 
-import sys
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.rpython.lltypesystem import lltype, llmemory, rclass
@@ -11,12 +10,11 @@
 from pypy.jit.metainterp import history
 from pypy.jit.metainterp.history import REF, INT, FLOAT
 from pypy.jit.metainterp.warmstate import unwrap
-from pypy.jit.metainterp.resoperation import ResOperation, rop
+from pypy.jit.metainterp.resoperation import rop
 from pypy.jit.backend import model
 from pypy.jit.backend.llgraph import llimpl, symbolic
 from pypy.jit.metainterp.typesystem import llhelper, oohelper
 from pypy.jit.codewriter import heaptracker, longlong
-from pypy.rlib import rgc
 
 class MiniStats:
     pass
@@ -177,8 +175,10 @@
             llimpl.compile_add(c, op.getopnum())
             descr = op.getdescr()
             if isinstance(descr, Descr):
-                llimpl.compile_add_descr(c, descr.ofs, descr.typeinfo, descr.arg_types)
-            if isinstance(descr, history.LoopToken) and op.getopnum() != rop.JUMP:
+                llimpl.compile_add_descr(c, descr.ofs, descr.typeinfo,
+                                         descr.arg_types)
+            if (isinstance(descr, history.LoopToken) and
+                op.getopnum() != rop.JUMP):
                 llimpl.compile_add_loop_token(c, descr)
             if self.is_oo and isinstance(descr, (OODescr, MethDescr)):
                 # hack hack, not rpython
@@ -193,6 +193,9 @@
                     llimpl.compile_add_ref_const(c, x.value, self.ts.BASETYPE)
                 elif isinstance(x, history.ConstFloat):
                     llimpl.compile_add_float_const(c, x.value)
+                elif isinstance(x, Descr):
+                    llimpl.compile_add_descr_arg(c, x.ofs, x.typeinfo,
+                                                 x.arg_types)
                 else:
                     raise Exception("'%s' args contain: %r" % (op.getopname(),
                                                                x))
@@ -316,6 +319,13 @@
         token = history.getkind(getattr(S, fieldname))
         return self.getdescr(ofs, token[0], name=fieldname)
 
+    def interiorfielddescrof(self, A, fieldname):
+        S = A.OF
+        ofs2 = symbolic.get_size(A)
+        ofs, size = symbolic.get_field_token(S, fieldname)
+        token = history.getkind(getattr(S, fieldname))
+        return self.getdescr(ofs, token[0], name=fieldname, extrainfo=ofs2)
+
     def calldescrof(self, FUNC, ARGS, RESULT, extrainfo):
         arg_types = []
         for ARG in ARGS:
@@ -353,8 +363,11 @@
     def arraydescrof(self, A):
         assert A.OF != lltype.Void
         size = symbolic.get_size(A)
-        token = history.getkind(A.OF)
-        return self.getdescr(size, token[0])
+        if isinstance(A.OF, lltype.Ptr) or isinstance(A.OF, lltype.Primitive):
+            token = history.getkind(A.OF)[0]
+        else:
+            token = '?'
+        return self.getdescr(size, token)
 
     # ---------- the backend-dependent operations ----------
 
@@ -406,6 +419,29 @@
         assert isinstance(fielddescr, Descr)
         return llimpl.do_getfield_raw_float(struct, fielddescr.ofs)
 
+    def bh_getinteriorfield_gc_i(self, array, index, descr):
+        assert isinstance(descr, Descr)
+        return llimpl.do_getinteriorfield_gc_int(array, index, descr.ofs)
+    def bh_getinteriorfield_gc_r(self, array, index, descr):
+        assert isinstance(descr, Descr)
+        return llimpl.do_getinteriorfield_gc_ptr(array, index, descr.ofs)
+    def bh_getinteriorfield_gc_f(self, array, index, descr):
+        assert isinstance(descr, Descr)
+        return llimpl.do_getinteriorfield_gc_float(array, index, descr.ofs)
+
+    def bh_setinteriorfield_gc_i(self, array, index, descr, value):
+        assert isinstance(descr, Descr)
+        return llimpl.do_setinteriorfield_gc_int(array, index, descr.ofs,
+                                                 value)
+    def bh_setinteriorfield_gc_r(self, array, index, descr, value):
+        assert isinstance(descr, Descr)
+        return llimpl.do_setinteriorfield_gc_ptr(array, index, descr.ofs,
+                                                 value)
+    def bh_setinteriorfield_gc_f(self, array, index, descr, value):
+        assert isinstance(descr, Descr)
+        return llimpl.do_setinteriorfield_gc_float(array, index, descr.ofs,
+                                                   value)
+
     def bh_new(self, sizedescr):
         assert isinstance(sizedescr, Descr)
         return llimpl.do_new(sizedescr.ofs)
@@ -418,7 +454,6 @@
 
     def bh_classof(self, struct):
         struct = lltype.cast_opaque_ptr(rclass.OBJECTPTR, struct)
-        result = struct.typeptr
         result_adr = llmemory.cast_ptr_to_adr(struct.typeptr)
         return heaptracker.adr2int(result_adr)
 
diff --git a/pypy/jit/backend/llsupport/descr.py b/pypy/jit/backend/llsupport/descr.py
--- a/pypy/jit/backend/llsupport/descr.py
+++ b/pypy/jit/backend/llsupport/descr.py
@@ -1,13 +1,10 @@
 import py
-from pypy.rpython.lltypesystem import lltype, rffi, llmemory, rclass
+from pypy.rpython.lltypesystem import lltype, rffi, llmemory
 from pypy.rpython.lltypesystem.lloperation import llop
 from pypy.jit.backend.llsupport import symbolic, support
-from pypy.jit.metainterp.history import AbstractDescr, getkind, BoxInt, BoxPtr
-from pypy.jit.metainterp.history import BasicFailDescr, LoopToken, BoxFloat
+from pypy.jit.metainterp.history import AbstractDescr, getkind
 from pypy.jit.metainterp import history
-from pypy.jit.metainterp.resoperation import ResOperation, rop
 from pypy.jit.codewriter import heaptracker, longlong
-from pypy.rlib.rarithmetic import r_longlong, r_ulonglong
 
 # The point of the class organization in this file is to make instances
 # as compact as possible.  This is done by not storing the field size or
@@ -23,6 +20,7 @@
         self._cache_field = {}
         self._cache_array = {}
         self._cache_call = {}
+        self._cache_interiorfield = {}
 
     def init_size_descr(self, STRUCT, sizedescr):
         assert isinstance(STRUCT, lltype.GcStruct)
@@ -142,7 +140,6 @@
         cachedict[fieldname] = fielddescr
         return fielddescr
 
-
 # ____________________________________________________________
 # ArrayDescrs
 
@@ -252,6 +249,36 @@
         cache[ARRAY] = arraydescr
         return arraydescr
 
+# ____________________________________________________________
+# InteriorFieldDescr
+
+class InteriorFieldDescr(AbstractDescr):
+    arraydescr = BaseArrayDescr()     # workaround for the annotator
+    fielddescr = BaseFieldDescr('', 0)
+
+    def __init__(self, arraydescr, fielddescr):
+        self.arraydescr = arraydescr
+        self.fielddescr = fielddescr
+
+    def is_pointer_field(self):
+        return self.fielddescr.is_pointer_field()
+
+    def is_float_field(self):
+        return self.fielddescr.is_float_field()
+
+    def repr_of_descr(self):
+        return '<InteriorFieldDescr %s>' % self.fielddescr.repr_of_descr()
+
+def get_interiorfield_descr(gc_ll_descr, ARRAY, FIELDTP, name):
+    cache = gc_ll_descr._cache_interiorfield
+    try:
+        return cache[(ARRAY, FIELDTP, name)]
+    except KeyError:
+        arraydescr = get_array_descr(gc_ll_descr, ARRAY)
+        fielddescr = get_field_descr(gc_ll_descr, FIELDTP, name)
+        descr = InteriorFieldDescr(arraydescr, fielddescr)
+        cache[(ARRAY, FIELDTP, name)] = descr
+        return descr
 
 # ____________________________________________________________
 # CallDescrs
@@ -525,7 +552,8 @@
         #
         if TYPE is lltype.Float or is_longlong(TYPE):
             setattr(Descr, floatattrname, True)
-        elif TYPE is not lltype.Bool and rffi.cast(TYPE, -1) == -1:
+        elif (TYPE is not lltype.Bool and isinstance(TYPE, lltype.Number) and
+              rffi.cast(TYPE, -1) == -1):
             setattr(Descr, signedattrname, True)
         #
         _cache[nameprefix, TYPE] = Descr
diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py
--- a/pypy/jit/backend/llsupport/gc.py
+++ b/pypy/jit/backend/llsupport/gc.py
@@ -45,6 +45,14 @@
     def freeing_block(self, start, stop):
         pass
 
+    def get_funcptr_for_newarray(self):
+        return llhelper(self.GC_MALLOC_ARRAY, self.malloc_array)
+    def get_funcptr_for_newstr(self):
+        return llhelper(self.GC_MALLOC_STR_UNICODE, self.malloc_str)
+    def get_funcptr_for_newunicode(self):
+        return llhelper(self.GC_MALLOC_STR_UNICODE, self.malloc_unicode)
+
+
     def record_constptrs(self, op, gcrefs_output_list):
         for i in range(op.numargs()):
             v = op.getarg(i)
@@ -96,6 +104,39 @@
         malloc_fn_ptr = self.configure_boehm_once()
         self.funcptr_for_new = malloc_fn_ptr
 
+        def malloc_array(basesize, itemsize, ofs_length, num_elem):
+            try:
+                size = ovfcheck(basesize + ovfcheck(itemsize * num_elem))
+            except OverflowError:
+                return lltype.nullptr(llmemory.GCREF.TO)
+            res = self.funcptr_for_new(size)
+            if not res:
+                return res
+            rffi.cast(rffi.CArrayPtr(lltype.Signed), res)[ofs_length/WORD] = num_elem
+            return res
+        self.malloc_array = malloc_array
+        self.GC_MALLOC_ARRAY = lltype.Ptr(lltype.FuncType(
+            [lltype.Signed] * 4, llmemory.GCREF))
+
+
+        (str_basesize, str_itemsize, str_ofs_length
+         ) = symbolic.get_array_token(rstr.STR, self.translate_support_code)
+        (unicode_basesize, unicode_itemsize, unicode_ofs_length
+         ) = symbolic.get_array_token(rstr.UNICODE, self.translate_support_code)
+        def malloc_str(length):
+            return self.malloc_array(
+                str_basesize, str_itemsize, str_ofs_length, length
+            )
+        def malloc_unicode(length):
+            return self.malloc_array(
+                unicode_basesize, unicode_itemsize, unicode_ofs_length, length
+            )
+        self.malloc_str = malloc_str
+        self.malloc_unicode = malloc_unicode
+        self.GC_MALLOC_STR_UNICODE = lltype.Ptr(lltype.FuncType(
+            [lltype.Signed], llmemory.GCREF))
+
+
         # on some platform GC_init is required before any other
         # GC_* functions, call it here for the benefit of tests
         # XXX move this to tests
@@ -116,39 +157,27 @@
         ofs_length = arraydescr.get_ofs_length(self.translate_support_code)
         basesize = arraydescr.get_base_size(self.translate_support_code)
         itemsize = arraydescr.get_item_size(self.translate_support_code)
-        size = basesize + itemsize * num_elem
-        res = self.funcptr_for_new(size)
-        rffi.cast(rffi.CArrayPtr(lltype.Signed), res)[ofs_length/WORD] = num_elem
-        return res
+        return self.malloc_array(basesize, itemsize, ofs_length, num_elem)
 
     def gc_malloc_str(self, num_elem):
-        basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR,
-                                                   self.translate_support_code)
-        assert itemsize == 1
-        size = basesize + num_elem
-        res = self.funcptr_for_new(size)
-        rffi.cast(rffi.CArrayPtr(lltype.Signed), res)[ofs_length/WORD] = num_elem
-        return res
+        return self.malloc_str(num_elem)
 
     def gc_malloc_unicode(self, num_elem):
-        basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE,
-                                                   self.translate_support_code)
-        size = basesize + num_elem * itemsize
-        res = self.funcptr_for_new(size)
-        rffi.cast(rffi.CArrayPtr(lltype.Signed), res)[ofs_length/WORD] = num_elem
-        return res
+        return self.malloc_unicode(num_elem)
 
     def args_for_new(self, sizedescr):
         assert isinstance(sizedescr, BaseSizeDescr)
         return [sizedescr.size]
 
+    def args_for_new_array(self, arraydescr):
+        ofs_length = arraydescr.get_ofs_length(self.translate_support_code)
+        basesize = arraydescr.get_base_size(self.translate_support_code)
+        itemsize = arraydescr.get_item_size(self.translate_support_code)
+        return [basesize, itemsize, ofs_length]
+
     def get_funcptr_for_new(self):
         return self.funcptr_for_new
 
-    get_funcptr_for_newarray = None
-    get_funcptr_for_newstr = None
-    get_funcptr_for_newunicode = None
-
     def rewrite_assembler(self, cpu, operations, gcrefs_output_list):
         # record all GCREFs too, because Boehm cannot see them and keep them
         # alive if they end up as constants in the assembler
@@ -752,15 +781,6 @@
     def get_funcptr_for_new(self):
         return llhelper(self.GC_MALLOC_BASIC, self.malloc_basic)
 
-    def get_funcptr_for_newarray(self):
-        return llhelper(self.GC_MALLOC_ARRAY, self.malloc_array)
-
-    def get_funcptr_for_newstr(self):
-        return llhelper(self.GC_MALLOC_STR_UNICODE, self.malloc_str)
-
-    def get_funcptr_for_newunicode(self):
-        return llhelper(self.GC_MALLOC_STR_UNICODE, self.malloc_unicode)
-
     def do_write_barrier(self, gcref_struct, gcref_newptr):
         hdr_addr = llmemory.cast_ptr_to_adr(gcref_struct)
         hdr_addr -= self.gcheaderbuilder.size_gc_header
diff --git a/pypy/jit/backend/llsupport/llmodel.py b/pypy/jit/backend/llsupport/llmodel.py
--- a/pypy/jit/backend/llsupport/llmodel.py
+++ b/pypy/jit/backend/llsupport/llmodel.py
@@ -1,24 +1,18 @@
-import sys
 from pypy.rpython.lltypesystem import lltype, llmemory, rffi, rclass, rstr
 from pypy.rpython.lltypesystem.lloperation import llop
-from pypy.rpython.llinterp import LLInterpreter, LLException
+from pypy.rpython.llinterp import LLInterpreter
 from pypy.rpython.annlowlevel import llhelper
 from pypy.rlib.objectmodel import we_are_translated, specialize
-from pypy.jit.metainterp.history import BoxInt, BoxPtr, set_future_values,\
-     BoxFloat
 from pypy.jit.metainterp import history
 from pypy.jit.codewriter import heaptracker, longlong
 from pypy.jit.backend.model import AbstractCPU
 from pypy.jit.backend.llsupport import symbolic
 from pypy.jit.backend.llsupport.symbolic import WORD, unroll_basic_sizes
-from pypy.jit.backend.llsupport.descr import get_size_descr,  BaseSizeDescr
-from pypy.jit.backend.llsupport.descr import get_field_descr, BaseFieldDescr
-from pypy.jit.backend.llsupport.descr import get_array_descr, BaseArrayDescr
-from pypy.jit.backend.llsupport.descr import get_call_descr
-from pypy.jit.backend.llsupport.descr import BaseIntCallDescr, GcPtrCallDescr
-from pypy.jit.backend.llsupport.descr import FloatCallDescr, VoidCallDescr
+from pypy.jit.backend.llsupport.descr import (get_size_descr,
+     get_field_descr, BaseFieldDescr, get_array_descr, BaseArrayDescr,
+     get_call_descr, BaseIntCallDescr, GcPtrCallDescr, FloatCallDescr,
+     VoidCallDescr, InteriorFieldDescr, get_interiorfield_descr)
 from pypy.jit.backend.llsupport.asmmemmgr import AsmMemoryManager
-from pypy.rpython.annlowlevel import cast_instance_to_base_ptr
 
 
 class AbstractLLCPU(AbstractCPU):
@@ -241,6 +235,9 @@
     def arraydescrof(self, A):
         return get_array_descr(self.gc_ll_descr, A)
 
+    def interiorfielddescrof(self, A, fieldname):
+        return get_interiorfield_descr(self.gc_ll_descr, A, A.OF, fieldname)
+
     def unpack_arraydescr(self, arraydescr):
         assert isinstance(arraydescr, BaseArrayDescr)
         return arraydescr.get_base_size(self.translate_support_code)
@@ -358,6 +355,100 @@
     bh_getarrayitem_raw_i = bh_getarrayitem_gc_i
     bh_getarrayitem_raw_f = bh_getarrayitem_gc_f
 
+    def bh_getinteriorfield_gc_i(self, gcref, itemindex, descr):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        fieldsize = descr.fielddescr.get_field_size(self.translate_support_code)
+        sign = descr.fielddescr.is_field_signed()
+        fullofs = itemindex * size + ofs
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), fullofs)
+        for STYPE, UTYPE, itemsize in unroll_basic_sizes:
+            if fieldsize == itemsize:
+                if sign:
+                    item = rffi.cast(rffi.CArrayPtr(STYPE), items)
+                    val = item[0]
+                    val = rffi.cast(lltype.Signed, val)
+                else:
+                    item = rffi.cast(rffi.CArrayPtr(UTYPE), items)
+                    val = item[0]
+                    val = rffi.cast(lltype.Signed, val)
+                # --- end of GC unsafe code ---
+                return val
+        else:
+            raise NotImplementedError("size = %d" % fieldsize)
+
+    def bh_getinteriorfield_gc_r(self, gcref, itemindex, descr):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs +
+                            size * itemindex)
+        items = rffi.cast(rffi.CArrayPtr(lltype.Signed), items)
+        pval = self._cast_int_to_gcref(items[0])
+        # --- end of GC unsafe code ---
+        return pval
+
+    def bh_getinteriorfield_gc_f(self, gcref, itemindex, descr):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs +
+                            size * itemindex)
+        items = rffi.cast(rffi.CArrayPtr(longlong.FLOATSTORAGE), items)
+        fval = items[0]
+        # --- end of GC unsafe code ---
+        return fval
+
+    def bh_setinteriorfield_gc_i(self, gcref, itemindex, descr, value):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        fieldsize = descr.fielddescr.get_field_size(self.translate_support_code)
+        ofs = itemindex * size + ofs
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs)
+        for TYPE, _, itemsize in unroll_basic_sizes:
+            if fieldsize == itemsize:
+                items = rffi.cast(rffi.CArrayPtr(TYPE), items)
+                items[0] = rffi.cast(TYPE, value)
+                # --- end of GC unsafe code ---
+                return
+        else:
+            raise NotImplementedError("size = %d" % fieldsize)
+
+    def bh_setinteriorfield_gc_r(self, gcref, itemindex, descr, newvalue):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        self.gc_ll_descr.do_write_barrier(gcref, newvalue)
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref),
+                            ofs + size * itemindex)
+        items = rffi.cast(rffi.CArrayPtr(lltype.Signed), items)
+        items[0] = self.cast_gcref_to_int(newvalue)
+        # --- end of GC unsafe code ---
+
+    def bh_setinteriorfield_gc_f(self, gcref, itemindex, descr, newvalue):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref),
+                            ofs + size * itemindex)
+        items = rffi.cast(rffi.CArrayPtr(longlong.FLOATSTORAGE), items)
+        items[0] = newvalue
+        # --- end of GC unsafe code ---
+
     def bh_strlen(self, string):
         s = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), string)
         return len(s.chars)
@@ -475,7 +566,6 @@
 
     def bh_classof(self, struct):
         struct = lltype.cast_opaque_ptr(rclass.OBJECTPTR, struct)
-        result = struct.typeptr
         result_adr = llmemory.cast_ptr_to_adr(struct.typeptr)
         return heaptracker.adr2int(result_adr)
 
diff --git a/pypy/jit/backend/llsupport/regalloc.py b/pypy/jit/backend/llsupport/regalloc.py
--- a/pypy/jit/backend/llsupport/regalloc.py
+++ b/pypy/jit/backend/llsupport/regalloc.py
@@ -287,7 +287,6 @@
         self.reg_bindings[to_v] = reg
 
     def _move_variable_away(self, v, prev_loc):
-        reg = None
         if self.free_regs:
             loc = self.free_regs.pop()
             self.reg_bindings[v] = loc
diff --git a/pypy/jit/backend/llsupport/test/test_asmmemmgr.py b/pypy/jit/backend/llsupport/test/test_asmmemmgr.py
--- a/pypy/jit/backend/llsupport/test/test_asmmemmgr.py
+++ b/pypy/jit/backend/llsupport/test/test_asmmemmgr.py
@@ -211,14 +211,14 @@
     debug._log = debug.DebugLog()
     try:
         mc._dump(addr, 'test-logname-section')
-        log = list(debug._log)
+        log = list(debug._log) 
     finally:
         debug._log = None
     encoded = ''.join(writtencode).encode('hex').upper()
     ataddr = '@%x' % addr
     assert log == [('test-logname-section',
                     [('debug_print', 'CODE_DUMP', ataddr, '+0 ', encoded)])]
-    #
+    
     lltype.free(p, flavor='raw')
 
 def test_blockbuildermixin2():
diff --git a/pypy/jit/backend/llsupport/test/test_descr.py b/pypy/jit/backend/llsupport/test/test_descr.py
--- a/pypy/jit/backend/llsupport/test/test_descr.py
+++ b/pypy/jit/backend/llsupport/test/test_descr.py
@@ -3,7 +3,6 @@
 from pypy.jit.backend.llsupport import symbolic
 from pypy.rlib.objectmodel import Symbolic
 from pypy.rpython.annlowlevel import llhelper
-from pypy.jit.metainterp.history import BoxInt, BoxFloat, BoxPtr
 from pypy.jit.metainterp import history
 from pypy.jit.codewriter import longlong
 import sys, struct, py
@@ -149,7 +148,9 @@
     A2 = lltype.GcArray(lltype.Ptr(T))
     A3 = lltype.GcArray(lltype.Ptr(U))
     A4 = lltype.GcArray(lltype.Float)
-    A5 = lltype.GcArray(lltype.SingleFloat)
+    A5 = lltype.GcArray(lltype.Struct('x', ('v', lltype.Signed),
+                                      ('k', lltype.Signed)))
+    A6 = lltype.GcArray(lltype.SingleFloat)
     assert getArrayDescrClass(A2) is GcPtrArrayDescr
     assert getArrayDescrClass(A3) is NonGcPtrArrayDescr
     cls = getArrayDescrClass(A1)
@@ -158,7 +159,7 @@
     clsf = getArrayDescrClass(A4)
     assert clsf != cls
     assert clsf == getArrayDescrClass(lltype.GcArray(lltype.Float))
-    clss = getArrayDescrClass(A5)
+    clss = getArrayDescrClass(A6)
     assert clss not in (clsf, cls)
     assert clss == getArrayDescrClass(lltype.GcArray(rffi.UINT))
     #
@@ -168,11 +169,12 @@
     descr3 = get_array_descr(c0, A3)
     descr4 = get_array_descr(c0, A4)
     descr5 = get_array_descr(c0, A5)
+    descr6 = get_array_descr(c0, A6)
     assert descr1.__class__ is cls
     assert descr2.__class__ is GcPtrArrayDescr
     assert descr3.__class__ is NonGcPtrArrayDescr
     assert descr4.__class__ is clsf
-    assert descr5.__class__ is clss
+    assert descr6.__class__ is clss
     assert descr1 == get_array_descr(c0, lltype.GcArray(lltype.Char))
     assert not descr1.is_array_of_pointers()
     assert     descr2.is_array_of_pointers()
@@ -202,7 +204,8 @@
     assert descr2.get_item_size(False) == rffi.sizeof(lltype.Ptr(T))
     assert descr3.get_item_size(False) == rffi.sizeof(lltype.Ptr(U))
     assert descr4.get_item_size(False) == rffi.sizeof(lltype.Float)
-    assert descr5.get_item_size(False) == rffi.sizeof(lltype.SingleFloat)
+    assert descr5.get_item_size(False) == rffi.sizeof(lltype.Signed) * 2
+    assert descr6.get_item_size(False) == rffi.sizeof(lltype.SingleFloat)
     #
     assert isinstance(descr1.get_base_size(True), Symbolic)
     assert isinstance(descr2.get_base_size(True), Symbolic)
@@ -348,7 +351,6 @@
                             (rffi.SHORT,      True), (rffi.USHORT, False),
                             (rffi.INT,        True), (rffi.UINT,   False),
                             (rffi.LONG,       True), (rffi.ULONG,  False)]:
-        A = lltype.GcArray(RESTYPE)
         for tsc in [False, True]:
             c2 = GcCache(tsc)
             descr1 = get_call_descr(c2, [], RESTYPE)
@@ -379,7 +381,6 @@
     descr3i = get_array_descr(c0, lltype.GcArray(lltype.Char))
     assert descr3i.repr_of_descr() == '<CharArrayDescr>'
     #
-    cache = {}
     descr4 = get_call_descr(c0, [lltype.Char, lltype.Ptr(S)], lltype.Ptr(S))
     assert 'GcPtrCallDescr' in descr4.repr_of_descr()
     #
@@ -412,10 +413,10 @@
     ARGS = [lltype.Float, lltype.Ptr(ARRAY)]
     RES = lltype.Float
 
-    def f(a, b):
+    def f2(a, b):
         return float(b[0]) + a
 
-    fnptr = llhelper(lltype.Ptr(lltype.FuncType(ARGS, RES)), f)
+    fnptr = llhelper(lltype.Ptr(lltype.FuncType(ARGS, RES)), f2)
     descr2 = get_call_descr(c0, ARGS, RES)
     a = lltype.malloc(ARRAY, 3)
     opaquea = lltype.cast_opaque_ptr(llmemory.GCREF, a)
diff --git a/pypy/jit/backend/model.py b/pypy/jit/backend/model.py
--- a/pypy/jit/backend/model.py
+++ b/pypy/jit/backend/model.py
@@ -1,5 +1,5 @@
 from pypy.rlib.debug import debug_start, debug_print, debug_stop
-from pypy.jit.metainterp import history, compile
+from pypy.jit.metainterp import history
 
 
 class AbstractCPU(object):
@@ -213,6 +213,10 @@
     def typedescrof(TYPE):
         raise NotImplementedError
 
+    @staticmethod
+    def interiorfielddescrof(A, fieldname):
+        raise NotImplementedError
+
     # ---------- the backend-dependent operations ----------
 
     # lltype specific operations
diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -5,7 +5,7 @@
                                          BoxInt, Box, BoxPtr,
                                          LoopToken,
                                          ConstInt, ConstPtr,
-                                         BoxObj, Const,
+                                         BoxObj,
                                          ConstObj, BoxFloat, ConstFloat)
 from pypy.jit.metainterp.resoperation import ResOperation, rop
 from pypy.jit.metainterp.typesystem import deref
@@ -111,7 +111,7 @@
         self.cpu.set_future_value_int(0, 2)
         fail = self.cpu.execute_token(looptoken)
         res = self.cpu.get_latest_value_int(0)
-        assert res == 3        
+        assert res == 3
         assert fail.identifier == 1
 
     def test_compile_loop(self):
@@ -127,7 +127,7 @@
             ]
         inputargs = [i0]
         operations[2].setfailargs([i1])
-        
+
         self.cpu.compile_loop(inputargs, operations, looptoken)
         self.cpu.set_future_value_int(0, 2)
         fail = self.cpu.execute_token(looptoken)
@@ -148,7 +148,7 @@
             ]
         inputargs = [i0]
         operations[2].setfailargs([None, None, i1, None])
-        
+
         self.cpu.compile_loop(inputargs, operations, looptoken)
         self.cpu.set_future_value_int(0, 2)
         fail = self.cpu.execute_token(looptoken)
@@ -372,7 +372,7 @@
         for opnum, boxargs, retvalue in get_int_tests():
             res = self.execute_operation(opnum, boxargs, 'int')
             assert res.value == retvalue
-        
+
     def test_float_operations(self):
         from pypy.jit.metainterp.test.test_executor import get_float_tests
         for opnum, boxargs, rettype, retvalue in get_float_tests(self.cpu):
@@ -438,7 +438,7 @@
 
     def test_ovf_operations_reversed(self):
         self.test_ovf_operations(reversed=True)
-        
+
     def test_bh_call(self):
         cpu = self.cpu
         #
@@ -503,7 +503,7 @@
                                          [funcbox, BoxInt(num), BoxInt(num)],
                                          'int', descr=dyn_calldescr)
             assert res.value == 2 * num
-            
+
 
         if cpu.supports_floats:
             def func(f0, f1, f2, f3, f4, f5, f6, i0, i1, f7, f8, f9):
@@ -543,7 +543,7 @@
         funcbox = self.get_funcbox(self.cpu, func_ptr)
         res = self.execute_operation(rop.CALL, [funcbox] + map(BoxInt, args), 'int', descr=calldescr)
         assert res.value == func(*args)
-        
+
     def test_call_stack_alignment(self):
         # test stack alignment issues, notably for Mac OS/X.
         # also test the ordering of the arguments.
@@ -615,7 +615,7 @@
         res = self.execute_operation(rop.GETFIELD_GC, [t_box],
                                      'int', descr=shortdescr)
         assert res.value == 1331
-        
+
         #
         u_box, U_box = self.alloc_instance(self.U)
         fielddescr2 = self.cpu.fielddescrof(self.S, 'next')
@@ -695,7 +695,7 @@
 
     def test_failing_guard_class(self):
         t_box, T_box = self.alloc_instance(self.T)
-        u_box, U_box = self.alloc_instance(self.U)        
+        u_box, U_box = self.alloc_instance(self.U)
         null_box = self.null_instance()
         for opname, args in [(rop.GUARD_CLASS, [t_box, U_box]),
                              (rop.GUARD_CLASS, [u_box, T_box]),
@@ -787,7 +787,7 @@
         r = self.execute_operation(rop.GETARRAYITEM_GC, [a_box, BoxInt(3)],
                                    'int', descr=arraydescr)
         assert r.value == 160
-        
+
         #
         if isinstance(A, lltype.GcArray):
             A = lltype.Ptr(A)
@@ -880,6 +880,73 @@
                                    'int', descr=arraydescr)
         assert r.value == 7441
 
+    def test_array_of_structs(self):
+        TP = lltype.GcStruct('x')
+        ITEM = lltype.Struct('x',
+                             ('vs', lltype.Signed),
+                             ('vu', lltype.Unsigned),
+                             ('vsc', rffi.SIGNEDCHAR),
+                             ('vuc', rffi.UCHAR),
+                             ('vss', rffi.SHORT),
+                             ('vus', rffi.USHORT),
+                             ('vsi', rffi.INT),
+                             ('vui', rffi.UINT),
+                             ('k', lltype.Float),
+                             ('p', lltype.Ptr(TP)))
+        a_box, A = self.alloc_array_of(ITEM, 15)
+        s_box, S = self.alloc_instance(TP)
+        kdescr = self.cpu.interiorfielddescrof(A, 'k')
+        pdescr = self.cpu.interiorfielddescrof(A, 'p')
+        self.execute_operation(rop.SETINTERIORFIELD_GC, [a_box, BoxInt(3),
+                                                         boxfloat(1.5)],
+                               'void', descr=kdescr)
+        f = self.cpu.bh_getinteriorfield_gc_f(a_box.getref_base(), 3, kdescr)
+        assert longlong.getrealfloat(f) == 1.5
+        self.cpu.bh_setinteriorfield_gc_f(a_box.getref_base(), 3, kdescr, longlong.getfloatstorage(2.5))
+        r = self.execute_operation(rop.GETINTERIORFIELD_GC, [a_box, BoxInt(3)],
+                                   'float', descr=kdescr)
+        assert r.getfloat() == 2.5
+        #
+        NUMBER_FIELDS = [('vs', lltype.Signed),
+                         ('vu', lltype.Unsigned),
+                         ('vsc', rffi.SIGNEDCHAR),
+                         ('vuc', rffi.UCHAR),
+                         ('vss', rffi.SHORT),
+                         ('vus', rffi.USHORT),
+                         ('vsi', rffi.INT),
+                         ('vui', rffi.UINT)]
+        for name, TYPE in NUMBER_FIELDS[::-1]:
+            vdescr = self.cpu.interiorfielddescrof(A, name)
+            self.execute_operation(rop.SETINTERIORFIELD_GC, [a_box, BoxInt(3),
+                                                             BoxInt(-15)],
+                                   'void', descr=vdescr)
+        for name, TYPE in NUMBER_FIELDS:
+            vdescr = self.cpu.interiorfielddescrof(A, name)
+            i = self.cpu.bh_getinteriorfield_gc_i(a_box.getref_base(), 3,
+                                                  vdescr)
+            assert i == rffi.cast(lltype.Signed, rffi.cast(TYPE, -15))
+        for name, TYPE in NUMBER_FIELDS[::-1]:
+            vdescr = self.cpu.interiorfielddescrof(A, name)
+            self.cpu.bh_setinteriorfield_gc_i(a_box.getref_base(), 3,
+                                              vdescr, -25)
+        for name, TYPE in NUMBER_FIELDS:
+            vdescr = self.cpu.interiorfielddescrof(A, name)
+            r = self.execute_operation(rop.GETINTERIORFIELD_GC,
+                                       [a_box, BoxInt(3)],
+                                       'int', descr=vdescr)
+            assert r.getint() == rffi.cast(lltype.Signed, rffi.cast(TYPE, -25))
+        #
+        self.execute_operation(rop.SETINTERIORFIELD_GC, [a_box, BoxInt(4),
+                                                         s_box],
+                               'void', descr=pdescr)
+        r = self.cpu.bh_getinteriorfield_gc_r(a_box.getref_base(), 4, pdescr)
+        assert r == s_box.getref_base()
+        self.cpu.bh_setinteriorfield_gc_r(a_box.getref_base(), 3, pdescr,
+                                          s_box.getref_base())
+        r = self.execute_operation(rop.GETINTERIORFIELD_GC, [a_box, BoxInt(3)],
+                                   'ref', descr=pdescr)
+        assert r.getref_base() == s_box.getref_base()
+
     def test_string_basic(self):
         s_box = self.alloc_string("hello\xfe")
         r = self.execute_operation(rop.STRLEN, [s_box], 'int')
@@ -1402,7 +1469,7 @@
         addr = llmemory.cast_ptr_to_adr(func_ptr)
         return ConstInt(heaptracker.adr2int(addr))
 
-    
+
     MY_VTABLE = rclass.OBJECT_VTABLE    # for tests only
 
     S = lltype.GcForwardReference()
@@ -1439,7 +1506,6 @@
         return BoxPtr(lltype.nullptr(llmemory.GCREF.TO))
 
     def alloc_array_of(self, ITEM, length):
-        cpu = self.cpu
         A = lltype.GcArray(ITEM)
         a = lltype.malloc(A, length)
         a_box = BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, a))
@@ -2318,7 +2384,7 @@
         for opname, arg, res in ops:
             self.execute_operation(opname, [arg], 'void')
             assert self.guard_failed == res
-        
+
         lltype.free(x, flavor='raw')
 
     def test_assembler_call(self):
@@ -2398,7 +2464,7 @@
         FakeJitDriverSD.portal_calldescr = self.cpu.calldescrof(
             lltype.Ptr(lltype.FuncType(ARGS, RES)), ARGS, RES,
             EffectInfo.MOST_GENERAL)
-        
+
         ops = '''
         [f0, f1]
         f2 = float_add(f0, f1)
@@ -2489,7 +2555,7 @@
         FakeJitDriverSD.portal_calldescr = self.cpu.calldescrof(
             lltype.Ptr(lltype.FuncType(ARGS, RES)), ARGS, RES,
             EffectInfo.MOST_GENERAL)
-        
+
         ops = '''
         [f0, f1]
         f2 = float_add(f0, f1)
@@ -2940,4 +3006,4 @@
 
     def alloc_unicode(self, unicode):
         py.test.skip("implement me")
-    
+
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1,7 +1,7 @@
 import sys, os
 from pypy.jit.backend.llsupport import symbolic
 from pypy.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper
-from pypy.jit.metainterp.history import Const, Box, BoxInt, BoxPtr, BoxFloat
+from pypy.jit.metainterp.history import Const, Box, BoxInt, ConstInt
 from pypy.jit.metainterp.history import (AbstractFailDescr, INT, REF, FLOAT,
                                          LoopToken)
 from pypy.rpython.lltypesystem import lltype, rffi, rstr, llmemory
@@ -36,7 +36,6 @@
 from pypy.rlib import rgc
 from pypy.rlib.clibffi import FFI_DEFAULT_ABI
 from pypy.jit.backend.x86.jump import remap_frame_layout
-from pypy.jit.metainterp.history import ConstInt, BoxInt
 from pypy.jit.codewriter.effectinfo import EffectInfo
 from pypy.jit.codewriter import longlong
 
@@ -729,8 +728,8 @@
         # Also, make sure this is consistent with FRAME_FIXED_SIZE.
         self.mc.PUSH_r(ebp.value)
         self.mc.MOV_rr(ebp.value, esp.value)
-        for regloc in self.cpu.CALLEE_SAVE_REGISTERS:
-            self.mc.PUSH_r(regloc.value)
+        for loc in self.cpu.CALLEE_SAVE_REGISTERS:
+            self.mc.PUSH_r(loc.value)
 
         gcrootmap = self.cpu.gc_ll_descr.gcrootmap
         if gcrootmap and gcrootmap.is_shadow_stack:
@@ -994,7 +993,7 @@
         effectinfo = op.getdescr().get_extra_info()
         oopspecindex = effectinfo.oopspecindex
         genop_llong_list[oopspecindex](self, op, arglocs, resloc)
-        
+
     def regalloc_perform_math(self, op, arglocs, resloc):
         effectinfo = op.getdescr().get_extra_info()
         oopspecindex = effectinfo.oopspecindex
@@ -1311,7 +1310,7 @@
     genop_guard_float_eq = _cmpop_guard_float("E", "E", "NE","NE")
     genop_guard_float_gt = _cmpop_guard_float("A", "B", "BE","AE")
     genop_guard_float_ge = _cmpop_guard_float("AE","BE", "B", "A")
-    
+
     def genop_math_sqrt(self, op, arglocs, resloc):
         self.mc.SQRTSD(arglocs[0], resloc)
 
@@ -1597,12 +1596,27 @@
     genop_getarrayitem_gc_pure = genop_getarrayitem_gc
     genop_getarrayitem_raw = genop_getarrayitem_gc
 
+    def genop_getinteriorfield_gc(self, op, arglocs, resloc):
+        base_loc, ofs_loc, itemsize_loc, fieldsize_loc, index_loc, sign_loc = arglocs
+        # XXX should not use IMUL in most cases
+        self.mc.IMUL(index_loc, itemsize_loc)
+        src_addr = AddressLoc(base_loc, index_loc, 0, ofs_loc.value)
+        self.load_from_mem(resloc, src_addr, fieldsize_loc, sign_loc)
+
+
     def genop_discard_setfield_gc(self, op, arglocs):
         base_loc, ofs_loc, size_loc, value_loc = arglocs
         assert isinstance(size_loc, ImmedLoc)
         dest_addr = AddressLoc(base_loc, ofs_loc)
         self.save_into_mem(dest_addr, value_loc, size_loc)
 
+    def genop_discard_setinteriorfield_gc(self, op, arglocs):
+        base_loc, ofs_loc, itemsize_loc, fieldsize_loc, index_loc, value_loc = arglocs
+        # XXX should not use IMUL in most cases
+        self.mc.IMUL(index_loc, itemsize_loc)
+        dest_addr = AddressLoc(base_loc, index_loc, 0, ofs_loc.value)
+        self.save_into_mem(dest_addr, value_loc, fieldsize_loc)
+
     def genop_discard_setarrayitem_gc(self, op, arglocs):
         base_loc, ofs_loc, value_loc, size_loc, baseofs = arglocs
         assert isinstance(baseofs, ImmedLoc)
diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py
--- a/pypy/jit/backend/x86/regalloc.py
+++ b/pypy/jit/backend/x86/regalloc.py
@@ -7,7 +7,7 @@
                                          ResOperation, BoxPtr, ConstFloat,
                                          BoxFloat, LoopToken, INT, REF, FLOAT)
 from pypy.jit.backend.x86.regloc import *
-from pypy.rpython.lltypesystem import lltype, ll2ctypes, rffi, rstr
+from pypy.rpython.lltypesystem import lltype, rffi, rstr
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.rlib import rgc
 from pypy.jit.backend.llsupport import symbolic
@@ -17,11 +17,12 @@
 from pypy.jit.metainterp.resoperation import rop
 from pypy.jit.backend.llsupport.descr import BaseFieldDescr, BaseArrayDescr
 from pypy.jit.backend.llsupport.descr import BaseCallDescr, BaseSizeDescr
+from pypy.jit.backend.llsupport.descr import InteriorFieldDescr
 from pypy.jit.backend.llsupport.regalloc import FrameManager, RegisterManager,\
      TempBox
 from pypy.jit.backend.x86.arch import WORD, FRAME_FIXED_SIZE
 from pypy.jit.backend.x86.arch import IS_X86_32, IS_X86_64, MY_COPY_OF_REGS
-from pypy.rlib.rarithmetic import r_longlong, r_uint
+from pypy.rlib.rarithmetic import r_longlong
 
 class X86RegisterManager(RegisterManager):
 
@@ -433,7 +434,7 @@
             if self.can_merge_with_next_guard(op, i, operations):
                 oplist_with_guard[op.getopnum()](self, op, operations[i + 1])
                 i += 1
-            elif not we_are_translated() and op.getopnum() == -124: 
+            elif not we_are_translated() and op.getopnum() == -124:
                 self._consider_force_spill(op)
             else:
                 oplist[op.getopnum()](self, op)
@@ -815,7 +816,7 @@
         save_all_regs = guard_not_forced_op is not None
         self.xrm.before_call(force_store, save_all_regs=save_all_regs)
         if not save_all_regs:
-            gcrootmap = gc_ll_descr = self.assembler.cpu.gc_ll_descr.gcrootmap
+            gcrootmap = self.assembler.cpu.gc_ll_descr.gcrootmap
             if gcrootmap and gcrootmap.is_shadow_stack:
                 save_all_regs = 2
         self.rm.before_call(force_store, save_all_regs=save_all_regs)
@@ -972,74 +973,27 @@
             return self._call(op, arglocs)
 
     def consider_newstr(self, op):
-        gc_ll_descr = self.assembler.cpu.gc_ll_descr
-        if gc_ll_descr.get_funcptr_for_newstr is not None:
-            # framework GC
-            loc = self.loc(op.getarg(0))
-            return self._call(op, [loc])
-        # boehm GC (XXX kill the following code at some point)
-        ofs_items, itemsize, ofs = symbolic.get_array_token(rstr.STR, self.translate_support_code)
-        assert itemsize == 1
-        return self._malloc_varsize(ofs_items, ofs, 0, op.getarg(0),
-                                    op.result)
+        loc = self.loc(op.getarg(0))
+        return self._call(op, [loc])
 
     def consider_newunicode(self, op):
-        gc_ll_descr = self.assembler.cpu.gc_ll_descr
-        if gc_ll_descr.get_funcptr_for_newunicode is not None:
-            # framework GC
-            loc = self.loc(op.getarg(0))
-            return self._call(op, [loc])
-        # boehm GC (XXX kill the following code at some point)
-        ofs_items, _, ofs = symbolic.get_array_token(rstr.UNICODE,
-                                                   self.translate_support_code)
-        scale = self._get_unicode_item_scale()
-        return self._malloc_varsize(ofs_items, ofs, scale, op.getarg(0),
-                                    op.result)
-
-    def _malloc_varsize(self, ofs_items, ofs_length, scale, v, res_v):
-        # XXX kill this function at some point
-        if isinstance(v, Box):
-            loc = self.rm.make_sure_var_in_reg(v, [v])
-            tempbox = TempBox()
-            other_loc = self.rm.force_allocate_reg(tempbox, [v])
-            self.assembler.load_effective_addr(loc, ofs_items,scale, other_loc)
-        else:
-            tempbox = None
-            other_loc = imm(ofs_items + (v.getint() << scale))
-        self._call(ResOperation(rop.NEW, [], res_v),
-                   [other_loc], [v])
-        loc = self.rm.make_sure_var_in_reg(v, [res_v])
-        assert self.loc(res_v) == eax
-        # now we have to reload length to some reasonable place
-        self.rm.possibly_free_var(v)
-        if tempbox is not None:
-            self.rm.possibly_free_var(tempbox)
-        self.PerformDiscard(ResOperation(rop.SETFIELD_GC, [None, None], None),
-                            [eax, imm(ofs_length), imm(WORD), loc])
+        loc = self.loc(op.getarg(0))
+        return self._call(op, [loc])
 
     def consider_new_array(self, op):
         gc_ll_descr = self.assembler.cpu.gc_ll_descr
-        if gc_ll_descr.get_funcptr_for_newarray is not None:
-            # framework GC
-            box_num_elem = op.getarg(0)
-            if isinstance(box_num_elem, ConstInt):
-                num_elem = box_num_elem.value
-                if gc_ll_descr.can_inline_malloc_varsize(op.getdescr(),
-                                                         num_elem):
-                    self.fastpath_malloc_varsize(op, op.getdescr(), num_elem)
-                    return
-            args = self.assembler.cpu.gc_ll_descr.args_for_new_array(
-                op.getdescr())
-            arglocs = [imm(x) for x in args]
-            arglocs.append(self.loc(box_num_elem))
-            self._call(op, arglocs)
-            return
-        # boehm GC (XXX kill the following code at some point)
-        itemsize, basesize, ofs_length, _, _ = (
-            self._unpack_arraydescr(op.getdescr()))
-        scale_of_field = _get_scale(itemsize)
-        self._malloc_varsize(basesize, ofs_length, scale_of_field,
-                             op.getarg(0), op.result)
+        box_num_elem = op.getarg(0)
+        if isinstance(box_num_elem, ConstInt):
+            num_elem = box_num_elem.value
+            if gc_ll_descr.can_inline_malloc_varsize(op.getdescr(),
+                                                     num_elem):
+                self.fastpath_malloc_varsize(op, op.getdescr(), num_elem)
+                return
+        args = self.assembler.cpu.gc_ll_descr.args_for_new_array(
+            op.getdescr())
+        arglocs = [imm(x) for x in args]
+        arglocs.append(self.loc(box_num_elem))
+        self._call(op, arglocs)
 
     def _unpack_arraydescr(self, arraydescr):
         assert isinstance(arraydescr, BaseArrayDescr)
@@ -1058,6 +1012,16 @@
         sign = fielddescr.is_field_signed()
         return imm(ofs), imm(size), ptr, sign
 
+    def _unpack_interiorfielddescr(self, descr):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs = arraydescr.get_base_size(self.translate_support_code)
+        itemsize = arraydescr.get_item_size(self.translate_support_code)
+        fieldsize = descr.fielddescr.get_field_size(self.translate_support_code)
+        sign = descr.fielddescr.is_field_signed()
+        ofs += descr.fielddescr.offset
+        return imm(ofs), imm(itemsize), imm(fieldsize), sign
+
     def consider_setfield_gc(self, op):
         ofs_loc, size_loc, _, _ = self._unpack_fielddescr(op.getdescr())
         assert isinstance(size_loc, ImmedLoc)
@@ -1074,6 +1038,21 @@
 
     consider_setfield_raw = consider_setfield_gc
 
+    def consider_setinteriorfield_gc(self, op):
+        t = self._unpack_interiorfielddescr(op.getdescr())
+        ofs, itemsize, fieldsize, _ = t
+        args = op.getarglist()
+        tmpvar = TempBox()
+        base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
+        index_loc = self.rm.force_result_in_reg(tmpvar, op.getarg(1),
+                                                args)
+        # we're free to modify index now
+        value_loc = self.make_sure_var_in_reg(op.getarg(2), args)
+        self.possibly_free_vars(args)
+        self.rm.possibly_free_var(tmpvar)
+        self.PerformDiscard(op, [base_loc, ofs, itemsize, fieldsize,
+                                 index_loc, value_loc])
+
     def consider_strsetitem(self, op):
         args = op.getarglist()
         base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
@@ -1135,6 +1114,24 @@
     consider_getarrayitem_raw = consider_getarrayitem_gc
     consider_getarrayitem_gc_pure = consider_getarrayitem_gc
 
+    def consider_getinteriorfield_gc(self, op):
+        t = self._unpack_interiorfielddescr(op.getdescr())
+        ofs, itemsize, fieldsize, sign = t
+        if sign:
+            sign_loc = imm1
+        else:
+            sign_loc = imm0
+        args = op.getarglist()
+        tmpvar = TempBox()
+        base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
+        index_loc = self.rm.force_result_in_reg(tmpvar, op.getarg(1),
+                                                args)
+        self.rm.possibly_free_vars_for_op(op)
+        self.rm.possibly_free_var(tmpvar)
+        result_loc = self.force_allocate_reg(op.result)
+        self.Perform(op, [base_loc, ofs, itemsize, fieldsize,
+                          index_loc, sign_loc], result_loc)
+
     def consider_int_is_true(self, op, guard_op):
         # doesn't need arg to be in a register
         argloc = self.loc(op.getarg(0))
@@ -1241,7 +1238,6 @@
         self.rm.possibly_free_var(srcaddr_box)
 
     def _gen_address_inside_string(self, baseloc, ofsloc, resloc, is_unicode):
-        cpu = self.assembler.cpu
         if is_unicode:
             ofs_items, _, _ = symbolic.get_array_token(rstr.UNICODE,
                                                   self.translate_support_code)
@@ -1300,7 +1296,7 @@
         tmpreg = X86RegisterManager.all_regs[0]
         tmploc = self.rm.force_allocate_reg(box, selected_reg=tmpreg)
         xmmtmp = X86XMMRegisterManager.all_regs[0]
-        xmmtmploc = self.xrm.force_allocate_reg(box1, selected_reg=xmmtmp)
+        self.xrm.force_allocate_reg(box1, selected_reg=xmmtmp)
         # Part about non-floats
         # XXX we don't need a copy, we only just the original list
         src_locations1 = [self.loc(op.getarg(i)) for i in range(op.numargs())
@@ -1380,7 +1376,7 @@
     return lambda self, op: fn(self, op, None)
 
 def is_comparison_or_ovf_op(opnum):
-    from pypy.jit.metainterp.resoperation import opclasses, AbstractResOp
+    from pypy.jit.metainterp.resoperation import opclasses
     cls = opclasses[opnum]
     # hack hack: in theory they are instance method, but they don't use
     # any instance field, we can use a fake object
diff --git a/pypy/jit/backend/x86/test/test_del.py b/pypy/jit/backend/x86/test/test_del.py
--- a/pypy/jit/backend/x86/test/test_del.py
+++ b/pypy/jit/backend/x86/test/test_del.py
@@ -1,5 +1,4 @@
 
-import py
 from pypy.jit.backend.x86.test.test_basic import Jit386Mixin
 from pypy.jit.metainterp.test.test_del import DelTests
 
diff --git a/pypy/jit/backend/x86/test/test_dict.py b/pypy/jit/backend/x86/test/test_dict.py
new file mode 100644
--- /dev/null
+++ b/pypy/jit/backend/x86/test/test_dict.py
@@ -0,0 +1,9 @@
+
+from pypy.jit.backend.x86.test.test_basic import Jit386Mixin
+from pypy.jit.metainterp.test.test_dict import DictTests
+
+
+class TestDict(Jit386Mixin, DictTests):
+    # for the individual tests see
+    # ====> ../../../metainterp/test/test_dict.py
+    pass
diff --git a/pypy/jit/backend/x86/test/test_runner.py b/pypy/jit/backend/x86/test/test_runner.py
--- a/pypy/jit/backend/x86/test/test_runner.py
+++ b/pypy/jit/backend/x86/test/test_runner.py
@@ -31,7 +31,7 @@
 
     # for the individual tests see
     # ====> ../../test/runner_test.py
-    
+
     def setup_method(self, meth):
         self.cpu = CPU(rtyper=None, stats=FakeStats())
         self.cpu.setup_once()
@@ -69,22 +69,16 @@
 
     def test_allocations(self):
         from pypy.rpython.lltypesystem import rstr
-        
+
         allocs = [None]
         all = []
+        orig_new = self.cpu.gc_ll_descr.funcptr_for_new
         def f(size):
             allocs.insert(0, size)
-            buf = ctypes.create_string_buffer(size)
-            all.append(buf)
-            return ctypes.cast(buf, ctypes.c_void_p).value
-        func = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int)(f)
-        addr = ctypes.cast(func, ctypes.c_void_p).value
-        # ctypes produces an unsigned value. We need it to be signed for, eg,
-        # relative addressing to work properly.
-        addr = rffi.cast(lltype.Signed, addr)
-        
+            return orig_new(size)
+
         self.cpu.assembler.setup_once()
-        self.cpu.assembler.malloc_func_addr = addr
+        self.cpu.gc_ll_descr.funcptr_for_new = f
         ofs = symbolic.get_field_token(rstr.STR, 'chars', False)[0]
 
         res = self.execute_operation(rop.NEWSTR, [ConstInt(7)], 'ref')
@@ -108,7 +102,7 @@
         res = self.execute_operation(rop.NEW_ARRAY, [ConstInt(10)],
                                          'ref', descr)
         assert allocs[0] == 10*WORD + ofs + WORD
-        resbuf = self._resbuf(res)            
+        resbuf = self._resbuf(res)
         assert resbuf[ofs/WORD] == 10
 
         # ------------------------------------------------------------
@@ -116,7 +110,7 @@
         res = self.execute_operation(rop.NEW_ARRAY, [BoxInt(10)],
                                          'ref', descr)
         assert allocs[0] == 10*WORD + ofs + WORD
-        resbuf = self._resbuf(res)                        
+        resbuf = self._resbuf(res)
         assert resbuf[ofs/WORD] == 10
 
     def test_stringitems(self):
@@ -146,7 +140,7 @@
                                                      ConstInt(2), BoxInt(38)],
                                'void', descr)
         assert resbuf[itemsofs/WORD + 2] == 38
-        
+
         self.execute_operation(rop.SETARRAYITEM_GC, [res,
                                                      BoxInt(3), BoxInt(42)],
                                'void', descr)
@@ -167,7 +161,7 @@
                                                          BoxInt(2)],
                                    'int', descr)
         assert r.value == 38
-        
+
         r = self.execute_operation(rop.GETARRAYITEM_GC, [res, BoxInt(3)],
                                    'int', descr)
         assert r.value == 42
@@ -226,7 +220,7 @@
         self.execute_operation(rop.SETFIELD_GC, [res, BoxInt(1234)], 'void', ofs_i)
         i = self.execute_operation(rop.GETFIELD_GC, [res], 'int', ofs_i)
         assert i.value == 1234
-        
+
         #u = self.execute_operation(rop.GETFIELD_GC, [res, ofs_u], 'int')
         #assert u.value == 5
         self.execute_operation(rop.SETFIELD_GC, [res, ConstInt(1)], 'void',
@@ -299,7 +293,7 @@
                     else:
                         assert result != execute(self.cpu, None,
                                                  op, None, b).value
-                    
+
 
     def test_stuff_followed_by_guard(self):
         boxes = [(BoxInt(1), BoxInt(0)),
@@ -523,7 +517,7 @@
     def test_debugger_on(self):
         from pypy.tool.logparser import parse_log_file, extract_category
         from pypy.rlib import debug
-        
+
         loop = """
         [i0]
         debug_merge_point('xyz', 0)
diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -52,9 +52,11 @@
         newoperations = []
         #
         def do_rename(var, var_or_const):
+            if var.concretetype is lltype.Void:
+                renamings[var] = Constant(None, lltype.Void)
+                return
             renamings[var] = var_or_const
-            if (isinstance(var_or_const, Constant)
-                and var.concretetype != lltype.Void):
+            if isinstance(var_or_const, Constant):
                 value = var_or_const.value
                 value = lltype._cast_whatever(var.concretetype, value)
                 renamings_constants[var] = Constant(value, var.concretetype)
@@ -735,29 +737,54 @@
         return SpaceOperation(opname, [op.args[0]], op.result)
 
     def rewrite_op_getinteriorfield(self, op):
-        # only supports strings and unicodes
         assert len(op.args) == 3
-        assert op.args[1].value == 'chars'
         optype = op.args[0].concretetype
         if optype == lltype.Ptr(rstr.STR):
             opname = "strgetitem"
+            return SpaceOperation(opname, [op.args[0], op.args[2]], op.result)
+        elif optype == lltype.Ptr(rstr.UNICODE):
+            opname = "unicodegetitem"
+            return SpaceOperation(opname, [op.args[0], op.args[2]], op.result)
         else:
-            assert optype == lltype.Ptr(rstr.UNICODE)
-            opname = "unicodegetitem"
-        return SpaceOperation(opname, [op.args[0], op.args[2]], op.result)
+            v_inst, v_index, c_field = op.args
+            if op.result.concretetype is lltype.Void:
+                return
+            # only GcArray of Struct supported
+            assert isinstance(v_inst.concretetype.TO, lltype.GcArray)
+            STRUCT = v_inst.concretetype.TO.OF
+            assert isinstance(STRUCT, lltype.Struct)
+            descr = self.cpu.interiorfielddescrof(v_inst.concretetype.TO,
+                                                  c_field.value)
+            args = [v_inst, v_index, descr]
+            kind = getkind(op.result.concretetype)[0]
+            return SpaceOperation('getinteriorfield_gc_%s' % kind, args,
+                                  op.result)
 
     def rewrite_op_setinteriorfield(self, op):
-        # only supports strings and unicodes
         assert len(op.args) == 4
-        assert op.args[1].value == 'chars'
         optype = op.args[0].concretetype
         if optype == lltype.Ptr(rstr.STR):
             opname = "strsetitem"
+            return SpaceOperation(opname, [op.args[0], op.args[2], op.args[3]],
+                                  op.result)
+        elif optype == lltype.Ptr(rstr.UNICODE):
+            opname = "unicodesetitem"
+            return SpaceOperation(opname, [op.args[0], op.args[2], op.args[3]],
+                                  op.result)
         else:
-            assert optype == lltype.Ptr(rstr.UNICODE)
-            opname = "unicodesetitem"
-        return SpaceOperation(opname, [op.args[0], op.args[2], op.args[3]],
-                              op.result)
+            v_inst, v_index, c_field, v_value = op.args
+            if v_value.concretetype is lltype.Void:
+                return
+            # only GcArray of Struct supported
+            assert isinstance(v_inst.concretetype.TO, lltype.GcArray)
+            STRUCT = v_inst.concretetype.TO.OF
+            assert isinstance(STRUCT, lltype.Struct)
+            descr = self.cpu.interiorfielddescrof(v_inst.concretetype.TO,
+                                                  c_field.value)
+            kind = getkind(v_value.concretetype)[0]
+            args = [v_inst, v_index, v_value, descr]
+            return SpaceOperation('setinteriorfield_gc_%s' % kind, args,
+                                  op.result)
 
     def _rewrite_equality(self, op, opname):
         arg0, arg1 = op.args
diff --git a/pypy/jit/codewriter/support.py b/pypy/jit/codewriter/support.py
--- a/pypy/jit/codewriter/support.py
+++ b/pypy/jit/codewriter/support.py
@@ -13,7 +13,6 @@
 from pypy.translator.simplify import get_funcobj
 from pypy.translator.unsimplify import split_block
 from pypy.objspace.flow.model import Constant
-from pypy import conftest
 from pypy.translator.translator import TranslationContext
 from pypy.annotation.policy import AnnotatorPolicy
 from pypy.annotation import model as annmodel
@@ -48,15 +47,13 @@
     a.build_types(func, argtypes, main_entry_point=True)
     rtyper = t.buildrtyper(type_system = type_system)
     rtyper.specialize()
-    if inline:
-        auto_inlining(t, threshold=inline)
+    #if inline:
+    #    auto_inlining(t, threshold=inline)
     if backendoptimize:
         from pypy.translator.backendopt.all import backend_optimizations
         backend_optimizations(t, inline_threshold=inline or 0,
                 remove_asserts=True, really_remove_asserts=True)
 
-    #if conftest.option.view:
-    #    t.view()
     return rtyper
 
 def getgraph(func, values):
@@ -456,6 +453,8 @@
         return LLtypeHelpers._dictnext_items(lltype.Ptr(RES), iter)
     _ll_1_dictiter_nextitems.need_result_type = True
 
+    _ll_1_dict_resize = ll_rdict.ll_dict_resize
+
     # ---------- strings and unicode ----------
 
     _ll_1_str_str2unicode = ll_rstr.LLHelpers.ll_str2unicode
diff --git a/pypy/jit/codewriter/test/test_jtransform.py b/pypy/jit/codewriter/test/test_jtransform.py
--- a/pypy/jit/codewriter/test/test_jtransform.py
+++ b/pypy/jit/codewriter/test/test_jtransform.py
@@ -1,4 +1,3 @@
-import py
 import random
 try:
     from itertools import product
@@ -16,13 +15,13 @@
 
 from pypy.objspace.flow.model import FunctionGraph, Block, Link
 from pypy.objspace.flow.model import SpaceOperation, Variable, Constant
-from pypy.jit.codewriter.jtransform import Transformer
-from pypy.jit.metainterp.history import getkind
-from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr, rlist
+from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr
 from pypy.rpython.lltypesystem.module import ll_math
 from pypy.translator.unsimplify import varoftype
 from pypy.jit.codewriter import heaptracker, effectinfo
 from pypy.jit.codewriter.flatten import ListOfKind
+from pypy.jit.codewriter.jtransform import Transformer
+from pypy.jit.metainterp.history import getkind
 
 def const(x):
     return Constant(x, lltype.typeOf(x))
@@ -37,6 +36,8 @@
         return ('calldescr', FUNC, ARGS, RESULT)
     def fielddescrof(self, STRUCT, name):
         return ('fielddescr', STRUCT, name)
+    def interiorfielddescrof(self, ARRAY, name):
+        return ('interiorfielddescr', ARRAY, name)
     def arraydescrof(self, ARRAY):
         return FakeDescr(('arraydescr', ARRAY))
     def sizeof(self, STRUCT):
@@ -539,7 +540,7 @@
 
 def test_rename_on_links():
     v1 = Variable()
-    v2 = Variable()
+    v2 = Variable(); v2.concretetype = llmemory.Address
     v3 = Variable()
     block = Block([v1])
     block.operations = [SpaceOperation('cast_pointer', [v1], v2)]
@@ -676,6 +677,22 @@
     assert op1.args == [v, v_index]
     assert op1.result == v_result
 
+def test_dict_getinteriorfield():
+    DICT = lltype.GcArray(lltype.Struct('ENTRY', ('v', lltype.Signed),
+                                        ('k', lltype.Signed)))
+    v = varoftype(lltype.Ptr(DICT))
+    i = varoftype(lltype.Signed)
+    v_result = varoftype(lltype.Signed)
+    op = SpaceOperation('getinteriorfield', [v, i, Constant('v', lltype.Void)],
+                        v_result)
+    op1 = Transformer(FakeCPU()).rewrite_operation(op)
+    assert op1.opname == 'getinteriorfield_gc_i'
+    assert op1.args == [v, i, ('interiorfielddescr', DICT, 'v')]
+    op = SpaceOperation('getinteriorfield', [v, i, Constant('v', lltype.Void)],
+                        Constant(None, lltype.Void))
+    op1 = Transformer(FakeCPU()).rewrite_operation(op)
+    assert op1 is None
+
 def test_str_setinteriorfield():
     v = varoftype(lltype.Ptr(rstr.STR))
     v_index = varoftype(lltype.Signed)
@@ -702,6 +719,23 @@
     assert op1.args == [v, v_index, v_newchr]
     assert op1.result == v_void
 
+def test_dict_setinteriorfield():
+    DICT = lltype.GcArray(lltype.Struct('ENTRY', ('v', lltype.Signed),
+                                        ('k', lltype.Signed)))
+    v = varoftype(lltype.Ptr(DICT))
+    i = varoftype(lltype.Signed)
+    v_void = varoftype(lltype.Void)
+    op = SpaceOperation('setinteriorfield', [v, i, Constant('v', lltype.Void),
+                                             i],
+                        v_void)
+    op1 = Transformer(FakeCPU()).rewrite_operation(op)
+    assert op1.opname == 'setinteriorfield_gc_i'
+    assert op1.args == [v, i, i, ('interiorfielddescr', DICT, 'v')]
+    op = SpaceOperation('setinteriorfield', [v, i, Constant('v', lltype.Void),
+                                             v_void], v_void)
+    op1 = Transformer(FakeCPU()).rewrite_operation(op)
+    assert not op1
+
 def test_promote_1():
     v1 = varoftype(lltype.Signed)
     v2 = varoftype(lltype.Signed)
diff --git a/pypy/jit/metainterp/blackhole.py b/pypy/jit/metainterp/blackhole.py
--- a/pypy/jit/metainterp/blackhole.py
+++ b/pypy/jit/metainterp/blackhole.py
@@ -6,7 +6,6 @@
 from pypy.rlib.debug import make_sure_not_resized
 from pypy.rpython.lltypesystem import lltype, llmemory, rclass
 from pypy.rpython.lltypesystem.lloperation import llop
-from pypy.rpython.llinterp import LLException
 from pypy.jit.codewriter.jitcode import JitCode, SwitchDictDescr
 from pypy.jit.codewriter import heaptracker, longlong
 from pypy.jit.metainterp.jitexc import JitException, get_llexception, reraise
@@ -1154,6 +1153,26 @@
         array = cpu.bh_getfield_gc_r(vable, fdescr)
         return cpu.bh_arraylen_gc(adescr, array)
 
+    @arguments("cpu", "r", "i", "d", returns="i")
+    def bhimpl_getinteriorfield_gc_i(cpu, array, index, descr):
+        return cpu.bh_getinteriorfield_gc_i(array, index, descr)
+    @arguments("cpu", "r", "i", "d", returns="r")
+    def bhimpl_getinteriorfield_gc_r(cpu, array, index, descr):
+        return cpu.bh_getinteriorfield_gc_r(array, index, descr)
+    @arguments("cpu", "r", "i", "d", returns="f")
+    def bhimpl_getinteriorfield_gc_f(cpu, array, index, descr):
+        return cpu.bh_getinteriorfield_gc_f(array, index, descr)
+
+    @arguments("cpu", "r", "i", "d", "i")
+    def bhimpl_setinteriorfield_gc_i(cpu, array, index, descr, value):
+        cpu.bh_setinteriorfield_gc_i(array, index, descr, value)
+    @arguments("cpu", "r", "i", "d", "r")
+    def bhimpl_setinteriorfield_gc_r(cpu, array, index, descr, value):
+        cpu.bh_setinteriorfield_gc_r(array, index, descr, value)
+    @arguments("cpu", "r", "i", "d", "f")
+    def bhimpl_setinteriorfield_gc_f(cpu, array, index, descr, value):
+        cpu.bh_setinteriorfield_gc_f(array, index, descr, value)
+
     @arguments("cpu", "r", "d", returns="i")
     def bhimpl_getfield_gc_i(cpu, struct, fielddescr):
         return cpu.bh_getfield_gc_i(struct, fielddescr)
diff --git a/pypy/jit/metainterp/executor.py b/pypy/jit/metainterp/executor.py
--- a/pypy/jit/metainterp/executor.py
+++ b/pypy/jit/metainterp/executor.py
@@ -1,11 +1,8 @@
 """This implements pyjitpl's execution of operations.
 """
 
-import py
-from pypy.rpython.lltypesystem import lltype, llmemory, rstr
-from pypy.rpython.ootypesystem import ootype
-from pypy.rpython.lltypesystem.lloperation import llop
-from pypy.rlib.rarithmetic import ovfcheck, r_uint, intmask, r_longlong
+from pypy.rpython.lltypesystem import lltype, rstr
+from pypy.rlib.rarithmetic import ovfcheck, r_longlong
 from pypy.rlib.rtimer import read_timestamp
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.jit.metainterp.history import BoxInt, BoxPtr, BoxFloat, check_descr
@@ -123,6 +120,29 @@
     else:
         cpu.bh_setarrayitem_raw_i(arraydescr, array, index, itembox.getint())
 
+def do_getinteriorfield_gc(cpu, _, arraybox, indexbox, descr):
+    array = arraybox.getref_base()
+    index = indexbox.getint()
+    if descr.is_pointer_field():
+        return BoxPtr(cpu.bh_getinteriorfield_gc_r(array, index, descr))
+    elif descr.is_float_field():
+        return BoxFloat(cpu.bh_getinteriorfield_gc_f(array, index, descr))
+    else:
+        return BoxInt(cpu.bh_getinteriorfield_gc_i(array, index, descr))
+
+def do_setinteriorfield_gc(cpu, _, arraybox, indexbox, valuebox, descr):
+    array = arraybox.getref_base()
+    index = indexbox.getint()
+    if descr.is_pointer_field():
+        cpu.bh_setinteriorfield_gc_r(array, index, descr,
+                                     valuebox.getref_base())
+    elif descr.is_float_field():
+        cpu.bh_setinteriorfield_gc_f(array, index, descr,
+                                     valuebox.getfloatstorage())
+    else:
+        cpu.bh_setinteriorfield_gc_i(array, index, descr,
+                                     valuebox.getint())
+
 def do_getfield_gc(cpu, _, structbox, fielddescr):
     struct = structbox.getref_base()
     if fielddescr.is_pointer_field():
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -36,6 +36,7 @@
 
 
 class MIFrame(object):
+    debug = False
 
     def __init__(self, metainterp):
         self.metainterp = metainterp
@@ -548,6 +549,14 @@
     opimpl_getfield_gc_r_pure = _opimpl_getfield_gc_pure_any
     opimpl_getfield_gc_f_pure = _opimpl_getfield_gc_pure_any
 
+    @arguments("box", "box", "descr")
+    def _opimpl_getinteriorfield_gc_any(self, array, index, descr):
+        return self.execute_with_descr(rop.GETINTERIORFIELD_GC, descr,
+                                       array, index)
+    opimpl_getinteriorfield_gc_i = _opimpl_getinteriorfield_gc_any
+    opimpl_getinteriorfield_gc_f = _opimpl_getinteriorfield_gc_any
+    opimpl_getinteriorfield_gc_r = _opimpl_getinteriorfield_gc_any
+
     @specialize.arg(1)
     def _opimpl_getfield_gc_any_pureornot(self, opnum, box, fielddescr):
         tobox = self.metainterp.heapcache.getfield(box, fielddescr)
@@ -588,6 +597,15 @@
     opimpl_setfield_gc_r = _opimpl_setfield_gc_any
     opimpl_setfield_gc_f = _opimpl_setfield_gc_any
 
+    @arguments("box", "box", "box", "descr")
+    def _opimpl_setinteriorfield_gc_any(self, array, index, value, descr):
+        self.execute_with_descr(rop.SETINTERIORFIELD_GC, descr,
+                                array, index, value)
+    opimpl_setinteriorfield_gc_i = _opimpl_setinteriorfield_gc_any
+    opimpl_setinteriorfield_gc_f = _opimpl_setinteriorfield_gc_any
+    opimpl_setinteriorfield_gc_r = _opimpl_setinteriorfield_gc_any
+                                
+
     @arguments("box", "descr")
     def _opimpl_getfield_raw_any(self, box, fielddescr):
         return self.execute_with_descr(rop.GETFIELD_RAW, fielddescr, box)
@@ -2588,17 +2606,21 @@
         self.pc = position
         #
         if not we_are_translated():
-            print '\tpyjitpl: %s(%s)' % (name, ', '.join(map(repr, args))),
+            if self.debug:
+                print '\tpyjitpl: %s(%s)' % (name, ', '.join(map(repr, args))),
             try:
                 resultbox = unboundmethod(self, *args)
             except Exception, e:
-                print '-> %s!' % e.__class__.__name__
+                if self.debug:
+                    print '-> %s!' % e.__class__.__name__
                 raise
             if num_return_args == 0:
-                print
+                if self.debug:
+                    print
                 assert resultbox is None
             else:
-                print '-> %r' % (resultbox,)
+                if self.debug:
+                    print '-> %r' % (resultbox,)
                 assert argcodes[next_argcode] == '>'
                 result_argcode = argcodes[next_argcode + 1]
                 assert resultbox.type == {'i': history.INT,
diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py
--- a/pypy/jit/metainterp/resoperation.py
+++ b/pypy/jit/metainterp/resoperation.py
@@ -1,5 +1,4 @@
 from pypy.rlib.objectmodel import we_are_translated
-from pypy.rlib.debug import make_sure_not_resized
 
 def ResOperation(opnum, args, result, descr=None):
     cls = opclasses[opnum]
@@ -457,6 +456,7 @@
 
     'GETARRAYITEM_GC/2d',
     'GETARRAYITEM_RAW/2d',
+    'GETINTERIORFIELD_GC/2d',
     'GETFIELD_GC/1d',
     'GETFIELD_RAW/1d',
     '_MALLOC_FIRST',
@@ -473,6 +473,7 @@
 
     'SETARRAYITEM_GC/3d',
     'SETARRAYITEM_RAW/3d',
+    'SETINTERIORFIELD_GC/3d',
     'SETFIELD_GC/2d',
     'SETFIELD_RAW/2d',
     'STRSETITEM/3',
diff --git a/pypy/jit/metainterp/test/test_dict.py b/pypy/jit/metainterp/test/test_dict.py
--- a/pypy/jit/metainterp/test/test_dict.py
+++ b/pypy/jit/metainterp/test/test_dict.py
@@ -91,7 +91,7 @@
         res1 = f(100)
         res2 = self.meta_interp(f, [100], listops=True)
         assert res1 == res2
-        self.check_loops(int_mod=1) # the hash was traced
+        self.check_loops(int_mod=1) # the hash was traced and eq, but cached
 
     def test_dict_setdefault(self):
         myjitdriver = JitDriver(greens = [], reds = ['total', 'dct'])
@@ -128,7 +128,7 @@
         assert f(100) == 50
         res = self.meta_interp(f, [100], listops=True)
         assert res == 50
-        self.check_loops(int_mod=1)
+        self.check_loops(int_mod=1) # key + eq, but cached
 
     def test_repeated_lookup(self):
         myjitdriver = JitDriver(greens = [], reds = ['n', 'd'])
@@ -153,10 +153,12 @@
 
         res = self.meta_interp(f, [100], listops=True)
         assert res == f(50)
-        self.check_loops({"call": 7, "guard_false": 1, "guard_no_exception": 6,
+        self.check_loops({"call": 5, "getfield_gc": 1, "getinteriorfield_gc": 1,
+                          "guard_false": 1, "guard_no_exception": 4,
                           "guard_true": 1, "int_and": 1, "int_gt": 1,
                           "int_is_true": 1, "int_sub": 1, "jump": 1,
-                          "new_with_vtable": 1, "setfield_gc": 1})
+                          "new_with_vtable": 1, "new": 1, "new_array": 1,
+                          "setfield_gc": 3, })
 
 
 class TestOOtype(DictTests, OOJitMixin):
diff --git a/pypy/jit/metainterp/warmspot.py b/pypy/jit/metainterp/warmspot.py
--- a/pypy/jit/metainterp/warmspot.py
+++ b/pypy/jit/metainterp/warmspot.py
@@ -206,7 +206,7 @@
         self.make_enter_functions()
         self.rewrite_jit_merge_points(policy)
 
-        verbose = not self.cpu.translate_support_code
+        verbose = False # not self.cpu.translate_support_code
         self.codewriter.make_jitcodes(verbose=verbose)
         self.rewrite_can_enter_jits()
         self.rewrite_set_param()
diff --git a/pypy/module/pypyjit/test_pypy_c/test_containers.py b/pypy/module/pypyjit/test_pypy_c/test_containers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_containers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_containers.py
@@ -46,7 +46,7 @@
         assert loop.match_by_id("getitem", """
             i28 = call(ConstClass(ll_dict_lookup__dicttablePtr_objectPtr_Signed), p18, p6, i25, descr=...)
             ...
-            p33 = call(ConstClass(ll_get_value__dicttablePtr_Signed), p18, i28, descr=...)
+            p33 = getinteriorfield_gc(p31, i26, <InteriorFieldDescr <GcPtrFieldDescr dictentry.value .*>>)
             ...
         """)
 
diff --git a/pypy/rpython/lltypesystem/ll2ctypes.py b/pypy/rpython/lltypesystem/ll2ctypes.py
--- a/pypy/rpython/lltypesystem/ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/ll2ctypes.py
@@ -15,12 +15,11 @@
     load_library_kwargs = {}
 
 import os
-from pypy import conftest
 from pypy.rpython.lltypesystem import lltype, llmemory
 from pypy.rpython.extfunc import ExtRegistryEntry
 from pypy.rlib.objectmodel import Symbolic, ComputedIntSymbolic
 from pypy.tool.uid import fixid
-from pypy.rlib.rarithmetic import r_uint, r_singlefloat, r_longfloat, base_int, intmask
+from pypy.rlib.rarithmetic import r_singlefloat, r_longfloat, base_int, intmask
 from pypy.annotation import model as annmodel
 from pypy.rpython.llinterp import LLInterpreter, LLException
 from pypy.rpython.lltypesystem.rclass import OBJECT, OBJECT_VTABLE
@@ -531,6 +530,10 @@
     def __str__(self):
         return repr(self)
 
+    def _setparentstructure(self, parent, parentindex):
+        super(_parentable_mixin, self)._setparentstructure(parent, parentindex)
+        self._keepparent = parent   # always keep a strong ref
+
 class _struct_mixin(_parentable_mixin):
     """Mixin added to _struct containers when they become ctypes-based."""
     __slots__ = ()
@@ -566,7 +569,10 @@
         return 0, sys.maxint
 
     def getitem(self, index, uninitialized_ok=False):
-        return self._storage.contents._getitem(index, boundscheck=False)
+        res = self._storage.contents._getitem(index, boundscheck=False)
+        if isinstance(self._TYPE.OF, lltype.ContainerType):
+            res._obj._setparentstructure(self, index)
+        return res
 
     def setitem(self, index, value):
         self._storage.contents._setitem(index, value, boundscheck=False)
@@ -1279,6 +1285,8 @@
             self.intval = intmask(void_p.value)
 
     def __eq__(self, other):
+        if not other:
+            return self.intval == 0
         if isinstance(other, _llgcopaque):
             return self.intval == other.intval
         storage = object()
diff --git a/pypy/rpython/lltypesystem/lltype.py b/pypy/rpython/lltypesystem/lltype.py
--- a/pypy/rpython/lltypesystem/lltype.py
+++ b/pypy/rpython/lltypesystem/lltype.py
@@ -1522,7 +1522,7 @@
             and parentindex in (self._parent_type._names[0], 0)
             and self._TYPE._gckind == typeOf(parent)._gckind):
             # keep strong reference to parent, we share the same allocation
-            self._keepparent = parent 
+            self._keepparent = parent
 
     def _parentstructure(self, check=True):
         if self._wrparent is not None:
@@ -1730,7 +1730,8 @@
         # Keep the parent array alive, we share the same allocation.
         # Don't do it if we are inside a GC object, though -- it's someone
         # else's job to keep the GC object alive
-        if typeOf(top_container(parent))._gckind == 'raw':
+        if (typeOf(top_container(parent))._gckind == 'raw' or
+            hasattr(top_container(parent)._storage, 'contents')):  # ll2ctypes
             self._keepparent = parent
 
     def __str__(self):
diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py
--- a/pypy/rpython/lltypesystem/rdict.py
+++ b/pypy/rpython/lltypesystem/rdict.py
@@ -1,16 +1,14 @@
 from pypy.tool.pairtype import pairtype
-from pypy.annotation import model as annmodel
 from pypy.objspace.flow.model import Constant
-from pypy.rpython.rdict import AbstractDictRepr, AbstractDictIteratorRepr,\
-     rtype_newdict
+from pypy.rpython.rdict import (AbstractDictRepr, AbstractDictIteratorRepr,
+     rtype_newdict)
 from pypy.rpython.lltypesystem import lltype
+from pypy.rlib import objectmodel, jit
 from pypy.rlib.rarithmetic import r_uint, intmask, LONG_BIT
-from pypy.rlib.objectmodel import hlinvoke
-from pypy.rpython import robject
-from pypy.rlib import objectmodel, jit
 from pypy.rpython import rmodel
 from pypy.rpython.error import TyperError
 
+
 HIGHEST_BIT = intmask(1 << (LONG_BIT - 1))
 MASK = intmask(HIGHEST_BIT - 1)
 
@@ -417,17 +415,16 @@
     ENTRIES = lltype.typeOf(entries).TO
     return ENTRIES.fasthashfn(entries[i].key)
 
- at jit.dont_look_inside
 def ll_get_value(d, i):
     return d.entries[i].value
 
 def ll_keyhash_custom(d, key):
     DICT = lltype.typeOf(d).TO
-    return hlinvoke(DICT.r_rdict_hashfn, d.fnkeyhash, key)
+    return objectmodel.hlinvoke(DICT.r_rdict_hashfn, d.fnkeyhash, key)
 
 def ll_keyeq_custom(d, key1, key2):
     DICT = lltype.typeOf(d).TO
-    return hlinvoke(DICT.r_rdict_eqfn, d.fnkeyeq, key1, key2)
+    return objectmodel.hlinvoke(DICT.r_rdict_eqfn, d.fnkeyeq, key1, key2)
 
 def ll_dict_len(d):
     return d.num_items
@@ -448,6 +445,8 @@
     i = ll_dict_lookup(d, key, hash)
     return _ll_dict_setitem_lookup_done(d, key, value, hash, i)
 
+# Leaving as dont_look_inside ATM, it has a few branches which could lead to
+# many bridges if we don't consider their possible frequency.
 @jit.dont_look_inside
 def _ll_dict_setitem_lookup_done(d, key, value, hash, i):
     valid = (i & HIGHEST_BIT) == 0
@@ -492,6 +491,8 @@
         raise KeyError
     _ll_dict_del(d, i)
 
+# XXX: Move the size checking and resize into a single call which is opauqe to
+# the JIT to avoid extra branches.
 @jit.dont_look_inside
 def _ll_dict_del(d, i):
     d.entries.mark_deleted(i)
@@ -532,6 +533,7 @@
 # ------- a port of CPython's dictobject.c's lookdict implementation -------
 PERTURB_SHIFT = 5
 
+ at jit.dont_look_inside
 def ll_dict_lookup(d, key, hash):
     entries = d.entries
     ENTRIES = lltype.typeOf(entries).TO
@@ -621,7 +623,6 @@
     d.num_items = 0
     d.resize_counter = DICT_INITSIZE * 2
     return d
-ll_newdict.oopspec = 'newdict()'
 
 def ll_newdict_size(DICT, length_estimate):
     length_estimate = (length_estimate // 2) * 3
@@ -633,7 +634,6 @@
     d.num_items = 0
     d.resize_counter = n * 2
     return d
-ll_newdict_size.oopspec = 'newdict()'
 
 # pypy.rpython.memory.lldict uses a dict based on Struct and Array
 # instead of GcStruct and GcArray, which is done by using different
@@ -866,7 +866,6 @@
         global_popitem_index.nextindex = base + counter
     return i
 
- at jit.dont_look_inside
 def ll_popitem(ELEM, dic):
     i = _ll_getnextitem(dic)
     entry = dic.entries[i]
diff --git a/pypy/rpython/lltypesystem/test/test_ll2ctypes.py b/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
--- a/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
@@ -8,6 +8,7 @@
 from pypy.rpython.lltypesystem.ll2ctypes import uninitialized2ctypes
 from pypy.rpython.lltypesystem.ll2ctypes import ALLOCATED, force_cast
 from pypy.rpython.lltypesystem.ll2ctypes import cast_adr_to_int, get_ctypes_type
+from pypy.rpython.lltypesystem.ll2ctypes import _llgcopaque
 from pypy.rpython.annlowlevel import llhelper
 from pypy.rlib import rposix
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
@@ -1349,6 +1350,11 @@
         round = ctypes2lltype(llmemory.GCREF, lltype2ctypes(opaque.hide()))
         assert Opaque.show(round) is opaque
 
+    def test_array_of_structs(self):
+        A = lltype.GcArray(lltype.Struct('x', ('v', lltype.Signed)))
+        a = lltype.malloc(A, 5)
+        a2 = ctypes2lltype(lltype.Ptr(A), lltype2ctypes(a))
+        assert a2._obj.getitem(0)._obj._parentstructure() is a2._obj
 
 class TestPlatform(object):
     def test_lib_on_libpaths(self):
@@ -1390,3 +1396,7 @@
         f = rffi.llexternal('f', [rffi.INT, rffi.INT], rffi.INT,
                             compilation_info=eci)
         assert f(3, 4) == 7
+
+    def test_llgcopaque_eq(self):
+        assert _llgcopaque(1) != None
+        assert _llgcopaque(0) == None

From noreply at buildbot.pypy.org  Sat Oct 22 00:28:34 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sat, 22 Oct 2011 00:28:34 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Use the new io stack for sys.stdout
	&friends
Message-ID: <20111021222834.DC3BD820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48328:8b87c77d1645
Date: 2011-10-22 00:24 +0200
http://bitbucket.org/pypy/pypy/changeset/8b87c77d1645/

Log:	Use the new io stack for sys.stdout &friends sys.stdout is not
	prebuilt anymore
	- translated code build it in app_main (to get all options & env vars)
	- untranslated code build it in sys.__init__ (easier for tests)

diff --git a/lib_pypy/_pypy_interact.py b/lib_pypy/_pypy_interact.py
--- a/lib_pypy/_pypy_interact.py
+++ b/lib_pypy/_pypy_interact.py
@@ -55,11 +55,7 @@
             else:
                 prompt = getattr(sys, 'ps1', '>>> ')
             try:
-                line = raw_input(prompt)
-                # Can be None if sys.stdin was redefined
-                encoding = getattr(sys.stdin, 'encoding', None)
-                if encoding and not isinstance(line, unicode):
-                    line = line.decode(encoding)
+                line = input(prompt)
             except EOFError:
                 console.write("\n")
                 break
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -489,16 +489,16 @@
         self.exceptions_module = Module(self, w_name)
         self.exceptions_module.install()
 
+        from pypy.module.imp import Module
+        w_name = self.wrap('imp')
+        mod = Module(self, w_name)
+        mod.install()
+
         from pypy.module.sys import Module
         w_name = self.wrap('sys')
         self.sys = Module(self, w_name)
         self.sys.install()
 
-        from pypy.module.imp import Module
-        w_name = self.wrap('imp')
-        mod = Module(self, w_name)
-        mod.install()
-
         from pypy.module.__builtin__ import Module
         w_name = self.wrap('builtins')
         self.builtin = Module(self, w_name)
diff --git a/pypy/module/__builtin__/__init__.py b/pypy/module/__builtin__/__init__.py
--- a/pypy/module/__builtin__/__init__.py
+++ b/pypy/module/__builtin__/__init__.py
@@ -13,7 +13,6 @@
 
     appleveldefs = {
         'execfile'      : 'app_io.execfile',
-        'raw_input'     : 'app_io.raw_input',
         'input'         : 'app_io.input',
         'print'         : 'app_io.print_',
 
diff --git a/pypy/module/__builtin__/app_io.py b/pypy/module/__builtin__/app_io.py
--- a/pypy/module/__builtin__/app_io.py
+++ b/pypy/module/__builtin__/app_io.py
@@ -27,8 +27,8 @@
     co = compile(source.rstrip()+"\n", filename, 'exec')
     exec(co, glob, loc)
 
-def raw_input(prompt=None):
-    """raw_input([prompt]) -> string
+def input(prompt=None):
+    """input([prompt]) -> string
 
 Read a string from standard input.  The trailing newline is stripped.
 If the user hits EOF (Unix: Ctl-D, Windows: Ctl-Z+Return), raise EOFError.
@@ -37,11 +37,11 @@
     try:
         stdin = sys.stdin
     except AttributeError:
-        raise RuntimeError("[raw_]input: lost sys.stdin")
+        raise RuntimeError("input: lost sys.stdin")
     try:
         stdout = sys.stdout
     except AttributeError:
-        raise RuntimeError("[raw_]input: lost sys.stdout")
+        raise RuntimeError("input: lost sys.stdout")
 
     # hook for the readline module
     if (hasattr(sys, '__raw_input__') and
@@ -66,10 +66,6 @@
         return line[:-1]
     return line
 
-def input(prompt=None):
-    """Equivalent to eval(raw_input(prompt))."""
-    return eval(raw_input(prompt))
-
 def print_(*args, **kwargs):
     """The new-style print function from py3k."""
     fp = kwargs.pop("file", sys.stdout)
@@ -78,8 +74,6 @@
     def write(data):
         if not isinstance(data, str):
             data = str(data)
-        if getattr(fp, 'encoding', None):
-            data = data.encode(fp.encoding)
         fp.write(data)
     sep = kwargs.pop("sep", None)
     if sep is not None:
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -28,12 +28,6 @@
         'maxsize'               : 'space.wrap(sys.maxint)',
         'byteorder'             : 'space.wrap(sys.byteorder)', 
         'maxunicode'            : 'space.wrap(vm.MAXUNICODE)',
-        'stdin'                 : 'state.getio(space).w_stdin',
-        '__stdin__'             : 'state.getio(space).w_stdin',
-        'stdout'                : 'state.getio(space).w_stdout',
-        '__stdout__'            : 'state.getio(space).w_stdout',
-        'stderr'                : 'state.getio(space).w_stderr',
-        '__stderr__'            : 'state.getio(space).w_stderr',
         'pypy_objspaceclass'    : 'space.wrap(repr(space))',
         #'prefix'               : # added by pypy_initial_path() when it 
         #'exec_prefix'          : # succeeds, pointing to trunk or /usr
@@ -116,6 +110,18 @@
                 w_handle = vm.get_dllhandle(space)
                 space.setitem(self.w_dict, space.wrap("dllhandle"), w_handle)
 
+        if not space.config.translating:
+            # Install standard streams for tests that don't call app_main
+            space.appexec([], """():
+                import sys, io
+                sys.stdin = sys.__stdin__ = io.open(0, "r", closefd=False)
+                sys.stdin.buffer.raw.name = "<stdin>"
+                sys.stdout = sys.__stdout__ = io.open(1, "w", closefd=False)
+                sys.stdout.buffer.raw.name = "<stdout>"
+                sys.stderr = sys.__stderr__ = io.open(2, "w", closefd=False)
+                sys.stderr.buffer.raw.name = "<stderr>"
+               """)
+
     def getmodule(self, name):
         space = self.space
         w_modules = self.get('modules') 
diff --git a/pypy/module/sys/state.py b/pypy/module/sys/state.py
--- a/pypy/module/sys/state.py
+++ b/pypy/module/sys/state.py
@@ -84,31 +84,6 @@
 def get(space):
     return space.fromcache(State)
 
-class IOState:
-    def __init__(self, space):
-        from pypy.module._file.interp_file import W_File
-        self.space = space
-
-        stdin = W_File(space)
-        stdin.file_fdopen(0, "r", 1)
-        stdin.name = '<stdin>'
-        self.w_stdin = space.wrap(stdin)
-
-        stdout = W_File(space)
-        stdout.file_fdopen(1, "w", 1)
-        stdout.name = '<stdout>'
-        self.w_stdout = space.wrap(stdout)
-
-        stderr = W_File(space)
-        stderr.file_fdopen(2, "w", 0)
-        stderr.name = '<stderr>'
-        self.w_stderr = space.wrap(stderr)
-
-        stdin._when_reading_first_flush(stdout)
-
-def getio(space):
-    return space.fromcache(IOState)
-
 def pypy_getudir(space):
     """NOT_RPYTHON
     (should be removed from interpleveldefs before translation)"""
diff --git a/pypy/translator/goal/app_main.py b/pypy/translator/goal/app_main.py
--- a/pypy/translator/goal/app_main.py
+++ b/pypy/translator/goal/app_main.py
@@ -33,7 +33,7 @@
         except:
             # not an integer: print it to stderr
             try:
-                print >> sys.stderr, exitcode
+                print(exitcode, file=sys.stderr)
             except:
                 pass   # too bad
             exitcode = 1
@@ -74,16 +74,16 @@
         # extra debugging info in case the code below goes very wrong
         if DEBUG and hasattr(sys, 'stderr'):
             s = getattr(etype, '__name__', repr(etype))
-            print >> sys.stderr, "debug: exception-type: ", s
-            print >> sys.stderr, "debug: exception-value:", str(evalue)
+            print("debug: exception-type: ", s, file=sys.stderr)
+            print("debug: exception-value:", str(evalue), file=sys.stderr)
             tbentry = etraceback
             if tbentry:
                 while tbentry.tb_next:
                     tbentry = tbentry.tb_next
                 lineno = tbentry.tb_lineno
                 filename = tbentry.tb_frame.f_code.co_filename
-                print >> sys.stderr, "debug: exception-tb:    %s:%d" % (
-                    filename, lineno)
+                print("debug: exception-tb:    %s:%d" % (filename, lineno),
+                      file=sys.stderr)
 
         # set the sys.last_xxx attributes
         sys.last_type = etype
@@ -101,10 +101,10 @@
         except AttributeError:
             pass   # too bad
         else:
-            print >> stderr, 'Error calling sys.excepthook:'
+            print('Error calling sys.excepthook:', file=stderr)
             originalexcepthook(*sys.exc_info())
-            print >> stderr
-            print >> stderr, 'Original exception was:'
+            print(file=stderr)
+            print('Original exception was:', file=stderr)
 
     # we only get here if sys.excepthook didn't do its job
     originalexcepthook(etype, evalue, etraceback)
@@ -117,7 +117,7 @@
     try:
         options = sys.pypy_translation_info
     except AttributeError:
-        print >> sys.stderr, 'no translation information found'
+        print('no translation information found', file=sys.stderr)
     else:
         optitems = options.items()
         optitems.sort()
@@ -170,14 +170,6 @@
         from os import fdopen
     return fdopen(fd, mode, bufsize)
 
-def set_unbuffered_io():
-    sys.stdin  = sys.__stdin__  = fdopen(0, 'rb', 0)
-    sys.stdout = sys.__stdout__ = fdopen(1, 'wb', 0)
-    sys.stderr = sys.__stderr__ = fdopen(2, 'wb', 0)
-
-def set_fully_buffered_io():
-    sys.stdout = sys.__stdout__ = fdopen(1, 'w')
-
 # ____________________________________________________________
 # Main entry point
 
@@ -257,6 +249,56 @@
             sys.path.append(dir)
             _seen[dir] = True
 
+def initstdio(encoding, unbuffered):
+    if ':' in encoding:
+        encoding, errors = encoding.split(':', 1)
+    else:
+        errors = None
+
+    sys.stdin = sys.__stdin__ = create_stdio(
+        0, False, "<stdin>", encoding, errors, unbuffered)
+    sys.stdout = sys.__stdout__ = create_stdio(
+        1, True, "<stdout>", encoding, errors, unbuffered)
+    sys.stderr = sys.__stderr__ = create_stdio(
+        2, True, "<stderr>", encoding, errors, unbuffered)
+
+def create_stdio(fd, writing, name, encoding, errors, unbuffered):
+    import io
+
+    if writing:
+        mode = "wb"
+    else:
+        mode= "rb"
+    # stdin is always opened in buffered mode, first because it
+    # shouldn't make a difference in common use cases, second because
+    # TextIOWrapper depends on the presence of a read1() method which
+    # only exists on buffered streams.
+    if writing:
+        buffering = 0
+    else:
+        buffering = -1
+    if sys.platform == 'win32' and not writing:
+        # translate \r\n to \n for sys.stdin on Windows
+        newline = None
+    else:
+        newline = '\n'
+    buf = io.open(fd, mode, buffering, closefd=False)
+
+    if buffering:
+        raw = buf.raw
+    else:
+        raw = buf
+    raw.name = name
+    if unbuffered or raw.isatty():
+        line_buffering = True
+    else:
+        line_buffering = False
+
+    stream = io.TextIOWrapper(buf, encoding, errors,
+                              newline=newline,
+                              line_buffering=line_buffering)
+    return stream
+    
 def set_io_encoding(io_encoding):
     try:
         import _file
@@ -467,10 +509,10 @@
     if '__pypy__' not in sys.builtin_module_names:
         sys.setrecursionlimit(5000)
 
-    if unbuffered:
-        set_unbuffered_io()
-    elif not sys.stdout.isatty():
-        set_fully_buffered_io()
+    readenv = not ignore_environment
+    io_encoding = ((readenv and os.getenv("PYTHONIOENCODING"))
+                   or sys.getfilesystemencoding())
+    initstdio(io_encoding, unbuffered)
 
     mainmodule = type(sys)('__main__')
     sys.modules['__main__'] = mainmodule
@@ -481,12 +523,6 @@
         except:
             print("'import site' failed", file=sys.stderr)
 
-    readenv = not ignore_environment
-    io_encoding = ((readenv and os.getenv("PYTHONIOENCODING"))
-                   or sys.getfilesystemencoding())
-    if io_encoding:
-        set_io_encoding(io_encoding)
-
     pythonwarnings = readenv and os.getenv('PYTHONWARNINGS')
     if pythonwarnings:
         warnoptions.extend(pythonwarnings.split(','))
@@ -606,6 +642,10 @@
                     args = (runpy._run_module_as_main, '__main__', False)
                 else:
                     # no.  That's the normal path, "pypy stuff.py".
+                    def execfile(filename, namespace):
+                        with open(filename) as f:
+                            code = f.read()
+                        exec(code, namespace)
                     args = (execfile, filename, mainmodule.__dict__)
             success = run_toplevel(*args)
 
@@ -653,29 +693,55 @@
     setup_sys_executable(executable, nanos)
     try:
         cmdline = parse_command_line(argv)
-    except CommandLineError, e:
+    except CommandLineError as e:
         print_error(str(e))
         return 2
-    except SystemExit, e:
+    except SystemExit as e:
         return e.code or 0
     setup_initial_paths(**cmdline)
     return run_command_line(**cmdline)
 
 
 if __name__ == '__main__':
-    import autopath
-    import nanos
-    # obscure! try removing the following line, see how it crashes, and
-    # guess why...
-    ImStillAroundDontForgetMe = sys.modules['__main__']
+    "For unit tests only"
+    import os as nanos
+    import os
 
-    # debugging only
+    if len(sys.argv) > 1 and sys.argv[1] == '--argparse-only':
+        import io
+        del sys.argv[:2]
+        sys.stdout = sys.stderr = io.StringIO()
+        try:
+            options = parse_command_line(sys.argv)
+        except SystemExit:
+            print('SystemExit', file=sys.__stdout__)
+            print(sys.stdout.getvalue(), file=sys.__stdout__)
+            raise
+        except BaseException as e:
+            print('Error', file=sys.__stdout__)
+            raise
+        else:
+            print('Return', file=sys.__stdout__)
+        print(options, file=sys.__stdout__)
+        print(sys.argv, file=sys.__stdout__)
+
+    # Testing python on python is hard:
+    # Some code above (run_command_line) will create a new module
+    # named __main__ and store it into sys.modules.  There it will
+    # replace the __main__ module that CPython created to execute the
+    # lines you are currently reading. This will free the module, and
+    # all globals (os, sys...) will be set to None.
+    # To avoid this we make a copy of our __main__ module.
+    sys.modules['__cpython_main__'] = sys.modules['__main__']
+
     def pypy_initial_path(s):
+        return ["../lib-python/3.2", "../lib_pypy", ".."]
         from pypy.module.sys.state import getinitialpath
         try:
             return getinitialpath(s)
         except OSError:
             return None
+    sys.pypy_initial_path = pypy_initial_path
 
     # add an emulator for these pypy-only or 2.7-only functions
     # (for test_pyc_commandline_argument)
@@ -711,10 +777,6 @@
     old_argv = sys.argv
     old_path = sys.path
 
-    from pypy.module.sys.version import PYPY_VERSION
-    sys.pypy_version_info = PYPY_VERSION
-    sys.pypy_initial_path = pypy_initial_path
-    os = nanos.os_module_for_testing
     try:
         sys.exit(int(entry_point(sys.argv[0], sys.argv[1:], os)))
     finally:
diff --git a/pypy/translator/goal/test2/mymodule.py b/pypy/translator/goal/test2/mymodule.py
--- a/pypy/translator/goal/test2/mymodule.py
+++ b/pypy/translator/goal/test2/mymodule.py
@@ -2,9 +2,9 @@
 
 import sys
 
-print 'mymodule running'
-print 'Name:', __name__
-print 'File:', __file__
-print 'Argv:', sys.argv
+print('mymodule running')
+print('Name:', __name__)
+print('File:', __file__)
+print('Argv:', sys.argv)
 
 somevalue = "foobar"
diff --git a/pypy/translator/goal/test2/test_app_main.py b/pypy/translator/goal/test2/test_app_main.py
--- a/pypy/translator/goal/test2/test_app_main.py
+++ b/pypy/translator/goal/test2/test_app_main.py
@@ -8,7 +8,15 @@
 from pypy.tool.udir import udir
 from contextlib import contextmanager
 
-banner = sys.version.splitlines()[0]
+python3 = os.environ.get("PYTHON3", "python3")
+
+def get_banner():
+    p = subprocess.Popen([python3, "-c",
+                          "import sys; print(sys.version.splitlines()[0])"],
+                         stdout=subprocess.PIPE)
+    return p.stdout.read().rstrip()
+banner = get_banner()
+print repr(banner)
 
 app_main = os.path.join(autopath.this_dir, os.pardir, 'app_main.py')
 app_main = os.path.abspath(app_main)
@@ -53,29 +61,28 @@
     return py.path.local().bestrelpath(pdir)
 
 demo_script = getscript("""
-    print 'hello'
-    print 'Name:', __name__
-    print 'File:', __file__
+    print('hello')
+    print('Name:', __name__)
+    print('File:', __file__)
     import sys
-    print 'Exec:', sys.executable
-    print 'Argv:', sys.argv
-    print 'goodbye'
+    print('Exec:', sys.executable)
+    print('Argv:', sys.argv)
+    print('goodbye')
     myvalue = 6*7
     """)
 
 crashing_demo_script = getscript("""
-    print 'Hello2'
+    print('Hello2')
     myvalue2 = 11
     ooups
     myvalue2 = 22
-    print 'Goodbye2'   # should not be reached
+    print('Goodbye2')  # should not be reached
     """)
 
 
 class TestParseCommandLine:
 
-    def check_options(self, options, sys_argv, **expected):
-        assert sys.argv == sys_argv
+    def check_options(self, options, sys_argv, expected):
         for key, value in expected.items():
             assert options[key] == value
         for key, value in options.items():
@@ -84,25 +91,20 @@
                     "option %r has unexpectedly the value %r" % (key, value))
 
     def check(self, argv, **expected):
-        import StringIO
-        from pypy.translator.goal import app_main
-        saved_sys_argv = sys.argv[:]
-        saved_sys_stdout = sys.stdout
-        saved_sys_stderr = sys.stdout
-        app_main.os = os
-        try:
-            sys.stdout = sys.stderr = StringIO.StringIO()
-            try:
-                options = app_main.parse_command_line(argv)
-            except SystemExit:
-                output = expected['output_contains']
-                assert output in sys.stdout.getvalue()
-            else:
-                self.check_options(options, **expected)
-        finally:
-            sys.argv[:] = saved_sys_argv
-            sys.stdout = saved_sys_stdout
-            sys.stderr = saved_sys_stderr
+        p = subprocess.Popen([python3, app_main,
+                              '--argparse-only'] + list(argv),
+                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+        res = p.wait()
+        outcome = p.stdout.readline()
+        if outcome == 'SystemExit\n':
+            output = p.stdout.read()
+            assert expected['output_contains'] in output
+        else:
+            app_options = eval(p.stdout.readline())
+            sys_argv = eval(p.stdout.readline())
+            app_options['sys_argv'] = sys_argv
+            self.check_options(app_options, sys_argv, expected)
 
     def test_all_combinations_I_can_think_of(self):
         self.check([], sys_argv=[''], run_stdin=True)
@@ -227,7 +229,7 @@
         return child
 
     def spawn(self, argv):
-        return self._spawn(sys.executable, [app_main] + argv)
+        return self._spawn(python3, [app_main] + argv)
 
     def test_interactive(self):
         child = self.spawn([])
@@ -336,7 +338,7 @@
     def test_atexit(self):
         child = self.spawn([])
         child.expect('>>> ')
-        child.sendline('def f(): print "foobye"')
+        child.sendline('def f(): print("foobye")')
         child.sendline('')
         child.sendline('import atexit; atexit.register(f)')
         child.sendline('6*7')
@@ -407,7 +409,8 @@
             os.environ['PYTHONPATH'] = old
 
     def test_unbuffered(self):
-        line = 'import os,sys;sys.stdout.write(str(789));os.read(0,1)'
+        # In Python3, -u affects the "binary layer" of sys.stdout.
+        line = 'import os,sys;sys.stdout.buffer.write(str(789).encode());os.read(0,1)'
         child = self.spawn(['-u', '-c', line])
         child.expect('789')    # expect to see it before the timeout hits
         child.sendline('X')
@@ -447,8 +450,7 @@
         if sys.platform == "win32":
             skip("close_fds is not supported on Windows platforms")
         import subprocess, select, os
-        python = sys.executable
-        pipe = subprocess.Popen([python, app_main, "-u", "-i"],
+        pipe = subprocess.Popen([python3, app_main, "-u", "-i"],
                                 stdout=subprocess.PIPE,
                                 stdin=subprocess.PIPE,
                                 stderr=subprocess.STDOUT,
@@ -472,7 +474,7 @@
         os.environ['PYTHONINSPECT_'] = '1'
         try:
             path = getscript("""
-                print 6*7
+                print(6*7)
                 """)
             child = self.spawn([path])
             child.expect('42')
@@ -484,7 +486,7 @@
         path = getscript("""
             import os
             os.environ['PYTHONINSPECT'] = '1'
-            print 6*7
+            print(6*7)
             """)
         child = self.spawn([path])
         child.expect('42')
@@ -503,6 +505,7 @@
             del os.environ['PYTHONINSPECT_']
 
     def test_stdout_flushes_before_stdin_blocks(self):
+        skip("Python3 does not implement this behavior")
         # This doesn't really test app_main.py, but a behavior that
         # can only be checked on top of py.py with pexpect.
         path = getscript("""
@@ -510,10 +513,10 @@
             sys.stdout.write('Are you suggesting coconuts migrate? ')
             line = sys.stdin.readline()
             assert line.rstrip() == 'Not at all. They could be carried.'
-            print 'A five ounce bird could not carry a one pound coconut.'
+            print('A five ounce bird could not carry a one pound coconut.')
             """)
         py_py = os.path.join(autopath.pypydir, 'bin', 'py.py')
-        child = self._spawn(sys.executable, [py_py, path])
+        child = self._spawn(sys.executable, [py_py, '-S', path])
         child.expect('Are you suggesting coconuts migrate?', timeout=120)
         child.sendline('Not at all. They could be carried.')
         child.expect('A five ounce bird could not carry a one pound coconut.')
@@ -521,14 +524,14 @@
     def test_no_space_before_argument(self):
         if not hasattr(runpy, '_run_module_as_main'):
             skip("requires CPython >= 2.6")
-        child = self.spawn(['-cprint "hel" + "lo"'])
+        child = self.spawn(['-cprint("hel" + "lo")'])
         child.expect('hello')
 
         child = self.spawn(['-mpypy.translator.goal.test2.mymodule'])
         child.expect('mymodule running')
 
     def test_ps1_only_if_interactive(self):
-        argv = ['-c', 'import sys; print hasattr(sys, "ps1")']
+        argv = ['-c', 'import sys; print(hasattr(sys, "ps1"))']
         child = self.spawn(argv)
         child.expect('False')
 
@@ -536,7 +539,7 @@
 
     def run_with_status_code(self, cmdline, senddata='', expect_prompt=False,
             expect_banner=False, python_flags='', env=None):
-        cmdline = '%s %s "%s" %s' % (sys.executable, python_flags,
+        cmdline = '%s %s "%s" %s' % (python3, python_flags,
                                      app_main, cmdline)
         print 'POPEN:', cmdline
         process = subprocess.Popen(
@@ -586,9 +589,9 @@
         assert 'Goodbye2' not in data
 
     def test_option_W(self):
-        data = self.run('-W d -c "print 42"')
+        data = self.run('-W d -c "print(42)"')
         assert '42' in data
-        data = self.run('-Wd -c "print 42"')
+        data = self.run('-Wd -c "print(42)"')
         assert '42' in data
 
     def test_option_W_crashing(self):
@@ -604,7 +607,7 @@
         assert "Invalid -W option ignored: invalid action:" in data
 
     def test_option_c(self):
-        data = self.run('-c "print 6**5"')
+        data = self.run('-c "print(6**5)"')
         assert '7776' in data
 
     def test_no_pythonstartup(self):
@@ -623,7 +626,7 @@
         try:
             os.environ['PYTHONWARNINGS'] = "once,error"
             data = self.run('-W ignore -W default '
-                            '-c "import sys; print sys.warnoptions"')
+                            '-c "import sys; print(sys.warnoptions)"')
             assert "['ignore', 'default', 'once', 'error']" in data
         finally:
             os.environ['PYTHONWARNINGS'] = old
@@ -644,7 +647,7 @@
     def test_pythoninspect_doesnt_override_isatty(self):
         os.environ['PYTHONINSPECT_'] = '1'
         try:
-            data = self.run('', senddata='6*7\nprint 2+3\n')
+            data = self.run('', senddata='6*7\nprint(2+3)\n')
             assert data == '5\n'
         finally:
             del os.environ['PYTHONINSPECT_']
@@ -656,7 +659,7 @@
         # if a file name is passed, the banner is never printed but
         # we get a prompt anyway
         cmdline = '-i %s' % getscript("""
-            print 'hello world'
+            print('hello world')
             """)
         data = self.run(cmdline, senddata='6*7\nraise SystemExit\n',
                                  expect_prompt=True, expect_banner=False)
@@ -672,7 +675,7 @@
             time.sleep(1)
             # stdout flushed automatically here
             """)
-        cmdline = '%s -u "%s" %s' % (sys.executable, app_main, path)
+        cmdline = '%s -u "%s" %s' % (python3, app_main, path)
         print 'POPEN:', cmdline
         child_in, child_out_err = os.popen4(cmdline)
         data = child_out_err.read(11)
@@ -697,35 +700,35 @@
                 if old_pythonpath is not None:
                     os.putenv('PYTHONPATH', old_pythonpath)
 
-        tmpdir.join('site.py').write('print "SHOULD NOT RUN"')
+        tmpdir.join('site.py').write('print("SHOULD NOT RUN")')
         runme_py = tmpdir.join('runme.py')
-        runme_py.write('print "some text"')
+        runme_py.write('print("some text")')
 
         cmdline = str(runme_py)
 
         with chdir_and_unset_pythonpath(tmpdir):
             data = self.run(cmdline, python_flags='-S')
 
-        assert data == "some text\n"
+        assert data in ("'import site' failed\nsome text\n")
 
         runme2_py = tmpdir.mkdir('otherpath').join('runme2.py')
-        runme2_py.write('print "some new text"\n'
+        runme2_py.write('print("some new text")\n'
                         'import sys\n'
-                        'print sys.path\n')
+                        'print(sys.path)\n')
 
         cmdline2 = str(runme2_py)
 
         with chdir_and_unset_pythonpath(tmpdir):
             data = self.run(cmdline2, python_flags='-S')
-        assert data.startswith("some new text\n")
+        assert data.startswith("'import site' failed\nsome new text\n")
         assert repr(str(tmpdir.join('otherpath'))) in data
         assert "''" not in data
 
-        data = self.run('-c "import sys; print sys.path"')
+        data = self.run('-c "import sys; print(sys.path)"')
         assert data.startswith("[''")
 
     def test_pyc_commandline_argument(self):
-        p = getscript_pyc(self.space, "print 6*7\n")
+        p = getscript_pyc(self.space, "print(6*7)\n")
         assert os.path.isfile(p) and p.endswith('.pyc')
         data = self.run(p)
         assert data == 'in _run_compiled_module\n'
@@ -733,7 +736,7 @@
     def test_main_in_dir_commandline_argument(self):
         if not hasattr(runpy, '_run_module_as_main'):
             skip("requires CPython >= 2.6")
-        p = getscript_in_dir('import sys; print sys.argv[0]\n')
+        p = getscript_in_dir('import sys; print(sys.argv[0])\n')
         data = self.run(p)
         assert data == p + '\n'
         data = self.run(p + os.sep)

From noreply at buildbot.pypy.org  Sat Oct 22 00:28:36 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sat, 22 Oct 2011 00:28:36 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Fix the cpyext module,
 at least py.py can import it without segfaulting
Message-ID: <20111021222836.2B4C6820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48329:928a4651aa4e
Date: 2011-10-22 00:24 +0200
http://bitbucket.org/pypy/pypy/changeset/928a4651aa4e/

Log:	Fix the cpyext module, at least py.py can import it without
	segfaulting

diff --git a/pypy/module/cpyext/funcobject.py b/pypy/module/cpyext/funcobject.py
--- a/pypy/module/cpyext/funcobject.py
+++ b/pypy/module/cpyext/funcobject.py
@@ -86,14 +86,13 @@
     w_code = space.wrap(func.code)
     return borrow_from(w_func, w_code)
 
- at cpython_api([PyObject, PyObject, PyObject], PyObject)
-def PyMethod_New(space, w_func, w_self, w_cls):
-    """Return a new method object, with func being any callable object; this is the
-    function that will be called when the method is called.  If this method should
-    be bound to an instance, self should be the instance and class should be the
-    class of self, otherwise self should be NULL and class should be the
-    class which provides the unbound method."""
-    return Method(space, w_func, w_self, w_cls)
+ at cpython_api([PyObject, PyObject], PyObject)
+def PyMethod_New(space, w_func, w_self):
+    """Return a new method object, with func being any callable object
+    and self the instance the method should be bound. func is the
+    function that will be called when the method is called. self must
+    not be NULL."""
+    return Method(space, w_func, w_self)
 
 @cpython_api([PyObject], PyObject)
 def PyMethod_Function(space, w_method):
diff --git a/pypy/module/cpyext/include/unicodeobject.h b/pypy/module/cpyext/include/unicodeobject.h
--- a/pypy/module/cpyext/include/unicodeobject.h
+++ b/pypy/module/cpyext/include/unicodeobject.h
@@ -22,6 +22,7 @@
     PyObject_HEAD
     Py_UNICODE *buffer;
     Py_ssize_t size;
+    char *utf8buffer;
 } PyUnicodeObject;
 
 
diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py
--- a/pypy/module/cpyext/methodobject.py
+++ b/pypy/module/cpyext/methodobject.py
@@ -186,13 +186,10 @@
     return ret
 
 def cmethod_descr_get(space, w_function, w_obj, w_cls=None):
-    asking_for_bound = (space.is_w(w_cls, space.w_None) or
-                        not space.is_w(w_obj, space.w_None) or
-                        space.is_w(w_cls, space.type(space.w_None)))
-    if asking_for_bound:
-        return space.wrap(Method(space, w_function, w_obj, w_cls))
+    if w_obj is None or space.is_w(w_obj, space.w_None):
+        return w_function
     else:
-        return w_function
+        return space.wrap(Method(space, w_function, w_obj))
 
 
 W_PyCFunctionObject.typedef = TypeDef(
diff --git a/pypy/module/cpyext/stringobject.py b/pypy/module/cpyext/stringobject.py
--- a/pypy/module/cpyext/stringobject.py
+++ b/pypy/module/cpyext/stringobject.py
@@ -73,8 +73,8 @@
     interpreter object.  The buffer may be mutated, until string_realize() is
     called.
     """
-    typedescr = get_typedescr(space.w_str.instancetypedef)
-    py_obj = typedescr.allocate(space, space.w_str)
+    typedescr = get_typedescr(space.w_bytes.instancetypedef)
+    py_obj = typedescr.allocate(space, space.w_bytes)
     py_str = rffi.cast(PyStringObject, py_obj)
 
     buflen = length + 1
@@ -89,7 +89,7 @@
     buffer must not be modified.
     """
     py_str = rffi.cast(PyStringObject, py_obj)
-    py_str.c_size = len(space.str_w(w_obj))
+    py_str.c_size = len(space.bytes_w(w_obj))
     py_str.c_buffer = lltype.nullptr(rffi.CCHARP.TO)
 
 def string_realize(space, py_obj):
@@ -119,14 +119,14 @@
 def PyString_FromStringAndSize(space, char_p, length):
     if char_p:
         s = rffi.charpsize2str(char_p, length)
-        return make_ref(space, space.wrap(s))
+        return make_ref(space, space.wrapbytes(s))
     else:
         return rffi.cast(PyObject, new_empty_str(space, length))
 
 @cpython_api([CONST_STRING], PyObject)
 def PyString_FromString(space, char_p):
     s = rffi.charp2str(char_p)
-    return space.wrap(s)
+    return space.wrapbytes(s)
 
 @cpython_api([PyObject], rffi.CCHARP, error=0)
 def PyString_AsString(space, ref):
@@ -134,7 +134,7 @@
     if not ref_str.c_buffer:
         # copy string buffer
         w_str = from_ref(space, ref)
-        s = space.str_w(w_str)
+        s = space.bytes_w(w_str)
         ref_str.c_buffer = rffi.str2charp(s)
     return ref_str.c_buffer
 
@@ -147,7 +147,7 @@
     if not ref_str.c_buffer:
         # copy string buffer
         w_str = from_ref(space, ref)
-        s = space.str_w(w_str)
+        s = space.bytes_w(w_str)
         ref_str.c_buffer = rffi.str2charp(s)
     buffer[0] = ref_str.c_buffer
     if length:
@@ -235,12 +235,6 @@
     PyString_Concat(space, ref, newpart)
     Py_DecRef(space, newpart)
 
- at cpython_api([PyObject, PyObject], PyObject)
-def PyString_Format(space, w_format, w_args):
-    """Return a new string object from format and args. Analogous to format %
-    args.  The args argument must be a tuple."""
-    return space.mod(w_format, w_args)
-
 @cpython_api([CONST_STRING], PyObject)
 def PyString_InternFromString(space, string):
     """A combination of PyString_FromString() and
@@ -250,25 +244,6 @@
     s = rffi.charp2str(string)
     return space.new_interned_str(s)
 
- at cpython_api([PyObject, rffi.CCHARP, rffi.CCHARP], PyObject)
-def PyString_AsEncodedObject(space, w_str, encoding, errors):
-    """Encode a string object using the codec registered for encoding and return
-    the result as Python object. encoding and errors have the same meaning as
-    the parameters of the same name in the string encode() method. The codec to
-    be used is looked up using the Python codec registry. Return NULL if an
-    exception was raised by the codec.
-
-    This function is not available in 3.x and does not have a PyBytes alias."""
-    if not PyString_Check(space, w_str):
-        PyErr_BadArgument(space)
-
-    w_encoding = w_errors = space.w_None
-    if encoding:
-        w_encoding = space.wrap(rffi.charp2str(encoding))
-    if errors:
-        w_errors = space.wrap(rffi.charp2str(errors))
-    return space.call_method(w_str, 'encode', w_encoding, w_errors)
-
 @cpython_api([PyObject, PyObject], PyObject)
 def _PyString_Join(space, w_sep, w_seq):
     return space.call_method(w_sep, 'join', w_seq)
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -203,6 +203,9 @@
         if filename is None, the module name will be used to construct the
         filename.
         """
+        name = name.encode()
+        init = init.encode()
+        body = body.encode()
         if init is not None:
             code = """
             #include <Python.h>
diff --git a/pypy/module/cpyext/test/test_funcobject.py b/pypy/module/cpyext/test/test_funcobject.py
--- a/pypy/module/cpyext/test/test_funcobject.py
+++ b/pypy/module/cpyext/test/test_funcobject.py
@@ -31,13 +31,11 @@
 
         w_function = space.getattr(w_method, space.wrap("im_func"))
         w_self = space.getattr(w_method, space.wrap("im_self"))
-        w_class = space.getattr(w_method, space.wrap("im_class"))
 
         assert space.is_w(api.PyMethod_Function(w_method), w_function)
         assert space.is_w(api.PyMethod_Self(w_method), w_self)
-        assert space.is_w(api.PyMethod_Class(w_method), w_class)
 
-        w_method2 = api.PyMethod_New(w_function, w_self, w_class)
+        w_method2 = api.PyMethod_New(w_function, w_self)
         assert space.eq_w(w_method, w_method2)
 
     def test_getcode(self, space, api):
@@ -67,7 +65,6 @@
             api.Py_DecRef(ref)
             return co_flags
         assert get_flags("x") == CO_NESTED | CO_OPTIMIZED | CO_NEWLOCALS
-        assert get_flags("x", "exec x") == CO_NESTED | CO_NEWLOCALS
         assert get_flags("x, *args") & CO_VARARGS
         assert get_flags("x, **kw") & CO_VARKEYWORDS
         assert get_flags("x", "yield x") & CO_GENERATOR
diff --git a/pypy/module/cpyext/test/test_stringobject.py b/pypy/module/cpyext/test/test_stringobject.py
--- a/pypy/module/cpyext/test/test_stringobject.py
+++ b/pypy/module/cpyext/test/test_stringobject.py
@@ -45,12 +45,12 @@
              """
                 return PyBool_FromLong(PyString_Check(PyTuple_GetItem(args, 0)));
              """)])
-        assert module.get_hello1() == 'Hello world'
-        assert module.get_hello2() == 'Hello world'
+        assert module.get_hello1() == b'Hello world'
+        assert module.get_hello2() == b'Hello world'
         assert module.test_Size()
         raises(TypeError, module.test_Size_exception)
 
-        assert module.test_is_string("")
+        assert module.test_is_string(b"")
         assert not module.test_is_string(())
 
     def test_string_buffer_init(self):
@@ -93,7 +93,7 @@
              """),
             ])
         s = module.getstring()
-        assert s == 'test'
+        assert s == b'test'
 
     def test_py_string_as_string(self):
         module = self.import_extension('foo', [
@@ -103,7 +103,7 @@
                        PyTuple_GetItem(args, 0)), 4);
              '''
             )])
-        assert module.string_as_string("huheduwe") == "huhe"
+        assert module.string_as_string(b"huheduwe") == b"huhe"
 
     def test_AsStringAndSize(self):
         module = self.import_extension('foo', [
@@ -150,7 +150,7 @@
               return res;
             }
             ''')
-        res = module.test_string_format_v(1, "xyz")
+        res = module.test_string_format_v(1, b"xyz")
         assert res == "bla 1 ble xyz\n"
 
     def test_format(self):
@@ -163,7 +163,7 @@
              '''
              )
             ])
-        res = module.test_string_format(1, "xyz")
+        res = module.test_string_format(1, b"xyz")
         assert res == "bla 1 ble xyz\n"
 
 class TestString(BaseApiTest):
@@ -205,41 +205,37 @@
         Py_DecRef(space, py_obj)
 
     def test_Concat(self, space, api):
-        ref = make_ref(space, space.wrap('abc'))
+        ref = make_ref(space, space.wrapbytes('abc'))
         ptr = lltype.malloc(PyObjectP.TO, 1, flavor='raw')
         ptr[0] = ref
-        api.PyString_Concat(ptr, space.wrap('def'))
-        assert space.str_w(from_ref(space, ptr[0])) == 'abcdef'
+        api.PyString_Concat(ptr, space.wrapbytes('def'))
+        assert space.bytes_w(from_ref(space, ptr[0])) == 'abcdef'
         api.PyString_Concat(ptr, space.w_None)
         assert not ptr[0]
         ptr[0] = lltype.nullptr(PyObject.TO)
-        api.PyString_Concat(ptr, space.wrap('def')) # should not crash
+        api.PyString_Concat(ptr, space.wrapbytes('def')) # should not crash
         lltype.free(ptr, flavor='raw')
 
     def test_ConcatAndDel(self, space, api):
-        ref1 = make_ref(space, space.wrap('abc'))
-        ref2 = make_ref(space, space.wrap('def'))
+        ref1 = make_ref(space, space.wrapbytes('abc'))
+        ref2 = make_ref(space, space.wrapbytes('def'))
         ptr = lltype.malloc(PyObjectP.TO, 1, flavor='raw')
         ptr[0] = ref1
         api.PyString_ConcatAndDel(ptr, ref2)
-        assert space.str_w(from_ref(space, ptr[0])) == 'abcdef'
+        assert space.bytes_w(from_ref(space, ptr[0])) == 'abcdef'
         assert ref2.c_ob_refcnt == 0
         Py_DecRef(space, ptr[0])
         ptr[0] = lltype.nullptr(PyObject.TO)
-        ref2 = make_ref(space, space.wrap('foo'))
+        ref2 = make_ref(space, space.wrapbytes('foo'))
         api.PyString_ConcatAndDel(ptr, ref2) # should not crash
         assert ref2.c_ob_refcnt == 0
         lltype.free(ptr, flavor='raw')
 
-    def test_format(self, space, api):
-        assert "1 2" == space.unwrap(
-            api.PyString_Format(space.wrap('%s %d'), space.wrap((1, 2))))
-
     def test_asbuffer(self, space, api):
         bufp = lltype.malloc(rffi.CCHARPP.TO, 1, flavor='raw')
         lenp = lltype.malloc(Py_ssize_tP.TO, 1, flavor='raw')
 
-        w_text = space.wrap("text")
+        w_text = space.wrapbytes("text")
         assert api.PyObject_AsCharBuffer(w_text, bufp, lenp) == 0
         assert lenp[0] == 4
         assert rffi.charp2str(bufp[0]) == 'text'
@@ -254,42 +250,12 @@
         rffi.free_charp(buf)
         assert w_s1 is w_s2
 
-    def test_AsEncodedObject(self, space, api):
-        ptr = space.wrap('abc')
-
-        errors = rffi.str2charp("strict")
-
-        encoding = rffi.str2charp("hex")
-        res = api.PyString_AsEncodedObject(
-            ptr, encoding, errors)
-        assert space.unwrap(res) == "616263"
-
-        res = api.PyString_AsEncodedObject(
-            ptr, encoding, lltype.nullptr(rffi.CCHARP.TO))
-        assert space.unwrap(res) == "616263"
-        rffi.free_charp(encoding)
-
-        encoding = rffi.str2charp("unknown_encoding")
-        self.raises(space, api, LookupError, api.PyString_AsEncodedObject,
-                    ptr, encoding, errors)
-        rffi.free_charp(encoding)
-
-        rffi.free_charp(errors)
-
-        res = api.PyString_AsEncodedObject(
-            ptr, lltype.nullptr(rffi.CCHARP.TO), lltype.nullptr(rffi.CCHARP.TO))
-        assert space.unwrap(res) == "abc"
-
-        self.raises(space, api, TypeError, api.PyString_AsEncodedObject,
-            space.wrap(2), lltype.nullptr(rffi.CCHARP.TO), lltype.nullptr(rffi.CCHARP.TO)
-        )
-
     def test_eq(self, space, api):
-        assert 1 == api._PyString_Eq(space.wrap("hello"), space.wrap("hello"))
-        assert 0 == api._PyString_Eq(space.wrap("hello"), space.wrap("world"))
+        assert 1 == api._PyString_Eq(space.wrapbytes("hello"), space.wrapbytes("hello"))
+        assert 0 == api._PyString_Eq(space.wrapbytes("hello"), space.wrapbytes("world"))
 
     def test_join(self, space, api):
-        w_sep = space.wrap('<sep>')
-        w_seq = space.wrap(['a', 'b'])
+        w_sep = space.wrapbytes('<sep>')
+        w_seq = space.newtuple([space.wrapbytes('a'), space.wrapbytes('b')])
         w_joined = api._PyString_Join(w_sep, w_seq)
-        assert space.unwrap(w_joined) == 'a<sep>b'
+        assert space.bytes_w(w_joined) == 'a<sep>b'
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -493,8 +493,8 @@
         w_typename = space.getattr(w_type, space.wrap('__name__'))
         heaptype = rffi.cast(PyHeapTypeObject, pto)
         heaptype.c_ht_name = make_ref(space, w_typename)
-        from pypy.module.cpyext.stringobject import PyString_AsString
-        pto.c_tp_name = PyString_AsString(space, heaptype.c_ht_name)
+        from pypy.module.cpyext.unicodeobject import _PyUnicode_AsString
+        pto.c_tp_name = _PyUnicode_AsString(space, heaptype.c_ht_name)
     else:
         pto.c_tp_name = rffi.str2charp(w_type.getname(space))
     pto.c_tp_basicsize = -1 # hopefully this makes malloc bail out
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -21,7 +21,8 @@
 PyUnicodeObjectStruct = lltype.ForwardReference()
 PyUnicodeObject = lltype.Ptr(PyUnicodeObjectStruct)
 PyUnicodeObjectFields = (PyObjectFields +
-    (("buffer", rffi.CWCHARP), ("size", Py_ssize_t)))
+    (("buffer", rffi.CWCHARP), ("size", Py_ssize_t),
+     ("utf8buffer", rffi.CCHARP)))
 cpython_struct("PyUnicodeObject", PyUnicodeObjectFields, PyUnicodeObjectStruct)
 
 @bootstrap_function
@@ -79,6 +80,8 @@
     py_unicode = rffi.cast(PyUnicodeObject, py_obj)
     if py_unicode.c_buffer:
         lltype.free(py_unicode.c_buffer, flavor="raw")
+    if py_unicode.c_utf8buffer:
+        lltype.free(py_unicode.c_utf8buffer, flavor="raw")
     from pypy.module.cpyext.object import PyObject_dealloc
     PyObject_dealloc(space, py_obj)
 
@@ -196,6 +199,17 @@
                              space.wrap("expected unicode object"))
     return PyUnicode_AS_UNICODE(space, ref)
 
+ at cpython_api([PyObject], rffi.CCHARP)
+def _PyUnicode_AsString(space, ref):
+    ref_unicode = rffi.cast(PyUnicodeObject, ref)
+    if not ref_unicode.c_utf8buffer:
+        # Copy unicode buffer
+        w_unicode = from_ref(space, ref)
+        w_encoded = unicodetype.encode_object(space, w_unicode, "utf-8", "strict")
+        s = space.bytes_w(w_encoded)
+        ref_unicode.c_utf8buffer = rffi.str2charp(s)
+    return ref_unicode.c_utf8buffer
+
 @cpython_api([PyObject], Py_ssize_t, error=-1)
 def PyUnicode_GetSize(space, ref):
     if from_ref(space, rffi.cast(PyObject, ref.c_ob_type)) is space.w_unicode:

From noreply at buildbot.pypy.org  Sat Oct 22 00:28:37 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sat, 22 Oct 2011 00:28:37 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: astcompiler: remove special code for the
	exec statement
Message-ID: <20111021222837.600C7820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48330:8e974d408b19
Date: 2011-10-22 00:24 +0200
http://bitbucket.org/pypy/pypy/changeset/8e974d408b19/

Log:	astcompiler: remove special code for the exec statement

diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -221,7 +221,7 @@
                                   "nested scopes: '%s'" % (identifier,))
             container = self.cell_vars
         elif scope == symtable.SCOPE_GLOBAL_IMPLICIT:
-            if self.scope.locals_fully_known:
+            if self.scope.optimized:
                 op = name_ops_global(ctx)
         elif scope == symtable.SCOPE_GLOBAL_EXPLICIT:
             op = name_ops_global(ctx)
@@ -1125,7 +1125,7 @@
         scope = self.scope
         assert isinstance(scope, symtable.FunctionScope)
         flags = 0
-        if scope.locals_fully_known:
+        if scope.optimized:
             flags |= consts.CO_OPTIMIZED
         if scope.nested:
             flags |= consts.CO_NESTED
diff --git a/pypy/interpreter/astcompiler/symtable.py b/pypy/interpreter/astcompiler/symtable.py
--- a/pypy/interpreter/astcompiler/symtable.py
+++ b/pypy/interpreter/astcompiler/symtable.py
@@ -32,14 +32,13 @@
         self.col_offset = col_offset
         self.parent = None
         self.name = name
-        self.locals_fully_known = False
+        self.optimized = False
         self.symbols = None
         self.roles = {}
         self.varnames = []
         self.children = []
         self.free_vars = []
         self.temp_name_counter = 1
-        self.has_exec = False
         self.has_free = False
         self.child_has_free = False
         self.nested = False
@@ -85,10 +84,6 @@
         raise SyntaxError("return outside function", ret.lineno,
                           ret.col_offset)
 
-    def note_exec(self, exc):
-        """Called when an exec statement is found."""
-        self.has_exec = True
-
     def note_import_star(self, imp):
         """Called when a star import is found."""
         return False
@@ -230,7 +225,6 @@
         self.optimized = True
         self.return_with_value = False
         self.import_star = None
-        self.bare_exec = None
 
     def note_symbol(self, identifier, role):
         # Special-case super: it counts as a use of __class__
@@ -252,12 +246,6 @@
             self.return_with_value = True
             self.ret = ret
 
-    def note_exec(self, exc):
-        Scope.note_exec(self, exc)
-        if not exc.globals:
-            self.optimized = False
-            self.bare_exec = exc
-
     def note_import_star(self, imp):
         self.optimized = False
         self.import_star = imp
@@ -293,20 +281,11 @@
             name = self.name
             if self.import_star:
                 node = self.import_star
-                if self.bare_exec:
-                    err = "function '%s' uses import * and bare exec, " \
-                        "which are illegal because it %s" % (name, trailer)
-                else:
-                    err = "import * is not allowed in function '%s' because " \
-                        "it %s" % (name, trailer)
-            elif self.bare_exec:
-                node = self.bare_exec
-                err = "unqualified exec is not allowed in function '%s' " \
-                    "because it %s" % (name, trailer)
+                err = "import * is not allowed in function '%s' because " \
+                    "it %s" % (name, trailer)
             else:
                 raise AssertionError("unknown reason for unoptimization")
             raise SyntaxError(err, node.lineno, node.col_offset)
-        self.locals_fully_known = self.optimized and not self.has_exec
 
 
 class ClassScope(Scope):
@@ -438,10 +417,6 @@
     def visit_alias(self, alias):
         self._visit_alias(alias)
 
-    def visit_Exec(self, exc):
-        self.scope.note_exec(exc)
-        ast.GenericASTVisitor.visit_Exec(self, exc)
-
     def visit_Yield(self, yie):
         self.scope.note_yield(yie)
         ast.GenericASTVisitor.visit_Yield(self, yie)
diff --git a/pypy/interpreter/astcompiler/test/test_symtable.py b/pypy/interpreter/astcompiler/test/test_symtable.py
--- a/pypy/interpreter/astcompiler/test/test_symtable.py
+++ b/pypy/interpreter/astcompiler/test/test_symtable.py
@@ -54,12 +54,12 @@
     def test_toplevel(self):
         scp = self.mod_scope("x = 4")
         assert scp.lookup("x") == symtable.SCOPE_LOCAL
-        assert not scp.locals_fully_known
+        assert not scp.optimized
         scp = self.mod_scope("x = 4", "single")
-        assert not scp.locals_fully_known
+        assert not scp.optimized
         assert scp.lookup("x") == symtable.SCOPE_LOCAL
         scp = self.mod_scope("x*4*6", "eval")
-        assert not scp.locals_fully_known
+        assert not scp.optimized
         assert scp.lookup("x") == symtable.SCOPE_GLOBAL_IMPLICIT
 
     def test_duplicate_argument(self):
@@ -278,12 +278,8 @@
 
     def test_unoptimization_with_nested_scopes(self):
         table = (
-            ("from x import *; exec 'hi'", "function 'f' uses import * " \
-                 "and bare exec, which are illegal because it"),
             ("from x import *", "import * is not allowed in function 'f' " \
                  "because it"),
-            ("exec 'hi'", "unqualified exec is not allowed in function 'f' " \
-                 "because it")
          )
         for line, error in table:
             input = """def n():
@@ -320,20 +316,6 @@
         assert     "import * only allowed at module level" in err1
         assert not "import * only allowed at module level" in err2
 
-    def test_exec(self):
-        self.mod_scope("exec 'hi'")
-        scp = self.func_scope("def f(): exec 'hi'")
-        assert not scp.optimized
-        assert not scp.locals_fully_known
-        assert isinstance(scp.bare_exec, ast.Exec)
-        assert scp.has_exec
-        for line in ("exec 'hi' in g", "exec 'hi' in g, h"):
-            scp = self.func_scope("def f(): " + line)
-            assert scp.optimized
-            assert not scp.locals_fully_known
-            assert scp.bare_exec is None
-            assert scp.has_exec
-
     def test_yield(self):
         scp = self.func_scope("def f(): yield x")
         assert scp.is_generator

From noreply at buildbot.pypy.org  Sat Oct 22 00:28:38 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sat, 22 Oct 2011 00:28:38 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Various translation fixes
Message-ID: <20111021222838.93D0A820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48331:dd152a06badd
Date: 2011-10-22 00:24 +0200
http://bitbucket.org/pypy/pypy/changeset/dd152a06badd/

Log:	Various translation fixes

diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -43,13 +43,11 @@
                             space.w_bytes)))):
                 if decode:
                     msg = ("decoding error handler must return "
-                           "(unicode, int) tuple, not %s")
+                           "(str, int) tuple")
                 else:
                     msg = ("encoding error handler must return "
-                           "(unicode, int) tuple, not %s")
-                raise operationerrfmt(
-                    space.w_TypeError, msg,
-                    space.unicode_w(space.repr(w_res)))
+                           "(str/bytes, int) tuple")
+                raise OperationError(space.w_TypeError, space.wrap(msg))
             w_replace, w_newpos = space.fixedview(w_res, 2)
             newpos = space.int_w(w_newpos)
             if newpos < 0:
diff --git a/pypy/module/_lsprof/interp_lsprof.py b/pypy/module/_lsprof/interp_lsprof.py
--- a/pypy/module/_lsprof/interp_lsprof.py
+++ b/pypy/module/_lsprof/interp_lsprof.py
@@ -194,7 +194,7 @@
         # try to get the real class that defines the method,
         # which is a superclass of the class of the instance
         from pypy.objspace.std.typeobject import W_TypeObject   # xxx
-        w_type = w_arg.w_class
+        w_type = space.type(w_arg.w_instance)
         class_name = w_type.getname(space)    # if the rest doesn't work
         if isinstance(w_type, W_TypeObject) and name != '?':
             w_realclass, _ = space.lookup_in_type_where(w_type, name)
diff --git a/pypy/module/_lsprof/test/test_cprofile.py b/pypy/module/_lsprof/test/test_cprofile.py
--- a/pypy/module/_lsprof/test/test_cprofile.py
+++ b/pypy/module/_lsprof/test/test_cprofile.py
@@ -41,7 +41,7 @@
         entries = {}
         for entry in stats:
             if not hasattr(entry.code, 'co_name'):
-                print entry.code
+                print(entry.code)
             else:
                 entries[entry.code.co_name] = entry
         efoo = entries['foo']
@@ -137,7 +137,7 @@
                 results.append(timer() - start_timer)
                 for methodname in methodnames:
                     import pstats
-                    from StringIO import StringIO
+                    from io import StringIO
                     s = StringIO()
                     stats = pstats.Stats(prof, stream=s)
                     stats.strip_dirs().sort_stats("stdname")
@@ -168,12 +168,12 @@
                             lines.remove(line)
                             break
                     else:
-                        print 'NOT FOUND:', pattern.rstrip('\n')
-                        print '--- GOT ---'
-                        print got
-                        print
-                        print '--- EXPECTED ---'
-                        print expected
+                        print('NOT FOUND:', pattern.rstrip('\n'))
+                        print('--- GOT ---')
+                        print(got)
+                        print()
+                        print('--- EXPECTED ---')
+                        print(expected)
                         assert False
                 assert not lines
         finally:
diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py
--- a/pypy/module/pyexpat/interp_pyexpat.py
+++ b/pypy/module/pyexpat/interp_pyexpat.py
@@ -453,7 +453,7 @@
             from pypy.interpreter.unicodehelper import PyUnicode_DecodeUTF8
             return space.wrap(PyUnicode_DecodeUTF8(space, s))
         else:
-            return space.wrap(s)
+            return space.wrapbytes(s)
 
     def w_convert_charp(self, space, data):
         if data:
@@ -552,7 +552,7 @@
         # Yes, supports only 8bit encodings
         translationmap = space.unicode_w(
             space.call_method(
-                space.wrap(self.all_chars), "decode",
+                space.wrapbytes(self.all_chars), "decode",
                 space.wrap(name), space.wrap("replace")))
 
         for i in range(256):
@@ -808,5 +808,5 @@
 def ErrorString(space, code):
     """ErrorString(errno) -> string
 Returns string error for given number."""
-    return space.wrap(rffi.charp2str(XML_ErrorString(code)))
+    return space.wrapbytes(rffi.charp2str(XML_ErrorString(code)))
 

From noreply at buildbot.pypy.org  Sat Oct 22 00:28:39 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sat, 22 Oct 2011 00:28:39 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Correctly implement isatty() on text files.
Message-ID: <20111021222839.C4836820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48332:c4c393a85de6
Date: 2011-10-22 00:24 +0200
http://bitbucket.org/pypy/pypy/changeset/c4c393a85de6/

Log:	Correctly implement isatty() on text files. Important for the
	interactive mode!

diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py
--- a/pypy/module/_io/interp_textio.py
+++ b/pypy/module/_io/interp_textio.py
@@ -448,6 +448,10 @@
             space.wrap("<_io.TextIOWrapper %sencoding=%r>"), w_args
         )
 
+    def isatty_w(self, space):
+        self._check_init(space)
+        return space.call_method(self.w_buffer, "isatty")
+
     def readable_w(self, space):
         self._check_init(space)
         return space.call_method(self.w_buffer, "readable")
@@ -993,6 +997,7 @@
     close = interp2app(W_TextIOWrapper.close_w),
 
     line_buffering = interp_attrproperty("line_buffering", W_TextIOWrapper),
+    isatty = interp2app(W_TextIOWrapper.isatty_w),
     readable = interp2app(W_TextIOWrapper.readable_w),
     writable = interp2app(W_TextIOWrapper.writable_w),
     seekable = interp2app(W_TextIOWrapper.seekable_w),
diff --git a/pypy/module/_io/test/test_textio.py b/pypy/module/_io/test/test_textio.py
--- a/pypy/module/_io/test/test_textio.py
+++ b/pypy/module/_io/test/test_textio.py
@@ -29,6 +29,14 @@
         assert t.readable()
         assert t.seekable()
 
+    def test_isatty(self):
+        import _io
+        class Tty(_io.BytesIO):
+            def isatty(self):
+                return True
+        txt = _io.TextIOWrapper(Tty())
+        assert txt.isatty()
+
     def test_unreadable(self):
         import _io
         class UnReadable(_io.BytesIO):

From noreply at buildbot.pypy.org  Sat Oct 22 00:28:41 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sat, 22 Oct 2011 00:28:41 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Implement _hashlib.openssl_md_meth_names,
 needed by hashlib.py
Message-ID: <20111021222841.054BD820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48333:08ea3258278e
Date: 2011-10-22 00:24 +0200
http://bitbucket.org/pypy/pypy/changeset/08ea3258278e/

Log:	Implement _hashlib.openssl_md_meth_names, needed by hashlib.py

diff --git a/pypy/module/_hashlib/__init__.py b/pypy/module/_hashlib/__init__.py
--- a/pypy/module/_hashlib/__init__.py
+++ b/pypy/module/_hashlib/__init__.py
@@ -5,6 +5,7 @@
 class Module(MixedModule):
     interpleveldefs = {
         'new' : 'interp_hashlib.new',
+        'openssl_md_meth_names': 'interp_hashlib.get(space).w_meth_names'
         }
 
     appleveldefs = {
diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py
--- a/pypy/module/_hashlib/interp_hashlib.py
+++ b/pypy/module/_hashlib/interp_hashlib.py
@@ -12,6 +12,47 @@
 
 algorithms = ('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512')
 
+def hash_name_mapper_callback(obj_name, userdata):
+    state = global_state[0] 
+    assert state is not None
+    if not obj_name:
+        return
+    # Ignore aliased names, they pollute the list and OpenSSL appears
+    # to have a its own definition of alias as the resulting list
+    # still contains duplicate and alternate names for several
+    # algorithms.
+    if obj_name[0].c_alias:
+        return
+    try:
+        w_name = state.space.wrap(rffi.charp2str(obj_name[0].c_name))
+        state.space.call_method(state.w_meth_names, "add", w_name)
+    except OperationError, e:
+        state.w_error = e
+
+# XXX make it threadlocal?
+global_state = [None]
+
+class State: 
+    def __init__(self, space): 
+        self.space = space
+        self.generate_method_names(space)
+    
+    def generate_method_names(self, space):
+        self.w_error = None
+        try:
+            global_state[0] = self
+            self.w_meth_names = space.call_function(space.w_set)
+            ropenssl.OBJ_NAME_do_all(
+                ropenssl.OBJ_NAME_TYPE_MD_METH, 
+                hash_name_mapper_callback, None)
+        finally:
+            global_state[0] = None
+        if self.w_error:
+            raise self.w_error
+
+def get(space):
+    return space.fromcache(State)
+
 class W_Hash(Wrappable):
     ctx = lltype.nullptr(ropenssl.EVP_MD_CTX.TO)
 
diff --git a/pypy/module/_hashlib/test/test_hashlib.py b/pypy/module/_hashlib/test/test_hashlib.py
--- a/pypy/module/_hashlib/test/test_hashlib.py
+++ b/pypy/module/_hashlib/test/test_hashlib.py
@@ -5,6 +5,11 @@
     def setup_class(cls):
         cls.space = gettestobjspace(usemodules=['_hashlib'])
 
+    def test_method_names(self):
+        import _hashlib
+        assert isinstance(_hashlib.openssl_md_meth_names, set)
+        assert "md5" in _hashlib.openssl_md_meth_names
+
     def test_simple(self):
         import _hashlib
         assert _hashlib.new('md5').__class__.__name__ == 'HASH'
diff --git a/pypy/rlib/ropenssl.py b/pypy/rlib/ropenssl.py
--- a/pypy/rlib/ropenssl.py
+++ b/pypy/rlib/ropenssl.py
@@ -89,6 +89,9 @@
 
     CRYPTO_LOCK = rffi_platform.ConstantInteger("CRYPTO_LOCK")
 
+    OBJ_NAME_TYPE_MD_METH = rffi_platform.ConstantInteger(
+        "OBJ_NAME_TYPE_MD_METH")
+
     # Some structures, with only the fields used in the _ssl module
     X509_name_entry_st = rffi_platform.Struct('struct X509_name_entry_st',
                                               [('set', rffi.INT)])
@@ -110,6 +113,12 @@
         [('type', rffi.INT),
          ]) 
 
+    OBJ_NAME_st = rffi_platform.Struct(
+        'OBJ_NAME',
+        [('alias', rffi.INT),
+         ('name', rffi.CCHARP),
+         ]) 
+
 
 for k, v in rffi_platform.configure(CConfig).items():
     globals()[k] = v
@@ -130,6 +139,7 @@
 ASN1_INTEGER = rffi.COpaquePtr('ASN1_INTEGER')
 GENERAL_NAMES = rffi.COpaquePtr('GENERAL_NAMES')
 GENERAL_NAME = rffi.CArrayPtr(GENERAL_NAME_st)
+OBJ_NAME = rffi.CArrayPtr(OBJ_NAME_st)
 
 HAVE_OPENSSL_RAND = OPENSSL_VERSION_NUMBER >= 0x0090500f
 
@@ -276,6 +286,11 @@
 EVP_MD_CTX_cleanup = external(
     'EVP_MD_CTX_cleanup', [EVP_MD_CTX], rffi.INT, threadsafe=False)
 
+OBJ_NAME_CALLBACK = lltype.Ptr(lltype.FuncType(
+        [OBJ_NAME, rffi.VOIDP], lltype.Void))
+OBJ_NAME_do_all = external(
+    'OBJ_NAME_do_all', [rffi.INT, OBJ_NAME_CALLBACK, rffi.VOIDP], lltype.Void)
+
 def init_ssl():
     libssl_SSL_load_error_strings()
     libssl_SSL_library_init()

From noreply at buildbot.pypy.org  Sat Oct 22 00:28:42 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sat, 22 Oct 2011 00:28:42 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Fixes for the hashlib module
Message-ID: <20111021222842.3EB45820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48334:7ab59b0b6fc2
Date: 2011-10-22 00:24 +0200
http://bitbucket.org/pypy/pypy/changeset/7ab59b0b6fc2/

Log:	Fixes for the hashlib module

diff --git a/lib_pypy/_sha.py b/lib_pypy/_sha.py
deleted file mode 100644
--- a/lib_pypy/_sha.py
+++ /dev/null
@@ -1,352 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: iso-8859-1
-
-# Note that PyPy contains also a built-in module 'sha' which will hide
-# this one if compiled in.
-
-"""A sample implementation of SHA-1 in pure Python.
-
-   Framework adapted from Dinu Gherman's MD5 implementation by
-   J. Hall&#65533;n and L. Creighton. SHA-1 implementation based directly on
-   the text of the NIST standard FIPS PUB 180-1.
-"""
-
-
-__date__    = '2004-11-17'
-__version__ = 0.91 # Modernised by J. Hall&#65533;n and L. Creighton for Pypy
-
-
-import struct, copy
-
-
-# ======================================================================
-# Bit-Manipulation helpers
-#
-#   _long2bytes() was contributed by Barry Warsaw
-#   and is reused here with tiny modifications.
-# ======================================================================
-
-def _long2bytesBigEndian(n, blocksize=0):
-    """Convert a long integer to a byte string.
-
-    If optional blocksize is given and greater than zero, pad the front
-    of the byte string with binary zeros so that the length is a multiple
-    of blocksize.
-    """
-
-    # After much testing, this algorithm was deemed to be the fastest.
-    s = ''
-    pack = struct.pack
-    while n > 0:
-        s = pack('>I', n & 0xffffffffL) + s
-        n = n >> 32
-
-    # Strip off leading zeros.
-    for i in range(len(s)):
-        if s[i] <> '\000':
-            break
-    else:
-        # Only happens when n == 0.
-        s = '\000'
-        i = 0
-
-    s = s[i:]
-
-    # Add back some pad bytes. This could be done more efficiently
-    # w.r.t. the de-padding being done above, but sigh...
-    if blocksize > 0 and len(s) % blocksize:
-        s = (blocksize - len(s) % blocksize) * '\000' + s
-
-    return s
-
-
-def _bytelist2longBigEndian(list):
-    "Transform a list of characters into a list of longs."
-
-    imax = len(list)/4
-    hl = [0L] * imax
-
-    j = 0
-    i = 0
-    while i < imax:
-        b0 = long(ord(list[j])) << 24
-        b1 = long(ord(list[j+1])) << 16
-        b2 = long(ord(list[j+2])) << 8
-        b3 = long(ord(list[j+3]))
-        hl[i] = b0 | b1 | b2 | b3
-        i = i+1
-        j = j+4
-
-    return hl
-
-
-def _rotateLeft(x, n):
-    "Rotate x (32 bit) left n bits circularly."
-
-    return (x << n) | (x >> (32-n))
-
-
-# ======================================================================
-# The SHA transformation functions
-#
-# ======================================================================
-
-def f0_19(B, C, D):
-    return (B & C) | ((~ B) & D)
-
-def f20_39(B, C, D):
-    return B ^ C ^ D
-
-def f40_59(B, C, D):
-    return (B & C) | (B & D) | (C & D)
-
-def f60_79(B, C, D):
-    return B ^ C ^ D
-
-
-f = [f0_19, f20_39, f40_59, f60_79]
-
-# Constants to be used
-K = [
-    0x5A827999L, # ( 0 <= t <= 19)
-    0x6ED9EBA1L, # (20 <= t <= 39)
-    0x8F1BBCDCL, # (40 <= t <= 59)
-    0xCA62C1D6L  # (60 <= t <= 79)
-    ]
-
-class sha:
-    "An implementation of the MD5 hash function in pure Python."
-
-    digest_size = digestsize = 20
-    block_size = 1
-
-    def __init__(self):
-        "Initialisation."
-        
-        # Initial message length in bits(!).
-        self.length = 0L
-        self.count = [0, 0]
-
-        # Initial empty message as a sequence of bytes (8 bit characters).
-        self.input = []
-
-        # Call a separate init function, that can be used repeatedly
-        # to start from scratch on the same object.
-        self.init()
-
-
-    def init(self):
-        "Initialize the message-digest and set all fields to zero."
-
-        self.length = 0L
-        self.input = []
-
-        # Initial 160 bit message digest (5 times 32 bit).
-        self.H0 = 0x67452301L
-        self.H1 = 0xEFCDAB89L
-        self.H2 = 0x98BADCFEL
-        self.H3 = 0x10325476L
-        self.H4 = 0xC3D2E1F0L
-
-    def _transform(self, W):
-
-        for t in range(16, 80):
-            W.append(_rotateLeft(
-                W[t-3] ^ W[t-8] ^ W[t-14] ^ W[t-16], 1) & 0xffffffffL)
-
-        A = self.H0
-        B = self.H1
-        C = self.H2
-        D = self.H3
-        E = self.H4
-
-        """
-        This loop was unrolled to gain about 10% in speed
-        for t in range(0, 80):
-            TEMP = _rotateLeft(A, 5) + f[t/20] + E + W[t] + K[t/20]
-            E = D
-            D = C
-            C = _rotateLeft(B, 30) & 0xffffffffL
-            B = A
-            A = TEMP & 0xffffffffL
-        """
-
-        for t in range(0, 20):
-            TEMP = _rotateLeft(A, 5) + ((B & C) | ((~ B) & D)) + E + W[t] + K[0]
-            E = D
-            D = C
-            C = _rotateLeft(B, 30) & 0xffffffffL
-            B = A
-            A = TEMP & 0xffffffffL
-
-        for t in range(20, 40):
-            TEMP = _rotateLeft(A, 5) + (B ^ C ^ D) + E + W[t] + K[1]
-            E = D
-            D = C
-            C = _rotateLeft(B, 30) & 0xffffffffL
-            B = A
-            A = TEMP & 0xffffffffL
-
-        for t in range(40, 60):
-            TEMP = _rotateLeft(A, 5) + ((B & C) | (B & D) | (C & D)) + E + W[t] + K[2]
-            E = D
-            D = C
-            C = _rotateLeft(B, 30) & 0xffffffffL
-            B = A
-            A = TEMP & 0xffffffffL
-
-        for t in range(60, 80):
-            TEMP = _rotateLeft(A, 5) + (B ^ C ^ D)  + E + W[t] + K[3]
-            E = D
-            D = C
-            C = _rotateLeft(B, 30) & 0xffffffffL
-            B = A
-            A = TEMP & 0xffffffffL
-
-
-        self.H0 = (self.H0 + A) & 0xffffffffL
-        self.H1 = (self.H1 + B) & 0xffffffffL
-        self.H2 = (self.H2 + C) & 0xffffffffL
-        self.H3 = (self.H3 + D) & 0xffffffffL
-        self.H4 = (self.H4 + E) & 0xffffffffL
-    
-
-    # Down from here all methods follow the Python Standard Library
-    # API of the sha module.
-
-    def update(self, inBuf):
-        """Add to the current message.
-
-        Update the md5 object with the string arg. Repeated calls
-        are equivalent to a single call with the concatenation of all
-        the arguments, i.e. m.update(a); m.update(b) is equivalent
-        to m.update(a+b).
-
-        The hash is immediately calculated for all full blocks. The final
-        calculation is made in digest(). It will calculate 1-2 blocks,
-        depending on how much padding we have to add. This allows us to
-        keep an intermediate value for the hash, so that we only need to
-        make minimal recalculation if we call update() to add more data
-        to the hashed string.
-        """
-
-        leninBuf = long(len(inBuf))
-
-        # Compute number of bytes mod 64.
-        index = (self.count[1] >> 3) & 0x3FL
-
-        # Update number of bits.
-        self.count[1] = self.count[1] + (leninBuf << 3)
-        if self.count[1] < (leninBuf << 3):
-            self.count[0] = self.count[0] + 1
-        self.count[0] = self.count[0] + (leninBuf >> 29)
-
-        partLen = 64 - index
-
-        if leninBuf >= partLen:
-            self.input[index:] = list(inBuf[:partLen])
-            self._transform(_bytelist2longBigEndian(self.input))
-            i = partLen
-            while i + 63 < leninBuf:
-                self._transform(_bytelist2longBigEndian(list(inBuf[i:i+64])))
-                i = i + 64
-            else:
-                self.input = list(inBuf[i:leninBuf])
-        else:
-            i = 0
-            self.input = self.input + list(inBuf)
-
-
-    def digest(self):
-        """Terminate the message-digest computation and return digest.
-
-        Return the digest of the strings passed to the update()
-        method so far. This is a 16-byte string which may contain
-        non-ASCII characters, including null bytes.
-        """
-
-        H0 = self.H0
-        H1 = self.H1
-        H2 = self.H2
-        H3 = self.H3
-        H4 = self.H4
-        input = [] + self.input
-        count = [] + self.count
-
-        index = (self.count[1] >> 3) & 0x3fL
-
-        if index < 56:
-            padLen = 56 - index
-        else:
-            padLen = 120 - index
-
-        padding = ['\200'] + ['\000'] * 63
-        self.update(padding[:padLen])
-
-        # Append length (before padding).
-        bits = _bytelist2longBigEndian(self.input[:56]) + count
-
-        self._transform(bits)
-
-        # Store state in digest.
-        digest = _long2bytesBigEndian(self.H0, 4) + \
-                 _long2bytesBigEndian(self.H1, 4) + \
-                 _long2bytesBigEndian(self.H2, 4) + \
-                 _long2bytesBigEndian(self.H3, 4) + \
-                 _long2bytesBigEndian(self.H4, 4)
-
-        self.H0 = H0 
-        self.H1 = H1 
-        self.H2 = H2
-        self.H3 = H3
-        self.H4 = H4
-        self.input = input 
-        self.count = count 
-
-        return digest
-
-
-    def hexdigest(self):
-        """Terminate and return digest in HEX form.
-
-        Like digest() except the digest is returned as a string of
-        length 32, containing only hexadecimal digits. This may be
-        used to exchange the value safely in email or other non-
-        binary environments.
-        """
-        return ''.join(['%02x' % ord(c) for c in self.digest()])
-
-    def copy(self):
-        """Return a clone object.
-
-        Return a copy ('clone') of the md5 object. This can be used
-        to efficiently compute the digests of strings that share
-        a common initial substring.
-        """
-
-        return copy.deepcopy(self)
-
-
-# ======================================================================
-# Mimic Python top-level functions from standard library API
-# for consistency with the _sha module of the standard library.
-# ======================================================================
-
-# These are mandatory variables in the module. They have constant values
-# in the SHA standard.
-
-digest_size = 20
-digestsize = 20
-blocksize = 1
-
-def new(arg=None):
-    """Return a new sha crypto object.
-
-    If arg is present, the method call update(arg) is made.
-    """
-
-    crypto = sha()
-    if arg:
-        crypto.update(arg)
-
-    return crypto
diff --git a/lib_pypy/_sha1.py b/lib_pypy/_sha1.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_sha1.py
@@ -0,0 +1,352 @@
+#!/usr/bin/env python
+# -*- coding: iso-8859-1
+
+# Note that PyPy contains also a built-in module 'sha' which will hide
+# this one if compiled in.
+
+"""A sample implementation of SHA-1 in pure Python.
+
+   Framework adapted from Dinu Gherman's MD5 implementation by
+   J. Hall&#65533;n and L. Creighton. SHA-1 implementation based directly on
+   the text of the NIST standard FIPS PUB 180-1.
+"""
+
+
+__date__    = '2004-11-17'
+__version__ = 0.91 # Modernised by J. Hall&#65533;n and L. Creighton for Pypy
+
+
+import struct, copy
+
+
+# ======================================================================
+# Bit-Manipulation helpers
+#
+#   _long2bytes() was contributed by Barry Warsaw
+#   and is reused here with tiny modifications.
+# ======================================================================
+
+def _long2bytesBigEndian(n, blocksize=0):
+    """Convert a long integer to a byte string.
+
+    If optional blocksize is given and greater than zero, pad the front
+    of the byte string with binary zeros so that the length is a multiple
+    of blocksize.
+    """
+
+    # After much testing, this algorithm was deemed to be the fastest.
+    s = ''
+    pack = struct.pack
+    while n > 0:
+        s = pack('>I', n & 0xffffffffL) + s
+        n = n >> 32
+
+    # Strip off leading zeros.
+    for i in range(len(s)):
+        if s[i] <> '\000':
+            break
+    else:
+        # Only happens when n == 0.
+        s = '\000'
+        i = 0
+
+    s = s[i:]
+
+    # Add back some pad bytes. This could be done more efficiently
+    # w.r.t. the de-padding being done above, but sigh...
+    if blocksize > 0 and len(s) % blocksize:
+        s = (blocksize - len(s) % blocksize) * '\000' + s
+
+    return s
+
+
+def _bytelist2longBigEndian(list):
+    "Transform a list of characters into a list of longs."
+
+    imax = len(list)/4
+    hl = [0L] * imax
+
+    j = 0
+    i = 0
+    while i < imax:
+        b0 = long(ord(list[j])) << 24
+        b1 = long(ord(list[j+1])) << 16
+        b2 = long(ord(list[j+2])) << 8
+        b3 = long(ord(list[j+3]))
+        hl[i] = b0 | b1 | b2 | b3
+        i = i+1
+        j = j+4
+
+    return hl
+
+
+def _rotateLeft(x, n):
+    "Rotate x (32 bit) left n bits circularly."
+
+    return (x << n) | (x >> (32-n))
+
+
+# ======================================================================
+# The SHA transformation functions
+#
+# ======================================================================
+
+def f0_19(B, C, D):
+    return (B & C) | ((~ B) & D)
+
+def f20_39(B, C, D):
+    return B ^ C ^ D
+
+def f40_59(B, C, D):
+    return (B & C) | (B & D) | (C & D)
+
+def f60_79(B, C, D):
+    return B ^ C ^ D
+
+
+f = [f0_19, f20_39, f40_59, f60_79]
+
+# Constants to be used
+K = [
+    0x5A827999L, # ( 0 <= t <= 19)
+    0x6ED9EBA1L, # (20 <= t <= 39)
+    0x8F1BBCDCL, # (40 <= t <= 59)
+    0xCA62C1D6L  # (60 <= t <= 79)
+    ]
+
+class sha:
+    "An implementation of the MD5 hash function in pure Python."
+
+    digest_size = digestsize = 20
+    block_size = 1
+
+    def __init__(self):
+        "Initialisation."
+        
+        # Initial message length in bits(!).
+        self.length = 0L
+        self.count = [0, 0]
+
+        # Initial empty message as a sequence of bytes (8 bit characters).
+        self.input = []
+
+        # Call a separate init function, that can be used repeatedly
+        # to start from scratch on the same object.
+        self.init()
+
+
+    def init(self):
+        "Initialize the message-digest and set all fields to zero."
+
+        self.length = 0L
+        self.input = []
+
+        # Initial 160 bit message digest (5 times 32 bit).
+        self.H0 = 0x67452301L
+        self.H1 = 0xEFCDAB89L
+        self.H2 = 0x98BADCFEL
+        self.H3 = 0x10325476L
+        self.H4 = 0xC3D2E1F0L
+
+    def _transform(self, W):
+
+        for t in range(16, 80):
+            W.append(_rotateLeft(
+                W[t-3] ^ W[t-8] ^ W[t-14] ^ W[t-16], 1) & 0xffffffffL)
+
+        A = self.H0
+        B = self.H1
+        C = self.H2
+        D = self.H3
+        E = self.H4
+
+        """
+        This loop was unrolled to gain about 10% in speed
+        for t in range(0, 80):
+            TEMP = _rotateLeft(A, 5) + f[t/20] + E + W[t] + K[t/20]
+            E = D
+            D = C
+            C = _rotateLeft(B, 30) & 0xffffffffL
+            B = A
+            A = TEMP & 0xffffffffL
+        """
+
+        for t in range(0, 20):
+            TEMP = _rotateLeft(A, 5) + ((B & C) | ((~ B) & D)) + E + W[t] + K[0]
+            E = D
+            D = C
+            C = _rotateLeft(B, 30) & 0xffffffffL
+            B = A
+            A = TEMP & 0xffffffffL
+
+        for t in range(20, 40):
+            TEMP = _rotateLeft(A, 5) + (B ^ C ^ D) + E + W[t] + K[1]
+            E = D
+            D = C
+            C = _rotateLeft(B, 30) & 0xffffffffL
+            B = A
+            A = TEMP & 0xffffffffL
+
+        for t in range(40, 60):
+            TEMP = _rotateLeft(A, 5) + ((B & C) | (B & D) | (C & D)) + E + W[t] + K[2]
+            E = D
+            D = C
+            C = _rotateLeft(B, 30) & 0xffffffffL
+            B = A
+            A = TEMP & 0xffffffffL
+
+        for t in range(60, 80):
+            TEMP = _rotateLeft(A, 5) + (B ^ C ^ D)  + E + W[t] + K[3]
+            E = D
+            D = C
+            C = _rotateLeft(B, 30) & 0xffffffffL
+            B = A
+            A = TEMP & 0xffffffffL
+
+
+        self.H0 = (self.H0 + A) & 0xffffffffL
+        self.H1 = (self.H1 + B) & 0xffffffffL
+        self.H2 = (self.H2 + C) & 0xffffffffL
+        self.H3 = (self.H3 + D) & 0xffffffffL
+        self.H4 = (self.H4 + E) & 0xffffffffL
+    
+
+    # Down from here all methods follow the Python Standard Library
+    # API of the sha module.
+
+    def update(self, inBuf):
+        """Add to the current message.
+
+        Update the md5 object with the string arg. Repeated calls
+        are equivalent to a single call with the concatenation of all
+        the arguments, i.e. m.update(a); m.update(b) is equivalent
+        to m.update(a+b).
+
+        The hash is immediately calculated for all full blocks. The final
+        calculation is made in digest(). It will calculate 1-2 blocks,
+        depending on how much padding we have to add. This allows us to
+        keep an intermediate value for the hash, so that we only need to
+        make minimal recalculation if we call update() to add more data
+        to the hashed string.
+        """
+
+        leninBuf = long(len(inBuf))
+
+        # Compute number of bytes mod 64.
+        index = (self.count[1] >> 3) & 0x3FL
+
+        # Update number of bits.
+        self.count[1] = self.count[1] + (leninBuf << 3)
+        if self.count[1] < (leninBuf << 3):
+            self.count[0] = self.count[0] + 1
+        self.count[0] = self.count[0] + (leninBuf >> 29)
+
+        partLen = 64 - index
+
+        if leninBuf >= partLen:
+            self.input[index:] = list(inBuf[:partLen])
+            self._transform(_bytelist2longBigEndian(self.input))
+            i = partLen
+            while i + 63 < leninBuf:
+                self._transform(_bytelist2longBigEndian(list(inBuf[i:i+64])))
+                i = i + 64
+            else:
+                self.input = list(inBuf[i:leninBuf])
+        else:
+            i = 0
+            self.input = self.input + list(inBuf)
+
+
+    def digest(self):
+        """Terminate the message-digest computation and return digest.
+
+        Return the digest of the strings passed to the update()
+        method so far. This is a 16-byte string which may contain
+        non-ASCII characters, including null bytes.
+        """
+
+        H0 = self.H0
+        H1 = self.H1
+        H2 = self.H2
+        H3 = self.H3
+        H4 = self.H4
+        input = [] + self.input
+        count = [] + self.count
+
+        index = (self.count[1] >> 3) & 0x3fL
+
+        if index < 56:
+            padLen = 56 - index
+        else:
+            padLen = 120 - index
+
+        padding = ['\200'] + ['\000'] * 63
+        self.update(padding[:padLen])
+
+        # Append length (before padding).
+        bits = _bytelist2longBigEndian(self.input[:56]) + count
+
+        self._transform(bits)
+
+        # Store state in digest.
+        digest = _long2bytesBigEndian(self.H0, 4) + \
+                 _long2bytesBigEndian(self.H1, 4) + \
+                 _long2bytesBigEndian(self.H2, 4) + \
+                 _long2bytesBigEndian(self.H3, 4) + \
+                 _long2bytesBigEndian(self.H4, 4)
+
+        self.H0 = H0 
+        self.H1 = H1 
+        self.H2 = H2
+        self.H3 = H3
+        self.H4 = H4
+        self.input = input 
+        self.count = count 
+
+        return digest
+
+
+    def hexdigest(self):
+        """Terminate and return digest in HEX form.
+
+        Like digest() except the digest is returned as a string of
+        length 32, containing only hexadecimal digits. This may be
+        used to exchange the value safely in email or other non-
+        binary environments.
+        """
+        return ''.join(['%02x' % ord(c) for c in self.digest()])
+
+    def copy(self):
+        """Return a clone object.
+
+        Return a copy ('clone') of the md5 object. This can be used
+        to efficiently compute the digests of strings that share
+        a common initial substring.
+        """
+
+        return copy.deepcopy(self)
+
+
+# ======================================================================
+# Mimic Python top-level functions from standard library API
+# for consistency with the _sha module of the standard library.
+# ======================================================================
+
+# These are mandatory variables in the module. They have constant values
+# in the SHA standard.
+
+digest_size = 20
+digestsize = 20
+blocksize = 1
+
+def sha1(arg=None):
+    """Return a new sha crypto object.
+
+    If arg is present, the method call update(arg) is made.
+    """
+
+    crypto = sha()
+    if arg:
+        crypto.update(arg)
+
+    return crypto
diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py
--- a/pypy/module/_hashlib/interp_hashlib.py
+++ b/pypy/module/_hashlib/interp_hashlib.py
@@ -100,7 +100,7 @@
     def digest(self, space):
         "Return the digest value as a string of binary data."
         digest = self._digest(space)
-        return space.wrap(digest)
+        return space.wrapbytes(digest)
 
     def hexdigest(self, space):
         "Return the digest value as a string of hexadecimal digits."
diff --git a/pypy/module/_hashlib/test/test_hashlib.py b/pypy/module/_hashlib/test/test_hashlib.py
--- a/pypy/module/_hashlib/test/test_hashlib.py
+++ b/pypy/module/_hashlib/test/test_hashlib.py
@@ -28,13 +28,13 @@
             assert h.digest_size == expected_size
             assert h.digestsize == expected_size
             #
-            h.update('abc')
+            h.update(b'abc')
             h2 = h.copy()
-            h.update('def')
+            h.update(b'def')
             digest = h.digest()
             hexdigest = h.hexdigest()
-            h2.update('d')
-            h2.update('ef')
+            h2.update(b'd')
+            h2.update(b'ef')
             assert digest    == h2.digest()
             assert hexdigest == h2.hexdigest()
             assert len(digest)    == h.digest_size
@@ -66,10 +66,6 @@
         import hashlib
         assert repr(hashlib.md5()).startswith("<md5 HASH object")
 
-    def test_unicode(self):
-        import _hashlib
-        assert _hashlib.new('sha1', u'xxx').__class__.__name__ == 'HASH'
-
     def test_uppercase(self):
         import _hashlib
         h = _hashlib.new('MD5')
@@ -78,9 +74,9 @@
 
     def test_buffer(self):
         import _hashlib, array
-        b = array.array('b', 'x' * 10)
+        b = array.array('b', b'x' * 10)
         h = _hashlib.new('md5', b)
         h.update(b)
-        assert h.digest() == _hashlib.openssl_md5('x' * 20).digest()
+        assert h.digest() == _hashlib.openssl_md5(b'x' * 20).digest()
         _hashlib.openssl_sha1(b).digest()
 
diff --git a/pypy/module/_sha/__init__.py b/pypy/module/_sha/__init__.py
--- a/pypy/module/_sha/__init__.py
+++ b/pypy/module/_sha/__init__.py
@@ -18,12 +18,10 @@
 the concatenation of the strings fed to it so far. SHA-1 digests are 160
 bits instead of MD5's 128 bits."""
 
+    applevel_name = '_sha1'
+
     interpleveldefs = {
-        'new': 'interp_sha.W_SHA',
-        'SHAType': 'interp_sha.W_SHA',
-        'blocksize': 'space.wrap(1)',
-        'digest_size': 'space.wrap(20)',
-        'digestsize': 'space.wrap(20)',
+        'sha1': 'interp_sha.W_SHA',
         }
 
     appleveldefs = {

From noreply at buildbot.pypy.org  Sat Oct 22 00:28:43 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sat, 22 Oct 2011 00:28:43 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: This tweak is not necessary anymore
Message-ID: <20111021222843.6D9F0820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48335:643e31c7752c
Date: 2011-10-22 00:24 +0200
http://bitbucket.org/pypy/pypy/changeset/643e31c7752c/

Log:	This tweak is not necessary anymore

diff --git a/lib-python/3.2/tokenize.py b/lib-python/3.2/tokenize.py
--- a/lib-python/3.2/tokenize.py
+++ b/lib-python/3.2/tokenize.py
@@ -30,7 +30,7 @@
 from token import *
 from codecs import lookup, BOM_UTF8
 import collections
-import io
+from io import TextIOWrapper
 cookie_re = re.compile("coding[:=]\s*([-\w.]+)")
 
 import token
@@ -340,10 +340,10 @@
     """Open a file in read only mode using the encoding detected by
     detect_encoding().
     """
-    buffer = io.open(filename, 'rb')
+    buffer = builtins.open(filename, 'rb')
     encoding, lines = detect_encoding(buffer.readline)
     buffer.seek(0)
-    text = io.TextIOWrapper(buffer, encoding, line_buffering=True)
+    text = TextIOWrapper(buffer, encoding, line_buffering=True)
     text.mode = 'r'
     return text
 

From noreply at buildbot.pypy.org  Sat Oct 22 00:28:44 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sat, 22 Oct 2011 00:28:44 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Reduce the list of non-working modules
Message-ID: <20111021222844.9E7D1820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48336:56294dacd7ca
Date: 2011-10-22 00:24 +0200
http://bitbucket.org/pypy/pypy/changeset/56294dacd7ca/

Log:	Reduce the list of non-working modules

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -37,16 +37,10 @@
      "_continuation"]
 ))
 
-# XXX Here is the list of modules not known to work yet
-for name in [
-    "mmap", "_locale", "pwd",
-    "zipimport", "_lsprof",
-    "crypt", "_rawffi", "termios", "zlib", "bz2",
-    "_hashlib", "_md5", "_sha", "_minimal_curses", "cStringIO",
-    "thread", "itertools", "pyexpat", "_ssl", "cpyext",
-    "_bisect", "_multiprocessing",
-    "_collections", "_multibytecodec", "micronumpy", "_ffi",
-    "_continuation"]:
+# Here is the list of modules known to not work yet
+for name in ["_rawffi", "_ffi", "cpyext",
+             "_hashlib", "_md5", "_sha",
+             ]:
     del working_modules[name]
 
 translation_modules = default_modules.copy()

From noreply at buildbot.pypy.org  Sat Oct 22 00:28:46 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sat, 22 Oct 2011 00:28:46 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Merge default
Message-ID: <20111021222846.76845820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48337:e53be0abbf48
Date: 2011-10-22 00:27 +0200
http://bitbucket.org/pypy/pypy/changeset/e53be0abbf48/

Log:	Merge default

diff --git a/lib-python/modified-2.7/json/encoder.py b/lib-python/modified-2.7/json/encoder.py
--- a/lib-python/modified-2.7/json/encoder.py
+++ b/lib-python/modified-2.7/json/encoder.py
@@ -2,14 +2,7 @@
 """
 import re
 
-try:
-    from _json import encode_basestring_ascii as c_encode_basestring_ascii
-except ImportError:
-    c_encode_basestring_ascii = None
-try:
-    from _json import make_encoder as c_make_encoder
-except ImportError:
-    c_make_encoder = None
+from __pypy__.builders import StringBuilder, UnicodeBuilder
 
 ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]')
 ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])')
@@ -24,8 +17,7 @@
     '\t': '\\t',
 }
 for i in range(0x20):
-    ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i))
-    #ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,))
+    ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,))
 
 # Assume this produces an infinity on all machines (probably not guaranteed)
 INFINITY = float('1e66666')
@@ -37,10 +29,9 @@
     """
     def replace(match):
         return ESCAPE_DCT[match.group(0)]
-    return '"' + ESCAPE.sub(replace, s) + '"'
+    return ESCAPE.sub(replace, s)
 
-
-def py_encode_basestring_ascii(s):
+def encode_basestring_ascii(s):
     """Return an ASCII-only JSON representation of a Python string
 
     """
@@ -53,20 +44,18 @@
         except KeyError:
             n = ord(s)
             if n < 0x10000:
-                return '\\u{0:04x}'.format(n)
-                #return '\\u%04x' % (n,)
+                return '\\u%04x' % (n,)
             else:
                 # surrogate pair
                 n -= 0x10000
                 s1 = 0xd800 | ((n >> 10) & 0x3ff)
                 s2 = 0xdc00 | (n & 0x3ff)
-                return '\\u{0:04x}\\u{1:04x}'.format(s1, s2)
-                #return '\\u%04x\\u%04x' % (s1, s2)
-    return '"' + str(ESCAPE_ASCII.sub(replace, s)) + '"'
-
-
-encode_basestring_ascii = (
-    c_encode_basestring_ascii or py_encode_basestring_ascii)
+                return '\\u%04x\\u%04x' % (s1, s2)
+    if ESCAPE_ASCII.search(s):
+        return str(ESCAPE_ASCII.sub(replace, s))
+    return s
+py_encode_basestring_ascii = lambda s: '"' + encode_basestring_ascii(s) + '"'
+c_encode_basestring_ascii = None
 
 class JSONEncoder(object):
     """Extensible JSON <http://json.org> encoder for Python data structures.
@@ -147,6 +136,17 @@
 
         self.skipkeys = skipkeys
         self.ensure_ascii = ensure_ascii
+        if ensure_ascii:
+            self.encoder = encode_basestring_ascii
+        else:
+            self.encoder = encode_basestring
+        if encoding != 'utf-8':
+            orig_encoder = self.encoder
+            def encoder(o):
+                if isinstance(o, str):
+                    o = o.decode(encoding)
+                return orig_encoder(o)
+            self.encoder = encoder
         self.check_circular = check_circular
         self.allow_nan = allow_nan
         self.sort_keys = sort_keys
@@ -184,24 +184,126 @@
         '{"foo": ["bar", "baz"]}'
 
         """
-        # This is for extremely simple cases and benchmarks.
+        if self.check_circular:
+            markers = {}
+        else:
+            markers = None
+        if self.ensure_ascii:
+            builder = StringBuilder()
+        else:
+            builder = UnicodeBuilder()
+        self._encode(o, markers, builder, 0)
+        return builder.build()
+
+    def _emit_indent(self, builder, _current_indent_level):
+        if self.indent is not None:
+            _current_indent_level += 1
+            newline_indent = '\n' + (' ' * (self.indent *
+                                            _current_indent_level))
+            separator = self.item_separator + newline_indent
+            builder.append(newline_indent)
+        else:
+            separator = self.item_separator
+        return separator, _current_indent_level
+
+    def _emit_unindent(self, builder, _current_indent_level):
+        if self.indent is not None:
+            builder.append('\n')
+            builder.append(' ' * (self.indent * (_current_indent_level - 1)))
+
+    def _encode(self, o, markers, builder, _current_indent_level):
         if isinstance(o, basestring):
-            if isinstance(o, str):
-                _encoding = self.encoding
-                if (_encoding is not None
-                        and not (_encoding == 'utf-8')):
-                    o = o.decode(_encoding)
-            if self.ensure_ascii:
-                return encode_basestring_ascii(o)
+            builder.append('"')
+            builder.append(self.encoder(o))
+            builder.append('"')
+        elif o is None:
+            builder.append('null')
+        elif o is True:
+            builder.append('true')
+        elif o is False:
+            builder.append('false')
+        elif isinstance(o, (int, long)):
+            builder.append(str(o))
+        elif isinstance(o, float):
+            builder.append(self._floatstr(o))
+        elif isinstance(o, (list, tuple)):
+            if not o:
+                builder.append('[]')
+                return
+            self._encode_list(o, markers, builder, _current_indent_level)
+        elif isinstance(o, dict):
+            if not o:
+                builder.append('{}')
+                return
+            self._encode_dict(o, markers, builder, _current_indent_level)
+        else:
+            self._mark_markers(markers, o)
+            res = self.default(o)
+            self._encode(res, markers, builder, _current_indent_level)
+            self._remove_markers(markers, o)
+            return res
+
+    def _encode_list(self, l, markers, builder, _current_indent_level):
+        self._mark_markers(markers, l)
+        builder.append('[')
+        first = True
+        separator, _current_indent_level = self._emit_indent(builder,
+                                                      _current_indent_level)
+        for elem in l:
+            if first:
+                first = False
             else:
-                return encode_basestring(o)
-        # This doesn't pass the iterator directly to ''.join() because the
-        # exceptions aren't as detailed.  The list call should be roughly
-        # equivalent to the PySequence_Fast that ''.join() would do.
-        chunks = self.iterencode(o, _one_shot=True)
-        if not isinstance(chunks, (list, tuple)):
-            chunks = list(chunks)
-        return ''.join(chunks)
+                builder.append(separator)
+            self._encode(elem, markers, builder, _current_indent_level)
+            del elem # XXX grumble
+        self._emit_unindent(builder, _current_indent_level)
+        builder.append(']')
+        self._remove_markers(markers, l)
+
+    def _encode_dict(self, d, markers, builder, _current_indent_level):
+        self._mark_markers(markers, d)
+        first = True
+        builder.append('{')
+        separator, _current_indent_level = self._emit_indent(builder,
+                                                         _current_indent_level)
+        if self.sort_keys:
+            items = sorted(d.items(), key=lambda kv: kv[0])
+        else:
+            items = d.iteritems()
+
+        for key, v in items:
+            if first:
+                first = False
+            else:
+                builder.append(separator)
+            if isinstance(key, basestring):
+                pass
+            # JavaScript is weakly typed for these, so it makes sense to
+            # also allow them.  Many encoders seem to do something like this.
+            elif isinstance(key, float):
+                key = self._floatstr(key)
+            elif key is True:
+                key = 'true'
+            elif key is False:
+                key = 'false'
+            elif key is None:
+                key = 'null'
+            elif isinstance(key, (int, long)):
+                key = str(key)
+            elif self.skipkeys:
+                continue
+            else:
+                raise TypeError("key " + repr(key) + " is not a string")
+            builder.append('"')
+            builder.append(self.encoder(key))
+            builder.append('"')
+            builder.append(self.key_separator)
+            self._encode(v, markers, builder, _current_indent_level)
+            del key
+            del v # XXX grumble
+        self._emit_unindent(builder, _current_indent_level)
+        builder.append('}')
+        self._remove_markers(markers, d)
 
     def iterencode(self, o, _one_shot=False):
         """Encode the given object and yield each string
@@ -217,86 +319,54 @@
             markers = {}
         else:
             markers = None
-        if self.ensure_ascii:
-            _encoder = encode_basestring_ascii
+        return self._iterencode(o, markers, 0)
+
+    def _floatstr(self, o):
+        # Check for specials.  Note that this type of test is processor
+        # and/or platform-specific, so do tests which don't depend on the
+        # internals.
+
+        if o != o:
+            text = 'NaN'
+        elif o == INFINITY:
+            text = 'Infinity'
+        elif o == -INFINITY:
+            text = '-Infinity'
         else:
-            _encoder = encode_basestring
-        if self.encoding != 'utf-8':
-            def _encoder(o, _orig_encoder=_encoder, _encoding=self.encoding):
-                if isinstance(o, str):
-                    o = o.decode(_encoding)
-                return _orig_encoder(o)
+            return FLOAT_REPR(o)
 
-        def floatstr(o, allow_nan=self.allow_nan,
-                _repr=FLOAT_REPR, _inf=INFINITY, _neginf=-INFINITY):
-            # Check for specials.  Note that this type of test is processor
-            # and/or platform-specific, so do tests which don't depend on the
-            # internals.
+        if not self.allow_nan:
+            raise ValueError(
+                "Out of range float values are not JSON compliant: " +
+                repr(o))
 
-            if o != o:
-                text = 'NaN'
-            elif o == _inf:
-                text = 'Infinity'
-            elif o == _neginf:
-                text = '-Infinity'
-            else:
-                return _repr(o)
+        return text
 
-            if not allow_nan:
-                raise ValueError(
-                    "Out of range float values are not JSON compliant: " +
-                    repr(o))
+    def _mark_markers(self, markers, o):
+        if markers is not None:
+            if id(o) in markers:
+                raise ValueError("Circular reference detected")
+            markers[id(o)] = None
 
-            return text
+    def _remove_markers(self, markers, o):
+        if markers is not None:
+            del markers[id(o)]
 
-
-        if (_one_shot and c_make_encoder is not None
-                and not self.indent and not self.sort_keys):
-            _iterencode = c_make_encoder(
-                markers, self.default, _encoder, self.indent,
-                self.key_separator, self.item_separator, self.sort_keys,
-                self.skipkeys, self.allow_nan)
-        else:
-            _iterencode = _make_iterencode(
-                markers, self.default, _encoder, self.indent, floatstr,
-                self.key_separator, self.item_separator, self.sort_keys,
-                self.skipkeys, _one_shot)
-        return _iterencode(o, 0)
-
-def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,
-        _key_separator, _item_separator, _sort_keys, _skipkeys, _one_shot,
-        ## HACK: hand-optimized bytecode; turn globals into locals
-        ValueError=ValueError,
-        basestring=basestring,
-        dict=dict,
-        float=float,
-        id=id,
-        int=int,
-        isinstance=isinstance,
-        list=list,
-        long=long,
-        str=str,
-        tuple=tuple,
-    ):
-
-    def _iterencode_list(lst, _current_indent_level):
+    def _iterencode_list(self, lst, markers, _current_indent_level):
         if not lst:
             yield '[]'
             return
-        if markers is not None:
-            markerid = id(lst)
-            if markerid in markers:
-                raise ValueError("Circular reference detected")
-            markers[markerid] = lst
+        self._mark_markers(markers, lst)
         buf = '['
-        if _indent is not None:
+        if self.indent is not None:
             _current_indent_level += 1
-            newline_indent = '\n' + (' ' * (_indent * _current_indent_level))
-            separator = _item_separator + newline_indent
+            newline_indent = '\n' + (' ' * (self.indent *
+                                            _current_indent_level))
+            separator = self.item_separator + newline_indent
             buf += newline_indent
         else:
             newline_indent = None
-            separator = _item_separator
+            separator = self.item_separator
         first = True
         for value in lst:
             if first:
@@ -304,7 +374,7 @@
             else:
                 buf = separator
             if isinstance(value, basestring):
-                yield buf + _encoder(value)
+                yield buf + '"' + self.encoder(value) + '"'
             elif value is None:
                 yield buf + 'null'
             elif value is True:
@@ -314,44 +384,43 @@
             elif isinstance(value, (int, long)):
                 yield buf + str(value)
             elif isinstance(value, float):
-                yield buf + _floatstr(value)
+                yield buf + self._floatstr(value)
             else:
                 yield buf
                 if isinstance(value, (list, tuple)):
-                    chunks = _iterencode_list(value, _current_indent_level)
+                    chunks = self._iterencode_list(value, markers,
+                                                   _current_indent_level)
                 elif isinstance(value, dict):
-                    chunks = _iterencode_dict(value, _current_indent_level)
+                    chunks = self._iterencode_dict(value, markers,
+                                                   _current_indent_level)
                 else:
-                    chunks = _iterencode(value, _current_indent_level)
+                    chunks = self._iterencode(value, markers,
+                                              _current_indent_level)
                 for chunk in chunks:
                     yield chunk
         if newline_indent is not None:
             _current_indent_level -= 1
-            yield '\n' + (' ' * (_indent * _current_indent_level))
+            yield '\n' + (' ' * (self.indent * _current_indent_level))
         yield ']'
-        if markers is not None:
-            del markers[markerid]
+        self._remove_markers(markers, lst)
 
-    def _iterencode_dict(dct, _current_indent_level):
+    def _iterencode_dict(self, dct, markers, _current_indent_level):
         if not dct:
             yield '{}'
             return
-        if markers is not None:
-            markerid = id(dct)
-            if markerid in markers:
-                raise ValueError("Circular reference detected")
-            markers[markerid] = dct
+        self._mark_markers(markers, dct)
         yield '{'
-        if _indent is not None:
+        if self.indent is not None:
             _current_indent_level += 1
-            newline_indent = '\n' + (' ' * (_indent * _current_indent_level))
-            item_separator = _item_separator + newline_indent
+            newline_indent = '\n' + (' ' * (self.indent *
+                                            _current_indent_level))
+            item_separator = self.item_separator + newline_indent
             yield newline_indent
         else:
             newline_indent = None
-            item_separator = _item_separator
+            item_separator = self.item_separator
         first = True
-        if _sort_keys:
+        if self.sort_keys:
             items = sorted(dct.items(), key=lambda kv: kv[0])
         else:
             items = dct.iteritems()
@@ -361,7 +430,7 @@
             # JavaScript is weakly typed for these, so it makes sense to
             # also allow them.  Many encoders seem to do something like this.
             elif isinstance(key, float):
-                key = _floatstr(key)
+                key = self._floatstr(key)
             elif key is True:
                 key = 'true'
             elif key is False:
@@ -370,7 +439,7 @@
                 key = 'null'
             elif isinstance(key, (int, long)):
                 key = str(key)
-            elif _skipkeys:
+            elif self.skipkeys:
                 continue
             else:
                 raise TypeError("key " + repr(key) + " is not a string")
@@ -378,10 +447,10 @@
                 first = False
             else:
                 yield item_separator
-            yield _encoder(key)
-            yield _key_separator
+            yield '"' + self.encoder(key) + '"'
+            yield self.key_separator
             if isinstance(value, basestring):
-                yield _encoder(value)
+                yield '"' + self.encoder(value) + '"'
             elif value is None:
                 yield 'null'
             elif value is True:
@@ -391,26 +460,28 @@
             elif isinstance(value, (int, long)):
                 yield str(value)
             elif isinstance(value, float):
-                yield _floatstr(value)
+                yield self._floatstr(value)
             else:
                 if isinstance(value, (list, tuple)):
-                    chunks = _iterencode_list(value, _current_indent_level)
+                    chunks = self._iterencode_list(value, markers,
+                                                   _current_indent_level)
                 elif isinstance(value, dict):
-                    chunks = _iterencode_dict(value, _current_indent_level)
+                    chunks = self._iterencode_dict(value, markers,
+                                                   _current_indent_level)
                 else:
-                    chunks = _iterencode(value, _current_indent_level)
+                    chunks = self._iterencode(value, markers,
+                                              _current_indent_level)
                 for chunk in chunks:
                     yield chunk
         if newline_indent is not None:
             _current_indent_level -= 1
-            yield '\n' + (' ' * (_indent * _current_indent_level))
+            yield '\n' + (' ' * (self.indent * _current_indent_level))
         yield '}'
-        if markers is not None:
-            del markers[markerid]
+        self._remove_markers(markers, dct)
 
-    def _iterencode(o, _current_indent_level):
+    def _iterencode(self, o, markers, _current_indent_level):
         if isinstance(o, basestring):
-            yield _encoder(o)
+            yield '"' + self.encoder(o) + '"'
         elif o is None:
             yield 'null'
         elif o is True:
@@ -420,23 +491,19 @@
         elif isinstance(o, (int, long)):
             yield str(o)
         elif isinstance(o, float):
-            yield _floatstr(o)
+            yield self._floatstr(o)
         elif isinstance(o, (list, tuple)):
-            for chunk in _iterencode_list(o, _current_indent_level):
+            for chunk in self._iterencode_list(o, markers,
+                                               _current_indent_level):
                 yield chunk
         elif isinstance(o, dict):
-            for chunk in _iterencode_dict(o, _current_indent_level):
+            for chunk in self._iterencode_dict(o, markers,
+                                               _current_indent_level):
                 yield chunk
         else:
-            if markers is not None:
-                markerid = id(o)
-                if markerid in markers:
-                    raise ValueError("Circular reference detected")
-                markers[markerid] = o
-            o = _default(o)
-            for chunk in _iterencode(o, _current_indent_level):
+            self._mark_markers(markers, o)
+            obj = self.default(o)
+            for chunk in self._iterencode(obj, markers,
+                                          _current_indent_level):
                 yield chunk
-            if markers is not None:
-                del markers[markerid]
-
-    return _iterencode
+            self._remove_markers(markers, o)
diff --git a/lib-python/modified-2.7/json/tests/test_unicode.py b/lib-python/modified-2.7/json/tests/test_unicode.py
--- a/lib-python/modified-2.7/json/tests/test_unicode.py
+++ b/lib-python/modified-2.7/json/tests/test_unicode.py
@@ -80,3 +80,9 @@
         self.assertEqual(type(json.loads(u'["a"]')[0]), unicode)
         # Issue 10038.
         self.assertEqual(type(json.loads('"foo"')), unicode)
+
+    def test_encode_not_utf_8(self):
+        self.assertEqual(json.dumps('\xb1\xe6', encoding='iso8859-2'),
+                         '"\\u0105\\u0107"')
+        self.assertEqual(json.dumps(['\xb1\xe6'], encoding='iso8859-2'),
+                         '["\\u0105\\u0107"]')
diff --git a/lib_pypy/pyrepl/readline.py b/lib_pypy/pyrepl/readline.py
--- a/lib_pypy/pyrepl/readline.py
+++ b/lib_pypy/pyrepl/readline.py
@@ -395,9 +395,21 @@
     _wrapper.f_in = f_in
     _wrapper.f_out = f_out
 
-    if hasattr(sys, '__raw_input__'):    # PyPy
-        _old_raw_input = sys.__raw_input__
+    if '__pypy__' in sys.builtin_module_names:    # PyPy
+
+        def _old_raw_input(prompt=''):
+            # sys.__raw_input__() is only called when stdin and stdout are
+            # as expected and are ttys.  If it is the case, then get_reader()
+            # should not really fail in _wrapper.raw_input().  If it still
+            # does, then we will just cancel the redirection and call again
+            # the built-in raw_input().
+            try:
+                del sys.__raw_input__
+            except AttributeError:
+                pass
+            return raw_input(prompt)
         sys.__raw_input__ = _wrapper.raw_input
+
     else:
         # this is not really what readline.c does.  Better than nothing I guess
         import __builtin__
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -78,6 +78,7 @@
     del working_modules['fcntl']  # LOCK_NB not defined
     del working_modules["_minimal_curses"]
     del working_modules["termios"]
+    del working_modules["_multiprocessing"]   # depends on rctime
 
 
 
diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -2,7 +2,7 @@
 from pypy.interpreter.baseobjspace import Wrappable
 from pypy.interpreter import typedef
 from pypy.interpreter.gateway import interp2app
-from pypy.interpreter.error import OperationError
+from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.tool.pairtype import extendabletype
 from pypy.tool.sourcetools import func_with_new_name
@@ -101,9 +101,7 @@
             missing = required[i]
             if missing is not None:
                  err = "required field \"%s\" missing from %s"
-                 err = err % (missing, host)
-                 w_err = space.wrap(err)
-                 raise OperationError(space.w_TypeError, w_err)
+                 raise operationerrfmt(space.w_TypeError, err, missing, host)
     raise AssertionError("should not reach here")
 
 
@@ -2857,14 +2855,13 @@
 def Module_get_body(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -2900,14 +2897,13 @@
 def Interactive_get_body(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -2947,8 +2943,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     return space.wrap(w_self.body)
 
 def Expression_set_body(space, w_self, w_new_value):
@@ -2989,14 +2984,13 @@
 def Suite_get_body(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3036,8 +3030,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._lineno_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'lineno'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lineno')
     return space.wrap(w_self.lineno)
 
 def stmt_set_lineno(space, w_self, w_new_value):
@@ -3058,8 +3051,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._col_offset_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'col_offset'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'col_offset')
     return space.wrap(w_self.col_offset)
 
 def stmt_set_col_offset(space, w_self, w_new_value):
@@ -3089,8 +3081,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'name'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name')
     return space.wrap(w_self.name)
 
 def FunctionDef_set_name(space, w_self, w_new_value):
@@ -3111,8 +3102,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'args'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args')
     return space.wrap(w_self.args)
 
 def FunctionDef_set_args(space, w_self, w_new_value):
@@ -3129,14 +3119,13 @@
 def FunctionDef_get_body(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3147,14 +3136,13 @@
 def FunctionDef_get_decorator_list(space, w_self):
     if not w_self.initialization_state & 8:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'decorator_list'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'decorator_list')
     if w_self.w_decorator_list is None:
         if w_self.decorator_list is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.decorator_list]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_decorator_list = w_list
     return w_self.w_decorator_list
 
@@ -3198,8 +3186,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'name'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name')
     return space.wrap(w_self.name)
 
 def ClassDef_set_name(space, w_self, w_new_value):
@@ -3216,14 +3203,13 @@
 def ClassDef_get_bases(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'bases'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'bases')
     if w_self.w_bases is None:
         if w_self.bases is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.bases]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_bases = w_list
     return w_self.w_bases
 
@@ -3234,14 +3220,13 @@
 def ClassDef_get_keywords(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'keywords'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'keywords')
     if w_self.w_keywords is None:
         if w_self.keywords is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.keywords]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_keywords = w_list
     return w_self.w_keywords
 
@@ -3256,8 +3241,7 @@
             return w_obj
     if not w_self.initialization_state & 8:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'starargs'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'starargs')
     return space.wrap(w_self.starargs)
 
 def ClassDef_set_starargs(space, w_self, w_new_value):
@@ -3278,8 +3262,7 @@
             return w_obj
     if not w_self.initialization_state & 16:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'kwargs'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'kwargs')
     return space.wrap(w_self.kwargs)
 
 def ClassDef_set_kwargs(space, w_self, w_new_value):
@@ -3296,14 +3279,13 @@
 def ClassDef_get_body(space, w_self):
     if not w_self.initialization_state & 32:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3314,14 +3296,13 @@
 def ClassDef_get_decorator_list(space, w_self):
     if not w_self.initialization_state & 64:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'decorator_list'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'decorator_list')
     if w_self.w_decorator_list is None:
         if w_self.decorator_list is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.decorator_list]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_decorator_list = w_list
     return w_self.w_decorator_list
 
@@ -3370,8 +3351,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Return_set_value(space, w_self, w_new_value):
@@ -3412,14 +3392,13 @@
 def Delete_get_targets(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'targets'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'targets')
     if w_self.w_targets is None:
         if w_self.targets is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.targets]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_targets = w_list
     return w_self.w_targets
 
@@ -3455,14 +3434,13 @@
 def Assign_get_targets(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'targets'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'targets')
     if w_self.w_targets is None:
         if w_self.targets is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.targets]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_targets = w_list
     return w_self.w_targets
 
@@ -3477,8 +3455,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Assign_set_value(space, w_self, w_new_value):
@@ -3525,8 +3502,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'target'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'target')
     return space.wrap(w_self.target)
 
 def AugAssign_set_target(space, w_self, w_new_value):
@@ -3547,8 +3523,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'op'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op')
     return operator_to_class[w_self.op - 1]()
 
 def AugAssign_set_op(space, w_self, w_new_value):
@@ -3571,8 +3546,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def AugAssign_set_value(space, w_self, w_new_value):
@@ -3619,8 +3593,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'target'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'target')
     return space.wrap(w_self.target)
 
 def For_set_target(space, w_self, w_new_value):
@@ -3641,8 +3614,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'iter'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'iter')
     return space.wrap(w_self.iter)
 
 def For_set_iter(space, w_self, w_new_value):
@@ -3659,14 +3631,13 @@
 def For_get_body(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3677,14 +3648,13 @@
 def For_get_orelse(space, w_self):
     if not w_self.initialization_state & 8:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'orelse'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
     if w_self.w_orelse is None:
         if w_self.orelse is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.orelse]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_orelse = w_list
     return w_self.w_orelse
 
@@ -3728,8 +3698,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'test'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test')
     return space.wrap(w_self.test)
 
 def While_set_test(space, w_self, w_new_value):
@@ -3746,14 +3715,13 @@
 def While_get_body(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3764,14 +3732,13 @@
 def While_get_orelse(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'orelse'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
     if w_self.w_orelse is None:
         if w_self.orelse is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.orelse]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_orelse = w_list
     return w_self.w_orelse
 
@@ -3814,8 +3781,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'test'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test')
     return space.wrap(w_self.test)
 
 def If_set_test(space, w_self, w_new_value):
@@ -3832,14 +3798,13 @@
 def If_get_body(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3850,14 +3815,13 @@
 def If_get_orelse(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'orelse'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
     if w_self.w_orelse is None:
         if w_self.orelse is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.orelse]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_orelse = w_list
     return w_self.w_orelse
 
@@ -3900,8 +3864,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'context_expr'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'context_expr')
     return space.wrap(w_self.context_expr)
 
 def With_set_context_expr(space, w_self, w_new_value):
@@ -3922,8 +3885,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'optional_vars'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'optional_vars')
     return space.wrap(w_self.optional_vars)
 
 def With_set_optional_vars(space, w_self, w_new_value):
@@ -3940,14 +3902,13 @@
 def With_get_body(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3989,8 +3950,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'exc'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'exc')
     return space.wrap(w_self.exc)
 
 def Raise_set_exc(space, w_self, w_new_value):
@@ -4011,8 +3971,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'cause'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'cause')
     return space.wrap(w_self.cause)
 
 def Raise_set_cause(space, w_self, w_new_value):
@@ -4054,14 +4013,13 @@
 def TryExcept_get_body(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -4072,14 +4030,13 @@
 def TryExcept_get_handlers(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'handlers'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'handlers')
     if w_self.w_handlers is None:
         if w_self.handlers is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.handlers]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_handlers = w_list
     return w_self.w_handlers
 
@@ -4090,14 +4047,13 @@
 def TryExcept_get_orelse(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'orelse'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
     if w_self.w_orelse is None:
         if w_self.orelse is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.orelse]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_orelse = w_list
     return w_self.w_orelse
 
@@ -4137,14 +4093,13 @@
 def TryFinally_get_body(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -4155,14 +4110,13 @@
 def TryFinally_get_finalbody(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'finalbody'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'finalbody')
     if w_self.w_finalbody is None:
         if w_self.finalbody is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.finalbody]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_finalbody = w_list
     return w_self.w_finalbody
 
@@ -4204,8 +4158,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'test'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test')
     return space.wrap(w_self.test)
 
 def Assert_set_test(space, w_self, w_new_value):
@@ -4226,8 +4179,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'msg'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'msg')
     return space.wrap(w_self.msg)
 
 def Assert_set_msg(space, w_self, w_new_value):
@@ -4269,14 +4221,13 @@
 def Import_get_names(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'names'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'names')
     if w_self.w_names is None:
         if w_self.names is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.names]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_names = w_list
     return w_self.w_names
 
@@ -4316,8 +4267,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'module'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'module')
     return space.wrap(w_self.module)
 
 def ImportFrom_set_module(space, w_self, w_new_value):
@@ -4337,14 +4287,13 @@
 def ImportFrom_get_names(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'names'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'names')
     if w_self.w_names is None:
         if w_self.names is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.names]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_names = w_list
     return w_self.w_names
 
@@ -4359,8 +4308,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'level'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'level')
     return space.wrap(w_self.level)
 
 def ImportFrom_set_level(space, w_self, w_new_value):
@@ -4404,14 +4352,13 @@
 def Global_get_names(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'names'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'names')
     if w_self.w_names is None:
         if w_self.names is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.names]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_names = w_list
     return w_self.w_names
 
@@ -4447,14 +4394,13 @@
 def Nonlocal_get_names(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'names'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'names')
     if w_self.w_names is None:
         if w_self.names is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.names]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_names = w_list
     return w_self.w_names
 
@@ -4494,8 +4440,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Expr_set_value(space, w_self, w_new_value):
@@ -4591,8 +4536,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._lineno_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'lineno'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lineno')
     return space.wrap(w_self.lineno)
 
 def expr_set_lineno(space, w_self, w_new_value):
@@ -4613,8 +4557,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._col_offset_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'col_offset'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'col_offset')
     return space.wrap(w_self.col_offset)
 
 def expr_set_col_offset(space, w_self, w_new_value):
@@ -4644,8 +4587,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'op'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op')
     return boolop_to_class[w_self.op - 1]()
 
 def BoolOp_set_op(space, w_self, w_new_value):
@@ -4664,14 +4606,13 @@
 def BoolOp_get_values(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'values'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'values')
     if w_self.w_values is None:
         if w_self.values is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.values]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_values = w_list
     return w_self.w_values
 
@@ -4712,8 +4653,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'left'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'left')
     return space.wrap(w_self.left)
 
 def BinOp_set_left(space, w_self, w_new_value):
@@ -4734,8 +4674,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'op'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op')
     return operator_to_class[w_self.op - 1]()
 
 def BinOp_set_op(space, w_self, w_new_value):
@@ -4758,8 +4697,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'right'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'right')
     return space.wrap(w_self.right)
 
 def BinOp_set_right(space, w_self, w_new_value):
@@ -4806,8 +4744,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'op'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op')
     return unaryop_to_class[w_self.op - 1]()
 
 def UnaryOp_set_op(space, w_self, w_new_value):
@@ -4830,8 +4767,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'operand'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'operand')
     return space.wrap(w_self.operand)
 
 def UnaryOp_set_operand(space, w_self, w_new_value):
@@ -4877,8 +4813,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'args'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args')
     return space.wrap(w_self.args)
 
 def Lambda_set_args(space, w_self, w_new_value):
@@ -4899,8 +4834,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     return space.wrap(w_self.body)
 
 def Lambda_set_body(space, w_self, w_new_value):
@@ -4946,8 +4880,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'test'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test')
     return space.wrap(w_self.test)
 
 def IfExp_set_test(space, w_self, w_new_value):
@@ -4968,8 +4901,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     return space.wrap(w_self.body)
 
 def IfExp_set_body(space, w_self, w_new_value):
@@ -4990,8 +4922,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'orelse'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
     return space.wrap(w_self.orelse)
 
 def IfExp_set_orelse(space, w_self, w_new_value):
@@ -5034,14 +4965,13 @@
 def Dict_get_keys(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'keys'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'keys')
     if w_self.w_keys is None:
         if w_self.keys is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.keys]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_keys = w_list
     return w_self.w_keys
 
@@ -5052,14 +4982,13 @@
 def Dict_get_values(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'values'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'values')
     if w_self.w_values is None:
         if w_self.values is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.values]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_values = w_list
     return w_self.w_values
 
@@ -5097,14 +5026,13 @@
 def Set_get_elts(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elts'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elts')
     if w_self.w_elts is None:
         if w_self.elts is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.elts]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_elts = w_list
     return w_self.w_elts
 
@@ -5144,8 +5072,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elt'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elt')
     return space.wrap(w_self.elt)
 
 def ListComp_set_elt(space, w_self, w_new_value):
@@ -5162,14 +5089,13 @@
 def ListComp_get_generators(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'generators'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators')
     if w_self.w_generators is None:
         if w_self.generators is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.generators]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_generators = w_list
     return w_self.w_generators
 
@@ -5210,8 +5136,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elt'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elt')
     return space.wrap(w_self.elt)
 
 def SetComp_set_elt(space, w_self, w_new_value):
@@ -5228,14 +5153,13 @@
 def SetComp_get_generators(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'generators'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators')
     if w_self.w_generators is None:
         if w_self.generators is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.generators]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_generators = w_list
     return w_self.w_generators
 
@@ -5276,8 +5200,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'key'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'key')
     return space.wrap(w_self.key)
 
 def DictComp_set_key(space, w_self, w_new_value):
@@ -5298,8 +5221,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def DictComp_set_value(space, w_self, w_new_value):
@@ -5316,14 +5238,13 @@
 def DictComp_get_generators(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'generators'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators')
     if w_self.w_generators is None:
         if w_self.generators is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.generators]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_generators = w_list
     return w_self.w_generators
 
@@ -5365,8 +5286,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elt'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elt')
     return space.wrap(w_self.elt)
 
 def GeneratorExp_set_elt(space, w_self, w_new_value):
@@ -5383,14 +5303,13 @@
 def GeneratorExp_get_generators(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'generators'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators')
     if w_self.w_generators is None:
         if w_self.generators is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.generators]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_generators = w_list
     return w_self.w_generators
 
@@ -5431,8 +5350,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Yield_set_value(space, w_self, w_new_value):
@@ -5477,8 +5395,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'left'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'left')
     return space.wrap(w_self.left)
 
 def Compare_set_left(space, w_self, w_new_value):
@@ -5495,14 +5412,13 @@
 def Compare_get_ops(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ops'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ops')
     if w_self.w_ops is None:
         if w_self.ops is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [cmpop_to_class[node - 1]() for node in w_self.ops]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_ops = w_list
     return w_self.w_ops
 
@@ -5513,14 +5429,13 @@
 def Compare_get_comparators(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'comparators'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'comparators')
     if w_self.w_comparators is None:
         if w_self.comparators is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.comparators]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_comparators = w_list
     return w_self.w_comparators
 
@@ -5563,8 +5478,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'func'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'func')
     return space.wrap(w_self.func)
 
 def Call_set_func(space, w_self, w_new_value):
@@ -5581,14 +5495,13 @@
 def Call_get_args(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'args'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args')
     if w_self.w_args is None:
         if w_self.args is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.args]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_args = w_list
     return w_self.w_args
 
@@ -5599,14 +5512,13 @@
 def Call_get_keywords(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'keywords'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'keywords')
     if w_self.w_keywords is None:
         if w_self.keywords is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.keywords]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_keywords = w_list
     return w_self.w_keywords
 
@@ -5621,8 +5533,7 @@
             return w_obj
     if not w_self.initialization_state & 8:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'starargs'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'starargs')
     return space.wrap(w_self.starargs)
 
 def Call_set_starargs(space, w_self, w_new_value):
@@ -5643,8 +5554,7 @@
             return w_obj
     if not w_self.initialization_state & 16:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'kwargs'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'kwargs')
     return space.wrap(w_self.kwargs)
 
 def Call_set_kwargs(space, w_self, w_new_value):
@@ -5695,8 +5605,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'n'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'n')
     return w_self.n
 
 def Num_set_n(space, w_self, w_new_value):
@@ -5741,8 +5650,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 's'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 's')
     return w_self.s
 
 def Str_set_s(space, w_self, w_new_value):
@@ -5787,8 +5695,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Attribute_set_value(space, w_self, w_new_value):
@@ -5809,8 +5716,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'attr'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'attr')
     return space.wrap(w_self.attr)
 
 def Attribute_set_attr(space, w_self, w_new_value):
@@ -5831,8 +5737,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ctx'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def Attribute_set_ctx(space, w_self, w_new_value):
@@ -5881,8 +5786,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Subscript_set_value(space, w_self, w_new_value):
@@ -5903,8 +5807,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'slice'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'slice')
     return space.wrap(w_self.slice)
 
 def Subscript_set_slice(space, w_self, w_new_value):
@@ -5925,8 +5828,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ctx'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def Subscript_set_ctx(space, w_self, w_new_value):
@@ -5975,8 +5877,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'id'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'id')
     return space.wrap(w_self.id)
 
 def Name_set_id(space, w_self, w_new_value):
@@ -5997,8 +5898,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ctx'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def Name_set_ctx(space, w_self, w_new_value):
@@ -6042,14 +5942,13 @@
 def List_get_elts(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elts'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elts')
     if w_self.w_elts is None:
         if w_self.elts is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.elts]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_elts = w_list
     return w_self.w_elts
 
@@ -6064,8 +5963,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ctx'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def List_set_ctx(space, w_self, w_new_value):
@@ -6110,14 +6008,13 @@
 def Tuple_get_elts(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elts'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elts')
     if w_self.w_elts is None:
         if w_self.elts is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.elts]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_elts = w_list
     return w_self.w_elts
 
@@ -6132,8 +6029,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ctx'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def Tuple_set_ctx(space, w_self, w_new_value):
@@ -6182,8 +6078,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return w_self.value
 
 def Const_set_value(space, w_self, w_new_value):
@@ -6301,8 +6196,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'lower'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lower')
     return space.wrap(w_self.lower)
 
 def Slice_set_lower(space, w_self, w_new_value):
@@ -6323,8 +6217,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'upper'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'upper')
     return space.wrap(w_self.upper)
 
 def Slice_set_upper(space, w_self, w_new_value):
@@ -6345,8 +6238,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'step'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'step')
     return space.wrap(w_self.step)
 
 def Slice_set_step(space, w_self, w_new_value):
@@ -6389,14 +6281,13 @@
 def ExtSlice_get_dims(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'dims'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'dims')
     if w_self.w_dims is None:
         if w_self.dims is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.dims]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_dims = w_list
     return w_self.w_dims
 
@@ -6436,8 +6327,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Index_set_value(space, w_self, w_new_value):
@@ -6706,8 +6596,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'target'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'target')
     return space.wrap(w_self.target)
 
 def comprehension_set_target(space, w_self, w_new_value):
@@ -6728,8 +6617,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'iter'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'iter')
     return space.wrap(w_self.iter)
 
 def comprehension_set_iter(space, w_self, w_new_value):
@@ -6746,14 +6634,13 @@
 def comprehension_get_ifs(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ifs'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ifs')
     if w_self.w_ifs is None:
         if w_self.ifs is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.ifs]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_ifs = w_list
     return w_self.w_ifs
 
@@ -6795,8 +6682,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._lineno_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'lineno'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lineno')
     return space.wrap(w_self.lineno)
 
 def excepthandler_set_lineno(space, w_self, w_new_value):
@@ -6817,8 +6703,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._col_offset_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'col_offset'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'col_offset')
     return space.wrap(w_self.col_offset)
 
 def excepthandler_set_col_offset(space, w_self, w_new_value):
@@ -6848,8 +6733,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'type'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'type')
     return space.wrap(w_self.type)
 
 def ExceptHandler_set_type(space, w_self, w_new_value):
@@ -6870,8 +6754,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'name'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name')
     return space.wrap(w_self.name)
 
 def ExceptHandler_set_name(space, w_self, w_new_value):
@@ -6888,14 +6771,13 @@
 def ExceptHandler_get_body(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -6933,14 +6815,13 @@
 def arguments_get_args(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'args'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args')
     if w_self.w_args is None:
         if w_self.args is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.args]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_args = w_list
     return w_self.w_args
 
@@ -6955,8 +6836,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'vararg'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'vararg')
     return space.wrap(w_self.vararg)
 
 def arguments_set_vararg(space, w_self, w_new_value):
@@ -6980,8 +6860,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'kwarg'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'kwarg')
     return space.wrap(w_self.kwarg)
 
 def arguments_set_kwarg(space, w_self, w_new_value):
@@ -7001,14 +6880,13 @@
 def arguments_get_defaults(space, w_self):
     if not w_self.initialization_state & 8:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'defaults'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'defaults')
     if w_self.w_defaults is None:
         if w_self.defaults is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.defaults]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_defaults = w_list
     return w_self.w_defaults
 
@@ -7052,8 +6930,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'arg'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'arg')
     return space.wrap(w_self.arg)
 
 def keyword_set_arg(space, w_self, w_new_value):
@@ -7074,8 +6951,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def keyword_set_value(space, w_self, w_new_value):
@@ -7121,8 +6997,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'name'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name')
     return space.wrap(w_self.name)
 
 def alias_set_name(space, w_self, w_new_value):
@@ -7143,8 +7018,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'asname'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'asname')
     return space.wrap(w_self.asname)
 
 def alias_set_asname(space, w_self, w_new_value):
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -414,13 +414,12 @@
             self.emit("        return w_obj", 1)
         self.emit("if not w_self.initialization_state & %s:" % (flag,), 1)
         self.emit("typename = space.type(w_self).getname(space)", 2)
-        self.emit("w_err = space.wrap(\"'%%s' object has no attribute '%s'\" %% typename)" %
+        self.emit("raise operationerrfmt(space.w_AttributeError, \"'%%s' object has no attribute '%%s'\", typename, '%s')" %
                   (field.name,), 2)
-        self.emit("raise OperationError(space.w_AttributeError, w_err)", 2)
         if field.seq:
             self.emit("if w_self.w_%s is None:" % (field.name,), 1)
             self.emit("if w_self.%s is None:" % (field.name,), 2)
-            self.emit("w_list = space.newlist([])", 3)
+            self.emit("list_w = []", 3)
             self.emit("else:", 2)
             if field.type.value in self.data.simple_types:
                 wrapper = "%s_to_class[node - 1]()" % (field.type,)
@@ -428,7 +427,7 @@
                 wrapper = "space.wrap(node)"
             self.emit("list_w = [%s for node in w_self.%s]" %
                       (wrapper, field.name), 3)
-            self.emit("w_list = space.newlist(list_w)", 3)
+            self.emit("w_list = space.newlist(list_w)", 2)
             self.emit("w_self.w_%s = w_list" % (field.name,), 2)
             self.emit("return w_self.w_%s" % (field.name,), 1)
         elif field.type.value in self.data.simple_types:
@@ -540,7 +539,7 @@
 from pypy.interpreter.baseobjspace import Wrappable
 from pypy.interpreter import typedef
 from pypy.interpreter.gateway import interp2app
-from pypy.interpreter.error import OperationError
+from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.tool.pairtype import extendabletype
 from pypy.tool.sourcetools import func_with_new_name
@@ -639,9 +638,7 @@
             missing = required[i]
             if missing is not None:
                  err = "required field \\"%s\\" missing from %s"
-                 err = err % (missing, host)
-                 w_err = space.wrap(err)
-                 raise OperationError(space.w_TypeError, w_err)
+                 raise operationerrfmt(space.w_TypeError, err, missing, host)
     raise AssertionError("should not reach here")
 
 
diff --git a/pypy/jit/backend/llgraph/llimpl.py b/pypy/jit/backend/llgraph/llimpl.py
--- a/pypy/jit/backend/llgraph/llimpl.py
+++ b/pypy/jit/backend/llgraph/llimpl.py
@@ -7,9 +7,7 @@
 import weakref
 from pypy.objspace.flow.model import Variable, Constant
 from pypy.annotation import model as annmodel
-from pypy.jit.metainterp.history import (ConstInt, ConstPtr,
-                                         BoxInt, BoxPtr, BoxObj, BoxFloat,
-                                         REF, INT, FLOAT)
+from pypy.jit.metainterp.history import REF, INT, FLOAT
 from pypy.jit.codewriter import heaptracker
 from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr, rffi
 from pypy.rpython.ootypesystem import ootype
@@ -17,7 +15,7 @@
 from pypy.rpython.llinterp import LLException
 from pypy.rpython.extregistry import ExtRegistryEntry
 
-from pypy.jit.metainterp import resoperation, executor
+from pypy.jit.metainterp import resoperation
 from pypy.jit.metainterp.resoperation import rop
 from pypy.jit.backend.llgraph import symbolic
 from pypy.jit.codewriter import longlong
@@ -334,6 +332,13 @@
     assert isinstance(type, str) and len(type) == 1
     op.descr = Descr(ofs, type, arg_types=arg_types)
 
+def compile_add_descr_arg(loop, ofs, type, arg_types):
+    from pypy.jit.backend.llgraph.runner import Descr
+    loop = _from_opaque(loop)
+    op = loop.operations[-1]
+    assert isinstance(type, str) and len(type) == 1
+    op.args.append(Descr(ofs, type, arg_types=arg_types))
+
 def compile_add_loop_token(loop, descr):
     if we_are_translated():
         raise ValueError("CALL_ASSEMBLER not supported")
@@ -438,8 +443,11 @@
         self._may_force = -1
 
     def getenv(self, v):
+        from pypy.jit.backend.llgraph.runner import Descr
         if isinstance(v, Constant):
             return v.value
+        elif isinstance(v, Descr):
+            return v
         else:
             return self.env[v]
 
@@ -807,6 +815,29 @@
         else:
             raise NotImplementedError
 
+    def op_getinteriorfield_gc(self, descr, array, index):
+        if descr.typeinfo == REF:
+            return do_getinteriorfield_gc_ptr(array, index, descr.ofs)
+        elif descr.typeinfo == INT:
+            return do_getinteriorfield_gc_int(array, index, descr.ofs)
+        elif descr.typeinfo == FLOAT:
+            return do_getinteriorfield_gc_float(array, index, descr.ofs)
+        else:
+            raise NotImplementedError
+
+    def op_setinteriorfield_gc(self, descr, array, index, newvalue):
+        if descr.typeinfo == REF:
+            return do_setinteriorfield_gc_ptr(array, index, descr.ofs,
+                                              newvalue)
+        elif descr.typeinfo == INT:
+            return do_setinteriorfield_gc_int(array, index, descr.ofs,
+                                              newvalue)
+        elif descr.typeinfo == FLOAT:
+            return do_setinteriorfield_gc_float(array, index, descr.ofs,
+                                                newvalue)
+        else:
+            raise NotImplementedError
+
     def op_setfield_gc(self, fielddescr, struct, newvalue):
         if fielddescr.typeinfo == REF:
             do_setfield_gc_ptr(struct, fielddescr.ofs, newvalue)
@@ -1354,6 +1385,22 @@
 def do_getfield_gc_ptr(struct, fieldnum):
     return cast_to_ptr(_getfield_gc(struct, fieldnum))
 
+def _getinteriorfield_gc(struct, fieldnum):
+    STRUCT, fieldname = symbolic.TokenToField[fieldnum]
+    return getattr(struct, fieldname)
+
+def do_getinteriorfield_gc_int(array, index, fieldnum):
+    struct = array._obj.container.getitem(index)
+    return cast_to_int(_getinteriorfield_gc(struct, fieldnum))
+
+def do_getinteriorfield_gc_float(array, index, fieldnum):
+    struct = array._obj.container.getitem(index)
+    return cast_to_floatstorage(_getinteriorfield_gc(struct, fieldnum))
+
+def do_getinteriorfield_gc_ptr(array, index, fieldnum):
+    struct = array._obj.container.getitem(index)
+    return cast_to_ptr(_getinteriorfield_gc(struct, fieldnum))
+
 def _getfield_raw(struct, fieldnum):
     STRUCT, fieldname = symbolic.TokenToField[fieldnum]
     ptr = cast_from_int(lltype.Ptr(STRUCT), struct)
@@ -1409,26 +1456,28 @@
     newvalue = cast_from_ptr(ITEMTYPE, newvalue)
     array.setitem(index, newvalue)
 
-def do_setfield_gc_int(struct, fieldnum, newvalue):
-    STRUCT, fieldname = symbolic.TokenToField[fieldnum]
-    ptr = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), struct)
-    FIELDTYPE = getattr(STRUCT, fieldname)
-    newvalue = cast_from_int(FIELDTYPE, newvalue)
-    setattr(ptr, fieldname, newvalue)
+def new_setfield_gc(cast_func):
+    def do_setfield_gc(struct, fieldnum, newvalue):
+        STRUCT, fieldname = symbolic.TokenToField[fieldnum]
+        ptr = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), struct)
+        FIELDTYPE = getattr(STRUCT, fieldname)
+        newvalue = cast_func(FIELDTYPE, newvalue)
+        setattr(ptr, fieldname, newvalue)
+    return do_setfield_gc
+do_setfield_gc_int = new_setfield_gc(cast_from_int)
+do_setfield_gc_float = new_setfield_gc(cast_from_floatstorage)
+do_setfield_gc_ptr = new_setfield_gc(cast_from_ptr)
 
-def do_setfield_gc_float(struct, fieldnum, newvalue):
-    STRUCT, fieldname = symbolic.TokenToField[fieldnum]
-    ptr = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), struct)
-    FIELDTYPE = getattr(STRUCT, fieldname)
-    newvalue = cast_from_floatstorage(FIELDTYPE, newvalue)
-    setattr(ptr, fieldname, newvalue)
-
-def do_setfield_gc_ptr(struct, fieldnum, newvalue):
-    STRUCT, fieldname = symbolic.TokenToField[fieldnum]
-    ptr = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), struct)
-    FIELDTYPE = getattr(STRUCT, fieldname)
-    newvalue = cast_from_ptr(FIELDTYPE, newvalue)
-    setattr(ptr, fieldname, newvalue)
+def new_setinteriorfield_gc(cast_func):
+    def do_setinteriorfield_gc(array, index, fieldnum, newvalue):
+        STRUCT, fieldname = symbolic.TokenToField[fieldnum]
+        struct = array._obj.container.getitem(index)
+        FIELDTYPE = getattr(STRUCT, fieldname)
+        setattr(struct, fieldname, cast_func(FIELDTYPE, newvalue))
+    return do_setinteriorfield_gc
+do_setinteriorfield_gc_int = new_setinteriorfield_gc(cast_from_int)
+do_setinteriorfield_gc_float = new_setinteriorfield_gc(cast_from_floatstorage)
+do_setinteriorfield_gc_ptr = new_setinteriorfield_gc(cast_from_ptr)        
 
 def do_setfield_raw_int(struct, fieldnum, newvalue):
     STRUCT, fieldname = symbolic.TokenToField[fieldnum]
@@ -1694,6 +1743,7 @@
 setannotation(compile_start_float_var, annmodel.SomeInteger())
 setannotation(compile_add, annmodel.s_None)
 setannotation(compile_add_descr, annmodel.s_None)
+setannotation(compile_add_descr_arg, annmodel.s_None)
 setannotation(compile_add_var, annmodel.s_None)
 setannotation(compile_add_int_const, annmodel.s_None)
 setannotation(compile_add_ref_const, annmodel.s_None)
@@ -1741,6 +1791,9 @@
 setannotation(do_getfield_raw_int, annmodel.SomeInteger())
 setannotation(do_getfield_raw_ptr, annmodel.SomePtr(llmemory.GCREF))
 setannotation(do_getfield_raw_float, s_FloatStorage)
+setannotation(do_getinteriorfield_gc_int, annmodel.SomeInteger())
+setannotation(do_getinteriorfield_gc_ptr, annmodel.SomePtr(llmemory.GCREF))
+setannotation(do_getinteriorfield_gc_float, s_FloatStorage)
 setannotation(do_new, annmodel.SomePtr(llmemory.GCREF))
 setannotation(do_new_array, annmodel.SomePtr(llmemory.GCREF))
 setannotation(do_setarrayitem_gc_int, annmodel.s_None)
@@ -1754,6 +1807,9 @@
 setannotation(do_setfield_raw_int, annmodel.s_None)
 setannotation(do_setfield_raw_ptr, annmodel.s_None)
 setannotation(do_setfield_raw_float, annmodel.s_None)
+setannotation(do_setinteriorfield_gc_int, annmodel.s_None)
+setannotation(do_setinteriorfield_gc_ptr, annmodel.s_None)
+setannotation(do_setinteriorfield_gc_float, annmodel.s_None)
 setannotation(do_newstr, annmodel.SomePtr(llmemory.GCREF))
 setannotation(do_strsetitem, annmodel.s_None)
 setannotation(do_newunicode, annmodel.SomePtr(llmemory.GCREF))
diff --git a/pypy/jit/backend/llgraph/runner.py b/pypy/jit/backend/llgraph/runner.py
--- a/pypy/jit/backend/llgraph/runner.py
+++ b/pypy/jit/backend/llgraph/runner.py
@@ -2,7 +2,6 @@
 Minimal-API wrapper around the llinterpreter to run operations.
 """
 
-import sys
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.rpython.lltypesystem import lltype, llmemory, rclass
@@ -11,12 +10,11 @@
 from pypy.jit.metainterp import history
 from pypy.jit.metainterp.history import REF, INT, FLOAT
 from pypy.jit.metainterp.warmstate import unwrap
-from pypy.jit.metainterp.resoperation import ResOperation, rop
+from pypy.jit.metainterp.resoperation import rop
 from pypy.jit.backend import model
 from pypy.jit.backend.llgraph import llimpl, symbolic
 from pypy.jit.metainterp.typesystem import llhelper, oohelper
 from pypy.jit.codewriter import heaptracker, longlong
-from pypy.rlib import rgc
 
 class MiniStats:
     pass
@@ -177,8 +175,10 @@
             llimpl.compile_add(c, op.getopnum())
             descr = op.getdescr()
             if isinstance(descr, Descr):
-                llimpl.compile_add_descr(c, descr.ofs, descr.typeinfo, descr.arg_types)
-            if isinstance(descr, history.LoopToken) and op.getopnum() != rop.JUMP:
+                llimpl.compile_add_descr(c, descr.ofs, descr.typeinfo,
+                                         descr.arg_types)
+            if (isinstance(descr, history.LoopToken) and
+                op.getopnum() != rop.JUMP):
                 llimpl.compile_add_loop_token(c, descr)
             if self.is_oo and isinstance(descr, (OODescr, MethDescr)):
                 # hack hack, not rpython
@@ -193,6 +193,9 @@
                     llimpl.compile_add_ref_const(c, x.value, self.ts.BASETYPE)
                 elif isinstance(x, history.ConstFloat):
                     llimpl.compile_add_float_const(c, x.value)
+                elif isinstance(x, Descr):
+                    llimpl.compile_add_descr_arg(c, x.ofs, x.typeinfo,
+                                                 x.arg_types)
                 else:
                     raise Exception("'%s' args contain: %r" % (op.getopname(),
                                                                x))
@@ -316,6 +319,13 @@
         token = history.getkind(getattr(S, fieldname))
         return self.getdescr(ofs, token[0], name=fieldname)
 
+    def interiorfielddescrof(self, A, fieldname):
+        S = A.OF
+        ofs2 = symbolic.get_size(A)
+        ofs, size = symbolic.get_field_token(S, fieldname)
+        token = history.getkind(getattr(S, fieldname))
+        return self.getdescr(ofs, token[0], name=fieldname, extrainfo=ofs2)
+
     def calldescrof(self, FUNC, ARGS, RESULT, extrainfo):
         arg_types = []
         for ARG in ARGS:
@@ -353,8 +363,11 @@
     def arraydescrof(self, A):
         assert A.OF != lltype.Void
         size = symbolic.get_size(A)
-        token = history.getkind(A.OF)
-        return self.getdescr(size, token[0])
+        if isinstance(A.OF, lltype.Ptr) or isinstance(A.OF, lltype.Primitive):
+            token = history.getkind(A.OF)[0]
+        else:
+            token = '?'
+        return self.getdescr(size, token)
 
     # ---------- the backend-dependent operations ----------
 
@@ -406,6 +419,29 @@
         assert isinstance(fielddescr, Descr)
         return llimpl.do_getfield_raw_float(struct, fielddescr.ofs)
 
+    def bh_getinteriorfield_gc_i(self, array, index, descr):
+        assert isinstance(descr, Descr)
+        return llimpl.do_getinteriorfield_gc_int(array, index, descr.ofs)
+    def bh_getinteriorfield_gc_r(self, array, index, descr):
+        assert isinstance(descr, Descr)
+        return llimpl.do_getinteriorfield_gc_ptr(array, index, descr.ofs)
+    def bh_getinteriorfield_gc_f(self, array, index, descr):
+        assert isinstance(descr, Descr)
+        return llimpl.do_getinteriorfield_gc_float(array, index, descr.ofs)
+
+    def bh_setinteriorfield_gc_i(self, array, index, descr, value):
+        assert isinstance(descr, Descr)
+        return llimpl.do_setinteriorfield_gc_int(array, index, descr.ofs,
+                                                 value)
+    def bh_setinteriorfield_gc_r(self, array, index, descr, value):
+        assert isinstance(descr, Descr)
+        return llimpl.do_setinteriorfield_gc_ptr(array, index, descr.ofs,
+                                                 value)
+    def bh_setinteriorfield_gc_f(self, array, index, descr, value):
+        assert isinstance(descr, Descr)
+        return llimpl.do_setinteriorfield_gc_float(array, index, descr.ofs,
+                                                   value)
+
     def bh_new(self, sizedescr):
         assert isinstance(sizedescr, Descr)
         return llimpl.do_new(sizedescr.ofs)
@@ -418,7 +454,6 @@
 
     def bh_classof(self, struct):
         struct = lltype.cast_opaque_ptr(rclass.OBJECTPTR, struct)
-        result = struct.typeptr
         result_adr = llmemory.cast_ptr_to_adr(struct.typeptr)
         return heaptracker.adr2int(result_adr)
 
diff --git a/pypy/jit/backend/llsupport/descr.py b/pypy/jit/backend/llsupport/descr.py
--- a/pypy/jit/backend/llsupport/descr.py
+++ b/pypy/jit/backend/llsupport/descr.py
@@ -1,13 +1,10 @@
 import py
-from pypy.rpython.lltypesystem import lltype, rffi, llmemory, rclass
+from pypy.rpython.lltypesystem import lltype, rffi, llmemory
 from pypy.rpython.lltypesystem.lloperation import llop
 from pypy.jit.backend.llsupport import symbolic, support
-from pypy.jit.metainterp.history import AbstractDescr, getkind, BoxInt, BoxPtr
-from pypy.jit.metainterp.history import BasicFailDescr, LoopToken, BoxFloat
+from pypy.jit.metainterp.history import AbstractDescr, getkind
 from pypy.jit.metainterp import history
-from pypy.jit.metainterp.resoperation import ResOperation, rop
 from pypy.jit.codewriter import heaptracker, longlong
-from pypy.rlib.rarithmetic import r_longlong, r_ulonglong
 
 # The point of the class organization in this file is to make instances
 # as compact as possible.  This is done by not storing the field size or
@@ -23,6 +20,7 @@
         self._cache_field = {}
         self._cache_array = {}
         self._cache_call = {}
+        self._cache_interiorfield = {}
 
     def init_size_descr(self, STRUCT, sizedescr):
         assert isinstance(STRUCT, lltype.GcStruct)
@@ -142,7 +140,6 @@
         cachedict[fieldname] = fielddescr
         return fielddescr
 
-
 # ____________________________________________________________
 # ArrayDescrs
 
@@ -252,6 +249,36 @@
         cache[ARRAY] = arraydescr
         return arraydescr
 
+# ____________________________________________________________
+# InteriorFieldDescr
+
+class InteriorFieldDescr(AbstractDescr):
+    arraydescr = BaseArrayDescr()     # workaround for the annotator
+    fielddescr = BaseFieldDescr('', 0)
+
+    def __init__(self, arraydescr, fielddescr):
+        self.arraydescr = arraydescr
+        self.fielddescr = fielddescr
+
+    def is_pointer_field(self):
+        return self.fielddescr.is_pointer_field()
+
+    def is_float_field(self):
+        return self.fielddescr.is_float_field()
+
+    def repr_of_descr(self):
+        return '<InteriorFieldDescr %s>' % self.fielddescr.repr_of_descr()
+
+def get_interiorfield_descr(gc_ll_descr, ARRAY, FIELDTP, name):
+    cache = gc_ll_descr._cache_interiorfield
+    try:
+        return cache[(ARRAY, FIELDTP, name)]
+    except KeyError:
+        arraydescr = get_array_descr(gc_ll_descr, ARRAY)
+        fielddescr = get_field_descr(gc_ll_descr, FIELDTP, name)
+        descr = InteriorFieldDescr(arraydescr, fielddescr)
+        cache[(ARRAY, FIELDTP, name)] = descr
+        return descr
 
 # ____________________________________________________________
 # CallDescrs
@@ -525,7 +552,8 @@
         #
         if TYPE is lltype.Float or is_longlong(TYPE):
             setattr(Descr, floatattrname, True)
-        elif TYPE is not lltype.Bool and rffi.cast(TYPE, -1) == -1:
+        elif (TYPE is not lltype.Bool and isinstance(TYPE, lltype.Number) and
+              rffi.cast(TYPE, -1) == -1):
             setattr(Descr, signedattrname, True)
         #
         _cache[nameprefix, TYPE] = Descr
diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py
--- a/pypy/jit/backend/llsupport/gc.py
+++ b/pypy/jit/backend/llsupport/gc.py
@@ -45,6 +45,14 @@
     def freeing_block(self, start, stop):
         pass
 
+    def get_funcptr_for_newarray(self):
+        return llhelper(self.GC_MALLOC_ARRAY, self.malloc_array)
+    def get_funcptr_for_newstr(self):
+        return llhelper(self.GC_MALLOC_STR_UNICODE, self.malloc_str)
+    def get_funcptr_for_newunicode(self):
+        return llhelper(self.GC_MALLOC_STR_UNICODE, self.malloc_unicode)
+
+
     def record_constptrs(self, op, gcrefs_output_list):
         for i in range(op.numargs()):
             v = op.getarg(i)
@@ -96,6 +104,39 @@
         malloc_fn_ptr = self.configure_boehm_once()
         self.funcptr_for_new = malloc_fn_ptr
 
+        def malloc_array(basesize, itemsize, ofs_length, num_elem):
+            try:
+                size = ovfcheck(basesize + ovfcheck(itemsize * num_elem))
+            except OverflowError:
+                return lltype.nullptr(llmemory.GCREF.TO)
+            res = self.funcptr_for_new(size)
+            if not res:
+                return res
+            rffi.cast(rffi.CArrayPtr(lltype.Signed), res)[ofs_length/WORD] = num_elem
+            return res
+        self.malloc_array = malloc_array
+        self.GC_MALLOC_ARRAY = lltype.Ptr(lltype.FuncType(
+            [lltype.Signed] * 4, llmemory.GCREF))
+
+
+        (str_basesize, str_itemsize, str_ofs_length
+         ) = symbolic.get_array_token(rstr.STR, self.translate_support_code)
+        (unicode_basesize, unicode_itemsize, unicode_ofs_length
+         ) = symbolic.get_array_token(rstr.UNICODE, self.translate_support_code)
+        def malloc_str(length):
+            return self.malloc_array(
+                str_basesize, str_itemsize, str_ofs_length, length
+            )
+        def malloc_unicode(length):
+            return self.malloc_array(
+                unicode_basesize, unicode_itemsize, unicode_ofs_length, length
+            )
+        self.malloc_str = malloc_str
+        self.malloc_unicode = malloc_unicode
+        self.GC_MALLOC_STR_UNICODE = lltype.Ptr(lltype.FuncType(
+            [lltype.Signed], llmemory.GCREF))
+
+
         # on some platform GC_init is required before any other
         # GC_* functions, call it here for the benefit of tests
         # XXX move this to tests
@@ -116,39 +157,27 @@
         ofs_length = arraydescr.get_ofs_length(self.translate_support_code)
         basesize = arraydescr.get_base_size(self.translate_support_code)
         itemsize = arraydescr.get_item_size(self.translate_support_code)
-        size = basesize + itemsize * num_elem
-        res = self.funcptr_for_new(size)
-        rffi.cast(rffi.CArrayPtr(lltype.Signed), res)[ofs_length/WORD] = num_elem
-        return res
+        return self.malloc_array(basesize, itemsize, ofs_length, num_elem)
 
     def gc_malloc_str(self, num_elem):
-        basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR,
-                                                   self.translate_support_code)
-        assert itemsize == 1
-        size = basesize + num_elem
-        res = self.funcptr_for_new(size)
-        rffi.cast(rffi.CArrayPtr(lltype.Signed), res)[ofs_length/WORD] = num_elem
-        return res
+        return self.malloc_str(num_elem)
 
     def gc_malloc_unicode(self, num_elem):
-        basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE,
-                                                   self.translate_support_code)
-        size = basesize + num_elem * itemsize
-        res = self.funcptr_for_new(size)
-        rffi.cast(rffi.CArrayPtr(lltype.Signed), res)[ofs_length/WORD] = num_elem
-        return res
+        return self.malloc_unicode(num_elem)
 
     def args_for_new(self, sizedescr):
         assert isinstance(sizedescr, BaseSizeDescr)
         return [sizedescr.size]
 
+    def args_for_new_array(self, arraydescr):
+        ofs_length = arraydescr.get_ofs_length(self.translate_support_code)
+        basesize = arraydescr.get_base_size(self.translate_support_code)
+        itemsize = arraydescr.get_item_size(self.translate_support_code)
+        return [basesize, itemsize, ofs_length]
+
     def get_funcptr_for_new(self):
         return self.funcptr_for_new
 
-    get_funcptr_for_newarray = None
-    get_funcptr_for_newstr = None
-    get_funcptr_for_newunicode = None
-
     def rewrite_assembler(self, cpu, operations, gcrefs_output_list):
         # record all GCREFs too, because Boehm cannot see them and keep them
         # alive if they end up as constants in the assembler
@@ -752,15 +781,6 @@
     def get_funcptr_for_new(self):
         return llhelper(self.GC_MALLOC_BASIC, self.malloc_basic)
 
-    def get_funcptr_for_newarray(self):
-        return llhelper(self.GC_MALLOC_ARRAY, self.malloc_array)
-
-    def get_funcptr_for_newstr(self):
-        return llhelper(self.GC_MALLOC_STR_UNICODE, self.malloc_str)
-
-    def get_funcptr_for_newunicode(self):
-        return llhelper(self.GC_MALLOC_STR_UNICODE, self.malloc_unicode)
-
     def do_write_barrier(self, gcref_struct, gcref_newptr):
         hdr_addr = llmemory.cast_ptr_to_adr(gcref_struct)
         hdr_addr -= self.gcheaderbuilder.size_gc_header
diff --git a/pypy/jit/backend/llsupport/llmodel.py b/pypy/jit/backend/llsupport/llmodel.py
--- a/pypy/jit/backend/llsupport/llmodel.py
+++ b/pypy/jit/backend/llsupport/llmodel.py
@@ -1,24 +1,18 @@
-import sys
 from pypy.rpython.lltypesystem import lltype, llmemory, rffi, rclass, rstr
 from pypy.rpython.lltypesystem.lloperation import llop
-from pypy.rpython.llinterp import LLInterpreter, LLException
+from pypy.rpython.llinterp import LLInterpreter
 from pypy.rpython.annlowlevel import llhelper
 from pypy.rlib.objectmodel import we_are_translated, specialize
-from pypy.jit.metainterp.history import BoxInt, BoxPtr, set_future_values,\
-     BoxFloat
 from pypy.jit.metainterp import history
 from pypy.jit.codewriter import heaptracker, longlong
 from pypy.jit.backend.model import AbstractCPU
 from pypy.jit.backend.llsupport import symbolic
 from pypy.jit.backend.llsupport.symbolic import WORD, unroll_basic_sizes
-from pypy.jit.backend.llsupport.descr import get_size_descr,  BaseSizeDescr
-from pypy.jit.backend.llsupport.descr import get_field_descr, BaseFieldDescr
-from pypy.jit.backend.llsupport.descr import get_array_descr, BaseArrayDescr
-from pypy.jit.backend.llsupport.descr import get_call_descr
-from pypy.jit.backend.llsupport.descr import BaseIntCallDescr, GcPtrCallDescr
-from pypy.jit.backend.llsupport.descr import FloatCallDescr, VoidCallDescr
+from pypy.jit.backend.llsupport.descr import (get_size_descr,
+     get_field_descr, BaseFieldDescr, get_array_descr, BaseArrayDescr,
+     get_call_descr, BaseIntCallDescr, GcPtrCallDescr, FloatCallDescr,
+     VoidCallDescr, InteriorFieldDescr, get_interiorfield_descr)
 from pypy.jit.backend.llsupport.asmmemmgr import AsmMemoryManager
-from pypy.rpython.annlowlevel import cast_instance_to_base_ptr
 
 
 class AbstractLLCPU(AbstractCPU):
@@ -241,6 +235,9 @@
     def arraydescrof(self, A):
         return get_array_descr(self.gc_ll_descr, A)
 
+    def interiorfielddescrof(self, A, fieldname):
+        return get_interiorfield_descr(self.gc_ll_descr, A, A.OF, fieldname)
+
     def unpack_arraydescr(self, arraydescr):
         assert isinstance(arraydescr, BaseArrayDescr)
         return arraydescr.get_base_size(self.translate_support_code)
@@ -358,6 +355,100 @@
     bh_getarrayitem_raw_i = bh_getarrayitem_gc_i
     bh_getarrayitem_raw_f = bh_getarrayitem_gc_f
 
+    def bh_getinteriorfield_gc_i(self, gcref, itemindex, descr):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        fieldsize = descr.fielddescr.get_field_size(self.translate_support_code)
+        sign = descr.fielddescr.is_field_signed()
+        fullofs = itemindex * size + ofs
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), fullofs)
+        for STYPE, UTYPE, itemsize in unroll_basic_sizes:
+            if fieldsize == itemsize:
+                if sign:
+                    item = rffi.cast(rffi.CArrayPtr(STYPE), items)
+                    val = item[0]
+                    val = rffi.cast(lltype.Signed, val)
+                else:
+                    item = rffi.cast(rffi.CArrayPtr(UTYPE), items)
+                    val = item[0]
+                    val = rffi.cast(lltype.Signed, val)
+                # --- end of GC unsafe code ---
+                return val
+        else:
+            raise NotImplementedError("size = %d" % fieldsize)
+
+    def bh_getinteriorfield_gc_r(self, gcref, itemindex, descr):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs +
+                            size * itemindex)
+        items = rffi.cast(rffi.CArrayPtr(lltype.Signed), items)
+        pval = self._cast_int_to_gcref(items[0])
+        # --- end of GC unsafe code ---
+        return pval
+
+    def bh_getinteriorfield_gc_f(self, gcref, itemindex, descr):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs +
+                            size * itemindex)
+        items = rffi.cast(rffi.CArrayPtr(longlong.FLOATSTORAGE), items)
+        fval = items[0]
+        # --- end of GC unsafe code ---
+        return fval
+
+    def bh_setinteriorfield_gc_i(self, gcref, itemindex, descr, value):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        fieldsize = descr.fielddescr.get_field_size(self.translate_support_code)
+        ofs = itemindex * size + ofs
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs)
+        for TYPE, _, itemsize in unroll_basic_sizes:
+            if fieldsize == itemsize:
+                items = rffi.cast(rffi.CArrayPtr(TYPE), items)
+                items[0] = rffi.cast(TYPE, value)
+                # --- end of GC unsafe code ---
+                return
+        else:
+            raise NotImplementedError("size = %d" % fieldsize)
+
+    def bh_setinteriorfield_gc_r(self, gcref, itemindex, descr, newvalue):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        self.gc_ll_descr.do_write_barrier(gcref, newvalue)
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref),
+                            ofs + size * itemindex)
+        items = rffi.cast(rffi.CArrayPtr(lltype.Signed), items)
+        items[0] = self.cast_gcref_to_int(newvalue)
+        # --- end of GC unsafe code ---
+
+    def bh_setinteriorfield_gc_f(self, gcref, itemindex, descr, newvalue):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref),
+                            ofs + size * itemindex)
+        items = rffi.cast(rffi.CArrayPtr(longlong.FLOATSTORAGE), items)
+        items[0] = newvalue
+        # --- end of GC unsafe code ---
+
     def bh_strlen(self, string):
         s = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), string)
         return len(s.chars)
@@ -475,7 +566,6 @@
 
     def bh_classof(self, struct):
         struct = lltype.cast_opaque_ptr(rclass.OBJECTPTR, struct)
-        result = struct.typeptr
         result_adr = llmemory.cast_ptr_to_adr(struct.typeptr)
         return heaptracker.adr2int(result_adr)
 
diff --git a/pypy/jit/backend/llsupport/regalloc.py b/pypy/jit/backend/llsupport/regalloc.py
--- a/pypy/jit/backend/llsupport/regalloc.py
+++ b/pypy/jit/backend/llsupport/regalloc.py
@@ -287,7 +287,6 @@
         self.reg_bindings[to_v] = reg
 
     def _move_variable_away(self, v, prev_loc):
-        reg = None
         if self.free_regs:
             loc = self.free_regs.pop()
             self.reg_bindings[v] = loc
diff --git a/pypy/jit/backend/llsupport/test/test_asmmemmgr.py b/pypy/jit/backend/llsupport/test/test_asmmemmgr.py
--- a/pypy/jit/backend/llsupport/test/test_asmmemmgr.py
+++ b/pypy/jit/backend/llsupport/test/test_asmmemmgr.py
@@ -211,14 +211,14 @@
     debug._log = debug.DebugLog()
     try:
         mc._dump(addr, 'test-logname-section')
-        log = list(debug._log)
+        log = list(debug._log) 
     finally:
         debug._log = None
     encoded = ''.join(writtencode).encode('hex').upper()
     ataddr = '@%x' % addr
     assert log == [('test-logname-section',
                     [('debug_print', 'CODE_DUMP', ataddr, '+0 ', encoded)])]
-    #
+    
     lltype.free(p, flavor='raw')
 
 def test_blockbuildermixin2():
diff --git a/pypy/jit/backend/llsupport/test/test_descr.py b/pypy/jit/backend/llsupport/test/test_descr.py
--- a/pypy/jit/backend/llsupport/test/test_descr.py
+++ b/pypy/jit/backend/llsupport/test/test_descr.py
@@ -3,7 +3,6 @@
 from pypy.jit.backend.llsupport import symbolic
 from pypy.rlib.objectmodel import Symbolic
 from pypy.rpython.annlowlevel import llhelper
-from pypy.jit.metainterp.history import BoxInt, BoxFloat, BoxPtr
 from pypy.jit.metainterp import history
 from pypy.jit.codewriter import longlong
 import sys, struct, py
@@ -149,7 +148,9 @@
     A2 = lltype.GcArray(lltype.Ptr(T))
     A3 = lltype.GcArray(lltype.Ptr(U))
     A4 = lltype.GcArray(lltype.Float)
-    A5 = lltype.GcArray(lltype.SingleFloat)
+    A5 = lltype.GcArray(lltype.Struct('x', ('v', lltype.Signed),
+                                      ('k', lltype.Signed)))
+    A6 = lltype.GcArray(lltype.SingleFloat)
     assert getArrayDescrClass(A2) is GcPtrArrayDescr
     assert getArrayDescrClass(A3) is NonGcPtrArrayDescr
     cls = getArrayDescrClass(A1)
@@ -158,7 +159,7 @@
     clsf = getArrayDescrClass(A4)
     assert clsf != cls
     assert clsf == getArrayDescrClass(lltype.GcArray(lltype.Float))
-    clss = getArrayDescrClass(A5)
+    clss = getArrayDescrClass(A6)
     assert clss not in (clsf, cls)
     assert clss == getArrayDescrClass(lltype.GcArray(rffi.UINT))
     #
@@ -168,11 +169,12 @@
     descr3 = get_array_descr(c0, A3)
     descr4 = get_array_descr(c0, A4)
     descr5 = get_array_descr(c0, A5)
+    descr6 = get_array_descr(c0, A6)
     assert descr1.__class__ is cls
     assert descr2.__class__ is GcPtrArrayDescr
     assert descr3.__class__ is NonGcPtrArrayDescr
     assert descr4.__class__ is clsf
-    assert descr5.__class__ is clss
+    assert descr6.__class__ is clss
     assert descr1 == get_array_descr(c0, lltype.GcArray(lltype.Char))
     assert not descr1.is_array_of_pointers()
     assert     descr2.is_array_of_pointers()
@@ -202,7 +204,8 @@
     assert descr2.get_item_size(False) == rffi.sizeof(lltype.Ptr(T))
     assert descr3.get_item_size(False) == rffi.sizeof(lltype.Ptr(U))
     assert descr4.get_item_size(False) == rffi.sizeof(lltype.Float)
-    assert descr5.get_item_size(False) == rffi.sizeof(lltype.SingleFloat)
+    assert descr5.get_item_size(False) == rffi.sizeof(lltype.Signed) * 2
+    assert descr6.get_item_size(False) == rffi.sizeof(lltype.SingleFloat)
     #
     assert isinstance(descr1.get_base_size(True), Symbolic)
     assert isinstance(descr2.get_base_size(True), Symbolic)
@@ -348,7 +351,6 @@
                             (rffi.SHORT,      True), (rffi.USHORT, False),
                             (rffi.INT,        True), (rffi.UINT,   False),
                             (rffi.LONG,       True), (rffi.ULONG,  False)]:
-        A = lltype.GcArray(RESTYPE)
         for tsc in [False, True]:
             c2 = GcCache(tsc)
             descr1 = get_call_descr(c2, [], RESTYPE)
@@ -379,7 +381,6 @@
     descr3i = get_array_descr(c0, lltype.GcArray(lltype.Char))
     assert descr3i.repr_of_descr() == '<CharArrayDescr>'
     #
-    cache = {}
     descr4 = get_call_descr(c0, [lltype.Char, lltype.Ptr(S)], lltype.Ptr(S))
     assert 'GcPtrCallDescr' in descr4.repr_of_descr()
     #
@@ -412,10 +413,10 @@
     ARGS = [lltype.Float, lltype.Ptr(ARRAY)]
     RES = lltype.Float
 
-    def f(a, b):
+    def f2(a, b):
         return float(b[0]) + a
 
-    fnptr = llhelper(lltype.Ptr(lltype.FuncType(ARGS, RES)), f)
+    fnptr = llhelper(lltype.Ptr(lltype.FuncType(ARGS, RES)), f2)
     descr2 = get_call_descr(c0, ARGS, RES)
     a = lltype.malloc(ARRAY, 3)
     opaquea = lltype.cast_opaque_ptr(llmemory.GCREF, a)
diff --git a/pypy/jit/backend/model.py b/pypy/jit/backend/model.py
--- a/pypy/jit/backend/model.py
+++ b/pypy/jit/backend/model.py
@@ -1,5 +1,5 @@
 from pypy.rlib.debug import debug_start, debug_print, debug_stop
-from pypy.jit.metainterp import history, compile
+from pypy.jit.metainterp import history
 
 
 class AbstractCPU(object):
@@ -213,6 +213,10 @@
     def typedescrof(TYPE):
         raise NotImplementedError
 
+    @staticmethod
+    def interiorfielddescrof(A, fieldname):
+        raise NotImplementedError
+
     # ---------- the backend-dependent operations ----------
 
     # lltype specific operations
diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -5,7 +5,7 @@
                                          BoxInt, Box, BoxPtr,
                                          LoopToken,
                                          ConstInt, ConstPtr,
-                                         BoxObj, Const,
+                                         BoxObj,
                                          ConstObj, BoxFloat, ConstFloat)
 from pypy.jit.metainterp.resoperation import ResOperation, rop
 from pypy.jit.metainterp.typesystem import deref
@@ -111,7 +111,7 @@
         self.cpu.set_future_value_int(0, 2)
         fail = self.cpu.execute_token(looptoken)
         res = self.cpu.get_latest_value_int(0)
-        assert res == 3        
+        assert res == 3
         assert fail.identifier == 1
 
     def test_compile_loop(self):
@@ -127,7 +127,7 @@
             ]
         inputargs = [i0]
         operations[2].setfailargs([i1])
-        
+
         self.cpu.compile_loop(inputargs, operations, looptoken)
         self.cpu.set_future_value_int(0, 2)
         fail = self.cpu.execute_token(looptoken)
@@ -148,7 +148,7 @@
             ]
         inputargs = [i0]
         operations[2].setfailargs([None, None, i1, None])
-        
+
         self.cpu.compile_loop(inputargs, operations, looptoken)
         self.cpu.set_future_value_int(0, 2)
         fail = self.cpu.execute_token(looptoken)
@@ -372,7 +372,7 @@
         for opnum, boxargs, retvalue in get_int_tests():
             res = self.execute_operation(opnum, boxargs, 'int')
             assert res.value == retvalue
-        
+
     def test_float_operations(self):
         from pypy.jit.metainterp.test.test_executor import get_float_tests
         for opnum, boxargs, rettype, retvalue in get_float_tests(self.cpu):
@@ -438,7 +438,7 @@
 
     def test_ovf_operations_reversed(self):
         self.test_ovf_operations(reversed=True)
-        
+
     def test_bh_call(self):
         cpu = self.cpu
         #
@@ -503,7 +503,7 @@
                                          [funcbox, BoxInt(num), BoxInt(num)],
                                          'int', descr=dyn_calldescr)
             assert res.value == 2 * num
-            
+
 
         if cpu.supports_floats:
             def func(f0, f1, f2, f3, f4, f5, f6, i0, i1, f7, f8, f9):
@@ -543,7 +543,7 @@
         funcbox = self.get_funcbox(self.cpu, func_ptr)
         res = self.execute_operation(rop.CALL, [funcbox] + map(BoxInt, args), 'int', descr=calldescr)
         assert res.value == func(*args)
-        
+
     def test_call_stack_alignment(self):
         # test stack alignment issues, notably for Mac OS/X.
         # also test the ordering of the arguments.
@@ -615,7 +615,7 @@
         res = self.execute_operation(rop.GETFIELD_GC, [t_box],
                                      'int', descr=shortdescr)
         assert res.value == 1331
-        
+
         #
         u_box, U_box = self.alloc_instance(self.U)
         fielddescr2 = self.cpu.fielddescrof(self.S, 'next')
@@ -695,7 +695,7 @@
 
     def test_failing_guard_class(self):
         t_box, T_box = self.alloc_instance(self.T)
-        u_box, U_box = self.alloc_instance(self.U)        
+        u_box, U_box = self.alloc_instance(self.U)
         null_box = self.null_instance()
         for opname, args in [(rop.GUARD_CLASS, [t_box, U_box]),
                              (rop.GUARD_CLASS, [u_box, T_box]),
@@ -787,7 +787,7 @@
         r = self.execute_operation(rop.GETARRAYITEM_GC, [a_box, BoxInt(3)],
                                    'int', descr=arraydescr)
         assert r.value == 160
-        
+
         #
         if isinstance(A, lltype.GcArray):
             A = lltype.Ptr(A)
@@ -880,6 +880,73 @@
                                    'int', descr=arraydescr)
         assert r.value == 7441
 
+    def test_array_of_structs(self):
+        TP = lltype.GcStruct('x')
+        ITEM = lltype.Struct('x',
+                             ('vs', lltype.Signed),
+                             ('vu', lltype.Unsigned),
+                             ('vsc', rffi.SIGNEDCHAR),
+                             ('vuc', rffi.UCHAR),
+                             ('vss', rffi.SHORT),
+                             ('vus', rffi.USHORT),
+                             ('vsi', rffi.INT),
+                             ('vui', rffi.UINT),
+                             ('k', lltype.Float),
+                             ('p', lltype.Ptr(TP)))
+        a_box, A = self.alloc_array_of(ITEM, 15)
+        s_box, S = self.alloc_instance(TP)
+        kdescr = self.cpu.interiorfielddescrof(A, 'k')
+        pdescr = self.cpu.interiorfielddescrof(A, 'p')
+        self.execute_operation(rop.SETINTERIORFIELD_GC, [a_box, BoxInt(3),
+                                                         boxfloat(1.5)],
+                               'void', descr=kdescr)
+        f = self.cpu.bh_getinteriorfield_gc_f(a_box.getref_base(), 3, kdescr)
+        assert longlong.getrealfloat(f) == 1.5
+        self.cpu.bh_setinteriorfield_gc_f(a_box.getref_base(), 3, kdescr, longlong.getfloatstorage(2.5))
+        r = self.execute_operation(rop.GETINTERIORFIELD_GC, [a_box, BoxInt(3)],
+                                   'float', descr=kdescr)
+        assert r.getfloat() == 2.5
+        #
+        NUMBER_FIELDS = [('vs', lltype.Signed),
+                         ('vu', lltype.Unsigned),
+                         ('vsc', rffi.SIGNEDCHAR),
+                         ('vuc', rffi.UCHAR),
+                         ('vss', rffi.SHORT),
+                         ('vus', rffi.USHORT),
+                         ('vsi', rffi.INT),
+                         ('vui', rffi.UINT)]
+        for name, TYPE in NUMBER_FIELDS[::-1]:
+            vdescr = self.cpu.interiorfielddescrof(A, name)
+            self.execute_operation(rop.SETINTERIORFIELD_GC, [a_box, BoxInt(3),
+                                                             BoxInt(-15)],
+                                   'void', descr=vdescr)
+        for name, TYPE in NUMBER_FIELDS:
+            vdescr = self.cpu.interiorfielddescrof(A, name)
+            i = self.cpu.bh_getinteriorfield_gc_i(a_box.getref_base(), 3,
+                                                  vdescr)
+            assert i == rffi.cast(lltype.Signed, rffi.cast(TYPE, -15))
+        for name, TYPE in NUMBER_FIELDS[::-1]:
+            vdescr = self.cpu.interiorfielddescrof(A, name)
+            self.cpu.bh_setinteriorfield_gc_i(a_box.getref_base(), 3,
+                                              vdescr, -25)
+        for name, TYPE in NUMBER_FIELDS:
+            vdescr = self.cpu.interiorfielddescrof(A, name)
+            r = self.execute_operation(rop.GETINTERIORFIELD_GC,
+                                       [a_box, BoxInt(3)],
+                                       'int', descr=vdescr)
+            assert r.getint() == rffi.cast(lltype.Signed, rffi.cast(TYPE, -25))
+        #
+        self.execute_operation(rop.SETINTERIORFIELD_GC, [a_box, BoxInt(4),
+                                                         s_box],
+                               'void', descr=pdescr)
+        r = self.cpu.bh_getinteriorfield_gc_r(a_box.getref_base(), 4, pdescr)
+        assert r == s_box.getref_base()
+        self.cpu.bh_setinteriorfield_gc_r(a_box.getref_base(), 3, pdescr,
+                                          s_box.getref_base())
+        r = self.execute_operation(rop.GETINTERIORFIELD_GC, [a_box, BoxInt(3)],
+                                   'ref', descr=pdescr)
+        assert r.getref_base() == s_box.getref_base()
+
     def test_string_basic(self):
         s_box = self.alloc_string("hello\xfe")
         r = self.execute_operation(rop.STRLEN, [s_box], 'int')
@@ -1402,7 +1469,7 @@
         addr = llmemory.cast_ptr_to_adr(func_ptr)
         return ConstInt(heaptracker.adr2int(addr))
 
-    
+
     MY_VTABLE = rclass.OBJECT_VTABLE    # for tests only
 
     S = lltype.GcForwardReference()
@@ -1439,7 +1506,6 @@
         return BoxPtr(lltype.nullptr(llmemory.GCREF.TO))
 
     def alloc_array_of(self, ITEM, length):
-        cpu = self.cpu
         A = lltype.GcArray(ITEM)
         a = lltype.malloc(A, length)
         a_box = BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, a))
@@ -2318,7 +2384,7 @@
         for opname, arg, res in ops:
             self.execute_operation(opname, [arg], 'void')
             assert self.guard_failed == res
-        
+
         lltype.free(x, flavor='raw')
 
     def test_assembler_call(self):
@@ -2398,7 +2464,7 @@
         FakeJitDriverSD.portal_calldescr = self.cpu.calldescrof(
             lltype.Ptr(lltype.FuncType(ARGS, RES)), ARGS, RES,
             EffectInfo.MOST_GENERAL)
-        
+
         ops = '''
         [f0, f1]
         f2 = float_add(f0, f1)
@@ -2489,7 +2555,7 @@
         FakeJitDriverSD.portal_calldescr = self.cpu.calldescrof(
             lltype.Ptr(lltype.FuncType(ARGS, RES)), ARGS, RES,
             EffectInfo.MOST_GENERAL)
-        
+
         ops = '''
         [f0, f1]
         f2 = float_add(f0, f1)
@@ -2940,4 +3006,4 @@
 
     def alloc_unicode(self, unicode):
         py.test.skip("implement me")
-    
+
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1,7 +1,7 @@
 import sys, os
 from pypy.jit.backend.llsupport import symbolic
 from pypy.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper
-from pypy.jit.metainterp.history import Const, Box, BoxInt, BoxPtr, BoxFloat
+from pypy.jit.metainterp.history import Const, Box, BoxInt, ConstInt
 from pypy.jit.metainterp.history import (AbstractFailDescr, INT, REF, FLOAT,
                                          LoopToken)
 from pypy.rpython.lltypesystem import lltype, rffi, rstr, llmemory
@@ -36,7 +36,6 @@
 from pypy.rlib import rgc
 from pypy.rlib.clibffi import FFI_DEFAULT_ABI
 from pypy.jit.backend.x86.jump import remap_frame_layout
-from pypy.jit.metainterp.history import ConstInt, BoxInt
 from pypy.jit.codewriter.effectinfo import EffectInfo
 from pypy.jit.codewriter import longlong
 
@@ -729,8 +728,8 @@
         # Also, make sure this is consistent with FRAME_FIXED_SIZE.
         self.mc.PUSH_r(ebp.value)
         self.mc.MOV_rr(ebp.value, esp.value)
-        for regloc in self.cpu.CALLEE_SAVE_REGISTERS:
-            self.mc.PUSH_r(regloc.value)
+        for loc in self.cpu.CALLEE_SAVE_REGISTERS:
+            self.mc.PUSH_r(loc.value)
 
         gcrootmap = self.cpu.gc_ll_descr.gcrootmap
         if gcrootmap and gcrootmap.is_shadow_stack:
@@ -994,7 +993,7 @@
         effectinfo = op.getdescr().get_extra_info()
         oopspecindex = effectinfo.oopspecindex
         genop_llong_list[oopspecindex](self, op, arglocs, resloc)
-        
+
     def regalloc_perform_math(self, op, arglocs, resloc):
         effectinfo = op.getdescr().get_extra_info()
         oopspecindex = effectinfo.oopspecindex
@@ -1311,7 +1310,7 @@
     genop_guard_float_eq = _cmpop_guard_float("E", "E", "NE","NE")
     genop_guard_float_gt = _cmpop_guard_float("A", "B", "BE","AE")
     genop_guard_float_ge = _cmpop_guard_float("AE","BE", "B", "A")
-    
+
     def genop_math_sqrt(self, op, arglocs, resloc):
         self.mc.SQRTSD(arglocs[0], resloc)
 
@@ -1597,12 +1596,27 @@
     genop_getarrayitem_gc_pure = genop_getarrayitem_gc
     genop_getarrayitem_raw = genop_getarrayitem_gc
 
+    def genop_getinteriorfield_gc(self, op, arglocs, resloc):
+        base_loc, ofs_loc, itemsize_loc, fieldsize_loc, index_loc, sign_loc = arglocs
+        # XXX should not use IMUL in most cases
+        self.mc.IMUL(index_loc, itemsize_loc)
+        src_addr = AddressLoc(base_loc, index_loc, 0, ofs_loc.value)
+        self.load_from_mem(resloc, src_addr, fieldsize_loc, sign_loc)
+
+
     def genop_discard_setfield_gc(self, op, arglocs):
         base_loc, ofs_loc, size_loc, value_loc = arglocs
         assert isinstance(size_loc, ImmedLoc)
         dest_addr = AddressLoc(base_loc, ofs_loc)
         self.save_into_mem(dest_addr, value_loc, size_loc)
 
+    def genop_discard_setinteriorfield_gc(self, op, arglocs):
+        base_loc, ofs_loc, itemsize_loc, fieldsize_loc, index_loc, value_loc = arglocs
+        # XXX should not use IMUL in most cases
+        self.mc.IMUL(index_loc, itemsize_loc)
+        dest_addr = AddressLoc(base_loc, index_loc, 0, ofs_loc.value)
+        self.save_into_mem(dest_addr, value_loc, fieldsize_loc)
+
     def genop_discard_setarrayitem_gc(self, op, arglocs):
         base_loc, ofs_loc, value_loc, size_loc, baseofs = arglocs
         assert isinstance(baseofs, ImmedLoc)
diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py
--- a/pypy/jit/backend/x86/regalloc.py
+++ b/pypy/jit/backend/x86/regalloc.py
@@ -7,7 +7,7 @@
                                          ResOperation, BoxPtr, ConstFloat,
                                          BoxFloat, LoopToken, INT, REF, FLOAT)
 from pypy.jit.backend.x86.regloc import *
-from pypy.rpython.lltypesystem import lltype, ll2ctypes, rffi, rstr
+from pypy.rpython.lltypesystem import lltype, rffi, rstr
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.rlib import rgc
 from pypy.jit.backend.llsupport import symbolic
@@ -17,11 +17,12 @@
 from pypy.jit.metainterp.resoperation import rop
 from pypy.jit.backend.llsupport.descr import BaseFieldDescr, BaseArrayDescr
 from pypy.jit.backend.llsupport.descr import BaseCallDescr, BaseSizeDescr
+from pypy.jit.backend.llsupport.descr import InteriorFieldDescr
 from pypy.jit.backend.llsupport.regalloc import FrameManager, RegisterManager,\
      TempBox
 from pypy.jit.backend.x86.arch import WORD, FRAME_FIXED_SIZE
 from pypy.jit.backend.x86.arch import IS_X86_32, IS_X86_64, MY_COPY_OF_REGS
-from pypy.rlib.rarithmetic import r_longlong, r_uint
+from pypy.rlib.rarithmetic import r_longlong
 
 class X86RegisterManager(RegisterManager):
 
@@ -433,7 +434,7 @@
             if self.can_merge_with_next_guard(op, i, operations):
                 oplist_with_guard[op.getopnum()](self, op, operations[i + 1])
                 i += 1
-            elif not we_are_translated() and op.getopnum() == -124: 
+            elif not we_are_translated() and op.getopnum() == -124:
                 self._consider_force_spill(op)
             else:
                 oplist[op.getopnum()](self, op)
@@ -815,7 +816,7 @@
         save_all_regs = guard_not_forced_op is not None
         self.xrm.before_call(force_store, save_all_regs=save_all_regs)
         if not save_all_regs:
-            gcrootmap = gc_ll_descr = self.assembler.cpu.gc_ll_descr.gcrootmap
+            gcrootmap = self.assembler.cpu.gc_ll_descr.gcrootmap
             if gcrootmap and gcrootmap.is_shadow_stack:
                 save_all_regs = 2
         self.rm.before_call(force_store, save_all_regs=save_all_regs)
@@ -972,74 +973,27 @@
             return self._call(op, arglocs)
 
     def consider_newstr(self, op):
-        gc_ll_descr = self.assembler.cpu.gc_ll_descr
-        if gc_ll_descr.get_funcptr_for_newstr is not None:
-            # framework GC
-            loc = self.loc(op.getarg(0))
-            return self._call(op, [loc])
-        # boehm GC (XXX kill the following code at some point)
-        ofs_items, itemsize, ofs = symbolic.get_array_token(rstr.STR, self.translate_support_code)
-        assert itemsize == 1
-        return self._malloc_varsize(ofs_items, ofs, 0, op.getarg(0),
-                                    op.result)
+        loc = self.loc(op.getarg(0))
+        return self._call(op, [loc])
 
     def consider_newunicode(self, op):
-        gc_ll_descr = self.assembler.cpu.gc_ll_descr
-        if gc_ll_descr.get_funcptr_for_newunicode is not None:
-            # framework GC
-            loc = self.loc(op.getarg(0))
-            return self._call(op, [loc])
-        # boehm GC (XXX kill the following code at some point)
-        ofs_items, _, ofs = symbolic.get_array_token(rstr.UNICODE,
-                                                   self.translate_support_code)
-        scale = self._get_unicode_item_scale()
-        return self._malloc_varsize(ofs_items, ofs, scale, op.getarg(0),
-                                    op.result)
-
-    def _malloc_varsize(self, ofs_items, ofs_length, scale, v, res_v):
-        # XXX kill this function at some point
-        if isinstance(v, Box):
-            loc = self.rm.make_sure_var_in_reg(v, [v])
-            tempbox = TempBox()
-            other_loc = self.rm.force_allocate_reg(tempbox, [v])
-            self.assembler.load_effective_addr(loc, ofs_items,scale, other_loc)
-        else:
-            tempbox = None
-            other_loc = imm(ofs_items + (v.getint() << scale))
-        self._call(ResOperation(rop.NEW, [], res_v),
-                   [other_loc], [v])
-        loc = self.rm.make_sure_var_in_reg(v, [res_v])
-        assert self.loc(res_v) == eax
-        # now we have to reload length to some reasonable place
-        self.rm.possibly_free_var(v)
-        if tempbox is not None:
-            self.rm.possibly_free_var(tempbox)
-        self.PerformDiscard(ResOperation(rop.SETFIELD_GC, [None, None], None),
-                            [eax, imm(ofs_length), imm(WORD), loc])
+        loc = self.loc(op.getarg(0))
+        return self._call(op, [loc])
 
     def consider_new_array(self, op):
         gc_ll_descr = self.assembler.cpu.gc_ll_descr
-        if gc_ll_descr.get_funcptr_for_newarray is not None:
-            # framework GC
-            box_num_elem = op.getarg(0)
-            if isinstance(box_num_elem, ConstInt):
-                num_elem = box_num_elem.value
-                if gc_ll_descr.can_inline_malloc_varsize(op.getdescr(),
-                                                         num_elem):
-                    self.fastpath_malloc_varsize(op, op.getdescr(), num_elem)
-                    return
-            args = self.assembler.cpu.gc_ll_descr.args_for_new_array(
-                op.getdescr())
-            arglocs = [imm(x) for x in args]
-            arglocs.append(self.loc(box_num_elem))
-            self._call(op, arglocs)
-            return
-        # boehm GC (XXX kill the following code at some point)
-        itemsize, basesize, ofs_length, _, _ = (
-            self._unpack_arraydescr(op.getdescr()))
-        scale_of_field = _get_scale(itemsize)
-        self._malloc_varsize(basesize, ofs_length, scale_of_field,
-                             op.getarg(0), op.result)
+        box_num_elem = op.getarg(0)
+        if isinstance(box_num_elem, ConstInt):
+            num_elem = box_num_elem.value
+            if gc_ll_descr.can_inline_malloc_varsize(op.getdescr(),
+                                                     num_elem):
+                self.fastpath_malloc_varsize(op, op.getdescr(), num_elem)
+                return
+        args = self.assembler.cpu.gc_ll_descr.args_for_new_array(
+            op.getdescr())
+        arglocs = [imm(x) for x in args]
+        arglocs.append(self.loc(box_num_elem))
+        self._call(op, arglocs)
 
     def _unpack_arraydescr(self, arraydescr):
         assert isinstance(arraydescr, BaseArrayDescr)
@@ -1058,6 +1012,16 @@
         sign = fielddescr.is_field_signed()
         return imm(ofs), imm(size), ptr, sign
 
+    def _unpack_interiorfielddescr(self, descr):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs = arraydescr.get_base_size(self.translate_support_code)
+        itemsize = arraydescr.get_item_size(self.translate_support_code)
+        fieldsize = descr.fielddescr.get_field_size(self.translate_support_code)
+        sign = descr.fielddescr.is_field_signed()
+        ofs += descr.fielddescr.offset
+        return imm(ofs), imm(itemsize), imm(fieldsize), sign
+
     def consider_setfield_gc(self, op):
         ofs_loc, size_loc, _, _ = self._unpack_fielddescr(op.getdescr())
         assert isinstance(size_loc, ImmedLoc)
@@ -1074,6 +1038,21 @@
 
     consider_setfield_raw = consider_setfield_gc
 
+    def consider_setinteriorfield_gc(self, op):
+        t = self._unpack_interiorfielddescr(op.getdescr())
+        ofs, itemsize, fieldsize, _ = t
+        args = op.getarglist()
+        tmpvar = TempBox()
+        base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
+        index_loc = self.rm.force_result_in_reg(tmpvar, op.getarg(1),
+                                                args)
+        # we're free to modify index now
+        value_loc = self.make_sure_var_in_reg(op.getarg(2), args)
+        self.possibly_free_vars(args)
+        self.rm.possibly_free_var(tmpvar)
+        self.PerformDiscard(op, [base_loc, ofs, itemsize, fieldsize,
+                                 index_loc, value_loc])
+
     def consider_strsetitem(self, op):
         args = op.getarglist()
         base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
@@ -1135,6 +1114,24 @@
     consider_getarrayitem_raw = consider_getarrayitem_gc
     consider_getarrayitem_gc_pure = consider_getarrayitem_gc
 
+    def consider_getinteriorfield_gc(self, op):
+        t = self._unpack_interiorfielddescr(op.getdescr())
+        ofs, itemsize, fieldsize, sign = t
+        if sign:
+            sign_loc = imm1
+        else:
+            sign_loc = imm0
+        args = op.getarglist()
+        tmpvar = TempBox()
+        base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
+        index_loc = self.rm.force_result_in_reg(tmpvar, op.getarg(1),
+                                                args)
+        self.rm.possibly_free_vars_for_op(op)
+        self.rm.possibly_free_var(tmpvar)
+        result_loc = self.force_allocate_reg(op.result)
+        self.Perform(op, [base_loc, ofs, itemsize, fieldsize,
+                          index_loc, sign_loc], result_loc)
+
     def consider_int_is_true(self, op, guard_op):
         # doesn't need arg to be in a register
         argloc = self.loc(op.getarg(0))
@@ -1241,7 +1238,6 @@
         self.rm.possibly_free_var(srcaddr_box)
 
     def _gen_address_inside_string(self, baseloc, ofsloc, resloc, is_unicode):
-        cpu = self.assembler.cpu
         if is_unicode:
             ofs_items, _, _ = symbolic.get_array_token(rstr.UNICODE,
                                                   self.translate_support_code)
@@ -1300,7 +1296,7 @@
         tmpreg = X86RegisterManager.all_regs[0]
         tmploc = self.rm.force_allocate_reg(box, selected_reg=tmpreg)
         xmmtmp = X86XMMRegisterManager.all_regs[0]
-        xmmtmploc = self.xrm.force_allocate_reg(box1, selected_reg=xmmtmp)
+        self.xrm.force_allocate_reg(box1, selected_reg=xmmtmp)
         # Part about non-floats
         # XXX we don't need a copy, we only just the original list
         src_locations1 = [self.loc(op.getarg(i)) for i in range(op.numargs())
@@ -1380,7 +1376,7 @@
     return lambda self, op: fn(self, op, None)
 
 def is_comparison_or_ovf_op(opnum):
-    from pypy.jit.metainterp.resoperation import opclasses, AbstractResOp
+    from pypy.jit.metainterp.resoperation import opclasses
     cls = opclasses[opnum]
     # hack hack: in theory they are instance method, but they don't use
     # any instance field, we can use a fake object
diff --git a/pypy/jit/backend/x86/test/test_del.py b/pypy/jit/backend/x86/test/test_del.py
--- a/pypy/jit/backend/x86/test/test_del.py
+++ b/pypy/jit/backend/x86/test/test_del.py
@@ -1,5 +1,4 @@
 
-import py
 from pypy.jit.backend.x86.test.test_basic import Jit386Mixin
 from pypy.jit.metainterp.test.test_del import DelTests
 
diff --git a/pypy/jit/backend/x86/test/test_dict.py b/pypy/jit/backend/x86/test/test_dict.py
new file mode 100644
--- /dev/null
+++ b/pypy/jit/backend/x86/test/test_dict.py
@@ -0,0 +1,9 @@
+
+from pypy.jit.backend.x86.test.test_basic import Jit386Mixin
+from pypy.jit.metainterp.test.test_dict import DictTests
+
+
+class TestDict(Jit386Mixin, DictTests):
+    # for the individual tests see
+    # ====> ../../../metainterp/test/test_dict.py
+    pass
diff --git a/pypy/jit/backend/x86/test/test_runner.py b/pypy/jit/backend/x86/test/test_runner.py
--- a/pypy/jit/backend/x86/test/test_runner.py
+++ b/pypy/jit/backend/x86/test/test_runner.py
@@ -31,7 +31,7 @@
 
     # for the individual tests see
     # ====> ../../test/runner_test.py
-    
+
     def setup_method(self, meth):
         self.cpu = CPU(rtyper=None, stats=FakeStats())
         self.cpu.setup_once()
@@ -69,22 +69,16 @@
 
     def test_allocations(self):
         from pypy.rpython.lltypesystem import rstr
-        
+
         allocs = [None]
         all = []
+        orig_new = self.cpu.gc_ll_descr.funcptr_for_new
         def f(size):
             allocs.insert(0, size)
-            buf = ctypes.create_string_buffer(size)
-            all.append(buf)
-            return ctypes.cast(buf, ctypes.c_void_p).value
-        func = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int)(f)
-        addr = ctypes.cast(func, ctypes.c_void_p).value
-        # ctypes produces an unsigned value. We need it to be signed for, eg,
-        # relative addressing to work properly.
-        addr = rffi.cast(lltype.Signed, addr)
-        
+            return orig_new(size)
+
         self.cpu.assembler.setup_once()
-        self.cpu.assembler.malloc_func_addr = addr
+        self.cpu.gc_ll_descr.funcptr_for_new = f
         ofs = symbolic.get_field_token(rstr.STR, 'chars', False)[0]
 
         res = self.execute_operation(rop.NEWSTR, [ConstInt(7)], 'ref')
@@ -108,7 +102,7 @@
         res = self.execute_operation(rop.NEW_ARRAY, [ConstInt(10)],
                                          'ref', descr)
         assert allocs[0] == 10*WORD + ofs + WORD
-        resbuf = self._resbuf(res)            
+        resbuf = self._resbuf(res)
         assert resbuf[ofs/WORD] == 10
 
         # ------------------------------------------------------------
@@ -116,7 +110,7 @@
         res = self.execute_operation(rop.NEW_ARRAY, [BoxInt(10)],
                                          'ref', descr)
         assert allocs[0] == 10*WORD + ofs + WORD
-        resbuf = self._resbuf(res)                        
+        resbuf = self._resbuf(res)
         assert resbuf[ofs/WORD] == 10
 
     def test_stringitems(self):
@@ -146,7 +140,7 @@
                                                      ConstInt(2), BoxInt(38)],
                                'void', descr)
         assert resbuf[itemsofs/WORD + 2] == 38
-        
+
         self.execute_operation(rop.SETARRAYITEM_GC, [res,
                                                      BoxInt(3), BoxInt(42)],
                                'void', descr)
@@ -167,7 +161,7 @@
                                                          BoxInt(2)],
                                    'int', descr)
         assert r.value == 38
-        
+
         r = self.execute_operation(rop.GETARRAYITEM_GC, [res, BoxInt(3)],
                                    'int', descr)
         assert r.value == 42
@@ -226,7 +220,7 @@
         self.execute_operation(rop.SETFIELD_GC, [res, BoxInt(1234)], 'void', ofs_i)
         i = self.execute_operation(rop.GETFIELD_GC, [res], 'int', ofs_i)
         assert i.value == 1234
-        
+
         #u = self.execute_operation(rop.GETFIELD_GC, [res, ofs_u], 'int')
         #assert u.value == 5
         self.execute_operation(rop.SETFIELD_GC, [res, ConstInt(1)], 'void',
@@ -299,7 +293,7 @@
                     else:
                         assert result != execute(self.cpu, None,
                                                  op, None, b).value
-                    
+
 
     def test_stuff_followed_by_guard(self):
         boxes = [(BoxInt(1), BoxInt(0)),
@@ -523,7 +517,7 @@
     def test_debugger_on(self):
         from pypy.tool.logparser import parse_log_file, extract_category
         from pypy.rlib import debug
-        
+
         loop = """
         [i0]
         debug_merge_point('xyz', 0)
diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -52,9 +52,11 @@
         newoperations = []
         #
         def do_rename(var, var_or_const):
+            if var.concretetype is lltype.Void:
+                renamings[var] = Constant(None, lltype.Void)
+                return
             renamings[var] = var_or_const
-            if (isinstance(var_or_const, Constant)
-                and var.concretetype != lltype.Void):
+            if isinstance(var_or_const, Constant):
                 value = var_or_const.value
                 value = lltype._cast_whatever(var.concretetype, value)
                 renamings_constants[var] = Constant(value, var.concretetype)
@@ -735,29 +737,54 @@
         return SpaceOperation(opname, [op.args[0]], op.result)
 
     def rewrite_op_getinteriorfield(self, op):
-        # only supports strings and unicodes
         assert len(op.args) == 3
-        assert op.args[1].value == 'chars'
         optype = op.args[0].concretetype
         if optype == lltype.Ptr(rstr.STR):
             opname = "strgetitem"
+            return SpaceOperation(opname, [op.args[0], op.args[2]], op.result)
+        elif optype == lltype.Ptr(rstr.UNICODE):
+            opname = "unicodegetitem"
+            return SpaceOperation(opname, [op.args[0], op.args[2]], op.result)
         else:
-            assert optype == lltype.Ptr(rstr.UNICODE)
-            opname = "unicodegetitem"
-        return SpaceOperation(opname, [op.args[0], op.args[2]], op.result)
+            v_inst, v_index, c_field = op.args
+            if op.result.concretetype is lltype.Void:
+                return
+            # only GcArray of Struct supported
+            assert isinstance(v_inst.concretetype.TO, lltype.GcArray)
+            STRUCT = v_inst.concretetype.TO.OF
+            assert isinstance(STRUCT, lltype.Struct)
+            descr = self.cpu.interiorfielddescrof(v_inst.concretetype.TO,
+                                                  c_field.value)
+            args = [v_inst, v_index, descr]
+            kind = getkind(op.result.concretetype)[0]
+            return SpaceOperation('getinteriorfield_gc_%s' % kind, args,
+                                  op.result)
 
     def rewrite_op_setinteriorfield(self, op):
-        # only supports strings and unicodes
         assert len(op.args) == 4
-        assert op.args[1].value == 'chars'
         optype = op.args[0].concretetype
         if optype == lltype.Ptr(rstr.STR):
             opname = "strsetitem"
+            return SpaceOperation(opname, [op.args[0], op.args[2], op.args[3]],
+                                  op.result)
+        elif optype == lltype.Ptr(rstr.UNICODE):
+            opname = "unicodesetitem"
+            return SpaceOperation(opname, [op.args[0], op.args[2], op.args[3]],
+                                  op.result)
         else:
-            assert optype == lltype.Ptr(rstr.UNICODE)
-            opname = "unicodesetitem"
-        return SpaceOperation(opname, [op.args[0], op.args[2], op.args[3]],
-                              op.result)
+            v_inst, v_index, c_field, v_value = op.args
+            if v_value.concretetype is lltype.Void:
+                return
+            # only GcArray of Struct supported
+            assert isinstance(v_inst.concretetype.TO, lltype.GcArray)
+            STRUCT = v_inst.concretetype.TO.OF
+            assert isinstance(STRUCT, lltype.Struct)
+            descr = self.cpu.interiorfielddescrof(v_inst.concretetype.TO,
+                                                  c_field.value)
+            kind = getkind(v_value.concretetype)[0]
+            args = [v_inst, v_index, v_value, descr]
+            return SpaceOperation('setinteriorfield_gc_%s' % kind, args,
+                                  op.result)
 
     def _rewrite_equality(self, op, opname):
         arg0, arg1 = op.args
diff --git a/pypy/jit/codewriter/support.py b/pypy/jit/codewriter/support.py
--- a/pypy/jit/codewriter/support.py
+++ b/pypy/jit/codewriter/support.py
@@ -13,7 +13,6 @@
 from pypy.translator.simplify import get_funcobj
 from pypy.translator.unsimplify import split_block
 from pypy.objspace.flow.model import Constant
-from pypy import conftest
 from pypy.translator.translator import TranslationContext
 from pypy.annotation.policy import AnnotatorPolicy
 from pypy.annotation import model as annmodel
@@ -48,15 +47,13 @@
     a.build_types(func, argtypes, main_entry_point=True)
     rtyper = t.buildrtyper(type_system = type_system)
     rtyper.specialize()
-    if inline:
-        auto_inlining(t, threshold=inline)
+    #if inline:
+    #    auto_inlining(t, threshold=inline)
     if backendoptimize:
         from pypy.translator.backendopt.all import backend_optimizations
         backend_optimizations(t, inline_threshold=inline or 0,
                 remove_asserts=True, really_remove_asserts=True)
 
-    #if conftest.option.view:
-    #    t.view()
     return rtyper
 
 def getgraph(func, values):
@@ -456,6 +453,8 @@
         return LLtypeHelpers._dictnext_items(lltype.Ptr(RES), iter)
     _ll_1_dictiter_nextitems.need_result_type = True
 
+    _ll_1_dict_resize = ll_rdict.ll_dict_resize
+
     # ---------- strings and unicode ----------
 
     _ll_1_str_str2unicode = ll_rstr.LLHelpers.ll_str2unicode
diff --git a/pypy/jit/codewriter/test/test_jtransform.py b/pypy/jit/codewriter/test/test_jtransform.py
--- a/pypy/jit/codewriter/test/test_jtransform.py
+++ b/pypy/jit/codewriter/test/test_jtransform.py
@@ -1,4 +1,3 @@
-import py
 import random
 try:
     from itertools import product
@@ -16,13 +15,13 @@
 
 from pypy.objspace.flow.model import FunctionGraph, Block, Link
 from pypy.objspace.flow.model import SpaceOperation, Variable, Constant
-from pypy.jit.codewriter.jtransform import Transformer
-from pypy.jit.metainterp.history import getkind
-from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr, rlist
+from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr
 from pypy.rpython.lltypesystem.module import ll_math
 from pypy.translator.unsimplify import varoftype
 from pypy.jit.codewriter import heaptracker, effectinfo
 from pypy.jit.codewriter.flatten import ListOfKind
+from pypy.jit.codewriter.jtransform import Transformer
+from pypy.jit.metainterp.history import getkind
 
 def const(x):
     return Constant(x, lltype.typeOf(x))
@@ -37,6 +36,8 @@
         return ('calldescr', FUNC, ARGS, RESULT)
     def fielddescrof(self, STRUCT, name):
         return ('fielddescr', STRUCT, name)
+    def interiorfielddescrof(self, ARRAY, name):
+        return ('interiorfielddescr', ARRAY, name)
     def arraydescrof(self, ARRAY):
         return FakeDescr(('arraydescr', ARRAY))
     def sizeof(self, STRUCT):
@@ -539,7 +540,7 @@
 
 def test_rename_on_links():
     v1 = Variable()
-    v2 = Variable()
+    v2 = Variable(); v2.concretetype = llmemory.Address
     v3 = Variable()
     block = Block([v1])
     block.operations = [SpaceOperation('cast_pointer', [v1], v2)]
@@ -676,6 +677,22 @@
     assert op1.args == [v, v_index]
     assert op1.result == v_result
 
+def test_dict_getinteriorfield():
+    DICT = lltype.GcArray(lltype.Struct('ENTRY', ('v', lltype.Signed),
+                                        ('k', lltype.Signed)))
+    v = varoftype(lltype.Ptr(DICT))
+    i = varoftype(lltype.Signed)
+    v_result = varoftype(lltype.Signed)
+    op = SpaceOperation('getinteriorfield', [v, i, Constant('v', lltype.Void)],
+                        v_result)
+    op1 = Transformer(FakeCPU()).rewrite_operation(op)
+    assert op1.opname == 'getinteriorfield_gc_i'
+    assert op1.args == [v, i, ('interiorfielddescr', DICT, 'v')]
+    op = SpaceOperation('getinteriorfield', [v, i, Constant('v', lltype.Void)],
+                        Constant(None, lltype.Void))
+    op1 = Transformer(FakeCPU()).rewrite_operation(op)
+    assert op1 is None
+
 def test_str_setinteriorfield():
     v = varoftype(lltype.Ptr(rstr.STR))
     v_index = varoftype(lltype.Signed)
@@ -702,6 +719,23 @@
     assert op1.args == [v, v_index, v_newchr]
     assert op1.result == v_void
 
+def test_dict_setinteriorfield():
+    DICT = lltype.GcArray(lltype.Struct('ENTRY', ('v', lltype.Signed),
+                                        ('k', lltype.Signed)))
+    v = varoftype(lltype.Ptr(DICT))
+    i = varoftype(lltype.Signed)
+    v_void = varoftype(lltype.Void)
+    op = SpaceOperation('setinteriorfield', [v, i, Constant('v', lltype.Void),
+                                             i],
+                        v_void)
+    op1 = Transformer(FakeCPU()).rewrite_operation(op)
+    assert op1.opname == 'setinteriorfield_gc_i'
+    assert op1.args == [v, i, i, ('interiorfielddescr', DICT, 'v')]
+    op = SpaceOperation('setinteriorfield', [v, i, Constant('v', lltype.Void),
+                                             v_void], v_void)
+    op1 = Transformer(FakeCPU()).rewrite_operation(op)
+    assert not op1
+
 def test_promote_1():
     v1 = varoftype(lltype.Signed)
     v2 = varoftype(lltype.Signed)
diff --git a/pypy/jit/metainterp/blackhole.py b/pypy/jit/metainterp/blackhole.py
--- a/pypy/jit/metainterp/blackhole.py
+++ b/pypy/jit/metainterp/blackhole.py
@@ -6,7 +6,6 @@
 from pypy.rlib.debug import make_sure_not_resized
 from pypy.rpython.lltypesystem import lltype, llmemory, rclass
 from pypy.rpython.lltypesystem.lloperation import llop
-from pypy.rpython.llinterp import LLException
 from pypy.jit.codewriter.jitcode import JitCode, SwitchDictDescr
 from pypy.jit.codewriter import heaptracker, longlong
 from pypy.jit.metainterp.jitexc import JitException, get_llexception, reraise
@@ -1154,6 +1153,26 @@
         array = cpu.bh_getfield_gc_r(vable, fdescr)
         return cpu.bh_arraylen_gc(adescr, array)
 
+    @arguments("cpu", "r", "i", "d", returns="i")
+    def bhimpl_getinteriorfield_gc_i(cpu, array, index, descr):
+        return cpu.bh_getinteriorfield_gc_i(array, index, descr)
+    @arguments("cpu", "r", "i", "d", returns="r")
+    def bhimpl_getinteriorfield_gc_r(cpu, array, index, descr):
+        return cpu.bh_getinteriorfield_gc_r(array, index, descr)
+    @arguments("cpu", "r", "i", "d", returns="f")
+    def bhimpl_getinteriorfield_gc_f(cpu, array, index, descr):
+        return cpu.bh_getinteriorfield_gc_f(array, index, descr)
+
+    @arguments("cpu", "r", "i", "d", "i")
+    def bhimpl_setinteriorfield_gc_i(cpu, array, index, descr, value):
+        cpu.bh_setinteriorfield_gc_i(array, index, descr, value)
+    @arguments("cpu", "r", "i", "d", "r")
+    def bhimpl_setinteriorfield_gc_r(cpu, array, index, descr, value):
+        cpu.bh_setinteriorfield_gc_r(array, index, descr, value)
+    @arguments("cpu", "r", "i", "d", "f")
+    def bhimpl_setinteriorfield_gc_f(cpu, array, index, descr, value):
+        cpu.bh_setinteriorfield_gc_f(array, index, descr, value)
+
     @arguments("cpu", "r", "d", returns="i")
     def bhimpl_getfield_gc_i(cpu, struct, fielddescr):
         return cpu.bh_getfield_gc_i(struct, fielddescr)
diff --git a/pypy/jit/metainterp/executor.py b/pypy/jit/metainterp/executor.py
--- a/pypy/jit/metainterp/executor.py
+++ b/pypy/jit/metainterp/executor.py
@@ -1,11 +1,8 @@
 """This implements pyjitpl's execution of operations.
 """
 
-import py
-from pypy.rpython.lltypesystem import lltype, llmemory, rstr
-from pypy.rpython.ootypesystem import ootype
-from pypy.rpython.lltypesystem.lloperation import llop
-from pypy.rlib.rarithmetic import ovfcheck, r_uint, intmask, r_longlong
+from pypy.rpython.lltypesystem import lltype, rstr
+from pypy.rlib.rarithmetic import ovfcheck, r_longlong
 from pypy.rlib.rtimer import read_timestamp
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.jit.metainterp.history import BoxInt, BoxPtr, BoxFloat, check_descr
@@ -123,6 +120,29 @@
     else:
         cpu.bh_setarrayitem_raw_i(arraydescr, array, index, itembox.getint())
 
+def do_getinteriorfield_gc(cpu, _, arraybox, indexbox, descr):
+    array = arraybox.getref_base()
+    index = indexbox.getint()
+    if descr.is_pointer_field():
+        return BoxPtr(cpu.bh_getinteriorfield_gc_r(array, index, descr))
+    elif descr.is_float_field():
+        return BoxFloat(cpu.bh_getinteriorfield_gc_f(array, index, descr))
+    else:
+        return BoxInt(cpu.bh_getinteriorfield_gc_i(array, index, descr))
+
+def do_setinteriorfield_gc(cpu, _, arraybox, indexbox, valuebox, descr):
+    array = arraybox.getref_base()
+    index = indexbox.getint()
+    if descr.is_pointer_field():
+        cpu.bh_setinteriorfield_gc_r(array, index, descr,
+                                     valuebox.getref_base())
+    elif descr.is_float_field():
+        cpu.bh_setinteriorfield_gc_f(array, index, descr,
+                                     valuebox.getfloatstorage())
+    else:
+        cpu.bh_setinteriorfield_gc_i(array, index, descr,
+                                     valuebox.getint())
+
 def do_getfield_gc(cpu, _, structbox, fielddescr):
     struct = structbox.getref_base()
     if fielddescr.is_pointer_field():
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -36,6 +36,7 @@
 
 
 class MIFrame(object):
+    debug = False
 
     def __init__(self, metainterp):
         self.metainterp = metainterp
@@ -548,6 +549,14 @@
     opimpl_getfield_gc_r_pure = _opimpl_getfield_gc_pure_any
     opimpl_getfield_gc_f_pure = _opimpl_getfield_gc_pure_any
 
+    @arguments("box", "box", "descr")
+    def _opimpl_getinteriorfield_gc_any(self, array, index, descr):
+        return self.execute_with_descr(rop.GETINTERIORFIELD_GC, descr,
+                                       array, index)
+    opimpl_getinteriorfield_gc_i = _opimpl_getinteriorfield_gc_any
+    opimpl_getinteriorfield_gc_f = _opimpl_getinteriorfield_gc_any
+    opimpl_getinteriorfield_gc_r = _opimpl_getinteriorfield_gc_any
+
     @specialize.arg(1)
     def _opimpl_getfield_gc_any_pureornot(self, opnum, box, fielddescr):
         tobox = self.metainterp.heapcache.getfield(box, fielddescr)
@@ -588,6 +597,15 @@
     opimpl_setfield_gc_r = _opimpl_setfield_gc_any
     opimpl_setfield_gc_f = _opimpl_setfield_gc_any
 
+    @arguments("box", "box", "box", "descr")
+    def _opimpl_setinteriorfield_gc_any(self, array, index, value, descr):
+        self.execute_with_descr(rop.SETINTERIORFIELD_GC, descr,
+                                array, index, value)
+    opimpl_setinteriorfield_gc_i = _opimpl_setinteriorfield_gc_any
+    opimpl_setinteriorfield_gc_f = _opimpl_setinteriorfield_gc_any
+    opimpl_setinteriorfield_gc_r = _opimpl_setinteriorfield_gc_any
+                                
+
     @arguments("box", "descr")
     def _opimpl_getfield_raw_any(self, box, fielddescr):
         return self.execute_with_descr(rop.GETFIELD_RAW, fielddescr, box)
@@ -2588,17 +2606,21 @@
         self.pc = position
         #
         if not we_are_translated():
-            print '\tpyjitpl: %s(%s)' % (name, ', '.join(map(repr, args))),
+            if self.debug:
+                print '\tpyjitpl: %s(%s)' % (name, ', '.join(map(repr, args))),
             try:
                 resultbox = unboundmethod(self, *args)
             except Exception, e:
-                print '-> %s!' % e.__class__.__name__
+                if self.debug:
+                    print '-> %s!' % e.__class__.__name__
                 raise
             if num_return_args == 0:
-                print
+                if self.debug:
+                    print
                 assert resultbox is None
             else:
-                print '-> %r' % (resultbox,)
+                if self.debug:
+                    print '-> %r' % (resultbox,)
                 assert argcodes[next_argcode] == '>'
                 result_argcode = argcodes[next_argcode + 1]
                 assert resultbox.type == {'i': history.INT,
diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py
--- a/pypy/jit/metainterp/resoperation.py
+++ b/pypy/jit/metainterp/resoperation.py
@@ -1,5 +1,4 @@
 from pypy.rlib.objectmodel import we_are_translated
-from pypy.rlib.debug import make_sure_not_resized
 
 def ResOperation(opnum, args, result, descr=None):
     cls = opclasses[opnum]
@@ -457,6 +456,7 @@
 
     'GETARRAYITEM_GC/2d',
     'GETARRAYITEM_RAW/2d',
+    'GETINTERIORFIELD_GC/2d',
     'GETFIELD_GC/1d',
     'GETFIELD_RAW/1d',
     '_MALLOC_FIRST',
@@ -473,6 +473,7 @@
 
     'SETARRAYITEM_GC/3d',
     'SETARRAYITEM_RAW/3d',
+    'SETINTERIORFIELD_GC/3d',
     'SETFIELD_GC/2d',
     'SETFIELD_RAW/2d',
     'STRSETITEM/3',
diff --git a/pypy/jit/metainterp/test/test_dict.py b/pypy/jit/metainterp/test/test_dict.py
--- a/pypy/jit/metainterp/test/test_dict.py
+++ b/pypy/jit/metainterp/test/test_dict.py
@@ -91,7 +91,7 @@
         res1 = f(100)
         res2 = self.meta_interp(f, [100], listops=True)
         assert res1 == res2
-        self.check_loops(int_mod=1) # the hash was traced
+        self.check_loops(int_mod=1) # the hash was traced and eq, but cached
 
     def test_dict_setdefault(self):
         myjitdriver = JitDriver(greens = [], reds = ['total', 'dct'])
@@ -128,7 +128,7 @@
         assert f(100) == 50
         res = self.meta_interp(f, [100], listops=True)
         assert res == 50
-        self.check_loops(int_mod=1)
+        self.check_loops(int_mod=1) # key + eq, but cached
 
     def test_repeated_lookup(self):
         myjitdriver = JitDriver(greens = [], reds = ['n', 'd'])
@@ -153,10 +153,12 @@
 
         res = self.meta_interp(f, [100], listops=True)
         assert res == f(50)
-        self.check_loops({"call": 7, "guard_false": 1, "guard_no_exception": 6,
+        self.check_loops({"call": 5, "getfield_gc": 1, "getinteriorfield_gc": 1,
+                          "guard_false": 1, "guard_no_exception": 4,
                           "guard_true": 1, "int_and": 1, "int_gt": 1,
                           "int_is_true": 1, "int_sub": 1, "jump": 1,
-                          "new_with_vtable": 1, "setfield_gc": 1})
+                          "new_with_vtable": 1, "new": 1, "new_array": 1,
+                          "setfield_gc": 3, })
 
 
 class TestOOtype(DictTests, OOJitMixin):
diff --git a/pypy/jit/metainterp/warmspot.py b/pypy/jit/metainterp/warmspot.py
--- a/pypy/jit/metainterp/warmspot.py
+++ b/pypy/jit/metainterp/warmspot.py
@@ -206,7 +206,7 @@
         self.make_enter_functions()
         self.rewrite_jit_merge_points(policy)
 
-        verbose = not self.cpu.translate_support_code
+        verbose = False # not self.cpu.translate_support_code
         self.codewriter.make_jitcodes(verbose=verbose)
         self.rewrite_can_enter_jits()
         self.rewrite_set_param()
diff --git a/pypy/module/__builtin__/__init__.py b/pypy/module/__builtin__/__init__.py
--- a/pypy/module/__builtin__/__init__.py
+++ b/pypy/module/__builtin__/__init__.py
@@ -24,6 +24,7 @@
         'map'           : 'app_functional.map',
         'reduce'        : 'app_functional.reduce',
         'filter'        : 'app_functional.filter',
+        'zip'           : 'app_functional.zip',
         'vars'          : 'app_inspect.vars',
         'dir'           : 'app_inspect.dir',
 
@@ -87,7 +88,6 @@
         'enumerate'     : 'functional.W_Enumerate',
         'min'           : 'functional.min',
         'max'           : 'functional.max',
-        'zip'           : 'functional.zip',
         'reversed'      : 'functional.reversed',
         'super'         : 'descriptor.W_Super',
         'staticmethod'  : 'descriptor.StaticMethod',
diff --git a/pypy/module/__builtin__/app_functional.py b/pypy/module/__builtin__/app_functional.py
--- a/pypy/module/__builtin__/app_functional.py
+++ b/pypy/module/__builtin__/app_functional.py
@@ -132,3 +132,20 @@
     for item in seq:
         if func(item):
             yield item
+
+def zip(*sequences):
+    """zip(seq1 [, seq2 [...]]) -> [(seq1[0], seq2[0] ...), (...)]
+
+Return a list of tuples, where each tuple contains the i-th element
+from each of the argument sequences.  The returned list is truncated
+in length to the length of the shortest argument sequence."""
+    if not sequences:
+        return []
+    result = []
+    iterators = [iter(seq) for seq in sequences]
+    while True:
+        try:
+            items = [next(it) for it in iterators]
+        except StopIteration:
+            return result
+        result.append(tuple(items))
diff --git a/pypy/module/__builtin__/app_io.py b/pypy/module/__builtin__/app_io.py
--- a/pypy/module/__builtin__/app_io.py
+++ b/pypy/module/__builtin__/app_io.py
@@ -27,7 +27,20 @@
     co = compile(source.rstrip()+"\n", filename, 'exec')
     exec(co, glob, loc)
 
-def input(prompt=None):
+def _write_prompt(stdout, prompt):
+    print(prompt, file=stdout, end='')
+    try:
+        flush = stdout.flush
+    except AttributeError:
+        pass
+    else:
+        flush()
+    try:
+        stdout.softspace = 0
+    except (AttributeError, TypeError):
+        pass
+
+def input(prompt=''):
     """input([prompt]) -> string
 
 Read a string from standard input.  The trailing newline is stripped.
@@ -47,18 +60,10 @@
     if (hasattr(sys, '__raw_input__') and
         isinstance(stdin, file)  and stdin.fileno() == 0 and stdin.isatty() and
         isinstance(stdout, file) and stdout.fileno() == 1):
-        if prompt is None:
-            prompt = ''
-        return sys.__raw_input__(prompt)
+        _write_prompt(stdout, '')
+        return sys.__raw_input__(str(prompt))
 
-    if prompt is not None:
-        stdout.write(prompt)
-        try:
-            flush = stdout.flush
-        except AttributeError:
-            pass
-        else:
-            flush()
+    _write_prompt(stdout, prompt)
     line = stdin.readline()
     if not line:    # inputting an empty line gives line == '\n'
         raise EOFError
diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py
--- a/pypy/module/__builtin__/functional.py
+++ b/pypy/module/__builtin__/functional.py
@@ -116,27 +116,6 @@
     """
     return min_max(space, __args__, "min")
 
- at unwrap_spec(sequences_w="args_w")
-def zip(space, sequences_w):
-    """Return a list of tuples, where the nth tuple contains every nth item of
-    each collection.
-
-    If the collections have different lengths, zip returns a list as long as the
-    shortest collection, ignoring the trailing items in the other collections.
-    """
-    if not sequences_w:
-        return space.newlist([])
-    result_w = []
-    iterators_w = [space.iter(w_seq) for w_seq in sequences_w]
-    while True:
-        try:
-            items_w = [space.next(w_it) for w_it in iterators_w]
-        except OperationError, e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            return space.newlist(result_w)
-        result_w.append(space.newtuple(items_w))
-
 class W_Enumerate(Wrappable):
 
     def __init__(self, w_iter, w_start):
diff --git a/pypy/module/__builtin__/test/test_rawinput.py b/pypy/module/__builtin__/test/test_rawinput.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/__builtin__/test/test_rawinput.py
@@ -0,0 +1,80 @@
+import autopath
+
+
+class AppTestRawInput():
+
+    def test_input_and_raw_input(self):
+        import sys, StringIO
+        for prompt, expected in [("def:", "abc/ def:/ghi\n"),
+                                 ("", "abc/ /ghi\n"),
+                                 (42, "abc/ 42/ghi\n"),
+                                 (None, "abc/ None/ghi\n"),
+                                 (Ellipsis, "abc/ /ghi\n")]:
+            for inputfn, inputtext, gottext in [
+                    (raw_input, "foo\nbar\n", "foo"),
+                    (input, "40+2\n", 42)]:
+                save = sys.stdin, sys.stdout
+                try:
+                    sys.stdin = StringIO.StringIO(inputtext)
+                    out = sys.stdout = StringIO.StringIO()
+                    print "abc",    # softspace = 1
+                    out.write('/')
+                    if prompt is Ellipsis:
+                        got = inputfn()
+                    else:
+                        got = inputfn(prompt)
+                    out.write('/')
+                    print "ghi"
+                finally:
+                    sys.stdin, sys.stdout = save
+                assert out.getvalue() == expected
+                assert got == gottext
+
+    def test_softspace(self):
+        import sys
+        import StringIO
+        fin = StringIO.StringIO()
+        fout = StringIO.StringIO()
+
+        fin.write("Coconuts\n")
+        fin.seek(0)
+
+        sys_stdin_orig = sys.stdin
+        sys_stdout_orig = sys.stdout
+
+        sys.stdin = fin
+        sys.stdout = fout
+
+        print "test",
+        raw_input("test")
+
+        sys.stdin = sys_stdin_orig
+        sys.stdout = sys_stdout_orig
+
+        fout.seek(0)
+        assert fout.read() == "test test"
+
+    def test_softspace_carryover(self):
+        import sys
+        import StringIO
+        fin = StringIO.StringIO()
+        fout = StringIO.StringIO()
+
+        fin.write("Coconuts\n")
+        fin.seek(0)
+
+        sys_stdin_orig = sys.stdin
+        sys_stdout_orig = sys.stdout
+
+        sys.stdin = fin
+        sys.stdout = fout
+
+        print "test",
+        raw_input("test")
+        print "test",
+
+        sys.stdin = sys_stdin_orig
+        sys.stdout = sys_stdout_orig
+
+        fout.seek(0)
+        assert fout.read() == "test testtest"
diff --git a/pypy/module/pypyjit/test_pypy_c/test_containers.py b/pypy/module/pypyjit/test_pypy_c/test_containers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_containers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_containers.py
@@ -46,7 +46,7 @@
         assert loop.match_by_id("getitem", """
             i28 = call(ConstClass(ll_dict_lookup__dicttablePtr_objectPtr_Signed), p18, p6, i25, descr=...)
             ...
-            p33 = call(ConstClass(ll_get_value__dicttablePtr_Signed), p18, i28, descr=...)
+            p33 = getinteriorfield_gc(p31, i26, <InteriorFieldDescr <GcPtrFieldDescr dictentry.value .*>>)
             ...
         """)
 
diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -93,7 +93,8 @@
             i46 = call(ConstClass(ll_startswith__rpy_stringPtr_rpy_stringPtr), p28, ConstPtr(ptr45), descr=<BoolCallDescr>)
             guard_false(i46, descr=...)
             p51 = new_with_vtable(21136408)
-            setfield_gc(p51, _, descr=...)    # 6 setfields, but the order is dict-order-dependent
+            setfield_gc(p51, _, descr=...)    # 7 setfields, but the order is dict-order-dependent
+            setfield_gc(p51, _, descr=...)
             setfield_gc(p51, _, descr=...)
             setfield_gc(p51, _, descr=...)
             setfield_gc(p51, _, descr=...)
diff --git a/pypy/rlib/_rsocket_rffi.py b/pypy/rlib/_rsocket_rffi.py
--- a/pypy/rlib/_rsocket_rffi.py
+++ b/pypy/rlib/_rsocket_rffi.py
@@ -16,6 +16,7 @@
 _MINGW = target_platform.name == "mingw32"
 _SOLARIS = sys.platform == "sunos5"
 _MACOSX = sys.platform == "darwin"
+_HAS_AF_PACKET = sys.platform.startswith('linux')   # only Linux for now
 
 if _POSIX:
     includes = ('sys/types.h',
@@ -34,11 +35,12 @@
                 'stdint.h', 
                 'errno.h',
                 )
+    if _HAS_AF_PACKET:
+        includes += ('netpacket/packet.h',
+                     'sys/ioctl.h',
+                     'net/if.h')
 
-    cond_includes = [('AF_NETLINK', 'linux/netlink.h'),
-                     ('AF_PACKET', 'netpacket/packet.h'),
-                     ('AF_PACKET', 'sys/ioctl.h'),
-                     ('AF_PACKET', 'net/if.h')]
+    cond_includes = [('AF_NETLINK', 'linux/netlink.h')]
     
     libraries = ()
     calling_conv = 'c'
@@ -320,18 +322,18 @@
                                              ('events', rffi.SHORT),
                                              ('revents', rffi.SHORT)])
 
-    CConfig.sockaddr_ll = platform.Struct('struct sockaddr_ll',
+    if _HAS_AF_PACKET:
+        CConfig.sockaddr_ll = platform.Struct('struct sockaddr_ll',
                               [('sll_ifindex', rffi.INT),
                                ('sll_protocol', rffi.INT),
                                ('sll_pkttype', rffi.INT),
                                ('sll_hatype', rffi.INT),
                                ('sll_addr', rffi.CFixedArray(rffi.CHAR, 8)),
-                               ('sll_halen', rffi.INT)],
-                              ifdef='AF_PACKET')
+                               ('sll_halen', rffi.INT)])
 
-    CConfig.ifreq = platform.Struct('struct ifreq', [('ifr_ifindex', rffi.INT),
-                                 ('ifr_name', rffi.CFixedArray(rffi.CHAR, 8))],
-                                    ifdef='AF_PACKET')
+        CConfig.ifreq = platform.Struct('struct ifreq',
+                                [('ifr_ifindex', rffi.INT),
+                                 ('ifr_name', rffi.CFixedArray(rffi.CHAR, 8))])
 
 if _WIN32:
     CConfig.WSAEVENT = platform.SimpleType('WSAEVENT', rffi.VOIDP)
@@ -386,6 +388,8 @@
         constants[name] = value
     else:
         constants[name] = default
+if not _HAS_AF_PACKET and 'AF_PACKET' in constants:
+    del constants['AF_PACKET']
 
 constants['has_ipv6'] = True # This is a configuration option in CPython
 for name, value in constants.items():
@@ -439,21 +443,14 @@
 if _POSIX:
     nfds_t = cConfig.nfds_t
     pollfd = cConfig.pollfd
-    if cConfig.sockaddr_ll is not None:
+    if _HAS_AF_PACKET:
         sockaddr_ll = cConfig.sockaddr_ll
-    ifreq = cConfig.ifreq
+        ifreq = cConfig.ifreq
 if WIN32:
     WSAEVENT = cConfig.WSAEVENT
     WSANETWORKEVENTS = cConfig.WSANETWORKEVENTS
 timeval = cConfig.timeval
 
-#if _POSIX:
-#    includes = list(includes)
-#    for _name, _header in cond_includes:
-#        if getattr(cConfig, _name) is not None:
-#            includes.append(_header)
-#    eci = ExternalCompilationInfo(includes=includes, libraries=libraries,
-#                                  separate_module_sources=sources)
 
 def external(name, args, result, **kwds):
     return rffi.llexternal(name, args, result, compilation_info=eci,
@@ -544,7 +541,7 @@
     socketpair_t = rffi.CArray(socketfd_type)
     socketpair = external('socketpair', [rffi.INT, rffi.INT, rffi.INT,
                           lltype.Ptr(socketpair_t)], rffi.INT)
-    if ifreq is not None:
+    if _HAS_AF_PACKET:
         ioctl = external('ioctl', [socketfd_type, rffi.INT, lltype.Ptr(ifreq)],
                          rffi.INT)
 
diff --git a/pypy/rlib/rerased.py b/pypy/rlib/rerased.py
--- a/pypy/rlib/rerased.py
+++ b/pypy/rlib/rerased.py
@@ -135,6 +135,8 @@
     _about_ = erase_int
 
     def compute_result_annotation(self, s_obj):
+        config = self.bookkeeper.annotator.translator.config
+        assert config.translation.taggedpointers, "need to enable tagged pointers to use erase_int"
         assert annmodel.SomeInteger().contains(s_obj)
         return SomeErased()
 
@@ -228,6 +230,8 @@
 
     def convert_const(self, value):
         if value._identity is _identity_for_ints:
+            config = self.rtyper.annotator.translator.config
+            assert config.translation.taggedpointers, "need to enable tagged pointers to use erase_int"
             return lltype.cast_int_to_ptr(self.lowleveltype, value._x * 2 + 1)
         bk = self.rtyper.annotator.bookkeeper
         s_obj = value._identity.get_input_annotation(bk)
diff --git a/pypy/rlib/rsocket.py b/pypy/rlib/rsocket.py
--- a/pypy/rlib/rsocket.py
+++ b/pypy/rlib/rsocket.py
@@ -8,6 +8,7 @@
 # Known missing features:
 #
 #   - address families other than AF_INET, AF_INET6, AF_UNIX, AF_PACKET
+#   - AF_PACKET is only supported on Linux
 #   - methods makefile(),
 #   - SSL
 #
diff --git a/pypy/rlib/test/test_rerased.py b/pypy/rlib/test/test_rerased.py
--- a/pypy/rlib/test/test_rerased.py
+++ b/pypy/rlib/test/test_rerased.py
@@ -10,6 +10,13 @@
 
 from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin
 
+def make_annotator():
+    a = RPythonAnnotator()
+    a.translator.config.translation.taggedpointers = True
+    return a
+
+
+
 class X(object):
     pass
 
@@ -55,7 +62,7 @@
 def test_annotate_1():
     def f():
         return eraseX(X())
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [])
     assert isinstance(s, SomeErased)
 
@@ -66,7 +73,7 @@
         #assert not is_integer(e)
         x2 = uneraseX(e)
         return x2
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [])
     assert isinstance(s, annmodel.SomeInstance)
     assert s.classdef == a.bookkeeper.getuniqueclassdef(X)
@@ -77,7 +84,7 @@
         #assert is_integer(e)
         x2 = unerase_int(e)
         return x2
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [])
     assert isinstance(s, annmodel.SomeInteger)
 
@@ -105,7 +112,7 @@
         x = make(n)
         return check(x, n)
     #
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [int])
     assert isinstance(s, annmodel.SomeInteger)
 
@@ -135,7 +142,7 @@
             else:
                 return inst
     #
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [])
     assert isinstance(s, annmodel.SomeInstance)
     assert s.classdef == a.bookkeeper.getuniqueclassdef(A)
@@ -155,7 +162,7 @@
             e = e2
         return unerase(e)
     #
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [int])
     assert isinstance(s, annmodel.SomeInstance)
     assert s.classdef == a.bookkeeper.getuniqueclassdef(X)
@@ -165,11 +172,14 @@
     e1 = erase_int(42)
     def f(i):
         return unerase_int(e1)
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [int])
     assert isinstance(s, annmodel.SomeInteger)
 
 class BaseTestRErased(BaseRtypingTest):
+    def interpret(self, *args, **kwargs):
+        kwargs["taggedpointers"] = True
+        return BaseRtypingTest.interpret(self, *args, **kwargs)
     def test_rtype_1(self):
         def f():
             return eraseX(X())
diff --git a/pypy/rpython/lltypesystem/ll2ctypes.py b/pypy/rpython/lltypesystem/ll2ctypes.py
--- a/pypy/rpython/lltypesystem/ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/ll2ctypes.py
@@ -15,12 +15,11 @@
     load_library_kwargs = {}
 
 import os
-from pypy import conftest
 from pypy.rpython.lltypesystem import lltype, llmemory
 from pypy.rpython.extfunc import ExtRegistryEntry
 from pypy.rlib.objectmodel import Symbolic, ComputedIntSymbolic
 from pypy.tool.uid import fixid
-from pypy.rlib.rarithmetic import r_uint, r_singlefloat, r_longfloat, base_int, intmask
+from pypy.rlib.rarithmetic import r_singlefloat, r_longfloat, base_int, intmask
 from pypy.annotation import model as annmodel
 from pypy.rpython.llinterp import LLInterpreter, LLException
 from pypy.rpython.lltypesystem.rclass import OBJECT, OBJECT_VTABLE
@@ -531,6 +530,10 @@
     def __str__(self):
         return repr(self)
 
+    def _setparentstructure(self, parent, parentindex):
+        super(_parentable_mixin, self)._setparentstructure(parent, parentindex)
+        self._keepparent = parent   # always keep a strong ref
+
 class _struct_mixin(_parentable_mixin):
     """Mixin added to _struct containers when they become ctypes-based."""
     __slots__ = ()
@@ -566,7 +569,10 @@
         return 0, sys.maxint
 
     def getitem(self, index, uninitialized_ok=False):
-        return self._storage.contents._getitem(index, boundscheck=False)
+        res = self._storage.contents._getitem(index, boundscheck=False)
+        if isinstance(self._TYPE.OF, lltype.ContainerType):
+            res._obj._setparentstructure(self, index)
+        return res
 
     def setitem(self, index, value):
         self._storage.contents._setitem(index, value, boundscheck=False)
@@ -1279,6 +1285,8 @@
             self.intval = intmask(void_p.value)
 
     def __eq__(self, other):
+        if not other:
+            return self.intval == 0
         if isinstance(other, _llgcopaque):
             return self.intval == other.intval
         storage = object()
diff --git a/pypy/rpython/lltypesystem/lltype.py b/pypy/rpython/lltypesystem/lltype.py
--- a/pypy/rpython/lltypesystem/lltype.py
+++ b/pypy/rpython/lltypesystem/lltype.py
@@ -1522,7 +1522,7 @@
             and parentindex in (self._parent_type._names[0], 0)
             and self._TYPE._gckind == typeOf(parent)._gckind):
             # keep strong reference to parent, we share the same allocation
-            self._keepparent = parent 
+            self._keepparent = parent
 
     def _parentstructure(self, check=True):
         if self._wrparent is not None:
@@ -1730,7 +1730,8 @@
         # Keep the parent array alive, we share the same allocation.
         # Don't do it if we are inside a GC object, though -- it's someone
         # else's job to keep the GC object alive
-        if typeOf(top_container(parent))._gckind == 'raw':
+        if (typeOf(top_container(parent))._gckind == 'raw' or
+            hasattr(top_container(parent)._storage, 'contents')):  # ll2ctypes
             self._keepparent = parent
 
     def __str__(self):
diff --git a/pypy/rpython/lltypesystem/rbuilder.py b/pypy/rpython/lltypesystem/rbuilder.py
--- a/pypy/rpython/lltypesystem/rbuilder.py
+++ b/pypy/rpython/lltypesystem/rbuilder.py
@@ -29,7 +29,7 @@
         except OverflowError:
             raise MemoryError
         newbuf = mallocfn(new_allocated)
-        copycontentsfn(ll_builder.buf, newbuf, 0, 0, ll_builder.allocated)
+        copycontentsfn(ll_builder.buf, newbuf, 0, 0, ll_builder.used)
         ll_builder.buf = newbuf
         ll_builder.allocated = new_allocated
     return func_with_new_name(stringbuilder_grow, name)
@@ -56,7 +56,7 @@
 class BaseStringBuilderRepr(AbstractStringBuilderRepr):
     def empty(self):
         return nullptr(self.lowleveltype.TO)
-    
+
     @classmethod
     def ll_new(cls, init_size):
         if init_size < 0 or init_size > MAX:
diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py
--- a/pypy/rpython/lltypesystem/rdict.py
+++ b/pypy/rpython/lltypesystem/rdict.py
@@ -1,16 +1,14 @@
 from pypy.tool.pairtype import pairtype
-from pypy.annotation import model as annmodel
 from pypy.objspace.flow.model import Constant
-from pypy.rpython.rdict import AbstractDictRepr, AbstractDictIteratorRepr,\
-     rtype_newdict
+from pypy.rpython.rdict import (AbstractDictRepr, AbstractDictIteratorRepr,
+     rtype_newdict)
 from pypy.rpython.lltypesystem import lltype
+from pypy.rlib import objectmodel, jit
 from pypy.rlib.rarithmetic import r_uint, intmask, LONG_BIT
-from pypy.rlib.objectmodel import hlinvoke
-from pypy.rpython import robject
-from pypy.rlib import objectmodel, jit
 from pypy.rpython import rmodel
 from pypy.rpython.error import TyperError
 
+
 HIGHEST_BIT = intmask(1 << (LONG_BIT - 1))
 MASK = intmask(HIGHEST_BIT - 1)
 
@@ -424,17 +422,16 @@
     ENTRIES = lltype.typeOf(entries).TO
     return ENTRIES.fasthashfn(entries[i].key)
 
- at jit.dont_look_inside
 def ll_get_value(d, i):
     return d.entries[i].value
 
 def ll_keyhash_custom(d, key):
     DICT = lltype.typeOf(d).TO
-    return hlinvoke(DICT.r_rdict_hashfn, d.fnkeyhash, key)
+    return objectmodel.hlinvoke(DICT.r_rdict_hashfn, d.fnkeyhash, key)
 
 def ll_keyeq_custom(d, key1, key2):
     DICT = lltype.typeOf(d).TO
-    return hlinvoke(DICT.r_rdict_eqfn, d.fnkeyeq, key1, key2)
+    return objectmodel.hlinvoke(DICT.r_rdict_eqfn, d.fnkeyeq, key1, key2)
 
 def ll_dict_len(d):
     return d.num_items
@@ -455,6 +452,8 @@
     i = ll_dict_lookup(d, key, hash)
     return _ll_dict_setitem_lookup_done(d, key, value, hash, i)
 
+# Leaving as dont_look_inside ATM, it has a few branches which could lead to
+# many bridges if we don't consider their possible frequency.
 @jit.dont_look_inside
 def _ll_dict_setitem_lookup_done(d, key, value, hash, i):
     valid = (i & HIGHEST_BIT) == 0
@@ -508,6 +507,8 @@
     else:
         return default
 
+# XXX: Move the size checking and resize into a single call which is opauqe to
+# the JIT to avoid extra branches.
 @jit.dont_look_inside
 def _ll_dict_del(d, i):
     d.entries.mark_deleted(i)
@@ -548,6 +549,7 @@
 # ------- a port of CPython's dictobject.c's lookdict implementation -------
 PERTURB_SHIFT = 5
 
+ at jit.dont_look_inside
 def ll_dict_lookup(d, key, hash):
     entries = d.entries
     ENTRIES = lltype.typeOf(entries).TO
@@ -637,7 +639,6 @@
     d.num_items = 0
     d.resize_counter = DICT_INITSIZE * 2
     return d
-ll_newdict.oopspec = 'newdict()'
 
 def ll_newdict_size(DICT, length_estimate):
     length_estimate = (length_estimate // 2) * 3
@@ -649,7 +650,6 @@
     d.num_items = 0
     d.resize_counter = n * 2
     return d
-ll_newdict_size.oopspec = 'newdict()'
 
 # pypy.rpython.memory.lldict uses a dict based on Struct and Array
 # instead of GcStruct and GcArray, which is done by using different
@@ -882,7 +882,6 @@
         global_popitem_index.nextindex = base + counter
     return i
 
- at jit.dont_look_inside
 def ll_popitem(ELEM, dic):
     i = _ll_getnextitem(dic)
     entry = dic.entries[i]
diff --git a/pypy/rpython/lltypesystem/test/test_ll2ctypes.py b/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
--- a/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
@@ -8,6 +8,7 @@
 from pypy.rpython.lltypesystem.ll2ctypes import uninitialized2ctypes
 from pypy.rpython.lltypesystem.ll2ctypes import ALLOCATED, force_cast
 from pypy.rpython.lltypesystem.ll2ctypes import cast_adr_to_int, get_ctypes_type
+from pypy.rpython.lltypesystem.ll2ctypes import _llgcopaque
 from pypy.rpython.annlowlevel import llhelper
 from pypy.rlib import rposix
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
@@ -1349,6 +1350,11 @@
         round = ctypes2lltype(llmemory.GCREF, lltype2ctypes(opaque.hide()))
         assert Opaque.show(round) is opaque
 
+    def test_array_of_structs(self):
+        A = lltype.GcArray(lltype.Struct('x', ('v', lltype.Signed)))
+        a = lltype.malloc(A, 5)
+        a2 = ctypes2lltype(lltype.Ptr(A), lltype2ctypes(a))
+        assert a2._obj.getitem(0)._obj._parentstructure() is a2._obj
 
 class TestPlatform(object):
     def test_lib_on_libpaths(self):
@@ -1390,3 +1396,7 @@
         f = rffi.llexternal('f', [rffi.INT, rffi.INT], rffi.INT,
                             compilation_info=eci)
         assert f(3, 4) == 7
+
+    def test_llgcopaque_eq(self):
+        assert _llgcopaque(1) != None
+        assert _llgcopaque(0) == None
diff --git a/pypy/translator/c/database.py b/pypy/translator/c/database.py
--- a/pypy/translator/c/database.py
+++ b/pypy/translator/c/database.py
@@ -176,7 +176,7 @@
                      # introduced by the GC transformer, or the type_info_table
         return node
 
-    def get(self, obj):
+    def get(self, obj, funcgen=None):
         if isinstance(obj, CConstant):
             return obj.c_name  # without further checks
         T = typeOf(obj)
@@ -228,6 +228,8 @@
                     return '((%s) %d)' % (cdecl(self.gettype(T), ''),
                                           obj._obj)
                 node = self.getcontainernode(container)
+                if node._funccodegen_owner is None:
+                    node._funccodegen_owner = funcgen
                 return node.getptrname()
             else:
                 return '((%s) NULL)' % (cdecl(self.gettype(T), ''), )
@@ -284,14 +286,16 @@
             finish_callbacks.append(('GC transformer: finished tables',
                                      self.gctransformer.get_finish_tables()))
 
-        def add_dependencies(newdependencies):
+        def add_dependencies(newdependencies, parent=None):
             for value in newdependencies:
                 #if isinstance(value, _uninitialized):
                 #    continue
                 if isinstance(typeOf(value), ContainerType):
-                    self.getcontainernode(value)
+                    node = self.getcontainernode(value)
+                    if parent and node._funccodegen_owner is not None:
+                        node._funccodegen_owner = parent._funccodegen_owner
                 else:
-                    self.get(value)
+                    self.get(value, parent and parent._funccodegen_owner)
 
         while True:
             while True:
@@ -303,7 +307,7 @@
                 if i == len(self.containerlist):
                     break
                 node = self.containerlist[i]
-                add_dependencies(node.enum_dependencies())
+                add_dependencies(node.enum_dependencies(), node)
                 i += 1
                 self.completedcontainers = i
                 if i == show_i:
diff --git a/pypy/translator/c/gc.py b/pypy/translator/c/gc.py
--- a/pypy/translator/c/gc.py
+++ b/pypy/translator/c/gc.py
@@ -170,6 +170,7 @@
     nodekind = 'refcnt rtti'
     globalcontainer = True
     typename = 'void (@)(void *)'
+    _funccodegen_owner = None
 
     def __init__(self, db, T, obj):
         assert T == RuntimeTypeInfo
@@ -266,6 +267,7 @@
     nodekind = 'boehm rtti'
     globalcontainer = True
     typename = 'char @'
+    _funccodegen_owner = None
 
     def __init__(self, db, T, obj):
         assert T == RuntimeTypeInfo
diff --git a/pypy/translator/c/gcc/test/test_asmgcroot.py b/pypy/translator/c/gcc/test/test_asmgcroot.py
--- a/pypy/translator/c/gcc/test/test_asmgcroot.py
+++ b/pypy/translator/c/gcc/test/test_asmgcroot.py
@@ -21,6 +21,7 @@
         config = get_pypy_config(translating=True)
         config.translation.gc = cls.gcpolicy
         config.translation.gcrootfinder = "asmgcc"
+        config.translation.taggedpointers = getattr(cls, "taggedpointers", False)
         return config
 
     @classmethod
diff --git a/pypy/translator/c/genc.py b/pypy/translator/c/genc.py
--- a/pypy/translator/c/genc.py
+++ b/pypy/translator/c/genc.py
@@ -682,8 +682,7 @@
     def getbasecfilefornode(self, node, basecname):
         # For FuncNode instances, use the python source filename (relative to
         # the top directory):
-        if hasattr(node.obj, 'graph'):
-            g = node.obj.graph
+        def invent_nice_name(g):
             # Lookup the filename from the function.
             # However, not all FunctionGraph objs actually have a "func":
             if hasattr(g, 'func'):
@@ -693,6 +692,15 @@
                     if pypkgpath:
                         relpypath =  localpath.relto(pypkgpath)
                         return relpypath.replace('.py', '.c')
+            return None
+        if hasattr(node.obj, 'graph'):
+            name = invent_nice_name(node.obj.graph)
+            if name is not None:
+                return name
+        elif node._funccodegen_owner is not None:
+            name = invent_nice_name(node._funccodegen_owner.graph)
+            if name is not None:
+                return "data_" + name
         return basecname
 
     def splitnodesimpl(self, basecname, nodes, nextra, nbetween,
diff --git a/pypy/translator/c/node.py b/pypy/translator/c/node.py
--- a/pypy/translator/c/node.py
+++ b/pypy/translator/c/node.py
@@ -485,6 +485,7 @@
         __slots__ = """db obj 
                        typename implementationtypename
                         name
+                        _funccodegen_owner
                         globalcontainer""".split()
     eci_name = '_compilation_info'
 
@@ -509,6 +510,7 @@
         if self.typename != self.implementationtypename:
             if db.gettypedefnode(T).extra_union_for_varlength:
                 self.name += '.b'
+        self._funccodegen_owner = None
 
     def getptrname(self):
         return '(&%s)' % self.name
@@ -842,6 +844,9 @@
         if self.funcgens:
             argnames = self.funcgens[0].argnames()  #Assume identical for all funcgens
             self.implementationtypename = self.db.gettype(self.T, argnames=argnames)
+            self._funccodegen_owner = self.funcgens[0]
+        else:
+            self._funccodegen_owner = None
 
     def basename(self):
         return self.obj._name
@@ -1005,6 +1010,7 @@
     globalcontainer = True
     typename = 'PyObject @'
     implementationtypename = 'PyObject *@'
+    _funccodegen_owner = None
 
     def __init__(self, db, T, obj):
         # obj is a _pyobject here; obj.value is the underlying CPython object
diff --git a/pypy/translator/c/primitive.py b/pypy/translator/c/primitive.py
--- a/pypy/translator/c/primitive.py
+++ b/pypy/translator/c/primitive.py
@@ -141,7 +141,12 @@
 
 def name_gcref(value, db):
     if value:
-        realobj = value._obj.container
+        obj = value._obj
+        if isinstance(obj, int):
+            # a tagged pointer
+            assert obj & 1 == 1
+            return '((%s) %d)' % (cdecl("void*", ''), obj)
+        realobj = obj.container
         realvalue = cast_opaque_ptr(Ptr(typeOf(realobj)), value)
         return db.get(realvalue)
     else:
diff --git a/pypy/translator/c/test/test_newgc.py b/pypy/translator/c/test/test_newgc.py
--- a/pypy/translator/c/test/test_newgc.py
+++ b/pypy/translator/c/test/test_newgc.py
@@ -1487,6 +1487,43 @@
         res = self.run("tagged")
         assert res == expected
 
+    def define_erased(cls):
+        from pypy.rlib import rerased
+        erase, unerase = rerased.new_erasing_pair("test")
+        class Unrelated(object):
+            pass
+
+        u = Unrelated()
+        u.tagged = True
+        u.x = rerased.erase_int(41)
+        class A(object):
+            pass
+        def fn():
+            n = 1
+            while n >= 0:
+                if u.tagged:
+                    n = rerased.unerase_int(u.x)
+                    a = A()
+                    a.n = n - 1
+                    u.x = erase(a)
+                    u.tagged = False
+                else:
+                    n = unerase(u.x).n
+                    u.x = rerased.erase_int(n - 1)
+                    u.tagged = True
+        def func():
+            rgc.collect() # check that a prebuilt erased integer doesn't explode
+            u.x = rerased.erase_int(1000)
+            u.tagged = True
+            fn()
+            return 1
+        return func
+
+    def test_erased(self):
+        expected = self.run_orig("erased")
+        res = self.run("erased")
+        assert res == expected
+
 from pypy.rlib.objectmodel import UnboxedValue
 
 class TaggedBase(object):

From noreply at buildbot.pypy.org  Sat Oct 22 06:27:36 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sat, 22 Oct 2011 06:27:36 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix test
Message-ID: <20111022042736.AD770820D7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r48338:f7b183bbffb3
Date: 2011-10-21 21:27 -0700
http://bitbucket.org/pypy/pypy/changeset/f7b183bbffb3/

Log:	fix test

diff --git a/pypy/module/pypyjit/test_pypy_c/test_containers.py b/pypy/module/pypyjit/test_pypy_c/test_containers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_containers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_containers.py
@@ -46,7 +46,7 @@
         assert loop.match_by_id("getitem", """
             i28 = call(ConstClass(ll_dict_lookup__dicttablePtr_objectPtr_Signed), p18, p6, i25, descr=...)
             ...
-            p33 = getinteriorfield_gc(p31, i26, <InteriorFieldDescr <GcPtrFieldDescr dictentry.value .*>>)
+            p33 = getinteriorfield_gc(p31, i26, descr=<InteriorFieldDescr <GcPtrFieldDescr dictentry.value .*>>)
             ...
         """)
 

From noreply at buildbot.pypy.org  Sat Oct 22 11:24:55 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 22 Oct 2011 11:24:55 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Add notes about the
 parallel generational mark-sweep GC.
Message-ID: <20111022092455.96A12820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r48339:90d75dc66ada
Date: 2011-10-22 11:20 +0200
http://bitbucket.org/pypy/pypy/changeset/90d75dc66ada/

Log:	Add notes about the parallel generational mark-sweep GC.

diff --git a/pypy/rpython/memory/gc/concurrentgen.py b/pypy/rpython/memory/gc/concurrentgen.py
new file mode 100644
--- /dev/null
+++ b/pypy/rpython/memory/gc/concurrentgen.py
@@ -0,0 +1,112 @@
+"""
+************************************************************
+  Minor collection cycles of the "concurrentgen" collector
+************************************************************
+
+
+Objects mark byte:
+
+    cym in 'mK': young objs (and all flagged objs)
+    cam in 'Km': aging objs
+    '#'        : old objs
+    'S'        : static prebuilt objs with no heap pointer
+
+'cym' is the current young marker
+'cam' is the current aging marker
+
+The write barrier activates when writing into an object whose
+mark byte is different from 'cym'.
+
+
+------------------------------------------------------------
+
+Step 1.  Only the mutator runs.
+
+   old obj    flagged obj     old obj
+                    |
+                    |
+                    v
+                young obj...
+
+Write barrier: change "old obj" to "flagged obj"
+    (if mark != cym:
+         mark = cym (used to be '#' or 'S')
+         record the object in the "flagged" list)
+    - note that we consider that flagged old objs are again young objects
+
+------------------------------------------------------------
+
+Step 2.  Preparation of running the collector.  (Still single-threaded.)
+
+   - young objs -> aging objs
+         (exchange the values of 'cam' and 'cym'.
+          there was no 'cam' object, so now there is no 'cym' object)
+
+   - collect roots; add roots and flagged objs to the "gray objs" list
+
+   - unflag objs (i.e. empty the "flagged" list)
+
+------------------------------------------------------------
+
+Step 3.  Parallel execution of the collector, mark phase
+
+   old obj    old obj     old obj
+
+         aging obj   aging obj
+
+   new young obj...
+
+
+Collector thread:
+
+    for each gray obj:
+        skip obj if not an aging obj    (i.e. if mark != cam: continue)
+        for each obj found by tracing:
+            add to gray objs      (if not an aging obj, will be skipped later)
+        gray obj -> black obj     (i.e. mark = '#')
+
+Write barrier:
+
+   - perform as a "deletion barrier", detecting changes done to aging objs
+        (i.e. if mark == cam,
+                  mark = '#'
+                  trace and add to gray objs)
+   - also flag old-or-aging objs that point to new young objs
+        (if mark != cym:
+             mark = cym (used to be '#' or 'S')
+             record the object in the "flagged" list)
+
+Threading issues:
+
+   - it's possible that both threads will trace the same object, if we're
+     unlucky, but it does not have buggy effects
+   - the "mark = '#'" in the collector thread can conflict with the
+     "mark = cym" in the mutator write barrier, but again, it should not
+     have buggy effects beyond occasionally triggering the write barrier
+     twice on the same object, adding it twice in "flagged" (and never more)
+   - it is essential to have "mark = '#'" _after_ tracing in the collector
+     thread; otherwise, the write barrier in the mutator thread would be
+     ignored in case it occurs between the two, and then the tracing done
+     by the collector thread doesn't see the original values any more.
+   - the detection of "we are done" in the collector thread needs to
+     account for the write barrier currently tracing and adding more
+     objects to "gray objs".
+
+------------------------------------------------------------
+
+Step 4.  Parallel execution of the collector, sweep phase
+
+    for obj in previous nursery:
+        if obj is "black":     (i.e. if mark != cam)
+            make the obj old   (         nothing to do here, mark already ok)
+        else:
+            clear the object space and return it to the available list
+    after this there are no more aging objects
+
+Write barrier:
+
+   - flag old objs that point to new young objs
+        (should not see any 'cam' object any more here)
+
+------------------------------------------------------------
+"""

From noreply at buildbot.pypy.org  Sat Oct 22 16:27:23 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sat, 22 Oct 2011 16:27:23 +0200 (CEST)
Subject: [pypy-commit] pypy default: mark a few functions in {str,
 unicode}.format as being unroll_safe.  "{} {}".format(i,
 i) now generates very nice code
Message-ID: <20111022142723.24A9A820D7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r48340:9b572a41522b
Date: 2011-10-22 07:27 -0700
http://bitbucket.org/pypy/pypy/changeset/9b572a41522b/

Log:	mark a few functions in {str, unicode}.format as being unroll_safe.
	"{} {}".format(i, i) now generates very nice code

diff --git a/pypy/objspace/std/newformat.py b/pypy/objspace/std/newformat.py
--- a/pypy/objspace/std/newformat.py
+++ b/pypy/objspace/std/newformat.py
@@ -120,6 +120,8 @@
             out.append_slice(s, last_literal, end)
             return out.build()
 
+        # This is only ever called if we're already unrolling _do_build_string
+        @jit.unroll_safe
         def _parse_field(self, start, end):
             s = self.template
             # Find ":" or "!"
@@ -149,6 +151,7 @@
                 i += 1
             return s[start:end], None, end
 
+        @jit.unroll_safe
         def _get_argument(self, name):
             # First, find the argument.
             space = self.space
@@ -207,6 +210,7 @@
                     raise OperationError(space.w_IndexError, w_msg)
             return self._resolve_lookups(w_arg, name, i, end)
 
+        @jit.unroll_safe
         def _resolve_lookups(self, w_obj, name, start, end):
             # Resolve attribute and item lookups.
             space = self.space

From noreply at buildbot.pypy.org  Sat Oct 22 16:38:42 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 22 Oct 2011 16:38:42 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Starting to write the
 "3/4th concurrent" generational mark&sweep GC.
Message-ID: <20111022143842.4355B820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r48341:1739e282bf89
Date: 2011-10-22 16:38 +0200
http://bitbucket.org/pypy/pypy/changeset/1739e282bf89/

Log:	Starting to write the "3/4th concurrent" generational mark&sweep GC.

diff --git a/pypy/rpython/memory/gc/concurrentgen.py b/pypy/rpython/memory/gc/concurrentgen.py
--- a/pypy/rpython/memory/gc/concurrentgen.py
+++ b/pypy/rpython/memory/gc/concurrentgen.py
@@ -1,112 +1,1240 @@
-"""
-************************************************************
-  Minor collection cycles of the "concurrentgen" collector
-************************************************************
+import time, sys
+from pypy.rpython.lltypesystem import lltype, llmemory, llarena, llgroup, rffi
+from pypy.rpython.lltypesystem.llmemory import raw_malloc_usage
+from pypy.rpython.lltypesystem.lloperation import llop
+from pypy.rpython.annlowlevel import llhelper
+from pypy.translator.tool.cbuild import ExternalCompilationInfo
+from pypy.rlib.objectmodel import we_are_translated, running_on_llinterp
+from pypy.rlib.debug import ll_assert, debug_print, debug_start, debug_stop
+from pypy.rlib.rarithmetic import ovfcheck, LONG_BIT, r_uint
+from pypy.rpython.memory.gc.base import GCBase
+from pypy.rpython.memory import gctypelayout
+from pypy.module.thread import ll_thread
 
+#
+# A "3/4th concurrent" generational mark&sweep GC.
+#
+# This uses a separate thread to run the minor collections in parallel,
+# as well as half of the major collections (the sweep phase).  The mark
+# phase is not parallelized.  See concurrentgen.txt for some details.
+#
+# Based on observations that the timing of collections with "minimark"
+# (on translate.py) is: about 15% of the time in minor collections
+# (including 2% in walk_roots), and about 7% in major collections (with
+# probably 3-4% in the marking phase).  So out of a total of 22% this
+# should parallelize 16-17%, i.e. 3/4th.
+#
+# This is an entirely non-moving collector, with a generational write
+# barrier adapted to the concurrent marking done by the collector thread.
+#
 
-Objects mark byte:
+WORD = LONG_BIT // 8
+WORD_POWER_2 = {32: 2, 64: 3}[LONG_BIT]
+assert 1 << WORD_POWER_2 == WORD
+MAXIMUM_SIZE = sys.maxint - (3*WORD-1)
 
-    cym in 'mK': young objs (and all flagged objs)
-    cam in 'Km': aging objs
-    '#'        : old objs
-    'S'        : static prebuilt objs with no heap pointer
 
-'cym' is the current young marker
-'cam' is the current aging marker
+# Objects start with an integer 'tid', which is decomposed as follows.
+# Lowest byte: one of the following values (which are all odd, so
+# let us know if the 'tid' is valid or is just a word-aligned address):
+MARK_BYTE_1       = 0x6D    # 'm', 109
+MARK_BYTE_2       = 0x4B    # 'K', 75
+MARK_BYTE_OLD     = 0x23    # '#', 35
+MARK_BYTE_STATIC  = 0x53    # 'S', 83
+# Next lower byte: a combination of flags.
+FL_WITHHASH       = 0x0100
+FL_EXTRA          = 0x0200
+# And the high half of the word contains the numeric typeid.
 
-The write barrier activates when writing into an object whose
-mark byte is different from 'cym'.
 
+class ConcurrentGenGC(GCBase):
+    _alloc_flavor_ = "raw"
+    inline_simple_malloc = True
+    inline_simple_malloc_varsize = True
+    needs_deletion_barrier = True
+    needs_weakref_read_barrier = True
+    prebuilt_gc_objects_are_static_roots = False
+    malloc_zero_filled = True
+    gcflag_extra = FL_EXTRA
 
-------------------------------------------------------------
+    HDR = lltype.Struct('header', ('tid', lltype.Signed))
+    HDRPTR = lltype.Ptr(HDR)
+    HDRSIZE = llmemory.sizeof(HDR)
+    NULL = lltype.nullptr(HDR)
+    typeid_is_in_field = 'tid', llgroup.HALFSHIFT
+    withhash_flag_is_in_field = 'tid', FL_WITHHASH
+    # ^^^ prebuilt objects may have the flag FL_WITHHASH;
+    #     then they are one word longer, the extra word storing the hash.
 
-Step 1.  Only the mutator runs.
+    TRANSLATION_PARAMS = {'page_size': 16384,
+                          'small_request_threshold': 35*WORD,
+                          }
 
-   old obj    flagged obj     old obj
-                    |
-                    |
-                    v
-                young obj...
+    def __init__(self, config, page_size=64, small_request_threshold=24,
+                 **kwds):
+        # 'small_request_threshold' is the largest size that we will
+        # satisfy using our own pages mecanism.  Larger requests just
+        # go to the system malloc().
+        self.addressstack_lock_object = SyncLock()
+        kwds['lock'] = self.addressstack_lock_object
+        GCBase.__init__(self, config, **kwds)
+        assert small_request_threshold % WORD == 0
+        self.small_request_threshold = small_request_threshold
+        self.page_size = page_size
+        self.pagelists_length = small_request_threshold // WORD + 1
+        #
+        # The following are arrays of 36 linked lists: the linked lists
+        # at indices 1 to 35 correspond to pages that store objects of
+        # size  1 * WORD  to  35 * WORD,  and the linked list at index 0
+        # is a list of all larger objects.
+        def list_of_addresses_per_small_size():
+            return lltype.malloc(rffi.CArray(self.HDRPTR),
+                                 self.pagelists_length, flavor='raw',
+                                 immortal=True)
+        # 1-35: a linked list of all pages; 0: a linked list of all larger objs
+        self.nonfree_pages = list_of_addresses_per_small_size()
+        # a snapshot of 'nonfree_pages' done when the collection starts
+        self.collect_pages = list_of_addresses_per_small_size()
+        # 1-35: free list of non-allocated locations; 0: unused
+        self.free_lists    = list_of_addresses_per_small_size()
+        # 1-35: head and tail of the free list built by the collector thread
+        # 0: head and tail of the linked list of surviving large objects
+        self.collect_heads = list_of_addresses_per_small_size()
+        self.collect_tails = list_of_addresses_per_small_size()
+        #
+        def collector_start():
+            if we_are_translated():
+                self.collector_run()
+            else:
+                self.collector_run_nontranslated()
+        #
+        collector_start._should_never_raise_ = True
+        self.collector_start = collector_start
+        #
+        self.gray_objects = self.AddressStack()
+        self.extra_objects_to_mark = self.AddressStack()
+        self.flagged_objects = self.AddressStack()
+        self.prebuilt_root_objects = self.AddressStack()
+        #
+        self._initialize()
+        #
+        # Write barrier: actually a deletion barrier, triggered when there
+        # is a collection running and the mutator tries to change an object
+        # that was not scanned yet.
+        self._init_writebarrier_logic()
 
-Write barrier: change "old obj" to "flagged obj"
-    (if mark != cym:
-         mark = cym (used to be '#' or 'S')
-         record the object in the "flagged" list)
-    - note that we consider that flagged old objs are again young objects
+    def _clear_list(self, array):
+        i = 0
+        while i < self.pagelists_length:
+            array[i] = self.NULL
+            i += 1
 
-------------------------------------------------------------
+    def _initialize(self):
+        self.free_pages = self.NULL
+        #
+        # Clear the lists
+        self._clear_list(self.nonfree_pages)
+        self._clear_list(self.collect_pages)
+        self._clear_list(self.free_lists)
+        self._clear_list(self.collect_heads)
+        self._clear_list(self.collect_tails)
+        #
+        self.finalizer_pages = self.NULL
+        self.collect_finalizer_pages = self.NULL
+        self.collect_finalizer_tails = self.NULL
+        self.collect_run_finalizers_head = self.NULL
+        self.collect_run_finalizers_tail = self.NULL
+        self.objects_with_finalizers_to_run = self.NULL
+        #
+        self.weakref_pages = self.NULL
+        self.collect_weakref_pages = self.NULL
+        self.collect_weakref_tails = self.NULL
+        #
+        # See concurrentgen.txt for more information about these fields.
+        self.current_young_marker = MARK_BYTE_1
+        self.current_aging_marker = MARK_BYTE_2
+        #
+        # When the mutator thread wants to trigger the next collection,
+        # it scans its own stack roots and prepares everything, then
+        # sets 'collection_running' to 1, and releases
+        # 'ready_to_start_lock'.  This triggers the collector thread,
+        # which re-acquires 'ready_to_start_lock' and does its job.
+        # When done it releases 'finished_lock'.  The mutator thread is
+        # responsible for resetting 'collection_running' to 0.
+        #
+        # The collector thread's state can be found (with careful locking)
+        # by inspecting the same variable from the mutator thread:
+        #   * collection_running == 1: Marking.  [Deletion barrier active.]
+        #   * collection_running == 2: Clearing weakrefs.
+        #   * collection_running == 3: Marking from unreachable finalizers.
+        #   * collection_running == 4: Sweeping.
+        #   * collection_running == -1: Done.
+        # The mutex_lock is acquired to go from 1 to 2, and from 2 to 3.
+        self.collection_running = 0
+        #self.ready_to_start_lock = ...built in setup()
+        #self.finished_lock = ...built in setup()
+        #
+        #self.mutex_lock = ...built in setup()
+        self.gray_objects.clear()
+        self.extra_objects_to_mark.clear()
+        self.flagged_objects.clear()
+        self.prebuilt_root_objects.clear()
 
-Step 2.  Preparation of running the collector.  (Still single-threaded.)
+    def setup(self):
+        "Start the concurrent collector thread."
+        # don't call GCBase.setup(self), because we don't need
+        # 'run_finalizers' as a deque
+        self.finalizer_lock_count = 0
+        #
+        self.main_thread_ident = ll_thread.get_ident()
+        self.ready_to_start_lock = ll_thread.allocate_ll_lock()
+        self.finished_lock = ll_thread.allocate_ll_lock()
+        self.mutex_lock = ll_thread.allocate_ll_lock()
+        self.addressstack_lock_object.setup()
+        #
+        self.acquire(self.finished_lock)
+        self.acquire(self.ready_to_start_lock)
+        #
+        self.collector_ident = ll_thread.c_thread_start_nowrapper(
+            llhelper(ll_thread.CALLBACK, self.collector_start))
+        assert self.collector_ident != -1
 
-   - young objs -> aging objs
-         (exchange the values of 'cam' and 'cym'.
-          there was no 'cam' object, so now there is no 'cym' object)
+    def _teardown(self):
+        "Stop the collector thread after tests have run."
+        self.wait_for_the_end_of_collection()
+        #
+        # start the next collection, but with collection_running set to 42,
+        # which should shut down the collector thread
+        self.collection_running = 42
+        debug_print("teardown!")
+        self.release(self.ready_to_start_lock)
+        self.acquire(self.finished_lock)
+        self._initialize()
 
-   - collect roots; add roots and flagged objs to the "gray objs" list
+    def get_type_id(self, obj):
+        tid = self.header(obj).tid
+        return llop.extract_high_ushort(llgroup.HALFWORD, tid)
 
-   - unflag objs (i.e. empty the "flagged" list)
+    def combine(self, typeid16, mark, flags):
+        return llop.combine_high_ushort(lltype.Signed, typeid16, mark | flags)
 
-------------------------------------------------------------
+    def init_gc_object_immortal(self, addr, typeid, flags=0):
+        # 'flags' is ignored here
+        hdr = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(self.HDR))
+        hdr.tid = self.combine(typeid, MARK_BYTE_STATIC, 0)
 
-Step 3.  Parallel execution of the collector, mark phase
+    def malloc_fixedsize_clear(self, typeid, size,
+                               needs_finalizer=False, contains_weakptr=False):
+        # contains_weakptr: detected during collection
+        #
+        # Case of finalizers (test constant-folded)
+        if needs_finalizer:
+            ll_assert(not contains_weakptr,
+                     "'needs_finalizer' and 'contains_weakptr' both specified")
+            return self._malloc_with_finalizer(typeid, size)
+        #
+        # Case of weakreferences (test constant-folded)
+        if contains_weakptr:
+            return self._malloc_weakref(typeid, size)
+        #
+        # Regular case
+        size_gc_header = self.gcheaderbuilder.size_gc_header
+        totalsize = size_gc_header + size
+        rawtotalsize = raw_malloc_usage(totalsize)
+        if rawtotalsize <= self.small_request_threshold:
+            ll_assert(rawtotalsize & (WORD - 1) == 0,
+                      "fixedsize not properly rounded")
+            #
+            n = rawtotalsize >> WORD_POWER_2
+            result = self.free_lists[n]
+            if result != self.NULL:
+                self.free_lists[n] = list_next(result)
+                obj = self.grow_reservation(result, totalsize)
+                hdr = self.header(obj)
+                hdr.tid = self.combine(typeid, self.current_young_marker, 0)
+                #debug_print("malloc_fixedsize_clear", obj)
+                return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
+                #
+        return self._malloc_slowpath(typeid, size)
 
-   old obj    old obj     old obj
+    def malloc_varsize_clear(self, typeid, length, size, itemsize,
+                             offset_to_length):
+        size_gc_header = self.gcheaderbuilder.size_gc_header
+        nonvarsize = size_gc_header + size
+        #
+        # Compute the maximal length that makes the object still below
+        # 'small_request_threshold'.  All the following logic is usually
+        # constant-folded because size and itemsize are constants (due
+        # to inlining).
+        maxsize = self.small_request_threshold - raw_malloc_usage(nonvarsize)
+        if maxsize < 0:
+            toobig = r_uint(0)    # the nonvarsize alone is too big
+        elif raw_malloc_usage(itemsize):
+            toobig = r_uint(maxsize // raw_malloc_usage(itemsize)) + 1
+        else:
+            toobig = r_uint(sys.maxint) + 1
 
-         aging obj   aging obj
+        if r_uint(length) < r_uint(toobig):
+            # With the above checks we know now that totalsize cannot be more
+            # than 'small_request_threshold'; in particular, the + and *
+            # cannot overflow.
+            totalsize = nonvarsize + itemsize * length
+            totalsize = llarena.round_up_for_allocation(totalsize)
+            rawtotalsize = raw_malloc_usage(totalsize)
+            ll_assert(rawtotalsize & (WORD - 1) == 0,
+                      "round_up_for_allocation failed")
+            #
+            n = rawtotalsize >> WORD_POWER_2
+            result = self.free_lists[n]
+            if result != self.NULL:
+                self.free_lists[n] = list_next(result)
+                obj = self.grow_reservation(result, totalsize)
+                hdr = self.header(obj)
+                hdr.tid = self.combine(typeid, self.current_young_marker, 0)
+                (obj + offset_to_length).signed[0] = length
+                #debug_print("malloc_varsize_clear", obj)
+                return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
+        #
+        # If the total size of the object would be larger than
+        # 'small_request_threshold', or if the free_list is empty,
+        # then allocate it externally.  We also go there if 'length'
+        # is actually negative.
+        return self._malloc_varsize_slowpath(typeid, length)
 
-   new young obj...
+    def _malloc_slowpath(self, typeid, size):
+        # Slow-path malloc.  Call this with 'size' being a valid and
+        # rounded number, between WORD and up to MAXIMUM_SIZE.
+        #
+        # For now, we always start the next collection immediately.
+        if self.collection_running <= 0:
+            self.trigger_next_collection()
+        #
+        size_gc_header = self.gcheaderbuilder.size_gc_header
+        totalsize = size_gc_header + size
+        rawtotalsize = raw_malloc_usage(totalsize)
+        #
+        if rawtotalsize <= self.small_request_threshold:
+            #
+            # Case 1: unless trigger_next_collection() happened to get us
+            # more locations in free_lists[n], we have run out of them
+            ll_assert(rawtotalsize & (WORD - 1) == 0,
+                      "malloc_slowpath: non-rounded size")
+            n = rawtotalsize >> WORD_POWER_2
+            head = self.free_lists[n]
+            if head:
+                self.free_lists[n] = list_next(head)
+                obj = self.grow_reservation(head, totalsize)
+                hdr = self.header(obj)
+                hdr.tid = self.combine(typeid, self.current_young_marker, 0)
+                return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
+            #
+            # We really have run out of the free list corresponding to
+            # the size.  Grab the next free page.
+            newpage = self.free_pages
+            if newpage == self.NULL:
+                self.allocate_next_arena()
+                newpage = self.free_pages
+            self.free_pages = list_next(newpage)
+            #
+            # Put the free page in the list 'nonfree_pages[n]'.  This is
+            # a linked list chained through the first word of each page.
+            set_next(newpage, self.nonfree_pages[n])
+            self.nonfree_pages[n] = newpage
+            #
+            # Initialize the free page to contain objects of the given
+            # size.  This requires setting up all object locations in the
+            # page, linking them in the free list.
+            i = self.page_size - rawtotalsize
+            limit = rawtotalsize + raw_malloc_usage(self.HDRSIZE)
+            newpageadr = llmemory.cast_ptr_to_adr(newpage)
+            newpageadr = llarena.getfakearenaaddress(newpageadr)
+            while i >= limit:
+                adr = newpageadr + i
+                llarena.arena_reserve(adr, self.HDRSIZE)
+                p = llmemory.cast_adr_to_ptr(adr, self.HDRPTR)
+                set_next(p, head)
+                head = p
+                i -= rawtotalsize
+            self.free_lists[n] = head
+            result = newpageadr + i
+            #
+            # Done: all object locations are linked, apart from
+            # 'result', which is the first object location in the page.
+            # Note that if the size is not an exact divisor of
+            # 4096-WORD, there are a few wasted WORDs, which we place at
+            # the start of the page rather than at the end (Hans Boehm,
+            # xxx ref).
+            #
+            return self._malloc_result(typeid, totalsize, result)
+        else:
+            # Case 2: the object is too large, so allocate it directly
+            # with the system malloc().
+            return self._malloc_large_object(typeid, size, 0)
+        #
+    _malloc_slowpath._dont_inline_ = True
 
+    def _malloc_result(self, typeid, totalsize, result):
+        llarena.arena_reserve(result, totalsize)
+        hdr = llmemory.cast_adr_to_ptr(result, self.HDRPTR)
+        hdr.tid = self.combine(typeid, self.current_young_marker, 0)
+        obj = result + self.gcheaderbuilder.size_gc_header
+        #debug_print("malloc_slowpath", obj)
+        return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
 
-Collector thread:
+    def _malloc_large_object(self, typeid, size, linked_list):
+        # xxx on 32-bit, we'll prefer 64-bit alignment of the object by
+        # always allocating an 8-bytes header
+        totalsize = self.gcheaderbuilder.size_gc_header + size
+        rawtotalsize = raw_malloc_usage(totalsize)
+        rawtotalsize += 8
+        block = llarena.arena_malloc(rawtotalsize, 2)
+        if not block:
+            raise MemoryError
+        llarena.arena_reserve(block, self.HDRSIZE)
+        blockhdr = llmemory.cast_adr_to_ptr(block, self.HDRPTR)
+        if linked_list == 0:
+            set_next(blockhdr, self.nonfree_pages[0])
+            self.nonfree_pages[0] = blockhdr
+        elif linked_list == 1:
+            set_next(blockhdr, self.finalizer_pages)
+            self.finalizer_pages = blockhdr
+        elif linked_list == 2:
+            set_next(blockhdr, self.weakref_pages)
+            self.weakref_pages = blockhdr
+        else:
+            ll_assert(0, "bad linked_list")
+        return self._malloc_result(typeid, totalsize, block + 8)
+    _malloc_large_object._annspecialcase_ = 'specialize:arg(3)'
+    _malloc_large_object._dont_inline_ = True
 
-    for each gray obj:
-        skip obj if not an aging obj    (i.e. if mark != cam: continue)
-        for each obj found by tracing:
-            add to gray objs      (if not an aging obj, will be skipped later)
-        gray obj -> black obj     (i.e. mark = '#')
+    def _malloc_varsize_slowpath(self, typeid, length):
+        #
+        if length < 0:
+            # negative length!  This likely comes from an overflow
+            # earlier.  We will just raise MemoryError here.
+            raise MemoryError
+        #
+        # Compute the total size, carefully checking for overflows.
+        nonvarsize = self.fixed_size(typeid)
+        itemsize = self.varsize_item_sizes(typeid)
+        try:
+            varsize = ovfcheck(itemsize * length)
+            totalsize = ovfcheck(nonvarsize + varsize)
+        except OverflowError:
+            raise MemoryError
+        #
+        # Detect very rare cases of overflows
+        if raw_malloc_usage(totalsize) > MAXIMUM_SIZE:
+            raise MemoryError("rare case of overflow")
+        #
+        totalsize = llarena.round_up_for_allocation(totalsize)
+        result = self._malloc_slowpath(typeid, totalsize)
+        #
+        offset_to_length = self.varsize_offset_to_length(typeid)
+        obj = llmemory.cast_ptr_to_adr(result)
+        (obj + offset_to_length).signed[0] = length
+        return result
+    _malloc_varsize_slowpath._dont_inline_ = True
 
-Write barrier:
+    def _malloc_with_finalizer(self, typeid, size):
+        return self._malloc_large_object(typeid, size, 1)
 
-   - perform as a "deletion barrier", detecting changes done to aging objs
-        (i.e. if mark == cam,
-                  mark = '#'
-                  trace and add to gray objs)
-   - also flag old-or-aging objs that point to new young objs
-        (if mark != cym:
-             mark = cym (used to be '#' or 'S')
-             record the object in the "flagged" list)
+    def _malloc_weakref(self, typeid, size):
+        return self._malloc_large_object(typeid, size, 2)
 
-Threading issues:
+    # ----------
+    # Other functions in the GC API
 
-   - it's possible that both threads will trace the same object, if we're
-     unlucky, but it does not have buggy effects
-   - the "mark = '#'" in the collector thread can conflict with the
-     "mark = cym" in the mutator write barrier, but again, it should not
-     have buggy effects beyond occasionally triggering the write barrier
-     twice on the same object, adding it twice in "flagged" (and never more)
-   - it is essential to have "mark = '#'" _after_ tracing in the collector
-     thread; otherwise, the write barrier in the mutator thread would be
-     ignored in case it occurs between the two, and then the tracing done
-     by the collector thread doesn't see the original values any more.
-   - the detection of "we are done" in the collector thread needs to
-     account for the write barrier currently tracing and adding more
-     objects to "gray objs".
+    #def set_max_heap_size(self, size):
+    #    XXX
 
-------------------------------------------------------------
+    #def raw_malloc_memory_pressure(self, sizehint):
+    #    XXX
 
-Step 4.  Parallel execution of the collector, sweep phase
+    #def shrink_array(self, obj, smallerlength):
+    #    no real point in supporting this, but if you think it's a good
+    #    idea, remember that changing the array length at run-time needs
+    #    extra care for the collector thread
 
-    for obj in previous nursery:
-        if obj is "black":     (i.e. if mark != cam)
-            make the obj old   (         nothing to do here, mark already ok)
+    def enumerate_all_roots(self, callback, arg):
+        self.flagged_objects.foreach(callback, arg)
+        GCBase.enumerate_all_roots(self, callback, arg)
+    enumerate_all_roots._annspecialcase_ = 'specialize:arg(1)'
+
+    def identityhash(self, obj):
+        obj = llmemory.cast_ptr_to_adr(obj)
+        if self.header(obj).tid & FL_WITHHASH:
+            obj += self.get_size(obj)
+            return obj.signed[0]
         else:
-            clear the object space and return it to the available list
-    after this there are no more aging objects
+            return llmemory.cast_adr_to_int(obj)
 
-Write barrier:
+    # ----------
 
-   - flag old objs that point to new young objs
-        (should not see any 'cam' object any more here)
+    def allocate_next_arena(self):
+        # xxx for now, allocate one page at a time with the system malloc()
+        page = llarena.arena_malloc(self.page_size, 2)     # zero-filled
+        if not page:
+            raise MemoryError
+        llarena.arena_reserve(page, self.HDRSIZE)
+        page = llmemory.cast_adr_to_ptr(page, self.HDRPTR)
+        page.tid = 0
+        self.free_pages = page
 
-------------------------------------------------------------
-"""
+    def grow_reservation(self, hdr, totalsize):
+        # Transform 'hdr', which used to point to just a HDR,
+        # into a pointer to a full object of size 'totalsize'.
+        # This is a no-op after translation.  Returns the
+        # address of the full object.
+        adr = llmemory.cast_ptr_to_adr(hdr)
+        adr = llarena.getfakearenaaddress(adr)
+        llarena.arena_reserve(adr, totalsize)
+        return adr + self.gcheaderbuilder.size_gc_header
+    grow_reservation._always_inline_ = True
+
+    def deletion_barrier(self, addr_struct):
+        # XXX check the assembler
+        mark = self.header(addr_struct).tid & 0xFF
+        if mark != self.current_young_marker:
+            self.force_scan(addr_struct)
+        #else:
+        #    debug_print("deletion_barrier (off)", addr_struct)
+
+    def assume_young_pointers(self, addr_struct):
+        pass # XXX
+
+    def _init_writebarrier_logic(self):
+        #
+        def force_scan(obj):
+            #debug_print("deletion_barrier  ON  ", obj)
+            cym = self.current_young_marker
+            mark = self.get_mark(obj)
+            #
+            if mark == MARK_BYTE_OLD:
+                #
+                self.set_mark(obj, cym)
+                #
+            elif mark == MARK_BYTE_STATIC:
+                # This is the first write into a prebuilt GC object.
+                # Record it in 'prebuilt_root_objects'.
+                self.set_mark(obj, cym)
+                self.prebuilt_root_objects.append(obj)
+                #
+            else:
+                #
+                # Only acquire the mutex_lock if necessary
+                self.acquire(self.mutex_lock)
+                #
+                # Reload the possibly changed marker from the object header,
+                # and set it to 'cym'
+                mark = self.get_mark(obj)
+                self.set_mark(obj, cym)
+                #
+                if mark == self.current_aging_marker:
+                    #
+                    # it is only possible to reach this point if there is
+                    # a collection running in collector_mark(), before it
+                    # does mutex_lock itself.  Check this:
+                    ll_assert(self.collection_running == 1,
+                              "write barrier: wrong call?")
+                    #
+                    # It's fine to set the mark before tracing, because
+                    # we are anyway in a 'mutex_lock' critical section.
+                    # The collector thread will not exit from the phase
+                    # 'collection_running == 1' here.
+                    self.trace(obj, self._barrier_add_extra, None)
+                    #
+                    # Still at 1:
+                    ll_assert(self.collection_running == 1,
+                              "write barrier: oups!?")
+                    #
+                else:
+                    # MARK_BYTE_OLD is possible here: the collector thread
+                    # sets it in parallel to objects.  In that case it has
+                    # been handled already.
+                    ll_assert(mark == MARK_BYTE_OLD,
+                              "write barrier: bogus object mark")
+                #
+                self.release(self.mutex_lock)
+            #
+            # In all cases, the object is now flagged
+            self.flagged_objects.append(obj)
+        #
+        force_scan._dont_inline_ = True
+        self.force_scan = force_scan
+
+    def _barrier_add_extra(self, root, ignored):
+        obj = root.address[0]
+        self.get_mark(obj)
+        self.extra_objects_to_mark.append(obj)
+
+
+    def wait_for_the_end_of_collection(self):
+        """In the mutator thread: wait for the minor collection currently
+        running (if any) to finish."""
+        if self.collection_running != 0:
+            debug_start("gc-stop")
+            #
+            self.acquire(self.finished_lock)
+            self.collection_running = 0
+            #debug_print("collection_running = 0")
+            #
+            # Check invariants
+            ll_assert(not self.extra_objects_to_mark.non_empty(),
+                      "objs left behind in extra_objects_to_mark")
+            ll_assert(not self.gray_objects.non_empty(),
+                      "objs left behind in gray_objects")
+            #
+            # Grab the results of the last collection: read the collector's
+            # 'collect_heads/collect_tails' and merge them with the mutator's
+            # 'free_lists'.
+            n = 1
+            while n < self.pagelists_length:
+                self.free_lists[n] = self.join_lists(self.free_lists[n],
+                                                     self.collect_heads[n],
+                                                     self.collect_tails[n])
+                n += 1
+            #
+            # Do the same with 'collect_heads[0]/collect_tails[0]'.
+            self.nonfree_pages[0] = self.join_lists(self.nonfree_pages[0],
+                                                    self.collect_heads[0],
+                                                    self.collect_tails[0])
+            #
+            # Do the same with 'collect_weakref_pages/tails'
+            self.weakref_pages = self.join_lists(self.weakref_pages,
+                                                 self.collect_weakref_pages,
+                                                 self.collect_weakref_tails)
+            #
+            # Do the same with 'collect_finalizer_pages/tails'
+            self.finalizer_pages = self.join_lists(self.finalizer_pages,
+                                                  self.collect_finalizer_pages,
+                                                  self.collect_finalizer_tails)
+            #
+            # Do the same with 'collect_run_finalizers_head/tail'
+            self.objects_with_finalizers_to_run = self.join_lists(
+                self.objects_with_finalizers_to_run,
+                self.collect_run_finalizers_head,
+                self.collect_run_finalizers_tail)
+            #
+            if self.DEBUG:
+                self.debug_check_lists()
+            #
+            debug_stop("gc-stop")
+            #
+            # We must *not* run execute_finalizers_ll() here, because it
+            # can start the next collection, and then this function returns
+            # with a collection in progress, which it should not.  Be careful
+            # to call execute_finalizers_ll() in the caller somewhere.
+            ll_assert(self.collection_running == 0,
+                      "collector thread not paused?")
+
+    def join_lists(self, list1, list2head, list2tail):
+        if list2tail == self.NULL:
+            ll_assert(list2head == self.NULL, "join_lists/1")
+            return list1
+        else:
+            ll_assert(list2head != self.NULL, "join_lists/2")
+            set_next(list2tail, list1)
+            return list2head
+
+
+    def execute_finalizers_ll(self):
+        self.finalizer_lock_count += 1
+        try:
+            while self.objects_with_finalizers_to_run != self.NULL:
+                if self.finalizer_lock_count > 1:
+                    # the outer invocation of execute_finalizers() will do it
+                    break
+                #
+                x = llmemory.cast_ptr_to_adr(
+                        self.objects_with_finalizers_to_run)
+                x = llarena.getfakearenaaddress(x) + 8
+                obj = x + self.gcheaderbuilder.size_gc_header
+                self.objects_with_finalizers_to_run = list_next(
+                    self.objects_with_finalizers_to_run)
+                #
+                finalizer = self.getfinalizer(self.get_type_id(obj))
+                finalizer(obj, llmemory.NULL)
+        finally:
+            self.finalizer_lock_count -= 1
+
+
+    def collect(self, gen=3):
+        """
+        gen=0: Trigger a minor collection if none is running.  Never blocks.
+        
+        gen=1: The same, but if a minor collection is running, wait for
+        it to finish before triggering the next one.  Guarantees that
+        young objects not reachable when collect() is called will soon
+        be freed.
+
+        gen=2: The same, but wait for the triggered collection to
+        finish.  Guarantees that young objects not reachable when
+        collect() is called will be freed by the time collect() returns.
+
+        gen>=3: Do a (synchronous) major collection.
+        """
+        if gen >= 1 or self.collection_running <= 0:
+            self.trigger_next_collection()
+            if gen >= 2:
+                self.wait_for_the_end_of_collection()
+                if gen >= 3:
+                    self.major_collection()
+        self.execute_finalizers_ll()
+
+    def trigger_next_collection(self):
+        """In the mutator thread: triggers the next minor collection."""
+        #
+        # In case the previous collection is not over yet, wait for it
+        self.wait_for_the_end_of_collection()
+        #
+        debug_start("gc-start")
+        #
+        # Scan the stack roots and the refs in non-GC objects
+        self.root_walker.walk_roots(
+            ConcurrentGenGC._add_stack_root,  # stack roots
+            ConcurrentGenGC._add_stack_root,  # in prebuilt non-gc
+            None)                         # static in prebuilt gc
+        #
+        # Add the prebuilt root objects that have been written to
+        self.flagged_objects.foreach(self._add_prebuilt_root, None)
+        #
+        # Add the objects still waiting in 'objects_with_finalizers_to_run'
+        p = self.objects_with_finalizers_to_run
+        while p != self.NULL:
+            x = llmemory.cast_ptr_to_adr(p)
+            x = llarena.getfakearenaaddress(x) + 8
+            obj = x + self.gcheaderbuilder.size_gc_header
+            #debug_print("_objects_with_finalizers_to_run", obj)
+            self.get_mark(obj)
+            self.gray_objects.append(obj)
+            p = list_next(p)
+        #
+        # Exchange the meanings of 'cym' and 'cam'
+        other = self.current_young_marker
+        self.current_young_marker = self.current_aging_marker
+        self.current_aging_marker = other
+        #
+        # Copy a few 'mutator' fields to 'collector' fields:
+        # 'collect_pages' make linked lists of all nonfree pages at the
+        # start of the collection (unlike the 'nonfree_pages' lists, which
+        # the mutator will continue to grow).
+        n = 0
+        while n < self.pagelists_length:
+            self.collect_pages[n] = self.nonfree_pages[n]
+            n += 1
+        self.collect_weakref_pages = self.weakref_pages
+        self.collect_finalizer_pages = self.finalizer_pages
+        #
+        # Clear the following lists.  When the collector thread finishes,
+        # it will give back (in collect_{pages,tails}[0] and
+        # collect_finalizer_{pages,tails}) all the original items that survive.
+        self.nonfree_pages[0] = self.NULL
+        self.weakref_pages = self.NULL
+        self.finalizer_pages = self.NULL
+        #
+        # Start the collector thread
+        self.collection_running = 1
+        #debug_print("collection_running = 1")
+        self.release(self.ready_to_start_lock)
+        #
+        debug_stop("gc-start")
+        #
+        self.execute_finalizers_ll()
+
+    def _add_stack_root(self, root):
+        obj = root.address[0]
+        #debug_print("_add_stack_root", obj)
+        self.get_mark(obj)
+        self.gray_objects.append(obj)
+
+    def _add_prebuilt_root(self, obj, ignored):
+        #debug_print("_add_prebuilt_root", obj)
+        self.get_mark(obj)
+        self.gray_objects.append(obj)
+
+    def debug_check_lists(self):
+        # just check that they are correct, non-infinite linked lists
+        self.debug_check_list(self.nonfree_pages[0])
+        n = 1
+        while n < self.pagelists_length:
+            self.debug_check_list(self.free_lists[n])
+            n += 1
+        self.debug_check_list(self.weakref_pages)
+        self.debug_check_list(self.finalizer_pages)
+        self.debug_check_list(self.objects_with_finalizers_to_run)
+
+    def debug_check_list(self, page):
+        try:
+            previous_page = self.NULL
+            count = 0
+            while page != self.NULL:
+                # prevent constant-folding, and detects loops of length 1
+                ll_assert(page != previous_page, "loop!")
+                previous_page = page
+                page = list_next(page)
+                count += 1
+            return count
+        except KeyboardInterrupt:
+            ll_assert(False, "interrupted")
+            raise
+
+    def acquire(self, lock):
+        if (we_are_translated() or
+                ll_thread.get_ident() != self.main_thread_ident):
+            ll_thread.c_thread_acquirelock(lock, 1)
+        else:
+            while rffi.cast(lltype.Signed,
+                            ll_thread.c_thread_acquirelock(lock, 0)) == 0:
+                time.sleep(0.05)
+                # ---------- EXCEPTION FROM THE COLLECTOR THREAD ----------
+                if hasattr(self, '_exc_info'):
+                    self._reraise_from_collector_thread()
+
+    def release(self, lock):
+        ll_thread.c_thread_releaselock(lock)
+
+    def _reraise_from_collector_thread(self):
+        exc, val, tb = self._exc_info
+        raise exc, val, tb
+
+
+    def collector_run_nontranslated(self):
+        try:
+            if not hasattr(self, 'currently_running_in_rtyper'):
+                self.collector_run()     # normal tests
+            else:
+                # this case is for test_transformed_gc: we need to spawn
+                # another LLInterpreter for this new thread.
+                from pypy.rpython.llinterp import LLInterpreter
+                llinterp = LLInterpreter(self.currently_running_in_rtyper)
+                # XXX FISH HORRIBLY for the graph...
+                graph = sys._getframe(2).f_locals['self']._obj.graph
+                llinterp.eval_graph(graph)
+        except Exception, e:
+            print 'Crash!', e.__class__.__name__, e
+            #import pdb; pdb.post_mortem(sys.exc_info()[2])
+            self._exc_info = sys.exc_info()
+
+    def collector_run(self):
+        """Main function of the collector's thread."""
+        #
+        # hack: this is an infinite loop in practice.  During tests it can
+        # be interrupted.  In all cases the annotator must not conclude that
+        # it is an infinite loop: otherwise, the caller would automatically
+        # end in a "raise AssertionError", annoyingly, because we don't want
+        # any exception in this thread
+        while True:
+            #
+            # Wait for the lock to be released
+            self.acquire(self.ready_to_start_lock)
+            #
+            # For tests: detect when we have to shut down
+            if self.collection_running == 42:
+                self.release(self.finished_lock)
+                break
+            #
+            # Mark                                # collection_running == 1
+            self.collector_mark()
+            #                                     # collection_running == 2
+            self.deal_with_weakrefs()
+            #                                     # collection_running == 3
+            self.deal_with_objects_with_finalizers()
+            # Sweep                               # collection_running == 4
+            self.collector_sweep()
+            # Done!                               # collection_running == -1
+            self.release(self.finished_lock)
+
+
+    def set_mark(self, obj, newmark):
+        _set_mark(self.header(obj), newmark)
+
+    def get_mark(self, obj):
+        mark = self.header(obj).tid & 0xFF
+        ll_assert(mark == MARK_BYTE_1 or
+                  mark == MARK_BYTE_2 or
+                  mark == MARK_BYTE_OLD or
+                  mark == MARK_BYTE_STATIC, "bad mark byte in object")
+        return mark
+
+    def collector_mark(self):
+        while True:
+            #
+            # Do marking.  The following function call is interrupted
+            # if the mutator's write barrier adds new objects to
+            # 'extra_objects_to_mark'.
+            self._collect_mark()
+            #
+            # Move the objects from 'extra_objects_to_mark' to
+            # 'gray_objects'.  This requires the mutex lock.
+            # There are typically only a few objects to move here,
+            # unless XXX we've hit the write barrier of a large array
+            self.acquire(self.mutex_lock)
+            #debug_print("...collector thread has mutex_lock")
+            while self.extra_objects_to_mark.non_empty():
+                obj = self.extra_objects_to_mark.pop()
+                self.get_mark(obj)
+                self.gray_objects.append(obj)
+            #
+            # If 'gray_objects' is empty, we are done: there should be
+            # no possible case in which more objects are being added to
+            # 'extra_objects_to_mark' concurrently, because 'gray_objects'
+            # and 'extra_objects_to_mark' were already empty before we
+            # acquired the 'mutex_lock', so all reachable objects have
+            # been marked.
+            if not self.gray_objects.non_empty():
+                break
+            #
+            # Else release mutex_lock and try again.
+            self.release(self.mutex_lock)
+        #
+        self.collection_running = 2
+        #debug_print("collection_running = 2")
+        self.release(self.mutex_lock)
+
+    def _collect_mark(self):
+        cam = self.current_aging_marker
+        while self.gray_objects.non_empty():
+            obj = self.gray_objects.pop()
+            if self.get_mark(obj) == cam:
+                #
+                # Scan the content of 'obj'.  We use a snapshot-at-the-
+                # beginning order, meaning that we want to scan the state
+                # of the object as it was at the beginning of the current
+                # collection --- and not the current state, which might have
+                # been modified.  That's why we have a deletion barrier:
+                # when the mutator thread is about to change an object that
+                # is not yet marked, it will itself do the scanning of just
+                # this object, and mark the object.  But this function is not
+                # synchronized, which means that in some rare cases it's
+                # possible that the object is scanned a second time here
+                # (harmlessly).
+                #
+                # The order of the next two lines is essential!  *First*
+                # scan the object, adding all objects found to gray_objects;
+                # and *only then* set the mark.  This is essential, because
+                # otherwise, we might set the mark, then the main thread
+                # thinks a force_scan() is not necessary and modifies the
+                # content of 'obj', and then here in the collector thread
+                # we scan a modified content --- and the original content
+                # is never scanned.
+                #
+                self.trace(obj, self._collect_add_pending, None)
+                self.set_mark(obj, MARK_BYTE_OLD)
+                #
+                # Interrupt early if the mutator's write barrier adds stuff
+                # to that list.  Note that the check is imprecise because
+                # it is not lock-protected, but that's good enough.  The
+                # idea is that we trace in priority objects flagged with
+                # the write barrier, because they are more likely to
+                # reference further objects that will soon be accessed too.
+                if self.extra_objects_to_mark.non_empty():
+                    return
+
+    def _collect_add_pending(self, root, ignored):
+        obj = root.address[0]
+        self.get_mark(obj)
+        self.gray_objects.append(obj)
+
+    def collector_sweep(self):
+        self._collect_sweep_large_objects()
+        #
+        n = 1
+        while n < self.pagelists_length:
+            self._collect_sweep_pages(n)
+            n += 1
+        #
+        self.collection_running = -1
+        #debug_print("collection_running = -1")
+
+    def _collect_sweep_large_objects(self):
+        block = self.collect_pages[0]
+        cam = self.current_aging_marker
+        linked_list = self.NULL
+        first_block_in_linked_list = self.NULL
+        while block != self.NULL:
+            nextblock = list_next(block)
+            blockadr = llmemory.cast_ptr_to_adr(block)
+            blockadr = llarena.getfakearenaaddress(blockadr)
+            hdr = llmemory.cast_adr_to_ptr(blockadr + 8, self.HDRPTR)
+            mark = hdr.tid & 0xFF
+            if mark == cam:
+                # the object is still not marked.  Free it.
+                llarena.arena_free(blockadr)
+                #
+            else:
+                # the object was marked: relink it
+                ll_assert(mark == MARK_BYTE_OLD,
+                          "bad mark in large object")
+                set_next(block, linked_list)
+                linked_list = block
+                if first_block_in_linked_list == self.NULL:
+                    first_block_in_linked_list = block
+            block = nextblock
+        #
+        self.collect_heads[0] = linked_list
+        self.collect_tails[0] = first_block_in_linked_list
+
+    def _collect_sweep_pages(self, n):
+        # sweep all pages from the linked list starting at 'page',
+        # containing objects of fixed size 'object_size'.
+        size_gc_header = self.gcheaderbuilder.size_gc_header
+        page = self.collect_pages[n]
+        object_size = n << WORD_POWER_2
+        linked_list = self.NULL
+        first_loc_in_linked_list = self.NULL
+        cam = self.current_aging_marker
+        while page != self.NULL:
+            i = self.page_size - object_size
+            limit = raw_malloc_usage(self.HDRSIZE)
+            pageadr = llmemory.cast_ptr_to_adr(page)
+            pageadr = llarena.getfakearenaaddress(pageadr)
+            while i >= limit:
+                adr = pageadr + i
+                hdr = llmemory.cast_adr_to_ptr(adr, self.HDRPTR)
+                mark = hdr.tid & 0xFF
+                #
+                if mark == cam:
+                    # the location contains really an object (and is not just
+                    # part of a linked list of free locations), and moreover
+                    # the object is still not marked.  Free it by inserting
+                    # it into the linked list.
+                    #debug_print("sweeps", adr + size_gc_header)
+                    llarena.arena_reset(adr, object_size, 0)
+                    llarena.arena_reserve(adr, self.HDRSIZE)
+                    hdr = llmemory.cast_adr_to_ptr(adr, self.HDRPTR)
+                    set_next(hdr, linked_list)
+                    linked_list = hdr
+                    if first_loc_in_linked_list == self.NULL:
+                        first_loc_in_linked_list = hdr
+                    # XXX detect when the whole page is freed again
+                    #
+                    # Clear the data, in prevision for the following
+                    # malloc_fixedsize_clear().
+                    size_of_int = raw_malloc_usage(
+                        llmemory.sizeof(lltype.Signed))
+                    llarena.arena_reset(adr + size_of_int,
+                                        object_size - size_of_int, 2)
+                #
+                i -= object_size
+            #
+            page = list_next(page)
+        #
+        self.collect_heads[n] = linked_list
+        self.collect_tails[n] = first_loc_in_linked_list
+
+
+    # ----------
+    # Major collections
+
+    def major_collection(self):
+        pass #XXX
+
+
+    # ----------
+    # Weakrefs
+
+    def weakref_deref(self, wrobj):
+        # Weakrefs need some care.  This code acts as a read barrier.
+        # The only way I found is to acquire the mutex_lock to prevent
+        # the collection thread from going from collection_running==1
+        # to collection_running==2, or from collection_running==2 to
+        # collection_running==3.
+        #
+        self.acquire(self.mutex_lock)
+        #
+        targetobj = gctypelayout.ll_weakref_deref(wrobj)
+        if targetobj != llmemory.NULL:
+            #
+            if self.collection_running == 1:
+                # If we are in the phase collection_running==1, we don't
+                # know if the object will be scanned a bit later or
+                # not; so we have to assume that it survives, and
+                # force it to be scanned.
+                self.get_mark(targetobj)
+                self.extra_objects_to_mark.append(targetobj)
+                #
+            elif self.collection_running == 2:
+                # In the phase collection_running==2, if the object is
+                # not marked it's too late; we have to detect that case
+                # and return NULL instead here, as if the corresponding
+                # collector phase was already finished (deal_with_weakrefs).
+                # Otherwise we would be returning an object that is about to
+                # be swept away.
+                if not self.is_marked_or_static(targetobj, self.current_mark):
+                    targetobj = llmemory.NULL
+                #
+            else:
+                # In other phases we are fine.
+                pass
+        #
+        self.release(self.mutex_lock)
+        #
+        return targetobj
+
+    def deal_with_weakrefs(self):
+        self.collection_running = 3; return
+        # ^XXX^
+        size_gc_header = self.gcheaderbuilder.size_gc_header
+        current_mark = self.current_mark
+        weakref_page = self.collect_weakref_pages
+        self.collect_weakref_pages = self.NULL
+        self.collect_weakref_tails = self.NULL
+        while weakref_page != self.NULL:
+            next_page = list_next(weakref_page)
+            #
+            # If the weakref points to a dead object, make it point to NULL.
+            x = llmemory.cast_ptr_to_adr(weakref_page)
+            x = llarena.getfakearenaaddress(x) + 8
+            hdr = llmemory.cast_adr_to_ptr(x, self.HDRPTR)
+            type_id = llop.extract_high_ushort(llgroup.HALFWORD, hdr.tid)
+            offset = self.weakpointer_offset(type_id)
+            ll_assert(offset >= 0, "bad weakref")
+            obj = x + size_gc_header
+            pointing_to = (obj + offset).address[0]
+            ll_assert(pointing_to != llmemory.NULL, "null weakref?")
+            if not self.is_marked_or_static(pointing_to, current_mark):
+                # 'pointing_to' dies: relink to self.collect_pages[0]
+                (obj + offset).address[0] = llmemory.NULL
+                set_next(weakref_page, self.collect_pages[0])
+                self.collect_pages[0] = weakref_page
+            else:
+                # the weakref stays alive
+                set_next(weakref_page, self.collect_weakref_pages)
+                self.collect_weakref_pages = weakref_page
+                if self.collect_weakref_tails == self.NULL:
+                    self.collect_weakref_tails = weakref_page
+            #
+            weakref_page = next_page
+        #
+        self.acquire(self.mutex_lock)
+        self.collection_running = 3
+        #debug_print("collection_running = 3")
+        self.release(self.mutex_lock)
+
+
+    # ----------
+    # Finalizers
+
+    def deal_with_objects_with_finalizers(self):
+        self.collection_running = 4; return
+        # ^XXX^
+        
+        # XXX needs to be done correctly; for now we'll call finalizers
+        # in random order
+        size_gc_header = self.gcheaderbuilder.size_gc_header
+        marked = self.current_mark
+        finalizer_page = self.collect_finalizer_pages
+        self.collect_run_finalizers_head = self.NULL
+        self.collect_run_finalizers_tail = self.NULL
+        self.collect_finalizer_pages = self.NULL
+        self.collect_finalizer_tails = self.NULL
+        while finalizer_page != self.NULL:
+            next_page = list_next(finalizer_page)
+            #
+            x = llmemory.cast_ptr_to_adr(finalizer_page)
+            x = llarena.getfakearenaaddress(x) + 8
+            hdr = llmemory.cast_adr_to_ptr(x, self.HDRPTR)
+            if (hdr.tid & 0xFF) != marked:
+                # non-marked: add to collect_run_finalizers,
+                # and mark the object and its dependencies
+                set_next(finalizer_page, self.collect_run_finalizers_head)
+                self.collect_run_finalizers_head = finalizer_page
+                if self.collect_run_finalizers_tail == self.NULL:
+                    self.collect_run_finalizers_tail = finalizer_page
+                obj = x + size_gc_header
+                self.get_mark(obj)
+                self.gray_objects.append(obj)
+            else:
+                # marked: relink into the collect_finalizer_pages list
+                set_next(finalizer_page, self.collect_finalizer_pages)
+                self.collect_finalizer_pages = finalizer_page
+                if self.collect_finalizer_tails == self.NULL:
+                    self.collect_finalizer_tails = finalizer_page
+            #
+            finalizer_page = next_page
+        #
+        self._collect_mark()
+        #
+        ll_assert(not self.extra_objects_to_mark.non_empty(),
+                  "should not see objects only reachable from finalizers "
+                  "before we run them")
+        #
+        self.collection_running = 4
+        #debug_print("collection_running = 4")
+
+
+# ____________________________________________________________
+#
+# Support for linked lists (used here because AddressStack is not thread-safe)
+
+def list_next(hdr):
+    return llmemory.cast_adr_to_ptr(llmemory.cast_int_to_adr(hdr.tid),
+                                    ConcurrentGenGC.HDRPTR)
+
+def set_next(hdr, nexthdr):
+    hdr.tid = llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(nexthdr),
+                                       "symbolic")
+
+
+# ____________________________________________________________
+#
+# Hack to write the 'mark' or the 'flags' bytes of an object header
+# without overwriting the whole word.  Essential in the rare case where
+# the other thread might be concurrently writing the other byte.
+
+concurrent_setter_lock = ll_thread.allocate_lock()
+
+def emulate_set_mark(p, v):
+    "NOT_RPYTHON"
+    assert v in (MARK_BYTE_1, MARK_BYTE_2, MARK_BYTE_OLD, MARK_BYTE_STATIC)
+    concurrent_setter_lock.acquire(True)
+    p.tid = (p.tid &~ 0xFF) | v
+    concurrent_setter_lock.release()
+
+def emulate_set_flags(p, v):
+    "NOT_RPYTHON"
+    assert (v & ~0xFF00) == 0
+    concurrent_setter_lock.acquire(True)
+    p.tid = (p.tid &~ 0xFF00) | v
+    concurrent_setter_lock.release()
+
+if sys.byteorder == 'little':
+    eci = ExternalCompilationInfo(
+        post_include_bits = ["""
+#define pypy_concurrentms_set_mark(p, v)   ((char*)p)[0] = v
+#define pypy_concurrentms_set_flags(p, v)  ((char*)p)[1] = v
+        """])
+elif sys.byteorder == 'big':
+    eci = ExternalCompilationInfo(
+        post_include_bits = [r"""
+#define pypy_concurrentms_set_mark(p, v)   ((char*)p)[sizeof(long)-1] = v
+#define pypy_concurrentms_set_flags(p, v)  ((char*)p)[sizeof(long)-2] = v
+        """])
+else:
+    raise NotImplementedError(sys.byteorder)
+
+_set_mark = rffi.llexternal("pypy_concurrentms_set_mark",
+                           [ConcurrentGenGC.HDRPTR, lltype.Signed],
+                           lltype.Void, compilation_info=eci, _nowrapper=True,
+                           _callable=emulate_set_mark)
+_set_flags = rffi.llexternal("pypy_concurrentms_set_flags",
+                           [ConcurrentGenGC.HDRPTR, lltype.Signed],
+                           lltype.Void, compilation_info=eci, _nowrapper=True,
+                           _callable=emulate_set_flags)
+
+# ____________________________________________________________
+#
+# A lock to synchronize access to AddressStack's free pages
+
+class SyncLock:
+    _alloc_flavor_ = "raw"
+    _lock = lltype.nullptr(ll_thread.TLOCKP.TO)
+    def setup(self):
+        self._lock = ll_thread.allocate_ll_lock()
+    def acquire(self):
+        if self._lock:
+            ll_thread.c_thread_acquirelock(self._lock, 1)
+    def release(self):
+        if self._lock:
+            ll_thread.c_thread_releaselock(self._lock)
diff --git a/pypy/rpython/memory/gc/concurrentgen.py b/pypy/rpython/memory/gc/concurrentgen.txt
copy from pypy/rpython/memory/gc/concurrentgen.py
copy to pypy/rpython/memory/gc/concurrentgen.txt
--- a/pypy/rpython/memory/gc/concurrentgen.py
+++ b/pypy/rpython/memory/gc/concurrentgen.txt
@@ -1,4 +1,3 @@
-"""
 ************************************************************
   Minor collection cycles of the "concurrentgen" collector
 ************************************************************
@@ -11,8 +10,8 @@
     '#'        : old objs
     'S'        : static prebuilt objs with no heap pointer
 
-'cym' is the current young marker
-'cam' is the current aging marker
+cym = current_young_marker
+cam = current_aging_marker
 
 The write barrier activates when writing into an object whose
 mark byte is different from 'cym'.
@@ -109,4 +108,3 @@
         (should not see any 'cam' object any more here)
 
 ------------------------------------------------------------
-"""
diff --git a/pypy/rpython/memory/gc/test/test_direct.py b/pypy/rpython/memory/gc/test/test_direct.py
--- a/pypy/rpython/memory/gc/test/test_direct.py
+++ b/pypy/rpython/memory/gc/test/test_direct.py
@@ -609,3 +609,7 @@
 class TestMostlyConcurrentMarkSweepGC(DirectGCTest):
     from pypy.rpython.memory.gc.concurrentms \
             import MostlyConcurrentMarkSweepGC as GCClass
+
+class TestConcurrentGenGC(DirectGCTest):
+    from pypy.rpython.memory.gc.concurrentgen \
+            import ConcurrentGenGC as GCClass

From noreply at buildbot.pypy.org  Sat Oct 22 16:40:13 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sat, 22 Oct 2011 16:40:13 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-dicts: Started on them,
 really just as simple as unrolling lookup and setitem for virtual
 dicts with constant keys. Next step is to teach virtualize.py about {get,
 set}interiorfield_gc.
Message-ID: <20111022144013.AB905820D7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: virtual-dicts
Changeset: r48342:15690256a200
Date: 2011-10-22 07:39 -0700
http://bitbucket.org/pypy/pypy/changeset/15690256a200/

Log:	Started on them, really just as simple as unrolling lookup and
	setitem for virtual dicts with constant keys. Next step is to teach
	virtualize.py about {get,set}interiorfield_gc.

diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -3435,7 +3435,20 @@
             return sa
         res = self.meta_interp(f, [16])
         assert res == f(16)
-        
+
+    def test_virtual_dict_constant_keys(self):
+        myjitdriver = JitDriver(greens = [], reds = ["n"])
+        def g(d):
+            return d["key"] - 1
+
+        def f(n):
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n)
+                n = g({"key": n})
+            return n
+        res = self.meta_interp(f, [10])
+        assert res == 0
+        self.check_loops({"int_sub": 1, "int_gt": 1, "guard_true": 1, "jump": 1})
 
 
 class TestLLtype(BaseLLtypeTests, LLJitMixin):
diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py
--- a/pypy/rpython/lltypesystem/rdict.py
+++ b/pypy/rpython/lltypesystem/rdict.py
@@ -445,9 +445,9 @@
     i = ll_dict_lookup(d, key, hash)
     return _ll_dict_setitem_lookup_done(d, key, value, hash, i)
 
-# Leaving as dont_look_inside ATM, it has a few branches which could lead to
-# many bridges if we don't consider their possible frequency.
- at jit.dont_look_inside
+# It may be safe to look inside always, it has a few branches though, and their
+# frequencies needs to be investigated.
+ at jit.look_inside_iff(lambda d, key, value, hash, i: jit.isvirtual(d) and jit.isconstant(key))
 def _ll_dict_setitem_lookup_done(d, key, value, hash, i):
     valid = (i & HIGHEST_BIT) == 0
     i = i & MASK
@@ -533,7 +533,7 @@
 # ------- a port of CPython's dictobject.c's lookdict implementation -------
 PERTURB_SHIFT = 5
 
- at jit.dont_look_inside
+ at jit.look_inside_iff(lambda d, key, hash: jit.isvirtual(d) and jit.isconstant(key))
 def ll_dict_lookup(d, key, hash):
     entries = d.entries
     ENTRIES = lltype.typeOf(entries).TO

From noreply at buildbot.pypy.org  Sat Oct 22 21:21:57 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sat, 22 Oct 2011 21:21:57 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-dicts: some progress,
 needs optimizeopt tests. also it blows up running the virtual dict
 test
Message-ID: <20111022192157.93C4C820D7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: virtual-dicts
Changeset: r48343:913dbc088237
Date: 2011-10-22 12:21 -0700
http://bitbucket.org/pypy/pypy/changeset/913dbc088237/

Log:	some progress, needs optimizeopt tests. also it blows up running
	the virtual dict test

diff --git a/pypy/jit/backend/llgraph/runner.py b/pypy/jit/backend/llgraph/runner.py
--- a/pypy/jit/backend/llgraph/runner.py
+++ b/pypy/jit/backend/llgraph/runner.py
@@ -8,7 +8,7 @@
 from pypy.rpython.ootypesystem import ootype
 from pypy.rpython.llinterp import LLInterpreter
 from pypy.jit.metainterp import history
-from pypy.jit.metainterp.history import REF, INT, FLOAT
+from pypy.jit.metainterp.history import REF, INT, FLOAT, STRUCT
 from pypy.jit.metainterp.warmstate import unwrap
 from pypy.jit.metainterp.resoperation import rop
 from pypy.jit.backend import model
@@ -60,6 +60,9 @@
     def is_array_of_floats(self):
         return self.typeinfo == FLOAT
 
+    def is_array_of_structs(self):
+        return self.typeinfo == STRUCT
+
     def as_vtable_size_descr(self):
         return self
 
@@ -365,6 +368,8 @@
         size = symbolic.get_size(A)
         if isinstance(A.OF, lltype.Ptr) or isinstance(A.OF, lltype.Primitive):
             token = history.getkind(A.OF)[0]
+        elif isinstance(A.OF, lltype.Struct):
+            token = 's'
         else:
             token = '?'
         return self.getdescr(size, token)
diff --git a/pypy/jit/metainterp/history.py b/pypy/jit/metainterp/history.py
--- a/pypy/jit/metainterp/history.py
+++ b/pypy/jit/metainterp/history.py
@@ -16,6 +16,7 @@
 INT   = 'i'
 REF   = 'r'
 FLOAT = 'f'
+STRUCT = 's'
 HOLE  = '_'
 VOID  = 'v'
 
diff --git a/pypy/jit/metainterp/optimizeopt/virtualize.py b/pypy/jit/metainterp/optimizeopt/virtualize.py
--- a/pypy/jit/metainterp/optimizeopt/virtualize.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualize.py
@@ -59,7 +59,7 @@
 
     def import_from(self, other, optimizer):
         raise NotImplementedError("should not be called at this level")
-    
+
 def get_fielddescrlist_cache(cpu):
     if not hasattr(cpu, '_optimizeopt_fielddescrlist_cache'):
         result = descrlist_dict()
@@ -113,7 +113,7 @@
         #
         if not we_are_translated():
             op.name = 'FORCE ' + self.source_op.name
-            
+
         if self._is_immutable_and_filled_with_constants(optforce):
             box = optforce.optimizer.constant_fold(op)
             self.make_constant(box)
@@ -239,12 +239,12 @@
         for index in range(len(self._items)):
             self._items[index] = self._items[index].force_at_end_of_preamble(already_forced, optforce)
         return self
-    
+
     def _really_force(self, optforce):
         assert self.source_op is not None
         if not we_are_translated():
             self.source_op.name = 'FORCE ' + self.source_op.name
-        optforce.emit_operation(self.source_op)        
+        optforce.emit_operation(self.source_op)
         self.box = box = self.source_op.result
         for index in range(len(self._items)):
             subvalue = self._items[index]
@@ -271,20 +271,61 @@
     def _make_virtual(self, modifier):
         return modifier.make_varray(self.arraydescr)
 
+class VArrayStructValue(AbstractVirtualValue):
+    def __init__(self, arraydescr, size, keybox, source_op=None):
+        AbstractVirtualValue.__init__(self, keybox, source_op)
+        self.arraydescr = arraydescr
+        self._items = [{} for _ in xrange(size)]
+
+    def getlength(self):
+        return len(self._items)
+
+    def getinteriorfield(self, index, descr, default):
+        return self._items[index].get(descr, default)
+
+    def setinteriorfield(self, index, descr, itemvalue):
+        assert isinstance(itemvalue, optimizer.OptValue)
+        self._items[index][descr] = itemvalue
+
+    def _really_force(self, optforce):
+        assert self.source_op is not None
+        if not we_are_translated():
+            self.source_op.name = 'FORCE ' + self.source_op.name
+        optforce.emit_operation(self.source_op)
+        self.box = box = self.source_op.result
+        for index in range(len(self._items)):
+            for descr, value in self._items[index].iteritems():
+                subbox = value.force_box(optforce)
+                op = ResOperation(rop.SETINTERIORFIELD_GC, [box, ConstInt(index), subbox], None, descr=descr)
+                optforce.emit_operation(op)
+
+    def force_at_end_of_preamble(self, already_forced, optforce):
+        if self in already_forced:
+            return self
+        already_forced[self] = self
+        for index in range(len(self._items)):
+            for descr in self._items[index].keys():
+                self._items[index][descr] = self._items[index][descr].force_at_end_of_preamble(already_forced, optforce)
+        return self
+
+
 class OptVirtualize(optimizer.Optimization):
     "Virtualize objects until they escape."
 
     def new(self):
         return OptVirtualize()
-        
+
     def make_virtual(self, known_class, box, source_op=None):
         vvalue = VirtualValue(self.optimizer.cpu, known_class, box, source_op)
         self.make_equal_to(box, vvalue)
         return vvalue
 
     def make_varray(self, arraydescr, size, box, source_op=None):
-        constvalue = self.new_const_item(arraydescr)
-        vvalue = VArrayValue(arraydescr, constvalue, size, box, source_op)
+        if arraydescr.is_array_of_structs():
+            vvalue = VArrayStructValue(arraydescr, size, box, source_op)
+        else:
+            constvalue = self.new_const_item(arraydescr)
+            vvalue = VArrayValue(arraydescr, constvalue, size, box, source_op)
         self.make_equal_to(box, vvalue)
         return vvalue
 
@@ -431,6 +472,32 @@
         value.ensure_nonnull()
         self.emit_operation(op)
 
+    def optimize_GETINTERIORFIELD_GC(self, op):
+        value = self.getvalue(op.getarg(0))
+        if value.is_virtual():
+            indexbox = self.get_constant_box(op.getarg(1))
+            if indexbox is not None:
+                descr = op.getdescr()
+                fieldvalue = value.getinteriorfield(
+                    indexbox.getint(), descr, self.new_const(descr)
+                )
+                self.make_equal_to(op.result, fieldvalue)
+                return
+        value.ensure_nonnull()
+        self.emit_operation(op)
+
+    def optimize_SETINTERIORFIELD_GC(self, op):
+        value = self.getvalue(op.getarg(0))
+        if value.is_virtual():
+            indexbox = self.get_constant_box(op.getarg(1))
+            if indexbox is not None:
+                value.setinteriorfield(
+                    indexbox.getint(), op.getdescr(), self.getvalue(op.getarg(2))
+                )
+                return
+        value.ensure_nonnull()
+        self.emit_operation(op)
+
 
 dispatch_opt = make_dispatcher_method(OptVirtualize, 'optimize_',
         default=OptVirtualize.emit_operation)

From noreply at buildbot.pypy.org  Sat Oct 22 21:40:32 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 22 Oct 2011 21:40:32 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Oups,
 forgot to add this file.  It's needed in debug_lltrace.
Message-ID: <20111022194032.AD134820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r48344:eb5d8279f60e
Date: 2011-10-22 21:36 +0200
http://bitbucket.org/pypy/pypy/changeset/eb5d8279f60e/

Log:	Oups, forgot to add this file. It's needed in debug_lltrace.

diff --git a/pypy/translator/c/src/atomic_ops.h b/pypy/translator/c/src/atomic_ops.h
new file mode 100644
--- /dev/null
+++ b/pypy/translator/c/src/atomic_ops.h
@@ -0,0 +1,45 @@
+
+
+/* "compiler fence" for preventing reordering of loads/stores to
+   non-volatiles */
+#define CFENCE          asm volatile ("":::"memory")
+
+
+#ifdef __llvm__
+#  define HAS_SYNC_BOOL_COMPARE_AND_SWAP
+#endif
+
+#ifdef __GNUC__
+#  if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 1)
+#    define HAS_SYNC_BOOL_COMPARE_AND_SWAP
+#  endif
+#endif
+
+
+#ifdef HAS_SYNC_BOOL_COMPARE_AND_SWAP
+#  define bool_cas __sync_bool_compare_and_swap
+#else
+/* x86 (32 bits and 64 bits) */
+static inline _Bool
+bool_cas(volatile unsigned long* ptr, unsigned long old, unsigned long _new)
+{
+    unsigned long prev;
+    asm volatile("lock;"
+#if defined(__amd64__)
+                 "cmpxchgq %1, %2;"
+#else
+                 "cmpxchgl %1, %2;"
+#endif
+                 : "=a"(prev)
+                 : "q"(_new), "m"(*ptr), "a"(old)
+                 : "memory");
+    return prev == old;
+}
+/* end */
+#endif
+
+
+static inline void spinloop(void)
+{
+  asm volatile ("pause");
+}

From noreply at buildbot.pypy.org  Sat Oct 22 21:40:33 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 22 Oct 2011 21:40:33 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: In the middle of slow
	progress.
Message-ID: <20111022194033.D93A7820D7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r48345:4e2e54d7f69e
Date: 2011-10-22 21:37 +0200
http://bitbucket.org/pypy/pypy/changeset/4e2e54d7f69e/

Log:	In the middle of slow progress.

diff --git a/pypy/rpython/memory/gc/concurrentgen.py b/pypy/rpython/memory/gc/concurrentgen.py
--- a/pypy/rpython/memory/gc/concurrentgen.py
+++ b/pypy/rpython/memory/gc/concurrentgen.py
@@ -14,9 +14,11 @@
 #
 # A "3/4th concurrent" generational mark&sweep GC.
 #
-# This uses a separate thread to run the minor collections in parallel,
-# as well as half of the major collections (the sweep phase).  The mark
-# phase is not parallelized.  See concurrentgen.txt for some details.
+# This uses a separate thread to run the minor collections in parallel.
+# See concurrentgen.txt for some details.
+#
+# Major collections are serialized for the mark phase, but the sweep
+# phase can be parallelized again.  XXX not done so far
 #
 # Based on observations that the timing of collections with "minimark"
 # (on translate.py) is: about 15% of the time in minor collections
@@ -85,22 +87,30 @@
         #
         # The following are arrays of 36 linked lists: the linked lists
         # at indices 1 to 35 correspond to pages that store objects of
-        # size  1 * WORD  to  35 * WORD,  and the linked list at index 0
-        # is a list of all larger objects.
+        # size  1 * WORD  to  35 * WORD.  The linked list at index 0
+        # contains the pages of objects that are larger than this limit
+        # (variable-sized pages, with exactly one object per page).
+        #
         def list_of_addresses_per_small_size():
             return lltype.malloc(rffi.CArray(self.HDRPTR),
                                  self.pagelists_length, flavor='raw',
                                  immortal=True)
-        # 1-35: a linked list of all pages; 0: a linked list of all larger objs
-        self.nonfree_pages = list_of_addresses_per_small_size()
-        # a snapshot of 'nonfree_pages' done when the collection starts
-        self.collect_pages = list_of_addresses_per_small_size()
-        # 1-35: free list of non-allocated locations; 0: unused
-        self.free_lists    = list_of_addresses_per_small_size()
-        # 1-35: head and tail of the free list built by the collector thread
-        # 0: head and tail of the linked list of surviving large objects
-        self.collect_heads = list_of_addresses_per_small_size()
-        self.collect_tails = list_of_addresses_per_small_size()
+        #
+        # pages that contain at least one new young object
+        self.new_young_objects_pages = list_of_addresses_per_small_size()
+        #
+        # when the collection starts and we make all young objects aging,
+        # we move the linked lists above into 'aging_objects_pages'
+        self.aging_objects_pages = list_of_addresses_per_small_size()
+        #
+        # free list of non-allocated locations within pages
+        # (at index 0: always empty)
+        self.location_free_lists = list_of_addresses_per_small_size()
+        #
+        # head and tail of the free list of locations built by the
+        # collector thread
+        self.collect_loc_heads = list_of_addresses_per_small_size()
+        self.collect_loc_tails = list_of_addresses_per_small_size()
         #
         def collector_start():
             if we_are_translated():
@@ -672,17 +682,20 @@
         finish.  Guarantees that young objects not reachable when
         collect() is called will be freed by the time collect() returns.
 
-        gen>=3: Do a (synchronous) major collection.
+        gen>=3: Major collection.
+
+        XXX later:
+           gen=3: Do a major collection, but don't wait for sweeping to finish.
+           The most useful default.
+           gen>=4: Do a full synchronous major collection.
         """
         if gen >= 1 or self.collection_running <= 0:
-            self.trigger_next_collection()
+            self.trigger_next_collection(gen >= 3)
             if gen >= 2:
                 self.wait_for_the_end_of_collection()
-                if gen >= 3:
-                    self.major_collection()
         self.execute_finalizers_ll()
 
-    def trigger_next_collection(self):
+    def trigger_next_collection(self, force_major_collection=False):
         """In the mutator thread: triggers the next minor collection."""
         #
         # In case the previous collection is not over yet, wait for it
@@ -694,10 +707,7 @@
         self.root_walker.walk_roots(
             ConcurrentGenGC._add_stack_root,  # stack roots
             ConcurrentGenGC._add_stack_root,  # in prebuilt non-gc
-            None)                         # static in prebuilt gc
-        #
-        # Add the prebuilt root objects that have been written to
-        self.flagged_objects.foreach(self._add_prebuilt_root, None)
+            None)                             # static in prebuilt gc
         #
         # Add the objects still waiting in 'objects_with_finalizers_to_run'
         p = self.objects_with_finalizers_to_run
@@ -710,13 +720,20 @@
             self.gray_objects.append(obj)
             p = list_next(p)
         #
+        # Add all old objects that have been written to since the last
+        # time trigger_next_collection was called
+        self.flagged_objects.foreach(self._add_prebuilt_root, None)
+        #
+        # Clear this list
+        self.flagged_objects.clear()
+        #
         # Exchange the meanings of 'cym' and 'cam'
         other = self.current_young_marker
         self.current_young_marker = self.current_aging_marker
         self.current_aging_marker = other
         #
-        # Copy a few 'mutator' fields to 'collector' fields:
-        # 'collect_pages' make linked lists of all nonfree pages at the
+        # Copy a few 'mutator' fields to 'collector' fields:  the
+        # 'collect_pages[n]' make linked lists of all nonfree pages at the
         # start of the collection (unlike the 'nonfree_pages' lists, which
         # the mutator will continue to grow).
         n = 0
@@ -1025,13 +1042,6 @@
 
 
     # ----------
-    # Major collections
-
-    def major_collection(self):
-        pass #XXX
-
-
-    # ----------
     # Weakrefs
 
     def weakref_deref(self, wrobj):

From noreply at buildbot.pypy.org  Sat Oct 22 23:03:16 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sat, 22 Oct 2011 23:03:16 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Python3 renamed thread to _thread
Message-ID: <20111022210316.7C14D820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48346:26fa76635221
Date: 2011-10-22 01:33 +0200
http://bitbucket.org/pypy/pypy/changeset/26fa76635221/

Log:	Python3 renamed thread to _thread

diff --git a/pypy/module/thread/__init__.py b/pypy/module/thread/__init__.py
--- a/pypy/module/thread/__init__.py
+++ b/pypy/module/thread/__init__.py
@@ -3,6 +3,8 @@
 from pypy.interpreter.mixedmodule import MixedModule
 
 class Module(MixedModule):
+    applevel_name = '_thread'
+    
     appleveldefs = {
     }
 
diff --git a/pypy/module/thread/test/test_fork.py b/pypy/module/thread/test/test_fork.py
--- a/pypy/module/thread/test/test_fork.py
+++ b/pypy/module/thread/test/test_fork.py
@@ -7,7 +7,7 @@
         # XXX This test depends on a multicore machine, as busy_thread must
         # aquire the GIL the instant that the main thread releases it.
         # It will incorrectly pass if the GIL is not grabbed in time.
-        import thread
+        import _thread
         import os
         import time
 
@@ -22,7 +22,7 @@
             done.append(None)
 
         try:
-            thread.start_new(busy_thread, ())
+            _thread.start_new(busy_thread, ())
 
             pid = os.fork()
 
@@ -39,18 +39,18 @@
 
     def test_forked_can_thread(self):
         "Checks that a forked interpreter can start a thread"
-        import os, thread, time
+        import os, _thread, time
 
         if not hasattr(os, 'fork'):
             skip("No fork on this platform")
 
         # pre-allocate some locks
-        thread.start_new_thread(lambda: None, ())
+        _thread.start_new_thread(lambda: None, ())
 
         pid = os.fork()
         if pid == 0:
-            print 'in child'
-            thread.start_new_thread(lambda: None, ())
+            print('in child')
+            _thread.start_new_thread(lambda: None, ())
             os._exit(0)
         else:
             self.timeout_killer(pid, 5)
diff --git a/pypy/module/thread/test/test_import_lock.py b/pypy/module/thread/test/test_import_lock.py
--- a/pypy/module/thread/test/test_import_lock.py
+++ b/pypy/module/thread/test/test_import_lock.py
@@ -12,26 +12,26 @@
     def test_import_lock(self):
         # XXX XXX XXX this test fails if run together with all other tests
         # of this directory, but not when run alone
-        import thread, imp
+        import _thread, imp
         assert not imp.lock_held()
         done = []
         def f(i):
-            print '[ENTER %d]' % i
+            print('[ENTER %d]' % i)
             from imghdr import testall
-            print '[LEAVE %d]' % i
+            print('[LEAVE %d]' % i)
             done.append(1)
         for i in range(5):
-            print '[RUN %d]' % i
-            thread.start_new_thread(f, (i,))
+            print('[RUN %d]' % i)
+            _thread.start_new_thread(f, (i,))
         self.waitfor(lambda: len(done) == 5)
         assert len(done) == 5
 
     def test_with_many_dependencies(self):
-        import thread
+        import _thread
         import re      # -> causes nested imports
 
     def test_manual_locking(self):
-        import thread, os, imp, time, sys
+        import _thread, os, imp, time, sys
         f = open(os.path.join(self.tmpdir, 'foobaz2.py'), 'w')
         f.close()   # empty
         done = []
@@ -44,7 +44,7 @@
         assert not imp.lock_held()
         imp.acquire_lock()
         assert imp.lock_held()
-        thread.start_new_thread(f, ())
+        _thread.start_new_thread(f, ())
         time.sleep(0.9)
         assert not done
         assert imp.lock_held()
diff --git a/pypy/module/thread/test/test_local.py b/pypy/module/thread/test/test_local.py
--- a/pypy/module/thread/test/test_local.py
+++ b/pypy/module/thread/test/test_local.py
@@ -4,8 +4,8 @@
 class AppTestLocal(GenericTestThread):
 
     def test_local_1(self):
-        import thread
-        from thread import _local as tlsobject
+        import _thread
+        from _thread import _local as tlsobject
         freed = []
         class X:
             def __del__(self):
@@ -31,7 +31,7 @@
             finally:
                 ok.append(success)
         for i in range(20):
-            thread.start_new_thread(f, (i,))
+            _thread.start_new_thread(f, (i,))
         self.waitfor(lambda: len(ok) == 20, delay=3)
         assert ok == 20*[True] # see stdout/stderr for failures in the threads
 
@@ -47,14 +47,14 @@
 
 
     def test_local_init(self):
-        import thread
+        import _thread
         tags = [1, 2, 3, 4, 5, 54321]
         seen = []
 
-        raises(TypeError, thread._local, a=1)
-        raises(TypeError, thread._local, 1)
+        raises(TypeError, _thread._local, a=1)
+        raises(TypeError, _thread._local, 1)
 
-        class X(thread._local):
+        class X(_thread._local):
             def __init__(self, n):
                 assert n == 42
                 self.tag = tags.pop()
@@ -64,7 +64,7 @@
         def f():
             seen.append(x.tag)
         for i in range(5):
-            thread.start_new_thread(f, ())
+            _thread.start_new_thread(f, ())
         self.waitfor(lambda: len(seen) == 5, delay=2)
         seen1 = seen[:]
         seen1.sort()
@@ -72,8 +72,8 @@
         assert tags == []
 
     def test_local_setdict(self):
-        import thread
-        x = thread._local()
+        import _thread
+        x = _thread._local()
         # XXX: On Cpython these are AttributeErrors
         raises(TypeError, "x.__dict__ = 42")
         raises(TypeError, "x.__dict__ = {}")
@@ -84,6 +84,6 @@
             assert x.__dict__["spam"] == n
             done.append(1)
         for i in range(5):
-            thread.start_new_thread(f, (i,))
+            _thread.start_new_thread(f, (i,))
         self.waitfor(lambda: len(done) == 5, delay=2)
         assert len(done) == 5
diff --git a/pypy/module/thread/test/test_lock.py b/pypy/module/thread/test/test_lock.py
--- a/pypy/module/thread/test/test_lock.py
+++ b/pypy/module/thread/test/test_lock.py
@@ -6,17 +6,17 @@
 class AppTestLock(GenericTestThread):
 
     def test_lock(self):
-        import thread
-        lock = thread.allocate_lock()
-        assert type(lock) is thread.LockType
+        import _thread
+        lock = _thread.allocate_lock()
+        assert type(lock) is _thread.LockType
         assert lock.locked() is False
-        raises(thread.error, lock.release)
+        raises(_thread.error, lock.release)
         assert lock.locked() is False
         lock.acquire()
         assert lock.locked() is True
         lock.release()
         assert lock.locked() is False
-        raises(thread.error, lock.release)
+        raises(_thread.error, lock.release)
         assert lock.locked() is False
         feedback = []
         lock.acquire()
@@ -25,14 +25,14 @@
             feedback.append(42)
             lock.release()
         assert lock.locked() is True
-        thread.start_new_thread(f, ())
+        _thread.start_new_thread(f, ())
         lock.acquire()
         assert lock.locked() is True
         assert feedback == [42]
 
     def test_lock_in_with(self):
-        import thread
-        lock = thread.allocate_lock()
+        import _thread
+        lock = _thread.allocate_lock()
         feedback = []
         lock.acquire()
         def f():
@@ -40,7 +40,7 @@
             feedback.append(42)
             lock.release()
         assert lock.locked() is True
-        thread.start_new_thread(f, ())
+        _thread.start_new_thread(f, ())
         with lock:
             assert lock.locked() is True
             assert feedback == [42]
diff --git a/pypy/module/thread/test/test_thread.py b/pypy/module/thread/test/test_thread.py
--- a/pypy/module/thread/test/test_thread.py
+++ b/pypy/module/thread/test/test_thread.py
@@ -27,34 +27,34 @@
             cls.w_can_start_many_threads = space.wrap(True)
 
     def test_start_new_thread(self):
-        import thread
+        import _thread
         feedback = []
         please_start = []
         def f(x, y, z):
             self.waitfor(lambda: please_start)
             feedback.append(42)
-        thread.start_new_thread(f, (1, 2), {'z': 3})
+        _thread.start_new_thread(f, (1, 2), {'z': 3})
         assert feedback == []   # still empty
         please_start.append(1)  # trigger
         self.waitfor(lambda: feedback)
         assert feedback == [42]
 
     def test_thread_count(self):
-        import thread, time
+        import _thread, time
         feedback = []
         please_start = []
         def f():
             feedback.append(42)
             self.waitfor(lambda: please_start)
-        assert thread._count() == 0
-        thread.start_new_thread(f, ())
+        assert _thread._count() == 0
+        _thread.start_new_thread(f, ())
         self.waitfor(lambda: feedback)
-        assert thread._count() == 1
+        assert _thread._count() == 1
         please_start.append(1)  # trigger
         # XXX joining a thread seems difficult at applevel.
 
     def test_start_new_thread_args(self):
-        import thread
+        import _thread
         def f():
             pass
         test_args = [
@@ -64,27 +64,27 @@
         ]
         for args in test_args:
             try:
-                thread.start_new_thread(*args)
+                _thread.start_new_thread(*args)
                 assert False
             except TypeError:
                 pass
 
     def test_get_ident(self):
-        import thread
-        ident = thread.get_ident()
+        import _thread
+        ident = _thread.get_ident()
         feedback = []
         def f():
-            feedback.append(thread.get_ident())
-        ident2 = thread.start_new_thread(f, ())
+            feedback.append(_thread.get_ident())
+        ident2 = _thread.start_new_thread(f, ())
         assert ident2 != ident
-        assert ident == thread.get_ident()
+        assert ident == _thread.get_ident()
         self.waitfor(lambda: feedback)
         assert feedback == [ident2]
 
     def test_sys_getframe(self):
         # this checks that each thread gets its own ExecutionContext.
         def main():
-            import thread, sys
+            import _thread, sys
             def dump_frames(feedback):
                 f = sys._getframe()
                 for i in range(3):
@@ -103,7 +103,7 @@
             feedbacks = []
             for i in range(3):
                 feedback = []
-                thread.start_new_thread(dummyfn, (feedback,))
+                _thread.start_new_thread(dummyfn, (feedback,))
                 feedbacks.append(feedback)
             expected = 3*[['dump_frames', 'dummyfn', None]]   # without 'main'
             self.waitfor(lambda: feedbacks == expected)
@@ -111,23 +111,23 @@
         main()
 
     def test_thread_exit(self):
-        import thread, sys, StringIO
+        import _thread, sys, io
         def fn1():
-            thread.exit()
+            _thread.exit()
         def fn2():
             raise SystemExit
         def fn3():
             raise ValueError("hello world")
         prev = sys.stderr
         try:
-            sys.stderr = StringIO.StringIO()
-            thread.start_new_thread(fn1, ())
-            thread.start_new_thread(fn2, ())
+            sys.stderr = io.StringIO()
+            _thread.start_new_thread(fn1, ())
+            _thread.start_new_thread(fn2, ())
             self.busywait(0.2)   # time for the threads to finish
             assert sys.stderr.getvalue() == ''
 
-            sys.stderr = StringIO.StringIO()
-            thread.start_new_thread(fn3, ())
+            sys.stderr = io.StringIO()
+            _thread.start_new_thread(fn3, ())
             self.waitfor(lambda: "ValueError" in sys.stderr.getvalue())
             result = sys.stderr.getvalue()
             assert "ValueError" in result
@@ -137,7 +137,7 @@
             sys.stderr = prev
 
     def test_perthread_excinfo(self):
-        import thread, sys
+        import _thread, sys
         done = []
         def fn1(n):
             success = False
@@ -160,12 +160,12 @@
             finally:
                 done.append(success)
         for i in range(20):
-            thread.start_new_thread(fn1, (i,))
+            _thread.start_new_thread(fn1, (i,))
         self.waitfor(lambda: len(done) == 20)
         assert done == 20*[True]  # see stderr for failures in the threads
 
     def test_no_corruption(self):
-        import thread
+        import _thread
         lst = []
         done_marker = []
         def f(x, done):
@@ -174,18 +174,18 @@
             done.append(True)
         for i in range(0, 120, 40):
             done = []
-            thread.start_new_thread(f, (i, done))
+            _thread.start_new_thread(f, (i, done))
             done_marker.append(done)
         for done in done_marker:
             self.waitfor(lambda: done, delay=3)
             assert done    # see stderr for failures in threads
-        assert sorted(lst) == range(120)
+        assert sorted(lst) == list(range(120))
 
     def test_many_threads(self):
-        import thread, time
+        import _thread, time
         if self.can_start_many_threads:
             skip("this OS supports too many threads to check (> 1000)")
-        lock = thread.allocate_lock()
+        lock = _thread.allocate_lock()
         lock.acquire()
         def f():
             lock.acquire()
@@ -193,43 +193,43 @@
         try:
             try:
                 for i in range(1000):
-                    thread.start_new_thread(f, ())
+                    _thread.start_new_thread(f, ())
             finally:
                 lock.release()
                 # wait a bit to allow most threads to finish now
                 self.busywait(2.0)
-        except (thread.error, MemoryError):
+        except (_thread.error, MemoryError):
             pass
         else:
             raise Exception("could unexpectedly start 1000 threads")
 
     def test_stack_size(self):
-        import thread
-        thread.stack_size(0)
-        res = thread.stack_size(0)
+        import _thread
+        _thread.stack_size(0)
+        res = _thread.stack_size(0)
         assert res == 0
-        res = thread.stack_size(1024*1024)
+        res = _thread.stack_size(1024*1024)
         assert res == 0
-        res = thread.stack_size(2*1024*1024)
+        res = _thread.stack_size(2*1024*1024)
         assert res == 1024*1024
-        res = thread.stack_size(0)
+        res = _thread.stack_size(0)
         assert res == 2*1024*1024
 
     def test_interrupt_main(self):
-        import thread, time
+        import _thread, time
         import signal
 
         def f():
             time.sleep(0.5)
-            thread.interrupt_main()
+            _thread.interrupt_main()
 
         def busy_wait():
             for x in range(1000):
-                print 'tick...', x  # <-force the GIL to be released, as
-                time.sleep(0.01)    #   time.sleep doesn't do non-translated
+                print('tick...', x)  # <-force the GIL to be released, as
+                time.sleep(0.01)     #   time.sleep doesn't do non-translated
 
         # This is normally called by app_main.py
         signal.signal(signal.SIGINT, signal.default_int_handler)
 
-        thread.start_new_thread(f, ())
+        _thread.start_new_thread(f, ())
         raises(KeyboardInterrupt, busy_wait)

From noreply at buildbot.pypy.org  Sat Oct 22 23:03:17 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sat, 22 Oct 2011 23:03:17 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Add _thread.TIMEOUT_MAX. Not used for now
Message-ID: <20111022210317.AD391820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48347:2e215c92d5ba
Date: 2011-10-22 11:49 +0200
http://bitbucket.org/pypy/pypy/changeset/2e215c92d5ba/

Log:	Add _thread.TIMEOUT_MAX. Not used for now

diff --git a/pypy/module/thread/__init__.py b/pypy/module/thread/__init__.py
--- a/pypy/module/thread/__init__.py
+++ b/pypy/module/thread/__init__.py
@@ -21,6 +21,7 @@
         'allocate':               'os_lock.allocate_lock',  # obsolete synonym
         'LockType':               'os_lock.Lock',
         '_local':                 'os_local.Local',
+        'TIMEOUT_MAX':            'space.wrap(float(os_lock.TIMEOUT_MAX) / 1000000.0)',
         'error':                  'space.fromcache(error.Cache).w_error',
     }
 
diff --git a/pypy/module/thread/os_lock.py b/pypy/module/thread/os_lock.py
--- a/pypy/module/thread/os_lock.py
+++ b/pypy/module/thread/os_lock.py
@@ -7,10 +7,14 @@
 from pypy.interpreter.baseobjspace import Wrappable
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.typedef import TypeDef
+from pypy.rlib.rarithmetic import r_longlong
 
 # Force the declaration of the type 'thread.LockType' for RPython
 #import pypy.module.thread.rpython.exttable
 
+LONGLONG_MAX = r_longlong(2 ** (r_longlong.BITS-1) - 1)
+TIMEOUT_MAX = LONGLONG_MAX
+
 
 ##import sys
 ##def debug(msg, n):
@@ -113,4 +117,4 @@
 def allocate_lock(space):
     """Create a new lock object.  (allocate() is an obsolete synonym.)
 See LockType.__doc__ for information about locks."""
-    return space.wrap(Lock(space))
\ No newline at end of file
+    return space.wrap(Lock(space))
diff --git a/pypy/module/thread/test/test_lock.py b/pypy/module/thread/test/test_lock.py
--- a/pypy/module/thread/test/test_lock.py
+++ b/pypy/module/thread/test/test_lock.py
@@ -46,6 +46,12 @@
             assert feedback == [42]
         assert lock.locked() is False
 
+    def test_timeout(self):
+        import _thread
+        assert isinstance(_thread.TIMEOUT_MAX, float)
+        assert _thread.TIMEOUT_MAX > 1000
+
+
 def test_compile_lock():
     from pypy.rlib import rgc
     from pypy.module.thread.ll_thread import allocate_lock

From noreply at buildbot.pypy.org  Sat Oct 22 23:03:18 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sat, 22 Oct 2011 23:03:18 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Implement timeout in thread.Lock.acquire()
Message-ID: <20111022210318.E3689820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48348:76b23a4c1483
Date: 2011-10-22 22:02 +0200
http://bitbucket.org/pypy/pypy/changeset/76b23a4c1483/

Log:	Implement timeout in thread.Lock.acquire()

diff --git a/pypy/module/thread/ll_thread.py b/pypy/module/thread/ll_thread.py
--- a/pypy/module/thread/ll_thread.py
+++ b/pypy/module/thread/ll_thread.py
@@ -58,6 +58,10 @@
 c_thread_acquirelock = llexternal('RPyThreadAcquireLock', [TLOCKP, rffi.INT],
                                   rffi.INT,
                                   threadsafe=True)    # release the GIL
+c_thread_acquirelock_timed = llexternal('RPyThreadAcquireLockTimed', 
+                                        [TLOCKP, rffi.LONGLONG, rffi.INT],
+                                        rffi.INT,
+                                        threadsafe=True)    # release the GIL
 c_thread_releaselock = llexternal('RPyThreadReleaseLock', [TLOCKP], lltype.Void,
                                   threadsafe=True)    # release the GIL
 
@@ -116,6 +120,11 @@
         res = rffi.cast(lltype.Signed, res)
         return bool(res)
 
+    def acquire_timed(self, timeout):
+        res = c_thread_acquirelock_timed(self._lock, timeout, 1)
+        res = rffi.cast(lltype.Signed, res)
+        return bool(res)
+
     def release(self):
         # Sanity check: the lock must be locked
         if self.acquire(False):
diff --git a/pypy/module/thread/os_lock.py b/pypy/module/thread/os_lock.py
--- a/pypy/module/thread/os_lock.py
+++ b/pypy/module/thread/os_lock.py
@@ -7,6 +7,7 @@
 from pypy.interpreter.baseobjspace import Wrappable
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.typedef import TypeDef
+from pypy.interpreter.error import OperationError
 from pypy.rlib.rarithmetic import r_longlong
 
 # Force the declaration of the type 'thread.LockType' for RPython
@@ -40,16 +41,32 @@
         except thread.error:
             raise wrap_thread_error(space, "out of resources")
 
-    @unwrap_spec(waitflag=int)
-    def descr_lock_acquire(self, space, waitflag=1):
+    @unwrap_spec(blocking=int, timeout=float)
+    def descr_lock_acquire(self, space, blocking=1, timeout=-1.0):
         """Lock the lock.  Without argument, this blocks if the lock is already
 locked (even by the same thread), waiting for another thread to release
 the lock, and return None once the lock is acquired.
 With an argument, this will only block if the argument is true,
 and the return value reflects whether the lock is acquired.
-The blocking operation is not interruptible."""
+The blocking operation is interruptible."""
+        if not blocking and timeout != -1.0:
+            raise OperationError(space.w_ValueError, space.wrap(
+                    "can't specify a timeout for a non-blocking call"))
+        if timeout < 0.0 and timeout != -1.0:
+            raise OperationError(space.w_ValueError, space.wrap(
+                    "timeout value must be strictly positive"))
+        if not blocking:
+            microseconds = 0
+        elif timeout == -1.0:
+            microseconds = -1
+        else:
+            timeout *= 1e6
+            if timeout > float(TIMEOUT_MAX):
+                raise OperationError(space.w_ValueError, space.wrap(
+                        "timeout value is too large"))
+            microseconds = r_longlong(timeout)
         mylock = self.lock
-        result = mylock.acquire(bool(waitflag))
+        result = mylock.acquire_timed(microseconds)
         return space.newbool(result)
 
     def descr_lock_release(self, space):
diff --git a/pypy/module/thread/test/test_lock.py b/pypy/module/thread/test/test_lock.py
--- a/pypy/module/thread/test/test_lock.py
+++ b/pypy/module/thread/test/test_lock.py
@@ -50,6 +50,10 @@
         import _thread
         assert isinstance(_thread.TIMEOUT_MAX, float)
         assert _thread.TIMEOUT_MAX > 1000
+        lock = _thread.allocate_lock()
+        assert lock.acquire() is True
+        assert lock.acquire(False) is False
+        assert lock.acquire(True, timeout=.1) is False
 
 
 def test_compile_lock():
diff --git a/pypy/translator/c/src/thread.h b/pypy/translator/c/src/thread.h
--- a/pypy/translator/c/src/thread.h
+++ b/pypy/translator/c/src/thread.h
@@ -5,6 +5,14 @@
 #define __PYPY_THREAD_H
 #include <assert.h>
 
+#define RPY_TIMEOUT_T long long
+
+typedef enum RPyLockStatus {
+    RPY_LOCK_FAILURE = 0,
+    RPY_LOCK_ACQUIRED = 1,
+    RPY_LOCK_INTR
+} RPyLockStatus;
+
 #ifdef _WIN32
 #include "thread_nt.h"
 #else
diff --git a/pypy/translator/c/src/thread_nt.h b/pypy/translator/c/src/thread_nt.h
--- a/pypy/translator/c/src/thread_nt.h
+++ b/pypy/translator/c/src/thread_nt.h
@@ -22,19 +22,19 @@
 } callobj;
 
 typedef struct RPyOpaque_ThreadLock {
-	LONG   owned ;
-	DWORD  thread_id ;
-	HANDLE hevent ;
-} NRMUTEX, *PNRMUTEX ;
+    HANDLE sem;
+} NRMUTEX, *PNRMUTEX;
 
 /* prototypes */
 long RPyThreadStart(void (*func)(void));
 BOOL InitializeNonRecursiveMutex(PNRMUTEX mutex);
 VOID DeleteNonRecursiveMutex(PNRMUTEX mutex);
-DWORD EnterNonRecursiveMutex(PNRMUTEX mutex, BOOL wait);
+DWORD EnterNonRecursiveMutex(PNRMUTEX mutex, DWORD milliseconds);
 BOOL LeaveNonRecursiveMutex(PNRMUTEX mutex);
 void RPyOpaqueDealloc_ThreadLock(struct RPyOpaque_ThreadLock *lock);
 int RPyThreadAcquireLock(struct RPyOpaque_ThreadLock *lock, int waitflag);
+RPyLockStatus RPyThreadAcquireLockTimed(struct RPyOpaque_ThreadLock *lock,
+					RPY_TIMEOUT_T timeout, int intr_flag);
 void RPyThreadReleaseLock(struct RPyOpaque_ThreadLock *lock);
 long RPyThreadGetStackSize(void);
 long RPyThreadSetStackSize(long);
@@ -125,47 +125,24 @@
 
 BOOL InitializeNonRecursiveMutex(PNRMUTEX mutex)
 {
-	mutex->owned = -1 ;  /* No threads have entered NonRecursiveMutex */
-	mutex->thread_id = 0 ;
-	mutex->hevent = CreateEvent(NULL, FALSE, FALSE, NULL) ;
-	return mutex->hevent != NULL ;	/* TRUE if the mutex is created */
+    mutex->sem = CreateSemaphore(NULL, 1, 1, NULL);
 }
 
 VOID DeleteNonRecursiveMutex(PNRMUTEX mutex)
 {
-	/* No in-use check */
-	CloseHandle(mutex->hevent) ;
-	mutex->hevent = NULL ; /* Just in case */
+    /* No in-use check */
+    CloseHandle(mutex->sem);
+    mutex->sem = NULL ; /* Just in case */
 }
 
 DWORD EnterNonRecursiveMutex(PNRMUTEX mutex, BOOL wait)
 {
-	/* Assume that the thread waits successfully */
-	DWORD ret ;
-
-	/* InterlockedIncrement(&mutex->owned) == 0 means that no thread currently owns the mutex */
-	if (!wait)
-	{
-		if (InterlockedCompareExchange(&mutex->owned, 0, -1) != -1)
-			return WAIT_TIMEOUT ;
-		ret = WAIT_OBJECT_0 ;
-	}
-	else
-		ret = InterlockedIncrement(&mutex->owned) ?
-			/* Some thread owns the mutex, let's wait... */
-			WaitForSingleObject(mutex->hevent, INFINITE) : WAIT_OBJECT_0 ;
-
-	mutex->thread_id = GetCurrentThreadId() ; /* We own it */
-	return ret ;
+    return WaitForSingleObject(mutex->sem, milliseconds);
 }
 
 BOOL LeaveNonRecursiveMutex(PNRMUTEX mutex)
 {
-	/* We don't own the mutex */
-	mutex->thread_id = 0 ;
-	return
-		InterlockedDecrement(&mutex->owned) < 0 ||
-		SetEvent(mutex->hevent) ; /* Other threads are waiting, wake one on them up */
+    return ReleaseSemaphore(mutex->sem, 1, NULL);
 }
 
 /************************************************************/
@@ -181,8 +158,8 @@
 
 void RPyOpaqueDealloc_ThreadLock(struct RPyOpaque_ThreadLock *lock)
 {
-	if (lock->hevent != NULL)
-		DeleteNonRecursiveMutex(lock);
+    if (lock->sem != NULL)
+	DeleteNonRecursiveMutex(lock);
 }
 
 /*
@@ -191,9 +168,40 @@
  * and 0 if the lock was not acquired. This means a 0 is returned
  * if the lock has already been acquired by this thread!
  */
+RPyLockStatus
+RPyThreadAcquireLockTimed(struct RPyOpaque_ThreadLock *lock,
+			  RPY_TIMEOUT_T microseconds, int intr_flag)
+{
+    /* Fow now, intr_flag does nothing on Windows, and lock acquires are
+     * uninterruptible.  */
+    PyLockStatus success;
+    PY_TIMEOUT_T milliseconds;
+
+    if (microseconds >= 0) {
+        milliseconds = microseconds / 1000;
+        if (microseconds % 1000 > 0)
+            ++milliseconds;
+        if ((DWORD) milliseconds != milliseconds)
+            Py_FatalError("Timeout too large for a DWORD, "
+                           "please check PY_TIMEOUT_MAX");
+    }
+    else
+        milliseconds = INFINITE;
+
+    if (lock && EnterNonRecursiveMutex(
+	    lock, (DWORD)milliseconds) == WAIT_OBJECT_0) {
+        success = PY_LOCK_ACQUIRED;
+    }
+    else {
+        success = PY_LOCK_FAILURE;
+    }
+
+    return success;
+}
+
 int RPyThreadAcquireLock(struct RPyOpaque_ThreadLock *lock, int waitflag)
 {
-	return EnterNonRecursiveMutex(lock, (waitflag != 0 ? INFINITE : 0)) == WAIT_OBJECT_0;
+    return RPyThreadAcquireLockTimed(lock, waitflag ? -1 : 0, /*intr_flag=*/0);
 }
 
 void RPyThreadReleaseLock(struct RPyOpaque_ThreadLock *lock)
diff --git a/pypy/translator/c/src/thread_pthread.h b/pypy/translator/c/src/thread_pthread.h
--- a/pypy/translator/c/src/thread_pthread.h
+++ b/pypy/translator/c/src/thread_pthread.h
@@ -7,6 +7,7 @@
 */
 
 #include <unistd.h>   /* for the _POSIX_xxx and _POSIX_THREAD_xxx defines */
+#include <sys/time.h> /* for gettimeofday() */
 #include <stdlib.h>
 #include <pthread.h>
 #include <signal.h>
@@ -97,6 +98,8 @@
 int RPyThreadLockInit(struct RPyOpaque_ThreadLock *lock);
 void RPyOpaqueDealloc_ThreadLock(struct RPyOpaque_ThreadLock *lock);
 int RPyThreadAcquireLock(struct RPyOpaque_ThreadLock *lock, int waitflag);
+RPyLockStatus RPyThreadAcquireLockTimed(struct RPyOpaque_ThreadLock *lock,
+					RPY_TIMEOUT_T timeout, int intr_flag);
 void RPyThreadReleaseLock(struct RPyOpaque_ThreadLock *lock);
 long RPyThreadGetStackSize(void);
 long RPyThreadSetStackSize(long);
@@ -238,6 +241,29 @@
 #endif
 }
 
+#ifdef GETTIMEOFDAY_NO_TZ
+#define RPY_GETTIMEOFDAY(ptv) gettimeofday(ptv)
+#else
+#define RPY_GETTIMEOFDAY(ptv) gettimeofday(ptv, (struct timezone *)NULL)
+#endif
+
+#define RPY_MICROSECONDS_TO_TIMESPEC(microseconds, ts) \
+do { \
+    struct timeval tv; \
+    RPY_GETTIMEOFDAY(&tv); \
+    tv.tv_usec += microseconds % 1000000; \
+    tv.tv_sec += microseconds / 1000000; \
+    tv.tv_sec += tv.tv_usec / 1000000; \
+    tv.tv_usec %= 1000000; \
+    ts.tv_sec = tv.tv_sec; \
+    ts.tv_nsec = tv.tv_usec * 1000; \
+} while(0)
+
+int RPyThreadAcquireLock(struct RPyOpaque_ThreadLock *lock, int waitflag)
+{
+    return RPyThreadAcquireLockTimed(lock, waitflag ? -1 : 0, /*intr_flag=*/0);
+}
+
 /************************************************************/
 #ifdef USE_SEMAPHORES
 /************************************************************/
@@ -283,26 +309,50 @@
 	return (status == -1) ? errno : status;
 }
 
-int RPyThreadAcquireLock(struct RPyOpaque_ThreadLock *lock, int waitflag)
+RPyLockStatus
+RPyThreadAcquireLockTimed(struct RPyOpaque_ThreadLock *lock,
+			  RPY_TIMEOUT_T microseconds, int intr_flag)
 {
-	int success;
+	RPyLockStatus success;
 	sem_t *thelock = &lock->sem;
 	int status, error = 0;
+	struct timespec ts;
 
+	if (microseconds > 0)
+		RPY_MICROSECONDS_TO_TIMESPEC(microseconds, ts);
 	do {
-		if (waitflag)
-			status = rpythread_fix_status(sem_wait(thelock));
-		else
-			status = rpythread_fix_status(sem_trywait(thelock));
-	} while (status == EINTR); /* Retry if interrupted by a signal */
+	    if (microseconds > 0)
+		status = rpythread_fix_status(sem_timedwait(thelock, &ts));
+	    else if (microseconds == 0)
+		status = rpythread_fix_status(sem_trywait(thelock));
+	    else
+		status = rpythread_fix_status(sem_wait(thelock));
+	    /* Retry if interrupted by a signal, unless the caller wants to be
+	       notified.  */
+	} while (!intr_flag && status == EINTR);
 
-	if (waitflag) {
+	/* Don't check the status if we're stopping because of an interrupt.  */
+	if (!(intr_flag && status == EINTR)) {
+	    if (microseconds > 0) {
+		if (status != ETIMEDOUT)
+		    CHECK_STATUS("sem_timedwait");
+	    }
+	    else if (microseconds == 0) {
+		if (status != EAGAIN)
+		    CHECK_STATUS("sem_trywait");
+	    }
+	    else {
 		CHECK_STATUS("sem_wait");
-	} else if (status != EAGAIN) {
-		CHECK_STATUS("sem_trywait");
+	    }
 	}
-	
-	success = (status == 0) ? 1 : 0;
+
+	if (status == 0) {
+	    success = RPY_LOCK_ACQUIRED;
+	} else if (intr_flag && status == EINTR) {
+	    success = RPY_LOCK_INTR;
+	} else {
+	    success = RPY_LOCK_FAILURE;
+	}
 	return success;
 }
 
@@ -394,32 +444,63 @@
 	}
 }
 
-int RPyThreadAcquireLock(struct RPyOpaque_ThreadLock *lock, int waitflag)
+RPyLockStatus
+RPyThreadAcquireLockTimed(struct RPyOpaque_ThreadLock *lock,
+			  RPY_TIMEOUT_T microseconds, int intr_flag)
 {
-	int success;
+	RPyLockStatus success;
 	int status, error = 0;
 
 	status = pthread_mutex_lock( &lock->mut );
 	CHECK_STATUS("pthread_mutex_lock[1]");
-	success = lock->locked == 0;
 
-	if ( !success && waitflag ) {
+	if (lock->locked == 0) {
+	    success = RPY_LOCK_ACQUIRED;
+	} else if (microseconds == 0) {
+	    success = RPY_LOCK_FAILURE;
+	} else {
+		struct timespec ts;
+		if (microseconds > 0)
+		    RPY_MICROSECONDS_TO_TIMESPEC(microseconds, ts);
 		/* continue trying until we get the lock */
 
 		/* mut must be locked by me -- part of the condition
 		 * protocol */
-		while ( lock->locked ) {
-			status = pthread_cond_wait(&lock->lock_released,
-						   &lock->mut);
+		success = RPY_LOCK_FAILURE;
+		while (success == RPY_LOCK_FAILURE) {
+		    if (microseconds > 0) {
+			status = pthread_cond_timedwait(
+			    &lock->lock_released,
+			    &lock->mut, &ts);
+			if (status == ETIMEDOUT)
+			    break;
+			CHECK_STATUS("pthread_cond_timed_wait");
+		    }
+		    else {
+			status = pthread_cond_wait(
+			    &lock->lock_released,
+			    &lock->mut);
 			CHECK_STATUS("pthread_cond_wait");
+		    }
+
+		    if (intr_flag && status == 0 && lock->locked) {
+			/* We were woken up, but didn't get the lock.  We probably received
+			 * a signal.  Return RPY_LOCK_INTR to allow the caller to handle
+			 * it and retry.  */
+			success = RPY_LOCK_INTR;
+			break;
+		    } else if (status == 0 && !lock->locked) {
+			success = RPY_LOCK_ACQUIRED;
+		    } else {
+			success = RPY_LOCK_FAILURE;
+		    }
 		}
-		success = 1;
 	}
-	if (success) lock->locked = 1;
+	if (success == RPY_LOCK_ACQUIRED) lock->locked = 1;
 	status = pthread_mutex_unlock( &lock->mut );
 	CHECK_STATUS("pthread_mutex_unlock[1]");
 
-	if (error) success = 0;
+	if (error) success = RPY_LOCK_FAILURE;
 	return success;
 }
 

From noreply at buildbot.pypy.org  Sat Oct 22 23:03:20 2011
From: noreply at buildbot.pypy.org (amauryfa)
Date: Sat, 22 Oct 2011 23:03:20 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: CPython now provides its own datetime.py, 
Message-ID: <20111022210320.31E7D820D7@wyvern.cs.uni-duesseldorf.de>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r48349:b36f48bf48f8
Date: 2011-10-22 23:02 +0200
http://bitbucket.org/pypy/pypy/changeset/b36f48bf48f8/

Log:	CPython now provides its own datetime.py, which is similar to the
	one we had.

diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py
deleted file mode 100644
--- a/lib_pypy/datetime.py
+++ /dev/null
@@ -1,2046 +0,0 @@
-"""Concrete date/time and related types -- prototype implemented in Python.
-
-See http://www.zope.org/Members/fdrake/DateTimeWiki/FrontPage
-
-See also http://dir.yahoo.com/Reference/calendars/
-
-For a primer on DST, including many current DST rules, see
-http://webexhibits.org/daylightsaving/
-
-For more about DST than you ever wanted to know, see
-ftp://elsie.nci.nih.gov/pub/
-
-Sources for time zone and DST data: http://www.twinsun.com/tz/tz-link.htm
-
-This was originally copied from the sandbox of the CPython CVS repository.
-Thanks to Tim Peters for suggesting using it. 
-"""
-
-import time as _time
-import math as _math
-
-MINYEAR = 1
-MAXYEAR = 9999
-
-# Utility functions, adapted from Python's Demo/classes/Dates.py, which
-# also assumes the current Gregorian calendar indefinitely extended in
-# both directions.  Difference:  Dates.py calls January 1 of year 0 day
-# number 1.  The code here calls January 1 of year 1 day number 1.  This is
-# to match the definition of the "proleptic Gregorian" calendar in Dershowitz
-# and Reingold's "Calendrical Calculations", where it's the base calendar
-# for all computations.  See the book for algorithms for converting between
-# proleptic Gregorian ordinals and many other calendar systems.
-
-_DAYS_IN_MONTH = [None, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
-
-_DAYS_BEFORE_MONTH = [None]
-dbm = 0
-for dim in _DAYS_IN_MONTH[1:]:
-    _DAYS_BEFORE_MONTH.append(dbm)
-    dbm += dim
-del dbm, dim
-
-def _is_leap(year):
-    "year -> 1 if leap year, else 0."
-    return year % 4 == 0 and (year % 100 != 0 or year % 400 == 0)
-
-def _days_in_year(year):
-    "year -> number of days in year (366 if a leap year, else 365)."
-    return 365 + _is_leap(year)
-
-def _days_before_year(year):
-    "year -> number of days before January 1st of year."
-    y = year - 1
-    return y*365 + y//4 - y//100 + y//400
-
-def _days_in_month(year, month):
-    "year, month -> number of days in that month in that year."
-    assert 1 <= month <= 12, month
-    if month == 2 and _is_leap(year):
-        return 29
-    return _DAYS_IN_MONTH[month]
-
-def _days_before_month(year, month):
-    "year, month -> number of days in year preceeding first day of month."
-    if not 1 <= month <= 12:
-        raise ValueError('month must be in 1..12', month)
-    return _DAYS_BEFORE_MONTH[month] + (month > 2 and _is_leap(year))
-
-def _ymd2ord(year, month, day):
-    "year, month, day -> ordinal, considering 01-Jan-0001 as day 1."
-    if not 1 <= month <= 12:
-        raise ValueError('month must be in 1..12', month)
-    dim = _days_in_month(year, month)
-    if not 1 <= day <= dim:
-        raise ValueError('day must be in 1..%d' % dim, day)
-    return (_days_before_year(year) +
-            _days_before_month(year, month) +
-            day)
-
-_DI400Y = _days_before_year(401)    # number of days in 400 years
-_DI100Y = _days_before_year(101)    #    "    "   "   " 100   "
-_DI4Y   = _days_before_year(5)      #    "    "   "   "   4   "
-
-# A 4-year cycle has an extra leap day over what we'd get from pasting
-# together 4 single years.
-assert _DI4Y == 4 * 365 + 1
-
-# Similarly, a 400-year cycle has an extra leap day over what we'd get from
-# pasting together 4 100-year cycles.
-assert _DI400Y == 4 * _DI100Y + 1
-
-# OTOH, a 100-year cycle has one fewer leap day than we'd get from
-# pasting together 25 4-year cycles.
-assert _DI100Y == 25 * _DI4Y - 1
-
-def _ord2ymd(n):
-    "ordinal -> (year, month, day), considering 01-Jan-0001 as day 1."
-
-    # n is a 1-based index, starting at 1-Jan-1.  The pattern of leap years
-    # repeats exactly every 400 years.  The basic strategy is to find the
-    # closest 400-year boundary at or before n, then work with the offset
-    # from that boundary to n.  Life is much clearer if we subtract 1 from
-    # n first -- then the values of n at 400-year boundaries are exactly
-    # those divisible by _DI400Y:
-    #
-    #     D  M   Y            n              n-1
-    #     -- --- ----        ----------     ----------------
-    #     31 Dec -400        -_DI400Y       -_DI400Y -1
-    #      1 Jan -399         -_DI400Y +1   -_DI400Y      400-year boundary
-    #     ...
-    #     30 Dec  000        -1             -2
-    #     31 Dec  000         0             -1
-    #      1 Jan  001         1              0            400-year boundary
-    #      2 Jan  001         2              1
-    #      3 Jan  001         3              2
-    #     ...
-    #     31 Dec  400         _DI400Y        _DI400Y -1
-    #      1 Jan  401         _DI400Y +1     _DI400Y      400-year boundary
-    n -= 1
-    n400, n = divmod(n, _DI400Y)
-    year = n400 * 400 + 1   # ..., -399, 1, 401, ...
-
-    # Now n is the (non-negative) offset, in days, from January 1 of year, to
-    # the desired date.  Now compute how many 100-year cycles precede n.
-    # Note that it's possible for n100 to equal 4!  In that case 4 full
-    # 100-year cycles precede the desired day, which implies the desired
-    # day is December 31 at the end of a 400-year cycle.
-    n100, n = divmod(n, _DI100Y)
-
-    # Now compute how many 4-year cycles precede it.
-    n4, n = divmod(n, _DI4Y)
-
-    # And now how many single years.  Again n1 can be 4, and again meaning
-    # that the desired day is December 31 at the end of the 4-year cycle.
-    n1, n = divmod(n, 365)
-
-    year += n100 * 100 + n4 * 4 + n1
-    if n1 == 4 or n100 == 4:
-        assert n == 0
-        return year-1, 12, 31
-
-    # Now the year is correct, and n is the offset from January 1.  We find
-    # the month via an estimate that's either exact or one too large.
-    leapyear = n1 == 3 and (n4 != 24 or n100 == 3)
-    assert leapyear == _is_leap(year)
-    month = (n + 50) >> 5
-    preceding = _DAYS_BEFORE_MONTH[month] + (month > 2 and leapyear)
-    if preceding > n:  # estimate is too large
-        month -= 1
-        preceding -= _DAYS_IN_MONTH[month] + (month == 2 and leapyear)
-    n -= preceding
-    assert 0 <= n < _days_in_month(year, month)
-
-    # Now the year and month are correct, and n is the offset from the
-    # start of that month:  we're done!
-    return year, month, n+1
-
-# Month and day names.  For localized versions, see the calendar module.
-_MONTHNAMES = [None, "Jan", "Feb", "Mar", "Apr", "May", "Jun",
-                     "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]
-_DAYNAMES = [None, "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"]
-
-
-def _build_struct_time(y, m, d, hh, mm, ss, dstflag):
-    wday = (_ymd2ord(y, m, d) + 6) % 7
-    dnum = _days_before_month(y, m) + d
-    return _time.struct_time((y, m, d, hh, mm, ss, wday, dnum, dstflag))
-
-def _format_time(hh, mm, ss, us):
-    # Skip trailing microseconds when us==0.
-    result = "%02d:%02d:%02d" % (hh, mm, ss)
-    if us:
-        result += ".%06d" % us
-    return result
-
-# Correctly substitute for %z and %Z escapes in strftime formats.
-def _wrap_strftime(object, format, timetuple):
-    year = timetuple[0]
-    if year < 1900:
-        raise ValueError("year=%d is before 1900; the datetime strftime() "
-                         "methods require year >= 1900" % year)
-    # Don't call _utcoffset() or tzname() unless actually needed.
-    zreplace = None # the string to use for %z
-    Zreplace = None # the string to use for %Z
-
-    # Scan format for %z and %Z escapes, replacing as needed.
-    newformat = []
-    push = newformat.append
-    i, n = 0, len(format)
-    while i < n:
-        ch = format[i]
-        i += 1
-        if ch == '%':
-            if i < n:
-                ch = format[i]
-                i += 1
-                if ch == 'z':
-                    if zreplace is None:
-                        zreplace = ""
-                        if hasattr(object, "_utcoffset"):
-                            offset = object._utcoffset()
-                            if offset is not None:
-                                sign = '+'
-                                if offset < 0:
-                                    offset = -offset
-                                    sign = '-'
-                                h, m = divmod(offset, 60)
-                                zreplace = '%c%02d%02d' % (sign, h, m)
-                    assert '%' not in zreplace
-                    newformat.append(zreplace)
-                elif ch == 'Z':
-                    if Zreplace is None:
-                        Zreplace = ""
-                        if hasattr(object, "tzname"):
-                            s = object.tzname()
-                            if s is not None:
-                                # strftime is going to have at this: escape %
-                                Zreplace = s.replace('%', '%%')
-                    newformat.append(Zreplace)
-                elif ch == 'f':
-                    if isinstance(object, (time, datetime)):
-                        newformat.append('%06d' % object.microsecond)
-                    else:
-                        newformat.append('000000')
-                else:
-                    push('%')
-                    push(ch)
-            else:
-                push('%')
-        else:
-            push(ch)
-    newformat = "".join(newformat)
-    return _time.strftime(newformat, timetuple)
-
-def _call_tzinfo_method(tzinfo, methname, tzinfoarg):
-    if tzinfo is None:
-        return None
-    return getattr(tzinfo, methname)(tzinfoarg)
-
-# Just raise TypeError if the arg isn't None or a string.
-def _check_tzname(name):
-    if name is not None and not isinstance(name, str):
-        raise TypeError("tzinfo.tzname() must return None or string, "
-                        "not '%s'" % type(name))
-
-# name is the offset-producing method, "utcoffset" or "dst".
-# offset is what it returned.
-# If offset isn't None or timedelta, raises TypeError.
-# If offset is None, returns None.
-# Else offset is checked for being in range, and a whole # of minutes.
-# If it is, its integer value is returned.  Else ValueError is raised.
-def _check_utc_offset(name, offset):
-    assert name in ("utcoffset", "dst")
-    if offset is None:
-        return None
-    if not isinstance(offset, timedelta):
-        raise TypeError("tzinfo.%s() must return None "
-                        "or timedelta, not '%s'" % (name, type(offset)))
-    days = offset.days
-    if days < -1 or days > 0:
-        offset = 1440  # trigger out-of-range
-    else:
-        seconds = days * 86400 + offset.seconds
-        minutes, seconds = divmod(seconds, 60)
-        if seconds or offset.microseconds:
-            raise ValueError("tzinfo.%s() must return a whole number "
-                             "of minutes" % name)
-        offset = minutes
-    if -1440 < offset < 1440:
-        return offset
-    raise ValueError("%s()=%d, must be in -1439..1439" % (name, offset))
-
-def _check_date_fields(year, month, day):
-    if not MINYEAR <= year <= MAXYEAR:
-        raise ValueError('year must be in %d..%d' % (MINYEAR, MAXYEAR), year)
-    if not 1 <= month <= 12:
-        raise ValueError('month must be in 1..12', month)
-    dim = _days_in_month(year, month)
-    if not 1 <= day <= dim:
-        raise ValueError('day must be in 1..%d' % dim, day)
-
-def _check_time_fields(hour, minute, second, microsecond):
-    if not 0 <= hour <= 23:
-        raise ValueError('hour must be in 0..23', hour)
-    if not 0 <= minute <= 59:
-        raise ValueError('minute must be in 0..59', minute)
-    if not 0 <= second <= 59:
-        raise ValueError('second must be in 0..59', second)
-    if not 0 <= microsecond <= 999999:
-        raise ValueError('microsecond must be in 0..999999', microsecond)
-
-def _check_tzinfo_arg(tz):
-    if tz is not None and not isinstance(tz, tzinfo):
-        raise TypeError("tzinfo argument must be None or of a tzinfo subclass")
-
-
-# Notes on comparison:  In general, datetime module comparison operators raise
-# TypeError when they don't know how to do a comparison themself.  If they
-# returned NotImplemented instead, comparison could (silently) fall back to
-# the default compare-objects-by-comparing-their-memory-addresses strategy,
-# and that's not helpful.  There are two exceptions:
-#
-# 1. For date and datetime, if the other object has a "timetuple" attr,
-#    NotImplemented is returned.  This is a hook to allow other kinds of
-#    datetime-like objects a chance to intercept the comparison.
-#
-# 2. Else __eq__ and __ne__ return False and True, respectively.  This is
-#    so opertaions like
-#
-#        x == y
-#        x != y
-#        x in sequence
-#        x not in sequence
-#        dict[x] = y
-#
-#    don't raise annoying TypeErrors just because a datetime object
-#    is part of a heterogeneous collection.  If there's no known way to
-#    compare X to a datetime, saying they're not equal is reasonable.
-
-def _cmperror(x, y):
-    raise TypeError("can't compare '%s' to '%s'" % (
-                    type(x).__name__, type(y).__name__))
-
-# This is a start at a struct tm workalike.  Goals:
-#
-# + Works the same way across platforms.
-# + Handles all the fields datetime needs handled, without 1970-2038 glitches.
-#
-# Note:  I suspect it's best if this flavor of tm does *not* try to
-# second-guess timezones or DST.  Instead fold whatever adjustments you want
-# into the minutes argument (and the constructor will normalize).
-
-_ORD1970 = _ymd2ord(1970, 1, 1) # base ordinal for UNIX epoch
-
-class tmxxx:
-
-    ordinal = None
-
-    def __init__(self, year, month, day, hour=0, minute=0, second=0,
-                 microsecond=0):
-        # Normalize all the inputs, and store the normalized values.
-        if not 0 <= microsecond <= 999999:
-            carry, microsecond = divmod(microsecond, 1000000)
-            second += carry
-        if not 0 <= second <= 59:
-            carry, second = divmod(second, 60)
-            minute += carry
-        if not 0 <= minute <= 59:
-            carry, minute = divmod(minute, 60)
-            hour += carry
-        if not 0 <= hour <= 23:
-            carry, hour = divmod(hour, 24)
-            day += carry
-
-        # That was easy.  Now it gets muddy:  the proper range for day
-        # can't be determined without knowing the correct month and year,
-        # but if day is, e.g., plus or minus a million, the current month
-        # and year values make no sense (and may also be out of bounds
-        # themselves).
-        # Saying 12 months == 1 year should be non-controversial.
-        if not 1 <= month <= 12:
-            carry, month = divmod(month-1, 12)
-            year += carry
-            month += 1
-            assert 1 <= month <= 12
-
-        # Now only day can be out of bounds (year may also be out of bounds
-        # for a datetime object, but we don't care about that here).
-        # If day is out of bounds, what to do is arguable, but at least the
-        # method here is principled and explainable.
-        dim = _days_in_month(year, month)
-        if not 1 <= day <= dim:
-            # Move day-1 days from the first of the month.  First try to
-            # get off cheap if we're only one day out of range (adjustments
-            # for timezone alone can't be worse than that).
-            if day == 0:    # move back a day
-                month -= 1
-                if month > 0:
-                    day = _days_in_month(year, month)
-                else:
-                    year, month, day = year-1, 12, 31
-            elif day == dim + 1:    # move forward a day
-                month += 1
-                day = 1
-                if month > 12:
-                    month = 1
-                    year += 1
-            else:
-                self.ordinal = _ymd2ord(year, month, 1) + (day - 1)
-                year, month, day = _ord2ymd(self.ordinal)
-
-        self.year, self.month, self.day = year, month, day
-        self.hour, self.minute, self.second = hour, minute, second
-        self.microsecond = microsecond
-
-    def toordinal(self):
-        """Return proleptic Gregorian ordinal for the year, month and day.
-
-        January 1 of year 1 is day 1.  Only the year, month and day values
-        contribute to the result.
-        """
-        if self.ordinal is None:
-            self.ordinal = _ymd2ord(self.year, self.month, self.day)
-        return self.ordinal
-
-    def time(self):
-        "Return Unixish timestamp, as a float (assuming UTC)."
-        days = self.toordinal() - _ORD1970   # convert to UNIX epoch
-        seconds = ((days * 24. + self.hour)*60. + self.minute)*60.
-        return seconds + self.second + self.microsecond / 1e6
-
-    def ctime(self):
-        "Return ctime() style string."
-        weekday = self.toordinal() % 7 or 7
-        return "%s %s %2d %02d:%02d:%02d %04d" % (
-            _DAYNAMES[weekday],
-            _MONTHNAMES[self.month],
-            self.day,
-            self.hour, self.minute, self.second,
-            self.year)
-
-class timedelta(object):
-    """Represent the difference between two datetime objects.
-
-    Supported operators:
-
-    - add, subtract timedelta
-    - unary plus, minus, abs
-    - compare to timedelta
-    - multiply, divide by int/long
-
-    In addition, datetime supports subtraction of two datetime objects
-    returning a timedelta, and addition or subtraction of a datetime
-    and a timedelta giving a datetime.
-
-    Representation: (days, seconds, microseconds).  Why?  Because I
-    felt like it.
-    """
-
-    def __new__(cls, days=0, seconds=0, microseconds=0,
-                # XXX The following should only be used as keyword args:
-                milliseconds=0, minutes=0, hours=0, weeks=0):
-        # Doing this efficiently and accurately in C is going to be difficult
-        # and error-prone, due to ubiquitous overflow possibilities, and that
-        # C double doesn't have enough bits of precision to represent
-        # microseconds over 10K years faithfully.  The code here tries to make
-        # explicit where go-fast assumptions can be relied on, in order to
-        # guide the C implementation; it's way more convoluted than speed-
-        # ignoring auto-overflow-to-long idiomatic Python could be.
-
-        # XXX Check that all inputs are ints, longs or floats.
-
-        # Final values, all integer.
-        # s and us fit in 32-bit signed ints; d isn't bounded.
-        d = s = us = 0
-
-        # Normalize everything to days, seconds, microseconds.
-        days += weeks*7
-        seconds += minutes*60 + hours*3600
-        microseconds += milliseconds*1000
-
-        # Get rid of all fractions, and normalize s and us.
-        # Take a deep breath <wink>.
-        if isinstance(days, float):
-            dayfrac, days = _math.modf(days)
-            daysecondsfrac, daysecondswhole = _math.modf(dayfrac * (24.*3600.))
-            assert daysecondswhole == int(daysecondswhole)  # can't overflow
-            s = int(daysecondswhole)
-            assert days == long(days)
-            d = long(days)
-        else:
-            daysecondsfrac = 0.0
-            d = days
-        assert isinstance(daysecondsfrac, float)
-        assert abs(daysecondsfrac) <= 1.0
-        assert isinstance(d, (int, long))
-        assert abs(s) <= 24 * 3600
-        # days isn't referenced again before redefinition
-
-        if isinstance(seconds, float):
-            secondsfrac, seconds = _math.modf(seconds)
-            assert seconds == long(seconds)
-            seconds = long(seconds)
-            secondsfrac += daysecondsfrac
-            assert abs(secondsfrac) <= 2.0
-        else:
-            secondsfrac = daysecondsfrac
-        # daysecondsfrac isn't referenced again
-        assert isinstance(secondsfrac, float)
-        assert abs(secondsfrac) <= 2.0
-
-        assert isinstance(seconds, (int, long))
-        days, seconds = divmod(seconds, 24*3600)
-        d += days
-        s += int(seconds)    # can't overflow
-        assert isinstance(s, int)
-        assert abs(s) <= 2 * 24 * 3600
-        # seconds isn't referenced again before redefinition
-
-        usdouble = secondsfrac * 1e6
-        assert abs(usdouble) < 2.1e6    # exact value not critical
-        # secondsfrac isn't referenced again
-
-        if isinstance(microseconds, float):
-            microseconds += usdouble
-            microseconds = round(microseconds)
-            seconds, microseconds = divmod(microseconds, 1e6)
-            assert microseconds == int(microseconds)
-            assert seconds == long(seconds)
-            days, seconds = divmod(seconds, 24.*3600.)
-            assert days == long(days)
-            assert seconds == int(seconds)
-            d += long(days)
-            s += int(seconds)   # can't overflow
-            assert isinstance(s, int)
-            assert abs(s) <= 3 * 24 * 3600
-        else:
-            seconds, microseconds = divmod(microseconds, 1000000)
-            days, seconds = divmod(seconds, 24*3600)
-            d += days
-            s += int(seconds)    # can't overflow
-            assert isinstance(s, int)
-            assert abs(s) <= 3 * 24 * 3600
-            microseconds = float(microseconds)
-            microseconds += usdouble
-            microseconds = round(microseconds)
-        assert abs(s) <= 3 * 24 * 3600
-        assert abs(microseconds) < 3.1e6
-
-        # Just a little bit of carrying possible for microseconds and seconds.
-        assert isinstance(microseconds, float)
-        assert int(microseconds) == microseconds
-        us = int(microseconds)
-        seconds, us = divmod(us, 1000000)
-        s += seconds    # cant't overflow
-        assert isinstance(s, int)
-        days, s = divmod(s, 24*3600)
-        d += days
-
-        assert isinstance(d, (int, long))
-        assert isinstance(s, int) and 0 <= s < 24*3600
-        assert isinstance(us, int) and 0 <= us < 1000000
-
-        self = object.__new__(cls)
-
-        self.__days = d
-        self.__seconds = s
-        self.__microseconds = us
-        if abs(d) > 999999999:
-            raise OverflowError("timedelta # of days is too large: %d" % d)
-
-        return self
-
-    def __repr__(self):
-        if self.__microseconds:
-            return "%s(%d, %d, %d)" % ('datetime.' + self.__class__.__name__,
-                                       self.__days,
-                                       self.__seconds,
-                                       self.__microseconds)
-        if self.__seconds:
-            return "%s(%d, %d)" % ('datetime.' + self.__class__.__name__,
-                                   self.__days,
-                                   self.__seconds)
-        return "%s(%d)" % ('datetime.' + self.__class__.__name__, self.__days)
-
-    def __str__(self):
-        mm, ss = divmod(self.__seconds, 60)
-        hh, mm = divmod(mm, 60)
-        s = "%d:%02d:%02d" % (hh, mm, ss)
-        if self.__days:
-            def plural(n):
-                return n, abs(n) != 1 and "s" or ""
-            s = ("%d day%s, " % plural(self.__days)) + s
-        if self.__microseconds:
-            s = s + ".%06d" % self.__microseconds
-        return s
-
-    days = property(lambda self: self.__days, doc="days")
-    seconds = property(lambda self: self.__seconds, doc="seconds")
-    microseconds = property(lambda self: self.__microseconds,
-                            doc="microseconds")
-
-    def total_seconds(self):
-        return ((self.days * 86400 + self.seconds) * 10**6
-                + self.microseconds) / 1e6
-
-    def __add__(self, other):
-        if isinstance(other, timedelta):
-            # for CPython compatibility, we cannot use
-            # our __class__ here, but need a real timedelta
-            return timedelta(self.__days + other.__days,
-                             self.__seconds + other.__seconds,
-                             self.__microseconds + other.__microseconds)
-        return NotImplemented
-
-    __radd__ = __add__
-
-    def __sub__(self, other):
-        if isinstance(other, timedelta):
-            return self + -other
-        return NotImplemented
-
-    def __rsub__(self, other):
-        if isinstance(other, timedelta):
-            return -self + other
-        return NotImplemented
-
-    def __neg__(self):
-            # for CPython compatibility, we cannot use
-            # our __class__ here, but need a real timedelta
-            return timedelta(-self.__days,
-                             -self.__seconds,
-                             -self.__microseconds)
-
-    def __pos__(self):
-        return self
-
-    def __abs__(self):
-        if self.__days < 0:
-            return -self
-        else:
-            return self
-
-    def __mul__(self, other):
-        if isinstance(other, (int, long)):
-            # for CPython compatibility, we cannot use
-            # our __class__ here, but need a real timedelta
-            return timedelta(self.__days * other,
-                             self.__seconds * other,
-                             self.__microseconds * other)
-        return NotImplemented
-
-    __rmul__ = __mul__
-
-    def __div__(self, other):
-        if isinstance(other, (int, long)):
-            usec = ((self.__days * (24*3600L) + self.__seconds) * 1000000 +
-                    self.__microseconds)
-            return timedelta(0, 0, usec // other)
-        return NotImplemented
-
-    __floordiv__ = __div__
-
-    # Comparisons.
-
-    def __eq__(self, other):
-        if isinstance(other, timedelta):
-            return self.__cmp(other) == 0
-        else:
-            return False
-
-    def __ne__(self, other):
-        if isinstance(other, timedelta):
-            return self.__cmp(other) != 0
-        else:
-            return True
-
-    def __le__(self, other):
-        if isinstance(other, timedelta):
-            return self.__cmp(other) <= 0
-        else:
-            _cmperror(self, other)
-
-    def __lt__(self, other):
-        if isinstance(other, timedelta):
-            return self.__cmp(other) < 0
-        else:
-            _cmperror(self, other)
-
-    def __ge__(self, other):
-        if isinstance(other, timedelta):
-            return self.__cmp(other) >= 0
-        else:
-            _cmperror(self, other)
-
-    def __gt__(self, other):
-        if isinstance(other, timedelta):
-            return self.__cmp(other) > 0
-        else:
-            _cmperror(self, other)
-
-    def __cmp(self, other):
-        assert isinstance(other, timedelta)
-        return cmp(self.__getstate(), other.__getstate())
-
-    def __hash__(self):
-        return hash(self.__getstate())
-
-    def __nonzero__(self):
-        return (self.__days != 0 or
-                self.__seconds != 0 or
-                self.__microseconds != 0)
-
-    # Pickle support.
-
-    __safe_for_unpickling__ = True      # For Python 2.2
-
-    def __getstate(self):
-        return (self.__days, self.__seconds, self.__microseconds)
-
-    def __reduce__(self):
-        return (self.__class__, self.__getstate())
-
-timedelta.min = timedelta(-999999999)
-timedelta.max = timedelta(days=999999999, hours=23, minutes=59, seconds=59,
-                          microseconds=999999)
-timedelta.resolution = timedelta(microseconds=1)
-
-class date(object):
-    """Concrete date type.
-
-    Constructors:
-
-    __new__()
-    fromtimestamp()
-    today()
-    fromordinal()
-
-    Operators:
-
-    __repr__, __str__
-    __cmp__, __hash__
-    __add__, __radd__, __sub__ (add/radd only with timedelta arg)
-
-    Methods:
-
-    timetuple()
-    toordinal()
-    weekday()
-    isoweekday(), isocalendar(), isoformat()
-    ctime()
-    strftime()
-
-    Properties (readonly):
-    year, month, day
-    """
-
-    def __new__(cls, year, month=None, day=None):
-        """Constructor.
-
-        Arguments:
-
-        year, month, day (required, base 1)
-        """
-        if isinstance(year, str):
-            # Pickle support
-            self = object.__new__(cls)
-            self.__setstate(year)
-            return self
-        _check_date_fields(year, month, day)
-        self = object.__new__(cls)
-        self.__year = year
-        self.__month = month
-        self.__day = day
-        return self
-
-    # Additional constructors
-
-    def fromtimestamp(cls, t):
-        "Construct a date from a POSIX timestamp (like time.time())."
-        y, m, d, hh, mm, ss, weekday, jday, dst = _time.localtime(t)
-        return cls(y, m, d)
-    fromtimestamp = classmethod(fromtimestamp)
-
-    def today(cls):
-        "Construct a date from time.time()."
-        t = _time.time()
-        return cls.fromtimestamp(t)
-    today = classmethod(today)
-
-    def fromordinal(cls, n):
-        """Contruct a date from a proleptic Gregorian ordinal.
-
-        January 1 of year 1 is day 1.  Only the year, month and day are
-        non-zero in the result.
-        """
-        y, m, d = _ord2ymd(n)
-        return cls(y, m, d)
-    fromordinal = classmethod(fromordinal)
-
-    # Conversions to string
-
-    def __repr__(self):
-        "Convert to formal string, for repr()."
-        return "%s(%d, %d, %d)" % ('datetime.' + self.__class__.__name__,
-                                   self.__year,
-                                   self.__month,
-                                   self.__day)
-    # XXX These shouldn't depend on time.localtime(), because that
-    # clips the usable dates to [1970 .. 2038).  At least ctime() is
-    # easily done without using strftime() -- that's better too because
-    # strftime("%c", ...) is locale specific.
-
-    def ctime(self):
-        "Format a la ctime()."
-        return tmxxx(self.__year, self.__month, self.__day).ctime()
-
-    def strftime(self, fmt):
-        "Format using strftime()."
-        return _wrap_strftime(self, fmt, self.timetuple())
-
-    def isoformat(self):
-        """Return the date formatted according to ISO.
-
-        This is 'YYYY-MM-DD'.
-
-        References:
-        - http://www.w3.org/TR/NOTE-datetime
-        - http://www.cl.cam.ac.uk/~mgk25/iso-time.html
-        """
-        return "%04d-%02d-%02d" % (self.__year, self.__month, self.__day)
-
-    __str__ = isoformat
-
-    def __format__(self, format):
-        if not isinstance(format, (str, unicode)):
-            raise ValueError("__format__ excepts str or unicode, not %s" %
-                             format.__class__.__name__)
-        if not format:
-            return str(self)
-        return self.strftime(format)
-
-    # Read-only field accessors
-    year = property(lambda self: self.__year,
-                    doc="year (%d-%d)" % (MINYEAR, MAXYEAR))
-    month = property(lambda self: self.__month, doc="month (1-12)")
-    day = property(lambda self: self.__day, doc="day (1-31)")
-
-    # Standard conversions, __cmp__, __hash__ (and helpers)
-
-    def timetuple(self):
-        "Return local time tuple compatible with time.localtime()."
-        return _build_struct_time(self.__year, self.__month, self.__day,
-                                  0, 0, 0, -1)
-
-    def toordinal(self):
-        """Return proleptic Gregorian ordinal for the year, month and day.
-
-        January 1 of year 1 is day 1.  Only the year, month and day values
-        contribute to the result.
-        """
-        return _ymd2ord(self.__year, self.__month, self.__day)
-
-    def replace(self, year=None, month=None, day=None):
-        """Return a new date with new values for the specified fields."""
-        if year is None:
-            year = self.__year
-        if month is None:
-            month = self.__month
-        if day is None:
-            day = self.__day
-        _check_date_fields(year, month, day)
-        return date(year, month, day)
-
-    # Comparisons.
-
-    def __eq__(self, other):
-        if isinstance(other, date):
-            return self.__cmp(other) == 0
-        elif hasattr(other, "timetuple"):
-            return NotImplemented
-        else:
-            return False
-
-    def __ne__(self, other):
-        if isinstance(other, date):
-            return self.__cmp(other) != 0
-        elif hasattr(other, "timetuple"):
-            return NotImplemented
-        else:
-            return True
-
-    def __le__(self, other):
-        if isinstance(other, date):
-            return self.__cmp(other) <= 0
-        elif hasattr(other, "timetuple"):
-            return NotImplemented
-        else:
-            _cmperror(self, other)
-
-    def __lt__(self, other):
-        if isinstance(other, date):
-            return self.__cmp(other) < 0
-        elif hasattr(other, "timetuple"):
-            return NotImplemented
-        else:
-            _cmperror(self, other)
-
-    def __ge__(self, other):
-        if isinstance(other, date):
-            return self.__cmp(other) >= 0
-        elif hasattr(other, "timetuple"):
-            return NotImplemented
-        else:
-            _cmperror(self, other)
-
-    def __gt__(self, other):
-        if isinstance(other, date):
-            return self.__cmp(other) > 0
-        elif hasattr(other, "timetuple"):
-            return NotImplemented
-        else:
-            _cmperror(self, other)
-
-    def __cmp(self, other):
-        assert isinstance(other, date)
-        y, m, d = self.__year, self.__month, self.__day
-        y2, m2, d2 = other.__year, other.__month, other.__day
-        return cmp((y, m, d), (y2, m2, d2))
-
-    def __hash__(self):
-        "Hash."
-        return hash(self.__getstate())
-
-    # Computations
-
-    def _checkOverflow(self, year):
-        if not MINYEAR <= year <= MAXYEAR:
-            raise OverflowError("date +/-: result year %d not in %d..%d" %
-                                (year, MINYEAR, MAXYEAR))
-
-    def __add__(self, other):
-        "Add a date to a timedelta."
-        if isinstance(other, timedelta):
-            t = tmxxx(self.__year,
-                      self.__month,
-                      self.__day + other.days)
-            self._checkOverflow(t.year)
-            result = date(t.year, t.month, t.day)
-            return result
-        raise TypeError
-        # XXX Should be 'return NotImplemented', but there's a bug in 2.2...
-
-    __radd__ = __add__
-
-    def __sub__(self, other):
-        """Subtract two dates, or a date and a timedelta."""
-        if isinstance(other, timedelta):
-            return self + timedelta(-other.days)
-        if isinstance(other, date):
-            days1 = self.toordinal()
-            days2 = other.toordinal()
-            return timedelta(days1 - days2)
-        return NotImplemented
-
-    def weekday(self):
-        "Return day of the week, where Monday == 0 ... Sunday == 6."
-        return (self.toordinal() + 6) % 7
-
-    # Day-of-the-week and week-of-the-year, according to ISO
-
-    def isoweekday(self):
-        "Return day of the week, where Monday == 1 ... Sunday == 7."
-        # 1-Jan-0001 is a Monday
-        return self.toordinal() % 7 or 7
-
-    def isocalendar(self):
-        """Return a 3-tuple containing ISO year, week number, and weekday.
-
-        The first ISO week of the year is the (Mon-Sun) week
-        containing the year's first Thursday; everything else derives
-        from that.
-
-        The first week is 1; Monday is 1 ... Sunday is 7.
-
-        ISO calendar algorithm taken from
-        http://www.phys.uu.nl/~vgent/calendar/isocalendar.htm
-        """
-        year = self.__year
-        week1monday = _isoweek1monday(year)
-        today = _ymd2ord(self.__year, self.__month, self.__day)
-        # Internally, week and day have origin 0
-        week, day = divmod(today - week1monday, 7)
-        if week < 0:
-            year -= 1
-            week1monday = _isoweek1monday(year)
-            week, day = divmod(today - week1monday, 7)
-        elif week >= 52:
-            if today >= _isoweek1monday(year+1):
-                year += 1
-                week = 0
-        return year, week+1, day+1
-
-    # Pickle support.
-
-    __safe_for_unpickling__ = True      # For Python 2.2
-
-    def __getstate(self):
-        yhi, ylo = divmod(self.__year, 256)
-        return ("%c%c%c%c" % (yhi, ylo, self.__month, self.__day), )
-
-    def __setstate(self, string):
-        if len(string) != 4 or not (1 <= ord(string[2]) <= 12):
-            raise TypeError("not enough arguments")
-        yhi, ylo, self.__month, self.__day = map(ord, string)
-        self.__year = yhi * 256 + ylo
-
-    def __reduce__(self):
-        return (self.__class__, self.__getstate())
-
-_date_class = date  # so functions w/ args named "date" can get at the class
-
-date.min = date(1, 1, 1)
-date.max = date(9999, 12, 31)
-date.resolution = timedelta(days=1)
-
-class tzinfo(object):
-    """Abstract base class for time zone info classes.
-
-    Subclasses must override the name(), utcoffset() and dst() methods.
-    """
-
-    def tzname(self, dt):
-        "datetime -> string name of time zone."
-        raise NotImplementedError("tzinfo subclass must override tzname()")
-
-    def utcoffset(self, dt):
-        "datetime -> minutes east of UTC (negative for west of UTC)"
-        raise NotImplementedError("tzinfo subclass must override utcoffset()")
-
-    def dst(self, dt):
-        """datetime -> DST offset in minutes east of UTC.
-
-        Return 0 if DST not in effect.  utcoffset() must include the DST
-        offset.
-        """
-        raise NotImplementedError("tzinfo subclass must override dst()")
-
-    def fromutc(self, dt):
-        "datetime in UTC -> datetime in local time."
-
-        if not isinstance(dt, datetime):
-            raise TypeError("fromutc() requires a datetime argument")
-        if dt.tzinfo is not self:
-            raise ValueError("dt.tzinfo is not self")
-
-        dtoff = dt.utcoffset()
-        if dtoff is None:
-            raise ValueError("fromutc() requires a non-None utcoffset() "
-                             "result")
-
-        # See the long comment block at the end of this file for an
-        # explanation of this algorithm.
-        dtdst = dt.dst()
-        if dtdst is None:
-            raise ValueError("fromutc() requires a non-None dst() result")
-        delta = dtoff - dtdst
-        if delta:
-            dt += delta
-            dtdst = dt.dst()
-            if dtdst is None:
-                raise ValueError("fromutc(): dt.dst gave inconsistent "
-                                 "results; cannot convert")
-        if dtdst:
-            return dt + dtdst
-        else:
-            return dt
-
-    # Pickle support.
-
-    __safe_for_unpickling__ = True      # For Python 2.2
-
-    def __reduce__(self):
-        getinitargs = getattr(self, "__getinitargs__", None)
-        if getinitargs:
-            args = getinitargs()
-        else:
-            args = ()
-        getstate = getattr(self, "__getstate__", None)
-        if getstate:
-            state = getstate()
-        else:
-            state = getattr(self, "__dict__", None) or None
-        if state is None:
-            return (self.__class__, args)
-        else:
-            return (self.__class__, args, state)
-
-_tzinfo_class = tzinfo   # so functions w/ args named "tinfo" can get at it
-
-class time(object):
-    """Time with time zone.
-
-    Constructors:
-
-    __new__()
-
-    Operators:
-
-    __repr__, __str__
-    __cmp__, __hash__
-
-    Methods:
-
-    strftime()
-    isoformat()
-    utcoffset()
-    tzname()
-    dst()
-
-    Properties (readonly):
-    hour, minute, second, microsecond, tzinfo
-    """
-
-    def __new__(cls, hour=0, minute=0, second=0, microsecond=0, tzinfo=None):
-        """Constructor.
-
-        Arguments:
-
-        hour, minute (required)
-        second, microsecond (default to zero)
-        tzinfo (default to None)
-        """
-        self = object.__new__(cls)
-        if isinstance(hour, str):
-            # Pickle support
-            self.__setstate(hour, minute or None)
-            return self
-        _check_tzinfo_arg(tzinfo)
-        _check_time_fields(hour, minute, second, microsecond)
-        self.__hour = hour
-        self.__minute = minute
-        self.__second = second
-        self.__microsecond = microsecond
-        self._tzinfo = tzinfo
-        return self
-
-    # Read-only field accessors
-    hour = property(lambda self: self.__hour, doc="hour (0-23)")
-    minute = property(lambda self: self.__minute, doc="minute (0-59)")
-    second = property(lambda self: self.__second, doc="second (0-59)")
-    microsecond = property(lambda self: self.__microsecond,
-                           doc="microsecond (0-999999)")
-    tzinfo = property(lambda self: self._tzinfo, doc="timezone info object")
-
-    # Standard conversions, __hash__ (and helpers)
-
-    # Comparisons.
-
-    def __eq__(self, other):
-        if isinstance(other, time):
-            return self.__cmp(other) == 0
-        else:
-            return False
-
-    def __ne__(self, other):
-        if isinstance(other, time):
-            return self.__cmp(other) != 0
-        else:
-            return True
-
-    def __le__(self, other):
-        if isinstance(other, time):
-            return self.__cmp(other) <= 0
-        else:
-            _cmperror(self, other)
-
-    def __lt__(self, other):
-        if isinstance(other, time):
-            return self.__cmp(other) < 0
-        else:
-            _cmperror(self, other)
-
-    def __ge__(self, other):
-        if isinstance(other, time):
-            return self.__cmp(other) >= 0
-        else:
-            _cmperror(self, other)
-
-    def __gt__(self, other):
-        if isinstance(other, time):
-            return self.__cmp(other) > 0
-        else:
-            _cmperror(self, other)
-
-    def __cmp(self, other):
-        assert isinstance(other, time)
-        mytz = self._tzinfo
-        ottz = other._tzinfo
-        myoff = otoff = None
-
-        if mytz is ottz:
-            base_compare = True
-        else:
-            myoff = self._utcoffset()
-            otoff = other._utcoffset()
-            base_compare = myoff == otoff
-
-        if base_compare:
-            return cmp((self.__hour, self.__minute, self.__second,
-                        self.__microsecond),
-                       (other.__hour, other.__minute, other.__second,
-                        other.__microsecond))
-        if myoff is None or otoff is None:
-            # XXX Buggy in 2.2.2.
-            raise TypeError("cannot compare naive and aware times")
-        myhhmm = self.__hour * 60 + self.__minute - myoff
-        othhmm = other.__hour * 60 + other.__minute - otoff
-        return cmp((myhhmm, self.__second, self.__microsecond),
-                   (othhmm, other.__second, other.__microsecond))
-
-    def __hash__(self):
-        """Hash."""
-        tzoff = self._utcoffset()
-        if not tzoff: # zero or None
-            return hash(self.__getstate()[0])
-        h, m = divmod(self.hour * 60 + self.minute - tzoff, 60)
-        if 0 <= h < 24:
-            return hash(time(h, m, self.second, self.microsecond))
-        return hash((h, m, self.second, self.microsecond))
-
-    # Conversion to string
-
-    def _tzstr(self, sep=":"):
-        """Return formatted timezone offset (+xx:xx) or None."""
-        off = self._utcoffset()
-        if off is not None:
-            if off < 0:
-                sign = "-"
-                off = -off
-            else:
-                sign = "+"
-            hh, mm = divmod(off, 60)
-            assert 0 <= hh < 24
-            off = "%s%02d%s%02d" % (sign, hh, sep, mm)
-        return off
-
-    def __repr__(self):
-        """Convert to formal string, for repr()."""
-        if self.__microsecond != 0:
-            s = ", %d, %d" % (self.__second, self.__microsecond)
-        elif self.__second != 0:
-            s = ", %d" % self.__second
-        else:
-            s = ""
-        s= "%s(%d, %d%s)" % ('datetime.' + self.__class__.__name__,
-                             self.__hour, self.__minute, s)
-        if self._tzinfo is not None:
-            assert s[-1:] == ")"
-            s = s[:-1] + ", tzinfo=%r" % self._tzinfo + ")"
-        return s
-
-    def isoformat(self):
-        """Return the time formatted according to ISO.
-
-        This is 'HH:MM:SS.mmmmmm+zz:zz', or 'HH:MM:SS+zz:zz' if
-        self.microsecond == 0.
-        """
-        s = _format_time(self.__hour, self.__minute, self.__second,
-                         self.__microsecond)
-        tz = self._tzstr()
-        if tz:
-            s += tz
-        return s
-
-    __str__ = isoformat
-
-    def __format__(self, format):
-        if not isinstance(format, (str, unicode)):
-            raise ValueError("__format__ excepts str or unicode, not %s" %
-                             format.__class__.__name__)
-        if not format:
-            return str(self)
-        return self.strftime(format)
-
-    def strftime(self, fmt):
-        """Format using strftime().  The date part of the timestamp passed
-        to underlying strftime should not be used.
-        """
-        # The year must be >= 1900 else Python's strftime implementation
-        # can raise a bogus exception.
-        timetuple = (1900, 1, 1,
-                     self.__hour, self.__minute, self.__second,
-                     0, 1, -1)
-        return _wrap_strftime(self, fmt, timetuple)
-
-    # Timezone functions
-
-    def utcoffset(self):
-        """Return the timezone offset in minutes east of UTC (negative west of
-        UTC)."""
-        offset = _call_tzinfo_method(self._tzinfo, "utcoffset", None)
-        offset = _check_utc_offset("utcoffset", offset)
-        if offset is not None:
-            offset = timedelta(minutes=offset)
-        return offset
-
-    # Return an integer (or None) instead of a timedelta (or None).
-    def _utcoffset(self):
-        offset = _call_tzinfo_method(self._tzinfo, "utcoffset", None)
-        offset = _check_utc_offset("utcoffset", offset)
-        return offset
-
-    def tzname(self):
-        """Return the timezone name.
-
-        Note that the name is 100% informational -- there's no requirement that
-        it mean anything in particular. For example, "GMT", "UTC", "-500",
-        "-5:00", "EDT", "US/Eastern", "America/New York" are all valid replies.
-        """
-        name = _call_tzinfo_method(self._tzinfo, "tzname", None)
-        _check_tzname(name)
-        return name
-
-    def dst(self):
-        """Return 0 if DST is not in effect, or the DST offset (in minutes
-        eastward) if DST is in effect.
-
-        This is purely informational; the DST offset has already been added to
-        the UTC offset returned by utcoffset() if applicable, so there's no
-        need to consult dst() unless you're interested in displaying the DST
-        info.
-        """
-        offset = _call_tzinfo_method(self._tzinfo, "dst", None)
-        offset = _check_utc_offset("dst", offset)
-        if offset is not None:
-            offset = timedelta(minutes=offset)
-        return offset
-
-    def replace(self, hour=None, minute=None, second=None, microsecond=None,
-                tzinfo=True):
-        """Return a new time with new values for the specified fields."""
-        if hour is None:
-            hour = self.hour
-        if minute is None:
-            minute = self.minute
-        if second is None:
-            second = self.second
-        if microsecond is None:
-            microsecond = self.microsecond
-        if tzinfo is True:
-            tzinfo = self.tzinfo
-        _check_time_fields(hour, minute, second, microsecond)
-        _check_tzinfo_arg(tzinfo)
-        return time(hour, minute, second, microsecond, tzinfo)
-
-    # Return an integer (or None) instead of a timedelta (or None).
-    def _dst(self):
-        offset = _call_tzinfo_method(self._tzinfo, "dst", None)
-        offset = _check_utc_offset("dst", offset)
-        return offset
-
-    def __nonzero__(self):
-        if self.second or self.microsecond:
-            return 1
-        offset = self._utcoffset() or 0
-        return self.hour * 60 + self.minute - offset != 0
-
-    # Pickle support.
-
-    __safe_for_unpickling__ = True      # For Python 2.2
-
-    def __getstate(self):
-        us2, us3 = divmod(self.__microsecond, 256)
-        us1, us2 = divmod(us2, 256)
-        basestate = ("%c" * 6) % (self.__hour, self.__minute, self.__second,
-                                  us1, us2, us3)
-        if self._tzinfo is None:
-            return (basestate,)
-        else:
-            return (basestate, self._tzinfo)
-
-    def __setstate(self, string, tzinfo):
-        if len(string) != 6 or ord(string[0]) >= 24:
-            raise TypeError("an integer is required")
-        self.__hour, self.__minute, self.__second, us1, us2, us3 = \
-                                                            map(ord, string)
-        self.__microsecond = (((us1 << 8) | us2) << 8) | us3
-        self._tzinfo = tzinfo
-
-    def __reduce__(self):
-        return (time, self.__getstate())
-
-_time_class = time  # so functions w/ args named "time" can get at the class
-
-time.min = time(0, 0, 0)
-time.max = time(23, 59, 59, 999999)
-time.resolution = timedelta(microseconds=1)
-
-class datetime(date):
-
-    # XXX needs docstrings
-    # See http://www.zope.org/Members/fdrake/DateTimeWiki/TimeZoneInfo
-
-    def __new__(cls, year, month=None, day=None, hour=0, minute=0, second=0,
-                microsecond=0, tzinfo=None):
-        if isinstance(year, str):
-            # Pickle support
-            self = date.__new__(cls, year[:4])
-            self.__setstate(year, month)
-            return self
-        _check_tzinfo_arg(tzinfo)
-        _check_time_fields(hour, minute, second, microsecond)
-        self = date.__new__(cls, year, month, day)
-        # XXX This duplicates __year, __month, __day for convenience :-(
-        self.__year = year
-        self.__month = month
-        self.__day = day
-        self.__hour = hour
-        self.__minute = minute
-        self.__second = second
-        self.__microsecond = microsecond
-        self._tzinfo = tzinfo
-        return self
-
-    # Read-only field accessors
-    hour = property(lambda self: self.__hour, doc="hour (0-23)")
-    minute = property(lambda self: self.__minute, doc="minute (0-59)")
-    second = property(lambda self: self.__second, doc="second (0-59)")
-    microsecond = property(lambda self: self.__microsecond,
-                           doc="microsecond (0-999999)")
-    tzinfo = property(lambda self: self._tzinfo, doc="timezone info object")
-
-    def fromtimestamp(cls, t, tz=None):
-        """Construct a datetime from a POSIX timestamp (like time.time()).
-
-        A timezone info object may be passed in as well.
-        """
-
-        _check_tzinfo_arg(tz)
-        if tz is None:
-            converter = _time.localtime
-        else:
-            converter = _time.gmtime
-        if t < 0.0:
-            us = int(round(((-t) % 1.0) * 1000000))
-            if us > 0:
-                us = 1000000 - us
-                t -= 1.0
-        else:
-            us = int(round((t % 1.0) * 1000000))
-            if us == 1000000:
-                us = 0
-                t += 1.0
-        y, m, d, hh, mm, ss, weekday, jday, dst = converter(t)
-        ss = min(ss, 59)    # clamp out leap seconds if the platform has them
-        result = cls(y, m, d, hh, mm, ss, us, tz)
-        if tz is not None:
-            result = tz.fromutc(result)
-        return result
-    fromtimestamp = classmethod(fromtimestamp)
-
-    def utcfromtimestamp(cls, t):
-        "Construct a UTC datetime from a POSIX timestamp (like time.time())."
-        if 1 - (t % 1.0) < 0.0000005:
-            t = float(int(t)) + 1
-        if t < 0:
-            t -= 1
-        y, m, d, hh, mm, ss, weekday, jday, dst = _time.gmtime(t)
-        us = int((t % 1.0) * 1000000)
-        ss = min(ss, 59)    # clamp out leap seconds if the platform has them
-        return cls(y, m, d, hh, mm, ss, us)
-    utcfromtimestamp = classmethod(utcfromtimestamp)
-
-    # XXX This is supposed to do better than we *can* do by using time.time(),
-    # XXX if the platform supports a more accurate way.  The C implementation
-    # XXX uses gettimeofday on platforms that have it, but that isn't
-    # XXX available from Python.  So now() may return different results
-    # XXX across the implementations.
-    def now(cls, tz=None):
-        "Construct a datetime from time.time() and optional time zone info."
-        t = _time.time()
-        return cls.fromtimestamp(t, tz)
-    now = classmethod(now)
-
-    def utcnow(cls):
-        "Construct a UTC datetime from time.time()."
-        t = _time.time()
-        return cls.utcfromtimestamp(t)
-    utcnow = classmethod(utcnow)
-
-    def combine(cls, date, time):
-        "Construct a datetime from a given date and a given time."
-        if not isinstance(date, _date_class):
-            raise TypeError("date argument must be a date instance")
-        if not isinstance(time, _time_class):
-            raise TypeError("time argument must be a time instance")
-        return cls(date.year, date.month, date.day,
-                   time.hour, time.minute, time.second, time.microsecond,
-                   time.tzinfo)
-    combine = classmethod(combine)
-
-    def timetuple(self):
-        "Return local time tuple compatible with time.localtime()."
-        dst = self._dst()
-        if dst is None:
-            dst = -1
-        elif dst:
-            dst = 1
-        return _build_struct_time(self.year, self.month, self.day,
-                                  self.hour, self.minute, self.second,
-                                  dst)
-
-    def utctimetuple(self):
-        "Return UTC time tuple compatible with time.gmtime()."
-        y, m, d = self.year, self.month, self.day
-        hh, mm, ss = self.hour, self.minute, self.second
-        offset = self._utcoffset()
-        if offset:  # neither None nor 0
-            tm = tmxxx(y, m, d, hh, mm - offset)
-            y, m, d = tm.year, tm.month, tm.day
-            hh, mm = tm.hour, tm.minute
-        return _build_struct_time(y, m, d, hh, mm, ss, 0)
-
-    def date(self):
-        "Return the date part."
-        return date(self.__year, self.__month, self.__day)
-
-    def time(self):
-        "Return the time part, with tzinfo None."
-        return time(self.hour, self.minute, self.second, self.microsecond)
-
-    def timetz(self):
-        "Return the time part, with same tzinfo."
-        return time(self.hour, self.minute, self.second, self.microsecond,
-                    self._tzinfo)
-
-    def replace(self, year=None, month=None, day=None, hour=None,
-                minute=None, second=None, microsecond=None, tzinfo=True):
-        """Return a new datetime with new values for the specified fields."""
-        if year is None:
-            year = self.year
-        if month is None:
-            month = self.month
-        if day is None:
-            day = self.day
-        if hour is None:
-            hour = self.hour
-        if minute is None:
-            minute = self.minute
-        if second is None:
-            second = self.second
-        if microsecond is None:
-            microsecond = self.microsecond
-        if tzinfo is True:
-            tzinfo = self.tzinfo
-        _check_date_fields(year, month, day)
-        _check_time_fields(hour, minute, second, microsecond)
-        _check_tzinfo_arg(tzinfo)
-        return datetime(year, month, day, hour, minute, second,
-                          microsecond, tzinfo)
-
-    def astimezone(self, tz):
-        if not isinstance(tz, tzinfo):
-            raise TypeError("tz argument must be an instance of tzinfo")
-
-        mytz = self.tzinfo
-        if mytz is None:
-            raise ValueError("astimezone() requires an aware datetime")
-
-        if tz is mytz:
-            return self
-
-        # Convert self to UTC, and attach the new time zone object.
-        myoffset = self.utcoffset()
-        if myoffset is None:
-            raise ValueError("astimezone() requires an aware datetime")
-        utc = (self - myoffset).replace(tzinfo=tz)
-
-        # Convert from UTC to tz's local time.
-        return tz.fromutc(utc)
-
-    # Ways to produce a string.
-
-    def ctime(self):
-        "Format a la ctime()."
-        t = tmxxx(self.__year, self.__month, self.__day, self.__hour,
-                  self.__minute, self.__second)
-        return t.ctime()
-
-    def isoformat(self, sep='T'):
-        """Return the time formatted according to ISO.
-
-        This is 'YYYY-MM-DD HH:MM:SS.mmmmmm', or 'YYYY-MM-DD HH:MM:SS' if
-        self.microsecond == 0.
-
-        If self.tzinfo is not None, the UTC offset is also attached, giving
-        'YYYY-MM-DD HH:MM:SS.mmmmmm+HH:MM' or 'YYYY-MM-DD HH:MM:SS+HH:MM'.
-
-        Optional argument sep specifies the separator between date and
-        time, default 'T'.
-        """
-        s = ("%04d-%02d-%02d%c" % (self.__year, self.__month, self.__day,
-                                  sep) +
-                _format_time(self.__hour, self.__minute, self.__second,
-                             self.__microsecond))
-        off = self._utcoffset()
-        if off is not None:
-            if off < 0:
-                sign = "-"
-                off = -off
-            else:
-                sign = "+"
-            hh, mm = divmod(off, 60)
-            s += "%s%02d:%02d" % (sign, hh, mm)
-        return s
-
-    def __repr__(self):
-        "Convert to formal string, for repr()."
-        L = [self.__year, self.__month, self.__day, # These are never zero
-             self.__hour, self.__minute, self.__second, self.__microsecond]
-        if L[-1] == 0:
-            del L[-1]
-        if L[-1] == 0:
-            del L[-1]            
-        s = ", ".join(map(str, L))
-        s = "%s(%s)" % ('datetime.' + self.__class__.__name__, s)
-        if self._tzinfo is not None:
-            assert s[-1:] == ")"
-            s = s[:-1] + ", tzinfo=%r" % self._tzinfo + ")"
-        return s
-
-    def __str__(self):
-        "Convert to string, for str()."
-        return self.isoformat(sep=' ')
-
-    @classmethod
-    def strptime(cls, date_string, format):
-        'string, format -> new datetime parsed from a string (like time.strptime()).'
-        from _strptime import _strptime
-        # _strptime._strptime returns a two-element tuple.  The first
-        # element is a time.struct_time object.  The second is the
-        # microseconds (which are not defined for time.struct_time).
-        struct, micros = _strptime(date_string, format)
-        return cls(*(struct[0:6] + (micros,)))
-
-    def utcoffset(self):
-        """Return the timezone offset in minutes east of UTC (negative west of
-        UTC)."""
-        offset = _call_tzinfo_method(self._tzinfo, "utcoffset", self)
-        offset = _check_utc_offset("utcoffset", offset)
-        if offset is not None:
-            offset = timedelta(minutes=offset)
-        return offset
-
-    # Return an integer (or None) instead of a timedelta (or None).
-    def _utcoffset(self):
-        offset = _call_tzinfo_method(self._tzinfo, "utcoffset", self)
-        offset = _check_utc_offset("utcoffset", offset)
-        return offset
-
-    def tzname(self):
-        """Return the timezone name.
-
-        Note that the name is 100% informational -- there's no requirement that
-        it mean anything in particular. For example, "GMT", "UTC", "-500",
-        "-5:00", "EDT", "US/Eastern", "America/New York" are all valid replies.
-        """
-        name = _call_tzinfo_method(self._tzinfo, "tzname", self)
-        _check_tzname(name)
-        return name
-
-    def dst(self):
-        """Return 0 if DST is not in effect, or the DST offset (in minutes
-        eastward) if DST is in effect.
-
-        This is purely informational; the DST offset has already been added to
-        the UTC offset returned by utcoffset() if applicable, so there's no
-        need to consult dst() unless you're interested in displaying the DST
-        info.
-        """
-        offset = _call_tzinfo_method(self._tzinfo, "dst", self)
-        offset = _check_utc_offset("dst", offset)
-        if offset is not None:
-            offset = timedelta(minutes=offset)
-        return offset
-
-    # Return an integer (or None) instead of a timedelta (or None).1573
-    def _dst(self):
-        offset = _call_tzinfo_method(self._tzinfo, "dst", self)
-        offset = _check_utc_offset("dst", offset)
-        return offset
-
-    # Comparisons.
-
-    def __eq__(self, other):
-        if isinstance(other, datetime):
-            return self.__cmp(other) == 0
-        elif hasattr(other, "timetuple") and not isinstance(other, date):
-            return NotImplemented
-        else:
-            return False
-
-    def __ne__(self, other):
-        if isinstance(other, datetime):
-            return self.__cmp(other) != 0
-        elif hasattr(other, "timetuple") and not isinstance(other, date):
-            return NotImplemented
-        else:
-            return True
-
-    def __le__(self, other):
-        if isinstance(other, datetime):
-            return self.__cmp(other) <= 0
-        elif hasattr(other, "timetuple") and not isinstance(other, date):
-            return NotImplemented
-        else:
-            _cmperror(self, other)
-
-    def __lt__(self, other):
-        if isinstance(other, datetime):
-            return self.__cmp(other) < 0
-        elif hasattr(other, "timetuple") and not isinstance(other, date):
-            return NotImplemented
-        else:
-            _cmperror(self, other)
-
-    def __ge__(self, other):
-        if isinstance(other, datetime):
-            return self.__cmp(other) >= 0
-        elif hasattr(other, "timetuple") and not isinstance(other, date):
-            return NotImplemented
-        else:
-            _cmperror(self, other)
-
-    def __gt__(self, other):
-        if isinstance(other, datetime):
-            return self.__cmp(other) > 0
-        elif hasattr(other, "timetuple") and not isinstance(other, date):
-            return NotImplemented
-        else:
-            _cmperror(self, other)
-
-    def __cmp(self, other):
-        assert isinstance(other, datetime)
-        mytz = self._tzinfo
-        ottz = other._tzinfo
-        myoff = otoff = None
-
-        if mytz is ottz:
-            base_compare = True
-        else:
-            if mytz is not None:
-                myoff = self._utcoffset()
-            if ottz is not None:
-                otoff = other._utcoffset()
-            base_compare = myoff == otoff
-
-        if base_compare:
-            return cmp((self.__year, self.__month, self.__day,
-                        self.__hour, self.__minute, self.__second,
-                        self.__microsecond),
-                       (other.__year, other.__month, other.__day,
-                        other.__hour, other.__minute, other.__second,
-                        other.__microsecond))
-        if myoff is None or otoff is None:
-            # XXX Buggy in 2.2.2.
-            raise TypeError("cannot compare naive and aware datetimes")
-        # XXX What follows could be done more efficiently...
-        diff = self - other     # this will take offsets into account
-        if diff.days < 0:
-            return -1
-        return diff and 1 or 0
-
-    def __add__(self, other):
-        "Add a datetime and a timedelta."
-        if not isinstance(other, timedelta):
-            return NotImplemented
-        t = tmxxx(self.__year,
-                  self.__month,
-                  self.__day + other.days,
-                  self.__hour,
-                  self.__minute,
-                  self.__second + other.seconds,
-                  self.__microsecond + other.microseconds)
-        self._checkOverflow(t.year)
-        result = datetime(t.year, t.month, t.day,
-                                t.hour, t.minute, t.second,
-                                t.microsecond, tzinfo=self._tzinfo)
-        return result
-
-    __radd__ = __add__
-
-    def __sub__(self, other):
-        "Subtract two datetimes, or a datetime and a timedelta."
-        if not isinstance(other, datetime):
-            if isinstance(other, timedelta):
-                return self + -other
-            return NotImplemented
-
-        days1 = self.toordinal()
-        days2 = other.toordinal()
-        secs1 = self.__second + self.__minute * 60 + self.__hour * 3600
-        secs2 = other.__second + other.__minute * 60 + other.__hour * 3600
-        base = timedelta(days1 - days2,
-                         secs1 - secs2,
-                         self.__microsecond - other.__microsecond)
-        if self._tzinfo is other._tzinfo:
-            return base
-        myoff = self._utcoffset()
-        otoff = other._utcoffset()
-        if myoff == otoff:
-            return base
-        if myoff is None or otoff is None:
-            raise TypeError, "cannot mix naive and timezone-aware time"
-        return base + timedelta(minutes = otoff-myoff)
-
-    def __hash__(self):
-        tzoff = self._utcoffset()
-        if tzoff is None:
-            return hash(self.__getstate()[0])
-        days = _ymd2ord(self.year, self.month, self.day)
-        seconds = self.hour * 3600 + (self.minute - tzoff) * 60 + self.second
-        return hash(timedelta(days, seconds, self.microsecond))
-
-    # Pickle support.
-
-    __safe_for_unpickling__ = True      # For Python 2.2
-
-    def __getstate(self):
-        yhi, ylo = divmod(self.__year, 256)
-        us2, us3 = divmod(self.__microsecond, 256)
-        us1, us2 = divmod(us2, 256)
-        basestate = ("%c" * 10) % (yhi, ylo, self.__month, self.__day,
-                                   self.__hour, self.__minute, self.__second,
-                                   us1, us2, us3)
-        if self._tzinfo is None:
-            return (basestate,)
-        else:
-            return (basestate, self._tzinfo)
-
-    def __setstate(self, string, tzinfo):
-        (yhi, ylo, self.__month, self.__day, self.__hour,
-         self.__minute, self.__second, us1, us2, us3) = map(ord, string)
-        self.__year = yhi * 256 + ylo
-        self.__microsecond = (((us1 << 8) | us2) << 8) | us3
-        self._tzinfo = tzinfo
-
-    def __reduce__(self):
-        return (self.__class__, self.__getstate())
-
-
-datetime.min = datetime(1, 1, 1)
-datetime.max = datetime(9999, 12, 31, 23, 59, 59, 999999)
-datetime.resolution = timedelta(microseconds=1)
-
-
-def _isoweek1monday(year):
-    # Helper to calculate the day number of the Monday starting week 1
-    # XXX This could be done more efficiently
-    THURSDAY = 3
-    firstday = _ymd2ord(year, 1, 1)
-    firstweekday = (firstday + 6) % 7 # See weekday() above
-    week1monday = firstday - firstweekday
-    if firstweekday > THURSDAY:
-        week1monday += 7
-    return week1monday
-
-"""
-Some time zone algebra.  For a datetime x, let
-    x.n = x stripped of its timezone -- its naive time.
-    x.o = x.utcoffset(), and assuming that doesn't raise an exception or
-          return None
-    x.d = x.dst(), and assuming that doesn't raise an exception or
-          return None
-    x.s = x's standard offset, x.o - x.d
-
-Now some derived rules, where k is a duration (timedelta).
-
-1. x.o = x.s + x.d
-   This follows from the definition of x.s.
-
-2. If x and y have the same tzinfo member, x.s = y.s.
-   This is actually a requirement, an assumption we need to make about
-   sane tzinfo classes.
-
-3. The naive UTC time corresponding to x is x.n - x.o.
-   This is again a requirement for a sane tzinfo class.
-
-4. (x+k).s = x.s
-   This follows from #2, and that datimetimetz+timedelta preserves tzinfo.
-
-5. (x+k).n = x.n + k
-   Again follows from how arithmetic is defined.
-
-Now we can explain tz.fromutc(x).  Let's assume it's an interesting case
-(meaning that the various tzinfo methods exist, and don't blow up or return
-None when called).
-
-The function wants to return a datetime y with timezone tz, equivalent to x.
-x is already in UTC.
-
-By #3, we want
-
-    y.n - y.o = x.n                             [1]
-
-The algorithm starts by attaching tz to x.n, and calling that y.  So
-x.n = y.n at the start.  Then it wants to add a duration k to y, so that [1]
-becomes true; in effect, we want to solve [2] for k:
-
-   (y+k).n - (y+k).o = x.n                      [2]
-
-By #1, this is the same as
-
-   (y+k).n - ((y+k).s + (y+k).d) = x.n          [3]
-
-By #5, (y+k).n = y.n + k, which equals x.n + k because x.n=y.n at the start.
-Substituting that into [3],
-
-   x.n + k - (y+k).s - (y+k).d = x.n; the x.n terms cancel, leaving
-   k - (y+k).s - (y+k).d = 0; rearranging,
-   k = (y+k).s - (y+k).d; by #4, (y+k).s == y.s, so
-   k = y.s - (y+k).d
-
-On the RHS, (y+k).d can't be computed directly, but y.s can be, and we
-approximate k by ignoring the (y+k).d term at first.  Note that k can't be
-very large, since all offset-returning methods return a duration of magnitude
-less than 24 hours.  For that reason, if y is firmly in std time, (y+k).d must
-be 0, so ignoring it has no consequence then.
-
-In any case, the new value is
-
-    z = y + y.s                                 [4]
-
-It's helpful to step back at look at [4] from a higher level:  it's simply
-mapping from UTC to tz's standard time.
-
-At this point, if
-
-    z.n - z.o = x.n                             [5]
-
-we have an equivalent time, and are almost done.  The insecurity here is
-at the start of daylight time.  Picture US Eastern for concreteness.  The wall
-time jumps from 1:59 to 3:00, and wall hours of the form 2:MM don't make good
-sense then.  The docs ask that an Eastern tzinfo class consider such a time to
-be EDT (because it's "after 2"), which is a redundant spelling of 1:MM EST
-on the day DST starts.  We want to return the 1:MM EST spelling because that's
-the only spelling that makes sense on the local wall clock.
-
-In fact, if [5] holds at this point, we do have the standard-time spelling,
-but that takes a bit of proof.  We first prove a stronger result.  What's the
-difference between the LHS and RHS of [5]?  Let
-
-    diff = x.n - (z.n - z.o)                    [6]
-
-Now
-    z.n =                       by [4]
-    (y + y.s).n =               by #5
-    y.n + y.s =                 since y.n = x.n
-    x.n + y.s =                 since z and y are have the same tzinfo member,
-                                    y.s = z.s by #2
-    x.n + z.s
-
-Plugging that back into [6] gives
-
-    diff =
-    x.n - ((x.n + z.s) - z.o) =     expanding
-    x.n - x.n - z.s + z.o =         cancelling
-    - z.s + z.o =                   by #2
-    z.d
-
-So diff = z.d.
-
-If [5] is true now, diff = 0, so z.d = 0 too, and we have the standard-time
-spelling we wanted in the endcase described above.  We're done.  Contrarily,
-if z.d = 0, then we have a UTC equivalent, and are also done.
-
-If [5] is not true now, diff = z.d != 0, and z.d is the offset we need to
-add to z (in effect, z is in tz's standard time, and we need to shift the
-local clock into tz's daylight time).
-
-Let
-
-    z' = z + z.d = z + diff                     [7]
-
-and we can again ask whether
-
-    z'.n - z'.o = x.n                           [8]
-
-If so, we're done.  If not, the tzinfo class is insane, according to the
-assumptions we've made.  This also requires a bit of proof.  As before, let's
-compute the difference between the LHS and RHS of [8] (and skipping some of
-the justifications for the kinds of substitutions we've done several times
-already):
-
-    diff' = x.n - (z'.n - z'.o) =           replacing z'.n via [7]
-            x.n  - (z.n + diff - z'.o) =    replacing diff via [6]
-            x.n - (z.n + x.n - (z.n - z.o) - z'.o) =
-            x.n - z.n - x.n + z.n - z.o + z'.o =    cancel x.n
-            - z.n + z.n - z.o + z'.o =              cancel z.n
-            - z.o + z'.o =                      #1 twice
-            -z.s - z.d + z'.s + z'.d =          z and z' have same tzinfo
-            z'.d - z.d
-
-So z' is UTC-equivalent to x iff z'.d = z.d at this point.  If they are equal,
-we've found the UTC-equivalent so are done.  In fact, we stop with [7] and
-return z', not bothering to compute z'.d.
-
-How could z.d and z'd differ?  z' = z + z.d [7], so merely moving z' by
-a dst() offset, and starting *from* a time already in DST (we know z.d != 0),
-would have to change the result dst() returns:  we start in DST, and moving
-a little further into it takes us out of DST.
-
-There isn't a sane case where this can happen.  The closest it gets is at
-the end of DST, where there's an hour in UTC with no spelling in a hybrid
-tzinfo class.  In US Eastern, that's 5:MM UTC = 0:MM EST = 1:MM EDT.  During
-that hour, on an Eastern clock 1:MM is taken as being in standard time (6:MM
-UTC) because the docs insist on that, but 0:MM is taken as being in daylight
-time (4:MM UTC).  There is no local time mapping to 5:MM UTC.  The local
-clock jumps from 1:59 back to 1:00 again, and repeats the 1:MM hour in
-standard time.  Since that's what the local clock *does*, we want to map both
-UTC hours 5:MM and 6:MM to 1:MM Eastern.  The result is ambiguous
-in local time, but so it goes -- it's the way the local clock works.
-
-When x = 5:MM UTC is the input to this algorithm, x.o=0, y.o=-5 and y.d=0,
-so z=0:MM.  z.d=60 (minutes) then, so [5] doesn't hold and we keep going.
-z' = z + z.d = 1:MM then, and z'.d=0, and z'.d - z.d = -60 != 0 so [8]
-(correctly) concludes that z' is not UTC-equivalent to x.
-
-Because we know z.d said z was in daylight time (else [5] would have held and
-we would have stopped then), and we know z.d != z'.d (else [8] would have held
-and we we have stopped then), and there are only 2 possible values dst() can
-return in Eastern, it follows that z'.d must be 0 (which it is in the example,
-but the reasoning doesn't depend on the example -- it depends on there being
-two possible dst() outcomes, one zero and the other non-zero).  Therefore
-z' must be in standard time, and is the spelling we want in this case.
-
-Note again that z' is not UTC-equivalent as far as the hybrid tzinfo class is
-concerned (because it takes z' as being in standard time rather than the
-daylight time we intend here), but returning it gives the real-life "local
-clock repeats an hour" behavior when mapping the "unspellable" UTC hour into
-tz.
-
-When the input is 6:MM, z=1:MM and z.d=0, and we stop at once, again with
-the 1:MM standard time spelling we want.
-
-So how can this break?  One of the assumptions must be violated.  Two
-possibilities:
-
-1) [2] effectively says that y.s is invariant across all y belong to a given
-   time zone.  This isn't true if, for political reasons or continental drift,
-   a region decides to change its base offset from UTC.
-
-2) There may be versions of "double daylight" time where the tail end of
-   the analysis gives up a step too early.  I haven't thought about that
-   enough to say.
-
-In any case, it's clear that the default fromutc() is strong enough to handle
-"almost all" time zones:  so long as the standard offset is invariant, it
-doesn't matter if daylight time transition points change from year to year, or
-if daylight time is skipped in some years; it doesn't matter how large or
-small dst() may get within its bounds; and it doesn't even matter if some
-perverse time zone returns a negative dst()).  So a breaking case must be
-pretty bizarre, and a tzinfo subclass can override fromutc() if it is.
-"""
-

From noreply at buildbot.pypy.org  Sun Oct 23 02:01:17 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sun, 23 Oct 2011 02:01:17 +0200 (CEST)
Subject: [pypy-commit] pypy default: optimize int_floordiv(0,
	i) to always return 0.
Message-ID: <20111023000117.8FFDB820D7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r48350:13caa6fd920e
Date: 2011-10-22 17:01 -0700
http://bitbucket.org/pypy/pypy/changeset/13caa6fd920e/

Log:	optimize int_floordiv(0, i) to always return 0.

diff --git a/pypy/jit/metainterp/optimizeopt/rewrite.py b/pypy/jit/metainterp/optimizeopt/rewrite.py
--- a/pypy/jit/metainterp/optimizeopt/rewrite.py
+++ b/pypy/jit/metainterp/optimizeopt/rewrite.py
@@ -448,6 +448,9 @@
         if v2.is_constant() and v2.box.getint() == 1:
             self.make_equal_to(op.result, v1)
             return
+        elif v1.is_constant() and v1.box.getint() == 0:
+            self.make_constant_int(op.result, 0)
+            return
         if v1.intbound.known_ge(IntBound(0, 0)) and v2.is_constant():
             val = v2.box.getint()
             if val & (val - 1) == 0 and val > 0: # val == 2**shift
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -2181,6 +2181,17 @@
         """
         self.optimize_loop(ops, expected)
 
+        ops = """
+        [i0]
+        i1 = int_floordiv(0, i0)
+        jump(i1)
+        """
+        expected = """
+        [i0]
+        jump(0)
+        """
+        self.optimize_loop(ops, expected)
+
     def test_fold_partially_constant_ops_ovf(self):
         ops = """
         [i0]

From noreply at buildbot.pypy.org  Sun Oct 23 02:05:16 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sun, 23 Oct 2011 02:05:16 +0200 (CEST)
Subject: [pypy-commit] pypy default: a failing JIT test. it seems to fail
 because we try to cast an rpy_str to rclass.OBJECT,
 perhaps this operation is not allowed,
 but then the underlying code shouldn't translate I think.
Message-ID: <20111023000516.39A06820D7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r48351:9b942c1b56a8
Date: 2011-10-22 17:05 -0700
http://bitbucket.org/pypy/pypy/changeset/9b942c1b56a8/

Log:	a failing JIT test. it seems to fail because we try to cast an
	rpy_str to rclass.OBJECT, perhaps this operation is not allowed, but
	then the underlying code shouldn't translate I think.

diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -3435,7 +3435,23 @@
             return sa
         res = self.meta_interp(f, [16])
         assert res == f(16)
-        
+
+    def test_ptr_eq_str_constants(self):
+        myjitdriver = JitDriver(greens = [], reds = ["n", "x"])
+        class A(object):
+            def __init__(self, v):
+                self.v = v
+        def f(n, x):
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, x=x)
+                z = 0 / x
+                a1 = A("key")
+                a2 = A("\x00")
+                n -= [a1, a2][z].v is not a2.v
+            return n
+        res = self.meta_interp(f, [10, 1])
+        assert res == 0
+
 
 
 class TestLLtype(BaseLLtypeTests, LLJitMixin):

From noreply at buildbot.pypy.org  Sun Oct 23 02:24:17 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sun, 23 Oct 2011 02:24:17 +0200 (CEST)
Subject: [pypy-commit] pypy default: failing test in optimizebasic form,
 if this code is invalid then this test can be removed
Message-ID: <20111023002417.E6AFD820D7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r48352:6be78cdf609b
Date: 2011-10-22 17:24 -0700
http://bitbucket.org/pypy/pypy/changeset/6be78cdf609b/

Log:	failing test in optimizebasic form, if this code is invalid then
	this test can be removed

diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -4800,6 +4800,18 @@
         """
         self.optimize_strunicode_loop(ops, expected)
 
+    def test_ptr_eq_str_constant(self):
+        ops = """
+        []
+        i0 = ptr_eq(s"abc", s"\x00")
+        finish(i0)
+        """
+        expected = """
+        []
+        finish(0)
+        """
+        self.optimize_loop(ops, expected)
+
 
 class TestLLtype(BaseTestOptimizeBasic, LLtypeMixin):
     pass

From noreply at buildbot.pypy.org  Sun Oct 23 05:55:31 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sun, 23 Oct 2011 05:55:31 +0200 (CEST)
Subject: [pypy-commit] pypy default: disable virtualization for arrays of
 structs,
 to be fixed by the virtual-dicts branch. in practice this wasn't very useful
 because it only worked on empty dicts.
Message-ID: <20111023035531.9958E820D7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r48353:d81036a102b3
Date: 2011-10-22 20:55 -0700
http://bitbucket.org/pypy/pypy/changeset/d81036a102b3/

Log:	disable virtualization for arrays of structs, to be fixed by the
	virtual-dicts branch. in practice this wasn't very useful because
	it only worked on empty dicts.

diff --git a/pypy/jit/backend/llgraph/runner.py b/pypy/jit/backend/llgraph/runner.py
--- a/pypy/jit/backend/llgraph/runner.py
+++ b/pypy/jit/backend/llgraph/runner.py
@@ -8,7 +8,7 @@
 from pypy.rpython.ootypesystem import ootype
 from pypy.rpython.llinterp import LLInterpreter
 from pypy.jit.metainterp import history
-from pypy.jit.metainterp.history import REF, INT, FLOAT
+from pypy.jit.metainterp.history import REF, INT, FLOAT, STRUCT
 from pypy.jit.metainterp.warmstate import unwrap
 from pypy.jit.metainterp.resoperation import rop
 from pypy.jit.backend import model
@@ -60,6 +60,9 @@
     def is_array_of_floats(self):
         return self.typeinfo == FLOAT
 
+    def is_array_of_structs(self):
+        return self.typeinfo == STRUCT
+
     def as_vtable_size_descr(self):
         return self
 
@@ -365,6 +368,8 @@
         size = symbolic.get_size(A)
         if isinstance(A.OF, lltype.Ptr) or isinstance(A.OF, lltype.Primitive):
             token = history.getkind(A.OF)[0]
+        elif isinstance(A.OF, lltype.Struct):
+            token = 's'
         else:
             token = '?'
         return self.getdescr(size, token)
diff --git a/pypy/jit/backend/llsupport/descr.py b/pypy/jit/backend/llsupport/descr.py
--- a/pypy/jit/backend/llsupport/descr.py
+++ b/pypy/jit/backend/llsupport/descr.py
@@ -164,6 +164,7 @@
 
     _is_array_of_pointers = False      # unless overridden by GcPtrArrayDescr
     _is_array_of_floats   = False      # unless overridden by FloatArrayDescr
+    _is_array_of_structs  = False      # unless overridden by StructArrayDescr
     _is_item_signed       = False      # unless overridden by XxxArrayDescr
 
     def is_array_of_pointers(self):
@@ -172,6 +173,9 @@
     def is_array_of_floats(self):
         return self._is_array_of_floats
 
+    def is_array_of_structs(self):
+        return self._is_array_of_structs
+
     def is_item_signed(self):
         return self._is_item_signed
 
@@ -196,6 +200,12 @@
     def get_item_size(self, translate_support_code):
         return symbolic.get_size(lltype.Float, translate_support_code)
 
+class StructArrayDescr(BaseArrayDescr):
+    _clsname = 'StructArrayDescr'
+    _is_array_of_structs = True
+    def get_item_size(self, translate_support_code):
+        return symbolic.get_size()
+
 class BaseArrayNoLengthDescr(BaseArrayDescr):
     def get_base_size(self, translate_support_code):
         return 0
@@ -215,6 +225,13 @@
 def getArrayDescrClass(ARRAY):
     if ARRAY.OF is lltype.Float:
         return FloatArrayDescr
+    elif isinstance(ARRAY.OF, lltype.Struct):
+        class Descr(StructArrayDescr):
+            _clsname = '%sArrayDescr' % ARRAY.OF._name
+            def get_item_size(self, translate_support_code):
+                return symbolic.get_size(ARRAY.OF, translate_support_code)
+        Descr.__name__ = Descr._clsname
+        return Descr
     return getDescrClass(ARRAY.OF, BaseArrayDescr, GcPtrArrayDescr,
                          NonGcPtrArrayDescr, 'Array', 'get_item_size',
                          '_is_array_of_floats', '_is_item_signed')
diff --git a/pypy/jit/metainterp/history.py b/pypy/jit/metainterp/history.py
--- a/pypy/jit/metainterp/history.py
+++ b/pypy/jit/metainterp/history.py
@@ -16,6 +16,7 @@
 INT   = 'i'
 REF   = 'r'
 FLOAT = 'f'
+STRUCT = 's'
 HOLE  = '_'
 VOID  = 'v'
 
@@ -172,6 +173,11 @@
         """
         raise NotImplementedError
 
+    def is_array_of_structs(self):
+        """ Implement for array descr
+        """
+        raise NotImplementedError
+
     def is_pointer_field(self):
         """ Implement for field descr
         """
diff --git a/pypy/jit/metainterp/optimizeopt/virtualize.py b/pypy/jit/metainterp/optimizeopt/virtualize.py
--- a/pypy/jit/metainterp/optimizeopt/virtualize.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualize.py
@@ -59,7 +59,7 @@
 
     def import_from(self, other, optimizer):
         raise NotImplementedError("should not be called at this level")
-    
+
 def get_fielddescrlist_cache(cpu):
     if not hasattr(cpu, '_optimizeopt_fielddescrlist_cache'):
         result = descrlist_dict()
@@ -113,7 +113,7 @@
         #
         if not we_are_translated():
             op.name = 'FORCE ' + self.source_op.name
-            
+
         if self._is_immutable_and_filled_with_constants(optforce):
             box = optforce.optimizer.constant_fold(op)
             self.make_constant(box)
@@ -239,12 +239,12 @@
         for index in range(len(self._items)):
             self._items[index] = self._items[index].force_at_end_of_preamble(already_forced, optforce)
         return self
-    
+
     def _really_force(self, optforce):
         assert self.source_op is not None
         if not we_are_translated():
             self.source_op.name = 'FORCE ' + self.source_op.name
-        optforce.emit_operation(self.source_op)        
+        optforce.emit_operation(self.source_op)
         self.box = box = self.source_op.result
         for index in range(len(self._items)):
             subvalue = self._items[index]
@@ -276,7 +276,7 @@
 
     def new(self):
         return OptVirtualize()
-        
+
     def make_virtual(self, known_class, box, source_op=None):
         vvalue = VirtualValue(self.optimizer.cpu, known_class, box, source_op)
         self.make_equal_to(box, vvalue)
@@ -386,7 +386,8 @@
 
     def optimize_NEW_ARRAY(self, op):
         sizebox = self.get_constant_box(op.getarg(0))
-        if sizebox is not None:
+        # For now we can't make arrays of structs virtual.
+        if sizebox is not None and not op.getdescr().is_array_of_structs():
             # if the original 'op' did not have a ConstInt as argument,
             # build a new one with the ConstInt argument
             if not isinstance(op.getarg(0), ConstInt):
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -3452,6 +3452,22 @@
         res = self.meta_interp(f, [10, 1])
         assert res == 0
 
+    def test_virtual_array_of_structs(self):
+        myjitdriver = JitDriver(greens = [], reds=["n", "d"])
+        def f(n):
+            d = None
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, d=d)
+                d = {}
+                if n % 2:
+                    d["k"] = n
+                else:
+                    d["z"] = n
+                n -= len(d)
+            return n
+        res = self.meta_interp(f, [10])
+        assert res == 0
+
 
 
 class TestLLtype(BaseLLtypeTests, LLJitMixin):

From noreply at buildbot.pypy.org  Sun Oct 23 07:35:06 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sun, 23 Oct 2011 07:35:06 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix translation
Message-ID: <20111023053506.72694820C7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r48354:10f7167b3e98
Date: 2011-10-22 22:34 -0700
http://bitbucket.org/pypy/pypy/changeset/10f7167b3e98/

Log:	fix translation

diff --git a/pypy/jit/backend/llsupport/descr.py b/pypy/jit/backend/llsupport/descr.py
--- a/pypy/jit/backend/llsupport/descr.py
+++ b/pypy/jit/backend/llsupport/descr.py
@@ -203,8 +203,6 @@
 class StructArrayDescr(BaseArrayDescr):
     _clsname = 'StructArrayDescr'
     _is_array_of_structs = True
-    def get_item_size(self, translate_support_code):
-        return symbolic.get_size()
 
 class BaseArrayNoLengthDescr(BaseArrayDescr):
     def get_base_size(self, translate_support_code):

From noreply at buildbot.pypy.org  Sun Oct 23 11:46:28 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 23 Oct 2011 11:46:28 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Fix enough to have the
 simplest test pass again.
Message-ID: <20111023094628.5A388820C7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r48355:174ecd921aae
Date: 2011-10-23 11:45 +0200
http://bitbucket.org/pypy/pypy/changeset/174ecd921aae/

Log:	Fix enough to have the simplest test pass again. For now, kill the
	logic of pages, because it's a mess to reason both about it and
	about concurrentgen.txt at the same time. All objects are just
	malloc()ed.

diff --git a/pypy/rpython/lltypesystem/llarena.py b/pypy/rpython/lltypesystem/llarena.py
--- a/pypy/rpython/lltypesystem/llarena.py
+++ b/pypy/rpython/lltypesystem/llarena.py
@@ -551,6 +551,8 @@
     has_protect = False
 
 
+# note: the concurrent GCs do mallocs in one thread and frees in another,
+# so make sure these point to a thread-safe implementation
 llimpl_malloc = rffi.llexternal('malloc', [lltype.Signed], llmemory.Address,
                                 sandboxsafe=True, _nowrapper=True)
 llimpl_free = rffi.llexternal('free', [llmemory.Address], lltype.Void,
diff --git a/pypy/rpython/memory/gc/concurrentgen.py b/pypy/rpython/memory/gc/concurrentgen.py
--- a/pypy/rpython/memory/gc/concurrentgen.py
+++ b/pypy/rpython/memory/gc/concurrentgen.py
@@ -9,6 +9,7 @@
 from pypy.rlib.rarithmetic import ovfcheck, LONG_BIT, r_uint
 from pypy.rpython.memory.gc.base import GCBase
 from pypy.rpython.memory import gctypelayout
+from pypy.rpython.memory.support import get_address_stack
 from pypy.module.thread import ll_thread
 
 #
@@ -18,7 +19,8 @@
 # See concurrentgen.txt for some details.
 #
 # Major collections are serialized for the mark phase, but the sweep
-# phase can be parallelized again.  XXX not done so far
+# phase can be parallelized again.  XXX not done so far, YYY investigate
+# also completely parallelizing them too
 #
 # Based on observations that the timing of collections with "minimark"
 # (on translate.py) is: about 15% of the time in minor collections
@@ -59,8 +61,10 @@
     malloc_zero_filled = True
     gcflag_extra = FL_EXTRA
 
-    HDR = lltype.Struct('header', ('tid', lltype.Signed))
-    HDRPTR = lltype.Ptr(HDR)
+    HDRPTR = lltype.Ptr(lltype.ForwardReference())
+    HDR = lltype.Struct('header', ('tid', lltype.Signed),
+                                  ('next', HDRPTR))   # <-- kill me later
+    HDRPTR.TO.become(HDR)
     HDRSIZE = llmemory.sizeof(HDR)
     NULL = lltype.nullptr(HDR)
     typeid_is_in_field = 'tid', llgroup.HALFSHIFT
@@ -68,63 +72,14 @@
     # ^^^ prebuilt objects may have the flag FL_WITHHASH;
     #     then they are one word longer, the extra word storing the hash.
 
-    TRANSLATION_PARAMS = {'page_size': 16384,
-                          'small_request_threshold': 35*WORD,
-                          }
+    TRANSLATION_PARAMS = {}
 
-    def __init__(self, config, page_size=64, small_request_threshold=24,
-                 **kwds):
-        # 'small_request_threshold' is the largest size that we will
-        # satisfy using our own pages mecanism.  Larger requests just
-        # go to the system malloc().
-        self.addressstack_lock_object = SyncLock()
-        kwds['lock'] = self.addressstack_lock_object
+    def __init__(self, config, **kwds):
         GCBase.__init__(self, config, **kwds)
-        assert small_request_threshold % WORD == 0
-        self.small_request_threshold = small_request_threshold
-        self.page_size = page_size
-        self.pagelists_length = small_request_threshold // WORD + 1
+        self.main_thread_ident = ll_thread.get_ident()
         #
-        # The following are arrays of 36 linked lists: the linked lists
-        # at indices 1 to 35 correspond to pages that store objects of
-        # size  1 * WORD  to  35 * WORD.  The linked list at index 0
-        # contains the pages of objects that are larger than this limit
-        # (variable-sized pages, with exactly one object per page).
-        #
-        def list_of_addresses_per_small_size():
-            return lltype.malloc(rffi.CArray(self.HDRPTR),
-                                 self.pagelists_length, flavor='raw',
-                                 immortal=True)
-        #
-        # pages that contain at least one new young object
-        self.new_young_objects_pages = list_of_addresses_per_small_size()
-        #
-        # when the collection starts and we make all young objects aging,
-        # we move the linked lists above into 'aging_objects_pages'
-        self.aging_objects_pages = list_of_addresses_per_small_size()
-        #
-        # free list of non-allocated locations within pages
-        # (at index 0: always empty)
-        self.location_free_lists = list_of_addresses_per_small_size()
-        #
-        # head and tail of the free list of locations built by the
-        # collector thread
-        self.collect_loc_heads = list_of_addresses_per_small_size()
-        self.collect_loc_tails = list_of_addresses_per_small_size()
-        #
-        def collector_start():
-            if we_are_translated():
-                self.collector_run()
-            else:
-                self.collector_run_nontranslated()
-        #
-        collector_start._should_never_raise_ = True
-        self.collector_start = collector_start
-        #
-        self.gray_objects = self.AddressStack()
-        self.extra_objects_to_mark = self.AddressStack()
-        self.flagged_objects = self.AddressStack()
-        self.prebuilt_root_objects = self.AddressStack()
+        # Create the CollectorThread object
+        self.collector = CollectorThread(self)
         #
         self._initialize()
         #
@@ -133,62 +88,31 @@
         # that was not scanned yet.
         self._init_writebarrier_logic()
 
-    def _clear_list(self, array):
-        i = 0
-        while i < self.pagelists_length:
-            array[i] = self.NULL
-            i += 1
-
     def _initialize(self):
-        self.free_pages = self.NULL
+        # Initialize the GC.  In normal translated program, this function
+        # is not translated but just called from __init__ ahead of time.
+        # During test_transformed_gc, it is translated, so that we can
+        # quickly reset the GC between tests.
         #
-        # Clear the lists
-        self._clear_list(self.nonfree_pages)
-        self._clear_list(self.collect_pages)
-        self._clear_list(self.free_lists)
-        self._clear_list(self.collect_heads)
-        self._clear_list(self.collect_tails)
+        self.extra_objects_to_mark = self.AddressStack()
+        self.flagged_objects = self.AddressStack()
+        self.prebuilt_root_objects = self.AddressStack()
         #
-        self.finalizer_pages = self.NULL
-        self.collect_finalizer_pages = self.NULL
-        self.collect_finalizer_tails = self.NULL
-        self.collect_run_finalizers_head = self.NULL
-        self.collect_run_finalizers_tail = self.NULL
-        self.objects_with_finalizers_to_run = self.NULL
-        #
-        self.weakref_pages = self.NULL
-        self.collect_weakref_pages = self.NULL
-        self.collect_weakref_tails = self.NULL
+        # the linked list of new young objects, and the linked list of
+        # all old objects.  note that the aging objects are not here
+        # but on 'collector.aging_objects'.
+        self.young_objects = self.NULL
+        self.old_objects = self.NULL
         #
         # See concurrentgen.txt for more information about these fields.
         self.current_young_marker = MARK_BYTE_1
         self.current_aging_marker = MARK_BYTE_2
         #
-        # When the mutator thread wants to trigger the next collection,
-        # it scans its own stack roots and prepares everything, then
-        # sets 'collection_running' to 1, and releases
-        # 'ready_to_start_lock'.  This triggers the collector thread,
-        # which re-acquires 'ready_to_start_lock' and does its job.
-        # When done it releases 'finished_lock'.  The mutator thread is
-        # responsible for resetting 'collection_running' to 0.
-        #
-        # The collector thread's state can be found (with careful locking)
-        # by inspecting the same variable from the mutator thread:
-        #   * collection_running == 1: Marking.  [Deletion barrier active.]
-        #   * collection_running == 2: Clearing weakrefs.
-        #   * collection_running == 3: Marking from unreachable finalizers.
-        #   * collection_running == 4: Sweeping.
-        #   * collection_running == -1: Done.
-        # The mutex_lock is acquired to go from 1 to 2, and from 2 to 3.
-        self.collection_running = 0
         #self.ready_to_start_lock = ...built in setup()
         #self.finished_lock = ...built in setup()
+        #self.mutex_lock = ...built in setup()
         #
-        #self.mutex_lock = ...built in setup()
-        self.gray_objects.clear()
-        self.extra_objects_to_mark.clear()
-        self.flagged_objects.clear()
-        self.prebuilt_root_objects.clear()
+        self.collector._initialize()
 
     def setup(self):
         "Start the concurrent collector thread."
@@ -196,26 +120,22 @@
         # 'run_finalizers' as a deque
         self.finalizer_lock_count = 0
         #
-        self.main_thread_ident = ll_thread.get_ident()
         self.ready_to_start_lock = ll_thread.allocate_ll_lock()
         self.finished_lock = ll_thread.allocate_ll_lock()
         self.mutex_lock = ll_thread.allocate_ll_lock()
-        self.addressstack_lock_object.setup()
         #
         self.acquire(self.finished_lock)
         self.acquire(self.ready_to_start_lock)
         #
-        self.collector_ident = ll_thread.c_thread_start_nowrapper(
-            llhelper(ll_thread.CALLBACK, self.collector_start))
-        assert self.collector_ident != -1
+        self.collector.setup()
 
     def _teardown(self):
         "Stop the collector thread after tests have run."
         self.wait_for_the_end_of_collection()
         #
-        # start the next collection, but with collection_running set to 42,
+        # start the next collection, but with collector.running set to 42,
         # which should shut down the collector thread
-        self.collection_running = 42
+        self.collector.running = 42
         debug_print("teardown!")
         self.release(self.ready_to_start_lock)
         self.acquire(self.finished_lock)
@@ -235,7 +155,11 @@
 
     def malloc_fixedsize_clear(self, typeid, size,
                                needs_finalizer=False, contains_weakptr=False):
-        # contains_weakptr: detected during collection
+        #
+        # For now, we always start the next collection as soon as the
+        # previous one is finished
+        if self.collector.running <= 0:
+            self.trigger_next_collection()
         #
         # Case of finalizers (test constant-folded)
         if needs_finalizer:
@@ -250,22 +174,14 @@
         # Regular case
         size_gc_header = self.gcheaderbuilder.size_gc_header
         totalsize = size_gc_header + size
-        rawtotalsize = raw_malloc_usage(totalsize)
-        if rawtotalsize <= self.small_request_threshold:
-            ll_assert(rawtotalsize & (WORD - 1) == 0,
-                      "fixedsize not properly rounded")
-            #
-            n = rawtotalsize >> WORD_POWER_2
-            result = self.free_lists[n]
-            if result != self.NULL:
-                self.free_lists[n] = list_next(result)
-                obj = self.grow_reservation(result, totalsize)
-                hdr = self.header(obj)
-                hdr.tid = self.combine(typeid, self.current_young_marker, 0)
-                #debug_print("malloc_fixedsize_clear", obj)
-                return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
-                #
-        return self._malloc_slowpath(typeid, size)
+        adr = llarena.arena_malloc(llmemory.raw_malloc_usage(totalsize), 2)
+        llarena.arena_reserve(adr, totalsize)
+        obj = adr + size_gc_header
+        hdr = self.header(obj)
+        hdr.tid = self.combine(typeid, self.current_young_marker, 0)
+        hdr.next = self.young_objects
+        self.young_objects = hdr
+        return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
 
     def malloc_varsize_clear(self, typeid, length, size, itemsize,
                              offset_to_length):
@@ -295,9 +211,9 @@
                       "round_up_for_allocation failed")
             #
             n = rawtotalsize >> WORD_POWER_2
-            result = self.free_lists[n]
+            result = self.location_free_lists[n]
             if result != self.NULL:
-                self.free_lists[n] = list_next(result)
+                self.location_free_lists[n] = list_next(result)
                 obj = self.grow_reservation(result, totalsize)
                 hdr = self.header(obj)
                 hdr.tid = self.combine(typeid, self.current_young_marker, 0)
@@ -311,147 +227,6 @@
         # is actually negative.
         return self._malloc_varsize_slowpath(typeid, length)
 
-    def _malloc_slowpath(self, typeid, size):
-        # Slow-path malloc.  Call this with 'size' being a valid and
-        # rounded number, between WORD and up to MAXIMUM_SIZE.
-        #
-        # For now, we always start the next collection immediately.
-        if self.collection_running <= 0:
-            self.trigger_next_collection()
-        #
-        size_gc_header = self.gcheaderbuilder.size_gc_header
-        totalsize = size_gc_header + size
-        rawtotalsize = raw_malloc_usage(totalsize)
-        #
-        if rawtotalsize <= self.small_request_threshold:
-            #
-            # Case 1: unless trigger_next_collection() happened to get us
-            # more locations in free_lists[n], we have run out of them
-            ll_assert(rawtotalsize & (WORD - 1) == 0,
-                      "malloc_slowpath: non-rounded size")
-            n = rawtotalsize >> WORD_POWER_2
-            head = self.free_lists[n]
-            if head:
-                self.free_lists[n] = list_next(head)
-                obj = self.grow_reservation(head, totalsize)
-                hdr = self.header(obj)
-                hdr.tid = self.combine(typeid, self.current_young_marker, 0)
-                return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
-            #
-            # We really have run out of the free list corresponding to
-            # the size.  Grab the next free page.
-            newpage = self.free_pages
-            if newpage == self.NULL:
-                self.allocate_next_arena()
-                newpage = self.free_pages
-            self.free_pages = list_next(newpage)
-            #
-            # Put the free page in the list 'nonfree_pages[n]'.  This is
-            # a linked list chained through the first word of each page.
-            set_next(newpage, self.nonfree_pages[n])
-            self.nonfree_pages[n] = newpage
-            #
-            # Initialize the free page to contain objects of the given
-            # size.  This requires setting up all object locations in the
-            # page, linking them in the free list.
-            i = self.page_size - rawtotalsize
-            limit = rawtotalsize + raw_malloc_usage(self.HDRSIZE)
-            newpageadr = llmemory.cast_ptr_to_adr(newpage)
-            newpageadr = llarena.getfakearenaaddress(newpageadr)
-            while i >= limit:
-                adr = newpageadr + i
-                llarena.arena_reserve(adr, self.HDRSIZE)
-                p = llmemory.cast_adr_to_ptr(adr, self.HDRPTR)
-                set_next(p, head)
-                head = p
-                i -= rawtotalsize
-            self.free_lists[n] = head
-            result = newpageadr + i
-            #
-            # Done: all object locations are linked, apart from
-            # 'result', which is the first object location in the page.
-            # Note that if the size is not an exact divisor of
-            # 4096-WORD, there are a few wasted WORDs, which we place at
-            # the start of the page rather than at the end (Hans Boehm,
-            # xxx ref).
-            #
-            return self._malloc_result(typeid, totalsize, result)
-        else:
-            # Case 2: the object is too large, so allocate it directly
-            # with the system malloc().
-            return self._malloc_large_object(typeid, size, 0)
-        #
-    _malloc_slowpath._dont_inline_ = True
-
-    def _malloc_result(self, typeid, totalsize, result):
-        llarena.arena_reserve(result, totalsize)
-        hdr = llmemory.cast_adr_to_ptr(result, self.HDRPTR)
-        hdr.tid = self.combine(typeid, self.current_young_marker, 0)
-        obj = result + self.gcheaderbuilder.size_gc_header
-        #debug_print("malloc_slowpath", obj)
-        return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
-
-    def _malloc_large_object(self, typeid, size, linked_list):
-        # xxx on 32-bit, we'll prefer 64-bit alignment of the object by
-        # always allocating an 8-bytes header
-        totalsize = self.gcheaderbuilder.size_gc_header + size
-        rawtotalsize = raw_malloc_usage(totalsize)
-        rawtotalsize += 8
-        block = llarena.arena_malloc(rawtotalsize, 2)
-        if not block:
-            raise MemoryError
-        llarena.arena_reserve(block, self.HDRSIZE)
-        blockhdr = llmemory.cast_adr_to_ptr(block, self.HDRPTR)
-        if linked_list == 0:
-            set_next(blockhdr, self.nonfree_pages[0])
-            self.nonfree_pages[0] = blockhdr
-        elif linked_list == 1:
-            set_next(blockhdr, self.finalizer_pages)
-            self.finalizer_pages = blockhdr
-        elif linked_list == 2:
-            set_next(blockhdr, self.weakref_pages)
-            self.weakref_pages = blockhdr
-        else:
-            ll_assert(0, "bad linked_list")
-        return self._malloc_result(typeid, totalsize, block + 8)
-    _malloc_large_object._annspecialcase_ = 'specialize:arg(3)'
-    _malloc_large_object._dont_inline_ = True
-
-    def _malloc_varsize_slowpath(self, typeid, length):
-        #
-        if length < 0:
-            # negative length!  This likely comes from an overflow
-            # earlier.  We will just raise MemoryError here.
-            raise MemoryError
-        #
-        # Compute the total size, carefully checking for overflows.
-        nonvarsize = self.fixed_size(typeid)
-        itemsize = self.varsize_item_sizes(typeid)
-        try:
-            varsize = ovfcheck(itemsize * length)
-            totalsize = ovfcheck(nonvarsize + varsize)
-        except OverflowError:
-            raise MemoryError
-        #
-        # Detect very rare cases of overflows
-        if raw_malloc_usage(totalsize) > MAXIMUM_SIZE:
-            raise MemoryError("rare case of overflow")
-        #
-        totalsize = llarena.round_up_for_allocation(totalsize)
-        result = self._malloc_slowpath(typeid, totalsize)
-        #
-        offset_to_length = self.varsize_offset_to_length(typeid)
-        obj = llmemory.cast_ptr_to_adr(result)
-        (obj + offset_to_length).signed[0] = length
-        return result
-    _malloc_varsize_slowpath._dont_inline_ = True
-
-    def _malloc_with_finalizer(self, typeid, size):
-        return self._malloc_large_object(typeid, size, 1)
-
-    def _malloc_weakref(self, typeid, size):
-        return self._malloc_large_object(typeid, size, 2)
-
     # ----------
     # Other functions in the GC API
 
@@ -545,17 +320,17 @@
                     # it is only possible to reach this point if there is
                     # a collection running in collector_mark(), before it
                     # does mutex_lock itself.  Check this:
-                    ll_assert(self.collection_running == 1,
+                    ll_assert(self.collector.running == 1,
                               "write barrier: wrong call?")
                     #
                     # It's fine to set the mark before tracing, because
                     # we are anyway in a 'mutex_lock' critical section.
                     # The collector thread will not exit from the phase
-                    # 'collection_running == 1' here.
+                    # 'collector.running == 1' here.
                     self.trace(obj, self._barrier_add_extra, None)
                     #
                     # Still at 1:
-                    ll_assert(self.collection_running == 1,
+                    ll_assert(self.collector.running == 1,
                               "write barrier: oups!?")
                     #
                 else:
@@ -582,50 +357,19 @@
     def wait_for_the_end_of_collection(self):
         """In the mutator thread: wait for the minor collection currently
         running (if any) to finish."""
-        if self.collection_running != 0:
+        if self.collector.running != 0:
             debug_start("gc-stop")
             #
             self.acquire(self.finished_lock)
-            self.collection_running = 0
-            #debug_print("collection_running = 0")
+            self.collector.running = 0
+            #debug_print("collector.running = 0")
             #
             # Check invariants
             ll_assert(not self.extra_objects_to_mark.non_empty(),
                       "objs left behind in extra_objects_to_mark")
-            ll_assert(not self.gray_objects.non_empty(),
+            ll_assert(not self.collector.gray_objects.non_empty(),
                       "objs left behind in gray_objects")
             #
-            # Grab the results of the last collection: read the collector's
-            # 'collect_heads/collect_tails' and merge them with the mutator's
-            # 'free_lists'.
-            n = 1
-            while n < self.pagelists_length:
-                self.free_lists[n] = self.join_lists(self.free_lists[n],
-                                                     self.collect_heads[n],
-                                                     self.collect_tails[n])
-                n += 1
-            #
-            # Do the same with 'collect_heads[0]/collect_tails[0]'.
-            self.nonfree_pages[0] = self.join_lists(self.nonfree_pages[0],
-                                                    self.collect_heads[0],
-                                                    self.collect_tails[0])
-            #
-            # Do the same with 'collect_weakref_pages/tails'
-            self.weakref_pages = self.join_lists(self.weakref_pages,
-                                                 self.collect_weakref_pages,
-                                                 self.collect_weakref_tails)
-            #
-            # Do the same with 'collect_finalizer_pages/tails'
-            self.finalizer_pages = self.join_lists(self.finalizer_pages,
-                                                  self.collect_finalizer_pages,
-                                                  self.collect_finalizer_tails)
-            #
-            # Do the same with 'collect_run_finalizers_head/tail'
-            self.objects_with_finalizers_to_run = self.join_lists(
-                self.objects_with_finalizers_to_run,
-                self.collect_run_finalizers_head,
-                self.collect_run_finalizers_tail)
-            #
             if self.DEBUG:
                 self.debug_check_lists()
             #
@@ -635,7 +379,7 @@
             # can start the next collection, and then this function returns
             # with a collection in progress, which it should not.  Be careful
             # to call execute_finalizers_ll() in the caller somewhere.
-            ll_assert(self.collection_running == 0,
+            ll_assert(self.collector.running == 0,
                       "collector thread not paused?")
 
     def join_lists(self, list1, list2head, list2tail):
@@ -649,6 +393,7 @@
 
 
     def execute_finalizers_ll(self):
+        return # XXX
         self.finalizer_lock_count += 1
         try:
             while self.objects_with_finalizers_to_run != self.NULL:
@@ -689,7 +434,7 @@
            The most useful default.
            gen>=4: Do a full synchronous major collection.
         """
-        if gen >= 1 or self.collection_running <= 0:
+        if gen >= 1 or self.collector.running <= 0:
             self.trigger_next_collection(gen >= 3)
             if gen >= 2:
                 self.wait_for_the_end_of_collection()
@@ -710,15 +455,15 @@
             None)                             # static in prebuilt gc
         #
         # Add the objects still waiting in 'objects_with_finalizers_to_run'
-        p = self.objects_with_finalizers_to_run
-        while p != self.NULL:
-            x = llmemory.cast_ptr_to_adr(p)
-            x = llarena.getfakearenaaddress(x) + 8
-            obj = x + self.gcheaderbuilder.size_gc_header
-            #debug_print("_objects_with_finalizers_to_run", obj)
-            self.get_mark(obj)
-            self.gray_objects.append(obj)
-            p = list_next(p)
+        #p = self.objects_with_finalizers_to_run
+        #while p != self.NULL:
+        #    x = llmemory.cast_ptr_to_adr(p)
+        #    x = llarena.getfakearenaaddress(x) + 8
+        #    obj = x + self.gcheaderbuilder.size_gc_header
+        #    #debug_print("_objects_with_finalizers_to_run", obj)
+        #    self.get_mark(obj)
+        #    self.gray_objects.append(obj)
+        #    p = list_next(p)
         #
         # Add all old objects that have been written to since the last
         # time trigger_next_collection was called
@@ -732,27 +477,16 @@
         self.current_young_marker = self.current_aging_marker
         self.current_aging_marker = other
         #
-        # Copy a few 'mutator' fields to 'collector' fields:  the
-        # 'collect_pages[n]' make linked lists of all nonfree pages at the
-        # start of the collection (unlike the 'nonfree_pages' lists, which
-        # the mutator will continue to grow).
-        n = 0
-        while n < self.pagelists_length:
-            self.collect_pages[n] = self.nonfree_pages[n]
-            n += 1
-        self.collect_weakref_pages = self.weakref_pages
-        self.collect_finalizer_pages = self.finalizer_pages
-        #
-        # Clear the following lists.  When the collector thread finishes,
-        # it will give back (in collect_{pages,tails}[0] and
-        # collect_finalizer_{pages,tails}) all the original items that survive.
-        self.nonfree_pages[0] = self.NULL
-        self.weakref_pages = self.NULL
-        self.finalizer_pages = self.NULL
+        # Copy a few 'mutator' fields to 'collector' fields
+        collector = self.collector
+        collector.aging_objects = self.young_objects
+        self.young_objects = self.NULL
+        #self.collect_weakref_pages = self.weakref_pages
+        #self.collect_finalizer_pages = self.finalizer_pages
         #
         # Start the collector thread
-        self.collection_running = 1
-        #debug_print("collection_running = 1")
+        self.collector.running = 1
+        #debug_print("collector.running = 1")
         self.release(self.ready_to_start_lock)
         #
         debug_stop("gc-start")
@@ -763,33 +497,29 @@
         obj = root.address[0]
         #debug_print("_add_stack_root", obj)
         self.get_mark(obj)
-        self.gray_objects.append(obj)
+        self.collector.gray_objects.append(obj)
 
     def _add_prebuilt_root(self, obj, ignored):
+        # NB. it's ok to edit 'gray_objects' from the mutator thread here,
+        # because the collector thread is not running yet
         #debug_print("_add_prebuilt_root", obj)
         self.get_mark(obj)
-        self.gray_objects.append(obj)
+        self.collector.gray_objects.append(obj)
 
     def debug_check_lists(self):
         # just check that they are correct, non-infinite linked lists
-        self.debug_check_list(self.nonfree_pages[0])
-        n = 1
-        while n < self.pagelists_length:
-            self.debug_check_list(self.free_lists[n])
-            n += 1
-        self.debug_check_list(self.weakref_pages)
-        self.debug_check_list(self.finalizer_pages)
-        self.debug_check_list(self.objects_with_finalizers_to_run)
+        self.debug_check_list(self.young_objects)
+        self.debug_check_list(self.old_objects)
 
-    def debug_check_list(self, page):
+    def debug_check_list(self, list):
         try:
-            previous_page = self.NULL
+            previous = self.NULL
             count = 0
-            while page != self.NULL:
+            while list != self.NULL:
                 # prevent constant-folding, and detects loops of length 1
-                ll_assert(page != previous_page, "loop!")
-                previous_page = page
-                page = list_next(page)
+                ll_assert(list != previous, "loop!")
+                previous = list
+                list = list.next
                 count += 1
             return count
         except KeyboardInterrupt:
@@ -797,24 +527,151 @@
             raise
 
     def acquire(self, lock):
-        if (we_are_translated() or
-                ll_thread.get_ident() != self.main_thread_ident):
+        if we_are_translated():
             ll_thread.c_thread_acquirelock(lock, 1)
         else:
+            assert ll_thread.get_ident() == self.main_thread_ident
             while rffi.cast(lltype.Signed,
                             ll_thread.c_thread_acquirelock(lock, 0)) == 0:
                 time.sleep(0.05)
                 # ---------- EXCEPTION FROM THE COLLECTOR THREAD ----------
-                if hasattr(self, '_exc_info'):
+                if hasattr(self.collector, '_exc_info'):
                     self._reraise_from_collector_thread()
 
     def release(self, lock):
         ll_thread.c_thread_releaselock(lock)
 
     def _reraise_from_collector_thread(self):
-        exc, val, tb = self._exc_info
+        exc, val, tb = self.collector._exc_info
         raise exc, val, tb
 
+    def set_mark(self, obj, newmark):
+        _set_mark(self.header(obj), newmark)
+
+    def get_mark(self, obj):
+        mark = self.header(obj).tid & 0xFF
+        ll_assert(mark == MARK_BYTE_1 or
+                  mark == MARK_BYTE_2 or
+                  mark == MARK_BYTE_OLD or
+                  mark == MARK_BYTE_STATIC, "bad mark byte in object")
+        return mark
+
+    # ----------
+    # Weakrefs
+
+    def weakref_deref(self, wrobj):
+        # Weakrefs need some care.  This code acts as a read barrier.
+        # The only way I found is to acquire the mutex_lock to prevent
+        # the collection thread from going from collector.running==1
+        # to collector.running==2, or from collector.running==2 to
+        # collector.running==3.
+        #
+        self.acquire(self.mutex_lock)
+        #
+        targetobj = gctypelayout.ll_weakref_deref(wrobj)
+        if targetobj != llmemory.NULL:
+            #
+            if self.collector.running == 1:
+                # If we are in the phase collector.running==1, we don't
+                # know if the object will be scanned a bit later or
+                # not; so we have to assume that it survives, and
+                # force it to be scanned.
+                self.get_mark(targetobj)
+                self.extra_objects_to_mark.append(targetobj)
+                #
+            elif self.collector.running == 2:
+                # In the phase collector.running==2, if the object is
+                # not marked it's too late; we have to detect that case
+                # and return NULL instead here, as if the corresponding
+                # collector phase was already finished (deal_with_weakrefs).
+                # Otherwise we would be returning an object that is about to
+                # be swept away.
+                if not self.is_marked_or_static(targetobj, self.current_mark):
+                    targetobj = llmemory.NULL
+                #
+            else:
+                # In other phases we are fine.
+                pass
+        #
+        self.release(self.mutex_lock)
+        #
+        return targetobj
+
+
+# ____________________________________________________________
+#
+# The collector thread is put on another class, in order to separate
+# it more cleanly (both from a code organization point of view and
+# from the point of view of cache locality).
+
+
+class CollectorThread(object):
+    _alloc_flavor_ = "raw"
+
+    NULL = ConcurrentGenGC.NULL
+
+    def __init__(self, gc):
+        self.gc = gc
+        #
+        # When the mutator thread wants to trigger the next collection,
+        # it scans its own stack roots and prepares everything, then
+        # sets 'collector.running' to 1, and releases
+        # 'ready_to_start_lock'.  This triggers the collector thread,
+        # which re-acquires 'ready_to_start_lock' and does its job.
+        # When done it releases 'finished_lock'.  The mutator thread is
+        # responsible for resetting 'collector.running' to 0.
+        #
+        # The collector thread's state can be found (with careful locking)
+        # by inspecting the same variable from the mutator thread:
+        #   * collector.running == 1: Marking.  [Deletion barrier active.]
+        #   * collector.running == 2: Clearing weakrefs.
+        #   * collector.running == 3: Marking from unreachable finalizers.
+        #   * collector.running == 4: Sweeping.
+        #   * collector.running == -1: Done.
+        # The mutex_lock is acquired to go from 1 to 2, and from 2 to 3.
+        self.running = 0
+        #
+        # a different AddressStack class, which uses a different pool
+        # of free pages than the regular one, so can run concurrently
+        self.CollectorAddressStack = get_address_stack(lock="collector")
+        #
+        # when the collection starts, we make all young objects aging and
+        # move 'young_objects' into 'aging_objects'
+        self.aging_objects = self.NULL
+        #
+        # The start function for the thread, as a function and not a method
+        def collector_start():
+            if we_are_translated():
+                self.collector_run()
+            else:
+                self.collector_run_nontranslated()
+        collector_start._should_never_raise_ = True
+        self.collector_start = collector_start
+
+    def _initialize(self):
+        self.gray_objects = self.CollectorAddressStack()
+
+    def setup(self):
+        self.ready_to_start_lock = self.gc.ready_to_start_lock
+        self.finished_lock = self.gc.finished_lock
+        self.mutex_lock = self.gc.mutex_lock
+        #
+        # start the thread
+        self.collector_ident = ll_thread.c_thread_start_nowrapper(
+            llhelper(ll_thread.CALLBACK, self.collector_start))
+        assert self.collector_ident != -1
+
+    def acquire(self, lock):
+        ll_thread.c_thread_acquirelock(lock, 1)
+
+    def release(self, lock):
+        ll_thread.c_thread_releaselock(lock)
+
+    def get_mark(self, obj):
+        return self.gc.get_mark(obj)
+
+    def set_mark(self, obj, newmark):
+        self.gc.set_mark(obj, newmark)
 
     def collector_run_nontranslated(self):
         try:
@@ -847,7 +704,7 @@
             self.acquire(self.ready_to_start_lock)
             #
             # For tests: detect when we have to shut down
-            if self.collection_running == 42:
+            if self.running == 42:
                 self.release(self.finished_lock)
                 break
             #
@@ -863,17 +720,6 @@
             self.release(self.finished_lock)
 
 
-    def set_mark(self, obj, newmark):
-        _set_mark(self.header(obj), newmark)
-
-    def get_mark(self, obj):
-        mark = self.header(obj).tid & 0xFF
-        ll_assert(mark == MARK_BYTE_1 or
-                  mark == MARK_BYTE_2 or
-                  mark == MARK_BYTE_OLD or
-                  mark == MARK_BYTE_STATIC, "bad mark byte in object")
-        return mark
-
     def collector_mark(self):
         while True:
             #
@@ -888,8 +734,9 @@
             # unless XXX we've hit the write barrier of a large array
             self.acquire(self.mutex_lock)
             #debug_print("...collector thread has mutex_lock")
-            while self.extra_objects_to_mark.non_empty():
-                obj = self.extra_objects_to_mark.pop()
+            extra_objects_to_mark = self.gc.extra_objects_to_mark
+            while extra_objects_to_mark.non_empty():
+                obj = extra_objects_to_mark.pop()
                 self.get_mark(obj)
                 self.gray_objects.append(obj)
             #
@@ -905,185 +752,89 @@
             # Else release mutex_lock and try again.
             self.release(self.mutex_lock)
         #
-        self.collection_running = 2
+        self.running = 2
         #debug_print("collection_running = 2")
         self.release(self.mutex_lock)
 
     def _collect_mark(self):
-        cam = self.current_aging_marker
+        extra_objects_to_mark = self.gc.extra_objects_to_mark
+        cam = self.gc.current_aging_marker
         while self.gray_objects.non_empty():
             obj = self.gray_objects.pop()
-            if self.get_mark(obj) == cam:
-                #
-                # Scan the content of 'obj'.  We use a snapshot-at-the-
-                # beginning order, meaning that we want to scan the state
-                # of the object as it was at the beginning of the current
-                # collection --- and not the current state, which might have
-                # been modified.  That's why we have a deletion barrier:
-                # when the mutator thread is about to change an object that
-                # is not yet marked, it will itself do the scanning of just
-                # this object, and mark the object.  But this function is not
-                # synchronized, which means that in some rare cases it's
-                # possible that the object is scanned a second time here
-                # (harmlessly).
-                #
-                # The order of the next two lines is essential!  *First*
-                # scan the object, adding all objects found to gray_objects;
-                # and *only then* set the mark.  This is essential, because
-                # otherwise, we might set the mark, then the main thread
-                # thinks a force_scan() is not necessary and modifies the
-                # content of 'obj', and then here in the collector thread
-                # we scan a modified content --- and the original content
-                # is never scanned.
-                #
-                self.trace(obj, self._collect_add_pending, None)
-                self.set_mark(obj, MARK_BYTE_OLD)
-                #
-                # Interrupt early if the mutator's write barrier adds stuff
-                # to that list.  Note that the check is imprecise because
-                # it is not lock-protected, but that's good enough.  The
-                # idea is that we trace in priority objects flagged with
-                # the write barrier, because they are more likely to
-                # reference further objects that will soon be accessed too.
-                if self.extra_objects_to_mark.non_empty():
-                    return
+            if self.get_mark(obj) != cam:
+                continue
+            #
+            # Scan the content of 'obj'.  We use a snapshot-at-the-
+            # beginning order, meaning that we want to scan the state
+            # of the object as it was at the beginning of the current
+            # collection --- and not the current state, which might have
+            # been modified.  That's why we have a deletion barrier:
+            # when the mutator thread is about to change an object that
+            # is not yet marked, it will itself do the scanning of just
+            # this object, and mark the object.  But this function is not
+            # synchronized, which means that in some rare cases it's
+            # possible that the object is scanned a second time here
+            # (harmlessly).
+            #
+            # The order of the next two lines is essential!  *First*
+            # scan the object, adding all objects found to gray_objects;
+            # and *only then* set the mark.  This is essential, because
+            # otherwise, we might set the mark, then the main thread
+            # thinks a force_scan() is not necessary and modifies the
+            # content of 'obj', and then here in the collector thread
+            # we scan a modified content --- and the original content
+            # is never scanned.
+            #
+            self.gc.trace(obj, self._collect_add_pending, None)
+            self.set_mark(obj, MARK_BYTE_OLD)
+            #
+            # Interrupt early if the mutator's write barrier adds stuff
+            # to that list.  Note that the check is imprecise because
+            # it is not lock-protected, but that's good enough.  The
+            # idea is that we trace in priority objects flagged with
+            # the write barrier, because they are more likely to
+            # reference further objects that will soon be accessed too.
+            if extra_objects_to_mark.non_empty():
+                return
 
     def _collect_add_pending(self, root, ignored):
         obj = root.address[0]
+        # these 'get_mark(obj) are here for debugging invalid marks.
+        # XXX check that the C compiler removes them if lldebug is off
         self.get_mark(obj)
         self.gray_objects.append(obj)
 
     def collector_sweep(self):
-        self._collect_sweep_large_objects()
-        #
-        n = 1
-        while n < self.pagelists_length:
-            self._collect_sweep_pages(n)
-            n += 1
-        #
-        self.collection_running = -1
-        #debug_print("collection_running = -1")
-
-    def _collect_sweep_large_objects(self):
-        block = self.collect_pages[0]
-        cam = self.current_aging_marker
-        linked_list = self.NULL
-        first_block_in_linked_list = self.NULL
-        while block != self.NULL:
-            nextblock = list_next(block)
-            blockadr = llmemory.cast_ptr_to_adr(block)
-            blockadr = llarena.getfakearenaaddress(blockadr)
-            hdr = llmemory.cast_adr_to_ptr(blockadr + 8, self.HDRPTR)
+        cam = self.gc.current_aging_marker
+        hdr = self.aging_objects
+        linked_list = self.gc.old_objects
+        while hdr != self.NULL:
+            nexthdr = hdr.next
             mark = hdr.tid & 0xFF
             if mark == cam:
                 # the object is still not marked.  Free it.
+                blockadr = llmemory.cast_ptr_to_adr(hdr)
                 llarena.arena_free(blockadr)
                 #
             else:
                 # the object was marked: relink it
                 ll_assert(mark == MARK_BYTE_OLD,
                           "bad mark in large object")
-                set_next(block, linked_list)
-                linked_list = block
-                if first_block_in_linked_list == self.NULL:
-                    first_block_in_linked_list = block
-            block = nextblock
+                hdr.next = linked_list
+                linked_list = hdr
+                #
+            hdr = nexthdr
         #
-        self.collect_heads[0] = linked_list
-        self.collect_tails[0] = first_block_in_linked_list
+        self.gc.old_objects = linked_list
+        #
+        self.running = -1
+        #debug_print("collection_running = -1")
 
-    def _collect_sweep_pages(self, n):
-        # sweep all pages from the linked list starting at 'page',
-        # containing objects of fixed size 'object_size'.
-        size_gc_header = self.gcheaderbuilder.size_gc_header
-        page = self.collect_pages[n]
-        object_size = n << WORD_POWER_2
-        linked_list = self.NULL
-        first_loc_in_linked_list = self.NULL
-        cam = self.current_aging_marker
-        while page != self.NULL:
-            i = self.page_size - object_size
-            limit = raw_malloc_usage(self.HDRSIZE)
-            pageadr = llmemory.cast_ptr_to_adr(page)
-            pageadr = llarena.getfakearenaaddress(pageadr)
-            while i >= limit:
-                adr = pageadr + i
-                hdr = llmemory.cast_adr_to_ptr(adr, self.HDRPTR)
-                mark = hdr.tid & 0xFF
-                #
-                if mark == cam:
-                    # the location contains really an object (and is not just
-                    # part of a linked list of free locations), and moreover
-                    # the object is still not marked.  Free it by inserting
-                    # it into the linked list.
-                    #debug_print("sweeps", adr + size_gc_header)
-                    llarena.arena_reset(adr, object_size, 0)
-                    llarena.arena_reserve(adr, self.HDRSIZE)
-                    hdr = llmemory.cast_adr_to_ptr(adr, self.HDRPTR)
-                    set_next(hdr, linked_list)
-                    linked_list = hdr
-                    if first_loc_in_linked_list == self.NULL:
-                        first_loc_in_linked_list = hdr
-                    # XXX detect when the whole page is freed again
-                    #
-                    # Clear the data, in prevision for the following
-                    # malloc_fixedsize_clear().
-                    size_of_int = raw_malloc_usage(
-                        llmemory.sizeof(lltype.Signed))
-                    llarena.arena_reset(adr + size_of_int,
-                                        object_size - size_of_int, 2)
-                #
-                i -= object_size
-            #
-            page = list_next(page)
-        #
-        self.collect_heads[n] = linked_list
-        self.collect_tails[n] = first_loc_in_linked_list
-
-
-    # ----------
-    # Weakrefs
-
-    def weakref_deref(self, wrobj):
-        # Weakrefs need some care.  This code acts as a read barrier.
-        # The only way I found is to acquire the mutex_lock to prevent
-        # the collection thread from going from collection_running==1
-        # to collection_running==2, or from collection_running==2 to
-        # collection_running==3.
-        #
-        self.acquire(self.mutex_lock)
-        #
-        targetobj = gctypelayout.ll_weakref_deref(wrobj)
-        if targetobj != llmemory.NULL:
-            #
-            if self.collection_running == 1:
-                # If we are in the phase collection_running==1, we don't
-                # know if the object will be scanned a bit later or
-                # not; so we have to assume that it survives, and
-                # force it to be scanned.
-                self.get_mark(targetobj)
-                self.extra_objects_to_mark.append(targetobj)
-                #
-            elif self.collection_running == 2:
-                # In the phase collection_running==2, if the object is
-                # not marked it's too late; we have to detect that case
-                # and return NULL instead here, as if the corresponding
-                # collector phase was already finished (deal_with_weakrefs).
-                # Otherwise we would be returning an object that is about to
-                # be swept away.
-                if not self.is_marked_or_static(targetobj, self.current_mark):
-                    targetobj = llmemory.NULL
-                #
-            else:
-                # In other phases we are fine.
-                pass
-        #
-        self.release(self.mutex_lock)
-        #
-        return targetobj
+    # -------------------------
+    # CollectorThread: Weakrefs
 
     def deal_with_weakrefs(self):
-        self.collection_running = 3; return
+        self.running = 3; return
         # ^XXX^
         size_gc_header = self.gcheaderbuilder.size_gc_header
         current_mark = self.current_mark
@@ -1118,16 +869,15 @@
             weakref_page = next_page
         #
         self.acquire(self.mutex_lock)
-        self.collection_running = 3
-        #debug_print("collection_running = 3")
+        self.collector.running = 3
+        #debug_print("collector.running = 3")
         self.release(self.mutex_lock)
 
-
-    # ----------
-    # Finalizers
+    # ---------------------------
+    # CollectorThread: Finalizers
 
     def deal_with_objects_with_finalizers(self):
-        self.collection_running = 4; return
+        self.running = 4; return
         # ^XXX^
         
         # XXX needs to be done correctly; for now we'll call finalizers
@@ -1170,25 +920,12 @@
                   "should not see objects only reachable from finalizers "
                   "before we run them")
         #
-        self.collection_running = 4
+        self.collector.running = 4
         #debug_print("collection_running = 4")
 
 
 # ____________________________________________________________
 #
-# Support for linked lists (used here because AddressStack is not thread-safe)
-
-def list_next(hdr):
-    return llmemory.cast_adr_to_ptr(llmemory.cast_int_to_adr(hdr.tid),
-                                    ConcurrentGenGC.HDRPTR)
-
-def set_next(hdr, nexthdr):
-    hdr.tid = llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(nexthdr),
-                                       "symbolic")
-
-
-# ____________________________________________________________
-#
 # Hack to write the 'mark' or the 'flags' bytes of an object header
 # without overwriting the whole word.  Essential in the rare case where
 # the other thread might be concurrently writing the other byte.
@@ -1232,19 +969,3 @@
                            [ConcurrentGenGC.HDRPTR, lltype.Signed],
                            lltype.Void, compilation_info=eci, _nowrapper=True,
                            _callable=emulate_set_flags)
-
-# ____________________________________________________________
-#
-# A lock to synchronize access to AddressStack's free pages
-
-class SyncLock:
-    _alloc_flavor_ = "raw"
-    _lock = lltype.nullptr(ll_thread.TLOCKP.TO)
-    def setup(self):
-        self._lock = ll_thread.allocate_ll_lock()
-    def acquire(self):
-        if self._lock:
-            ll_thread.c_thread_acquirelock(self._lock, 1)
-    def release(self):
-        if self._lock:
-            ll_thread.c_thread_releaselock(self._lock)
diff --git a/pypy/rpython/memory/support.py b/pypy/rpython/memory/support.py
--- a/pypy/rpython/memory/support.py
+++ b/pypy/rpython/memory/support.py
@@ -69,7 +69,7 @@
                 lltype.free(chunk, flavor="raw", track_allocation=False)
             self._unlock()
 
-        if lock is not None:
+        if lock is not None and not isinstance(lock, str):
             def _lock(self):   lock.acquire()
             def _unlock(self): lock.release()
         else:

From noreply at buildbot.pypy.org  Sun Oct 23 11:53:02 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 23 Oct 2011 11:53:02 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Varsized.
Message-ID: <20111023095302.2C84C820C7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r48356:cba48e6cce2e
Date: 2011-10-23 11:51 +0200
http://bitbucket.org/pypy/pypy/changeset/cba48e6cce2e/

Log:	Varsized.

diff --git a/pypy/rpython/memory/gc/concurrentgen.py b/pypy/rpython/memory/gc/concurrentgen.py
--- a/pypy/rpython/memory/gc/concurrentgen.py
+++ b/pypy/rpython/memory/gc/concurrentgen.py
@@ -175,6 +175,8 @@
         size_gc_header = self.gcheaderbuilder.size_gc_header
         totalsize = size_gc_header + size
         adr = llarena.arena_malloc(llmemory.raw_malloc_usage(totalsize), 2)
+        if adr == llmemory.NULL:
+            raise MemoryError
         llarena.arena_reserve(adr, totalsize)
         obj = adr + size_gc_header
         hdr = self.header(obj)
@@ -185,47 +187,33 @@
 
     def malloc_varsize_clear(self, typeid, length, size, itemsize,
                              offset_to_length):
+        #
+        # For now, we always start the next collection as soon as the
+        # previous one is finished
+        if self.collector.running <= 0:
+            self.trigger_next_collection()
+        #
         size_gc_header = self.gcheaderbuilder.size_gc_header
         nonvarsize = size_gc_header + size
         #
-        # Compute the maximal length that makes the object still below
-        # 'small_request_threshold'.  All the following logic is usually
-        # constant-folded because size and itemsize are constants (due
-        # to inlining).
-        maxsize = self.small_request_threshold - raw_malloc_usage(nonvarsize)
-        if maxsize < 0:
-            toobig = r_uint(0)    # the nonvarsize alone is too big
-        elif raw_malloc_usage(itemsize):
-            toobig = r_uint(maxsize // raw_malloc_usage(itemsize)) + 1
-        else:
-            toobig = r_uint(sys.maxint) + 1
-
-        if r_uint(length) < r_uint(toobig):
-            # With the above checks we know now that totalsize cannot be more
-            # than 'small_request_threshold'; in particular, the + and *
-            # cannot overflow.
-            totalsize = nonvarsize + itemsize * length
-            totalsize = llarena.round_up_for_allocation(totalsize)
-            rawtotalsize = raw_malloc_usage(totalsize)
-            ll_assert(rawtotalsize & (WORD - 1) == 0,
-                      "round_up_for_allocation failed")
-            #
-            n = rawtotalsize >> WORD_POWER_2
-            result = self.location_free_lists[n]
-            if result != self.NULL:
-                self.location_free_lists[n] = list_next(result)
-                obj = self.grow_reservation(result, totalsize)
-                hdr = self.header(obj)
-                hdr.tid = self.combine(typeid, self.current_young_marker, 0)
-                (obj + offset_to_length).signed[0] = length
-                #debug_print("malloc_varsize_clear", obj)
-                return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
+        if length < 0:
+            raise MemoryError
+        try:
+            totalsize = ovfcheck(nonvarsize + ovfcheck(itemsize * length))
+        except OverflowError:
+            raise MemoryError
         #
-        # If the total size of the object would be larger than
-        # 'small_request_threshold', or if the free_list is empty,
-        # then allocate it externally.  We also go there if 'length'
-        # is actually negative.
-        return self._malloc_varsize_slowpath(typeid, length)
+        adr = llarena.arena_malloc(llmemory.raw_malloc_usage(totalsize), 2)
+        if adr == llmemory.NULL:
+            raise MemoryError
+        llarena.arena_reserve(adr, totalsize)
+        obj = adr + size_gc_header
+        (obj + offset_to_length).signed[0] = length
+        hdr = self.header(obj)
+        hdr.tid = self.combine(typeid, self.current_young_marker, 0)
+        hdr.next = self.young_objects
+        self.young_objects = hdr
+        return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
 
     # ----------
     # Other functions in the GC API
@@ -814,12 +802,13 @@
             if mark == cam:
                 # the object is still not marked.  Free it.
                 blockadr = llmemory.cast_ptr_to_adr(hdr)
+                blockadr = llarena.getfakearenaaddress(blockadr)
                 llarena.arena_free(blockadr)
                 #
             else:
                 # the object was marked: relink it
-                ll_assert(mark == MARK_BYTE_OLD,
-                          "bad mark in large object")
+                ll_assert(mark == self.gc.current_young_marker or
+                          mark == MARK_BYTE_OLD, "sweep: bad mark")
                 hdr.next = linked_list
                 linked_list = hdr
                 #

From noreply at buildbot.pypy.org  Sun Oct 23 12:46:03 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 23 Oct 2011 12:46:03 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Bug fixes.
Message-ID: <20111023104603.7D8A2820C7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r48357:ca2fc56c81dd
Date: 2011-10-23 12:44 +0200
http://bitbucket.org/pypy/pypy/changeset/ca2fc56c81dd/

Log:	Bug fixes.

diff --git a/pypy/rpython/memory/gc/base.py b/pypy/rpython/memory/gc/base.py
--- a/pypy/rpython/memory/gc/base.py
+++ b/pypy/rpython/memory/gc/base.py
@@ -100,9 +100,6 @@
     def set_root_walker(self, root_walker):
         self.root_walker = root_walker
 
-    def write_barrier(self, newvalue, addr_struct):
-        pass
-
     def statistics(self, index):
         return -1
 
diff --git a/pypy/rpython/memory/gc/concurrentgen.py b/pypy/rpython/memory/gc/concurrentgen.py
--- a/pypy/rpython/memory/gc/concurrentgen.py
+++ b/pypy/rpython/memory/gc/concurrentgen.py
@@ -98,15 +98,17 @@
         self.flagged_objects = self.AddressStack()
         self.prebuilt_root_objects = self.AddressStack()
         #
-        # the linked list of new young objects, and the linked list of
-        # all old objects.  note that the aging objects are not here
-        # but on 'collector.aging_objects'.
-        self.young_objects = self.NULL
+        # The linked list of new young objects, and the linked list of
+        # all old objects.  Note that the aging objects are not here
+        # but on 'collector.aging_objects'.  Note also that 'old_objects'
+        # contains the objects that the write barrier re-marked as young
+        # (so they are "old young objects").
+        self.new_young_objects = self.NULL
         self.old_objects = self.NULL
         #
         # See concurrentgen.txt for more information about these fields.
         self.current_young_marker = MARK_BYTE_1
-        self.current_aging_marker = MARK_BYTE_2
+        self.collector.current_aging_marker = MARK_BYTE_2
         #
         #self.ready_to_start_lock = ...built in setup()
         #self.finished_lock = ...built in setup()
@@ -181,8 +183,8 @@
         obj = adr + size_gc_header
         hdr = self.header(obj)
         hdr.tid = self.combine(typeid, self.current_young_marker, 0)
-        hdr.next = self.young_objects
-        self.young_objects = hdr
+        hdr.next = self.new_young_objects
+        self.new_young_objects = hdr
         return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
 
     def malloc_varsize_clear(self, typeid, length, size, itemsize,
@@ -211,8 +213,8 @@
         (obj + offset_to_length).signed[0] = length
         hdr = self.header(obj)
         hdr.tid = self.combine(typeid, self.current_young_marker, 0)
-        hdr.next = self.young_objects
-        self.young_objects = hdr
+        hdr.next = self.new_young_objects
+        self.new_young_objects = hdr
         return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
 
     # ----------
@@ -270,8 +272,6 @@
         mark = self.header(addr_struct).tid & 0xFF
         if mark != self.current_young_marker:
             self.force_scan(addr_struct)
-        #else:
-        #    debug_print("deletion_barrier (off)", addr_struct)
 
     def assume_young_pointers(self, addr_struct):
         pass # XXX
@@ -279,9 +279,9 @@
     def _init_writebarrier_logic(self):
         #
         def force_scan(obj):
-            #debug_print("deletion_barrier  ON  ", obj)
             cym = self.current_young_marker
             mark = self.get_mark(obj)
+            #debug_print("deletion_barrier:", mark, obj)
             #
             if mark == MARK_BYTE_OLD:
                 #
@@ -303,7 +303,7 @@
                 mark = self.get_mark(obj)
                 self.set_mark(obj, cym)
                 #
-                if mark == self.current_aging_marker:
+                if mark == self.collector.current_aging_marker:
                     #
                     # it is only possible to reach this point if there is
                     # a collection running in collector_mark(), before it
@@ -422,11 +422,14 @@
            The most useful default.
            gen>=4: Do a full synchronous major collection.
         """
+        debug_start("gc-forced-collect")
+        debug_print("collect, gen =", gen)
         if gen >= 1 or self.collector.running <= 0:
             self.trigger_next_collection(gen >= 3)
             if gen >= 2:
                 self.wait_for_the_end_of_collection()
         self.execute_finalizers_ll()
+        debug_stop("gc-forced-collect")
 
     def trigger_next_collection(self, force_major_collection=False):
         """In the mutator thread: triggers the next minor collection."""
@@ -462,13 +465,13 @@
         #
         # Exchange the meanings of 'cym' and 'cam'
         other = self.current_young_marker
-        self.current_young_marker = self.current_aging_marker
-        self.current_aging_marker = other
+        self.current_young_marker = self.collector.current_aging_marker
+        self.collector.current_aging_marker = other
         #
         # Copy a few 'mutator' fields to 'collector' fields
         collector = self.collector
-        collector.aging_objects = self.young_objects
-        self.young_objects = self.NULL
+        collector.aging_objects = self.new_young_objects
+        self.new_young_objects = self.NULL
         #self.collect_weakref_pages = self.weakref_pages
         #self.collect_finalizer_pages = self.finalizer_pages
         #
@@ -496,7 +499,7 @@
 
     def debug_check_lists(self):
         # just check that they are correct, non-infinite linked lists
-        self.debug_check_list(self.young_objects)
+        self.debug_check_list(self.new_young_objects)
         self.debug_check_list(self.old_objects)
 
     def debug_check_list(self, list):
@@ -601,6 +604,22 @@
     def __init__(self, gc):
         self.gc = gc
         #
+        # a different AddressStack class, which uses a different pool
+        # of free pages than the regular one, so can run concurrently
+        self.CollectorAddressStack = get_address_stack(lock="collector")
+        #
+        # The start function for the thread, as a function and not a method
+        def collector_start():
+            if we_are_translated():
+                self.collector_run()
+            else:
+                self.collector_run_nontranslated()
+        collector_start._should_never_raise_ = True
+        self.collector_start = collector_start
+
+    def _initialize(self):
+        self.gray_objects = self.CollectorAddressStack()
+        #
         # When the mutator thread wants to trigger the next collection,
         # it scans its own stack roots and prepares everything, then
         # sets 'collector.running' to 1, and releases
@@ -619,25 +638,9 @@
         # The mutex_lock is acquired to go from 1 to 2, and from 2 to 3.
         self.running = 0
         #
-        # a different AddressStack class, which uses a different pool
-        # of free pages than the regular one, so can run concurrently
-        self.CollectorAddressStack = get_address_stack(lock="collector")
-        #
         # when the collection starts, we make all young objects aging and
-        # move 'young_objects' into 'aging_objects'
+        # move 'new_young_objects' into 'aging_objects'
         self.aging_objects = self.NULL
-        #
-        # The start function for the thread, as a function and not a method
-        def collector_start():
-            if we_are_translated():
-                self.collector_run()
-            else:
-                self.collector_run_nontranslated()
-        collector_start._should_never_raise_ = True
-        self.collector_start = collector_start
-
-    def _initialize(self):
-        self.gray_objects = self.CollectorAddressStack()
 
     def setup(self):
         self.ready_to_start_lock = self.gc.ready_to_start_lock
@@ -746,7 +749,7 @@
 
     def _collect_mark(self):
         extra_objects_to_mark = self.gc.extra_objects_to_mark
-        cam = self.gc.current_aging_marker
+        cam = self.current_aging_marker
         while self.gray_objects.non_empty():
             obj = self.gray_objects.pop()
             if self.get_mark(obj) != cam:
@@ -793,7 +796,7 @@
         self.gray_objects.append(obj)
 
     def collector_sweep(self):
-        cam = self.gc.current_aging_marker
+        cam = self.current_aging_marker
         hdr = self.aging_objects
         linked_list = self.gc.old_objects
         while hdr != self.NULL:
diff --git a/pypy/rpython/memory/gcwrapper.py b/pypy/rpython/memory/gcwrapper.py
--- a/pypy/rpython/memory/gcwrapper.py
+++ b/pypy/rpython/memory/gcwrapper.py
@@ -107,9 +107,13 @@
                         break
             #
             if wb:
-                self.gc.write_barrier(
-                    llmemory.cast_ptr_to_adr(newvalue),
-                    llmemory.cast_ptr_to_adr(toplevelcontainer))
+                if self.gc.needs_write_barrier:
+                    self.gc.write_barrier(
+                        llmemory.cast_ptr_to_adr(newvalue),
+                        llmemory.cast_ptr_to_adr(toplevelcontainer))
+                elif self.gc.needs_deletion_barrier:
+                    self.gc.deletion_barrier(
+                        llmemory.cast_ptr_to_adr(toplevelcontainer))
         llheap.setinterior(toplevelcontainer, inneraddr, INNERTYPE, newvalue)
 
     def collect(self, *gen):
diff --git a/pypy/rpython/memory/support.py b/pypy/rpython/memory/support.py
--- a/pypy/rpython/memory/support.py
+++ b/pypy/rpython/memory/support.py
@@ -46,6 +46,8 @@
                 else:
                     zero = 2
                 size = llmemory.raw_malloc_usage(llmemory.sizeof(CHUNK))
+                # use arena_malloc to directly call the system 'malloc',
+                # so no locking issue in case of concurrent usage
                 addr = llarena.arena_malloc(size, zero)
                 if not addr:
                     fatalerror("out of memory in GC support code")
diff --git a/pypy/rpython/memory/test/test_gc.py b/pypy/rpython/memory/test/test_gc.py
--- a/pypy/rpython/memory/test/test_gc.py
+++ b/pypy/rpython/memory/test/test_gc.py
@@ -937,3 +937,6 @@
 
     def test_weakref_to_object_with_finalizer_ordering(self):
         py.test.skip("frees weakrefs before calling finalizers")
+
+class TestConcurrentGenGC(GCTest):
+    from pypy.rpython.memory.gc.concurrentgen import ConcurrentGenGC as GCClass
diff --git a/pypy/rpython/memory/test/test_transformed_gc.py b/pypy/rpython/memory/test/test_transformed_gc.py
--- a/pypy/rpython/memory/test/test_transformed_gc.py
+++ b/pypy/rpython/memory/test/test_transformed_gc.py
@@ -1437,3 +1437,12 @@
             GC_PARAMS = {'page_size': 128*WORD,
                          'translated_to_c': False}
             root_stack_depth = 200
+
+class TestConcurrentGenGC(GCTest):
+    gcname = "concurrentgen"
+    class gcpolicy(gc.FrameworkGcPolicy):
+        class transformerclass(framework.FrameworkGCTransformer):
+            from pypy.rpython.memory.gc.concurrentgen \
+                    import ConcurrentGenGC as GCClass
+            GC_PARAMS = {'translated_to_c': False}
+            root_stack_depth = 200

From noreply at buildbot.pypy.org  Sun Oct 23 13:22:31 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 23 Oct 2011 13:22:31 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Fixes.
Message-ID: <20111023112231.CFD54820C7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r48358:6c61df9c6d64
Date: 2011-10-23 13:19 +0200
http://bitbucket.org/pypy/pypy/changeset/6c61df9c6d64/

Log:	Fixes.

diff --git a/pypy/config/translationoption.py b/pypy/config/translationoption.py
--- a/pypy/config/translationoption.py
+++ b/pypy/config/translationoption.py
@@ -59,7 +59,7 @@
     ChoiceOption("gc", "Garbage Collection Strategy",
                  ["boehm", "ref", "marksweep", "semispace", "statistics",
                   "generation", "hybrid", "markcompact", "minimark",
-                  "concurrentms", "none"],
+                  "concurrentms", "concurrentgen", "none"],
                   "ref", requires={
                      "ref": [("translation.rweakref", False), # XXX
                              ("translation.gctransformer", "ref")],
@@ -75,6 +75,7 @@
                      "markcompact": [("translation.gctransformer", "framework")],
                      "minimark": [("translation.gctransformer", "framework")],
                      "concurrentms": [("translation.gctransformer", "framework")],
+                     "concurrentgen": [("translation.gctransformer", "framework")],
                      },
                   cmdline="--gc"),
     ChoiceOption("gctransformer", "GC transformer that is used - internal",
diff --git a/pypy/rpython/memory/gc/base.py b/pypy/rpython/memory/gc/base.py
--- a/pypy/rpython/memory/gc/base.py
+++ b/pypy/rpython/memory/gc/base.py
@@ -439,6 +439,7 @@
                "markcompact" : "markcompact.MarkCompactGC",
                "minimark" : "minimark.MiniMarkGC",
                "concurrentms": "concurrentms.MostlyConcurrentMarkSweepGC",
+               "concurrentgen": "concurrentgen.ConcurrentGenGC",
                }
     try:
         modulename, classname = classes[config.translation.gc].split('.')
diff --git a/pypy/rpython/memory/gc/concurrentgen.py b/pypy/rpython/memory/gc/concurrentgen.py
--- a/pypy/rpython/memory/gc/concurrentgen.py
+++ b/pypy/rpython/memory/gc/concurrentgen.py
@@ -78,6 +78,10 @@
         GCBase.__init__(self, config, **kwds)
         self.main_thread_ident = ll_thread.get_ident()
         #
+        self.extra_objects_to_mark = self.AddressStack()
+        self.flagged_objects = self.AddressStack()
+        self.prebuilt_root_objects = self.AddressStack()
+        #
         # Create the CollectorThread object
         self.collector = CollectorThread(self)
         #
@@ -94,9 +98,9 @@
         # During test_transformed_gc, it is translated, so that we can
         # quickly reset the GC between tests.
         #
-        self.extra_objects_to_mark = self.AddressStack()
-        self.flagged_objects = self.AddressStack()
-        self.prebuilt_root_objects = self.AddressStack()
+        self.extra_objects_to_mark.clear()
+        self.flagged_objects.clear()
+        self.prebuilt_root_objects.clear()
         #
         # The linked list of new young objects, and the linked list of
         # all old objects.  Note that the aging objects are not here
@@ -165,12 +169,14 @@
         #
         # Case of finalizers (test constant-folded)
         if needs_finalizer:
+            raise NotImplementedError
             ll_assert(not contains_weakptr,
                      "'needs_finalizer' and 'contains_weakptr' both specified")
             return self._malloc_with_finalizer(typeid, size)
         #
         # Case of weakreferences (test constant-folded)
         if contains_weakptr:
+            raise NotImplementedError
             return self._malloc_weakref(typeid, size)
         #
         # Regular case
@@ -246,27 +252,6 @@
 
     # ----------
 
-    def allocate_next_arena(self):
-        # xxx for now, allocate one page at a time with the system malloc()
-        page = llarena.arena_malloc(self.page_size, 2)     # zero-filled
-        if not page:
-            raise MemoryError
-        llarena.arena_reserve(page, self.HDRSIZE)
-        page = llmemory.cast_adr_to_ptr(page, self.HDRPTR)
-        page.tid = 0
-        self.free_pages = page
-
-    def grow_reservation(self, hdr, totalsize):
-        # Transform 'hdr', which used to point to just a HDR,
-        # into a pointer to a full object of size 'totalsize'.
-        # This is a no-op after translation.  Returns the
-        # address of the full object.
-        adr = llmemory.cast_ptr_to_adr(hdr)
-        adr = llarena.getfakearenaaddress(adr)
-        llarena.arena_reserve(adr, totalsize)
-        return adr + self.gcheaderbuilder.size_gc_header
-    grow_reservation._always_inline_ = True
-
     def deletion_barrier(self, addr_struct):
         # XXX check the assembler
         mark = self.header(addr_struct).tid & 0xFF
@@ -551,6 +536,7 @@
     # Weakrefs
 
     def weakref_deref(self, wrobj):
+        raise NotImplementedError
         # Weakrefs need some care.  This code acts as a read barrier.
         # The only way I found is to acquire the mutex_lock to prevent
         # the collection thread from going from collector.running==1
@@ -607,6 +593,7 @@
         # a different AddressStack class, which uses a different pool
         # of free pages than the regular one, so can run concurrently
         self.CollectorAddressStack = get_address_stack(lock="collector")
+        self.gray_objects = self.CollectorAddressStack()
         #
         # The start function for the thread, as a function and not a method
         def collector_start():
@@ -618,7 +605,7 @@
         self.collector_start = collector_start
 
     def _initialize(self):
-        self.gray_objects = self.CollectorAddressStack()
+        self.gray_objects.clear()
         #
         # When the mutator thread wants to trigger the next collection,
         # it scans its own stack roots and prepares everything, then
@@ -666,13 +653,13 @@
 
     def collector_run_nontranslated(self):
         try:
-            if not hasattr(self, 'currently_running_in_rtyper'):
+            if not hasattr(self.gc, 'currently_running_in_rtyper'):
                 self.collector_run()     # normal tests
             else:
                 # this case is for test_transformed_gc: we need to spawn
                 # another LLInterpreter for this new thread.
                 from pypy.rpython.llinterp import LLInterpreter
-                llinterp = LLInterpreter(self.currently_running_in_rtyper)
+                llinterp = LLInterpreter(self.gc.currently_running_in_rtyper)
                 # XXX FISH HORRIBLY for the graph...
                 graph = sys._getframe(2).f_locals['self']._obj.graph
                 llinterp.eval_graph(graph)
diff --git a/pypy/rpython/memory/support.py b/pypy/rpython/memory/support.py
--- a/pypy/rpython/memory/support.py
+++ b/pypy/rpython/memory/support.py
@@ -68,7 +68,7 @@
                 # Don't cache the old chunks but free them immediately.
                 # Helps debugging, and avoids that old chunks full of
                 # addresses left behind by a test end up in genc...
-                lltype.free(chunk, flavor="raw", track_allocation=False)
+                llarena.arena_free(chunk)
             self._unlock()
 
         if lock is not None and not isinstance(lock, str):
diff --git a/pypy/rpython/memory/test/test_transformed_gc.py b/pypy/rpython/memory/test/test_transformed_gc.py
--- a/pypy/rpython/memory/test/test_transformed_gc.py
+++ b/pypy/rpython/memory/test/test_transformed_gc.py
@@ -1438,7 +1438,7 @@
                          'translated_to_c': False}
             root_stack_depth = 200
 
-class TestConcurrentGenGC(GCTest):
+class TestConcurrentGenGC(GenericGCTests):
     gcname = "concurrentgen"
     class gcpolicy(gc.FrameworkGcPolicy):
         class transformerclass(framework.FrameworkGCTransformer):

From noreply at buildbot.pypy.org  Sun Oct 23 16:05:04 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 23 Oct 2011 16:05:04 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: With this,
 targetgcbench works (but is terribly bad for now).
Message-ID: <20111023140504.9D58A820C7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r48359:429e30ddfdfc
Date: 2011-10-23 16:02 +0200
http://bitbucket.org/pypy/pypy/changeset/429e30ddfdfc/

Log:	With this, targetgcbench works (but is terribly bad for now).

diff --git a/pypy/rpython/memory/gc/concurrentgen.py b/pypy/rpython/memory/gc/concurrentgen.py
--- a/pypy/rpython/memory/gc/concurrentgen.py
+++ b/pypy/rpython/memory/gc/concurrentgen.py
@@ -259,7 +259,11 @@
             self.force_scan(addr_struct)
 
     def assume_young_pointers(self, addr_struct):
-        pass # XXX
+        raise NotImplementedError
+
+    def writebarrier_before_copy(self, source_addr, dest_addr,
+                                 source_start, dest_start, length):
+        return False  # XXX implement
 
     def _init_writebarrier_logic(self):
         #
@@ -319,6 +323,7 @@
             self.flagged_objects.append(obj)
         #
         force_scan._dont_inline_ = True
+        force_scan._should_never_raise_ = True
         self.force_scan = force_scan
 
     def _barrier_add_extra(self, root, ignored):
@@ -443,7 +448,7 @@
         #
         # Add all old objects that have been written to since the last
         # time trigger_next_collection was called
-        self.flagged_objects.foreach(self._add_prebuilt_root, None)
+        self.flagged_objects.foreach(self._add_flagged_root, None)
         #
         # Clear this list
         self.flagged_objects.clear()
@@ -470,16 +475,26 @@
         self.execute_finalizers_ll()
 
     def _add_stack_root(self, root):
+        # NB. it's ok to edit 'gray_objects' from the mutator thread here,
+        # because the collector thread is not running yet
         obj = root.address[0]
         #debug_print("_add_stack_root", obj)
         self.get_mark(obj)
         self.collector.gray_objects.append(obj)
 
-    def _add_prebuilt_root(self, obj, ignored):
-        # NB. it's ok to edit 'gray_objects' from the mutator thread here,
-        # because the collector thread is not running yet
-        #debug_print("_add_prebuilt_root", obj)
-        self.get_mark(obj)
+    def _add_flagged_root(self, obj, ignored):
+        #debug_print("_add_flagged_root", obj)
+        #
+        # Important: the mark on 'obj' must be 'cym', otherwise it will not
+        # be scanned at all.  It should generally be, except in rare cases
+        # where it was reset to '#' by the collector thread.
+        mark = self.get_mark(obj)
+        if mark == MARK_BYTE_OLD:
+            self.set_mark(obj, self.current_young_marker)
+        else:
+            ll_assert(mark == self.current_young_marker,
+                      "add_flagged: bad mark")
+        #
         self.collector.gray_objects.append(obj)
 
     def debug_check_lists(self):
diff --git a/pypy/rpython/memory/gc/test/test_direct.py b/pypy/rpython/memory/gc/test/test_direct.py
--- a/pypy/rpython/memory/gc/test/test_direct.py
+++ b/pypy/rpython/memory/gc/test/test_direct.py
@@ -613,3 +613,37 @@
 class TestConcurrentGenGC(DirectGCTest):
     from pypy.rpython.memory.gc.concurrentgen \
             import ConcurrentGenGC as GCClass
+
+    def test_prebuilt_object_must_not_be_freed(self):
+        k = lltype.malloc(S, immortal=True)
+        k.x = 42
+        k.next = k2 = lltype.malloc(S, immortal=True)
+        k.next.x = 43
+        self.consider_constant(k)
+        self.consider_constant(k2)
+        all_s = []
+        all_s2 = []
+        #
+        for i in range(10):
+            s = self.malloc(S); s.x = 31
+            self.stackroots.append(s)
+            s2 = self.malloc(S); s2.x = 32
+            self.stackroots.append(s2)
+            #
+            all_s.append(s)
+            all_s2.append(s)
+            self.gc.collect(2)
+            #
+            self.write(k2, 'next', s)
+            self.write(k, 'next', s2)
+            self.gc.collect()
+            self.gc.collect()
+            assert k.x == 42
+            assert k.next.x == 32
+            assert k2.x == 43
+            assert k2.next.x == 31
+            #
+            for s in all_s:       # still all alive
+                assert s.x == 31
+            for s2 in all_s2:
+                assert s2.x == 31
diff --git a/pypy/rpython/memory/gctransform/framework.py b/pypy/rpython/memory/gctransform/framework.py
--- a/pypy/rpython/memory/gctransform/framework.py
+++ b/pypy/rpython/memory/gctransform/framework.py
@@ -323,7 +323,7 @@
                     getfn(GCClass.writebarrier_before_copy.im_func,
                     [s_gc] + [annmodel.SomeAddress()] * 2 +
                     [annmodel.SomeInteger()] * 3, annmodel.SomeBool())
-        elif GCClass.needs_write_barrier:
+        elif GCClass.needs_write_barrier or GCClass.needs_deletion_barrier:
             raise NotImplementedError("GC needs write barrier, but does not provide writebarrier_before_copy functionality")
 
         # in some GCs we can inline the common case of
diff --git a/pypy/translator/c/src/debug_lltrace.h b/pypy/translator/c/src/debug_lltrace.h
--- a/pypy/translator/c/src/debug_lltrace.h
+++ b/pypy/translator/c/src/debug_lltrace.h
@@ -12,6 +12,13 @@
 #  endif
 
 
+/* enable this for sending it out to stderr
+#undef RPyTraceSet
+#define RPyTraceSet(field, n)  {fprintf(stderr,"s %p->%p\n",\
+                                &(field),(void*)(long)field);}
+*/
+
+
 #else /*******************************************************/
 
 #  include "src/atomic_ops.h"
diff --git a/pypy/translator/c/test/test_newgc.py b/pypy/translator/c/test/test_newgc.py
--- a/pypy/translator/c/test/test_newgc.py
+++ b/pypy/translator/c/test/test_newgc.py
@@ -1541,3 +1541,7 @@
 class TestMostlyConcurrentMarkSweepGC(TestUsingFramework):
     gcpolicy = "concurrentms"
     repetitions = 100
+
+class TestConcurrentGenGC(TestUsingFramework):
+    gcpolicy = "concurrentgen"
+    repetitions = 100

From noreply at buildbot.pypy.org  Sun Oct 23 16:23:56 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 23 Oct 2011 16:23:56 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Only trigger the next
 minor collection after enough data was allocated.
Message-ID: <20111023142356.041FA820C7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r48360:230e2318e67c
Date: 2011-10-23 16:19 +0200
http://bitbucket.org/pypy/pypy/changeset/230e2318e67c/

Log:	Only trigger the next minor collection after enough data was
	allocated.

diff --git a/pypy/rpython/memory/gc/concurrentgen.py b/pypy/rpython/memory/gc/concurrentgen.py
--- a/pypy/rpython/memory/gc/concurrentgen.py
+++ b/pypy/rpython/memory/gc/concurrentgen.py
@@ -8,6 +8,7 @@
 from pypy.rlib.debug import ll_assert, debug_print, debug_start, debug_stop
 from pypy.rlib.rarithmetic import ovfcheck, LONG_BIT, r_uint
 from pypy.rpython.memory.gc.base import GCBase
+from pypy.rpython.memory.gc import env
 from pypy.rpython.memory import gctypelayout
 from pypy.rpython.memory.support import get_address_stack
 from pypy.module.thread import ll_thread
@@ -72,11 +73,25 @@
     # ^^^ prebuilt objects may have the flag FL_WITHHASH;
     #     then they are one word longer, the extra word storing the hash.
 
-    TRANSLATION_PARAMS = {}
+    TRANSLATION_PARAMS = {
+        # Automatically adjust the remaining parameters from the environment.
+        "read_from_env": True,
 
-    def __init__(self, config, **kwds):
+        # The default size of the nursery: use 6 MB by default.
+        # Environment variable: PYPY_GC_NURSERY
+        "nursery_size": 6*1024*1024,
+        }
+
+
+    def __init__(self, config,
+                 read_from_env=False,
+                 nursery_size=16*WORD,
+                 **kwds):
         GCBase.__init__(self, config, **kwds)
-        self.main_thread_ident = ll_thread.get_ident()
+        self.read_from_env = read_from_env
+        self.nursery_size = nursery_size
+        #
+        self.main_thread_ident = ll_thread.get_ident() # non-transl. debug only
         #
         self.extra_objects_to_mark = self.AddressStack()
         self.flagged_objects = self.AddressStack()
@@ -91,6 +106,13 @@
         # is a collection running and the mutator tries to change an object
         # that was not scanned yet.
         self._init_writebarrier_logic()
+        #
+        def trigger_collection_now():
+            # a hack to reduce the code size in _account_for_nursery():
+            # avoids both 'self' and the default argument value to be passed
+            self.trigger_next_collection()
+        trigger_collection_now._dont_inline_ = True
+        self.trigger_collection_now = trigger_collection_now
 
     def _initialize(self):
         # Initialize the GC.  In normal translated program, this function
@@ -134,6 +156,14 @@
         self.acquire(self.ready_to_start_lock)
         #
         self.collector.setup()
+        #
+        self.nursery_size_still_available = r_uint(self.nursery_size)
+        #
+        if self.read_from_env:
+            newsize = env.read_from_env('PYPY_GC_NURSERY')
+            if newsize > 0:
+                self.nursery_size = newsize
+                self.nursery_size_still_available = r_uint(newsize)
 
     def _teardown(self):
         "Stop the collector thread after tests have run."
@@ -162,11 +192,6 @@
     def malloc_fixedsize_clear(self, typeid, size,
                                needs_finalizer=False, contains_weakptr=False):
         #
-        # For now, we always start the next collection as soon as the
-        # previous one is finished
-        if self.collector.running <= 0:
-            self.trigger_next_collection()
-        #
         # Case of finalizers (test constant-folded)
         if needs_finalizer:
             raise NotImplementedError
@@ -182,7 +207,9 @@
         # Regular case
         size_gc_header = self.gcheaderbuilder.size_gc_header
         totalsize = size_gc_header + size
-        adr = llarena.arena_malloc(llmemory.raw_malloc_usage(totalsize), 2)
+        rawtotalsize = llmemory.raw_malloc_usage(totalsize)
+        self._account_for_nursery(rawtotalsize)
+        adr = llarena.arena_malloc(rawtotalsize, 2)
         if adr == llmemory.NULL:
             raise MemoryError
         llarena.arena_reserve(adr, totalsize)
@@ -195,12 +222,6 @@
 
     def malloc_varsize_clear(self, typeid, length, size, itemsize,
                              offset_to_length):
-        #
-        # For now, we always start the next collection as soon as the
-        # previous one is finished
-        if self.collector.running <= 0:
-            self.trigger_next_collection()
-        #
         size_gc_header = self.gcheaderbuilder.size_gc_header
         nonvarsize = size_gc_header + size
         #
@@ -211,7 +232,9 @@
         except OverflowError:
             raise MemoryError
         #
-        adr = llarena.arena_malloc(llmemory.raw_malloc_usage(totalsize), 2)
+        rawtotalsize = llmemory.raw_malloc_usage(totalsize)
+        self._account_for_nursery(rawtotalsize)
+        adr = llarena.arena_malloc(rawtotalsize, 2)
         if adr == llmemory.NULL:
             raise MemoryError
         llarena.arena_reserve(adr, totalsize)
@@ -223,6 +246,13 @@
         self.new_young_objects = hdr
         return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
 
+    def _account_for_nursery(self, additional_size):
+        if r_uint(additional_size) > self.nursery_size_still_available:
+            self.trigger_collection_now()
+        else:
+            self.nursery_size_still_available -= additional_size
+    _account_for_nursery._always_inline_ = True
+
     # ----------
     # Other functions in the GC API
 
@@ -470,6 +500,8 @@
         #debug_print("collector.running = 1")
         self.release(self.ready_to_start_lock)
         #
+        self.nursery_size_still_available = r_uint(self.nursery_size)
+        #
         debug_stop("gc-start")
         #
         self.execute_finalizers_ll()

From pullrequests-noreply at bitbucket.org  Sun Oct 23 18:03:00 2011
From: pullrequests-noreply at bitbucket.org (Bitbucket)
Date: Sun, 23 Oct 2011 16:03:00 -0000
Subject: [pypy-commit] [OPEN] Pull request #12 for pypy/pypy: Don't lose
 data when doing non-blocking I/O
Message-ID: <pullreq-2f16a8889242@bitbucket.org>

A new pull request has been opened by Stefano Rivera.

stefanor/pypy has changes to be pulled into pypy/pypy.

https://bitbucket.org/pypy/pypy/pull-request/12/dont-lose-data-when-doing-non-blocking-i-o

Title: Don't lose data when doing non-blocking I/O

Continuing on from 3f96afe7cdc2, I looked for all other possible data loss, when EAGAIN's could be thrown.

The common scenario is that many layers of the stream stack temporarily cache read data, while read()ing in a loop, and then return the data, concatenated. This breaks when you receive an EAGAIN.

I started by auditing the code to find possible problems, then writing tests that could expose them, then fixing the bugs. There isn't a strict relationship between tests and patches, rather, vaguley-comprehensive tests.

The tests are a bit ugly, I can't see any way to avoid massive code duplication or multiple asserts per test. Suggestions welcome :)

Changes to be pulled:

2f16a8889242 by Stefano Rivera: "Merge default"
f07445d2e8d2 by Stefano Rivera: "Catch EAGAIN in Stream.readline()"
808debcd8f67 by Stefano Rivera: "Catch EAGAIN in TextCRLFFilter.read()"
220c023da717 by Stefano Rivera: "Add AppTestCRLFFilterNonblocking to test TextCRLFilter on non-Windows"
0f58b4a9bb8b by Stefano Rivera: "Catch EAGAIN in W_File.direct_readline()"
3b966a3257d5 by Stefano Rivera: "Extend AppTestNonblocking to exercise limited size readlines()s, and reads that ?"
52f9fa5f396a by Stefano Rivera: "Catch EAGAIN in W_File.direct_read()"
ca15c5b74132 by Stefano Rivera: "Catch EAGAIN in BufferingInputStream.readline()"
71219b370d01 by Stefano Rivera: "Catch EAGAIN in BufferingInputStream.read()"
c3e17ec33491 by Stefano Rivera: "Extend AppTestNonblocking to exercise limited size read()s"
bf0c77b5c906 by Stefano Rivera: "Catch EAGAIN in TextInputFilter.readline()"
e5179b65e805 by Stefano Rivera: "Extend AppTestNonblocking to create a larger variety of test file objects (excer?"
30fe8615eee1 by Stefano Rivera: "Handle EAGAIN correctly for unbuffered files too"
a2025a6c6031 by Stefano Rivera: "Handle EAGAIN correctly for unbuffered files too"


--
This is an issue notification from bitbucket.org.
You are receiving this either because you are the participating
in a pull request, or you are following it.


From notifications-noreply at bitbucket.org  Sun Oct 23 22:30:38 2011
From: notifications-noreply at bitbucket.org (Bitbucket)
Date: Sun, 23 Oct 2011 20:30:38 -0000
Subject: [pypy-commit] [COMMENT] Pull request #12 for pypy/pypy: Don't lose
 data when doing non-blocking I/O
Message-ID: <20111023203038.10557.212@bitbucket13.managed.contegix.com>

New comment on pull request:

https://bitbucket.org/pypy/pypy/pull-request/12/dont-lose-data-when-doing-non-blocking-i-o#comment-540

Ronny Pfannschmidt (RonnyPfannschmidt) said:

it seems that most of your read loops can be generalized to some kind of lambda fd:... and a function that does the actual loop

--
This is a pull request comment notification from bitbucket.org.
You are receiving this either because you are participating
in a pull request, or you are following it.

From notifications-noreply at bitbucket.org  Sun Oct 23 23:09:09 2011
From: notifications-noreply at bitbucket.org (Bitbucket)
Date: Sun, 23 Oct 2011 21:09:09 -0000
Subject: [pypy-commit] [COMMENT] Pull request #12 for pypy/pypy: Don't lose
 data when doing non-blocking I/O
Message-ID: <20111023210909.25540.81212@bitbucket03.managed.contegix.com>

New comment on pull request:

https://bitbucket.org/pypy/pypy/pull-request/12/dont-lose-data-when-doing-non-blocking-i-o#comment-543

Amaury Forgeot d'Arc (amauryfa) said:

- Could you also add tests that run on a translated pypy-c? It's important, because python 3 uses a very different code (the _io module) and your changes are likely to be lost.
- How does CPython (2.7 and 3.2 behave in this area?)

--
This is a pull request comment notification from bitbucket.org.
You are receiving this either because you are participating
in a pull request, or you are following it.

From notifications-noreply at bitbucket.org  Sun Oct 23 23:33:41 2011
From: notifications-noreply at bitbucket.org (Bitbucket)
Date: Sun, 23 Oct 2011 21:33:41 -0000
Subject: [pypy-commit] [COMMENT] Pull request #12 for pypy/pypy: Don't lose
 data when doing non-blocking I/O
Message-ID: <20111023213341.10400.4987@bitbucket05.managed.contegix.com>

New comment on pull request:

https://bitbucket.org/pypy/pypy/pull-request/12/dont-lose-data-when-doing-non-blocking-i-o#comment-544

Stefano Rivera (stefanor) said:

Yes, but where can I define that function. One can't call neighbor methods from the test methods. (I've never met pytest before, and it seems rather magicky. Esp with the multiple objectspaces in pypy)

--
This is a pull request comment notification from bitbucket.org.
You are receiving this either because you are participating
in a pull request, or you are following it.

From notifications-noreply at bitbucket.org  Sun Oct 23 23:42:58 2011
From: notifications-noreply at bitbucket.org (Bitbucket)
Date: Sun, 23 Oct 2011 21:42:58 -0000
Subject: [pypy-commit] [COMMENT] Pull request #12 for pypy/pypy: Don't lose
 data when doing non-blocking I/O
Message-ID: <20111023214258.10915.13713@bitbucket13.managed.contegix.com>

New comment on pull request:

https://bitbucket.org/pypy/pypy/pull-request/12/dont-lose-data-when-doing-non-blocking-i-o#comment-545

Stefano Rivera (stefanor) said:

I did my best to copy C-python (2.7) and general C behavior. Basically, any read() can throw EAGAIN if there's nothing to be read. And a readline() can return a partial line if it runs out of data. Of course, this isn't quite as easy to test in cpython, because we can't play with internals, but I get the same results when running bursty jobs in subprocess.

In fact, the reason I did any of this at all was so that we could get pygame's test suite to run under Pypy. Without this patch, it randomly looses data that subprocesses output. Here's a slightly modified example of that test suite's problem (the sleep may need tweaking between cpython and pypy to get nice bursty reads): http://paste.pocoo.org/show/497065/

--
This is a pull request comment notification from bitbucket.org.
You are receiving this either because you are participating
in a pull request, or you are following it.

From notifications-noreply at bitbucket.org  Sun Oct 23 23:46:58 2011
From: notifications-noreply at bitbucket.org (Bitbucket)
Date: Sun, 23 Oct 2011 21:46:58 -0000
Subject: [pypy-commit] [COMMENT] Pull request #12 for pypy/pypy: Don't lose
 data when doing non-blocking I/O
Message-ID: <20111023214658.19781.31494@bitbucket03.managed.contegix.com>

New comment on pull request:

https://bitbucket.org/pypy/pypy/pull-request/12/dont-lose-data-when-doing-non-blocking-i-o#comment-546

Stefano Rivera (stefanor) said:

Also, ideally, I'd want one test per assert. But I can't work out where I should be generating that.

--
This is a pull request comment notification from bitbucket.org.
You are receiving this either because you are participating
in a pull request, or you are following it.

From noreply at buildbot.pypy.org  Mon Oct 24 10:48:23 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 24 Oct 2011 10:48:23 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Major collections.
Message-ID: <20111024084823.CDEB7820C7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r48361:6a06b46f0e3f
Date: 2011-10-24 10:39 +0200
http://bitbucket.org/pypy/pypy/changeset/6a06b46f0e3f/

Log:	Major collections.

diff --git a/pypy/rpython/memory/gc/concurrentgen.py b/pypy/rpython/memory/gc/concurrentgen.py
--- a/pypy/rpython/memory/gc/concurrentgen.py
+++ b/pypy/rpython/memory/gc/concurrentgen.py
@@ -6,7 +6,7 @@
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
 from pypy.rlib.objectmodel import we_are_translated, running_on_llinterp
 from pypy.rlib.debug import ll_assert, debug_print, debug_start, debug_stop
-from pypy.rlib.rarithmetic import ovfcheck, LONG_BIT, r_uint
+from pypy.rlib.rarithmetic import ovfcheck, LONG_BIT, r_uint, intmask
 from pypy.rpython.memory.gc.base import GCBase
 from pypy.rpython.memory.gc import env
 from pypy.rpython.memory import gctypelayout
@@ -23,8 +23,8 @@
 # phase can be parallelized again.  XXX not done so far, YYY investigate
 # also completely parallelizing them too
 #
-# Based on observations that the timing of collections with "minimark"
-# (on translate.py) is: about 15% of the time in minor collections
+# Based on observations of the timing of collections with "minimark"
+# (on translate.py): about 15% of the time in minor collections
 # (including 2% in walk_roots), and about 7% in major collections (with
 # probably 3-4% in the marking phase).  So out of a total of 22% this
 # should parallelize 16-17%, i.e. 3/4th.
@@ -36,7 +36,7 @@
 WORD = LONG_BIT // 8
 WORD_POWER_2 = {32: 2, 64: 3}[LONG_BIT]
 assert 1 << WORD_POWER_2 == WORD
-MAXIMUM_SIZE = sys.maxint - (3*WORD-1)
+FLOAT_ALMOST_MAXINT = float(sys.maxint) * 0.9999
 
 
 # Objects start with an integer 'tid', which is decomposed as follows.
@@ -44,8 +44,11 @@
 # let us know if the 'tid' is valid or is just a word-aligned address):
 MARK_BYTE_1       = 0x6D    # 'm', 109
 MARK_BYTE_2       = 0x4B    # 'K', 75
-MARK_BYTE_OLD     = 0x23    # '#', 35
-MARK_BYTE_STATIC  = 0x53    # 'S', 83
+MARK_BYTE_OLD_1   = 0x23    # '#', 35
+MARK_BYTE_OLD_2   = 0x2F    # '/', 47
+MARK_BYTE_STATIC  = 0x35    # '5', 53
+mark_byte_is_old  = lambda n: n <= MARK_BYTE_OLD_2
+mark_byte_is_old_or_static = lambda n: n <= MARK_BYTE_STATIC
 # Next lower byte: a combination of flags.
 FL_WITHHASH       = 0x0100
 FL_EXTRA          = 0x0200
@@ -80,16 +83,24 @@
         # The default size of the nursery: use 6 MB by default.
         # Environment variable: PYPY_GC_NURSERY
         "nursery_size": 6*1024*1024,
+
+        # Trigger another major collection when 'N+(F-1)*P' bytes survived
+        # minor collections, where N = nursery_size, P = bytes surviving
+        # the previous major collection, and F is the fill_factor here.
+        # Environment variable: PYPY_GC_MAJOR_COLLECT
+        "fill_factor": 1.75,
         }
 
 
     def __init__(self, config,
                  read_from_env=False,
-                 nursery_size=16*WORD,
+                 nursery_size=32*WORD,
+                 fill_factor=2.0,
                  **kwds):
         GCBase.__init__(self, config, **kwds)
         self.read_from_env = read_from_env
         self.nursery_size = nursery_size
+        self.fill_factor = fill_factor
         #
         self.main_thread_ident = ll_thread.get_ident() # non-transl. debug only
         #
@@ -157,13 +168,21 @@
         #
         self.collector.setup()
         #
-        self.nursery_size_still_available = r_uint(self.nursery_size)
-        #
+        self.set_nursery_size(self.nursery_size)
         if self.read_from_env:
+            #
             newsize = env.read_from_env('PYPY_GC_NURSERY')
             if newsize > 0:
-                self.nursery_size = newsize
-                self.nursery_size_still_available = r_uint(newsize)
+                self.set_nursery_size(newsize)
+            #
+            fillfact = env.read_float_from_env('PYPY_GC_MAJOR_COLLECT')
+            if fillfact > 1.0:
+                self.fill_factor = fillfact
+
+    def set_nursery_size(self, newsize):
+        self.nursery_size = newsize
+        self.nursery_size_still_available = newsize
+        self.size_still_available_before_major = newsize
 
     def _teardown(self):
         "Stop the collector thread after tests have run."
@@ -207,7 +226,7 @@
         # Regular case
         size_gc_header = self.gcheaderbuilder.size_gc_header
         totalsize = size_gc_header + size
-        rawtotalsize = llmemory.raw_malloc_usage(totalsize)
+        rawtotalsize = raw_malloc_usage(totalsize)
         self._account_for_nursery(rawtotalsize)
         adr = llarena.arena_malloc(rawtotalsize, 2)
         if adr == llmemory.NULL:
@@ -232,7 +251,7 @@
         except OverflowError:
             raise MemoryError
         #
-        rawtotalsize = llmemory.raw_malloc_usage(totalsize)
+        rawtotalsize = raw_malloc_usage(totalsize)
         self._account_for_nursery(rawtotalsize)
         adr = llarena.arena_malloc(rawtotalsize, 2)
         if adr == llmemory.NULL:
@@ -247,10 +266,9 @@
         return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
 
     def _account_for_nursery(self, additional_size):
-        if r_uint(additional_size) > self.nursery_size_still_available:
+        self.nursery_size_still_available -= additional_size
+        if self.nursery_size_still_available < 0:
             self.trigger_collection_now()
-        else:
-            self.nursery_size_still_available -= additional_size
     _account_for_nursery._always_inline_ = True
 
     # ----------
@@ -302,15 +320,14 @@
             mark = self.get_mark(obj)
             #debug_print("deletion_barrier:", mark, obj)
             #
-            if mark == MARK_BYTE_OLD:
+            if mark_byte_is_old_or_static(mark):
                 #
                 self.set_mark(obj, cym)
                 #
-            elif mark == MARK_BYTE_STATIC:
-                # This is the first write into a prebuilt GC object.
-                # Record it in 'prebuilt_root_objects'.
-                self.set_mark(obj, cym)
-                self.prebuilt_root_objects.append(obj)
+                if mark == MARK_BYTE_STATIC:
+                    # This is the first write into a prebuilt GC object.
+                    # Record it in 'prebuilt_root_objects'.
+                    self.prebuilt_root_objects.append(obj)
                 #
             else:
                 #
@@ -341,10 +358,10 @@
                               "write barrier: oups!?")
                     #
                 else:
-                    # MARK_BYTE_OLD is possible here: the collector thread
+                    # MARK_BYTE_OLD_* is possible here: the collector thread
                     # sets it in parallel to objects.  In that case it has
                     # been handled already.
-                    ll_assert(mark == MARK_BYTE_OLD,
+                    ll_assert(mark_byte_is_old(mark),
                               "write barrier: bogus object mark")
                 #
                 self.release(self.mutex_lock)
@@ -367,20 +384,7 @@
         running (if any) to finish."""
         if self.collector.running != 0:
             debug_start("gc-stop")
-            #
-            self.acquire(self.finished_lock)
-            self.collector.running = 0
-            #debug_print("collector.running = 0")
-            #
-            # Check invariants
-            ll_assert(not self.extra_objects_to_mark.non_empty(),
-                      "objs left behind in extra_objects_to_mark")
-            ll_assert(not self.collector.gray_objects.non_empty(),
-                      "objs left behind in gray_objects")
-            #
-            if self.DEBUG:
-                self.debug_check_lists()
-            #
+            self._stop_collection()
             debug_stop("gc-stop")
             #
             # We must *not* run execute_finalizers_ll() here, because it
@@ -390,14 +394,19 @@
             ll_assert(self.collector.running == 0,
                       "collector thread not paused?")
 
-    def join_lists(self, list1, list2head, list2tail):
-        if list2tail == self.NULL:
-            ll_assert(list2head == self.NULL, "join_lists/1")
-            return list1
-        else:
-            ll_assert(list2head != self.NULL, "join_lists/2")
-            set_next(list2tail, list1)
-            return list2head
+    def _stop_collection(self):
+        self.acquire(self.finished_lock)
+        self.collector.running = 0
+        #debug_print("collector.running = 0")
+        #
+        # Check invariants
+        ll_assert(not self.extra_objects_to_mark.non_empty(),
+                  "objs left behind in extra_objects_to_mark")
+        ll_assert(not self.collector.gray_objects.non_empty(),
+                  "objs left behind in gray_objects")
+        #
+        if self.DEBUG:
+            self.debug_check_lists()
 
 
     def execute_finalizers_ll(self):
@@ -457,6 +466,18 @@
         # In case the previous collection is not over yet, wait for it
         self.wait_for_the_end_of_collection()
         #
+        # Choose between a minor and a major collection
+        if (force_major_collection or
+                self.size_still_available_before_major < 0):
+            self._start_major_collection()
+        else:
+            self._start_minor_collection()
+        #
+        self.execute_finalizers_ll()
+
+
+    def _start_minor_collection(self):
+        #
         debug_start("gc-start")
         #
         # Scan the stack roots and the refs in non-GC objects
@@ -496,15 +517,54 @@
         #self.collect_finalizer_pages = self.finalizer_pages
         #
         # Start the collector thread
+        self._start_collection_common(False)
+        #
+        debug_stop("gc-start")
+
+    def _start_major_collection(self):
+        #
+        debug_start("gc-major-collection")
+        #
+        # Clear this list, which is not relevant for major collections
+        self.flagged_objects.clear()
+        #
+        # Scan the stack roots and the refs in non-GC objects
+        self.root_walker.walk_roots(
+            ConcurrentGenGC._add_stack_root,  # stack roots
+            ConcurrentGenGC._add_stack_root,  # in prebuilt non-gc
+            None)                             # static in prebuilt gc
+        #
+        # Add the objects still waiting in 'objects_with_finalizers_to_run'
+        #xxx
+        #
+        # Add all prebuilt objects that have ever been mutated
+        self.prebuilt_root_objects.foreach(self._add_prebuilt_root, None)
+        #
+        # Copy a few 'mutator' fields to 'collector' fields
+        collector = self.collector
+        collector.aging_objects = self.new_young_objects
+        self.new_young_objects = self.NULL
+
+        collector.collect_old_objects = self.old_objects
+        self.old_objects = self.NULL
+
+        #self.collect_weakref_pages = self.weakref_pages
+        #self.collect_finalizer_pages = self.finalizer_pages
+        #
+        # Start the collector thread
+        self._start_collection_common(True)
+        #
+        # Pause the mutator thread while a major collection is running
+        self._stop_collection()
+        #
+        debug_stop("gc-major-collection")
+
+    def _start_collection_common(self, is_major):
+        self.collector.is_major_collection = is_major
         self.collector.running = 1
         #debug_print("collector.running = 1")
         self.release(self.ready_to_start_lock)
-        #
-        self.nursery_size_still_available = r_uint(self.nursery_size)
-        #
-        debug_stop("gc-start")
-        #
-        self.execute_finalizers_ll()
+        self.nursery_size_still_available = self.nursery_size
 
     def _add_stack_root(self, root):
         # NB. it's ok to edit 'gray_objects' from the mutator thread here,
@@ -519,9 +579,9 @@
         #
         # Important: the mark on 'obj' must be 'cym', otherwise it will not
         # be scanned at all.  It should generally be, except in rare cases
-        # where it was reset to '#' by the collector thread.
+        # where it was reset to MARK_BYTE_OLD_* by the collector thread.
         mark = self.get_mark(obj)
-        if mark == MARK_BYTE_OLD:
+        if mark_byte_is_old(mark):
             self.set_mark(obj, self.current_young_marker)
         else:
             ll_assert(mark == self.current_young_marker,
@@ -529,6 +589,10 @@
         #
         self.collector.gray_objects.append(obj)
 
+    def _add_prebuilt_root(self, obj, ignored):
+        self.get_mark(obj)
+        self.collector.gray_objects.append(obj)
+
     def debug_check_lists(self):
         # just check that they are correct, non-infinite linked lists
         self.debug_check_list(self.new_young_objects)
@@ -575,7 +639,8 @@
         mark = self.header(obj).tid & 0xFF
         ll_assert(mark == MARK_BYTE_1 or
                   mark == MARK_BYTE_2 or
-                  mark == MARK_BYTE_OLD or
+                  mark == MARK_BYTE_OLD_1 or
+                  mark == MARK_BYTE_OLD_2 or
                   mark == MARK_BYTE_STATIC, "bad mark byte in object")
         return mark
 
@@ -675,6 +740,9 @@
         # when the collection starts, we make all young objects aging and
         # move 'new_young_objects' into 'aging_objects'
         self.aging_objects = self.NULL
+        self.collect_old_objects = self.NULL
+        self.current_old_marker = MARK_BYTE_OLD_1
+        self.num_major_collects = 0
 
     def setup(self):
         self.ready_to_start_lock = self.gc.ready_to_start_lock
@@ -746,12 +814,18 @@
 
 
     def collector_mark(self):
+        if self.is_major_collection:
+            self.collector_mark_major()
+            return
+        #
+        surviving_size = r_uint(0)
+        #
         while True:
             #
             # Do marking.  The following function call is interrupted
             # if the mutator's write barrier adds new objects to
             # 'extra_objects_to_mark'.
-            self._collect_mark()
+            surviving_size += self._collect_mark()
             #
             # Move the objects from 'extra_objects_to_mark' to
             # 'gray_objects'.  This requires the mutex lock.
@@ -777,18 +851,42 @@
             # Else release mutex_lock and try again.
             self.release(self.mutex_lock)
         #
+        # When we sweep during minor collections, we subtract the size of
+        # the surviving now-old objects to the following field.  Note
+        # that the write barrier may make objects young again, without
+        # decreasing the value here.  During the following minor
+        # collection this variable will be decreased *again*.  When the
+        # write barrier triggers on an aging object, it is random whether
+        # its size ends up being accounted here or not --- but it will
+        # be at the following minor collection, because the object is
+        # young again.  So, careful about underflows.
+        sz = self.gc.size_still_available_before_major
+        if sz < 0 or r_uint(sz) < surviving_size:
+            sz = -1    # trigger major collect the next time
+        else:
+            # do the difference, which results in a non-negative number
+            # (in particular, surviving_size <= sys.maxint here)
+            sz -= intmask(surviving_size)
+        #
+        self.gc.size_still_available_before_major = sz
+        #
         self.running = 2
         #debug_print("collection_running = 2")
         self.release(self.mutex_lock)
 
     def _collect_mark(self):
+        surviving_size = r_uint(0)
         extra_objects_to_mark = self.gc.extra_objects_to_mark
         cam = self.current_aging_marker
+        com = self.current_old_marker
         while self.gray_objects.non_empty():
             obj = self.gray_objects.pop()
             if self.get_mark(obj) != cam:
                 continue
             #
+            # Record the object's size
+            surviving_size += raw_malloc_usage(self.gc.get_size(obj))
+            #
             # Scan the content of 'obj'.  We use a snapshot-at-the-
             # beginning order, meaning that we want to scan the state
             # of the object as it was at the beginning of the current
@@ -811,7 +909,7 @@
             # is never scanned.
             #
             self.gc.trace(obj, self._collect_add_pending, None)
-            self.set_mark(obj, MARK_BYTE_OLD)
+            self.set_mark(obj, com)
             #
             # Interrupt early if the mutator's write barrier adds stuff
             # to that list.  Note that the check is imprecise because
@@ -820,7 +918,9 @@
             # the write barrier, because they are more likely to
             # reference further objects that will soon be accessed too.
             if extra_objects_to_mark.non_empty():
-                return
+                break
+        #
+        return surviving_size
 
     def _collect_add_pending(self, root, ignored):
         obj = root.address[0]
@@ -829,14 +929,82 @@
         self.get_mark(obj)
         self.gray_objects.append(obj)
 
+    def collector_mark_major(self):
+        # Marking for a major collection.  Differs from marking for
+        # a minor collection, because we have to follow references
+        # to objects whose mark is 'cym' or 'oom', and replace them
+        # with 'nom'.  We must stop if objects have already 'nom',
+        # or if they have MARK_BYTE_STATIC.  For now they cannot
+        # have 'cam'.
+        #
+        # Get 'oom' and 'nom' from current_old_marker, and switch
+        # the value in that field:
+        oom = self.current_old_marker
+        nom = oom ^ (MARK_BYTE_OLD_1 ^ MARK_BYTE_OLD_2)
+        self.current_old_marker = nom
+        #
+        debug_print()
+        debug_print(".----------- Full collection ------------------")
+        #
+        self.num_major_collects += 1
+        debug_print("| number of major collects:   ", self.num_major_collects)
+        #
+        surviving_size = r_uint(0)
+        #
+        while self.gray_objects.non_empty():
+            obj = self.gray_objects.pop()
+            mark = self.get_mark(obj)
+            if mark == nom or mark == MARK_BYTE_STATIC:
+                continue
+            #
+            # Record the object's size
+            surviving_size += raw_malloc_usage(self.gc.get_size(obj))
+            #
+            # Scan the content of 'obj'.
+            self.gc.trace(obj, self._collect_add_pending, None)
+            self.set_mark(obj, nom)
+        #
+        next_major_collection_limit = (      # as a float
+            self.gc.nursery_size +
+            (self.gc.fill_factor - 1.0) * float(surviving_size))
+        if next_major_collection_limit > FLOAT_ALMOST_MAXINT:
+            next_major_collection_limit = FLOAT_ALMOST_MAXINT
+        #
+        self.gc.size_still_available_before_major = int(
+            next_major_collection_limit)
+        #
+        debug_print("| surviving size:             ", surviving_size)
+        debug_print("| next major collection after:",
+                    self.gc.size_still_available_before_major)
+
+
     def collector_sweep(self):
-        cam = self.current_aging_marker
-        hdr = self.aging_objects
+        if self.is_major_collection:
+            #
+            cym = self.gc.current_young_marker
+            nom = self.current_old_marker
+            oom = nom ^ (MARK_BYTE_OLD_1 ^ MARK_BYTE_OLD_2)
+            self._collect_do_sweep(self.aging_objects, cym, False)
+            self._collect_do_sweep(self.collect_old_objects, oom, False)
+            #
+            debug_print("`----------------------------------------------")
+            #
+        else:
+            #
+            cam = self.current_aging_marker
+            self._collect_do_sweep(self.aging_objects, cam, True)
+            #
+        #
+        self.running = -1
+        #debug_print("collection_running = -1")
+
+    def _collect_do_sweep(self, hdr, still_not_marked, write_barrier_active):
         linked_list = self.gc.old_objects
+        #
         while hdr != self.NULL:
             nexthdr = hdr.next
             mark = hdr.tid & 0xFF
-            if mark == cam:
+            if mark == still_not_marked:
                 # the object is still not marked.  Free it.
                 blockadr = llmemory.cast_ptr_to_adr(hdr)
                 blockadr = llarena.getfakearenaaddress(blockadr)
@@ -844,17 +1012,17 @@
                 #
             else:
                 # the object was marked: relink it
-                ll_assert(mark == self.gc.current_young_marker or
-                          mark == MARK_BYTE_OLD, "sweep: bad mark")
+                ll_assert(mark == self.current_old_marker or
+                          (write_barrier_active and
+                              mark == self.gc.current_young_marker),
+                          "sweep: bad mark")
                 hdr.next = linked_list
                 linked_list = hdr
                 #
             hdr = nexthdr
         #
         self.gc.old_objects = linked_list
-        #
-        self.running = -1
-        #debug_print("collection_running = -1")
+
 
     # -------------------------
     # CollectorThread: Weakrefs
@@ -960,7 +1128,8 @@
 
 def emulate_set_mark(p, v):
     "NOT_RPYTHON"
-    assert v in (MARK_BYTE_1, MARK_BYTE_2, MARK_BYTE_OLD, MARK_BYTE_STATIC)
+    assert v in (MARK_BYTE_1, MARK_BYTE_2,
+                 MARK_BYTE_OLD_1, MARK_BYTE_OLD_2, MARK_BYTE_STATIC)
     concurrent_setter_lock.acquire(True)
     p.tid = (p.tid &~ 0xFF) | v
     concurrent_setter_lock.release()

From noreply at buildbot.pypy.org  Mon Oct 24 10:48:25 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 24 Oct 2011 10:48:25 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Fixes.
Message-ID: <20111024084825.0617D820C7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r48362:2c6f2010d4af
Date: 2011-10-24 10:47 +0200
http://bitbucket.org/pypy/pypy/changeset/2c6f2010d4af/

Log:	Fixes.

diff --git a/pypy/rpython/memory/gc/concurrentgen.py b/pypy/rpython/memory/gc/concurrentgen.py
--- a/pypy/rpython/memory/gc/concurrentgen.py
+++ b/pypy/rpython/memory/gc/concurrentgen.py
@@ -525,7 +525,10 @@
         #
         debug_start("gc-major-collection")
         #
-        # Clear this list, which is not relevant for major collections
+        # Clear this list, which is not relevant for major collections.
+        # For simplicity we first reset the markers on the objects it
+        # contains, which are all originally old objects.
+        self.flagged_objects.foreach(self._reset_flagged_root, None)
         self.flagged_objects.clear()
         #
         # Scan the stack roots and the refs in non-GC objects
@@ -589,6 +592,9 @@
         #
         self.collector.gray_objects.append(obj)
 
+    def _reset_flagged_root(self, obj, ignored):
+        self.set_mark(obj, self.collector.current_old_marker)
+
     def _add_prebuilt_root(self, obj, ignored):
         self.get_mark(obj)
         self.collector.gray_objects.append(obj)
diff --git a/pypy/rpython/memory/support.py b/pypy/rpython/memory/support.py
--- a/pypy/rpython/memory/support.py
+++ b/pypy/rpython/memory/support.py
@@ -68,6 +68,8 @@
                 # Don't cache the old chunks but free them immediately.
                 # Helps debugging, and avoids that old chunks full of
                 # addresses left behind by a test end up in genc...
+                chunk = llmemory.cast_ptr_to_adr(chunk)
+                chunk = llarena.getfakearenaaddress(chunk)
                 llarena.arena_free(chunk)
             self._unlock()
 

From noreply at buildbot.pypy.org  Mon Oct 24 11:21:28 2011
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 24 Oct 2011 11:21:28 +0200 (CEST)
Subject: [pypy-commit] pypy int-tag-untag-as-operations: a branch where to
 make int tagging and untagging explicit operations
Message-ID: <20111024092128.AE8C6820C7@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: int-tag-untag-as-operations
Changeset: r48363:63dfc29454a2
Date: 2011-10-22 14:13 +0200
http://bitbucket.org/pypy/pypy/changeset/63dfc29454a2/

Log:	a branch where to make int tagging and untagging explicit operations


From noreply at buildbot.pypy.org  Mon Oct 24 11:21:30 2011
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 24 Oct 2011 11:21:30 +0200 (CEST)
Subject: [pypy-commit] pypy int-tag-untag-as-operations: introduce int_tag
 (which can overflow) and int_untag
Message-ID: <20111024092130.0138E82A87@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: int-tag-untag-as-operations
Changeset: r48364:300e42e0ee80
Date: 2011-10-22 14:13 +0200
http://bitbucket.org/pypy/pypy/changeset/300e42e0ee80/

Log:	introduce int_tag (which can overflow) and int_untag

diff --git a/pypy/jit/backend/llgraph/llimpl.py b/pypy/jit/backend/llgraph/llimpl.py
--- a/pypy/jit/backend/llgraph/llimpl.py
+++ b/pypy/jit/backend/llgraph/llimpl.py
@@ -692,6 +692,17 @@
         if not flag:
             raise GuardFailed
 
+    def op_int_tag(self, _, x):
+        try:
+            z = ovfcheck(x << 1) + 1
+        except OverflowError:
+            ovf = True
+            z = 0
+        else:
+            ovf = False
+        self.overflow_flag = ovf
+        return z
+
     def op_int_add_ovf(self, _, x, y):
         try:
             z = ovfcheck(x + y)
diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -269,6 +269,10 @@
         op1 = SpaceOperation('-live-', [], None)
         return [op, op1]
 
+    def rewrite_op_int_tag(self, op):
+        op1 = SpaceOperation('-live-', [], None)
+        return [op, op1]
+
     # ----------
     # Various kinds of calls
 
diff --git a/pypy/jit/codewriter/support.py b/pypy/jit/codewriter/support.py
--- a/pypy/jit/codewriter/support.py
+++ b/pypy/jit/codewriter/support.py
@@ -38,9 +38,10 @@
     return a.typeannotation(t)
 
 def annotate(func, values, inline=None, backendoptimize=True,
-             type_system="lltype"):
+             type_system="lltype", taggedpointers=False):
     # build the normal ll graphs for ll_function
     t = TranslationContext()
+    t.config.translation.taggedpointers = taggedpointers
     annpolicy = AnnotatorPolicy()
     annpolicy.allow_someobjects = False
     a = t.buildannotator(policy=annpolicy)
diff --git a/pypy/jit/metainterp/blackhole.py b/pypy/jit/metainterp/blackhole.py
--- a/pypy/jit/metainterp/blackhole.py
+++ b/pypy/jit/metainterp/blackhole.py
@@ -447,6 +447,14 @@
     def bhimpl_int_invert(a):
         return intmask(~a)
 
+    @arguments("i", returns="i")
+    def bhimpl_int_untag(a):
+        return a >> 1
+    @arguments("i", returns="i")
+    def bhimpl_int_tag(a):
+        return ovfcheck(a << 1) + 1
+
+
     @arguments("i", "i", returns="i")
     def bhimpl_int_lt(a, b):
         return a < b
diff --git a/pypy/jit/metainterp/executor.py b/pypy/jit/metainterp/executor.py
--- a/pypy/jit/metainterp/executor.py
+++ b/pypy/jit/metainterp/executor.py
@@ -204,6 +204,18 @@
         z = 0
     return BoxInt(z)
 
+def do_int_tag(cpu, metainterp, box1):
+    # the overflow operations can be called without a metainterp, if an
+    # overflow cannot occur
+    a = box1.getint()
+    try:
+        z = ovfcheck(a << 1)
+    except OverflowError:
+        assert metainterp is not None
+        metainterp.execute_raised(OverflowError(), constant=True)
+        z = 0
+    return BoxInt(z + 1)
+
 def do_same_as(cpu, _, box):
     return box.clonebox()
 
diff --git a/pypy/jit/metainterp/optimizeopt/intbounds.py b/pypy/jit/metainterp/optimizeopt/intbounds.py
--- a/pypy/jit/metainterp/optimizeopt/intbounds.py
+++ b/pypy/jit/metainterp/optimizeopt/intbounds.py
@@ -284,6 +284,10 @@
         else:
             self.emit_operation(op)
 
+    def optimize_INT_TAG(self, op):
+        self.emit_operation(op) # XXX for now
+        self.emit_operation(self.nextop)
+
     def optimize_ARRAYLEN_GC(self, op):
         self.emit_operation(op)
         array  = self.getvalue(op.getarg(0))
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -218,7 +218,7 @@
                 return resbox
         ''' % (_opimpl, _opimpl.upper())).compile()
 
-    for _opimpl in ['int_is_true', 'int_is_zero', 'int_neg', 'int_invert',
+    for _opimpl in ['int_is_true', 'int_is_zero', 'int_neg', 'int_invert', 'int_untag',
                     'cast_float_to_int', 'cast_int_to_float',
                     'cast_float_to_singlefloat', 'cast_singlefloat_to_float',
                     'float_neg', 'float_abs',
@@ -231,6 +231,15 @@
         ''' % (_opimpl, _opimpl.upper())).compile()
 
     @arguments("box")
+    def opimpl_int_tag(self, b1):
+        self.metainterp.clear_exception()
+        resbox = self.execute(rop.INT_TAG, b1)
+        self.make_result_of_lastop(resbox)
+        if not isinstance(resbox, Const):
+            self.metainterp.handle_possible_overflow_error()
+        return resbox
+
+    @arguments("box")
     def opimpl_ptr_nonzero(self, box):
         return self.execute(rop.PTR_NE, box, history.CONST_NULL)
 
diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py
--- a/pypy/jit/metainterp/resoperation.py
+++ b/pypy/jit/metainterp/resoperation.py
@@ -431,6 +431,7 @@
     'INT_IS_TRUE/1b',
     'INT_NEG/1',
     'INT_INVERT/1',
+    'INT_UNTAG/1',
     #
     'SAME_AS/1',      # gets a Const or a Box, turns it into another Box
     'CAST_PTR_TO_INT/1',
@@ -504,6 +505,7 @@
     'INT_ADD_OVF/2',
     'INT_SUB_OVF/2',
     'INT_MUL_OVF/2',
+    'INT_TAG/1',
     '_OVF_LAST', # ----- end of is_ovf operations -----
     '_LAST',     # for the backend to add more internal operations
 ]
diff --git a/pypy/jit/metainterp/test/support.py b/pypy/jit/metainterp/test/support.py
--- a/pypy/jit/metainterp/test/support.py
+++ b/pypy/jit/metainterp/test/support.py
@@ -42,7 +42,7 @@
         enable_opts = ALL_OPTS_DICT
 
     func._jit_unroll_safe_ = True
-    rtyper = support.annotate(func, values, type_system=type_system)
+    rtyper = support.annotate(func, values, type_system=type_system, **kwds)
     graphs = rtyper.annotator.translator.graphs
     testself.all_graphs = graphs
     result_kind = history.getkind(graphs[0].getreturnvar().concretetype)[0]
diff --git a/pypy/rlib/rerased.py b/pypy/rlib/rerased.py
--- a/pypy/rlib/rerased.py
+++ b/pypy/rlib/rerased.py
@@ -160,7 +160,7 @@
     from pypy.rlib.debug import ll_assert
     x = llop.cast_ptr_to_int(lltype.Signed, gcref)
     ll_assert((x&1) != 0, "unerased_int(): not an integer")
-    return x >> 1
+    return llop.int_untag(lltype.Signed, x)
 
 
 class Entry(ExtRegistryEntry):
@@ -220,11 +220,9 @@
         [v_value] = hop.inputargs(lltype.Signed)
         c_one = hop.inputconst(lltype.Signed, 1)
         hop.exception_is_here()
-        v2 = hop.genop('int_add_ovf', [v_value, v_value],
+        v2 = hop.genop('int_tag', [v_value],
                        resulttype = lltype.Signed)
-        v2p1 = hop.genop('int_add', [v2, c_one],
-                         resulttype = lltype.Signed)
-        v_instance = hop.genop('cast_int_to_ptr', [v2p1],
+        v_instance = hop.genop('cast_int_to_ptr', [v2],
                                resulttype=self.lowleveltype)
         return v_instance
 
diff --git a/pypy/rpython/llinterp.py b/pypy/rpython/llinterp.py
--- a/pypy/rpython/llinterp.py
+++ b/pypy/rpython/llinterp.py
@@ -1095,6 +1095,16 @@
             assert y >= 0
         return self.op_int_add_ovf(x, y)
 
+    def op_int_tag(self, x):
+        try:
+            return ovfcheck(x + x + 1)
+        except OverflowError:
+            self.make_llexception()
+
+    def op_int_untag(self, x):
+        assert x & 1, "argument has to be tagged!"
+        return x >> 1
+
     def op_cast_float_to_int(self, f):
         assert type(f) is float
         try:
diff --git a/pypy/rpython/lltypesystem/lloperation.py b/pypy/rpython/lltypesystem/lloperation.py
--- a/pypy/rpython/lltypesystem/lloperation.py
+++ b/pypy/rpython/lltypesystem/lloperation.py
@@ -207,6 +207,8 @@
     'int_abs':              LLOp(canfold=True),
     'int_abs_ovf':          LLOp(canraise=(OverflowError,), tryfold=True),
     'int_invert':           LLOp(canfold=True),
+    'int_tag':              LLOp(canraise=(OverflowError, ), tryfold=True),
+    'int_untag':            LLOp(canfold=True),
 
     'int_add':              LLOp(canfold=True),
     'int_sub':              LLOp(canfold=True),
diff --git a/pypy/translator/c/src/int.h b/pypy/translator/c/src/int.h
--- a/pypy/translator/c/src/int.h
+++ b/pypy/translator/c/src/int.h
@@ -19,6 +19,14 @@
 	if ((x) == LONG_MIN) FAIL_OVF("integer absolute"); \
 	OP_INT_ABS(x,r)
 
+#define OP_INT_TAG(x, r) \
+    r = (long)((unsigned long)x << 1); \
+    if ((r ^ x) < 0) FAIL_OVF("integer tagging"); \
+    r = r + 1
+
+#define OP_INT_UNTAG(x, r) \
+    r = x >> 1
+
 /***  binary operations ***/
 
 #define OP_INT_EQ(x,y,r)	  r = ((x) == (y))

From noreply at buildbot.pypy.org  Mon Oct 24 11:21:31 2011
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 24 Oct 2011 11:21:31 +0200 (CEST)
Subject: [pypy-commit] pypy int-tag-untag-as-operations: each the optimizer
 that int_tag and int_untag are each others opposites
Message-ID: <20111024092131.2CE6E820C7@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: int-tag-untag-as-operations
Changeset: r48365:52e92200bf93
Date: 2011-10-22 14:23 +0200
http://bitbucket.org/pypy/pypy/changeset/52e92200bf93/

Log:	each the optimizer that int_tag and int_untag are each others
	opposites

diff --git a/pypy/jit/metainterp/optimizeopt/intbounds.py b/pypy/jit/metainterp/optimizeopt/intbounds.py
--- a/pypy/jit/metainterp/optimizeopt/intbounds.py
+++ b/pypy/jit/metainterp/optimizeopt/intbounds.py
@@ -287,6 +287,13 @@
     def optimize_INT_TAG(self, op):
         self.emit_operation(op) # XXX for now
         self.emit_operation(self.nextop)
+        if self.nextop.getopnum() == rop.GUARD_NO_OVERFLOW:
+            self.pure(rop.INT_UNTAG, [op.result], op.getarg(0))
+
+    def optimize_INT_UNTAG(self, op):
+        self.emit_operation(op)
+        self.pure(rop.INT_TAG, [op.result], op.getarg(0))
+
 
     def optimize_ARRAYLEN_GC(self, op):
         self.emit_operation(op)
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -5074,6 +5074,40 @@
         """
         self.optimize_loop(ops, ops, ops)
 
+    def test_int_tag_untag_reverses(self):
+        ops = """
+        [i0]
+        i1 = int_tag(i0)
+        guard_no_overflow() []
+        i2 = int_untag(i1)
+        i3 = int_add(i2, 1)
+        jump(i3)
+        """
+        expected = """
+        [i0]
+        i1 = int_tag(i0)
+        guard_no_overflow() []
+        i2 = int_add(i0, 1)
+        jump(i2)
+        """
+        self.optimize_loop(ops, expected)
+        ops = """
+        [i0]
+        i1 = int_untag(i0)
+        i2 = int_tag(i1)
+        guard_no_overflow() []
+        i3 = int_untag(i2)
+        i4 = int_add(i3, 1)
+        jump(i4)
+        """
+        expected = """
+        [i0]
+        i1 = int_untag(i0)
+        i2 = int_add(i1, 1)
+        jump(i2)
+        """
+        self.optimize_loop(ops, expected)
+
 
     def test_mul_ovf(self):
         ops = """

From noreply at buildbot.pypy.org  Mon Oct 24 11:21:32 2011
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 24 Oct 2011 11:21:32 +0200 (CEST)
Subject: [pypy-commit] pypy int-tag-untag-as-operations: add a more precise
 .mul operation on integer bounds.
Message-ID: <20111024092132.54E42820C7@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: int-tag-untag-as-operations
Changeset: r48366:c499ddaefd52
Date: 2011-10-24 08:21 +0200
http://bitbucket.org/pypy/pypy/changeset/c499ddaefd52/

Log:	add a more precise .mul operation on integer bounds.

diff --git a/pypy/jit/metainterp/optimizeopt/intutils.py b/pypy/jit/metainterp/optimizeopt/intutils.py
--- a/pypy/jit/metainterp/optimizeopt/intutils.py
+++ b/pypy/jit/metainterp/optimizeopt/intutils.py
@@ -96,8 +96,32 @@
         return res
 
     def mul(self, value):
-        return self.mul_bound(IntBound(value, value))
-    
+        upper = 0
+        has_upper = False
+        if self.has_upper:
+            try:
+                upper = ovfcheck(self.upper * value)
+            except OverflowError:
+                pass
+            else:
+                has_upper = True
+        lower = 0
+        has_lower = False
+        if self.has_lower:
+            try:
+                lower = ovfcheck(self.lower * value)
+            except OverflowError:
+                pass
+            else:
+                has_lower = True
+        if value < 0:
+            has_upper, has_lower = has_lower, has_upper
+            upper, lower = lower, upper
+        result = IntBound(lower, upper)
+        result.has_lower = has_lower
+        result.has_upper = has_upper
+        return result
+
     def add_bound(self, other):
         res = self.clone()
         if other.has_upper:
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_intutils.py b/pypy/jit/metainterp/optimizeopt/test/test_intutils.py
new file mode 100644
--- /dev/null
+++ b/pypy/jit/metainterp/optimizeopt/test/test_intutils.py
@@ -0,0 +1,29 @@
+from pypy.jit.metainterp.optimizeopt import intutils
+
+# XXX this file should really be filled with tests for all operations!
+
+def test_mul_with_constant():
+    x = intutils.IntBound(0, 100)
+    y = x.mul(2)
+    assert y.has_lower
+    assert y.has_upper
+    assert y.lower == 0
+    assert y.upper == 200
+
+    y = x.mul(-5)
+    assert y.has_lower
+    assert y.has_upper
+    assert y.lower == -500
+    assert y.upper == 0
+
+    x = intutils.IntUpperBound(100)
+    y = x.mul(2)
+    assert not y.has_lower
+    assert y.has_upper
+    assert y.upper == 200
+
+    y = x.mul(-5)
+    assert y.has_lower
+    assert not y.has_upper
+    assert y.lower == -500
+    assert y.upper == 0

From noreply at buildbot.pypy.org  Mon Oct 24 11:21:33 2011
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 24 Oct 2011 11:21:33 +0200 (CEST)
Subject: [pypy-commit] pypy int-tag-untag-as-operations: check that
 operations on untagged integers get their overflow checks removed
Message-ID: <20111024092133.82572820C7@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: int-tag-untag-as-operations
Changeset: r48367:c7b823622f57
Date: 2011-10-24 08:35 +0200
http://bitbucket.org/pypy/pypy/changeset/c7b823622f57/

Log:	check that operations on untagged integers get their overflow checks
	removed

diff --git a/pypy/jit/metainterp/optimizeopt/intbounds.py b/pypy/jit/metainterp/optimizeopt/intbounds.py
--- a/pypy/jit/metainterp/optimizeopt/intbounds.py
+++ b/pypy/jit/metainterp/optimizeopt/intbounds.py
@@ -1,3 +1,4 @@
+import sys
 from pypy.jit.metainterp.optimizeopt.optimizer import Optimization, CONST_1, CONST_0, \
                                                   MODE_ARRAY, MODE_STR, MODE_UNICODE
 from pypy.jit.metainterp.history import ConstInt
@@ -286,14 +287,25 @@
 
     def optimize_INT_TAG(self, op):
         self.emit_operation(op) # XXX for now
-        self.emit_operation(self.nextop)
+        v1 = self.getvalue(op.getarg(0))
+        r = self.getvalue(op.result)
+        resbound = v1.intbound.mul(2).add(1)
+        r.intbound.intersect(resbound)
+        no_guard = False
         if self.nextop.getopnum() == rop.GUARD_NO_OVERFLOW:
+            maxbounds = IntBound((-sys.maxint-1) >> 1, sys.maxint >> 1)
+            v1.intbound.intersect(maxbounds)
             self.pure(rop.INT_UNTAG, [op.result], op.getarg(0))
+            no_guard = resbound.has_lower and resbound.has_upper
+        if not no_guard:
+            self.emit_operation(self.nextop)
 
     def optimize_INT_UNTAG(self, op):
+        v1 = self.getvalue(op.getarg(0))
+        self.pure(rop.INT_TAG, [op.result], op.getarg(0))
+        r = self.getvalue(op.result)
+        r.intbound.intersect(v1.intbound.rshift_bound(IntBound(1, 1)))
         self.emit_operation(op)
-        self.pure(rop.INT_TAG, [op.result], op.getarg(0))
-
 
     def optimize_ARRAYLEN_GC(self, op):
         self.emit_operation(op)
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -5108,6 +5108,32 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_int_tag_remove_overflow_checking(self):
+        ops = """
+        [i0]
+        i2 = int_tag(i0)
+        guard_no_overflow() []
+        i3 = int_untag(i2)
+        i4 = int_add_ovf(i3, 1)
+        guard_no_overflow() []
+        jump(i4)
+        """
+        expected = """
+        [i0]
+        i2 = int_tag(i0)
+        guard_no_overflow() []
+        i3 = int_add(i0, 1)
+        jump(i3)
+        """
+        preamble = """
+        [i0]
+        i2 = int_tag(i0)
+        guard_no_overflow() []
+        i3 = int_add(i0, 1)
+        jump(i3)
+        """
+        self.optimize_loop(ops, expected, preamble)
+
 
     def test_mul_ovf(self):
         ops = """

From noreply at buildbot.pypy.org  Mon Oct 24 11:21:34 2011
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 24 Oct 2011 11:21:34 +0200 (CEST)
Subject: [pypy-commit] pypy int-tag-untag-as-operations: rename the
	operation to int_tag_ovf
Message-ID: <20111024092134.C9C43820C7@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: int-tag-untag-as-operations
Changeset: r48368:766299f6042c
Date: 2011-10-24 09:01 +0200
http://bitbucket.org/pypy/pypy/changeset/766299f6042c/

Log:	rename the operation to int_tag_ovf

diff --git a/pypy/jit/backend/llgraph/llimpl.py b/pypy/jit/backend/llgraph/llimpl.py
--- a/pypy/jit/backend/llgraph/llimpl.py
+++ b/pypy/jit/backend/llgraph/llimpl.py
@@ -692,7 +692,7 @@
         if not flag:
             raise GuardFailed
 
-    def op_int_tag(self, _, x):
+    def op_int_tag_ovf(self, _, x):
         try:
             z = ovfcheck(x << 1) + 1
         except OverflowError:
diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -269,7 +269,7 @@
         op1 = SpaceOperation('-live-', [], None)
         return [op, op1]
 
-    def rewrite_op_int_tag(self, op):
+    def rewrite_op_int_tag_ovf(self, op):
         op1 = SpaceOperation('-live-', [], None)
         return [op, op1]
 
diff --git a/pypy/jit/metainterp/blackhole.py b/pypy/jit/metainterp/blackhole.py
--- a/pypy/jit/metainterp/blackhole.py
+++ b/pypy/jit/metainterp/blackhole.py
@@ -451,7 +451,7 @@
     def bhimpl_int_untag(a):
         return a >> 1
     @arguments("i", returns="i")
-    def bhimpl_int_tag(a):
+    def bhimpl_int_tag_ovf(a):
         return ovfcheck(a << 1) + 1
 
 
diff --git a/pypy/jit/metainterp/executor.py b/pypy/jit/metainterp/executor.py
--- a/pypy/jit/metainterp/executor.py
+++ b/pypy/jit/metainterp/executor.py
@@ -204,7 +204,7 @@
         z = 0
     return BoxInt(z)
 
-def do_int_tag(cpu, metainterp, box1):
+def do_int_tag_ovf(cpu, metainterp, box1):
     # the overflow operations can be called without a metainterp, if an
     # overflow cannot occur
     a = box1.getint()
diff --git a/pypy/jit/metainterp/optimizeopt/intbounds.py b/pypy/jit/metainterp/optimizeopt/intbounds.py
--- a/pypy/jit/metainterp/optimizeopt/intbounds.py
+++ b/pypy/jit/metainterp/optimizeopt/intbounds.py
@@ -285,7 +285,7 @@
         else:
             self.emit_operation(op)
 
-    def optimize_INT_TAG(self, op):
+    def optimize_INT_TAG_OVF(self, op):
         self.emit_operation(op) # XXX for now
         v1 = self.getvalue(op.getarg(0))
         r = self.getvalue(op.result)
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -5077,7 +5077,7 @@
     def test_int_tag_untag_reverses(self):
         ops = """
         [i0]
-        i1 = int_tag(i0)
+        i1 = int_tag_ovf(i0)
         guard_no_overflow() []
         i2 = int_untag(i1)
         i3 = int_add(i2, 1)
@@ -5085,7 +5085,7 @@
         """
         expected = """
         [i0]
-        i1 = int_tag(i0)
+        i1 = int_tag_ovf(i0)
         guard_no_overflow() []
         i2 = int_add(i0, 1)
         jump(i2)
@@ -5094,7 +5094,7 @@
         ops = """
         [i0]
         i1 = int_untag(i0)
-        i2 = int_tag(i1)
+        i2 = int_tag_ovf(i1)
         guard_no_overflow() []
         i3 = int_untag(i2)
         i4 = int_add(i3, 1)
@@ -5111,7 +5111,7 @@
     def test_int_tag_remove_overflow_checking(self):
         ops = """
         [i0]
-        i2 = int_tag(i0)
+        i2 = int_tag_ovf(i0)
         guard_no_overflow() []
         i3 = int_untag(i2)
         i4 = int_add_ovf(i3, 1)
@@ -5120,14 +5120,14 @@
         """
         expected = """
         [i0]
-        i2 = int_tag(i0)
+        i2 = int_tag_ovf(i0)
         guard_no_overflow() []
         i3 = int_add(i0, 1)
         jump(i3)
         """
         preamble = """
         [i0]
-        i2 = int_tag(i0)
+        i2 = int_tag_ovf(i0)
         guard_no_overflow() []
         i3 = int_add(i0, 1)
         jump(i3)
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -231,9 +231,9 @@
         ''' % (_opimpl, _opimpl.upper())).compile()
 
     @arguments("box")
-    def opimpl_int_tag(self, b1):
+    def opimpl_int_tag_ovf(self, b1):
         self.metainterp.clear_exception()
-        resbox = self.execute(rop.INT_TAG, b1)
+        resbox = self.execute(rop.INT_TAG_OVF, b1)
         self.make_result_of_lastop(resbox)
         if not isinstance(resbox, Const):
             self.metainterp.handle_possible_overflow_error()
diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py
--- a/pypy/jit/metainterp/resoperation.py
+++ b/pypy/jit/metainterp/resoperation.py
@@ -505,7 +505,7 @@
     'INT_ADD_OVF/2',
     'INT_SUB_OVF/2',
     'INT_MUL_OVF/2',
-    'INT_TAG/1',
+    'INT_TAG_OVF/1',
     '_OVF_LAST', # ----- end of is_ovf operations -----
     '_LAST',     # for the backend to add more internal operations
 ]
diff --git a/pypy/rlib/rerased.py b/pypy/rlib/rerased.py
--- a/pypy/rlib/rerased.py
+++ b/pypy/rlib/rerased.py
@@ -220,7 +220,7 @@
         [v_value] = hop.inputargs(lltype.Signed)
         c_one = hop.inputconst(lltype.Signed, 1)
         hop.exception_is_here()
-        v2 = hop.genop('int_tag', [v_value],
+        v2 = hop.genop('int_tag_ovf', [v_value],
                        resulttype = lltype.Signed)
         v_instance = hop.genop('cast_int_to_ptr', [v2],
                                resulttype=self.lowleveltype)
diff --git a/pypy/rpython/llinterp.py b/pypy/rpython/llinterp.py
--- a/pypy/rpython/llinterp.py
+++ b/pypy/rpython/llinterp.py
@@ -1095,7 +1095,7 @@
             assert y >= 0
         return self.op_int_add_ovf(x, y)
 
-    def op_int_tag(self, x):
+    def op_int_tag_ovf(self, x):
         try:
             return ovfcheck(x + x + 1)
         except OverflowError:
diff --git a/pypy/rpython/lltypesystem/lloperation.py b/pypy/rpython/lltypesystem/lloperation.py
--- a/pypy/rpython/lltypesystem/lloperation.py
+++ b/pypy/rpython/lltypesystem/lloperation.py
@@ -207,7 +207,7 @@
     'int_abs':              LLOp(canfold=True),
     'int_abs_ovf':          LLOp(canraise=(OverflowError,), tryfold=True),
     'int_invert':           LLOp(canfold=True),
-    'int_tag':              LLOp(canraise=(OverflowError, ), tryfold=True),
+    'int_tag_ovf':          LLOp(canraise=(OverflowError, ), tryfold=True),
     'int_untag':            LLOp(canfold=True),
 
     'int_add':              LLOp(canfold=True),
diff --git a/pypy/translator/c/src/int.h b/pypy/translator/c/src/int.h
--- a/pypy/translator/c/src/int.h
+++ b/pypy/translator/c/src/int.h
@@ -19,11 +19,10 @@
 	if ((x) == LONG_MIN) FAIL_OVF("integer absolute"); \
 	OP_INT_ABS(x,r)
 
-#define OP_INT_TAG(x, r) \
+#define OP_INT_TAG_OVF(x, r) \
     r = (long)((unsigned long)x << 1); \
     if ((r ^ x) < 0) FAIL_OVF("integer tagging"); \
     r = r + 1
-
 #define OP_INT_UNTAG(x, r) \
     r = x >> 1
 

From noreply at buildbot.pypy.org  Mon Oct 24 11:21:36 2011
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 24 Oct 2011 11:21:36 +0200 (CEST)
Subject: [pypy-commit] pypy int-tag-untag-as-operations: introduce an
 no-overflow-check variant int_tag for when the bounds-optimizer
Message-ID: <20111024092136.074AF820C7@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: int-tag-untag-as-operations
Changeset: r48369:1a183750ebf7
Date: 2011-10-24 09:07 +0200
http://bitbucket.org/pypy/pypy/changeset/1a183750ebf7/

Log:	introduce an no-overflow-check variant int_tag for when the bounds-
	optimizer finds out that something cannot overflow.

diff --git a/pypy/jit/metainterp/blackhole.py b/pypy/jit/metainterp/blackhole.py
--- a/pypy/jit/metainterp/blackhole.py
+++ b/pypy/jit/metainterp/blackhole.py
@@ -451,6 +451,9 @@
     def bhimpl_int_untag(a):
         return a >> 1
     @arguments("i", returns="i")
+    def bhimpl_int_tag(a):
+        return (a << 1) + 1
+    @arguments("i", returns="i")
     def bhimpl_int_tag_ovf(a):
         return ovfcheck(a << 1) + 1
 
diff --git a/pypy/jit/metainterp/optimizeopt/intbounds.py b/pypy/jit/metainterp/optimizeopt/intbounds.py
--- a/pypy/jit/metainterp/optimizeopt/intbounds.py
+++ b/pypy/jit/metainterp/optimizeopt/intbounds.py
@@ -286,7 +286,6 @@
             self.emit_operation(op)
 
     def optimize_INT_TAG_OVF(self, op):
-        self.emit_operation(op) # XXX for now
         v1 = self.getvalue(op.getarg(0))
         r = self.getvalue(op.result)
         resbound = v1.intbound.mul(2).add(1)
@@ -297,9 +296,22 @@
             v1.intbound.intersect(maxbounds)
             self.pure(rop.INT_UNTAG, [op.result], op.getarg(0))
             no_guard = resbound.has_lower and resbound.has_upper
-        if not no_guard:
+        if no_guard:
+            op = op.copy_and_change(rop.INT_TAG)
+            self.optimize_INT_TAG(op) # emit the op
+        else:
+            self.emit_operation(op)
             self.emit_operation(self.nextop)
 
+    def optimize_INT_TAG(self, op):
+        v1 = self.getvalue(op.getarg(0))
+        r = self.getvalue(op.result)
+        resbound = v1.intbound.mul(2).add(1)
+        r.intbound.intersect(resbound)
+        maxbounds = IntBound((-sys.maxint-1) >> 1, sys.maxint >> 1)
+        v1.intbound.intersect(maxbounds)
+        self.pure(rop.INT_UNTAG, [op.result], op.getarg(0))
+
     def optimize_INT_UNTAG(self, op):
         v1 = self.getvalue(op.getarg(0))
         self.pure(rop.INT_TAG, [op.result], op.getarg(0))
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -5111,6 +5111,8 @@
     def test_int_tag_remove_overflow_checking(self):
         ops = """
         [i0]
+        i1 = int_lt(i0, 1000)
+        guard_true(i1), []
         i2 = int_tag_ovf(i0)
         guard_no_overflow() []
         i3 = int_untag(i2)
@@ -5120,13 +5122,15 @@
         """
         expected = """
         [i0]
-        i2 = int_tag_ovf(i0)
-        guard_no_overflow() []
+        i1 = int_lt(i0, 1000)
+        guard_true(i1), []
         i3 = int_add(i0, 1)
         jump(i3)
         """
         preamble = """
         [i0]
+        i1 = int_lt(i0, 1000)
+        guard_true(i1), []
         i2 = int_tag_ovf(i0)
         guard_no_overflow() []
         i3 = int_add(i0, 1)
diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py
--- a/pypy/jit/metainterp/resoperation.py
+++ b/pypy/jit/metainterp/resoperation.py
@@ -432,6 +432,7 @@
     'INT_NEG/1',
     'INT_INVERT/1',
     'INT_UNTAG/1',
+    'INT_TAG/1',
     #
     'SAME_AS/1',      # gets a Const or a Box, turns it into another Box
     'CAST_PTR_TO_INT/1',

From noreply at buildbot.pypy.org  Mon Oct 24 12:41:57 2011
From: noreply at buildbot.pypy.org (cfbolz)
Date: Mon, 24 Oct 2011 12:41:57 +0200 (CEST)
Subject: [pypy-commit] pypy int-tag-untag-as-operations: (arigo,
 cfbolz): implement int_tag, int_tag_ovf and int_untag in the x86 backend
Message-ID: <20111024104157.62378820C7@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: int-tag-untag-as-operations
Changeset: r48370:30319db8738f
Date: 2011-10-24 12:41 +0200
http://bitbucket.org/pypy/pypy/changeset/30319db8738f/

Log:	(arigo, cfbolz): implement int_tag, int_tag_ovf and int_untag in the
	x86 backend

diff --git a/pypy/jit/backend/test/test_random.py b/pypy/jit/backend/test/test_random.py
--- a/pypy/jit/backend/test/test_random.py
+++ b/pypy/jit/backend/test/test_random.py
@@ -293,6 +293,9 @@
 class BinaryOvfOperation(AbstractOvfOperation, BinaryOperation):
     pass
 
+class UnaryOvfOperation(AbstractOvfOperation, UnaryOperation):
+    pass
+
 class AbstractFloatOperation(AbstractOperation):
     def filter(self, builder):
         if not builder.cpu.supports_floats:
@@ -421,6 +424,8 @@
 
 for _op in [rop.INT_NEG,
             rop.INT_INVERT,
+            rop.INT_TAG,
+            rop.INT_UNTAG,
             ]:
     OPERATIONS.append(UnaryOperation(_op))
 
@@ -433,6 +438,7 @@
             rop.INT_MUL_OVF,
             ]:
     OPERATIONS.append(BinaryOvfOperation(_op))
+OPERATIONS.append(UnaryOvfOperation(rop.INT_TAG_OVF))
 
 for _op in [rop.FLOAT_ADD,
             rop.FLOAT_SUB,
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1311,7 +1311,19 @@
     genop_guard_float_eq = _cmpop_guard_float("E", "E", "NE","NE")
     genop_guard_float_gt = _cmpop_guard_float("A", "B", "BE","AE")
     genop_guard_float_ge = _cmpop_guard_float("AE","BE", "B", "A")
-    
+
+
+    def genop_int_tag(self, op, arglocs, resloc):
+        loc, = arglocs
+        assert isinstance(loc, RegLoc)
+        assert isinstance(resloc, RegLoc)
+        # res = loc + (loc << 0) + 1
+        self.mc.LEA_ra(resloc.value, (loc.value, loc.value, 0, 1))
+
+    def genop_int_untag(self, op, arglocs, resloc):
+        loc, = arglocs
+        self.mc.SAR(loc, imm1)
+
     def genop_math_sqrt(self, op, arglocs, resloc):
         self.mc.SQRTSD(arglocs[0], resloc)
 
@@ -1730,6 +1742,11 @@
         self.genop_int_mul(op, arglocs, result_loc)
         return self._gen_guard_overflow(guard_op, guard_token)
 
+    def genop_guard_int_tag_ovf(self, op, guard_op, guard_token, arglocs, result_loc):
+        self.mc.STC()
+        self.mc.ADC(arglocs[0], arglocs[0])
+        return self._gen_guard_overflow(guard_op, guard_token)
+
     def genop_guard_guard_false(self, ign_1, guard_op, guard_token, locs, ign_2):
         loc = locs[0]
         self.mc.TEST(loc, loc)
diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py
--- a/pypy/jit/backend/x86/regalloc.py
+++ b/pypy/jit/backend/x86/regalloc.py
@@ -579,11 +579,22 @@
     consider_int_sub_ovf = _consider_binop_with_guard
     consider_int_add_ovf = _consider_binop_with_guard
 
+    def consider_int_tag_ovf(self, op, guard_op):
+        loc = self.rm.force_result_in_reg(op.result, op.getarg(0))
+        self.perform_with_guard(op, guard_op, [loc], loc)
+
+    def consider_int_tag(self, op):
+        loc = self.rm.make_sure_var_in_reg(op.getarg(0))
+        self.rm.possibly_free_vars_for_op(op)
+        res = self.rm.force_allocate_reg(op.result)
+        self.Perform(op, [loc], res)
+
     def consider_int_neg(self, op):
         res = self.rm.force_result_in_reg(op.result, op.getarg(0))
         self.Perform(op, [res], res)
 
     consider_int_invert = consider_int_neg
+    consider_int_untag = consider_int_neg
 
     def consider_int_lshift(self, op):
         if isinstance(op.getarg(1), Const):
diff --git a/pypy/jit/backend/x86/regloc.py b/pypy/jit/backend/x86/regloc.py
--- a/pypy/jit/backend/x86/regloc.py
+++ b/pypy/jit/backend/x86/regloc.py
@@ -487,6 +487,7 @@
     BTS = _binaryop('BTS')
 
     ADD = _binaryop('ADD')
+    ADC = _binaryop('ADC')
     SUB = _binaryop('SUB')
     IMUL = _binaryop('IMUL')
     NEG = _unaryop('NEG')
diff --git a/pypy/jit/backend/x86/rx86.py b/pypy/jit/backend/x86/rx86.py
--- a/pypy/jit/backend/x86/rx86.py
+++ b/pypy/jit/backend/x86/rx86.py
@@ -470,6 +470,7 @@
     # ------------------------------ Arithmetic ------------------------------
 
     ADD_ri,ADD_rr,ADD_rb,_,_,ADD_rm,ADD_rj,_,_ = common_modes(0)
+    ADC_ri,ADC_rr,ADC_rb,_,_,ADC_rm,ADC_rj,_,_ = common_modes(2)
     OR_ri, OR_rr, OR_rb, _,_,OR_rm, OR_rj, _,_ = common_modes(1)
     AND_ri,AND_rr,AND_rb,_,_,AND_rm,AND_rj,_,_ = common_modes(4)
     SUB_ri,SUB_rr,SUB_rb,_,_,SUB_rm,SUB_rj,SUB_ji8,SUB_mi8 = common_modes(5)
@@ -577,6 +578,8 @@
     FSTPL_b = insn('\xDD', orbyte(3<<3), stack_bp(1)) # rffi.DOUBLE ('as' wants L??)
     FSTPS_s = insn('\xD9', orbyte(3<<3), stack_sp(1)) # lltype.SingleFloat
 
+    STC = insn('\xF9')
+
     # ------------------------------ Random mess -----------------------
     RDTSC = insn('\x0F\x31')
 
diff --git a/pypy/jit/backend/x86/test/test_regalloc.py b/pypy/jit/backend/x86/test/test_regalloc.py
--- a/pypy/jit/backend/x86/test/test_regalloc.py
+++ b/pypy/jit/backend/x86/test/test_regalloc.py
@@ -660,3 +660,27 @@
         self.run(loop)
         assert self.getint(0) == 29
 
+    def test_int_untag(self):
+        ops = '''
+        [i0]
+        i1 = int_untag(i0)
+        finish(i1)
+        '''
+        self.interpret(ops, [1129])
+        assert self.getint(0) == 564
+        self.interpret(ops, [-1129])
+        assert self.getint(0) == -565
+
+    def test_int_tag(self):
+        ops = '''
+        [i0]
+        i1 = int_tag(i0)
+        i2 = int_tag(i0)
+        finish(i1, i2)
+        '''
+        self.interpret(ops, [1129])
+        assert self.getint(0) == 1129 * 2 + 1
+        assert self.getint(1) == 1129 * 2 + 1
+        self.interpret(ops, [-1129])
+        assert self.getint(0) == -1129 * 2 + 1
+        assert self.getint(1) == -1129 * 2 + 1
diff --git a/pypy/jit/metainterp/blackhole.py b/pypy/jit/metainterp/blackhole.py
--- a/pypy/jit/metainterp/blackhole.py
+++ b/pypy/jit/metainterp/blackhole.py
@@ -452,7 +452,7 @@
         return a >> 1
     @arguments("i", returns="i")
     def bhimpl_int_tag(a):
-        return (a << 1) + 1
+        return intmask(a << 1) + 1 # mostly there for test_random
     @arguments("i", returns="i")
     def bhimpl_int_tag_ovf(a):
         return ovfcheck(a << 1) + 1
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -3435,10 +3435,7 @@
             return sa
         res = self.meta_interp(f, [16])
         assert res == f(16)
-        
-
-
-class TestLLtype(BaseLLtypeTests, LLJitMixin):
+
     def test_tagged(self):
         from pypy.rlib.objectmodel import UnboxedValue
         class Base(object):
@@ -3523,3 +3520,8 @@
         assert x == -42
         x = self.interp_operations(f, [1000, 1], taggedpointers=True)
         assert x == 999
+
+
+
+class TestLLtype(BaseLLtypeTests, LLJitMixin):
+    pass # should be empty

From noreply at buildbot.pypy.org  Mon Oct 24 14:43:42 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 24 Oct 2011 14:43:42 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Reorder the tests for
	performance.
Message-ID: <20111024124342.3213E820C7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r48371:d5d9c2cef342
Date: 2011-10-24 10:53 +0200
http://bitbucket.org/pypy/pypy/changeset/d5d9c2cef342/

Log:	Reorder the tests for performance.

diff --git a/pypy/rpython/memory/gc/concurrentgen.py b/pypy/rpython/memory/gc/concurrentgen.py
--- a/pypy/rpython/memory/gc/concurrentgen.py
+++ b/pypy/rpython/memory/gc/concurrentgen.py
@@ -48,7 +48,6 @@
 MARK_BYTE_OLD_2   = 0x2F    # '/', 47
 MARK_BYTE_STATIC  = 0x35    # '5', 53
 mark_byte_is_old  = lambda n: n <= MARK_BYTE_OLD_2
-mark_byte_is_old_or_static = lambda n: n <= MARK_BYTE_STATIC
 # Next lower byte: a combination of flags.
 FL_WITHHASH       = 0x0100
 FL_EXTRA          = 0x0200
@@ -320,14 +319,15 @@
             mark = self.get_mark(obj)
             #debug_print("deletion_barrier:", mark, obj)
             #
-            if mark_byte_is_old_or_static(mark):
+            if mark_byte_is_old(mark):     # most common case, make it fast
                 #
                 self.set_mark(obj, cym)
                 #
-                if mark == MARK_BYTE_STATIC:
-                    # This is the first write into a prebuilt GC object.
-                    # Record it in 'prebuilt_root_objects'.
-                    self.prebuilt_root_objects.append(obj)
+            elif mark == MARK_BYTE_STATIC:
+                # This is the first write into a prebuilt GC object.
+                # Record it in 'prebuilt_root_objects'.
+                self.set_mark(obj, cym)
+                self.prebuilt_root_objects.append(obj)
                 #
             else:
                 #

From noreply at buildbot.pypy.org  Mon Oct 24 14:43:43 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 24 Oct 2011 14:43:43 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix the declaration of the lloperation
 'cast_float_to_int' to
Message-ID: <20111024124343.6AED2820C7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48372:4241a19c0141
Date: 2011-10-24 14:35 +0200
http://bitbucket.org/pypy/pypy/changeset/4241a19c0141/

Log:	Fix the declaration of the lloperation 'cast_float_to_int' to no
	longer pretend it raises OverflowError, which is wrong.

	Fix the JIT by refusing to jtransform operationsl like
	'cast_float_to_uint' and 'cast_uint_to_float' and their longlong and
	ulonglong equivalents. These would each require their own custom
	messy code in the backends. (If really needed, they are probably
	better written as '_ll_1_cast_x_to_y' support functions.)

diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -847,6 +847,12 @@
             return self._float_to_float_cast(v_arg, v_result)
         elif not float_arg and float_res:
             # some int -> some float
+            size1, unsigned1 = rffi.size_and_sign(v_arg.concretetype)
+            assert size1 <= rffi.sizeof(lltype.Signed), (
+                "not implemented: cast_longlong_to_float")
+            assert size1 < rffi.sizeof(lltype.Signed) or not unsigned1, (
+                "not implemented: cast_uint_to_float")
+            #
             ops = []
             v1 = varoftype(lltype.Signed)
             oplist = self.rewrite_operation(
@@ -871,6 +877,12 @@
             return ops
         elif float_arg and not float_res:
             # some float -> some int
+            size2, unsigned2 = rffi.size_and_sign(v_result.concretetype)
+            assert size2 <= rffi.sizeof(lltype.Signed), (
+                "not implemented: cast_float_to_longlong")
+            assert size2 < rffi.sizeof(lltype.Signed) or not unsigned2, (
+                "not implemented: cast_float_to_uint")
+            #
             ops = []
             v1 = varoftype(lltype.Float)
             op1 = self.rewrite_operation(
@@ -1097,8 +1109,8 @@
     # The new operation is optionally further processed by rewrite_operation().
     for _old, _new in [('bool_not', 'int_is_zero'),
                        ('cast_bool_to_float', 'cast_int_to_float'),
-                       ('cast_uint_to_float', 'cast_int_to_float'),
-                       ('cast_float_to_uint', 'cast_float_to_int'),
+                       ('cast_uint_to_float', 'cast_primitive'),
+                       ('cast_float_to_uint', 'cast_primitive'),
 
                        ('int_add_nonneg_ovf', 'int_add_ovf'),
                        ('keepalive', '-live-'),
diff --git a/pypy/jit/metainterp/blackhole.py b/pypy/jit/metainterp/blackhole.py
--- a/pypy/jit/metainterp/blackhole.py
+++ b/pypy/jit/metainterp/blackhole.py
@@ -630,6 +630,9 @@
         a = longlong.getrealfloat(a)
         # note: we need to call int() twice to care for the fact that
         # int(-2147483648.0) returns a long :-(
+        # we could also call intmask() instead of the outermost int(), but
+        # it's probably better to explicitly crash (by getting a long) if a
+        # non-translated version tries to cast a too large float to an int.
         return int(int(a))
 
     @arguments("i", returns="f")
diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py
--- a/pypy/jit/metainterp/resoperation.py
+++ b/pypy/jit/metainterp/resoperation.py
@@ -404,8 +404,8 @@
     'FLOAT_TRUEDIV/2',
     'FLOAT_NEG/1',
     'FLOAT_ABS/1',
-    'CAST_FLOAT_TO_INT/1',
-    'CAST_INT_TO_FLOAT/1',
+    'CAST_FLOAT_TO_INT/1',          # don't use for unsigned ints; we would
+    'CAST_INT_TO_FLOAT/1',          # need some messy code in the backend
     'CAST_FLOAT_TO_SINGLEFLOAT/1',
     'CAST_SINGLEFLOAT_TO_FLOAT/1',
     #
diff --git a/pypy/jit/metainterp/test/test_float.py b/pypy/jit/metainterp/test/test_float.py
--- a/pypy/jit/metainterp/test/test_float.py
+++ b/pypy/jit/metainterp/test/test_float.py
@@ -1,5 +1,6 @@
-import math
+import math, sys
 from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin
+from pypy.rlib.rarithmetic import intmask, r_uint
 
 
 class FloatTests:
@@ -45,6 +46,41 @@
         res = self.interp_operations(f, [-2.0])
         assert res == -8.5
 
+    def test_cast_float_to_int(self):
+        def g(f):
+            return int(f)
+        res = self.interp_operations(g, [-12345.9])
+        assert res == -12345
+
+    def test_cast_float_to_uint(self):
+        def g(f):
+            return intmask(r_uint(f))
+        raises(AssertionError, self.interp_operations, g, [0.0])   # for now
+        #res = self.interp_operations(g, [sys.maxint*2.0])
+        #assert res == intmask(long(sys.maxint*2.0))
+        #res = self.interp_operations(g, [-12345.9])
+        #assert res == -12345
+
+    def test_cast_int_to_float(self):
+        def g(i):
+            return float(i)
+        res = self.interp_operations(g, [-12345])
+        assert type(res) is float and res == -12345.0
+
+    def test_cast_uint_to_float(self):
+        def g(i):
+            return float(r_uint(i))
+        raises(AssertionError, self.interp_operations, g, [0])    # for now
+        #res = self.interp_operations(g, [sys.maxint*2])
+        #assert type(res) is float and res == float(sys.maxint*2)
+        #res = self.interp_operations(g, [-12345])
+        #assert type(res) is float and res == float(long(r_uint(-12345)))
+
+    #def test_cast_longlong_to_float(self):
+    #def test_cast_ulonglong_to_float(self):
+    #def test_cast_float_to_longlong(self):
+    #def test_cast_float_to_ulonglong(self):
+
 
 class TestOOtype(FloatTests, OOJitMixin):
     pass
diff --git a/pypy/rpython/llinterp.py b/pypy/rpython/llinterp.py
--- a/pypy/rpython/llinterp.py
+++ b/pypy/rpython/llinterp.py
@@ -1095,13 +1095,6 @@
             assert y >= 0
         return self.op_int_add_ovf(x, y)
 
-    def op_cast_float_to_int(self, f):
-        assert type(f) is float
-        try:
-            return ovfcheck(int(f))
-        except OverflowError:
-            self.make_llexception()
-
     def op_int_is_true(self, x):
         # special case
         if type(x) is CDefinedIntSymbolic:
diff --git a/pypy/rpython/lltypesystem/lloperation.py b/pypy/rpython/lltypesystem/lloperation.py
--- a/pypy/rpython/lltypesystem/lloperation.py
+++ b/pypy/rpython/lltypesystem/lloperation.py
@@ -343,8 +343,8 @@
     'cast_uint_to_float':   LLOp(canfold=True),
     'cast_longlong_to_float' :LLOp(canfold=True),
     'cast_ulonglong_to_float':LLOp(canfold=True),
-    'cast_float_to_int':    LLOp(canraise=(OverflowError,), tryfold=True),
-    'cast_float_to_uint':   LLOp(canfold=True),    # XXX need OverflowError?
+    'cast_float_to_int':    LLOp(canfold=True),
+    'cast_float_to_uint':   LLOp(canfold=True),
     'cast_float_to_longlong' :LLOp(canfold=True),
     'cast_float_to_ulonglong':LLOp(canfold=True),
     'truncate_longlong_to_int':LLOp(canfold=True),
diff --git a/pypy/rpython/lltypesystem/opimpl.py b/pypy/rpython/lltypesystem/opimpl.py
--- a/pypy/rpython/lltypesystem/opimpl.py
+++ b/pypy/rpython/lltypesystem/opimpl.py
@@ -355,6 +355,10 @@
     assert type(b) is bool
     return float(b)
 
+def op_cast_float_to_int(f):
+    assert type(f) is float
+    return intmask(int(f))
+
 def op_cast_float_to_uint(f):
     assert type(f) is float
     return r_uint(long(f))

From noreply at buildbot.pypy.org  Mon Oct 24 14:55:11 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 24 Oct 2011 14:55:11 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: hg merge default
Message-ID: <20111024125511.580CD820C7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r48373:503589dfa9f4
Date: 2011-10-24 14:46 +0200
http://bitbucket.org/pypy/pypy/changeset/503589dfa9f4/

Log:	hg merge default

diff --git a/lib-python/modified-2.7/json/encoder.py b/lib-python/modified-2.7/json/encoder.py
--- a/lib-python/modified-2.7/json/encoder.py
+++ b/lib-python/modified-2.7/json/encoder.py
@@ -2,14 +2,7 @@
 """
 import re
 
-try:
-    from _json import encode_basestring_ascii as c_encode_basestring_ascii
-except ImportError:
-    c_encode_basestring_ascii = None
-try:
-    from _json import make_encoder as c_make_encoder
-except ImportError:
-    c_make_encoder = None
+from __pypy__.builders import StringBuilder, UnicodeBuilder
 
 ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]')
 ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])')
@@ -24,8 +17,7 @@
     '\t': '\\t',
 }
 for i in range(0x20):
-    ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i))
-    #ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,))
+    ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,))
 
 # Assume this produces an infinity on all machines (probably not guaranteed)
 INFINITY = float('1e66666')
@@ -37,10 +29,9 @@
     """
     def replace(match):
         return ESCAPE_DCT[match.group(0)]
-    return '"' + ESCAPE.sub(replace, s) + '"'
+    return ESCAPE.sub(replace, s)
 
-
-def py_encode_basestring_ascii(s):
+def encode_basestring_ascii(s):
     """Return an ASCII-only JSON representation of a Python string
 
     """
@@ -53,20 +44,18 @@
         except KeyError:
             n = ord(s)
             if n < 0x10000:
-                return '\\u{0:04x}'.format(n)
-                #return '\\u%04x' % (n,)
+                return '\\u%04x' % (n,)
             else:
                 # surrogate pair
                 n -= 0x10000
                 s1 = 0xd800 | ((n >> 10) & 0x3ff)
                 s2 = 0xdc00 | (n & 0x3ff)
-                return '\\u{0:04x}\\u{1:04x}'.format(s1, s2)
-                #return '\\u%04x\\u%04x' % (s1, s2)
-    return '"' + str(ESCAPE_ASCII.sub(replace, s)) + '"'
-
-
-encode_basestring_ascii = (
-    c_encode_basestring_ascii or py_encode_basestring_ascii)
+                return '\\u%04x\\u%04x' % (s1, s2)
+    if ESCAPE_ASCII.search(s):
+        return str(ESCAPE_ASCII.sub(replace, s))
+    return s
+py_encode_basestring_ascii = lambda s: '"' + encode_basestring_ascii(s) + '"'
+c_encode_basestring_ascii = None
 
 class JSONEncoder(object):
     """Extensible JSON <http://json.org> encoder for Python data structures.
@@ -147,6 +136,17 @@
 
         self.skipkeys = skipkeys
         self.ensure_ascii = ensure_ascii
+        if ensure_ascii:
+            self.encoder = encode_basestring_ascii
+        else:
+            self.encoder = encode_basestring
+        if encoding != 'utf-8':
+            orig_encoder = self.encoder
+            def encoder(o):
+                if isinstance(o, str):
+                    o = o.decode(encoding)
+                return orig_encoder(o)
+            self.encoder = encoder
         self.check_circular = check_circular
         self.allow_nan = allow_nan
         self.sort_keys = sort_keys
@@ -184,24 +184,126 @@
         '{"foo": ["bar", "baz"]}'
 
         """
-        # This is for extremely simple cases and benchmarks.
+        if self.check_circular:
+            markers = {}
+        else:
+            markers = None
+        if self.ensure_ascii:
+            builder = StringBuilder()
+        else:
+            builder = UnicodeBuilder()
+        self._encode(o, markers, builder, 0)
+        return builder.build()
+
+    def _emit_indent(self, builder, _current_indent_level):
+        if self.indent is not None:
+            _current_indent_level += 1
+            newline_indent = '\n' + (' ' * (self.indent *
+                                            _current_indent_level))
+            separator = self.item_separator + newline_indent
+            builder.append(newline_indent)
+        else:
+            separator = self.item_separator
+        return separator, _current_indent_level
+
+    def _emit_unindent(self, builder, _current_indent_level):
+        if self.indent is not None:
+            builder.append('\n')
+            builder.append(' ' * (self.indent * (_current_indent_level - 1)))
+
+    def _encode(self, o, markers, builder, _current_indent_level):
         if isinstance(o, basestring):
-            if isinstance(o, str):
-                _encoding = self.encoding
-                if (_encoding is not None
-                        and not (_encoding == 'utf-8')):
-                    o = o.decode(_encoding)
-            if self.ensure_ascii:
-                return encode_basestring_ascii(o)
+            builder.append('"')
+            builder.append(self.encoder(o))
+            builder.append('"')
+        elif o is None:
+            builder.append('null')
+        elif o is True:
+            builder.append('true')
+        elif o is False:
+            builder.append('false')
+        elif isinstance(o, (int, long)):
+            builder.append(str(o))
+        elif isinstance(o, float):
+            builder.append(self._floatstr(o))
+        elif isinstance(o, (list, tuple)):
+            if not o:
+                builder.append('[]')
+                return
+            self._encode_list(o, markers, builder, _current_indent_level)
+        elif isinstance(o, dict):
+            if not o:
+                builder.append('{}')
+                return
+            self._encode_dict(o, markers, builder, _current_indent_level)
+        else:
+            self._mark_markers(markers, o)
+            res = self.default(o)
+            self._encode(res, markers, builder, _current_indent_level)
+            self._remove_markers(markers, o)
+            return res
+
+    def _encode_list(self, l, markers, builder, _current_indent_level):
+        self._mark_markers(markers, l)
+        builder.append('[')
+        first = True
+        separator, _current_indent_level = self._emit_indent(builder,
+                                                      _current_indent_level)
+        for elem in l:
+            if first:
+                first = False
             else:
-                return encode_basestring(o)
-        # This doesn't pass the iterator directly to ''.join() because the
-        # exceptions aren't as detailed.  The list call should be roughly
-        # equivalent to the PySequence_Fast that ''.join() would do.
-        chunks = self.iterencode(o, _one_shot=True)
-        if not isinstance(chunks, (list, tuple)):
-            chunks = list(chunks)
-        return ''.join(chunks)
+                builder.append(separator)
+            self._encode(elem, markers, builder, _current_indent_level)
+            del elem # XXX grumble
+        self._emit_unindent(builder, _current_indent_level)
+        builder.append(']')
+        self._remove_markers(markers, l)
+
+    def _encode_dict(self, d, markers, builder, _current_indent_level):
+        self._mark_markers(markers, d)
+        first = True
+        builder.append('{')
+        separator, _current_indent_level = self._emit_indent(builder,
+                                                         _current_indent_level)
+        if self.sort_keys:
+            items = sorted(d.items(), key=lambda kv: kv[0])
+        else:
+            items = d.iteritems()
+
+        for key, v in items:
+            if first:
+                first = False
+            else:
+                builder.append(separator)
+            if isinstance(key, basestring):
+                pass
+            # JavaScript is weakly typed for these, so it makes sense to
+            # also allow them.  Many encoders seem to do something like this.
+            elif isinstance(key, float):
+                key = self._floatstr(key)
+            elif key is True:
+                key = 'true'
+            elif key is False:
+                key = 'false'
+            elif key is None:
+                key = 'null'
+            elif isinstance(key, (int, long)):
+                key = str(key)
+            elif self.skipkeys:
+                continue
+            else:
+                raise TypeError("key " + repr(key) + " is not a string")
+            builder.append('"')
+            builder.append(self.encoder(key))
+            builder.append('"')
+            builder.append(self.key_separator)
+            self._encode(v, markers, builder, _current_indent_level)
+            del key
+            del v # XXX grumble
+        self._emit_unindent(builder, _current_indent_level)
+        builder.append('}')
+        self._remove_markers(markers, d)
 
     def iterencode(self, o, _one_shot=False):
         """Encode the given object and yield each string
@@ -217,86 +319,54 @@
             markers = {}
         else:
             markers = None
-        if self.ensure_ascii:
-            _encoder = encode_basestring_ascii
+        return self._iterencode(o, markers, 0)
+
+    def _floatstr(self, o):
+        # Check for specials.  Note that this type of test is processor
+        # and/or platform-specific, so do tests which don't depend on the
+        # internals.
+
+        if o != o:
+            text = 'NaN'
+        elif o == INFINITY:
+            text = 'Infinity'
+        elif o == -INFINITY:
+            text = '-Infinity'
         else:
-            _encoder = encode_basestring
-        if self.encoding != 'utf-8':
-            def _encoder(o, _orig_encoder=_encoder, _encoding=self.encoding):
-                if isinstance(o, str):
-                    o = o.decode(_encoding)
-                return _orig_encoder(o)
+            return FLOAT_REPR(o)
 
-        def floatstr(o, allow_nan=self.allow_nan,
-                _repr=FLOAT_REPR, _inf=INFINITY, _neginf=-INFINITY):
-            # Check for specials.  Note that this type of test is processor
-            # and/or platform-specific, so do tests which don't depend on the
-            # internals.
+        if not self.allow_nan:
+            raise ValueError(
+                "Out of range float values are not JSON compliant: " +
+                repr(o))
 
-            if o != o:
-                text = 'NaN'
-            elif o == _inf:
-                text = 'Infinity'
-            elif o == _neginf:
-                text = '-Infinity'
-            else:
-                return _repr(o)
+        return text
 
-            if not allow_nan:
-                raise ValueError(
-                    "Out of range float values are not JSON compliant: " +
-                    repr(o))
+    def _mark_markers(self, markers, o):
+        if markers is not None:
+            if id(o) in markers:
+                raise ValueError("Circular reference detected")
+            markers[id(o)] = None
 
-            return text
+    def _remove_markers(self, markers, o):
+        if markers is not None:
+            del markers[id(o)]
 
-
-        if (_one_shot and c_make_encoder is not None
-                and not self.indent and not self.sort_keys):
-            _iterencode = c_make_encoder(
-                markers, self.default, _encoder, self.indent,
-                self.key_separator, self.item_separator, self.sort_keys,
-                self.skipkeys, self.allow_nan)
-        else:
-            _iterencode = _make_iterencode(
-                markers, self.default, _encoder, self.indent, floatstr,
-                self.key_separator, self.item_separator, self.sort_keys,
-                self.skipkeys, _one_shot)
-        return _iterencode(o, 0)
-
-def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,
-        _key_separator, _item_separator, _sort_keys, _skipkeys, _one_shot,
-        ## HACK: hand-optimized bytecode; turn globals into locals
-        ValueError=ValueError,
-        basestring=basestring,
-        dict=dict,
-        float=float,
-        id=id,
-        int=int,
-        isinstance=isinstance,
-        list=list,
-        long=long,
-        str=str,
-        tuple=tuple,
-    ):
-
-    def _iterencode_list(lst, _current_indent_level):
+    def _iterencode_list(self, lst, markers, _current_indent_level):
         if not lst:
             yield '[]'
             return
-        if markers is not None:
-            markerid = id(lst)
-            if markerid in markers:
-                raise ValueError("Circular reference detected")
-            markers[markerid] = lst
+        self._mark_markers(markers, lst)
         buf = '['
-        if _indent is not None:
+        if self.indent is not None:
             _current_indent_level += 1
-            newline_indent = '\n' + (' ' * (_indent * _current_indent_level))
-            separator = _item_separator + newline_indent
+            newline_indent = '\n' + (' ' * (self.indent *
+                                            _current_indent_level))
+            separator = self.item_separator + newline_indent
             buf += newline_indent
         else:
             newline_indent = None
-            separator = _item_separator
+            separator = self.item_separator
         first = True
         for value in lst:
             if first:
@@ -304,7 +374,7 @@
             else:
                 buf = separator
             if isinstance(value, basestring):
-                yield buf + _encoder(value)
+                yield buf + '"' + self.encoder(value) + '"'
             elif value is None:
                 yield buf + 'null'
             elif value is True:
@@ -314,44 +384,43 @@
             elif isinstance(value, (int, long)):
                 yield buf + str(value)
             elif isinstance(value, float):
-                yield buf + _floatstr(value)
+                yield buf + self._floatstr(value)
             else:
                 yield buf
                 if isinstance(value, (list, tuple)):
-                    chunks = _iterencode_list(value, _current_indent_level)
+                    chunks = self._iterencode_list(value, markers,
+                                                   _current_indent_level)
                 elif isinstance(value, dict):
-                    chunks = _iterencode_dict(value, _current_indent_level)
+                    chunks = self._iterencode_dict(value, markers,
+                                                   _current_indent_level)
                 else:
-                    chunks = _iterencode(value, _current_indent_level)
+                    chunks = self._iterencode(value, markers,
+                                              _current_indent_level)
                 for chunk in chunks:
                     yield chunk
         if newline_indent is not None:
             _current_indent_level -= 1
-            yield '\n' + (' ' * (_indent * _current_indent_level))
+            yield '\n' + (' ' * (self.indent * _current_indent_level))
         yield ']'
-        if markers is not None:
-            del markers[markerid]
+        self._remove_markers(markers, lst)
 
-    def _iterencode_dict(dct, _current_indent_level):
+    def _iterencode_dict(self, dct, markers, _current_indent_level):
         if not dct:
             yield '{}'
             return
-        if markers is not None:
-            markerid = id(dct)
-            if markerid in markers:
-                raise ValueError("Circular reference detected")
-            markers[markerid] = dct
+        self._mark_markers(markers, dct)
         yield '{'
-        if _indent is not None:
+        if self.indent is not None:
             _current_indent_level += 1
-            newline_indent = '\n' + (' ' * (_indent * _current_indent_level))
-            item_separator = _item_separator + newline_indent
+            newline_indent = '\n' + (' ' * (self.indent *
+                                            _current_indent_level))
+            item_separator = self.item_separator + newline_indent
             yield newline_indent
         else:
             newline_indent = None
-            item_separator = _item_separator
+            item_separator = self.item_separator
         first = True
-        if _sort_keys:
+        if self.sort_keys:
             items = sorted(dct.items(), key=lambda kv: kv[0])
         else:
             items = dct.iteritems()
@@ -361,7 +430,7 @@
             # JavaScript is weakly typed for these, so it makes sense to
             # also allow them.  Many encoders seem to do something like this.
             elif isinstance(key, float):
-                key = _floatstr(key)
+                key = self._floatstr(key)
             elif key is True:
                 key = 'true'
             elif key is False:
@@ -370,7 +439,7 @@
                 key = 'null'
             elif isinstance(key, (int, long)):
                 key = str(key)
-            elif _skipkeys:
+            elif self.skipkeys:
                 continue
             else:
                 raise TypeError("key " + repr(key) + " is not a string")
@@ -378,10 +447,10 @@
                 first = False
             else:
                 yield item_separator
-            yield _encoder(key)
-            yield _key_separator
+            yield '"' + self.encoder(key) + '"'
+            yield self.key_separator
             if isinstance(value, basestring):
-                yield _encoder(value)
+                yield '"' + self.encoder(value) + '"'
             elif value is None:
                 yield 'null'
             elif value is True:
@@ -391,26 +460,28 @@
             elif isinstance(value, (int, long)):
                 yield str(value)
             elif isinstance(value, float):
-                yield _floatstr(value)
+                yield self._floatstr(value)
             else:
                 if isinstance(value, (list, tuple)):
-                    chunks = _iterencode_list(value, _current_indent_level)
+                    chunks = self._iterencode_list(value, markers,
+                                                   _current_indent_level)
                 elif isinstance(value, dict):
-                    chunks = _iterencode_dict(value, _current_indent_level)
+                    chunks = self._iterencode_dict(value, markers,
+                                                   _current_indent_level)
                 else:
-                    chunks = _iterencode(value, _current_indent_level)
+                    chunks = self._iterencode(value, markers,
+                                              _current_indent_level)
                 for chunk in chunks:
                     yield chunk
         if newline_indent is not None:
             _current_indent_level -= 1
-            yield '\n' + (' ' * (_indent * _current_indent_level))
+            yield '\n' + (' ' * (self.indent * _current_indent_level))
         yield '}'
-        if markers is not None:
-            del markers[markerid]
+        self._remove_markers(markers, dct)
 
-    def _iterencode(o, _current_indent_level):
+    def _iterencode(self, o, markers, _current_indent_level):
         if isinstance(o, basestring):
-            yield _encoder(o)
+            yield '"' + self.encoder(o) + '"'
         elif o is None:
             yield 'null'
         elif o is True:
@@ -420,23 +491,19 @@
         elif isinstance(o, (int, long)):
             yield str(o)
         elif isinstance(o, float):
-            yield _floatstr(o)
+            yield self._floatstr(o)
         elif isinstance(o, (list, tuple)):
-            for chunk in _iterencode_list(o, _current_indent_level):
+            for chunk in self._iterencode_list(o, markers,
+                                               _current_indent_level):
                 yield chunk
         elif isinstance(o, dict):
-            for chunk in _iterencode_dict(o, _current_indent_level):
+            for chunk in self._iterencode_dict(o, markers,
+                                               _current_indent_level):
                 yield chunk
         else:
-            if markers is not None:
-                markerid = id(o)
-                if markerid in markers:
-                    raise ValueError("Circular reference detected")
-                markers[markerid] = o
-            o = _default(o)
-            for chunk in _iterencode(o, _current_indent_level):
+            self._mark_markers(markers, o)
+            obj = self.default(o)
+            for chunk in self._iterencode(obj, markers,
+                                          _current_indent_level):
                 yield chunk
-            if markers is not None:
-                del markers[markerid]
-
-    return _iterencode
+            self._remove_markers(markers, o)
diff --git a/lib-python/modified-2.7/json/tests/test_unicode.py b/lib-python/modified-2.7/json/tests/test_unicode.py
--- a/lib-python/modified-2.7/json/tests/test_unicode.py
+++ b/lib-python/modified-2.7/json/tests/test_unicode.py
@@ -80,3 +80,9 @@
         self.assertEqual(type(json.loads(u'["a"]')[0]), unicode)
         # Issue 10038.
         self.assertEqual(type(json.loads('"foo"')), unicode)
+
+    def test_encode_not_utf_8(self):
+        self.assertEqual(json.dumps('\xb1\xe6', encoding='iso8859-2'),
+                         '"\\u0105\\u0107"')
+        self.assertEqual(json.dumps(['\xb1\xe6'], encoding='iso8859-2'),
+                         '["\\u0105\\u0107"]')
diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -2,7 +2,7 @@
 from pypy.interpreter.baseobjspace import Wrappable
 from pypy.interpreter import typedef
 from pypy.interpreter.gateway import interp2app
-from pypy.interpreter.error import OperationError
+from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.tool.pairtype import extendabletype
 from pypy.tool.sourcetools import func_with_new_name
@@ -2925,14 +2925,13 @@
 def Module_get_body(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -2968,14 +2967,13 @@
 def Interactive_get_body(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3015,8 +3013,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     return space.wrap(w_self.body)
 
 def Expression_set_body(space, w_self, w_new_value):
@@ -3057,14 +3054,13 @@
 def Suite_get_body(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3104,8 +3100,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._lineno_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'lineno'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lineno')
     return space.wrap(w_self.lineno)
 
 def stmt_set_lineno(space, w_self, w_new_value):
@@ -3126,8 +3121,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._col_offset_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'col_offset'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'col_offset')
     return space.wrap(w_self.col_offset)
 
 def stmt_set_col_offset(space, w_self, w_new_value):
@@ -3157,8 +3151,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'name'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name')
     return space.wrap(w_self.name)
 
 def FunctionDef_set_name(space, w_self, w_new_value):
@@ -3179,8 +3172,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'args'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args')
     return space.wrap(w_self.args)
 
 def FunctionDef_set_args(space, w_self, w_new_value):
@@ -3197,14 +3189,13 @@
 def FunctionDef_get_body(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3215,14 +3206,13 @@
 def FunctionDef_get_decorator_list(space, w_self):
     if not w_self.initialization_state & 8:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'decorator_list'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'decorator_list')
     if w_self.w_decorator_list is None:
         if w_self.decorator_list is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.decorator_list]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_decorator_list = w_list
     return w_self.w_decorator_list
 
@@ -3266,8 +3256,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'name'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name')
     return space.wrap(w_self.name)
 
 def ClassDef_set_name(space, w_self, w_new_value):
@@ -3284,14 +3273,13 @@
 def ClassDef_get_bases(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'bases'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'bases')
     if w_self.w_bases is None:
         if w_self.bases is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.bases]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_bases = w_list
     return w_self.w_bases
 
@@ -3302,14 +3290,13 @@
 def ClassDef_get_body(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3320,14 +3307,13 @@
 def ClassDef_get_decorator_list(space, w_self):
     if not w_self.initialization_state & 8:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'decorator_list'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'decorator_list')
     if w_self.w_decorator_list is None:
         if w_self.decorator_list is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.decorator_list]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_decorator_list = w_list
     return w_self.w_decorator_list
 
@@ -3372,8 +3358,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Return_set_value(space, w_self, w_new_value):
@@ -3414,14 +3399,13 @@
 def Delete_get_targets(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'targets'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'targets')
     if w_self.w_targets is None:
         if w_self.targets is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.targets]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_targets = w_list
     return w_self.w_targets
 
@@ -3457,14 +3441,13 @@
 def Assign_get_targets(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'targets'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'targets')
     if w_self.w_targets is None:
         if w_self.targets is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.targets]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_targets = w_list
     return w_self.w_targets
 
@@ -3479,8 +3462,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Assign_set_value(space, w_self, w_new_value):
@@ -3527,8 +3509,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'target'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'target')
     return space.wrap(w_self.target)
 
 def AugAssign_set_target(space, w_self, w_new_value):
@@ -3549,8 +3530,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'op'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op')
     return operator_to_class[w_self.op - 1]()
 
 def AugAssign_set_op(space, w_self, w_new_value):
@@ -3573,8 +3553,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def AugAssign_set_value(space, w_self, w_new_value):
@@ -3621,8 +3600,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'dest'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'dest')
     return space.wrap(w_self.dest)
 
 def Print_set_dest(space, w_self, w_new_value):
@@ -3639,14 +3617,13 @@
 def Print_get_values(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'values'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'values')
     if w_self.w_values is None:
         if w_self.values is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.values]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_values = w_list
     return w_self.w_values
 
@@ -3661,8 +3638,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'nl'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'nl')
     return space.wrap(w_self.nl)
 
 def Print_set_nl(space, w_self, w_new_value):
@@ -3710,8 +3686,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'target'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'target')
     return space.wrap(w_self.target)
 
 def For_set_target(space, w_self, w_new_value):
@@ -3732,8 +3707,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'iter'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'iter')
     return space.wrap(w_self.iter)
 
 def For_set_iter(space, w_self, w_new_value):
@@ -3750,14 +3724,13 @@
 def For_get_body(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3768,14 +3741,13 @@
 def For_get_orelse(space, w_self):
     if not w_self.initialization_state & 8:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'orelse'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
     if w_self.w_orelse is None:
         if w_self.orelse is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.orelse]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_orelse = w_list
     return w_self.w_orelse
 
@@ -3819,8 +3791,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'test'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test')
     return space.wrap(w_self.test)
 
 def While_set_test(space, w_self, w_new_value):
@@ -3837,14 +3808,13 @@
 def While_get_body(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3855,14 +3825,13 @@
 def While_get_orelse(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'orelse'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
     if w_self.w_orelse is None:
         if w_self.orelse is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.orelse]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_orelse = w_list
     return w_self.w_orelse
 
@@ -3905,8 +3874,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'test'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test')
     return space.wrap(w_self.test)
 
 def If_set_test(space, w_self, w_new_value):
@@ -3923,14 +3891,13 @@
 def If_get_body(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3941,14 +3908,13 @@
 def If_get_orelse(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'orelse'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
     if w_self.w_orelse is None:
         if w_self.orelse is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.orelse]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_orelse = w_list
     return w_self.w_orelse
 
@@ -3991,8 +3957,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'context_expr'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'context_expr')
     return space.wrap(w_self.context_expr)
 
 def With_set_context_expr(space, w_self, w_new_value):
@@ -4013,8 +3978,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'optional_vars'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'optional_vars')
     return space.wrap(w_self.optional_vars)
 
 def With_set_optional_vars(space, w_self, w_new_value):
@@ -4031,14 +3995,13 @@
 def With_get_body(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -4080,8 +4043,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'type'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'type')
     return space.wrap(w_self.type)
 
 def Raise_set_type(space, w_self, w_new_value):
@@ -4102,8 +4064,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'inst'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'inst')
     return space.wrap(w_self.inst)
 
 def Raise_set_inst(space, w_self, w_new_value):
@@ -4124,8 +4085,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'tback'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'tback')
     return space.wrap(w_self.tback)
 
 def Raise_set_tback(space, w_self, w_new_value):
@@ -4168,14 +4128,13 @@
 def TryExcept_get_body(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -4186,14 +4145,13 @@
 def TryExcept_get_handlers(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'handlers'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'handlers')
     if w_self.w_handlers is None:
         if w_self.handlers is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.handlers]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_handlers = w_list
     return w_self.w_handlers
 
@@ -4204,14 +4162,13 @@
 def TryExcept_get_orelse(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'orelse'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
     if w_self.w_orelse is None:
         if w_self.orelse is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.orelse]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_orelse = w_list
     return w_self.w_orelse
 
@@ -4251,14 +4208,13 @@
 def TryFinally_get_body(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -4269,14 +4225,13 @@
 def TryFinally_get_finalbody(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'finalbody'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'finalbody')
     if w_self.w_finalbody is None:
         if w_self.finalbody is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.finalbody]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_finalbody = w_list
     return w_self.w_finalbody
 
@@ -4318,8 +4273,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'test'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test')
     return space.wrap(w_self.test)
 
 def Assert_set_test(space, w_self, w_new_value):
@@ -4340,8 +4294,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'msg'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'msg')
     return space.wrap(w_self.msg)
 
 def Assert_set_msg(space, w_self, w_new_value):
@@ -4383,14 +4336,13 @@
 def Import_get_names(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'names'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'names')
     if w_self.w_names is None:
         if w_self.names is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.names]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_names = w_list
     return w_self.w_names
 
@@ -4430,8 +4382,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'module'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'module')
     return space.wrap(w_self.module)
 
 def ImportFrom_set_module(space, w_self, w_new_value):
@@ -4451,14 +4402,13 @@
 def ImportFrom_get_names(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'names'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'names')
     if w_self.w_names is None:
         if w_self.names is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.names]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_names = w_list
     return w_self.w_names
 
@@ -4473,8 +4423,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'level'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'level')
     return space.wrap(w_self.level)
 
 def ImportFrom_set_level(space, w_self, w_new_value):
@@ -4522,8 +4471,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     return space.wrap(w_self.body)
 
 def Exec_set_body(space, w_self, w_new_value):
@@ -4544,8 +4492,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'globals'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'globals')
     return space.wrap(w_self.globals)
 
 def Exec_set_globals(space, w_self, w_new_value):
@@ -4566,8 +4513,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'locals'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'locals')
     return space.wrap(w_self.locals)
 
 def Exec_set_locals(space, w_self, w_new_value):
@@ -4610,14 +4556,13 @@
 def Global_get_names(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'names'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'names')
     if w_self.w_names is None:
         if w_self.names is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.names]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_names = w_list
     return w_self.w_names
 
@@ -4657,8 +4602,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Expr_set_value(space, w_self, w_new_value):
@@ -4754,8 +4698,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._lineno_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'lineno'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lineno')
     return space.wrap(w_self.lineno)
 
 def expr_set_lineno(space, w_self, w_new_value):
@@ -4776,8 +4719,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._col_offset_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'col_offset'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'col_offset')
     return space.wrap(w_self.col_offset)
 
 def expr_set_col_offset(space, w_self, w_new_value):
@@ -4807,8 +4749,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'op'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op')
     return boolop_to_class[w_self.op - 1]()
 
 def BoolOp_set_op(space, w_self, w_new_value):
@@ -4827,14 +4768,13 @@
 def BoolOp_get_values(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'values'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'values')
     if w_self.w_values is None:
         if w_self.values is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.values]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_values = w_list
     return w_self.w_values
 
@@ -4875,8 +4815,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'left'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'left')
     return space.wrap(w_self.left)
 
 def BinOp_set_left(space, w_self, w_new_value):
@@ -4897,8 +4836,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'op'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op')
     return operator_to_class[w_self.op - 1]()
 
 def BinOp_set_op(space, w_self, w_new_value):
@@ -4921,8 +4859,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'right'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'right')
     return space.wrap(w_self.right)
 
 def BinOp_set_right(space, w_self, w_new_value):
@@ -4969,8 +4906,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'op'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op')
     return unaryop_to_class[w_self.op - 1]()
 
 def UnaryOp_set_op(space, w_self, w_new_value):
@@ -4993,8 +4929,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'operand'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'operand')
     return space.wrap(w_self.operand)
 
 def UnaryOp_set_operand(space, w_self, w_new_value):
@@ -5040,8 +4975,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'args'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args')
     return space.wrap(w_self.args)
 
 def Lambda_set_args(space, w_self, w_new_value):
@@ -5062,8 +4996,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     return space.wrap(w_self.body)
 
 def Lambda_set_body(space, w_self, w_new_value):
@@ -5109,8 +5042,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'test'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test')
     return space.wrap(w_self.test)
 
 def IfExp_set_test(space, w_self, w_new_value):
@@ -5131,8 +5063,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     return space.wrap(w_self.body)
 
 def IfExp_set_body(space, w_self, w_new_value):
@@ -5153,8 +5084,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'orelse'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
     return space.wrap(w_self.orelse)
 
 def IfExp_set_orelse(space, w_self, w_new_value):
@@ -5197,14 +5127,13 @@
 def Dict_get_keys(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'keys'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'keys')
     if w_self.w_keys is None:
         if w_self.keys is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.keys]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_keys = w_list
     return w_self.w_keys
 
@@ -5215,14 +5144,13 @@
 def Dict_get_values(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'values'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'values')
     if w_self.w_values is None:
         if w_self.values is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.values]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_values = w_list
     return w_self.w_values
 
@@ -5260,14 +5188,13 @@
 def Set_get_elts(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elts'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elts')
     if w_self.w_elts is None:
         if w_self.elts is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.elts]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_elts = w_list
     return w_self.w_elts
 
@@ -5307,8 +5234,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elt'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elt')
     return space.wrap(w_self.elt)
 
 def ListComp_set_elt(space, w_self, w_new_value):
@@ -5325,14 +5251,13 @@
 def ListComp_get_generators(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'generators'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators')
     if w_self.w_generators is None:
         if w_self.generators is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.generators]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_generators = w_list
     return w_self.w_generators
 
@@ -5373,8 +5298,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elt'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elt')
     return space.wrap(w_self.elt)
 
 def SetComp_set_elt(space, w_self, w_new_value):
@@ -5391,14 +5315,13 @@
 def SetComp_get_generators(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'generators'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators')
     if w_self.w_generators is None:
         if w_self.generators is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.generators]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_generators = w_list
     return w_self.w_generators
 
@@ -5439,8 +5362,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'key'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'key')
     return space.wrap(w_self.key)
 
 def DictComp_set_key(space, w_self, w_new_value):
@@ -5461,8 +5383,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def DictComp_set_value(space, w_self, w_new_value):
@@ -5479,14 +5400,13 @@
 def DictComp_get_generators(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'generators'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators')
     if w_self.w_generators is None:
         if w_self.generators is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.generators]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_generators = w_list
     return w_self.w_generators
 
@@ -5528,8 +5448,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elt'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elt')
     return space.wrap(w_self.elt)
 
 def GeneratorExp_set_elt(space, w_self, w_new_value):
@@ -5546,14 +5465,13 @@
 def GeneratorExp_get_generators(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'generators'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators')
     if w_self.w_generators is None:
         if w_self.generators is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.generators]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_generators = w_list
     return w_self.w_generators
 
@@ -5594,8 +5512,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Yield_set_value(space, w_self, w_new_value):
@@ -5640,8 +5557,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'left'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'left')
     return space.wrap(w_self.left)
 
 def Compare_set_left(space, w_self, w_new_value):
@@ -5658,14 +5574,13 @@
 def Compare_get_ops(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ops'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ops')
     if w_self.w_ops is None:
         if w_self.ops is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [cmpop_to_class[node - 1]() for node in w_self.ops]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_ops = w_list
     return w_self.w_ops
 
@@ -5676,14 +5591,13 @@
 def Compare_get_comparators(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'comparators'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'comparators')
     if w_self.w_comparators is None:
         if w_self.comparators is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.comparators]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_comparators = w_list
     return w_self.w_comparators
 
@@ -5726,8 +5640,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'func'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'func')
     return space.wrap(w_self.func)
 
 def Call_set_func(space, w_self, w_new_value):
@@ -5744,14 +5657,13 @@
 def Call_get_args(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'args'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args')
     if w_self.w_args is None:
         if w_self.args is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.args]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_args = w_list
     return w_self.w_args
 
@@ -5762,14 +5674,13 @@
 def Call_get_keywords(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'keywords'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'keywords')
     if w_self.w_keywords is None:
         if w_self.keywords is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.keywords]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_keywords = w_list
     return w_self.w_keywords
 
@@ -5784,8 +5695,7 @@
             return w_obj
     if not w_self.initialization_state & 8:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'starargs'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'starargs')
     return space.wrap(w_self.starargs)
 
 def Call_set_starargs(space, w_self, w_new_value):
@@ -5806,8 +5716,7 @@
             return w_obj
     if not w_self.initialization_state & 16:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'kwargs'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'kwargs')
     return space.wrap(w_self.kwargs)
 
 def Call_set_kwargs(space, w_self, w_new_value):
@@ -5858,8 +5767,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Repr_set_value(space, w_self, w_new_value):
@@ -5904,8 +5812,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'n'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'n')
     return w_self.n
 
 def Num_set_n(space, w_self, w_new_value):
@@ -5950,8 +5857,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 's'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 's')
     return w_self.s
 
 def Str_set_s(space, w_self, w_new_value):
@@ -5996,8 +5902,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Attribute_set_value(space, w_self, w_new_value):
@@ -6018,8 +5923,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'attr'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'attr')
     return space.wrap(w_self.attr)
 
 def Attribute_set_attr(space, w_self, w_new_value):
@@ -6040,8 +5944,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ctx'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def Attribute_set_ctx(space, w_self, w_new_value):
@@ -6090,8 +5993,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Subscript_set_value(space, w_self, w_new_value):
@@ -6112,8 +6014,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'slice'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'slice')
     return space.wrap(w_self.slice)
 
 def Subscript_set_slice(space, w_self, w_new_value):
@@ -6134,8 +6035,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ctx'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def Subscript_set_ctx(space, w_self, w_new_value):
@@ -6184,8 +6084,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'id'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'id')
     return space.wrap(w_self.id)
 
 def Name_set_id(space, w_self, w_new_value):
@@ -6206,8 +6105,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ctx'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def Name_set_ctx(space, w_self, w_new_value):
@@ -6251,14 +6149,13 @@
 def List_get_elts(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elts'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elts')
     if w_self.w_elts is None:
         if w_self.elts is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.elts]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_elts = w_list
     return w_self.w_elts
 
@@ -6273,8 +6170,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ctx'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def List_set_ctx(space, w_self, w_new_value):
@@ -6319,14 +6215,13 @@
 def Tuple_get_elts(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elts'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elts')
     if w_self.w_elts is None:
         if w_self.elts is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.elts]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_elts = w_list
     return w_self.w_elts
 
@@ -6341,8 +6236,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ctx'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def Tuple_set_ctx(space, w_self, w_new_value):
@@ -6391,8 +6285,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return w_self.value
 
 def Const_set_value(space, w_self, w_new_value):
@@ -6510,8 +6403,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'lower'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lower')
     return space.wrap(w_self.lower)
 
 def Slice_set_lower(space, w_self, w_new_value):
@@ -6532,8 +6424,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'upper'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'upper')
     return space.wrap(w_self.upper)
 
 def Slice_set_upper(space, w_self, w_new_value):
@@ -6554,8 +6445,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'step'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'step')
     return space.wrap(w_self.step)
 
 def Slice_set_step(space, w_self, w_new_value):
@@ -6598,14 +6488,13 @@
 def ExtSlice_get_dims(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'dims'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'dims')
     if w_self.w_dims is None:
         if w_self.dims is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.dims]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_dims = w_list
     return w_self.w_dims
 
@@ -6645,8 +6534,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Index_set_value(space, w_self, w_new_value):
@@ -6915,8 +6803,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'target'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'target')
     return space.wrap(w_self.target)
 
 def comprehension_set_target(space, w_self, w_new_value):
@@ -6937,8 +6824,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'iter'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'iter')
     return space.wrap(w_self.iter)
 
 def comprehension_set_iter(space, w_self, w_new_value):
@@ -6955,14 +6841,13 @@
 def comprehension_get_ifs(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ifs'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ifs')
     if w_self.w_ifs is None:
         if w_self.ifs is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.ifs]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_ifs = w_list
     return w_self.w_ifs
 
@@ -7004,8 +6889,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._lineno_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'lineno'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lineno')
     return space.wrap(w_self.lineno)
 
 def excepthandler_set_lineno(space, w_self, w_new_value):
@@ -7026,8 +6910,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._col_offset_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'col_offset'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'col_offset')
     return space.wrap(w_self.col_offset)
 
 def excepthandler_set_col_offset(space, w_self, w_new_value):
@@ -7057,8 +6940,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'type'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'type')
     return space.wrap(w_self.type)
 
 def ExceptHandler_set_type(space, w_self, w_new_value):
@@ -7079,8 +6961,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'name'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name')
     return space.wrap(w_self.name)
 
 def ExceptHandler_set_name(space, w_self, w_new_value):
@@ -7097,14 +6978,13 @@
 def ExceptHandler_get_body(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -7142,14 +7022,13 @@
 def arguments_get_args(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'args'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args')
     if w_self.w_args is None:
         if w_self.args is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.args]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_args = w_list
     return w_self.w_args
 
@@ -7164,8 +7043,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'vararg'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'vararg')
     return space.wrap(w_self.vararg)
 
 def arguments_set_vararg(space, w_self, w_new_value):
@@ -7189,8 +7067,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'kwarg'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'kwarg')
     return space.wrap(w_self.kwarg)
 
 def arguments_set_kwarg(space, w_self, w_new_value):
@@ -7210,14 +7087,13 @@
 def arguments_get_defaults(space, w_self):
     if not w_self.initialization_state & 8:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'defaults'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'defaults')
     if w_self.w_defaults is None:
         if w_self.defaults is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.defaults]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_defaults = w_list
     return w_self.w_defaults
 
@@ -7261,8 +7137,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'arg'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'arg')
     return space.wrap(w_self.arg)
 
 def keyword_set_arg(space, w_self, w_new_value):
@@ -7283,8 +7158,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def keyword_set_value(space, w_self, w_new_value):
@@ -7330,8 +7204,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'name'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name')
     return space.wrap(w_self.name)
 
 def alias_set_name(space, w_self, w_new_value):
@@ -7352,8 +7225,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'asname'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'asname')
     return space.wrap(w_self.asname)
 
 def alias_set_asname(space, w_self, w_new_value):
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -414,13 +414,12 @@
             self.emit("        return w_obj", 1)
         self.emit("if not w_self.initialization_state & %s:" % (flag,), 1)
         self.emit("typename = space.type(w_self).getname(space)", 2)
-        self.emit("w_err = space.wrap(\"'%%s' object has no attribute '%s'\" %% typename)" %
+        self.emit("raise operationerrfmt(space.w_AttributeError, \"'%%s' object has no attribute '%%s'\", typename, '%s')" %
                   (field.name,), 2)
-        self.emit("raise OperationError(space.w_AttributeError, w_err)", 2)
         if field.seq:
             self.emit("if w_self.w_%s is None:" % (field.name,), 1)
             self.emit("if w_self.%s is None:" % (field.name,), 2)
-            self.emit("w_list = space.newlist([])", 3)
+            self.emit("list_w = []", 3)
             self.emit("else:", 2)
             if field.type.value in self.data.simple_types:
                 wrapper = "%s_to_class[node - 1]()" % (field.type,)
@@ -428,7 +427,7 @@
                 wrapper = "space.wrap(node)"
             self.emit("list_w = [%s for node in w_self.%s]" %
                       (wrapper, field.name), 3)
-            self.emit("w_list = space.newlist(list_w)", 3)
+            self.emit("w_list = space.newlist(list_w)", 2)
             self.emit("w_self.w_%s = w_list" % (field.name,), 2)
             self.emit("return w_self.w_%s" % (field.name,), 1)
         elif field.type.value in self.data.simple_types:
@@ -540,7 +539,7 @@
 from pypy.interpreter.baseobjspace import Wrappable
 from pypy.interpreter import typedef
 from pypy.interpreter.gateway import interp2app
-from pypy.interpreter.error import OperationError
+from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.tool.pairtype import extendabletype
 from pypy.tool.sourcetools import func_with_new_name
@@ -639,9 +638,7 @@
             missing = required[i]
             if missing is not None:
                  err = "required field \\"%s\\" missing from %s"
-                 err = err % (missing, host)
-                 w_err = space.wrap(err)
-                 raise OperationError(space.w_TypeError, w_err)
+                 raise operationerrfmt(space.w_TypeError, err, missing, host)
     raise AssertionError("should not reach here")
 
 
diff --git a/pypy/jit/backend/llgraph/llimpl.py b/pypy/jit/backend/llgraph/llimpl.py
--- a/pypy/jit/backend/llgraph/llimpl.py
+++ b/pypy/jit/backend/llgraph/llimpl.py
@@ -7,9 +7,7 @@
 import weakref
 from pypy.objspace.flow.model import Variable, Constant
 from pypy.annotation import model as annmodel
-from pypy.jit.metainterp.history import (ConstInt, ConstPtr,
-                                         BoxInt, BoxPtr, BoxObj, BoxFloat,
-                                         REF, INT, FLOAT)
+from pypy.jit.metainterp.history import REF, INT, FLOAT
 from pypy.jit.codewriter import heaptracker
 from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr, rffi
 from pypy.rpython.ootypesystem import ootype
@@ -17,7 +15,7 @@
 from pypy.rpython.llinterp import LLException
 from pypy.rpython.extregistry import ExtRegistryEntry
 
-from pypy.jit.metainterp import resoperation, executor
+from pypy.jit.metainterp import resoperation
 from pypy.jit.metainterp.resoperation import rop
 from pypy.jit.backend.llgraph import symbolic
 from pypy.jit.codewriter import longlong
@@ -334,6 +332,13 @@
     assert isinstance(type, str) and len(type) == 1
     op.descr = Descr(ofs, type, arg_types=arg_types)
 
+def compile_add_descr_arg(loop, ofs, type, arg_types):
+    from pypy.jit.backend.llgraph.runner import Descr
+    loop = _from_opaque(loop)
+    op = loop.operations[-1]
+    assert isinstance(type, str) and len(type) == 1
+    op.args.append(Descr(ofs, type, arg_types=arg_types))
+
 def compile_add_loop_token(loop, descr):
     if we_are_translated():
         raise ValueError("CALL_ASSEMBLER not supported")
@@ -438,8 +443,11 @@
         self._may_force = -1
 
     def getenv(self, v):
+        from pypy.jit.backend.llgraph.runner import Descr
         if isinstance(v, Constant):
             return v.value
+        elif isinstance(v, Descr):
+            return v
         else:
             return self.env[v]
 
@@ -807,6 +815,29 @@
         else:
             raise NotImplementedError
 
+    def op_getinteriorfield_gc(self, descr, array, index):
+        if descr.typeinfo == REF:
+            return do_getinteriorfield_gc_ptr(array, index, descr.ofs)
+        elif descr.typeinfo == INT:
+            return do_getinteriorfield_gc_int(array, index, descr.ofs)
+        elif descr.typeinfo == FLOAT:
+            return do_getinteriorfield_gc_float(array, index, descr.ofs)
+        else:
+            raise NotImplementedError
+
+    def op_setinteriorfield_gc(self, descr, array, index, newvalue):
+        if descr.typeinfo == REF:
+            return do_setinteriorfield_gc_ptr(array, index, descr.ofs,
+                                              newvalue)
+        elif descr.typeinfo == INT:
+            return do_setinteriorfield_gc_int(array, index, descr.ofs,
+                                              newvalue)
+        elif descr.typeinfo == FLOAT:
+            return do_setinteriorfield_gc_float(array, index, descr.ofs,
+                                                newvalue)
+        else:
+            raise NotImplementedError
+
     def op_setfield_gc(self, fielddescr, struct, newvalue):
         if fielddescr.typeinfo == REF:
             do_setfield_gc_ptr(struct, fielddescr.ofs, newvalue)
@@ -1354,6 +1385,22 @@
 def do_getfield_gc_ptr(struct, fieldnum):
     return cast_to_ptr(_getfield_gc(struct, fieldnum))
 
+def _getinteriorfield_gc(struct, fieldnum):
+    STRUCT, fieldname = symbolic.TokenToField[fieldnum]
+    return getattr(struct, fieldname)
+
+def do_getinteriorfield_gc_int(array, index, fieldnum):
+    struct = array._obj.container.getitem(index)
+    return cast_to_int(_getinteriorfield_gc(struct, fieldnum))
+
+def do_getinteriorfield_gc_float(array, index, fieldnum):
+    struct = array._obj.container.getitem(index)
+    return cast_to_floatstorage(_getinteriorfield_gc(struct, fieldnum))
+
+def do_getinteriorfield_gc_ptr(array, index, fieldnum):
+    struct = array._obj.container.getitem(index)
+    return cast_to_ptr(_getinteriorfield_gc(struct, fieldnum))
+
 def _getfield_raw(struct, fieldnum):
     STRUCT, fieldname = symbolic.TokenToField[fieldnum]
     ptr = cast_from_int(lltype.Ptr(STRUCT), struct)
@@ -1409,26 +1456,28 @@
     newvalue = cast_from_ptr(ITEMTYPE, newvalue)
     array.setitem(index, newvalue)
 
-def do_setfield_gc_int(struct, fieldnum, newvalue):
-    STRUCT, fieldname = symbolic.TokenToField[fieldnum]
-    ptr = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), struct)
-    FIELDTYPE = getattr(STRUCT, fieldname)
-    newvalue = cast_from_int(FIELDTYPE, newvalue)
-    setattr(ptr, fieldname, newvalue)
+def new_setfield_gc(cast_func):
+    def do_setfield_gc(struct, fieldnum, newvalue):
+        STRUCT, fieldname = symbolic.TokenToField[fieldnum]
+        ptr = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), struct)
+        FIELDTYPE = getattr(STRUCT, fieldname)
+        newvalue = cast_func(FIELDTYPE, newvalue)
+        setattr(ptr, fieldname, newvalue)
+    return do_setfield_gc
+do_setfield_gc_int = new_setfield_gc(cast_from_int)
+do_setfield_gc_float = new_setfield_gc(cast_from_floatstorage)
+do_setfield_gc_ptr = new_setfield_gc(cast_from_ptr)
 
-def do_setfield_gc_float(struct, fieldnum, newvalue):
-    STRUCT, fieldname = symbolic.TokenToField[fieldnum]
-    ptr = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), struct)
-    FIELDTYPE = getattr(STRUCT, fieldname)
-    newvalue = cast_from_floatstorage(FIELDTYPE, newvalue)
-    setattr(ptr, fieldname, newvalue)
-
-def do_setfield_gc_ptr(struct, fieldnum, newvalue):
-    STRUCT, fieldname = symbolic.TokenToField[fieldnum]
-    ptr = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), struct)
-    FIELDTYPE = getattr(STRUCT, fieldname)
-    newvalue = cast_from_ptr(FIELDTYPE, newvalue)
-    setattr(ptr, fieldname, newvalue)
+def new_setinteriorfield_gc(cast_func):
+    def do_setinteriorfield_gc(array, index, fieldnum, newvalue):
+        STRUCT, fieldname = symbolic.TokenToField[fieldnum]
+        struct = array._obj.container.getitem(index)
+        FIELDTYPE = getattr(STRUCT, fieldname)
+        setattr(struct, fieldname, cast_func(FIELDTYPE, newvalue))
+    return do_setinteriorfield_gc
+do_setinteriorfield_gc_int = new_setinteriorfield_gc(cast_from_int)
+do_setinteriorfield_gc_float = new_setinteriorfield_gc(cast_from_floatstorage)
+do_setinteriorfield_gc_ptr = new_setinteriorfield_gc(cast_from_ptr)        
 
 def do_setfield_raw_int(struct, fieldnum, newvalue):
     STRUCT, fieldname = symbolic.TokenToField[fieldnum]
@@ -1694,6 +1743,7 @@
 setannotation(compile_start_float_var, annmodel.SomeInteger())
 setannotation(compile_add, annmodel.s_None)
 setannotation(compile_add_descr, annmodel.s_None)
+setannotation(compile_add_descr_arg, annmodel.s_None)
 setannotation(compile_add_var, annmodel.s_None)
 setannotation(compile_add_int_const, annmodel.s_None)
 setannotation(compile_add_ref_const, annmodel.s_None)
@@ -1741,6 +1791,9 @@
 setannotation(do_getfield_raw_int, annmodel.SomeInteger())
 setannotation(do_getfield_raw_ptr, annmodel.SomePtr(llmemory.GCREF))
 setannotation(do_getfield_raw_float, s_FloatStorage)
+setannotation(do_getinteriorfield_gc_int, annmodel.SomeInteger())
+setannotation(do_getinteriorfield_gc_ptr, annmodel.SomePtr(llmemory.GCREF))
+setannotation(do_getinteriorfield_gc_float, s_FloatStorage)
 setannotation(do_new, annmodel.SomePtr(llmemory.GCREF))
 setannotation(do_new_array, annmodel.SomePtr(llmemory.GCREF))
 setannotation(do_setarrayitem_gc_int, annmodel.s_None)
@@ -1754,6 +1807,9 @@
 setannotation(do_setfield_raw_int, annmodel.s_None)
 setannotation(do_setfield_raw_ptr, annmodel.s_None)
 setannotation(do_setfield_raw_float, annmodel.s_None)
+setannotation(do_setinteriorfield_gc_int, annmodel.s_None)
+setannotation(do_setinteriorfield_gc_ptr, annmodel.s_None)
+setannotation(do_setinteriorfield_gc_float, annmodel.s_None)
 setannotation(do_newstr, annmodel.SomePtr(llmemory.GCREF))
 setannotation(do_strsetitem, annmodel.s_None)
 setannotation(do_newunicode, annmodel.SomePtr(llmemory.GCREF))
diff --git a/pypy/jit/backend/llgraph/runner.py b/pypy/jit/backend/llgraph/runner.py
--- a/pypy/jit/backend/llgraph/runner.py
+++ b/pypy/jit/backend/llgraph/runner.py
@@ -2,21 +2,19 @@
 Minimal-API wrapper around the llinterpreter to run operations.
 """
 
-import sys
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.rpython.lltypesystem import lltype, llmemory, rclass
 from pypy.rpython.ootypesystem import ootype
 from pypy.rpython.llinterp import LLInterpreter
 from pypy.jit.metainterp import history
-from pypy.jit.metainterp.history import REF, INT, FLOAT
+from pypy.jit.metainterp.history import REF, INT, FLOAT, STRUCT
 from pypy.jit.metainterp.warmstate import unwrap
-from pypy.jit.metainterp.resoperation import ResOperation, rop
+from pypy.jit.metainterp.resoperation import rop
 from pypy.jit.backend import model
 from pypy.jit.backend.llgraph import llimpl, symbolic
 from pypy.jit.metainterp.typesystem import llhelper, oohelper
 from pypy.jit.codewriter import heaptracker, longlong
-from pypy.rlib import rgc
 
 class MiniStats:
     pass
@@ -62,6 +60,9 @@
     def is_array_of_floats(self):
         return self.typeinfo == FLOAT
 
+    def is_array_of_structs(self):
+        return self.typeinfo == STRUCT
+
     def as_vtable_size_descr(self):
         return self
 
@@ -177,8 +178,10 @@
             llimpl.compile_add(c, op.getopnum())
             descr = op.getdescr()
             if isinstance(descr, Descr):
-                llimpl.compile_add_descr(c, descr.ofs, descr.typeinfo, descr.arg_types)
-            if isinstance(descr, history.LoopToken) and op.getopnum() != rop.JUMP:
+                llimpl.compile_add_descr(c, descr.ofs, descr.typeinfo,
+                                         descr.arg_types)
+            if (isinstance(descr, history.LoopToken) and
+                op.getopnum() != rop.JUMP):
                 llimpl.compile_add_loop_token(c, descr)
             if self.is_oo and isinstance(descr, (OODescr, MethDescr)):
                 # hack hack, not rpython
@@ -193,6 +196,9 @@
                     llimpl.compile_add_ref_const(c, x.value, self.ts.BASETYPE)
                 elif isinstance(x, history.ConstFloat):
                     llimpl.compile_add_float_const(c, x.value)
+                elif isinstance(x, Descr):
+                    llimpl.compile_add_descr_arg(c, x.ofs, x.typeinfo,
+                                                 x.arg_types)
                 else:
                     raise Exception("'%s' args contain: %r" % (op.getopname(),
                                                                x))
@@ -316,6 +322,13 @@
         token = history.getkind(getattr(S, fieldname))
         return self.getdescr(ofs, token[0], name=fieldname)
 
+    def interiorfielddescrof(self, A, fieldname):
+        S = A.OF
+        ofs2 = symbolic.get_size(A)
+        ofs, size = symbolic.get_field_token(S, fieldname)
+        token = history.getkind(getattr(S, fieldname))
+        return self.getdescr(ofs, token[0], name=fieldname, extrainfo=ofs2)
+
     def calldescrof(self, FUNC, ARGS, RESULT, extrainfo):
         arg_types = []
         for ARG in ARGS:
@@ -353,8 +366,13 @@
     def arraydescrof(self, A):
         assert A.OF != lltype.Void
         size = symbolic.get_size(A)
-        token = history.getkind(A.OF)
-        return self.getdescr(size, token[0])
+        if isinstance(A.OF, lltype.Ptr) or isinstance(A.OF, lltype.Primitive):
+            token = history.getkind(A.OF)[0]
+        elif isinstance(A.OF, lltype.Struct):
+            token = 's'
+        else:
+            token = '?'
+        return self.getdescr(size, token)
 
     # ---------- the backend-dependent operations ----------
 
@@ -406,6 +424,29 @@
         assert isinstance(fielddescr, Descr)
         return llimpl.do_getfield_raw_float(struct, fielddescr.ofs)
 
+    def bh_getinteriorfield_gc_i(self, array, index, descr):
+        assert isinstance(descr, Descr)
+        return llimpl.do_getinteriorfield_gc_int(array, index, descr.ofs)
+    def bh_getinteriorfield_gc_r(self, array, index, descr):
+        assert isinstance(descr, Descr)
+        return llimpl.do_getinteriorfield_gc_ptr(array, index, descr.ofs)
+    def bh_getinteriorfield_gc_f(self, array, index, descr):
+        assert isinstance(descr, Descr)
+        return llimpl.do_getinteriorfield_gc_float(array, index, descr.ofs)
+
+    def bh_setinteriorfield_gc_i(self, array, index, descr, value):
+        assert isinstance(descr, Descr)
+        return llimpl.do_setinteriorfield_gc_int(array, index, descr.ofs,
+                                                 value)
+    def bh_setinteriorfield_gc_r(self, array, index, descr, value):
+        assert isinstance(descr, Descr)
+        return llimpl.do_setinteriorfield_gc_ptr(array, index, descr.ofs,
+                                                 value)
+    def bh_setinteriorfield_gc_f(self, array, index, descr, value):
+        assert isinstance(descr, Descr)
+        return llimpl.do_setinteriorfield_gc_float(array, index, descr.ofs,
+                                                   value)
+
     def bh_new(self, sizedescr):
         assert isinstance(sizedescr, Descr)
         return llimpl.do_new(sizedescr.ofs)
@@ -418,7 +459,6 @@
 
     def bh_classof(self, struct):
         struct = lltype.cast_opaque_ptr(rclass.OBJECTPTR, struct)
-        result = struct.typeptr
         result_adr = llmemory.cast_ptr_to_adr(struct.typeptr)
         return heaptracker.adr2int(result_adr)
 
diff --git a/pypy/jit/backend/llsupport/descr.py b/pypy/jit/backend/llsupport/descr.py
--- a/pypy/jit/backend/llsupport/descr.py
+++ b/pypy/jit/backend/llsupport/descr.py
@@ -1,13 +1,10 @@
 import py
-from pypy.rpython.lltypesystem import lltype, rffi, llmemory, rclass
+from pypy.rpython.lltypesystem import lltype, rffi, llmemory
 from pypy.rpython.lltypesystem.lloperation import llop
 from pypy.jit.backend.llsupport import symbolic, support
-from pypy.jit.metainterp.history import AbstractDescr, getkind, BoxInt, BoxPtr
-from pypy.jit.metainterp.history import BasicFailDescr, LoopToken, BoxFloat
+from pypy.jit.metainterp.history import AbstractDescr, getkind
 from pypy.jit.metainterp import history
-from pypy.jit.metainterp.resoperation import ResOperation, rop
 from pypy.jit.codewriter import heaptracker, longlong
-from pypy.rlib.rarithmetic import r_longlong, r_ulonglong
 
 # The point of the class organization in this file is to make instances
 # as compact as possible.  This is done by not storing the field size or
@@ -23,6 +20,7 @@
         self._cache_field = {}
         self._cache_array = {}
         self._cache_call = {}
+        self._cache_interiorfield = {}
 
     def init_size_descr(self, STRUCT, sizedescr):
         assert isinstance(STRUCT, lltype.GcStruct)
@@ -142,7 +140,6 @@
         cachedict[fieldname] = fielddescr
         return fielddescr
 
-
 # ____________________________________________________________
 # ArrayDescrs
 
@@ -167,6 +164,7 @@
 
     _is_array_of_pointers = False      # unless overridden by GcPtrArrayDescr
     _is_array_of_floats   = False      # unless overridden by FloatArrayDescr
+    _is_array_of_structs  = False      # unless overridden by StructArrayDescr
     _is_item_signed       = False      # unless overridden by XxxArrayDescr
 
     def is_array_of_pointers(self):
@@ -175,6 +173,9 @@
     def is_array_of_floats(self):
         return self._is_array_of_floats
 
+    def is_array_of_structs(self):
+        return self._is_array_of_structs
+
     def is_item_signed(self):
         return self._is_item_signed
 
@@ -199,6 +200,10 @@
     def get_item_size(self, translate_support_code):
         return symbolic.get_size(lltype.Float, translate_support_code)
 
+class StructArrayDescr(BaseArrayDescr):
+    _clsname = 'StructArrayDescr'
+    _is_array_of_structs = True
+
 class BaseArrayNoLengthDescr(BaseArrayDescr):
     def get_base_size(self, translate_support_code):
         return 0
@@ -218,6 +223,13 @@
 def getArrayDescrClass(ARRAY):
     if ARRAY.OF is lltype.Float:
         return FloatArrayDescr
+    elif isinstance(ARRAY.OF, lltype.Struct):
+        class Descr(StructArrayDescr):
+            _clsname = '%sArrayDescr' % ARRAY.OF._name
+            def get_item_size(self, translate_support_code):
+                return symbolic.get_size(ARRAY.OF, translate_support_code)
+        Descr.__name__ = Descr._clsname
+        return Descr
     return getDescrClass(ARRAY.OF, BaseArrayDescr, GcPtrArrayDescr,
                          NonGcPtrArrayDescr, 'Array', 'get_item_size',
                          '_is_array_of_floats', '_is_item_signed')
@@ -252,6 +264,36 @@
         cache[ARRAY] = arraydescr
         return arraydescr
 
+# ____________________________________________________________
+# InteriorFieldDescr
+
+class InteriorFieldDescr(AbstractDescr):
+    arraydescr = BaseArrayDescr()     # workaround for the annotator
+    fielddescr = BaseFieldDescr('', 0)
+
+    def __init__(self, arraydescr, fielddescr):
+        self.arraydescr = arraydescr
+        self.fielddescr = fielddescr
+
+    def is_pointer_field(self):
+        return self.fielddescr.is_pointer_field()
+
+    def is_float_field(self):
+        return self.fielddescr.is_float_field()
+
+    def repr_of_descr(self):
+        return '<InteriorFieldDescr %s>' % self.fielddescr.repr_of_descr()
+
+def get_interiorfield_descr(gc_ll_descr, ARRAY, FIELDTP, name):
+    cache = gc_ll_descr._cache_interiorfield
+    try:
+        return cache[(ARRAY, FIELDTP, name)]
+    except KeyError:
+        arraydescr = get_array_descr(gc_ll_descr, ARRAY)
+        fielddescr = get_field_descr(gc_ll_descr, FIELDTP, name)
+        descr = InteriorFieldDescr(arraydescr, fielddescr)
+        cache[(ARRAY, FIELDTP, name)] = descr
+        return descr
 
 # ____________________________________________________________
 # CallDescrs
@@ -525,7 +567,8 @@
         #
         if TYPE is lltype.Float or is_longlong(TYPE):
             setattr(Descr, floatattrname, True)
-        elif TYPE is not lltype.Bool and rffi.cast(TYPE, -1) == -1:
+        elif (TYPE is not lltype.Bool and isinstance(TYPE, lltype.Number) and
+              rffi.cast(TYPE, -1) == -1):
             setattr(Descr, signedattrname, True)
         #
         _cache[nameprefix, TYPE] = Descr
diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py
--- a/pypy/jit/backend/llsupport/gc.py
+++ b/pypy/jit/backend/llsupport/gc.py
@@ -45,6 +45,14 @@
     def freeing_block(self, start, stop):
         pass
 
+    def get_funcptr_for_newarray(self):
+        return llhelper(self.GC_MALLOC_ARRAY, self.malloc_array)
+    def get_funcptr_for_newstr(self):
+        return llhelper(self.GC_MALLOC_STR_UNICODE, self.malloc_str)
+    def get_funcptr_for_newunicode(self):
+        return llhelper(self.GC_MALLOC_STR_UNICODE, self.malloc_unicode)
+
+
     def record_constptrs(self, op, gcrefs_output_list):
         for i in range(op.numargs()):
             v = op.getarg(i)
@@ -96,6 +104,39 @@
         malloc_fn_ptr = self.configure_boehm_once()
         self.funcptr_for_new = malloc_fn_ptr
 
+        def malloc_array(basesize, itemsize, ofs_length, num_elem):
+            try:
+                size = ovfcheck(basesize + ovfcheck(itemsize * num_elem))
+            except OverflowError:
+                return lltype.nullptr(llmemory.GCREF.TO)
+            res = self.funcptr_for_new(size)
+            if not res:
+                return res
+            rffi.cast(rffi.CArrayPtr(lltype.Signed), res)[ofs_length/WORD] = num_elem
+            return res
+        self.malloc_array = malloc_array
+        self.GC_MALLOC_ARRAY = lltype.Ptr(lltype.FuncType(
+            [lltype.Signed] * 4, llmemory.GCREF))
+
+
+        (str_basesize, str_itemsize, str_ofs_length
+         ) = symbolic.get_array_token(rstr.STR, self.translate_support_code)
+        (unicode_basesize, unicode_itemsize, unicode_ofs_length
+         ) = symbolic.get_array_token(rstr.UNICODE, self.translate_support_code)
+        def malloc_str(length):
+            return self.malloc_array(
+                str_basesize, str_itemsize, str_ofs_length, length
+            )
+        def malloc_unicode(length):
+            return self.malloc_array(
+                unicode_basesize, unicode_itemsize, unicode_ofs_length, length
+            )
+        self.malloc_str = malloc_str
+        self.malloc_unicode = malloc_unicode
+        self.GC_MALLOC_STR_UNICODE = lltype.Ptr(lltype.FuncType(
+            [lltype.Signed], llmemory.GCREF))
+
+
         # on some platform GC_init is required before any other
         # GC_* functions, call it here for the benefit of tests
         # XXX move this to tests
@@ -116,39 +157,27 @@
         ofs_length = arraydescr.get_ofs_length(self.translate_support_code)
         basesize = arraydescr.get_base_size(self.translate_support_code)
         itemsize = arraydescr.get_item_size(self.translate_support_code)
-        size = basesize + itemsize * num_elem
-        res = self.funcptr_for_new(size)
-        rffi.cast(rffi.CArrayPtr(lltype.Signed), res)[ofs_length/WORD] = num_elem
-        return res
+        return self.malloc_array(basesize, itemsize, ofs_length, num_elem)
 
     def gc_malloc_str(self, num_elem):
-        basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR,
-                                                   self.translate_support_code)
-        assert itemsize == 1
-        size = basesize + num_elem
-        res = self.funcptr_for_new(size)
-        rffi.cast(rffi.CArrayPtr(lltype.Signed), res)[ofs_length/WORD] = num_elem
-        return res
+        return self.malloc_str(num_elem)
 
     def gc_malloc_unicode(self, num_elem):
-        basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE,
-                                                   self.translate_support_code)
-        size = basesize + num_elem * itemsize
-        res = self.funcptr_for_new(size)
-        rffi.cast(rffi.CArrayPtr(lltype.Signed), res)[ofs_length/WORD] = num_elem
-        return res
+        return self.malloc_unicode(num_elem)
 
     def args_for_new(self, sizedescr):
         assert isinstance(sizedescr, BaseSizeDescr)
         return [sizedescr.size]
 
+    def args_for_new_array(self, arraydescr):
+        ofs_length = arraydescr.get_ofs_length(self.translate_support_code)
+        basesize = arraydescr.get_base_size(self.translate_support_code)
+        itemsize = arraydescr.get_item_size(self.translate_support_code)
+        return [basesize, itemsize, ofs_length]
+
     def get_funcptr_for_new(self):
         return self.funcptr_for_new
 
-    get_funcptr_for_newarray = None
-    get_funcptr_for_newstr = None
-    get_funcptr_for_newunicode = None
-
     def rewrite_assembler(self, cpu, operations, gcrefs_output_list):
         # record all GCREFs too, because Boehm cannot see them and keep them
         # alive if they end up as constants in the assembler
@@ -752,15 +781,6 @@
     def get_funcptr_for_new(self):
         return llhelper(self.GC_MALLOC_BASIC, self.malloc_basic)
 
-    def get_funcptr_for_newarray(self):
-        return llhelper(self.GC_MALLOC_ARRAY, self.malloc_array)
-
-    def get_funcptr_for_newstr(self):
-        return llhelper(self.GC_MALLOC_STR_UNICODE, self.malloc_str)
-
-    def get_funcptr_for_newunicode(self):
-        return llhelper(self.GC_MALLOC_STR_UNICODE, self.malloc_unicode)
-
     def do_write_barrier(self, gcref_struct, gcref_newptr):
         hdr_addr = llmemory.cast_ptr_to_adr(gcref_struct)
         hdr_addr -= self.gcheaderbuilder.size_gc_header
diff --git a/pypy/jit/backend/llsupport/llmodel.py b/pypy/jit/backend/llsupport/llmodel.py
--- a/pypy/jit/backend/llsupport/llmodel.py
+++ b/pypy/jit/backend/llsupport/llmodel.py
@@ -1,24 +1,18 @@
-import sys
 from pypy.rpython.lltypesystem import lltype, llmemory, rffi, rclass, rstr
 from pypy.rpython.lltypesystem.lloperation import llop
-from pypy.rpython.llinterp import LLInterpreter, LLException
+from pypy.rpython.llinterp import LLInterpreter
 from pypy.rpython.annlowlevel import llhelper
 from pypy.rlib.objectmodel import we_are_translated, specialize
-from pypy.jit.metainterp.history import BoxInt, BoxPtr, set_future_values,\
-     BoxFloat
 from pypy.jit.metainterp import history
 from pypy.jit.codewriter import heaptracker, longlong
 from pypy.jit.backend.model import AbstractCPU
 from pypy.jit.backend.llsupport import symbolic
 from pypy.jit.backend.llsupport.symbolic import WORD, unroll_basic_sizes
-from pypy.jit.backend.llsupport.descr import get_size_descr,  BaseSizeDescr
-from pypy.jit.backend.llsupport.descr import get_field_descr, BaseFieldDescr
-from pypy.jit.backend.llsupport.descr import get_array_descr, BaseArrayDescr
-from pypy.jit.backend.llsupport.descr import get_call_descr
-from pypy.jit.backend.llsupport.descr import BaseIntCallDescr, GcPtrCallDescr
-from pypy.jit.backend.llsupport.descr import FloatCallDescr, VoidCallDescr
+from pypy.jit.backend.llsupport.descr import (get_size_descr,
+     get_field_descr, BaseFieldDescr, get_array_descr, BaseArrayDescr,
+     get_call_descr, BaseIntCallDescr, GcPtrCallDescr, FloatCallDescr,
+     VoidCallDescr, InteriorFieldDescr, get_interiorfield_descr)
 from pypy.jit.backend.llsupport.asmmemmgr import AsmMemoryManager
-from pypy.rpython.annlowlevel import cast_instance_to_base_ptr
 
 
 class AbstractLLCPU(AbstractCPU):
@@ -241,6 +235,9 @@
     def arraydescrof(self, A):
         return get_array_descr(self.gc_ll_descr, A)
 
+    def interiorfielddescrof(self, A, fieldname):
+        return get_interiorfield_descr(self.gc_ll_descr, A, A.OF, fieldname)
+
     def unpack_arraydescr(self, arraydescr):
         assert isinstance(arraydescr, BaseArrayDescr)
         return arraydescr.get_base_size(self.translate_support_code)
@@ -358,6 +355,100 @@
     bh_getarrayitem_raw_i = bh_getarrayitem_gc_i
     bh_getarrayitem_raw_f = bh_getarrayitem_gc_f
 
+    def bh_getinteriorfield_gc_i(self, gcref, itemindex, descr):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        fieldsize = descr.fielddescr.get_field_size(self.translate_support_code)
+        sign = descr.fielddescr.is_field_signed()
+        fullofs = itemindex * size + ofs
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), fullofs)
+        for STYPE, UTYPE, itemsize in unroll_basic_sizes:
+            if fieldsize == itemsize:
+                if sign:
+                    item = rffi.cast(rffi.CArrayPtr(STYPE), items)
+                    val = item[0]
+                    val = rffi.cast(lltype.Signed, val)
+                else:
+                    item = rffi.cast(rffi.CArrayPtr(UTYPE), items)
+                    val = item[0]
+                    val = rffi.cast(lltype.Signed, val)
+                # --- end of GC unsafe code ---
+                return val
+        else:
+            raise NotImplementedError("size = %d" % fieldsize)
+
+    def bh_getinteriorfield_gc_r(self, gcref, itemindex, descr):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs +
+                            size * itemindex)
+        items = rffi.cast(rffi.CArrayPtr(lltype.Signed), items)
+        pval = self._cast_int_to_gcref(items[0])
+        # --- end of GC unsafe code ---
+        return pval
+
+    def bh_getinteriorfield_gc_f(self, gcref, itemindex, descr):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs +
+                            size * itemindex)
+        items = rffi.cast(rffi.CArrayPtr(longlong.FLOATSTORAGE), items)
+        fval = items[0]
+        # --- end of GC unsafe code ---
+        return fval
+
+    def bh_setinteriorfield_gc_i(self, gcref, itemindex, descr, value):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        fieldsize = descr.fielddescr.get_field_size(self.translate_support_code)
+        ofs = itemindex * size + ofs
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs)
+        for TYPE, _, itemsize in unroll_basic_sizes:
+            if fieldsize == itemsize:
+                items = rffi.cast(rffi.CArrayPtr(TYPE), items)
+                items[0] = rffi.cast(TYPE, value)
+                # --- end of GC unsafe code ---
+                return
+        else:
+            raise NotImplementedError("size = %d" % fieldsize)
+
+    def bh_setinteriorfield_gc_r(self, gcref, itemindex, descr, newvalue):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        self.gc_ll_descr.do_write_barrier(gcref, newvalue)
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref),
+                            ofs + size * itemindex)
+        items = rffi.cast(rffi.CArrayPtr(lltype.Signed), items)
+        items[0] = self.cast_gcref_to_int(newvalue)
+        # --- end of GC unsafe code ---
+
+    def bh_setinteriorfield_gc_f(self, gcref, itemindex, descr, newvalue):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref),
+                            ofs + size * itemindex)
+        items = rffi.cast(rffi.CArrayPtr(longlong.FLOATSTORAGE), items)
+        items[0] = newvalue
+        # --- end of GC unsafe code ---
+
     def bh_strlen(self, string):
         s = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), string)
         return len(s.chars)
@@ -475,7 +566,6 @@
 
     def bh_classof(self, struct):
         struct = lltype.cast_opaque_ptr(rclass.OBJECTPTR, struct)
-        result = struct.typeptr
         result_adr = llmemory.cast_ptr_to_adr(struct.typeptr)
         return heaptracker.adr2int(result_adr)
 
diff --git a/pypy/jit/backend/llsupport/regalloc.py b/pypy/jit/backend/llsupport/regalloc.py
--- a/pypy/jit/backend/llsupport/regalloc.py
+++ b/pypy/jit/backend/llsupport/regalloc.py
@@ -287,7 +287,6 @@
         self.reg_bindings[to_v] = reg
 
     def _move_variable_away(self, v, prev_loc):
-        reg = None
         if self.free_regs:
             loc = self.free_regs.pop()
             self.reg_bindings[v] = loc
diff --git a/pypy/jit/backend/llsupport/test/test_asmmemmgr.py b/pypy/jit/backend/llsupport/test/test_asmmemmgr.py
--- a/pypy/jit/backend/llsupport/test/test_asmmemmgr.py
+++ b/pypy/jit/backend/llsupport/test/test_asmmemmgr.py
@@ -211,14 +211,14 @@
     debug._log = debug.DebugLog()
     try:
         mc._dump(addr, 'test-logname-section')
-        log = list(debug._log)
+        log = list(debug._log) 
     finally:
         debug._log = None
     encoded = ''.join(writtencode).encode('hex').upper()
     ataddr = '@%x' % addr
     assert log == [('test-logname-section',
                     [('debug_print', 'CODE_DUMP', ataddr, '+0 ', encoded)])]
-    #
+    
     lltype.free(p, flavor='raw')
 
 def test_blockbuildermixin2():
diff --git a/pypy/jit/backend/llsupport/test/test_descr.py b/pypy/jit/backend/llsupport/test/test_descr.py
--- a/pypy/jit/backend/llsupport/test/test_descr.py
+++ b/pypy/jit/backend/llsupport/test/test_descr.py
@@ -3,7 +3,6 @@
 from pypy.jit.backend.llsupport import symbolic
 from pypy.rlib.objectmodel import Symbolic
 from pypy.rpython.annlowlevel import llhelper
-from pypy.jit.metainterp.history import BoxInt, BoxFloat, BoxPtr
 from pypy.jit.metainterp import history
 from pypy.jit.codewriter import longlong
 import sys, struct, py
@@ -149,7 +148,9 @@
     A2 = lltype.GcArray(lltype.Ptr(T))
     A3 = lltype.GcArray(lltype.Ptr(U))
     A4 = lltype.GcArray(lltype.Float)
-    A5 = lltype.GcArray(lltype.SingleFloat)
+    A5 = lltype.GcArray(lltype.Struct('x', ('v', lltype.Signed),
+                                      ('k', lltype.Signed)))
+    A6 = lltype.GcArray(lltype.SingleFloat)
     assert getArrayDescrClass(A2) is GcPtrArrayDescr
     assert getArrayDescrClass(A3) is NonGcPtrArrayDescr
     cls = getArrayDescrClass(A1)
@@ -158,7 +159,7 @@
     clsf = getArrayDescrClass(A4)
     assert clsf != cls
     assert clsf == getArrayDescrClass(lltype.GcArray(lltype.Float))
-    clss = getArrayDescrClass(A5)
+    clss = getArrayDescrClass(A6)
     assert clss not in (clsf, cls)
     assert clss == getArrayDescrClass(lltype.GcArray(rffi.UINT))
     #
@@ -168,11 +169,12 @@
     descr3 = get_array_descr(c0, A3)
     descr4 = get_array_descr(c0, A4)
     descr5 = get_array_descr(c0, A5)
+    descr6 = get_array_descr(c0, A6)
     assert descr1.__class__ is cls
     assert descr2.__class__ is GcPtrArrayDescr
     assert descr3.__class__ is NonGcPtrArrayDescr
     assert descr4.__class__ is clsf
-    assert descr5.__class__ is clss
+    assert descr6.__class__ is clss
     assert descr1 == get_array_descr(c0, lltype.GcArray(lltype.Char))
     assert not descr1.is_array_of_pointers()
     assert     descr2.is_array_of_pointers()
@@ -202,7 +204,8 @@
     assert descr2.get_item_size(False) == rffi.sizeof(lltype.Ptr(T))
     assert descr3.get_item_size(False) == rffi.sizeof(lltype.Ptr(U))
     assert descr4.get_item_size(False) == rffi.sizeof(lltype.Float)
-    assert descr5.get_item_size(False) == rffi.sizeof(lltype.SingleFloat)
+    assert descr5.get_item_size(False) == rffi.sizeof(lltype.Signed) * 2
+    assert descr6.get_item_size(False) == rffi.sizeof(lltype.SingleFloat)
     #
     assert isinstance(descr1.get_base_size(True), Symbolic)
     assert isinstance(descr2.get_base_size(True), Symbolic)
@@ -348,7 +351,6 @@
                             (rffi.SHORT,      True), (rffi.USHORT, False),
                             (rffi.INT,        True), (rffi.UINT,   False),
                             (rffi.LONG,       True), (rffi.ULONG,  False)]:
-        A = lltype.GcArray(RESTYPE)
         for tsc in [False, True]:
             c2 = GcCache(tsc)
             descr1 = get_call_descr(c2, [], RESTYPE)
@@ -379,7 +381,6 @@
     descr3i = get_array_descr(c0, lltype.GcArray(lltype.Char))
     assert descr3i.repr_of_descr() == '<CharArrayDescr>'
     #
-    cache = {}
     descr4 = get_call_descr(c0, [lltype.Char, lltype.Ptr(S)], lltype.Ptr(S))
     assert 'GcPtrCallDescr' in descr4.repr_of_descr()
     #
@@ -412,10 +413,10 @@
     ARGS = [lltype.Float, lltype.Ptr(ARRAY)]
     RES = lltype.Float
 
-    def f(a, b):
+    def f2(a, b):
         return float(b[0]) + a
 
-    fnptr = llhelper(lltype.Ptr(lltype.FuncType(ARGS, RES)), f)
+    fnptr = llhelper(lltype.Ptr(lltype.FuncType(ARGS, RES)), f2)
     descr2 = get_call_descr(c0, ARGS, RES)
     a = lltype.malloc(ARRAY, 3)
     opaquea = lltype.cast_opaque_ptr(llmemory.GCREF, a)
diff --git a/pypy/jit/backend/model.py b/pypy/jit/backend/model.py
--- a/pypy/jit/backend/model.py
+++ b/pypy/jit/backend/model.py
@@ -1,5 +1,5 @@
 from pypy.rlib.debug import debug_start, debug_print, debug_stop
-from pypy.jit.metainterp import history, compile
+from pypy.jit.metainterp import history
 
 
 class AbstractCPU(object):
@@ -213,6 +213,10 @@
     def typedescrof(TYPE):
         raise NotImplementedError
 
+    @staticmethod
+    def interiorfielddescrof(A, fieldname):
+        raise NotImplementedError
+
     # ---------- the backend-dependent operations ----------
 
     # lltype specific operations
diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -5,7 +5,7 @@
                                          BoxInt, Box, BoxPtr,
                                          LoopToken,
                                          ConstInt, ConstPtr,
-                                         BoxObj, Const,
+                                         BoxObj,
                                          ConstObj, BoxFloat, ConstFloat)
 from pypy.jit.metainterp.resoperation import ResOperation, rop
 from pypy.jit.metainterp.typesystem import deref
@@ -111,7 +111,7 @@
         self.cpu.set_future_value_int(0, 2)
         fail = self.cpu.execute_token(looptoken)
         res = self.cpu.get_latest_value_int(0)
-        assert res == 3        
+        assert res == 3
         assert fail.identifier == 1
 
     def test_compile_loop(self):
@@ -127,7 +127,7 @@
             ]
         inputargs = [i0]
         operations[2].setfailargs([i1])
-        
+
         self.cpu.compile_loop(inputargs, operations, looptoken)
         self.cpu.set_future_value_int(0, 2)
         fail = self.cpu.execute_token(looptoken)
@@ -148,7 +148,7 @@
             ]
         inputargs = [i0]
         operations[2].setfailargs([None, None, i1, None])
-        
+
         self.cpu.compile_loop(inputargs, operations, looptoken)
         self.cpu.set_future_value_int(0, 2)
         fail = self.cpu.execute_token(looptoken)
@@ -372,7 +372,7 @@
         for opnum, boxargs, retvalue in get_int_tests():
             res = self.execute_operation(opnum, boxargs, 'int')
             assert res.value == retvalue
-        
+
     def test_float_operations(self):
         from pypy.jit.metainterp.test.test_executor import get_float_tests
         for opnum, boxargs, rettype, retvalue in get_float_tests(self.cpu):
@@ -438,7 +438,7 @@
 
     def test_ovf_operations_reversed(self):
         self.test_ovf_operations(reversed=True)
-        
+
     def test_bh_call(self):
         cpu = self.cpu
         #
@@ -503,7 +503,7 @@
                                          [funcbox, BoxInt(num), BoxInt(num)],
                                          'int', descr=dyn_calldescr)
             assert res.value == 2 * num
-            
+
 
         if cpu.supports_floats:
             def func(f0, f1, f2, f3, f4, f5, f6, i0, i1, f7, f8, f9):
@@ -543,7 +543,7 @@
         funcbox = self.get_funcbox(self.cpu, func_ptr)
         res = self.execute_operation(rop.CALL, [funcbox] + map(BoxInt, args), 'int', descr=calldescr)
         assert res.value == func(*args)
-        
+
     def test_call_stack_alignment(self):
         # test stack alignment issues, notably for Mac OS/X.
         # also test the ordering of the arguments.
@@ -615,7 +615,7 @@
         res = self.execute_operation(rop.GETFIELD_GC, [t_box],
                                      'int', descr=shortdescr)
         assert res.value == 1331
-        
+
         #
         u_box, U_box = self.alloc_instance(self.U)
         fielddescr2 = self.cpu.fielddescrof(self.S, 'next')
@@ -695,7 +695,7 @@
 
     def test_failing_guard_class(self):
         t_box, T_box = self.alloc_instance(self.T)
-        u_box, U_box = self.alloc_instance(self.U)        
+        u_box, U_box = self.alloc_instance(self.U)
         null_box = self.null_instance()
         for opname, args in [(rop.GUARD_CLASS, [t_box, U_box]),
                              (rop.GUARD_CLASS, [u_box, T_box]),
@@ -787,7 +787,7 @@
         r = self.execute_operation(rop.GETARRAYITEM_GC, [a_box, BoxInt(3)],
                                    'int', descr=arraydescr)
         assert r.value == 160
-        
+
         #
         if isinstance(A, lltype.GcArray):
             A = lltype.Ptr(A)
@@ -880,6 +880,73 @@
                                    'int', descr=arraydescr)
         assert r.value == 7441
 
+    def test_array_of_structs(self):
+        TP = lltype.GcStruct('x')
+        ITEM = lltype.Struct('x',
+                             ('vs', lltype.Signed),
+                             ('vu', lltype.Unsigned),
+                             ('vsc', rffi.SIGNEDCHAR),
+                             ('vuc', rffi.UCHAR),
+                             ('vss', rffi.SHORT),
+                             ('vus', rffi.USHORT),
+                             ('vsi', rffi.INT),
+                             ('vui', rffi.UINT),
+                             ('k', lltype.Float),
+                             ('p', lltype.Ptr(TP)))
+        a_box, A = self.alloc_array_of(ITEM, 15)
+        s_box, S = self.alloc_instance(TP)
+        kdescr = self.cpu.interiorfielddescrof(A, 'k')
+        pdescr = self.cpu.interiorfielddescrof(A, 'p')
+        self.execute_operation(rop.SETINTERIORFIELD_GC, [a_box, BoxInt(3),
+                                                         boxfloat(1.5)],
+                               'void', descr=kdescr)
+        f = self.cpu.bh_getinteriorfield_gc_f(a_box.getref_base(), 3, kdescr)
+        assert longlong.getrealfloat(f) == 1.5
+        self.cpu.bh_setinteriorfield_gc_f(a_box.getref_base(), 3, kdescr, longlong.getfloatstorage(2.5))
+        r = self.execute_operation(rop.GETINTERIORFIELD_GC, [a_box, BoxInt(3)],
+                                   'float', descr=kdescr)
+        assert r.getfloat() == 2.5
+        #
+        NUMBER_FIELDS = [('vs', lltype.Signed),
+                         ('vu', lltype.Unsigned),
+                         ('vsc', rffi.SIGNEDCHAR),
+                         ('vuc', rffi.UCHAR),
+                         ('vss', rffi.SHORT),
+                         ('vus', rffi.USHORT),
+                         ('vsi', rffi.INT),
+                         ('vui', rffi.UINT)]
+        for name, TYPE in NUMBER_FIELDS[::-1]:
+            vdescr = self.cpu.interiorfielddescrof(A, name)
+            self.execute_operation(rop.SETINTERIORFIELD_GC, [a_box, BoxInt(3),
+                                                             BoxInt(-15)],
+                                   'void', descr=vdescr)
+        for name, TYPE in NUMBER_FIELDS:
+            vdescr = self.cpu.interiorfielddescrof(A, name)
+            i = self.cpu.bh_getinteriorfield_gc_i(a_box.getref_base(), 3,
+                                                  vdescr)
+            assert i == rffi.cast(lltype.Signed, rffi.cast(TYPE, -15))
+        for name, TYPE in NUMBER_FIELDS[::-1]:
+            vdescr = self.cpu.interiorfielddescrof(A, name)
+            self.cpu.bh_setinteriorfield_gc_i(a_box.getref_base(), 3,
+                                              vdescr, -25)
+        for name, TYPE in NUMBER_FIELDS:
+            vdescr = self.cpu.interiorfielddescrof(A, name)
+            r = self.execute_operation(rop.GETINTERIORFIELD_GC,
+                                       [a_box, BoxInt(3)],
+                                       'int', descr=vdescr)
+            assert r.getint() == rffi.cast(lltype.Signed, rffi.cast(TYPE, -25))
+        #
+        self.execute_operation(rop.SETINTERIORFIELD_GC, [a_box, BoxInt(4),
+                                                         s_box],
+                               'void', descr=pdescr)
+        r = self.cpu.bh_getinteriorfield_gc_r(a_box.getref_base(), 4, pdescr)
+        assert r == s_box.getref_base()
+        self.cpu.bh_setinteriorfield_gc_r(a_box.getref_base(), 3, pdescr,
+                                          s_box.getref_base())
+        r = self.execute_operation(rop.GETINTERIORFIELD_GC, [a_box, BoxInt(3)],
+                                   'ref', descr=pdescr)
+        assert r.getref_base() == s_box.getref_base()
+
     def test_string_basic(self):
         s_box = self.alloc_string("hello\xfe")
         r = self.execute_operation(rop.STRLEN, [s_box], 'int')
@@ -1402,7 +1469,7 @@
         addr = llmemory.cast_ptr_to_adr(func_ptr)
         return ConstInt(heaptracker.adr2int(addr))
 
-    
+
     MY_VTABLE = rclass.OBJECT_VTABLE    # for tests only
 
     S = lltype.GcForwardReference()
@@ -1439,7 +1506,6 @@
         return BoxPtr(lltype.nullptr(llmemory.GCREF.TO))
 
     def alloc_array_of(self, ITEM, length):
-        cpu = self.cpu
         A = lltype.GcArray(ITEM)
         a = lltype.malloc(A, length)
         a_box = BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, a))
@@ -2318,7 +2384,7 @@
         for opname, arg, res in ops:
             self.execute_operation(opname, [arg], 'void')
             assert self.guard_failed == res
-        
+
         lltype.free(x, flavor='raw')
 
     def test_assembler_call(self):
@@ -2398,7 +2464,7 @@
         FakeJitDriverSD.portal_calldescr = self.cpu.calldescrof(
             lltype.Ptr(lltype.FuncType(ARGS, RES)), ARGS, RES,
             EffectInfo.MOST_GENERAL)
-        
+
         ops = '''
         [f0, f1]
         f2 = float_add(f0, f1)
@@ -2489,7 +2555,7 @@
         FakeJitDriverSD.portal_calldescr = self.cpu.calldescrof(
             lltype.Ptr(lltype.FuncType(ARGS, RES)), ARGS, RES,
             EffectInfo.MOST_GENERAL)
-        
+
         ops = '''
         [f0, f1]
         f2 = float_add(f0, f1)
@@ -2940,4 +3006,4 @@
 
     def alloc_unicode(self, unicode):
         py.test.skip("implement me")
-    
+
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1,7 +1,7 @@
 import sys, os
 from pypy.jit.backend.llsupport import symbolic
 from pypy.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper
-from pypy.jit.metainterp.history import Const, Box, BoxInt, BoxPtr, BoxFloat
+from pypy.jit.metainterp.history import Const, Box, BoxInt, ConstInt
 from pypy.jit.metainterp.history import (AbstractFailDescr, INT, REF, FLOAT,
                                          LoopToken)
 from pypy.rpython.lltypesystem import lltype, rffi, rstr, llmemory
@@ -36,7 +36,6 @@
 from pypy.rlib import rgc
 from pypy.rlib.clibffi import FFI_DEFAULT_ABI
 from pypy.jit.backend.x86.jump import remap_frame_layout
-from pypy.jit.metainterp.history import ConstInt, BoxInt
 from pypy.jit.codewriter.effectinfo import EffectInfo
 from pypy.jit.codewriter import longlong
 
@@ -729,8 +728,8 @@
         # Also, make sure this is consistent with FRAME_FIXED_SIZE.
         self.mc.PUSH_r(ebp.value)
         self.mc.MOV_rr(ebp.value, esp.value)
-        for regloc in self.cpu.CALLEE_SAVE_REGISTERS:
-            self.mc.PUSH_r(regloc.value)
+        for loc in self.cpu.CALLEE_SAVE_REGISTERS:
+            self.mc.PUSH_r(loc.value)
 
         gcrootmap = self.cpu.gc_ll_descr.gcrootmap
         if gcrootmap and gcrootmap.is_shadow_stack:
@@ -994,7 +993,7 @@
         effectinfo = op.getdescr().get_extra_info()
         oopspecindex = effectinfo.oopspecindex
         genop_llong_list[oopspecindex](self, op, arglocs, resloc)
-        
+
     def regalloc_perform_math(self, op, arglocs, resloc):
         effectinfo = op.getdescr().get_extra_info()
         oopspecindex = effectinfo.oopspecindex
@@ -1311,7 +1310,7 @@
     genop_guard_float_eq = _cmpop_guard_float("E", "E", "NE","NE")
     genop_guard_float_gt = _cmpop_guard_float("A", "B", "BE","AE")
     genop_guard_float_ge = _cmpop_guard_float("AE","BE", "B", "A")
-    
+
     def genop_math_sqrt(self, op, arglocs, resloc):
         self.mc.SQRTSD(arglocs[0], resloc)
 
@@ -1597,12 +1596,27 @@
     genop_getarrayitem_gc_pure = genop_getarrayitem_gc
     genop_getarrayitem_raw = genop_getarrayitem_gc
 
+    def genop_getinteriorfield_gc(self, op, arglocs, resloc):
+        base_loc, ofs_loc, itemsize_loc, fieldsize_loc, index_loc, sign_loc = arglocs
+        # XXX should not use IMUL in most cases
+        self.mc.IMUL(index_loc, itemsize_loc)
+        src_addr = AddressLoc(base_loc, index_loc, 0, ofs_loc.value)
+        self.load_from_mem(resloc, src_addr, fieldsize_loc, sign_loc)
+
+
     def genop_discard_setfield_gc(self, op, arglocs):
         base_loc, ofs_loc, size_loc, value_loc = arglocs
         assert isinstance(size_loc, ImmedLoc)
         dest_addr = AddressLoc(base_loc, ofs_loc)
         self.save_into_mem(dest_addr, value_loc, size_loc)
 
+    def genop_discard_setinteriorfield_gc(self, op, arglocs):
+        base_loc, ofs_loc, itemsize_loc, fieldsize_loc, index_loc, value_loc = arglocs
+        # XXX should not use IMUL in most cases
+        self.mc.IMUL(index_loc, itemsize_loc)
+        dest_addr = AddressLoc(base_loc, index_loc, 0, ofs_loc.value)
+        self.save_into_mem(dest_addr, value_loc, fieldsize_loc)
+
     def genop_discard_setarrayitem_gc(self, op, arglocs):
         base_loc, ofs_loc, value_loc, size_loc, baseofs = arglocs
         assert isinstance(baseofs, ImmedLoc)
diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py
--- a/pypy/jit/backend/x86/regalloc.py
+++ b/pypy/jit/backend/x86/regalloc.py
@@ -7,7 +7,7 @@
                                          ResOperation, BoxPtr, ConstFloat,
                                          BoxFloat, LoopToken, INT, REF, FLOAT)
 from pypy.jit.backend.x86.regloc import *
-from pypy.rpython.lltypesystem import lltype, ll2ctypes, rffi, rstr
+from pypy.rpython.lltypesystem import lltype, rffi, rstr
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.rlib import rgc
 from pypy.jit.backend.llsupport import symbolic
@@ -17,11 +17,12 @@
 from pypy.jit.metainterp.resoperation import rop
 from pypy.jit.backend.llsupport.descr import BaseFieldDescr, BaseArrayDescr
 from pypy.jit.backend.llsupport.descr import BaseCallDescr, BaseSizeDescr
+from pypy.jit.backend.llsupport.descr import InteriorFieldDescr
 from pypy.jit.backend.llsupport.regalloc import FrameManager, RegisterManager,\
      TempBox
 from pypy.jit.backend.x86.arch import WORD, FRAME_FIXED_SIZE
 from pypy.jit.backend.x86.arch import IS_X86_32, IS_X86_64, MY_COPY_OF_REGS
-from pypy.rlib.rarithmetic import r_longlong, r_uint
+from pypy.rlib.rarithmetic import r_longlong
 
 class X86RegisterManager(RegisterManager):
 
@@ -433,7 +434,7 @@
             if self.can_merge_with_next_guard(op, i, operations):
                 oplist_with_guard[op.getopnum()](self, op, operations[i + 1])
                 i += 1
-            elif not we_are_translated() and op.getopnum() == -124: 
+            elif not we_are_translated() and op.getopnum() == -124:
                 self._consider_force_spill(op)
             else:
                 oplist[op.getopnum()](self, op)
@@ -815,7 +816,7 @@
         save_all_regs = guard_not_forced_op is not None
         self.xrm.before_call(force_store, save_all_regs=save_all_regs)
         if not save_all_regs:
-            gcrootmap = gc_ll_descr = self.assembler.cpu.gc_ll_descr.gcrootmap
+            gcrootmap = self.assembler.cpu.gc_ll_descr.gcrootmap
             if gcrootmap and gcrootmap.is_shadow_stack:
                 save_all_regs = 2
         self.rm.before_call(force_store, save_all_regs=save_all_regs)
@@ -972,74 +973,27 @@
             return self._call(op, arglocs)
 
     def consider_newstr(self, op):
-        gc_ll_descr = self.assembler.cpu.gc_ll_descr
-        if gc_ll_descr.get_funcptr_for_newstr is not None:
-            # framework GC
-            loc = self.loc(op.getarg(0))
-            return self._call(op, [loc])
-        # boehm GC (XXX kill the following code at some point)
-        ofs_items, itemsize, ofs = symbolic.get_array_token(rstr.STR, self.translate_support_code)
-        assert itemsize == 1
-        return self._malloc_varsize(ofs_items, ofs, 0, op.getarg(0),
-                                    op.result)
+        loc = self.loc(op.getarg(0))
+        return self._call(op, [loc])
 
     def consider_newunicode(self, op):
-        gc_ll_descr = self.assembler.cpu.gc_ll_descr
-        if gc_ll_descr.get_funcptr_for_newunicode is not None:
-            # framework GC
-            loc = self.loc(op.getarg(0))
-            return self._call(op, [loc])
-        # boehm GC (XXX kill the following code at some point)
-        ofs_items, _, ofs = symbolic.get_array_token(rstr.UNICODE,
-                                                   self.translate_support_code)
-        scale = self._get_unicode_item_scale()
-        return self._malloc_varsize(ofs_items, ofs, scale, op.getarg(0),
-                                    op.result)
-
-    def _malloc_varsize(self, ofs_items, ofs_length, scale, v, res_v):
-        # XXX kill this function at some point
-        if isinstance(v, Box):
-            loc = self.rm.make_sure_var_in_reg(v, [v])
-            tempbox = TempBox()
-            other_loc = self.rm.force_allocate_reg(tempbox, [v])
-            self.assembler.load_effective_addr(loc, ofs_items,scale, other_loc)
-        else:
-            tempbox = None
-            other_loc = imm(ofs_items + (v.getint() << scale))
-        self._call(ResOperation(rop.NEW, [], res_v),
-                   [other_loc], [v])
-        loc = self.rm.make_sure_var_in_reg(v, [res_v])
-        assert self.loc(res_v) == eax
-        # now we have to reload length to some reasonable place
-        self.rm.possibly_free_var(v)
-        if tempbox is not None:
-            self.rm.possibly_free_var(tempbox)
-        self.PerformDiscard(ResOperation(rop.SETFIELD_GC, [None, None], None),
-                            [eax, imm(ofs_length), imm(WORD), loc])
+        loc = self.loc(op.getarg(0))
+        return self._call(op, [loc])
 
     def consider_new_array(self, op):
         gc_ll_descr = self.assembler.cpu.gc_ll_descr
-        if gc_ll_descr.get_funcptr_for_newarray is not None:
-            # framework GC
-            box_num_elem = op.getarg(0)
-            if isinstance(box_num_elem, ConstInt):
-                num_elem = box_num_elem.value
-                if gc_ll_descr.can_inline_malloc_varsize(op.getdescr(),
-                                                         num_elem):
-                    self.fastpath_malloc_varsize(op, op.getdescr(), num_elem)
-                    return
-            args = self.assembler.cpu.gc_ll_descr.args_for_new_array(
-                op.getdescr())
-            arglocs = [imm(x) for x in args]
-            arglocs.append(self.loc(box_num_elem))
-            self._call(op, arglocs)
-            return
-        # boehm GC (XXX kill the following code at some point)
-        itemsize, basesize, ofs_length, _, _ = (
-            self._unpack_arraydescr(op.getdescr()))
-        scale_of_field = _get_scale(itemsize)
-        self._malloc_varsize(basesize, ofs_length, scale_of_field,
-                             op.getarg(0), op.result)
+        box_num_elem = op.getarg(0)
+        if isinstance(box_num_elem, ConstInt):
+            num_elem = box_num_elem.value
+            if gc_ll_descr.can_inline_malloc_varsize(op.getdescr(),
+                                                     num_elem):
+                self.fastpath_malloc_varsize(op, op.getdescr(), num_elem)
+                return
+        args = self.assembler.cpu.gc_ll_descr.args_for_new_array(
+            op.getdescr())
+        arglocs = [imm(x) for x in args]
+        arglocs.append(self.loc(box_num_elem))
+        self._call(op, arglocs)
 
     def _unpack_arraydescr(self, arraydescr):
         assert isinstance(arraydescr, BaseArrayDescr)
@@ -1058,6 +1012,16 @@
         sign = fielddescr.is_field_signed()
         return imm(ofs), imm(size), ptr, sign
 
+    def _unpack_interiorfielddescr(self, descr):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs = arraydescr.get_base_size(self.translate_support_code)
+        itemsize = arraydescr.get_item_size(self.translate_support_code)
+        fieldsize = descr.fielddescr.get_field_size(self.translate_support_code)
+        sign = descr.fielddescr.is_field_signed()
+        ofs += descr.fielddescr.offset
+        return imm(ofs), imm(itemsize), imm(fieldsize), sign
+
     def consider_setfield_gc(self, op):
         ofs_loc, size_loc, _, _ = self._unpack_fielddescr(op.getdescr())
         assert isinstance(size_loc, ImmedLoc)
@@ -1074,6 +1038,21 @@
 
     consider_setfield_raw = consider_setfield_gc
 
+    def consider_setinteriorfield_gc(self, op):
+        t = self._unpack_interiorfielddescr(op.getdescr())
+        ofs, itemsize, fieldsize, _ = t
+        args = op.getarglist()
+        tmpvar = TempBox()
+        base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
+        index_loc = self.rm.force_result_in_reg(tmpvar, op.getarg(1),
+                                                args)
+        # we're free to modify index now
+        value_loc = self.make_sure_var_in_reg(op.getarg(2), args)
+        self.possibly_free_vars(args)
+        self.rm.possibly_free_var(tmpvar)
+        self.PerformDiscard(op, [base_loc, ofs, itemsize, fieldsize,
+                                 index_loc, value_loc])
+
     def consider_strsetitem(self, op):
         args = op.getarglist()
         base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
@@ -1135,6 +1114,24 @@
     consider_getarrayitem_raw = consider_getarrayitem_gc
     consider_getarrayitem_gc_pure = consider_getarrayitem_gc
 
+    def consider_getinteriorfield_gc(self, op):
+        t = self._unpack_interiorfielddescr(op.getdescr())
+        ofs, itemsize, fieldsize, sign = t
+        if sign:
+            sign_loc = imm1
+        else:
+            sign_loc = imm0
+        args = op.getarglist()
+        tmpvar = TempBox()
+        base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
+        index_loc = self.rm.force_result_in_reg(tmpvar, op.getarg(1),
+                                                args)
+        self.rm.possibly_free_vars_for_op(op)
+        self.rm.possibly_free_var(tmpvar)
+        result_loc = self.force_allocate_reg(op.result)
+        self.Perform(op, [base_loc, ofs, itemsize, fieldsize,
+                          index_loc, sign_loc], result_loc)
+
     def consider_int_is_true(self, op, guard_op):
         # doesn't need arg to be in a register
         argloc = self.loc(op.getarg(0))
@@ -1241,7 +1238,6 @@
         self.rm.possibly_free_var(srcaddr_box)
 
     def _gen_address_inside_string(self, baseloc, ofsloc, resloc, is_unicode):
-        cpu = self.assembler.cpu
         if is_unicode:
             ofs_items, _, _ = symbolic.get_array_token(rstr.UNICODE,
                                                   self.translate_support_code)
@@ -1300,7 +1296,7 @@
         tmpreg = X86RegisterManager.all_regs[0]
         tmploc = self.rm.force_allocate_reg(box, selected_reg=tmpreg)
         xmmtmp = X86XMMRegisterManager.all_regs[0]
-        xmmtmploc = self.xrm.force_allocate_reg(box1, selected_reg=xmmtmp)
+        self.xrm.force_allocate_reg(box1, selected_reg=xmmtmp)
         # Part about non-floats
         # XXX we don't need a copy, we only just the original list
         src_locations1 = [self.loc(op.getarg(i)) for i in range(op.numargs())
@@ -1380,7 +1376,7 @@
     return lambda self, op: fn(self, op, None)
 
 def is_comparison_or_ovf_op(opnum):
-    from pypy.jit.metainterp.resoperation import opclasses, AbstractResOp
+    from pypy.jit.metainterp.resoperation import opclasses
     cls = opclasses[opnum]
     # hack hack: in theory they are instance method, but they don't use
     # any instance field, we can use a fake object
diff --git a/pypy/jit/backend/x86/test/test_del.py b/pypy/jit/backend/x86/test/test_del.py
--- a/pypy/jit/backend/x86/test/test_del.py
+++ b/pypy/jit/backend/x86/test/test_del.py
@@ -1,5 +1,4 @@
 
-import py
 from pypy.jit.backend.x86.test.test_basic import Jit386Mixin
 from pypy.jit.metainterp.test.test_del import DelTests
 
diff --git a/pypy/jit/backend/x86/test/test_dict.py b/pypy/jit/backend/x86/test/test_dict.py
new file mode 100644
--- /dev/null
+++ b/pypy/jit/backend/x86/test/test_dict.py
@@ -0,0 +1,9 @@
+
+from pypy.jit.backend.x86.test.test_basic import Jit386Mixin
+from pypy.jit.metainterp.test.test_dict import DictTests
+
+
+class TestDict(Jit386Mixin, DictTests):
+    # for the individual tests see
+    # ====> ../../../metainterp/test/test_dict.py
+    pass
diff --git a/pypy/jit/backend/x86/test/test_runner.py b/pypy/jit/backend/x86/test/test_runner.py
--- a/pypy/jit/backend/x86/test/test_runner.py
+++ b/pypy/jit/backend/x86/test/test_runner.py
@@ -31,7 +31,7 @@
 
     # for the individual tests see
     # ====> ../../test/runner_test.py
-    
+
     def setup_method(self, meth):
         self.cpu = CPU(rtyper=None, stats=FakeStats())
         self.cpu.setup_once()
@@ -69,22 +69,16 @@
 
     def test_allocations(self):
         from pypy.rpython.lltypesystem import rstr
-        
+
         allocs = [None]
         all = []
+        orig_new = self.cpu.gc_ll_descr.funcptr_for_new
         def f(size):
             allocs.insert(0, size)
-            buf = ctypes.create_string_buffer(size)
-            all.append(buf)
-            return ctypes.cast(buf, ctypes.c_void_p).value
-        func = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int)(f)
-        addr = ctypes.cast(func, ctypes.c_void_p).value
-        # ctypes produces an unsigned value. We need it to be signed for, eg,
-        # relative addressing to work properly.
-        addr = rffi.cast(lltype.Signed, addr)
-        
+            return orig_new(size)
+
         self.cpu.assembler.setup_once()
-        self.cpu.assembler.malloc_func_addr = addr
+        self.cpu.gc_ll_descr.funcptr_for_new = f
         ofs = symbolic.get_field_token(rstr.STR, 'chars', False)[0]
 
         res = self.execute_operation(rop.NEWSTR, [ConstInt(7)], 'ref')
@@ -108,7 +102,7 @@
         res = self.execute_operation(rop.NEW_ARRAY, [ConstInt(10)],
                                          'ref', descr)
         assert allocs[0] == 10*WORD + ofs + WORD
-        resbuf = self._resbuf(res)            
+        resbuf = self._resbuf(res)
         assert resbuf[ofs/WORD] == 10
 
         # ------------------------------------------------------------
@@ -116,7 +110,7 @@
         res = self.execute_operation(rop.NEW_ARRAY, [BoxInt(10)],
                                          'ref', descr)
         assert allocs[0] == 10*WORD + ofs + WORD
-        resbuf = self._resbuf(res)                        
+        resbuf = self._resbuf(res)
         assert resbuf[ofs/WORD] == 10
 
     def test_stringitems(self):
@@ -146,7 +140,7 @@
                                                      ConstInt(2), BoxInt(38)],
                                'void', descr)
         assert resbuf[itemsofs/WORD + 2] == 38
-        
+
         self.execute_operation(rop.SETARRAYITEM_GC, [res,
                                                      BoxInt(3), BoxInt(42)],
                                'void', descr)
@@ -167,7 +161,7 @@
                                                          BoxInt(2)],
                                    'int', descr)
         assert r.value == 38
-        
+
         r = self.execute_operation(rop.GETARRAYITEM_GC, [res, BoxInt(3)],
                                    'int', descr)
         assert r.value == 42
@@ -226,7 +220,7 @@
         self.execute_operation(rop.SETFIELD_GC, [res, BoxInt(1234)], 'void', ofs_i)
         i = self.execute_operation(rop.GETFIELD_GC, [res], 'int', ofs_i)
         assert i.value == 1234
-        
+
         #u = self.execute_operation(rop.GETFIELD_GC, [res, ofs_u], 'int')
         #assert u.value == 5
         self.execute_operation(rop.SETFIELD_GC, [res, ConstInt(1)], 'void',
@@ -299,7 +293,7 @@
                     else:
                         assert result != execute(self.cpu, None,
                                                  op, None, b).value
-                    
+
 
     def test_stuff_followed_by_guard(self):
         boxes = [(BoxInt(1), BoxInt(0)),
@@ -523,7 +517,7 @@
     def test_debugger_on(self):
         from pypy.tool.logparser import parse_log_file, extract_category
         from pypy.rlib import debug
-        
+
         loop = """
         [i0]
         debug_merge_point('xyz', 0)
diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -52,9 +52,11 @@
         newoperations = []
         #
         def do_rename(var, var_or_const):
+            if var.concretetype is lltype.Void:
+                renamings[var] = Constant(None, lltype.Void)
+                return
             renamings[var] = var_or_const
-            if (isinstance(var_or_const, Constant)
-                and var.concretetype != lltype.Void):
+            if isinstance(var_or_const, Constant):
                 value = var_or_const.value
                 value = lltype._cast_whatever(var.concretetype, value)
                 renamings_constants[var] = Constant(value, var.concretetype)
@@ -735,29 +737,54 @@
         return SpaceOperation(opname, [op.args[0]], op.result)
 
     def rewrite_op_getinteriorfield(self, op):
-        # only supports strings and unicodes
         assert len(op.args) == 3
-        assert op.args[1].value == 'chars'
         optype = op.args[0].concretetype
         if optype == lltype.Ptr(rstr.STR):
             opname = "strgetitem"
+            return SpaceOperation(opname, [op.args[0], op.args[2]], op.result)
+        elif optype == lltype.Ptr(rstr.UNICODE):
+            opname = "unicodegetitem"
+            return SpaceOperation(opname, [op.args[0], op.args[2]], op.result)
         else:
-            assert optype == lltype.Ptr(rstr.UNICODE)
-            opname = "unicodegetitem"
-        return SpaceOperation(opname, [op.args[0], op.args[2]], op.result)
+            v_inst, v_index, c_field = op.args
+            if op.result.concretetype is lltype.Void:
+                return
+            # only GcArray of Struct supported
+            assert isinstance(v_inst.concretetype.TO, lltype.GcArray)
+            STRUCT = v_inst.concretetype.TO.OF
+            assert isinstance(STRUCT, lltype.Struct)
+            descr = self.cpu.interiorfielddescrof(v_inst.concretetype.TO,
+                                                  c_field.value)
+            args = [v_inst, v_index, descr]
+            kind = getkind(op.result.concretetype)[0]
+            return SpaceOperation('getinteriorfield_gc_%s' % kind, args,
+                                  op.result)
 
     def rewrite_op_setinteriorfield(self, op):
-        # only supports strings and unicodes
         assert len(op.args) == 4
-        assert op.args[1].value == 'chars'
         optype = op.args[0].concretetype
         if optype == lltype.Ptr(rstr.STR):
             opname = "strsetitem"
+            return SpaceOperation(opname, [op.args[0], op.args[2], op.args[3]],
+                                  op.result)
+        elif optype == lltype.Ptr(rstr.UNICODE):
+            opname = "unicodesetitem"
+            return SpaceOperation(opname, [op.args[0], op.args[2], op.args[3]],
+                                  op.result)
         else:
-            assert optype == lltype.Ptr(rstr.UNICODE)
-            opname = "unicodesetitem"
-        return SpaceOperation(opname, [op.args[0], op.args[2], op.args[3]],
-                              op.result)
+            v_inst, v_index, c_field, v_value = op.args
+            if v_value.concretetype is lltype.Void:
+                return
+            # only GcArray of Struct supported
+            assert isinstance(v_inst.concretetype.TO, lltype.GcArray)
+            STRUCT = v_inst.concretetype.TO.OF
+            assert isinstance(STRUCT, lltype.Struct)
+            descr = self.cpu.interiorfielddescrof(v_inst.concretetype.TO,
+                                                  c_field.value)
+            kind = getkind(v_value.concretetype)[0]
+            args = [v_inst, v_index, v_value, descr]
+            return SpaceOperation('setinteriorfield_gc_%s' % kind, args,
+                                  op.result)
 
     def _rewrite_equality(self, op, opname):
         arg0, arg1 = op.args
@@ -820,6 +847,12 @@
             return self._float_to_float_cast(v_arg, v_result)
         elif not float_arg and float_res:
             # some int -> some float
+            size1, unsigned1 = rffi.size_and_sign(v_arg.concretetype)
+            assert size1 <= rffi.sizeof(lltype.Signed), (
+                "not implemented: cast_longlong_to_float")
+            assert size1 < rffi.sizeof(lltype.Signed) or not unsigned1, (
+                "not implemented: cast_uint_to_float")
+            #
             ops = []
             v1 = varoftype(lltype.Signed)
             oplist = self.rewrite_operation(
@@ -844,6 +877,12 @@
             return ops
         elif float_arg and not float_res:
             # some float -> some int
+            size2, unsigned2 = rffi.size_and_sign(v_result.concretetype)
+            assert size2 <= rffi.sizeof(lltype.Signed), (
+                "not implemented: cast_float_to_longlong")
+            assert size2 < rffi.sizeof(lltype.Signed) or not unsigned2, (
+                "not implemented: cast_float_to_uint")
+            #
             ops = []
             v1 = varoftype(lltype.Float)
             op1 = self.rewrite_operation(
@@ -1070,8 +1109,8 @@
     # The new operation is optionally further processed by rewrite_operation().
     for _old, _new in [('bool_not', 'int_is_zero'),
                        ('cast_bool_to_float', 'cast_int_to_float'),
-                       ('cast_uint_to_float', 'cast_int_to_float'),
-                       ('cast_float_to_uint', 'cast_float_to_int'),
+                       ('cast_uint_to_float', 'cast_primitive'),
+                       ('cast_float_to_uint', 'cast_primitive'),
 
                        ('int_add_nonneg_ovf', 'int_add_ovf'),
                        ('keepalive', '-live-'),
diff --git a/pypy/jit/codewriter/support.py b/pypy/jit/codewriter/support.py
--- a/pypy/jit/codewriter/support.py
+++ b/pypy/jit/codewriter/support.py
@@ -13,7 +13,6 @@
 from pypy.translator.simplify import get_funcobj
 from pypy.translator.unsimplify import split_block
 from pypy.objspace.flow.model import Constant
-from pypy import conftest
 from pypy.translator.translator import TranslationContext
 from pypy.annotation.policy import AnnotatorPolicy
 from pypy.annotation import model as annmodel
@@ -48,15 +47,13 @@
     a.build_types(func, argtypes, main_entry_point=True)
     rtyper = t.buildrtyper(type_system = type_system)
     rtyper.specialize()
-    if inline:
-        auto_inlining(t, threshold=inline)
+    #if inline:
+    #    auto_inlining(t, threshold=inline)
     if backendoptimize:
         from pypy.translator.backendopt.all import backend_optimizations
         backend_optimizations(t, inline_threshold=inline or 0,
                 remove_asserts=True, really_remove_asserts=True)
 
-    #if conftest.option.view:
-    #    t.view()
     return rtyper
 
 def getgraph(func, values):
@@ -456,6 +453,8 @@
         return LLtypeHelpers._dictnext_items(lltype.Ptr(RES), iter)
     _ll_1_dictiter_nextitems.need_result_type = True
 
+    _ll_1_dict_resize = ll_rdict.ll_dict_resize
+
     # ---------- strings and unicode ----------
 
     _ll_1_str_str2unicode = ll_rstr.LLHelpers.ll_str2unicode
diff --git a/pypy/jit/codewriter/test/test_jtransform.py b/pypy/jit/codewriter/test/test_jtransform.py
--- a/pypy/jit/codewriter/test/test_jtransform.py
+++ b/pypy/jit/codewriter/test/test_jtransform.py
@@ -1,4 +1,3 @@
-import py
 import random
 try:
     from itertools import product
@@ -16,13 +15,13 @@
 
 from pypy.objspace.flow.model import FunctionGraph, Block, Link
 from pypy.objspace.flow.model import SpaceOperation, Variable, Constant
-from pypy.jit.codewriter.jtransform import Transformer
-from pypy.jit.metainterp.history import getkind
-from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr, rlist
+from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr
 from pypy.rpython.lltypesystem.module import ll_math
 from pypy.translator.unsimplify import varoftype
 from pypy.jit.codewriter import heaptracker, effectinfo
 from pypy.jit.codewriter.flatten import ListOfKind
+from pypy.jit.codewriter.jtransform import Transformer
+from pypy.jit.metainterp.history import getkind
 
 def const(x):
     return Constant(x, lltype.typeOf(x))
@@ -37,6 +36,8 @@
         return ('calldescr', FUNC, ARGS, RESULT)
     def fielddescrof(self, STRUCT, name):
         return ('fielddescr', STRUCT, name)
+    def interiorfielddescrof(self, ARRAY, name):
+        return ('interiorfielddescr', ARRAY, name)
     def arraydescrof(self, ARRAY):
         return FakeDescr(('arraydescr', ARRAY))
     def sizeof(self, STRUCT):
@@ -539,7 +540,7 @@
 
 def test_rename_on_links():
     v1 = Variable()
-    v2 = Variable()
+    v2 = Variable(); v2.concretetype = llmemory.Address
     v3 = Variable()
     block = Block([v1])
     block.operations = [SpaceOperation('cast_pointer', [v1], v2)]
@@ -676,6 +677,22 @@
     assert op1.args == [v, v_index]
     assert op1.result == v_result
 
+def test_dict_getinteriorfield():
+    DICT = lltype.GcArray(lltype.Struct('ENTRY', ('v', lltype.Signed),
+                                        ('k', lltype.Signed)))
+    v = varoftype(lltype.Ptr(DICT))
+    i = varoftype(lltype.Signed)
+    v_result = varoftype(lltype.Signed)
+    op = SpaceOperation('getinteriorfield', [v, i, Constant('v', lltype.Void)],
+                        v_result)
+    op1 = Transformer(FakeCPU()).rewrite_operation(op)
+    assert op1.opname == 'getinteriorfield_gc_i'
+    assert op1.args == [v, i, ('interiorfielddescr', DICT, 'v')]
+    op = SpaceOperation('getinteriorfield', [v, i, Constant('v', lltype.Void)],
+                        Constant(None, lltype.Void))
+    op1 = Transformer(FakeCPU()).rewrite_operation(op)
+    assert op1 is None
+
 def test_str_setinteriorfield():
     v = varoftype(lltype.Ptr(rstr.STR))
     v_index = varoftype(lltype.Signed)
@@ -702,6 +719,23 @@
     assert op1.args == [v, v_index, v_newchr]
     assert op1.result == v_void
 
+def test_dict_setinteriorfield():
+    DICT = lltype.GcArray(lltype.Struct('ENTRY', ('v', lltype.Signed),
+                                        ('k', lltype.Signed)))
+    v = varoftype(lltype.Ptr(DICT))
+    i = varoftype(lltype.Signed)
+    v_void = varoftype(lltype.Void)
+    op = SpaceOperation('setinteriorfield', [v, i, Constant('v', lltype.Void),
+                                             i],
+                        v_void)
+    op1 = Transformer(FakeCPU()).rewrite_operation(op)
+    assert op1.opname == 'setinteriorfield_gc_i'
+    assert op1.args == [v, i, i, ('interiorfielddescr', DICT, 'v')]
+    op = SpaceOperation('setinteriorfield', [v, i, Constant('v', lltype.Void),
+                                             v_void], v_void)
+    op1 = Transformer(FakeCPU()).rewrite_operation(op)
+    assert not op1
+
 def test_promote_1():
     v1 = varoftype(lltype.Signed)
     v2 = varoftype(lltype.Signed)
diff --git a/pypy/jit/metainterp/blackhole.py b/pypy/jit/metainterp/blackhole.py
--- a/pypy/jit/metainterp/blackhole.py
+++ b/pypy/jit/metainterp/blackhole.py
@@ -6,7 +6,6 @@
 from pypy.rlib.debug import make_sure_not_resized
 from pypy.rpython.lltypesystem import lltype, llmemory, rclass
 from pypy.rpython.lltypesystem.lloperation import llop
-from pypy.rpython.llinterp import LLException
 from pypy.jit.codewriter.jitcode import JitCode, SwitchDictDescr
 from pypy.jit.codewriter import heaptracker, longlong
 from pypy.jit.metainterp.jitexc import JitException, get_llexception, reraise
@@ -631,6 +630,9 @@
         a = longlong.getrealfloat(a)
         # note: we need to call int() twice to care for the fact that
         # int(-2147483648.0) returns a long :-(
+        # we could also call intmask() instead of the outermost int(), but
+        # it's probably better to explicitly crash (by getting a long) if a
+        # non-translated version tries to cast a too large float to an int.
         return int(int(a))
 
     @arguments("i", returns="f")
@@ -1154,6 +1156,26 @@
         array = cpu.bh_getfield_gc_r(vable, fdescr)
         return cpu.bh_arraylen_gc(adescr, array)
 
+    @arguments("cpu", "r", "i", "d", returns="i")
+    def bhimpl_getinteriorfield_gc_i(cpu, array, index, descr):
+        return cpu.bh_getinteriorfield_gc_i(array, index, descr)
+    @arguments("cpu", "r", "i", "d", returns="r")
+    def bhimpl_getinteriorfield_gc_r(cpu, array, index, descr):
+        return cpu.bh_getinteriorfield_gc_r(array, index, descr)
+    @arguments("cpu", "r", "i", "d", returns="f")
+    def bhimpl_getinteriorfield_gc_f(cpu, array, index, descr):
+        return cpu.bh_getinteriorfield_gc_f(array, index, descr)
+
+    @arguments("cpu", "r", "i", "d", "i")
+    def bhimpl_setinteriorfield_gc_i(cpu, array, index, descr, value):
+        cpu.bh_setinteriorfield_gc_i(array, index, descr, value)
+    @arguments("cpu", "r", "i", "d", "r")
+    def bhimpl_setinteriorfield_gc_r(cpu, array, index, descr, value):
+        cpu.bh_setinteriorfield_gc_r(array, index, descr, value)
+    @arguments("cpu", "r", "i", "d", "f")
+    def bhimpl_setinteriorfield_gc_f(cpu, array, index, descr, value):
+        cpu.bh_setinteriorfield_gc_f(array, index, descr, value)
+
     @arguments("cpu", "r", "d", returns="i")
     def bhimpl_getfield_gc_i(cpu, struct, fielddescr):
         return cpu.bh_getfield_gc_i(struct, fielddescr)
diff --git a/pypy/jit/metainterp/executor.py b/pypy/jit/metainterp/executor.py
--- a/pypy/jit/metainterp/executor.py
+++ b/pypy/jit/metainterp/executor.py
@@ -1,11 +1,8 @@
 """This implements pyjitpl's execution of operations.
 """
 
-import py
-from pypy.rpython.lltypesystem import lltype, llmemory, rstr
-from pypy.rpython.ootypesystem import ootype
-from pypy.rpython.lltypesystem.lloperation import llop
-from pypy.rlib.rarithmetic import ovfcheck, r_uint, intmask, r_longlong
+from pypy.rpython.lltypesystem import lltype, rstr
+from pypy.rlib.rarithmetic import ovfcheck, r_longlong
 from pypy.rlib.rtimer import read_timestamp
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.jit.metainterp.history import BoxInt, BoxPtr, BoxFloat, check_descr
@@ -123,6 +120,29 @@
     else:
         cpu.bh_setarrayitem_raw_i(arraydescr, array, index, itembox.getint())
 
+def do_getinteriorfield_gc(cpu, _, arraybox, indexbox, descr):
+    array = arraybox.getref_base()
+    index = indexbox.getint()
+    if descr.is_pointer_field():
+        return BoxPtr(cpu.bh_getinteriorfield_gc_r(array, index, descr))
+    elif descr.is_float_field():
+        return BoxFloat(cpu.bh_getinteriorfield_gc_f(array, index, descr))
+    else:
+        return BoxInt(cpu.bh_getinteriorfield_gc_i(array, index, descr))
+
+def do_setinteriorfield_gc(cpu, _, arraybox, indexbox, valuebox, descr):
+    array = arraybox.getref_base()
+    index = indexbox.getint()
+    if descr.is_pointer_field():
+        cpu.bh_setinteriorfield_gc_r(array, index, descr,
+                                     valuebox.getref_base())
+    elif descr.is_float_field():
+        cpu.bh_setinteriorfield_gc_f(array, index, descr,
+                                     valuebox.getfloatstorage())
+    else:
+        cpu.bh_setinteriorfield_gc_i(array, index, descr,
+                                     valuebox.getint())
+
 def do_getfield_gc(cpu, _, structbox, fielddescr):
     struct = structbox.getref_base()
     if fielddescr.is_pointer_field():
diff --git a/pypy/jit/metainterp/history.py b/pypy/jit/metainterp/history.py
--- a/pypy/jit/metainterp/history.py
+++ b/pypy/jit/metainterp/history.py
@@ -16,6 +16,7 @@
 INT   = 'i'
 REF   = 'r'
 FLOAT = 'f'
+STRUCT = 's'
 HOLE  = '_'
 VOID  = 'v'
 
@@ -172,6 +173,11 @@
         """
         raise NotImplementedError
 
+    def is_array_of_structs(self):
+        """ Implement for array descr
+        """
+        raise NotImplementedError
+
     def is_pointer_field(self):
         """ Implement for field descr
         """
diff --git a/pypy/jit/metainterp/optimizeopt/rewrite.py b/pypy/jit/metainterp/optimizeopt/rewrite.py
--- a/pypy/jit/metainterp/optimizeopt/rewrite.py
+++ b/pypy/jit/metainterp/optimizeopt/rewrite.py
@@ -448,6 +448,9 @@
         if v2.is_constant() and v2.box.getint() == 1:
             self.make_equal_to(op.result, v1)
             return
+        elif v1.is_constant() and v1.box.getint() == 0:
+            self.make_constant_int(op.result, 0)
+            return
         if v1.intbound.known_ge(IntBound(0, 0)) and v2.is_constant():
             val = v2.box.getint()
             if val & (val - 1) == 0 and val > 0: # val == 2**shift
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -2181,6 +2181,17 @@
         """
         self.optimize_loop(ops, expected)
 
+        ops = """
+        [i0]
+        i1 = int_floordiv(0, i0)
+        jump(i1)
+        """
+        expected = """
+        [i0]
+        jump(0)
+        """
+        self.optimize_loop(ops, expected)
+
     def test_fold_partially_constant_ops_ovf(self):
         ops = """
         [i0]
@@ -4789,6 +4800,18 @@
         """
         self.optimize_strunicode_loop(ops, expected)
 
+    def test_ptr_eq_str_constant(self):
+        ops = """
+        []
+        i0 = ptr_eq(s"abc", s"\x00")
+        finish(i0)
+        """
+        expected = """
+        []
+        finish(0)
+        """
+        self.optimize_loop(ops, expected)
+
 
 class TestLLtype(BaseTestOptimizeBasic, LLtypeMixin):
     pass
diff --git a/pypy/jit/metainterp/optimizeopt/virtualize.py b/pypy/jit/metainterp/optimizeopt/virtualize.py
--- a/pypy/jit/metainterp/optimizeopt/virtualize.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualize.py
@@ -59,7 +59,7 @@
 
     def import_from(self, other, optimizer):
         raise NotImplementedError("should not be called at this level")
-    
+
 def get_fielddescrlist_cache(cpu):
     if not hasattr(cpu, '_optimizeopt_fielddescrlist_cache'):
         result = descrlist_dict()
@@ -113,7 +113,7 @@
         #
         if not we_are_translated():
             op.name = 'FORCE ' + self.source_op.name
-            
+
         if self._is_immutable_and_filled_with_constants(optforce):
             box = optforce.optimizer.constant_fold(op)
             self.make_constant(box)
@@ -239,12 +239,12 @@
         for index in range(len(self._items)):
             self._items[index] = self._items[index].force_at_end_of_preamble(already_forced, optforce)
         return self
-    
+
     def _really_force(self, optforce):
         assert self.source_op is not None
         if not we_are_translated():
             self.source_op.name = 'FORCE ' + self.source_op.name
-        optforce.emit_operation(self.source_op)        
+        optforce.emit_operation(self.source_op)
         self.box = box = self.source_op.result
         for index in range(len(self._items)):
             subvalue = self._items[index]
@@ -276,7 +276,7 @@
 
     def new(self):
         return OptVirtualize()
-        
+
     def make_virtual(self, known_class, box, source_op=None):
         vvalue = VirtualValue(self.optimizer.cpu, known_class, box, source_op)
         self.make_equal_to(box, vvalue)
@@ -386,7 +386,8 @@
 
     def optimize_NEW_ARRAY(self, op):
         sizebox = self.get_constant_box(op.getarg(0))
-        if sizebox is not None:
+        # For now we can't make arrays of structs virtual.
+        if sizebox is not None and not op.getdescr().is_array_of_structs():
             # if the original 'op' did not have a ConstInt as argument,
             # build a new one with the ConstInt argument
             if not isinstance(op.getarg(0), ConstInt):
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -36,6 +36,7 @@
 
 
 class MIFrame(object):
+    debug = False
 
     def __init__(self, metainterp):
         self.metainterp = metainterp
@@ -548,6 +549,14 @@
     opimpl_getfield_gc_r_pure = _opimpl_getfield_gc_pure_any
     opimpl_getfield_gc_f_pure = _opimpl_getfield_gc_pure_any
 
+    @arguments("box", "box", "descr")
+    def _opimpl_getinteriorfield_gc_any(self, array, index, descr):
+        return self.execute_with_descr(rop.GETINTERIORFIELD_GC, descr,
+                                       array, index)
+    opimpl_getinteriorfield_gc_i = _opimpl_getinteriorfield_gc_any
+    opimpl_getinteriorfield_gc_f = _opimpl_getinteriorfield_gc_any
+    opimpl_getinteriorfield_gc_r = _opimpl_getinteriorfield_gc_any
+
     @specialize.arg(1)
     def _opimpl_getfield_gc_any_pureornot(self, opnum, box, fielddescr):
         tobox = self.metainterp.heapcache.getfield(box, fielddescr)
@@ -588,6 +597,15 @@
     opimpl_setfield_gc_r = _opimpl_setfield_gc_any
     opimpl_setfield_gc_f = _opimpl_setfield_gc_any
 
+    @arguments("box", "box", "box", "descr")
+    def _opimpl_setinteriorfield_gc_any(self, array, index, value, descr):
+        self.execute_with_descr(rop.SETINTERIORFIELD_GC, descr,
+                                array, index, value)
+    opimpl_setinteriorfield_gc_i = _opimpl_setinteriorfield_gc_any
+    opimpl_setinteriorfield_gc_f = _opimpl_setinteriorfield_gc_any
+    opimpl_setinteriorfield_gc_r = _opimpl_setinteriorfield_gc_any
+                                
+
     @arguments("box", "descr")
     def _opimpl_getfield_raw_any(self, box, fielddescr):
         return self.execute_with_descr(rop.GETFIELD_RAW, fielddescr, box)
@@ -2588,17 +2606,21 @@
         self.pc = position
         #
         if not we_are_translated():
-            print '\tpyjitpl: %s(%s)' % (name, ', '.join(map(repr, args))),
+            if self.debug:
+                print '\tpyjitpl: %s(%s)' % (name, ', '.join(map(repr, args))),
             try:
                 resultbox = unboundmethod(self, *args)
             except Exception, e:
-                print '-> %s!' % e.__class__.__name__
+                if self.debug:
+                    print '-> %s!' % e.__class__.__name__
                 raise
             if num_return_args == 0:
-                print
+                if self.debug:
+                    print
                 assert resultbox is None
             else:
-                print '-> %r' % (resultbox,)
+                if self.debug:
+                    print '-> %r' % (resultbox,)
                 assert argcodes[next_argcode] == '>'
                 result_argcode = argcodes[next_argcode + 1]
                 assert resultbox.type == {'i': history.INT,
diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py
--- a/pypy/jit/metainterp/resoperation.py
+++ b/pypy/jit/metainterp/resoperation.py
@@ -1,5 +1,4 @@
 from pypy.rlib.objectmodel import we_are_translated
-from pypy.rlib.debug import make_sure_not_resized
 
 def ResOperation(opnum, args, result, descr=None):
     cls = opclasses[opnum]
@@ -405,8 +404,8 @@
     'FLOAT_TRUEDIV/2',
     'FLOAT_NEG/1',
     'FLOAT_ABS/1',
-    'CAST_FLOAT_TO_INT/1',
-    'CAST_INT_TO_FLOAT/1',
+    'CAST_FLOAT_TO_INT/1',          # don't use for unsigned ints; we would
+    'CAST_INT_TO_FLOAT/1',          # need some messy code in the backend
     'CAST_FLOAT_TO_SINGLEFLOAT/1',
     'CAST_SINGLEFLOAT_TO_FLOAT/1',
     #
@@ -457,6 +456,7 @@
 
     'GETARRAYITEM_GC/2d',
     'GETARRAYITEM_RAW/2d',
+    'GETINTERIORFIELD_GC/2d',
     'GETFIELD_GC/1d',
     'GETFIELD_RAW/1d',
     '_MALLOC_FIRST',
@@ -473,6 +473,7 @@
 
     'SETARRAYITEM_GC/3d',
     'SETARRAYITEM_RAW/3d',
+    'SETINTERIORFIELD_GC/3d',
     'SETFIELD_GC/2d',
     'SETFIELD_RAW/2d',
     'STRSETITEM/3',
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -3435,7 +3435,39 @@
             return sa
         res = self.meta_interp(f, [16])
         assert res == f(16)
-        
+
+    def test_ptr_eq_str_constants(self):
+        myjitdriver = JitDriver(greens = [], reds = ["n", "x"])
+        class A(object):
+            def __init__(self, v):
+                self.v = v
+        def f(n, x):
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, x=x)
+                z = 0 / x
+                a1 = A("key")
+                a2 = A("\x00")
+                n -= [a1, a2][z].v is not a2.v
+            return n
+        res = self.meta_interp(f, [10, 1])
+        assert res == 0
+
+    def test_virtual_array_of_structs(self):
+        myjitdriver = JitDriver(greens = [], reds=["n", "d"])
+        def f(n):
+            d = None
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, d=d)
+                d = {}
+                if n % 2:
+                    d["k"] = n
+                else:
+                    d["z"] = n
+                n -= len(d)
+            return n
+        res = self.meta_interp(f, [10])
+        assert res == 0
+
 
 
 class TestLLtype(BaseLLtypeTests, LLJitMixin):
diff --git a/pypy/jit/metainterp/test/test_dict.py b/pypy/jit/metainterp/test/test_dict.py
--- a/pypy/jit/metainterp/test/test_dict.py
+++ b/pypy/jit/metainterp/test/test_dict.py
@@ -91,7 +91,7 @@
         res1 = f(100)
         res2 = self.meta_interp(f, [100], listops=True)
         assert res1 == res2
-        self.check_loops(int_mod=1) # the hash was traced
+        self.check_loops(int_mod=1) # the hash was traced and eq, but cached
 
     def test_dict_setdefault(self):
         myjitdriver = JitDriver(greens = [], reds = ['total', 'dct'])
@@ -128,7 +128,7 @@
         assert f(100) == 50
         res = self.meta_interp(f, [100], listops=True)
         assert res == 50
-        self.check_loops(int_mod=1)
+        self.check_loops(int_mod=1) # key + eq, but cached
 
     def test_repeated_lookup(self):
         myjitdriver = JitDriver(greens = [], reds = ['n', 'd'])
@@ -153,10 +153,12 @@
 
         res = self.meta_interp(f, [100], listops=True)
         assert res == f(50)
-        self.check_loops({"call": 7, "guard_false": 1, "guard_no_exception": 6,
+        self.check_loops({"call": 5, "getfield_gc": 1, "getinteriorfield_gc": 1,
+                          "guard_false": 1, "guard_no_exception": 4,
                           "guard_true": 1, "int_and": 1, "int_gt": 1,
                           "int_is_true": 1, "int_sub": 1, "jump": 1,
-                          "new_with_vtable": 1, "setfield_gc": 1})
+                          "new_with_vtable": 1, "new": 1, "new_array": 1,
+                          "setfield_gc": 3, })
 
 
 class TestOOtype(DictTests, OOJitMixin):
diff --git a/pypy/jit/metainterp/test/test_float.py b/pypy/jit/metainterp/test/test_float.py
--- a/pypy/jit/metainterp/test/test_float.py
+++ b/pypy/jit/metainterp/test/test_float.py
@@ -1,5 +1,6 @@
-import math
+import math, sys
 from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin
+from pypy.rlib.rarithmetic import intmask, r_uint
 
 
 class FloatTests:
@@ -45,6 +46,41 @@
         res = self.interp_operations(f, [-2.0])
         assert res == -8.5
 
+    def test_cast_float_to_int(self):
+        def g(f):
+            return int(f)
+        res = self.interp_operations(g, [-12345.9])
+        assert res == -12345
+
+    def test_cast_float_to_uint(self):
+        def g(f):
+            return intmask(r_uint(f))
+        raises(AssertionError, self.interp_operations, g, [0.0])   # for now
+        #res = self.interp_operations(g, [sys.maxint*2.0])
+        #assert res == intmask(long(sys.maxint*2.0))
+        #res = self.interp_operations(g, [-12345.9])
+        #assert res == -12345
+
+    def test_cast_int_to_float(self):
+        def g(i):
+            return float(i)
+        res = self.interp_operations(g, [-12345])
+        assert type(res) is float and res == -12345.0
+
+    def test_cast_uint_to_float(self):
+        def g(i):
+            return float(r_uint(i))
+        raises(AssertionError, self.interp_operations, g, [0])    # for now
+        #res = self.interp_operations(g, [sys.maxint*2])
+        #assert type(res) is float and res == float(sys.maxint*2)
+        #res = self.interp_operations(g, [-12345])
+        #assert type(res) is float and res == float(long(r_uint(-12345)))
+
+    #def test_cast_longlong_to_float(self):
+    #def test_cast_ulonglong_to_float(self):
+    #def test_cast_float_to_longlong(self):
+    #def test_cast_float_to_ulonglong(self):
+
 
 class TestOOtype(FloatTests, OOJitMixin):
     pass
diff --git a/pypy/jit/metainterp/warmspot.py b/pypy/jit/metainterp/warmspot.py
--- a/pypy/jit/metainterp/warmspot.py
+++ b/pypy/jit/metainterp/warmspot.py
@@ -206,7 +206,7 @@
         self.make_enter_functions()
         self.rewrite_jit_merge_points(policy)
 
-        verbose = not self.cpu.translate_support_code
+        verbose = False # not self.cpu.translate_support_code
         self.codewriter.make_jitcodes(verbose=verbose)
         self.rewrite_can_enter_jits()
         self.rewrite_set_param()
diff --git a/pypy/module/__builtin__/app_io.py b/pypy/module/__builtin__/app_io.py
--- a/pypy/module/__builtin__/app_io.py
+++ b/pypy/module/__builtin__/app_io.py
@@ -71,7 +71,7 @@
         return line[:-1]
     return line
 
-def input(prompt=None):
+def input(prompt=''):
     """Equivalent to eval(raw_input(prompt))."""
     return eval(raw_input(prompt))
 
diff --git a/pypy/module/__builtin__/test/test_rawinput.py b/pypy/module/__builtin__/test/test_rawinput.py
--- a/pypy/module/__builtin__/test/test_rawinput.py
+++ b/pypy/module/__builtin__/test/test_rawinput.py
@@ -3,29 +3,32 @@
 
 class AppTestRawInput():
 
-    def test_raw_input(self):
+    def test_input_and_raw_input(self):
         import sys, StringIO
         for prompt, expected in [("def:", "abc/ def:/ghi\n"),
                                  ("", "abc/ /ghi\n"),
                                  (42, "abc/ 42/ghi\n"),
                                  (None, "abc/ None/ghi\n"),
                                  (Ellipsis, "abc/ /ghi\n")]:
-            save = sys.stdin, sys.stdout
-            try:
-                sys.stdin = StringIO.StringIO("foo\nbar\n")
-                out = sys.stdout = StringIO.StringIO()
-                print "abc",    # softspace = 1
-                out.write('/')
-                if prompt is Ellipsis:
-                    got = raw_input()
-                else:
-                    got = raw_input(prompt)
-                out.write('/')
-                print "ghi"
-            finally:
-                sys.stdin, sys.stdout = save
-            assert out.getvalue() == expected
-            assert got == "foo"
+            for inputfn, inputtext, gottext in [
+                    (raw_input, "foo\nbar\n", "foo"),
+                    (input, "40+2\n", 42)]:
+                save = sys.stdin, sys.stdout
+                try:
+                    sys.stdin = StringIO.StringIO(inputtext)
+                    out = sys.stdout = StringIO.StringIO()
+                    print "abc",    # softspace = 1
+                    out.write('/')
+                    if prompt is Ellipsis:
+                        got = inputfn()
+                    else:
+                        got = inputfn(prompt)
+                    out.write('/')
+                    print "ghi"
+                finally:
+                    sys.stdin, sys.stdout = save
+                assert out.getvalue() == expected
+                assert got == gottext
 
     def test_softspace(self):
         import sys
diff --git a/pypy/module/pypyjit/test_pypy_c/test_containers.py b/pypy/module/pypyjit/test_pypy_c/test_containers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_containers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_containers.py
@@ -46,7 +46,7 @@
         assert loop.match_by_id("getitem", """
             i28 = call(ConstClass(ll_dict_lookup__dicttablePtr_objectPtr_Signed), p18, p6, i25, descr=...)
             ...
-            p33 = call(ConstClass(ll_get_value__dicttablePtr_Signed), p18, i28, descr=...)
+            p33 = getinteriorfield_gc(p31, i26, descr=<InteriorFieldDescr <GcPtrFieldDescr dictentry.value .*>>)
             ...
         """)
 
diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -93,7 +93,8 @@
             i46 = call(ConstClass(ll_startswith__rpy_stringPtr_rpy_stringPtr), p28, ConstPtr(ptr45), descr=<BoolCallDescr>)
             guard_false(i46, descr=...)
             p51 = new_with_vtable(21136408)
-            setfield_gc(p51, _, descr=...)    # 6 setfields, but the order is dict-order-dependent
+            setfield_gc(p51, _, descr=...)    # 7 setfields, but the order is dict-order-dependent
+            setfield_gc(p51, _, descr=...)
             setfield_gc(p51, _, descr=...)
             setfield_gc(p51, _, descr=...)
             setfield_gc(p51, _, descr=...)
diff --git a/pypy/objspace/std/newformat.py b/pypy/objspace/std/newformat.py
--- a/pypy/objspace/std/newformat.py
+++ b/pypy/objspace/std/newformat.py
@@ -120,6 +120,8 @@
             out.append_slice(s, last_literal, end)
             return out.build()
 
+        # This is only ever called if we're already unrolling _do_build_string
+        @jit.unroll_safe
         def _parse_field(self, start, end):
             s = self.template
             # Find ":" or "!"
@@ -149,6 +151,7 @@
                 i += 1
             return s[start:end], None, end
 
+        @jit.unroll_safe
         def _get_argument(self, name):
             # First, find the argument.
             space = self.space
@@ -207,6 +210,7 @@
                     raise OperationError(space.w_IndexError, w_msg)
             return self._resolve_lookups(w_arg, name, i, end)
 
+        @jit.unroll_safe
         def _resolve_lookups(self, w_obj, name, start, end):
             # Resolve attribute and item lookups.
             space = self.space
diff --git a/pypy/rlib/rerased.py b/pypy/rlib/rerased.py
--- a/pypy/rlib/rerased.py
+++ b/pypy/rlib/rerased.py
@@ -135,6 +135,8 @@
     _about_ = erase_int
 
     def compute_result_annotation(self, s_obj):
+        config = self.bookkeeper.annotator.translator.config
+        assert config.translation.taggedpointers, "need to enable tagged pointers to use erase_int"
         assert annmodel.SomeInteger().contains(s_obj)
         return SomeErased()
 
@@ -228,6 +230,8 @@
 
     def convert_const(self, value):
         if value._identity is _identity_for_ints:
+            config = self.rtyper.annotator.translator.config
+            assert config.translation.taggedpointers, "need to enable tagged pointers to use erase_int"
             return lltype.cast_int_to_ptr(self.lowleveltype, value._x * 2 + 1)
         bk = self.rtyper.annotator.bookkeeper
         s_obj = value._identity.get_input_annotation(bk)
diff --git a/pypy/rlib/test/test_rerased.py b/pypy/rlib/test/test_rerased.py
--- a/pypy/rlib/test/test_rerased.py
+++ b/pypy/rlib/test/test_rerased.py
@@ -10,6 +10,13 @@
 
 from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin
 
+def make_annotator():
+    a = RPythonAnnotator()
+    a.translator.config.translation.taggedpointers = True
+    return a
+
+
+
 class X(object):
     pass
 
@@ -55,7 +62,7 @@
 def test_annotate_1():
     def f():
         return eraseX(X())
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [])
     assert isinstance(s, SomeErased)
 
@@ -66,7 +73,7 @@
         #assert not is_integer(e)
         x2 = uneraseX(e)
         return x2
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [])
     assert isinstance(s, annmodel.SomeInstance)
     assert s.classdef == a.bookkeeper.getuniqueclassdef(X)
@@ -77,7 +84,7 @@
         #assert is_integer(e)
         x2 = unerase_int(e)
         return x2
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [])
     assert isinstance(s, annmodel.SomeInteger)
 
@@ -105,7 +112,7 @@
         x = make(n)
         return check(x, n)
     #
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [int])
     assert isinstance(s, annmodel.SomeInteger)
 
@@ -135,7 +142,7 @@
             else:
                 return inst
     #
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [])
     assert isinstance(s, annmodel.SomeInstance)
     assert s.classdef == a.bookkeeper.getuniqueclassdef(A)
@@ -155,7 +162,7 @@
             e = e2
         return unerase(e)
     #
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [int])
     assert isinstance(s, annmodel.SomeInstance)
     assert s.classdef == a.bookkeeper.getuniqueclassdef(X)
@@ -165,11 +172,14 @@
     e1 = erase_int(42)
     def f(i):
         return unerase_int(e1)
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [int])
     assert isinstance(s, annmodel.SomeInteger)
 
 class BaseTestRErased(BaseRtypingTest):
+    def interpret(self, *args, **kwargs):
+        kwargs["taggedpointers"] = True
+        return BaseRtypingTest.interpret(self, *args, **kwargs)
     def test_rtype_1(self):
         def f():
             return eraseX(X())
diff --git a/pypy/rpython/llinterp.py b/pypy/rpython/llinterp.py
--- a/pypy/rpython/llinterp.py
+++ b/pypy/rpython/llinterp.py
@@ -1101,13 +1101,6 @@
             assert y >= 0
         return self.op_int_add_ovf(x, y)
 
-    def op_cast_float_to_int(self, f):
-        assert type(f) is float
-        try:
-            return ovfcheck(int(f))
-        except OverflowError:
-            self.make_llexception()
-
     def op_int_is_true(self, x):
         # special case
         if type(x) is CDefinedIntSymbolic:
diff --git a/pypy/rpython/lltypesystem/ll2ctypes.py b/pypy/rpython/lltypesystem/ll2ctypes.py
--- a/pypy/rpython/lltypesystem/ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/ll2ctypes.py
@@ -15,12 +15,11 @@
     load_library_kwargs = {}
 
 import os
-from pypy import conftest
 from pypy.rpython.lltypesystem import lltype, llmemory
 from pypy.rpython.extfunc import ExtRegistryEntry
 from pypy.rlib.objectmodel import Symbolic, ComputedIntSymbolic
 from pypy.tool.uid import fixid
-from pypy.rlib.rarithmetic import r_uint, r_singlefloat, r_longfloat, base_int, intmask
+from pypy.rlib.rarithmetic import r_singlefloat, r_longfloat, base_int, intmask
 from pypy.annotation import model as annmodel
 from pypy.rpython.llinterp import LLInterpreter, LLException
 from pypy.rpython.lltypesystem.rclass import OBJECT, OBJECT_VTABLE
@@ -526,6 +525,10 @@
     def __str__(self):
         return repr(self)
 
+    def _setparentstructure(self, parent, parentindex):
+        super(_parentable_mixin, self)._setparentstructure(parent, parentindex)
+        self._keepparent = parent   # always keep a strong ref
+
 class _struct_mixin(_parentable_mixin):
     """Mixin added to _struct containers when they become ctypes-based."""
     __slots__ = ()
@@ -561,7 +564,10 @@
         return 0, sys.maxint
 
     def getitem(self, index, uninitialized_ok=False):
-        return self._storage.contents._getitem(index, boundscheck=False)
+        res = self._storage.contents._getitem(index, boundscheck=False)
+        if isinstance(self._TYPE.OF, lltype.ContainerType):
+            res._obj._setparentstructure(self, index)
+        return res
 
     def setitem(self, index, value):
         self._storage.contents._setitem(index, value, boundscheck=False)
@@ -1274,6 +1280,8 @@
             self.intval = intmask(void_p.value)
 
     def __eq__(self, other):
+        if not other:
+            return self.intval == 0
         if isinstance(other, _llgcopaque):
             return self.intval == other.intval
         storage = object()
diff --git a/pypy/rpython/lltypesystem/lloperation.py b/pypy/rpython/lltypesystem/lloperation.py
--- a/pypy/rpython/lltypesystem/lloperation.py
+++ b/pypy/rpython/lltypesystem/lloperation.py
@@ -344,8 +344,8 @@
     'cast_uint_to_float':   LLOp(canfold=True),
     'cast_longlong_to_float' :LLOp(canfold=True),
     'cast_ulonglong_to_float':LLOp(canfold=True),
-    'cast_float_to_int':    LLOp(canraise=(OverflowError,), tryfold=True),
-    'cast_float_to_uint':   LLOp(canfold=True),    # XXX need OverflowError?
+    'cast_float_to_int':    LLOp(canfold=True),
+    'cast_float_to_uint':   LLOp(canfold=True),
     'cast_float_to_longlong' :LLOp(canfold=True),
     'cast_float_to_ulonglong':LLOp(canfold=True),
     'truncate_longlong_to_int':LLOp(canfold=True),
diff --git a/pypy/rpython/lltypesystem/lltype.py b/pypy/rpython/lltypesystem/lltype.py
--- a/pypy/rpython/lltypesystem/lltype.py
+++ b/pypy/rpython/lltypesystem/lltype.py
@@ -1525,7 +1525,7 @@
             and parentindex in (self._parent_type._names[0], 0)
             and self._TYPE._gckind == typeOf(parent)._gckind):
             # keep strong reference to parent, we share the same allocation
-            self._keepparent = parent 
+            self._keepparent = parent
 
     def _parentstructure(self, check=True):
         if self._wrparent is not None:
@@ -1733,7 +1733,8 @@
         # Keep the parent array alive, we share the same allocation.
         # Don't do it if we are inside a GC object, though -- it's someone
         # else's job to keep the GC object alive
-        if typeOf(top_container(parent))._gckind == 'raw':
+        if (typeOf(top_container(parent))._gckind == 'raw' or
+            hasattr(top_container(parent)._storage, 'contents')):  # ll2ctypes
             self._keepparent = parent
 
     def __str__(self):
diff --git a/pypy/rpython/lltypesystem/opimpl.py b/pypy/rpython/lltypesystem/opimpl.py
--- a/pypy/rpython/lltypesystem/opimpl.py
+++ b/pypy/rpython/lltypesystem/opimpl.py
@@ -366,6 +366,10 @@
     assert type(b) is bool
     return float(b)
 
+def op_cast_float_to_int(f):
+    assert type(f) is float
+    return intmask(int(f))
+
 def op_cast_float_to_uint(f):
     assert type(f) is float
     return r_uint(long(f))
diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py
--- a/pypy/rpython/lltypesystem/rdict.py
+++ b/pypy/rpython/lltypesystem/rdict.py
@@ -1,16 +1,14 @@
 from pypy.tool.pairtype import pairtype
-from pypy.annotation import model as annmodel
 from pypy.objspace.flow.model import Constant
-from pypy.rpython.rdict import AbstractDictRepr, AbstractDictIteratorRepr,\
-     rtype_newdict
+from pypy.rpython.rdict import (AbstractDictRepr, AbstractDictIteratorRepr,
+     rtype_newdict)
 from pypy.rpython.lltypesystem import lltype
+from pypy.rlib import objectmodel, jit
 from pypy.rlib.rarithmetic import r_uint, intmask, LONG_BIT
-from pypy.rlib.objectmodel import hlinvoke
-from pypy.rpython import robject
-from pypy.rlib import objectmodel, jit
 from pypy.rpython import rmodel
 from pypy.rpython.error import TyperError
 
+
 HIGHEST_BIT = intmask(1 << (LONG_BIT - 1))
 MASK = intmask(HIGHEST_BIT - 1)
 
@@ -417,17 +415,16 @@
     ENTRIES = lltype.typeOf(entries).TO
     return ENTRIES.fasthashfn(entries[i].key)
 
- at jit.dont_look_inside
 def ll_get_value(d, i):
     return d.entries[i].value
 
 def ll_keyhash_custom(d, key):
     DICT = lltype.typeOf(d).TO
-    return hlinvoke(DICT.r_rdict_hashfn, d.fnkeyhash, key)
+    return objectmodel.hlinvoke(DICT.r_rdict_hashfn, d.fnkeyhash, key)
 
 def ll_keyeq_custom(d, key1, key2):
     DICT = lltype.typeOf(d).TO
-    return hlinvoke(DICT.r_rdict_eqfn, d.fnkeyeq, key1, key2)
+    return objectmodel.hlinvoke(DICT.r_rdict_eqfn, d.fnkeyeq, key1, key2)
 
 def ll_dict_len(d):
     return d.num_items
@@ -448,6 +445,8 @@
     i = ll_dict_lookup(d, key, hash)
     return _ll_dict_setitem_lookup_done(d, key, value, hash, i)
 
+# Leaving as dont_look_inside ATM, it has a few branches which could lead to
+# many bridges if we don't consider their possible frequency.
 @jit.dont_look_inside
 def _ll_dict_setitem_lookup_done(d, key, value, hash, i):
     valid = (i & HIGHEST_BIT) == 0
@@ -492,6 +491,8 @@
         raise KeyError
     _ll_dict_del(d, i)
 
+# XXX: Move the size checking and resize into a single call which is opauqe to
+# the JIT to avoid extra branches.
 @jit.dont_look_inside
 def _ll_dict_del(d, i):
     d.entries.mark_deleted(i)
@@ -532,6 +533,7 @@
 # ------- a port of CPython's dictobject.c's lookdict implementation -------
 PERTURB_SHIFT = 5
 
+ at jit.dont_look_inside
 def ll_dict_lookup(d, key, hash):
     entries = d.entries
     ENTRIES = lltype.typeOf(entries).TO
@@ -621,7 +623,6 @@
     d.num_items = 0
     d.resize_counter = DICT_INITSIZE * 2
     return d
-ll_newdict.oopspec = 'newdict()'
 
 def ll_newdict_size(DICT, length_estimate):
     length_estimate = (length_estimate // 2) * 3
@@ -633,7 +634,6 @@
     d.num_items = 0
     d.resize_counter = n * 2
     return d
-ll_newdict_size.oopspec = 'newdict()'
 
 # pypy.rpython.memory.lldict uses a dict based on Struct and Array
 # instead of GcStruct and GcArray, which is done by using different
@@ -866,7 +866,6 @@
         global_popitem_index.nextindex = base + counter
     return i
 
- at jit.dont_look_inside
 def ll_popitem(ELEM, dic):
     i = _ll_getnextitem(dic)
     entry = dic.entries[i]
diff --git a/pypy/rpython/lltypesystem/test/test_ll2ctypes.py b/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
--- a/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
@@ -8,6 +8,7 @@
 from pypy.rpython.lltypesystem.ll2ctypes import uninitialized2ctypes
 from pypy.rpython.lltypesystem.ll2ctypes import ALLOCATED, force_cast
 from pypy.rpython.lltypesystem.ll2ctypes import cast_adr_to_int, get_ctypes_type
+from pypy.rpython.lltypesystem.ll2ctypes import _llgcopaque
 from pypy.rpython.annlowlevel import llhelper
 from pypy.rlib import rposix
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
@@ -1349,6 +1350,11 @@
         round = ctypes2lltype(llmemory.GCREF, lltype2ctypes(opaque.hide()))
         assert Opaque.show(round) is opaque
 
+    def test_array_of_structs(self):
+        A = lltype.GcArray(lltype.Struct('x', ('v', lltype.Signed)))
+        a = lltype.malloc(A, 5)
+        a2 = ctypes2lltype(lltype.Ptr(A), lltype2ctypes(a))
+        assert a2._obj.getitem(0)._obj._parentstructure() is a2._obj
 
 class TestPlatform(object):
     def test_lib_on_libpaths(self):
@@ -1390,3 +1396,7 @@
         f = rffi.llexternal('f', [rffi.INT, rffi.INT], rffi.INT,
                             compilation_info=eci)
         assert f(3, 4) == 7
+
+    def test_llgcopaque_eq(self):
+        assert _llgcopaque(1) != None
+        assert _llgcopaque(0) == None
diff --git a/pypy/translator/c/database.py b/pypy/translator/c/database.py
--- a/pypy/translator/c/database.py
+++ b/pypy/translator/c/database.py
@@ -176,7 +176,7 @@
                      # introduced by the GC transformer, or the type_info_table
         return node
 
-    def get(self, obj):
+    def get(self, obj, funcgen=None):
         if isinstance(obj, CConstant):
             return obj.c_name  # without further checks
         T = typeOf(obj)
@@ -228,6 +228,8 @@
                     return '((%s) %d)' % (cdecl(self.gettype(T), ''),
                                           obj._obj)
                 node = self.getcontainernode(container)
+                if node._funccodegen_owner is None:
+                    node._funccodegen_owner = funcgen
                 return node.getptrname()
             else:
                 return '((%s) NULL)' % (cdecl(self.gettype(T), ''), )
@@ -284,14 +286,16 @@
             finish_callbacks.append(('GC transformer: finished tables',
                                      self.gctransformer.get_finish_tables()))
 
-        def add_dependencies(newdependencies):
+        def add_dependencies(newdependencies, parent=None):
             for value in newdependencies:
                 #if isinstance(value, _uninitialized):
                 #    continue
                 if isinstance(typeOf(value), ContainerType):
-                    self.getcontainernode(value)
+                    node = self.getcontainernode(value)
+                    if parent and node._funccodegen_owner is not None:
+                        node._funccodegen_owner = parent._funccodegen_owner
                 else:
-                    self.get(value)
+                    self.get(value, parent and parent._funccodegen_owner)
 
         while True:
             while True:
@@ -303,7 +307,7 @@
                 if i == len(self.containerlist):
                     break
                 node = self.containerlist[i]
-                add_dependencies(node.enum_dependencies())
+                add_dependencies(node.enum_dependencies(), node)
                 i += 1
                 self.completedcontainers = i
                 if i == show_i:
diff --git a/pypy/translator/c/gc.py b/pypy/translator/c/gc.py
--- a/pypy/translator/c/gc.py
+++ b/pypy/translator/c/gc.py
@@ -170,6 +170,7 @@
     nodekind = 'refcnt rtti'
     globalcontainer = True
     typename = 'void (@)(void *)'
+    _funccodegen_owner = None
 
     def __init__(self, db, T, obj):
         assert T == RuntimeTypeInfo
@@ -266,6 +267,7 @@
     nodekind = 'boehm rtti'
     globalcontainer = True
     typename = 'char @'
+    _funccodegen_owner = None
 
     def __init__(self, db, T, obj):
         assert T == RuntimeTypeInfo
diff --git a/pypy/translator/c/gcc/test/test_asmgcroot.py b/pypy/translator/c/gcc/test/test_asmgcroot.py
--- a/pypy/translator/c/gcc/test/test_asmgcroot.py
+++ b/pypy/translator/c/gcc/test/test_asmgcroot.py
@@ -21,6 +21,7 @@
         config = get_pypy_config(translating=True)
         config.translation.gc = cls.gcpolicy
         config.translation.gcrootfinder = "asmgcc"
+        config.translation.taggedpointers = getattr(cls, "taggedpointers", False)
         return config
 
     @classmethod
diff --git a/pypy/translator/c/genc.py b/pypy/translator/c/genc.py
--- a/pypy/translator/c/genc.py
+++ b/pypy/translator/c/genc.py
@@ -707,8 +707,7 @@
     def getbasecfilefornode(self, node, basecname):
         # For FuncNode instances, use the python source filename (relative to
         # the top directory):
-        if hasattr(node.obj, 'graph'):
-            g = node.obj.graph
+        def invent_nice_name(g):
             # Lookup the filename from the function.
             # However, not all FunctionGraph objs actually have a "func":
             if hasattr(g, 'func'):
@@ -718,6 +717,15 @@
                     if pypkgpath:
                         relpypath =  localpath.relto(pypkgpath)
                         return relpypath.replace('.py', '.c')
+            return None
+        if hasattr(node.obj, 'graph'):
+            name = invent_nice_name(node.obj.graph)
+            if name is not None:
+                return name
+        elif node._funccodegen_owner is not None:
+            name = invent_nice_name(node._funccodegen_owner.graph)
+            if name is not None:
+                return "data_" + name
         return basecname
 
     def splitnodesimpl(self, basecname, nodes, nextra, nbetween,
diff --git a/pypy/translator/c/node.py b/pypy/translator/c/node.py
--- a/pypy/translator/c/node.py
+++ b/pypy/translator/c/node.py
@@ -485,6 +485,7 @@
         __slots__ = """db obj 
                        typename implementationtypename
                         name
+                        _funccodegen_owner
                         globalcontainer""".split()
     eci_name = '_compilation_info'
 
@@ -509,6 +510,7 @@
         if self.typename != self.implementationtypename:
             if db.gettypedefnode(T).extra_union_for_varlength:
                 self.name += '.b'
+        self._funccodegen_owner = None
 
     def getptrname(self):
         return '(&%s)' % self.name
@@ -842,6 +844,9 @@
         if self.funcgens:
             argnames = self.funcgens[0].argnames()  #Assume identical for all funcgens
             self.implementationtypename = self.db.gettype(self.T, argnames=argnames)
+            self._funccodegen_owner = self.funcgens[0]
+        else:
+            self._funccodegen_owner = None
 
     def basename(self):
         return self.obj._name
@@ -1005,6 +1010,7 @@
     globalcontainer = True
     typename = 'PyObject @'
     implementationtypename = 'PyObject *@'
+    _funccodegen_owner = None
 
     def __init__(self, db, T, obj):
         # obj is a _pyobject here; obj.value is the underlying CPython object
diff --git a/pypy/translator/c/primitive.py b/pypy/translator/c/primitive.py
--- a/pypy/translator/c/primitive.py
+++ b/pypy/translator/c/primitive.py
@@ -146,7 +146,12 @@
 
 def name_gcref(value, db):
     if value:
-        realobj = value._obj.container
+        obj = value._obj
+        if isinstance(obj, int):
+            # a tagged pointer
+            assert obj & 1 == 1
+            return '((%s) %d)' % (cdecl("void*", ''), obj)
+        realobj = obj.container
         realvalue = cast_opaque_ptr(Ptr(typeOf(realobj)), value)
         return db.get(realvalue)
     else:
diff --git a/pypy/translator/c/test/test_newgc.py b/pypy/translator/c/test/test_newgc.py
--- a/pypy/translator/c/test/test_newgc.py
+++ b/pypy/translator/c/test/test_newgc.py
@@ -1509,6 +1509,43 @@
         res = self.run("tagged")
         assert res == expected
 
+    def define_erased(cls):
+        from pypy.rlib import rerased
+        erase, unerase = rerased.new_erasing_pair("test")
+        class Unrelated(object):
+            pass
+
+        u = Unrelated()
+        u.tagged = True
+        u.x = rerased.erase_int(41)
+        class A(object):
+            pass
+        def fn():
+            n = 1
+            while n >= 0:
+                if u.tagged:
+                    n = rerased.unerase_int(u.x)
+                    a = A()
+                    a.n = n - 1
+                    u.x = erase(a)
+                    u.tagged = False
+                else:
+                    n = unerase(u.x).n
+                    u.x = rerased.erase_int(n - 1)
+                    u.tagged = True
+        def func():
+            rgc.collect() # check that a prebuilt erased integer doesn't explode
+            u.x = rerased.erase_int(1000)
+            u.tagged = True
+            fn()
+            return 1
+        return func
+
+    def test_erased(self):
+        expected = self.run_orig("erased")
+        res = self.run("erased")
+        assert res == expected
+
 from pypy.rlib.objectmodel import UnboxedValue
 
 class TaggedBase(object):

From noreply at buildbot.pypy.org  Mon Oct 24 15:32:26 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 24 Oct 2011 15:32:26 +0200 (CEST)
Subject: [pypy-commit] pypy default: cast_uint_to_float and
	cast_float_to_uint are needed for
Message-ID: <20111024133226.56E7A820C7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48374:9977c322a23b
Date: 2011-10-24 15:32 +0200
http://bitbucket.org/pypy/pypy/changeset/9977c322a23b/

Log:	cast_uint_to_float and cast_float_to_uint are needed for micronumpy.
	Implement them (hopefully correctly now) as a residual call for now.

diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -443,6 +443,8 @@
     rewrite_op_gc_identityhash = _do_builtin_call
     rewrite_op_gc_id           = _do_builtin_call
     rewrite_op_uint_mod        = _do_builtin_call
+    rewrite_op_cast_float_to_uint = _do_builtin_call
+    rewrite_op_cast_uint_to_float = _do_builtin_call
 
     # ----------
     # getfield/setfield/mallocs etc.
@@ -850,38 +852,43 @@
             size1, unsigned1 = rffi.size_and_sign(v_arg.concretetype)
             assert size1 <= rffi.sizeof(lltype.Signed), (
                 "not implemented: cast_longlong_to_float")
-            assert size1 < rffi.sizeof(lltype.Signed) or not unsigned1, (
-                "not implemented: cast_uint_to_float")
+            from_uint = (unsigned1 and size1 == rffi.sizeof(lltype.Signed))
             #
             ops = []
-            v1 = varoftype(lltype.Signed)
-            oplist = self.rewrite_operation(
-                SpaceOperation('force_cast', [v_arg], v1)
-            )
-            if oplist:
-                ops.extend(oplist)
+            v2 = varoftype(lltype.Float)
+            if not from_uint:
+                v1 = varoftype(lltype.Signed)
+                oplist = self.rewrite_operation(
+                    SpaceOperation('force_cast', [v_arg], v1)
+                )
+                if oplist:
+                    ops.extend(oplist)
+                else:
+                    v1 = v_arg
+                op = self.rewrite_operation(
+                    SpaceOperation('cast_int_to_float', [v1], v2)
+                )
+                ops.append(op)
             else:
-                v1 = v_arg
-            v2 = varoftype(lltype.Float)
-            op = self.rewrite_operation(
-                SpaceOperation('cast_int_to_float', [v1], v2)
-            )
-            ops.append(op)
+                ops1 = self.rewrite_operation(
+                    SpaceOperation('cast_uint_to_float', [v_arg], v2)
+                )
+                if not isinstance(ops1, list): ops1 = [ops1]
+                ops.extend(ops1)
             op2 = self.rewrite_operation(
                 SpaceOperation('force_cast', [v2], v_result)
             )
             if op2:
                 ops.append(op2)
             else:
-                op.result = v_result
+                ops[-1].result = v_result
             return ops
         elif float_arg and not float_res:
             # some float -> some int
             size2, unsigned2 = rffi.size_and_sign(v_result.concretetype)
             assert size2 <= rffi.sizeof(lltype.Signed), (
                 "not implemented: cast_float_to_longlong")
-            assert size2 < rffi.sizeof(lltype.Signed) or not unsigned2, (
-                "not implemented: cast_float_to_uint")
+            to_uint = (unsigned2 and size2 == rffi.sizeof(lltype.Signed))
             #
             ops = []
             v1 = varoftype(lltype.Float)
@@ -892,18 +899,25 @@
                 ops.append(op1)
             else:
                 v1 = v_arg
-            v2 = varoftype(lltype.Signed)
-            op = self.rewrite_operation(
-                SpaceOperation('cast_float_to_int', [v1], v2)
-            )
-            ops.append(op)
-            oplist = self.rewrite_operation(
-                SpaceOperation('force_cast', [v2], v_result)
-            )
-            if oplist:
-                ops.extend(oplist)
+            if not to_uint:
+                v2 = varoftype(lltype.Signed)
+                op = self.rewrite_operation(
+                    SpaceOperation('cast_float_to_int', [v1], v2)
+                )
+                ops.append(op)
+                oplist = self.rewrite_operation(
+                    SpaceOperation('force_cast', [v2], v_result)
+                )
+                if oplist:
+                    ops.extend(oplist)
+                else:
+                    op.result = v_result
             else:
-                op.result = v_result
+                ops1 = self.rewrite_operation(
+                    SpaceOperation('cast_float_to_uint', [v1], v_result)
+                )
+                if not isinstance(ops1, list): ops1 = [ops1]
+                ops.extend(ops1)
             return ops
         else:
             assert False
@@ -1109,8 +1123,6 @@
     # The new operation is optionally further processed by rewrite_operation().
     for _old, _new in [('bool_not', 'int_is_zero'),
                        ('cast_bool_to_float', 'cast_int_to_float'),
-                       ('cast_uint_to_float', 'cast_primitive'),
-                       ('cast_float_to_uint', 'cast_primitive'),
 
                        ('int_add_nonneg_ovf', 'int_add_ovf'),
                        ('keepalive', '-live-'),
diff --git a/pypy/jit/codewriter/support.py b/pypy/jit/codewriter/support.py
--- a/pypy/jit/codewriter/support.py
+++ b/pypy/jit/codewriter/support.py
@@ -229,6 +229,13 @@
     else:
         return x
 
+def _ll_1_cast_uint_to_float(x):
+    return llop.cast_uint_to_float(lltype.Float, x)
+
+def _ll_1_cast_float_to_uint(x):
+    return llop.cast_float_to_uint(lltype.Unsigned, x)
+
+
 # math support
 # ------------
 
diff --git a/pypy/jit/codewriter/test/test_flatten.py b/pypy/jit/codewriter/test/test_flatten.py
--- a/pypy/jit/codewriter/test/test_flatten.py
+++ b/pypy/jit/codewriter/test/test_flatten.py
@@ -70,7 +70,8 @@
         return 'residual'
     def getcalldescr(self, op, oopspecindex=None, extraeffect=None):
         try:
-            if 'cannot_raise' in op.args[0].value._obj.graph.name:
+            name = op.args[0].value._obj._name
+            if 'cannot_raise' in name or name.startswith('cast_'):
                 return self._descr_cannot_raise
         except AttributeError:
             pass
@@ -900,6 +901,35 @@
             int_return %i4
         """, transform=True)
 
+        def f(dbl):
+            return rffi.cast(rffi.UCHAR, dbl)
+        self.encoding_test(f, [12.456], """
+            cast_float_to_int %f0 -> %i0
+            int_and %i0, $255 -> %i1
+            int_return %i1
+        """, transform=True)
+
+        def f(dbl):
+            return rffi.cast(lltype.Unsigned, dbl)
+        self.encoding_test(f, [12.456], """
+            residual_call_irf_i $<* fn cast_float_to_uint>, <Descr>, I[], R[], F[%f0] -> %i0
+            int_return %i0
+        """, transform=True)
+
+        def f(i):
+            return rffi.cast(lltype.Float, chr(i))    # "char -> float"
+        self.encoding_test(f, [12], """
+            cast_int_to_float %i0 -> %f0
+            float_return %f0
+        """, transform=True)
+
+        def f(i):
+            return rffi.cast(lltype.Float, r_uint(i))    # "uint -> float"
+        self.encoding_test(f, [12], """
+            residual_call_irf_f $<* fn cast_uint_to_float>, <Descr>, I[%i0], R[], F[] -> %f0
+            float_return %f0
+        """, transform=True)
+
 
     def test_direct_ptradd(self):
         from pypy.rpython.lltypesystem import rffi
diff --git a/pypy/jit/metainterp/test/test_float.py b/pypy/jit/metainterp/test/test_float.py
--- a/pypy/jit/metainterp/test/test_float.py
+++ b/pypy/jit/metainterp/test/test_float.py
@@ -55,11 +55,10 @@
     def test_cast_float_to_uint(self):
         def g(f):
             return intmask(r_uint(f))
-        raises(AssertionError, self.interp_operations, g, [0.0])   # for now
-        #res = self.interp_operations(g, [sys.maxint*2.0])
-        #assert res == intmask(long(sys.maxint*2.0))
-        #res = self.interp_operations(g, [-12345.9])
-        #assert res == -12345
+        res = self.interp_operations(g, [sys.maxint*2.0])
+        assert res == intmask(long(sys.maxint*2.0))
+        res = self.interp_operations(g, [-12345.9])
+        assert res == -12345
 
     def test_cast_int_to_float(self):
         def g(i):
@@ -70,11 +69,10 @@
     def test_cast_uint_to_float(self):
         def g(i):
             return float(r_uint(i))
-        raises(AssertionError, self.interp_operations, g, [0])    # for now
-        #res = self.interp_operations(g, [sys.maxint*2])
-        #assert type(res) is float and res == float(sys.maxint*2)
-        #res = self.interp_operations(g, [-12345])
-        #assert type(res) is float and res == float(long(r_uint(-12345)))
+        res = self.interp_operations(g, [intmask(sys.maxint*2)])
+        assert type(res) is float and res == float(sys.maxint*2)
+        res = self.interp_operations(g, [-12345])
+        assert type(res) is float and res == float(long(r_uint(-12345)))
 
     #def test_cast_longlong_to_float(self):
     #def test_cast_ulonglong_to_float(self):

From notifications-noreply at bitbucket.org  Mon Oct 24 15:39:13 2011
From: notifications-noreply at bitbucket.org (Bitbucket)
Date: Mon, 24 Oct 2011 13:39:13 -0000
Subject: [pypy-commit] [COMMENT] Pull request #12 for pypy/pypy: Don't lose
 data when doing non-blocking I/O
Message-ID: <20111024133913.10511.92659@bitbucket05.managed.contegix.com>

New comment on pull request:

https://bitbucket.org/pypy/pypy/pull-request/12/dont-lose-data-when-doing-non-blocking-i-o#comment-552

arigo said:

You should open a bug (or a patch) on http://bugs.pypy.org/, it is less likely to be lost.

--
This is a pull request comment notification from bitbucket.org.
You are receiving this either because you are participating
in a pull request, or you are following it.

From noreply at buildbot.pypy.org  Mon Oct 24 17:27:45 2011
From: noreply at buildbot.pypy.org (edelsohn)
Date: Mon, 24 Oct 2011 17:27:45 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: Update test_call_function for
 PPC64 to use load_from_addr
Message-ID: <20111024152745.C8761820C7@wyvern.cs.uni-duesseldorf.de>

Author: edelsohn
Branch: ppc-jit-backend
Changeset: r48375:b0d393de7dd6
Date: 2011-10-24 11:20 -0400
http://bitbucket.org/pypy/pypy/changeset/b0d393de7dd6/

Log:	Update test_call_function for PPC64 to use load_from_addr

diff --git a/pypy/jit/backend/ppc/ppcgen/test/test_ppc.py b/pypy/jit/backend/ppc/ppcgen/test/test_ppc.py
--- a/pypy/jit/backend/ppc/ppcgen/test/test_ppc.py
+++ b/pypy/jit/backend/ppc/ppcgen/test/test_ppc.py
@@ -194,9 +194,9 @@
         if IS_PPC_32:
             a.load_imm(r10, call_addr)
         else:
-            a.load_from(10, call_addr)
-            a.load_from(2, call_addr+WORD)
-            a.load_from(11, call_addr+2*WORD)
+            a.load_from_addr(r10, call_addr)
+            a.load_from_addr(r2, call_addr+WORD)
+            a.load_from_addr(r11, call_addr+2*WORD)
         a.mtctr(10)
         a.bctr()
         a.blr()

From noreply at buildbot.pypy.org  Mon Oct 24 17:27:46 2011
From: noreply at buildbot.pypy.org (edelsohn)
Date: Mon, 24 Oct 2011 17:27:46 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: Update load_from_addr and
	store_reg for PPC64
Message-ID: <20111024152746.EEF9E820C7@wyvern.cs.uni-duesseldorf.de>

Author: edelsohn
Branch: ppc-jit-backend
Changeset: r48376:128dccb72575
Date: 2011-10-24 11:24 -0400
http://bitbucket.org/pypy/pypy/changeset/128dccb72575/

Log:	Update load_from_addr and store_reg for PPC64

diff --git a/pypy/jit/backend/ppc/ppcgen/codebuilder.py b/pypy/jit/backend/ppc/ppcgen/codebuilder.py
--- a/pypy/jit/backend/ppc/ppcgen/codebuilder.py
+++ b/pypy/jit/backend/ppc/ppcgen/codebuilder.py
@@ -954,15 +954,15 @@
             self.load_imm(rD, addr)
             self.lwzx(rD.value, 0, rD.value)
         else:
-            self.load_word(rD, addr)
-            self.ld(rD, rD, 0)
+            self.load_imm(rD, addr)
+            self.ldx(rD.value, 0, rD.value)
 
     def store_reg(self, source_reg, addr):
         self.load_imm(r.r0, addr)
         if IS_PPC_32:
             self.stwx(source_reg.value, 0, 0)
         else:
-            self.std(source_reg.value, 0, 0)
+            self.stdx(source_reg.value, 0, 0)
 
     def b_cond_offset(self, offset, condition):
         pos = self.currpos()

From noreply at buildbot.pypy.org  Mon Oct 24 17:27:48 2011
From: noreply at buildbot.pypy.org (edelsohn)
Date: Mon, 24 Oct 2011 17:27:48 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: Add PPC64 support to
 emit_setfield_gc and emit_getfield_gc.
Message-ID: <20111024152748.26F05820C7@wyvern.cs.uni-duesseldorf.de>

Author: edelsohn
Branch: ppc-jit-backend
Changeset: r48377:90737207852a
Date: 2011-10-24 11:26 -0400
http://bitbucket.org/pypy/pypy/changeset/90737207852a/

Log:	Add PPC64 support to emit_setfield_gc and emit_getfield_gc. Correct
	typo in emit_setfield_gc for size == 4.

diff --git a/pypy/jit/backend/ppc/ppcgen/opassembler.py b/pypy/jit/backend/ppc/ppcgen/opassembler.py
--- a/pypy/jit/backend/ppc/ppcgen/opassembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/opassembler.py
@@ -261,12 +261,15 @@
     def emit_setfield_gc(self, op, arglocs, regalloc):
         value_loc, base_loc, ofs, size = arglocs
         if size.value == 8:
-            assert 0, "not implemented yet"
+            if ofs.is_imm():
+                self.mc.std(value_loc.value, base_loc.value, ofs.value)
+            else:
+                self.mc.stdx(value_loc.value, base_loc.value, ofs.value)
         elif size.value == 4:
             if ofs.is_imm():
                 self.mc.stw(value_loc.value, base_loc.value, ofs.value)
             else:
-                self.mc.stw(value_loc.value, base_loc.value, ofs.value)
+                self.mc.stwx(value_loc.value, base_loc.value, ofs.value)
         elif size.value == 2:
             if ofs.is_imm():
                 self.mc.sth(value_loc.value, base_loc.value, ofs.value)
@@ -284,7 +287,10 @@
     def emit_getfield_gc(self, op, arglocs, regalloc):
         base_loc, ofs, res, size = arglocs
         if size.value == 8:
-            assert 0, "not implemented yet"
+            if ofs.is_imm():
+                self.mc.ld(res.value, base_loc.value, ofs.value)
+            else:
+                self.mc.ldx(res.value, base_loc.value, ofs.value)
         elif size.value == 4:
             if ofs.is_imm():
                 self.mc.lwz(res.value, base_loc.value, ofs.value)

From noreply at buildbot.pypy.org  Mon Oct 24 17:57:58 2011
From: noreply at buildbot.pypy.org (hager)
Date: Mon, 24 Oct 2011 17:57:58 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: Implemented basic array
	operations.
Message-ID: <20111024155758.82489820C7@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r48378:155a57d1d251
Date: 2011-10-24 17:52 +0200
http://bitbucket.org/pypy/pypy/changeset/155a57d1d251/

Log:	Implemented basic array operations.

diff --git a/pypy/jit/backend/ppc/ppcgen/opassembler.py b/pypy/jit/backend/ppc/ppcgen/opassembler.py
--- a/pypy/jit/backend/ppc/ppcgen/opassembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/opassembler.py
@@ -5,6 +5,7 @@
 from pypy.jit.backend.ppc.ppcgen.arch import GPR_SAVE_AREA, IS_PPC_32, WORD
 
 from pypy.jit.metainterp.history import LoopToken, AbstractFailDescr
+from pypy.rlib.objectmodel import we_are_translated
 
 class GuardToken(object):
     def __init__(self, descr, failargs, faillocs, offset, fcond=c.NE,
@@ -303,5 +304,66 @@
         else:
             assert 0, "size not supported"
 
+    # XXX 64 bit adjustment
+    def emit_arraylen_gc(self, op, arglocs, regalloc):
+        res, base_loc, ofs = arglocs
+        self.mc.lwz(res.value, base_loc.value, ofs.value)
+
+    # XXX 64 bit adjustment
+    def emit_setarrayitem_gc(self, op, arglocs, regalloc):
+        value_loc, base_loc, ofs_loc, scale, ofs = arglocs
+        if scale.value > 0:
+            scale_loc = r.r0
+            self.mc.load_imm(r.r0, scale.value)
+            self.mc.slw(r.r0.value, ofs_loc.value, r.r0.value)
+        else:
+            scale_loc = ofs_loc
+
+        if ofs.value > 0:
+            self.mc.addi(r.r0.value, scale_loc.value, ofs.value)
+            scale_loc = r.r0
+
+        if scale.value == 3:
+            assert 0, "not implemented yet"
+        elif scale.value == 2:
+            self.mc.stwx(value_loc.value, base_loc.value, scale_loc.value)
+        elif scale.value == 1:
+            self.mc.sthx(value_loc.value, base_loc.value, scale_loc.value)
+        elif scale.value == 0:
+            self.mc.stbx(value_loc.value, base_loc.value, scale_loc.value)
+        else:
+            assert 0, "scale %s not supported" % (scale.value)
+
+    # XXX 64 bit adjustment
+    def emit_getarrayitem_gc(self, op, arglocs, regalloc):
+        res, base_loc, ofs_loc, scale, ofs = arglocs
+        if scale.value > 0:
+            scale_loc = r.r0
+            self.mc.load_imm(r.r0, scale.value)
+            self.mc.slw(r.r0.value, ofs_loc.value, scale.value)
+        else:
+            scale_loc = ofs_loc
+        if ofs.value > 0:
+            self.mc.addi(r.r0.value, scale_loc.value, ofs.value)
+            scale_loc = r.r0
+
+        if scale.value == 3:
+            assert 0, "not implemented yet"
+        elif scale.value == 2:
+            self.mc.lwzx(res.value, base_loc.value, scale_loc.value)
+        elif scale.value == 1:
+            self.mc.lhzx(res.value, base_loc.value, scale_loc.value)
+        elif scale.value == 0:
+            self.mc.lbzx(res.value, base_loc.value, scale_loc.value)
+        else:
+            assert 0
+
+        #XXX Hack, Hack, Hack
+        if not we_are_translated():
+            descr = op.getdescr()
+            size =  descr.get_item_size(False)
+            signed = descr.is_item_signed()
+            self._ensure_result_bit_extension(res, size, signed)
+
     def nop(self):
         self.mc.ori(0, 0, 0)
diff --git a/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py b/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py
--- a/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py
@@ -652,6 +652,27 @@
             assert 0, "not supported location"
         assert 0, "not supported location"
 
+    def _ensure_result_bit_extension(self, resloc, size, signed):
+        if size == 4:
+            return
+        if size == 1:
+            if not signed: #unsigned char
+                self.mc.load_imm(r.r0, 0xFF)
+                self.mc.and_(resloc.value, resloc.value, r.r0.value)
+            else:
+                self.mc.load_imm(r.r0, 24)
+                self.mc.slw(resloc.value, resloc.value, r.r0.value)
+                self.mc.sraw(resloc.value, resloc.value, r.r0.value)
+        elif size == 2:
+            if not signed:
+                self.mc.load_imm(r.r0, 16)
+                self.mc.slw(resloc.value, resloc.value, r.r0.value)
+                self.mc.srw(resloc.value, resloc.value, r.r0.value)
+            else:
+                self.mc.load_imm(r.r0, 16)
+                self.mc.slw(resloc.value, resloc.value, r.r0.value)
+                self.mc.sraw(resloc.value, resloc.value, r.r0.value)
+
 def make_operations():
     def not_implemented(builder, trace_op, cpu, *rest_args):
         raise NotImplementedError, trace_op
diff --git a/pypy/jit/backend/ppc/ppcgen/regalloc.py b/pypy/jit/backend/ppc/ppcgen/regalloc.py
--- a/pypy/jit/backend/ppc/ppcgen/regalloc.py
+++ b/pypy/jit/backend/ppc/ppcgen/regalloc.py
@@ -374,6 +374,45 @@
         self.possibly_free_var(op.result)
         return [base_loc, ofs_loc, res, imm(size)]
 
+    def prepare_arraylen_gc(self, op):
+        arraydescr = op.getdescr()
+        assert isinstance(arraydescr, BaseArrayDescr)
+        ofs = arraydescr.get_ofs_length(self.cpu.translate_support_code)
+        arg = op.getarg(0)
+        base_loc, base_box = self._ensure_value_is_boxed(arg)
+        self.possibly_free_vars([arg, base_box])
+
+        res = self.force_allocate_reg(op.result)
+        self.possibly_free_var(op.result)
+        return [res, base_loc, imm(ofs)]
+
+    def prepare_setarrayitem_gc(self, op):
+        b0, b1, b2 = boxes = list(op.getarglist())
+        _, scale, ofs, _, ptr = self._unpack_arraydescr(op.getdescr())
+
+        base_loc, base_box  = self._ensure_value_is_boxed(b0, boxes)
+        boxes.append(base_box)
+        ofs_loc, ofs_box = self._ensure_value_is_boxed(b1, boxes)
+        boxes.append(ofs_box)
+        #XXX check if imm would be fine here
+        value_loc, value_box = self._ensure_value_is_boxed(b2, boxes)
+        boxes.append(value_box)
+        self.possibly_free_vars(boxes)
+        return [value_loc, base_loc, ofs_loc, imm(scale), imm(ofs)]
+
+    def prepare_getarrayitem_gc(self, op):
+        a0, a1 = boxes = list(op.getarglist())
+        _, scale, ofs, _, ptr = self._unpack_arraydescr(op.getdescr())
+
+        base_loc, base_box  = self._ensure_value_is_boxed(a0, boxes)
+        boxes.append(base_box)
+        ofs_loc, ofs_box = self._ensure_value_is_boxed(a1, boxes)
+        boxes.append(ofs_box)
+        self.possibly_free_vars(boxes)
+        res = self.force_allocate_reg(op.result)
+        self.possibly_free_var(op.result)
+        return [res, base_loc, ofs_loc, imm(scale), imm(ofs)]
+
     # from ../x86/regalloc.py:791
     def _unpack_fielddescr(self, fielddescr):
         assert isinstance(fielddescr, BaseFieldDescr)
@@ -382,6 +421,22 @@
         ptr = fielddescr.is_pointer_field()
         return ofs, size, ptr
 
+    # from ../x86/regalloc.py:779
+    def _unpack_arraydescr(self, arraydescr):
+        assert isinstance(arraydescr, BaseArrayDescr)
+        cpu = self.cpu
+        ofs_length = arraydescr.get_ofs_length(cpu.translate_support_code)
+        ofs = arraydescr.get_base_size(cpu.translate_support_code)
+        size = arraydescr.get_item_size(cpu.translate_support_code)
+        ptr = arraydescr.is_array_of_pointers()
+        scale = 0
+        # XXX HACK, improve!
+        if not arraydescr._clsname.startswith("BoolArrayDescr"):
+            while (1 << scale) < size:
+                scale += 1
+            assert (1 << scale) == size
+        return size, scale, ofs, ofs_length, ptr
+
 def make_operation_list():
     def not_implemented(self, op, *args):
         raise NotImplementedError, op

From noreply at buildbot.pypy.org  Mon Oct 24 17:57:59 2011
From: noreply at buildbot.pypy.org (hager)
Date: Mon, 24 Oct 2011 17:57:59 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: merge
Message-ID: <20111024155759.AB074820C7@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r48379:38f0e1f8b91e
Date: 2011-10-24 17:57 +0200
http://bitbucket.org/pypy/pypy/changeset/38f0e1f8b91e/

Log:	merge

diff --git a/pypy/jit/backend/ppc/ppcgen/codebuilder.py b/pypy/jit/backend/ppc/ppcgen/codebuilder.py
--- a/pypy/jit/backend/ppc/ppcgen/codebuilder.py
+++ b/pypy/jit/backend/ppc/ppcgen/codebuilder.py
@@ -954,15 +954,15 @@
             self.load_imm(rD, addr)
             self.lwzx(rD.value, 0, rD.value)
         else:
-            self.load_word(rD, addr)
-            self.ld(rD, rD, 0)
+            self.load_imm(rD, addr)
+            self.ldx(rD.value, 0, rD.value)
 
     def store_reg(self, source_reg, addr):
         self.load_imm(r.r0, addr)
         if IS_PPC_32:
             self.stwx(source_reg.value, 0, 0)
         else:
-            self.std(source_reg.value, 0, 0)
+            self.stdx(source_reg.value, 0, 0)
 
     def b_cond_offset(self, offset, condition):
         pos = self.currpos()
diff --git a/pypy/jit/backend/ppc/ppcgen/opassembler.py b/pypy/jit/backend/ppc/ppcgen/opassembler.py
--- a/pypy/jit/backend/ppc/ppcgen/opassembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/opassembler.py
@@ -262,12 +262,15 @@
     def emit_setfield_gc(self, op, arglocs, regalloc):
         value_loc, base_loc, ofs, size = arglocs
         if size.value == 8:
-            assert 0, "not implemented yet"
+            if ofs.is_imm():
+                self.mc.std(value_loc.value, base_loc.value, ofs.value)
+            else:
+                self.mc.stdx(value_loc.value, base_loc.value, ofs.value)
         elif size.value == 4:
             if ofs.is_imm():
                 self.mc.stw(value_loc.value, base_loc.value, ofs.value)
             else:
-                self.mc.stw(value_loc.value, base_loc.value, ofs.value)
+                self.mc.stwx(value_loc.value, base_loc.value, ofs.value)
         elif size.value == 2:
             if ofs.is_imm():
                 self.mc.sth(value_loc.value, base_loc.value, ofs.value)
@@ -285,7 +288,10 @@
     def emit_getfield_gc(self, op, arglocs, regalloc):
         base_loc, ofs, res, size = arglocs
         if size.value == 8:
-            assert 0, "not implemented yet"
+            if ofs.is_imm():
+                self.mc.ld(res.value, base_loc.value, ofs.value)
+            else:
+                self.mc.ldx(res.value, base_loc.value, ofs.value)
         elif size.value == 4:
             if ofs.is_imm():
                 self.mc.lwz(res.value, base_loc.value, ofs.value)
diff --git a/pypy/jit/backend/ppc/ppcgen/test/test_ppc.py b/pypy/jit/backend/ppc/ppcgen/test/test_ppc.py
--- a/pypy/jit/backend/ppc/ppcgen/test/test_ppc.py
+++ b/pypy/jit/backend/ppc/ppcgen/test/test_ppc.py
@@ -194,9 +194,9 @@
         if IS_PPC_32:
             a.load_imm(r10, call_addr)
         else:
-            a.load_from(10, call_addr)
-            a.load_from(2, call_addr+WORD)
-            a.load_from(11, call_addr+2*WORD)
+            a.load_from_addr(r10, call_addr)
+            a.load_from_addr(r2, call_addr+WORD)
+            a.load_from_addr(r11, call_addr+2*WORD)
         a.mtctr(10)
         a.bctr()
         a.blr()

From noreply at buildbot.pypy.org  Mon Oct 24 17:58:24 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 24 Oct 2011 17:58:24 +0200 (CEST)
Subject: [pypy-commit] pypy default: Support casts between floats and
 (u)longlongs written as a force_cast.
Message-ID: <20111024155824.6BBED820C7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48380:054d434f5e82
Date: 2011-10-24 17:58 +0200
http://bitbucket.org/pypy/pypy/changeset/054d434f5e82/

Log:	Support casts between floats and (u)longlongs written as a
	force_cast.

diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -849,14 +849,12 @@
             return self._float_to_float_cast(v_arg, v_result)
         elif not float_arg and float_res:
             # some int -> some float
-            size1, unsigned1 = rffi.size_and_sign(v_arg.concretetype)
-            assert size1 <= rffi.sizeof(lltype.Signed), (
-                "not implemented: cast_longlong_to_float")
-            from_uint = (unsigned1 and size1 == rffi.sizeof(lltype.Signed))
-            #
             ops = []
             v2 = varoftype(lltype.Float)
-            if not from_uint:
+            sizesign = rffi.size_and_sign(v_arg.concretetype)
+            if sizesign <= rffi.size_and_sign(lltype.Signed):
+                # cast from a type that fits in an int: either the size is
+                # smaller, or it is equal and it is not unsigned
                 v1 = varoftype(lltype.Signed)
                 oplist = self.rewrite_operation(
                     SpaceOperation('force_cast', [v_arg], v1)
@@ -870,8 +868,16 @@
                 )
                 ops.append(op)
             else:
+                if sizesign == rffi.size_and_sign(lltype.Unsigned):
+                    opname = 'cast_uint_to_float'
+                elif sizesign == rffi.size_and_sign(lltype.SignedLongLong):
+                    opname = 'cast_longlong_to_float'
+                elif sizesign == rffi.size_and_sign(lltype.UnsignedLongLong):
+                    opname = 'cast_ulonglong_to_float'
+                else:
+                    raise AssertionError('cast_x_to_float: %r' % (sizesign,))
                 ops1 = self.rewrite_operation(
-                    SpaceOperation('cast_uint_to_float', [v_arg], v2)
+                    SpaceOperation(opname, [v_arg], v2)
                 )
                 if not isinstance(ops1, list): ops1 = [ops1]
                 ops.extend(ops1)
@@ -885,11 +891,6 @@
             return ops
         elif float_arg and not float_res:
             # some float -> some int
-            size2, unsigned2 = rffi.size_and_sign(v_result.concretetype)
-            assert size2 <= rffi.sizeof(lltype.Signed), (
-                "not implemented: cast_float_to_longlong")
-            to_uint = (unsigned2 and size2 == rffi.sizeof(lltype.Signed))
-            #
             ops = []
             v1 = varoftype(lltype.Float)
             op1 = self.rewrite_operation(
@@ -899,7 +900,10 @@
                 ops.append(op1)
             else:
                 v1 = v_arg
-            if not to_uint:
+            sizesign = rffi.size_and_sign(v_result.concretetype)
+            if sizesign <= rffi.size_and_sign(lltype.Signed):
+                # cast to a type that fits in an int: either the size is
+                # smaller, or it is equal and it is not unsigned
                 v2 = varoftype(lltype.Signed)
                 op = self.rewrite_operation(
                     SpaceOperation('cast_float_to_int', [v1], v2)
@@ -913,8 +917,16 @@
                 else:
                     op.result = v_result
             else:
+                if sizesign == rffi.size_and_sign(lltype.Unsigned):
+                    opname = 'cast_float_to_uint'
+                elif sizesign == rffi.size_and_sign(lltype.SignedLongLong):
+                    opname = 'cast_float_to_longlong'
+                elif sizesign == rffi.size_and_sign(lltype.UnsignedLongLong):
+                    opname = 'cast_float_to_ulonglong'
+                else:
+                    raise AssertionError('cast_float_to_x: %r' % (sizesign,))
                 ops1 = self.rewrite_operation(
-                    SpaceOperation('cast_float_to_uint', [v1], v_result)
+                    SpaceOperation(opname, [v1], v_result)
                 )
                 if not isinstance(ops1, list): ops1 = [ops1]
                 ops.extend(ops1)
diff --git a/pypy/jit/codewriter/support.py b/pypy/jit/codewriter/support.py
--- a/pypy/jit/codewriter/support.py
+++ b/pypy/jit/codewriter/support.py
@@ -230,9 +230,13 @@
         return x
 
 def _ll_1_cast_uint_to_float(x):
+    # XXX on 32-bit platforms, this should be done using cast_longlong_to_float
+    # (which is a residual call right now in the x86 backend)
     return llop.cast_uint_to_float(lltype.Float, x)
 
 def _ll_1_cast_float_to_uint(x):
+    # XXX on 32-bit platforms, this should be done using cast_float_to_longlong
+    # (which is a residual call right now in the x86 backend)
     return llop.cast_float_to_uint(lltype.Unsigned, x)
 
 
diff --git a/pypy/jit/codewriter/test/test_flatten.py b/pypy/jit/codewriter/test/test_flatten.py
--- a/pypy/jit/codewriter/test/test_flatten.py
+++ b/pypy/jit/codewriter/test/test_flatten.py
@@ -8,7 +8,7 @@
 from pypy.rpython.lltypesystem import lltype, rclass, rstr
 from pypy.objspace.flow.model import SpaceOperation, Variable, Constant
 from pypy.translator.unsimplify import varoftype
-from pypy.rlib.rarithmetic import ovfcheck, r_uint
+from pypy.rlib.rarithmetic import ovfcheck, r_uint, r_longlong, r_ulonglong
 from pypy.rlib.jit import dont_look_inside, _we_are_jitted, JitDriver
 from pypy.rlib.objectmodel import keepalive_until_here
 from pypy.rlib import jit
@@ -930,6 +930,38 @@
             float_return %f0
         """, transform=True)
 
+        if not longlong.is_64_bit:
+            def f(dbl):
+                return rffi.cast(lltype.SignedLongLong, dbl)
+            self.encoding_test(f, [12.3], """
+                residual_call_irf_f $<* fn llong_from_float>, <Descr>, I[], R[], F[%f0] -> %f1
+                float_return %f1
+            """, transform=True)
+
+            def f(dbl):
+                return rffi.cast(lltype.UnsignedLongLong, dbl)
+            self.encoding_test(f, [12.3], """
+                residual_call_irf_f $<* fn ullong_from_float>, <Descr>, I[], R[], F[%f0] -> %f1
+                float_return %f1
+            """, transform=True)
+
+            def f(x):
+                ll = r_longlong(x)
+                return rffi.cast(lltype.Float, ll)
+            self.encoding_test(f, [12], """
+                residual_call_irf_f $<* fn llong_from_int>, <Descr>, I[%i0], R[], F[] -> %f0
+                residual_call_irf_f $<* fn llong_to_float>, <Descr>, I[], R[], F[%f0] -> %f1
+                float_return %f1
+            """, transform=True)
+
+            def f(x):
+                ll = r_ulonglong(x)
+                return rffi.cast(lltype.Float, ll)
+            self.encoding_test(f, [12], """
+                residual_call_irf_f $<* fn ullong_from_int>, <Descr>, I[%i0], R[], F[] -> %f0
+                residual_call_irf_f $<* fn ullong_u_to_float>, <Descr>, I[], R[], F[%f0] -> %f1
+                float_return %f1
+            """, transform=True)
 
     def test_direct_ptradd(self):
         from pypy.rpython.lltypesystem import rffi
diff --git a/pypy/jit/metainterp/test/test_float.py b/pypy/jit/metainterp/test/test_float.py
--- a/pypy/jit/metainterp/test/test_float.py
+++ b/pypy/jit/metainterp/test/test_float.py
@@ -74,11 +74,6 @@
         res = self.interp_operations(g, [-12345])
         assert type(res) is float and res == float(long(r_uint(-12345)))
 
-    #def test_cast_longlong_to_float(self):
-    #def test_cast_ulonglong_to_float(self):
-    #def test_cast_float_to_longlong(self):
-    #def test_cast_float_to_ulonglong(self):
-
 
 class TestOOtype(FloatTests, OOJitMixin):
     pass

From noreply at buildbot.pypy.org  Mon Oct 24 18:48:43 2011
From: noreply at buildbot.pypy.org (hager)
Date: Mon, 24 Oct 2011 18:48:43 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: Fixed hack in treating bools.
Message-ID: <20111024164843.96B54820C7@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r48381:064df4d72ec4
Date: 2011-10-24 18:48 +0200
http://bitbucket.org/pypy/pypy/changeset/064df4d72ec4/

Log:	Fixed hack in treating bools.

diff --git a/pypy/jit/backend/ppc/ppcgen/regalloc.py b/pypy/jit/backend/ppc/ppcgen/regalloc.py
--- a/pypy/jit/backend/ppc/ppcgen/regalloc.py
+++ b/pypy/jit/backend/ppc/ppcgen/regalloc.py
@@ -431,10 +431,10 @@
         ptr = arraydescr.is_array_of_pointers()
         scale = 0
         # XXX HACK, improve!
-        if not arraydescr._clsname.startswith("BoolArrayDescr"):
-            while (1 << scale) < size:
-                scale += 1
-            assert (1 << scale) == size
+        #if not arraydescr._clsname.startswith("BoolArrayDescr"):
+        while (1 << scale) < size:
+            scale += 1
+        assert (1 << scale) == size
         return size, scale, ofs, ofs_length, ptr
 
 def make_operation_list():
diff --git a/pypy/rpython/lltypesystem/ll2ctypes.py b/pypy/rpython/lltypesystem/ll2ctypes.py
--- a/pypy/rpython/lltypesystem/ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/ll2ctypes.py
@@ -107,7 +107,7 @@
         rffi.LONGLONG:   ctypes.c_longlong,
         rffi.ULONGLONG:  ctypes.c_ulonglong,
         rffi.SIZE_T:     ctypes.c_size_t,
-        lltype.Bool:     ctypes.c_long, # XXX
+        lltype.Bool:     getattr(ctypes, "c_bool", ctypes.c_long),
         llmemory.Address:  ctypes.c_void_p,
         llmemory.GCREF:    ctypes.c_void_p,
         llmemory.WeakRef:  ctypes.c_void_p, # XXX

From noreply at buildbot.pypy.org  Mon Oct 24 18:49:58 2011
From: noreply at buildbot.pypy.org (hager)
Date: Mon, 24 Oct 2011 18:49:58 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: Remove comment
Message-ID: <20111024164958.31246820C7@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r48382:92cd323471d9
Date: 2011-10-24 18:49 +0200
http://bitbucket.org/pypy/pypy/changeset/92cd323471d9/

Log:	Remove comment

diff --git a/pypy/jit/backend/ppc/ppcgen/regalloc.py b/pypy/jit/backend/ppc/ppcgen/regalloc.py
--- a/pypy/jit/backend/ppc/ppcgen/regalloc.py
+++ b/pypy/jit/backend/ppc/ppcgen/regalloc.py
@@ -430,8 +430,6 @@
         size = arraydescr.get_item_size(cpu.translate_support_code)
         ptr = arraydescr.is_array_of_pointers()
         scale = 0
-        # XXX HACK, improve!
-        #if not arraydescr._clsname.startswith("BoolArrayDescr"):
         while (1 << scale) < size:
             scale += 1
         assert (1 << scale) == size

From noreply at buildbot.pypy.org  Mon Oct 24 19:38:25 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 24 Oct 2011 19:38:25 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Tweaks.
Message-ID: <20111024173825.152C2820C7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r48383:92beb8ad501b
Date: 2011-10-24 19:34 +0200
http://bitbucket.org/pypy/pypy/changeset/92beb8ad501b/

Log:	Tweaks.

diff --git a/pypy/rpython/memory/gc/concurrentgen.py b/pypy/rpython/memory/gc/concurrentgen.py
--- a/pypy/rpython/memory/gc/concurrentgen.py
+++ b/pypy/rpython/memory/gc/concurrentgen.py
@@ -156,6 +156,7 @@
         "Start the concurrent collector thread."
         # don't call GCBase.setup(self), because we don't need
         # 'run_finalizers' as a deque
+        debug_start("gc-startup")
         self.finalizer_lock_count = 0
         #
         self.ready_to_start_lock = ll_thread.allocate_ll_lock()
@@ -177,6 +178,10 @@
             fillfact = env.read_float_from_env('PYPY_GC_MAJOR_COLLECT')
             if fillfact > 1.0:
                 self.fill_factor = fillfact
+        #
+        debug_print("nursery size:", self.nursery_size)
+        debug_print("fill factor: ", self.fill_factor)
+        debug_stop("gc-startup")
 
     def set_nursery_size(self, newsize):
         self.nursery_size = newsize
@@ -605,19 +610,16 @@
         self.debug_check_list(self.old_objects)
 
     def debug_check_list(self, list):
-        try:
-            previous = self.NULL
-            count = 0
-            while list != self.NULL:
-                # prevent constant-folding, and detects loops of length 1
-                ll_assert(list != previous, "loop!")
-                previous = list
-                list = list.next
-                count += 1
-            return count
-        except KeyboardInterrupt:
-            ll_assert(False, "interrupted")
-            raise
+        previous = self.NULL
+        count = 0
+        while list != self.NULL:
+            # prevent constant-folding, and detects loops
+            ll_assert(list != previous, "loop!")
+            count += 1
+            if count & (count-1) == 0:    # only on powers of two, to
+                previous = list           # detect loops of any size
+            list = list.next
+        return count
 
     def acquire(self, lock):
         if we_are_translated():
@@ -875,6 +877,7 @@
             sz -= intmask(surviving_size)
         #
         self.gc.size_still_available_before_major = sz
+        debug_print("size_still_available_before_major =", sz)
         #
         self.running = 2
         #debug_print("collection_running = 2")

From noreply at buildbot.pypy.org  Mon Oct 24 19:38:26 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 24 Oct 2011 19:38:26 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: It seems that not using
 "borrowed vars" is much more efficient.
Message-ID: <20111024173826.48F6B82A87@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r48384:229af044431a
Date: 2011-10-24 19:35 +0200
http://bitbucket.org/pypy/pypy/changeset/229af044431a/

Log:	It seems that not using "borrowed vars" is much more efficient.

diff --git a/pypy/rpython/memory/gctransform/framework.py b/pypy/rpython/memory/gctransform/framework.py
--- a/pypy/rpython/memory/gctransform/framework.py
+++ b/pypy/rpython/memory/gctransform/framework.py
@@ -1222,9 +1222,8 @@
         pass
 
     def get_livevars_for_roots(self, hop, keep_current_args=False):
-        if self.gcdata.gc.moving_gc and not keep_current_args:
-            # moving GCs don't borrow, so the caller does not need to keep
-            # the arguments alive
+        if not keep_current_args:
+            # the caller does not need to keep the arguments alive
             livevars = [var for var in hop.livevars_after_op()
                             if not var_ispyobj(var)]
         else:
@@ -1267,10 +1266,9 @@
                 hop.genop("gc_reload_possibly_moved", [v_newaddr, var])
 
     def compute_borrowed_vars(self, graph):
-        # XXX temporary workaround, should be done more correctly
-        if self.gcdata.gc.moving_gc:
-            return lambda v: False
-        return super(FrameworkGCTransformer, self).compute_borrowed_vars(graph)
+        # we don't use borrowed vars with our own GCs, because it seems
+        # to be always more efficient not to.
+        return lambda v: False
 
 
 class TransformerLayoutBuilder(gctypelayout.TypeLayoutBuilder):

From noreply at buildbot.pypy.org  Mon Oct 24 19:38:27 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 24 Oct 2011 19:38:27 +0200 (CEST)
Subject: [pypy-commit] pypy concurrent-marksweep: Move this outside the
 collector thread, to ensure that it is correctly
Message-ID: <20111024173827.7174D820C7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: concurrent-marksweep
Changeset: r48385:b661552e1f80
Date: 2011-10-24 19:38 +0200
http://bitbucket.org/pypy/pypy/changeset/b661552e1f80/

Log:	Move this outside the collector thread, to ensure that it is
	correctly nested in a debug_print("gc-*")

diff --git a/pypy/rpython/memory/gc/concurrentgen.py b/pypy/rpython/memory/gc/concurrentgen.py
--- a/pypy/rpython/memory/gc/concurrentgen.py
+++ b/pypy/rpython/memory/gc/concurrentgen.py
@@ -390,6 +390,8 @@
         if self.collector.running != 0:
             debug_start("gc-stop")
             self._stop_collection()
+            debug_print("size_still_available_before_major =",
+                        self.size_still_available_before_major)
             debug_stop("gc-stop")
             #
             # We must *not* run execute_finalizers_ll() here, because it
@@ -877,7 +879,6 @@
             sz -= intmask(surviving_size)
         #
         self.gc.size_still_available_before_major = sz
-        debug_print("size_still_available_before_major =", sz)
         #
         self.running = 2
         #debug_print("collection_running = 2")

From noreply at buildbot.pypy.org  Mon Oct 24 20:19:23 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 24 Oct 2011 20:19:23 +0200 (CEST)
Subject: [pypy-commit] pypy lightweight-finalizers: rewrite obvious nonsense,
	needs tests
Message-ID: <20111024181923.3BC42820C7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: lightweight-finalizers
Changeset: r48386:6949b37f7cb8
Date: 2011-10-24 16:33 +0200
http://bitbucket.org/pypy/pypy/changeset/6949b37f7cb8/

Log:	rewrite obvious nonsense, needs tests

diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -1845,16 +1845,12 @@
         don't do anything fancy and *just* call them. Among other things
         they won't resurrect objects
         """
-        new_objects = self.AddressStack()
         while self.young_objects_with_light_finalizers.non_empty():
             obj = self.young_objects_with_light_finalizers.pop()
-            if self.is_forwarded(obj):
-                new_objects.append(self.get_forwarding_address(obj))
-            else:
+            if not self.is_forwarded(obj):
                 finalizer = self.getlightfinalizer(self.get_type_id(obj))
                 ll_assert(bool(finalizer), "no light finalizer found")
                 finalizer(obj, llmemory.NULL)
-        self.objects_with_light_finalizers = new_objects
 
     def deal_with_objects_with_finalizers(self):
         # Walk over list of objects with finalizers.

From noreply at buildbot.pypy.org  Mon Oct 24 20:19:24 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 24 Oct 2011 20:19:24 +0200 (CEST)
Subject: [pypy-commit] pypy default: add some options
Message-ID: <20111024181924.6584A820C7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r48387:9ea261642156
Date: 2011-10-24 16:34 +0200
http://bitbucket.org/pypy/pypy/changeset/9ea261642156/

Log:	add some options

diff --git a/pypy/pytest.ini b/pypy/pytest.ini
--- a/pypy/pytest.ini
+++ b/pypy/pytest.ini
@@ -1,2 +1,2 @@
 [pytest]
-addopts = --assertmode=old
\ No newline at end of file
+addopts = --assertmode=old -rf

From noreply at buildbot.pypy.org  Mon Oct 24 20:19:26 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 24 Oct 2011 20:19:26 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20111024181926.273B6820C7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r48388:76d0bea6c4e7
Date: 2011-10-24 20:19 +0200
http://bitbucket.org/pypy/pypy/changeset/76d0bea6c4e7/

Log:	merge heads

diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -2,7 +2,7 @@
 from pypy.interpreter.baseobjspace import Wrappable
 from pypy.interpreter import typedef
 from pypy.interpreter.gateway import interp2app
-from pypy.interpreter.error import OperationError
+from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.tool.pairtype import extendabletype
 from pypy.tool.sourcetools import func_with_new_name
@@ -2925,14 +2925,13 @@
 def Module_get_body(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -2968,14 +2967,13 @@
 def Interactive_get_body(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3015,8 +3013,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     return space.wrap(w_self.body)
 
 def Expression_set_body(space, w_self, w_new_value):
@@ -3057,14 +3054,13 @@
 def Suite_get_body(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3104,8 +3100,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._lineno_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'lineno'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lineno')
     return space.wrap(w_self.lineno)
 
 def stmt_set_lineno(space, w_self, w_new_value):
@@ -3126,8 +3121,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._col_offset_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'col_offset'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'col_offset')
     return space.wrap(w_self.col_offset)
 
 def stmt_set_col_offset(space, w_self, w_new_value):
@@ -3157,8 +3151,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'name'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name')
     return space.wrap(w_self.name)
 
 def FunctionDef_set_name(space, w_self, w_new_value):
@@ -3179,8 +3172,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'args'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args')
     return space.wrap(w_self.args)
 
 def FunctionDef_set_args(space, w_self, w_new_value):
@@ -3197,14 +3189,13 @@
 def FunctionDef_get_body(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3215,14 +3206,13 @@
 def FunctionDef_get_decorator_list(space, w_self):
     if not w_self.initialization_state & 8:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'decorator_list'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'decorator_list')
     if w_self.w_decorator_list is None:
         if w_self.decorator_list is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.decorator_list]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_decorator_list = w_list
     return w_self.w_decorator_list
 
@@ -3266,8 +3256,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'name'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name')
     return space.wrap(w_self.name)
 
 def ClassDef_set_name(space, w_self, w_new_value):
@@ -3284,14 +3273,13 @@
 def ClassDef_get_bases(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'bases'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'bases')
     if w_self.w_bases is None:
         if w_self.bases is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.bases]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_bases = w_list
     return w_self.w_bases
 
@@ -3302,14 +3290,13 @@
 def ClassDef_get_body(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3320,14 +3307,13 @@
 def ClassDef_get_decorator_list(space, w_self):
     if not w_self.initialization_state & 8:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'decorator_list'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'decorator_list')
     if w_self.w_decorator_list is None:
         if w_self.decorator_list is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.decorator_list]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_decorator_list = w_list
     return w_self.w_decorator_list
 
@@ -3372,8 +3358,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Return_set_value(space, w_self, w_new_value):
@@ -3414,14 +3399,13 @@
 def Delete_get_targets(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'targets'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'targets')
     if w_self.w_targets is None:
         if w_self.targets is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.targets]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_targets = w_list
     return w_self.w_targets
 
@@ -3457,14 +3441,13 @@
 def Assign_get_targets(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'targets'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'targets')
     if w_self.w_targets is None:
         if w_self.targets is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.targets]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_targets = w_list
     return w_self.w_targets
 
@@ -3479,8 +3462,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Assign_set_value(space, w_self, w_new_value):
@@ -3527,8 +3509,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'target'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'target')
     return space.wrap(w_self.target)
 
 def AugAssign_set_target(space, w_self, w_new_value):
@@ -3549,8 +3530,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'op'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op')
     return operator_to_class[w_self.op - 1]()
 
 def AugAssign_set_op(space, w_self, w_new_value):
@@ -3573,8 +3553,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def AugAssign_set_value(space, w_self, w_new_value):
@@ -3621,8 +3600,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'dest'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'dest')
     return space.wrap(w_self.dest)
 
 def Print_set_dest(space, w_self, w_new_value):
@@ -3639,14 +3617,13 @@
 def Print_get_values(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'values'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'values')
     if w_self.w_values is None:
         if w_self.values is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.values]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_values = w_list
     return w_self.w_values
 
@@ -3661,8 +3638,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'nl'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'nl')
     return space.wrap(w_self.nl)
 
 def Print_set_nl(space, w_self, w_new_value):
@@ -3710,8 +3686,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'target'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'target')
     return space.wrap(w_self.target)
 
 def For_set_target(space, w_self, w_new_value):
@@ -3732,8 +3707,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'iter'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'iter')
     return space.wrap(w_self.iter)
 
 def For_set_iter(space, w_self, w_new_value):
@@ -3750,14 +3724,13 @@
 def For_get_body(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3768,14 +3741,13 @@
 def For_get_orelse(space, w_self):
     if not w_self.initialization_state & 8:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'orelse'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
     if w_self.w_orelse is None:
         if w_self.orelse is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.orelse]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_orelse = w_list
     return w_self.w_orelse
 
@@ -3819,8 +3791,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'test'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test')
     return space.wrap(w_self.test)
 
 def While_set_test(space, w_self, w_new_value):
@@ -3837,14 +3808,13 @@
 def While_get_body(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3855,14 +3825,13 @@
 def While_get_orelse(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'orelse'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
     if w_self.w_orelse is None:
         if w_self.orelse is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.orelse]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_orelse = w_list
     return w_self.w_orelse
 
@@ -3905,8 +3874,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'test'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test')
     return space.wrap(w_self.test)
 
 def If_set_test(space, w_self, w_new_value):
@@ -3923,14 +3891,13 @@
 def If_get_body(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3941,14 +3908,13 @@
 def If_get_orelse(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'orelse'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
     if w_self.w_orelse is None:
         if w_self.orelse is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.orelse]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_orelse = w_list
     return w_self.w_orelse
 
@@ -3991,8 +3957,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'context_expr'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'context_expr')
     return space.wrap(w_self.context_expr)
 
 def With_set_context_expr(space, w_self, w_new_value):
@@ -4013,8 +3978,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'optional_vars'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'optional_vars')
     return space.wrap(w_self.optional_vars)
 
 def With_set_optional_vars(space, w_self, w_new_value):
@@ -4031,14 +3995,13 @@
 def With_get_body(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -4080,8 +4043,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'type'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'type')
     return space.wrap(w_self.type)
 
 def Raise_set_type(space, w_self, w_new_value):
@@ -4102,8 +4064,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'inst'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'inst')
     return space.wrap(w_self.inst)
 
 def Raise_set_inst(space, w_self, w_new_value):
@@ -4124,8 +4085,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'tback'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'tback')
     return space.wrap(w_self.tback)
 
 def Raise_set_tback(space, w_self, w_new_value):
@@ -4168,14 +4128,13 @@
 def TryExcept_get_body(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -4186,14 +4145,13 @@
 def TryExcept_get_handlers(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'handlers'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'handlers')
     if w_self.w_handlers is None:
         if w_self.handlers is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.handlers]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_handlers = w_list
     return w_self.w_handlers
 
@@ -4204,14 +4162,13 @@
 def TryExcept_get_orelse(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'orelse'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
     if w_self.w_orelse is None:
         if w_self.orelse is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.orelse]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_orelse = w_list
     return w_self.w_orelse
 
@@ -4251,14 +4208,13 @@
 def TryFinally_get_body(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -4269,14 +4225,13 @@
 def TryFinally_get_finalbody(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'finalbody'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'finalbody')
     if w_self.w_finalbody is None:
         if w_self.finalbody is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.finalbody]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_finalbody = w_list
     return w_self.w_finalbody
 
@@ -4318,8 +4273,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'test'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test')
     return space.wrap(w_self.test)
 
 def Assert_set_test(space, w_self, w_new_value):
@@ -4340,8 +4294,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'msg'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'msg')
     return space.wrap(w_self.msg)
 
 def Assert_set_msg(space, w_self, w_new_value):
@@ -4383,14 +4336,13 @@
 def Import_get_names(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'names'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'names')
     if w_self.w_names is None:
         if w_self.names is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.names]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_names = w_list
     return w_self.w_names
 
@@ -4430,8 +4382,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'module'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'module')
     return space.wrap(w_self.module)
 
 def ImportFrom_set_module(space, w_self, w_new_value):
@@ -4451,14 +4402,13 @@
 def ImportFrom_get_names(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'names'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'names')
     if w_self.w_names is None:
         if w_self.names is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.names]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_names = w_list
     return w_self.w_names
 
@@ -4473,8 +4423,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'level'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'level')
     return space.wrap(w_self.level)
 
 def ImportFrom_set_level(space, w_self, w_new_value):
@@ -4522,8 +4471,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     return space.wrap(w_self.body)
 
 def Exec_set_body(space, w_self, w_new_value):
@@ -4544,8 +4492,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'globals'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'globals')
     return space.wrap(w_self.globals)
 
 def Exec_set_globals(space, w_self, w_new_value):
@@ -4566,8 +4513,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'locals'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'locals')
     return space.wrap(w_self.locals)
 
 def Exec_set_locals(space, w_self, w_new_value):
@@ -4610,14 +4556,13 @@
 def Global_get_names(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'names'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'names')
     if w_self.w_names is None:
         if w_self.names is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.names]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_names = w_list
     return w_self.w_names
 
@@ -4657,8 +4602,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Expr_set_value(space, w_self, w_new_value):
@@ -4754,8 +4698,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._lineno_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'lineno'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lineno')
     return space.wrap(w_self.lineno)
 
 def expr_set_lineno(space, w_self, w_new_value):
@@ -4776,8 +4719,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._col_offset_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'col_offset'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'col_offset')
     return space.wrap(w_self.col_offset)
 
 def expr_set_col_offset(space, w_self, w_new_value):
@@ -4807,8 +4749,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'op'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op')
     return boolop_to_class[w_self.op - 1]()
 
 def BoolOp_set_op(space, w_self, w_new_value):
@@ -4827,14 +4768,13 @@
 def BoolOp_get_values(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'values'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'values')
     if w_self.w_values is None:
         if w_self.values is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.values]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_values = w_list
     return w_self.w_values
 
@@ -4875,8 +4815,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'left'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'left')
     return space.wrap(w_self.left)
 
 def BinOp_set_left(space, w_self, w_new_value):
@@ -4897,8 +4836,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'op'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op')
     return operator_to_class[w_self.op - 1]()
 
 def BinOp_set_op(space, w_self, w_new_value):
@@ -4921,8 +4859,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'right'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'right')
     return space.wrap(w_self.right)
 
 def BinOp_set_right(space, w_self, w_new_value):
@@ -4969,8 +4906,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'op'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op')
     return unaryop_to_class[w_self.op - 1]()
 
 def UnaryOp_set_op(space, w_self, w_new_value):
@@ -4993,8 +4929,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'operand'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'operand')
     return space.wrap(w_self.operand)
 
 def UnaryOp_set_operand(space, w_self, w_new_value):
@@ -5040,8 +4975,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'args'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args')
     return space.wrap(w_self.args)
 
 def Lambda_set_args(space, w_self, w_new_value):
@@ -5062,8 +4996,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     return space.wrap(w_self.body)
 
 def Lambda_set_body(space, w_self, w_new_value):
@@ -5109,8 +5042,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'test'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test')
     return space.wrap(w_self.test)
 
 def IfExp_set_test(space, w_self, w_new_value):
@@ -5131,8 +5063,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     return space.wrap(w_self.body)
 
 def IfExp_set_body(space, w_self, w_new_value):
@@ -5153,8 +5084,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'orelse'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
     return space.wrap(w_self.orelse)
 
 def IfExp_set_orelse(space, w_self, w_new_value):
@@ -5197,14 +5127,13 @@
 def Dict_get_keys(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'keys'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'keys')
     if w_self.w_keys is None:
         if w_self.keys is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.keys]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_keys = w_list
     return w_self.w_keys
 
@@ -5215,14 +5144,13 @@
 def Dict_get_values(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'values'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'values')
     if w_self.w_values is None:
         if w_self.values is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.values]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_values = w_list
     return w_self.w_values
 
@@ -5260,14 +5188,13 @@
 def Set_get_elts(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elts'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elts')
     if w_self.w_elts is None:
         if w_self.elts is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.elts]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_elts = w_list
     return w_self.w_elts
 
@@ -5307,8 +5234,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elt'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elt')
     return space.wrap(w_self.elt)
 
 def ListComp_set_elt(space, w_self, w_new_value):
@@ -5325,14 +5251,13 @@
 def ListComp_get_generators(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'generators'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators')
     if w_self.w_generators is None:
         if w_self.generators is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.generators]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_generators = w_list
     return w_self.w_generators
 
@@ -5373,8 +5298,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elt'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elt')
     return space.wrap(w_self.elt)
 
 def SetComp_set_elt(space, w_self, w_new_value):
@@ -5391,14 +5315,13 @@
 def SetComp_get_generators(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'generators'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators')
     if w_self.w_generators is None:
         if w_self.generators is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.generators]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_generators = w_list
     return w_self.w_generators
 
@@ -5439,8 +5362,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'key'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'key')
     return space.wrap(w_self.key)
 
 def DictComp_set_key(space, w_self, w_new_value):
@@ -5461,8 +5383,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def DictComp_set_value(space, w_self, w_new_value):
@@ -5479,14 +5400,13 @@
 def DictComp_get_generators(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'generators'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators')
     if w_self.w_generators is None:
         if w_self.generators is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.generators]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_generators = w_list
     return w_self.w_generators
 
@@ -5528,8 +5448,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elt'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elt')
     return space.wrap(w_self.elt)
 
 def GeneratorExp_set_elt(space, w_self, w_new_value):
@@ -5546,14 +5465,13 @@
 def GeneratorExp_get_generators(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'generators'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators')
     if w_self.w_generators is None:
         if w_self.generators is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.generators]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_generators = w_list
     return w_self.w_generators
 
@@ -5594,8 +5512,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Yield_set_value(space, w_self, w_new_value):
@@ -5640,8 +5557,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'left'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'left')
     return space.wrap(w_self.left)
 
 def Compare_set_left(space, w_self, w_new_value):
@@ -5658,14 +5574,13 @@
 def Compare_get_ops(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ops'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ops')
     if w_self.w_ops is None:
         if w_self.ops is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [cmpop_to_class[node - 1]() for node in w_self.ops]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_ops = w_list
     return w_self.w_ops
 
@@ -5676,14 +5591,13 @@
 def Compare_get_comparators(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'comparators'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'comparators')
     if w_self.w_comparators is None:
         if w_self.comparators is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.comparators]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_comparators = w_list
     return w_self.w_comparators
 
@@ -5726,8 +5640,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'func'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'func')
     return space.wrap(w_self.func)
 
 def Call_set_func(space, w_self, w_new_value):
@@ -5744,14 +5657,13 @@
 def Call_get_args(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'args'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args')
     if w_self.w_args is None:
         if w_self.args is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.args]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_args = w_list
     return w_self.w_args
 
@@ -5762,14 +5674,13 @@
 def Call_get_keywords(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'keywords'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'keywords')
     if w_self.w_keywords is None:
         if w_self.keywords is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.keywords]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_keywords = w_list
     return w_self.w_keywords
 
@@ -5784,8 +5695,7 @@
             return w_obj
     if not w_self.initialization_state & 8:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'starargs'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'starargs')
     return space.wrap(w_self.starargs)
 
 def Call_set_starargs(space, w_self, w_new_value):
@@ -5806,8 +5716,7 @@
             return w_obj
     if not w_self.initialization_state & 16:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'kwargs'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'kwargs')
     return space.wrap(w_self.kwargs)
 
 def Call_set_kwargs(space, w_self, w_new_value):
@@ -5858,8 +5767,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Repr_set_value(space, w_self, w_new_value):
@@ -5904,8 +5812,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'n'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'n')
     return w_self.n
 
 def Num_set_n(space, w_self, w_new_value):
@@ -5950,8 +5857,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 's'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 's')
     return w_self.s
 
 def Str_set_s(space, w_self, w_new_value):
@@ -5996,8 +5902,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Attribute_set_value(space, w_self, w_new_value):
@@ -6018,8 +5923,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'attr'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'attr')
     return space.wrap(w_self.attr)
 
 def Attribute_set_attr(space, w_self, w_new_value):
@@ -6040,8 +5944,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ctx'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def Attribute_set_ctx(space, w_self, w_new_value):
@@ -6090,8 +5993,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Subscript_set_value(space, w_self, w_new_value):
@@ -6112,8 +6014,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'slice'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'slice')
     return space.wrap(w_self.slice)
 
 def Subscript_set_slice(space, w_self, w_new_value):
@@ -6134,8 +6035,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ctx'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def Subscript_set_ctx(space, w_self, w_new_value):
@@ -6184,8 +6084,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'id'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'id')
     return space.wrap(w_self.id)
 
 def Name_set_id(space, w_self, w_new_value):
@@ -6206,8 +6105,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ctx'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def Name_set_ctx(space, w_self, w_new_value):
@@ -6251,14 +6149,13 @@
 def List_get_elts(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elts'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elts')
     if w_self.w_elts is None:
         if w_self.elts is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.elts]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_elts = w_list
     return w_self.w_elts
 
@@ -6273,8 +6170,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ctx'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def List_set_ctx(space, w_self, w_new_value):
@@ -6319,14 +6215,13 @@
 def Tuple_get_elts(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elts'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elts')
     if w_self.w_elts is None:
         if w_self.elts is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.elts]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_elts = w_list
     return w_self.w_elts
 
@@ -6341,8 +6236,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ctx'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def Tuple_set_ctx(space, w_self, w_new_value):
@@ -6391,8 +6285,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return w_self.value
 
 def Const_set_value(space, w_self, w_new_value):
@@ -6510,8 +6403,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'lower'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lower')
     return space.wrap(w_self.lower)
 
 def Slice_set_lower(space, w_self, w_new_value):
@@ -6532,8 +6424,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'upper'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'upper')
     return space.wrap(w_self.upper)
 
 def Slice_set_upper(space, w_self, w_new_value):
@@ -6554,8 +6445,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'step'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'step')
     return space.wrap(w_self.step)
 
 def Slice_set_step(space, w_self, w_new_value):
@@ -6598,14 +6488,13 @@
 def ExtSlice_get_dims(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'dims'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'dims')
     if w_self.w_dims is None:
         if w_self.dims is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.dims]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_dims = w_list
     return w_self.w_dims
 
@@ -6645,8 +6534,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Index_set_value(space, w_self, w_new_value):
@@ -6915,8 +6803,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'target'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'target')
     return space.wrap(w_self.target)
 
 def comprehension_set_target(space, w_self, w_new_value):
@@ -6937,8 +6824,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'iter'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'iter')
     return space.wrap(w_self.iter)
 
 def comprehension_set_iter(space, w_self, w_new_value):
@@ -6955,14 +6841,13 @@
 def comprehension_get_ifs(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ifs'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ifs')
     if w_self.w_ifs is None:
         if w_self.ifs is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.ifs]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_ifs = w_list
     return w_self.w_ifs
 
@@ -7004,8 +6889,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._lineno_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'lineno'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lineno')
     return space.wrap(w_self.lineno)
 
 def excepthandler_set_lineno(space, w_self, w_new_value):
@@ -7026,8 +6910,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._col_offset_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'col_offset'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'col_offset')
     return space.wrap(w_self.col_offset)
 
 def excepthandler_set_col_offset(space, w_self, w_new_value):
@@ -7057,8 +6940,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'type'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'type')
     return space.wrap(w_self.type)
 
 def ExceptHandler_set_type(space, w_self, w_new_value):
@@ -7079,8 +6961,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'name'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name')
     return space.wrap(w_self.name)
 
 def ExceptHandler_set_name(space, w_self, w_new_value):
@@ -7097,14 +6978,13 @@
 def ExceptHandler_get_body(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -7142,14 +7022,13 @@
 def arguments_get_args(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'args'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args')
     if w_self.w_args is None:
         if w_self.args is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.args]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_args = w_list
     return w_self.w_args
 
@@ -7164,8 +7043,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'vararg'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'vararg')
     return space.wrap(w_self.vararg)
 
 def arguments_set_vararg(space, w_self, w_new_value):
@@ -7189,8 +7067,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'kwarg'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'kwarg')
     return space.wrap(w_self.kwarg)
 
 def arguments_set_kwarg(space, w_self, w_new_value):
@@ -7210,14 +7087,13 @@
 def arguments_get_defaults(space, w_self):
     if not w_self.initialization_state & 8:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'defaults'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'defaults')
     if w_self.w_defaults is None:
         if w_self.defaults is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.defaults]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_defaults = w_list
     return w_self.w_defaults
 
@@ -7261,8 +7137,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'arg'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'arg')
     return space.wrap(w_self.arg)
 
 def keyword_set_arg(space, w_self, w_new_value):
@@ -7283,8 +7158,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def keyword_set_value(space, w_self, w_new_value):
@@ -7330,8 +7204,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'name'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name')
     return space.wrap(w_self.name)
 
 def alias_set_name(space, w_self, w_new_value):
@@ -7352,8 +7225,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'asname'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'asname')
     return space.wrap(w_self.asname)
 
 def alias_set_asname(space, w_self, w_new_value):
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -414,13 +414,12 @@
             self.emit("        return w_obj", 1)
         self.emit("if not w_self.initialization_state & %s:" % (flag,), 1)
         self.emit("typename = space.type(w_self).getname(space)", 2)
-        self.emit("w_err = space.wrap(\"'%%s' object has no attribute '%s'\" %% typename)" %
+        self.emit("raise operationerrfmt(space.w_AttributeError, \"'%%s' object has no attribute '%%s'\", typename, '%s')" %
                   (field.name,), 2)
-        self.emit("raise OperationError(space.w_AttributeError, w_err)", 2)
         if field.seq:
             self.emit("if w_self.w_%s is None:" % (field.name,), 1)
             self.emit("if w_self.%s is None:" % (field.name,), 2)
-            self.emit("w_list = space.newlist([])", 3)
+            self.emit("list_w = []", 3)
             self.emit("else:", 2)
             if field.type.value in self.data.simple_types:
                 wrapper = "%s_to_class[node - 1]()" % (field.type,)
@@ -428,7 +427,7 @@
                 wrapper = "space.wrap(node)"
             self.emit("list_w = [%s for node in w_self.%s]" %
                       (wrapper, field.name), 3)
-            self.emit("w_list = space.newlist(list_w)", 3)
+            self.emit("w_list = space.newlist(list_w)", 2)
             self.emit("w_self.w_%s = w_list" % (field.name,), 2)
             self.emit("return w_self.w_%s" % (field.name,), 1)
         elif field.type.value in self.data.simple_types:
@@ -540,7 +539,7 @@
 from pypy.interpreter.baseobjspace import Wrappable
 from pypy.interpreter import typedef
 from pypy.interpreter.gateway import interp2app
-from pypy.interpreter.error import OperationError
+from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.tool.pairtype import extendabletype
 from pypy.tool.sourcetools import func_with_new_name
@@ -639,9 +638,7 @@
             missing = required[i]
             if missing is not None:
                  err = "required field \\"%s\\" missing from %s"
-                 err = err % (missing, host)
-                 w_err = space.wrap(err)
-                 raise OperationError(space.w_TypeError, w_err)
+                 raise operationerrfmt(space.w_TypeError, err, missing, host)
     raise AssertionError("should not reach here")
 
 
diff --git a/pypy/jit/backend/llgraph/llimpl.py b/pypy/jit/backend/llgraph/llimpl.py
--- a/pypy/jit/backend/llgraph/llimpl.py
+++ b/pypy/jit/backend/llgraph/llimpl.py
@@ -7,9 +7,7 @@
 import weakref
 from pypy.objspace.flow.model import Variable, Constant
 from pypy.annotation import model as annmodel
-from pypy.jit.metainterp.history import (ConstInt, ConstPtr,
-                                         BoxInt, BoxPtr, BoxObj, BoxFloat,
-                                         REF, INT, FLOAT)
+from pypy.jit.metainterp.history import REF, INT, FLOAT
 from pypy.jit.codewriter import heaptracker
 from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr, rffi
 from pypy.rpython.ootypesystem import ootype
@@ -17,7 +15,7 @@
 from pypy.rpython.llinterp import LLException
 from pypy.rpython.extregistry import ExtRegistryEntry
 
-from pypy.jit.metainterp import resoperation, executor
+from pypy.jit.metainterp import resoperation
 from pypy.jit.metainterp.resoperation import rop
 from pypy.jit.backend.llgraph import symbolic
 from pypy.jit.codewriter import longlong
@@ -334,6 +332,13 @@
     assert isinstance(type, str) and len(type) == 1
     op.descr = Descr(ofs, type, arg_types=arg_types)
 
+def compile_add_descr_arg(loop, ofs, type, arg_types):
+    from pypy.jit.backend.llgraph.runner import Descr
+    loop = _from_opaque(loop)
+    op = loop.operations[-1]
+    assert isinstance(type, str) and len(type) == 1
+    op.args.append(Descr(ofs, type, arg_types=arg_types))
+
 def compile_add_loop_token(loop, descr):
     if we_are_translated():
         raise ValueError("CALL_ASSEMBLER not supported")
@@ -438,8 +443,11 @@
         self._may_force = -1
 
     def getenv(self, v):
+        from pypy.jit.backend.llgraph.runner import Descr
         if isinstance(v, Constant):
             return v.value
+        elif isinstance(v, Descr):
+            return v
         else:
             return self.env[v]
 
@@ -807,6 +815,29 @@
         else:
             raise NotImplementedError
 
+    def op_getinteriorfield_gc(self, descr, array, index):
+        if descr.typeinfo == REF:
+            return do_getinteriorfield_gc_ptr(array, index, descr.ofs)
+        elif descr.typeinfo == INT:
+            return do_getinteriorfield_gc_int(array, index, descr.ofs)
+        elif descr.typeinfo == FLOAT:
+            return do_getinteriorfield_gc_float(array, index, descr.ofs)
+        else:
+            raise NotImplementedError
+
+    def op_setinteriorfield_gc(self, descr, array, index, newvalue):
+        if descr.typeinfo == REF:
+            return do_setinteriorfield_gc_ptr(array, index, descr.ofs,
+                                              newvalue)
+        elif descr.typeinfo == INT:
+            return do_setinteriorfield_gc_int(array, index, descr.ofs,
+                                              newvalue)
+        elif descr.typeinfo == FLOAT:
+            return do_setinteriorfield_gc_float(array, index, descr.ofs,
+                                                newvalue)
+        else:
+            raise NotImplementedError
+
     def op_setfield_gc(self, fielddescr, struct, newvalue):
         if fielddescr.typeinfo == REF:
             do_setfield_gc_ptr(struct, fielddescr.ofs, newvalue)
@@ -1354,6 +1385,22 @@
 def do_getfield_gc_ptr(struct, fieldnum):
     return cast_to_ptr(_getfield_gc(struct, fieldnum))
 
+def _getinteriorfield_gc(struct, fieldnum):
+    STRUCT, fieldname = symbolic.TokenToField[fieldnum]
+    return getattr(struct, fieldname)
+
+def do_getinteriorfield_gc_int(array, index, fieldnum):
+    struct = array._obj.container.getitem(index)
+    return cast_to_int(_getinteriorfield_gc(struct, fieldnum))
+
+def do_getinteriorfield_gc_float(array, index, fieldnum):
+    struct = array._obj.container.getitem(index)
+    return cast_to_floatstorage(_getinteriorfield_gc(struct, fieldnum))
+
+def do_getinteriorfield_gc_ptr(array, index, fieldnum):
+    struct = array._obj.container.getitem(index)
+    return cast_to_ptr(_getinteriorfield_gc(struct, fieldnum))
+
 def _getfield_raw(struct, fieldnum):
     STRUCT, fieldname = symbolic.TokenToField[fieldnum]
     ptr = cast_from_int(lltype.Ptr(STRUCT), struct)
@@ -1409,26 +1456,28 @@
     newvalue = cast_from_ptr(ITEMTYPE, newvalue)
     array.setitem(index, newvalue)
 
-def do_setfield_gc_int(struct, fieldnum, newvalue):
-    STRUCT, fieldname = symbolic.TokenToField[fieldnum]
-    ptr = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), struct)
-    FIELDTYPE = getattr(STRUCT, fieldname)
-    newvalue = cast_from_int(FIELDTYPE, newvalue)
-    setattr(ptr, fieldname, newvalue)
+def new_setfield_gc(cast_func):
+    def do_setfield_gc(struct, fieldnum, newvalue):
+        STRUCT, fieldname = symbolic.TokenToField[fieldnum]
+        ptr = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), struct)
+        FIELDTYPE = getattr(STRUCT, fieldname)
+        newvalue = cast_func(FIELDTYPE, newvalue)
+        setattr(ptr, fieldname, newvalue)
+    return do_setfield_gc
+do_setfield_gc_int = new_setfield_gc(cast_from_int)
+do_setfield_gc_float = new_setfield_gc(cast_from_floatstorage)
+do_setfield_gc_ptr = new_setfield_gc(cast_from_ptr)
 
-def do_setfield_gc_float(struct, fieldnum, newvalue):
-    STRUCT, fieldname = symbolic.TokenToField[fieldnum]
-    ptr = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), struct)
-    FIELDTYPE = getattr(STRUCT, fieldname)
-    newvalue = cast_from_floatstorage(FIELDTYPE, newvalue)
-    setattr(ptr, fieldname, newvalue)
-
-def do_setfield_gc_ptr(struct, fieldnum, newvalue):
-    STRUCT, fieldname = symbolic.TokenToField[fieldnum]
-    ptr = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), struct)
-    FIELDTYPE = getattr(STRUCT, fieldname)
-    newvalue = cast_from_ptr(FIELDTYPE, newvalue)
-    setattr(ptr, fieldname, newvalue)
+def new_setinteriorfield_gc(cast_func):
+    def do_setinteriorfield_gc(array, index, fieldnum, newvalue):
+        STRUCT, fieldname = symbolic.TokenToField[fieldnum]
+        struct = array._obj.container.getitem(index)
+        FIELDTYPE = getattr(STRUCT, fieldname)
+        setattr(struct, fieldname, cast_func(FIELDTYPE, newvalue))
+    return do_setinteriorfield_gc
+do_setinteriorfield_gc_int = new_setinteriorfield_gc(cast_from_int)
+do_setinteriorfield_gc_float = new_setinteriorfield_gc(cast_from_floatstorage)
+do_setinteriorfield_gc_ptr = new_setinteriorfield_gc(cast_from_ptr)        
 
 def do_setfield_raw_int(struct, fieldnum, newvalue):
     STRUCT, fieldname = symbolic.TokenToField[fieldnum]
@@ -1694,6 +1743,7 @@
 setannotation(compile_start_float_var, annmodel.SomeInteger())
 setannotation(compile_add, annmodel.s_None)
 setannotation(compile_add_descr, annmodel.s_None)
+setannotation(compile_add_descr_arg, annmodel.s_None)
 setannotation(compile_add_var, annmodel.s_None)
 setannotation(compile_add_int_const, annmodel.s_None)
 setannotation(compile_add_ref_const, annmodel.s_None)
@@ -1741,6 +1791,9 @@
 setannotation(do_getfield_raw_int, annmodel.SomeInteger())
 setannotation(do_getfield_raw_ptr, annmodel.SomePtr(llmemory.GCREF))
 setannotation(do_getfield_raw_float, s_FloatStorage)
+setannotation(do_getinteriorfield_gc_int, annmodel.SomeInteger())
+setannotation(do_getinteriorfield_gc_ptr, annmodel.SomePtr(llmemory.GCREF))
+setannotation(do_getinteriorfield_gc_float, s_FloatStorage)
 setannotation(do_new, annmodel.SomePtr(llmemory.GCREF))
 setannotation(do_new_array, annmodel.SomePtr(llmemory.GCREF))
 setannotation(do_setarrayitem_gc_int, annmodel.s_None)
@@ -1754,6 +1807,9 @@
 setannotation(do_setfield_raw_int, annmodel.s_None)
 setannotation(do_setfield_raw_ptr, annmodel.s_None)
 setannotation(do_setfield_raw_float, annmodel.s_None)
+setannotation(do_setinteriorfield_gc_int, annmodel.s_None)
+setannotation(do_setinteriorfield_gc_ptr, annmodel.s_None)
+setannotation(do_setinteriorfield_gc_float, annmodel.s_None)
 setannotation(do_newstr, annmodel.SomePtr(llmemory.GCREF))
 setannotation(do_strsetitem, annmodel.s_None)
 setannotation(do_newunicode, annmodel.SomePtr(llmemory.GCREF))
diff --git a/pypy/jit/backend/llgraph/runner.py b/pypy/jit/backend/llgraph/runner.py
--- a/pypy/jit/backend/llgraph/runner.py
+++ b/pypy/jit/backend/llgraph/runner.py
@@ -2,21 +2,19 @@
 Minimal-API wrapper around the llinterpreter to run operations.
 """
 
-import sys
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.rpython.lltypesystem import lltype, llmemory, rclass
 from pypy.rpython.ootypesystem import ootype
 from pypy.rpython.llinterp import LLInterpreter
 from pypy.jit.metainterp import history
-from pypy.jit.metainterp.history import REF, INT, FLOAT
+from pypy.jit.metainterp.history import REF, INT, FLOAT, STRUCT
 from pypy.jit.metainterp.warmstate import unwrap
-from pypy.jit.metainterp.resoperation import ResOperation, rop
+from pypy.jit.metainterp.resoperation import rop
 from pypy.jit.backend import model
 from pypy.jit.backend.llgraph import llimpl, symbolic
 from pypy.jit.metainterp.typesystem import llhelper, oohelper
 from pypy.jit.codewriter import heaptracker, longlong
-from pypy.rlib import rgc
 
 class MiniStats:
     pass
@@ -62,6 +60,9 @@
     def is_array_of_floats(self):
         return self.typeinfo == FLOAT
 
+    def is_array_of_structs(self):
+        return self.typeinfo == STRUCT
+
     def as_vtable_size_descr(self):
         return self
 
@@ -177,8 +178,10 @@
             llimpl.compile_add(c, op.getopnum())
             descr = op.getdescr()
             if isinstance(descr, Descr):
-                llimpl.compile_add_descr(c, descr.ofs, descr.typeinfo, descr.arg_types)
-            if isinstance(descr, history.LoopToken) and op.getopnum() != rop.JUMP:
+                llimpl.compile_add_descr(c, descr.ofs, descr.typeinfo,
+                                         descr.arg_types)
+            if (isinstance(descr, history.LoopToken) and
+                op.getopnum() != rop.JUMP):
                 llimpl.compile_add_loop_token(c, descr)
             if self.is_oo and isinstance(descr, (OODescr, MethDescr)):
                 # hack hack, not rpython
@@ -193,6 +196,9 @@
                     llimpl.compile_add_ref_const(c, x.value, self.ts.BASETYPE)
                 elif isinstance(x, history.ConstFloat):
                     llimpl.compile_add_float_const(c, x.value)
+                elif isinstance(x, Descr):
+                    llimpl.compile_add_descr_arg(c, x.ofs, x.typeinfo,
+                                                 x.arg_types)
                 else:
                     raise Exception("'%s' args contain: %r" % (op.getopname(),
                                                                x))
@@ -316,6 +322,13 @@
         token = history.getkind(getattr(S, fieldname))
         return self.getdescr(ofs, token[0], name=fieldname)
 
+    def interiorfielddescrof(self, A, fieldname):
+        S = A.OF
+        ofs2 = symbolic.get_size(A)
+        ofs, size = symbolic.get_field_token(S, fieldname)
+        token = history.getkind(getattr(S, fieldname))
+        return self.getdescr(ofs, token[0], name=fieldname, extrainfo=ofs2)
+
     def calldescrof(self, FUNC, ARGS, RESULT, extrainfo):
         arg_types = []
         for ARG in ARGS:
@@ -353,8 +366,13 @@
     def arraydescrof(self, A):
         assert A.OF != lltype.Void
         size = symbolic.get_size(A)
-        token = history.getkind(A.OF)
-        return self.getdescr(size, token[0])
+        if isinstance(A.OF, lltype.Ptr) or isinstance(A.OF, lltype.Primitive):
+            token = history.getkind(A.OF)[0]
+        elif isinstance(A.OF, lltype.Struct):
+            token = 's'
+        else:
+            token = '?'
+        return self.getdescr(size, token)
 
     # ---------- the backend-dependent operations ----------
 
@@ -406,6 +424,29 @@
         assert isinstance(fielddescr, Descr)
         return llimpl.do_getfield_raw_float(struct, fielddescr.ofs)
 
+    def bh_getinteriorfield_gc_i(self, array, index, descr):
+        assert isinstance(descr, Descr)
+        return llimpl.do_getinteriorfield_gc_int(array, index, descr.ofs)
+    def bh_getinteriorfield_gc_r(self, array, index, descr):
+        assert isinstance(descr, Descr)
+        return llimpl.do_getinteriorfield_gc_ptr(array, index, descr.ofs)
+    def bh_getinteriorfield_gc_f(self, array, index, descr):
+        assert isinstance(descr, Descr)
+        return llimpl.do_getinteriorfield_gc_float(array, index, descr.ofs)
+
+    def bh_setinteriorfield_gc_i(self, array, index, descr, value):
+        assert isinstance(descr, Descr)
+        return llimpl.do_setinteriorfield_gc_int(array, index, descr.ofs,
+                                                 value)
+    def bh_setinteriorfield_gc_r(self, array, index, descr, value):
+        assert isinstance(descr, Descr)
+        return llimpl.do_setinteriorfield_gc_ptr(array, index, descr.ofs,
+                                                 value)
+    def bh_setinteriorfield_gc_f(self, array, index, descr, value):
+        assert isinstance(descr, Descr)
+        return llimpl.do_setinteriorfield_gc_float(array, index, descr.ofs,
+                                                   value)
+
     def bh_new(self, sizedescr):
         assert isinstance(sizedescr, Descr)
         return llimpl.do_new(sizedescr.ofs)
@@ -418,7 +459,6 @@
 
     def bh_classof(self, struct):
         struct = lltype.cast_opaque_ptr(rclass.OBJECTPTR, struct)
-        result = struct.typeptr
         result_adr = llmemory.cast_ptr_to_adr(struct.typeptr)
         return heaptracker.adr2int(result_adr)
 
diff --git a/pypy/jit/backend/llsupport/descr.py b/pypy/jit/backend/llsupport/descr.py
--- a/pypy/jit/backend/llsupport/descr.py
+++ b/pypy/jit/backend/llsupport/descr.py
@@ -1,13 +1,10 @@
 import py
-from pypy.rpython.lltypesystem import lltype, rffi, llmemory, rclass
+from pypy.rpython.lltypesystem import lltype, rffi, llmemory
 from pypy.rpython.lltypesystem.lloperation import llop
 from pypy.jit.backend.llsupport import symbolic, support
-from pypy.jit.metainterp.history import AbstractDescr, getkind, BoxInt, BoxPtr
-from pypy.jit.metainterp.history import BasicFailDescr, LoopToken, BoxFloat
+from pypy.jit.metainterp.history import AbstractDescr, getkind
 from pypy.jit.metainterp import history
-from pypy.jit.metainterp.resoperation import ResOperation, rop
 from pypy.jit.codewriter import heaptracker, longlong
-from pypy.rlib.rarithmetic import r_longlong, r_ulonglong
 
 # The point of the class organization in this file is to make instances
 # as compact as possible.  This is done by not storing the field size or
@@ -23,6 +20,7 @@
         self._cache_field = {}
         self._cache_array = {}
         self._cache_call = {}
+        self._cache_interiorfield = {}
 
     def init_size_descr(self, STRUCT, sizedescr):
         assert isinstance(STRUCT, lltype.GcStruct)
@@ -142,7 +140,6 @@
         cachedict[fieldname] = fielddescr
         return fielddescr
 
-
 # ____________________________________________________________
 # ArrayDescrs
 
@@ -167,6 +164,7 @@
 
     _is_array_of_pointers = False      # unless overridden by GcPtrArrayDescr
     _is_array_of_floats   = False      # unless overridden by FloatArrayDescr
+    _is_array_of_structs  = False      # unless overridden by StructArrayDescr
     _is_item_signed       = False      # unless overridden by XxxArrayDescr
 
     def is_array_of_pointers(self):
@@ -175,6 +173,9 @@
     def is_array_of_floats(self):
         return self._is_array_of_floats
 
+    def is_array_of_structs(self):
+        return self._is_array_of_structs
+
     def is_item_signed(self):
         return self._is_item_signed
 
@@ -199,6 +200,10 @@
     def get_item_size(self, translate_support_code):
         return symbolic.get_size(lltype.Float, translate_support_code)
 
+class StructArrayDescr(BaseArrayDescr):
+    _clsname = 'StructArrayDescr'
+    _is_array_of_structs = True
+
 class BaseArrayNoLengthDescr(BaseArrayDescr):
     def get_base_size(self, translate_support_code):
         return 0
@@ -218,6 +223,13 @@
 def getArrayDescrClass(ARRAY):
     if ARRAY.OF is lltype.Float:
         return FloatArrayDescr
+    elif isinstance(ARRAY.OF, lltype.Struct):
+        class Descr(StructArrayDescr):
+            _clsname = '%sArrayDescr' % ARRAY.OF._name
+            def get_item_size(self, translate_support_code):
+                return symbolic.get_size(ARRAY.OF, translate_support_code)
+        Descr.__name__ = Descr._clsname
+        return Descr
     return getDescrClass(ARRAY.OF, BaseArrayDescr, GcPtrArrayDescr,
                          NonGcPtrArrayDescr, 'Array', 'get_item_size',
                          '_is_array_of_floats', '_is_item_signed')
@@ -252,6 +264,36 @@
         cache[ARRAY] = arraydescr
         return arraydescr
 
+# ____________________________________________________________
+# InteriorFieldDescr
+
+class InteriorFieldDescr(AbstractDescr):
+    arraydescr = BaseArrayDescr()     # workaround for the annotator
+    fielddescr = BaseFieldDescr('', 0)
+
+    def __init__(self, arraydescr, fielddescr):
+        self.arraydescr = arraydescr
+        self.fielddescr = fielddescr
+
+    def is_pointer_field(self):
+        return self.fielddescr.is_pointer_field()
+
+    def is_float_field(self):
+        return self.fielddescr.is_float_field()
+
+    def repr_of_descr(self):
+        return '<InteriorFieldDescr %s>' % self.fielddescr.repr_of_descr()
+
+def get_interiorfield_descr(gc_ll_descr, ARRAY, FIELDTP, name):
+    cache = gc_ll_descr._cache_interiorfield
+    try:
+        return cache[(ARRAY, FIELDTP, name)]
+    except KeyError:
+        arraydescr = get_array_descr(gc_ll_descr, ARRAY)
+        fielddescr = get_field_descr(gc_ll_descr, FIELDTP, name)
+        descr = InteriorFieldDescr(arraydescr, fielddescr)
+        cache[(ARRAY, FIELDTP, name)] = descr
+        return descr
 
 # ____________________________________________________________
 # CallDescrs
@@ -525,7 +567,8 @@
         #
         if TYPE is lltype.Float or is_longlong(TYPE):
             setattr(Descr, floatattrname, True)
-        elif TYPE is not lltype.Bool and rffi.cast(TYPE, -1) == -1:
+        elif (TYPE is not lltype.Bool and isinstance(TYPE, lltype.Number) and
+              rffi.cast(TYPE, -1) == -1):
             setattr(Descr, signedattrname, True)
         #
         _cache[nameprefix, TYPE] = Descr
diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py
--- a/pypy/jit/backend/llsupport/gc.py
+++ b/pypy/jit/backend/llsupport/gc.py
@@ -45,6 +45,14 @@
     def freeing_block(self, start, stop):
         pass
 
+    def get_funcptr_for_newarray(self):
+        return llhelper(self.GC_MALLOC_ARRAY, self.malloc_array)
+    def get_funcptr_for_newstr(self):
+        return llhelper(self.GC_MALLOC_STR_UNICODE, self.malloc_str)
+    def get_funcptr_for_newunicode(self):
+        return llhelper(self.GC_MALLOC_STR_UNICODE, self.malloc_unicode)
+
+
     def record_constptrs(self, op, gcrefs_output_list):
         for i in range(op.numargs()):
             v = op.getarg(i)
@@ -96,6 +104,39 @@
         malloc_fn_ptr = self.configure_boehm_once()
         self.funcptr_for_new = malloc_fn_ptr
 
+        def malloc_array(basesize, itemsize, ofs_length, num_elem):
+            try:
+                size = ovfcheck(basesize + ovfcheck(itemsize * num_elem))
+            except OverflowError:
+                return lltype.nullptr(llmemory.GCREF.TO)
+            res = self.funcptr_for_new(size)
+            if not res:
+                return res
+            rffi.cast(rffi.CArrayPtr(lltype.Signed), res)[ofs_length/WORD] = num_elem
+            return res
+        self.malloc_array = malloc_array
+        self.GC_MALLOC_ARRAY = lltype.Ptr(lltype.FuncType(
+            [lltype.Signed] * 4, llmemory.GCREF))
+
+
+        (str_basesize, str_itemsize, str_ofs_length
+         ) = symbolic.get_array_token(rstr.STR, self.translate_support_code)
+        (unicode_basesize, unicode_itemsize, unicode_ofs_length
+         ) = symbolic.get_array_token(rstr.UNICODE, self.translate_support_code)
+        def malloc_str(length):
+            return self.malloc_array(
+                str_basesize, str_itemsize, str_ofs_length, length
+            )
+        def malloc_unicode(length):
+            return self.malloc_array(
+                unicode_basesize, unicode_itemsize, unicode_ofs_length, length
+            )
+        self.malloc_str = malloc_str
+        self.malloc_unicode = malloc_unicode
+        self.GC_MALLOC_STR_UNICODE = lltype.Ptr(lltype.FuncType(
+            [lltype.Signed], llmemory.GCREF))
+
+
         # on some platform GC_init is required before any other
         # GC_* functions, call it here for the benefit of tests
         # XXX move this to tests
@@ -116,39 +157,27 @@
         ofs_length = arraydescr.get_ofs_length(self.translate_support_code)
         basesize = arraydescr.get_base_size(self.translate_support_code)
         itemsize = arraydescr.get_item_size(self.translate_support_code)
-        size = basesize + itemsize * num_elem
-        res = self.funcptr_for_new(size)
-        rffi.cast(rffi.CArrayPtr(lltype.Signed), res)[ofs_length/WORD] = num_elem
-        return res
+        return self.malloc_array(basesize, itemsize, ofs_length, num_elem)
 
     def gc_malloc_str(self, num_elem):
-        basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR,
-                                                   self.translate_support_code)
-        assert itemsize == 1
-        size = basesize + num_elem
-        res = self.funcptr_for_new(size)
-        rffi.cast(rffi.CArrayPtr(lltype.Signed), res)[ofs_length/WORD] = num_elem
-        return res
+        return self.malloc_str(num_elem)
 
     def gc_malloc_unicode(self, num_elem):
-        basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE,
-                                                   self.translate_support_code)
-        size = basesize + num_elem * itemsize
-        res = self.funcptr_for_new(size)
-        rffi.cast(rffi.CArrayPtr(lltype.Signed), res)[ofs_length/WORD] = num_elem
-        return res
+        return self.malloc_unicode(num_elem)
 
     def args_for_new(self, sizedescr):
         assert isinstance(sizedescr, BaseSizeDescr)
         return [sizedescr.size]
 
+    def args_for_new_array(self, arraydescr):
+        ofs_length = arraydescr.get_ofs_length(self.translate_support_code)
+        basesize = arraydescr.get_base_size(self.translate_support_code)
+        itemsize = arraydescr.get_item_size(self.translate_support_code)
+        return [basesize, itemsize, ofs_length]
+
     def get_funcptr_for_new(self):
         return self.funcptr_for_new
 
-    get_funcptr_for_newarray = None
-    get_funcptr_for_newstr = None
-    get_funcptr_for_newunicode = None
-
     def rewrite_assembler(self, cpu, operations, gcrefs_output_list):
         # record all GCREFs too, because Boehm cannot see them and keep them
         # alive if they end up as constants in the assembler
@@ -752,15 +781,6 @@
     def get_funcptr_for_new(self):
         return llhelper(self.GC_MALLOC_BASIC, self.malloc_basic)
 
-    def get_funcptr_for_newarray(self):
-        return llhelper(self.GC_MALLOC_ARRAY, self.malloc_array)
-
-    def get_funcptr_for_newstr(self):
-        return llhelper(self.GC_MALLOC_STR_UNICODE, self.malloc_str)
-
-    def get_funcptr_for_newunicode(self):
-        return llhelper(self.GC_MALLOC_STR_UNICODE, self.malloc_unicode)
-
     def do_write_barrier(self, gcref_struct, gcref_newptr):
         hdr_addr = llmemory.cast_ptr_to_adr(gcref_struct)
         hdr_addr -= self.gcheaderbuilder.size_gc_header
diff --git a/pypy/jit/backend/llsupport/llmodel.py b/pypy/jit/backend/llsupport/llmodel.py
--- a/pypy/jit/backend/llsupport/llmodel.py
+++ b/pypy/jit/backend/llsupport/llmodel.py
@@ -1,24 +1,18 @@
-import sys
 from pypy.rpython.lltypesystem import lltype, llmemory, rffi, rclass, rstr
 from pypy.rpython.lltypesystem.lloperation import llop
-from pypy.rpython.llinterp import LLInterpreter, LLException
+from pypy.rpython.llinterp import LLInterpreter
 from pypy.rpython.annlowlevel import llhelper
 from pypy.rlib.objectmodel import we_are_translated, specialize
-from pypy.jit.metainterp.history import BoxInt, BoxPtr, set_future_values,\
-     BoxFloat
 from pypy.jit.metainterp import history
 from pypy.jit.codewriter import heaptracker, longlong
 from pypy.jit.backend.model import AbstractCPU
 from pypy.jit.backend.llsupport import symbolic
 from pypy.jit.backend.llsupport.symbolic import WORD, unroll_basic_sizes
-from pypy.jit.backend.llsupport.descr import get_size_descr,  BaseSizeDescr
-from pypy.jit.backend.llsupport.descr import get_field_descr, BaseFieldDescr
-from pypy.jit.backend.llsupport.descr import get_array_descr, BaseArrayDescr
-from pypy.jit.backend.llsupport.descr import get_call_descr
-from pypy.jit.backend.llsupport.descr import BaseIntCallDescr, GcPtrCallDescr
-from pypy.jit.backend.llsupport.descr import FloatCallDescr, VoidCallDescr
+from pypy.jit.backend.llsupport.descr import (get_size_descr,
+     get_field_descr, BaseFieldDescr, get_array_descr, BaseArrayDescr,
+     get_call_descr, BaseIntCallDescr, GcPtrCallDescr, FloatCallDescr,
+     VoidCallDescr, InteriorFieldDescr, get_interiorfield_descr)
 from pypy.jit.backend.llsupport.asmmemmgr import AsmMemoryManager
-from pypy.rpython.annlowlevel import cast_instance_to_base_ptr
 
 
 class AbstractLLCPU(AbstractCPU):
@@ -241,6 +235,9 @@
     def arraydescrof(self, A):
         return get_array_descr(self.gc_ll_descr, A)
 
+    def interiorfielddescrof(self, A, fieldname):
+        return get_interiorfield_descr(self.gc_ll_descr, A, A.OF, fieldname)
+
     def unpack_arraydescr(self, arraydescr):
         assert isinstance(arraydescr, BaseArrayDescr)
         return arraydescr.get_base_size(self.translate_support_code)
@@ -358,6 +355,100 @@
     bh_getarrayitem_raw_i = bh_getarrayitem_gc_i
     bh_getarrayitem_raw_f = bh_getarrayitem_gc_f
 
+    def bh_getinteriorfield_gc_i(self, gcref, itemindex, descr):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        fieldsize = descr.fielddescr.get_field_size(self.translate_support_code)
+        sign = descr.fielddescr.is_field_signed()
+        fullofs = itemindex * size + ofs
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), fullofs)
+        for STYPE, UTYPE, itemsize in unroll_basic_sizes:
+            if fieldsize == itemsize:
+                if sign:
+                    item = rffi.cast(rffi.CArrayPtr(STYPE), items)
+                    val = item[0]
+                    val = rffi.cast(lltype.Signed, val)
+                else:
+                    item = rffi.cast(rffi.CArrayPtr(UTYPE), items)
+                    val = item[0]
+                    val = rffi.cast(lltype.Signed, val)
+                # --- end of GC unsafe code ---
+                return val
+        else:
+            raise NotImplementedError("size = %d" % fieldsize)
+
+    def bh_getinteriorfield_gc_r(self, gcref, itemindex, descr):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs +
+                            size * itemindex)
+        items = rffi.cast(rffi.CArrayPtr(lltype.Signed), items)
+        pval = self._cast_int_to_gcref(items[0])
+        # --- end of GC unsafe code ---
+        return pval
+
+    def bh_getinteriorfield_gc_f(self, gcref, itemindex, descr):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs +
+                            size * itemindex)
+        items = rffi.cast(rffi.CArrayPtr(longlong.FLOATSTORAGE), items)
+        fval = items[0]
+        # --- end of GC unsafe code ---
+        return fval
+
+    def bh_setinteriorfield_gc_i(self, gcref, itemindex, descr, value):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        fieldsize = descr.fielddescr.get_field_size(self.translate_support_code)
+        ofs = itemindex * size + ofs
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs)
+        for TYPE, _, itemsize in unroll_basic_sizes:
+            if fieldsize == itemsize:
+                items = rffi.cast(rffi.CArrayPtr(TYPE), items)
+                items[0] = rffi.cast(TYPE, value)
+                # --- end of GC unsafe code ---
+                return
+        else:
+            raise NotImplementedError("size = %d" % fieldsize)
+
+    def bh_setinteriorfield_gc_r(self, gcref, itemindex, descr, newvalue):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        self.gc_ll_descr.do_write_barrier(gcref, newvalue)
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref),
+                            ofs + size * itemindex)
+        items = rffi.cast(rffi.CArrayPtr(lltype.Signed), items)
+        items[0] = self.cast_gcref_to_int(newvalue)
+        # --- end of GC unsafe code ---
+
+    def bh_setinteriorfield_gc_f(self, gcref, itemindex, descr, newvalue):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref),
+                            ofs + size * itemindex)
+        items = rffi.cast(rffi.CArrayPtr(longlong.FLOATSTORAGE), items)
+        items[0] = newvalue
+        # --- end of GC unsafe code ---
+
     def bh_strlen(self, string):
         s = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), string)
         return len(s.chars)
@@ -475,7 +566,6 @@
 
     def bh_classof(self, struct):
         struct = lltype.cast_opaque_ptr(rclass.OBJECTPTR, struct)
-        result = struct.typeptr
         result_adr = llmemory.cast_ptr_to_adr(struct.typeptr)
         return heaptracker.adr2int(result_adr)
 
diff --git a/pypy/jit/backend/llsupport/regalloc.py b/pypy/jit/backend/llsupport/regalloc.py
--- a/pypy/jit/backend/llsupport/regalloc.py
+++ b/pypy/jit/backend/llsupport/regalloc.py
@@ -287,7 +287,6 @@
         self.reg_bindings[to_v] = reg
 
     def _move_variable_away(self, v, prev_loc):
-        reg = None
         if self.free_regs:
             loc = self.free_regs.pop()
             self.reg_bindings[v] = loc
diff --git a/pypy/jit/backend/llsupport/test/test_asmmemmgr.py b/pypy/jit/backend/llsupport/test/test_asmmemmgr.py
--- a/pypy/jit/backend/llsupport/test/test_asmmemmgr.py
+++ b/pypy/jit/backend/llsupport/test/test_asmmemmgr.py
@@ -211,14 +211,14 @@
     debug._log = debug.DebugLog()
     try:
         mc._dump(addr, 'test-logname-section')
-        log = list(debug._log)
+        log = list(debug._log) 
     finally:
         debug._log = None
     encoded = ''.join(writtencode).encode('hex').upper()
     ataddr = '@%x' % addr
     assert log == [('test-logname-section',
                     [('debug_print', 'CODE_DUMP', ataddr, '+0 ', encoded)])]
-    #
+    
     lltype.free(p, flavor='raw')
 
 def test_blockbuildermixin2():
diff --git a/pypy/jit/backend/llsupport/test/test_descr.py b/pypy/jit/backend/llsupport/test/test_descr.py
--- a/pypy/jit/backend/llsupport/test/test_descr.py
+++ b/pypy/jit/backend/llsupport/test/test_descr.py
@@ -3,7 +3,6 @@
 from pypy.jit.backend.llsupport import symbolic
 from pypy.rlib.objectmodel import Symbolic
 from pypy.rpython.annlowlevel import llhelper
-from pypy.jit.metainterp.history import BoxInt, BoxFloat, BoxPtr
 from pypy.jit.metainterp import history
 from pypy.jit.codewriter import longlong
 import sys, struct, py
@@ -149,7 +148,9 @@
     A2 = lltype.GcArray(lltype.Ptr(T))
     A3 = lltype.GcArray(lltype.Ptr(U))
     A4 = lltype.GcArray(lltype.Float)
-    A5 = lltype.GcArray(lltype.SingleFloat)
+    A5 = lltype.GcArray(lltype.Struct('x', ('v', lltype.Signed),
+                                      ('k', lltype.Signed)))
+    A6 = lltype.GcArray(lltype.SingleFloat)
     assert getArrayDescrClass(A2) is GcPtrArrayDescr
     assert getArrayDescrClass(A3) is NonGcPtrArrayDescr
     cls = getArrayDescrClass(A1)
@@ -158,7 +159,7 @@
     clsf = getArrayDescrClass(A4)
     assert clsf != cls
     assert clsf == getArrayDescrClass(lltype.GcArray(lltype.Float))
-    clss = getArrayDescrClass(A5)
+    clss = getArrayDescrClass(A6)
     assert clss not in (clsf, cls)
     assert clss == getArrayDescrClass(lltype.GcArray(rffi.UINT))
     #
@@ -168,11 +169,12 @@
     descr3 = get_array_descr(c0, A3)
     descr4 = get_array_descr(c0, A4)
     descr5 = get_array_descr(c0, A5)
+    descr6 = get_array_descr(c0, A6)
     assert descr1.__class__ is cls
     assert descr2.__class__ is GcPtrArrayDescr
     assert descr3.__class__ is NonGcPtrArrayDescr
     assert descr4.__class__ is clsf
-    assert descr5.__class__ is clss
+    assert descr6.__class__ is clss
     assert descr1 == get_array_descr(c0, lltype.GcArray(lltype.Char))
     assert not descr1.is_array_of_pointers()
     assert     descr2.is_array_of_pointers()
@@ -202,7 +204,8 @@
     assert descr2.get_item_size(False) == rffi.sizeof(lltype.Ptr(T))
     assert descr3.get_item_size(False) == rffi.sizeof(lltype.Ptr(U))
     assert descr4.get_item_size(False) == rffi.sizeof(lltype.Float)
-    assert descr5.get_item_size(False) == rffi.sizeof(lltype.SingleFloat)
+    assert descr5.get_item_size(False) == rffi.sizeof(lltype.Signed) * 2
+    assert descr6.get_item_size(False) == rffi.sizeof(lltype.SingleFloat)
     #
     assert isinstance(descr1.get_base_size(True), Symbolic)
     assert isinstance(descr2.get_base_size(True), Symbolic)
@@ -348,7 +351,6 @@
                             (rffi.SHORT,      True), (rffi.USHORT, False),
                             (rffi.INT,        True), (rffi.UINT,   False),
                             (rffi.LONG,       True), (rffi.ULONG,  False)]:
-        A = lltype.GcArray(RESTYPE)
         for tsc in [False, True]:
             c2 = GcCache(tsc)
             descr1 = get_call_descr(c2, [], RESTYPE)
@@ -379,7 +381,6 @@
     descr3i = get_array_descr(c0, lltype.GcArray(lltype.Char))
     assert descr3i.repr_of_descr() == '<CharArrayDescr>'
     #
-    cache = {}
     descr4 = get_call_descr(c0, [lltype.Char, lltype.Ptr(S)], lltype.Ptr(S))
     assert 'GcPtrCallDescr' in descr4.repr_of_descr()
     #
@@ -412,10 +413,10 @@
     ARGS = [lltype.Float, lltype.Ptr(ARRAY)]
     RES = lltype.Float
 
-    def f(a, b):
+    def f2(a, b):
         return float(b[0]) + a
 
-    fnptr = llhelper(lltype.Ptr(lltype.FuncType(ARGS, RES)), f)
+    fnptr = llhelper(lltype.Ptr(lltype.FuncType(ARGS, RES)), f2)
     descr2 = get_call_descr(c0, ARGS, RES)
     a = lltype.malloc(ARRAY, 3)
     opaquea = lltype.cast_opaque_ptr(llmemory.GCREF, a)
diff --git a/pypy/jit/backend/model.py b/pypy/jit/backend/model.py
--- a/pypy/jit/backend/model.py
+++ b/pypy/jit/backend/model.py
@@ -1,5 +1,5 @@
 from pypy.rlib.debug import debug_start, debug_print, debug_stop
-from pypy.jit.metainterp import history, compile
+from pypy.jit.metainterp import history
 
 
 class AbstractCPU(object):
@@ -213,6 +213,10 @@
     def typedescrof(TYPE):
         raise NotImplementedError
 
+    @staticmethod
+    def interiorfielddescrof(A, fieldname):
+        raise NotImplementedError
+
     # ---------- the backend-dependent operations ----------
 
     # lltype specific operations
diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -5,7 +5,7 @@
                                          BoxInt, Box, BoxPtr,
                                          LoopToken,
                                          ConstInt, ConstPtr,
-                                         BoxObj, Const,
+                                         BoxObj,
                                          ConstObj, BoxFloat, ConstFloat)
 from pypy.jit.metainterp.resoperation import ResOperation, rop
 from pypy.jit.metainterp.typesystem import deref
@@ -111,7 +111,7 @@
         self.cpu.set_future_value_int(0, 2)
         fail = self.cpu.execute_token(looptoken)
         res = self.cpu.get_latest_value_int(0)
-        assert res == 3        
+        assert res == 3
         assert fail.identifier == 1
 
     def test_compile_loop(self):
@@ -127,7 +127,7 @@
             ]
         inputargs = [i0]
         operations[2].setfailargs([i1])
-        
+
         self.cpu.compile_loop(inputargs, operations, looptoken)
         self.cpu.set_future_value_int(0, 2)
         fail = self.cpu.execute_token(looptoken)
@@ -148,7 +148,7 @@
             ]
         inputargs = [i0]
         operations[2].setfailargs([None, None, i1, None])
-        
+
         self.cpu.compile_loop(inputargs, operations, looptoken)
         self.cpu.set_future_value_int(0, 2)
         fail = self.cpu.execute_token(looptoken)
@@ -372,7 +372,7 @@
         for opnum, boxargs, retvalue in get_int_tests():
             res = self.execute_operation(opnum, boxargs, 'int')
             assert res.value == retvalue
-        
+
     def test_float_operations(self):
         from pypy.jit.metainterp.test.test_executor import get_float_tests
         for opnum, boxargs, rettype, retvalue in get_float_tests(self.cpu):
@@ -438,7 +438,7 @@
 
     def test_ovf_operations_reversed(self):
         self.test_ovf_operations(reversed=True)
-        
+
     def test_bh_call(self):
         cpu = self.cpu
         #
@@ -503,7 +503,7 @@
                                          [funcbox, BoxInt(num), BoxInt(num)],
                                          'int', descr=dyn_calldescr)
             assert res.value == 2 * num
-            
+
 
         if cpu.supports_floats:
             def func(f0, f1, f2, f3, f4, f5, f6, i0, i1, f7, f8, f9):
@@ -543,7 +543,7 @@
         funcbox = self.get_funcbox(self.cpu, func_ptr)
         res = self.execute_operation(rop.CALL, [funcbox] + map(BoxInt, args), 'int', descr=calldescr)
         assert res.value == func(*args)
-        
+
     def test_call_stack_alignment(self):
         # test stack alignment issues, notably for Mac OS/X.
         # also test the ordering of the arguments.
@@ -615,7 +615,7 @@
         res = self.execute_operation(rop.GETFIELD_GC, [t_box],
                                      'int', descr=shortdescr)
         assert res.value == 1331
-        
+
         #
         u_box, U_box = self.alloc_instance(self.U)
         fielddescr2 = self.cpu.fielddescrof(self.S, 'next')
@@ -695,7 +695,7 @@
 
     def test_failing_guard_class(self):
         t_box, T_box = self.alloc_instance(self.T)
-        u_box, U_box = self.alloc_instance(self.U)        
+        u_box, U_box = self.alloc_instance(self.U)
         null_box = self.null_instance()
         for opname, args in [(rop.GUARD_CLASS, [t_box, U_box]),
                              (rop.GUARD_CLASS, [u_box, T_box]),
@@ -787,7 +787,7 @@
         r = self.execute_operation(rop.GETARRAYITEM_GC, [a_box, BoxInt(3)],
                                    'int', descr=arraydescr)
         assert r.value == 160
-        
+
         #
         if isinstance(A, lltype.GcArray):
             A = lltype.Ptr(A)
@@ -880,6 +880,73 @@
                                    'int', descr=arraydescr)
         assert r.value == 7441
 
+    def test_array_of_structs(self):
+        TP = lltype.GcStruct('x')
+        ITEM = lltype.Struct('x',
+                             ('vs', lltype.Signed),
+                             ('vu', lltype.Unsigned),
+                             ('vsc', rffi.SIGNEDCHAR),
+                             ('vuc', rffi.UCHAR),
+                             ('vss', rffi.SHORT),
+                             ('vus', rffi.USHORT),
+                             ('vsi', rffi.INT),
+                             ('vui', rffi.UINT),
+                             ('k', lltype.Float),
+                             ('p', lltype.Ptr(TP)))
+        a_box, A = self.alloc_array_of(ITEM, 15)
+        s_box, S = self.alloc_instance(TP)
+        kdescr = self.cpu.interiorfielddescrof(A, 'k')
+        pdescr = self.cpu.interiorfielddescrof(A, 'p')
+        self.execute_operation(rop.SETINTERIORFIELD_GC, [a_box, BoxInt(3),
+                                                         boxfloat(1.5)],
+                               'void', descr=kdescr)
+        f = self.cpu.bh_getinteriorfield_gc_f(a_box.getref_base(), 3, kdescr)
+        assert longlong.getrealfloat(f) == 1.5
+        self.cpu.bh_setinteriorfield_gc_f(a_box.getref_base(), 3, kdescr, longlong.getfloatstorage(2.5))
+        r = self.execute_operation(rop.GETINTERIORFIELD_GC, [a_box, BoxInt(3)],
+                                   'float', descr=kdescr)
+        assert r.getfloat() == 2.5
+        #
+        NUMBER_FIELDS = [('vs', lltype.Signed),
+                         ('vu', lltype.Unsigned),
+                         ('vsc', rffi.SIGNEDCHAR),
+                         ('vuc', rffi.UCHAR),
+                         ('vss', rffi.SHORT),
+                         ('vus', rffi.USHORT),
+                         ('vsi', rffi.INT),
+                         ('vui', rffi.UINT)]
+        for name, TYPE in NUMBER_FIELDS[::-1]:
+            vdescr = self.cpu.interiorfielddescrof(A, name)
+            self.execute_operation(rop.SETINTERIORFIELD_GC, [a_box, BoxInt(3),
+                                                             BoxInt(-15)],
+                                   'void', descr=vdescr)
+        for name, TYPE in NUMBER_FIELDS:
+            vdescr = self.cpu.interiorfielddescrof(A, name)
+            i = self.cpu.bh_getinteriorfield_gc_i(a_box.getref_base(), 3,
+                                                  vdescr)
+            assert i == rffi.cast(lltype.Signed, rffi.cast(TYPE, -15))
+        for name, TYPE in NUMBER_FIELDS[::-1]:
+            vdescr = self.cpu.interiorfielddescrof(A, name)
+            self.cpu.bh_setinteriorfield_gc_i(a_box.getref_base(), 3,
+                                              vdescr, -25)
+        for name, TYPE in NUMBER_FIELDS:
+            vdescr = self.cpu.interiorfielddescrof(A, name)
+            r = self.execute_operation(rop.GETINTERIORFIELD_GC,
+                                       [a_box, BoxInt(3)],
+                                       'int', descr=vdescr)
+            assert r.getint() == rffi.cast(lltype.Signed, rffi.cast(TYPE, -25))
+        #
+        self.execute_operation(rop.SETINTERIORFIELD_GC, [a_box, BoxInt(4),
+                                                         s_box],
+                               'void', descr=pdescr)
+        r = self.cpu.bh_getinteriorfield_gc_r(a_box.getref_base(), 4, pdescr)
+        assert r == s_box.getref_base()
+        self.cpu.bh_setinteriorfield_gc_r(a_box.getref_base(), 3, pdescr,
+                                          s_box.getref_base())
+        r = self.execute_operation(rop.GETINTERIORFIELD_GC, [a_box, BoxInt(3)],
+                                   'ref', descr=pdescr)
+        assert r.getref_base() == s_box.getref_base()
+
     def test_string_basic(self):
         s_box = self.alloc_string("hello\xfe")
         r = self.execute_operation(rop.STRLEN, [s_box], 'int')
@@ -1402,7 +1469,7 @@
         addr = llmemory.cast_ptr_to_adr(func_ptr)
         return ConstInt(heaptracker.adr2int(addr))
 
-    
+
     MY_VTABLE = rclass.OBJECT_VTABLE    # for tests only
 
     S = lltype.GcForwardReference()
@@ -1439,7 +1506,6 @@
         return BoxPtr(lltype.nullptr(llmemory.GCREF.TO))
 
     def alloc_array_of(self, ITEM, length):
-        cpu = self.cpu
         A = lltype.GcArray(ITEM)
         a = lltype.malloc(A, length)
         a_box = BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, a))
@@ -2318,7 +2384,7 @@
         for opname, arg, res in ops:
             self.execute_operation(opname, [arg], 'void')
             assert self.guard_failed == res
-        
+
         lltype.free(x, flavor='raw')
 
     def test_assembler_call(self):
@@ -2398,7 +2464,7 @@
         FakeJitDriverSD.portal_calldescr = self.cpu.calldescrof(
             lltype.Ptr(lltype.FuncType(ARGS, RES)), ARGS, RES,
             EffectInfo.MOST_GENERAL)
-        
+
         ops = '''
         [f0, f1]
         f2 = float_add(f0, f1)
@@ -2489,7 +2555,7 @@
         FakeJitDriverSD.portal_calldescr = self.cpu.calldescrof(
             lltype.Ptr(lltype.FuncType(ARGS, RES)), ARGS, RES,
             EffectInfo.MOST_GENERAL)
-        
+
         ops = '''
         [f0, f1]
         f2 = float_add(f0, f1)
@@ -2940,4 +3006,4 @@
 
     def alloc_unicode(self, unicode):
         py.test.skip("implement me")
-    
+
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1,7 +1,7 @@
 import sys, os
 from pypy.jit.backend.llsupport import symbolic
 from pypy.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper
-from pypy.jit.metainterp.history import Const, Box, BoxInt, BoxPtr, BoxFloat
+from pypy.jit.metainterp.history import Const, Box, BoxInt, ConstInt
 from pypy.jit.metainterp.history import (AbstractFailDescr, INT, REF, FLOAT,
                                          LoopToken)
 from pypy.rpython.lltypesystem import lltype, rffi, rstr, llmemory
@@ -36,7 +36,6 @@
 from pypy.rlib import rgc
 from pypy.rlib.clibffi import FFI_DEFAULT_ABI
 from pypy.jit.backend.x86.jump import remap_frame_layout
-from pypy.jit.metainterp.history import ConstInt, BoxInt
 from pypy.jit.codewriter.effectinfo import EffectInfo
 from pypy.jit.codewriter import longlong
 
@@ -729,8 +728,8 @@
         # Also, make sure this is consistent with FRAME_FIXED_SIZE.
         self.mc.PUSH_r(ebp.value)
         self.mc.MOV_rr(ebp.value, esp.value)
-        for regloc in self.cpu.CALLEE_SAVE_REGISTERS:
-            self.mc.PUSH_r(regloc.value)
+        for loc in self.cpu.CALLEE_SAVE_REGISTERS:
+            self.mc.PUSH_r(loc.value)
 
         gcrootmap = self.cpu.gc_ll_descr.gcrootmap
         if gcrootmap and gcrootmap.is_shadow_stack:
@@ -994,7 +993,7 @@
         effectinfo = op.getdescr().get_extra_info()
         oopspecindex = effectinfo.oopspecindex
         genop_llong_list[oopspecindex](self, op, arglocs, resloc)
-        
+
     def regalloc_perform_math(self, op, arglocs, resloc):
         effectinfo = op.getdescr().get_extra_info()
         oopspecindex = effectinfo.oopspecindex
@@ -1311,7 +1310,7 @@
     genop_guard_float_eq = _cmpop_guard_float("E", "E", "NE","NE")
     genop_guard_float_gt = _cmpop_guard_float("A", "B", "BE","AE")
     genop_guard_float_ge = _cmpop_guard_float("AE","BE", "B", "A")
-    
+
     def genop_math_sqrt(self, op, arglocs, resloc):
         self.mc.SQRTSD(arglocs[0], resloc)
 
@@ -1597,12 +1596,27 @@
     genop_getarrayitem_gc_pure = genop_getarrayitem_gc
     genop_getarrayitem_raw = genop_getarrayitem_gc
 
+    def genop_getinteriorfield_gc(self, op, arglocs, resloc):
+        base_loc, ofs_loc, itemsize_loc, fieldsize_loc, index_loc, sign_loc = arglocs
+        # XXX should not use IMUL in most cases
+        self.mc.IMUL(index_loc, itemsize_loc)
+        src_addr = AddressLoc(base_loc, index_loc, 0, ofs_loc.value)
+        self.load_from_mem(resloc, src_addr, fieldsize_loc, sign_loc)
+
+
     def genop_discard_setfield_gc(self, op, arglocs):
         base_loc, ofs_loc, size_loc, value_loc = arglocs
         assert isinstance(size_loc, ImmedLoc)
         dest_addr = AddressLoc(base_loc, ofs_loc)
         self.save_into_mem(dest_addr, value_loc, size_loc)
 
+    def genop_discard_setinteriorfield_gc(self, op, arglocs):
+        base_loc, ofs_loc, itemsize_loc, fieldsize_loc, index_loc, value_loc = arglocs
+        # XXX should not use IMUL in most cases
+        self.mc.IMUL(index_loc, itemsize_loc)
+        dest_addr = AddressLoc(base_loc, index_loc, 0, ofs_loc.value)
+        self.save_into_mem(dest_addr, value_loc, fieldsize_loc)
+
     def genop_discard_setarrayitem_gc(self, op, arglocs):
         base_loc, ofs_loc, value_loc, size_loc, baseofs = arglocs
         assert isinstance(baseofs, ImmedLoc)
diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py
--- a/pypy/jit/backend/x86/regalloc.py
+++ b/pypy/jit/backend/x86/regalloc.py
@@ -7,7 +7,7 @@
                                          ResOperation, BoxPtr, ConstFloat,
                                          BoxFloat, LoopToken, INT, REF, FLOAT)
 from pypy.jit.backend.x86.regloc import *
-from pypy.rpython.lltypesystem import lltype, ll2ctypes, rffi, rstr
+from pypy.rpython.lltypesystem import lltype, rffi, rstr
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.rlib import rgc
 from pypy.jit.backend.llsupport import symbolic
@@ -17,11 +17,12 @@
 from pypy.jit.metainterp.resoperation import rop
 from pypy.jit.backend.llsupport.descr import BaseFieldDescr, BaseArrayDescr
 from pypy.jit.backend.llsupport.descr import BaseCallDescr, BaseSizeDescr
+from pypy.jit.backend.llsupport.descr import InteriorFieldDescr
 from pypy.jit.backend.llsupport.regalloc import FrameManager, RegisterManager,\
      TempBox
 from pypy.jit.backend.x86.arch import WORD, FRAME_FIXED_SIZE
 from pypy.jit.backend.x86.arch import IS_X86_32, IS_X86_64, MY_COPY_OF_REGS
-from pypy.rlib.rarithmetic import r_longlong, r_uint
+from pypy.rlib.rarithmetic import r_longlong
 
 class X86RegisterManager(RegisterManager):
 
@@ -433,7 +434,7 @@
             if self.can_merge_with_next_guard(op, i, operations):
                 oplist_with_guard[op.getopnum()](self, op, operations[i + 1])
                 i += 1
-            elif not we_are_translated() and op.getopnum() == -124: 
+            elif not we_are_translated() and op.getopnum() == -124:
                 self._consider_force_spill(op)
             else:
                 oplist[op.getopnum()](self, op)
@@ -815,7 +816,7 @@
         save_all_regs = guard_not_forced_op is not None
         self.xrm.before_call(force_store, save_all_regs=save_all_regs)
         if not save_all_regs:
-            gcrootmap = gc_ll_descr = self.assembler.cpu.gc_ll_descr.gcrootmap
+            gcrootmap = self.assembler.cpu.gc_ll_descr.gcrootmap
             if gcrootmap and gcrootmap.is_shadow_stack:
                 save_all_regs = 2
         self.rm.before_call(force_store, save_all_regs=save_all_regs)
@@ -972,74 +973,27 @@
             return self._call(op, arglocs)
 
     def consider_newstr(self, op):
-        gc_ll_descr = self.assembler.cpu.gc_ll_descr
-        if gc_ll_descr.get_funcptr_for_newstr is not None:
-            # framework GC
-            loc = self.loc(op.getarg(0))
-            return self._call(op, [loc])
-        # boehm GC (XXX kill the following code at some point)
-        ofs_items, itemsize, ofs = symbolic.get_array_token(rstr.STR, self.translate_support_code)
-        assert itemsize == 1
-        return self._malloc_varsize(ofs_items, ofs, 0, op.getarg(0),
-                                    op.result)
+        loc = self.loc(op.getarg(0))
+        return self._call(op, [loc])
 
     def consider_newunicode(self, op):
-        gc_ll_descr = self.assembler.cpu.gc_ll_descr
-        if gc_ll_descr.get_funcptr_for_newunicode is not None:
-            # framework GC
-            loc = self.loc(op.getarg(0))
-            return self._call(op, [loc])
-        # boehm GC (XXX kill the following code at some point)
-        ofs_items, _, ofs = symbolic.get_array_token(rstr.UNICODE,
-                                                   self.translate_support_code)
-        scale = self._get_unicode_item_scale()
-        return self._malloc_varsize(ofs_items, ofs, scale, op.getarg(0),
-                                    op.result)
-
-    def _malloc_varsize(self, ofs_items, ofs_length, scale, v, res_v):
-        # XXX kill this function at some point
-        if isinstance(v, Box):
-            loc = self.rm.make_sure_var_in_reg(v, [v])
-            tempbox = TempBox()
-            other_loc = self.rm.force_allocate_reg(tempbox, [v])
-            self.assembler.load_effective_addr(loc, ofs_items,scale, other_loc)
-        else:
-            tempbox = None
-            other_loc = imm(ofs_items + (v.getint() << scale))
-        self._call(ResOperation(rop.NEW, [], res_v),
-                   [other_loc], [v])
-        loc = self.rm.make_sure_var_in_reg(v, [res_v])
-        assert self.loc(res_v) == eax
-        # now we have to reload length to some reasonable place
-        self.rm.possibly_free_var(v)
-        if tempbox is not None:
-            self.rm.possibly_free_var(tempbox)
-        self.PerformDiscard(ResOperation(rop.SETFIELD_GC, [None, None], None),
-                            [eax, imm(ofs_length), imm(WORD), loc])
+        loc = self.loc(op.getarg(0))
+        return self._call(op, [loc])
 
     def consider_new_array(self, op):
         gc_ll_descr = self.assembler.cpu.gc_ll_descr
-        if gc_ll_descr.get_funcptr_for_newarray is not None:
-            # framework GC
-            box_num_elem = op.getarg(0)
-            if isinstance(box_num_elem, ConstInt):
-                num_elem = box_num_elem.value
-                if gc_ll_descr.can_inline_malloc_varsize(op.getdescr(),
-                                                         num_elem):
-                    self.fastpath_malloc_varsize(op, op.getdescr(), num_elem)
-                    return
-            args = self.assembler.cpu.gc_ll_descr.args_for_new_array(
-                op.getdescr())
-            arglocs = [imm(x) for x in args]
-            arglocs.append(self.loc(box_num_elem))
-            self._call(op, arglocs)
-            return
-        # boehm GC (XXX kill the following code at some point)
-        itemsize, basesize, ofs_length, _, _ = (
-            self._unpack_arraydescr(op.getdescr()))
-        scale_of_field = _get_scale(itemsize)
-        self._malloc_varsize(basesize, ofs_length, scale_of_field,
-                             op.getarg(0), op.result)
+        box_num_elem = op.getarg(0)
+        if isinstance(box_num_elem, ConstInt):
+            num_elem = box_num_elem.value
+            if gc_ll_descr.can_inline_malloc_varsize(op.getdescr(),
+                                                     num_elem):
+                self.fastpath_malloc_varsize(op, op.getdescr(), num_elem)
+                return
+        args = self.assembler.cpu.gc_ll_descr.args_for_new_array(
+            op.getdescr())
+        arglocs = [imm(x) for x in args]
+        arglocs.append(self.loc(box_num_elem))
+        self._call(op, arglocs)
 
     def _unpack_arraydescr(self, arraydescr):
         assert isinstance(arraydescr, BaseArrayDescr)
@@ -1058,6 +1012,16 @@
         sign = fielddescr.is_field_signed()
         return imm(ofs), imm(size), ptr, sign
 
+    def _unpack_interiorfielddescr(self, descr):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs = arraydescr.get_base_size(self.translate_support_code)
+        itemsize = arraydescr.get_item_size(self.translate_support_code)
+        fieldsize = descr.fielddescr.get_field_size(self.translate_support_code)
+        sign = descr.fielddescr.is_field_signed()
+        ofs += descr.fielddescr.offset
+        return imm(ofs), imm(itemsize), imm(fieldsize), sign
+
     def consider_setfield_gc(self, op):
         ofs_loc, size_loc, _, _ = self._unpack_fielddescr(op.getdescr())
         assert isinstance(size_loc, ImmedLoc)
@@ -1074,6 +1038,21 @@
 
     consider_setfield_raw = consider_setfield_gc
 
+    def consider_setinteriorfield_gc(self, op):
+        t = self._unpack_interiorfielddescr(op.getdescr())
+        ofs, itemsize, fieldsize, _ = t
+        args = op.getarglist()
+        tmpvar = TempBox()
+        base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
+        index_loc = self.rm.force_result_in_reg(tmpvar, op.getarg(1),
+                                                args)
+        # we're free to modify index now
+        value_loc = self.make_sure_var_in_reg(op.getarg(2), args)
+        self.possibly_free_vars(args)
+        self.rm.possibly_free_var(tmpvar)
+        self.PerformDiscard(op, [base_loc, ofs, itemsize, fieldsize,
+                                 index_loc, value_loc])
+
     def consider_strsetitem(self, op):
         args = op.getarglist()
         base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
@@ -1135,6 +1114,24 @@
     consider_getarrayitem_raw = consider_getarrayitem_gc
     consider_getarrayitem_gc_pure = consider_getarrayitem_gc
 
+    def consider_getinteriorfield_gc(self, op):
+        t = self._unpack_interiorfielddescr(op.getdescr())
+        ofs, itemsize, fieldsize, sign = t
+        if sign:
+            sign_loc = imm1
+        else:
+            sign_loc = imm0
+        args = op.getarglist()
+        tmpvar = TempBox()
+        base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
+        index_loc = self.rm.force_result_in_reg(tmpvar, op.getarg(1),
+                                                args)
+        self.rm.possibly_free_vars_for_op(op)
+        self.rm.possibly_free_var(tmpvar)
+        result_loc = self.force_allocate_reg(op.result)
+        self.Perform(op, [base_loc, ofs, itemsize, fieldsize,
+                          index_loc, sign_loc], result_loc)
+
     def consider_int_is_true(self, op, guard_op):
         # doesn't need arg to be in a register
         argloc = self.loc(op.getarg(0))
@@ -1241,7 +1238,6 @@
         self.rm.possibly_free_var(srcaddr_box)
 
     def _gen_address_inside_string(self, baseloc, ofsloc, resloc, is_unicode):
-        cpu = self.assembler.cpu
         if is_unicode:
             ofs_items, _, _ = symbolic.get_array_token(rstr.UNICODE,
                                                   self.translate_support_code)
@@ -1300,7 +1296,7 @@
         tmpreg = X86RegisterManager.all_regs[0]
         tmploc = self.rm.force_allocate_reg(box, selected_reg=tmpreg)
         xmmtmp = X86XMMRegisterManager.all_regs[0]
-        xmmtmploc = self.xrm.force_allocate_reg(box1, selected_reg=xmmtmp)
+        self.xrm.force_allocate_reg(box1, selected_reg=xmmtmp)
         # Part about non-floats
         # XXX we don't need a copy, we only just the original list
         src_locations1 = [self.loc(op.getarg(i)) for i in range(op.numargs())
@@ -1380,7 +1376,7 @@
     return lambda self, op: fn(self, op, None)
 
 def is_comparison_or_ovf_op(opnum):
-    from pypy.jit.metainterp.resoperation import opclasses, AbstractResOp
+    from pypy.jit.metainterp.resoperation import opclasses
     cls = opclasses[opnum]
     # hack hack: in theory they are instance method, but they don't use
     # any instance field, we can use a fake object
diff --git a/pypy/jit/backend/x86/test/test_del.py b/pypy/jit/backend/x86/test/test_del.py
--- a/pypy/jit/backend/x86/test/test_del.py
+++ b/pypy/jit/backend/x86/test/test_del.py
@@ -1,5 +1,4 @@
 
-import py
 from pypy.jit.backend.x86.test.test_basic import Jit386Mixin
 from pypy.jit.metainterp.test.test_del import DelTests
 
diff --git a/pypy/jit/backend/x86/test/test_dict.py b/pypy/jit/backend/x86/test/test_dict.py
new file mode 100644
--- /dev/null
+++ b/pypy/jit/backend/x86/test/test_dict.py
@@ -0,0 +1,9 @@
+
+from pypy.jit.backend.x86.test.test_basic import Jit386Mixin
+from pypy.jit.metainterp.test.test_dict import DictTests
+
+
+class TestDict(Jit386Mixin, DictTests):
+    # for the individual tests see
+    # ====> ../../../metainterp/test/test_dict.py
+    pass
diff --git a/pypy/jit/backend/x86/test/test_runner.py b/pypy/jit/backend/x86/test/test_runner.py
--- a/pypy/jit/backend/x86/test/test_runner.py
+++ b/pypy/jit/backend/x86/test/test_runner.py
@@ -31,7 +31,7 @@
 
     # for the individual tests see
     # ====> ../../test/runner_test.py
-    
+
     def setup_method(self, meth):
         self.cpu = CPU(rtyper=None, stats=FakeStats())
         self.cpu.setup_once()
@@ -69,22 +69,16 @@
 
     def test_allocations(self):
         from pypy.rpython.lltypesystem import rstr
-        
+
         allocs = [None]
         all = []
+        orig_new = self.cpu.gc_ll_descr.funcptr_for_new
         def f(size):
             allocs.insert(0, size)
-            buf = ctypes.create_string_buffer(size)
-            all.append(buf)
-            return ctypes.cast(buf, ctypes.c_void_p).value
-        func = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int)(f)
-        addr = ctypes.cast(func, ctypes.c_void_p).value
-        # ctypes produces an unsigned value. We need it to be signed for, eg,
-        # relative addressing to work properly.
-        addr = rffi.cast(lltype.Signed, addr)
-        
+            return orig_new(size)
+
         self.cpu.assembler.setup_once()
-        self.cpu.assembler.malloc_func_addr = addr
+        self.cpu.gc_ll_descr.funcptr_for_new = f
         ofs = symbolic.get_field_token(rstr.STR, 'chars', False)[0]
 
         res = self.execute_operation(rop.NEWSTR, [ConstInt(7)], 'ref')
@@ -108,7 +102,7 @@
         res = self.execute_operation(rop.NEW_ARRAY, [ConstInt(10)],
                                          'ref', descr)
         assert allocs[0] == 10*WORD + ofs + WORD
-        resbuf = self._resbuf(res)            
+        resbuf = self._resbuf(res)
         assert resbuf[ofs/WORD] == 10
 
         # ------------------------------------------------------------
@@ -116,7 +110,7 @@
         res = self.execute_operation(rop.NEW_ARRAY, [BoxInt(10)],
                                          'ref', descr)
         assert allocs[0] == 10*WORD + ofs + WORD
-        resbuf = self._resbuf(res)                        
+        resbuf = self._resbuf(res)
         assert resbuf[ofs/WORD] == 10
 
     def test_stringitems(self):
@@ -146,7 +140,7 @@
                                                      ConstInt(2), BoxInt(38)],
                                'void', descr)
         assert resbuf[itemsofs/WORD + 2] == 38
-        
+
         self.execute_operation(rop.SETARRAYITEM_GC, [res,
                                                      BoxInt(3), BoxInt(42)],
                                'void', descr)
@@ -167,7 +161,7 @@
                                                          BoxInt(2)],
                                    'int', descr)
         assert r.value == 38
-        
+
         r = self.execute_operation(rop.GETARRAYITEM_GC, [res, BoxInt(3)],
                                    'int', descr)
         assert r.value == 42
@@ -226,7 +220,7 @@
         self.execute_operation(rop.SETFIELD_GC, [res, BoxInt(1234)], 'void', ofs_i)
         i = self.execute_operation(rop.GETFIELD_GC, [res], 'int', ofs_i)
         assert i.value == 1234
-        
+
         #u = self.execute_operation(rop.GETFIELD_GC, [res, ofs_u], 'int')
         #assert u.value == 5
         self.execute_operation(rop.SETFIELD_GC, [res, ConstInt(1)], 'void',
@@ -299,7 +293,7 @@
                     else:
                         assert result != execute(self.cpu, None,
                                                  op, None, b).value
-                    
+
 
     def test_stuff_followed_by_guard(self):
         boxes = [(BoxInt(1), BoxInt(0)),
@@ -523,7 +517,7 @@
     def test_debugger_on(self):
         from pypy.tool.logparser import parse_log_file, extract_category
         from pypy.rlib import debug
-        
+
         loop = """
         [i0]
         debug_merge_point('xyz', 0)
diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -52,9 +52,11 @@
         newoperations = []
         #
         def do_rename(var, var_or_const):
+            if var.concretetype is lltype.Void:
+                renamings[var] = Constant(None, lltype.Void)
+                return
             renamings[var] = var_or_const
-            if (isinstance(var_or_const, Constant)
-                and var.concretetype != lltype.Void):
+            if isinstance(var_or_const, Constant):
                 value = var_or_const.value
                 value = lltype._cast_whatever(var.concretetype, value)
                 renamings_constants[var] = Constant(value, var.concretetype)
@@ -441,6 +443,8 @@
     rewrite_op_gc_identityhash = _do_builtin_call
     rewrite_op_gc_id           = _do_builtin_call
     rewrite_op_uint_mod        = _do_builtin_call
+    rewrite_op_cast_float_to_uint = _do_builtin_call
+    rewrite_op_cast_uint_to_float = _do_builtin_call
 
     # ----------
     # getfield/setfield/mallocs etc.
@@ -735,29 +739,54 @@
         return SpaceOperation(opname, [op.args[0]], op.result)
 
     def rewrite_op_getinteriorfield(self, op):
-        # only supports strings and unicodes
         assert len(op.args) == 3
-        assert op.args[1].value == 'chars'
         optype = op.args[0].concretetype
         if optype == lltype.Ptr(rstr.STR):
             opname = "strgetitem"
+            return SpaceOperation(opname, [op.args[0], op.args[2]], op.result)
+        elif optype == lltype.Ptr(rstr.UNICODE):
+            opname = "unicodegetitem"
+            return SpaceOperation(opname, [op.args[0], op.args[2]], op.result)
         else:
-            assert optype == lltype.Ptr(rstr.UNICODE)
-            opname = "unicodegetitem"
-        return SpaceOperation(opname, [op.args[0], op.args[2]], op.result)
+            v_inst, v_index, c_field = op.args
+            if op.result.concretetype is lltype.Void:
+                return
+            # only GcArray of Struct supported
+            assert isinstance(v_inst.concretetype.TO, lltype.GcArray)
+            STRUCT = v_inst.concretetype.TO.OF
+            assert isinstance(STRUCT, lltype.Struct)
+            descr = self.cpu.interiorfielddescrof(v_inst.concretetype.TO,
+                                                  c_field.value)
+            args = [v_inst, v_index, descr]
+            kind = getkind(op.result.concretetype)[0]
+            return SpaceOperation('getinteriorfield_gc_%s' % kind, args,
+                                  op.result)
 
     def rewrite_op_setinteriorfield(self, op):
-        # only supports strings and unicodes
         assert len(op.args) == 4
-        assert op.args[1].value == 'chars'
         optype = op.args[0].concretetype
         if optype == lltype.Ptr(rstr.STR):
             opname = "strsetitem"
+            return SpaceOperation(opname, [op.args[0], op.args[2], op.args[3]],
+                                  op.result)
+        elif optype == lltype.Ptr(rstr.UNICODE):
+            opname = "unicodesetitem"
+            return SpaceOperation(opname, [op.args[0], op.args[2], op.args[3]],
+                                  op.result)
         else:
-            assert optype == lltype.Ptr(rstr.UNICODE)
-            opname = "unicodesetitem"
-        return SpaceOperation(opname, [op.args[0], op.args[2], op.args[3]],
-                              op.result)
+            v_inst, v_index, c_field, v_value = op.args
+            if v_value.concretetype is lltype.Void:
+                return
+            # only GcArray of Struct supported
+            assert isinstance(v_inst.concretetype.TO, lltype.GcArray)
+            STRUCT = v_inst.concretetype.TO.OF
+            assert isinstance(STRUCT, lltype.Struct)
+            descr = self.cpu.interiorfielddescrof(v_inst.concretetype.TO,
+                                                  c_field.value)
+            kind = getkind(v_value.concretetype)[0]
+            args = [v_inst, v_index, v_value, descr]
+            return SpaceOperation('setinteriorfield_gc_%s' % kind, args,
+                                  op.result)
 
     def _rewrite_equality(self, op, opname):
         arg0, arg1 = op.args
@@ -821,26 +850,44 @@
         elif not float_arg and float_res:
             # some int -> some float
             ops = []
-            v1 = varoftype(lltype.Signed)
-            oplist = self.rewrite_operation(
-                SpaceOperation('force_cast', [v_arg], v1)
-            )
-            if oplist:
-                ops.extend(oplist)
+            v2 = varoftype(lltype.Float)
+            sizesign = rffi.size_and_sign(v_arg.concretetype)
+            if sizesign <= rffi.size_and_sign(lltype.Signed):
+                # cast from a type that fits in an int: either the size is
+                # smaller, or it is equal and it is not unsigned
+                v1 = varoftype(lltype.Signed)
+                oplist = self.rewrite_operation(
+                    SpaceOperation('force_cast', [v_arg], v1)
+                )
+                if oplist:
+                    ops.extend(oplist)
+                else:
+                    v1 = v_arg
+                op = self.rewrite_operation(
+                    SpaceOperation('cast_int_to_float', [v1], v2)
+                )
+                ops.append(op)
             else:
-                v1 = v_arg
-            v2 = varoftype(lltype.Float)
-            op = self.rewrite_operation(
-                SpaceOperation('cast_int_to_float', [v1], v2)
-            )
-            ops.append(op)
+                if sizesign == rffi.size_and_sign(lltype.Unsigned):
+                    opname = 'cast_uint_to_float'
+                elif sizesign == rffi.size_and_sign(lltype.SignedLongLong):
+                    opname = 'cast_longlong_to_float'
+                elif sizesign == rffi.size_and_sign(lltype.UnsignedLongLong):
+                    opname = 'cast_ulonglong_to_float'
+                else:
+                    raise AssertionError('cast_x_to_float: %r' % (sizesign,))
+                ops1 = self.rewrite_operation(
+                    SpaceOperation(opname, [v_arg], v2)
+                )
+                if not isinstance(ops1, list): ops1 = [ops1]
+                ops.extend(ops1)
             op2 = self.rewrite_operation(
                 SpaceOperation('force_cast', [v2], v_result)
             )
             if op2:
                 ops.append(op2)
             else:
-                op.result = v_result
+                ops[-1].result = v_result
             return ops
         elif float_arg and not float_res:
             # some float -> some int
@@ -853,18 +900,36 @@
                 ops.append(op1)
             else:
                 v1 = v_arg
-            v2 = varoftype(lltype.Signed)
-            op = self.rewrite_operation(
-                SpaceOperation('cast_float_to_int', [v1], v2)
-            )
-            ops.append(op)
-            oplist = self.rewrite_operation(
-                SpaceOperation('force_cast', [v2], v_result)
-            )
-            if oplist:
-                ops.extend(oplist)
+            sizesign = rffi.size_and_sign(v_result.concretetype)
+            if sizesign <= rffi.size_and_sign(lltype.Signed):
+                # cast to a type that fits in an int: either the size is
+                # smaller, or it is equal and it is not unsigned
+                v2 = varoftype(lltype.Signed)
+                op = self.rewrite_operation(
+                    SpaceOperation('cast_float_to_int', [v1], v2)
+                )
+                ops.append(op)
+                oplist = self.rewrite_operation(
+                    SpaceOperation('force_cast', [v2], v_result)
+                )
+                if oplist:
+                    ops.extend(oplist)
+                else:
+                    op.result = v_result
             else:
-                op.result = v_result
+                if sizesign == rffi.size_and_sign(lltype.Unsigned):
+                    opname = 'cast_float_to_uint'
+                elif sizesign == rffi.size_and_sign(lltype.SignedLongLong):
+                    opname = 'cast_float_to_longlong'
+                elif sizesign == rffi.size_and_sign(lltype.UnsignedLongLong):
+                    opname = 'cast_float_to_ulonglong'
+                else:
+                    raise AssertionError('cast_float_to_x: %r' % (sizesign,))
+                ops1 = self.rewrite_operation(
+                    SpaceOperation(opname, [v1], v_result)
+                )
+                if not isinstance(ops1, list): ops1 = [ops1]
+                ops.extend(ops1)
             return ops
         else:
             assert False
@@ -1070,8 +1135,6 @@
     # The new operation is optionally further processed by rewrite_operation().
     for _old, _new in [('bool_not', 'int_is_zero'),
                        ('cast_bool_to_float', 'cast_int_to_float'),
-                       ('cast_uint_to_float', 'cast_int_to_float'),
-                       ('cast_float_to_uint', 'cast_float_to_int'),
 
                        ('int_add_nonneg_ovf', 'int_add_ovf'),
                        ('keepalive', '-live-'),
diff --git a/pypy/jit/codewriter/support.py b/pypy/jit/codewriter/support.py
--- a/pypy/jit/codewriter/support.py
+++ b/pypy/jit/codewriter/support.py
@@ -13,7 +13,6 @@
 from pypy.translator.simplify import get_funcobj
 from pypy.translator.unsimplify import split_block
 from pypy.objspace.flow.model import Constant
-from pypy import conftest
 from pypy.translator.translator import TranslationContext
 from pypy.annotation.policy import AnnotatorPolicy
 from pypy.annotation import model as annmodel
@@ -48,15 +47,13 @@
     a.build_types(func, argtypes, main_entry_point=True)
     rtyper = t.buildrtyper(type_system = type_system)
     rtyper.specialize()
-    if inline:
-        auto_inlining(t, threshold=inline)
+    #if inline:
+    #    auto_inlining(t, threshold=inline)
     if backendoptimize:
         from pypy.translator.backendopt.all import backend_optimizations
         backend_optimizations(t, inline_threshold=inline or 0,
                 remove_asserts=True, really_remove_asserts=True)
 
-    #if conftest.option.view:
-    #    t.view()
     return rtyper
 
 def getgraph(func, values):
@@ -232,6 +229,17 @@
     else:
         return x
 
+def _ll_1_cast_uint_to_float(x):
+    # XXX on 32-bit platforms, this should be done using cast_longlong_to_float
+    # (which is a residual call right now in the x86 backend)
+    return llop.cast_uint_to_float(lltype.Float, x)
+
+def _ll_1_cast_float_to_uint(x):
+    # XXX on 32-bit platforms, this should be done using cast_float_to_longlong
+    # (which is a residual call right now in the x86 backend)
+    return llop.cast_float_to_uint(lltype.Unsigned, x)
+
+
 # math support
 # ------------
 
@@ -456,6 +464,8 @@
         return LLtypeHelpers._dictnext_items(lltype.Ptr(RES), iter)
     _ll_1_dictiter_nextitems.need_result_type = True
 
+    _ll_1_dict_resize = ll_rdict.ll_dict_resize
+
     # ---------- strings and unicode ----------
 
     _ll_1_str_str2unicode = ll_rstr.LLHelpers.ll_str2unicode
diff --git a/pypy/jit/codewriter/test/test_flatten.py b/pypy/jit/codewriter/test/test_flatten.py
--- a/pypy/jit/codewriter/test/test_flatten.py
+++ b/pypy/jit/codewriter/test/test_flatten.py
@@ -8,7 +8,7 @@
 from pypy.rpython.lltypesystem import lltype, rclass, rstr
 from pypy.objspace.flow.model import SpaceOperation, Variable, Constant
 from pypy.translator.unsimplify import varoftype
-from pypy.rlib.rarithmetic import ovfcheck, r_uint
+from pypy.rlib.rarithmetic import ovfcheck, r_uint, r_longlong, r_ulonglong
 from pypy.rlib.jit import dont_look_inside, _we_are_jitted, JitDriver
 from pypy.rlib.objectmodel import keepalive_until_here
 from pypy.rlib import jit
@@ -70,7 +70,8 @@
         return 'residual'
     def getcalldescr(self, op, oopspecindex=None, extraeffect=None):
         try:
-            if 'cannot_raise' in op.args[0].value._obj.graph.name:
+            name = op.args[0].value._obj._name
+            if 'cannot_raise' in name or name.startswith('cast_'):
                 return self._descr_cannot_raise
         except AttributeError:
             pass
@@ -900,6 +901,67 @@
             int_return %i4
         """, transform=True)
 
+        def f(dbl):
+            return rffi.cast(rffi.UCHAR, dbl)
+        self.encoding_test(f, [12.456], """
+            cast_float_to_int %f0 -> %i0
+            int_and %i0, $255 -> %i1
+            int_return %i1
+        """, transform=True)
+
+        def f(dbl):
+            return rffi.cast(lltype.Unsigned, dbl)
+        self.encoding_test(f, [12.456], """
+            residual_call_irf_i $<* fn cast_float_to_uint>, <Descr>, I[], R[], F[%f0] -> %i0
+            int_return %i0
+        """, transform=True)
+
+        def f(i):
+            return rffi.cast(lltype.Float, chr(i))    # "char -> float"
+        self.encoding_test(f, [12], """
+            cast_int_to_float %i0 -> %f0
+            float_return %f0
+        """, transform=True)
+
+        def f(i):
+            return rffi.cast(lltype.Float, r_uint(i))    # "uint -> float"
+        self.encoding_test(f, [12], """
+            residual_call_irf_f $<* fn cast_uint_to_float>, <Descr>, I[%i0], R[], F[] -> %f0
+            float_return %f0
+        """, transform=True)
+
+        if not longlong.is_64_bit:
+            def f(dbl):
+                return rffi.cast(lltype.SignedLongLong, dbl)
+            self.encoding_test(f, [12.3], """
+                residual_call_irf_f $<* fn llong_from_float>, <Descr>, I[], R[], F[%f0] -> %f1
+                float_return %f1
+            """, transform=True)
+
+            def f(dbl):
+                return rffi.cast(lltype.UnsignedLongLong, dbl)
+            self.encoding_test(f, [12.3], """
+                residual_call_irf_f $<* fn ullong_from_float>, <Descr>, I[], R[], F[%f0] -> %f1
+                float_return %f1
+            """, transform=True)
+
+            def f(x):
+                ll = r_longlong(x)
+                return rffi.cast(lltype.Float, ll)
+            self.encoding_test(f, [12], """
+                residual_call_irf_f $<* fn llong_from_int>, <Descr>, I[%i0], R[], F[] -> %f0
+                residual_call_irf_f $<* fn llong_to_float>, <Descr>, I[], R[], F[%f0] -> %f1
+                float_return %f1
+            """, transform=True)
+
+            def f(x):
+                ll = r_ulonglong(x)
+                return rffi.cast(lltype.Float, ll)
+            self.encoding_test(f, [12], """
+                residual_call_irf_f $<* fn ullong_from_int>, <Descr>, I[%i0], R[], F[] -> %f0
+                residual_call_irf_f $<* fn ullong_u_to_float>, <Descr>, I[], R[], F[%f0] -> %f1
+                float_return %f1
+            """, transform=True)
 
     def test_direct_ptradd(self):
         from pypy.rpython.lltypesystem import rffi
diff --git a/pypy/jit/codewriter/test/test_jtransform.py b/pypy/jit/codewriter/test/test_jtransform.py
--- a/pypy/jit/codewriter/test/test_jtransform.py
+++ b/pypy/jit/codewriter/test/test_jtransform.py
@@ -1,4 +1,3 @@
-import py
 import random
 try:
     from itertools import product
@@ -16,13 +15,13 @@
 
 from pypy.objspace.flow.model import FunctionGraph, Block, Link
 from pypy.objspace.flow.model import SpaceOperation, Variable, Constant
-from pypy.jit.codewriter.jtransform import Transformer
-from pypy.jit.metainterp.history import getkind
-from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr, rlist
+from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr
 from pypy.rpython.lltypesystem.module import ll_math
 from pypy.translator.unsimplify import varoftype
 from pypy.jit.codewriter import heaptracker, effectinfo
 from pypy.jit.codewriter.flatten import ListOfKind
+from pypy.jit.codewriter.jtransform import Transformer
+from pypy.jit.metainterp.history import getkind
 
 def const(x):
     return Constant(x, lltype.typeOf(x))
@@ -37,6 +36,8 @@
         return ('calldescr', FUNC, ARGS, RESULT)
     def fielddescrof(self, STRUCT, name):
         return ('fielddescr', STRUCT, name)
+    def interiorfielddescrof(self, ARRAY, name):
+        return ('interiorfielddescr', ARRAY, name)
     def arraydescrof(self, ARRAY):
         return FakeDescr(('arraydescr', ARRAY))
     def sizeof(self, STRUCT):
@@ -539,7 +540,7 @@
 
 def test_rename_on_links():
     v1 = Variable()
-    v2 = Variable()
+    v2 = Variable(); v2.concretetype = llmemory.Address
     v3 = Variable()
     block = Block([v1])
     block.operations = [SpaceOperation('cast_pointer', [v1], v2)]
@@ -676,6 +677,22 @@
     assert op1.args == [v, v_index]
     assert op1.result == v_result
 
+def test_dict_getinteriorfield():
+    DICT = lltype.GcArray(lltype.Struct('ENTRY', ('v', lltype.Signed),
+                                        ('k', lltype.Signed)))
+    v = varoftype(lltype.Ptr(DICT))
+    i = varoftype(lltype.Signed)
+    v_result = varoftype(lltype.Signed)
+    op = SpaceOperation('getinteriorfield', [v, i, Constant('v', lltype.Void)],
+                        v_result)
+    op1 = Transformer(FakeCPU()).rewrite_operation(op)
+    assert op1.opname == 'getinteriorfield_gc_i'
+    assert op1.args == [v, i, ('interiorfielddescr', DICT, 'v')]
+    op = SpaceOperation('getinteriorfield', [v, i, Constant('v', lltype.Void)],
+                        Constant(None, lltype.Void))
+    op1 = Transformer(FakeCPU()).rewrite_operation(op)
+    assert op1 is None
+
 def test_str_setinteriorfield():
     v = varoftype(lltype.Ptr(rstr.STR))
     v_index = varoftype(lltype.Signed)
@@ -702,6 +719,23 @@
     assert op1.args == [v, v_index, v_newchr]
     assert op1.result == v_void
 
+def test_dict_setinteriorfield():
+    DICT = lltype.GcArray(lltype.Struct('ENTRY', ('v', lltype.Signed),
+                                        ('k', lltype.Signed)))
+    v = varoftype(lltype.Ptr(DICT))
+    i = varoftype(lltype.Signed)
+    v_void = varoftype(lltype.Void)
+    op = SpaceOperation('setinteriorfield', [v, i, Constant('v', lltype.Void),
+                                             i],
+                        v_void)
+    op1 = Transformer(FakeCPU()).rewrite_operation(op)
+    assert op1.opname == 'setinteriorfield_gc_i'
+    assert op1.args == [v, i, i, ('interiorfielddescr', DICT, 'v')]
+    op = SpaceOperation('setinteriorfield', [v, i, Constant('v', lltype.Void),
+                                             v_void], v_void)
+    op1 = Transformer(FakeCPU()).rewrite_operation(op)
+    assert not op1
+
 def test_promote_1():
     v1 = varoftype(lltype.Signed)
     v2 = varoftype(lltype.Signed)
diff --git a/pypy/jit/metainterp/blackhole.py b/pypy/jit/metainterp/blackhole.py
--- a/pypy/jit/metainterp/blackhole.py
+++ b/pypy/jit/metainterp/blackhole.py
@@ -6,7 +6,6 @@
 from pypy.rlib.debug import make_sure_not_resized
 from pypy.rpython.lltypesystem import lltype, llmemory, rclass
 from pypy.rpython.lltypesystem.lloperation import llop
-from pypy.rpython.llinterp import LLException
 from pypy.jit.codewriter.jitcode import JitCode, SwitchDictDescr
 from pypy.jit.codewriter import heaptracker, longlong
 from pypy.jit.metainterp.jitexc import JitException, get_llexception, reraise
@@ -631,6 +630,9 @@
         a = longlong.getrealfloat(a)
         # note: we need to call int() twice to care for the fact that
         # int(-2147483648.0) returns a long :-(
+        # we could also call intmask() instead of the outermost int(), but
+        # it's probably better to explicitly crash (by getting a long) if a
+        # non-translated version tries to cast a too large float to an int.
         return int(int(a))
 
     @arguments("i", returns="f")
@@ -1154,6 +1156,26 @@
         array = cpu.bh_getfield_gc_r(vable, fdescr)
         return cpu.bh_arraylen_gc(adescr, array)
 
+    @arguments("cpu", "r", "i", "d", returns="i")
+    def bhimpl_getinteriorfield_gc_i(cpu, array, index, descr):
+        return cpu.bh_getinteriorfield_gc_i(array, index, descr)
+    @arguments("cpu", "r", "i", "d", returns="r")
+    def bhimpl_getinteriorfield_gc_r(cpu, array, index, descr):
+        return cpu.bh_getinteriorfield_gc_r(array, index, descr)
+    @arguments("cpu", "r", "i", "d", returns="f")
+    def bhimpl_getinteriorfield_gc_f(cpu, array, index, descr):
+        return cpu.bh_getinteriorfield_gc_f(array, index, descr)
+
+    @arguments("cpu", "r", "i", "d", "i")
+    def bhimpl_setinteriorfield_gc_i(cpu, array, index, descr, value):
+        cpu.bh_setinteriorfield_gc_i(array, index, descr, value)
+    @arguments("cpu", "r", "i", "d", "r")
+    def bhimpl_setinteriorfield_gc_r(cpu, array, index, descr, value):
+        cpu.bh_setinteriorfield_gc_r(array, index, descr, value)
+    @arguments("cpu", "r", "i", "d", "f")
+    def bhimpl_setinteriorfield_gc_f(cpu, array, index, descr, value):
+        cpu.bh_setinteriorfield_gc_f(array, index, descr, value)
+
     @arguments("cpu", "r", "d", returns="i")
     def bhimpl_getfield_gc_i(cpu, struct, fielddescr):
         return cpu.bh_getfield_gc_i(struct, fielddescr)
diff --git a/pypy/jit/metainterp/executor.py b/pypy/jit/metainterp/executor.py
--- a/pypy/jit/metainterp/executor.py
+++ b/pypy/jit/metainterp/executor.py
@@ -1,11 +1,8 @@
 """This implements pyjitpl's execution of operations.
 """
 
-import py
-from pypy.rpython.lltypesystem import lltype, llmemory, rstr
-from pypy.rpython.ootypesystem import ootype
-from pypy.rpython.lltypesystem.lloperation import llop
-from pypy.rlib.rarithmetic import ovfcheck, r_uint, intmask, r_longlong
+from pypy.rpython.lltypesystem import lltype, rstr
+from pypy.rlib.rarithmetic import ovfcheck, r_longlong
 from pypy.rlib.rtimer import read_timestamp
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.jit.metainterp.history import BoxInt, BoxPtr, BoxFloat, check_descr
@@ -123,6 +120,29 @@
     else:
         cpu.bh_setarrayitem_raw_i(arraydescr, array, index, itembox.getint())
 
+def do_getinteriorfield_gc(cpu, _, arraybox, indexbox, descr):
+    array = arraybox.getref_base()
+    index = indexbox.getint()
+    if descr.is_pointer_field():
+        return BoxPtr(cpu.bh_getinteriorfield_gc_r(array, index, descr))
+    elif descr.is_float_field():
+        return BoxFloat(cpu.bh_getinteriorfield_gc_f(array, index, descr))
+    else:
+        return BoxInt(cpu.bh_getinteriorfield_gc_i(array, index, descr))
+
+def do_setinteriorfield_gc(cpu, _, arraybox, indexbox, valuebox, descr):
+    array = arraybox.getref_base()
+    index = indexbox.getint()
+    if descr.is_pointer_field():
+        cpu.bh_setinteriorfield_gc_r(array, index, descr,
+                                     valuebox.getref_base())
+    elif descr.is_float_field():
+        cpu.bh_setinteriorfield_gc_f(array, index, descr,
+                                     valuebox.getfloatstorage())
+    else:
+        cpu.bh_setinteriorfield_gc_i(array, index, descr,
+                                     valuebox.getint())
+
 def do_getfield_gc(cpu, _, structbox, fielddescr):
     struct = structbox.getref_base()
     if fielddescr.is_pointer_field():
diff --git a/pypy/jit/metainterp/history.py b/pypy/jit/metainterp/history.py
--- a/pypy/jit/metainterp/history.py
+++ b/pypy/jit/metainterp/history.py
@@ -16,6 +16,7 @@
 INT   = 'i'
 REF   = 'r'
 FLOAT = 'f'
+STRUCT = 's'
 HOLE  = '_'
 VOID  = 'v'
 
@@ -172,6 +173,11 @@
         """
         raise NotImplementedError
 
+    def is_array_of_structs(self):
+        """ Implement for array descr
+        """
+        raise NotImplementedError
+
     def is_pointer_field(self):
         """ Implement for field descr
         """
diff --git a/pypy/jit/metainterp/optimizeopt/rewrite.py b/pypy/jit/metainterp/optimizeopt/rewrite.py
--- a/pypy/jit/metainterp/optimizeopt/rewrite.py
+++ b/pypy/jit/metainterp/optimizeopt/rewrite.py
@@ -448,6 +448,9 @@
         if v2.is_constant() and v2.box.getint() == 1:
             self.make_equal_to(op.result, v1)
             return
+        elif v1.is_constant() and v1.box.getint() == 0:
+            self.make_constant_int(op.result, 0)
+            return
         if v1.intbound.known_ge(IntBound(0, 0)) and v2.is_constant():
             val = v2.box.getint()
             if val & (val - 1) == 0 and val > 0: # val == 2**shift
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -2181,6 +2181,17 @@
         """
         self.optimize_loop(ops, expected)
 
+        ops = """
+        [i0]
+        i1 = int_floordiv(0, i0)
+        jump(i1)
+        """
+        expected = """
+        [i0]
+        jump(0)
+        """
+        self.optimize_loop(ops, expected)
+
     def test_fold_partially_constant_ops_ovf(self):
         ops = """
         [i0]
@@ -4789,6 +4800,18 @@
         """
         self.optimize_strunicode_loop(ops, expected)
 
+    def test_ptr_eq_str_constant(self):
+        ops = """
+        []
+        i0 = ptr_eq(s"abc", s"\x00")
+        finish(i0)
+        """
+        expected = """
+        []
+        finish(0)
+        """
+        self.optimize_loop(ops, expected)
+
 
 class TestLLtype(BaseTestOptimizeBasic, LLtypeMixin):
     pass
diff --git a/pypy/jit/metainterp/optimizeopt/virtualize.py b/pypy/jit/metainterp/optimizeopt/virtualize.py
--- a/pypy/jit/metainterp/optimizeopt/virtualize.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualize.py
@@ -59,7 +59,7 @@
 
     def import_from(self, other, optimizer):
         raise NotImplementedError("should not be called at this level")
-    
+
 def get_fielddescrlist_cache(cpu):
     if not hasattr(cpu, '_optimizeopt_fielddescrlist_cache'):
         result = descrlist_dict()
@@ -113,7 +113,7 @@
         #
         if not we_are_translated():
             op.name = 'FORCE ' + self.source_op.name
-            
+
         if self._is_immutable_and_filled_with_constants(optforce):
             box = optforce.optimizer.constant_fold(op)
             self.make_constant(box)
@@ -239,12 +239,12 @@
         for index in range(len(self._items)):
             self._items[index] = self._items[index].force_at_end_of_preamble(already_forced, optforce)
         return self
-    
+
     def _really_force(self, optforce):
         assert self.source_op is not None
         if not we_are_translated():
             self.source_op.name = 'FORCE ' + self.source_op.name
-        optforce.emit_operation(self.source_op)        
+        optforce.emit_operation(self.source_op)
         self.box = box = self.source_op.result
         for index in range(len(self._items)):
             subvalue = self._items[index]
@@ -276,7 +276,7 @@
 
     def new(self):
         return OptVirtualize()
-        
+
     def make_virtual(self, known_class, box, source_op=None):
         vvalue = VirtualValue(self.optimizer.cpu, known_class, box, source_op)
         self.make_equal_to(box, vvalue)
@@ -386,7 +386,8 @@
 
     def optimize_NEW_ARRAY(self, op):
         sizebox = self.get_constant_box(op.getarg(0))
-        if sizebox is not None:
+        # For now we can't make arrays of structs virtual.
+        if sizebox is not None and not op.getdescr().is_array_of_structs():
             # if the original 'op' did not have a ConstInt as argument,
             # build a new one with the ConstInt argument
             if not isinstance(op.getarg(0), ConstInt):
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -36,6 +36,7 @@
 
 
 class MIFrame(object):
+    debug = False
 
     def __init__(self, metainterp):
         self.metainterp = metainterp
@@ -548,6 +549,14 @@
     opimpl_getfield_gc_r_pure = _opimpl_getfield_gc_pure_any
     opimpl_getfield_gc_f_pure = _opimpl_getfield_gc_pure_any
 
+    @arguments("box", "box", "descr")
+    def _opimpl_getinteriorfield_gc_any(self, array, index, descr):
+        return self.execute_with_descr(rop.GETINTERIORFIELD_GC, descr,
+                                       array, index)
+    opimpl_getinteriorfield_gc_i = _opimpl_getinteriorfield_gc_any
+    opimpl_getinteriorfield_gc_f = _opimpl_getinteriorfield_gc_any
+    opimpl_getinteriorfield_gc_r = _opimpl_getinteriorfield_gc_any
+
     @specialize.arg(1)
     def _opimpl_getfield_gc_any_pureornot(self, opnum, box, fielddescr):
         tobox = self.metainterp.heapcache.getfield(box, fielddescr)
@@ -588,6 +597,15 @@
     opimpl_setfield_gc_r = _opimpl_setfield_gc_any
     opimpl_setfield_gc_f = _opimpl_setfield_gc_any
 
+    @arguments("box", "box", "box", "descr")
+    def _opimpl_setinteriorfield_gc_any(self, array, index, value, descr):
+        self.execute_with_descr(rop.SETINTERIORFIELD_GC, descr,
+                                array, index, value)
+    opimpl_setinteriorfield_gc_i = _opimpl_setinteriorfield_gc_any
+    opimpl_setinteriorfield_gc_f = _opimpl_setinteriorfield_gc_any
+    opimpl_setinteriorfield_gc_r = _opimpl_setinteriorfield_gc_any
+                                
+
     @arguments("box", "descr")
     def _opimpl_getfield_raw_any(self, box, fielddescr):
         return self.execute_with_descr(rop.GETFIELD_RAW, fielddescr, box)
@@ -2588,17 +2606,21 @@
         self.pc = position
         #
         if not we_are_translated():
-            print '\tpyjitpl: %s(%s)' % (name, ', '.join(map(repr, args))),
+            if self.debug:
+                print '\tpyjitpl: %s(%s)' % (name, ', '.join(map(repr, args))),
             try:
                 resultbox = unboundmethod(self, *args)
             except Exception, e:
-                print '-> %s!' % e.__class__.__name__
+                if self.debug:
+                    print '-> %s!' % e.__class__.__name__
                 raise
             if num_return_args == 0:
-                print
+                if self.debug:
+                    print
                 assert resultbox is None
             else:
-                print '-> %r' % (resultbox,)
+                if self.debug:
+                    print '-> %r' % (resultbox,)
                 assert argcodes[next_argcode] == '>'
                 result_argcode = argcodes[next_argcode + 1]
                 assert resultbox.type == {'i': history.INT,
diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py
--- a/pypy/jit/metainterp/resoperation.py
+++ b/pypy/jit/metainterp/resoperation.py
@@ -1,5 +1,4 @@
 from pypy.rlib.objectmodel import we_are_translated
-from pypy.rlib.debug import make_sure_not_resized
 
 def ResOperation(opnum, args, result, descr=None):
     cls = opclasses[opnum]
@@ -405,8 +404,8 @@
     'FLOAT_TRUEDIV/2',
     'FLOAT_NEG/1',
     'FLOAT_ABS/1',
-    'CAST_FLOAT_TO_INT/1',
-    'CAST_INT_TO_FLOAT/1',
+    'CAST_FLOAT_TO_INT/1',          # don't use for unsigned ints; we would
+    'CAST_INT_TO_FLOAT/1',          # need some messy code in the backend
     'CAST_FLOAT_TO_SINGLEFLOAT/1',
     'CAST_SINGLEFLOAT_TO_FLOAT/1',
     #
@@ -457,6 +456,7 @@
 
     'GETARRAYITEM_GC/2d',
     'GETARRAYITEM_RAW/2d',
+    'GETINTERIORFIELD_GC/2d',
     'GETFIELD_GC/1d',
     'GETFIELD_RAW/1d',
     '_MALLOC_FIRST',
@@ -473,6 +473,7 @@
 
     'SETARRAYITEM_GC/3d',
     'SETARRAYITEM_RAW/3d',
+    'SETINTERIORFIELD_GC/3d',
     'SETFIELD_GC/2d',
     'SETFIELD_RAW/2d',
     'STRSETITEM/3',
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -3435,7 +3435,39 @@
             return sa
         res = self.meta_interp(f, [16])
         assert res == f(16)
-        
+
+    def test_ptr_eq_str_constants(self):
+        myjitdriver = JitDriver(greens = [], reds = ["n", "x"])
+        class A(object):
+            def __init__(self, v):
+                self.v = v
+        def f(n, x):
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, x=x)
+                z = 0 / x
+                a1 = A("key")
+                a2 = A("\x00")
+                n -= [a1, a2][z].v is not a2.v
+            return n
+        res = self.meta_interp(f, [10, 1])
+        assert res == 0
+
+    def test_virtual_array_of_structs(self):
+        myjitdriver = JitDriver(greens = [], reds=["n", "d"])
+        def f(n):
+            d = None
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, d=d)
+                d = {}
+                if n % 2:
+                    d["k"] = n
+                else:
+                    d["z"] = n
+                n -= len(d)
+            return n
+        res = self.meta_interp(f, [10])
+        assert res == 0
+
 
 
 class TestLLtype(BaseLLtypeTests, LLJitMixin):
diff --git a/pypy/jit/metainterp/test/test_dict.py b/pypy/jit/metainterp/test/test_dict.py
--- a/pypy/jit/metainterp/test/test_dict.py
+++ b/pypy/jit/metainterp/test/test_dict.py
@@ -91,7 +91,7 @@
         res1 = f(100)
         res2 = self.meta_interp(f, [100], listops=True)
         assert res1 == res2
-        self.check_loops(int_mod=1) # the hash was traced
+        self.check_loops(int_mod=1) # the hash was traced and eq, but cached
 
     def test_dict_setdefault(self):
         myjitdriver = JitDriver(greens = [], reds = ['total', 'dct'])
@@ -128,7 +128,7 @@
         assert f(100) == 50
         res = self.meta_interp(f, [100], listops=True)
         assert res == 50
-        self.check_loops(int_mod=1)
+        self.check_loops(int_mod=1) # key + eq, but cached
 
     def test_repeated_lookup(self):
         myjitdriver = JitDriver(greens = [], reds = ['n', 'd'])
@@ -153,10 +153,12 @@
 
         res = self.meta_interp(f, [100], listops=True)
         assert res == f(50)
-        self.check_loops({"call": 7, "guard_false": 1, "guard_no_exception": 6,
+        self.check_loops({"call": 5, "getfield_gc": 1, "getinteriorfield_gc": 1,
+                          "guard_false": 1, "guard_no_exception": 4,
                           "guard_true": 1, "int_and": 1, "int_gt": 1,
                           "int_is_true": 1, "int_sub": 1, "jump": 1,
-                          "new_with_vtable": 1, "setfield_gc": 1})
+                          "new_with_vtable": 1, "new": 1, "new_array": 1,
+                          "setfield_gc": 3, })
 
 
 class TestOOtype(DictTests, OOJitMixin):
diff --git a/pypy/jit/metainterp/test/test_float.py b/pypy/jit/metainterp/test/test_float.py
--- a/pypy/jit/metainterp/test/test_float.py
+++ b/pypy/jit/metainterp/test/test_float.py
@@ -1,5 +1,6 @@
-import math
+import math, sys
 from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin
+from pypy.rlib.rarithmetic import intmask, r_uint
 
 
 class FloatTests:
@@ -45,6 +46,34 @@
         res = self.interp_operations(f, [-2.0])
         assert res == -8.5
 
+    def test_cast_float_to_int(self):
+        def g(f):
+            return int(f)
+        res = self.interp_operations(g, [-12345.9])
+        assert res == -12345
+
+    def test_cast_float_to_uint(self):
+        def g(f):
+            return intmask(r_uint(f))
+        res = self.interp_operations(g, [sys.maxint*2.0])
+        assert res == intmask(long(sys.maxint*2.0))
+        res = self.interp_operations(g, [-12345.9])
+        assert res == -12345
+
+    def test_cast_int_to_float(self):
+        def g(i):
+            return float(i)
+        res = self.interp_operations(g, [-12345])
+        assert type(res) is float and res == -12345.0
+
+    def test_cast_uint_to_float(self):
+        def g(i):
+            return float(r_uint(i))
+        res = self.interp_operations(g, [intmask(sys.maxint*2)])
+        assert type(res) is float and res == float(sys.maxint*2)
+        res = self.interp_operations(g, [-12345])
+        assert type(res) is float and res == float(long(r_uint(-12345)))
+
 
 class TestOOtype(FloatTests, OOJitMixin):
     pass
diff --git a/pypy/jit/metainterp/warmspot.py b/pypy/jit/metainterp/warmspot.py
--- a/pypy/jit/metainterp/warmspot.py
+++ b/pypy/jit/metainterp/warmspot.py
@@ -206,7 +206,7 @@
         self.make_enter_functions()
         self.rewrite_jit_merge_points(policy)
 
-        verbose = not self.cpu.translate_support_code
+        verbose = False # not self.cpu.translate_support_code
         self.codewriter.make_jitcodes(verbose=verbose)
         self.rewrite_can_enter_jits()
         self.rewrite_set_param()
diff --git a/pypy/module/__builtin__/app_io.py b/pypy/module/__builtin__/app_io.py
--- a/pypy/module/__builtin__/app_io.py
+++ b/pypy/module/__builtin__/app_io.py
@@ -71,7 +71,7 @@
         return line[:-1]
     return line
 
-def input(prompt=None):
+def input(prompt=''):
     """Equivalent to eval(raw_input(prompt))."""
     return eval(raw_input(prompt))
 
diff --git a/pypy/module/__builtin__/test/test_rawinput.py b/pypy/module/__builtin__/test/test_rawinput.py
--- a/pypy/module/__builtin__/test/test_rawinput.py
+++ b/pypy/module/__builtin__/test/test_rawinput.py
@@ -3,29 +3,32 @@
 
 class AppTestRawInput():
 
-    def test_raw_input(self):
+    def test_input_and_raw_input(self):
         import sys, StringIO
         for prompt, expected in [("def:", "abc/ def:/ghi\n"),
                                  ("", "abc/ /ghi\n"),
                                  (42, "abc/ 42/ghi\n"),
                                  (None, "abc/ None/ghi\n"),
                                  (Ellipsis, "abc/ /ghi\n")]:
-            save = sys.stdin, sys.stdout
-            try:
-                sys.stdin = StringIO.StringIO("foo\nbar\n")
-                out = sys.stdout = StringIO.StringIO()
-                print "abc",    # softspace = 1
-                out.write('/')
-                if prompt is Ellipsis:
-                    got = raw_input()
-                else:
-                    got = raw_input(prompt)
-                out.write('/')
-                print "ghi"
-            finally:
-                sys.stdin, sys.stdout = save
-            assert out.getvalue() == expected
-            assert got == "foo"
+            for inputfn, inputtext, gottext in [
+                    (raw_input, "foo\nbar\n", "foo"),
+                    (input, "40+2\n", 42)]:
+                save = sys.stdin, sys.stdout
+                try:
+                    sys.stdin = StringIO.StringIO(inputtext)
+                    out = sys.stdout = StringIO.StringIO()
+                    print "abc",    # softspace = 1
+                    out.write('/')
+                    if prompt is Ellipsis:
+                        got = inputfn()
+                    else:
+                        got = inputfn(prompt)
+                    out.write('/')
+                    print "ghi"
+                finally:
+                    sys.stdin, sys.stdout = save
+                assert out.getvalue() == expected
+                assert got == gottext
 
     def test_softspace(self):
         import sys
diff --git a/pypy/module/pypyjit/test_pypy_c/test_containers.py b/pypy/module/pypyjit/test_pypy_c/test_containers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_containers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_containers.py
@@ -46,7 +46,7 @@
         assert loop.match_by_id("getitem", """
             i28 = call(ConstClass(ll_dict_lookup__dicttablePtr_objectPtr_Signed), p18, p6, i25, descr=...)
             ...
-            p33 = call(ConstClass(ll_get_value__dicttablePtr_Signed), p18, i28, descr=...)
+            p33 = getinteriorfield_gc(p31, i26, descr=<InteriorFieldDescr <GcPtrFieldDescr dictentry.value .*>>)
             ...
         """)
 
diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -93,7 +93,8 @@
             i46 = call(ConstClass(ll_startswith__rpy_stringPtr_rpy_stringPtr), p28, ConstPtr(ptr45), descr=<BoolCallDescr>)
             guard_false(i46, descr=...)
             p51 = new_with_vtable(21136408)
-            setfield_gc(p51, _, descr=...)    # 6 setfields, but the order is dict-order-dependent
+            setfield_gc(p51, _, descr=...)    # 7 setfields, but the order is dict-order-dependent
+            setfield_gc(p51, _, descr=...)
             setfield_gc(p51, _, descr=...)
             setfield_gc(p51, _, descr=...)
             setfield_gc(p51, _, descr=...)
diff --git a/pypy/objspace/std/newformat.py b/pypy/objspace/std/newformat.py
--- a/pypy/objspace/std/newformat.py
+++ b/pypy/objspace/std/newformat.py
@@ -120,6 +120,8 @@
             out.append_slice(s, last_literal, end)
             return out.build()
 
+        # This is only ever called if we're already unrolling _do_build_string
+        @jit.unroll_safe
         def _parse_field(self, start, end):
             s = self.template
             # Find ":" or "!"
@@ -149,6 +151,7 @@
                 i += 1
             return s[start:end], None, end
 
+        @jit.unroll_safe
         def _get_argument(self, name):
             # First, find the argument.
             space = self.space
@@ -207,6 +210,7 @@
                     raise OperationError(space.w_IndexError, w_msg)
             return self._resolve_lookups(w_arg, name, i, end)
 
+        @jit.unroll_safe
         def _resolve_lookups(self, w_obj, name, start, end):
             # Resolve attribute and item lookups.
             space = self.space
diff --git a/pypy/rlib/rerased.py b/pypy/rlib/rerased.py
--- a/pypy/rlib/rerased.py
+++ b/pypy/rlib/rerased.py
@@ -135,6 +135,8 @@
     _about_ = erase_int
 
     def compute_result_annotation(self, s_obj):
+        config = self.bookkeeper.annotator.translator.config
+        assert config.translation.taggedpointers, "need to enable tagged pointers to use erase_int"
         assert annmodel.SomeInteger().contains(s_obj)
         return SomeErased()
 
@@ -228,6 +230,8 @@
 
     def convert_const(self, value):
         if value._identity is _identity_for_ints:
+            config = self.rtyper.annotator.translator.config
+            assert config.translation.taggedpointers, "need to enable tagged pointers to use erase_int"
             return lltype.cast_int_to_ptr(self.lowleveltype, value._x * 2 + 1)
         bk = self.rtyper.annotator.bookkeeper
         s_obj = value._identity.get_input_annotation(bk)
diff --git a/pypy/rlib/test/test_rerased.py b/pypy/rlib/test/test_rerased.py
--- a/pypy/rlib/test/test_rerased.py
+++ b/pypy/rlib/test/test_rerased.py
@@ -10,6 +10,13 @@
 
 from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin
 
+def make_annotator():
+    a = RPythonAnnotator()
+    a.translator.config.translation.taggedpointers = True
+    return a
+
+
+
 class X(object):
     pass
 
@@ -55,7 +62,7 @@
 def test_annotate_1():
     def f():
         return eraseX(X())
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [])
     assert isinstance(s, SomeErased)
 
@@ -66,7 +73,7 @@
         #assert not is_integer(e)
         x2 = uneraseX(e)
         return x2
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [])
     assert isinstance(s, annmodel.SomeInstance)
     assert s.classdef == a.bookkeeper.getuniqueclassdef(X)
@@ -77,7 +84,7 @@
         #assert is_integer(e)
         x2 = unerase_int(e)
         return x2
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [])
     assert isinstance(s, annmodel.SomeInteger)
 
@@ -105,7 +112,7 @@
         x = make(n)
         return check(x, n)
     #
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [int])
     assert isinstance(s, annmodel.SomeInteger)
 
@@ -135,7 +142,7 @@
             else:
                 return inst
     #
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [])
     assert isinstance(s, annmodel.SomeInstance)
     assert s.classdef == a.bookkeeper.getuniqueclassdef(A)
@@ -155,7 +162,7 @@
             e = e2
         return unerase(e)
     #
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [int])
     assert isinstance(s, annmodel.SomeInstance)
     assert s.classdef == a.bookkeeper.getuniqueclassdef(X)
@@ -165,11 +172,14 @@
     e1 = erase_int(42)
     def f(i):
         return unerase_int(e1)
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [int])
     assert isinstance(s, annmodel.SomeInteger)
 
 class BaseTestRErased(BaseRtypingTest):
+    def interpret(self, *args, **kwargs):
+        kwargs["taggedpointers"] = True
+        return BaseRtypingTest.interpret(self, *args, **kwargs)
     def test_rtype_1(self):
         def f():
             return eraseX(X())
diff --git a/pypy/rpython/llinterp.py b/pypy/rpython/llinterp.py
--- a/pypy/rpython/llinterp.py
+++ b/pypy/rpython/llinterp.py
@@ -1095,13 +1095,6 @@
             assert y >= 0
         return self.op_int_add_ovf(x, y)
 
-    def op_cast_float_to_int(self, f):
-        assert type(f) is float
-        try:
-            return ovfcheck(int(f))
-        except OverflowError:
-            self.make_llexception()
-
     def op_int_is_true(self, x):
         # special case
         if type(x) is CDefinedIntSymbolic:
diff --git a/pypy/rpython/lltypesystem/ll2ctypes.py b/pypy/rpython/lltypesystem/ll2ctypes.py
--- a/pypy/rpython/lltypesystem/ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/ll2ctypes.py
@@ -15,12 +15,11 @@
     load_library_kwargs = {}
 
 import os
-from pypy import conftest
 from pypy.rpython.lltypesystem import lltype, llmemory
 from pypy.rpython.extfunc import ExtRegistryEntry
 from pypy.rlib.objectmodel import Symbolic, ComputedIntSymbolic
 from pypy.tool.uid import fixid
-from pypy.rlib.rarithmetic import r_uint, r_singlefloat, r_longfloat, base_int, intmask
+from pypy.rlib.rarithmetic import r_singlefloat, r_longfloat, base_int, intmask
 from pypy.annotation import model as annmodel
 from pypy.rpython.llinterp import LLInterpreter, LLException
 from pypy.rpython.lltypesystem.rclass import OBJECT, OBJECT_VTABLE
@@ -531,6 +530,10 @@
     def __str__(self):
         return repr(self)
 
+    def _setparentstructure(self, parent, parentindex):
+        super(_parentable_mixin, self)._setparentstructure(parent, parentindex)
+        self._keepparent = parent   # always keep a strong ref
+
 class _struct_mixin(_parentable_mixin):
     """Mixin added to _struct containers when they become ctypes-based."""
     __slots__ = ()
@@ -566,7 +569,10 @@
         return 0, sys.maxint
 
     def getitem(self, index, uninitialized_ok=False):
-        return self._storage.contents._getitem(index, boundscheck=False)
+        res = self._storage.contents._getitem(index, boundscheck=False)
+        if isinstance(self._TYPE.OF, lltype.ContainerType):
+            res._obj._setparentstructure(self, index)
+        return res
 
     def setitem(self, index, value):
         self._storage.contents._setitem(index, value, boundscheck=False)
@@ -1279,6 +1285,8 @@
             self.intval = intmask(void_p.value)
 
     def __eq__(self, other):
+        if not other:
+            return self.intval == 0
         if isinstance(other, _llgcopaque):
             return self.intval == other.intval
         storage = object()
diff --git a/pypy/rpython/lltypesystem/lloperation.py b/pypy/rpython/lltypesystem/lloperation.py
--- a/pypy/rpython/lltypesystem/lloperation.py
+++ b/pypy/rpython/lltypesystem/lloperation.py
@@ -343,8 +343,8 @@
     'cast_uint_to_float':   LLOp(canfold=True),
     'cast_longlong_to_float' :LLOp(canfold=True),
     'cast_ulonglong_to_float':LLOp(canfold=True),
-    'cast_float_to_int':    LLOp(canraise=(OverflowError,), tryfold=True),
-    'cast_float_to_uint':   LLOp(canfold=True),    # XXX need OverflowError?
+    'cast_float_to_int':    LLOp(canfold=True),
+    'cast_float_to_uint':   LLOp(canfold=True),
     'cast_float_to_longlong' :LLOp(canfold=True),
     'cast_float_to_ulonglong':LLOp(canfold=True),
     'truncate_longlong_to_int':LLOp(canfold=True),
diff --git a/pypy/rpython/lltypesystem/lltype.py b/pypy/rpython/lltypesystem/lltype.py
--- a/pypy/rpython/lltypesystem/lltype.py
+++ b/pypy/rpython/lltypesystem/lltype.py
@@ -1522,7 +1522,7 @@
             and parentindex in (self._parent_type._names[0], 0)
             and self._TYPE._gckind == typeOf(parent)._gckind):
             # keep strong reference to parent, we share the same allocation
-            self._keepparent = parent 
+            self._keepparent = parent
 
     def _parentstructure(self, check=True):
         if self._wrparent is not None:
@@ -1730,7 +1730,8 @@
         # Keep the parent array alive, we share the same allocation.
         # Don't do it if we are inside a GC object, though -- it's someone
         # else's job to keep the GC object alive
-        if typeOf(top_container(parent))._gckind == 'raw':
+        if (typeOf(top_container(parent))._gckind == 'raw' or
+            hasattr(top_container(parent)._storage, 'contents')):  # ll2ctypes
             self._keepparent = parent
 
     def __str__(self):
diff --git a/pypy/rpython/lltypesystem/opimpl.py b/pypy/rpython/lltypesystem/opimpl.py
--- a/pypy/rpython/lltypesystem/opimpl.py
+++ b/pypy/rpython/lltypesystem/opimpl.py
@@ -355,6 +355,10 @@
     assert type(b) is bool
     return float(b)
 
+def op_cast_float_to_int(f):
+    assert type(f) is float
+    return intmask(int(f))
+
 def op_cast_float_to_uint(f):
     assert type(f) is float
     return r_uint(long(f))
diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py
--- a/pypy/rpython/lltypesystem/rdict.py
+++ b/pypy/rpython/lltypesystem/rdict.py
@@ -1,16 +1,14 @@
 from pypy.tool.pairtype import pairtype
-from pypy.annotation import model as annmodel
 from pypy.objspace.flow.model import Constant
-from pypy.rpython.rdict import AbstractDictRepr, AbstractDictIteratorRepr,\
-     rtype_newdict
+from pypy.rpython.rdict import (AbstractDictRepr, AbstractDictIteratorRepr,
+     rtype_newdict)
 from pypy.rpython.lltypesystem import lltype
+from pypy.rlib import objectmodel, jit
 from pypy.rlib.rarithmetic import r_uint, intmask, LONG_BIT
-from pypy.rlib.objectmodel import hlinvoke
-from pypy.rpython import robject
-from pypy.rlib import objectmodel, jit
 from pypy.rpython import rmodel
 from pypy.rpython.error import TyperError
 
+
 HIGHEST_BIT = intmask(1 << (LONG_BIT - 1))
 MASK = intmask(HIGHEST_BIT - 1)
 
@@ -417,17 +415,16 @@
     ENTRIES = lltype.typeOf(entries).TO
     return ENTRIES.fasthashfn(entries[i].key)
 
- at jit.dont_look_inside
 def ll_get_value(d, i):
     return d.entries[i].value
 
 def ll_keyhash_custom(d, key):
     DICT = lltype.typeOf(d).TO
-    return hlinvoke(DICT.r_rdict_hashfn, d.fnkeyhash, key)
+    return objectmodel.hlinvoke(DICT.r_rdict_hashfn, d.fnkeyhash, key)
 
 def ll_keyeq_custom(d, key1, key2):
     DICT = lltype.typeOf(d).TO
-    return hlinvoke(DICT.r_rdict_eqfn, d.fnkeyeq, key1, key2)
+    return objectmodel.hlinvoke(DICT.r_rdict_eqfn, d.fnkeyeq, key1, key2)
 
 def ll_dict_len(d):
     return d.num_items
@@ -448,6 +445,8 @@
     i = ll_dict_lookup(d, key, hash)
     return _ll_dict_setitem_lookup_done(d, key, value, hash, i)
 
+# Leaving as dont_look_inside ATM, it has a few branches which could lead to
+# many bridges if we don't consider their possible frequency.
 @jit.dont_look_inside
 def _ll_dict_setitem_lookup_done(d, key, value, hash, i):
     valid = (i & HIGHEST_BIT) == 0
@@ -492,6 +491,8 @@
         raise KeyError
     _ll_dict_del(d, i)
 
+# XXX: Move the size checking and resize into a single call which is opauqe to
+# the JIT to avoid extra branches.
 @jit.dont_look_inside
 def _ll_dict_del(d, i):
     d.entries.mark_deleted(i)
@@ -532,6 +533,7 @@
 # ------- a port of CPython's dictobject.c's lookdict implementation -------
 PERTURB_SHIFT = 5
 
+ at jit.dont_look_inside
 def ll_dict_lookup(d, key, hash):
     entries = d.entries
     ENTRIES = lltype.typeOf(entries).TO
@@ -621,7 +623,6 @@
     d.num_items = 0
     d.resize_counter = DICT_INITSIZE * 2
     return d
-ll_newdict.oopspec = 'newdict()'
 
 def ll_newdict_size(DICT, length_estimate):
     length_estimate = (length_estimate // 2) * 3
@@ -633,7 +634,6 @@
     d.num_items = 0
     d.resize_counter = n * 2
     return d
-ll_newdict_size.oopspec = 'newdict()'
 
 # pypy.rpython.memory.lldict uses a dict based on Struct and Array
 # instead of GcStruct and GcArray, which is done by using different
@@ -866,7 +866,6 @@
         global_popitem_index.nextindex = base + counter
     return i
 
- at jit.dont_look_inside
 def ll_popitem(ELEM, dic):
     i = _ll_getnextitem(dic)
     entry = dic.entries[i]
diff --git a/pypy/rpython/lltypesystem/test/test_ll2ctypes.py b/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
--- a/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
@@ -8,6 +8,7 @@
 from pypy.rpython.lltypesystem.ll2ctypes import uninitialized2ctypes
 from pypy.rpython.lltypesystem.ll2ctypes import ALLOCATED, force_cast
 from pypy.rpython.lltypesystem.ll2ctypes import cast_adr_to_int, get_ctypes_type
+from pypy.rpython.lltypesystem.ll2ctypes import _llgcopaque
 from pypy.rpython.annlowlevel import llhelper
 from pypy.rlib import rposix
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
@@ -1349,6 +1350,11 @@
         round = ctypes2lltype(llmemory.GCREF, lltype2ctypes(opaque.hide()))
         assert Opaque.show(round) is opaque
 
+    def test_array_of_structs(self):
+        A = lltype.GcArray(lltype.Struct('x', ('v', lltype.Signed)))
+        a = lltype.malloc(A, 5)
+        a2 = ctypes2lltype(lltype.Ptr(A), lltype2ctypes(a))
+        assert a2._obj.getitem(0)._obj._parentstructure() is a2._obj
 
 class TestPlatform(object):
     def test_lib_on_libpaths(self):
@@ -1390,3 +1396,7 @@
         f = rffi.llexternal('f', [rffi.INT, rffi.INT], rffi.INT,
                             compilation_info=eci)
         assert f(3, 4) == 7
+
+    def test_llgcopaque_eq(self):
+        assert _llgcopaque(1) != None
+        assert _llgcopaque(0) == None
diff --git a/pypy/translator/c/database.py b/pypy/translator/c/database.py
--- a/pypy/translator/c/database.py
+++ b/pypy/translator/c/database.py
@@ -176,7 +176,7 @@
                      # introduced by the GC transformer, or the type_info_table
         return node
 
-    def get(self, obj):
+    def get(self, obj, funcgen=None):
         if isinstance(obj, CConstant):
             return obj.c_name  # without further checks
         T = typeOf(obj)
@@ -228,6 +228,8 @@
                     return '((%s) %d)' % (cdecl(self.gettype(T), ''),
                                           obj._obj)
                 node = self.getcontainernode(container)
+                if node._funccodegen_owner is None:
+                    node._funccodegen_owner = funcgen
                 return node.getptrname()
             else:
                 return '((%s) NULL)' % (cdecl(self.gettype(T), ''), )
@@ -284,14 +286,16 @@
             finish_callbacks.append(('GC transformer: finished tables',
                                      self.gctransformer.get_finish_tables()))
 
-        def add_dependencies(newdependencies):
+        def add_dependencies(newdependencies, parent=None):
             for value in newdependencies:
                 #if isinstance(value, _uninitialized):
                 #    continue
                 if isinstance(typeOf(value), ContainerType):
-                    self.getcontainernode(value)
+                    node = self.getcontainernode(value)
+                    if parent and node._funccodegen_owner is not None:
+                        node._funccodegen_owner = parent._funccodegen_owner
                 else:
-                    self.get(value)
+                    self.get(value, parent and parent._funccodegen_owner)
 
         while True:
             while True:
@@ -303,7 +307,7 @@
                 if i == len(self.containerlist):
                     break
                 node = self.containerlist[i]
-                add_dependencies(node.enum_dependencies())
+                add_dependencies(node.enum_dependencies(), node)
                 i += 1
                 self.completedcontainers = i
                 if i == show_i:
diff --git a/pypy/translator/c/gc.py b/pypy/translator/c/gc.py
--- a/pypy/translator/c/gc.py
+++ b/pypy/translator/c/gc.py
@@ -170,6 +170,7 @@
     nodekind = 'refcnt rtti'
     globalcontainer = True
     typename = 'void (@)(void *)'
+    _funccodegen_owner = None
 
     def __init__(self, db, T, obj):
         assert T == RuntimeTypeInfo
@@ -266,6 +267,7 @@
     nodekind = 'boehm rtti'
     globalcontainer = True
     typename = 'char @'
+    _funccodegen_owner = None
 
     def __init__(self, db, T, obj):
         assert T == RuntimeTypeInfo
diff --git a/pypy/translator/c/gcc/test/test_asmgcroot.py b/pypy/translator/c/gcc/test/test_asmgcroot.py
--- a/pypy/translator/c/gcc/test/test_asmgcroot.py
+++ b/pypy/translator/c/gcc/test/test_asmgcroot.py
@@ -21,6 +21,7 @@
         config = get_pypy_config(translating=True)
         config.translation.gc = cls.gcpolicy
         config.translation.gcrootfinder = "asmgcc"
+        config.translation.taggedpointers = getattr(cls, "taggedpointers", False)
         return config
 
     @classmethod
diff --git a/pypy/translator/c/genc.py b/pypy/translator/c/genc.py
--- a/pypy/translator/c/genc.py
+++ b/pypy/translator/c/genc.py
@@ -682,8 +682,7 @@
     def getbasecfilefornode(self, node, basecname):
         # For FuncNode instances, use the python source filename (relative to
         # the top directory):
-        if hasattr(node.obj, 'graph'):
-            g = node.obj.graph
+        def invent_nice_name(g):
             # Lookup the filename from the function.
             # However, not all FunctionGraph objs actually have a "func":
             if hasattr(g, 'func'):
@@ -693,6 +692,15 @@
                     if pypkgpath:
                         relpypath =  localpath.relto(pypkgpath)
                         return relpypath.replace('.py', '.c')
+            return None
+        if hasattr(node.obj, 'graph'):
+            name = invent_nice_name(node.obj.graph)
+            if name is not None:
+                return name
+        elif node._funccodegen_owner is not None:
+            name = invent_nice_name(node._funccodegen_owner.graph)
+            if name is not None:
+                return "data_" + name
         return basecname
 
     def splitnodesimpl(self, basecname, nodes, nextra, nbetween,
diff --git a/pypy/translator/c/node.py b/pypy/translator/c/node.py
--- a/pypy/translator/c/node.py
+++ b/pypy/translator/c/node.py
@@ -485,6 +485,7 @@
         __slots__ = """db obj 
                        typename implementationtypename
                         name
+                        _funccodegen_owner
                         globalcontainer""".split()
     eci_name = '_compilation_info'
 
@@ -509,6 +510,7 @@
         if self.typename != self.implementationtypename:
             if db.gettypedefnode(T).extra_union_for_varlength:
                 self.name += '.b'
+        self._funccodegen_owner = None
 
     def getptrname(self):
         return '(&%s)' % self.name
@@ -842,6 +844,9 @@
         if self.funcgens:
             argnames = self.funcgens[0].argnames()  #Assume identical for all funcgens
             self.implementationtypename = self.db.gettype(self.T, argnames=argnames)
+            self._funccodegen_owner = self.funcgens[0]
+        else:
+            self._funccodegen_owner = None
 
     def basename(self):
         return self.obj._name
@@ -1005,6 +1010,7 @@
     globalcontainer = True
     typename = 'PyObject @'
     implementationtypename = 'PyObject *@'
+    _funccodegen_owner = None
 
     def __init__(self, db, T, obj):
         # obj is a _pyobject here; obj.value is the underlying CPython object
diff --git a/pypy/translator/c/primitive.py b/pypy/translator/c/primitive.py
--- a/pypy/translator/c/primitive.py
+++ b/pypy/translator/c/primitive.py
@@ -141,7 +141,12 @@
 
 def name_gcref(value, db):
     if value:
-        realobj = value._obj.container
+        obj = value._obj
+        if isinstance(obj, int):
+            # a tagged pointer
+            assert obj & 1 == 1
+            return '((%s) %d)' % (cdecl("void*", ''), obj)
+        realobj = obj.container
         realvalue = cast_opaque_ptr(Ptr(typeOf(realobj)), value)
         return db.get(realvalue)
     else:
diff --git a/pypy/translator/c/test/test_newgc.py b/pypy/translator/c/test/test_newgc.py
--- a/pypy/translator/c/test/test_newgc.py
+++ b/pypy/translator/c/test/test_newgc.py
@@ -1487,6 +1487,43 @@
         res = self.run("tagged")
         assert res == expected
 
+    def define_erased(cls):
+        from pypy.rlib import rerased
+        erase, unerase = rerased.new_erasing_pair("test")
+        class Unrelated(object):
+            pass
+
+        u = Unrelated()
+        u.tagged = True
+        u.x = rerased.erase_int(41)
+        class A(object):
+            pass
+        def fn():
+            n = 1
+            while n >= 0:
+                if u.tagged:
+                    n = rerased.unerase_int(u.x)
+                    a = A()
+                    a.n = n - 1
+                    u.x = erase(a)
+                    u.tagged = False
+                else:
+                    n = unerase(u.x).n
+                    u.x = rerased.erase_int(n - 1)
+                    u.tagged = True
+        def func():
+            rgc.collect() # check that a prebuilt erased integer doesn't explode
+            u.x = rerased.erase_int(1000)
+            u.tagged = True
+            fn()
+            return 1
+        return func
+
+    def test_erased(self):
+        expected = self.run_orig("erased")
+        res = self.run("erased")
+        assert res == expected
+
 from pypy.rlib.objectmodel import UnboxedValue
 
 class TaggedBase(object):

From noreply at buildbot.pypy.org  Mon Oct 24 21:28:11 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Mon, 24 Oct 2011 21:28:11 +0200 (CEST)
Subject: [pypy-commit] pypy default: (alex,
 armin/carl for idea and discussion, fijal review) create instance_ptr_{eq,
 ne} resops, which are for comparins instances,
 as compared to ptr_eq which is for arbitrary gc pointers. this is needed so
 that the rewrite optimizer doesn't go around trying to get the rclass of an
 rstr (which doesn't have one)
Message-ID: <20111024192811.3999A820C7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r48389:2d1620b150fa
Date: 2011-10-24 15:26 -0400
http://bitbucket.org/pypy/pypy/changeset/2d1620b150fa/

Log:	(alex, armin/carl for idea and discussion, fijal review) create
	instance_ptr_{eq,ne} resops, which are for comparins instances, as
	compared to ptr_eq which is for arbitrary gc pointers. this is
	needed so that the rewrite optimizer doesn't go around trying to get
	the rclass of an rstr (which doesn't have one)

diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1276,8 +1276,8 @@
     genop_int_ne = _cmpop("NE", "NE")
     genop_int_gt = _cmpop("G", "L")
     genop_int_ge = _cmpop("GE", "LE")
-    genop_ptr_eq = genop_int_eq
-    genop_ptr_ne = genop_int_ne
+    genop_ptr_eq = genop_instance_ptr_eq = genop_int_eq
+    genop_ptr_ne = genop_instance_ptr_ne = genop_int_ne
 
     genop_float_lt = _cmpop_float('B', 'A')
     genop_float_le = _cmpop_float('BE', 'AE')
diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py
--- a/pypy/jit/backend/x86/regalloc.py
+++ b/pypy/jit/backend/x86/regalloc.py
@@ -651,8 +651,8 @@
     consider_uint_lt = _consider_compop
     consider_uint_le = _consider_compop
     consider_uint_ge = _consider_compop
-    consider_ptr_eq = _consider_compop
-    consider_ptr_ne = _consider_compop
+    consider_ptr_eq = consider_instance_ptr_eq = _consider_compop
+    consider_ptr_ne = consider_instance_ptr_ne = _consider_compop
 
     def _consider_float_op(self, op):
         loc1 = self.xrm.loc(op.getarg(1))
diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -800,6 +800,9 @@
     def _is_gc(self, v):
         return getattr(getattr(v.concretetype, "TO", None), "_gckind", "?") == 'gc'
 
+    def _is_rclass_instance(self, v):
+        return lltype._castdepth(v.concretetype.TO, rclass.OBJECT) >= 0
+
     def _rewrite_cmp_ptrs(self, op):
         if self._is_gc(op.args[0]):
             return op
@@ -817,11 +820,21 @@
         return self._rewrite_equality(op, 'int_is_true')
 
     def rewrite_op_ptr_eq(self, op):
-        op1 = self._rewrite_equality(op, 'ptr_iszero')
+        prefix = ''
+        if self._is_rclass_instance(op.args[0]):
+            assert self._is_rclass_instance(op.args[1])
+            op = SpaceOperation('instance_ptr_eq', op.args, op.result)
+            prefix = 'instance_'
+        op1 = self._rewrite_equality(op, prefix + 'ptr_iszero')
         return self._rewrite_cmp_ptrs(op1)
 
     def rewrite_op_ptr_ne(self, op):
-        op1 = self._rewrite_equality(op, 'ptr_nonzero')
+        prefix = ''
+        if self._is_rclass_instance(op.args[0]):
+            assert self._is_rclass_instance(op.args[1])
+            op = SpaceOperation('instance_ptr_ne', op.args, op.result)
+            prefix = 'instance_'
+        op1 = self._rewrite_equality(op, prefix + 'ptr_nonzero')
         return self._rewrite_cmp_ptrs(op1)
 
     rewrite_op_ptr_iszero = _rewrite_cmp_ptrs
diff --git a/pypy/jit/codewriter/test/test_jtransform.py b/pypy/jit/codewriter/test/test_jtransform.py
--- a/pypy/jit/codewriter/test/test_jtransform.py
+++ b/pypy/jit/codewriter/test/test_jtransform.py
@@ -576,10 +576,10 @@
         assert op1.args == [v2]
 
 def test_ptr_eq():
-    v1 = varoftype(rclass.OBJECTPTR)
-    v2 = varoftype(rclass.OBJECTPTR)
+    v1 = varoftype(lltype.Ptr(rstr.STR))
+    v2 = varoftype(lltype.Ptr(rstr.STR))
     v3 = varoftype(lltype.Bool)
-    c0 = const(lltype.nullptr(rclass.OBJECT))
+    c0 = const(lltype.nullptr(rstr.STR))
     #
     for opname, reducedname in [('ptr_eq', 'ptr_iszero'),
                                 ('ptr_ne', 'ptr_nonzero')]:
@@ -598,6 +598,31 @@
         assert op1.opname == reducedname
         assert op1.args == [v2]
 
+def test_instance_ptr_eq():
+    v1 = varoftype(rclass.OBJECTPTR)
+    v2 = varoftype(rclass.OBJECTPTR)
+    v3 = varoftype(lltype.Bool)
+    c0 = const(lltype.nullptr(rclass.OBJECT))
+
+    for opname, newopname, reducedname in [
+        ('ptr_eq', 'instance_ptr_eq', 'instance_ptr_iszero'),
+        ('ptr_ne', 'instance_ptr_ne', 'instance_ptr_nonzero')
+    ]:
+        op = SpaceOperation(opname, [v1, v2], v3)
+        op1 = Transformer().rewrite_operation(op)
+        assert op1.opname == newopname
+        assert op1.args == [v1, v2]
+
+        op = SpaceOperation(opname, [v1, c0], v3)
+        op1 = Transformer().rewrite_operation(op)
+        assert op1.opname == reducedname
+        assert op1.args == [v1]
+
+        op = SpaceOperation(opname, [c0, v1], v3)
+        op1 = Transformer().rewrite_operation(op)
+        assert op1.opname == reducedname
+        assert op1.args == [v1]
+
 def test_nongc_ptr_eq():
     v1 = varoftype(rclass.NONGCOBJECTPTR)
     v2 = varoftype(rclass.NONGCOBJECTPTR)
diff --git a/pypy/jit/metainterp/blackhole.py b/pypy/jit/metainterp/blackhole.py
--- a/pypy/jit/metainterp/blackhole.py
+++ b/pypy/jit/metainterp/blackhole.py
@@ -499,6 +499,12 @@
     @arguments("r", returns="i")
     def bhimpl_ptr_nonzero(a):
         return bool(a)
+    @arguments("r", "r", returns="i")
+    def bhimpl_instance_ptr_eq(a, b):
+        return a == b
+    @arguments("r", "r", returns="i")
+    def bhimpl_instance_ptr_ne(a, b):
+        return a != b
     @arguments("r", returns="r")
     def bhimpl_cast_opaque_ptr(a):
         return a
diff --git a/pypy/jit/metainterp/optimizeopt/rewrite.py b/pypy/jit/metainterp/optimizeopt/rewrite.py
--- a/pypy/jit/metainterp/optimizeopt/rewrite.py
+++ b/pypy/jit/metainterp/optimizeopt/rewrite.py
@@ -337,7 +337,7 @@
     def optimize_INT_IS_ZERO(self, op):
         self._optimize_nullness(op, op.getarg(0), False)
 
-    def _optimize_oois_ooisnot(self, op, expect_isnot):
+    def _optimize_oois_ooisnot(self, op, expect_isnot, instance):
         value0 = self.getvalue(op.getarg(0))
         value1 = self.getvalue(op.getarg(1))
         if value0.is_virtual():
@@ -355,21 +355,28 @@
         elif value0 is value1:
             self.make_constant_int(op.result, not expect_isnot)
         else:
-            cls0 = value0.get_constant_class(self.optimizer.cpu)
-            if cls0 is not None:
-                cls1 = value1.get_constant_class(self.optimizer.cpu)
-                if cls1 is not None and not cls0.same_constant(cls1):
-                    # cannot be the same object, as we know that their
-                    # class is different
-                    self.make_constant_int(op.result, expect_isnot)
-                    return
+            if instance:
+                cls0 = value0.get_constant_class(self.optimizer.cpu)
+                if cls0 is not None:
+                    cls1 = value1.get_constant_class(self.optimizer.cpu)
+                    if cls1 is not None and not cls0.same_constant(cls1):
+                        # cannot be the same object, as we know that their
+                        # class is different
+                        self.make_constant_int(op.result, expect_isnot)
+                        return
             self.emit_operation(op)
 
+    def optimize_PTR_EQ(self, op):
+        self._optimize_oois_ooisnot(op, False, False)
+
     def optimize_PTR_NE(self, op):
-        self._optimize_oois_ooisnot(op, True)
+        self._optimize_oois_ooisnot(op, True, False)
 
-    def optimize_PTR_EQ(self, op):
-        self._optimize_oois_ooisnot(op, False)
+    def optimize_INSTANCE_PTR_EQ(self, op):
+        self._optimize_oois_ooisnot(op, False, True)
+
+    def optimize_INSTANCE_PTR_NE(self, op):
+        self._optimize_oois_ooisnot(op, True, True)
 
 ##    def optimize_INSTANCEOF(self, op):
 ##        value = self.getvalue(op.args[0])
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -508,13 +508,13 @@
         ops = """
         [p0]
         guard_class(p0, ConstClass(node_vtable)) []
-        i0 = ptr_ne(p0, NULL)
+        i0 = instance_ptr_ne(p0, NULL)
         guard_true(i0) []
-        i1 = ptr_eq(p0, NULL)
+        i1 = instance_ptr_eq(p0, NULL)
         guard_false(i1) []
-        i2 = ptr_ne(NULL, p0)
+        i2 = instance_ptr_ne(NULL, p0)
         guard_true(i0) []
-        i3 = ptr_eq(NULL, p0)
+        i3 = instance_ptr_eq(NULL, p0)
         guard_false(i1) []
         jump(p0)
         """
@@ -2026,7 +2026,7 @@
         ops = """
         [p1]
         guard_class(p1, ConstClass(node_vtable2)) []
-        i = ptr_ne(ConstPtr(myptr), p1)
+        i = instance_ptr_ne(ConstPtr(myptr), p1)
         guard_true(i) []
         jump(p1)
         """
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -2683,7 +2683,7 @@
         ops = """
         [p1]
         guard_class(p1, ConstClass(node_vtable2)) []
-        i = ptr_ne(ConstPtr(myptr), p1)
+        i = instance_ptr_ne(ConstPtr(myptr), p1)
         guard_true(i) []
         jump(p1)
         """
@@ -3331,7 +3331,7 @@
         jump(p1, i1, i2, i6)
         '''
         self.optimize_loop(ops, expected, preamble)
-        
+
 
     # ----------
 
@@ -7280,7 +7280,7 @@
         ops = """
         [p1, p2]
         setarrayitem_gc(p1, 2, 10, descr=arraydescr)
-        setarrayitem_gc(p2, 3, 13, descr=arraydescr)        
+        setarrayitem_gc(p2, 3, 13, descr=arraydescr)
         call(0, p1, p2, 0, 0, 10, descr=arraycopydescr)
         jump(p1, p2)
         """
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -165,7 +165,7 @@
         if not we_are_translated():
             for b in registers[count:]:
                 assert not oldbox.same_box(b)
-                
+
 
     def make_result_of_lastop(self, resultbox):
         got_type = resultbox.type
@@ -199,7 +199,7 @@
                     'float_add', 'float_sub', 'float_mul', 'float_truediv',
                     'float_lt', 'float_le', 'float_eq',
                     'float_ne', 'float_gt', 'float_ge',
-                    'ptr_eq', 'ptr_ne',
+                    'ptr_eq', 'ptr_ne', 'instance_ptr_eq', 'instance_ptr_ne',
                     ]:
         exec py.code.Source('''
             @arguments("box", "box")
@@ -604,7 +604,7 @@
     opimpl_setinteriorfield_gc_i = _opimpl_setinteriorfield_gc_any
     opimpl_setinteriorfield_gc_f = _opimpl_setinteriorfield_gc_any
     opimpl_setinteriorfield_gc_r = _opimpl_setinteriorfield_gc_any
-                                
+
 
     @arguments("box", "descr")
     def _opimpl_getfield_raw_any(self, box, fielddescr):
diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py
--- a/pypy/jit/metainterp/resoperation.py
+++ b/pypy/jit/metainterp/resoperation.py
@@ -437,6 +437,8 @@
     #
     'PTR_EQ/2b',
     'PTR_NE/2b',
+    'INSTANCE_PTR_EQ/2b',
+    'INSTANCE_PTR_NE/2b',
     'CAST_OPAQUE_PTR/1b',
     #
     'ARRAYLEN_GC/1d',
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -3436,7 +3436,7 @@
         res = self.meta_interp(f, [16])
         assert res == f(16)
 
-    def test_ptr_eq_str_constants(self):
+    def test_ptr_eq(self):
         myjitdriver = JitDriver(greens = [], reds = ["n", "x"])
         class A(object):
             def __init__(self, v):
@@ -3452,6 +3452,26 @@
         res = self.meta_interp(f, [10, 1])
         assert res == 0
 
+    def test_instance_ptr_eq(self):
+        myjitdriver = JitDriver(greens = [], reds = ["n", "i", "a1", "a2"])
+        class A(object):
+            pass
+        def f(n):
+            a1 = A()
+            a2 = A()
+            i = 0
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, i=i, a1=a1, a2=a2)
+                if n % 2:
+                    a = a2
+                else:
+                    a = a1
+                i += a is a1
+                n -= 1
+            return i
+        res = self.meta_interp(f, [10])
+        assert res == f(10)
+
     def test_virtual_array_of_structs(self):
         myjitdriver = JitDriver(greens = [], reds=["n", "d"])
         def f(n):

From noreply at buildbot.pypy.org  Mon Oct 24 21:28:12 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Mon, 24 Oct 2011 21:28:12 +0200 (CEST)
Subject: [pypy-commit] pypy default: merged default
Message-ID: <20111024192812.6081C820C7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r48390:4c9bd9aa3b11
Date: 2011-10-24 15:27 -0400
http://bitbucket.org/pypy/pypy/changeset/4c9bd9aa3b11/

Log:	merged default

diff --git a/pypy/pytest.ini b/pypy/pytest.ini
--- a/pypy/pytest.ini
+++ b/pypy/pytest.ini
@@ -1,2 +1,2 @@
 [pytest]
-addopts = --assertmode=old
\ No newline at end of file
+addopts = --assertmode=old -rf

From noreply at buildbot.pypy.org  Mon Oct 24 21:32:13 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Mon, 24 Oct 2011 21:32:13 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-dicts: merged in default
Message-ID: <20111024193213.08AF6820C7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: virtual-dicts
Changeset: r48391:71c19063db1e
Date: 2011-10-24 15:32 -0400
http://bitbucket.org/pypy/pypy/changeset/71c19063db1e/

Log:	merged in default

diff --git a/pypy/jit/backend/llsupport/descr.py b/pypy/jit/backend/llsupport/descr.py
--- a/pypy/jit/backend/llsupport/descr.py
+++ b/pypy/jit/backend/llsupport/descr.py
@@ -164,6 +164,7 @@
 
     _is_array_of_pointers = False      # unless overridden by GcPtrArrayDescr
     _is_array_of_floats   = False      # unless overridden by FloatArrayDescr
+    _is_array_of_structs  = False      # unless overridden by StructArrayDescr
     _is_item_signed       = False      # unless overridden by XxxArrayDescr
 
     def is_array_of_pointers(self):
@@ -172,6 +173,9 @@
     def is_array_of_floats(self):
         return self._is_array_of_floats
 
+    def is_array_of_structs(self):
+        return self._is_array_of_structs
+
     def is_item_signed(self):
         return self._is_item_signed
 
@@ -196,6 +200,10 @@
     def get_item_size(self, translate_support_code):
         return symbolic.get_size(lltype.Float, translate_support_code)
 
+class StructArrayDescr(BaseArrayDescr):
+    _clsname = 'StructArrayDescr'
+    _is_array_of_structs = True
+
 class BaseArrayNoLengthDescr(BaseArrayDescr):
     def get_base_size(self, translate_support_code):
         return 0
@@ -215,6 +223,13 @@
 def getArrayDescrClass(ARRAY):
     if ARRAY.OF is lltype.Float:
         return FloatArrayDescr
+    elif isinstance(ARRAY.OF, lltype.Struct):
+        class Descr(StructArrayDescr):
+            _clsname = '%sArrayDescr' % ARRAY.OF._name
+            def get_item_size(self, translate_support_code):
+                return symbolic.get_size(ARRAY.OF, translate_support_code)
+        Descr.__name__ = Descr._clsname
+        return Descr
     return getDescrClass(ARRAY.OF, BaseArrayDescr, GcPtrArrayDescr,
                          NonGcPtrArrayDescr, 'Array', 'get_item_size',
                          '_is_array_of_floats', '_is_item_signed')
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1276,8 +1276,8 @@
     genop_int_ne = _cmpop("NE", "NE")
     genop_int_gt = _cmpop("G", "L")
     genop_int_ge = _cmpop("GE", "LE")
-    genop_ptr_eq = genop_int_eq
-    genop_ptr_ne = genop_int_ne
+    genop_ptr_eq = genop_instance_ptr_eq = genop_int_eq
+    genop_ptr_ne = genop_instance_ptr_ne = genop_int_ne
 
     genop_float_lt = _cmpop_float('B', 'A')
     genop_float_le = _cmpop_float('BE', 'AE')
diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py
--- a/pypy/jit/backend/x86/regalloc.py
+++ b/pypy/jit/backend/x86/regalloc.py
@@ -651,8 +651,8 @@
     consider_uint_lt = _consider_compop
     consider_uint_le = _consider_compop
     consider_uint_ge = _consider_compop
-    consider_ptr_eq = _consider_compop
-    consider_ptr_ne = _consider_compop
+    consider_ptr_eq = consider_instance_ptr_eq = _consider_compop
+    consider_ptr_ne = consider_instance_ptr_ne = _consider_compop
 
     def _consider_float_op(self, op):
         loc1 = self.xrm.loc(op.getarg(1))
diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -443,6 +443,8 @@
     rewrite_op_gc_identityhash = _do_builtin_call
     rewrite_op_gc_id           = _do_builtin_call
     rewrite_op_uint_mod        = _do_builtin_call
+    rewrite_op_cast_float_to_uint = _do_builtin_call
+    rewrite_op_cast_uint_to_float = _do_builtin_call
 
     # ----------
     # getfield/setfield/mallocs etc.
@@ -798,6 +800,9 @@
     def _is_gc(self, v):
         return getattr(getattr(v.concretetype, "TO", None), "_gckind", "?") == 'gc'
 
+    def _is_rclass_instance(self, v):
+        return lltype._castdepth(v.concretetype.TO, rclass.OBJECT) >= 0
+
     def _rewrite_cmp_ptrs(self, op):
         if self._is_gc(op.args[0]):
             return op
@@ -815,11 +820,21 @@
         return self._rewrite_equality(op, 'int_is_true')
 
     def rewrite_op_ptr_eq(self, op):
-        op1 = self._rewrite_equality(op, 'ptr_iszero')
+        prefix = ''
+        if self._is_rclass_instance(op.args[0]):
+            assert self._is_rclass_instance(op.args[1])
+            op = SpaceOperation('instance_ptr_eq', op.args, op.result)
+            prefix = 'instance_'
+        op1 = self._rewrite_equality(op, prefix + 'ptr_iszero')
         return self._rewrite_cmp_ptrs(op1)
 
     def rewrite_op_ptr_ne(self, op):
-        op1 = self._rewrite_equality(op, 'ptr_nonzero')
+        prefix = ''
+        if self._is_rclass_instance(op.args[0]):
+            assert self._is_rclass_instance(op.args[1])
+            op = SpaceOperation('instance_ptr_ne', op.args, op.result)
+            prefix = 'instance_'
+        op1 = self._rewrite_equality(op, prefix + 'ptr_nonzero')
         return self._rewrite_cmp_ptrs(op1)
 
     rewrite_op_ptr_iszero = _rewrite_cmp_ptrs
@@ -848,26 +863,44 @@
         elif not float_arg and float_res:
             # some int -> some float
             ops = []
-            v1 = varoftype(lltype.Signed)
-            oplist = self.rewrite_operation(
-                SpaceOperation('force_cast', [v_arg], v1)
-            )
-            if oplist:
-                ops.extend(oplist)
+            v2 = varoftype(lltype.Float)
+            sizesign = rffi.size_and_sign(v_arg.concretetype)
+            if sizesign <= rffi.size_and_sign(lltype.Signed):
+                # cast from a type that fits in an int: either the size is
+                # smaller, or it is equal and it is not unsigned
+                v1 = varoftype(lltype.Signed)
+                oplist = self.rewrite_operation(
+                    SpaceOperation('force_cast', [v_arg], v1)
+                )
+                if oplist:
+                    ops.extend(oplist)
+                else:
+                    v1 = v_arg
+                op = self.rewrite_operation(
+                    SpaceOperation('cast_int_to_float', [v1], v2)
+                )
+                ops.append(op)
             else:
-                v1 = v_arg
-            v2 = varoftype(lltype.Float)
-            op = self.rewrite_operation(
-                SpaceOperation('cast_int_to_float', [v1], v2)
-            )
-            ops.append(op)
+                if sizesign == rffi.size_and_sign(lltype.Unsigned):
+                    opname = 'cast_uint_to_float'
+                elif sizesign == rffi.size_and_sign(lltype.SignedLongLong):
+                    opname = 'cast_longlong_to_float'
+                elif sizesign == rffi.size_and_sign(lltype.UnsignedLongLong):
+                    opname = 'cast_ulonglong_to_float'
+                else:
+                    raise AssertionError('cast_x_to_float: %r' % (sizesign,))
+                ops1 = self.rewrite_operation(
+                    SpaceOperation(opname, [v_arg], v2)
+                )
+                if not isinstance(ops1, list): ops1 = [ops1]
+                ops.extend(ops1)
             op2 = self.rewrite_operation(
                 SpaceOperation('force_cast', [v2], v_result)
             )
             if op2:
                 ops.append(op2)
             else:
-                op.result = v_result
+                ops[-1].result = v_result
             return ops
         elif float_arg and not float_res:
             # some float -> some int
@@ -880,18 +913,36 @@
                 ops.append(op1)
             else:
                 v1 = v_arg
-            v2 = varoftype(lltype.Signed)
-            op = self.rewrite_operation(
-                SpaceOperation('cast_float_to_int', [v1], v2)
-            )
-            ops.append(op)
-            oplist = self.rewrite_operation(
-                SpaceOperation('force_cast', [v2], v_result)
-            )
-            if oplist:
-                ops.extend(oplist)
+            sizesign = rffi.size_and_sign(v_result.concretetype)
+            if sizesign <= rffi.size_and_sign(lltype.Signed):
+                # cast to a type that fits in an int: either the size is
+                # smaller, or it is equal and it is not unsigned
+                v2 = varoftype(lltype.Signed)
+                op = self.rewrite_operation(
+                    SpaceOperation('cast_float_to_int', [v1], v2)
+                )
+                ops.append(op)
+                oplist = self.rewrite_operation(
+                    SpaceOperation('force_cast', [v2], v_result)
+                )
+                if oplist:
+                    ops.extend(oplist)
+                else:
+                    op.result = v_result
             else:
-                op.result = v_result
+                if sizesign == rffi.size_and_sign(lltype.Unsigned):
+                    opname = 'cast_float_to_uint'
+                elif sizesign == rffi.size_and_sign(lltype.SignedLongLong):
+                    opname = 'cast_float_to_longlong'
+                elif sizesign == rffi.size_and_sign(lltype.UnsignedLongLong):
+                    opname = 'cast_float_to_ulonglong'
+                else:
+                    raise AssertionError('cast_float_to_x: %r' % (sizesign,))
+                ops1 = self.rewrite_operation(
+                    SpaceOperation(opname, [v1], v_result)
+                )
+                if not isinstance(ops1, list): ops1 = [ops1]
+                ops.extend(ops1)
             return ops
         else:
             assert False
@@ -1097,8 +1148,6 @@
     # The new operation is optionally further processed by rewrite_operation().
     for _old, _new in [('bool_not', 'int_is_zero'),
                        ('cast_bool_to_float', 'cast_int_to_float'),
-                       ('cast_uint_to_float', 'cast_int_to_float'),
-                       ('cast_float_to_uint', 'cast_float_to_int'),
 
                        ('int_add_nonneg_ovf', 'int_add_ovf'),
                        ('keepalive', '-live-'),
diff --git a/pypy/jit/codewriter/support.py b/pypy/jit/codewriter/support.py
--- a/pypy/jit/codewriter/support.py
+++ b/pypy/jit/codewriter/support.py
@@ -229,6 +229,17 @@
     else:
         return x
 
+def _ll_1_cast_uint_to_float(x):
+    # XXX on 32-bit platforms, this should be done using cast_longlong_to_float
+    # (which is a residual call right now in the x86 backend)
+    return llop.cast_uint_to_float(lltype.Float, x)
+
+def _ll_1_cast_float_to_uint(x):
+    # XXX on 32-bit platforms, this should be done using cast_float_to_longlong
+    # (which is a residual call right now in the x86 backend)
+    return llop.cast_float_to_uint(lltype.Unsigned, x)
+
+
 # math support
 # ------------
 
diff --git a/pypy/jit/codewriter/test/test_flatten.py b/pypy/jit/codewriter/test/test_flatten.py
--- a/pypy/jit/codewriter/test/test_flatten.py
+++ b/pypy/jit/codewriter/test/test_flatten.py
@@ -8,7 +8,7 @@
 from pypy.rpython.lltypesystem import lltype, rclass, rstr
 from pypy.objspace.flow.model import SpaceOperation, Variable, Constant
 from pypy.translator.unsimplify import varoftype
-from pypy.rlib.rarithmetic import ovfcheck, r_uint
+from pypy.rlib.rarithmetic import ovfcheck, r_uint, r_longlong, r_ulonglong
 from pypy.rlib.jit import dont_look_inside, _we_are_jitted, JitDriver
 from pypy.rlib.objectmodel import keepalive_until_here
 from pypy.rlib import jit
@@ -70,7 +70,8 @@
         return 'residual'
     def getcalldescr(self, op, oopspecindex=None, extraeffect=None):
         try:
-            if 'cannot_raise' in op.args[0].value._obj.graph.name:
+            name = op.args[0].value._obj._name
+            if 'cannot_raise' in name or name.startswith('cast_'):
                 return self._descr_cannot_raise
         except AttributeError:
             pass
@@ -900,6 +901,67 @@
             int_return %i4
         """, transform=True)
 
+        def f(dbl):
+            return rffi.cast(rffi.UCHAR, dbl)
+        self.encoding_test(f, [12.456], """
+            cast_float_to_int %f0 -> %i0
+            int_and %i0, $255 -> %i1
+            int_return %i1
+        """, transform=True)
+
+        def f(dbl):
+            return rffi.cast(lltype.Unsigned, dbl)
+        self.encoding_test(f, [12.456], """
+            residual_call_irf_i $<* fn cast_float_to_uint>, <Descr>, I[], R[], F[%f0] -> %i0
+            int_return %i0
+        """, transform=True)
+
+        def f(i):
+            return rffi.cast(lltype.Float, chr(i))    # "char -> float"
+        self.encoding_test(f, [12], """
+            cast_int_to_float %i0 -> %f0
+            float_return %f0
+        """, transform=True)
+
+        def f(i):
+            return rffi.cast(lltype.Float, r_uint(i))    # "uint -> float"
+        self.encoding_test(f, [12], """
+            residual_call_irf_f $<* fn cast_uint_to_float>, <Descr>, I[%i0], R[], F[] -> %f0
+            float_return %f0
+        """, transform=True)
+
+        if not longlong.is_64_bit:
+            def f(dbl):
+                return rffi.cast(lltype.SignedLongLong, dbl)
+            self.encoding_test(f, [12.3], """
+                residual_call_irf_f $<* fn llong_from_float>, <Descr>, I[], R[], F[%f0] -> %f1
+                float_return %f1
+            """, transform=True)
+
+            def f(dbl):
+                return rffi.cast(lltype.UnsignedLongLong, dbl)
+            self.encoding_test(f, [12.3], """
+                residual_call_irf_f $<* fn ullong_from_float>, <Descr>, I[], R[], F[%f0] -> %f1
+                float_return %f1
+            """, transform=True)
+
+            def f(x):
+                ll = r_longlong(x)
+                return rffi.cast(lltype.Float, ll)
+            self.encoding_test(f, [12], """
+                residual_call_irf_f $<* fn llong_from_int>, <Descr>, I[%i0], R[], F[] -> %f0
+                residual_call_irf_f $<* fn llong_to_float>, <Descr>, I[], R[], F[%f0] -> %f1
+                float_return %f1
+            """, transform=True)
+
+            def f(x):
+                ll = r_ulonglong(x)
+                return rffi.cast(lltype.Float, ll)
+            self.encoding_test(f, [12], """
+                residual_call_irf_f $<* fn ullong_from_int>, <Descr>, I[%i0], R[], F[] -> %f0
+                residual_call_irf_f $<* fn ullong_u_to_float>, <Descr>, I[], R[], F[%f0] -> %f1
+                float_return %f1
+            """, transform=True)
 
     def test_direct_ptradd(self):
         from pypy.rpython.lltypesystem import rffi
diff --git a/pypy/jit/codewriter/test/test_jtransform.py b/pypy/jit/codewriter/test/test_jtransform.py
--- a/pypy/jit/codewriter/test/test_jtransform.py
+++ b/pypy/jit/codewriter/test/test_jtransform.py
@@ -576,10 +576,10 @@
         assert op1.args == [v2]
 
 def test_ptr_eq():
-    v1 = varoftype(rclass.OBJECTPTR)
-    v2 = varoftype(rclass.OBJECTPTR)
+    v1 = varoftype(lltype.Ptr(rstr.STR))
+    v2 = varoftype(lltype.Ptr(rstr.STR))
     v3 = varoftype(lltype.Bool)
-    c0 = const(lltype.nullptr(rclass.OBJECT))
+    c0 = const(lltype.nullptr(rstr.STR))
     #
     for opname, reducedname in [('ptr_eq', 'ptr_iszero'),
                                 ('ptr_ne', 'ptr_nonzero')]:
@@ -598,6 +598,31 @@
         assert op1.opname == reducedname
         assert op1.args == [v2]
 
+def test_instance_ptr_eq():
+    v1 = varoftype(rclass.OBJECTPTR)
+    v2 = varoftype(rclass.OBJECTPTR)
+    v3 = varoftype(lltype.Bool)
+    c0 = const(lltype.nullptr(rclass.OBJECT))
+
+    for opname, newopname, reducedname in [
+        ('ptr_eq', 'instance_ptr_eq', 'instance_ptr_iszero'),
+        ('ptr_ne', 'instance_ptr_ne', 'instance_ptr_nonzero')
+    ]:
+        op = SpaceOperation(opname, [v1, v2], v3)
+        op1 = Transformer().rewrite_operation(op)
+        assert op1.opname == newopname
+        assert op1.args == [v1, v2]
+
+        op = SpaceOperation(opname, [v1, c0], v3)
+        op1 = Transformer().rewrite_operation(op)
+        assert op1.opname == reducedname
+        assert op1.args == [v1]
+
+        op = SpaceOperation(opname, [c0, v1], v3)
+        op1 = Transformer().rewrite_operation(op)
+        assert op1.opname == reducedname
+        assert op1.args == [v1]
+
 def test_nongc_ptr_eq():
     v1 = varoftype(rclass.NONGCOBJECTPTR)
     v2 = varoftype(rclass.NONGCOBJECTPTR)
diff --git a/pypy/jit/metainterp/blackhole.py b/pypy/jit/metainterp/blackhole.py
--- a/pypy/jit/metainterp/blackhole.py
+++ b/pypy/jit/metainterp/blackhole.py
@@ -499,6 +499,12 @@
     @arguments("r", returns="i")
     def bhimpl_ptr_nonzero(a):
         return bool(a)
+    @arguments("r", "r", returns="i")
+    def bhimpl_instance_ptr_eq(a, b):
+        return a == b
+    @arguments("r", "r", returns="i")
+    def bhimpl_instance_ptr_ne(a, b):
+        return a != b
     @arguments("r", returns="r")
     def bhimpl_cast_opaque_ptr(a):
         return a
@@ -630,6 +636,9 @@
         a = longlong.getrealfloat(a)
         # note: we need to call int() twice to care for the fact that
         # int(-2147483648.0) returns a long :-(
+        # we could also call intmask() instead of the outermost int(), but
+        # it's probably better to explicitly crash (by getting a long) if a
+        # non-translated version tries to cast a too large float to an int.
         return int(int(a))
 
     @arguments("i", returns="f")
diff --git a/pypy/jit/metainterp/history.py b/pypy/jit/metainterp/history.py
--- a/pypy/jit/metainterp/history.py
+++ b/pypy/jit/metainterp/history.py
@@ -173,6 +173,11 @@
         """
         raise NotImplementedError
 
+    def is_array_of_structs(self):
+        """ Implement for array descr
+        """
+        raise NotImplementedError
+
     def is_pointer_field(self):
         """ Implement for field descr
         """
diff --git a/pypy/jit/metainterp/optimizeopt/rewrite.py b/pypy/jit/metainterp/optimizeopt/rewrite.py
--- a/pypy/jit/metainterp/optimizeopt/rewrite.py
+++ b/pypy/jit/metainterp/optimizeopt/rewrite.py
@@ -337,7 +337,7 @@
     def optimize_INT_IS_ZERO(self, op):
         self._optimize_nullness(op, op.getarg(0), False)
 
-    def _optimize_oois_ooisnot(self, op, expect_isnot):
+    def _optimize_oois_ooisnot(self, op, expect_isnot, instance):
         value0 = self.getvalue(op.getarg(0))
         value1 = self.getvalue(op.getarg(1))
         if value0.is_virtual():
@@ -355,21 +355,28 @@
         elif value0 is value1:
             self.make_constant_int(op.result, not expect_isnot)
         else:
-            cls0 = value0.get_constant_class(self.optimizer.cpu)
-            if cls0 is not None:
-                cls1 = value1.get_constant_class(self.optimizer.cpu)
-                if cls1 is not None and not cls0.same_constant(cls1):
-                    # cannot be the same object, as we know that their
-                    # class is different
-                    self.make_constant_int(op.result, expect_isnot)
-                    return
+            if instance:
+                cls0 = value0.get_constant_class(self.optimizer.cpu)
+                if cls0 is not None:
+                    cls1 = value1.get_constant_class(self.optimizer.cpu)
+                    if cls1 is not None and not cls0.same_constant(cls1):
+                        # cannot be the same object, as we know that their
+                        # class is different
+                        self.make_constant_int(op.result, expect_isnot)
+                        return
             self.emit_operation(op)
 
+    def optimize_PTR_EQ(self, op):
+        self._optimize_oois_ooisnot(op, False, False)
+
     def optimize_PTR_NE(self, op):
-        self._optimize_oois_ooisnot(op, True)
+        self._optimize_oois_ooisnot(op, True, False)
 
-    def optimize_PTR_EQ(self, op):
-        self._optimize_oois_ooisnot(op, False)
+    def optimize_INSTANCE_PTR_EQ(self, op):
+        self._optimize_oois_ooisnot(op, False, True)
+
+    def optimize_INSTANCE_PTR_NE(self, op):
+        self._optimize_oois_ooisnot(op, True, True)
 
 ##    def optimize_INSTANCEOF(self, op):
 ##        value = self.getvalue(op.args[0])
@@ -448,6 +455,9 @@
         if v2.is_constant() and v2.box.getint() == 1:
             self.make_equal_to(op.result, v1)
             return
+        elif v1.is_constant() and v1.box.getint() == 0:
+            self.make_constant_int(op.result, 0)
+            return
         if v1.intbound.known_ge(IntBound(0, 0)) and v2.is_constant():
             val = v2.box.getint()
             if val & (val - 1) == 0 and val > 0: # val == 2**shift
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -508,13 +508,13 @@
         ops = """
         [p0]
         guard_class(p0, ConstClass(node_vtable)) []
-        i0 = ptr_ne(p0, NULL)
+        i0 = instance_ptr_ne(p0, NULL)
         guard_true(i0) []
-        i1 = ptr_eq(p0, NULL)
+        i1 = instance_ptr_eq(p0, NULL)
         guard_false(i1) []
-        i2 = ptr_ne(NULL, p0)
+        i2 = instance_ptr_ne(NULL, p0)
         guard_true(i0) []
-        i3 = ptr_eq(NULL, p0)
+        i3 = instance_ptr_eq(NULL, p0)
         guard_false(i1) []
         jump(p0)
         """
@@ -2026,7 +2026,7 @@
         ops = """
         [p1]
         guard_class(p1, ConstClass(node_vtable2)) []
-        i = ptr_ne(ConstPtr(myptr), p1)
+        i = instance_ptr_ne(ConstPtr(myptr), p1)
         guard_true(i) []
         jump(p1)
         """
@@ -2181,6 +2181,17 @@
         """
         self.optimize_loop(ops, expected)
 
+        ops = """
+        [i0]
+        i1 = int_floordiv(0, i0)
+        jump(i1)
+        """
+        expected = """
+        [i0]
+        jump(0)
+        """
+        self.optimize_loop(ops, expected)
+
     def test_fold_partially_constant_ops_ovf(self):
         ops = """
         [i0]
@@ -4789,6 +4800,18 @@
         """
         self.optimize_strunicode_loop(ops, expected)
 
+    def test_ptr_eq_str_constant(self):
+        ops = """
+        []
+        i0 = ptr_eq(s"abc", s"\x00")
+        finish(i0)
+        """
+        expected = """
+        []
+        finish(0)
+        """
+        self.optimize_loop(ops, expected)
+
 
 class TestLLtype(BaseTestOptimizeBasic, LLtypeMixin):
     pass
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -2683,7 +2683,7 @@
         ops = """
         [p1]
         guard_class(p1, ConstClass(node_vtable2)) []
-        i = ptr_ne(ConstPtr(myptr), p1)
+        i = instance_ptr_ne(ConstPtr(myptr), p1)
         guard_true(i) []
         jump(p1)
         """
@@ -3331,7 +3331,7 @@
         jump(p1, i1, i2, i6)
         '''
         self.optimize_loop(ops, expected, preamble)
-        
+
 
     # ----------
 
@@ -7280,7 +7280,7 @@
         ops = """
         [p1, p2]
         setarrayitem_gc(p1, 2, 10, descr=arraydescr)
-        setarrayitem_gc(p2, 3, 13, descr=arraydescr)        
+        setarrayitem_gc(p2, 3, 13, descr=arraydescr)
         call(0, p1, p2, 0, 0, 10, descr=arraycopydescr)
         jump(p1, p2)
         """
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -165,7 +165,7 @@
         if not we_are_translated():
             for b in registers[count:]:
                 assert not oldbox.same_box(b)
-                
+
 
     def make_result_of_lastop(self, resultbox):
         got_type = resultbox.type
@@ -199,7 +199,7 @@
                     'float_add', 'float_sub', 'float_mul', 'float_truediv',
                     'float_lt', 'float_le', 'float_eq',
                     'float_ne', 'float_gt', 'float_ge',
-                    'ptr_eq', 'ptr_ne',
+                    'ptr_eq', 'ptr_ne', 'instance_ptr_eq', 'instance_ptr_ne',
                     ]:
         exec py.code.Source('''
             @arguments("box", "box")
@@ -604,7 +604,7 @@
     opimpl_setinteriorfield_gc_i = _opimpl_setinteriorfield_gc_any
     opimpl_setinteriorfield_gc_f = _opimpl_setinteriorfield_gc_any
     opimpl_setinteriorfield_gc_r = _opimpl_setinteriorfield_gc_any
-                                
+
 
     @arguments("box", "descr")
     def _opimpl_getfield_raw_any(self, box, fielddescr):
diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py
--- a/pypy/jit/metainterp/resoperation.py
+++ b/pypy/jit/metainterp/resoperation.py
@@ -404,8 +404,8 @@
     'FLOAT_TRUEDIV/2',
     'FLOAT_NEG/1',
     'FLOAT_ABS/1',
-    'CAST_FLOAT_TO_INT/1',
-    'CAST_INT_TO_FLOAT/1',
+    'CAST_FLOAT_TO_INT/1',          # don't use for unsigned ints; we would
+    'CAST_INT_TO_FLOAT/1',          # need some messy code in the backend
     'CAST_FLOAT_TO_SINGLEFLOAT/1',
     'CAST_SINGLEFLOAT_TO_FLOAT/1',
     #
@@ -437,6 +437,8 @@
     #
     'PTR_EQ/2b',
     'PTR_NE/2b',
+    'INSTANCE_PTR_EQ/2b',
+    'INSTANCE_PTR_NE/2b',
     'CAST_OPAQUE_PTR/1b',
     #
     'ARRAYLEN_GC/1d',
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -3436,6 +3436,58 @@
         res = self.meta_interp(f, [16])
         assert res == f(16)
 
+    def test_ptr_eq(self):
+        myjitdriver = JitDriver(greens = [], reds = ["n", "x"])
+        class A(object):
+            def __init__(self, v):
+                self.v = v
+        def f(n, x):
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, x=x)
+                z = 0 / x
+                a1 = A("key")
+                a2 = A("\x00")
+                n -= [a1, a2][z].v is not a2.v
+            return n
+        res = self.meta_interp(f, [10, 1])
+        assert res == 0
+
+    def test_instance_ptr_eq(self):
+        myjitdriver = JitDriver(greens = [], reds = ["n", "i", "a1", "a2"])
+        class A(object):
+            pass
+        def f(n):
+            a1 = A()
+            a2 = A()
+            i = 0
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, i=i, a1=a1, a2=a2)
+                if n % 2:
+                    a = a2
+                else:
+                    a = a1
+                i += a is a1
+                n -= 1
+            return i
+        res = self.meta_interp(f, [10])
+        assert res == f(10)
+
+    def test_virtual_array_of_structs(self):
+        myjitdriver = JitDriver(greens = [], reds=["n", "d"])
+        def f(n):
+            d = None
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, d=d)
+                d = {}
+                if n % 2:
+                    d["k"] = n
+                else:
+                    d["z"] = n
+                n -= len(d)
+            return n
+        res = self.meta_interp(f, [10])
+        assert res == 0
+
     def test_virtual_dict_constant_keys(self):
         myjitdriver = JitDriver(greens = [], reds = ["n"])
         def g(d):
@@ -3446,6 +3498,7 @@
                 myjitdriver.jit_merge_point(n=n)
                 n = g({"key": n})
             return n
+
         res = self.meta_interp(f, [10])
         assert res == 0
         self.check_loops({"int_sub": 1, "int_gt": 1, "guard_true": 1, "jump": 1})
diff --git a/pypy/jit/metainterp/test/test_float.py b/pypy/jit/metainterp/test/test_float.py
--- a/pypy/jit/metainterp/test/test_float.py
+++ b/pypy/jit/metainterp/test/test_float.py
@@ -1,5 +1,6 @@
-import math
+import math, sys
 from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin
+from pypy.rlib.rarithmetic import intmask, r_uint
 
 
 class FloatTests:
@@ -45,6 +46,34 @@
         res = self.interp_operations(f, [-2.0])
         assert res == -8.5
 
+    def test_cast_float_to_int(self):
+        def g(f):
+            return int(f)
+        res = self.interp_operations(g, [-12345.9])
+        assert res == -12345
+
+    def test_cast_float_to_uint(self):
+        def g(f):
+            return intmask(r_uint(f))
+        res = self.interp_operations(g, [sys.maxint*2.0])
+        assert res == intmask(long(sys.maxint*2.0))
+        res = self.interp_operations(g, [-12345.9])
+        assert res == -12345
+
+    def test_cast_int_to_float(self):
+        def g(i):
+            return float(i)
+        res = self.interp_operations(g, [-12345])
+        assert type(res) is float and res == -12345.0
+
+    def test_cast_uint_to_float(self):
+        def g(i):
+            return float(r_uint(i))
+        res = self.interp_operations(g, [intmask(sys.maxint*2)])
+        assert type(res) is float and res == float(sys.maxint*2)
+        res = self.interp_operations(g, [-12345])
+        assert type(res) is float and res == float(long(r_uint(-12345)))
+
 
 class TestOOtype(FloatTests, OOJitMixin):
     pass
diff --git a/pypy/pytest.ini b/pypy/pytest.ini
--- a/pypy/pytest.ini
+++ b/pypy/pytest.ini
@@ -1,2 +1,2 @@
 [pytest]
-addopts = --assertmode=old
\ No newline at end of file
+addopts = --assertmode=old -rf
diff --git a/pypy/rpython/llinterp.py b/pypy/rpython/llinterp.py
--- a/pypy/rpython/llinterp.py
+++ b/pypy/rpython/llinterp.py
@@ -1095,13 +1095,6 @@
             assert y >= 0
         return self.op_int_add_ovf(x, y)
 
-    def op_cast_float_to_int(self, f):
-        assert type(f) is float
-        try:
-            return ovfcheck(int(f))
-        except OverflowError:
-            self.make_llexception()
-
     def op_int_is_true(self, x):
         # special case
         if type(x) is CDefinedIntSymbolic:
diff --git a/pypy/rpython/lltypesystem/lloperation.py b/pypy/rpython/lltypesystem/lloperation.py
--- a/pypy/rpython/lltypesystem/lloperation.py
+++ b/pypy/rpython/lltypesystem/lloperation.py
@@ -343,8 +343,8 @@
     'cast_uint_to_float':   LLOp(canfold=True),
     'cast_longlong_to_float' :LLOp(canfold=True),
     'cast_ulonglong_to_float':LLOp(canfold=True),
-    'cast_float_to_int':    LLOp(canraise=(OverflowError,), tryfold=True),
-    'cast_float_to_uint':   LLOp(canfold=True),    # XXX need OverflowError?
+    'cast_float_to_int':    LLOp(canfold=True),
+    'cast_float_to_uint':   LLOp(canfold=True),
     'cast_float_to_longlong' :LLOp(canfold=True),
     'cast_float_to_ulonglong':LLOp(canfold=True),
     'truncate_longlong_to_int':LLOp(canfold=True),
diff --git a/pypy/rpython/lltypesystem/opimpl.py b/pypy/rpython/lltypesystem/opimpl.py
--- a/pypy/rpython/lltypesystem/opimpl.py
+++ b/pypy/rpython/lltypesystem/opimpl.py
@@ -355,6 +355,10 @@
     assert type(b) is bool
     return float(b)
 
+def op_cast_float_to_int(f):
+    assert type(f) is float
+    return intmask(int(f))
+
 def op_cast_float_to_uint(f):
     assert type(f) is float
     return r_uint(long(f))

From noreply at buildbot.pypy.org  Mon Oct 24 22:01:34 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Mon, 24 Oct 2011 22:01:34 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-dicts: progress
Message-ID: <20111024200134.BAA41820C7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: virtual-dicts
Changeset: r48392:7e285a4099fa
Date: 2011-10-24 16:01 -0400
http://bitbucket.org/pypy/pypy/changeset/7e285a4099fa/

Log:	progress

diff --git a/pypy/jit/metainterp/optimizeopt/virtualize.py b/pypy/jit/metainterp/optimizeopt/virtualize.py
--- a/pypy/jit/metainterp/optimizeopt/virtualize.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualize.py
@@ -288,6 +288,7 @@
         self._items[index][descr] = itemvalue
 
     def _really_force(self, optforce):
+        raise NotImplementedError
         assert self.source_op is not None
         if not we_are_translated():
             self.source_op.name = 'FORCE ' + self.source_op.name
@@ -308,6 +309,9 @@
                 self._items[index][descr] = self._items[index][descr].force_at_end_of_preamble(already_forced, optforce)
         return self
 
+    def _make_virtual(self, modifier):
+        return modifier.make_varraystruct(self.arraydescr)
+
 
 class OptVirtualize(optimizer.Optimization):
     "Virtualize objects until they escape."

From noreply at buildbot.pypy.org  Mon Oct 24 22:27:26 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 24 Oct 2011 22:27:26 +0200 (CEST)
Subject: [pypy-commit] pypy sse-vectorization: break test zjit. This gives
 traces roughly how I would like them.
Message-ID: <20111024202726.43BEB820C7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: sse-vectorization
Changeset: r48393:d1e954a0e84e
Date: 2011-10-24 22:26 +0200
http://bitbucket.org/pypy/pypy/changeset/d1e954a0e84e/

Log:	break test zjit. This gives traces roughly how I would like them.

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -9,7 +9,8 @@
 
 
 numpy_driver = jit.JitDriver(greens = ['signature'],
-                             reds = ['result_size', 'i', 'self', 'result'])
+                             reds = ['result_size', 'i', 'counter', 'c',
+                                     'finish', 'self', 'result'])
 all_driver = jit.JitDriver(greens=['signature'], reds=['i', 'size', 'self', 'dtype'])
 any_driver = jit.JitDriver(greens=['signature'], reds=['i', 'size', 'self', 'dtype'])
 slice_driver = jit.JitDriver(greens=['signature'], reds=['i', 'j', 'step', 'stop', 'source', 'dest'])
@@ -349,12 +350,28 @@
         signature = self.signature
         result_size = self.find_size()
         result = SingleDimArray(result_size, self.find_dtype())
-        while i < result_size:
+        finish = result_size // 4
+        counter = 0
+        c = 0
+        while counter < finish:
             numpy_driver.jit_merge_point(signature=signature,
                                          result_size=result_size, i=i,
-                                         self=self, result=result)
+                                         self=self, result=result,
+                                         counter=counter, finish=finish, c=c)
             result.dtype.setitem(result.storage, i, self.eval(i))
             i += 1
+            c += 1
+            if c == 4:
+                counter += 1
+                c = 0
+                numpy_driver.can_enter_jit(signature=signature,
+                                           result_size=result_size, i=i,
+                                           self=self, result=result,
+                                           counter=counter, finish=finish, c=c)
+        # tail
+        while i < result_size:
+            result.dtype.setitem(result.storage, i, self.eval(i))
+            i += 1            
         return result
 
     def force_if_needed(self):
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -26,11 +26,11 @@
             v = interp_ufuncs.get(self.space).add.call(self.space, [ar, ar])
             return v.get_concrete().eval(3).val
 
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
+        result = self.meta_interp(f, [20], listops=True, backendopt=True)
         self.check_loops({'getarrayitem_raw': 2, 'float_add': 1,
                           'setarrayitem_raw': 1, 'int_add': 1,
                           'int_lt': 1, 'guard_true': 1, 'jump': 1})
-        assert result == f(5)
+        assert result == f(20)
 
     def test_floatadd(self):
         def f(i):

From noreply at buildbot.pypy.org  Mon Oct 24 22:51:21 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Mon, 24 Oct 2011 22:51:21 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-dicts: small amount of progress
Message-ID: <20111024205121.19985820C7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: virtual-dicts
Changeset: r48394:c1ae43540abe
Date: 2011-10-24 16:49 -0400
http://bitbucket.org/pypy/pypy/changeset/c1ae43540abe/

Log:	small amount of progress

diff --git a/pypy/jit/metainterp/optimizeopt/virtualstate.py b/pypy/jit/metainterp/optimizeopt/virtualstate.py
--- a/pypy/jit/metainterp/optimizeopt/virtualstate.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualstate.py
@@ -193,6 +193,10 @@
     def debug_header(self, indent):
         debug_print(indent + 'VArrayStateInfo(%d):' % self.position)
             
+class VArrayStructStateInfo(AbstractVirtualStateInfo):
+    def __init__(self, arraydescr):
+        self.arraydescr = arraydescr
+
         
 class NotVirtualStateInfo(AbstractVirtualStateInfo):
     def __init__(self, value):
@@ -463,6 +467,9 @@
     def make_varray(self, arraydescr):
         return VArrayStateInfo(arraydescr)
 
+    def make_varraystruct(self, arraydescr):
+        return VArrayStructStateInfo(arraydescr)
+
 class BoxNotProducable(Exception):
     pass
 

From noreply at buildbot.pypy.org  Mon Oct 24 22:51:22 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Mon, 24 Oct 2011 22:51:22 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-dicts: whitespace
Message-ID: <20111024205122.42C09820C7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: virtual-dicts
Changeset: r48395:3f982c63e966
Date: 2011-10-24 16:50 -0400
http://bitbucket.org/pypy/pypy/changeset/3f982c63e966/

Log:	whitespace

diff --git a/pypy/jit/metainterp/optimizeopt/virtualstate.py b/pypy/jit/metainterp/optimizeopt/virtualstate.py
--- a/pypy/jit/metainterp/optimizeopt/virtualstate.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualstate.py
@@ -16,7 +16,7 @@
 
 class AbstractVirtualStateInfo(resume.AbstractVirtualInfo):
     position = -1
-    
+
     def generalization_of(self, other, renum, bad):
         raise NotImplementedError
 
@@ -54,7 +54,7 @@
                 s.debug_print(indent + "    ", seen, bad)
         else:
             debug_print(indent + "    ...")
-                
+
 
     def debug_header(self, indent):
         raise NotImplementedError
@@ -83,7 +83,7 @@
             bad[self] = True
             bad[other] = True
             return False
-        
+
         for i in range(len(self.fielddescrs)):
             if other.fielddescrs[i] is not self.fielddescrs[i]:
                 bad[self] = True
@@ -112,8 +112,8 @@
     def _enum(self, virtual_state):
         for s in self.fieldstate:
             s.enum(virtual_state)
-        
-        
+
+
 class VirtualStateInfo(AbstractVirtualStructStateInfo):
     def __init__(self, known_class, fielddescrs):
         AbstractVirtualStructStateInfo.__init__(self, fielddescrs)
@@ -128,13 +128,13 @@
 
     def debug_header(self, indent):
         debug_print(indent + 'VirtualStateInfo(%d):' % self.position)
-        
+
 class VStructStateInfo(AbstractVirtualStructStateInfo):
     def __init__(self, typedescr, fielddescrs):
         AbstractVirtualStructStateInfo.__init__(self, fielddescrs)
         self.typedescr = typedescr
 
-    def _generalization_of(self, other):        
+    def _generalization_of(self, other):
         if not isinstance(other, VStructStateInfo):
             return False
         if self.typedescr is not other.typedescr:
@@ -143,7 +143,7 @@
 
     def debug_header(self, indent):
         debug_print(indent + 'VStructStateInfo(%d):' % self.position)
-        
+
 class VArrayStateInfo(AbstractVirtualStateInfo):
     def __init__(self, arraydescr):
         self.arraydescr = arraydescr
@@ -192,12 +192,12 @@
 
     def debug_header(self, indent):
         debug_print(indent + 'VArrayStateInfo(%d):' % self.position)
-            
+
 class VArrayStructStateInfo(AbstractVirtualStateInfo):
     def __init__(self, arraydescr):
         self.arraydescr = arraydescr
 
-        
+
 class NotVirtualStateInfo(AbstractVirtualStateInfo):
     def __init__(self, value):
         self.known_class = value.known_class
@@ -281,7 +281,7 @@
             op = ResOperation(rop.GUARD_CLASS, [box, self.known_class], None)
             extra_guards.append(op)
             return
-        
+
         if self.level == LEVEL_NONNULL and \
                other.level == LEVEL_UNKNOWN and \
                isinstance(box, BoxPtr) and \
@@ -289,7 +289,7 @@
             op = ResOperation(rop.GUARD_NONNULL, [box], None)
             extra_guards.append(op)
             return
-        
+
         if self.level == LEVEL_UNKNOWN and \
                other.level == LEVEL_UNKNOWN and \
                isinstance(box, BoxInt) and \
@@ -313,7 +313,7 @@
                     op = ResOperation(rop.GUARD_TRUE, [res], None)
                     extra_guards.append(op)
             return
-        
+
         # Remaining cases are probably not interesting
         raise InvalidLoop
         if self.level == LEVEL_CONSTANT:
@@ -323,7 +323,7 @@
     def enum_forced_boxes(self, boxes, value, optimizer):
         if self.level == LEVEL_CONSTANT:
             return
-        assert 0 <= self.position_in_notvirtuals 
+        assert 0 <= self.position_in_notvirtuals
         boxes[self.position_in_notvirtuals] = value.force_box(optimizer)
 
     def _enum(self, virtual_state):
@@ -352,7 +352,7 @@
         lb = ''
         if self.lenbound:
             lb = ', ' + self.lenbound.bound.__repr__()
-        
+
         debug_print(indent + mark + 'NotVirtualInfo(%d' % self.position +
                     ', ' + l + ', ' + self.intbound.__repr__() + lb + ')')
 
@@ -374,7 +374,7 @@
                 return False
         return True
 
-    def generate_guards(self, other, args, cpu, extra_guards):        
+    def generate_guards(self, other, args, cpu, extra_guards):
         assert len(self.state) == len(other.state) == len(args)
         renum = {}
         for i in range(len(self.state)):
@@ -397,7 +397,7 @@
                     inputargs.append(box)
 
         assert None not in inputargs
-            
+
         return inputargs
 
     def debug_print(self, hdr='', bad=None):
@@ -416,7 +416,7 @@
 
     def register_virtual_fields(self, keybox, fieldboxes):
         self.fieldboxes[keybox] = fieldboxes
-        
+
     def already_seen_virtual(self, keybox):
         return keybox in self.fieldboxes
 
@@ -508,12 +508,12 @@
             else: # Low priority
                 lo -= 1
         return alts
-            
+
     def renamed(self, box):
         if box in self.rename:
             return self.rename[box]
         return box
-    
+
     def add_to_short(self, box, op):
         if op:
             op = op.clone()
@@ -535,12 +535,12 @@
             self.optimizer.make_equal_to(newbox, value)
         else:
             self.short_boxes[box] = op
-        
+
     def produce_short_preamble_box(self, box):
         if box in self.short_boxes:
-            return 
+            return
         if isinstance(box, Const):
-            return 
+            return
         if box in self.potential_ops:
             ops = self.prioritized_alternatives(box)
             produced_one = False
@@ -577,7 +577,7 @@
             else:
                 debug_print(logops.repr_of_arg(box) + ': None')
         debug_stop('jit-short-boxes')
-        
+
     def operations(self):
         if not we_are_translated(): # For tests
             ops = self.short_boxes.values()
@@ -595,7 +595,7 @@
         if not isinstance(oldbox, Const) and newbox not in self.short_boxes:
             self.short_boxes[newbox] = self.short_boxes[oldbox]
         self.aliases[newbox] = oldbox
-        
+
     def original(self, box):
         while box in self.aliases:
             box = self.aliases[box]

From noreply at buildbot.pypy.org  Mon Oct 24 23:17:54 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Mon, 24 Oct 2011 23:17:54 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-dicts: better error
Message-ID: <20111024211754.0E482820C7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: virtual-dicts
Changeset: r48396:ae6a1aa152ae
Date: 2011-10-24 17:17 -0400
http://bitbucket.org/pypy/pypy/changeset/ae6a1aa152ae/

Log:	better error

diff --git a/pypy/jit/metainterp/optimizeopt/virtualize.py b/pypy/jit/metainterp/optimizeopt/virtualize.py
--- a/pypy/jit/metainterp/optimizeopt/virtualize.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualize.py
@@ -300,6 +300,12 @@
                 op = ResOperation(rop.SETINTERIORFIELD_GC, [box, ConstInt(index), subbox], None, descr=descr)
                 optforce.emit_operation(op)
 
+    def get_args_for_fail(self, modifier):
+        if self.box is None and not modifier.already_seen_virtual(self.keybox):
+            # Need a way to store data as pairs of (index, descr), not just
+            # descr or index.
+            raise NotImplementedError
+
     def force_at_end_of_preamble(self, already_forced, optforce):
         if self in already_forced:
             return self

From notifications-noreply at bitbucket.org  Mon Oct 24 23:27:45 2011
From: notifications-noreply at bitbucket.org (Bitbucket)
Date: Mon, 24 Oct 2011 21:27:45 -0000
Subject: [pypy-commit] [COMMENT] Pull request #12 for pypy/pypy: Don't lose
 data when doing non-blocking I/O
Message-ID: <20111024212745.3129.48680@bitbucket03.managed.contegix.com>

New comment on pull request:

https://bitbucket.org/pypy/pypy/pull-request/12/dont-lose-data-when-doing-non-blocking-i-o#comment-563

Stefano Rivera (stefanor) said:

Good point, thanks: https://bugs.pypy.org/issue920

--
This is a pull request comment notification from bitbucket.org.
You are receiving this either because you are participating
in a pull request, or you are following it.

From notifications-noreply at bitbucket.org  Mon Oct 24 23:28:00 2011
From: notifications-noreply at bitbucket.org (Bitbucket)
Date: Mon, 24 Oct 2011 21:28:00 -0000
Subject: [pypy-commit] [COMMENT] Pull request #12 for pypy/pypy: Don't lose
 data when doing non-blocking I/O
Message-ID: <20111024212800.9043.20112@bitbucket03.managed.contegix.com>

New comment on pull request:

https://bitbucket.org/pypy/pypy/pull-request/12/dont-lose-data-when-doing-non-blocking-i-o#comment-564

Stefano Rivera (stefanor) said:

Solved. App tests weren't the way to go here.

--
This is a pull request comment notification from bitbucket.org.
You are receiving this either because you are participating
in a pull request, or you are following it.

From noreply at buildbot.pypy.org  Mon Oct 24 23:30:45 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Mon, 24 Oct 2011 23:30:45 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-dicts: progress I think
Message-ID: <20111024213045.BA688820C7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: virtual-dicts
Changeset: r48397:325af29a17b8
Date: 2011-10-24 17:30 -0400
http://bitbucket.org/pypy/pypy/changeset/325af29a17b8/

Log:	progress I think

diff --git a/pypy/jit/metainterp/optimizeopt/virtualize.py b/pypy/jit/metainterp/optimizeopt/virtualize.py
--- a/pypy/jit/metainterp/optimizeopt/virtualize.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualize.py
@@ -302,9 +302,16 @@
 
     def get_args_for_fail(self, modifier):
         if self.box is None and not modifier.already_seen_virtual(self.keybox):
-            # Need a way to store data as pairs of (index, descr), not just
-            # descr or index.
-            raise NotImplementedError
+            itemboxes = []
+            for items in self._items:
+                descrs = items.keys()
+                sort_descrs(descrs)
+                for descr in descrs:
+                    itemboxes.append(items[descr])
+            modifier.register_virtual_fields(self.keybox, itemboxes)
+            for item in itemboxes:
+                item.get_args_for_fail(modifier)
+
 
     def force_at_end_of_preamble(self, already_forced, optforce):
         if self in already_forced:
diff --git a/pypy/jit/metainterp/resume.py b/pypy/jit/metainterp/resume.py
--- a/pypy/jit/metainterp/resume.py
+++ b/pypy/jit/metainterp/resume.py
@@ -139,7 +139,7 @@
         self.numberings = {}
         self.cached_boxes = {}
         self.cached_virtuals = {}
-    
+
         self.nvirtuals = 0
         self.nvholes = 0
         self.nvreused = 0
@@ -273,6 +273,9 @@
     def make_varray(self, arraydescr):
         return VArrayInfo(arraydescr)
 
+    def make_varraystruct(self, arraydescr):
+        return VArrayStructInfo(arraydescr)
+
     def make_vstrplain(self, is_unicode=False):
         if is_unicode:
             return VUniPlainInfo()
@@ -402,7 +405,7 @@
                 virtuals[num] = vinfo
 
         if self._invalidation_needed(len(liveboxes), nholes):
-            memo.clear_box_virtual_numbers()           
+            memo.clear_box_virtual_numbers()
 
     def _invalidation_needed(self, nliveboxes, nholes):
         memo = self.memo
@@ -455,7 +458,7 @@
 
     def debug_prints(self):
         raise NotImplementedError
-        
+
 class AbstractVirtualStructInfo(AbstractVirtualInfo):
     def __init__(self, fielddescrs):
         self.fielddescrs = fielddescrs
@@ -537,6 +540,15 @@
         for i in self.fieldnums:
             debug_print("\t\t", str(untag(i)))
 
+class VArrayStructInfo(AbstractVirtualInfo):
+    def __init__(self, arraydescr):
+        self.arraydescr = arraydescr
+
+    def debug_prints(self):
+        debug_print("\tvarraystructinfo", self.arraydescr)
+        for i in self.fieldnums:
+            debug_print("\t\t", str(untag(i)))
+
 class VStrPlainInfo(AbstractVirtualInfo):
     """Stands for the string made out of the characters of all fieldnums."""
 

From noreply at buildbot.pypy.org  Tue Oct 25 00:36:55 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue, 25 Oct 2011 00:36:55 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-dicts: semantic fix
Message-ID: <20111024223655.DAC39820C7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: virtual-dicts
Changeset: r48398:365af1bb7686
Date: 2011-10-24 18:34 -0400
http://bitbucket.org/pypy/pypy/changeset/365af1bb7686/

Log:	semantic fix

diff --git a/pypy/jit/metainterp/optimizeopt/virtualize.py b/pypy/jit/metainterp/optimizeopt/virtualize.py
--- a/pypy/jit/metainterp/optimizeopt/virtualize.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualize.py
@@ -303,13 +303,15 @@
     def get_args_for_fail(self, modifier):
         if self.box is None and not modifier.already_seen_virtual(self.keybox):
             itemboxes = []
+            values = []
             for items in self._items:
                 descrs = items.keys()
                 sort_descrs(descrs)
                 for descr in descrs:
-                    itemboxes.append(items[descr])
+                    itemboxes.append(items[descr].get_key_box())
+                    values.append(items[descr])
             modifier.register_virtual_fields(self.keybox, itemboxes)
-            for item in itemboxes:
+            for item in values:
                 item.get_args_for_fail(modifier)
 
 

From noreply at buildbot.pypy.org  Tue Oct 25 00:36:57 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue, 25 Oct 2011 00:36:57 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-dicts: progress (these commit messages
	suck)
Message-ID: <20111024223657.0E31C820C7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: virtual-dicts
Changeset: r48399:a6e23621d2bf
Date: 2011-10-24 18:36 -0400
http://bitbucket.org/pypy/pypy/changeset/a6e23621d2bf/

Log:	progress (these commit messages suck)

diff --git a/pypy/jit/metainterp/optimizeopt/virtualstate.py b/pypy/jit/metainterp/optimizeopt/virtualstate.py
--- a/pypy/jit/metainterp/optimizeopt/virtualstate.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualstate.py
@@ -197,6 +197,10 @@
     def __init__(self, arraydescr):
         self.arraydescr = arraydescr
 
+    def _enum(self, virtual_state):
+        for s in self.fieldstate:
+            s.enum(virtual_state)
+
 
 class NotVirtualStateInfo(AbstractVirtualStateInfo):
     def __init__(self, value):

From noreply at buildbot.pypy.org  Tue Oct 25 00:59:12 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue, 25 Oct 2011 00:59:12 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix
Message-ID: <20111024225912.7534B820C7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r48400:adab424acda7
Date: 2011-10-24 18:58 -0400
http://bitbucket.org/pypy/pypy/changeset/adab424acda7/

Log:	fix

diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1297,8 +1297,8 @@
     genop_guard_int_ne = _cmpop_guard("NE", "NE", "E", "E")
     genop_guard_int_gt = _cmpop_guard("G", "L", "LE", "GE")
     genop_guard_int_ge = _cmpop_guard("GE", "LE", "L", "G")
-    genop_guard_ptr_eq = genop_guard_int_eq
-    genop_guard_ptr_ne = genop_guard_int_ne
+    genop_guard_ptr_eq = genop_guard_instance_ptr_eq = genop_guard_int_eq
+    genop_guard_ptr_ne = genop_guard_instance_ptr_ne = genop_guard_int_ne
 
     genop_guard_uint_gt = _cmpop_guard("A", "B", "BE", "AE")
     genop_guard_uint_lt = _cmpop_guard("B", "A", "AE", "BE")
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -3471,6 +3471,22 @@
             return i
         res = self.meta_interp(f, [10])
         assert res == f(10)
+        def f(n):
+            a1 = A()
+            a2 = A()
+            i = 0
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, i=i, a1=a1, a2=a2)
+                if n % 2:
+                    a = a2
+                else:
+                    a = a1
+                if a is a2:
+                    i += 1
+                n -= 1
+            return i
+        res = self.meta_interp(f, [10])
+        assert res == f(10)
 
     def test_virtual_array_of_structs(self):
         myjitdriver = JitDriver(greens = [], reds=["n", "d"])

From noreply at buildbot.pypy.org  Tue Oct 25 01:00:32 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue, 25 Oct 2011 01:00:32 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-dicts: merge default in
Message-ID: <20111024230032.3AD5E820C7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: virtual-dicts
Changeset: r48401:7a95c14349c0
Date: 2011-10-24 19:00 -0400
http://bitbucket.org/pypy/pypy/changeset/7a95c14349c0/

Log:	merge default in

diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1297,8 +1297,8 @@
     genop_guard_int_ne = _cmpop_guard("NE", "NE", "E", "E")
     genop_guard_int_gt = _cmpop_guard("G", "L", "LE", "GE")
     genop_guard_int_ge = _cmpop_guard("GE", "LE", "L", "G")
-    genop_guard_ptr_eq = genop_guard_int_eq
-    genop_guard_ptr_ne = genop_guard_int_ne
+    genop_guard_ptr_eq = genop_guard_instance_ptr_eq = genop_guard_int_eq
+    genop_guard_ptr_ne = genop_guard_instance_ptr_ne = genop_guard_int_ne
 
     genop_guard_uint_gt = _cmpop_guard("A", "B", "BE", "AE")
     genop_guard_uint_lt = _cmpop_guard("B", "A", "AE", "BE")
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -3471,6 +3471,22 @@
             return i
         res = self.meta_interp(f, [10])
         assert res == f(10)
+        def f(n):
+            a1 = A()
+            a2 = A()
+            i = 0
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, i=i, a1=a1, a2=a2)
+                if n % 2:
+                    a = a2
+                else:
+                    a = a1
+                if a is a2:
+                    i += 1
+                n -= 1
+            return i
+        res = self.meta_interp(f, [10])
+        assert res == f(10)
 
     def test_virtual_array_of_structs(self):
         myjitdriver = JitDriver(greens = [], reds=["n", "d"])

From noreply at buildbot.pypy.org  Tue Oct 25 01:21:31 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue, 25 Oct 2011 01:21:31 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-dicts: we now have a representation of a
 list of fielddescrs,
 I don't understand what generalization_of is supposed to do.
Message-ID: <20111024232131.103BA820C7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: virtual-dicts
Changeset: r48402:068b20a399e7
Date: 2011-10-24 19:21 -0400
http://bitbucket.org/pypy/pypy/changeset/068b20a399e7/

Log:	we now have a representation of a list of fielddescrs, I don't
	understand what generalization_of is supposed to do.

diff --git a/pypy/jit/metainterp/optimizeopt/virtualize.py b/pypy/jit/metainterp/optimizeopt/virtualize.py
--- a/pypy/jit/metainterp/optimizeopt/virtualize.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualize.py
@@ -288,7 +288,6 @@
         self._items[index][descr] = itemvalue
 
     def _really_force(self, optforce):
-        raise NotImplementedError
         assert self.source_op is not None
         if not we_are_translated():
             self.source_op.name = 'FORCE ' + self.source_op.name
@@ -300,20 +299,25 @@
                 op = ResOperation(rop.SETINTERIORFIELD_GC, [box, ConstInt(index), subbox], None, descr=descr)
                 optforce.emit_operation(op)
 
+    def _get_list_of_descrs(self):
+        descrs = []
+        for item in self._items:
+            item_descrs = item.keys()
+            sort_descrs(item_descrs)
+            descrs.append(item_descrs)
+        return descrs
+
     def get_args_for_fail(self, modifier):
         if self.box is None and not modifier.already_seen_virtual(self.keybox):
+            itemdescrs = self._get_list_of_descrs()
             itemboxes = []
-            values = []
-            for items in self._items:
-                descrs = items.keys()
-                sort_descrs(descrs)
-                for descr in descrs:
-                    itemboxes.append(items[descr].get_key_box())
-                    values.append(items[descr])
+            for i in range(len(self._items)):
+                for descr in itemdescrs[i]:
+                    itemboxes.append(self._items[i][descr].get_key_box())
             modifier.register_virtual_fields(self.keybox, itemboxes)
-            for item in values:
-                item.get_args_for_fail(modifier)
-
+            for i in range(len(self._items)):
+                for descr in itemdescrs[i]:
+                    self._items[i][descr].get_args_for_fail(modifier)
 
     def force_at_end_of_preamble(self, already_forced, optforce):
         if self in already_forced:
@@ -325,7 +329,7 @@
         return self
 
     def _make_virtual(self, modifier):
-        return modifier.make_varraystruct(self.arraydescr)
+        return modifier.make_varraystruct(self.arraydescr, self._get_list_of_descrs())
 
 
 class OptVirtualize(optimizer.Optimization):
diff --git a/pypy/jit/metainterp/optimizeopt/virtualstate.py b/pypy/jit/metainterp/optimizeopt/virtualstate.py
--- a/pypy/jit/metainterp/optimizeopt/virtualstate.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualstate.py
@@ -194,13 +194,29 @@
         debug_print(indent + 'VArrayStateInfo(%d):' % self.position)
 
 class VArrayStructStateInfo(AbstractVirtualStateInfo):
-    def __init__(self, arraydescr):
+    def __init__(self, arraydescr, fielddescrs):
         self.arraydescr = arraydescr
+        self.fielddescrs = fielddescrs
 
     def _enum(self, virtual_state):
         for s in self.fieldstate:
             s.enum(virtual_state)
 
+    def enum_forced_boxes(self, boxes, value, optimizer):
+        assert isinstance(value, virtualize.VArrayStructValue)
+        assert value.is_virtual()
+        p = 0
+        for i in range(len(self.fielddescrs)):
+            for j in range(len(self.fielddescrs[i])):
+                v = value._items[i][self.fielddescrs[i][j]]
+                s = self.fieldstate[p]
+                if s.position > self.position:
+                    s.enum_forced_boxes(boxes, v, optimizer)
+                p += 1
+
+    def debug_header(self, indent):
+        debug_print(indent + 'VArrayStructStateInfo(%d):' % self.position)
+
 
 class NotVirtualStateInfo(AbstractVirtualStateInfo):
     def __init__(self, value):
@@ -471,8 +487,8 @@
     def make_varray(self, arraydescr):
         return VArrayStateInfo(arraydescr)
 
-    def make_varraystruct(self, arraydescr):
-        return VArrayStructStateInfo(arraydescr)
+    def make_varraystruct(self, arraydescr, fielddescrs):
+        return VArrayStructStateInfo(arraydescr, fielddescrs)
 
 class BoxNotProducable(Exception):
     pass
diff --git a/pypy/jit/metainterp/resume.py b/pypy/jit/metainterp/resume.py
--- a/pypy/jit/metainterp/resume.py
+++ b/pypy/jit/metainterp/resume.py
@@ -273,8 +273,8 @@
     def make_varray(self, arraydescr):
         return VArrayInfo(arraydescr)
 
-    def make_varraystruct(self, arraydescr):
-        return VArrayStructInfo(arraydescr)
+    def make_varraystruct(self, arraydescr, fielddescrs):
+        return VArrayStructInfo(arraydescr, fielddescrs)
 
     def make_vstrplain(self, is_unicode=False):
         if is_unicode:
@@ -541,8 +541,9 @@
             debug_print("\t\t", str(untag(i)))
 
 class VArrayStructInfo(AbstractVirtualInfo):
-    def __init__(self, arraydescr):
+    def __init__(self, arraydescr, fielddescrs):
         self.arraydescr = arraydescr
+        self.fielddescrs = fielddescrs
 
     def debug_prints(self):
         debug_print("\tvarraystructinfo", self.arraydescr)

From noreply at buildbot.pypy.org  Tue Oct 25 03:18:47 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue, 25 Oct 2011 03:18:47 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-dicts: it... all seems to work
Message-ID: <20111025011847.6F56B820C7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: virtual-dicts
Changeset: r48403:912afce3954e
Date: 2011-10-24 21:18 -0400
http://bitbucket.org/pypy/pypy/changeset/912afce3954e/

Log:	it... all seems to work

diff --git a/pypy/jit/metainterp/optimizeopt/virtualstate.py b/pypy/jit/metainterp/optimizeopt/virtualstate.py
--- a/pypy/jit/metainterp/optimizeopt/virtualstate.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualstate.py
@@ -157,11 +157,7 @@
             bad[other] = True
             return False
         renum[self.position] = other.position
-        if not isinstance(other, VArrayStateInfo):
-            bad[self] = True
-            bad[other] = True
-            return False
-        if self.arraydescr is not other.arraydescr:
+        if not self._generalization_of(other):
             bad[self] = True
             bad[other] = True
             return False
@@ -177,6 +173,10 @@
                 return False
         return True
 
+    def _generalization_of(self, other):
+        return (isinstance(other, VArrayStateInfo) and
+            self.arraydescr is other.arraydescr)
+
     def enum_forced_boxes(self, boxes, value, optimizer):
         assert isinstance(value, virtualize.VArrayValue)
         assert value.is_virtual()
@@ -198,6 +198,48 @@
         self.arraydescr = arraydescr
         self.fielddescrs = fielddescrs
 
+    def generalization_of(self, other, renum, bad):
+        assert self.position != -1
+        if self.position in renum:
+            if renum[self.position] == other.position:
+                return True
+            bad[self] = True
+            bad[other] = True
+            return False
+        renum[self.position] = other.position
+        if not self._generalization_of(other):
+            bad[self] = True
+            bad[other] = True
+            return False
+
+        if len(self.fielddescrs) != len(other.fielddescrs):
+            bad[self] = True
+            bad[other] = True
+            return False
+
+        p = 0
+        for i in range(len(self.fielddescrs)):
+            if len(self.fielddescrs[i]) != len(other.fielddescrs[i]):
+                bad[self] = True
+                bad[other] = True
+                return False
+            for j in range(len(self.fielddescrs[i])):
+                if self.fielddescrs[i][j] is not other.fielddescrs[i][j]:
+                    bad[self] = True
+                    bad[other] = True
+                    return False
+                if not self.fieldstate[p].generalization_of(other.fieldstate[p],
+                                                            renum, bad):
+                    bad[self] = True
+                    bad[other] = True
+                    return False
+                p += 1
+        return True
+
+    def _generalization_of(self, other):
+        return (isinstance(other, VArrayStructStateInfo) and
+            self.arraydescr is other.arraydescr)
+
     def _enum(self, virtual_state):
         for s in self.fieldstate:
             s.enum(virtual_state)
diff --git a/pypy/jit/metainterp/resume.py b/pypy/jit/metainterp/resume.py
--- a/pypy/jit/metainterp/resume.py
+++ b/pypy/jit/metainterp/resume.py
@@ -540,6 +540,7 @@
         for i in self.fieldnums:
             debug_print("\t\t", str(untag(i)))
 
+
 class VArrayStructInfo(AbstractVirtualInfo):
     def __init__(self, arraydescr, fielddescrs):
         self.arraydescr = arraydescr
@@ -550,6 +551,18 @@
         for i in self.fieldnums:
             debug_print("\t\t", str(untag(i)))
 
+    @specialize.argtype(1)
+    def allocate(self, decoder, index):
+        array = decoder.allocate_array(self.arraydescr, len(self.fielddescrs))
+        decoder.virtuals_cache[index] = array
+        p = 0
+        for i in range(len(self.fielddescrs)):
+            for j in range(len(self.fielddescrs[i])):
+                decoder.setinteriorfield(i, self.fielddescrs[i][j], array, self.fieldnums[p])
+                p += 1
+        return array
+
+
 class VStrPlainInfo(AbstractVirtualInfo):
     """Stands for the string made out of the characters of all fieldnums."""
 
@@ -897,6 +910,17 @@
         self.metainterp.execute_and_record(rop.SETFIELD_GC, descr,
                                            structbox, fieldbox)
 
+    def setinteriorfield(self, index, descr, array, fieldnum):
+        if descr.is_pointer_field():
+            kind = REF
+        elif descr.is_float_field():
+            kind = FLOAT
+        else:
+            kind = INT
+        fieldbox = self.decode_box(fieldnum, kind)
+        self.metainterp.execute_and_record(rop.SETINTERIORFIELD_GC, descr,
+                                           array, ConstInt(index), fieldbox)
+
     def setarrayitem_int(self, arraydescr, arraybox, index, fieldnum):
         self._setarrayitem(arraydescr, arraybox, index, fieldnum, INT)
 
@@ -1177,6 +1201,17 @@
             newvalue = self.decode_int(fieldnum)
             self.cpu.bh_setfield_gc_i(struct, descr, newvalue)
 
+    def setinteriorfield(self, index, descr, array, fieldnum):
+        if descr.is_pointer_field():
+            newvalue = self.decode_ref(fieldnum)
+            self.cpu.bh_setinteriorfield_gc_r(array, index, descr, newvalue)
+        elif descr.is_float_field():
+            newvalue = self.decode_float(fieldnum)
+            self.cpu.bh_setinteriorfield_gc_f(array, index, descr, newvalue)
+        else:
+            newvalue = self.decode_int(fieldnum)
+            self.cpu.bh_setinteriorfield_gc_i(array, index, descr, newvalue)
+
     def setarrayitem_int(self, arraydescr, array, index, fieldnum):
         newvalue = self.decode_int(fieldnum)
         self.cpu.bh_setarrayitem_gc_i(arraydescr, array, index, newvalue)
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -3494,12 +3494,12 @@
             d = None
             while n > 0:
                 myjitdriver.jit_merge_point(n=n, d=d)
-                d = {}
+                d = {"q": 1}
                 if n % 2:
                     d["k"] = n
                 else:
                     d["z"] = n
-                n -= len(d)
+                n -= len(d) - d["q"]
             return n
         res = self.meta_interp(f, [10])
         assert res == 0

From noreply at buildbot.pypy.org  Tue Oct 25 03:35:47 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue, 25 Oct 2011 03:35:47 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-dicts: fix translation
Message-ID: <20111025013547.7038B820C7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: virtual-dicts
Changeset: r48404:8cae4e3f42e9
Date: 2011-10-24 21:35 -0400
http://bitbucket.org/pypy/pypy/changeset/8cae4e3f42e9/

Log:	fix translation

diff --git a/pypy/jit/metainterp/optimizeopt/virtualstate.py b/pypy/jit/metainterp/optimizeopt/virtualstate.py
--- a/pypy/jit/metainterp/optimizeopt/virtualstate.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualstate.py
@@ -77,6 +77,8 @@
             bad[self] = True
             bad[other] = True
             return False
+
+        assert isinstance(other, AbstractVirtualStructStateInfo)
         assert len(self.fielddescrs) == len(self.fieldstate)
         assert len(other.fielddescrs) == len(other.fieldstate)
         if len(self.fielddescrs) != len(other.fielddescrs):
@@ -212,6 +214,7 @@
             bad[other] = True
             return False
 
+        assert isinstance(other, VArrayStructStateInfo)
         if len(self.fielddescrs) != len(other.fielddescrs):
             bad[self] = True
             bad[other] = True

From noreply at buildbot.pypy.org  Tue Oct 25 06:21:55 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue, 25 Oct 2011 06:21:55 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-dicts: next test that I need to pass.
 for virtual dicts to be useful in python they need to work through opaque
 pointers, which are used by dictmultiobject
Message-ID: <20111025042155.65E90820C7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: virtual-dicts
Changeset: r48405:fd60bd44e563
Date: 2011-10-25 00:21 -0400
http://bitbucket.org/pypy/pypy/changeset/fd60bd44e563/

Log:	next test that I need to pass. for virtual dicts to be useful in
	python they need to work through opaque pointers, which are used by
	dictmultiobject

diff --git a/pypy/jit/metainterp/heapcache.py b/pypy/jit/metainterp/heapcache.py
--- a/pypy/jit/metainterp/heapcache.py
+++ b/pypy/jit/metainterp/heapcache.py
@@ -34,7 +34,6 @@
         self.clear_caches(opnum, descr, argboxes)
 
     def mark_escaped(self, opnum, argboxes):
-        idx = 0
         if opnum == rop.SETFIELD_GC:
             assert len(argboxes) == 2
             box, valuebox = argboxes
@@ -44,6 +43,7 @@
                 self._escape(valuebox)
         # GETFIELD_GC doesn't escape it's argument
         elif opnum != rop.GETFIELD_GC:
+            idx = 0
             for box in argboxes:
                 # setarrayitem_gc don't escape its first argument
                 if not (idx == 0 and opnum in [rop.SETARRAYITEM_GC]):
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -10,6 +10,7 @@
 from pypy.jit.metainterp.typesystem import LLTypeHelper, OOTypeHelper
 from pypy.jit.metainterp.warmspot import get_stats
 from pypy.jit.metainterp.warmstate import set_future_value
+from pypy.rlib import rerased
 from pypy.rlib.jit import (JitDriver, we_are_jitted, hint, dont_look_inside,
     loop_invariant, elidable, promote, jit_debug, assert_green,
     AssertGreenFailed, unroll_safe, current_trace_length, look_inside_iff,
@@ -3519,6 +3520,25 @@
         assert res == 0
         self.check_loops({"int_sub": 1, "int_gt": 1, "guard_true": 1, "jump": 1})
 
+    def test_virtual_opaque_ptr(self):
+        myjitdriver = JitDriver(greens = [], reds=["n"])
+        erase, unerase = rerased.new_erasing_pair("x")
+        @look_inside_iff(lambda x: isvirtual(x))
+        def g(x):
+            return x[0]
+        def f(n):
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n)
+                x = []
+                y = erase(x)
+                z = unerase(y)
+                z.append(1)
+                n -= g(z)
+            return n
+        res = self.meta_interp(f, [10])
+        assert res == 0
+        self.check_loops({"int_sub": 1, "int_gt": 1, "guard_true": 1, "jump": 1})
+
 
 class TestLLtype(BaseLLtypeTests, LLJitMixin):
     def test_tagged(self):
@@ -3575,8 +3595,7 @@
         res = self.meta_interp(main, [False, 100, True], taggedpointers=True)
 
     def test_rerased(self):
-        from pypy.rlib.rerased import erase_int, unerase_int, new_erasing_pair
-        eraseX, uneraseX = new_erasing_pair("X")
+        eraseX, uneraseX = rerased,new_erasing_pair("X")
         #
         class X:
             def __init__(self, a, b):
@@ -3589,14 +3608,14 @@
                 e = eraseX(X(i, j))
             else:
                 try:
-                    e = erase_int(i)
+                    e = rerased.erase_int(i)
                 except OverflowError:
                     return -42
             if j & 1:
                 x = uneraseX(e)
                 return x.a - x.b
             else:
-                return unerase_int(e)
+                return rerased.unerase_int(e)
         #
         x = self.interp_operations(f, [-128, 0], taggedpointers=True)
         assert x == -128

From noreply at buildbot.pypy.org  Tue Oct 25 06:55:18 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue, 25 Oct 2011 06:55:18 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-dicts: replace cast_opaque_ptr resop,
 which never did anything except be used to mark its argument with
 mark_opaque_ptr,
 which helps reduce the confusion of the metainterp about the unescaped-ness
 of things
Message-ID: <20111025045518.1B75A820C7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: virtual-dicts
Changeset: r48406:56876d48b184
Date: 2011-10-25 00:55 -0400
http://bitbucket.org/pypy/pypy/changeset/56876d48b184/

Log:	replace cast_opaque_ptr resop, which never did anything except be
	used to mark its argument with mark_opaque_ptr, which helps reduce
	the confusion of the metainterp about the unescaped-ness of things

diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -844,6 +844,10 @@
         if self._is_gc(op.args[0]):
             return op
 
+    def rewrite_op_cast_opaque_ptr(self, op):
+        # None causes the result of this op to get aliased to op.args[0]
+        return [SpaceOperation('mark_opaque_ptr', op.args, None), None]
+
     def rewrite_op_force_cast(self, op):
         v_arg = op.args[0]
         v_result = op.result
diff --git a/pypy/jit/codewriter/test/test_jtransform.py b/pypy/jit/codewriter/test/test_jtransform.py
--- a/pypy/jit/codewriter/test/test_jtransform.py
+++ b/pypy/jit/codewriter/test/test_jtransform.py
@@ -1128,3 +1128,16 @@
                         varoftype(lltype.Signed))
     tr = Transformer(None, None)
     raises(NotImplementedError, tr.rewrite_operation, op)
+
+def test_cast_opaque_ptr():
+    S = lltype.GcStruct("S", ("x", lltype.Signed))
+    v1 = varoftype(lltype.Ptr(S))
+    v2 = varoftype(lltype.Ptr(rclass.OBJECT))
+
+    op = SpaceOperation('cast_opaque_ptr', [v1], v2)
+    tr = Transformer()
+    [op1, op2] = tr.rewrite_operation(op)
+    assert op1.opname == 'mark_opaque_ptr'
+    assert op1.args == [v1]
+    assert op1.result is None
+    assert op2 is None
\ No newline at end of file
diff --git a/pypy/jit/metainterp/blackhole.py b/pypy/jit/metainterp/blackhole.py
--- a/pypy/jit/metainterp/blackhole.py
+++ b/pypy/jit/metainterp/blackhole.py
@@ -505,9 +505,6 @@
     @arguments("r", "r", returns="i")
     def bhimpl_instance_ptr_ne(a, b):
         return a != b
-    @arguments("r", returns="r")
-    def bhimpl_cast_opaque_ptr(a):
-        return a
     @arguments("r", returns="i")
     def bhimpl_cast_ptr_to_int(a):
         i = lltype.cast_ptr_to_int(a)
@@ -518,6 +515,10 @@
         ll_assert((i & 1) == 1, "bhimpl_cast_int_to_ptr: not an odd int")
         return lltype.cast_int_to_ptr(llmemory.GCREF, i)
 
+    @arguments("r")
+    def bhimpl_mark_opaque_ptr(a):
+        pass
+
     @arguments("i", returns="i")
     def bhimpl_int_copy(a):
         return a
diff --git a/pypy/jit/metainterp/heapcache.py b/pypy/jit/metainterp/heapcache.py
--- a/pypy/jit/metainterp/heapcache.py
+++ b/pypy/jit/metainterp/heapcache.py
@@ -41,8 +41,8 @@
                 self.dependencies.setdefault(box, []).append(valuebox)
             else:
                 self._escape(valuebox)
-        # GETFIELD_GC doesn't escape it's argument
-        elif opnum != rop.GETFIELD_GC:
+        # GETFIELD_GC and MARK_OPAQUE_PTR doesn't escape their arguments
+        elif opnum != rop.GETFIELD_GC and opnum != rop.MARK_OPAQUE_PTR:
             idx = 0
             for box in argboxes:
                 # setarrayitem_gc don't escape its first argument
diff --git a/pypy/jit/metainterp/optimizeopt/rewrite.py b/pypy/jit/metainterp/optimizeopt/rewrite.py
--- a/pypy/jit/metainterp/optimizeopt/rewrite.py
+++ b/pypy/jit/metainterp/optimizeopt/rewrite.py
@@ -465,10 +465,9 @@
                                         args = [op.getarg(0), ConstInt(highest_bit(val))])
         self.emit_operation(op)
 
-    def optimize_CAST_OPAQUE_PTR(self, op):
+    def optimize_MARK_OPAQUE_PTR(self, op):
         value = self.getvalue(op.getarg(0))
         self.optimizer.opaque_pointers[value] = True
-        self.make_equal_to(op.result, value)
 
     def optimize_CAST_PTR_TO_INT(self, op):
         self.pure(rop.CAST_INT_TO_PTR, [op.result], op.getarg(0))
diff --git a/pypy/jit/metainterp/optimizeopt/simplify.py b/pypy/jit/metainterp/optimizeopt/simplify.py
--- a/pypy/jit/metainterp/optimizeopt/simplify.py
+++ b/pypy/jit/metainterp/optimizeopt/simplify.py
@@ -25,7 +25,8 @@
         #     but it's a bit hard to implement robustly if heap.py is also run
         pass
 
-    optimize_CAST_OPAQUE_PTR = optimize_VIRTUAL_REF
+    def optimize_MARK_OPAQUE_PTR(self, op):
+        pass
 
 
 dispatch_opt = make_dispatcher_method(OptSimplify, 'optimize_',
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -240,8 +240,8 @@
         return self.execute(rop.PTR_EQ, box, history.CONST_NULL)
 
     @arguments("box")
-    def opimpl_cast_opaque_ptr(self, box):
-        return self.execute(rop.CAST_OPAQUE_PTR, box)
+    def opimpl_mark_opaque_ptr(self, box):
+        return self.execute(rop.MARK_OPAQUE_PTR, box)
 
     @arguments("box")
     def _opimpl_any_return(self, box):
diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py
--- a/pypy/jit/metainterp/resoperation.py
+++ b/pypy/jit/metainterp/resoperation.py
@@ -439,7 +439,6 @@
     'PTR_NE/2b',
     'INSTANCE_PTR_EQ/2b',
     'INSTANCE_PTR_NE/2b',
-    'CAST_OPAQUE_PTR/1b',
     #
     'ARRAYLEN_GC/1d',
     'STRLEN/1',
@@ -471,6 +470,7 @@
     'FORCE_TOKEN/0',
     'VIRTUAL_REF/2',         # removed before it's passed to the backend
     'READ_TIMESTAMP/0',
+    'MARK_OPAQUE_PTR/1b',
     '_NOSIDEEFFECT_LAST', # ----- end of no_side_effect operations -----
 
     'SETARRAYITEM_GC/3d',

From noreply at buildbot.pypy.org  Tue Oct 25 09:47:19 2011
From: noreply at buildbot.pypy.org (hager)
Date: Tue, 25 Oct 2011 09:47:19 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: Implemented STRLEN, STRGETITEM,
	STRSETITEM.
Message-ID: <20111025074719.D53D3820C7@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r48407:7c2f8272c677
Date: 2011-10-25 09:46 +0200
http://bitbucket.org/pypy/pypy/changeset/7c2f8272c677/

Log:	Implemented STRLEN, STRGETITEM, STRSETITEM.

diff --git a/pypy/jit/backend/ppc/ppcgen/opassembler.py b/pypy/jit/backend/ppc/ppcgen/opassembler.py
--- a/pypy/jit/backend/ppc/ppcgen/opassembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/opassembler.py
@@ -371,5 +371,32 @@
             signed = descr.is_item_signed()
             self._ensure_result_bit_extension(res, size, signed)
 
+    # XXX 64 bit adjustment needed
+    def emit_strlen(self, op, arglocs, regalloc):
+        l0, l1, res = arglocs
+        if l1.is_imm():
+            self.mc.lwz(res.value, l0.value, l1.getint())
+        else:
+            self.mc.lwzx(res.value, l0.value, l1.value)
+
+    def emit_strgetitem(self, op, arglocs, regalloc):
+        res, base_loc, ofs_loc, basesize = arglocs
+        if ofs_loc.is_imm():
+            self.mc.addi(res.value, base_loc.value, ofs_loc.getint())
+        else:
+            self.mc.add(res.value, base_loc.value, ofs_loc.value)
+        
+        self.mc.load_imm(r.r0, basesize.value)
+        self.mc.lbzx(res.value, res.value, r.r0.value)
+
+    def emit_strsetitem(self, op, arglocs, regalloc):
+        value_loc, base_loc, ofs_loc, basesize = arglocs
+        if ofs_loc.is_imm():
+            self.mc.addi(base_loc.value, base_loc.value, ofs_loc.getint())
+        else:
+            self.mc.add(base_loc.value, base_loc.value, ofs_loc.value)
+        self.mc.load_imm(r.r0, basesize.value)
+        self.mc.stbx(value_loc.value, base_loc.value, r.r0.value)
+
     def nop(self):
         self.mc.ori(0, 0, 0)
diff --git a/pypy/jit/backend/ppc/ppcgen/regalloc.py b/pypy/jit/backend/ppc/ppcgen/regalloc.py
--- a/pypy/jit/backend/ppc/ppcgen/regalloc.py
+++ b/pypy/jit/backend/ppc/ppcgen/regalloc.py
@@ -16,7 +16,8 @@
                                              BaseCallDescr, BaseSizeDescr
 from pypy.jit.metainterp.resoperation import rop
 from pypy.jit.backend.ppc.ppcgen import locations
-from pypy.rpython.lltypesystem import rffi, lltype
+from pypy.rpython.lltypesystem import rffi, lltype, rstr
+from pypy.jit.backend.llsupport import symbolic
 import pypy.jit.backend.ppc.ppcgen.register as r
 
 class TempInt(TempBox):
@@ -413,6 +414,67 @@
         self.possibly_free_var(op.result)
         return [res, base_loc, ofs_loc, imm(scale), imm(ofs)]
 
+    def prepare_strlen(self, op):
+        l0, box = self._ensure_value_is_boxed(op.getarg(0))
+        boxes = [box]
+
+        basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR,
+                                         self.cpu.translate_support_code)
+        ofs_box = ConstInt(ofs_length)
+        imm_ofs = _check_imm_arg(ofs_box)
+
+        if imm_ofs:
+            l1 = self.make_sure_var_in_reg(ofs_box, boxes)
+        else:
+            l1, box1 = self._ensure_value_is_boxed(ofs_box, boxes)
+            boxes.append(box1)
+
+        self.possibly_free_vars(boxes)
+        res = self.force_allocate_reg(op.result)
+        self.possibly_free_var(op.result)
+        return [l0, l1, res]
+
+    def prepare_strgetitem(self, op):
+        boxes = list(op.getarglist())
+        base_loc, box = self._ensure_value_is_boxed(boxes[0])
+        boxes.append(box)
+
+        a1 = boxes[1]
+        imm_a1 = _check_imm_arg(a1)
+        if imm_a1:
+            ofs_loc = self.make_sure_var_in_reg(a1, boxes)
+        else:
+            ofs_loc, box = self._ensure_value_is_boxed(a1, boxes)
+            boxes.append(box)
+
+        self.possibly_free_vars(boxes)
+        res = self.force_allocate_reg(op.result)
+        self.possibly_free_var(op.result)
+
+        basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR,
+                                         self.cpu.translate_support_code)
+        assert itemsize == 1
+        return [res, base_loc, ofs_loc, imm(basesize)]
+
+    def prepare_strsetitem(self, op):
+        boxes = list(op.getarglist())
+
+        base_loc, box = self._ensure_value_is_boxed(boxes[0], boxes)
+        boxes.append(box)
+
+        ofs_loc, box = self._ensure_value_is_boxed(boxes[1], boxes)
+        boxes.append(box)
+
+        value_loc, box = self._ensure_value_is_boxed(boxes[2], boxes)
+        boxes.append(box)
+
+        self.possibly_free_vars(boxes)
+
+        basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR,
+                                         self.cpu.translate_support_code)
+        assert itemsize == 1
+        return [value_loc, base_loc, ofs_loc, imm(basesize)]
+
     # from ../x86/regalloc.py:791
     def _unpack_fielddescr(self, fielddescr):
         assert isinstance(fielddescr, BaseFieldDescr)

From noreply at buildbot.pypy.org  Tue Oct 25 10:47:19 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 25 Oct 2011 10:47:19 +0200 (CEST)
Subject: [pypy-commit] benchmarks default: improve cpython saver
Message-ID: <20111025084719.2612B820C7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r148:05f934e964ab
Date: 2011-10-25 10:47 +0200
http://bitbucket.org/pypy/benchmarks/changeset/05f934e964ab/

Log:	improve cpython saver

diff --git a/savecpython.py b/savecpython.py
--- a/savecpython.py
+++ b/savecpython.py
@@ -4,11 +4,13 @@
 import json
 import urllib, urllib2
 from datetime import datetime
+import optparse
 
 #SPEEDURL = 'http://127.0.0.1:8000/'
 SPEEDURL = 'http://speed.pypy.org/'
 
-def save(project, revision, results, options, executable, host, testing=False):
+def save(project, revision, results, options, executable, host, testing=False,
+         base=False):
     testparams = []
     #Parse data
     data = {}
@@ -20,9 +22,15 @@
         results = b[2]
         value = 0
         if res_type == "SimpleComparisonResult":
-            value = results['base_time']
+            if base:
+                value = results['base_time']
+            else:
+                value = results['changed_time']
         elif res_type == "ComparisonResult":
-            value = results['avg_base']
+            if base:
+                value = results['avg_base']
+            else:
+                value = results['avg_changed']
         else:
             print("ERROR: result type unknown " + b[1])
             return 1
@@ -36,7 +44,10 @@
             'result_date': current_date,
         }
         if res_type == "ComparisonResult":
-            data['std_dev'] = results['std_changed']
+            if base:
+                data['std_dev'] = results['std_base']
+            else:
+                data['std_dev'] = results['std_changed']
         if testing: testparams.append(data)
         else: send(data)
     if testing: return testparams
@@ -62,9 +73,19 @@
             response = '\n  The server couldn\'t fulfill the request\n'
             response += '  Error code: ' + str(e)
         print("Server (%s) response: %s\n" % (SPEEDURL, response))
+        if hasattr(e, 'fp'):
+            print e.fp.read(), "\n"
         return 1
     return 0
 
 if __name__ == '__main__':
-    results = json.load(open(sys.argv[1]))['results']
-    save('cpython', 100, results, None, 'cpython', 'tannit', testing=False)
+    parser = optparse.OptionParser()
+    parser.add_option('-b', '--base', action='store_true',
+                      help='take base values instead of modified')
+    options, args = parser.parse_args(sys.argv)
+    if len(args) != 2:
+        print parser.usage
+        sys.exit(1)
+    results = json.load(open(args[1]))['results']
+    save('cpython', 100, results, None, 'cpython', 'tannit', testing=False,
+         base=options.base)

From noreply at buildbot.pypy.org  Tue Oct 25 11:00:10 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 25 Oct 2011 11:00:10 +0200 (CEST)
Subject: [pypy-commit] benchmarks default: finish fighting
Message-ID: <20111025090010.ED261820C7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r149:354e7ef9ccde
Date: 2011-10-25 10:59 +0200
http://bitbucket.org/pypy/benchmarks/changeset/354e7ef9ccde/

Log:	finish fighting

diff --git a/savecpython.py b/savecpython.py
--- a/savecpython.py
+++ b/savecpython.py
@@ -21,11 +21,14 @@
         res_type = b[1]
         results = b[2]
         value = 0
-        if res_type == "SimpleComparisonResult":
+        if res_type == "SimpleComparisonResult" or res_type == 'RawResult':
             if base:
-                value = results['base_time']
+                value = results['base_times']
             else:
-                value = results['changed_time']
+                value = results['changed_times']
+            if value is None:
+                continue
+            value = value[0]
         elif res_type == "ComparisonResult":
             if base:
                 value = results['avg_base']
@@ -42,6 +45,7 @@
             'environment': host,
             'result_value': value,
             'result_date': current_date,
+            'branch': 'default',
         }
         if res_type == "ComparisonResult":
             if base:
@@ -82,10 +86,13 @@
     parser = optparse.OptionParser()
     parser.add_option('-b', '--base', action='store_true',
                       help='take base values instead of modified')
+    parser.add_option('--revision', help='revision number', type=int,
+                      default=100)
     options, args = parser.parse_args(sys.argv)
     if len(args) != 2:
         print parser.usage
         sys.exit(1)
     results = json.load(open(args[1]))['results']
-    save('cpython', 100, results, None, 'cpython', 'tannit', testing=False,
+    save('cpython', options.revision, results, None, 'cpython', 'tannit',
+         testing=False,
          base=options.base)

From noreply at buildbot.pypy.org  Tue Oct 25 11:00:12 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 25 Oct 2011 11:00:12 +0200 (CEST)
Subject: [pypy-commit] benchmarks default: clarify comment
Message-ID: <20111025090012.3AEC9820C7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r150:9cd4b2c5429c
Date: 2011-10-25 10:59 +0200
http://bitbucket.org/pypy/benchmarks/changeset/9cd4b2c5429c/

Log:	clarify comment

diff --git a/savecpython.py b/savecpython.py
--- a/savecpython.py
+++ b/savecpython.py
@@ -86,7 +86,7 @@
     parser = optparse.OptionParser()
     parser.add_option('-b', '--base', action='store_true',
                       help='take base values instead of modified')
-    parser.add_option('--revision', help='revision number', type=int,
+    parser.add_option('--revision', help='revision number (100 for cpythono 2.6.2, 101 for 2.7.2, edit admin interface to add more)', type=int,
                       default=100)
     options, args = parser.parse_args(sys.argv)
     if len(args) != 2:

From noreply at buildbot.pypy.org  Tue Oct 25 11:07:37 2011
From: noreply at buildbot.pypy.org (bivab)
Date: Tue, 25 Oct 2011 11:07:37 +0200 (CEST)
Subject: [pypy-commit] pypy arm-backend-2: extend call_assembler tests to
 check the fail_descr_number and ensure it is non-zero in the tests
Message-ID: <20111025090737.12CB0820C7@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: arm-backend-2
Changeset: r48408:45d7621d2e2a
Date: 2011-10-04 14:34 +0200
http://bitbucket.org/pypy/pypy/changeset/45d7621d2e2a/

Log:	extend call_assembler tests to check the fail_descr_number and
	ensure it is non-zero in the tests

diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -2376,6 +2376,9 @@
             assembler_helper_adr = llmemory.cast_ptr_to_adr(
                 _assembler_helper_ptr)
 
+        # ensure the fail_descr_number is not zero
+        for _ in range(10):
+            self.cpu.reserve_some_free_fail_descr_number()
         ops = '''
         [i0, i1, i2, i3, i4, i5, i6, i7, i8, i9]
         i10 = int_add(i0, i1)
@@ -2416,7 +2419,7 @@
             self.cpu.set_future_value_int(i, i+1)
         res = self.cpu.execute_token(othertoken)
         assert self.cpu.get_latest_value_int(0) == 13
-        assert called
+        assert called == [done_number]
 
         # test the fast path, which should not call assembler_helper()
         del called[:]
@@ -2455,7 +2458,8 @@
         FakeJitDriverSD.portal_calldescr = self.cpu.calldescrof(
             lltype.Ptr(lltype.FuncType(ARGS, RES)), ARGS, RES,
             EffectInfo.MOST_GENERAL)
-        
+        for _ in range(10):
+            self.cpu.reserve_some_free_fail_descr_number()
         ops = '''
         [f0, f1]
         f2 = float_add(f0, f1)
@@ -2484,7 +2488,7 @@
         res = self.cpu.execute_token(othertoken)
         x = self.cpu.get_latest_value_float(0)
         assert longlong.getrealfloat(x) == 13.5
-        assert called
+        assert called == [done_number]
 
         # test the fast path, which should not call assembler_helper()
         del called[:]
@@ -2548,7 +2552,8 @@
         FakeJitDriverSD.portal_calldescr = self.cpu.calldescrof(
             lltype.Ptr(lltype.FuncType(ARGS, RES)), ARGS, RES,
             EffectInfo.MOST_GENERAL)
-        
+        for _ in range(10):
+            self.cpu.reserve_some_free_fail_descr_number()
         ops = '''
         [f0, f1]
         f2 = float_add(f0, f1)
@@ -2557,6 +2562,7 @@
         looptoken = LoopToken()
         looptoken.outermost_jitdriver_sd = FakeJitDriverSD()
         self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken)
+        done_number = self.cpu.get_fail_descr_number(loop.operations[-1].getdescr())
         self.cpu.set_future_value_float(0, longlong.getfloatstorage(1.25))
         self.cpu.set_future_value_float(1, longlong.getfloatstorage(2.35))
         res = self.cpu.execute_token(looptoken)
@@ -2580,7 +2586,7 @@
         res = self.cpu.execute_token(othertoken)
         x = self.cpu.get_latest_value_float(0)
         assert longlong.getrealfloat(x) == 13.5
-        assert called
+        assert called == [done_number]
         del called[:]
 
         # compile a replacement
@@ -2592,6 +2598,7 @@
         looptoken2 = LoopToken()
         looptoken2.outermost_jitdriver_sd = FakeJitDriverSD()
         self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken2)
+        done_number = self.cpu.get_fail_descr_number(loop.operations[-1].getdescr())
 
         # install it
         self.cpu.redirect_call_assembler(looptoken, looptoken2)
@@ -2603,7 +2610,7 @@
         res = self.cpu.execute_token(othertoken)
         x = self.cpu.get_latest_value_float(0)
         assert longlong.getrealfloat(x) == 13.5
-        assert called
+        assert called == [done_number]
 
     def test_short_result_of_getfield_direct(self):
         # Test that a getfield that returns a CHAR, SHORT or INT, signed

From noreply at buildbot.pypy.org  Tue Oct 25 11:07:41 2011
From: noreply at buildbot.pypy.org (bivab)
Date: Tue, 25 Oct 2011 11:07:41 +0200 (CEST)
Subject: [pypy-commit] pypy arm-backend-2: merge default
Message-ID: <20111025090741.C950C820C7@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: arm-backend-2
Changeset: r48409:f534b61e5913
Date: 2011-10-10 09:24 +0200
http://bitbucket.org/pypy/pypy/changeset/f534b61e5913/

Log:	merge default

diff too long, truncating to 10000 out of 17472 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -1,2 +1,3 @@
 b590cf6de4190623aad9aa698694c22e614d67b9 release-1.5
 b48df0bf4e75b81d98f19ce89d4a7dc3e1dab5e5 benchmarked
+d8ac7d23d3ec5f9a0fa1264972f74a010dbfd07f release-1.6
diff --git a/dotviewer/graphparse.py b/dotviewer/graphparse.py
--- a/dotviewer/graphparse.py
+++ b/dotviewer/graphparse.py
@@ -36,48 +36,45 @@
     print >> sys.stderr, "Warning: could not guess file type, using 'dot'"
     return 'unknown'
 
-def dot2plain(content, contenttype, use_codespeak=False):
-    if contenttype == 'plain':
-        # already a .plain file
-        return content
+def dot2plain_graphviz(content, contenttype, use_codespeak=False):
+    if contenttype != 'neato':
+        cmdline = 'dot -Tplain'
+    else:
+        cmdline = 'neato -Tplain'
+    #print >> sys.stderr, '* running:', cmdline
+    close_fds = sys.platform != 'win32'
+    p = subprocess.Popen(cmdline, shell=True, close_fds=close_fds,
+                         stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+    (child_in, child_out) = (p.stdin, p.stdout)
+    try:
+        import thread
+    except ImportError:
+        bkgndwrite(child_in, content)
+    else:
+        thread.start_new_thread(bkgndwrite, (child_in, content))
+    plaincontent = child_out.read()
+    child_out.close()
+    if not plaincontent:    # 'dot' is likely not installed
+        raise PlainParseError("no result from running 'dot'")
+    return plaincontent
 
-    if not use_codespeak:
-        if contenttype != 'neato':
-            cmdline = 'dot -Tplain'
-        else:
-            cmdline = 'neato -Tplain'
-        #print >> sys.stderr, '* running:', cmdline
-        close_fds = sys.platform != 'win32'
-        p = subprocess.Popen(cmdline, shell=True, close_fds=close_fds,
-                             stdin=subprocess.PIPE, stdout=subprocess.PIPE)
-        (child_in, child_out) = (p.stdin, p.stdout)
-        try:
-            import thread
-        except ImportError:
-            bkgndwrite(child_in, content)
-        else:
-            thread.start_new_thread(bkgndwrite, (child_in, content))
-        plaincontent = child_out.read()
-        child_out.close()
-        if not plaincontent:    # 'dot' is likely not installed
-            raise PlainParseError("no result from running 'dot'")
-    else:
-        import urllib
-        request = urllib.urlencode({'dot': content})
-        url = 'http://codespeak.net/pypy/convertdot.cgi'
-        print >> sys.stderr, '* posting:', url
-        g = urllib.urlopen(url, data=request)
-        result = []
-        while True:
-            data = g.read(16384)
-            if not data:
-                break
-            result.append(data)
-        g.close()
-        plaincontent = ''.join(result)
-        # very simple-minded way to give a somewhat better error message
-        if plaincontent.startswith('<body'):
-            raise Exception("the dot on codespeak has very likely crashed")
+def dot2plain_codespeak(content, contenttype):
+    import urllib
+    request = urllib.urlencode({'dot': content})
+    url = 'http://codespeak.net/pypy/convertdot.cgi'
+    print >> sys.stderr, '* posting:', url
+    g = urllib.urlopen(url, data=request)
+    result = []
+    while True:
+        data = g.read(16384)
+        if not data:
+            break
+        result.append(data)
+    g.close()
+    plaincontent = ''.join(result)
+    # very simple-minded way to give a somewhat better error message
+    if plaincontent.startswith('<body'):
+        raise Exception("the dot on codespeak has very likely crashed")
     return plaincontent
 
 def bkgndwrite(f, data):
@@ -148,10 +145,13 @@
 
 def parse_dot(graph_id, content, links={}, fixedfont=False):
     contenttype = guess_type(content)
-    try:
-        plaincontent = dot2plain(content, contenttype, use_codespeak=False)
-        return list(parse_plain(graph_id, plaincontent, links, fixedfont))
-    except PlainParseError:
-        # failed, retry via codespeak
-        plaincontent = dot2plain(content, contenttype, use_codespeak=True)
-        return list(parse_plain(graph_id, plaincontent, links, fixedfont))
+    if contenttype == 'plain':
+        plaincontent = content
+    else:
+        try:
+            plaincontent = dot2plain_graphviz(content, contenttype)
+        except PlainParseError, e:
+            print e
+            # failed, retry via codespeak
+            plaincontent = dot2plain_codespeak(content, contenttype)
+    return list(parse_plain(graph_id, plaincontent, links, fixedfont))
diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -317,7 +317,7 @@
     RegrTest('test_multibytecodec.py', usemodules='_multibytecodec'),
     RegrTest('test_multibytecodec_support.py', skip="not a test"),
     RegrTest('test_multifile.py'),
-    RegrTest('test_multiprocessing.py', skip='FIXME leaves subprocesses'),
+    RegrTest('test_multiprocessing.py', skip="FIXME leaves subprocesses"),
     RegrTest('test_mutants.py', core="possibly"),
     RegrTest('test_mutex.py'),
     RegrTest('test_netrc.py'),
diff --git a/lib-python/modified-2.7/gzip.py b/lib-python/modified-2.7/gzip.py
deleted file mode 100644
--- a/lib-python/modified-2.7/gzip.py
+++ /dev/null
@@ -1,514 +0,0 @@
-"""Functions that read and write gzipped files.
-
-The user of the file doesn't have to worry about the compression,
-but random access is not allowed."""
-
-# based on Andrew Kuchling's minigzip.py distributed with the zlib module
-
-import struct, sys, time, os
-import zlib
-import io
-import __builtin__
-
-__all__ = ["GzipFile","open"]
-
-FTEXT, FHCRC, FEXTRA, FNAME, FCOMMENT = 1, 2, 4, 8, 16
-
-READ, WRITE = 1, 2
-
-def write32u(output, value):
-    # The L format writes the bit pattern correctly whether signed
-    # or unsigned.
-    output.write(struct.pack("<L", value))
-
-def read32(input):
-    return struct.unpack("<I", input.read(4))[0]
-
-def open(filename, mode="rb", compresslevel=9):
-    """Shorthand for GzipFile(filename, mode, compresslevel).
-
-    The filename argument is required; mode defaults to 'rb'
-    and compresslevel defaults to 9.
-
-    """
-    return GzipFile(filename, mode, compresslevel)
-
-class GzipFile(io.BufferedIOBase):
-    """The GzipFile class simulates most of the methods of a file object with
-    the exception of the readinto() and truncate() methods.
-
-    """
-
-    myfileobj = None
-    max_read_chunk = 10 * 1024 * 1024   # 10Mb
-
-    def __init__(self, filename=None, mode=None,
-                 compresslevel=9, fileobj=None, mtime=None):
-        """Constructor for the GzipFile class.
-
-        At least one of fileobj and filename must be given a
-        non-trivial value.
-
-        The new class instance is based on fileobj, which can be a regular
-        file, a StringIO object, or any other object which simulates a file.
-        It defaults to None, in which case filename is opened to provide
-        a file object.
-
-        When fileobj is not None, the filename argument is only used to be
-        included in the gzip file header, which may includes the original
-        filename of the uncompressed file.  It defaults to the filename of
-        fileobj, if discernible; otherwise, it defaults to the empty string,
-        and in this case the original filename is not included in the header.
-
-        The mode argument can be any of 'r', 'rb', 'a', 'ab', 'w', or 'wb',
-        depending on whether the file will be read or written.  The default
-        is the mode of fileobj if discernible; otherwise, the default is 'rb'.
-        Be aware that only the 'rb', 'ab', and 'wb' values should be used
-        for cross-platform portability.
-
-        The compresslevel argument is an integer from 1 to 9 controlling the
-        level of compression; 1 is fastest and produces the least compression,
-        and 9 is slowest and produces the most compression.  The default is 9.
-
-        The mtime argument is an optional numeric timestamp to be written
-        to the stream when compressing.  All gzip compressed streams
-        are required to contain a timestamp.  If omitted or None, the
-        current time is used.  This module ignores the timestamp when
-        decompressing; however, some programs, such as gunzip, make use
-        of it.  The format of the timestamp is the same as that of the
-        return value of time.time() and of the st_mtime member of the
-        object returned by os.stat().
-
-        """
-
-        # guarantee the file is opened in binary mode on platforms
-        # that care about that sort of thing
-        if mode and 'b' not in mode:
-            mode += 'b'
-        if fileobj is None:
-            fileobj = self.myfileobj = __builtin__.open(filename, mode or 'rb')
-        if filename is None:
-            if hasattr(fileobj, 'name'): filename = fileobj.name
-            else: filename = ''
-        if mode is None:
-            if hasattr(fileobj, 'mode'): mode = fileobj.mode
-            else: mode = 'rb'
-
-        if mode[0:1] == 'r':
-            self.mode = READ
-            # Set flag indicating start of a new member
-            self._new_member = True
-            # Buffer data read from gzip file. extrastart is offset in
-            # stream where buffer starts. extrasize is number of
-            # bytes remaining in buffer from current stream position.
-            self.extrabuf = ""
-            self.extrasize = 0
-            self.extrastart = 0
-            self.name = filename
-            # Starts small, scales exponentially
-            self.min_readsize = 100
-
-        elif mode[0:1] == 'w' or mode[0:1] == 'a':
-            self.mode = WRITE
-            self._init_write(filename)
-            self.compress = zlib.compressobj(compresslevel,
-                                             zlib.DEFLATED,
-                                             -zlib.MAX_WBITS,
-                                             zlib.DEF_MEM_LEVEL,
-                                             0)
-        else:
-            raise IOError, "Mode " + mode + " not supported"
-
-        self.fileobj = fileobj
-        self.offset = 0
-        self.mtime = mtime
-
-        if self.mode == WRITE:
-            self._write_gzip_header()
-
-    @property
-    def filename(self):
-        import warnings
-        warnings.warn("use the name attribute", DeprecationWarning, 2)
-        if self.mode == WRITE and self.name[-3:] != ".gz":
-            return self.name + ".gz"
-        return self.name
-
-    def __repr__(self):
-        s = repr(self.fileobj)
-        return '<gzip ' + s[1:-1] + ' ' + hex(id(self)) + '>'
-
-    def _check_closed(self):
-        """Raises a ValueError if the underlying file object has been closed.
-
-        """
-        if self.closed:
-            raise ValueError('I/O operation on closed file.')
-
-    def _init_write(self, filename):
-        self.name = filename
-        self.crc = zlib.crc32("") & 0xffffffffL
-        self.size = 0
-        self.writebuf = []
-        self.bufsize = 0
-
-    def _write_gzip_header(self):
-        self.fileobj.write('\037\213')             # magic header
-        self.fileobj.write('\010')                 # compression method
-        fname = os.path.basename(self.name)
-        if fname.endswith(".gz"):
-            fname = fname[:-3]
-        flags = 0
-        if fname:
-            flags = FNAME
-        self.fileobj.write(chr(flags))
-        mtime = self.mtime
-        if mtime is None:
-            mtime = time.time()
-        write32u(self.fileobj, long(mtime))
-        self.fileobj.write('\002')
-        self.fileobj.write('\377')
-        if fname:
-            self.fileobj.write(fname + '\000')
-
-    def _init_read(self):
-        self.crc = zlib.crc32("") & 0xffffffffL
-        self.size = 0
-
-    def _read_gzip_header(self):
-        magic = self.fileobj.read(2)
-        if magic != '\037\213':
-            raise IOError, 'Not a gzipped file'
-        method = ord( self.fileobj.read(1) )
-        if method != 8:
-            raise IOError, 'Unknown compression method'
-        flag = ord( self.fileobj.read(1) )
-        self.mtime = read32(self.fileobj)
-        # extraflag = self.fileobj.read(1)
-        # os = self.fileobj.read(1)
-        self.fileobj.read(2)
-
-        if flag & FEXTRA:
-            # Read & discard the extra field, if present
-            xlen = ord(self.fileobj.read(1))
-            xlen = xlen + 256*ord(self.fileobj.read(1))
-            self.fileobj.read(xlen)
-        if flag & FNAME:
-            # Read and discard a null-terminated string containing the filename
-            while True:
-                s = self.fileobj.read(1)
-                if not s or s=='\000':
-                    break
-        if flag & FCOMMENT:
-            # Read and discard a null-terminated string containing a comment
-            while True:
-                s = self.fileobj.read(1)
-                if not s or s=='\000':
-                    break
-        if flag & FHCRC:
-            self.fileobj.read(2)     # Read & discard the 16-bit header CRC
-
-    def write(self,data):
-        self._check_closed()
-        if self.mode != WRITE:
-            import errno
-            raise IOError(errno.EBADF, "write() on read-only GzipFile object")
-
-        if self.fileobj is None:
-            raise ValueError, "write() on closed GzipFile object"
-
-        # Convert data type if called by io.BufferedWriter.
-        if isinstance(data, memoryview):
-            data = data.tobytes()
-
-        if len(data) > 0:
-            self.size = self.size + len(data)
-            self.crc = zlib.crc32(data, self.crc) & 0xffffffffL
-            self.fileobj.write( self.compress.compress(data) )
-            self.offset += len(data)
-
-        return len(data)
-
-    def read(self, size=-1):
-        self._check_closed()
-        if self.mode != READ:
-            import errno
-            raise IOError(errno.EBADF, "read() on write-only GzipFile object")
-
-        if self.extrasize <= 0 and self.fileobj is None:
-            return ''
-
-        readsize = 1024
-        if size < 0:        # get the whole thing
-            try:
-                while True:
-                    self._read(readsize)
-                    readsize = min(self.max_read_chunk, readsize * 2)
-            except EOFError:
-                size = self.extrasize
-        elif size == 0:
-            return ""
-        else:               # just get some more of it
-            try:
-                while size > self.extrasize:
-                    self._read(readsize)
-                    readsize = min(self.max_read_chunk, readsize * 2)
-            except EOFError:
-                if size > self.extrasize:
-                    size = self.extrasize
-
-        offset = self.offset - self.extrastart
-        chunk = self.extrabuf[offset: offset + size]
-        self.extrasize = self.extrasize - size
-
-        self.offset += size
-        return chunk
-
-    def _unread(self, buf):
-        self.extrasize = len(buf) + self.extrasize
-        self.offset -= len(buf)
-
-    def _read(self, size=1024):
-        if self.fileobj is None:
-            raise EOFError, "Reached EOF"
-
-        if self._new_member:
-            # If the _new_member flag is set, we have to
-            # jump to the next member, if there is one.
-            #
-            # First, check if we're at the end of the file;
-            # if so, it's time to stop; no more members to read.
-            pos = self.fileobj.tell()   # Save current position
-            self.fileobj.seek(0, 2)     # Seek to end of file
-            if pos == self.fileobj.tell():
-                raise EOFError, "Reached EOF"
-            else:
-                self.fileobj.seek( pos ) # Return to original position
-
-            self._init_read()
-            self._read_gzip_header()
-            self.decompress = zlib.decompressobj(-zlib.MAX_WBITS)
-            self._new_member = False
-
-        # Read a chunk of data from the file
-        buf = self.fileobj.read(size)
-
-        # If the EOF has been reached, flush the decompression object
-        # and mark this object as finished.
-
-        if buf == "":
-            uncompress = self.decompress.flush()
-            self._read_eof()
-            self._add_read_data( uncompress )
-            raise EOFError, 'Reached EOF'
-
-        uncompress = self.decompress.decompress(buf)
-        self._add_read_data( uncompress )
-
-        if self.decompress.unused_data != "":
-            # Ending case: we've come to the end of a member in the file,
-            # so seek back to the start of the unused data, finish up
-            # this member, and read a new gzip header.
-            # (The number of bytes to seek back is the length of the unused
-            # data, minus 8 because _read_eof() will rewind a further 8 bytes)
-            self.fileobj.seek( -len(self.decompress.unused_data)+8, 1)
-
-            # Check the CRC and file size, and set the flag so we read
-            # a new member on the next call
-            self._read_eof()
-            self._new_member = True
-
-    def _add_read_data(self, data):
-        self.crc = zlib.crc32(data, self.crc) & 0xffffffffL
-        offset = self.offset - self.extrastart
-        self.extrabuf = self.extrabuf[offset:] + data
-        self.extrasize = self.extrasize + len(data)
-        self.extrastart = self.offset
-        self.size = self.size + len(data)
-
-    def _read_eof(self):
-        # We've read to the end of the file, so we have to rewind in order
-        # to reread the 8 bytes containing the CRC and the file size.
-        # We check the that the computed CRC and size of the
-        # uncompressed data matches the stored values.  Note that the size
-        # stored is the true file size mod 2**32.
-        self.fileobj.seek(-8, 1)
-        crc32 = read32(self.fileobj)
-        isize = read32(self.fileobj)  # may exceed 2GB
-        if crc32 != self.crc:
-            raise IOError("CRC check failed %s != %s" % (hex(crc32),
-                                                         hex(self.crc)))
-        elif isize != (self.size & 0xffffffffL):
-            raise IOError, "Incorrect length of data produced"
-
-        # Gzip files can be padded with zeroes and still have archives.
-        # Consume all zero bytes and set the file position to the first
-        # non-zero byte. See http://www.gzip.org/#faq8
-        c = "\x00"
-        while c == "\x00":
-            c = self.fileobj.read(1)
-        if c:
-            self.fileobj.seek(-1, 1)
-
-    @property
-    def closed(self):
-        return self.fileobj is None
-
-    def close(self):
-        if self.fileobj is None:
-            return
-        if self.mode == WRITE:
-            self.fileobj.write(self.compress.flush())
-            write32u(self.fileobj, self.crc)
-            # self.size may exceed 2GB, or even 4GB
-            write32u(self.fileobj, self.size & 0xffffffffL)
-            self.fileobj = None
-        elif self.mode == READ:
-            self.fileobj = None
-        if self.myfileobj:
-            self.myfileobj.close()
-            self.myfileobj = None
-
-    def flush(self,zlib_mode=zlib.Z_SYNC_FLUSH):
-        self._check_closed()
-        if self.mode == WRITE:
-            # Ensure the compressor's buffer is flushed
-            self.fileobj.write(self.compress.flush(zlib_mode))
-            self.fileobj.flush()
-
-    def fileno(self):
-        """Invoke the underlying file object's fileno() method.
-
-        This will raise AttributeError if the underlying file object
-        doesn't support fileno().
-        """
-        return self.fileobj.fileno()
-
-    def rewind(self):
-        '''Return the uncompressed stream file position indicator to the
-        beginning of the file'''
-        if self.mode != READ:
-            raise IOError("Can't rewind in write mode")
-        self.fileobj.seek(0)
-        self._new_member = True
-        self.extrabuf = ""
-        self.extrasize = 0
-        self.extrastart = 0
-        self.offset = 0
-
-    def readable(self):
-        return self.mode == READ
-
-    def writable(self):
-        return self.mode == WRITE
-
-    def seekable(self):
-        return True
-
-    def seek(self, offset, whence=0):
-        if whence:
-            if whence == 1:
-                offset = self.offset + offset
-            else:
-                raise ValueError('Seek from end not supported')
-        if self.mode == WRITE:
-            if offset < self.offset:
-                raise IOError('Negative seek in write mode')
-            count = offset - self.offset
-            for i in range(count // 1024):
-                self.write(1024 * '\0')
-            self.write((count % 1024) * '\0')
-        elif self.mode == READ:
-            if offset == self.offset:
-                self.read(0) # to make sure that this file is open
-                return self.offset
-            if offset < self.offset:
-                # for negative seek, rewind and do positive seek
-                self.rewind()
-            count = offset - self.offset
-            for i in range(count // 1024):
-                self.read(1024)
-            self.read(count % 1024)
-
-        return self.offset
-
-    def readline(self, size=-1):
-        if size < 0:
-            # Shortcut common case - newline found in buffer.
-            offset = self.offset - self.extrastart
-            i = self.extrabuf.find('\n', offset) + 1
-            if i > 0:
-                self.extrasize -= i - offset
-                self.offset += i - offset
-                return self.extrabuf[offset: i]
-
-            size = sys.maxint
-            readsize = self.min_readsize
-        else:
-            readsize = size
-        bufs = []
-        while size != 0:
-            c = self.read(readsize)
-            i = c.find('\n')
-
-            # We set i=size to break out of the loop under two
-            # conditions: 1) there's no newline, and the chunk is
-            # larger than size, or 2) there is a newline, but the
-            # resulting line would be longer than 'size'.
-            if (size <= i) or (i == -1 and len(c) > size):
-                i = size - 1
-
-            if i >= 0 or c == '':
-                bufs.append(c[:i + 1])    # Add portion of last chunk
-                self._unread(c[i + 1:])   # Push back rest of chunk
-                break
-
-            # Append chunk to list, decrease 'size',
-            bufs.append(c)
-            size = size - len(c)
-            readsize = min(size, readsize * 2)
-        if readsize > self.min_readsize:
-            self.min_readsize = min(readsize, self.min_readsize * 2, 512)
-        return ''.join(bufs) # Return resulting line
-
-
-def _test():
-    # Act like gzip; with -d, act like gunzip.
-    # The input file is not deleted, however, nor are any other gzip
-    # options or features supported.
-    args = sys.argv[1:]
-    decompress = args and args[0] == "-d"
-    if decompress:
-        args = args[1:]
-    if not args:
-        args = ["-"]
-    for arg in args:
-        if decompress:
-            if arg == "-":
-                f = GzipFile(filename="", mode="rb", fileobj=sys.stdin)
-                g = sys.stdout
-            else:
-                if arg[-3:] != ".gz":
-                    print "filename doesn't end in .gz:", repr(arg)
-                    continue
-                f = open(arg, "rb")
-                g = __builtin__.open(arg[:-3], "wb")
-        else:
-            if arg == "-":
-                f = sys.stdin
-                g = GzipFile(filename="", mode="wb", fileobj=sys.stdout)
-            else:
-                f = __builtin__.open(arg, "rb")
-                g = open(arg + ".gz", "wb")
-        while True:
-            chunk = f.read(1024)
-            if not chunk:
-                break
-            g.write(chunk)
-        if g is not sys.stdout:
-            g.close()
-        if f is not sys.stdin:
-            f.close()
-
-if __name__ == '__main__':
-    _test()
diff --git a/lib-python/modified-2.7/httplib.py b/lib-python/modified-2.7/httplib.py
new file mode 100644
--- /dev/null
+++ b/lib-python/modified-2.7/httplib.py
@@ -0,0 +1,1377 @@
+"""HTTP/1.1 client library
+
+<intro stuff goes here>
+<other stuff, too>
+
+HTTPConnection goes through a number of "states", which define when a client
+may legally make another request or fetch the response for a particular
+request. This diagram details these state transitions:
+
+    (null)
+      |
+      | HTTPConnection()
+      v
+    Idle
+      |
+      | putrequest()
+      v
+    Request-started
+      |
+      | ( putheader() )*  endheaders()
+      v
+    Request-sent
+      |
+      | response = getresponse()
+      v
+    Unread-response   [Response-headers-read]
+      |\____________________
+      |                     |
+      | response.read()     | putrequest()
+      v                     v
+    Idle                  Req-started-unread-response
+                     ______/|
+                   /        |
+   response.read() |        | ( putheader() )*  endheaders()
+                   v        v
+       Request-started    Req-sent-unread-response
+                            |
+                            | response.read()
+                            v
+                          Request-sent
+
+This diagram presents the following rules:
+  -- a second request may not be started until {response-headers-read}
+  -- a response [object] cannot be retrieved until {request-sent}
+  -- there is no differentiation between an unread response body and a
+     partially read response body
+
+Note: this enforcement is applied by the HTTPConnection class. The
+      HTTPResponse class does not enforce this state machine, which
+      implies sophisticated clients may accelerate the request/response
+      pipeline. Caution should be taken, though: accelerating the states
+      beyond the above pattern may imply knowledge of the server's
+      connection-close behavior for certain requests. For example, it
+      is impossible to tell whether the server will close the connection
+      UNTIL the response headers have been read; this means that further
+      requests cannot be placed into the pipeline until it is known that
+      the server will NOT be closing the connection.
+
+Logical State                  __state            __response
+-------------                  -------            ----------
+Idle                           _CS_IDLE           None
+Request-started                _CS_REQ_STARTED    None
+Request-sent                   _CS_REQ_SENT       None
+Unread-response                _CS_IDLE           <response_class>
+Req-started-unread-response    _CS_REQ_STARTED    <response_class>
+Req-sent-unread-response       _CS_REQ_SENT       <response_class>
+"""
+
+from array import array
+import os
+import socket
+from sys import py3kwarning
+from urlparse import urlsplit
+import warnings
+with warnings.catch_warnings():
+    if py3kwarning:
+        warnings.filterwarnings("ignore", ".*mimetools has been removed",
+                                DeprecationWarning)
+    import mimetools
+
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from StringIO import StringIO
+
+__all__ = ["HTTP", "HTTPResponse", "HTTPConnection",
+           "HTTPException", "NotConnected", "UnknownProtocol",
+           "UnknownTransferEncoding", "UnimplementedFileMode",
+           "IncompleteRead", "InvalidURL", "ImproperConnectionState",
+           "CannotSendRequest", "CannotSendHeader", "ResponseNotReady",
+           "BadStatusLine", "error", "responses"]
+
+HTTP_PORT = 80
+HTTPS_PORT = 443
+
+_UNKNOWN = 'UNKNOWN'
+
+# connection states
+_CS_IDLE = 'Idle'
+_CS_REQ_STARTED = 'Request-started'
+_CS_REQ_SENT = 'Request-sent'
+
+# status codes
+# informational
+CONTINUE = 100
+SWITCHING_PROTOCOLS = 101
+PROCESSING = 102
+
+# successful
+OK = 200
+CREATED = 201
+ACCEPTED = 202
+NON_AUTHORITATIVE_INFORMATION = 203
+NO_CONTENT = 204
+RESET_CONTENT = 205
+PARTIAL_CONTENT = 206
+MULTI_STATUS = 207
+IM_USED = 226
+
+# redirection
+MULTIPLE_CHOICES = 300
+MOVED_PERMANENTLY = 301
+FOUND = 302
+SEE_OTHER = 303
+NOT_MODIFIED = 304
+USE_PROXY = 305
+TEMPORARY_REDIRECT = 307
+
+# client error
+BAD_REQUEST = 400
+UNAUTHORIZED = 401
+PAYMENT_REQUIRED = 402
+FORBIDDEN = 403
+NOT_FOUND = 404
+METHOD_NOT_ALLOWED = 405
+NOT_ACCEPTABLE = 406
+PROXY_AUTHENTICATION_REQUIRED = 407
+REQUEST_TIMEOUT = 408
+CONFLICT = 409
+GONE = 410
+LENGTH_REQUIRED = 411
+PRECONDITION_FAILED = 412
+REQUEST_ENTITY_TOO_LARGE = 413
+REQUEST_URI_TOO_LONG = 414
+UNSUPPORTED_MEDIA_TYPE = 415
+REQUESTED_RANGE_NOT_SATISFIABLE = 416
+EXPECTATION_FAILED = 417
+UNPROCESSABLE_ENTITY = 422
+LOCKED = 423
+FAILED_DEPENDENCY = 424
+UPGRADE_REQUIRED = 426
+
+# server error
+INTERNAL_SERVER_ERROR = 500
+NOT_IMPLEMENTED = 501
+BAD_GATEWAY = 502
+SERVICE_UNAVAILABLE = 503
+GATEWAY_TIMEOUT = 504
+HTTP_VERSION_NOT_SUPPORTED = 505
+INSUFFICIENT_STORAGE = 507
+NOT_EXTENDED = 510
+
+# Mapping status codes to official W3C names
+responses = {
+    100: 'Continue',
+    101: 'Switching Protocols',
+
+    200: 'OK',
+    201: 'Created',
+    202: 'Accepted',
+    203: 'Non-Authoritative Information',
+    204: 'No Content',
+    205: 'Reset Content',
+    206: 'Partial Content',
+
+    300: 'Multiple Choices',
+    301: 'Moved Permanently',
+    302: 'Found',
+    303: 'See Other',
+    304: 'Not Modified',
+    305: 'Use Proxy',
+    306: '(Unused)',
+    307: 'Temporary Redirect',
+
+    400: 'Bad Request',
+    401: 'Unauthorized',
+    402: 'Payment Required',
+    403: 'Forbidden',
+    404: 'Not Found',
+    405: 'Method Not Allowed',
+    406: 'Not Acceptable',
+    407: 'Proxy Authentication Required',
+    408: 'Request Timeout',
+    409: 'Conflict',
+    410: 'Gone',
+    411: 'Length Required',
+    412: 'Precondition Failed',
+    413: 'Request Entity Too Large',
+    414: 'Request-URI Too Long',
+    415: 'Unsupported Media Type',
+    416: 'Requested Range Not Satisfiable',
+    417: 'Expectation Failed',
+
+    500: 'Internal Server Error',
+    501: 'Not Implemented',
+    502: 'Bad Gateway',
+    503: 'Service Unavailable',
+    504: 'Gateway Timeout',
+    505: 'HTTP Version Not Supported',
+}
+
+# maximal amount of data to read at one time in _safe_read
+MAXAMOUNT = 1048576
+
+class HTTPMessage(mimetools.Message):
+
+    def addheader(self, key, value):
+        """Add header for field key handling repeats."""
+        prev = self.dict.get(key)
+        if prev is None:
+            self.dict[key] = value
+        else:
+            combined = ", ".join((prev, value))
+            self.dict[key] = combined
+
+    def addcontinue(self, key, more):
+        """Add more field data from a continuation line."""
+        prev = self.dict[key]
+        self.dict[key] = prev + "\n " + more
+
+    def readheaders(self):
+        """Read header lines.
+
+        Read header lines up to the entirely blank line that terminates them.
+        The (normally blank) line that ends the headers is skipped, but not
+        included in the returned list.  If a non-header line ends the headers,
+        (which is an error), an attempt is made to backspace over it; it is
+        never included in the returned list.
+
+        The variable self.status is set to the empty string if all went well,
+        otherwise it is an error message.  The variable self.headers is a
+        completely uninterpreted list of lines contained in the header (so
+        printing them will reproduce the header exactly as it appears in the
+        file).
+
+        If multiple header fields with the same name occur, they are combined
+        according to the rules in RFC 2616 sec 4.2:
+
+        Appending each subsequent field-value to the first, each separated
+        by a comma. The order in which header fields with the same field-name
+        are received is significant to the interpretation of the combined
+        field value.
+        """
+        # XXX The implementation overrides the readheaders() method of
+        # rfc822.Message.  The base class design isn't amenable to
+        # customized behavior here so the method here is a copy of the
+        # base class code with a few small changes.
+
+        self.dict = {}
+        self.unixfrom = ''
+        self.headers = hlist = []
+        self.status = ''
+        headerseen = ""
+        firstline = 1
+        startofline = unread = tell = None
+        if hasattr(self.fp, 'unread'):
+            unread = self.fp.unread
+        elif self.seekable:
+            tell = self.fp.tell
+        while True:
+            if tell:
+                try:
+                    startofline = tell()
+                except IOError:
+                    startofline = tell = None
+                    self.seekable = 0
+            line = self.fp.readline()
+            if not line:
+                self.status = 'EOF in headers'
+                break
+            # Skip unix From name time lines
+            if firstline and line.startswith('From '):
+                self.unixfrom = self.unixfrom + line
+                continue
+            firstline = 0
+            if headerseen and line[0] in ' \t':
+                # XXX Not sure if continuation lines are handled properly
+                # for http and/or for repeating headers
+                # It's a continuation line.
+                hlist.append(line)
+                self.addcontinue(headerseen, line.strip())
+                continue
+            elif self.iscomment(line):
+                # It's a comment.  Ignore it.
+                continue
+            elif self.islast(line):
+                # Note! No pushback here!  The delimiter line gets eaten.
+                break
+            headerseen = self.isheader(line)
+            if headerseen:
+                # It's a legal header line, save it.
+                hlist.append(line)
+                self.addheader(headerseen, line[len(headerseen)+1:].strip())
+                continue
+            else:
+                # It's not a header line; throw it back and stop here.
+                if not self.dict:
+                    self.status = 'No headers'
+                else:
+                    self.status = 'Non-header line where header expected'
+                # Try to undo the read.
+                if unread:
+                    unread(line)
+                elif tell:
+                    self.fp.seek(startofline)
+                else:
+                    self.status = self.status + '; bad seek'
+                break
+
+class HTTPResponse:
+
+    # strict: If true, raise BadStatusLine if the status line can't be
+    # parsed as a valid HTTP/1.0 or 1.1 status line.  By default it is
+    # false because it prevents clients from talking to HTTP/0.9
+    # servers.  Note that a response with a sufficiently corrupted
+    # status line will look like an HTTP/0.9 response.
+
+    # See RFC 2616 sec 19.6 and RFC 1945 sec 6 for details.
+
+    def __init__(self, sock, debuglevel=0, strict=0, method=None, buffering=False):
+        if buffering:
+            # The caller won't be using any sock.recv() calls, so buffering
+            # is fine and recommended for performance.
+            self.fp = sock.makefile('rb')
+        else:
+            # The buffer size is specified as zero, because the headers of
+            # the response are read with readline().  If the reads were
+            # buffered the readline() calls could consume some of the
+            # response, which make be read via a recv() on the underlying
+            # socket.
+            self.fp = sock.makefile('rb', 0)
+        self.debuglevel = debuglevel
+        self.strict = strict
+        self._method = method
+
+        self.msg = None
+
+        # from the Status-Line of the response
+        self.version = _UNKNOWN # HTTP-Version
+        self.status = _UNKNOWN  # Status-Code
+        self.reason = _UNKNOWN  # Reason-Phrase
+
+        self.chunked = _UNKNOWN         # is "chunked" being used?
+        self.chunk_left = _UNKNOWN      # bytes left to read in current chunk
+        self.length = _UNKNOWN          # number of bytes left in response
+        self.will_close = _UNKNOWN      # conn will close at end of response
+
+    def _read_status(self):
+        # Initialize with Simple-Response defaults
+        line = self.fp.readline()
+        if self.debuglevel > 0:
+            print "reply:", repr(line)
+        if not line:
+            # Presumably, the server closed the connection before
+            # sending a valid response.
+            raise BadStatusLine(line)
+        try:
+            [version, status, reason] = line.split(None, 2)
+        except ValueError:
+            try:
+                [version, status] = line.split(None, 1)
+                reason = ""
+            except ValueError:
+                # empty version will cause next test to fail and status
+                # will be treated as 0.9 response.
+                version = ""
+        if not version.startswith('HTTP/'):
+            if self.strict:
+                self.close()
+                raise BadStatusLine(line)
+            else:
+                # assume it's a Simple-Response from an 0.9 server
+                self.fp = LineAndFileWrapper(line, self.fp)
+                return "HTTP/0.9", 200, ""
+
+        # The status code is a three-digit number
+        try:
+            status = int(status)
+            if status < 100 or status > 999:
+                raise BadStatusLine(line)
+        except ValueError:
+            raise BadStatusLine(line)
+        return version, status, reason
+
+    def begin(self):
+        if self.msg is not None:
+            # we've already started reading the response
+            return
+
+        # read until we get a non-100 response
+        while True:
+            version, status, reason = self._read_status()
+            if status != CONTINUE:
+                break
+            # skip the header from the 100 response
+            while True:
+                skip = self.fp.readline().strip()
+                if not skip:
+                    break
+                if self.debuglevel > 0:
+                    print "header:", skip
+
+        self.status = status
+        self.reason = reason.strip()
+        if version == 'HTTP/1.0':
+            self.version = 10
+        elif version.startswith('HTTP/1.'):
+            self.version = 11   # use HTTP/1.1 code for HTTP/1.x where x>=1
+        elif version == 'HTTP/0.9':
+            self.version = 9
+        else:
+            raise UnknownProtocol(version)
+
+        if self.version == 9:
+            self.length = None
+            self.chunked = 0
+            self.will_close = 1
+            self.msg = HTTPMessage(StringIO())
+            return
+
+        self.msg = HTTPMessage(self.fp, 0)
+        if self.debuglevel > 0:
+            for hdr in self.msg.headers:
+                print "header:", hdr,
+
+        # don't let the msg keep an fp
+        self.msg.fp = None
+
+        # are we using the chunked-style of transfer encoding?
+        tr_enc = self.msg.getheader('transfer-encoding')
+        if tr_enc and tr_enc.lower() == "chunked":
+            self.chunked = 1
+            self.chunk_left = None
+        else:
+            self.chunked = 0
+
+        # will the connection close at the end of the response?
+        self.will_close = self._check_close()
+
+        # do we have a Content-Length?
+        # NOTE: RFC 2616, S4.4, #3 says we ignore this if tr_enc is "chunked"
+        length = self.msg.getheader('content-length')
+        if length and not self.chunked:
+            try:
+                self.length = int(length)
+            except ValueError:
+                self.length = None
+            else:
+                if self.length < 0:  # ignore nonsensical negative lengths
+                    self.length = None
+        else:
+            self.length = None
+
+        # does the body have a fixed length? (of zero)
+        if (status == NO_CONTENT or status == NOT_MODIFIED or
+            100 <= status < 200 or      # 1xx codes
+            self._method == 'HEAD'):
+            self.length = 0
+
+        # if the connection remains open, and we aren't using chunked, and
+        # a content-length was not provided, then assume that the connection
+        # WILL close.
+        if not self.will_close and \
+           not self.chunked and \
+           self.length is None:
+            self.will_close = 1
+
+    def _check_close(self):
+        conn = self.msg.getheader('connection')
+        if self.version == 11:
+            # An HTTP/1.1 proxy is assumed to stay open unless
+            # explicitly closed.
+            conn = self.msg.getheader('connection')
+            if conn and "close" in conn.lower():
+                return True
+            return False
+
+        # Some HTTP/1.0 implementations have support for persistent
+        # connections, using rules different than HTTP/1.1.
+
+        # For older HTTP, Keep-Alive indicates persistent connection.
+        if self.msg.getheader('keep-alive'):
+            return False
+
+        # At least Akamai returns a "Connection: Keep-Alive" header,
+        # which was supposed to be sent by the client.
+        if conn and "keep-alive" in conn.lower():
+            return False
+
+        # Proxy-Connection is a netscape hack.
+        pconn = self.msg.getheader('proxy-connection')
+        if pconn and "keep-alive" in pconn.lower():
+            return False
+
+        # otherwise, assume it will close
+        return True
+
+    def close(self):
+        if self.fp:
+            self.fp.close()
+            self.fp = None
+
+    def isclosed(self):
+        # NOTE: it is possible that we will not ever call self.close(). This
+        #       case occurs when will_close is TRUE, length is None, and we
+        #       read up to the last byte, but NOT past it.
+        #
+        # IMPLIES: if will_close is FALSE, then self.close() will ALWAYS be
+        #          called, meaning self.isclosed() is meaningful.
+        return self.fp is None
+
+    # XXX It would be nice to have readline and __iter__ for this, too.
+
+    def read(self, amt=None):
+        if self.fp is None:
+            return ''
+
+        if self._method == 'HEAD':
+            self.close()
+            return ''
+
+        if self.chunked:
+            return self._read_chunked(amt)
+
+        if amt is None:
+            # unbounded read
+            if self.length is None:
+                s = self.fp.read()
+            else:
+                s = self._safe_read(self.length)
+                self.length = 0
+            self.close()        # we read everything
+            return s
+
+        if self.length is not None:
+            if amt > self.length:
+                # clip the read to the "end of response"
+                amt = self.length
+
+        # we do not use _safe_read() here because this may be a .will_close
+        # connection, and the user is reading more bytes than will be provided
+        # (for example, reading in 1k chunks)
+        s = self.fp.read(amt)
+        if self.length is not None:
+            self.length -= len(s)
+            if not self.length:
+                self.close()
+        return s
+
+    def _read_chunked(self, amt):
+        assert self.chunked != _UNKNOWN
+        chunk_left = self.chunk_left
+        value = []
+        while True:
+            if chunk_left is None:
+                line = self.fp.readline()
+                i = line.find(';')
+                if i >= 0:
+                    line = line[:i] # strip chunk-extensions
+                try:
+                    chunk_left = int(line, 16)
+                except ValueError:
+                    # close the connection as protocol synchronisation is
+                    # probably lost
+                    self.close()
+                    raise IncompleteRead(''.join(value))
+                if chunk_left == 0:
+                    break
+            if amt is None:
+                value.append(self._safe_read(chunk_left))
+            elif amt < chunk_left:
+                value.append(self._safe_read(amt))
+                self.chunk_left = chunk_left - amt
+                return ''.join(value)
+            elif amt == chunk_left:
+                value.append(self._safe_read(amt))
+                self._safe_read(2)  # toss the CRLF at the end of the chunk
+                self.chunk_left = None
+                return ''.join(value)
+            else:
+                value.append(self._safe_read(chunk_left))
+                amt -= chunk_left
+
+            # we read the whole chunk, get another
+            self._safe_read(2)      # toss the CRLF at the end of the chunk
+            chunk_left = None
+
+        # read and discard trailer up to the CRLF terminator
+        ### note: we shouldn't have any trailers!
+        while True:
+            line = self.fp.readline()
+            if not line:
+                # a vanishingly small number of sites EOF without
+                # sending the trailer
+                break
+            if line == '\r\n':
+                break
+
+        # we read everything; close the "file"
+        self.close()
+
+        return ''.join(value)
+
+    def _safe_read(self, amt):
+        """Read the number of bytes requested, compensating for partial reads.
+
+        Normally, we have a blocking socket, but a read() can be interrupted
+        by a signal (resulting in a partial read).
+
+        Note that we cannot distinguish between EOF and an interrupt when zero
+        bytes have been read. IncompleteRead() will be raised in this
+        situation.
+
+        This function should be used when <amt> bytes "should" be present for
+        reading. If the bytes are truly not available (due to EOF), then the
+        IncompleteRead exception can be used to detect the problem.
+        """
+        # NOTE(gps): As of svn r74426 socket._fileobject.read(x) will never
+        # return less than x bytes unless EOF is encountered.  It now handles
+        # signal interruptions (socket.error EINTR) internally.  This code
+        # never caught that exception anyways.  It seems largely pointless.
+        # self.fp.read(amt) will work fine.
+        s = []
+        while amt > 0:
+            chunk = self.fp.read(min(amt, MAXAMOUNT))
+            if not chunk:
+                raise IncompleteRead(''.join(s), amt)
+            s.append(chunk)
+            amt -= len(chunk)
+        return ''.join(s)
+
+    def fileno(self):
+        return self.fp.fileno()
+
+    def getheader(self, name, default=None):
+        if self.msg is None:
+            raise ResponseNotReady()
+        return self.msg.getheader(name, default)
+
+    def getheaders(self):
+        """Return list of (header, value) tuples."""
+        if self.msg is None:
+            raise ResponseNotReady()
+        return self.msg.items()
+
+
+class HTTPConnection:
+
+    _http_vsn = 11
+    _http_vsn_str = 'HTTP/1.1'
+
+    response_class = HTTPResponse
+    default_port = HTTP_PORT
+    auto_open = 1
+    debuglevel = 0
+    strict = 0
+
+    def __init__(self, host, port=None, strict=None,
+                 timeout=socket._GLOBAL_DEFAULT_TIMEOUT, source_address=None):
+        self.timeout = timeout
+        self.source_address = source_address
+        self.sock = None
+        self._buffer = []
+        self.__response = None
+        self.__state = _CS_IDLE
+        self._method = None
+        self._tunnel_host = None
+        self._tunnel_port = None
+        self._tunnel_headers = {}
+
+        self._set_hostport(host, port)
+        if strict is not None:
+            self.strict = strict
+
+    def set_tunnel(self, host, port=None, headers=None):
+        """ Sets up the host and the port for the HTTP CONNECT Tunnelling.
+
+        The headers argument should be a mapping of extra HTTP headers
+        to send with the CONNECT request.
+        """
+        self._tunnel_host = host
+        self._tunnel_port = port
+        if headers:
+            self._tunnel_headers = headers
+        else:
+            self._tunnel_headers.clear()
+
+    def _set_hostport(self, host, port):
+        if port is None:
+            i = host.rfind(':')
+            j = host.rfind(']')         # ipv6 addresses have [...]
+            if i > j:
+                try:
+                    port = int(host[i+1:])
+                except ValueError:
+                    raise InvalidURL("nonnumeric port: '%s'" % host[i+1:])
+                host = host[:i]
+            else:
+                port = self.default_port
+            if host and host[0] == '[' and host[-1] == ']':
+                host = host[1:-1]
+        self.host = host
+        self.port = port
+
+    def set_debuglevel(self, level):
+        self.debuglevel = level
+
+    def _tunnel(self):
+        self._set_hostport(self._tunnel_host, self._tunnel_port)
+        self.send("CONNECT %s:%d HTTP/1.0\r\n" % (self.host, self.port))
+        for header, value in self._tunnel_headers.iteritems():
+            self.send("%s: %s\r\n" % (header, value))
+        self.send("\r\n")
+        response = self.response_class(self.sock, strict = self.strict,
+                                       method = self._method)
+        (version, code, message) = response._read_status()
+
+        if code != 200:
+            self.close()
+            raise socket.error("Tunnel connection failed: %d %s" % (code,
+                                                                    message.strip()))
+        while True:
+            line = response.fp.readline()
+            if line == '\r\n': break
+
+
+    def connect(self):
+        """Connect to the host and port specified in __init__."""
+        self.sock = socket.create_connection((self.host,self.port),
+                                             self.timeout, self.source_address)
+
+        if self._tunnel_host:
+            self._tunnel()
+
+    def close(self):
+        """Close the connection to the HTTP server."""
+        if self.sock:
+            self.sock.close()   # close it manually... there may be other refs
+            self.sock = None
+        if self.__response:
+            self.__response.close()
+            self.__response = None
+        self.__state = _CS_IDLE
+
+    def send(self, data):
+        """Send `data' to the server."""
+        if self.sock is None:
+            if self.auto_open:
+                self.connect()
+            else:
+                raise NotConnected()
+
+        if self.debuglevel > 0:
+            print "send:", repr(data)
+        blocksize = 8192
+        if hasattr(data,'read') and not isinstance(data, array):
+            if self.debuglevel > 0: print "sendIng a read()able"
+            datablock = data.read(blocksize)
+            while datablock:
+                self.sock.sendall(datablock)
+                datablock = data.read(blocksize)
+        else:
+            self.sock.sendall(data)
+
+    def _output(self, s):
+        """Add a line of output to the current request buffer.
+
+        Assumes that the line does *not* end with \\r\\n.
+        """
+        self._buffer.append(s)
+
+    def _send_output(self, message_body=None):
+        """Send the currently buffered request and clear the buffer.
+
+        Appends an extra \\r\\n to the buffer.
+        A message_body may be specified, to be appended to the request.
+        """
+        self._buffer.extend(("", ""))
+        msg = "\r\n".join(self._buffer)
+        del self._buffer[:]
+        # If msg and message_body are sent in a single send() call,
+        # it will avoid performance problems caused by the interaction
+        # between delayed ack and the Nagle algorithim.
+        if isinstance(message_body, str):
+            msg += message_body
+            message_body = None
+        self.send(msg)
+        if message_body is not None:
+            #message_body was not a string (i.e. it is a file) and
+            #we must run the risk of Nagle
+            self.send(message_body)
+
+    def putrequest(self, method, url, skip_host=0, skip_accept_encoding=0):
+        """Send a request to the server.
+
+        `method' specifies an HTTP request method, e.g. 'GET'.
+        `url' specifies the object being requested, e.g. '/index.html'.
+        `skip_host' if True does not add automatically a 'Host:' header
+        `skip_accept_encoding' if True does not add automatically an
+           'Accept-Encoding:' header
+        """
+
+        # if a prior response has been completed, then forget about it.
+        if self.__response and self.__response.isclosed():
+            self.__response = None
+
+
+        # in certain cases, we cannot issue another request on this connection.
+        # this occurs when:
+        #   1) we are in the process of sending a request.   (_CS_REQ_STARTED)
+        #   2) a response to a previous request has signalled that it is going
+        #      to close the connection upon completion.
+        #   3) the headers for the previous response have not been read, thus
+        #      we cannot determine whether point (2) is true.   (_CS_REQ_SENT)
+        #
+        # if there is no prior response, then we can request at will.
+        #
+        # if point (2) is true, then we will have passed the socket to the
+        # response (effectively meaning, "there is no prior response"), and
+        # will open a new one when a new request is made.
+        #
+        # Note: if a prior response exists, then we *can* start a new request.
+        #       We are not allowed to begin fetching the response to this new
+        #       request, however, until that prior response is complete.
+        #
+        if self.__state == _CS_IDLE:
+            self.__state = _CS_REQ_STARTED
+        else:
+            raise CannotSendRequest()
+
+        # Save the method we use, we need it later in the response phase
+        self._method = method
+        if not url:
+            url = '/'
+        hdr = '%s %s %s' % (method, url, self._http_vsn_str)
+
+        self._output(hdr)
+
+        if self._http_vsn == 11:
+            # Issue some standard headers for better HTTP/1.1 compliance
+
+            if not skip_host:
+                # this header is issued *only* for HTTP/1.1
+                # connections. more specifically, this means it is
+                # only issued when the client uses the new
+                # HTTPConnection() class. backwards-compat clients
+                # will be using HTTP/1.0 and those clients may be
+                # issuing this header themselves. we should NOT issue
+                # it twice; some web servers (such as Apache) barf
+                # when they see two Host: headers
+
+                # If we need a non-standard port,include it in the
+                # header.  If the request is going through a proxy,
+                # but the host of the actual URL, not the host of the
+                # proxy.
+
+                netloc = ''
+                if url.startswith('http'):
+                    nil, netloc, nil, nil, nil = urlsplit(url)
+
+                if netloc:
+                    try:
+                        netloc_enc = netloc.encode("ascii")
+                    except UnicodeEncodeError:
+                        netloc_enc = netloc.encode("idna")
+                    self.putheader('Host', netloc_enc)
+                else:
+                    try:
+                        host_enc = self.host.encode("ascii")
+                    except UnicodeEncodeError:
+                        host_enc = self.host.encode("idna")
+                    # Wrap the IPv6 Host Header with [] (RFC 2732)
+                    if host_enc.find(':') >= 0:
+                        host_enc = "[" + host_enc + "]"
+                    if self.port == self.default_port:
+                        self.putheader('Host', host_enc)
+                    else:
+                        self.putheader('Host', "%s:%s" % (host_enc, self.port))
+
+            # note: we are assuming that clients will not attempt to set these
+            #       headers since *this* library must deal with the
+            #       consequences. this also means that when the supporting
+            #       libraries are updated to recognize other forms, then this
+            #       code should be changed (removed or updated).
+
+            # we only want a Content-Encoding of "identity" since we don't
+            # support encodings such as x-gzip or x-deflate.
+            if not skip_accept_encoding:
+                self.putheader('Accept-Encoding', 'identity')
+
+            # we can accept "chunked" Transfer-Encodings, but no others
+            # NOTE: no TE header implies *only* "chunked"
+            #self.putheader('TE', 'chunked')
+
+            # if TE is supplied in the header, then it must appear in a
+            # Connection header.
+            #self.putheader('Connection', 'TE')
+
+        else:
+            # For HTTP/1.0, the server will assume "not chunked"
+            pass
+
+    def putheader(self, header, *values):
+        """Send a request header line to the server.
+
+        For example: h.putheader('Accept', 'text/html')
+        """
+        if self.__state != _CS_REQ_STARTED:
+            raise CannotSendHeader()
+
+        hdr = '%s: %s' % (header, '\r\n\t'.join([str(v) for v in values]))
+        self._output(hdr)
+
+    def endheaders(self, message_body=None):
+        """Indicate that the last header line has been sent to the server.
+
+        This method sends the request to the server.  The optional
+        message_body argument can be used to pass message body
+        associated with the request.  The message body will be sent in
+        the same packet as the message headers if possible.  The
+        message_body should be a string.
+        """
+        if self.__state == _CS_REQ_STARTED:
+            self.__state = _CS_REQ_SENT
+        else:
+            raise CannotSendHeader()
+        self._send_output(message_body)
+
+    def request(self, method, url, body=None, headers={}):
+        """Send a complete request to the server."""
+        self._send_request(method, url, body, headers)
+
+    def _set_content_length(self, body):
+        # Set the content-length based on the body.
+        thelen = None
+        try:
+            thelen = str(len(body))
+        except TypeError, te:
+            # If this is a file-like object, try to
+            # fstat its file descriptor
+            try:
+                thelen = str(os.fstat(body.fileno()).st_size)
+            except (AttributeError, OSError):
+                # Don't send a length if this failed
+                if self.debuglevel > 0: print "Cannot stat!!"
+
+        if thelen is not None:
+            self.putheader('Content-Length', thelen)
+
+    def _send_request(self, method, url, body, headers):
+        # Honor explicitly requested Host: and Accept-Encoding: headers.
+        header_names = dict.fromkeys([k.lower() for k in headers])
+        skips = {}
+        if 'host' in header_names:
+            skips['skip_host'] = 1
+        if 'accept-encoding' in header_names:
+            skips['skip_accept_encoding'] = 1
+
+        self.putrequest(method, url, **skips)
+
+        if body and ('content-length' not in header_names):
+            self._set_content_length(body)
+        for hdr, value in headers.iteritems():
+            self.putheader(hdr, value)
+        self.endheaders(body)
+
+    def getresponse(self, buffering=False):
+        "Get the response from the server."
+
+        # if a prior response has been completed, then forget about it.
+        if self.__response and self.__response.isclosed():
+            self.__response = None
+
+        #
+        # if a prior response exists, then it must be completed (otherwise, we
+        # cannot read this response's header to determine the connection-close
+        # behavior)
+        #
+        # note: if a prior response existed, but was connection-close, then the
+        # socket and response were made independent of this HTTPConnection
+        # object since a new request requires that we open a whole new
+        # connection
+        #
+        # this means the prior response had one of two states:
+        #   1) will_close: this connection was reset and the prior socket and
+        #                  response operate independently
+        #   2) persistent: the response was retained and we await its
+        #                  isclosed() status to become true.
+        #
+        if self.__state != _CS_REQ_SENT or self.__response:
+            raise ResponseNotReady()
+
+        args = (self.sock,)
+        kwds = {"strict":self.strict, "method":self._method}
+        if self.debuglevel > 0:
+            args += (self.debuglevel,)
+        if buffering:
+            #only add this keyword if non-default, for compatibility with
+            #other response_classes.
+            kwds["buffering"] = True;
+        response = self.response_class(*args, **kwds)
+
+        try:
+            response.begin()
+        except:
+            response.close()
+            raise
+        assert response.will_close != _UNKNOWN
+        self.__state = _CS_IDLE
+
+        if response.will_close:
+            # this effectively passes the connection to the response
+            self.close()
+        else:
+            # remember this, so we can tell when it is complete
+            self.__response = response
+
+        return response
+
+
+class HTTP:
+    "Compatibility class with httplib.py from 1.5."
+
+    _http_vsn = 10
+    _http_vsn_str = 'HTTP/1.0'
+
+    debuglevel = 0
+
+    _connection_class = HTTPConnection
+
+    def __init__(self, host='', port=None, strict=None):
+        "Provide a default host, since the superclass requires one."
+
+        # some joker passed 0 explicitly, meaning default port
+        if port == 0:
+            port = None
+
+        # Note that we may pass an empty string as the host; this will throw
+        # an error when we attempt to connect. Presumably, the client code
+        # will call connect before then, with a proper host.
+        self._setup(self._connection_class(host, port, strict))
+
+    def _setup(self, conn):
+        self._conn = conn
+
+        # set up delegation to flesh out interface
+        self.send = conn.send
+        self.putrequest = conn.putrequest
+        self.putheader = conn.putheader
+        self.endheaders = conn.endheaders
+        self.set_debuglevel = conn.set_debuglevel
+
+        conn._http_vsn = self._http_vsn
+        conn._http_vsn_str = self._http_vsn_str
+
+        self.file = None
+
+    def connect(self, host=None, port=None):
+        "Accept arguments to set the host/port, since the superclass doesn't."
+
+        if host is not None:
+            self._conn._set_hostport(host, port)
+        self._conn.connect()
+
+    def getfile(self):
+        "Provide a getfile, since the superclass' does not use this concept."
+        return self.file
+
+    def getreply(self, buffering=False):
+        """Compat definition since superclass does not define it.
+
+        Returns a tuple consisting of:
+        - server status code (e.g. '200' if all goes well)
+        - server "reason" corresponding to status code
+        - any RFC822 headers in the response from the server
+        """
+        try:
+            if not buffering:
+                response = self._conn.getresponse()
+            else:
+                #only add this keyword if non-default for compatibility
+                #with other connection classes
+                response = self._conn.getresponse(buffering)
+        except BadStatusLine, e:
+            ### hmm. if getresponse() ever closes the socket on a bad request,
+            ### then we are going to have problems with self.sock
+
+            ### should we keep this behavior? do people use it?
+            # keep the socket open (as a file), and return it
+            self.file = self._conn.sock.makefile('rb', 0)
+
+            # close our socket -- we want to restart after any protocol error
+            self.close()
+
+            self.headers = None
+            return -1, e.line, None
+
+        self.headers = response.msg
+        self.file = response.fp
+        return response.status, response.reason, response.msg
+
+    def close(self):
+        self._conn.close()
+
+        # note that self.file == response.fp, which gets closed by the
+        # superclass. just clear the object ref here.
+        ### hmm. messy. if status==-1, then self.file is owned by us.
+        ### well... we aren't explicitly closing, but losing this ref will
+        ### do it
+        self.file = None
+
+try:
+    import ssl
+except ImportError:
+    pass
+else:
+    class HTTPSConnection(HTTPConnection):
+        "This class allows communication via SSL."
+
+        default_port = HTTPS_PORT
+
+        def __init__(self, host, port=None, key_file=None, cert_file=None,
+                     strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
+                     source_address=None):
+            HTTPConnection.__init__(self, host, port, strict, timeout,
+                                    source_address)
+            self.key_file = key_file
+            self.cert_file = cert_file
+
+        def connect(self):
+            "Connect to a host on a given (SSL) port."
+
+            sock = socket.create_connection((self.host, self.port),
+                                            self.timeout, self.source_address)
+            if self._tunnel_host:
+                self.sock = sock
+                self._tunnel()
+            self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file)
+
+    __all__.append("HTTPSConnection")
+
+    class HTTPS(HTTP):
+        """Compatibility with 1.5 httplib interface
+
+        Python 1.5.2 did not have an HTTPS class, but it defined an
+        interface for sending http requests that is also useful for
+        https.
+        """
+
+        _connection_class = HTTPSConnection
+
+        def __init__(self, host='', port=None, key_file=None, cert_file=None,
+                     strict=None):
+            # provide a default host, pass the X509 cert info
+
+            # urf. compensate for bad input.
+            if port == 0:
+                port = None
+            self._setup(self._connection_class(host, port, key_file,
+                                               cert_file, strict))
+
+            # we never actually use these for anything, but we keep them
+            # here for compatibility with post-1.5.2 CVS.
+            self.key_file = key_file
+            self.cert_file = cert_file
+
+
+    def FakeSocket (sock, sslobj):
+        warnings.warn("FakeSocket is deprecated, and won't be in 3.x.  " +
+                      "Use the result of ssl.wrap_socket() directly instead.",
+                      DeprecationWarning, stacklevel=2)
+        return sslobj
+
+
+class HTTPException(Exception):
+    # Subclasses that define an __init__ must call Exception.__init__
+    # or define self.args.  Otherwise, str() will fail.
+    pass
+
+class NotConnected(HTTPException):
+    pass
+
+class InvalidURL(HTTPException):
+    pass
+
+class UnknownProtocol(HTTPException):
+    def __init__(self, version):
+        self.args = version,
+        self.version = version
+
+class UnknownTransferEncoding(HTTPException):
+    pass
+
+class UnimplementedFileMode(HTTPException):
+    pass
+
+class IncompleteRead(HTTPException):
+    def __init__(self, partial, expected=None):
+        self.args = partial,
+        self.partial = partial
+        self.expected = expected
+    def __repr__(self):
+        if self.expected is not None:
+            e = ', %i more expected' % self.expected
+        else:
+            e = ''
+        return 'IncompleteRead(%i bytes read%s)' % (len(self.partial), e)
+    def __str__(self):
+        return repr(self)
+
+class ImproperConnectionState(HTTPException):
+    pass
+
+class CannotSendRequest(ImproperConnectionState):
+    pass
+
+class CannotSendHeader(ImproperConnectionState):
+    pass
+
+class ResponseNotReady(ImproperConnectionState):
+    pass
+
+class BadStatusLine(HTTPException):
+    def __init__(self, line):
+        if not line:
+            line = repr(line)
+        self.args = line,
+        self.line = line
+
+# for backwards compatibility
+error = HTTPException
+
+class LineAndFileWrapper:
+    """A limited file-like object for HTTP/0.9 responses."""
+
+    # The status-line parsing code calls readline(), which normally
+    # get the HTTP status line.  For a 0.9 response, however, this is
+    # actually the first line of the body!  Clients need to get a
+    # readable file object that contains that line.
+
+    def __init__(self, line, file):
+        self._line = line
+        self._file = file
+        self._line_consumed = 0
+        self._line_offset = 0
+        self._line_left = len(line)
+
+    def __getattr__(self, attr):
+        return getattr(self._file, attr)
+
+    def _done(self):
+        # called when the last byte is read from the line.  After the
+        # call, all read methods are delegated to the underlying file
+        # object.
+        self._line_consumed = 1
+        self.read = self._file.read
+        self.readline = self._file.readline
+        self.readlines = self._file.readlines
+
+    def read(self, amt=None):
+        if self._line_consumed:
+            return self._file.read(amt)
+        assert self._line_left
+        if amt is None or amt > self._line_left:
+            s = self._line[self._line_offset:]
+            self._done()
+            if amt is None:
+                return s + self._file.read()
+            else:
+                return s + self._file.read(amt - len(s))
+        else:
+            assert amt <= self._line_left
+            i = self._line_offset
+            j = i + amt
+            s = self._line[i:j]
+            self._line_offset = j
+            self._line_left -= amt
+            if self._line_left == 0:
+                self._done()
+            return s
+
+    def readline(self):
+        if self._line_consumed:
+            return self._file.readline()
+        assert self._line_left
+        s = self._line[self._line_offset:]
+        self._done()
+        return s
+
+    def readlines(self, size=None):
+        if self._line_consumed:
+            return self._file.readlines(size)
+        assert self._line_left
+        L = [self._line[self._line_offset:]]
+        self._done()
+        if size is None:
+            return L + self._file.readlines()
+        else:
+            return L + self._file.readlines(size)
+
+def test():
+    """Test this module.
+
+    A hodge podge of tests collected here, because they have too many
+    external dependencies for the regular test suite.
+    """
+
+    import sys
+    import getopt
+    opts, args = getopt.getopt(sys.argv[1:], 'd')
+    dl = 0
+    for o, a in opts:
+        if o == '-d': dl = dl + 1
+    host = 'www.python.org'
+    selector = '/'
+    if args[0:]: host = args[0]
+    if args[1:]: selector = args[1]
+    h = HTTP()
+    h.set_debuglevel(dl)
+    h.connect(host)
+    h.putrequest('GET', selector)
+    h.endheaders()
+    status, reason, headers = h.getreply()
+    print 'status =', status
+    print 'reason =', reason
+    print "read", len(h.getfile().read())
+    print
+    if headers:
+        for header in headers.headers: print header.strip()
+    print
+
+    # minimal test that code to extract host from url works
+    class HTTP11(HTTP):
+        _http_vsn = 11
+        _http_vsn_str = 'HTTP/1.1'
+
+    h = HTTP11('www.python.org')
+    h.putrequest('GET', 'http://www.python.org/~jeremy/')
+    h.endheaders()
+    h.getreply()
+    h.close()
+
+    try:
+        import ssl
+    except ImportError:
+        pass
+    else:
+
+        for host, selector in (('sourceforge.net', '/projects/python'),
+                               ):
+            print "https://%s%s" % (host, selector)
+            hs = HTTPS()
+            hs.set_debuglevel(dl)
+            hs.connect(host)
+            hs.putrequest('GET', selector)
+            hs.endheaders()
+            status, reason, headers = hs.getreply()
+            print 'status =', status
+            print 'reason =', reason
+            print "read", len(hs.getfile().read())
+            print
+            if headers:
+                for header in headers.headers: print header.strip()
+            print
+
+if __name__ == '__main__':
+    test()
diff --git a/lib-python/modified-2.7/tarfile.py b/lib-python/modified-2.7/tarfile.py
--- a/lib-python/modified-2.7/tarfile.py
+++ b/lib-python/modified-2.7/tarfile.py
@@ -252,8 +252,8 @@
        the high bit set. So we calculate two checksums, unsigned and
        signed.
     """
-    unsigned_chksum = 256 + sum(struct.unpack("148B8x356B", buf[:512]))
-    signed_chksum = 256 + sum(struct.unpack("148b8x356b", buf[:512]))
+    unsigned_chksum = 256 + sum(struct.unpack("148B", buf[:148]) + struct.unpack("356B", buf[156:512]))
+    signed_chksum = 256 + sum(struct.unpack("148b", buf[:148]) + struct.unpack("356b", buf[156:512]))
     return unsigned_chksum, signed_chksum
 
 def copyfileobj(src, dst, length=None):
@@ -265,6 +265,7 @@
     if length is None:
         shutil.copyfileobj(src, dst)
         return
+
     BUFSIZE = 16 * 1024
     blocks, remainder = divmod(length, BUFSIZE)
     for b in xrange(blocks):
@@ -801,19 +802,19 @@
         if self.closed:
             raise ValueError("I/O operation on closed file")
 
+        buf = ""
         if self.buffer:
             if size is None:
-                buf = self.buffer + self.fileobj.read()
+                buf = self.buffer
                 self.buffer = ""
             else:
                 buf = self.buffer[:size]
                 self.buffer = self.buffer[size:]
-                buf += self.fileobj.read(size - len(buf))
+
+        if size is None:
+            buf += self.fileobj.read()
         else:
-            if size is None:
-                buf = self.fileobj.read()
-            else:
-                buf = self.fileobj.read(size)
+            buf += self.fileobj.read(size - len(buf))
 
         self.position += len(buf)
         return buf
diff --git a/lib-python/modified-2.7/test/test_multiprocessing.py b/lib-python/modified-2.7/test/test_multiprocessing.py
--- a/lib-python/modified-2.7/test/test_multiprocessing.py
+++ b/lib-python/modified-2.7/test/test_multiprocessing.py
@@ -510,7 +510,6 @@
 
         p.join()
 
-    @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
     def test_qsize(self):
         q = self.Queue()
         try:
@@ -532,7 +531,6 @@
             time.sleep(DELTA)
             q.task_done()
 
-    @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
     def test_task_done(self):
         queue = self.JoinableQueue()
 
@@ -1091,7 +1089,6 @@
 class _TestPoolWorkerLifetime(BaseTestCase):
 
     ALLOWED_TYPES = ('processes', )
-    @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
     def test_pool_worker_lifetime(self):
         p = multiprocessing.Pool(3, maxtasksperchild=10)
         self.assertEqual(3, len(p._pool))
@@ -1280,7 +1277,6 @@
         queue = manager.get_queue()
         queue.put('hello world')
 
-    @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
     def test_rapid_restart(self):
         authkey = os.urandom(32)
         manager = QueueManager(
@@ -1297,6 +1293,7 @@
         queue = manager.get_queue()
         self.assertEqual(queue.get(), 'hello world')
         del queue
+        test_support.gc_collect()
         manager.shutdown()
         manager = QueueManager(
             address=addr, authkey=authkey, serializer=SERIALIZER)
@@ -1573,7 +1570,6 @@
 
     ALLOWED_TYPES = ('processes',)
 
-    @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
     def test_heap(self):
         iterations = 5000
         maxblocks = 50
diff --git a/lib-python/modified-2.7/test/test_sys_settrace.py b/lib-python/modified-2.7/test/test_sys_settrace.py
--- a/lib-python/modified-2.7/test/test_sys_settrace.py
+++ b/lib-python/modified-2.7/test/test_sys_settrace.py
@@ -286,11 +286,11 @@
         self.compare_events(func.func_code.co_firstlineno,
                             tracer.events, func.events)
 
-    def set_and_retrieve_none(self):
+    def test_set_and_retrieve_none(self):
         sys.settrace(None)
         assert sys.gettrace() is None
 
-    def set_and_retrieve_func(self):
+    def test_set_and_retrieve_func(self):
         def fn(*args):
             pass
 
diff --git a/lib-python/modified-2.7/test/test_urllib2.py b/lib-python/modified-2.7/test/test_urllib2.py
--- a/lib-python/modified-2.7/test/test_urllib2.py
+++ b/lib-python/modified-2.7/test/test_urllib2.py
@@ -307,6 +307,9 @@
     def getresponse(self):
         return MockHTTPResponse(MockFile(), {}, 200, "OK")
 
+    def close(self):
+        pass
+
 class MockHandler:
     # useful for testing handler machinery
     # see add_ordered_mock_handlers() docstring
diff --git a/lib-python/modified-2.7/urllib2.py b/lib-python/modified-2.7/urllib2.py
new file mode 100644
--- /dev/null
+++ b/lib-python/modified-2.7/urllib2.py
@@ -0,0 +1,1440 @@
+"""An extensible library for opening URLs using a variety of protocols
+
+The simplest way to use this module is to call the urlopen function,
+which accepts a string containing a URL or a Request object (described
+below).  It opens the URL and returns the results as file-like
+object; the returned object has some extra methods described below.
+
+The OpenerDirector manages a collection of Handler objects that do
+all the actual work.  Each Handler implements a particular protocol or
+option.  The OpenerDirector is a composite object that invokes the
+Handlers needed to open the requested URL.  For example, the
+HTTPHandler performs HTTP GET and POST requests and deals with
+non-error returns.  The HTTPRedirectHandler automatically deals with
+HTTP 301, 302, 303 and 307 redirect errors, and the HTTPDigestAuthHandler
+deals with digest authentication.
+
+urlopen(url, data=None) -- Basic usage is the same as original
+urllib.  pass the url and optionally data to post to an HTTP URL, and
+get a file-like object back.  One difference is that you can also pass
+a Request instance instead of URL.  Raises a URLError (subclass of
+IOError); for HTTP errors, raises an HTTPError, which can also be
+treated as a valid response.
+
+build_opener -- Function that creates a new OpenerDirector instance.
+Will install the default handlers.  Accepts one or more Handlers as
+arguments, either instances or Handler classes that it will
+instantiate.  If one of the argument is a subclass of the default
+handler, the argument will be installed instead of the default.
+
+install_opener -- Installs a new opener as the default opener.
+
+objects of interest:
+
+OpenerDirector -- Sets up the User Agent as the Python-urllib client and manages
+the Handler classes, while dealing with requests and responses.
+
+Request -- An object that encapsulates the state of a request.  The
+state can be as simple as the URL.  It can also include extra HTTP
+headers, e.g. a User-Agent.
+
+BaseHandler --
+
+exceptions:
+URLError -- A subclass of IOError, individual protocols have their own
+specific subclass.
+
+HTTPError -- Also a valid HTTP response, so you can treat an HTTP error
+as an exceptional event or valid response.
+
+internals:
+BaseHandler and parent
+_call_chain conventions
+
+Example usage:
+
+import urllib2
+
+# set up authentication info
+authinfo = urllib2.HTTPBasicAuthHandler()
+authinfo.add_password(realm='PDQ Application',
+                      uri='https://mahler:8092/site-updates.py',
+                      user='klem',
+                      passwd='geheim$parole')
+
+proxy_support = urllib2.ProxyHandler({"http" : "http://ahad-haam:3128"})
+
+# build a new opener that adds authentication and caching FTP handlers
+opener = urllib2.build_opener(proxy_support, authinfo, urllib2.CacheFTPHandler)
+
+# install it
+urllib2.install_opener(opener)
+
+f = urllib2.urlopen('http://www.python.org/')
+
+
+"""
+
+# XXX issues:
+# If an authentication error handler that tries to perform
+# authentication for some reason but fails, how should the error be
+# signalled?  The client needs to know the HTTP error code.  But if
+# the handler knows that the problem was, e.g., that it didn't know
+# that hash algo that requested in the challenge, it would be good to
+# pass that information along to the client, too.
+# ftp errors aren't handled cleanly
+# check digest against correct (i.e. non-apache) implementation
+
+# Possible extensions:
+# complex proxies  XXX not sure what exactly was meant by this
+# abstract factory for opener
+
+import base64
+import hashlib
+import httplib
+import mimetools
+import os
+import posixpath
+import random
+import re
+import socket
+import sys
+import time
+import urlparse
+import bisect
+
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from StringIO import StringIO
+
+from urllib import (unwrap, unquote, splittype, splithost, quote,
+     addinfourl, splitport, splittag,
+     splitattr, ftpwrapper, splituser, splitpasswd, splitvalue)
+
+# support for FileHandler, proxies via environment variables
+from urllib import localhost, url2pathname, getproxies, proxy_bypass
+
+# used in User-Agent header sent
+__version__ = sys.version[:3]
+
+_opener = None
+def urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
+    global _opener
+    if _opener is None:
+        _opener = build_opener()
+    return _opener.open(url, data, timeout)
+
+def install_opener(opener):
+    global _opener
+    _opener = opener
+
+# do these error classes make sense?
+# make sure all of the IOError stuff is overridden.  we just want to be
+# subtypes.
+
+class URLError(IOError):
+    # URLError is a sub-type of IOError, but it doesn't share any of
+    # the implementation.  need to override __init__ and __str__.
+    # It sets self.args for compatibility with other EnvironmentError
+    # subclasses, but args doesn't have the typical format with errno in
+    # slot 0 and strerror in slot 1.  This may be better than nothing.
+    def __init__(self, reason):
+        self.args = reason,
+        self.reason = reason
+
+    def __str__(self):
+        return '<urlopen error %s>' % self.reason
+
+class HTTPError(URLError, addinfourl):
+    """Raised when HTTP error occurs, but also acts like non-error return"""
+    __super_init = addinfourl.__init__
+
+    def __init__(self, url, code, msg, hdrs, fp):
+        self.code = code
+        self.msg = msg
+        self.hdrs = hdrs
+        self.fp = fp
+        self.filename = url
+        # The addinfourl classes depend on fp being a valid file
+        # object.  In some cases, the HTTPError may not have a valid
+        # file object.  If this happens, the simplest workaround is to
+        # not initialize the base classes.
+        if fp is not None:
+            self.__super_init(fp, hdrs, url, code)
+
+    def __str__(self):
+        return 'HTTP Error %s: %s' % (self.code, self.msg)
+
+# copied from cookielib.py
+_cut_port_re = re.compile(r":\d+$")
+def request_host(request):
+    """Return request-host, as defined by RFC 2965.
+
+    Variation from RFC: returned value is lowercased, for convenient
+    comparison.
+
+    """
+    url = request.get_full_url()
+    host = urlparse.urlparse(url)[1]
+    if host == "":
+        host = request.get_header("Host", "")
+
+    # remove port, if present
+    host = _cut_port_re.sub("", host, 1)
+    return host.lower()
+
+class Request:
+
+    def __init__(self, url, data=None, headers={},
+                 origin_req_host=None, unverifiable=False):
+        # unwrap('<URL:type://host/path>') --> 'type://host/path'
+        self.__original = unwrap(url)
+        self.__original, fragment = splittag(self.__original)
+        self.type = None
+        # self.__r_type is what's left after doing the splittype
+        self.host = None
+        self.port = None
+        self._tunnel_host = None
+        self.data = data
+        self.headers = {}
+        for key, value in headers.items():
+            self.add_header(key, value)
+        self.unredirected_hdrs = {}
+        if origin_req_host is None:
+            origin_req_host = request_host(self)
+        self.origin_req_host = origin_req_host
+        self.unverifiable = unverifiable
+
+    def __getattr__(self, attr):
+        # XXX this is a fallback mechanism to guard against these
+        # methods getting called in a non-standard order.  this may be
+        # too complicated and/or unnecessary.
+        # XXX should the __r_XXX attributes be public?
+        if attr[:12] == '_Request__r_':
+            name = attr[12:]
+            if hasattr(Request, 'get_' + name):
+                getattr(self, 'get_' + name)()
+                return getattr(self, attr)
+        raise AttributeError, attr
+
+    def get_method(self):
+        if self.has_data():
+            return "POST"
+        else:
+            return "GET"
+
+    # XXX these helper methods are lame
+
+    def add_data(self, data):
+        self.data = data
+
+    def has_data(self):
+        return self.data is not None
+
+    def get_data(self):
+        return self.data
+
+    def get_full_url(self):
+        return self.__original
+
+    def get_type(self):
+        if self.type is None:
+            self.type, self.__r_type = splittype(self.__original)
+            if self.type is None:
+                raise ValueError, "unknown url type: %s" % self.__original
+        return self.type
+
+    def get_host(self):
+        if self.host is None:
+            self.host, self.__r_host = splithost(self.__r_type)
+            if self.host:
+                self.host = unquote(self.host)
+        return self.host
+
+    def get_selector(self):
+        return self.__r_host
+
+    def set_proxy(self, host, type):
+        if self.type == 'https' and not self._tunnel_host:
+            self._tunnel_host = self.host
+        else:
+            self.type = type
+            self.__r_host = self.__original
+
+        self.host = host
+
+    def has_proxy(self):
+        return self.__r_host == self.__original
+
+    def get_origin_req_host(self):
+        return self.origin_req_host
+
+    def is_unverifiable(self):
+        return self.unverifiable
+
+    def add_header(self, key, val):
+        # useful for something like authentication
+        self.headers[key.capitalize()] = val
+
+    def add_unredirected_header(self, key, val):
+        # will not be added to a redirected request
+        self.unredirected_hdrs[key.capitalize()] = val
+
+    def has_header(self, header_name):
+        return (header_name in self.headers or
+                header_name in self.unredirected_hdrs)
+
+    def get_header(self, header_name, default=None):
+        return self.headers.get(
+            header_name,
+            self.unredirected_hdrs.get(header_name, default))
+
+    def header_items(self):
+        hdrs = self.unredirected_hdrs.copy()
+        hdrs.update(self.headers)
+        return hdrs.items()
+
+class OpenerDirector:
+    def __init__(self):
+        client_version = "Python-urllib/%s" % __version__
+        self.addheaders = [('User-agent', client_version)]
+        # manage the individual handlers
+        self.handlers = []
+        self.handle_open = {}
+        self.handle_error = {}
+        self.process_response = {}
+        self.process_request = {}
+
+    def add_handler(self, handler):
+        if not hasattr(handler, "add_parent"):
+            raise TypeError("expected BaseHandler instance, got %r" %
+                            type(handler))
+
+        added = False
+        for meth in dir(handler):
+            if meth in ["redirect_request", "do_open", "proxy_open"]:
+                # oops, coincidental match
+                continue
+
+            i = meth.find("_")
+            protocol = meth[:i]
+            condition = meth[i+1:]
+
+            if condition.startswith("error"):
+                j = condition.find("_") + i + 1
+                kind = meth[j+1:]
+                try:
+                    kind = int(kind)
+                except ValueError:
+                    pass
+                lookup = self.handle_error.get(protocol, {})
+                self.handle_error[protocol] = lookup
+            elif condition == "open":
+                kind = protocol
+                lookup = self.handle_open
+            elif condition == "response":
+                kind = protocol
+                lookup = self.process_response
+            elif condition == "request":
+                kind = protocol
+                lookup = self.process_request
+            else:
+                continue
+
+            handlers = lookup.setdefault(kind, [])
+            if handlers:
+                bisect.insort(handlers, handler)
+            else:
+                handlers.append(handler)
+            added = True
+
+        if added:
+            # the handlers must work in an specific order, the order
+            # is specified in a Handler attribute
+            bisect.insort(self.handlers, handler)
+            handler.add_parent(self)
+
+    def close(self):
+        # Only exists for backwards compatibility.
+        pass
+
+    def _call_chain(self, chain, kind, meth_name, *args):
+        # Handlers raise an exception if no one else should try to handle
+        # the request, or return None if they can't but another handler
+        # could.  Otherwise, they return the response.
+        handlers = chain.get(kind, ())
+        for handler in handlers:
+            func = getattr(handler, meth_name)
+
+            result = func(*args)
+            if result is not None:
+                return result
+
+    def open(self, fullurl, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
+        # accept a URL or a Request object
+        if isinstance(fullurl, basestring):
+            req = Request(fullurl, data)
+        else:
+            req = fullurl
+            if data is not None:
+                req.add_data(data)
+
+        req.timeout = timeout
+        protocol = req.get_type()
+
+        # pre-process request
+        meth_name = protocol+"_request"
+        for processor in self.process_request.get(protocol, []):
+            meth = getattr(processor, meth_name)
+            req = meth(req)
+
+        response = self._open(req, data)
+
+        # post-process response
+        meth_name = protocol+"_response"
+        for processor in self.process_response.get(protocol, []):
+            meth = getattr(processor, meth_name)
+            try:
+                response = meth(req, response)
+            except:
+                response.close()
+                raise
+
+        return response
+
+    def _open(self, req, data=None):
+        result = self._call_chain(self.handle_open, 'default',
+                                  'default_open', req)
+        if result:
+            return result
+
+        protocol = req.get_type()
+        result = self._call_chain(self.handle_open, protocol, protocol +
+                                  '_open', req)
+        if result:
+            return result
+
+        return self._call_chain(self.handle_open, 'unknown',
+                                'unknown_open', req)
+
+    def error(self, proto, *args):
+        if proto in ('http', 'https'):
+            # XXX http[s] protocols are special-cased
+            dict = self.handle_error['http'] # https is not different than http
+            proto = args[2]  # YUCK!
+            meth_name = 'http_error_%s' % proto
+            http_err = 1
+            orig_args = args
+        else:
+            dict = self.handle_error
+            meth_name = proto + '_error'
+            http_err = 0
+        args = (dict, proto, meth_name) + args
+        result = self._call_chain(*args)
+        if result:
+            return result
+
+        if http_err:
+            args = (dict, 'default', 'http_error_default') + orig_args
+            return self._call_chain(*args)
+
+# XXX probably also want an abstract factory that knows when it makes
+# sense to skip a superclass in favor of a subclass and when it might
+# make sense to include both
+
+def build_opener(*handlers):
+    """Create an opener object from a list of handlers.
+
+    The opener will use several default handlers, including support
+    for HTTP, FTP and when applicable, HTTPS.
+
+    If any of the handlers passed as arguments are subclasses of the
+    default handlers, the default handlers will not be used.
+    """
+    import types
+    def isclass(obj):
+        return isinstance(obj, (types.ClassType, type))
+
+    opener = OpenerDirector()
+    default_classes = [ProxyHandler, UnknownHandler, HTTPHandler,
+                       HTTPDefaultErrorHandler, HTTPRedirectHandler,
+                       FTPHandler, FileHandler, HTTPErrorProcessor]
+    if hasattr(httplib, 'HTTPS'):
+        default_classes.append(HTTPSHandler)
+    skip = set()
+    for klass in default_classes:
+        for check in handlers:
+            if isclass(check):
+                if issubclass(check, klass):
+                    skip.add(klass)
+            elif isinstance(check, klass):
+                skip.add(klass)
+    for klass in skip:
+        default_classes.remove(klass)
+
+    for klass in default_classes:
+        opener.add_handler(klass())
+
+    for h in handlers:
+        if isclass(h):
+            h = h()
+        opener.add_handler(h)
+    return opener
+
+class BaseHandler:
+    handler_order = 500
+
+    def add_parent(self, parent):
+        self.parent = parent
+
+    def close(self):
+        # Only exists for backwards compatibility
+        pass
+
+    def __lt__(self, other):
+        if not hasattr(other, "handler_order"):
+            # Try to preserve the old behavior of having custom classes
+            # inserted after default ones (works only for custom user
+            # classes which are not aware of handler_order).
+            return True
+        return self.handler_order < other.handler_order
+
+
+class HTTPErrorProcessor(BaseHandler):
+    """Process HTTP error responses."""
+    handler_order = 1000  # after all other processing
+
+    def http_response(self, request, response):
+        code, msg, hdrs = response.code, response.msg, response.info()
+
+        # According to RFC 2616, "2xx" code indicates that the client's
+        # request was successfully received, understood, and accepted.
+        if not (200 <= code < 300):
+            response = self.parent.error(
+                'http', request, response, code, msg, hdrs)
+
+        return response
+
+    https_response = http_response
+
+class HTTPDefaultErrorHandler(BaseHandler):
+    def http_error_default(self, req, fp, code, msg, hdrs):
+        raise HTTPError(req.get_full_url(), code, msg, hdrs, fp)
+
+class HTTPRedirectHandler(BaseHandler):
+    # maximum number of redirections to any single URL
+    # this is needed because of the state that cookies introduce
+    max_repeats = 4
+    # maximum total number of redirections (regardless of URL) before
+    # assuming we're in a loop
+    max_redirections = 10
+
+    def redirect_request(self, req, fp, code, msg, headers, newurl):
+        """Return a Request or None in response to a redirect.
+
+        This is called by the http_error_30x methods when a
+        redirection response is received.  If a redirection should
+        take place, return a new Request to allow http_error_30x to
+        perform the redirect.  Otherwise, raise HTTPError if no-one
+        else should try to handle this url.  Return None if you can't
+        but another Handler might.
+        """
+        m = req.get_method()
+        if (code in (301, 302, 303, 307) and m in ("GET", "HEAD")
+            or code in (301, 302, 303) and m == "POST"):
+            # Strictly (according to RFC 2616), 301 or 302 in response
+            # to a POST MUST NOT cause a redirection without confirmation
+            # from the user (of urllib2, in this case).  In practice,
+            # essentially all clients do redirect in this case, so we
+            # do the same.
+            # be conciliant with URIs containing a space
+            newurl = newurl.replace(' ', '%20')
+            newheaders = dict((k,v) for k,v in req.headers.items()
+                              if k.lower() not in ("content-length", "content-type")
+                             )
+            return Request(newurl,
+                           headers=newheaders,
+                           origin_req_host=req.get_origin_req_host(),
+                           unverifiable=True)
+        else:
+            raise HTTPError(req.get_full_url(), code, msg, headers, fp)
+
+    # Implementation note: To avoid the server sending us into an
+    # infinite loop, the request object needs to track what URLs we
+    # have already seen.  Do this by adding a handler-specific
+    # attribute to the Request object.
+    def http_error_302(self, req, fp, code, msg, headers):
+        # Some servers (incorrectly) return multiple Location headers
+        # (so probably same goes for URI).  Use first header.
+        if 'location' in headers:
+            newurl = headers.getheaders('location')[0]
+        elif 'uri' in headers:
+            newurl = headers.getheaders('uri')[0]
+        else:
+            return
+
+        # fix a possible malformed URL
+        urlparts = urlparse.urlparse(newurl)
+        if not urlparts.path:
+            urlparts = list(urlparts)
+            urlparts[2] = "/"
+        newurl = urlparse.urlunparse(urlparts)
+
+        newurl = urlparse.urljoin(req.get_full_url(), newurl)
+
+        # XXX Probably want to forget about the state of the current
+        # request, although that might interact poorly with other
+        # handlers that also use handler-specific request attributes
+        new = self.redirect_request(req, fp, code, msg, headers, newurl)
+        if new is None:
+            return
+
+        # loop detection
+        # .redirect_dict has a key url if url was previously visited.
+        if hasattr(req, 'redirect_dict'):
+            visited = new.redirect_dict = req.redirect_dict
+            if (visited.get(newurl, 0) >= self.max_repeats or
+                len(visited) >= self.max_redirections):
+                raise HTTPError(req.get_full_url(), code,
+                                self.inf_msg + msg, headers, fp)
+        else:
+            visited = new.redirect_dict = req.redirect_dict = {}
+        visited[newurl] = visited.get(newurl, 0) + 1
+
+        # Don't close the fp until we are sure that we won't use it
+        # with HTTPError.
+        fp.read()
+        fp.close()
+
+        return self.parent.open(new, timeout=req.timeout)
+
+    http_error_301 = http_error_303 = http_error_307 = http_error_302
+
+    inf_msg = "The HTTP server returned a redirect error that would " \
+              "lead to an infinite loop.\n" \
+              "The last 30x error message was:\n"
+
+
+def _parse_proxy(proxy):
+    """Return (scheme, user, password, host/port) given a URL or an authority.
+
+    If a URL is supplied, it must have an authority (host:port) component.
+    According to RFC 3986, having an authority component means the URL must
+    have two slashes after the scheme:
+
+    >>> _parse_proxy('file:/ftp.example.com/')
+    Traceback (most recent call last):
+    ValueError: proxy URL with no authority: 'file:/ftp.example.com/'
+
+    The first three items of the returned tuple may be None.
+
+    Examples of authority parsing:
+
+    >>> _parse_proxy('proxy.example.com')
+    (None, None, None, 'proxy.example.com')
+    >>> _parse_proxy('proxy.example.com:3128')
+    (None, None, None, 'proxy.example.com:3128')
+
+    The authority component may optionally include userinfo (assumed to be
+    username:password):
+
+    >>> _parse_proxy('joe:password at proxy.example.com')
+    (None, 'joe', 'password', 'proxy.example.com')
+    >>> _parse_proxy('joe:password at proxy.example.com:3128')
+    (None, 'joe', 'password', 'proxy.example.com:3128')
+
+    Same examples, but with URLs instead:
+
+    >>> _parse_proxy('http://proxy.example.com/')
+    ('http', None, None, 'proxy.example.com')
+    >>> _parse_proxy('http://proxy.example.com:3128/')
+    ('http', None, None, 'proxy.example.com:3128')
+    >>> _parse_proxy('http://joe:password at proxy.example.com/')
+    ('http', 'joe', 'password', 'proxy.example.com')
+    >>> _parse_proxy('http://joe:password at proxy.example.com:3128')
+    ('http', 'joe', 'password', 'proxy.example.com:3128')
+
+    Everything after the authority is ignored:
+
+    >>> _parse_proxy('ftp://joe:password at proxy.example.com/rubbish:3128')
+    ('ftp', 'joe', 'password', 'proxy.example.com')
+
+    Test for no trailing '/' case:
+
+    >>> _parse_proxy('http://joe:password at proxy.example.com')
+    ('http', 'joe', 'password', 'proxy.example.com')
+
+    """
+    scheme, r_scheme = splittype(proxy)
+    if not r_scheme.startswith("/"):
+        # authority
+        scheme = None
+        authority = proxy
+    else:
+        # URL
+        if not r_scheme.startswith("//"):
+            raise ValueError("proxy URL with no authority: %r" % proxy)
+        # We have an authority, so for RFC 3986-compliant URLs (by ss 3.
+        # and 3.3.), path is empty or starts with '/'
+        end = r_scheme.find("/", 2)
+        if end == -1:
+            end = None
+        authority = r_scheme[2:end]
+    userinfo, hostport = splituser(authority)
+    if userinfo is not None:
+        user, password = splitpasswd(userinfo)
+    else:
+        user = password = None
+    return scheme, user, password, hostport
+
+class ProxyHandler(BaseHandler):
+    # Proxies must be in front
+    handler_order = 100
+
+    def __init__(self, proxies=None):
+        if proxies is None:
+            proxies = getproxies()
+        assert hasattr(proxies, 'has_key'), "proxies must be a mapping"
+        self.proxies = proxies
+        for type, url in proxies.items():
+            setattr(self, '%s_open' % type,
+                    lambda r, proxy=url, type=type, meth=self.proxy_open: \
+                    meth(r, proxy, type))
+
+    def proxy_open(self, req, proxy, type):
+        orig_type = req.get_type()
+        proxy_type, user, password, hostport = _parse_proxy(proxy)
+
+        if proxy_type is None:
+            proxy_type = orig_type
+
+        if req.host and proxy_bypass(req.host):
+            return None
+
+        if user and password:
+            user_pass = '%s:%s' % (unquote(user), unquote(password))
+            creds = base64.b64encode(user_pass).strip()
+            req.add_header('Proxy-authorization', 'Basic ' + creds)
+        hostport = unquote(hostport)
+        req.set_proxy(hostport, proxy_type)
+
+        if orig_type == proxy_type or orig_type == 'https':
+            # let other handlers take care of it
+            return None
+        else:
+            # need to start over, because the other handlers don't
+            # grok the proxy's URL type
+            # e.g. if we have a constructor arg proxies like so:
+            # {'http': 'ftp://proxy.example.com'}, we may end up turning
+            # a request for http://acme.example.com/a into one for
+            # ftp://proxy.example.com/a
+            return self.parent.open(req, timeout=req.timeout)
+
+class HTTPPasswordMgr:
+
+    def __init__(self):
+        self.passwd = {}
+
+    def add_password(self, realm, uri, user, passwd):
+        # uri could be a single URI or a sequence
+        if isinstance(uri, basestring):
+            uri = [uri]
+        if not realm in self.passwd:
+            self.passwd[realm] = {}
+        for default_port in True, False:
+            reduced_uri = tuple(
+                [self.reduce_uri(u, default_port) for u in uri])
+            self.passwd[realm][reduced_uri] = (user, passwd)
+
+    def find_user_password(self, realm, authuri):
+        domains = self.passwd.get(realm, {})
+        for default_port in True, False:
+            reduced_authuri = self.reduce_uri(authuri, default_port)
+            for uris, authinfo in domains.iteritems():
+                for uri in uris:
+                    if self.is_suburi(uri, reduced_authuri):
+                        return authinfo
+        return None, None
+
+    def reduce_uri(self, uri, default_port=True):
+        """Accept authority or URI and extract only the authority and path."""
+        # note HTTP URLs do not have a userinfo component
+        parts = urlparse.urlsplit(uri)
+        if parts[1]:
+            # URI
+            scheme = parts[0]
+            authority = parts[1]
+            path = parts[2] or '/'
+        else:
+            # host or host:port
+            scheme = None
+            authority = uri
+            path = '/'
+        host, port = splitport(authority)
+        if default_port and port is None and scheme is not None:
+            dport = {"http": 80,
+                     "https": 443,
+                     }.get(scheme)
+            if dport is not None:
+                authority = "%s:%d" % (host, dport)
+        return authority, path
+
+    def is_suburi(self, base, test):
+        """Check if test is below base in a URI tree
+
+        Both args must be URIs in reduced form.
+        """
+        if base == test:
+            return True
+        if base[0] != test[0]:
+            return False
+        common = posixpath.commonprefix((base[1], test[1]))
+        if len(common) == len(base[1]):
+            return True
+        return False
+
+
+class HTTPPasswordMgrWithDefaultRealm(HTTPPasswordMgr):
+
+    def find_user_password(self, realm, authuri):
+        user, password = HTTPPasswordMgr.find_user_password(self, realm,
+                                                            authuri)
+        if user is not None:
+            return user, password
+        return HTTPPasswordMgr.find_user_password(self, None, authuri)
+
+
+class AbstractBasicAuthHandler:
+
+    # XXX this allows for multiple auth-schemes, but will stupidly pick
+    # the last one with a realm specified.
+
+    # allow for double- and single-quoted realm values
+    # (single quotes are a violation of the RFC, but appear in the wild)
+    rx = re.compile('(?:.*,)*[ \t]*([^ \t]+)[ \t]+'
+                    'realm=(["\'])(.*?)\\2', re.I)
+
+    # XXX could pre-emptively send auth info already accepted (RFC 2617,
+    # end of section 2, and section 1.2 immediately after "credentials"
+    # production).
+
+    def __init__(self, password_mgr=None):
+        if password_mgr is None:
+            password_mgr = HTTPPasswordMgr()
+        self.passwd = password_mgr
+        self.add_password = self.passwd.add_password
+        self.retried = 0
+
+    def reset_retry_count(self):
+        self.retried = 0
+
+    def http_error_auth_reqed(self, authreq, host, req, headers):
+        # host may be an authority (without userinfo) or a URL with an
+        # authority
+        # XXX could be multiple headers
+        authreq = headers.get(authreq, None)
+
+        if self.retried > 5:
+            # retry sending the username:password 5 times before failing.
+            raise HTTPError(req.get_full_url(), 401, "basic auth failed",
+                            headers, None)
+        else:
+            self.retried += 1
+
+        if authreq:
+            mo = AbstractBasicAuthHandler.rx.search(authreq)
+            if mo:
+                scheme, quote, realm = mo.groups()
+                if scheme.lower() == 'basic':
+                    response = self.retry_http_basic_auth(host, req, realm)
+                    if response and response.code != 401:
+                        self.retried = 0
+                    return response
+
+    def retry_http_basic_auth(self, host, req, realm):
+        user, pw = self.passwd.find_user_password(realm, host)
+        if pw is not None:
+            raw = "%s:%s" % (user, pw)
+            auth = 'Basic %s' % base64.b64encode(raw).strip()
+            if req.headers.get(self.auth_header, None) == auth:
+                return None
+            req.add_unredirected_header(self.auth_header, auth)
+            return self.parent.open(req, timeout=req.timeout)
+        else:
+            return None
+
+
+class HTTPBasicAuthHandler(AbstractBasicAuthHandler, BaseHandler):
+
+    auth_header = 'Authorization'
+
+    def http_error_401(self, req, fp, code, msg, headers):
+        url = req.get_full_url()
+        response = self.http_error_auth_reqed('www-authenticate',
+                                              url, req, headers)
+        self.reset_retry_count()
+        return response
+
+
+class ProxyBasicAuthHandler(AbstractBasicAuthHandler, BaseHandler):
+
+    auth_header = 'Proxy-authorization'
+
+    def http_error_407(self, req, fp, code, msg, headers):
+        # http_error_auth_reqed requires that there is no userinfo component in
+        # authority.  Assume there isn't one, since urllib2 does not (and
+        # should not, RFC 3986 s. 3.2.1) support requests for URLs containing
+        # userinfo.
+        authority = req.get_host()
+        response = self.http_error_auth_reqed('proxy-authenticate',
+                                          authority, req, headers)
+        self.reset_retry_count()
+        return response
+
+
+def randombytes(n):
+    """Return n random bytes."""
+    # Use /dev/urandom if it is available.  Fall back to random module
+    # if not.  It might be worthwhile to extend this function to use
+    # other platform-specific mechanisms for getting random bytes.
+    if os.path.exists("/dev/urandom"):
+        f = open("/dev/urandom")
+        s = f.read(n)
+        f.close()
+        return s
+    else:
+        L = [chr(random.randrange(0, 256)) for i in range(n)]
+        return "".join(L)
+
+class AbstractDigestAuthHandler:
+    # Digest authentication is specified in RFC 2617.
+
+    # XXX The client does not inspect the Authentication-Info header
+    # in a successful response.
+
+    # XXX It should be possible to test this implementation against
+    # a mock server that just generates a static set of challenges.
+
+    # XXX qop="auth-int" supports is shaky
+
+    def __init__(self, passwd=None):
+        if passwd is None:
+            passwd = HTTPPasswordMgr()
+        self.passwd = passwd
+        self.add_password = self.passwd.add_password
+        self.retried = 0
+        self.nonce_count = 0
+        self.last_nonce = None
+
+    def reset_retry_count(self):
+        self.retried = 0
+
+    def http_error_auth_reqed(self, auth_header, host, req, headers):
+        authreq = headers.get(auth_header, None)
+        if self.retried > 5:
+            # Don't fail endlessly - if we failed once, we'll probably
+            # fail a second time. Hm. Unless the Password Manager is
+            # prompting for the information. Crap. This isn't great
+            # but it's better than the current 'repeat until recursion
+            # depth exceeded' approach <wink>
+            raise HTTPError(req.get_full_url(), 401, "digest auth failed",
+                            headers, None)
+        else:
+            self.retried += 1
+        if authreq:
+            scheme = authreq.split()[0]
+            if scheme.lower() == 'digest':
+                return self.retry_http_digest_auth(req, authreq)
+
+    def retry_http_digest_auth(self, req, auth):
+        token, challenge = auth.split(' ', 1)
+        chal = parse_keqv_list(parse_http_list(challenge))
+        auth = self.get_authorization(req, chal)
+        if auth:
+            auth_val = 'Digest %s' % auth
+            if req.headers.get(self.auth_header, None) == auth_val:
+                return None
+            req.add_unredirected_header(self.auth_header, auth_val)
+            resp = self.parent.open(req, timeout=req.timeout)
+            return resp
+
+    def get_cnonce(self, nonce):
+        # The cnonce-value is an opaque
+        # quoted string value provided by the client and used by both client
+        # and server to avoid chosen plaintext attacks, to provide mutual
+        # authentication, and to provide some message integrity protection.
+        # This isn't a fabulous effort, but it's probably Good Enough.
+        dig = hashlib.sha1("%s:%s:%s:%s" % (self.nonce_count, nonce, time.ctime(),
+                                            randombytes(8))).hexdigest()
+        return dig[:16]
+
+    def get_authorization(self, req, chal):
+        try:
+            realm = chal['realm']
+            nonce = chal['nonce']
+            qop = chal.get('qop')
+            algorithm = chal.get('algorithm', 'MD5')
+            # mod_digest doesn't send an opaque, even though it isn't
+            # supposed to be optional
+            opaque = chal.get('opaque', None)
+        except KeyError:
+            return None
+
+        H, KD = self.get_algorithm_impls(algorithm)
+        if H is None:
+            return None
+
+        user, pw = self.passwd.find_user_password(realm, req.get_full_url())
+        if user is None:
+            return None
+
+        # XXX not implemented yet
+        if req.has_data():
+            entdig = self.get_entity_digest(req.get_data(), chal)
+        else:
+            entdig = None
+
+        A1 = "%s:%s:%s" % (user, realm, pw)
+        A2 = "%s:%s" % (req.get_method(),
+                        # XXX selector: what about proxies and full urls
+                        req.get_selector())
+        if qop == 'auth':
+            if nonce == self.last_nonce:
+                self.nonce_count += 1
+            else:
+                self.nonce_count = 1
+                self.last_nonce = nonce
+
+            ncvalue = '%08x' % self.nonce_count
+            cnonce = self.get_cnonce(nonce)
+            noncebit = "%s:%s:%s:%s:%s" % (nonce, ncvalue, cnonce, qop, H(A2))
+            respdig = KD(H(A1), noncebit)
+        elif qop is None:
+            respdig = KD(H(A1), "%s:%s" % (nonce, H(A2)))
+        else:
+            # XXX handle auth-int.
+            raise URLError("qop '%s' is not supported." % qop)
+
+        # XXX should the partial digests be encoded too?
+
+        base = 'username="%s", realm="%s", nonce="%s", uri="%s", ' \
+               'response="%s"' % (user, realm, nonce, req.get_selector(),
+                                  respdig)
+        if opaque:
+            base += ', opaque="%s"' % opaque
+        if entdig:
+            base += ', digest="%s"' % entdig
+        base += ', algorithm="%s"' % algorithm
+        if qop:
+            base += ', qop=auth, nc=%s, cnonce="%s"' % (ncvalue, cnonce)
+        return base
+
+    def get_algorithm_impls(self, algorithm):
+        # algorithm should be case-insensitive according to RFC2617
+        algorithm = algorithm.upper()
+        # lambdas assume digest modules are imported at the top level
+        if algorithm == 'MD5':
+            H = lambda x: hashlib.md5(x).hexdigest()
+        elif algorithm == 'SHA':
+            H = lambda x: hashlib.sha1(x).hexdigest()
+        # XXX MD5-sess
+        KD = lambda s, d: H("%s:%s" % (s, d))
+        return H, KD
+
+    def get_entity_digest(self, data, chal):
+        # XXX not implemented yet
+        return None
+
+
+class HTTPDigestAuthHandler(BaseHandler, AbstractDigestAuthHandler):
+    """An authentication protocol defined by RFC 2069
+
+    Digest authentication improves on basic authentication because it
+    does not transmit passwords in the clear.
+    """
+
+    auth_header = 'Authorization'
+    handler_order = 490  # before Basic auth
+
+    def http_error_401(self, req, fp, code, msg, headers):
+        host = urlparse.urlparse(req.get_full_url())[1]
+        retry = self.http_error_auth_reqed('www-authenticate',
+                                           host, req, headers)
+        self.reset_retry_count()
+        return retry
+
+
+class ProxyDigestAuthHandler(BaseHandler, AbstractDigestAuthHandler):
+
+    auth_header = 'Proxy-Authorization'
+    handler_order = 490  # before Basic auth
+
+    def http_error_407(self, req, fp, code, msg, headers):
+        host = req.get_host()
+        retry = self.http_error_auth_reqed('proxy-authenticate',
+                                           host, req, headers)
+        self.reset_retry_count()
+        return retry
+
+class AbstractHTTPHandler(BaseHandler):
+
+    def __init__(self, debuglevel=0):
+        self._debuglevel = debuglevel
+
+    def set_http_debuglevel(self, level):
+        self._debuglevel = level
+
+    def do_request_(self, request):
+        host = request.get_host()
+        if not host:
+            raise URLError('no host given')
+
+        if request.has_data():  # POST
+            data = request.get_data()
+            if not request.has_header('Content-type'):
+                request.add_unredirected_header(
+                    'Content-type',
+                    'application/x-www-form-urlencoded')
+            if not request.has_header('Content-length'):
+                request.add_unredirected_header(
+                    'Content-length', '%d' % len(data))
+
+        sel_host = host
+        if request.has_proxy():
+            scheme, sel = splittype(request.get_selector())
+            sel_host, sel_path = splithost(sel)
+
+        if not request.has_header('Host'):
+            request.add_unredirected_header('Host', sel_host)
+        for name, value in self.parent.addheaders:
+            name = name.capitalize()
+            if not request.has_header(name):
+                request.add_unredirected_header(name, value)
+
+        return request
+
+    def do_open(self, http_class, req):
+        """Return an addinfourl object for the request, using http_class.
+
+        http_class must implement the HTTPConnection API from httplib.
+        The addinfourl return value is a file-like object.  It also
+        has methods and attributes including:
+            - info(): return a mimetools.Message object for the headers
+            - geturl(): return the original request URL
+            - code: HTTP status code
+        """
+        host = req.get_host()
+        if not host:
+            raise URLError('no host given')
+
+        h = http_class(host, timeout=req.timeout) # will parse host:port
+        h.set_debuglevel(self._debuglevel)
+
+        headers = dict(req.unredirected_hdrs)
+        headers.update(dict((k, v) for k, v in req.headers.items()
+                            if k not in headers))
+
+        # We want to make an HTTP/1.1 request, but the addinfourl
+        # class isn't prepared to deal with a persistent connection.
+        # It will try to read all remaining data from the socket,
+        # which will block while the server waits for the next request.
+        # So make sure the connection gets closed after the (only)
+        # request.
+        headers["Connection"] = "close"
+        headers = dict(
+            (name.title(), val) for name, val in headers.items())
+
+        if req._tunnel_host:
+            tunnel_headers = {}
+            proxy_auth_hdr = "Proxy-Authorization"
+            if proxy_auth_hdr in headers:
+                tunnel_headers[proxy_auth_hdr] = headers[proxy_auth_hdr]
+                # Proxy-Authorization should not be sent to origin
+                # server.
+                del headers[proxy_auth_hdr]
+            h.set_tunnel(req._tunnel_host, headers=tunnel_headers)
+
+        try:
+            h.request(req.get_method(), req.get_selector(), req.data, headers)
+            try:
+                r = h.getresponse(buffering=True)
+            except TypeError: #buffering kw not supported
+                r = h.getresponse()
+        except socket.error, err: # XXX what error?
+            h.close()
+            raise URLError(err)
+
+        # Pick apart the HTTPResponse object to get the addinfourl
+        # object initialized properly.
+
+        # Wrap the HTTPResponse object in socket's file object adapter
+        # for Windows.  That adapter calls recv(), so delegate recv()
+        # to read().  This weird wrapping allows the returned object to
+        # have readline() and readlines() methods.
+
+        # XXX It might be better to extract the read buffering code
+        # out of socket._fileobject() and into a base class.
+
+        r.recv = r.read
+        fp = socket._fileobject(r, close=True)
+
+        resp = addinfourl(fp, r.msg, req.get_full_url())
+        resp.code = r.status
+        resp.msg = r.reason
+        return resp
+
+
+class HTTPHandler(AbstractHTTPHandler):
+
+    def http_open(self, req):
+        return self.do_open(httplib.HTTPConnection, req)
+
+    http_request = AbstractHTTPHandler.do_request_
+
+if hasattr(httplib, 'HTTPS'):
+    class HTTPSHandler(AbstractHTTPHandler):
+
+        def https_open(self, req):
+            return self.do_open(httplib.HTTPSConnection, req)
+
+        https_request = AbstractHTTPHandler.do_request_
+
+class HTTPCookieProcessor(BaseHandler):
+    def __init__(self, cookiejar=None):
+        import cookielib
+        if cookiejar is None:
+            cookiejar = cookielib.CookieJar()
+        self.cookiejar = cookiejar
+
+    def http_request(self, request):
+        self.cookiejar.add_cookie_header(request)
+        return request
+
+    def http_response(self, request, response):
+        self.cookiejar.extract_cookies(response, request)
+        return response
+
+    https_request = http_request
+    https_response = http_response
+
+class UnknownHandler(BaseHandler):
+    def unknown_open(self, req):
+        type = req.get_type()
+        raise URLError('unknown url type: %s' % type)
+
+def parse_keqv_list(l):
+    """Parse list of key=value strings where keys are not duplicated."""
+    parsed = {}
+    for elt in l:
+        k, v = elt.split('=', 1)
+        if v[0] == '"' and v[-1] == '"':
+            v = v[1:-1]
+        parsed[k] = v
+    return parsed
+
+def parse_http_list(s):
+    """Parse lists as described by RFC 2068 Section 2.
+
+    In particular, parse comma-separated lists where the elements of
+    the list may include quoted-strings.  A quoted-string could
+    contain a comma.  A non-quoted string could have quotes in the
+    middle.  Neither commas nor quotes count if they are escaped.
+    Only double-quotes count, not single-quotes.
+    """
+    res = []
+    part = ''
+
+    escape = quote = False
+    for cur in s:
+        if escape:
+            part += cur
+            escape = False
+            continue
+        if quote:
+            if cur == '\\':
+                escape = True
+                continue
+            elif cur == '"':
+                quote = False
+            part += cur
+            continue
+
+        if cur == ',':
+            res.append(part)
+            part = ''
+            continue
+
+        if cur == '"':
+            quote = True
+
+        part += cur
+
+    # append last part
+    if part:
+        res.append(part)
+
+    return [part.strip() for part in res]
+
+def _safe_gethostbyname(host):
+    try:
+        return socket.gethostbyname(host)
+    except socket.gaierror:
+        return None
+
+class FileHandler(BaseHandler):
+    # Use local file or FTP depending on form of URL
+    def file_open(self, req):
+        url = req.get_selector()
+        if url[:2] == '//' and url[2:3] != '/' and (req.host and
+                req.host != 'localhost'):
+            req.type = 'ftp'
+            return self.parent.open(req)
+        else:
+            return self.open_local_file(req)
+
+    # names for the localhost
+    names = None
+    def get_names(self):
+        if FileHandler.names is None:
+            try:
+                FileHandler.names = tuple(
+                    socket.gethostbyname_ex('localhost')[2] +
+                    socket.gethostbyname_ex(socket.gethostname())[2])
+            except socket.gaierror:
+                FileHandler.names = (socket.gethostbyname('localhost'),)
+        return FileHandler.names
+
+    # not entirely sure what the rules are here
+    def open_local_file(self, req):
+        import email.utils
+        import mimetypes
+        host = req.get_host()
+        filename = req.get_selector()
+        localfile = url2pathname(filename)
+        try:
+            stats = os.stat(localfile)
+            size = stats.st_size
+            modified = email.utils.formatdate(stats.st_mtime, usegmt=True)
+            mtype = mimetypes.guess_type(filename)[0]
+            headers = mimetools.Message(StringIO(
+                'Content-type: %s\nContent-length: %d\nLast-modified: %s\n' %
+                (mtype or 'text/plain', size, modified)))
+            if host:
+                host, port = splitport(host)
+            if not host or \
+                (not port and _safe_gethostbyname(host) in self.get_names()):
+                if host:
+                    origurl = 'file://' + host + filename
+                else:
+                    origurl = 'file://' + filename
+                return addinfourl(open(localfile, 'rb'), headers, origurl)
+        except OSError, msg:
+            # urllib2 users shouldn't expect OSErrors coming from urlopen()
+            raise URLError(msg)
+        raise URLError('file not on local host')
+
+class FTPHandler(BaseHandler):
+    def ftp_open(self, req):
+        import ftplib
+        import mimetypes
+        host = req.get_host()
+        if not host:
+            raise URLError('ftp error: no host given')
+        host, port = splitport(host)
+        if port is None:
+            port = ftplib.FTP_PORT
+        else:
+            port = int(port)
+
+        # username/password handling
+        user, host = splituser(host)
+        if user:
+            user, passwd = splitpasswd(user)
+        else:
+            passwd = None
+        host = unquote(host)
+        user = user or ''
+        passwd = passwd or ''
+
+        try:
+            host = socket.gethostbyname(host)
+        except socket.error, msg:
+            raise URLError(msg)
+        path, attrs = splitattr(req.get_selector())
+        dirs = path.split('/')
+        dirs = map(unquote, dirs)
+        dirs, file = dirs[:-1], dirs[-1]
+        if dirs and not dirs[0]:
+            dirs = dirs[1:]
+        try:
+            fw = self.connect_ftp(user, passwd, host, port, dirs, req.timeout)
+            type = file and 'I' or 'D'
+            for attr in attrs:
+                attr, value = splitvalue(attr)
+                if attr.lower() == 'type' and \
+                   value in ('a', 'A', 'i', 'I', 'd', 'D'):
+                    type = value.upper()
+            fp, retrlen = fw.retrfile(file, type)
+            headers = ""
+            mtype = mimetypes.guess_type(req.get_full_url())[0]
+            if mtype:
+                headers += "Content-type: %s\n" % mtype
+            if retrlen is not None and retrlen >= 0:
+                headers += "Content-length: %d\n" % retrlen
+            sf = StringIO(headers)
+            headers = mimetools.Message(sf)
+            return addinfourl(fp, headers, req.get_full_url())
+        except ftplib.all_errors, msg:
+            raise URLError, ('ftp error: %s' % msg), sys.exc_info()[2]
+
+    def connect_ftp(self, user, passwd, host, port, dirs, timeout):
+        fw = ftpwrapper(user, passwd, host, port, dirs, timeout)
+##        fw.ftp.set_debuglevel(1)
+        return fw
+
+class CacheFTPHandler(FTPHandler):
+    # XXX would be nice to have pluggable cache strategies
+    # XXX this stuff is definitely not thread safe
+    def __init__(self):
+        self.cache = {}
+        self.timeout = {}
+        self.soonest = 0
+        self.delay = 60
+        self.max_conns = 16
+
+    def setTimeout(self, t):
+        self.delay = t
+
+    def setMaxConns(self, m):
+        self.max_conns = m
+
+    def connect_ftp(self, user, passwd, host, port, dirs, timeout):
+        key = user, host, port, '/'.join(dirs), timeout
+        if key in self.cache:
+            self.timeout[key] = time.time() + self.delay
+        else:
+            self.cache[key] = ftpwrapper(user, passwd, host, port, dirs, timeout)
+            self.timeout[key] = time.time() + self.delay
+        self.check_cache()
+        return self.cache[key]
+
+    def check_cache(self):
+        # first check for old ones
+        t = time.time()
+        if self.soonest <= t:
+            for k, v in self.timeout.items():
+                if v < t:
+                    self.cache[k].close()
+                    del self.cache[k]
+                    del self.timeout[k]
+        self.soonest = min(self.timeout.values())
+
+        # then check the size
+        if len(self.cache) == self.max_conns:
+            for k, v in self.timeout.items():
+                if v == self.soonest:
+                    del self.cache[k]
+                    del self.timeout[k]
+                    break
+            self.soonest = min(self.timeout.values())
diff --git a/lib_pypy/_functools.py b/lib_pypy/_functools.py
--- a/lib_pypy/_functools.py
+++ b/lib_pypy/_functools.py
@@ -14,10 +14,9 @@
             raise TypeError("the first argument must be callable")
         self.func = func
         self.args = args
-        self.keywords = keywords
+        self.keywords = keywords or None
 
     def __call__(self, *fargs, **fkeywords):
-        newkeywords = self.keywords.copy()
-        newkeywords.update(fkeywords)
-        return self.func(*(self.args + fargs), **newkeywords)
-
+        if self.keywords is not None:
+            fkeywords = dict(self.keywords, **fkeywords)
+        return self.func(*(self.args + fargs), **fkeywords)
diff --git a/lib_pypy/greenlet.py b/lib_pypy/greenlet.py
--- a/lib_pypy/greenlet.py
+++ b/lib_pypy/greenlet.py
@@ -96,7 +96,16 @@
 
     @property
     def gr_frame(self):
-        raise NotImplementedError("attribute 'gr_frame' of greenlet objects")
+        # xxx this doesn't work when called on either the current or
+        # the main greenlet of another thread
+        if self is getcurrent():
+            return None
+        if self.__main:
+            self = getcurrent()
+        f = _continulet.__reduce__(self)[2][0]
+        if not f:
+            return None
+        return f.f_back.f_back.f_back   # go past start(), __switch(), switch()
 
 # ____________________________________________________________
 # Internal stuff
diff --git a/lib_pypy/pypy_test/test_stackless_pickling.py b/lib_pypy/pypy_test/test_stackless_pickling.py
--- a/lib_pypy/pypy_test/test_stackless_pickling.py
+++ b/lib_pypy/pypy_test/test_stackless_pickling.py
@@ -1,7 +1,3 @@
-"""
-this test should probably not run from CPython or py.py.
-I'm not entirely sure, how to do that.
-"""
 from __future__ import absolute_import
 from py.test import skip
 try:
@@ -16,11 +12,15 @@
 
 class Test_StacklessPickling:
 
+    def test_pickle_main_coroutine(self):
+        import stackless, pickle
+        s = pickle.dumps(stackless.coroutine.getcurrent())
+        print s
+        c = pickle.loads(s)
+        assert c is stackless.coroutine.getcurrent()
+
     def test_basic_tasklet_pickling(self):
-        try:
-            import stackless
-        except ImportError:
-            skip("can't load stackless and don't know why!!!")
+        import stackless
         from stackless import run, schedule, tasklet
         import pickle
 
diff --git a/lib_pypy/pyrepl/completing_reader.py b/lib_pypy/pyrepl/completing_reader.py
--- a/lib_pypy/pyrepl/completing_reader.py
+++ b/lib_pypy/pyrepl/completing_reader.py
@@ -229,7 +229,8 @@
 
     def after_command(self, cmd):
         super(CompletingReader, self).after_command(cmd)
-        if not isinstance(cmd, complete) and not isinstance(cmd, self_insert):
+        if not isinstance(cmd, self.commands['complete']) \
+           and not isinstance(cmd, self.commands['self_insert']):
             self.cmpltn_reset()
 
     def calc_screen(self):
diff --git a/lib_pypy/stackless.py b/lib_pypy/stackless.py
--- a/lib_pypy/stackless.py
+++ b/lib_pypy/stackless.py
@@ -5,7 +5,6 @@
 """
 
 
-import traceback
 import _continuation
 
 class TaskletExit(Exception):
@@ -14,33 +13,46 @@
 CoroutineExit = TaskletExit
 
 
+def _coroutine_getcurrent():
+    "Returns the current coroutine (i.e. the one which called this function)."
+    try:
+        return _tls.current_coroutine
+    except AttributeError:
+        # first call in this thread: current == main
+        return _coroutine_getmain()
+
+def _coroutine_getmain():
+    try:
+        return _tls.main_coroutine
+    except AttributeError:
+        # create the main coroutine for this thread
+        continulet = _continuation.continulet
+        main = coroutine()
+        main._frame = continulet.__new__(continulet)
+        main._is_started = -1
+        _tls.current_coroutine = _tls.main_coroutine = main
+        return _tls.main_coroutine
+
+
 class coroutine(object):
-    "we can't have continulet as a base, because continulets can't be rebound"
+    _is_started = 0      # 0=no, 1=yes, -1=main
 
     def __init__(self):
         self._frame = None
-        self.is_zombie = False
-
-    def __getattr__(self, attr):
-        return getattr(self._frame, attr)
-
-    def __del__(self):
-        self.is_zombie = True
-        del self._frame
-        self._frame = None
 
     def bind(self, func, *argl, **argd):
         """coro.bind(f, *argl, **argd) -> None.
            binds function f to coro. f will be called with
            arguments *argl, **argd
         """
-        if self._frame is None or not self._frame.is_pending():
-            def run(c):
-                _tls.current_coroutine = self
-                return func(*argl, **argd)
-            self._frame = frame = _continuation.continulet(run)
-        else:
+        if self.is_alive:
             raise ValueError("cannot bind a bound coroutine")
+        def run(c):
+            _tls.current_coroutine = self
+            self._is_started = 1
+            return func(*argl, **argd)
+        self._is_started = 0
+        self._frame = _continuation.continulet(run)
 
     def switch(self):
         """coro.switch() -> returnvalue
@@ -48,7 +60,7 @@
            f finishes, the returnvalue is that of f, otherwise
            None is returned
         """
-        current = _getcurrent()
+        current = _coroutine_getcurrent()
         try:
             current._frame.switch(to=self._frame)
         finally:
@@ -56,37 +68,30 @@
 
     def kill(self):
         """coro.kill() : kill coroutine coro"""
-        current = _getcurrent()
+        current = _coroutine_getcurrent()
         try:
             current._frame.throw(CoroutineExit, to=self._frame)
         finally:
             _tls.current_coroutine = current
 
-    def _is_alive(self):
-        if self._frame is None:
-            return False
-        return not self._frame.is_pending()
-    is_alive = property(_is_alive)
-    del _is_alive
+    @property
+    def is_alive(self):
+        return self._is_started < 0 or (
+            self._frame is not None and self._frame.is_pending())
 
-    def getcurrent():
-        """coroutine.getcurrent() -> the currently running coroutine"""
-        return _getcurrent()
-    getcurrent = staticmethod(getcurrent)
+    @property
+    def is_zombie(self):
+        return self._is_started > 0 and not self._frame.is_pending()
+
+    getcurrent = staticmethod(_coroutine_getcurrent)
 
     def __reduce__(self):
-        raise TypeError, 'pickling is not possible based upon continulets'
+        if self._is_started < 0:
+            return _coroutine_getmain, ()
+        else:
+            return type(self), (), self.__dict__
 
 
-def _getcurrent():
-    "Returns the current coroutine (i.e. the one which called this function)."
-    try:
-        return _tls.current_coroutine
-    except AttributeError:
-        # first call in this thread: current == main
-        _coroutine_create_main()
-        return _tls.current_coroutine
-
 try:
     from thread import _local
 except ImportError:
@@ -95,14 +100,8 @@
 
 _tls = _local()
 
-def _coroutine_create_main():
-    # create the main coroutine for this thread
-    _tls.current_coroutine = None
-    main_coroutine = coroutine()
-    typ = _continuation.continulet
-    main_coroutine._frame = typ.__new__(typ)
-    _tls.main_coroutine = main_coroutine
-    _tls.current_coroutine = main_coroutine
+
+# ____________________________________________________________
 
 
 from collections import deque
@@ -148,10 +147,7 @@
     _last_task = next
     assert not next.blocked
     if next is not current:
-        #try:
-            next.switch()
-        #except CoroutineExit:  --- they are the same anyway
-        #    raise TaskletExit
+        next.switch()
     return current
 
 def set_schedule_callback(callback):
@@ -175,34 +171,6 @@
         raise self.type, self.value, self.traceback
 
 #
-# helpers for pickling
-#
-
-_stackless_primitive_registry = {}
-
-def register_stackless_primitive(thang, retval_expr='None'):
-    import types
-    func = thang
-    if isinstance(thang, types.MethodType):
-        func = thang.im_func
-    code = func.func_code
-    _stackless_primitive_registry[code] = retval_expr
-    # It is not too nice to attach info via the code object, but
-    # I can't think of a better solution without a real transform.
-
-def rewrite_stackless_primitive(coro_state, alive, tempval):
-    flags, frame, thunk, parent = coro_state
-    while frame is not None:
-        retval_expr = _stackless_primitive_registry.get(frame.f_code)
-        if retval_expr:
-            # this tasklet needs to stop pickling here and return its value.
-            tempval = eval(retval_expr, globals(), frame.f_locals)
-            coro_state = flags, frame, thunk, parent
-            break
-        frame = frame.f_back
-    return coro_state, alive, tempval
-
-#
 #
 
 class channel(object):
@@ -354,8 +322,6 @@
         """
         return self._channel_action(None, -1)
 
-    register_stackless_primitive(receive, retval_expr='receiver.tempval')
-
     def send_exception(self, exp_type, msg):
         self.send(bomb(exp_type, exp_type(msg)))
 
@@ -372,9 +338,8 @@
         the runnables list.
         """
         return self._channel_action(msg, 1)
-            
-    register_stackless_primitive(send)
-            
+
+
 class tasklet(coroutine):
     """
     A tasklet object represents a tiny task in a Python thread.
@@ -456,7 +421,7 @@
 
         self.func = None
         coroutine.bind(self, _func)
-        back = _getcurrent()
+        back = _coroutine_getcurrent()
         coroutine.switch(self)
         self.alive = True
         _scheduler_append(self)
@@ -480,39 +445,6 @@
             raise RuntimeError, "The current tasklet cannot be removed."
             # not sure if I will revive this  " Use t=tasklet().capture()"
         _scheduler_remove(self)
-        
-    def __reduce__(self):
-        one, two, coro_state = coroutine.__reduce__(self)
-        assert one is coroutine
-        assert two == ()
-        # we want to get rid of the parent thing.
-        # for now, we just drop it
-        a, frame, c, d = coro_state
-
-        # Removing all frames related to stackless.py.
-        # They point to stuff we don't want to be pickled.
-
-        pickleframe = frame
-        while frame is not None:
-            if frame.f_code == schedule.func_code:
-                # Removing everything including and after the
-                # call to stackless.schedule()
-                pickleframe = frame.f_back
-                break
-            frame = frame.f_back
-        if d:
-            assert isinstance(d, coroutine)
-        coro_state = a, pickleframe, c, None
-        coro_state, alive, tempval = rewrite_stackless_primitive(coro_state, self.alive, self.tempval)
-        inst_dict = self.__dict__.copy()
-        inst_dict.pop('tempval', None)
-        return self.__class__, (), (coro_state, alive, tempval, inst_dict)
-
-    def __setstate__(self, (coro_state, alive, tempval, inst_dict)):
-        coroutine.__setstate__(self, coro_state)
-        self.__dict__.update(inst_dict)
-        self.alive = alive
-        self.tempval = tempval
 
 def getmain():
     """
@@ -601,30 +533,7 @@
     global _last_task
     _global_task_id = 0
     _main_tasklet = coroutine.getcurrent()
-    try:
-        _main_tasklet.__class__ = tasklet
-    except TypeError: # we are running pypy-c
-        class TaskletProxy(object):
-            """TaskletProxy is needed to give the _main_coroutine tasklet behaviour"""
-            def __init__(self, coro):
-                self._coro = coro
-
-            def __getattr__(self,attr):
-                return getattr(self._coro,attr)
-
-            def __str__(self):
-                return '<tasklet %s a:%s>' % (self._task_id, self.is_alive)
-
-            def __reduce__(self):
-                return getmain, ()
-
-            __repr__ = __str__
-
-
-        global _main_coroutine
-        _main_coroutine = _main_tasklet
-        _main_tasklet = TaskletProxy(_main_tasklet)
-        assert _main_tasklet.is_alive and not _main_tasklet.is_zombie
+    _main_tasklet.__class__ = tasklet         # XXX HAAAAAAAAAAAAAAAAAAAAACK
     _last_task = _main_tasklet
     tasklet._init.im_func(_main_tasklet, label='main')
     _squeue = deque()
diff --git a/py/_code/source.py b/py/_code/source.py
--- a/py/_code/source.py
+++ b/py/_code/source.py
@@ -139,7 +139,7 @@
                 trysource = self[start:end]
                 if trysource.isparseable():
                     return start, end
-        return start, end
+        return start, len(self)
 
     def getblockend(self, lineno):
         # XXX
diff --git a/pypy/annotation/annrpython.py b/pypy/annotation/annrpython.py
--- a/pypy/annotation/annrpython.py
+++ b/pypy/annotation/annrpython.py
@@ -149,7 +149,7 @@
         desc = olddesc.bind_self(classdef)
         args = self.bookkeeper.build_args("simple_call", args_s[:])
         desc.consider_call_site(self.bookkeeper, desc.getcallfamily(), [desc],
-            args, annmodel.s_ImpossibleValue)
+            args, annmodel.s_ImpossibleValue, None)
         result = []
         def schedule(graph, inputcells):
             result.append((graph, inputcells))
diff --git a/pypy/annotation/bookkeeper.py b/pypy/annotation/bookkeeper.py
--- a/pypy/annotation/bookkeeper.py
+++ b/pypy/annotation/bookkeeper.py
@@ -209,8 +209,8 @@
                 self.consider_call_site(call_op)
 
             for pbc, args_s in self.emulated_pbc_calls.itervalues():
-                self.consider_call_site_for_pbc(pbc, 'simple_call', 
-                                                args_s, s_ImpossibleValue)
+                self.consider_call_site_for_pbc(pbc, 'simple_call',
+                                                args_s, s_ImpossibleValue, None)
             self.emulated_pbc_calls = {}
         finally:
             self.leave()
@@ -257,18 +257,18 @@
             args_s = [lltype_to_annotation(adtmeth.ll_ptrtype)] + args_s
         if isinstance(s_callable, SomePBC):
             s_result = binding(call_op.result, s_ImpossibleValue)
-            self.consider_call_site_for_pbc(s_callable,
-                                            call_op.opname,
-                                            args_s, s_result)
+            self.consider_call_site_for_pbc(s_callable, call_op.opname, args_s,
+                                            s_result, call_op)
 
-    def consider_call_site_for_pbc(self, s_callable, opname, args_s, s_result):
+    def consider_call_site_for_pbc(self, s_callable, opname, args_s, s_result,
+                                   call_op):
         descs = list(s_callable.descriptions)
         if not descs:
             return
         family = descs[0].getcallfamily()
         args = self.build_args(opname, args_s)
         s_callable.getKind().consider_call_site(self, family, descs, args,
-                                                s_result)
+                                                s_result, call_op)
 
     def getuniqueclassdef(self, cls):
         """Get the ClassDef associated with the given user cls.
@@ -656,6 +656,7 @@
                 whence = None
             else:
                 whence = emulated # callback case
+            op = None
             s_previous_result = s_ImpossibleValue
 
         def schedule(graph, inputcells):
@@ -663,7 +664,7 @@
 
         results = []
         for desc in descs:
-            results.append(desc.pycall(schedule, args, s_previous_result))
+            results.append(desc.pycall(schedule, args, s_previous_result, op))
         s_result = unionof(*results)
         return s_result
 
diff --git a/pypy/annotation/description.py b/pypy/annotation/description.py
--- a/pypy/annotation/description.py
+++ b/pypy/annotation/description.py
@@ -255,7 +255,11 @@
             raise TypeError, "signature mismatch: %s" % e.getmsg(self.name)
         return inputcells
 
-    def specialize(self, inputcells):
+    def specialize(self, inputcells, op=None):
+        if (op is None and
+            getattr(self.bookkeeper, "position_key", None) is not None):
+            _, block, i = self.bookkeeper.position_key
+            op = block.operations[i]
         if self.specializer is None:
             # get the specializer based on the tag of the 'pyobj'
             # (if any), according to the current policy
@@ -269,11 +273,14 @@
                 enforceargs = Sig(*enforceargs)
                 self.pyobj._annenforceargs_ = enforceargs
             enforceargs(self, inputcells) # can modify inputcells in-place
-        return self.specializer(self, inputcells)
+        if getattr(self.pyobj, '_annspecialcase_', '').endswith("call_location"):
+            return self.specializer(self, inputcells, op)
+        else:
+            return self.specializer(self, inputcells)
 
-    def pycall(self, schedule, args, s_previous_result):
+    def pycall(self, schedule, args, s_previous_result, op=None):
         inputcells = self.parse_arguments(args)
-        result = self.specialize(inputcells)
+        result = self.specialize(inputcells, op)
         if isinstance(result, FunctionGraph):
             graph = result         # common case
             # if that graph has a different signature, we need to re-parse
@@ -296,17 +303,17 @@
                                              None,       # selfclassdef
                                              name)
 
-    def consider_call_site(bookkeeper, family, descs, args, s_result):
+    def consider_call_site(bookkeeper, family, descs, args, s_result, op):
         shape = rawshape(args)
-        row = FunctionDesc.row_to_consider(descs, args)
+        row = FunctionDesc.row_to_consider(descs, args, op)
         family.calltable_add_row(shape, row)
     consider_call_site = staticmethod(consider_call_site)
 
-    def variant_for_call_site(bookkeeper, family, descs, args):
+    def variant_for_call_site(bookkeeper, family, descs, args, op):
         shape = rawshape(args)
         bookkeeper.enter(None)
         try:
-            row = FunctionDesc.row_to_consider(descs, args)
+            row = FunctionDesc.row_to_consider(descs, args, op)
         finally:
             bookkeeper.leave()
         index = family.calltable_lookup_row(shape, row)
@@ -316,7 +323,7 @@
     def rowkey(self):
         return self
 
-    def row_to_consider(descs, args):
+    def row_to_consider(descs, args, op):
         # see comments in CallFamily
         from pypy.annotation.model import s_ImpossibleValue
         row = {}
@@ -324,7 +331,7 @@
             def enlist(graph, ignore):
                 row[desc.rowkey()] = graph
                 return s_ImpossibleValue   # meaningless
-            desc.pycall(enlist, args, s_ImpossibleValue)
+            desc.pycall(enlist, args, s_ImpossibleValue, op)
         return row
     row_to_consider = staticmethod(row_to_consider)
 
@@ -521,7 +528,7 @@
                             "specialization" % (self.name,))
         return self.getclassdef(None)
 
-    def pycall(self, schedule, args, s_previous_result):
+    def pycall(self, schedule, args, s_previous_result, op=None):
         from pypy.annotation.model import SomeInstance, SomeImpossibleValue
         if self.specialize:
             if self.specialize == 'specialize:ctr_location':
@@ -664,7 +671,7 @@
             cdesc = cdesc.basedesc
         return s_result     # common case
 
-    def consider_call_site(bookkeeper, family, descs, args, s_result):
+    def consider_call_site(bookkeeper, family, descs, args, s_result, op):
         from pypy.annotation.model import SomeInstance, SomePBC, s_None
         if len(descs) == 1:
             # call to a single class, look at the result annotation
@@ -709,7 +716,7 @@
             initdescs[0].mergecallfamilies(*initdescs[1:])
             initfamily = initdescs[0].getcallfamily()
             MethodDesc.consider_call_site(bookkeeper, initfamily, initdescs,
-                                          args, s_None)
+                                          args, s_None, op)
     consider_call_site = staticmethod(consider_call_site)
 
     def getallbases(self):
@@ -782,13 +789,13 @@
     def getuniquegraph(self):
         return self.funcdesc.getuniquegraph()
 
-    def pycall(self, schedule, args, s_previous_result):
+    def pycall(self, schedule, args, s_previous_result, op=None):
         from pypy.annotation.model import SomeInstance
         if self.selfclassdef is None:
             raise Exception("calling %r" % (self,))
         s_instance = SomeInstance(self.selfclassdef, flags = self.flags)
         args = args.prepend(s_instance)
-        return self.funcdesc.pycall(schedule, args, s_previous_result)
+        return self.funcdesc.pycall(schedule, args, s_previous_result, op)
 
     def bind_under(self, classdef, name):
         self.bookkeeper.warning("rebinding an already bound %r" % (self,))
@@ -801,10 +808,10 @@
                                              self.name,
                                              flags)
 
-    def consider_call_site(bookkeeper, family, descs, args, s_result):
+    def consider_call_site(bookkeeper, family, descs, args, s_result, op):
         shape = rawshape(args, nextra=1)     # account for the extra 'self'
         funcdescs = [methoddesc.funcdesc for methoddesc in descs]
-        row = FunctionDesc.row_to_consider(descs, args)
+        row = FunctionDesc.row_to_consider(descs, args, op)
         family.calltable_add_row(shape, row)
     consider_call_site = staticmethod(consider_call_site)
 
@@ -956,16 +963,16 @@
         return '<MethodOfFrozenDesc %r of %r>' % (self.funcdesc,
                                                   self.frozendesc)
 
-    def pycall(self, schedule, args, s_previous_result):
+    def pycall(self, schedule, args, s_previous_result, op=None):
         from pypy.annotation.model import SomePBC
         s_self = SomePBC([self.frozendesc])
         args = args.prepend(s_self)
-        return self.funcdesc.pycall(schedule, args, s_previous_result)
+        return self.funcdesc.pycall(schedule, args, s_previous_result, op)
 
-    def consider_call_site(bookkeeper, family, descs, args, s_result):
+    def consider_call_site(bookkeeper, family, descs, args, s_result, op):
         shape = rawshape(args, nextra=1)    # account for the extra 'self'
         funcdescs = [mofdesc.funcdesc for mofdesc in descs]
-        row = FunctionDesc.row_to_consider(descs, args)
+        row = FunctionDesc.row_to_consider(descs, args, op)
         family.calltable_add_row(shape, row)
     consider_call_site = staticmethod(consider_call_site)
 
diff --git a/pypy/annotation/policy.py b/pypy/annotation/policy.py
--- a/pypy/annotation/policy.py
+++ b/pypy/annotation/policy.py
@@ -1,7 +1,7 @@
 # base annotation policy for specialization
 from pypy.annotation.specialize import default_specialize as default
-from pypy.annotation.specialize import specialize_argvalue, specialize_argtype, specialize_arglistitemtype
-from pypy.annotation.specialize import memo
+from pypy.annotation.specialize import specialize_argvalue, specialize_argtype, specialize_arglistitemtype, specialize_arg_or_var
+from pypy.annotation.specialize import memo, specialize_call_location
 # for some reason, model must be imported first,
 # or we create a cycle.
 from pypy.annotation import model as annmodel
@@ -73,8 +73,10 @@
     default_specialize = staticmethod(default)
     specialize__memo = staticmethod(memo)
     specialize__arg = staticmethod(specialize_argvalue) # specialize:arg(N)
+    specialize__arg_or_var = staticmethod(specialize_arg_or_var)
     specialize__argtype = staticmethod(specialize_argtype) # specialize:argtype(N)
     specialize__arglistitemtype = staticmethod(specialize_arglistitemtype)
+    specialize__call_location = staticmethod(specialize_call_location)
 
     def specialize__ll(pol, *args):
         from pypy.rpython.annlowlevel import LowLevelAnnotatorPolicy
diff --git a/pypy/annotation/specialize.py b/pypy/annotation/specialize.py
--- a/pypy/annotation/specialize.py
+++ b/pypy/annotation/specialize.py
@@ -353,6 +353,16 @@
     key = tuple(key)
     return maybe_star_args(funcdesc, key, args_s)
 
+def specialize_arg_or_var(funcdesc, args_s, *argindices):
+    for argno in argindices:
+        if not args_s[argno].is_constant():
+            break
+    else:
+        # all constant
+        return specialize_argvalue(funcdesc, args_s, *argindices)
+    # some not constant
+    return maybe_star_args(funcdesc, None, args_s)
+
 def specialize_argtype(funcdesc, args_s, *argindices):
     key = tuple([args_s[i].knowntype for i in argindices])
     for cls in key:
@@ -370,3 +380,7 @@
     else:
         key = s.listdef.listitem.s_value.knowntype
     return maybe_star_args(funcdesc, key, args_s)
+
+def specialize_call_location(funcdesc, args_s, op):
+    assert op is not None
+    return maybe_star_args(funcdesc, op, args_s)
diff --git a/pypy/annotation/test/test_annrpython.py b/pypy/annotation/test/test_annrpython.py
--- a/pypy/annotation/test/test_annrpython.py
+++ b/pypy/annotation/test/test_annrpython.py
@@ -1099,8 +1099,8 @@
         allocdesc = a.bookkeeper.getdesc(alloc)
         s_C1 = a.bookkeeper.immutablevalue(C1)
         s_C2 = a.bookkeeper.immutablevalue(C2)
-        graph1 = allocdesc.specialize([s_C1])
-        graph2 = allocdesc.specialize([s_C2])
+        graph1 = allocdesc.specialize([s_C1], None)
+        graph2 = allocdesc.specialize([s_C2], None)
         assert a.binding(graph1.getreturnvar()).classdef == C1df
         assert a.binding(graph2.getreturnvar()).classdef == C2df
         assert graph1 in a.translator.graphs
@@ -1135,8 +1135,8 @@
         allocdesc = a.bookkeeper.getdesc(alloc)
         s_C1 = a.bookkeeper.immutablevalue(C1)
         s_C2 = a.bookkeeper.immutablevalue(C2)
-        graph1 = allocdesc.specialize([s_C1, s_C2])
-        graph2 = allocdesc.specialize([s_C2, s_C2])
+        graph1 = allocdesc.specialize([s_C1, s_C2], None)
+        graph2 = allocdesc.specialize([s_C2, s_C2], None)
         assert a.binding(graph1.getreturnvar()).classdef == C1df
         assert a.binding(graph2.getreturnvar()).classdef == C2df
         assert graph1 in a.translator.graphs
@@ -1194,6 +1194,33 @@
         assert len(executedesc._cache[(0, 'star', 2)].startblock.inputargs) == 4
         assert len(executedesc._cache[(1, 'star', 3)].startblock.inputargs) == 5
 
+    def test_specialize_arg_or_var(self):
+        def f(a):
+            return 1
+        f._annspecialcase_ = 'specialize:arg_or_var(0)'
+
+        def fn(a):
+            return f(3) + f(a)
+
+        a = self.RPythonAnnotator()
+        a.build_types(fn, [int])
+        executedesc = a.bookkeeper.getdesc(f)
+        assert sorted(executedesc._cache.keys()) == [None, (3,)]
+        # we got two different special
+
+    def test_specialize_call_location(self):
+        def g(a):
+            return a
+        g._annspecialcase_ = "specialize:call_location"
+        def f(x):
+            return g(x)
+        f._annspecialcase_ = "specialize:argtype(0)"
+        def h(y):
+            w = f(y)
+            return int(f(str(y))) + w
+        a = self.RPythonAnnotator()
+        assert a.build_types(h, [int]) == annmodel.SomeInteger()
+
     def test_assert_list_doesnt_lose_info(self):
         class T(object):
             pass
@@ -3177,6 +3204,8 @@
         s = a.build_types(f, [])
         assert isinstance(s, annmodel.SomeList)
         assert not s.listdef.listitem.resized
+        assert not s.listdef.listitem.immutable
+        assert s.listdef.listitem.mutated
 
     def test_delslice(self):
         def f():
diff --git a/pypy/annotation/unaryop.py b/pypy/annotation/unaryop.py
--- a/pypy/annotation/unaryop.py
+++ b/pypy/annotation/unaryop.py
@@ -352,6 +352,7 @@
         check_negative_slice(s_start, s_stop)
         if not isinstance(s_iterable, SomeList):
             raise Exception("list[start:stop] = x: x must be a list")
+        lst.listdef.mutate()
         lst.listdef.agree(s_iterable.listdef)
         # note that setslice is not allowed to resize a list in RPython
 
diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst
--- a/pypy/doc/index.rst
+++ b/pypy/doc/index.rst
@@ -21,8 +21,6 @@
 
 * `Papers`_: Academic papers, talks, and related projects
 
-* `Videos`_: Videos of PyPy talks and presentations
-
 * `speed.pypy.org`_: Daily benchmarks of how fast PyPy is
 
 * `potential project ideas`_: In case you want to get your feet wet...
diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -53,6 +53,18 @@
 this is an ideal task to get started, because it does not require any deep
 knowledge of the internals.
 
+Optimized Unicode Representation
+--------------------------------
+
+CPython 3.3 will use an `optimized unicode representation`_ which switches between
+different ways to represent a unicode string, depending on whether the string
+fits into ASCII, has only two-byte characters or needs four-byte characters.
+
+The actual details would be rather differen in PyPy, but we would like to have
+the same optimization implemented.
+
+.. _`optimized unicode representation`: http://www.python.org/dev/peps/pep-0393/
+
 Translation Toolchain
 ---------------------
 
diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py
--- a/pypy/interpreter/argument.py
+++ b/pypy/interpreter/argument.py
@@ -125,6 +125,7 @@
 
     ###  Manipulation  ###
 
+    @jit.look_inside_iff(lambda self: not self._dont_jit)
     def unpack(self): # slowish
         "Return a ([w1,w2...], {'kw':w3...}) pair."
         kwds_w = {}
@@ -245,6 +246,8 @@
 
     ###  Parsing for function calls  ###
 
+    # XXX: this should be @jit.look_inside_iff, but we need key word arguments,
+    # and it doesn't support them for now.
     def _match_signature(self, w_firstarg, scope_w, signature, defaults_w=None,
                          blindargs=0):
         """Parse args and kwargs according to the signature of a code object,
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -3,12 +3,12 @@
 from pypy.interpreter.executioncontext import ExecutionContext, ActionFlag
 from pypy.interpreter.executioncontext import UserDelAction, FrameTraceAction
 from pypy.interpreter.error import OperationError, operationerrfmt
-from pypy.interpreter.error import new_exception_class
+from pypy.interpreter.error import new_exception_class, typed_unwrap_error_msg
 from pypy.interpreter.argument import Arguments
 from pypy.interpreter.miscutils import ThreadLocals
 from pypy.tool.cache import Cache
 from pypy.tool.uid import HUGEVAL_BYTES
-from pypy.rlib.objectmodel import we_are_translated, newlist
+from pypy.rlib.objectmodel import we_are_translated, newlist, compute_unique_id
 from pypy.rlib.debug import make_sure_not_resized
 from pypy.rlib.timer import DummyTimer, Timer
 from pypy.rlib.rarithmetic import r_uint
@@ -186,6 +186,28 @@
     def _set_mapdict_storage_and_map(self, storage, map):
         raise NotImplementedError
 
+    # -------------------------------------------------------------------
+
+    def str_w(self, space):
+        w_msg = typed_unwrap_error_msg(space, "string", self)
+        raise OperationError(space.w_TypeError, w_msg)
+
+    def unicode_w(self, space):
+        raise OperationError(space.w_TypeError,
+                             typed_unwrap_error_msg(space, "unicode", self))
+
+    def int_w(self, space):
+        raise OperationError(space.w_TypeError,
+                             typed_unwrap_error_msg(space, "integer", self))
+    
+    def uint_w(self, space):
+        raise OperationError(space.w_TypeError,
+                             typed_unwrap_error_msg(space, "integer", self))
+    
+    def bigint_w(self, space):
+        raise OperationError(space.w_TypeError,
+                             typed_unwrap_error_msg(space, "integer", self))
+
 
 class Wrappable(W_Root):
     """A subclass of Wrappable is an internal, interpreter-level class
@@ -901,7 +923,7 @@
         ec.c_call_trace(frame, w_func, args)
         try:
             w_res = self.call_args(w_func, args)
-        except OperationError, e:
+        except OperationError:
             ec.c_exception_trace(frame, w_func)
             raise
         ec.c_return_trace(frame, w_func, args)
@@ -947,6 +969,9 @@
     def isinstance_w(self, w_obj, w_type):
         return self.is_true(self.isinstance(w_obj, w_type))
 
+    def id(self, w_obj):
+        return self.wrap(compute_unique_id(w_obj))
+
     # The code below only works
     # for the simple case (new-style instance).
     # These methods are patched with the full logic by the __builtin__
@@ -999,8 +1024,6 @@
 
     def eval(self, expression, w_globals, w_locals, hidden_applevel=False):
         "NOT_RPYTHON: For internal debugging."
-        import types
-        from pypy.interpreter.pycode import PyCode
         if isinstance(expression, str):
             compiler = self.createcompiler()
             expression = compiler.compile(expression, '?', 'eval', 0,
@@ -1012,7 +1035,6 @@
     def exec_(self, statement, w_globals, w_locals, hidden_applevel=False,
               filename=None):
         "NOT_RPYTHON: For internal debugging."
-        import types
         if filename is None:
             filename = '?'
         from pypy.interpreter.pycode import PyCode
@@ -1210,6 +1232,18 @@
             return None
         return self.str_w(w_obj)
 
+    def str_w(self, w_obj):
+        return w_obj.str_w(self)
+
+    def int_w(self, w_obj):
+        return w_obj.int_w(self)
+
+    def uint_w(self, w_obj):
+        return w_obj.uint_w(self)
+
+    def bigint_w(self, w_obj):
+        return w_obj.bigint_w(self)
+
     def realstr_w(self, w_obj):
         # Like str_w, but only works if w_obj is really of type 'str'.
         if not self.is_true(self.isinstance(w_obj, self.w_str)):
@@ -1217,6 +1251,9 @@
                                  self.wrap('argument must be a string'))
         return self.str_w(w_obj)
 
+    def unicode_w(self, w_obj):
+        return w_obj.unicode_w(self)
+
     def realunicode_w(self, w_obj):
         # Like unicode_w, but only works if w_obj is really of type
         # 'unicode'.
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -458,3 +458,7 @@
     if module:
         space.setattr(w_exc, space.wrap("__module__"), space.wrap(module))
     return w_exc
+
+def typed_unwrap_error_msg(space, expected, w_obj):
+    type_name = space.type(w_obj).getname(space)
+    return space.wrap("expected %s, got %s object" % (expected, type_name))
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -1,5 +1,4 @@
 import sys
-from pypy.interpreter.miscutils import Stack
 from pypy.interpreter.error import OperationError
 from pypy.rlib.rarithmetic import LONG_BIT
 from pypy.rlib.unroll import unrolling_iterable
@@ -176,6 +175,9 @@
             self.w_tracefunc = w_func
             self.space.frame_trace_action.fire()
 
+    def gettrace(self):
+        return self.w_tracefunc
+
     def setprofile(self, w_func):
         """Set the global trace function."""
         if self.space.is_w(w_func, self.space.w_None):
@@ -308,7 +310,11 @@
         self._nonperiodic_actions = []
         self.has_bytecode_counter = False
         self.fired_actions = None
-        self.checkinterval_scaled = 100 * TICK_COUNTER_STEP
+        # the default value is not 100, unlike CPython 2.7, but a much
+        # larger value, because we use a technique that not only allows
+        # but actually *forces* another thread to run whenever the counter
+        # reaches zero.
+        self.checkinterval_scaled = 10000 * TICK_COUNTER_STEP
         self._rebuild_action_dispatcher()
 
     def fire(self, action):
@@ -347,6 +353,7 @@
         elif interval > MAX:
             interval = MAX
         self.checkinterval_scaled = interval * TICK_COUNTER_STEP
+        self.reset_ticker(-1)
 
     def _rebuild_action_dispatcher(self):
         periodic_actions = unrolling_iterable(self._periodic_actions)
diff --git a/pypy/interpreter/miscutils.py b/pypy/interpreter/miscutils.py
--- a/pypy/interpreter/miscutils.py
+++ b/pypy/interpreter/miscutils.py
@@ -2,154 +2,6 @@
 Miscellaneous utilities.
 """
 
-import types
-
-from pypy.rlib.rarithmetic import r_uint
-
-class RootStack:
-    pass
-
-class Stack(RootStack):
-    """Utility class implementing a stack."""
-
-    _annspecialcase_ = "specialize:ctr_location" # polymorphic
-
-    def __init__(self):
-        self.items = []
-
-    def clone(self):
-        s = self.__class__()
-        for item in self.items:
-            try:
-                item = item.clone()
-            except AttributeError:
-                pass
-            s.push(item)
-        return s
-
-    def push(self, item):
-        self.items.append(item)
-
-    def pop(self):
-        return self.items.pop()
-
-    def drop(self, n):
-        if n > 0:
-            del self.items[-n:]
-
-    def top(self, position=0):
-        """'position' is 0 for the top of the stack, 1 for the item below,
-        and so on.  It must not be negative."""
-        if position < 0:
-            raise ValueError, 'negative stack position'
-        if position >= len(self.items):
-            raise IndexError, 'not enough entries in stack'
-        return self.items[~position]
-
-    def set_top(self, value, position=0):
-        """'position' is 0 for the top of the stack, 1 for the item below,
-        and so on.  It must not be negative."""
-        if position < 0:
-            raise ValueError, 'negative stack position'
-        if position >= len(self.items):
-            raise IndexError, 'not enough entries in stack'
-        self.items[~position] = value
-
-    def depth(self):
-        return len(self.items)
-
-    def empty(self):
-        return len(self.items) == 0
-
-
-class FixedStack(RootStack):
-    _annspecialcase_ = "specialize:ctr_location" # polymorphic
-
-    # unfortunately, we have to re-do everything
-    def __init__(self):
-        pass
-
-    def setup(self, stacksize):
-        self.ptr = r_uint(0) # we point after the last element
-        self.items = [None] * stacksize
-
-    def clone(self):
-        # this is only needed if we support flow space
-        s = self.__class__()
-        s.setup(len(self.items))
-        for item in self.items[:self.ptr]:
-            try:
-                item = item.clone()
-            except AttributeError:
-                pass
-            s.push(item)
-        return s
-
-    def push(self, item):
-        ptr = self.ptr
-        self.items[ptr] = item
-        self.ptr = ptr + 1
-
-    def pop(self):
-        ptr = self.ptr - 1
-        ret = self.items[ptr]   # you get OverflowError if the stack is empty
-        self.items[ptr] = None
-        self.ptr = ptr
-        return ret
-
-    def drop(self, n):
-        while n > 0:
-            n -= 1
-            self.ptr -= 1
-            self.items[self.ptr] = None
-
-    def top(self, position=0):
-        # for a fixed stack, we assume correct indices
-        return self.items[self.ptr + ~position]
-
-    def set_top(self, value, position=0):
-        # for a fixed stack, we assume correct indices
-        self.items[self.ptr + ~position] = value
-
-    def depth(self):
-        return self.ptr
-
-    def empty(self):
-        return not self.ptr
-
-
-class InitializedClass(type):
-    """NOT_RPYTHON.  A meta-class that allows a class to initialize itself (or
-    its subclasses) by calling __initclass__() as a class method."""
-    def __init__(self, name, bases, dict):
-        super(InitializedClass, self).__init__(name, bases, dict)
-        for basecls in self.__mro__:
-            raw = basecls.__dict__.get('__initclass__')
-            if isinstance(raw, types.FunctionType):
-                raw(self)   # call it as a class method
-
-
-class RwDictProxy(object):
-    """NOT_RPYTHON.  A dict-like class standing for 'cls.__dict__', to work
-    around the fact that the latter is a read-only proxy for new-style
-    classes."""
-    
-    def __init__(self, cls):
-        self.cls = cls
-
-    def __getitem__(self, attr):
-        return self.cls.__dict__[attr]
-
-    def __setitem__(self, attr, value):
-        setattr(self.cls, attr, value)
-
-    def __contains__(self, value):
-        return value in self.cls.__dict__
-
-    def items(self):
-        return self.cls.__dict__.items()
-
-
 class ThreadLocals:
     """Pseudo thread-local storage, for 'space.threadlocals'.
     This is not really thread-local at all; the intention is that the PyPy
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -66,7 +66,7 @@
         make_sure_not_resized(self.locals_stack_w)
         check_nonneg(self.nlocals)
         #
-        if space.config.objspace.honor__builtins__ and w_globals is not None:
+        if space.config.objspace.honor__builtins__:
             self.builtin = space.builtin.pick_builtin(w_globals)
         # regular functions always have CO_OPTIMIZED and CO_NEWLOCALS.
         # class bodies only have CO_NEWLOCALS.
diff --git a/pypy/interpreter/pyparser/future.py b/pypy/interpreter/pyparser/future.py
--- a/pypy/interpreter/pyparser/future.py
+++ b/pypy/interpreter/pyparser/future.py
@@ -225,14 +225,16 @@
             raise DoneException
         self.consume_whitespace()
 
-    def consume_whitespace(self):
+    def consume_whitespace(self, newline_ok=False):
         while 1:
             c = self.getc()
             if c in whitespace:
                 self.pos += 1
                 continue
-            elif c == '\\':
-                self.pos += 1
+            elif c == '\\' or newline_ok:
+                slash = c == '\\'
+                if slash:
+                    self.pos += 1
                 c = self.getc()
                 if c == '\n':
                     self.pos += 1
@@ -243,8 +245,10 @@
                     if self.getc() == '\n':
                         self.pos += 1
                         self.atbol()
+                elif slash:
+                    raise DoneException
                 else:
-                    raise DoneException
+                    return
             else:
                 return
 
@@ -281,7 +285,7 @@
             return
         else:
             self.pos += 1
-            self.consume_whitespace()
+            self.consume_whitespace(paren_list)
             if paren_list and self.getc() == ')':
                 self.pos += 1
                 return # Handles trailing comma inside parenthesis
diff --git a/pypy/interpreter/pyparser/test/test_futureautomaton.py b/pypy/interpreter/pyparser/test/test_futureautomaton.py
--- a/pypy/interpreter/pyparser/test/test_futureautomaton.py
+++ b/pypy/interpreter/pyparser/test/test_futureautomaton.py
@@ -3,7 +3,7 @@
 from pypy.tool import stdlib___future__ as fut
 
 def run(s):
-    f = future.FutureAutomaton(future.futureFlags_2_5, s)
+    f = future.FutureAutomaton(future.futureFlags_2_7, s)
     try:
         f.start()
     except future.DoneException:
@@ -113,6 +113,14 @@
     assert f.lineno == 1
     assert f.col_offset == 0
 
+def test_paren_with_newline():
+    s = 'from __future__ import (division,\nabsolute_import)\n'
+    f = run(s)
+    assert f.pos == len(s)
+    assert f.flags == (fut.CO_FUTURE_DIVISION | fut.CO_FUTURE_ABSOLUTE_IMPORT)
+    assert f.lineno == 1
+    assert f.col_offset == 0
+
 def test_multiline():
     s = '"abc" #def\n  #ghi\nfrom  __future__ import (division as b, generators,)\nfrom __future__ import with_statement\n'
     f = run(s)
diff --git a/pypy/interpreter/test/test_executioncontext.py b/pypy/interpreter/test/test_executioncontext.py
--- a/pypy/interpreter/test/test_executioncontext.py
+++ b/pypy/interpreter/test/test_executioncontext.py
@@ -42,6 +42,7 @@
         assert i == 9
 
     def test_periodic_action(self):
+        from pypy.interpreter.executioncontext import ActionFlag
 
         class DemoAction(executioncontext.PeriodicAsyncAction):
             counter = 0
@@ -53,17 +54,20 @@
 
         space = self.space
         a2 = DemoAction(space)
-        space.actionflag.register_periodic_action(a2, True)
         try:
-            for i in range(500):
-                space.appexec([], """():
-                    n = 5
-                    return n + 2
-                """)
-        except Finished:
-            pass
-        checkinterval = space.actionflag.getcheckinterval()
-        assert checkinterval / 10 < i < checkinterval * 1.1
+            space.actionflag.setcheckinterval(100)
+            space.actionflag.register_periodic_action(a2, True)
+            try:
+                for i in range(500):
+                    space.appexec([], """():
+                        n = 5
+                        return n + 2
+                    """)
+            except Finished:
+                pass
+        finally:
+            space.actionflag = ActionFlag()   # reset to default
+        assert 10 < i < 110
 
     def test_llprofile(self):
         l = []
diff --git a/pypy/jit/backend/llsupport/llmodel.py b/pypy/jit/backend/llsupport/llmodel.py
--- a/pypy/jit/backend/llsupport/llmodel.py
+++ b/pypy/jit/backend/llsupport/llmodel.py
@@ -498,6 +498,16 @@
         u = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), string)
         u.chars[index] = unichr(newvalue)
 
+    def bh_copystrcontent(self, src, dst, srcstart, dststart, length):
+        src = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), src)
+        dst = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), dst)
+        rstr.copy_string_contents(src, dst, srcstart, dststart, length)
+
+    def bh_copyunicodecontent(self, src, dst, srcstart, dststart, length):
+        src = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), src)
+        dst = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), dst)
+        rstr.copy_unicode_contents(src, dst, srcstart, dststart, length)
+
     def bh_call_i(self, func, calldescr, args_i, args_r, args_f):
         assert isinstance(calldescr, BaseIntCallDescr)
         if not we_are_translated():
diff --git a/pypy/jit/backend/model.py b/pypy/jit/backend/model.py
--- a/pypy/jit/backend/model.py
+++ b/pypy/jit/backend/model.py
@@ -303,6 +303,10 @@
         raise NotImplementedError
     def bh_unicodesetitem(self, string, index, newvalue):
         raise NotImplementedError
+    def bh_copystrcontent(self, src, dst, srcstart, dststart, length):
+        raise NotImplementedError
+    def bh_copyunicodecontent(self, src, dst, srcstart, dststart, length):
+        raise NotImplementedError
 
     def force(self, force_token):
         raise NotImplementedError
diff --git a/pypy/jit/codewriter/effectinfo.py b/pypy/jit/codewriter/effectinfo.py
--- a/pypy/jit/codewriter/effectinfo.py
+++ b/pypy/jit/codewriter/effectinfo.py
@@ -74,6 +74,7 @@
     OS_LLONG_UGE                = 91
     OS_LLONG_URSHIFT            = 92
     OS_LLONG_FROM_UINT          = 93
+    OS_LLONG_U_TO_FLOAT         = 94
     #
     OS_MATH_SQRT                = 100
 
diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -440,6 +440,7 @@
     rewrite_op_ullong_mod_zer      = _do_builtin_call
     rewrite_op_gc_identityhash = _do_builtin_call
     rewrite_op_gc_id           = _do_builtin_call
+    rewrite_op_uint_mod        = _do_builtin_call
 
     # ----------
     # getfield/setfield/mallocs etc.
@@ -883,9 +884,15 @@
                 v = v_arg
                 oplist = []
             if unsigned1:
-                opname = 'cast_uint_to_longlong'
+                if unsigned2:
+                    opname = 'cast_uint_to_ulonglong'
+                else:
+                    opname = 'cast_uint_to_longlong'
             else:
-                opname = 'cast_int_to_longlong'
+                if unsigned2:
+                    opname = 'cast_int_to_ulonglong'
+                else:
+                    opname = 'cast_int_to_longlong'
             op2 = self.rewrite_operation(
                 SpaceOperation(opname, [v], v_result)
             )
@@ -995,6 +1002,21 @@
                 return op2
         ''' % (_op, _oopspec.lower(), _oopspec, _oopspec)).compile()
 
+    for _op, _oopspec in [('cast_int_to_ulonglong',     'FROM_INT'),
+                          ('cast_uint_to_ulonglong',    'FROM_UINT'),
+                          ('cast_float_to_ulonglong',   'FROM_FLOAT'),
+                          ('cast_ulonglong_to_float',   'U_TO_FLOAT'),
+                         ]:
+        exec py.code.Source('''
+            def rewrite_op_%s(self, op):
+                args = op.args
+                op1 = self.prepare_builtin_call(op, "ullong_%s", args)
+                op2 = self._handle_oopspec_call(op1, args,
+                                                EffectInfo.OS_LLONG_%s,
+                                           EffectInfo.EF_ELIDABLE_CANNOT_RAISE)
+                return op2
+        ''' % (_op, _oopspec.lower(), _oopspec)).compile()
+
     def _normalize(self, oplist):
         if isinstance(oplist, SpaceOperation):
             return [oplist]
@@ -1158,6 +1180,12 @@
             return SpaceOperation('%s_assert_green' % kind, args, None)
         elif oopspec_name == 'jit.current_trace_length':
             return SpaceOperation('current_trace_length', [], op.result)
+        elif oopspec_name == 'jit.isconstant':
+            kind = getkind(args[0].concretetype)
+            return SpaceOperation('%s_isconstant' % kind, args, op.result)
+        elif oopspec_name == 'jit.isvirtual':
+            kind = getkind(args[0].concretetype)
+            return SpaceOperation('%s_isvirtual' % kind, args, op.result)
         else:
             raise AssertionError("missing support for %r" % oopspec_name)
 
@@ -1415,6 +1443,14 @@
         else:
             assert 0, "args[0].concretetype must be STR or UNICODE"
         #
+        if oopspec_name == 'stroruni.copy_contents':
+            if SoU.TO == rstr.STR:
+                new_op = 'copystrcontent'
+            elif SoU.TO == rstr.UNICODE:
+                new_op = 'copyunicodecontent'
+            else:
+                assert 0
+            return SpaceOperation(new_op, args, op.result)
         if oopspec_name == "stroruni.equal":
             for otherindex, othername, argtypes, resulttype in [
                 (EffectInfo.OS_STREQ_SLICE_CHECKNULL,
diff --git a/pypy/jit/codewriter/support.py b/pypy/jit/codewriter/support.py
--- a/pypy/jit/codewriter/support.py
+++ b/pypy/jit/codewriter/support.py
@@ -315,18 +315,30 @@
 def _ll_1_llong_from_int(x):
     return r_longlong(intmask(x))
 
+def _ll_1_ullong_from_int(x):
+    return r_ulonglong(intmask(x))
+
 def _ll_1_llong_from_uint(x):
     return r_longlong(r_uint(x))
 
+def _ll_1_ullong_from_uint(x):
+    return r_ulonglong(r_uint(x))
+
 def _ll_1_llong_to_int(xll):
     return intmask(xll)
 
 def _ll_1_llong_from_float(xf):
     return r_longlong(xf)
 
+def _ll_1_ullong_from_float(xf):
+    return r_ulonglong(xf)
+
 def _ll_1_llong_to_float(xll):
     return float(rffi.cast(lltype.SignedLongLong, xll))
 
+def _ll_1_ullong_u_to_float(xull):
+    return float(rffi.cast(lltype.UnsignedLongLong, xull))
+
 
 def _ll_1_llong_abs(xll):
     if xll < 0:
@@ -351,20 +363,23 @@
     return llop.llong_mod(lltype.SignedLongLong, xll, yll)
 
 def _ll_2_ullong_floordiv(xll, yll):
-    return llop.ullong_floordiv(lltype.SignedLongLong, xll, yll)
+    return llop.ullong_floordiv(lltype.UnsignedLongLong, xll, yll)
 
 def _ll_2_ullong_floordiv_zer(xll, yll):
     if yll == 0:
         raise ZeroDivisionError
-    return llop.ullong_floordiv(lltype.SignedLongLong, xll, yll)
+    return llop.ullong_floordiv(lltype.UnsignedLongLong, xll, yll)
 
 def _ll_2_ullong_mod(xll, yll):
-    return llop.ullong_mod(lltype.SignedLongLong, xll, yll)
+    return llop.ullong_mod(lltype.UnsignedLongLong, xll, yll)
 
 def _ll_2_ullong_mod_zer(xll, yll):
     if yll == 0:
         raise ZeroDivisionError
-    return llop.ullong_mod(lltype.SignedLongLong, xll, yll)
+    return llop.ullong_mod(lltype.UnsignedLongLong, xll, yll)
+
+def _ll_2_uint_mod(xll, yll):
+    return llop.uint_mod(lltype.Unsigned, xll, yll)
 
 
 # libffi support
diff --git a/pypy/jit/codewriter/test/test_flatten.py b/pypy/jit/codewriter/test/test_flatten.py
--- a/pypy/jit/codewriter/test/test_flatten.py
+++ b/pypy/jit/codewriter/test/test_flatten.py
@@ -829,14 +829,15 @@
                     self.encoding_test(f, [rffi.cast(FROM, 42)], expectedstr,
                                        transform=True)
                 elif TO in (rffi.LONG, rffi.ULONG):
+                    if rffi.cast(FROM, -1) < 0:
+                        fnname = "llong_from_int"
+                    else:
+                        fnname = "llong_from_uint"
                     if TO == rffi.LONG:
                         TO = rffi.LONGLONG
                     else:
                         TO = rffi.ULONGLONG
-                    if rffi.cast(FROM, -1) < 0:
-                        fnname = "llong_from_int"
-                    else:
-                        fnname = "llong_from_uint"
+                        fnname = "u" + fnname
                     expected.pop()   # remove int_return
                     expected.append(
                         "residual_call_irf_f $<* fn %s>, <Descr>, I[%s], R[], F[] -> %%f0"
diff --git a/pypy/jit/codewriter/test/test_jtransform.py b/pypy/jit/codewriter/test/test_jtransform.py
--- a/pypy/jit/codewriter/test/test_jtransform.py
+++ b/pypy/jit/codewriter/test/test_jtransform.py
@@ -1,5 +1,19 @@
 import py
 import random
+try:
+    from itertools import product
+except ImportError:
+    # Python 2.5, this is taken from the CPython docs, but simplified.
+    def product(*args):
+        # product('ABCD', 'xy') --> Ax Ay Bx By Cx Cy Dx Dy
+        # product(range(2), repeat=3) --> 000 001 010 011 100 101 110 111
+        pools = map(tuple, args)
+        result = [[]]
+        for pool in pools:
+            result = [x+[y] for x in result for y in pool]
+        for prod in result:
+            yield tuple(prod)
+
 from pypy.objspace.flow.model import FunctionGraph, Block, Link
 from pypy.objspace.flow.model import SpaceOperation, Variable, Constant
 from pypy.jit.codewriter.jtransform import Transformer
@@ -254,26 +268,35 @@
             assert op1.result is None
 
 def test_calls():
-    for RESTYPE in [lltype.Signed, rclass.OBJECTPTR,
-                    lltype.Float, lltype.Void]:
-      for with_void in [False, True]:
-        for with_i in [False, True]:
-          for with_r in [False, True]:
-            for with_f in [False, True]:
-              ARGS = []
-              if with_void: ARGS += [lltype.Void, lltype.Void]
-              if with_i: ARGS += [lltype.Signed, lltype.Char]
-              if with_r: ARGS += [rclass.OBJECTPTR, lltype.Ptr(rstr.STR)]
-              if with_f: ARGS += [lltype.Float, lltype.Float]
-              random.shuffle(ARGS)
-              if RESTYPE == lltype.Float: with_f = True
-              if with_f: expectedkind = 'irf'   # all kinds
-              elif with_i: expectedkind = 'ir'  # integers and references
-              else: expectedkind = 'r'          # only references
-              yield residual_call_test, ARGS, RESTYPE, expectedkind
-              yield direct_call_test, ARGS, RESTYPE, expectedkind
-              yield indirect_residual_call_test, ARGS, RESTYPE, expectedkind
-              yield indirect_regular_call_test, ARGS, RESTYPE, expectedkind
+    for RESTYPE, with_void, with_i, with_r, with_f in product(
+        [lltype.Signed, rclass.OBJECTPTR, lltype.Float, lltype.Void],
+        [False, True],
+        [False, True],
+        [False, True],
+        [False, True],
+    ):
+        ARGS = []
+        if with_void:
+            ARGS += [lltype.Void, lltype.Void]
+        if with_i:
+            ARGS += [lltype.Signed, lltype.Char]
+        if with_r:
+            ARGS += [rclass.OBJECTPTR, lltype.Ptr(rstr.STR)]
+        if with_f:
+            ARGS += [lltype.Float, lltype.Float]
+        random.shuffle(ARGS)
+        if RESTYPE == lltype.Float:
+            with_f = True
+        if with_f:
+            expectedkind = 'irf'   # all kinds
+        elif with_i:
+            expectedkind = 'ir'  # integers and references
+        else:
+            expectedkind = 'r'          # only references
+        yield residual_call_test, ARGS, RESTYPE, expectedkind
+        yield direct_call_test, ARGS, RESTYPE, expectedkind
+        yield indirect_residual_call_test, ARGS, RESTYPE, expectedkind
+        yield indirect_regular_call_test, ARGS, RESTYPE, expectedkind
 
 def get_direct_call_op(argtypes, restype):
     FUNC = lltype.FuncType(argtypes, restype)
diff --git a/pypy/jit/codewriter/test/test_longlong.py b/pypy/jit/codewriter/test/test_longlong.py
--- a/pypy/jit/codewriter/test/test_longlong.py
+++ b/pypy/jit/codewriter/test/test_longlong.py
@@ -57,7 +57,8 @@
             assert op1.opname == 'residual_call_irf_f'
         else:
             assert op1.opname == 'residual_call_irf_i'
-        gotindex = getattr(EffectInfo, 'OS_' + op1.args[0].value.upper())
+        gotindex = getattr(EffectInfo,
+                           'OS_' + op1.args[0].value.upper().lstrip('U'))
         assert gotindex == oopspecindex
         assert op1.args[1] == 'calldescr-%d' % oopspecindex
         assert list(op1.args[2]) == [v for v in vlist
@@ -192,8 +193,12 @@
                       [lltype.SignedLongLong], lltype.Signed)
         self.do_check('cast_float_to_longlong', EffectInfo.OS_LLONG_FROM_FLOAT,
                       [lltype.Float], lltype.SignedLongLong)
+        self.do_check('cast_float_to_ulonglong', EffectInfo.OS_LLONG_FROM_FLOAT,
+                      [lltype.Float], lltype.UnsignedLongLong)
         self.do_check('cast_longlong_to_float', EffectInfo.OS_LLONG_TO_FLOAT,
                       [lltype.SignedLongLong], lltype.Float)
+        self.do_check('cast_ulonglong_to_float', EffectInfo.OS_LLONG_U_TO_FLOAT,
+                      [lltype.UnsignedLongLong], lltype.Float)
         for T1 in [lltype.SignedLongLong, lltype.UnsignedLongLong]:
             for T2 in [lltype.Signed, lltype.Unsigned]:
                 self.do_check('cast_primitive', EffectInfo.OS_LLONG_TO_INT,
diff --git a/pypy/jit/metainterp/blackhole.py b/pypy/jit/metainterp/blackhole.py
--- a/pypy/jit/metainterp/blackhole.py
+++ b/pypy/jit/metainterp/blackhole.py
@@ -835,6 +835,18 @@
     def bhimpl_current_trace_length():
         return -1
 
+    @arguments("i", returns="i")
+    def bhimpl_int_isconstant(x):
+        return False
+
+    @arguments("r", returns="i")
+    def bhimpl_ref_isconstant(x):
+        return False
+
+    @arguments("r", returns="i")
+    def bhimpl_ref_isvirtual(x):
+        return False
+
     # ----------
     # the main hints and recursive calls
 
@@ -1224,6 +1236,9 @@
     @arguments("cpu", "r", "i", "i")
     def bhimpl_strsetitem(cpu, string, index, newchr):
         cpu.bh_strsetitem(string, index, newchr)
+    @arguments("cpu", "r", "r", "i", "i", "i")
+    def bhimpl_copystrcontent(cpu, src, dst, srcstart, dststart, length):
+        cpu.bh_copystrcontent(src, dst, srcstart, dststart, length)
 
     @arguments("cpu", "i", returns="r")
     def bhimpl_newunicode(cpu, length):
@@ -1237,6 +1252,9 @@
     @arguments("cpu", "r", "i", "i")
     def bhimpl_unicodesetitem(cpu, unicode, index, newchr):
         cpu.bh_unicodesetitem(unicode, index, newchr)
+    @arguments("cpu", "r", "r", "i", "i", "i")
+    def bhimpl_copyunicodecontent(cpu, src, dst, srcstart, dststart, length):
+        cpu.bh_copyunicodecontent(src, dst, srcstart, dststart, length)
 
     @arguments(returns=(longlong.is_64_bit and "i" or "f"))
     def bhimpl_ll_read_timestamp():
diff --git a/pypy/jit/metainterp/heapcache.py b/pypy/jit/metainterp/heapcache.py
--- a/pypy/jit/metainterp/heapcache.py
+++ b/pypy/jit/metainterp/heapcache.py
@@ -11,9 +11,13 @@
         self.known_class_boxes = {}
         # store the boxes that contain newly allocated objects, this maps the
         # boxes to a bool, the bool indicates whether or not the object has
-        # escaped the trace or not, its presences in the mapping shows that it
-        # was allocated inside the trace
+        # escaped the trace or not (True means the box never escaped, False
+        # means it did escape), its presences in the mapping shows that it was
+        # allocated inside the trace
         self.new_boxes = {}
+        # Tracks which boxes should be marked as escaped when the key box
+        # escapes.
+        self.dependencies = {}
         # contains frame boxes that are not virtualizables
         self.nonstandard_virtualizables = {}
         # heap cache
@@ -30,11 +34,29 @@
         self.clear_caches(opnum, descr, argboxes)
 
     def mark_escaped(self, opnum, argboxes):
-        for idx, box in enumerate(argboxes):
-            # setfield_gc and setarrayitem_gc don't escape their first argument
-            if not (idx == 0 and opnum in [rop.SETFIELD_GC, rop.SETARRAYITEM_GC]):
-                if box in self.new_boxes:
-                    self.new_boxes[box] = False
+        idx = 0
+        if opnum == rop.SETFIELD_GC:
+            assert len(argboxes) == 2
+            box, valuebox = argboxes
+            if self.is_unescaped(box) and self.is_unescaped(valuebox):
+                self.dependencies.setdefault(box, []).append(valuebox)
+            else:
+                self._escape(valuebox)
+        # GETFIELD_GC doesn't escape it's argument
+        elif opnum != rop.GETFIELD_GC:
+            for box in argboxes:
+                # setarrayitem_gc don't escape its first argument
+                if not (idx == 0 and opnum in [rop.SETARRAYITEM_GC]):
+                    self._escape(box)
+                idx += 1
+
+    def _escape(self, box):
+        if box in self.new_boxes:
+            self.new_boxes[box] = False
+        if box in self.dependencies:
+            for dep in self.dependencies[box]:
+                self._escape(dep)
+            del self.dependencies[box]
 
     def clear_caches(self, opnum, descr, argboxes):
         if opnum == rop.SETFIELD_GC:
diff --git a/pypy/jit/metainterp/optimizeopt/__init__.py b/pypy/jit/metainterp/optimizeopt/__init__.py
--- a/pypy/jit/metainterp/optimizeopt/__init__.py
+++ b/pypy/jit/metainterp/optimizeopt/__init__.py
@@ -7,6 +7,8 @@
 from pypy.jit.metainterp.optimizeopt.unroll import optimize_unroll, OptInlineShortPreamble
 from pypy.jit.metainterp.optimizeopt.fficall import OptFfiCall
 from pypy.jit.metainterp.optimizeopt.simplify import OptSimplify
+from pypy.jit.metainterp.optimizeopt.pure import OptPure
+from pypy.jit.metainterp.optimizeopt.earlyforce import OptEarlyForce
 from pypy.rlib.jit import PARAMETERS
 from pypy.rlib.unroll import unrolling_iterable
 
@@ -14,6 +16,8 @@
             ('rewrite', OptRewrite),
             ('virtualize', OptVirtualize),
             ('string', OptString),
+            ('earlyforce', OptEarlyForce),
+            ('pure', OptPure),
             ('heap', OptHeap),
             ('ffi', None),
             ('unroll', None)]
diff --git a/pypy/jit/metainterp/optimizeopt/earlyforce.py b/pypy/jit/metainterp/optimizeopt/earlyforce.py
new file mode 100644
--- /dev/null
+++ b/pypy/jit/metainterp/optimizeopt/earlyforce.py
@@ -0,0 +1,24 @@
+from pypy.jit.metainterp.optimizeopt.optimizer import Optimization
+from pypy.jit.metainterp.optimizeopt.vstring import VAbstractStringValue
+from pypy.jit.metainterp.resoperation import rop, ResOperation
+
+class OptEarlyForce(Optimization):
+    def propagate_forward(self, op):
+        opnum = op.getopnum()
+        if (opnum != rop.SETFIELD_GC and 
+            opnum != rop.SETARRAYITEM_GC and
+            opnum != rop.QUASIIMMUT_FIELD):
+               
+            for arg in op.getarglist():
+                if arg in self.optimizer.values:
+                    value = self.getvalue(arg)
+                    value.force_box(self)
+        self.emit_operation(op)
+
+    def new(self):
+        return OptEarlyForce()
+
+    def setup(self):
+        self.optimizer.optearlyforce = self
+
+    
diff --git a/pypy/jit/metainterp/optimizeopt/fficall.py b/pypy/jit/metainterp/optimizeopt/fficall.py
--- a/pypy/jit/metainterp/optimizeopt/fficall.py
+++ b/pypy/jit/metainterp/optimizeopt/fficall.py
@@ -177,10 +177,10 @@
             funcinfo.descr is None):
             return [op] # cannot optimize
         funcsymval = self.getvalue(op.getarg(2))
-        arglist = [funcsymval.force_box()]
+        arglist = [funcsymval.get_key_box()]
         for push_op in funcinfo.opargs:
             argval = self.getvalue(push_op.getarg(2))
-            arglist.append(argval.force_box())
+            arglist.append(argval.get_key_box())
         newop = ResOperation(rop.CALL_RELEASE_GIL, arglist, op.result,
                              descr=funcinfo.descr)
         self.commit_optimization()
diff --git a/pypy/jit/metainterp/optimizeopt/heap.py b/pypy/jit/metainterp/optimizeopt/heap.py
--- a/pypy/jit/metainterp/optimizeopt/heap.py
+++ b/pypy/jit/metainterp/optimizeopt/heap.py
@@ -50,6 +50,7 @@
             if not self._lazy_setfield_registered:
                 optheap._lazy_setfields_and_arrayitems.append(self)
                 self._lazy_setfield_registered = True
+            
         else:
             # this is the case where the pending setfield ends up
             # storing precisely the value that is already there,
@@ -158,12 +159,17 @@
         self._lazy_setfields_and_arrayitems = []
         self._remove_guard_not_invalidated = False
         self._seen_guard_not_invalidated = False
+        self.posponedop = None
 
     def force_at_end_of_preamble(self):
         self.force_all_lazy_setfields_and_arrayitems()
 
     def flush(self):
         self.force_all_lazy_setfields_and_arrayitems()
+        if self.posponedop:
+            posponedop = self.posponedop
+            self.posponedop = None
+            self.next_optimization.propagate_forward(posponedop)
 
     def new(self):
         return OptHeap()
@@ -211,7 +217,15 @@
 
     def emit_operation(self, op):
         self.emitting_operation(op)
-        self.next_optimization.propagate_forward(op)
+        if self.posponedop:
+            posponedop = self.posponedop
+            self.posponedop = None
+            self.next_optimization.propagate_forward(posponedop)
+        if (op.is_comparison() or op.getopnum() == rop.CALL_MAY_FORCE
+            or op.is_ovf()):
+            self.posponedop = op
+        else:
+            self.next_optimization.propagate_forward(op)
 
     def emitting_operation(self, op):
         if op.has_no_side_effect():
@@ -293,30 +307,6 @@
             if indexvalue is None or indexvalue.intbound.contains(idx):
                 cf.force_lazy_setfield(self, can_cache)
 
-    def fixup_guard_situation(self):
-        # hackish: reverse the order of the last two operations if it makes
-        # sense to avoid a situation like "int_eq/setfield_gc/guard_true",
-        # which the backend (at least the x86 backend) does not handle well.
-        newoperations = self.optimizer.newoperations
-        if len(newoperations) < 2:
-            return
-        lastop = newoperations[-1]
-        if (lastop.getopnum() != rop.SETFIELD_GC and
-            lastop.getopnum() != rop.SETARRAYITEM_GC):
-            return
-        # - is_comparison() for cases like "int_eq/setfield_gc/guard_true"
-        # - CALL_MAY_FORCE: "call_may_force/setfield_gc/guard_not_forced"
-        # - is_ovf(): "int_add_ovf/setfield_gc/guard_no_overflow"
-        prevop = newoperations[-2]
-        opnum = prevop.getopnum()
-        if not (prevop.is_comparison() or opnum == rop.CALL_MAY_FORCE
-                or prevop.is_ovf()):
-            return
-        if prevop.result in lastop.getarglist():
-            return
-        newoperations[-2] = lastop
-        newoperations[-1] = prevop
-
     def _assert_valid_cf(self, cf):
         # check that 'cf' is in cached_fields or cached_arrayitems
         if not we_are_translated():
@@ -362,7 +352,6 @@
                                       fieldvalue.get_key_box(), itemindex))
             else:
                 cf.force_lazy_setfield(self)
-                self.fixup_guard_situation()
         return pendingfields
 
     def optimize_GETFIELD_GC(self, op):
@@ -374,12 +363,22 @@
             return
         # default case: produce the operation
         structvalue.ensure_nonnull()
-        ###self.optimizer.optimize_default(op)
         self.emit_operation(op)
         # then remember the result of reading the field
         fieldvalue = self.getvalue(op.result)
         cf.remember_field_value(structvalue, fieldvalue, op)
 
+    def optimize_GETFIELD_GC_PURE(self, op):
+        structvalue = self.getvalue(op.getarg(0))
+        cf = self.field_cache(op.getdescr())
+        fieldvalue = cf.getfield_from_cache(self, structvalue)
+        if fieldvalue is not None:
+            self.make_equal_to(op.result, fieldvalue)
+            return
+        # default case: produce the operation
+        structvalue.ensure_nonnull()
+        self.emit_operation(op)
+
     def optimize_SETFIELD_GC(self, op):
         if self.has_pure_result(rop.GETFIELD_GC_PURE, [op.getarg(0)],
                                 op.getdescr()):
@@ -389,6 +388,7 @@
         #
         cf = self.field_cache(op.getdescr())
         cf.do_setfield(self, op)
+        
 
     def optimize_GETARRAYITEM_GC(self, op):
         arrayvalue = self.getvalue(op.getarg(0))
@@ -413,6 +413,25 @@
             fieldvalue = self.getvalue(op.result)
             cf.remember_field_value(arrayvalue, fieldvalue, op)
 
+    def optimize_GETARRAYITEM_GC_PURE(self, op):
+        arrayvalue = self.getvalue(op.getarg(0))
+        indexvalue = self.getvalue(op.getarg(1))
+        cf = None
+        if indexvalue.is_constant():
+            arrayvalue.make_len_gt(MODE_ARRAY, op.getdescr(), indexvalue.box.getint())
+            # use the cache on (arraydescr, index), which is a constant
+            cf = self.arrayitem_cache(op.getdescr(), indexvalue.box.getint())
+            fieldvalue = cf.getfield_from_cache(self, arrayvalue)
+            if fieldvalue is not None:
+                self.make_equal_to(op.result, fieldvalue)
+                return
+        else:
+            # variable index, so make sure the lazy setarrayitems are done
+            self.force_lazy_setarrayitem(op.getdescr(), indexvalue=indexvalue)
+        # default case: produce the operation
+        arrayvalue.ensure_nonnull()
+        self.emit_operation(op)
+
     def optimize_SETARRAYITEM_GC(self, op):
         if self.has_pure_result(rop.GETARRAYITEM_GC_PURE, [op.getarg(0),
                                                            op.getarg(1)],
diff --git a/pypy/jit/metainterp/optimizeopt/optimizer.py b/pypy/jit/metainterp/optimizeopt/optimizer.py
--- a/pypy/jit/metainterp/optimizeopt/optimizer.py
+++ b/pypy/jit/metainterp/optimizeopt/optimizer.py
@@ -31,8 +31,8 @@
 
 class OptValue(object):
     __metaclass__ = extendabletype
-    _attrs_ = ('box', 'known_class', 'last_guard_index', 'level', 'intbound', 'lenbound')
-    last_guard_index = -1
+    _attrs_ = ('box', 'known_class', 'last_guard', 'level', 'intbound', 'lenbound')
+    last_guard = None
 
     level = LEVEL_UNKNOWN
     known_class = None
@@ -71,7 +71,7 @@
             guards.append(op)
         elif self.level == LEVEL_KNOWNCLASS:
             op = ResOperation(rop.GUARD_NONNULL, [box], None)
-            guards.append(op)            
+            guards.append(op)
             op = ResOperation(rop.GUARD_CLASS, [box, self.known_class], None)
             guards.append(op)
         else:
@@ -100,7 +100,7 @@
             self.make_constant(other.get_key_box())
             optimizer.turned_constant(self)
         elif other.level == LEVEL_KNOWNCLASS:
-            self.make_constant_class(other.known_class, -1)
+            self.make_constant_class(other.known_class, None)
         else:
             if other.level == LEVEL_NONNULL:
                 self.ensure_nonnull()
@@ -112,15 +112,15 @@
                     self.lenbound.bound.intersect(other.lenbound.bound)
                 else:
                     self.lenbound = other.lenbound.clone()
-                    
 
-    def force_box(self):
+
+    def force_box(self, optforce):
         return self.box
 
     def get_key_box(self):
         return self.box
 
-    def force_at_end_of_preamble(self, already_forced):
+    def force_at_end_of_preamble(self, already_forced, optforce):
         return self
 
     def get_args_for_fail(self, modifier):
@@ -146,7 +146,7 @@
         assert isinstance(constbox, Const)
         self.box = constbox
         self.level = LEVEL_CONSTANT
-        
+
         if isinstance(constbox, ConstInt):
             val = constbox.getint()
             self.intbound = IntBound(val, val)
@@ -162,16 +162,16 @@
         else:
             return None
 
-    def make_constant_class(self, classbox, opindex):
+    def make_constant_class(self, classbox, guardop):
         assert self.level < LEVEL_KNOWNCLASS
         self.known_class = classbox
         self.level = LEVEL_KNOWNCLASS
-        self.last_guard_index = opindex
+        self.last_guard = guardop
 
-    def make_nonnull(self, opindex):
+    def make_nonnull(self, guardop):
         assert self.level < LEVEL_NONNULL
         self.level = LEVEL_NONNULL
-        self.last_guard_index = opindex
+        self.last_guard = guardop
 
     def is_nonnull(self):
         level = self.level
@@ -223,6 +223,9 @@
     def __init__(self, box):
         self.make_constant(box)
 
+    def __repr__(self):
+        return 'Constant(%r)' % (self.box,)
+
 CONST_0      = ConstInt(0)
 CONST_1      = ConstInt(1)
 CVAL_ZERO    = ConstantValue(CONST_0)
@@ -237,26 +240,12 @@
     def __init__(self):
         pass # make rpython happy
 
-    def propagate_begin_forward(self):
-        if self.next_optimization:
-            self.next_optimization.propagate_begin_forward()
-
-    def propagate_end_forward(self):
-        if self.next_optimization:
-            self.next_optimization.propagate_end_forward()
-
     def propagate_forward(self, op):
         raise NotImplementedError
 
     def emit_operation(self, op):
         self.next_optimization.propagate_forward(op)
 
-    def test_emittable(self, op):
-        return self.is_emittable(op)
-
-    def is_emittable(self, op):
-        return self.next_optimization.test_emittable(op)
-
     # FIXME: Move some of these here?
     def getvalue(self, box):
         return self.optimizer.getvalue(box)
@@ -286,19 +275,19 @@
         return self.optimizer.new_const_item(arraydescr)
 
     def pure(self, opnum, args, result):
-        op = ResOperation(opnum, args, result)
-        key = self.optimizer.make_args_key(op)
-        if key not in self.optimizer.pure_operations:
-            self.optimizer.pure_operations[key] = op
+        if self.optimizer.optpure:
+            self.optimizer.optpure.pure(opnum, args, result)
 
     def has_pure_result(self, opnum, args, descr):
-        op = ResOperation(opnum, args, None, descr)
-        key = self.optimizer.make_args_key(op)
-        op = self.optimizer.pure_operations.get(key, None)
-        if op is None:
-            return False
-        return op.getdescr() is descr
+        if self.optimizer.optpure:
+            return self.optimizer.optpure.has_pure_result(opnum, args, descr)
+        return False
 
+    def get_pure_result(self, key):    
+        if self.optimizer.optpure:
+            return self.optimizer.optpure.get_pure_result(key)
+        return None
+    
     def setup(self):
         pass
 
@@ -319,6 +308,9 @@
     def produce_potential_short_preamble_ops(self, potential_ops):
         pass
 
+    def forget_numberings(self, box):
+        self.optimizer.forget_numberings(box)
+
 class Optimizer(Optimization):
 
     def __init__(self, metainterp_sd, loop, optimizations=None, bridge=False):
@@ -330,14 +322,16 @@
         self.interned_refs = self.cpu.ts.new_ref_dict()
         self.resumedata_memo = resume.ResumeDataLoopMemo(metainterp_sd)
         self.bool_boxes = {}
-        self.pure_operations = args_dict()
         self.producer = {}
         self.pendingfields = []
-        self.posponedop = None
         self.exception_might_have_happened = False
         self.quasi_immutable_deps = None
         self.opaque_pointers = {}
-        self.newoperations = []
+        self.replaces_guard = {}
+        self._newoperations = []
+        self.optimizer = self
+        self.optpure = None
+        self.optearlyforce = None
         if loop is not None:
             self.call_pure_results = loop.call_pure_results
 
@@ -366,21 +360,20 @@
     def flush(self):
         for o in self.optimizations:
             o.flush()
-        assert self.posponedop is None
 
     def new(self):
         new = Optimizer(self.metainterp_sd, self.loop)
         return self._new(new)
 
     def _new(self, new):
-        assert self.posponedop is None
         optimizations = [o.new() for o in self.optimizations]
         new.set_optimizations(optimizations)
         new.quasi_immutable_deps = self.quasi_immutable_deps
         return new
-        
+
     def produce_potential_short_preamble_ops(self, sb):
-        raise NotImplementedError('This is implemented in unroll.UnrollableOptimizer')
+        for opt in self.optimizations:
+            opt.produce_potential_short_preamble_ops(sb)
 
     def turned_constant(self, value):
         for o in self.optimizations:
@@ -430,6 +423,13 @@
             return constbox
         return None
 
+    def get_newoperations(self):
+        self.flush()
+        return self._newoperations
+
+    def clear_newoperations(self):
+        self._newoperations = []
+
     def make_equal_to(self, box, value, replace=False):
         assert isinstance(value, OptValue)
         assert replace or box not in self.values
@@ -478,15 +478,10 @@
 
     def propagate_all_forward(self):
         self.exception_might_have_happened = self.bridge
-        self.newoperations = []
-        self.first_optimization.propagate_begin_forward()
-        self.i = 0
-        while self.i < len(self.loop.operations):
-            op = self.loop.operations[self.i]
+        self.clear_newoperations()
+        for op in self.loop.operations:
             self.first_optimization.propagate_forward(op)
-            self.i += 1
-        self.first_optimization.propagate_end_forward()
-        self.loop.operations = self.newoperations
+        self.loop.operations = self.get_newoperations()
         self.loop.quasi_immutable_deps = self.quasi_immutable_deps
         # accumulate counters
         self.resumedata_memo.update_counters(self.metainterp_sd.profiler)
@@ -498,16 +493,13 @@
         self.producer[op.result] = op
         dispatch_opt(self, op)
 
-    def test_emittable(self, op):
-        return True
-
     def emit_operation(self, op):
         if op.returns_bool_result():
             self.bool_boxes[self.getvalue(op.result)] = None
         self._emit_operation(op)
-        
+
     @specialize.argtype(0)
-    def _emit_operation(self, op):        
+    def _emit_operation(self, op):
         for i in range(op.numargs()):
             arg = op.getarg(i)
             try:
@@ -516,14 +508,30 @@
                 pass
             else:
                 self.ensure_imported(value)
-                op.setarg(i, value.force_box())
+                op.setarg(i, value.force_box(self))
         self.metainterp_sd.profiler.count(jitprof.OPT_OPS)
         if op.is_guard():
             self.metainterp_sd.profiler.count(jitprof.OPT_GUARDS)
-            op = self.store_final_boxes_in_guard(op)
+            if self.replaces_guard and op in self.replaces_guard:
+                self.replace_op(self.replaces_guard[op], op)
+                del self.replaces_guard[op]
+                return
+            else:
+                op = self.store_final_boxes_in_guard(op)
         elif op.can_raise():
             self.exception_might_have_happened = True
-        self.newoperations.append(op)
+        self._newoperations.append(op)
+
+    def replace_op(self, old_op, new_op):
+        # XXX: Do we want to cache indexes to prevent search?
+        i = len(self._newoperations) 
+        while i > 0:
+            i -= 1
+            if self._newoperations[i] is old_op:
+                self._newoperations[i] = new_op
+                break
+        else:
+            assert False
 
     def store_final_boxes_in_guard(self, op):
         descr = op.getdescr()
@@ -568,55 +576,12 @@
                 arg = value.get_key_box()
             args[i] = arg
         args[n] = ConstInt(op.getopnum())
-        args[n+1] = op.getdescr()
+        args[n + 1] = op.getdescr()
         return args
 
-    @specialize.argtype(0)
     def optimize_default(self, op):
-        canfold = op.is_always_pure()
-        if op.is_ovf():
-            self.posponedop = op
-            return
-        if self.posponedop:
-            nextop = op
-            op = self.posponedop
-            self.posponedop = None
-            canfold = nextop.getopnum() == rop.GUARD_NO_OVERFLOW
-        else:
-            nextop = None
+        self.emit_operation(op)
 
-        if canfold:
-            for i in range(op.numargs()):
-                if self.get_constant_box(op.getarg(i)) is None:
-                    break
-            else:
-                # all constant arguments: constant-fold away
-                resbox = self.constant_fold(op)
-                # note that INT_xxx_OVF is not done from here, and the
-                # overflows in the INT_xxx operations are ignored
-                self.make_constant(op.result, resbox)
-                return
-
-            # did we do the exact same operation already?
-            args = self.make_args_key(op)
-            oldop = self.pure_operations.get(args, None)
-            if oldop is not None and oldop.getdescr() is op.getdescr():
-                assert oldop.getopnum() == op.getopnum()
-                self.make_equal_to(op.result, self.getvalue(oldop.result),
-                                   True)
-                return
-            else:
-                self.pure_operations[args] = op
-                self.remember_emitting_pure(op)
-
-        # otherwise, the operation remains
-        self.emit_operation(op)
-        if nextop:
-            self.emit_operation(nextop)
-
-    def remember_emitting_pure(self, op):
-        pass
-    
     def constant_fold(self, op):
         argboxes = [self.get_constant_box(op.getarg(i))
                     for i in range(op.numargs())]
@@ -633,11 +598,6 @@
     def optimize_DEBUG_MERGE_POINT(self, op):
         self.emit_operation(op)
 
-    def optimize_CAST_OPAQUE_PTR(self, op):
-        value = self.getvalue(op.getarg(0))
-        self.opaque_pointers[value] = True
-        self.make_equal_to(op.result, value)
-
     def optimize_GETARRAYITEM_GC_PURE(self, op):
         indexvalue = self.getvalue(op.getarg(1))
         if indexvalue.is_constant():
@@ -658,9 +618,9 @@
             arrayvalue = self.getvalue(op.getarg(0))
             arrayvalue.make_len_gt(MODE_UNICODE, op.getdescr(), indexvalue.box.getint())
         self.optimize_default(op)
-        
 
-    
+
+
 
 dispatch_opt = make_dispatcher_method(Optimizer, 'optimize_',
         default=Optimizer.optimize_default)
diff --git a/pypy/jit/metainterp/optimizeopt/pure.py b/pypy/jit/metainterp/optimizeopt/pure.py
new file mode 100644
--- /dev/null
+++ b/pypy/jit/metainterp/optimizeopt/pure.py
@@ -0,0 +1,115 @@
+from pypy.jit.metainterp.optimizeopt.optimizer import Optimization
+from pypy.jit.metainterp.resoperation import rop, ResOperation
+from pypy.jit.metainterp.optimizeopt.util import (make_dispatcher_method,
+    args_dict)
+
+class OptPure(Optimization):
+    def __init__(self):
+        self.posponedop = None
+        self.pure_operations = args_dict()
+        self.emitted_pure_operations = {}
+
+    def propagate_forward(self, op):
+        dispatch_opt(self, op)
+
+    def optimize_default(self, op):
+        canfold = op.is_always_pure()
+        if op.is_ovf():
+            self.posponedop = op
+            return
+        if self.posponedop:
+            nextop = op
+            op = self.posponedop
+            self.posponedop = None
+            canfold = nextop.getopnum() == rop.GUARD_NO_OVERFLOW
+        else:
+            nextop = None
+
+        if canfold:
+            for i in range(op.numargs()):
+                if self.get_constant_box(op.getarg(i)) is None:
+                    break
+            else:
+                # all constant arguments: constant-fold away
+                resbox = self.optimizer.constant_fold(op)
+                # note that INT_xxx_OVF is not done from here, and the
+                # overflows in the INT_xxx operations are ignored
+                self.optimizer.make_constant(op.result, resbox)
+                return
+
+            # did we do the exact same operation already?
+            args = self.optimizer.make_args_key(op)
+            oldop = self.pure_operations.get(args, None)
+            if oldop is not None and oldop.getdescr() is op.getdescr():
+                assert oldop.getopnum() == op.getopnum()
+                self.optimizer.make_equal_to(op.result, self.getvalue(oldop.result),
+                                   True)
+                return
+            else:
+                self.pure_operations[args] = op
+                self.remember_emitting_pure(op)
+
+        # otherwise, the operation remains
+        self.emit_operation(op)
+        if op.returns_bool_result():
+            self.optimizer.bool_boxes[self.getvalue(op.result)] = None        
+        if nextop:
+            self.emit_operation(nextop)
+
+    def optimize_CALL_PURE(self, op):
+        args = self.optimizer.make_args_key(op)
+        oldop = self.pure_operations.get(args, None)
+        if oldop is not None and oldop.getdescr() is op.getdescr():
+            assert oldop.getopnum() == op.getopnum()
+            self.make_equal_to(op.result, self.getvalue(oldop.result))
+            return
+        else:
+            self.pure_operations[args] = op
+            self.remember_emitting_pure(op)
+
+        # replace CALL_PURE with just CALL
+        args = op.getarglist()
+        self.emit_operation(ResOperation(rop.CALL, args, op.result,
+                                         op.getdescr()))
+
+    def flush(self):
+        assert self.posponedop is None
+
+    def new(self):
+        assert self.posponedop is None
+        return OptPure()
+
+    def setup(self):
+        self.optimizer.optpure = self
+
+    def pure(self, opnum, args, result):
+        op = ResOperation(opnum, args, result)
+        key = self.optimizer.make_args_key(op)
+        if key not in self.pure_operations:
+            self.pure_operations[key] = op
+
+    def has_pure_result(self, opnum, args, descr):
+        op = ResOperation(opnum, args, None, descr)
+        key = self.optimizer.make_args_key(op)
+        op = self.pure_operations.get(key, None)
+        if op is None:
+            return False
+        return op.getdescr() is descr
+
+    def get_pure_result(self, key):
+        return self.pure_operations.get(key, None)
+
+    def remember_emitting_pure(self, op):
+        self.emitted_pure_operations[op] = True
+
+    def produce_potential_short_preamble_ops(self, sb):
+        for op in self.emitted_pure_operations:
+            if op.getopnum() == rop.GETARRAYITEM_GC_PURE or \
+               op.getopnum() == rop.STRGETITEM or \
+               op.getopnum() == rop.UNICODEGETITEM:
+                if not self.getvalue(op.getarg(1)).is_constant():
+                    continue
+            sb.add_potential(op)
+
+dispatch_opt = make_dispatcher_method(OptPure, 'optimize_',
+                                      default=OptPure.optimize_default)
diff --git a/pypy/jit/metainterp/optimizeopt/rewrite.py b/pypy/jit/metainterp/optimizeopt/rewrite.py
--- a/pypy/jit/metainterp/optimizeopt/rewrite.py
+++ b/pypy/jit/metainterp/optimizeopt/rewrite.py
@@ -19,7 +19,7 @@
 
     def new(self):
         return OptRewrite()
-        
+
     def produce_potential_short_preamble_ops(self, sb):
         for op in self.loop_invariant_producer.values():
             sb.add_potential(op)
@@ -31,21 +31,8 @@
 
         dispatch_opt(self, op)
 
-    def test_emittable(self, op):
-        opnum = op.getopnum()
-        for value, cls, func in optimize_guards:
-            if opnum == value:
-                assert isinstance(op, cls)
-                try:
-                    func(self, op, dryrun=True)
-                    return self.is_emittable(op)
-                except InvalidLoop:
-                    return False
-        return self.is_emittable(op)
-
-
     def try_boolinvers(self, op, targs):
-        oldop = self.optimizer.pure_operations.get(targs, None)
+        oldop = self.get_pure_result(targs)
         if oldop is not None and oldop.getdescr() is op.getdescr():
             value = self.getvalue(oldop.result)
             if value.is_constant():
@@ -62,32 +49,35 @@
     def find_rewritable_bool(self, op, args):
         try:
             oldopnum = opboolinvers[op.getopnum()]
+        except KeyError:
+            pass
+        else:
             targs = self.optimizer.make_args_key(ResOperation(oldopnum, [args[0], args[1]],
                                                               None))
             if self.try_boolinvers(op, targs):
                 return True
-        except KeyError:
-            pass
 
         try:
             oldopnum = opboolreflex[op.getopnum()] # FIXME: add INT_ADD, INT_MUL
+        except KeyError:
+            pass
+        else:
             targs = self.optimizer.make_args_key(ResOperation(oldopnum, [args[1], args[0]],
                                                               None))
-            oldop = self.optimizer.pure_operations.get(targs, None)
+            oldop = self.get_pure_result(targs)
             if oldop is not None and oldop.getdescr() is op.getdescr():
                 self.make_equal_to(op.result, self.getvalue(oldop.result))
                 return True
-        except KeyError:
-            pass
 
         try:
             oldopnum = opboolinvers[opboolreflex[op.getopnum()]]
+        except KeyError:
+            pass
+        else:
             targs = self.optimizer.make_args_key(ResOperation(oldopnum, [args[1], args[0]],
                                                               None))
             if self.try_boolinvers(op, targs):
                 return True
-        except KeyError:
-            pass
 
         return False
 
@@ -214,29 +204,7 @@
         self.emit_operation(op)
         self.pure(rop.FLOAT_NEG, [op.result], v1)
 
-    def optimize_CALL_PURE(self, op):
-        arg_consts = []
-        for i in range(op.numargs()):
-            arg = op.getarg(i)
-            const = self.get_constant_box(arg)
-            if const is None:
-                break
-            arg_consts.append(const)
-        else:
-            # all constant arguments: check if we already know the result
-            try:
-                result = self.optimizer.call_pure_results[arg_consts]
-            except KeyError:
-                pass
-            else:
-                self.make_constant(op.result, result)
-                return
-        # replace CALL_PURE with just CALL
-        args = op.getarglist()
-        self.emit_operation(ResOperation(rop.CALL, args, op.result,
-                                         op.getdescr()))
-
-    def optimize_guard(self, op, constbox, emit_operation=True, dryrun=False):
+    def optimize_guard(self, op, constbox, emit_operation=True):
         value = self.getvalue(op.getarg(0))
         if value.is_constant():
             box = value.box
@@ -244,62 +212,57 @@
             if not box.same_constant(constbox):
                 raise InvalidLoop
             return
-        if dryrun: return
         if emit_operation:
             self.emit_operation(op)
         value.make_constant(constbox)
         self.optimizer.turned_constant(value)
 
-    def optimize_GUARD_ISNULL(self, op, dryrun=False):
+    def optimize_GUARD_ISNULL(self, op):
         value = self.getvalue(op.getarg(0))
         if value.is_null():
             return
         elif value.is_nonnull():
             raise InvalidLoop
-        if dryrun: return
         self.emit_operation(op)
         value.make_constant(self.optimizer.cpu.ts.CONST_NULL)
 
-    def optimize_GUARD_NONNULL(self, op, dryrun=False):
+    def optimize_GUARD_NONNULL(self, op):
         value = self.getvalue(op.getarg(0))
         if value.is_nonnull():
             return
         elif value.is_null():
             raise InvalidLoop
-        if dryrun: return
         self.emit_operation(op)
-        value.make_nonnull(len(self.optimizer.newoperations) - 1)
+        value.make_nonnull(op)
 
-    def optimize_GUARD_VALUE(self, op, dryrun=False):
+    def optimize_GUARD_VALUE(self, op):
         value = self.getvalue(op.getarg(0))
-        emit_operation = True
-        if not dryrun and value.last_guard_index != -1:
+        if value.last_guard:
             # there already has been a guard_nonnull or guard_class or
             # guard_nonnull_class on this value, which is rather silly.
             # replace the original guard with a guard_value
-            old_guard_op = self.optimizer.newoperations[value.last_guard_index]
-            new_guard_op = old_guard_op.copy_and_change(rop.GUARD_VALUE,
-                                             args = [old_guard_op.getarg(0), op.getarg(1)])
-            self.optimizer.newoperations[value.last_guard_index] = new_guard_op
+            old_guard_op = value.last_guard
+            op = old_guard_op.copy_and_change(rop.GUARD_VALUE,
+                                      args = [old_guard_op.getarg(0), op.getarg(1)])
+            self.optimizer.replaces_guard[op] = old_guard_op
             # hack hack hack.  Change the guard_opnum on
             # new_guard_op.getdescr() so that when resuming,
             # the operation is not skipped by pyjitpl.py.
-            descr = new_guard_op.getdescr()
+            descr = op.getdescr()
             assert isinstance(descr, compile.ResumeGuardDescr)
             descr.guard_opnum = rop.GUARD_VALUE
-            descr.make_a_counter_per_value(new_guard_op)
-            emit_operation = False
+            descr.make_a_counter_per_value(op)
         constbox = op.getarg(1)
         assert isinstance(constbox, Const)
-        self.optimize_guard(op, constbox, emit_operation, dryrun)
+        self.optimize_guard(op, constbox)
 
-    def optimize_GUARD_TRUE(self, op, dryrun=False):
-        self.optimize_guard(op, CONST_1, dryrun=dryrun)
+    def optimize_GUARD_TRUE(self, op):
+        self.optimize_guard(op, CONST_1)
 
-    def optimize_GUARD_FALSE(self, op, dryrun=False):
-        self.optimize_guard(op, CONST_0, dryrun=dryrun)
+    def optimize_GUARD_FALSE(self, op):
+        self.optimize_guard(op, CONST_0)
 
-    def optimize_GUARD_CLASS(self, op, dryrun=False):
+    def optimize_GUARD_CLASS(self, op):
         value = self.getvalue(op.getarg(0))
         expectedclassbox = op.getarg(1)
         assert isinstance(expectedclassbox, Const)
@@ -308,38 +271,32 @@
             if realclassbox.same_constant(expectedclassbox):
                 return
             raise InvalidLoop
-        if dryrun: return
-        emit_operation = True
-        if value.last_guard_index != -1:
+        if value.last_guard:
             # there already has been a guard_nonnull or guard_class or
             # guard_nonnull_class on this value.
-            old_guard_op = self.optimizer.newoperations[value.last_guard_index]
+            old_guard_op = value.last_guard
             if old_guard_op.getopnum() == rop.GUARD_NONNULL:
                 # it was a guard_nonnull, which we replace with a
                 # guard_nonnull_class.
-                new_guard_op = old_guard_op.copy_and_change (rop.GUARD_NONNULL_CLASS,
+                op = old_guard_op.copy_and_change (rop.GUARD_NONNULL_CLASS,
                                          args = [old_guard_op.getarg(0), op.getarg(1)])
-                self.optimizer.newoperations[value.last_guard_index] = new_guard_op
+                self.optimizer.replaces_guard[op] = old_guard_op
                 # hack hack hack.  Change the guard_opnum on
                 # new_guard_op.getdescr() so that when resuming,
                 # the operation is not skipped by pyjitpl.py.
-                descr = new_guard_op.getdescr()
+                descr = op.getdescr()
                 assert isinstance(descr, compile.ResumeGuardDescr)
                 descr.guard_opnum = rop.GUARD_NONNULL_CLASS
-                emit_operation = False
-        if emit_operation:
-            self.emit_operation(op)
-            last_guard_index = len(self.optimizer.newoperations) - 1
-        else:
-            last_guard_index = value.last_guard_index
-        value.make_constant_class(expectedclassbox, last_guard_index)
+        self.emit_operation(op)
+        value.make_constant_class(expectedclassbox, op)
 
-    def optimize_GUARD_NONNULL_CLASS(self, op, dryrun=False):
-        self.optimize_GUARD_NONNULL(op, True)
-        self.optimize_GUARD_CLASS(op, dryrun)
+    def optimize_GUARD_NONNULL_CLASS(self, op):
+        value = self.getvalue(op.getarg(0))
+        if value.is_null():
+            raise InvalidLoop
+        self.optimize_GUARD_CLASS(op)
 
-    def optimize_GUARD_NO_EXCEPTION(self, op, dryrun=False):
-        if dryrun: return
+    def optimize_GUARD_NO_EXCEPTION(self, op):
         if not self.optimizer.exception_might_have_happened:
             return
         self.emit_operation(op)
@@ -351,7 +308,7 @@
         # expects a compile-time constant
         assert isinstance(arg, Const)
         key = make_hashable_int(arg.getint())
-        
+
         resvalue = self.loop_invariant_results.get(key, None)
         if resvalue is not None:
             self.make_equal_to(op.result, resvalue)
@@ -459,7 +416,7 @@
                     newop = ResOperation(rop.SETARRAYITEM_GC,
                                          [op.getarg(2),
                                           ConstInt(index + dest_start),
-                                          val.force_box()], None,
+                                          val.get_key_box()], None,
                                          descr=source_value.arraydescr)
                     self.emit_operation(newop)
             return True
@@ -467,6 +424,25 @@
             return True # 0-length arraycopy
         return False
 
+    def optimize_CALL_PURE(self, op):
+        arg_consts = []
+        for i in range(op.numargs()):
+            arg = op.getarg(i)
+            const = self.get_constant_box(arg)
+            if const is None:
+                break
+            arg_consts.append(const)
+        else:
+            # all constant arguments: check if we already know the result
+            try:
+                result = self.optimizer.call_pure_results[arg_consts]
+            except KeyError:
+                pass
+            else:
+                self.make_constant(op.result, result)
+                return
+        self.emit_operation(op)
+
     def optimize_INT_FLOORDIV(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
@@ -481,6 +457,10 @@
                                         args = [op.getarg(0), ConstInt(highest_bit(val))])
         self.emit_operation(op)
 
+    def optimize_CAST_OPAQUE_PTR(self, op):
+        value = self.getvalue(op.getarg(0))
+        self.optimizer.opaque_pointers[value] = True
+        self.make_equal_to(op.result, value)
 
 dispatch_opt = make_dispatcher_method(OptRewrite, 'optimize_',
         default=OptRewrite.emit_operation)
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -39,8 +39,9 @@
 
 def test_sharing_field_lists_of_virtual():
     class FakeOptimizer(object):
-        class cpu(object):
-            pass
+        class optimizer(object):
+            class cpu(object):
+                pass
     opt = FakeOptimizer()
     virt1 = virtualize.AbstractVirtualStructValue(opt, None)
     lst1 = virt1._get_field_descr_list()
@@ -69,7 +70,7 @@
     class FakeVirtualValue(virtualize.AbstractVirtualValue):
         def _make_virtual(self, *args):
             return FakeVInfo()
-    v1 = FakeVirtualValue(None, None, None)
+    v1 = FakeVirtualValue(None, None)
     vinfo1 = v1.make_virtual_info(None, [1, 2, 4])
     vinfo2 = v1.make_virtual_info(None, [1, 2, 4])
     assert vinfo1 is vinfo2
@@ -111,7 +112,7 @@
 
 class BaseTestBasic(BaseTest):
 
-    enable_opts = "intbounds:rewrite:virtualize:string:heap"
+    enable_opts = "intbounds:rewrite:virtualize:string:earlyforce:pure:heap"
 
     def optimize_loop(self, ops, optops, call_pure_results=None):
 
@@ -651,8 +652,8 @@
         i3 = getfield_gc(p3, descr=valuedescr)
         escape(i3)
         p1 = new_with_vtable(ConstClass(node_vtable))
+        p1sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p1, i1, descr=valuedescr)
-        p1sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p1sub, i1, descr=valuedescr)
         setfield_gc(p1, p1sub, descr=nextdescr)
         jump(i1, p1, p2)
@@ -667,10 +668,10 @@
         p3sub = getfield_gc(p3, descr=nextdescr)
         i3 = getfield_gc(p3sub, descr=valuedescr)
         escape(i3)
+        p1 = new_with_vtable(ConstClass(node_vtable))
         p2sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p2sub, i1, descr=valuedescr)
         setfield_gc(p2, p2sub, descr=nextdescr)
-        p1 = new_with_vtable(ConstClass(node_vtable))
         jump(i1, p1, p2)
         """
         # The same as test_p123_simple, but in the end the "old" p2 contains
@@ -4711,6 +4712,35 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_empty_copystrunicontent(self):
+        ops = """
+        [p0, p1, i0, i2, i3]
+        i4 = int_eq(i3, 0)
+        guard_true(i4) []
+        copystrcontent(p0, p1, i0, i2, i3)
+        jump(p0, p1, i0, i2, i3)
+        """
+        expected = """
+        [p0, p1, i0, i2, i3]
+        i4 = int_eq(i3, 0)
+        guard_true(i4) []
+        jump(p0, p1, i0, i2, 0)
+        """
+        self.optimize_strunicode_loop(ops, expected)
+
+    def test_empty_copystrunicontent_virtual(self):
+        ops = """
+        [p0]
+        p1 = newstr(23)
+        copystrcontent(p0, p1, 0, 0, 0)
+        jump(p0)
+        """
+        expected = """
+        [p0]
+        jump(p0)
+        """
+        self.optimize_strunicode_loop(ops, expected)
+
     def test_forced_virtuals_aliasing(self):
         ops = """
         [i0, i1]
@@ -4738,6 +4768,27 @@
         # other
         self.optimize_loop(ops, expected)
 
+    def test_plain_virtual_string_copy_content(self):
+        ops = """
+        []
+        p0 = newstr(6)
+        copystrcontent(s"hello!", p0, 0, 0, 6)
+        p1 = call(0, p0, s"abc123", descr=strconcatdescr)
+        i0 = strgetitem(p1, 0)
+        finish(i0)
+        """
+        expected = """
+        []
+        p0 = newstr(6)
+        copystrcontent(s"hello!", p0, 0, 0, 6)
+        p1 = newstr(12)
+        copystrcontent(p0, p1, 0, 0, 6)
+        copystrcontent(s"abc123", p1, 0, 6, 6)
+        i0 = strgetitem(p1, 0)
+        finish(i0)
+        """
+        self.optimize_strunicode_loop(ops, expected)
+
 
 class TestLLtype(BaseTestOptimizeBasic, LLtypeMixin):
     pass
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizefficall.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizefficall.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizefficall.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizefficall.py
@@ -36,7 +36,7 @@
 
 class TestFfiCall(BaseTestBasic, LLtypeMixin):
 
-    enable_opts = "intbounds:rewrite:virtualize:string:heap:ffi"
+    enable_opts = "intbounds:rewrite:virtualize:string:pure:earlyforce:heap:ffi"
 
     class namespace:
         cpu = LLtypeMixin.cpu
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -68,7 +68,7 @@
 
 class BaseTestWithUnroll(BaseTest):
 
-    enable_opts = "intbounds:rewrite:virtualize:string:heap:unroll"
+    enable_opts = "intbounds:rewrite:virtualize:string:earlyforce:pure:heap:unroll"
 
     def optimize_loop(self, ops, expected, expected_preamble=None,
                       call_pure_results=None, expected_short=None):
@@ -102,9 +102,9 @@
             print "Short Preamble:"
             short = loop.preamble.token.short_preamble[0]
             print short.inputargs
-            print '\n'.join([str(o) for o in short.operations])        
+            print '\n'.join([str(o) for o in short.operations])
             print
-        
+
         assert expected != "crash!", "should have raised an exception"
         self.assert_equal(loop, expected)
         if expected_preamble:
@@ -113,7 +113,7 @@
         if expected_short:
             self.assert_equal(short, expected_short,
                               text_right='expected short preamble')
-            
+
         return loop
 
 class OptimizeOptTest(BaseTestWithUnroll):
@@ -825,8 +825,8 @@
         i3 = getfield_gc(p2, descr=valuedescr)
         escape(i3)
         p4 = new_with_vtable(ConstClass(node_vtable))
+        p1sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p4, i1, descr=valuedescr)
-        p1sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p1sub, i1, descr=valuedescr)
         setfield_gc(p4, p1sub, descr=nextdescr)
         jump(i1, p4)
@@ -865,13 +865,7 @@
         p3sub = new_with_vtable(ConstClass(node_vtable2))
         setfield_gc(p3sub, i1, descr=valuedescr)
         setfield_gc(p1, p3sub, descr=nextdescr)
-        # XXX: We get two extra operations here because the setfield
-        #      above is the result of forcing p1 and thus not 
-        #      registered with the heap optimizer. I've makred tests
-        #      below with VIRTUALHEAP if they suffer from this issue
-        p3sub2 = getfield_gc(p1, descr=nextdescr) 
-        guard_nonnull_class(p3sub2, ConstClass(node_vtable2)) []
-        jump(i1, p1, p3sub2)
+        jump(i1, p1, p3sub)
         """
         self.optimize_loop(ops, expected, preamble)
 
@@ -902,9 +896,7 @@
         guard_true(i2b) []
         p3 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3, i2, descr=nextdescr)
-        # XXX: VIRTUALHEAP (see above)
-        i3 = getfield_gc(p3, descr=nextdescr)
-        jump(p3, i3)
+        jump(p3, i2)
         """
         self.optimize_loop(ops, expected, preamble)
 
@@ -1219,7 +1211,15 @@
         setfield_gc(p3, p30, descr=valuedescr)
         jump(i29, p30, p3)
         """
-        expected = preamble
+        expected = """
+        [i0, p1, p3]
+        i28 = int_add(i0, 1)
+        i29 = int_add(i28, 1)
+        p30 = new_with_vtable(ConstClass(node_vtable))
+        setfield_gc(p3, p30, descr=valuedescr)
+        setfield_gc(p30, i28, descr=nextdescr)
+        jump(i29, p30, p3)
+        """
         self.optimize_loop(ops, expected, preamble)
 
     def test_nonvirtual_1(self):
@@ -1411,7 +1411,7 @@
         guard_isnull(p18) [p0, p8]
         p31 = new(descr=ssize)
         p35 = new_with_vtable(ConstClass(node_vtable))
-        setfield_gc(p35, p31, descr=valuedescr)        
+        setfield_gc(p35, p31, descr=valuedescr)
         jump(p0, p35)
         """
         expected = """
@@ -1426,7 +1426,7 @@
         guard_isnull(p18) [p0, p8]
         p31 = new(descr=ssize)
         p35 = new_with_vtable(ConstClass(node_vtable))
-        setfield_gc(p35, p31, descr=valuedescr)        
+        setfield_gc(p35, p31, descr=valuedescr)
         jump(p0, p35, p19, p18)
         """
         expected = """
@@ -1435,7 +1435,7 @@
         jump(p0, NULL)
         """
         self.optimize_loop(ops, expected)
-        
+
     def test_varray_1(self):
         ops = """
         [i1]
@@ -2181,7 +2181,7 @@
         jump(p1)
         """
         self.optimize_loop(ops, expected)
-        
+
     def test_duplicate_getarrayitem_2(self):
         ops = """
         [p1, i0]
@@ -2199,7 +2199,7 @@
         jump(p1, i7, i6)
         """
         self.optimize_loop(ops, expected)
-        
+
     def test_duplicate_getarrayitem_after_setarrayitem_1(self):
         ops = """
         [p1, p2]
@@ -2408,8 +2408,8 @@
         guard_class(p2, ConstClass(node_vtable)) []
         p3 = getfield_gc(p1, descr=otherdescr)
         guard_class(p3, ConstClass(node_vtable)) []
+        p3a = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3, p2, descr=otherdescr)
-        p3a = new_with_vtable(ConstClass(node_vtable))
         escape(p3a)
         jump(p3a)
         """
@@ -2421,9 +2421,9 @@
         # setfield_gc(p3, p2, descr=otherdescr) # p3a.other = p2a
         # p1a = new_with_vtable(ConstClass(node_vtable2))
         # p2a = new_with_vtable(ConstClass(node_vtable))
+        p3anew = new_with_vtable(ConstClass(node_vtable))
         p2 = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3a, p2, descr=otherdescr) # p3a.other = p2a
-        p3anew = new_with_vtable(ConstClass(node_vtable))
         escape(p3anew)
         jump(p3anew)
         """
@@ -2458,9 +2458,9 @@
         p3 = getfield_gc(p1, descr=otherdescr)
         guard_class(p3, ConstClass(node_vtable)) []
         # p1a = new_with_vtable(ConstClass(node_vtable2))
+        p3a = new_with_vtable(ConstClass(node_vtable))
         p2a = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3, p2a, descr=otherdescr)
-        p3a = new_with_vtable(ConstClass(node_vtable))
         escape(p3a)
         # setfield_gc(p1a, p2a, descr=nextdescr)
         # setfield_gc(p1a, p3a, descr=otherdescr)
@@ -2468,9 +2468,9 @@
         """
         expected = """
         [p2, p3]
+        p3a = new_with_vtable(ConstClass(node_vtable))
         p2a = new_with_vtable(ConstClass(node_vtable))
         setfield_gc(p3, p2a, descr=otherdescr)
-        p3a = new_with_vtable(ConstClass(node_vtable))
         escape(p3a)
         jump(p2a, p3a)
         """
@@ -2790,7 +2790,6 @@
         self.optimize_loop(ops, expected, preamble)
 
     def test_remove_duplicate_pure_op_ovf_with_lazy_setfield(self):
-        py.test.skip('this optimization is not yet supprted')
         ops = """
         [i1, p1]
         i3 = int_add_ovf(i1, 1)
@@ -2812,14 +2811,14 @@
         guard_no_overflow() []
         i3b = int_is_true(i3)
         guard_true(i3b) []
-        setfield_gc(p1, i1, descr=valuedescr)        
+        setfield_gc(p1, i1, descr=valuedescr)
         escape(i3)
         escape(i3)
         jump(i1, p1, i3)
         """
         expected = """
         [i1, p1, i3]
-        setfield_gc(p1, i1, descr=valuedescr)        
+        setfield_gc(p1, i1, descr=valuedescr)
         escape(i3)
         escape(i3)
         jump(i1, p1, i3)
@@ -2830,7 +2829,7 @@
         ops = """
         [p8, p11, i24]
         p26 = new_with_vtable(ConstClass(node_vtable))
-        setfield_gc(p26, i24, descr=adescr)        
+        setfield_gc(p26, i24, descr=adescr)
         i34 = getfield_gc_pure(p11, descr=valuedescr)
         i35 = getfield_gc_pure(p26, descr=adescr)
         i36 = int_add_ovf(i34, i35)
@@ -2839,10 +2838,10 @@
         """
         expected = """
         [p8, p11, i26]
-        jump(p8, p11, i26)        
-        """
-        self.optimize_loop(ops, expected)
-        
+        jump(p8, p11, i26)
+        """
+        self.optimize_loop(ops, expected)
+
     def test_ovf_guard_in_short_preamble2(self):
         ops = """
         [p8, p11, p12]
@@ -3191,13 +3190,18 @@
         jump(p1, i4, i3)
         '''
         expected = '''
+        [p1, i4, i3, i5]
+        setfield_gc(p1, i5, descr=valuedescr)
+        jump(p1, i3, i5, i5)
+        '''
+        preamble = '''
         [p1, i1, i4]
         setfield_gc(p1, i1, descr=valuedescr)
         i3 = call(p1, descr=plaincalldescr)
         setfield_gc(p1, i3, descr=valuedescr)
-        jump(p1, i4, i3)
+        jump(p1, i4, i3, i3)
         '''
-        self.optimize_loop(ops, expected, expected)
+        self.optimize_loop(ops, expected, preamble)
 
     def test_call_pure_invalidates_heap_knowledge(self):
         # CALL_PURE should still force the setfield_gc() to occur before it
@@ -3209,21 +3213,20 @@
         jump(p1, i4, i3)
         '''
         expected = '''
+        [p1, i4, i3, i5]
+        setfield_gc(p1, i4, descr=valuedescr)
+        jump(p1, i3, i5, i5)
+        '''
+        preamble = '''
         [p1, i1, i4]
         setfield_gc(p1, i1, descr=valuedescr)
         i3 = call(p1, descr=plaincalldescr)
         setfield_gc(p1, i1, descr=valuedescr)
-        jump(p1, i4, i3)
+        jump(p1, i4, i3, i3)
         '''
-        self.optimize_loop(ops, expected, expected)
+        self.optimize_loop(ops, expected, preamble)
 
     def test_call_pure_constant_folding(self):
-        # CALL_PURE is not marked as is_always_pure(), because it is wrong
-        # to call the function arbitrary many times at arbitrary points in
-        # time.  Check that it is either constant-folded (and replaced by
-        # the result of the call, recorded as the first arg), or turned into
-        # a regular CALL.
-        # XXX can this test be improved with unrolling?
         arg_consts = [ConstInt(i) for i in (123456, 4, 5, 6)]
         call_pure_results = {tuple(arg_consts): ConstInt(42)}
         ops = '''
@@ -3239,14 +3242,13 @@
         escape(i1)
         escape(i2)
         i4 = call(123456, 4, i0, 6, descr=plaincalldescr)
-        jump(i0, i4)
+        jump(i0, i4, i4)
         '''
         expected = '''
-        [i0, i2]
+        [i0, i4, i5]
         escape(42)
-        escape(i2)
-        i4 = call(123456, 4, i0, 6, descr=plaincalldescr)
-        jump(i0, i4)
+        escape(i4)
+        jump(i0, i5, i5)
         '''
         self.optimize_loop(ops, expected, preamble, call_pure_results)
 
@@ -3270,18 +3272,43 @@
         escape(i2)
         i4 = call(123456, 4, i0, 6, descr=plaincalldescr)
         guard_no_exception() []
-        jump(i0, i4)
+        jump(i0, i4, i4)
         '''
         expected = '''
-        [i0, i2]
+        [i0, i2, i3]
         escape(42)
         escape(i2)
-        i4 = call(123456, 4, i0, 6, descr=plaincalldescr)
-        guard_no_exception() []
-        jump(i0, i4)
+        jump(i0, i3, i3)
         '''
         self.optimize_loop(ops, expected, preamble, call_pure_results)
 
+    def test_call_pure_returning_virtual(self):
+        # XXX: This kind of loop invaraint call_pure will be forced
+        #      both in the preamble and in the peeled loop
+        ops = '''
+        [p1, i1, i2]
+        p2 = call_pure(0, p1, i1, i2, descr=strslicedescr)
+        escape(p2)
+        jump(p1, i1, i2)
+        '''
+        preamble = '''
+        [p1, i1, i2]
+        i6 = int_sub(i2, i1)
+        p2 = newstr(i6)
+        copystrcontent(p1, p2, i1, 0, i6)
+        escape(p2)
+        jump(p1, i1, i2, i6)
+        '''
+        expected = '''
+        [p1, i1, i2, i6]
+        p2 = newstr(i6)
+        copystrcontent(p1, p2, i1, 0, i6)
+        escape(p2)
+        jump(p1, i1, i2, i6)
+        '''
+        self.optimize_loop(ops, expected, preamble)
+        
+
     # ----------
 
     def test_vref_nonvirtual_nonescape(self):
@@ -5150,14 +5177,14 @@
         [i0, i1, i10, i11, i2, i3, i4]
         escape(i2)
         escape(i3)
-        escape(i4)        
+        escape(i4)
         i24 = int_mul_ovf(i10, i11)
         guard_no_overflow() []
         i23 = int_sub_ovf(i10, i11)
         guard_no_overflow() []
         i22 = int_add_ovf(i10, i11)
         guard_no_overflow() []
-        jump(i0, i1, i10, i11, i2, i3, i4) 
+        jump(i0, i1, i10, i11, i2, i3, i4)
         """
         self.optimize_loop(ops, expected)
 
@@ -5366,6 +5393,8 @@
         """
         self.optimize_strunicode_loop(ops, expected, expected)
 
+    # XXX Should some of the call's below now be call_pure?
+
     def test_str_concat_1(self):
         ops = """
         [p1, p2]
@@ -5699,14 +5728,14 @@
         ops = """
         [p0, i0]
         i1 = unicodegetitem(p0, i0)
-        i10 = unicodegetitem(p0, i0)        
+        i10 = unicodegetitem(p0, i0)
         i2 = int_lt(i1, 0)
         guard_false(i2) []
         jump(p0, i0)
         """
         expected = """
         [p0, i0]
-        i1 = unicodegetitem(p0, i0)        
+        i1 = unicodegetitem(p0, i0)
         jump(p0, i0)
         """
         self.optimize_loop(ops, expected)
@@ -5865,7 +5894,7 @@
         """
         preamble = """
         [p1, i1, i2, p3]
-        guard_nonnull(p3) []        
+        guard_nonnull(p3) []
         i4 = int_sub(i2, i1)
         i0 = call(0, p1, i1, i4, p3, descr=streq_slice_nonnull_descr)
         escape(i0)
@@ -5931,13 +5960,18 @@
         escape(i0)
         jump(p1)
         """
-        expected = """
+        preamble = """
         [p1]
         i0 = ptr_eq(p1, NULL)
         escape(i0)
-        jump(p1)
-        """
-        self.optimize_strunicode_loop_extradescrs(ops, expected, expected)
+        jump(p1, i0)
+        """
+        expected = """
+        [p1, i0]
+        escape(i0)
+        jump(p1, i0)
+        """
+        self.optimize_strunicode_loop_extradescrs(ops, expected, preamble)
 
     def test_str_equal_none2(self):
         ops = """
@@ -5946,13 +5980,18 @@
         escape(i0)
         jump(p1)
         """
-        expected = """
+        preamble = """
         [p1]
         i0 = ptr_eq(p1, NULL)
         escape(i0)
-        jump(p1)
-        """
-        self.optimize_strunicode_loop_extradescrs(ops, expected, expected)
+        jump(p1, i0)
+        """
+        expected = """
+        [p1, i0]
+        escape(i0)
+        jump(p1, i0)
+        """
+        self.optimize_strunicode_loop_extradescrs(ops, expected, preamble)
 
     def test_str_equal_nonnull1(self):
         ops = """
@@ -6474,7 +6513,7 @@
         setfield_gc(p3, i1, descr=adescr)
         setfield_gc(p3, i2, descr=bdescr)
         i5 = int_gt(ii, 42)
-        guard_true(i5) []        
+        guard_true(i5) []
         jump(p0, p1, p3, ii2, ii, i1, i2)
         """
         self.optimize_loop(ops, expected)
@@ -6500,7 +6539,7 @@
         p1 = getfield_gc(p0, descr=nextdescr)
         guard_nonnull_class(p1, ConstClass(node_vtable)) []
         p2 = getfield_gc(p1, descr=nextdescr)
-        guard_nonnull_class(p2, ConstClass(node_vtable)) []        
+        guard_nonnull_class(p2, ConstClass(node_vtable)) []
         jump(p0)
         """
         expected = """
@@ -6514,11 +6553,11 @@
         guard_class(p1, ConstClass(node_vtable)) []
         p2 = getfield_gc(p1, descr=nextdescr)
         guard_nonnull(p2) []
-        guard_class(p2, ConstClass(node_vtable)) []        
+        guard_class(p2, ConstClass(node_vtable)) []
         jump(p0)
         """
         self.optimize_loop(ops, expected, expected_short=short)
-        
+
     def test_forced_virtual_pure_getfield(self):
         ops = """
         [p0]
@@ -6582,7 +6621,7 @@
         jump(p1, i2)
         """
         self.optimize_loop(ops, expected)
-        
+
     def test_loopinvariant_strlen(self):
         ops = """
         [p9]
@@ -6715,7 +6754,7 @@
         [p0, p1]
         p2 = new_with_vtable(ConstClass(node_vtable))
         p3 = new_with_vtable(ConstClass(node_vtable))
-        setfield_gc(p2, p3, descr=nextdescr) 
+        setfield_gc(p2, p3, descr=nextdescr)
         jump(p2, p3)
         """
         expected = """
@@ -6734,7 +6773,7 @@
         jump(p2, i2)
         """
         expected = """
-        [p1]        
+        [p1]
         p2 = getarrayitem_gc(p1, 7, descr=<GcPtrArrayDescr>)
         i1 = arraylen_gc(p1)
         jump(p2)
@@ -6775,8 +6814,8 @@
         jump(p0, p2, p1)
         """
         self.optimize_loop(ops, expected, expected_short=short)
-        
-        
+
+
     def test_loopinvariant_constant_strgetitem(self):
         ops = """
         [p0]
@@ -6830,11 +6869,11 @@
         expected = """
         [p0, i22, p1]
         call(i22, descr=nonwritedescr)
-        i3 = unicodelen(p1) # Should be killed by backend        
+        i3 = unicodelen(p1) # Should be killed by backend
         jump(p0, i22, p1)
         """
         self.optimize_loop(ops, expected, expected_short=short)
-        
+
     def test_propagate_virtual_arryalen(self):
         ops = """
         [p0]
@@ -6903,7 +6942,7 @@
         [p0, p1, p10, p11]
         i1 = arraylen_gc(p10, descr=arraydescr)
         getarrayitem_gc(p11, 1, descr=arraydescr)
-        call(i1, descr=nonwritedescr)        
+        call(i1, descr=nonwritedescr)
         jump(p1, p0, p11, p10)
         """
         self.optimize_loop(ops, expected)
@@ -6912,20 +6951,20 @@
         ops = """
         [p5]
         i10 = getfield_gc(p5, descr=valuedescr)
-        call(i10, descr=nonwritedescr) 
+        call(i10, descr=nonwritedescr)
         setfield_gc(p5, 1, descr=valuedescr)
         jump(p5)
         """
         preamble = """
         [p5]
         i10 = getfield_gc(p5, descr=valuedescr)
-        call(i10, descr=nonwritedescr) 
+        call(i10, descr=nonwritedescr)
         setfield_gc(p5, 1, descr=valuedescr)
         jump(p5)
         """
         expected = """
         [p5]
-        call(1, descr=nonwritedescr) 
+        call(1, descr=nonwritedescr)
         jump(p5)
         """
         self.optimize_loop(ops, expected, preamble)
@@ -6963,7 +7002,7 @@
         [p9]
         call_assembler(0, descr=asmdescr)
         i18 = getfield_gc(p9, descr=valuedescr)
-        guard_value(i18, 0) []        
+        guard_value(i18, 0) []
         jump(p9)
         """
         self.optimize_loop(ops, expected)
@@ -6992,17 +7031,17 @@
         i10 = getfield_gc(p5, descr=valuedescr)
         i11 = getfield_gc(p6, descr=nextdescr)
         call(i10, i11, descr=nonwritedescr)
-        setfield_gc(p6, i10, descr=nextdescr)        
+        setfield_gc(p6, i10, descr=nextdescr)
         jump(p5, p6)
         """
         expected = """
         [p5, p6, i10, i11]
         call(i10, i11, descr=nonwritedescr)
-        setfield_gc(p6, i10, descr=nextdescr)        
+        setfield_gc(p6, i10, descr=nextdescr)
         jump(p5, p6, i10, i10)
         """
         self.optimize_loop(ops, expected)
-        
+
     def test_cached_pure_func_of_equal_fields(self):
         ops = """
         [p5, p6]
@@ -7011,18 +7050,18 @@
         i12 = int_add(i10, 7)
         i13 = int_add(i11, 7)
         call(i12, i13, descr=nonwritedescr)
-        setfield_gc(p6, i10, descr=nextdescr)        
+        setfield_gc(p6, i10, descr=nextdescr)
         jump(p5, p6)
         """
         expected = """
         [p5, p6, i14, i12, i10]
         i13 = int_add(i14, 7)
         call(i12, i13, descr=nonwritedescr)
-        setfield_gc(p6, i10, descr=nextdescr)        
+        setfield_gc(p6, i10, descr=nextdescr)
         jump(p5, p6, i10, i12, i10)
         """
         self.optimize_loop(ops, expected)
-        
+
     def test_forced_counter(self):
         # XXX: VIRTUALHEAP (see above)
         py.test.skip("would be fixed by make heap optimizer aware of virtual setfields")
@@ -7165,7 +7204,7 @@
         expected = """
         [p1, p2, i2, i1]
         call(i2, descr=nonwritedescr)
-        setfield_gc(p2, i1, descr=nextdescr)        
+        setfield_gc(p2, i1, descr=nextdescr)
         jump(p1, p2, i2, i1)
         """
         self.optimize_loop(ops, expected)
@@ -7185,11 +7224,65 @@
         expected = """
         [p1, p2, i2, i1]
         call(i2, descr=nonwritedescr)
-        setfield_gc(p2, i1, descr=valuedescr)        
+        setfield_gc(p2, i1, descr=valuedescr)
         jump(p1, p2, i2, i1)
         """
         self.optimize_loop(ops, expected)
-        
+
+    def test_heap_cache_forced_virtuals(self):
+        ops = """
+        [i1, i2, p0]
+        p1 = new(descr=ssize)
+        setfield_gc(p1, i1, descr=adescr)
+        setfield_gc(p1, i2, descr=bdescr)
+        call(p0, p1, descr=writeadescr)
+        i3 = getfield_gc(p1, descr=adescr)
+        i4 = getfield_gc(p1, descr=bdescr)
+        jump(i3, i4, p0)
+        """
+        expected = """
+        [i1, i2, p0]
+        p1 = new(descr=ssize)
+        setfield_gc(p1, i1, descr=adescr)
+        call(p0, p1, descr=writeadescr)
+        i3 = getfield_gc(p1, descr=adescr)
+        setfield_gc(p1, i2, descr=bdescr)
+        jump(i3, i2, p0)
+        """
+        self.optimize_loop(ops, expected)
+
+
+    def test_setarrayitem_followed_by_arraycopy(self):
+        ops = """
+        [p1, p2]
+        setarrayitem_gc(p1, 2, 10, descr=arraydescr)
+        setarrayitem_gc(p2, 3, 13, descr=arraydescr)        
+        call(0, p1, p2, 0, 0, 10, descr=arraycopydescr)
+        jump(p1, p2)
+        """
+        self.optimize_loop(ops, ops)
+
+    def test_heap_cache_virtuals_forced_by_delayed_setfield(self):
+        py.test.skip('not yet supoprted')
+        ops = """
+        [i1, p0]
+        p1 = new(descr=ssize)
+        setfield_gc(p1, i1, descr=valuedescr)
+        setfield_gc(p0, p1, descr=adescr)
+        call(p0, descr=writeadescr)
+        i2 = getfield_gc(p1, descr=valuedescr)
+        jump(i2, p0)
+        """
+        expected = """
+        [i1, p0]
+        p1 = new(descr=ssize)
+        setfield_gc(p1, i1, descr=valuedescr)
+        setfield_gc(p0, p1, descr=adescr)
+        call(p0, descr=writeadescr)
+        jump(i1, p0)
+        """
+        self.optimize_loop(ops, expected)
+
 class TestLLtype(OptimizeOptTest, LLtypeMixin):
     pass
-        
+
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_util.py b/pypy/jit/metainterp/optimizeopt/test/test_util.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_util.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_util.py
@@ -182,7 +182,8 @@
                             EffectInfo.EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE,
                             can_invalidate=True))
     arraycopydescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT,
-             EffectInfo([], [], [], [], oopspecindex=EffectInfo.OS_ARRAYCOPY))
+             EffectInfo([], [arraydescr], [], [arraydescr],
+                        oopspecindex=EffectInfo.OS_ARRAYCOPY))
 
     for _name, _os in [
         ('strconcatdescr',               'OS_STR_CONCAT'),
diff --git a/pypy/jit/metainterp/optimizeopt/unroll.py b/pypy/jit/metainterp/optimizeopt/unroll.py
--- a/pypy/jit/metainterp/optimizeopt/unroll.py
+++ b/pypy/jit/metainterp/optimizeopt/unroll.py
@@ -75,7 +75,6 @@
         self.importable_values = {}
         self.emitting_dissabled = False
         self.emitted_guards = 0
-        self.emitted_pure_operations = {}
 
     def ensure_imported(self, value):
         if not self.emitting_dissabled and value in self.importable_values:
@@ -96,21 +95,6 @@
         new = UnrollableOptimizer(self.metainterp_sd, self.loop)
         return self._new(new)
 
-    def remember_emitting_pure(self, op):
-        self.emitted_pure_operations[op] = True
-
-    def produce_potential_short_preamble_ops(self, sb):
-        for op in self.emitted_pure_operations:
-            if op.getopnum() == rop.GETARRAYITEM_GC_PURE or \
-               op.getopnum() == rop.STRGETITEM or \
-               op.getopnum() == rop.UNICODEGETITEM:
-                if not self.getvalue(op.getarg(1)).is_constant():
-                    continue
-            sb.add_potential(op)
-        for opt in self.optimizations:
-            opt.produce_potential_short_preamble_ops(sb)
-
-
 
 class UnrollOptimizer(Optimization):
     """Unroll the loop into two iterations. The first one will
@@ -154,7 +138,7 @@
 
             KillHugeIntBounds(self.optimizer).apply()
             
-            loop.preamble.operations = self.optimizer.newoperations
+            loop.preamble.operations = self.optimizer.get_newoperations()
             jump_args = [self.getvalue(a).get_key_box() for a in jump_args]
 
             start_resumedescr = loop.preamble.start_resumedescr.clone_if_mutable()
@@ -167,8 +151,9 @@
             virtual_state = modifier.get_virtual_state(jump_args)
             
             values = [self.getvalue(arg) for arg in jump_args]
-            inputargs = virtual_state.make_inputargs(values)
-            short_inputargs = virtual_state.make_inputargs(values, keyboxes=True)
+            inputargs = virtual_state.make_inputargs(values, self.optimizer)
+            short_inputargs = virtual_state.make_inputargs(values, self.optimizer,
+                                                           keyboxes=True)
 
             self.constant_inputargs = {}
             for box in jump_args: 
@@ -197,7 +182,7 @@
             # operations needed to setup the proper state of those virtuals
             # in the peeled loop
             inputarg_setup_ops = []
-            preamble_optimizer.newoperations = []
+            preamble_optimizer.clear_newoperations()
             seen = {}
             for box in inputargs:
                 if box in seen:
@@ -211,9 +196,8 @@
                     continue
                 seen[box] = True
                 value = preamble_optimizer.getvalue(box)
-                value.force_box()
-            preamble_optimizer.flush()
-            inputarg_setup_ops += preamble_optimizer.newoperations
+                value.force_box(preamble_optimizer)
+            inputarg_setup_ops += preamble_optimizer.get_newoperations()
 
             # Setup the state of the new optimizer by emiting the
             # short preamble operations and discarding the result
@@ -226,8 +210,9 @@
                 if op and op.result:
                     preamble_value = preamble_optimizer.getvalue(op.result)
                     value = self.optimizer.getvalue(op.result)
-                    imp = ValueImporter(self, preamble_value, op)
-                    self.optimizer.importable_values[value] = imp
+                    if not value.is_virtual():
+                        imp = ValueImporter(self, preamble_value, op)
+                        self.optimizer.importable_values[value] = imp
                     newresult = self.optimizer.getvalue(op.result).get_key_box()
                     if newresult is not op.result:
                         self.short_boxes.alias(newresult, op.result)
@@ -243,13 +228,13 @@
                                 virtual_state)
             
             loop.inputargs = inputargs
-            args = [preamble_optimizer.getvalue(self.short_boxes.original(a)).force_box()\
+            args = [preamble_optimizer.getvalue(self.short_boxes.original(a)).force_box(preamble_optimizer)\
                     for a in inputargs]
             jmp = ResOperation(rop.JUMP, args, None)
             jmp.setdescr(loop.token)
             loop.preamble.operations.append(jmp)
 
-            loop.operations = self.optimizer.newoperations
+            loop.operations = self.optimizer.get_newoperations()
             maxguards = self.optimizer.metainterp_sd.warmrunnerdesc.memory_manager.max_retrace_guards
             
             if self.optimizer.emitted_guards > maxguards:
@@ -334,9 +319,10 @@
         assert jumpop
         original_jumpargs = jumpop.getarglist()[:]
         values = [self.getvalue(arg) for arg in jumpop.getarglist()]
-        jumpargs = virtual_state.make_inputargs(values)
+        jumpargs = virtual_state.make_inputargs(values, self.optimizer)
         jumpop.initarglist(jumpargs)
-        jmp_to_short_args = virtual_state.make_inputargs(values, keyboxes=True)
+        jmp_to_short_args = virtual_state.make_inputargs(values, self.optimizer,
+                                                         keyboxes=True)
         self.short_inliner = Inliner(short_inputargs, jmp_to_short_args)
         
         for box, const in self.constant_inputargs.items():
@@ -346,11 +332,11 @@
             newop = self.short_inliner.inline_op(op)
             self.optimizer.send_extra_operation(newop)
         
-        self.optimizer.flush()
+        newoperations = self.optimizer.get_newoperations()
 
         i = j = 0
-        while i < len(self.optimizer.newoperations) or j < len(jumpargs):
-            if i == len(self.optimizer.newoperations):
+        while i < len(newoperations) or j < len(jumpargs):
+            if i == len(newoperations):
                 while j < len(jumpargs):
                     a = jumpargs[j]
                     if self.optimizer.loop.logops:
@@ -359,7 +345,7 @@
                                     jumpargs, short_seen)
                     j += 1
             else:
-                op = self.optimizer.newoperations[i]
+                op = newoperations[i]
 
                 self.boxes_created_this_iteration[op.result] = True
                 args = op.getarglist()
@@ -374,6 +360,7 @@
                     self.import_box(a, inputargs, short, short_jumpargs,
                                     jumpargs, short_seen)
                 i += 1
+            newoperations = self.optimizer.get_newoperations()
 
         jumpop.initarglist(jumpargs)
         self.optimizer.send_extra_operation(jumpop)
@@ -467,7 +454,7 @@
         inputargs.append(box)
         box = newresult
         if box in self.optimizer.values:
-            box = self.optimizer.values[box].force_box()
+            box = self.optimizer.values[box].force_box(self.optimizer)
         jumpargs.append(box)
         
 
@@ -482,11 +469,6 @@
         if op.getopnum() == rop.JUMP:
             loop_token = op.getdescr()
             assert isinstance(loop_token, LoopToken)
-            # FIXME: Use a tree, similar to the tree formed by the full
-            # preamble and it's bridges, instead of a list to save time and
-            # memory. This should also allow better behaviour in
-            # situations that the is_emittable() chain currently cant
-            # handle and the inlining fails unexpectedly belwo.
             short = loop_token.short_preamble
             if short:
                 args = op.getarglist()
@@ -522,7 +504,7 @@
 
                         values = [self.getvalue(arg)
                                   for arg in op.getarglist()]
-                        args = sh.virtual_state.make_inputargs(values,
+                        args = sh.virtual_state.make_inputargs(values, self.optimizer,
                                                                keyboxes=True)
                         inliner = Inliner(sh.inputargs, args)
                         
diff --git a/pypy/jit/metainterp/optimizeopt/virtualize.py b/pypy/jit/metainterp/optimizeopt/virtualize.py
--- a/pypy/jit/metainterp/optimizeopt/virtualize.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualize.py
@@ -10,13 +10,12 @@
 
 
 class AbstractVirtualValue(optimizer.OptValue):
-    _attrs_ = ('optimizer', 'keybox', 'source_op', '_cached_vinfo')
+    _attrs_ = ('keybox', 'source_op', '_cached_vinfo')
     box = None
     level = optimizer.LEVEL_NONNULL
     _cached_vinfo = None
 
-    def __init__(self, optimizer, keybox, source_op=None):
-        self.optimizer = optimizer
+    def __init__(self, keybox, source_op=None):
         self.keybox = keybox   # only used as a key in dictionaries
         self.source_op = source_op  # the NEW_WITH_VTABLE/NEW_ARRAY operation
                                     # that builds this box
@@ -29,17 +28,17 @@
             return self.keybox
         return self.box
 
-    def force_box(self):
+    def force_box(self, optforce):
         if self.box is None:
-            self.optimizer.forget_numberings(self.keybox)
-            self._really_force()
+            optforce.forget_numberings(self.keybox)
+            self._really_force(optforce)
         return self.box
 
-    def force_at_end_of_preamble(self, already_forced):
+    def force_at_end_of_preamble(self, already_forced, optforce):
         value = already_forced.get(self, None)
         if value:
             return value
-        return OptValue(self.force_box())
+        return OptValue(self.force_box(optforce))
 
     def make_virtual_info(self, modifier, fieldnums):
         if fieldnums is None:
@@ -55,7 +54,7 @@
     def _make_virtual(self, modifier):
         raise NotImplementedError("abstract base")
 
-    def _really_force(self):
+    def _really_force(self, optforce):
         raise NotImplementedError("abstract base")
 
     def import_from(self, other, optimizer):
@@ -70,10 +69,11 @@
 get_fielddescrlist_cache._annspecialcase_ = "specialize:memo"
 
 class AbstractVirtualStructValue(AbstractVirtualValue):
-    _attrs_ = ('_fields', '_cached_sorted_fields')
+    _attrs_ = ('_fields', 'cpu', '_cached_sorted_fields')
 
-    def __init__(self, optimizer, keybox, source_op=None):
-        AbstractVirtualValue.__init__(self, optimizer, keybox, source_op)
+    def __init__(self, cpu, keybox, source_op=None):
+        AbstractVirtualValue.__init__(self, keybox, source_op)
+        self.cpu = cpu
         self._fields = {}
         self._cached_sorted_fields = None
 
@@ -87,45 +87,44 @@
     def _get_descr(self):
         raise NotImplementedError
 
-    def _is_immutable_and_filled_with_constants(self):
+    def _is_immutable_and_filled_with_constants(self, optforce):
         count = self._get_descr().count_fields_if_immutable()
         if count != len(self._fields):    # always the case if count == -1
             return False
         for value in self._fields.itervalues():
-            subbox = value.force_box()
+            subbox = value.force_box(optforce)
             if not isinstance(subbox, Const):
                 return False
         return True
 
-    def force_at_end_of_preamble(self, already_forced):
+    def force_at_end_of_preamble(self, already_forced, optforce):
         if self in already_forced:
             return self
         already_forced[self] = self
         if self._fields:
             for ofs in self._fields.keys():
-                self._fields[ofs] = self._fields[ofs].force_at_end_of_preamble(already_forced)
+                self._fields[ofs] = self._fields[ofs].force_at_end_of_preamble(already_forced, optforce)
         return self
 
-    def _really_force(self):
+    def _really_force(self, optforce):
         op = self.source_op
         assert op is not None
         # ^^^ This case should not occur any more (see test_bug_3).
         #
         if not we_are_translated():
             op.name = 'FORCE ' + self.source_op.name
-
-        if self._is_immutable_and_filled_with_constants():
-            box = self.optimizer.constant_fold(op)
+            
+        if self._is_immutable_and_filled_with_constants(optforce):
+            box = optforce.optimizer.constant_fold(op)
             self.make_constant(box)
             for ofs, value in self._fields.iteritems():
-                subbox = value.force_box()
+                subbox = value.force_box(optforce)
                 assert isinstance(subbox, Const)
-                execute(self.optimizer.cpu, None, rop.SETFIELD_GC,
+                execute(optforce.optimizer.cpu, None, rop.SETFIELD_GC,
                         ofs, box, subbox)
             # keep self._fields, because it's all immutable anyway
         else:
-            newoperations = self.optimizer.newoperations
-            newoperations.append(op)
+            optforce.emit_operation(op)
             self.box = box = op.result
             #
             iteritems = self._fields.iteritems()
@@ -135,10 +134,11 @@
             for ofs, value in iteritems:
                 if value.is_null():
                     continue
-                subbox = value.force_box()
+                subbox = value.force_box(optforce)
                 op = ResOperation(rop.SETFIELD_GC, [box, subbox], None,
                                   descr=ofs)
-                newoperations.append(op)
+
+                optforce.emit_operation(op)
 
     def _get_field_descr_list(self):
         _cached_sorted_fields = self._cached_sorted_fields
@@ -155,7 +155,7 @@
             else:
                 lst = self._fields.keys()
             sort_descrs(lst)
-            cache = get_fielddescrlist_cache(self.optimizer.cpu)
+            cache = get_fielddescrlist_cache(self.cpu)
             result = cache.get(lst, None)
             if result is None:
                 cache[lst] = lst
@@ -180,8 +180,8 @@
 class VirtualValue(AbstractVirtualStructValue):
     level = optimizer.LEVEL_KNOWNCLASS
 
-    def __init__(self, optimizer, known_class, keybox, source_op=None):
-        AbstractVirtualStructValue.__init__(self, optimizer, keybox, source_op)
+    def __init__(self, cpu, known_class, keybox, source_op=None):
+        AbstractVirtualStructValue.__init__(self, cpu, keybox, source_op)
         assert isinstance(known_class, Const)
         self.known_class = known_class
 
@@ -190,7 +190,7 @@
         return modifier.make_virtual(self.known_class, fielddescrs)
 
     def _get_descr(self):
-        return vtable2descr(self.optimizer.cpu, self.known_class.getint())
+        return vtable2descr(self.cpu, self.known_class.getint())
 
     def __repr__(self):
         cls_name = self.known_class.value.adr.ptr._obj._TYPE._name
@@ -201,8 +201,8 @@
 
 class VStructValue(AbstractVirtualStructValue):
 
-    def __init__(self, optimizer, structdescr, keybox, source_op=None):
-        AbstractVirtualStructValue.__init__(self, optimizer, keybox, source_op)
+    def __init__(self, cpu, structdescr, keybox, source_op=None):
+        AbstractVirtualStructValue.__init__(self, cpu, keybox, source_op)
         self.structdescr = structdescr
 
     def _make_virtual(self, modifier):
@@ -215,10 +215,10 @@
 
 class VArrayValue(AbstractVirtualValue):
 
-    def __init__(self, optimizer, arraydescr, size, keybox, source_op=None):
-        AbstractVirtualValue.__init__(self, optimizer, keybox, source_op)
+    def __init__(self, arraydescr, constvalue, size, keybox, source_op=None):
+        AbstractVirtualValue.__init__(self, keybox, source_op)
         self.arraydescr = arraydescr
-        self.constvalue = optimizer.new_const_item(arraydescr)
+        self.constvalue = constvalue
         self._items = [self.constvalue] * size
 
     def getlength(self):
@@ -232,31 +232,30 @@
         assert isinstance(itemvalue, optimizer.OptValue)
         self._items[index] = itemvalue
 
-    def force_at_end_of_preamble(self, already_forced):
+    def force_at_end_of_preamble(self, already_forced, optforce):
         if self in already_forced:
             return self
         already_forced[self] = self
         for index in range(len(self._items)):
-            self._items[index] = self._items[index].force_at_end_of_preamble(already_forced)
+            self._items[index] = self._items[index].force_at_end_of_preamble(already_forced, optforce)
         return self
     
-    def _really_force(self):
+    def _really_force(self, optforce):
         assert self.source_op is not None
         if not we_are_translated():
             self.source_op.name = 'FORCE ' + self.source_op.name
-        newoperations = self.optimizer.newoperations
-        newoperations.append(self.source_op)
+        optforce.emit_operation(self.source_op)        
         self.box = box = self.source_op.result
         for index in range(len(self._items)):
             subvalue = self._items[index]
             if subvalue is not self.constvalue:
                 if subvalue.is_null():
                     continue
-                subbox = subvalue.force_box()
+                subbox = subvalue.force_box(optforce)
                 op = ResOperation(rop.SETARRAYITEM_GC,
                                   [box, ConstInt(index), subbox], None,
                                   descr=self.arraydescr)
-                newoperations.append(op)
+                optforce.emit_operation(op)
 
     def get_args_for_fail(self, modifier):
         if self.box is None and not modifier.already_seen_virtual(self.keybox):
@@ -279,17 +278,18 @@
         return OptVirtualize()
         
     def make_virtual(self, known_class, box, source_op=None):
-        vvalue = VirtualValue(self.optimizer, known_class, box, source_op)
+        vvalue = VirtualValue(self.optimizer.cpu, known_class, box, source_op)
         self.make_equal_to(box, vvalue)
         return vvalue
 
     def make_varray(self, arraydescr, size, box, source_op=None):
-        vvalue = VArrayValue(self.optimizer, arraydescr, size, box, source_op)
+        constvalue = self.new_const_item(arraydescr)
+        vvalue = VArrayValue(arraydescr, constvalue, size, box, source_op)
         self.make_equal_to(box, vvalue)
         return vvalue
 
     def make_vstruct(self, structdescr, box, source_op=None):
-        vvalue = VStructValue(self.optimizer, structdescr, box, source_op)
+        vvalue = VStructValue(self.optimizer.cpu, structdescr, box, source_op)
         self.make_equal_to(box, vvalue)
         return vvalue
 
@@ -362,7 +362,6 @@
             self.make_equal_to(op.result, fieldvalue)
         else:
             value.ensure_nonnull()
-            ###self.heap_op_optimizer.optimize_GETFIELD_GC(op, value)
             self.emit_operation(op)
 
     # note: the following line does not mean that the two operations are
@@ -377,7 +376,6 @@
             value.setfield(op.getdescr(), fieldvalue)
         else:
             value.ensure_nonnull()
-            ###self.heap_op_optimizer.optimize_SETFIELD_GC(op, value, fieldvalue)
             self.emit_operation(op)
 
     def optimize_NEW_WITH_VTABLE(self, op):
@@ -417,7 +415,6 @@
                 self.make_equal_to(op.result, itemvalue)
                 return
         value.ensure_nonnull()
-        ###self.heap_op_optimizer.optimize_GETARRAYITEM_GC(op, value)
         self.emit_operation(op)
 
     # note: the following line does not mean that the two operations are
@@ -432,7 +429,6 @@
                 value.setitem(indexbox.getint(), self.getvalue(op.getarg(2)))
                 return
         value.ensure_nonnull()
-        ###self.heap_op_optimizer.optimize_SETARRAYITEM_GC(op, value, fieldvalue)
         self.emit_operation(op)
 
 
diff --git a/pypy/jit/metainterp/optimizeopt/virtualstate.py b/pypy/jit/metainterp/optimizeopt/virtualstate.py
--- a/pypy/jit/metainterp/optimizeopt/virtualstate.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualstate.py
@@ -30,7 +30,7 @@
     def _generate_guards(self, other, box, cpu, extra_guards):
         raise InvalidLoop
 
-    def enum_forced_boxes(self, boxes, value):
+    def enum_forced_boxes(self, boxes, value, optimizer):
         raise NotImplementedError
 
     def enum(self, virtual_state):
@@ -100,14 +100,14 @@
     def _generalization_of(self, other):
         raise NotImplementedError
 
-    def enum_forced_boxes(self, boxes, value):
+    def enum_forced_boxes(self, boxes, value, optimizer):
         assert isinstance(value, virtualize.AbstractVirtualStructValue)
         assert value.is_virtual()
         for i in range(len(self.fielddescrs)):
             v = value._fields[self.fielddescrs[i]]
             s = self.fieldstate[i]
             if s.position > self.position:
-                s.enum_forced_boxes(boxes, v)
+                s.enum_forced_boxes(boxes, v, optimizer)
 
     def _enum(self, virtual_state):
         for s in self.fieldstate:
@@ -177,14 +177,14 @@
                 return False
         return True
 
-    def enum_forced_boxes(self, boxes, value):
+    def enum_forced_boxes(self, boxes, value, optimizer):
         assert isinstance(value, virtualize.VArrayValue)
         assert value.is_virtual()
         for i in range(len(self.fieldstate)):
             v = value._items[i]
             s = self.fieldstate[i]
             if s.position > self.position:
-                s.enum_forced_boxes(boxes, v)
+                s.enum_forced_boxes(boxes, v, optimizer)
 
     def _enum(self, virtual_state):
         for s in self.fieldstate:
@@ -316,11 +316,11 @@
             import pdb; pdb.set_trace()
             raise NotImplementedError
 
-    def enum_forced_boxes(self, boxes, value):
+    def enum_forced_boxes(self, boxes, value, optimizer):
         if self.level == LEVEL_CONSTANT:
             return
         assert 0 <= self.position_in_notvirtuals 
-        boxes[self.position_in_notvirtuals] = value.force_box()
+        boxes[self.position_in_notvirtuals] = value.force_box(optimizer)
 
     def _enum(self, virtual_state):
         if self.level == LEVEL_CONSTANT:
@@ -377,11 +377,13 @@
             self.state[i].generate_guards(other.state[i], args[i],
                                           cpu, extra_guards, renum)
 
-    def make_inputargs(self, values, keyboxes=False):
+    def make_inputargs(self, values, optimizer, keyboxes=False):
+        if optimizer.optearlyforce:
+            optimizer = optimizer.optearlyforce
         assert len(values) == len(self.state)
         inputargs = [None] * len(self.notvirtuals)
         for i in range(len(values)):
-            self.state[i].enum_forced_boxes(inputargs, values[i])
+            self.state[i].enum_forced_boxes(inputargs, values[i], optimizer)
 
         if keyboxes:
             for i in range(len(values)):
@@ -434,7 +436,12 @@
     def get_virtual_state(self, jump_args):
         self.optimizer.force_at_end_of_preamble()
         already_forced = {}
-        values = [self.getvalue(box).force_at_end_of_preamble(already_forced)
+        if self.optimizer.optearlyforce:
+            opt = self.optimizer.optearlyforce
+        else:
+            opt = self.optimizer
+        values = [self.getvalue(box).force_at_end_of_preamble(already_forced,
+                                                              opt)
                   for box in jump_args]
 
         for value in values:
diff --git a/pypy/jit/metainterp/optimizeopt/vstring.py b/pypy/jit/metainterp/optimizeopt/vstring.py
--- a/pypy/jit/metainterp/optimizeopt/vstring.py
+++ b/pypy/jit/metainterp/optimizeopt/vstring.py
@@ -43,7 +43,7 @@
 class __extend__(optimizer.OptValue):
     """New methods added to the base class OptValue for this file."""
 
-    def getstrlen(self, optimization, mode):
+    def getstrlen(self, string_optimizer, mode):
         if mode is mode_string:
             s = self.get_constant_string_spec(mode_string)
             if s is not None:
@@ -52,12 +52,12 @@
             s = self.get_constant_string_spec(mode_unicode)
             if s is not None:
                 return ConstInt(len(s))
-        if optimization is None:
+        if string_optimizer is None:
             return None
         self.ensure_nonnull()
-        box = self.force_box()
+        box = self.force_box(string_optimizer)
         lengthbox = BoxInt()
-        optimization.propagate_forward(ResOperation(mode.STRLEN, [box], lengthbox))
+        string_optimizer.emit_operation(ResOperation(mode.STRLEN, [box], lengthbox))
         return lengthbox
 
     @specialize.arg(1)
@@ -68,25 +68,25 @@
         else:
             return None
 
-    def string_copy_parts(self, optimizer, targetbox, offsetbox, mode):
+    def string_copy_parts(self, string_optimizer, targetbox, offsetbox, mode):
         # Copies the pointer-to-string 'self' into the target string
         # given by 'targetbox', at the specified offset.  Returns the offset
         # at the end of the copy.
-        lengthbox = self.getstrlen(optimizer, mode)
-        srcbox = self.force_box()
-        return copy_str_content(optimizer, srcbox, targetbox,
+        lengthbox = self.getstrlen(string_optimizer, mode)
+        srcbox = self.force_box(string_optimizer)
+        return copy_str_content(string_optimizer, srcbox, targetbox,
                                 CONST_0, offsetbox, lengthbox, mode)
 
 
 class VAbstractStringValue(virtualize.AbstractVirtualValue):
     _attrs_ = ('mode',)
 
-    def __init__(self, optimizer, keybox, source_op, mode):
-        virtualize.AbstractVirtualValue.__init__(self, optimizer, keybox,
+    def __init__(self, keybox, source_op, mode):
+        virtualize.AbstractVirtualValue.__init__(self, keybox,
                                                  source_op)
         self.mode = mode
 
-    def _really_force(self):
+    def _really_force(self, optforce):
         if self.mode is mode_string:
             s = self.get_constant_string_spec(mode_string)
             if s is not None:
@@ -101,12 +101,12 @@
                 return
         assert self.source_op is not None
         self.box = box = self.source_op.result
-        lengthbox = self.getstrlen(self.optimizer, self.mode)
+        lengthbox = self.getstrlen(optforce, self.mode)
         op = ResOperation(self.mode.NEWSTR, [lengthbox], box)
         if not we_are_translated():
             op.name = 'FORCE'
-        self.optimizer.emit_operation(op)
-        self.string_copy_parts(self.optimizer, box, CONST_0, self.mode)
+        optforce.emit_operation(op)
+        self.string_copy_parts(optforce, box, CONST_0, self.mode)
 
 
 class VStringPlainValue(VAbstractStringValue):
@@ -140,15 +140,20 @@
         return mode.emptystr.join([mode.chr(c.box.getint())
                                    for c in self._chars])
 
-    def string_copy_parts(self, optimizer, targetbox, offsetbox, mode):
+    def string_copy_parts(self, string_optimizer, targetbox, offsetbox, mode):
+        if not self.is_virtual() and targetbox is not self.box:
+            lengthbox = self.getstrlen(string_optimizer, mode)
+            srcbox = self.force_box(string_optimizer)
+            return copy_str_content(string_optimizer, srcbox, targetbox,
+                                CONST_0, offsetbox, lengthbox, mode)
         for i in range(len(self._chars)):
-            charbox = self._chars[i].force_box()
+            charbox = self._chars[i].force_box(string_optimizer)
             if not (isinstance(charbox, Const) and charbox.same_constant(CONST_0)):
-                optimizer.emit_operation(ResOperation(mode.STRSETITEM, [targetbox,
-                                                                    offsetbox,
-                                                                    charbox],
+                string_optimizer.emit_operation(ResOperation(mode.STRSETITEM, [targetbox,
+                                                                               offsetbox,
+                                                                               charbox],
                                                   None))
-            offsetbox = _int_add(optimizer, offsetbox, CONST_1)
+            offsetbox = _int_add(string_optimizer, offsetbox, CONST_1)
         return offsetbox
 
     def get_args_for_fail(self, modifier):
@@ -182,16 +187,16 @@
         self.left = left
         self.right = right
 
-    def getstrlen(self, optimizer, mode):
+    def getstrlen(self, string_optimizer, mode):
         if self.lengthbox is None:
-            len1box = self.left.getstrlen(optimizer, mode)
+            len1box = self.left.getstrlen(string_optimizer, mode)
             if len1box is None:
                 return None
-            len2box = self.right.getstrlen(optimizer, mode)
+            len2box = self.right.getstrlen(string_optimizer, mode)
             if len2box is None:
                 return None
-            self.lengthbox = _int_add(optimizer, len1box, len2box)
-            # ^^^ may still be None, if optimizer is None
+            self.lengthbox = _int_add(string_optimizer, len1box, len2box)
+            # ^^^ may still be None, if string_optimizer is None
         return self.lengthbox
 
     @specialize.arg(1)
@@ -204,10 +209,10 @@
             return None
         return s1 + s2
 
-    def string_copy_parts(self, optimizer, targetbox, offsetbox, mode):
-        offsetbox = self.left.string_copy_parts(optimizer, targetbox,
+    def string_copy_parts(self, string_optimizer, targetbox, offsetbox, mode):
+        offsetbox = self.left.string_copy_parts(string_optimizer, targetbox,
                                                 offsetbox, mode)
-        offsetbox = self.right.string_copy_parts(optimizer, targetbox,
+        offsetbox = self.right.string_copy_parts(string_optimizer, targetbox,
                                                  offsetbox, mode)
         return offsetbox
 
@@ -246,8 +251,8 @@
         self.vstart = vstart
         self.vlength = vlength
 
-    def getstrlen(self, _, mode):
-        return self.vlength.force_box()
+    def getstrlen(self, optforce, mode):
+        return self.vlength.force_box(optforce)
 
     @specialize.arg(1)
     def get_constant_string_spec(self, mode):
@@ -262,11 +267,11 @@
             return s1[start : start + length]
         return None
 
-    def string_copy_parts(self, optimizer, targetbox, offsetbox, mode):
-        lengthbox = self.getstrlen(optimizer, mode)
-        return copy_str_content(optimizer,
-                                self.vstr.force_box(), targetbox,
-                                self.vstart.force_box(), offsetbox,
+    def string_copy_parts(self, string_optimizer, targetbox, offsetbox, mode):
+        lengthbox = self.getstrlen(string_optimizer, mode)
+        return copy_str_content(string_optimizer,
+                                self.vstr.force_box(string_optimizer), targetbox,
+                                self.vstart.force_box(string_optimizer), offsetbox,
                                 lengthbox, mode)
 
     def get_args_for_fail(self, modifier):
@@ -295,8 +300,8 @@
         return modifier.make_vstrslice(self.mode is mode_unicode)
 
 
-def copy_str_content(optimizer, srcbox, targetbox,
-                     srcoffsetbox, offsetbox, lengthbox, mode):
+def copy_str_content(string_optimizer, srcbox, targetbox,
+                     srcoffsetbox, offsetbox, lengthbox, mode, need_next_offset=True):
     if isinstance(srcbox, ConstPtr) and isinstance(srcoffsetbox, Const):
         M = 5
     else:
@@ -305,23 +310,26 @@
         # up to M characters are done "inline", i.e. with STRGETITEM/STRSETITEM
         # instead of just a COPYSTRCONTENT.
         for i in range(lengthbox.value):
-            charbox = _strgetitem(optimizer, srcbox, srcoffsetbox, mode)
-            srcoffsetbox = _int_add(optimizer, srcoffsetbox, CONST_1)
-            optimizer.emit_operation(ResOperation(mode.STRSETITEM, [targetbox,
-                                                                offsetbox,
-                                                                charbox],
+            charbox = _strgetitem(string_optimizer, srcbox, srcoffsetbox, mode)
+            srcoffsetbox = _int_add(string_optimizer, srcoffsetbox, CONST_1)
+            string_optimizer.emit_operation(ResOperation(mode.STRSETITEM, [targetbox,
+                                                                           offsetbox,
+                                                                           charbox],
                                               None))
-            offsetbox = _int_add(optimizer, offsetbox, CONST_1)
+            offsetbox = _int_add(string_optimizer, offsetbox, CONST_1)
     else:
-        nextoffsetbox = _int_add(optimizer, offsetbox, lengthbox)
+        if need_next_offset:
+            nextoffsetbox = _int_add(string_optimizer, offsetbox, lengthbox)
+        else:
+            nextoffsetbox = None
         op = ResOperation(mode.COPYSTRCONTENT, [srcbox, targetbox,
                                                 srcoffsetbox, offsetbox,
                                                 lengthbox], None)
-        optimizer.emit_operation(op)
+        string_optimizer.emit_operation(op)
         offsetbox = nextoffsetbox
     return offsetbox
 
-def _int_add(optimizer, box1, box2):
+def _int_add(string_optimizer, box1, box2):
     if isinstance(box1, ConstInt):
         if box1.value == 0:
             return box2
@@ -329,23 +337,23 @@
             return ConstInt(box1.value + box2.value)
     elif isinstance(box2, ConstInt) and box2.value == 0:
         return box1
-    if optimizer is None:
+    if string_optimizer is None:
         return None
     resbox = BoxInt()
-    optimizer.propagate_forward(ResOperation(rop.INT_ADD, [box1, box2], resbox))
+    string_optimizer.emit_operation(ResOperation(rop.INT_ADD, [box1, box2], resbox))
     return resbox
 
-def _int_sub(optimizer, box1, box2):
+def _int_sub(string_optimizer, box1, box2):
     if isinstance(box2, ConstInt):
         if box2.value == 0:
             return box1
         if isinstance(box1, ConstInt):
             return ConstInt(box1.value - box2.value)
     resbox = BoxInt()
-    optimizer.propagate_forward(ResOperation(rop.INT_SUB, [box1, box2], resbox))
+    string_optimizer.emit_operation(ResOperation(rop.INT_SUB, [box1, box2], resbox))
     return resbox
 
-def _strgetitem(optimizer, strbox, indexbox, mode):
+def _strgetitem(string_optimizer, strbox, indexbox, mode):
     if isinstance(strbox, ConstPtr) and isinstance(indexbox, ConstInt):
         if mode is mode_string:
             s = strbox.getref(lltype.Ptr(rstr.STR))
@@ -354,30 +362,28 @@
             s = strbox.getref(lltype.Ptr(rstr.UNICODE))
             return ConstInt(ord(s.chars[indexbox.getint()]))
     resbox = BoxInt()
-    optimizer.propagate_forward(ResOperation(mode.STRGETITEM, [strbox, indexbox],
-                                      resbox))
+    string_optimizer.emit_operation(ResOperation(mode.STRGETITEM, [strbox, indexbox],
+                                                 resbox))
     return resbox
 
 
 class OptString(optimizer.Optimization):
     "Handling of strings and unicodes."
-    enabled = True
-
     def new(self):
         return OptString()
-    
+
     def make_vstring_plain(self, box, source_op, mode):
-        vvalue = VStringPlainValue(self.optimizer, box, source_op, mode)
+        vvalue = VStringPlainValue(box, source_op, mode)
         self.make_equal_to(box, vvalue)
         return vvalue
 
     def make_vstring_concat(self, box, source_op, mode):
-        vvalue = VStringConcatValue(self.optimizer, box, source_op, mode)
+        vvalue = VStringConcatValue(box, source_op, mode)
         self.make_equal_to(box, vvalue)
         return vvalue
 
     def make_vstring_slice(self, box, source_op, mode):
-        vvalue = VStringSliceValue(self.optimizer, box, source_op, mode)
+        vvalue = VStringSliceValue(box, source_op, mode)
         self.make_equal_to(box, vvalue)
         return vvalue
 
@@ -427,17 +433,21 @@
         value.ensure_nonnull()
         #
         if value.is_virtual() and isinstance(value, VStringSliceValue):
-            fullindexbox = _int_add(self.optimizer,
-                                    value.vstart.force_box(),
-                                    vindex.force_box())
+            fullindexbox = _int_add(self,
+                                    value.vstart.force_box(self),
+                                    vindex.force_box(self))
             value = value.vstr
             vindex = self.getvalue(fullindexbox)
         #
         if isinstance(value, VStringPlainValue):  # even if no longer virtual
             if vindex.is_constant():
-                return value.getitem(vindex.box.getint())
+                res = value.getitem(vindex.box.getint())
+                # If it is uninitialized we can't return it, it was set by a
+                # COPYSTRCONTENT, not a STRSETITEM
+                if res is not optimizer.CVAL_UNINITIALIZED_ZERO:
+                    return res
         #
-        resbox = _strgetitem(self.optimizer, value.force_box(), vindex.force_box(), mode)
+        resbox = _strgetitem(self, value.force_box(self), vindex.force_box(self), mode)
         return self.getvalue(resbox)
 
     def optimize_STRLEN(self, op):
@@ -447,9 +457,33 @@
 
     def _optimize_STRLEN(self, op, mode):
         value = self.getvalue(op.getarg(0))
-        lengthbox = value.getstrlen(self.optimizer, mode)
+        lengthbox = value.getstrlen(self, mode)
         self.make_equal_to(op.result, self.getvalue(lengthbox))
 
+    def optimize_COPYSTRCONTENT(self, op):
+        self._optimize_COPYSTRCONTENT(op, mode_string)
+    def optimize_COPYUNICODECONTENT(self, op):
+        self._optimize_COPYSTRCONTENT(op, mode_unicode)
+
+    def _optimize_COPYSTRCONTENT(self, op, mode):
+        # args: src dst srcstart dststart length
+        src = self.getvalue(op.getarg(0))
+        dst = self.getvalue(op.getarg(1))
+        srcstart = self.getvalue(op.getarg(2))
+        dststart = self.getvalue(op.getarg(3))
+        length = self.getvalue(op.getarg(4))
+
+        if length.is_constant() and length.box.getint() == 0:
+            return
+        copy_str_content(self,
+            src.force_box(self),
+            dst.force_box(self),
+            srcstart.force_box(self),
+            dststart.force_box(self),
+            length.force_box(self),
+            mode, need_next_offset=False
+        )
+
     def optimize_CALL(self, op):
         # dispatch based on 'oopspecindex' to a method that handles
         # specifically the given oopspec call.  For non-oopspec calls,
@@ -472,6 +506,8 @@
                     return
         self.emit_operation(op)
 
+    optimize_CALL_PURE = optimize_CALL
+
     def opt_call_str_STR2UNICODE(self, op):
         # Constant-fold unicode("constant string").
         # More generally, supporting non-constant but virtual cases is
@@ -511,16 +547,16 @@
             return True
         #
         vstr.ensure_nonnull()
-        lengthbox = _int_sub(self.optimizer, vstop.force_box(),
-                                            vstart.force_box())
+        lengthbox = _int_sub(self, vstop.force_box(self),
+                                   vstart.force_box(self))
         #
         if isinstance(vstr, VStringSliceValue):
             # double slicing  s[i:j][k:l]
             vintermediate = vstr
             vstr = vintermediate.vstr
-            startbox = _int_add(self.optimizer,
-                                vintermediate.vstart.force_box(),
-                                vstart.force_box())
+            startbox = _int_add(self,
+                                vintermediate.vstart.force_box(self),
+                                vstart.force_box(self))
             vstart = self.getvalue(startbox)
         #
         value = self.make_vstring_slice(op.result, op, mode)
@@ -558,8 +594,8 @@
                 do = EffectInfo.OS_STREQ_LENGTHOK
             else:
                 do = EffectInfo.OS_STREQ_NONNULL
-            self.generate_modified_call(do, [v1.force_box(),
-                                             v2.force_box()], op.result, mode)
+            self.generate_modified_call(do, [v1.force_box(self),
+                                             v2.force_box(self)], op.result, mode)
             return True
         return False
 
@@ -567,7 +603,7 @@
         l2box = v2.getstrlen(None, mode)
         if isinstance(l2box, ConstInt):
             if l2box.value == 0:
-                lengthbox = v1.getstrlen(self.optimizer, mode)
+                lengthbox = v1.getstrlen(self, mode)
                 seo = self.optimizer.send_extra_operation
                 seo(ResOperation(rop.INT_EQ, [lengthbox, CONST_0], resultbox))
                 return True
@@ -578,17 +614,17 @@
                     vchar1 = self.strgetitem(v1, optimizer.CVAL_ZERO, mode)
                     vchar2 = self.strgetitem(v2, optimizer.CVAL_ZERO, mode)
                     seo = self.optimizer.send_extra_operation
-                    seo(ResOperation(rop.INT_EQ, [vchar1.force_box(),
-                                                  vchar2.force_box()],
+                    seo(ResOperation(rop.INT_EQ, [vchar1.force_box(self),
+                                                  vchar2.force_box(self)],
                                      resultbox))
                     return True
                 if isinstance(v1, VStringSliceValue):
                     vchar = self.strgetitem(v2, optimizer.CVAL_ZERO, mode)
                     do = EffectInfo.OS_STREQ_SLICE_CHAR
-                    self.generate_modified_call(do, [v1.vstr.force_box(),
-                                                     v1.vstart.force_box(),
-                                                     v1.vlength.force_box(),
-                                                     vchar.force_box()],
+                    self.generate_modified_call(do, [v1.vstr.force_box(self),
+                                                     v1.vstart.force_box(self),
+                                                     v1.vlength.force_box(self),
+                                                     vchar.force_box(self)],
                                                 resultbox, mode)
                     return True
         #
@@ -599,10 +635,10 @@
             if v1.is_null():
                 self.make_constant(resultbox, CONST_1)
                 return True
-            op = ResOperation(rop.PTR_EQ, [v1.force_box(),
+            op = ResOperation(rop.PTR_EQ, [v1.force_box(self),
                                            llhelper.CONST_NULL],
                               resultbox)
-            self.optimizer.emit_operation(op)
+            self.emit_operation(op)
             return True
         #
         return False
@@ -616,8 +652,8 @@
                     do = EffectInfo.OS_STREQ_NONNULL_CHAR
                 else:
                     do = EffectInfo.OS_STREQ_CHECKNULL_CHAR
-                self.generate_modified_call(do, [v1.force_box(),
-                                                 vchar.force_box()], resultbox,
+                self.generate_modified_call(do, [v1.force_box(self),
+                                                 vchar.force_box(self)], resultbox,
                                             mode)
                 return True
         #
@@ -626,10 +662,10 @@
                 do = EffectInfo.OS_STREQ_SLICE_NONNULL
             else:
                 do = EffectInfo.OS_STREQ_SLICE_CHECKNULL
-            self.generate_modified_call(do, [v1.vstr.force_box(),
-                                             v1.vstart.force_box(),
-                                             v1.vlength.force_box(),
-                                             v2.force_box()], resultbox, mode)
+            self.generate_modified_call(do, [v1.vstr.force_box(self),
+                                             v1.vstart.force_box(self),
+                                             v1.vlength.force_box(self),
+                                             v2.force_box(self)], resultbox, mode)
             return True
         return False
 
@@ -639,16 +675,11 @@
         calldescr, func = cic.callinfo_for_oopspec(oopspecindex)
         op = ResOperation(rop.CALL, [ConstInt(func)] + args, result,
                           descr=calldescr)
-        self.optimizer.emit_operation(op)
+        self.emit_operation(op)
 
     def propagate_forward(self, op):
-        if not self.enabled:
-            self.emit_operation(op)
-            return
-
         dispatch_opt(self, op)
 
-
 dispatch_opt = make_dispatcher_method(OptString, 'optimize_',
         default=OptString.emit_operation)
 
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -210,7 +210,8 @@
                 self.metainterp.clear_exception()
                 resbox = self.execute(rop.%s, b1, b2)
                 self.make_result_of_lastop(resbox)  # same as execute_varargs()
-                self.metainterp.handle_possible_overflow_error()
+                if not isinstance(resbox, Const):
+                    self.metainterp.handle_possible_overflow_error()
                 return resbox
         ''' % (_opimpl, _opimpl.upper())).compile()
 
@@ -401,23 +402,25 @@
         self.metainterp.heapcache.new_array(resbox, lengthbox)
         return resbox
 
-    @arguments("box", "descr", "box")
-    def _opimpl_getarrayitem_gc_any(self, arraybox, arraydescr, indexbox):
+    @specialize.arg(1)
+    def _do_getarrayitem_gc_any(self, op, arraybox, arraydescr, indexbox):
         tobox = self.metainterp.heapcache.getarrayitem(
                 arraybox, arraydescr, indexbox)
         if tobox:
             # sanity check: see whether the current array value
             # corresponds to what the cache thinks the value is
-            resbox = executor.execute(self.metainterp.cpu, self.metainterp,
-                                      rop.GETARRAYITEM_GC, arraydescr, arraybox, indexbox)
+            resbox = executor.execute(self.metainterp.cpu, self.metainterp, op,
+                                      arraydescr, arraybox, indexbox)
             assert resbox.constbox().same_constant(tobox.constbox())
             return tobox
-        resbox = self.execute_with_descr(rop.GETARRAYITEM_GC,
-                                         arraydescr, arraybox, indexbox)
+        resbox = self.execute_with_descr(op, arraydescr, arraybox, indexbox)
         self.metainterp.heapcache.getarrayitem_now_known(
                 arraybox, arraydescr, indexbox, resbox)
         return resbox
 
+    @arguments("box", "descr", "box")
+    def _opimpl_getarrayitem_gc_any(self, arraybox, arraydescr, indexbox):
+        return self._do_getarrayitem_gc_any(rop.GETARRAYITEM_GC, arraybox, arraydescr, indexbox)
 
     opimpl_getarrayitem_gc_i = _opimpl_getarrayitem_gc_any
     opimpl_getarrayitem_gc_r = _opimpl_getarrayitem_gc_any
@@ -433,8 +436,7 @@
 
     @arguments("box", "descr", "box")
     def _opimpl_getarrayitem_gc_pure_any(self, arraybox, arraydescr, indexbox):
-        return self.execute_with_descr(rop.GETARRAYITEM_GC_PURE,
-                                       arraydescr, arraybox, indexbox)
+        return self._do_getarrayitem_gc_any(rop.GETARRAYITEM_GC_PURE, arraybox, arraydescr, indexbox)
 
     opimpl_getarrayitem_gc_pure_i = _opimpl_getarrayitem_gc_pure_any
     opimpl_getarrayitem_gc_pure_r = _opimpl_getarrayitem_gc_pure_any
@@ -866,6 +868,14 @@
     def opimpl_newunicode(self, lengthbox):
         return self.execute(rop.NEWUNICODE, lengthbox)
 
+    @arguments("box", "box", "box", "box", "box")
+    def opimpl_copystrcontent(self, srcbox, dstbox, srcstartbox, dststartbox, lengthbox):
+        return self.execute(rop.COPYSTRCONTENT, srcbox, dstbox, srcstartbox, dststartbox, lengthbox)
+
+    @arguments("box", "box", "box", "box", "box")
+    def opimpl_copyunicodecontent(self, srcbox, dstbox, srcstartbox, dststartbox, lengthbox):
+        return self.execute(rop.COPYUNICODECONTENT, srcbox, dstbox, srcstartbox, dststartbox, lengthbox)
+
 ##    @FixME  #arguments("descr", "varargs")
 ##    def opimpl_residual_oosend_canraise(self, methdescr, varargs):
 ##        return self.execute_varargs(rop.OOSEND, varargs, descr=methdescr,
@@ -1060,6 +1070,18 @@
         return ConstInt(trace_length)
 
     @arguments("box")
+    def _opimpl_isconstant(self, box):
+        return ConstInt(isinstance(box, Const))
+
+    opimpl_int_isconstant = opimpl_ref_isconstant = _opimpl_isconstant
+
+    @arguments("box")
+    def _opimpl_isvirtual(self, box):
+        return ConstInt(self.metainterp.heapcache.is_unescaped(box))
+
+    opimpl_ref_isvirtual = _opimpl_isvirtual
+
+    @arguments("box")
     def opimpl_virtual_ref(self, box):
         # Details on the content of metainterp.virtualref_boxes:
         #
@@ -1673,6 +1695,10 @@
 
     def _record_helper_nonpure_varargs(self, opnum, resbox, descr, argboxes):
         assert resbox is None or isinstance(resbox, Box)
+        if (rop._OVF_FIRST <= opnum <= rop._OVF_LAST and
+            self.last_exc_value_box is None and
+            self._all_constants_varargs(argboxes)):
+            return resbox.constbox()
         # record the operation
         profiler = self.staticdata.profiler
         profiler.count_ops(opnum, RECORDED_OPS)
diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py
--- a/pypy/jit/metainterp/resoperation.py
+++ b/pypy/jit/metainterp/resoperation.py
@@ -221,7 +221,6 @@
         newop.setfailargs(self.getfailargs())
         return newop
 
-
 # ============
 # arity mixins
 # ============
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -1,23 +1,25 @@
+import sys
+
 import py
-import sys
-from pypy.rlib.jit import JitDriver, we_are_jitted, hint, dont_look_inside
-from pypy.rlib.jit import loop_invariant, elidable, promote
-from pypy.rlib.jit import jit_debug, assert_green, AssertGreenFailed
-from pypy.rlib.jit import unroll_safe, current_trace_length
+
+from pypy import conftest
+from pypy.jit.codewriter.policy import JitPolicy, StopAtXPolicy
 from pypy.jit.metainterp import pyjitpl, history
+from pypy.jit.metainterp.optimizeopt import ALL_OPTS_DICT
+from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin, noConst
+from pypy.jit.metainterp.typesystem import LLTypeHelper, OOTypeHelper
+from pypy.jit.metainterp.warmspot import get_stats
 from pypy.jit.metainterp.warmstate import set_future_value
-from pypy.jit.metainterp.warmspot import get_stats
-from pypy.jit.codewriter.policy import JitPolicy, StopAtXPolicy
-from pypy import conftest
+from pypy.rlib.jit import (JitDriver, we_are_jitted, hint, dont_look_inside,
+    loop_invariant, elidable, promote, jit_debug, assert_green,
+    AssertGreenFailed, unroll_safe, current_trace_length, look_inside_iff,
+    isconstant, isvirtual)
 from pypy.rlib.rarithmetic import ovfcheck
-from pypy.jit.metainterp.typesystem import LLTypeHelper, OOTypeHelper
 from pypy.rpython.lltypesystem import lltype, llmemory, rffi
 from pypy.rpython.ootypesystem import ootype
-from pypy.jit.metainterp.optimizeopt import ALL_OPTS_DICT
-from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin, noConst
+
 
 class BasicTests:
-
     def test_basic(self):
         def f(x, y):
             return x + y
@@ -99,14 +101,14 @@
                 myjitdriver.jit_merge_point(x=x, y=y, res=res)
                 res += x * x
                 x += 1
-                res += x * x                
+                res += x * x
                 y -= 1
             return res
         res = self.meta_interp(f, [6, 7])
         assert res == 1323
         self.check_loop_count(1)
         self.check_loops(int_mul=1)
-        
+
     def test_loop_variant_mul_ovf(self):
         myjitdriver = JitDriver(greens = [], reds = ['y', 'res', 'x'])
         def f(x, y):
@@ -1372,7 +1374,7 @@
             return x
         res = self.meta_interp(f, [299], listops=True)
         assert res == f(299)
-        self.check_loops(guard_class=0, guard_value=3)        
+        self.check_loops(guard_class=0, guard_value=3)
         self.check_loops(guard_class=0, guard_value=6, everywhere=True)
 
     def test_merge_guardnonnull_guardclass(self):
@@ -2118,7 +2120,7 @@
             return sa
         res = self.meta_interp(f, [32, 7])
         assert res == f(32, 7)
-        
+
     def test_caching_setarrayitem_fixed(self):
         myjitdriver = JitDriver(greens = [], reds = ['sa', 'i', 'n', 'a', 'node'])
         def f(n, a):
@@ -2138,7 +2140,7 @@
             return sa
         res = self.meta_interp(f, [32, 7])
         assert res == f(32, 7)
-        
+
     def test_caching_setarrayitem_var(self):
         myjitdriver = JitDriver(greens = [], reds = ['sa', 'i', 'n', 'a', 'b', 'node'])
         def f(n, a, b):
@@ -2668,7 +2670,7 @@
             myjitdriver.set_param('threshold', 3)
             myjitdriver.set_param('trace_eagerness', 1)
             myjitdriver.set_param('retrace_limit', 5)
-            myjitdriver.set_param('function_threshold', -1)            
+            myjitdriver.set_param('function_threshold', -1)
             pc = sa = i = 0
             while pc < len(bytecode):
                 myjitdriver.jit_merge_point(pc=pc, n=n, sa=sa, i=i)
@@ -2693,12 +2695,12 @@
         def g(n1, n2):
             for i in range(10):
                 f(n1)
-            for i in range(10):                
+            for i in range(10):
                 f(n2)
 
         nn = [10, 3]
         assert self.meta_interp(g, nn) == g(*nn)
-        
+
         # The attempts of retracing first loop will end up retracing the
         # second and thus fail 5 times, saturating the retrace_count. Instead a
         # bridge back to the preamble of the first loop is produced. A guard in
@@ -2709,7 +2711,7 @@
         self.check_tree_loop_count(2 + 3)
 
         # FIXME: Add a gloabl retrace counter and test that we are not trying more than 5 times.
-        
+
         def g(n):
             for i in range(n):
                 for j in range(10):
@@ -2945,15 +2947,27 @@
             a = [0, 1, 2, 3, 4]
             while i < n:
                 myjitdriver.jit_merge_point(sa=sa, n=n, a=a, i=i)
-                if i < n/2:
+                if i < n / 2:
                     sa += a[4]
-                elif i == n/2:
+                elif i == n / 2:
                     a.pop()
                 i += 1
         res = self.meta_interp(f, [32])
         assert res == f(32)
         self.check_loops(arraylen_gc=2)
-        
+
+    def test_ulonglong_mod(self):
+        myjitdriver = JitDriver(greens = [], reds = ['n', 'sa', 'i'])
+        def f(n):
+            sa = i = rffi.cast(rffi.ULONGLONG, 1)
+            while i < rffi.cast(rffi.ULONGLONG, n):
+                myjitdriver.jit_merge_point(sa=sa, n=n, i=i)
+                sa += sa % i
+                i += 1
+        res = self.meta_interp(f, [32])
+        assert res == f(32)
+
+
 class TestOOtype(BasicTests, OOJitMixin):
 
     def test_oohash(self):
@@ -3173,7 +3187,7 @@
         res = self.meta_interp(f, [32])
         assert res == f(32)
         self.check_tree_loop_count(3)
-        
+
     def test_two_loopinvariant_arrays3(self):
         from pypy.rpython.lltypesystem import lltype, llmemory, rffi
         myjitdriver = JitDriver(greens = [], reds = ['sa', 'n', 'i', 'a'])
@@ -3197,7 +3211,7 @@
         res = self.meta_interp(f, [32])
         assert res == f(32)
         self.check_tree_loop_count(2)
-        
+
     def test_two_loopinvariant_arrays_boxed(self):
         class A(object):
             def __init__(self, a):
@@ -3222,7 +3236,7 @@
         res = self.meta_interp(f, [32])
         assert res == f(32)
         self.check_loops(arraylen_gc=2, everywhere=True)
-        
+
     def test_release_gil_flush_heap_cache(self):
         if sys.platform == "win32":
             py.test.skip("needs 'time'")
@@ -3298,5 +3312,183 @@
 
         self.meta_interp(main, [10])
 
+    def test_look_inside_iff_const(self):
+        @look_inside_iff(lambda arg: isconstant(arg))
+        def f(arg):
+            s = 0
+            while arg > 0:
+                s += arg
+                arg -= 1
+            return s
+
+        driver = JitDriver(greens = ['code'], reds = ['n', 'arg', 's'])
+
+        def main(code, n, arg):
+            s = 0
+            while n > 0:
+                driver.jit_merge_point(code=code, n=n, arg=arg, s=s)
+                if code == 0:
+                    s += f(arg)
+                else:
+                    s += f(1)
+                n -= 1
+            return s
+
+        res = self.meta_interp(main, [0, 10, 2], enable_opts='')
+        assert res == main(0, 10, 2)
+        self.check_loops(call=1)
+        res = self.meta_interp(main, [1, 10, 2], enable_opts='')
+        assert res == main(1, 10, 2)
+        self.check_loops(call=0)
+
+    def test_look_inside_iff_virtual(self):
+        # There's no good reason for this to be look_inside_iff, but it's a test!
+        @look_inside_iff(lambda arg, n: isvirtual(arg))
+        def f(arg, n):
+            if n == 100:
+                for i in xrange(n):
+                    n += i
+            return arg.x
+        class A(object):
+            def __init__(self, x):
+                self.x = x
+        driver = JitDriver(greens=['n'], reds=['i', 'a'])
+        def main(n):
+            i = 0
+            a = A(3)
+            while i < 20:
+                driver.jit_merge_point(i=i, n=n, a=a)
+                if n == 0:
+                    i += f(a, n)
+                else:
+                    i += f(A(2), n)
+        res = self.meta_interp(main, [0], enable_opts='')
+        assert res == main(0)
+        self.check_loops(call=1, getfield_gc=0)
+        res = self.meta_interp(main, [1], enable_opts='')
+        assert res == main(1)
+        self.check_loops(call=0, getfield_gc=0)
+
+    def test_reuse_elidable_result(self):
+        driver = JitDriver(reds=['n', 's'], greens = [])
+        def main(n):
+            s = 0
+            while n > 0:
+                driver.jit_merge_point(s=s, n=n)
+                s += len(str(n)) + len(str(n))
+                n -= 1
+            return s
+        res = self.meta_interp(main, [10])
+        assert res == main(10)
+        self.check_loops({
+            'call': 1, 'guard_no_exception': 1, 'guard_true': 1, 'int_add': 2,
+            'int_gt': 1, 'int_sub': 1, 'strlen': 1, 'jump': 1,
+        })
+
+    def test_look_inside_iff_const_getarrayitem_gc_pure(self):
+        driver = JitDriver(greens=['unroll'], reds=['s', 'n'])
+
+        class A(object):
+            _immutable_fields_ = ["x[*]"]
+            def __init__(self, x):
+                self.x = [x]
+
+        @look_inside_iff(lambda x: isconstant(x))
+        def f(x):
+            i = 0
+            for c in x:
+                i += 1
+            return i
+
+        def main(unroll, n):
+            s = 0
+            while n > 0:
+                driver.jit_merge_point(s=s, n=n, unroll=unroll)
+                if unroll:
+                    x = A("xx")
+                else:
+                    x = A("x" * n)
+                s += f(x.x[0])
+                n -= 1
+            return s
+
+        res = self.meta_interp(main, [0, 10])
+        assert res == main(0, 10)
+        # 2 calls, one for f() and one for char_mul
+        self.check_loops(call=2)
+        res = self.meta_interp(main, [1, 10])
+        assert res == main(1, 10)
+        self.check_loops(call=0)
+
+    def test_setarrayitem_followed_by_arraycopy(self):
+        myjitdriver = JitDriver(greens = [], reds = ['n', 'sa', 'x', 'y'])
+        def f(n):
+            sa = 0
+            x = [1,2,n]
+            y = [1,2,3]
+            while n > 0:
+                myjitdriver.jit_merge_point(sa=sa, n=n, x=x, y=y)
+                y[0] = n
+                x[0:3] = y
+                sa += x[0]
+                n -= 1
+            return sa
+        res = self.meta_interp(f, [16])
+        assert res == f(16)
+        
+
+
 class TestLLtype(BaseLLtypeTests, LLJitMixin):
-    pass
+    def test_tagged(self):
+        py.test.skip("implement me")
+        from pypy.rlib.objectmodel import UnboxedValue
+        class Base(object):
+            __slots__ = ()
+
+        class Int(UnboxedValue, Base):
+            __slots__ = ["a"]
+
+            def is_pos(self):
+                return self.a > 0
+
+            def dec(self):
+                return Int(self.a - 1)
+
+
+        class Float(Base):
+            def __init__(self, a):
+                self.a = a
+
+            def is_pos(self):
+                return self.a > 0
+
+            def dec(self):
+                return Float(self.a - 1)
+
+        driver = JitDriver(greens=['pc', 's'], reds=['o'])
+
+        def main(fl, n, s):
+            if s:
+                s = "--j"
+            else:
+                s = "---j"
+            if fl:
+                o = Float(float(n))
+            else:
+                o = Int(n)
+            pc = 0
+            while True:
+                driver.jit_merge_point(s=s, pc=pc, o=o)
+                c = s[pc]
+                if c == "j":
+                    driver.can_enter_jit(s=s, pc=pc, o=o)
+                    if o.is_pos():
+                        pc = 0
+                        continue
+                    else:
+                        break
+                elif c == "-":
+                    o = o.dec()
+                pc += 1
+            return pc
+        res = self.meta_interp(main, [False, 100, True], taggedpointers=True)
diff --git a/pypy/jit/metainterp/test/test_dict.py b/pypy/jit/metainterp/test/test_dict.py
--- a/pypy/jit/metainterp/test/test_dict.py
+++ b/pypy/jit/metainterp/test/test_dict.py
@@ -153,11 +153,7 @@
 
         res = self.meta_interp(f, [100], listops=True)
         assert res == f(50)
-        # XXX: ideally there would be 7 calls here, but repeated CALL_PURE with
-        # the same arguments are not folded, because we have conflicting
-        # definitions of pure, once strhash can be appropriately folded
-        # this should be decreased to seven.
-        self.check_loops({"call": 8, "guard_false": 1, "guard_no_exception": 6,
+        self.check_loops({"call": 7, "guard_false": 1, "guard_no_exception": 6,
                           "guard_true": 1, "int_and": 1, "int_gt": 1,
                           "int_is_true": 1, "int_sub": 1, "jump": 1,
                           "new_with_vtable": 1, "setfield_gc": 1})
diff --git a/pypy/jit/metainterp/test/test_heapcache.py b/pypy/jit/metainterp/test/test_heapcache.py
--- a/pypy/jit/metainterp/test/test_heapcache.py
+++ b/pypy/jit/metainterp/test/test_heapcache.py
@@ -337,6 +337,24 @@
         h.invalidate_caches(rop.SETFIELD_GC, None, [box1, box2])
         assert not h.is_unescaped(box2)
 
+    def test_unescaped_testing(self):
+        h = HeapCache()
+        h.new(box1)
+        h.new(box2)
+        assert h.is_unescaped(box1)
+        assert h.is_unescaped(box2)
+        # Putting a virtual inside of another virtual doesn't escape it.
+        h.invalidate_caches(rop.SETFIELD_GC, None, [box1, box2])
+        assert h.is_unescaped(box2)
+        # Reading a field from a virtual doesn't escape it.
+        h.invalidate_caches(rop.GETFIELD_GC, None, [box1])
+        assert h.is_unescaped(box1)
+        # Escaping a virtual transitively escapes anything inside of it.
+        assert not h.is_unescaped(box3)
+        h.invalidate_caches(rop.SETFIELD_GC, None, [box3, box1])
+        assert not h.is_unescaped(box1)
+        assert not h.is_unescaped(box2)
+
     def test_unescaped_array(self):
         h = HeapCache()
         h.new_array(box1, lengthbox1)
diff --git a/pypy/jit/metainterp/test/test_list.py b/pypy/jit/metainterp/test/test_list.py
--- a/pypy/jit/metainterp/test/test_list.py
+++ b/pypy/jit/metainterp/test/test_list.py
@@ -34,7 +34,7 @@
                 l = [x + 1]
                 n -= 1
             return l[0]
-        
+
         res = self.meta_interp(f, [10], listops=True)
         assert res == f(10)
         self.check_all_virtualized()
@@ -60,7 +60,7 @@
 
     def test_ll_fixed_setitem_fast(self):
         jitdriver = JitDriver(greens = [], reds = ['n', 'l'])
-        
+
         def f(n):
             l = [1, 2, 3]
 
@@ -116,7 +116,7 @@
         assert res == f(10)
         py.test.skip("'[non-null] * n' gives a residual call so far")
         self.check_loops(setarrayitem_gc=0, getarrayitem_gc=0, call=0)
-    
+
     def test_arraycopy_simpleoptimize(self):
         def f():
             l = [1, 2, 3, 4]
@@ -208,6 +208,26 @@
         assert res == f(15)
         self.check_loops(guard_exception=0)
 
+    def test_virtual_resize(self):
+        jitdriver = JitDriver(greens = [], reds = ['n', 's'])
+        def f(n):
+            s = 0
+            while n > 0:
+                jitdriver.jit_merge_point(n=n, s=s)
+                lst = []
+                lst += [1]
+                n -= len(lst)
+                s += lst[0]
+                lst.pop()
+                lst.append(1)
+                s /= lst.pop()
+            return s
+        res = self.meta_interp(f, [15], listops=True)
+        assert res == f(15)
+        self.check_loops({"int_add": 1, "int_sub": 1, "int_gt": 1,
+                          "guard_true": 1, "jump": 1})
+
+
 class TestOOtype(ListTests, OOJitMixin):
     pass
 
@@ -236,8 +256,6 @@
             return a * b
         res = self.meta_interp(f, [37])
         assert res == f(37)
-        # There is the one actual field on a, plus 2 getfield's from the list
-        # itself, 1 to get the length (which is then incremented and passed to
-        # the resize func), and then a read of the items field to actually
-        # perform the setarrayitem on
-        self.check_loops(getfield_gc=5, everywhere=True)
+        # There is the one actual field on a, plus several fields on the list
+        # itself
+        self.check_loops(getfield_gc=10, everywhere=True)
diff --git a/pypy/jit/metainterp/test/test_longlong.py b/pypy/jit/metainterp/test/test_longlong.py
--- a/pypy/jit/metainterp/test/test_longlong.py
+++ b/pypy/jit/metainterp/test/test_longlong.py
@@ -118,6 +118,26 @@
         res = self.interp_operations(f, [1000000000])
         assert res == 123500000000.0
 
+    def test_floats_negative(self):
+        def f(i):
+            # i == 1000000000
+            f = i * -123.5
+            n = r_longlong(f)
+            compare(n, -29, 1054051584)
+            return float(n)
+        res = self.interp_operations(f, [1000000000])
+        assert res == -123500000000.0
+
+    def test_floats_ulonglong(self):
+        def f(i):
+            # i == 1000000000
+            f = i * 12350000000.0
+            n = r_ulonglong(f)
+            compare(n, -1419508847, 538116096)
+            return float(n)
+        res = self.interp_operations(f, [1000000000])
+        assert res == 12350000000000000000.0
+
     def test_unsigned_compare_ops(self):
         def f(n1, n2):
             # n == 30002000000000
diff --git a/pypy/jit/metainterp/test/test_resume.py b/pypy/jit/metainterp/test/test_resume.py
--- a/pypy/jit/metainterp/test/test_resume.py
+++ b/pypy/jit/metainterp/test/test_resume.py
@@ -576,8 +576,9 @@
 
 
 class FakeOptimizer_VirtualValue(object):
-    class cpu:
-        pass
+    class optimizer:
+        class cpu:
+            pass
 fakeoptimizer = FakeOptimizer_VirtualValue()
 
 def ConstAddr(addr, cpu):   # compatibility
@@ -1135,12 +1136,7 @@
     modifier.liveboxes = {}
     modifier.vfieldboxes = {}
 
-    class FakeOptimizer(object):
-        class cpu:
-            pass
-        def new_const_item(self, descr):
-            return None
-    v2 = VArrayValue(FakeOptimizer(), LLtypeMixin.arraydescr, 2, b2s)
+    v2 = VArrayValue(LLtypeMixin.arraydescr, None, 2, b2s)
     v2._items = [b4s, c1s]
     modifier.register_virtual_fields(b2s, [b4s, c1s])
     liveboxes = []
diff --git a/pypy/jit/metainterp/test/test_slist.py b/pypy/jit/metainterp/test/test_slist.py
--- a/pypy/jit/metainterp/test/test_slist.py
+++ b/pypy/jit/metainterp/test/test_slist.py
@@ -5,7 +5,6 @@
 class ListTests(object):
 
     def test_basic_list(self):
-        py.test.skip("not yet")
         myjitdriver = JitDriver(greens = [], reds = ['n', 'lst'])
         def f(n):
             lst = []
@@ -34,7 +33,7 @@
             return m
         res = self.interp_operations(f, [11], listops=True)
         assert res == 49
-        self.check_operations_history(call=5)
+        self.check_operations_history(call=3)
 
     def test_list_of_voids(self):
         myjitdriver = JitDriver(greens = [], reds = ['n', 'lst'])
@@ -93,7 +92,7 @@
             return x
         res = self.meta_interp(f, [-2], listops=True)
         assert res == 41
-        self.check_loops(call=1, guard_value=0)
+        self.check_loops(call=0, guard_value=0)
 
 # we don't support resizable lists on ootype
 #class TestOOtype(ListTests, OOJitMixin):
diff --git a/pypy/jit/metainterp/test/test_string.py b/pypy/jit/metainterp/test/test_string.py
--- a/pypy/jit/metainterp/test/test_string.py
+++ b/pypy/jit/metainterp/test/test_string.py
@@ -1,9 +1,11 @@
 import py
+
+from pypy.jit.codewriter.policy import StopAtXPolicy
+from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin
+from pypy.rlib.debug import debug_print
 from pypy.rlib.jit import JitDriver, dont_look_inside, we_are_jitted
-from pypy.rlib.debug import debug_print
-from pypy.jit.codewriter.policy import StopAtXPolicy
+from pypy.rlib.rstring import StringBuilder
 from pypy.rpython.ootypesystem import ootype
-from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin
 
 
 class StringTests:
@@ -27,7 +29,7 @@
             return i
         res = self.meta_interp(f, [10, True, _str('h')], listops=True)
         assert res == 5
-        self.check_loops(**{self.CALL: 1, self.CALL_PURE: 0})
+        self.check_loops(**{self.CALL: 1, self.CALL_PURE: 0, 'everywhere': True})
 
     def test_eq_folded(self):
         _str = self._str
@@ -327,7 +329,7 @@
     def test_str_slice_len_surviving(self):
         _str = self._str
         longstring = _str("Unrolling Trouble")
-        mydriver = JitDriver(reds = ['i', 'a', 'sa'], greens = []) 
+        mydriver = JitDriver(reds = ['i', 'a', 'sa'], greens = [])
         def f(a):
             i = sa = a
             while i < len(longstring):
@@ -343,7 +345,7 @@
         fillers = _str("abcdefghijklmnopqrstuvwxyz")
         data = _str("ABCDEFGHIJKLMNOPQRSTUVWXYZ")
 
-        mydriver = JitDriver(reds = ['line', 'noise', 'res'], greens = []) 
+        mydriver = JitDriver(reds = ['line', 'noise', 'res'], greens = [])
         def f():
             line = data
             noise = fillers
@@ -370,7 +372,7 @@
             def __init__(self, value):
                 self.value = value
         mydriver = JitDriver(reds = ['ratio', 'line', 'noise', 'res'],
-                             greens = []) 
+                             greens = [])
         def f():
             line = Str(data)
             noise = Str(fillers)
@@ -408,7 +410,7 @@
             return len(sa)
         assert self.meta_interp(f, [16]) == f(16)
 
-    def test_loop_invariant_string_slize(self):
+    def test_loop_invariant_string_slice(self):
         _str = self._str
         mydriver = JitDriver(reds = ['i', 'n', 'sa', 's', 's1'], greens = [])
         def f(n, c):
@@ -425,7 +427,7 @@
             return sa
         assert self.meta_interp(f, [16, 'a']) == f(16, 'a')
 
-    def test_loop_invariant_string_slize_boxed(self):
+    def test_loop_invariant_string_slice_boxed(self):
         class Str(object):
             def __init__(self, value):
                 self.value = value
@@ -445,7 +447,7 @@
             return sa
         assert self.meta_interp(f, [16, 'a']) == f(16, 'a')
 
-    def test_loop_invariant_string_slize_in_array(self):
+    def test_loop_invariant_string_slice_in_array(self):
         _str = self._str
         mydriver = JitDriver(reds = ['i', 'n', 'sa', 's', 's1'], greens = [])
         def f(n, c):
@@ -482,6 +484,30 @@
             return len(sa.val)
         assert self.meta_interp(f, ['a']) == f('a')
 
+    def test_string_comepare_quasiimmutable(self):
+        class Sys(object):
+            _immutable_fields_ = ["defaultencoding?"]
+            def __init__(self, s):
+                self.defaultencoding = s
+        _str = self._str
+        sys = Sys(_str('ascii'))        
+        mydriver = JitDriver(reds = ['n', 'sa'], greens = [])
+        def f(n):
+            sa = 0
+            sys.defaultencoding = _str('ascii')
+            while n:
+                mydriver.jit_merge_point(n=n, sa=sa)
+                if sys.defaultencoding == _str('ascii'):
+                    sa += 1
+                n -= 1
+            sys.defaultencoding = _str('utf-8')
+            return sa
+        assert self.meta_interp(f, [8]) == f(8)
+        self.check_loops({'int_add': 1, 'guard_true': 1, 'int_sub': 1,
+                          'jump': 1, 'int_is_true': 1,
+                          'guard_not_invalidated': 1})
+
+
 #class TestOOtype(StringTests, OOJitMixin):
 #    CALL = "oosend"
 #    CALL_PURE = "oosend_pure"
@@ -513,7 +539,7 @@
                 m -= 1
             return 42
         self.meta_interp(f, [6, 7])
-        self.check_loops(call=3,    # str(), _str(), escape()
+        self.check_loops(call=1,    # escape()
                          newunicode=1, unicodegetitem=0,
                          unicodesetitem=1, copyunicodecontent=1)
 
@@ -536,3 +562,55 @@
         self.check_loops(call_pure=0, call=1,
                          newunicode=0, unicodegetitem=0,
                          unicodesetitem=0, copyunicodecontent=0)
+
+    def test_join_chars(self):
+        jitdriver = JitDriver(reds=['a', 'b', 'c', 'i'], greens=[])
+        def f(a, b, c):
+            i = 0
+            while i < 10:
+                jitdriver.jit_merge_point(a=a, b=b, c=c, i=i)
+                x = []
+                if a:
+                    x.append("a")
+                if b:
+                    x.append("b")
+                if c:
+                    x.append("c")
+                i += len("".join(x))
+            return i
+        res = self.meta_interp(f, [1, 1, 1])
+        assert res == f(True, True, True)
+        # The "".join should be unrolled, since the length of x is known since
+        # it is virtual, ensure there are no calls to ll_join_chars, or
+        # allocations.
+        self.check_loops({
+            "guard_true": 5, "int_is_true": 3, "int_lt": 2, "int_add": 2, "jump": 2,
+        }, everywhere=True)
+
+    def test_virtual_copystringcontent(self):
+        jitdriver = JitDriver(reds=['n', 'result'], greens=[])
+        def main(n):
+            result = 0
+            while n >= 0:
+                jitdriver.jit_merge_point(n=n, result=result)
+                b = StringBuilder(6)
+                b.append("Hello!")
+                result += ord(b.build()[0])
+                n -= 1
+            return result
+        res = self.meta_interp(main, [9])
+        assert res == main(9)
+
+    def test_virtual_copystringcontent2(self):
+        jitdriver = JitDriver(reds=['n', 'result'], greens=[])
+        def main(n):
+            result = 0
+            while n >= 0:
+                jitdriver.jit_merge_point(n=n, result=result)
+                b = StringBuilder(6)
+                b.append("Hello!")
+                result += ord((b.build() + "xyz")[0])
+                n -= 1
+            return result
+        res = self.meta_interp(main, [9])
+        assert res == main(9)
diff --git a/pypy/jit/metainterp/test/test_tracingopts.py b/pypy/jit/metainterp/test/test_tracingopts.py
--- a/pypy/jit/metainterp/test/test_tracingopts.py
+++ b/pypy/jit/metainterp/test/test_tracingopts.py
@@ -1,7 +1,10 @@
+import sys
+
+from pypy.jit.metainterp.test.support import LLJitMixin
+from pypy.rlib import jit
+from pypy.rlib.rarithmetic import ovfcheck
+
 import py
-import sys
-from pypy.rlib import jit
-from pypy.jit.metainterp.test.support import LLJitMixin
 
 
 class TestLLtype(LLJitMixin):
@@ -573,3 +576,18 @@
         res = self.interp_operations(fn, [3])
         assert res == 24
         self.check_operations_history(getarrayitem_gc=0)
+
+    def test_fold_int_add_ovf(self):
+        def fn(n):
+            jit.promote(n)
+            try:
+                n = ovfcheck(n + 1)
+            except OverflowError:
+                return 12
+            else:
+                return n
+        res = self.interp_operations(fn, [3])
+        assert res == 4
+        self.check_operations_history(int_add_ovf=0)
+        res = self.interp_operations(fn, [sys.maxint])
+        assert res == 12
\ No newline at end of file
diff --git a/pypy/jit/metainterp/test/test_virtualstate.py b/pypy/jit/metainterp/test/test_virtualstate.py
--- a/pypy/jit/metainterp/test/test_virtualstate.py
+++ b/pypy/jit/metainterp/test/test_virtualstate.py
@@ -431,7 +431,7 @@
         
 
 class BaseTestBridges(BaseTest):
-    enable_opts = "intbounds:rewrite:virtualize:string:heap:unroll"
+    enable_opts = "intbounds:rewrite:virtualize:string:pure:heap:unroll"
 
     def _do_optimize_bridge(self, bridge, call_pure_results):
         from pypy.jit.metainterp.optimizeopt import optimize_bridge_1, build_opt_chain
diff --git a/pypy/jit/metainterp/warmspot.py b/pypy/jit/metainterp/warmspot.py
--- a/pypy/jit/metainterp/warmspot.py
+++ b/pypy/jit/metainterp/warmspot.py
@@ -53,6 +53,8 @@
         extraconfigopts = {'translation.list_comprehension_operations': True}
     else:
         extraconfigopts = {}
+    if kwds.pop("taggedpointers", False):
+        extraconfigopts["translation.taggedpointers"] = True
     interp, graph = get_interpreter(function, args,
                                     backendopt=False,  # will be done below
                                     type_system=type_system,
diff --git a/pypy/jit/tl/pypyjit.py b/pypy/jit/tl/pypyjit.py
--- a/pypy/jit/tl/pypyjit.py
+++ b/pypy/jit/tl/pypyjit.py
@@ -40,7 +40,7 @@
 config.objspace.usemodules.array = False
 config.objspace.usemodules._weakref = True
 config.objspace.usemodules._sre = False
-config.objspace.usemodules._lsprof = True
+config.objspace.usemodules._lsprof = False
 #
 config.objspace.usemodules._ffi = True
 config.objspace.usemodules.micronumpy = False
@@ -77,7 +77,7 @@
 
 def read_code():
     from pypy.module.marshal.interp_marshal import dumps
-    
+
     filename = 'pypyjit_demo.py'
     source = readfile(filename)
     ec = space.getexecutioncontext()
diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py
--- a/pypy/module/__builtin__/functional.py
+++ b/pypy/module/__builtin__/functional.py
@@ -3,13 +3,13 @@
 
 """
 
+from pypy.interpreter.baseobjspace import Wrappable
 from pypy.interpreter.error import OperationError
-from pypy.interpreter.gateway import NoneNotWrapped
-from pypy.interpreter.gateway import interp2app, unwrap_spec
+from pypy.interpreter.gateway import NoneNotWrapped, interp2app, unwrap_spec
 from pypy.interpreter.typedef import TypeDef
-from pypy.interpreter.baseobjspace import Wrappable
+from pypy.rlib import jit
+from pypy.rlib.objectmodel import specialize
 from pypy.rlib.rarithmetic import r_uint, intmask
-from pypy.rlib.objectmodel import specialize
 from pypy.rlib.rbigint import rbigint
 
 
@@ -134,29 +134,15 @@
 
 
 @specialize.arg(2)
+ at jit.look_inside_iff(lambda space, args, implementation_of:
+    jit.isconstant(len(args.arguments_w)) and
+    len(args.arguments_w) == 2
+)
 def min_max(space, args, implementation_of):
     if implementation_of == "max":
         compare = space.gt
     else:
         compare = space.lt
-
-    args_w = args.arguments_w
-    if len(args_w) == 2 and not args.keywords:
-        # simple case, suitable for the JIT
-        w_arg0, w_arg1 = args_w
-        if space.is_true(compare(w_arg0, w_arg1)):
-            return w_arg0
-        else:
-            return w_arg1
-    else:
-        return min_max_loop(space, args, implementation_of)
-
- at specialize.arg(2)
-def min_max_loop(space, args, implementation_of):
-    if implementation_of == "max":
-        compare = space.gt
-    else:
-        compare = space.lt
     args_w = args.arguments_w
     if len(args_w) > 1:
         w_sequence = space.newtuple(args_w)
@@ -207,9 +193,11 @@
     return min_max(space, __args__, "max")
 
 def min(space, __args__):
-    """Return the smallest item in a sequence.
+    """min(iterable[, key=func]) -> value
+    min(a, b, c, ...[, key=func]) -> value
 
-    If more than one argument is passed, return the minimum of them.
+    With a single iterable argument, return its smallest item.
+    With two or more arguments, return the smallest argument.
     """
     return min_max(space, __args__, "min")
 
diff --git a/pypy/module/__pypy__/interp_builders.py b/pypy/module/__pypy__/interp_builders.py
--- a/pypy/module/__pypy__/interp_builders.py
+++ b/pypy/module/__pypy__/interp_builders.py
@@ -16,7 +16,8 @@
 
         def _check_done(self, space):
             if self.builder is None:
-                raise OperationError(space.w_ValueError, space.wrap("Can't operate on a done builder"))
+                raise OperationError(space.w_ValueError, space.wrap(
+                        "Can't operate on a built builder"))
 
         @unwrap_spec(size=int)
         def descr__new__(space, w_subtype, size=-1):
@@ -31,7 +32,8 @@
         def descr_append_slice(self, space, s, start, end):
             self._check_done(space)
             if not 0 <= start <= end <= len(s):
-                raise OperationError(space.w_ValueError, space.wrap("bad start/stop"))
+                raise OperationError(space.w_ValueError, space.wrap(
+                        "bad start/stop"))
             self.builder.append_slice(s, start, end)
 
         def descr_build(self, space):
@@ -40,6 +42,12 @@
             self.builder = None
             return w_s
 
+        def descr_len(self, space):
+            if self.builder is None:
+                raise OperationError(space.w_ValueError, space.wrap(
+                        "no length of built builder"))
+            return space.wrap(self.builder.getlength())
+
     W_Builder.__name__ = "W_%s" % name
     W_Builder.typedef = TypeDef(name,
         __new__ = interp2app(func_with_new_name(
@@ -48,6 +56,7 @@
         append = interp2app(W_Builder.descr_append),
         append_slice = interp2app(W_Builder.descr_append_slice),
         build = interp2app(W_Builder.descr_build),
+        __len__ = interp2app(W_Builder.descr_len),
     )
     W_Builder.typedef.acceptable_as_base_class = False
     return W_Builder
diff --git a/pypy/module/__pypy__/test/test_builders.py b/pypy/module/__pypy__/test/test_builders.py
--- a/pypy/module/__pypy__/test/test_builders.py
+++ b/pypy/module/__pypy__/test/test_builders.py
@@ -38,7 +38,9 @@
         b = StringBuilder()
         b.append("abc")
         b.append("123")
+        assert len(b) == 6
         b.append("you and me")
         s = b.build()
+        raises(ValueError, len, b)
         assert s == "abc123you and me"
-        raises(ValueError, b.build)
\ No newline at end of file
+        raises(ValueError, b.build)
diff --git a/pypy/module/_continuation/__init__.py b/pypy/module/_continuation/__init__.py
--- a/pypy/module/_continuation/__init__.py
+++ b/pypy/module/_continuation/__init__.py
@@ -37,4 +37,5 @@
     interpleveldefs = {
         'continulet': 'interp_continuation.W_Continulet',
         'permute': 'interp_continuation.permute',
+        '_p': 'interp_continuation.unpickle',      # pickle support
     }
diff --git a/pypy/module/_continuation/interp_continuation.py b/pypy/module/_continuation/interp_continuation.py
--- a/pypy/module/_continuation/interp_continuation.py
+++ b/pypy/module/_continuation/interp_continuation.py
@@ -6,6 +6,7 @@
 from pypy.interpreter.typedef import TypeDef
 from pypy.interpreter.gateway import interp2app
 from pypy.interpreter.pycode import PyCode
+from pypy.interpreter.pyframe import PyFrame
 
 
 class W_Continulet(Wrappable):
@@ -23,24 +24,27 @@
         if ec.stacklet_thread is not self.sthread:
             global_state.clear()
             raise geterror(self.space, "inter-thread support is missing")
-        return ec
 
     def descr_init(self, w_callable, __args__):
         if self.sthread is not None:
             raise geterror(self.space, "continulet already __init__ialized")
+        #
+        # hackish: build the frame "by hand", passing it the correct arguments
+        space = self.space
+        w_args, w_kwds = __args__.topacked()
+        bottomframe = space.createframe(get_entrypoint_pycode(space),
+                                        get_w_module_dict(space), None)
+        bottomframe.locals_stack_w[0] = space.wrap(self)
+        bottomframe.locals_stack_w[1] = w_callable
+        bottomframe.locals_stack_w[2] = w_args
+        bottomframe.locals_stack_w[3] = w_kwds
+        self.bottomframe = bottomframe
+        #
         global_state.origin = self
-        global_state.w_callable = w_callable
-        global_state.args = __args__
-        self.bottomframe = make_fresh_frame(self.space)
-        self.sthread = build_sthread(self.space)
-        try:
-            self.h = self.sthread.new(new_stacklet_callback)
-            if self.sthread.is_empty_handle(self.h):    # early return
-                raise MemoryError
-        except MemoryError:
-            self.sthread = None
-            global_state.clear()
-            raise getmemoryerror(self.space)
+        sthread = build_sthread(self.space)
+        self.sthread = sthread
+        h = sthread.new(new_stacklet_callback)
+        post_switch(sthread, h)
 
     def switch(self, w_to):
         sthread = self.sthread
@@ -66,7 +70,7 @@
             if sthread.is_empty_handle(to.h):
                 global_state.clear()
                 raise geterror(self.space, "continulet already finished")
-        ec = self.check_sthread()
+        self.check_sthread()
         #
         global_state.origin = self
         if to is None:
@@ -76,13 +80,8 @@
             # double switch: the final destination is to.h
             global_state.destination = to
         #
-        try:
-            do_switch(sthread, global_state.destination.h)
-        except MemoryError:
-            global_state.clear()
-            raise getmemoryerror(self.space)
-        #
-        return get_result()
+        h = sthread.switch(global_state.destination.h)
+        return post_switch(sthread, h)
 
     def descr_switch(self, w_value=None, w_to=None):
         global_state.w_value = w_value
@@ -109,12 +108,26 @@
                  and not self.sthread.is_empty_handle(self.h))
         return self.space.newbool(valid)
 
+    def descr__reduce__(self):
+        from pypy.module._continuation import interp_pickle
+        return interp_pickle.reduce(self)
+
+    def descr__setstate__(self, w_args):
+        from pypy.module._continuation import interp_pickle
+        interp_pickle.setstate(self, w_args)
+
 
 def W_Continulet___new__(space, w_subtype, __args__):
     r = space.allocate_instance(W_Continulet, w_subtype)
     r.__init__(space)
     return space.wrap(r)
 
+def unpickle(space, w_subtype):
+    """Pickle support."""
+    r = space.allocate_instance(W_Continulet, w_subtype)
+    r.__init__(space)
+    return space.wrap(r)
+
 
 W_Continulet.typedef = TypeDef(
     'continulet',
@@ -124,9 +137,10 @@
     switch      = interp2app(W_Continulet.descr_switch),
     throw       = interp2app(W_Continulet.descr_throw),
     is_pending  = interp2app(W_Continulet.descr_is_pending),
+    __reduce__  = interp2app(W_Continulet.descr__reduce__),
+    __setstate__= interp2app(W_Continulet.descr__setstate__),
     )
 
-
 # ____________________________________________________________
 
 # Continulet objects maintain a dummy frame object in order to ensure
@@ -135,27 +149,40 @@
 
 class State:
     def __init__(self, space):
-        from pypy.interpreter.astcompiler.consts import CO_OPTIMIZED
-        self.space = space 
+        self.space = space
         w_module = space.getbuiltinmodule('_continuation')
         self.w_error = space.getattr(w_module, space.wrap('error'))
-        self.w_memoryerror = OperationError(space.w_MemoryError, space.w_None)
-        self.dummy_pycode = PyCode(space, 0, 0, 0, CO_OPTIMIZED,
-                                   '', [], [], [], '',
-                                   '<bottom of continulet>', 0, '', [], [],
-                                   hidden_applevel=True)
+        # the following function switches away immediately, so that
+        # continulet.__init__() doesn't immediately run func(), but it
+        # also has the hidden purpose of making sure we have a single
+        # bottomframe for the whole duration of the continulet's run.
+        # Hackish: only the func_code is used, and used in the context
+        # of w_globals == this module, so we can access the name
+        # 'continulet' directly.
+        w_code = space.appexec([], '''():
+            def start(c, func, args, kwds):
+                if continulet.switch(c) is not None:
+                    raise TypeError(
+                     "can\'t send non-None value to a just-started continulet")
+                return func(c, *args, **kwds)
+            return start.func_code
+        ''')
+        self.entrypoint_pycode = space.interp_w(PyCode, w_code)
+        self.entrypoint_pycode.hidden_applevel = True
+        self.w_unpickle = w_module.get('_p')
+        self.w_module_dict = w_module.getdict(space)
 
 def geterror(space, message):
     cs = space.fromcache(State)
     return OperationError(cs.w_error, space.wrap(message))
 
-def getmemoryerror(space):
+def get_entrypoint_pycode(space):
     cs = space.fromcache(State)
-    return cs.w_memoryerror
+    return cs.entrypoint_pycode
 
-def make_fresh_frame(space):
+def get_w_module_dict(space):
     cs = space.fromcache(State)
-    return space.FrameClass(space, cs.dummy_pycode, None, None)
+    return cs.w_module_dict
 
 # ____________________________________________________________
 
@@ -166,6 +193,9 @@
         StackletThread.__init__(self, space.config)
         self.space = space
         self.ec = ec
+        # for unpickling
+        from pypy.rlib.rweakref import RWeakKeyDictionary
+        self.frame2continulet = RWeakKeyDictionary(PyFrame, W_Continulet)
 
 ExecutionContext.stacklet_thread = None
 
@@ -176,8 +206,6 @@
     def clear(self):
         self.origin = None
         self.destination = None
-        self.w_callable = None
-        self.args = None
         self.w_value = None
         self.propagate_exception = None
 global_state = GlobalState()
@@ -185,27 +213,13 @@
 
 
 def new_stacklet_callback(h, arg):
-    self       = global_state.origin
-    w_callable = global_state.w_callable
-    args       = global_state.args
+    self = global_state.origin
+    self.h = h
     global_state.clear()
-    try:
-        do_switch(self.sthread, h)
-    except MemoryError:
-        return h       # oups!  do an early return in this case
-    #
     space = self.space
     try:
-        assert self.sthread.ec.topframeref() is None
-        self.sthread.ec.topframeref = jit.non_virtual_ref(self.bottomframe)
-        if global_state.propagate_exception is not None:
-            raise global_state.propagate_exception  # just propagate it further
-        if global_state.w_value is not space.w_None:
-            raise OperationError(space.w_TypeError, space.wrap(
-                "can't send non-None value to a just-started continulet"))
-
-        args = args.prepend(self.space.wrap(self))
-        w_result = space.call_args(w_callable, args)
+        frame = self.bottomframe
+        w_result = frame.execute_frame()
     except Exception, e:
         global_state.propagate_exception = e
     else:
@@ -215,9 +229,7 @@
     global_state.destination = self
     return self.h
 
-
-def do_switch(sthread, h):
-    h = sthread.switch(h)
+def post_switch(sthread, h):
     origin = global_state.origin
     self = global_state.destination
     global_state.origin = None
@@ -228,6 +240,8 @@
     sthread.ec.topframeref = self.bottomframe.f_backref
     self.bottomframe.f_backref = origin.bottomframe.f_backref
     origin.bottomframe.f_backref = current
+    #
+    return get_result()
 
 def get_result():
     if global_state.propagate_exception:
diff --git a/pypy/module/_continuation/interp_pickle.py b/pypy/module/_continuation/interp_pickle.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_continuation/interp_pickle.py
@@ -0,0 +1,128 @@
+from pypy.tool import stdlib_opcode as pythonopcode
+from pypy.rlib import jit
+from pypy.interpreter.error import OperationError
+from pypy.interpreter.pyframe import PyFrame
+from pypy.module._continuation.interp_continuation import State, global_state
+from pypy.module._continuation.interp_continuation import build_sthread
+from pypy.module._continuation.interp_continuation import post_switch
+from pypy.module._continuation.interp_continuation import get_result, geterror
+
+
+def getunpickle(space):
+    cs = space.fromcache(State)
+    return cs.w_unpickle
+
+
+def reduce(self):
+    # xxx this is known to be not completely correct with respect
+    # to subclasses, e.g. no __slots__ support, no looking for a
+    # __getnewargs__ or __getstate__ defined in the subclass, etc.
+    # Doing the right thing looks involved, though...
+    space = self.space
+    if self.sthread is None:
+        w_frame = space.w_False
+    elif self.sthread.is_empty_handle(self.h):
+        w_frame = space.w_None
+    else:
+        w_frame = space.wrap(self.bottomframe)
+    w_continulet_type = space.type(space.wrap(self))
+    w_dict = self.getdict(space) or space.w_None
+    args = [getunpickle(space),
+            space.newtuple([w_continulet_type]),
+            space.newtuple([w_frame, w_dict]),
+            ]
+    return space.newtuple(args)
+
+def setstate(self, w_args):
+    space = self.space
+    if self.sthread is not None:
+        raise geterror(space, "continulet.__setstate__() on an already-"
+                              "initialized continulet")
+    w_frame, w_dict = space.fixedview(w_args, expected_length=2)
+    if not space.is_w(w_dict, space.w_None):
+        self.setdict(space, w_dict)
+    if space.is_w(w_frame, space.w_False):
+        return    # not initialized
+    sthread = build_sthread(self.space)
+    self.sthread = sthread
+    self.bottomframe = space.interp_w(PyFrame, w_frame, can_be_None=True)
+    #
+    global_state.origin = self
+    if self.bottomframe is not None:
+        sthread.frame2continulet.set(self.bottomframe, self)
+    self.h = sthread.new(resume_trampoline_callback)
+    get_result()    # propagate the eventual MemoryError
+
+# ____________________________________________________________
+
+def resume_trampoline_callback(h, arg):
+    self = global_state.origin
+    self.h = h
+    space = self.space
+    sthread = self.sthread
+    try:
+        global_state.clear()
+        if self.bottomframe is None:
+            w_result = space.w_None
+        else:
+            h = sthread.switch(self.h)
+            try:
+                w_result = post_switch(sthread, h)
+                operr = None
+            except OperationError, e:
+                w_result = None
+                operr = e
+            #
+            while True:
+                ec = sthread.ec
+                frame = ec.topframeref()
+                assert frame is not None     # XXX better error message
+                exit_continulet = sthread.frame2continulet.get(frame)
+                #
+                continue_after_call(frame)
+                #
+                # small hack: unlink frame out of the execution context,
+                # because execute_frame will add it there again
+                ec.topframeref = frame.f_backref
+                #
+                try:
+                    w_result = frame.execute_frame(w_result, operr)
+                    operr = None
+                except OperationError, e:
+                    w_result = None
+                    operr = e
+                if exit_continulet is not None:
+                    self = exit_continulet
+                    break
+            sthread.ec.topframeref = jit.vref_None
+            if operr:
+                raise operr
+    except Exception, e:
+        global_state.propagate_exception = e
+    else:
+        global_state.w_value = w_result
+    global_state.origin = self
+    global_state.destination = self
+    return self.h
+
+def continue_after_call(frame):
+    code = frame.pycode.co_code
+    instr = frame.last_instr
+    opcode = ord(code[instr])
+    map = pythonopcode.opmap
+    call_ops = [map['CALL_FUNCTION'], map['CALL_FUNCTION_KW'],
+                map['CALL_FUNCTION_VAR'], map['CALL_FUNCTION_VAR_KW'],
+                map['CALL_METHOD']]
+    assert opcode in call_ops   # XXX check better, and complain better
+    instr += 1
+    oparg = ord(code[instr]) | ord(code[instr + 1]) << 8
+    nargs = oparg & 0xff
+    nkwds = (oparg >> 8) & 0xff
+    if nkwds == 0:     # only positional arguments
+        # fast paths leaves things on the stack, pop them
+        if (frame.space.config.objspace.opcodes.CALL_METHOD and
+            opcode == map['CALL_METHOD']):
+            frame.dropvalues(nargs + 2)
+        elif opcode == map['CALL_FUNCTION']:
+            frame.dropvalues(nargs + 1)
+    frame.last_instr = instr + 1    # continue after the call
diff --git a/pypy/module/_continuation/test/test_stacklet.py b/pypy/module/_continuation/test/test_stacklet.py
--- a/pypy/module/_continuation/test/test_stacklet.py
+++ b/pypy/module/_continuation/test/test_stacklet.py
@@ -13,7 +13,7 @@
         from _continuation import continulet
         #
         def empty_callback(c):
-            pass
+            never_called
         #
         c = continulet(empty_callback)
         assert type(c) is continulet
@@ -36,7 +36,7 @@
         from _continuation import continulet, error
         #
         def empty_callback(c1):
-            pass
+            never_called
         #
         c = continulet(empty_callback)
         raises(error, c.__init__, empty_callback)
diff --git a/pypy/module/_continuation/test/test_zpickle.py b/pypy/module/_continuation/test/test_zpickle.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_continuation/test/test_zpickle.py
@@ -0,0 +1,262 @@
+from pypy.conftest import gettestobjspace
+
+
+class AppTestCopy:
+    def setup_class(cls):
+        cls.space = gettestobjspace(usemodules=('_continuation',),
+                                    CALL_METHOD=True)
+        cls.space.config.translation.continuation = True
+
+    def test_basic_setup(self):
+        from _continuation import continulet
+        lst = [4]
+        co = continulet(lst.append)
+        assert lst == [4]
+        res = co.switch()
+        assert res is None
+        assert lst == [4, co]
+
+    def test_copy_continulet_not_started(self):
+        from _continuation import continulet, error
+        import copy
+        lst = []
+        co = continulet(lst.append)
+        co2, lst2 = copy.deepcopy((co, lst))
+        #
+        assert lst == []
+        co.switch()
+        assert lst == [co]
+        #
+        assert lst2 == []
+        co2.switch()
+        assert lst2 == [co2]
+
+    def test_copy_continulet_not_started_multiple(self):
+        from _continuation import continulet, error
+        import copy
+        lst = []
+        co = continulet(lst.append)
+        co2, lst2 = copy.deepcopy((co, lst))
+        co3, lst3 = copy.deepcopy((co, lst))
+        co4, lst4 = copy.deepcopy((co, lst))
+        #
+        assert lst == []
+        co.switch()
+        assert lst == [co]
+        #
+        assert lst2 == []
+        co2.switch()
+        assert lst2 == [co2]
+        #
+        assert lst3 == []
+        co3.switch()
+        assert lst3 == [co3]
+        #
+        assert lst4 == []
+        co4.switch()
+        assert lst4 == [co4]
+
+    def test_copy_continulet_real(self):
+        import new, sys
+        mod = new.module('test_copy_continulet_real')
+        sys.modules['test_copy_continulet_real'] = mod
+        exec '''if 1:
+            from _continuation import continulet
+            import copy
+            def f(co, x):
+                co.switch(x + 1)
+                co.switch(x + 2)
+                return x + 3
+            co = continulet(f, 40)
+            res = co.switch()
+            assert res == 41
+            co2 = copy.deepcopy(co)
+            #
+            res = co2.switch()
+            assert res == 42
+            assert co2.is_pending()
+            res = co2.switch()
+            assert res == 43
+            assert not co2.is_pending()
+            #
+            res = co.switch()
+            assert res == 42
+            assert co.is_pending()
+            res = co.switch()
+            assert res == 43
+            assert not co.is_pending()
+        ''' in mod.__dict__
+
+    def test_copy_continulet_already_finished(self):
+        from _continuation import continulet, error
+        import copy
+        lst = []
+        co = continulet(lst.append)
+        co.switch()
+        co2 = copy.deepcopy(co)
+        assert not co.is_pending()
+        assert not co2.is_pending()
+        raises(error, co.__init__, lst.append)
+        raises(error, co2.__init__, lst.append)
+        raises(error, co.switch)
+        raises(error, co2.switch)
+
+
+class AppTestPickle:
+    version = 0
+
+    def setup_class(cls):
+        cls.space = gettestobjspace(usemodules=('_continuation',),
+                                    CALL_METHOD=True)
+        cls.space.appexec([], """():
+            global continulet, A, __name__
+
+            import sys
+            __name__ = 'test_pickle_continulet'
+            thismodule = type(sys)(__name__)
+            sys.modules[__name__] = thismodule
+
+            from _continuation import continulet
+            class A(continulet):
+                pass
+
+            thismodule.__dict__.update(globals())
+        """)
+        cls.w_version = cls.space.wrap(cls.version)
+
+    def test_pickle_continulet_empty(self):
+        from _continuation import continulet
+        lst = [4]
+        co = continulet.__new__(continulet)
+        import pickle
+        pckl = pickle.dumps(co, self.version)
+        print repr(pckl)
+        co2 = pickle.loads(pckl)
+        assert co2 is not co
+        assert not co.is_pending()
+        assert not co2.is_pending()
+        # the empty unpickled coroutine can still be used:
+        result = [5]
+        co2.__init__(result.append)
+        res = co2.switch()
+        assert res is None
+        assert result == [5, co2]
+
+    def test_pickle_continulet_empty_subclass(self):
+        from test_pickle_continulet import continulet, A
+        lst = [4]
+        co = continulet.__new__(A)
+        co.foo = 'bar'
+        co.bar = 'baz'
+        import pickle
+        pckl = pickle.dumps(co, self.version)
+        print repr(pckl)
+        co2 = pickle.loads(pckl)
+        assert co2 is not co
+        assert not co.is_pending()
+        assert not co2.is_pending()
+        assert type(co) is type(co2) is A
+        assert co.foo == co2.foo == 'bar'
+        assert co.bar == co2.bar == 'baz'
+        # the empty unpickled coroutine can still be used:
+        result = [5]
+        co2.__init__(result.append)
+        res = co2.switch()
+        assert res is None
+        assert result == [5, co2]
+
+    def test_pickle_continulet_not_started(self):
+        from _continuation import continulet, error
+        import pickle
+        lst = []
+        co = continulet(lst.append)
+        pckl = pickle.dumps((co, lst))
+        print pckl
+        del co, lst
+        for i in range(2):
+            print 'resume...'
+            co2, lst2 = pickle.loads(pckl)
+            assert lst2 == []
+            co2.switch()
+            assert lst2 == [co2]
+
+    def test_pickle_continulet_real(self):
+        import new, sys
+        mod = new.module('test_pickle_continulet_real')
+        sys.modules['test_pickle_continulet_real'] = mod
+        mod.version = self.version
+        exec '''if 1:
+            from _continuation import continulet
+            import pickle
+            def f(co, x):
+                co.switch(x + 1)
+                co.switch(x + 2)
+                return x + 3
+            co = continulet(f, 40)
+            res = co.switch()
+            assert res == 41
+            pckl = pickle.dumps(co, version)
+            print repr(pckl)
+            co2 = pickle.loads(pckl)
+            #
+            res = co2.switch()
+            assert res == 42
+            assert co2.is_pending()
+            res = co2.switch()
+            assert res == 43
+            assert not co2.is_pending()
+            #
+            res = co.switch()
+            assert res == 42
+            assert co.is_pending()
+            res = co.switch()
+            assert res == 43
+            assert not co.is_pending()
+        ''' in mod.__dict__
+
+    def test_pickle_continulet_real_subclass(self):
+        import new, sys
+        mod = new.module('test_pickle_continulet_real_subclass')
+        sys.modules['test_pickle_continulet_real_subclass'] = mod
+        mod.version = self.version
+        exec '''if 1:
+            from _continuation import continulet
+            import pickle
+            class A(continulet):
+                def __init__(self):
+                    crash
+            def f(co):
+                co.switch(co.x + 1)
+                co.switch(co.x + 2)
+                return co.x + 3
+            co = A.__new__(A)
+            continulet.__init__(co, f)
+            co.x = 40
+            res = co.switch()
+            assert res == 41
+            pckl = pickle.dumps(co, version)
+            print repr(pckl)
+            co2 = pickle.loads(pckl)
+            #
+            assert type(co2) is A
+            res = co2.switch()
+            assert res == 42
+            assert co2.is_pending()
+            res = co2.switch()
+            assert res == 43
+            assert not co2.is_pending()
+            #
+            res = co.switch()
+            assert res == 42
+            assert co.is_pending()
+            res = co.switch()
+            assert res == 43
+            assert not co.is_pending()
+        ''' in mod.__dict__
+
+
+class AppTestPickle_v1(AppTestPickle):
+    version = 1
+
+class AppTestPickle_v2(AppTestPickle):
+    version = 2
diff --git a/pypy/module/_file/test/test_file.py b/pypy/module/_file/test/test_file.py
--- a/pypy/module/_file/test/test_file.py
+++ b/pypy/module/_file/test/test_file.py
@@ -1,5 +1,5 @@
 from __future__ import with_statement
-import py
+import py, os, errno
 
 from pypy.conftest import gettestobjspace, option
 
@@ -257,6 +257,35 @@
             assert self.temppath in g.getvalue()
 
 
+class AppTestNonblocking(object):
+    def setup_class(cls):
+        from pypy.module._file.interp_file import W_File
+        
+        cls.old_read = os.read
+        state = [0]
+        def read(fd, n=None):
+            if fd != 42:
+                return cls.old_read(fd, n)
+            if state[0] == 0:
+                state[0] += 1
+                return "xyz"
+            if state[0] < 3:
+                state[0] += 1
+                raise OSError(errno.EAGAIN, "xyz")
+            return ''
+        os.read = read
+        stdin = W_File(cls.space)
+        stdin.file_fdopen(42, "r", 1)
+        stdin.name = '<stdin>'
+        cls.w_stream = stdin
+
+    def teardown_class(cls):
+        os.read = cls.old_read
+        
+    def test_nonblocking_file(self):
+        res = self.stream.read()
+        assert res == 'xyz'
+
 class AppTestConcurrency(object):
     # these tests only really make sense on top of a translated pypy-c,
     # because on top of py.py the inner calls to os.write() don't
diff --git a/pypy/module/_multiprocessing/interp_connection.py b/pypy/module/_multiprocessing/interp_connection.py
--- a/pypy/module/_multiprocessing/interp_connection.py
+++ b/pypy/module/_multiprocessing/interp_connection.py
@@ -225,7 +225,9 @@
             except OSError:
                 pass
 
-    def __init__(self, fd, flags):
+    def __init__(self, space, fd, flags):
+        if fd == self.INVALID_HANDLE_VALUE or fd < 0:
+            raise OperationError(space.w_IOError, space.wrap("invalid handle %d" % fd))
         W_BaseConnection.__init__(self, flags)
         self.fd = fd
 
@@ -234,7 +236,7 @@
         flags = (readable and READABLE) | (writable and WRITABLE)
 
         self = space.allocate_instance(W_FileConnection, w_subtype)
-        W_FileConnection.__init__(self, fd, flags)
+        W_FileConnection.__init__(self, space, fd, flags)
         return space.wrap(self)
 
     def fileno(self, space):
diff --git a/pypy/module/_multiprocessing/interp_semaphore.py b/pypy/module/_multiprocessing/interp_semaphore.py
--- a/pypy/module/_multiprocessing/interp_semaphore.py
+++ b/pypy/module/_multiprocessing/interp_semaphore.py
@@ -468,6 +468,9 @@
 
         self.count -= 1
 
+    def after_fork(self):
+        self.count = 0
+
     @unwrap_spec(kind=int, maxvalue=int)
     def rebuild(space, w_cls, w_handle, kind, maxvalue):
         self = space.allocate_instance(W_SemLock, w_cls)
@@ -512,6 +515,7 @@
     acquire = interp2app(W_SemLock.acquire),
     release = interp2app(W_SemLock.release),
     _rebuild = interp2app(W_SemLock.rebuild.im_func, as_classmethod=True),
+    _after_fork = interp2app(W_SemLock.after_fork),
     __enter__=interp2app(W_SemLock.enter),
     __exit__=interp2app(W_SemLock.exit),
     SEM_VALUE_MAX=SEM_VALUE_MAX,
diff --git a/pypy/module/_multiprocessing/test/test_connection.py b/pypy/module/_multiprocessing/test/test_connection.py
--- a/pypy/module/_multiprocessing/test/test_connection.py
+++ b/pypy/module/_multiprocessing/test/test_connection.py
@@ -145,3 +145,9 @@
             else:
                 c.close()
         space.delslice(w_connections, space.wrap(0), space.wrap(100))
+
+    def test_bad_fd(self):
+        import _multiprocessing
+
+        raises(IOError, _multiprocessing.Connection, -1)
+        raises(IOError, _multiprocessing.Connection, -15)
\ No newline at end of file
diff --git a/pypy/module/_multiprocessing/test/test_semaphore.py b/pypy/module/_multiprocessing/test/test_semaphore.py
--- a/pypy/module/_multiprocessing/test/test_semaphore.py
+++ b/pypy/module/_multiprocessing/test/test_semaphore.py
@@ -39,6 +39,10 @@
         sem.release()
         assert sem._count() == 0
 
+        sem.acquire()
+        sem._after_fork()
+        assert sem._count() == 0
+
     def test_recursive(self):
         from _multiprocessing import SemLock
         kind = self.RECURSIVE
diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py
--- a/pypy/module/_sre/interp_sre.py
+++ b/pypy/module/_sre/interp_sre.py
@@ -99,6 +99,7 @@
 # SRE_Pattern class
 
 class W_SRE_Pattern(Wrappable):
+    _immutable_fields_ = ["code", "flags"]
 
     def cannot_copy_w(self):
         space = self.space
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -57,6 +57,9 @@
         compile_extra=['-DPy_BUILD_CORE'],
         )
 
+class CConfig2:
+    _compilation_info_ = CConfig._compilation_info_
+
 class CConfig_constants:
     _compilation_info_ = CConfig._compilation_info_
 
@@ -300,9 +303,13 @@
         return unwrapper_raise # used in 'normal' RPython code.
     return decorate
 
-def cpython_struct(name, fields, forward=None):
+def cpython_struct(name, fields, forward=None, level=1):
     configname = name.replace(' ', '__')
-    setattr(CConfig, configname, rffi_platform.Struct(name, fields))
+    if level == 1:
+        config = CConfig
+    else:
+        config = CConfig2
+    setattr(config, configname, rffi_platform.Struct(name, fields))
     if forward is None:
         forward = lltype.ForwardReference()
     TYPES[configname] = forward
@@ -445,9 +452,10 @@

From noreply at buildbot.pypy.org  Tue Oct 25 11:07:43 2011
From: noreply at buildbot.pypy.org (bivab)
Date: Tue, 25 Oct 2011 11:07:43 +0200 (CEST)
Subject: [pypy-commit] pypy arm-backend-2: there was a word missing here
Message-ID: <20111025090743.0B8C5820C7@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: arm-backend-2
Changeset: r48410:beae95ba795c
Date: 2011-10-13 17:54 +0200
http://bitbucket.org/pypy/pypy/changeset/beae95ba795c/

Log:	there was a word missing here

diff --git a/pypy/jit/backend/arm/assembler.py b/pypy/jit/backend/arm/assembler.py
--- a/pypy/jit/backend/arm/assembler.py
+++ b/pypy/jit/backend/arm/assembler.py
@@ -740,7 +740,7 @@
         return l
 
     def _patch_sp_offset(self, pos, frame_depth):
-        cb = OverwritingBuilder(self.mc, pos, OverwritingBuilder.size_of_gen_load_int)
+        cb = OverwritingBuilder(self.mc, pos, OverwritingBuilder.size_of_gen_load_int+WORD)
         # Note: the frame_depth is one less than the value stored in the frame
         # manager
         if frame_depth == 1:

From noreply at buildbot.pypy.org  Tue Oct 25 11:07:44 2011
From: noreply at buildbot.pypy.org (bivab)
Date: Tue, 25 Oct 2011 11:07:44 +0200 (CEST)
Subject: [pypy-commit] pypy arm-backend-2: (arigo,
 bivab) refactor a bit and fix decode_inputargs when checking
 spilled floating point values
Message-ID: <20111025090744.35759820C7@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: arm-backend-2
Changeset: r48411:744854db4b1c
Date: 2011-10-19 10:51 +0200
http://bitbucket.org/pypy/pypy/changeset/744854db4b1c/

Log:	(arigo, bivab) refactor a bit and fix decode_inputargs when checking
	spilled floating point values

diff --git a/pypy/jit/backend/arm/assembler.py b/pypy/jit/backend/arm/assembler.py
--- a/pypy/jit/backend/arm/assembler.py
+++ b/pypy/jit/backend/arm/assembler.py
@@ -36,6 +36,7 @@
     """
     Encoding for locations in memory
     types:
+    \xED = FLOAT
     \xEE = REF
     \xEF = INT
     location:
@@ -98,6 +99,7 @@
         self._regalloc = None
         self.mc = None
         self.pending_guards = None
+        assert self.datablockwrapper is None
 
     def setup_once(self):
         # Addresses of functions called by new_xxx operations
@@ -267,16 +269,14 @@
         self.fail_force_index = frame_loc
         return descr
 
-    def decode_inputargs(self, enc, inputargs, regalloc):
+    def decode_inputargs(self, enc, regalloc):
         locs = []
         j = 0
-        for i in range(len(inputargs)):
+        while enc[j] != self.END_OF_LOCS:
             res = enc[j]
-            if res == self.END_OF_LOCS:
-                assert 0, 'reached end of encoded area'
-            while res == self.EMPTY_LOC:
+            if res == self.EMPTY_LOC:
                 j += 1
-                res = enc[j]
+                continue
 
             assert res in [self.FLOAT_TYPE, self.INT_TYPE, self.REF_TYPE], 'location type is not supported'
             res_type = res
@@ -286,10 +286,14 @@
                 # XXX decode imm if necessary
                 assert 0, 'Imm Locations are not supported'
             elif res == self.STACK_LOC:
-                if res_type == FLOAT:
-                    assert 0, 'float on stack'
+                if res_type == self.FLOAT_TYPE:
+                    t = FLOAT
+                elif res_type == self.INT_TYPE:
+                    t = INT
+                else:
+                    t = REF
                 stack_loc = decode32(enc, j+1)
-                loc = regalloc.frame_manager.frame_pos(stack_loc, INT)
+                loc = regalloc.frame_manager.frame_pos(stack_loc, t)
                 j += 4
             else: # REG_LOC
                 if res_type == self.FLOAT_TYPE:
@@ -665,7 +669,8 @@
 
         sp_patch_location = self._prepare_sp_patch_position()
         frame_depth = faildescr._arm_frame_depth
-        locs = self.decode_inputargs(enc, inputargs, regalloc)
+        locs = self.decode_inputargs(enc, regalloc)
+        assert len(inputargs) == len(locs)
         regalloc.update_bindings(locs, frame_depth, inputargs)
 
         self._walk_operations(operations, regalloc)
@@ -862,7 +867,7 @@
             self.mc.VLDR(loc.value, r.ip.value)
 
     def _mov_imm_to_loc(self, prev_loc, loc, cond=c.AL):
-        if not loc.is_reg() and not (loc.is_stack() and loc.type == INT):
+        if not loc.is_reg() and not (loc.is_stack() and loc.type != FLOAT):
             raise AssertionError("invalid target for move from imm value")
         if loc.is_reg():
             new_loc = loc

From noreply at buildbot.pypy.org  Tue Oct 25 11:07:45 2011
From: noreply at buildbot.pypy.org (bivab)
Date: Tue, 25 Oct 2011 11:07:45 +0200 (CEST)
Subject: [pypy-commit] pypy arm-backend-2: set the name of generated
 functions for floatint point operations
Message-ID: <20111025090745.5F8EC820C7@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: arm-backend-2
Changeset: r48412:411d754b81a2
Date: 2011-10-19 10:53 +0200
http://bitbucket.org/pypy/pypy/changeset/411d754b81a2/

Log:	set the name of generated functions for floatint point operations

diff --git a/pypy/jit/backend/arm/helper/regalloc.py b/pypy/jit/backend/arm/helper/regalloc.py
--- a/pypy/jit/backend/arm/helper/regalloc.py
+++ b/pypy/jit/backend/arm/helper/regalloc.py
@@ -1,7 +1,7 @@
 from pypy.jit.backend.arm import conditions as c
 from pypy.jit.backend.arm import registers as r
 from pypy.jit.backend.arm.codebuilder import AbstractARMv7Builder
-from pypy.jit.metainterp.history import ConstInt, BoxInt, Box
+from pypy.jit.metainterp.history import ConstInt, BoxInt, Box, FLOAT
 from pypy.jit.metainterp.history import ConstInt
 
 # XXX create a version that does not need a ConstInt
@@ -56,7 +56,7 @@
         f.__name__ = name
     return f
 
-def prepare_float_op(base=True, float_result=True):
+def prepare_float_op(name=None, base=True, float_result=True):
     def f(self, op, fcond):
         locs = []
         loc1, box1 = self._ensure_value_is_boxed(op.getarg(0))
@@ -66,13 +66,13 @@
             locs.append(loc2)
             self.possibly_free_var(box2)
         self.possibly_free_var(box1)
-        if float_result:
-            res  = self.vfprm.force_allocate_reg(op.result)
-        else:
-            res  = self.rm.force_allocate_reg(op.result)
+        res = self.force_allocate_reg(op.result)
+        assert float_result == (op.result.type == FLOAT)
         self.possibly_free_var(op.result)
         locs.append(res)
         return locs
+    if name:
+        f.__name__ = name
     return f
 
 def prepare_op_by_helper_call(name):
diff --git a/pypy/jit/backend/arm/regalloc.py b/pypy/jit/backend/arm/regalloc.py
--- a/pypy/jit/backend/arm/regalloc.py
+++ b/pypy/jit/backend/arm/regalloc.py
@@ -1072,18 +1072,18 @@
         return size, scale, ofs, ofs_length, ptr
 
 
-    prepare_op_float_add = prepare_float_op()
-    prepare_op_float_sub = prepare_float_op()
-    prepare_op_float_mul = prepare_float_op()
-    prepare_op_float_truediv = prepare_float_op()
-    prepare_op_float_lt = prepare_float_op(float_result=False)
-    prepare_op_float_le = prepare_float_op(float_result=False)
-    prepare_op_float_eq = prepare_float_op(float_result=False)
-    prepare_op_float_ne = prepare_float_op(float_result=False)
-    prepare_op_float_gt = prepare_float_op(float_result=False)
-    prepare_op_float_ge = prepare_float_op(float_result=False)
-    prepare_op_float_neg = prepare_float_op(base=False)
-    prepare_op_float_abs = prepare_float_op(base=False)
+    prepare_op_float_add = prepare_float_op(name='float_add')
+    prepare_op_float_sub = prepare_float_op(name='float_sub')
+    prepare_op_float_mul = prepare_float_op(name='float_mul')
+    prepare_op_float_truediv = prepare_float_op(name='float_truediv')
+    prepare_op_float_lt = prepare_float_op(float_result=False, name='float_lt')
+    prepare_op_float_le = prepare_float_op(float_result=False, name='float_le')
+    prepare_op_float_eq = prepare_float_op(float_result=False, name='float_eq')
+    prepare_op_float_ne = prepare_float_op(float_result=False, name='float_ne')
+    prepare_op_float_gt = prepare_float_op(float_result=False, name='float_gt')
+    prepare_op_float_ge = prepare_float_op(float_result=False, name='float_ge')
+    prepare_op_float_neg = prepare_float_op(base=False, name='float_neg')
+    prepare_op_float_abs = prepare_float_op(base=False, name='float_abs')
 
     def prepare_op_math_sqrt(self, op, fcond):
         loc, box = self._ensure_value_is_boxed(op.getarg(1))

From noreply at buildbot.pypy.org  Tue Oct 25 11:07:46 2011
From: noreply at buildbot.pypy.org (bivab)
Date: Tue, 25 Oct 2011 11:07:46 +0200 (CEST)
Subject: [pypy-commit] pypy arm-backend-2: update comment
Message-ID: <20111025090746.8C000820C7@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: arm-backend-2
Changeset: r48413:1aeb2c211a8a
Date: 2011-10-19 10:57 +0200
http://bitbucket.org/pypy/pypy/changeset/1aeb2c211a8a/

Log:	update comment

diff --git a/pypy/jit/backend/arm/opassembler.py b/pypy/jit/backend/arm/opassembler.py
--- a/pypy/jit/backend/arm/opassembler.py
+++ b/pypy/jit/backend/arm/opassembler.py
@@ -935,7 +935,7 @@
     # ../x86/assembler.py:668
     def redirect_call_assembler(self, oldlooptoken, newlooptoken):
         # we overwrite the instructions at the old _x86_direct_bootstrap_code
-        # to start with a JMP to the new _x86_direct_bootstrap_code.
+        # to start with a JMP to the new _arm_direct_bootstrap_code.
         # Ideally we should rather patch all existing CALLs, but well.
         oldadr = oldlooptoken._arm_direct_bootstrap_code
         target = newlooptoken._arm_direct_bootstrap_code

From noreply at buildbot.pypy.org  Tue Oct 25 11:07:49 2011
From: noreply at buildbot.pypy.org (bivab)
Date: Tue, 25 Oct 2011 11:07:49 +0200 (CEST)
Subject: [pypy-commit] pypy arm-backend-2: merge default
Message-ID: <20111025090749.A6CF0820C7@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: arm-backend-2
Changeset: r48414:310898482c7b
Date: 2011-10-24 09:53 +0200
http://bitbucket.org/pypy/pypy/changeset/310898482c7b/

Log:	merge default

diff --git a/lib-python/modified-2.7/json/encoder.py b/lib-python/modified-2.7/json/encoder.py
--- a/lib-python/modified-2.7/json/encoder.py
+++ b/lib-python/modified-2.7/json/encoder.py
@@ -2,14 +2,7 @@
 """
 import re
 
-try:
-    from _json import encode_basestring_ascii as c_encode_basestring_ascii
-except ImportError:
-    c_encode_basestring_ascii = None
-try:
-    from _json import make_encoder as c_make_encoder
-except ImportError:
-    c_make_encoder = None
+from __pypy__.builders import StringBuilder, UnicodeBuilder
 
 ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]')
 ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])')
@@ -24,8 +17,7 @@
     '\t': '\\t',
 }
 for i in range(0x20):
-    ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i))
-    #ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,))
+    ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,))
 
 # Assume this produces an infinity on all machines (probably not guaranteed)
 INFINITY = float('1e66666')
@@ -37,10 +29,9 @@
     """
     def replace(match):
         return ESCAPE_DCT[match.group(0)]
-    return '"' + ESCAPE.sub(replace, s) + '"'
+    return ESCAPE.sub(replace, s)
 
-
-def py_encode_basestring_ascii(s):
+def encode_basestring_ascii(s):
     """Return an ASCII-only JSON representation of a Python string
 
     """
@@ -53,20 +44,18 @@
         except KeyError:
             n = ord(s)
             if n < 0x10000:
-                return '\\u{0:04x}'.format(n)
-                #return '\\u%04x' % (n,)
+                return '\\u%04x' % (n,)
             else:
                 # surrogate pair
                 n -= 0x10000
                 s1 = 0xd800 | ((n >> 10) & 0x3ff)
                 s2 = 0xdc00 | (n & 0x3ff)
-                return '\\u{0:04x}\\u{1:04x}'.format(s1, s2)
-                #return '\\u%04x\\u%04x' % (s1, s2)
-    return '"' + str(ESCAPE_ASCII.sub(replace, s)) + '"'
-
-
-encode_basestring_ascii = (
-    c_encode_basestring_ascii or py_encode_basestring_ascii)
+                return '\\u%04x\\u%04x' % (s1, s2)
+    if ESCAPE_ASCII.search(s):
+        return str(ESCAPE_ASCII.sub(replace, s))
+    return s
+py_encode_basestring_ascii = lambda s: '"' + encode_basestring_ascii(s) + '"'
+c_encode_basestring_ascii = None
 
 class JSONEncoder(object):
     """Extensible JSON <http://json.org> encoder for Python data structures.
@@ -147,6 +136,17 @@
 
         self.skipkeys = skipkeys
         self.ensure_ascii = ensure_ascii
+        if ensure_ascii:
+            self.encoder = encode_basestring_ascii
+        else:
+            self.encoder = encode_basestring
+        if encoding != 'utf-8':
+            orig_encoder = self.encoder
+            def encoder(o):
+                if isinstance(o, str):
+                    o = o.decode(encoding)
+                return orig_encoder(o)
+            self.encoder = encoder
         self.check_circular = check_circular
         self.allow_nan = allow_nan
         self.sort_keys = sort_keys
@@ -184,24 +184,126 @@
         '{"foo": ["bar", "baz"]}'
 
         """
-        # This is for extremely simple cases and benchmarks.
+        if self.check_circular:
+            markers = {}
+        else:
+            markers = None
+        if self.ensure_ascii:
+            builder = StringBuilder()
+        else:
+            builder = UnicodeBuilder()
+        self._encode(o, markers, builder, 0)
+        return builder.build()
+
+    def _emit_indent(self, builder, _current_indent_level):
+        if self.indent is not None:
+            _current_indent_level += 1
+            newline_indent = '\n' + (' ' * (self.indent *
+                                            _current_indent_level))
+            separator = self.item_separator + newline_indent
+            builder.append(newline_indent)
+        else:
+            separator = self.item_separator
+        return separator, _current_indent_level
+
+    def _emit_unindent(self, builder, _current_indent_level):
+        if self.indent is not None:
+            builder.append('\n')
+            builder.append(' ' * (self.indent * (_current_indent_level - 1)))
+
+    def _encode(self, o, markers, builder, _current_indent_level):
         if isinstance(o, basestring):
-            if isinstance(o, str):
-                _encoding = self.encoding
-                if (_encoding is not None
-                        and not (_encoding == 'utf-8')):
-                    o = o.decode(_encoding)
-            if self.ensure_ascii:
-                return encode_basestring_ascii(o)
+            builder.append('"')
+            builder.append(self.encoder(o))
+            builder.append('"')
+        elif o is None:
+            builder.append('null')
+        elif o is True:
+            builder.append('true')
+        elif o is False:
+            builder.append('false')
+        elif isinstance(o, (int, long)):
+            builder.append(str(o))
+        elif isinstance(o, float):
+            builder.append(self._floatstr(o))
+        elif isinstance(o, (list, tuple)):
+            if not o:
+                builder.append('[]')
+                return
+            self._encode_list(o, markers, builder, _current_indent_level)
+        elif isinstance(o, dict):
+            if not o:
+                builder.append('{}')
+                return
+            self._encode_dict(o, markers, builder, _current_indent_level)
+        else:
+            self._mark_markers(markers, o)
+            res = self.default(o)
+            self._encode(res, markers, builder, _current_indent_level)
+            self._remove_markers(markers, o)
+            return res
+
+    def _encode_list(self, l, markers, builder, _current_indent_level):
+        self._mark_markers(markers, l)
+        builder.append('[')
+        first = True
+        separator, _current_indent_level = self._emit_indent(builder,
+                                                      _current_indent_level)
+        for elem in l:
+            if first:
+                first = False
             else:
-                return encode_basestring(o)
-        # This doesn't pass the iterator directly to ''.join() because the
-        # exceptions aren't as detailed.  The list call should be roughly
-        # equivalent to the PySequence_Fast that ''.join() would do.
-        chunks = self.iterencode(o, _one_shot=True)
-        if not isinstance(chunks, (list, tuple)):
-            chunks = list(chunks)
-        return ''.join(chunks)
+                builder.append(separator)
+            self._encode(elem, markers, builder, _current_indent_level)
+            del elem # XXX grumble
+        self._emit_unindent(builder, _current_indent_level)
+        builder.append(']')
+        self._remove_markers(markers, l)
+
+    def _encode_dict(self, d, markers, builder, _current_indent_level):
+        self._mark_markers(markers, d)
+        first = True
+        builder.append('{')
+        separator, _current_indent_level = self._emit_indent(builder,
+                                                         _current_indent_level)
+        if self.sort_keys:
+            items = sorted(d.items(), key=lambda kv: kv[0])
+        else:
+            items = d.iteritems()
+
+        for key, v in items:
+            if first:
+                first = False
+            else:
+                builder.append(separator)
+            if isinstance(key, basestring):
+                pass
+            # JavaScript is weakly typed for these, so it makes sense to
+            # also allow them.  Many encoders seem to do something like this.
+            elif isinstance(key, float):
+                key = self._floatstr(key)
+            elif key is True:
+                key = 'true'
+            elif key is False:
+                key = 'false'
+            elif key is None:
+                key = 'null'
+            elif isinstance(key, (int, long)):
+                key = str(key)
+            elif self.skipkeys:
+                continue
+            else:
+                raise TypeError("key " + repr(key) + " is not a string")
+            builder.append('"')
+            builder.append(self.encoder(key))
+            builder.append('"')
+            builder.append(self.key_separator)
+            self._encode(v, markers, builder, _current_indent_level)
+            del key
+            del v # XXX grumble
+        self._emit_unindent(builder, _current_indent_level)
+        builder.append('}')
+        self._remove_markers(markers, d)
 
     def iterencode(self, o, _one_shot=False):
         """Encode the given object and yield each string
@@ -217,86 +319,54 @@
             markers = {}
         else:
             markers = None
-        if self.ensure_ascii:
-            _encoder = encode_basestring_ascii
+        return self._iterencode(o, markers, 0)
+
+    def _floatstr(self, o):
+        # Check for specials.  Note that this type of test is processor
+        # and/or platform-specific, so do tests which don't depend on the
+        # internals.
+
+        if o != o:
+            text = 'NaN'
+        elif o == INFINITY:
+            text = 'Infinity'
+        elif o == -INFINITY:
+            text = '-Infinity'
         else:
-            _encoder = encode_basestring
-        if self.encoding != 'utf-8':
-            def _encoder(o, _orig_encoder=_encoder, _encoding=self.encoding):
-                if isinstance(o, str):
-                    o = o.decode(_encoding)
-                return _orig_encoder(o)
+            return FLOAT_REPR(o)
 
-        def floatstr(o, allow_nan=self.allow_nan,
-                _repr=FLOAT_REPR, _inf=INFINITY, _neginf=-INFINITY):
-            # Check for specials.  Note that this type of test is processor
-            # and/or platform-specific, so do tests which don't depend on the
-            # internals.
+        if not self.allow_nan:
+            raise ValueError(
+                "Out of range float values are not JSON compliant: " +
+                repr(o))
 
-            if o != o:
-                text = 'NaN'
-            elif o == _inf:
-                text = 'Infinity'
-            elif o == _neginf:
-                text = '-Infinity'
-            else:
-                return _repr(o)
+        return text
 
-            if not allow_nan:
-                raise ValueError(
-                    "Out of range float values are not JSON compliant: " +
-                    repr(o))
+    def _mark_markers(self, markers, o):
+        if markers is not None:
+            if id(o) in markers:
+                raise ValueError("Circular reference detected")
+            markers[id(o)] = None
 
-            return text
+    def _remove_markers(self, markers, o):
+        if markers is not None:
+            del markers[id(o)]
 
-
-        if (_one_shot and c_make_encoder is not None
-                and not self.indent and not self.sort_keys):
-            _iterencode = c_make_encoder(
-                markers, self.default, _encoder, self.indent,
-                self.key_separator, self.item_separator, self.sort_keys,
-                self.skipkeys, self.allow_nan)
-        else:
-            _iterencode = _make_iterencode(
-                markers, self.default, _encoder, self.indent, floatstr,
-                self.key_separator, self.item_separator, self.sort_keys,
-                self.skipkeys, _one_shot)
-        return _iterencode(o, 0)
-
-def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,
-        _key_separator, _item_separator, _sort_keys, _skipkeys, _one_shot,
-        ## HACK: hand-optimized bytecode; turn globals into locals
-        ValueError=ValueError,
-        basestring=basestring,
-        dict=dict,
-        float=float,
-        id=id,
-        int=int,
-        isinstance=isinstance,
-        list=list,
-        long=long,
-        str=str,
-        tuple=tuple,
-    ):
-
-    def _iterencode_list(lst, _current_indent_level):
+    def _iterencode_list(self, lst, markers, _current_indent_level):
         if not lst:
             yield '[]'
             return
-        if markers is not None:
-            markerid = id(lst)
-            if markerid in markers:
-                raise ValueError("Circular reference detected")
-            markers[markerid] = lst
+        self._mark_markers(markers, lst)
         buf = '['
-        if _indent is not None:
+        if self.indent is not None:
             _current_indent_level += 1
-            newline_indent = '\n' + (' ' * (_indent * _current_indent_level))
-            separator = _item_separator + newline_indent
+            newline_indent = '\n' + (' ' * (self.indent *
+                                            _current_indent_level))
+            separator = self.item_separator + newline_indent
             buf += newline_indent
         else:
             newline_indent = None
-            separator = _item_separator
+            separator = self.item_separator
         first = True
         for value in lst:
             if first:
@@ -304,7 +374,7 @@
             else:
                 buf = separator
             if isinstance(value, basestring):
-                yield buf + _encoder(value)
+                yield buf + '"' + self.encoder(value) + '"'
             elif value is None:
                 yield buf + 'null'
             elif value is True:
@@ -314,44 +384,43 @@
             elif isinstance(value, (int, long)):
                 yield buf + str(value)
             elif isinstance(value, float):
-                yield buf + _floatstr(value)
+                yield buf + self._floatstr(value)
             else:
                 yield buf
                 if isinstance(value, (list, tuple)):
-                    chunks = _iterencode_list(value, _current_indent_level)
+                    chunks = self._iterencode_list(value, markers,
+                                                   _current_indent_level)
                 elif isinstance(value, dict):
-                    chunks = _iterencode_dict(value, _current_indent_level)
+                    chunks = self._iterencode_dict(value, markers,
+                                                   _current_indent_level)
                 else:
-                    chunks = _iterencode(value, _current_indent_level)
+                    chunks = self._iterencode(value, markers,
+                                              _current_indent_level)
                 for chunk in chunks:
                     yield chunk
         if newline_indent is not None:
             _current_indent_level -= 1
-            yield '\n' + (' ' * (_indent * _current_indent_level))
+            yield '\n' + (' ' * (self.indent * _current_indent_level))
         yield ']'
-        if markers is not None:
-            del markers[markerid]
+        self._remove_markers(markers, lst)
 
-    def _iterencode_dict(dct, _current_indent_level):
+    def _iterencode_dict(self, dct, markers, _current_indent_level):
         if not dct:
             yield '{}'
             return
-        if markers is not None:
-            markerid = id(dct)
-            if markerid in markers:
-                raise ValueError("Circular reference detected")
-            markers[markerid] = dct
+        self._mark_markers(markers, dct)
         yield '{'
-        if _indent is not None:
+        if self.indent is not None:
             _current_indent_level += 1
-            newline_indent = '\n' + (' ' * (_indent * _current_indent_level))
-            item_separator = _item_separator + newline_indent
+            newline_indent = '\n' + (' ' * (self.indent *
+                                            _current_indent_level))
+            item_separator = self.item_separator + newline_indent
             yield newline_indent
         else:
             newline_indent = None
-            item_separator = _item_separator
+            item_separator = self.item_separator
         first = True
-        if _sort_keys:
+        if self.sort_keys:
             items = sorted(dct.items(), key=lambda kv: kv[0])
         else:
             items = dct.iteritems()
@@ -361,7 +430,7 @@
             # JavaScript is weakly typed for these, so it makes sense to
             # also allow them.  Many encoders seem to do something like this.
             elif isinstance(key, float):
-                key = _floatstr(key)
+                key = self._floatstr(key)
             elif key is True:
                 key = 'true'
             elif key is False:
@@ -370,7 +439,7 @@
                 key = 'null'
             elif isinstance(key, (int, long)):
                 key = str(key)
-            elif _skipkeys:
+            elif self.skipkeys:
                 continue
             else:
                 raise TypeError("key " + repr(key) + " is not a string")
@@ -378,10 +447,10 @@
                 first = False
             else:
                 yield item_separator
-            yield _encoder(key)
-            yield _key_separator
+            yield '"' + self.encoder(key) + '"'
+            yield self.key_separator
             if isinstance(value, basestring):
-                yield _encoder(value)
+                yield '"' + self.encoder(value) + '"'
             elif value is None:
                 yield 'null'
             elif value is True:
@@ -391,26 +460,28 @@
             elif isinstance(value, (int, long)):
                 yield str(value)
             elif isinstance(value, float):
-                yield _floatstr(value)
+                yield self._floatstr(value)
             else:
                 if isinstance(value, (list, tuple)):
-                    chunks = _iterencode_list(value, _current_indent_level)
+                    chunks = self._iterencode_list(value, markers,
+                                                   _current_indent_level)
                 elif isinstance(value, dict):
-                    chunks = _iterencode_dict(value, _current_indent_level)
+                    chunks = self._iterencode_dict(value, markers,
+                                                   _current_indent_level)
                 else:
-                    chunks = _iterencode(value, _current_indent_level)
+                    chunks = self._iterencode(value, markers,
+                                              _current_indent_level)
                 for chunk in chunks:
                     yield chunk
         if newline_indent is not None:
             _current_indent_level -= 1
-            yield '\n' + (' ' * (_indent * _current_indent_level))
+            yield '\n' + (' ' * (self.indent * _current_indent_level))
         yield '}'
-        if markers is not None:
-            del markers[markerid]
+        self._remove_markers(markers, dct)
 
-    def _iterencode(o, _current_indent_level):
+    def _iterencode(self, o, markers, _current_indent_level):
         if isinstance(o, basestring):
-            yield _encoder(o)
+            yield '"' + self.encoder(o) + '"'
         elif o is None:
             yield 'null'
         elif o is True:
@@ -420,23 +491,19 @@
         elif isinstance(o, (int, long)):
             yield str(o)
         elif isinstance(o, float):
-            yield _floatstr(o)
+            yield self._floatstr(o)
         elif isinstance(o, (list, tuple)):
-            for chunk in _iterencode_list(o, _current_indent_level):
+            for chunk in self._iterencode_list(o, markers,
+                                               _current_indent_level):
                 yield chunk
         elif isinstance(o, dict):
-            for chunk in _iterencode_dict(o, _current_indent_level):
+            for chunk in self._iterencode_dict(o, markers,
+                                               _current_indent_level):
                 yield chunk
         else:
-            if markers is not None:
-                markerid = id(o)
-                if markerid in markers:
-                    raise ValueError("Circular reference detected")
-                markers[markerid] = o
-            o = _default(o)
-            for chunk in _iterencode(o, _current_indent_level):
+            self._mark_markers(markers, o)
+            obj = self.default(o)
+            for chunk in self._iterencode(obj, markers,
+                                          _current_indent_level):
                 yield chunk
-            if markers is not None:
-                del markers[markerid]
-
-    return _iterencode
+            self._remove_markers(markers, o)
diff --git a/lib-python/modified-2.7/json/tests/test_unicode.py b/lib-python/modified-2.7/json/tests/test_unicode.py
--- a/lib-python/modified-2.7/json/tests/test_unicode.py
+++ b/lib-python/modified-2.7/json/tests/test_unicode.py
@@ -80,3 +80,9 @@
         self.assertEqual(type(json.loads(u'["a"]')[0]), unicode)
         # Issue 10038.
         self.assertEqual(type(json.loads('"foo"')), unicode)
+
+    def test_encode_not_utf_8(self):
+        self.assertEqual(json.dumps('\xb1\xe6', encoding='iso8859-2'),
+                         '"\\u0105\\u0107"')
+        self.assertEqual(json.dumps(['\xb1\xe6'], encoding='iso8859-2'),
+                         '["\\u0105\\u0107"]')
diff --git a/lib-python/modified-2.7/test/test_array.py b/lib-python/modified-2.7/test/test_array.py
--- a/lib-python/modified-2.7/test/test_array.py
+++ b/lib-python/modified-2.7/test/test_array.py
@@ -295,9 +295,10 @@
         )
 
         b = array.array(self.badtypecode())
-        self.assertRaises(TypeError, "a + b")
-
-        self.assertRaises(TypeError, "a + 'bad'")
+        with self.assertRaises(TypeError):
+            a + b
+        with self.assertRaises(TypeError):
+            a + 'bad'
 
     def test_iadd(self):
         a = array.array(self.typecode, self.example[::-1])
@@ -316,9 +317,10 @@
         )
 
         b = array.array(self.badtypecode())
-        self.assertRaises(TypeError, "a += b")
-
-        self.assertRaises(TypeError, "a += 'bad'")
+        with self.assertRaises(TypeError):
+            a += b
+        with self.assertRaises(TypeError):
+            a += 'bad'
 
     def test_mul(self):
         a = 5*array.array(self.typecode, self.example)
@@ -345,7 +347,8 @@
             array.array(self.typecode)
         )
 
-        self.assertRaises(TypeError, "a * 'bad'")
+        with self.assertRaises(TypeError):
+            a * 'bad'
 
     def test_imul(self):
         a = array.array(self.typecode, self.example)
@@ -374,7 +377,8 @@
         a *= -1
         self.assertEqual(a, array.array(self.typecode))
 
-        self.assertRaises(TypeError, "a *= 'bad'")
+        with self.assertRaises(TypeError):
+            a *= 'bad'
 
     def test_getitem(self):
         a = array.array(self.typecode, self.example)
diff --git a/lib_pypy/pyrepl/readline.py b/lib_pypy/pyrepl/readline.py
--- a/lib_pypy/pyrepl/readline.py
+++ b/lib_pypy/pyrepl/readline.py
@@ -395,9 +395,21 @@
     _wrapper.f_in = f_in
     _wrapper.f_out = f_out
 
-    if hasattr(sys, '__raw_input__'):    # PyPy
-        _old_raw_input = sys.__raw_input__
+    if '__pypy__' in sys.builtin_module_names:    # PyPy
+
+        def _old_raw_input(prompt=''):
+            # sys.__raw_input__() is only called when stdin and stdout are
+            # as expected and are ttys.  If it is the case, then get_reader()
+            # should not really fail in _wrapper.raw_input().  If it still
+            # does, then we will just cancel the redirection and call again
+            # the built-in raw_input().
+            try:
+                del sys.__raw_input__
+            except AttributeError:
+                pass
+            return raw_input(prompt)
         sys.__raw_input__ = _wrapper.raw_input
+
     else:
         # this is not really what readline.c does.  Better than nothing I guess
         import __builtin__
diff --git a/lib_pypy/resource.py b/lib_pypy/resource.py
--- a/lib_pypy/resource.py
+++ b/lib_pypy/resource.py
@@ -7,7 +7,7 @@
 
 from ctypes_support import standard_c_lib as libc
 from ctypes_support import get_errno
-from ctypes import Structure, c_int, c_long, byref, sizeof, POINTER
+from ctypes import Structure, c_int, c_long, byref, POINTER
 from errno import EINVAL, EPERM
 import _structseq
 
@@ -165,7 +165,6 @@
 
 @builtinify
 def getpagesize():
-    pagesize = 0
     if _getpagesize:
         return _getpagesize()
     else:
diff --git a/pypy/annotation/classdef.py b/pypy/annotation/classdef.py
--- a/pypy/annotation/classdef.py
+++ b/pypy/annotation/classdef.py
@@ -276,8 +276,8 @@
         # create the Attribute and do the generalization asked for
         newattr = Attribute(attr, self.bookkeeper)
         if s_value:
-            if newattr.name == 'intval' and getattr(s_value, 'unsigned', False):
-                import pdb; pdb.set_trace()
+            #if newattr.name == 'intval' and getattr(s_value, 'unsigned', False):
+            #    import pdb; pdb.set_trace()
             newattr.s_value = s_value
 
         # keep all subattributes' values
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -72,6 +72,7 @@
     del working_modules['fcntl']  # LOCK_NB not defined
     del working_modules["_minimal_curses"]
     del working_modules["termios"]
+    del working_modules["_multiprocessing"]   # depends on rctime
 
 
 
@@ -127,7 +128,7 @@
 
 pypy_optiondescription = OptionDescription("objspace", "Object Space Options", [
     ChoiceOption("name", "Object Space name",
-                 ["std", "flow", "thunk", "dump", "taint"],
+                 ["std", "flow", "thunk", "dump"],
                  "std",
                  cmdline='--objspace -o'),
 
diff --git a/pypy/doc/__pypy__-module.rst b/pypy/doc/__pypy__-module.rst
--- a/pypy/doc/__pypy__-module.rst
+++ b/pypy/doc/__pypy__-module.rst
@@ -37,29 +37,6 @@
 .. _`thunk object space docs`: objspace-proxies.html#thunk
 .. _`interface section of the thunk object space docs`: objspace-proxies.html#thunk-interface
 
-.. broken:
-
-    Taint Object Space Functionality
-    ================================
-
-    When the taint object space is used (choose with :config:`objspace.name`),
-    the following names are put into ``__pypy__``:
-
-     - ``taint``
-     - ``is_tainted``
-     - ``untaint``
-     - ``taint_atomic``
-     - ``_taint_debug``
-     - ``_taint_look``
-     - ``TaintError``
-
-    Those are all described in the `interface section of the taint object space
-    docs`_.
-
-    For more detailed explanations and examples see the `taint object space docs`_.
-
-    .. _`taint object space docs`: objspace-proxies.html#taint
-    .. _`interface section of the taint object space docs`: objspace-proxies.html#taint-interface
 
 Transparent Proxy Functionality
 ===============================
diff --git a/pypy/doc/config/objspace.name.txt b/pypy/doc/config/objspace.name.txt
--- a/pypy/doc/config/objspace.name.txt
+++ b/pypy/doc/config/objspace.name.txt
@@ -4,7 +4,6 @@
 for normal usage):
 
   * thunk_: The thunk object space adds lazy evaluation to PyPy.
-  * taint_: The taint object space adds soft security features.
   * dump_:  Using this object spaces results in the dumpimp of all operations
     to a log.
 
@@ -12,5 +11,4 @@
 .. _`Object Space Proxies`: ../objspace-proxies.html
 .. _`Standard Object Space`: ../objspace.html#standard-object-space
 .. _thunk: ../objspace-proxies.html#thunk
-.. _taint: ../objspace-proxies.html#taint
 .. _dump: ../objspace-proxies.html#dump
diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst
--- a/pypy/doc/index.rst
+++ b/pypy/doc/index.rst
@@ -309,7 +309,6 @@
 .. _`object space`: objspace.html
 .. _FlowObjSpace: objspace.html#the-flow-object-space 
 .. _`trace object space`: objspace.html#the-trace-object-space 
-.. _`taint object space`: objspace-proxies.html#taint
 .. _`thunk object space`: objspace-proxies.html#thunk
 .. _`transparent proxies`: objspace-proxies.html#tproxy
 .. _`Differences between PyPy and CPython`: cpython_differences.html
diff --git a/pypy/doc/objspace-proxies.rst b/pypy/doc/objspace-proxies.rst
--- a/pypy/doc/objspace-proxies.rst
+++ b/pypy/doc/objspace-proxies.rst
@@ -129,297 +129,6 @@
    function behaves lazily: all calls to it return a thunk object.
 
 
-.. broken right now:
-
-    .. _taint:
-
-    The Taint Object Space
-    ======================
-
-    Motivation
-    ----------
-
-    The Taint Object Space provides a form of security: "tainted objects",
-    inspired by various sources, see [D12.1]_ for a more detailed discussion. 
-
-    The basic idea of this kind of security is not to protect against
-    malicious code but to help with handling and boxing sensitive data. 
-    It covers two kinds of sensitive data: secret data which should not leak, 
-    and untrusted data coming from an external source and that must be 
-    validated before it is used.
-
-    The idea is that, considering a large application that handles these
-    kinds of sensitive data, there are typically only a small number of
-    places that need to explicitly manipulate that sensitive data; all the
-    other places merely pass it around, or do entirely unrelated things.
-
-    Nevertheless, if a large application needs to be reviewed for security,
-    it must be entirely carefully checked, because it is possible that a
-    bug at some apparently unrelated place could lead to a leak of sensitive
-    information in a way that an external attacker could exploit.  For
-    example, if any part of the application provides web services, an
-    attacker might be able to issue unexpected requests with a regular web
-    browser and deduce secret information from the details of the answers he
-    gets.  Another example is the common CGI attack where an attacker sends
-    malformed inputs and causes the CGI script to do unintended things.
-
-    An approach like that of the Taint Object Space allows the small parts
-    of the program that manipulate sensitive data to be explicitly marked.
-    The effect of this is that although these small parts still need a
-    careful security review, the rest of the application no longer does,
-    because even a bug would be unable to leak the information.
-
-    We have implemented a simple two-level model: objects are either
-    regular (untainted), or sensitive (tainted).  Objects are marked as
-    sensitive if they are secret or untrusted, and only declassified at
-    carefully-checked positions (e.g. where the secret data is needed, or
-    after the untrusted data has been fully validated).
-
-    It would be simple to extend the code for more fine-grained scales of
-    secrecy.  For example it is typical in the literature to consider
-    user-specified lattices of secrecy levels, corresponding to multiple
-    "owners" that cannot access data belonging to another "owner" unless
-    explicitly authorized to do so.
-
-    Tainting and untainting
-    -----------------------
-
-    Start a py.py with the Taint Object Space and try the following example::
-
-        $ py.py -o taint
-        >>>> from __pypy__ import taint
-        >>>> x = taint(6)
-
-        # x is hidden from now on.  We can pass it around and
-        # even operate on it, but not inspect it.  Taintness
-        # is propagated to operation results.
-
-        >>>> x
-        TaintError
-
-        >>>> if x > 5: y = 2   # see below
-        TaintError
-
-        >>>> y = x + 5         # ok
-        >>>> lst = [x, y]
-        >>>> z = lst.pop()
-        >>>> t = type(z)       # type() works too, tainted answer
-        >>>> t
-        TaintError
-        >>>> u = t is int      # even 'is' works
-        >>>> u
-        TaintError
-
-    Notice that using a tainted boolean like ``x > 5`` in an ``if``
-    statement is forbidden.  This is because knowing which path is followed
-    would give away a hint about ``x``; in the example above, if the
-    statement ``if x > 5: y = 2`` was allowed to run, we would know
-    something about the value of ``x`` by looking at the (untainted) value
-    in the variable ``y``.
-
-    Of course, there is a way to inspect tainted objects.  The basic way is
-    to explicitly "declassify" it with the ``untaint()`` function.  In an
-    application, the places that use ``untaint()`` are the places that need
-    careful security review.  To avoid unexpected objects showing up, the
-    ``untaint()`` function must be called with the exact type of the object
-    to declassify.  It will raise ``TaintError`` if the type doesn't match::
-
-        >>>> from __pypy__ import taint
-        >>>> untaint(int, x)
-        6
-        >>>> untaint(int, z)
-        11
-        >>>> untaint(bool, x > 5)
-        True
-        >>>> untaint(int, x > 5)
-        TaintError
-
-
-    Taint Bombs
-    -----------
-
-    In this area, a common problem is what to do about failing operations.
-    If an operation raises an exception when manipulating a tainted object,
-    then the very presence of the exception can leak information about the
-    tainted object itself.  Consider::
-
-        >>>> 5 / (x-6)
-
-    By checking if this raises ``ZeroDivisionError`` or not, we would know
-    if ``x`` was equal to 6 or not.  The solution to this problem in the
-    Taint Object Space is to introduce *Taint Bombs*.  They are a kind of
-    tainted object that doesn't contain a real object, but a pending
-    exception.  Taint Bombs are indistinguishable from normal tainted
-    objects to unprivileged code. See::
-
-        >>>> x = taint(6)
-        >>>> i = 5 / (x-6)     # no exception here
-        >>>> j = i + 1         # nor here
-        >>>> k = j + 5         # nor here
-        >>>> untaint(int, k)
-        TaintError
-
-    In the above example, all of ``i``, ``j`` and ``k`` contain a Taint
-    Bomb.  Trying to untaint it raises an exception - a generic
-    ``TaintError``.  What we win is that the exception gives little away,
-    and most importantly it occurs at the point where ``untaint()`` is
-    called, not where the operation failed.  This means that all calls to
-    ``untaint()`` - but not the rest of the code - must be carefully
-    reviewed for what occurs if they receive a Taint Bomb; they might catch
-    the ``TaintError`` and give the user a generic message that something
-    went wrong, if we are reasonably careful that the message or even its
-    presence doesn't give information away.  This might be a
-    problem by itself, but there is no satisfying general solution here:
-    it must be considered on a case-by-case basis.  Again, what the
-    Taint Object Space approach achieves is not solving these problems, but
-    localizing them to well-defined small parts of the application - namely,
-    around calls to ``untaint()``.
-
-    The ``TaintError`` exception deliberately does not include any
-    useful error messages, because they might give information away.
-    Of course, this makes debugging quite a bit harder; a difficult
-    problem to solve properly.  So far we have implemented a way to peek in a Taint
-    Box or Bomb, ``__pypy__._taint_look(x)``, and a "debug mode" that
-    prints the exception as soon as a Bomb is created - both write
-    information to the low-level stderr of the application, where we hope
-    that it is unlikely to be seen by anyone but the application
-    developer.
-
-
-    Taint Atomic functions
-    ----------------------
-
-    Occasionally, a more complicated computation must be performed on a
-    tainted object.  This requires first untainting the object, performing the
-    computations, and then carefully tainting the result again (including
-    hiding all exceptions into Bombs).
-
-    There is a built-in decorator that does this for you::
-
-        >>>> @__pypy__.taint_atomic
-        >>>> def myop(x, y):
-        ....     while x > 0:
-        ....         x -= y
-        ....     return x
-        ....
-        >>>> myop(42, 10)
-        -8
-        >>>> z = myop(taint(42), 10)
-        >>>> z
-        TaintError
-        >>>> untaint(int, z)
-        -8
-
-    The decorator makes a whole function behave like a built-in operation.
-    If no tainted argument is passed in, the function behaves normally.  But
-    if any of the arguments is tainted, it is automatically untainted - so
-    the function body always sees untainted arguments - and the eventual
-    result is tainted again (possibly in a Taint Bomb).
-
-    It is important for the function marked as ``taint_atomic`` to have no
-    visible side effects, as these could cause information leakage.
-    This is currently not enforced, which means that all ``taint_atomic``
-    functions have to be carefully reviewed for security (but not the
-    callers of ``taint_atomic`` functions).
-
-    A possible future extension would be to forbid side-effects on
-    non-tainted objects from all ``taint_atomic`` functions.
-
-    An example of usage: given a tainted object ``passwords_db`` that
-    references a database of passwords, we can write a function
-    that checks if a password is valid as follows::
-
-        @taint_atomic
-        def validate(passwords_db, username, password):
-            assert type(passwords_db) is PasswordDatabase
-            assert type(username) is str
-            assert type(password) is str
-            ...load username entry from passwords_db...
-            return expected_password == password
-
-    It returns a tainted boolean answer, or a Taint Bomb if something
-    went wrong.  A caller can do::
-
-        ok = validate(passwords_db, 'john', '1234')
-        ok = untaint(bool, ok)
-
-    This can give three outcomes: ``True``, ``False``, or a ``TaintError``
-    exception (with no information on it) if anything went wrong.  If even
-    this is considered giving too much information away, the ``False`` case
-    can be made indistinguishable from the ``TaintError`` case (simply by
-    raising an exception in ``validate()`` if the password is wrong).
-
-    In the above example, the security results achieved are the following:
-    as long as ``validate()`` does not leak information, no other part of
-    the code can obtain more information about a passwords database than a
-    Yes/No answer to a precise query.
-
-    A possible extension of the ``taint_atomic`` decorator would be to check
-    the argument types, as ``untaint()`` does, for the same reason: to
-    prevent bugs where a function like ``validate()`` above is accidentally
-    called with the wrong kind of tainted object, which would make it
-    misbehave.  For now, all ``taint_atomic`` functions should be
-    conservative and carefully check all assumptions on their input
-    arguments.
-
-
-    .. _`taint-interface`:
-
-    Interface
-    ---------
-
-    .. _`like a built-in operation`:
-
-    The basic rule of the Tainted Object Space is that it introduces two new
-    kinds of objects, Tainted Boxes and Tainted Bombs (which are not types
-    in the Python sense).  Each box internally contains a regular object;
-    each bomb internally contains an exception object.  An operation
-    involving Tainted Boxes is performed on the objects contained in the
-    boxes, and gives a Tainted Box or a Tainted Bomb as a result (such an
-    operation does not let an exception be raised).  An operation called
-    with a Tainted Bomb argument immediately returns the same Tainted Bomb.
-
-    In a PyPy running with (or translated with) the Taint Object Space,
-    the ``__pypy__`` module exposes the following interface:
-
-    * ``taint(obj)``
-
-        Return a new Tainted Box wrapping ``obj``.  Return ``obj`` itself
-        if it is already tainted (a Box or a Bomb).
-
-    * ``is_tainted(obj)``
-
-        Check if ``obj`` is tainted (a Box or a Bomb).
-
-    * ``untaint(type, obj)``
-
-        Untaints ``obj`` if it is tainted.  Raise ``TaintError`` if the type
-        of the untainted object is not exactly ``type``, or if ``obj`` is a
-        Bomb.
-
-    * ``taint_atomic(func)``
-
-        Return a wrapper function around the callable ``func``.  The wrapper
-        behaves `like a built-in operation`_ with respect to untainting the
-        arguments, tainting the result, and returning a Bomb.
-
-    * ``TaintError``
-
-        Exception.  On purpose, it provides no attribute or error message.
-
-    * ``_taint_debug(level)``
-
-        Set the debugging level to ``level`` (0=off).  At level 1 or above,
-        all Taint Bombs print a diagnostic message to stderr when they are
-        created.
-
-    * ``_taint_look(obj)``
-
-        For debugging purposes: prints (to stderr) the type and address of
-        the object in a Tainted Box, or prints the exception if ``obj`` is
-        a Taint Bomb.
-
-
 .. _dump:
 
 The Dump Object Space
diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -2,7 +2,7 @@
 from pypy.interpreter.baseobjspace import Wrappable
 from pypy.interpreter import typedef
 from pypy.interpreter.gateway import interp2app
-from pypy.interpreter.error import OperationError
+from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.tool.pairtype import extendabletype
 from pypy.tool.sourcetools import func_with_new_name
@@ -2925,14 +2925,13 @@
 def Module_get_body(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -2968,14 +2967,13 @@
 def Interactive_get_body(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3015,8 +3013,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     return space.wrap(w_self.body)
 
 def Expression_set_body(space, w_self, w_new_value):
@@ -3057,14 +3054,13 @@
 def Suite_get_body(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3104,8 +3100,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._lineno_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'lineno'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lineno')
     return space.wrap(w_self.lineno)
 
 def stmt_set_lineno(space, w_self, w_new_value):
@@ -3126,8 +3121,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._col_offset_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'col_offset'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'col_offset')
     return space.wrap(w_self.col_offset)
 
 def stmt_set_col_offset(space, w_self, w_new_value):
@@ -3157,8 +3151,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'name'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name')
     return space.wrap(w_self.name)
 
 def FunctionDef_set_name(space, w_self, w_new_value):
@@ -3179,8 +3172,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'args'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args')
     return space.wrap(w_self.args)
 
 def FunctionDef_set_args(space, w_self, w_new_value):
@@ -3197,14 +3189,13 @@
 def FunctionDef_get_body(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3215,14 +3206,13 @@
 def FunctionDef_get_decorator_list(space, w_self):
     if not w_self.initialization_state & 8:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'decorator_list'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'decorator_list')
     if w_self.w_decorator_list is None:
         if w_self.decorator_list is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.decorator_list]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_decorator_list = w_list
     return w_self.w_decorator_list
 
@@ -3266,8 +3256,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'name'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name')
     return space.wrap(w_self.name)
 
 def ClassDef_set_name(space, w_self, w_new_value):
@@ -3284,14 +3273,13 @@
 def ClassDef_get_bases(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'bases'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'bases')
     if w_self.w_bases is None:
         if w_self.bases is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.bases]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_bases = w_list
     return w_self.w_bases
 
@@ -3302,14 +3290,13 @@
 def ClassDef_get_body(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3320,14 +3307,13 @@
 def ClassDef_get_decorator_list(space, w_self):
     if not w_self.initialization_state & 8:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'decorator_list'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'decorator_list')
     if w_self.w_decorator_list is None:
         if w_self.decorator_list is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.decorator_list]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_decorator_list = w_list
     return w_self.w_decorator_list
 
@@ -3372,8 +3358,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Return_set_value(space, w_self, w_new_value):
@@ -3414,14 +3399,13 @@
 def Delete_get_targets(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'targets'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'targets')
     if w_self.w_targets is None:
         if w_self.targets is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.targets]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_targets = w_list
     return w_self.w_targets
 
@@ -3457,14 +3441,13 @@
 def Assign_get_targets(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'targets'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'targets')
     if w_self.w_targets is None:
         if w_self.targets is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.targets]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_targets = w_list
     return w_self.w_targets
 
@@ -3479,8 +3462,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Assign_set_value(space, w_self, w_new_value):
@@ -3527,8 +3509,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'target'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'target')
     return space.wrap(w_self.target)
 
 def AugAssign_set_target(space, w_self, w_new_value):
@@ -3549,8 +3530,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'op'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op')
     return operator_to_class[w_self.op - 1]()
 
 def AugAssign_set_op(space, w_self, w_new_value):
@@ -3573,8 +3553,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def AugAssign_set_value(space, w_self, w_new_value):
@@ -3621,8 +3600,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'dest'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'dest')
     return space.wrap(w_self.dest)
 
 def Print_set_dest(space, w_self, w_new_value):
@@ -3639,14 +3617,13 @@
 def Print_get_values(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'values'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'values')
     if w_self.w_values is None:
         if w_self.values is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.values]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_values = w_list
     return w_self.w_values
 
@@ -3661,8 +3638,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'nl'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'nl')
     return space.wrap(w_self.nl)
 
 def Print_set_nl(space, w_self, w_new_value):
@@ -3710,8 +3686,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'target'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'target')
     return space.wrap(w_self.target)
 
 def For_set_target(space, w_self, w_new_value):
@@ -3732,8 +3707,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'iter'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'iter')
     return space.wrap(w_self.iter)
 
 def For_set_iter(space, w_self, w_new_value):
@@ -3750,14 +3724,13 @@
 def For_get_body(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3768,14 +3741,13 @@
 def For_get_orelse(space, w_self):
     if not w_self.initialization_state & 8:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'orelse'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
     if w_self.w_orelse is None:
         if w_self.orelse is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.orelse]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_orelse = w_list
     return w_self.w_orelse
 
@@ -3819,8 +3791,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'test'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test')
     return space.wrap(w_self.test)
 
 def While_set_test(space, w_self, w_new_value):
@@ -3837,14 +3808,13 @@
 def While_get_body(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3855,14 +3825,13 @@
 def While_get_orelse(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'orelse'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
     if w_self.w_orelse is None:
         if w_self.orelse is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.orelse]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_orelse = w_list
     return w_self.w_orelse
 
@@ -3905,8 +3874,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'test'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test')
     return space.wrap(w_self.test)
 
 def If_set_test(space, w_self, w_new_value):
@@ -3923,14 +3891,13 @@
 def If_get_body(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3941,14 +3908,13 @@
 def If_get_orelse(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'orelse'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
     if w_self.w_orelse is None:
         if w_self.orelse is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.orelse]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_orelse = w_list
     return w_self.w_orelse
 
@@ -3991,8 +3957,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'context_expr'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'context_expr')
     return space.wrap(w_self.context_expr)
 
 def With_set_context_expr(space, w_self, w_new_value):
@@ -4013,8 +3978,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'optional_vars'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'optional_vars')
     return space.wrap(w_self.optional_vars)
 
 def With_set_optional_vars(space, w_self, w_new_value):
@@ -4031,14 +3995,13 @@
 def With_get_body(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -4080,8 +4043,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'type'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'type')
     return space.wrap(w_self.type)
 
 def Raise_set_type(space, w_self, w_new_value):
@@ -4102,8 +4064,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'inst'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'inst')
     return space.wrap(w_self.inst)
 
 def Raise_set_inst(space, w_self, w_new_value):
@@ -4124,8 +4085,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'tback'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'tback')
     return space.wrap(w_self.tback)
 
 def Raise_set_tback(space, w_self, w_new_value):
@@ -4168,14 +4128,13 @@
 def TryExcept_get_body(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -4186,14 +4145,13 @@
 def TryExcept_get_handlers(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'handlers'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'handlers')
     if w_self.w_handlers is None:
         if w_self.handlers is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.handlers]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_handlers = w_list
     return w_self.w_handlers
 
@@ -4204,14 +4162,13 @@
 def TryExcept_get_orelse(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'orelse'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
     if w_self.w_orelse is None:
         if w_self.orelse is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.orelse]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_orelse = w_list
     return w_self.w_orelse
 
@@ -4251,14 +4208,13 @@
 def TryFinally_get_body(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -4269,14 +4225,13 @@
 def TryFinally_get_finalbody(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'finalbody'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'finalbody')
     if w_self.w_finalbody is None:
         if w_self.finalbody is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.finalbody]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_finalbody = w_list
     return w_self.w_finalbody
 
@@ -4318,8 +4273,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'test'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test')
     return space.wrap(w_self.test)
 
 def Assert_set_test(space, w_self, w_new_value):
@@ -4340,8 +4294,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'msg'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'msg')
     return space.wrap(w_self.msg)
 
 def Assert_set_msg(space, w_self, w_new_value):
@@ -4383,14 +4336,13 @@
 def Import_get_names(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'names'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'names')
     if w_self.w_names is None:
         if w_self.names is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.names]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_names = w_list
     return w_self.w_names
 
@@ -4430,8 +4382,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'module'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'module')
     return space.wrap(w_self.module)
 
 def ImportFrom_set_module(space, w_self, w_new_value):
@@ -4451,14 +4402,13 @@
 def ImportFrom_get_names(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'names'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'names')
     if w_self.w_names is None:
         if w_self.names is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.names]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_names = w_list
     return w_self.w_names
 
@@ -4473,8 +4423,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'level'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'level')
     return space.wrap(w_self.level)
 
 def ImportFrom_set_level(space, w_self, w_new_value):
@@ -4522,8 +4471,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     return space.wrap(w_self.body)
 
 def Exec_set_body(space, w_self, w_new_value):
@@ -4544,8 +4492,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'globals'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'globals')
     return space.wrap(w_self.globals)
 
 def Exec_set_globals(space, w_self, w_new_value):
@@ -4566,8 +4513,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'locals'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'locals')
     return space.wrap(w_self.locals)
 
 def Exec_set_locals(space, w_self, w_new_value):
@@ -4610,14 +4556,13 @@
 def Global_get_names(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'names'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'names')
     if w_self.w_names is None:
         if w_self.names is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.names]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_names = w_list
     return w_self.w_names
 
@@ -4657,8 +4602,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Expr_set_value(space, w_self, w_new_value):
@@ -4754,8 +4698,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._lineno_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'lineno'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lineno')
     return space.wrap(w_self.lineno)
 
 def expr_set_lineno(space, w_self, w_new_value):
@@ -4776,8 +4719,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._col_offset_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'col_offset'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'col_offset')
     return space.wrap(w_self.col_offset)
 
 def expr_set_col_offset(space, w_self, w_new_value):
@@ -4807,8 +4749,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'op'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op')
     return boolop_to_class[w_self.op - 1]()
 
 def BoolOp_set_op(space, w_self, w_new_value):
@@ -4827,14 +4768,13 @@
 def BoolOp_get_values(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'values'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'values')
     if w_self.w_values is None:
         if w_self.values is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.values]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_values = w_list
     return w_self.w_values
 
@@ -4875,8 +4815,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'left'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'left')
     return space.wrap(w_self.left)
 
 def BinOp_set_left(space, w_self, w_new_value):
@@ -4897,8 +4836,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'op'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op')
     return operator_to_class[w_self.op - 1]()
 
 def BinOp_set_op(space, w_self, w_new_value):
@@ -4921,8 +4859,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'right'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'right')
     return space.wrap(w_self.right)
 
 def BinOp_set_right(space, w_self, w_new_value):
@@ -4969,8 +4906,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'op'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op')
     return unaryop_to_class[w_self.op - 1]()
 
 def UnaryOp_set_op(space, w_self, w_new_value):
@@ -4993,8 +4929,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'operand'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'operand')
     return space.wrap(w_self.operand)
 
 def UnaryOp_set_operand(space, w_self, w_new_value):
@@ -5040,8 +4975,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'args'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args')
     return space.wrap(w_self.args)
 
 def Lambda_set_args(space, w_self, w_new_value):
@@ -5062,8 +4996,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     return space.wrap(w_self.body)
 
 def Lambda_set_body(space, w_self, w_new_value):
@@ -5109,8 +5042,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'test'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test')
     return space.wrap(w_self.test)
 
 def IfExp_set_test(space, w_self, w_new_value):
@@ -5131,8 +5063,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     return space.wrap(w_self.body)
 
 def IfExp_set_body(space, w_self, w_new_value):
@@ -5153,8 +5084,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'orelse'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
     return space.wrap(w_self.orelse)
 
 def IfExp_set_orelse(space, w_self, w_new_value):
@@ -5197,14 +5127,13 @@
 def Dict_get_keys(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'keys'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'keys')
     if w_self.w_keys is None:
         if w_self.keys is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.keys]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_keys = w_list
     return w_self.w_keys
 
@@ -5215,14 +5144,13 @@
 def Dict_get_values(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'values'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'values')
     if w_self.w_values is None:
         if w_self.values is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.values]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_values = w_list
     return w_self.w_values
 
@@ -5260,14 +5188,13 @@
 def Set_get_elts(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elts'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elts')
     if w_self.w_elts is None:
         if w_self.elts is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.elts]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_elts = w_list
     return w_self.w_elts
 
@@ -5307,8 +5234,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elt'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elt')
     return space.wrap(w_self.elt)
 
 def ListComp_set_elt(space, w_self, w_new_value):
@@ -5325,14 +5251,13 @@
 def ListComp_get_generators(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'generators'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators')
     if w_self.w_generators is None:
         if w_self.generators is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.generators]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_generators = w_list
     return w_self.w_generators
 
@@ -5373,8 +5298,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elt'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elt')
     return space.wrap(w_self.elt)
 
 def SetComp_set_elt(space, w_self, w_new_value):
@@ -5391,14 +5315,13 @@
 def SetComp_get_generators(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'generators'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators')
     if w_self.w_generators is None:
         if w_self.generators is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.generators]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_generators = w_list
     return w_self.w_generators
 
@@ -5439,8 +5362,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'key'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'key')
     return space.wrap(w_self.key)
 
 def DictComp_set_key(space, w_self, w_new_value):
@@ -5461,8 +5383,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def DictComp_set_value(space, w_self, w_new_value):
@@ -5479,14 +5400,13 @@
 def DictComp_get_generators(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'generators'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators')
     if w_self.w_generators is None:
         if w_self.generators is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.generators]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_generators = w_list
     return w_self.w_generators
 
@@ -5528,8 +5448,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elt'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elt')
     return space.wrap(w_self.elt)
 
 def GeneratorExp_set_elt(space, w_self, w_new_value):
@@ -5546,14 +5465,13 @@
 def GeneratorExp_get_generators(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'generators'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators')
     if w_self.w_generators is None:
         if w_self.generators is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.generators]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_generators = w_list
     return w_self.w_generators
 
@@ -5594,8 +5512,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Yield_set_value(space, w_self, w_new_value):
@@ -5640,8 +5557,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'left'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'left')
     return space.wrap(w_self.left)
 
 def Compare_set_left(space, w_self, w_new_value):
@@ -5658,14 +5574,13 @@
 def Compare_get_ops(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ops'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ops')
     if w_self.w_ops is None:
         if w_self.ops is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [cmpop_to_class[node - 1]() for node in w_self.ops]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_ops = w_list
     return w_self.w_ops
 
@@ -5676,14 +5591,13 @@
 def Compare_get_comparators(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'comparators'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'comparators')
     if w_self.w_comparators is None:
         if w_self.comparators is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.comparators]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_comparators = w_list
     return w_self.w_comparators
 
@@ -5726,8 +5640,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'func'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'func')
     return space.wrap(w_self.func)
 
 def Call_set_func(space, w_self, w_new_value):
@@ -5744,14 +5657,13 @@
 def Call_get_args(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'args'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args')
     if w_self.w_args is None:
         if w_self.args is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.args]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_args = w_list
     return w_self.w_args
 
@@ -5762,14 +5674,13 @@
 def Call_get_keywords(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'keywords'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'keywords')
     if w_self.w_keywords is None:
         if w_self.keywords is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.keywords]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_keywords = w_list
     return w_self.w_keywords
 
@@ -5784,8 +5695,7 @@
             return w_obj
     if not w_self.initialization_state & 8:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'starargs'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'starargs')
     return space.wrap(w_self.starargs)
 
 def Call_set_starargs(space, w_self, w_new_value):
@@ -5806,8 +5716,7 @@
             return w_obj
     if not w_self.initialization_state & 16:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'kwargs'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'kwargs')
     return space.wrap(w_self.kwargs)
 
 def Call_set_kwargs(space, w_self, w_new_value):
@@ -5858,8 +5767,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Repr_set_value(space, w_self, w_new_value):
@@ -5904,8 +5812,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'n'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'n')
     return w_self.n
 
 def Num_set_n(space, w_self, w_new_value):
@@ -5950,8 +5857,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 's'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 's')
     return w_self.s
 
 def Str_set_s(space, w_self, w_new_value):
@@ -5996,8 +5902,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Attribute_set_value(space, w_self, w_new_value):
@@ -6018,8 +5923,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'attr'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'attr')
     return space.wrap(w_self.attr)
 
 def Attribute_set_attr(space, w_self, w_new_value):
@@ -6040,8 +5944,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ctx'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def Attribute_set_ctx(space, w_self, w_new_value):
@@ -6090,8 +5993,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Subscript_set_value(space, w_self, w_new_value):
@@ -6112,8 +6014,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'slice'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'slice')
     return space.wrap(w_self.slice)
 
 def Subscript_set_slice(space, w_self, w_new_value):
@@ -6134,8 +6035,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ctx'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def Subscript_set_ctx(space, w_self, w_new_value):
@@ -6184,8 +6084,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'id'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'id')
     return space.wrap(w_self.id)
 
 def Name_set_id(space, w_self, w_new_value):
@@ -6206,8 +6105,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ctx'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def Name_set_ctx(space, w_self, w_new_value):
@@ -6251,14 +6149,13 @@
 def List_get_elts(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elts'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elts')
     if w_self.w_elts is None:
         if w_self.elts is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.elts]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_elts = w_list
     return w_self.w_elts
 
@@ -6273,8 +6170,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ctx'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def List_set_ctx(space, w_self, w_new_value):
@@ -6319,14 +6215,13 @@
 def Tuple_get_elts(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elts'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elts')
     if w_self.w_elts is None:
         if w_self.elts is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.elts]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_elts = w_list
     return w_self.w_elts
 
@@ -6341,8 +6236,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ctx'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def Tuple_set_ctx(space, w_self, w_new_value):
@@ -6391,8 +6285,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return w_self.value
 
 def Const_set_value(space, w_self, w_new_value):
@@ -6510,8 +6403,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'lower'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lower')
     return space.wrap(w_self.lower)
 
 def Slice_set_lower(space, w_self, w_new_value):
@@ -6532,8 +6424,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'upper'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'upper')
     return space.wrap(w_self.upper)
 
 def Slice_set_upper(space, w_self, w_new_value):
@@ -6554,8 +6445,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'step'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'step')
     return space.wrap(w_self.step)
 
 def Slice_set_step(space, w_self, w_new_value):
@@ -6598,14 +6488,13 @@
 def ExtSlice_get_dims(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'dims'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'dims')
     if w_self.w_dims is None:
         if w_self.dims is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.dims]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_dims = w_list
     return w_self.w_dims
 
@@ -6645,8 +6534,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Index_set_value(space, w_self, w_new_value):
@@ -6915,8 +6803,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'target'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'target')
     return space.wrap(w_self.target)
 
 def comprehension_set_target(space, w_self, w_new_value):
@@ -6937,8 +6824,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'iter'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'iter')
     return space.wrap(w_self.iter)
 
 def comprehension_set_iter(space, w_self, w_new_value):
@@ -6955,14 +6841,13 @@
 def comprehension_get_ifs(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ifs'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ifs')
     if w_self.w_ifs is None:
         if w_self.ifs is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.ifs]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_ifs = w_list
     return w_self.w_ifs
 
@@ -7004,8 +6889,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._lineno_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'lineno'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lineno')
     return space.wrap(w_self.lineno)
 
 def excepthandler_set_lineno(space, w_self, w_new_value):
@@ -7026,8 +6910,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._col_offset_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'col_offset'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'col_offset')
     return space.wrap(w_self.col_offset)
 
 def excepthandler_set_col_offset(space, w_self, w_new_value):
@@ -7057,8 +6940,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'type'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'type')
     return space.wrap(w_self.type)
 
 def ExceptHandler_set_type(space, w_self, w_new_value):
@@ -7079,8 +6961,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'name'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name')
     return space.wrap(w_self.name)
 
 def ExceptHandler_set_name(space, w_self, w_new_value):
@@ -7097,14 +6978,13 @@
 def ExceptHandler_get_body(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -7142,14 +7022,13 @@
 def arguments_get_args(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'args'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args')
     if w_self.w_args is None:
         if w_self.args is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.args]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_args = w_list
     return w_self.w_args
 
@@ -7164,8 +7043,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'vararg'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'vararg')
     return space.wrap(w_self.vararg)
 
 def arguments_set_vararg(space, w_self, w_new_value):
@@ -7189,8 +7067,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'kwarg'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'kwarg')
     return space.wrap(w_self.kwarg)
 
 def arguments_set_kwarg(space, w_self, w_new_value):
@@ -7210,14 +7087,13 @@
 def arguments_get_defaults(space, w_self):
     if not w_self.initialization_state & 8:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'defaults'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'defaults')
     if w_self.w_defaults is None:
         if w_self.defaults is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.defaults]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_defaults = w_list
     return w_self.w_defaults
 
@@ -7261,8 +7137,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'arg'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'arg')
     return space.wrap(w_self.arg)
 
 def keyword_set_arg(space, w_self, w_new_value):
@@ -7283,8 +7158,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def keyword_set_value(space, w_self, w_new_value):
@@ -7330,8 +7204,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'name'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name')
     return space.wrap(w_self.name)
 
 def alias_set_name(space, w_self, w_new_value):
@@ -7352,8 +7225,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'asname'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'asname')
     return space.wrap(w_self.asname)
 
 def alias_set_asname(space, w_self, w_new_value):
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -414,13 +414,12 @@
             self.emit("        return w_obj", 1)
         self.emit("if not w_self.initialization_state & %s:" % (flag,), 1)
         self.emit("typename = space.type(w_self).getname(space)", 2)
-        self.emit("w_err = space.wrap(\"'%%s' object has no attribute '%s'\" %% typename)" %
+        self.emit("raise operationerrfmt(space.w_AttributeError, \"'%%s' object has no attribute '%%s'\", typename, '%s')" %
                   (field.name,), 2)
-        self.emit("raise OperationError(space.w_AttributeError, w_err)", 2)
         if field.seq:
             self.emit("if w_self.w_%s is None:" % (field.name,), 1)
             self.emit("if w_self.%s is None:" % (field.name,), 2)
-            self.emit("w_list = space.newlist([])", 3)
+            self.emit("list_w = []", 3)
             self.emit("else:", 2)
             if field.type.value in self.data.simple_types:
                 wrapper = "%s_to_class[node - 1]()" % (field.type,)
@@ -428,7 +427,7 @@
                 wrapper = "space.wrap(node)"
             self.emit("list_w = [%s for node in w_self.%s]" %
                       (wrapper, field.name), 3)
-            self.emit("w_list = space.newlist(list_w)", 3)
+            self.emit("w_list = space.newlist(list_w)", 2)
             self.emit("w_self.w_%s = w_list" % (field.name,), 2)
             self.emit("return w_self.w_%s" % (field.name,), 1)
         elif field.type.value in self.data.simple_types:
@@ -540,7 +539,7 @@
 from pypy.interpreter.baseobjspace import Wrappable
 from pypy.interpreter import typedef
 from pypy.interpreter.gateway import interp2app
-from pypy.interpreter.error import OperationError
+from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.tool.pairtype import extendabletype
 from pypy.tool.sourcetools import func_with_new_name
@@ -639,9 +638,7 @@
             missing = required[i]
             if missing is not None:
                  err = "required field \\"%s\\" missing from %s"
-                 err = err % (missing, host)
-                 w_err = space.wrap(err)
-                 raise OperationError(space.w_TypeError, w_err)
+                 raise operationerrfmt(space.w_TypeError, err, missing, host)
     raise AssertionError("should not reach here")
 
 
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -391,8 +391,11 @@
     def decrement_ticker(self, by):
         value = self._ticker
         if self.has_bytecode_counter:    # this 'if' is constant-folded
-            value -= by
-            self._ticker = value
+            if jit.isconstant(by) and by == 0:
+                pass     # normally constant-folded too
+            else:
+                value -= by
+                self._ticker = value
         return value
 
 
diff --git a/pypy/interpreter/pyparser/pytokenizer.py b/pypy/interpreter/pyparser/pytokenizer.py
--- a/pypy/interpreter/pyparser/pytokenizer.py
+++ b/pypy/interpreter/pyparser/pytokenizer.py
@@ -226,7 +226,7 @@
                         parenlev = parenlev - 1
                         if parenlev < 0:
                             raise TokenError("unmatched '%s'" % initial, line,
-                                             lnum-1, 0, token_list)
+                                             lnum, start + 1, token_list)
                     if token in python_opmap:
                         punct = python_opmap[token]
                     else:
diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py
--- a/pypy/interpreter/pyparser/test/test_pyparse.py
+++ b/pypy/interpreter/pyparser/test/test_pyparse.py
@@ -87,6 +87,10 @@
         assert exc.lineno == 1
         assert exc.offset == 5
         assert exc.lastlineno == 5
+        exc = py.test.raises(SyntaxError, parse, "abc)").value
+        assert exc.msg == "unmatched ')'"
+        assert exc.lineno == 1
+        assert exc.offset == 4
 
     def test_is(self):
         self.parse("x is y")
diff --git a/pypy/jit/backend/llgraph/llimpl.py b/pypy/jit/backend/llgraph/llimpl.py
--- a/pypy/jit/backend/llgraph/llimpl.py
+++ b/pypy/jit/backend/llgraph/llimpl.py
@@ -7,9 +7,7 @@
 import weakref
 from pypy.objspace.flow.model import Variable, Constant
 from pypy.annotation import model as annmodel
-from pypy.jit.metainterp.history import (ConstInt, ConstPtr,
-                                         BoxInt, BoxPtr, BoxObj, BoxFloat,
-                                         REF, INT, FLOAT)
+from pypy.jit.metainterp.history import REF, INT, FLOAT
 from pypy.jit.codewriter import heaptracker
 from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr, rffi
 from pypy.rpython.ootypesystem import ootype
@@ -17,7 +15,7 @@
 from pypy.rpython.llinterp import LLException
 from pypy.rpython.extregistry import ExtRegistryEntry
 
-from pypy.jit.metainterp import resoperation, executor
+from pypy.jit.metainterp import resoperation
 from pypy.jit.metainterp.resoperation import rop
 from pypy.jit.backend.llgraph import symbolic
 from pypy.jit.codewriter import longlong
@@ -165,6 +163,7 @@
     'unicodegetitem'  : (('ref', 'int'), 'int'),
     'unicodesetitem'  : (('ref', 'int', 'int'), 'int'),
     'cast_ptr_to_int' : (('ref',), 'int'),
+    'cast_int_to_ptr' : (('int',), 'ref'),
     'debug_merge_point': (('ref', 'int'), None),
     'force_token'     : ((), 'int'),
     'call_may_force'  : (('int', 'varargs'), 'intorptr'),
@@ -333,6 +332,13 @@
     assert isinstance(type, str) and len(type) == 1
     op.descr = Descr(ofs, type, arg_types=arg_types)
 
+def compile_add_descr_arg(loop, ofs, type, arg_types):
+    from pypy.jit.backend.llgraph.runner import Descr
+    loop = _from_opaque(loop)
+    op = loop.operations[-1]
+    assert isinstance(type, str) and len(type) == 1
+    op.args.append(Descr(ofs, type, arg_types=arg_types))
+
 def compile_add_loop_token(loop, descr):
     if we_are_translated():
         raise ValueError("CALL_ASSEMBLER not supported")
@@ -437,8 +443,11 @@
         self._may_force = -1
 
     def getenv(self, v):
+        from pypy.jit.backend.llgraph.runner import Descr
         if isinstance(v, Constant):
             return v.value
+        elif isinstance(v, Descr):
+            return v
         else:
             return self.env[v]
 
@@ -806,6 +815,29 @@
         else:
             raise NotImplementedError
 
+    def op_getinteriorfield_gc(self, descr, array, index):
+        if descr.typeinfo == REF:
+            return do_getinteriorfield_gc_ptr(array, index, descr.ofs)
+        elif descr.typeinfo == INT:
+            return do_getinteriorfield_gc_int(array, index, descr.ofs)
+        elif descr.typeinfo == FLOAT:
+            return do_getinteriorfield_gc_float(array, index, descr.ofs)
+        else:
+            raise NotImplementedError
+
+    def op_setinteriorfield_gc(self, descr, array, index, newvalue):
+        if descr.typeinfo == REF:
+            return do_setinteriorfield_gc_ptr(array, index, descr.ofs,
+                                              newvalue)
+        elif descr.typeinfo == INT:
+            return do_setinteriorfield_gc_int(array, index, descr.ofs,
+                                              newvalue)
+        elif descr.typeinfo == FLOAT:
+            return do_setinteriorfield_gc_float(array, index, descr.ofs,
+                                                newvalue)
+        else:
+            raise NotImplementedError
+
     def op_setfield_gc(self, fielddescr, struct, newvalue):
         if fielddescr.typeinfo == REF:
             do_setfield_gc_ptr(struct, fielddescr.ofs, newvalue)
@@ -875,9 +907,6 @@
     def op_new_array(self, arraydescr, count):
         return do_new_array(arraydescr.ofs, count)
 
-    def op_cast_ptr_to_int(self, descr, ptr):
-        return cast_to_int(ptr)
-
     def op_force_token(self, descr):
         opaque_frame = _to_opaque(self)
         return llmemory.cast_ptr_to_adr(opaque_frame)
@@ -1356,6 +1385,22 @@
 def do_getfield_gc_ptr(struct, fieldnum):
     return cast_to_ptr(_getfield_gc(struct, fieldnum))
 
+def _getinteriorfield_gc(struct, fieldnum):
+    STRUCT, fieldname = symbolic.TokenToField[fieldnum]
+    return getattr(struct, fieldname)
+
+def do_getinteriorfield_gc_int(array, index, fieldnum):
+    struct = array._obj.container.getitem(index)
+    return cast_to_int(_getinteriorfield_gc(struct, fieldnum))
+
+def do_getinteriorfield_gc_float(array, index, fieldnum):
+    struct = array._obj.container.getitem(index)
+    return cast_to_floatstorage(_getinteriorfield_gc(struct, fieldnum))
+
+def do_getinteriorfield_gc_ptr(array, index, fieldnum):
+    struct = array._obj.container.getitem(index)
+    return cast_to_ptr(_getinteriorfield_gc(struct, fieldnum))
+
 def _getfield_raw(struct, fieldnum):
     STRUCT, fieldname = symbolic.TokenToField[fieldnum]
     ptr = cast_from_int(lltype.Ptr(STRUCT), struct)
@@ -1411,26 +1456,28 @@
     newvalue = cast_from_ptr(ITEMTYPE, newvalue)
     array.setitem(index, newvalue)
 
-def do_setfield_gc_int(struct, fieldnum, newvalue):
-    STRUCT, fieldname = symbolic.TokenToField[fieldnum]
-    ptr = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), struct)
-    FIELDTYPE = getattr(STRUCT, fieldname)
-    newvalue = cast_from_int(FIELDTYPE, newvalue)
-    setattr(ptr, fieldname, newvalue)
+def new_setfield_gc(cast_func):
+    def do_setfield_gc(struct, fieldnum, newvalue):
+        STRUCT, fieldname = symbolic.TokenToField[fieldnum]
+        ptr = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), struct)
+        FIELDTYPE = getattr(STRUCT, fieldname)
+        newvalue = cast_func(FIELDTYPE, newvalue)
+        setattr(ptr, fieldname, newvalue)
+    return do_setfield_gc
+do_setfield_gc_int = new_setfield_gc(cast_from_int)
+do_setfield_gc_float = new_setfield_gc(cast_from_floatstorage)
+do_setfield_gc_ptr = new_setfield_gc(cast_from_ptr)
 
-def do_setfield_gc_float(struct, fieldnum, newvalue):
-    STRUCT, fieldname = symbolic.TokenToField[fieldnum]
-    ptr = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), struct)
-    FIELDTYPE = getattr(STRUCT, fieldname)
-    newvalue = cast_from_floatstorage(FIELDTYPE, newvalue)
-    setattr(ptr, fieldname, newvalue)
-
-def do_setfield_gc_ptr(struct, fieldnum, newvalue):
-    STRUCT, fieldname = symbolic.TokenToField[fieldnum]
-    ptr = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), struct)
-    FIELDTYPE = getattr(STRUCT, fieldname)
-    newvalue = cast_from_ptr(FIELDTYPE, newvalue)
-    setattr(ptr, fieldname, newvalue)
+def new_setinteriorfield_gc(cast_func):
+    def do_setinteriorfield_gc(array, index, fieldnum, newvalue):
+        STRUCT, fieldname = symbolic.TokenToField[fieldnum]
+        struct = array._obj.container.getitem(index)
+        FIELDTYPE = getattr(STRUCT, fieldname)
+        setattr(struct, fieldname, cast_func(FIELDTYPE, newvalue))
+    return do_setinteriorfield_gc
+do_setinteriorfield_gc_int = new_setinteriorfield_gc(cast_from_int)
+do_setinteriorfield_gc_float = new_setinteriorfield_gc(cast_from_floatstorage)
+do_setinteriorfield_gc_ptr = new_setinteriorfield_gc(cast_from_ptr)        
 
 def do_setfield_raw_int(struct, fieldnum, newvalue):
     STRUCT, fieldname = symbolic.TokenToField[fieldnum]
@@ -1696,6 +1743,7 @@
 setannotation(compile_start_float_var, annmodel.SomeInteger())
 setannotation(compile_add, annmodel.s_None)
 setannotation(compile_add_descr, annmodel.s_None)
+setannotation(compile_add_descr_arg, annmodel.s_None)
 setannotation(compile_add_var, annmodel.s_None)
 setannotation(compile_add_int_const, annmodel.s_None)
 setannotation(compile_add_ref_const, annmodel.s_None)
@@ -1743,6 +1791,9 @@
 setannotation(do_getfield_raw_int, annmodel.SomeInteger())
 setannotation(do_getfield_raw_ptr, annmodel.SomePtr(llmemory.GCREF))
 setannotation(do_getfield_raw_float, s_FloatStorage)
+setannotation(do_getinteriorfield_gc_int, annmodel.SomeInteger())
+setannotation(do_getinteriorfield_gc_ptr, annmodel.SomePtr(llmemory.GCREF))
+setannotation(do_getinteriorfield_gc_float, s_FloatStorage)
 setannotation(do_new, annmodel.SomePtr(llmemory.GCREF))
 setannotation(do_new_array, annmodel.SomePtr(llmemory.GCREF))
 setannotation(do_setarrayitem_gc_int, annmodel.s_None)
@@ -1756,6 +1807,9 @@
 setannotation(do_setfield_raw_int, annmodel.s_None)
 setannotation(do_setfield_raw_ptr, annmodel.s_None)
 setannotation(do_setfield_raw_float, annmodel.s_None)
+setannotation(do_setinteriorfield_gc_int, annmodel.s_None)
+setannotation(do_setinteriorfield_gc_ptr, annmodel.s_None)
+setannotation(do_setinteriorfield_gc_float, annmodel.s_None)
 setannotation(do_newstr, annmodel.SomePtr(llmemory.GCREF))
 setannotation(do_strsetitem, annmodel.s_None)
 setannotation(do_newunicode, annmodel.SomePtr(llmemory.GCREF))
diff --git a/pypy/jit/backend/llgraph/runner.py b/pypy/jit/backend/llgraph/runner.py
--- a/pypy/jit/backend/llgraph/runner.py
+++ b/pypy/jit/backend/llgraph/runner.py
@@ -2,21 +2,19 @@
 Minimal-API wrapper around the llinterpreter to run operations.
 """
 
-import sys
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.rpython.lltypesystem import lltype, llmemory, rclass
 from pypy.rpython.ootypesystem import ootype
 from pypy.rpython.llinterp import LLInterpreter
 from pypy.jit.metainterp import history
-from pypy.jit.metainterp.history import REF, INT, FLOAT
+from pypy.jit.metainterp.history import REF, INT, FLOAT, STRUCT
 from pypy.jit.metainterp.warmstate import unwrap
-from pypy.jit.metainterp.resoperation import ResOperation, rop
+from pypy.jit.metainterp.resoperation import rop
 from pypy.jit.backend import model
 from pypy.jit.backend.llgraph import llimpl, symbolic
 from pypy.jit.metainterp.typesystem import llhelper, oohelper
 from pypy.jit.codewriter import heaptracker, longlong
-from pypy.rlib import rgc
 
 class MiniStats:
     pass
@@ -62,6 +60,9 @@
     def is_array_of_floats(self):
         return self.typeinfo == FLOAT
 
+    def is_array_of_structs(self):
+        return self.typeinfo == STRUCT
+
     def as_vtable_size_descr(self):
         return self
 
@@ -177,8 +178,10 @@
             llimpl.compile_add(c, op.getopnum())
             descr = op.getdescr()
             if isinstance(descr, Descr):
-                llimpl.compile_add_descr(c, descr.ofs, descr.typeinfo, descr.arg_types)
-            if isinstance(descr, history.LoopToken) and op.getopnum() != rop.JUMP:
+                llimpl.compile_add_descr(c, descr.ofs, descr.typeinfo,
+                                         descr.arg_types)
+            if (isinstance(descr, history.LoopToken) and
+                op.getopnum() != rop.JUMP):
                 llimpl.compile_add_loop_token(c, descr)
             if self.is_oo and isinstance(descr, (OODescr, MethDescr)):
                 # hack hack, not rpython
@@ -193,6 +196,9 @@
                     llimpl.compile_add_ref_const(c, x.value, self.ts.BASETYPE)
                 elif isinstance(x, history.ConstFloat):
                     llimpl.compile_add_float_const(c, x.value)
+                elif isinstance(x, Descr):
+                    llimpl.compile_add_descr_arg(c, x.ofs, x.typeinfo,
+                                                 x.arg_types)
                 else:
                     raise Exception("'%s' args contain: %r" % (op.getopname(),
                                                                x))
@@ -316,6 +322,13 @@
         token = history.getkind(getattr(S, fieldname))
         return self.getdescr(ofs, token[0], name=fieldname)
 
+    def interiorfielddescrof(self, A, fieldname):
+        S = A.OF
+        ofs2 = symbolic.get_size(A)
+        ofs, size = symbolic.get_field_token(S, fieldname)
+        token = history.getkind(getattr(S, fieldname))
+        return self.getdescr(ofs, token[0], name=fieldname, extrainfo=ofs2)
+
     def calldescrof(self, FUNC, ARGS, RESULT, extrainfo):
         arg_types = []
         for ARG in ARGS:
@@ -353,8 +366,13 @@
     def arraydescrof(self, A):
         assert A.OF != lltype.Void
         size = symbolic.get_size(A)
-        token = history.getkind(A.OF)
-        return self.getdescr(size, token[0])
+        if isinstance(A.OF, lltype.Ptr) or isinstance(A.OF, lltype.Primitive):
+            token = history.getkind(A.OF)[0]
+        elif isinstance(A.OF, lltype.Struct):
+            token = 's'
+        else:
+            token = '?'
+        return self.getdescr(size, token)
 
     # ---------- the backend-dependent operations ----------
 
@@ -406,6 +424,29 @@
         assert isinstance(fielddescr, Descr)
         return llimpl.do_getfield_raw_float(struct, fielddescr.ofs)
 
+    def bh_getinteriorfield_gc_i(self, array, index, descr):
+        assert isinstance(descr, Descr)
+        return llimpl.do_getinteriorfield_gc_int(array, index, descr.ofs)
+    def bh_getinteriorfield_gc_r(self, array, index, descr):
+        assert isinstance(descr, Descr)
+        return llimpl.do_getinteriorfield_gc_ptr(array, index, descr.ofs)
+    def bh_getinteriorfield_gc_f(self, array, index, descr):
+        assert isinstance(descr, Descr)
+        return llimpl.do_getinteriorfield_gc_float(array, index, descr.ofs)
+
+    def bh_setinteriorfield_gc_i(self, array, index, descr, value):
+        assert isinstance(descr, Descr)
+        return llimpl.do_setinteriorfield_gc_int(array, index, descr.ofs,
+                                                 value)
+    def bh_setinteriorfield_gc_r(self, array, index, descr, value):
+        assert isinstance(descr, Descr)
+        return llimpl.do_setinteriorfield_gc_ptr(array, index, descr.ofs,
+                                                 value)
+    def bh_setinteriorfield_gc_f(self, array, index, descr, value):
+        assert isinstance(descr, Descr)
+        return llimpl.do_setinteriorfield_gc_float(array, index, descr.ofs,
+                                                   value)
+
     def bh_new(self, sizedescr):
         assert isinstance(sizedescr, Descr)
         return llimpl.do_new(sizedescr.ofs)
@@ -418,7 +459,6 @@
 
     def bh_classof(self, struct):
         struct = lltype.cast_opaque_ptr(rclass.OBJECTPTR, struct)
-        result = struct.typeptr
         result_adr = llmemory.cast_ptr_to_adr(struct.typeptr)
         return heaptracker.adr2int(result_adr)
 
diff --git a/pypy/jit/backend/llsupport/descr.py b/pypy/jit/backend/llsupport/descr.py
--- a/pypy/jit/backend/llsupport/descr.py
+++ b/pypy/jit/backend/llsupport/descr.py
@@ -1,13 +1,10 @@
 import py
-from pypy.rpython.lltypesystem import lltype, rffi, llmemory, rclass
+from pypy.rpython.lltypesystem import lltype, rffi, llmemory
 from pypy.rpython.lltypesystem.lloperation import llop
 from pypy.jit.backend.llsupport import symbolic, support
-from pypy.jit.metainterp.history import AbstractDescr, getkind, BoxInt, BoxPtr
-from pypy.jit.metainterp.history import BasicFailDescr, LoopToken, BoxFloat
+from pypy.jit.metainterp.history import AbstractDescr, getkind
 from pypy.jit.metainterp import history
-from pypy.jit.metainterp.resoperation import ResOperation, rop
 from pypy.jit.codewriter import heaptracker, longlong
-from pypy.rlib.rarithmetic import r_longlong, r_ulonglong
 
 # The point of the class organization in this file is to make instances
 # as compact as possible.  This is done by not storing the field size or
@@ -23,6 +20,7 @@
         self._cache_field = {}
         self._cache_array = {}
         self._cache_call = {}
+        self._cache_interiorfield = {}
 
     def init_size_descr(self, STRUCT, sizedescr):
         assert isinstance(STRUCT, lltype.GcStruct)
@@ -142,7 +140,6 @@
         cachedict[fieldname] = fielddescr
         return fielddescr
 
-
 # ____________________________________________________________
 # ArrayDescrs
 
@@ -167,6 +164,7 @@
 
     _is_array_of_pointers = False      # unless overridden by GcPtrArrayDescr
     _is_array_of_floats   = False      # unless overridden by FloatArrayDescr
+    _is_array_of_structs  = False      # unless overridden by StructArrayDescr
     _is_item_signed       = False      # unless overridden by XxxArrayDescr
 
     def is_array_of_pointers(self):
@@ -175,6 +173,9 @@
     def is_array_of_floats(self):
         return self._is_array_of_floats
 
+    def is_array_of_structs(self):
+        return self._is_array_of_structs
+
     def is_item_signed(self):
         return self._is_item_signed
 
@@ -199,6 +200,10 @@
     def get_item_size(self, translate_support_code):
         return symbolic.get_size(lltype.Float, translate_support_code)
 
+class StructArrayDescr(BaseArrayDescr):
+    _clsname = 'StructArrayDescr'
+    _is_array_of_structs = True
+
 class BaseArrayNoLengthDescr(BaseArrayDescr):
     def get_base_size(self, translate_support_code):
         return 0
@@ -218,6 +223,13 @@
 def getArrayDescrClass(ARRAY):
     if ARRAY.OF is lltype.Float:
         return FloatArrayDescr
+    elif isinstance(ARRAY.OF, lltype.Struct):
+        class Descr(StructArrayDescr):
+            _clsname = '%sArrayDescr' % ARRAY.OF._name
+            def get_item_size(self, translate_support_code):
+                return symbolic.get_size(ARRAY.OF, translate_support_code)
+        Descr.__name__ = Descr._clsname
+        return Descr
     return getDescrClass(ARRAY.OF, BaseArrayDescr, GcPtrArrayDescr,
                          NonGcPtrArrayDescr, 'Array', 'get_item_size',
                          '_is_array_of_floats', '_is_item_signed')
@@ -252,6 +264,36 @@
         cache[ARRAY] = arraydescr
         return arraydescr
 
+# ____________________________________________________________
+# InteriorFieldDescr
+
+class InteriorFieldDescr(AbstractDescr):
+    arraydescr = BaseArrayDescr()     # workaround for the annotator
+    fielddescr = BaseFieldDescr('', 0)
+
+    def __init__(self, arraydescr, fielddescr):
+        self.arraydescr = arraydescr
+        self.fielddescr = fielddescr
+
+    def is_pointer_field(self):
+        return self.fielddescr.is_pointer_field()
+
+    def is_float_field(self):
+        return self.fielddescr.is_float_field()
+
+    def repr_of_descr(self):
+        return '<InteriorFieldDescr %s>' % self.fielddescr.repr_of_descr()
+
+def get_interiorfield_descr(gc_ll_descr, ARRAY, FIELDTP, name):
+    cache = gc_ll_descr._cache_interiorfield
+    try:
+        return cache[(ARRAY, FIELDTP, name)]
+    except KeyError:
+        arraydescr = get_array_descr(gc_ll_descr, ARRAY)
+        fielddescr = get_field_descr(gc_ll_descr, FIELDTP, name)
+        descr = InteriorFieldDescr(arraydescr, fielddescr)
+        cache[(ARRAY, FIELDTP, name)] = descr
+        return descr
 
 # ____________________________________________________________
 # CallDescrs
@@ -525,7 +567,8 @@
         #
         if TYPE is lltype.Float or is_longlong(TYPE):
             setattr(Descr, floatattrname, True)
-        elif TYPE is not lltype.Bool and rffi.cast(TYPE, -1) == -1:
+        elif (TYPE is not lltype.Bool and isinstance(TYPE, lltype.Number) and
+              rffi.cast(TYPE, -1) == -1):
             setattr(Descr, signedattrname, True)
         #
         _cache[nameprefix, TYPE] = Descr
diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py
--- a/pypy/jit/backend/llsupport/gc.py
+++ b/pypy/jit/backend/llsupport/gc.py
@@ -45,6 +45,22 @@
     def freeing_block(self, start, stop):
         pass
 
+    def get_funcptr_for_newarray(self):
+        return llhelper(self.GC_MALLOC_ARRAY, self.malloc_array)
+    def get_funcptr_for_newstr(self):
+        return llhelper(self.GC_MALLOC_STR_UNICODE, self.malloc_str)
+    def get_funcptr_for_newunicode(self):
+        return llhelper(self.GC_MALLOC_STR_UNICODE, self.malloc_unicode)
+
+
+    def record_constptrs(self, op, gcrefs_output_list):
+        for i in range(op.numargs()):
+            v = op.getarg(i)
+            if isinstance(v, ConstPtr) and bool(v.value):
+                p = v.value
+                rgc._make_sure_does_not_move(p)
+                gcrefs_output_list.append(p)
+
 # ____________________________________________________________
 
 class GcLLDescr_boehm(GcLLDescription):
@@ -88,6 +104,39 @@
         malloc_fn_ptr = self.configure_boehm_once()
         self.funcptr_for_new = malloc_fn_ptr
 
+        def malloc_array(basesize, itemsize, ofs_length, num_elem):
+            try:
+                size = ovfcheck(basesize + ovfcheck(itemsize * num_elem))
+            except OverflowError:
+                return lltype.nullptr(llmemory.GCREF.TO)
+            res = self.funcptr_for_new(size)
+            if not res:
+                return res
+            rffi.cast(rffi.CArrayPtr(lltype.Signed), res)[ofs_length/WORD] = num_elem
+            return res
+        self.malloc_array = malloc_array
+        self.GC_MALLOC_ARRAY = lltype.Ptr(lltype.FuncType(
+            [lltype.Signed] * 4, llmemory.GCREF))
+
+
+        (str_basesize, str_itemsize, str_ofs_length
+         ) = symbolic.get_array_token(rstr.STR, self.translate_support_code)
+        (unicode_basesize, unicode_itemsize, unicode_ofs_length
+         ) = symbolic.get_array_token(rstr.UNICODE, self.translate_support_code)
+        def malloc_str(length):
+            return self.malloc_array(
+                str_basesize, str_itemsize, str_ofs_length, length
+            )
+        def malloc_unicode(length):
+            return self.malloc_array(
+                unicode_basesize, unicode_itemsize, unicode_ofs_length, length
+            )
+        self.malloc_str = malloc_str
+        self.malloc_unicode = malloc_unicode
+        self.GC_MALLOC_STR_UNICODE = lltype.Ptr(lltype.FuncType(
+            [lltype.Signed], llmemory.GCREF))
+
+
         # on some platform GC_init is required before any other
         # GC_* functions, call it here for the benefit of tests
         # XXX move this to tests
@@ -108,38 +157,34 @@
         ofs_length = arraydescr.get_ofs_length(self.translate_support_code)
         basesize = arraydescr.get_base_size(self.translate_support_code)
         itemsize = arraydescr.get_item_size(self.translate_support_code)
-        size = basesize + itemsize * num_elem
-        res = self.funcptr_for_new(size)
-        rffi.cast(rffi.CArrayPtr(lltype.Signed), res)[ofs_length/WORD] = num_elem
-        return res
+        return self.malloc_array(basesize, itemsize, ofs_length, num_elem)
 
     def gc_malloc_str(self, num_elem):
-        basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR,
-                                                   self.translate_support_code)
-        assert itemsize == 1
-        size = basesize + num_elem
-        res = self.funcptr_for_new(size)
-        rffi.cast(rffi.CArrayPtr(lltype.Signed), res)[ofs_length/WORD] = num_elem
-        return res
+        return self.malloc_str(num_elem)
 
     def gc_malloc_unicode(self, num_elem):
-        basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE,
-                                                   self.translate_support_code)
-        size = basesize + num_elem * itemsize
-        res = self.funcptr_for_new(size)
-        rffi.cast(rffi.CArrayPtr(lltype.Signed), res)[ofs_length/WORD] = num_elem
-        return res
+        return self.malloc_unicode(num_elem)
 
     def args_for_new(self, sizedescr):
         assert isinstance(sizedescr, BaseSizeDescr)
         return [sizedescr.size]
 
+    def args_for_new_array(self, arraydescr):
+        ofs_length = arraydescr.get_ofs_length(self.translate_support_code)
+        basesize = arraydescr.get_base_size(self.translate_support_code)
+        itemsize = arraydescr.get_item_size(self.translate_support_code)
+        return [basesize, itemsize, ofs_length]
+
     def get_funcptr_for_new(self):
         return self.funcptr_for_new
 
-    get_funcptr_for_newarray = None
-    get_funcptr_for_newstr = None
-    get_funcptr_for_newunicode = None
+    def rewrite_assembler(self, cpu, operations, gcrefs_output_list):
+        # record all GCREFs too, because Boehm cannot see them and keep them
+        # alive if they end up as constants in the assembler
+        for op in operations:
+            self.record_constptrs(op, gcrefs_output_list)
+        return GcLLDescription.rewrite_assembler(self, cpu, operations,
+                                                 gcrefs_output_list)
 
 
 # ____________________________________________________________
@@ -738,15 +783,6 @@
     def get_funcptr_for_new(self):
         return llhelper(self.GC_MALLOC_BASIC, self.malloc_basic)
 
-    def get_funcptr_for_newarray(self):
-        return llhelper(self.GC_MALLOC_ARRAY, self.malloc_array)
-
-    def get_funcptr_for_newstr(self):
-        return llhelper(self.GC_MALLOC_STR_UNICODE, self.malloc_str)
-
-    def get_funcptr_for_newunicode(self):
-        return llhelper(self.GC_MALLOC_STR_UNICODE, self.malloc_unicode)
-
     def do_write_barrier(self, gcref_struct, gcref_newptr):
         hdr_addr = llmemory.cast_ptr_to_adr(gcref_struct)
         hdr_addr -= self.gcheaderbuilder.size_gc_header
@@ -759,14 +795,6 @@
             funcptr(llmemory.cast_ptr_to_adr(gcref_struct),
                     llmemory.cast_ptr_to_adr(gcref_newptr))
 
-    def record_constptrs(self, op, gcrefs_output_list):
-        for i in range(op.numargs()):
-            v = op.getarg(i)
-            if isinstance(v, ConstPtr) and bool(v.value):
-                p = v.value
-                rgc._make_sure_does_not_move(p)
-                gcrefs_output_list.append(p)
-
     def rewrite_assembler(self, cpu, operations, gcrefs_output_list):
         # Perform two kinds of rewrites in parallel:
         #
diff --git a/pypy/jit/backend/llsupport/llmodel.py b/pypy/jit/backend/llsupport/llmodel.py
--- a/pypy/jit/backend/llsupport/llmodel.py
+++ b/pypy/jit/backend/llsupport/llmodel.py
@@ -1,24 +1,18 @@
-import sys
 from pypy.rpython.lltypesystem import lltype, llmemory, rffi, rclass, rstr
 from pypy.rpython.lltypesystem.lloperation import llop
-from pypy.rpython.llinterp import LLInterpreter, LLException
+from pypy.rpython.llinterp import LLInterpreter
 from pypy.rpython.annlowlevel import llhelper
 from pypy.rlib.objectmodel import we_are_translated, specialize
-from pypy.jit.metainterp.history import BoxInt, BoxPtr, set_future_values,\
-     BoxFloat
 from pypy.jit.metainterp import history
 from pypy.jit.codewriter import heaptracker, longlong
 from pypy.jit.backend.model import AbstractCPU
 from pypy.jit.backend.llsupport import symbolic
 from pypy.jit.backend.llsupport.symbolic import WORD, unroll_basic_sizes
-from pypy.jit.backend.llsupport.descr import get_size_descr,  BaseSizeDescr
-from pypy.jit.backend.llsupport.descr import get_field_descr, BaseFieldDescr
-from pypy.jit.backend.llsupport.descr import get_array_descr, BaseArrayDescr
-from pypy.jit.backend.llsupport.descr import get_call_descr
-from pypy.jit.backend.llsupport.descr import BaseIntCallDescr, GcPtrCallDescr
-from pypy.jit.backend.llsupport.descr import FloatCallDescr, VoidCallDescr
+from pypy.jit.backend.llsupport.descr import (get_size_descr,
+     get_field_descr, BaseFieldDescr, get_array_descr, BaseArrayDescr,
+     get_call_descr, BaseIntCallDescr, GcPtrCallDescr, FloatCallDescr,
+     VoidCallDescr, InteriorFieldDescr, get_interiorfield_descr)
 from pypy.jit.backend.llsupport.asmmemmgr import AsmMemoryManager
-from pypy.rpython.annlowlevel import cast_instance_to_base_ptr
 
 
 class AbstractLLCPU(AbstractCPU):
@@ -243,6 +237,9 @@
     def arraydescrof(self, A):
         return get_array_descr(self.gc_ll_descr, A)
 
+    def interiorfielddescrof(self, A, fieldname):
+        return get_interiorfield_descr(self.gc_ll_descr, A, A.OF, fieldname)
+
     def unpack_arraydescr(self, arraydescr):
         assert isinstance(arraydescr, BaseArrayDescr)
         return arraydescr.get_base_size(self.translate_support_code)
@@ -360,6 +357,100 @@
     bh_getarrayitem_raw_i = bh_getarrayitem_gc_i
     bh_getarrayitem_raw_f = bh_getarrayitem_gc_f
 
+    def bh_getinteriorfield_gc_i(self, gcref, itemindex, descr):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        fieldsize = descr.fielddescr.get_field_size(self.translate_support_code)
+        sign = descr.fielddescr.is_field_signed()
+        fullofs = itemindex * size + ofs
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), fullofs)
+        for STYPE, UTYPE, itemsize in unroll_basic_sizes:
+            if fieldsize == itemsize:
+                if sign:
+                    item = rffi.cast(rffi.CArrayPtr(STYPE), items)
+                    val = item[0]
+                    val = rffi.cast(lltype.Signed, val)
+                else:
+                    item = rffi.cast(rffi.CArrayPtr(UTYPE), items)
+                    val = item[0]
+                    val = rffi.cast(lltype.Signed, val)
+                # --- end of GC unsafe code ---
+                return val
+        else:
+            raise NotImplementedError("size = %d" % fieldsize)
+
+    def bh_getinteriorfield_gc_r(self, gcref, itemindex, descr):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs +
+                            size * itemindex)
+        items = rffi.cast(rffi.CArrayPtr(lltype.Signed), items)
+        pval = self._cast_int_to_gcref(items[0])
+        # --- end of GC unsafe code ---
+        return pval
+
+    def bh_getinteriorfield_gc_f(self, gcref, itemindex, descr):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs +
+                            size * itemindex)
+        items = rffi.cast(rffi.CArrayPtr(longlong.FLOATSTORAGE), items)
+        fval = items[0]
+        # --- end of GC unsafe code ---
+        return fval
+
+    def bh_setinteriorfield_gc_i(self, gcref, itemindex, descr, value):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        fieldsize = descr.fielddescr.get_field_size(self.translate_support_code)
+        ofs = itemindex * size + ofs
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs)
+        for TYPE, _, itemsize in unroll_basic_sizes:
+            if fieldsize == itemsize:
+                items = rffi.cast(rffi.CArrayPtr(TYPE), items)
+                items[0] = rffi.cast(TYPE, value)
+                # --- end of GC unsafe code ---
+                return
+        else:
+            raise NotImplementedError("size = %d" % fieldsize)
+
+    def bh_setinteriorfield_gc_r(self, gcref, itemindex, descr, newvalue):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        self.gc_ll_descr.do_write_barrier(gcref, newvalue)
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref),
+                            ofs + size * itemindex)
+        items = rffi.cast(rffi.CArrayPtr(lltype.Signed), items)
+        items[0] = self.cast_gcref_to_int(newvalue)
+        # --- end of GC unsafe code ---
+
+    def bh_setinteriorfield_gc_f(self, gcref, itemindex, descr, newvalue):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref),
+                            ofs + size * itemindex)
+        items = rffi.cast(rffi.CArrayPtr(longlong.FLOATSTORAGE), items)
+        items[0] = newvalue
+        # --- end of GC unsafe code ---
+
     def bh_strlen(self, string):
         s = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), string)
         return len(s.chars)
@@ -477,7 +568,6 @@
 
     def bh_classof(self, struct):
         struct = lltype.cast_opaque_ptr(rclass.OBJECTPTR, struct)
-        result = struct.typeptr
         result_adr = llmemory.cast_ptr_to_adr(struct.typeptr)
         return heaptracker.adr2int(result_adr)
 
diff --git a/pypy/jit/backend/llsupport/regalloc.py b/pypy/jit/backend/llsupport/regalloc.py
--- a/pypy/jit/backend/llsupport/regalloc.py
+++ b/pypy/jit/backend/llsupport/regalloc.py
@@ -288,7 +288,6 @@
         self.reg_bindings[to_v] = reg
 
     def _move_variable_away(self, v, prev_loc):
-        reg = None
         if self.free_regs:
             loc = self.free_regs.pop()
             self.reg_bindings[v] = loc
diff --git a/pypy/jit/backend/llsupport/test/test_asmmemmgr.py b/pypy/jit/backend/llsupport/test/test_asmmemmgr.py
--- a/pypy/jit/backend/llsupport/test/test_asmmemmgr.py
+++ b/pypy/jit/backend/llsupport/test/test_asmmemmgr.py
@@ -211,14 +211,14 @@
     debug._log = debug.DebugLog()
     try:
         mc._dump(addr, 'test-logname-section')
-        log = list(debug._log)
+        log = list(debug._log) 
     finally:
         debug._log = None
     encoded = ''.join(writtencode).encode('hex').upper()
     ataddr = '@%x' % addr
     assert log == [('test-logname-section',
                     [('debug_print', 'CODE_DUMP', ataddr, '+0 ', encoded)])]
-    #
+    
     lltype.free(p, flavor='raw')
 
 def test_blockbuildermixin2():
diff --git a/pypy/jit/backend/llsupport/test/test_descr.py b/pypy/jit/backend/llsupport/test/test_descr.py
--- a/pypy/jit/backend/llsupport/test/test_descr.py
+++ b/pypy/jit/backend/llsupport/test/test_descr.py
@@ -3,7 +3,6 @@
 from pypy.jit.backend.llsupport import symbolic
 from pypy.rlib.objectmodel import Symbolic
 from pypy.rpython.annlowlevel import llhelper
-from pypy.jit.metainterp.history import BoxInt, BoxFloat, BoxPtr
 from pypy.jit.metainterp import history
 from pypy.jit.codewriter import longlong
 import sys, struct, py
@@ -149,7 +148,9 @@
     A2 = lltype.GcArray(lltype.Ptr(T))
     A3 = lltype.GcArray(lltype.Ptr(U))
     A4 = lltype.GcArray(lltype.Float)
-    A5 = lltype.GcArray(lltype.SingleFloat)
+    A5 = lltype.GcArray(lltype.Struct('x', ('v', lltype.Signed),
+                                      ('k', lltype.Signed)))
+    A6 = lltype.GcArray(lltype.SingleFloat)
     assert getArrayDescrClass(A2) is GcPtrArrayDescr
     assert getArrayDescrClass(A3) is NonGcPtrArrayDescr
     cls = getArrayDescrClass(A1)
@@ -158,7 +159,7 @@
     clsf = getArrayDescrClass(A4)
     assert clsf != cls
     assert clsf == getArrayDescrClass(lltype.GcArray(lltype.Float))
-    clss = getArrayDescrClass(A5)
+    clss = getArrayDescrClass(A6)
     assert clss not in (clsf, cls)
     assert clss == getArrayDescrClass(lltype.GcArray(rffi.UINT))
     #
@@ -168,11 +169,12 @@
     descr3 = get_array_descr(c0, A3)
     descr4 = get_array_descr(c0, A4)
     descr5 = get_array_descr(c0, A5)
+    descr6 = get_array_descr(c0, A6)
     assert descr1.__class__ is cls
     assert descr2.__class__ is GcPtrArrayDescr
     assert descr3.__class__ is NonGcPtrArrayDescr
     assert descr4.__class__ is clsf
-    assert descr5.__class__ is clss
+    assert descr6.__class__ is clss
     assert descr1 == get_array_descr(c0, lltype.GcArray(lltype.Char))
     assert not descr1.is_array_of_pointers()
     assert     descr2.is_array_of_pointers()
@@ -202,7 +204,8 @@
     assert descr2.get_item_size(False) == rffi.sizeof(lltype.Ptr(T))
     assert descr3.get_item_size(False) == rffi.sizeof(lltype.Ptr(U))
     assert descr4.get_item_size(False) == rffi.sizeof(lltype.Float)
-    assert descr5.get_item_size(False) == rffi.sizeof(lltype.SingleFloat)
+    assert descr5.get_item_size(False) == rffi.sizeof(lltype.Signed) * 2
+    assert descr6.get_item_size(False) == rffi.sizeof(lltype.SingleFloat)
     #
     assert isinstance(descr1.get_base_size(True), Symbolic)
     assert isinstance(descr2.get_base_size(True), Symbolic)
@@ -348,7 +351,6 @@
                             (rffi.SHORT,      True), (rffi.USHORT, False),
                             (rffi.INT,        True), (rffi.UINT,   False),
                             (rffi.LONG,       True), (rffi.ULONG,  False)]:
-        A = lltype.GcArray(RESTYPE)
         for tsc in [False, True]:
             c2 = GcCache(tsc)
             descr1 = get_call_descr(c2, [], RESTYPE)
@@ -379,7 +381,6 @@
     descr3i = get_array_descr(c0, lltype.GcArray(lltype.Char))
     assert descr3i.repr_of_descr() == '<CharArrayDescr>'
     #
-    cache = {}
     descr4 = get_call_descr(c0, [lltype.Char, lltype.Ptr(S)], lltype.Ptr(S))
     assert 'GcPtrCallDescr' in descr4.repr_of_descr()
     #
@@ -412,10 +413,10 @@
     ARGS = [lltype.Float, lltype.Ptr(ARRAY)]
     RES = lltype.Float
 
-    def f(a, b):
+    def f2(a, b):
         return float(b[0]) + a
 
-    fnptr = llhelper(lltype.Ptr(lltype.FuncType(ARGS, RES)), f)
+    fnptr = llhelper(lltype.Ptr(lltype.FuncType(ARGS, RES)), f2)
     descr2 = get_call_descr(c0, ARGS, RES)
     a = lltype.malloc(ARRAY, 3)
     opaquea = lltype.cast_opaque_ptr(llmemory.GCREF, a)
diff --git a/pypy/jit/backend/model.py b/pypy/jit/backend/model.py
--- a/pypy/jit/backend/model.py
+++ b/pypy/jit/backend/model.py
@@ -1,5 +1,5 @@
 from pypy.rlib.debug import debug_start, debug_print, debug_stop
-from pypy.jit.metainterp import history, compile
+from pypy.jit.metainterp import history
 
 
 class AbstractCPU(object):
@@ -218,6 +218,10 @@
     def typedescrof(TYPE):
         raise NotImplementedError
 
+    @staticmethod
+    def interiorfielddescrof(A, fieldname):
+        raise NotImplementedError
+
     # ---------- the backend-dependent operations ----------
 
     # lltype specific operations
diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -143,7 +143,7 @@
             ]
         inputargs = [i0]
         operations[2].setfailargs([i1])
-        
+
         self.cpu.compile_loop(inputargs, operations, looptoken)
         self.cpu.set_future_value_int(0, 2)
         fail = self.cpu.execute_token(looptoken)
@@ -164,7 +164,7 @@
             ]
         inputargs = [i0]
         operations[2].setfailargs([None, None, i1, None])
-        
+
         self.cpu.compile_loop(inputargs, operations, looptoken)
         self.cpu.set_future_value_int(0, 2)
         fail = self.cpu.execute_token(looptoken)
@@ -388,7 +388,7 @@
         for opnum, boxargs, retvalue in get_int_tests():
             res = self.execute_operation(opnum, boxargs, 'int')
             assert res.value == retvalue
-        
+
     def test_float_operations(self):
         from pypy.jit.metainterp.test.test_executor import get_float_tests
         for opnum, boxargs, rettype, retvalue in get_float_tests(self.cpu):
@@ -454,7 +454,7 @@
 
     def test_ovf_operations_reversed(self):
         self.test_ovf_operations(reversed=True)
-        
+
     def test_bh_call(self):
         cpu = self.cpu
         #
@@ -566,7 +566,7 @@
         funcbox = self.get_funcbox(self.cpu, func_ptr)
         res = self.execute_operation(rop.CALL, [funcbox] + map(BoxInt, args), 'int', descr=calldescr)
         assert res.value == func(*args)
-        
+
     def test_call_stack_alignment(self):
         # test stack alignment issues, notably for Mac OS/X.
         # also test the ordering of the arguments.
@@ -640,7 +640,7 @@
         res = self.execute_operation(rop.GETFIELD_GC, [t_box],
                                      'int', descr=shortdescr)
         assert res.value == 1331
-        
+
         #
         u_box, U_box = self.alloc_instance(self.U)
         fielddescr2 = self.cpu.fielddescrof(self.S, 'next')
@@ -720,7 +720,7 @@
 
     def test_failing_guard_class(self):
         t_box, T_box = self.alloc_instance(self.T)
-        u_box, U_box = self.alloc_instance(self.U)        
+        u_box, U_box = self.alloc_instance(self.U)
         null_box = self.null_instance()
         for opname, args in [(rop.GUARD_CLASS, [t_box, U_box]),
                              (rop.GUARD_CLASS, [u_box, T_box]),
@@ -812,7 +812,7 @@
         r = self.execute_operation(rop.GETARRAYITEM_GC, [a_box, BoxInt(3)],
                                    'int', descr=arraydescr)
         assert r.value == 160
-        
+
         #
         if isinstance(A, lltype.GcArray):
             A = lltype.Ptr(A)
@@ -905,6 +905,73 @@
                                    'int', descr=arraydescr)
         assert r.value == 7441
 
+    def test_array_of_structs(self):
+        TP = lltype.GcStruct('x')
+        ITEM = lltype.Struct('x',
+                             ('vs', lltype.Signed),
+                             ('vu', lltype.Unsigned),
+                             ('vsc', rffi.SIGNEDCHAR),
+                             ('vuc', rffi.UCHAR),
+                             ('vss', rffi.SHORT),
+                             ('vus', rffi.USHORT),
+                             ('vsi', rffi.INT),
+                             ('vui', rffi.UINT),
+                             ('k', lltype.Float),
+                             ('p', lltype.Ptr(TP)))
+        a_box, A = self.alloc_array_of(ITEM, 15)
+        s_box, S = self.alloc_instance(TP)
+        kdescr = self.cpu.interiorfielddescrof(A, 'k')
+        pdescr = self.cpu.interiorfielddescrof(A, 'p')
+        self.execute_operation(rop.SETINTERIORFIELD_GC, [a_box, BoxInt(3),
+                                                         boxfloat(1.5)],
+                               'void', descr=kdescr)
+        f = self.cpu.bh_getinteriorfield_gc_f(a_box.getref_base(), 3, kdescr)
+        assert longlong.getrealfloat(f) == 1.5
+        self.cpu.bh_setinteriorfield_gc_f(a_box.getref_base(), 3, kdescr, longlong.getfloatstorage(2.5))
+        r = self.execute_operation(rop.GETINTERIORFIELD_GC, [a_box, BoxInt(3)],
+                                   'float', descr=kdescr)
+        assert r.getfloat() == 2.5
+        #
+        NUMBER_FIELDS = [('vs', lltype.Signed),
+                         ('vu', lltype.Unsigned),
+                         ('vsc', rffi.SIGNEDCHAR),
+                         ('vuc', rffi.UCHAR),
+                         ('vss', rffi.SHORT),
+                         ('vus', rffi.USHORT),
+                         ('vsi', rffi.INT),
+                         ('vui', rffi.UINT)]
+        for name, TYPE in NUMBER_FIELDS[::-1]:
+            vdescr = self.cpu.interiorfielddescrof(A, name)
+            self.execute_operation(rop.SETINTERIORFIELD_GC, [a_box, BoxInt(3),
+                                                             BoxInt(-15)],
+                                   'void', descr=vdescr)
+        for name, TYPE in NUMBER_FIELDS:
+            vdescr = self.cpu.interiorfielddescrof(A, name)
+            i = self.cpu.bh_getinteriorfield_gc_i(a_box.getref_base(), 3,
+                                                  vdescr)
+            assert i == rffi.cast(lltype.Signed, rffi.cast(TYPE, -15))
+        for name, TYPE in NUMBER_FIELDS[::-1]:
+            vdescr = self.cpu.interiorfielddescrof(A, name)
+            self.cpu.bh_setinteriorfield_gc_i(a_box.getref_base(), 3,
+                                              vdescr, -25)
+        for name, TYPE in NUMBER_FIELDS:
+            vdescr = self.cpu.interiorfielddescrof(A, name)
+            r = self.execute_operation(rop.GETINTERIORFIELD_GC,
+                                       [a_box, BoxInt(3)],
+                                       'int', descr=vdescr)
+            assert r.getint() == rffi.cast(lltype.Signed, rffi.cast(TYPE, -25))
+        #
+        self.execute_operation(rop.SETINTERIORFIELD_GC, [a_box, BoxInt(4),
+                                                         s_box],
+                               'void', descr=pdescr)
+        r = self.cpu.bh_getinteriorfield_gc_r(a_box.getref_base(), 4, pdescr)
+        assert r == s_box.getref_base()
+        self.cpu.bh_setinteriorfield_gc_r(a_box.getref_base(), 3, pdescr,
+                                          s_box.getref_base())
+        r = self.execute_operation(rop.GETINTERIORFIELD_GC, [a_box, BoxInt(3)],
+                                   'ref', descr=pdescr)
+        assert r.getref_base() == s_box.getref_base()
+
     def test_string_basic(self):
         s_box = self.alloc_string("hello\xfe")
         r = self.execute_operation(rop.STRLEN, [s_box], 'int')
@@ -1429,7 +1496,7 @@
         addr = llmemory.cast_ptr_to_adr(func_ptr)
         return ConstInt(heaptracker.adr2int(addr))
 
-    
+
     MY_VTABLE = rclass.OBJECT_VTABLE    # for tests only
 
     S = lltype.GcForwardReference()
@@ -1495,20 +1562,16 @@
         return u''.join(u.chars)
 
 
-    def test_casts(self):
-        py.test.skip("xxx fix or kill")
-        from pypy.rpython.lltypesystem import lltype, llmemory
-        TP = lltype.GcStruct('x')
-        x = lltype.malloc(TP)        
-        x = lltype.cast_opaque_ptr(llmemory.GCREF, x)
+    def test_cast_int_to_ptr(self):
+        res = self.execute_operation(rop.CAST_INT_TO_PTR,
+                                     [BoxInt(-17)],  'ref').value
+        assert lltype.cast_ptr_to_int(res) == -17
+
+    def test_cast_ptr_to_int(self):
+        x = lltype.cast_int_to_ptr(llmemory.GCREF, -19)
         res = self.execute_operation(rop.CAST_PTR_TO_INT,
-                                     [BoxPtr(x)],  'int').value
-        expected = self.cpu.cast_adr_to_int(llmemory.cast_ptr_to_adr(x))
-        assert rffi.get_real_int(res) == rffi.get_real_int(expected)
-        res = self.execute_operation(rop.CAST_PTR_TO_INT,
-                                     [ConstPtr(x)],  'int').value
-        expected = self.cpu.cast_adr_to_int(llmemory.cast_ptr_to_adr(x))
-        assert rffi.get_real_int(res) == rffi.get_real_int(expected)
+                                     [BoxPtr(x)], 'int').value
+        assert res == -19
 
     def test_ooops_non_gc(self):
         x = lltype.malloc(lltype.Struct('x'), flavor='raw')
@@ -2328,13 +2391,6 @@
         #
         cpu.bh_strsetitem(x, 4, ord('/'))
         assert str.chars[4] == '/'
-        #
-##        x = cpu.bh_newstr(5)
-##        y = cpu.bh_cast_ptr_to_int(x)
-##        z = cpu.bh_cast_ptr_to_int(x)
-##        y = rffi.get_real_int(y)
-##        z = rffi.get_real_int(z)
-##        assert type(y) == type(z) == int and y == z
 
     def test_sorting_of_fields(self):
         S = self.S
@@ -2358,7 +2414,7 @@
         for opname, arg, res in ops:
             self.execute_operation(opname, [arg], 'void')
             assert self.guard_failed == res
-        
+
         lltype.free(x, flavor='raw')
 
     def test_assembler_call(self):
@@ -3028,4 +3084,4 @@
 
     def alloc_unicode(self, unicode):
         py.test.skip("implement me")
-    
+
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1,7 +1,7 @@
 import sys, os
 from pypy.jit.backend.llsupport import symbolic
 from pypy.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper
-from pypy.jit.metainterp.history import Const, Box, BoxInt, BoxPtr, BoxFloat
+from pypy.jit.metainterp.history import Const, Box, BoxInt, ConstInt
 from pypy.jit.metainterp.history import (AbstractFailDescr, INT, REF, FLOAT,
                                          LoopToken)
 from pypy.rpython.lltypesystem import lltype, rffi, rstr, llmemory
@@ -36,7 +36,6 @@
 from pypy.rlib import rgc
 from pypy.rlib.clibffi import FFI_DEFAULT_ABI
 from pypy.jit.backend.x86.jump import remap_frame_layout
-from pypy.jit.metainterp.history import ConstInt, BoxInt
 from pypy.jit.codewriter.effectinfo import EffectInfo
 from pypy.jit.codewriter import longlong
 
@@ -729,8 +728,8 @@
         # Also, make sure this is consistent with FRAME_FIXED_SIZE.
         self.mc.PUSH_r(ebp.value)
         self.mc.MOV_rr(ebp.value, esp.value)
-        for regloc in self.cpu.CALLEE_SAVE_REGISTERS:
-            self.mc.PUSH_r(regloc.value)
+        for loc in self.cpu.CALLEE_SAVE_REGISTERS:
+            self.mc.PUSH_r(loc.value)
 
         gcrootmap = self.cpu.gc_ll_descr.gcrootmap
         if gcrootmap and gcrootmap.is_shadow_stack:
@@ -994,7 +993,7 @@
         effectinfo = op.getdescr().get_extra_info()
         oopspecindex = effectinfo.oopspecindex
         genop_llong_list[oopspecindex](self, op, arglocs, resloc)
-        
+
     def regalloc_perform_math(self, op, arglocs, resloc):
         effectinfo = op.getdescr().get_extra_info()
         oopspecindex = effectinfo.oopspecindex
@@ -1311,7 +1310,7 @@
     genop_guard_float_eq = _cmpop_guard_float("E", "E", "NE","NE")
     genop_guard_float_gt = _cmpop_guard_float("A", "B", "BE","AE")
     genop_guard_float_ge = _cmpop_guard_float("AE","BE", "B", "A")
-    
+
     def genop_math_sqrt(self, op, arglocs, resloc):
         self.mc.SQRTSD(arglocs[0], resloc)
 
@@ -1387,7 +1386,8 @@
 
     def genop_same_as(self, op, arglocs, resloc):
         self.mov(arglocs[0], resloc)
-    #genop_cast_ptr_to_int = genop_same_as
+    genop_cast_ptr_to_int = genop_same_as
+    genop_cast_int_to_ptr = genop_same_as
 
     def genop_int_mod(self, op, arglocs, resloc):
         if IS_X86_32:
@@ -1596,12 +1596,27 @@
     genop_getarrayitem_gc_pure = genop_getarrayitem_gc
     genop_getarrayitem_raw = genop_getarrayitem_gc
 
+    def genop_getinteriorfield_gc(self, op, arglocs, resloc):
+        base_loc, ofs_loc, itemsize_loc, fieldsize_loc, index_loc, sign_loc = arglocs
+        # XXX should not use IMUL in most cases
+        self.mc.IMUL(index_loc, itemsize_loc)
+        src_addr = AddressLoc(base_loc, index_loc, 0, ofs_loc.value)
+        self.load_from_mem(resloc, src_addr, fieldsize_loc, sign_loc)
+
+
     def genop_discard_setfield_gc(self, op, arglocs):
         base_loc, ofs_loc, size_loc, value_loc = arglocs
         assert isinstance(size_loc, ImmedLoc)
         dest_addr = AddressLoc(base_loc, ofs_loc)
         self.save_into_mem(dest_addr, value_loc, size_loc)
 
+    def genop_discard_setinteriorfield_gc(self, op, arglocs):
+        base_loc, ofs_loc, itemsize_loc, fieldsize_loc, index_loc, value_loc = arglocs
+        # XXX should not use IMUL in most cases
+        self.mc.IMUL(index_loc, itemsize_loc)
+        dest_addr = AddressLoc(base_loc, index_loc, 0, ofs_loc.value)
+        self.save_into_mem(dest_addr, value_loc, fieldsize_loc)
+
     def genop_discard_setarrayitem_gc(self, op, arglocs):
         base_loc, ofs_loc, value_loc, size_loc, baseofs = arglocs
         assert isinstance(baseofs, ImmedLoc)
diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py
--- a/pypy/jit/backend/x86/regalloc.py
+++ b/pypy/jit/backend/x86/regalloc.py
@@ -7,7 +7,7 @@
                                          ResOperation, BoxPtr, ConstFloat,
                                          BoxFloat, LoopToken, INT, REF, FLOAT)
 from pypy.jit.backend.x86.regloc import *
-from pypy.rpython.lltypesystem import lltype, ll2ctypes, rffi, rstr
+from pypy.rpython.lltypesystem import lltype, rffi, rstr
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.rlib import rgc
 from pypy.jit.backend.llsupport import symbolic
@@ -17,11 +17,12 @@
 from pypy.jit.metainterp.resoperation import rop
 from pypy.jit.backend.llsupport.descr import BaseFieldDescr, BaseArrayDescr
 from pypy.jit.backend.llsupport.descr import BaseCallDescr, BaseSizeDescr
+from pypy.jit.backend.llsupport.descr import InteriorFieldDescr
 from pypy.jit.backend.llsupport.regalloc import FrameManager, RegisterManager,\
      TempBox, compute_vars_longevity, compute_loop_consts
 from pypy.jit.backend.x86.arch import WORD, FRAME_FIXED_SIZE
 from pypy.jit.backend.x86.arch import IS_X86_32, IS_X86_64, MY_COPY_OF_REGS
-from pypy.rlib.rarithmetic import r_longlong, r_uint
+from pypy.rlib.rarithmetic import r_longlong
 
 class X86RegisterManager(RegisterManager):
 
@@ -423,7 +424,7 @@
             if self.can_merge_with_next_guard(op, i, operations):
                 oplist_with_guard[op.getopnum()](self, op, operations[i + 1])
                 i += 1
-            elif not we_are_translated() and op.getopnum() == -124: 
+            elif not we_are_translated() and op.getopnum() == -124:
                 self._consider_force_spill(op)
             else:
                 oplist[op.getopnum()](self, op)
@@ -765,7 +766,7 @@
         save_all_regs = guard_not_forced_op is not None
         self.xrm.before_call(force_store, save_all_regs=save_all_regs)
         if not save_all_regs:
-            gcrootmap = gc_ll_descr = self.assembler.cpu.gc_ll_descr.gcrootmap
+            gcrootmap = self.assembler.cpu.gc_ll_descr.gcrootmap
             if gcrootmap and gcrootmap.is_shadow_stack:
                 save_all_regs = 2
         self.rm.before_call(force_store, save_all_regs=save_all_regs)
@@ -922,74 +923,27 @@
             return self._call(op, arglocs)
 
     def consider_newstr(self, op):
-        gc_ll_descr = self.assembler.cpu.gc_ll_descr
-        if gc_ll_descr.get_funcptr_for_newstr is not None:
-            # framework GC
-            loc = self.loc(op.getarg(0))
-            return self._call(op, [loc])
-        # boehm GC (XXX kill the following code at some point)
-        ofs_items, itemsize, ofs = symbolic.get_array_token(rstr.STR, self.translate_support_code)
-        assert itemsize == 1
-        return self._malloc_varsize(ofs_items, ofs, 0, op.getarg(0),
-                                    op.result)
+        loc = self.loc(op.getarg(0))
+        return self._call(op, [loc])
 
     def consider_newunicode(self, op):
-        gc_ll_descr = self.assembler.cpu.gc_ll_descr
-        if gc_ll_descr.get_funcptr_for_newunicode is not None:
-            # framework GC
-            loc = self.loc(op.getarg(0))
-            return self._call(op, [loc])
-        # boehm GC (XXX kill the following code at some point)
-        ofs_items, _, ofs = symbolic.get_array_token(rstr.UNICODE,
-                                                   self.translate_support_code)
-        scale = self._get_unicode_item_scale()
-        return self._malloc_varsize(ofs_items, ofs, scale, op.getarg(0),
-                                    op.result)
-
-    def _malloc_varsize(self, ofs_items, ofs_length, scale, v, res_v):
-        # XXX kill this function at some point
-        if isinstance(v, Box):
-            loc = self.rm.make_sure_var_in_reg(v, [v])
-            tempbox = TempBox()
-            other_loc = self.rm.force_allocate_reg(tempbox, [v])
-            self.assembler.load_effective_addr(loc, ofs_items,scale, other_loc)
-        else:
-            tempbox = None
-            other_loc = imm(ofs_items + (v.getint() << scale))
-        self._call(ResOperation(rop.NEW, [], res_v),
-                   [other_loc], [v])
-        loc = self.rm.make_sure_var_in_reg(v, [res_v])
-        assert self.loc(res_v) == eax
-        # now we have to reload length to some reasonable place
-        self.rm.possibly_free_var(v)
-        if tempbox is not None:
-            self.rm.possibly_free_var(tempbox)
-        self.PerformDiscard(ResOperation(rop.SETFIELD_GC, [None, None], None),
-                            [eax, imm(ofs_length), imm(WORD), loc])
+        loc = self.loc(op.getarg(0))
+        return self._call(op, [loc])
 
     def consider_new_array(self, op):
         gc_ll_descr = self.assembler.cpu.gc_ll_descr
-        if gc_ll_descr.get_funcptr_for_newarray is not None:
-            # framework GC
-            box_num_elem = op.getarg(0)
-            if isinstance(box_num_elem, ConstInt):
-                num_elem = box_num_elem.value
-                if gc_ll_descr.can_inline_malloc_varsize(op.getdescr(),
-                                                         num_elem):
-                    self.fastpath_malloc_varsize(op, op.getdescr(), num_elem)
-                    return
-            args = self.assembler.cpu.gc_ll_descr.args_for_new_array(
-                op.getdescr())
-            arglocs = [imm(x) for x in args]
-            arglocs.append(self.loc(box_num_elem))
-            self._call(op, arglocs)
-            return
-        # boehm GC (XXX kill the following code at some point)
-        itemsize, basesize, ofs_length, _, _ = (
-            self._unpack_arraydescr(op.getdescr()))
-        scale_of_field = _get_scale(itemsize)
-        self._malloc_varsize(basesize, ofs_length, scale_of_field,
-                             op.getarg(0), op.result)
+        box_num_elem = op.getarg(0)
+        if isinstance(box_num_elem, ConstInt):
+            num_elem = box_num_elem.value
+            if gc_ll_descr.can_inline_malloc_varsize(op.getdescr(),
+                                                     num_elem):
+                self.fastpath_malloc_varsize(op, op.getdescr(), num_elem)
+                return
+        args = self.assembler.cpu.gc_ll_descr.args_for_new_array(
+            op.getdescr())
+        arglocs = [imm(x) for x in args]
+        arglocs.append(self.loc(box_num_elem))
+        self._call(op, arglocs)
 
     def _unpack_arraydescr(self, arraydescr):
         assert isinstance(arraydescr, BaseArrayDescr)
@@ -1008,6 +962,16 @@
         sign = fielddescr.is_field_signed()
         return imm(ofs), imm(size), ptr, sign
 
+    def _unpack_interiorfielddescr(self, descr):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs = arraydescr.get_base_size(self.translate_support_code)
+        itemsize = arraydescr.get_item_size(self.translate_support_code)
+        fieldsize = descr.fielddescr.get_field_size(self.translate_support_code)
+        sign = descr.fielddescr.is_field_signed()
+        ofs += descr.fielddescr.offset
+        return imm(ofs), imm(itemsize), imm(fieldsize), sign
+
     def consider_setfield_gc(self, op):
         ofs_loc, size_loc, _, _ = self._unpack_fielddescr(op.getdescr())
         assert isinstance(size_loc, ImmedLoc)
@@ -1024,6 +988,21 @@
 
     consider_setfield_raw = consider_setfield_gc
 
+    def consider_setinteriorfield_gc(self, op):
+        t = self._unpack_interiorfielddescr(op.getdescr())
+        ofs, itemsize, fieldsize, _ = t
+        args = op.getarglist()
+        tmpvar = TempBox()
+        base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
+        index_loc = self.rm.force_result_in_reg(tmpvar, op.getarg(1),
+                                                args)
+        # we're free to modify index now
+        value_loc = self.make_sure_var_in_reg(op.getarg(2), args)
+        self.possibly_free_vars(args)
+        self.rm.possibly_free_var(tmpvar)
+        self.PerformDiscard(op, [base_loc, ofs, itemsize, fieldsize,
+                                 index_loc, value_loc])
+
     def consider_strsetitem(self, op):
         args = op.getarglist()
         base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
@@ -1085,6 +1064,24 @@
     consider_getarrayitem_raw = consider_getarrayitem_gc
     consider_getarrayitem_gc_pure = consider_getarrayitem_gc
 
+    def consider_getinteriorfield_gc(self, op):
+        t = self._unpack_interiorfielddescr(op.getdescr())
+        ofs, itemsize, fieldsize, sign = t
+        if sign:
+            sign_loc = imm1
+        else:
+            sign_loc = imm0
+        args = op.getarglist()
+        tmpvar = TempBox()
+        base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
+        index_loc = self.rm.force_result_in_reg(tmpvar, op.getarg(1),
+                                                args)
+        self.rm.possibly_free_vars_for_op(op)
+        self.rm.possibly_free_var(tmpvar)
+        result_loc = self.force_allocate_reg(op.result)
+        self.Perform(op, [base_loc, ofs, itemsize, fieldsize,
+                          index_loc, sign_loc], result_loc)
+
     def consider_int_is_true(self, op, guard_op):
         # doesn't need arg to be in a register
         argloc = self.loc(op.getarg(0))
@@ -1102,7 +1099,8 @@
         self.possibly_free_var(op.getarg(0))
         resloc = self.force_allocate_reg(op.result)
         self.Perform(op, [argloc], resloc)
-    #consider_cast_ptr_to_int = consider_same_as
+    consider_cast_ptr_to_int = consider_same_as
+    consider_cast_int_to_ptr = consider_same_as
 
     def consider_strlen(self, op):
         args = op.getarglist()
@@ -1190,7 +1188,6 @@
         self.rm.possibly_free_var(srcaddr_box)
 
     def _gen_address_inside_string(self, baseloc, ofsloc, resloc, is_unicode):
-        cpu = self.assembler.cpu
         if is_unicode:
             ofs_items, _, _ = symbolic.get_array_token(rstr.UNICODE,
                                                   self.translate_support_code)
@@ -1249,7 +1246,7 @@
         tmpreg = X86RegisterManager.all_regs[0]
         tmploc = self.rm.force_allocate_reg(box, selected_reg=tmpreg)
         xmmtmp = X86XMMRegisterManager.all_regs[0]
-        xmmtmploc = self.xrm.force_allocate_reg(box1, selected_reg=xmmtmp)
+        self.xrm.force_allocate_reg(box1, selected_reg=xmmtmp)
         # Part about non-floats
         # XXX we don't need a copy, we only just the original list
         src_locations1 = [self.loc(op.getarg(i)) for i in range(op.numargs())
@@ -1329,7 +1326,7 @@
     return lambda self, op: fn(self, op, None)
 
 def is_comparison_or_ovf_op(opnum):
-    from pypy.jit.metainterp.resoperation import opclasses, AbstractResOp
+    from pypy.jit.metainterp.resoperation import opclasses
     cls = opclasses[opnum]
     # hack hack: in theory they are instance method, but they don't use
     # any instance field, we can use a fake object
diff --git a/pypy/jit/backend/x86/test/test_del.py b/pypy/jit/backend/x86/test/test_del.py
--- a/pypy/jit/backend/x86/test/test_del.py
+++ b/pypy/jit/backend/x86/test/test_del.py
@@ -1,5 +1,4 @@
 
-import py
 from pypy.jit.backend.x86.test.test_basic import Jit386Mixin
 from pypy.jit.metainterp.test.test_del import DelTests
 
diff --git a/pypy/jit/backend/x86/test/test_dict.py b/pypy/jit/backend/x86/test/test_dict.py
new file mode 100644
--- /dev/null
+++ b/pypy/jit/backend/x86/test/test_dict.py
@@ -0,0 +1,9 @@
+
+from pypy.jit.backend.x86.test.test_basic import Jit386Mixin
+from pypy.jit.metainterp.test.test_dict import DictTests
+
+
+class TestDict(Jit386Mixin, DictTests):
+    # for the individual tests see
+    # ====> ../../../metainterp/test/test_dict.py
+    pass
diff --git a/pypy/jit/backend/x86/test/test_runner.py b/pypy/jit/backend/x86/test/test_runner.py
--- a/pypy/jit/backend/x86/test/test_runner.py
+++ b/pypy/jit/backend/x86/test/test_runner.py
@@ -31,7 +31,7 @@
 
     # for the individual tests see
     # ====> ../../test/runner_test.py
-    
+
     def setup_method(self, meth):
         self.cpu = CPU(rtyper=None, stats=FakeStats())
         self.cpu.setup_once()
@@ -69,22 +69,16 @@
 
     def test_allocations(self):
         from pypy.rpython.lltypesystem import rstr
-        
+
         allocs = [None]
         all = []
+        orig_new = self.cpu.gc_ll_descr.funcptr_for_new
         def f(size):
             allocs.insert(0, size)
-            buf = ctypes.create_string_buffer(size)
-            all.append(buf)
-            return ctypes.cast(buf, ctypes.c_void_p).value
-        func = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int)(f)
-        addr = ctypes.cast(func, ctypes.c_void_p).value
-        # ctypes produces an unsigned value. We need it to be signed for, eg,
-        # relative addressing to work properly.
-        addr = rffi.cast(lltype.Signed, addr)
-        
+            return orig_new(size)
+
         self.cpu.assembler.setup_once()
-        self.cpu.assembler.malloc_func_addr = addr
+        self.cpu.gc_ll_descr.funcptr_for_new = f
         ofs = symbolic.get_field_token(rstr.STR, 'chars', False)[0]
 
         res = self.execute_operation(rop.NEWSTR, [ConstInt(7)], 'ref')
@@ -108,7 +102,7 @@
         res = self.execute_operation(rop.NEW_ARRAY, [ConstInt(10)],
                                          'ref', descr)
         assert allocs[0] == 10*WORD + ofs + WORD
-        resbuf = self._resbuf(res)            
+        resbuf = self._resbuf(res)
         assert resbuf[ofs/WORD] == 10
 
         # ------------------------------------------------------------
@@ -116,7 +110,7 @@
         res = self.execute_operation(rop.NEW_ARRAY, [BoxInt(10)],
                                          'ref', descr)
         assert allocs[0] == 10*WORD + ofs + WORD
-        resbuf = self._resbuf(res)                        
+        resbuf = self._resbuf(res)
         assert resbuf[ofs/WORD] == 10
 
     def test_stringitems(self):
@@ -146,7 +140,7 @@
                                                      ConstInt(2), BoxInt(38)],
                                'void', descr)
         assert resbuf[itemsofs/WORD + 2] == 38
-        
+
         self.execute_operation(rop.SETARRAYITEM_GC, [res,
                                                      BoxInt(3), BoxInt(42)],
                                'void', descr)
@@ -167,7 +161,7 @@
                                                          BoxInt(2)],
                                    'int', descr)
         assert r.value == 38
-        
+
         r = self.execute_operation(rop.GETARRAYITEM_GC, [res, BoxInt(3)],
                                    'int', descr)
         assert r.value == 42
@@ -226,7 +220,7 @@
         self.execute_operation(rop.SETFIELD_GC, [res, BoxInt(1234)], 'void', ofs_i)
         i = self.execute_operation(rop.GETFIELD_GC, [res], 'int', ofs_i)
         assert i.value == 1234
-        
+
         #u = self.execute_operation(rop.GETFIELD_GC, [res, ofs_u], 'int')
         #assert u.value == 5
         self.execute_operation(rop.SETFIELD_GC, [res, ConstInt(1)], 'void',
@@ -299,7 +293,7 @@
                     else:
                         assert result != execute(self.cpu, None,
                                                  op, None, b).value
-                    
+
 
     def test_stuff_followed_by_guard(self):
         boxes = [(BoxInt(1), BoxInt(0)),
@@ -523,7 +517,7 @@
     def test_debugger_on(self):
         from pypy.tool.logparser import parse_log_file, extract_category
         from pypy.rlib import debug
-        
+
         loop = """
         [i0]
         debug_merge_point('xyz', 0)
diff --git a/pypy/jit/backend/x86/tool/viewcode.py b/pypy/jit/backend/x86/tool/viewcode.py
--- a/pypy/jit/backend/x86/tool/viewcode.py
+++ b/pypy/jit/backend/x86/tool/viewcode.py
@@ -9,7 +9,12 @@
 """
 
 import autopath
-import operator, sys, os, re, py, new
+import new
+import operator
+import py
+import re
+import sys
+import subprocess
 from bisect import bisect_left
 
 # don't use pypy.tool.udir here to avoid removing old usessions which
@@ -44,14 +49,16 @@
     f = open(tmpfile, 'wb')
     f.write(data)
     f.close()
-    g = os.popen(objdump % {
+    p = subprocess.Popen(objdump % {
         'file': tmpfile,
         'origin': originaddr,
         'backend': objdump_backend_option[backend_name],
-    }, 'r')
-    result = g.readlines()
-    g.close()
-    lines = result[6:]   # drop some objdump cruft
+    }, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    stdout, stderr = p.communicate()
+    assert not p.returncode, ('Encountered an error running objdump: %s' %
+                              stderr)
+    # drop some objdump cruft
+    lines = stdout.splitlines()[6:]
     return format_code_dump_with_labels(originaddr, lines, label_list)
 
 def format_code_dump_with_labels(originaddr, lines, label_list):
@@ -85,8 +92,12 @@
     #
     print 'loading symbols from %s...' % (filename,)
     symbols = {}
-    g = os.popen(symbollister % filename, "r")
-    for line in g:
+    p = subprocess.Popen(symbollister % filename, shell=True,
+                         stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    stdout, stderr = p.communicate()
+    assert not p.returncode, ('Encountered an error running nm: %s' %
+                              stderr)
+    for line in stdout.splitlines():
         match = re_symbolentry.match(line)
         if match:
             addr = long(match.group(1), 16)
@@ -94,7 +105,6 @@
             if name.startswith('pypy_g_'):
                 name = '\xb7' + name[7:]
             symbols[addr] = name
-    g.close()
     print '%d symbols found' % (len(symbols),)
     return symbols
 
diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -52,9 +52,11 @@
         newoperations = []
         #
         def do_rename(var, var_or_const):
+            if var.concretetype is lltype.Void:
+                renamings[var] = Constant(None, lltype.Void)
+                return
             renamings[var] = var_or_const
-            if (isinstance(var_or_const, Constant)
-                and var.concretetype != lltype.Void):
+            if isinstance(var_or_const, Constant):
                 value = var_or_const.value
                 value = lltype._cast_whatever(var.concretetype, value)
                 renamings_constants[var] = Constant(value, var.concretetype)
@@ -455,6 +457,23 @@
             # the special return value None forces op.result to be considered
             # equal to op.args[0]
             return [op0, op1, None]
+        if (hints.get('promote_string') and
+            op.args[0].concretetype is not lltype.Void):
+            S = lltype.Ptr(rstr.STR)
+            assert op.args[0].concretetype == S
+            self._register_extra_helper(EffectInfo.OS_STREQ_NONNULL,
+                                        "str.eq_nonnull",
+                                        [S, S],
+                                        lltype.Signed,
+                                        EffectInfo.EF_ELIDABLE_CANNOT_RAISE)
+            descr, p = self.callcontrol.callinfocollection.callinfo_for_oopspec(
+                EffectInfo.OS_STREQ_NONNULL)
+            # XXX this is fairly ugly way of creating a constant,
+            #     however, callinfocollection has no better interface
+            c = Constant(p.adr.ptr, lltype.typeOf(p.adr.ptr))
+            op1 = SpaceOperation('str_guard_value', [op.args[0], c, descr],
+                                 op.result)
+            return [SpaceOperation('-live-', [], None), op1, None]
         else:
             log.WARNING('ignoring hint %r at %r' % (hints, self.graph))
 
@@ -718,29 +737,54 @@
         return SpaceOperation(opname, [op.args[0]], op.result)
 
     def rewrite_op_getinteriorfield(self, op):
-        # only supports strings and unicodes
         assert len(op.args) == 3
-        assert op.args[1].value == 'chars'
         optype = op.args[0].concretetype
         if optype == lltype.Ptr(rstr.STR):
             opname = "strgetitem"
+            return SpaceOperation(opname, [op.args[0], op.args[2]], op.result)
+        elif optype == lltype.Ptr(rstr.UNICODE):
+            opname = "unicodegetitem"
+            return SpaceOperation(opname, [op.args[0], op.args[2]], op.result)
         else:
-            assert optype == lltype.Ptr(rstr.UNICODE)
-            opname = "unicodegetitem"
-        return SpaceOperation(opname, [op.args[0], op.args[2]], op.result)
+            v_inst, v_index, c_field = op.args
+            if op.result.concretetype is lltype.Void:
+                return
+            # only GcArray of Struct supported
+            assert isinstance(v_inst.concretetype.TO, lltype.GcArray)
+            STRUCT = v_inst.concretetype.TO.OF
+            assert isinstance(STRUCT, lltype.Struct)
+            descr = self.cpu.interiorfielddescrof(v_inst.concretetype.TO,
+                                                  c_field.value)
+            args = [v_inst, v_index, descr]
+            kind = getkind(op.result.concretetype)[0]
+            return SpaceOperation('getinteriorfield_gc_%s' % kind, args,
+                                  op.result)
 
     def rewrite_op_setinteriorfield(self, op):
-        # only supports strings and unicodes
         assert len(op.args) == 4
-        assert op.args[1].value == 'chars'
         optype = op.args[0].concretetype
         if optype == lltype.Ptr(rstr.STR):
             opname = "strsetitem"
+            return SpaceOperation(opname, [op.args[0], op.args[2], op.args[3]],
+                                  op.result)
+        elif optype == lltype.Ptr(rstr.UNICODE):
+            opname = "unicodesetitem"
+            return SpaceOperation(opname, [op.args[0], op.args[2], op.args[3]],
+                                  op.result)
         else:
-            assert optype == lltype.Ptr(rstr.UNICODE)
-            opname = "unicodesetitem"
-        return SpaceOperation(opname, [op.args[0], op.args[2], op.args[3]],
-                              op.result)
+            v_inst, v_index, c_field, v_value = op.args
+            if v_value.concretetype is lltype.Void:
+                return
+            # only GcArray of Struct supported
+            assert isinstance(v_inst.concretetype.TO, lltype.GcArray)
+            STRUCT = v_inst.concretetype.TO.OF
+            assert isinstance(STRUCT, lltype.Struct)
+            descr = self.cpu.interiorfielddescrof(v_inst.concretetype.TO,
+                                                  c_field.value)
+            kind = getkind(v_value.concretetype)[0]
+            args = [v_inst, v_index, v_value, descr]
+            return SpaceOperation('setinteriorfield_gc_%s' % kind, args,
+                                  op.result)
 
     def _rewrite_equality(self, op, opname):
         arg0, arg1 = op.args
@@ -783,8 +827,7 @@
 
     def rewrite_op_cast_ptr_to_int(self, op):
         if self._is_gc(op.args[0]):
-            #return op
-            raise NotImplementedError("cast_ptr_to_int")
+            return op
 
     def rewrite_op_force_cast(self, op):
         v_arg = op.args[0]
@@ -1526,6 +1569,10 @@
     def rewrite_op_jit_force_virtual(self, op):
         return self._do_builtin_call(op)
 
+    def rewrite_op_jit_is_virtual(self, op):
+        raise Exception, (
+            "'vref.virtual' should not be used from jit-visible code")
+
     def rewrite_op_jit_force_virtualizable(self, op):
         # this one is for virtualizables
         vinfo = self.get_vinfo(op.args[0])
diff --git a/pypy/jit/codewriter/support.py b/pypy/jit/codewriter/support.py
--- a/pypy/jit/codewriter/support.py
+++ b/pypy/jit/codewriter/support.py
@@ -13,7 +13,6 @@
 from pypy.translator.simplify import get_funcobj
 from pypy.translator.unsimplify import split_block
 from pypy.objspace.flow.model import Constant
-from pypy import conftest
 from pypy.translator.translator import TranslationContext
 from pypy.annotation.policy import AnnotatorPolicy
 from pypy.annotation import model as annmodel
@@ -48,15 +47,13 @@
     a.build_types(func, argtypes, main_entry_point=True)
     rtyper = t.buildrtyper(type_system = type_system)
     rtyper.specialize()
-    if inline:
-        auto_inlining(t, threshold=inline)
+    #if inline:
+    #    auto_inlining(t, threshold=inline)
     if backendoptimize:
         from pypy.translator.backendopt.all import backend_optimizations
         backend_optimizations(t, inline_threshold=inline or 0,
                 remove_asserts=True, really_remove_asserts=True)
 
-    #if conftest.option.view:
-    #    t.view()
     return rtyper
 
 def getgraph(func, values):
@@ -456,6 +453,8 @@
         return LLtypeHelpers._dictnext_items(lltype.Ptr(RES), iter)
     _ll_1_dictiter_nextitems.need_result_type = True
 
+    _ll_1_dict_resize = ll_rdict.ll_dict_resize
+
     # ---------- strings and unicode ----------
 
     _ll_1_str_str2unicode = ll_rstr.LLHelpers.ll_str2unicode
diff --git a/pypy/jit/codewriter/test/test_jtransform.py b/pypy/jit/codewriter/test/test_jtransform.py
--- a/pypy/jit/codewriter/test/test_jtransform.py
+++ b/pypy/jit/codewriter/test/test_jtransform.py
@@ -1,4 +1,3 @@
-import py
 import random
 try:
     from itertools import product
@@ -16,13 +15,13 @@
 
 from pypy.objspace.flow.model import FunctionGraph, Block, Link
 from pypy.objspace.flow.model import SpaceOperation, Variable, Constant
-from pypy.jit.codewriter.jtransform import Transformer
-from pypy.jit.metainterp.history import getkind
-from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr, rlist
+from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr
 from pypy.rpython.lltypesystem.module import ll_math
 from pypy.translator.unsimplify import varoftype
 from pypy.jit.codewriter import heaptracker, effectinfo
 from pypy.jit.codewriter.flatten import ListOfKind
+from pypy.jit.codewriter.jtransform import Transformer
+from pypy.jit.metainterp.history import getkind
 
 def const(x):
     return Constant(x, lltype.typeOf(x))
@@ -37,6 +36,8 @@
         return ('calldescr', FUNC, ARGS, RESULT)
     def fielddescrof(self, STRUCT, name):
         return ('fielddescr', STRUCT, name)
+    def interiorfielddescrof(self, ARRAY, name):
+        return ('interiorfielddescr', ARRAY, name)
     def arraydescrof(self, ARRAY):
         return FakeDescr(('arraydescr', ARRAY))
     def sizeof(self, STRUCT):
@@ -99,6 +100,12 @@
             if i == oopspecindex:
                 return True
         return False
+    def callinfo_for_oopspec(self, oopspecindex):
+        assert oopspecindex == effectinfo.EffectInfo.OS_STREQ_NONNULL
+        class c:
+            class adr:
+                ptr = 1
+        return ('calldescr', c)
 
 class FakeBuiltinCallControl:
     def __init__(self):
@@ -119,6 +126,7 @@
              EI.OS_STR2UNICODE:([PSTR], PUNICODE),
              EI.OS_STR_CONCAT: ([PSTR, PSTR], PSTR),
              EI.OS_STR_SLICE:  ([PSTR, INT, INT], PSTR),
+             EI.OS_STREQ_NONNULL:  ([PSTR, PSTR], INT),
              EI.OS_UNI_CONCAT: ([PUNICODE, PUNICODE], PUNICODE),
              EI.OS_UNI_SLICE:  ([PUNICODE, INT, INT], PUNICODE),
              EI.OS_UNI_EQUAL:  ([PUNICODE, PUNICODE], lltype.Bool),
@@ -532,7 +540,7 @@
 
 def test_rename_on_links():
     v1 = Variable()
-    v2 = Variable()
+    v2 = Variable(); v2.concretetype = llmemory.Address
     v3 = Variable()
     block = Block([v1])
     block.operations = [SpaceOperation('cast_pointer', [v1], v2)]
@@ -669,6 +677,22 @@
     assert op1.args == [v, v_index]
     assert op1.result == v_result
 
+def test_dict_getinteriorfield():
+    DICT = lltype.GcArray(lltype.Struct('ENTRY', ('v', lltype.Signed),
+                                        ('k', lltype.Signed)))
+    v = varoftype(lltype.Ptr(DICT))
+    i = varoftype(lltype.Signed)
+    v_result = varoftype(lltype.Signed)
+    op = SpaceOperation('getinteriorfield', [v, i, Constant('v', lltype.Void)],
+                        v_result)
+    op1 = Transformer(FakeCPU()).rewrite_operation(op)
+    assert op1.opname == 'getinteriorfield_gc_i'
+    assert op1.args == [v, i, ('interiorfielddescr', DICT, 'v')]
+    op = SpaceOperation('getinteriorfield', [v, i, Constant('v', lltype.Void)],
+                        Constant(None, lltype.Void))
+    op1 = Transformer(FakeCPU()).rewrite_operation(op)
+    assert op1 is None
+
 def test_str_setinteriorfield():
     v = varoftype(lltype.Ptr(rstr.STR))
     v_index = varoftype(lltype.Signed)
@@ -695,6 +719,23 @@
     assert op1.args == [v, v_index, v_newchr]
     assert op1.result == v_void
 
+def test_dict_setinteriorfield():
+    DICT = lltype.GcArray(lltype.Struct('ENTRY', ('v', lltype.Signed),
+                                        ('k', lltype.Signed)))
+    v = varoftype(lltype.Ptr(DICT))
+    i = varoftype(lltype.Signed)
+    v_void = varoftype(lltype.Void)
+    op = SpaceOperation('setinteriorfield', [v, i, Constant('v', lltype.Void),
+                                             i],
+                        v_void)
+    op1 = Transformer(FakeCPU()).rewrite_operation(op)
+    assert op1.opname == 'setinteriorfield_gc_i'
+    assert op1.args == [v, i, i, ('interiorfielddescr', DICT, 'v')]
+    op = SpaceOperation('setinteriorfield', [v, i, Constant('v', lltype.Void),
+                                             v_void], v_void)
+    op1 = Transformer(FakeCPU()).rewrite_operation(op)
+    assert not op1
+
 def test_promote_1():
     v1 = varoftype(lltype.Signed)
     v2 = varoftype(lltype.Signed)
@@ -844,6 +885,21 @@
     assert op1.args[2] == ListOfKind('ref', [v1, v2])
     assert op1.result == v3
 
+def test_str_promote():
+    PSTR = lltype.Ptr(rstr.STR)
+    v1 = varoftype(PSTR)
+    v2 = varoftype(PSTR)
+    op = SpaceOperation('hint',
+                        [v1, Constant({'promote_string': True}, lltype.Void)],
+                        v2)
+    tr = Transformer(FakeCPU(), FakeBuiltinCallControl())
+    op0, op1, _ = tr.rewrite_operation(op)
+    assert op1.opname == 'str_guard_value'
+    assert op1.args[0] == v1
+    assert op1.args[2] == 'calldescr'
+    assert op1.result == v2
+    assert op0.opname == '-live-'
+
 def test_unicode_concat():
     # test that the oopspec is present and correctly transformed
     PSTR = lltype.Ptr(rstr.UNICODE)
diff --git a/pypy/jit/metainterp/blackhole.py b/pypy/jit/metainterp/blackhole.py
--- a/pypy/jit/metainterp/blackhole.py
+++ b/pypy/jit/metainterp/blackhole.py
@@ -2,11 +2,10 @@
 from pypy.rlib.rtimer import read_timestamp
 from pypy.rlib.rarithmetic import intmask, LONG_BIT, r_uint, ovfcheck
 from pypy.rlib.objectmodel import we_are_translated
-from pypy.rlib.debug import debug_start, debug_stop
+from pypy.rlib.debug import debug_start, debug_stop, ll_assert
 from pypy.rlib.debug import make_sure_not_resized
 from pypy.rpython.lltypesystem import lltype, llmemory, rclass
 from pypy.rpython.lltypesystem.lloperation import llop
-from pypy.rpython.llinterp import LLException
 from pypy.jit.codewriter.jitcode import JitCode, SwitchDictDescr
 from pypy.jit.codewriter import heaptracker, longlong
 from pypy.jit.metainterp.jitexc import JitException, get_llexception, reraise
@@ -503,6 +502,15 @@
     @arguments("r", returns="r")
     def bhimpl_cast_opaque_ptr(a):
         return a
+    @arguments("r", returns="i")
+    def bhimpl_cast_ptr_to_int(a):
+        i = lltype.cast_ptr_to_int(a)
+        ll_assert((i & 1) == 1, "bhimpl_cast_ptr_to_int: not an odd int")
+        return i
+    @arguments("i", returns="r")
+    def bhimpl_cast_int_to_ptr(i):
+        ll_assert((i & 1) == 1, "bhimpl_cast_int_to_ptr: not an odd int")
+        return lltype.cast_int_to_ptr(llmemory.GCREF, i)
 
     @arguments("i", returns="i")
     def bhimpl_int_copy(a):
@@ -523,6 +531,9 @@
     @arguments("f")
     def bhimpl_float_guard_value(a):
         pass
+    @arguments("r", "i", "d", returns="r")
+    def bhimpl_str_guard_value(a, i, d):
+        return a
 
     @arguments("self", "i")
     def bhimpl_int_push(self, a):
@@ -1142,6 +1153,26 @@
         array = cpu.bh_getfield_gc_r(vable, fdescr)
         return cpu.bh_arraylen_gc(adescr, array)
 
+    @arguments("cpu", "r", "i", "d", returns="i")
+    def bhimpl_getinteriorfield_gc_i(cpu, array, index, descr):
+        return cpu.bh_getinteriorfield_gc_i(array, index, descr)
+    @arguments("cpu", "r", "i", "d", returns="r")
+    def bhimpl_getinteriorfield_gc_r(cpu, array, index, descr):
+        return cpu.bh_getinteriorfield_gc_r(array, index, descr)
+    @arguments("cpu", "r", "i", "d", returns="f")
+    def bhimpl_getinteriorfield_gc_f(cpu, array, index, descr):
+        return cpu.bh_getinteriorfield_gc_f(array, index, descr)
+
+    @arguments("cpu", "r", "i", "d", "i")
+    def bhimpl_setinteriorfield_gc_i(cpu, array, index, descr, value):
+        cpu.bh_setinteriorfield_gc_i(array, index, descr, value)
+    @arguments("cpu", "r", "i", "d", "r")
+    def bhimpl_setinteriorfield_gc_r(cpu, array, index, descr, value):
+        cpu.bh_setinteriorfield_gc_r(array, index, descr, value)
+    @arguments("cpu", "r", "i", "d", "f")
+    def bhimpl_setinteriorfield_gc_f(cpu, array, index, descr, value):
+        cpu.bh_setinteriorfield_gc_f(array, index, descr, value)
+
     @arguments("cpu", "r", "d", returns="i")
     def bhimpl_getfield_gc_i(cpu, struct, fielddescr):
         return cpu.bh_getfield_gc_i(struct, fielddescr)
diff --git a/pypy/jit/metainterp/executor.py b/pypy/jit/metainterp/executor.py
--- a/pypy/jit/metainterp/executor.py
+++ b/pypy/jit/metainterp/executor.py
@@ -1,11 +1,8 @@
 """This implements pyjitpl's execution of operations.
 """
 
-import py
-from pypy.rpython.lltypesystem import lltype, llmemory, rstr
-from pypy.rpython.ootypesystem import ootype
-from pypy.rpython.lltypesystem.lloperation import llop
-from pypy.rlib.rarithmetic import ovfcheck, r_uint, intmask, r_longlong
+from pypy.rpython.lltypesystem import lltype, rstr
+from pypy.rlib.rarithmetic import ovfcheck, r_longlong
 from pypy.rlib.rtimer import read_timestamp
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.jit.metainterp.history import BoxInt, BoxPtr, BoxFloat, check_descr
@@ -123,6 +120,29 @@
     else:
         cpu.bh_setarrayitem_raw_i(arraydescr, array, index, itembox.getint())
 
+def do_getinteriorfield_gc(cpu, _, arraybox, indexbox, descr):
+    array = arraybox.getref_base()
+    index = indexbox.getint()
+    if descr.is_pointer_field():
+        return BoxPtr(cpu.bh_getinteriorfield_gc_r(array, index, descr))
+    elif descr.is_float_field():
+        return BoxFloat(cpu.bh_getinteriorfield_gc_f(array, index, descr))
+    else:
+        return BoxInt(cpu.bh_getinteriorfield_gc_i(array, index, descr))
+
+def do_setinteriorfield_gc(cpu, _, arraybox, indexbox, valuebox, descr):
+    array = arraybox.getref_base()
+    index = indexbox.getint()
+    if descr.is_pointer_field():
+        cpu.bh_setinteriorfield_gc_r(array, index, descr,
+                                     valuebox.getref_base())
+    elif descr.is_float_field():
+        cpu.bh_setinteriorfield_gc_f(array, index, descr,
+                                     valuebox.getfloatstorage())
+    else:
+        cpu.bh_setinteriorfield_gc_i(array, index, descr,
+                                     valuebox.getint())
+
 def do_getfield_gc(cpu, _, structbox, fielddescr):
     struct = structbox.getref_base()
     if fielddescr.is_pointer_field():
diff --git a/pypy/jit/metainterp/graphpage.py b/pypy/jit/metainterp/graphpage.py
--- a/pypy/jit/metainterp/graphpage.py
+++ b/pypy/jit/metainterp/graphpage.py
@@ -12,8 +12,8 @@
     def get_display_text(self):
         return None
 
-def display_loops(loops, errmsg=None, highlight_loops=()):
-    graphs = [(loop, loop in highlight_loops) for loop in loops]    
+def display_loops(loops, errmsg=None, highlight_loops={}):
+    graphs = [(loop, highlight_loops.get(loop, 0)) for loop in loops]    
     for graph, highlight in graphs:
         for op in graph.get_operations():
             if is_interesting_guard(op):
@@ -65,8 +65,7 @@
     def add_graph(self, graph, highlight=False):
         graphindex = len(self.graphs)
         self.graphs.append(graph)
-        if highlight:
-            self.highlight_graphs[graph] = True
+        self.highlight_graphs[graph] = highlight
         for i, op in enumerate(graph.get_operations()):
             self.all_operations[op] = graphindex, i
 
@@ -126,10 +125,13 @@
             self.dotgen.emit('subgraph cluster%d {' % graphindex)
         label = graph.get_display_text()
         if label is not None:
-            if self.highlight_graphs.get(graph):
-                fillcolor = '#f084c2'
+            colorindex = self.highlight_graphs.get(graph, 0)
+            if colorindex == 1:
+                fillcolor = '#f084c2'    # highlighted graph
+            elif colorindex == 2:
+                fillcolor = '#808080'    # invalidated graph
             else:
-                fillcolor = '#84f0c2'
+                fillcolor = '#84f0c2'    # normal color
             self.dotgen.emit_node(graphname, shape="octagon",
                                   label=label, fillcolor=fillcolor)
             self.pendingedges.append((graphname,
diff --git a/pypy/jit/metainterp/heapcache.py b/pypy/jit/metainterp/heapcache.py
--- a/pypy/jit/metainterp/heapcache.py
+++ b/pypy/jit/metainterp/heapcache.py
@@ -54,9 +54,10 @@
         if box in self.new_boxes:
             self.new_boxes[box] = False
         if box in self.dependencies:
-            for dep in self.dependencies[box]:
+            deps = self.dependencies[box]
+            del self.dependencies[box]
+            for dep in deps:
                 self._escape(dep)
-            del self.dependencies[box]
 
     def clear_caches(self, opnum, descr, argboxes):
         if opnum == rop.SETFIELD_GC:
diff --git a/pypy/jit/metainterp/history.py b/pypy/jit/metainterp/history.py
--- a/pypy/jit/metainterp/history.py
+++ b/pypy/jit/metainterp/history.py
@@ -9,12 +9,14 @@
 
 from pypy.jit.metainterp.resoperation import ResOperation, rop
 from pypy.jit.codewriter import heaptracker, longlong
+from pypy.rlib.objectmodel import compute_identity_hash
 
 # ____________________________________________________________
 
 INT   = 'i'
 REF   = 'r'
 FLOAT = 'f'
+STRUCT = 's'
 HOLE  = '_'
 VOID  = 'v'
 
@@ -104,7 +106,7 @@
     getref._annspecialcase_ = 'specialize:arg(1)'
 
     def _get_hash_(self):
-        raise NotImplementedError
+        return compute_identity_hash(self)
 
     def clonebox(self):
         raise NotImplementedError
@@ -133,6 +135,9 @@
     def _get_str(self):
         raise NotImplementedError
 
+    def same_box(self, other):
+        return self is other
+
 class AbstractDescr(AbstractValue):
     __slots__ = ()
 
@@ -168,6 +173,11 @@
         """
         raise NotImplementedError
 
+    def is_array_of_structs(self):
+        """ Implement for array descr
+        """
+        raise NotImplementedError
+
     def is_pointer_field(self):
         """ Implement for field descr
         """
@@ -241,32 +251,15 @@
     def constbox(self):
         return self
 
+    def same_box(self, other):
+        return self.same_constant(other)
+
     def same_constant(self, other):
         raise NotImplementedError
 
     def __repr__(self):
         return 'Const(%s)' % self._getrepr_()
 
-    def __eq__(self, other):
-        "NOT_RPYTHON"
-        # Remember that you should not compare Consts with '==' in RPython.
-        # Consts have no special __hash__, in order to force different Consts
-        # from being considered as different keys when stored in dicts
-        # (as they always are after translation).  Use a dict_equal_consts()
-        # to get the other behavior (i.e. using this __eq__).
-        if self.__class__ is not other.__class__:
-            return False
-        try:
-            return self.value == other.value
-        except TypeError:
-            if (isinstance(self.value, Symbolic) and
-                isinstance(other.value, Symbolic)):
-                return self.value is other.value
-            raise
-
-    def __ne__(self, other):
-        return not (self == other)
-
 
 class ConstInt(Const):
     type = INT
@@ -688,33 +681,6 @@
 
 # ____________________________________________________________
 
-def dict_equal_consts():
-    "NOT_RPYTHON"
-    # Returns a dict in which Consts that compare as equal
-    # are identified when used as keys.
-    return r_dict(dc_eq, dc_hash)
-
-def dc_eq(c1, c2):
-    return c1 == c2
-
-def dc_hash(c):
-    "NOT_RPYTHON"
-    # This is called during translation only.  Avoid using identityhash(),
-    # to avoid forcing a hash, at least on lltype objects.
-    if not isinstance(c, Const):
-        return hash(c)
-    if isinstance(c.value, Symbolic):
-        return id(c.value)
-    try:
-        if isinstance(c, ConstPtr):
-            p = lltype.normalizeptr(c.value)
-            if p is not None:
-                return hash(p._obj)
-            else:
-                return 0
-        return c._get_hash_()
-    except lltype.DelayedPointer:
-        return -2      # xxx risk of changing hash...
 
 def make_hashable_int(i):
     from pypy.rpython.lltypesystem.ll2ctypes import NotCtypesAllocatedStructure
@@ -772,6 +738,7 @@
     failed_states = None
     retraced_count = 0
     terminating = False # see TerminatingLoopToken in compile.py
+    invalidated = False
     outermost_jitdriver_sd = None
     # and more data specified by the backend when the loop is compiled
     number = -1
@@ -974,6 +941,7 @@
         self.loops = []
         self.locations = []
         self.aborted_keys = []
+        self.invalidated_token_numbers = set()
 
     def set_history(self, history):
         self.operations = history.operations
@@ -1052,7 +1020,12 @@
             if loop in loops:
                 loops.remove(loop)
             loops.append(loop)
-        display_loops(loops, errmsg, extraloops)
+        highlight_loops = dict.fromkeys(extraloops, 1)
+        for loop in loops:
+            if hasattr(loop, '_looptoken_number') and (
+                    loop._looptoken_number in self.invalidated_token_numbers):
+                highlight_loops.setdefault(loop, 2)
+        display_loops(loops, errmsg, highlight_loops)
 
 # ----------------------------------------------------------------
 
diff --git a/pypy/jit/metainterp/memmgr.py b/pypy/jit/metainterp/memmgr.py
--- a/pypy/jit/metainterp/memmgr.py
+++ b/pypy/jit/metainterp/memmgr.py
@@ -68,7 +68,8 @@
         debug_print("Loop tokens before:", oldtotal)
         max_generation = self.current_generation - (self.max_age-1)
         for looptoken in self.alive_loops.keys():
-            if 0 <= looptoken.generation < max_generation:
+            if (0 <= looptoken.generation < max_generation or
+                looptoken.invalidated):
                 del self.alive_loops[looptoken]
         newtotal = len(self.alive_loops)
         debug_print("Loop tokens freed: ", oldtotal - newtotal)
diff --git a/pypy/jit/metainterp/optimizeopt/rewrite.py b/pypy/jit/metainterp/optimizeopt/rewrite.py
--- a/pypy/jit/metainterp/optimizeopt/rewrite.py
+++ b/pypy/jit/metainterp/optimizeopt/rewrite.py
@@ -106,10 +106,9 @@
             self.make_equal_to(op.result, v1)
         else:
             self.emit_operation(op)
-
-        # Synthesize the reverse ops for optimize_default to reuse
-        self.pure(rop.INT_ADD, [op.result, op.getarg(1)], op.getarg(0))
-        self.pure(rop.INT_SUB, [op.getarg(0), op.result], op.getarg(1))
+            # Synthesize the reverse ops for optimize_default to reuse
+            self.pure(rop.INT_ADD, [op.result, op.getarg(1)], op.getarg(0))
+            self.pure(rop.INT_SUB, [op.getarg(0), op.result], op.getarg(1))
 
     def optimize_INT_ADD(self, op):
         v1 = self.getvalue(op.getarg(0))
@@ -122,10 +121,9 @@
             self.make_equal_to(op.result, v1)
         else:
             self.emit_operation(op)
-
-        # Synthesize the reverse op for optimize_default to reuse
-        self.pure(rop.INT_SUB, [op.result, op.getarg(1)], op.getarg(0))
-        self.pure(rop.INT_SUB, [op.result, op.getarg(0)], op.getarg(1))
+            # Synthesize the reverse op for optimize_default to reuse
+            self.pure(rop.INT_SUB, [op.result, op.getarg(1)], op.getarg(0))
+            self.pure(rop.INT_SUB, [op.result, op.getarg(0)], op.getarg(1))
 
     def optimize_INT_MUL(self, op):
         v1 = self.getvalue(op.getarg(0))
@@ -141,13 +139,13 @@
             self.make_constant_int(op.result, 0)
         else:
             for lhs, rhs in [(v1, v2), (v2, v1)]:
-                # x & (x -1) == 0 is a quick test for power of 2
-                if (lhs.is_constant() and
-                    (lhs.box.getint() & (lhs.box.getint() - 1)) == 0):
-                    new_rhs = ConstInt(highest_bit(lhs.box.getint()))
-                    op = op.copy_and_change(rop.INT_LSHIFT, args=[rhs.box, new_rhs])
-                    break
-
+                if lhs.is_constant():
+                    x = lhs.box.getint()
+                    # x & (x - 1) == 0 is a quick test for power of 2
+                    if x & (x - 1) == 0:
+                        new_rhs = ConstInt(highest_bit(lhs.box.getint()))
+                        op = op.copy_and_change(rop.INT_LSHIFT, args=[rhs.box, new_rhs])
+                        break
             self.emit_operation(op)
 
     def optimize_UINT_FLOORDIV(self, op):
@@ -450,6 +448,9 @@
         if v2.is_constant() and v2.box.getint() == 1:
             self.make_equal_to(op.result, v1)
             return
+        elif v1.is_constant() and v1.box.getint() == 0:
+            self.make_constant_int(op.result, 0)
+            return
         if v1.intbound.known_ge(IntBound(0, 0)) and v2.is_constant():
             val = v2.box.getint()
             if val & (val - 1) == 0 and val > 0: # val == 2**shift
@@ -462,6 +463,14 @@
         self.optimizer.opaque_pointers[value] = True
         self.make_equal_to(op.result, value)
 
+    def optimize_CAST_PTR_TO_INT(self, op):
+        self.pure(rop.CAST_INT_TO_PTR, [op.result], op.getarg(0))
+        self.emit_operation(op)
+
+    def optimize_CAST_INT_TO_PTR(self, op):
+        self.pure(rop.CAST_PTR_TO_INT, [op.result], op.getarg(0))
+        self.emit_operation(op)
+
 dispatch_opt = make_dispatcher_method(OptRewrite, 'optimize_',
         default=OptRewrite.emit_operation)
 optimize_guards = _findall(OptRewrite, 'optimize_', 'GUARD')
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -2181,6 +2181,17 @@
         """
         self.optimize_loop(ops, expected)
 
+        ops = """
+        [i0]
+        i1 = int_floordiv(0, i0)
+        jump(i1)
+        """
+        expected = """
+        [i0]
+        jump(0)
+        """
+        self.optimize_loop(ops, expected)
+
     def test_fold_partially_constant_ops_ovf(self):
         ops = """
         [i0]
@@ -2329,7 +2340,7 @@
         def _variables_equal(box, varname, strict):
             if varname not in virtuals:
                 if strict:
-                    assert box == oparse.getvar(varname)
+                    assert box.same_box(oparse.getvar(varname))
                 else:
                     assert box.value == oparse.getvar(varname).value
             else:
@@ -4789,6 +4800,18 @@
         """
         self.optimize_strunicode_loop(ops, expected)
 
+    def test_ptr_eq_str_constant(self):
+        ops = """
+        []
+        i0 = ptr_eq(s"abc", s"\x00")
+        finish(i0)
+        """
+        expected = """
+        []
+        finish(0)
+        """
+        self.optimize_loop(ops, expected)
+
 
 class TestLLtype(BaseTestOptimizeBasic, LLtypeMixin):
     pass
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -234,6 +234,30 @@
             """ % expected_value
             self.optimize_loop(ops, expected)
 
+    def test_reverse_of_cast(self):
+        ops = """
+        [i0]
+        p0 = cast_int_to_ptr(i0)
+        i1 = cast_ptr_to_int(p0)
+        jump(i1)
+        """
+        expected = """
+        [i0]
+        jump(i0)
+        """
+        self.optimize_loop(ops, expected)
+        ops = """
+        [p0]
+        i1 = cast_ptr_to_int(p0)
+        p1 = cast_int_to_ptr(i1)
+        jump(p1)
+        """
+        expected = """
+        [p0]
+        jump(p0)
+        """
+        self.optimize_loop(ops, expected)
+
     # ----------
 
     def test_remove_guard_class_1(self):
diff --git a/pypy/jit/metainterp/optimizeopt/util.py b/pypy/jit/metainterp/optimizeopt/util.py
--- a/pypy/jit/metainterp/optimizeopt/util.py
+++ b/pypy/jit/metainterp/optimizeopt/util.py
@@ -90,14 +90,11 @@
     for i in range(len(args1)):
         arg1 = args1[i]
         arg2 = args2[i]
-        if isinstance(arg1, history.Const):
-            if arg1.__class__ is not arg2.__class__:
+        if arg1 is None:
+            if arg2 is not None:
                 return False
-            if not arg1.same_constant(arg2):
-                return False
-        else:
-            if not arg1 is arg2:
-                return False
+        elif not arg1.same_box(arg2):
+            return False
     return True
 
 def args_hash(args):
@@ -106,10 +103,8 @@
     for arg in args:
         if arg is None:
             y = 17
-        elif isinstance(arg, history.Const):
+        else:
             y = arg._get_hash_()
-        else:
-            y = compute_identity_hash(arg)
         res = intmask((1000003 * res) ^ y)
     return res
 
@@ -145,9 +140,12 @@
         for i in range(op1.numargs()):
             x = op1.getarg(i)
             y = op2.getarg(i)
-            assert x == remap.get(y, y)
+            assert x.same_box(remap.get(y, y))
         if op2.result in remap:
-            assert op1.result == remap[op2.result]
+            if op2.result is None:
+                assert op1.result == remap[op2.result]
+            else:
+                assert op1.result.same_box(remap[op2.result])
         else:
             remap[op2.result] = op1.result
         if op1.getopnum() != rop.JUMP:      # xxx obscure
@@ -156,11 +154,20 @@
             assert len(op1.getfailargs()) == len(op2.getfailargs())
             if strict_fail_args:
                 for x, y in zip(op1.getfailargs(), op2.getfailargs()):
-                    assert x == remap.get(y, y)
+                    if x is None:
+                        assert remap.get(y, y) is None
+                    else:
+                        assert x.same_box(remap.get(y, y))
             else:
                 fail_args1 = set(op1.getfailargs())
                 fail_args2 = set([remap.get(y, y) for y in op2.getfailargs()])
-                assert fail_args1 == fail_args2
+                for x in fail_args1:
+                    for y in fail_args2:
+                        if x.same_box(y):
+                            fail_args2.remove(y)
+                            break
+                    else:
+                        assert False
     assert len(oplist1) == len(oplist2)
     print '-'*totwidth
     return True
diff --git a/pypy/jit/metainterp/optimizeopt/virtualize.py b/pypy/jit/metainterp/optimizeopt/virtualize.py
--- a/pypy/jit/metainterp/optimizeopt/virtualize.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualize.py
@@ -59,7 +59,7 @@
 
     def import_from(self, other, optimizer):
         raise NotImplementedError("should not be called at this level")
-    
+
 def get_fielddescrlist_cache(cpu):
     if not hasattr(cpu, '_optimizeopt_fielddescrlist_cache'):
         result = descrlist_dict()
@@ -113,7 +113,7 @@
         #
         if not we_are_translated():
             op.name = 'FORCE ' + self.source_op.name
-            
+
         if self._is_immutable_and_filled_with_constants(optforce):
             box = optforce.optimizer.constant_fold(op)
             self.make_constant(box)
@@ -239,12 +239,12 @@
         for index in range(len(self._items)):
             self._items[index] = self._items[index].force_at_end_of_preamble(already_forced, optforce)
         return self
-    
+
     def _really_force(self, optforce):
         assert self.source_op is not None
         if not we_are_translated():
             self.source_op.name = 'FORCE ' + self.source_op.name
-        optforce.emit_operation(self.source_op)        
+        optforce.emit_operation(self.source_op)
         self.box = box = self.source_op.result
         for index in range(len(self._items)):
             subvalue = self._items[index]
@@ -276,7 +276,7 @@
 
     def new(self):
         return OptVirtualize()
-        
+
     def make_virtual(self, known_class, box, source_op=None):
         vvalue = VirtualValue(self.optimizer.cpu, known_class, box, source_op)
         self.make_equal_to(box, vvalue)
@@ -386,7 +386,8 @@
 
     def optimize_NEW_ARRAY(self, op):
         sizebox = self.get_constant_box(op.getarg(0))
-        if sizebox is not None:
+        # For now we can't make arrays of structs virtual.
+        if sizebox is not None and not op.getdescr().is_array_of_structs():
             # if the original 'op' did not have a ConstInt as argument,
             # build a new one with the ConstInt argument
             if not isinstance(op.getarg(0), ConstInt):
diff --git a/pypy/jit/metainterp/optimizeopt/vstring.py b/pypy/jit/metainterp/optimizeopt/vstring.py
--- a/pypy/jit/metainterp/optimizeopt/vstring.py
+++ b/pypy/jit/metainterp/optimizeopt/vstring.py
@@ -587,10 +587,7 @@
             return True
         #
         if v1.is_nonnull() and v2.is_nonnull():
-            if l1box is not None and l2box is not None and (
-                l1box == l2box or (isinstance(l1box, ConstInt) and
-                                   isinstance(l2box, ConstInt) and
-                                   l1box.value == l2box.value)):
+            if l1box is not None and l2box is not None and l1box.same_box(l2box):
                 do = EffectInfo.OS_STREQ_LENGTHOK
             else:
                 do = EffectInfo.OS_STREQ_NONNULL
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -36,6 +36,7 @@
 
 
 class MIFrame(object):
+    debug = False
 
     def __init__(self, metainterp):
         self.metainterp = metainterp
@@ -162,15 +163,18 @@
             if registers[i] is oldbox:
                 registers[i] = newbox
         if not we_are_translated():
-            assert oldbox not in registers[count:]
+            for b in registers[count:]:
+                assert not oldbox.same_box(b)
+                
 
     def make_result_of_lastop(self, resultbox):
         got_type = resultbox.type
-        if not we_are_translated():
-            typeof = {'i': history.INT,
-                      'r': history.REF,
-                      'f': history.FLOAT}
-            assert typeof[self.jitcode._resulttypes[self.pc]] == got_type
+        # XXX disabled for now, conflicts with str_guard_value
+        #if not we_are_translated():
+        #    typeof = {'i': history.INT,
+        #              'r': history.REF,
+        #              'f': history.FLOAT}
+        #    assert typeof[self.jitcode._resulttypes[self.pc]] == got_type
         target_index = ord(self.bytecode[self.pc-1])
         if got_type == history.INT:
             self.registers_i[target_index] = resultbox
@@ -219,6 +223,7 @@
                     'cast_float_to_int', 'cast_int_to_float',
                     'cast_float_to_singlefloat', 'cast_singlefloat_to_float',
                     'float_neg', 'float_abs',
+                    'cast_ptr_to_int', 'cast_int_to_ptr',
                     ]:
         exec py.code.Source('''
             @arguments("box")
@@ -544,6 +549,14 @@
     opimpl_getfield_gc_r_pure = _opimpl_getfield_gc_pure_any
     opimpl_getfield_gc_f_pure = _opimpl_getfield_gc_pure_any
 
+    @arguments("box", "box", "descr")
+    def _opimpl_getinteriorfield_gc_any(self, array, index, descr):
+        return self.execute_with_descr(rop.GETINTERIORFIELD_GC, descr,
+                                       array, index)
+    opimpl_getinteriorfield_gc_i = _opimpl_getinteriorfield_gc_any
+    opimpl_getinteriorfield_gc_f = _opimpl_getinteriorfield_gc_any
+    opimpl_getinteriorfield_gc_r = _opimpl_getinteriorfield_gc_any
+
     @specialize.arg(1)
     def _opimpl_getfield_gc_any_pureornot(self, opnum, box, fielddescr):
         tobox = self.metainterp.heapcache.getfield(box, fielddescr)
@@ -584,6 +597,15 @@
     opimpl_setfield_gc_r = _opimpl_setfield_gc_any
     opimpl_setfield_gc_f = _opimpl_setfield_gc_any
 
+    @arguments("box", "box", "box", "descr")
+    def _opimpl_setinteriorfield_gc_any(self, array, index, value, descr):
+        self.execute_with_descr(rop.SETINTERIORFIELD_GC, descr,
+                                array, index, value)
+    opimpl_setinteriorfield_gc_i = _opimpl_setinteriorfield_gc_any
+    opimpl_setinteriorfield_gc_f = _opimpl_setinteriorfield_gc_any
+    opimpl_setinteriorfield_gc_r = _opimpl_setinteriorfield_gc_any
+                                
+
     @arguments("box", "descr")
     def _opimpl_getfield_raw_any(self, box, fielddescr):
         return self.execute_with_descr(rop.GETFIELD_RAW, fielddescr, box)
@@ -895,6 +917,21 @@
     def _opimpl_guard_value(self, orgpc, box):
         self.implement_guard_value(orgpc, box)
 
+    @arguments("orgpc", "box", "box", "descr")
+    def opimpl_str_guard_value(self, orgpc, box, funcbox, descr):
+        if isinstance(box, Const):
+            return box     # no promotion needed, already a Const
+        else:
+            constbox = box.constbox()
+            resbox = self.do_residual_call(funcbox, descr, [box, constbox])
+            promoted_box = resbox.constbox()
+            # This is GUARD_VALUE because GUARD_TRUE assumes the existance
+            # of a label when computing resumepc
+            self.generate_guard(rop.GUARD_VALUE, resbox, [promoted_box],
+                                resumepc=orgpc)
+            self.metainterp.replace_box(box, constbox)
+            return constbox
+
     opimpl_int_guard_value = _opimpl_guard_value
     opimpl_ref_guard_value = _opimpl_guard_value
     opimpl_float_guard_value = _opimpl_guard_value
@@ -2569,17 +2606,21 @@
         self.pc = position
         #
         if not we_are_translated():
-            print '\tpyjitpl: %s(%s)' % (name, ', '.join(map(repr, args))),
+            if self.debug:
+                print '\tpyjitpl: %s(%s)' % (name, ', '.join(map(repr, args))),
             try:
                 resultbox = unboundmethod(self, *args)
             except Exception, e:
-                print '-> %s!' % e.__class__.__name__
+                if self.debug:
+                    print '-> %s!' % e.__class__.__name__
                 raise
             if num_return_args == 0:
-                print
+                if self.debug:
+                    print
                 assert resultbox is None
             else:
-                print '-> %r' % (resultbox,)
+                if self.debug:
+                    print '-> %r' % (resultbox,)
                 assert argcodes[next_argcode] == '>'
                 result_argcode = argcodes[next_argcode + 1]
                 assert resultbox.type == {'i': history.INT,
diff --git a/pypy/jit/metainterp/quasiimmut.py b/pypy/jit/metainterp/quasiimmut.py
--- a/pypy/jit/metainterp/quasiimmut.py
+++ b/pypy/jit/metainterp/quasiimmut.py
@@ -2,6 +2,7 @@
 from pypy.rpython.lltypesystem import lltype, rclass
 from pypy.rpython.annlowlevel import cast_base_ptr_to_instance
 from pypy.jit.metainterp.history import AbstractDescr
+from pypy.rlib.objectmodel import we_are_translated
 
 
 def get_mutate_field_name(fieldname):
@@ -50,13 +51,13 @@
 
 class QuasiImmut(object):
     llopaque = True
+    compress_limit = 30
     
     def __init__(self, cpu):
         self.cpu = cpu
         # list of weakrefs to the LoopTokens that must be invalidated if
         # this value ever changes
         self.looptokens_wrefs = []
-        self.compress_limit = 30
 
     def hide(self):
         qmut_ptr = self.cpu.ts.cast_instance_to_base_ref(self)
@@ -75,6 +76,8 @@
     def compress_looptokens_list(self):
         self.looptokens_wrefs = [wref for wref in self.looptokens_wrefs
                                       if wref() is not None]
+        # NB. we must keep around the looptoken_wrefs that are
+        # already invalidated; see below
         self.compress_limit = (len(self.looptokens_wrefs) + 15) * 2
 
     def invalidate(self):
@@ -86,7 +89,16 @@
         for wref in wrefs:
             looptoken = wref()
             if looptoken is not None:
+                looptoken.invalidated = True
                 self.cpu.invalidate_loop(looptoken)
+                # NB. we must call cpu.invalidate_loop() even if
+                # looptoken.invalidated was already set to True.
+                # It's possible to invalidate several times the
+                # same looptoken; see comments in jit.backend.model
+                # in invalidate_loop().
+                if not we_are_translated():
+                    self.cpu.stats.invalidated_token_numbers.add(
+                        looptoken.number)
 
 
 class QuasiImmutDescr(AbstractDescr):
diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py
--- a/pypy/jit/metainterp/resoperation.py
+++ b/pypy/jit/metainterp/resoperation.py
@@ -1,5 +1,4 @@
 from pypy.rlib.objectmodel import we_are_translated
-from pypy.rlib.debug import make_sure_not_resized
 
 def ResOperation(opnum, args, result, descr=None):
     cls = opclasses[opnum]
@@ -433,6 +432,8 @@
     'INT_INVERT/1',
     #
     'SAME_AS/1',      # gets a Const or a Box, turns it into another Box
+    'CAST_PTR_TO_INT/1',
+    'CAST_INT_TO_PTR/1',
     #
     'PTR_EQ/2b',
     'PTR_NE/2b',
@@ -455,6 +456,7 @@
 
     'GETARRAYITEM_GC/2d',
     'GETARRAYITEM_RAW/2d',
+    'GETINTERIORFIELD_GC/2d',
     'GETFIELD_GC/1d',
     'GETFIELD_RAW/1d',
     '_MALLOC_FIRST',
@@ -471,6 +473,7 @@
 
     'SETARRAYITEM_GC/3d',
     'SETARRAYITEM_RAW/3d',
+    'SETINTERIORFIELD_GC/3d',
     'SETFIELD_GC/2d',
     'SETFIELD_RAW/2d',
     'STRSETITEM/3',
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -13,7 +13,7 @@
 from pypy.rlib.jit import (JitDriver, we_are_jitted, hint, dont_look_inside,
     loop_invariant, elidable, promote, jit_debug, assert_green,
     AssertGreenFailed, unroll_safe, current_trace_length, look_inside_iff,
-    isconstant, isvirtual)
+    isconstant, isvirtual, promote_string)
 from pypy.rlib.rarithmetic import ovfcheck
 from pypy.rpython.lltypesystem import lltype, llmemory, rffi
 from pypy.rpython.ootypesystem import ootype
@@ -3435,12 +3435,43 @@
             return sa
         res = self.meta_interp(f, [16])
         assert res == f(16)
-        
+
+    def test_ptr_eq_str_constants(self):
+        myjitdriver = JitDriver(greens = [], reds = ["n", "x"])
+        class A(object):
+            def __init__(self, v):
+                self.v = v
+        def f(n, x):
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, x=x)
+                z = 0 / x
+                a1 = A("key")
+                a2 = A("\x00")
+                n -= [a1, a2][z].v is not a2.v
+            return n
+        res = self.meta_interp(f, [10, 1])
+        assert res == 0
+
+    def test_virtual_array_of_structs(self):
+        myjitdriver = JitDriver(greens = [], reds=["n", "d"])
+        def f(n):
+            d = None
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, d=d)
+                d = {}
+                if n % 2:
+                    d["k"] = n
+                else:
+                    d["z"] = n
+                n -= len(d)
+            return n
+        res = self.meta_interp(f, [10])
+        assert res == 0
+
 
 
 class TestLLtype(BaseLLtypeTests, LLJitMixin):
     def test_tagged(self):
-        py.test.skip("implement me")
         from pypy.rlib.objectmodel import UnboxedValue
         class Base(object):
             __slots__ = ()
@@ -3492,3 +3523,35 @@
                 pc += 1
             return pc
         res = self.meta_interp(main, [False, 100, True], taggedpointers=True)
+
+    def test_rerased(self):
+        from pypy.rlib.rerased import erase_int, unerase_int, new_erasing_pair
+        eraseX, uneraseX = new_erasing_pair("X")
+        #
+        class X:
+            def __init__(self, a, b):
+                self.a = a
+                self.b = b
+        #
+        def f(i, j):
+            # 'j' should be 0 or 1, not other values
+            if j > 0:
+                e = eraseX(X(i, j))
+            else:
+                try:
+                    e = erase_int(i)
+                except OverflowError:
+                    return -42
+            if j & 1:
+                x = uneraseX(e)
+                return x.a - x.b
+            else:
+                return unerase_int(e)
+        #
+        x = self.interp_operations(f, [-128, 0], taggedpointers=True)
+        assert x == -128
+        bigint = sys.maxint//2 + 1
+        x = self.interp_operations(f, [bigint, 0], taggedpointers=True)
+        assert x == -42
+        x = self.interp_operations(f, [1000, 1], taggedpointers=True)
+        assert x == 999
diff --git a/pypy/jit/metainterp/test/test_dict.py b/pypy/jit/metainterp/test/test_dict.py
--- a/pypy/jit/metainterp/test/test_dict.py
+++ b/pypy/jit/metainterp/test/test_dict.py
@@ -91,7 +91,7 @@
         res1 = f(100)
         res2 = self.meta_interp(f, [100], listops=True)
         assert res1 == res2
-        self.check_loops(int_mod=1) # the hash was traced
+        self.check_loops(int_mod=1) # the hash was traced and eq, but cached
 
     def test_dict_setdefault(self):
         myjitdriver = JitDriver(greens = [], reds = ['total', 'dct'])
@@ -128,7 +128,7 @@
         assert f(100) == 50
         res = self.meta_interp(f, [100], listops=True)
         assert res == 50
-        self.check_loops(int_mod=1)
+        self.check_loops(int_mod=1) # key + eq, but cached
 
     def test_repeated_lookup(self):
         myjitdriver = JitDriver(greens = [], reds = ['n', 'd'])
@@ -153,10 +153,12 @@
 
         res = self.meta_interp(f, [100], listops=True)
         assert res == f(50)
-        self.check_loops({"call": 7, "guard_false": 1, "guard_no_exception": 6,
+        self.check_loops({"call": 5, "getfield_gc": 1, "getinteriorfield_gc": 1,
+                          "guard_false": 1, "guard_no_exception": 4,
                           "guard_true": 1, "int_and": 1, "int_gt": 1,
                           "int_is_true": 1, "int_sub": 1, "jump": 1,
-                          "new_with_vtable": 1, "setfield_gc": 1})
+                          "new_with_vtable": 1, "new": 1, "new_array": 1,
+                          "setfield_gc": 3, })
 
 
 class TestOOtype(DictTests, OOJitMixin):
diff --git a/pypy/jit/metainterp/test/test_heapcache.py b/pypy/jit/metainterp/test/test_heapcache.py
--- a/pypy/jit/metainterp/test/test_heapcache.py
+++ b/pypy/jit/metainterp/test/test_heapcache.py
@@ -355,6 +355,14 @@
         assert not h.is_unescaped(box1)
         assert not h.is_unescaped(box2)
 
+    def test_circular_virtuals(self):
+        h = HeapCache()
+        h.new(box1)
+        h.new(box2)
+        h.invalidate_caches(rop.SETFIELD_GC, None, [box1, box2])
+        h.invalidate_caches(rop.SETFIELD_GC, None, [box2, box1])
+        h.invalidate_caches(rop.SETFIELD_GC, None, [box3, box1]) # does not crash
+
     def test_unescaped_array(self):
         h = HeapCache()
         h.new_array(box1, lengthbox1)
@@ -362,4 +370,4 @@
         h.invalidate_caches(rop.SETARRAYITEM_GC, None, [box1, index1, box2])
         assert h.is_unescaped(box1)
         h.invalidate_caches(rop.SETARRAYITEM_GC, None, [box2, index1, box1])
-        assert not h.is_unescaped(box1)
\ No newline at end of file
+        assert not h.is_unescaped(box1)
diff --git a/pypy/jit/metainterp/test/test_memmgr.py b/pypy/jit/metainterp/test/test_memmgr.py
--- a/pypy/jit/metainterp/test/test_memmgr.py
+++ b/pypy/jit/metainterp/test/test_memmgr.py
@@ -18,6 +18,7 @@
 
 class FakeLoopToken:
     generation = 0
+    invalidated = False
 
 
 class _TestMemoryManager:
diff --git a/pypy/jit/metainterp/test/test_quasiimmut.py b/pypy/jit/metainterp/test/test_quasiimmut.py
--- a/pypy/jit/metainterp/test/test_quasiimmut.py
+++ b/pypy/jit/metainterp/test/test_quasiimmut.py
@@ -48,6 +48,13 @@
 
 class QuasiImmutTests(object):
 
+    def setup_method(self, meth):
+        self.prev_compress_limit = QuasiImmut.compress_limit
+        QuasiImmut.compress_limit = 1
+
+    def teardown_method(self, meth):
+        QuasiImmut.compress_limit = self.prev_compress_limit
+
     def test_simple_1(self):
         myjitdriver = JitDriver(greens=['foo'], reds=['x', 'total'])
         class Foo:
@@ -289,7 +296,7 @@
             return total
 
         res = self.meta_interp(main, [])
-        self.check_loop_count(9)
+        self.check_tree_loop_count(6)
         assert res == main()
 
     def test_change_during_running(self):
@@ -317,7 +324,7 @@
         assert f(100, 15) == 3009
         res = self.meta_interp(f, [100, 15])
         assert res == 3009
-        self.check_loops(guard_not_invalidated=2, getfield_gc=0,
+        self.check_loops(guard_not_invalidated=4, getfield_gc=0,
                          call_may_force=0, guard_not_forced=0)
 
     def test_list_simple_1(self):
@@ -453,10 +460,30 @@
         assert f(100, 15) == 3009
         res = self.meta_interp(f, [100, 15])
         assert res == 3009
-        self.check_loops(guard_not_invalidated=2, getfield_gc=0,
+        self.check_loops(guard_not_invalidated=4, getfield_gc=0,
                          getarrayitem_gc=0, getarrayitem_gc_pure=0,
                          call_may_force=0, guard_not_forced=0)
 
+    def test_invalidated_loop_is_not_used_any_more_as_target(self):
+        myjitdriver = JitDriver(greens=['foo'], reds=['x'])
+        class Foo:
+            _immutable_fields_ = ['step?']
+        @dont_look_inside
+        def residual(x, foo):
+            if x == 20:
+                foo.step = 1
+        def f(x):
+            foo = Foo()
+            foo.step = 2
+            while x > 0:
+                myjitdriver.jit_merge_point(foo=foo, x=x)
+                residual(x, foo)
+                x -= foo.step
+            return foo.step
+        res = self.meta_interp(f, [60])
+        assert res == 1
+        self.check_tree_loop_count(4)   # at least not 2 like before
+
 
 class TestLLtypeGreenFieldsTests(QuasiImmutTests, LLJitMixin):
     pass
diff --git a/pypy/jit/metainterp/test/test_resume.py b/pypy/jit/metainterp/test/test_resume.py
--- a/pypy/jit/metainterp/test/test_resume.py
+++ b/pypy/jit/metainterp/test/test_resume.py
@@ -1,3 +1,4 @@
+from __future__ import with_statement
 import py
 from pypy.rpython.lltypesystem import lltype, llmemory, rffi
 from pypy.jit.metainterp.optimizeopt.optimizer import OptValue
@@ -180,22 +181,27 @@
     reader.consume_boxes(info, bi, br, bf)
     b1s = reader.liveboxes[0]
     b2s = reader.liveboxes[1]
-    assert bi == [b1s, ConstInt(111), b1s]
-    assert br == [ConstPtr(gcrefnull), b2s]
+    assert_same(bi, [b1s, ConstInt(111), b1s])
+    assert_same(br, [ConstPtr(gcrefnull), b2s])
     bi, br, bf = [None]*2, [None]*0, [None]*0
     info = MyBlackholeInterp([lltype.Signed,
                               lltype.Signed]).get_current_position_info()
     reader.consume_boxes(info, bi, br, bf)
-    assert bi == [ConstInt(222), ConstInt(333)]
+    assert_same(bi, [ConstInt(222), ConstInt(333)])
     bi, br, bf = [None]*2, [None]*1, [None]*0
     info = MyBlackholeInterp([lltype.Signed, llmemory.GCREF,
                               lltype.Signed]).get_current_position_info()
     reader.consume_boxes(info, bi, br, bf)
     b3s = reader.liveboxes[2]
-    assert bi == [b1s, b3s]
-    assert br == [b2s]
+    assert_same(bi, [b1s, b3s])
+    assert_same(br, [b2s])
     #
 
+def assert_same(list1, list2):
+    assert len(list1) == len(list2)
+    for b1, b2 in zip(list1, list2):
+        assert b1.same_box(b2)
+
 def test_simple_read_tagged_ints():
     storage = Storage()
     storage.rd_consts = []
@@ -1023,11 +1029,11 @@
     metainterp = MyMetaInterp()
     reader = ResumeDataFakeReader(storage, newboxes, metainterp)
     lst = reader.consume_boxes()
-    assert lst == [b1t, b1t, b3t]
+    assert_same(lst, [b1t, b1t, b3t])
     lst = reader.consume_boxes()
-    assert lst == [ConstInt(2), ConstInt(3)]
+    assert_same(lst, [ConstInt(2), ConstInt(3)])
     lst = reader.consume_boxes()
-    assert lst == [b1t, ConstInt(1), b1t, b1t]
+    assert_same(lst, [b1t, ConstInt(1), b1t, b1t])
     assert metainterp.trace == []    
 
 
@@ -1044,11 +1050,11 @@
     reader = ResumeDataFakeReader(storage, newboxes, metainterp)
     lst = reader.consume_boxes()
     c1t = ConstInt(111)
-    assert lst == [c1t, b2t, b3t]
+    assert_same(lst, [c1t, b2t, b3t])
     lst = reader.consume_boxes()
-    assert lst == [ConstInt(2), ConstInt(3)]
+    assert_same(lst, [ConstInt(2), ConstInt(3)])
     lst = reader.consume_boxes()
-    assert lst == [c1t, ConstInt(1), c1t, b2t]
+    assert_same(lst, [c1t, ConstInt(1), c1t, b2t])
     assert metainterp.trace == []
 
 
@@ -1114,9 +1120,11 @@
     # check that we get the operations in 'expected', in a possibly different
     # order.
     assert len(trace) == len(expected)
-    for x in trace:
-        assert x in expected
-        expected.remove(x)
+    with CompareableConsts():
+        for x in trace:
+            assert x in expected
+            expected.remove(x)
+
     ptr = b2t.value._obj.container._as_ptr()
     assert lltype.typeOf(ptr) == lltype.Ptr(LLtypeMixin.NODE)
     assert ptr.value == 111
@@ -1126,6 +1134,18 @@
     assert ptr2.parent.value == 33
     assert ptr2.parent.next == ptr
 
+class CompareableConsts(object):
+    def __init__(self):
+        self.oldeq = None
+        
+    def __enter__(self):
+        assert self.oldeq is None
+        self.oldeq = Const.__eq__
+        Const.__eq__ = Const.same_box
+        
+    def __exit__(self, type, value, traceback):
+        Const.__eq__ = self.oldeq
+
 def test_virtual_adder_make_varray():
     b2s, b4s = [BoxPtr(), BoxInt(4)]
     c1s = ConstInt(111)
@@ -1163,8 +1183,9 @@
         (rop.SETARRAYITEM_GC, [b2t,ConstInt(1), c1s], None,
                               LLtypeMixin.arraydescr),
         ]
-    for x, y in zip(expected, trace):
-        assert x == y
+    with CompareableConsts():
+        for x, y in zip(expected, trace):
+            assert x == y
     #
     ptr = b2t.value._obj.container._as_ptr()
     assert lltype.typeOf(ptr) == lltype.Ptr(lltype.GcArray(lltype.Signed))
@@ -1207,8 +1228,9 @@
         (rop.SETFIELD_GC, [b2t, c1s],  None, LLtypeMixin.adescr),
         (rop.SETFIELD_GC, [b2t, b4t], None, LLtypeMixin.bdescr),
         ]
-    for x, y in zip(expected, trace):
-        assert x == y
+    with CompareableConsts():
+        for x, y in zip(expected, trace):
+            assert x == y
     #
     ptr = b2t.value._obj.container._as_ptr()
     assert lltype.typeOf(ptr) == lltype.Ptr(LLtypeMixin.S)
diff --git a/pypy/jit/metainterp/test/test_string.py b/pypy/jit/metainterp/test/test_string.py
--- a/pypy/jit/metainterp/test/test_string.py
+++ b/pypy/jit/metainterp/test/test_string.py
@@ -3,7 +3,8 @@
 from pypy.jit.codewriter.policy import StopAtXPolicy
 from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin
 from pypy.rlib.debug import debug_print
-from pypy.rlib.jit import JitDriver, dont_look_inside, we_are_jitted
+from pypy.rlib.jit import JitDriver, dont_look_inside, we_are_jitted,\
+     promote_string
 from pypy.rlib.rstring import StringBuilder
 from pypy.rpython.ootypesystem import ootype
 
@@ -507,6 +508,18 @@
                           'jump': 1, 'int_is_true': 1,
                           'guard_not_invalidated': 1})
 
+    def test_promote_string(self):
+        driver = JitDriver(greens = [], reds = ['n'])
+        
+        def f(n):
+            while n < 21:
+                driver.jit_merge_point(n=n)
+                promote_string(str(n % 3))
+                n += 1
+            return 0
+
+        self.meta_interp(f, [0])
+        self.check_loops(call=3 + 1) # one for int2str
 
 #class TestOOtype(StringTests, OOJitMixin):
 #    CALL = "oosend"
diff --git a/pypy/jit/metainterp/test/test_virtualref.py b/pypy/jit/metainterp/test/test_virtualref.py
--- a/pypy/jit/metainterp/test/test_virtualref.py
+++ b/pypy/jit/metainterp/test/test_virtualref.py
@@ -3,6 +3,7 @@
 from pypy.rpython.llinterp import LLException
 from pypy.rlib.jit import JitDriver, dont_look_inside, vref_None
 from pypy.rlib.jit import virtual_ref, virtual_ref_finish, InvalidVirtualRef
+from pypy.rlib.jit import non_virtual_ref
 from pypy.rlib.objectmodel import compute_unique_id
 from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin, _get_jitcodes
 from pypy.jit.metainterp.resoperation import rop
@@ -595,6 +596,65 @@
         res = self.meta_interp(fn, [10])
         assert res == 6
 
+    def test_is_virtual(self):
+        myjitdriver = JitDriver(greens=[], reds=['n', 'res1'])
+        class X:
+            pass
+        @dont_look_inside
+        def residual(vref):
+            return vref.virtual
+        #
+        def f(n):
+            res1 = -42
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, res1=res1)
+                x = X()
+                vref = virtual_ref(x)
+                res1 = residual(vref)
+                virtual_ref_finish(vref, x)
+                n -= 1
+            return res1
+        #
+        res = self.meta_interp(f, [10])
+        assert res == 1
+
+    def test_is_not_virtual_none(self):
+        myjitdriver = JitDriver(greens=[], reds=['n', 'res1'])
+        @dont_look_inside
+        def residual(vref):
+            return vref.virtual
+        #
+        def f(n):
+            res1 = -42
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, res1=res1)
+                res1 = residual(vref_None)
+                n -= 1
+            return res1
+        #
+        res = self.meta_interp(f, [10])
+        assert res == 0
+
+    def test_is_not_virtual_non_none(self):
+        myjitdriver = JitDriver(greens=[], reds=['n', 'res1'])
+        class X:
+            pass
+        @dont_look_inside
+        def residual(vref):
+            return vref.virtual
+        #
+        def f(n):
+            res1 = -42
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, res1=res1)
+                x = X()
+                res1 = residual(non_virtual_ref(x))
+                n -= 1
+            return res1
+        #
+        res = self.meta_interp(f, [10])
+        assert res == 0
+
 
 class TestLLtype(VRefTests, LLJitMixin):
     pass
diff --git a/pypy/jit/metainterp/virtualref.py b/pypy/jit/metainterp/virtualref.py
--- a/pypy/jit/metainterp/virtualref.py
+++ b/pypy/jit/metainterp/virtualref.py
@@ -39,6 +39,7 @@
     def replace_force_virtual_with_call(self, graphs):
         # similar to rvirtualizable2.replace_force_virtualizable_with_call().
         c_force_virtual_ptr = None
+        c_is_virtual_ptr = None
         force_virtual_count = 0
         for graph in graphs:
             for block in graph.iterblocks():
@@ -52,6 +53,13 @@
                         op.opname = 'direct_call'
                         op.args = [c_force_virtual_ptr, op.args[0]]
                         force_virtual_count += 1
+                    #
+                    if op.opname == 'jit_is_virtual':
+                        if c_is_virtual_ptr is None:
+                            c_is_virtual_ptr = self.get_is_virtual_fnptr()
+                        #
+                        op.opname = 'direct_call'
+                        op.args = [c_is_virtual_ptr, op.args[0]]
         #
         if c_force_virtual_ptr is not None:
             log("replaced %d 'jit_force_virtual' with %r" % (force_virtual_count,
@@ -129,6 +137,17 @@
             force_virtual_if_necessary)
         return inputconst(lltype.typeOf(funcptr), funcptr)
 
+    def get_is_virtual_fnptr(self):
+        #
+        def is_virtual(inst):
+            if not inst:
+                return False
+            return inst.typeptr == self.jit_virtual_ref_vtable
+        #
+        FUNC = lltype.FuncType([rclass.OBJECTPTR], lltype.Bool)
+        funcptr = self.warmrunnerdesc.helper_func(lltype.Ptr(FUNC), is_virtual)
+        return inputconst(lltype.typeOf(funcptr), funcptr)
+
     def force_virtual(self, inst):
         vref = lltype.cast_pointer(lltype.Ptr(self.JIT_VIRTUAL_REF), inst)
         token = vref.virtual_token
diff --git a/pypy/jit/metainterp/warmspot.py b/pypy/jit/metainterp/warmspot.py
--- a/pypy/jit/metainterp/warmspot.py
+++ b/pypy/jit/metainterp/warmspot.py
@@ -206,7 +206,7 @@
         self.make_enter_functions()
         self.rewrite_jit_merge_points(policy)
 
-        verbose = not self.cpu.translate_support_code
+        verbose = False # not self.cpu.translate_support_code
         self.codewriter.make_jitcodes(verbose=verbose)
         self.rewrite_can_enter_jits()
         self.rewrite_set_param()
diff --git a/pypy/jit/metainterp/warmstate.py b/pypy/jit/metainterp/warmstate.py
--- a/pypy/jit/metainterp/warmstate.py
+++ b/pypy/jit/metainterp/warmstate.py
@@ -178,7 +178,7 @@
         if self.compiled_merge_points_wref is not None:
             for wref in self.compiled_merge_points_wref:
                 looptoken = wref()
-                if looptoken is not None:
+                if looptoken is not None and not looptoken.invalidated:
                     result.append(looptoken)
         return result
 
diff --git a/pypy/module/__builtin__/__init__.py b/pypy/module/__builtin__/__init__.py
--- a/pypy/module/__builtin__/__init__.py
+++ b/pypy/module/__builtin__/__init__.py
@@ -20,6 +20,10 @@
         'any'           : 'app_functional.any',
         'all'           : 'app_functional.all',
         'sum'           : 'app_functional.sum',
+        'map'           : 'app_functional.map',
+        'reduce'        : 'app_functional.reduce',
+        'filter'        : 'app_functional.filter',
+        'zip'           : 'app_functional.zip',
         'vars'          : 'app_inspect.vars',
         'dir'           : 'app_inspect.dir',
 
@@ -86,11 +90,7 @@
         'enumerate'     : 'functional.W_Enumerate',
         'min'           : 'functional.min',
         'max'           : 'functional.max',
-        'map'           : 'functional.map',
-        'zip'           : 'functional.zip',
-        'reduce'        : 'functional.reduce',
         'reversed'      : 'functional.reversed',
-        'filter'        : 'functional.filter',
         'super'         : 'descriptor.W_Super',
         'staticmethod'  : 'descriptor.StaticMethod',
         'classmethod'   : 'descriptor.ClassMethod',
@@ -119,7 +119,7 @@
            builtin = space.interpclass_w(w_builtin)
            if isinstance(builtin, module.Module):
                return builtin
-       # no builtin! make a default one.  Given them None, at least.
+       # no builtin! make a default one.  Give them None, at least.
        builtin = module.Module(space, None)
        space.setitem(builtin.w_dict, space.wrap('None'), space.w_None)
        return builtin
diff --git a/pypy/module/__builtin__/app_functional.py b/pypy/module/__builtin__/app_functional.py
--- a/pypy/module/__builtin__/app_functional.py
+++ b/pypy/module/__builtin__/app_functional.py
@@ -48,4 +48,135 @@
         # Very intentionally *not* +=, that would have different semantics if
         # start was a mutable type, such as a list
         last = last + x
-    return last
\ No newline at end of file
+    return last
+
+def map(func, *collections):
+    """map(function, sequence[, sequence, ...]) -> list
+
+Return a list of the results of applying the function to the items of
+the argument sequence(s).  If more than one sequence is given, the
+function is called with an argument list consisting of the corresponding
+item of each sequence, substituting None for missing values when not all
+sequences have the same length.  If the function is None, return a list of
+the items of the sequence (or a list of tuples if more than one sequence)."""
+    if not collections:
+        raise TypeError("map() requires at least two arguments")
+    num_collections = len(collections)
+    none_func = func is None
+    if num_collections == 1:
+        if none_func:
+            return list(collections[0])
+        else:
+            # Special case for the really common case of a single collection,
+            # this can be eliminated if we could unroll that loop that creates
+            # `args` based on whether or not len(collections) was constant
+            result = []
+            for item in collections[0]:
+                result.append(func(item))
+            return result
+    result = []
+    # Pair of (iterator, has_finished)
+    iterators = [(iter(seq), False) for seq in collections]
+    while True:
+        cont = False
+        args = []
+        for idx, (iterator, has_finished) in enumerate(iterators):
+            val = None
+            if not has_finished:
+                try:
+                    val = next(iterator)
+                except StopIteration:
+                    iterators[idx] = (None, True)
+                else:
+                    cont = True
+            args.append(val)
+        args = tuple(args)
+        if cont:
+            if none_func:
+                result.append(args)
+            else:
+                result.append(func(*args))
+        else:
+            return result
+
+sentinel = object()
+
+def reduce(func, sequence, initial=sentinel):
+    """reduce(function, sequence[, initial]) -> value
+
+Apply a function of two arguments cumulatively to the items of a sequence,
+from left to right, so as to reduce the sequence to a single value.
+For example, reduce(lambda x, y: x+y, [1, 2, 3, 4, 5]) calculates
+((((1+2)+3)+4)+5).  If initial is present, it is placed before the items
+of the sequence in the calculation, and serves as a default when the
+sequence is empty."""
+    iterator = iter(sequence)
+    if initial is sentinel:
+        try:
+            initial = next(iterator)
+        except StopIteration:
+            raise TypeError("reduce() of empty sequence with no initial value")
+    result = initial
+    for item in iterator:
+        result = func(result, item)
+    return result
+
+def filter(func, seq):
+    """filter(function or None, sequence) -> list, tuple, or string
+
+Return those items of sequence for which function(item) is true.  If
+function is None, return the items that are true.  If sequence is a tuple
+or string, return the same type, else return a list."""
+    if func is None:
+        func = bool
+    if isinstance(seq, str):
+        return _filter_string(func, seq, str)
+    elif isinstance(seq, unicode):
+        return _filter_string(func, seq, unicode)
+    elif isinstance(seq, tuple):
+        return _filter_tuple(func, seq)
+    result = []
+    for item in seq:
+        if func(item):
+            result.append(item)
+    return result
+
+def _filter_string(func, string, str_type):
+    if func is bool and type(string) is str_type:
+        return string
+    result = []
+    for i in range(len(string)):
+        # You must call __getitem__ on the strings, simply iterating doesn't
+        # work :/
+        item = string[i]
+        if func(item):
+            if not isinstance(item, str_type):
+                raise TypeError("__getitem__ returned a non-string type")
+            result.append(item)
+    return str_type().join(result)
+
+def _filter_tuple(func, seq):
+    result = []
+    for i in range(len(seq)):
+        # Again, must call __getitem__, at least there are tests.
+        item = seq[i]
+        if func(item):
+            result.append(item)
+    return tuple(result)
+
+def zip(*sequences):
+    """zip(seq1 [, seq2 [...]]) -> [(seq1[0], seq2[0] ...), (...)]
+
+Return a list of tuples, where each tuple contains the i-th element
+from each of the argument sequences.  The returned list is truncated
+in length to the length of the shortest argument sequence."""
+    if not sequences:
+        return []
+    result = []
+    iterators = [iter(seq) for seq in sequences]
+    while True:
+        try:
+            items = [next(it) for it in iterators]
+        except StopIteration:
+            return result
+        result.append(tuple(items))
diff --git a/pypy/module/__builtin__/app_io.py b/pypy/module/__builtin__/app_io.py
--- a/pypy/module/__builtin__/app_io.py
+++ b/pypy/module/__builtin__/app_io.py
@@ -27,7 +27,20 @@
     co = compile(source.rstrip()+"\n", filename, 'exec')
     exec co in glob, loc
 
-def raw_input(prompt=None):
+def _write_prompt(stdout, prompt):
+    print >> stdout, prompt,
+    try:
+        flush = stdout.flush
+    except AttributeError:
+        pass
+    else:
+        flush()
+    try:
+        stdout.softspace = 0
+    except (AttributeError, TypeError):
+        pass
+
+def raw_input(prompt=''):
     """raw_input([prompt]) -> string
 
 Read a string from standard input.  The trailing newline is stripped.
@@ -47,18 +60,10 @@
     if (hasattr(sys, '__raw_input__') and
         isinstance(stdin, file)  and stdin.fileno() == 0 and stdin.isatty() and
         isinstance(stdout, file) and stdout.fileno() == 1):
-        if prompt is None:
-            prompt = ''
-        return sys.__raw_input__(prompt)
+        _write_prompt(stdout, '')
+        return sys.__raw_input__(str(prompt))
 
-    if prompt is not None:
-        stdout.write(prompt)
-        try:
-            flush = stdout.flush
-        except AttributeError:
-            pass
-        else:
-            flush()
+    _write_prompt(stdout, prompt)
     line = stdin.readline()
     if not line:    # inputting an empty line gives line == '\n'
         raise EOFError
@@ -66,7 +71,7 @@
         return line[:-1]
     return line
 
-def input(prompt=None):
+def input(prompt=''):
     """Equivalent to eval(raw_input(prompt))."""
     return eval(raw_input(prompt))
 
diff --git a/pypy/module/__builtin__/compiling.py b/pypy/module/__builtin__/compiling.py
--- a/pypy/module/__builtin__/compiling.py
+++ b/pypy/module/__builtin__/compiling.py
@@ -84,8 +84,8 @@
         raise OperationError(space.w_TypeError,
               w('eval() arg 1 must be a string or code object'))
 
-    caller = space.getexecutioncontext().gettopframe_nohidden()
     if space.is_w(w_globals, space.w_None):
+        caller = space.getexecutioncontext().gettopframe_nohidden()
         if caller is None:
             w_globals = space.newdict()
             if space.is_w(w_locals, space.w_None):
@@ -97,13 +97,9 @@
     elif space.is_w(w_locals, space.w_None):
         w_locals = w_globals
 
-    try:
-        space.getitem(w_globals, space.wrap('__builtins__'))
-    except OperationError, e:
-        if not e.match(space, space.w_KeyError):
-            raise
-        if caller is not None:
-            w_builtin = space.builtin.pick_builtin(caller.w_globals)
-            space.setitem(w_globals, space.wrap('__builtins__'), w_builtin)
+    # xxx removed: adding '__builtins__' to the w_globals dict, if there
+    # is none.  This logic was removed as costly (it requires to get at
+    # the gettopframe_nohidden()).  I bet no test fails, and it's a really
+    # obscure case.
 
     return codeobj.exec_code(space, w_globals, w_locals)
diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py
--- a/pypy/module/__builtin__/functional.py
+++ b/pypy/module/__builtin__/functional.py
@@ -201,223 +201,6 @@
     """
     return min_max(space, __args__, "min")
 
- at unwrap_spec(collections_w="args_w")
-def map(space, w_func, collections_w):
-    """does 3 separate things, hence this enormous docstring.
-       1.  if function is None, return a list of tuples, each with one
-           item from each collection.  If the collections have different
-           lengths,  shorter ones are padded with None.
-
-       2.  if function is not None, and there is only one collection,
-           apply function to every item in the collection and return a
-           list of the results.
-
-       3.  if function is not None, and there are several collections,
-           repeatedly call the function with one argument from each
-           collection.  If the collections have different lengths,
-           shorter ones are padded with None
-    """
-    if not collections_w:
-        msg = "map() requires at least two arguments"
-        raise OperationError(space.w_TypeError, space.wrap(msg))
-    none_func = space.is_w(w_func, space.w_None)
-    if len(collections_w) == 1:
-        w_collection = collections_w[0]
-        if none_func:
-            result_w = space.unpackiterable(w_collection)
-        else:
-            result_w = map_single_collection(space, w_func, w_collection)
-    else:
-        result_w = map_multiple_collections(space, w_func, collections_w,
-                                            none_func)
-    return space.newlist(result_w)
-
-def map_single_collection(space, w_func, w_collection):
-    """Special case for 'map(func, coll)', where 'func' is not None and there
-    is only one 'coll' argument."""
-    w_iter = space.iter(w_collection)
-    # xxx special hacks for speed
-    from pypy.interpreter import function, pycode
-    if isinstance(w_func, function.Function):
-        # xxx compatibility issue: what if func_code is modified in the
-        # middle of running map()??  That's far too obscure for me to care...
-        code = w_func.getcode()
-        fast_natural_arity = code.fast_natural_arity
-        if fast_natural_arity == (1|pycode.PyCode.FLATPYCALL):
-            assert isinstance(code, pycode.PyCode)
-            return map_single_user_function(code, w_func, w_iter)
-    # /xxx end of special hacks
-    return map_single_other_callable(space, w_func, w_iter)
-
-def map_single_other_callable(space, w_func, w_iter):
-    result_w = []
-    while True:
-        try:
-            w_item = space.next(w_iter)
-        except OperationError, e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            break
-        result_w.append(space.call_function(w_func, w_item))
-    return result_w
-map_single_other_callable._dont_inline_ = True
-
-from pypy.rlib.jit import JitDriver
-mapjitdriver = JitDriver(greens = ['code'],
-                         reds = ['w_func', 'w_iter', 'result_w'])
-def map_single_user_function(code, w_func, w_iter):
-    result_w = []
-    while True:
-        mapjitdriver.can_enter_jit(code=code, w_func=w_func,
-                                   w_iter=w_iter, result_w=result_w)
-        mapjitdriver.jit_merge_point(code=code, w_func=w_func,
-                                     w_iter=w_iter, result_w=result_w)
-        space = w_func.space
-        try:
-            w_item = space.next(w_iter)
-        except OperationError, e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            break
-        new_frame = space.createframe(code, w_func.w_func_globals,
-                                      w_func)
-        new_frame.locals_stack_w[0] = w_item
-        w_res = new_frame.run()
-        result_w.append(w_res)
-    return result_w
-
-def map_multiple_collections(space, w_func, collections_w, none_func):
-    result_w = []
-    iterators_w = [space.iter(w_seq) for w_seq in collections_w]
-    num_iterators = len(iterators_w)
-    while True:
-        cont = False
-        args_w = [space.w_None] * num_iterators
-        for i in range(num_iterators):
-            if iterators_w[i] is not None:
-                try:
-                    args_w[i] = space.next(iterators_w[i])
-                except OperationError, e:
-                    if not e.match(space, space.w_StopIteration):
-                        raise
-                    iterators_w[i] = None
-                else:
-                    cont = True
-        if not cont:
-            break
-        w_args = space.newtuple(args_w)
-        if none_func:
-            w_res = w_args
-        else:
-            w_res = space.call(w_func, w_args)
-        result_w.append(w_res)
-    return result_w
-
- at unwrap_spec(sequences_w="args_w")
-def zip(space, sequences_w):
-    """Return a list of tuples, where the nth tuple contains every nth item of
-    each collection.
-
-    If the collections have different lengths, zip returns a list as long as the
-    shortest collection, ignoring the trailing items in the other collections.
-    """
-    if not sequences_w:
-        return space.newlist([])
-    result_w = []
-    iterators_w = [space.iter(w_seq) for w_seq in sequences_w]
-    while True:
-        try:
-            items_w = [space.next(w_it) for w_it in iterators_w]
-        except OperationError, e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            return space.newlist(result_w)
-        result_w.append(space.newtuple(items_w))
-
-def reduce(space, w_func, w_sequence, w_initial=NoneNotWrapped):
-    """ Apply function of two arguments cumulatively to the items of sequence,
-        from left to right, so as to reduce the sequence to a single value.
-        Optionally begin with an initial value.
-    """
-    w_iter = space.iter(w_sequence)
-    if w_initial is None:
-        try:
-            w_initial = space.next(w_iter)
-        except OperationError, e:
-            if e.match(space, space.w_StopIteration):
-                msg = "reduce() of empty sequence with no initial value"
-                raise OperationError(space.w_TypeError, space.wrap(msg))
-            raise
-    w_result = w_initial
-    while True:
-        try:
-            w_next = space.next(w_iter)
-        except OperationError, e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            break
-        w_result = space.call_function(w_func, w_result, w_next)
-    return w_result
-
-def filter(space, w_func, w_seq):
-    """construct a list of those elements of collection for which function
-       is True.  If function is None, then return the items in the sequence
-       which are True.
-    """
-    if space.is_true(space.isinstance(w_seq, space.w_str)):
-        return _filter_string(space, w_func, w_seq, space.w_str)
-    if space.is_true(space.isinstance(w_seq, space.w_unicode)):
-        return _filter_string(space, w_func, w_seq, space.w_unicode)
-    if space.is_true(space.isinstance(w_seq, space.w_tuple)):
-        return _filter_tuple(space, w_func, w_seq)
-    w_iter = space.iter(w_seq)
-    result_w = []
-    none_func = space.is_w(w_func, space.w_None)
-    while True:
-        try:
-            w_next = space.next(w_iter)
-        except OperationError, e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            break
-        if none_func:
-            w_keep = w_next
-        else:
-            w_keep = space.call_function(w_func, w_next)
-        if space.is_true(w_keep):
-            result_w.append(w_next)
-    return space.newlist(result_w)
-
-def _filter_tuple(space, w_func, w_tuple):
-    none_func = space.is_w(w_func, space.w_None)
-    length = space.len_w(w_tuple)
-    result_w = []
-    for i in range(length):
-        w_item = space.getitem(w_tuple, space.wrap(i))
-        if none_func:
-            w_keep = w_item
-        else:
-            w_keep = space.call_function(w_func, w_item)
-        if space.is_true(w_keep):
-            result_w.append(w_item)
-    return space.newtuple(result_w)
-
-def _filter_string(space, w_func, w_string, w_str_type):
-    none_func = space.is_w(w_func, space.w_None)
-    if none_func and space.is_w(space.type(w_string), w_str_type):
-        return w_string
-    length = space.len_w(w_string)
-    result_w = []
-    for i in range(length):
-        w_item = space.getitem(w_string, space.wrap(i))
-        if none_func or space.is_true(space.call_function(w_func, w_item)):
-            if not space.is_true(space.isinstance(w_item, w_str_type)):
-                msg = "__getitem__ returned a non-string type"
-                raise OperationError(space.w_TypeError, space.wrap(msg))
-            result_w.append(w_item)
-    w_empty = space.call_function(w_str_type)
-    return space.call_method(w_empty, "join", space.newlist(result_w))
-
 class W_Enumerate(Wrappable):
 
     def __init__(self, w_iter, w_start):
diff --git a/pypy/module/__builtin__/test/test_functional.py b/pypy/module/__builtin__/test/test_functional.py
--- a/pypy/module/__builtin__/test/test_functional.py
+++ b/pypy/module/__builtin__/test/test_functional.py
@@ -147,7 +147,7 @@
        assert list(xrange(A())) == [0, 1, 2, 3, 4]
        assert list(xrange(0, A())) == [0, 1, 2, 3, 4]
        assert list(xrange(0, 10, A())) == [0, 5]
- 
+
    def test_xrange_float(self):
       assert list(xrange(0.1, 2.0, 1.1)) == [0, 1]
 
diff --git a/pypy/module/__builtin__/test/test_rawinput.py b/pypy/module/__builtin__/test/test_rawinput.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/__builtin__/test/test_rawinput.py
@@ -0,0 +1,80 @@
+import autopath
+
+
+class AppTestRawInput():
+
+    def test_input_and_raw_input(self):
+        import sys, StringIO
+        for prompt, expected in [("def:", "abc/ def:/ghi\n"),
+                                 ("", "abc/ /ghi\n"),
+                                 (42, "abc/ 42/ghi\n"),
+                                 (None, "abc/ None/ghi\n"),
+                                 (Ellipsis, "abc/ /ghi\n")]:
+            for inputfn, inputtext, gottext in [
+                    (raw_input, "foo\nbar\n", "foo"),
+                    (input, "40+2\n", 42)]:
+                save = sys.stdin, sys.stdout
+                try:
+                    sys.stdin = StringIO.StringIO(inputtext)
+                    out = sys.stdout = StringIO.StringIO()
+                    print "abc",    # softspace = 1
+                    out.write('/')
+                    if prompt is Ellipsis:
+                        got = inputfn()
+                    else:
+                        got = inputfn(prompt)
+                    out.write('/')
+                    print "ghi"
+                finally:
+                    sys.stdin, sys.stdout = save
+                assert out.getvalue() == expected
+                assert got == gottext
+
+    def test_softspace(self):
+        import sys
+        import StringIO
+        fin = StringIO.StringIO()
+        fout = StringIO.StringIO()
+
+        fin.write("Coconuts\n")
+        fin.seek(0)
+
+        sys_stdin_orig = sys.stdin
+        sys_stdout_orig = sys.stdout
+
+        sys.stdin = fin
+        sys.stdout = fout
+
+        print "test",
+        raw_input("test")
+
+        sys.stdin = sys_stdin_orig
+        sys.stdout = sys_stdout_orig
+
+        fout.seek(0)
+        assert fout.read() == "test test"
+
+    def test_softspace_carryover(self):
+        import sys
+        import StringIO
+        fin = StringIO.StringIO()
+        fout = StringIO.StringIO()
+
+        fin.write("Coconuts\n")
+        fin.seek(0)
+
+        sys_stdin_orig = sys.stdin
+        sys_stdout_orig = sys.stdout
+
+        sys.stdin = fin
+        sys.stdout = fout
+
+        print "test",
+        raw_input("test")
+        print "test",
+
+        sys.stdin = sys_stdin_orig
+        sys.stdout = sys_stdout_orig
+
+        fout.seek(0)
+        assert fout.read() == "test testtest"
diff --git a/pypy/module/_continuation/interp_continuation.py b/pypy/module/_continuation/interp_continuation.py
--- a/pypy/module/_continuation/interp_continuation.py
+++ b/pypy/module/_continuation/interp_continuation.py
@@ -19,6 +19,11 @@
         #  - normal:      self.sthread != None, not is_empty_handle(self.h)
         #  - finished:    self.sthread != None, is_empty_handle(self.h)
 
+    def __del__(self):
+        sthread = self.sthread
+        if sthread is not None and not sthread.is_empty_handle(self.h):
+            sthread.destroy(self.h)
+
     def check_sthread(self):
         ec = self.space.getexecutioncontext()
         if ec.stacklet_thread is not self.sthread:
@@ -28,6 +33,8 @@
     def descr_init(self, w_callable, __args__):
         if self.sthread is not None:
             raise geterror(self.space, "continulet already __init__ialized")
+        sthread = build_sthread(self.space)
+        workaround_disable_jit(sthread)
         #
         # hackish: build the frame "by hand", passing it the correct arguments
         space = self.space
@@ -41,7 +48,6 @@
         self.bottomframe = bottomframe
         #
         global_state.origin = self
-        sthread = build_sthread(self.space)
         self.sthread = sthread
         h = sthread.new(new_stacklet_callback)
         post_switch(sthread, h)
@@ -71,6 +77,7 @@
                 global_state.clear()
                 raise geterror(self.space, "continulet already finished")
         self.check_sthread()
+        workaround_disable_jit(self.sthread)
         #
         global_state.origin = self
         if to is None:
@@ -259,6 +266,16 @@
         sthread = ec.stacklet_thread = SThread(space, ec)
     return sthread
 
+def workaround_disable_jit(sthread):
+    # A bad workaround to kill the JIT anywhere in this thread.
+    # This forces all the frames.  It's a bad workaround because
+    # it takes O(depth) time, and it will cause some "abort:
+    # vable escape" in the JIT.  The goal is to prevent any frame
+    # from being still virtuals, because the JIT generates code
+    # to un-virtualizable them "on demand" by loading values based
+    # on FORCE_TOKEN, which is an address in the stack.
+    sthread.ec.force_all_frames()
+
 # ____________________________________________________________
 
 def permute(space, args_w):
diff --git a/pypy/module/_file/test/test_file.py b/pypy/module/_file/test/test_file.py
--- a/pypy/module/_file/test/test_file.py
+++ b/pypy/module/_file/test/test_file.py
@@ -260,8 +260,12 @@
 class AppTestNonblocking(object):
     def setup_class(cls):
         from pypy.module._file.interp_file import W_File
-        
+
         cls.old_read = os.read
+
+        if option.runappdirect:
+            py.test.skip("works with internals of _file impl on py.py")
+
         state = [0]
         def read(fd, n=None):
             if fd != 42:
@@ -281,7 +285,7 @@
 
     def teardown_class(cls):
         os.read = cls.old_read
-        
+
     def test_nonblocking_file(self):
         res = self.stream.read()
         assert res == 'xyz'
@@ -403,24 +407,24 @@
         with self.file(self.temppath, 'w') as f:
             f.write('foo')
         assert f.closed
-        
+
         with self.file(self.temppath, 'r') as f:
             s = f.readline()
 
         assert s == "foo"
         assert f.closed
-    
+
     def test_subclass_with(self):
         file = self.file
         class C(file):
             def __init__(self, *args, **kwargs):
                 self.subclass_closed = False
                 file.__init__(self, *args, **kwargs)
-            
+
             def close(self):
                 self.subclass_closed = True
                 file.close(self)
-        
+
         with C(self.temppath, 'w') as f:
             pass
         assert f.subclass_closed
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -161,9 +161,6 @@
     @binop
     def mul(self, v1, v2):
         return v1 * v2
-    @binop
-    def div(self, v1, v2):
-        return v1 / v2
 
     @unaryop
     def pos(self, v):
@@ -217,6 +214,14 @@
         return float2string(self.for_computation(self.unbox(item)), 'g', rfloat.DTSF_STR_PRECISION)
 
     @binop
+    def div(self, v1, v2):
+        try:
+            return v1 / v2
+        except ZeroDivisionError:
+            if v1 == v2 == 0.0:
+                return rfloat.NAN
+            return rfloat.copysign(rfloat.INFINITY, v1 * v2)
+    @binop
     def mod(self, v1, v2):
         return math.fmod(v1, v2)
     @binop
@@ -295,6 +300,11 @@
         return str(widen(self.unbox(item)))
 
     @binop
+    def div(self, v1, v2):
+        if v2 == 0:
+            return 0
+        return v1 / v2
+    @binop
     def mod(self, v1, v2):
         return v1 % v2
 
@@ -426,23 +436,22 @@
     pass
 
 if LONG_BIT == 32:
-    class W_LongDtype(W_Int32Dtype):
-        pass
+    long_dtype = W_Int32Dtype
+    ulong_dtype = W_UInt32Dtype
+elif LONG_BIT == 64:
+    long_dtype = W_Int64Dtype
+    ulong_dtype = W_UInt64Dtype
+else:
+    assert False
 
-    class W_ULongDtype(W_UInt32Dtype):
-        pass
-else:
-    class W_LongDtype(W_Int64Dtype):
-        pass
+class W_LongDtype(long_dtype):
+    num = 7
+    aliases = ["l"]
+    applevel_types = ["int"]
 
-    class W_ULongDtype(W_UInt64Dtype):
-        pass
-
-W_LongDtype.num = 7
-W_LongDtype.aliases = ["l"]
-W_LongDtype.applevel_types = ["int"]
-W_ULongDtype.num = 8
-W_ULongDtype.aliases = ["L"]
+class W_ULongDtype(ulong_dtype):
+    num = 8
+    aliases = ["L"]
 
 W_Float32Dtype = create_low_level_dtype(
     num = 11, kind = FLOATINGLTR, name = "float32",
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -285,7 +285,9 @@
             assert b[i] == i * 5
 
     def test_div(self):
-        from numpy import array, dtype
+        from math import isnan
+        from numpy import array, dtype, inf
+
         a = array(range(1, 6))
         b = a / a
         for i in range(5):
@@ -297,6 +299,24 @@
         for i in range(5):
             assert b[i] == 1
 
+        a = array([-1, 0, 1])
+        b = array([0, 0, 0])
+        c = a / b
+        assert (c == [0, 0, 0]).all()
+
+        a = array([-1.0, 0.0, 1.0])
+        b = array([0.0, 0.0, 0.0])
+        c = a / b
+        assert c[0] == -inf
+        assert isnan(c[1])
+        assert c[2] == inf
+
+        b = array([-0.0, -0.0, -0.0])
+        c = a / b
+        assert c[0] == inf
+        assert isnan(c[1])
+        assert c[2] == -inf
+
     def test_div_other(self):
         from numpy import array
         a = array(range(5))
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -420,12 +420,15 @@
             import sys
             if not hasattr(os, "fork"):
                 skip("Need fork() to test execv()")
+            try:
+                output = u"caf\xe9 \u1234\n".encode(sys.getfilesystemencoding())
+            except UnicodeEncodeError:
+                skip("encoding not good enough")
             pid = os.fork()
             if pid == 0:
                 os.execv(u"/bin/sh", ["sh", "-c",
                                       u"echo caf\xe9 \u1234 > onefile"])
             os.waitpid(pid, 0)
-            output = u"caf\xe9 \u1234\n".encode(sys.getfilesystemencoding())
             assert open("onefile").read() == output
             os.unlink("onefile")
 
@@ -445,13 +448,16 @@
             import sys
             if not hasattr(os, "fork"):
                 skip("Need fork() to test execve()")
+            try:
+                output = u"caf\xe9 \u1234\n".encode(sys.getfilesystemencoding())
+            except UnicodeEncodeError:
+                skip("encoding not good enough")
             pid = os.fork()
             if pid == 0:
                 os.execve(u"/bin/sh", ["sh", "-c",
                                       u"echo caf\xe9 \u1234 > onefile"],
                           {'ddd': 'xxx'})
             os.waitpid(pid, 0)
-            output = u"caf\xe9 \u1234\n".encode(sys.getfilesystemencoding())
             assert open("onefile").read() == output
             os.unlink("onefile")
         pass # <- please, inspect.getsource(), don't crash
diff --git a/pypy/module/pyexpat/__init__.py b/pypy/module/pyexpat/__init__.py
--- a/pypy/module/pyexpat/__init__.py
+++ b/pypy/module/pyexpat/__init__.py
@@ -4,12 +4,8 @@
 
 class ErrorsModule(MixedModule):
     "Definition of pyexpat.errors module."
-
-    appleveldefs = {
-        }
-
-    interpleveldefs = {
-        }
+    appleveldefs = {}
+    interpleveldefs = {}
 
     def setup_after_space_initialization(self):
         from pypy.module.pyexpat import interp_pyexpat
@@ -18,6 +14,18 @@
                     interp_pyexpat.ErrorString(self.space,
                     getattr(interp_pyexpat, name)))
 
+class ModelModule(MixedModule):
+    "Definition of pyexpat.model module."
+    appleveldefs = {}
+    interpleveldefs = {}
+
+    def setup_after_space_initialization(self):
+        from pypy.module.pyexpat import interp_pyexpat
+        space = self.space
+        for name in interp_pyexpat.xml_model_list:
+            value = getattr(interp_pyexpat, name)
+            space.setattr(self, space.wrap(name), space.wrap(value))
+
 class Module(MixedModule):
     "Python wrapper for Expat parser."
 
@@ -39,6 +47,7 @@
 
     submodules = {
         'errors': ErrorsModule,
+        'model':  ModelModule,
     }
 
     for name in ['XML_PARAM_ENTITY_PARSING_NEVER',
diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py
--- a/pypy/module/pyexpat/interp_pyexpat.py
+++ b/pypy/module/pyexpat/interp_pyexpat.py
@@ -76,6 +76,18 @@
     "XML_ERROR_FINISHED",
     "XML_ERROR_SUSPEND_PE",
     ]
+xml_model_list = [
+    "XML_CTYPE_EMPTY",
+    "XML_CTYPE_ANY",
+    "XML_CTYPE_MIXED",
+    "XML_CTYPE_NAME",
+    "XML_CTYPE_CHOICE",
+    "XML_CTYPE_SEQ",
+    "XML_CQUANT_NONE",
+    "XML_CQUANT_OPT",
+    "XML_CQUANT_REP",
+    "XML_CQUANT_PLUS",
+    ]
 
 class CConfigure:
     _compilation_info_ = eci
@@ -104,6 +116,8 @@
 
     for name in xml_error_list:
         locals()[name] = rffi_platform.ConstantInteger(name)
+    for name in xml_model_list:
+        locals()[name] = rffi_platform.ConstantInteger(name)
 
 for k, v in rffi_platform.configure(CConfigure).items():
     globals()[k] = v
diff --git a/pypy/module/pyexpat/test/test_parser.py b/pypy/module/pyexpat/test/test_parser.py
--- a/pypy/module/pyexpat/test/test_parser.py
+++ b/pypy/module/pyexpat/test/test_parser.py
@@ -131,3 +131,7 @@
                'encoding specified in XML declaration is incorrect')
         assert (pyexpat.errors.XML_ERROR_XML_DECL ==
                 'XML declaration not well-formed')
+
+    def test_model(self):
+        import pyexpat
+        assert isinstance(pyexpat.model.XML_CTYPE_EMPTY, int)
diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py
--- a/pypy/module/pypyjit/interp_jit.py
+++ b/pypy/module/pypyjit/interp_jit.py
@@ -137,20 +137,15 @@
 
     def jump_absolute(self, jumpto, _, ec=None):
         if we_are_jitted():
-            # Normally, the tick counter is decremented by 100 for every
-            # Python opcode.  Here, to better support JIT compilation of
-            # small loops, we decrement it by a possibly smaller constant.
-            # We get the maximum 100 when the (unoptimized) trace length
-            # is at least 3200 (a bit randomly).
-            trace_length = r_uint(current_trace_length())
-            decr_by = trace_length // 32
-            if decr_by < 1:
-                decr_by = 1
-            elif decr_by > 100:    # also if current_trace_length() returned -1
-                decr_by = 100
+            #
+            # assume that only threads are using the bytecode counter
+            decr_by = 0
+            if self.space.actionflag.has_bytecode_counter:   # constant-folded
+                if self.space.threadlocals.gil_ready:   # quasi-immutable field
+                    decr_by = _get_adapted_tick_counter()
             #
             self.last_instr = intmask(jumpto)
-            ec.bytecode_trace(self, intmask(decr_by))
+            ec.bytecode_trace(self, decr_by)
             jumpto = r_uint(self.last_instr)
         #
         pypyjitdriver.can_enter_jit(frame=self, ec=ec, next_instr=jumpto,
@@ -158,6 +153,20 @@
                                     is_being_profiled=self.is_being_profiled)
         return jumpto
 
+def _get_adapted_tick_counter():
+    # Normally, the tick counter is decremented by 100 for every
+    # Python opcode.  Here, to better support JIT compilation of
+    # small loops, we decrement it by a possibly smaller constant.
+    # We get the maximum 100 when the (unoptimized) trace length
+    # is at least 3200 (a bit randomly).
+    trace_length = r_uint(current_trace_length())
+    decr_by = trace_length // 32
+    if decr_by < 1:
+        decr_by = 1
+    elif decr_by > 100:    # also if current_trace_length() returned -1
+        decr_by = 100
+    return intmask(decr_by)
+
 
 PyCode__initialize = PyCode._initialize
 
diff --git a/pypy/module/pypyjit/test_pypy_c/model.py b/pypy/module/pypyjit/test_pypy_c/model.py
--- a/pypy/module/pypyjit/test_pypy_c/model.py
+++ b/pypy/module/pypyjit/test_pypy_c/model.py
@@ -225,6 +225,8 @@
         # strip comment
         if '#' in line:
             line = line[:line.index('#')]
+        if line.strip() == 'guard_not_invalidated?':
+            return 'guard_not_invalidated', None, [], '...', False
         # find the resvar, if any
         if ' = ' in line:
             resvar, _, line = line.partition(' = ')
@@ -249,7 +251,7 @@
             descr = descr[len('descr='):]
         else:
             descr = None
-        return opname, resvar, args, descr
+        return opname, resvar, args, descr, True
 
     @classmethod
     def preprocess_expected_src(cls, src):
@@ -258,13 +260,23 @@
         # replaced with the corresponding operations, so that tests don't have
         # to repeat it every time
         ticker_check = """
+            guard_not_invalidated?
+            ticker0 = getfield_raw(ticker_address, descr=<SignedFieldDescr pypysig_long_struct.c_value .*>)
+            ticker_cond0 = int_lt(ticker0, 0)
+            guard_false(ticker_cond0, descr=...)
+        """
+        src = src.replace('--TICK--', ticker_check)
+        #
+        # this is the ticker check generated if we have threads
+        thread_ticker_check = """
+            guard_not_invalidated?
             ticker0 = getfield_raw(ticker_address, descr=<SignedFieldDescr pypysig_long_struct.c_value .*>)
             ticker1 = int_sub(ticker0, 1)
             setfield_raw(ticker_address, ticker1, descr=<SignedFieldDescr pypysig_long_struct.c_value .*>)
             ticker_cond0 = int_lt(ticker1, 0)
             guard_false(ticker_cond0, descr=...)
         """
-        src = src.replace('--TICK--', ticker_check)
+        src = src.replace('--THREAD-TICK--', thread_ticker_check)
         #
         # this is the ticker check generated in PyFrame.handle_operation_error
         exc_ticker_check = """
@@ -298,7 +310,7 @@
         if not cond:
             raise InvalidMatch(message, frame=sys._getframe(1))
 
-    def match_op(self, op, (exp_opname, exp_res, exp_args, exp_descr)):
+    def match_op(self, op, (exp_opname, exp_res, exp_args, exp_descr, _)):
         self._assert(op.name == exp_opname, "operation mismatch")
         self.match_var(op.res, exp_res)
         if exp_args != ['...']:
@@ -341,7 +353,7 @@
         what is after the '...'
         """
         iter_exp_ops = iter(expected_ops)
-        iter_ops = iter(self.ops)
+        iter_ops = RevertableIterator(self.ops)
         for opindex, exp_op in enumerate(iter_exp_ops):
             try:
                 if exp_op == '...':
@@ -360,6 +372,9 @@
                             break
                 self.match_op(op, exp_op)
             except InvalidMatch, e:
+                if exp_op[4] is False:    # optional operation
+                    iter_ops.revert_one()
+                    continue       # try to match with the next exp_op
                 e.opindex = opindex
                 raise
         #
@@ -372,8 +387,8 @@
                 return ''
             text = str(py.code.Source(src).deindent().indent())
             lines = text.splitlines(True)
-            if opindex is not None and 0 <= opindex < len(lines):
-                lines[opindex] = lines[opindex].rstrip() + '\t<=====\n'
+            if opindex is not None and 0 <= opindex <= len(lines):
+                lines.insert(opindex, '\n\t===== HERE =====\n')
             return ''.join(lines)
         #
         expected_src = self.preprocess_expected_src(expected_src)
@@ -398,3 +413,18 @@
         else:
             return True
 
+
+class RevertableIterator(object):
+    def __init__(self, sequence):
+        self.sequence = sequence
+        self.index = 0
+    def __iter__(self):
+        return self
+    def next(self):
+        index = self.index
+        if index == len(self.sequence):
+            raise StopIteration
+        self.index = index + 1
+        return self.sequence[index]
+    def revert_one(self):
+        self.index -= 1
diff --git a/pypy/module/pypyjit/test_pypy_c/test_00_model.py b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
--- a/pypy/module/pypyjit/test_pypy_c/test_00_model.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
@@ -145,15 +145,17 @@
 
     def test_parse_op(self):
         res = OpMatcher.parse_op("  a =   int_add(  b,  3 ) # foo")
-        assert res == ("int_add", "a", ["b", "3"], None)
+        assert res == ("int_add", "a", ["b", "3"], None, True)
         res = OpMatcher.parse_op("guard_true(a)")
-        assert res == ("guard_true", None, ["a"], None)
+        assert res == ("guard_true", None, ["a"], None, True)
         res = OpMatcher.parse_op("setfield_gc(p0, i0, descr=<foobar>)")
-        assert res == ("setfield_gc", None, ["p0", "i0"], "<foobar>")
+        assert res == ("setfield_gc", None, ["p0", "i0"], "<foobar>", True)
         res = OpMatcher.parse_op("i1 = getfield_gc(p0, descr=<foobar>)")
-        assert res == ("getfield_gc", "i1", ["p0"], "<foobar>")
+        assert res == ("getfield_gc", "i1", ["p0"], "<foobar>", True)
         res = OpMatcher.parse_op("p0 = force_token()")
-        assert res == ("force_token", "p0", [], None)
+        assert res == ("force_token", "p0", [], None, True)
+        res = OpMatcher.parse_op("guard_not_invalidated?")
+        assert res == ("guard_not_invalidated", None, [], '...', False)
 
     def test_exact_match(self):
         loop = """
@@ -341,7 +343,7 @@
             # this is the actual loop
             'int_lt', 'guard_true', 'int_add',
             # this is the signal checking stuff
-            'getfield_raw', 'int_sub', 'setfield_raw', 'int_lt', 'guard_false',
+            'guard_not_invalidated', 'getfield_raw', 'int_lt', 'guard_false',
             'jump'
             ]
 
@@ -407,7 +409,7 @@
             # this is the actual loop
             'int_lt', 'guard_true', 'force_token', 'int_add',
             # this is the signal checking stuff
-            'getfield_raw', 'int_sub', 'setfield_raw', 'int_lt', 'guard_false',
+            'guard_not_invalidated', 'getfield_raw', 'int_lt', 'guard_false',
             'jump'
             ]
 
@@ -425,10 +427,9 @@
             guard_true(i6, descr=...)
             i8 = int_add(i4, 1)
             # signal checking stuff
+            guard_not_invalidated(descr=...)
             i10 = getfield_raw(37212896, descr=<.* pypysig_long_struct.c_value .*>)
-            i12 = int_sub(i10, 1)
-            setfield_raw(37212896, i12, descr=<.* pypysig_long_struct.c_value .*>)
-            i14 = int_lt(i12, 0)
+            i14 = int_lt(i10, 0)
             guard_false(i14, descr=...)
             jump(p0, p1, p2, p3, i8, descr=...)
         """)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_containers.py b/pypy/module/pypyjit/test_pypy_c/test_containers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_containers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_containers.py
@@ -46,7 +46,7 @@
         assert loop.match_by_id("getitem", """
             i28 = call(ConstClass(ll_dict_lookup__dicttablePtr_objectPtr_Signed), p18, p6, i25, descr=...)
             ...
-            p33 = call(ConstClass(ll_get_value__dicttablePtr_Signed), p18, i28, descr=...)
+            p33 = getinteriorfield_gc(p31, i26, descr=<InteriorFieldDescr <GcPtrFieldDescr dictentry.value .*>>)
             ...
         """)
 
diff --git a/pypy/module/pypyjit/test_pypy_c/test_misc.py b/pypy/module/pypyjit/test_pypy_c/test_misc.py
--- a/pypy/module/pypyjit/test_pypy_c/test_misc.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_misc.py
@@ -329,4 +329,20 @@
             guard_false(i28, descr=...)
             i30 = int_lshift(i20, 24)
             i31 = int_or(i26, i30)
-        """ % {"32_bit_only": extra})
\ No newline at end of file
+        """ % {"32_bit_only": extra})
+
+    def test_eval(self):
+        def main():
+            i = 1
+            a = compile('x+x+x+x+x+x', 'eval', 'eval')
+            b = {'x': 7}
+            while i < 1000:
+                y = eval(a,b,b)  # ID: eval
+                i += 1
+            return y
+
+        log = self.run(main)
+        assert log.result == 42
+        # the following assertion fails if the loop was cancelled due
+        # to "abort: vable escape"
+        assert len(log.loops_by_id("eval")) == 1
diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -41,7 +41,7 @@
             guard_true(i32, descr=...)
             i34 = int_add(i6, 1)
             --TICK--
-            jump(p0, p1, p2, p3, p4, p5, i34, p7, p8, i9, i10, p11, i12, p13, descr=<Loop4>)
+            jump(p0, p1, p2, p3, p4, p5, i34, p7, p8, i9, i10, p11, i12, p13, descr=...)
         """)
 
     def test_long(self):
@@ -93,7 +93,8 @@
             i46 = call(ConstClass(ll_startswith__rpy_stringPtr_rpy_stringPtr), p28, ConstPtr(ptr45), descr=<BoolCallDescr>)
             guard_false(i46, descr=...)
             p51 = new_with_vtable(21136408)
-            setfield_gc(p51, _, descr=...)    # 6 setfields, but the order is dict-order-dependent
+            setfield_gc(p51, _, descr=...)    # 7 setfields, but the order is dict-order-dependent
+            setfield_gc(p51, _, descr=...)
             setfield_gc(p51, _, descr=...)
             setfield_gc(p51, _, descr=...)
             setfield_gc(p51, _, descr=...)
@@ -106,7 +107,7 @@
             i58 = int_add_ovf(i6, i57)
             guard_no_overflow(descr=...)
             --TICK--
-            jump(p0, p1, p2, p3, p4, p5, i58, i7, descr=<Loop4>)
+            jump(p0, p1, p2, p3, p4, p5, i58, i7, descr=...)
         """)
 
     def test_str_mod(self):
@@ -156,4 +157,41 @@
             i40 = int_sub(i4, 1)
             --TICK--
             jump(p0, p1, p2, p3, i40, i38, descr=<Loop0>)
-        """)
\ No newline at end of file
+        """)
+
+    def test_getattr_promote(self):
+        def main(n):
+            class A(object):
+                def meth_a(self):
+                    return 1
+                def meth_b(self):
+                    return 2
+            a = A()
+
+            l = ['a', 'b']
+            s = 0
+            for i in range(n):
+                name = 'meth_' + l[i & 1]
+                meth = getattr(a, name) # ID: getattr
+                s += meth()
+            return s
+
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loops = log.loops_by_filename(self.filepath)
+        assert len(loops) == 2
+        for loop in loops:
+            loop.match_by_id('getattr','''
+            guard_not_invalidated(descr=...)
+            i32 = strlen(p31)
+            i34 = int_add(5, i32)
+            p35 = newstr(i34)
+            strsetitem(p35, 0, 109)
+            strsetitem(p35, 1, 101)
+            strsetitem(p35, 2, 116)
+            strsetitem(p35, 3, 104)
+            strsetitem(p35, 4, 95)
+            copystrcontent(p31, p35, 0, 5, i32)
+            i49 = call(ConstClass(_ll_2_str_eq_nonnull__rpy_stringPtr_rpy_stringPtr), p35, ConstPtr(ptr48), descr=<SignedCallDescr>)
+            guard_value(i49, 1, descr=<Guard8>)
+            ''')
diff --git a/pypy/module/pypyjit/test_pypy_c/test_thread.py b/pypy/module/pypyjit/test_pypy_c/test_thread.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/pypyjit/test_pypy_c/test_thread.py
@@ -0,0 +1,28 @@
+from pypy.module.pypyjit.test_pypy_c.test_00_model import BaseTestPyPyC
+
+
+class TestThread(BaseTestPyPyC):
+    def test_simple(self):
+        def main(n):
+            import thread
+            def f():
+                i = 0
+                while i < n:
+                    i += 1
+                done.release()
+
+            done = thread.allocate_lock()
+            done.acquire()
+            thread.start_new_thread(f, ())
+            done.acquire()
+            return 0
+        log = self.run(main, [500])
+        assert round(log.result, 6) == round(main(500), 6)
+        loop, = log.loops_by_filename(self.filepath)
+        assert loop.match("""
+            i2 = int_lt(i0, i1)
+            guard_true(i2, descr=...)
+            i3 = int_add(i0, 1)
+            --THREAD-TICK--
+            jump(..., descr=<Loop0>)
+        """)
diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py
--- a/pypy/module/signal/interp_signal.py
+++ b/pypy/module/signal/interp_signal.py
@@ -109,8 +109,11 @@
         p = pypysig_getaddr_occurred()
         value = p.c_value
         if self.has_bytecode_counter:    # this 'if' is constant-folded
-            value -= by
-            p.c_value = value
+            if jit.isconstant(by) and by == 0:
+                pass     # normally constant-folded too
+            else:
+                value -= by
+                p.c_value = value
         return value
 
 
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -47,7 +47,7 @@
         'pypy_initial_path'     : 'state.pypy_initial_path',
 
         '_getframe'             : 'vm._getframe', 
-        '_current_frames'       : 'vm._current_frames', 
+        '_current_frames'       : 'currentframes._current_frames', 
         'setrecursionlimit'     : 'vm.setrecursionlimit', 
         'getrecursionlimit'     : 'vm.getrecursionlimit', 
         'setcheckinterval'      : 'vm.setcheckinterval', 
diff --git a/pypy/module/sys/currentframes.py b/pypy/module/sys/currentframes.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/sys/currentframes.py
@@ -0,0 +1,78 @@
+"""
+Implementation of the 'sys._current_frames()' routine.
+"""
+from pypy.interpreter import gateway
+
+app = gateway.applevel('''
+"NOT_RPYTHON"
+import __builtin__
+
+class fake_code(object):
+    co_name = "?"
+    co_filename = "?"
+    co_firstlineno = 0
+
+class fake_frame(object):
+    f_back = None
+    f_builtins = __builtin__.__dict__
+    f_code = fake_code()
+    f_exc_traceback = None
+    f_exc_type = None
+    f_exc_value = None
+    f_globals = {}
+    f_lasti = -1
+    f_lineno = 0
+    f_locals = {}
+    f_restricted = False
+    f_trace = None
+
+    def __init__(self, f):
+        if f is not None:
+            for name in ["f_builtins", "f_code", "f_globals", "f_lasti",
+                         "f_lineno"]:
+                setattr(self, name, getattr(f, name))
+''')
+
+def _current_frames(space):
+    """_current_frames() -> dictionary
+
+    Return a dictionary mapping each current thread T's thread id to T's
+    current stack "frame".  Functions in the traceback module can build the
+    call stack given such a frame.
+
+    Note that in PyPy this returns fake frame objects, to avoid a runtime
+    penalty everywhere with the JIT.  (So far these fake frames can be
+    completely uninformative depending on the JIT state; we could return
+    more with more efforts.)
+
+    This function should be used for specialized purposes only."""
+    w_result = space.newdict()
+    w_fake_frame = app.wget(space, "fake_frame")
+    w_fake_code  = app.wget(space, "fake_code")
+    ecs = space.threadlocals.getallvalues()
+    for thread_ident, ec in ecs.items():
+        vref = ec.topframeref
+        frames = []
+        while not vref.virtual:
+            f = vref()
+            if f is None:
+                break
+            frames.append(f)
+            vref = f.f_backref
+        else:
+            frames.append(None)
+        #
+        w_topframe = space.wrap(None)
+        w_prevframe = None
+        for f in frames:
+            w_nextframe = space.call_function(w_fake_frame, space.wrap(f))
+            if w_prevframe is None:
+                w_topframe = w_nextframe
+            else:
+                space.setattr(w_prevframe, space.wrap('f_back'), w_nextframe)
+            w_prevframe = w_nextframe
+        #
+        space.setitem(w_result,
+                      space.wrap(thread_ident),
+                      w_topframe)
+    return w_result
diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -556,7 +556,7 @@
             return sys._current_frames()
         frames = f()
         assert frames.keys() == [0]
-        assert frames[0].f_code.co_name == 'f'
+        assert frames[0].f_code.co_name in ('f', '?')
 
 class AppTestCurrentFramesWithThread(AppTestCurrentFrames):
     def setup_class(cls):
@@ -568,23 +568,25 @@
         import thread
 
         thread_id = thread.get_ident()
-        self.ready = False
         def other_thread():
-            self.ready = True
             print "thread started"
-            time.sleep(5)
+            lock2.release()
+            lock1.acquire()
+        lock1 = thread.allocate_lock()
+        lock2 = thread.allocate_lock()
+        lock1.acquire()
+        lock2.acquire()
         thread.start_new_thread(other_thread, ())
 
         def f():
-            for i in range(100):
-                if self.ready: break
-                time.sleep(0.1)
+            lock2.acquire()
             return sys._current_frames()
 
         frames = f()
+        lock1.release()
         thisframe = frames.pop(thread_id)
-        assert thisframe.f_code.co_name == 'f'
+        assert thisframe.f_code.co_name in ('f', '?')
 
         assert len(frames) == 1
         _, other_frame = frames.popitem()
-        assert other_frame.f_code.co_name == 'other_thread'
+        assert other_frame.f_code.co_name in ('other_thread', '?')
diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -45,25 +45,6 @@
     f.mark_as_escaped()
     return space.wrap(f)
 
-def _current_frames(space):
-    """_current_frames() -> dictionary
-
-    Return a dictionary mapping each current thread T's thread id to T's
-    current stack frame.
-
-    This function should be used for specialized purposes only."""
-    raise OperationError(space.w_NotImplementedError,
-        space.wrap("XXX sys._current_frames() incompatible with the JIT"))
-    w_result = space.newdict()
-    ecs = space.threadlocals.getallvalues()
-    for thread_ident, ec in ecs.items():
-        f = ec.gettopframe_nohidden()
-        f.mark_as_escaped()
-        space.setitem(w_result,
-                      space.wrap(thread_ident),
-                      space.wrap(f))
-    return w_result
-
 def setrecursionlimit(space, w_new_limit):
     """setrecursionlimit() sets the maximum number of nested calls that
 can occur before a RuntimeError is raised.  On PyPy the limit is
diff --git a/pypy/module/thread/gil.py b/pypy/module/thread/gil.py
--- a/pypy/module/thread/gil.py
+++ b/pypy/module/thread/gil.py
@@ -17,6 +17,7 @@
 class GILThreadLocals(OSThreadLocals):
     """A version of OSThreadLocals that enforces a GIL."""
     gil_ready = False
+    _immutable_fields_ = ['gil_ready?']
 
     def initialize(self, space):
         # add the GIL-releasing callback as an action on the space
diff --git a/pypy/module/thread/threadlocals.py b/pypy/module/thread/threadlocals.py
--- a/pypy/module/thread/threadlocals.py
+++ b/pypy/module/thread/threadlocals.py
@@ -8,9 +8,14 @@
 
     def __init__(self):
         self._valuedict = {}   # {thread_ident: ExecutionContext()}
+        self._freeze_()
+
+    def _freeze_(self):
+        self._valuedict.clear()
         self._mainthreadident = 0
         self._mostrecentkey = 0        # fast minicaching for the common case
         self._mostrecentvalue = None   # fast minicaching for the common case
+        return False
 
     def getvalue(self):
         ident = thread.get_ident()
diff --git a/pypy/objspace/std/newformat.py b/pypy/objspace/std/newformat.py
--- a/pypy/objspace/std/newformat.py
+++ b/pypy/objspace/std/newformat.py
@@ -120,6 +120,8 @@
             out.append_slice(s, last_literal, end)
             return out.build()
 
+        # This is only ever called if we're already unrolling _do_build_string
+        @jit.unroll_safe
         def _parse_field(self, start, end):
             s = self.template
             # Find ":" or "!"
@@ -149,6 +151,7 @@
                 i += 1
             return s[start:end], None, end
 
+        @jit.unroll_safe
         def _get_argument(self, name):
             # First, find the argument.
             space = self.space
@@ -207,6 +210,7 @@
                     raise OperationError(space.w_IndexError, w_msg)
             return self._resolve_lookups(w_arg, name, i, end)
 
+        @jit.unroll_safe
         def _resolve_lookups(self, w_obj, name, start, end):
             # Resolve attribute and item lookups.
             space = self.space
diff --git a/pypy/objspace/std/objecttype.py b/pypy/objspace/std/objecttype.py
--- a/pypy/objspace/std/objecttype.py
+++ b/pypy/objspace/std/objecttype.py
@@ -44,7 +44,6 @@
         raise OperationError(space.w_TypeError,
                              space.wrap("__class__ assignment: only for heap types"))
     w_oldcls = space.type(w_obj)
-    # XXX taint space should raise a TaintError here if w_oldcls is tainted
     assert isinstance(w_oldcls, W_TypeObject)
     if w_oldcls.get_full_instance_layout() == w_newcls.get_full_instance_layout():
         w_obj.setclass(space, w_newcls)
diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py
--- a/pypy/objspace/std/stringobject.py
+++ b/pypy/objspace/std/stringobject.py
@@ -161,57 +161,59 @@
 
 def str_swapcase__String(space, w_self):
     self = w_self._value
-    res = [' '] * len(self)
+    builder = StringBuilder(len(self))
     for i in range(len(self)):
         ch = self[i]
         if ch.isupper():
             o = ord(ch) + 32
-            res[i] = chr(o)
+            builder.append(chr(o))
         elif ch.islower():
             o = ord(ch) - 32
-            res[i] = chr(o)
+            builder.append(chr(o))
         else:
-            res[i] = ch
+            builder.append(ch)
 
-    return space.wrap("".join(res))
+    return space.wrap(builder.build())
 
 
 def str_capitalize__String(space, w_self):
     input = w_self._value
-    buffer = [' '] * len(input)
+    builder = StringBuilder(len(input))
     if len(input) > 0:
         ch = input[0]
         if ch.islower():
             o = ord(ch) - 32
-            buffer[0] = chr(o)
+            builder.append(chr(o))
         else:
-            buffer[0] = ch
+            builder.append(ch)
 
         for i in range(1, len(input)):
             ch = input[i]
             if ch.isupper():
                 o = ord(ch) + 32
-                buffer[i] = chr(o)
+                builder.append(chr(o))
             else:
-                buffer[i] = ch
+                builder.append(ch)
 
-    return space.wrap("".join(buffer))
+    return space.wrap(builder.build())
 
 def str_title__String(space, w_self):
     input = w_self._value
-    buffer = [' '] * len(input)
+    builder = StringBuilder(len(input))
     prev_letter=' '
 
-    for pos in range(0, len(input)):
+    for pos in range(len(input)):
         ch = input[pos]
         if not prev_letter.isalpha():
-            buffer[pos] = _upper(ch)
+            ch = _upper(ch)
+            builder.append(ch)
         else:
-            buffer[pos] = _lower(ch)
+            ch = _lower(ch)
+            builder.append(ch)
 
-        prev_letter = buffer[pos]
+        prev_letter = ch
 
-    return space.wrap("".join(buffer))
+    return space.wrap(builder.build())
 
 def str_split__String_None_ANY(space, w_self, w_none, w_maxsplit=-1):
     maxsplit = space.int_w(w_maxsplit)
@@ -754,27 +756,21 @@
     input = w_self._value
     width = space.int_w(w_width)
 
-    if len(input) >= width:
+    num_zeros = width - len(input)
+    if num_zeros <= 0:
         # cannot return w_self, in case it is a subclass of str
         return space.wrap(input)
 
-    buf = [' '] * width
+    builder = StringBuilder(width)
     if len(input) > 0 and (input[0] == '+' or input[0] == '-'):
-        buf[0] = input[0]
+        builder.append(input[0])
         start = 1
-        middle = width - len(input) + 1
     else:
         start = 0
-        middle = width - len(input)
 
-    for i in range(start, middle):
-        buf[i] = '0'
-
-    for i in range(middle, width):
-        buf[i] = input[start]
-        start = start + 1
-
-    return space.wrap("".join(buf))
+    builder.append_multiple_char('0', num_zeros)
+    builder.append_slice(input, start, len(input))
+    return space.wrap(builder.build())
 
 
 def hash__String(space, w_str):
diff --git a/pypy/objspace/std/strutil.py b/pypy/objspace/std/strutil.py
--- a/pypy/objspace/std/strutil.py
+++ b/pypy/objspace/std/strutil.py
@@ -35,7 +35,7 @@
 
     def error(self):
         raise ParseStringError("invalid literal for %s() with base %d: '%s'" %
-                               (self.fname, self.base, self.literal))
+                               (self.fname, self.original_base, self.literal))
 
     def __init__(self, s, literal, base, fname):
         self.literal = literal
@@ -47,7 +47,8 @@
         elif s.startswith('+'):
             s = strip_spaces(s[1:])
         self.sign = sign
-        
+        self.original_base = base
+
         if base == 0:
             if s.startswith('0x') or s.startswith('0X'):
                 base = 16
diff --git a/pypy/objspace/std/test/test_strutil.py b/pypy/objspace/std/test/test_strutil.py
--- a/pypy/objspace/std/test/test_strutil.py
+++ b/pypy/objspace/std/test/test_strutil.py
@@ -89,6 +89,8 @@
 
         exc = raises(ParseStringError, string_to_int, '')
         assert exc.value.msg == "invalid literal for int() with base 10: ''"
+        exc = raises(ParseStringError, string_to_int, '', 0)
+        assert exc.value.msg == "invalid literal for int() with base 0: ''"
 
     def test_string_to_int_overflow(self):
         import sys
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -10,7 +10,8 @@
 from pypy.objspace.std import identitydict
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.rlib.objectmodel import current_object_addr_as_int, compute_hash
-from pypy.rlib.jit import promote, elidable_promote, we_are_jitted
+from pypy.rlib.jit import promote, elidable_promote, we_are_jitted,\
+     promote_string
 from pypy.rlib.jit import elidable, dont_look_inside, unroll_safe
 from pypy.rlib.rarithmetic import intmask, r_uint
 
@@ -101,6 +102,7 @@
                           'instancetypedef',
                           'terminator',
                           '_version_tag?',
+                          'interplevel_cls',
                           ]
 
     # for config.objspace.std.getattributeshortcut
@@ -399,6 +401,7 @@
         if version_tag is None:
             tup = w_self._lookup_where(name)
             return tup
+        name = promote_string(name)
         w_class, w_value = w_self._pure_lookup_where_with_method_cache(name, version_tag)
         return w_class, unwrap_cell(space, w_value)
 
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -417,54 +417,54 @@
     input = w_self._value
     if len(input) == 0:
         return W_UnicodeObject.EMPTY
-    result = [u'\0'] * len(input)
-    result[0] = unichr(unicodedb.toupper(ord(input[0])))
+    builder = UnicodeBuilder(len(input))
+    builder.append(unichr(unicodedb.toupper(ord(input[0]))))
     for i in range(1, len(input)):
-        result[i] = unichr(unicodedb.tolower(ord(input[i])))
-    return W_UnicodeObject(u''.join(result))
+        builder.append(unichr(unicodedb.tolower(ord(input[i]))))
+    return W_UnicodeObject(builder.build())
 
 def unicode_title__Unicode(space, w_self):
     input = w_self._value
     if len(input) == 0:
         return w_self
-    result = [u'\0'] * len(input)
+    builder = UnicodeBuilder(len(input))
 
     previous_is_cased = False
     for i in range(len(input)):
         unichar = ord(input[i])
         if previous_is_cased:
-            result[i] = unichr(unicodedb.tolower(unichar))
+            builder.append(unichr(unicodedb.tolower(unichar)))
         else:
-            result[i] = unichr(unicodedb.totitle(unichar))
+            builder.append(unichr(unicodedb.totitle(unichar)))
         previous_is_cased = unicodedb.iscased(unichar)
-    return W_UnicodeObject(u''.join(result))
+    return W_UnicodeObject(builder.build())
 
 def unicode_lower__Unicode(space, w_self):
     input = w_self._value
-    result = [u'\0'] * len(input)
+    builder = UnicodeBuilder(len(input))
     for i in range(len(input)):
-        result[i] = unichr(unicodedb.tolower(ord(input[i])))
-    return W_UnicodeObject(u''.join(result))
+        builder.append(unichr(unicodedb.tolower(ord(input[i]))))
+    return W_UnicodeObject(builder.build())
 
 def unicode_upper__Unicode(space, w_self):
     input = w_self._value
-    result = [u'\0'] * len(input)
+    builder = UnicodeBuilder(len(input))
     for i in range(len(input)):
-        result[i] = unichr(unicodedb.toupper(ord(input[i])))
-    return W_UnicodeObject(u''.join(result))
+        builder.append(unichr(unicodedb.toupper(ord(input[i]))))
+    return W_UnicodeObject(builder.build())
 
 def unicode_swapcase__Unicode(space, w_self):
     input = w_self._value
-    result = [u'\0'] * len(input)
+    builder = UnicodeBuilder(len(input))
     for i in range(len(input)):
         unichar = ord(input[i])
         if unicodedb.islower(unichar):
-            result[i] = unichr(unicodedb.toupper(unichar))
+            builder.append(unichr(unicodedb.toupper(unichar)))
         elif unicodedb.isupper(unichar):
-            result[i] = unichr(unicodedb.tolower(unichar))
+            builder.append(unichr(unicodedb.tolower(unichar)))
         else:
-            result[i] = input[i]
-    return W_UnicodeObject(u''.join(result))
+            builder.append(input[i])
+    return W_UnicodeObject(builder.build())
 
 def _normalize_index(length, index):
     if index < 0:
diff --git a/pypy/objspace/taint.py b/pypy/objspace/taint.py
deleted file mode 100644
--- a/pypy/objspace/taint.py
+++ /dev/null
@@ -1,294 +0,0 @@
-"""
-Just an experiment.
-"""
-import os
-from pypy.objspace.std.objspace import StdObjSpace
-from pypy.objspace.proxy import patch_space_in_place
-from pypy.objspace.thunk import nb_forcing_args
-from pypy.interpreter.error import OperationError
-from pypy.interpreter import baseobjspace, gateway, executioncontext
-from pypy.interpreter.function import Method
-from pypy.interpreter.pyframe import PyFrame
-from pypy.tool.sourcetools import func_with_new_name
-from pypy.rlib.unroll import unrolling_iterable
-
-
-class W_Tainted(baseobjspace.W_Root):
-    def __init__(self, w_obj):
-        self.w_obj = w_obj
-
-##    def getdict(self, space):
-##        return taint(self.w_obj.getdict(space))
-
-##    def getdictvalue(self, space, attr):
-##        return taint(self.w_obj.getdictvalue(space, attr))
-
-##    def setdictvalue(self, space, attr, w_value):
-##        return self.w_obj.setdictvalue(space, attr, w_value)
-
-##    ...
-
-class W_TaintBomb(baseobjspace.W_Root):
-    filename = '?'
-    codename = '?'
-    codeline = 0
-
-    def __init__(self, space, operr):
-        self.space = space
-        self.operr = operr
-        self.record_debug_info()
-
-    def record_debug_info(self):
-        ec = self.space.getexecutioncontext()
-        frame = ec.gettopframe_nohidden()
-        if isinstance(frame, PyFrame):     # and, in particular, frame != None
-            self.filename = frame.pycode.co_filename
-            self.codename = frame.pycode.co_name
-            self.codeline = frame.get_last_lineno()
-        if get_debug_level(self.space) > 0:
-            self.debug_dump()
-
-    def debug_dump(self):
-        os.write(2, 'Taint Bomb from file "%s", line %d, in %s\n    %s\n' % (
-            self.filename, self.codeline, self.codename,
-            self.operr.errorstr(self.space)))
-
-    def explode(self):
-        #msg = self.operr.errorstr(space)
-        raise OperationError(self.space.w_TaintError, self.space.w_None)
-
-
-def taint(w_obj):
-    """Return a tainted version of the argument."""
-    if w_obj is None or isinstance(w_obj, W_Tainted):
-        return w_obj
-    else:
-        return W_Tainted(w_obj)
-app_taint = gateway.interp2app(taint)
-
-def is_tainted(space, w_obj):
-    """Return whether the argument is tainted."""
-    res = isinstance(w_obj, W_Tainted) or isinstance(w_obj, W_TaintBomb)
-    return space.wrap(res)
-app_is_tainted = gateway.interp2app(is_tainted)
-
-def untaint(space, w_expectedtype, w_obj):
-    """untaint(expectedtype, tainted_obj) -> obj
-Untaint untainted_obj and return it. If the result is not of expectedtype,
-raise a type error."""
-    if (isinstance(w_expectedtype, W_Tainted) or
-        isinstance(w_expectedtype, W_TaintBomb)):
-        raise OperationError(space.w_TypeError,
-                  space.wrap("untaint() arg 1 must be an untainted type"))
-    if not space.is_true(space.isinstance(w_expectedtype, space.w_type)):
-        raise OperationError(space.w_TypeError,
-                             space.wrap("untaint() arg 1 must be a type"))
-    if isinstance(w_obj, W_Tainted):
-        w_obj = w_obj.w_obj
-    elif isinstance(w_obj, W_TaintBomb):
-        w_obj.explode()
-    #if isinstance(w_expectedtype, W_Tainted):
-    #    w_expectedtype = w_expectedtype.w_obj
-    w_realtype = space.type(w_obj)
-    if not space.is_w(w_realtype, w_expectedtype):
-        #msg = "expected an object of type '%s'" % (
-        #    w_expectedtype.getname(space),)
-        #    #w_realtype.getname(space))
-        raise OperationError(space.w_TaintError, space.w_None)
-    return w_obj
-app_untaint = gateway.interp2app(untaint)
-
-# ____________________________________________________________
-
- at gateway.unwrap_spec(args_w='args_w')
-def taint_atomic_function(space, w_func, args_w):
-    newargs_w = []
-    tainted = False
-    for w_arg in args_w:
-        if isinstance(w_arg, W_Tainted):
-            tainted = True
-            w_arg = w_arg.w_obj
-        elif isinstance(w_arg, W_TaintBomb):
-            return w_arg
-        newargs_w.append(w_arg)
-    w_newargs = space.newtuple(newargs_w)
-    try:
-        w_res = space.call(w_func, w_newargs)
-    except OperationError, operr:
-        if not tainted:
-            raise
-        return W_TaintBomb(space, operr)
-    if tainted:
-        w_res = taint(w_res)
-    return w_res
-
-app_taint_atomic_function = gateway.interp2app(taint_atomic_function)
-
-def taint_atomic(space, w_callable):
-    """decorator to make a callable "taint-atomic": if the function is called
-with tainted arguments, those are untainted. The result of the function is
-tainted again.  All exceptions that the callable raises are turned into
-taint bombs."""
-    meth = Method(space, space.w_fn_taint_atomic_function,
-                  w_callable, space.type(w_callable))
-    return space.wrap(meth)
-app_taint_atomic = gateway.interp2app(taint_atomic)
-
-# ____________________________________________________________
-
-executioncontext.ExecutionContext.taint_debug = 0
-
- at gateway.unwrap_spec(level=int)
-def taint_debug(space, level):
-    """Set the debug level. If the debug level is greater than 0, the creation
-of taint bombs will print debug information. For debugging purposes
-only!"""
-    space.getexecutioncontext().taint_debug = level
-app_taint_debug = gateway.interp2app(taint_debug)
-
-def taint_look(space, w_obj):
-    """Print some info about the taintedness of an object. For debugging
-purposes only!"""
-    if isinstance(w_obj, W_Tainted):
-        info = space.type(w_obj.w_obj).getname(space)
-        msg = space.str_w(w_obj.w_obj.getrepr(space, info))
-        msg = 'Taint Box %s\n' % msg
-        os.write(2, msg)
-    elif isinstance(w_obj, W_TaintBomb):
-        w_obj.debug_dump()
-    else:
-        os.write(2, 'not tainted\n')
-app_taint_look = gateway.interp2app(taint_look)
-
-def get_debug_level(space):
-    return space.getexecutioncontext().taint_debug
-
-def debug_bomb(space, operr):
-    ec = space.getexecutioncontext()
-    filename = '?'
-    codename = '?'
-    codeline = 0
-    frame = ec.gettopframe_nohidden()
-    if isinstance(frame, PyFrame):     # and, in particular, frame != None
-        filename = frame.pycode.co_filename
-        codename = frame.pycode.co_name
-        codeline = frame.get_last_lineno()
-    os.write(2, 'Taint Bomb in file "%s", line %d, in %s\n    %s\n' % (
-        filename, codeline, codename, operr.errorstr(space)))
-
-# ____________________________________________________________
-
-
-class TaintSpace(StdObjSpace):
-
-    def __init__(self, *args, **kwds):
-        StdObjSpace.__init__(self, *args, **kwds)
-        w_dict = self.newdict()
-        self.setitem(w_dict, self.wrap("__doc__"), self.wrap("""\
-Exception that is raised when an operation revealing information on a tainted
-object is performed."""))
-        self.w_TaintError = self.call_function(
-            self.w_type,
-            self.wrap("TaintError"),
-            self.newtuple([self.w_Exception]),
-            w_dict
-            )
-        w___pypy__ = self.getbuiltinmodule("__pypy__")
-        self.setattr(w___pypy__, self.wrap('taint'),
-                     self.wrap(app_taint))
-        self.setattr(w___pypy__, self.wrap('is_tainted'),
-                     self.wrap(app_is_tainted))
-        self.setattr(w___pypy__, self.wrap('untaint'),
-                     self.wrap(app_untaint))
-        self.w_fn_taint_atomic_function = self.wrap(app_taint_atomic_function)
-        self.setattr(w___pypy__, self.wrap('taint_atomic'),
-                     self.wrap(app_taint_atomic))
-        self.setattr(w___pypy__, self.wrap('TaintError'),
-                     self.w_TaintError)
-        self.setattr(w___pypy__, self.wrap('_taint_debug'),
-                     self.wrap(app_taint_debug))
-        self.setattr(w___pypy__, self.wrap('_taint_look'),
-                     self.wrap(app_taint_look))
-        patch_space_in_place(self, 'taint', proxymaker)
-
-        # XXX may leak info, perfomance hit, what about taint bombs?
-        from pypy.objspace.std.typeobject import W_TypeObject
-
-        def taint_lookup(w_obj, name):
-            if isinstance(w_obj, W_Tainted):
-                w_obj = w_obj.w_obj
-            w_type = self.type(w_obj)
-            assert isinstance(w_type, W_TypeObject)
-            return w_type.lookup(name)
-
-        def taint_lookup_in_type_where(w_obj, name):
-            if isinstance(w_obj, W_Tainted):
-                w_type = w_obj.w_obj
-            else:
-                w_type = w_obj
-            assert isinstance(w_type, W_TypeObject)
-            return w_type.lookup_where(name)
-
-        self.lookup = taint_lookup
-        self.lookup_in_type_where = taint_lookup_in_type_where
-
-
-Space = TaintSpace
-
-
-def tainted_error(space, name):
-    #msg = "operation '%s' forbidden on tainted object" % (name,)
-    raise OperationError(space.w_TaintError, space.w_None)# space.wrap(msg))
-
-
-RegularMethods = dict.fromkeys(
-    [name for name, _, _, _ in baseobjspace.ObjSpace.MethodTable])
-
-TaintResultIrregularMethods = dict.fromkeys(
-    ['wrap', 'call_args'] +
-    [name for name in baseobjspace.ObjSpace.IrregularOpTable
-          if name.startswith('new')])
-
-def proxymaker(space, name, parentfn):
-    arity = nb_forcing_args[name]
-    indices = unrolling_iterable(range(arity))
-    if name in RegularMethods:
-
-        def proxy(*args_w):
-            newargs_w = ()
-            tainted = False
-            for i in indices:
-                w_arg = args_w[i]
-                if isinstance(w_arg, W_Tainted):
-                    tainted = True
-                    w_arg = w_arg.w_obj
-                elif isinstance(w_arg, W_TaintBomb):
-                    return w_arg
-                newargs_w += (w_arg,)
-            newargs_w += args_w[arity:]
-            try:
-                w_res = parentfn(*newargs_w)
-            except OperationError, operr:
-                if not tainted:
-                    raise
-                return W_TaintBomb(space, operr)
-            if tainted:
-                w_res = taint(w_res)
-            return w_res
-
-    elif arity == 0:
-        return None
-
-    else:
-
-        def proxy(*args_w):
-            for i in indices:
-                w_arg = args_w[i]
-                if isinstance(w_arg, W_Tainted):
-                    tainted_error(space, name)
-                elif isinstance(w_arg, W_TaintBomb):
-                    w_arg.explode()
-            return parentfn(*args_w)
-
-    proxy = func_with_new_name(proxy, '%s_proxy' % name)
-    return proxy
diff --git a/pypy/objspace/test/test_taintobjspace.py b/pypy/objspace/test/test_taintobjspace.py
deleted file mode 100644
--- a/pypy/objspace/test/test_taintobjspace.py
+++ /dev/null
@@ -1,77 +0,0 @@
-from pypy.conftest import gettestobjspace
-
-class AppTest_Taint:
-
-    def setup_class(cls):
-        cls.space = gettestobjspace('taint')
-
-    def test_simple(self):
-        from __pypy__ import taint, untaint, TaintError
-        x = taint(6)
-        x = x * 7
-        raises(TaintError, "if x: y = 1")
-        t = type(x)
-        raises(TaintError, "if t is int: y = 1")
-        assert untaint(int, x) == 42
-        raises(TaintError, "untaint(float, x)")
-
-    def test_bomb(self):
-        from __pypy__ import taint, untaint, TaintError
-        x = taint(6)
-        x = x / 0
-        raises(TaintError, "if x: y = 1")
-        t = type(x)
-        raises(TaintError, "if t is int: y = 1")
-        raises(TaintError, "untaint(int, x)")
-        raises(TaintError, "untaint(float, x)")
-
-    def test_taint_atomic(self):
-        from __pypy__ import taint, untaint, TaintError, taint_atomic
-        x = taint(6)
-        x *= 7
-
-        def dummy(x):
-            if x > 40:
-                return 5
-            else:
-                return 3
-        dummy = taint_atomic(dummy)
-
-        y = dummy(x)
-        raises(TaintError, "if y == 3: z = 1")
-        assert untaint(int, y) == 5
-
-    def test_taint_atomic_exception(self):
-        from __pypy__ import taint, untaint, TaintError, taint_atomic
-        x = taint(6)
-        x *= 7
-
-        def dummy(x):
-            if x + "world" == "hello world":
-                return 5
-            else:
-                return 3
-        dummy = taint_atomic(dummy)
-
-        y = dummy(x)
-        raises(TaintError, "if y == 3: z = 1")
-        raises(TaintError, "untaint(int, y)")
-
-    def test_taint_atomic_incoming_bomb(self):
-        from __pypy__ import taint, untaint, TaintError, taint_atomic
-        x = taint(6)
-        x /= 0
-        lst = []
-
-        def dummy(x):
-            lst.append("running!")
-            if x > 40:
-                return 5
-            else:
-                return 3
-        dummy = taint_atomic(dummy)
-
-        y = dummy(x)
-        raises(TaintError, "if y == 3: z = 1")
-        assert lst == []
-        raises(TaintError, "untaint(int, y)")
diff --git a/pypy/rlib/_jit_vref.py b/pypy/rlib/_jit_vref.py
--- a/pypy/rlib/_jit_vref.py
+++ b/pypy/rlib/_jit_vref.py
@@ -25,6 +25,10 @@
     def simple_call(self):
         return self.s_instance
 
+    def getattr(self, s_attr):
+        assert s_attr.const == 'virtual'
+        return annmodel.s_Bool
+
     def rtyper_makerepr(self, rtyper):
         if rtyper.type_system.name == 'lltypesystem':
             return vrefrepr
@@ -61,6 +65,13 @@
                              " prebuilt virtual_ref")
         return lltype.nullptr(OBJECTPTR.TO)
 
+    def rtype_getattr(self, hop):
+        s_attr = hop.args_s[1]
+        assert s_attr.const == 'virtual'
+        v = hop.inputarg(self, arg=0)
+        hop.exception_cannot_occur()
+        return hop.genop('jit_is_virtual', [v], resulttype = lltype.Bool)
+
 from pypy.rpython.ootypesystem.rclass import OBJECT
 
 class OOVRefRepr(VRefRepr):
diff --git a/pypy/rlib/_rsocket_rffi.py b/pypy/rlib/_rsocket_rffi.py
--- a/pypy/rlib/_rsocket_rffi.py
+++ b/pypy/rlib/_rsocket_rffi.py
@@ -16,6 +16,7 @@
 _MINGW = target_platform.name == "mingw32"
 _SOLARIS = sys.platform == "sunos5"
 _MACOSX = sys.platform == "darwin"
+_HAS_AF_PACKET = sys.platform.startswith('linux')   # only Linux for now
 
 if _POSIX:
     includes = ('sys/types.h',
@@ -34,11 +35,12 @@
                 'stdint.h', 
                 'errno.h',
                 )
+    if _HAS_AF_PACKET:
+        includes += ('netpacket/packet.h',
+                     'sys/ioctl.h',
+                     'net/if.h')
 
-    cond_includes = [('AF_NETLINK', 'linux/netlink.h'),
-                     ('AF_PACKET', 'netpacket/packet.h'),
-                     ('AF_PACKET', 'sys/ioctl.h'),
-                     ('AF_PACKET', 'net/if.h')]
+    cond_includes = [('AF_NETLINK', 'linux/netlink.h')]
     
     libraries = ()
     calling_conv = 'c'
@@ -320,18 +322,18 @@
                                              ('events', rffi.SHORT),
                                              ('revents', rffi.SHORT)])
 
-    CConfig.sockaddr_ll = platform.Struct('struct sockaddr_ll',
+    if _HAS_AF_PACKET:
+        CConfig.sockaddr_ll = platform.Struct('struct sockaddr_ll',
                               [('sll_ifindex', rffi.INT),
                                ('sll_protocol', rffi.INT),
                                ('sll_pkttype', rffi.INT),
                                ('sll_hatype', rffi.INT),
                                ('sll_addr', rffi.CFixedArray(rffi.CHAR, 8)),
-                               ('sll_halen', rffi.INT)],
-                              ifdef='AF_PACKET')
+                               ('sll_halen', rffi.INT)])
 
-    CConfig.ifreq = platform.Struct('struct ifreq', [('ifr_ifindex', rffi.INT),
-                                 ('ifr_name', rffi.CFixedArray(rffi.CHAR, 8))],
-                                    ifdef='AF_PACKET')
+        CConfig.ifreq = platform.Struct('struct ifreq',
+                                [('ifr_ifindex', rffi.INT),
+                                 ('ifr_name', rffi.CFixedArray(rffi.CHAR, 8))])
 
 if _WIN32:
     CConfig.WSAEVENT = platform.SimpleType('WSAEVENT', rffi.VOIDP)
@@ -386,6 +388,8 @@
         constants[name] = value
     else:
         constants[name] = default
+if not _HAS_AF_PACKET and 'AF_PACKET' in constants:
+    del constants['AF_PACKET']
 
 constants['has_ipv6'] = True # This is a configuration option in CPython
 for name, value in constants.items():
@@ -439,21 +443,14 @@
 if _POSIX:
     nfds_t = cConfig.nfds_t
     pollfd = cConfig.pollfd
-    if cConfig.sockaddr_ll is not None:
+    if _HAS_AF_PACKET:
         sockaddr_ll = cConfig.sockaddr_ll
-    ifreq = cConfig.ifreq
+        ifreq = cConfig.ifreq
 if WIN32:
     WSAEVENT = cConfig.WSAEVENT
     WSANETWORKEVENTS = cConfig.WSANETWORKEVENTS
 timeval = cConfig.timeval
 
-#if _POSIX:
-#    includes = list(includes)
-#    for _name, _header in cond_includes:
-#        if getattr(cConfig, _name) is not None:
-#            includes.append(_header)
-#    eci = ExternalCompilationInfo(includes=includes, libraries=libraries,
-#                                  separate_module_sources=sources)
 
 def external(name, args, result, **kwds):
     return rffi.llexternal(name, args, result, compilation_info=eci,
@@ -544,7 +541,7 @@
     socketpair_t = rffi.CArray(socketfd_type)
     socketpair = external('socketpair', [rffi.INT, rffi.INT, rffi.INT,
                           lltype.Ptr(socketpair_t)], rffi.INT)
-    if ifreq is not None:
+    if _HAS_AF_PACKET:
         ioctl = external('ioctl', [socketfd_type, rffi.INT, lltype.Ptr(ifreq)],
                          rffi.INT)
 
diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -8,6 +8,8 @@
 from pypy.rpython.extregistry import ExtRegistryEntry
 from pypy.tool.sourcetools import func_with_new_name
 
+DEBUG_ELIDABLE_FUNCTIONS = False
+
 
 def elidable(func):
     """ Decorate a function as "trace-elidable". This means precisely that:
@@ -24,6 +26,18 @@
     If a particular call to this function ends up raising an exception, then it
     is handled like a normal function call (this decorator is ignored).
     """
+    if DEBUG_ELIDABLE_FUNCTIONS:
+        cache = {}
+        oldfunc = func
+        def func(*args):
+            result = oldfunc(*args)    # if it raises, no caching
+            try:
+                oldresult = cache.setdefault(args, result)
+            except TypeError:
+                pass           # unhashable args
+            else:
+                assert oldresult == result
+            return result
     func._elidable_function_ = True
     return func
 
@@ -38,6 +52,7 @@
     possible arguments are:
 
     * promote - promote the argument from a variable into a constant
+    * promote_string - same, but promote string by *value*
     * access_directly - directly access a virtualizable, as a structure
                         and don't treat it as a virtualizable
     * fresh_virtualizable - means that virtualizable was just allocated.
@@ -51,6 +66,9 @@
 def promote(x):
     return hint(x, promote=True)
 
+def promote_string(x):
+    return hint(x, promote_string=True)
+
 def dont_look_inside(func):
     """ Make sure the JIT does not trace inside decorated function
     (it becomes a call instead)
@@ -313,6 +331,12 @@
             raise InvalidVirtualRef
         return self._x
 
+    @property
+    def virtual(self):
+        """A property that is True if the vref contains a virtual that would
+        be forced by the '()' operator."""
+        return self._state == 'non-forced'
+
     def _finish(self):
         if self._state == 'non-forced':
             self._state = 'invalid'
diff --git a/pypy/rlib/rerased.py b/pypy/rlib/rerased.py
--- a/pypy/rlib/rerased.py
+++ b/pypy/rlib/rerased.py
@@ -135,6 +135,8 @@
     _about_ = erase_int
 
     def compute_result_annotation(self, s_obj):
+        config = self.bookkeeper.annotator.translator.config
+        assert config.translation.taggedpointers, "need to enable tagged pointers to use erase_int"
         assert annmodel.SomeInteger().contains(s_obj)
         return SomeErased()
 
@@ -218,18 +220,18 @@
         [v_value] = hop.inputargs(lltype.Signed)
         c_one = hop.inputconst(lltype.Signed, 1)
         hop.exception_is_here()
-        v2 = hop.genop('int_lshift_ovf', [v_value, c_one],
+        v2 = hop.genop('int_add_ovf', [v_value, v_value],
                        resulttype = lltype.Signed)
         v2p1 = hop.genop('int_add', [v2, c_one],
                          resulttype = lltype.Signed)
         v_instance = hop.genop('cast_int_to_ptr', [v2p1],
                                resulttype=self.lowleveltype)
-        v = hop.genop('cast_opaque_ptr', [v_instance],
-                      resulttype=self.lowleveltype)
-        return v
+        return v_instance
 
     def convert_const(self, value):
         if value._identity is _identity_for_ints:
+            config = self.rtyper.annotator.translator.config
+            assert config.translation.taggedpointers, "need to enable tagged pointers to use erase_int"
             return lltype.cast_int_to_ptr(self.lowleveltype, value._x * 2 + 1)
         bk = self.rtyper.annotator.bookkeeper
         s_obj = value._identity.get_input_annotation(bk)
@@ -266,10 +268,10 @@
         return hop.genop('int_rshift', [v2, c_one], resulttype=lltype.Signed)
 
     def rtype_erase_int(self, hop):
-        hop.exception_is_here()
         [v_value] = hop.inputargs(lltype.Signed)
         c_one = hop.inputconst(lltype.Signed, 1)
-        v2 = hop.genop('int_lshift_ovf', [v_value, c_one],
+        hop.exception_is_here()
+        v2 = hop.genop('int_add_ovf', [v_value, v_value],
                        resulttype = lltype.Signed)
         v2p1 = hop.genop('int_add', [v2, c_one],
                          resulttype = lltype.Signed)
diff --git a/pypy/rlib/rsocket.py b/pypy/rlib/rsocket.py
--- a/pypy/rlib/rsocket.py
+++ b/pypy/rlib/rsocket.py
@@ -8,6 +8,7 @@
 # Known missing features:
 #
 #   - address families other than AF_INET, AF_INET6, AF_UNIX, AF_PACKET
+#   - AF_PACKET is only supported on Linux
 #   - methods makefile(),
 #   - SSL
 #
diff --git a/pypy/rlib/test/test__jit_vref.py b/pypy/rlib/test/test__jit_vref.py
--- a/pypy/rlib/test/test__jit_vref.py
+++ b/pypy/rlib/test/test__jit_vref.py
@@ -27,10 +27,13 @@
     x1 = X()
     vref = virtual_ref(x1)
     assert vref._state == 'non-forced'
+    assert vref.virtual is True
     assert vref() is x1
     assert vref._state == 'forced'
+    assert vref.virtual is False
     virtual_ref_finish(vref, x1)
     assert vref._state == 'forced'
+    assert vref.virtual is False
     assert vref() is x1
 
 def test_direct_invalid():
@@ -135,6 +138,13 @@
         x = self.interpret(f, [])
         assert x == 42
 
+    def test_rtype_virtualattr(self):
+        def f():
+            vref = virtual_ref(X())
+            return vref.virtual
+        x = self.interpret(f, [])
+        assert x is False
+
 
 class TestLLtype(BaseTestVRef, LLRtypeMixin):
     OBJECTTYPE = OBJECTPTR
diff --git a/pypy/rlib/test/test_jit.py b/pypy/rlib/test/test_jit.py
--- a/pypy/rlib/test/test_jit.py
+++ b/pypy/rlib/test/test_jit.py
@@ -139,12 +139,11 @@
 
     def test_isconstant(self):
         def f(n):
-            assert n >= 0
             assert isconstant(n) is False
             l = []
             l.append(n)
             return len(l)
-        res = self.interpret(f, [234])
+        res = self.interpret(f, [-234])
         assert res == 1
 
 
diff --git a/pypy/rlib/test/test_rerased.py b/pypy/rlib/test/test_rerased.py
--- a/pypy/rlib/test/test_rerased.py
+++ b/pypy/rlib/test/test_rerased.py
@@ -10,6 +10,13 @@
 
 from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin
 
+def make_annotator():
+    a = RPythonAnnotator()
+    a.translator.config.translation.taggedpointers = True
+    return a
+
+
+
 class X(object):
     pass
 
@@ -55,7 +62,7 @@
 def test_annotate_1():
     def f():
         return eraseX(X())
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [])
     assert isinstance(s, SomeErased)
 
@@ -66,7 +73,7 @@
         #assert not is_integer(e)
         x2 = uneraseX(e)
         return x2
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [])
     assert isinstance(s, annmodel.SomeInstance)
     assert s.classdef == a.bookkeeper.getuniqueclassdef(X)
@@ -77,7 +84,7 @@
         #assert is_integer(e)
         x2 = unerase_int(e)
         return x2
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [])
     assert isinstance(s, annmodel.SomeInteger)
 
@@ -105,7 +112,7 @@
         x = make(n)
         return check(x, n)
     #
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [int])
     assert isinstance(s, annmodel.SomeInteger)
 
@@ -135,7 +142,7 @@
             else:
                 return inst
     #
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [])
     assert isinstance(s, annmodel.SomeInstance)
     assert s.classdef == a.bookkeeper.getuniqueclassdef(A)
@@ -155,7 +162,7 @@
             e = e2
         return unerase(e)
     #
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [int])
     assert isinstance(s, annmodel.SomeInstance)
     assert s.classdef == a.bookkeeper.getuniqueclassdef(X)
@@ -165,11 +172,14 @@
     e1 = erase_int(42)
     def f(i):
         return unerase_int(e1)
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [int])
     assert isinstance(s, annmodel.SomeInteger)
 
 class BaseTestRErased(BaseRtypingTest):
+    def interpret(self, *args, **kwargs):
+        kwargs["taggedpointers"] = True
+        return BaseRtypingTest.interpret(self, *args, **kwargs)
     def test_rtype_1(self):
         def f():
             return eraseX(X())
diff --git a/pypy/rpython/lltypesystem/ll2ctypes.py b/pypy/rpython/lltypesystem/ll2ctypes.py
--- a/pypy/rpython/lltypesystem/ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/ll2ctypes.py
@@ -15,12 +15,11 @@
     load_library_kwargs = {}
 
 import os
-from pypy import conftest
 from pypy.rpython.lltypesystem import lltype, llmemory
 from pypy.rpython.extfunc import ExtRegistryEntry
 from pypy.rlib.objectmodel import Symbolic, ComputedIntSymbolic
 from pypy.tool.uid import fixid
-from pypy.rlib.rarithmetic import r_uint, r_singlefloat, r_longfloat, base_int, intmask
+from pypy.rlib.rarithmetic import r_singlefloat, r_longfloat, base_int, intmask
 from pypy.annotation import model as annmodel
 from pypy.rpython.llinterp import LLInterpreter, LLException
 from pypy.rpython.lltypesystem.rclass import OBJECT, OBJECT_VTABLE
@@ -531,6 +530,10 @@
     def __str__(self):
         return repr(self)
 
+    def _setparentstructure(self, parent, parentindex):
+        super(_parentable_mixin, self)._setparentstructure(parent, parentindex)
+        self._keepparent = parent   # always keep a strong ref
+
 class _struct_mixin(_parentable_mixin):
     """Mixin added to _struct containers when they become ctypes-based."""
     __slots__ = ()
@@ -566,7 +569,10 @@
         return 0, sys.maxint
 
     def getitem(self, index, uninitialized_ok=False):
-        return self._storage.contents._getitem(index, boundscheck=False)
+        res = self._storage.contents._getitem(index, boundscheck=False)
+        if isinstance(self._TYPE.OF, lltype.ContainerType):
+            res._obj._setparentstructure(self, index)
+        return res
 
     def setitem(self, index, value):
         self._storage.contents._setitem(index, value, boundscheck=False)
@@ -658,6 +664,8 @@
         if T == llmemory.GCREF:
             if isinstance(llobj._obj, _llgcopaque):
                 return ctypes.c_void_p(llobj._obj.intval)
+            if isinstance(llobj._obj, int):    # tagged pointer
+                return ctypes.c_void_p(llobj._obj)
             container = llobj._obj.container
             T = lltype.Ptr(lltype.typeOf(container))
             # otherwise it came from integer and we want a c_void_p with
@@ -1271,6 +1279,7 @@
 class _llgcopaque(lltype._container):
     _TYPE = llmemory.GCREF.TO
     _name = "_llgcopaque"
+    _read_directly_intval = True     # for _ptr._cast_to_int()
 
     def __init__(self, void_p):
         if isinstance(void_p, (int, long)):
@@ -1279,6 +1288,8 @@
             self.intval = intmask(void_p.value)
 
     def __eq__(self, other):
+        if not other:
+            return self.intval == 0
         if isinstance(other, _llgcopaque):
             return self.intval == other.intval
         storage = object()
@@ -1302,11 +1313,6 @@
             return _opaque_objs[self.intval // 2]
         return force_cast(PTRTYPE, self.intval)
 
-##     def _cast_to_int(self):
-##         return self.intval
-
-##     def _cast_to_adr(self):
-##         return _lladdress(self.intval)
 
 def cast_adr_to_int(addr):
     if isinstance(addr, llmemory.fakeaddress):
diff --git a/pypy/rpython/lltypesystem/llmemory.py b/pypy/rpython/lltypesystem/llmemory.py
--- a/pypy/rpython/lltypesystem/llmemory.py
+++ b/pypy/rpython/lltypesystem/llmemory.py
@@ -498,6 +498,8 @@
 
     def _cast_to_int(self, symbolic=False):
         if self:
+            if isinstance(self.ptr._obj0, int):    # tagged integer
+                return self.ptr._obj0
             if symbolic:
                 return AddressAsInt(self)
             else:
diff --git a/pypy/rpython/lltypesystem/lloperation.py b/pypy/rpython/lltypesystem/lloperation.py
--- a/pypy/rpython/lltypesystem/lloperation.py
+++ b/pypy/rpython/lltypesystem/lloperation.py
@@ -428,6 +428,7 @@
     'jit_marker':           LLOp(),
     'jit_force_virtualizable':LLOp(canrun=True),
     'jit_force_virtual':    LLOp(canrun=True),
+    'jit_is_virtual':       LLOp(canrun=True),
     'jit_force_quasi_immutable': LLOp(canrun=True),
     'get_exception_addr':   LLOp(),
     'get_exc_value_addr':   LLOp(),
diff --git a/pypy/rpython/lltypesystem/lltype.py b/pypy/rpython/lltypesystem/lltype.py
--- a/pypy/rpython/lltypesystem/lltype.py
+++ b/pypy/rpython/lltypesystem/lltype.py
@@ -1360,6 +1360,8 @@
         obj = normalizeptr(self, check)._getobj(check)
         if isinstance(obj, int):
             return obj     # special case for cast_int_to_ptr() results put into opaques
+        if getattr(obj, '_read_directly_intval', False):
+            return obj.intval   # special case for _llgcopaque
         result = intmask(obj._getid())
         # assume that id() returns an addressish value which is
         # not zero and aligned to at least a multiple of 4
@@ -1520,7 +1522,7 @@
             and parentindex in (self._parent_type._names[0], 0)
             and self._TYPE._gckind == typeOf(parent)._gckind):
             # keep strong reference to parent, we share the same allocation
-            self._keepparent = parent 
+            self._keepparent = parent
 
     def _parentstructure(self, check=True):
         if self._wrparent is not None:
@@ -1728,7 +1730,8 @@
         # Keep the parent array alive, we share the same allocation.
         # Don't do it if we are inside a GC object, though -- it's someone
         # else's job to keep the GC object alive
-        if typeOf(top_container(parent))._gckind == 'raw':
+        if (typeOf(top_container(parent))._gckind == 'raw' or
+            hasattr(top_container(parent)._storage, 'contents')):  # ll2ctypes
             self._keepparent = parent
 
     def __str__(self):
diff --git a/pypy/rpython/lltypesystem/opimpl.py b/pypy/rpython/lltypesystem/opimpl.py
--- a/pypy/rpython/lltypesystem/opimpl.py
+++ b/pypy/rpython/lltypesystem/opimpl.py
@@ -538,6 +538,9 @@
 def op_jit_force_virtual(x):
     return x
 
+def op_jit_is_virtual(x):
+    return False
+
 def op_jit_force_quasi_immutable(*args):
     pass
 
diff --git a/pypy/rpython/lltypesystem/rbuilder.py b/pypy/rpython/lltypesystem/rbuilder.py
--- a/pypy/rpython/lltypesystem/rbuilder.py
+++ b/pypy/rpython/lltypesystem/rbuilder.py
@@ -29,7 +29,7 @@
         except OverflowError:
             raise MemoryError
         newbuf = mallocfn(new_allocated)
-        copycontentsfn(ll_builder.buf, newbuf, 0, 0, ll_builder.allocated)
+        copycontentsfn(ll_builder.buf, newbuf, 0, 0, ll_builder.used)
         ll_builder.buf = newbuf
         ll_builder.allocated = new_allocated
     return func_with_new_name(stringbuilder_grow, name)
@@ -56,7 +56,7 @@
 class BaseStringBuilderRepr(AbstractStringBuilderRepr):
     def empty(self):
         return nullptr(self.lowleveltype.TO)
-    
+
     @classmethod
     def ll_new(cls, init_size):
         if init_size < 0 or init_size > MAX:
diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py
--- a/pypy/rpython/lltypesystem/rdict.py
+++ b/pypy/rpython/lltypesystem/rdict.py
@@ -1,16 +1,14 @@
 from pypy.tool.pairtype import pairtype
-from pypy.annotation import model as annmodel
 from pypy.objspace.flow.model import Constant
-from pypy.rpython.rdict import AbstractDictRepr, AbstractDictIteratorRepr,\
-     rtype_newdict
+from pypy.rpython.rdict import (AbstractDictRepr, AbstractDictIteratorRepr,
+     rtype_newdict)
 from pypy.rpython.lltypesystem import lltype
+from pypy.rlib import objectmodel, jit
 from pypy.rlib.rarithmetic import r_uint, intmask, LONG_BIT
-from pypy.rlib.objectmodel import hlinvoke
-from pypy.rpython import robject
-from pypy.rlib import objectmodel, jit
 from pypy.rpython import rmodel
 from pypy.rpython.error import TyperError
 
+
 HIGHEST_BIT = intmask(1 << (LONG_BIT - 1))
 MASK = intmask(HIGHEST_BIT - 1)
 
@@ -417,17 +415,16 @@
     ENTRIES = lltype.typeOf(entries).TO
     return ENTRIES.fasthashfn(entries[i].key)
 
- at jit.dont_look_inside
 def ll_get_value(d, i):
     return d.entries[i].value
 
 def ll_keyhash_custom(d, key):
     DICT = lltype.typeOf(d).TO
-    return hlinvoke(DICT.r_rdict_hashfn, d.fnkeyhash, key)
+    return objectmodel.hlinvoke(DICT.r_rdict_hashfn, d.fnkeyhash, key)
 
 def ll_keyeq_custom(d, key1, key2):
     DICT = lltype.typeOf(d).TO
-    return hlinvoke(DICT.r_rdict_eqfn, d.fnkeyeq, key1, key2)
+    return objectmodel.hlinvoke(DICT.r_rdict_eqfn, d.fnkeyeq, key1, key2)
 
 def ll_dict_len(d):
     return d.num_items
@@ -448,6 +445,8 @@
     i = ll_dict_lookup(d, key, hash)
     return _ll_dict_setitem_lookup_done(d, key, value, hash, i)
 
+# Leaving as dont_look_inside ATM, it has a few branches which could lead to
+# many bridges if we don't consider their possible frequency.
 @jit.dont_look_inside
 def _ll_dict_setitem_lookup_done(d, key, value, hash, i):
     valid = (i & HIGHEST_BIT) == 0
@@ -492,6 +491,8 @@
         raise KeyError
     _ll_dict_del(d, i)
 
+# XXX: Move the size checking and resize into a single call which is opauqe to
+# the JIT to avoid extra branches.
 @jit.dont_look_inside
 def _ll_dict_del(d, i):
     d.entries.mark_deleted(i)
@@ -532,6 +533,7 @@
 # ------- a port of CPython's dictobject.c's lookdict implementation -------
 PERTURB_SHIFT = 5
 
+ at jit.dont_look_inside
 def ll_dict_lookup(d, key, hash):
     entries = d.entries
     ENTRIES = lltype.typeOf(entries).TO
@@ -621,7 +623,6 @@
     d.num_items = 0
     d.resize_counter = DICT_INITSIZE * 2
     return d
-ll_newdict.oopspec = 'newdict()'
 
 def ll_newdict_size(DICT, length_estimate):
     length_estimate = (length_estimate // 2) * 3
@@ -633,7 +634,6 @@
     d.num_items = 0
     d.resize_counter = n * 2
     return d
-ll_newdict_size.oopspec = 'newdict()'
 
 # pypy.rpython.memory.lldict uses a dict based on Struct and Array
 # instead of GcStruct and GcArray, which is done by using different
@@ -866,7 +866,6 @@
         global_popitem_index.nextindex = base + counter
     return i
 
- at jit.dont_look_inside
 def ll_popitem(ELEM, dic):
     i = _ll_getnextitem(dic)
     entry = dic.entries[i]
diff --git a/pypy/rpython/lltypesystem/rtagged.py b/pypy/rpython/lltypesystem/rtagged.py
--- a/pypy/rpython/lltypesystem/rtagged.py
+++ b/pypy/rpython/lltypesystem/rtagged.py
@@ -43,7 +43,7 @@
         v_value = hop.inputarg(lltype.Signed, arg=1)
         c_one = hop.inputconst(lltype.Signed, 1)
         hop.exception_is_here()
-        v2 = hop.genop('int_lshift_ovf', [v_value, c_one],
+        v2 = hop.genop('int_add_ovf', [v_value, v_value],
                        resulttype = lltype.Signed)
         v2p1 = hop.genop('int_add', [v2, c_one],
                          resulttype = lltype.Signed)
diff --git a/pypy/rpython/lltypesystem/test/test_ll2ctypes.py b/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
--- a/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
@@ -8,6 +8,7 @@
 from pypy.rpython.lltypesystem.ll2ctypes import uninitialized2ctypes
 from pypy.rpython.lltypesystem.ll2ctypes import ALLOCATED, force_cast
 from pypy.rpython.lltypesystem.ll2ctypes import cast_adr_to_int, get_ctypes_type
+from pypy.rpython.lltypesystem.ll2ctypes import _llgcopaque
 from pypy.rpython.annlowlevel import llhelper
 from pypy.rlib import rposix
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
@@ -1123,6 +1124,9 @@
 
         #assert lltype.cast_ptr_to_int(ref1) == intval
 
+        x = rffi.cast(llmemory.GCREF, -17)
+        assert lltype.cast_ptr_to_int(x) == -17
+
     def test_ptr_truth(self):
         abc = rffi.cast(lltype.Ptr(lltype.FuncType([], lltype.Void)), 0)
         assert not abc
@@ -1346,6 +1350,11 @@
         round = ctypes2lltype(llmemory.GCREF, lltype2ctypes(opaque.hide()))
         assert Opaque.show(round) is opaque
 
+    def test_array_of_structs(self):
+        A = lltype.GcArray(lltype.Struct('x', ('v', lltype.Signed)))
+        a = lltype.malloc(A, 5)
+        a2 = ctypes2lltype(lltype.Ptr(A), lltype2ctypes(a))
+        assert a2._obj.getitem(0)._obj._parentstructure() is a2._obj
 
 class TestPlatform(object):
     def test_lib_on_libpaths(self):
@@ -1387,3 +1396,7 @@
         f = rffi.llexternal('f', [rffi.INT, rffi.INT], rffi.INT,
                             compilation_info=eci)
         assert f(3, 4) == 7
+
+    def test_llgcopaque_eq(self):
+        assert _llgcopaque(1) != None
+        assert _llgcopaque(0) == None
diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -1292,10 +1292,12 @@
         # if a prebuilt GcStruct contains a pointer to a young object,
         # then the write_barrier must have ensured that the prebuilt
         # GcStruct is in the list self.old_objects_pointing_to_young.
+        debug_start("gc-minor-walkroots")
         self.root_walker.walk_roots(
             MiniMarkGC._trace_drag_out1,  # stack roots
             MiniMarkGC._trace_drag_out1,  # static in prebuilt non-gc
             None)                         # static in prebuilt gc
+        debug_stop("gc-minor-walkroots")
 
     def collect_cardrefs_to_nursery(self):
         size_gc_header = self.gcheaderbuilder.size_gc_header
diff --git a/pypy/rpython/rclass.py b/pypy/rpython/rclass.py
--- a/pypy/rpython/rclass.py
+++ b/pypy/rpython/rclass.py
@@ -191,6 +191,10 @@
                     "class %r inherits from its parent _immutable_=True, "
                     "so it should also declare _immutable_=True" % (
                     self.classdef,))
+            if loc.classdict.get('_immutable_').value is not True:
+                raise TyperError(
+                    "class %r: _immutable_ = something else than True" % (
+                    self.classdef,))
             hints = hints.copy()
             hints['immutable'] = True
         self.immutable_field_set = set()  # unless overwritten below
diff --git a/pypy/tool/logparser.py b/pypy/tool/logparser.py
--- a/pypy/tool/logparser.py
+++ b/pypy/tool/logparser.py
@@ -298,6 +298,8 @@
         image.paste(textpercent, (t1x, 5), textpercent)
         image.paste(textlabel,   (t2x, 5), textlabel)
         images.append(image)
+    if not images:
+        return None
     return combine(images, spacing=0, border=1, horizontal=False)
 
 def get_timesummary_single_image(totaltimes, totaltime0, componentdict,
@@ -333,6 +335,8 @@
         del totaltimes[None]
     img2 = render_histogram(totaltimes, totaltime0, {},
                             width, summarybarheight)
+    if img2 is None:
+        return img1
     return combine([img1, img2], spacing=spacing, horizontal=True)
 
 # ----------
diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -56,8 +56,8 @@
     binaries = [(pypy_c, rename_pypy_c)]
     #
     if sys.platform == 'win32':
-        # Can't rename a DLL: it is always called 'libpypy_c.dll'
-        for extra in ['libpypy_c.dll',
+        # Can't rename a DLL: it is always called 'libpypy-c.dll'
+        for extra in ['libpypy-c.dll',
                       'libexpat.dll', 'sqlite3.dll', 'msvcr90.dll']:
             p = pypy_c.dirpath().join(extra)
             if not p.check():
diff --git a/pypy/translator/c/database.py b/pypy/translator/c/database.py
--- a/pypy/translator/c/database.py
+++ b/pypy/translator/c/database.py
@@ -176,7 +176,7 @@
                      # introduced by the GC transformer, or the type_info_table
         return node
 
-    def get(self, obj):
+    def get(self, obj, funcgen=None):
         if isinstance(obj, CConstant):
             return obj.c_name  # without further checks
         T = typeOf(obj)
@@ -228,6 +228,8 @@
                     return '((%s) %d)' % (cdecl(self.gettype(T), ''),
                                           obj._obj)
                 node = self.getcontainernode(container)
+                if node._funccodegen_owner is None:
+                    node._funccodegen_owner = funcgen
                 return node.getptrname()
             else:
                 return '((%s) NULL)' % (cdecl(self.gettype(T), ''), )
@@ -284,14 +286,16 @@
             finish_callbacks.append(('GC transformer: finished tables',
                                      self.gctransformer.get_finish_tables()))
 
-        def add_dependencies(newdependencies):
+        def add_dependencies(newdependencies, parent=None):
             for value in newdependencies:
                 #if isinstance(value, _uninitialized):
                 #    continue
                 if isinstance(typeOf(value), ContainerType):
-                    self.getcontainernode(value)
+                    node = self.getcontainernode(value)
+                    if parent and node._funccodegen_owner is not None:
+                        node._funccodegen_owner = parent._funccodegen_owner
                 else:
-                    self.get(value)
+                    self.get(value, parent and parent._funccodegen_owner)
 
         while True:
             while True:
@@ -303,7 +307,7 @@
                 if i == len(self.containerlist):
                     break
                 node = self.containerlist[i]
-                add_dependencies(node.enum_dependencies())
+                add_dependencies(node.enum_dependencies(), node)
                 i += 1
                 self.completedcontainers = i
                 if i == show_i:
diff --git a/pypy/translator/c/funcgen.py b/pypy/translator/c/funcgen.py
--- a/pypy/translator/c/funcgen.py
+++ b/pypy/translator/c/funcgen.py
@@ -833,7 +833,7 @@
         return 'INSTRUMENT_COUNT(%s);' % counter_label
             
     def OP_IS_EARLY_CONSTANT(self, op):
-        return self.expr(op.result)  + ' = 0;' # Allways false
+        return '%s = 0; /* IS_EARLY_CONSTANT */' % (self.expr(op.result),)
 
     def OP_JIT_MARKER(self, op):
         return '/* JIT_MARKER %s */' % op
@@ -845,6 +845,9 @@
         return '%s = %s; /* JIT_FORCE_VIRTUAL */' % (self.expr(op.result),
                                                      self.expr(op.args[0]))
 
+    def OP_JIT_IS_VIRTUAL(self, op):
+        return '%s = 0; /* JIT_IS_VIRTUAL */' % (self.expr(op.result),)
+
     def OP_JIT_FORCE_QUASI_IMMUTABLE(self, op):
         return '/* JIT_FORCE_QUASI_IMMUTABLE %s */' % op
 
diff --git a/pypy/translator/c/gc.py b/pypy/translator/c/gc.py
--- a/pypy/translator/c/gc.py
+++ b/pypy/translator/c/gc.py
@@ -170,6 +170,7 @@
     nodekind = 'refcnt rtti'
     globalcontainer = True
     typename = 'void (@)(void *)'
+    _funccodegen_owner = None
 
     def __init__(self, db, T, obj):
         assert T == RuntimeTypeInfo
@@ -266,6 +267,7 @@
     nodekind = 'boehm rtti'
     globalcontainer = True
     typename = 'char @'
+    _funccodegen_owner = None
 
     def __init__(self, db, T, obj):
         assert T == RuntimeTypeInfo
diff --git a/pypy/translator/c/gcc/test/test_asmgcroot.py b/pypy/translator/c/gcc/test/test_asmgcroot.py
--- a/pypy/translator/c/gcc/test/test_asmgcroot.py
+++ b/pypy/translator/c/gcc/test/test_asmgcroot.py
@@ -21,6 +21,7 @@
         config = get_pypy_config(translating=True)
         config.translation.gc = cls.gcpolicy
         config.translation.gcrootfinder = "asmgcc"
+        config.translation.taggedpointers = getattr(cls, "taggedpointers", False)
         return config
 
     @classmethod
diff --git a/pypy/translator/c/genc.py b/pypy/translator/c/genc.py
--- a/pypy/translator/c/genc.py
+++ b/pypy/translator/c/genc.py
@@ -682,8 +682,7 @@
     def getbasecfilefornode(self, node, basecname):
         # For FuncNode instances, use the python source filename (relative to
         # the top directory):
-        if hasattr(node.obj, 'graph'):
-            g = node.obj.graph
+        def invent_nice_name(g):
             # Lookup the filename from the function.
             # However, not all FunctionGraph objs actually have a "func":
             if hasattr(g, 'func'):
@@ -693,6 +692,15 @@
                     if pypkgpath:
                         relpypath =  localpath.relto(pypkgpath)
                         return relpypath.replace('.py', '.c')
+            return None
+        if hasattr(node.obj, 'graph'):
+            name = invent_nice_name(node.obj.graph)
+            if name is not None:
+                return name
+        elif node._funccodegen_owner is not None:
+            name = invent_nice_name(node._funccodegen_owner.graph)
+            if name is not None:
+                return "data_" + name
         return basecname
 
     def splitnodesimpl(self, basecname, nodes, nextra, nbetween,
diff --git a/pypy/translator/c/node.py b/pypy/translator/c/node.py
--- a/pypy/translator/c/node.py
+++ b/pypy/translator/c/node.py
@@ -485,6 +485,7 @@
         __slots__ = """db obj 
                        typename implementationtypename
                         name
+                        _funccodegen_owner
                         globalcontainer""".split()
     eci_name = '_compilation_info'
 
@@ -509,6 +510,7 @@
         if self.typename != self.implementationtypename:
             if db.gettypedefnode(T).extra_union_for_varlength:
                 self.name += '.b'
+        self._funccodegen_owner = None
 
     def getptrname(self):
         return '(&%s)' % self.name
@@ -842,6 +844,9 @@
         if self.funcgens:
             argnames = self.funcgens[0].argnames()  #Assume identical for all funcgens
             self.implementationtypename = self.db.gettype(self.T, argnames=argnames)
+            self._funccodegen_owner = self.funcgens[0]
+        else:
+            self._funccodegen_owner = None
 
     def basename(self):
         return self.obj._name
@@ -1005,6 +1010,7 @@
     globalcontainer = True
     typename = 'PyObject @'
     implementationtypename = 'PyObject *@'
+    _funccodegen_owner = None
 
     def __init__(self, db, T, obj):
         # obj is a _pyobject here; obj.value is the underlying CPython object
diff --git a/pypy/translator/c/primitive.py b/pypy/translator/c/primitive.py
--- a/pypy/translator/c/primitive.py
+++ b/pypy/translator/c/primitive.py
@@ -141,7 +141,12 @@
 
 def name_gcref(value, db):
     if value:
-        realobj = value._obj.container
+        obj = value._obj
+        if isinstance(obj, int):
+            # a tagged pointer
+            assert obj & 1 == 1
+            return '((%s) %d)' % (cdecl("void*", ''), obj)
+        realobj = obj.container
         realvalue = cast_opaque_ptr(Ptr(typeOf(realobj)), value)
         return db.get(realvalue)
     else:
diff --git a/pypy/translator/c/src/thread_nt.h b/pypy/translator/c/src/thread_nt.h
--- a/pypy/translator/c/src/thread_nt.h
+++ b/pypy/translator/c/src/thread_nt.h
@@ -245,7 +245,7 @@
     if (pending_acquires <= 0)
         return 0;
     InterlockedIncrement(&pending_acquires);
-    PulseEvent(&cond_gil);
+    PulseEvent(cond_gil);
 
     /* hack: the three following lines do a pthread_cond_wait(), and
        normally specifying a timeout of INFINITE would be fine.  But the
@@ -253,7 +253,7 @@
        (small) risk that PulseEvent misses the WaitForSingleObject().
        In this case the process will just sleep a few milliseconds. */
     LeaveCriticalSection(&mutex_gil);
-    WaitForSingleObject(&cond_gil, 15);
+    WaitForSingleObject(cond_gil, 15);
     EnterCriticalSection(&mutex_gil);
 
     InterlockedDecrement(&pending_acquires);
@@ -263,7 +263,7 @@
 void RPyGilRelease(void)
 {
     LeaveCriticalSection(&mutex_gil);
-    PulseEvent(&cond_gil);
+    PulseEvent(cond_gil);
 }
 
 void RPyGilAcquire(void)
diff --git a/pypy/translator/c/test/test_newgc.py b/pypy/translator/c/test/test_newgc.py
--- a/pypy/translator/c/test/test_newgc.py
+++ b/pypy/translator/c/test/test_newgc.py
@@ -1487,6 +1487,43 @@
         res = self.run("tagged")
         assert res == expected
 
+    def define_erased(cls):
+        from pypy.rlib import rerased
+        erase, unerase = rerased.new_erasing_pair("test")
+        class Unrelated(object):
+            pass
+
+        u = Unrelated()
+        u.tagged = True
+        u.x = rerased.erase_int(41)
+        class A(object):
+            pass
+        def fn():
+            n = 1
+            while n >= 0:
+                if u.tagged:
+                    n = rerased.unerase_int(u.x)
+                    a = A()
+                    a.n = n - 1
+                    u.x = erase(a)
+                    u.tagged = False
+                else:
+                    n = unerase(u.x).n
+                    u.x = rerased.erase_int(n - 1)
+                    u.tagged = True
+        def func():
+            rgc.collect() # check that a prebuilt erased integer doesn't explode
+            u.x = rerased.erase_int(1000)
+            u.tagged = True
+            fn()
+            return 1
+        return func
+
+    def test_erased(self):
+        expected = self.run_orig("erased")
+        res = self.run("erased")
+        assert res == expected
+
 from pypy.rlib.objectmodel import UnboxedValue
 
 class TaggedBase(object):
diff --git a/pypy/translator/platform/linux.py b/pypy/translator/platform/linux.py
--- a/pypy/translator/platform/linux.py
+++ b/pypy/translator/platform/linux.py
@@ -1,5 +1,6 @@
 """Support for Linux."""
 
+import sys
 from pypy.translator.platform.posix import BasePosix
 
 class BaseLinux(BasePosix):
@@ -24,15 +25,18 @@
         return self._pkg_config("libffi", "--libs-only-L",
                                 ['/usr/lib/libffi'])
 
+    def library_dirs_for_libffi_a(self):
+        # places where we need to look for libffi.a
+        # XXX obscuuure!  only look for libffi.a if run with translate.py
+        if 'translate' in sys.modules:
+            return self.library_dirs_for_libffi() + ['/usr/lib']
+        else:
+            return []
+
 
 class Linux(BaseLinux):
     shared_only = ()    # it seems that on 32-bit linux, compiling with -fPIC
                         # gives assembler that asmgcc is not happy about.
 
-    def library_dirs_for_libffi_a(self):
-        # places where we need to look for libffi.a
-        return self.library_dirs_for_libffi() + ['/usr/lib']
-
-
 class Linux64(BaseLinux):
     pass
diff --git a/pypy/translator/platform/posix.py b/pypy/translator/platform/posix.py
--- a/pypy/translator/platform/posix.py
+++ b/pypy/translator/platform/posix.py
@@ -157,7 +157,7 @@
 
         rules = [
             ('all', '$(DEFAULT_TARGET)', []),
-            ('$(TARGET)', '$(OBJECTS)', '$(CC_LINK) $(LDFLAGS) $(LDFLAGSEXTRA) -o $@ $(OBJECTS) $(LIBDIRS) $(LIBS) $(LINKFILES)'),
+            ('$(TARGET)', '$(OBJECTS)', '$(CC_LINK) $(LDFLAGSEXTRA) -o $@ $(OBJECTS) $(LIBDIRS) $(LIBS) $(LINKFILES) $(LDFLAGS)'),
             ('%.o', '%.c', '$(CC) $(CFLAGS) $(CFLAGSEXTRA) -o $@ -c $< $(INCLUDEDIRS)'),
             ]
 

From noreply at buildbot.pypy.org  Tue Oct 25 11:07:50 2011
From: noreply at buildbot.pypy.org (bivab)
Date: Tue, 25 Oct 2011 11:07:50 +0200 (CEST)
Subject: [pypy-commit] pypy arm-backend-2: implement getinteriorfield_gc and
 setinteriorfield_gc
Message-ID: <20111025090750.DA56F820C7@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: arm-backend-2
Changeset: r48415:6d558e62094e
Date: 2011-10-24 12:37 +0200
http://bitbucket.org/pypy/pypy/changeset/6d558e62094e/

Log:	implement getinteriorfield_gc and setinteriorfield_gc

diff --git a/pypy/jit/backend/arm/opassembler.py b/pypy/jit/backend/arm/opassembler.py
--- a/pypy/jit/backend/arm/opassembler.py
+++ b/pypy/jit/backend/arm/opassembler.py
@@ -565,6 +565,64 @@
     emit_op_getfield_raw_pure = emit_op_getfield_gc
     emit_op_getfield_gc_pure = emit_op_getfield_gc
 
+    def emit_op_getinteriorfield_gc(self, op, arglocs, regalloc, fcond):
+        base_loc, index_loc, res_loc, ofs_loc, ofs, itemsize, fieldsize = arglocs
+        self.mc.gen_load_int(r.ip.value, itemsize.value)
+        self.mc.MUL(r.ip.value, index_loc.value, r.ip.value)
+        if ofs.value > 0:
+            if ofs_loc.is_imm():
+                self.mc.ADD_ri(r.ip.value, r.ip.value, ofs_loc.value)
+            else:
+                self.mc.ADD_rr(r.ip.value, r.ip.value, ofs_loc.value)
+
+        if fieldsize.value == 8:
+            # vldr only supports imm offsets
+            # so if the ofset is too large we add it to the base and use an
+            # offset of 0
+            assert res_loc.is_vfp_reg()
+            self.mc.ADD_rr(r.ip.value, base_loc.value, r.ip.value)
+            self.mc.VLDR(res_loc.value, r.ip.value, 0)
+        elif fieldsize.value == 4:
+            self.mc.LDR_rr(res_loc.value, base_loc.value, r.ip.value)
+        elif fieldsize.value == 2:
+            self.mc.LDRH_rr(res_loc.value, base_loc.value, r.ip.value)
+        elif fieldsize.value == 1:
+            self.mc.LDRB_rr(res_loc.value, base_loc.value, r.ip.value)
+        else:
+            assert 0
+
+        #XXX Hack, Hack, Hack
+        if not we_are_translated():
+            signed = op.getdescr().fielddescr.is_field_signed()
+            self._ensure_result_bit_extension(res_loc, fieldsize.value, signed)
+        return fcond
+
+    def emit_op_setinteriorfield_gc(self, op, arglocs, regalloc, fcond):
+        base_loc, index_loc, value_loc, ofs_loc, ofs, itemsize, fieldsize = arglocs
+        self.mc.gen_load_int(r.ip.value, itemsize.value)
+        self.mc.MUL(r.ip.value, index_loc.value, r.ip.value)
+        if ofs.value > 0:
+            if ofs_loc.is_imm():
+                self.mc.ADD_ri(r.ip.value, r.ip.value, ofs_loc.value)
+            else:
+                self.mc.ADD_rr(r.ip.value, r.ip.value, ofs_loc.value)
+        if fieldsize.value == 8:
+            # vstr only supports imm offsets
+            # so if the ofset is too large we add it to the base and use an
+            # offset of 0
+            assert value_loc.is_vfp_reg()
+            self.mc.ADD_rr(r.ip.value, base_loc.value, r.ip.value)
+            self.mc.VSTR(value_loc.value, r.ip.value, 0)
+        elif fieldsize.value == 4:
+            self.mc.STR_rr(value_loc.value, base_loc.value, r.ip.value)
+        elif fieldsize.value == 2:
+            self.mc.STRH_rr(value_loc.value, base_loc.value, r.ip.value)
+        elif fieldsize.value == 1:
+            self.mc.STRB_rr(value_loc.value, base_loc.value, r.ip.value)
+        else:
+            assert 0
+        return fcond
+
 
 
 
diff --git a/pypy/jit/backend/arm/regalloc.py b/pypy/jit/backend/arm/regalloc.py
--- a/pypy/jit/backend/arm/regalloc.py
+++ b/pypy/jit/backend/arm/regalloc.py
@@ -17,7 +17,8 @@
                                         INT, REF, FLOAT, LoopToken)
 from pypy.jit.metainterp.resoperation import rop
 from pypy.jit.backend.llsupport.descr import BaseFieldDescr, BaseArrayDescr, \
-                                             BaseCallDescr, BaseSizeDescr
+                                             BaseCallDescr, BaseSizeDescr, \
+                                             InteriorFieldDescr
 from pypy.jit.backend.llsupport import symbolic
 from pypy.rpython.lltypesystem import lltype, rffi, rstr, llmemory
 from pypy.jit.codewriter import heaptracker
@@ -637,6 +638,46 @@
     prepare_op_getfield_raw_pure = prepare_op_getfield_gc
     prepare_op_getfield_gc_pure = prepare_op_getfield_gc
 
+    def prepare_op_getinteriorfield_gc(self, op, fcond):
+        t = self._unpack_interiorfielddescr(op.getdescr())
+        ofs, itemsize, fieldsize, sign = t
+        args = op.getarglist()
+        base_loc, base_box = self._ensure_value_is_boxed(op.getarg(0), args)
+        index_loc, index_box = self._ensure_value_is_boxed(op.getarg(1), args)
+        c_ofs = ConstInt(ofs)
+        if _check_imm_arg(c_ofs):
+            ofs_loc = imm(ofs)
+        else:
+            ofs_loc, ofs_box = self._ensure_value_is_boxed(c_ofs, [base_box, index_box])
+            self.possibly_free_var(ofs_box)
+        self.possibly_free_vars(args)
+        self.possibly_free_var(base_box)
+        self.possibly_free_var(index_box)
+        result_loc = self.force_allocate_reg(op.result)
+        return [base_loc, index_loc, result_loc, ofs_loc, imm(ofs), 
+                                        imm(itemsize), imm(fieldsize)]
+
+    
+    def prepare_op_setinteriorfield_gc(self, op, fcond):
+        t = self._unpack_interiorfielddescr(op.getdescr())
+        ofs, itemsize, fieldsize, sign = t
+        boxes = [None]*3
+        base_loc, base_box = self._ensure_value_is_boxed(op.getarg(0), boxes)
+        boxes[0] = base_box
+        index_loc, index_box = self._ensure_value_is_boxed(op.getarg(1), boxes)
+        boxes[1] = index_box
+        value_loc, value_box = self._ensure_value_is_boxed(op.getarg(2), boxes)
+        boxes[2] = value_box
+        c_ofs = ConstInt(ofs)
+        if _check_imm_arg(c_ofs):
+            ofs_loc = imm(ofs)
+        else:
+            ofs_loc, ofs_box = self._ensure_value_is_boxed(c_ofs, boxes)
+            self.possibly_free_var(ofs_box)
+        self.possibly_free_vars(boxes)
+        return [base_loc, index_loc, value_loc, ofs_loc, imm(ofs),
+                                        imm(itemsize), imm(fieldsize)]
+
     def prepare_op_arraylen_gc(self, op, fcond):
         arraydescr = op.getdescr()
         assert isinstance(arraydescr, BaseArrayDescr)
@@ -657,7 +698,6 @@
         boxes.append(base_box)
         ofs_loc, ofs_box = self._ensure_value_is_boxed(a1, boxes)
         boxes.append(ofs_box)
-        #XXX check if imm would be fine here
         value_loc, value_box = self._ensure_value_is_boxed(a2, boxes)
         boxes.append(value_box)
         self.possibly_free_vars(boxes)
@@ -1071,6 +1111,16 @@
         assert (1 << scale) == size
         return size, scale, ofs, ofs_length, ptr
 
+    # from ../x86/regalloc.py:965
+    def _unpack_interiorfielddescr(self, descr):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs = arraydescr.get_base_size(self.cpu.translate_support_code)
+        itemsize = arraydescr.get_item_size(self.cpu.translate_support_code)
+        fieldsize = descr.fielddescr.get_field_size(self.cpu.translate_support_code)
+        sign = descr.fielddescr.is_field_signed()
+        ofs += descr.fielddescr.offset
+        return ofs, itemsize, fieldsize, sign
 
     prepare_op_float_add = prepare_float_op(name='float_add')
     prepare_op_float_sub = prepare_float_op(name='float_sub')

From noreply at buildbot.pypy.org  Tue Oct 25 11:07:52 2011
From: noreply at buildbot.pypy.org (bivab)
Date: Tue, 25 Oct 2011 11:07:52 +0200 (CEST)
Subject: [pypy-commit] pypy arm-backend-2: add cast_ptr_to_int and
	cast_int_to_ptr
Message-ID: <20111025090752.0FF5C820C7@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: arm-backend-2
Changeset: r48416:3bae2cc9ba15
Date: 2011-10-24 12:38 +0200
http://bitbucket.org/pypy/pypy/changeset/3bae2cc9ba15/

Log:	add cast_ptr_to_int and cast_int_to_ptr

diff --git a/pypy/jit/backend/arm/opassembler.py b/pypy/jit/backend/arm/opassembler.py
--- a/pypy/jit/backend/arm/opassembler.py
+++ b/pypy/jit/backend/arm/opassembler.py
@@ -408,6 +408,9 @@
         self.mov_loc_loc(argloc, resloc)
         return fcond
 
+    emit_op_cast_ptr_to_int = emit_op_same_as
+    emit_op_cast_int_to_ptr = emit_op_same_as
+
     def emit_op_guard_no_exception(self, op, arglocs, regalloc, fcond):
         loc = arglocs[0]
         failargs = arglocs[1:]

From noreply at buildbot.pypy.org  Tue Oct 25 11:07:53 2011
From: noreply at buildbot.pypy.org (bivab)
Date: Tue, 25 Oct 2011 11:07:53 +0200 (CEST)
Subject: [pypy-commit] pypy arm-backend-2: add some asserts
Message-ID: <20111025090753.3C453820C7@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: arm-backend-2
Changeset: r48417:8754b1a53808
Date: 2011-10-24 14:32 +0200
http://bitbucket.org/pypy/pypy/changeset/8754b1a53808/

Log:	add some asserts

diff --git a/pypy/jit/backend/arm/opassembler.py b/pypy/jit/backend/arm/opassembler.py
--- a/pypy/jit/backend/arm/opassembler.py
+++ b/pypy/jit/backend/arm/opassembler.py
@@ -640,15 +640,16 @@
 
     def emit_op_setarrayitem_gc(self, op, arglocs, regalloc, fcond):
         value_loc, base_loc, ofs_loc, scale, ofs = arglocs
-
+        assert ofs_loc.is_reg()
         if scale.value > 0:
             scale_loc = r.ip
             self.mc.LSL_ri(r.ip.value, ofs_loc.value, scale.value)
         else:
             scale_loc = ofs_loc
 
+        # add the base offset  
         if ofs.value > 0:
-            self.mc.ADD_ri(r.ip.value, scale_loc.value, ofs.value)
+            self.mc.ADD_ri(r.ip.value, scale_loc.value, imm=ofs.value)
             scale_loc = r.ip
 
         if scale.value == 3:
@@ -670,11 +671,14 @@
 
     def emit_op_getarrayitem_gc(self, op, arglocs, regalloc, fcond):
         res, base_loc, ofs_loc, scale, ofs = arglocs
+        assert ofs_loc.is_reg()
         if scale.value > 0:
             scale_loc = r.ip
             self.mc.LSL_ri(r.ip.value, ofs_loc.value, scale.value)
         else:
             scale_loc = ofs_loc
+
+        # add the base offset  
         if ofs.value > 0:
             self.mc.ADD_ri(r.ip.value, scale_loc.value, imm=ofs.value)
             scale_loc = r.ip
diff --git a/pypy/jit/backend/arm/regalloc.py b/pypy/jit/backend/arm/regalloc.py
--- a/pypy/jit/backend/arm/regalloc.py
+++ b/pypy/jit/backend/arm/regalloc.py
@@ -692,7 +692,7 @@
 
     def prepare_op_setarrayitem_gc(self, op, fcond):
         a0, a1, a2 = boxes = list(op.getarglist())
-        _, scale, ofs, _, ptr = self._unpack_arraydescr(op.getdescr())
+        _, scale, base_ofs, _, ptr = self._unpack_arraydescr(op.getdescr())
 
         base_loc, base_box  = self._ensure_value_is_boxed(a0, boxes)
         boxes.append(base_box)
@@ -701,12 +701,13 @@
         value_loc, value_box = self._ensure_value_is_boxed(a2, boxes)
         boxes.append(value_box)
         self.possibly_free_vars(boxes)
-        return [value_loc, base_loc, ofs_loc, imm(scale), imm(ofs)]
+        assert _check_imm_arg(ConstInt(base_ofs))
+        return [value_loc, base_loc, ofs_loc, imm(scale), imm(base_ofs)]
     prepare_op_setarrayitem_raw = prepare_op_setarrayitem_gc
 
     def prepare_op_getarrayitem_gc(self, op, fcond):
         a0, a1 = boxes = list(op.getarglist())
-        _, scale, ofs, _, ptr = self._unpack_arraydescr(op.getdescr())
+        _, scale, base_ofs, _, ptr = self._unpack_arraydescr(op.getdescr())
 
         base_loc, base_box  = self._ensure_value_is_boxed(a0, boxes)
         boxes.append(base_box)
@@ -715,7 +716,8 @@
         self.possibly_free_vars(boxes)
         res = self.force_allocate_reg(op.result)
         self.possibly_free_var(op.result)
-        return [res, base_loc, ofs_loc, imm(scale), imm(ofs)]
+        assert _check_imm_arg(ConstInt(base_ofs))
+        return [res, base_loc, ofs_loc, imm(scale), imm(base_ofs)]
 
     prepare_op_getarrayitem_raw = prepare_op_getarrayitem_gc
     prepare_op_getarrayitem_gc_pure = prepare_op_getarrayitem_gc

From noreply at buildbot.pypy.org  Tue Oct 25 11:07:54 2011
From: noreply at buildbot.pypy.org (bivab)
Date: Tue, 25 Oct 2011 11:07:54 +0200 (CEST)
Subject: [pypy-commit] pypy arm-backend-2: fix an error when setting and
 reading float fields from an object with a large offset
Message-ID: <20111025090754.666DD820C7@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: arm-backend-2
Changeset: r48418:0dc87b084c4a
Date: 2011-10-24 14:59 +0200
http://bitbucket.org/pypy/pypy/changeset/0dc87b084c4a/

Log:	fix an error when setting and reading float fields from an object
	with a large offset

diff --git a/pypy/jit/backend/arm/opassembler.py b/pypy/jit/backend/arm/opassembler.py
--- a/pypy/jit/backend/arm/opassembler.py
+++ b/pypy/jit/backend/arm/opassembler.py
@@ -505,10 +505,15 @@
         value_loc, base_loc, ofs, size = arglocs
         if size.value == 8:
             assert value_loc.is_vfp_reg()
+            # vstr only supports imm offsets
+            # so if the ofset is too large we add it to the base and use an
+            # offset of 0
             if ofs.is_reg():
+                self.mc.ADD_rr(r.ip.value, base_loc.value, ofs.value)
                 base_loc = r.ip
                 ofs = imm(0)
-                self.mc.ADD_rr(r.ip.value, base_loc.value, ofs.value)
+            else:
+                assert ofs.value % 4 == 0
             self.mc.VSTR(value_loc.value, base_loc.value, ofs.value)
         elif size.value == 4:
             if ofs.is_imm():
@@ -535,10 +540,15 @@
         base_loc, ofs, res, size = arglocs
         if size.value == 8:
             assert res.is_vfp_reg()
+            # vldr only supports imm offsets
+            # so if the ofset is too large we add it to the base and use an
+            # offset of 0
             if ofs.is_reg():
+                self.mc.ADD_rr(r.ip.value, base_loc.value, ofs.value)
                 base_loc = r.ip
                 ofs = imm(0)
-                self.mc.ADD_rr(r.ip.value, base_loc.value, ofs.value)
+            else:
+                assert ofs.value % 4 == 0
             self.mc.VLDR(res.value, base_loc.value, ofs.value)
         elif size.value == 4:
             if ofs.is_imm():
diff --git a/pypy/jit/backend/arm/test/test_runner.py b/pypy/jit/backend/arm/test/test_runner.py
--- a/pypy/jit/backend/arm/test/test_runner.py
+++ b/pypy/jit/backend/arm/test/test_runner.py
@@ -1,6 +1,8 @@
 import py
 from pypy.jit.backend.arm.runner import ArmCPU
-from pypy.jit.backend.test.runner_test import LLtypeBackendTest
+from pypy.jit.backend.test.runner_test import LLtypeBackendTest, \
+                                                boxfloat, \
+                                                constfloat
 from pypy.jit.backend.arm.test.support import skip_unless_arm
 from pypy.jit.metainterp.history import (AbstractFailDescr,
                                          AbstractDescr,
@@ -12,7 +14,7 @@
                                          ConstObj, BoxFloat, ConstFloat)
 from pypy.jit.metainterp.resoperation import ResOperation, rop
 from pypy.jit.tool.oparser import parse
-from pypy.rpython.lltypesystem import lltype, llmemory
+from pypy.rpython.lltypesystem import lltype, llmemory, rclass
 from pypy.rpython.annlowlevel import llhelper
 from pypy.jit.codewriter.effectinfo import EffectInfo
 
@@ -117,3 +119,69 @@
 
     def test_cond_call_gc_wb_array_card_marking_fast_path(self):
         py.test.skip('ignore this fast path for now')
+
+    SFloat = lltype.GcForwardReference()
+    SFloat.become(lltype.GcStruct('SFloat', ('parent', rclass.OBJECT),
+          ('v1', lltype.Signed), ('v2', lltype.Signed),
+          ('v3', lltype.Signed), ('v4', lltype.Signed),
+          ('v5', lltype.Signed), ('v6', lltype.Signed),
+          ('v7', lltype.Signed), ('v8', lltype.Signed),
+          ('v9', lltype.Signed), ('v10', lltype.Signed),
+          ('v11', lltype.Signed), ('v12', lltype.Signed),
+          ('v13', lltype.Signed), ('v14', lltype.Signed),
+          ('v15', lltype.Signed), ('v16', lltype.Signed),
+          ('v17', lltype.Signed), ('v18', lltype.Signed),
+          ('v19', lltype.Signed), ('v20', lltype.Signed),
+
+          ('w1', lltype.Signed), ('w2', lltype.Signed),
+          ('w3', lltype.Signed), ('w4', lltype.Signed),
+          ('w5', lltype.Signed), ('w6', lltype.Signed),
+          ('w7', lltype.Signed), ('w8', lltype.Signed),
+          ('w9', lltype.Signed), ('w10', lltype.Signed),
+          ('w11', lltype.Signed), ('w12', lltype.Signed),
+          ('w13', lltype.Signed), ('w14', lltype.Signed),
+          ('w15', lltype.Signed), ('w16', lltype.Signed),
+          ('w17', lltype.Signed), ('w18', lltype.Signed),
+          ('w19', lltype.Signed), ('w20', lltype.Signed),
+
+          ('x1', lltype.Signed), ('x2', lltype.Signed),
+          ('x3', lltype.Signed), ('x4', lltype.Signed),
+          ('x5', lltype.Signed), ('x6', lltype.Signed),
+          ('x7', lltype.Signed), ('x8', lltype.Signed),
+          ('x9', lltype.Signed), ('x10', lltype.Signed),
+          ('x11', lltype.Signed), ('x12', lltype.Signed),
+          ('x13', lltype.Signed), ('x14', lltype.Signed),
+          ('x15', lltype.Signed), ('x16', lltype.Signed),
+          ('x17', lltype.Signed), ('x18', lltype.Signed),
+          ('x19', lltype.Signed), ('x20', lltype.Signed),
+
+          ('y1', lltype.Signed), ('y2', lltype.Signed),
+          ('y3', lltype.Signed), ('y4', lltype.Signed),
+          ('y5', lltype.Signed), ('y6', lltype.Signed),
+          ('y7', lltype.Signed), ('y8', lltype.Signed),
+          ('y9', lltype.Signed), ('y10', lltype.Signed),
+          ('y11', lltype.Signed), ('y12', lltype.Signed),
+          ('y13', lltype.Signed), ('y14', lltype.Signed),
+          ('y15', lltype.Signed), ('y16', lltype.Signed),
+          ('y17', lltype.Signed), ('y18', lltype.Signed),
+          ('y19', lltype.Signed), ('y20', lltype.Signed),
+          ('float', lltype.Float)))
+
+    T = lltype.GcStruct('T', ('parent', SFloat),
+                             ('next', lltype.Ptr(SFloat)))
+    def test_float_field(self):
+        if not self.cpu.supports_floats:
+            py.test.skip('requires floats')
+        floatdescr = self.cpu.fielddescrof(self.SFloat, 'float')
+        t_box, T_box = self.alloc_instance(self.T)
+        self.execute_operation(rop.SETFIELD_GC, [t_box, boxfloat(3.4)],
+                               'void', descr=floatdescr)
+        res = self.execute_operation(rop.GETFIELD_GC, [t_box],
+                                     'float', descr=floatdescr)
+        assert res.getfloat() == 3.4
+        #
+        self.execute_operation(rop.SETFIELD_GC, [t_box, constfloat(-3.6)],
+                               'void', descr=floatdescr)
+        res = self.execute_operation(rop.GETFIELD_GC, [t_box],
+                                     'float', descr=floatdescr)
+        assert res.getfloat() == -3.6

From noreply at buildbot.pypy.org  Tue Oct 25 11:07:55 2011
From: noreply at buildbot.pypy.org (bivab)
Date: Tue, 25 Oct 2011 11:07:55 +0200 (CEST)
Subject: [pypy-commit] pypy arm-backend-2: forgot to add these methods
Message-ID: <20111025090755.8FD1E820C7@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: arm-backend-2
Changeset: r48419:0868f9fb2793
Date: 2011-10-24 15:58 +0200
http://bitbucket.org/pypy/pypy/changeset/0868f9fb2793/

Log:	forgot to add these methods

diff --git a/pypy/jit/backend/arm/regalloc.py b/pypy/jit/backend/arm/regalloc.py
--- a/pypy/jit/backend/arm/regalloc.py
+++ b/pypy/jit/backend/arm/regalloc.py
@@ -850,6 +850,8 @@
         resloc = self.force_allocate_reg(op.result)
         self.possibly_free_var(op.result)
         return [argloc, resloc]
+    prepare_op_cast_ptr_to_int = prepare_op_same_as
+    prepare_op_cast_int_to_ptr = prepare_op_same_as
 
     def prepare_op_new(self, op, fcond):
         gc_ll_descr = self.assembler.cpu.gc_ll_descr

From noreply at buildbot.pypy.org  Tue Oct 25 11:07:56 2011
From: noreply at buildbot.pypy.org (bivab)
Date: Tue, 25 Oct 2011 11:07:56 +0200 (CEST)
Subject: [pypy-commit] pypy arm-backend-2: implement merging of comparison
 operations with following guards
Message-ID: <20111025090756.C6353820C7@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: arm-backend-2
Changeset: r48420:f29ee3944161
Date: 2011-10-24 17:14 +0200
http://bitbucket.org/pypy/pypy/changeset/f29ee3944161/

Log:	implement merging of comparison operations with following guards

diff --git a/pypy/jit/backend/arm/assembler.py b/pypy/jit/backend/arm/assembler.py
--- a/pypy/jit/backend/arm/assembler.py
+++ b/pypy/jit/backend/arm/assembler.py
@@ -8,7 +8,10 @@
 from pypy.jit.backend.arm.arch import WORD, FUNC_ALIGN, PC_OFFSET, N_REGISTERS_SAVED_BY_MALLOC
 from pypy.jit.backend.arm.codebuilder import ARMv7Builder, OverwritingBuilder
 from pypy.jit.backend.arm.regalloc import (Regalloc, ARMFrameManager, ARMv7RegisterMananger,
-                                                    _check_imm_arg, TempInt, TempPtr)
+                                        _check_imm_arg, TempInt,
+                                        TempPtr,
+                                        operations as regalloc_operations,
+                                        operations_with_guard as regalloc_operations_with_guard)
 from pypy.jit.backend.arm.jump import remap_frame_layout
 from pypy.jit.backend.llsupport.regalloc import compute_vars_longevity, TempBox
 from pypy.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper
@@ -791,34 +794,45 @@
                 regalloc.possibly_free_vars_for_op(op)
             elif self.can_merge_with_next_guard(op, i, operations):
                 regalloc.next_instruction()
-                arglocs = regalloc.operations_with_guard[opnum](regalloc, op,
+                arglocs = regalloc_operations_with_guard[opnum](regalloc, op,
                                         operations[i+1], fcond)
-                fcond = self.operations_with_guard[opnum](self, op,
+                fcond = asm_operations_with_guard[opnum](self, op,
                                         operations[i+1], arglocs, regalloc, fcond)
             elif not we_are_translated() and op.getopnum() == -124:
                 regalloc.prepare_force_spill(op, fcond)
             else:
-                arglocs = regalloc.operations[opnum](regalloc, op, fcond)
+                arglocs = regalloc_operations[opnum](regalloc, op, fcond)
                 if arglocs is not None:
-                    fcond = self.operations[opnum](self, op, arglocs, regalloc, fcond)
+                    fcond = asm_operations[opnum](self, op, arglocs, regalloc, fcond)
             if op.result:
                 regalloc.possibly_free_var(op.result)
             regalloc.possibly_free_vars_for_op(op)
             regalloc._check_invariants()
 
+    # from ../x86/regalloc.py
     def can_merge_with_next_guard(self, op, i, operations):
-        num = op.getopnum()
-        if num == rop.CALL_MAY_FORCE or num == rop.CALL_ASSEMBLER:
+        if (op.getopnum() == rop.CALL_MAY_FORCE or
+            op.getopnum() == rop.CALL_ASSEMBLER or
+            op.getopnum() == rop.CALL_RELEASE_GIL):
             assert operations[i + 1].getopnum() == rop.GUARD_NOT_FORCED
             return True
-        if num == rop.INT_MUL_OVF or num == rop.INT_ADD_OVF or num == rop.INT_SUB_OVF:
-            opnum = operations[i + 1].getopnum()
-            assert opnum  == rop.GUARD_OVERFLOW or opnum == rop.GUARD_NO_OVERFLOW
-            return True
-        if num == rop.CALL_RELEASE_GIL:
-            return True
-        return False
-
+        if not op.is_comparison():
+            if op.is_ovf():
+                if (operations[i + 1].getopnum() != rop.GUARD_NO_OVERFLOW and
+                    operations[i + 1].getopnum() != rop.GUARD_OVERFLOW):
+                    not_implemented("int_xxx_ovf not followed by "
+                                    "guard_(no)_overflow")
+                return True
+            return False
+        if (operations[i + 1].getopnum() != rop.GUARD_TRUE and
+            operations[i + 1].getopnum() != rop.GUARD_FALSE):
+            return False
+        if operations[i + 1].getarg(0) is not op.result:
+            return False
+        if (self._regalloc.longevity[op.result][1] > i + 1 or
+            op.result in operations[i + 1].getfailargs()):
+            return False
+        return True
 
     def _insert_checks(self, mc=None):
         if not we_are_translated():
@@ -1148,36 +1162,29 @@
         else:
             return 0
 
-def make_operation_list():
-    def notimplemented(self, op, arglocs, regalloc, fcond):
-        raise NotImplementedError, op
+def notimplemented(self, op, arglocs, regalloc, fcond):
+    raise NotImplementedError, op
+def notimplemented_with_guard(self, op, guard_op, arglocs, regalloc, fcond):
+    raise NotImplementedError, op
 
-    operations = [None] * (rop._LAST+1)
-    for key, value in rop.__dict__.items():
-        key = key.lower()
-        if key.startswith('_'):
-            continue
-        methname = 'emit_op_%s' % key
-        if hasattr(AssemblerARM, methname):
-            func = getattr(AssemblerARM, methname).im_func
-        else:
-            func = notimplemented
-        operations[value] = func
-    return operations
+asm_operations = [notimplemented] * (rop._LAST + 1)
+asm_operations_with_guard = [notimplemented_with_guard] * (rop._LAST + 1)
 
-def make_guard_operation_list():
-    def notimplemented(self, op, guard_op, arglocs, regalloc, fcond):
-        raise NotImplementedError, op
-    guard_operations = [notimplemented] * rop._LAST
-    for key, value in rop.__dict__.items():
-        key = key.lower()
-        if key.startswith('_'):
-            continue
-        methname = 'emit_guard_%s' % key
-        if hasattr(AssemblerARM, methname):
-            func = getattr(AssemblerARM, methname).im_func
-            guard_operations[value] = func
-    return guard_operations
+for key, value in rop.__dict__.items():
+    key = key.lower()
+    if key.startswith('_'):
+        continue
+    methname = 'emit_op_%s' % key
+    if hasattr(AssemblerARM, methname):
+        func = getattr(AssemblerARM, methname).im_func
+        asm_operations[value] = func
 
-AssemblerARM.operations = make_operation_list()
-AssemblerARM.operations_with_guard = make_guard_operation_list()
+for key, value in rop.__dict__.items():
+    key = key.lower()
+    if key.startswith('_'):
+        continue
+    methname = 'emit_guard_%s' % key
+    if hasattr(AssemblerARM, methname):
+        func = getattr(AssemblerARM, methname).im_func
+        asm_operations_with_guard[value] = func
+
diff --git a/pypy/jit/backend/arm/helper/assembler.py b/pypy/jit/backend/arm/helper/assembler.py
--- a/pypy/jit/backend/arm/helper/assembler.py
+++ b/pypy/jit/backend/arm/helper/assembler.py
@@ -4,6 +4,7 @@
 from pypy.jit.backend.arm.codebuilder import AbstractARMv7Builder
 from pypy.jit.metainterp.history import ConstInt, BoxInt, FLOAT
 from pypy.rlib.rarithmetic import r_uint, r_longlong, intmask
+from pypy.jit.metainterp.resoperation import rop
 
 def gen_emit_op_unary_cmp(true_cond, false_cond):
     def f(self, op, arglocs, regalloc, fcond):
@@ -55,6 +56,24 @@
         return fcond
     return f
 
+def gen_emit_cmp_op_guard(condition):
+    def f(self, op, guard, arglocs, regalloc, fcond):
+        l0 = arglocs[0]
+        l1 = arglocs[1]
+
+        inv = c.get_opposite_of(condition)
+        if l1.is_imm():
+            self.mc.CMP_ri(l0.value, imm=l1.getint(), cond=fcond)
+        else:
+            self.mc.CMP_rr(l0.value, l1.value, cond=fcond)
+        guard_opnum = guard.getopnum()
+        cond = condition
+        if guard_opnum == rop.GUARD_FALSE:
+            cond = inv
+        self._emit_guard(guard, arglocs[2:], cond)
+        return fcond
+    return f
+
 def gen_emit_float_op(opname):
     op_rr = getattr(AbstractARMv7Builder, opname)
     def f(self, op, arglocs, regalloc, fcond):
diff --git a/pypy/jit/backend/arm/helper/regalloc.py b/pypy/jit/backend/arm/helper/regalloc.py
--- a/pypy/jit/backend/arm/helper/regalloc.py
+++ b/pypy/jit/backend/arm/helper/regalloc.py
@@ -95,7 +95,7 @@
     return f
 
 def prepare_cmp_op(name=None, inverse=False):
-    def f(self, op, fcond):
+    def f(self, op, guard_op, fcond):
         assert fcond is not None
         boxes = list(op.getarglist())
         if not inverse:
@@ -114,9 +114,14 @@
             l1, box = self._ensure_value_is_boxed(arg1, forbidden_vars=boxes)
             boxes.append(box)
         self.possibly_free_vars(boxes)
-        res = self.force_allocate_reg(op.result)
-        self.possibly_free_var(op.result)
-        return [l0, l1, res]
+        if guard_op is None:
+            res = self.force_allocate_reg(op.result)
+            self.possibly_free_var(op.result)
+            return [l0, l1, res]
+        else:
+            args = self._prepare_guard(guard_op, [l0, l1])
+            self.possibly_free_vars(guard_op.getfailargs())
+            return args
     if name:
         f.__name__ = name
     return f
diff --git a/pypy/jit/backend/arm/opassembler.py b/pypy/jit/backend/arm/opassembler.py
--- a/pypy/jit/backend/arm/opassembler.py
+++ b/pypy/jit/backend/arm/opassembler.py
@@ -11,6 +11,7 @@
                                                     gen_emit_op_unary_cmp,
                                                     gen_emit_op_ri,
                                                     gen_emit_cmp_op,
+                                                    gen_emit_cmp_op_guard,
                                                     gen_emit_float_op,
                                                     gen_emit_float_cmp_op,
                                                     gen_emit_unary_float_op, 
@@ -139,11 +140,28 @@
     emit_op_uint_lt = gen_emit_cmp_op(c.HI)
     emit_op_uint_ge = gen_emit_cmp_op(c.LS)
 
+    emit_op_ptr_eq = emit_op_int_eq
+    emit_op_ptr_ne = emit_op_int_ne
+
+    emit_guard_int_lt = gen_emit_cmp_op_guard(c.LT)
+    emit_guard_int_le = gen_emit_cmp_op_guard(c.LE)
+    emit_guard_int_eq = gen_emit_cmp_op_guard(c.EQ)
+    emit_guard_int_ne = gen_emit_cmp_op_guard(c.NE)
+    emit_guard_int_gt = gen_emit_cmp_op_guard(c.GT)
+    emit_guard_int_ge = gen_emit_cmp_op_guard(c.GE)
+
+    emit_guard_uint_le = gen_emit_cmp_op_guard(c.LS)
+    emit_guard_uint_gt = gen_emit_cmp_op_guard(c.HI)
+
+    emit_guard_uint_lt = gen_emit_cmp_op_guard(c.HI)
+    emit_guard_uint_ge = gen_emit_cmp_op_guard(c.LS)
+
+    emit_guard_ptr_eq = emit_guard_int_eq
+    emit_guard_ptr_ne = emit_guard_int_ne
+
     emit_op_int_add_ovf = emit_op_int_add
     emit_op_int_sub_ovf = emit_op_int_sub
 
-    emit_op_ptr_eq = emit_op_int_eq
-    emit_op_ptr_ne = emit_op_int_ne
 
 
 class UnaryIntOpAssembler(object):
diff --git a/pypy/jit/backend/arm/regalloc.py b/pypy/jit/backend/arm/regalloc.py
--- a/pypy/jit/backend/arm/regalloc.py
+++ b/pypy/jit/backend/arm/regalloc.py
@@ -425,11 +425,28 @@
     prepare_op_uint_lt = prepare_cmp_op('uint_lt', inverse=True)
     prepare_op_uint_ge = prepare_cmp_op('uint_ge', inverse=True)
 
+    prepare_op_ptr_eq = prepare_op_int_eq
+    prepare_op_ptr_ne = prepare_op_int_ne
+
+    prepare_guard_int_lt = prepare_cmp_op('guard_int_lt')
+    prepare_guard_int_le = prepare_cmp_op('guard_int_le')
+    prepare_guard_int_eq = prepare_cmp_op('guard_int_eq')
+    prepare_guard_int_ne = prepare_cmp_op('guard_int_ne')
+    prepare_guard_int_gt = prepare_cmp_op('guard_int_gt')
+    prepare_guard_int_ge = prepare_cmp_op('guard_int_ge')
+
+    prepare_guard_uint_le = prepare_cmp_op('guard_uint_le')
+    prepare_guard_uint_gt = prepare_cmp_op('guard_uint_gt')
+
+    prepare_guard_uint_lt = prepare_cmp_op('guard_uint_lt', inverse=True)
+    prepare_guard_uint_ge = prepare_cmp_op('guard_uint_ge', inverse=True)
+
+    prepare_guard_ptr_eq = prepare_guard_int_eq
+    prepare_guard_ptr_ne = prepare_guard_int_ne
+
     prepare_op_int_add_ovf = prepare_op_int_add
     prepare_op_int_sub_ovf = prepare_op_int_sub
 
-    prepare_op_ptr_eq = prepare_op_int_eq
-    prepare_op_ptr_ne = prepare_op_int_ne
 
     prepare_op_int_is_true = prepare_op_unary_cmp('int_is_true')
     prepare_op_int_is_zero = prepare_op_unary_cmp('int_is_zero')
@@ -1186,36 +1203,32 @@
         self.force_spill_var(op.getarg(0))
         return []
 
-def make_operation_list():
-    def notimplemented(self, op, fcond):
-        raise NotImplementedError, op
+def add_none_argument(fn):
+    return lambda self, op, fcond: fn(self, op, None, fcond)
 
-    operations = [None] * (rop._LAST+1)
-    for key, value in rop.__dict__.items():
-        key = key.lower()
-        if key.startswith('_'):
-            continue
-        methname = 'prepare_op_%s' % key
-        if hasattr(Regalloc, methname):
-            func = getattr(Regalloc, methname).im_func
-        else:
-            func = notimplemented
+def notimplemented(self, op, fcond):
+    raise NotImplementedError, op
+def notimplemented_with_guard(self, op, guard_op, fcond):
+    raise NotImplementedError, op
+
+operations = [notimplemented] * (rop._LAST + 1)
+operations_with_guard = [notimplemented_with_guard] * (rop._LAST + 1)
+
+for key, value in rop.__dict__.items():
+    key = key.lower()
+    if key.startswith('_'):
+        continue
+    methname = 'prepare_op_%s' % key
+    if hasattr(Regalloc, methname):
+        func = getattr(Regalloc, methname).im_func
         operations[value] = func
-    return operations
 
-def make_guard_operation_list():
-    def notimplemented(self, op, guard_op, fcond):
-        raise NotImplementedError, op
-    guard_operations = [notimplemented] * rop._LAST
-    for key, value in rop.__dict__.items():
-        key = key.lower()
-        if key.startswith('_'):
-            continue
-        methname = 'prepare_guard_%s' % key
-        if hasattr(Regalloc, methname):
-            func = getattr(Regalloc, methname).im_func
-            guard_operations[value] = func
-    return guard_operations
-
-Regalloc.operations = make_operation_list()
-Regalloc.operations_with_guard = make_guard_operation_list()
+for key, value in rop.__dict__.items():
+    key = key.lower()
+    if key.startswith('_'):
+        continue
+    methname = 'prepare_guard_%s' % key
+    if hasattr(Regalloc, methname):
+        func = getattr(Regalloc, methname).im_func
+        operations_with_guard[value] = func
+        operations[value] = add_none_argument(func)

From noreply at buildbot.pypy.org  Tue Oct 25 11:07:58 2011
From: noreply at buildbot.pypy.org (bivab)
Date: Tue, 25 Oct 2011 11:07:58 +0200 (CEST)
Subject: [pypy-commit] pypy arm-backend-2: merge guards with cmp ops for
	floats
Message-ID: <20111025090758.003F0820C7@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: arm-backend-2
Changeset: r48421:7b6405557495
Date: 2011-10-25 11:07 +0200
http://bitbucket.org/pypy/pypy/changeset/7b6405557495/

Log:	merge guards with cmp ops for floats

diff --git a/pypy/jit/backend/arm/helper/assembler.py b/pypy/jit/backend/arm/helper/assembler.py
--- a/pypy/jit/backend/arm/helper/assembler.py
+++ b/pypy/jit/backend/arm/helper/assembler.py
@@ -100,6 +100,21 @@
         return fcond
     return f
 
+def gen_emit_float_cmp_op_guard(guard_cond):
+    def f(self, op, guard, arglocs, regalloc, fcond):
+        arg1 = arglocs[0]
+        arg2 = arglocs[1]
+        inv = c.get_opposite_of(guard_cond)
+        self.mc.VCMP(arg1.value, arg2.value)
+        self.mc.VMRS(cond=fcond)
+        cond = guard_cond
+        guard_opnum = guard.getopnum()
+        if guard_opnum == rop.GUARD_FALSE:
+            cond = inv
+        self._emit_guard(guard, arglocs[2:], cond)
+        return fcond
+    return f
+
 class saved_registers(object):
     def __init__(self, assembler, regs_to_save, vfp_regs_to_save=None, regalloc=None):
         assert regalloc is None
diff --git a/pypy/jit/backend/arm/helper/regalloc.py b/pypy/jit/backend/arm/helper/regalloc.py
--- a/pypy/jit/backend/arm/helper/regalloc.py
+++ b/pypy/jit/backend/arm/helper/regalloc.py
@@ -56,21 +56,42 @@
         f.__name__ = name
     return f
 
-def prepare_float_op(name=None, base=True, float_result=True):
-    def f(self, op, fcond):
-        locs = []
-        loc1, box1 = self._ensure_value_is_boxed(op.getarg(0))
-        locs.append(loc1)
-        if base:
-            loc2, box2 = self._ensure_value_is_boxed(op.getarg(1))
-            locs.append(loc2)
-            self.possibly_free_var(box2)
-        self.possibly_free_var(box1)
-        res = self.force_allocate_reg(op.result)
-        assert float_result == (op.result.type == FLOAT)
-        self.possibly_free_var(op.result)
-        locs.append(res)
-        return locs
+def prepare_float_op(name=None, base=True, float_result=True, guard=False):
+    if guard:
+        def f(self, op, guard_op, fcond):
+            locs = []
+            loc1, box1 = self._ensure_value_is_boxed(op.getarg(0))
+            locs.append(loc1)
+            if base:
+                loc2, box2 = self._ensure_value_is_boxed(op.getarg(1))
+                locs.append(loc2)
+                self.possibly_free_var(box2)
+            self.possibly_free_var(box1)
+            if guard_op is None:
+                res = self.force_allocate_reg(op.result)
+                assert float_result == (op.result.type == FLOAT)
+                self.possibly_free_var(op.result)
+                locs.append(res)
+                return locs
+            else:
+                args = self._prepare_guard(guard_op, locs)
+                self.possibly_free_vars(guard_op.getfailargs())
+                return args
+    else:
+        def f(self, op, fcond):
+            locs = []
+            loc1, box1 = self._ensure_value_is_boxed(op.getarg(0))
+            locs.append(loc1)
+            if base:
+                loc2, box2 = self._ensure_value_is_boxed(op.getarg(1))
+                locs.append(loc2)
+                self.possibly_free_var(box2)
+            self.possibly_free_var(box1)
+            res = self.force_allocate_reg(op.result)
+            assert float_result == (op.result.type == FLOAT)
+            self.possibly_free_var(op.result)
+            locs.append(res)
+            return locs
     if name:
         f.__name__ = name
     return f
diff --git a/pypy/jit/backend/arm/opassembler.py b/pypy/jit/backend/arm/opassembler.py
--- a/pypy/jit/backend/arm/opassembler.py
+++ b/pypy/jit/backend/arm/opassembler.py
@@ -14,6 +14,7 @@
                                                     gen_emit_cmp_op_guard,
                                                     gen_emit_float_op,
                                                     gen_emit_float_cmp_op,
+                                                    gen_emit_float_cmp_op_guard,
                                                     gen_emit_unary_float_op, 
                                                     saved_registers,
                                                     count_reg_args)
@@ -1157,6 +1158,13 @@
     emit_op_float_gt = gen_emit_float_cmp_op(c.GT)
     emit_op_float_ge = gen_emit_float_cmp_op(c.GE)
 
+    emit_guard_float_lt = gen_emit_float_cmp_op_guard(c.VFP_LT)
+    emit_guard_float_le = gen_emit_float_cmp_op_guard(c.VFP_LE)
+    emit_guard_float_eq = gen_emit_float_cmp_op_guard(c.EQ)
+    emit_guard_float_ne = gen_emit_float_cmp_op_guard(c.NE)
+    emit_guard_float_gt = gen_emit_float_cmp_op_guard(c.GT)
+    emit_guard_float_ge = gen_emit_float_cmp_op_guard(c.GE)
+
     def emit_op_cast_float_to_int(self, op, arglocs, regalloc, fcond):
         arg, temp, res = arglocs
         self.mc.VCVT_float_to_int(temp.value, arg.value)
diff --git a/pypy/jit/backend/arm/regalloc.py b/pypy/jit/backend/arm/regalloc.py
--- a/pypy/jit/backend/arm/regalloc.py
+++ b/pypy/jit/backend/arm/regalloc.py
@@ -1143,18 +1143,25 @@
         ofs += descr.fielddescr.offset
         return ofs, itemsize, fieldsize, sign
 
-    prepare_op_float_add = prepare_float_op(name='float_add')
-    prepare_op_float_sub = prepare_float_op(name='float_sub')
-    prepare_op_float_mul = prepare_float_op(name='float_mul')
-    prepare_op_float_truediv = prepare_float_op(name='float_truediv')
-    prepare_op_float_lt = prepare_float_op(float_result=False, name='float_lt')
-    prepare_op_float_le = prepare_float_op(float_result=False, name='float_le')
-    prepare_op_float_eq = prepare_float_op(float_result=False, name='float_eq')
-    prepare_op_float_ne = prepare_float_op(float_result=False, name='float_ne')
-    prepare_op_float_gt = prepare_float_op(float_result=False, name='float_gt')
-    prepare_op_float_ge = prepare_float_op(float_result=False, name='float_ge')
-    prepare_op_float_neg = prepare_float_op(base=False, name='float_neg')
-    prepare_op_float_abs = prepare_float_op(base=False, name='float_abs')
+    prepare_op_float_add = prepare_float_op(name='prepare_op_float_add')
+    prepare_op_float_sub = prepare_float_op(name='prepare_op_float_sub')
+    prepare_op_float_mul = prepare_float_op(name='prepare_op_float_mul')
+    prepare_op_float_truediv = prepare_float_op(name='prepare_op_float_truediv')
+    prepare_op_float_lt = prepare_float_op(float_result=False, name='prepare_op_float_lt')
+    prepare_op_float_le = prepare_float_op(float_result=False, name='prepare_op_float_le')
+    prepare_op_float_eq = prepare_float_op(float_result=False, name='prepare_op_float_eq')
+    prepare_op_float_ne = prepare_float_op(float_result=False, name='prepare_op_float_ne')
+    prepare_op_float_gt = prepare_float_op(float_result=False, name='prepare_op_float_gt')
+    prepare_op_float_ge = prepare_float_op(float_result=False, name='prepare_op_float_ge')
+    prepare_op_float_neg = prepare_float_op(base=False, name='prepare_op_float_neg')
+    prepare_op_float_abs = prepare_float_op(base=False, name='prepare_op_float_abs')
+
+    prepare_guard_float_lt = prepare_float_op(guard=True, float_result=False, name='prepare_guard_float_lt')
+    prepare_guard_float_le = prepare_float_op(guard=True, float_result=False, name='prepare_guard_float_le')
+    prepare_guard_float_eq = prepare_float_op(guard=True, float_result=False, name='prepare_guard_float_eq')
+    prepare_guard_float_ne = prepare_float_op(guard=True, float_result=False, name='prepare_guard_float_ne')
+    prepare_guard_float_gt = prepare_float_op(guard=True, float_result=False, name='prepare_guard_float_gt')
+    prepare_guard_float_ge = prepare_float_op(guard=True, float_result=False, name='prepare_guard_float_ge')
 
     def prepare_op_math_sqrt(self, op, fcond):
         loc, box = self._ensure_value_is_boxed(op.getarg(1))

From noreply at buildbot.pypy.org  Tue Oct 25 11:26:03 2011
From: noreply at buildbot.pypy.org (hager)
Date: Tue, 25 Oct 2011 11:26:03 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: Implemented UNICODEGETITEM,
 UNICODESETITEM, UNICODELEN.
Message-ID: <20111025092603.45CD4820C7@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r48422:ef1c13357210
Date: 2011-10-25 11:25 +0200
http://bitbucket.org/pypy/pypy/changeset/ef1c13357210/

Log:	Implemented UNICODEGETITEM, UNICODESETITEM, UNICODELEN.

diff --git a/pypy/jit/backend/ppc/ppcgen/opassembler.py b/pypy/jit/backend/ppc/ppcgen/opassembler.py
--- a/pypy/jit/backend/ppc/ppcgen/opassembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/opassembler.py
@@ -398,5 +398,40 @@
         self.mc.load_imm(r.r0, basesize.value)
         self.mc.stbx(value_loc.value, base_loc.value, r.r0.value)
 
+    emit_unicodelen = emit_strlen
+
+    # XXX 64 bit adjustment
+    def emit_unicodegetitem(self, op, arglocs, regalloc):
+        res, base_loc, ofs_loc, scale, basesize, itemsize = arglocs
+
+        # XXX arrrrgh, why does PPC not have an SLWI instruction ?
+        self.mc.li(r.r0.value, scale.value)
+        self.mc.slw(ofs_loc.value, ofs_loc.value, r.r0.value)
+        self.mc.add(res.value, base_loc.value, ofs_loc.value)
+        self.mc.li(r.r0.value, basesize.value)
+
+        if scale.value == 2:
+            self.mc.lwzx(res.value, res.value, r.r0.value)
+        elif scale.value == 1:
+            self.mc.lhzx(res.value, res.value, r.r0.value)
+        else:
+            assert 0, itemsize.value
+
+    # XXX 64 bit adjustment
+    def emit_unicodesetitem(self, op, arglocs, regalloc):
+        value_loc, base_loc, ofs_loc, scale, basesize, itemsize = arglocs
+
+        self.mc.li(r.r0.value, scale.value)
+        self.mc.slw(ofs_loc.value, ofs_loc.value, r.r0.value)
+        self.mc.add(base_loc.value, base_loc.value, ofs_loc.value)
+        self.mc.li(r.r0.value, basesize.value)
+
+        if scale.value == 2:
+            self.mc.stwx(value_loc.value, base_loc.value, r.r0.value)
+        elif scale.value == 1:
+            self.mc.sthx(value_loc.value, base_loc.value, r.r0.value)
+        else:
+            assert 0, itemsize.value
+
     def nop(self):
         self.mc.ori(0, 0, 0)
diff --git a/pypy/jit/backend/ppc/ppcgen/regalloc.py b/pypy/jit/backend/ppc/ppcgen/regalloc.py
--- a/pypy/jit/backend/ppc/ppcgen/regalloc.py
+++ b/pypy/jit/backend/ppc/ppcgen/regalloc.py
@@ -475,6 +475,57 @@
         assert itemsize == 1
         return [value_loc, base_loc, ofs_loc, imm(basesize)]
 
+    def prepare_unicodelen(self, op):
+        l0, box = self._ensure_value_is_boxed(op.getarg(0))
+        boxes = [box]
+        basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE,
+                                         self.cpu.translate_support_code)
+        ofs_box = ConstInt(ofs_length)
+        imm_ofs = _check_imm_arg(ofs_box)
+
+        if imm_ofs:
+            l1 = imm(ofs_length)
+        else:
+            l1, box1 = self._ensure_value_is_boxed(ofs_box, boxes)
+            boxes.append(box1)
+
+        self.possibly_free_vars(boxes)
+        res = self.force_allocate_reg(op.result)
+        self.possibly_free_var(op.result)
+        return [l0, l1, res]
+
+    def prepare_unicodegetitem(self, op):
+        boxes = list(op.getarglist())
+        base_loc, box = self._ensure_value_is_boxed(boxes[0], boxes)
+        boxes.append(box)
+        ofs_loc, box = self._ensure_value_is_boxed(boxes[1], boxes)
+        boxes.append(box)
+        self.possibly_free_vars(boxes)
+
+        res = self.force_allocate_reg(op.result)
+        self.possibly_free_var(op.result)
+
+        basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE,
+                                         self.cpu.translate_support_code)
+        scale = itemsize/2
+        return [res, base_loc, ofs_loc, imm(scale), imm(basesize), imm(itemsize)]
+
+    def prepare_unicodesetitem(self, op):
+        boxes = list(op.getarglist())
+        base_loc, box = self._ensure_value_is_boxed(boxes[0], boxes)
+        boxes.append(box)
+        ofs_loc, box = self._ensure_value_is_boxed(boxes[1], boxes)
+        boxes.append(box)
+        value_loc, box = self._ensure_value_is_boxed(boxes[2], boxes)
+        boxes.append(box)
+
+        self.possibly_free_vars(boxes)
+
+        basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE,
+                                         self.cpu.translate_support_code)
+        scale = itemsize/2
+        return [value_loc, base_loc, ofs_loc, imm(scale), imm(basesize), imm(itemsize)]
+
     # from ../x86/regalloc.py:791
     def _unpack_fielddescr(self, fielddescr):
         assert isinstance(fielddescr, BaseFieldDescr)
diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -1056,6 +1056,9 @@
         r = self.execute_operation(rop.UNICODEGETITEM, [u_box, BoxInt(5)],
                                    'int')
         assert r.value == 0x1234
+        r = self.execute_operation(rop.UNICODEGETITEM, [u_box, BoxInt(4)],
+                                   'int')
+        assert r.value == 0x6F # 0x6F = 'o'
         r = self.execute_operation(rop.UNICODESETITEM, [u_box, BoxInt(4),
                                                         BoxInt(31313)], 'void')
         assert r is None

From noreply at buildbot.pypy.org  Tue Oct 25 11:47:17 2011
From: noreply at buildbot.pypy.org (hager)
Date: Tue, 25 Oct 2011 11:47:17 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: Removed unnecessary operations
 in emit_unicodegetitem and emit_unicodesetitem.
Message-ID: <20111025094717.5085C820C7@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r48423:2aa87669180f
Date: 2011-10-25 11:46 +0200
http://bitbucket.org/pypy/pypy/changeset/2aa87669180f/

Log:	Removed unnecessary operations in emit_unicodegetitem and
	emit_unicodesetitem.

diff --git a/pypy/jit/backend/ppc/ppcgen/opassembler.py b/pypy/jit/backend/ppc/ppcgen/opassembler.py
--- a/pypy/jit/backend/ppc/ppcgen/opassembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/opassembler.py
@@ -408,12 +408,11 @@
         self.mc.li(r.r0.value, scale.value)
         self.mc.slw(ofs_loc.value, ofs_loc.value, r.r0.value)
         self.mc.add(res.value, base_loc.value, ofs_loc.value)
-        self.mc.li(r.r0.value, basesize.value)
 
         if scale.value == 2:
-            self.mc.lwzx(res.value, res.value, r.r0.value)
+            self.mc.lwz(res.value, res.value, basesize.value)
         elif scale.value == 1:
-            self.mc.lhzx(res.value, res.value, r.r0.value)
+            self.mc.lhz(res.value, res.value, basesize.value)
         else:
             assert 0, itemsize.value
 
@@ -424,12 +423,11 @@
         self.mc.li(r.r0.value, scale.value)
         self.mc.slw(ofs_loc.value, ofs_loc.value, r.r0.value)
         self.mc.add(base_loc.value, base_loc.value, ofs_loc.value)
-        self.mc.li(r.r0.value, basesize.value)
 
         if scale.value == 2:
-            self.mc.stwx(value_loc.value, base_loc.value, r.r0.value)
+            self.mc.stw(value_loc.value, base_loc.value, basesize.value)
         elif scale.value == 1:
-            self.mc.sthx(value_loc.value, base_loc.value, r.r0.value)
+            self.mc.sth(value_loc.value, base_loc.value, basesize.value)
         else:
             assert 0, itemsize.value
 

From noreply at buildbot.pypy.org  Tue Oct 25 11:53:57 2011
From: noreply at buildbot.pypy.org (hager)
Date: Tue, 25 Oct 2011 11:53:57 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: Removed unnecessary operations
 from emit_strgetitem and emit_strsetitem.
Message-ID: <20111025095357.5E1AB820C7@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r48424:52d3d714f611
Date: 2011-10-25 11:53 +0200
http://bitbucket.org/pypy/pypy/changeset/52d3d714f611/

Log:	Removed unnecessary operations from emit_strgetitem and
	emit_strsetitem.

diff --git a/pypy/jit/backend/ppc/ppcgen/opassembler.py b/pypy/jit/backend/ppc/ppcgen/opassembler.py
--- a/pypy/jit/backend/ppc/ppcgen/opassembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/opassembler.py
@@ -385,9 +385,7 @@
             self.mc.addi(res.value, base_loc.value, ofs_loc.getint())
         else:
             self.mc.add(res.value, base_loc.value, ofs_loc.value)
-        
-        self.mc.load_imm(r.r0, basesize.value)
-        self.mc.lbzx(res.value, res.value, r.r0.value)
+        self.mc.lbz(res.value, res.value, basesize.value)
 
     def emit_strsetitem(self, op, arglocs, regalloc):
         value_loc, base_loc, ofs_loc, basesize = arglocs
@@ -395,8 +393,7 @@
             self.mc.addi(base_loc.value, base_loc.value, ofs_loc.getint())
         else:
             self.mc.add(base_loc.value, base_loc.value, ofs_loc.value)
-        self.mc.load_imm(r.r0, basesize.value)
-        self.mc.stbx(value_loc.value, base_loc.value, r.r0.value)
+        self.mc.stb(value_loc.value, base_loc.value, basesize.value)
 
     emit_unicodelen = emit_strlen
 

From noreply at buildbot.pypy.org  Tue Oct 25 12:20:33 2011
From: noreply at buildbot.pypy.org (hager)
Date: Tue, 25 Oct 2011 12:20:33 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: Implemented SAME_AS operation.
Message-ID: <20111025102033.64AAE820C7@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r48425:be68ff457064
Date: 2011-10-25 12:19 +0200
http://bitbucket.org/pypy/pypy/changeset/be68ff457064/

Log:	Implemented SAME_AS operation.

diff --git a/pypy/jit/backend/ppc/ppcgen/opassembler.py b/pypy/jit/backend/ppc/ppcgen/opassembler.py
--- a/pypy/jit/backend/ppc/ppcgen/opassembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/opassembler.py
@@ -428,5 +428,9 @@
         else:
             assert 0, itemsize.value
 
+    def emit_same_as(self, op, arglocs, regalloc):
+        argloc, resloc = arglocs
+        self.regalloc_mov(argloc, resloc)
+
     def nop(self):
         self.mc.ori(0, 0, 0)
diff --git a/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py b/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py
--- a/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py
@@ -613,8 +613,7 @@
             value = prev_loc.getint()
             # move immediate value to register
             if loc.is_reg():
-                reg = loc.as_key()
-                self.mc.load_imm(reg, value)
+                self.mc.load_imm(loc, value)
                 return
             # move immediate value to memory
             elif loc.is_stack():
diff --git a/pypy/jit/backend/ppc/ppcgen/regalloc.py b/pypy/jit/backend/ppc/ppcgen/regalloc.py
--- a/pypy/jit/backend/ppc/ppcgen/regalloc.py
+++ b/pypy/jit/backend/ppc/ppcgen/regalloc.py
@@ -526,6 +526,19 @@
         scale = itemsize/2
         return [value_loc, base_loc, ofs_loc, imm(scale), imm(basesize), imm(itemsize)]
 
+    def prepare_same_as(self, op):
+        arg = op.getarg(0)
+        imm_arg = _check_imm_arg(arg)
+        if imm_arg:
+            argloc = self.make_sure_var_in_reg(arg)
+        else:
+            argloc, box = self._ensure_value_is_boxed(arg)
+            self.possibly_free_var(box)
+
+        resloc = self.force_allocate_reg(op.result)
+        self.possibly_free_var(op.result)
+        return [argloc, resloc]
+
     # from ../x86/regalloc.py:791
     def _unpack_fielddescr(self, fielddescr):
         assert isinstance(fielddescr, BaseFieldDescr)
diff --git a/pypy/jit/backend/ppc/ppcgen/test/test_regalloc.py b/pypy/jit/backend/ppc/ppcgen/test/test_regalloc.py
--- a/pypy/jit/backend/ppc/ppcgen/test/test_regalloc.py
+++ b/pypy/jit/backend/ppc/ppcgen/test/test_regalloc.py
@@ -81,12 +81,12 @@
         self.asm.mc = self.builder
 
     def test_immediate_to_reg(self):
-        self.asm.regalloc_mov(imm(5), reg(10))
+        self.asm.regalloc_mov(imm(5), r10)
         big = 2 << 28
-        self.asm.regalloc_mov(imm(big), reg(0))
+        self.asm.regalloc_mov(imm(big), r0)
 
-        exp_instr = [MI("load_imm", 10, 5), 
-                     MI("load_imm", 0, big)]
+        exp_instr = [MI("load_imm", r10, 5), 
+                     MI("load_imm", r0, big)]
         assert self.asm.mc.instrs == exp_instr
 
     def test_immediate_to_mem(self):

From noreply at buildbot.pypy.org  Tue Oct 25 12:20:53 2011
From: noreply at buildbot.pypy.org (bivab)
Date: Tue, 25 Oct 2011 12:20:53 +0200 (CEST)
Subject: [pypy-commit] pypy arm-backend-2: add missing not_implemented
	implementation
Message-ID: <20111025102053.F2973820C7@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: arm-backend-2
Changeset: r48426:fccb38718397
Date: 2011-10-25 12:03 +0200
http://bitbucket.org/pypy/pypy/changeset/fccb38718397/

Log:	add missing not_implemented implementation

diff --git a/pypy/jit/backend/arm/assembler.py b/pypy/jit/backend/arm/assembler.py
--- a/pypy/jit/backend/arm/assembler.py
+++ b/pypy/jit/backend/arm/assembler.py
@@ -1,4 +1,5 @@
 from __future__ import with_statement
+import os
 from pypy.jit.backend.arm.helper.assembler import saved_registers, count_reg_args, \
                                                     decode32, encode32, \
                                                     decode64, encode64
@@ -1162,13 +1163,17 @@
         else:
             return 0
 
-def notimplemented(self, op, arglocs, regalloc, fcond):
+def not_implemented(msg):
+    os.write(2, '[ARM/asm] %s\n' % msg)
+    raise NotImplementedError(msg)
+
+def notimplemented_op(self, op, arglocs, regalloc, fcond):
     raise NotImplementedError, op
-def notimplemented_with_guard(self, op, guard_op, arglocs, regalloc, fcond):
+def notimplemented_op_with_guard(self, op, guard_op, arglocs, regalloc, fcond):
     raise NotImplementedError, op
 
-asm_operations = [notimplemented] * (rop._LAST + 1)
-asm_operations_with_guard = [notimplemented_with_guard] * (rop._LAST + 1)
+asm_operations = [notimplemented_op] * (rop._LAST + 1)
+asm_operations_with_guard = [notimplemented_op_with_guard] * (rop._LAST + 1)
 
 for key, value in rop.__dict__.items():
     key = key.lower()

From noreply at buildbot.pypy.org  Tue Oct 25 12:20:55 2011
From: noreply at buildbot.pypy.org (bivab)
Date: Tue, 25 Oct 2011 12:20:55 +0200 (CEST)
Subject: [pypy-commit] pypy arm-backend-2: fix test
Message-ID: <20111025102055.29859820C7@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: arm-backend-2
Changeset: r48427:2df0dec0a518
Date: 2011-10-25 12:11 +0200
http://bitbucket.org/pypy/pypy/changeset/2df0dec0a518/

Log:	fix test

diff --git a/pypy/jit/backend/arm/test/test_runner.py b/pypy/jit/backend/arm/test/test_runner.py
--- a/pypy/jit/backend/arm/test/test_runner.py
+++ b/pypy/jit/backend/arm/test/test_runner.py
@@ -167,13 +167,13 @@
           ('y19', lltype.Signed), ('y20', lltype.Signed),
           ('float', lltype.Float)))
 
-    T = lltype.GcStruct('T', ('parent', SFloat),
+    TFloat = lltype.GcStruct('TFloat', ('parent', SFloat),
                              ('next', lltype.Ptr(SFloat)))
     def test_float_field(self):
         if not self.cpu.supports_floats:
             py.test.skip('requires floats')
         floatdescr = self.cpu.fielddescrof(self.SFloat, 'float')
-        t_box, T_box = self.alloc_instance(self.T)
+        t_box, T_box = self.alloc_instance(self.TFloat)
         self.execute_operation(rop.SETFIELD_GC, [t_box, boxfloat(3.4)],
                                'void', descr=floatdescr)
         res = self.execute_operation(rop.GETFIELD_GC, [t_box],

From noreply at buildbot.pypy.org  Tue Oct 25 12:27:44 2011
From: noreply at buildbot.pypy.org (cfbolz)
Date: Tue, 25 Oct 2011 12:27:44 +0200 (CEST)
Subject: [pypy-commit] pypy int-tag-untag-as-operations: woops
Message-ID: <20111025102744.D2CD6820C7@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: int-tag-untag-as-operations
Changeset: r48428:0ed5fb78f4be
Date: 2011-10-25 12:27 +0200
http://bitbucket.org/pypy/pypy/changeset/0ed5fb78f4be/

Log:	woops

diff --git a/pypy/jit/metainterp/optimizeopt/intbounds.py b/pypy/jit/metainterp/optimizeopt/intbounds.py
--- a/pypy/jit/metainterp/optimizeopt/intbounds.py
+++ b/pypy/jit/metainterp/optimizeopt/intbounds.py
@@ -311,6 +311,7 @@
         maxbounds = IntBound((-sys.maxint-1) >> 1, sys.maxint >> 1)
         v1.intbound.intersect(maxbounds)
         self.pure(rop.INT_UNTAG, [op.result], op.getarg(0))
+        self.emit_operation(op)
 
     def optimize_INT_UNTAG(self, op):
         v1 = self.getvalue(op.getarg(0))
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -5138,6 +5138,29 @@
         """
         self.optimize_loop(ops, expected, preamble)
 
+    def test_int_tag_remove_overflow_checking2(self):
+        ops = """
+        [i0]
+        i1 = int_lt(i0, 1000)
+        guard_true(i1), []
+        i2 = int_gt(i0, 0)
+        guard_true(i2), []
+        i3 = int_tag_ovf(i0)
+        guard_no_overflow() []
+        i4 = escape(i3)
+        jump(i4)
+        """
+        expected = """
+        [i0]
+        i1 = int_lt(i0, 1000)
+        guard_true(i1), []
+        i2 = int_gt(i0, 0)
+        guard_true(i2), []
+        i3 = int_tag(i0)
+        i4 = escape(i3)
+        jump(i4)
+        """
+        self.optimize_loop(ops, expected)
 
     def test_mul_ovf(self):
         ops = """

From notifications-noreply at bitbucket.org  Tue Oct 25 14:38:59 2011
From: notifications-noreply at bitbucket.org (Bitbucket)
Date: Tue, 25 Oct 2011 12:38:59 -0000
Subject: [pypy-commit] [COMMENT] Pull request #12 for pypy/pypy: Don't lose
 data when doing non-blocking I/O
Message-ID: <20111025123859.11545.80985@bitbucket05.managed.contegix.com>

New comment on pull request:

https://bitbucket.org/pypy/pypy/pull-request/12/dont-lose-data-when-doing-non-blocking-i-o#comment-569

Stefano Rivera (stefanor) said:

> Could you also add tests that run on a translated pypy-c?

Don't think so. Could create one or two tests, showing the type of problem, but all the stream classes need to be tested, and one can't create them all from app-space (or simulate this problem with all the types of stream that one *can* create, from app space).

--
This is a pull request comment notification from bitbucket.org.
You are receiving this either because you are participating
in a pull request, or you are following it.

From notifications-noreply at bitbucket.org  Tue Oct 25 14:42:53 2011
From: notifications-noreply at bitbucket.org (Bitbucket)
Date: Tue, 25 Oct 2011 12:42:53 -0000
Subject: [pypy-commit] [COMMENT] Pull request #12 for pypy/pypy: Don't lose
 data when doing non-blocking I/O
Message-ID: <20111025124253.7080.21037@bitbucket05.managed.contegix.com>

New comment on pull request:

https://bitbucket.org/pypy/pypy/pull-request/12/dont-lose-data-when-doing-non-blocking-i-o#comment-570

Amaury Forgeot d'Arc (amauryfa) said:

In this case, could you investigate module/_io as well?  This is likely to be the only implementation of files in the Python3 world.

--
This is a pull request comment notification from bitbucket.org.
You are receiving this either because you are participating
in a pull request, or you are following it.

From noreply at buildbot.pypy.org  Tue Oct 25 15:16:13 2011
From: noreply at buildbot.pypy.org (hager)
Date: Tue, 25 Oct 2011 15:16:13 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: (arigo,
	hager): merge with branch arm-backend-2
Message-ID: <20111025131613.15483820C7@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r48429:23ce29218564
Date: 2011-10-25 15:15 +0200
http://bitbucket.org/pypy/pypy/changeset/23ce29218564/

Log:	(arigo, hager): merge with branch arm-backend-2

diff too long, truncating to 10000 out of 110919 lines

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -2,6 +2,7 @@
 *.py[co]
 *.sw[po]
 *~
+.*.swp
 
 syntax: regexp
 ^testresult$
@@ -39,6 +40,8 @@
 ^pypy/translator/benchmark/shootout_benchmarks$
 ^pypy/translator/goal/pypy-translation-snapshot$
 ^pypy/translator/goal/pypy-c
+^pypy/translator/goal/pypy-jvm
+^pypy/translator/goal/pypy-jvm.jar
 ^pypy/translator/goal/.+\.exe$
 ^pypy/translator/goal/.+\.dll$
 ^pypy/translator/goal/target.+-c$
@@ -65,6 +68,7 @@
 ^pypy/doc/image/lattice3\.png$
 ^pypy/doc/image/stackless_informal\.png$
 ^pypy/doc/image/parsing_example.+\.png$
+^pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test\.o$
 ^compiled
 ^.git/
 ^release/
diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -1,1 +1,3 @@
 b590cf6de4190623aad9aa698694c22e614d67b9 release-1.5
+b48df0bf4e75b81d98f19ce89d4a7dc3e1dab5e5 benchmarked
+d8ac7d23d3ec5f9a0fa1264972f74a010dbfd07f release-1.6
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -37,22 +37,22 @@
     Armin Rigo
     Maciej Fijalkowski
     Carl Friedrich Bolz
+    Antonio Cuni
     Amaury Forgeot d'Arc
-    Antonio Cuni
     Samuele Pedroni
     Michael Hudson
     Holger Krekel
+    Benjamin Peterson
     Christian Tismer
-    Benjamin Peterson
+    Hakan Ardo
+    Alex Gaynor
     Eric van Riet Paap
-    Anders Chrigstr&#246;m
-    H&#229;kan Ard&#246;
+    Anders Chrigstrom
+    David Schneider
     Richard Emslie
     Dan Villiom Podlaski Christiansen
     Alexander Schremmer
-    Alex Gaynor
-    David Schneider
-    Aureli&#233;n Campeas
+    Aurelien Campeas
     Anders Lehmann
     Camillo Bruni
     Niklaus Haldimann
@@ -63,16 +63,17 @@
     Bartosz Skowron
     Jakub Gustak
     Guido Wesdorp
+    Daniel Roberts
     Adrien Di Mascio
     Laura Creighton
     Ludovic Aubry
     Niko Matsakis
-    Daniel Roberts
     Jason Creighton
-    Jacob Hall&#233;n
+    Jacob Hallen
     Alex Martelli
     Anders Hammarquist
     Jan de Mooij
+    Wim Lavrijsen
     Stephan Diehl
     Michael Foord
     Stefan Schwarzer
@@ -83,9 +84,13 @@
     Alexandre Fayolle
     Marius Gedminas
     Simon Burton
+    Justin Peel
     Jean-Paul Calderone
     John Witulski
+    Lukas Diekmann
+    holger krekel
     Wim Lavrijsen
+    Dario Bertini
     Andreas St&#252;hrk
     Jean-Philippe St. Pierre
     Guido van Rossum
@@ -97,15 +102,16 @@
     Georg Brandl
     Gerald Klix
     Wanja Saatkamp
+    Ronny Pfannschmidt
     Boris Feigin
     Oscar Nierstrasz
-    Dario Bertini
     David Malcolm
     Eugene Oden
     Henry Mason
+    Sven Hager
     Lukas Renggli
+    Ilya Osadchiy
     Guenter Jantzen
-    Ronny Pfannschmidt
     Bert Freudenberg
     Amit Regmi
     Ben Young
@@ -122,8 +128,8 @@
     Jared Grubb
     Karl Bartel
     Gabriel Lavoie
+    Victor Stinner
     Brian Dorsey
-    Victor Stinner
     Stuart Williams
     Toby Watson
     Antoine Pitrou
@@ -134,19 +140,23 @@
     Jonathan David Riehl
     Elmo M&#228;ntynen
     Anders Qvist
-    Beatrice D&#252;ring
+    Beatrice During
     Alexander Sedov
+    Timo Paulssen
+    Corbin Simpson
     Vincent Legoll
+    Romain Guillebert
     Alan McIntyre
-    Romain Guillebert
     Alex Perry
     Jens-Uwe Mager
+    Simon Cross
     Dan Stromberg
-    Lukas Diekmann
+    Guillebert Romain
     Carl Meyer
     Pieter Zieschang
     Alejandro J. Cura
     Sylvain Thenault
+    Christoph Gerum
     Travis Francis Athougies
     Henrik Vendelbo
     Lutz Paelike
@@ -157,6 +167,7 @@
     Miguel de Val Borro
     Ignas Mikalajunas
     Artur Lisiecki
+    Philip Jenvey
     Joshua Gilbert
     Godefroid Chappelle
     Yusei Tahara
@@ -165,26 +176,31 @@
     Gustavo Niemeyer
     William Leslie
     Akira Li
-    Kristj&#225;n Valur J&#243;nsson
+    Kristjan Valur Jonsson
     Bobby Impollonia
+    Michael Hudson-Doyle
     Andrew Thompson
     Anders Sigfridsson
+    Floris Bruynooghe
     Jacek Generowicz
     Dan Colish
-    Sven Hager
     Zooko Wilcox-O Hearn
+    Dan Villiom Podlaski Christiansen
     Anders Hammarquist
+    Chris Lambacher
     Dinu Gherman
     Dan Colish
+    Brett Cannon
     Daniel Neuh&#228;user
     Michael Chermside
     Konrad Delong
     Anna Ravencroft
     Greg Price
     Armin Ronacher
+    Christian Muirhead
     Jim Baker
-    Philip Jenvey
     Rodrigo Ara&#250;jo
+    Romain Guillebert
 
     Heinrich-Heine University, Germany 
     Open End AB (formerly AB Strakt), Sweden
diff --git a/_pytest/__init__.py b/_pytest/__init__.py
--- a/_pytest/__init__.py
+++ b/_pytest/__init__.py
@@ -1,2 +1,2 @@
 #
-__version__ = '2.0.3'
+__version__ = '2.1.0.dev4'
diff --git a/_pytest/assertion.py b/_pytest/assertion.py
deleted file mode 100644
--- a/_pytest/assertion.py
+++ /dev/null
@@ -1,177 +0,0 @@
-"""
-support for presented detailed information in failing assertions.
-"""
-import py
-import sys
-from _pytest.monkeypatch import monkeypatch
-
-def pytest_addoption(parser):
-    group = parser.getgroup("debugconfig")
-    group._addoption('--no-assert', action="store_true", default=False,
-        dest="noassert",
-        help="disable python assert expression reinterpretation."),
-
-def pytest_configure(config):
-    # The _reprcompare attribute on the py.code module is used by
-    # py._code._assertionnew to detect this plugin was loaded and in
-    # turn call the hooks defined here as part of the
-    # DebugInterpreter.
-    m = monkeypatch()
-    config._cleanup.append(m.undo)
-    warn_about_missing_assertion()
-    if not config.getvalue("noassert") and not config.getvalue("nomagic"):
-        def callbinrepr(op, left, right):
-            hook_result = config.hook.pytest_assertrepr_compare(
-                config=config, op=op, left=left, right=right)
-            for new_expl in hook_result:
-                if new_expl:
-                    return '\n~'.join(new_expl)
-        m.setattr(py.builtin.builtins,
-                  'AssertionError', py.code._AssertionError)
-        m.setattr(py.code, '_reprcompare', callbinrepr)
-
-def warn_about_missing_assertion():
-    try:
-        assert False
-    except AssertionError:
-        pass
-    else:
-        sys.stderr.write("WARNING: failing tests may report as passing because "
-        "assertions are turned off!  (are you using python -O?)\n")
-
-# Provide basestring in python3
-try:
-    basestring = basestring
-except NameError:
-    basestring = str
-
-
-def pytest_assertrepr_compare(op, left, right):
-    """return specialised explanations for some operators/operands"""
-    width = 80 - 15 - len(op) - 2 # 15 chars indentation, 1 space around op
-    left_repr = py.io.saferepr(left, maxsize=int(width/2))
-    right_repr = py.io.saferepr(right, maxsize=width-len(left_repr))
-    summary = '%s %s %s' % (left_repr, op, right_repr)
-
-    issequence = lambda x: isinstance(x, (list, tuple))
-    istext = lambda x: isinstance(x, basestring)
-    isdict = lambda x: isinstance(x, dict)
-    isset = lambda x: isinstance(x, set)
-
-    explanation = None
-    try:
-        if op == '==':
-            if istext(left) and istext(right):
-                explanation = _diff_text(left, right)
-            elif issequence(left) and issequence(right):
-                explanation = _compare_eq_sequence(left, right)
-            elif isset(left) and isset(right):
-                explanation = _compare_eq_set(left, right)
-            elif isdict(left) and isdict(right):
-                explanation = _diff_text(py.std.pprint.pformat(left),
-                                         py.std.pprint.pformat(right))
-        elif op == 'not in':
-            if istext(left) and istext(right):
-                explanation = _notin_text(left, right)
-    except py.builtin._sysex:
-        raise
-    except:
-        excinfo = py.code.ExceptionInfo()
-        explanation = ['(pytest_assertion plugin: representation of '
-            'details failed. Probably an object has a faulty __repr__.)',
-            str(excinfo)
-            ]
-
-
-    if not explanation:
-        return None
-
-    # Don't include pageloads of data, should be configurable
-    if len(''.join(explanation)) > 80*8:
-        explanation = ['Detailed information too verbose, truncated']
-
-    return [summary] + explanation
-
-
-def _diff_text(left, right):
-    """Return the explanation for the diff between text
-
-    This will skip leading and trailing characters which are
-    identical to keep the diff minimal.
-    """
-    explanation = []
-    i = 0 # just in case left or right has zero length
-    for i in range(min(len(left), len(right))):
-        if left[i] != right[i]:
-            break
-    if i > 42:
-        i -= 10                 # Provide some context
-        explanation = ['Skipping %s identical '
-                       'leading characters in diff' % i]
-        left = left[i:]
-        right = right[i:]
-    if len(left) == len(right):
-        for i in range(len(left)):
-            if left[-i] != right[-i]:
-                break
-        if i > 42:
-            i -= 10     # Provide some context
-            explanation += ['Skipping %s identical '
-                            'trailing characters in diff' % i]
-            left = left[:-i]
-            right = right[:-i]
-    explanation += [line.strip('\n')
-                    for line in py.std.difflib.ndiff(left.splitlines(),
-                                                     right.splitlines())]
-    return explanation
-
-
-def _compare_eq_sequence(left, right):
-    explanation = []
-    for i in range(min(len(left), len(right))):
-        if left[i] != right[i]:
-            explanation += ['At index %s diff: %r != %r' %
-                            (i, left[i], right[i])]
-            break
-    if len(left) > len(right):
-        explanation += ['Left contains more items, '
-            'first extra item: %s' % py.io.saferepr(left[len(right)],)]
-    elif len(left) < len(right):
-        explanation += ['Right contains more items, '
-            'first extra item: %s' % py.io.saferepr(right[len(left)],)]
-    return explanation # + _diff_text(py.std.pprint.pformat(left),
-                       #             py.std.pprint.pformat(right))
-
-
-def _compare_eq_set(left, right):
-    explanation = []
-    diff_left = left - right
-    diff_right = right - left
-    if diff_left:
-        explanation.append('Extra items in the left set:')
-        for item in diff_left:
-            explanation.append(py.io.saferepr(item))
-    if diff_right:
-        explanation.append('Extra items in the right set:')
-        for item in diff_right:
-            explanation.append(py.io.saferepr(item))
-    return explanation
-
-
-def _notin_text(term, text):
-    index = text.find(term)
-    head = text[:index]
-    tail = text[index+len(term):]
-    correct_text = head + tail
-    diff = _diff_text(correct_text, text)
-    newdiff = ['%s is contained here:' % py.io.saferepr(term, maxsize=42)]
-    for line in diff:
-        if line.startswith('Skipping'):
-            continue
-        if line.startswith('- '):
-            continue
-        if line.startswith('+ '):
-            newdiff.append('  ' + line[2:])
-        else:
-            newdiff.append(line)
-    return newdiff
diff --git a/_pytest/assertion/__init__.py b/_pytest/assertion/__init__.py
new file mode 100644
--- /dev/null
+++ b/_pytest/assertion/__init__.py
@@ -0,0 +1,128 @@
+"""
+support for presenting detailed information in failing assertions.
+"""
+import py
+import imp
+import marshal
+import struct
+import sys
+import pytest
+from _pytest.monkeypatch import monkeypatch
+from _pytest.assertion import reinterpret, util
+
+try:
+    from _pytest.assertion.rewrite import rewrite_asserts
+except ImportError:
+    rewrite_asserts = None
+else:
+    import ast
+
+def pytest_addoption(parser):
+    group = parser.getgroup("debugconfig")
+    group.addoption('--assertmode', action="store", dest="assertmode",
+                    choices=("on", "old", "off", "default"), default="default",
+                    metavar="on|old|off",
+                    help="""control assertion debugging tools.
+'off' performs no assertion debugging.
+'old' reinterprets the expressions in asserts to glean information.
+'on' (the default) rewrites the assert statements in test modules to provide
+sub-expression results.""")
+    group.addoption('--no-assert', action="store_true", default=False,
+        dest="noassert", help="DEPRECATED equivalent to --assertmode=off")
+    group.addoption('--nomagic', action="store_true", default=False,
+        dest="nomagic", help="DEPRECATED equivalent to --assertmode=off")
+
+class AssertionState:
+    """State for the assertion plugin."""
+
+    def __init__(self, config, mode):
+        self.mode = mode
+        self.trace = config.trace.root.get("assertion")
+
+def pytest_configure(config):
+    warn_about_missing_assertion()
+    mode = config.getvalue("assertmode")
+    if config.getvalue("noassert") or config.getvalue("nomagic"):
+        if mode not in ("off", "default"):
+            raise pytest.UsageError("assertion options conflict")
+        mode = "off"
+    elif mode == "default":
+        mode = "on"
+    if mode != "off":
+        def callbinrepr(op, left, right):
+            hook_result = config.hook.pytest_assertrepr_compare(
+                config=config, op=op, left=left, right=right)
+            for new_expl in hook_result:
+                if new_expl:
+                    return '\n~'.join(new_expl)
+        m = monkeypatch()
+        config._cleanup.append(m.undo)
+        m.setattr(py.builtin.builtins, 'AssertionError',
+                  reinterpret.AssertionError)
+        m.setattr(util, '_reprcompare', callbinrepr)
+    if mode == "on" and rewrite_asserts is None:
+        mode = "old"
+    config._assertstate = AssertionState(config, mode)
+    config._assertstate.trace("configured with mode set to %r" % (mode,))
+
+def _write_pyc(co, source_path):
+    if hasattr(imp, "cache_from_source"):
+        # Handle PEP 3147 pycs.
+        pyc = py.path.local(imp.cache_from_source(str(source_path)))
+        pyc.ensure()
+    else:
+        pyc = source_path + "c"
+    mtime = int(source_path.mtime())
+    fp = pyc.open("wb")
+    try:
+        fp.write(imp.get_magic())
+        fp.write(struct.pack("<l", mtime))
+        marshal.dump(co, fp)
+    finally:
+        fp.close()
+    return pyc
+
+def before_module_import(mod):
+    if mod.config._assertstate.mode != "on":
+        return
+    # Some deep magic: load the source, rewrite the asserts, and write a
+    # fake pyc, so that it'll be loaded when the module is imported.
+    source = mod.fspath.read()
+    try:
+        tree = ast.parse(source)
+    except SyntaxError:
+        # Let this pop up again in the real import.
+        mod.config._assertstate.trace("failed to parse: %r" % (mod.fspath,))
+        return
+    rewrite_asserts(tree)
+    try:
+        co = compile(tree, str(mod.fspath), "exec")
+    except SyntaxError:
+        # It's possible that this error is from some bug in the assertion
+        # rewriting, but I don't know of a fast way to tell.
+        mod.config._assertstate.trace("failed to compile: %r" % (mod.fspath,))
+        return
+    mod._pyc = _write_pyc(co, mod.fspath)
+    mod.config._assertstate.trace("wrote pyc: %r" % (mod._pyc,))
+
+def after_module_import(mod):
+    if not hasattr(mod, "_pyc"):
+        return
+    state = mod.config._assertstate
+    try:
+        mod._pyc.remove()
+    except py.error.ENOENT:
+        state.trace("couldn't find pyc: %r" % (mod._pyc,))
+    else:
+        state.trace("removed pyc: %r" % (mod._pyc,))
+
+def warn_about_missing_assertion():
+    try:
+        assert False
+    except AssertionError:
+        pass
+    else:
+        sys.stderr.write("WARNING: failing tests may report as passing because "
+        "assertions are turned off!  (are you using python -O?)\n")
+
+pytest_assertrepr_compare = util.assertrepr_compare
diff --git a/_pytest/assertion/newinterpret.py b/_pytest/assertion/newinterpret.py
new file mode 100644
--- /dev/null
+++ b/_pytest/assertion/newinterpret.py
@@ -0,0 +1,333 @@
+"""
+Find intermediate evalutation results in assert statements through builtin AST.
+This should replace oldinterpret.py eventually.
+"""
+
+import sys
+import ast
+
+import py
+from _pytest.assertion import util
+from _pytest.assertion.reinterpret import BuiltinAssertionError
+
+
+if sys.platform.startswith("java") and sys.version_info < (2, 5, 2):
+    # See http://bugs.jython.org/issue1497
+    _exprs = ("BoolOp", "BinOp", "UnaryOp", "Lambda", "IfExp", "Dict",
+              "ListComp", "GeneratorExp", "Yield", "Compare", "Call",
+              "Repr", "Num", "Str", "Attribute", "Subscript", "Name",
+              "List", "Tuple")
+    _stmts = ("FunctionDef", "ClassDef", "Return", "Delete", "Assign",
+              "AugAssign", "Print", "For", "While", "If", "With", "Raise",
+              "TryExcept", "TryFinally", "Assert", "Import", "ImportFrom",
+              "Exec", "Global", "Expr", "Pass", "Break", "Continue")
+    _expr_nodes = set(getattr(ast, name) for name in _exprs)
+    _stmt_nodes = set(getattr(ast, name) for name in _stmts)
+    def _is_ast_expr(node):
+        return node.__class__ in _expr_nodes
+    def _is_ast_stmt(node):
+        return node.__class__ in _stmt_nodes
+else:
+    def _is_ast_expr(node):
+        return isinstance(node, ast.expr)
+    def _is_ast_stmt(node):
+        return isinstance(node, ast.stmt)
+
+
+class Failure(Exception):
+    """Error found while interpreting AST."""
+
+    def __init__(self, explanation=""):
+        self.cause = sys.exc_info()
+        self.explanation = explanation
+
+
+def interpret(source, frame, should_fail=False):
+    mod = ast.parse(source)
+    visitor = DebugInterpreter(frame)
+    try:
+        visitor.visit(mod)
+    except Failure:
+        failure = sys.exc_info()[1]
+        return getfailure(failure)
+    if should_fail:
+        return ("(assertion failed, but when it was re-run for "
+                "printing intermediate values, it did not fail.  Suggestions: "
+                "compute assert expression before the assert or use --no-assert)")
+
+def run(offending_line, frame=None):
+    if frame is None:
+        frame = py.code.Frame(sys._getframe(1))
+    return interpret(offending_line, frame)
+
+def getfailure(e):
+    explanation = util.format_explanation(e.explanation)
+    value = e.cause[1]
+    if str(value):
+        lines = explanation.split('\n')
+        lines[0] += "  << %s" % (value,)
+        explanation = '\n'.join(lines)
+    text = "%s: %s" % (e.cause[0].__name__, explanation)
+    if text.startswith('AssertionError: assert '):
+        text = text[16:]
+    return text
+
+operator_map = {
+    ast.BitOr : "|",
+    ast.BitXor : "^",
+    ast.BitAnd : "&",
+    ast.LShift : "<<",
+    ast.RShift : ">>",
+    ast.Add : "+",
+    ast.Sub : "-",
+    ast.Mult : "*",
+    ast.Div : "/",
+    ast.FloorDiv : "//",
+    ast.Mod : "%",
+    ast.Eq : "==",
+    ast.NotEq : "!=",
+    ast.Lt : "<",
+    ast.LtE : "<=",
+    ast.Gt : ">",
+    ast.GtE : ">=",
+    ast.Pow : "**",
+    ast.Is : "is",
+    ast.IsNot : "is not",
+    ast.In : "in",
+    ast.NotIn : "not in"
+}
+
+unary_map = {
+    ast.Not : "not %s",
+    ast.Invert : "~%s",
+    ast.USub : "-%s",
+    ast.UAdd : "+%s"
+}
+
+
+class DebugInterpreter(ast.NodeVisitor):
+    """Interpret AST nodes to gleam useful debugging information. """
+
+    def __init__(self, frame):
+        self.frame = frame
+
+    def generic_visit(self, node):
+        # Fallback when we don't have a special implementation.
+        if _is_ast_expr(node):
+            mod = ast.Expression(node)
+            co = self._compile(mod)
+            try:
+                result = self.frame.eval(co)
+            except Exception:
+                raise Failure()
+            explanation = self.frame.repr(result)
+            return explanation, result
+        elif _is_ast_stmt(node):
+            mod = ast.Module([node])
+            co = self._compile(mod, "exec")
+            try:
+                self.frame.exec_(co)
+            except Exception:
+                raise Failure()
+            return None, None
+        else:
+            raise AssertionError("can't handle %s" %(node,))
+
+    def _compile(self, source, mode="eval"):
+        return compile(source, "<assertion interpretation>", mode)
+
+    def visit_Expr(self, expr):
+        return self.visit(expr.value)
+
+    def visit_Module(self, mod):
+        for stmt in mod.body:
+            self.visit(stmt)
+
+    def visit_Name(self, name):
+        explanation, result = self.generic_visit(name)
+        # See if the name is local.
+        source = "%r in locals() is not globals()" % (name.id,)
+        co = self._compile(source)
+        try:
+            local = self.frame.eval(co)
+        except Exception:
+            # have to assume it isn't
+            local = None
+        if local is None or not self.frame.is_true(local):
+            return name.id, result
+        return explanation, result
+
+    def visit_Compare(self, comp):
+        left = comp.left
+        left_explanation, left_result = self.visit(left)
+        for op, next_op in zip(comp.ops, comp.comparators):
+            next_explanation, next_result = self.visit(next_op)
+            op_symbol = operator_map[op.__class__]
+            explanation = "%s %s %s" % (left_explanation, op_symbol,
+                                        next_explanation)
+            source = "__exprinfo_left %s __exprinfo_right" % (op_symbol,)
+            co = self._compile(source)
+            try:
+                result = self.frame.eval(co, __exprinfo_left=left_result,
+                                         __exprinfo_right=next_result)
+            except Exception:
+                raise Failure(explanation)
+            try:
+                if not self.frame.is_true(result):
+                    break
+            except KeyboardInterrupt:
+                raise
+            except:
+                break
+            left_explanation, left_result = next_explanation, next_result
+
+        if util._reprcompare is not None:
+            res = util._reprcompare(op_symbol, left_result, next_result)
+            if res:
+                explanation = res
+        return explanation, result
+
+    def visit_BoolOp(self, boolop):
+        is_or = isinstance(boolop.op, ast.Or)
+        explanations = []
+        for operand in boolop.values:
+            explanation, result = self.visit(operand)
+            explanations.append(explanation)
+            if result == is_or:
+                break
+        name = is_or and " or " or " and "
+        explanation = "(" + name.join(explanations) + ")"
+        return explanation, result
+
+    def visit_UnaryOp(self, unary):
+        pattern = unary_map[unary.op.__class__]
+        operand_explanation, operand_result = self.visit(unary.operand)
+        explanation = pattern % (operand_explanation,)
+        co = self._compile(pattern % ("__exprinfo_expr",))
+        try:
+            result = self.frame.eval(co, __exprinfo_expr=operand_result)
+        except Exception:
+            raise Failure(explanation)
+        return explanation, result
+
+    def visit_BinOp(self, binop):
+        left_explanation, left_result = self.visit(binop.left)
+        right_explanation, right_result = self.visit(binop.right)
+        symbol = operator_map[binop.op.__class__]
+        explanation = "(%s %s %s)" % (left_explanation, symbol,
+                                      right_explanation)
+        source = "__exprinfo_left %s __exprinfo_right" % (symbol,)
+        co = self._compile(source)
+        try:
+            result = self.frame.eval(co, __exprinfo_left=left_result,
+                                     __exprinfo_right=right_result)
+        except Exception:
+            raise Failure(explanation)
+        return explanation, result
+
+    def visit_Call(self, call):
+        func_explanation, func = self.visit(call.func)
+        arg_explanations = []
+        ns = {"__exprinfo_func" : func}
+        arguments = []
+        for arg in call.args:
+            arg_explanation, arg_result = self.visit(arg)
+            arg_name = "__exprinfo_%s" % (len(ns),)
+            ns[arg_name] = arg_result
+            arguments.append(arg_name)
+            arg_explanations.append(arg_explanation)
+        for keyword in call.keywords:
+            arg_explanation, arg_result = self.visit(keyword.value)
+            arg_name = "__exprinfo_%s" % (len(ns),)
+            ns[arg_name] = arg_result
+            keyword_source = "%s=%%s" % (keyword.arg)
+            arguments.append(keyword_source % (arg_name,))
+            arg_explanations.append(keyword_source % (arg_explanation,))
+        if call.starargs:
+            arg_explanation, arg_result = self.visit(call.starargs)
+            arg_name = "__exprinfo_star"
+            ns[arg_name] = arg_result
+            arguments.append("*%s" % (arg_name,))
+            arg_explanations.append("*%s" % (arg_explanation,))
+        if call.kwargs:
+            arg_explanation, arg_result = self.visit(call.kwargs)
+            arg_name = "__exprinfo_kwds"
+            ns[arg_name] = arg_result
+            arguments.append("**%s" % (arg_name,))
+            arg_explanations.append("**%s" % (arg_explanation,))
+        args_explained = ", ".join(arg_explanations)
+        explanation = "%s(%s)" % (func_explanation, args_explained)
+        args = ", ".join(arguments)
+        source = "__exprinfo_func(%s)" % (args,)
+        co = self._compile(source)
+        try:
+            result = self.frame.eval(co, **ns)
+        except Exception:
+            raise Failure(explanation)
+        pattern = "%s\n{%s = %s\n}"
+        rep = self.frame.repr(result)
+        explanation = pattern % (rep, rep, explanation)
+        return explanation, result
+
+    def _is_builtin_name(self, name):
+        pattern = "%r not in globals() and %r not in locals()"
+        source = pattern % (name.id, name.id)
+        co = self._compile(source)
+        try:
+            return self.frame.eval(co)
+        except Exception:
+            return False
+
+    def visit_Attribute(self, attr):
+        if not isinstance(attr.ctx, ast.Load):
+            return self.generic_visit(attr)
+        source_explanation, source_result = self.visit(attr.value)
+        explanation = "%s.%s" % (source_explanation, attr.attr)
+        source = "__exprinfo_expr.%s" % (attr.attr,)
+        co = self._compile(source)
+        try:
+            result = self.frame.eval(co, __exprinfo_expr=source_result)
+        except Exception:
+            raise Failure(explanation)
+        explanation = "%s\n{%s = %s.%s\n}" % (self.frame.repr(result),
+                                              self.frame.repr(result),
+                                              source_explanation, attr.attr)
+        # Check if the attr is from an instance.
+        source = "%r in getattr(__exprinfo_expr, '__dict__', {})"
+        source = source % (attr.attr,)
+        co = self._compile(source)
+        try:
+            from_instance = self.frame.eval(co, __exprinfo_expr=source_result)
+        except Exception:
+            from_instance = None
+        if from_instance is None or self.frame.is_true(from_instance):
+            rep = self.frame.repr(result)
+            pattern = "%s\n{%s = %s\n}"
+            explanation = pattern % (rep, rep, explanation)
+        return explanation, result
+
+    def visit_Assert(self, assrt):
+        test_explanation, test_result = self.visit(assrt.test)
+        explanation = "assert %s" % (test_explanation,)
+        if not self.frame.is_true(test_result):
+            try:
+                raise BuiltinAssertionError
+            except Exception:
+                raise Failure(explanation)
+        return explanation, test_result
+
+    def visit_Assign(self, assign):
+        value_explanation, value_result = self.visit(assign.value)
+        explanation = "... = %s" % (value_explanation,)
+        name = ast.Name("__exprinfo_expr", ast.Load(),
+                        lineno=assign.value.lineno,
+                        col_offset=assign.value.col_offset)
+        new_assign = ast.Assign(assign.targets, name, lineno=assign.lineno,
+                                col_offset=assign.col_offset)
+        mod = ast.Module([new_assign])
+        co = self._compile(mod, "exec")
+        try:
+            self.frame.exec_(co, __exprinfo_expr=value_result)
+        except Exception:
+            raise Failure(explanation)
+        return explanation, value_result
diff --git a/_pytest/assertion/oldinterpret.py b/_pytest/assertion/oldinterpret.py
new file mode 100644
--- /dev/null
+++ b/_pytest/assertion/oldinterpret.py
@@ -0,0 +1,552 @@
+import py
+import sys, inspect
+from compiler import parse, ast, pycodegen
+from _pytest.assertion.util import format_explanation
+from _pytest.assertion.reinterpret import BuiltinAssertionError
+
+passthroughex = py.builtin._sysex
+
+class Failure:
+    def __init__(self, node):
+        self.exc, self.value, self.tb = sys.exc_info()
+        self.node = node
+
+class View(object):
+    """View base class.
+
+    If C is a subclass of View, then C(x) creates a proxy object around
+    the object x.  The actual class of the proxy is not C in general,
+    but a *subclass* of C determined by the rules below.  To avoid confusion
+    we call view class the class of the proxy (a subclass of C, so of View)
+    and object class the class of x.
+
+    Attributes and methods not found in the proxy are automatically read on x.
+    Other operations like setting attributes are performed on the proxy, as
+    determined by its view class.  The object x is available from the proxy
+    as its __obj__ attribute.
+
+    The view class selection is determined by the __view__ tuples and the
+    optional __viewkey__ method.  By default, the selected view class is the
+    most specific subclass of C whose __view__ mentions the class of x.
+    If no such subclass is found, the search proceeds with the parent
+    object classes.  For example, C(True) will first look for a subclass
+    of C with __view__ = (..., bool, ...) and only if it doesn't find any
+    look for one with __view__ = (..., int, ...), and then ..., object,...
+    If everything fails the class C itself is considered to be the default.
+
+    Alternatively, the view class selection can be driven by another aspect
+    of the object x, instead of the class of x, by overriding __viewkey__.
+    See last example at the end of this module.
+    """
+
+    _viewcache = {}
+    __view__ = ()
+
+    def __new__(rootclass, obj, *args, **kwds):
+        self = object.__new__(rootclass)
+        self.__obj__ = obj
+        self.__rootclass__ = rootclass
+        key = self.__viewkey__()
+        try:
+            self.__class__ = self._viewcache[key]
+        except KeyError:
+            self.__class__ = self._selectsubclass(key)
+        return self
+
+    def __getattr__(self, attr):
+        # attributes not found in the normal hierarchy rooted on View
+        # are looked up in the object's real class
+        return getattr(self.__obj__, attr)
+
+    def __viewkey__(self):
+        return self.__obj__.__class__
+
+    def __matchkey__(self, key, subclasses):
+        if inspect.isclass(key):
+            keys = inspect.getmro(key)
+        else:
+            keys = [key]
+        for key in keys:
+            result = [C for C in subclasses if key in C.__view__]
+            if result:
+                return result
+        return []
+
+    def _selectsubclass(self, key):
+        subclasses = list(enumsubclasses(self.__rootclass__))
+        for C in subclasses:
+            if not isinstance(C.__view__, tuple):
+                C.__view__ = (C.__view__,)
+        choices = self.__matchkey__(key, subclasses)
+        if not choices:
+            return self.__rootclass__
+        elif len(choices) == 1:
+            return choices[0]
+        else:
+            # combine the multiple choices
+            return type('?', tuple(choices), {})
+
+    def __repr__(self):
+        return '%s(%r)' % (self.__rootclass__.__name__, self.__obj__)
+
+
+def enumsubclasses(cls):
+    for subcls in cls.__subclasses__():
+        for subsubclass in enumsubclasses(subcls):
+            yield subsubclass
+    yield cls
+
+
+class Interpretable(View):
+    """A parse tree node with a few extra methods."""
+    explanation = None
+
+    def is_builtin(self, frame):
+        return False
+
+    def eval(self, frame):
+        # fall-back for unknown expression nodes
+        try:
+            expr = ast.Expression(self.__obj__)
+            expr.filename = '<eval>'
+            self.__obj__.filename = '<eval>'
+            co = pycodegen.ExpressionCodeGenerator(expr).getCode()
+            result = frame.eval(co)
+        except passthroughex:
+            raise
+        except:
+            raise Failure(self)
+        self.result = result
+        self.explanation = self.explanation or frame.repr(self.result)
+
+    def run(self, frame):
+        # fall-back for unknown statement nodes
+        try:
+            expr = ast.Module(None, ast.Stmt([self.__obj__]))
+            expr.filename = '<run>'
+            co = pycodegen.ModuleCodeGenerator(expr).getCode()
+            frame.exec_(co)
+        except passthroughex:
+            raise
+        except:
+            raise Failure(self)
+
+    def nice_explanation(self):
+        return format_explanation(self.explanation)
+
+
+class Name(Interpretable):
+    __view__ = ast.Name
+
+    def is_local(self, frame):
+        source = '%r in locals() is not globals()' % self.name
+        try:
+            return frame.is_true(frame.eval(source))
+        except passthroughex:
+            raise
+        except:
+            return False
+
+    def is_global(self, frame):
+        source = '%r in globals()' % self.name
+        try:
+            return frame.is_true(frame.eval(source))
+        except passthroughex:
+            raise
+        except:
+            return False
+
+    def is_builtin(self, frame):
+        source = '%r not in locals() and %r not in globals()' % (
+            self.name, self.name)
+        try:
+            return frame.is_true(frame.eval(source))
+        except passthroughex:
+            raise
+        except:
+            return False
+
+    def eval(self, frame):
+        super(Name, self).eval(frame)
+        if not self.is_local(frame):
+            self.explanation = self.name
+
+class Compare(Interpretable):
+    __view__ = ast.Compare
+
+    def eval(self, frame):
+        expr = Interpretable(self.expr)
+        expr.eval(frame)
+        for operation, expr2 in self.ops:
+            if hasattr(self, 'result'):
+                # shortcutting in chained expressions
+                if not frame.is_true(self.result):
+                    break
+            expr2 = Interpretable(expr2)
+            expr2.eval(frame)
+            self.explanation = "%s %s %s" % (
+                expr.explanation, operation, expr2.explanation)
+            source = "__exprinfo_left %s __exprinfo_right" % operation
+            try:
+                self.result = frame.eval(source,
+                                         __exprinfo_left=expr.result,
+                                         __exprinfo_right=expr2.result)
+            except passthroughex:
+                raise
+            except:
+                raise Failure(self)
+            expr = expr2
+
+class And(Interpretable):
+    __view__ = ast.And
+
+    def eval(self, frame):
+        explanations = []
+        for expr in self.nodes:
+            expr = Interpretable(expr)
+            expr.eval(frame)
+            explanations.append(expr.explanation)
+            self.result = expr.result
+            if not frame.is_true(expr.result):
+                break
+        self.explanation = '(' + ' and '.join(explanations) + ')'
+
+class Or(Interpretable):
+    __view__ = ast.Or
+
+    def eval(self, frame):
+        explanations = []
+        for expr in self.nodes:
+            expr = Interpretable(expr)
+            expr.eval(frame)
+            explanations.append(expr.explanation)
+            self.result = expr.result
+            if frame.is_true(expr.result):
+                break
+        self.explanation = '(' + ' or '.join(explanations) + ')'
+
+
+# == Unary operations ==
+keepalive = []
+for astclass, astpattern in {
+    ast.Not    : 'not __exprinfo_expr',
+    ast.Invert : '(~__exprinfo_expr)',
+    }.items():
+
+    class UnaryArith(Interpretable):
+        __view__ = astclass
+
+        def eval(self, frame, astpattern=astpattern):
+            expr = Interpretable(self.expr)
+            expr.eval(frame)
+            self.explanation = astpattern.replace('__exprinfo_expr',
+                                                  expr.explanation)
+            try:
+                self.result = frame.eval(astpattern,
+                                         __exprinfo_expr=expr.result)
+            except passthroughex:
+                raise
+            except:
+                raise Failure(self)
+
+    keepalive.append(UnaryArith)
+
+# == Binary operations ==
+for astclass, astpattern in {
+    ast.Add    : '(__exprinfo_left + __exprinfo_right)',
+    ast.Sub    : '(__exprinfo_left - __exprinfo_right)',
+    ast.Mul    : '(__exprinfo_left * __exprinfo_right)',
+    ast.Div    : '(__exprinfo_left / __exprinfo_right)',
+    ast.Mod    : '(__exprinfo_left % __exprinfo_right)',
+    ast.Power  : '(__exprinfo_left ** __exprinfo_right)',
+    }.items():
+
+    class BinaryArith(Interpretable):
+        __view__ = astclass
+
+        def eval(self, frame, astpattern=astpattern):
+            left = Interpretable(self.left)
+            left.eval(frame)
+            right = Interpretable(self.right)
+            right.eval(frame)
+            self.explanation = (astpattern
+                                .replace('__exprinfo_left',  left .explanation)
+                                .replace('__exprinfo_right', right.explanation))
+            try:
+                self.result = frame.eval(astpattern,
+                                         __exprinfo_left=left.result,
+                                         __exprinfo_right=right.result)
+            except passthroughex:
+                raise
+            except:
+                raise Failure(self)
+
+    keepalive.append(BinaryArith)
+
+
+class CallFunc(Interpretable):
+    __view__ = ast.CallFunc
+
+    def is_bool(self, frame):
+        source = 'isinstance(__exprinfo_value, bool)'
+        try:
+            return frame.is_true(frame.eval(source,
+                                            __exprinfo_value=self.result))
+        except passthroughex:
+            raise
+        except:
+            return False
+
+    def eval(self, frame):
+        node = Interpretable(self.node)
+        node.eval(frame)
+        explanations = []
+        vars = {'__exprinfo_fn': node.result}
+        source = '__exprinfo_fn('
+        for a in self.args:
+            if isinstance(a, ast.Keyword):
+                keyword = a.name
+                a = a.expr
+            else:
+                keyword = None
+            a = Interpretable(a)
+            a.eval(frame)
+            argname = '__exprinfo_%d' % len(vars)
+            vars[argname] = a.result
+            if keyword is None:
+                source += argname + ','
+                explanations.append(a.explanation)
+            else:
+                source += '%s=%s,' % (keyword, argname)
+                explanations.append('%s=%s' % (keyword, a.explanation))
+        if self.star_args:
+            star_args = Interpretable(self.star_args)
+            star_args.eval(frame)
+            argname = '__exprinfo_star'
+            vars[argname] = star_args.result
+            source += '*' + argname + ','
+            explanations.append('*' + star_args.explanation)
+        if self.dstar_args:
+            dstar_args = Interpretable(self.dstar_args)
+            dstar_args.eval(frame)
+            argname = '__exprinfo_kwds'
+            vars[argname] = dstar_args.result
+            source += '**' + argname + ','
+            explanations.append('**' + dstar_args.explanation)
+        self.explanation = "%s(%s)" % (
+            node.explanation, ', '.join(explanations))
+        if source.endswith(','):
+            source = source[:-1]
+        source += ')'
+        try:
+            self.result = frame.eval(source, **vars)
+        except passthroughex:
+            raise
+        except:
+            raise Failure(self)
+        if not node.is_builtin(frame) or not self.is_bool(frame):
+            r = frame.repr(self.result)
+            self.explanation = '%s\n{%s = %s\n}' % (r, r, self.explanation)
+
+class Getattr(Interpretable):
+    __view__ = ast.Getattr
+
+    def eval(self, frame):
+        expr = Interpretable(self.expr)
+        expr.eval(frame)
+        source = '__exprinfo_expr.%s' % self.attrname
+        try:
+            self.result = frame.eval(source, __exprinfo_expr=expr.result)
+        except passthroughex:
+            raise
+        except:
+            raise Failure(self)
+        self.explanation = '%s.%s' % (expr.explanation, self.attrname)
+        # if the attribute comes from the instance, its value is interesting
+        source = ('hasattr(__exprinfo_expr, "__dict__") and '
+                  '%r in __exprinfo_expr.__dict__' % self.attrname)
+        try:
+            from_instance = frame.is_true(
+                frame.eval(source, __exprinfo_expr=expr.result))
+        except passthroughex:
+            raise
+        except:
+            from_instance = True
+        if from_instance:
+            r = frame.repr(self.result)
+            self.explanation = '%s\n{%s = %s\n}' % (r, r, self.explanation)
+
+# == Re-interpretation of full statements ==
+
+class Assert(Interpretable):
+    __view__ = ast.Assert
+
+    def run(self, frame):
+        test = Interpretable(self.test)
+        test.eval(frame)
+        # print the result as  'assert <explanation>'
+        self.result = test.result
+        self.explanation = 'assert ' + test.explanation
+        if not frame.is_true(test.result):
+            try:
+                raise BuiltinAssertionError
+            except passthroughex:
+                raise
+            except:
+                raise Failure(self)
+
+class Assign(Interpretable):
+    __view__ = ast.Assign
+
+    def run(self, frame):
+        expr = Interpretable(self.expr)
+        expr.eval(frame)
+        self.result = expr.result
+        self.explanation = '... = ' + expr.explanation
+        # fall-back-run the rest of the assignment
+        ass = ast.Assign(self.nodes, ast.Name('__exprinfo_expr'))
+        mod = ast.Module(None, ast.Stmt([ass]))
+        mod.filename = '<run>'
+        co = pycodegen.ModuleCodeGenerator(mod).getCode()
+        try:
+            frame.exec_(co, __exprinfo_expr=expr.result)
+        except passthroughex:
+            raise
+        except:
+            raise Failure(self)
+
+class Discard(Interpretable):
+    __view__ = ast.Discard
+
+    def run(self, frame):
+        expr = Interpretable(self.expr)
+        expr.eval(frame)
+        self.result = expr.result
+        self.explanation = expr.explanation
+
+class Stmt(Interpretable):
+    __view__ = ast.Stmt
+
+    def run(self, frame):
+        for stmt in self.nodes:
+            stmt = Interpretable(stmt)
+            stmt.run(frame)
+
+
+def report_failure(e):
+    explanation = e.node.nice_explanation()
+    if explanation:
+        explanation = ", in: " + explanation
+    else:
+        explanation = ""
+    sys.stdout.write("%s: %s%s\n" % (e.exc.__name__, e.value, explanation))
+
+def check(s, frame=None):
+    if frame is None:
+        frame = sys._getframe(1)
+        frame = py.code.Frame(frame)
+    expr = parse(s, 'eval')
+    assert isinstance(expr, ast.Expression)
+    node = Interpretable(expr.node)
+    try:
+        node.eval(frame)
+    except passthroughex:
+        raise
+    except Failure:
+        e = sys.exc_info()[1]
+        report_failure(e)
+    else:
+        if not frame.is_true(node.result):
+            sys.stderr.write("assertion failed: %s\n" % node.nice_explanation())
+
+
+###########################################################
+# API / Entry points
+# #########################################################
+
+def interpret(source, frame, should_fail=False):
+    module = Interpretable(parse(source, 'exec').node)
+    #print "got module", module
+    if isinstance(frame, py.std.types.FrameType):
+        frame = py.code.Frame(frame)
+    try:
+        module.run(frame)
+    except Failure:
+        e = sys.exc_info()[1]
+        return getfailure(e)
+    except passthroughex:
+        raise
+    except:
+        import traceback
+        traceback.print_exc()
+    if should_fail:
+        return ("(assertion failed, but when it was re-run for "
+                "printing intermediate values, it did not fail.  Suggestions: "
+                "compute assert expression before the assert or use --nomagic)")
+    else:
+        return None
+
+def getmsg(excinfo):
+    if isinstance(excinfo, tuple):
+        excinfo = py.code.ExceptionInfo(excinfo)
+    #frame, line = gettbline(tb)
+    #frame = py.code.Frame(frame)
+    #return interpret(line, frame)
+
+    tb = excinfo.traceback[-1]
+    source = str(tb.statement).strip()
+    x = interpret(source, tb.frame, should_fail=True)
+    if not isinstance(x, str):
+        raise TypeError("interpret returned non-string %r" % (x,))
+    return x
+
+def getfailure(e):
+    explanation = e.node.nice_explanation()
+    if str(e.value):
+        lines = explanation.split('\n')
+        lines[0] += "  << %s" % (e.value,)
+        explanation = '\n'.join(lines)
+    text = "%s: %s" % (e.exc.__name__, explanation)
+    if text.startswith('AssertionError: assert '):
+        text = text[16:]
+    return text
+
+def run(s, frame=None):
+    if frame is None:
+        frame = sys._getframe(1)
+        frame = py.code.Frame(frame)
+    module = Interpretable(parse(s, 'exec').node)
+    try:
+        module.run(frame)
+    except Failure:
+        e = sys.exc_info()[1]
+        report_failure(e)
+
+
+if __name__ == '__main__':
+    # example:
+    def f():
+        return 5
+    def g():
+        return 3
+    def h(x):
+        return 'never'
+    check("f() * g() == 5")
+    check("not f()")
+    check("not (f() and g() or 0)")
+    check("f() == g()")
+    i = 4
+    check("i == f()")
+    check("len(f()) == 0")
+    check("isinstance(2+3+4, float)")
+
+    run("x = i")
+    check("x == 5")
+
+    run("assert not f(), 'oops'")
+    run("a, b, c = 1, 2")
+    run("a, b, c = f()")
+
+    check("max([f(),g()]) == 4")
+    check("'hello'[g()] == 'h'")
+    run("'guk%d' % h(f())")
diff --git a/_pytest/assertion/reinterpret.py b/_pytest/assertion/reinterpret.py
new file mode 100644
--- /dev/null
+++ b/_pytest/assertion/reinterpret.py
@@ -0,0 +1,48 @@
+import sys
+import py
+
+BuiltinAssertionError = py.builtin.builtins.AssertionError
+
+class AssertionError(BuiltinAssertionError):
+    def __init__(self, *args):
+        BuiltinAssertionError.__init__(self, *args)
+        if args:
+            try:
+                self.msg = str(args[0])
+            except py.builtin._sysex:
+                raise
+            except:
+                self.msg = "<[broken __repr__] %s at %0xd>" %(
+                    args[0].__class__, id(args[0]))
+        else:
+            f = py.code.Frame(sys._getframe(1))
+            try:
+                source = f.code.fullsource
+                if source is not None:
+                    try:
+                        source = source.getstatement(f.lineno, assertion=True)
+                    except IndexError:
+                        source = None
+                    else:
+                        source = str(source.deindent()).strip()
+            except py.error.ENOENT:
+                source = None
+                # this can also occur during reinterpretation, when the
+                # co_filename is set to "<run>".
+            if source:
+                self.msg = reinterpret(source, f, should_fail=True)
+            else:
+                self.msg = "<could not determine information>"
+            if not self.args:
+                self.args = (self.msg,)
+
+if sys.version_info > (3, 0):
+    AssertionError.__module__ = "builtins"
+    reinterpret_old = "old reinterpretation not available for py3"
+else:
+    from _pytest.assertion.oldinterpret import interpret as reinterpret_old
+if sys.version_info >= (2, 6) or (sys.platform.startswith("java")):
+    from _pytest.assertion.newinterpret import interpret as reinterpret
+else:
+    reinterpret = reinterpret_old
+
diff --git a/_pytest/assertion/rewrite.py b/_pytest/assertion/rewrite.py
new file mode 100644
--- /dev/null
+++ b/_pytest/assertion/rewrite.py
@@ -0,0 +1,340 @@
+"""Rewrite assertion AST to produce nice error messages"""
+
+import ast
+import collections
+import itertools
+import sys
+
+import py
+from _pytest.assertion import util
+
+
+def rewrite_asserts(mod):
+    """Rewrite the assert statements in mod."""
+    AssertionRewriter().run(mod)
+
+
+_saferepr = py.io.saferepr
+from _pytest.assertion.util import format_explanation as _format_explanation
+
+def _format_boolop(operands, explanations, is_or):
+    show_explanations = []
+    for operand, expl in zip(operands, explanations):
+        show_explanations.append(expl)
+        if operand == is_or:
+            break
+    return "(" + (is_or and " or " or " and ").join(show_explanations) + ")"
+
+def _call_reprcompare(ops, results, expls, each_obj):
+    for i, res, expl in zip(range(len(ops)), results, expls):
+        try:
+            done = not res
+        except Exception:
+            done = True
+        if done:
+            break
+    if util._reprcompare is not None:
+        custom = util._reprcompare(ops[i], each_obj[i], each_obj[i + 1])
+        if custom is not None:
+            return custom
+    return expl
+
+
+unary_map = {
+    ast.Not : "not %s",
+    ast.Invert : "~%s",
+    ast.USub : "-%s",
+    ast.UAdd : "+%s"
+}
+
+binop_map = {
+    ast.BitOr : "|",
+    ast.BitXor : "^",
+    ast.BitAnd : "&",
+    ast.LShift : "<<",
+    ast.RShift : ">>",
+    ast.Add : "+",
+    ast.Sub : "-",
+    ast.Mult : "*",
+    ast.Div : "/",
+    ast.FloorDiv : "//",
+    ast.Mod : "%",
+    ast.Eq : "==",
+    ast.NotEq : "!=",
+    ast.Lt : "<",
+    ast.LtE : "<=",
+    ast.Gt : ">",
+    ast.GtE : ">=",
+    ast.Pow : "**",
+    ast.Is : "is",
+    ast.IsNot : "is not",
+    ast.In : "in",
+    ast.NotIn : "not in"
+}
+
+
+def set_location(node, lineno, col_offset):
+    """Set node location information recursively."""
+    def _fix(node, lineno, col_offset):
+        if "lineno" in node._attributes:
+            node.lineno = lineno
+        if "col_offset" in node._attributes:
+            node.col_offset = col_offset
+        for child in ast.iter_child_nodes(node):
+            _fix(child, lineno, col_offset)
+    _fix(node, lineno, col_offset)
+    return node
+
+
+class AssertionRewriter(ast.NodeVisitor):
+
+    def run(self, mod):
+        """Find all assert statements in *mod* and rewrite them."""
+        if not mod.body:
+            # Nothing to do.
+            return
+        # Insert some special imports at the top of the module but after any
+        # docstrings and __future__ imports.
+        aliases = [ast.alias(py.builtin.builtins.__name__, "@py_builtins"),
+                   ast.alias("_pytest.assertion.rewrite", "@pytest_ar")]
+        expect_docstring = True
+        pos = 0
+        lineno = 0
+        for item in mod.body:
+            if (expect_docstring and isinstance(item, ast.Expr) and
+                isinstance(item.value, ast.Str)):
+                doc = item.value.s
+                if "PYTEST_DONT_REWRITE" in doc:
+                    # The module has disabled assertion rewriting.
+                    return
+                lineno += len(doc) - 1
+                expect_docstring = False
+            elif (not isinstance(item, ast.ImportFrom) or item.level > 0 and
+                  item.identifier != "__future__"):
+                lineno = item.lineno
+                break
+            pos += 1
+        imports = [ast.Import([alias], lineno=lineno, col_offset=0)
+                   for alias in aliases]
+        mod.body[pos:pos] = imports
+        # Collect asserts.
+        nodes = collections.deque([mod])
+        while nodes:
+            node = nodes.popleft()
+            for name, field in ast.iter_fields(node):
+                if isinstance(field, list):
+                    new = []
+                    for i, child in enumerate(field):
+                        if isinstance(child, ast.Assert):
+                            # Transform assert.
+                            new.extend(self.visit(child))
+                        else:
+                            new.append(child)
+                            if isinstance(child, ast.AST):
+                                nodes.append(child)
+                    setattr(node, name, new)
+                elif (isinstance(field, ast.AST) and
+                      # Don't recurse into expressions as they can't contain
+                      # asserts.
+                      not isinstance(field, ast.expr)):
+                    nodes.append(field)
+
+    def variable(self):
+        """Get a new variable."""
+        # Use a character invalid in python identifiers to avoid clashing.
+        name = "@py_assert" + str(next(self.variable_counter))
+        self.variables.add(name)
+        return name
+
+    def assign(self, expr):
+        """Give *expr* a name."""
+        name = self.variable()
+        self.statements.append(ast.Assign([ast.Name(name, ast.Store())], expr))
+        return ast.Name(name, ast.Load())
+
+    def display(self, expr):
+        """Call py.io.saferepr on the expression."""
+        return self.helper("saferepr", expr)
+
+    def helper(self, name, *args):
+        """Call a helper in this module."""
+        py_name = ast.Name("@pytest_ar", ast.Load())
+        attr = ast.Attribute(py_name, "_" + name, ast.Load())
+        return ast.Call(attr, list(args), [], None, None)
+
+    def builtin(self, name):
+        """Return the builtin called *name*."""
+        builtin_name = ast.Name("@py_builtins", ast.Load())
+        return ast.Attribute(builtin_name, name, ast.Load())
+
+    def explanation_param(self, expr):
+        specifier = "py" + str(next(self.variable_counter))
+        self.explanation_specifiers[specifier] = expr
+        return "%(" + specifier + ")s"
+
+    def push_format_context(self):
+        self.explanation_specifiers = {}
+        self.stack.append(self.explanation_specifiers)
+
+    def pop_format_context(self, expl_expr):
+        current = self.stack.pop()
+        if self.stack:
+            self.explanation_specifiers = self.stack[-1]
+        keys = [ast.Str(key) for key in current.keys()]
+        format_dict = ast.Dict(keys, list(current.values()))
+        form = ast.BinOp(expl_expr, ast.Mod(), format_dict)
+        name = "@py_format" + str(next(self.variable_counter))
+        self.on_failure.append(ast.Assign([ast.Name(name, ast.Store())], form))
+        return ast.Name(name, ast.Load())
+
+    def generic_visit(self, node):
+        """Handle expressions we don't have custom code for."""
+        assert isinstance(node, ast.expr)
+        res = self.assign(node)
+        return res, self.explanation_param(self.display(res))
+
+    def visit_Assert(self, assert_):
+        if assert_.msg:
+            # There's already a message. Don't mess with it.
+            return [assert_]
+        self.statements = []
+        self.variables = set()
+        self.variable_counter = itertools.count()
+        self.stack = []
+        self.on_failure = []
+        self.push_format_context()
+        # Rewrite assert into a bunch of statements.
+        top_condition, explanation = self.visit(assert_.test)
+        # Create failure message.
+        body = self.on_failure
+        negation = ast.UnaryOp(ast.Not(), top_condition)
+        self.statements.append(ast.If(negation, body, []))
+        explanation = "assert " + explanation
+        template = ast.Str(explanation)
+        msg = self.pop_format_context(template)
+        fmt = self.helper("format_explanation", msg)
+        err_name = ast.Name("AssertionError", ast.Load())
+        exc = ast.Call(err_name, [fmt], [], None, None)
+        if sys.version_info[0] >= 3:
+            raise_ = ast.Raise(exc, None)
+        else:
+            raise_ = ast.Raise(exc, None, None)
+        body.append(raise_)
+        # Delete temporary variables.
+        names = [ast.Name(name, ast.Del()) for name in self.variables]
+        if names:
+            delete = ast.Delete(names)
+            self.statements.append(delete)
+        # Fix line numbers.
+        for stmt in self.statements:
+            set_location(stmt, assert_.lineno, assert_.col_offset)
+        return self.statements
+
+    def visit_Name(self, name):
+        # Check if the name is local or not.
+        locs = ast.Call(self.builtin("locals"), [], [], None, None)
+        globs = ast.Call(self.builtin("globals"), [], [], None, None)
+        ops = [ast.In(), ast.IsNot()]
+        test = ast.Compare(ast.Str(name.id), ops, [locs, globs])
+        expr = ast.IfExp(test, self.display(name), ast.Str(name.id))
+        return name, self.explanation_param(expr)
+
+    def visit_BoolOp(self, boolop):
+        operands = []
+        explanations = []
+        self.push_format_context()
+        for operand in boolop.values:
+            res, explanation = self.visit(operand)
+            operands.append(res)
+            explanations.append(explanation)
+        expls = ast.Tuple([ast.Str(expl) for expl in explanations], ast.Load())
+        is_or = ast.Num(isinstance(boolop.op, ast.Or))
+        expl_template = self.helper("format_boolop",
+                                    ast.Tuple(operands, ast.Load()), expls,
+                                    is_or)
+        expl = self.pop_format_context(expl_template)
+        res = self.assign(ast.BoolOp(boolop.op, operands))
+        return res, self.explanation_param(expl)
+
+    def visit_UnaryOp(self, unary):
+        pattern = unary_map[unary.op.__class__]
+        operand_res, operand_expl = self.visit(unary.operand)
+        res = self.assign(ast.UnaryOp(unary.op, operand_res))
+        return res, pattern % (operand_expl,)
+
+    def visit_BinOp(self, binop):
+        symbol = binop_map[binop.op.__class__]
+        left_expr, left_expl = self.visit(binop.left)
+        right_expr, right_expl = self.visit(binop.right)
+        explanation = "(%s %s %s)" % (left_expl, symbol, right_expl)
+        res = self.assign(ast.BinOp(left_expr, binop.op, right_expr))
+        return res, explanation
+
+    def visit_Call(self, call):
+        new_func, func_expl = self.visit(call.func)
+        arg_expls = []
+        new_args = []
+        new_kwargs = []
+        new_star = new_kwarg = None
+        for arg in call.args:
+            res, expl = self.visit(arg)
+            new_args.append(res)
+            arg_expls.append(expl)
+        for keyword in call.keywords:
+            res, expl = self.visit(keyword.value)
+            new_kwargs.append(ast.keyword(keyword.arg, res))
+            arg_expls.append(keyword.arg + "=" + expl)
+        if call.starargs:
+            new_star, expl = self.visit(call.starargs)
+            arg_expls.append("*" + expl)
+        if call.kwargs:
+            new_kwarg, expl = self.visit(call.kwarg)
+            arg_expls.append("**" + expl)
+        expl = "%s(%s)" % (func_expl, ', '.join(arg_expls))
+        new_call = ast.Call(new_func, new_args, new_kwargs, new_star, new_kwarg)
+        res = self.assign(new_call)
+        res_expl = self.explanation_param(self.display(res))
+        outer_expl = "%s\n{%s = %s\n}" % (res_expl, res_expl, expl)
+        return res, outer_expl
+
+    def visit_Attribute(self, attr):
+        if not isinstance(attr.ctx, ast.Load):
+            return self.generic_visit(attr)
+        value, value_expl = self.visit(attr.value)
+        res = self.assign(ast.Attribute(value, attr.attr, ast.Load()))
+        res_expl = self.explanation_param(self.display(res))
+        pat = "%s\n{%s = %s.%s\n}"
+        expl = pat % (res_expl, res_expl, value_expl, attr.attr)
+        return res, expl
+
+    def visit_Compare(self, comp):
+        self.push_format_context()
+        left_res, left_expl = self.visit(comp.left)
+        res_variables = [self.variable() for i in range(len(comp.ops))]
+        load_names = [ast.Name(v, ast.Load()) for v in res_variables]
+        store_names = [ast.Name(v, ast.Store()) for v in res_variables]
+        it = zip(range(len(comp.ops)), comp.ops, comp.comparators)
+        expls = []
+        syms = []
+        results = [left_res]
+        for i, op, next_operand in it:
+            next_res, next_expl = self.visit(next_operand)
+            results.append(next_res)
+            sym = binop_map[op.__class__]
+            syms.append(ast.Str(sym))
+            expl = "%s %s %s" % (left_expl, sym, next_expl)
+            expls.append(ast.Str(expl))
+            res_expr = ast.Compare(left_res, [op], [next_res])
+            self.statements.append(ast.Assign([store_names[i]], res_expr))
+            left_res, left_expl = next_res, next_expl
+        # Use py.code._reprcompare if that's available.
+        expl_call = self.helper("call_reprcompare",
+                                ast.Tuple(syms, ast.Load()),
+                                ast.Tuple(load_names, ast.Load()),
+                                ast.Tuple(expls, ast.Load()),
+                                ast.Tuple(results, ast.Load()))
+        if len(comp.ops) > 1:
+            res = ast.BoolOp(ast.And(), load_names)
+        else:
+            res = load_names[0]
+        return res, self.explanation_param(self.pop_format_context(expl_call))
diff --git a/_pytest/assertion/util.py b/_pytest/assertion/util.py
new file mode 100644
--- /dev/null
+++ b/_pytest/assertion/util.py
@@ -0,0 +1,213 @@
+"""Utilities for assertion debugging"""
+
+import py
+
+
+# The _reprcompare attribute on the util module is used by the new assertion
+# interpretation code and assertion rewriter to detect this plugin was
+# loaded and in turn call the hooks defined here as part of the
+# DebugInterpreter.
+_reprcompare = None
+
+def format_explanation(explanation):
+    """This formats an explanation
+
+    Normally all embedded newlines are escaped, however there are
+    three exceptions: \n{, \n} and \n~.  The first two are intended
+    cover nested explanations, see function and attribute explanations
+    for examples (.visit_Call(), visit_Attribute()).  The last one is
+    for when one explanation needs to span multiple lines, e.g. when
+    displaying diffs.
+    """
+    # simplify 'assert False where False = ...'
+    where = 0
+    while True:
+        start = where = explanation.find("False\n{False = ", where)
+        if where == -1:
+            break
+        level = 0
+        for i, c in enumerate(explanation[start:]):
+            if c == "{":
+                level += 1
+            elif c == "}":
+                level -= 1
+                if not level:
+                    break
+        else:
+            raise AssertionError("unbalanced braces: %r" % (explanation,))
+        end = start + i
+        where = end
+        if explanation[end - 1] == '\n':
+            explanation = (explanation[:start] + explanation[start+15:end-1] +
+                           explanation[end+1:])
+            where -= 17
+    raw_lines = (explanation or '').split('\n')
+    # escape newlines not followed by {, } and ~
+    lines = [raw_lines[0]]
+    for l in raw_lines[1:]:
+        if l.startswith('{') or l.startswith('}') or l.startswith('~'):
+            lines.append(l)
+        else:
+            lines[-1] += '\\n' + l
+
+    result = lines[:1]
+    stack = [0]
+    stackcnt = [0]
+    for line in lines[1:]:
+        if line.startswith('{'):
+            if stackcnt[-1]:
+                s = 'and   '
+            else:
+                s = 'where '
+            stack.append(len(result))
+            stackcnt[-1] += 1
+            stackcnt.append(0)
+            result.append(' +' + '  '*(len(stack)-1) + s + line[1:])
+        elif line.startswith('}'):
+            assert line.startswith('}')
+            stack.pop()
+            stackcnt.pop()
+            result[stack[-1]] += line[1:]
+        else:
+            assert line.startswith('~')
+            result.append('  '*len(stack) + line[1:])
+    assert len(stack) == 1
+    return '\n'.join(result)
+
+
+# Provide basestring in python3
+try:
+    basestring = basestring
+except NameError:
+    basestring = str
+
+
+def assertrepr_compare(op, left, right):
+    """return specialised explanations for some operators/operands"""
+    width = 80 - 15 - len(op) - 2 # 15 chars indentation, 1 space around op
+    left_repr = py.io.saferepr(left, maxsize=int(width/2))
+    right_repr = py.io.saferepr(right, maxsize=width-len(left_repr))
+    summary = '%s %s %s' % (left_repr, op, right_repr)
+
+    issequence = lambda x: isinstance(x, (list, tuple))
+    istext = lambda x: isinstance(x, basestring)
+    isdict = lambda x: isinstance(x, dict)
+    isset = lambda x: isinstance(x, set)
+
+    explanation = None
+    try:
+        if op == '==':
+            if istext(left) and istext(right):
+                explanation = _diff_text(left, right)
+            elif issequence(left) and issequence(right):
+                explanation = _compare_eq_sequence(left, right)
+            elif isset(left) and isset(right):
+                explanation = _compare_eq_set(left, right)
+            elif isdict(left) and isdict(right):
+                explanation = _diff_text(py.std.pprint.pformat(left),
+                                         py.std.pprint.pformat(right))
+        elif op == 'not in':
+            if istext(left) and istext(right):
+                explanation = _notin_text(left, right)
+    except py.builtin._sysex:
+        raise
+    except:
+        excinfo = py.code.ExceptionInfo()
+        explanation = ['(pytest_assertion plugin: representation of '
+            'details failed. Probably an object has a faulty __repr__.)',
+            str(excinfo)
+            ]
+
+
+    if not explanation:
+        return None
+
+    # Don't include pageloads of data, should be configurable
+    if len(''.join(explanation)) > 80*8:
+        explanation = ['Detailed information too verbose, truncated']
+
+    return [summary] + explanation
+
+
+def _diff_text(left, right):
+    """Return the explanation for the diff between text
+
+    This will skip leading and trailing characters which are
+    identical to keep the diff minimal.
+    """
+    explanation = []
+    i = 0 # just in case left or right has zero length
+    for i in range(min(len(left), len(right))):
+        if left[i] != right[i]:
+            break
+    if i > 42:
+        i -= 10                 # Provide some context
+        explanation = ['Skipping %s identical '
+                       'leading characters in diff' % i]
+        left = left[i:]
+        right = right[i:]
+    if len(left) == len(right):
+        for i in range(len(left)):
+            if left[-i] != right[-i]:
+                break
+        if i > 42:
+            i -= 10     # Provide some context
+            explanation += ['Skipping %s identical '
+                            'trailing characters in diff' % i]
+            left = left[:-i]
+            right = right[:-i]
+    explanation += [line.strip('\n')
+                    for line in py.std.difflib.ndiff(left.splitlines(),
+                                                     right.splitlines())]
+    return explanation
+
+
+def _compare_eq_sequence(left, right):
+    explanation = []
+    for i in range(min(len(left), len(right))):
+        if left[i] != right[i]:
+            explanation += ['At index %s diff: %r != %r' %
+                            (i, left[i], right[i])]
+            break
+    if len(left) > len(right):
+        explanation += ['Left contains more items, '
+            'first extra item: %s' % py.io.saferepr(left[len(right)],)]
+    elif len(left) < len(right):
+        explanation += ['Right contains more items, '
+            'first extra item: %s' % py.io.saferepr(right[len(left)],)]
+    return explanation # + _diff_text(py.std.pprint.pformat(left),
+                       #             py.std.pprint.pformat(right))
+
+
+def _compare_eq_set(left, right):
+    explanation = []
+    diff_left = left - right
+    diff_right = right - left
+    if diff_left:
+        explanation.append('Extra items in the left set:')
+        for item in diff_left:
+            explanation.append(py.io.saferepr(item))
+    if diff_right:
+        explanation.append('Extra items in the right set:')
+        for item in diff_right:
+            explanation.append(py.io.saferepr(item))
+    return explanation
+
+
+def _notin_text(term, text):
+    index = text.find(term)
+    head = text[:index]
+    tail = text[index+len(term):]
+    correct_text = head + tail
+    diff = _diff_text(correct_text, text)
+    newdiff = ['%s is contained here:' % py.io.saferepr(term, maxsize=42)]
+    for line in diff:
+        if line.startswith('Skipping'):
+            continue
+        if line.startswith('- '):
+            continue
+        if line.startswith('+ '):
+            newdiff.append('  ' + line[2:])
+        else:
+            newdiff.append(line)
+    return newdiff
diff --git a/_pytest/doctest.py b/_pytest/doctest.py
--- a/_pytest/doctest.py
+++ b/_pytest/doctest.py
@@ -59,7 +59,7 @@
                 inner_excinfo = py.code.ExceptionInfo(excinfo.value.exc_info)
                 lines += ["UNEXPECTED EXCEPTION: %s" %
                             repr(inner_excinfo.value)]
-
+                lines += py.std.traceback.format_exception(*excinfo.value.exc_info)
             return ReprFailDoctest(reprlocation, lines)
         else:
             return super(DoctestItem, self).repr_failure(excinfo)
diff --git a/_pytest/helpconfig.py b/_pytest/helpconfig.py
--- a/_pytest/helpconfig.py
+++ b/_pytest/helpconfig.py
@@ -16,9 +16,6 @@
     group.addoption('--traceconfig',
                action="store_true", dest="traceconfig", default=False,
                help="trace considerations of conftest.py files."),
-    group._addoption('--nomagic',
-               action="store_true", dest="nomagic", default=False,
-               help="don't reinterpret asserts, no traceback cutting. ")
     group.addoption('--debug',
                action="store_true", dest="debug", default=False,
                help="generate and show internal debugging information.")
diff --git a/_pytest/junitxml.py b/_pytest/junitxml.py
--- a/_pytest/junitxml.py
+++ b/_pytest/junitxml.py
@@ -65,7 +65,8 @@
 
 class LogXML(object):
     def __init__(self, logfile, prefix):
-        self.logfile = logfile
+        logfile = os.path.expanduser(os.path.expandvars(logfile))
+        self.logfile = os.path.normpath(logfile)
         self.prefix = prefix
         self.test_logs = []
         self.passed = self.skipped = 0
@@ -76,7 +77,7 @@
         names = report.nodeid.split("::")
         names[0] = names[0].replace("/", '.')
         names = tuple(names)
-        d = {'time': self._durations.pop(names, "0")}
+        d = {'time': self._durations.pop(report.nodeid, "0")}
         names = [x.replace(".py", "") for x in names if x != "()"]
         classnames = names[:-1]
         if self.prefix:
@@ -170,12 +171,11 @@
             self.append_skipped(report)
 
     def pytest_runtest_call(self, item, __multicall__):
-        names = tuple(item.listnames())
         start = time.time()
         try:
             return __multicall__.execute()
         finally:
-            self._durations[names] = time.time() - start
+            self._durations[item.nodeid] = time.time() - start
 
     def pytest_collectreport(self, report):
         if not report.passed:
diff --git a/_pytest/main.py b/_pytest/main.py
--- a/_pytest/main.py
+++ b/_pytest/main.py
@@ -46,23 +46,25 @@
 
 
 def pytest_namespace():
-    return dict(collect=dict(Item=Item, Collector=Collector, File=File))
+    collect = dict(Item=Item, Collector=Collector, File=File, Session=Session)
+    return dict(collect=collect)
         
 def pytest_configure(config):
     py.test.config = config # compatibiltiy
     if config.option.exitfirst:
         config.option.maxfail = 1
 
-def pytest_cmdline_main(config):
-    """ default command line protocol for initialization, session,
-    running tests and reporting. """
+def wrap_session(config, doit):
+    """Skeleton command line program"""
     session = Session(config)
     session.exitstatus = EXIT_OK
+    initstate = 0
     try:
         config.pluginmanager.do_configure(config)
+        initstate = 1
         config.hook.pytest_sessionstart(session=session)
-        config.hook.pytest_collection(session=session)
-        config.hook.pytest_runtestloop(session=session)
+        initstate = 2
+        doit(config, session)
     except pytest.UsageError:
         raise
     except KeyboardInterrupt:
@@ -77,18 +79,24 @@
             sys.stderr.write("mainloop: caught Spurious SystemExit!\n")
     if not session.exitstatus and session._testsfailed:
         session.exitstatus = EXIT_TESTSFAILED
-    config.hook.pytest_sessionfinish(session=session,
-        exitstatus=session.exitstatus)
-    config.pluginmanager.do_unconfigure(config)
+    if initstate >= 2:
+        config.hook.pytest_sessionfinish(session=session,
+            exitstatus=session.exitstatus)
+    if initstate >= 1:
+        config.pluginmanager.do_unconfigure(config)
     return session.exitstatus
 
+def pytest_cmdline_main(config):
+    return wrap_session(config, _main)
+
+def _main(config, session):
+    """ default command line protocol for initialization, session,
+    running tests and reporting. """
+    config.hook.pytest_collection(session=session)
+    config.hook.pytest_runtestloop(session=session)
+
 def pytest_collection(session):
-    session.perform_collect()
-    hook = session.config.hook
-    hook.pytest_collection_modifyitems(session=session,
-        config=session.config, items=session.items)
-    hook.pytest_collection_finish(session=session)
-    return True
+    return session.perform_collect()
 
 def pytest_runtestloop(session):
     if session.config.option.collectonly:
@@ -374,6 +382,16 @@
         return HookProxy(fspath, self.config)
 
     def perform_collect(self, args=None, genitems=True):
+        hook = self.config.hook
+        try:
+            items = self._perform_collect(args, genitems)
+            hook.pytest_collection_modifyitems(session=self,
+                config=self.config, items=items)
+        finally:
+            hook.pytest_collection_finish(session=self)
+        return items
+
+    def _perform_collect(self, args, genitems):
         if args is None:
             args = self.config.args
         self.trace("perform_collect", self, args)
diff --git a/_pytest/mark.py b/_pytest/mark.py
--- a/_pytest/mark.py
+++ b/_pytest/mark.py
@@ -153,7 +153,7 @@
 
     def __repr__(self):
         return "<MarkInfo %r args=%r kwargs=%r>" % (
-                self._name, self.args, self.kwargs)
+                self.name, self.args, self.kwargs)
 
 def pytest_itemcollected(item):
     if not isinstance(item, pytest.Function):
diff --git a/_pytest/pytester.py b/_pytest/pytester.py
--- a/_pytest/pytester.py
+++ b/_pytest/pytester.py
@@ -6,7 +6,7 @@
 import inspect
 import time
 from fnmatch import fnmatch
-from _pytest.main import Session
+from _pytest.main import Session, EXIT_OK
 from py.builtin import print_
 from _pytest.core import HookRelay
 
@@ -292,13 +292,19 @@
         assert '::' not in str(arg)
         p = py.path.local(arg)
         x = session.fspath.bestrelpath(p)
-        return session.perform_collect([x], genitems=False)[0]
+        config.hook.pytest_sessionstart(session=session)
+        res = session.perform_collect([x], genitems=False)[0]
+        config.hook.pytest_sessionfinish(session=session, exitstatus=EXIT_OK)
+        return res
 
     def getpathnode(self, path):
-        config = self.parseconfig(path)
+        config = self.parseconfigure(path)
         session = Session(config)
         x = session.fspath.bestrelpath(path)
-        return session.perform_collect([x], genitems=False)[0]
+        config.hook.pytest_sessionstart(session=session)
+        res = session.perform_collect([x], genitems=False)[0]
+        config.hook.pytest_sessionfinish(session=session, exitstatus=EXIT_OK)
+        return res
 
     def genitems(self, colitems):
         session = colitems[0].session
@@ -312,7 +318,9 @@
         config = self.parseconfigure(*args)
         rec = self.getreportrecorder(config)
         session = Session(config)
+        config.hook.pytest_sessionstart(session=session)
         session.perform_collect()
+        config.hook.pytest_sessionfinish(session=session, exitstatus=EXIT_OK)
         return session.items, rec
 
     def runitem(self, source):
@@ -382,6 +390,8 @@
             c.basetemp = py.path.local.make_numbered_dir(prefix="reparse",
                 keep=0, rootdir=self.tmpdir, lock_timeout=None)
             c.parse(args)
+            c.pluginmanager.do_configure(c)
+            self.request.addfinalizer(lambda: c.pluginmanager.do_unconfigure(c))
             return c
         finally:
             py.test.config = oldconfig
diff --git a/_pytest/python.py b/_pytest/python.py
--- a/_pytest/python.py
+++ b/_pytest/python.py
@@ -226,8 +226,13 @@
 
     def _importtestmodule(self):
         # we assume we are only called once per module
+        from _pytest import assertion
+        assertion.before_module_import(self)
         try:
-            mod = self.fspath.pyimport(ensuresyspath=True)
+            try:
+                mod = self.fspath.pyimport(ensuresyspath=True)
+            finally:
+                assertion.after_module_import(self)
         except SyntaxError:
             excinfo = py.code.ExceptionInfo()
             raise self.CollectError(excinfo.getrepr(style="short"))
@@ -374,7 +379,7 @@
         # test generators are seen as collectors but they also
         # invoke setup/teardown on popular request
         # (induced by the common "test_*" naming shared with normal tests)
-        self.config._setupstate.prepare(self)
+        self.session._setupstate.prepare(self)
         # see FunctionMixin.setup and test_setupstate_is_preserved_134
         self._preservedparent = self.parent.obj
         l = []
@@ -721,7 +726,7 @@
 
     def _addfinalizer(self, finalizer, scope):
         colitem = self._getscopeitem(scope)
-        self.config._setupstate.addfinalizer(
+        self._pyfuncitem.session._setupstate.addfinalizer(
             finalizer=finalizer, colitem=colitem)
 
     def __repr__(self):
@@ -742,8 +747,10 @@
         raise self.LookupError(msg)
 
 def showfuncargs(config):
-    from _pytest.main import Session
-    session = Session(config)
+    from _pytest.main import wrap_session
+    return wrap_session(config, _showfuncargs_main)
+
+def _showfuncargs_main(config, session):
     session.perform_collect()
     if session.items:
         plugins = session.items[0].getplugins()
diff --git a/_pytest/runner.py b/_pytest/runner.py
--- a/_pytest/runner.py
+++ b/_pytest/runner.py
@@ -14,17 +14,15 @@
 #
 # pytest plugin hooks
 
-# XXX move to pytest_sessionstart and fix py.test owns tests
-def pytest_configure(config):
-    config._setupstate = SetupState()
+def pytest_sessionstart(session):
+    session._setupstate = SetupState()
 
 def pytest_sessionfinish(session, exitstatus):
-    if hasattr(session.config, '_setupstate'):
-        hook = session.config.hook
-        rep = hook.pytest__teardown_final(session=session)
-        if rep:
-            hook.pytest__teardown_final_logerror(session=session, report=rep)
-            session.exitstatus = 1
+    hook = session.config.hook
+    rep = hook.pytest__teardown_final(session=session)
+    if rep:
+        hook.pytest__teardown_final_logerror(session=session, report=rep)
+        session.exitstatus = 1
 
 class NodeInfo:
     def __init__(self, location):
@@ -46,16 +44,16 @@
     return reports
 
 def pytest_runtest_setup(item):
-    item.config._setupstate.prepare(item)
+    item.session._setupstate.prepare(item)
 
 def pytest_runtest_call(item):
     item.runtest()
 
 def pytest_runtest_teardown(item):
-    item.config._setupstate.teardown_exact(item)
+    item.session._setupstate.teardown_exact(item)
 
 def pytest__teardown_final(session):
-    call = CallInfo(session.config._setupstate.teardown_all, when="teardown")
+    call = CallInfo(session._setupstate.teardown_all, when="teardown")
     if call.excinfo:
         ntraceback = call.excinfo.traceback .cut(excludepath=py._pydir)
         call.excinfo.traceback = ntraceback.filter()
diff --git a/ctypes_configure/configure.py b/ctypes_configure/configure.py
--- a/ctypes_configure/configure.py
+++ b/ctypes_configure/configure.py
@@ -559,7 +559,9 @@
 C_HEADER = """
 #include <stdio.h>
 #include <stddef.h>   /* for offsetof() */
-#include <stdint.h>   /* FreeBSD: for uint64_t */
+#ifndef _WIN32
+#  include <stdint.h>   /* FreeBSD: for uint64_t */
+#endif
 
 void dump(char* key, int value) {
     printf("%s: %d\\n", key, value);
diff --git a/ctypes_configure/stdoutcapture.py b/ctypes_configure/stdoutcapture.py
--- a/ctypes_configure/stdoutcapture.py
+++ b/ctypes_configure/stdoutcapture.py
@@ -15,6 +15,15 @@
             not hasattr(os, 'fdopen')):
             self.dummy = 1
         else:
+            try:
+                self.tmpout = os.tmpfile()
+                if mixed_out_err:
+                    self.tmperr = self.tmpout
+                else:
+                    self.tmperr = os.tmpfile()
+            except OSError:     # bah?  on at least one Windows box
+                self.dummy = 1
+                return
             self.dummy = 0
             # make new stdout/stderr files if needed
             self.localoutfd = os.dup(1)
@@ -29,11 +38,6 @@
                 sys.stderr = os.fdopen(self.localerrfd, 'w', 0)
             else:
                 self.saved_stderr = None
-            self.tmpout = os.tmpfile()
-            if mixed_out_err:
-                self.tmperr = self.tmpout
-            else:
-                self.tmperr = os.tmpfile()
             os.dup2(self.tmpout.fileno(), 1)
             os.dup2(self.tmperr.fileno(), 2)
 
diff --git a/dotviewer/graphparse.py b/dotviewer/graphparse.py
--- a/dotviewer/graphparse.py
+++ b/dotviewer/graphparse.py
@@ -36,48 +36,45 @@
     print >> sys.stderr, "Warning: could not guess file type, using 'dot'"
     return 'unknown'
 
-def dot2plain(content, contenttype, use_codespeak=False):
-    if contenttype == 'plain':
-        # already a .plain file
-        return content
+def dot2plain_graphviz(content, contenttype, use_codespeak=False):
+    if contenttype != 'neato':
+        cmdline = 'dot -Tplain'
+    else:
+        cmdline = 'neato -Tplain'
+    #print >> sys.stderr, '* running:', cmdline
+    close_fds = sys.platform != 'win32'
+    p = subprocess.Popen(cmdline, shell=True, close_fds=close_fds,
+                         stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+    (child_in, child_out) = (p.stdin, p.stdout)
+    try:
+        import thread
+    except ImportError:
+        bkgndwrite(child_in, content)
+    else:
+        thread.start_new_thread(bkgndwrite, (child_in, content))
+    plaincontent = child_out.read()
+    child_out.close()
+    if not plaincontent:    # 'dot' is likely not installed
+        raise PlainParseError("no result from running 'dot'")
+    return plaincontent
 
-    if not use_codespeak:
-        if contenttype != 'neato':
-            cmdline = 'dot -Tplain'
-        else:
-            cmdline = 'neato -Tplain'
-        #print >> sys.stderr, '* running:', cmdline
-        close_fds = sys.platform != 'win32'
-        p = subprocess.Popen(cmdline, shell=True, close_fds=close_fds,
-                             stdin=subprocess.PIPE, stdout=subprocess.PIPE)
-        (child_in, child_out) = (p.stdin, p.stdout)
-        try:
-            import thread
-        except ImportError:
-            bkgndwrite(child_in, content)
-        else:
-            thread.start_new_thread(bkgndwrite, (child_in, content))
-        plaincontent = child_out.read()
-        child_out.close()
-        if not plaincontent:    # 'dot' is likely not installed
-            raise PlainParseError("no result from running 'dot'")
-    else:
-        import urllib
-        request = urllib.urlencode({'dot': content})
-        url = 'http://codespeak.net/pypy/convertdot.cgi'
-        print >> sys.stderr, '* posting:', url
-        g = urllib.urlopen(url, data=request)
-        result = []
-        while True:
-            data = g.read(16384)
-            if not data:
-                break
-            result.append(data)
-        g.close()
-        plaincontent = ''.join(result)
-        # very simple-minded way to give a somewhat better error message
-        if plaincontent.startswith('<body'):
-            raise Exception("the dot on codespeak has very likely crashed")
+def dot2plain_codespeak(content, contenttype):
+    import urllib
+    request = urllib.urlencode({'dot': content})
+    url = 'http://codespeak.net/pypy/convertdot.cgi'
+    print >> sys.stderr, '* posting:', url
+    g = urllib.urlopen(url, data=request)
+    result = []
+    while True:
+        data = g.read(16384)
+        if not data:
+            break
+        result.append(data)
+    g.close()
+    plaincontent = ''.join(result)
+    # very simple-minded way to give a somewhat better error message
+    if plaincontent.startswith('<body'):
+        raise Exception("the dot on codespeak has very likely crashed")
     return plaincontent
 
 def bkgndwrite(f, data):
@@ -148,10 +145,13 @@
 
 def parse_dot(graph_id, content, links={}, fixedfont=False):
     contenttype = guess_type(content)
-    try:
-        plaincontent = dot2plain(content, contenttype, use_codespeak=False)
-        return list(parse_plain(graph_id, plaincontent, links, fixedfont))
-    except PlainParseError:
-        # failed, retry via codespeak
-        plaincontent = dot2plain(content, contenttype, use_codespeak=True)
-        return list(parse_plain(graph_id, plaincontent, links, fixedfont))
+    if contenttype == 'plain':
+        plaincontent = content
+    else:
+        try:
+            plaincontent = dot2plain_graphviz(content, contenttype)
+        except PlainParseError, e:
+            print e
+            # failed, retry via codespeak
+            plaincontent = dot2plain_codespeak(content, contenttype)
+    return list(parse_plain(graph_id, plaincontent, links, fixedfont))
diff --git a/lib-python/2.7/ssl.py b/lib-python/2.7/ssl.py
--- a/lib-python/2.7/ssl.py
+++ b/lib-python/2.7/ssl.py
@@ -62,7 +62,6 @@
 from _ssl import OPENSSL_VERSION_NUMBER, OPENSSL_VERSION_INFO, OPENSSL_VERSION
 from _ssl import SSLError
 from _ssl import CERT_NONE, CERT_OPTIONAL, CERT_REQUIRED
-from _ssl import PROTOCOL_SSLv2, PROTOCOL_SSLv3, PROTOCOL_SSLv23, PROTOCOL_TLSv1
 from _ssl import RAND_status, RAND_egd, RAND_add
 from _ssl import \
      SSL_ERROR_ZERO_RETURN, \
@@ -74,6 +73,18 @@
      SSL_ERROR_WANT_CONNECT, \
      SSL_ERROR_EOF, \
      SSL_ERROR_INVALID_ERROR_CODE
+from _ssl import PROTOCOL_SSLv3, PROTOCOL_SSLv23, PROTOCOL_TLSv1
+_PROTOCOL_NAMES = {
+    PROTOCOL_TLSv1: "TLSv1",
+    PROTOCOL_SSLv23: "SSLv23",
+    PROTOCOL_SSLv3: "SSLv3",
+}
+try:
+    from _ssl import PROTOCOL_SSLv2
+except ImportError:
+    pass
+else:
+    _PROTOCOL_NAMES[PROTOCOL_SSLv2] = "SSLv2"
 
 from socket import socket, _fileobject, _delegate_methods, error as socket_error
 from socket import getnameinfo as _getnameinfo
@@ -408,16 +419,7 @@
     return DER_cert_to_PEM_cert(dercert)
 
 def get_protocol_name(protocol_code):
-    if protocol_code == PROTOCOL_TLSv1:
-        return "TLSv1"
-    elif protocol_code == PROTOCOL_SSLv23:
-        return "SSLv23"
-    elif protocol_code == PROTOCOL_SSLv2:
-        return "SSLv2"
-    elif protocol_code == PROTOCOL_SSLv3:
-        return "SSLv3"
-    else:
-        return "<unknown>"
+    return _PROTOCOL_NAMES.get(protocol_code, '<unknown>')
 
 
 # a replacement for the old socket.ssl function
diff --git a/lib-python/2.7/test/test_ssl.py b/lib-python/2.7/test/test_ssl.py
--- a/lib-python/2.7/test/test_ssl.py
+++ b/lib-python/2.7/test/test_ssl.py
@@ -58,32 +58,35 @@
 
 # Issue #9415: Ubuntu hijacks their OpenSSL and forcefully disables SSLv2
 def skip_if_broken_ubuntu_ssl(func):
-    # We need to access the lower-level wrapper in order to create an
-    # implicit SSL context without trying to connect or listen.
-    try:
-        import _ssl
-    except ImportError:
-        # The returned function won't get executed, just ignore the error
-        pass
-    @functools.wraps(func)
-    def f(*args, **kwargs):
+    if hasattr(ssl, 'PROTOCOL_SSLv2'):
+        # We need to access the lower-level wrapper in order to create an
+        # implicit SSL context without trying to connect or listen.
         try:
-            s = socket.socket(socket.AF_INET)
-            _ssl.sslwrap(s._sock, 0, None, None,
-                         ssl.CERT_NONE, ssl.PROTOCOL_SSLv2, None, None)
-        except ssl.SSLError as e:
-            if (ssl.OPENSSL_VERSION_INFO == (0, 9, 8, 15, 15) and
-                platform.linux_distribution() == ('debian', 'squeeze/sid', '')
-                and 'Invalid SSL protocol variant specified' in str(e)):
-                raise unittest.SkipTest("Patched Ubuntu OpenSSL breaks behaviour")
-        return func(*args, **kwargs)
-    return f
+            import _ssl
+        except ImportError:
+            # The returned function won't get executed, just ignore the error
+            pass
+        @functools.wraps(func)
+        def f(*args, **kwargs):
+            try:
+                s = socket.socket(socket.AF_INET)
+                _ssl.sslwrap(s._sock, 0, None, None,
+                             ssl.CERT_NONE, ssl.PROTOCOL_SSLv2, None, None)
+            except ssl.SSLError as e:
+                if (ssl.OPENSSL_VERSION_INFO == (0, 9, 8, 15, 15) and
+                    platform.linux_distribution() == ('debian', 'squeeze/sid', '')
+                    and 'Invalid SSL protocol variant specified' in str(e)):
+                    raise unittest.SkipTest("Patched Ubuntu OpenSSL breaks behaviour")
+            return func(*args, **kwargs)
+        return f
+    else:
+        return func
 
 
 class BasicSocketTests(unittest.TestCase):
 
     def test_constants(self):
-        ssl.PROTOCOL_SSLv2
+        #ssl.PROTOCOL_SSLv2
         ssl.PROTOCOL_SSLv23
         ssl.PROTOCOL_SSLv3
         ssl.PROTOCOL_TLSv1
@@ -964,7 +967,8 @@
             try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, True)
             try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, True, ssl.CERT_OPTIONAL)
             try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, True, ssl.CERT_REQUIRED)
-            try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv2, False)
+            if hasattr(ssl, 'PROTOCOL_SSLv2'):
+                try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv2, False)
             try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv23, False)
             try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_TLSv1, False)
 
@@ -976,7 +980,8 @@
             try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, True)
             try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, True, ssl.CERT_OPTIONAL)
             try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, True, ssl.CERT_REQUIRED)
-            try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv2, False)
+            if hasattr(ssl, 'PROTOCOL_SSLv2'):
+                try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv2, False)
             try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv3, False)
             try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv23, False)
 
diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -154,18 +154,18 @@
     RegrTest('test_cmd.py'),
     RegrTest('test_cmd_line_script.py'),
     RegrTest('test_codeccallbacks.py', core=True),
-    RegrTest('test_codecencodings_cn.py'),
-    RegrTest('test_codecencodings_hk.py'),
-    RegrTest('test_codecencodings_jp.py'),
-    RegrTest('test_codecencodings_kr.py'),
-    RegrTest('test_codecencodings_tw.py'),
+    RegrTest('test_codecencodings_cn.py', usemodules='_multibytecodec'),
+    RegrTest('test_codecencodings_hk.py', usemodules='_multibytecodec'),
+    RegrTest('test_codecencodings_jp.py', usemodules='_multibytecodec'),
+    RegrTest('test_codecencodings_kr.py', usemodules='_multibytecodec'),
+    RegrTest('test_codecencodings_tw.py', usemodules='_multibytecodec'),
 
-    RegrTest('test_codecmaps_cn.py'),
-    RegrTest('test_codecmaps_hk.py'),
-    RegrTest('test_codecmaps_jp.py'),
-    RegrTest('test_codecmaps_kr.py'),
-    RegrTest('test_codecmaps_tw.py'),
-    RegrTest('test_codecs.py', core=True),
+    RegrTest('test_codecmaps_cn.py', usemodules='_multibytecodec'),
+    RegrTest('test_codecmaps_hk.py', usemodules='_multibytecodec'),
+    RegrTest('test_codecmaps_jp.py', usemodules='_multibytecodec'),
+    RegrTest('test_codecmaps_kr.py', usemodules='_multibytecodec'),
+    RegrTest('test_codecmaps_tw.py', usemodules='_multibytecodec'),
+    RegrTest('test_codecs.py', core=True, usemodules='_multibytecodec'),
     RegrTest('test_codeop.py', core=True),
     RegrTest('test_coercion.py', core=True),
     RegrTest('test_collections.py'),
@@ -314,10 +314,10 @@
     RegrTest('test_mmap.py'),
     RegrTest('test_module.py', core=True),
     RegrTest('test_modulefinder.py'),
-    RegrTest('test_multibytecodec.py'),
+    RegrTest('test_multibytecodec.py', usemodules='_multibytecodec'),
     RegrTest('test_multibytecodec_support.py', skip="not a test"),
     RegrTest('test_multifile.py'),
-    RegrTest('test_multiprocessing.py', skip='FIXME leaves subprocesses'),
+    RegrTest('test_multiprocessing.py', skip="FIXME leaves subprocesses"),
     RegrTest('test_mutants.py', core="possibly"),
     RegrTest('test_mutex.py'),
     RegrTest('test_netrc.py'),
@@ -359,7 +359,7 @@
     RegrTest('test_property.py', core=True),
     RegrTest('test_pstats.py'),
     RegrTest('test_pty.py', skip="unsupported extension module"),
-    RegrTest('test_pwd.py', skip=skip_win32),
+    RegrTest('test_pwd.py', usemodules="pwd", skip=skip_win32),
     RegrTest('test_py3kwarn.py'),
     RegrTest('test_pyclbr.py'),
     RegrTest('test_pydoc.py'),
@@ -569,7 +569,6 @@
 #
 import os
 import time
-import socket
 import getpass
 
 class ReallyRunFileExternal(py.test.collect.Item): 
diff --git a/lib-python/modified-2.7/ctypes/__init__.py b/lib-python/modified-2.7/ctypes/__init__.py
--- a/lib-python/modified-2.7/ctypes/__init__.py
+++ b/lib-python/modified-2.7/ctypes/__init__.py
@@ -7,6 +7,7 @@
 
 __version__ = "1.1.0"
 
+import _ffi
 from _ctypes import Union, Structure, Array
 from _ctypes import _Pointer
 from _ctypes import CFuncPtr as _CFuncPtr
@@ -350,7 +351,7 @@
         self._FuncPtr = _FuncPtr
 
         if handle is None:
-            self._handle = _dlopen(self._name, mode)
+            self._handle = _ffi.CDLL(name)
         else:
             self._handle = handle
 
@@ -488,9 +489,12 @@
         _flags_ = _FUNCFLAG_CDECL | _FUNCFLAG_PYTHONAPI
     return CFunctionType
 
-_cast = PYFUNCTYPE(py_object, c_void_p, py_object, py_object)(_cast_addr)
 def cast(obj, typ):
-    return _cast(obj, obj, typ)
+    try:
+        c_void_p.from_param(obj)
+    except TypeError, e:
+        raise ArgumentError(str(e))
+    return _cast_addr(obj, obj, typ)
 
 _string_at = PYFUNCTYPE(py_object, c_void_p, c_int)(_string_at_addr)
 def string_at(ptr, size=-1):
diff --git a/lib-python/modified-2.7/ctypes/test/test_cfuncs.py b/lib-python/modified-2.7/ctypes/test/test_cfuncs.py
--- a/lib-python/modified-2.7/ctypes/test/test_cfuncs.py
+++ b/lib-python/modified-2.7/ctypes/test/test_cfuncs.py
@@ -3,8 +3,8 @@
 
 import unittest
 from ctypes import *
-
 import _ctypes_test
+from test.test_support import impl_detail
 
 class CFunctions(unittest.TestCase):
     _dll = CDLL(_ctypes_test.__file__)
@@ -158,12 +158,14 @@
         self.assertEqual(self._dll.tf_bd(0, 42.), 14.)
         self.assertEqual(self.S(), 42)
 
+    @impl_detail('long double not supported by PyPy', pypy=False)
     def test_longdouble(self):
         self._dll.tf_D.restype = c_longdouble
         self._dll.tf_D.argtypes = (c_longdouble,)
         self.assertEqual(self._dll.tf_D(42.), 14.)
         self.assertEqual(self.S(), 42)
-
+        
+    @impl_detail('long double not supported by PyPy', pypy=False)
     def test_longdouble_plus(self):
         self._dll.tf_bD.restype = c_longdouble
         self._dll.tf_bD.argtypes = (c_byte, c_longdouble)
diff --git a/lib-python/modified-2.7/ctypes/test/test_functions.py b/lib-python/modified-2.7/ctypes/test/test_functions.py
--- a/lib-python/modified-2.7/ctypes/test/test_functions.py
+++ b/lib-python/modified-2.7/ctypes/test/test_functions.py
@@ -8,6 +8,7 @@
 from ctypes import *
 import sys, unittest
 from ctypes.test import xfail
+from test.test_support import impl_detail
 
 try:
     WINFUNCTYPE
@@ -144,6 +145,7 @@
         self.assertEqual(result, -21)
         self.assertEqual(type(result), float)
 
+    @impl_detail('long double not supported by PyPy', pypy=False)
     def test_longdoubleresult(self):
         f = dll._testfunc_D_bhilfD
         f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_longdouble]
diff --git a/lib-python/modified-2.7/ctypes/test/test_libc.py b/lib-python/modified-2.7/ctypes/test/test_libc.py
--- a/lib-python/modified-2.7/ctypes/test/test_libc.py
+++ b/lib-python/modified-2.7/ctypes/test/test_libc.py
@@ -26,6 +26,7 @@
         self.assertEqual(chars.raw, "   ,,aaaadmmmnpppsss\x00")
 
     def test_no_more_xfail(self):
+        import socket
         import ctypes.test
         self.assertTrue(not hasattr(ctypes.test, 'xfail'),
                         "You should incrementally grep for '@xfail' and remove them, they are real failures")
diff --git a/lib-python/modified-2.7/ctypes/util.py b/lib-python/modified-2.7/ctypes/util.py
--- a/lib-python/modified-2.7/ctypes/util.py
+++ b/lib-python/modified-2.7/ctypes/util.py
@@ -72,8 +72,8 @@
         return name
 
 if os.name == "posix" and sys.platform == "darwin":
-    from ctypes.macholib.dyld import dyld_find as _dyld_find
     def find_library(name):
+        from ctypes.macholib.dyld import dyld_find as _dyld_find
         possible = ['lib%s.dylib' % name,
                     '%s.dylib' % name,
                     '%s.framework/%s' % (name, name)]
diff --git a/lib-python/modified-2.7/distutils/cygwinccompiler.py b/lib-python/modified-2.7/distutils/cygwinccompiler.py
--- a/lib-python/modified-2.7/distutils/cygwinccompiler.py
+++ b/lib-python/modified-2.7/distutils/cygwinccompiler.py
@@ -75,6 +75,9 @@
         elif msc_ver == '1500':
             # VS2008 / MSVC 9.0
             return ['msvcr90']
+        elif msc_ver == '1600':
+            # VS2010 / MSVC 10.0
+            return ['msvcr100']
         else:
             raise ValueError("Unknown MS Compiler version %s " % msc_ver)
 
diff --git a/lib-python/modified-2.7/distutils/sysconfig_pypy.py b/lib-python/modified-2.7/distutils/sysconfig_pypy.py
--- a/lib-python/modified-2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/modified-2.7/distutils/sysconfig_pypy.py
@@ -116,6 +116,12 @@
     if compiler.compiler_type == "unix":
         compiler.compiler_so.extend(['-fPIC', '-Wimplicit'])
         compiler.shared_lib_extension = get_config_var('SO')
+        if "CFLAGS" in os.environ:
+            cflags = os.environ["CFLAGS"]
+            compiler.compiler.append(cflags)
+            compiler.compiler_so.append(cflags)
+            compiler.linker_so.append(cflags)
+
 
 from sysconfig_cpython import (
     parse_makefile, _variable_rx, expand_makefile_vars)
diff --git a/lib-python/modified-2.7/distutils/unixccompiler.py b/lib-python/modified-2.7/distutils/unixccompiler.py
--- a/lib-python/modified-2.7/distutils/unixccompiler.py
+++ b/lib-python/modified-2.7/distutils/unixccompiler.py
@@ -324,7 +324,7 @@
             # On OSX users can specify an alternate SDK using
             # '-isysroot', calculate the SDK root if it is specified
             # (and use it further on)
-            cflags = sysconfig.get_config_var('CFLAGS')
+            cflags = sysconfig.get_config_var('CFLAGS') or ''
             m = re.search(r'-isysroot\s+(\S+)', cflags)
             if m is None:
                 sysroot = '/'
diff --git a/lib-python/modified-2.7/httplib.py b/lib-python/modified-2.7/httplib.py
new file mode 100644
--- /dev/null
+++ b/lib-python/modified-2.7/httplib.py
@@ -0,0 +1,1377 @@
+"""HTTP/1.1 client library
+
+<intro stuff goes here>
+<other stuff, too>
+
+HTTPConnection goes through a number of "states", which define when a client
+may legally make another request or fetch the response for a particular
+request. This diagram details these state transitions:
+
+    (null)
+      |
+      | HTTPConnection()
+      v
+    Idle
+      |
+      | putrequest()
+      v
+    Request-started
+      |
+      | ( putheader() )*  endheaders()
+      v
+    Request-sent
+      |
+      | response = getresponse()
+      v
+    Unread-response   [Response-headers-read]
+      |\____________________
+      |                     |
+      | response.read()     | putrequest()
+      v                     v
+    Idle                  Req-started-unread-response
+                     ______/|
+                   /        |
+   response.read() |        | ( putheader() )*  endheaders()
+                   v        v
+       Request-started    Req-sent-unread-response
+                            |
+                            | response.read()
+                            v
+                          Request-sent
+
+This diagram presents the following rules:
+  -- a second request may not be started until {response-headers-read}
+  -- a response [object] cannot be retrieved until {request-sent}
+  -- there is no differentiation between an unread response body and a
+     partially read response body
+
+Note: this enforcement is applied by the HTTPConnection class. The
+      HTTPResponse class does not enforce this state machine, which
+      implies sophisticated clients may accelerate the request/response
+      pipeline. Caution should be taken, though: accelerating the states
+      beyond the above pattern may imply knowledge of the server's
+      connection-close behavior for certain requests. For example, it
+      is impossible to tell whether the server will close the connection
+      UNTIL the response headers have been read; this means that further
+      requests cannot be placed into the pipeline until it is known that
+      the server will NOT be closing the connection.
+
+Logical State                  __state            __response
+-------------                  -------            ----------
+Idle                           _CS_IDLE           None
+Request-started                _CS_REQ_STARTED    None
+Request-sent                   _CS_REQ_SENT       None
+Unread-response                _CS_IDLE           <response_class>
+Req-started-unread-response    _CS_REQ_STARTED    <response_class>
+Req-sent-unread-response       _CS_REQ_SENT       <response_class>
+"""
+
+from array import array
+import os
+import socket
+from sys import py3kwarning
+from urlparse import urlsplit
+import warnings
+with warnings.catch_warnings():
+    if py3kwarning:
+        warnings.filterwarnings("ignore", ".*mimetools has been removed",
+                                DeprecationWarning)
+    import mimetools
+
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from StringIO import StringIO
+
+__all__ = ["HTTP", "HTTPResponse", "HTTPConnection",
+           "HTTPException", "NotConnected", "UnknownProtocol",
+           "UnknownTransferEncoding", "UnimplementedFileMode",
+           "IncompleteRead", "InvalidURL", "ImproperConnectionState",
+           "CannotSendRequest", "CannotSendHeader", "ResponseNotReady",
+           "BadStatusLine", "error", "responses"]
+
+HTTP_PORT = 80
+HTTPS_PORT = 443
+
+_UNKNOWN = 'UNKNOWN'
+
+# connection states
+_CS_IDLE = 'Idle'
+_CS_REQ_STARTED = 'Request-started'
+_CS_REQ_SENT = 'Request-sent'
+
+# status codes
+# informational
+CONTINUE = 100
+SWITCHING_PROTOCOLS = 101
+PROCESSING = 102
+
+# successful
+OK = 200
+CREATED = 201
+ACCEPTED = 202
+NON_AUTHORITATIVE_INFORMATION = 203
+NO_CONTENT = 204
+RESET_CONTENT = 205
+PARTIAL_CONTENT = 206
+MULTI_STATUS = 207
+IM_USED = 226
+
+# redirection
+MULTIPLE_CHOICES = 300
+MOVED_PERMANENTLY = 301
+FOUND = 302
+SEE_OTHER = 303
+NOT_MODIFIED = 304
+USE_PROXY = 305
+TEMPORARY_REDIRECT = 307
+
+# client error
+BAD_REQUEST = 400
+UNAUTHORIZED = 401
+PAYMENT_REQUIRED = 402
+FORBIDDEN = 403
+NOT_FOUND = 404
+METHOD_NOT_ALLOWED = 405
+NOT_ACCEPTABLE = 406
+PROXY_AUTHENTICATION_REQUIRED = 407
+REQUEST_TIMEOUT = 408
+CONFLICT = 409
+GONE = 410
+LENGTH_REQUIRED = 411
+PRECONDITION_FAILED = 412
+REQUEST_ENTITY_TOO_LARGE = 413
+REQUEST_URI_TOO_LONG = 414
+UNSUPPORTED_MEDIA_TYPE = 415
+REQUESTED_RANGE_NOT_SATISFIABLE = 416
+EXPECTATION_FAILED = 417
+UNPROCESSABLE_ENTITY = 422
+LOCKED = 423
+FAILED_DEPENDENCY = 424
+UPGRADE_REQUIRED = 426
+
+# server error
+INTERNAL_SERVER_ERROR = 500
+NOT_IMPLEMENTED = 501
+BAD_GATEWAY = 502
+SERVICE_UNAVAILABLE = 503
+GATEWAY_TIMEOUT = 504
+HTTP_VERSION_NOT_SUPPORTED = 505
+INSUFFICIENT_STORAGE = 507
+NOT_EXTENDED = 510
+
+# Mapping status codes to official W3C names
+responses = {
+    100: 'Continue',
+    101: 'Switching Protocols',
+
+    200: 'OK',
+    201: 'Created',
+    202: 'Accepted',
+    203: 'Non-Authoritative Information',
+    204: 'No Content',
+    205: 'Reset Content',
+    206: 'Partial Content',
+
+    300: 'Multiple Choices',
+    301: 'Moved Permanently',
+    302: 'Found',
+    303: 'See Other',
+    304: 'Not Modified',
+    305: 'Use Proxy',
+    306: '(Unused)',
+    307: 'Temporary Redirect',
+
+    400: 'Bad Request',
+    401: 'Unauthorized',
+    402: 'Payment Required',
+    403: 'Forbidden',
+    404: 'Not Found',
+    405: 'Method Not Allowed',
+    406: 'Not Acceptable',
+    407: 'Proxy Authentication Required',
+    408: 'Request Timeout',
+    409: 'Conflict',
+    410: 'Gone',
+    411: 'Length Required',
+    412: 'Precondition Failed',
+    413: 'Request Entity Too Large',
+    414: 'Request-URI Too Long',
+    415: 'Unsupported Media Type',
+    416: 'Requested Range Not Satisfiable',
+    417: 'Expectation Failed',
+
+    500: 'Internal Server Error',
+    501: 'Not Implemented',
+    502: 'Bad Gateway',
+    503: 'Service Unavailable',
+    504: 'Gateway Timeout',
+    505: 'HTTP Version Not Supported',
+}
+
+# maximal amount of data to read at one time in _safe_read
+MAXAMOUNT = 1048576
+
+class HTTPMessage(mimetools.Message):
+
+    def addheader(self, key, value):
+        """Add header for field key handling repeats."""
+        prev = self.dict.get(key)
+        if prev is None:
+            self.dict[key] = value
+        else:
+            combined = ", ".join((prev, value))
+            self.dict[key] = combined
+
+    def addcontinue(self, key, more):
+        """Add more field data from a continuation line."""
+        prev = self.dict[key]
+        self.dict[key] = prev + "\n " + more
+
+    def readheaders(self):
+        """Read header lines.
+
+        Read header lines up to the entirely blank line that terminates them.
+        The (normally blank) line that ends the headers is skipped, but not
+        included in the returned list.  If a non-header line ends the headers,
+        (which is an error), an attempt is made to backspace over it; it is
+        never included in the returned list.
+
+        The variable self.status is set to the empty string if all went well,
+        otherwise it is an error message.  The variable self.headers is a
+        completely uninterpreted list of lines contained in the header (so
+        printing them will reproduce the header exactly as it appears in the
+        file).
+
+        If multiple header fields with the same name occur, they are combined
+        according to the rules in RFC 2616 sec 4.2:
+
+        Appending each subsequent field-value to the first, each separated
+        by a comma. The order in which header fields with the same field-name
+        are received is significant to the interpretation of the combined
+        field value.
+        """
+        # XXX The implementation overrides the readheaders() method of
+        # rfc822.Message.  The base class design isn't amenable to
+        # customized behavior here so the method here is a copy of the
+        # base class code with a few small changes.
+
+        self.dict = {}
+        self.unixfrom = ''
+        self.headers = hlist = []
+        self.status = ''
+        headerseen = ""
+        firstline = 1
+        startofline = unread = tell = None
+        if hasattr(self.fp, 'unread'):
+            unread = self.fp.unread
+        elif self.seekable:
+            tell = self.fp.tell
+        while True:
+            if tell:
+                try:
+                    startofline = tell()
+                except IOError:
+                    startofline = tell = None
+                    self.seekable = 0
+            line = self.fp.readline()
+            if not line:
+                self.status = 'EOF in headers'
+                break
+            # Skip unix From name time lines
+            if firstline and line.startswith('From '):
+                self.unixfrom = self.unixfrom + line
+                continue
+            firstline = 0
+            if headerseen and line[0] in ' \t':
+                # XXX Not sure if continuation lines are handled properly
+                # for http and/or for repeating headers
+                # It's a continuation line.
+                hlist.append(line)
+                self.addcontinue(headerseen, line.strip())
+                continue
+            elif self.iscomment(line):
+                # It's a comment.  Ignore it.
+                continue
+            elif self.islast(line):
+                # Note! No pushback here!  The delimiter line gets eaten.
+                break
+            headerseen = self.isheader(line)
+            if headerseen:
+                # It's a legal header line, save it.
+                hlist.append(line)
+                self.addheader(headerseen, line[len(headerseen)+1:].strip())
+                continue
+            else:
+                # It's not a header line; throw it back and stop here.
+                if not self.dict:
+                    self.status = 'No headers'
+                else:
+                    self.status = 'Non-header line where header expected'
+                # Try to undo the read.
+                if unread:
+                    unread(line)
+                elif tell:
+                    self.fp.seek(startofline)
+                else:
+                    self.status = self.status + '; bad seek'
+                break
+
+class HTTPResponse:
+
+    # strict: If true, raise BadStatusLine if the status line can't be
+    # parsed as a valid HTTP/1.0 or 1.1 status line.  By default it is
+    # false because it prevents clients from talking to HTTP/0.9
+    # servers.  Note that a response with a sufficiently corrupted
+    # status line will look like an HTTP/0.9 response.
+
+    # See RFC 2616 sec 19.6 and RFC 1945 sec 6 for details.
+
+    def __init__(self, sock, debuglevel=0, strict=0, method=None, buffering=False):
+        if buffering:
+            # The caller won't be using any sock.recv() calls, so buffering
+            # is fine and recommended for performance.
+            self.fp = sock.makefile('rb')
+        else:
+            # The buffer size is specified as zero, because the headers of
+            # the response are read with readline().  If the reads were
+            # buffered the readline() calls could consume some of the
+            # response, which make be read via a recv() on the underlying
+            # socket.
+            self.fp = sock.makefile('rb', 0)
+        self.debuglevel = debuglevel
+        self.strict = strict
+        self._method = method
+
+        self.msg = None
+
+        # from the Status-Line of the response
+        self.version = _UNKNOWN # HTTP-Version
+        self.status = _UNKNOWN  # Status-Code
+        self.reason = _UNKNOWN  # Reason-Phrase
+
+        self.chunked = _UNKNOWN         # is "chunked" being used?
+        self.chunk_left = _UNKNOWN      # bytes left to read in current chunk
+        self.length = _UNKNOWN          # number of bytes left in response
+        self.will_close = _UNKNOWN      # conn will close at end of response
+
+    def _read_status(self):
+        # Initialize with Simple-Response defaults
+        line = self.fp.readline()
+        if self.debuglevel > 0:
+            print "reply:", repr(line)
+        if not line:
+            # Presumably, the server closed the connection before
+            # sending a valid response.
+            raise BadStatusLine(line)
+        try:
+            [version, status, reason] = line.split(None, 2)
+        except ValueError:
+            try:
+                [version, status] = line.split(None, 1)
+                reason = ""
+            except ValueError:
+                # empty version will cause next test to fail and status
+                # will be treated as 0.9 response.
+                version = ""
+        if not version.startswith('HTTP/'):
+            if self.strict:
+                self.close()
+                raise BadStatusLine(line)
+            else:
+                # assume it's a Simple-Response from an 0.9 server
+                self.fp = LineAndFileWrapper(line, self.fp)
+                return "HTTP/0.9", 200, ""
+
+        # The status code is a three-digit number
+        try:
+            status = int(status)
+            if status < 100 or status > 999:
+                raise BadStatusLine(line)
+        except ValueError:
+            raise BadStatusLine(line)
+        return version, status, reason
+
+    def begin(self):
+        if self.msg is not None:
+            # we've already started reading the response
+            return
+
+        # read until we get a non-100 response
+        while True:
+            version, status, reason = self._read_status()
+            if status != CONTINUE:
+                break
+            # skip the header from the 100 response
+            while True:
+                skip = self.fp.readline().strip()
+                if not skip:
+                    break
+                if self.debuglevel > 0:
+                    print "header:", skip
+
+        self.status = status
+        self.reason = reason.strip()
+        if version == 'HTTP/1.0':
+            self.version = 10
+        elif version.startswith('HTTP/1.'):
+            self.version = 11   # use HTTP/1.1 code for HTTP/1.x where x>=1
+        elif version == 'HTTP/0.9':
+            self.version = 9
+        else:
+            raise UnknownProtocol(version)
+
+        if self.version == 9:
+            self.length = None
+            self.chunked = 0
+            self.will_close = 1
+            self.msg = HTTPMessage(StringIO())
+            return
+
+        self.msg = HTTPMessage(self.fp, 0)
+        if self.debuglevel > 0:
+            for hdr in self.msg.headers:
+                print "header:", hdr,
+
+        # don't let the msg keep an fp
+        self.msg.fp = None
+
+        # are we using the chunked-style of transfer encoding?
+        tr_enc = self.msg.getheader('transfer-encoding')
+        if tr_enc and tr_enc.lower() == "chunked":
+            self.chunked = 1
+            self.chunk_left = None
+        else:
+            self.chunked = 0
+
+        # will the connection close at the end of the response?
+        self.will_close = self._check_close()
+
+        # do we have a Content-Length?
+        # NOTE: RFC 2616, S4.4, #3 says we ignore this if tr_enc is "chunked"
+        length = self.msg.getheader('content-length')
+        if length and not self.chunked:
+            try:
+                self.length = int(length)
+            except ValueError:
+                self.length = None
+            else:
+                if self.length < 0:  # ignore nonsensical negative lengths
+                    self.length = None
+        else:
+            self.length = None
+
+        # does the body have a fixed length? (of zero)
+        if (status == NO_CONTENT or status == NOT_MODIFIED or
+            100 <= status < 200 or      # 1xx codes
+            self._method == 'HEAD'):
+            self.length = 0
+
+        # if the connection remains open, and we aren't using chunked, and
+        # a content-length was not provided, then assume that the connection
+        # WILL close.
+        if not self.will_close and \
+           not self.chunked and \
+           self.length is None:
+            self.will_close = 1
+
+    def _check_close(self):
+        conn = self.msg.getheader('connection')
+        if self.version == 11:
+            # An HTTP/1.1 proxy is assumed to stay open unless
+            # explicitly closed.
+            conn = self.msg.getheader('connection')
+            if conn and "close" in conn.lower():
+                return True
+            return False
+
+        # Some HTTP/1.0 implementations have support for persistent
+        # connections, using rules different than HTTP/1.1.
+
+        # For older HTTP, Keep-Alive indicates persistent connection.
+        if self.msg.getheader('keep-alive'):
+            return False
+
+        # At least Akamai returns a "Connection: Keep-Alive" header,
+        # which was supposed to be sent by the client.
+        if conn and "keep-alive" in conn.lower():
+            return False
+
+        # Proxy-Connection is a netscape hack.
+        pconn = self.msg.getheader('proxy-connection')
+        if pconn and "keep-alive" in pconn.lower():
+            return False
+
+        # otherwise, assume it will close
+        return True
+
+    def close(self):
+        if self.fp:
+            self.fp.close()
+            self.fp = None
+
+    def isclosed(self):
+        # NOTE: it is possible that we will not ever call self.close(). This
+        #       case occurs when will_close is TRUE, length is None, and we
+        #       read up to the last byte, but NOT past it.
+        #
+        # IMPLIES: if will_close is FALSE, then self.close() will ALWAYS be
+        #          called, meaning self.isclosed() is meaningful.
+        return self.fp is None
+
+    # XXX It would be nice to have readline and __iter__ for this, too.
+
+    def read(self, amt=None):
+        if self.fp is None:
+            return ''
+
+        if self._method == 'HEAD':
+            self.close()
+            return ''
+
+        if self.chunked:
+            return self._read_chunked(amt)
+
+        if amt is None:
+            # unbounded read
+            if self.length is None:
+                s = self.fp.read()
+            else:
+                s = self._safe_read(self.length)
+                self.length = 0
+            self.close()        # we read everything
+            return s
+
+        if self.length is not None:
+            if amt > self.length:
+                # clip the read to the "end of response"
+                amt = self.length
+
+        # we do not use _safe_read() here because this may be a .will_close
+        # connection, and the user is reading more bytes than will be provided
+        # (for example, reading in 1k chunks)
+        s = self.fp.read(amt)
+        if self.length is not None:
+            self.length -= len(s)
+            if not self.length:
+                self.close()
+        return s
+
+    def _read_chunked(self, amt):
+        assert self.chunked != _UNKNOWN
+        chunk_left = self.chunk_left
+        value = []
+        while True:
+            if chunk_left is None:
+                line = self.fp.readline()
+                i = line.find(';')
+                if i >= 0:
+                    line = line[:i] # strip chunk-extensions
+                try:
+                    chunk_left = int(line, 16)
+                except ValueError:
+                    # close the connection as protocol synchronisation is
+                    # probably lost
+                    self.close()
+                    raise IncompleteRead(''.join(value))
+                if chunk_left == 0:
+                    break
+            if amt is None:
+                value.append(self._safe_read(chunk_left))
+            elif amt < chunk_left:
+                value.append(self._safe_read(amt))
+                self.chunk_left = chunk_left - amt
+                return ''.join(value)
+            elif amt == chunk_left:
+                value.append(self._safe_read(amt))
+                self._safe_read(2)  # toss the CRLF at the end of the chunk
+                self.chunk_left = None
+                return ''.join(value)
+            else:
+                value.append(self._safe_read(chunk_left))
+                amt -= chunk_left
+
+            # we read the whole chunk, get another
+            self._safe_read(2)      # toss the CRLF at the end of the chunk
+            chunk_left = None
+
+        # read and discard trailer up to the CRLF terminator
+        ### note: we shouldn't have any trailers!
+        while True:
+            line = self.fp.readline()
+            if not line:
+                # a vanishingly small number of sites EOF without
+                # sending the trailer
+                break
+            if line == '\r\n':
+                break
+
+        # we read everything; close the "file"
+        self.close()
+
+        return ''.join(value)
+
+    def _safe_read(self, amt):
+        """Read the number of bytes requested, compensating for partial reads.
+
+        Normally, we have a blocking socket, but a read() can be interrupted
+        by a signal (resulting in a partial read).
+
+        Note that we cannot distinguish between EOF and an interrupt when zero
+        bytes have been read. IncompleteRead() will be raised in this
+        situation.
+
+        This function should be used when <amt> bytes "should" be present for
+        reading. If the bytes are truly not available (due to EOF), then the
+        IncompleteRead exception can be used to detect the problem.
+        """
+        # NOTE(gps): As of svn r74426 socket._fileobject.read(x) will never
+        # return less than x bytes unless EOF is encountered.  It now handles
+        # signal interruptions (socket.error EINTR) internally.  This code
+        # never caught that exception anyways.  It seems largely pointless.
+        # self.fp.read(amt) will work fine.
+        s = []
+        while amt > 0:
+            chunk = self.fp.read(min(amt, MAXAMOUNT))
+            if not chunk:
+                raise IncompleteRead(''.join(s), amt)
+            s.append(chunk)
+            amt -= len(chunk)
+        return ''.join(s)
+
+    def fileno(self):
+        return self.fp.fileno()
+
+    def getheader(self, name, default=None):
+        if self.msg is None:
+            raise ResponseNotReady()
+        return self.msg.getheader(name, default)
+
+    def getheaders(self):
+        """Return list of (header, value) tuples."""
+        if self.msg is None:
+            raise ResponseNotReady()
+        return self.msg.items()
+
+
+class HTTPConnection:
+
+    _http_vsn = 11
+    _http_vsn_str = 'HTTP/1.1'
+
+    response_class = HTTPResponse
+    default_port = HTTP_PORT
+    auto_open = 1
+    debuglevel = 0
+    strict = 0
+
+    def __init__(self, host, port=None, strict=None,
+                 timeout=socket._GLOBAL_DEFAULT_TIMEOUT, source_address=None):
+        self.timeout = timeout
+        self.source_address = source_address
+        self.sock = None
+        self._buffer = []
+        self.__response = None
+        self.__state = _CS_IDLE
+        self._method = None
+        self._tunnel_host = None
+        self._tunnel_port = None
+        self._tunnel_headers = {}
+
+        self._set_hostport(host, port)
+        if strict is not None:
+            self.strict = strict
+
+    def set_tunnel(self, host, port=None, headers=None):
+        """ Sets up the host and the port for the HTTP CONNECT Tunnelling.
+
+        The headers argument should be a mapping of extra HTTP headers
+        to send with the CONNECT request.
+        """
+        self._tunnel_host = host
+        self._tunnel_port = port
+        if headers:
+            self._tunnel_headers = headers
+        else:
+            self._tunnel_headers.clear()
+
+    def _set_hostport(self, host, port):
+        if port is None:
+            i = host.rfind(':')
+            j = host.rfind(']')         # ipv6 addresses have [...]
+            if i > j:
+                try:
+                    port = int(host[i+1:])
+                except ValueError:
+                    raise InvalidURL("nonnumeric port: '%s'" % host[i+1:])
+                host = host[:i]
+            else:
+                port = self.default_port
+            if host and host[0] == '[' and host[-1] == ']':
+                host = host[1:-1]
+        self.host = host
+        self.port = port
+
+    def set_debuglevel(self, level):
+        self.debuglevel = level
+
+    def _tunnel(self):
+        self._set_hostport(self._tunnel_host, self._tunnel_port)
+        self.send("CONNECT %s:%d HTTP/1.0\r\n" % (self.host, self.port))
+        for header, value in self._tunnel_headers.iteritems():
+            self.send("%s: %s\r\n" % (header, value))
+        self.send("\r\n")
+        response = self.response_class(self.sock, strict = self.strict,
+                                       method = self._method)
+        (version, code, message) = response._read_status()
+
+        if code != 200:
+            self.close()
+            raise socket.error("Tunnel connection failed: %d %s" % (code,
+                                                                    message.strip()))
+        while True:
+            line = response.fp.readline()
+            if line == '\r\n': break
+
+
+    def connect(self):
+        """Connect to the host and port specified in __init__."""
+        self.sock = socket.create_connection((self.host,self.port),
+                                             self.timeout, self.source_address)
+
+        if self._tunnel_host:
+            self._tunnel()
+
+    def close(self):
+        """Close the connection to the HTTP server."""
+        if self.sock:
+            self.sock.close()   # close it manually... there may be other refs
+            self.sock = None
+        if self.__response:
+            self.__response.close()
+            self.__response = None
+        self.__state = _CS_IDLE
+
+    def send(self, data):
+        """Send `data' to the server."""
+        if self.sock is None:
+            if self.auto_open:
+                self.connect()
+            else:
+                raise NotConnected()
+
+        if self.debuglevel > 0:
+            print "send:", repr(data)
+        blocksize = 8192
+        if hasattr(data,'read') and not isinstance(data, array):
+            if self.debuglevel > 0: print "sendIng a read()able"
+            datablock = data.read(blocksize)
+            while datablock:
+                self.sock.sendall(datablock)
+                datablock = data.read(blocksize)
+        else:
+            self.sock.sendall(data)
+
+    def _output(self, s):
+        """Add a line of output to the current request buffer.
+
+        Assumes that the line does *not* end with \\r\\n.
+        """
+        self._buffer.append(s)
+
+    def _send_output(self, message_body=None):
+        """Send the currently buffered request and clear the buffer.
+
+        Appends an extra \\r\\n to the buffer.
+        A message_body may be specified, to be appended to the request.
+        """
+        self._buffer.extend(("", ""))
+        msg = "\r\n".join(self._buffer)
+        del self._buffer[:]
+        # If msg and message_body are sent in a single send() call,
+        # it will avoid performance problems caused by the interaction
+        # between delayed ack and the Nagle algorithim.
+        if isinstance(message_body, str):
+            msg += message_body
+            message_body = None
+        self.send(msg)
+        if message_body is not None:
+            #message_body was not a string (i.e. it is a file) and
+            #we must run the risk of Nagle
+            self.send(message_body)
+
+    def putrequest(self, method, url, skip_host=0, skip_accept_encoding=0):
+        """Send a request to the server.
+
+        `method' specifies an HTTP request method, e.g. 'GET'.
+        `url' specifies the object being requested, e.g. '/index.html'.
+        `skip_host' if True does not add automatically a 'Host:' header
+        `skip_accept_encoding' if True does not add automatically an
+           'Accept-Encoding:' header
+        """
+
+        # if a prior response has been completed, then forget about it.
+        if self.__response and self.__response.isclosed():
+            self.__response = None
+
+
+        # in certain cases, we cannot issue another request on this connection.
+        # this occurs when:
+        #   1) we are in the process of sending a request.   (_CS_REQ_STARTED)
+        #   2) a response to a previous request has signalled that it is going
+        #      to close the connection upon completion.
+        #   3) the headers for the previous response have not been read, thus
+        #      we cannot determine whether point (2) is true.   (_CS_REQ_SENT)
+        #
+        # if there is no prior response, then we can request at will.
+        #
+        # if point (2) is true, then we will have passed the socket to the
+        # response (effectively meaning, "there is no prior response"), and
+        # will open a new one when a new request is made.
+        #
+        # Note: if a prior response exists, then we *can* start a new request.
+        #       We are not allowed to begin fetching the response to this new
+        #       request, however, until that prior response is complete.
+        #
+        if self.__state == _CS_IDLE:
+            self.__state = _CS_REQ_STARTED
+        else:
+            raise CannotSendRequest()
+
+        # Save the method we use, we need it later in the response phase
+        self._method = method
+        if not url:
+            url = '/'
+        hdr = '%s %s %s' % (method, url, self._http_vsn_str)
+
+        self._output(hdr)
+
+        if self._http_vsn == 11:
+            # Issue some standard headers for better HTTP/1.1 compliance
+
+            if not skip_host:
+                # this header is issued *only* for HTTP/1.1
+                # connections. more specifically, this means it is
+                # only issued when the client uses the new
+                # HTTPConnection() class. backwards-compat clients
+                # will be using HTTP/1.0 and those clients may be
+                # issuing this header themselves. we should NOT issue
+                # it twice; some web servers (such as Apache) barf
+                # when they see two Host: headers
+
+                # If we need a non-standard port,include it in the
+                # header.  If the request is going through a proxy,
+                # but the host of the actual URL, not the host of the
+                # proxy.
+
+                netloc = ''
+                if url.startswith('http'):
+                    nil, netloc, nil, nil, nil = urlsplit(url)
+
+                if netloc:
+                    try:
+                        netloc_enc = netloc.encode("ascii")
+                    except UnicodeEncodeError:
+                        netloc_enc = netloc.encode("idna")
+                    self.putheader('Host', netloc_enc)
+                else:
+                    try:
+                        host_enc = self.host.encode("ascii")
+                    except UnicodeEncodeError:
+                        host_enc = self.host.encode("idna")
+                    # Wrap the IPv6 Host Header with [] (RFC 2732)
+                    if host_enc.find(':') >= 0:
+                        host_enc = "[" + host_enc + "]"
+                    if self.port == self.default_port:
+                        self.putheader('Host', host_enc)
+                    else:
+                        self.putheader('Host', "%s:%s" % (host_enc, self.port))
+
+            # note: we are assuming that clients will not attempt to set these
+            #       headers since *this* library must deal with the
+            #       consequences. this also means that when the supporting
+            #       libraries are updated to recognize other forms, then this
+            #       code should be changed (removed or updated).
+
+            # we only want a Content-Encoding of "identity" since we don't
+            # support encodings such as x-gzip or x-deflate.
+            if not skip_accept_encoding:
+                self.putheader('Accept-Encoding', 'identity')
+
+            # we can accept "chunked" Transfer-Encodings, but no others
+            # NOTE: no TE header implies *only* "chunked"
+            #self.putheader('TE', 'chunked')
+
+            # if TE is supplied in the header, then it must appear in a
+            # Connection header.
+            #self.putheader('Connection', 'TE')
+
+        else:
+            # For HTTP/1.0, the server will assume "not chunked"
+            pass
+
+    def putheader(self, header, *values):
+        """Send a request header line to the server.
+
+        For example: h.putheader('Accept', 'text/html')
+        """
+        if self.__state != _CS_REQ_STARTED:
+            raise CannotSendHeader()
+
+        hdr = '%s: %s' % (header, '\r\n\t'.join([str(v) for v in values]))
+        self._output(hdr)
+
+    def endheaders(self, message_body=None):
+        """Indicate that the last header line has been sent to the server.
+
+        This method sends the request to the server.  The optional
+        message_body argument can be used to pass message body
+        associated with the request.  The message body will be sent in
+        the same packet as the message headers if possible.  The
+        message_body should be a string.
+        """
+        if self.__state == _CS_REQ_STARTED:
+            self.__state = _CS_REQ_SENT
+        else:
+            raise CannotSendHeader()
+        self._send_output(message_body)
+
+    def request(self, method, url, body=None, headers={}):
+        """Send a complete request to the server."""
+        self._send_request(method, url, body, headers)
+
+    def _set_content_length(self, body):
+        # Set the content-length based on the body.
+        thelen = None
+        try:
+            thelen = str(len(body))
+        except TypeError, te:
+            # If this is a file-like object, try to
+            # fstat its file descriptor
+            try:
+                thelen = str(os.fstat(body.fileno()).st_size)
+            except (AttributeError, OSError):
+                # Don't send a length if this failed
+                if self.debuglevel > 0: print "Cannot stat!!"
+
+        if thelen is not None:
+            self.putheader('Content-Length', thelen)
+
+    def _send_request(self, method, url, body, headers):
+        # Honor explicitly requested Host: and Accept-Encoding: headers.
+        header_names = dict.fromkeys([k.lower() for k in headers])
+        skips = {}
+        if 'host' in header_names:
+            skips['skip_host'] = 1
+        if 'accept-encoding' in header_names:
+            skips['skip_accept_encoding'] = 1
+
+        self.putrequest(method, url, **skips)
+
+        if body and ('content-length' not in header_names):
+            self._set_content_length(body)
+        for hdr, value in headers.iteritems():
+            self.putheader(hdr, value)
+        self.endheaders(body)
+
+    def getresponse(self, buffering=False):
+        "Get the response from the server."
+
+        # if a prior response has been completed, then forget about it.
+        if self.__response and self.__response.isclosed():
+            self.__response = None
+
+        #
+        # if a prior response exists, then it must be completed (otherwise, we
+        # cannot read this response's header to determine the connection-close
+        # behavior)
+        #
+        # note: if a prior response existed, but was connection-close, then the
+        # socket and response were made independent of this HTTPConnection
+        # object since a new request requires that we open a whole new
+        # connection
+        #
+        # this means the prior response had one of two states:
+        #   1) will_close: this connection was reset and the prior socket and
+        #                  response operate independently
+        #   2) persistent: the response was retained and we await its
+        #                  isclosed() status to become true.
+        #
+        if self.__state != _CS_REQ_SENT or self.__response:
+            raise ResponseNotReady()
+
+        args = (self.sock,)
+        kwds = {"strict":self.strict, "method":self._method}
+        if self.debuglevel > 0:
+            args += (self.debuglevel,)
+        if buffering:
+            #only add this keyword if non-default, for compatibility with
+            #other response_classes.
+            kwds["buffering"] = True;
+        response = self.response_class(*args, **kwds)
+
+        try:
+            response.begin()
+        except:
+            response.close()
+            raise
+        assert response.will_close != _UNKNOWN
+        self.__state = _CS_IDLE
+
+        if response.will_close:
+            # this effectively passes the connection to the response
+            self.close()
+        else:
+            # remember this, so we can tell when it is complete
+            self.__response = response
+
+        return response
+
+
+class HTTP:
+    "Compatibility class with httplib.py from 1.5."
+
+    _http_vsn = 10
+    _http_vsn_str = 'HTTP/1.0'
+
+    debuglevel = 0
+
+    _connection_class = HTTPConnection
+
+    def __init__(self, host='', port=None, strict=None):
+        "Provide a default host, since the superclass requires one."
+
+        # some joker passed 0 explicitly, meaning default port
+        if port == 0:
+            port = None
+
+        # Note that we may pass an empty string as the host; this will throw
+        # an error when we attempt to connect. Presumably, the client code
+        # will call connect before then, with a proper host.
+        self._setup(self._connection_class(host, port, strict))
+
+    def _setup(self, conn):
+        self._conn = conn
+
+        # set up delegation to flesh out interface
+        self.send = conn.send
+        self.putrequest = conn.putrequest
+        self.putheader = conn.putheader
+        self.endheaders = conn.endheaders
+        self.set_debuglevel = conn.set_debuglevel
+
+        conn._http_vsn = self._http_vsn
+        conn._http_vsn_str = self._http_vsn_str
+
+        self.file = None
+
+    def connect(self, host=None, port=None):
+        "Accept arguments to set the host/port, since the superclass doesn't."
+
+        if host is not None:
+            self._conn._set_hostport(host, port)
+        self._conn.connect()
+
+    def getfile(self):
+        "Provide a getfile, since the superclass' does not use this concept."
+        return self.file
+
+    def getreply(self, buffering=False):
+        """Compat definition since superclass does not define it.
+
+        Returns a tuple consisting of:
+        - server status code (e.g. '200' if all goes well)
+        - server "reason" corresponding to status code
+        - any RFC822 headers in the response from the server
+        """
+        try:
+            if not buffering:
+                response = self._conn.getresponse()
+            else:
+                #only add this keyword if non-default for compatibility
+                #with other connection classes
+                response = self._conn.getresponse(buffering)
+        except BadStatusLine, e:
+            ### hmm. if getresponse() ever closes the socket on a bad request,
+            ### then we are going to have problems with self.sock
+
+            ### should we keep this behavior? do people use it?
+            # keep the socket open (as a file), and return it
+            self.file = self._conn.sock.makefile('rb', 0)
+
+            # close our socket -- we want to restart after any protocol error
+            self.close()
+
+            self.headers = None
+            return -1, e.line, None
+
+        self.headers = response.msg
+        self.file = response.fp
+        return response.status, response.reason, response.msg
+
+    def close(self):
+        self._conn.close()
+
+        # note that self.file == response.fp, which gets closed by the
+        # superclass. just clear the object ref here.
+        ### hmm. messy. if status==-1, then self.file is owned by us.
+        ### well... we aren't explicitly closing, but losing this ref will
+        ### do it
+        self.file = None
+
+try:
+    import ssl
+except ImportError:
+    pass
+else:
+    class HTTPSConnection(HTTPConnection):
+        "This class allows communication via SSL."
+
+        default_port = HTTPS_PORT
+
+        def __init__(self, host, port=None, key_file=None, cert_file=None,
+                     strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
+                     source_address=None):
+            HTTPConnection.__init__(self, host, port, strict, timeout,
+                                    source_address)
+            self.key_file = key_file
+            self.cert_file = cert_file
+
+        def connect(self):
+            "Connect to a host on a given (SSL) port."
+
+            sock = socket.create_connection((self.host, self.port),
+                                            self.timeout, self.source_address)
+            if self._tunnel_host:
+                self.sock = sock
+                self._tunnel()
+            self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file)
+
+    __all__.append("HTTPSConnection")
+
+    class HTTPS(HTTP):
+        """Compatibility with 1.5 httplib interface
+
+        Python 1.5.2 did not have an HTTPS class, but it defined an
+        interface for sending http requests that is also useful for
+        https.
+        """
+
+        _connection_class = HTTPSConnection
+
+        def __init__(self, host='', port=None, key_file=None, cert_file=None,
+                     strict=None):
+            # provide a default host, pass the X509 cert info
+
+            # urf. compensate for bad input.
+            if port == 0:
+                port = None
+            self._setup(self._connection_class(host, port, key_file,
+                                               cert_file, strict))
+
+            # we never actually use these for anything, but we keep them
+            # here for compatibility with post-1.5.2 CVS.
+            self.key_file = key_file
+            self.cert_file = cert_file
+
+
+    def FakeSocket (sock, sslobj):
+        warnings.warn("FakeSocket is deprecated, and won't be in 3.x.  " +
+                      "Use the result of ssl.wrap_socket() directly instead.",
+                      DeprecationWarning, stacklevel=2)
+        return sslobj
+
+
+class HTTPException(Exception):
+    # Subclasses that define an __init__ must call Exception.__init__
+    # or define self.args.  Otherwise, str() will fail.
+    pass
+
+class NotConnected(HTTPException):
+    pass
+
+class InvalidURL(HTTPException):
+    pass
+
+class UnknownProtocol(HTTPException):
+    def __init__(self, version):
+        self.args = version,
+        self.version = version
+
+class UnknownTransferEncoding(HTTPException):
+    pass
+
+class UnimplementedFileMode(HTTPException):
+    pass
+
+class IncompleteRead(HTTPException):
+    def __init__(self, partial, expected=None):
+        self.args = partial,
+        self.partial = partial
+        self.expected = expected
+    def __repr__(self):
+        if self.expected is not None:
+            e = ', %i more expected' % self.expected
+        else:
+            e = ''
+        return 'IncompleteRead(%i bytes read%s)' % (len(self.partial), e)
+    def __str__(self):
+        return repr(self)
+
+class ImproperConnectionState(HTTPException):
+    pass
+
+class CannotSendRequest(ImproperConnectionState):
+    pass
+
+class CannotSendHeader(ImproperConnectionState):
+    pass
+
+class ResponseNotReady(ImproperConnectionState):
+    pass
+
+class BadStatusLine(HTTPException):
+    def __init__(self, line):
+        if not line:
+            line = repr(line)
+        self.args = line,
+        self.line = line
+
+# for backwards compatibility
+error = HTTPException
+
+class LineAndFileWrapper:
+    """A limited file-like object for HTTP/0.9 responses."""
+
+    # The status-line parsing code calls readline(), which normally
+    # get the HTTP status line.  For a 0.9 response, however, this is
+    # actually the first line of the body!  Clients need to get a
+    # readable file object that contains that line.
+
+    def __init__(self, line, file):
+        self._line = line
+        self._file = file
+        self._line_consumed = 0
+        self._line_offset = 0
+        self._line_left = len(line)
+
+    def __getattr__(self, attr):
+        return getattr(self._file, attr)
+
+    def _done(self):
+        # called when the last byte is read from the line.  After the
+        # call, all read methods are delegated to the underlying file
+        # object.
+        self._line_consumed = 1
+        self.read = self._file.read
+        self.readline = self._file.readline
+        self.readlines = self._file.readlines
+
+    def read(self, amt=None):
+        if self._line_consumed:
+            return self._file.read(amt)
+        assert self._line_left
+        if amt is None or amt > self._line_left:
+            s = self._line[self._line_offset:]
+            self._done()
+            if amt is None:
+                return s + self._file.read()
+            else:
+                return s + self._file.read(amt - len(s))
+        else:
+            assert amt <= self._line_left
+            i = self._line_offset
+            j = i + amt
+            s = self._line[i:j]
+            self._line_offset = j
+            self._line_left -= amt
+            if self._line_left == 0:
+                self._done()
+            return s
+
+    def readline(self):
+        if self._line_consumed:
+            return self._file.readline()
+        assert self._line_left
+        s = self._line[self._line_offset:]
+        self._done()
+        return s
+
+    def readlines(self, size=None):
+        if self._line_consumed:
+            return self._file.readlines(size)
+        assert self._line_left
+        L = [self._line[self._line_offset:]]
+        self._done()
+        if size is None:
+            return L + self._file.readlines()
+        else:
+            return L + self._file.readlines(size)
+
+def test():
+    """Test this module.
+
+    A hodge podge of tests collected here, because they have too many
+    external dependencies for the regular test suite.
+    """
+
+    import sys
+    import getopt
+    opts, args = getopt.getopt(sys.argv[1:], 'd')
+    dl = 0
+    for o, a in opts:
+        if o == '-d': dl = dl + 1
+    host = 'www.python.org'
+    selector = '/'
+    if args[0:]: host = args[0]
+    if args[1:]: selector = args[1]
+    h = HTTP()
+    h.set_debuglevel(dl)
+    h.connect(host)
+    h.putrequest('GET', selector)
+    h.endheaders()
+    status, reason, headers = h.getreply()
+    print 'status =', status
+    print 'reason =', reason
+    print "read", len(h.getfile().read())
+    print
+    if headers:
+        for header in headers.headers: print header.strip()
+    print
+
+    # minimal test that code to extract host from url works
+    class HTTP11(HTTP):
+        _http_vsn = 11
+        _http_vsn_str = 'HTTP/1.1'
+
+    h = HTTP11('www.python.org')
+    h.putrequest('GET', 'http://www.python.org/~jeremy/')
+    h.endheaders()
+    h.getreply()
+    h.close()
+
+    try:
+        import ssl
+    except ImportError:
+        pass
+    else:
+
+        for host, selector in (('sourceforge.net', '/projects/python'),
+                               ):
+            print "https://%s%s" % (host, selector)
+            hs = HTTPS()
+            hs.set_debuglevel(dl)
+            hs.connect(host)
+            hs.putrequest('GET', selector)
+            hs.endheaders()
+            status, reason, headers = hs.getreply()
+            print 'status =', status
+            print 'reason =', reason
+            print "read", len(hs.getfile().read())
+            print
+            if headers:
+                for header in headers.headers: print header.strip()
+            print
+
+if __name__ == '__main__':
+    test()
diff --git a/lib-python/modified-2.7/json/encoder.py b/lib-python/modified-2.7/json/encoder.py
--- a/lib-python/modified-2.7/json/encoder.py
+++ b/lib-python/modified-2.7/json/encoder.py
@@ -2,14 +2,7 @@
 """
 import re
 
-try:
-    from _json import encode_basestring_ascii as c_encode_basestring_ascii
-except ImportError:
-    c_encode_basestring_ascii = None
-try:
-    from _json import make_encoder as c_make_encoder
-except ImportError:
-    c_make_encoder = None
+from __pypy__.builders import StringBuilder, UnicodeBuilder
 
 ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]')
 ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])')
@@ -24,8 +17,7 @@
     '\t': '\\t',
 }
 for i in range(0x20):
-    ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i))
-    #ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,))
+    ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,))
 
 # Assume this produces an infinity on all machines (probably not guaranteed)
 INFINITY = float('1e66666')
@@ -37,10 +29,9 @@
     """
     def replace(match):
         return ESCAPE_DCT[match.group(0)]
-    return '"' + ESCAPE.sub(replace, s) + '"'
+    return ESCAPE.sub(replace, s)
 
-
-def py_encode_basestring_ascii(s):
+def encode_basestring_ascii(s):
     """Return an ASCII-only JSON representation of a Python string
 
     """
@@ -53,20 +44,18 @@
         except KeyError:
             n = ord(s)
             if n < 0x10000:
-                return '\\u{0:04x}'.format(n)
-                #return '\\u%04x' % (n,)
+                return '\\u%04x' % (n,)
             else:
                 # surrogate pair
                 n -= 0x10000
                 s1 = 0xd800 | ((n >> 10) & 0x3ff)
                 s2 = 0xdc00 | (n & 0x3ff)
-                return '\\u{0:04x}\\u{1:04x}'.format(s1, s2)
-                #return '\\u%04x\\u%04x' % (s1, s2)
-    return '"' + str(ESCAPE_ASCII.sub(replace, s)) + '"'
-
-
-encode_basestring_ascii = (
-    c_encode_basestring_ascii or py_encode_basestring_ascii)
+                return '\\u%04x\\u%04x' % (s1, s2)
+    if ESCAPE_ASCII.search(s):
+        return str(ESCAPE_ASCII.sub(replace, s))
+    return s
+py_encode_basestring_ascii = lambda s: '"' + encode_basestring_ascii(s) + '"'
+c_encode_basestring_ascii = None
 
 class JSONEncoder(object):
     """Extensible JSON <http://json.org> encoder for Python data structures.
@@ -147,6 +136,17 @@
 
         self.skipkeys = skipkeys
         self.ensure_ascii = ensure_ascii
+        if ensure_ascii:
+            self.encoder = encode_basestring_ascii
+        else:
+            self.encoder = encode_basestring
+        if encoding != 'utf-8':
+            orig_encoder = self.encoder
+            def encoder(o):
+                if isinstance(o, str):
+                    o = o.decode(encoding)
+                return orig_encoder(o)
+            self.encoder = encoder
         self.check_circular = check_circular
         self.allow_nan = allow_nan
         self.sort_keys = sort_keys
@@ -184,24 +184,126 @@
         '{"foo": ["bar", "baz"]}'
 
         """
-        # This is for extremely simple cases and benchmarks.
+        if self.check_circular:
+            markers = {}
+        else:
+            markers = None
+        if self.ensure_ascii:
+            builder = StringBuilder()
+        else:
+            builder = UnicodeBuilder()
+        self._encode(o, markers, builder, 0)
+        return builder.build()
+
+    def _emit_indent(self, builder, _current_indent_level):
+        if self.indent is not None:
+            _current_indent_level += 1
+            newline_indent = '\n' + (' ' * (self.indent *
+                                            _current_indent_level))
+            separator = self.item_separator + newline_indent
+            builder.append(newline_indent)
+        else:
+            separator = self.item_separator
+        return separator, _current_indent_level
+
+    def _emit_unindent(self, builder, _current_indent_level):
+        if self.indent is not None:
+            builder.append('\n')
+            builder.append(' ' * (self.indent * (_current_indent_level - 1)))
+
+    def _encode(self, o, markers, builder, _current_indent_level):
         if isinstance(o, basestring):
-            if isinstance(o, str):
-                _encoding = self.encoding
-                if (_encoding is not None
-                        and not (_encoding == 'utf-8')):
-                    o = o.decode(_encoding)
-            if self.ensure_ascii:
-                return encode_basestring_ascii(o)
+            builder.append('"')
+            builder.append(self.encoder(o))
+            builder.append('"')
+        elif o is None:
+            builder.append('null')
+        elif o is True:
+            builder.append('true')
+        elif o is False:
+            builder.append('false')
+        elif isinstance(o, (int, long)):
+            builder.append(str(o))
+        elif isinstance(o, float):
+            builder.append(self._floatstr(o))
+        elif isinstance(o, (list, tuple)):
+            if not o:
+                builder.append('[]')
+                return
+            self._encode_list(o, markers, builder, _current_indent_level)
+        elif isinstance(o, dict):
+            if not o:
+                builder.append('{}')
+                return
+            self._encode_dict(o, markers, builder, _current_indent_level)
+        else:
+            self._mark_markers(markers, o)
+            res = self.default(o)
+            self._encode(res, markers, builder, _current_indent_level)
+            self._remove_markers(markers, o)
+            return res
+
+    def _encode_list(self, l, markers, builder, _current_indent_level):
+        self._mark_markers(markers, l)
+        builder.append('[')
+        first = True
+        separator, _current_indent_level = self._emit_indent(builder,
+                                                      _current_indent_level)
+        for elem in l:
+            if first:
+                first = False
             else:
-                return encode_basestring(o)
-        # This doesn't pass the iterator directly to ''.join() because the
-        # exceptions aren't as detailed.  The list call should be roughly
-        # equivalent to the PySequence_Fast that ''.join() would do.
-        chunks = self.iterencode(o, _one_shot=True)
-        if not isinstance(chunks, (list, tuple)):
-            chunks = list(chunks)
-        return ''.join(chunks)
+                builder.append(separator)
+            self._encode(elem, markers, builder, _current_indent_level)
+            del elem # XXX grumble
+        self._emit_unindent(builder, _current_indent_level)
+        builder.append(']')
+        self._remove_markers(markers, l)
+
+    def _encode_dict(self, d, markers, builder, _current_indent_level):
+        self._mark_markers(markers, d)
+        first = True
+        builder.append('{')
+        separator, _current_indent_level = self._emit_indent(builder,
+                                                         _current_indent_level)
+        if self.sort_keys:
+            items = sorted(d.items(), key=lambda kv: kv[0])
+        else:
+            items = d.iteritems()
+
+        for key, v in items:
+            if first:
+                first = False
+            else:
+                builder.append(separator)
+            if isinstance(key, basestring):
+                pass
+            # JavaScript is weakly typed for these, so it makes sense to
+            # also allow them.  Many encoders seem to do something like this.
+            elif isinstance(key, float):
+                key = self._floatstr(key)
+            elif key is True:
+                key = 'true'
+            elif key is False:
+                key = 'false'
+            elif key is None:
+                key = 'null'
+            elif isinstance(key, (int, long)):
+                key = str(key)
+            elif self.skipkeys:
+                continue
+            else:
+                raise TypeError("key " + repr(key) + " is not a string")
+            builder.append('"')
+            builder.append(self.encoder(key))
+            builder.append('"')
+            builder.append(self.key_separator)
+            self._encode(v, markers, builder, _current_indent_level)
+            del key
+            del v # XXX grumble
+        self._emit_unindent(builder, _current_indent_level)
+        builder.append('}')
+        self._remove_markers(markers, d)
 
     def iterencode(self, o, _one_shot=False):
         """Encode the given object and yield each string
@@ -217,86 +319,54 @@
             markers = {}
         else:
             markers = None
-        if self.ensure_ascii:
-            _encoder = encode_basestring_ascii
+        return self._iterencode(o, markers, 0)
+
+    def _floatstr(self, o):
+        # Check for specials.  Note that this type of test is processor
+        # and/or platform-specific, so do tests which don't depend on the
+        # internals.
+
+        if o != o:
+            text = 'NaN'
+        elif o == INFINITY:
+            text = 'Infinity'
+        elif o == -INFINITY:
+            text = '-Infinity'
         else:
-            _encoder = encode_basestring
-        if self.encoding != 'utf-8':
-            def _encoder(o, _orig_encoder=_encoder, _encoding=self.encoding):
-                if isinstance(o, str):
-                    o = o.decode(_encoding)
-                return _orig_encoder(o)
+            return FLOAT_REPR(o)
 
-        def floatstr(o, allow_nan=self.allow_nan,
-                _repr=FLOAT_REPR, _inf=INFINITY, _neginf=-INFINITY):
-            # Check for specials.  Note that this type of test is processor
-            # and/or platform-specific, so do tests which don't depend on the
-            # internals.
+        if not self.allow_nan:
+            raise ValueError(
+                "Out of range float values are not JSON compliant: " +
+                repr(o))
 
-            if o != o:
-                text = 'NaN'
-            elif o == _inf:
-                text = 'Infinity'
-            elif o == _neginf:
-                text = '-Infinity'
-            else:
-                return _repr(o)
+        return text
 
-            if not allow_nan:
-                raise ValueError(
-                    "Out of range float values are not JSON compliant: " +
-                    repr(o))
+    def _mark_markers(self, markers, o):
+        if markers is not None:
+            if id(o) in markers:
+                raise ValueError("Circular reference detected")
+            markers[id(o)] = None
 
-            return text
+    def _remove_markers(self, markers, o):
+        if markers is not None:
+            del markers[id(o)]
 
-
-        if (_one_shot and c_make_encoder is not None
-                and not self.indent and not self.sort_keys):
-            _iterencode = c_make_encoder(
-                markers, self.default, _encoder, self.indent,
-                self.key_separator, self.item_separator, self.sort_keys,
-                self.skipkeys, self.allow_nan)
-        else:
-            _iterencode = _make_iterencode(
-                markers, self.default, _encoder, self.indent, floatstr,
-                self.key_separator, self.item_separator, self.sort_keys,
-                self.skipkeys, _one_shot)
-        return _iterencode(o, 0)
-
-def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,
-        _key_separator, _item_separator, _sort_keys, _skipkeys, _one_shot,
-        ## HACK: hand-optimized bytecode; turn globals into locals
-        ValueError=ValueError,
-        basestring=basestring,
-        dict=dict,
-        float=float,
-        id=id,
-        int=int,
-        isinstance=isinstance,
-        list=list,
-        long=long,
-        str=str,
-        tuple=tuple,
-    ):
-
-    def _iterencode_list(lst, _current_indent_level):
+    def _iterencode_list(self, lst, markers, _current_indent_level):
         if not lst:
             yield '[]'
             return
-        if markers is not None:
-            markerid = id(lst)
-            if markerid in markers:
-                raise ValueError("Circular reference detected")
-            markers[markerid] = lst
+        self._mark_markers(markers, lst)
         buf = '['
-        if _indent is not None:
+        if self.indent is not None:
             _current_indent_level += 1
-            newline_indent = '\n' + (' ' * (_indent * _current_indent_level))
-            separator = _item_separator + newline_indent
+            newline_indent = '\n' + (' ' * (self.indent *
+                                            _current_indent_level))
+            separator = self.item_separator + newline_indent
             buf += newline_indent
         else:
             newline_indent = None
-            separator = _item_separator
+            separator = self.item_separator
         first = True
         for value in lst:
             if first:
@@ -304,7 +374,7 @@
             else:
                 buf = separator
             if isinstance(value, basestring):
-                yield buf + _encoder(value)
+                yield buf + '"' + self.encoder(value) + '"'
             elif value is None:
                 yield buf + 'null'
             elif value is True:
@@ -314,44 +384,43 @@
             elif isinstance(value, (int, long)):
                 yield buf + str(value)
             elif isinstance(value, float):
-                yield buf + _floatstr(value)
+                yield buf + self._floatstr(value)
             else:
                 yield buf
                 if isinstance(value, (list, tuple)):
-                    chunks = _iterencode_list(value, _current_indent_level)
+                    chunks = self._iterencode_list(value, markers,
+                                                   _current_indent_level)
                 elif isinstance(value, dict):
-                    chunks = _iterencode_dict(value, _current_indent_level)
+                    chunks = self._iterencode_dict(value, markers,
+                                                   _current_indent_level)
                 else:
-                    chunks = _iterencode(value, _current_indent_level)
+                    chunks = self._iterencode(value, markers,
+                                              _current_indent_level)
                 for chunk in chunks:
                     yield chunk
         if newline_indent is not None:
             _current_indent_level -= 1
-            yield '\n' + (' ' * (_indent * _current_indent_level))
+            yield '\n' + (' ' * (self.indent * _current_indent_level))
         yield ']'
-        if markers is not None:
-            del markers[markerid]
+        self._remove_markers(markers, lst)
 
-    def _iterencode_dict(dct, _current_indent_level):
+    def _iterencode_dict(self, dct, markers, _current_indent_level):
         if not dct:
             yield '{}'
             return
-        if markers is not None:
-            markerid = id(dct)
-            if markerid in markers:
-                raise ValueError("Circular reference detected")
-            markers[markerid] = dct
+        self._mark_markers(markers, dct)
         yield '{'
-        if _indent is not None:
+        if self.indent is not None:
             _current_indent_level += 1
-            newline_indent = '\n' + (' ' * (_indent * _current_indent_level))
-            item_separator = _item_separator + newline_indent
+            newline_indent = '\n' + (' ' * (self.indent *
+                                            _current_indent_level))
+            item_separator = self.item_separator + newline_indent
             yield newline_indent
         else:
             newline_indent = None
-            item_separator = _item_separator
+            item_separator = self.item_separator
         first = True
-        if _sort_keys:
+        if self.sort_keys:
             items = sorted(dct.items(), key=lambda kv: kv[0])
         else:
             items = dct.iteritems()
@@ -361,7 +430,7 @@
             # JavaScript is weakly typed for these, so it makes sense to
             # also allow them.  Many encoders seem to do something like this.
             elif isinstance(key, float):
-                key = _floatstr(key)
+                key = self._floatstr(key)
             elif key is True:
                 key = 'true'
             elif key is False:
@@ -370,7 +439,7 @@
                 key = 'null'
             elif isinstance(key, (int, long)):
                 key = str(key)
-            elif _skipkeys:
+            elif self.skipkeys:
                 continue
             else:
                 raise TypeError("key " + repr(key) + " is not a string")
@@ -378,10 +447,10 @@
                 first = False
             else:
                 yield item_separator
-            yield _encoder(key)
-            yield _key_separator
+            yield '"' + self.encoder(key) + '"'
+            yield self.key_separator
             if isinstance(value, basestring):
-                yield _encoder(value)
+                yield '"' + self.encoder(value) + '"'
             elif value is None:
                 yield 'null'
             elif value is True:
@@ -391,26 +460,28 @@
             elif isinstance(value, (int, long)):
                 yield str(value)
             elif isinstance(value, float):
-                yield _floatstr(value)
+                yield self._floatstr(value)
             else:
                 if isinstance(value, (list, tuple)):
-                    chunks = _iterencode_list(value, _current_indent_level)
+                    chunks = self._iterencode_list(value, markers,
+                                                   _current_indent_level)
                 elif isinstance(value, dict):
-                    chunks = _iterencode_dict(value, _current_indent_level)
+                    chunks = self._iterencode_dict(value, markers,
+                                                   _current_indent_level)
                 else:
-                    chunks = _iterencode(value, _current_indent_level)
+                    chunks = self._iterencode(value, markers,
+                                              _current_indent_level)
                 for chunk in chunks:
                     yield chunk
         if newline_indent is not None:
             _current_indent_level -= 1
-            yield '\n' + (' ' * (_indent * _current_indent_level))
+            yield '\n' + (' ' * (self.indent * _current_indent_level))
         yield '}'
-        if markers is not None:
-            del markers[markerid]
+        self._remove_markers(markers, dct)
 
-    def _iterencode(o, _current_indent_level):
+    def _iterencode(self, o, markers, _current_indent_level):
         if isinstance(o, basestring):
-            yield _encoder(o)
+            yield '"' + self.encoder(o) + '"'
         elif o is None:
             yield 'null'
         elif o is True:
@@ -420,23 +491,19 @@
         elif isinstance(o, (int, long)):
             yield str(o)
         elif isinstance(o, float):
-            yield _floatstr(o)
+            yield self._floatstr(o)
         elif isinstance(o, (list, tuple)):
-            for chunk in _iterencode_list(o, _current_indent_level):
+            for chunk in self._iterencode_list(o, markers,
+                                               _current_indent_level):
                 yield chunk
         elif isinstance(o, dict):
-            for chunk in _iterencode_dict(o, _current_indent_level):
+            for chunk in self._iterencode_dict(o, markers,
+                                               _current_indent_level):
                 yield chunk
         else:
-            if markers is not None:
-                markerid = id(o)
-                if markerid in markers:
-                    raise ValueError("Circular reference detected")
-                markers[markerid] = o
-            o = _default(o)
-            for chunk in _iterencode(o, _current_indent_level):
+            self._mark_markers(markers, o)
+            obj = self.default(o)
+            for chunk in self._iterencode(obj, markers,
+                                          _current_indent_level):
                 yield chunk
-            if markers is not None:
-                del markers[markerid]
-
-    return _iterencode
+            self._remove_markers(markers, o)
diff --git a/lib-python/modified-2.7/json/tests/test_unicode.py b/lib-python/modified-2.7/json/tests/test_unicode.py
--- a/lib-python/modified-2.7/json/tests/test_unicode.py
+++ b/lib-python/modified-2.7/json/tests/test_unicode.py
@@ -80,3 +80,9 @@
         self.assertEqual(type(json.loads(u'["a"]')[0]), unicode)
         # Issue 10038.
         self.assertEqual(type(json.loads('"foo"')), unicode)
+
+    def test_encode_not_utf_8(self):
+        self.assertEqual(json.dumps('\xb1\xe6', encoding='iso8859-2'),
+                         '"\\u0105\\u0107"')
+        self.assertEqual(json.dumps(['\xb1\xe6'], encoding='iso8859-2'),
+                         '["\\u0105\\u0107"]')
diff --git a/lib-python/modified-2.7/opcode.py b/lib-python/modified-2.7/opcode.py
--- a/lib-python/modified-2.7/opcode.py
+++ b/lib-python/modified-2.7/opcode.py
@@ -189,7 +189,6 @@
 def_op('MAP_ADD', 147)
 
 # pypy modification, experimental bytecode
-def_op('CALL_LIKELY_BUILTIN', 200)    # #args + (#kwargs << 8)
 def_op('LOOKUP_METHOD', 201)          # Index in name list
 hasname.append(201)
 def_op('CALL_METHOD', 202)            # #args not including 'self'
diff --git a/lib-python/modified-2.7/pickle.py b/lib-python/modified-2.7/pickle.py
--- a/lib-python/modified-2.7/pickle.py
+++ b/lib-python/modified-2.7/pickle.py
@@ -168,7 +168,7 @@
 
 # Pickling machinery
 
-class Pickler:
+class Pickler(object):
 
     def __init__(self, file, protocol=None):
         """This takes a file-like object for writing a pickle data stream.
@@ -873,7 +873,7 @@
 
 # Unpickling machinery
 
-class Unpickler:
+class Unpickler(object):
 
     def __init__(self, file):
         """This takes a file-like object for reading a pickle data stream.
diff --git a/lib-python/modified-2.7/sqlite3/test/regression.py b/lib-python/modified-2.7/sqlite3/test/regression.py
--- a/lib-python/modified-2.7/sqlite3/test/regression.py
+++ b/lib-python/modified-2.7/sqlite3/test/regression.py
@@ -274,6 +274,18 @@
         cur.execute("UPDATE foo SET id = 3 WHERE id = 1")
         self.assertEqual(cur.description, None)
 
+    def CheckStatementCache(self):
+        cur = self.con.cursor()
+        cur.execute("CREATE TABLE foo (id INTEGER)")
+        values = [(i,) for i in xrange(5)]
+        cur.executemany("INSERT INTO foo (id) VALUES (?)", values)
+
+        cur.execute("SELECT id FROM foo")
+        self.assertEqual(list(cur), values)
+        self.con.commit()
+        cur.execute("SELECT id FROM foo")
+        self.assertEqual(list(cur), values)
+
 def suite():
     regression_suite = unittest.makeSuite(RegressionTests, "Check")
     return unittest.TestSuite((regression_suite,))
diff --git a/lib-python/modified-2.7/ssl.py b/lib-python/modified-2.7/ssl.py
--- a/lib-python/modified-2.7/ssl.py
+++ b/lib-python/modified-2.7/ssl.py
@@ -62,7 +62,6 @@
 from _ssl import OPENSSL_VERSION_NUMBER, OPENSSL_VERSION_INFO, OPENSSL_VERSION
 from _ssl import SSLError
 from _ssl import CERT_NONE, CERT_OPTIONAL, CERT_REQUIRED
-from _ssl import PROTOCOL_SSLv2, PROTOCOL_SSLv3, PROTOCOL_SSLv23, PROTOCOL_TLSv1
 from _ssl import RAND_status, RAND_egd, RAND_add
 from _ssl import \
      SSL_ERROR_ZERO_RETURN, \
@@ -74,6 +73,18 @@
      SSL_ERROR_WANT_CONNECT, \
      SSL_ERROR_EOF, \
      SSL_ERROR_INVALID_ERROR_CODE
+from _ssl import PROTOCOL_SSLv3, PROTOCOL_SSLv23, PROTOCOL_TLSv1
+_PROTOCOL_NAMES = {
+    PROTOCOL_TLSv1: "TLSv1",
+    PROTOCOL_SSLv23: "SSLv23",
+    PROTOCOL_SSLv3: "SSLv3",
+}
+try:
+    from _ssl import PROTOCOL_SSLv2
+except ImportError:
+    pass
+else:
+    _PROTOCOL_NAMES[PROTOCOL_SSLv2] = "SSLv2"
 
 from socket import socket, _fileobject, error as socket_error
 from socket import getnameinfo as _getnameinfo
@@ -400,16 +411,7 @@
     return DER_cert_to_PEM_cert(dercert)
 
 def get_protocol_name(protocol_code):
-    if protocol_code == PROTOCOL_TLSv1:
-        return "TLSv1"
-    elif protocol_code == PROTOCOL_SSLv23:
-        return "SSLv23"
-    elif protocol_code == PROTOCOL_SSLv2:
-        return "SSLv2"
-    elif protocol_code == PROTOCOL_SSLv3:
-        return "SSLv3"
-    else:
-        return "<unknown>"
+    return _PROTOCOL_NAMES.get(protocol_code, '<unknown>')
 
 
 # a replacement for the old socket.ssl function
diff --git a/lib-python/modified-2.7/test/regrtest.py b/lib-python/modified-2.7/test/regrtest.py
--- a/lib-python/modified-2.7/test/regrtest.py
+++ b/lib-python/modified-2.7/test/regrtest.py
@@ -1403,7 +1403,26 @@
         test_zipimport
         test_zlib
         """,
-    'openbsd3':
+    'openbsd4':
+        """
+        test_ascii_formatd
+        test_bsddb
+        test_bsddb3
+        test_ctypes
+        test_dl
+        test_epoll
+        test_gdbm
+        test_locale
+        test_normalization
+        test_ossaudiodev
+        test_pep277
+        test_tcl
+        test_tk
+        test_ttk_guionly
+        test_ttk_textonly
+        test_multiprocessing
+        """,
+    'openbsd5':
         """
         test_ascii_formatd
         test_bsddb
diff --git a/lib-python/modified-2.7/test/test_array.py b/lib-python/modified-2.7/test/test_array.py
--- a/lib-python/modified-2.7/test/test_array.py
+++ b/lib-python/modified-2.7/test/test_array.py
@@ -295,9 +295,10 @@
         )
 
         b = array.array(self.badtypecode())
-        self.assertRaises(TypeError, "a + b")
-
-        self.assertRaises(TypeError, "a + 'bad'")
+        with self.assertRaises(TypeError):
+            a + b
+        with self.assertRaises(TypeError):
+            a + 'bad'
 
     def test_iadd(self):
         a = array.array(self.typecode, self.example[::-1])
@@ -316,9 +317,10 @@
         )
 
         b = array.array(self.badtypecode())
-        self.assertRaises(TypeError, "a += b")
-
-        self.assertRaises(TypeError, "a += 'bad'")
+        with self.assertRaises(TypeError):
+            a += b
+        with self.assertRaises(TypeError):
+            a += 'bad'
 
     def test_mul(self):
         a = 5*array.array(self.typecode, self.example)
@@ -345,7 +347,8 @@
             array.array(self.typecode)
         )
 
-        self.assertRaises(TypeError, "a * 'bad'")
+        with self.assertRaises(TypeError):
+            a * 'bad'
 
     def test_imul(self):
         a = array.array(self.typecode, self.example)
@@ -374,7 +377,8 @@
         a *= -1
         self.assertEqual(a, array.array(self.typecode))
 
-        self.assertRaises(TypeError, "a *= 'bad'")
+        with self.assertRaises(TypeError):
+            a *= 'bad'
 
     def test_getitem(self):
         a = array.array(self.typecode, self.example)
diff --git a/lib-python/modified-2.7/test/test_bz2.py b/lib-python/modified-2.7/test/test_bz2.py
--- a/lib-python/modified-2.7/test/test_bz2.py
+++ b/lib-python/modified-2.7/test/test_bz2.py
@@ -50,6 +50,7 @@
         self.filename = TESTFN
 
     def tearDown(self):
+        test_support.gc_collect()
         if os.path.isfile(self.filename):
             os.unlink(self.filename)
 
diff --git a/lib-python/modified-2.7/test/test_descr.py b/lib-python/modified-2.7/test/test_descr.py
--- a/lib-python/modified-2.7/test/test_descr.py
+++ b/lib-python/modified-2.7/test/test_descr.py
@@ -4399,13 +4399,10 @@
         self.assertTrue(l.__add__ != [5].__add__)
         self.assertTrue(l.__add__ != l.__mul__)
         self.assertTrue(l.__add__.__name__ == '__add__')
-        if hasattr(l.__add__, '__self__'):
-            # CPython
-            self.assertTrue(l.__add__.__self__ is l)
+        self.assertTrue(l.__add__.__self__ is l)
+        if hasattr(l.__add__, '__objclass__'):   # CPython
             self.assertTrue(l.__add__.__objclass__ is list)
-        else:
-            # Python implementations where [].__add__ is a normal bound method
-            self.assertTrue(l.__add__.im_self is l)
+        else:                                    # PyPy
             self.assertTrue(l.__add__.im_class is list)
         self.assertEqual(l.__add__.__doc__, list.__add__.__doc__)
         try:
diff --git a/lib-python/modified-2.7/test/test_dis.py b/lib-python/modified-2.7/test/test_dis.py
deleted file mode 100644
--- a/lib-python/modified-2.7/test/test_dis.py
+++ /dev/null
@@ -1,152 +0,0 @@
-# Minimal tests for dis module
-
-from test.test_support import run_unittest
-import unittest
-import sys
-import dis
-import StringIO
-
-
-def _f(a):
-    print a
-    return 1
-
-dis_f = """\
- %-4d         0 LOAD_FAST                0 (a)
-              3 PRINT_ITEM
-              4 PRINT_NEWLINE
-
- %-4d         5 LOAD_CONST               1 (1)
-              8 RETURN_VALUE
-"""%(_f.func_code.co_firstlineno + 1,
-     _f.func_code.co_firstlineno + 2)
-
-
-# we "call" rangexxx() instead of range() to disable the
-# pypy optimization that turns it into CALL_LIKELY_BUILTIN.
-def bug708901():
-    for res in rangexxx(1,
-                        10):
-        pass
-
-dis_bug708901 = """\
- %-4d         0 SETUP_LOOP              23 (to 26)
-              3 LOAD_GLOBAL              0 (rangexxx)
-              6 LOAD_CONST               1 (1)
-
- %-4d         9 LOAD_CONST               2 (10)
-             12 CALL_FUNCTION            2
-             15 GET_ITER
-        >>   16 FOR_ITER                 6 (to 25)
-             19 STORE_FAST               0 (res)
-
- %-4d        22 JUMP_ABSOLUTE           16
-        >>   25 POP_BLOCK
-        >>   26 LOAD_CONST               0 (None)
-             29 RETURN_VALUE
-"""%(bug708901.func_code.co_firstlineno + 1,
-     bug708901.func_code.co_firstlineno + 2,
-     bug708901.func_code.co_firstlineno + 3)
-
-
-def bug1333982(x=[]):
-    assert 0, ([s for s in x] +
-              1)
-    pass
-
-dis_bug1333982 = """\
- %-4d         0 LOAD_CONST               1 (0)
-              3 POP_JUMP_IF_TRUE        38
-              6 LOAD_GLOBAL              0 (AssertionError)
-              9 BUILD_LIST               0
-             12 LOAD_FAST                0 (x)
-             15 GET_ITER
-        >>   16 FOR_ITER                12 (to 31)
-             19 STORE_FAST               1 (s)
-             22 LOAD_FAST                1 (s)
-             25 LIST_APPEND              2
-             28 JUMP_ABSOLUTE           16
-
- %-4d   >>   31 LOAD_CONST               2 (1)
-             34 BINARY_ADD
-             35 RAISE_VARARGS            2
-
- %-4d   >>   38 LOAD_CONST               0 (None)
-             41 RETURN_VALUE
-"""%(bug1333982.func_code.co_firstlineno + 1,
-     bug1333982.func_code.co_firstlineno + 2,
-     bug1333982.func_code.co_firstlineno + 3)
-
-_BIG_LINENO_FORMAT = """\
-%3d           0 LOAD_GLOBAL              0 (spam)
-              3 POP_TOP
-              4 LOAD_CONST               0 (None)
-              7 RETURN_VALUE
-"""
-
-class DisTests(unittest.TestCase):
-    def do_disassembly_test(self, func, expected):
-        s = StringIO.StringIO()
-        save_stdout = sys.stdout
-        sys.stdout = s
-        dis.dis(func)
-        sys.stdout = save_stdout
-        got = s.getvalue()
-        # Trim trailing blanks (if any).
-        lines = got.split('\n')
-        lines = [line.rstrip() for line in lines]
-        expected = expected.split("\n")
-        import difflib
-        if expected != lines:
-            self.fail(
-                "events did not match expectation:\n" +
-                "\n".join(difflib.ndiff(expected,
-                                        lines)))
-
-    def test_opmap(self):
-        self.assertEqual(dis.opmap["STOP_CODE"], 0)
-        self.assertIn(dis.opmap["LOAD_CONST"], dis.hasconst)
-        self.assertIn(dis.opmap["STORE_NAME"], dis.hasname)
-
-    def test_opname(self):
-        self.assertEqual(dis.opname[dis.opmap["LOAD_FAST"]], "LOAD_FAST")
-
-    def test_boundaries(self):
-        self.assertEqual(dis.opmap["EXTENDED_ARG"], dis.EXTENDED_ARG)
-        self.assertEqual(dis.opmap["STORE_NAME"], dis.HAVE_ARGUMENT)
-
-    def test_dis(self):
-        self.do_disassembly_test(_f, dis_f)
-
-    def test_bug_708901(self):
-        self.do_disassembly_test(bug708901, dis_bug708901)
-
-    def test_bug_1333982(self):
-        # This one is checking bytecodes generated for an `assert` statement,
-        # so fails if the tests are run with -O.  Skip this test then.
-        if __debug__:
-            self.do_disassembly_test(bug1333982, dis_bug1333982)
-
-    def test_big_linenos(self):
-        def func(count):
-            namespace = {}
-            func = "def foo():\n " + "".join(["\n "] * count + ["spam\n"])
-            exec func in namespace
-            return namespace['foo']
-
-        # Test all small ranges
-        for i in xrange(1, 300):
-            expected = _BIG_LINENO_FORMAT % (i + 2)
-            self.do_disassembly_test(func(i), expected)
-
-        # Test some larger ranges too
-        for i in xrange(300, 5000, 10):
-            expected = _BIG_LINENO_FORMAT % (i + 2)
-            self.do_disassembly_test(func(i), expected)
-
-def test_main():
-    run_unittest(DisTests)
-
-
-if __name__ == "__main__":
-    test_main()
diff --git a/lib-python/modified-2.7/test/test_extcall.py b/lib-python/modified-2.7/test/test_extcall.py
--- a/lib-python/modified-2.7/test/test_extcall.py
+++ b/lib-python/modified-2.7/test/test_extcall.py
@@ -299,7 +299,7 @@
         def f(a):
             return a
         self.assertEqual(f(**{u'a': 4}), 4)
-        self.assertRaises(TypeError, lambda: f(**{u'st&#246;ren': 4}))
+        self.assertRaises(TypeError, f, **{u'st&#246;ren': 4})
         self.assertRaises(TypeError, f, **{u'someLongString':2})
         try:
             f(a=4, **{u'a': 4})
diff --git a/lib-python/modified-2.7/test/test_fcntl.py b/lib-python/modified-2.7/test/test_fcntl.py
new file mode 100644
--- /dev/null
+++ b/lib-python/modified-2.7/test/test_fcntl.py
@@ -0,0 +1,108 @@
+"""Test program for the fcntl C module.
+
+OS/2+EMX doesn't support the file locking operations.
+
+"""
+import os
+import struct
+import sys
+import unittest
+from test.test_support import (verbose, TESTFN, unlink, run_unittest,
+    import_module)
+
+# Skip test if no fnctl module.
+fcntl = import_module('fcntl')
+
+
+# TODO - Write tests for flock() and lockf().
+
+def get_lockdata():
+    if sys.platform.startswith('atheos'):
+        start_len = "qq"
+    else:
+        try:
+            os.O_LARGEFILE
+        except AttributeError:
+            start_len = "ll"
+        else:
+            start_len = "qq"
+
+    if sys.platform in ('netbsd1', 'netbsd2', 'netbsd3',
+                        'Darwin1.2', 'darwin',
+                        'freebsd2', 'freebsd3', 'freebsd4', 'freebsd5',
+                        'freebsd6', 'freebsd7', 'freebsd8',
+                        'bsdos2', 'bsdos3', 'bsdos4',
+                        'openbsd', 'openbsd2', 'openbsd3', 'openbsd4', 'openbsd5'):
+        if struct.calcsize('l') == 8:
+            off_t = 'l'
+            pid_t = 'i'
+        else:
+            off_t = 'lxxxx'
+            pid_t = 'l'
+        lockdata = struct.pack(off_t + off_t + pid_t + 'hh', 0, 0, 0,
+                               fcntl.F_WRLCK, 0)
+    elif sys.platform in ['aix3', 'aix4', 'hp-uxB', 'unixware7']:
+        lockdata = struct.pack('hhlllii', fcntl.F_WRLCK, 0, 0, 0, 0, 0, 0)
+    elif sys.platform in ['os2emx']:
+        lockdata = None
+    else:
+        lockdata = struct.pack('hh'+start_len+'hh', fcntl.F_WRLCK, 0, 0, 0, 0, 0)
+    if lockdata:
+        if verbose:
+            print 'struct.pack: ', repr(lockdata)
+    return lockdata
+
+lockdata = get_lockdata()
+
+
+class TestFcntl(unittest.TestCase):
+
+    def setUp(self):
+        self.f = None
+
+    def tearDown(self):
+        if self.f and not self.f.closed:
+            self.f.close()
+        unlink(TESTFN)
+
+    def test_fcntl_fileno(self):
+        # the example from the library docs
+        self.f = open(TESTFN, 'w')
+        rv = fcntl.fcntl(self.f.fileno(), fcntl.F_SETFL, os.O_NONBLOCK)
+        if verbose:
+            print 'Status from fcntl with O_NONBLOCK: ', rv
+        if sys.platform not in ['os2emx']:
+            rv = fcntl.fcntl(self.f.fileno(), fcntl.F_SETLKW, lockdata)
+            if verbose:
+                print 'String from fcntl with F_SETLKW: ', repr(rv)
+        self.f.close()
+
+    def test_fcntl_file_descriptor(self):
+        # again, but pass the file rather than numeric descriptor
+        self.f = open(TESTFN, 'w')
+        rv = fcntl.fcntl(self.f, fcntl.F_SETFL, os.O_NONBLOCK)
+        if sys.platform not in ['os2emx']:
+            rv = fcntl.fcntl(self.f, fcntl.F_SETLKW, lockdata)
+        self.f.close()
+
+    def test_fcntl_64_bit(self):
+        # Issue #1309352: fcntl shouldn't fail when the third arg fits in a
+        # C 'long' but not in a C 'int'.
+        try:
+            cmd = fcntl.F_NOTIFY
+            # This flag is larger than 2**31 in 64-bit builds
+            flags = fcntl.DN_MULTISHOT
+        except AttributeError:
+            self.skipTest("F_NOTIFY or DN_MULTISHOT unavailable")
+        fd = os.open(os.path.dirname(os.path.abspath(TESTFN)), os.O_RDONLY)
+        try:
+            fcntl.fcntl(fd, cmd, flags)
+        finally:
+            os.close(fd)
+
+
+def test_main():
+    run_unittest(TestFcntl)
+
+if __name__ == '__main__':
+    test_main()
diff --git a/lib-python/2.7/test/test_multibytecodec.py b/lib-python/modified-2.7/test/test_multibytecodec.py
copy from lib-python/2.7/test/test_multibytecodec.py
copy to lib-python/modified-2.7/test/test_multibytecodec.py
--- a/lib-python/2.7/test/test_multibytecodec.py
+++ b/lib-python/modified-2.7/test/test_multibytecodec.py
@@ -42,7 +42,7 @@
         dec = codecs.getdecoder('euc-kr')
         myreplace  = lambda exc: (u'', sys.maxint+1)
         codecs.register_error('test.cjktest', myreplace)
-        self.assertRaises(IndexError, dec,
+        self.assertRaises((IndexError, OverflowError), dec,
                           'apple\x92ham\x93spam', 'test.cjktest')
 
     def test_codingspec(self):
@@ -148,7 +148,8 @@
 class Test_StreamReader(unittest.TestCase):
     def test_bug1728403(self):
         try:
-            open(TESTFN, 'w').write('\xa1')
+            with open(TESTFN, 'w') as f:
+                f.write('\xa1')
             f = codecs.open(TESTFN, encoding='cp949')
             self.assertRaises(UnicodeDecodeError, f.read, 2)
         finally:
diff --git a/lib-python/2.7/test/test_multibytecodec_support.py b/lib-python/modified-2.7/test/test_multibytecodec_support.py
copy from lib-python/2.7/test/test_multibytecodec_support.py
copy to lib-python/modified-2.7/test/test_multibytecodec_support.py
--- a/lib-python/2.7/test/test_multibytecodec_support.py
+++ b/lib-python/modified-2.7/test/test_multibytecodec_support.py
@@ -107,8 +107,8 @@
         def myreplace(exc):
             return (u'x', sys.maxint + 1)
         codecs.register_error("test.cjktest", myreplace)
-        self.assertRaises(IndexError, self.encode, self.unmappedunicode,
-                          'test.cjktest')
+        self.assertRaises((IndexError, OverflowError), self.encode,
+                          self.unmappedunicode, 'test.cjktest')
 
     def test_callback_None_index(self):
         def myreplace(exc):
diff --git a/lib-python/modified-2.7/test/test_multiprocessing.py b/lib-python/modified-2.7/test/test_multiprocessing.py
--- a/lib-python/modified-2.7/test/test_multiprocessing.py
+++ b/lib-python/modified-2.7/test/test_multiprocessing.py
@@ -510,7 +510,6 @@
 
         p.join()
 
-    @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
     def test_qsize(self):
         q = self.Queue()
         try:
@@ -532,7 +531,6 @@
             time.sleep(DELTA)
             q.task_done()
 
-    @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
     def test_task_done(self):
         queue = self.JoinableQueue()
 
@@ -1091,7 +1089,6 @@
 class _TestPoolWorkerLifetime(BaseTestCase):
 
     ALLOWED_TYPES = ('processes', )
-    @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
     def test_pool_worker_lifetime(self):
         p = multiprocessing.Pool(3, maxtasksperchild=10)
         self.assertEqual(3, len(p._pool))
@@ -1280,7 +1277,6 @@
         queue = manager.get_queue()
         queue.put('hello world')
 
-    @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
     def test_rapid_restart(self):
         authkey = os.urandom(32)
         manager = QueueManager(
@@ -1297,6 +1293,7 @@
         queue = manager.get_queue()
         self.assertEqual(queue.get(), 'hello world')
         del queue
+        test_support.gc_collect()
         manager.shutdown()
         manager = QueueManager(
             address=addr, authkey=authkey, serializer=SERIALIZER)
@@ -1573,7 +1570,6 @@
 
     ALLOWED_TYPES = ('processes',)
 
-    @unittest.skipIf(os.name == 'posix', "PYPY: FIXME")
     def test_heap(self):
         iterations = 5000
         maxblocks = 50
diff --git a/lib-python/2.7/test/test_sets.py b/lib-python/modified-2.7/test/test_sets.py
copy from lib-python/2.7/test/test_sets.py
copy to lib-python/modified-2.7/test/test_sets.py
--- a/lib-python/2.7/test/test_sets.py
+++ b/lib-python/modified-2.7/test/test_sets.py
@@ -686,7 +686,9 @@
         set_list = sorted(self.set)
         self.assertEqual(len(dup_list), len(set_list))
         for i, el in enumerate(dup_list):
-            self.assertIs(el, set_list[i])
+            # Object identity is not guarnteed for immutable objects, so we
+            # can't use assertIs here.
+            self.assertEqual(el, set_list[i])
 
     def test_deep_copy(self):
         dup = copy.deepcopy(self.set)
diff --git a/lib-python/modified-2.7/test/test_ssl.py b/lib-python/modified-2.7/test/test_ssl.py
--- a/lib-python/modified-2.7/test/test_ssl.py
+++ b/lib-python/modified-2.7/test/test_ssl.py
@@ -58,32 +58,35 @@
 
 # Issue #9415: Ubuntu hijacks their OpenSSL and forcefully disables SSLv2
 def skip_if_broken_ubuntu_ssl(func):
-    # We need to access the lower-level wrapper in order to create an
-    # implicit SSL context without trying to connect or listen.
-    try:
-        import _ssl
-    except ImportError:
-        # The returned function won't get executed, just ignore the error
-        pass
-    @functools.wraps(func)
-    def f(*args, **kwargs):
+    if hasattr(ssl, 'PROTOCOL_SSLv2'):
+        # We need to access the lower-level wrapper in order to create an
+        # implicit SSL context without trying to connect or listen.
         try:
-            s = socket.socket(socket.AF_INET)
-            _ssl.sslwrap(s._sock, 0, None, None,
-                         ssl.CERT_NONE, ssl.PROTOCOL_SSLv2, None, None)
-        except ssl.SSLError as e:
-            if (ssl.OPENSSL_VERSION_INFO == (0, 9, 8, 15, 15) and
-                platform.linux_distribution() == ('debian', 'squeeze/sid', '')
-                and 'Invalid SSL protocol variant specified' in str(e)):
-                raise unittest.SkipTest("Patched Ubuntu OpenSSL breaks behaviour")
-        return func(*args, **kwargs)
-    return f
+            import _ssl
+        except ImportError:
+            # The returned function won't get executed, just ignore the error
+            pass
+        @functools.wraps(func)
+        def f(*args, **kwargs):
+            try:
+                s = socket.socket(socket.AF_INET)
+                _ssl.sslwrap(s._sock, 0, None, None,
+                             ssl.CERT_NONE, ssl.PROTOCOL_SSLv2, None, None)
+            except ssl.SSLError as e:
+                if (ssl.OPENSSL_VERSION_INFO == (0, 9, 8, 15, 15) and
+                    platform.linux_distribution() == ('debian', 'squeeze/sid', '')
+                    and 'Invalid SSL protocol variant specified' in str(e)):
+                    raise unittest.SkipTest("Patched Ubuntu OpenSSL breaks behaviour")
+            return func(*args, **kwargs)
+        return f
+    else:
+        return func
 
 
 class BasicSocketTests(unittest.TestCase):
 
     def test_constants(self):
-        ssl.PROTOCOL_SSLv2
+        #ssl.PROTOCOL_SSLv2
         ssl.PROTOCOL_SSLv23
         ssl.PROTOCOL_SSLv3
         ssl.PROTOCOL_TLSv1
@@ -966,7 +969,8 @@
             try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, True)
             try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, True, ssl.CERT_OPTIONAL)
             try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, True, ssl.CERT_REQUIRED)
-            try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv2, False)
+            if hasattr(ssl, 'PROTOCOL_SSLv2'):
+                try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv2, False)
             try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv23, False)
             try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_TLSv1, False)
 
@@ -978,7 +982,8 @@
             try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, True)
             try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, True, ssl.CERT_OPTIONAL)
             try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, True, ssl.CERT_REQUIRED)
-            try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv2, False)
+            if hasattr(ssl, 'PROTOCOL_SSLv2'):
+                try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv2, False)
             try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv3, False)
             try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv23, False)
 
diff --git a/lib-python/modified-2.7/test/test_support.py b/lib-python/modified-2.7/test/test_support.py
--- a/lib-python/modified-2.7/test/test_support.py
+++ b/lib-python/modified-2.7/test/test_support.py
@@ -1066,7 +1066,7 @@
         if '--pdb' in sys.argv:
             import pdb, traceback
             traceback.print_tb(exc_info[2])
-            pdb.post_mortem(exc_info[2], pdb.Pdb)
+            pdb.post_mortem(exc_info[2])
 
 # ----------------------------------
 
diff --git a/lib-python/modified-2.7/test/test_sys_settrace.py b/lib-python/modified-2.7/test/test_sys_settrace.py
--- a/lib-python/modified-2.7/test/test_sys_settrace.py
+++ b/lib-python/modified-2.7/test/test_sys_settrace.py
@@ -286,11 +286,11 @@
         self.compare_events(func.func_code.co_firstlineno,
                             tracer.events, func.events)
 
-    def set_and_retrieve_none(self):
+    def test_set_and_retrieve_none(self):
         sys.settrace(None)
         assert sys.gettrace() is None
 
-    def set_and_retrieve_func(self):
+    def test_set_and_retrieve_func(self):
         def fn(*args):
             pass
 
diff --git a/lib-python/2.7/test/test_tarfile.py b/lib-python/modified-2.7/test/test_tarfile.py
copy from lib-python/2.7/test/test_tarfile.py
copy to lib-python/modified-2.7/test/test_tarfile.py
--- a/lib-python/2.7/test/test_tarfile.py
+++ b/lib-python/modified-2.7/test/test_tarfile.py
@@ -169,6 +169,7 @@
         except tarfile.ReadError:
             self.fail("tarfile.open() failed on empty archive")
         self.assertListEqual(tar.getmembers(), [])
+        tar.close()
 
     def test_null_tarfile(self):
         # Test for issue6123: Allow opening empty archives.
@@ -207,16 +208,21 @@
         fobj = open(self.tarname, "rb")
         tar = tarfile.open(fileobj=fobj, mode=self.mode)
         self.assertEqual(tar.name, os.path.abspath(fobj.name))
+        tar.close()
 
     def test_no_name_attribute(self):
-        data = open(self.tarname, "rb").read()
+        f = open(self.tarname, "rb")
+        data = f.read()
+        f.close()
         fobj = StringIO.StringIO(data)
         self.assertRaises(AttributeError, getattr, fobj, "name")
         tar = tarfile.open(fileobj=fobj, mode=self.mode)
         self.assertEqual(tar.name, None)
 
     def test_empty_name_attribute(self):
-        data = open(self.tarname, "rb").read()
+        f = open(self.tarname, "rb")
+        data = f.read()
+        f.close()
         fobj = StringIO.StringIO(data)
         fobj.name = ""
         tar = tarfile.open(fileobj=fobj, mode=self.mode)
@@ -515,6 +521,7 @@
         self.tar = tarfile.open(self.tarname, mode=self.mode, encoding="iso8859-1")
         tarinfo = self.tar.getmember("pax/umlauts-&#65533;&#65533;&#65533;&#65533;&#65533;&#65533;&#65533;")
         self._test_member(tarinfo, size=7011, chksum=md5_regtype)
+        self.tar.close()
 
 
 class LongnameTest(ReadTest):
@@ -675,6 +682,7 @@
             tar = tarfile.open(tmpname, self.mode)
             tarinfo = tar.gettarinfo(path)
             self.assertEqual(tarinfo.size, 0)
+            tar.close()
         finally:
             os.rmdir(path)
 
@@ -692,6 +700,7 @@
                 tar.gettarinfo(target)
                 tarinfo = tar.gettarinfo(link)
                 self.assertEqual(tarinfo.size, 0)
+                tar.close()
             finally:
                 os.remove(target)
                 os.remove(link)
@@ -704,6 +713,7 @@
                 tar = tarfile.open(tmpname, self.mode)
                 tarinfo = tar.gettarinfo(path)
                 self.assertEqual(tarinfo.size, 0)
+                tar.close()
             finally:
                 os.remove(path)
 
@@ -722,6 +732,7 @@
         tar.add(dstname)
         os.chdir(cwd)
         self.assertTrue(tar.getnames() == [], "added the archive to itself")
+        tar.close()
 
     def test_exclude(self):
         tempdir = os.path.join(TEMPDIR, "exclude")
@@ -742,6 +753,7 @@
             tar = tarfile.open(tmpname, "r")
             self.assertEqual(len(tar.getmembers()), 1)
             self.assertEqual(tar.getnames()[0], "empty_dir")
+            tar.close()
         finally:
             shutil.rmtree(tempdir)
 
@@ -859,7 +871,9 @@
             fobj.close()
         elif self.mode.endswith("bz2"):
             dec = bz2.BZ2Decompressor()
-            data = open(tmpname, "rb").read()
+            f = open(tmpname, "rb")
+            data = f.read()
+            f.close()
             data = dec.decompress(data)
             self.assertTrue(len(dec.unused_data) == 0,
                     "found trailing data")
@@ -938,6 +952,7 @@
                 "unable to read longname member")
         self.assertEqual(tarinfo.linkname, member.linkname,
                 "unable to read longname member")
+        tar.close()
 
     def test_longname_1023(self):
         self._test(("longnam/" * 127) + "longnam")
@@ -1030,6 +1045,7 @@
         else:
             n = tar.getmembers()[0].name
             self.assertTrue(name == n, "PAX longname creation failed")
+        tar.close()
 
     def test_pax_global_header(self):
         pax_headers = {
@@ -1058,6 +1074,7 @@
                     tarfile.PAX_NUMBER_FIELDS[key](val)
                 except (TypeError, ValueError):
                     self.fail("unable to convert pax header field")
+        tar.close()
 
     def test_pax_extended_header(self):
         # The fields from the pax header have priority over the
@@ -1077,6 +1094,7 @@
         self.assertEqual(t.pax_headers, pax_headers)
         self.assertEqual(t.name, "foo")
         self.assertEqual(t.uid, 123)
+        tar.close()
 
 
 class UstarUnicodeTest(unittest.TestCase):
@@ -1120,6 +1138,7 @@
         tarinfo.name = "foo"
         tarinfo.uname = u"&#65533;&#65533;&#65533;"
         self.assertRaises(UnicodeError, tar.addfile, tarinfo)
+        tar.close()
 
     def test_unicode_argument(self):
         tar = tarfile.open(tarname, "r", encoding="iso8859-1", errors="strict")
@@ -1174,6 +1193,7 @@
             tar = tarfile.open(tmpname, format=self.format, encoding="ascii",
                     errors=handler)
             self.assertEqual(tar.getnames()[0], name)
+            tar.close()
 
         self.assertRaises(UnicodeError, tarfile.open, tmpname,
                 encoding="ascii", errors="strict")
@@ -1186,6 +1206,7 @@
         tar = tarfile.open(tmpname, format=self.format, encoding="iso8859-1",
                 errors="utf-8")
         self.assertEqual(tar.getnames()[0], "&#65533;&#65533;&#65533;/" + u"&#65533;".encode("utf8"))
+        tar.close()
 
 
 class AppendTest(unittest.TestCase):
@@ -1213,6 +1234,7 @@
     def _test(self, names=["bar"], fileobj=None):
         tar = tarfile.open(self.tarname, fileobj=fileobj)
         self.assertEqual(tar.getnames(), names)
+        tar.close()
 
     def test_non_existing(self):
         self._add_testfile()
@@ -1231,7 +1253,9 @@
 
     def test_fileobj(self):
         self._create_testtar()
-        data = open(self.tarname).read()
+        f = open(self.tarname)
+        data = f.read()
+        f.close()
         fobj = StringIO.StringIO(data)
         self._add_testfile(fobj)
         fobj.seek(0)
@@ -1257,7 +1281,9 @@
     # Append mode is supposed to fail if the tarfile to append to
     # does not end with a zero block.
     def _test_error(self, data):
-        open(self.tarname, "wb").write(data)
+        f = open(self.tarname, "wb")
+        f.write(data)
+        f.close()
         self.assertRaises(tarfile.ReadError, self._add_testfile)
 
     def test_null(self):
diff --git a/lib-python/modified-2.7/test/test_tempfile.py b/lib-python/modified-2.7/test/test_tempfile.py
--- a/lib-python/modified-2.7/test/test_tempfile.py
+++ b/lib-python/modified-2.7/test/test_tempfile.py
@@ -23,8 +23,8 @@
 
 # TEST_FILES may need to be tweaked for systems depending on the maximum
 # number of files that can be opened at one time (see ulimit -n)
-if sys.platform in ('openbsd3', 'openbsd4'):
-    TEST_FILES = 48
+if sys.platform.startswith("openbsd"):
+    TEST_FILES = 64 # ulimit -n defaults to 128 for normal users
 else:
     TEST_FILES = 100
 
diff --git a/lib-python/modified-2.7/test/test_urllib2.py b/lib-python/modified-2.7/test/test_urllib2.py
--- a/lib-python/modified-2.7/test/test_urllib2.py
+++ b/lib-python/modified-2.7/test/test_urllib2.py
@@ -307,6 +307,9 @@
     def getresponse(self):
         return MockHTTPResponse(MockFile(), {}, 200, "OK")
 
+    def close(self):
+        pass
+
 class MockHandler:
     # useful for testing handler machinery
     # see add_ordered_mock_handlers() docstring
diff --git a/lib-python/modified-2.7/test/test_weakref.py b/lib-python/modified-2.7/test/test_weakref.py
--- a/lib-python/modified-2.7/test/test_weakref.py
+++ b/lib-python/modified-2.7/test/test_weakref.py
@@ -993,13 +993,13 @@
         self.assertTrue(len(weakdict) == 2)
         k, v = weakdict.popitem()
         self.assertTrue(len(weakdict) == 1)
-        if k is key1:
+        if k == key1:
             self.assertTrue(v is value1)
         else:
             self.assertTrue(v is value2)
         k, v = weakdict.popitem()
         self.assertTrue(len(weakdict) == 0)
-        if k is key1:
+        if k == key1:
             self.assertTrue(v is value1)
         else:
             self.assertTrue(v is value2)
diff --git a/lib-python/modified-2.7/urllib2.py b/lib-python/modified-2.7/urllib2.py
new file mode 100644
--- /dev/null
+++ b/lib-python/modified-2.7/urllib2.py
@@ -0,0 +1,1440 @@
+"""An extensible library for opening URLs using a variety of protocols
+
+The simplest way to use this module is to call the urlopen function,
+which accepts a string containing a URL or a Request object (described
+below).  It opens the URL and returns the results as file-like
+object; the returned object has some extra methods described below.
+
+The OpenerDirector manages a collection of Handler objects that do
+all the actual work.  Each Handler implements a particular protocol or
+option.  The OpenerDirector is a composite object that invokes the
+Handlers needed to open the requested URL.  For example, the
+HTTPHandler performs HTTP GET and POST requests and deals with
+non-error returns.  The HTTPRedirectHandler automatically deals with
+HTTP 301, 302, 303 and 307 redirect errors, and the HTTPDigestAuthHandler
+deals with digest authentication.
+
+urlopen(url, data=None) -- Basic usage is the same as original
+urllib.  pass the url and optionally data to post to an HTTP URL, and
+get a file-like object back.  One difference is that you can also pass
+a Request instance instead of URL.  Raises a URLError (subclass of
+IOError); for HTTP errors, raises an HTTPError, which can also be
+treated as a valid response.
+
+build_opener -- Function that creates a new OpenerDirector instance.
+Will install the default handlers.  Accepts one or more Handlers as
+arguments, either instances or Handler classes that it will
+instantiate.  If one of the argument is a subclass of the default
+handler, the argument will be installed instead of the default.
+
+install_opener -- Installs a new opener as the default opener.
+
+objects of interest:
+
+OpenerDirector -- Sets up the User Agent as the Python-urllib client and manages
+the Handler classes, while dealing with requests and responses.
+
+Request -- An object that encapsulates the state of a request.  The
+state can be as simple as the URL.  It can also include extra HTTP
+headers, e.g. a User-Agent.
+
+BaseHandler --
+
+exceptions:
+URLError -- A subclass of IOError, individual protocols have their own
+specific subclass.
+
+HTTPError -- Also a valid HTTP response, so you can treat an HTTP error
+as an exceptional event or valid response.
+
+internals:
+BaseHandler and parent
+_call_chain conventions
+
+Example usage:
+
+import urllib2
+
+# set up authentication info
+authinfo = urllib2.HTTPBasicAuthHandler()
+authinfo.add_password(realm='PDQ Application',
+                      uri='https://mahler:8092/site-updates.py',
+                      user='klem',
+                      passwd='geheim$parole')
+
+proxy_support = urllib2.ProxyHandler({"http" : "http://ahad-haam:3128"})
+
+# build a new opener that adds authentication and caching FTP handlers
+opener = urllib2.build_opener(proxy_support, authinfo, urllib2.CacheFTPHandler)
+
+# install it
+urllib2.install_opener(opener)
+
+f = urllib2.urlopen('http://www.python.org/')
+
+
+"""
+
+# XXX issues:
+# If an authentication error handler that tries to perform
+# authentication for some reason but fails, how should the error be
+# signalled?  The client needs to know the HTTP error code.  But if
+# the handler knows that the problem was, e.g., that it didn't know
+# that hash algo that requested in the challenge, it would be good to
+# pass that information along to the client, too.
+# ftp errors aren't handled cleanly
+# check digest against correct (i.e. non-apache) implementation
+
+# Possible extensions:
+# complex proxies  XXX not sure what exactly was meant by this
+# abstract factory for opener
+
+import base64
+import hashlib
+import httplib
+import mimetools
+import os
+import posixpath
+import random
+import re
+import socket
+import sys
+import time
+import urlparse
+import bisect
+
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from StringIO import StringIO
+
+from urllib import (unwrap, unquote, splittype, splithost, quote,
+     addinfourl, splitport, splittag,
+     splitattr, ftpwrapper, splituser, splitpasswd, splitvalue)
+
+# support for FileHandler, proxies via environment variables
+from urllib import localhost, url2pathname, getproxies, proxy_bypass
+
+# used in User-Agent header sent
+__version__ = sys.version[:3]
+
+_opener = None
+def urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
+    global _opener
+    if _opener is None:
+        _opener = build_opener()
+    return _opener.open(url, data, timeout)
+
+def install_opener(opener):
+    global _opener
+    _opener = opener
+
+# do these error classes make sense?
+# make sure all of the IOError stuff is overridden.  we just want to be
+# subtypes.
+
+class URLError(IOError):
+    # URLError is a sub-type of IOError, but it doesn't share any of
+    # the implementation.  need to override __init__ and __str__.
+    # It sets self.args for compatibility with other EnvironmentError
+    # subclasses, but args doesn't have the typical format with errno in
+    # slot 0 and strerror in slot 1.  This may be better than nothing.
+    def __init__(self, reason):
+        self.args = reason,
+        self.reason = reason
+
+    def __str__(self):
+        return '<urlopen error %s>' % self.reason
+
+class HTTPError(URLError, addinfourl):
+    """Raised when HTTP error occurs, but also acts like non-error return"""
+    __super_init = addinfourl.__init__
+
+    def __init__(self, url, code, msg, hdrs, fp):
+        self.code = code
+        self.msg = msg
+        self.hdrs = hdrs
+        self.fp = fp
+        self.filename = url
+        # The addinfourl classes depend on fp being a valid file
+        # object.  In some cases, the HTTPError may not have a valid
+        # file object.  If this happens, the simplest workaround is to
+        # not initialize the base classes.
+        if fp is not None:
+            self.__super_init(fp, hdrs, url, code)
+
+    def __str__(self):
+        return 'HTTP Error %s: %s' % (self.code, self.msg)
+
+# copied from cookielib.py
+_cut_port_re = re.compile(r":\d+$")
+def request_host(request):
+    """Return request-host, as defined by RFC 2965.
+
+    Variation from RFC: returned value is lowercased, for convenient
+    comparison.
+
+    """
+    url = request.get_full_url()
+    host = urlparse.urlparse(url)[1]
+    if host == "":
+        host = request.get_header("Host", "")
+
+    # remove port, if present
+    host = _cut_port_re.sub("", host, 1)
+    return host.lower()
+
+class Request:
+
+    def __init__(self, url, data=None, headers={},
+                 origin_req_host=None, unverifiable=False):
+        # unwrap('<URL:type://host/path>') --> 'type://host/path'
+        self.__original = unwrap(url)
+        self.__original, fragment = splittag(self.__original)
+        self.type = None
+        # self.__r_type is what's left after doing the splittype
+        self.host = None
+        self.port = None
+        self._tunnel_host = None
+        self.data = data
+        self.headers = {}
+        for key, value in headers.items():
+            self.add_header(key, value)
+        self.unredirected_hdrs = {}
+        if origin_req_host is None:
+            origin_req_host = request_host(self)
+        self.origin_req_host = origin_req_host
+        self.unverifiable = unverifiable
+
+    def __getattr__(self, attr):
+        # XXX this is a fallback mechanism to guard against these
+        # methods getting called in a non-standard order.  this may be
+        # too complicated and/or unnecessary.
+        # XXX should the __r_XXX attributes be public?
+        if attr[:12] == '_Request__r_':
+            name = attr[12:]
+            if hasattr(Request, 'get_' + name):
+                getattr(self, 'get_' + name)()
+                return getattr(self, attr)
+        raise AttributeError, attr
+
+    def get_method(self):
+        if self.has_data():
+            return "POST"
+        else:
+            return "GET"
+
+    # XXX these helper methods are lame
+
+    def add_data(self, data):
+        self.data = data
+
+    def has_data(self):
+        return self.data is not None
+
+    def get_data(self):
+        return self.data
+
+    def get_full_url(self):
+        return self.__original
+
+    def get_type(self):
+        if self.type is None:
+            self.type, self.__r_type = splittype(self.__original)
+            if self.type is None:
+                raise ValueError, "unknown url type: %s" % self.__original
+        return self.type
+
+    def get_host(self):
+        if self.host is None:
+            self.host, self.__r_host = splithost(self.__r_type)
+            if self.host:
+                self.host = unquote(self.host)
+        return self.host
+
+    def get_selector(self):
+        return self.__r_host
+
+    def set_proxy(self, host, type):
+        if self.type == 'https' and not self._tunnel_host:
+            self._tunnel_host = self.host
+        else:
+            self.type = type
+            self.__r_host = self.__original
+
+        self.host = host
+
+    def has_proxy(self):
+        return self.__r_host == self.__original
+
+    def get_origin_req_host(self):
+        return self.origin_req_host
+
+    def is_unverifiable(self):
+        return self.unverifiable
+
+    def add_header(self, key, val):
+        # useful for something like authentication
+        self.headers[key.capitalize()] = val
+
+    def add_unredirected_header(self, key, val):
+        # will not be added to a redirected request
+        self.unredirected_hdrs[key.capitalize()] = val
+
+    def has_header(self, header_name):
+        return (header_name in self.headers or
+                header_name in self.unredirected_hdrs)
+
+    def get_header(self, header_name, default=None):
+        return self.headers.get(
+            header_name,
+            self.unredirected_hdrs.get(header_name, default))
+
+    def header_items(self):
+        hdrs = self.unredirected_hdrs.copy()
+        hdrs.update(self.headers)
+        return hdrs.items()
+
+class OpenerDirector:
+    def __init__(self):
+        client_version = "Python-urllib/%s" % __version__
+        self.addheaders = [('User-agent', client_version)]
+        # manage the individual handlers
+        self.handlers = []
+        self.handle_open = {}
+        self.handle_error = {}
+        self.process_response = {}
+        self.process_request = {}
+
+    def add_handler(self, handler):
+        if not hasattr(handler, "add_parent"):
+            raise TypeError("expected BaseHandler instance, got %r" %
+                            type(handler))
+
+        added = False
+        for meth in dir(handler):
+            if meth in ["redirect_request", "do_open", "proxy_open"]:
+                # oops, coincidental match
+                continue
+
+            i = meth.find("_")
+            protocol = meth[:i]
+            condition = meth[i+1:]
+
+            if condition.startswith("error"):
+                j = condition.find("_") + i + 1
+                kind = meth[j+1:]
+                try:
+                    kind = int(kind)
+                except ValueError:
+                    pass
+                lookup = self.handle_error.get(protocol, {})
+                self.handle_error[protocol] = lookup
+            elif condition == "open":
+                kind = protocol
+                lookup = self.handle_open
+            elif condition == "response":
+                kind = protocol
+                lookup = self.process_response
+            elif condition == "request":
+                kind = protocol
+                lookup = self.process_request
+            else:
+                continue
+
+            handlers = lookup.setdefault(kind, [])
+            if handlers:
+                bisect.insort(handlers, handler)
+            else:
+                handlers.append(handler)
+            added = True
+
+        if added:
+            # the handlers must work in an specific order, the order
+            # is specified in a Handler attribute
+            bisect.insort(self.handlers, handler)
+            handler.add_parent(self)
+
+    def close(self):
+        # Only exists for backwards compatibility.
+        pass
+
+    def _call_chain(self, chain, kind, meth_name, *args):
+        # Handlers raise an exception if no one else should try to handle
+        # the request, or return None if they can't but another handler
+        # could.  Otherwise, they return the response.
+        handlers = chain.get(kind, ())
+        for handler in handlers:
+            func = getattr(handler, meth_name)
+
+            result = func(*args)
+            if result is not None:
+                return result
+
+    def open(self, fullurl, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
+        # accept a URL or a Request object
+        if isinstance(fullurl, basestring):
+            req = Request(fullurl, data)
+        else:
+            req = fullurl
+            if data is not None:
+                req.add_data(data)
+
+        req.timeout = timeout
+        protocol = req.get_type()
+
+        # pre-process request
+        meth_name = protocol+"_request"
+        for processor in self.process_request.get(protocol, []):
+            meth = getattr(processor, meth_name)
+            req = meth(req)
+
+        response = self._open(req, data)
+
+        # post-process response
+        meth_name = protocol+"_response"
+        for processor in self.process_response.get(protocol, []):
+            meth = getattr(processor, meth_name)
+            try:
+                response = meth(req, response)
+            except:
+                response.close()
+                raise
+
+        return response
+
+    def _open(self, req, data=None):
+        result = self._call_chain(self.handle_open, 'default',
+                                  'default_open', req)
+        if result:
+            return result
+
+        protocol = req.get_type()
+        result = self._call_chain(self.handle_open, protocol, protocol +
+                                  '_open', req)
+        if result:
+            return result
+
+        return self._call_chain(self.handle_open, 'unknown',
+                                'unknown_open', req)
+
+    def error(self, proto, *args):
+        if proto in ('http', 'https'):
+            # XXX http[s] protocols are special-cased
+            dict = self.handle_error['http'] # https is not different than http
+            proto = args[2]  # YUCK!
+            meth_name = 'http_error_%s' % proto
+            http_err = 1
+            orig_args = args
+        else:
+            dict = self.handle_error
+            meth_name = proto + '_error'
+            http_err = 0
+        args = (dict, proto, meth_name) + args
+        result = self._call_chain(*args)
+        if result:
+            return result
+
+        if http_err:
+            args = (dict, 'default', 'http_error_default') + orig_args
+            return self._call_chain(*args)
+
+# XXX probably also want an abstract factory that knows when it makes
+# sense to skip a superclass in favor of a subclass and when it might
+# make sense to include both
+
+def build_opener(*handlers):
+    """Create an opener object from a list of handlers.
+
+    The opener will use several default handlers, including support
+    for HTTP, FTP and when applicable, HTTPS.
+
+    If any of the handlers passed as arguments are subclasses of the
+    default handlers, the default handlers will not be used.
+    """
+    import types
+    def isclass(obj):
+        return isinstance(obj, (types.ClassType, type))
+
+    opener = OpenerDirector()
+    default_classes = [ProxyHandler, UnknownHandler, HTTPHandler,
+                       HTTPDefaultErrorHandler, HTTPRedirectHandler,
+                       FTPHandler, FileHandler, HTTPErrorProcessor]
+    if hasattr(httplib, 'HTTPS'):
+        default_classes.append(HTTPSHandler)
+    skip = set()
+    for klass in default_classes:
+        for check in handlers:
+            if isclass(check):
+                if issubclass(check, klass):
+                    skip.add(klass)
+            elif isinstance(check, klass):
+                skip.add(klass)
+    for klass in skip:
+        default_classes.remove(klass)
+
+    for klass in default_classes:
+        opener.add_handler(klass())
+
+    for h in handlers:
+        if isclass(h):
+            h = h()
+        opener.add_handler(h)
+    return opener
+
+class BaseHandler:
+    handler_order = 500
+
+    def add_parent(self, parent):
+        self.parent = parent
+
+    def close(self):
+        # Only exists for backwards compatibility
+        pass
+
+    def __lt__(self, other):
+        if not hasattr(other, "handler_order"):
+            # Try to preserve the old behavior of having custom classes
+            # inserted after default ones (works only for custom user
+            # classes which are not aware of handler_order).
+            return True
+        return self.handler_order < other.handler_order
+
+
+class HTTPErrorProcessor(BaseHandler):
+    """Process HTTP error responses."""
+    handler_order = 1000  # after all other processing
+
+    def http_response(self, request, response):
+        code, msg, hdrs = response.code, response.msg, response.info()
+
+        # According to RFC 2616, "2xx" code indicates that the client's
+        # request was successfully received, understood, and accepted.
+        if not (200 <= code < 300):
+            response = self.parent.error(
+                'http', request, response, code, msg, hdrs)
+
+        return response
+
+    https_response = http_response
+
+class HTTPDefaultErrorHandler(BaseHandler):
+    def http_error_default(self, req, fp, code, msg, hdrs):
+        raise HTTPError(req.get_full_url(), code, msg, hdrs, fp)
+
+class HTTPRedirectHandler(BaseHandler):
+    # maximum number of redirections to any single URL
+    # this is needed because of the state that cookies introduce
+    max_repeats = 4
+    # maximum total number of redirections (regardless of URL) before
+    # assuming we're in a loop
+    max_redirections = 10
+
+    def redirect_request(self, req, fp, code, msg, headers, newurl):
+        """Return a Request or None in response to a redirect.
+
+        This is called by the http_error_30x methods when a
+        redirection response is received.  If a redirection should
+        take place, return a new Request to allow http_error_30x to
+        perform the redirect.  Otherwise, raise HTTPError if no-one
+        else should try to handle this url.  Return None if you can't
+        but another Handler might.
+        """
+        m = req.get_method()
+        if (code in (301, 302, 303, 307) and m in ("GET", "HEAD")
+            or code in (301, 302, 303) and m == "POST"):
+            # Strictly (according to RFC 2616), 301 or 302 in response
+            # to a POST MUST NOT cause a redirection without confirmation
+            # from the user (of urllib2, in this case).  In practice,
+            # essentially all clients do redirect in this case, so we
+            # do the same.
+            # be conciliant with URIs containing a space
+            newurl = newurl.replace(' ', '%20')
+            newheaders = dict((k,v) for k,v in req.headers.items()
+                              if k.lower() not in ("content-length", "content-type")
+                             )
+            return Request(newurl,
+                           headers=newheaders,
+                           origin_req_host=req.get_origin_req_host(),
+                           unverifiable=True)
+        else:
+            raise HTTPError(req.get_full_url(), code, msg, headers, fp)
+
+    # Implementation note: To avoid the server sending us into an
+    # infinite loop, the request object needs to track what URLs we
+    # have already seen.  Do this by adding a handler-specific
+    # attribute to the Request object.
+    def http_error_302(self, req, fp, code, msg, headers):
+        # Some servers (incorrectly) return multiple Location headers
+        # (so probably same goes for URI).  Use first header.
+        if 'location' in headers:
+            newurl = headers.getheaders('location')[0]
+        elif 'uri' in headers:
+            newurl = headers.getheaders('uri')[0]
+        else:
+            return
+
+        # fix a possible malformed URL
+        urlparts = urlparse.urlparse(newurl)
+        if not urlparts.path:
+            urlparts = list(urlparts)
+            urlparts[2] = "/"
+        newurl = urlparse.urlunparse(urlparts)
+
+        newurl = urlparse.urljoin(req.get_full_url(), newurl)
+
+        # XXX Probably want to forget about the state of the current
+        # request, although that might interact poorly with other
+        # handlers that also use handler-specific request attributes
+        new = self.redirect_request(req, fp, code, msg, headers, newurl)
+        if new is None:
+            return
+
+        # loop detection
+        # .redirect_dict has a key url if url was previously visited.
+        if hasattr(req, 'redirect_dict'):
+            visited = new.redirect_dict = req.redirect_dict
+            if (visited.get(newurl, 0) >= self.max_repeats or
+                len(visited) >= self.max_redirections):
+                raise HTTPError(req.get_full_url(), code,
+                                self.inf_msg + msg, headers, fp)
+        else:
+            visited = new.redirect_dict = req.redirect_dict = {}
+        visited[newurl] = visited.get(newurl, 0) + 1
+
+        # Don't close the fp until we are sure that we won't use it
+        # with HTTPError.
+        fp.read()
+        fp.close()
+
+        return self.parent.open(new, timeout=req.timeout)
+
+    http_error_301 = http_error_303 = http_error_307 = http_error_302
+
+    inf_msg = "The HTTP server returned a redirect error that would " \
+              "lead to an infinite loop.\n" \
+              "The last 30x error message was:\n"
+
+
+def _parse_proxy(proxy):
+    """Return (scheme, user, password, host/port) given a URL or an authority.
+
+    If a URL is supplied, it must have an authority (host:port) component.
+    According to RFC 3986, having an authority component means the URL must
+    have two slashes after the scheme:
+
+    >>> _parse_proxy('file:/ftp.example.com/')
+    Traceback (most recent call last):
+    ValueError: proxy URL with no authority: 'file:/ftp.example.com/'
+
+    The first three items of the returned tuple may be None.
+
+    Examples of authority parsing:
+
+    >>> _parse_proxy('proxy.example.com')
+    (None, None, None, 'proxy.example.com')
+    >>> _parse_proxy('proxy.example.com:3128')
+    (None, None, None, 'proxy.example.com:3128')
+
+    The authority component may optionally include userinfo (assumed to be
+    username:password):
+
+    >>> _parse_proxy('joe:password at proxy.example.com')
+    (None, 'joe', 'password', 'proxy.example.com')
+    >>> _parse_proxy('joe:password at proxy.example.com:3128')
+    (None, 'joe', 'password', 'proxy.example.com:3128')
+
+    Same examples, but with URLs instead:
+
+    >>> _parse_proxy('http://proxy.example.com/')
+    ('http', None, None, 'proxy.example.com')
+    >>> _parse_proxy('http://proxy.example.com:3128/')
+    ('http', None, None, 'proxy.example.com:3128')
+    >>> _parse_proxy('http://joe:password at proxy.example.com/')
+    ('http', 'joe', 'password', 'proxy.example.com')
+    >>> _parse_proxy('http://joe:password at proxy.example.com:3128')
+    ('http', 'joe', 'password', 'proxy.example.com:3128')
+
+    Everything after the authority is ignored:
+
+    >>> _parse_proxy('ftp://joe:password at proxy.example.com/rubbish:3128')
+    ('ftp', 'joe', 'password', 'proxy.example.com')
+
+    Test for no trailing '/' case:
+
+    >>> _parse_proxy('http://joe:password at proxy.example.com')
+    ('http', 'joe', 'password', 'proxy.example.com')
+
+    """
+    scheme, r_scheme = splittype(proxy)
+    if not r_scheme.startswith("/"):
+        # authority
+        scheme = None
+        authority = proxy
+    else:
+        # URL
+        if not r_scheme.startswith("//"):
+            raise ValueError("proxy URL with no authority: %r" % proxy)
+        # We have an authority, so for RFC 3986-compliant URLs (by ss 3.
+        # and 3.3.), path is empty or starts with '/'
+        end = r_scheme.find("/", 2)
+        if end == -1:
+            end = None
+        authority = r_scheme[2:end]
+    userinfo, hostport = splituser(authority)
+    if userinfo is not None:
+        user, password = splitpasswd(userinfo)
+    else:
+        user = password = None
+    return scheme, user, password, hostport
+
+class ProxyHandler(BaseHandler):
+    # Proxies must be in front
+    handler_order = 100
+
+    def __init__(self, proxies=None):
+        if proxies is None:
+            proxies = getproxies()
+        assert hasattr(proxies, 'has_key'), "proxies must be a mapping"
+        self.proxies = proxies
+        for type, url in proxies.items():
+            setattr(self, '%s_open' % type,
+                    lambda r, proxy=url, type=type, meth=self.proxy_open: \
+                    meth(r, proxy, type))
+
+    def proxy_open(self, req, proxy, type):
+        orig_type = req.get_type()
+        proxy_type, user, password, hostport = _parse_proxy(proxy)
+
+        if proxy_type is None:
+            proxy_type = orig_type
+
+        if req.host and proxy_bypass(req.host):
+            return None
+
+        if user and password:
+            user_pass = '%s:%s' % (unquote(user), unquote(password))
+            creds = base64.b64encode(user_pass).strip()
+            req.add_header('Proxy-authorization', 'Basic ' + creds)
+        hostport = unquote(hostport)
+        req.set_proxy(hostport, proxy_type)
+
+        if orig_type == proxy_type or orig_type == 'https':
+            # let other handlers take care of it
+            return None
+        else:
+            # need to start over, because the other handlers don't
+            # grok the proxy's URL type
+            # e.g. if we have a constructor arg proxies like so:
+            # {'http': 'ftp://proxy.example.com'}, we may end up turning
+            # a request for http://acme.example.com/a into one for
+            # ftp://proxy.example.com/a
+            return self.parent.open(req, timeout=req.timeout)
+
+class HTTPPasswordMgr:
+
+    def __init__(self):
+        self.passwd = {}
+
+    def add_password(self, realm, uri, user, passwd):
+        # uri could be a single URI or a sequence
+        if isinstance(uri, basestring):
+            uri = [uri]
+        if not realm in self.passwd:
+            self.passwd[realm] = {}
+        for default_port in True, False:
+            reduced_uri = tuple(
+                [self.reduce_uri(u, default_port) for u in uri])
+            self.passwd[realm][reduced_uri] = (user, passwd)
+
+    def find_user_password(self, realm, authuri):
+        domains = self.passwd.get(realm, {})
+        for default_port in True, False:
+            reduced_authuri = self.reduce_uri(authuri, default_port)
+            for uris, authinfo in domains.iteritems():
+                for uri in uris:
+                    if self.is_suburi(uri, reduced_authuri):
+                        return authinfo
+        return None, None
+
+    def reduce_uri(self, uri, default_port=True):
+        """Accept authority or URI and extract only the authority and path."""
+        # note HTTP URLs do not have a userinfo component
+        parts = urlparse.urlsplit(uri)
+        if parts[1]:
+            # URI
+            scheme = parts[0]
+            authority = parts[1]
+            path = parts[2] or '/'
+        else:
+            # host or host:port
+            scheme = None
+            authority = uri
+            path = '/'
+        host, port = splitport(authority)
+        if default_port and port is None and scheme is not None:
+            dport = {"http": 80,
+                     "https": 443,
+                     }.get(scheme)
+            if dport is not None:
+                authority = "%s:%d" % (host, dport)
+        return authority, path
+
+    def is_suburi(self, base, test):
+        """Check if test is below base in a URI tree
+
+        Both args must be URIs in reduced form.
+        """
+        if base == test:
+            return True
+        if base[0] != test[0]:
+            return False
+        common = posixpath.commonprefix((base[1], test[1]))
+        if len(common) == len(base[1]):
+            return True
+        return False
+
+
+class HTTPPasswordMgrWithDefaultRealm(HTTPPasswordMgr):
+
+    def find_user_password(self, realm, authuri):
+        user, password = HTTPPasswordMgr.find_user_password(self, realm,
+                                                            authuri)
+        if user is not None:
+            return user, password
+        return HTTPPasswordMgr.find_user_password(self, None, authuri)
+
+
+class AbstractBasicAuthHandler:
+
+    # XXX this allows for multiple auth-schemes, but will stupidly pick
+    # the last one with a realm specified.
+
+    # allow for double- and single-quoted realm values
+    # (single quotes are a violation of the RFC, but appear in the wild)
+    rx = re.compile('(?:.*,)*[ \t]*([^ \t]+)[ \t]+'
+                    'realm=(["\'])(.*?)\\2', re.I)
+
+    # XXX could pre-emptively send auth info already accepted (RFC 2617,
+    # end of section 2, and section 1.2 immediately after "credentials"
+    # production).
+
+    def __init__(self, password_mgr=None):
+        if password_mgr is None:
+            password_mgr = HTTPPasswordMgr()
+        self.passwd = password_mgr
+        self.add_password = self.passwd.add_password
+        self.retried = 0
+
+    def reset_retry_count(self):
+        self.retried = 0
+
+    def http_error_auth_reqed(self, authreq, host, req, headers):
+        # host may be an authority (without userinfo) or a URL with an
+        # authority
+        # XXX could be multiple headers
+        authreq = headers.get(authreq, None)
+
+        if self.retried > 5:
+            # retry sending the username:password 5 times before failing.
+            raise HTTPError(req.get_full_url(), 401, "basic auth failed",
+                            headers, None)
+        else:
+            self.retried += 1
+
+        if authreq:
+            mo = AbstractBasicAuthHandler.rx.search(authreq)
+            if mo:
+                scheme, quote, realm = mo.groups()
+                if scheme.lower() == 'basic':
+                    response = self.retry_http_basic_auth(host, req, realm)
+                    if response and response.code != 401:
+                        self.retried = 0
+                    return response
+
+    def retry_http_basic_auth(self, host, req, realm):
+        user, pw = self.passwd.find_user_password(realm, host)
+        if pw is not None:
+            raw = "%s:%s" % (user, pw)
+            auth = 'Basic %s' % base64.b64encode(raw).strip()
+            if req.headers.get(self.auth_header, None) == auth:
+                return None
+            req.add_unredirected_header(self.auth_header, auth)
+            return self.parent.open(req, timeout=req.timeout)
+        else:
+            return None
+
+
+class HTTPBasicAuthHandler(AbstractBasicAuthHandler, BaseHandler):
+
+    auth_header = 'Authorization'
+
+    def http_error_401(self, req, fp, code, msg, headers):
+        url = req.get_full_url()
+        response = self.http_error_auth_reqed('www-authenticate',
+                                              url, req, headers)
+        self.reset_retry_count()
+        return response
+
+
+class ProxyBasicAuthHandler(AbstractBasicAuthHandler, BaseHandler):
+
+    auth_header = 'Proxy-authorization'
+
+    def http_error_407(self, req, fp, code, msg, headers):
+        # http_error_auth_reqed requires that there is no userinfo component in
+        # authority.  Assume there isn't one, since urllib2 does not (and
+        # should not, RFC 3986 s. 3.2.1) support requests for URLs containing
+        # userinfo.
+        authority = req.get_host()
+        response = self.http_error_auth_reqed('proxy-authenticate',
+                                          authority, req, headers)
+        self.reset_retry_count()
+        return response
+
+
+def randombytes(n):
+    """Return n random bytes."""
+    # Use /dev/urandom if it is available.  Fall back to random module
+    # if not.  It might be worthwhile to extend this function to use
+    # other platform-specific mechanisms for getting random bytes.
+    if os.path.exists("/dev/urandom"):
+        f = open("/dev/urandom")
+        s = f.read(n)
+        f.close()
+        return s
+    else:
+        L = [chr(random.randrange(0, 256)) for i in range(n)]
+        return "".join(L)
+
+class AbstractDigestAuthHandler:
+    # Digest authentication is specified in RFC 2617.
+
+    # XXX The client does not inspect the Authentication-Info header
+    # in a successful response.
+
+    # XXX It should be possible to test this implementation against
+    # a mock server that just generates a static set of challenges.
+
+    # XXX qop="auth-int" supports is shaky
+
+    def __init__(self, passwd=None):
+        if passwd is None:
+            passwd = HTTPPasswordMgr()
+        self.passwd = passwd
+        self.add_password = self.passwd.add_password
+        self.retried = 0
+        self.nonce_count = 0
+        self.last_nonce = None
+
+    def reset_retry_count(self):
+        self.retried = 0
+
+    def http_error_auth_reqed(self, auth_header, host, req, headers):
+        authreq = headers.get(auth_header, None)
+        if self.retried > 5:
+            # Don't fail endlessly - if we failed once, we'll probably
+            # fail a second time. Hm. Unless the Password Manager is
+            # prompting for the information. Crap. This isn't great
+            # but it's better than the current 'repeat until recursion
+            # depth exceeded' approach <wink>
+            raise HTTPError(req.get_full_url(), 401, "digest auth failed",
+                            headers, None)
+        else:
+            self.retried += 1
+        if authreq:
+            scheme = authreq.split()[0]
+            if scheme.lower() == 'digest':
+                return self.retry_http_digest_auth(req, authreq)
+
+    def retry_http_digest_auth(self, req, auth):
+        token, challenge = auth.split(' ', 1)
+        chal = parse_keqv_list(parse_http_list(challenge))
+        auth = self.get_authorization(req, chal)
+        if auth:
+            auth_val = 'Digest %s' % auth
+            if req.headers.get(self.auth_header, None) == auth_val:
+                return None
+            req.add_unredirected_header(self.auth_header, auth_val)
+            resp = self.parent.open(req, timeout=req.timeout)
+            return resp
+
+    def get_cnonce(self, nonce):
+        # The cnonce-value is an opaque
+        # quoted string value provided by the client and used by both client
+        # and server to avoid chosen plaintext attacks, to provide mutual
+        # authentication, and to provide some message integrity protection.
+        # This isn't a fabulous effort, but it's probably Good Enough.
+        dig = hashlib.sha1("%s:%s:%s:%s" % (self.nonce_count, nonce, time.ctime(),
+                                            randombytes(8))).hexdigest()
+        return dig[:16]
+
+    def get_authorization(self, req, chal):
+        try:
+            realm = chal['realm']
+            nonce = chal['nonce']
+            qop = chal.get('qop')
+            algorithm = chal.get('algorithm', 'MD5')
+            # mod_digest doesn't send an opaque, even though it isn't
+            # supposed to be optional
+            opaque = chal.get('opaque', None)
+        except KeyError:
+            return None
+
+        H, KD = self.get_algorithm_impls(algorithm)
+        if H is None:
+            return None
+
+        user, pw = self.passwd.find_user_password(realm, req.get_full_url())
+        if user is None:
+            return None
+
+        # XXX not implemented yet
+        if req.has_data():
+            entdig = self.get_entity_digest(req.get_data(), chal)
+        else:
+            entdig = None
+
+        A1 = "%s:%s:%s" % (user, realm, pw)
+        A2 = "%s:%s" % (req.get_method(),
+                        # XXX selector: what about proxies and full urls
+                        req.get_selector())
+        if qop == 'auth':
+            if nonce == self.last_nonce:
+                self.nonce_count += 1
+            else:
+                self.nonce_count = 1
+                self.last_nonce = nonce
+
+            ncvalue = '%08x' % self.nonce_count
+            cnonce = self.get_cnonce(nonce)
+            noncebit = "%s:%s:%s:%s:%s" % (nonce, ncvalue, cnonce, qop, H(A2))
+            respdig = KD(H(A1), noncebit)
+        elif qop is None:
+            respdig = KD(H(A1), "%s:%s" % (nonce, H(A2)))
+        else:
+            # XXX handle auth-int.
+            raise URLError("qop '%s' is not supported." % qop)
+
+        # XXX should the partial digests be encoded too?
+
+        base = 'username="%s", realm="%s", nonce="%s", uri="%s", ' \
+               'response="%s"' % (user, realm, nonce, req.get_selector(),
+                                  respdig)
+        if opaque:
+            base += ', opaque="%s"' % opaque
+        if entdig:
+            base += ', digest="%s"' % entdig
+        base += ', algorithm="%s"' % algorithm
+        if qop:
+            base += ', qop=auth, nc=%s, cnonce="%s"' % (ncvalue, cnonce)
+        return base
+
+    def get_algorithm_impls(self, algorithm):
+        # algorithm should be case-insensitive according to RFC2617
+        algorithm = algorithm.upper()
+        # lambdas assume digest modules are imported at the top level
+        if algorithm == 'MD5':
+            H = lambda x: hashlib.md5(x).hexdigest()
+        elif algorithm == 'SHA':
+            H = lambda x: hashlib.sha1(x).hexdigest()
+        # XXX MD5-sess
+        KD = lambda s, d: H("%s:%s" % (s, d))
+        return H, KD
+
+    def get_entity_digest(self, data, chal):
+        # XXX not implemented yet
+        return None
+
+
+class HTTPDigestAuthHandler(BaseHandler, AbstractDigestAuthHandler):
+    """An authentication protocol defined by RFC 2069
+
+    Digest authentication improves on basic authentication because it
+    does not transmit passwords in the clear.
+    """
+
+    auth_header = 'Authorization'
+    handler_order = 490  # before Basic auth
+
+    def http_error_401(self, req, fp, code, msg, headers):
+        host = urlparse.urlparse(req.get_full_url())[1]
+        retry = self.http_error_auth_reqed('www-authenticate',
+                                           host, req, headers)
+        self.reset_retry_count()
+        return retry
+
+
+class ProxyDigestAuthHandler(BaseHandler, AbstractDigestAuthHandler):
+
+    auth_header = 'Proxy-Authorization'
+    handler_order = 490  # before Basic auth
+
+    def http_error_407(self, req, fp, code, msg, headers):
+        host = req.get_host()
+        retry = self.http_error_auth_reqed('proxy-authenticate',
+                                           host, req, headers)
+        self.reset_retry_count()
+        return retry
+
+class AbstractHTTPHandler(BaseHandler):
+
+    def __init__(self, debuglevel=0):
+        self._debuglevel = debuglevel
+
+    def set_http_debuglevel(self, level):
+        self._debuglevel = level
+
+    def do_request_(self, request):
+        host = request.get_host()
+        if not host:
+            raise URLError('no host given')
+
+        if request.has_data():  # POST
+            data = request.get_data()
+            if not request.has_header('Content-type'):
+                request.add_unredirected_header(
+                    'Content-type',
+                    'application/x-www-form-urlencoded')
+            if not request.has_header('Content-length'):
+                request.add_unredirected_header(
+                    'Content-length', '%d' % len(data))
+
+        sel_host = host
+        if request.has_proxy():
+            scheme, sel = splittype(request.get_selector())
+            sel_host, sel_path = splithost(sel)
+
+        if not request.has_header('Host'):
+            request.add_unredirected_header('Host', sel_host)
+        for name, value in self.parent.addheaders:
+            name = name.capitalize()
+            if not request.has_header(name):
+                request.add_unredirected_header(name, value)
+
+        return request
+
+    def do_open(self, http_class, req):
+        """Return an addinfourl object for the request, using http_class.
+
+        http_class must implement the HTTPConnection API from httplib.
+        The addinfourl return value is a file-like object.  It also
+        has methods and attributes including:
+            - info(): return a mimetools.Message object for the headers
+            - geturl(): return the original request URL
+            - code: HTTP status code
+        """
+        host = req.get_host()
+        if not host:
+            raise URLError('no host given')
+
+        h = http_class(host, timeout=req.timeout) # will parse host:port
+        h.set_debuglevel(self._debuglevel)
+
+        headers = dict(req.unredirected_hdrs)
+        headers.update(dict((k, v) for k, v in req.headers.items()
+                            if k not in headers))
+
+        # We want to make an HTTP/1.1 request, but the addinfourl
+        # class isn't prepared to deal with a persistent connection.
+        # It will try to read all remaining data from the socket,
+        # which will block while the server waits for the next request.
+        # So make sure the connection gets closed after the (only)
+        # request.
+        headers["Connection"] = "close"
+        headers = dict(
+            (name.title(), val) for name, val in headers.items())
+
+        if req._tunnel_host:
+            tunnel_headers = {}
+            proxy_auth_hdr = "Proxy-Authorization"
+            if proxy_auth_hdr in headers:
+                tunnel_headers[proxy_auth_hdr] = headers[proxy_auth_hdr]
+                # Proxy-Authorization should not be sent to origin
+                # server.
+                del headers[proxy_auth_hdr]
+            h.set_tunnel(req._tunnel_host, headers=tunnel_headers)
+
+        try:
+            h.request(req.get_method(), req.get_selector(), req.data, headers)
+            try:
+                r = h.getresponse(buffering=True)
+            except TypeError: #buffering kw not supported
+                r = h.getresponse()
+        except socket.error, err: # XXX what error?
+            h.close()
+            raise URLError(err)
+
+        # Pick apart the HTTPResponse object to get the addinfourl
+        # object initialized properly.
+
+        # Wrap the HTTPResponse object in socket's file object adapter
+        # for Windows.  That adapter calls recv(), so delegate recv()
+        # to read().  This weird wrapping allows the returned object to
+        # have readline() and readlines() methods.
+
+        # XXX It might be better to extract the read buffering code
+        # out of socket._fileobject() and into a base class.
+
+        r.recv = r.read
+        fp = socket._fileobject(r, close=True)
+
+        resp = addinfourl(fp, r.msg, req.get_full_url())
+        resp.code = r.status
+        resp.msg = r.reason
+        return resp
+
+
+class HTTPHandler(AbstractHTTPHandler):
+
+    def http_open(self, req):
+        return self.do_open(httplib.HTTPConnection, req)
+
+    http_request = AbstractHTTPHandler.do_request_
+
+if hasattr(httplib, 'HTTPS'):
+    class HTTPSHandler(AbstractHTTPHandler):
+
+        def https_open(self, req):
+            return self.do_open(httplib.HTTPSConnection, req)
+
+        https_request = AbstractHTTPHandler.do_request_
+
+class HTTPCookieProcessor(BaseHandler):
+    def __init__(self, cookiejar=None):
+        import cookielib
+        if cookiejar is None:
+            cookiejar = cookielib.CookieJar()
+        self.cookiejar = cookiejar
+
+    def http_request(self, request):
+        self.cookiejar.add_cookie_header(request)
+        return request
+
+    def http_response(self, request, response):
+        self.cookiejar.extract_cookies(response, request)
+        return response
+
+    https_request = http_request
+    https_response = http_response
+
+class UnknownHandler(BaseHandler):
+    def unknown_open(self, req):
+        type = req.get_type()
+        raise URLError('unknown url type: %s' % type)
+
+def parse_keqv_list(l):
+    """Parse list of key=value strings where keys are not duplicated."""
+    parsed = {}
+    for elt in l:
+        k, v = elt.split('=', 1)
+        if v[0] == '"' and v[-1] == '"':
+            v = v[1:-1]
+        parsed[k] = v
+    return parsed
+
+def parse_http_list(s):
+    """Parse lists as described by RFC 2068 Section 2.
+
+    In particular, parse comma-separated lists where the elements of
+    the list may include quoted-strings.  A quoted-string could
+    contain a comma.  A non-quoted string could have quotes in the
+    middle.  Neither commas nor quotes count if they are escaped.
+    Only double-quotes count, not single-quotes.
+    """
+    res = []
+    part = ''
+
+    escape = quote = False
+    for cur in s:
+        if escape:
+            part += cur
+            escape = False
+            continue
+        if quote:
+            if cur == '\\':
+                escape = True
+                continue
+            elif cur == '"':
+                quote = False
+            part += cur
+            continue
+
+        if cur == ',':
+            res.append(part)
+            part = ''
+            continue
+
+        if cur == '"':
+            quote = True
+
+        part += cur
+
+    # append last part
+    if part:
+        res.append(part)
+
+    return [part.strip() for part in res]
+
+def _safe_gethostbyname(host):
+    try:
+        return socket.gethostbyname(host)
+    except socket.gaierror:
+        return None
+
+class FileHandler(BaseHandler):
+    # Use local file or FTP depending on form of URL
+    def file_open(self, req):
+        url = req.get_selector()
+        if url[:2] == '//' and url[2:3] != '/' and (req.host and
+                req.host != 'localhost'):
+            req.type = 'ftp'
+            return self.parent.open(req)
+        else:
+            return self.open_local_file(req)
+
+    # names for the localhost
+    names = None
+    def get_names(self):
+        if FileHandler.names is None:
+            try:
+                FileHandler.names = tuple(
+                    socket.gethostbyname_ex('localhost')[2] +
+                    socket.gethostbyname_ex(socket.gethostname())[2])
+            except socket.gaierror:
+                FileHandler.names = (socket.gethostbyname('localhost'),)
+        return FileHandler.names
+
+    # not entirely sure what the rules are here
+    def open_local_file(self, req):
+        import email.utils
+        import mimetypes
+        host = req.get_host()
+        filename = req.get_selector()
+        localfile = url2pathname(filename)
+        try:
+            stats = os.stat(localfile)
+            size = stats.st_size
+            modified = email.utils.formatdate(stats.st_mtime, usegmt=True)
+            mtype = mimetypes.guess_type(filename)[0]
+            headers = mimetools.Message(StringIO(
+                'Content-type: %s\nContent-length: %d\nLast-modified: %s\n' %
+                (mtype or 'text/plain', size, modified)))
+            if host:
+                host, port = splitport(host)
+            if not host or \
+                (not port and _safe_gethostbyname(host) in self.get_names()):
+                if host:
+                    origurl = 'file://' + host + filename
+                else:
+                    origurl = 'file://' + filename
+                return addinfourl(open(localfile, 'rb'), headers, origurl)
+        except OSError, msg:
+            # urllib2 users shouldn't expect OSErrors coming from urlopen()
+            raise URLError(msg)
+        raise URLError('file not on local host')
+
+class FTPHandler(BaseHandler):
+    def ftp_open(self, req):
+        import ftplib
+        import mimetypes
+        host = req.get_host()
+        if not host:
+            raise URLError('ftp error: no host given')
+        host, port = splitport(host)
+        if port is None:
+            port = ftplib.FTP_PORT
+        else:
+            port = int(port)
+
+        # username/password handling
+        user, host = splituser(host)
+        if user:
+            user, passwd = splitpasswd(user)
+        else:
+            passwd = None
+        host = unquote(host)
+        user = user or ''
+        passwd = passwd or ''
+
+        try:
+            host = socket.gethostbyname(host)
+        except socket.error, msg:
+            raise URLError(msg)
+        path, attrs = splitattr(req.get_selector())
+        dirs = path.split('/')
+        dirs = map(unquote, dirs)
+        dirs, file = dirs[:-1], dirs[-1]
+        if dirs and not dirs[0]:
+            dirs = dirs[1:]
+        try:
+            fw = self.connect_ftp(user, passwd, host, port, dirs, req.timeout)
+            type = file and 'I' or 'D'
+            for attr in attrs:
+                attr, value = splitvalue(attr)
+                if attr.lower() == 'type' and \
+                   value in ('a', 'A', 'i', 'I', 'd', 'D'):
+                    type = value.upper()
+            fp, retrlen = fw.retrfile(file, type)
+            headers = ""
+            mtype = mimetypes.guess_type(req.get_full_url())[0]
+            if mtype:
+                headers += "Content-type: %s\n" % mtype
+            if retrlen is not None and retrlen >= 0:
+                headers += "Content-length: %d\n" % retrlen
+            sf = StringIO(headers)
+            headers = mimetools.Message(sf)
+            return addinfourl(fp, headers, req.get_full_url())
+        except ftplib.all_errors, msg:
+            raise URLError, ('ftp error: %s' % msg), sys.exc_info()[2]
+
+    def connect_ftp(self, user, passwd, host, port, dirs, timeout):
+        fw = ftpwrapper(user, passwd, host, port, dirs, timeout)
+##        fw.ftp.set_debuglevel(1)
+        return fw
+
+class CacheFTPHandler(FTPHandler):
+    # XXX would be nice to have pluggable cache strategies
+    # XXX this stuff is definitely not thread safe
+    def __init__(self):
+        self.cache = {}
+        self.timeout = {}
+        self.soonest = 0
+        self.delay = 60
+        self.max_conns = 16
+
+    def setTimeout(self, t):
+        self.delay = t
+
+    def setMaxConns(self, m):
+        self.max_conns = m
+
+    def connect_ftp(self, user, passwd, host, port, dirs, timeout):
+        key = user, host, port, '/'.join(dirs), timeout
+        if key in self.cache:
+            self.timeout[key] = time.time() + self.delay
+        else:
+            self.cache[key] = ftpwrapper(user, passwd, host, port, dirs, timeout)
+            self.timeout[key] = time.time() + self.delay
+        self.check_cache()
+        return self.cache[key]
+
+    def check_cache(self):
+        # first check for old ones
+        t = time.time()
+        if self.soonest <= t:
+            for k, v in self.timeout.items():
+                if v < t:
+                    self.cache[k].close()
+                    del self.cache[k]
+                    del self.timeout[k]
+        self.soonest = min(self.timeout.values())
+
+        # then check the size
+        if len(self.cache) == self.max_conns:
+            for k, v in self.timeout.items():
+                if v == self.soonest:
+                    del self.cache[k]
+                    del self.timeout[k]
+                    break
+            self.soonest = min(self.timeout.values())
diff --git a/lib_pypy/_ctypes/__init__.py b/lib_pypy/_ctypes/__init__.py
--- a/lib_pypy/_ctypes/__init__.py
+++ b/lib_pypy/_ctypes/__init__.py
@@ -18,7 +18,16 @@
 if _os.name in ("nt", "ce"):
     from _rawffi import FormatError
     from _rawffi import check_HRESULT as _check_HRESULT
-    CopyComPointer = None # XXX
+
+    def CopyComPointer(src, dst):
+        from ctypes import c_void_p, cast
+        if src:
+            hr = src[0][0].AddRef(src)
+            if hr & 0x80000000:
+                return hr
+        dst[0] = cast(src, c_void_p).value
+        return 0
+
     LoadLibrary = dlopen
 
 from _rawffi import FUNCFLAG_STDCALL, FUNCFLAG_CDECL, FUNCFLAG_PYTHONAPI
diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py
--- a/lib_pypy/_ctypes/array.py
+++ b/lib_pypy/_ctypes/array.py
@@ -208,6 +208,9 @@
     def _get_buffer_value(self):
         return self._buffer.buffer
 
+    def _to_ffi_param(self):
+        return self._get_buffer_value()
+
 ARRAY_CACHE = {}
 
 def create_array_type(base, length):
diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -1,5 +1,6 @@
 
 import _rawffi
+import _ffi
 import sys
 
 keepalive_key = str # XXX fix this when provided with test
@@ -46,6 +47,16 @@
         else:
             return self.from_param(as_parameter)
 
+    def get_ffi_param(self, value):
+        cdata = self.from_param(value)
+        return cdata, cdata._to_ffi_param()
+
+    def get_ffi_argtype(self):
+        if self._ffiargtype:
+            return self._ffiargtype
+        self._ffiargtype = _shape_to_ffi_type(self._ffiargshape)
+        return self._ffiargtype
+
     def _CData_output(self, resbuffer, base=None, index=-1):
         #assert isinstance(resbuffer, _rawffi.ArrayInstance)
         """Used when data exits ctypes and goes into user code.
@@ -99,6 +110,7 @@
     """
     __metaclass__ = _CDataMeta
     _objects = None
+    _ffiargtype = None
 
     def __init__(self, *args, **kwds):
         raise TypeError("%s has no type" % (type(self),))
@@ -119,11 +131,20 @@
     def _get_buffer_value(self):
         return self._buffer[0]
 
+    def _to_ffi_param(self):
+        if self.__class__._is_pointer_like():
+            return self._get_buffer_value()
+        else:
+            return self.value
+
     def __buffer__(self):
         return buffer(self._buffer)
 
     def _get_b_base(self):
-        return self._base
+        try:
+            return self._base
+        except AttributeError:
+            return None
     _b_base_ = property(_get_b_base)
     _b_needsfree_ = False
 
@@ -146,11 +167,12 @@
     return tp._alignmentofinstances()
 
 def byref(cdata):
-    from ctypes import pointer
+    # "pointer" is imported at the end of this module to avoid circular
+    # imports
     return pointer(cdata)
 
 def cdata_from_address(self, address):
-    # fix the address, in case it's unsigned
+    # fix the address: turn it into as unsigned, in case it's a negative number
     address = address & (sys.maxint * 2 + 1)
     instance = self.__new__(self)
     lgt = getattr(self, '_length_', 1)
@@ -159,3 +181,54 @@
 
 def addressof(tp):
     return tp._buffer.buffer
+
+
+# ----------------------------------------------------------------------
+
+def is_struct_shape(shape):
+    # see the corresponding code to set the shape in
+    # _ctypes.structure._set_shape
+    return (isinstance(shape, tuple) and
+            len(shape) == 2 and
+            isinstance(shape[0], _rawffi.Structure) and
+            shape[1] == 1)
+
+def _shape_to_ffi_type(shape):
+    try:
+        return _shape_to_ffi_type.typemap[shape]
+    except KeyError:
+        pass
+    if is_struct_shape(shape):
+        return shape[0].get_ffi_type()
+    #
+    assert False, 'unknown shape %s' % (shape,)
+
+
+_shape_to_ffi_type.typemap =  {
+    'c' : _ffi.types.char,
+    'b' : _ffi.types.sbyte,
+    'B' : _ffi.types.ubyte,
+    'h' : _ffi.types.sshort,
+    'u' : _ffi.types.unichar,
+    'H' : _ffi.types.ushort,
+    'i' : _ffi.types.sint,
+    'I' : _ffi.types.uint,
+    'l' : _ffi.types.slong,
+    'L' : _ffi.types.ulong,
+    'q' : _ffi.types.slonglong,
+    'Q' : _ffi.types.ulonglong,
+    'f' : _ffi.types.float,
+    'd' : _ffi.types.double,
+    's' : _ffi.types.void_p,
+    'P' : _ffi.types.void_p,
+    'z' : _ffi.types.void_p,
+    'O' : _ffi.types.void_p,
+    'Z' : _ffi.types.void_p,
+    'X' : _ffi.types.void_p,
+    'v' : _ffi.types.sshort,
+    '?' : _ffi.types.ubyte,
+    }
+
+
+# used by "byref"
+from _ctypes.pointer import pointer
diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py
--- a/lib_pypy/_ctypes/function.py
+++ b/lib_pypy/_ctypes/function.py
@@ -1,12 +1,15 @@
+
+from _ctypes.basics import _CData, _CDataMeta, cdata_from_address
+from _ctypes.primitive import SimpleType, _SimpleCData
+from _ctypes.basics import ArgumentError, keepalive_key
+from _ctypes.basics import is_struct_shape
+from _ctypes.builtin import set_errno, set_last_error
 import _rawffi
+import _ffi
 import sys
 import traceback
 import warnings
 
-from _ctypes.basics import ArgumentError, keepalive_key
-from _ctypes.basics import _CData, _CDataMeta, cdata_from_address
-from _ctypes.builtin import set_errno, set_last_error
-from _ctypes.primitive import SimpleType
 
 # XXX this file needs huge refactoring I fear
 
@@ -24,6 +27,7 @@
 
 WIN64 = sys.platform == 'win32' and sys.maxint == 2**63 - 1
 
+
 def get_com_error(errcode, riid, pIunk):
     "Win32 specific: build a COM Error exception"
     # XXX need C support code
@@ -36,6 +40,7 @@
     funcptr.restype = int
     return funcptr(*args)
 
+
 class CFuncPtrType(_CDataMeta):
     # XXX write down here defaults and such things
 
@@ -50,6 +55,7 @@
 
     from_address = cdata_from_address
 
+
 class CFuncPtr(_CData):
     __metaclass__ = CFuncPtrType
 
@@ -65,12 +71,12 @@
     callable = None
     _ptr = None
     _buffer = None
+    _address = None
     # win32 COM properties
     _paramflags = None
     _com_index = None
     _com_iid = None
-
-    __restype_set = False
+    _is_fastpath = False
 
     def _getargtypes(self):
         return self._argtypes_
@@ -85,9 +91,14 @@
                     raise TypeError(
                         "item %d in _argtypes_ has no from_param method" % (
                             i + 1,))
-            self._argtypes_ = argtypes
+            self._argtypes_ = list(argtypes)
+            self._check_argtypes_for_fastpath()
+    argtypes = property(_getargtypes, _setargtypes)
 
-    argtypes = property(_getargtypes, _setargtypes)
+    def _check_argtypes_for_fastpath(self):
+        if all([hasattr(argtype, '_ffiargshape') for argtype in self._argtypes_]):
+            fastpath_cls = make_fastpath_subclass(self.__class__)
+            fastpath_cls.enable_fastpath_maybe(self)
 
     def _getparamflags(self):
         return self._paramflags
@@ -133,11 +144,11 @@
 
     paramflags = property(_getparamflags, _setparamflags)
 
+
     def _getrestype(self):
         return self._restype_
 
     def _setrestype(self, restype):
-        self.__restype_set = True
         self._ptr = None
         if restype is int:
             from ctypes import c_int
@@ -146,27 +157,24 @@
                 callable(restype)):
             raise TypeError("restype must be a type, a callable, or None")
         self._restype_ = restype
-
+        
     def _delrestype(self):
         self._ptr = None
         del self._restype_
-
+        
     restype = property(_getrestype, _setrestype, _delrestype)
 
     def _geterrcheck(self):
         return getattr(self, '_errcheck_', None)
-
     def _seterrcheck(self, errcheck):
         if not callable(errcheck):
             raise TypeError("The errcheck attribute must be callable")
         self._errcheck_ = errcheck
-
     def _delerrcheck(self):
         try:
             del self._errcheck_
         except AttributeError:
             pass
-
     errcheck = property(_geterrcheck, _seterrcheck, _delerrcheck)
 
     def _ffishapes(self, args, restype):
@@ -181,6 +189,14 @@
             restype = 'O' # void
         return argtypes, restype
 
+    def _set_address(self, address):
+        if not self._buffer:
+            self._buffer = _rawffi.Array('P')(1)
+        self._buffer[0] = address
+
+    def _get_address(self):
+        return self._buffer[0]
+
     def __init__(self, *args):
         self.name = None
         self._objects = {keepalive_key(0):self}
@@ -188,7 +204,7 @@
 
         # Empty function object -- this is needed for casts
         if not args:
-            self._buffer = _rawffi.Array('P')(1)
+            self._set_address(0)
             return
 
         argsl = list(args)
@@ -196,20 +212,25 @@
 
         # Direct construction from raw address
         if isinstance(argument, (int, long)) and not argsl:
-            ffiargs, ffires = self._ffishapes(self._argtypes_, self._restype_)
-            self._ptr = _rawffi.FuncPtr(argument, ffiargs, ffires, self._flags_)
-            self._buffer = self._ptr.byptr()
+            self._set_address(argument)
+            restype = self._restype_
+            if restype is None:
+                import ctypes
+                restype = ctypes.c_int
+            self._ptr = self._getfuncptr_fromaddress(self._argtypes_, restype)
+            self._check_argtypes_for_fastpath()
             return
 
-        # A callback into Python
+        
+        # A callback into python
         if callable(argument) and not argsl:
             self.callable = argument
             ffiargs, ffires = self._ffishapes(self._argtypes_, self._restype_)
             if self._restype_ is None:
                 ffires = None
-            self._ptr = _rawffi.CallbackPtr(self._wrap_callable(
-                argument, self.argtypes
-                ), ffiargs, ffires, self._flags_)
+            self._ptr = _rawffi.CallbackPtr(self._wrap_callable(argument,
+                                                                self.argtypes),
+                                            ffiargs, ffires, self._flags_)
             self._buffer = self._ptr.byptr()
             return
 
@@ -218,7 +239,7 @@
             import ctypes
             self.name, dll = argument
             if isinstance(dll, str):
-                self.dll = ctypes.CDLL(dll)
+                self.dll = ctypes.CDLL(self.dll)
             else:
                 self.dll = dll
             if argsl:
@@ -227,7 +248,7 @@
                     raise TypeError("Unknown constructor %s" % (args,))
             # We need to check dll anyway
             ptr = self._getfuncptr([], ctypes.c_int)
-            self._buffer = ptr.byptr()
+            self._set_address(ptr.getaddr())
             return
 
         # A COM function call, by index
@@ -270,18 +291,17 @@
                     # than the length of the argtypes tuple.
                     args = args[:len(self._argtypes_)]
             else:
-                plural = len(argtypes) > 1 and "s" or ""
+                plural = len(self._argtypes_) > 1 and "s" or ""
                 raise TypeError(
                     "This function takes %d argument%s (%s given)"
-                    % (len(argtypes), plural, len(args)))
-
-            # check that arguments are convertible
-            ## XXX Not as long as ctypes.cast is a callback function with
-            ## py_object arguments...
-            ## self._convert_args(argtypes, args, {})
+                    % (len(self._argtypes_), plural, len(args)))
 
             try:
-                res = self.callable(*args)
+                newargs = self._convert_args_for_callback(argtypes, args)
+            except (UnicodeError, TypeError, ValueError), e:
+                raise ArgumentError(str(e))
+            try:
+                res = self.callable(*newargs)
             except:
                 exc_info = sys.exc_info()
                 traceback.print_tb(exc_info[2], file=sys.stderr)
@@ -295,10 +315,6 @@
             warnings.warn('C function without declared arguments called',
                           RuntimeWarning, stacklevel=2)
             argtypes = []
-            
-        if not self.__restype_set:
-            warnings.warn('C function without declared return type called',
-                          RuntimeWarning, stacklevel=2)
 
         if self._com_index:
             from ctypes import cast, c_void_p, POINTER
@@ -306,83 +322,78 @@
                 raise ValueError(
                     "native COM method call without 'this' parameter"
                     )
-            thisarg = cast(args[0], POINTER(POINTER(c_void_p))).contents
-            argtypes = [c_void_p] + list(argtypes)
-            args = list(args)
-            args[0] = args[0].value
+            thisarg = cast(args[0], POINTER(POINTER(c_void_p)))
+            keepalives, newargs, argtypes, outargs = self._convert_args(argtypes,
+                                                                        args[1:], kwargs)
+            newargs.insert(0, args[0].value)
+            argtypes.insert(0, c_void_p)
         else:
             thisarg = None
+            keepalives, newargs, argtypes, outargs = self._convert_args(argtypes,
+                                                                        args, kwargs)
 
-        args, outargs = self._convert_args(argtypes, args, kwargs)
-        argtypes = [type(arg) for arg in args]
+        funcptr = self._getfuncptr(argtypes, self._restype_, thisarg)
+        result = self._call_funcptr(funcptr, *newargs)
+        result = self._do_errcheck(result, args)
 
-        restype = self._restype_
-        funcptr = self._getfuncptr(argtypes, restype, thisarg)
+        if not outargs:
+            return result
+
+        simple_cdata = type(c_void_p()).__bases__[0]
+        outargs = [x.value if type(x).__bases__[0] is simple_cdata else x
+                   for x in outargs]
+
+        if len(outargs) == 1:
+            return outargs[0]
+        return tuple(outargs)
+
+    def _call_funcptr(self, funcptr, *newargs):
+
         if self._flags_ & _rawffi.FUNCFLAG_USE_ERRNO:
             set_errno(_rawffi.get_errno())
         if self._flags_ & _rawffi.FUNCFLAG_USE_LASTERROR:
             set_last_error(_rawffi.get_last_error())
         try:
-            resbuffer = funcptr(*[arg._get_buffer_for_param()._buffer
-                                  for arg in args])
+            result = funcptr(*newargs)
         finally:
             if self._flags_ & _rawffi.FUNCFLAG_USE_ERRNO:
                 set_errno(_rawffi.get_errno())
             if self._flags_ & _rawffi.FUNCFLAG_USE_LASTERROR:
                 set_last_error(_rawffi.get_last_error())
+        #
+        try:
+            return self._build_result(self._restype_, result, newargs)
+        finally:
+            funcptr.free_temp_buffers()
 
-        result = None
-        if self._com_index:
-            if resbuffer[0] & 0x80000000:
-                raise get_com_error(resbuffer[0],
-                                    self._com_iid, args[0])
-            else:
-                result = int(resbuffer[0])
-        elif restype is not None:
-            checker = getattr(self.restype, '_check_retval_', None)
-            if checker:
-                val = restype(resbuffer[0])
-                # the original ctypes seems to make the distinction between
-                # classes defining a new type, and their subclasses
-                if '_type_' in restype.__dict__:
-                    val = val.value
-                result = checker(val)
-            elif not isinstance(restype, _CDataMeta):
-                result = restype(resbuffer[0])
-            else:
-                result = restype._CData_retval(resbuffer)
-
+    def _do_errcheck(self, result, args):
         # The 'errcheck' protocol
         if self._errcheck_:
             v = self._errcheck_(result, self, args)
             # If the errcheck funtion failed, let it throw
-            # If the errcheck function returned callargs unchanged,
+            # If the errcheck function returned newargs unchanged,
             # continue normal processing.
             # If the errcheck function returned something else,
             # use that as result.
             if v is not args:
-                result = v
+                return v
+        return result
 
-        if not outargs:
-            return result
-
-        if len(outargs) == 1:
-            return outargs[0]
-
-        return tuple(outargs)
+    def _getfuncptr_fromaddress(self, argtypes, restype):
+        address = self._get_address()
+        ffiargs = [argtype.get_ffi_argtype() for argtype in argtypes]
+        ffires = restype.get_ffi_argtype()
+        return _ffi.FuncPtr.fromaddr(address, '', ffiargs, ffires)
 
     def _getfuncptr(self, argtypes, restype, thisarg=None):
-        if self._ptr is not None and argtypes is self._argtypes_:
+        if self._ptr is not None and (argtypes is self._argtypes_ or argtypes == self._argtypes_):
             return self._ptr
         if restype is None or not isinstance(restype, _CDataMeta):
             import ctypes
             restype = ctypes.c_int
-        argshapes = [arg._ffiargshape for arg in argtypes]
-        resshape = restype._ffiargshape
         if self._buffer is not None:
-            ptr = _rawffi.FuncPtr(self._buffer[0], argshapes, resshape,
-                                  self._flags_)
-            if argtypes is self._argtypes_:
+            ptr = self._getfuncptr_fromaddress(argtypes, restype)
+            if argtypes == self._argtypes_:
                 self._ptr = ptr
             return ptr
 
@@ -390,15 +401,21 @@
             # extract the address from the object's virtual table
             if not thisarg:
                 raise ValueError("COM method call without VTable")
-            ptr = thisarg[self._com_index - 0x1000]
-            return _rawffi.FuncPtr(ptr, argshapes, resshape, self._flags_)
-
+            ptr = thisarg[0][self._com_index - 0x1000]
+            ffiargs = [argtype.get_ffi_argtype() for argtype in argtypes]
+            ffires = restype.get_ffi_argtype()
+            return _ffi.FuncPtr.fromaddr(ptr, '', ffiargs, ffires)
+        
         cdll = self.dll._handle
         try:
-            return cdll.ptr(self.name, argshapes, resshape, self._flags_)
+            ffi_argtypes = [argtype.get_ffi_argtype() for argtype in argtypes]
+            ffi_restype = restype.get_ffi_argtype()
+            self._ptr = cdll.getfunc(self.name, ffi_argtypes, ffi_restype)
+            return self._ptr
         except AttributeError:
             if self._flags_ & _rawffi.FUNCFLAG_CDECL:
                 raise
+
             # Win64 has no stdcall calling conv, so it should also not have the
             # name mangling of it.
             if WIN64:
@@ -409,23 +426,32 @@
             for i in range(33):
                 mangled_name = "_%s@%d" % (self.name, i*4)
                 try:
-                    return cdll.ptr(mangled_name, argshapes, resshape,
-                                    self._flags_)
+                    return cdll.getfunc(mangled_name,
+                                        ffi_argtypes, ffi_restype,
+                                        # XXX self._flags_
+                                        )
                 except AttributeError:
                     pass
             raise
 
-    @staticmethod
-    def _conv_param(argtype, arg):
-        from ctypes import c_char_p, c_wchar_p, c_void_p, c_int
+    @classmethod
+    def _conv_param(cls, argtype, arg):
+        if isinstance(argtype, _CDataMeta):
+            cobj, ffiparam = argtype.get_ffi_param(arg)
+            return cobj, ffiparam, argtype
+        
         if argtype is not None:
             arg = argtype.from_param(arg)
         if hasattr(arg, '_as_parameter_'):
             arg = arg._as_parameter_
         if isinstance(arg, _CData):
-            # The usual case when argtype is defined
-            cobj = arg
-        elif isinstance(arg, str):
+            return arg, arg._to_ffi_param(), type(arg)
+        #
+        # non-usual case: we do the import here to save a lot of code in the
+        # jit trace of the normal case
+        from ctypes import c_char_p, c_wchar_p, c_void_p, c_int
+        #
+        if isinstance(arg, str):
             cobj = c_char_p(arg)
         elif isinstance(arg, unicode):
             cobj = c_wchar_p(arg)
@@ -435,18 +461,30 @@
             cobj = c_int(arg)
         else:
             raise TypeError("Don't know how to handle %s" % (arg,))
-        return cobj
+
+        return cobj, cobj._to_ffi_param(), type(cobj)
+
+    def _convert_args_for_callback(self, argtypes, args):
+        assert len(argtypes) == len(args)
+        newargs = []
+        for argtype, arg in zip(argtypes, args):
+            param = argtype.from_param(arg)
+            _type_ = getattr(argtype, '_type_', None)
+            if _type_ == 'P': # special-case for c_void_p
+                param = param._get_buffer_value()
+            elif self._is_primitive(argtype):
+                param = param.value
+            newargs.append(param)
+        return newargs
 
     def _convert_args(self, argtypes, args, kwargs, marker=object()):
-        callargs = []
+        newargs = []
         outargs = []
+        keepalives = []
+        newargtypes = []
         total = len(args)
         paramflags = self._paramflags
-
-        if self._com_index:
-            inargs_idx = 1
-        else:
-            inargs_idx = 0
+        inargs_idx = 0
 
         if not paramflags and total < len(argtypes):
             raise TypeError("not enough arguments")
@@ -470,8 +508,10 @@
                     val = defval
                     if val is marker:
                         val = 0
-                    wrapped = self._conv_param(argtype, val)
-                    callargs.append(wrapped)
+                    keepalive, newarg, newargtype = self._conv_param(argtype, val)
+                    keepalives.append(keepalive)
+                    newargs.append(newarg)
+                    newargtypes.append(newargtype)
                 elif flag in (0, PARAMFLAG_FIN):
                     if inargs_idx < total:
                         val = args[inargs_idx]
@@ -485,38 +525,110 @@
                         raise TypeError("required argument '%s' missing" % name)
                     else:
                         raise TypeError("not enough arguments")
-                    wrapped = self._conv_param(argtype, val)
-                    callargs.append(wrapped)
+                    keepalive, newarg, newargtype = self._conv_param(argtype, val)
+                    keepalives.append(keepalive)
+                    newargs.append(newarg)
+                    newargtypes.append(newargtype)
                 elif flag == PARAMFLAG_FOUT:
                     if defval is not marker:
                         outargs.append(defval)
-                        wrapped = self._conv_param(argtype, defval)
+                        keepalive, newarg, newargtype = self._conv_param(argtype, defval)
                     else:
                         import ctypes
                         val = argtype._type_()
                         outargs.append(val)
-                        wrapped = ctypes.byref(val)
-                    callargs.append(wrapped)
+                        keepalive = None
+                        newarg = ctypes.byref(val)
+                        newargtype = type(newarg)
+                    keepalives.append(keepalive)
+                    newargs.append(newarg)
+                    newargtypes.append(newargtype)
                 else:
                     raise ValueError("paramflag %d not yet implemented" % flag)
             else:
                 try:
-                    wrapped = self._conv_param(argtype, args[i])
+                    keepalive, newarg, newargtype = self._conv_param(argtype, args[i])
                 except (UnicodeError, TypeError, ValueError), e:
                     raise ArgumentError(str(e))
-                callargs.append(wrapped)
+                keepalives.append(keepalive)
+                newargs.append(newarg)
+                newargtypes.append(newargtype)
                 inargs_idx += 1
 
-        if len(callargs) < total:
-            extra = args[len(callargs):]
+        if len(newargs) < len(args):
+            extra = args[len(newargs):]
             for i, arg in enumerate(extra):
                 try:
-                    wrapped = self._conv_param(None, arg)
+                    keepalive, newarg, newargtype = self._conv_param(None, arg)
                 except (UnicodeError, TypeError, ValueError), e:
                     raise ArgumentError(str(e))
-                callargs.append(wrapped)
+                keepalives.append(keepalive)
+                newargs.append(newarg)
+                newargtypes.append(newargtype)
+        return keepalives, newargs, newargtypes, outargs
 
-        return callargs, outargs
+    @staticmethod
+    def _is_primitive(argtype):
+        return argtype.__bases__[0] is _SimpleCData
+    
+    def _wrap_result(self, restype, result):
+        """
+        Convert from low-level repr of the result to the high-level python
+        one.
+        """
+        # hack for performance: if restype is a "simple" primitive type, don't
+        # allocate the buffer because it's going to be thrown away immediately
+        if self._is_primitive(restype) and not restype._is_pointer_like():
+            return result
+        #
+        shape = restype._ffishape
+        if is_struct_shape(shape):
+            buf = result
+        else:
+            buf = _rawffi.Array(shape)(1, autofree=True)
+            buf[0] = result
+        retval = restype._CData_retval(buf)
+        return retval
+
+    def _build_result(self, restype, result, argsandobjs):
+        """Build the function result:
+           If there is no OUT parameter, return the actual function result
+           If there is one OUT parameter, return it
+           If there are many OUT parameters, return a tuple"""
+
+        # XXX: note for the future: the function used to take a "resbuffer",
+        # i.e. an array of ints. Now it takes a result, which is already a
+        # python object. All places that do "resbuffer[0]" should check that
+        # result is actually an int and just use it.
+        #
+        # Also, argsandobjs used to be "args" in __call__, now it's "newargs"
+        # (i.e., the already unwrapped objects). It's used only when we have a
+        # PARAMFLAG_FOUT and it's probably wrong, I'll fix it when I find a
+        # failing test
+
+        retval = None
+
+        if restype is not None:
+            checker = getattr(self.restype, '_check_retval_', None)
+            if checker:
+                val = restype(result)
+                # the original ctypes seems to make the distinction between
+                # classes defining a new type, and their subclasses
+                if '_type_' in restype.__dict__:
+                    val = val.value
+                # XXX Raise a COMError when restype is HRESULT and
+                # checker(val) fails.  How to check for restype == HRESULT?
+                if self._com_index:
+                    if result & 0x80000000:
+                        raise get_com_error(result, None, None)
+                else:
+                    retval = checker(val)
+            elif not isinstance(restype, _CDataMeta):
+                retval = restype(result)
+            else:
+                retval = self._wrap_result(restype, result)
+
+        return retval
 
     def __nonzero__(self):
         return self._com_index is not None or bool(self._buffer[0])
@@ -532,3 +644,61 @@
                 self._ptr.free()
                 self._ptr = None
             self._needs_free = False
+
+
+def make_fastpath_subclass(CFuncPtr):
+    if CFuncPtr._is_fastpath:
+        return CFuncPtr
+    #
+    try:
+        return make_fastpath_subclass.memo[CFuncPtr]
+    except KeyError:
+        pass
+
+    class CFuncPtrFast(CFuncPtr):
+
+        _is_fastpath = True
+        _slowpath_allowed = True # set to False by tests
+
+        @classmethod
+        def enable_fastpath_maybe(cls, obj):
+            if (obj.callable is None and
+                obj._com_index is None):
+                obj.__class__ = cls
+
+        def __rollback(self):
+            assert self._slowpath_allowed
+            self.__class__ = CFuncPtr
+
+        # disable the fast path if we reset argtypes
+        def _setargtypes(self, argtypes):
+            self.__rollback()
+            self._setargtypes(argtypes)
+        argtypes = property(CFuncPtr._getargtypes, _setargtypes)
+
+        def _setcallable(self, func):
+            self.__rollback()
+            self.callable = func
+        callable = property(lambda x: None, _setcallable)
+
+        def _setcom_index(self, idx):
+            self.__rollback()
+            self._com_index = idx
+        _com_index = property(lambda x: None, _setcom_index)
+
+        def __call__(self, *args):
+            thisarg = None
+            argtypes = self._argtypes_
+            restype = self._restype_
+            funcptr = self._getfuncptr(argtypes, restype, thisarg)
+            try:
+                result = self._call_funcptr(funcptr, *args)
+                result = self._do_errcheck(result, args)
+            except (TypeError, ArgumentError, UnicodeDecodeError):
+                assert self._slowpath_allowed
+                return CFuncPtr.__call__(self, *args)
+            return result
+
+    make_fastpath_subclass.memo[CFuncPtr] = CFuncPtrFast
+    return CFuncPtrFast
+make_fastpath_subclass.memo = {}
diff --git a/lib_pypy/_ctypes/pointer.py b/lib_pypy/_ctypes/pointer.py
--- a/lib_pypy/_ctypes/pointer.py
+++ b/lib_pypy/_ctypes/pointer.py
@@ -1,6 +1,7 @@
 
 import _rawffi
-from _ctypes.basics import _CData, _CDataMeta, cdata_from_address
+import _ffi
+from _ctypes.basics import _CData, _CDataMeta, cdata_from_address, ArgumentError
 from _ctypes.basics import keepalive_key, store_reference, ensure_objects
 from _ctypes.basics import sizeof, byref
 from _ctypes.array import Array, array_get_slice_params, array_slice_getitem,\
@@ -19,7 +20,7 @@
             length     = 1,
             _ffiargshape = 'P',
             _ffishape  = 'P',
-            _fficompositesize = None
+            _fficompositesize = None,
         )
         # XXX check if typedict['_type_'] is any sane
         # XXX remember about paramfunc
@@ -66,6 +67,7 @@
         self._ffiarray = ffiarray
         self.__init__ = __init__
         self._type_ = TP
+        self._ffiargtype = _ffi.types.Pointer(TP.get_ffi_argtype())
 
     from_address = cdata_from_address
 
@@ -114,6 +116,17 @@
 
     contents = property(getcontents, setcontents)
 
+    def _as_ffi_pointer_(self, ffitype):
+        return as_ffi_pointer(self, ffitype)
+
+def as_ffi_pointer(value, ffitype):
+    my_ffitype = type(value).get_ffi_argtype()
+    # for now, we always allow types.pointer, else a lot of tests
+    # break. We need to rethink how pointers are represented, though
+    if my_ffitype is not ffitype and ffitype is not _ffi.types.void_p:
+        raise ArgumentError, "expected %s instance, got %s" % (type(value), ffitype)
+    return value._get_buffer_value()
+
 def _cast_addr(obj, _, tp):
     if not (isinstance(tp, _CDataMeta) and tp._is_pointer_like()):
         raise TypeError("cast() argument 2 must be a pointer type, not %s"
diff --git a/lib_pypy/_ctypes/primitive.py b/lib_pypy/_ctypes/primitive.py
--- a/lib_pypy/_ctypes/primitive.py
+++ b/lib_pypy/_ctypes/primitive.py
@@ -1,3 +1,4 @@
+import _ffi
 import _rawffi
 import weakref
 import sys
@@ -8,7 +9,9 @@
      CArgObject
 from _ctypes.builtin import ConvMode
 from _ctypes.array import Array
-from _ctypes.pointer import _Pointer
+from _ctypes.pointer import _Pointer, as_ffi_pointer
+#from _ctypes.function import CFuncPtr # this import is moved at the bottom
+                                       # because else it's circular
 
 class NULL(object):
     pass
@@ -85,7 +88,7 @@
         return res
     if isinstance(value, Array):
         return value
-    if isinstance(value, _Pointer):
+    if isinstance(value, (_Pointer, CFuncPtr)):
         return cls.from_address(value._buffer.buffer)
     if isinstance(value, (int, long)):
         return cls(value)
@@ -140,6 +143,8 @@
                     value = 0
                 self._buffer[0] = value
             result.value = property(_getvalue, _setvalue)
+            result._ffiargtype = _ffi.types.Pointer(_ffi.types.char)
+
         elif tp == 'Z':
             # c_wchar_p
             def _getvalue(self):
@@ -162,6 +167,7 @@
                     value = 0
                 self._buffer[0] = value
             result.value = property(_getvalue, _setvalue)
+            result._ffiargtype = _ffi.types.Pointer(_ffi.types.unichar)
 
         elif tp == 'P':
             # c_void_p
@@ -212,10 +218,15 @@
             result.value = property(_getvalue, _setvalue)
 
         elif tp == 'X':
-            from ctypes import windll
-            SysAllocStringLen = windll.oleaut32.SysAllocStringLen
-            SysStringLen = windll.oleaut32.SysStringLen
-            SysFreeString = windll.oleaut32.SysFreeString
+            from ctypes import WinDLL
+            # Use WinDLL("oleaut32") instead of windll.oleaut32
+            # because the latter is a shared (cached) object; and
+            # other code may set their own restypes. We need out own
+            # restype here.
+            oleaut32 = WinDLL("oleaut32")
+            SysAllocStringLen = oleaut32.SysAllocStringLen
+            SysStringLen = oleaut32.SysStringLen
+            SysFreeString = oleaut32.SysFreeString
             def _getvalue(self):
                 addr = self._buffer[0]
                 if addr == 0:
@@ -248,6 +259,12 @@
                     self._buffer[0] = 0  # VARIANT_FALSE
             result.value = property(_getvalue, _setvalue)
 
+        # make pointer-types compatible with the _ffi fast path
+        if result._is_pointer_like():
+            def _as_ffi_pointer_(self, ffitype):
+                return as_ffi_pointer(self, ffitype)
+            result._as_ffi_pointer_ = _as_ffi_pointer_
+            
         return result
 
     from_address = cdata_from_address
@@ -323,3 +340,5 @@
 
     def __nonzero__(self):
         return self._buffer[0] not in (0, '\x00')
+
+from _ctypes.function import CFuncPtr
diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py
--- a/lib_pypy/_ctypes/structure.py
+++ b/lib_pypy/_ctypes/structure.py
@@ -14,6 +14,15 @@
             raise TypeError("Expected CData subclass, got %s" % (tp,))
         if isinstance(tp, StructOrUnionMeta):
             tp._make_final()
+        if len(f) == 3:
+            if (not hasattr(tp, '_type_')
+                or not isinstance(tp._type_, str)
+                or tp._type_ not in "iIhHbBlL"):
+                #XXX: are those all types?
+                #     we just dont get the type name
+                #     in the interp levle thrown TypeError
+                #     from rawffi if there are more
+                raise TypeError('bit fields not allowed for type ' + tp.__name__)
 
     all_fields = []
     for cls in reversed(inspect.getmro(superclass)):
@@ -34,34 +43,37 @@
     for i, field in enumerate(all_fields):
         name = field[0]
         value = field[1]
+        is_bitfield = (len(field) == 3)
         fields[name] = Field(name,
                              self._ffistruct.fieldoffset(name),
                              self._ffistruct.fieldsize(name),
-                             value, i)
+                             value, i, is_bitfield)
 
     if anonymous_fields:
         resnames = []
         for i, field in enumerate(all_fields):
             name = field[0]
             value = field[1]
+            is_bitfield = (len(field) == 3)
             startpos = self._ffistruct.fieldoffset(name)
             if name in anonymous_fields:
                 for subname in value._names:
                     resnames.append(subname)
-                    relpos = startpos + value._fieldtypes[subname].offset
-                    subvalue = value._fieldtypes[subname].ctype
+                    subfield = getattr(value, subname)
+                    relpos = startpos + subfield.offset
+                    subvalue = subfield.ctype
                     fields[subname] = Field(subname,
                                             relpos, subvalue._sizeofinstances(),
-                                            subvalue, i)
+                                            subvalue, i, is_bitfield)
             else:
                 resnames.append(name)
         names = resnames
     self._names = names
-    self._fieldtypes = fields
+    self.__dict__.update(fields)
 
 class Field(object):
-    def __init__(self, name, offset, size, ctype, num):
-        for k in ('name', 'offset', 'size', 'ctype', 'num'):
+    def __init__(self, name, offset, size, ctype, num, is_bitfield):
+        for k in ('name', 'offset', 'size', 'ctype', 'num', 'is_bitfield'):
             self.__dict__[k] = locals()[k]
 
     def __setattr__(self, name, value):
@@ -71,6 +83,35 @@
         return "<Field '%s' offset=%d size=%d>" % (self.name, self.offset,
                                                    self.size)
 
+    def __get__(self, obj, cls=None):
+        if obj is None:
+            return self
+        if self.is_bitfield:
+            # bitfield member, use direct access
+            return obj._buffer.__getattr__(self.name)
+        else:
+            fieldtype = self.ctype
+            offset = self.num
+            suba = obj._subarray(fieldtype, self.name)
+            return fieldtype._CData_output(suba, obj, offset)
+
+
+    def __set__(self, obj, value):
+        fieldtype = self.ctype
+        cobj = fieldtype.from_param(value)
+        if ensure_objects(cobj) is not None:
+            key = keepalive_key(self.num)
+            store_reference(obj, key, cobj._objects)
+        arg = cobj._get_buffer_value()
+        if fieldtype._fficompositesize is not None:
+            from ctypes import memmove
+            dest = obj._buffer.fieldaddress(self.name)
+            memmove(dest, arg, fieldtype._fficompositesize)
+        else:
+            obj._buffer.__setattr__(self.name, arg)
+
+
+
 # ________________________________________________________________
 
 def _set_shape(tp, rawfields, is_union=False):
@@ -79,17 +120,12 @@
     tp._ffiargshape = tp._ffishape = (tp._ffistruct, 1)
     tp._fficompositesize = tp._ffistruct.size
 
-def struct_getattr(self, name):
-    if name not in ('_fields_', '_fieldtypes'):
-        if hasattr(self, '_fieldtypes') and name in self._fieldtypes:
-            return self._fieldtypes[name]
-    return _CDataMeta.__getattribute__(self, name)
 
 def struct_setattr(self, name, value):
     if name == '_fields_':
         if self.__dict__.get('_fields_', None) is not None:
             raise AttributeError("_fields_ is final")
-        if self in [v for k, v in value]:
+        if self in [f[1] for f in value]:
             raise AttributeError("Structure or union cannot contain itself")
         names_and_fields(
             self,
@@ -127,14 +163,14 @@
         if '_fields_' not in self.__dict__:
             self._fields_ = []
             self._names = []
-            self._fieldtypes = {}
             _set_shape(self, [], self._is_union)
 
-    __getattr__ = struct_getattr
     __setattr__ = struct_setattr
 
     def from_address(self, address):
         instance = StructOrUnion.__new__(self)
+        if isinstance(address, _rawffi.StructureInstance):
+            address = address.buffer
         instance.__dict__['_buffer'] = self._ffistruct.fromaddress(address)
         return instance
 
@@ -200,46 +236,15 @@
         A = _rawffi.Array(fieldtype._ffishape)
         return A.fromaddress(address, 1)
 
-    def __setattr__(self, name, value):
-        try:
-            field = self._fieldtypes[name]
-        except KeyError:
-            return _CData.__setattr__(self, name, value)
-        fieldtype = field.ctype
-        cobj = fieldtype.from_param(value)
-        if ensure_objects(cobj) is not None:
-            key = keepalive_key(field.num)
-            store_reference(self, key, cobj._objects)
-        arg = cobj._get_buffer_value()
-        if fieldtype._fficompositesize is not None:
-            from ctypes import memmove
-            dest = self._buffer.fieldaddress(name)
-            memmove(dest, arg, fieldtype._fficompositesize)
-        else:
-            self._buffer.__setattr__(name, arg)
-
-    def __getattribute__(self, name):
-        if name == '_fieldtypes':
-            return _CData.__getattribute__(self, '_fieldtypes')
-        try:
-            field = self._fieldtypes[name]
-        except KeyError:
-            return _CData.__getattribute__(self, name)
-        if field.size >> 16:
-            # bitfield member, use direct access
-            return self._buffer.__getattr__(name)
-        else:
-            fieldtype = field.ctype
-            offset = field.num
-            suba = self._subarray(fieldtype, name)
-            return fieldtype._CData_output(suba, self, offset)
-
     def _get_buffer_for_param(self):
         return self
 
     def _get_buffer_value(self):
         return self._buffer.buffer
 
+    def _to_ffi_param(self):
+        return self._buffer
+
 
 class StructureMeta(StructOrUnionMeta):
     _is_union = False
diff --git a/lib_pypy/_elementtree.py b/lib_pypy/_elementtree.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_elementtree.py
@@ -0,0 +1,6 @@
+# Just use ElementTree.
+
+from xml.etree import ElementTree
+
+globals().update(ElementTree.__dict__)
+del __all__
diff --git a/lib_pypy/_functools.py b/lib_pypy/_functools.py
--- a/lib_pypy/_functools.py
+++ b/lib_pypy/_functools.py
@@ -14,10 +14,9 @@
             raise TypeError("the first argument must be callable")
         self.func = func
         self.args = args
-        self.keywords = keywords
+        self.keywords = keywords or None
 
     def __call__(self, *fargs, **fkeywords):
-        newkeywords = self.keywords.copy()
-        newkeywords.update(fkeywords)
-        return self.func(*(self.args + fargs), **newkeywords)
-
+        if self.keywords is not None:
+            fkeywords = dict(self.keywords, **fkeywords)
+        return self.func(*(self.args + fargs), **fkeywords)
diff --git a/lib_pypy/_pypy_interact.py b/lib_pypy/_pypy_interact.py
--- a/lib_pypy/_pypy_interact.py
+++ b/lib_pypy/_pypy_interact.py
@@ -56,6 +56,10 @@
                 prompt = getattr(sys, 'ps1', '>>> ')
             try:
                 line = raw_input(prompt)
+                # Can be None if sys.stdin was redefined
+                encoding = getattr(sys.stdin, 'encoding', None)
+                if encoding and not isinstance(line, unicode):
+                    line = line.decode(encoding)
             except EOFError:
                 console.write("\n")
                 break
diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py
--- a/lib_pypy/_sqlite3.py
+++ b/lib_pypy/_sqlite3.py
@@ -24,6 +24,7 @@
 from ctypes import c_void_p, c_int, c_double, c_int64, c_char_p, cdll
 from ctypes import POINTER, byref, string_at, CFUNCTYPE, cast
 from ctypes import sizeof, c_ssize_t
+from collections import OrderedDict
 import datetime
 import sys
 import time
@@ -274,8 +275,31 @@
 def unicode_text_factory(x):
     return unicode(x, 'utf-8')
 
+
+class StatementCache(object):
+    def __init__(self, connection, maxcount):
+        self.connection = connection
+        self.maxcount = maxcount
+        self.cache = OrderedDict()
+
+    def get(self, sql, cursor, row_factory):
+        try:
+            stat = self.cache[sql]
+        except KeyError:
+            stat = Statement(self.connection, sql)
+            self.cache[sql] = stat
+            if len(self.cache) > self.maxcount:
+                self.cache.popitem(0)
+        #
+        if stat.in_use:
+            stat = Statement(self.connection, sql)
+        stat.set_row_factory(row_factory)
+        return stat
+
+
 class Connection(object):
-    def __init__(self, database, isolation_level="", detect_types=0, timeout=None, cached_statements=None, factory=None):
+    def __init__(self, database, timeout=5.0, detect_types=0, isolation_level="",
+                 check_same_thread=True, factory=None, cached_statements=100):
         self.db = c_void_p()
         if sqlite.sqlite3_open(database, byref(self.db)) != SQLITE_OK:
             raise OperationalError("Could not open database")
@@ -290,6 +314,7 @@
         self.row_factory = None
         self._isolation_level = isolation_level
         self.detect_types = detect_types
+        self.statement_cache = StatementCache(self, cached_statements)
 
         self.cursors = []
 
@@ -308,7 +333,8 @@
         self._aggregates = {}
         self.aggregate_instances = {}
         self._collations = {}
-        self.thread_ident = thread_get_ident()
+        if check_same_thread:
+            self.thread_ident = thread_get_ident()
 
     def _get_exception(self, error_code = None):
         if error_code is None:
@@ -397,7 +423,7 @@
         cur = Cursor(self)
         if not isinstance(sql, (str, unicode)):
             raise Warning("SQL is of wrong type. Must be string or unicode.")
-        statement = Statement(cur, sql, self.row_factory)
+        statement = self.statement_cache.get(sql, cur, self.row_factory)
         return statement
 
     def _get_isolation_level(self):
@@ -679,6 +705,8 @@
         from sqlite3.dump import _iterdump
         return _iterdump(self)
 
+DML, DQL, DDL = range(3)
+
 class Cursor(object):
     def __init__(self, con):
         if not isinstance(con, Connection):
@@ -706,12 +734,12 @@
         if type(sql) is unicode:
             sql = sql.encode("utf-8")
         self._check_closed()
-        self.statement = Statement(self, sql, self.row_factory)
+        self.statement = self.connection.statement_cache.get(sql, self, self.row_factory)
 
         if self.connection._isolation_level is not None:
-            if self.statement.kind == "DDL":
+            if self.statement.kind == DDL:
                 self.connection.commit()
-            elif self.statement.kind == "DML":
+            elif self.statement.kind == DML:
                 self.connection._begin()
 
         self.statement.set_params(params)
@@ -722,19 +750,18 @@
             self.statement.reset()
             raise self.connection._get_exception(ret)
 
-        if self.statement.kind == "DQL":
-            if ret == SQLITE_ROW:
-                self.statement._build_row_cast_map()
-                self.statement._readahead()
-            else:
-                self.statement.item = None
-                self.statement.exhausted = True
+        if self.statement.kind == DQL and ret == SQLITE_ROW:
+            self.statement._build_row_cast_map()
+            self.statement._readahead(self)
+        else:
+            self.statement.item = None
+            self.statement.exhausted = True
 
-        if self.statement.kind in ("DML", "DDL"):
+        if self.statement.kind == DML or self.statement.kind == DDL:
             self.statement.reset()
 
         self.rowcount = -1
-        if self.statement.kind == "DML":
+        if self.statement.kind == DML:
             self.rowcount = sqlite.sqlite3_changes(self.connection.db)
 
         return self
@@ -745,8 +772,9 @@
         if type(sql) is unicode:
             sql = sql.encode("utf-8")
         self._check_closed()
-        self.statement = Statement(self, sql, self.row_factory)
-        if self.statement.kind == "DML":
+        self.statement = self.connection.statement_cache.get(sql, self, self.row_factory)
+
+        if self.statement.kind == DML:
             self.connection._begin()
         else:
             raise ProgrammingError, "executemany is only for DML statements"
@@ -798,7 +826,7 @@
         return self
 
     def __iter__(self):
-        return self.statement
+        return iter(self.fetchone, None)
 
     def _check_reset(self):
         if self.reset:
@@ -815,7 +843,7 @@
             return None
 
         try:
-            return self.statement.next()
+            return self.statement.next(self)
         except StopIteration:
             return None
 
@@ -829,7 +857,7 @@
         if size is None:
             size = self.arraysize
         lst = []
-        for row in self.statement:
+        for row in self:
             lst.append(row)
             if len(lst) == size:
                 break
@@ -840,7 +868,7 @@
         self._check_reset()
         if self.statement is None:
             return []
-        return list(self.statement)
+        return list(self)
 
     def _getdescription(self):
         if self._description is None:
@@ -870,39 +898,47 @@
     lastrowid = property(_getlastrowid)
 
 class Statement(object):
-    def __init__(self, cur, sql, row_factory):
+    def __init__(self, connection, sql):
         self.statement = None
         if not isinstance(sql, str):
             raise ValueError, "sql must be a string"
-        self.con = cur.connection
-        self.cur = weakref.ref(cur)
+        self.con = connection
         self.sql = sql # DEBUG ONLY
-        self.row_factory = row_factory
         first_word = self._statement_kind = sql.lstrip().split(" ")[0].upper()
         if first_word in ("INSERT", "UPDATE", "DELETE", "REPLACE"):
-            self.kind = "DML"
+            self.kind = DML
         elif first_word in ("SELECT", "PRAGMA"):
-            self.kind = "DQL"
+            self.kind = DQL
         else:
-            self.kind = "DDL"
+            self.kind = DDL
         self.exhausted = False
+        self.in_use = False
+        #
+        # set by set_row_factory
+        self.row_factory = None
 
         self.statement = c_void_p()
         next_char = c_char_p()
-        ret = sqlite.sqlite3_prepare_v2(self.con.db, sql, -1, byref(self.statement), byref(next_char))
+        sql_char = c_char_p(sql)
+        ret = sqlite.sqlite3_prepare_v2(self.con.db, sql_char, -1, byref(self.statement), byref(next_char))
         if ret == SQLITE_OK and self.statement.value is None:
             # an empty statement, we work around that, as it's the least trouble
             ret = sqlite.sqlite3_prepare_v2(self.con.db, "select 42", -1, byref(self.statement), byref(next_char))
-            self.kind = "DQL"
+            self.kind = DQL
 
         if ret != SQLITE_OK:
             raise self.con._get_exception(ret)
         self.con._remember_statement(self)
         if _check_remaining_sql(next_char.value):
-            raise Warning, "One and only one statement required"
+            raise Warning, "One and only one statement required: %r" % (
+                next_char.value,)
+        # sql_char should remain alive until here
 
         self._build_row_cast_map()
 
+    def set_row_factory(self, row_factory):
+        self.row_factory = row_factory
+
     def _build_row_cast_map(self):
         self.row_cast_map = []
         for i in xrange(sqlite.sqlite3_column_count(self.statement)):
@@ -972,6 +1008,7 @@
         ret = sqlite.sqlite3_reset(self.statement)
         if ret != SQLITE_OK:
             raise self.con._get_exception(ret)
+        self.mark_dirty()
 
         if params is None:
             if sqlite.sqlite3_bind_parameter_count(self.statement) != 0:
@@ -1002,10 +1039,7 @@
                     raise ProgrammingError("missing parameter '%s'" %param)
                 self.set_param(idx, param)
 
-    def __iter__(self):
-        return self
-
-    def next(self):
+    def next(self, cursor):
         self.con._check_closed()
         self.con._check_thread()
         if self.exhausted:
@@ -1021,10 +1055,10 @@
             sqlite.sqlite3_reset(self.statement)
             raise exc
 
-        self._readahead()
+        self._readahead(cursor)
         return item
 
-    def _readahead(self):
+    def _readahead(self, cursor):
         self.column_count = sqlite.sqlite3_column_count(self.statement)
         row = []
         for i in xrange(self.column_count):
@@ -1059,23 +1093,30 @@
 
         row = tuple(row)
         if self.row_factory is not None:
-            row = self.row_factory(self.cur(), row)
+            row = self.row_factory(cursor, row)
         self.item = row
 
     def reset(self):
         self.row_cast_map = None
-        return sqlite.sqlite3_reset(self.statement)
+        ret = sqlite.sqlite3_reset(self.statement)
+        self.in_use = False
+        self.exhausted = False
+        return ret
 
     def finalize(self):
         sqlite.sqlite3_finalize(self.statement)
         self.statement = None
+        self.in_use = False
+
+    def mark_dirty(self):
+        self.in_use = True
 
     def __del__(self):
         sqlite.sqlite3_finalize(self.statement)
         self.statement = None
 
     def _get_description(self):
-        if self.kind == "DML":
+        if self.kind == DML:
             return None
         desc = []
         for i in xrange(sqlite.sqlite3_column_count(self.statement)):
diff --git a/lib_pypy/_subprocess.py b/lib_pypy/_subprocess.py
--- a/lib_pypy/_subprocess.py
+++ b/lib_pypy/_subprocess.py
@@ -35,7 +35,7 @@
 _DuplicateHandle.restype = ctypes.c_int
     
 _WaitForSingleObject = _kernel32.WaitForSingleObject
-_WaitForSingleObject.argtypes = [ctypes.c_int, ctypes.c_int]
+_WaitForSingleObject.argtypes = [ctypes.c_int, ctypes.c_uint]
 _WaitForSingleObject.restype = ctypes.c_int
 
 _GetExitCodeProcess = _kernel32.GetExitCodeProcess
diff --git a/lib_pypy/binascii.py b/lib_pypy/binascii.py
--- a/lib_pypy/binascii.py
+++ b/lib_pypy/binascii.py
@@ -659,7 +659,7 @@
         crc = crc_32_tab[(crc ^ long(ord(c))) & 0xffL] ^ (crc >> 8)
         #/* Note:  (crc >> 8) MUST zero fill on left
 
-        result = crc ^ 0xffffffffL
+    result = crc ^ 0xffffffffL
     
     if result > 2**31:
         result = ((result + 2**31) % 2**32) - 2**31
diff --git a/lib_pypy/cPickle.py b/lib_pypy/cPickle.py
--- a/lib_pypy/cPickle.py
+++ b/lib_pypy/cPickle.py
@@ -27,9 +27,9 @@
             PythonPickler.__init__(self, self.__f, args[0], **kw)
         else:
             PythonPickler.__init__(self, *args, **kw)
-            
+
     def memoize(self, obj):
-        self.memo[None] = None   # cPickle starts counting at one
+        self.memo[id(None)] = None   # cPickle starts counting at one
         return PythonPickler.memoize(self, obj)
 
     def getvalue(self):
diff --git a/lib_pypy/ctypes_support.py b/lib_pypy/ctypes_support.py
--- a/lib_pypy/ctypes_support.py
+++ b/lib_pypy/ctypes_support.py
@@ -10,8 +10,8 @@
 # __________ the standard C library __________
 
 if sys.platform == 'win32':
-    import _rawffi
-    standard_c_lib = ctypes.CDLL('msvcrt', handle=_rawffi.get_libc())
+    import _ffi
+    standard_c_lib = ctypes.CDLL('msvcrt', handle=_ffi.get_libc())
 else:
     standard_c_lib = ctypes.CDLL(ctypes.util.find_library('c'))
 
diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py
--- a/lib_pypy/datetime.py
+++ b/lib_pypy/datetime.py
@@ -1422,12 +1422,17 @@
             converter = _time.localtime
         else:
             converter = _time.gmtime
-        if 1 - (t % 1.0) < 0.000001:
-            t = float(int(t)) + 1
-        if t < 0:
-            t -= 1
+        if t < 0.0:
+            us = int(round(((-t) % 1.0) * 1000000))
+            if us > 0:
+                us = 1000000 - us
+                t -= 1.0
+        else:
+            us = int(round((t % 1.0) * 1000000))
+            if us == 1000000:
+                us = 0
+                t += 1.0
         y, m, d, hh, mm, ss, weekday, jday, dst = converter(t)
-        us = int((t % 1.0) * 1000000)
         ss = min(ss, 59)    # clamp out leap seconds if the platform has them
         result = cls(y, m, d, hh, mm, ss, us, tz)
         if tz is not None:
diff --git a/lib_pypy/distributed/test/test_distributed.py b/lib_pypy/distributed/test/test_distributed.py
--- a/lib_pypy/distributed/test/test_distributed.py
+++ b/lib_pypy/distributed/test/test_distributed.py
@@ -9,7 +9,7 @@
 class AppTestDistributed(object):
     def setup_class(cls):
         cls.space = gettestobjspace(**{"objspace.std.withtproxy": True,
-            "usemodules":("_stackless",)})
+            "usemodules":("_continuation",)})
 
     def test_init(self):
         import distributed
@@ -91,10 +91,8 @@
 
 class AppTestDistributedTasklets(object):
     spaceconfig = {"objspace.std.withtproxy": True,
-                   "objspace.usemodules._stackless": True}
+                   "objspace.usemodules._continuation": True}
     def setup_class(cls):
-        #cls.space = gettestobjspace(**{"objspace.std.withtproxy": True,
-        #    "usemodules":("_stackless",)})
         cls.w_test_env = cls.space.appexec([], """():
         from distributed import test_env
         return test_env
diff --git a/lib_pypy/distributed/test/test_greensock.py b/lib_pypy/distributed/test/test_greensock.py
--- a/lib_pypy/distributed/test/test_greensock.py
+++ b/lib_pypy/distributed/test/test_greensock.py
@@ -10,7 +10,7 @@
         if not option.runappdirect:
             py.test.skip("Cannot run this on top of py.py because of PopenGateway")
         cls.space = gettestobjspace(**{"objspace.std.withtproxy": True,
-                                       "usemodules":("_stackless",)})
+                                       "usemodules":("_continuation",)})
         cls.w_remote_side_code = cls.space.appexec([], """():
         import sys
         sys.path.insert(0, '%s')
diff --git a/lib_pypy/distributed/test/test_socklayer.py b/lib_pypy/distributed/test/test_socklayer.py
--- a/lib_pypy/distributed/test/test_socklayer.py
+++ b/lib_pypy/distributed/test/test_socklayer.py
@@ -9,7 +9,8 @@
 class AppTestSocklayer:
     def setup_class(cls):
         cls.space = gettestobjspace(**{"objspace.std.withtproxy": True,
-                                       "usemodules":("_stackless","_socket", "select")})
+                                       "usemodules":("_continuation",
+                                                     "_socket", "select")})
     
     def test_socklayer(self):
         class X(object):
diff --git a/lib_pypy/greenlet.py b/lib_pypy/greenlet.py
--- a/lib_pypy/greenlet.py
+++ b/lib_pypy/greenlet.py
@@ -1,1 +1,144 @@
-from _stackless import greenlet
+import _continuation, sys
+
+
+# ____________________________________________________________
+# Exceptions
+
+class GreenletExit(Exception):
+    """This special exception does not propagate to the parent greenlet; it
+can be used to kill a single greenlet."""
+
+error = _continuation.error
+
+# ____________________________________________________________
+# Helper function
+
+def getcurrent():
+    "Returns the current greenlet (i.e. the one which called this function)."
+    try:
+        return _tls.current
+    except AttributeError:
+        # first call in this thread: current == main
+        _green_create_main()
+        return _tls.current
+
+# ____________________________________________________________
+# The 'greenlet' class
+
+_continulet = _continuation.continulet
+
+class greenlet(_continulet):
+    getcurrent = staticmethod(getcurrent)
+    error = error
+    GreenletExit = GreenletExit
+    __main = False
+    __started = False
+
+    def __new__(cls, *args, **kwds):
+        self = _continulet.__new__(cls)
+        self.parent = getcurrent()
+        return self
+
+    def __init__(self, run=None, parent=None):
+        if run is not None:
+            self.run = run
+        if parent is not None:
+            self.parent = parent
+
+    def switch(self, *args):
+        "Switch execution to this greenlet, optionally passing the values "
+        "given as argument(s).  Returns the value passed when switching back."
+        return self.__switch('switch', args)
+
+    def throw(self, typ=GreenletExit, val=None, tb=None):
+        "raise exception in greenlet, return value passed when switching back"
+        return self.__switch('throw', typ, val, tb)
+
+    def __switch(target, methodname, *args):
+        current = getcurrent()
+        #
+        while not target:
+            if not target.__started:
+                if methodname == 'switch':
+                    greenlet_func = _greenlet_start
+                else:
+                    greenlet_func = _greenlet_throw
+                _continulet.__init__(target, greenlet_func, *args)
+                methodname = 'switch'
+                args = ()
+                target.__started = True
+                break
+            # already done, go to the parent instead
+            # (NB. infinite loop possible, but unlikely, unless you mess
+            # up the 'parent' explicitly.  Good enough, because a Ctrl-C
+            # will show that the program is caught in this loop here.)
+            target = target.parent
+        #
+        try:
+            unbound_method = getattr(_continulet, methodname)
+            args = unbound_method(current, *args, to=target)
+        except GreenletExit, e:
+            args = (e,)
+        finally:
+            _tls.current = current
+        #
+        if len(args) == 1:
+            return args[0]
+        else:
+            return args
+
+    def __nonzero__(self):
+        return self.__main or _continulet.is_pending(self)
+
+    @property
+    def dead(self):
+        return self.__started and not self
+
+    @property
+    def gr_frame(self):
+        # xxx this doesn't work when called on either the current or
+        # the main greenlet of another thread
+        if self is getcurrent():
+            return None
+        if self.__main:
+            self = getcurrent()
+        f = _continulet.__reduce__(self)[2][0]
+        if not f:
+            return None
+        return f.f_back.f_back.f_back   # go past start(), __switch(), switch()
+
+# ____________________________________________________________
+# Internal stuff
+
+try:
+    from thread import _local
+except ImportError:
+    class _local(object):    # assume no threads
+        pass
+
+_tls = _local()
+
+def _green_create_main():
+    # create the main greenlet for this thread
+    _tls.current = None
+    gmain = greenlet.__new__(greenlet)
+    gmain._greenlet__main = True
+    gmain._greenlet__started = True
+    assert gmain.parent is None
+    _tls.main = gmain
+    _tls.current = gmain
+
+def _greenlet_start(greenlet, args):
+    _tls.current = greenlet
+    try:
+        res = greenlet.run(*args)
+    finally:
+        _continuation.permute(greenlet, greenlet.parent)
+    return (res,)
+
+def _greenlet_throw(greenlet, exc, value, tb):
+    _tls.current = greenlet
+    try:
+        raise exc, value, tb
+    finally:
+        _continuation.permute(greenlet, greenlet.parent)
diff --git a/lib_pypy/msvcrt.py b/lib_pypy/msvcrt.py
--- a/lib_pypy/msvcrt.py
+++ b/lib_pypy/msvcrt.py
@@ -46,4 +46,42 @@
         e = get_errno()
         raise IOError(e, errno.errorcode[e])
 
+# Console I/O routines
+
+kbhit = _c._kbhit
+kbhit.argtypes = []
+kbhit.restype = ctypes.c_int
+
+getch = _c._getch
+getch.argtypes = []
+getch.restype = ctypes.c_char
+
+getwch = _c._getwch
+getwch.argtypes = []
+getwch.restype = ctypes.c_wchar
+
+getche = _c._getche
+getche.argtypes = []
+getche.restype = ctypes.c_char
+
+getwche = _c._getwche
+getwche.argtypes = []
+getwche.restype = ctypes.c_wchar
+
+putch = _c._putch
+putch.argtypes = [ctypes.c_char]
+putch.restype = None
+
+putwch = _c._putwch
+putwch.argtypes = [ctypes.c_wchar]
+putwch.restype = None
+
+ungetch = _c._ungetch
+ungetch.argtypes = [ctypes.c_char]
+ungetch.restype = None
+
+ungetwch = _c._ungetwch
+ungetwch.argtypes = [ctypes.c_wchar]
+ungetwch.restype = None
+
 del ctypes
diff --git a/lib_pypy/pwd.py b/lib_pypy/pwd.py
--- a/lib_pypy/pwd.py
+++ b/lib_pypy/pwd.py
@@ -16,6 +16,7 @@
 
 from ctypes_support import standard_c_lib as libc
 from ctypes import Structure, POINTER, c_int, c_char_p, c_long
+from _structseq import structseqtype, structseqfield
 
 try: from __pypy__ import builtinify
 except ImportError: builtinify = lambda f: f
@@ -68,7 +69,7 @@
             yield self.pw_dir
             yield self.pw_shell
 
-class struct_passwd(tuple):
+class struct_passwd:
     """
     pwd.struct_passwd: Results from getpw*() routines.
 
@@ -76,15 +77,15 @@
       (pw_name,pw_passwd,pw_uid,pw_gid,pw_gecos,pw_dir,pw_shell)
     or via the object attributes as named in the above tuple.
     """
-    def __init__(self, passwd):
-        self.pw_name = passwd.pw_name
-        self.pw_passwd = passwd.pw_passwd
-        self.pw_uid = passwd.pw_uid
-        self.pw_gid = passwd.pw_gid
-        self.pw_gecos = passwd.pw_gecos
-        self.pw_dir = passwd.pw_dir
-        self.pw_shell = passwd.pw_shell
-        tuple.__init__(self, passwd)
+    __metaclass__ = structseqtype
+    name = "pwd.struct_passwd"
+    pw_name = structseqfield(0)
+    pw_passwd = structseqfield(1)
+    pw_uid = structseqfield(2)
+    pw_gid = structseqfield(3)
+    pw_gecos = structseqfield(4)
+    pw_dir = structseqfield(5)
+    pw_shell = structseqfield(6)
 
 passwd_p = POINTER(passwd)
 
diff --git a/lib_pypy/pypy_test/test_coroutine.py b/lib_pypy/pypy_test/test_coroutine.py
--- a/lib_pypy/pypy_test/test_coroutine.py
+++ b/lib_pypy/pypy_test/test_coroutine.py
@@ -2,7 +2,7 @@
 from py.test import skip, raises
 
 try:
-    from lib_pypy.stackless import coroutine, CoroutineExit
+    from stackless import coroutine, CoroutineExit
 except ImportError, e:
     skip('cannot import stackless: %s' % (e,))
 
@@ -20,10 +20,6 @@
         assert not co.is_zombie
 
     def test_is_zombie_del_without_frame(self):
-        try:
-            import _stackless # are we on pypy with a stackless build?
-        except ImportError:
-            skip("only works on pypy-c-stackless")
         import gc
         res = []
         class MyCoroutine(coroutine):
@@ -45,10 +41,6 @@
         assert res[0], "is_zombie was False in __del__"
 
     def test_is_zombie_del_with_frame(self):
-        try:
-            import _stackless # are we on pypy with a stackless build?
-        except ImportError:
-            skip("only works on pypy-c-stackless")
         import gc
         res = []
         class MyCoroutine(coroutine):
diff --git a/lib_pypy/pypy_test/test_datetime.py b/lib_pypy/pypy_test/test_datetime.py
--- a/lib_pypy/pypy_test/test_datetime.py
+++ b/lib_pypy/pypy_test/test_datetime.py
@@ -32,4 +32,28 @@
     assert datetime.datetime.utcfromtimestamp(a).microsecond == 0
     assert datetime.datetime.utcfromtimestamp(a).second == 1
 
-    
+def test_more_datetime_rounding():
+    # this test verified on top of CPython 2.7 (using a plain
+    # "import datetime" above)
+    expected_results = {
+        -1000.0: 'datetime.datetime(1970, 1, 1, 0, 43, 20)',
+        -999.9999996: 'datetime.datetime(1970, 1, 1, 0, 43, 20)',
+        -999.4: 'datetime.datetime(1970, 1, 1, 0, 43, 20, 600000)',
+        -999.0000004: 'datetime.datetime(1970, 1, 1, 0, 43, 21)',
+        -1.0: 'datetime.datetime(1970, 1, 1, 0, 59, 59)',
+        -0.9999996: 'datetime.datetime(1970, 1, 1, 0, 59, 59)',
+        -0.4: 'datetime.datetime(1970, 1, 1, 0, 59, 59, 600000)',
+        -0.0000004: 'datetime.datetime(1970, 1, 1, 1, 0)',
+        0.0: 'datetime.datetime(1970, 1, 1, 1, 0)',
+        0.0000004: 'datetime.datetime(1970, 1, 1, 1, 0)',
+        0.4: 'datetime.datetime(1970, 1, 1, 1, 0, 0, 400000)',
+        0.9999996: 'datetime.datetime(1970, 1, 1, 1, 0, 1)',
+        1000.0: 'datetime.datetime(1970, 1, 1, 1, 16, 40)',
+        1000.0000004: 'datetime.datetime(1970, 1, 1, 1, 16, 40)',
+        1000.4: 'datetime.datetime(1970, 1, 1, 1, 16, 40, 400000)',
+        1000.9999996: 'datetime.datetime(1970, 1, 1, 1, 16, 41)',
+        1293843661.191: 'datetime.datetime(2011, 1, 1, 2, 1, 1, 191000)',
+        }
+    for t in sorted(expected_results):
+        dt = datetime.datetime.fromtimestamp(t)
+        assert repr(dt) == expected_results[t]
diff --git a/lib_pypy/pypy_test/test_stackless_pickling.py b/lib_pypy/pypy_test/test_stackless_pickling.py
--- a/lib_pypy/pypy_test/test_stackless_pickling.py
+++ b/lib_pypy/pypy_test/test_stackless_pickling.py
@@ -1,7 +1,3 @@
-"""
-this test should probably not run from CPython or py.py.
-I'm not entirely sure, how to do that.
-"""
 from __future__ import absolute_import
 from py.test import skip
 try:
@@ -16,11 +12,15 @@
 
 class Test_StacklessPickling:
 
+    def test_pickle_main_coroutine(self):
+        import stackless, pickle
+        s = pickle.dumps(stackless.coroutine.getcurrent())
+        print s
+        c = pickle.loads(s)
+        assert c is stackless.coroutine.getcurrent()
+
     def test_basic_tasklet_pickling(self):
-        try:
-            import stackless
-        except ImportError:
-            skip("can't load stackless and don't know why!!!")
+        import stackless
         from stackless import run, schedule, tasklet
         import pickle
 
diff --git a/lib_pypy/pyrepl/completing_reader.py b/lib_pypy/pyrepl/completing_reader.py
--- a/lib_pypy/pyrepl/completing_reader.py
+++ b/lib_pypy/pyrepl/completing_reader.py
@@ -229,7 +229,8 @@
 
     def after_command(self, cmd):
         super(CompletingReader, self).after_command(cmd)
-        if not isinstance(cmd, complete) and not isinstance(cmd, self_insert):
+        if not isinstance(cmd, self.commands['complete']) \
+           and not isinstance(cmd, self.commands['self_insert']):
             self.cmpltn_reset()
 
     def calc_screen(self):
diff --git a/lib_pypy/pyrepl/reader.py b/lib_pypy/pyrepl/reader.py
--- a/lib_pypy/pyrepl/reader.py
+++ b/lib_pypy/pyrepl/reader.py
@@ -401,13 +401,19 @@
             return "(arg: %s) "%self.arg
         if "\n" in self.buffer:
             if lineno == 0:
-                return self._ps2
+                res = self.ps2
             elif lineno == self.buffer.count("\n"):
-                return self._ps4
+                res = self.ps4
             else:
-                return self._ps3
+                res = self.ps3
         else:
-            return self._ps1
+            res = self.ps1
+        # Lazily call str() on self.psN, and cache the results using as key
+        # the object on which str() was called.  This ensures that even if the
+        # same object is used e.g. for ps1 and ps2, str() is called only once.
+        if res not in self._pscache:
+            self._pscache[res] = str(res)
+        return self._pscache[res]
 
     def push_input_trans(self, itrans):
         self.input_trans_stack.append(self.input_trans)
@@ -473,8 +479,7 @@
             self.pos = 0
             self.dirty = 1
             self.last_command = None
-            self._ps1, self._ps2, self._ps3, self._ps4 = \
-                           map(str, [self.ps1, self.ps2, self.ps3, self.ps4])
+            self._pscache = {}
         except:
             self.restore()
             raise
@@ -571,7 +576,7 @@
         self.console.push_char(char)
         self.handle1(0)
     
-    def readline(self):
+    def readline(self, returns_unicode=False):
         """Read a line.  The implementation of this method also shows
         how to drive Reader if you want more control over the event
         loop."""
@@ -580,6 +585,8 @@
             self.refresh()
             while not self.finished:
                 self.handle1()
+            if returns_unicode:
+                return self.get_unicode()
             return self.get_buffer()
         finally:
             self.restore()
diff --git a/lib_pypy/pyrepl/readline.py b/lib_pypy/pyrepl/readline.py
--- a/lib_pypy/pyrepl/readline.py
+++ b/lib_pypy/pyrepl/readline.py
@@ -33,7 +33,7 @@
 from pyrepl.unix_console import UnixConsole, _error
 
 
-ENCODING = 'latin1'     # XXX hard-coded
+ENCODING = sys.getfilesystemencoding() or 'latin1'     # XXX review
 
 __all__ = ['add_history',
            'clear_history',
@@ -198,7 +198,7 @@
         reader.ps1 = prompt
         return reader.readline()
 
-    def multiline_input(self, more_lines, ps1, ps2):
+    def multiline_input(self, more_lines, ps1, ps2, returns_unicode=False):
         """Read an input on possibly multiple lines, asking for more
         lines as long as 'more_lines(unicodetext)' returns an object whose
         boolean value is true.
@@ -209,7 +209,7 @@
             reader.more_lines = more_lines
             reader.ps1 = reader.ps2 = ps1
             reader.ps3 = reader.ps4 = ps2
-            return reader.readline()
+            return reader.readline(returns_unicode=returns_unicode)
         finally:
             reader.more_lines = saved
 
@@ -395,9 +395,21 @@
     _wrapper.f_in = f_in
     _wrapper.f_out = f_out
 
-    if hasattr(sys, '__raw_input__'):    # PyPy
-        _old_raw_input = sys.__raw_input__
+    if '__pypy__' in sys.builtin_module_names:    # PyPy
+
+        def _old_raw_input(prompt=''):
+            # sys.__raw_input__() is only called when stdin and stdout are
+            # as expected and are ttys.  If it is the case, then get_reader()
+            # should not really fail in _wrapper.raw_input().  If it still
+            # does, then we will just cancel the redirection and call again
+            # the built-in raw_input().
+            try:
+                del sys.__raw_input__
+            except AttributeError:
+                pass
+            return raw_input(prompt)
         sys.__raw_input__ = _wrapper.raw_input
+
     else:
         # this is not really what readline.c does.  Better than nothing I guess
         import __builtin__
diff --git a/lib_pypy/pyrepl/simple_interact.py b/lib_pypy/pyrepl/simple_interact.py
--- a/lib_pypy/pyrepl/simple_interact.py
+++ b/lib_pypy/pyrepl/simple_interact.py
@@ -54,7 +54,8 @@
             ps1 = getattr(sys, 'ps1', '>>> ')
             ps2 = getattr(sys, 'ps2', '... ')
             try:
-                statement = multiline_input(more_lines, ps1, ps2)
+                statement = multiline_input(more_lines, ps1, ps2,
+                                            returns_unicode=True)
             except EOFError:
                 break
             more = console.push(statement)
diff --git a/lib_pypy/pyrepl/unix_console.py b/lib_pypy/pyrepl/unix_console.py
--- a/lib_pypy/pyrepl/unix_console.py
+++ b/lib_pypy/pyrepl/unix_console.py
@@ -384,15 +384,19 @@
 
         self.__maybe_write_code(self._smkx)
 
-        self.old_sigwinch = signal.signal(
-            signal.SIGWINCH, self.__sigwinch)
+        try:
+            self.old_sigwinch = signal.signal(
+                signal.SIGWINCH, self.__sigwinch)
+        except ValueError:
+            pass
 
     def restore(self):
         self.__maybe_write_code(self._rmkx)
         self.flushoutput()
         tcsetattr(self.input_fd, termios.TCSADRAIN, self.__svtermstate)
 
-        signal.signal(signal.SIGWINCH, self.old_sigwinch)
+        if hasattr(self, 'old_sigwinch'):
+            signal.signal(signal.SIGWINCH, self.old_sigwinch)
 
     def __sigwinch(self, signum, frame):
         self.height, self.width = self.getheightwidth()
diff --git a/lib_pypy/resource.py b/lib_pypy/resource.py
--- a/lib_pypy/resource.py
+++ b/lib_pypy/resource.py
@@ -7,7 +7,7 @@
 
 from ctypes_support import standard_c_lib as libc
 from ctypes_support import get_errno
-from ctypes import Structure, c_int, c_long, byref, sizeof
+from ctypes import Structure, c_int, c_long, byref, POINTER
 from errno import EINVAL, EPERM
 import _structseq
 
@@ -25,6 +25,8 @@
 _setrlimit = libc.setrlimit
 try:
     _getpagesize = libc.getpagesize
+    _getpagesize.argtypes = ()
+    _getpagesize.restype = c_int
 except AttributeError:
     from os import sysconf
     _getpagesize = None
@@ -61,6 +63,10 @@
         ("ru_nivcsw", c_long),
     )
 
+_getrusage.argtypes = (c_int, POINTER(_struct_rusage))
+_getrusage.restype = c_int
+
+
 class struct_rusage:
     __metaclass__ = _structseq.structseqtype
 
@@ -94,6 +100,12 @@
         ("rlim_max", rlim_t),
     )
 
+_getrlimit.argtypes = (c_int, POINTER(rlimit))
+_getrlimit.restype = c_int
+_setrlimit.argtypes = (c_int, POINTER(rlimit))
+_setrlimit.restype = c_int
+
+
 @builtinify
 def getrusage(who):
     ru = _struct_rusage()
@@ -153,7 +165,6 @@
 
 @builtinify
 def getpagesize():
-    pagesize = 0
     if _getpagesize:
         return _getpagesize()
     else:
diff --git a/lib_pypy/stackless.py b/lib_pypy/stackless.py
--- a/lib_pypy/stackless.py
+++ b/lib_pypy/stackless.py
@@ -4,121 +4,110 @@
 Please refer to their documentation.
 """
 
-DEBUG = True
 
-def dprint(*args):
-    for arg in args:
-        print arg,
-    print
+import _continuation
 
-import traceback
-import sys
+class TaskletExit(Exception):
+    pass
+
+CoroutineExit = TaskletExit
+
+
+def _coroutine_getcurrent():
+    "Returns the current coroutine (i.e. the one which called this function)."
+    try:
+        return _tls.current_coroutine
+    except AttributeError:
+        # first call in this thread: current == main
+        return _coroutine_getmain()
+
+def _coroutine_getmain():
+    try:
+        return _tls.main_coroutine
+    except AttributeError:
+        # create the main coroutine for this thread
+        continulet = _continuation.continulet
+        main = coroutine()
+        main._frame = continulet.__new__(continulet)
+        main._is_started = -1
+        _tls.current_coroutine = _tls.main_coroutine = main
+        return _tls.main_coroutine
+
+
+class coroutine(object):
+    _is_started = 0      # 0=no, 1=yes, -1=main
+
+    def __init__(self):
+        self._frame = None
+
+    def bind(self, func, *argl, **argd):
+        """coro.bind(f, *argl, **argd) -> None.
+           binds function f to coro. f will be called with
+           arguments *argl, **argd
+        """
+        if self.is_alive:
+            raise ValueError("cannot bind a bound coroutine")
+        def run(c):
+            _tls.current_coroutine = self
+            self._is_started = 1
+            return func(*argl, **argd)
+        self._is_started = 0
+        self._frame = _continuation.continulet(run)
+
+    def switch(self):
+        """coro.switch() -> returnvalue
+           switches to coroutine coro. If the bound function
+           f finishes, the returnvalue is that of f, otherwise
+           None is returned
+        """
+        current = _coroutine_getcurrent()
+        try:
+            current._frame.switch(to=self._frame)
+        finally:
+            _tls.current_coroutine = current
+
+    def kill(self):
+        """coro.kill() : kill coroutine coro"""
+        current = _coroutine_getcurrent()
+        try:
+            current._frame.throw(CoroutineExit, to=self._frame)
+        finally:
+            _tls.current_coroutine = current
+
+    @property
+    def is_alive(self):
+        return self._is_started < 0 or (
+            self._frame is not None and self._frame.is_pending())
+
+    @property
+    def is_zombie(self):
+        return self._is_started > 0 and not self._frame.is_pending()
+
+    getcurrent = staticmethod(_coroutine_getcurrent)
+
+    def __reduce__(self):
+        if self._is_started < 0:
+            return _coroutine_getmain, ()
+        else:
+            return type(self), (), self.__dict__
+
+
 try:
-    # If _stackless can be imported then TaskletExit and CoroutineExit are 
-    # automatically added to the builtins.
-    from _stackless import coroutine, greenlet
-except ImportError: # we are running from CPython
-    from greenlet import greenlet, GreenletExit
-    TaskletExit = CoroutineExit = GreenletExit
-    del GreenletExit
-    try:
-        from functools import partial
-    except ImportError: # we are not running python 2.5
-        class partial(object):
-            # just enough of 'partial' to be usefull
-            def __init__(self, func, *argl, **argd):
-                self.func = func
-                self.argl = argl
-                self.argd = argd
+    from thread import _local
+except ImportError:
+    class _local(object):    # assume no threads
+        pass
 
-            def __call__(self):
-                return self.func(*self.argl, **self.argd)
+_tls = _local()
 
-    class GWrap(greenlet):
-        """This is just a wrapper around greenlets to allow
-           to stick additional attributes to a greenlet.
-           To be more concrete, we need a backreference to
-           the coroutine object"""
 
-    class MWrap(object):
-        def __init__(self,something):
-            self.something = something
+# ____________________________________________________________
 
-        def __getattr__(self, attr):
-            return getattr(self.something, attr)
-
-    class coroutine(object):
-        "we can't have greenlet as a base, because greenlets can't be rebound"
-
-        def __init__(self):
-            self._frame = None
-            self.is_zombie = False
-
-        def __getattr__(self, attr):
-            return getattr(self._frame, attr)
-
-        def __del__(self):
-            self.is_zombie = True
-            del self._frame
-            self._frame = None
-
-        def bind(self, func, *argl, **argd):
-            """coro.bind(f, *argl, **argd) -> None.
-               binds function f to coro. f will be called with
-               arguments *argl, **argd
-            """
-            if self._frame is None or self._frame.dead:
-                self._frame = frame = GWrap()
-                frame.coro = self
-            if hasattr(self._frame, 'run') and self._frame.run:
-                raise ValueError("cannot bind a bound coroutine")
-            self._frame.run = partial(func, *argl, **argd)
-
-        def switch(self):
-            """coro.switch() -> returnvalue
-               switches to coroutine coro. If the bound function
-               f finishes, the returnvalue is that of f, otherwise
-               None is returned
-            """
-            try:
-                return greenlet.switch(self._frame)
-            except TypeError, exp: # self._frame is the main coroutine
-                return greenlet.switch(self._frame.something)
-
-        def kill(self):
-            """coro.kill() : kill coroutine coro"""
-            self._frame.throw()
-
-        def _is_alive(self):
-            if self._frame is None:
-                return False
-            return not self._frame.dead
-        is_alive = property(_is_alive)
-        del _is_alive
-
-        def getcurrent():
-            """coroutine.getcurrent() -> the currently running coroutine"""
-            try:
-                return greenlet.getcurrent().coro
-            except AttributeError:
-                return _maincoro
-        getcurrent = staticmethod(getcurrent)
-
-        def __reduce__(self):
-            raise TypeError, 'pickling is not possible based upon greenlets'
-
-    _maincoro = coroutine()
-    maingreenlet = greenlet.getcurrent()
-    _maincoro._frame = frame = MWrap(maingreenlet)
-    frame.coro = _maincoro
-    del frame
-    del maingreenlet
 
 from collections import deque
 
 import operator
-__all__ = 'run getcurrent getmain schedule tasklet channel coroutine \
-                greenlet'.split()
+__all__ = 'run getcurrent getmain schedule tasklet channel coroutine'.split()
 
 _global_task_id = 0
 _squeue = None
@@ -131,7 +120,8 @@
 def _scheduler_remove(value):
     try:
         del _squeue[operator.indexOf(_squeue, value)]
-    except ValueError:pass
+    except ValueError:
+        pass
 
 def _scheduler_append(value, normal=True):
     if normal:
@@ -157,10 +147,7 @@
     _last_task = next
     assert not next.blocked
     if next is not current:
-        try:
-            next.switch()
-        except CoroutineExit:
-            raise TaskletExit
+        next.switch()
     return current
 
 def set_schedule_callback(callback):
@@ -184,34 +171,6 @@
         raise self.type, self.value, self.traceback
 
 #
-# helpers for pickling
-#
-
-_stackless_primitive_registry = {}
-
-def register_stackless_primitive(thang, retval_expr='None'):
-    import types
-    func = thang
-    if isinstance(thang, types.MethodType):
-        func = thang.im_func
-    code = func.func_code
-    _stackless_primitive_registry[code] = retval_expr
-    # It is not too nice to attach info via the code object, but
-    # I can't think of a better solution without a real transform.
-
-def rewrite_stackless_primitive(coro_state, alive, tempval):
-    flags, frame, thunk, parent = coro_state
-    while frame is not None:
-        retval_expr = _stackless_primitive_registry.get(frame.f_code)
-        if retval_expr:
-            # this tasklet needs to stop pickling here and return its value.
-            tempval = eval(retval_expr, globals(), frame.f_locals)
-            coro_state = flags, frame, thunk, parent
-            break
-        frame = frame.f_back
-    return coro_state, alive, tempval
-
-#
 #
 
 class channel(object):
@@ -363,8 +322,6 @@
         """
         return self._channel_action(None, -1)
 
-    register_stackless_primitive(receive, retval_expr='receiver.tempval')
-
     def send_exception(self, exp_type, msg):
         self.send(bomb(exp_type, exp_type(msg)))
 
@@ -381,9 +338,8 @@
         the runnables list.
         """
         return self._channel_action(msg, 1)
-            
-    register_stackless_primitive(send)
-            
+
+
 class tasklet(coroutine):
     """
     A tasklet object represents a tiny task in a Python thread.
@@ -455,6 +411,7 @@
         def _func():
             try:
                 try:
+                    coroutine.switch(back)
                     func(*argl, **argd)
                 except TaskletExit:
                     pass
@@ -464,6 +421,8 @@
 
         self.func = None
         coroutine.bind(self, _func)
+        back = _coroutine_getcurrent()
+        coroutine.switch(self)
         self.alive = True
         _scheduler_append(self)
         return self
@@ -486,39 +445,6 @@
             raise RuntimeError, "The current tasklet cannot be removed."
             # not sure if I will revive this  " Use t=tasklet().capture()"
         _scheduler_remove(self)
-        
-    def __reduce__(self):
-        one, two, coro_state = coroutine.__reduce__(self)
-        assert one is coroutine
-        assert two == ()
-        # we want to get rid of the parent thing.
-        # for now, we just drop it
-        a, frame, c, d = coro_state
-
-        # Removing all frames related to stackless.py.
-        # They point to stuff we don't want to be pickled.
-
-        pickleframe = frame
-        while frame is not None:
-            if frame.f_code == schedule.func_code:
-                # Removing everything including and after the
-                # call to stackless.schedule()
-                pickleframe = frame.f_back
-                break
-            frame = frame.f_back
-        if d:
-            assert isinstance(d, coroutine)
-        coro_state = a, pickleframe, c, None
-        coro_state, alive, tempval = rewrite_stackless_primitive(coro_state, self.alive, self.tempval)
-        inst_dict = self.__dict__.copy()
-        inst_dict.pop('tempval', None)
-        return self.__class__, (), (coro_state, alive, tempval, inst_dict)
-
-    def __setstate__(self, (coro_state, alive, tempval, inst_dict)):
-        coroutine.__setstate__(self, coro_state)
-        self.__dict__.update(inst_dict)
-        self.alive = alive
-        self.tempval = tempval
 
 def getmain():
     """
@@ -607,30 +533,7 @@
     global _last_task
     _global_task_id = 0
     _main_tasklet = coroutine.getcurrent()
-    try:
-        _main_tasklet.__class__ = tasklet
-    except TypeError: # we are running pypy-c
-        class TaskletProxy(object):
-            """TaskletProxy is needed to give the _main_coroutine tasklet behaviour"""
-            def __init__(self, coro):
-                self._coro = coro
-
-            def __getattr__(self,attr):
-                return getattr(self._coro,attr)
-
-            def __str__(self):
-                return '<tasklet %s a:%s>' % (self._task_id, self.is_alive)
-
-            def __reduce__(self):
-                return getmain, ()
-
-            __repr__ = __str__
-
-
-        global _main_coroutine
-        _main_coroutine = _main_tasklet
-        _main_tasklet = TaskletProxy(_main_tasklet)
-        assert _main_tasklet.is_alive and not _main_tasklet.is_zombie
+    _main_tasklet.__class__ = tasklet         # XXX HAAAAAAAAAAAAAAAAAAAAACK
     _last_task = _main_tasklet
     tasklet._init.im_func(_main_tasklet, label='main')
     _squeue = deque()
diff --git a/py/__init__.py b/py/__init__.py
--- a/py/__init__.py
+++ b/py/__init__.py
@@ -8,7 +8,7 @@
 
 (c) Holger Krekel and others, 2004-2010
 """
-__version__ = '1.4.3'
+__version__ = '1.4.4.dev1'
 
 from py import _apipkg
 
@@ -70,10 +70,6 @@
         'getrawcode'        : '._code.code:getrawcode',
         'patch_builtins'    : '._code.code:patch_builtins',
         'unpatch_builtins'  : '._code.code:unpatch_builtins',
-        '_AssertionError'   : '._code.assertion:AssertionError',
-        '_reinterpret_old'  : '._code.assertion:reinterpret_old',
-        '_reinterpret'      : '._code.assertion:reinterpret',
-        '_reprcompare'      : '._code.assertion:_reprcompare',
     },
 
     # backports and additions of builtins
diff --git a/py/_code/_assertionnew.py b/py/_code/_assertionnew.py
deleted file mode 100644
--- a/py/_code/_assertionnew.py
+++ /dev/null
@@ -1,339 +0,0 @@
-"""
-Find intermediate evalutation results in assert statements through builtin AST.
-This should replace _assertionold.py eventually.
-"""
-
-import sys
-import ast
-
-import py
-from py._code.assertion import _format_explanation, BuiltinAssertionError
-
-
-if sys.platform.startswith("java") and sys.version_info < (2, 5, 2):
-    # See http://bugs.jython.org/issue1497
-    _exprs = ("BoolOp", "BinOp", "UnaryOp", "Lambda", "IfExp", "Dict",
-              "ListComp", "GeneratorExp", "Yield", "Compare", "Call",
-              "Repr", "Num", "Str", "Attribute", "Subscript", "Name",
-              "List", "Tuple")
-    _stmts = ("FunctionDef", "ClassDef", "Return", "Delete", "Assign",
-              "AugAssign", "Print", "For", "While", "If", "With", "Raise",
-              "TryExcept", "TryFinally", "Assert", "Import", "ImportFrom",
-              "Exec", "Global", "Expr", "Pass", "Break", "Continue")
-    _expr_nodes = set(getattr(ast, name) for name in _exprs)
-    _stmt_nodes = set(getattr(ast, name) for name in _stmts)
-    def _is_ast_expr(node):
-        return node.__class__ in _expr_nodes
-    def _is_ast_stmt(node):
-        return node.__class__ in _stmt_nodes
-else:
-    def _is_ast_expr(node):
-        return isinstance(node, ast.expr)
-    def _is_ast_stmt(node):
-        return isinstance(node, ast.stmt)
-
-
-class Failure(Exception):
-    """Error found while interpreting AST."""
-
-    def __init__(self, explanation=""):
-        self.cause = sys.exc_info()
-        self.explanation = explanation
-
-
-def interpret(source, frame, should_fail=False):
-    mod = ast.parse(source)
-    visitor = DebugInterpreter(frame)
-    try:
-        visitor.visit(mod)
-    except Failure:
-        failure = sys.exc_info()[1]
-        return getfailure(failure)
-    if should_fail:
-        return ("(assertion failed, but when it was re-run for "
-                "printing intermediate values, it did not fail.  Suggestions: "
-                "compute assert expression before the assert or use --no-assert)")
-
-def run(offending_line, frame=None):
-    if frame is None:
-        frame = py.code.Frame(sys._getframe(1))
-    return interpret(offending_line, frame)
-
-def getfailure(failure):
-    explanation = _format_explanation(failure.explanation)
-    value = failure.cause[1]
-    if str(value):
-        lines = explanation.splitlines()
-        if not lines:
-            lines.append("")
-        lines[0] += " << %s" % (value,)
-        explanation = "\n".join(lines)
-    text = "%s: %s" % (failure.cause[0].__name__, explanation)
-    if text.startswith("AssertionError: assert "):
-        text = text[16:]
-    return text
-
-
-operator_map = {
-    ast.BitOr : "|",
-    ast.BitXor : "^",
-    ast.BitAnd : "&",
-    ast.LShift : "<<",
-    ast.RShift : ">>",
-    ast.Add : "+",
-    ast.Sub : "-",
-    ast.Mult : "*",
-    ast.Div : "/",
-    ast.FloorDiv : "//",
-    ast.Mod : "%",
-    ast.Eq : "==",
-    ast.NotEq : "!=",
-    ast.Lt : "<",
-    ast.LtE : "<=",
-    ast.Gt : ">",
-    ast.GtE : ">=",
-    ast.Pow : "**",
-    ast.Is : "is",
-    ast.IsNot : "is not",
-    ast.In : "in",
-    ast.NotIn : "not in"
-}
-
-unary_map = {
-    ast.Not : "not %s",
-    ast.Invert : "~%s",
-    ast.USub : "-%s",
-    ast.UAdd : "+%s"
-}
-
-
-class DebugInterpreter(ast.NodeVisitor):
-    """Interpret AST nodes to gleam useful debugging information. """
-
-    def __init__(self, frame):
-        self.frame = frame
-
-    def generic_visit(self, node):
-        # Fallback when we don't have a special implementation.
-        if _is_ast_expr(node):
-            mod = ast.Expression(node)
-            co = self._compile(mod)
-            try:
-                result = self.frame.eval(co)
-            except Exception:
-                raise Failure()
-            explanation = self.frame.repr(result)
-            return explanation, result
-        elif _is_ast_stmt(node):
-            mod = ast.Module([node])
-            co = self._compile(mod, "exec")
-            try:
-                self.frame.exec_(co)
-            except Exception:
-                raise Failure()
-            return None, None
-        else:
-            raise AssertionError("can't handle %s" %(node,))
-
-    def _compile(self, source, mode="eval"):
-        return compile(source, "<assertion interpretation>", mode)
-
-    def visit_Expr(self, expr):
-        return self.visit(expr.value)
-
-    def visit_Module(self, mod):
-        for stmt in mod.body:
-            self.visit(stmt)
-
-    def visit_Name(self, name):
-        explanation, result = self.generic_visit(name)
-        # See if the name is local.
-        source = "%r in locals() is not globals()" % (name.id,)
-        co = self._compile(source)
-        try:
-            local = self.frame.eval(co)
-        except Exception:
-            # have to assume it isn't
-            local = False
-        if not local:
-            return name.id, result
-        return explanation, result
-
-    def visit_Compare(self, comp):
-        left = comp.left
-        left_explanation, left_result = self.visit(left)
-        for op, next_op in zip(comp.ops, comp.comparators):
-            next_explanation, next_result = self.visit(next_op)
-            op_symbol = operator_map[op.__class__]
-            explanation = "%s %s %s" % (left_explanation, op_symbol,
-                                        next_explanation)
-            source = "__exprinfo_left %s __exprinfo_right" % (op_symbol,)
-            co = self._compile(source)
-            try:
-                result = self.frame.eval(co, __exprinfo_left=left_result,
-                                         __exprinfo_right=next_result)
-            except Exception:
-                raise Failure(explanation)
-            try:
-                if not result:
-                    break
-            except KeyboardInterrupt:
-                raise
-            except:
-                break
-            left_explanation, left_result = next_explanation, next_result
-
-        rcomp = py.code._reprcompare
-        if rcomp:
-            res = rcomp(op_symbol, left_result, next_result)
-            if res:
-                explanation = res
-        return explanation, result
-
-    def visit_BoolOp(self, boolop):
-        is_or = isinstance(boolop.op, ast.Or)
-        explanations = []
-        for operand in boolop.values:
-            explanation, result = self.visit(operand)
-            explanations.append(explanation)
-            if result == is_or:
-                break
-        name = is_or and " or " or " and "
-        explanation = "(" + name.join(explanations) + ")"
-        return explanation, result
-
-    def visit_UnaryOp(self, unary):
-        pattern = unary_map[unary.op.__class__]
-        operand_explanation, operand_result = self.visit(unary.operand)
-        explanation = pattern % (operand_explanation,)
-        co = self._compile(pattern % ("__exprinfo_expr",))
-        try:
-            result = self.frame.eval(co, __exprinfo_expr=operand_result)
-        except Exception:
-            raise Failure(explanation)
-        return explanation, result
-
-    def visit_BinOp(self, binop):
-        left_explanation, left_result = self.visit(binop.left)
-        right_explanation, right_result = self.visit(binop.right)
-        symbol = operator_map[binop.op.__class__]
-        explanation = "(%s %s %s)" % (left_explanation, symbol,
-                                      right_explanation)
-        source = "__exprinfo_left %s __exprinfo_right" % (symbol,)
-        co = self._compile(source)
-        try:
-            result = self.frame.eval(co, __exprinfo_left=left_result,
-                                     __exprinfo_right=right_result)
-        except Exception:
-            raise Failure(explanation)
-        return explanation, result
-
-    def visit_Call(self, call):
-        func_explanation, func = self.visit(call.func)
-        arg_explanations = []
-        ns = {"__exprinfo_func" : func}
-        arguments = []
-        for arg in call.args:
-            arg_explanation, arg_result = self.visit(arg)
-            arg_name = "__exprinfo_%s" % (len(ns),)
-            ns[arg_name] = arg_result
-            arguments.append(arg_name)
-            arg_explanations.append(arg_explanation)
-        for keyword in call.keywords:
-            arg_explanation, arg_result = self.visit(keyword.value)
-            arg_name = "__exprinfo_%s" % (len(ns),)
-            ns[arg_name] = arg_result
-            keyword_source = "%s=%%s" % (keyword.arg)
-            arguments.append(keyword_source % (arg_name,))
-            arg_explanations.append(keyword_source % (arg_explanation,))
-        if call.starargs:
-            arg_explanation, arg_result = self.visit(call.starargs)
-            arg_name = "__exprinfo_star"
-            ns[arg_name] = arg_result
-            arguments.append("*%s" % (arg_name,))
-            arg_explanations.append("*%s" % (arg_explanation,))
-        if call.kwargs:
-            arg_explanation, arg_result = self.visit(call.kwargs)
-            arg_name = "__exprinfo_kwds"
-            ns[arg_name] = arg_result
-            arguments.append("**%s" % (arg_name,))
-            arg_explanations.append("**%s" % (arg_explanation,))
-        args_explained = ", ".join(arg_explanations)
-        explanation = "%s(%s)" % (func_explanation, args_explained)
-        args = ", ".join(arguments)
-        source = "__exprinfo_func(%s)" % (args,)
-        co = self._compile(source)
-        try:
-            result = self.frame.eval(co, **ns)
-        except Exception:
-            raise Failure(explanation)
-        pattern = "%s\n{%s = %s\n}"
-        rep = self.frame.repr(result)
-        explanation = pattern % (rep, rep, explanation)
-        return explanation, result
-
-    def _is_builtin_name(self, name):
-        pattern = "%r not in globals() and %r not in locals()"
-        source = pattern % (name.id, name.id)
-        co = self._compile(source)
-        try:
-            return self.frame.eval(co)
-        except Exception:
-            return False

From noreply at buildbot.pypy.org  Tue Oct 25 15:36:33 2011
From: noreply at buildbot.pypy.org (hager)
Date: Tue, 25 Oct 2011 15:36:33 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: (bivab,
	hager): fix interface access after merge
Message-ID: <20111025133634.00DD0820C7@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r48430:c00c0dca8cfc
Date: 2011-10-25 15:36 +0200
http://bitbucket.org/pypy/pypy/changeset/c00c0dca8cfc/

Log:	(bivab, hager): fix interface access after merge

diff --git a/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py b/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py
--- a/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py
@@ -81,6 +81,7 @@
         self.datablockwrapper = None
         self.memcpy_addr = 0
         self.fail_boxes_count = 0
+        self.current_clt = None
 
     def load_imm(self, rD, word):
         if word <= 32767 and word >= -32768:
@@ -378,10 +379,10 @@
                 self.regalloc_mov(r.r0, loc)
 
     def setup(self, looptoken, operations):
+        assert self.memcpy_addr != 0
+        self.current_clt = looptoken.compiled_loop_token 
         operations = self.cpu.gc_ll_descr.rewrite_assembler(self.cpu, 
-                                                            operations)
-        assert self.memcpy_addr != 0
-        self.current_clt = looptoken.compiled_loop_token
+                operations, self.current_clt.allgcrefs)
         self.mc = PPCBuilder()
         self.pending_guards = []
         assert self.datablockwrapper is None
@@ -399,6 +400,7 @@
     def assemble_loop(self, inputargs, operations, looptoken, log):
 
         clt = CompiledLoopToken(self.cpu, looptoken.number)
+        clt.allgcrefs = []
         looptoken.compiled_loop_token = clt
 
         self.setup(looptoken, operations)

From noreply at buildbot.pypy.org  Tue Oct 25 15:41:46 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 25 Oct 2011 15:41:46 +0200 (CEST)
Subject: [pypy-commit] pypy default: (fijal,
 arigo) Remove call to clear_all_weakrefs with an explanation. A test
Message-ID: <20111025134146.29E47820C7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r48431:9079d6cb7394
Date: 2011-10-25 15:38 +0200
http://bitbucket.org/pypy/pypy/changeset/9079d6cb7394/

Log:	(fijal, arigo) Remove call to clear_all_weakrefs with an
	explanation. A test that might potentially fail with -A if things go
	wrong.

diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -211,7 +211,9 @@
             return result
 
         def __del__(self):
-            self.clear_all_weakrefs()
+            # note that we don't call clear_all_weakrefs here because
+            # an array with freed buffer is ok to see - it's just empty with 0
+            # length
             self.setlen(0)
 
         def setlen(self, size):
diff --git a/pypy/module/array/test/test_array.py b/pypy/module/array/test/test_array.py
--- a/pypy/module/array/test/test_array.py
+++ b/pypy/module/array/test/test_array.py
@@ -824,6 +824,22 @@
         r = weakref.ref(a)
         assert r() is a
 
+    def test_subclass_del(self):
+        import array, gc, weakref
+        l = []
+        
+        class A(array.array):
+            pass
+
+        a = A('d')
+        a.append(3.0)
+        r = weakref.ref(a, lambda a: l.append(a()))
+        del a
+        gc.collect()
+        assert l
+        assert l[0] is None or len(l[0]) == 0
+
+
 class TestCPythonsOwnArray(BaseArrayTests):
 
     def setup_class(cls):
@@ -844,11 +860,7 @@
         cls.w_tempfile = cls.space.wrap(
             str(py.test.ensuretemp('array').join('tmpfile')))
         cls.w_maxint = cls.space.wrap(sys.maxint)
-
-
-
-
-
+    
     def test_buffer_info(self):
         a = self.array('c', 'Hi!')
         bi = a.buffer_info()

From noreply at buildbot.pypy.org  Tue Oct 25 15:41:47 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 25 Oct 2011 15:41:47 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge default
Message-ID: <20111025134147.7F33F820C7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r48432:f5825eff38c3
Date: 2011-10-25 15:41 +0200
http://bitbucket.org/pypy/pypy/changeset/f5825eff38c3/

Log:	merge default

diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1276,8 +1276,8 @@
     genop_int_ne = _cmpop("NE", "NE")
     genop_int_gt = _cmpop("G", "L")
     genop_int_ge = _cmpop("GE", "LE")
-    genop_ptr_eq = genop_int_eq
-    genop_ptr_ne = genop_int_ne
+    genop_ptr_eq = genop_instance_ptr_eq = genop_int_eq
+    genop_ptr_ne = genop_instance_ptr_ne = genop_int_ne
 
     genop_float_lt = _cmpop_float('B', 'A')
     genop_float_le = _cmpop_float('BE', 'AE')
@@ -1297,8 +1297,8 @@
     genop_guard_int_ne = _cmpop_guard("NE", "NE", "E", "E")
     genop_guard_int_gt = _cmpop_guard("G", "L", "LE", "GE")
     genop_guard_int_ge = _cmpop_guard("GE", "LE", "L", "G")
-    genop_guard_ptr_eq = genop_guard_int_eq
-    genop_guard_ptr_ne = genop_guard_int_ne
+    genop_guard_ptr_eq = genop_guard_instance_ptr_eq = genop_guard_int_eq
+    genop_guard_ptr_ne = genop_guard_instance_ptr_ne = genop_guard_int_ne
 
     genop_guard_uint_gt = _cmpop_guard("A", "B", "BE", "AE")
     genop_guard_uint_lt = _cmpop_guard("B", "A", "AE", "BE")
diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py
--- a/pypy/jit/backend/x86/regalloc.py
+++ b/pypy/jit/backend/x86/regalloc.py
@@ -651,8 +651,8 @@
     consider_uint_lt = _consider_compop
     consider_uint_le = _consider_compop
     consider_uint_ge = _consider_compop
-    consider_ptr_eq = _consider_compop
-    consider_ptr_ne = _consider_compop
+    consider_ptr_eq = consider_instance_ptr_eq = _consider_compop
+    consider_ptr_ne = consider_instance_ptr_ne = _consider_compop
 
     def _consider_float_op(self, op):
         loc1 = self.xrm.loc(op.getarg(1))
diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -800,6 +800,9 @@
     def _is_gc(self, v):
         return getattr(getattr(v.concretetype, "TO", None), "_gckind", "?") == 'gc'
 
+    def _is_rclass_instance(self, v):
+        return lltype._castdepth(v.concretetype.TO, rclass.OBJECT) >= 0
+
     def _rewrite_cmp_ptrs(self, op):
         if self._is_gc(op.args[0]):
             return op
@@ -817,11 +820,21 @@
         return self._rewrite_equality(op, 'int_is_true')
 
     def rewrite_op_ptr_eq(self, op):
-        op1 = self._rewrite_equality(op, 'ptr_iszero')
+        prefix = ''
+        if self._is_rclass_instance(op.args[0]):
+            assert self._is_rclass_instance(op.args[1])
+            op = SpaceOperation('instance_ptr_eq', op.args, op.result)
+            prefix = 'instance_'
+        op1 = self._rewrite_equality(op, prefix + 'ptr_iszero')
         return self._rewrite_cmp_ptrs(op1)
 
     def rewrite_op_ptr_ne(self, op):
-        op1 = self._rewrite_equality(op, 'ptr_nonzero')
+        prefix = ''
+        if self._is_rclass_instance(op.args[0]):
+            assert self._is_rclass_instance(op.args[1])
+            op = SpaceOperation('instance_ptr_ne', op.args, op.result)
+            prefix = 'instance_'
+        op1 = self._rewrite_equality(op, prefix + 'ptr_nonzero')
         return self._rewrite_cmp_ptrs(op1)
 
     rewrite_op_ptr_iszero = _rewrite_cmp_ptrs
diff --git a/pypy/jit/codewriter/test/test_jtransform.py b/pypy/jit/codewriter/test/test_jtransform.py
--- a/pypy/jit/codewriter/test/test_jtransform.py
+++ b/pypy/jit/codewriter/test/test_jtransform.py
@@ -576,10 +576,10 @@
         assert op1.args == [v2]
 
 def test_ptr_eq():
-    v1 = varoftype(rclass.OBJECTPTR)
-    v2 = varoftype(rclass.OBJECTPTR)
+    v1 = varoftype(lltype.Ptr(rstr.STR))
+    v2 = varoftype(lltype.Ptr(rstr.STR))
     v3 = varoftype(lltype.Bool)
-    c0 = const(lltype.nullptr(rclass.OBJECT))
+    c0 = const(lltype.nullptr(rstr.STR))
     #
     for opname, reducedname in [('ptr_eq', 'ptr_iszero'),
                                 ('ptr_ne', 'ptr_nonzero')]:
@@ -598,6 +598,31 @@
         assert op1.opname == reducedname
         assert op1.args == [v2]
 
+def test_instance_ptr_eq():
+    v1 = varoftype(rclass.OBJECTPTR)
+    v2 = varoftype(rclass.OBJECTPTR)
+    v3 = varoftype(lltype.Bool)
+    c0 = const(lltype.nullptr(rclass.OBJECT))
+
+    for opname, newopname, reducedname in [
+        ('ptr_eq', 'instance_ptr_eq', 'instance_ptr_iszero'),
+        ('ptr_ne', 'instance_ptr_ne', 'instance_ptr_nonzero')
+    ]:
+        op = SpaceOperation(opname, [v1, v2], v3)
+        op1 = Transformer().rewrite_operation(op)
+        assert op1.opname == newopname
+        assert op1.args == [v1, v2]
+
+        op = SpaceOperation(opname, [v1, c0], v3)
+        op1 = Transformer().rewrite_operation(op)
+        assert op1.opname == reducedname
+        assert op1.args == [v1]
+
+        op = SpaceOperation(opname, [c0, v1], v3)
+        op1 = Transformer().rewrite_operation(op)
+        assert op1.opname == reducedname
+        assert op1.args == [v1]
+
 def test_nongc_ptr_eq():
     v1 = varoftype(rclass.NONGCOBJECTPTR)
     v2 = varoftype(rclass.NONGCOBJECTPTR)
diff --git a/pypy/jit/metainterp/blackhole.py b/pypy/jit/metainterp/blackhole.py
--- a/pypy/jit/metainterp/blackhole.py
+++ b/pypy/jit/metainterp/blackhole.py
@@ -499,6 +499,12 @@
     @arguments("r", returns="i")
     def bhimpl_ptr_nonzero(a):
         return bool(a)
+    @arguments("r", "r", returns="i")
+    def bhimpl_instance_ptr_eq(a, b):
+        return a == b
+    @arguments("r", "r", returns="i")
+    def bhimpl_instance_ptr_ne(a, b):
+        return a != b
     @arguments("r", returns="r")
     def bhimpl_cast_opaque_ptr(a):
         return a
diff --git a/pypy/jit/metainterp/optimizeopt/rewrite.py b/pypy/jit/metainterp/optimizeopt/rewrite.py
--- a/pypy/jit/metainterp/optimizeopt/rewrite.py
+++ b/pypy/jit/metainterp/optimizeopt/rewrite.py
@@ -337,7 +337,7 @@
     def optimize_INT_IS_ZERO(self, op):
         self._optimize_nullness(op, op.getarg(0), False)
 
-    def _optimize_oois_ooisnot(self, op, expect_isnot):
+    def _optimize_oois_ooisnot(self, op, expect_isnot, instance):
         value0 = self.getvalue(op.getarg(0))
         value1 = self.getvalue(op.getarg(1))
         if value0.is_virtual():
@@ -355,21 +355,28 @@
         elif value0 is value1:
             self.make_constant_int(op.result, not expect_isnot)
         else:
-            cls0 = value0.get_constant_class(self.optimizer.cpu)
-            if cls0 is not None:
-                cls1 = value1.get_constant_class(self.optimizer.cpu)
-                if cls1 is not None and not cls0.same_constant(cls1):
-                    # cannot be the same object, as we know that their
-                    # class is different
-                    self.make_constant_int(op.result, expect_isnot)
-                    return
+            if instance:
+                cls0 = value0.get_constant_class(self.optimizer.cpu)
+                if cls0 is not None:
+                    cls1 = value1.get_constant_class(self.optimizer.cpu)
+                    if cls1 is not None and not cls0.same_constant(cls1):
+                        # cannot be the same object, as we know that their
+                        # class is different
+                        self.make_constant_int(op.result, expect_isnot)
+                        return
             self.emit_operation(op)
 
+    def optimize_PTR_EQ(self, op):
+        self._optimize_oois_ooisnot(op, False, False)
+
     def optimize_PTR_NE(self, op):
-        self._optimize_oois_ooisnot(op, True)
+        self._optimize_oois_ooisnot(op, True, False)
 
-    def optimize_PTR_EQ(self, op):
-        self._optimize_oois_ooisnot(op, False)
+    def optimize_INSTANCE_PTR_EQ(self, op):
+        self._optimize_oois_ooisnot(op, False, True)
+
+    def optimize_INSTANCE_PTR_NE(self, op):
+        self._optimize_oois_ooisnot(op, True, True)
 
 ##    def optimize_INSTANCEOF(self, op):
 ##        value = self.getvalue(op.args[0])
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -508,13 +508,13 @@
         ops = """
         [p0]
         guard_class(p0, ConstClass(node_vtable)) []
-        i0 = ptr_ne(p0, NULL)
+        i0 = instance_ptr_ne(p0, NULL)
         guard_true(i0) []
-        i1 = ptr_eq(p0, NULL)
+        i1 = instance_ptr_eq(p0, NULL)
         guard_false(i1) []
-        i2 = ptr_ne(NULL, p0)
+        i2 = instance_ptr_ne(NULL, p0)
         guard_true(i0) []
-        i3 = ptr_eq(NULL, p0)
+        i3 = instance_ptr_eq(NULL, p0)
         guard_false(i1) []
         jump(p0)
         """
@@ -2026,7 +2026,7 @@
         ops = """
         [p1]
         guard_class(p1, ConstClass(node_vtable2)) []
-        i = ptr_ne(ConstPtr(myptr), p1)
+        i = instance_ptr_ne(ConstPtr(myptr), p1)
         guard_true(i) []
         jump(p1)
         """
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -2683,7 +2683,7 @@
         ops = """
         [p1]
         guard_class(p1, ConstClass(node_vtable2)) []
-        i = ptr_ne(ConstPtr(myptr), p1)
+        i = instance_ptr_ne(ConstPtr(myptr), p1)
         guard_true(i) []
         jump(p1)
         """
@@ -3331,7 +3331,7 @@
         jump(p1, i1, i2, i6)
         '''
         self.optimize_loop(ops, expected, preamble)
-        
+
 
     # ----------
 
@@ -7280,7 +7280,7 @@
         ops = """
         [p1, p2]
         setarrayitem_gc(p1, 2, 10, descr=arraydescr)
-        setarrayitem_gc(p2, 3, 13, descr=arraydescr)        
+        setarrayitem_gc(p2, 3, 13, descr=arraydescr)
         call(0, p1, p2, 0, 0, 10, descr=arraycopydescr)
         jump(p1, p2)
         """
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -165,7 +165,7 @@
         if not we_are_translated():
             for b in registers[count:]:
                 assert not oldbox.same_box(b)
-                
+
 
     def make_result_of_lastop(self, resultbox):
         got_type = resultbox.type
@@ -199,7 +199,7 @@
                     'float_add', 'float_sub', 'float_mul', 'float_truediv',
                     'float_lt', 'float_le', 'float_eq',
                     'float_ne', 'float_gt', 'float_ge',
-                    'ptr_eq', 'ptr_ne',
+                    'ptr_eq', 'ptr_ne', 'instance_ptr_eq', 'instance_ptr_ne',
                     ]:
         exec py.code.Source('''
             @arguments("box", "box")
@@ -604,7 +604,7 @@
     opimpl_setinteriorfield_gc_i = _opimpl_setinteriorfield_gc_any
     opimpl_setinteriorfield_gc_f = _opimpl_setinteriorfield_gc_any
     opimpl_setinteriorfield_gc_r = _opimpl_setinteriorfield_gc_any
-                                
+
 
     @arguments("box", "descr")
     def _opimpl_getfield_raw_any(self, box, fielddescr):
diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py
--- a/pypy/jit/metainterp/resoperation.py
+++ b/pypy/jit/metainterp/resoperation.py
@@ -437,6 +437,8 @@
     #
     'PTR_EQ/2b',
     'PTR_NE/2b',
+    'INSTANCE_PTR_EQ/2b',
+    'INSTANCE_PTR_NE/2b',
     'CAST_OPAQUE_PTR/1b',
     #
     'ARRAYLEN_GC/1d',
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -3436,7 +3436,7 @@
         res = self.meta_interp(f, [16])
         assert res == f(16)
 
-    def test_ptr_eq_str_constants(self):
+    def test_ptr_eq(self):
         myjitdriver = JitDriver(greens = [], reds = ["n", "x"])
         class A(object):
             def __init__(self, v):
@@ -3452,6 +3452,42 @@
         res = self.meta_interp(f, [10, 1])
         assert res == 0
 
+    def test_instance_ptr_eq(self):
+        myjitdriver = JitDriver(greens = [], reds = ["n", "i", "a1", "a2"])
+        class A(object):
+            pass
+        def f(n):
+            a1 = A()
+            a2 = A()
+            i = 0
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, i=i, a1=a1, a2=a2)
+                if n % 2:
+                    a = a2
+                else:
+                    a = a1
+                i += a is a1
+                n -= 1
+            return i
+        res = self.meta_interp(f, [10])
+        assert res == f(10)
+        def f(n):
+            a1 = A()
+            a2 = A()
+            i = 0
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, i=i, a1=a1, a2=a2)
+                if n % 2:
+                    a = a2
+                else:
+                    a = a1
+                if a is a2:
+                    i += 1
+                n -= 1
+            return i
+        res = self.meta_interp(f, [10])
+        assert res == f(10)
+
     def test_virtual_array_of_structs(self):
         myjitdriver = JitDriver(greens = [], reds=["n", "d"])
         def f(n):

From noreply at buildbot.pypy.org  Tue Oct 25 16:07:27 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 25 Oct 2011 16:07:27 +0200 (CEST)
Subject: [pypy-commit] pypy lightweight-finalizers: remove traces of
	previous approach
Message-ID: <20111025140727.9D6B2820C7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: lightweight-finalizers
Changeset: r48433:0554cbf80d3c
Date: 2011-10-25 16:05 +0200
http://bitbucket.org/pypy/pypy/changeset/0554cbf80d3c/

Log:	remove traces of previous approach

diff --git a/pypy/rpython/lltypesystem/lltype.py b/pypy/rpython/lltypesystem/lltype.py
--- a/pypy/rpython/lltypesystem/lltype.py
+++ b/pypy/rpython/lltypesystem/lltype.py
@@ -363,7 +363,7 @@
         Struct._install_extras(self, **kwds)
 
     def _attach_runtime_type_info_funcptr(self, funcptr, destrptr,
-                                          customtraceptr, raw_mem_attr_name):
+                                          customtraceptr):
         if self._runtime_type_info is None:
             raise TypeError("attachRuntimeTypeInfo: %r must have been built "
                             "with the rtti=True argument" % (self,))
@@ -399,8 +399,6 @@
                 raise TypeError("expected a custom trace function "
                                 "implementation, got: %s" % customtraceptr)
             self._runtime_type_info.custom_trace_funcptr = customtraceptr
-        if raw_mem_attr_name is not None:
-            self._runtime_type_info.raw_mem_attr_name = raw_mem_attr_name
 
 class GcStruct(RttiStruct):
     _gckind = 'gc'
@@ -2060,12 +2058,11 @@
     return _ptr(PTRTYPE, oddint, solid=True)
 
 def attachRuntimeTypeInfo(GCSTRUCT, funcptr=None, destrptr=None,
-                          customtraceptr=None, raw_mem_attr_name=None):
+                          customtraceptr=None):
     if not isinstance(GCSTRUCT, RttiStruct):
         raise TypeError, "expected a RttiStruct: %s" % GCSTRUCT
     GCSTRUCT._attach_runtime_type_info_funcptr(funcptr, destrptr,
-                                               customtraceptr,
-                                               raw_mem_attr_name)
+                                               customtraceptr)
     return _ptr(Ptr(RuntimeTypeInfo), GCSTRUCT._runtime_type_info)
 
 def getRuntimeTypeInfo(GCSTRUCT):
diff --git a/pypy/rpython/rtyper.py b/pypy/rpython/rtyper.py
--- a/pypy/rpython/rtyper.py
+++ b/pypy/rpython/rtyper.py
@@ -705,7 +705,7 @@
         return self.getcallable(graph)
 
     def attachRuntimeTypeInfoFunc(self, GCSTRUCT, func, ARG_GCSTRUCT=None,
-                                  destrptr=None, raw_mem_attr_name=None):
+                                  destrptr=None):
         self.call_all_setups()  # compute ForwardReferences now
         if ARG_GCSTRUCT is None:
             ARG_GCSTRUCT = GCSTRUCT
@@ -717,8 +717,7 @@
             raise TyperError("runtime type info function %r returns %r, "
                              "excepted Ptr(RuntimeTypeInfo)" % (func, s))
         funcptr = self.getcallable(graph)
-        attachRuntimeTypeInfo(GCSTRUCT, funcptr, destrptr, None,
-                              raw_mem_attr_name)
+        attachRuntimeTypeInfo(GCSTRUCT, funcptr, destrptr, None)
 
 # register operations from annotation model
 RPythonTyper._registeroperations(annmodel)

From noreply at buildbot.pypy.org  Tue Oct 25 16:11:45 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 25 Oct 2011 16:11:45 +0200 (CEST)
Subject: [pypy-commit] pypy lightweight-finalizers: rename argument to
 malloc_fixedsize_clear to reflect what it really means
Message-ID: <20111025141145.6356B820C7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: lightweight-finalizers
Changeset: r48434:cde38a4297fc
Date: 2011-10-25 16:11 +0200
http://bitbucket.org/pypy/pypy/changeset/cde38a4297fc/

Log:	rename argument to malloc_fixedsize_clear to reflect what it really
	means (it implies has_finalizer)

diff --git a/pypy/rpython/memory/gc/generation.py b/pypy/rpython/memory/gc/generation.py
--- a/pypy/rpython/memory/gc/generation.py
+++ b/pypy/rpython/memory/gc/generation.py
@@ -168,7 +168,7 @@
 
     def malloc_fixedsize_clear(self, typeid, size,
                                has_finalizer=False,
-                               has_light_finalizer=False,
+                               is_finalizer_light=False,
                                contains_weakptr=False):
         if (has_finalizer or
             (raw_malloc_usage(size) > self.lb_young_fixedsize and
diff --git a/pypy/rpython/memory/gc/marksweep.py b/pypy/rpython/memory/gc/marksweep.py
--- a/pypy/rpython/memory/gc/marksweep.py
+++ b/pypy/rpython/memory/gc/marksweep.py
@@ -93,7 +93,7 @@
         pass
 
     def malloc_fixedsize(self, typeid16, size,
-                         has_finalizer=False, has_light_finalizer=False,
+                         has_finalizer=False, is_finalizer_light=False,
                          contains_weakptr=False):
         self.maybe_collect()
         size_gc_header = self.gcheaderbuilder.size_gc_header
@@ -130,7 +130,7 @@
 
     def malloc_fixedsize_clear(self, typeid16, size,
                                has_finalizer=False,
-                               has_light_finalizer=False,
+                               is_finalizer_light=False,
                                contains_weakptr=False):
         self.maybe_collect()
         size_gc_header = self.gcheaderbuilder.size_gc_header
diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -459,7 +459,7 @@
 
     def malloc_fixedsize_clear(self, typeid, size,
                                needs_finalizer=False,
-                               has_light_finalizer=False,
+                               is_finalizer_light=False,
                                contains_weakptr=False):
         size_gc_header = self.gcheaderbuilder.size_gc_header
         totalsize = size_gc_header + size
@@ -467,7 +467,7 @@
         #
         # If the object needs a finalizer, ask for a rawmalloc.
         # The following check should be constant-folded.
-        if needs_finalizer and not has_light_finalizer:
+        if needs_finalizer and not is_finalizer_light:
             ll_assert(not contains_weakptr,
                      "'needs_finalizer' and 'contains_weakptr' both specified")
             obj = self.external_malloc(typeid, 0, can_make_young=False)
@@ -498,7 +498,7 @@
             # Build the object.
             llarena.arena_reserve(result, totalsize)
             obj = result + size_gc_header
-            if has_light_finalizer:
+            if is_finalizer_light:
                 self.young_objects_with_light_finalizers.append(obj)
             self.init_gc_object(result, typeid, flags=0)
             #
diff --git a/pypy/rpython/memory/gc/semispace.py b/pypy/rpython/memory/gc/semispace.py
--- a/pypy/rpython/memory/gc/semispace.py
+++ b/pypy/rpython/memory/gc/semispace.py
@@ -95,7 +95,7 @@
 
     def malloc_fixedsize_clear(self, typeid16, size,
                                has_finalizer=False,
-                               has_light_finalizer=False,
+                               is_finalizer_light=False,
                                contains_weakptr=False):
         size_gc_header = self.gcheaderbuilder.size_gc_header
         totalsize = size_gc_header + size
@@ -105,7 +105,7 @@
         llarena.arena_reserve(result, totalsize)
         self.init_gc_object(result, typeid16)
         self.free = result + totalsize
-        if has_light_finalizer:
+        if is_finalizer_light:
             self.objects_with_light_finalizers.append(result + size_gc_header)
         elif has_finalizer:
             self.objects_with_finalizers.append(result + size_gc_header)
diff --git a/pypy/rpython/memory/test/test_transformed_gc.py b/pypy/rpython/memory/test/test_transformed_gc.py
--- a/pypy/rpython/memory/test/test_transformed_gc.py
+++ b/pypy/rpython/memory/test/test_transformed_gc.py
@@ -807,7 +807,7 @@
                     op.args = [Constant(type_id, llgroup.HALFWORD),
                                Constant(llmemory.sizeof(P), lltype.Signed),
                                Constant(False, lltype.Bool), # has_finalizer
-                               Constant(False, lltype.Bool), # has_light_finalizer
+                               Constant(False, lltype.Bool), # is_finalizer_light
                                Constant(False, lltype.Bool)] # contains_weakptr
                     break
             else:
@@ -844,7 +844,7 @@
                     op.args = [Constant(type_id, llgroup.HALFWORD),
                                Constant(llmemory.sizeof(P), lltype.Signed),
                                Constant(False, lltype.Bool), # has_finalizer
-                               Constant(False, lltype.Bool), # has_light_finalizer
+                               Constant(False, lltype.Bool), # is_finalizer_light
                                Constant(False, lltype.Bool)] # contains_weakptr
                     break
             else:

From noreply at buildbot.pypy.org  Tue Oct 25 16:17:12 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue, 25 Oct 2011 16:17:12 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-dicts: translation fixes for some tests,
	fix x86 tests
Message-ID: <20111025141712.5A3EB820C7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: virtual-dicts
Changeset: r48435:ef72b02f3457
Date: 2011-10-25 10:16 -0400
http://bitbucket.org/pypy/pypy/changeset/ef72b02f3457/

Log:	translation fixes for some tests, fix x86 tests

diff --git a/pypy/jit/backend/llsupport/descr.py b/pypy/jit/backend/llsupport/descr.py
--- a/pypy/jit/backend/llsupport/descr.py
+++ b/pypy/jit/backend/llsupport/descr.py
@@ -281,6 +281,9 @@
     def is_float_field(self):
         return self.fielddescr.is_float_field()
 
+    def sort_key(self):
+        return self.fielddescr.sort_key()
+
     def repr_of_descr(self):
         return '<InteriorFieldDescr %s>' % self.fielddescr.repr_of_descr()
 
diff --git a/pypy/jit/metainterp/optimizeopt/optimizer.py b/pypy/jit/metainterp/optimizeopt/optimizer.py
--- a/pypy/jit/metainterp/optimizeopt/optimizer.py
+++ b/pypy/jit/metainterp/optimizeopt/optimizer.py
@@ -209,13 +209,19 @@
     def setfield(self, ofs, value):
         raise NotImplementedError
 
+    def getlength(self):
+        raise NotImplementedError
+
     def getitem(self, index):
         raise NotImplementedError
 
-    def getlength(self):
+    def setitem(self, index, value):
         raise NotImplementedError
 
-    def setitem(self, index, value):
+    def getinteriorfield(self, index, ofs, default):
+        raise NotImplementedError
+
+    def setinteriorfield(self, index, ofs, value):
         raise NotImplementedError
 
 
@@ -283,11 +289,11 @@
             return self.optimizer.optpure.has_pure_result(opnum, args, descr)
         return False
 
-    def get_pure_result(self, key):    
+    def get_pure_result(self, key):
         if self.optimizer.optpure:
             return self.optimizer.optpure.get_pure_result(key)
         return None
-    
+
     def setup(self):
         pass
 
@@ -524,7 +530,7 @@
 
     def replace_op(self, old_op, new_op):
         # XXX: Do we want to cache indexes to prevent search?
-        i = len(self._newoperations) 
+        i = len(self._newoperations)
         while i > 0:
             i -= 1
             if self._newoperations[i] is old_op:
diff --git a/pypy/jit/metainterp/optimizeopt/virtualize.py b/pypy/jit/metainterp/optimizeopt/virtualize.py
--- a/pypy/jit/metainterp/optimizeopt/virtualize.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualize.py
@@ -280,12 +280,12 @@
     def getlength(self):
         return len(self._items)
 
-    def getinteriorfield(self, index, descr, default):
-        return self._items[index].get(descr, default)
+    def getinteriorfield(self, index, ofs, default):
+        return self._items[index].get(ofs, default)
 
-    def setinteriorfield(self, index, descr, itemvalue):
+    def setinteriorfield(self, index, ofs, itemvalue):
         assert isinstance(itemvalue, optimizer.OptValue)
-        self._items[index][descr] = itemvalue
+        self._items[index][ofs] = itemvalue
 
     def _really_force(self, optforce):
         assert self.source_op is not None
@@ -296,7 +296,9 @@
         for index in range(len(self._items)):
             for descr, value in self._items[index].iteritems():
                 subbox = value.force_box(optforce)
-                op = ResOperation(rop.SETINTERIORFIELD_GC, [box, ConstInt(index), subbox], None, descr=descr)
+                op = ResOperation(rop.SETINTERIORFIELD_GC,
+                    [box, ConstInt(index), subbox], None, descr=descr
+                )
                 optforce.emit_operation(op)
 
     def _get_list_of_descrs(self):

From noreply at buildbot.pypy.org  Tue Oct 25 16:25:55 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 25 Oct 2011 16:25:55 +0200 (CEST)
Subject: [pypy-commit] pypy lightweight-finalizers: a missed call
Message-ID: <20111025142555.018EA820C7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: lightweight-finalizers
Changeset: r48436:fadf05e6c02a
Date: 2011-10-25 16:25 +0200
http://bitbucket.org/pypy/pypy/changeset/fadf05e6c02a/

Log:	a missed call

diff --git a/pypy/rpython/memory/gc/generation.py b/pypy/rpython/memory/gc/generation.py
--- a/pypy/rpython/memory/gc/generation.py
+++ b/pypy/rpython/memory/gc/generation.py
@@ -181,6 +181,7 @@
             # "non-simple" case or object too big: don't use the nursery
             return SemiSpaceGC.malloc_fixedsize_clear(self, typeid, size,
                                                       has_finalizer,
+                                                      is_finalizer_light,
                                                       contains_weakptr)
         size_gc_header = self.gcheaderbuilder.size_gc_header
         totalsize = size_gc_header + size

From noreply at buildbot.pypy.org  Tue Oct 25 16:28:54 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 25 Oct 2011 16:28:54 +0200 (CEST)
Subject: [pypy-commit] pypy lightweight-finalizers: add a delete here
Message-ID: <20111025142854.A1BA5820C7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: lightweight-finalizers
Changeset: r48437:98da45c0aeb5
Date: 2011-10-25 16:28 +0200
http://bitbucket.org/pypy/pypy/changeset/98da45c0aeb5/

Log:	add a delete here

diff --git a/pypy/rpython/memory/gc/semispace.py b/pypy/rpython/memory/gc/semispace.py
--- a/pypy/rpython/memory/gc/semispace.py
+++ b/pypy/rpython/memory/gc/semispace.py
@@ -492,6 +492,7 @@
             else:
                 finalizer = self.getfinalizer(self.get_type_id(obj))
                 finalizer(obj, llmemory.NULL)
+        self.objects_with_light_finalizers.delete()
         self.objects_with_light_finalizers = new_objects
 
     def deal_with_objects_with_finalizers(self, scan):

From noreply at buildbot.pypy.org  Tue Oct 25 16:31:53 2011
From: noreply at buildbot.pypy.org (bivab)
Date: Tue, 25 Oct 2011 16:31:53 +0200 (CEST)
Subject: [pypy-commit] pypy arm-backend-2: add names to the functions
 generated to emit code in the assembler
Message-ID: <20111025143153.CEA8A820C7@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: arm-backend-2
Changeset: r48438:e9b9f641f506
Date: 2011-10-25 16:30 +0200
http://bitbucket.org/pypy/pypy/changeset/e9b9f641f506/

Log:	add names to the functions generated to emit code in the assembler

diff --git a/pypy/jit/backend/arm/helper/assembler.py b/pypy/jit/backend/arm/helper/assembler.py
--- a/pypy/jit/backend/arm/helper/assembler.py
+++ b/pypy/jit/backend/arm/helper/assembler.py
@@ -6,7 +6,7 @@
 from pypy.rlib.rarithmetic import r_uint, r_longlong, intmask
 from pypy.jit.metainterp.resoperation import rop
 
-def gen_emit_op_unary_cmp(true_cond, false_cond):
+def gen_emit_op_unary_cmp(name, true_cond, false_cond):
     def f(self, op, arglocs, regalloc, fcond):
         assert fcond is not None
         reg, res = arglocs
@@ -14,9 +14,10 @@
         self.mc.MOV_ri(res.value, 1, true_cond)
         self.mc.MOV_ri(res.value, 0, false_cond)
         return fcond
+    f.__name__ = 'emit_op_%s' % name
     return f
 
-def gen_emit_op_ri(opname):
+def gen_emit_op_ri(name, opname):
     ri_op = getattr(AbstractARMv7Builder, '%s_ri' % opname)
     rr_op = getattr(AbstractARMv7Builder, '%s_rr' % opname)
     def f(self, op, arglocs, regalloc, fcond):
@@ -27,9 +28,10 @@
         else:
             rr_op(self.mc, res.value, l0.value, l1.value)
         return fcond
+    f.__name__ = 'emit_op_%s' % name
     return f
 
-def gen_emit_op_by_helper_call(opname):
+def gen_emit_op_by_helper_call(name, opname):
     helper = getattr(AbstractARMv7Builder, opname)
     def f(self, op, arglocs, regalloc, fcond):
         assert fcond is not None
@@ -40,9 +42,10 @@
         with saved_registers(self.mc, regs, r.caller_vfp_resp):
             helper(self.mc, fcond)
         return fcond
+    f.__name__ = 'emit_op_%s' % name
     return f
 
-def gen_emit_cmp_op(condition):
+def gen_emit_cmp_op(name, condition):
     def f(self, op, arglocs, regalloc, fcond):
         l0, l1, res = arglocs
 
@@ -54,9 +57,10 @@
         self.mc.MOV_ri(res.value, 1, cond=condition)
         self.mc.MOV_ri(res.value, 0, cond=inv)
         return fcond
+    f.__name__ = 'emit_op_%s' % name
     return f
 
-def gen_emit_cmp_op_guard(condition):
+def gen_emit_cmp_op_guard(name, condition):
     def f(self, op, guard, arglocs, regalloc, fcond):
         l0 = arglocs[0]
         l1 = arglocs[1]
@@ -72,24 +76,27 @@
             cond = inv
         self._emit_guard(guard, arglocs[2:], cond)
         return fcond
+    f.__name__ = 'emit_guard_%s' % name
+    f.__name__ = 'emit_op_%s' % name
     return f
 
-def gen_emit_float_op(opname):
+def gen_emit_float_op(name, opname):
     op_rr = getattr(AbstractARMv7Builder, opname)
     def f(self, op, arglocs, regalloc, fcond):
         arg1, arg2, result = arglocs
         op_rr(self.mc, result.value, arg1.value, arg2.value)
         return fcond
     return f
-def gen_emit_unary_float_op(opname):
+def gen_emit_unary_float_op(name, opname):
     op_rr = getattr(AbstractARMv7Builder, opname)
     def f(self, op, arglocs, regalloc, fcond):
         arg1, result = arglocs
         op_rr(self.mc, result.value, arg1.value)
         return fcond
+    f.__name__ = 'emit_op_%s' % name
     return f
 
-def gen_emit_float_cmp_op(cond):
+def gen_emit_float_cmp_op(name, cond):
     def f(self, op, arglocs, regalloc, fcond):
         arg1, arg2, res = arglocs
         inv = c.get_opposite_of(cond)
@@ -98,9 +105,10 @@
         self.mc.MOV_ri(res.value, 1, cond=cond)
         self.mc.MOV_ri(res.value, 0, cond=inv)
         return fcond
+    f.__name__ = 'emit_op_%s' % name
     return f
 
-def gen_emit_float_cmp_op_guard(guard_cond):
+def gen_emit_float_cmp_op_guard(name, guard_cond):
     def f(self, op, guard, arglocs, regalloc, fcond):
         arg1 = arglocs[0]
         arg2 = arglocs[1]
@@ -113,6 +121,7 @@
             cond = inv
         self._emit_guard(guard, arglocs[2:], cond)
         return fcond
+    f.__name__ = 'emit_guard_%s' % name
     return f
 
 class saved_registers(object):
diff --git a/pypy/jit/backend/arm/opassembler.py b/pypy/jit/backend/arm/opassembler.py
--- a/pypy/jit/backend/arm/opassembler.py
+++ b/pypy/jit/backend/arm/opassembler.py
@@ -9,6 +9,7 @@
 
 from pypy.jit.backend.arm.helper.assembler import (gen_emit_op_by_helper_call,
                                                     gen_emit_op_unary_cmp,
+                                                    gen_emit_guard_unary_cmp,
                                                     gen_emit_op_ri,
                                                     gen_emit_cmp_op,
                                                     gen_emit_cmp_op_guard,
@@ -117,45 +118,45 @@
         self._emit_guard_overflow(guard, arglocs[3:], fcond)
         return fcond
 
-    emit_op_int_floordiv = gen_emit_op_by_helper_call('DIV')
-    emit_op_int_mod = gen_emit_op_by_helper_call('MOD')
-    emit_op_uint_floordiv = gen_emit_op_by_helper_call('UDIV')
+    emit_op_int_floordiv = gen_emit_op_by_helper_call('int_floordiv', 'DIV')
+    emit_op_int_mod = gen_emit_op_by_helper_call('int_mod', 'MOD')
+    emit_op_uint_floordiv = gen_emit_op_by_helper_call('uint_floordiv', 'UDIV')
 
-    emit_op_int_and = gen_emit_op_ri('AND')
-    emit_op_int_or = gen_emit_op_ri('ORR')
-    emit_op_int_xor = gen_emit_op_ri('EOR')
-    emit_op_int_lshift = gen_emit_op_ri('LSL')
-    emit_op_int_rshift = gen_emit_op_ri('ASR')
-    emit_op_uint_rshift = gen_emit_op_ri('LSR')
+    emit_op_int_and = gen_emit_op_ri('int_and', 'AND')
+    emit_op_int_or = gen_emit_op_ri('int_or', 'ORR')
+    emit_op_int_xor = gen_emit_op_ri('int_xor', 'EOR')
+    emit_op_int_lshift = gen_emit_op_ri('int_lshift', 'LSL')
+    emit_op_int_rshift = gen_emit_op_ri('int_rshift', 'ASR')
+    emit_op_uint_rshift = gen_emit_op_ri('uint_rshift', 'LSR')
 
-    emit_op_int_lt = gen_emit_cmp_op(c.LT)
-    emit_op_int_le = gen_emit_cmp_op(c.LE)
-    emit_op_int_eq = gen_emit_cmp_op(c.EQ)
-    emit_op_int_ne = gen_emit_cmp_op(c.NE)
-    emit_op_int_gt = gen_emit_cmp_op(c.GT)
-    emit_op_int_ge = gen_emit_cmp_op(c.GE)
+    emit_op_int_lt = gen_emit_cmp_op('int_lt', c.LT)
+    emit_op_int_le = gen_emit_cmp_op('int_le', c.LE)
+    emit_op_int_eq = gen_emit_cmp_op('int_eq', c.EQ)
+    emit_op_int_ne = gen_emit_cmp_op('int_ne', c.NE)
+    emit_op_int_gt = gen_emit_cmp_op('int_gt', c.GT)
+    emit_op_int_ge = gen_emit_cmp_op('int_ge', c.GE)
 
-    emit_op_uint_le = gen_emit_cmp_op(c.LS)
-    emit_op_uint_gt = gen_emit_cmp_op(c.HI)
+    emit_op_uint_le = gen_emit_cmp_op('uint_le', c.LS)
+    emit_op_uint_gt = gen_emit_cmp_op('uint_gt', c.HI)
 
-    emit_op_uint_lt = gen_emit_cmp_op(c.HI)
-    emit_op_uint_ge = gen_emit_cmp_op(c.LS)
+    emit_op_uint_lt = gen_emit_cmp_op('uint_lt', c.HI)
+    emit_op_uint_ge = gen_emit_cmp_op('uint_ge', c.LS)
 
     emit_op_ptr_eq = emit_op_int_eq
     emit_op_ptr_ne = emit_op_int_ne
 
-    emit_guard_int_lt = gen_emit_cmp_op_guard(c.LT)
-    emit_guard_int_le = gen_emit_cmp_op_guard(c.LE)
-    emit_guard_int_eq = gen_emit_cmp_op_guard(c.EQ)
-    emit_guard_int_ne = gen_emit_cmp_op_guard(c.NE)
-    emit_guard_int_gt = gen_emit_cmp_op_guard(c.GT)
-    emit_guard_int_ge = gen_emit_cmp_op_guard(c.GE)
+    emit_guard_int_lt = gen_emit_cmp_op_guard('int_lt', c.LT)
+    emit_guard_int_le = gen_emit_cmp_op_guard('int_le', c.LE)
+    emit_guard_int_eq = gen_emit_cmp_op_guard('int_eq', c.EQ)
+    emit_guard_int_ne = gen_emit_cmp_op_guard('int_ne', c.NE)
+    emit_guard_int_gt = gen_emit_cmp_op_guard('int_gt', c.GT)
+    emit_guard_int_ge = gen_emit_cmp_op_guard('int_ge', c.GE)
 
-    emit_guard_uint_le = gen_emit_cmp_op_guard(c.LS)
-    emit_guard_uint_gt = gen_emit_cmp_op_guard(c.HI)
+    emit_guard_uint_le = gen_emit_cmp_op_guard('uint_le', c.LS)
+    emit_guard_uint_gt = gen_emit_cmp_op_guard('uint_gt', c.HI)
 
-    emit_guard_uint_lt = gen_emit_cmp_op_guard(c.HI)
-    emit_guard_uint_ge = gen_emit_cmp_op_guard(c.LS)
+    emit_guard_uint_lt = gen_emit_cmp_op_guard('uint_lt', c.HI)
+    emit_guard_uint_ge = gen_emit_cmp_op_guard('uint_ge', c.LS)
 
     emit_guard_ptr_eq = emit_guard_int_eq
     emit_guard_ptr_ne = emit_guard_int_ne
@@ -169,8 +170,8 @@
 
     _mixin_ = True
 
-    emit_op_int_is_true = gen_emit_op_unary_cmp(c.NE, c.EQ)
-    emit_op_int_is_zero = gen_emit_op_unary_cmp(c.EQ, c.NE)
+    emit_op_int_is_true = gen_emit_op_unary_cmp('int_is_true', c.NE, c.EQ)
+    emit_op_int_is_zero = gen_emit_op_unary_cmp('int_is_zero', c.EQ, c.NE)
 
     def emit_op_int_invert(self, op, arglocs, regalloc, fcond):
         reg, res = arglocs
@@ -1142,28 +1143,28 @@
 class FloatOpAssemlber(object):
     _mixin_ = True
 
-    emit_op_float_add = gen_emit_float_op('VADD')
-    emit_op_float_sub = gen_emit_float_op('VSUB')
-    emit_op_float_mul = gen_emit_float_op('VMUL')
-    emit_op_float_truediv = gen_emit_float_op('VDIV')
+    emit_op_float_add = gen_emit_float_op('float_add', 'VADD')
+    emit_op_float_sub = gen_emit_float_op('float_sub', 'VSUB')
+    emit_op_float_mul = gen_emit_float_op('float_mul', 'VMUL')
+    emit_op_float_truediv = gen_emit_float_op('float_truediv', 'VDIV')
 
-    emit_op_float_neg = gen_emit_unary_float_op('VNEG')
-    emit_op_float_abs = gen_emit_unary_float_op('VABS')
-    emit_op_math_sqrt = gen_emit_unary_float_op('VSQRT')
+    emit_op_float_neg = gen_emit_unary_float_op('float_neg', 'VNEG')
+    emit_op_float_abs = gen_emit_unary_float_op('float_abs', 'VABS')
+    emit_op_math_sqrt = gen_emit_unary_float_op('math_sqrt', 'VSQRT')
 
-    emit_op_float_lt = gen_emit_float_cmp_op(c.VFP_LT)
-    emit_op_float_le = gen_emit_float_cmp_op(c.VFP_LE)
-    emit_op_float_eq = gen_emit_float_cmp_op(c.EQ)
-    emit_op_float_ne = gen_emit_float_cmp_op(c.NE)
-    emit_op_float_gt = gen_emit_float_cmp_op(c.GT)
-    emit_op_float_ge = gen_emit_float_cmp_op(c.GE)
+    emit_op_float_lt = gen_emit_float_cmp_op('float_lt', c.VFP_LT)
+    emit_op_float_le = gen_emit_float_cmp_op('float_le', c.VFP_LE)
+    emit_op_float_eq = gen_emit_float_cmp_op('float_eq', c.EQ)
+    emit_op_float_ne = gen_emit_float_cmp_op('float_ne', c.NE)
+    emit_op_float_gt = gen_emit_float_cmp_op('float_gt', c.GT)
+    emit_op_float_ge = gen_emit_float_cmp_op('float_ge', c.GE)
 
-    emit_guard_float_lt = gen_emit_float_cmp_op_guard(c.VFP_LT)
-    emit_guard_float_le = gen_emit_float_cmp_op_guard(c.VFP_LE)
-    emit_guard_float_eq = gen_emit_float_cmp_op_guard(c.EQ)
-    emit_guard_float_ne = gen_emit_float_cmp_op_guard(c.NE)
-    emit_guard_float_gt = gen_emit_float_cmp_op_guard(c.GT)
-    emit_guard_float_ge = gen_emit_float_cmp_op_guard(c.GE)
+    emit_guard_float_lt = gen_emit_float_cmp_op_guard('float_lt', c.VFP_LT)
+    emit_guard_float_le = gen_emit_float_cmp_op_guard('float_le', c.VFP_LE)
+    emit_guard_float_eq = gen_emit_float_cmp_op_guard('float_eq', c.EQ)
+    emit_guard_float_ne = gen_emit_float_cmp_op_guard('float_ne', c.NE)
+    emit_guard_float_gt = gen_emit_float_cmp_op_guard('float_gt', c.GT)
+    emit_guard_float_ge = gen_emit_float_cmp_op_guard('float_ge', c.GE)
 
     def emit_op_cast_float_to_int(self, op, arglocs, regalloc, fcond):
         arg, temp, res = arglocs

From noreply at buildbot.pypy.org  Tue Oct 25 16:31:55 2011
From: noreply at buildbot.pypy.org (bivab)
Date: Tue, 25 Oct 2011 16:31:55 +0200 (CEST)
Subject: [pypy-commit] pypy arm-backend-2: add functions to merge unary cmp
 operatios with guards
Message-ID: <20111025143155.074C0820C7@wyvern.cs.uni-duesseldorf.de>

Author: David Schneider <david.schneider at picle.org>
Branch: arm-backend-2
Changeset: r48439:3c00e205750e
Date: 2011-10-25 16:31 +0200
http://bitbucket.org/pypy/pypy/changeset/3c00e205750e/

Log:	add functions to merge unary cmp operatios with guards

diff --git a/pypy/jit/backend/arm/helper/assembler.py b/pypy/jit/backend/arm/helper/assembler.py
--- a/pypy/jit/backend/arm/helper/assembler.py
+++ b/pypy/jit/backend/arm/helper/assembler.py
@@ -17,6 +17,21 @@
     f.__name__ = 'emit_op_%s' % name
     return f
 
+def gen_emit_guard_unary_cmp(name, true_cond, false_cond):
+    def f(self, op, guard, arglocs, regalloc, fcond):
+        assert fcond is not None
+        assert guard is not None
+        reg = arglocs[0]
+        self.mc.CMP_ri(reg.value, 0)
+        cond = true_cond
+        guard_opnum = guard.getopnum()
+        if guard_opnum == rop.GUARD_FALSE:
+            cond = false_cond
+        self._emit_guard(guard, arglocs[1:], cond)
+        return fcond
+    f.__name__ = 'emit_guard_%s' % name
+    return f
+
 def gen_emit_op_ri(name, opname):
     ri_op = getattr(AbstractARMv7Builder, '%s_ri' % opname)
     rr_op = getattr(AbstractARMv7Builder, '%s_rr' % opname)
@@ -77,7 +92,6 @@
         self._emit_guard(guard, arglocs[2:], cond)
         return fcond
     f.__name__ = 'emit_guard_%s' % name
-    f.__name__ = 'emit_op_%s' % name
     return f
 
 def gen_emit_float_op(name, opname):
@@ -86,6 +100,7 @@
         arg1, arg2, result = arglocs
         op_rr(self.mc, result.value, arg1.value, arg2.value)
         return fcond
+    f.__name__ = 'emit_op_%s' % name
     return f
 def gen_emit_unary_float_op(name, opname):
     op_rr = getattr(AbstractARMv7Builder, opname)
diff --git a/pypy/jit/backend/arm/helper/regalloc.py b/pypy/jit/backend/arm/helper/regalloc.py
--- a/pypy/jit/backend/arm/helper/regalloc.py
+++ b/pypy/jit/backend/arm/helper/regalloc.py
@@ -15,18 +15,6 @@
         return i <= size and lower_bound
     return False
 
-def prepare_op_unary_cmp(name=None):
-    def f(self, op, fcond):
-        assert fcond is not None
-        a0 = op.getarg(0)
-        reg, box = self._ensure_value_is_boxed(a0)
-        res = self.force_allocate_reg(op.result, [box])
-        self.possibly_free_vars([a0, box, op.result])
-        return [reg, res]
-    if name:
-        f.__name__ = name
-    return f
-
 def prepare_op_ri(name=None, imm_size=0xFF, commutative=True, allow_zero=True):
     def f(self, op, fcond):
         assert fcond is not None
@@ -146,3 +134,20 @@
     if name:
         f.__name__ = name
     return f
+
+def prepare_op_unary_cmp(name=None):
+    def f(self, op, guard_op, fcond):
+        assert fcond is not None
+        a0 = op.getarg(0)
+        reg, box = self._ensure_value_is_boxed(a0)
+        if guard_op is None:
+            res = self.force_allocate_reg(op.result, [box])
+            self.possibly_free_vars([a0, box, op.result])
+            return [reg, res]
+        else:
+            args = self._prepare_guard(guard_op, [reg])
+            self.possibly_free_vars(guard_op.getfailargs())
+            return args
+    if name:
+        f.__name__ = name
+    return f
diff --git a/pypy/jit/backend/arm/opassembler.py b/pypy/jit/backend/arm/opassembler.py
--- a/pypy/jit/backend/arm/opassembler.py
+++ b/pypy/jit/backend/arm/opassembler.py
@@ -173,6 +173,9 @@
     emit_op_int_is_true = gen_emit_op_unary_cmp('int_is_true', c.NE, c.EQ)
     emit_op_int_is_zero = gen_emit_op_unary_cmp('int_is_zero', c.EQ, c.NE)
 
+    emit_guard_int_is_true = gen_emit_guard_unary_cmp('int_is_true', c.NE, c.EQ)
+    emit_guard_int_is_zero = gen_emit_guard_unary_cmp('int_is_zero', c.EQ, c.NE)
+
     def emit_op_int_invert(self, op, arglocs, regalloc, fcond):
         reg, res = arglocs
 
diff --git a/pypy/jit/backend/arm/regalloc.py b/pypy/jit/backend/arm/regalloc.py
--- a/pypy/jit/backend/arm/regalloc.py
+++ b/pypy/jit/backend/arm/regalloc.py
@@ -451,6 +451,9 @@
     prepare_op_int_is_true = prepare_op_unary_cmp('int_is_true')
     prepare_op_int_is_zero = prepare_op_unary_cmp('int_is_zero')
 
+    prepare_guard_int_is_true = prepare_op_unary_cmp('int_is_true')
+    prepare_guard_int_is_zero = prepare_op_unary_cmp('int_is_zero')
+
     def prepare_op_int_neg(self, op, fcond):
         l0, box = self._ensure_value_is_boxed(op.getarg(0))
         self.possibly_free_var(box)

From noreply at buildbot.pypy.org  Tue Oct 25 16:35:33 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 25 Oct 2011 16:35:33 +0200 (CEST)
Subject: [pypy-commit] pypy lightweight-finalizers: remove unused imports
Message-ID: <20111025143533.BC268820C7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: lightweight-finalizers
Changeset: r48440:e916dec7191d
Date: 2011-10-25 16:35 +0200
http://bitbucket.org/pypy/pypy/changeset/e916dec7191d/

Log:	remove unused imports

diff --git a/pypy/rpython/test/test_rclass.py b/pypy/rpython/test/test_rclass.py
--- a/pypy/rpython/test/test_rclass.py
+++ b/pypy/rpython/test/test_rclass.py
@@ -3,8 +3,7 @@
 from pypy.translator.translator import TranslationContext, graphof
 from pypy.rpython.lltypesystem.lltype import *
 from pypy.rpython.ootypesystem import ootype
-from pypy.rlib.rarithmetic import intmask, r_longlong
-from pypy.rlib import rgc
+from pypy.rlib.rarithmetic import r_longlong
 from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin
 from pypy.rpython.rclass import IR_IMMUTABLE, IR_IMMUTABLE_ARRAY
 from pypy.rpython.rclass import IR_QUASIIMMUTABLE, IR_QUASIIMMUTABLE_ARRAY

From noreply at buildbot.pypy.org  Tue Oct 25 16:54:16 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 25 Oct 2011 16:54:16 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: add myself
Message-ID: <20111025145416.61425820C7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: extradoc
Changeset: r3938:1fc0097464a4
Date: 2011-10-25 16:54 +0200
http://bitbucket.org/pypy/extradoc/changeset/1fc0097464a4/

Log:	add myself

diff --git a/sprintinfo/gothenburg-2011-2/people.txt b/sprintinfo/gothenburg-2011-2/people.txt
--- a/sprintinfo/gothenburg-2011-2/people.txt
+++ b/sprintinfo/gothenburg-2011-2/people.txt
@@ -11,6 +11,7 @@
 Jacob Hallen         lives there
 Laura Creighton      lives there
 Armin Rigo           ?              ?
+Maciej Fija&#322;kowski   2-8 Nov        ?                     vegetarian
 Antonio Cuni         4-10 november  Laura's and Jacob's
 H&#229;kan Ard&#246;           3-8            ?
 Sam Lade             1-9            Hotel Poseidon

From noreply at buildbot.pypy.org  Tue Oct 25 16:59:35 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 25 Oct 2011 16:59:35 +0200 (CEST)
Subject: [pypy-commit] pypy lightweight-finalizers: remove track_alloc_stop
 support, not needed any more
Message-ID: <20111025145935.A6C17820C7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: lightweight-finalizers
Changeset: r48441:d80d4e3f457a
Date: 2011-10-25 16:58 +0200
http://bitbucket.org/pypy/pypy/changeset/d80d4e3f457a/

Log:	remove track_alloc_stop support, not needed any more

diff --git a/pypy/rpython/lltypesystem/lloperation.py b/pypy/rpython/lltypesystem/lloperation.py
--- a/pypy/rpython/lltypesystem/lloperation.py
+++ b/pypy/rpython/lltypesystem/lloperation.py
@@ -400,7 +400,7 @@
     'raw_store':            LLOp(),
     'stack_malloc':         LLOp(), # mmh
     'track_alloc_start':    LLOp(),
-    'track_alloc_stop':     LLOp(canrun=True),
+    'track_alloc_stop':     LLOp(),
     'adr_add':              LLOp(canfold=True),
     'adr_sub':              LLOp(canfold=True),
     'adr_delta':            LLOp(canfold=True),
diff --git a/pypy/rpython/lltypesystem/opimpl.py b/pypy/rpython/lltypesystem/opimpl.py
--- a/pypy/rpython/lltypesystem/opimpl.py
+++ b/pypy/rpython/lltypesystem/opimpl.py
@@ -4,7 +4,6 @@
 from pypy.rpython.lltypesystem import lltype, llmemory
 from pypy.rpython.lltypesystem.lloperation import opimpls
 from pypy.rlib import debug
-from pypy.tool import leakfinder
 
 # ____________________________________________________________
 # Implementation of the 'canfold' operations
@@ -599,9 +598,6 @@
     from pypy.rlib.rtimer import read_timestamp
     return read_timestamp()
 
-def op_track_alloc_stop(addr):
-    leakfinder.remember_free(addr.ptr._obj)
-
 # ____________________________________________________________
 
 def get_op_impl(opname):

From noreply at buildbot.pypy.org  Tue Oct 25 16:59:36 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 25 Oct 2011 16:59:36 +0200 (CEST)
Subject: [pypy-commit] pypy lightweight-finalizers: Use a list instead of
 checking each time the type id data
Message-ID: <20111025145936.D4263820C7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: lightweight-finalizers
Changeset: r48442:e09c7b3e1296
Date: 2011-10-25 16:59 +0200
http://bitbucket.org/pypy/pypy/changeset/e09c7b3e1296/

Log:	Use a list instead of checking each time the type id data

diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -291,6 +291,7 @@
         # A list of all objects with finalizers (these are never young).
         self.objects_with_finalizers = self.AddressDeque()
         self.young_objects_with_light_finalizers = self.AddressStack()
+        self.old_objects_with_light_finalizers = self.AddressStack()
         #
         # Two lists of the objects with weakrefs.  No weakref can be an
         # old object weakly pointing to a young object: indeed, weakrefs
@@ -1588,6 +1589,9 @@
         # Weakref support: clear the weak pointers to dying objects
         if self.old_objects_with_weakrefs.non_empty():
             self.invalidate_old_weakrefs()
+        if self.old_objects_with_light_finalizers.non_empty():
+            self.deal_with_old_objects_with_finalizers()
+
         #
         # Walk all rawmalloced objects and free the ones that don't
         # have the GCFLAG_VISITED flag.
@@ -1653,11 +1657,7 @@
         if self.header(obj).tid & GCFLAG_VISITED:
             self.header(obj).tid &= ~GCFLAG_VISITED
             return False     # survives
-        else:
-            finalizer = self.getlightfinalizer(self.get_type_id(obj))
-            if finalizer:
-                finalizer(obj, llmemory.NULL)
-            return True      # dies
+        return True      # dies
 
     def _reset_gcflag_visited(self, obj, ignored):
         self.header(obj).tid &= ~GCFLAG_VISITED
@@ -1670,9 +1670,6 @@
             size_gc_header = self.gcheaderbuilder.size_gc_header
             totalsize = size_gc_header + self.get_size(obj)
             allocsize = raw_malloc_usage(totalsize)
-            finalizer = self.getlightfinalizer(self.get_type_id(obj))
-            if finalizer:
-                finalizer(obj, llmemory.NULL)            
             arena = llarena.getfakearenaaddress(obj - size_gc_header)
             #
             # Must also include the card marker area, if any
@@ -1852,6 +1849,26 @@
                 ll_assert(bool(finalizer), "no light finalizer found")
                 finalizer(obj, llmemory.NULL)
 
+    def deal_with_old_objects_with_finalizers(self):
+        """ This is a much simpler version of dealing with finalizers
+        and an optimization - we can reasonably assume that those finalizers
+        don't do anything fancy and *just* call them. Among other things
+        they won't resurrect objects
+        """
+        new_objects = self.AddressStack()
+        while self.old_objects_with_light_finalizers.non_empty():
+            obj = self.old_objects_with_light_finalizers.pop()
+            if self.header(obj).tid & GCFLAG_VISITED:
+                # surviving
+                new_objects.append(obj)
+            else:
+                # dying
+                finalizer = self.getlightfinalizer(self.get_type_id(obj))
+                ll_assert(bool(finalizer), "no light finalizer found")
+                finalizer(obj, llmemory.NULL)
+        self.old_objects_with_light_finalizers.delete()
+        self.old_objects_with_light_finalizers = new_objects
+
     def deal_with_objects_with_finalizers(self):
         # Walk over list of objects with finalizers.
         # If it is not surviving, add it to the list of to-be-called

From noreply at buildbot.pypy.org  Tue Oct 25 17:02:08 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 25 Oct 2011 17:02:08 +0200 (CEST)
Subject: [pypy-commit] pypy lightweight-finalizers: close merged branch
Message-ID: <20111025150208.51FA5820C7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: lightweight-finalizers
Changeset: r48443:630cc8b2cb4c
Date: 2011-10-25 17:00 +0200
http://bitbucket.org/pypy/pypy/changeset/630cc8b2cb4c/

Log:	close merged branch


From noreply at buildbot.pypy.org  Tue Oct 25 17:02:09 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 25 Oct 2011 17:02:09 +0200 (CEST)
Subject: [pypy-commit] pypy default: (fijal,
 arigo reviewing) Merge lightweight-finalizers branch. This branch
Message-ID: <20111025150209.CEFE9820C7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r48444:b3d0ebe10a5d
Date: 2011-10-25 17:01 +0200
http://bitbucket.org/pypy/pypy/changeset/b3d0ebe10a5d/

Log:	(fijal, arigo reviewing) Merge lightweight-finalizers branch. This
	branch adds a possitibility for a finalizer to be "lightweight".
	This means it won't resurrect an object or call arbitrary code. Such
	finalizers are detected automatically and can be dealt with more
	efficiently (such objects can live in the nursery for example).

diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py
--- a/pypy/jit/backend/llsupport/gc.py
+++ b/pypy/jit/backend/llsupport/gc.py
@@ -649,10 +649,13 @@
         def malloc_basic(size, tid):
             type_id = llop.extract_ushort(llgroup.HALFWORD, tid)
             has_finalizer = bool(tid & (1<<llgroup.HALFSHIFT))
+            has_light_finalizer = bool(tid & (1<<(llgroup.HALFSHIFT + 1)))
             check_typeid(type_id)
             res = llop1.do_malloc_fixedsize_clear(llmemory.GCREF,
                                                   type_id, size,
-                                                  has_finalizer, False)
+                                                  has_finalizer,
+                                                  has_light_finalizer,
+                                                  False)
             # In case the operation above failed, we are returning NULL
             # from this function to assembler.  There is also an RPython
             # exception set, typically MemoryError; but it's easier and
@@ -723,7 +726,7 @@
             # also use it to allocate varsized objects.  The tid
             # and possibly the length are both set afterward.
             gcref = llop1.do_malloc_fixedsize_clear(llmemory.GCREF,
-                                        0, size, False, False)
+                                        0, size, False, False, False)
             return rffi.cast(lltype.Signed, gcref)
         self.malloc_slowpath = malloc_slowpath
         self.MALLOC_SLOWPATH = lltype.FuncType([lltype.Signed], lltype.Signed)
@@ -747,7 +750,9 @@
         type_id = self.layoutbuilder.get_type_id(S)
         assert not self.layoutbuilder.is_weakref_type(S)
         has_finalizer = bool(self.layoutbuilder.has_finalizer(S))
-        flags = int(has_finalizer) << llgroup.HALFSHIFT
+        has_light_finalizer = bool(self.layoutbuilder.has_light_finalizer(S))
+        flags = (int(has_finalizer) << llgroup.HALFSHIFT |
+                 int(has_light_finalizer) << (llgroup.HALFSHIFT + 1))
         descr.tid = llop.combine_ushort(lltype.Signed, type_id, flags)
 
     def init_array_descr(self, A, descr):
diff --git a/pypy/jit/backend/llsupport/test/test_gc.py b/pypy/jit/backend/llsupport/test/test_gc.py
--- a/pypy/jit/backend/llsupport/test/test_gc.py
+++ b/pypy/jit/backend/llsupport/test/test_gc.py
@@ -247,11 +247,12 @@
         self.record = []
 
     def do_malloc_fixedsize_clear(self, RESTYPE, type_id, size,
-                                  has_finalizer, contains_weakptr):
+                                  has_finalizer, has_light_finalizer,
+                                  contains_weakptr):
         assert not contains_weakptr
         p = llmemory.raw_malloc(size)
         p = llmemory.cast_adr_to_ptr(p, RESTYPE)
-        flags = int(has_finalizer) << 16
+        flags = (int(has_finalizer) << 16) | (int(has_light_finalizer) << 17)
         tid = llop.combine_ushort(lltype.Signed, type_id, flags)
         self.record.append(("fixedsize", repr(size), tid, p))
         return p
diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -19,7 +19,7 @@
 class W_RSocket(Wrappable, RSocket):
     def __del__(self):
         self.clear_all_weakrefs()
-        self.close()
+        RSocket.__del__(self)
 
     def accept_w(self, space):
         """accept() -> (socket object, address info)
diff --git a/pypy/rlib/rmmap.py b/pypy/rlib/rmmap.py
--- a/pypy/rlib/rmmap.py
+++ b/pypy/rlib/rmmap.py
@@ -292,6 +292,9 @@
         elif _POSIX:
             self.closed = True
             if self.fd != -1:
+                # XXX this is buggy - raising in an RPython del is not a good
+                #     idea, we should swallow the exception or ignore the
+                #     underlaying close error code
                 os.close(self.fd)
                 self.fd = -1
             if self.size > 0:
diff --git a/pypy/rlib/rsocket.py b/pypy/rlib/rsocket.py
--- a/pypy/rlib/rsocket.py
+++ b/pypy/rlib/rsocket.py
@@ -56,6 +56,7 @@
 
 
 _FAMILIES = {}
+
 class Address(object):
     """The base class for RPython-level objects representing addresses.
     Fields:  addr    - a _c.sockaddr_ptr (memory owned by the Address instance)
@@ -77,9 +78,8 @@
         self.addrlen = addrlen
 
     def __del__(self):
-        addr = self.addr_p
-        if addr:
-            lltype.free(addr, flavor='raw')
+        if self.addr_p:
+            lltype.free(self.addr_p, flavor='raw')
 
     def setdata(self, addr, addrlen):
         # initialize self.addr and self.addrlen.  'addr' can be a different
@@ -613,7 +613,10 @@
         self.timeout = defaults.timeout
         
     def __del__(self):
-        self.close()
+        fd = self.fd
+        if fd != _c.INVALID_SOCKET:
+            self.fd = _c.INVALID_SOCKET
+            _c.socketclose(fd)
 
     if hasattr(_c, 'fcntl'):
         def _setblocking(self, block):
diff --git a/pypy/rpython/memory/gc/base.py b/pypy/rpython/memory/gc/base.py
--- a/pypy/rpython/memory/gc/base.py
+++ b/pypy/rpython/memory/gc/base.py
@@ -1,4 +1,5 @@
-from pypy.rpython.lltypesystem import lltype, llmemory, llarena
+from pypy.rpython.lltypesystem import lltype, llmemory, llarena, rffi
+from pypy.rpython.lltypesystem.lloperation import llop
 from pypy.rlib.debug import ll_assert
 from pypy.rpython.memory.gcheader import GCHeaderBuilder
 from pypy.rpython.memory.support import DEFAULT_CHUNK_SIZE
@@ -62,6 +63,7 @@
     def set_query_functions(self, is_varsize, has_gcptr_in_varsize,
                             is_gcarrayofgcptr,
                             getfinalizer,
+                            getlightfinalizer,
                             offsets_to_gc_pointers,
                             fixed_size, varsize_item_sizes,
                             varsize_offset_to_variable_part,
@@ -74,6 +76,7 @@
                             get_custom_trace,
                             fast_path_tracing):
         self.getfinalizer = getfinalizer
+        self.getlightfinalizer = getlightfinalizer
         self.is_varsize = is_varsize
         self.has_gcptr_in_varsize = has_gcptr_in_varsize
         self.is_gcarrayofgcptr = is_gcarrayofgcptr
@@ -139,6 +142,7 @@
 
         size = self.fixed_size(typeid)
         needs_finalizer = bool(self.getfinalizer(typeid))
+        finalizer_is_light = bool(self.getlightfinalizer(typeid))
         contains_weakptr = self.weakpointer_offset(typeid) >= 0
         assert not (needs_finalizer and contains_weakptr)
         if self.is_varsize(typeid):
@@ -158,6 +162,7 @@
             else:
                 malloc_fixedsize = self.malloc_fixedsize
             ref = malloc_fixedsize(typeid, size, needs_finalizer,
+                                   finalizer_is_light,
                                    contains_weakptr)
         # lots of cast and reverse-cast around...
         return llmemory.cast_ptr_to_adr(ref)
diff --git a/pypy/rpython/memory/gc/generation.py b/pypy/rpython/memory/gc/generation.py
--- a/pypy/rpython/memory/gc/generation.py
+++ b/pypy/rpython/memory/gc/generation.py
@@ -167,7 +167,9 @@
         return self.nursery <= addr < self.nursery_top
 
     def malloc_fixedsize_clear(self, typeid, size,
-                               has_finalizer=False, contains_weakptr=False):
+                               has_finalizer=False,
+                               is_finalizer_light=False,
+                               contains_weakptr=False):
         if (has_finalizer or
             (raw_malloc_usage(size) > self.lb_young_fixedsize and
              raw_malloc_usage(size) > self.largest_young_fixedsize)):
@@ -179,6 +181,7 @@
             # "non-simple" case or object too big: don't use the nursery
             return SemiSpaceGC.malloc_fixedsize_clear(self, typeid, size,
                                                       has_finalizer,
+                                                      is_finalizer_light,
                                                       contains_weakptr)
         size_gc_header = self.gcheaderbuilder.size_gc_header
         totalsize = size_gc_header + size
diff --git a/pypy/rpython/memory/gc/marksweep.py b/pypy/rpython/memory/gc/marksweep.py
--- a/pypy/rpython/memory/gc/marksweep.py
+++ b/pypy/rpython/memory/gc/marksweep.py
@@ -93,7 +93,8 @@
         pass
 
     def malloc_fixedsize(self, typeid16, size,
-                         has_finalizer=False, contains_weakptr=False):
+                         has_finalizer=False, is_finalizer_light=False,
+                         contains_weakptr=False):
         self.maybe_collect()
         size_gc_header = self.gcheaderbuilder.size_gc_header
         try:
@@ -128,7 +129,9 @@
     malloc_fixedsize._dont_inline_ = True
 
     def malloc_fixedsize_clear(self, typeid16, size,
-                               has_finalizer=False, contains_weakptr=False):
+                               has_finalizer=False,
+                               is_finalizer_light=False,
+                               contains_weakptr=False):
         self.maybe_collect()
         size_gc_header = self.gcheaderbuilder.size_gc_header
         try:
diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -290,6 +290,8 @@
         #
         # A list of all objects with finalizers (these are never young).
         self.objects_with_finalizers = self.AddressDeque()
+        self.young_objects_with_light_finalizers = self.AddressStack()
+        self.old_objects_with_light_finalizers = self.AddressStack()
         #
         # Two lists of the objects with weakrefs.  No weakref can be an
         # old object weakly pointing to a young object: indeed, weakrefs
@@ -457,14 +459,16 @@
 
 
     def malloc_fixedsize_clear(self, typeid, size,
-                               needs_finalizer=False, contains_weakptr=False):
+                               needs_finalizer=False,
+                               is_finalizer_light=False,
+                               contains_weakptr=False):
         size_gc_header = self.gcheaderbuilder.size_gc_header
         totalsize = size_gc_header + size
         rawtotalsize = raw_malloc_usage(totalsize)
         #
         # If the object needs a finalizer, ask for a rawmalloc.
         # The following check should be constant-folded.
-        if needs_finalizer:
+        if needs_finalizer and not is_finalizer_light:
             ll_assert(not contains_weakptr,
                      "'needs_finalizer' and 'contains_weakptr' both specified")
             obj = self.external_malloc(typeid, 0, can_make_young=False)
@@ -494,13 +498,14 @@
             #
             # Build the object.
             llarena.arena_reserve(result, totalsize)
+            obj = result + size_gc_header
+            if is_finalizer_light:
+                self.young_objects_with_light_finalizers.append(obj)
             self.init_gc_object(result, typeid, flags=0)
             #
             # If it is a weakref, record it (check constant-folded).
             if contains_weakptr:
-                self.young_objects_with_weakrefs.append(result+size_gc_header)
-            #
-            obj = result + size_gc_header
+                self.young_objects_with_weakrefs.append(obj)
         #
         return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
 
@@ -1264,6 +1269,8 @@
         # weakrefs' targets.
         if self.young_objects_with_weakrefs.non_empty():
             self.invalidate_young_weakrefs()
+        if self.young_objects_with_light_finalizers.non_empty():
+            self.deal_with_young_objects_with_finalizers()
         #
         # Clear this mapping.
         if self.nursery_objects_shadows.length() > 0:
@@ -1584,6 +1591,9 @@
         # Weakref support: clear the weak pointers to dying objects
         if self.old_objects_with_weakrefs.non_empty():
             self.invalidate_old_weakrefs()
+        if self.old_objects_with_light_finalizers.non_empty():
+            self.deal_with_old_objects_with_finalizers()
+
         #
         # Walk all rawmalloced objects and free the ones that don't
         # have the GCFLAG_VISITED flag.
@@ -1649,8 +1659,7 @@
         if self.header(obj).tid & GCFLAG_VISITED:
             self.header(obj).tid &= ~GCFLAG_VISITED
             return False     # survives
-        else:
-            return True      # dies
+        return True      # dies
 
     def _reset_gcflag_visited(self, obj, ignored):
         self.header(obj).tid &= ~GCFLAG_VISITED
@@ -1829,6 +1838,39 @@
     # ----------
     # Finalizers
 
+    def deal_with_young_objects_with_finalizers(self):
+        """ This is a much simpler version of dealing with finalizers
+        and an optimization - we can reasonably assume that those finalizers
+        don't do anything fancy and *just* call them. Among other things
+        they won't resurrect objects
+        """
+        while self.young_objects_with_light_finalizers.non_empty():
+            obj = self.young_objects_with_light_finalizers.pop()
+            if not self.is_forwarded(obj):
+                finalizer = self.getlightfinalizer(self.get_type_id(obj))
+                ll_assert(bool(finalizer), "no light finalizer found")
+                finalizer(obj, llmemory.NULL)
+
+    def deal_with_old_objects_with_finalizers(self):
+        """ This is a much simpler version of dealing with finalizers
+        and an optimization - we can reasonably assume that those finalizers
+        don't do anything fancy and *just* call them. Among other things
+        they won't resurrect objects
+        """
+        new_objects = self.AddressStack()
+        while self.old_objects_with_light_finalizers.non_empty():
+            obj = self.old_objects_with_light_finalizers.pop()
+            if self.header(obj).tid & GCFLAG_VISITED:
+                # surviving
+                new_objects.append(obj)
+            else:
+                # dying
+                finalizer = self.getlightfinalizer(self.get_type_id(obj))
+                ll_assert(bool(finalizer), "no light finalizer found")
+                finalizer(obj, llmemory.NULL)
+        self.old_objects_with_light_finalizers.delete()
+        self.old_objects_with_light_finalizers = new_objects
+
     def deal_with_objects_with_finalizers(self):
         # Walk over list of objects with finalizers.
         # If it is not surviving, add it to the list of to-be-called
@@ -1959,7 +2001,6 @@
             #
             self.old_objects_with_weakrefs.append(obj)
 
-
     def invalidate_old_weakrefs(self):
         """Called during a major collection."""
         # walk over list of objects that contain weakrefs
diff --git a/pypy/rpython/memory/gc/semispace.py b/pypy/rpython/memory/gc/semispace.py
--- a/pypy/rpython/memory/gc/semispace.py
+++ b/pypy/rpython/memory/gc/semispace.py
@@ -82,6 +82,7 @@
         self.free = self.tospace
         MovingGCBase.setup(self)
         self.objects_with_finalizers = self.AddressDeque()
+        self.objects_with_light_finalizers = self.AddressStack()
         self.objects_with_weakrefs = self.AddressStack()
 
     def _teardown(self):
@@ -93,7 +94,9 @@
     # because the spaces are filled with zeroes in advance.
 
     def malloc_fixedsize_clear(self, typeid16, size,
-                               has_finalizer=False, contains_weakptr=False):
+                               has_finalizer=False,
+                               is_finalizer_light=False,
+                               contains_weakptr=False):
         size_gc_header = self.gcheaderbuilder.size_gc_header
         totalsize = size_gc_header + size
         result = self.free
@@ -102,7 +105,9 @@
         llarena.arena_reserve(result, totalsize)
         self.init_gc_object(result, typeid16)
         self.free = result + totalsize
-        if has_finalizer:
+        if is_finalizer_light:
+            self.objects_with_light_finalizers.append(result + size_gc_header)
+        elif has_finalizer:
             self.objects_with_finalizers.append(result + size_gc_header)
         if contains_weakptr:
             self.objects_with_weakrefs.append(result + size_gc_header)
@@ -263,6 +268,8 @@
         if self.run_finalizers.non_empty():
             self.update_run_finalizers()
         scan = self.scan_copied(scan)
+        if self.objects_with_light_finalizers.non_empty():
+            self.deal_with_objects_with_light_finalizers()
         if self.objects_with_finalizers.non_empty():
             scan = self.deal_with_objects_with_finalizers(scan)
         if self.objects_with_weakrefs.non_empty():
@@ -471,6 +478,23 @@
         # immortal objects always have GCFLAG_FORWARDED set;
         # see get_forwarding_address().
 
+    def deal_with_objects_with_light_finalizers(self):
+        """ This is a much simpler version of dealing with finalizers
+        and an optimization - we can reasonably assume that those finalizers
+        don't do anything fancy and *just* call them. Among other things
+        they won't resurrect objects
+        """
+        new_objects = self.AddressStack()
+        while self.objects_with_light_finalizers.non_empty():
+            obj = self.objects_with_light_finalizers.pop()
+            if self.surviving(obj):
+                new_objects.append(self.get_forwarding_address(obj))
+            else:
+                finalizer = self.getfinalizer(self.get_type_id(obj))
+                finalizer(obj, llmemory.NULL)
+        self.objects_with_light_finalizers.delete()
+        self.objects_with_light_finalizers = new_objects
+
     def deal_with_objects_with_finalizers(self, scan):
         # walk over list of objects with finalizers
         # if it is not copied, add it to the list of to-be-called finalizers
diff --git a/pypy/rpython/memory/gctransform/framework.py b/pypy/rpython/memory/gctransform/framework.py
--- a/pypy/rpython/memory/gctransform/framework.py
+++ b/pypy/rpython/memory/gctransform/framework.py
@@ -12,6 +12,7 @@
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.translator.backendopt import graphanalyze
 from pypy.translator.backendopt.support import var_needsgc
+from pypy.translator.backendopt.finalizer import FinalizerAnalyzer
 from pypy.annotation import model as annmodel
 from pypy.rpython import annlowlevel
 from pypy.rpython.rbuiltin import gen_cast
@@ -258,6 +259,7 @@
             [s_gc, s_typeid16,
              annmodel.SomeInteger(nonneg=True),
              annmodel.SomeBool(),
+             annmodel.SomeBool(),
              annmodel.SomeBool()], s_gcref,
             inline = False)
         if hasattr(GCClass, 'malloc_fixedsize'):
@@ -267,6 +269,7 @@
                 [s_gc, s_typeid16,
                  annmodel.SomeInteger(nonneg=True),
                  annmodel.SomeBool(),
+                 annmodel.SomeBool(),
                  annmodel.SomeBool()], s_gcref,
                 inline = False)
         else:
@@ -319,7 +322,7 @@
             raise NotImplementedError("GC needs write barrier, but does not provide writebarrier_before_copy functionality")
 
         # in some GCs we can inline the common case of
-        # malloc_fixedsize(typeid, size, True, False, False)
+        # malloc_fixedsize(typeid, size, False, False, False)
         if getattr(GCClass, 'inline_simple_malloc', False):
             # make a copy of this function so that it gets annotated
             # independently and the constants are folded inside
@@ -337,7 +340,7 @@
                 malloc_fast,
                 [s_gc, s_typeid16,
                  annmodel.SomeInteger(nonneg=True),
-                 s_False, s_False], s_gcref,
+                 s_False, s_False, s_False], s_gcref,
                 inline = True)
         else:
             self.malloc_fast_ptr = None
@@ -668,7 +671,13 @@
         kind_and_fptr = self.special_funcptr_for_type(TYPE)
         has_finalizer = (kind_and_fptr is not None and
                          kind_and_fptr[0] == "finalizer")
+        has_light_finalizer = (kind_and_fptr is not None and
+                               kind_and_fptr[0] == "light_finalizer")
+        if has_light_finalizer:
+            has_finalizer = True
         c_has_finalizer = rmodel.inputconst(lltype.Bool, has_finalizer)
+        c_has_light_finalizer = rmodel.inputconst(lltype.Bool,
+                                                  has_light_finalizer)
 
         if not op.opname.endswith('_varsize') and not flags.get('varsize'):
             #malloc_ptr = self.malloc_fixedsize_ptr
@@ -682,7 +691,8 @@
             else:
                 malloc_ptr = self.malloc_fixedsize_ptr
             args = [self.c_const_gc, c_type_id, c_size,
-                    c_has_finalizer, rmodel.inputconst(lltype.Bool, False)]
+                    c_has_finalizer, c_has_light_finalizer,
+                    rmodel.inputconst(lltype.Bool, False)]
         else:
             assert not c_has_finalizer.value
             info_varsize = self.layoutbuilder.get_info_varsize(type_id)
@@ -847,12 +857,13 @@
         # used by the JIT (see pypy.jit.backend.llsupport.gc)
         op = hop.spaceop
         [v_typeid, v_size,
-         v_has_finalizer, v_contains_weakptr] = op.args
+         v_has_finalizer, v_has_light_finalizer, v_contains_weakptr] = op.args
         livevars = self.push_roots(hop)
         hop.genop("direct_call",
                   [self.malloc_fixedsize_clear_ptr, self.c_const_gc,
                    v_typeid, v_size,
-                   v_has_finalizer, v_contains_weakptr],
+                   v_has_finalizer, v_has_light_finalizer,
+                   v_contains_weakptr],
                   resultvar=op.result)
         self.pop_roots(hop, livevars)
 
@@ -912,10 +923,10 @@
         info = self.layoutbuilder.get_info(type_id)
         c_size = rmodel.inputconst(lltype.Signed, info.fixedsize)
         malloc_ptr = self.malloc_fixedsize_ptr
-        c_has_finalizer = rmodel.inputconst(lltype.Bool, False)
+        c_false = rmodel.inputconst(lltype.Bool, False)
         c_has_weakptr = rmodel.inputconst(lltype.Bool, True)
         args = [self.c_const_gc, c_type_id, c_size,
-                c_has_finalizer, c_has_weakptr]
+                c_false, c_false, c_has_weakptr]
 
         # push and pop the current live variables *including* the argument
         # to the weakref_create operation, which must be kept alive and
@@ -1250,6 +1261,7 @@
             lltype2vtable = translator.rtyper.lltype2vtable
         else:
             lltype2vtable = None
+        self.translator = translator
         super(TransformerLayoutBuilder, self).__init__(GCClass, lltype2vtable)
 
     def has_finalizer(self, TYPE):
@@ -1257,6 +1269,10 @@
         return rtti is not None and getattr(rtti._obj, 'destructor_funcptr',
                                             None)
 
+    def has_light_finalizer(self, TYPE):
+        special = self.special_funcptr_for_type(TYPE)
+        return special is not None and special[0] == 'light_finalizer'
+
     def has_custom_trace(self, TYPE):
         rtti = get_rtti(TYPE)
         return rtti is not None and getattr(rtti._obj, 'custom_trace_funcptr',
@@ -1264,7 +1280,7 @@
 
     def make_finalizer_funcptr_for_type(self, TYPE):
         if not self.has_finalizer(TYPE):
-            return None
+            return None, False
         rtti = get_rtti(TYPE)
         destrptr = rtti._obj.destructor_funcptr
         DESTR_ARG = lltype.typeOf(destrptr).TO.ARGS[0]
@@ -1276,7 +1292,9 @@
             return llmemory.NULL
         fptr = self.transformer.annotate_finalizer(ll_finalizer,
                 [llmemory.Address, llmemory.Address], llmemory.Address)
-        return fptr
+        g = destrptr._obj.graph
+        light = not FinalizerAnalyzer(self.translator).analyze_direct_call(g)
+        return fptr, light
 
     def make_custom_trace_funcptr_for_type(self, TYPE):
         if not self.has_custom_trace(TYPE):
diff --git a/pypy/rpython/memory/gctypelayout.py b/pypy/rpython/memory/gctypelayout.py
--- a/pypy/rpython/memory/gctypelayout.py
+++ b/pypy/rpython/memory/gctypelayout.py
@@ -1,7 +1,6 @@
 from pypy.rpython.lltypesystem import lltype, llmemory, llarena, llgroup
 from pypy.rpython.lltypesystem import rclass
 from pypy.rpython.lltypesystem.lloperation import llop
-from pypy.rlib.objectmodel import we_are_translated
 from pypy.rlib.debug import ll_assert
 from pypy.rlib.rarithmetic import intmask
 from pypy.tool.identity_dict import identity_dict
@@ -85,6 +84,13 @@
         else:
             return lltype.nullptr(GCData.FINALIZER_OR_CT_FUNC)
 
+    def q_light_finalizer(self, typeid):
+        typeinfo = self.get(typeid)
+        if typeinfo.infobits & T_HAS_LIGHTWEIGHT_FINALIZER:
+            return typeinfo.finalizer_or_customtrace
+        else:
+            return lltype.nullptr(GCData.FINALIZER_OR_CT_FUNC)        
+
     def q_offsets_to_gc_pointers(self, typeid):
         return self.get(typeid).ofstoptrs
 
@@ -142,6 +148,7 @@
             self.q_has_gcptr_in_varsize,
             self.q_is_gcarrayofgcptr,
             self.q_finalizer,
+            self.q_light_finalizer,
             self.q_offsets_to_gc_pointers,
             self.q_fixed_size,
             self.q_varsize_item_sizes,
@@ -157,16 +164,17 @@
 
 
 # the lowest 16bits are used to store group member index
-T_MEMBER_INDEX         =   0xffff
-T_IS_VARSIZE           = 0x010000
-T_HAS_GCPTR_IN_VARSIZE = 0x020000
-T_IS_GCARRAY_OF_GCPTR  = 0x040000
-T_IS_WEAKREF           = 0x080000
-T_IS_RPYTHON_INSTANCE  = 0x100000    # the type is a subclass of OBJECT
-T_HAS_FINALIZER        = 0x200000
-T_HAS_CUSTOM_TRACE     = 0x400000
-T_KEY_MASK             = intmask(0xFF000000)
-T_KEY_VALUE            = intmask(0x5A000000)    # bug detection only
+T_MEMBER_INDEX              =   0xffff
+T_IS_VARSIZE                = 0x010000
+T_HAS_GCPTR_IN_VARSIZE      = 0x020000
+T_IS_GCARRAY_OF_GCPTR       = 0x040000
+T_IS_WEAKREF                = 0x080000
+T_IS_RPYTHON_INSTANCE       = 0x100000 # the type is a subclass of OBJECT
+T_HAS_FINALIZER             = 0x200000
+T_HAS_CUSTOM_TRACE          = 0x400000
+T_HAS_LIGHTWEIGHT_FINALIZER = 0x800000
+T_KEY_MASK                  = intmask(0xFF000000)
+T_KEY_VALUE                 = intmask(0x5A000000) # bug detection only
 
 def _check_valid_type_info(p):
     ll_assert(p.infobits & T_KEY_MASK == T_KEY_VALUE, "invalid type_id")
@@ -194,6 +202,8 @@
         info.finalizer_or_customtrace = fptr
         if kind == "finalizer":
             infobits |= T_HAS_FINALIZER
+        elif kind == 'light_finalizer':
+            infobits |= T_HAS_FINALIZER | T_HAS_LIGHTWEIGHT_FINALIZER
         elif kind == "custom_trace":
             infobits |= T_HAS_CUSTOM_TRACE
         else:
@@ -367,12 +377,15 @@
     def special_funcptr_for_type(self, TYPE):
         if TYPE in self._special_funcptrs:
             return self._special_funcptrs[TYPE]
-        fptr1 = self.make_finalizer_funcptr_for_type(TYPE)
+        fptr1, is_lightweight = self.make_finalizer_funcptr_for_type(TYPE)
         fptr2 = self.make_custom_trace_funcptr_for_type(TYPE)
         assert not (fptr1 and fptr2), (
             "type %r needs both a finalizer and a custom tracer" % (TYPE,))
         if fptr1:
-            kind_and_fptr = "finalizer", fptr1
+            if is_lightweight:
+                kind_and_fptr = "light_finalizer", fptr1
+            else:
+                kind_and_fptr = "finalizer", fptr1
         elif fptr2:
             kind_and_fptr = "custom_trace", fptr2
         else:
@@ -382,7 +395,7 @@
 
     def make_finalizer_funcptr_for_type(self, TYPE):
         # must be overridden for proper finalizer support
-        return None
+        return None, False
 
     def make_custom_trace_funcptr_for_type(self, TYPE):
         # must be overridden for proper custom tracer support
diff --git a/pypy/rpython/memory/gcwrapper.py b/pypy/rpython/memory/gcwrapper.py
--- a/pypy/rpython/memory/gcwrapper.py
+++ b/pypy/rpython/memory/gcwrapper.py
@@ -1,3 +1,4 @@
+from pypy.translator.backendopt.finalizer import FinalizerAnalyzer
 from pypy.rpython.lltypesystem import lltype, llmemory, llheap
 from pypy.rpython import llinterp
 from pypy.rpython.annlowlevel import llhelper
@@ -196,9 +197,11 @@
             DESTR_ARG = lltype.typeOf(destrptr).TO.ARGS[0]
             destrgraph = destrptr._obj.graph
         else:
-            return None
+            return None, False
 
         assert not type_contains_pyobjs(TYPE), "not implemented"
+        t = self.llinterp.typer.annotator.translator
+        light = not FinalizerAnalyzer(t).analyze_direct_call(destrgraph)
         def ll_finalizer(addr, dummy):
             assert dummy == llmemory.NULL
             try:
@@ -208,7 +211,7 @@
                 raise RuntimeError(
                     "a finalizer raised an exception, shouldn't happen")
             return llmemory.NULL
-        return llhelper(gctypelayout.GCData.FINALIZER_OR_CT, ll_finalizer)
+        return llhelper(gctypelayout.GCData.FINALIZER_OR_CT, ll_finalizer), light
 
     def make_custom_trace_funcptr_for_type(self, TYPE):
         from pypy.rpython.memory.gctransform.support import get_rtti, \
diff --git a/pypy/rpython/memory/test/test_gc.py b/pypy/rpython/memory/test/test_gc.py
--- a/pypy/rpython/memory/test/test_gc.py
+++ b/pypy/rpython/memory/test/test_gc.py
@@ -5,7 +5,6 @@
 from pypy.rpython.memory.test import snippet
 from pypy.rpython.test.test_llinterp import get_interpreter
 from pypy.rpython.lltypesystem import lltype
-from pypy.rpython.lltypesystem.rstr import STR
 from pypy.rpython.lltypesystem.lloperation import llop
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.rlib.objectmodel import compute_unique_id
@@ -57,7 +56,7 @@
                 while j < 20:
                     j += 1
                     a.append(j)
-        res = self.interpret(malloc_a_lot, [])
+        self.interpret(malloc_a_lot, [])
         #assert simulator.current_size - curr < 16000 * INT_SIZE / 4
         #print "size before: %s, size after %s" % (curr, simulator.current_size)
 
@@ -73,7 +72,7 @@
                 while j < 20:
                     j += 1
                     b.append((1, j, i))
-        res = self.interpret(malloc_a_lot, [])
+        self.interpret(malloc_a_lot, [])
         #assert simulator.current_size - curr < 16000 * INT_SIZE / 4
         #print "size before: %s, size after %s" % (curr, simulator.current_size)
 
@@ -129,7 +128,7 @@
         res = self.interpret(concat, [100])
         assert res == concat(100)
         #assert simulator.current_size - curr < 16000 * INT_SIZE / 4
-        
+
     def test_finalizer(self):
         class B(object):
             pass
@@ -278,7 +277,7 @@
                                self.interpret, f, [])
 
     def test_weakref(self):
-        import weakref, gc
+        import weakref
         class A(object):
             pass
         def g():
@@ -299,7 +298,7 @@
         assert res
 
     def test_weakref_to_object_with_finalizer(self):
-        import weakref, gc
+        import weakref
         class A(object):
             count = 0
         a = A()
@@ -338,7 +337,7 @@
         assert res
 
     def test_cycle_with_weakref_and_del(self):
-        import weakref, gc
+        import weakref
         class A(object):
             count = 0
         a = A()
@@ -367,7 +366,7 @@
         assert res == 11
 
     def test_weakref_to_object_with_finalizer_ordering(self):
-        import weakref, gc
+        import weakref
         class A(object):
             count = 0
         a = A()
@@ -616,7 +615,7 @@
                     assert not rgc.can_move(a)
                     return 1
                 return 0
-            except Exception, e:
+            except Exception:
                 return 2
 
         assert self.interpret(func, []) == int(self.GC_CAN_MALLOC_NONMOVABLE)
@@ -647,8 +646,6 @@
         assert self.interpret(f, [bigsize, 0, flag]) == 0x62024241
 
     def test_tagged_simple(self):
-        from pypy.rlib.objectmodel import UnboxedValue
-
         class Unrelated(object):
             pass
 
@@ -689,8 +686,6 @@
         assert res == -897
 
     def test_tagged_id(self):
-        from pypy.rlib.objectmodel import UnboxedValue, compute_unique_id
-
         class Unrelated(object):
             pass
 
diff --git a/pypy/rpython/memory/test/test_transformed_gc.py b/pypy/rpython/memory/test/test_transformed_gc.py
--- a/pypy/rpython/memory/test/test_transformed_gc.py
+++ b/pypy/rpython/memory/test/test_transformed_gc.py
@@ -345,22 +345,22 @@
         b = B()
         b.nextid = 0
         b.num_deleted = 0
-        class A(object):
+        class AAA(object):
             def __init__(self):
                 self.id = b.nextid
                 b.nextid += 1
             def __del__(self):
                 b.num_deleted += 1
                 C()
-        class C(A):
+        class C(AAA):
             def __del__(self):
                 b.num_deleted += 1
         def f(x, y):
-            a = A()
+            a = AAA()
             i = 0
             while i < x:
                 i += 1
-                a = A()
+                a = AAA()
             llop.gc__collect(lltype.Void)
             llop.gc__collect(lltype.Void)
             return b.num_deleted
@@ -807,6 +807,7 @@
                     op.args = [Constant(type_id, llgroup.HALFWORD),
                                Constant(llmemory.sizeof(P), lltype.Signed),
                                Constant(False, lltype.Bool), # has_finalizer
+                               Constant(False, lltype.Bool), # is_finalizer_light
                                Constant(False, lltype.Bool)] # contains_weakptr
                     break
             else:
@@ -843,6 +844,7 @@
                     op.args = [Constant(type_id, llgroup.HALFWORD),
                                Constant(llmemory.sizeof(P), lltype.Signed),
                                Constant(False, lltype.Bool), # has_finalizer
+                               Constant(False, lltype.Bool), # is_finalizer_light
                                Constant(False, lltype.Bool)] # contains_weakptr
                     break
             else:
diff --git a/pypy/rpython/rtyper.py b/pypy/rpython/rtyper.py
--- a/pypy/rpython/rtyper.py
+++ b/pypy/rpython/rtyper.py
@@ -717,7 +717,7 @@
             raise TyperError("runtime type info function %r returns %r, "
                              "excepted Ptr(RuntimeTypeInfo)" % (func, s))
         funcptr = self.getcallable(graph)
-        attachRuntimeTypeInfo(GCSTRUCT, funcptr, destrptr)
+        attachRuntimeTypeInfo(GCSTRUCT, funcptr, destrptr, None)
 
 # register operations from annotation model
 RPythonTyper._registeroperations(annmodel)
diff --git a/pypy/rpython/test/test_rclass.py b/pypy/rpython/test/test_rclass.py
--- a/pypy/rpython/test/test_rclass.py
+++ b/pypy/rpython/test/test_rclass.py
@@ -3,7 +3,7 @@
 from pypy.translator.translator import TranslationContext, graphof
 from pypy.rpython.lltypesystem.lltype import *
 from pypy.rpython.ootypesystem import ootype
-from pypy.rlib.rarithmetic import intmask, r_longlong
+from pypy.rlib.rarithmetic import r_longlong
 from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin
 from pypy.rpython.rclass import IR_IMMUTABLE, IR_IMMUTABLE_ARRAY
 from pypy.rpython.rclass import IR_QUASIIMMUTABLE, IR_QUASIIMMUTABLE_ARRAY
@@ -972,10 +972,10 @@
         graph = graphof(t, f)
         TYPE = graph.startblock.operations[0].args[0].value
         RTTI = getRuntimeTypeInfo(TYPE)
-        queryptr = RTTI._obj.query_funcptr # should not raise
+        RTTI._obj.query_funcptr # should not raise
         destrptr = RTTI._obj.destructor_funcptr
         assert destrptr is not None
-    
+
     def test_del_inheritance(self):
         from pypy.rlib import rgc
         class State:
diff --git a/pypy/translator/backendopt/finalizer.py b/pypy/translator/backendopt/finalizer.py
new file mode 100644
--- /dev/null
+++ b/pypy/translator/backendopt/finalizer.py
@@ -0,0 +1,34 @@
+
+from pypy.translator.backendopt import graphanalyze
+from pypy.rpython.lltypesystem import lltype
+
+class FinalizerAnalyzer(graphanalyze.BoolGraphAnalyzer):
+    """ Analyzer that determines whether a finalizer is lightweight enough
+    so it can be called without all the complicated logic in the garbage
+    collector. The set of operations here is restrictive for a good reason
+    - it's better to be safe. Specifically disallowed operations:
+
+    * anything that escapes self
+    * anything that can allocate
+    """
+    ok_operations = ['ptr_nonzero', 'ptr_eq', 'ptr_ne', 'free', 'same_as',
+                     'direct_ptradd', 'force_cast', 'track_alloc_stop',
+                     'raw_free']
+    
+    def analyze_simple_operation(self, op, graphinfo):
+        if op.opname in self.ok_operations:
+            return self.bottom_result()
+        if (op.opname.startswith('int_') or op.opname.startswith('float_')
+            or op.opname.startswith('cast_')):
+            return self.bottom_result()
+        if op.opname == 'setfield' or op.opname == 'bare_setfield':
+            TP = op.args[2].concretetype
+            if not isinstance(TP, lltype.Ptr) or TP.TO._gckind == 'raw':
+                # primitive type
+                return self.bottom_result()
+        if op.opname == 'getfield':
+            TP = op.result.concretetype
+            if not isinstance(TP, lltype.Ptr) or TP.TO._gckind == 'raw':
+                # primitive type
+                return self.bottom_result()            
+        return self.top_result()
diff --git a/pypy/translator/backendopt/test/test_finalizer.py b/pypy/translator/backendopt/test/test_finalizer.py
new file mode 100644
--- /dev/null
+++ b/pypy/translator/backendopt/test/test_finalizer.py
@@ -0,0 +1,138 @@
+
+import py
+from pypy.translator.backendopt.finalizer import FinalizerAnalyzer
+from pypy.translator.translator import TranslationContext, graphof
+from pypy.translator.backendopt.all import backend_optimizations
+from pypy.translator.unsimplify import varoftype
+from pypy.rpython.lltypesystem import lltype, rffi
+from pypy.conftest import option
+
+
+class BaseFinalizerAnalyzerTests(object):
+    """ Below are typical destructors that we encounter in pypy
+    """
+
+    type_system = None
+    
+    def analyze(self, func, sig, func_to_analyze=None, backendopt=False):
+        if func_to_analyze is None:
+            func_to_analyze = func
+        t = TranslationContext()
+        t.buildannotator().build_types(func, sig)
+        t.buildrtyper(type_system=self.type_system).specialize()
+        if backendopt:
+            backend_optimizations(t)
+        if option.view:
+            t.view()
+        a = FinalizerAnalyzer(t)
+        fgraph = graphof(t, func_to_analyze)
+        result = a.analyze_direct_call(fgraph)
+        return result
+
+    def test_nothing(self):
+        def f():
+            pass
+        r = self.analyze(f, [])
+        assert not r
+
+def test_various_ops():
+    from pypy.objspace.flow.model import SpaceOperation, Constant
+
+    X = lltype.Ptr(lltype.GcStruct('X'))
+    Z = lltype.Ptr(lltype.Struct('Z'))
+    S = lltype.GcStruct('S', ('x', lltype.Signed),
+                        ('y', X),
+                        ('z', Z))
+    v1 = varoftype(lltype.Bool)
+    v2 = varoftype(lltype.Signed)
+    f = FinalizerAnalyzer(None)
+    r = f.analyze(SpaceOperation('cast_int_to_bool', [v2],
+                                                       v1))
+    assert not r
+    v1 = varoftype(lltype.Ptr(S))
+    v2 = varoftype(lltype.Signed)
+    v3 = varoftype(X)
+    v4 = varoftype(Z)
+    assert not f.analyze(SpaceOperation('bare_setfield', [v1, Constant('x'),
+                                                          v2], None))
+    assert     f.analyze(SpaceOperation('bare_setfield', [v1, Constant('y'),
+                                                          v3], None))
+    assert not f.analyze(SpaceOperation('bare_setfield', [v1, Constant('z'),
+                                                          v4], None))
+    
+        
+class TestLLType(BaseFinalizerAnalyzerTests):
+    type_system = 'lltype'
+
+    def test_malloc(self):
+        S = lltype.GcStruct('S')
+        
+        def f():
+            return lltype.malloc(S)
+
+        r = self.analyze(f, [])
+        assert r
+
+    def test_raw_free_getfield(self):
+        S = lltype.Struct('S')
+        
+        class A(object):
+            def __init__(self):
+                self.x = lltype.malloc(S, flavor='raw')
+
+            def __del__(self):
+                if self.x:
+                    self.x = lltype.nullptr(S)
+                    lltype.free(self.x, flavor='raw')
+
+        def f():
+            return A()
+
+        r = self.analyze(f, [], A.__del__.im_func)
+        assert not r
+
+    def test_c_call(self):
+        C = rffi.CArray(lltype.Signed)
+        c = rffi.llexternal('x', [lltype.Ptr(C)], lltype.Signed)
+
+        def g():
+            p = lltype.malloc(C, 3, flavor='raw')
+            f(p)
+        
+        def f(p):
+            c(rffi.ptradd(p, 0))
+            lltype.free(p, flavor='raw')
+
+        r = self.analyze(g, [], f, backendopt=True)
+        assert not r
+
+    def test_chain(self):
+        class B(object):
+            def __init__(self):
+                self.counter = 1
+        
+        class A(object):
+            def __init__(self):
+                self.x = B()
+
+            def __del__(self):
+                self.x.counter += 1
+
+        def f():
+            A()
+
+        r = self.analyze(f, [], A.__del__.im_func)
+        assert r
+
+    def test_os_call(self):
+        py.test.skip("can allocate OSError, but also can raise, ignore for now")
+        import os
+        
+        def f(i):
+            os.close(i)
+
+        r = self.analyze(f, [int], backendopt=True)
+        assert not r
+
+class TestOOType(BaseFinalizerAnalyzerTests):
+    type_system = 'ootype'

From noreply at buildbot.pypy.org  Tue Oct 25 18:21:33 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 25 Oct 2011 18:21:33 +0200 (CEST)
Subject: [pypy-commit] pypy default: Provide a hint that can be specified as
 a decorator - @rgc.is_light_finalizer
Message-ID: <20111025162133.B72F2820C7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r48445:9d13b202cb4b
Date: 2011-10-25 18:21 +0200
http://bitbucket.org/pypy/pypy/changeset/9d13b202cb4b/

Log:	Provide a hint that can be specified as a decorator -
	@rgc.is_light_finalizer

diff --git a/pypy/rlib/rgc.py b/pypy/rlib/rgc.py
--- a/pypy/rlib/rgc.py
+++ b/pypy/rlib/rgc.py
@@ -214,6 +214,10 @@
     func._gc_no_collect_ = True
     return func
 
+def is_light_finalizer(func):
+    func._is_light_finalizer_ = True
+    return func
+
 # ____________________________________________________________
 
 def get_rpy_roots():
diff --git a/pypy/rpython/memory/gctransform/framework.py b/pypy/rpython/memory/gctransform/framework.py
--- a/pypy/rpython/memory/gctransform/framework.py
+++ b/pypy/rpython/memory/gctransform/framework.py
@@ -1293,7 +1293,7 @@
         fptr = self.transformer.annotate_finalizer(ll_finalizer,
                 [llmemory.Address, llmemory.Address], llmemory.Address)
         g = destrptr._obj.graph
-        light = not FinalizerAnalyzer(self.translator).analyze_direct_call(g)
+        light = not FinalizerAnalyzer(self.translator).analyze_light_finalizer(g)
         return fptr, light
 
     def make_custom_trace_funcptr_for_type(self, TYPE):
diff --git a/pypy/rpython/memory/gcwrapper.py b/pypy/rpython/memory/gcwrapper.py
--- a/pypy/rpython/memory/gcwrapper.py
+++ b/pypy/rpython/memory/gcwrapper.py
@@ -201,7 +201,7 @@
 
         assert not type_contains_pyobjs(TYPE), "not implemented"
         t = self.llinterp.typer.annotator.translator
-        light = not FinalizerAnalyzer(t).analyze_direct_call(destrgraph)
+        light = not FinalizerAnalyzer(t).analyze_light_finalizer(destrgraph)
         def ll_finalizer(addr, dummy):
             assert dummy == llmemory.NULL
             try:
diff --git a/pypy/translator/backendopt/finalizer.py b/pypy/translator/backendopt/finalizer.py
--- a/pypy/translator/backendopt/finalizer.py
+++ b/pypy/translator/backendopt/finalizer.py
@@ -2,6 +2,11 @@
 from pypy.translator.backendopt import graphanalyze
 from pypy.rpython.lltypesystem import lltype
 
+class FinalizerError(Exception):
+    """ __del__ marked as lightweight finalizer, but the analyzer did
+    not agreed
+    """
+
 class FinalizerAnalyzer(graphanalyze.BoolGraphAnalyzer):
     """ Analyzer that determines whether a finalizer is lightweight enough
     so it can be called without all the complicated logic in the garbage
@@ -15,6 +20,13 @@
                      'direct_ptradd', 'force_cast', 'track_alloc_stop',
                      'raw_free']
     
+    def analyze_light_finalizer(self, graph):
+        result = self.analyze_direct_call(graph)
+        if (result is self.top_result() and
+            getattr(graph.func, '_is_light_finalizer_', False)):
+            raise FinalizerError(FinalizerError.__doc__, graph)
+        return result
+    
     def analyze_simple_operation(self, op, graphinfo):
         if op.opname in self.ok_operations:
             return self.bottom_result()
diff --git a/pypy/translator/backendopt/test/test_finalizer.py b/pypy/translator/backendopt/test/test_finalizer.py
--- a/pypy/translator/backendopt/test/test_finalizer.py
+++ b/pypy/translator/backendopt/test/test_finalizer.py
@@ -1,11 +1,13 @@
 
 import py
-from pypy.translator.backendopt.finalizer import FinalizerAnalyzer
+from pypy.translator.backendopt.finalizer import FinalizerAnalyzer,\
+     FinalizerError
 from pypy.translator.translator import TranslationContext, graphof
 from pypy.translator.backendopt.all import backend_optimizations
 from pypy.translator.unsimplify import varoftype
 from pypy.rpython.lltypesystem import lltype, rffi
 from pypy.conftest import option
+from pypy.rlib import rgc
 
 
 class BaseFinalizerAnalyzerTests(object):
@@ -26,7 +28,7 @@
             t.view()
         a = FinalizerAnalyzer(t)
         fgraph = graphof(t, func_to_analyze)
-        result = a.analyze_direct_call(fgraph)
+        result = a.analyze_light_finalizer(fgraph)
         return result
 
     def test_nothing(self):
@@ -124,15 +126,17 @@
         r = self.analyze(f, [], A.__del__.im_func)
         assert r
 
-    def test_os_call(self):
-        py.test.skip("can allocate OSError, but also can raise, ignore for now")
-        import os
-        
-        def f(i):
-            os.close(i)
+    def test_is_light_finalizer_decorator(self):
+        S = lltype.GcStruct('S')
 
-        r = self.analyze(f, [int], backendopt=True)
-        assert not r
+        @rgc.is_light_finalizer
+        def f():
+            lltype.malloc(S)
+        @rgc.is_light_finalizer
+        def g():
+            pass
+        self.analyze(g, []) # did not explode
+        py.test.raises(FinalizerError, self.analyze, f, [])
 
 class TestOOType(BaseFinalizerAnalyzerTests):
     type_system = 'ootype'

From noreply at buildbot.pypy.org  Tue Oct 25 19:01:03 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 25 Oct 2011 19:01:03 +0200 (CEST)
Subject: [pypy-commit] pypy default: I swear I did that change before -
 remove the incorrect usage of close
Message-ID: <20111025170103.CCF1D820C7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r48446:2612cae35357
Date: 2011-10-25 19:00 +0200
http://bitbucket.org/pypy/pypy/changeset/2612cae35357/

Log:	I swear I did that change before - remove the incorrect usage of
	close introduced by chance when experimenting with closing urllib2
	requests.

diff --git a/lib-python/modified-2.7/urllib2.py b/lib-python/modified-2.7/urllib2.py
--- a/lib-python/modified-2.7/urllib2.py
+++ b/lib-python/modified-2.7/urllib2.py
@@ -395,11 +395,7 @@
         meth_name = protocol+"_response"
         for processor in self.process_response.get(protocol, []):
             meth = getattr(processor, meth_name)
-            try:
-                response = meth(req, response)
-            except:
-                response.close()
-                raise
+            response = meth(req, response)
 
         return response
 

From noreply at buildbot.pypy.org  Tue Oct 25 20:33:24 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue, 25 Oct 2011 20:33:24 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-dicts: make the heapcache a bit smarter
 about how stuff escaped
Message-ID: <20111025183324.20EAE820C7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: virtual-dicts
Changeset: r48447:d58706c7bdb6
Date: 2011-10-25 14:33 -0400
http://bitbucket.org/pypy/pypy/changeset/d58706c7bdb6/

Log:	make the heapcache a bit smarter about how stuff escaped

diff --git a/pypy/jit/metainterp/heapcache.py b/pypy/jit/metainterp/heapcache.py
--- a/pypy/jit/metainterp/heapcache.py
+++ b/pypy/jit/metainterp/heapcache.py
@@ -41,6 +41,13 @@
                 self.dependencies.setdefault(box, []).append(valuebox)
             else:
                 self._escape(valuebox)
+        elif opnum == rop.SETARRAYITEM_GC:
+            assert len(argboxes) == 3
+            box, indexbox, valuebox = argboxes
+            if self.is_unescaped(box) and self.is_unescaped(valuebox):
+                self.dependencies.setdefault(box, []).append(valuebox)
+            else:
+                self._escape(valuebox)
         # GETFIELD_GC and MARK_OPAQUE_PTR doesn't escape their arguments
         elif opnum != rop.GETFIELD_GC and opnum != rop.MARK_OPAQUE_PTR:
             idx = 0
@@ -60,14 +67,11 @@
                 self._escape(dep)
 
     def clear_caches(self, opnum, descr, argboxes):
-        if opnum == rop.SETFIELD_GC:
-            return
-        if opnum == rop.SETARRAYITEM_GC:
-            return
-        if opnum == rop.SETFIELD_RAW:
-            return
-        if opnum == rop.SETARRAYITEM_RAW:
-            return
+        if (opnum == rop.SETFIELD_GC or
+            opnum == rop.SETARRAYITEM_GC or
+            opnum == rop.SETFIELD_RAW or
+            opnum == rop.SETARRAYITEM_RAW or
+            opnum == rop.SETINTERIORFIELD_GC):
         if rop._OVF_FIRST <= opnum <= rop._OVF_LAST:
             return
         if rop._NOSIDEEFFECT_FIRST <= opnum <= rop._NOSIDEEFFECT_LAST:
@@ -75,9 +79,9 @@
         if opnum == rop.CALL or opnum == rop.CALL_LOOPINVARIANT:
             effectinfo = descr.get_extra_info()
             ef = effectinfo.extraeffect
-            if ef == effectinfo.EF_LOOPINVARIANT or \
-               ef == effectinfo.EF_ELIDABLE_CANNOT_RAISE or \
-               ef == effectinfo.EF_ELIDABLE_CAN_RAISE:
+            if (ef == effectinfo.EF_LOOPINVARIANT or
+                ef == effectinfo.EF_ELIDABLE_CANNOT_RAISE or
+                ef == effectinfo.EF_ELIDABLE_CAN_RAISE):
                 return
             # A special case for ll_arraycopy, because it is so common, and its
             # effects are so well defined.
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -3521,7 +3521,7 @@
         self.check_loops({"int_sub": 1, "int_gt": 1, "guard_true": 1, "jump": 1})
 
     def test_virtual_opaque_ptr(self):
-        myjitdriver = JitDriver(greens = [], reds=["n"])
+        myjitdriver = JitDriver(greens = [], reds = ["n"])
         erase, unerase = rerased.new_erasing_pair("x")
         @look_inside_iff(lambda x: isvirtual(x))
         def g(x):
@@ -3539,6 +3539,27 @@
         assert res == 0
         self.check_loops({"int_sub": 1, "int_gt": 1, "guard_true": 1, "jump": 1})
 
+    def test_virtual_opaque_dict(self):
+        myjitdriver = JitDriver(greens = [], reds = ["n"])
+        erase, unerase = rerased.new_erasing_pair("x")
+        @look_inside_iff(lambda x: isvirtual(x))
+        def g(x):
+            return x[0]["key"] - 1
+        def f(n):
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n)
+                x = [{}]
+                x[0]["key"] = n
+                x[0]["other key"] = n
+                y = erase(x)
+                z = unerase(y)
+                n = g(x)
+            return n
+        res = self.meta_interp(f, [10])
+        assert res == 0
+        self.check_loops({"int_sub": 1, "int_gt": 1, "guard_true": 1, "jump": 1})
+
+
 
 class TestLLtype(BaseLLtypeTests, LLJitMixin):
     def test_tagged(self):

From noreply at buildbot.pypy.org  Tue Oct 25 20:36:37 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue, 25 Oct 2011 20:36:37 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-dicts: add a unittest for the previous
	commit
Message-ID: <20111025183637.80A4B820C7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: virtual-dicts
Changeset: r48448:c1dbbe1bd86e
Date: 2011-10-25 14:36 -0400
http://bitbucket.org/pypy/pypy/changeset/c1dbbe1bd86e/

Log:	add a unittest for the previous commit

diff --git a/pypy/jit/metainterp/heapcache.py b/pypy/jit/metainterp/heapcache.py
--- a/pypy/jit/metainterp/heapcache.py
+++ b/pypy/jit/metainterp/heapcache.py
@@ -72,6 +72,7 @@
             opnum == rop.SETFIELD_RAW or
             opnum == rop.SETARRAYITEM_RAW or
             opnum == rop.SETINTERIORFIELD_GC):
+            return
         if rop._OVF_FIRST <= opnum <= rop._OVF_LAST:
             return
         if rop._NOSIDEEFFECT_FIRST <= opnum <= rop._NOSIDEEFFECT_LAST:
diff --git a/pypy/jit/metainterp/test/test_heapcache.py b/pypy/jit/metainterp/test/test_heapcache.py
--- a/pypy/jit/metainterp/test/test_heapcache.py
+++ b/pypy/jit/metainterp/test/test_heapcache.py
@@ -371,3 +371,17 @@
         assert h.is_unescaped(box1)
         h.invalidate_caches(rop.SETARRAYITEM_GC, None, [box2, index1, box1])
         assert not h.is_unescaped(box1)
+
+        h = HeapCache()
+        h.new_array(box1, lengthbox1)
+        h.new(box2)
+        assert h.is_unescaped(box1)
+        assert h.is_unescaped(box2)
+        h.invalidate_caches(rop.SETARRAYITEM_GC, None, [box1, lengthbox2, box2])
+        assert h.is_unescaped(box1)
+        assert h.is_unescaped(box2)
+        h.invalidate_caches(
+            rop.CALL, FakeCallDescr(FakeEffektinfo.EF_RANDOM_EFFECTS), [box1]
+        )
+        assert not h.is_unescaped(box1)
+        assert not h.is_unescaped(box2)

From noreply at buildbot.pypy.org  Tue Oct 25 20:38:21 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue, 25 Oct 2011 20:38:21 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-dicts: merged default
Message-ID: <20111025183821.15C8E820C7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: virtual-dicts
Changeset: r48449:d876b901547e
Date: 2011-10-25 14:38 -0400
http://bitbucket.org/pypy/pypy/changeset/d876b901547e/

Log:	merged default

diff --git a/lib-python/modified-2.7/urllib2.py b/lib-python/modified-2.7/urllib2.py
--- a/lib-python/modified-2.7/urllib2.py
+++ b/lib-python/modified-2.7/urllib2.py
@@ -395,11 +395,7 @@
         meth_name = protocol+"_response"
         for processor in self.process_response.get(protocol, []):
             meth = getattr(processor, meth_name)
-            try:
-                response = meth(req, response)
-            except:
-                response.close()
-                raise
+            response = meth(req, response)
 
         return response
 
diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py
--- a/pypy/jit/backend/llsupport/gc.py
+++ b/pypy/jit/backend/llsupport/gc.py
@@ -649,10 +649,13 @@
         def malloc_basic(size, tid):
             type_id = llop.extract_ushort(llgroup.HALFWORD, tid)
             has_finalizer = bool(tid & (1<<llgroup.HALFSHIFT))
+            has_light_finalizer = bool(tid & (1<<(llgroup.HALFSHIFT + 1)))
             check_typeid(type_id)
             res = llop1.do_malloc_fixedsize_clear(llmemory.GCREF,
                                                   type_id, size,
-                                                  has_finalizer, False)
+                                                  has_finalizer,
+                                                  has_light_finalizer,
+                                                  False)
             # In case the operation above failed, we are returning NULL
             # from this function to assembler.  There is also an RPython
             # exception set, typically MemoryError; but it's easier and
@@ -723,7 +726,7 @@
             # also use it to allocate varsized objects.  The tid
             # and possibly the length are both set afterward.
             gcref = llop1.do_malloc_fixedsize_clear(llmemory.GCREF,
-                                        0, size, False, False)
+                                        0, size, False, False, False)
             return rffi.cast(lltype.Signed, gcref)
         self.malloc_slowpath = malloc_slowpath
         self.MALLOC_SLOWPATH = lltype.FuncType([lltype.Signed], lltype.Signed)
@@ -747,7 +750,9 @@
         type_id = self.layoutbuilder.get_type_id(S)
         assert not self.layoutbuilder.is_weakref_type(S)
         has_finalizer = bool(self.layoutbuilder.has_finalizer(S))
-        flags = int(has_finalizer) << llgroup.HALFSHIFT
+        has_light_finalizer = bool(self.layoutbuilder.has_light_finalizer(S))
+        flags = (int(has_finalizer) << llgroup.HALFSHIFT |
+                 int(has_light_finalizer) << (llgroup.HALFSHIFT + 1))
         descr.tid = llop.combine_ushort(lltype.Signed, type_id, flags)
 
     def init_array_descr(self, A, descr):
diff --git a/pypy/jit/backend/llsupport/test/test_gc.py b/pypy/jit/backend/llsupport/test/test_gc.py
--- a/pypy/jit/backend/llsupport/test/test_gc.py
+++ b/pypy/jit/backend/llsupport/test/test_gc.py
@@ -247,11 +247,12 @@
         self.record = []
 
     def do_malloc_fixedsize_clear(self, RESTYPE, type_id, size,
-                                  has_finalizer, contains_weakptr):
+                                  has_finalizer, has_light_finalizer,
+                                  contains_weakptr):
         assert not contains_weakptr
         p = llmemory.raw_malloc(size)
         p = llmemory.cast_adr_to_ptr(p, RESTYPE)
-        flags = int(has_finalizer) << 16
+        flags = (int(has_finalizer) << 16) | (int(has_light_finalizer) << 17)
         tid = llop.combine_ushort(lltype.Signed, type_id, flags)
         self.record.append(("fixedsize", repr(size), tid, p))
         return p
diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -19,7 +19,7 @@
 class W_RSocket(Wrappable, RSocket):
     def __del__(self):
         self.clear_all_weakrefs()
-        self.close()
+        RSocket.__del__(self)
 
     def accept_w(self, space):
         """accept() -> (socket object, address info)
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -211,7 +211,9 @@
             return result
 
         def __del__(self):
-            self.clear_all_weakrefs()
+            # note that we don't call clear_all_weakrefs here because
+            # an array with freed buffer is ok to see - it's just empty with 0
+            # length
             self.setlen(0)
 
         def setlen(self, size):
diff --git a/pypy/module/array/test/test_array.py b/pypy/module/array/test/test_array.py
--- a/pypy/module/array/test/test_array.py
+++ b/pypy/module/array/test/test_array.py
@@ -824,6 +824,22 @@
         r = weakref.ref(a)
         assert r() is a
 
+    def test_subclass_del(self):
+        import array, gc, weakref
+        l = []
+        
+        class A(array.array):
+            pass
+
+        a = A('d')
+        a.append(3.0)
+        r = weakref.ref(a, lambda a: l.append(a()))
+        del a
+        gc.collect()
+        assert l
+        assert l[0] is None or len(l[0]) == 0
+
+
 class TestCPythonsOwnArray(BaseArrayTests):
 
     def setup_class(cls):
@@ -844,11 +860,7 @@
         cls.w_tempfile = cls.space.wrap(
             str(py.test.ensuretemp('array').join('tmpfile')))
         cls.w_maxint = cls.space.wrap(sys.maxint)
-
-
-
-
-
+    
     def test_buffer_info(self):
         a = self.array('c', 'Hi!')
         bi = a.buffer_info()
diff --git a/pypy/rlib/rgc.py b/pypy/rlib/rgc.py
--- a/pypy/rlib/rgc.py
+++ b/pypy/rlib/rgc.py
@@ -214,6 +214,10 @@
     func._gc_no_collect_ = True
     return func
 
+def is_light_finalizer(func):
+    func._is_light_finalizer_ = True
+    return func
+
 # ____________________________________________________________
 
 def get_rpy_roots():
diff --git a/pypy/rlib/rmmap.py b/pypy/rlib/rmmap.py
--- a/pypy/rlib/rmmap.py
+++ b/pypy/rlib/rmmap.py
@@ -292,6 +292,9 @@
         elif _POSIX:
             self.closed = True
             if self.fd != -1:
+                # XXX this is buggy - raising in an RPython del is not a good
+                #     idea, we should swallow the exception or ignore the
+                #     underlaying close error code
                 os.close(self.fd)
                 self.fd = -1
             if self.size > 0:
diff --git a/pypy/rlib/rsocket.py b/pypy/rlib/rsocket.py
--- a/pypy/rlib/rsocket.py
+++ b/pypy/rlib/rsocket.py
@@ -56,6 +56,7 @@
 
 
 _FAMILIES = {}
+
 class Address(object):
     """The base class for RPython-level objects representing addresses.
     Fields:  addr    - a _c.sockaddr_ptr (memory owned by the Address instance)
@@ -77,9 +78,8 @@
         self.addrlen = addrlen
 
     def __del__(self):
-        addr = self.addr_p
-        if addr:
-            lltype.free(addr, flavor='raw')
+        if self.addr_p:
+            lltype.free(self.addr_p, flavor='raw')
 
     def setdata(self, addr, addrlen):
         # initialize self.addr and self.addrlen.  'addr' can be a different
@@ -613,7 +613,10 @@
         self.timeout = defaults.timeout
         
     def __del__(self):
-        self.close()
+        fd = self.fd
+        if fd != _c.INVALID_SOCKET:
+            self.fd = _c.INVALID_SOCKET
+            _c.socketclose(fd)
 
     if hasattr(_c, 'fcntl'):
         def _setblocking(self, block):
diff --git a/pypy/rpython/memory/gc/base.py b/pypy/rpython/memory/gc/base.py
--- a/pypy/rpython/memory/gc/base.py
+++ b/pypy/rpython/memory/gc/base.py
@@ -1,4 +1,5 @@
-from pypy.rpython.lltypesystem import lltype, llmemory, llarena
+from pypy.rpython.lltypesystem import lltype, llmemory, llarena, rffi
+from pypy.rpython.lltypesystem.lloperation import llop
 from pypy.rlib.debug import ll_assert
 from pypy.rpython.memory.gcheader import GCHeaderBuilder
 from pypy.rpython.memory.support import DEFAULT_CHUNK_SIZE
@@ -62,6 +63,7 @@
     def set_query_functions(self, is_varsize, has_gcptr_in_varsize,
                             is_gcarrayofgcptr,
                             getfinalizer,
+                            getlightfinalizer,
                             offsets_to_gc_pointers,
                             fixed_size, varsize_item_sizes,
                             varsize_offset_to_variable_part,
@@ -74,6 +76,7 @@
                             get_custom_trace,
                             fast_path_tracing):
         self.getfinalizer = getfinalizer
+        self.getlightfinalizer = getlightfinalizer
         self.is_varsize = is_varsize
         self.has_gcptr_in_varsize = has_gcptr_in_varsize
         self.is_gcarrayofgcptr = is_gcarrayofgcptr
@@ -139,6 +142,7 @@
 
         size = self.fixed_size(typeid)
         needs_finalizer = bool(self.getfinalizer(typeid))
+        finalizer_is_light = bool(self.getlightfinalizer(typeid))
         contains_weakptr = self.weakpointer_offset(typeid) >= 0
         assert not (needs_finalizer and contains_weakptr)
         if self.is_varsize(typeid):
@@ -158,6 +162,7 @@
             else:
                 malloc_fixedsize = self.malloc_fixedsize
             ref = malloc_fixedsize(typeid, size, needs_finalizer,
+                                   finalizer_is_light,
                                    contains_weakptr)
         # lots of cast and reverse-cast around...
         return llmemory.cast_ptr_to_adr(ref)
diff --git a/pypy/rpython/memory/gc/generation.py b/pypy/rpython/memory/gc/generation.py
--- a/pypy/rpython/memory/gc/generation.py
+++ b/pypy/rpython/memory/gc/generation.py
@@ -167,7 +167,9 @@
         return self.nursery <= addr < self.nursery_top
 
     def malloc_fixedsize_clear(self, typeid, size,
-                               has_finalizer=False, contains_weakptr=False):
+                               has_finalizer=False,
+                               is_finalizer_light=False,
+                               contains_weakptr=False):
         if (has_finalizer or
             (raw_malloc_usage(size) > self.lb_young_fixedsize and
              raw_malloc_usage(size) > self.largest_young_fixedsize)):
@@ -179,6 +181,7 @@
             # "non-simple" case or object too big: don't use the nursery
             return SemiSpaceGC.malloc_fixedsize_clear(self, typeid, size,
                                                       has_finalizer,
+                                                      is_finalizer_light,
                                                       contains_weakptr)
         size_gc_header = self.gcheaderbuilder.size_gc_header
         totalsize = size_gc_header + size
diff --git a/pypy/rpython/memory/gc/marksweep.py b/pypy/rpython/memory/gc/marksweep.py
--- a/pypy/rpython/memory/gc/marksweep.py
+++ b/pypy/rpython/memory/gc/marksweep.py
@@ -93,7 +93,8 @@
         pass
 
     def malloc_fixedsize(self, typeid16, size,
-                         has_finalizer=False, contains_weakptr=False):
+                         has_finalizer=False, is_finalizer_light=False,
+                         contains_weakptr=False):
         self.maybe_collect()
         size_gc_header = self.gcheaderbuilder.size_gc_header
         try:
@@ -128,7 +129,9 @@
     malloc_fixedsize._dont_inline_ = True
 
     def malloc_fixedsize_clear(self, typeid16, size,
-                               has_finalizer=False, contains_weakptr=False):
+                               has_finalizer=False,
+                               is_finalizer_light=False,
+                               contains_weakptr=False):
         self.maybe_collect()
         size_gc_header = self.gcheaderbuilder.size_gc_header
         try:
diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -290,6 +290,8 @@
         #
         # A list of all objects with finalizers (these are never young).
         self.objects_with_finalizers = self.AddressDeque()
+        self.young_objects_with_light_finalizers = self.AddressStack()
+        self.old_objects_with_light_finalizers = self.AddressStack()
         #
         # Two lists of the objects with weakrefs.  No weakref can be an
         # old object weakly pointing to a young object: indeed, weakrefs
@@ -457,14 +459,16 @@
 
 
     def malloc_fixedsize_clear(self, typeid, size,
-                               needs_finalizer=False, contains_weakptr=False):
+                               needs_finalizer=False,
+                               is_finalizer_light=False,
+                               contains_weakptr=False):
         size_gc_header = self.gcheaderbuilder.size_gc_header
         totalsize = size_gc_header + size
         rawtotalsize = raw_malloc_usage(totalsize)
         #
         # If the object needs a finalizer, ask for a rawmalloc.
         # The following check should be constant-folded.
-        if needs_finalizer:
+        if needs_finalizer and not is_finalizer_light:
             ll_assert(not contains_weakptr,
                      "'needs_finalizer' and 'contains_weakptr' both specified")
             obj = self.external_malloc(typeid, 0, can_make_young=False)
@@ -494,13 +498,14 @@
             #
             # Build the object.
             llarena.arena_reserve(result, totalsize)
+            obj = result + size_gc_header
+            if is_finalizer_light:
+                self.young_objects_with_light_finalizers.append(obj)
             self.init_gc_object(result, typeid, flags=0)
             #
             # If it is a weakref, record it (check constant-folded).
             if contains_weakptr:
-                self.young_objects_with_weakrefs.append(result+size_gc_header)
-            #
-            obj = result + size_gc_header
+                self.young_objects_with_weakrefs.append(obj)
         #
         return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
 
@@ -1264,6 +1269,8 @@
         # weakrefs' targets.
         if self.young_objects_with_weakrefs.non_empty():
             self.invalidate_young_weakrefs()
+        if self.young_objects_with_light_finalizers.non_empty():
+            self.deal_with_young_objects_with_finalizers()
         #
         # Clear this mapping.
         if self.nursery_objects_shadows.length() > 0:
@@ -1584,6 +1591,9 @@
         # Weakref support: clear the weak pointers to dying objects
         if self.old_objects_with_weakrefs.non_empty():
             self.invalidate_old_weakrefs()
+        if self.old_objects_with_light_finalizers.non_empty():
+            self.deal_with_old_objects_with_finalizers()
+
         #
         # Walk all rawmalloced objects and free the ones that don't
         # have the GCFLAG_VISITED flag.
@@ -1649,8 +1659,7 @@
         if self.header(obj).tid & GCFLAG_VISITED:
             self.header(obj).tid &= ~GCFLAG_VISITED
             return False     # survives
-        else:
-            return True      # dies
+        return True      # dies
 
     def _reset_gcflag_visited(self, obj, ignored):
         self.header(obj).tid &= ~GCFLAG_VISITED
@@ -1829,6 +1838,39 @@
     # ----------
     # Finalizers
 
+    def deal_with_young_objects_with_finalizers(self):
+        """ This is a much simpler version of dealing with finalizers
+        and an optimization - we can reasonably assume that those finalizers
+        don't do anything fancy and *just* call them. Among other things
+        they won't resurrect objects
+        """
+        while self.young_objects_with_light_finalizers.non_empty():
+            obj = self.young_objects_with_light_finalizers.pop()
+            if not self.is_forwarded(obj):
+                finalizer = self.getlightfinalizer(self.get_type_id(obj))
+                ll_assert(bool(finalizer), "no light finalizer found")
+                finalizer(obj, llmemory.NULL)
+
+    def deal_with_old_objects_with_finalizers(self):
+        """ This is a much simpler version of dealing with finalizers
+        and an optimization - we can reasonably assume that those finalizers
+        don't do anything fancy and *just* call them. Among other things
+        they won't resurrect objects
+        """
+        new_objects = self.AddressStack()
+        while self.old_objects_with_light_finalizers.non_empty():
+            obj = self.old_objects_with_light_finalizers.pop()
+            if self.header(obj).tid & GCFLAG_VISITED:
+                # surviving
+                new_objects.append(obj)
+            else:
+                # dying
+                finalizer = self.getlightfinalizer(self.get_type_id(obj))
+                ll_assert(bool(finalizer), "no light finalizer found")
+                finalizer(obj, llmemory.NULL)
+        self.old_objects_with_light_finalizers.delete()
+        self.old_objects_with_light_finalizers = new_objects
+
     def deal_with_objects_with_finalizers(self):
         # Walk over list of objects with finalizers.
         # If it is not surviving, add it to the list of to-be-called
@@ -1959,7 +2001,6 @@
             #
             self.old_objects_with_weakrefs.append(obj)
 
-
     def invalidate_old_weakrefs(self):
         """Called during a major collection."""
         # walk over list of objects that contain weakrefs
diff --git a/pypy/rpython/memory/gc/semispace.py b/pypy/rpython/memory/gc/semispace.py
--- a/pypy/rpython/memory/gc/semispace.py
+++ b/pypy/rpython/memory/gc/semispace.py
@@ -82,6 +82,7 @@
         self.free = self.tospace
         MovingGCBase.setup(self)
         self.objects_with_finalizers = self.AddressDeque()
+        self.objects_with_light_finalizers = self.AddressStack()
         self.objects_with_weakrefs = self.AddressStack()
 
     def _teardown(self):
@@ -93,7 +94,9 @@
     # because the spaces are filled with zeroes in advance.
 
     def malloc_fixedsize_clear(self, typeid16, size,
-                               has_finalizer=False, contains_weakptr=False):
+                               has_finalizer=False,
+                               is_finalizer_light=False,
+                               contains_weakptr=False):
         size_gc_header = self.gcheaderbuilder.size_gc_header
         totalsize = size_gc_header + size
         result = self.free
@@ -102,7 +105,9 @@
         llarena.arena_reserve(result, totalsize)
         self.init_gc_object(result, typeid16)
         self.free = result + totalsize
-        if has_finalizer:
+        if is_finalizer_light:
+            self.objects_with_light_finalizers.append(result + size_gc_header)
+        elif has_finalizer:
             self.objects_with_finalizers.append(result + size_gc_header)
         if contains_weakptr:
             self.objects_with_weakrefs.append(result + size_gc_header)
@@ -263,6 +268,8 @@
         if self.run_finalizers.non_empty():
             self.update_run_finalizers()
         scan = self.scan_copied(scan)
+        if self.objects_with_light_finalizers.non_empty():
+            self.deal_with_objects_with_light_finalizers()
         if self.objects_with_finalizers.non_empty():
             scan = self.deal_with_objects_with_finalizers(scan)
         if self.objects_with_weakrefs.non_empty():
@@ -471,6 +478,23 @@
         # immortal objects always have GCFLAG_FORWARDED set;
         # see get_forwarding_address().
 
+    def deal_with_objects_with_light_finalizers(self):
+        """ This is a much simpler version of dealing with finalizers
+        and an optimization - we can reasonably assume that those finalizers
+        don't do anything fancy and *just* call them. Among other things
+        they won't resurrect objects
+        """
+        new_objects = self.AddressStack()
+        while self.objects_with_light_finalizers.non_empty():
+            obj = self.objects_with_light_finalizers.pop()
+            if self.surviving(obj):
+                new_objects.append(self.get_forwarding_address(obj))
+            else:
+                finalizer = self.getfinalizer(self.get_type_id(obj))
+                finalizer(obj, llmemory.NULL)
+        self.objects_with_light_finalizers.delete()
+        self.objects_with_light_finalizers = new_objects
+
     def deal_with_objects_with_finalizers(self, scan):
         # walk over list of objects with finalizers
         # if it is not copied, add it to the list of to-be-called finalizers
diff --git a/pypy/rpython/memory/gctransform/framework.py b/pypy/rpython/memory/gctransform/framework.py
--- a/pypy/rpython/memory/gctransform/framework.py
+++ b/pypy/rpython/memory/gctransform/framework.py
@@ -12,6 +12,7 @@
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.translator.backendopt import graphanalyze
 from pypy.translator.backendopt.support import var_needsgc
+from pypy.translator.backendopt.finalizer import FinalizerAnalyzer
 from pypy.annotation import model as annmodel
 from pypy.rpython import annlowlevel
 from pypy.rpython.rbuiltin import gen_cast
@@ -258,6 +259,7 @@
             [s_gc, s_typeid16,
              annmodel.SomeInteger(nonneg=True),
              annmodel.SomeBool(),
+             annmodel.SomeBool(),
              annmodel.SomeBool()], s_gcref,
             inline = False)
         if hasattr(GCClass, 'malloc_fixedsize'):
@@ -267,6 +269,7 @@
                 [s_gc, s_typeid16,
                  annmodel.SomeInteger(nonneg=True),
                  annmodel.SomeBool(),
+                 annmodel.SomeBool(),
                  annmodel.SomeBool()], s_gcref,
                 inline = False)
         else:
@@ -319,7 +322,7 @@
             raise NotImplementedError("GC needs write barrier, but does not provide writebarrier_before_copy functionality")
 
         # in some GCs we can inline the common case of
-        # malloc_fixedsize(typeid, size, True, False, False)
+        # malloc_fixedsize(typeid, size, False, False, False)
         if getattr(GCClass, 'inline_simple_malloc', False):
             # make a copy of this function so that it gets annotated
             # independently and the constants are folded inside
@@ -337,7 +340,7 @@
                 malloc_fast,
                 [s_gc, s_typeid16,
                  annmodel.SomeInteger(nonneg=True),
-                 s_False, s_False], s_gcref,
+                 s_False, s_False, s_False], s_gcref,
                 inline = True)
         else:
             self.malloc_fast_ptr = None
@@ -668,7 +671,13 @@
         kind_and_fptr = self.special_funcptr_for_type(TYPE)
         has_finalizer = (kind_and_fptr is not None and
                          kind_and_fptr[0] == "finalizer")
+        has_light_finalizer = (kind_and_fptr is not None and
+                               kind_and_fptr[0] == "light_finalizer")
+        if has_light_finalizer:
+            has_finalizer = True
         c_has_finalizer = rmodel.inputconst(lltype.Bool, has_finalizer)
+        c_has_light_finalizer = rmodel.inputconst(lltype.Bool,
+                                                  has_light_finalizer)
 
         if not op.opname.endswith('_varsize') and not flags.get('varsize'):
             #malloc_ptr = self.malloc_fixedsize_ptr
@@ -682,7 +691,8 @@
             else:
                 malloc_ptr = self.malloc_fixedsize_ptr
             args = [self.c_const_gc, c_type_id, c_size,
-                    c_has_finalizer, rmodel.inputconst(lltype.Bool, False)]
+                    c_has_finalizer, c_has_light_finalizer,
+                    rmodel.inputconst(lltype.Bool, False)]
         else:
             assert not c_has_finalizer.value
             info_varsize = self.layoutbuilder.get_info_varsize(type_id)
@@ -847,12 +857,13 @@
         # used by the JIT (see pypy.jit.backend.llsupport.gc)
         op = hop.spaceop
         [v_typeid, v_size,
-         v_has_finalizer, v_contains_weakptr] = op.args
+         v_has_finalizer, v_has_light_finalizer, v_contains_weakptr] = op.args
         livevars = self.push_roots(hop)
         hop.genop("direct_call",
                   [self.malloc_fixedsize_clear_ptr, self.c_const_gc,
                    v_typeid, v_size,
-                   v_has_finalizer, v_contains_weakptr],
+                   v_has_finalizer, v_has_light_finalizer,
+                   v_contains_weakptr],
                   resultvar=op.result)
         self.pop_roots(hop, livevars)
 
@@ -912,10 +923,10 @@
         info = self.layoutbuilder.get_info(type_id)
         c_size = rmodel.inputconst(lltype.Signed, info.fixedsize)
         malloc_ptr = self.malloc_fixedsize_ptr
-        c_has_finalizer = rmodel.inputconst(lltype.Bool, False)
+        c_false = rmodel.inputconst(lltype.Bool, False)
         c_has_weakptr = rmodel.inputconst(lltype.Bool, True)
         args = [self.c_const_gc, c_type_id, c_size,
-                c_has_finalizer, c_has_weakptr]
+                c_false, c_false, c_has_weakptr]
 
         # push and pop the current live variables *including* the argument
         # to the weakref_create operation, which must be kept alive and
@@ -1250,6 +1261,7 @@
             lltype2vtable = translator.rtyper.lltype2vtable
         else:
             lltype2vtable = None
+        self.translator = translator
         super(TransformerLayoutBuilder, self).__init__(GCClass, lltype2vtable)
 
     def has_finalizer(self, TYPE):
@@ -1257,6 +1269,10 @@
         return rtti is not None and getattr(rtti._obj, 'destructor_funcptr',
                                             None)
 
+    def has_light_finalizer(self, TYPE):
+        special = self.special_funcptr_for_type(TYPE)
+        return special is not None and special[0] == 'light_finalizer'
+
     def has_custom_trace(self, TYPE):
         rtti = get_rtti(TYPE)
         return rtti is not None and getattr(rtti._obj, 'custom_trace_funcptr',
@@ -1264,7 +1280,7 @@
 
     def make_finalizer_funcptr_for_type(self, TYPE):
         if not self.has_finalizer(TYPE):
-            return None
+            return None, False
         rtti = get_rtti(TYPE)
         destrptr = rtti._obj.destructor_funcptr
         DESTR_ARG = lltype.typeOf(destrptr).TO.ARGS[0]
@@ -1276,7 +1292,9 @@
             return llmemory.NULL
         fptr = self.transformer.annotate_finalizer(ll_finalizer,
                 [llmemory.Address, llmemory.Address], llmemory.Address)
-        return fptr
+        g = destrptr._obj.graph
+        light = not FinalizerAnalyzer(self.translator).analyze_light_finalizer(g)
+        return fptr, light
 
     def make_custom_trace_funcptr_for_type(self, TYPE):
         if not self.has_custom_trace(TYPE):
diff --git a/pypy/rpython/memory/gctypelayout.py b/pypy/rpython/memory/gctypelayout.py
--- a/pypy/rpython/memory/gctypelayout.py
+++ b/pypy/rpython/memory/gctypelayout.py
@@ -1,7 +1,6 @@
 from pypy.rpython.lltypesystem import lltype, llmemory, llarena, llgroup
 from pypy.rpython.lltypesystem import rclass
 from pypy.rpython.lltypesystem.lloperation import llop
-from pypy.rlib.objectmodel import we_are_translated
 from pypy.rlib.debug import ll_assert
 from pypy.rlib.rarithmetic import intmask
 from pypy.tool.identity_dict import identity_dict
@@ -85,6 +84,13 @@
         else:
             return lltype.nullptr(GCData.FINALIZER_OR_CT_FUNC)
 
+    def q_light_finalizer(self, typeid):
+        typeinfo = self.get(typeid)
+        if typeinfo.infobits & T_HAS_LIGHTWEIGHT_FINALIZER:
+            return typeinfo.finalizer_or_customtrace
+        else:
+            return lltype.nullptr(GCData.FINALIZER_OR_CT_FUNC)        
+
     def q_offsets_to_gc_pointers(self, typeid):
         return self.get(typeid).ofstoptrs
 
@@ -142,6 +148,7 @@
             self.q_has_gcptr_in_varsize,
             self.q_is_gcarrayofgcptr,
             self.q_finalizer,
+            self.q_light_finalizer,
             self.q_offsets_to_gc_pointers,
             self.q_fixed_size,
             self.q_varsize_item_sizes,
@@ -157,16 +164,17 @@
 
 
 # the lowest 16bits are used to store group member index
-T_MEMBER_INDEX         =   0xffff
-T_IS_VARSIZE           = 0x010000
-T_HAS_GCPTR_IN_VARSIZE = 0x020000
-T_IS_GCARRAY_OF_GCPTR  = 0x040000
-T_IS_WEAKREF           = 0x080000
-T_IS_RPYTHON_INSTANCE  = 0x100000    # the type is a subclass of OBJECT
-T_HAS_FINALIZER        = 0x200000
-T_HAS_CUSTOM_TRACE     = 0x400000
-T_KEY_MASK             = intmask(0xFF000000)
-T_KEY_VALUE            = intmask(0x5A000000)    # bug detection only
+T_MEMBER_INDEX              =   0xffff
+T_IS_VARSIZE                = 0x010000
+T_HAS_GCPTR_IN_VARSIZE      = 0x020000
+T_IS_GCARRAY_OF_GCPTR       = 0x040000
+T_IS_WEAKREF                = 0x080000
+T_IS_RPYTHON_INSTANCE       = 0x100000 # the type is a subclass of OBJECT
+T_HAS_FINALIZER             = 0x200000
+T_HAS_CUSTOM_TRACE          = 0x400000
+T_HAS_LIGHTWEIGHT_FINALIZER = 0x800000
+T_KEY_MASK                  = intmask(0xFF000000)
+T_KEY_VALUE                 = intmask(0x5A000000) # bug detection only
 
 def _check_valid_type_info(p):
     ll_assert(p.infobits & T_KEY_MASK == T_KEY_VALUE, "invalid type_id")
@@ -194,6 +202,8 @@
         info.finalizer_or_customtrace = fptr
         if kind == "finalizer":
             infobits |= T_HAS_FINALIZER
+        elif kind == 'light_finalizer':
+            infobits |= T_HAS_FINALIZER | T_HAS_LIGHTWEIGHT_FINALIZER
         elif kind == "custom_trace":
             infobits |= T_HAS_CUSTOM_TRACE
         else:
@@ -367,12 +377,15 @@
     def special_funcptr_for_type(self, TYPE):
         if TYPE in self._special_funcptrs:
             return self._special_funcptrs[TYPE]
-        fptr1 = self.make_finalizer_funcptr_for_type(TYPE)
+        fptr1, is_lightweight = self.make_finalizer_funcptr_for_type(TYPE)
         fptr2 = self.make_custom_trace_funcptr_for_type(TYPE)
         assert not (fptr1 and fptr2), (
             "type %r needs both a finalizer and a custom tracer" % (TYPE,))
         if fptr1:
-            kind_and_fptr = "finalizer", fptr1
+            if is_lightweight:
+                kind_and_fptr = "light_finalizer", fptr1
+            else:
+                kind_and_fptr = "finalizer", fptr1
         elif fptr2:
             kind_and_fptr = "custom_trace", fptr2
         else:
@@ -382,7 +395,7 @@
 
     def make_finalizer_funcptr_for_type(self, TYPE):
         # must be overridden for proper finalizer support
-        return None
+        return None, False
 
     def make_custom_trace_funcptr_for_type(self, TYPE):
         # must be overridden for proper custom tracer support
diff --git a/pypy/rpython/memory/gcwrapper.py b/pypy/rpython/memory/gcwrapper.py
--- a/pypy/rpython/memory/gcwrapper.py
+++ b/pypy/rpython/memory/gcwrapper.py
@@ -1,3 +1,4 @@
+from pypy.translator.backendopt.finalizer import FinalizerAnalyzer
 from pypy.rpython.lltypesystem import lltype, llmemory, llheap
 from pypy.rpython import llinterp
 from pypy.rpython.annlowlevel import llhelper
@@ -196,9 +197,11 @@
             DESTR_ARG = lltype.typeOf(destrptr).TO.ARGS[0]
             destrgraph = destrptr._obj.graph
         else:
-            return None
+            return None, False
 
         assert not type_contains_pyobjs(TYPE), "not implemented"
+        t = self.llinterp.typer.annotator.translator
+        light = not FinalizerAnalyzer(t).analyze_light_finalizer(destrgraph)
         def ll_finalizer(addr, dummy):
             assert dummy == llmemory.NULL
             try:
@@ -208,7 +211,7 @@
                 raise RuntimeError(
                     "a finalizer raised an exception, shouldn't happen")
             return llmemory.NULL
-        return llhelper(gctypelayout.GCData.FINALIZER_OR_CT, ll_finalizer)
+        return llhelper(gctypelayout.GCData.FINALIZER_OR_CT, ll_finalizer), light
 
     def make_custom_trace_funcptr_for_type(self, TYPE):
         from pypy.rpython.memory.gctransform.support import get_rtti, \
diff --git a/pypy/rpython/memory/test/test_gc.py b/pypy/rpython/memory/test/test_gc.py
--- a/pypy/rpython/memory/test/test_gc.py
+++ b/pypy/rpython/memory/test/test_gc.py
@@ -5,7 +5,6 @@
 from pypy.rpython.memory.test import snippet
 from pypy.rpython.test.test_llinterp import get_interpreter
 from pypy.rpython.lltypesystem import lltype
-from pypy.rpython.lltypesystem.rstr import STR
 from pypy.rpython.lltypesystem.lloperation import llop
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.rlib.objectmodel import compute_unique_id
@@ -57,7 +56,7 @@
                 while j < 20:
                     j += 1
                     a.append(j)
-        res = self.interpret(malloc_a_lot, [])
+        self.interpret(malloc_a_lot, [])
         #assert simulator.current_size - curr < 16000 * INT_SIZE / 4
         #print "size before: %s, size after %s" % (curr, simulator.current_size)
 
@@ -73,7 +72,7 @@
                 while j < 20:
                     j += 1
                     b.append((1, j, i))
-        res = self.interpret(malloc_a_lot, [])
+        self.interpret(malloc_a_lot, [])
         #assert simulator.current_size - curr < 16000 * INT_SIZE / 4
         #print "size before: %s, size after %s" % (curr, simulator.current_size)
 
@@ -129,7 +128,7 @@
         res = self.interpret(concat, [100])
         assert res == concat(100)
         #assert simulator.current_size - curr < 16000 * INT_SIZE / 4
-        
+
     def test_finalizer(self):
         class B(object):
             pass
@@ -278,7 +277,7 @@
                                self.interpret, f, [])
 
     def test_weakref(self):
-        import weakref, gc
+        import weakref
         class A(object):
             pass
         def g():
@@ -299,7 +298,7 @@
         assert res
 
     def test_weakref_to_object_with_finalizer(self):
-        import weakref, gc
+        import weakref
         class A(object):
             count = 0
         a = A()
@@ -338,7 +337,7 @@
         assert res
 
     def test_cycle_with_weakref_and_del(self):
-        import weakref, gc
+        import weakref
         class A(object):
             count = 0
         a = A()
@@ -367,7 +366,7 @@
         assert res == 11
 
     def test_weakref_to_object_with_finalizer_ordering(self):
-        import weakref, gc
+        import weakref
         class A(object):
             count = 0
         a = A()
@@ -616,7 +615,7 @@
                     assert not rgc.can_move(a)
                     return 1
                 return 0
-            except Exception, e:
+            except Exception:
                 return 2
 
         assert self.interpret(func, []) == int(self.GC_CAN_MALLOC_NONMOVABLE)
@@ -647,8 +646,6 @@
         assert self.interpret(f, [bigsize, 0, flag]) == 0x62024241
 
     def test_tagged_simple(self):
-        from pypy.rlib.objectmodel import UnboxedValue
-
         class Unrelated(object):
             pass
 
@@ -689,8 +686,6 @@
         assert res == -897
 
     def test_tagged_id(self):
-        from pypy.rlib.objectmodel import UnboxedValue, compute_unique_id
-
         class Unrelated(object):
             pass
 
diff --git a/pypy/rpython/memory/test/test_transformed_gc.py b/pypy/rpython/memory/test/test_transformed_gc.py
--- a/pypy/rpython/memory/test/test_transformed_gc.py
+++ b/pypy/rpython/memory/test/test_transformed_gc.py
@@ -345,22 +345,22 @@
         b = B()
         b.nextid = 0
         b.num_deleted = 0
-        class A(object):
+        class AAA(object):
             def __init__(self):
                 self.id = b.nextid
                 b.nextid += 1
             def __del__(self):
                 b.num_deleted += 1
                 C()
-        class C(A):
+        class C(AAA):
             def __del__(self):
                 b.num_deleted += 1
         def f(x, y):
-            a = A()
+            a = AAA()
             i = 0
             while i < x:
                 i += 1
-                a = A()
+                a = AAA()
             llop.gc__collect(lltype.Void)
             llop.gc__collect(lltype.Void)
             return b.num_deleted
@@ -807,6 +807,7 @@
                     op.args = [Constant(type_id, llgroup.HALFWORD),
                                Constant(llmemory.sizeof(P), lltype.Signed),
                                Constant(False, lltype.Bool), # has_finalizer
+                               Constant(False, lltype.Bool), # is_finalizer_light
                                Constant(False, lltype.Bool)] # contains_weakptr
                     break
             else:
@@ -843,6 +844,7 @@
                     op.args = [Constant(type_id, llgroup.HALFWORD),
                                Constant(llmemory.sizeof(P), lltype.Signed),
                                Constant(False, lltype.Bool), # has_finalizer
+                               Constant(False, lltype.Bool), # is_finalizer_light
                                Constant(False, lltype.Bool)] # contains_weakptr
                     break
             else:
diff --git a/pypy/rpython/rtyper.py b/pypy/rpython/rtyper.py
--- a/pypy/rpython/rtyper.py
+++ b/pypy/rpython/rtyper.py
@@ -717,7 +717,7 @@
             raise TyperError("runtime type info function %r returns %r, "
                              "excepted Ptr(RuntimeTypeInfo)" % (func, s))
         funcptr = self.getcallable(graph)
-        attachRuntimeTypeInfo(GCSTRUCT, funcptr, destrptr)
+        attachRuntimeTypeInfo(GCSTRUCT, funcptr, destrptr, None)
 
 # register operations from annotation model
 RPythonTyper._registeroperations(annmodel)
diff --git a/pypy/rpython/test/test_rclass.py b/pypy/rpython/test/test_rclass.py
--- a/pypy/rpython/test/test_rclass.py
+++ b/pypy/rpython/test/test_rclass.py
@@ -3,7 +3,7 @@
 from pypy.translator.translator import TranslationContext, graphof
 from pypy.rpython.lltypesystem.lltype import *
 from pypy.rpython.ootypesystem import ootype
-from pypy.rlib.rarithmetic import intmask, r_longlong
+from pypy.rlib.rarithmetic import r_longlong
 from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin
 from pypy.rpython.rclass import IR_IMMUTABLE, IR_IMMUTABLE_ARRAY
 from pypy.rpython.rclass import IR_QUASIIMMUTABLE, IR_QUASIIMMUTABLE_ARRAY
@@ -972,10 +972,10 @@
         graph = graphof(t, f)
         TYPE = graph.startblock.operations[0].args[0].value
         RTTI = getRuntimeTypeInfo(TYPE)
-        queryptr = RTTI._obj.query_funcptr # should not raise
+        RTTI._obj.query_funcptr # should not raise
         destrptr = RTTI._obj.destructor_funcptr
         assert destrptr is not None
-    
+
     def test_del_inheritance(self):
         from pypy.rlib import rgc
         class State:
diff --git a/pypy/translator/backendopt/finalizer.py b/pypy/translator/backendopt/finalizer.py
new file mode 100644
--- /dev/null
+++ b/pypy/translator/backendopt/finalizer.py
@@ -0,0 +1,46 @@
+
+from pypy.translator.backendopt import graphanalyze
+from pypy.rpython.lltypesystem import lltype
+
+class FinalizerError(Exception):
+    """ __del__ marked as lightweight finalizer, but the analyzer did
+    not agreed
+    """
+
+class FinalizerAnalyzer(graphanalyze.BoolGraphAnalyzer):
+    """ Analyzer that determines whether a finalizer is lightweight enough
+    so it can be called without all the complicated logic in the garbage
+    collector. The set of operations here is restrictive for a good reason
+    - it's better to be safe. Specifically disallowed operations:
+
+    * anything that escapes self
+    * anything that can allocate
+    """
+    ok_operations = ['ptr_nonzero', 'ptr_eq', 'ptr_ne', 'free', 'same_as',
+                     'direct_ptradd', 'force_cast', 'track_alloc_stop',
+                     'raw_free']
+    
+    def analyze_light_finalizer(self, graph):
+        result = self.analyze_direct_call(graph)
+        if (result is self.top_result() and
+            getattr(graph.func, '_is_light_finalizer_', False)):
+            raise FinalizerError(FinalizerError.__doc__, graph)
+        return result
+    
+    def analyze_simple_operation(self, op, graphinfo):
+        if op.opname in self.ok_operations:
+            return self.bottom_result()
+        if (op.opname.startswith('int_') or op.opname.startswith('float_')
+            or op.opname.startswith('cast_')):
+            return self.bottom_result()
+        if op.opname == 'setfield' or op.opname == 'bare_setfield':
+            TP = op.args[2].concretetype
+            if not isinstance(TP, lltype.Ptr) or TP.TO._gckind == 'raw':
+                # primitive type
+                return self.bottom_result()
+        if op.opname == 'getfield':
+            TP = op.result.concretetype
+            if not isinstance(TP, lltype.Ptr) or TP.TO._gckind == 'raw':
+                # primitive type
+                return self.bottom_result()            
+        return self.top_result()
diff --git a/pypy/translator/backendopt/test/test_finalizer.py b/pypy/translator/backendopt/test/test_finalizer.py
new file mode 100644
--- /dev/null
+++ b/pypy/translator/backendopt/test/test_finalizer.py
@@ -0,0 +1,142 @@
+
+import py
+from pypy.translator.backendopt.finalizer import FinalizerAnalyzer,\
+     FinalizerError
+from pypy.translator.translator import TranslationContext, graphof
+from pypy.translator.backendopt.all import backend_optimizations
+from pypy.translator.unsimplify import varoftype
+from pypy.rpython.lltypesystem import lltype, rffi
+from pypy.conftest import option
+from pypy.rlib import rgc
+
+
+class BaseFinalizerAnalyzerTests(object):
+    """ Below are typical destructors that we encounter in pypy
+    """
+
+    type_system = None
+    
+    def analyze(self, func, sig, func_to_analyze=None, backendopt=False):
+        if func_to_analyze is None:
+            func_to_analyze = func
+        t = TranslationContext()
+        t.buildannotator().build_types(func, sig)
+        t.buildrtyper(type_system=self.type_system).specialize()
+        if backendopt:
+            backend_optimizations(t)
+        if option.view:
+            t.view()
+        a = FinalizerAnalyzer(t)
+        fgraph = graphof(t, func_to_analyze)
+        result = a.analyze_light_finalizer(fgraph)
+        return result
+
+    def test_nothing(self):
+        def f():
+            pass
+        r = self.analyze(f, [])
+        assert not r
+
+def test_various_ops():
+    from pypy.objspace.flow.model import SpaceOperation, Constant
+
+    X = lltype.Ptr(lltype.GcStruct('X'))
+    Z = lltype.Ptr(lltype.Struct('Z'))
+    S = lltype.GcStruct('S', ('x', lltype.Signed),
+                        ('y', X),
+                        ('z', Z))
+    v1 = varoftype(lltype.Bool)
+    v2 = varoftype(lltype.Signed)
+    f = FinalizerAnalyzer(None)
+    r = f.analyze(SpaceOperation('cast_int_to_bool', [v2],
+                                                       v1))
+    assert not r
+    v1 = varoftype(lltype.Ptr(S))
+    v2 = varoftype(lltype.Signed)
+    v3 = varoftype(X)
+    v4 = varoftype(Z)
+    assert not f.analyze(SpaceOperation('bare_setfield', [v1, Constant('x'),
+                                                          v2], None))
+    assert     f.analyze(SpaceOperation('bare_setfield', [v1, Constant('y'),
+                                                          v3], None))
+    assert not f.analyze(SpaceOperation('bare_setfield', [v1, Constant('z'),
+                                                          v4], None))
+    
+        
+class TestLLType(BaseFinalizerAnalyzerTests):
+    type_system = 'lltype'
+
+    def test_malloc(self):
+        S = lltype.GcStruct('S')
+        
+        def f():
+            return lltype.malloc(S)
+
+        r = self.analyze(f, [])
+        assert r
+
+    def test_raw_free_getfield(self):
+        S = lltype.Struct('S')
+        
+        class A(object):
+            def __init__(self):
+                self.x = lltype.malloc(S, flavor='raw')
+
+            def __del__(self):
+                if self.x:
+                    self.x = lltype.nullptr(S)
+                    lltype.free(self.x, flavor='raw')
+
+        def f():
+            return A()
+
+        r = self.analyze(f, [], A.__del__.im_func)
+        assert not r
+
+    def test_c_call(self):
+        C = rffi.CArray(lltype.Signed)
+        c = rffi.llexternal('x', [lltype.Ptr(C)], lltype.Signed)
+
+        def g():
+            p = lltype.malloc(C, 3, flavor='raw')
+            f(p)
+        
+        def f(p):
+            c(rffi.ptradd(p, 0))
+            lltype.free(p, flavor='raw')
+
+        r = self.analyze(g, [], f, backendopt=True)
+        assert not r
+
+    def test_chain(self):
+        class B(object):
+            def __init__(self):
+                self.counter = 1
+        
+        class A(object):
+            def __init__(self):
+                self.x = B()
+
+            def __del__(self):
+                self.x.counter += 1
+
+        def f():
+            A()
+
+        r = self.analyze(f, [], A.__del__.im_func)
+        assert r
+
+    def test_is_light_finalizer_decorator(self):
+        S = lltype.GcStruct('S')
+
+        @rgc.is_light_finalizer
+        def f():
+            lltype.malloc(S)
+        @rgc.is_light_finalizer
+        def g():
+            pass
+        self.analyze(g, []) # did not explode
+        py.test.raises(FinalizerError, self.analyze, f, [])
+
+class TestOOType(BaseFinalizerAnalyzerTests):
+    type_system = 'ootype'

From noreply at buildbot.pypy.org  Tue Oct 25 21:00:48 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 25 Oct 2011 21:00:48 +0200 (CEST)
Subject: [pypy-commit] pypy jit-simplify-backendintf: A branch to simplify
 the backend interface, killing set_future_value_xx().
Message-ID: <20111025190048.DA303820C7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: jit-simplify-backendintf
Changeset: r48450:26419ef5e993
Date: 2011-10-25 20:07 +0200
http://bitbucket.org/pypy/pypy/changeset/26419ef5e993/

Log:	A branch to simplify the backend interface, killing
	set_future_value_xx().


From noreply at buildbot.pypy.org  Tue Oct 25 21:00:50 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 25 Oct 2011 21:00:50 +0200 (CEST)
Subject: [pypy-commit] pypy jit-simplify-backendintf: Change the interface
 and fix the llgraph backend.
Message-ID: <20111025190050.1F857820C7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: jit-simplify-backendintf
Changeset: r48451:6e1590cee5e1
Date: 2011-10-25 20:30 +0200
http://bitbucket.org/pypy/pypy/changeset/6e1590cee5e1/

Log:	Change the interface and fix the llgraph backend.

diff --git a/pypy/jit/backend/llgraph/runner.py b/pypy/jit/backend/llgraph/runner.py
--- a/pypy/jit/backend/llgraph/runner.py
+++ b/pypy/jit/backend/llgraph/runner.py
@@ -257,22 +257,25 @@
         self.latest_frame = frame
         return fail_index
 
-    def execute_token(self, loop_token):
+    def execute_token(self, loop_token, *args):
         """Calls the assembler generated for the given loop.
         Returns the ResOperation that failed, of type rop.FAIL.
         """
+        # XXX RPythonize me
+        for index, x in enumerate(args):
+            TYPE = lltype.typeOf(x)
+            if TYPE == lltype.Signed:
+                llimpl.set_future_value_int(index, x)
+            elif TYPE == llmemory.GCREF:
+                llimpl.set_future_value_ref(index, x)
+            elif TYPE == longlong.FLOATSTORAGE:
+                llimpl.set_future_value_float(index, x)
+            else:
+                raise ValueError(TYPE)
+        #
         fail_index = self._execute_token(loop_token)
         return self.get_fail_descr_from_number(fail_index)
 
-    def set_future_value_int(self, index, intvalue):
-        llimpl.set_future_value_int(index, intvalue)
-
-    def set_future_value_ref(self, index, objvalue):
-        llimpl.set_future_value_ref(index, objvalue)
-
-    def set_future_value_float(self, index, floatvalue):
-        llimpl.set_future_value_float(index, floatvalue)
-
     def get_latest_value_int(self, index):
         return llimpl.frame_int_getvalue(self.latest_frame, index)
 
diff --git a/pypy/jit/backend/model.py b/pypy/jit/backend/model.py
--- a/pypy/jit/backend/model.py
+++ b/pypy/jit/backend/model.py
@@ -84,26 +84,16 @@
         """Print a disassembled version of looptoken to stdout"""
         raise NotImplementedError
 
-    def execute_token(self, looptoken):
+    def execute_token(self, looptoken, *args):
         """Execute the generated code referenced by the looptoken.
         Returns the descr of the last executed operation: either the one
         attached to the failing guard, or the one attached to the FINISH.
-        Use set_future_value_xxx() before, and get_latest_value_xxx() after.
+        Use get_latest_value_xxx() afterwards to read the result(s).
+        (This method is automatically specialized by the front-end if
+        needed, for various types and numbers of *args.)
         """
         raise NotImplementedError
 
-    def set_future_value_int(self, index, intvalue):
-        """Set the value for the index'th argument for the loop to run."""
-        raise NotImplementedError
-
-    def set_future_value_float(self, index, floatvalue):
-        """Set the value for the index'th argument for the loop to run."""
-        raise NotImplementedError
-
-    def set_future_value_ref(self, index, objvalue):
-        """Set the value for the index'th argument for the loop to run."""
-        raise NotImplementedError
-
     def get_latest_value_int(self, index):
         """Returns the value for the index'th argument to the
         last executed operation (from 'fail_args' if it was a guard,
diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -34,20 +34,17 @@
                                                                 descr)
         looptoken = LoopToken()
         self.cpu.compile_loop(inputargs, operations, looptoken)
-        j = 0
+        args = []
         for box in inputargs:
             if isinstance(box, BoxInt):
-                self.cpu.set_future_value_int(j, box.getint())
-                j += 1
+                args.append(box.getint())
             elif isinstance(box, (BoxPtr, BoxObj)):
-                self.cpu.set_future_value_ref(j, box.getref_base())
-                j += 1
+                args.append(box.getref_base())
             elif isinstance(box, BoxFloat):
-                self.cpu.set_future_value_float(j, box.getfloatstorage())
-                j += 1
+                args.append(box.getfloatstorage())
             else:
                 raise NotImplementedError(box)
-        res = self.cpu.execute_token(looptoken)
+        res = self.cpu.execute_token(looptoken, *args)
         if res is operations[-1].getdescr():
             self.guard_failed = False
         else:
@@ -108,8 +105,7 @@
         inputargs = [i0]
         looptoken = LoopToken()
         self.cpu.compile_loop(inputargs, operations, looptoken)
-        self.cpu.set_future_value_int(0, 2)
-        fail = self.cpu.execute_token(looptoken)
+        fail = self.cpu.execute_token(looptoken, 2)
         res = self.cpu.get_latest_value_int(0)
         assert res == 3
         assert fail.identifier == 1
@@ -129,8 +125,7 @@
         operations[2].setfailargs([i1])
 
         self.cpu.compile_loop(inputargs, operations, looptoken)
-        self.cpu.set_future_value_int(0, 2)
-        fail = self.cpu.execute_token(looptoken)
+        fail = self.cpu.execute_token(looptoken, 2)
         assert fail.identifier == 2
         res = self.cpu.get_latest_value_int(0)
         assert res == 10
@@ -150,8 +145,7 @@
         operations[2].setfailargs([None, None, i1, None])
 
         self.cpu.compile_loop(inputargs, operations, looptoken)
-        self.cpu.set_future_value_int(0, 2)
-        fail = self.cpu.execute_token(looptoken)
+        fail = self.cpu.execute_token(looptoken, 2)
         assert fail.identifier == 2
         res = self.cpu.get_latest_value_int(2)
         assert res == 10
@@ -212,8 +206,7 @@
 
         self.cpu.compile_bridge(faildescr1, [i1b], bridge, looptoken)
 
-        self.cpu.set_future_value_int(0, 2)
-        fail = self.cpu.execute_token(looptoken)
+        fail = self.cpu.execute_token(looptoken, 2)
         assert fail.identifier == 2
         res = self.cpu.get_latest_value_int(0)
         assert res == 20
@@ -250,8 +243,7 @@
 
         self.cpu.compile_bridge(faildescr1, [i1b], bridge, looptoken)
 
-        self.cpu.set_future_value_int(0, 2)
-        fail = self.cpu.execute_token(looptoken)
+        fail = self.cpu.execute_token(looptoken, 2)
         assert fail.identifier == 2
         res = self.cpu.get_latest_value_int(0)
         assert res == 20
@@ -272,8 +264,7 @@
         operations[2].setfailargs([None, i1, None])
         self.cpu.compile_loop(inputargs, operations, looptoken)
 
-        self.cpu.set_future_value_int(0, 2)
-        fail = self.cpu.execute_token(looptoken)
+        fail = self.cpu.execute_token(looptoken, 2)
         assert fail is faildescr1
 
         count = self.cpu.get_latest_value_count()
@@ -295,8 +286,7 @@
             ResOperation(rop.FINISH, [i0], None, descr=faildescr)
             ]
         self.cpu.compile_loop([i0], operations, looptoken)
-        self.cpu.set_future_value_int(0, 99)
-        fail = self.cpu.execute_token(looptoken)
+        fail = self.cpu.execute_token(looptoken, 99)
         assert fail is faildescr
         res = self.cpu.get_latest_value_int(0)
         assert res == 99
@@ -327,8 +317,7 @@
                 ]
             self.cpu.compile_loop([f0], operations, looptoken)
             value = longlong.getfloatstorage(-61.25)
-            self.cpu.set_future_value_float(0, value)
-            fail = self.cpu.execute_token(looptoken)
+            fail = self.cpu.execute_token(looptoken, value)
             assert fail is faildescr
             res = self.cpu.get_latest_value_float(0)
             assert longlong.getrealfloat(res) == -61.25
@@ -361,9 +350,7 @@
             ]
         operations[-2].setfailargs([t, z])
         cpu.compile_loop([x, y], operations, looptoken)
-        self.cpu.set_future_value_int(0, 0)
-        self.cpu.set_future_value_int(1, 10)
-        res = self.cpu.execute_token(looptoken)
+        res = self.cpu.execute_token(looptoken, 0, 10)
         assert self.cpu.get_latest_value_int(0) == 0
         assert self.cpu.get_latest_value_int(1) == 55
 
@@ -424,9 +411,7 @@
             for x, y, z in testcases:
                 excvalue = self.cpu.grab_exc_value()
                 assert not excvalue
-                self.cpu.set_future_value_int(0, x)
-                self.cpu.set_future_value_int(1, y)
-                fail = self.cpu.execute_token(looptoken)
+                fail = self.cpu.execute_token(looptoken, x, y)
                 if (z == boom) ^ reversed:
                     assert fail.identifier == 1
                 else:
@@ -1109,17 +1094,7 @@
                     assert 0
             values[index_counter] = 11
             #
-            for i, (box, val) in enumerate(zip(inputargs, values)):
-                if isinstance(box, BoxInt):
-                    self.cpu.set_future_value_int(i, val)
-                elif isinstance(box, BoxPtr):
-                    self.cpu.set_future_value_ref(i, val)
-                elif isinstance(box, BoxFloat):
-                    self.cpu.set_future_value_float(i, val)
-                else:
-                    assert 0
-            #
-            fail = self.cpu.execute_token(looptoken)
+            fail = self.cpu.execute_token(looptoken, *values)
             assert fail.identifier == 15
             #
             dstvalues = values[:]
@@ -1169,10 +1144,11 @@
 
         self.cpu.compile_bridge(faildescr1, fboxes2, bridge, looptoken)
 
+        args = []
         for i in range(len(fboxes)):
             x = 13.5 + 6.73 * i
-            self.cpu.set_future_value_float(i, longlong.getfloatstorage(x))
-        fail = self.cpu.execute_token(looptoken)
+            args.append(longlong.getfloatstorage(x))
+        fail = self.cpu.execute_token(looptoken, *args)
         assert fail.identifier == 2
         res = self.cpu.get_latest_value_float(0)
         assert longlong.getrealfloat(res) == 8.5
@@ -1222,14 +1198,12 @@
                         if test1 == -42 or combinaison[0] == 'b':
                             for test2 in [-65, -42, -11]:
                                 if test2 == -42 or combinaison[1] == 'b':
-                                    n = 0
+                                    args = []
                                     if combinaison[0] == 'b':
-                                        cpu.set_future_value_int(n, test1)
-                                        n += 1
+                                        args.append(test1)
                                     if combinaison[1] == 'b':
-                                        cpu.set_future_value_int(n, test2)
-                                        n += 1
-                                    fail = cpu.execute_token(looptoken)
+                                        args.append(test2)
+                                    fail = cpu.execute_token(looptoken, *args)
                                     #
                                     expected = compare(test1, test2)
                                     expected ^= guard_case
@@ -1281,16 +1255,14 @@
                         if test1 == -4.5 or combinaison[0] == 'b':
                             for test2 in [-6.5, -4.5, -2.5, nan]:
                                 if test2 == -4.5 or combinaison[1] == 'b':
-                                    n = 0
+                                    args = []
                                     if combinaison[0] == 'b':
-                                        cpu.set_future_value_float(
-                                            n, longlong.getfloatstorage(test1))
-                                        n += 1
+                                        args.append(
+                                            longlong.getfloatstorage(test1))
                                     if combinaison[1] == 'b':
-                                        cpu.set_future_value_float(
-                                            n, longlong.getfloatstorage(test2))
-                                        n += 1
-                                    fail = cpu.execute_token(looptoken)
+                                        args.append(
+                                            longlong.getfloatstorage(test2))
+                                    fail = cpu.execute_token(looptoken, *args)
                                     #
                                     expected = compare(test1, test2)
                                     expected ^= guard_case
@@ -1334,15 +1306,16 @@
         #
         self.cpu.compile_loop(inputargs, operations, looptoken)
         #
-        for i, box in enumerate(inputargs):
+        args = []
+        for box in inputargs:
             if isinstance(box, BoxInt):
-                self.cpu.set_future_value_int(i, box.getint())
+                args.append(box.getint())
             elif isinstance(box, BoxFloat):
-                self.cpu.set_future_value_float(i, box.getfloatstorage())
+                args.append(box.getfloatstorage())
             else:
                 assert 0
         #
-        fail = self.cpu.execute_token(looptoken)
+        fail = self.cpu.execute_token(looptoken, *args)
         assert fail.identifier == 1
 
     def test_nan_and_infinity(self):
@@ -1405,10 +1378,9 @@
                         unique_testcase_list = list(set(testcase))
                         self.cpu.compile_loop(unique_testcase_list, operations,
                                               looptoken)
-                        for i, box in enumerate(unique_testcase_list):
-                            self.cpu.set_future_value_float(
-                                i, box.getfloatstorage())
-                        fail = self.cpu.execute_token(looptoken)
+                        args = [box.getfloatstorage()
+                                for box in unique_testcase_list]
+                        fail = self.cpu.execute_token(looptoken, *args)
                         if fail.identifier != 5 - (expected_id^expected):
                             if fail.identifier == 4:
                                 msg = "was taken"
@@ -1676,14 +1648,12 @@
         exc_ptr = xptr
         loop = parse(ops, self.cpu, namespace=locals())
         self.cpu.compile_loop(loop.inputargs, loop.operations, loop.token)
-        self.cpu.set_future_value_int(0, 1)
-        self.cpu.execute_token(loop.token)
+        self.cpu.execute_token(loop.token, 1)
         assert self.cpu.get_latest_value_int(0) == 0
         assert self.cpu.get_latest_value_ref(1) == xptr
         excvalue = self.cpu.grab_exc_value()
         assert not excvalue
-        self.cpu.set_future_value_int(0, 0)
-        self.cpu.execute_token(loop.token)
+        self.cpu.execute_token(loop.token, 0)
         assert self.cpu.get_latest_value_int(0) == 1
         excvalue = self.cpu.grab_exc_value()
         assert not excvalue
@@ -1701,8 +1671,7 @@
         exc_ptr = yptr
         loop = parse(ops, self.cpu, namespace=locals())
         self.cpu.compile_loop(loop.inputargs, loop.operations, loop.token)
-        self.cpu.set_future_value_int(0, 1)
-        self.cpu.execute_token(loop.token)
+        self.cpu.execute_token(loop.token, 1)
         assert self.cpu.get_latest_value_int(0) == 1
         excvalue = self.cpu.grab_exc_value()
         assert excvalue == yptr
@@ -1719,13 +1688,11 @@
         '''
         loop = parse(ops, self.cpu, namespace=locals())
         self.cpu.compile_loop(loop.inputargs, loop.operations, loop.token)
-        self.cpu.set_future_value_int(0, 1)
-        self.cpu.execute_token(loop.token)
+        self.cpu.execute_token(loop.token, 1)
         assert self.cpu.get_latest_value_int(0) == 1
         excvalue = self.cpu.grab_exc_value()
         assert excvalue == xptr
-        self.cpu.set_future_value_int(0, 0)
-        self.cpu.execute_token(loop.token)
+        self.cpu.execute_token(loop.token, 0)
         assert self.cpu.get_latest_value_int(0) == 0
         excvalue = self.cpu.grab_exc_value()
         assert not excvalue
@@ -1897,16 +1864,12 @@
         ops[2].setfailargs([i1, i0])
         looptoken = LoopToken()
         self.cpu.compile_loop([i0, i1], ops, looptoken)
-        self.cpu.set_future_value_int(0, 20)
-        self.cpu.set_future_value_int(1, 0)
-        fail = self.cpu.execute_token(looptoken)
+        fail = self.cpu.execute_token(looptoken, 20, 0)
         assert fail.identifier == 0
         assert self.cpu.get_latest_value_int(0) == 20
         assert values == []
 
-        self.cpu.set_future_value_int(0, 10)
-        self.cpu.set_future_value_int(1, 1)
-        fail = self.cpu.execute_token(looptoken)
+        fail = self.cpu.execute_token(looptoken, 10, 1)
         assert fail.identifier == 1
         assert self.cpu.get_latest_value_int(0) == 1
         assert self.cpu.get_latest_value_int(1) == 10
@@ -1942,16 +1905,12 @@
         ops[2].setfailargs([i1, i2, i0])
         looptoken = LoopToken()
         self.cpu.compile_loop([i0, i1], ops, looptoken)
-        self.cpu.set_future_value_int(0, 20)
-        self.cpu.set_future_value_int(1, 0)
-        fail = self.cpu.execute_token(looptoken)
+        fail = self.cpu.execute_token(looptoken, 20, 0)
         assert fail.identifier == 0
         assert self.cpu.get_latest_value_int(0) == 42
         assert values == []
 
-        self.cpu.set_future_value_int(0, 10)
-        self.cpu.set_future_value_int(1, 1)
-        fail = self.cpu.execute_token(looptoken)
+        fail = self.cpu.execute_token(looptoken, 10, 1)
         assert fail.identifier == 1
         assert self.cpu.get_latest_value_int(0) == 1
         assert self.cpu.get_latest_value_int(1) == 42
@@ -1988,17 +1947,13 @@
         ops[2].setfailargs([i1, f2, i0])
         looptoken = LoopToken()
         self.cpu.compile_loop([i0, i1], ops, looptoken)
-        self.cpu.set_future_value_int(0, 20)
-        self.cpu.set_future_value_int(1, 0)
-        fail = self.cpu.execute_token(looptoken)
+        fail = self.cpu.execute_token(looptoken, 20, 0)
         assert fail.identifier == 0
         x = self.cpu.get_latest_value_float(0)
         assert longlong.getrealfloat(x) == 42.5
         assert values == []
 
-        self.cpu.set_future_value_int(0, 10)
-        self.cpu.set_future_value_int(1, 1)
-        fail = self.cpu.execute_token(looptoken)
+        fail = self.cpu.execute_token(looptoken, 10, 1)
         assert fail.identifier == 1
         assert self.cpu.get_latest_value_int(0) == 1
         x = self.cpu.get_latest_value_float(1)
@@ -2033,8 +1988,7 @@
         ops[1].setfailargs([i1, i2])
         looptoken = LoopToken()
         self.cpu.compile_loop([i1], ops, looptoken)
-        self.cpu.set_future_value_int(0, ord('G'))
-        fail = self.cpu.execute_token(looptoken)
+        fail = self.cpu.execute_token(looptoken, ord('G'))
         assert fail.identifier == 0
         assert self.cpu.get_latest_value_int(0) == ord('g')
 
@@ -2093,12 +2047,12 @@
         ops[1].setfailargs([])
         looptoken = LoopToken()
         self.cpu.compile_loop([i0, i1, i2, i3], ops, looptoken)
-        self.cpu.set_future_value_int(0, rffi.cast(lltype.Signed, raw))
-        self.cpu.set_future_value_int(1, 2)
-        self.cpu.set_future_value_int(2, 4)
-        self.cpu.set_future_value_int(3, rffi.cast(lltype.Signed, fn))
+        args = [rffi.cast(lltype.Signed, raw),
+                2,
+                4,
+                rffi.cast(lltype.Signed, fn)]
         assert glob.lst == []
-        fail = self.cpu.execute_token(looptoken)
+        fail = self.cpu.execute_token(looptoken, *args)
         assert fail.identifier == 0
         assert len(glob.lst) > 0
         lltype.free(raw, flavor='raw')
@@ -2151,9 +2105,8 @@
         self.cpu.compile_loop([i1, i2], ops, looptoken)
 
         buffer = lltype.malloc(rffi.CCHARP.TO, buflen, flavor='raw')
-        self.cpu.set_future_value_int(0, buflen)
-        self.cpu.set_future_value_int(1, rffi.cast(lltype.Signed, buffer))
-        fail = self.cpu.execute_token(looptoken)
+        args = [buflen, rffi.cast(lltype.Signed, buffer)]
+        fail = self.cpu.execute_token(looptoken, *args)
         assert fail.identifier == 0
         assert self.cpu.get_latest_value_int(0) == len(cwd)
         assert rffi.charp2strn(buffer, buflen) == cwd
@@ -2172,9 +2125,7 @@
         looptoken = LoopToken()
         self.cpu.compile_loop([i0, i1], ops, looptoken)
 
-        self.cpu.set_future_value_int(0, -42)
-        self.cpu.set_future_value_int(1, 9)
-        fail = self.cpu.execute_token(looptoken)
+        fail = self.cpu.execute_token(looptoken, -42, 9)
         assert fail.identifier == 0
         assert self.cpu.get_latest_value_int(0) == -42
         print 'step 1 ok'
@@ -2183,9 +2134,7 @@
         # mark as failing
         self.cpu.invalidate_loop(looptoken)
 
-        self.cpu.set_future_value_int(0, -42)
-        self.cpu.set_future_value_int(1, 9)
-        fail = self.cpu.execute_token(looptoken)
+        fail = self.cpu.execute_token(looptoken, -42, 9)
         assert fail is faildescr
         assert self.cpu.get_latest_value_int(0) == 9
         print 'step 2 ok'
@@ -2201,9 +2150,7 @@
         ops[0].setfailargs([])
         self.cpu.compile_bridge(faildescr, [i2], ops, looptoken)
 
-        self.cpu.set_future_value_int(0, -42)
-        self.cpu.set_future_value_int(1, 9)
-        fail = self.cpu.execute_token(looptoken)
+        fail = self.cpu.execute_token(looptoken, -42, 9)
         assert fail.identifier == 3
         assert self.cpu.get_latest_value_int(0) == 9
         print 'step 3 ok'
@@ -2212,9 +2159,7 @@
         # mark as failing again
         self.cpu.invalidate_loop(looptoken)
 
-        self.cpu.set_future_value_int(0, -42)
-        self.cpu.set_future_value_int(1, 9)
-        fail = self.cpu.execute_token(looptoken)
+        fail = self.cpu.execute_token(looptoken, -42, 9)
         assert fail is faildescr2
         print 'step 4 ok'
         print '-'*79
@@ -2423,9 +2368,8 @@
         FakeJitDriverSD.portal_calldescr = self.cpu.calldescrof(
             lltype.Ptr(lltype.FuncType(ARGS, RES)), ARGS, RES,
             EffectInfo.MOST_GENERAL)
-        for i in range(10):
-            self.cpu.set_future_value_int(i, i+1)
-        res = self.cpu.execute_token(looptoken)
+        args = [i+1 for i in range(10)]
+        res = self.cpu.execute_token(looptoken, *args)
         assert self.cpu.get_latest_value_int(0) == 55
         ops = '''
         [i0, i1, i2, i3, i4, i5, i6, i7, i8, i9]
@@ -2437,9 +2381,8 @@
         loop = parse(ops, namespace=locals())
         othertoken = LoopToken()
         self.cpu.compile_loop(loop.inputargs, loop.operations, othertoken)
-        for i in range(10):
-            self.cpu.set_future_value_int(i, i+1)
-        res = self.cpu.execute_token(othertoken)
+        args = [i+1 for i in range(10)]
+        res = self.cpu.execute_token(othertoken, *args)
         assert self.cpu.get_latest_value_int(0) == 13
         assert called
 
@@ -2474,9 +2417,9 @@
         looptoken = LoopToken()
         looptoken.outermost_jitdriver_sd = FakeJitDriverSD()
         self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken)
-        self.cpu.set_future_value_float(0, longlong.getfloatstorage(1.2))
-        self.cpu.set_future_value_float(1, longlong.getfloatstorage(2.3))
-        res = self.cpu.execute_token(looptoken)
+        args = [longlong.getfloatstorage(1.2),
+                longlong.getfloatstorage(2.3)]
+        res = self.cpu.execute_token(looptoken, *args)
         x = self.cpu.get_latest_value_float(0)
         assert longlong.getrealfloat(x) == 1.2 + 2.3
         ops = '''
@@ -2488,9 +2431,9 @@
         loop = parse(ops, namespace=locals())
         othertoken = LoopToken()
         self.cpu.compile_loop(loop.inputargs, loop.operations, othertoken)
-        self.cpu.set_future_value_float(0, longlong.getfloatstorage(1.2))
-        self.cpu.set_future_value_float(1, longlong.getfloatstorage(3.2))
-        res = self.cpu.execute_token(othertoken)
+        args = [longlong.getfloatstorage(1.2),
+                longlong.getfloatstorage(3.2)]
+        res = self.cpu.execute_token(othertoken, *args)
         x = self.cpu.get_latest_value_float(0)
         assert longlong.getrealfloat(x) == 13.5
         assert called
@@ -2501,9 +2444,9 @@
         try:
             othertoken = LoopToken()
             self.cpu.compile_loop(loop.inputargs, loop.operations, othertoken)
-            self.cpu.set_future_value_float(0, longlong.getfloatstorage(1.2))
-            self.cpu.set_future_value_float(1, longlong.getfloatstorage(3.2))
-            res = self.cpu.execute_token(othertoken)
+            args = [longlong.getfloatstorage(1.2),
+                    longlong.getfloatstorage(3.2)]
+            res = self.cpu.execute_token(othertoken, *args)
             x = self.cpu.get_latest_value_float(0)
             assert longlong.getrealfloat(x) == 1.2 + 3.2
             assert not called
@@ -2564,9 +2507,9 @@
         looptoken = LoopToken()
         looptoken.outermost_jitdriver_sd = FakeJitDriverSD()
         self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken)
-        self.cpu.set_future_value_float(0, longlong.getfloatstorage(1.25))
-        self.cpu.set_future_value_float(1, longlong.getfloatstorage(2.35))
-        res = self.cpu.execute_token(looptoken)
+        args = [longlong.getfloatstorage(1.25),
+                longlong.getfloatstorage(2.35)]
+        res = self.cpu.execute_token(looptoken, *args)
         x = self.cpu.get_latest_value_float(0)
         assert longlong.getrealfloat(x) == 1.25 + 2.35
         assert not called
@@ -2582,9 +2525,9 @@
         self.cpu.compile_loop(loop.inputargs, loop.operations, othertoken)
 
         # normal call_assembler: goes to looptoken
-        self.cpu.set_future_value_float(0, longlong.getfloatstorage(1.25))
-        self.cpu.set_future_value_float(1, longlong.getfloatstorage(3.25))
-        res = self.cpu.execute_token(othertoken)
+        args = [longlong.getfloatstorage(1.25),
+                longlong.getfloatstorage(3.25)]
+        res = self.cpu.execute_token(othertoken, *args)
         x = self.cpu.get_latest_value_float(0)
         assert longlong.getrealfloat(x) == 13.5
         assert called
@@ -2604,10 +2547,9 @@
         self.cpu.redirect_call_assembler(looptoken, looptoken2)
 
         # now, our call_assembler should go to looptoken2
-        self.cpu.set_future_value_float(0, longlong.getfloatstorage(6.0))
-        self.cpu.set_future_value_float(1, longlong.getfloatstorage(1.5))
-                                                       # 6.0-1.5 == 1.25+3.25
-        res = self.cpu.execute_token(othertoken)
+        args = [longlong.getfloatstorage(6.0),
+                longlong.getfloatstorage(1.5)]         # 6.0-1.5 == 1.25+3.25
+        res = self.cpu.execute_token(othertoken, *args)
         x = self.cpu.get_latest_value_float(0)
         assert longlong.getrealfloat(x) == 13.5
         assert called
@@ -2961,8 +2903,7 @@
         looptoken = LoopToken()
         self.cpu.compile_loop(inputargs, operations, looptoken)
         # overflowing value:
-        self.cpu.set_future_value_int(0, sys.maxint // 4 + 1)
-        fail = self.cpu.execute_token(looptoken)
+        fail = self.cpu.execute_token(looptoken, sys.maxint // 4 + 1)
         assert fail.identifier == excdescr.identifier
 
 

From noreply at buildbot.pypy.org  Tue Oct 25 21:05:20 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 25 Oct 2011 21:05:20 +0200 (CEST)
Subject: [pypy-commit] pypy default: Make this test closer to what was
	intended.
Message-ID: <20111025190520.375E7820C7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48452:9a4ee13149cd
Date: 2011-10-25 15:54 +0200
http://bitbucket.org/pypy/pypy/changeset/9a4ee13149cd/

Log:	Make this test closer to what was intended.

diff --git a/pypy/jit/backend/llsupport/test/test_gc.py b/pypy/jit/backend/llsupport/test/test_gc.py
--- a/pypy/jit/backend/llsupport/test/test_gc.py
+++ b/pypy/jit/backend/llsupport/test/test_gc.py
@@ -250,10 +250,11 @@
                                   has_finalizer, has_light_finalizer,
                                   contains_weakptr):
         assert not contains_weakptr
+        assert not has_finalizer           # in these tests
+        assert not has_light_finalizer     # in these tests
         p = llmemory.raw_malloc(size)
         p = llmemory.cast_adr_to_ptr(p, RESTYPE)
-        flags = (int(has_finalizer) << 16) | (int(has_light_finalizer) << 17)
-        tid = llop.combine_ushort(lltype.Signed, type_id, flags)
+        tid = llop.combine_ushort(lltype.Signed, type_id, 0)
         self.record.append(("fixedsize", repr(size), tid, p))
         return p
 

From noreply at buildbot.pypy.org  Tue Oct 25 21:24:11 2011
From: noreply at buildbot.pypy.org (edelsoh)
Date: Tue, 25 Oct 2011 21:24:11 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: More PPC64 support for
 arraylen_gc, setarrayitem_gc, getarrayitem_gc
Message-ID: <20111025192411.C0BCE820C7@wyvern.cs.uni-duesseldorf.de>

Author: edelsoh
Branch: ppc-jit-backend
Changeset: r48453:d4c1290fd460
Date: 2011-10-25 15:23 -0400
http://bitbucket.org/pypy/pypy/changeset/d4c1290fd460/

Log:	More PPC64 support for arraylen_gc, setarrayitem_gc, getarrayitem_gc

diff --git a/pypy/jit/backend/ppc/ppcgen/opassembler.py b/pypy/jit/backend/ppc/ppcgen/opassembler.py
--- a/pypy/jit/backend/ppc/ppcgen/opassembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/opassembler.py
@@ -222,9 +222,15 @@
         offset = locs[2]
         if offset is not None:
             if offset.is_imm():
-                self.mc.lwz(r.r0.value, locs[0].value, offset.value)
+                if IS_PPC_32:
+                    self.mc.lwz(r.r0.value, locs[0].value, offset.value)
+                else:
+                    self.mc.ld(r.r0.value, locs[0].value, offset.value)
             else:
-                self.mc.lwzx(r.r0.value, locs[0].value, offset.value)
+                if IS_PPC_32:
+                    self.mc.lwzx(r.r0.value, locs[0].value, offset.value)
+                else:
+                    self.mc.ldx(r.r0.value, locs[0].value, offset.value)
             self.mc.cmp(r.r0.value, locs[1].value)
         else:
             assert 0, "not implemented yet"
@@ -313,7 +319,10 @@
     # XXX 64 bit adjustment
     def emit_arraylen_gc(self, op, arglocs, regalloc):
         res, base_loc, ofs = arglocs
-        self.mc.lwz(res.value, base_loc.value, ofs.value)
+        if IS_PPC_32:
+            self.mc.lwz(res.value, base_loc.value, ofs.value)
+        else:
+            self.mc.ld(res.value, base_loc.value, ofs.value)
 
     # XXX 64 bit adjustment
     def emit_setarrayitem_gc(self, op, arglocs, regalloc):
@@ -330,7 +339,7 @@
             scale_loc = r.r0
 
         if scale.value == 3:
-            assert 0, "not implemented yet"
+            self.mc.stdx(value_loc.value, base_loc.value, scale_loc.value)
         elif scale.value == 2:
             self.mc.stwx(value_loc.value, base_loc.value, scale_loc.value)
         elif scale.value == 1:
@@ -354,7 +363,7 @@
             scale_loc = r.r0
 
         if scale.value == 3:
-            assert 0, "not implemented yet"
+            self.mc.ldx(res.value, base_loc.value, scale_loc.value)
         elif scale.value == 2:
             self.mc.lwzx(res.value, base_loc.value, scale_loc.value)
         elif scale.value == 1:

From noreply at buildbot.pypy.org  Tue Oct 25 22:20:29 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue, 25 Oct 2011 22:20:29 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-dicts: fix a really emberassing typo
Message-ID: <20111025202029.65A23820C7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: virtual-dicts
Changeset: r48454:6283d742ab74
Date: 2011-10-25 16:20 -0400
http://bitbucket.org/pypy/pypy/changeset/6283d742ab74/

Log:	fix a really emberassing typo

diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -3616,7 +3616,7 @@
         res = self.meta_interp(main, [False, 100, True], taggedpointers=True)
 
     def test_rerased(self):
-        eraseX, uneraseX = rerased,new_erasing_pair("X")
+        eraseX, uneraseX = rerased.new_erasing_pair("X")
         #
         class X:
             def __init__(self, a, b):

From noreply at buildbot.pypy.org  Tue Oct 25 22:33:51 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 25 Oct 2011 22:33:51 +0200 (CEST)
Subject: [pypy-commit] pypy default: progress towards get/set interiorfield
	in test_ll_random
Message-ID: <20111025203351.093B0820C7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r48455:de02b302dd89
Date: 2011-10-25 22:32 +0200
http://bitbucket.org/pypy/pypy/changeset/de02b302dd89/

Log:	progress towards get/set interiorfield in test_ll_random

diff --git a/pypy/jit/backend/test/test_ll_random.py b/pypy/jit/backend/test/test_ll_random.py
--- a/pypy/jit/backend/test/test_ll_random.py
+++ b/pypy/jit/backend/test/test_ll_random.py
@@ -28,16 +28,27 @@
         fork.structure_types_and_vtables = self.structure_types_and_vtables
         return fork
 
-    def get_structptr_var(self, r, must_have_vtable=False, type=lltype.Struct):
+    def _choose_ptr_vars(self, from_, type, array_of_structs):
+        ptrvars = []
+        for i in range(len(from_)):
+            v, S = from_[i][:2]
+            if not isinstance(S, type):
+                continue
+            if (isinstance(S, lltype.Array) and
+                isinstance(S.OF, lltype.Struct) == array_of_structs):
+                ptrvars.append((v, S))
+        return ptrvars
+
+    def get_structptr_var(self, r, must_have_vtable=False, type=lltype.Struct,
+                          array_of_structs=False):
         while True:
-            ptrvars = [(v, S) for (v, S) in self.ptrvars
-                              if isinstance(S, type)]
+            ptrvars = self._choose_ptr_vars(self.ptrvars, type,
+                                            array_of_structs)
             if ptrvars and r.random() < 0.8:
                 v, S = r.choice(ptrvars)
             else:
-                prebuilt_ptr_consts = [(v, S)
-                                 for (v, S, _) in self.prebuilt_ptr_consts
-                                 if isinstance(S, type)]
+                prebuilt_ptr_consts = self._choose_ptr_vars(
+                    self.prebuilt_ptr_consts, type, array_of_structs)
                 if prebuilt_ptr_consts and r.random() < 0.7:
                     v, S = r.choice(prebuilt_ptr_consts)
                 else:
@@ -48,7 +59,8 @@
                                                 has_vtable=must_have_vtable)
                     else:
                         # create a new constant array
-                        p = self.get_random_array(r)
+                        p = self.get_random_array(r,
+                                    must_be_array_of_structs=array_of_structs)
                     S = lltype.typeOf(p).TO
                     v = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, p))
                     self.prebuilt_ptr_consts.append((v, S,
@@ -74,7 +86,8 @@
                 TYPE = lltype.Signed
         return TYPE
 
-    def get_random_structure_type(self, r, with_vtable=None, cache=True):
+    def get_random_structure_type(self, r, with_vtable=None, cache=True,
+                                  type=lltype.GcStruct):
         if cache and self.structure_types and r.random() < 0.5:
             return r.choice(self.structure_types)
         fields = []
@@ -85,7 +98,7 @@
         for i in range(r.randrange(1, 5)):
             TYPE = self.get_random_primitive_type(r)
             fields.append(('f%d' % i, TYPE))
-        S = lltype.GcStruct('S%d' % self.counter, *fields, **kwds)
+        S = type('S%d' % self.counter, *fields, **kwds)
         self.counter += 1
         if cache:
             self.structure_types.append(S)
@@ -125,17 +138,29 @@
                 setattr(p, fieldname, rffi.cast(TYPE, r.random_integer()))
         return p
 
-    def get_random_array_type(self, r):
-        TYPE = self.get_random_primitive_type(r)
+    def get_random_array_type(self, r, can_be_array_of_struct=False,
+                              must_be_array_of_structs=False):
+        if ((can_be_array_of_struct and r.random() < 0.1) or
+            must_be_array_of_structs):
+            TYPE = self.get_random_structure_type(r, cache=False,
+                                                  type=lltype.Struct)
+        else:
+            TYPE = self.get_random_primitive_type(r)
         return lltype.GcArray(TYPE)
 
-    def get_random_array(self, r):
-        A = self.get_random_array_type(r)
+    def get_random_array(self, r, must_be_array_of_structs=False):
+        A = self.get_random_array_type(r,
+                           must_be_array_of_structs=must_be_array_of_structs)
         length = (r.random_integer() // 15) % 300  # length: between 0 and 299
                                                    # likely to be small
         p = lltype.malloc(A, length)
-        for i in range(length):
-            p[i] = rffi.cast(A.OF, r.random_integer())
+        if isinstance(A.OF, lltype.Primitive):
+            for i in range(length):
+                p[i] = rffi.cast(A.OF, r.random_integer())
+        else:
+            for i in range(length):
+                for fname, TP in A.OF._flds.iteritems():
+                    setattr(p[i], fname, rffi.cast(TP, r.random_integer()))
         return p
 
     def get_index(self, length, r):
@@ -220,7 +245,7 @@
 
 class GetFieldOperation(test_random.AbstractOperation):
     def field_descr(self, builder, r):
-        v, S = builder.get_structptr_var(r)
+        v, S = builder.get_structptr_var(r, )
         names = S._names
         if names[0] == 'parent':
             names = names[1:]
@@ -239,6 +264,31 @@
                 continue
             break
 
+class GetInteriorFieldOperation(test_random.AbstractOperation):
+    def field_descr(self, builder, r):
+        v, A = builder.get_structptr_var(r, type=lltype.Array,
+                                         array_of_structs=True)
+        array = v.getref(lltype.Ptr(A))
+        v_index = builder.get_index(len(array), r)
+        names = A.OF._names
+        if names[0] == 'parent':
+            names = names[1:]
+        name = r.choice(names)
+        descr = builder.cpu.interiorfielddescrof(A, name)
+        descr._random_info = 'cpu.interiorfielddescrof(%s, %r)' % (A.OF._name,
+                                                                   name)
+        TYPE = getattr(A.OF, name)
+        return v, v_index, descr, TYPE
+
+    def produce_into(self, builder, r):
+        while True:
+            try:
+                v, v_index, descr, _ = self.field_descr(builder, r)
+                self.put(builder, [v, v_index], descr)
+            except lltype.UninitializedMemoryAccess:
+                continue
+            break
+
 class SetFieldOperation(GetFieldOperation):
     def produce_into(self, builder, r):
         v, descr, TYPE = self.field_descr(builder, r)
@@ -251,6 +301,20 @@
                 break
         builder.do(self.opnum, [v, w], descr)
 
+class SetInteriorFieldOperation(GetFieldOperation):
+    def produce_into(self, builder, r):
+        import pdb
+        pdb.set_trace()
+        v, descr, TYPE = self.field_descr(builder, r)
+        while True:
+            if r.random() < 0.3:
+                w = ConstInt(r.random_integer())
+            else:
+                w = r.choice(builder.intvars)
+            if rffi.cast(lltype.Signed, rffi.cast(TYPE, w.value)) == w.value:
+                break
+        builder.do(self.opnum, [v, w], descr)
+
 class NewOperation(test_random.AbstractOperation):
     def size_descr(self, builder, S):
         descr = builder.cpu.sizeof(S)
@@ -306,7 +370,7 @@
 
 class NewArrayOperation(ArrayOperation):
     def produce_into(self, builder, r):
-        A = builder.get_random_array_type(r)
+        A = builder.get_random_array_type(r, can_be_array_of_struct=True)
         v_size = builder.get_index(300, r)
         v_ptr = builder.do(self.opnum, [v_size], self.array_descr(builder, A))
         builder.ptrvars.append((v_ptr, A))
@@ -586,7 +650,9 @@
 for i in range(4):      # make more common
     OPERATIONS.append(GetFieldOperation(rop.GETFIELD_GC))
     OPERATIONS.append(GetFieldOperation(rop.GETFIELD_GC))
+    OPERATIONS.append(GetInteriorFieldOperation(rop.GETINTERIORFIELD_GC))
     OPERATIONS.append(SetFieldOperation(rop.SETFIELD_GC))
+    #OPERATIONS.append(SetInteriorFieldOperation(rop.GETINTERIORFIELD_GC))
     OPERATIONS.append(NewOperation(rop.NEW))
     OPERATIONS.append(NewOperation(rop.NEW_WITH_VTABLE))
 

From noreply at buildbot.pypy.org  Tue Oct 25 22:33:52 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 25 Oct 2011 22:33:52 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge default
Message-ID: <20111025203352.36903820C7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r48456:13932f534c82
Date: 2011-10-25 22:33 +0200
http://bitbucket.org/pypy/pypy/changeset/13932f534c82/

Log:	merge default

diff --git a/pypy/jit/backend/llsupport/test/test_gc.py b/pypy/jit/backend/llsupport/test/test_gc.py
--- a/pypy/jit/backend/llsupport/test/test_gc.py
+++ b/pypy/jit/backend/llsupport/test/test_gc.py
@@ -250,10 +250,11 @@
                                   has_finalizer, has_light_finalizer,
                                   contains_weakptr):
         assert not contains_weakptr
+        assert not has_finalizer           # in these tests
+        assert not has_light_finalizer     # in these tests
         p = llmemory.raw_malloc(size)
         p = llmemory.cast_adr_to_ptr(p, RESTYPE)
-        flags = (int(has_finalizer) << 16) | (int(has_light_finalizer) << 17)
-        tid = llop.combine_ushort(lltype.Signed, type_id, flags)
+        tid = llop.combine_ushort(lltype.Signed, type_id, 0)
         self.record.append(("fixedsize", repr(size), tid, p))
         return p
 

From noreply at buildbot.pypy.org  Tue Oct 25 22:36:53 2011
From: noreply at buildbot.pypy.org (edelsohn)
Date: Tue, 25 Oct 2011 22:36:53 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: Add PPC64 support for strlen
Message-ID: <20111025203653.9799E820C7@wyvern.cs.uni-duesseldorf.de>

Author: edelsohn
Branch: ppc-jit-backend
Changeset: r48457:c58fa0a4d970
Date: 2011-10-25 16:36 -0400
http://bitbucket.org/pypy/pypy/changeset/c58fa0a4d970/

Log:	Add PPC64 support for strlen

diff --git a/pypy/jit/backend/ppc/ppcgen/opassembler.py b/pypy/jit/backend/ppc/ppcgen/opassembler.py
--- a/pypy/jit/backend/ppc/ppcgen/opassembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/opassembler.py
@@ -384,9 +384,15 @@
     def emit_strlen(self, op, arglocs, regalloc):
         l0, l1, res = arglocs
         if l1.is_imm():
-            self.mc.lwz(res.value, l0.value, l1.getint())
+            if IS_PPC_32:
+                self.mc.lwz(res.value, l0.value, l1.getint())
+            else:
+                self.mc.ld(res.value, l0.value, l1.getint())
         else:
-            self.mc.lwzx(res.value, l0.value, l1.value)
+            if IS_PPC_32:
+                self.mc.lwzx(res.value, l0.value, l1.value)
+            else:
+                self.mc.ldx(res.value, l0.value, l1.value)
 
     def emit_strgetitem(self, op, arglocs, regalloc):
         res, base_loc, ofs_loc, basesize = arglocs

From noreply at buildbot.pypy.org  Tue Oct 25 22:38:29 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 25 Oct 2011 22:38:29 +0200 (CEST)
Subject: [pypy-commit] pypy jit-simplify-backendintf: Fix the front-end.
 Some reduction in the total
Message-ID: <20111025203829.669B2820C7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: jit-simplify-backendintf
Changeset: r48458:cce408b68723
Date: 2011-10-25 22:04 +0200
http://bitbucket.org/pypy/pypy/changeset/cce408b68723/

Log:	Fix the front-end. Some reduction in the total number of lines, but
	due to a change --- when tracing is done, we raise
	ContinueRunningNormally instead of building the arguments directly
	for the next iteration, including virtuals --- a number of tests
	fail now :-(

diff --git a/pypy/jit/metainterp/blackhole.py b/pypy/jit/metainterp/blackhole.py
--- a/pypy/jit/metainterp/blackhole.py
+++ b/pypy/jit/metainterp/blackhole.py
@@ -1500,7 +1500,6 @@
                         all_virtuals=None):
     from pypy.jit.metainterp.resume import blackhole_from_resumedata
     #debug_start('jit-blackhole')
-    metainterp_sd.profiler.start_blackhole()
     blackholeinterp = blackhole_from_resumedata(
         metainterp_sd.blackholeinterpbuilder,
         jitdriver_sd,
@@ -1514,10 +1513,9 @@
     current_exc = blackholeinterp._prepare_resume_from_failure(
         resumedescr.guard_opnum, dont_change_position)
 
-    try:
-        _run_forever(blackholeinterp, current_exc)
-    finally:
-        metainterp_sd.profiler.end_blackhole()
+    #try:
+    _run_forever(blackholeinterp, current_exc)
+    #finally:
         #debug_stop('jit-blackhole')
 
 def convert_and_run_from_pyjitpl(metainterp, raising_exception=False):
@@ -1525,7 +1523,6 @@
     # 'metainterp.framestack'.
     #debug_start('jit-blackhole')
     metainterp_sd = metainterp.staticdata
-    metainterp_sd.profiler.start_blackhole()
     nextbh = None
     for frame in metainterp.framestack:
         curbh = metainterp_sd.blackholeinterpbuilder.acquire_interp()
@@ -1542,8 +1539,7 @@
         firstbh.exception_last_value = current_exc
         current_exc = lltype.nullptr(rclass.OBJECTPTR.TO)
     #
-    try:
-        _run_forever(firstbh, current_exc)
-    finally:
-        metainterp_sd.profiler.end_blackhole()
+    #try:
+    _run_forever(firstbh, current_exc)
+    #finally:
         #debug_stop('jit-blackhole')
diff --git a/pypy/jit/metainterp/compile.py b/pypy/jit/metainterp/compile.py
--- a/pypy/jit/metainterp/compile.py
+++ b/pypy/jit/metainterp/compile.py
@@ -340,12 +340,11 @@
 
     def handle_fail(self, metainterp_sd, jitdriver_sd):
         if self.must_compile(metainterp_sd, jitdriver_sd):
-            return self._trace_and_compile_from_bridge(metainterp_sd,
-                                                       jitdriver_sd)
+            self._trace_and_compile_from_bridge(metainterp_sd, jitdriver_sd)
         else:
             from pypy.jit.metainterp.blackhole import resume_in_blackhole
             resume_in_blackhole(metainterp_sd, jitdriver_sd, self)
-            assert 0, "unreachable"
+        assert 0, "unreachable"
 
     def _trace_and_compile_from_bridge(self, metainterp_sd, jitdriver_sd):
         # 'jitdriver_sd' corresponds to the outermost one, i.e. the one
@@ -354,7 +353,7 @@
         # jitdrivers.
         from pypy.jit.metainterp.pyjitpl import MetaInterp
         metainterp = MetaInterp(metainterp_sd, jitdriver_sd)
-        return metainterp.handle_guard_failure(self)
+        metainterp.handle_guard_failure(self)
     _trace_and_compile_from_bridge._dont_inline_ = True
 
     def must_compile(self, metainterp_sd, jitdriver_sd):
diff --git a/pypy/jit/metainterp/history.py b/pypy/jit/metainterp/history.py
--- a/pypy/jit/metainterp/history.py
+++ b/pypy/jit/metainterp/history.py
@@ -123,9 +123,6 @@
     def sort_key(self):
         raise NotImplementedError
 
-    def set_future_value(self, cpu, j):
-        raise NotImplementedError
-
     def nonnull(self):
         raise NotImplementedError
 
@@ -288,9 +285,6 @@
     def _get_hash_(self):
         return make_hashable_int(self.value)
 
-    def set_future_value(self, cpu, j):
-        cpu.set_future_value_int(j, self.value)
-
     def same_constant(self, other):
         if isinstance(other, ConstInt):
             return self.value == other.value
@@ -328,9 +322,6 @@
     def _get_hash_(self):
         return longlong.gethash(self.value)
 
-    def set_future_value(self, cpu, j):
-        cpu.set_future_value_float(j, self.value)
-
     def same_constant(self, other):
         if isinstance(other, ConstFloat):
             return self.value == other.value
@@ -377,9 +368,6 @@
     def getaddr(self):
         return llmemory.cast_ptr_to_adr(self.value)
 
-    def set_future_value(self, cpu, j):
-        cpu.set_future_value_ref(j, self.value)
-
     def same_constant(self, other):
         if isinstance(other, ConstPtr):
             return self.value == other.value
@@ -431,9 +419,6 @@
         else:
             return 0
 
-    def set_future_value(self, cpu, j):
-        cpu.set_future_value_ref(j, self.value)
-
 ##    def getaddr(self):
 ##        # so far this is used only when calling
 ##        # CodeWriter.IndirectCallset.bytecode_for_address.  We don't need a
@@ -539,9 +524,6 @@
     def _get_hash_(self):
         return make_hashable_int(self.value)
 
-    def set_future_value(self, cpu, j):
-        cpu.set_future_value_int(j, self.value)
-
     def nonnull(self):
         return self.value != 0
 
@@ -574,9 +556,6 @@
     def _get_hash_(self):
         return longlong.gethash(self.value)
 
-    def set_future_value(self, cpu, j):
-        cpu.set_future_value_float(j, self.value)
-
     def nonnull(self):
         return self.value != longlong.ZEROF
 
@@ -619,9 +598,6 @@
         else:
             return 0
 
-    def set_future_value(self, cpu, j):
-        cpu.set_future_value_ref(j, self.value)
-
     def nonnull(self):
         return bool(self.value)
 
@@ -666,19 +642,12 @@
     def nonnull(self):
         return bool(self.value)
 
-    def set_future_value(self, cpu, j):
-        cpu.set_future_value_ref(j, self.value)
-
     def repr_rpython(self):
         return repr_rpython(self, 'bo')
 
     _getrepr_ = repr_object
 
 
-def set_future_values(cpu, boxes):
-    for j in range(len(boxes)):
-        boxes[j].set_future_value(cpu, j)
-
 # ____________________________________________________________
 
 
diff --git a/pypy/jit/metainterp/jitprof.py b/pypy/jit/metainterp/jitprof.py
--- a/pypy/jit/metainterp/jitprof.py
+++ b/pypy/jit/metainterp/jitprof.py
@@ -10,8 +10,6 @@
 counters="""
 TRACING
 BACKEND
-RUNNING
-BLACKHOLE
 OPS
 RECORDED_OPS
 GUARDS
@@ -67,18 +65,6 @@
     def end_backend(self):
         pass
 
-    def start_running(self):
-        pass
-
-    def end_running(self):
-        pass
-
-    def start_blackhole(self):
-        pass
-
-    def end_blackhole(self):
-        pass
-
     def count(self, kind, inc=1):
         pass
 
@@ -134,16 +120,6 @@
     def start_backend(self):   self._start(BACKEND)
     def end_backend(self):     self._end  (BACKEND)
 
-    # Don't record times for 'running' and 'blackhole' because there are
-    # too many of them: calling time.time() is a major blocker.
-    # If you are interested in these numbers, use 'PYPYLOG=file' and
-    # look at the resulting file with pypy/tool/logparser.py.
-    def start_running(self): self.count(RUNNING)
-    def end_running(self):   pass
-
-    def start_blackhole(self): self.count(BLACKHOLE)
-    def end_blackhole(self):   pass
-
     def count(self, kind, inc=1):
         self.counters[kind] += inc        
     
@@ -165,8 +141,6 @@
         calls = self.calls
         self._print_line_time("Tracing", cnt[TRACING],   tim[TRACING])
         self._print_line_time("Backend", cnt[BACKEND],   tim[BACKEND])
-        self._print_intline("Running asm", cnt[RUNNING])
-        self._print_intline("Blackhole", cnt[BLACKHOLE])
         line = "TOTAL:      \t\t%f" % (self.tk - self.starttime, )
         debug_print(line)
         self._print_intline("ops", cnt[OPS])
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -1795,7 +1795,7 @@
 
     def _interpret(self):
         # Execute the frames forward until we raise a DoneWithThisFrame,
-        # a ExitFrameWithException, or a GenerateMergePoint exception.
+        # a ExitFrameWithException, or a ContinueRunningNormally exception.
         self.staticdata.stats.entered()
         while True:
             self.framestack[-1].run_one_step()
@@ -1843,8 +1843,6 @@
         self.seen_loop_header_for_jdindex = -1
         try:
             self.interpret()
-        except GenerateMergePoint, gmp:
-            return self.designate_target_loop(gmp)
         except SwitchToBlackhole, stb:
             self.run_blackhole_interp_to_cancel_tracing(stb)
         assert False, "should always raise"
@@ -1879,8 +1877,6 @@
             if self.resumekey_original_loop_token is None:   # very rare case
                 raise SwitchToBlackhole(ABORT_BRIDGE)
             self.interpret()
-        except GenerateMergePoint, gmp:
-            return self.designate_target_loop(gmp)
         except SwitchToBlackhole, stb:
             self.run_blackhole_interp_to_cancel_tracing(stb)
         assert False, "should always raise"
@@ -1974,12 +1970,25 @@
         start = len(self.history.operations)
         self.current_merge_points.append((live_arg_boxes, start))
 
-    def designate_target_loop(self, gmp):
-        loop_token = gmp.target_loop_token
+    def _unpack_boxes(self, boxes, start, stop):
+        ints = []; refs = []; floats = []
+        for i in range(start, stop):
+            box = boxes[i]
+            if   box.type == history.INT: ints.append(box.getint())
+            elif box.type == history.REF: refs.append(box.getref_base())
+            elif box.type == history.FLOAT:floats.append(box.getfloatstorage())
+            else: assert 0
+        return ints[:], refs[:], floats[:]
+
+    def raise_continue_running_normally(self, live_arg_boxes):
+        self.history.inputargs = None
+        self.history.operations = None
         num_green_args = self.jitdriver_sd.num_green_args
-        residual_args = gmp.argboxes[num_green_args:]
-        history.set_future_values(self.cpu, residual_args)
-        return loop_token
+        gi, gr, gf = self._unpack_boxes(live_arg_boxes, 0, num_green_args)
+        ri, rr, rf = self._unpack_boxes(live_arg_boxes, num_green_args,
+                                        len(live_arg_boxes))
+        CRN = self.staticdata.ContinueRunningNormally
+        raise CRN(gi, gr, gf, ri, rr, rf)
 
     def prepare_resume_from_failure(self, opnum, dont_change_position=False):
         frame = self.framestack[-1]
@@ -2042,9 +2051,7 @@
                                               greenkey, start, start_resumedescr)
         if loop_token is not None: # raise if it *worked* correctly
             self.set_compiled_merge_points(greenkey, old_loop_tokens)
-            self.history.inputargs = None
-            self.history.operations = None
-            raise GenerateMergePoint(live_arg_boxes, loop_token)
+            self.raise_continue_running_normally(live_arg_boxes)
 
         self.history.inputargs = original_inputargs
         self.history.operations.pop()     # remove the JUMP
@@ -2065,9 +2072,7 @@
         finally:
             self.history.operations.pop()     # remove the JUMP
         if target_loop_token is not None: # raise if it *worked* correctly
-            self.history.inputargs = None
-            self.history.operations = None
-            raise GenerateMergePoint(live_arg_boxes, target_loop_token)
+            self.raise_continue_running_normally(live_arg_boxes)
 
     def compile_bridge_and_loop(self, original_boxes, live_arg_boxes, start,
                                 bridge_arg_boxes, start_resumedescr):
@@ -2103,10 +2108,7 @@
         except RetraceLoop:
             assert False
         assert target_loop_token is not None
-
-        self.history.inputargs = None
-        self.history.operations = None
-        raise GenerateMergePoint(live_arg_boxes, old_loop_tokens[0])
+        self.raise_continue_running_normally(live_arg_boxes)
 
     def compile_done_with_this_frame(self, exitbox):
         self.gen_store_back_in_virtualizable()
@@ -2484,12 +2486,6 @@
 
 # ____________________________________________________________
 
-class GenerateMergePoint(JitException):
-    def __init__(self, args, target_loop_token):
-        assert target_loop_token is not None
-        self.argboxes = args
-        self.target_loop_token = target_loop_token
-
 class ChangeFrame(JitException):
     """Raised after we mutated metainterp.framestack, in order to force
     it to reload the current top-of-stack frame that gets interpreted."""
diff --git a/pypy/jit/metainterp/test/support.py b/pypy/jit/metainterp/test/support.py
--- a/pypy/jit/metainterp/test/support.py
+++ b/pypy/jit/metainterp/test/support.py
@@ -4,9 +4,9 @@
 from pypy.rpython.ootypesystem import ootype
 from pypy.jit.backend.llgraph import runner
 from pypy.jit.metainterp.warmspot import ll_meta_interp, get_stats
+from pypy.jit.metainterp.warmstate import unspecialize_value
 from pypy.jit.metainterp.optimizeopt import ALL_OPTS_DICT
 from pypy.jit.metainterp import pyjitpl, history
-from pypy.jit.metainterp.warmstate import set_future_value
 from pypy.jit.codewriter.policy import JitPolicy
 from pypy.jit.codewriter import codewriter, longlong
 from pypy.rlib.rfloat import isnan
@@ -136,11 +136,11 @@
         return NotImplemented
     # a loop was successfully created by _run_with_pyjitpl(); call it
     cpu = metainterp.cpu
+    args1 = []
     for i in range(len(args) - num_green_args):
         x = args[num_green_args + i]
-        typecode = history.getkind(lltype.typeOf(x))
-        set_future_value(cpu, i, x, typecode)
-    faildescr = cpu.execute_token(loop_tokens[0])
+        args1.append(unspecialize_value(x))
+    faildescr = cpu.execute_token(loop_tokens[0], *args1)
     assert faildescr.__class__.__name__.startswith('DoneWithThisFrameDescr')
     if metainterp.jitdriver_sd.result_type == history.INT:
         return cpu.get_latest_value_int(0)
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -9,7 +9,6 @@
 from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin, noConst
 from pypy.jit.metainterp.typesystem import LLTypeHelper, OOTypeHelper
 from pypy.jit.metainterp.warmspot import get_stats
-from pypy.jit.metainterp.warmstate import set_future_value
 from pypy.rlib.jit import (JitDriver, we_are_jitted, hint, dont_look_inside,
     loop_invariant, elidable, promote, jit_debug, assert_green,
     AssertGreenFailed, unroll_safe, current_trace_length, look_inside_iff,
diff --git a/pypy/jit/metainterp/warmspot.py b/pypy/jit/metainterp/warmspot.py
--- a/pypy/jit/metainterp/warmspot.py
+++ b/pypy/jit/metainterp/warmspot.py
@@ -895,10 +895,3 @@
         graphs = self.translator.graphs
         for graph, block, i in find_force_quasi_immutable(graphs):
             self.replace_force_quasiimmut_with_direct_call(block.operations[i])
-
-    # ____________________________________________________________
-
-    def execute_token(self, loop_token):
-        fail_descr = self.cpu.execute_token(loop_token)
-        self.memory_manager.keep_loop_alive(loop_token)
-        return fail_descr
diff --git a/pypy/jit/metainterp/warmstate.py b/pypy/jit/metainterp/warmstate.py
--- a/pypy/jit/metainterp/warmstate.py
+++ b/pypy/jit/metainterp/warmstate.py
@@ -142,26 +142,6 @@
     else:
         return rffi.cast(lltype.Signed, x)
 
- at specialize.ll_and_arg(3)
-def set_future_value(cpu, j, value, typecode):
-    if typecode == 'ref':
-        refvalue = cpu.ts.cast_to_ref(value)
-        cpu.set_future_value_ref(j, refvalue)
-    elif typecode == 'int':
-        if isinstance(lltype.typeOf(value), lltype.Ptr):
-            intvalue = llmemory.AddressAsInt(llmemory.cast_ptr_to_adr(value))
-        else:
-            intvalue = lltype.cast_primitive(lltype.Signed, value)
-        cpu.set_future_value_int(j, intvalue)
-    elif typecode == 'float':
-        if lltype.typeOf(value) is lltype.Float:
-            value = longlong.getfloatstorage(value)
-        else:
-            assert longlong.is_longlong(lltype.typeOf(value))
-            value = rffi.cast(lltype.SignedLongLong, value)
-        cpu.set_future_value_float(j, value)
-    else:
-        assert False
 
 class JitCell(BaseJitCell):
     # the counter can mean the following things:
@@ -310,20 +290,36 @@
         index_of_virtualizable = jitdriver_sd.index_of_virtualizable
         num_green_args = jitdriver_sd.num_green_args
         get_jitcell = self.make_jitcell_getter()
-        set_future_values = self.make_set_future_values()
         self.make_jitdriver_callbacks()
         confirm_enter_jit = self.confirm_enter_jit
+        range_red_args = unrolling_iterable(
+            range(num_green_args, num_green_args + jitdriver_sd.num_red_args))
+
+        def execute_assembler(loop_token, *args):
+            # Call the backend to run the 'looptoken' with the given
+            # input args.
+            fail_descr = self.cpu.execute_token(loop_token, *args)
+            #
+            # If we have a virtualizable, we have to reset its
+            # 'vable_token' field afterwards
+            if vinfo is not None:
+                virtualizable = args[index_of_virtualizable]
+                virtualizable = vinfo.cast_to_vtype(virtualizable)
+                vinfo.reset_vable_token(virtualizable)
+            #
+            # Record in the memmgr that we just ran this loop,
+            # so that it will keep it alive for a longer time
+            warmrunnerdesc.memory_manager.keep_loop_alive(loop_token)
+            #
+            # Handle the failure
+            fail_descr.handle_fail(metainterp_sd, jitdriver_sd)
+            #
+            assert 0, "should have raised"
 
         def maybe_compile_and_run(threshold, *args):
             """Entry point to the JIT.  Called at the point with the
             can_enter_jit() hint.
             """
-            if vinfo is not None:
-                virtualizable = args[num_green_args + index_of_virtualizable]
-                virtualizable = vinfo.cast_to_vtype(virtualizable)
-            else:
-                virtualizable = None
-
             # look for the cell corresponding to the current greenargs
             greenargs = args[:num_green_args]
             cell = get_jitcell(True, *greenargs)
@@ -343,42 +339,36 @@
                 # set counter to -2, to mean "tracing in effect"
                 cell.counter = -2
                 try:
-                    loop_token = metainterp.compile_and_run_once(jitdriver_sd,
-                                                                 *args)
+                    metainterp.compile_and_run_once(jitdriver_sd, *args)
                 finally:
                     if cell.counter == -2:
                         cell.counter = 0
             else:
-                if cell.counter == -2:
+                if cell.counter != -1:
+                    assert cell.counter == -2
                     # tracing already happening in some outer invocation of
                     # this function. don't trace a second time.
                     return
-                assert cell.counter == -1
                 if not confirm_enter_jit(*args):
                     return
+                # machine code was already compiled for these greenargs
                 loop_token = cell.get_entry_loop_token()
                 if loop_token is None:   # it was a weakref that has been freed
                     cell.counter = 0
                     return
-                # machine code was already compiled for these greenargs
-                # get the assembler and fill in the boxes
-                set_future_values(*args[num_green_args:])
-
-            # ---------- execute assembler ----------
-            while True:     # until interrupted by an exception
-                metainterp_sd.profiler.start_running()
-                #debug_start("jit-running")
-                fail_descr = warmrunnerdesc.execute_token(loop_token)
-                #debug_stop("jit-running")
-                metainterp_sd.profiler.end_running()
-                loop_token = None     # for test_memmgr
-                if vinfo is not None:
-                    vinfo.reset_vable_token(virtualizable)
-                loop_token = fail_descr.handle_fail(metainterp_sd,
-                                                    jitdriver_sd)
+                # extract and unspecialize the red arguments to pass to
+                # the assembler
+                execute_args = ()
+                for i in range_red_args:
+                    execute_args += (unspecialize_value(args[i]), )
+                # run it!  this executes until interrupted by an exception
+                execute_assembler(loop_token, *execute_args)
+            #
+            assert 0, "should not reach this point"
 
         maybe_compile_and_run._dont_inline_ = True
         self.maybe_compile_and_run = maybe_compile_and_run
+        self.execute_assembler = execute_assembler
         return maybe_compile_and_run
 
     # ----------
@@ -515,56 +505,6 @@
 
     # ----------
 
-    def make_set_future_values(self):
-        "NOT_RPYTHON"
-        if hasattr(self, 'set_future_values'):
-            return self.set_future_values
-
-        jitdriver_sd   = self.jitdriver_sd
-        cpu = self.cpu
-        vinfo = jitdriver_sd.virtualizable_info
-        red_args_types = unrolling_iterable(jitdriver_sd._red_args_types)
-        #
-        def set_future_values(*redargs):
-            i = 0
-            for typecode in red_args_types:
-                set_future_value(cpu, i, redargs[i], typecode)
-                i = i + 1
-            if vinfo is not None:
-                set_future_values_from_vinfo(*redargs)
-        #
-        if vinfo is not None:
-            i0 = len(jitdriver_sd._red_args_types)
-            index_of_virtualizable = jitdriver_sd.index_of_virtualizable
-            vable_static_fields = unrolling_iterable(
-                zip(vinfo.static_extra_types, vinfo.static_fields))
-            vable_array_fields = unrolling_iterable(
-                zip(vinfo.arrayitem_extra_types, vinfo.array_fields))
-            getlength = cpu.ts.getlength
-            getarrayitem = cpu.ts.getarrayitem
-            #
-            def set_future_values_from_vinfo(*redargs):
-                i = i0
-                virtualizable = redargs[index_of_virtualizable]
-                virtualizable = vinfo.cast_to_vtype(virtualizable)
-                for typecode, fieldname in vable_static_fields:
-                    x = getattr(virtualizable, fieldname)
-                    set_future_value(cpu, i, x, typecode)
-                    i = i + 1
-                for typecode, fieldname in vable_array_fields:
-                    lst = getattr(virtualizable, fieldname)
-                    for j in range(getlength(lst)):
-                        x = getarrayitem(lst, j)
-                        set_future_value(cpu, i, x, typecode)
-                        i = i + 1
-        else:
-            set_future_values_from_vinfo = None
-        #
-        self.set_future_values = set_future_values
-        return set_future_values
-
-    # ----------
-
     def make_jitdriver_callbacks(self):
         if hasattr(self, 'get_location_str'):
             return

From noreply at buildbot.pypy.org  Tue Oct 25 22:38:30 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 25 Oct 2011 22:38:30 +0200 (CEST)
Subject: [pypy-commit] pypy default: Typo
Message-ID: <20111025203830.9A187820C7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48459:89631bbf04fe
Date: 2011-10-25 22:38 +0200
http://bitbucket.org/pypy/pypy/changeset/89631bbf04fe/

Log:	Typo

diff --git a/pypy/jit/backend/test/test_ll_random.py b/pypy/jit/backend/test/test_ll_random.py
--- a/pypy/jit/backend/test/test_ll_random.py
+++ b/pypy/jit/backend/test/test_ll_random.py
@@ -652,7 +652,7 @@
     OPERATIONS.append(GetFieldOperation(rop.GETFIELD_GC))
     OPERATIONS.append(GetInteriorFieldOperation(rop.GETINTERIORFIELD_GC))
     OPERATIONS.append(SetFieldOperation(rop.SETFIELD_GC))
-    #OPERATIONS.append(SetInteriorFieldOperation(rop.GETINTERIORFIELD_GC))
+    #OPERATIONS.append(SetInteriorFieldOperation(rop.SETINTERIORFIELD_GC))
     OPERATIONS.append(NewOperation(rop.NEW))
     OPERATIONS.append(NewOperation(rop.NEW_WITH_VTABLE))
 

From noreply at buildbot.pypy.org  Tue Oct 25 22:42:22 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue, 25 Oct 2011 22:42:22 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-dicts: ptr_eq and ptr_ne don't escape
	things.
Message-ID: <20111025204222.A5A6E820C7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: virtual-dicts
Changeset: r48460:8dfa8646de2c
Date: 2011-10-25 16:42 -0400
http://bitbucket.org/pypy/pypy/changeset/8dfa8646de2c/

Log:	ptr_eq and ptr_ne don't escape things.

diff --git a/pypy/jit/metainterp/heapcache.py b/pypy/jit/metainterp/heapcache.py
--- a/pypy/jit/metainterp/heapcache.py
+++ b/pypy/jit/metainterp/heapcache.py
@@ -48,8 +48,12 @@
                 self.dependencies.setdefault(box, []).append(valuebox)
             else:
                 self._escape(valuebox)
-        # GETFIELD_GC and MARK_OPAQUE_PTR doesn't escape their arguments
-        elif opnum != rop.GETFIELD_GC and opnum != rop.MARK_OPAQUE_PTR:
+        # GETFIELD_GC, MARK_OPAQUE_PTR, PTR_EQ, and PTR_NE don't escape their
+        # arguments
+        elif (opnum != rop.GETFIELD_GC and
+              opnum != rop.MARK_OPAQUE_PTR and
+              opnum != rop.PTR_EQ and
+              opnum != rop.PTR_NE):
             idx = 0
             for box in argboxes:
                 # setarrayitem_gc don't escape its first argument

From noreply at buildbot.pypy.org  Tue Oct 25 23:20:36 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 25 Oct 2011 23:20:36 +0200 (CEST)
Subject: [pypy-commit] pypy releasegil-effectinfo: Initial JIT test for the
 manifestation of this issue.
Message-ID: <20111025212036.64517820C7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: releasegil-effectinfo
Changeset: r48461:ce357df2a5c2
Date: 2011-07-18 23:45 +0200
http://bitbucket.org/pypy/pypy/changeset/ce357df2a5c2/

Log:	Initial JIT test for the manifestation of this issue.

diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -2619,5 +2619,42 @@
         self.meta_interp(f, [], enable_opts='')
         self.check_loops(new_with_vtable=1)
 
+
+    def test_release_gil_flush_heap_cache(self):
+        from pypy.rpython.lltypesystem import rffi
+
+        T = rffi.CArrayPtr(rffi.TIME_T)
+
+        external = rffi.llexternal("time", [T], rffi.TIME_T, threadsafe=True)
+        class Lock(object):
+            @dont_look_inside
+            def acquire(self):
+                external(lltype.nullptr(T.TO))
+            @dont_look_inside
+            def release(self):
+                external(lltype.nullptr(T.TO))
+            def dealloc(self):
+                pass
+        @dont_look_inside
+        def get_lst():
+            return [0]
+        myjitdriver = JitDriver(greens=[], reds=["n", "l", "lock"])
+        def f(n):
+            lock = Lock()
+            l = 0
+            while n > 0:
+                myjitdriver.jit_merge_point(lock=lock, l=l, n=n)
+                x = get_lst()
+                l += len(x)
+                lock.acquire()
+                # Thist must not reuse the previous one.
+                n -= len(x)
+                lock.release()
+            lock.dealloc()
+            return n
+        res = self.meta_interp(f, [10])
+        self.check_loops(arraylen_gc=2)
+
+
 class TestLLtype(BaseLLtypeTests, LLJitMixin):
     pass

From noreply at buildbot.pypy.org  Tue Oct 25 23:20:37 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 25 Oct 2011 23:20:37 +0200 (CEST)
Subject: [pypy-commit] pypy default: look into mmap
Message-ID: <20111025212037.9016C820C7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r48462:fc37961a668f
Date: 2011-10-25 23:17 +0200
http://bitbucket.org/pypy/pypy/changeset/fc37961a668f/

Log:	look into mmap

diff --git a/pypy/module/pypyjit/policy.py b/pypy/module/pypyjit/policy.py
--- a/pypy/module/pypyjit/policy.py
+++ b/pypy/module/pypyjit/policy.py
@@ -16,7 +16,8 @@
         if modname in ['pypyjit', 'signal', 'micronumpy', 'math', 'exceptions',
                        'imp', 'sys', 'array', '_ffi', 'itertools', 'operator',
                        'posix', '_socket', '_sre', '_lsprof', '_weakref',
-                       '__pypy__', 'cStringIO', '_collections', 'struct']:
+                       '__pypy__', 'cStringIO', '_collections', 'struct',
+                       'mmap']:
             return True
         return False
 

From noreply at buildbot.pypy.org  Tue Oct 25 23:28:01 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Tue, 25 Oct 2011 23:28:01 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-dicts: copy{str,
	unicode}content can't affect heap caches
Message-ID: <20111025212801.D1322820C7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: virtual-dicts
Changeset: r48463:90eed59807bd
Date: 2011-10-25 17:27 -0400
http://bitbucket.org/pypy/pypy/changeset/90eed59807bd/

Log:	copy{str,unicode}content can't affect heap caches

diff --git a/pypy/jit/metainterp/heapcache.py b/pypy/jit/metainterp/heapcache.py
--- a/pypy/jit/metainterp/heapcache.py
+++ b/pypy/jit/metainterp/heapcache.py
@@ -75,7 +75,9 @@
             opnum == rop.SETARRAYITEM_GC or
             opnum == rop.SETFIELD_RAW or
             opnum == rop.SETARRAYITEM_RAW or
-            opnum == rop.SETINTERIORFIELD_GC):
+            opnum == rop.SETINTERIORFIELD_GC or
+            opnum == rop.COPYSTRCONTENT or
+            opnum == rop.COPYUNICODECONTENT):
             return
         if rop._OVF_FIRST <= opnum <= rop._OVF_LAST:
             return
diff --git a/pypy/jit/metainterp/test/test_tracingopts.py b/pypy/jit/metainterp/test/test_tracingopts.py
--- a/pypy/jit/metainterp/test/test_tracingopts.py
+++ b/pypy/jit/metainterp/test/test_tracingopts.py
@@ -3,6 +3,7 @@
 from pypy.jit.metainterp.test.support import LLJitMixin
 from pypy.rlib import jit
 from pypy.rlib.rarithmetic import ovfcheck
+from pypy.rlib.rstring import StringBuilder
 
 import py
 
@@ -590,4 +591,14 @@
         assert res == 4
         self.check_operations_history(int_add_ovf=0)
         res = self.interp_operations(fn, [sys.maxint])
-        assert res == 12
\ No newline at end of file
+        assert res == 12
+
+    def test_copy_str_content(self):
+        def fn(n):
+            a = StringBuilder()
+            x = [1]
+            a.append("hello world")
+            return x[0]
+        res = self.interp_operations(fn, [0])
+        assert res == 1
+        self.check_operations_history(getarrayitem_gc=0, getarrayitem_gc_pure=0 )
\ No newline at end of file

From noreply at buildbot.pypy.org  Tue Oct 25 23:30:28 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 25 Oct 2011 23:30:28 +0200 (CEST)
Subject: [pypy-commit] pypy releasegil-effectinfo: close long abandoned
	branch
Message-ID: <20111025213028.61FA3820C7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: releasegil-effectinfo
Changeset: r48464:48639d927f25
Date: 2011-10-25 23:30 +0200
http://bitbucket.org/pypy/pypy/changeset/48639d927f25/

Log:	close long abandoned branch


From noreply at buildbot.pypy.org  Wed Oct 26 03:00:10 2011
From: noreply at buildbot.pypy.org (gutworth)
Date: Wed, 26 Oct 2011 03:00:10 +0200 (CEST)
Subject: [pypy-commit] pypy default: I fixed this ages ago.
Message-ID: <20111026010010.617BE820C7@wyvern.cs.uni-duesseldorf.de>

Author: Benjamin Peterson <benjamin at python.org>
Branch: 
Changeset: r48465:7eea9ac595bd
Date: 2011-10-25 20:59 -0400
http://bitbucket.org/pypy/pypy/changeset/7eea9ac595bd/

Log:	I fixed this ages ago.

diff --git a/pypy/rpython/module/ll_os.py b/pypy/rpython/module/ll_os.py
--- a/pypy/rpython/module/ll_os.py
+++ b/pypy/rpython/module/ll_os.py
@@ -959,8 +959,6 @@
                             os_ftruncate(rffi.cast(rffi.INT, fd),
                                          rffi.cast(rffi.LONGLONG, length)))
             if res < 0:
-                # Note: for consistency we raise OSError, but CPython
-                # raises IOError here
                 raise OSError(rposix.get_errno(), "os_ftruncate failed")
 
         return extdef([int, r_longlong], s_None,

From noreply at buildbot.pypy.org  Wed Oct 26 03:03:20 2011
From: noreply at buildbot.pypy.org (timo_jbo)
Date: Wed, 26 Oct 2011 03:03:20 +0200 (CEST)
Subject: [pypy-commit] pypy default: make the test for time.sleep with
 negative values fail (Issue922)
Message-ID: <20111026010320.E252B820C7@wyvern.cs.uni-duesseldorf.de>

Author: Timo Paulssen <timonator at perpetuum-immobile.de>
Branch: 
Changeset: r48466:1ee24bbb619b
Date: 2011-10-26 02:57 +0200
http://bitbucket.org/pypy/pypy/changeset/1ee24bbb619b/

Log:	make the test for time.sleep with negative values fail (Issue922)

diff --git a/pypy/module/rctime/test/test_rctime.py b/pypy/module/rctime/test/test_rctime.py
--- a/pypy/module/rctime/test/test_rctime.py
+++ b/pypy/module/rctime/test/test_rctime.py
@@ -21,7 +21,8 @@
         import os
         raises(TypeError, rctime.sleep, "foo")
         rctime.sleep(1.2345)
-        
+        raises(IOError, rctime.sleep, -1.0)
+
     def test_clock(self):
         import time as rctime
         rctime.clock()

From noreply at buildbot.pypy.org  Wed Oct 26 03:03:22 2011
From: noreply at buildbot.pypy.org (timo_jbo)
Date: Wed, 26 Oct 2011 03:03:22 +0200 (CEST)
Subject: [pypy-commit] pypy default: raise an IOError when sleeping for
 negative time (Issue922)
Message-ID: <20111026010322.1BFC9820C7@wyvern.cs.uni-duesseldorf.de>

Author: Timo Paulssen <timonator at perpetuum-immobile.de>
Branch: 
Changeset: r48467:b5bd720602fb
Date: 2011-10-26 03:00 +0200
http://bitbucket.org/pypy/pypy/changeset/b5bd720602fb/

Log:	raise an IOError when sleeping for negative time (Issue922)

diff --git a/pypy/module/rctime/interp_time.py b/pypy/module/rctime/interp_time.py
--- a/pypy/module/rctime/interp_time.py
+++ b/pypy/module/rctime/interp_time.py
@@ -245,6 +245,9 @@
 if sys.platform != 'win32':
     @unwrap_spec(secs=float)
     def sleep(space, secs):
+        if secs < 0:
+            raise space.OperationError(space.w_IOError,
+                                       space.wrap("Invalid argument: negative time in sleep"))
         pytime.sleep(secs)
 else:
     from pypy.rlib import rwin32
@@ -265,6 +268,9 @@
                                    OSError(EINTR, "sleep() interrupted"))
     @unwrap_spec(secs=float)
     def sleep(space, secs):
+        if secs < 0:
+            raise space.OperationError(space.w_IOError,
+                                       space.wrap("Invalid argument: negative time in sleep"))
         # as decreed by Guido, only the main thread can be
         # interrupted.
         main_thread = space.fromcache(State).main_thread

From noreply at buildbot.pypy.org  Wed Oct 26 03:03:23 2011
From: noreply at buildbot.pypy.org (timo_jbo)
Date: Wed, 26 Oct 2011 03:03:23 +0200 (CEST)
Subject: [pypy-commit] pypy default: Merge heads
Message-ID: <20111026010323.4B1D7820C7@wyvern.cs.uni-duesseldorf.de>

Author: Timo Paulssen <timonator at perpetuum-immobile.de>
Branch: 
Changeset: r48468:c0c84696f00b
Date: 2011-10-26 03:01 +0200
http://bitbucket.org/pypy/pypy/changeset/c0c84696f00b/

Log:	Merge heads

diff --git a/pypy/rpython/module/ll_os.py b/pypy/rpython/module/ll_os.py
--- a/pypy/rpython/module/ll_os.py
+++ b/pypy/rpython/module/ll_os.py
@@ -959,8 +959,6 @@
                             os_ftruncate(rffi.cast(rffi.INT, fd),
                                          rffi.cast(rffi.LONGLONG, length)))
             if res < 0:
-                # Note: for consistency we raise OSError, but CPython
-                # raises IOError here
                 raise OSError(rposix.get_errno(), "os_ftruncate failed")
 
         return extdef([int, r_longlong], s_None,

From noreply at buildbot.pypy.org  Wed Oct 26 09:38:01 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 26 Oct 2011 09:38:01 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix tests and translation. decrease the
	wait time
Message-ID: <20111026073801.3A9E3820C7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r48469:d40090f37f98
Date: 2011-10-26 09:37 +0200
http://bitbucket.org/pypy/pypy/changeset/d40090f37f98/

Log:	fix tests and translation. decrease the wait time

diff --git a/pypy/module/rctime/interp_time.py b/pypy/module/rctime/interp_time.py
--- a/pypy/module/rctime/interp_time.py
+++ b/pypy/module/rctime/interp_time.py
@@ -246,8 +246,8 @@
     @unwrap_spec(secs=float)
     def sleep(space, secs):
         if secs < 0:
-            raise space.OperationError(space.w_IOError,
-                                       space.wrap("Invalid argument: negative time in sleep"))
+            raise OperationError(space.w_IOError,
+                                 space.wrap("Invalid argument: negative time in sleep"))
         pytime.sleep(secs)
 else:
     from pypy.rlib import rwin32
@@ -269,8 +269,8 @@
     @unwrap_spec(secs=float)
     def sleep(space, secs):
         if secs < 0:
-            raise space.OperationError(space.w_IOError,
-                                       space.wrap("Invalid argument: negative time in sleep"))
+            raise OperationError(space.w_IOError,
+                                 space.wrap("Invalid argument: negative time in sleep"))
         # as decreed by Guido, only the main thread can be
         # interrupted.
         main_thread = space.fromcache(State).main_thread
diff --git a/pypy/module/rctime/test/test_rctime.py b/pypy/module/rctime/test/test_rctime.py
--- a/pypy/module/rctime/test/test_rctime.py
+++ b/pypy/module/rctime/test/test_rctime.py
@@ -20,7 +20,7 @@
         import sys
         import os
         raises(TypeError, rctime.sleep, "foo")
-        rctime.sleep(1.2345)
+        rctime.sleep(0.12345)
         raises(IOError, rctime.sleep, -1.0)
 
     def test_clock(self):

From noreply at buildbot.pypy.org  Wed Oct 26 14:19:05 2011
From: noreply at buildbot.pypy.org (cfbolz)
Date: Wed, 26 Oct 2011 14:19:05 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix(?) small ints by doing more
 aggressive copying and by implementing some
Message-ID: <20111026121905.96DB482A87@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r48471:f060ae084c2e
Date: 2011-10-26 14:08 +0200
http://bitbucket.org/pypy/pypy/changeset/f060ae084c2e/

Log:	fix(?) small ints by doing more aggressive copying and by
	implementing some operations that don't need overflow checking
	directly.

diff --git a/pypy/objspace/std/smallintobject.py b/pypy/objspace/std/smallintobject.py
--- a/pypy/objspace/std/smallintobject.py
+++ b/pypy/objspace/std/smallintobject.py
@@ -12,6 +12,7 @@
 from pypy.rlib.rbigint import rbigint
 from pypy.rlib.rarithmetic import r_uint
 from pypy.tool.sourcetools import func_with_new_name
+from pypy.objspace.std.inttype import wrapint
 
 class W_SmallIntObject(W_Object, UnboxedValue):
     __slots__ = 'intval'
@@ -48,14 +49,36 @@
 def delegate_SmallInt2Complex(space, w_small):
     return space.newcomplex(float(w_small.intval), 0.0)
 
+def add__SmallInt_SmallInt(space, w_a, w_b):
+    return wrapint(space, w_a.intval + w_b.intval) # cannot overflow
+
+def sub__SmallInt_SmallInt(space, w_a, w_b):
+    return wrapint(space, w_a.intval - w_b.intval) # cannot overflow
+
+def floordiv__SmallInt_SmallInt(space, w_a, w_b):
+    return wrapint(space, w_a.intval // w_b.intval) # cannot overflow
+
+div__SmallInt_SmallInt = floordiv__SmallInt_SmallInt
+
+def mod__SmallInt_SmallInt(space, w_a, w_b):
+    return wrapint(space, w_a.intval % w_b.intval) # cannot overflow
+
+def divmod__SmallInt_SmallInt(space, w_a, w_b):
+    w = wrapint(space, w_a.intval // w_b.intval) # cannot overflow
+    z = wrapint(space, w_a.intval % w_b.intval)
+    return space.newtuple([w, z])
+
 def copy_multimethods(ns):
     """Copy integer multimethods for small int."""
     for name, func in intobject.__dict__.iteritems():
         if "__Int" in name:
             new_name = name.replace("Int", "SmallInt")
-            # Copy the function, so the annotator specializes it for
-            # W_SmallIntObject.
-            ns[new_name] = func_with_new_name(func, new_name)
+            if new_name not in ns:
+                # Copy the function, so the annotator specializes it for
+                # W_SmallIntObject.
+                ns[new_name] = func = func_with_new_name(func, new_name, globals=ns)
+        else:
+            ns[name] = func
     ns["get_integer"] = ns["pos__SmallInt"] = ns["int__SmallInt"]
     ns["get_negint"] = ns["neg__SmallInt"]
 
diff --git a/pypy/tool/sourcetools.py b/pypy/tool/sourcetools.py
--- a/pypy/tool/sourcetools.py
+++ b/pypy/tool/sourcetools.py
@@ -216,9 +216,11 @@
 
 # ____________________________________________________________
 
-def func_with_new_name(func, newname):
+def func_with_new_name(func, newname, globals=None):
     """Make a renamed copy of a function."""
-    f = new.function(func.func_code, func.func_globals,
+    if globals is None:
+        globals = func.func_globals
+    f = new.function(func.func_code, globals,
                         newname, func.func_defaults,
                         func.func_closure)
     if func.func_dict:

From noreply at buildbot.pypy.org  Wed Oct 26 14:19:04 2011
From: noreply at buildbot.pypy.org (cfbolz)
Date: Wed, 26 Oct 2011 14:19:04 +0200 (CEST)
Subject: [pypy-commit] pypy default: use a specialize:memo instead of adding
 yet another word to the already huge W_TypeObjects.
Message-ID: <20111026121904.5BC20820C7@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r48470:bdebff21909a
Date: 2011-10-25 15:08 +0200
http://bitbucket.org/pypy/pypy/changeset/bdebff21909a/

Log:	use a specialize:memo instead of adding yet another word to the
	already huge W_TypeObjects.

diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -83,11 +83,12 @@
         if self.config.objspace.std.withtproxy:
             transparent.setup(self)
 
+        interplevel_classes = {}
         for type, classes in self.model.typeorder.iteritems():
-            if len(classes) >= 3:
+            if len(classes) >= 3: # XXX what does this 3 mean??!
                 # W_Root, AnyXxx and actual object
-                self.gettypefor(type).interplevel_cls = classes[0][0]
-
+                interplevel_classes[self.gettypefor(type)] = classes[0][0]
+        self._interplevel_classes = interplevel_classes
 
     def get_builtin_types(self):
         return self.builtin_types
@@ -579,7 +580,7 @@
             raise OperationError(self.w_TypeError,
                                  self.wrap("need type object"))
         if is_annotation_constant(w_type):
-            cls = w_type.interplevel_cls
+            cls = self._get_interplevel_cls(w_type)
             if cls is not None:
                 assert w_inst is not None
                 if isinstance(w_inst, cls):
@@ -589,3 +590,9 @@
     @specialize.arg_or_var(2)
     def isinstance_w(space, w_inst, w_type):
         return space._type_isinstance(w_inst, w_type)
+
+    @specialize.memo()
+    def _get_interplevel_cls(self, w_type):
+        if not hasattr(self, "_interplevel_classes"):
+            return None # before running initialize
+        return self._interplevel_classes.get(w_type, None)
diff --git a/pypy/objspace/std/test/test_obj.py b/pypy/objspace/std/test/test_obj.py
--- a/pypy/objspace/std/test/test_obj.py
+++ b/pypy/objspace/std/test/test_obj.py
@@ -102,3 +102,11 @@
             def __repr__(self):
                 return 123456
         assert A().__str__() == 123456
+
+def test_isinstance_shortcut():
+    from pypy.objspace.std import objspace
+    space = objspace.StdObjSpace()
+    w_a = space.wrap("a")
+    space.type = None
+    space.isinstance_w(w_a, space.w_str) # does not crash
+
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -102,7 +102,6 @@
                           'instancetypedef',
                           'terminator',
                           '_version_tag?',
-                          'interplevel_cls',
                           ]
 
     # for config.objspace.std.getattributeshortcut
@@ -117,9 +116,6 @@
     # of the __new__ is an instance of the type
     w_bltin_new = None
 
-    interplevel_cls = None # not None for prebuilt instances of
-                           # interpreter-level types
-
     @dont_look_inside
     def __init__(w_self, space, name, bases_w, dict_w,
                  overridetypedef=None):

From noreply at buildbot.pypy.org  Wed Oct 26 14:19:08 2011
From: noreply at buildbot.pypy.org (cfbolz)
Date: Wed, 26 Oct 2011 14:19:08 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge
Message-ID: <20111026121908.2562B820C7@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r48472:2c94dfea4323
Date: 2011-10-26 14:18 +0200
http://bitbucket.org/pypy/pypy/changeset/2c94dfea4323/

Log:	merge

diff --git a/lib-python/modified-2.7/urllib2.py b/lib-python/modified-2.7/urllib2.py
--- a/lib-python/modified-2.7/urllib2.py
+++ b/lib-python/modified-2.7/urllib2.py
@@ -395,11 +395,7 @@
         meth_name = protocol+"_response"
         for processor in self.process_response.get(protocol, []):
             meth = getattr(processor, meth_name)
-            try:
-                response = meth(req, response)
-            except:
-                response.close()
-                raise
+            response = meth(req, response)
 
         return response
 
diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py
--- a/pypy/jit/backend/llsupport/gc.py
+++ b/pypy/jit/backend/llsupport/gc.py
@@ -649,10 +649,13 @@
         def malloc_basic(size, tid):
             type_id = llop.extract_ushort(llgroup.HALFWORD, tid)
             has_finalizer = bool(tid & (1<<llgroup.HALFSHIFT))
+            has_light_finalizer = bool(tid & (1<<(llgroup.HALFSHIFT + 1)))
             check_typeid(type_id)
             res = llop1.do_malloc_fixedsize_clear(llmemory.GCREF,
                                                   type_id, size,
-                                                  has_finalizer, False)
+                                                  has_finalizer,
+                                                  has_light_finalizer,
+                                                  False)
             # In case the operation above failed, we are returning NULL
             # from this function to assembler.  There is also an RPython
             # exception set, typically MemoryError; but it's easier and
@@ -723,7 +726,7 @@
             # also use it to allocate varsized objects.  The tid
             # and possibly the length are both set afterward.
             gcref = llop1.do_malloc_fixedsize_clear(llmemory.GCREF,
-                                        0, size, False, False)
+                                        0, size, False, False, False)
             return rffi.cast(lltype.Signed, gcref)
         self.malloc_slowpath = malloc_slowpath
         self.MALLOC_SLOWPATH = lltype.FuncType([lltype.Signed], lltype.Signed)
@@ -747,7 +750,9 @@
         type_id = self.layoutbuilder.get_type_id(S)
         assert not self.layoutbuilder.is_weakref_type(S)
         has_finalizer = bool(self.layoutbuilder.has_finalizer(S))
-        flags = int(has_finalizer) << llgroup.HALFSHIFT
+        has_light_finalizer = bool(self.layoutbuilder.has_light_finalizer(S))
+        flags = (int(has_finalizer) << llgroup.HALFSHIFT |
+                 int(has_light_finalizer) << (llgroup.HALFSHIFT + 1))
         descr.tid = llop.combine_ushort(lltype.Signed, type_id, flags)
 
     def init_array_descr(self, A, descr):
diff --git a/pypy/jit/backend/llsupport/test/test_gc.py b/pypy/jit/backend/llsupport/test/test_gc.py
--- a/pypy/jit/backend/llsupport/test/test_gc.py
+++ b/pypy/jit/backend/llsupport/test/test_gc.py
@@ -247,12 +247,14 @@
         self.record = []
 
     def do_malloc_fixedsize_clear(self, RESTYPE, type_id, size,
-                                  has_finalizer, contains_weakptr):
+                                  has_finalizer, has_light_finalizer,
+                                  contains_weakptr):
         assert not contains_weakptr
+        assert not has_finalizer           # in these tests
+        assert not has_light_finalizer     # in these tests
         p = llmemory.raw_malloc(size)
         p = llmemory.cast_adr_to_ptr(p, RESTYPE)
-        flags = int(has_finalizer) << 16
-        tid = llop.combine_ushort(lltype.Signed, type_id, flags)
+        tid = llop.combine_ushort(lltype.Signed, type_id, 0)
         self.record.append(("fixedsize", repr(size), tid, p))
         return p
 
diff --git a/pypy/jit/backend/test/test_ll_random.py b/pypy/jit/backend/test/test_ll_random.py
--- a/pypy/jit/backend/test/test_ll_random.py
+++ b/pypy/jit/backend/test/test_ll_random.py
@@ -28,16 +28,27 @@
         fork.structure_types_and_vtables = self.structure_types_and_vtables
         return fork
 
-    def get_structptr_var(self, r, must_have_vtable=False, type=lltype.Struct):
+    def _choose_ptr_vars(self, from_, type, array_of_structs):
+        ptrvars = []
+        for i in range(len(from_)):
+            v, S = from_[i][:2]
+            if not isinstance(S, type):
+                continue
+            if (isinstance(S, lltype.Array) and
+                isinstance(S.OF, lltype.Struct) == array_of_structs):
+                ptrvars.append((v, S))
+        return ptrvars
+
+    def get_structptr_var(self, r, must_have_vtable=False, type=lltype.Struct,
+                          array_of_structs=False):
         while True:
-            ptrvars = [(v, S) for (v, S) in self.ptrvars
-                              if isinstance(S, type)]
+            ptrvars = self._choose_ptr_vars(self.ptrvars, type,
+                                            array_of_structs)
             if ptrvars and r.random() < 0.8:
                 v, S = r.choice(ptrvars)
             else:
-                prebuilt_ptr_consts = [(v, S)
-                                 for (v, S, _) in self.prebuilt_ptr_consts
-                                 if isinstance(S, type)]
+                prebuilt_ptr_consts = self._choose_ptr_vars(
+                    self.prebuilt_ptr_consts, type, array_of_structs)
                 if prebuilt_ptr_consts and r.random() < 0.7:
                     v, S = r.choice(prebuilt_ptr_consts)
                 else:
@@ -48,7 +59,8 @@
                                                 has_vtable=must_have_vtable)
                     else:
                         # create a new constant array
-                        p = self.get_random_array(r)
+                        p = self.get_random_array(r,
+                                    must_be_array_of_structs=array_of_structs)
                     S = lltype.typeOf(p).TO
                     v = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, p))
                     self.prebuilt_ptr_consts.append((v, S,
@@ -74,7 +86,8 @@
                 TYPE = lltype.Signed
         return TYPE
 
-    def get_random_structure_type(self, r, with_vtable=None, cache=True):
+    def get_random_structure_type(self, r, with_vtable=None, cache=True,
+                                  type=lltype.GcStruct):
         if cache and self.structure_types and r.random() < 0.5:
             return r.choice(self.structure_types)
         fields = []
@@ -85,7 +98,7 @@
         for i in range(r.randrange(1, 5)):
             TYPE = self.get_random_primitive_type(r)
             fields.append(('f%d' % i, TYPE))
-        S = lltype.GcStruct('S%d' % self.counter, *fields, **kwds)
+        S = type('S%d' % self.counter, *fields, **kwds)
         self.counter += 1
         if cache:
             self.structure_types.append(S)
@@ -125,17 +138,29 @@
                 setattr(p, fieldname, rffi.cast(TYPE, r.random_integer()))
         return p
 
-    def get_random_array_type(self, r):
-        TYPE = self.get_random_primitive_type(r)
+    def get_random_array_type(self, r, can_be_array_of_struct=False,
+                              must_be_array_of_structs=False):
+        if ((can_be_array_of_struct and r.random() < 0.1) or
+            must_be_array_of_structs):
+            TYPE = self.get_random_structure_type(r, cache=False,
+                                                  type=lltype.Struct)
+        else:
+            TYPE = self.get_random_primitive_type(r)
         return lltype.GcArray(TYPE)
 
-    def get_random_array(self, r):
-        A = self.get_random_array_type(r)
+    def get_random_array(self, r, must_be_array_of_structs=False):
+        A = self.get_random_array_type(r,
+                           must_be_array_of_structs=must_be_array_of_structs)
         length = (r.random_integer() // 15) % 300  # length: between 0 and 299
                                                    # likely to be small
         p = lltype.malloc(A, length)
-        for i in range(length):
-            p[i] = rffi.cast(A.OF, r.random_integer())
+        if isinstance(A.OF, lltype.Primitive):
+            for i in range(length):
+                p[i] = rffi.cast(A.OF, r.random_integer())
+        else:
+            for i in range(length):
+                for fname, TP in A.OF._flds.iteritems():
+                    setattr(p[i], fname, rffi.cast(TP, r.random_integer()))
         return p
 
     def get_index(self, length, r):
@@ -220,7 +245,7 @@
 
 class GetFieldOperation(test_random.AbstractOperation):
     def field_descr(self, builder, r):
-        v, S = builder.get_structptr_var(r)
+        v, S = builder.get_structptr_var(r, )
         names = S._names
         if names[0] == 'parent':
             names = names[1:]
@@ -239,6 +264,31 @@
                 continue
             break
 
+class GetInteriorFieldOperation(test_random.AbstractOperation):
+    def field_descr(self, builder, r):
+        v, A = builder.get_structptr_var(r, type=lltype.Array,
+                                         array_of_structs=True)
+        array = v.getref(lltype.Ptr(A))
+        v_index = builder.get_index(len(array), r)
+        names = A.OF._names
+        if names[0] == 'parent':
+            names = names[1:]
+        name = r.choice(names)
+        descr = builder.cpu.interiorfielddescrof(A, name)
+        descr._random_info = 'cpu.interiorfielddescrof(%s, %r)' % (A.OF._name,
+                                                                   name)
+        TYPE = getattr(A.OF, name)
+        return v, v_index, descr, TYPE
+
+    def produce_into(self, builder, r):
+        while True:
+            try:
+                v, v_index, descr, _ = self.field_descr(builder, r)
+                self.put(builder, [v, v_index], descr)
+            except lltype.UninitializedMemoryAccess:
+                continue
+            break
+
 class SetFieldOperation(GetFieldOperation):
     def produce_into(self, builder, r):
         v, descr, TYPE = self.field_descr(builder, r)
@@ -251,6 +301,20 @@
                 break
         builder.do(self.opnum, [v, w], descr)
 
+class SetInteriorFieldOperation(GetFieldOperation):
+    def produce_into(self, builder, r):
+        import pdb
+        pdb.set_trace()
+        v, descr, TYPE = self.field_descr(builder, r)
+        while True:
+            if r.random() < 0.3:
+                w = ConstInt(r.random_integer())
+            else:
+                w = r.choice(builder.intvars)
+            if rffi.cast(lltype.Signed, rffi.cast(TYPE, w.value)) == w.value:
+                break
+        builder.do(self.opnum, [v, w], descr)
+
 class NewOperation(test_random.AbstractOperation):
     def size_descr(self, builder, S):
         descr = builder.cpu.sizeof(S)
@@ -306,7 +370,7 @@
 
 class NewArrayOperation(ArrayOperation):
     def produce_into(self, builder, r):
-        A = builder.get_random_array_type(r)
+        A = builder.get_random_array_type(r, can_be_array_of_struct=True)
         v_size = builder.get_index(300, r)
         v_ptr = builder.do(self.opnum, [v_size], self.array_descr(builder, A))
         builder.ptrvars.append((v_ptr, A))
@@ -586,7 +650,9 @@
 for i in range(4):      # make more common
     OPERATIONS.append(GetFieldOperation(rop.GETFIELD_GC))
     OPERATIONS.append(GetFieldOperation(rop.GETFIELD_GC))
+    OPERATIONS.append(GetInteriorFieldOperation(rop.GETINTERIORFIELD_GC))
     OPERATIONS.append(SetFieldOperation(rop.SETFIELD_GC))
+    #OPERATIONS.append(SetInteriorFieldOperation(rop.SETINTERIORFIELD_GC))
     OPERATIONS.append(NewOperation(rop.NEW))
     OPERATIONS.append(NewOperation(rop.NEW_WITH_VTABLE))
 
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1276,8 +1276,8 @@
     genop_int_ne = _cmpop("NE", "NE")
     genop_int_gt = _cmpop("G", "L")
     genop_int_ge = _cmpop("GE", "LE")
-    genop_ptr_eq = genop_int_eq
-    genop_ptr_ne = genop_int_ne
+    genop_ptr_eq = genop_instance_ptr_eq = genop_int_eq
+    genop_ptr_ne = genop_instance_ptr_ne = genop_int_ne
 
     genop_float_lt = _cmpop_float('B', 'A')
     genop_float_le = _cmpop_float('BE', 'AE')
@@ -1297,8 +1297,8 @@
     genop_guard_int_ne = _cmpop_guard("NE", "NE", "E", "E")
     genop_guard_int_gt = _cmpop_guard("G", "L", "LE", "GE")
     genop_guard_int_ge = _cmpop_guard("GE", "LE", "L", "G")
-    genop_guard_ptr_eq = genop_guard_int_eq
-    genop_guard_ptr_ne = genop_guard_int_ne
+    genop_guard_ptr_eq = genop_guard_instance_ptr_eq = genop_guard_int_eq
+    genop_guard_ptr_ne = genop_guard_instance_ptr_ne = genop_guard_int_ne
 
     genop_guard_uint_gt = _cmpop_guard("A", "B", "BE", "AE")
     genop_guard_uint_lt = _cmpop_guard("B", "A", "AE", "BE")
diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py
--- a/pypy/jit/backend/x86/regalloc.py
+++ b/pypy/jit/backend/x86/regalloc.py
@@ -651,8 +651,8 @@
     consider_uint_lt = _consider_compop
     consider_uint_le = _consider_compop
     consider_uint_ge = _consider_compop
-    consider_ptr_eq = _consider_compop
-    consider_ptr_ne = _consider_compop
+    consider_ptr_eq = consider_instance_ptr_eq = _consider_compop
+    consider_ptr_ne = consider_instance_ptr_ne = _consider_compop
 
     def _consider_float_op(self, op):
         loc1 = self.xrm.loc(op.getarg(1))
diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -443,6 +443,8 @@
     rewrite_op_gc_identityhash = _do_builtin_call
     rewrite_op_gc_id           = _do_builtin_call
     rewrite_op_uint_mod        = _do_builtin_call
+    rewrite_op_cast_float_to_uint = _do_builtin_call
+    rewrite_op_cast_uint_to_float = _do_builtin_call
 
     # ----------
     # getfield/setfield/mallocs etc.
@@ -798,6 +800,9 @@
     def _is_gc(self, v):
         return getattr(getattr(v.concretetype, "TO", None), "_gckind", "?") == 'gc'
 
+    def _is_rclass_instance(self, v):
+        return lltype._castdepth(v.concretetype.TO, rclass.OBJECT) >= 0
+
     def _rewrite_cmp_ptrs(self, op):
         if self._is_gc(op.args[0]):
             return op
@@ -815,11 +820,21 @@
         return self._rewrite_equality(op, 'int_is_true')
 
     def rewrite_op_ptr_eq(self, op):
-        op1 = self._rewrite_equality(op, 'ptr_iszero')
+        prefix = ''
+        if self._is_rclass_instance(op.args[0]):
+            assert self._is_rclass_instance(op.args[1])
+            op = SpaceOperation('instance_ptr_eq', op.args, op.result)
+            prefix = 'instance_'
+        op1 = self._rewrite_equality(op, prefix + 'ptr_iszero')
         return self._rewrite_cmp_ptrs(op1)
 
     def rewrite_op_ptr_ne(self, op):
-        op1 = self._rewrite_equality(op, 'ptr_nonzero')
+        prefix = ''
+        if self._is_rclass_instance(op.args[0]):
+            assert self._is_rclass_instance(op.args[1])
+            op = SpaceOperation('instance_ptr_ne', op.args, op.result)
+            prefix = 'instance_'
+        op1 = self._rewrite_equality(op, prefix + 'ptr_nonzero')
         return self._rewrite_cmp_ptrs(op1)
 
     rewrite_op_ptr_iszero = _rewrite_cmp_ptrs
@@ -848,26 +863,44 @@
         elif not float_arg and float_res:
             # some int -> some float
             ops = []
-            v1 = varoftype(lltype.Signed)
-            oplist = self.rewrite_operation(
-                SpaceOperation('force_cast', [v_arg], v1)
-            )
-            if oplist:
-                ops.extend(oplist)
+            v2 = varoftype(lltype.Float)
+            sizesign = rffi.size_and_sign(v_arg.concretetype)
+            if sizesign <= rffi.size_and_sign(lltype.Signed):
+                # cast from a type that fits in an int: either the size is
+                # smaller, or it is equal and it is not unsigned
+                v1 = varoftype(lltype.Signed)
+                oplist = self.rewrite_operation(
+                    SpaceOperation('force_cast', [v_arg], v1)
+                )
+                if oplist:
+                    ops.extend(oplist)
+                else:
+                    v1 = v_arg
+                op = self.rewrite_operation(
+                    SpaceOperation('cast_int_to_float', [v1], v2)
+                )
+                ops.append(op)
             else:
-                v1 = v_arg
-            v2 = varoftype(lltype.Float)
-            op = self.rewrite_operation(
-                SpaceOperation('cast_int_to_float', [v1], v2)
-            )
-            ops.append(op)
+                if sizesign == rffi.size_and_sign(lltype.Unsigned):
+                    opname = 'cast_uint_to_float'
+                elif sizesign == rffi.size_and_sign(lltype.SignedLongLong):
+                    opname = 'cast_longlong_to_float'
+                elif sizesign == rffi.size_and_sign(lltype.UnsignedLongLong):
+                    opname = 'cast_ulonglong_to_float'
+                else:
+                    raise AssertionError('cast_x_to_float: %r' % (sizesign,))
+                ops1 = self.rewrite_operation(
+                    SpaceOperation(opname, [v_arg], v2)
+                )
+                if not isinstance(ops1, list): ops1 = [ops1]
+                ops.extend(ops1)
             op2 = self.rewrite_operation(
                 SpaceOperation('force_cast', [v2], v_result)
             )
             if op2:
                 ops.append(op2)
             else:
-                op.result = v_result
+                ops[-1].result = v_result
             return ops
         elif float_arg and not float_res:
             # some float -> some int
@@ -880,18 +913,36 @@
                 ops.append(op1)
             else:
                 v1 = v_arg
-            v2 = varoftype(lltype.Signed)
-            op = self.rewrite_operation(
-                SpaceOperation('cast_float_to_int', [v1], v2)
-            )
-            ops.append(op)
-            oplist = self.rewrite_operation(
-                SpaceOperation('force_cast', [v2], v_result)
-            )
-            if oplist:
-                ops.extend(oplist)
+            sizesign = rffi.size_and_sign(v_result.concretetype)
+            if sizesign <= rffi.size_and_sign(lltype.Signed):
+                # cast to a type that fits in an int: either the size is
+                # smaller, or it is equal and it is not unsigned
+                v2 = varoftype(lltype.Signed)
+                op = self.rewrite_operation(
+                    SpaceOperation('cast_float_to_int', [v1], v2)
+                )
+                ops.append(op)
+                oplist = self.rewrite_operation(
+                    SpaceOperation('force_cast', [v2], v_result)
+                )
+                if oplist:
+                    ops.extend(oplist)
+                else:
+                    op.result = v_result
             else:
-                op.result = v_result
+                if sizesign == rffi.size_and_sign(lltype.Unsigned):
+                    opname = 'cast_float_to_uint'
+                elif sizesign == rffi.size_and_sign(lltype.SignedLongLong):
+                    opname = 'cast_float_to_longlong'
+                elif sizesign == rffi.size_and_sign(lltype.UnsignedLongLong):
+                    opname = 'cast_float_to_ulonglong'
+                else:
+                    raise AssertionError('cast_float_to_x: %r' % (sizesign,))
+                ops1 = self.rewrite_operation(
+                    SpaceOperation(opname, [v1], v_result)
+                )
+                if not isinstance(ops1, list): ops1 = [ops1]
+                ops.extend(ops1)
             return ops
         else:
             assert False
@@ -1097,8 +1148,6 @@
     # The new operation is optionally further processed by rewrite_operation().
     for _old, _new in [('bool_not', 'int_is_zero'),
                        ('cast_bool_to_float', 'cast_int_to_float'),
-                       ('cast_uint_to_float', 'cast_int_to_float'),
-                       ('cast_float_to_uint', 'cast_float_to_int'),
 
                        ('int_add_nonneg_ovf', 'int_add_ovf'),
                        ('keepalive', '-live-'),
diff --git a/pypy/jit/codewriter/support.py b/pypy/jit/codewriter/support.py
--- a/pypy/jit/codewriter/support.py
+++ b/pypy/jit/codewriter/support.py
@@ -229,6 +229,17 @@
     else:
         return x
 
+def _ll_1_cast_uint_to_float(x):
+    # XXX on 32-bit platforms, this should be done using cast_longlong_to_float
+    # (which is a residual call right now in the x86 backend)
+    return llop.cast_uint_to_float(lltype.Float, x)
+
+def _ll_1_cast_float_to_uint(x):
+    # XXX on 32-bit platforms, this should be done using cast_float_to_longlong
+    # (which is a residual call right now in the x86 backend)
+    return llop.cast_float_to_uint(lltype.Unsigned, x)
+
+
 # math support
 # ------------
 
diff --git a/pypy/jit/codewriter/test/test_flatten.py b/pypy/jit/codewriter/test/test_flatten.py
--- a/pypy/jit/codewriter/test/test_flatten.py
+++ b/pypy/jit/codewriter/test/test_flatten.py
@@ -8,7 +8,7 @@
 from pypy.rpython.lltypesystem import lltype, rclass, rstr
 from pypy.objspace.flow.model import SpaceOperation, Variable, Constant
 from pypy.translator.unsimplify import varoftype
-from pypy.rlib.rarithmetic import ovfcheck, r_uint
+from pypy.rlib.rarithmetic import ovfcheck, r_uint, r_longlong, r_ulonglong
 from pypy.rlib.jit import dont_look_inside, _we_are_jitted, JitDriver
 from pypy.rlib.objectmodel import keepalive_until_here
 from pypy.rlib import jit
@@ -70,7 +70,8 @@
         return 'residual'
     def getcalldescr(self, op, oopspecindex=None, extraeffect=None):
         try:
-            if 'cannot_raise' in op.args[0].value._obj.graph.name:
+            name = op.args[0].value._obj._name
+            if 'cannot_raise' in name or name.startswith('cast_'):
                 return self._descr_cannot_raise
         except AttributeError:
             pass
@@ -900,6 +901,67 @@
             int_return %i4
         """, transform=True)
 
+        def f(dbl):
+            return rffi.cast(rffi.UCHAR, dbl)
+        self.encoding_test(f, [12.456], """
+            cast_float_to_int %f0 -> %i0
+            int_and %i0, $255 -> %i1
+            int_return %i1
+        """, transform=True)
+
+        def f(dbl):
+            return rffi.cast(lltype.Unsigned, dbl)
+        self.encoding_test(f, [12.456], """
+            residual_call_irf_i $<* fn cast_float_to_uint>, <Descr>, I[], R[], F[%f0] -> %i0
+            int_return %i0
+        """, transform=True)
+
+        def f(i):
+            return rffi.cast(lltype.Float, chr(i))    # "char -> float"
+        self.encoding_test(f, [12], """
+            cast_int_to_float %i0 -> %f0
+            float_return %f0
+        """, transform=True)
+
+        def f(i):
+            return rffi.cast(lltype.Float, r_uint(i))    # "uint -> float"
+        self.encoding_test(f, [12], """
+            residual_call_irf_f $<* fn cast_uint_to_float>, <Descr>, I[%i0], R[], F[] -> %f0
+            float_return %f0
+        """, transform=True)
+
+        if not longlong.is_64_bit:
+            def f(dbl):
+                return rffi.cast(lltype.SignedLongLong, dbl)
+            self.encoding_test(f, [12.3], """
+                residual_call_irf_f $<* fn llong_from_float>, <Descr>, I[], R[], F[%f0] -> %f1
+                float_return %f1
+            """, transform=True)
+
+            def f(dbl):
+                return rffi.cast(lltype.UnsignedLongLong, dbl)
+            self.encoding_test(f, [12.3], """
+                residual_call_irf_f $<* fn ullong_from_float>, <Descr>, I[], R[], F[%f0] -> %f1
+                float_return %f1
+            """, transform=True)
+
+            def f(x):
+                ll = r_longlong(x)
+                return rffi.cast(lltype.Float, ll)
+            self.encoding_test(f, [12], """
+                residual_call_irf_f $<* fn llong_from_int>, <Descr>, I[%i0], R[], F[] -> %f0
+                residual_call_irf_f $<* fn llong_to_float>, <Descr>, I[], R[], F[%f0] -> %f1
+                float_return %f1
+            """, transform=True)
+
+            def f(x):
+                ll = r_ulonglong(x)
+                return rffi.cast(lltype.Float, ll)
+            self.encoding_test(f, [12], """
+                residual_call_irf_f $<* fn ullong_from_int>, <Descr>, I[%i0], R[], F[] -> %f0
+                residual_call_irf_f $<* fn ullong_u_to_float>, <Descr>, I[], R[], F[%f0] -> %f1
+                float_return %f1
+            """, transform=True)
 
     def test_direct_ptradd(self):
         from pypy.rpython.lltypesystem import rffi
diff --git a/pypy/jit/codewriter/test/test_jtransform.py b/pypy/jit/codewriter/test/test_jtransform.py
--- a/pypy/jit/codewriter/test/test_jtransform.py
+++ b/pypy/jit/codewriter/test/test_jtransform.py
@@ -576,10 +576,10 @@
         assert op1.args == [v2]
 
 def test_ptr_eq():
-    v1 = varoftype(rclass.OBJECTPTR)
-    v2 = varoftype(rclass.OBJECTPTR)
+    v1 = varoftype(lltype.Ptr(rstr.STR))
+    v2 = varoftype(lltype.Ptr(rstr.STR))
     v3 = varoftype(lltype.Bool)
-    c0 = const(lltype.nullptr(rclass.OBJECT))
+    c0 = const(lltype.nullptr(rstr.STR))
     #
     for opname, reducedname in [('ptr_eq', 'ptr_iszero'),
                                 ('ptr_ne', 'ptr_nonzero')]:
@@ -598,6 +598,31 @@
         assert op1.opname == reducedname
         assert op1.args == [v2]
 
+def test_instance_ptr_eq():
+    v1 = varoftype(rclass.OBJECTPTR)
+    v2 = varoftype(rclass.OBJECTPTR)
+    v3 = varoftype(lltype.Bool)
+    c0 = const(lltype.nullptr(rclass.OBJECT))
+
+    for opname, newopname, reducedname in [
+        ('ptr_eq', 'instance_ptr_eq', 'instance_ptr_iszero'),
+        ('ptr_ne', 'instance_ptr_ne', 'instance_ptr_nonzero')
+    ]:
+        op = SpaceOperation(opname, [v1, v2], v3)
+        op1 = Transformer().rewrite_operation(op)
+        assert op1.opname == newopname
+        assert op1.args == [v1, v2]
+
+        op = SpaceOperation(opname, [v1, c0], v3)
+        op1 = Transformer().rewrite_operation(op)
+        assert op1.opname == reducedname
+        assert op1.args == [v1]
+
+        op = SpaceOperation(opname, [c0, v1], v3)
+        op1 = Transformer().rewrite_operation(op)
+        assert op1.opname == reducedname
+        assert op1.args == [v1]
+
 def test_nongc_ptr_eq():
     v1 = varoftype(rclass.NONGCOBJECTPTR)
     v2 = varoftype(rclass.NONGCOBJECTPTR)
diff --git a/pypy/jit/metainterp/blackhole.py b/pypy/jit/metainterp/blackhole.py
--- a/pypy/jit/metainterp/blackhole.py
+++ b/pypy/jit/metainterp/blackhole.py
@@ -499,6 +499,12 @@
     @arguments("r", returns="i")
     def bhimpl_ptr_nonzero(a):
         return bool(a)
+    @arguments("r", "r", returns="i")
+    def bhimpl_instance_ptr_eq(a, b):
+        return a == b
+    @arguments("r", "r", returns="i")
+    def bhimpl_instance_ptr_ne(a, b):
+        return a != b
     @arguments("r", returns="r")
     def bhimpl_cast_opaque_ptr(a):
         return a
@@ -630,6 +636,9 @@
         a = longlong.getrealfloat(a)
         # note: we need to call int() twice to care for the fact that
         # int(-2147483648.0) returns a long :-(
+        # we could also call intmask() instead of the outermost int(), but
+        # it's probably better to explicitly crash (by getting a long) if a
+        # non-translated version tries to cast a too large float to an int.
         return int(int(a))
 
     @arguments("i", returns="f")
diff --git a/pypy/jit/metainterp/optimizeopt/rewrite.py b/pypy/jit/metainterp/optimizeopt/rewrite.py
--- a/pypy/jit/metainterp/optimizeopt/rewrite.py
+++ b/pypy/jit/metainterp/optimizeopt/rewrite.py
@@ -337,7 +337,7 @@
     def optimize_INT_IS_ZERO(self, op):
         self._optimize_nullness(op, op.getarg(0), False)
 
-    def _optimize_oois_ooisnot(self, op, expect_isnot):
+    def _optimize_oois_ooisnot(self, op, expect_isnot, instance):
         value0 = self.getvalue(op.getarg(0))
         value1 = self.getvalue(op.getarg(1))
         if value0.is_virtual():
@@ -355,21 +355,28 @@
         elif value0 is value1:
             self.make_constant_int(op.result, not expect_isnot)
         else:
-            cls0 = value0.get_constant_class(self.optimizer.cpu)
-            if cls0 is not None:
-                cls1 = value1.get_constant_class(self.optimizer.cpu)
-                if cls1 is not None and not cls0.same_constant(cls1):
-                    # cannot be the same object, as we know that their
-                    # class is different
-                    self.make_constant_int(op.result, expect_isnot)
-                    return
+            if instance:
+                cls0 = value0.get_constant_class(self.optimizer.cpu)
+                if cls0 is not None:
+                    cls1 = value1.get_constant_class(self.optimizer.cpu)
+                    if cls1 is not None and not cls0.same_constant(cls1):
+                        # cannot be the same object, as we know that their
+                        # class is different
+                        self.make_constant_int(op.result, expect_isnot)
+                        return
             self.emit_operation(op)
 
+    def optimize_PTR_EQ(self, op):
+        self._optimize_oois_ooisnot(op, False, False)
+
     def optimize_PTR_NE(self, op):
-        self._optimize_oois_ooisnot(op, True)
+        self._optimize_oois_ooisnot(op, True, False)
 
-    def optimize_PTR_EQ(self, op):
-        self._optimize_oois_ooisnot(op, False)
+    def optimize_INSTANCE_PTR_EQ(self, op):
+        self._optimize_oois_ooisnot(op, False, True)
+
+    def optimize_INSTANCE_PTR_NE(self, op):
+        self._optimize_oois_ooisnot(op, True, True)
 
 ##    def optimize_INSTANCEOF(self, op):
 ##        value = self.getvalue(op.args[0])
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -508,13 +508,13 @@
         ops = """
         [p0]
         guard_class(p0, ConstClass(node_vtable)) []
-        i0 = ptr_ne(p0, NULL)
+        i0 = instance_ptr_ne(p0, NULL)
         guard_true(i0) []
-        i1 = ptr_eq(p0, NULL)
+        i1 = instance_ptr_eq(p0, NULL)
         guard_false(i1) []
-        i2 = ptr_ne(NULL, p0)
+        i2 = instance_ptr_ne(NULL, p0)
         guard_true(i0) []
-        i3 = ptr_eq(NULL, p0)
+        i3 = instance_ptr_eq(NULL, p0)
         guard_false(i1) []
         jump(p0)
         """
@@ -2026,7 +2026,7 @@
         ops = """
         [p1]
         guard_class(p1, ConstClass(node_vtable2)) []
-        i = ptr_ne(ConstPtr(myptr), p1)
+        i = instance_ptr_ne(ConstPtr(myptr), p1)
         guard_true(i) []
         jump(p1)
         """
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -2683,7 +2683,7 @@
         ops = """
         [p1]
         guard_class(p1, ConstClass(node_vtable2)) []
-        i = ptr_ne(ConstPtr(myptr), p1)
+        i = instance_ptr_ne(ConstPtr(myptr), p1)
         guard_true(i) []
         jump(p1)
         """
@@ -3331,7 +3331,7 @@
         jump(p1, i1, i2, i6)
         '''
         self.optimize_loop(ops, expected, preamble)
-        
+
 
     # ----------
 
@@ -7280,7 +7280,7 @@
         ops = """
         [p1, p2]
         setarrayitem_gc(p1, 2, 10, descr=arraydescr)
-        setarrayitem_gc(p2, 3, 13, descr=arraydescr)        
+        setarrayitem_gc(p2, 3, 13, descr=arraydescr)
         call(0, p1, p2, 0, 0, 10, descr=arraycopydescr)
         jump(p1, p2)
         """
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -165,7 +165,7 @@
         if not we_are_translated():
             for b in registers[count:]:
                 assert not oldbox.same_box(b)
-                
+
 
     def make_result_of_lastop(self, resultbox):
         got_type = resultbox.type
@@ -199,7 +199,7 @@
                     'float_add', 'float_sub', 'float_mul', 'float_truediv',
                     'float_lt', 'float_le', 'float_eq',
                     'float_ne', 'float_gt', 'float_ge',
-                    'ptr_eq', 'ptr_ne',
+                    'ptr_eq', 'ptr_ne', 'instance_ptr_eq', 'instance_ptr_ne',
                     ]:
         exec py.code.Source('''
             @arguments("box", "box")
@@ -604,7 +604,7 @@
     opimpl_setinteriorfield_gc_i = _opimpl_setinteriorfield_gc_any
     opimpl_setinteriorfield_gc_f = _opimpl_setinteriorfield_gc_any
     opimpl_setinteriorfield_gc_r = _opimpl_setinteriorfield_gc_any
-                                
+
 
     @arguments("box", "descr")
     def _opimpl_getfield_raw_any(self, box, fielddescr):
diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py
--- a/pypy/jit/metainterp/resoperation.py
+++ b/pypy/jit/metainterp/resoperation.py
@@ -404,8 +404,8 @@
     'FLOAT_TRUEDIV/2',
     'FLOAT_NEG/1',
     'FLOAT_ABS/1',
-    'CAST_FLOAT_TO_INT/1',
-    'CAST_INT_TO_FLOAT/1',
+    'CAST_FLOAT_TO_INT/1',          # don't use for unsigned ints; we would
+    'CAST_INT_TO_FLOAT/1',          # need some messy code in the backend
     'CAST_FLOAT_TO_SINGLEFLOAT/1',
     'CAST_SINGLEFLOAT_TO_FLOAT/1',
     #
@@ -437,6 +437,8 @@
     #
     'PTR_EQ/2b',
     'PTR_NE/2b',
+    'INSTANCE_PTR_EQ/2b',
+    'INSTANCE_PTR_NE/2b',
     'CAST_OPAQUE_PTR/1b',
     #
     'ARRAYLEN_GC/1d',
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -3436,7 +3436,7 @@
         res = self.meta_interp(f, [16])
         assert res == f(16)
 
-    def test_ptr_eq_str_constants(self):
+    def test_ptr_eq(self):
         myjitdriver = JitDriver(greens = [], reds = ["n", "x"])
         class A(object):
             def __init__(self, v):
@@ -3452,6 +3452,42 @@
         res = self.meta_interp(f, [10, 1])
         assert res == 0
 
+    def test_instance_ptr_eq(self):
+        myjitdriver = JitDriver(greens = [], reds = ["n", "i", "a1", "a2"])
+        class A(object):
+            pass
+        def f(n):
+            a1 = A()
+            a2 = A()
+            i = 0
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, i=i, a1=a1, a2=a2)
+                if n % 2:
+                    a = a2
+                else:
+                    a = a1
+                i += a is a1
+                n -= 1
+            return i
+        res = self.meta_interp(f, [10])
+        assert res == f(10)
+        def f(n):
+            a1 = A()
+            a2 = A()
+            i = 0
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, i=i, a1=a1, a2=a2)
+                if n % 2:
+                    a = a2
+                else:
+                    a = a1
+                if a is a2:
+                    i += 1
+                n -= 1
+            return i
+        res = self.meta_interp(f, [10])
+        assert res == f(10)
+
     def test_virtual_array_of_structs(self):
         myjitdriver = JitDriver(greens = [], reds=["n", "d"])
         def f(n):
diff --git a/pypy/jit/metainterp/test/test_float.py b/pypy/jit/metainterp/test/test_float.py
--- a/pypy/jit/metainterp/test/test_float.py
+++ b/pypy/jit/metainterp/test/test_float.py
@@ -1,5 +1,6 @@
-import math
+import math, sys
 from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin
+from pypy.rlib.rarithmetic import intmask, r_uint
 
 
 class FloatTests:
@@ -45,6 +46,34 @@
         res = self.interp_operations(f, [-2.0])
         assert res == -8.5
 
+    def test_cast_float_to_int(self):
+        def g(f):
+            return int(f)
+        res = self.interp_operations(g, [-12345.9])
+        assert res == -12345
+
+    def test_cast_float_to_uint(self):
+        def g(f):
+            return intmask(r_uint(f))
+        res = self.interp_operations(g, [sys.maxint*2.0])
+        assert res == intmask(long(sys.maxint*2.0))
+        res = self.interp_operations(g, [-12345.9])
+        assert res == -12345
+
+    def test_cast_int_to_float(self):
+        def g(i):
+            return float(i)
+        res = self.interp_operations(g, [-12345])
+        assert type(res) is float and res == -12345.0
+
+    def test_cast_uint_to_float(self):
+        def g(i):
+            return float(r_uint(i))
+        res = self.interp_operations(g, [intmask(sys.maxint*2)])
+        assert type(res) is float and res == float(sys.maxint*2)
+        res = self.interp_operations(g, [-12345])
+        assert type(res) is float and res == float(long(r_uint(-12345)))
+
 
 class TestOOtype(FloatTests, OOJitMixin):
     pass
diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -19,7 +19,7 @@
 class W_RSocket(Wrappable, RSocket):
     def __del__(self):
         self.clear_all_weakrefs()
-        self.close()
+        RSocket.__del__(self)
 
     def accept_w(self, space):
         """accept() -> (socket object, address info)
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -211,7 +211,9 @@
             return result
 
         def __del__(self):
-            self.clear_all_weakrefs()
+            # note that we don't call clear_all_weakrefs here because
+            # an array with freed buffer is ok to see - it's just empty with 0
+            # length
             self.setlen(0)
 
         def setlen(self, size):
diff --git a/pypy/module/array/test/test_array.py b/pypy/module/array/test/test_array.py
--- a/pypy/module/array/test/test_array.py
+++ b/pypy/module/array/test/test_array.py
@@ -824,6 +824,22 @@
         r = weakref.ref(a)
         assert r() is a
 
+    def test_subclass_del(self):
+        import array, gc, weakref
+        l = []
+        
+        class A(array.array):
+            pass
+
+        a = A('d')
+        a.append(3.0)
+        r = weakref.ref(a, lambda a: l.append(a()))
+        del a
+        gc.collect()
+        assert l
+        assert l[0] is None or len(l[0]) == 0
+
+
 class TestCPythonsOwnArray(BaseArrayTests):
 
     def setup_class(cls):
@@ -844,11 +860,7 @@
         cls.w_tempfile = cls.space.wrap(
             str(py.test.ensuretemp('array').join('tmpfile')))
         cls.w_maxint = cls.space.wrap(sys.maxint)
-
-
-
-
-
+    
     def test_buffer_info(self):
         a = self.array('c', 'Hi!')
         bi = a.buffer_info()
diff --git a/pypy/module/pypyjit/policy.py b/pypy/module/pypyjit/policy.py
--- a/pypy/module/pypyjit/policy.py
+++ b/pypy/module/pypyjit/policy.py
@@ -16,7 +16,8 @@
         if modname in ['pypyjit', 'signal', 'micronumpy', 'math', 'exceptions',
                        'imp', 'sys', 'array', '_ffi', 'itertools', 'operator',
                        'posix', '_socket', '_sre', '_lsprof', '_weakref',
-                       '__pypy__', 'cStringIO', '_collections', 'struct']:
+                       '__pypy__', 'cStringIO', '_collections', 'struct',
+                       'mmap']:
             return True
         return False
 
diff --git a/pypy/module/rctime/interp_time.py b/pypy/module/rctime/interp_time.py
--- a/pypy/module/rctime/interp_time.py
+++ b/pypy/module/rctime/interp_time.py
@@ -245,6 +245,9 @@
 if sys.platform != 'win32':
     @unwrap_spec(secs=float)
     def sleep(space, secs):
+        if secs < 0:
+            raise OperationError(space.w_IOError,
+                                 space.wrap("Invalid argument: negative time in sleep"))
         pytime.sleep(secs)
 else:
     from pypy.rlib import rwin32
@@ -265,6 +268,9 @@
                                    OSError(EINTR, "sleep() interrupted"))
     @unwrap_spec(secs=float)
     def sleep(space, secs):
+        if secs < 0:
+            raise OperationError(space.w_IOError,
+                                 space.wrap("Invalid argument: negative time in sleep"))
         # as decreed by Guido, only the main thread can be
         # interrupted.
         main_thread = space.fromcache(State).main_thread
diff --git a/pypy/module/rctime/test/test_rctime.py b/pypy/module/rctime/test/test_rctime.py
--- a/pypy/module/rctime/test/test_rctime.py
+++ b/pypy/module/rctime/test/test_rctime.py
@@ -20,8 +20,9 @@
         import sys
         import os
         raises(TypeError, rctime.sleep, "foo")
-        rctime.sleep(1.2345)
-        
+        rctime.sleep(0.12345)
+        raises(IOError, rctime.sleep, -1.0)
+
     def test_clock(self):
         import time as rctime
         rctime.clock()
diff --git a/pypy/pytest.ini b/pypy/pytest.ini
--- a/pypy/pytest.ini
+++ b/pypy/pytest.ini
@@ -1,2 +1,2 @@
 [pytest]
-addopts = --assertmode=old
\ No newline at end of file
+addopts = --assertmode=old -rf
diff --git a/pypy/rlib/rgc.py b/pypy/rlib/rgc.py
--- a/pypy/rlib/rgc.py
+++ b/pypy/rlib/rgc.py
@@ -214,6 +214,10 @@
     func._gc_no_collect_ = True
     return func
 
+def is_light_finalizer(func):
+    func._is_light_finalizer_ = True
+    return func
+
 # ____________________________________________________________
 
 def get_rpy_roots():
diff --git a/pypy/rlib/rmmap.py b/pypy/rlib/rmmap.py
--- a/pypy/rlib/rmmap.py
+++ b/pypy/rlib/rmmap.py
@@ -292,6 +292,9 @@
         elif _POSIX:
             self.closed = True
             if self.fd != -1:
+                # XXX this is buggy - raising in an RPython del is not a good
+                #     idea, we should swallow the exception or ignore the
+                #     underlaying close error code
                 os.close(self.fd)
                 self.fd = -1
             if self.size > 0:
diff --git a/pypy/rlib/rsocket.py b/pypy/rlib/rsocket.py
--- a/pypy/rlib/rsocket.py
+++ b/pypy/rlib/rsocket.py
@@ -56,6 +56,7 @@
 
 
 _FAMILIES = {}
+
 class Address(object):
     """The base class for RPython-level objects representing addresses.
     Fields:  addr    - a _c.sockaddr_ptr (memory owned by the Address instance)
@@ -77,9 +78,8 @@
         self.addrlen = addrlen
 
     def __del__(self):
-        addr = self.addr_p
-        if addr:
-            lltype.free(addr, flavor='raw')
+        if self.addr_p:
+            lltype.free(self.addr_p, flavor='raw')
 
     def setdata(self, addr, addrlen):
         # initialize self.addr and self.addrlen.  'addr' can be a different
@@ -613,7 +613,10 @@
         self.timeout = defaults.timeout
         
     def __del__(self):
-        self.close()
+        fd = self.fd
+        if fd != _c.INVALID_SOCKET:
+            self.fd = _c.INVALID_SOCKET
+            _c.socketclose(fd)
 
     if hasattr(_c, 'fcntl'):
         def _setblocking(self, block):
diff --git a/pypy/rpython/llinterp.py b/pypy/rpython/llinterp.py
--- a/pypy/rpython/llinterp.py
+++ b/pypy/rpython/llinterp.py
@@ -1095,13 +1095,6 @@
             assert y >= 0
         return self.op_int_add_ovf(x, y)
 
-    def op_cast_float_to_int(self, f):
-        assert type(f) is float
-        try:
-            return ovfcheck(int(f))
-        except OverflowError:
-            self.make_llexception()
-
     def op_int_is_true(self, x):
         # special case
         if type(x) is CDefinedIntSymbolic:
diff --git a/pypy/rpython/lltypesystem/lloperation.py b/pypy/rpython/lltypesystem/lloperation.py
--- a/pypy/rpython/lltypesystem/lloperation.py
+++ b/pypy/rpython/lltypesystem/lloperation.py
@@ -343,8 +343,8 @@
     'cast_uint_to_float':   LLOp(canfold=True),
     'cast_longlong_to_float' :LLOp(canfold=True),
     'cast_ulonglong_to_float':LLOp(canfold=True),
-    'cast_float_to_int':    LLOp(canraise=(OverflowError,), tryfold=True),
-    'cast_float_to_uint':   LLOp(canfold=True),    # XXX need OverflowError?
+    'cast_float_to_int':    LLOp(canfold=True),
+    'cast_float_to_uint':   LLOp(canfold=True),
     'cast_float_to_longlong' :LLOp(canfold=True),
     'cast_float_to_ulonglong':LLOp(canfold=True),
     'truncate_longlong_to_int':LLOp(canfold=True),
diff --git a/pypy/rpython/lltypesystem/opimpl.py b/pypy/rpython/lltypesystem/opimpl.py
--- a/pypy/rpython/lltypesystem/opimpl.py
+++ b/pypy/rpython/lltypesystem/opimpl.py
@@ -355,6 +355,10 @@
     assert type(b) is bool
     return float(b)
 
+def op_cast_float_to_int(f):
+    assert type(f) is float
+    return intmask(int(f))
+
 def op_cast_float_to_uint(f):
     assert type(f) is float
     return r_uint(long(f))
diff --git a/pypy/rpython/memory/gc/base.py b/pypy/rpython/memory/gc/base.py
--- a/pypy/rpython/memory/gc/base.py
+++ b/pypy/rpython/memory/gc/base.py
@@ -1,4 +1,5 @@
-from pypy.rpython.lltypesystem import lltype, llmemory, llarena
+from pypy.rpython.lltypesystem import lltype, llmemory, llarena, rffi
+from pypy.rpython.lltypesystem.lloperation import llop
 from pypy.rlib.debug import ll_assert
 from pypy.rpython.memory.gcheader import GCHeaderBuilder
 from pypy.rpython.memory.support import DEFAULT_CHUNK_SIZE
@@ -62,6 +63,7 @@
     def set_query_functions(self, is_varsize, has_gcptr_in_varsize,
                             is_gcarrayofgcptr,
                             getfinalizer,
+                            getlightfinalizer,
                             offsets_to_gc_pointers,
                             fixed_size, varsize_item_sizes,
                             varsize_offset_to_variable_part,
@@ -74,6 +76,7 @@
                             get_custom_trace,
                             fast_path_tracing):
         self.getfinalizer = getfinalizer
+        self.getlightfinalizer = getlightfinalizer
         self.is_varsize = is_varsize
         self.has_gcptr_in_varsize = has_gcptr_in_varsize
         self.is_gcarrayofgcptr = is_gcarrayofgcptr
@@ -139,6 +142,7 @@
 
         size = self.fixed_size(typeid)
         needs_finalizer = bool(self.getfinalizer(typeid))
+        finalizer_is_light = bool(self.getlightfinalizer(typeid))
         contains_weakptr = self.weakpointer_offset(typeid) >= 0
         assert not (needs_finalizer and contains_weakptr)
         if self.is_varsize(typeid):
@@ -158,6 +162,7 @@
             else:
                 malloc_fixedsize = self.malloc_fixedsize
             ref = malloc_fixedsize(typeid, size, needs_finalizer,
+                                   finalizer_is_light,
                                    contains_weakptr)
         # lots of cast and reverse-cast around...
         return llmemory.cast_ptr_to_adr(ref)
diff --git a/pypy/rpython/memory/gc/generation.py b/pypy/rpython/memory/gc/generation.py
--- a/pypy/rpython/memory/gc/generation.py
+++ b/pypy/rpython/memory/gc/generation.py
@@ -167,7 +167,9 @@
         return self.nursery <= addr < self.nursery_top
 
     def malloc_fixedsize_clear(self, typeid, size,
-                               has_finalizer=False, contains_weakptr=False):
+                               has_finalizer=False,
+                               is_finalizer_light=False,
+                               contains_weakptr=False):
         if (has_finalizer or
             (raw_malloc_usage(size) > self.lb_young_fixedsize and
              raw_malloc_usage(size) > self.largest_young_fixedsize)):
@@ -179,6 +181,7 @@
             # "non-simple" case or object too big: don't use the nursery
             return SemiSpaceGC.malloc_fixedsize_clear(self, typeid, size,
                                                       has_finalizer,
+                                                      is_finalizer_light,
                                                       contains_weakptr)
         size_gc_header = self.gcheaderbuilder.size_gc_header
         totalsize = size_gc_header + size
diff --git a/pypy/rpython/memory/gc/marksweep.py b/pypy/rpython/memory/gc/marksweep.py
--- a/pypy/rpython/memory/gc/marksweep.py
+++ b/pypy/rpython/memory/gc/marksweep.py
@@ -93,7 +93,8 @@
         pass
 
     def malloc_fixedsize(self, typeid16, size,
-                         has_finalizer=False, contains_weakptr=False):
+                         has_finalizer=False, is_finalizer_light=False,
+                         contains_weakptr=False):
         self.maybe_collect()
         size_gc_header = self.gcheaderbuilder.size_gc_header
         try:
@@ -128,7 +129,9 @@
     malloc_fixedsize._dont_inline_ = True
 
     def malloc_fixedsize_clear(self, typeid16, size,
-                               has_finalizer=False, contains_weakptr=False):
+                               has_finalizer=False,
+                               is_finalizer_light=False,
+                               contains_weakptr=False):
         self.maybe_collect()
         size_gc_header = self.gcheaderbuilder.size_gc_header
         try:
diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -290,6 +290,8 @@
         #
         # A list of all objects with finalizers (these are never young).
         self.objects_with_finalizers = self.AddressDeque()
+        self.young_objects_with_light_finalizers = self.AddressStack()
+        self.old_objects_with_light_finalizers = self.AddressStack()
         #
         # Two lists of the objects with weakrefs.  No weakref can be an
         # old object weakly pointing to a young object: indeed, weakrefs
@@ -457,14 +459,16 @@
 
 
     def malloc_fixedsize_clear(self, typeid, size,
-                               needs_finalizer=False, contains_weakptr=False):
+                               needs_finalizer=False,
+                               is_finalizer_light=False,
+                               contains_weakptr=False):
         size_gc_header = self.gcheaderbuilder.size_gc_header
         totalsize = size_gc_header + size
         rawtotalsize = raw_malloc_usage(totalsize)
         #
         # If the object needs a finalizer, ask for a rawmalloc.
         # The following check should be constant-folded.
-        if needs_finalizer:
+        if needs_finalizer and not is_finalizer_light:
             ll_assert(not contains_weakptr,
                      "'needs_finalizer' and 'contains_weakptr' both specified")
             obj = self.external_malloc(typeid, 0, can_make_young=False)
@@ -494,13 +498,14 @@
             #
             # Build the object.
             llarena.arena_reserve(result, totalsize)
+            obj = result + size_gc_header
+            if is_finalizer_light:
+                self.young_objects_with_light_finalizers.append(obj)
             self.init_gc_object(result, typeid, flags=0)
             #
             # If it is a weakref, record it (check constant-folded).
             if contains_weakptr:
-                self.young_objects_with_weakrefs.append(result+size_gc_header)
-            #
-            obj = result + size_gc_header
+                self.young_objects_with_weakrefs.append(obj)
         #
         return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
 
@@ -1264,6 +1269,8 @@
         # weakrefs' targets.
         if self.young_objects_with_weakrefs.non_empty():
             self.invalidate_young_weakrefs()
+        if self.young_objects_with_light_finalizers.non_empty():
+            self.deal_with_young_objects_with_finalizers()
         #
         # Clear this mapping.
         if self.nursery_objects_shadows.length() > 0:
@@ -1584,6 +1591,9 @@
         # Weakref support: clear the weak pointers to dying objects
         if self.old_objects_with_weakrefs.non_empty():
             self.invalidate_old_weakrefs()
+        if self.old_objects_with_light_finalizers.non_empty():
+            self.deal_with_old_objects_with_finalizers()
+
         #
         # Walk all rawmalloced objects and free the ones that don't
         # have the GCFLAG_VISITED flag.
@@ -1649,8 +1659,7 @@
         if self.header(obj).tid & GCFLAG_VISITED:
             self.header(obj).tid &= ~GCFLAG_VISITED
             return False     # survives
-        else:
-            return True      # dies
+        return True      # dies
 
     def _reset_gcflag_visited(self, obj, ignored):
         self.header(obj).tid &= ~GCFLAG_VISITED
@@ -1829,6 +1838,39 @@
     # ----------
     # Finalizers
 
+    def deal_with_young_objects_with_finalizers(self):
+        """ This is a much simpler version of dealing with finalizers
+        and an optimization - we can reasonably assume that those finalizers
+        don't do anything fancy and *just* call them. Among other things
+        they won't resurrect objects
+        """
+        while self.young_objects_with_light_finalizers.non_empty():
+            obj = self.young_objects_with_light_finalizers.pop()
+            if not self.is_forwarded(obj):
+                finalizer = self.getlightfinalizer(self.get_type_id(obj))
+                ll_assert(bool(finalizer), "no light finalizer found")
+                finalizer(obj, llmemory.NULL)
+
+    def deal_with_old_objects_with_finalizers(self):
+        """ This is a much simpler version of dealing with finalizers
+        and an optimization - we can reasonably assume that those finalizers
+        don't do anything fancy and *just* call them. Among other things
+        they won't resurrect objects
+        """
+        new_objects = self.AddressStack()
+        while self.old_objects_with_light_finalizers.non_empty():
+            obj = self.old_objects_with_light_finalizers.pop()
+            if self.header(obj).tid & GCFLAG_VISITED:
+                # surviving
+                new_objects.append(obj)
+            else:
+                # dying
+                finalizer = self.getlightfinalizer(self.get_type_id(obj))
+                ll_assert(bool(finalizer), "no light finalizer found")
+                finalizer(obj, llmemory.NULL)
+        self.old_objects_with_light_finalizers.delete()
+        self.old_objects_with_light_finalizers = new_objects
+
     def deal_with_objects_with_finalizers(self):
         # Walk over list of objects with finalizers.
         # If it is not surviving, add it to the list of to-be-called
@@ -1959,7 +2001,6 @@
             #
             self.old_objects_with_weakrefs.append(obj)
 
-
     def invalidate_old_weakrefs(self):
         """Called during a major collection."""
         # walk over list of objects that contain weakrefs
diff --git a/pypy/rpython/memory/gc/semispace.py b/pypy/rpython/memory/gc/semispace.py
--- a/pypy/rpython/memory/gc/semispace.py
+++ b/pypy/rpython/memory/gc/semispace.py
@@ -82,6 +82,7 @@
         self.free = self.tospace
         MovingGCBase.setup(self)
         self.objects_with_finalizers = self.AddressDeque()
+        self.objects_with_light_finalizers = self.AddressStack()
         self.objects_with_weakrefs = self.AddressStack()
 
     def _teardown(self):
@@ -93,7 +94,9 @@
     # because the spaces are filled with zeroes in advance.
 
     def malloc_fixedsize_clear(self, typeid16, size,
-                               has_finalizer=False, contains_weakptr=False):
+                               has_finalizer=False,
+                               is_finalizer_light=False,
+                               contains_weakptr=False):
         size_gc_header = self.gcheaderbuilder.size_gc_header
         totalsize = size_gc_header + size
         result = self.free
@@ -102,7 +105,9 @@
         llarena.arena_reserve(result, totalsize)
         self.init_gc_object(result, typeid16)
         self.free = result + totalsize
-        if has_finalizer:
+        if is_finalizer_light:
+            self.objects_with_light_finalizers.append(result + size_gc_header)
+        elif has_finalizer:
             self.objects_with_finalizers.append(result + size_gc_header)
         if contains_weakptr:
             self.objects_with_weakrefs.append(result + size_gc_header)
@@ -263,6 +268,8 @@
         if self.run_finalizers.non_empty():
             self.update_run_finalizers()
         scan = self.scan_copied(scan)
+        if self.objects_with_light_finalizers.non_empty():
+            self.deal_with_objects_with_light_finalizers()
         if self.objects_with_finalizers.non_empty():
             scan = self.deal_with_objects_with_finalizers(scan)
         if self.objects_with_weakrefs.non_empty():
@@ -471,6 +478,23 @@
         # immortal objects always have GCFLAG_FORWARDED set;
         # see get_forwarding_address().
 
+    def deal_with_objects_with_light_finalizers(self):
+        """ This is a much simpler version of dealing with finalizers
+        and an optimization - we can reasonably assume that those finalizers
+        don't do anything fancy and *just* call them. Among other things
+        they won't resurrect objects
+        """
+        new_objects = self.AddressStack()
+        while self.objects_with_light_finalizers.non_empty():
+            obj = self.objects_with_light_finalizers.pop()
+            if self.surviving(obj):
+                new_objects.append(self.get_forwarding_address(obj))
+            else:
+                finalizer = self.getfinalizer(self.get_type_id(obj))
+                finalizer(obj, llmemory.NULL)
+        self.objects_with_light_finalizers.delete()
+        self.objects_with_light_finalizers = new_objects
+
     def deal_with_objects_with_finalizers(self, scan):
         # walk over list of objects with finalizers
         # if it is not copied, add it to the list of to-be-called finalizers
diff --git a/pypy/rpython/memory/gctransform/framework.py b/pypy/rpython/memory/gctransform/framework.py
--- a/pypy/rpython/memory/gctransform/framework.py
+++ b/pypy/rpython/memory/gctransform/framework.py
@@ -12,6 +12,7 @@
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.translator.backendopt import graphanalyze
 from pypy.translator.backendopt.support import var_needsgc
+from pypy.translator.backendopt.finalizer import FinalizerAnalyzer
 from pypy.annotation import model as annmodel
 from pypy.rpython import annlowlevel
 from pypy.rpython.rbuiltin import gen_cast
@@ -258,6 +259,7 @@
             [s_gc, s_typeid16,
              annmodel.SomeInteger(nonneg=True),
              annmodel.SomeBool(),
+             annmodel.SomeBool(),
              annmodel.SomeBool()], s_gcref,
             inline = False)
         if hasattr(GCClass, 'malloc_fixedsize'):
@@ -267,6 +269,7 @@
                 [s_gc, s_typeid16,
                  annmodel.SomeInteger(nonneg=True),
                  annmodel.SomeBool(),
+                 annmodel.SomeBool(),
                  annmodel.SomeBool()], s_gcref,
                 inline = False)
         else:
@@ -319,7 +322,7 @@
             raise NotImplementedError("GC needs write barrier, but does not provide writebarrier_before_copy functionality")
 
         # in some GCs we can inline the common case of
-        # malloc_fixedsize(typeid, size, True, False, False)
+        # malloc_fixedsize(typeid, size, False, False, False)
         if getattr(GCClass, 'inline_simple_malloc', False):
             # make a copy of this function so that it gets annotated
             # independently and the constants are folded inside
@@ -337,7 +340,7 @@
                 malloc_fast,
                 [s_gc, s_typeid16,
                  annmodel.SomeInteger(nonneg=True),
-                 s_False, s_False], s_gcref,
+                 s_False, s_False, s_False], s_gcref,
                 inline = True)
         else:
             self.malloc_fast_ptr = None
@@ -668,7 +671,13 @@
         kind_and_fptr = self.special_funcptr_for_type(TYPE)
         has_finalizer = (kind_and_fptr is not None and
                          kind_and_fptr[0] == "finalizer")
+        has_light_finalizer = (kind_and_fptr is not None and
+                               kind_and_fptr[0] == "light_finalizer")
+        if has_light_finalizer:
+            has_finalizer = True
         c_has_finalizer = rmodel.inputconst(lltype.Bool, has_finalizer)
+        c_has_light_finalizer = rmodel.inputconst(lltype.Bool,
+                                                  has_light_finalizer)
 
         if not op.opname.endswith('_varsize') and not flags.get('varsize'):
             #malloc_ptr = self.malloc_fixedsize_ptr
@@ -682,7 +691,8 @@
             else:
                 malloc_ptr = self.malloc_fixedsize_ptr
             args = [self.c_const_gc, c_type_id, c_size,
-                    c_has_finalizer, rmodel.inputconst(lltype.Bool, False)]
+                    c_has_finalizer, c_has_light_finalizer,
+                    rmodel.inputconst(lltype.Bool, False)]
         else:
             assert not c_has_finalizer.value
             info_varsize = self.layoutbuilder.get_info_varsize(type_id)
@@ -847,12 +857,13 @@
         # used by the JIT (see pypy.jit.backend.llsupport.gc)
         op = hop.spaceop
         [v_typeid, v_size,
-         v_has_finalizer, v_contains_weakptr] = op.args
+         v_has_finalizer, v_has_light_finalizer, v_contains_weakptr] = op.args
         livevars = self.push_roots(hop)
         hop.genop("direct_call",
                   [self.malloc_fixedsize_clear_ptr, self.c_const_gc,
                    v_typeid, v_size,
-                   v_has_finalizer, v_contains_weakptr],
+                   v_has_finalizer, v_has_light_finalizer,
+                   v_contains_weakptr],
                   resultvar=op.result)
         self.pop_roots(hop, livevars)
 
@@ -912,10 +923,10 @@
         info = self.layoutbuilder.get_info(type_id)
         c_size = rmodel.inputconst(lltype.Signed, info.fixedsize)
         malloc_ptr = self.malloc_fixedsize_ptr
-        c_has_finalizer = rmodel.inputconst(lltype.Bool, False)
+        c_false = rmodel.inputconst(lltype.Bool, False)
         c_has_weakptr = rmodel.inputconst(lltype.Bool, True)
         args = [self.c_const_gc, c_type_id, c_size,
-                c_has_finalizer, c_has_weakptr]
+                c_false, c_false, c_has_weakptr]
 
         # push and pop the current live variables *including* the argument
         # to the weakref_create operation, which must be kept alive and
@@ -1250,6 +1261,7 @@
             lltype2vtable = translator.rtyper.lltype2vtable
         else:
             lltype2vtable = None
+        self.translator = translator
         super(TransformerLayoutBuilder, self).__init__(GCClass, lltype2vtable)
 
     def has_finalizer(self, TYPE):
@@ -1257,6 +1269,10 @@
         return rtti is not None and getattr(rtti._obj, 'destructor_funcptr',
                                             None)
 
+    def has_light_finalizer(self, TYPE):
+        special = self.special_funcptr_for_type(TYPE)
+        return special is not None and special[0] == 'light_finalizer'
+
     def has_custom_trace(self, TYPE):
         rtti = get_rtti(TYPE)
         return rtti is not None and getattr(rtti._obj, 'custom_trace_funcptr',
@@ -1264,7 +1280,7 @@
 
     def make_finalizer_funcptr_for_type(self, TYPE):
         if not self.has_finalizer(TYPE):
-            return None
+            return None, False
         rtti = get_rtti(TYPE)
         destrptr = rtti._obj.destructor_funcptr
         DESTR_ARG = lltype.typeOf(destrptr).TO.ARGS[0]
@@ -1276,7 +1292,9 @@
             return llmemory.NULL
         fptr = self.transformer.annotate_finalizer(ll_finalizer,
                 [llmemory.Address, llmemory.Address], llmemory.Address)
-        return fptr
+        g = destrptr._obj.graph
+        light = not FinalizerAnalyzer(self.translator).analyze_light_finalizer(g)
+        return fptr, light
 
     def make_custom_trace_funcptr_for_type(self, TYPE):
         if not self.has_custom_trace(TYPE):
diff --git a/pypy/rpython/memory/gctypelayout.py b/pypy/rpython/memory/gctypelayout.py
--- a/pypy/rpython/memory/gctypelayout.py
+++ b/pypy/rpython/memory/gctypelayout.py
@@ -1,7 +1,6 @@
 from pypy.rpython.lltypesystem import lltype, llmemory, llarena, llgroup
 from pypy.rpython.lltypesystem import rclass
 from pypy.rpython.lltypesystem.lloperation import llop
-from pypy.rlib.objectmodel import we_are_translated
 from pypy.rlib.debug import ll_assert
 from pypy.rlib.rarithmetic import intmask
 from pypy.tool.identity_dict import identity_dict
@@ -85,6 +84,13 @@
         else:
             return lltype.nullptr(GCData.FINALIZER_OR_CT_FUNC)
 
+    def q_light_finalizer(self, typeid):
+        typeinfo = self.get(typeid)
+        if typeinfo.infobits & T_HAS_LIGHTWEIGHT_FINALIZER:
+            return typeinfo.finalizer_or_customtrace
+        else:
+            return lltype.nullptr(GCData.FINALIZER_OR_CT_FUNC)        
+
     def q_offsets_to_gc_pointers(self, typeid):
         return self.get(typeid).ofstoptrs
 
@@ -142,6 +148,7 @@
             self.q_has_gcptr_in_varsize,
             self.q_is_gcarrayofgcptr,
             self.q_finalizer,
+            self.q_light_finalizer,
             self.q_offsets_to_gc_pointers,
             self.q_fixed_size,
             self.q_varsize_item_sizes,
@@ -157,16 +164,17 @@
 
 
 # the lowest 16bits are used to store group member index
-T_MEMBER_INDEX         =   0xffff
-T_IS_VARSIZE           = 0x010000
-T_HAS_GCPTR_IN_VARSIZE = 0x020000
-T_IS_GCARRAY_OF_GCPTR  = 0x040000
-T_IS_WEAKREF           = 0x080000
-T_IS_RPYTHON_INSTANCE  = 0x100000    # the type is a subclass of OBJECT
-T_HAS_FINALIZER        = 0x200000
-T_HAS_CUSTOM_TRACE     = 0x400000
-T_KEY_MASK             = intmask(0xFF000000)
-T_KEY_VALUE            = intmask(0x5A000000)    # bug detection only
+T_MEMBER_INDEX              =   0xffff
+T_IS_VARSIZE                = 0x010000
+T_HAS_GCPTR_IN_VARSIZE      = 0x020000
+T_IS_GCARRAY_OF_GCPTR       = 0x040000
+T_IS_WEAKREF                = 0x080000
+T_IS_RPYTHON_INSTANCE       = 0x100000 # the type is a subclass of OBJECT
+T_HAS_FINALIZER             = 0x200000
+T_HAS_CUSTOM_TRACE          = 0x400000
+T_HAS_LIGHTWEIGHT_FINALIZER = 0x800000
+T_KEY_MASK                  = intmask(0xFF000000)
+T_KEY_VALUE                 = intmask(0x5A000000) # bug detection only
 
 def _check_valid_type_info(p):
     ll_assert(p.infobits & T_KEY_MASK == T_KEY_VALUE, "invalid type_id")
@@ -194,6 +202,8 @@
         info.finalizer_or_customtrace = fptr
         if kind == "finalizer":
             infobits |= T_HAS_FINALIZER
+        elif kind == 'light_finalizer':
+            infobits |= T_HAS_FINALIZER | T_HAS_LIGHTWEIGHT_FINALIZER
         elif kind == "custom_trace":
             infobits |= T_HAS_CUSTOM_TRACE
         else:
@@ -367,12 +377,15 @@
     def special_funcptr_for_type(self, TYPE):
         if TYPE in self._special_funcptrs:
             return self._special_funcptrs[TYPE]
-        fptr1 = self.make_finalizer_funcptr_for_type(TYPE)
+        fptr1, is_lightweight = self.make_finalizer_funcptr_for_type(TYPE)
         fptr2 = self.make_custom_trace_funcptr_for_type(TYPE)
         assert not (fptr1 and fptr2), (
             "type %r needs both a finalizer and a custom tracer" % (TYPE,))
         if fptr1:
-            kind_and_fptr = "finalizer", fptr1
+            if is_lightweight:
+                kind_and_fptr = "light_finalizer", fptr1
+            else:
+                kind_and_fptr = "finalizer", fptr1
         elif fptr2:
             kind_and_fptr = "custom_trace", fptr2
         else:
@@ -382,7 +395,7 @@
 
     def make_finalizer_funcptr_for_type(self, TYPE):
         # must be overridden for proper finalizer support
-        return None
+        return None, False
 
     def make_custom_trace_funcptr_for_type(self, TYPE):
         # must be overridden for proper custom tracer support
diff --git a/pypy/rpython/memory/gcwrapper.py b/pypy/rpython/memory/gcwrapper.py
--- a/pypy/rpython/memory/gcwrapper.py
+++ b/pypy/rpython/memory/gcwrapper.py
@@ -1,3 +1,4 @@
+from pypy.translator.backendopt.finalizer import FinalizerAnalyzer
 from pypy.rpython.lltypesystem import lltype, llmemory, llheap
 from pypy.rpython import llinterp
 from pypy.rpython.annlowlevel import llhelper
@@ -196,9 +197,11 @@
             DESTR_ARG = lltype.typeOf(destrptr).TO.ARGS[0]
             destrgraph = destrptr._obj.graph
         else:
-            return None
+            return None, False
 
         assert not type_contains_pyobjs(TYPE), "not implemented"
+        t = self.llinterp.typer.annotator.translator
+        light = not FinalizerAnalyzer(t).analyze_light_finalizer(destrgraph)
         def ll_finalizer(addr, dummy):
             assert dummy == llmemory.NULL
             try:
@@ -208,7 +211,7 @@
                 raise RuntimeError(
                     "a finalizer raised an exception, shouldn't happen")
             return llmemory.NULL
-        return llhelper(gctypelayout.GCData.FINALIZER_OR_CT, ll_finalizer)
+        return llhelper(gctypelayout.GCData.FINALIZER_OR_CT, ll_finalizer), light
 
     def make_custom_trace_funcptr_for_type(self, TYPE):
         from pypy.rpython.memory.gctransform.support import get_rtti, \
diff --git a/pypy/rpython/memory/test/test_gc.py b/pypy/rpython/memory/test/test_gc.py
--- a/pypy/rpython/memory/test/test_gc.py
+++ b/pypy/rpython/memory/test/test_gc.py
@@ -5,7 +5,6 @@
 from pypy.rpython.memory.test import snippet
 from pypy.rpython.test.test_llinterp import get_interpreter
 from pypy.rpython.lltypesystem import lltype
-from pypy.rpython.lltypesystem.rstr import STR
 from pypy.rpython.lltypesystem.lloperation import llop
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.rlib.objectmodel import compute_unique_id
@@ -57,7 +56,7 @@
                 while j < 20:
                     j += 1
                     a.append(j)
-        res = self.interpret(malloc_a_lot, [])
+        self.interpret(malloc_a_lot, [])
         #assert simulator.current_size - curr < 16000 * INT_SIZE / 4
         #print "size before: %s, size after %s" % (curr, simulator.current_size)
 
@@ -73,7 +72,7 @@
                 while j < 20:
                     j += 1
                     b.append((1, j, i))
-        res = self.interpret(malloc_a_lot, [])
+        self.interpret(malloc_a_lot, [])
         #assert simulator.current_size - curr < 16000 * INT_SIZE / 4
         #print "size before: %s, size after %s" % (curr, simulator.current_size)
 
@@ -129,7 +128,7 @@
         res = self.interpret(concat, [100])
         assert res == concat(100)
         #assert simulator.current_size - curr < 16000 * INT_SIZE / 4
-        
+
     def test_finalizer(self):
         class B(object):
             pass
@@ -278,7 +277,7 @@
                                self.interpret, f, [])
 
     def test_weakref(self):
-        import weakref, gc
+        import weakref
         class A(object):
             pass
         def g():
@@ -299,7 +298,7 @@
         assert res
 
     def test_weakref_to_object_with_finalizer(self):
-        import weakref, gc
+        import weakref
         class A(object):
             count = 0
         a = A()
@@ -338,7 +337,7 @@
         assert res
 
     def test_cycle_with_weakref_and_del(self):
-        import weakref, gc
+        import weakref
         class A(object):
             count = 0
         a = A()
@@ -367,7 +366,7 @@
         assert res == 11
 
     def test_weakref_to_object_with_finalizer_ordering(self):
-        import weakref, gc
+        import weakref
         class A(object):
             count = 0
         a = A()
@@ -616,7 +615,7 @@
                     assert not rgc.can_move(a)
                     return 1
                 return 0
-            except Exception, e:
+            except Exception:
                 return 2
 
         assert self.interpret(func, []) == int(self.GC_CAN_MALLOC_NONMOVABLE)
@@ -647,8 +646,6 @@
         assert self.interpret(f, [bigsize, 0, flag]) == 0x62024241
 
     def test_tagged_simple(self):
-        from pypy.rlib.objectmodel import UnboxedValue
-
         class Unrelated(object):
             pass
 
@@ -689,8 +686,6 @@
         assert res == -897
 
     def test_tagged_id(self):
-        from pypy.rlib.objectmodel import UnboxedValue, compute_unique_id
-
         class Unrelated(object):
             pass
 
diff --git a/pypy/rpython/memory/test/test_transformed_gc.py b/pypy/rpython/memory/test/test_transformed_gc.py
--- a/pypy/rpython/memory/test/test_transformed_gc.py
+++ b/pypy/rpython/memory/test/test_transformed_gc.py
@@ -345,22 +345,22 @@
         b = B()
         b.nextid = 0
         b.num_deleted = 0
-        class A(object):
+        class AAA(object):
             def __init__(self):
                 self.id = b.nextid
                 b.nextid += 1
             def __del__(self):
                 b.num_deleted += 1
                 C()
-        class C(A):
+        class C(AAA):
             def __del__(self):
                 b.num_deleted += 1
         def f(x, y):
-            a = A()
+            a = AAA()
             i = 0
             while i < x:
                 i += 1
-                a = A()
+                a = AAA()
             llop.gc__collect(lltype.Void)
             llop.gc__collect(lltype.Void)
             return b.num_deleted
@@ -807,6 +807,7 @@
                     op.args = [Constant(type_id, llgroup.HALFWORD),
                                Constant(llmemory.sizeof(P), lltype.Signed),
                                Constant(False, lltype.Bool), # has_finalizer
+                               Constant(False, lltype.Bool), # is_finalizer_light
                                Constant(False, lltype.Bool)] # contains_weakptr
                     break
             else:
@@ -843,6 +844,7 @@
                     op.args = [Constant(type_id, llgroup.HALFWORD),
                                Constant(llmemory.sizeof(P), lltype.Signed),
                                Constant(False, lltype.Bool), # has_finalizer
+                               Constant(False, lltype.Bool), # is_finalizer_light
                                Constant(False, lltype.Bool)] # contains_weakptr
                     break
             else:
diff --git a/pypy/rpython/module/ll_os.py b/pypy/rpython/module/ll_os.py
--- a/pypy/rpython/module/ll_os.py
+++ b/pypy/rpython/module/ll_os.py
@@ -959,8 +959,6 @@
                             os_ftruncate(rffi.cast(rffi.INT, fd),
                                          rffi.cast(rffi.LONGLONG, length)))
             if res < 0:
-                # Note: for consistency we raise OSError, but CPython
-                # raises IOError here
                 raise OSError(rposix.get_errno(), "os_ftruncate failed")
 
         return extdef([int, r_longlong], s_None,
diff --git a/pypy/rpython/rtyper.py b/pypy/rpython/rtyper.py
--- a/pypy/rpython/rtyper.py
+++ b/pypy/rpython/rtyper.py
@@ -717,7 +717,7 @@
             raise TyperError("runtime type info function %r returns %r, "
                              "excepted Ptr(RuntimeTypeInfo)" % (func, s))
         funcptr = self.getcallable(graph)
-        attachRuntimeTypeInfo(GCSTRUCT, funcptr, destrptr)
+        attachRuntimeTypeInfo(GCSTRUCT, funcptr, destrptr, None)
 
 # register operations from annotation model
 RPythonTyper._registeroperations(annmodel)
diff --git a/pypy/rpython/test/test_rclass.py b/pypy/rpython/test/test_rclass.py
--- a/pypy/rpython/test/test_rclass.py
+++ b/pypy/rpython/test/test_rclass.py
@@ -3,7 +3,7 @@
 from pypy.translator.translator import TranslationContext, graphof
 from pypy.rpython.lltypesystem.lltype import *
 from pypy.rpython.ootypesystem import ootype
-from pypy.rlib.rarithmetic import intmask, r_longlong
+from pypy.rlib.rarithmetic import r_longlong
 from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin
 from pypy.rpython.rclass import IR_IMMUTABLE, IR_IMMUTABLE_ARRAY
 from pypy.rpython.rclass import IR_QUASIIMMUTABLE, IR_QUASIIMMUTABLE_ARRAY
@@ -972,10 +972,10 @@
         graph = graphof(t, f)
         TYPE = graph.startblock.operations[0].args[0].value
         RTTI = getRuntimeTypeInfo(TYPE)
-        queryptr = RTTI._obj.query_funcptr # should not raise
+        RTTI._obj.query_funcptr # should not raise
         destrptr = RTTI._obj.destructor_funcptr
         assert destrptr is not None
-    
+
     def test_del_inheritance(self):
         from pypy.rlib import rgc
         class State:
diff --git a/pypy/translator/backendopt/finalizer.py b/pypy/translator/backendopt/finalizer.py
new file mode 100644
--- /dev/null
+++ b/pypy/translator/backendopt/finalizer.py
@@ -0,0 +1,46 @@
+
+from pypy.translator.backendopt import graphanalyze
+from pypy.rpython.lltypesystem import lltype
+
+class FinalizerError(Exception):
+    """ __del__ marked as lightweight finalizer, but the analyzer did
+    not agreed
+    """
+
+class FinalizerAnalyzer(graphanalyze.BoolGraphAnalyzer):
+    """ Analyzer that determines whether a finalizer is lightweight enough
+    so it can be called without all the complicated logic in the garbage
+    collector. The set of operations here is restrictive for a good reason
+    - it's better to be safe. Specifically disallowed operations:
+
+    * anything that escapes self
+    * anything that can allocate
+    """
+    ok_operations = ['ptr_nonzero', 'ptr_eq', 'ptr_ne', 'free', 'same_as',
+                     'direct_ptradd', 'force_cast', 'track_alloc_stop',
+                     'raw_free']
+    
+    def analyze_light_finalizer(self, graph):
+        result = self.analyze_direct_call(graph)
+        if (result is self.top_result() and
+            getattr(graph.func, '_is_light_finalizer_', False)):
+            raise FinalizerError(FinalizerError.__doc__, graph)
+        return result
+    
+    def analyze_simple_operation(self, op, graphinfo):
+        if op.opname in self.ok_operations:
+            return self.bottom_result()
+        if (op.opname.startswith('int_') or op.opname.startswith('float_')
+            or op.opname.startswith('cast_')):
+            return self.bottom_result()
+        if op.opname == 'setfield' or op.opname == 'bare_setfield':
+            TP = op.args[2].concretetype
+            if not isinstance(TP, lltype.Ptr) or TP.TO._gckind == 'raw':
+                # primitive type
+                return self.bottom_result()
+        if op.opname == 'getfield':
+            TP = op.result.concretetype
+            if not isinstance(TP, lltype.Ptr) or TP.TO._gckind == 'raw':
+                # primitive type
+                return self.bottom_result()            
+        return self.top_result()
diff --git a/pypy/translator/backendopt/test/test_finalizer.py b/pypy/translator/backendopt/test/test_finalizer.py
new file mode 100644
--- /dev/null
+++ b/pypy/translator/backendopt/test/test_finalizer.py
@@ -0,0 +1,142 @@
+
+import py
+from pypy.translator.backendopt.finalizer import FinalizerAnalyzer,\
+     FinalizerError
+from pypy.translator.translator import TranslationContext, graphof
+from pypy.translator.backendopt.all import backend_optimizations
+from pypy.translator.unsimplify import varoftype
+from pypy.rpython.lltypesystem import lltype, rffi
+from pypy.conftest import option
+from pypy.rlib import rgc
+
+
+class BaseFinalizerAnalyzerTests(object):
+    """ Below are typical destructors that we encounter in pypy
+    """
+
+    type_system = None
+    
+    def analyze(self, func, sig, func_to_analyze=None, backendopt=False):
+        if func_to_analyze is None:
+            func_to_analyze = func
+        t = TranslationContext()
+        t.buildannotator().build_types(func, sig)
+        t.buildrtyper(type_system=self.type_system).specialize()
+        if backendopt:
+            backend_optimizations(t)
+        if option.view:
+            t.view()
+        a = FinalizerAnalyzer(t)
+        fgraph = graphof(t, func_to_analyze)
+        result = a.analyze_light_finalizer(fgraph)
+        return result
+
+    def test_nothing(self):
+        def f():
+            pass
+        r = self.analyze(f, [])
+        assert not r
+
+def test_various_ops():
+    from pypy.objspace.flow.model import SpaceOperation, Constant
+
+    X = lltype.Ptr(lltype.GcStruct('X'))
+    Z = lltype.Ptr(lltype.Struct('Z'))
+    S = lltype.GcStruct('S', ('x', lltype.Signed),
+                        ('y', X),
+                        ('z', Z))
+    v1 = varoftype(lltype.Bool)
+    v2 = varoftype(lltype.Signed)
+    f = FinalizerAnalyzer(None)
+    r = f.analyze(SpaceOperation('cast_int_to_bool', [v2],
+                                                       v1))
+    assert not r
+    v1 = varoftype(lltype.Ptr(S))
+    v2 = varoftype(lltype.Signed)
+    v3 = varoftype(X)
+    v4 = varoftype(Z)
+    assert not f.analyze(SpaceOperation('bare_setfield', [v1, Constant('x'),
+                                                          v2], None))
+    assert     f.analyze(SpaceOperation('bare_setfield', [v1, Constant('y'),
+                                                          v3], None))
+    assert not f.analyze(SpaceOperation('bare_setfield', [v1, Constant('z'),
+                                                          v4], None))
+    
+        
+class TestLLType(BaseFinalizerAnalyzerTests):
+    type_system = 'lltype'
+
+    def test_malloc(self):
+        S = lltype.GcStruct('S')
+        
+        def f():
+            return lltype.malloc(S)
+
+        r = self.analyze(f, [])
+        assert r
+
+    def test_raw_free_getfield(self):
+        S = lltype.Struct('S')
+        
+        class A(object):
+            def __init__(self):
+                self.x = lltype.malloc(S, flavor='raw')
+
+            def __del__(self):
+                if self.x:
+                    self.x = lltype.nullptr(S)
+                    lltype.free(self.x, flavor='raw')
+
+        def f():
+            return A()
+
+        r = self.analyze(f, [], A.__del__.im_func)
+        assert not r
+
+    def test_c_call(self):
+        C = rffi.CArray(lltype.Signed)
+        c = rffi.llexternal('x', [lltype.Ptr(C)], lltype.Signed)
+
+        def g():
+            p = lltype.malloc(C, 3, flavor='raw')
+            f(p)
+        
+        def f(p):
+            c(rffi.ptradd(p, 0))
+            lltype.free(p, flavor='raw')
+
+        r = self.analyze(g, [], f, backendopt=True)
+        assert not r
+
+    def test_chain(self):
+        class B(object):
+            def __init__(self):
+                self.counter = 1
+        
+        class A(object):
+            def __init__(self):
+                self.x = B()
+
+            def __del__(self):
+                self.x.counter += 1
+
+        def f():
+            A()
+
+        r = self.analyze(f, [], A.__del__.im_func)
+        assert r
+
+    def test_is_light_finalizer_decorator(self):
+        S = lltype.GcStruct('S')
+
+        @rgc.is_light_finalizer
+        def f():
+            lltype.malloc(S)
+        @rgc.is_light_finalizer
+        def g():
+            pass
+        self.analyze(g, []) # did not explode
+        py.test.raises(FinalizerError, self.analyze, f, [])
+
+class TestOOType(BaseFinalizerAnalyzerTests):
+    type_system = 'ootype'

From noreply at buildbot.pypy.org  Wed Oct 26 16:31:57 2011
From: noreply at buildbot.pypy.org (cfbolz)
Date: Wed, 26 Oct 2011 16:31:57 +0200 (CEST)
Subject: [pypy-commit] pypy default: yet another case of tagged pointer
 handling in the C backend.
Message-ID: <20111026143157.AFDC9820C7@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r48473:990f8f82e063
Date: 2011-10-26 16:31 +0200
http://bitbucket.org/pypy/pypy/changeset/990f8f82e063/

Log:	yet another case of tagged pointer handling in the C backend.

diff --git a/pypy/translator/c/primitive.py b/pypy/translator/c/primitive.py
--- a/pypy/translator/c/primitive.py
+++ b/pypy/translator/c/primitive.py
@@ -144,14 +144,19 @@
         obj = value._obj
         if isinstance(obj, int):
             # a tagged pointer
-            assert obj & 1 == 1
-            return '((%s) %d)' % (cdecl("void*", ''), obj)
+            return _name_tagged(obj, db)
         realobj = obj.container
+        if isinstance(realobj, int):
+            return _name_tagged(realobj, db)
         realvalue = cast_opaque_ptr(Ptr(typeOf(realobj)), value)
         return db.get(realvalue)
     else:
         return 'NULL'
 
+def _name_tagged(obj, db):
+    assert obj & 1 == 1
+    return '((%s) %d)' % (cdecl("void*", ''), obj)
+
 def name_small_integer(value, db):
     """Works for integers of size at most INT or UINT."""
     if isinstance(value, Symbolic):
diff --git a/pypy/translator/c/test/test_rtagged.py b/pypy/translator/c/test/test_rtagged.py
--- a/pypy/translator/c/test/test_rtagged.py
+++ b/pypy/translator/c/test/test_rtagged.py
@@ -77,3 +77,12 @@
     data = g.read()
     g.close()
     assert data.rstrip().endswith('ALL OK')
+
+def test_name_gcref():
+    from pypy.rpython.lltypesystem import lltype, llmemory, rclass
+    from pypy.translator.c import primitive
+    from pypy.translator.c.database import LowLevelDatabase
+    x = lltype.cast_int_to_ptr(rclass.OBJECTPTR, 19)
+    y = lltype.cast_opaque_ptr(llmemory.GCREF, x)
+    db = LowLevelDatabase()
+    assert primitive.name_gcref(y, db) == "((void*) 19)"

From noreply at buildbot.pypy.org  Wed Oct 26 17:00:31 2011
From: noreply at buildbot.pypy.org (cfbolz)
Date: Wed, 26 Oct 2011 17:00:31 +0200 (CEST)
Subject: [pypy-commit] pypy default: add longs as a project idea
Message-ID: <20111026150031.2265A820C7@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r48474:e8b365c58a38
Date: 2011-10-26 17:00 +0200
http://bitbucket.org/pypy/pypy/changeset/e8b365c58a38/

Log:	add longs as a project idea

diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -17,6 +17,12 @@
 projects, or anything else in PyPy, pop up on IRC or write to us on the
 `mailing list`_.
 
+Make big integers faster
+-------------------------
+
+PyPy's implementation of the Python ``long`` type is slower than CPython's.
+Find out why and optimize them.
+
 Numpy improvements
 ------------------
 

From noreply at buildbot.pypy.org  Wed Oct 26 17:37:15 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 26 Oct 2011 17:37:15 +0200 (CEST)
Subject: [pypy-commit] pypy jit-simplify-backendintf: Fix some tests by
 using a slightly different logic in non-translated
Message-ID: <20111026153715.96097820C7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: jit-simplify-backendintf
Changeset: r48475:bfb70bc8d71f
Date: 2011-10-26 17:36 +0200
http://bitbucket.org/pypy/pypy/changeset/bfb70bc8d71f/

Log:	Fix some tests by using a slightly different logic in non-translated
	and in translated versions (horror!).

diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -1980,15 +1980,37 @@
             else: assert 0
         return ints[:], refs[:], floats[:]
 
-    def raise_continue_running_normally(self, live_arg_boxes):
+    def raise_continue_running_normally(self, live_arg_boxes, loop_token):
         self.history.inputargs = None
         self.history.operations = None
+        # For simplicity, we just raise ContinueRunningNormally here and
+        # ignore the loop_token passed in.  It means that we go back to
+        # interpreted mode, but it should come back very quickly to the
+        # JIT, find probably the same 'loop_token', and execute it.
+        if we_are_translated():
+            num_green_args = self.jitdriver_sd.num_green_args
+            gi, gr, gf = self._unpack_boxes(live_arg_boxes, 0, num_green_args)
+            ri, rr, rf = self._unpack_boxes(live_arg_boxes, num_green_args,
+                                            len(live_arg_boxes))
+            CRN = self.staticdata.ContinueRunningNormally
+            raise CRN(gi, gr, gf, ri, rr, rf)
+        else:
+            # However, in order to keep the existing tests working
+            # (which are based on the assumption that 'loop_token' is
+            # directly used here), a bit of custom non-translatable code...
+            self._nontranslated_run_directly(live_arg_boxes, loop_token)
+            assert 0, "unreachable"
+
+    def _nontranslated_run_directly(self, live_arg_boxes, loop_token):
+        "NOT_RPYTHON"
+        args = []
         num_green_args = self.jitdriver_sd.num_green_args
-        gi, gr, gf = self._unpack_boxes(live_arg_boxes, 0, num_green_args)
-        ri, rr, rf = self._unpack_boxes(live_arg_boxes, num_green_args,
-                                        len(live_arg_boxes))
-        CRN = self.staticdata.ContinueRunningNormally
-        raise CRN(gi, gr, gf, ri, rr, rf)
+        for box in live_arg_boxes[num_green_args:]:
+            if   box.type == history.INT: args.append(box.getint())
+            elif box.type == history.REF: args.append(box.getref_base())
+            elif box.type == history.FLOAT: args.append(box.getfloatstorage())
+            else: assert 0
+        self.jitdriver_sd.warmstate.execute_assembler(loop_token, *args)
 
     def prepare_resume_from_failure(self, opnum, dont_change_position=False):
         frame = self.framestack[-1]
@@ -2051,7 +2073,7 @@
                                               greenkey, start, start_resumedescr)
         if loop_token is not None: # raise if it *worked* correctly
             self.set_compiled_merge_points(greenkey, old_loop_tokens)
-            self.raise_continue_running_normally(live_arg_boxes)
+            self.raise_continue_running_normally(live_arg_boxes, loop_token)
 
         self.history.inputargs = original_inputargs
         self.history.operations.pop()     # remove the JUMP
@@ -2072,7 +2094,8 @@
         finally:
             self.history.operations.pop()     # remove the JUMP
         if target_loop_token is not None: # raise if it *worked* correctly
-            self.raise_continue_running_normally(live_arg_boxes)
+            self.raise_continue_running_normally(live_arg_boxes,
+                                                 target_loop_token)
 
     def compile_bridge_and_loop(self, original_boxes, live_arg_boxes, start,
                                 bridge_arg_boxes, start_resumedescr):
@@ -2108,7 +2131,8 @@
         except RetraceLoop:
             assert False
         assert target_loop_token is not None
-        self.raise_continue_running_normally(live_arg_boxes)
+        self.raise_continue_running_normally(live_arg_boxes,
+                                             old_loop_tokens[0])
 
     def compile_done_with_this_frame(self, exitbox):
         self.gen_store_back_in_virtualizable()
diff --git a/pypy/jit/metainterp/warmstate.py b/pypy/jit/metainterp/warmstate.py
--- a/pypy/jit/metainterp/warmstate.py
+++ b/pypy/jit/metainterp/warmstate.py
@@ -12,6 +12,7 @@
 from pypy.rlib.debug import debug_start, debug_stop, debug_print
 from pypy.jit.metainterp import history
 from pypy.jit.codewriter import support, heaptracker, longlong
+from pypy.tool.sourcetools import func_with_new_name
 
 # ____________________________________________________________
 
@@ -294,11 +295,15 @@
         confirm_enter_jit = self.confirm_enter_jit
         range_red_args = unrolling_iterable(
             range(num_green_args, num_green_args + jitdriver_sd.num_red_args))
+        # hack: make a new copy of the method
+        func_execute_token = self.cpu.execute_token.im_func
+        func_execute_token = func_with_new_name(func_execute_token,
+                                                "execute_token_spec")
 
         def execute_assembler(loop_token, *args):
             # Call the backend to run the 'looptoken' with the given
             # input args.
-            fail_descr = self.cpu.execute_token(loop_token, *args)
+            fail_descr = func_execute_token(self.cpu, loop_token, *args)
             #
             # If we have a virtualizable, we have to reset its
             # 'vable_token' field afterwards

From noreply at buildbot.pypy.org  Wed Oct 26 17:39:52 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 26 Oct 2011 17:39:52 +0200 (CEST)
Subject: [pypy-commit] pypy jit-simplify-backendintf: Fix test.
Message-ID: <20111026153952.C9C46820C7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: jit-simplify-backendintf
Changeset: r48476:3a49ad1be7d9
Date: 2011-10-26 17:38 +0200
http://bitbucket.org/pypy/pypy/changeset/3a49ad1be7d9/

Log:	Fix test.

diff --git a/pypy/jit/metainterp/test/test_compile.py b/pypy/jit/metainterp/test/test_compile.py
--- a/pypy/jit/metainterp/test/test_compile.py
+++ b/pypy/jit/metainterp/test/test_compile.py
@@ -200,19 +200,14 @@
                                       [BoxInt(56), ConstInt(78), BoxInt(90)])
     #
     raiseme = None
-    cpu.set_future_value_int(0, -156)
-    cpu.set_future_value_int(1, -178)
-    cpu.set_future_value_int(2, -190)     # passed in, but dropped
-    fail_descr = cpu.execute_token(loop_token)
+    # arg -190 is passed in, but dropped
+    fail_descr = cpu.execute_token(loop_token, -156, -178, -190)
     assert fail_descr is FakeJitDriverSD().portal_finishtoken
     #
     EXC = lltype.GcStruct('EXC')
     llexc = lltype.malloc(EXC)
     raiseme = LLException("exception class", llexc)
-    cpu.set_future_value_int(0, -156)
-    cpu.set_future_value_int(1, -178)
-    cpu.set_future_value_int(2, -190)
-    fail_descr = cpu.execute_token(loop_token)
+    fail_descr = cpu.execute_token(loop_token, -156, -178, -190)
     assert isinstance(fail_descr, compile.PropagateExceptionDescr)
     got = cpu.grab_exc_value()
     assert lltype.cast_opaque_ptr(lltype.Ptr(EXC), got) == llexc
@@ -221,10 +216,7 @@
         class ExitFrameWithExceptionRef(Exception):
             pass
     FakeMetaInterpSD.cpu = cpu
-    cpu.set_future_value_int(0, -156)
-    cpu.set_future_value_int(1, -178)
-    cpu.set_future_value_int(2, -190)
-    fail_descr = cpu.execute_token(loop_token)
+    fail_descr = cpu.execute_token(loop_token, -156, -178, -190)
     try:
         fail_descr.handle_fail(FakeMetaInterpSD(), None)
     except FakeMetaInterpSD.ExitFrameWithExceptionRef, e:

From noreply at buildbot.pypy.org  Wed Oct 26 17:39:54 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 26 Oct 2011 17:39:54 +0200 (CEST)
Subject: [pypy-commit] pypy jit-simplify-backendintf: Kill outdated test.
Message-ID: <20111026153954.032CC820C7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: jit-simplify-backendintf
Changeset: r48477:c28a86f87724
Date: 2011-10-26 17:39 +0200
http://bitbucket.org/pypy/pypy/changeset/c28a86f87724/

Log:	Kill outdated test.

diff --git a/pypy/jit/metainterp/test/test_warmstate.py b/pypy/jit/metainterp/test/test_warmstate.py
--- a/pypy/jit/metainterp/test/test_warmstate.py
+++ b/pypy/jit/metainterp/test/test_warmstate.py
@@ -151,29 +151,6 @@
     assert get_jitcell(False, 42, 0.25) is cell4
     assert cell1 is not cell3 is not cell4 is not cell1
 
-def test_make_set_future_values():
-    future_values = {}
-    class FakeCPU:
-        def set_future_value_int(self, j, value):
-            future_values[j] = "int", value
-        def set_future_value_float(self, j, value):
-            future_values[j] = "float", value
-    class FakeWarmRunnerDesc:
-        cpu = FakeCPU()
-        memory_manager = None
-    class FakeJitDriverSD:
-        _red_args_types = ["int", "float"]
-        virtualizable_info = None
-    #
-    state = WarmEnterState(FakeWarmRunnerDesc(), FakeJitDriverSD())
-    set_future_values = state.make_set_future_values()
-    set_future_values(5, 42.5)
-    assert future_values == {
-        0: ("int", 5),
-        1: ("float", longlong.getfloatstorage(42.5)),
-    }
-    assert set_future_values is state.make_set_future_values()
-
 def test_make_unwrap_greenkey():
     class FakeJitDriverSD:
         _green_args_spec = [lltype.Signed, lltype.Float]

From noreply at buildbot.pypy.org  Wed Oct 26 17:43:41 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 26 Oct 2011 17:43:41 +0200 (CEST)
Subject: [pypy-commit] pypy jit-simplify-backendintf: Fix test.
Message-ID: <20111026154341.0FC1E820C7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: jit-simplify-backendintf
Changeset: r48478:40312ab5a8ff
Date: 2011-10-26 17:41 +0200
http://bitbucket.org/pypy/pypy/changeset/40312ab5a8ff/

Log:	Fix test.

diff --git a/pypy/jit/metainterp/test/test_jitprof.py b/pypy/jit/metainterp/test/test_jitprof.py
--- a/pypy/jit/metainterp/test/test_jitprof.py
+++ b/pypy/jit/metainterp/test/test_jitprof.py
@@ -10,7 +10,7 @@
         self.counter = 123456
         Profiler.start(self)
         self.events = []
-        self.times = [0, 0, 0, 0]
+        self.times = [0, 0]
     
     def timer(self):
         self.counter += 1
@@ -24,12 +24,6 @@
         Profiler._end(self, event)
         self.events.append(~event)
 
-    def start_running(self):   self._start(RUNNING)
-    def end_running(self):     self._end(RUNNING)
-
-    def start_blackhole(self): self._start(BLACKHOLE)
-    def end_blackhole(self):   self._end(BLACKHOLE)
-
 class ProfilerMixin(LLJitMixin):
     def meta_interp(self, *args, **kwds):
         kwds = kwds.copy()
@@ -58,14 +52,10 @@
             BACKEND,
             ~ BACKEND,
             ~ TRACING,
-            RUNNING,
-            ~ RUNNING,
-            BLACKHOLE,
-            ~ BLACKHOLE
             ]
         assert profiler.events == expected
-        assert profiler.times == [3, 2, 1, 1]
-        assert profiler.counters == [1, 2, 1, 1, 3, 3, 1, 13, 2, 0, 0, 0, 0,
+        assert profiler.times == [3, 2]
+        assert profiler.counters == [1, 2, 3, 3, 1, 13, 2, 0, 0, 0, 0,
                                      0, 0, 0, 0, 0]
 
     def test_simple_loop_with_call(self):

From noreply at buildbot.pypy.org  Wed Oct 26 19:01:04 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 26 Oct 2011 19:01:04 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix test_ll_random. Add an assert in
 lltype to check the type of
Message-ID: <20111026170104.2A1FE82A87@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48480:c84880f2e5ca
Date: 2011-10-26 19:00 +0200
http://bitbucket.org/pypy/pypy/changeset/c84880f2e5ca/

Log:	Fix test_ll_random. Add an assert in lltype to check the type of
	the values we try to set in an array with array.setitem().

diff --git a/pypy/jit/backend/test/test_ll_random.py b/pypy/jit/backend/test/test_ll_random.py
--- a/pypy/jit/backend/test/test_ll_random.py
+++ b/pypy/jit/backend/test/test_ll_random.py
@@ -180,8 +180,16 @@
                     dic[fieldname] = getattr(p, fieldname)
         else:
             assert isinstance(S, lltype.Array)
-            for i in range(len(p)):
-                dic[i] = p[i]
+            if isinstance(S.OF, lltype.Struct):
+                for i in range(len(p)):
+                    item = p[i]
+                    s1 = {}
+                    for fieldname in S.OF._names:
+                        s1[fieldname] = getattr(item, fieldname)
+                    dic[i] = s1
+            else:
+                for i in range(len(p)):
+                    dic[i] = p[i]
         return dic
 
     def print_loop_prebuilt(self, names, writevar, s):
diff --git a/pypy/jit/backend/test/test_random.py b/pypy/jit/backend/test/test_random.py
--- a/pypy/jit/backend/test/test_random.py
+++ b/pypy/jit/backend/test/test_random.py
@@ -595,6 +595,10 @@
             for name, value in fields.items():
                 if isinstance(name, str):
                     setattr(container, name, value)
+                elif isinstance(value, dict):
+                    item = container.getitem(name)
+                    for key1, value1 in value.items():
+                        setattr(item, key1, value1)
                 else:
                     container.setitem(name, value)
 
diff --git a/pypy/rpython/lltypesystem/lltype.py b/pypy/rpython/lltypesystem/lltype.py
--- a/pypy/rpython/lltypesystem/lltype.py
+++ b/pypy/rpython/lltypesystem/lltype.py
@@ -1713,6 +1713,7 @@
         return v
 
     def setitem(self, index, value):
+        assert typeOf(value) == self._TYPE.OF
         self.items[index] = value
 
 assert not '__dict__' in dir(_array)

From noreply at buildbot.pypy.org  Wed Oct 26 19:01:02 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 26 Oct 2011 19:01:02 +0200 (CEST)
Subject: [pypy-commit] pypy default: Random fixes (no pun intended)
Message-ID: <20111026170102.EB608820C7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48479:2268df591186
Date: 2011-10-26 18:17 +0200
http://bitbucket.org/pypy/pypy/changeset/2268df591186/

Log:	Random fixes (no pun intended)

diff --git a/pypy/jit/backend/test/test_ll_random.py b/pypy/jit/backend/test/test_ll_random.py
--- a/pypy/jit/backend/test/test_ll_random.py
+++ b/pypy/jit/backend/test/test_ll_random.py
@@ -34,8 +34,8 @@
             v, S = from_[i][:2]
             if not isinstance(S, type):
                 continue
-            if (isinstance(S, lltype.Array) and
-                isinstance(S.OF, lltype.Struct) == array_of_structs):
+            if ((isinstance(S, lltype.Array) and
+                 isinstance(S.OF, lltype.Struct)) == array_of_structs):
                 ptrvars.append((v, S))
         return ptrvars
 
@@ -270,10 +270,7 @@
                                          array_of_structs=True)
         array = v.getref(lltype.Ptr(A))
         v_index = builder.get_index(len(array), r)
-        names = A.OF._names
-        if names[0] == 'parent':
-            names = names[1:]
-        name = r.choice(names)
+        name = r.choice(A.OF._names)
         descr = builder.cpu.interiorfielddescrof(A, name)
         descr._random_info = 'cpu.interiorfielddescrof(%s, %r)' % (A.OF._name,
                                                                    name)

From noreply at buildbot.pypy.org  Wed Oct 26 19:06:06 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 26 Oct 2011 19:06:06 +0200 (CEST)
Subject: [pypy-commit] pypy default: SetInteriorFieldOperation.
Message-ID: <20111026170606.6D4EF820C7@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48481:37be72baea57
Date: 2011-10-26 19:05 +0200
http://bitbucket.org/pypy/pypy/changeset/37be72baea57/

Log:	SetInteriorFieldOperation.

diff --git a/pypy/jit/backend/test/test_ll_random.py b/pypy/jit/backend/test/test_ll_random.py
--- a/pypy/jit/backend/test/test_ll_random.py
+++ b/pypy/jit/backend/test/test_ll_random.py
@@ -306,11 +306,9 @@
                 break
         builder.do(self.opnum, [v, w], descr)
 
-class SetInteriorFieldOperation(GetFieldOperation):
+class SetInteriorFieldOperation(GetInteriorFieldOperation):
     def produce_into(self, builder, r):
-        import pdb
-        pdb.set_trace()
-        v, descr, TYPE = self.field_descr(builder, r)
+        v, v_index, descr, TYPE = self.field_descr(builder, r)
         while True:
             if r.random() < 0.3:
                 w = ConstInt(r.random_integer())
@@ -318,7 +316,7 @@
                 w = r.choice(builder.intvars)
             if rffi.cast(lltype.Signed, rffi.cast(TYPE, w.value)) == w.value:
                 break
-        builder.do(self.opnum, [v, w], descr)
+        builder.do(self.opnum, [v, v_index, w], descr)
 
 class NewOperation(test_random.AbstractOperation):
     def size_descr(self, builder, S):
@@ -657,7 +655,7 @@
     OPERATIONS.append(GetFieldOperation(rop.GETFIELD_GC))
     OPERATIONS.append(GetInteriorFieldOperation(rop.GETINTERIORFIELD_GC))
     OPERATIONS.append(SetFieldOperation(rop.SETFIELD_GC))
-    #OPERATIONS.append(SetInteriorFieldOperation(rop.SETINTERIORFIELD_GC))
+    OPERATIONS.append(SetInteriorFieldOperation(rop.SETINTERIORFIELD_GC))
     OPERATIONS.append(NewOperation(rop.NEW))
     OPERATIONS.append(NewOperation(rop.NEW_WITH_VTABLE))
 

From noreply at buildbot.pypy.org  Wed Oct 26 19:36:00 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 26 Oct 2011 19:36:00 +0200 (CEST)
Subject: [pypy-commit] pypy default: Few fixes for test_zll_random on 32bit
Message-ID: <20111026173600.6097E820C7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r48482:a2b911e61392
Date: 2011-10-26 19:35 +0200
http://bitbucket.org/pypy/pypy/changeset/a2b911e61392/

Log:	Few fixes for test_zll_random on 32bit

diff --git a/pypy/jit/backend/llsupport/regalloc.py b/pypy/jit/backend/llsupport/regalloc.py
--- a/pypy/jit/backend/llsupport/regalloc.py
+++ b/pypy/jit/backend/llsupport/regalloc.py
@@ -178,6 +178,8 @@
         cur_max_age = -1
         candidate = None
         for next in self.reg_bindings:
+            if isinstance(next, TempBox):
+                continue
             reg = self.reg_bindings[next]
             if next in forbidden_vars:
                 continue
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1613,7 +1613,10 @@
     def genop_discard_setinteriorfield_gc(self, op, arglocs):
         base_loc, ofs_loc, itemsize_loc, fieldsize_loc, index_loc, value_loc = arglocs
         # XXX should not use IMUL in most cases
-        self.mc.IMUL(index_loc, itemsize_loc)
+        if isinstance(index_loc, ImmedLoc):
+            index_loc = imm(index_loc.value * itemsize_loc.value)
+        else:
+            self.mc.IMUL(index_loc, itemsize_loc)
         dest_addr = AddressLoc(base_loc, index_loc, 0, ofs_loc.value)
         self.save_into_mem(dest_addr, value_loc, fieldsize_loc)
 
diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py
--- a/pypy/jit/backend/x86/regalloc.py
+++ b/pypy/jit/backend/x86/regalloc.py
@@ -1042,14 +1042,18 @@
         t = self._unpack_interiorfielddescr(op.getdescr())
         ofs, itemsize, fieldsize, _ = t
         args = op.getarglist()
-        tmpvar = TempBox()
+        if fieldsize.value == 1:
+            need_lower_byte = True
+        else:
+            need_lower_byte = False
         base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
-        index_loc = self.rm.force_result_in_reg(tmpvar, op.getarg(1),
-                                                args)
+        tempvar = TempBox()
+        index_loc = self.rm.force_result_in_reg(tempvar, op.getarg(1), args)
         # we're free to modify index now
-        value_loc = self.make_sure_var_in_reg(op.getarg(2), args)
+        value_loc = self.make_sure_var_in_reg(op.getarg(2), args,
+                                              need_lower_byte=need_lower_byte)
+        self.rm.possibly_free_var(tempvar)
         self.possibly_free_vars(args)
-        self.rm.possibly_free_var(tmpvar)
         self.PerformDiscard(op, [base_loc, ofs, itemsize, fieldsize,
                                  index_loc, value_loc])
 

From noreply at buildbot.pypy.org  Wed Oct 26 21:07:44 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Wed, 26 Oct 2011 21:07:44 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-dicts: kill some dead code
Message-ID: <20111026190744.D9770820C7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: virtual-dicts
Changeset: r48483:3789a09bff0f
Date: 2011-10-25 21:17 -0400
http://bitbucket.org/pypy/pypy/changeset/3789a09bff0f/

Log:	kill some dead code

diff --git a/pypy/jit/metainterp/optimizeopt/vstring.py b/pypy/jit/metainterp/optimizeopt/vstring.py
--- a/pypy/jit/metainterp/optimizeopt/vstring.py
+++ b/pypy/jit/metainterp/optimizeopt/vstring.py
@@ -163,17 +163,6 @@
             for value in self._chars:
                 value.get_args_for_fail(modifier)
 
-    def FIXME_enum_forced_boxes(self, boxes, already_seen):
-        key = self.get_key_box()
-        if key in already_seen:
-            return
-        already_seen[key] = None
-        if self.box is None:
-            for box in self._chars:
-                box.enum_forced_boxes(boxes, already_seen)
-        else:
-            boxes.append(self.box)
-
     def _make_virtual(self, modifier):
         return modifier.make_vstrplain(self.mode is mode_unicode)
 
@@ -226,18 +215,6 @@
             self.left.get_args_for_fail(modifier)
             self.right.get_args_for_fail(modifier)
 
-    def FIXME_enum_forced_boxes(self, boxes, already_seen):
-        key = self.get_key_box()
-        if key in already_seen:
-            return
-        already_seen[key] = None
-        if self.box is None:
-            self.left.enum_forced_boxes(boxes, already_seen)
-            self.right.enum_forced_boxes(boxes, already_seen)
-            self.lengthbox = None
-        else:
-            boxes.append(self.box)
-
     def _make_virtual(self, modifier):
         return modifier.make_vstrconcat(self.mode is mode_unicode)
 
@@ -284,18 +261,6 @@
             self.vstart.get_args_for_fail(modifier)
             self.vlength.get_args_for_fail(modifier)
 
-    def FIXME_enum_forced_boxes(self, boxes, already_seen):
-        key = self.get_key_box()
-        if key in already_seen:
-            return
-        already_seen[key] = None
-        if self.box is None:
-            self.vstr.enum_forced_boxes(boxes, already_seen)
-            self.vstart.enum_forced_boxes(boxes, already_seen)
-            self.vlength.enum_forced_boxes(boxes, already_seen)
-        else:
-            boxes.append(self.box)
-
     def _make_virtual(self, modifier):
         return modifier.make_vstrslice(self.mode is mode_unicode)
 

From noreply at buildbot.pypy.org  Wed Oct 26 21:07:46 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Wed, 26 Oct 2011 21:07:46 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-dicts: merged default
Message-ID: <20111026190746.3EA53820C7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: virtual-dicts
Changeset: r48484:5e5b44d30598
Date: 2011-10-26 14:18 -0400
http://bitbucket.org/pypy/pypy/changeset/5e5b44d30598/

Log:	merged default

diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -17,6 +17,12 @@
 projects, or anything else in PyPy, pop up on IRC or write to us on the
 `mailing list`_.
 
+Make big integers faster
+-------------------------
+
+PyPy's implementation of the Python ``long`` type is slower than CPython's.
+Find out why and optimize them.
+
 Numpy improvements
 ------------------
 
diff --git a/pypy/jit/backend/llsupport/regalloc.py b/pypy/jit/backend/llsupport/regalloc.py
--- a/pypy/jit/backend/llsupport/regalloc.py
+++ b/pypy/jit/backend/llsupport/regalloc.py
@@ -178,6 +178,8 @@
         cur_max_age = -1
         candidate = None
         for next in self.reg_bindings:
+            if isinstance(next, TempBox):
+                continue
             reg = self.reg_bindings[next]
             if next in forbidden_vars:
                 continue
diff --git a/pypy/jit/backend/llsupport/test/test_gc.py b/pypy/jit/backend/llsupport/test/test_gc.py
--- a/pypy/jit/backend/llsupport/test/test_gc.py
+++ b/pypy/jit/backend/llsupport/test/test_gc.py
@@ -250,10 +250,11 @@
                                   has_finalizer, has_light_finalizer,
                                   contains_weakptr):
         assert not contains_weakptr
+        assert not has_finalizer           # in these tests
+        assert not has_light_finalizer     # in these tests
         p = llmemory.raw_malloc(size)
         p = llmemory.cast_adr_to_ptr(p, RESTYPE)
-        flags = (int(has_finalizer) << 16) | (int(has_light_finalizer) << 17)
-        tid = llop.combine_ushort(lltype.Signed, type_id, flags)
+        tid = llop.combine_ushort(lltype.Signed, type_id, 0)
         self.record.append(("fixedsize", repr(size), tid, p))
         return p
 
diff --git a/pypy/jit/backend/test/test_ll_random.py b/pypy/jit/backend/test/test_ll_random.py
--- a/pypy/jit/backend/test/test_ll_random.py
+++ b/pypy/jit/backend/test/test_ll_random.py
@@ -28,16 +28,27 @@
         fork.structure_types_and_vtables = self.structure_types_and_vtables
         return fork
 
-    def get_structptr_var(self, r, must_have_vtable=False, type=lltype.Struct):
+    def _choose_ptr_vars(self, from_, type, array_of_structs):
+        ptrvars = []
+        for i in range(len(from_)):
+            v, S = from_[i][:2]
+            if not isinstance(S, type):
+                continue
+            if ((isinstance(S, lltype.Array) and
+                 isinstance(S.OF, lltype.Struct)) == array_of_structs):
+                ptrvars.append((v, S))
+        return ptrvars
+
+    def get_structptr_var(self, r, must_have_vtable=False, type=lltype.Struct,
+                          array_of_structs=False):
         while True:
-            ptrvars = [(v, S) for (v, S) in self.ptrvars
-                              if isinstance(S, type)]
+            ptrvars = self._choose_ptr_vars(self.ptrvars, type,
+                                            array_of_structs)
             if ptrvars and r.random() < 0.8:
                 v, S = r.choice(ptrvars)
             else:
-                prebuilt_ptr_consts = [(v, S)
-                                 for (v, S, _) in self.prebuilt_ptr_consts
-                                 if isinstance(S, type)]
+                prebuilt_ptr_consts = self._choose_ptr_vars(
+                    self.prebuilt_ptr_consts, type, array_of_structs)
                 if prebuilt_ptr_consts and r.random() < 0.7:
                     v, S = r.choice(prebuilt_ptr_consts)
                 else:
@@ -48,7 +59,8 @@
                                                 has_vtable=must_have_vtable)
                     else:
                         # create a new constant array
-                        p = self.get_random_array(r)
+                        p = self.get_random_array(r,
+                                    must_be_array_of_structs=array_of_structs)
                     S = lltype.typeOf(p).TO
                     v = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, p))
                     self.prebuilt_ptr_consts.append((v, S,
@@ -74,7 +86,8 @@
                 TYPE = lltype.Signed
         return TYPE
 
-    def get_random_structure_type(self, r, with_vtable=None, cache=True):
+    def get_random_structure_type(self, r, with_vtable=None, cache=True,
+                                  type=lltype.GcStruct):
         if cache and self.structure_types and r.random() < 0.5:
             return r.choice(self.structure_types)
         fields = []
@@ -85,7 +98,7 @@
         for i in range(r.randrange(1, 5)):
             TYPE = self.get_random_primitive_type(r)
             fields.append(('f%d' % i, TYPE))
-        S = lltype.GcStruct('S%d' % self.counter, *fields, **kwds)
+        S = type('S%d' % self.counter, *fields, **kwds)
         self.counter += 1
         if cache:
             self.structure_types.append(S)
@@ -125,17 +138,29 @@
                 setattr(p, fieldname, rffi.cast(TYPE, r.random_integer()))
         return p
 
-    def get_random_array_type(self, r):
-        TYPE = self.get_random_primitive_type(r)
+    def get_random_array_type(self, r, can_be_array_of_struct=False,
+                              must_be_array_of_structs=False):
+        if ((can_be_array_of_struct and r.random() < 0.1) or
+            must_be_array_of_structs):
+            TYPE = self.get_random_structure_type(r, cache=False,
+                                                  type=lltype.Struct)
+        else:
+            TYPE = self.get_random_primitive_type(r)
         return lltype.GcArray(TYPE)
 
-    def get_random_array(self, r):
-        A = self.get_random_array_type(r)
+    def get_random_array(self, r, must_be_array_of_structs=False):
+        A = self.get_random_array_type(r,
+                           must_be_array_of_structs=must_be_array_of_structs)
         length = (r.random_integer() // 15) % 300  # length: between 0 and 299
                                                    # likely to be small
         p = lltype.malloc(A, length)
-        for i in range(length):
-            p[i] = rffi.cast(A.OF, r.random_integer())
+        if isinstance(A.OF, lltype.Primitive):
+            for i in range(length):
+                p[i] = rffi.cast(A.OF, r.random_integer())
+        else:
+            for i in range(length):
+                for fname, TP in A.OF._flds.iteritems():
+                    setattr(p[i], fname, rffi.cast(TP, r.random_integer()))
         return p
 
     def get_index(self, length, r):
@@ -155,8 +180,16 @@
                     dic[fieldname] = getattr(p, fieldname)
         else:
             assert isinstance(S, lltype.Array)
-            for i in range(len(p)):
-                dic[i] = p[i]
+            if isinstance(S.OF, lltype.Struct):
+                for i in range(len(p)):
+                    item = p[i]
+                    s1 = {}
+                    for fieldname in S.OF._names:
+                        s1[fieldname] = getattr(item, fieldname)
+                    dic[i] = s1
+            else:
+                for i in range(len(p)):
+                    dic[i] = p[i]
         return dic
 
     def print_loop_prebuilt(self, names, writevar, s):
@@ -220,7 +253,7 @@
 
 class GetFieldOperation(test_random.AbstractOperation):
     def field_descr(self, builder, r):
-        v, S = builder.get_structptr_var(r)
+        v, S = builder.get_structptr_var(r, )
         names = S._names
         if names[0] == 'parent':
             names = names[1:]
@@ -239,6 +272,28 @@
                 continue
             break
 
+class GetInteriorFieldOperation(test_random.AbstractOperation):
+    def field_descr(self, builder, r):
+        v, A = builder.get_structptr_var(r, type=lltype.Array,
+                                         array_of_structs=True)
+        array = v.getref(lltype.Ptr(A))
+        v_index = builder.get_index(len(array), r)
+        name = r.choice(A.OF._names)
+        descr = builder.cpu.interiorfielddescrof(A, name)
+        descr._random_info = 'cpu.interiorfielddescrof(%s, %r)' % (A.OF._name,
+                                                                   name)
+        TYPE = getattr(A.OF, name)
+        return v, v_index, descr, TYPE
+
+    def produce_into(self, builder, r):
+        while True:
+            try:
+                v, v_index, descr, _ = self.field_descr(builder, r)
+                self.put(builder, [v, v_index], descr)
+            except lltype.UninitializedMemoryAccess:
+                continue
+            break
+
 class SetFieldOperation(GetFieldOperation):
     def produce_into(self, builder, r):
         v, descr, TYPE = self.field_descr(builder, r)
@@ -251,6 +306,18 @@
                 break
         builder.do(self.opnum, [v, w], descr)
 
+class SetInteriorFieldOperation(GetInteriorFieldOperation):
+    def produce_into(self, builder, r):
+        v, v_index, descr, TYPE = self.field_descr(builder, r)
+        while True:
+            if r.random() < 0.3:
+                w = ConstInt(r.random_integer())
+            else:
+                w = r.choice(builder.intvars)
+            if rffi.cast(lltype.Signed, rffi.cast(TYPE, w.value)) == w.value:
+                break
+        builder.do(self.opnum, [v, v_index, w], descr)
+
 class NewOperation(test_random.AbstractOperation):
     def size_descr(self, builder, S):
         descr = builder.cpu.sizeof(S)
@@ -306,7 +373,7 @@
 
 class NewArrayOperation(ArrayOperation):
     def produce_into(self, builder, r):
-        A = builder.get_random_array_type(r)
+        A = builder.get_random_array_type(r, can_be_array_of_struct=True)
         v_size = builder.get_index(300, r)
         v_ptr = builder.do(self.opnum, [v_size], self.array_descr(builder, A))
         builder.ptrvars.append((v_ptr, A))
@@ -586,7 +653,9 @@
 for i in range(4):      # make more common
     OPERATIONS.append(GetFieldOperation(rop.GETFIELD_GC))
     OPERATIONS.append(GetFieldOperation(rop.GETFIELD_GC))
+    OPERATIONS.append(GetInteriorFieldOperation(rop.GETINTERIORFIELD_GC))
     OPERATIONS.append(SetFieldOperation(rop.SETFIELD_GC))
+    OPERATIONS.append(SetInteriorFieldOperation(rop.SETINTERIORFIELD_GC))
     OPERATIONS.append(NewOperation(rop.NEW))
     OPERATIONS.append(NewOperation(rop.NEW_WITH_VTABLE))
 
diff --git a/pypy/jit/backend/test/test_random.py b/pypy/jit/backend/test/test_random.py
--- a/pypy/jit/backend/test/test_random.py
+++ b/pypy/jit/backend/test/test_random.py
@@ -595,6 +595,10 @@
             for name, value in fields.items():
                 if isinstance(name, str):
                     setattr(container, name, value)
+                elif isinstance(value, dict):
+                    item = container.getitem(name)
+                    for key1, value1 in value.items():
+                        setattr(item, key1, value1)
                 else:
                     container.setitem(name, value)
 
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1613,7 +1613,10 @@
     def genop_discard_setinteriorfield_gc(self, op, arglocs):
         base_loc, ofs_loc, itemsize_loc, fieldsize_loc, index_loc, value_loc = arglocs
         # XXX should not use IMUL in most cases
-        self.mc.IMUL(index_loc, itemsize_loc)
+        if isinstance(index_loc, ImmedLoc):
+            index_loc = imm(index_loc.value * itemsize_loc.value)
+        else:
+            self.mc.IMUL(index_loc, itemsize_loc)
         dest_addr = AddressLoc(base_loc, index_loc, 0, ofs_loc.value)
         self.save_into_mem(dest_addr, value_loc, fieldsize_loc)
 
diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py
--- a/pypy/jit/backend/x86/regalloc.py
+++ b/pypy/jit/backend/x86/regalloc.py
@@ -1042,14 +1042,18 @@
         t = self._unpack_interiorfielddescr(op.getdescr())
         ofs, itemsize, fieldsize, _ = t
         args = op.getarglist()
-        tmpvar = TempBox()
+        if fieldsize.value == 1:
+            need_lower_byte = True
+        else:
+            need_lower_byte = False
         base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
-        index_loc = self.rm.force_result_in_reg(tmpvar, op.getarg(1),
-                                                args)
+        tempvar = TempBox()
+        index_loc = self.rm.force_result_in_reg(tempvar, op.getarg(1), args)
         # we're free to modify index now
-        value_loc = self.make_sure_var_in_reg(op.getarg(2), args)
+        value_loc = self.make_sure_var_in_reg(op.getarg(2), args,
+                                              need_lower_byte=need_lower_byte)
+        self.rm.possibly_free_var(tempvar)
         self.possibly_free_vars(args)
-        self.rm.possibly_free_var(tmpvar)
         self.PerformDiscard(op, [base_loc, ofs, itemsize, fieldsize,
                                  index_loc, value_loc])
 
diff --git a/pypy/module/pypyjit/policy.py b/pypy/module/pypyjit/policy.py
--- a/pypy/module/pypyjit/policy.py
+++ b/pypy/module/pypyjit/policy.py
@@ -16,7 +16,8 @@
         if modname in ['pypyjit', 'signal', 'micronumpy', 'math', 'exceptions',
                        'imp', 'sys', 'array', '_ffi', 'itertools', 'operator',
                        'posix', '_socket', '_sre', '_lsprof', '_weakref',
-                       '__pypy__', 'cStringIO', '_collections', 'struct']:
+                       '__pypy__', 'cStringIO', '_collections', 'struct',
+                       'mmap']:
             return True
         return False
 
diff --git a/pypy/module/rctime/interp_time.py b/pypy/module/rctime/interp_time.py
--- a/pypy/module/rctime/interp_time.py
+++ b/pypy/module/rctime/interp_time.py
@@ -245,6 +245,9 @@
 if sys.platform != 'win32':
     @unwrap_spec(secs=float)
     def sleep(space, secs):
+        if secs < 0:
+            raise OperationError(space.w_IOError,
+                                 space.wrap("Invalid argument: negative time in sleep"))
         pytime.sleep(secs)
 else:
     from pypy.rlib import rwin32
@@ -265,6 +268,9 @@
                                    OSError(EINTR, "sleep() interrupted"))
     @unwrap_spec(secs=float)
     def sleep(space, secs):
+        if secs < 0:
+            raise OperationError(space.w_IOError,
+                                 space.wrap("Invalid argument: negative time in sleep"))
         # as decreed by Guido, only the main thread can be
         # interrupted.
         main_thread = space.fromcache(State).main_thread
diff --git a/pypy/module/rctime/test/test_rctime.py b/pypy/module/rctime/test/test_rctime.py
--- a/pypy/module/rctime/test/test_rctime.py
+++ b/pypy/module/rctime/test/test_rctime.py
@@ -20,8 +20,9 @@
         import sys
         import os
         raises(TypeError, rctime.sleep, "foo")
-        rctime.sleep(1.2345)
-        
+        rctime.sleep(0.12345)
+        raises(IOError, rctime.sleep, -1.0)
+
     def test_clock(self):
         import time as rctime
         rctime.clock()
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -83,11 +83,12 @@
         if self.config.objspace.std.withtproxy:
             transparent.setup(self)
 
+        interplevel_classes = {}
         for type, classes in self.model.typeorder.iteritems():
-            if len(classes) >= 3:
+            if len(classes) >= 3: # XXX what does this 3 mean??!
                 # W_Root, AnyXxx and actual object
-                self.gettypefor(type).interplevel_cls = classes[0][0]
-
+                interplevel_classes[self.gettypefor(type)] = classes[0][0]
+        self._interplevel_classes = interplevel_classes
 
     def get_builtin_types(self):
         return self.builtin_types
@@ -579,7 +580,7 @@
             raise OperationError(self.w_TypeError,
                                  self.wrap("need type object"))
         if is_annotation_constant(w_type):
-            cls = w_type.interplevel_cls
+            cls = self._get_interplevel_cls(w_type)
             if cls is not None:
                 assert w_inst is not None
                 if isinstance(w_inst, cls):
@@ -589,3 +590,9 @@
     @specialize.arg_or_var(2)
     def isinstance_w(space, w_inst, w_type):
         return space._type_isinstance(w_inst, w_type)
+
+    @specialize.memo()
+    def _get_interplevel_cls(self, w_type):
+        if not hasattr(self, "_interplevel_classes"):
+            return None # before running initialize
+        return self._interplevel_classes.get(w_type, None)
diff --git a/pypy/objspace/std/smallintobject.py b/pypy/objspace/std/smallintobject.py
--- a/pypy/objspace/std/smallintobject.py
+++ b/pypy/objspace/std/smallintobject.py
@@ -12,6 +12,7 @@
 from pypy.rlib.rbigint import rbigint
 from pypy.rlib.rarithmetic import r_uint
 from pypy.tool.sourcetools import func_with_new_name
+from pypy.objspace.std.inttype import wrapint
 
 class W_SmallIntObject(W_Object, UnboxedValue):
     __slots__ = 'intval'
@@ -48,14 +49,36 @@
 def delegate_SmallInt2Complex(space, w_small):
     return space.newcomplex(float(w_small.intval), 0.0)
 
+def add__SmallInt_SmallInt(space, w_a, w_b):
+    return wrapint(space, w_a.intval + w_b.intval) # cannot overflow
+
+def sub__SmallInt_SmallInt(space, w_a, w_b):
+    return wrapint(space, w_a.intval - w_b.intval) # cannot overflow
+
+def floordiv__SmallInt_SmallInt(space, w_a, w_b):
+    return wrapint(space, w_a.intval // w_b.intval) # cannot overflow
+
+div__SmallInt_SmallInt = floordiv__SmallInt_SmallInt
+
+def mod__SmallInt_SmallInt(space, w_a, w_b):
+    return wrapint(space, w_a.intval % w_b.intval) # cannot overflow
+
+def divmod__SmallInt_SmallInt(space, w_a, w_b):
+    w = wrapint(space, w_a.intval // w_b.intval) # cannot overflow
+    z = wrapint(space, w_a.intval % w_b.intval)
+    return space.newtuple([w, z])
+
 def copy_multimethods(ns):
     """Copy integer multimethods for small int."""
     for name, func in intobject.__dict__.iteritems():
         if "__Int" in name:
             new_name = name.replace("Int", "SmallInt")
-            # Copy the function, so the annotator specializes it for
-            # W_SmallIntObject.
-            ns[new_name] = func_with_new_name(func, new_name)
+            if new_name not in ns:
+                # Copy the function, so the annotator specializes it for
+                # W_SmallIntObject.
+                ns[new_name] = func = func_with_new_name(func, new_name, globals=ns)
+        else:
+            ns[name] = func
     ns["get_integer"] = ns["pos__SmallInt"] = ns["int__SmallInt"]
     ns["get_negint"] = ns["neg__SmallInt"]
 
diff --git a/pypy/objspace/std/test/test_obj.py b/pypy/objspace/std/test/test_obj.py
--- a/pypy/objspace/std/test/test_obj.py
+++ b/pypy/objspace/std/test/test_obj.py
@@ -102,3 +102,11 @@
             def __repr__(self):
                 return 123456
         assert A().__str__() == 123456
+
+def test_isinstance_shortcut():
+    from pypy.objspace.std import objspace
+    space = objspace.StdObjSpace()
+    w_a = space.wrap("a")
+    space.type = None
+    space.isinstance_w(w_a, space.w_str) # does not crash
+
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -102,7 +102,6 @@
                           'instancetypedef',
                           'terminator',
                           '_version_tag?',
-                          'interplevel_cls',
                           ]
 
     # for config.objspace.std.getattributeshortcut
@@ -117,9 +116,6 @@
     # of the __new__ is an instance of the type
     w_bltin_new = None
 
-    interplevel_cls = None # not None for prebuilt instances of
-                           # interpreter-level types
-
     @dont_look_inside
     def __init__(w_self, space, name, bases_w, dict_w,
                  overridetypedef=None):
diff --git a/pypy/rpython/lltypesystem/lltype.py b/pypy/rpython/lltypesystem/lltype.py
--- a/pypy/rpython/lltypesystem/lltype.py
+++ b/pypy/rpython/lltypesystem/lltype.py
@@ -1713,6 +1713,7 @@
         return v
 
     def setitem(self, index, value):
+        assert typeOf(value) == self._TYPE.OF
         self.items[index] = value
 
 assert not '__dict__' in dir(_array)
diff --git a/pypy/rpython/module/ll_os.py b/pypy/rpython/module/ll_os.py
--- a/pypy/rpython/module/ll_os.py
+++ b/pypy/rpython/module/ll_os.py
@@ -959,8 +959,6 @@
                             os_ftruncate(rffi.cast(rffi.INT, fd),
                                          rffi.cast(rffi.LONGLONG, length)))
             if res < 0:
-                # Note: for consistency we raise OSError, but CPython
-                # raises IOError here
                 raise OSError(rposix.get_errno(), "os_ftruncate failed")
 
         return extdef([int, r_longlong], s_None,
diff --git a/pypy/tool/sourcetools.py b/pypy/tool/sourcetools.py
--- a/pypy/tool/sourcetools.py
+++ b/pypy/tool/sourcetools.py
@@ -216,9 +216,11 @@
 
 # ____________________________________________________________
 
-def func_with_new_name(func, newname):
+def func_with_new_name(func, newname, globals=None):
     """Make a renamed copy of a function."""
-    f = new.function(func.func_code, func.func_globals,
+    if globals is None:
+        globals = func.func_globals
+    f = new.function(func.func_code, globals,
                         newname, func.func_defaults,
                         func.func_closure)
     if func.func_dict:
diff --git a/pypy/translator/c/primitive.py b/pypy/translator/c/primitive.py
--- a/pypy/translator/c/primitive.py
+++ b/pypy/translator/c/primitive.py
@@ -144,14 +144,19 @@
         obj = value._obj
         if isinstance(obj, int):
             # a tagged pointer
-            assert obj & 1 == 1
-            return '((%s) %d)' % (cdecl("void*", ''), obj)
+            return _name_tagged(obj, db)
         realobj = obj.container
+        if isinstance(realobj, int):
+            return _name_tagged(realobj, db)
         realvalue = cast_opaque_ptr(Ptr(typeOf(realobj)), value)
         return db.get(realvalue)
     else:
         return 'NULL'
 
+def _name_tagged(obj, db):
+    assert obj & 1 == 1
+    return '((%s) %d)' % (cdecl("void*", ''), obj)
+
 def name_small_integer(value, db):
     """Works for integers of size at most INT or UINT."""
     if isinstance(value, Symbolic):
diff --git a/pypy/translator/c/test/test_rtagged.py b/pypy/translator/c/test/test_rtagged.py
--- a/pypy/translator/c/test/test_rtagged.py
+++ b/pypy/translator/c/test/test_rtagged.py
@@ -77,3 +77,12 @@
     data = g.read()
     g.close()
     assert data.rstrip().endswith('ALL OK')
+
+def test_name_gcref():
+    from pypy.rpython.lltypesystem import lltype, llmemory, rclass
+    from pypy.translator.c import primitive
+    from pypy.translator.c.database import LowLevelDatabase
+    x = lltype.cast_int_to_ptr(rclass.OBJECTPTR, 19)
+    y = lltype.cast_opaque_ptr(llmemory.GCREF, x)
+    db = LowLevelDatabase()
+    assert primitive.name_gcref(y, db) == "((void*) 19)"

From noreply at buildbot.pypy.org  Wed Oct 26 21:15:39 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Wed, 26 Oct 2011 21:15:39 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-dicts: make this consistant with the
 other version of this code
Message-ID: <20111026191539.20C49820C7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: virtual-dicts
Changeset: r48485:072b6ee49f10
Date: 2011-10-26 15:15 -0400
http://bitbucket.org/pypy/pypy/changeset/072b6ee49f10/

Log:	make this consistant with the other version of this code

diff --git a/pypy/jit/metainterp/optimizeopt/virtualize.py b/pypy/jit/metainterp/optimizeopt/virtualize.py
--- a/pypy/jit/metainterp/optimizeopt/virtualize.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualize.py
@@ -504,8 +504,10 @@
             if indexbox is not None:
                 descr = op.getdescr()
                 fieldvalue = value.getinteriorfield(
-                    indexbox.getint(), descr, self.new_const(descr)
+                    indexbox.getint(), descr, None
                 )
+                if fieldvalue is None:
+                    fieldvalue = self.new_const(descr)
                 self.make_equal_to(op.result, fieldvalue)
                 return
         value.ensure_nonnull()

From noreply at buildbot.pypy.org  Wed Oct 26 21:27:10 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 26 Oct 2011 21:27:10 +0200 (CEST)
Subject: [pypy-commit] pypy numpy NDimArray: merge default
Message-ID: <20111026192710.1E79D820C7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: numpy NDimArray
Changeset: r48486:9283585d7084
Date: 2011-10-26 21:15 +0200
http://bitbucket.org/pypy/pypy/changeset/9283585d7084/

Log:	merge default

diff --git a/lib-python/modified-2.7/json/encoder.py b/lib-python/modified-2.7/json/encoder.py
--- a/lib-python/modified-2.7/json/encoder.py
+++ b/lib-python/modified-2.7/json/encoder.py
@@ -2,14 +2,7 @@
 """
 import re
 
-try:
-    from _json import encode_basestring_ascii as c_encode_basestring_ascii
-except ImportError:
-    c_encode_basestring_ascii = None
-try:
-    from _json import make_encoder as c_make_encoder
-except ImportError:
-    c_make_encoder = None
+from __pypy__.builders import StringBuilder, UnicodeBuilder
 
 ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]')
 ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])')
@@ -24,8 +17,7 @@
     '\t': '\\t',
 }
 for i in range(0x20):
-    ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i))
-    #ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,))
+    ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,))
 
 # Assume this produces an infinity on all machines (probably not guaranteed)
 INFINITY = float('1e66666')
@@ -37,10 +29,9 @@
     """
     def replace(match):
         return ESCAPE_DCT[match.group(0)]
-    return '"' + ESCAPE.sub(replace, s) + '"'
+    return ESCAPE.sub(replace, s)
 
-
-def py_encode_basestring_ascii(s):
+def encode_basestring_ascii(s):
     """Return an ASCII-only JSON representation of a Python string
 
     """
@@ -53,20 +44,18 @@
         except KeyError:
             n = ord(s)
             if n < 0x10000:
-                return '\\u{0:04x}'.format(n)
-                #return '\\u%04x' % (n,)
+                return '\\u%04x' % (n,)
             else:
                 # surrogate pair
                 n -= 0x10000
                 s1 = 0xd800 | ((n >> 10) & 0x3ff)
                 s2 = 0xdc00 | (n & 0x3ff)
-                return '\\u{0:04x}\\u{1:04x}'.format(s1, s2)
-                #return '\\u%04x\\u%04x' % (s1, s2)
-    return '"' + str(ESCAPE_ASCII.sub(replace, s)) + '"'
-
-
-encode_basestring_ascii = (
-    c_encode_basestring_ascii or py_encode_basestring_ascii)
+                return '\\u%04x\\u%04x' % (s1, s2)
+    if ESCAPE_ASCII.search(s):
+        return str(ESCAPE_ASCII.sub(replace, s))
+    return s
+py_encode_basestring_ascii = lambda s: '"' + encode_basestring_ascii(s) + '"'
+c_encode_basestring_ascii = None
 
 class JSONEncoder(object):
     """Extensible JSON <http://json.org> encoder for Python data structures.
@@ -147,6 +136,17 @@
 
         self.skipkeys = skipkeys
         self.ensure_ascii = ensure_ascii
+        if ensure_ascii:
+            self.encoder = encode_basestring_ascii
+        else:
+            self.encoder = encode_basestring
+        if encoding != 'utf-8':
+            orig_encoder = self.encoder
+            def encoder(o):
+                if isinstance(o, str):
+                    o = o.decode(encoding)
+                return orig_encoder(o)
+            self.encoder = encoder
         self.check_circular = check_circular
         self.allow_nan = allow_nan
         self.sort_keys = sort_keys
@@ -184,24 +184,126 @@
         '{"foo": ["bar", "baz"]}'
 
         """
-        # This is for extremely simple cases and benchmarks.
+        if self.check_circular:
+            markers = {}
+        else:
+            markers = None
+        if self.ensure_ascii:
+            builder = StringBuilder()
+        else:
+            builder = UnicodeBuilder()
+        self._encode(o, markers, builder, 0)
+        return builder.build()
+
+    def _emit_indent(self, builder, _current_indent_level):
+        if self.indent is not None:
+            _current_indent_level += 1
+            newline_indent = '\n' + (' ' * (self.indent *
+                                            _current_indent_level))
+            separator = self.item_separator + newline_indent
+            builder.append(newline_indent)
+        else:
+            separator = self.item_separator
+        return separator, _current_indent_level
+
+    def _emit_unindent(self, builder, _current_indent_level):
+        if self.indent is not None:
+            builder.append('\n')
+            builder.append(' ' * (self.indent * (_current_indent_level - 1)))
+
+    def _encode(self, o, markers, builder, _current_indent_level):
         if isinstance(o, basestring):
-            if isinstance(o, str):
-                _encoding = self.encoding
-                if (_encoding is not None
-                        and not (_encoding == 'utf-8')):
-                    o = o.decode(_encoding)
-            if self.ensure_ascii:
-                return encode_basestring_ascii(o)
+            builder.append('"')
+            builder.append(self.encoder(o))
+            builder.append('"')
+        elif o is None:
+            builder.append('null')
+        elif o is True:
+            builder.append('true')
+        elif o is False:
+            builder.append('false')
+        elif isinstance(o, (int, long)):
+            builder.append(str(o))
+        elif isinstance(o, float):
+            builder.append(self._floatstr(o))
+        elif isinstance(o, (list, tuple)):
+            if not o:
+                builder.append('[]')
+                return
+            self._encode_list(o, markers, builder, _current_indent_level)
+        elif isinstance(o, dict):
+            if not o:
+                builder.append('{}')
+                return
+            self._encode_dict(o, markers, builder, _current_indent_level)
+        else:
+            self._mark_markers(markers, o)
+            res = self.default(o)
+            self._encode(res, markers, builder, _current_indent_level)
+            self._remove_markers(markers, o)
+            return res
+
+    def _encode_list(self, l, markers, builder, _current_indent_level):
+        self._mark_markers(markers, l)
+        builder.append('[')
+        first = True
+        separator, _current_indent_level = self._emit_indent(builder,
+                                                      _current_indent_level)
+        for elem in l:
+            if first:
+                first = False
             else:
-                return encode_basestring(o)
-        # This doesn't pass the iterator directly to ''.join() because the
-        # exceptions aren't as detailed.  The list call should be roughly
-        # equivalent to the PySequence_Fast that ''.join() would do.
-        chunks = self.iterencode(o, _one_shot=True)
-        if not isinstance(chunks, (list, tuple)):
-            chunks = list(chunks)
-        return ''.join(chunks)
+                builder.append(separator)
+            self._encode(elem, markers, builder, _current_indent_level)
+            del elem # XXX grumble
+        self._emit_unindent(builder, _current_indent_level)
+        builder.append(']')
+        self._remove_markers(markers, l)
+
+    def _encode_dict(self, d, markers, builder, _current_indent_level):
+        self._mark_markers(markers, d)
+        first = True
+        builder.append('{')
+        separator, _current_indent_level = self._emit_indent(builder,
+                                                         _current_indent_level)
+        if self.sort_keys:
+            items = sorted(d.items(), key=lambda kv: kv[0])
+        else:
+            items = d.iteritems()
+
+        for key, v in items:
+            if first:
+                first = False
+            else:
+                builder.append(separator)
+            if isinstance(key, basestring):
+                pass
+            # JavaScript is weakly typed for these, so it makes sense to
+            # also allow them.  Many encoders seem to do something like this.
+            elif isinstance(key, float):
+                key = self._floatstr(key)
+            elif key is True:
+                key = 'true'
+            elif key is False:
+                key = 'false'
+            elif key is None:
+                key = 'null'
+            elif isinstance(key, (int, long)):
+                key = str(key)
+            elif self.skipkeys:
+                continue
+            else:
+                raise TypeError("key " + repr(key) + " is not a string")
+            builder.append('"')
+            builder.append(self.encoder(key))
+            builder.append('"')
+            builder.append(self.key_separator)
+            self._encode(v, markers, builder, _current_indent_level)
+            del key
+            del v # XXX grumble
+        self._emit_unindent(builder, _current_indent_level)
+        builder.append('}')
+        self._remove_markers(markers, d)
 
     def iterencode(self, o, _one_shot=False):
         """Encode the given object and yield each string
@@ -217,86 +319,54 @@
             markers = {}
         else:
             markers = None
-        if self.ensure_ascii:
-            _encoder = encode_basestring_ascii
+        return self._iterencode(o, markers, 0)
+
+    def _floatstr(self, o):
+        # Check for specials.  Note that this type of test is processor
+        # and/or platform-specific, so do tests which don't depend on the
+        # internals.
+
+        if o != o:
+            text = 'NaN'
+        elif o == INFINITY:
+            text = 'Infinity'
+        elif o == -INFINITY:
+            text = '-Infinity'
         else:
-            _encoder = encode_basestring
-        if self.encoding != 'utf-8':
-            def _encoder(o, _orig_encoder=_encoder, _encoding=self.encoding):
-                if isinstance(o, str):
-                    o = o.decode(_encoding)
-                return _orig_encoder(o)
+            return FLOAT_REPR(o)
 
-        def floatstr(o, allow_nan=self.allow_nan,
-                _repr=FLOAT_REPR, _inf=INFINITY, _neginf=-INFINITY):
-            # Check for specials.  Note that this type of test is processor
-            # and/or platform-specific, so do tests which don't depend on the
-            # internals.
+        if not self.allow_nan:
+            raise ValueError(
+                "Out of range float values are not JSON compliant: " +
+                repr(o))
 
-            if o != o:
-                text = 'NaN'
-            elif o == _inf:
-                text = 'Infinity'
-            elif o == _neginf:
-                text = '-Infinity'
-            else:
-                return _repr(o)
+        return text
 
-            if not allow_nan:
-                raise ValueError(
-                    "Out of range float values are not JSON compliant: " +
-                    repr(o))
+    def _mark_markers(self, markers, o):
+        if markers is not None:
+            if id(o) in markers:
+                raise ValueError("Circular reference detected")
+            markers[id(o)] = None
 
-            return text
+    def _remove_markers(self, markers, o):
+        if markers is not None:
+            del markers[id(o)]
 
-
-        if (_one_shot and c_make_encoder is not None
-                and not self.indent and not self.sort_keys):
-            _iterencode = c_make_encoder(
-                markers, self.default, _encoder, self.indent,
-                self.key_separator, self.item_separator, self.sort_keys,
-                self.skipkeys, self.allow_nan)
-        else:
-            _iterencode = _make_iterencode(
-                markers, self.default, _encoder, self.indent, floatstr,
-                self.key_separator, self.item_separator, self.sort_keys,
-                self.skipkeys, _one_shot)
-        return _iterencode(o, 0)
-
-def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,
-        _key_separator, _item_separator, _sort_keys, _skipkeys, _one_shot,
-        ## HACK: hand-optimized bytecode; turn globals into locals
-        ValueError=ValueError,
-        basestring=basestring,
-        dict=dict,
-        float=float,
-        id=id,
-        int=int,
-        isinstance=isinstance,
-        list=list,
-        long=long,
-        str=str,
-        tuple=tuple,
-    ):
-
-    def _iterencode_list(lst, _current_indent_level):
+    def _iterencode_list(self, lst, markers, _current_indent_level):
         if not lst:
             yield '[]'
             return
-        if markers is not None:
-            markerid = id(lst)
-            if markerid in markers:
-                raise ValueError("Circular reference detected")
-            markers[markerid] = lst
+        self._mark_markers(markers, lst)
         buf = '['
-        if _indent is not None:
+        if self.indent is not None:
             _current_indent_level += 1
-            newline_indent = '\n' + (' ' * (_indent * _current_indent_level))
-            separator = _item_separator + newline_indent
+            newline_indent = '\n' + (' ' * (self.indent *
+                                            _current_indent_level))
+            separator = self.item_separator + newline_indent
             buf += newline_indent
         else:
             newline_indent = None
-            separator = _item_separator
+            separator = self.item_separator
         first = True
         for value in lst:
             if first:
@@ -304,7 +374,7 @@
             else:
                 buf = separator
             if isinstance(value, basestring):
-                yield buf + _encoder(value)
+                yield buf + '"' + self.encoder(value) + '"'
             elif value is None:
                 yield buf + 'null'
             elif value is True:
@@ -314,44 +384,43 @@
             elif isinstance(value, (int, long)):
                 yield buf + str(value)
             elif isinstance(value, float):
-                yield buf + _floatstr(value)
+                yield buf + self._floatstr(value)
             else:
                 yield buf
                 if isinstance(value, (list, tuple)):
-                    chunks = _iterencode_list(value, _current_indent_level)
+                    chunks = self._iterencode_list(value, markers,
+                                                   _current_indent_level)
                 elif isinstance(value, dict):
-                    chunks = _iterencode_dict(value, _current_indent_level)
+                    chunks = self._iterencode_dict(value, markers,
+                                                   _current_indent_level)
                 else:
-                    chunks = _iterencode(value, _current_indent_level)
+                    chunks = self._iterencode(value, markers,
+                                              _current_indent_level)
                 for chunk in chunks:
                     yield chunk
         if newline_indent is not None:
             _current_indent_level -= 1
-            yield '\n' + (' ' * (_indent * _current_indent_level))
+            yield '\n' + (' ' * (self.indent * _current_indent_level))
         yield ']'
-        if markers is not None:
-            del markers[markerid]
+        self._remove_markers(markers, lst)
 
-    def _iterencode_dict(dct, _current_indent_level):
+    def _iterencode_dict(self, dct, markers, _current_indent_level):
         if not dct:
             yield '{}'
             return
-        if markers is not None:
-            markerid = id(dct)
-            if markerid in markers:
-                raise ValueError("Circular reference detected")
-            markers[markerid] = dct
+        self._mark_markers(markers, dct)
         yield '{'
-        if _indent is not None:
+        if self.indent is not None:
             _current_indent_level += 1
-            newline_indent = '\n' + (' ' * (_indent * _current_indent_level))
-            item_separator = _item_separator + newline_indent
+            newline_indent = '\n' + (' ' * (self.indent *
+                                            _current_indent_level))
+            item_separator = self.item_separator + newline_indent
             yield newline_indent
         else:
             newline_indent = None
-            item_separator = _item_separator
+            item_separator = self.item_separator
         first = True
-        if _sort_keys:
+        if self.sort_keys:
             items = sorted(dct.items(), key=lambda kv: kv[0])
         else:
             items = dct.iteritems()
@@ -361,7 +430,7 @@
             # JavaScript is weakly typed for these, so it makes sense to
             # also allow them.  Many encoders seem to do something like this.
             elif isinstance(key, float):
-                key = _floatstr(key)
+                key = self._floatstr(key)
             elif key is True:
                 key = 'true'
             elif key is False:
@@ -370,7 +439,7 @@
                 key = 'null'
             elif isinstance(key, (int, long)):
                 key = str(key)
-            elif _skipkeys:
+            elif self.skipkeys:
                 continue
             else:
                 raise TypeError("key " + repr(key) + " is not a string")
@@ -378,10 +447,10 @@
                 first = False
             else:
                 yield item_separator
-            yield _encoder(key)
-            yield _key_separator
+            yield '"' + self.encoder(key) + '"'
+            yield self.key_separator
             if isinstance(value, basestring):
-                yield _encoder(value)
+                yield '"' + self.encoder(value) + '"'
             elif value is None:
                 yield 'null'
             elif value is True:
@@ -391,26 +460,28 @@
             elif isinstance(value, (int, long)):
                 yield str(value)
             elif isinstance(value, float):
-                yield _floatstr(value)
+                yield self._floatstr(value)
             else:
                 if isinstance(value, (list, tuple)):
-                    chunks = _iterencode_list(value, _current_indent_level)
+                    chunks = self._iterencode_list(value, markers,
+                                                   _current_indent_level)
                 elif isinstance(value, dict):
-                    chunks = _iterencode_dict(value, _current_indent_level)
+                    chunks = self._iterencode_dict(value, markers,
+                                                   _current_indent_level)
                 else:
-                    chunks = _iterencode(value, _current_indent_level)
+                    chunks = self._iterencode(value, markers,
+                                              _current_indent_level)
                 for chunk in chunks:
                     yield chunk
         if newline_indent is not None:
             _current_indent_level -= 1
-            yield '\n' + (' ' * (_indent * _current_indent_level))
+            yield '\n' + (' ' * (self.indent * _current_indent_level))
         yield '}'
-        if markers is not None:
-            del markers[markerid]
+        self._remove_markers(markers, dct)
 
-    def _iterencode(o, _current_indent_level):
+    def _iterencode(self, o, markers, _current_indent_level):
         if isinstance(o, basestring):
-            yield _encoder(o)
+            yield '"' + self.encoder(o) + '"'
         elif o is None:
             yield 'null'
         elif o is True:
@@ -420,23 +491,19 @@
         elif isinstance(o, (int, long)):
             yield str(o)
         elif isinstance(o, float):
-            yield _floatstr(o)
+            yield self._floatstr(o)
         elif isinstance(o, (list, tuple)):
-            for chunk in _iterencode_list(o, _current_indent_level):
+            for chunk in self._iterencode_list(o, markers,
+                                               _current_indent_level):
                 yield chunk
         elif isinstance(o, dict):
-            for chunk in _iterencode_dict(o, _current_indent_level):
+            for chunk in self._iterencode_dict(o, markers,
+                                               _current_indent_level):
                 yield chunk
         else:
-            if markers is not None:
-                markerid = id(o)
-                if markerid in markers:
-                    raise ValueError("Circular reference detected")
-                markers[markerid] = o
-            o = _default(o)
-            for chunk in _iterencode(o, _current_indent_level):
+            self._mark_markers(markers, o)
+            obj = self.default(o)
+            for chunk in self._iterencode(obj, markers,
+                                          _current_indent_level):
                 yield chunk
-            if markers is not None:
-                del markers[markerid]
-
-    return _iterencode
+            self._remove_markers(markers, o)
diff --git a/lib-python/modified-2.7/json/tests/test_unicode.py b/lib-python/modified-2.7/json/tests/test_unicode.py
--- a/lib-python/modified-2.7/json/tests/test_unicode.py
+++ b/lib-python/modified-2.7/json/tests/test_unicode.py
@@ -80,3 +80,9 @@
         self.assertEqual(type(json.loads(u'["a"]')[0]), unicode)
         # Issue 10038.
         self.assertEqual(type(json.loads('"foo"')), unicode)
+
+    def test_encode_not_utf_8(self):
+        self.assertEqual(json.dumps('\xb1\xe6', encoding='iso8859-2'),
+                         '"\\u0105\\u0107"')
+        self.assertEqual(json.dumps(['\xb1\xe6'], encoding='iso8859-2'),
+                         '["\\u0105\\u0107"]')
diff --git a/lib-python/modified-2.7/urllib2.py b/lib-python/modified-2.7/urllib2.py
--- a/lib-python/modified-2.7/urllib2.py
+++ b/lib-python/modified-2.7/urllib2.py
@@ -395,11 +395,7 @@
         meth_name = protocol+"_response"
         for processor in self.process_response.get(protocol, []):
             meth = getattr(processor, meth_name)
-            try:
-                response = meth(req, response)
-            except:
-                response.close()
-                raise
+            response = meth(req, response)
 
         return response
 
diff --git a/lib_pypy/pyrepl/readline.py b/lib_pypy/pyrepl/readline.py
--- a/lib_pypy/pyrepl/readline.py
+++ b/lib_pypy/pyrepl/readline.py
@@ -395,9 +395,21 @@
     _wrapper.f_in = f_in
     _wrapper.f_out = f_out
 
-    if hasattr(sys, '__raw_input__'):    # PyPy
-        _old_raw_input = sys.__raw_input__
+    if '__pypy__' in sys.builtin_module_names:    # PyPy
+
+        def _old_raw_input(prompt=''):
+            # sys.__raw_input__() is only called when stdin and stdout are
+            # as expected and are ttys.  If it is the case, then get_reader()
+            # should not really fail in _wrapper.raw_input().  If it still
+            # does, then we will just cancel the redirection and call again
+            # the built-in raw_input().
+            try:
+                del sys.__raw_input__
+            except AttributeError:
+                pass
+            return raw_input(prompt)
         sys.__raw_input__ = _wrapper.raw_input
+
     else:
         # this is not really what readline.c does.  Better than nothing I guess
         import __builtin__
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -72,6 +72,7 @@
     del working_modules['fcntl']  # LOCK_NB not defined
     del working_modules["_minimal_curses"]
     del working_modules["termios"]
+    del working_modules["_multiprocessing"]   # depends on rctime
 
 
 
diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -17,6 +17,12 @@
 projects, or anything else in PyPy, pop up on IRC or write to us on the
 `mailing list`_.
 
+Make big integers faster
+-------------------------
+
+PyPy's implementation of the Python ``long`` type is slower than CPython's.
+Find out why and optimize them.
+
 Numpy improvements
 ------------------
 
diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -2,7 +2,7 @@
 from pypy.interpreter.baseobjspace import Wrappable
 from pypy.interpreter import typedef
 from pypy.interpreter.gateway import interp2app
-from pypy.interpreter.error import OperationError
+from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.tool.pairtype import extendabletype
 from pypy.tool.sourcetools import func_with_new_name
@@ -2925,14 +2925,13 @@
 def Module_get_body(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -2968,14 +2967,13 @@
 def Interactive_get_body(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3015,8 +3013,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     return space.wrap(w_self.body)
 
 def Expression_set_body(space, w_self, w_new_value):
@@ -3057,14 +3054,13 @@
 def Suite_get_body(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3104,8 +3100,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._lineno_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'lineno'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lineno')
     return space.wrap(w_self.lineno)
 
 def stmt_set_lineno(space, w_self, w_new_value):
@@ -3126,8 +3121,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._col_offset_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'col_offset'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'col_offset')
     return space.wrap(w_self.col_offset)
 
 def stmt_set_col_offset(space, w_self, w_new_value):
@@ -3157,8 +3151,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'name'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name')
     return space.wrap(w_self.name)
 
 def FunctionDef_set_name(space, w_self, w_new_value):
@@ -3179,8 +3172,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'args'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args')
     return space.wrap(w_self.args)
 
 def FunctionDef_set_args(space, w_self, w_new_value):
@@ -3197,14 +3189,13 @@
 def FunctionDef_get_body(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3215,14 +3206,13 @@
 def FunctionDef_get_decorator_list(space, w_self):
     if not w_self.initialization_state & 8:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'decorator_list'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'decorator_list')
     if w_self.w_decorator_list is None:
         if w_self.decorator_list is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.decorator_list]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_decorator_list = w_list
     return w_self.w_decorator_list
 
@@ -3266,8 +3256,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'name'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name')
     return space.wrap(w_self.name)
 
 def ClassDef_set_name(space, w_self, w_new_value):
@@ -3284,14 +3273,13 @@
 def ClassDef_get_bases(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'bases'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'bases')
     if w_self.w_bases is None:
         if w_self.bases is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.bases]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_bases = w_list
     return w_self.w_bases
 
@@ -3302,14 +3290,13 @@
 def ClassDef_get_body(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3320,14 +3307,13 @@
 def ClassDef_get_decorator_list(space, w_self):
     if not w_self.initialization_state & 8:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'decorator_list'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'decorator_list')
     if w_self.w_decorator_list is None:
         if w_self.decorator_list is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.decorator_list]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_decorator_list = w_list
     return w_self.w_decorator_list
 
@@ -3372,8 +3358,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Return_set_value(space, w_self, w_new_value):
@@ -3414,14 +3399,13 @@
 def Delete_get_targets(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'targets'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'targets')
     if w_self.w_targets is None:
         if w_self.targets is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.targets]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_targets = w_list
     return w_self.w_targets
 
@@ -3457,14 +3441,13 @@
 def Assign_get_targets(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'targets'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'targets')
     if w_self.w_targets is None:
         if w_self.targets is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.targets]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_targets = w_list
     return w_self.w_targets
 
@@ -3479,8 +3462,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Assign_set_value(space, w_self, w_new_value):
@@ -3527,8 +3509,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'target'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'target')
     return space.wrap(w_self.target)
 
 def AugAssign_set_target(space, w_self, w_new_value):
@@ -3549,8 +3530,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'op'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op')
     return operator_to_class[w_self.op - 1]()
 
 def AugAssign_set_op(space, w_self, w_new_value):
@@ -3573,8 +3553,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def AugAssign_set_value(space, w_self, w_new_value):
@@ -3621,8 +3600,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'dest'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'dest')
     return space.wrap(w_self.dest)
 
 def Print_set_dest(space, w_self, w_new_value):
@@ -3639,14 +3617,13 @@
 def Print_get_values(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'values'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'values')
     if w_self.w_values is None:
         if w_self.values is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.values]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_values = w_list
     return w_self.w_values
 
@@ -3661,8 +3638,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'nl'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'nl')
     return space.wrap(w_self.nl)
 
 def Print_set_nl(space, w_self, w_new_value):
@@ -3710,8 +3686,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'target'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'target')
     return space.wrap(w_self.target)
 
 def For_set_target(space, w_self, w_new_value):
@@ -3732,8 +3707,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'iter'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'iter')
     return space.wrap(w_self.iter)
 
 def For_set_iter(space, w_self, w_new_value):
@@ -3750,14 +3724,13 @@
 def For_get_body(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3768,14 +3741,13 @@
 def For_get_orelse(space, w_self):
     if not w_self.initialization_state & 8:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'orelse'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
     if w_self.w_orelse is None:
         if w_self.orelse is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.orelse]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_orelse = w_list
     return w_self.w_orelse
 
@@ -3819,8 +3791,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'test'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test')
     return space.wrap(w_self.test)
 
 def While_set_test(space, w_self, w_new_value):
@@ -3837,14 +3808,13 @@
 def While_get_body(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3855,14 +3825,13 @@
 def While_get_orelse(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'orelse'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
     if w_self.w_orelse is None:
         if w_self.orelse is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.orelse]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_orelse = w_list
     return w_self.w_orelse
 
@@ -3905,8 +3874,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'test'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test')
     return space.wrap(w_self.test)
 
 def If_set_test(space, w_self, w_new_value):
@@ -3923,14 +3891,13 @@
 def If_get_body(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3941,14 +3908,13 @@
 def If_get_orelse(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'orelse'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
     if w_self.w_orelse is None:
         if w_self.orelse is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.orelse]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_orelse = w_list
     return w_self.w_orelse
 
@@ -3991,8 +3957,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'context_expr'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'context_expr')
     return space.wrap(w_self.context_expr)
 
 def With_set_context_expr(space, w_self, w_new_value):
@@ -4013,8 +3978,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'optional_vars'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'optional_vars')
     return space.wrap(w_self.optional_vars)
 
 def With_set_optional_vars(space, w_self, w_new_value):
@@ -4031,14 +3995,13 @@
 def With_get_body(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -4080,8 +4043,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'type'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'type')
     return space.wrap(w_self.type)
 
 def Raise_set_type(space, w_self, w_new_value):
@@ -4102,8 +4064,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'inst'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'inst')
     return space.wrap(w_self.inst)
 
 def Raise_set_inst(space, w_self, w_new_value):
@@ -4124,8 +4085,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'tback'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'tback')
     return space.wrap(w_self.tback)
 
 def Raise_set_tback(space, w_self, w_new_value):
@@ -4168,14 +4128,13 @@
 def TryExcept_get_body(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -4186,14 +4145,13 @@
 def TryExcept_get_handlers(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'handlers'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'handlers')
     if w_self.w_handlers is None:
         if w_self.handlers is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.handlers]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_handlers = w_list
     return w_self.w_handlers
 
@@ -4204,14 +4162,13 @@
 def TryExcept_get_orelse(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'orelse'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
     if w_self.w_orelse is None:
         if w_self.orelse is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.orelse]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_orelse = w_list
     return w_self.w_orelse
 
@@ -4251,14 +4208,13 @@
 def TryFinally_get_body(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -4269,14 +4225,13 @@
 def TryFinally_get_finalbody(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'finalbody'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'finalbody')
     if w_self.w_finalbody is None:
         if w_self.finalbody is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.finalbody]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_finalbody = w_list
     return w_self.w_finalbody
 
@@ -4318,8 +4273,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'test'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test')
     return space.wrap(w_self.test)
 
 def Assert_set_test(space, w_self, w_new_value):
@@ -4340,8 +4294,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'msg'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'msg')
     return space.wrap(w_self.msg)
 
 def Assert_set_msg(space, w_self, w_new_value):
@@ -4383,14 +4336,13 @@
 def Import_get_names(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'names'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'names')
     if w_self.w_names is None:
         if w_self.names is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.names]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_names = w_list
     return w_self.w_names
 
@@ -4430,8 +4382,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'module'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'module')
     return space.wrap(w_self.module)
 
 def ImportFrom_set_module(space, w_self, w_new_value):
@@ -4451,14 +4402,13 @@
 def ImportFrom_get_names(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'names'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'names')
     if w_self.w_names is None:
         if w_self.names is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.names]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_names = w_list
     return w_self.w_names
 
@@ -4473,8 +4423,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'level'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'level')
     return space.wrap(w_self.level)
 
 def ImportFrom_set_level(space, w_self, w_new_value):
@@ -4522,8 +4471,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     return space.wrap(w_self.body)
 
 def Exec_set_body(space, w_self, w_new_value):
@@ -4544,8 +4492,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'globals'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'globals')
     return space.wrap(w_self.globals)
 
 def Exec_set_globals(space, w_self, w_new_value):
@@ -4566,8 +4513,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'locals'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'locals')
     return space.wrap(w_self.locals)
 
 def Exec_set_locals(space, w_self, w_new_value):
@@ -4610,14 +4556,13 @@
 def Global_get_names(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'names'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'names')
     if w_self.w_names is None:
         if w_self.names is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.names]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_names = w_list
     return w_self.w_names
 
@@ -4657,8 +4602,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Expr_set_value(space, w_self, w_new_value):
@@ -4754,8 +4698,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._lineno_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'lineno'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lineno')
     return space.wrap(w_self.lineno)
 
 def expr_set_lineno(space, w_self, w_new_value):
@@ -4776,8 +4719,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._col_offset_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'col_offset'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'col_offset')
     return space.wrap(w_self.col_offset)
 
 def expr_set_col_offset(space, w_self, w_new_value):
@@ -4807,8 +4749,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'op'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op')
     return boolop_to_class[w_self.op - 1]()
 
 def BoolOp_set_op(space, w_self, w_new_value):
@@ -4827,14 +4768,13 @@
 def BoolOp_get_values(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'values'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'values')
     if w_self.w_values is None:
         if w_self.values is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.values]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_values = w_list
     return w_self.w_values
 
@@ -4875,8 +4815,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'left'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'left')
     return space.wrap(w_self.left)
 
 def BinOp_set_left(space, w_self, w_new_value):
@@ -4897,8 +4836,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'op'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op')
     return operator_to_class[w_self.op - 1]()
 
 def BinOp_set_op(space, w_self, w_new_value):
@@ -4921,8 +4859,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'right'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'right')
     return space.wrap(w_self.right)
 
 def BinOp_set_right(space, w_self, w_new_value):
@@ -4969,8 +4906,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'op'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op')
     return unaryop_to_class[w_self.op - 1]()
 
 def UnaryOp_set_op(space, w_self, w_new_value):
@@ -4993,8 +4929,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'operand'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'operand')
     return space.wrap(w_self.operand)
 
 def UnaryOp_set_operand(space, w_self, w_new_value):
@@ -5040,8 +4975,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'args'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args')
     return space.wrap(w_self.args)
 
 def Lambda_set_args(space, w_self, w_new_value):
@@ -5062,8 +4996,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     return space.wrap(w_self.body)
 
 def Lambda_set_body(space, w_self, w_new_value):
@@ -5109,8 +5042,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'test'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test')
     return space.wrap(w_self.test)
 
 def IfExp_set_test(space, w_self, w_new_value):
@@ -5131,8 +5063,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     return space.wrap(w_self.body)
 
 def IfExp_set_body(space, w_self, w_new_value):
@@ -5153,8 +5084,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'orelse'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
     return space.wrap(w_self.orelse)
 
 def IfExp_set_orelse(space, w_self, w_new_value):
@@ -5197,14 +5127,13 @@
 def Dict_get_keys(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'keys'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'keys')
     if w_self.w_keys is None:
         if w_self.keys is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.keys]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_keys = w_list
     return w_self.w_keys
 
@@ -5215,14 +5144,13 @@
 def Dict_get_values(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'values'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'values')
     if w_self.w_values is None:
         if w_self.values is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.values]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_values = w_list
     return w_self.w_values
 
@@ -5260,14 +5188,13 @@
 def Set_get_elts(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elts'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elts')
     if w_self.w_elts is None:
         if w_self.elts is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.elts]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_elts = w_list
     return w_self.w_elts
 
@@ -5307,8 +5234,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elt'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elt')
     return space.wrap(w_self.elt)
 
 def ListComp_set_elt(space, w_self, w_new_value):
@@ -5325,14 +5251,13 @@
 def ListComp_get_generators(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'generators'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators')
     if w_self.w_generators is None:
         if w_self.generators is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.generators]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_generators = w_list
     return w_self.w_generators
 
@@ -5373,8 +5298,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elt'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elt')
     return space.wrap(w_self.elt)
 
 def SetComp_set_elt(space, w_self, w_new_value):
@@ -5391,14 +5315,13 @@
 def SetComp_get_generators(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'generators'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators')
     if w_self.w_generators is None:
         if w_self.generators is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.generators]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_generators = w_list
     return w_self.w_generators
 
@@ -5439,8 +5362,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'key'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'key')
     return space.wrap(w_self.key)
 
 def DictComp_set_key(space, w_self, w_new_value):
@@ -5461,8 +5383,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def DictComp_set_value(space, w_self, w_new_value):
@@ -5479,14 +5400,13 @@
 def DictComp_get_generators(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'generators'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators')
     if w_self.w_generators is None:
         if w_self.generators is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.generators]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_generators = w_list
     return w_self.w_generators
 
@@ -5528,8 +5448,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elt'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elt')
     return space.wrap(w_self.elt)
 
 def GeneratorExp_set_elt(space, w_self, w_new_value):
@@ -5546,14 +5465,13 @@
 def GeneratorExp_get_generators(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'generators'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators')
     if w_self.w_generators is None:
         if w_self.generators is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.generators]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_generators = w_list
     return w_self.w_generators
 
@@ -5594,8 +5512,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Yield_set_value(space, w_self, w_new_value):
@@ -5640,8 +5557,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'left'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'left')
     return space.wrap(w_self.left)
 
 def Compare_set_left(space, w_self, w_new_value):
@@ -5658,14 +5574,13 @@
 def Compare_get_ops(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ops'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ops')
     if w_self.w_ops is None:
         if w_self.ops is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [cmpop_to_class[node - 1]() for node in w_self.ops]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_ops = w_list
     return w_self.w_ops
 
@@ -5676,14 +5591,13 @@
 def Compare_get_comparators(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'comparators'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'comparators')
     if w_self.w_comparators is None:
         if w_self.comparators is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.comparators]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_comparators = w_list
     return w_self.w_comparators
 
@@ -5726,8 +5640,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'func'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'func')
     return space.wrap(w_self.func)
 
 def Call_set_func(space, w_self, w_new_value):
@@ -5744,14 +5657,13 @@
 def Call_get_args(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'args'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args')
     if w_self.w_args is None:
         if w_self.args is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.args]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_args = w_list
     return w_self.w_args
 
@@ -5762,14 +5674,13 @@
 def Call_get_keywords(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'keywords'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'keywords')
     if w_self.w_keywords is None:
         if w_self.keywords is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.keywords]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_keywords = w_list
     return w_self.w_keywords
 
@@ -5784,8 +5695,7 @@
             return w_obj
     if not w_self.initialization_state & 8:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'starargs'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'starargs')
     return space.wrap(w_self.starargs)
 
 def Call_set_starargs(space, w_self, w_new_value):
@@ -5806,8 +5716,7 @@
             return w_obj
     if not w_self.initialization_state & 16:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'kwargs'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'kwargs')
     return space.wrap(w_self.kwargs)
 
 def Call_set_kwargs(space, w_self, w_new_value):
@@ -5858,8 +5767,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Repr_set_value(space, w_self, w_new_value):
@@ -5904,8 +5812,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'n'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'n')
     return w_self.n
 
 def Num_set_n(space, w_self, w_new_value):
@@ -5950,8 +5857,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 's'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 's')
     return w_self.s
 
 def Str_set_s(space, w_self, w_new_value):
@@ -5996,8 +5902,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Attribute_set_value(space, w_self, w_new_value):
@@ -6018,8 +5923,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'attr'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'attr')
     return space.wrap(w_self.attr)
 
 def Attribute_set_attr(space, w_self, w_new_value):
@@ -6040,8 +5944,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ctx'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def Attribute_set_ctx(space, w_self, w_new_value):
@@ -6090,8 +5993,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Subscript_set_value(space, w_self, w_new_value):
@@ -6112,8 +6014,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'slice'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'slice')
     return space.wrap(w_self.slice)
 
 def Subscript_set_slice(space, w_self, w_new_value):
@@ -6134,8 +6035,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ctx'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def Subscript_set_ctx(space, w_self, w_new_value):
@@ -6184,8 +6084,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'id'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'id')
     return space.wrap(w_self.id)
 
 def Name_set_id(space, w_self, w_new_value):
@@ -6206,8 +6105,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ctx'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def Name_set_ctx(space, w_self, w_new_value):
@@ -6251,14 +6149,13 @@
 def List_get_elts(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elts'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elts')
     if w_self.w_elts is None:
         if w_self.elts is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.elts]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_elts = w_list
     return w_self.w_elts
 
@@ -6273,8 +6170,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ctx'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def List_set_ctx(space, w_self, w_new_value):
@@ -6319,14 +6215,13 @@
 def Tuple_get_elts(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elts'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elts')
     if w_self.w_elts is None:
         if w_self.elts is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.elts]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_elts = w_list
     return w_self.w_elts
 
@@ -6341,8 +6236,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ctx'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def Tuple_set_ctx(space, w_self, w_new_value):
@@ -6391,8 +6285,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return w_self.value
 
 def Const_set_value(space, w_self, w_new_value):
@@ -6510,8 +6403,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'lower'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lower')
     return space.wrap(w_self.lower)
 
 def Slice_set_lower(space, w_self, w_new_value):
@@ -6532,8 +6424,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'upper'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'upper')
     return space.wrap(w_self.upper)
 
 def Slice_set_upper(space, w_self, w_new_value):
@@ -6554,8 +6445,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'step'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'step')
     return space.wrap(w_self.step)
 
 def Slice_set_step(space, w_self, w_new_value):
@@ -6598,14 +6488,13 @@
 def ExtSlice_get_dims(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'dims'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'dims')
     if w_self.w_dims is None:
         if w_self.dims is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.dims]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_dims = w_list
     return w_self.w_dims
 
@@ -6645,8 +6534,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Index_set_value(space, w_self, w_new_value):
@@ -6915,8 +6803,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'target'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'target')
     return space.wrap(w_self.target)
 
 def comprehension_set_target(space, w_self, w_new_value):
@@ -6937,8 +6824,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'iter'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'iter')
     return space.wrap(w_self.iter)
 
 def comprehension_set_iter(space, w_self, w_new_value):
@@ -6955,14 +6841,13 @@
 def comprehension_get_ifs(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ifs'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ifs')
     if w_self.w_ifs is None:
         if w_self.ifs is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.ifs]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_ifs = w_list
     return w_self.w_ifs
 
@@ -7004,8 +6889,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._lineno_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'lineno'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lineno')
     return space.wrap(w_self.lineno)
 
 def excepthandler_set_lineno(space, w_self, w_new_value):
@@ -7026,8 +6910,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._col_offset_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'col_offset'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'col_offset')
     return space.wrap(w_self.col_offset)
 
 def excepthandler_set_col_offset(space, w_self, w_new_value):
@@ -7057,8 +6940,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'type'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'type')
     return space.wrap(w_self.type)
 
 def ExceptHandler_set_type(space, w_self, w_new_value):
@@ -7079,8 +6961,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'name'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name')
     return space.wrap(w_self.name)
 
 def ExceptHandler_set_name(space, w_self, w_new_value):
@@ -7097,14 +6978,13 @@
 def ExceptHandler_get_body(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -7142,14 +7022,13 @@
 def arguments_get_args(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'args'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args')
     if w_self.w_args is None:
         if w_self.args is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.args]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_args = w_list
     return w_self.w_args
 
@@ -7164,8 +7043,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'vararg'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'vararg')
     return space.wrap(w_self.vararg)
 
 def arguments_set_vararg(space, w_self, w_new_value):
@@ -7189,8 +7067,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'kwarg'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'kwarg')
     return space.wrap(w_self.kwarg)
 
 def arguments_set_kwarg(space, w_self, w_new_value):
@@ -7210,14 +7087,13 @@
 def arguments_get_defaults(space, w_self):
     if not w_self.initialization_state & 8:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'defaults'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'defaults')
     if w_self.w_defaults is None:
         if w_self.defaults is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.defaults]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_defaults = w_list
     return w_self.w_defaults
 
@@ -7261,8 +7137,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'arg'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'arg')
     return space.wrap(w_self.arg)
 
 def keyword_set_arg(space, w_self, w_new_value):
@@ -7283,8 +7158,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def keyword_set_value(space, w_self, w_new_value):
@@ -7330,8 +7204,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'name'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name')
     return space.wrap(w_self.name)
 
 def alias_set_name(space, w_self, w_new_value):
@@ -7352,8 +7225,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'asname'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'asname')
     return space.wrap(w_self.asname)
 
 def alias_set_asname(space, w_self, w_new_value):
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -414,13 +414,12 @@
             self.emit("        return w_obj", 1)
         self.emit("if not w_self.initialization_state & %s:" % (flag,), 1)
         self.emit("typename = space.type(w_self).getname(space)", 2)
-        self.emit("w_err = space.wrap(\"'%%s' object has no attribute '%s'\" %% typename)" %
+        self.emit("raise operationerrfmt(space.w_AttributeError, \"'%%s' object has no attribute '%%s'\", typename, '%s')" %
                   (field.name,), 2)
-        self.emit("raise OperationError(space.w_AttributeError, w_err)", 2)
         if field.seq:
             self.emit("if w_self.w_%s is None:" % (field.name,), 1)
             self.emit("if w_self.%s is None:" % (field.name,), 2)
-            self.emit("w_list = space.newlist([])", 3)
+            self.emit("list_w = []", 3)
             self.emit("else:", 2)
             if field.type.value in self.data.simple_types:
                 wrapper = "%s_to_class[node - 1]()" % (field.type,)
@@ -428,7 +427,7 @@
                 wrapper = "space.wrap(node)"
             self.emit("list_w = [%s for node in w_self.%s]" %
                       (wrapper, field.name), 3)
-            self.emit("w_list = space.newlist(list_w)", 3)
+            self.emit("w_list = space.newlist(list_w)", 2)
             self.emit("w_self.w_%s = w_list" % (field.name,), 2)
             self.emit("return w_self.w_%s" % (field.name,), 1)
         elif field.type.value in self.data.simple_types:
@@ -540,7 +539,7 @@
 from pypy.interpreter.baseobjspace import Wrappable
 from pypy.interpreter import typedef
 from pypy.interpreter.gateway import interp2app
-from pypy.interpreter.error import OperationError
+from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.tool.pairtype import extendabletype
 from pypy.tool.sourcetools import func_with_new_name
@@ -639,9 +638,7 @@
             missing = required[i]
             if missing is not None:
                  err = "required field \\"%s\\" missing from %s"
-                 err = err % (missing, host)
-                 w_err = space.wrap(err)
-                 raise OperationError(space.w_TypeError, w_err)
+                 raise operationerrfmt(space.w_TypeError, err, missing, host)
     raise AssertionError("should not reach here")
 
 
diff --git a/pypy/jit/backend/llgraph/llimpl.py b/pypy/jit/backend/llgraph/llimpl.py
--- a/pypy/jit/backend/llgraph/llimpl.py
+++ b/pypy/jit/backend/llgraph/llimpl.py
@@ -7,9 +7,7 @@
 import weakref
 from pypy.objspace.flow.model import Variable, Constant
 from pypy.annotation import model as annmodel
-from pypy.jit.metainterp.history import (ConstInt, ConstPtr,
-                                         BoxInt, BoxPtr, BoxObj, BoxFloat,
-                                         REF, INT, FLOAT)
+from pypy.jit.metainterp.history import REF, INT, FLOAT
 from pypy.jit.codewriter import heaptracker
 from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr, rffi
 from pypy.rpython.ootypesystem import ootype
@@ -17,7 +15,7 @@
 from pypy.rpython.llinterp import LLException
 from pypy.rpython.extregistry import ExtRegistryEntry
 
-from pypy.jit.metainterp import resoperation, executor
+from pypy.jit.metainterp import resoperation
 from pypy.jit.metainterp.resoperation import rop
 from pypy.jit.backend.llgraph import symbolic
 from pypy.jit.codewriter import longlong
@@ -334,6 +332,13 @@
     assert isinstance(type, str) and len(type) == 1
     op.descr = Descr(ofs, type, arg_types=arg_types)
 
+def compile_add_descr_arg(loop, ofs, type, arg_types):
+    from pypy.jit.backend.llgraph.runner import Descr
+    loop = _from_opaque(loop)
+    op = loop.operations[-1]
+    assert isinstance(type, str) and len(type) == 1
+    op.args.append(Descr(ofs, type, arg_types=arg_types))
+
 def compile_add_loop_token(loop, descr):
     if we_are_translated():
         raise ValueError("CALL_ASSEMBLER not supported")
@@ -438,8 +443,11 @@
         self._may_force = -1
 
     def getenv(self, v):
+        from pypy.jit.backend.llgraph.runner import Descr
         if isinstance(v, Constant):
             return v.value
+        elif isinstance(v, Descr):
+            return v
         else:
             return self.env[v]
 
@@ -807,6 +815,29 @@
         else:
             raise NotImplementedError
 
+    def op_getinteriorfield_gc(self, descr, array, index):
+        if descr.typeinfo == REF:
+            return do_getinteriorfield_gc_ptr(array, index, descr.ofs)
+        elif descr.typeinfo == INT:
+            return do_getinteriorfield_gc_int(array, index, descr.ofs)
+        elif descr.typeinfo == FLOAT:
+            return do_getinteriorfield_gc_float(array, index, descr.ofs)
+        else:
+            raise NotImplementedError
+
+    def op_setinteriorfield_gc(self, descr, array, index, newvalue):
+        if descr.typeinfo == REF:
+            return do_setinteriorfield_gc_ptr(array, index, descr.ofs,
+                                              newvalue)
+        elif descr.typeinfo == INT:
+            return do_setinteriorfield_gc_int(array, index, descr.ofs,
+                                              newvalue)
+        elif descr.typeinfo == FLOAT:
+            return do_setinteriorfield_gc_float(array, index, descr.ofs,
+                                                newvalue)
+        else:
+            raise NotImplementedError
+
     def op_setfield_gc(self, fielddescr, struct, newvalue):
         if fielddescr.typeinfo == REF:
             do_setfield_gc_ptr(struct, fielddescr.ofs, newvalue)
@@ -1354,6 +1385,22 @@
 def do_getfield_gc_ptr(struct, fieldnum):
     return cast_to_ptr(_getfield_gc(struct, fieldnum))
 
+def _getinteriorfield_gc(struct, fieldnum):
+    STRUCT, fieldname = symbolic.TokenToField[fieldnum]
+    return getattr(struct, fieldname)
+
+def do_getinteriorfield_gc_int(array, index, fieldnum):
+    struct = array._obj.container.getitem(index)
+    return cast_to_int(_getinteriorfield_gc(struct, fieldnum))
+
+def do_getinteriorfield_gc_float(array, index, fieldnum):
+    struct = array._obj.container.getitem(index)
+    return cast_to_floatstorage(_getinteriorfield_gc(struct, fieldnum))
+
+def do_getinteriorfield_gc_ptr(array, index, fieldnum):
+    struct = array._obj.container.getitem(index)
+    return cast_to_ptr(_getinteriorfield_gc(struct, fieldnum))
+
 def _getfield_raw(struct, fieldnum):
     STRUCT, fieldname = symbolic.TokenToField[fieldnum]
     ptr = cast_from_int(lltype.Ptr(STRUCT), struct)
@@ -1409,26 +1456,28 @@
     newvalue = cast_from_ptr(ITEMTYPE, newvalue)
     array.setitem(index, newvalue)
 
-def do_setfield_gc_int(struct, fieldnum, newvalue):
-    STRUCT, fieldname = symbolic.TokenToField[fieldnum]
-    ptr = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), struct)
-    FIELDTYPE = getattr(STRUCT, fieldname)
-    newvalue = cast_from_int(FIELDTYPE, newvalue)
-    setattr(ptr, fieldname, newvalue)
+def new_setfield_gc(cast_func):
+    def do_setfield_gc(struct, fieldnum, newvalue):
+        STRUCT, fieldname = symbolic.TokenToField[fieldnum]
+        ptr = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), struct)
+        FIELDTYPE = getattr(STRUCT, fieldname)
+        newvalue = cast_func(FIELDTYPE, newvalue)
+        setattr(ptr, fieldname, newvalue)
+    return do_setfield_gc
+do_setfield_gc_int = new_setfield_gc(cast_from_int)
+do_setfield_gc_float = new_setfield_gc(cast_from_floatstorage)
+do_setfield_gc_ptr = new_setfield_gc(cast_from_ptr)
 
-def do_setfield_gc_float(struct, fieldnum, newvalue):
-    STRUCT, fieldname = symbolic.TokenToField[fieldnum]
-    ptr = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), struct)
-    FIELDTYPE = getattr(STRUCT, fieldname)
-    newvalue = cast_from_floatstorage(FIELDTYPE, newvalue)
-    setattr(ptr, fieldname, newvalue)
-
-def do_setfield_gc_ptr(struct, fieldnum, newvalue):
-    STRUCT, fieldname = symbolic.TokenToField[fieldnum]
-    ptr = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), struct)
-    FIELDTYPE = getattr(STRUCT, fieldname)
-    newvalue = cast_from_ptr(FIELDTYPE, newvalue)
-    setattr(ptr, fieldname, newvalue)
+def new_setinteriorfield_gc(cast_func):
+    def do_setinteriorfield_gc(array, index, fieldnum, newvalue):
+        STRUCT, fieldname = symbolic.TokenToField[fieldnum]
+        struct = array._obj.container.getitem(index)
+        FIELDTYPE = getattr(STRUCT, fieldname)
+        setattr(struct, fieldname, cast_func(FIELDTYPE, newvalue))
+    return do_setinteriorfield_gc
+do_setinteriorfield_gc_int = new_setinteriorfield_gc(cast_from_int)
+do_setinteriorfield_gc_float = new_setinteriorfield_gc(cast_from_floatstorage)
+do_setinteriorfield_gc_ptr = new_setinteriorfield_gc(cast_from_ptr)        
 
 def do_setfield_raw_int(struct, fieldnum, newvalue):
     STRUCT, fieldname = symbolic.TokenToField[fieldnum]
@@ -1694,6 +1743,7 @@
 setannotation(compile_start_float_var, annmodel.SomeInteger())
 setannotation(compile_add, annmodel.s_None)
 setannotation(compile_add_descr, annmodel.s_None)
+setannotation(compile_add_descr_arg, annmodel.s_None)
 setannotation(compile_add_var, annmodel.s_None)
 setannotation(compile_add_int_const, annmodel.s_None)
 setannotation(compile_add_ref_const, annmodel.s_None)
@@ -1741,6 +1791,9 @@
 setannotation(do_getfield_raw_int, annmodel.SomeInteger())
 setannotation(do_getfield_raw_ptr, annmodel.SomePtr(llmemory.GCREF))
 setannotation(do_getfield_raw_float, s_FloatStorage)
+setannotation(do_getinteriorfield_gc_int, annmodel.SomeInteger())
+setannotation(do_getinteriorfield_gc_ptr, annmodel.SomePtr(llmemory.GCREF))
+setannotation(do_getinteriorfield_gc_float, s_FloatStorage)
 setannotation(do_new, annmodel.SomePtr(llmemory.GCREF))
 setannotation(do_new_array, annmodel.SomePtr(llmemory.GCREF))
 setannotation(do_setarrayitem_gc_int, annmodel.s_None)
@@ -1754,6 +1807,9 @@
 setannotation(do_setfield_raw_int, annmodel.s_None)
 setannotation(do_setfield_raw_ptr, annmodel.s_None)
 setannotation(do_setfield_raw_float, annmodel.s_None)
+setannotation(do_setinteriorfield_gc_int, annmodel.s_None)
+setannotation(do_setinteriorfield_gc_ptr, annmodel.s_None)
+setannotation(do_setinteriorfield_gc_float, annmodel.s_None)
 setannotation(do_newstr, annmodel.SomePtr(llmemory.GCREF))
 setannotation(do_strsetitem, annmodel.s_None)
 setannotation(do_newunicode, annmodel.SomePtr(llmemory.GCREF))
diff --git a/pypy/jit/backend/llgraph/runner.py b/pypy/jit/backend/llgraph/runner.py
--- a/pypy/jit/backend/llgraph/runner.py
+++ b/pypy/jit/backend/llgraph/runner.py
@@ -2,21 +2,19 @@
 Minimal-API wrapper around the llinterpreter to run operations.
 """
 
-import sys
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.rpython.lltypesystem import lltype, llmemory, rclass
 from pypy.rpython.ootypesystem import ootype
 from pypy.rpython.llinterp import LLInterpreter
 from pypy.jit.metainterp import history
-from pypy.jit.metainterp.history import REF, INT, FLOAT
+from pypy.jit.metainterp.history import REF, INT, FLOAT, STRUCT
 from pypy.jit.metainterp.warmstate import unwrap
-from pypy.jit.metainterp.resoperation import ResOperation, rop
+from pypy.jit.metainterp.resoperation import rop
 from pypy.jit.backend import model
 from pypy.jit.backend.llgraph import llimpl, symbolic
 from pypy.jit.metainterp.typesystem import llhelper, oohelper
 from pypy.jit.codewriter import heaptracker, longlong
-from pypy.rlib import rgc
 
 class MiniStats:
     pass
@@ -62,6 +60,9 @@
     def is_array_of_floats(self):
         return self.typeinfo == FLOAT
 
+    def is_array_of_structs(self):
+        return self.typeinfo == STRUCT
+
     def as_vtable_size_descr(self):
         return self
 
@@ -177,8 +178,10 @@
             llimpl.compile_add(c, op.getopnum())
             descr = op.getdescr()
             if isinstance(descr, Descr):
-                llimpl.compile_add_descr(c, descr.ofs, descr.typeinfo, descr.arg_types)
-            if isinstance(descr, history.LoopToken) and op.getopnum() != rop.JUMP:
+                llimpl.compile_add_descr(c, descr.ofs, descr.typeinfo,
+                                         descr.arg_types)
+            if (isinstance(descr, history.LoopToken) and
+                op.getopnum() != rop.JUMP):
                 llimpl.compile_add_loop_token(c, descr)
             if self.is_oo and isinstance(descr, (OODescr, MethDescr)):
                 # hack hack, not rpython
@@ -193,6 +196,9 @@
                     llimpl.compile_add_ref_const(c, x.value, self.ts.BASETYPE)
                 elif isinstance(x, history.ConstFloat):
                     llimpl.compile_add_float_const(c, x.value)
+                elif isinstance(x, Descr):
+                    llimpl.compile_add_descr_arg(c, x.ofs, x.typeinfo,
+                                                 x.arg_types)
                 else:
                     raise Exception("'%s' args contain: %r" % (op.getopname(),
                                                                x))
@@ -316,6 +322,13 @@
         token = history.getkind(getattr(S, fieldname))
         return self.getdescr(ofs, token[0], name=fieldname)
 
+    def interiorfielddescrof(self, A, fieldname):
+        S = A.OF
+        ofs2 = symbolic.get_size(A)
+        ofs, size = symbolic.get_field_token(S, fieldname)
+        token = history.getkind(getattr(S, fieldname))
+        return self.getdescr(ofs, token[0], name=fieldname, extrainfo=ofs2)
+
     def calldescrof(self, FUNC, ARGS, RESULT, extrainfo):
         arg_types = []
         for ARG in ARGS:
@@ -353,8 +366,13 @@
     def arraydescrof(self, A):
         assert A.OF != lltype.Void
         size = symbolic.get_size(A)
-        token = history.getkind(A.OF)
-        return self.getdescr(size, token[0])
+        if isinstance(A.OF, lltype.Ptr) or isinstance(A.OF, lltype.Primitive):
+            token = history.getkind(A.OF)[0]
+        elif isinstance(A.OF, lltype.Struct):
+            token = 's'
+        else:
+            token = '?'
+        return self.getdescr(size, token)
 
     # ---------- the backend-dependent operations ----------
 
@@ -406,6 +424,29 @@
         assert isinstance(fielddescr, Descr)
         return llimpl.do_getfield_raw_float(struct, fielddescr.ofs)
 
+    def bh_getinteriorfield_gc_i(self, array, index, descr):
+        assert isinstance(descr, Descr)
+        return llimpl.do_getinteriorfield_gc_int(array, index, descr.ofs)
+    def bh_getinteriorfield_gc_r(self, array, index, descr):
+        assert isinstance(descr, Descr)
+        return llimpl.do_getinteriorfield_gc_ptr(array, index, descr.ofs)
+    def bh_getinteriorfield_gc_f(self, array, index, descr):
+        assert isinstance(descr, Descr)
+        return llimpl.do_getinteriorfield_gc_float(array, index, descr.ofs)
+
+    def bh_setinteriorfield_gc_i(self, array, index, descr, value):
+        assert isinstance(descr, Descr)
+        return llimpl.do_setinteriorfield_gc_int(array, index, descr.ofs,
+                                                 value)
+    def bh_setinteriorfield_gc_r(self, array, index, descr, value):
+        assert isinstance(descr, Descr)
+        return llimpl.do_setinteriorfield_gc_ptr(array, index, descr.ofs,
+                                                 value)
+    def bh_setinteriorfield_gc_f(self, array, index, descr, value):
+        assert isinstance(descr, Descr)
+        return llimpl.do_setinteriorfield_gc_float(array, index, descr.ofs,
+                                                   value)
+
     def bh_new(self, sizedescr):
         assert isinstance(sizedescr, Descr)
         return llimpl.do_new(sizedescr.ofs)
@@ -418,7 +459,6 @@
 
     def bh_classof(self, struct):
         struct = lltype.cast_opaque_ptr(rclass.OBJECTPTR, struct)
-        result = struct.typeptr
         result_adr = llmemory.cast_ptr_to_adr(struct.typeptr)
         return heaptracker.adr2int(result_adr)
 
diff --git a/pypy/jit/backend/llsupport/descr.py b/pypy/jit/backend/llsupport/descr.py
--- a/pypy/jit/backend/llsupport/descr.py
+++ b/pypy/jit/backend/llsupport/descr.py
@@ -1,13 +1,10 @@
 import py
-from pypy.rpython.lltypesystem import lltype, rffi, llmemory, rclass
+from pypy.rpython.lltypesystem import lltype, rffi, llmemory
 from pypy.rpython.lltypesystem.lloperation import llop
 from pypy.jit.backend.llsupport import symbolic, support
-from pypy.jit.metainterp.history import AbstractDescr, getkind, BoxInt, BoxPtr
-from pypy.jit.metainterp.history import BasicFailDescr, LoopToken, BoxFloat
+from pypy.jit.metainterp.history import AbstractDescr, getkind
 from pypy.jit.metainterp import history
-from pypy.jit.metainterp.resoperation import ResOperation, rop
 from pypy.jit.codewriter import heaptracker, longlong
-from pypy.rlib.rarithmetic import r_longlong, r_ulonglong
 
 # The point of the class organization in this file is to make instances
 # as compact as possible.  This is done by not storing the field size or
@@ -23,6 +20,7 @@
         self._cache_field = {}
         self._cache_array = {}
         self._cache_call = {}
+        self._cache_interiorfield = {}
 
     def init_size_descr(self, STRUCT, sizedescr):
         assert isinstance(STRUCT, lltype.GcStruct)
@@ -142,7 +140,6 @@
         cachedict[fieldname] = fielddescr
         return fielddescr
 
-
 # ____________________________________________________________
 # ArrayDescrs
 
@@ -167,6 +164,7 @@
 
     _is_array_of_pointers = False      # unless overridden by GcPtrArrayDescr
     _is_array_of_floats   = False      # unless overridden by FloatArrayDescr
+    _is_array_of_structs  = False      # unless overridden by StructArrayDescr
     _is_item_signed       = False      # unless overridden by XxxArrayDescr
 
     def is_array_of_pointers(self):
@@ -175,6 +173,9 @@
     def is_array_of_floats(self):
         return self._is_array_of_floats
 
+    def is_array_of_structs(self):
+        return self._is_array_of_structs
+
     def is_item_signed(self):
         return self._is_item_signed
 
@@ -199,6 +200,10 @@
     def get_item_size(self, translate_support_code):
         return symbolic.get_size(lltype.Float, translate_support_code)
 
+class StructArrayDescr(BaseArrayDescr):
+    _clsname = 'StructArrayDescr'
+    _is_array_of_structs = True
+
 class BaseArrayNoLengthDescr(BaseArrayDescr):
     def get_base_size(self, translate_support_code):
         return 0
@@ -218,6 +223,13 @@
 def getArrayDescrClass(ARRAY):
     if ARRAY.OF is lltype.Float:
         return FloatArrayDescr
+    elif isinstance(ARRAY.OF, lltype.Struct):
+        class Descr(StructArrayDescr):
+            _clsname = '%sArrayDescr' % ARRAY.OF._name
+            def get_item_size(self, translate_support_code):
+                return symbolic.get_size(ARRAY.OF, translate_support_code)
+        Descr.__name__ = Descr._clsname
+        return Descr
     return getDescrClass(ARRAY.OF, BaseArrayDescr, GcPtrArrayDescr,
                          NonGcPtrArrayDescr, 'Array', 'get_item_size',
                          '_is_array_of_floats', '_is_item_signed')
@@ -252,6 +264,36 @@
         cache[ARRAY] = arraydescr
         return arraydescr
 
+# ____________________________________________________________
+# InteriorFieldDescr
+
+class InteriorFieldDescr(AbstractDescr):
+    arraydescr = BaseArrayDescr()     # workaround for the annotator
+    fielddescr = BaseFieldDescr('', 0)
+
+    def __init__(self, arraydescr, fielddescr):
+        self.arraydescr = arraydescr
+        self.fielddescr = fielddescr
+
+    def is_pointer_field(self):
+        return self.fielddescr.is_pointer_field()
+
+    def is_float_field(self):
+        return self.fielddescr.is_float_field()
+
+    def repr_of_descr(self):
+        return '<InteriorFieldDescr %s>' % self.fielddescr.repr_of_descr()
+
+def get_interiorfield_descr(gc_ll_descr, ARRAY, FIELDTP, name):
+    cache = gc_ll_descr._cache_interiorfield
+    try:
+        return cache[(ARRAY, FIELDTP, name)]
+    except KeyError:
+        arraydescr = get_array_descr(gc_ll_descr, ARRAY)
+        fielddescr = get_field_descr(gc_ll_descr, FIELDTP, name)
+        descr = InteriorFieldDescr(arraydescr, fielddescr)
+        cache[(ARRAY, FIELDTP, name)] = descr
+        return descr
 
 # ____________________________________________________________
 # CallDescrs
@@ -525,7 +567,8 @@
         #
         if TYPE is lltype.Float or is_longlong(TYPE):
             setattr(Descr, floatattrname, True)
-        elif TYPE is not lltype.Bool and rffi.cast(TYPE, -1) == -1:
+        elif (TYPE is not lltype.Bool and isinstance(TYPE, lltype.Number) and
+              rffi.cast(TYPE, -1) == -1):
             setattr(Descr, signedattrname, True)
         #
         _cache[nameprefix, TYPE] = Descr
diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py
--- a/pypy/jit/backend/llsupport/gc.py
+++ b/pypy/jit/backend/llsupport/gc.py
@@ -45,6 +45,14 @@
     def freeing_block(self, start, stop):
         pass
 
+    def get_funcptr_for_newarray(self):
+        return llhelper(self.GC_MALLOC_ARRAY, self.malloc_array)
+    def get_funcptr_for_newstr(self):
+        return llhelper(self.GC_MALLOC_STR_UNICODE, self.malloc_str)
+    def get_funcptr_for_newunicode(self):
+        return llhelper(self.GC_MALLOC_STR_UNICODE, self.malloc_unicode)
+
+
     def record_constptrs(self, op, gcrefs_output_list):
         for i in range(op.numargs()):
             v = op.getarg(i)
@@ -96,6 +104,39 @@
         malloc_fn_ptr = self.configure_boehm_once()
         self.funcptr_for_new = malloc_fn_ptr
 
+        def malloc_array(basesize, itemsize, ofs_length, num_elem):
+            try:
+                size = ovfcheck(basesize + ovfcheck(itemsize * num_elem))
+            except OverflowError:
+                return lltype.nullptr(llmemory.GCREF.TO)
+            res = self.funcptr_for_new(size)
+            if not res:
+                return res
+            rffi.cast(rffi.CArrayPtr(lltype.Signed), res)[ofs_length/WORD] = num_elem
+            return res
+        self.malloc_array = malloc_array
+        self.GC_MALLOC_ARRAY = lltype.Ptr(lltype.FuncType(
+            [lltype.Signed] * 4, llmemory.GCREF))
+
+
+        (str_basesize, str_itemsize, str_ofs_length
+         ) = symbolic.get_array_token(rstr.STR, self.translate_support_code)
+        (unicode_basesize, unicode_itemsize, unicode_ofs_length
+         ) = symbolic.get_array_token(rstr.UNICODE, self.translate_support_code)
+        def malloc_str(length):
+            return self.malloc_array(
+                str_basesize, str_itemsize, str_ofs_length, length
+            )
+        def malloc_unicode(length):
+            return self.malloc_array(
+                unicode_basesize, unicode_itemsize, unicode_ofs_length, length
+            )
+        self.malloc_str = malloc_str
+        self.malloc_unicode = malloc_unicode
+        self.GC_MALLOC_STR_UNICODE = lltype.Ptr(lltype.FuncType(
+            [lltype.Signed], llmemory.GCREF))
+
+
         # on some platform GC_init is required before any other
         # GC_* functions, call it here for the benefit of tests
         # XXX move this to tests
@@ -116,39 +157,27 @@
         ofs_length = arraydescr.get_ofs_length(self.translate_support_code)
         basesize = arraydescr.get_base_size(self.translate_support_code)
         itemsize = arraydescr.get_item_size(self.translate_support_code)
-        size = basesize + itemsize * num_elem
-        res = self.funcptr_for_new(size)
-        rffi.cast(rffi.CArrayPtr(lltype.Signed), res)[ofs_length/WORD] = num_elem
-        return res
+        return self.malloc_array(basesize, itemsize, ofs_length, num_elem)
 
     def gc_malloc_str(self, num_elem):
-        basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR,
-                                                   self.translate_support_code)
-        assert itemsize == 1
-        size = basesize + num_elem
-        res = self.funcptr_for_new(size)
-        rffi.cast(rffi.CArrayPtr(lltype.Signed), res)[ofs_length/WORD] = num_elem
-        return res
+        return self.malloc_str(num_elem)
 
     def gc_malloc_unicode(self, num_elem):
-        basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE,
-                                                   self.translate_support_code)
-        size = basesize + num_elem * itemsize
-        res = self.funcptr_for_new(size)
-        rffi.cast(rffi.CArrayPtr(lltype.Signed), res)[ofs_length/WORD] = num_elem
-        return res
+        return self.malloc_unicode(num_elem)
 
     def args_for_new(self, sizedescr):
         assert isinstance(sizedescr, BaseSizeDescr)
         return [sizedescr.size]
 
+    def args_for_new_array(self, arraydescr):
+        ofs_length = arraydescr.get_ofs_length(self.translate_support_code)
+        basesize = arraydescr.get_base_size(self.translate_support_code)
+        itemsize = arraydescr.get_item_size(self.translate_support_code)
+        return [basesize, itemsize, ofs_length]
+
     def get_funcptr_for_new(self):
         return self.funcptr_for_new
 
-    get_funcptr_for_newarray = None
-    get_funcptr_for_newstr = None
-    get_funcptr_for_newunicode = None
-
     def rewrite_assembler(self, cpu, operations, gcrefs_output_list):
         # record all GCREFs too, because Boehm cannot see them and keep them
         # alive if they end up as constants in the assembler
@@ -620,10 +649,13 @@
         def malloc_basic(size, tid):
             type_id = llop.extract_ushort(llgroup.HALFWORD, tid)
             has_finalizer = bool(tid & (1<<llgroup.HALFSHIFT))
+            has_light_finalizer = bool(tid & (1<<(llgroup.HALFSHIFT + 1)))
             check_typeid(type_id)
             res = llop1.do_malloc_fixedsize_clear(llmemory.GCREF,
                                                   type_id, size,
-                                                  has_finalizer, False)
+                                                  has_finalizer,
+                                                  has_light_finalizer,
+                                                  False)
             # In case the operation above failed, we are returning NULL
             # from this function to assembler.  There is also an RPython
             # exception set, typically MemoryError; but it's easier and
@@ -694,7 +726,7 @@
             # also use it to allocate varsized objects.  The tid
             # and possibly the length are both set afterward.
             gcref = llop1.do_malloc_fixedsize_clear(llmemory.GCREF,
-                                        0, size, False, False)
+                                        0, size, False, False, False)
             return rffi.cast(lltype.Signed, gcref)
         self.malloc_slowpath = malloc_slowpath
         self.MALLOC_SLOWPATH = lltype.FuncType([lltype.Signed], lltype.Signed)
@@ -718,7 +750,9 @@
         type_id = self.layoutbuilder.get_type_id(S)
         assert not self.layoutbuilder.is_weakref_type(S)
         has_finalizer = bool(self.layoutbuilder.has_finalizer(S))
-        flags = int(has_finalizer) << llgroup.HALFSHIFT
+        has_light_finalizer = bool(self.layoutbuilder.has_light_finalizer(S))
+        flags = (int(has_finalizer) << llgroup.HALFSHIFT |
+                 int(has_light_finalizer) << (llgroup.HALFSHIFT + 1))
         descr.tid = llop.combine_ushort(lltype.Signed, type_id, flags)
 
     def init_array_descr(self, A, descr):
@@ -752,15 +786,6 @@
     def get_funcptr_for_new(self):
         return llhelper(self.GC_MALLOC_BASIC, self.malloc_basic)
 
-    def get_funcptr_for_newarray(self):
-        return llhelper(self.GC_MALLOC_ARRAY, self.malloc_array)
-
-    def get_funcptr_for_newstr(self):
-        return llhelper(self.GC_MALLOC_STR_UNICODE, self.malloc_str)
-
-    def get_funcptr_for_newunicode(self):
-        return llhelper(self.GC_MALLOC_STR_UNICODE, self.malloc_unicode)
-
     def do_write_barrier(self, gcref_struct, gcref_newptr):
         hdr_addr = llmemory.cast_ptr_to_adr(gcref_struct)
         hdr_addr -= self.gcheaderbuilder.size_gc_header
diff --git a/pypy/jit/backend/llsupport/llmodel.py b/pypy/jit/backend/llsupport/llmodel.py
--- a/pypy/jit/backend/llsupport/llmodel.py
+++ b/pypy/jit/backend/llsupport/llmodel.py
@@ -1,24 +1,18 @@
-import sys
 from pypy.rpython.lltypesystem import lltype, llmemory, rffi, rclass, rstr
 from pypy.rpython.lltypesystem.lloperation import llop
-from pypy.rpython.llinterp import LLInterpreter, LLException
+from pypy.rpython.llinterp import LLInterpreter
 from pypy.rpython.annlowlevel import llhelper
 from pypy.rlib.objectmodel import we_are_translated, specialize
-from pypy.jit.metainterp.history import BoxInt, BoxPtr, set_future_values,\
-     BoxFloat
 from pypy.jit.metainterp import history
 from pypy.jit.codewriter import heaptracker, longlong
 from pypy.jit.backend.model import AbstractCPU
 from pypy.jit.backend.llsupport import symbolic
 from pypy.jit.backend.llsupport.symbolic import WORD, unroll_basic_sizes
-from pypy.jit.backend.llsupport.descr import get_size_descr,  BaseSizeDescr
-from pypy.jit.backend.llsupport.descr import get_field_descr, BaseFieldDescr
-from pypy.jit.backend.llsupport.descr import get_array_descr, BaseArrayDescr
-from pypy.jit.backend.llsupport.descr import get_call_descr
-from pypy.jit.backend.llsupport.descr import BaseIntCallDescr, GcPtrCallDescr
-from pypy.jit.backend.llsupport.descr import FloatCallDescr, VoidCallDescr
+from pypy.jit.backend.llsupport.descr import (get_size_descr,
+     get_field_descr, BaseFieldDescr, get_array_descr, BaseArrayDescr,
+     get_call_descr, BaseIntCallDescr, GcPtrCallDescr, FloatCallDescr,
+     VoidCallDescr, InteriorFieldDescr, get_interiorfield_descr)
 from pypy.jit.backend.llsupport.asmmemmgr import AsmMemoryManager
-from pypy.rpython.annlowlevel import cast_instance_to_base_ptr
 
 
 class AbstractLLCPU(AbstractCPU):
@@ -241,6 +235,9 @@
     def arraydescrof(self, A):
         return get_array_descr(self.gc_ll_descr, A)
 
+    def interiorfielddescrof(self, A, fieldname):
+        return get_interiorfield_descr(self.gc_ll_descr, A, A.OF, fieldname)
+
     def unpack_arraydescr(self, arraydescr):
         assert isinstance(arraydescr, BaseArrayDescr)
         return arraydescr.get_base_size(self.translate_support_code)
@@ -358,6 +355,100 @@
     bh_getarrayitem_raw_i = bh_getarrayitem_gc_i
     bh_getarrayitem_raw_f = bh_getarrayitem_gc_f
 
+    def bh_getinteriorfield_gc_i(self, gcref, itemindex, descr):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        fieldsize = descr.fielddescr.get_field_size(self.translate_support_code)
+        sign = descr.fielddescr.is_field_signed()
+        fullofs = itemindex * size + ofs
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), fullofs)
+        for STYPE, UTYPE, itemsize in unroll_basic_sizes:
+            if fieldsize == itemsize:
+                if sign:
+                    item = rffi.cast(rffi.CArrayPtr(STYPE), items)
+                    val = item[0]
+                    val = rffi.cast(lltype.Signed, val)
+                else:
+                    item = rffi.cast(rffi.CArrayPtr(UTYPE), items)
+                    val = item[0]
+                    val = rffi.cast(lltype.Signed, val)
+                # --- end of GC unsafe code ---
+                return val
+        else:
+            raise NotImplementedError("size = %d" % fieldsize)
+
+    def bh_getinteriorfield_gc_r(self, gcref, itemindex, descr):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs +
+                            size * itemindex)
+        items = rffi.cast(rffi.CArrayPtr(lltype.Signed), items)
+        pval = self._cast_int_to_gcref(items[0])
+        # --- end of GC unsafe code ---
+        return pval
+
+    def bh_getinteriorfield_gc_f(self, gcref, itemindex, descr):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs +
+                            size * itemindex)
+        items = rffi.cast(rffi.CArrayPtr(longlong.FLOATSTORAGE), items)
+        fval = items[0]
+        # --- end of GC unsafe code ---
+        return fval
+
+    def bh_setinteriorfield_gc_i(self, gcref, itemindex, descr, value):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        fieldsize = descr.fielddescr.get_field_size(self.translate_support_code)
+        ofs = itemindex * size + ofs
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs)
+        for TYPE, _, itemsize in unroll_basic_sizes:
+            if fieldsize == itemsize:
+                items = rffi.cast(rffi.CArrayPtr(TYPE), items)
+                items[0] = rffi.cast(TYPE, value)
+                # --- end of GC unsafe code ---
+                return
+        else:
+            raise NotImplementedError("size = %d" % fieldsize)
+
+    def bh_setinteriorfield_gc_r(self, gcref, itemindex, descr, newvalue):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        self.gc_ll_descr.do_write_barrier(gcref, newvalue)
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref),
+                            ofs + size * itemindex)
+        items = rffi.cast(rffi.CArrayPtr(lltype.Signed), items)
+        items[0] = self.cast_gcref_to_int(newvalue)
+        # --- end of GC unsafe code ---
+
+    def bh_setinteriorfield_gc_f(self, gcref, itemindex, descr, newvalue):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref),
+                            ofs + size * itemindex)
+        items = rffi.cast(rffi.CArrayPtr(longlong.FLOATSTORAGE), items)
+        items[0] = newvalue
+        # --- end of GC unsafe code ---
+
     def bh_strlen(self, string):
         s = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), string)
         return len(s.chars)
@@ -475,7 +566,6 @@
 
     def bh_classof(self, struct):
         struct = lltype.cast_opaque_ptr(rclass.OBJECTPTR, struct)
-        result = struct.typeptr
         result_adr = llmemory.cast_ptr_to_adr(struct.typeptr)
         return heaptracker.adr2int(result_adr)
 
diff --git a/pypy/jit/backend/llsupport/regalloc.py b/pypy/jit/backend/llsupport/regalloc.py
--- a/pypy/jit/backend/llsupport/regalloc.py
+++ b/pypy/jit/backend/llsupport/regalloc.py
@@ -178,6 +178,8 @@
         cur_max_age = -1
         candidate = None
         for next in self.reg_bindings:
+            if isinstance(next, TempBox):
+                continue
             reg = self.reg_bindings[next]
             if next in forbidden_vars:
                 continue
@@ -287,7 +289,6 @@
         self.reg_bindings[to_v] = reg
 
     def _move_variable_away(self, v, prev_loc):
-        reg = None
         if self.free_regs:
             loc = self.free_regs.pop()
             self.reg_bindings[v] = loc
diff --git a/pypy/jit/backend/llsupport/test/test_asmmemmgr.py b/pypy/jit/backend/llsupport/test/test_asmmemmgr.py
--- a/pypy/jit/backend/llsupport/test/test_asmmemmgr.py
+++ b/pypy/jit/backend/llsupport/test/test_asmmemmgr.py
@@ -211,14 +211,14 @@
     debug._log = debug.DebugLog()
     try:
         mc._dump(addr, 'test-logname-section')
-        log = list(debug._log)
+        log = list(debug._log) 
     finally:
         debug._log = None
     encoded = ''.join(writtencode).encode('hex').upper()
     ataddr = '@%x' % addr
     assert log == [('test-logname-section',
                     [('debug_print', 'CODE_DUMP', ataddr, '+0 ', encoded)])]
-    #
+    
     lltype.free(p, flavor='raw')
 
 def test_blockbuildermixin2():
diff --git a/pypy/jit/backend/llsupport/test/test_descr.py b/pypy/jit/backend/llsupport/test/test_descr.py
--- a/pypy/jit/backend/llsupport/test/test_descr.py
+++ b/pypy/jit/backend/llsupport/test/test_descr.py
@@ -3,7 +3,6 @@
 from pypy.jit.backend.llsupport import symbolic
 from pypy.rlib.objectmodel import Symbolic
 from pypy.rpython.annlowlevel import llhelper
-from pypy.jit.metainterp.history import BoxInt, BoxFloat, BoxPtr
 from pypy.jit.metainterp import history
 from pypy.jit.codewriter import longlong
 import sys, struct, py
@@ -149,7 +148,9 @@
     A2 = lltype.GcArray(lltype.Ptr(T))
     A3 = lltype.GcArray(lltype.Ptr(U))
     A4 = lltype.GcArray(lltype.Float)
-    A5 = lltype.GcArray(lltype.SingleFloat)
+    A5 = lltype.GcArray(lltype.Struct('x', ('v', lltype.Signed),
+                                      ('k', lltype.Signed)))
+    A6 = lltype.GcArray(lltype.SingleFloat)
     assert getArrayDescrClass(A2) is GcPtrArrayDescr
     assert getArrayDescrClass(A3) is NonGcPtrArrayDescr
     cls = getArrayDescrClass(A1)
@@ -158,7 +159,7 @@
     clsf = getArrayDescrClass(A4)
     assert clsf != cls
     assert clsf == getArrayDescrClass(lltype.GcArray(lltype.Float))
-    clss = getArrayDescrClass(A5)
+    clss = getArrayDescrClass(A6)
     assert clss not in (clsf, cls)
     assert clss == getArrayDescrClass(lltype.GcArray(rffi.UINT))
     #
@@ -168,11 +169,12 @@
     descr3 = get_array_descr(c0, A3)
     descr4 = get_array_descr(c0, A4)
     descr5 = get_array_descr(c0, A5)
+    descr6 = get_array_descr(c0, A6)
     assert descr1.__class__ is cls
     assert descr2.__class__ is GcPtrArrayDescr
     assert descr3.__class__ is NonGcPtrArrayDescr
     assert descr4.__class__ is clsf
-    assert descr5.__class__ is clss
+    assert descr6.__class__ is clss
     assert descr1 == get_array_descr(c0, lltype.GcArray(lltype.Char))
     assert not descr1.is_array_of_pointers()
     assert     descr2.is_array_of_pointers()
@@ -202,7 +204,8 @@
     assert descr2.get_item_size(False) == rffi.sizeof(lltype.Ptr(T))
     assert descr3.get_item_size(False) == rffi.sizeof(lltype.Ptr(U))
     assert descr4.get_item_size(False) == rffi.sizeof(lltype.Float)
-    assert descr5.get_item_size(False) == rffi.sizeof(lltype.SingleFloat)
+    assert descr5.get_item_size(False) == rffi.sizeof(lltype.Signed) * 2
+    assert descr6.get_item_size(False) == rffi.sizeof(lltype.SingleFloat)
     #
     assert isinstance(descr1.get_base_size(True), Symbolic)
     assert isinstance(descr2.get_base_size(True), Symbolic)
@@ -348,7 +351,6 @@
                             (rffi.SHORT,      True), (rffi.USHORT, False),
                             (rffi.INT,        True), (rffi.UINT,   False),
                             (rffi.LONG,       True), (rffi.ULONG,  False)]:
-        A = lltype.GcArray(RESTYPE)
         for tsc in [False, True]:
             c2 = GcCache(tsc)
             descr1 = get_call_descr(c2, [], RESTYPE)
@@ -379,7 +381,6 @@
     descr3i = get_array_descr(c0, lltype.GcArray(lltype.Char))
     assert descr3i.repr_of_descr() == '<CharArrayDescr>'
     #
-    cache = {}
     descr4 = get_call_descr(c0, [lltype.Char, lltype.Ptr(S)], lltype.Ptr(S))
     assert 'GcPtrCallDescr' in descr4.repr_of_descr()
     #
@@ -412,10 +413,10 @@
     ARGS = [lltype.Float, lltype.Ptr(ARRAY)]
     RES = lltype.Float
 
-    def f(a, b):
+    def f2(a, b):
         return float(b[0]) + a
 
-    fnptr = llhelper(lltype.Ptr(lltype.FuncType(ARGS, RES)), f)
+    fnptr = llhelper(lltype.Ptr(lltype.FuncType(ARGS, RES)), f2)
     descr2 = get_call_descr(c0, ARGS, RES)
     a = lltype.malloc(ARRAY, 3)
     opaquea = lltype.cast_opaque_ptr(llmemory.GCREF, a)
diff --git a/pypy/jit/backend/llsupport/test/test_gc.py b/pypy/jit/backend/llsupport/test/test_gc.py
--- a/pypy/jit/backend/llsupport/test/test_gc.py
+++ b/pypy/jit/backend/llsupport/test/test_gc.py
@@ -247,12 +247,14 @@
         self.record = []
 
     def do_malloc_fixedsize_clear(self, RESTYPE, type_id, size,
-                                  has_finalizer, contains_weakptr):
+                                  has_finalizer, has_light_finalizer,
+                                  contains_weakptr):
         assert not contains_weakptr
+        assert not has_finalizer           # in these tests
+        assert not has_light_finalizer     # in these tests
         p = llmemory.raw_malloc(size)
         p = llmemory.cast_adr_to_ptr(p, RESTYPE)
-        flags = int(has_finalizer) << 16
-        tid = llop.combine_ushort(lltype.Signed, type_id, flags)
+        tid = llop.combine_ushort(lltype.Signed, type_id, 0)
         self.record.append(("fixedsize", repr(size), tid, p))
         return p
 
diff --git a/pypy/jit/backend/model.py b/pypy/jit/backend/model.py
--- a/pypy/jit/backend/model.py
+++ b/pypy/jit/backend/model.py
@@ -1,5 +1,5 @@
 from pypy.rlib.debug import debug_start, debug_print, debug_stop
-from pypy.jit.metainterp import history, compile
+from pypy.jit.metainterp import history
 
 
 class AbstractCPU(object):
@@ -213,6 +213,10 @@
     def typedescrof(TYPE):
         raise NotImplementedError
 
+    @staticmethod
+    def interiorfielddescrof(A, fieldname):
+        raise NotImplementedError
+
     # ---------- the backend-dependent operations ----------
 
     # lltype specific operations
diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -5,7 +5,7 @@
                                          BoxInt, Box, BoxPtr,
                                          LoopToken,
                                          ConstInt, ConstPtr,
-                                         BoxObj, Const,
+                                         BoxObj,
                                          ConstObj, BoxFloat, ConstFloat)
 from pypy.jit.metainterp.resoperation import ResOperation, rop
 from pypy.jit.metainterp.typesystem import deref
@@ -111,7 +111,7 @@
         self.cpu.set_future_value_int(0, 2)
         fail = self.cpu.execute_token(looptoken)
         res = self.cpu.get_latest_value_int(0)
-        assert res == 3        
+        assert res == 3
         assert fail.identifier == 1
 
     def test_compile_loop(self):
@@ -127,7 +127,7 @@
             ]
         inputargs = [i0]
         operations[2].setfailargs([i1])
-        
+
         self.cpu.compile_loop(inputargs, operations, looptoken)
         self.cpu.set_future_value_int(0, 2)
         fail = self.cpu.execute_token(looptoken)
@@ -148,7 +148,7 @@
             ]
         inputargs = [i0]
         operations[2].setfailargs([None, None, i1, None])
-        
+
         self.cpu.compile_loop(inputargs, operations, looptoken)
         self.cpu.set_future_value_int(0, 2)
         fail = self.cpu.execute_token(looptoken)
@@ -372,7 +372,7 @@
         for opnum, boxargs, retvalue in get_int_tests():
             res = self.execute_operation(opnum, boxargs, 'int')
             assert res.value == retvalue
-        
+
     def test_float_operations(self):
         from pypy.jit.metainterp.test.test_executor import get_float_tests
         for opnum, boxargs, rettype, retvalue in get_float_tests(self.cpu):
@@ -438,7 +438,7 @@
 
     def test_ovf_operations_reversed(self):
         self.test_ovf_operations(reversed=True)
-        
+
     def test_bh_call(self):
         cpu = self.cpu
         #
@@ -503,7 +503,7 @@
                                          [funcbox, BoxInt(num), BoxInt(num)],
                                          'int', descr=dyn_calldescr)
             assert res.value == 2 * num
-            
+
 
         if cpu.supports_floats:
             def func(f0, f1, f2, f3, f4, f5, f6, i0, i1, f7, f8, f9):
@@ -543,7 +543,7 @@
         funcbox = self.get_funcbox(self.cpu, func_ptr)
         res = self.execute_operation(rop.CALL, [funcbox] + map(BoxInt, args), 'int', descr=calldescr)
         assert res.value == func(*args)
-        
+
     def test_call_stack_alignment(self):
         # test stack alignment issues, notably for Mac OS/X.
         # also test the ordering of the arguments.
@@ -615,7 +615,7 @@
         res = self.execute_operation(rop.GETFIELD_GC, [t_box],
                                      'int', descr=shortdescr)
         assert res.value == 1331
-        
+
         #
         u_box, U_box = self.alloc_instance(self.U)
         fielddescr2 = self.cpu.fielddescrof(self.S, 'next')
@@ -695,7 +695,7 @@
 
     def test_failing_guard_class(self):
         t_box, T_box = self.alloc_instance(self.T)
-        u_box, U_box = self.alloc_instance(self.U)        
+        u_box, U_box = self.alloc_instance(self.U)
         null_box = self.null_instance()
         for opname, args in [(rop.GUARD_CLASS, [t_box, U_box]),
                              (rop.GUARD_CLASS, [u_box, T_box]),
@@ -787,7 +787,7 @@
         r = self.execute_operation(rop.GETARRAYITEM_GC, [a_box, BoxInt(3)],
                                    'int', descr=arraydescr)
         assert r.value == 160
-        
+
         #
         if isinstance(A, lltype.GcArray):
             A = lltype.Ptr(A)
@@ -880,6 +880,73 @@
                                    'int', descr=arraydescr)
         assert r.value == 7441
 
+    def test_array_of_structs(self):
+        TP = lltype.GcStruct('x')
+        ITEM = lltype.Struct('x',
+                             ('vs', lltype.Signed),
+                             ('vu', lltype.Unsigned),
+                             ('vsc', rffi.SIGNEDCHAR),
+                             ('vuc', rffi.UCHAR),
+                             ('vss', rffi.SHORT),
+                             ('vus', rffi.USHORT),
+                             ('vsi', rffi.INT),
+                             ('vui', rffi.UINT),
+                             ('k', lltype.Float),
+                             ('p', lltype.Ptr(TP)))
+        a_box, A = self.alloc_array_of(ITEM, 15)
+        s_box, S = self.alloc_instance(TP)
+        kdescr = self.cpu.interiorfielddescrof(A, 'k')
+        pdescr = self.cpu.interiorfielddescrof(A, 'p')
+        self.execute_operation(rop.SETINTERIORFIELD_GC, [a_box, BoxInt(3),
+                                                         boxfloat(1.5)],
+                               'void', descr=kdescr)
+        f = self.cpu.bh_getinteriorfield_gc_f(a_box.getref_base(), 3, kdescr)
+        assert longlong.getrealfloat(f) == 1.5
+        self.cpu.bh_setinteriorfield_gc_f(a_box.getref_base(), 3, kdescr, longlong.getfloatstorage(2.5))
+        r = self.execute_operation(rop.GETINTERIORFIELD_GC, [a_box, BoxInt(3)],
+                                   'float', descr=kdescr)
+        assert r.getfloat() == 2.5
+        #
+        NUMBER_FIELDS = [('vs', lltype.Signed),
+                         ('vu', lltype.Unsigned),
+                         ('vsc', rffi.SIGNEDCHAR),
+                         ('vuc', rffi.UCHAR),
+                         ('vss', rffi.SHORT),
+                         ('vus', rffi.USHORT),
+                         ('vsi', rffi.INT),
+                         ('vui', rffi.UINT)]
+        for name, TYPE in NUMBER_FIELDS[::-1]:
+            vdescr = self.cpu.interiorfielddescrof(A, name)
+            self.execute_operation(rop.SETINTERIORFIELD_GC, [a_box, BoxInt(3),
+                                                             BoxInt(-15)],
+                                   'void', descr=vdescr)
+        for name, TYPE in NUMBER_FIELDS:
+            vdescr = self.cpu.interiorfielddescrof(A, name)
+            i = self.cpu.bh_getinteriorfield_gc_i(a_box.getref_base(), 3,
+                                                  vdescr)
+            assert i == rffi.cast(lltype.Signed, rffi.cast(TYPE, -15))
+        for name, TYPE in NUMBER_FIELDS[::-1]:
+            vdescr = self.cpu.interiorfielddescrof(A, name)
+            self.cpu.bh_setinteriorfield_gc_i(a_box.getref_base(), 3,
+                                              vdescr, -25)
+        for name, TYPE in NUMBER_FIELDS:
+            vdescr = self.cpu.interiorfielddescrof(A, name)
+            r = self.execute_operation(rop.GETINTERIORFIELD_GC,
+                                       [a_box, BoxInt(3)],
+                                       'int', descr=vdescr)
+            assert r.getint() == rffi.cast(lltype.Signed, rffi.cast(TYPE, -25))
+        #
+        self.execute_operation(rop.SETINTERIORFIELD_GC, [a_box, BoxInt(4),
+                                                         s_box],
+                               'void', descr=pdescr)
+        r = self.cpu.bh_getinteriorfield_gc_r(a_box.getref_base(), 4, pdescr)
+        assert r == s_box.getref_base()
+        self.cpu.bh_setinteriorfield_gc_r(a_box.getref_base(), 3, pdescr,
+                                          s_box.getref_base())
+        r = self.execute_operation(rop.GETINTERIORFIELD_GC, [a_box, BoxInt(3)],
+                                   'ref', descr=pdescr)
+        assert r.getref_base() == s_box.getref_base()
+
     def test_string_basic(self):
         s_box = self.alloc_string("hello\xfe")
         r = self.execute_operation(rop.STRLEN, [s_box], 'int')
@@ -1402,7 +1469,7 @@
         addr = llmemory.cast_ptr_to_adr(func_ptr)
         return ConstInt(heaptracker.adr2int(addr))
 
-    
+
     MY_VTABLE = rclass.OBJECT_VTABLE    # for tests only
 
     S = lltype.GcForwardReference()
@@ -1439,7 +1506,6 @@
         return BoxPtr(lltype.nullptr(llmemory.GCREF.TO))
 
     def alloc_array_of(self, ITEM, length):
-        cpu = self.cpu
         A = lltype.GcArray(ITEM)
         a = lltype.malloc(A, length)
         a_box = BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, a))
@@ -2318,7 +2384,7 @@
         for opname, arg, res in ops:
             self.execute_operation(opname, [arg], 'void')
             assert self.guard_failed == res
-        
+
         lltype.free(x, flavor='raw')
 
     def test_assembler_call(self):
@@ -2398,7 +2464,7 @@
         FakeJitDriverSD.portal_calldescr = self.cpu.calldescrof(
             lltype.Ptr(lltype.FuncType(ARGS, RES)), ARGS, RES,
             EffectInfo.MOST_GENERAL)
-        
+
         ops = '''
         [f0, f1]
         f2 = float_add(f0, f1)
@@ -2489,7 +2555,7 @@
         FakeJitDriverSD.portal_calldescr = self.cpu.calldescrof(
             lltype.Ptr(lltype.FuncType(ARGS, RES)), ARGS, RES,
             EffectInfo.MOST_GENERAL)
-        
+
         ops = '''
         [f0, f1]
         f2 = float_add(f0, f1)
@@ -2940,4 +3006,4 @@
 
     def alloc_unicode(self, unicode):
         py.test.skip("implement me")
-    
+
diff --git a/pypy/jit/backend/test/test_ll_random.py b/pypy/jit/backend/test/test_ll_random.py
--- a/pypy/jit/backend/test/test_ll_random.py
+++ b/pypy/jit/backend/test/test_ll_random.py
@@ -28,16 +28,27 @@
         fork.structure_types_and_vtables = self.structure_types_and_vtables
         return fork
 
-    def get_structptr_var(self, r, must_have_vtable=False, type=lltype.Struct):
+    def _choose_ptr_vars(self, from_, type, array_of_structs):
+        ptrvars = []
+        for i in range(len(from_)):
+            v, S = from_[i][:2]
+            if not isinstance(S, type):
+                continue
+            if ((isinstance(S, lltype.Array) and
+                 isinstance(S.OF, lltype.Struct)) == array_of_structs):
+                ptrvars.append((v, S))
+        return ptrvars
+
+    def get_structptr_var(self, r, must_have_vtable=False, type=lltype.Struct,
+                          array_of_structs=False):
         while True:
-            ptrvars = [(v, S) for (v, S) in self.ptrvars
-                              if isinstance(S, type)]
+            ptrvars = self._choose_ptr_vars(self.ptrvars, type,
+                                            array_of_structs)
             if ptrvars and r.random() < 0.8:
                 v, S = r.choice(ptrvars)
             else:
-                prebuilt_ptr_consts = [(v, S)
-                                 for (v, S, _) in self.prebuilt_ptr_consts
-                                 if isinstance(S, type)]
+                prebuilt_ptr_consts = self._choose_ptr_vars(
+                    self.prebuilt_ptr_consts, type, array_of_structs)
                 if prebuilt_ptr_consts and r.random() < 0.7:
                     v, S = r.choice(prebuilt_ptr_consts)
                 else:
@@ -48,7 +59,8 @@
                                                 has_vtable=must_have_vtable)
                     else:
                         # create a new constant array
-                        p = self.get_random_array(r)
+                        p = self.get_random_array(r,
+                                    must_be_array_of_structs=array_of_structs)
                     S = lltype.typeOf(p).TO
                     v = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, p))
                     self.prebuilt_ptr_consts.append((v, S,
@@ -74,7 +86,8 @@
                 TYPE = lltype.Signed
         return TYPE
 
-    def get_random_structure_type(self, r, with_vtable=None, cache=True):
+    def get_random_structure_type(self, r, with_vtable=None, cache=True,
+                                  type=lltype.GcStruct):
         if cache and self.structure_types and r.random() < 0.5:
             return r.choice(self.structure_types)
         fields = []
@@ -85,7 +98,7 @@
         for i in range(r.randrange(1, 5)):
             TYPE = self.get_random_primitive_type(r)
             fields.append(('f%d' % i, TYPE))
-        S = lltype.GcStruct('S%d' % self.counter, *fields, **kwds)
+        S = type('S%d' % self.counter, *fields, **kwds)
         self.counter += 1
         if cache:
             self.structure_types.append(S)
@@ -125,17 +138,29 @@
                 setattr(p, fieldname, rffi.cast(TYPE, r.random_integer()))
         return p
 
-    def get_random_array_type(self, r):
-        TYPE = self.get_random_primitive_type(r)
+    def get_random_array_type(self, r, can_be_array_of_struct=False,
+                              must_be_array_of_structs=False):
+        if ((can_be_array_of_struct and r.random() < 0.1) or
+            must_be_array_of_structs):
+            TYPE = self.get_random_structure_type(r, cache=False,
+                                                  type=lltype.Struct)
+        else:
+            TYPE = self.get_random_primitive_type(r)
         return lltype.GcArray(TYPE)
 
-    def get_random_array(self, r):
-        A = self.get_random_array_type(r)
+    def get_random_array(self, r, must_be_array_of_structs=False):
+        A = self.get_random_array_type(r,
+                           must_be_array_of_structs=must_be_array_of_structs)
         length = (r.random_integer() // 15) % 300  # length: between 0 and 299
                                                    # likely to be small
         p = lltype.malloc(A, length)
-        for i in range(length):
-            p[i] = rffi.cast(A.OF, r.random_integer())
+        if isinstance(A.OF, lltype.Primitive):
+            for i in range(length):
+                p[i] = rffi.cast(A.OF, r.random_integer())
+        else:
+            for i in range(length):
+                for fname, TP in A.OF._flds.iteritems():
+                    setattr(p[i], fname, rffi.cast(TP, r.random_integer()))
         return p
 
     def get_index(self, length, r):
@@ -155,8 +180,16 @@
                     dic[fieldname] = getattr(p, fieldname)
         else:
             assert isinstance(S, lltype.Array)
-            for i in range(len(p)):
-                dic[i] = p[i]
+            if isinstance(S.OF, lltype.Struct):
+                for i in range(len(p)):
+                    item = p[i]
+                    s1 = {}
+                    for fieldname in S.OF._names:
+                        s1[fieldname] = getattr(item, fieldname)
+                    dic[i] = s1
+            else:
+                for i in range(len(p)):
+                    dic[i] = p[i]
         return dic
 
     def print_loop_prebuilt(self, names, writevar, s):
@@ -220,7 +253,7 @@
 
 class GetFieldOperation(test_random.AbstractOperation):
     def field_descr(self, builder, r):
-        v, S = builder.get_structptr_var(r)
+        v, S = builder.get_structptr_var(r, )
         names = S._names
         if names[0] == 'parent':
             names = names[1:]
@@ -239,6 +272,28 @@
                 continue
             break
 
+class GetInteriorFieldOperation(test_random.AbstractOperation):
+    def field_descr(self, builder, r):
+        v, A = builder.get_structptr_var(r, type=lltype.Array,
+                                         array_of_structs=True)
+        array = v.getref(lltype.Ptr(A))
+        v_index = builder.get_index(len(array), r)
+        name = r.choice(A.OF._names)
+        descr = builder.cpu.interiorfielddescrof(A, name)
+        descr._random_info = 'cpu.interiorfielddescrof(%s, %r)' % (A.OF._name,
+                                                                   name)
+        TYPE = getattr(A.OF, name)
+        return v, v_index, descr, TYPE
+
+    def produce_into(self, builder, r):
+        while True:
+            try:
+                v, v_index, descr, _ = self.field_descr(builder, r)
+                self.put(builder, [v, v_index], descr)
+            except lltype.UninitializedMemoryAccess:
+                continue
+            break
+
 class SetFieldOperation(GetFieldOperation):
     def produce_into(self, builder, r):
         v, descr, TYPE = self.field_descr(builder, r)
@@ -251,6 +306,18 @@
                 break
         builder.do(self.opnum, [v, w], descr)
 
+class SetInteriorFieldOperation(GetInteriorFieldOperation):
+    def produce_into(self, builder, r):
+        v, v_index, descr, TYPE = self.field_descr(builder, r)
+        while True:
+            if r.random() < 0.3:
+                w = ConstInt(r.random_integer())
+            else:
+                w = r.choice(builder.intvars)
+            if rffi.cast(lltype.Signed, rffi.cast(TYPE, w.value)) == w.value:
+                break
+        builder.do(self.opnum, [v, v_index, w], descr)
+
 class NewOperation(test_random.AbstractOperation):
     def size_descr(self, builder, S):
         descr = builder.cpu.sizeof(S)
@@ -306,7 +373,7 @@
 
 class NewArrayOperation(ArrayOperation):
     def produce_into(self, builder, r):
-        A = builder.get_random_array_type(r)
+        A = builder.get_random_array_type(r, can_be_array_of_struct=True)
         v_size = builder.get_index(300, r)
         v_ptr = builder.do(self.opnum, [v_size], self.array_descr(builder, A))
         builder.ptrvars.append((v_ptr, A))
@@ -586,7 +653,9 @@
 for i in range(4):      # make more common
     OPERATIONS.append(GetFieldOperation(rop.GETFIELD_GC))
     OPERATIONS.append(GetFieldOperation(rop.GETFIELD_GC))
+    OPERATIONS.append(GetInteriorFieldOperation(rop.GETINTERIORFIELD_GC))
     OPERATIONS.append(SetFieldOperation(rop.SETFIELD_GC))
+    OPERATIONS.append(SetInteriorFieldOperation(rop.SETINTERIORFIELD_GC))
     OPERATIONS.append(NewOperation(rop.NEW))
     OPERATIONS.append(NewOperation(rop.NEW_WITH_VTABLE))
 
diff --git a/pypy/jit/backend/test/test_random.py b/pypy/jit/backend/test/test_random.py
--- a/pypy/jit/backend/test/test_random.py
+++ b/pypy/jit/backend/test/test_random.py
@@ -595,6 +595,10 @@
             for name, value in fields.items():
                 if isinstance(name, str):
                     setattr(container, name, value)
+                elif isinstance(value, dict):
+                    item = container.getitem(name)
+                    for key1, value1 in value.items():
+                        setattr(item, key1, value1)
                 else:
                     container.setitem(name, value)
 
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1,7 +1,7 @@
 import sys, os
 from pypy.jit.backend.llsupport import symbolic
 from pypy.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper
-from pypy.jit.metainterp.history import Const, Box, BoxInt, BoxPtr, BoxFloat
+from pypy.jit.metainterp.history import Const, Box, BoxInt, ConstInt
 from pypy.jit.metainterp.history import (AbstractFailDescr, INT, REF, FLOAT,
                                          LoopToken)
 from pypy.rpython.lltypesystem import lltype, rffi, rstr, llmemory
@@ -36,7 +36,6 @@
 from pypy.rlib import rgc
 from pypy.rlib.clibffi import FFI_DEFAULT_ABI
 from pypy.jit.backend.x86.jump import remap_frame_layout
-from pypy.jit.metainterp.history import ConstInt, BoxInt
 from pypy.jit.codewriter.effectinfo import EffectInfo
 from pypy.jit.codewriter import longlong
 
@@ -729,8 +728,8 @@
         # Also, make sure this is consistent with FRAME_FIXED_SIZE.
         self.mc.PUSH_r(ebp.value)
         self.mc.MOV_rr(ebp.value, esp.value)
-        for regloc in self.cpu.CALLEE_SAVE_REGISTERS:
-            self.mc.PUSH_r(regloc.value)
+        for loc in self.cpu.CALLEE_SAVE_REGISTERS:
+            self.mc.PUSH_r(loc.value)
 
         gcrootmap = self.cpu.gc_ll_descr.gcrootmap
         if gcrootmap and gcrootmap.is_shadow_stack:
@@ -994,7 +993,7 @@
         effectinfo = op.getdescr().get_extra_info()
         oopspecindex = effectinfo.oopspecindex
         genop_llong_list[oopspecindex](self, op, arglocs, resloc)
-        
+
     def regalloc_perform_math(self, op, arglocs, resloc):
         effectinfo = op.getdescr().get_extra_info()
         oopspecindex = effectinfo.oopspecindex
@@ -1277,8 +1276,8 @@
     genop_int_ne = _cmpop("NE", "NE")
     genop_int_gt = _cmpop("G", "L")
     genop_int_ge = _cmpop("GE", "LE")
-    genop_ptr_eq = genop_int_eq
-    genop_ptr_ne = genop_int_ne
+    genop_ptr_eq = genop_instance_ptr_eq = genop_int_eq
+    genop_ptr_ne = genop_instance_ptr_ne = genop_int_ne
 
     genop_float_lt = _cmpop_float('B', 'A')
     genop_float_le = _cmpop_float('BE', 'AE')
@@ -1298,8 +1297,8 @@
     genop_guard_int_ne = _cmpop_guard("NE", "NE", "E", "E")
     genop_guard_int_gt = _cmpop_guard("G", "L", "LE", "GE")
     genop_guard_int_ge = _cmpop_guard("GE", "LE", "L", "G")
-    genop_guard_ptr_eq = genop_guard_int_eq
-    genop_guard_ptr_ne = genop_guard_int_ne
+    genop_guard_ptr_eq = genop_guard_instance_ptr_eq = genop_guard_int_eq
+    genop_guard_ptr_ne = genop_guard_instance_ptr_ne = genop_guard_int_ne
 
     genop_guard_uint_gt = _cmpop_guard("A", "B", "BE", "AE")
     genop_guard_uint_lt = _cmpop_guard("B", "A", "AE", "BE")
@@ -1311,7 +1310,7 @@
     genop_guard_float_eq = _cmpop_guard_float("E", "E", "NE","NE")
     genop_guard_float_gt = _cmpop_guard_float("A", "B", "BE","AE")
     genop_guard_float_ge = _cmpop_guard_float("AE","BE", "B", "A")
-    
+
     def genop_math_sqrt(self, op, arglocs, resloc):
         self.mc.SQRTSD(arglocs[0], resloc)
 
@@ -1597,12 +1596,30 @@
     genop_getarrayitem_gc_pure = genop_getarrayitem_gc
     genop_getarrayitem_raw = genop_getarrayitem_gc
 
+    def genop_getinteriorfield_gc(self, op, arglocs, resloc):
+        base_loc, ofs_loc, itemsize_loc, fieldsize_loc, index_loc, sign_loc = arglocs
+        # XXX should not use IMUL in most cases
+        self.mc.IMUL(index_loc, itemsize_loc)
+        src_addr = AddressLoc(base_loc, index_loc, 0, ofs_loc.value)
+        self.load_from_mem(resloc, src_addr, fieldsize_loc, sign_loc)
+
+
     def genop_discard_setfield_gc(self, op, arglocs):
         base_loc, ofs_loc, size_loc, value_loc = arglocs
         assert isinstance(size_loc, ImmedLoc)
         dest_addr = AddressLoc(base_loc, ofs_loc)
         self.save_into_mem(dest_addr, value_loc, size_loc)
 
+    def genop_discard_setinteriorfield_gc(self, op, arglocs):
+        base_loc, ofs_loc, itemsize_loc, fieldsize_loc, index_loc, value_loc = arglocs
+        # XXX should not use IMUL in most cases
+        if isinstance(index_loc, ImmedLoc):
+            index_loc = imm(index_loc.value * itemsize_loc.value)
+        else:
+            self.mc.IMUL(index_loc, itemsize_loc)
+        dest_addr = AddressLoc(base_loc, index_loc, 0, ofs_loc.value)
+        self.save_into_mem(dest_addr, value_loc, fieldsize_loc)
+
     def genop_discard_setarrayitem_gc(self, op, arglocs):
         base_loc, ofs_loc, value_loc, size_loc, baseofs = arglocs
         assert isinstance(baseofs, ImmedLoc)
diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py
--- a/pypy/jit/backend/x86/regalloc.py
+++ b/pypy/jit/backend/x86/regalloc.py
@@ -7,7 +7,7 @@
                                          ResOperation, BoxPtr, ConstFloat,
                                          BoxFloat, LoopToken, INT, REF, FLOAT)
 from pypy.jit.backend.x86.regloc import *
-from pypy.rpython.lltypesystem import lltype, ll2ctypes, rffi, rstr
+from pypy.rpython.lltypesystem import lltype, rffi, rstr
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.rlib import rgc
 from pypy.jit.backend.llsupport import symbolic
@@ -17,11 +17,12 @@
 from pypy.jit.metainterp.resoperation import rop
 from pypy.jit.backend.llsupport.descr import BaseFieldDescr, BaseArrayDescr
 from pypy.jit.backend.llsupport.descr import BaseCallDescr, BaseSizeDescr
+from pypy.jit.backend.llsupport.descr import InteriorFieldDescr
 from pypy.jit.backend.llsupport.regalloc import FrameManager, RegisterManager,\
      TempBox
 from pypy.jit.backend.x86.arch import WORD, FRAME_FIXED_SIZE
 from pypy.jit.backend.x86.arch import IS_X86_32, IS_X86_64, MY_COPY_OF_REGS
-from pypy.rlib.rarithmetic import r_longlong, r_uint
+from pypy.rlib.rarithmetic import r_longlong
 
 class X86RegisterManager(RegisterManager):
 
@@ -433,7 +434,7 @@
             if self.can_merge_with_next_guard(op, i, operations):
                 oplist_with_guard[op.getopnum()](self, op, operations[i + 1])
                 i += 1
-            elif not we_are_translated() and op.getopnum() == -124: 
+            elif not we_are_translated() and op.getopnum() == -124:
                 self._consider_force_spill(op)
             else:
                 oplist[op.getopnum()](self, op)
@@ -650,8 +651,8 @@
     consider_uint_lt = _consider_compop
     consider_uint_le = _consider_compop
     consider_uint_ge = _consider_compop
-    consider_ptr_eq = _consider_compop
-    consider_ptr_ne = _consider_compop
+    consider_ptr_eq = consider_instance_ptr_eq = _consider_compop
+    consider_ptr_ne = consider_instance_ptr_ne = _consider_compop
 
     def _consider_float_op(self, op):
         loc1 = self.xrm.loc(op.getarg(1))
@@ -815,7 +816,7 @@
         save_all_regs = guard_not_forced_op is not None
         self.xrm.before_call(force_store, save_all_regs=save_all_regs)
         if not save_all_regs:
-            gcrootmap = gc_ll_descr = self.assembler.cpu.gc_ll_descr.gcrootmap
+            gcrootmap = self.assembler.cpu.gc_ll_descr.gcrootmap
             if gcrootmap and gcrootmap.is_shadow_stack:
                 save_all_regs = 2
         self.rm.before_call(force_store, save_all_regs=save_all_regs)
@@ -972,74 +973,27 @@
             return self._call(op, arglocs)
 
     def consider_newstr(self, op):
-        gc_ll_descr = self.assembler.cpu.gc_ll_descr
-        if gc_ll_descr.get_funcptr_for_newstr is not None:
-            # framework GC
-            loc = self.loc(op.getarg(0))
-            return self._call(op, [loc])
-        # boehm GC (XXX kill the following code at some point)
-        ofs_items, itemsize, ofs = symbolic.get_array_token(rstr.STR, self.translate_support_code)
-        assert itemsize == 1
-        return self._malloc_varsize(ofs_items, ofs, 0, op.getarg(0),
-                                    op.result)
+        loc = self.loc(op.getarg(0))
+        return self._call(op, [loc])
 
     def consider_newunicode(self, op):
-        gc_ll_descr = self.assembler.cpu.gc_ll_descr
-        if gc_ll_descr.get_funcptr_for_newunicode is not None:
-            # framework GC
-            loc = self.loc(op.getarg(0))
-            return self._call(op, [loc])
-        # boehm GC (XXX kill the following code at some point)
-        ofs_items, _, ofs = symbolic.get_array_token(rstr.UNICODE,
-                                                   self.translate_support_code)
-        scale = self._get_unicode_item_scale()
-        return self._malloc_varsize(ofs_items, ofs, scale, op.getarg(0),
-                                    op.result)
-
-    def _malloc_varsize(self, ofs_items, ofs_length, scale, v, res_v):
-        # XXX kill this function at some point
-        if isinstance(v, Box):
-            loc = self.rm.make_sure_var_in_reg(v, [v])
-            tempbox = TempBox()
-            other_loc = self.rm.force_allocate_reg(tempbox, [v])
-            self.assembler.load_effective_addr(loc, ofs_items,scale, other_loc)
-        else:
-            tempbox = None
-            other_loc = imm(ofs_items + (v.getint() << scale))
-        self._call(ResOperation(rop.NEW, [], res_v),
-                   [other_loc], [v])
-        loc = self.rm.make_sure_var_in_reg(v, [res_v])
-        assert self.loc(res_v) == eax
-        # now we have to reload length to some reasonable place
-        self.rm.possibly_free_var(v)
-        if tempbox is not None:
-            self.rm.possibly_free_var(tempbox)
-        self.PerformDiscard(ResOperation(rop.SETFIELD_GC, [None, None], None),
-                            [eax, imm(ofs_length), imm(WORD), loc])
+        loc = self.loc(op.getarg(0))
+        return self._call(op, [loc])
 
     def consider_new_array(self, op):
         gc_ll_descr = self.assembler.cpu.gc_ll_descr
-        if gc_ll_descr.get_funcptr_for_newarray is not None:
-            # framework GC
-            box_num_elem = op.getarg(0)
-            if isinstance(box_num_elem, ConstInt):
-                num_elem = box_num_elem.value
-                if gc_ll_descr.can_inline_malloc_varsize(op.getdescr(),
-                                                         num_elem):
-                    self.fastpath_malloc_varsize(op, op.getdescr(), num_elem)
-                    return
-            args = self.assembler.cpu.gc_ll_descr.args_for_new_array(
-                op.getdescr())
-            arglocs = [imm(x) for x in args]
-            arglocs.append(self.loc(box_num_elem))
-            self._call(op, arglocs)
-            return
-        # boehm GC (XXX kill the following code at some point)
-        itemsize, basesize, ofs_length, _, _ = (
-            self._unpack_arraydescr(op.getdescr()))
-        scale_of_field = _get_scale(itemsize)
-        self._malloc_varsize(basesize, ofs_length, scale_of_field,
-                             op.getarg(0), op.result)
+        box_num_elem = op.getarg(0)
+        if isinstance(box_num_elem, ConstInt):
+            num_elem = box_num_elem.value
+            if gc_ll_descr.can_inline_malloc_varsize(op.getdescr(),
+                                                     num_elem):
+                self.fastpath_malloc_varsize(op, op.getdescr(), num_elem)
+                return
+        args = self.assembler.cpu.gc_ll_descr.args_for_new_array(
+            op.getdescr())
+        arglocs = [imm(x) for x in args]
+        arglocs.append(self.loc(box_num_elem))
+        self._call(op, arglocs)
 
     def _unpack_arraydescr(self, arraydescr):
         assert isinstance(arraydescr, BaseArrayDescr)
@@ -1058,6 +1012,16 @@
         sign = fielddescr.is_field_signed()
         return imm(ofs), imm(size), ptr, sign
 
+    def _unpack_interiorfielddescr(self, descr):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs = arraydescr.get_base_size(self.translate_support_code)
+        itemsize = arraydescr.get_item_size(self.translate_support_code)
+        fieldsize = descr.fielddescr.get_field_size(self.translate_support_code)
+        sign = descr.fielddescr.is_field_signed()
+        ofs += descr.fielddescr.offset
+        return imm(ofs), imm(itemsize), imm(fieldsize), sign
+
     def consider_setfield_gc(self, op):
         ofs_loc, size_loc, _, _ = self._unpack_fielddescr(op.getdescr())
         assert isinstance(size_loc, ImmedLoc)
@@ -1074,6 +1038,25 @@
 
     consider_setfield_raw = consider_setfield_gc
 
+    def consider_setinteriorfield_gc(self, op):
+        t = self._unpack_interiorfielddescr(op.getdescr())
+        ofs, itemsize, fieldsize, _ = t
+        args = op.getarglist()
+        if fieldsize.value == 1:
+            need_lower_byte = True
+        else:
+            need_lower_byte = False
+        base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
+        tempvar = TempBox()
+        index_loc = self.rm.force_result_in_reg(tempvar, op.getarg(1), args)
+        # we're free to modify index now
+        value_loc = self.make_sure_var_in_reg(op.getarg(2), args,
+                                              need_lower_byte=need_lower_byte)
+        self.rm.possibly_free_var(tempvar)
+        self.possibly_free_vars(args)
+        self.PerformDiscard(op, [base_loc, ofs, itemsize, fieldsize,
+                                 index_loc, value_loc])
+
     def consider_strsetitem(self, op):
         args = op.getarglist()
         base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
@@ -1135,6 +1118,24 @@
     consider_getarrayitem_raw = consider_getarrayitem_gc
     consider_getarrayitem_gc_pure = consider_getarrayitem_gc
 
+    def consider_getinteriorfield_gc(self, op):
+        t = self._unpack_interiorfielddescr(op.getdescr())
+        ofs, itemsize, fieldsize, sign = t
+        if sign:
+            sign_loc = imm1
+        else:
+            sign_loc = imm0
+        args = op.getarglist()
+        tmpvar = TempBox()
+        base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
+        index_loc = self.rm.force_result_in_reg(tmpvar, op.getarg(1),
+                                                args)
+        self.rm.possibly_free_vars_for_op(op)
+        self.rm.possibly_free_var(tmpvar)
+        result_loc = self.force_allocate_reg(op.result)
+        self.Perform(op, [base_loc, ofs, itemsize, fieldsize,
+                          index_loc, sign_loc], result_loc)
+
     def consider_int_is_true(self, op, guard_op):
         # doesn't need arg to be in a register
         argloc = self.loc(op.getarg(0))
@@ -1241,7 +1242,6 @@
         self.rm.possibly_free_var(srcaddr_box)
 
     def _gen_address_inside_string(self, baseloc, ofsloc, resloc, is_unicode):
-        cpu = self.assembler.cpu
         if is_unicode:
             ofs_items, _, _ = symbolic.get_array_token(rstr.UNICODE,
                                                   self.translate_support_code)
@@ -1300,7 +1300,7 @@
         tmpreg = X86RegisterManager.all_regs[0]
         tmploc = self.rm.force_allocate_reg(box, selected_reg=tmpreg)
         xmmtmp = X86XMMRegisterManager.all_regs[0]
-        xmmtmploc = self.xrm.force_allocate_reg(box1, selected_reg=xmmtmp)
+        self.xrm.force_allocate_reg(box1, selected_reg=xmmtmp)
         # Part about non-floats
         # XXX we don't need a copy, we only just the original list
         src_locations1 = [self.loc(op.getarg(i)) for i in range(op.numargs())
@@ -1380,7 +1380,7 @@
     return lambda self, op: fn(self, op, None)
 
 def is_comparison_or_ovf_op(opnum):
-    from pypy.jit.metainterp.resoperation import opclasses, AbstractResOp
+    from pypy.jit.metainterp.resoperation import opclasses
     cls = opclasses[opnum]
     # hack hack: in theory they are instance method, but they don't use
     # any instance field, we can use a fake object
diff --git a/pypy/jit/backend/x86/test/test_del.py b/pypy/jit/backend/x86/test/test_del.py
--- a/pypy/jit/backend/x86/test/test_del.py
+++ b/pypy/jit/backend/x86/test/test_del.py
@@ -1,5 +1,4 @@
 
-import py
 from pypy.jit.backend.x86.test.test_basic import Jit386Mixin
 from pypy.jit.metainterp.test.test_del import DelTests
 
diff --git a/pypy/jit/backend/x86/test/test_dict.py b/pypy/jit/backend/x86/test/test_dict.py
new file mode 100644
--- /dev/null
+++ b/pypy/jit/backend/x86/test/test_dict.py
@@ -0,0 +1,9 @@
+
+from pypy.jit.backend.x86.test.test_basic import Jit386Mixin
+from pypy.jit.metainterp.test.test_dict import DictTests
+
+
+class TestDict(Jit386Mixin, DictTests):
+    # for the individual tests see
+    # ====> ../../../metainterp/test/test_dict.py
+    pass
diff --git a/pypy/jit/backend/x86/test/test_runner.py b/pypy/jit/backend/x86/test/test_runner.py
--- a/pypy/jit/backend/x86/test/test_runner.py
+++ b/pypy/jit/backend/x86/test/test_runner.py
@@ -31,7 +31,7 @@
 
     # for the individual tests see
     # ====> ../../test/runner_test.py
-    
+
     def setup_method(self, meth):
         self.cpu = CPU(rtyper=None, stats=FakeStats())
         self.cpu.setup_once()
@@ -69,22 +69,16 @@
 
     def test_allocations(self):
         from pypy.rpython.lltypesystem import rstr
-        
+
         allocs = [None]
         all = []
+        orig_new = self.cpu.gc_ll_descr.funcptr_for_new
         def f(size):
             allocs.insert(0, size)
-            buf = ctypes.create_string_buffer(size)
-            all.append(buf)
-            return ctypes.cast(buf, ctypes.c_void_p).value
-        func = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int)(f)
-        addr = ctypes.cast(func, ctypes.c_void_p).value
-        # ctypes produces an unsigned value. We need it to be signed for, eg,
-        # relative addressing to work properly.
-        addr = rffi.cast(lltype.Signed, addr)
-        
+            return orig_new(size)
+
         self.cpu.assembler.setup_once()
-        self.cpu.assembler.malloc_func_addr = addr
+        self.cpu.gc_ll_descr.funcptr_for_new = f
         ofs = symbolic.get_field_token(rstr.STR, 'chars', False)[0]
 
         res = self.execute_operation(rop.NEWSTR, [ConstInt(7)], 'ref')
@@ -108,7 +102,7 @@
         res = self.execute_operation(rop.NEW_ARRAY, [ConstInt(10)],
                                          'ref', descr)
         assert allocs[0] == 10*WORD + ofs + WORD
-        resbuf = self._resbuf(res)            
+        resbuf = self._resbuf(res)
         assert resbuf[ofs/WORD] == 10
 
         # ------------------------------------------------------------
@@ -116,7 +110,7 @@
         res = self.execute_operation(rop.NEW_ARRAY, [BoxInt(10)],
                                          'ref', descr)
         assert allocs[0] == 10*WORD + ofs + WORD
-        resbuf = self._resbuf(res)                        
+        resbuf = self._resbuf(res)
         assert resbuf[ofs/WORD] == 10
 
     def test_stringitems(self):
@@ -146,7 +140,7 @@
                                                      ConstInt(2), BoxInt(38)],
                                'void', descr)
         assert resbuf[itemsofs/WORD + 2] == 38
-        
+
         self.execute_operation(rop.SETARRAYITEM_GC, [res,
                                                      BoxInt(3), BoxInt(42)],
                                'void', descr)
@@ -167,7 +161,7 @@
                                                          BoxInt(2)],
                                    'int', descr)
         assert r.value == 38
-        
+
         r = self.execute_operation(rop.GETARRAYITEM_GC, [res, BoxInt(3)],
                                    'int', descr)
         assert r.value == 42
@@ -226,7 +220,7 @@
         self.execute_operation(rop.SETFIELD_GC, [res, BoxInt(1234)], 'void', ofs_i)
         i = self.execute_operation(rop.GETFIELD_GC, [res], 'int', ofs_i)
         assert i.value == 1234
-        
+
         #u = self.execute_operation(rop.GETFIELD_GC, [res, ofs_u], 'int')
         #assert u.value == 5
         self.execute_operation(rop.SETFIELD_GC, [res, ConstInt(1)], 'void',
@@ -299,7 +293,7 @@
                     else:
                         assert result != execute(self.cpu, None,
                                                  op, None, b).value
-                    
+
 
     def test_stuff_followed_by_guard(self):
         boxes = [(BoxInt(1), BoxInt(0)),
@@ -523,7 +517,7 @@
     def test_debugger_on(self):
         from pypy.tool.logparser import parse_log_file, extract_category
         from pypy.rlib import debug
-        
+
         loop = """
         [i0]
         debug_merge_point('xyz', 0)
diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -52,9 +52,11 @@
         newoperations = []
         #
         def do_rename(var, var_or_const):
+            if var.concretetype is lltype.Void:
+                renamings[var] = Constant(None, lltype.Void)
+                return
             renamings[var] = var_or_const
-            if (isinstance(var_or_const, Constant)
-                and var.concretetype != lltype.Void):
+            if isinstance(var_or_const, Constant):
                 value = var_or_const.value
                 value = lltype._cast_whatever(var.concretetype, value)
                 renamings_constants[var] = Constant(value, var.concretetype)
@@ -441,6 +443,8 @@
     rewrite_op_gc_identityhash = _do_builtin_call
     rewrite_op_gc_id           = _do_builtin_call
     rewrite_op_uint_mod        = _do_builtin_call
+    rewrite_op_cast_float_to_uint = _do_builtin_call
+    rewrite_op_cast_uint_to_float = _do_builtin_call
 
     # ----------
     # getfield/setfield/mallocs etc.
@@ -735,29 +739,54 @@
         return SpaceOperation(opname, [op.args[0]], op.result)
 
     def rewrite_op_getinteriorfield(self, op):
-        # only supports strings and unicodes
         assert len(op.args) == 3
-        assert op.args[1].value == 'chars'
         optype = op.args[0].concretetype
         if optype == lltype.Ptr(rstr.STR):
             opname = "strgetitem"
+            return SpaceOperation(opname, [op.args[0], op.args[2]], op.result)
+        elif optype == lltype.Ptr(rstr.UNICODE):
+            opname = "unicodegetitem"
+            return SpaceOperation(opname, [op.args[0], op.args[2]], op.result)
         else:
-            assert optype == lltype.Ptr(rstr.UNICODE)
-            opname = "unicodegetitem"
-        return SpaceOperation(opname, [op.args[0], op.args[2]], op.result)
+            v_inst, v_index, c_field = op.args
+            if op.result.concretetype is lltype.Void:
+                return
+            # only GcArray of Struct supported
+            assert isinstance(v_inst.concretetype.TO, lltype.GcArray)
+            STRUCT = v_inst.concretetype.TO.OF
+            assert isinstance(STRUCT, lltype.Struct)
+            descr = self.cpu.interiorfielddescrof(v_inst.concretetype.TO,
+                                                  c_field.value)
+            args = [v_inst, v_index, descr]
+            kind = getkind(op.result.concretetype)[0]
+            return SpaceOperation('getinteriorfield_gc_%s' % kind, args,
+                                  op.result)
 
     def rewrite_op_setinteriorfield(self, op):
-        # only supports strings and unicodes
         assert len(op.args) == 4
-        assert op.args[1].value == 'chars'
         optype = op.args[0].concretetype
         if optype == lltype.Ptr(rstr.STR):
             opname = "strsetitem"
+            return SpaceOperation(opname, [op.args[0], op.args[2], op.args[3]],
+                                  op.result)
+        elif optype == lltype.Ptr(rstr.UNICODE):
+            opname = "unicodesetitem"
+            return SpaceOperation(opname, [op.args[0], op.args[2], op.args[3]],
+                                  op.result)
         else:
-            assert optype == lltype.Ptr(rstr.UNICODE)
-            opname = "unicodesetitem"
-        return SpaceOperation(opname, [op.args[0], op.args[2], op.args[3]],
-                              op.result)
+            v_inst, v_index, c_field, v_value = op.args
+            if v_value.concretetype is lltype.Void:
+                return
+            # only GcArray of Struct supported
+            assert isinstance(v_inst.concretetype.TO, lltype.GcArray)
+            STRUCT = v_inst.concretetype.TO.OF
+            assert isinstance(STRUCT, lltype.Struct)
+            descr = self.cpu.interiorfielddescrof(v_inst.concretetype.TO,
+                                                  c_field.value)
+            kind = getkind(v_value.concretetype)[0]
+            args = [v_inst, v_index, v_value, descr]
+            return SpaceOperation('setinteriorfield_gc_%s' % kind, args,
+                                  op.result)
 
     def _rewrite_equality(self, op, opname):
         arg0, arg1 = op.args
@@ -771,6 +800,9 @@
     def _is_gc(self, v):
         return getattr(getattr(v.concretetype, "TO", None), "_gckind", "?") == 'gc'
 
+    def _is_rclass_instance(self, v):
+        return lltype._castdepth(v.concretetype.TO, rclass.OBJECT) >= 0
+
     def _rewrite_cmp_ptrs(self, op):
         if self._is_gc(op.args[0]):
             return op
@@ -788,11 +820,21 @@
         return self._rewrite_equality(op, 'int_is_true')
 
     def rewrite_op_ptr_eq(self, op):
-        op1 = self._rewrite_equality(op, 'ptr_iszero')
+        prefix = ''
+        if self._is_rclass_instance(op.args[0]):
+            assert self._is_rclass_instance(op.args[1])
+            op = SpaceOperation('instance_ptr_eq', op.args, op.result)
+            prefix = 'instance_'
+        op1 = self._rewrite_equality(op, prefix + 'ptr_iszero')
         return self._rewrite_cmp_ptrs(op1)
 
     def rewrite_op_ptr_ne(self, op):
-        op1 = self._rewrite_equality(op, 'ptr_nonzero')
+        prefix = ''
+        if self._is_rclass_instance(op.args[0]):
+            assert self._is_rclass_instance(op.args[1])
+            op = SpaceOperation('instance_ptr_ne', op.args, op.result)
+            prefix = 'instance_'
+        op1 = self._rewrite_equality(op, prefix + 'ptr_nonzero')
         return self._rewrite_cmp_ptrs(op1)
 
     rewrite_op_ptr_iszero = _rewrite_cmp_ptrs
@@ -821,26 +863,44 @@
         elif not float_arg and float_res:
             # some int -> some float
             ops = []
-            v1 = varoftype(lltype.Signed)
-            oplist = self.rewrite_operation(
-                SpaceOperation('force_cast', [v_arg], v1)
-            )
-            if oplist:
-                ops.extend(oplist)
+            v2 = varoftype(lltype.Float)
+            sizesign = rffi.size_and_sign(v_arg.concretetype)
+            if sizesign <= rffi.size_and_sign(lltype.Signed):
+                # cast from a type that fits in an int: either the size is
+                # smaller, or it is equal and it is not unsigned
+                v1 = varoftype(lltype.Signed)
+                oplist = self.rewrite_operation(
+                    SpaceOperation('force_cast', [v_arg], v1)
+                )
+                if oplist:
+                    ops.extend(oplist)
+                else:
+                    v1 = v_arg
+                op = self.rewrite_operation(
+                    SpaceOperation('cast_int_to_float', [v1], v2)
+                )
+                ops.append(op)
             else:
-                v1 = v_arg
-            v2 = varoftype(lltype.Float)
-            op = self.rewrite_operation(
-                SpaceOperation('cast_int_to_float', [v1], v2)
-            )
-            ops.append(op)
+                if sizesign == rffi.size_and_sign(lltype.Unsigned):
+                    opname = 'cast_uint_to_float'
+                elif sizesign == rffi.size_and_sign(lltype.SignedLongLong):
+                    opname = 'cast_longlong_to_float'
+                elif sizesign == rffi.size_and_sign(lltype.UnsignedLongLong):
+                    opname = 'cast_ulonglong_to_float'
+                else:
+                    raise AssertionError('cast_x_to_float: %r' % (sizesign,))
+                ops1 = self.rewrite_operation(
+                    SpaceOperation(opname, [v_arg], v2)
+                )
+                if not isinstance(ops1, list): ops1 = [ops1]
+                ops.extend(ops1)
             op2 = self.rewrite_operation(
                 SpaceOperation('force_cast', [v2], v_result)
             )
             if op2:
                 ops.append(op2)
             else:
-                op.result = v_result
+                ops[-1].result = v_result
             return ops
         elif float_arg and not float_res:
             # some float -> some int
@@ -853,18 +913,36 @@
                 ops.append(op1)
             else:
                 v1 = v_arg
-            v2 = varoftype(lltype.Signed)
-            op = self.rewrite_operation(
-                SpaceOperation('cast_float_to_int', [v1], v2)
-            )
-            ops.append(op)
-            oplist = self.rewrite_operation(
-                SpaceOperation('force_cast', [v2], v_result)
-            )
-            if oplist:
-                ops.extend(oplist)
+            sizesign = rffi.size_and_sign(v_result.concretetype)
+            if sizesign <= rffi.size_and_sign(lltype.Signed):
+                # cast to a type that fits in an int: either the size is
+                # smaller, or it is equal and it is not unsigned
+                v2 = varoftype(lltype.Signed)
+                op = self.rewrite_operation(
+                    SpaceOperation('cast_float_to_int', [v1], v2)
+                )
+                ops.append(op)
+                oplist = self.rewrite_operation(
+                    SpaceOperation('force_cast', [v2], v_result)
+                )
+                if oplist:
+                    ops.extend(oplist)
+                else:
+                    op.result = v_result
             else:
-                op.result = v_result
+                if sizesign == rffi.size_and_sign(lltype.Unsigned):
+                    opname = 'cast_float_to_uint'
+                elif sizesign == rffi.size_and_sign(lltype.SignedLongLong):
+                    opname = 'cast_float_to_longlong'
+                elif sizesign == rffi.size_and_sign(lltype.UnsignedLongLong):
+                    opname = 'cast_float_to_ulonglong'
+                else:
+                    raise AssertionError('cast_float_to_x: %r' % (sizesign,))
+                ops1 = self.rewrite_operation(
+                    SpaceOperation(opname, [v1], v_result)
+                )
+                if not isinstance(ops1, list): ops1 = [ops1]
+                ops.extend(ops1)
             return ops
         else:
             assert False
@@ -1070,8 +1148,6 @@
     # The new operation is optionally further processed by rewrite_operation().
     for _old, _new in [('bool_not', 'int_is_zero'),
                        ('cast_bool_to_float', 'cast_int_to_float'),
-                       ('cast_uint_to_float', 'cast_int_to_float'),
-                       ('cast_float_to_uint', 'cast_float_to_int'),
 
                        ('int_add_nonneg_ovf', 'int_add_ovf'),
                        ('keepalive', '-live-'),
diff --git a/pypy/jit/codewriter/support.py b/pypy/jit/codewriter/support.py
--- a/pypy/jit/codewriter/support.py
+++ b/pypy/jit/codewriter/support.py
@@ -13,7 +13,6 @@
 from pypy.translator.simplify import get_funcobj
 from pypy.translator.unsimplify import split_block
 from pypy.objspace.flow.model import Constant
-from pypy import conftest
 from pypy.translator.translator import TranslationContext
 from pypy.annotation.policy import AnnotatorPolicy
 from pypy.annotation import model as annmodel
@@ -48,15 +47,13 @@
     a.build_types(func, argtypes, main_entry_point=True)
     rtyper = t.buildrtyper(type_system = type_system)
     rtyper.specialize()
-    if inline:
-        auto_inlining(t, threshold=inline)
+    #if inline:
+    #    auto_inlining(t, threshold=inline)
     if backendoptimize:
         from pypy.translator.backendopt.all import backend_optimizations
         backend_optimizations(t, inline_threshold=inline or 0,
                 remove_asserts=True, really_remove_asserts=True)
 
-    #if conftest.option.view:
-    #    t.view()
     return rtyper
 
 def getgraph(func, values):
@@ -232,6 +229,17 @@
     else:
         return x
 
+def _ll_1_cast_uint_to_float(x):
+    # XXX on 32-bit platforms, this should be done using cast_longlong_to_float
+    # (which is a residual call right now in the x86 backend)
+    return llop.cast_uint_to_float(lltype.Float, x)
+
+def _ll_1_cast_float_to_uint(x):
+    # XXX on 32-bit platforms, this should be done using cast_float_to_longlong
+    # (which is a residual call right now in the x86 backend)
+    return llop.cast_float_to_uint(lltype.Unsigned, x)
+
+
 # math support
 # ------------
 
@@ -456,6 +464,8 @@
         return LLtypeHelpers._dictnext_items(lltype.Ptr(RES), iter)
     _ll_1_dictiter_nextitems.need_result_type = True
 
+    _ll_1_dict_resize = ll_rdict.ll_dict_resize
+
     # ---------- strings and unicode ----------
 
     _ll_1_str_str2unicode = ll_rstr.LLHelpers.ll_str2unicode
diff --git a/pypy/jit/codewriter/test/test_flatten.py b/pypy/jit/codewriter/test/test_flatten.py
--- a/pypy/jit/codewriter/test/test_flatten.py
+++ b/pypy/jit/codewriter/test/test_flatten.py
@@ -8,7 +8,7 @@
 from pypy.rpython.lltypesystem import lltype, rclass, rstr
 from pypy.objspace.flow.model import SpaceOperation, Variable, Constant
 from pypy.translator.unsimplify import varoftype
-from pypy.rlib.rarithmetic import ovfcheck, r_uint
+from pypy.rlib.rarithmetic import ovfcheck, r_uint, r_longlong, r_ulonglong
 from pypy.rlib.jit import dont_look_inside, _we_are_jitted, JitDriver
 from pypy.rlib.objectmodel import keepalive_until_here
 from pypy.rlib import jit
@@ -70,7 +70,8 @@
         return 'residual'
     def getcalldescr(self, op, oopspecindex=None, extraeffect=None):
         try:
-            if 'cannot_raise' in op.args[0].value._obj.graph.name:
+            name = op.args[0].value._obj._name
+            if 'cannot_raise' in name or name.startswith('cast_'):
                 return self._descr_cannot_raise
         except AttributeError:
             pass
@@ -900,6 +901,67 @@
             int_return %i4
         """, transform=True)
 
+        def f(dbl):
+            return rffi.cast(rffi.UCHAR, dbl)
+        self.encoding_test(f, [12.456], """
+            cast_float_to_int %f0 -> %i0
+            int_and %i0, $255 -> %i1
+            int_return %i1
+        """, transform=True)
+
+        def f(dbl):
+            return rffi.cast(lltype.Unsigned, dbl)
+        self.encoding_test(f, [12.456], """
+            residual_call_irf_i $<* fn cast_float_to_uint>, <Descr>, I[], R[], F[%f0] -> %i0
+            int_return %i0
+        """, transform=True)
+
+        def f(i):
+            return rffi.cast(lltype.Float, chr(i))    # "char -> float"
+        self.encoding_test(f, [12], """
+            cast_int_to_float %i0 -> %f0
+            float_return %f0
+        """, transform=True)
+
+        def f(i):
+            return rffi.cast(lltype.Float, r_uint(i))    # "uint -> float"
+        self.encoding_test(f, [12], """
+            residual_call_irf_f $<* fn cast_uint_to_float>, <Descr>, I[%i0], R[], F[] -> %f0
+            float_return %f0
+        """, transform=True)
+
+        if not longlong.is_64_bit:
+            def f(dbl):
+                return rffi.cast(lltype.SignedLongLong, dbl)
+            self.encoding_test(f, [12.3], """
+                residual_call_irf_f $<* fn llong_from_float>, <Descr>, I[], R[], F[%f0] -> %f1
+                float_return %f1
+            """, transform=True)
+
+            def f(dbl):
+                return rffi.cast(lltype.UnsignedLongLong, dbl)
+            self.encoding_test(f, [12.3], """
+                residual_call_irf_f $<* fn ullong_from_float>, <Descr>, I[], R[], F[%f0] -> %f1
+                float_return %f1
+            """, transform=True)
+
+            def f(x):
+                ll = r_longlong(x)
+                return rffi.cast(lltype.Float, ll)
+            self.encoding_test(f, [12], """
+                residual_call_irf_f $<* fn llong_from_int>, <Descr>, I[%i0], R[], F[] -> %f0
+                residual_call_irf_f $<* fn llong_to_float>, <Descr>, I[], R[], F[%f0] -> %f1
+                float_return %f1
+            """, transform=True)
+
+            def f(x):
+                ll = r_ulonglong(x)
+                return rffi.cast(lltype.Float, ll)
+            self.encoding_test(f, [12], """
+                residual_call_irf_f $<* fn ullong_from_int>, <Descr>, I[%i0], R[], F[] -> %f0
+                residual_call_irf_f $<* fn ullong_u_to_float>, <Descr>, I[], R[], F[%f0] -> %f1
+                float_return %f1
+            """, transform=True)
 
     def test_direct_ptradd(self):
         from pypy.rpython.lltypesystem import rffi
diff --git a/pypy/jit/codewriter/test/test_jtransform.py b/pypy/jit/codewriter/test/test_jtransform.py
--- a/pypy/jit/codewriter/test/test_jtransform.py
+++ b/pypy/jit/codewriter/test/test_jtransform.py
@@ -1,4 +1,3 @@
-import py
 import random
 try:
     from itertools import product
@@ -16,13 +15,13 @@
 
 from pypy.objspace.flow.model import FunctionGraph, Block, Link
 from pypy.objspace.flow.model import SpaceOperation, Variable, Constant
-from pypy.jit.codewriter.jtransform import Transformer
-from pypy.jit.metainterp.history import getkind
-from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr, rlist
+from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr
 from pypy.rpython.lltypesystem.module import ll_math
 from pypy.translator.unsimplify import varoftype
 from pypy.jit.codewriter import heaptracker, effectinfo
 from pypy.jit.codewriter.flatten import ListOfKind
+from pypy.jit.codewriter.jtransform import Transformer
+from pypy.jit.metainterp.history import getkind
 
 def const(x):
     return Constant(x, lltype.typeOf(x))
@@ -37,6 +36,8 @@
         return ('calldescr', FUNC, ARGS, RESULT)
     def fielddescrof(self, STRUCT, name):
         return ('fielddescr', STRUCT, name)
+    def interiorfielddescrof(self, ARRAY, name):
+        return ('interiorfielddescr', ARRAY, name)
     def arraydescrof(self, ARRAY):
         return FakeDescr(('arraydescr', ARRAY))
     def sizeof(self, STRUCT):
@@ -539,7 +540,7 @@
 
 def test_rename_on_links():
     v1 = Variable()
-    v2 = Variable()
+    v2 = Variable(); v2.concretetype = llmemory.Address
     v3 = Variable()
     block = Block([v1])
     block.operations = [SpaceOperation('cast_pointer', [v1], v2)]
@@ -575,10 +576,10 @@
         assert op1.args == [v2]
 
 def test_ptr_eq():
-    v1 = varoftype(rclass.OBJECTPTR)
-    v2 = varoftype(rclass.OBJECTPTR)
+    v1 = varoftype(lltype.Ptr(rstr.STR))
+    v2 = varoftype(lltype.Ptr(rstr.STR))
     v3 = varoftype(lltype.Bool)
-    c0 = const(lltype.nullptr(rclass.OBJECT))
+    c0 = const(lltype.nullptr(rstr.STR))
     #
     for opname, reducedname in [('ptr_eq', 'ptr_iszero'),
                                 ('ptr_ne', 'ptr_nonzero')]:
@@ -597,6 +598,31 @@
         assert op1.opname == reducedname
         assert op1.args == [v2]
 
+def test_instance_ptr_eq():
+    v1 = varoftype(rclass.OBJECTPTR)
+    v2 = varoftype(rclass.OBJECTPTR)
+    v3 = varoftype(lltype.Bool)
+    c0 = const(lltype.nullptr(rclass.OBJECT))
+
+    for opname, newopname, reducedname in [
+        ('ptr_eq', 'instance_ptr_eq', 'instance_ptr_iszero'),
+        ('ptr_ne', 'instance_ptr_ne', 'instance_ptr_nonzero')
+    ]:
+        op = SpaceOperation(opname, [v1, v2], v3)
+        op1 = Transformer().rewrite_operation(op)
+        assert op1.opname == newopname
+        assert op1.args == [v1, v2]
+
+        op = SpaceOperation(opname, [v1, c0], v3)
+        op1 = Transformer().rewrite_operation(op)
+        assert op1.opname == reducedname
+        assert op1.args == [v1]
+
+        op = SpaceOperation(opname, [c0, v1], v3)
+        op1 = Transformer().rewrite_operation(op)
+        assert op1.opname == reducedname
+        assert op1.args == [v1]
+
 def test_nongc_ptr_eq():
     v1 = varoftype(rclass.NONGCOBJECTPTR)
     v2 = varoftype(rclass.NONGCOBJECTPTR)
@@ -676,6 +702,22 @@
     assert op1.args == [v, v_index]
     assert op1.result == v_result
 
+def test_dict_getinteriorfield():
+    DICT = lltype.GcArray(lltype.Struct('ENTRY', ('v', lltype.Signed),
+                                        ('k', lltype.Signed)))
+    v = varoftype(lltype.Ptr(DICT))
+    i = varoftype(lltype.Signed)
+    v_result = varoftype(lltype.Signed)
+    op = SpaceOperation('getinteriorfield', [v, i, Constant('v', lltype.Void)],
+                        v_result)
+    op1 = Transformer(FakeCPU()).rewrite_operation(op)
+    assert op1.opname == 'getinteriorfield_gc_i'
+    assert op1.args == [v, i, ('interiorfielddescr', DICT, 'v')]
+    op = SpaceOperation('getinteriorfield', [v, i, Constant('v', lltype.Void)],
+                        Constant(None, lltype.Void))
+    op1 = Transformer(FakeCPU()).rewrite_operation(op)
+    assert op1 is None
+
 def test_str_setinteriorfield():
     v = varoftype(lltype.Ptr(rstr.STR))
     v_index = varoftype(lltype.Signed)
@@ -702,6 +744,23 @@
     assert op1.args == [v, v_index, v_newchr]
     assert op1.result == v_void
 
+def test_dict_setinteriorfield():
+    DICT = lltype.GcArray(lltype.Struct('ENTRY', ('v', lltype.Signed),
+                                        ('k', lltype.Signed)))
+    v = varoftype(lltype.Ptr(DICT))
+    i = varoftype(lltype.Signed)
+    v_void = varoftype(lltype.Void)
+    op = SpaceOperation('setinteriorfield', [v, i, Constant('v', lltype.Void),
+                                             i],
+                        v_void)
+    op1 = Transformer(FakeCPU()).rewrite_operation(op)
+    assert op1.opname == 'setinteriorfield_gc_i'
+    assert op1.args == [v, i, i, ('interiorfielddescr', DICT, 'v')]
+    op = SpaceOperation('setinteriorfield', [v, i, Constant('v', lltype.Void),
+                                             v_void], v_void)
+    op1 = Transformer(FakeCPU()).rewrite_operation(op)
+    assert not op1
+
 def test_promote_1():
     v1 = varoftype(lltype.Signed)
     v2 = varoftype(lltype.Signed)
diff --git a/pypy/jit/metainterp/blackhole.py b/pypy/jit/metainterp/blackhole.py
--- a/pypy/jit/metainterp/blackhole.py
+++ b/pypy/jit/metainterp/blackhole.py
@@ -6,7 +6,6 @@
 from pypy.rlib.debug import make_sure_not_resized
 from pypy.rpython.lltypesystem import lltype, llmemory, rclass
 from pypy.rpython.lltypesystem.lloperation import llop
-from pypy.rpython.llinterp import LLException
 from pypy.jit.codewriter.jitcode import JitCode, SwitchDictDescr
 from pypy.jit.codewriter import heaptracker, longlong
 from pypy.jit.metainterp.jitexc import JitException, get_llexception, reraise
@@ -500,6 +499,12 @@
     @arguments("r", returns="i")
     def bhimpl_ptr_nonzero(a):
         return bool(a)
+    @arguments("r", "r", returns="i")
+    def bhimpl_instance_ptr_eq(a, b):
+        return a == b
+    @arguments("r", "r", returns="i")
+    def bhimpl_instance_ptr_ne(a, b):
+        return a != b
     @arguments("r", returns="r")
     def bhimpl_cast_opaque_ptr(a):
         return a
@@ -631,6 +636,9 @@
         a = longlong.getrealfloat(a)
         # note: we need to call int() twice to care for the fact that
         # int(-2147483648.0) returns a long :-(
+        # we could also call intmask() instead of the outermost int(), but
+        # it's probably better to explicitly crash (by getting a long) if a
+        # non-translated version tries to cast a too large float to an int.
         return int(int(a))
 
     @arguments("i", returns="f")
@@ -1154,6 +1162,26 @@
         array = cpu.bh_getfield_gc_r(vable, fdescr)
         return cpu.bh_arraylen_gc(adescr, array)
 
+    @arguments("cpu", "r", "i", "d", returns="i")
+    def bhimpl_getinteriorfield_gc_i(cpu, array, index, descr):
+        return cpu.bh_getinteriorfield_gc_i(array, index, descr)
+    @arguments("cpu", "r", "i", "d", returns="r")
+    def bhimpl_getinteriorfield_gc_r(cpu, array, index, descr):
+        return cpu.bh_getinteriorfield_gc_r(array, index, descr)
+    @arguments("cpu", "r", "i", "d", returns="f")
+    def bhimpl_getinteriorfield_gc_f(cpu, array, index, descr):
+        return cpu.bh_getinteriorfield_gc_f(array, index, descr)
+
+    @arguments("cpu", "r", "i", "d", "i")
+    def bhimpl_setinteriorfield_gc_i(cpu, array, index, descr, value):
+        cpu.bh_setinteriorfield_gc_i(array, index, descr, value)
+    @arguments("cpu", "r", "i", "d", "r")
+    def bhimpl_setinteriorfield_gc_r(cpu, array, index, descr, value):
+        cpu.bh_setinteriorfield_gc_r(array, index, descr, value)
+    @arguments("cpu", "r", "i", "d", "f")
+    def bhimpl_setinteriorfield_gc_f(cpu, array, index, descr, value):
+        cpu.bh_setinteriorfield_gc_f(array, index, descr, value)
+
     @arguments("cpu", "r", "d", returns="i")
     def bhimpl_getfield_gc_i(cpu, struct, fielddescr):
         return cpu.bh_getfield_gc_i(struct, fielddescr)
diff --git a/pypy/jit/metainterp/executor.py b/pypy/jit/metainterp/executor.py
--- a/pypy/jit/metainterp/executor.py
+++ b/pypy/jit/metainterp/executor.py
@@ -1,11 +1,8 @@
 """This implements pyjitpl's execution of operations.
 """
 
-import py
-from pypy.rpython.lltypesystem import lltype, llmemory, rstr
-from pypy.rpython.ootypesystem import ootype
-from pypy.rpython.lltypesystem.lloperation import llop
-from pypy.rlib.rarithmetic import ovfcheck, r_uint, intmask, r_longlong
+from pypy.rpython.lltypesystem import lltype, rstr
+from pypy.rlib.rarithmetic import ovfcheck, r_longlong
 from pypy.rlib.rtimer import read_timestamp
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.jit.metainterp.history import BoxInt, BoxPtr, BoxFloat, check_descr
@@ -123,6 +120,29 @@
     else:
         cpu.bh_setarrayitem_raw_i(arraydescr, array, index, itembox.getint())
 
+def do_getinteriorfield_gc(cpu, _, arraybox, indexbox, descr):
+    array = arraybox.getref_base()
+    index = indexbox.getint()
+    if descr.is_pointer_field():
+        return BoxPtr(cpu.bh_getinteriorfield_gc_r(array, index, descr))
+    elif descr.is_float_field():
+        return BoxFloat(cpu.bh_getinteriorfield_gc_f(array, index, descr))
+    else:
+        return BoxInt(cpu.bh_getinteriorfield_gc_i(array, index, descr))
+
+def do_setinteriorfield_gc(cpu, _, arraybox, indexbox, valuebox, descr):
+    array = arraybox.getref_base()
+    index = indexbox.getint()
+    if descr.is_pointer_field():
+        cpu.bh_setinteriorfield_gc_r(array, index, descr,
+                                     valuebox.getref_base())
+    elif descr.is_float_field():
+        cpu.bh_setinteriorfield_gc_f(array, index, descr,
+                                     valuebox.getfloatstorage())
+    else:
+        cpu.bh_setinteriorfield_gc_i(array, index, descr,
+                                     valuebox.getint())
+
 def do_getfield_gc(cpu, _, structbox, fielddescr):
     struct = structbox.getref_base()
     if fielddescr.is_pointer_field():
diff --git a/pypy/jit/metainterp/graphpage.py b/pypy/jit/metainterp/graphpage.py
--- a/pypy/jit/metainterp/graphpage.py
+++ b/pypy/jit/metainterp/graphpage.py
@@ -12,8 +12,8 @@
     def get_display_text(self):
         return None
 
-def display_loops(loops, errmsg=None, highlight_loops=()):
-    graphs = [(loop, loop in highlight_loops) for loop in loops]    
+def display_loops(loops, errmsg=None, highlight_loops={}):
+    graphs = [(loop, highlight_loops.get(loop, 0)) for loop in loops]    
     for graph, highlight in graphs:
         for op in graph.get_operations():
             if is_interesting_guard(op):
@@ -65,8 +65,7 @@
     def add_graph(self, graph, highlight=False):
         graphindex = len(self.graphs)
         self.graphs.append(graph)
-        if highlight:
-            self.highlight_graphs[graph] = True
+        self.highlight_graphs[graph] = highlight
         for i, op in enumerate(graph.get_operations()):
             self.all_operations[op] = graphindex, i
 
@@ -126,10 +125,13 @@
             self.dotgen.emit('subgraph cluster%d {' % graphindex)
         label = graph.get_display_text()
         if label is not None:
-            if self.highlight_graphs.get(graph):
-                fillcolor = '#f084c2'
+            colorindex = self.highlight_graphs.get(graph, 0)
+            if colorindex == 1:
+                fillcolor = '#f084c2'    # highlighted graph
+            elif colorindex == 2:
+                fillcolor = '#808080'    # invalidated graph
             else:
-                fillcolor = '#84f0c2'
+                fillcolor = '#84f0c2'    # normal color
             self.dotgen.emit_node(graphname, shape="octagon",
                                   label=label, fillcolor=fillcolor)
             self.pendingedges.append((graphname,
diff --git a/pypy/jit/metainterp/history.py b/pypy/jit/metainterp/history.py
--- a/pypy/jit/metainterp/history.py
+++ b/pypy/jit/metainterp/history.py
@@ -16,6 +16,7 @@
 INT   = 'i'
 REF   = 'r'
 FLOAT = 'f'
+STRUCT = 's'
 HOLE  = '_'
 VOID  = 'v'
 
@@ -172,6 +173,11 @@
         """
         raise NotImplementedError
 
+    def is_array_of_structs(self):
+        """ Implement for array descr
+        """
+        raise NotImplementedError
+
     def is_pointer_field(self):
         """ Implement for field descr
         """
@@ -732,6 +738,7 @@
     failed_states = None
     retraced_count = 0
     terminating = False # see TerminatingLoopToken in compile.py
+    invalidated = False
     outermost_jitdriver_sd = None
     # and more data specified by the backend when the loop is compiled
     number = -1
@@ -934,6 +941,7 @@
         self.loops = []
         self.locations = []
         self.aborted_keys = []
+        self.invalidated_token_numbers = set()
 
     def set_history(self, history):
         self.operations = history.operations
@@ -1012,7 +1020,12 @@
             if loop in loops:
                 loops.remove(loop)
             loops.append(loop)
-        display_loops(loops, errmsg, extraloops)
+        highlight_loops = dict.fromkeys(extraloops, 1)
+        for loop in loops:
+            if hasattr(loop, '_looptoken_number') and (
+                    loop._looptoken_number in self.invalidated_token_numbers):
+                highlight_loops.setdefault(loop, 2)
+        display_loops(loops, errmsg, highlight_loops)
 
 # ----------------------------------------------------------------
 
diff --git a/pypy/jit/metainterp/memmgr.py b/pypy/jit/metainterp/memmgr.py
--- a/pypy/jit/metainterp/memmgr.py
+++ b/pypy/jit/metainterp/memmgr.py
@@ -68,7 +68,8 @@
         debug_print("Loop tokens before:", oldtotal)
         max_generation = self.current_generation - (self.max_age-1)
         for looptoken in self.alive_loops.keys():
-            if 0 <= looptoken.generation < max_generation:
+            if (0 <= looptoken.generation < max_generation or
+                looptoken.invalidated):
                 del self.alive_loops[looptoken]
         newtotal = len(self.alive_loops)
         debug_print("Loop tokens freed: ", oldtotal - newtotal)
diff --git a/pypy/jit/metainterp/optimizeopt/rewrite.py b/pypy/jit/metainterp/optimizeopt/rewrite.py
--- a/pypy/jit/metainterp/optimizeopt/rewrite.py
+++ b/pypy/jit/metainterp/optimizeopt/rewrite.py
@@ -106,10 +106,9 @@
             self.make_equal_to(op.result, v1)
         else:
             self.emit_operation(op)
-
-        # Synthesize the reverse ops for optimize_default to reuse
-        self.pure(rop.INT_ADD, [op.result, op.getarg(1)], op.getarg(0))
-        self.pure(rop.INT_SUB, [op.getarg(0), op.result], op.getarg(1))
+            # Synthesize the reverse ops for optimize_default to reuse
+            self.pure(rop.INT_ADD, [op.result, op.getarg(1)], op.getarg(0))
+            self.pure(rop.INT_SUB, [op.getarg(0), op.result], op.getarg(1))
 
     def optimize_INT_ADD(self, op):
         v1 = self.getvalue(op.getarg(0))
@@ -122,10 +121,9 @@
             self.make_equal_to(op.result, v1)
         else:
             self.emit_operation(op)
-
-        # Synthesize the reverse op for optimize_default to reuse
-        self.pure(rop.INT_SUB, [op.result, op.getarg(1)], op.getarg(0))
-        self.pure(rop.INT_SUB, [op.result, op.getarg(0)], op.getarg(1))
+            # Synthesize the reverse op for optimize_default to reuse
+            self.pure(rop.INT_SUB, [op.result, op.getarg(1)], op.getarg(0))
+            self.pure(rop.INT_SUB, [op.result, op.getarg(0)], op.getarg(1))
 
     def optimize_INT_MUL(self, op):
         v1 = self.getvalue(op.getarg(0))
@@ -141,13 +139,13 @@
             self.make_constant_int(op.result, 0)
         else:
             for lhs, rhs in [(v1, v2), (v2, v1)]:
-                # x & (x -1) == 0 is a quick test for power of 2
-                if (lhs.is_constant() and
-                    (lhs.box.getint() & (lhs.box.getint() - 1)) == 0):
-                    new_rhs = ConstInt(highest_bit(lhs.box.getint()))
-                    op = op.copy_and_change(rop.INT_LSHIFT, args=[rhs.box, new_rhs])
-                    break
-
+                if lhs.is_constant():
+                    x = lhs.box.getint()
+                    # x & (x - 1) == 0 is a quick test for power of 2
+                    if x & (x - 1) == 0:
+                        new_rhs = ConstInt(highest_bit(lhs.box.getint()))
+                        op = op.copy_and_change(rop.INT_LSHIFT, args=[rhs.box, new_rhs])
+                        break
             self.emit_operation(op)
 
     def optimize_UINT_FLOORDIV(self, op):
@@ -339,7 +337,7 @@
     def optimize_INT_IS_ZERO(self, op):
         self._optimize_nullness(op, op.getarg(0), False)
 
-    def _optimize_oois_ooisnot(self, op, expect_isnot):
+    def _optimize_oois_ooisnot(self, op, expect_isnot, instance):
         value0 = self.getvalue(op.getarg(0))
         value1 = self.getvalue(op.getarg(1))
         if value0.is_virtual():
@@ -357,21 +355,28 @@
         elif value0 is value1:
             self.make_constant_int(op.result, not expect_isnot)
         else:
-            cls0 = value0.get_constant_class(self.optimizer.cpu)
-            if cls0 is not None:
-                cls1 = value1.get_constant_class(self.optimizer.cpu)
-                if cls1 is not None and not cls0.same_constant(cls1):
-                    # cannot be the same object, as we know that their
-                    # class is different
-                    self.make_constant_int(op.result, expect_isnot)
-                    return
+            if instance:
+                cls0 = value0.get_constant_class(self.optimizer.cpu)
+                if cls0 is not None:
+                    cls1 = value1.get_constant_class(self.optimizer.cpu)
+                    if cls1 is not None and not cls0.same_constant(cls1):
+                        # cannot be the same object, as we know that their
+                        # class is different
+                        self.make_constant_int(op.result, expect_isnot)
+                        return
             self.emit_operation(op)
 
+    def optimize_PTR_EQ(self, op):
+        self._optimize_oois_ooisnot(op, False, False)
+
     def optimize_PTR_NE(self, op):
-        self._optimize_oois_ooisnot(op, True)
+        self._optimize_oois_ooisnot(op, True, False)
 
-    def optimize_PTR_EQ(self, op):
-        self._optimize_oois_ooisnot(op, False)
+    def optimize_INSTANCE_PTR_EQ(self, op):
+        self._optimize_oois_ooisnot(op, False, True)
+
+    def optimize_INSTANCE_PTR_NE(self, op):
+        self._optimize_oois_ooisnot(op, True, True)
 
 ##    def optimize_INSTANCEOF(self, op):
 ##        value = self.getvalue(op.args[0])
@@ -450,6 +455,9 @@
         if v2.is_constant() and v2.box.getint() == 1:
             self.make_equal_to(op.result, v1)
             return
+        elif v1.is_constant() and v1.box.getint() == 0:
+            self.make_constant_int(op.result, 0)
+            return
         if v1.intbound.known_ge(IntBound(0, 0)) and v2.is_constant():
             val = v2.box.getint()
             if val & (val - 1) == 0 and val > 0: # val == 2**shift
@@ -462,6 +470,14 @@
         self.optimizer.opaque_pointers[value] = True
         self.make_equal_to(op.result, value)
 
+    def optimize_CAST_PTR_TO_INT(self, op):
+        self.pure(rop.CAST_INT_TO_PTR, [op.result], op.getarg(0))
+        self.emit_operation(op)
+
+    def optimize_CAST_INT_TO_PTR(self, op):
+        self.pure(rop.CAST_PTR_TO_INT, [op.result], op.getarg(0))
+        self.emit_operation(op)
+
 dispatch_opt = make_dispatcher_method(OptRewrite, 'optimize_',
         default=OptRewrite.emit_operation)
 optimize_guards = _findall(OptRewrite, 'optimize_', 'GUARD')
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -508,13 +508,13 @@
         ops = """
         [p0]
         guard_class(p0, ConstClass(node_vtable)) []
-        i0 = ptr_ne(p0, NULL)
+        i0 = instance_ptr_ne(p0, NULL)
         guard_true(i0) []
-        i1 = ptr_eq(p0, NULL)
+        i1 = instance_ptr_eq(p0, NULL)
         guard_false(i1) []
-        i2 = ptr_ne(NULL, p0)
+        i2 = instance_ptr_ne(NULL, p0)
         guard_true(i0) []
-        i3 = ptr_eq(NULL, p0)
+        i3 = instance_ptr_eq(NULL, p0)
         guard_false(i1) []
         jump(p0)
         """
@@ -2026,7 +2026,7 @@
         ops = """
         [p1]
         guard_class(p1, ConstClass(node_vtable2)) []
-        i = ptr_ne(ConstPtr(myptr), p1)
+        i = instance_ptr_ne(ConstPtr(myptr), p1)
         guard_true(i) []
         jump(p1)
         """
@@ -2181,6 +2181,17 @@
         """
         self.optimize_loop(ops, expected)
 
+        ops = """
+        [i0]
+        i1 = int_floordiv(0, i0)
+        jump(i1)
+        """
+        expected = """
+        [i0]
+        jump(0)
+        """
+        self.optimize_loop(ops, expected)
+
     def test_fold_partially_constant_ops_ovf(self):
         ops = """
         [i0]
@@ -4789,6 +4800,18 @@
         """
         self.optimize_strunicode_loop(ops, expected)
 
+    def test_ptr_eq_str_constant(self):
+        ops = """
+        []
+        i0 = ptr_eq(s"abc", s"\x00")
+        finish(i0)
+        """
+        expected = """
+        []
+        finish(0)
+        """
+        self.optimize_loop(ops, expected)
+
 
 class TestLLtype(BaseTestOptimizeBasic, LLtypeMixin):
     pass
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -234,6 +234,30 @@
             """ % expected_value
             self.optimize_loop(ops, expected)
 
+    def test_reverse_of_cast(self):
+        ops = """
+        [i0]
+        p0 = cast_int_to_ptr(i0)
+        i1 = cast_ptr_to_int(p0)
+        jump(i1)
+        """
+        expected = """
+        [i0]
+        jump(i0)
+        """
+        self.optimize_loop(ops, expected)
+        ops = """
+        [p0]
+        i1 = cast_ptr_to_int(p0)
+        p1 = cast_int_to_ptr(i1)
+        jump(p1)
+        """
+        expected = """
+        [p0]
+        jump(p0)
+        """
+        self.optimize_loop(ops, expected)
+
     # ----------
 
     def test_remove_guard_class_1(self):
@@ -2659,7 +2683,7 @@
         ops = """
         [p1]
         guard_class(p1, ConstClass(node_vtable2)) []
-        i = ptr_ne(ConstPtr(myptr), p1)
+        i = instance_ptr_ne(ConstPtr(myptr), p1)
         guard_true(i) []
         jump(p1)
         """
@@ -3307,7 +3331,7 @@
         jump(p1, i1, i2, i6)
         '''
         self.optimize_loop(ops, expected, preamble)
-        
+
 
     # ----------
 
@@ -7256,7 +7280,7 @@
         ops = """
         [p1, p2]
         setarrayitem_gc(p1, 2, 10, descr=arraydescr)
-        setarrayitem_gc(p2, 3, 13, descr=arraydescr)        
+        setarrayitem_gc(p2, 3, 13, descr=arraydescr)
         call(0, p1, p2, 0, 0, 10, descr=arraycopydescr)
         jump(p1, p2)
         """
diff --git a/pypy/jit/metainterp/optimizeopt/virtualize.py b/pypy/jit/metainterp/optimizeopt/virtualize.py
--- a/pypy/jit/metainterp/optimizeopt/virtualize.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualize.py
@@ -59,7 +59,7 @@
 
     def import_from(self, other, optimizer):
         raise NotImplementedError("should not be called at this level")
-    
+
 def get_fielddescrlist_cache(cpu):
     if not hasattr(cpu, '_optimizeopt_fielddescrlist_cache'):
         result = descrlist_dict()
@@ -113,7 +113,7 @@
         #
         if not we_are_translated():
             op.name = 'FORCE ' + self.source_op.name
-            
+
         if self._is_immutable_and_filled_with_constants(optforce):
             box = optforce.optimizer.constant_fold(op)
             self.make_constant(box)
@@ -239,12 +239,12 @@
         for index in range(len(self._items)):
             self._items[index] = self._items[index].force_at_end_of_preamble(already_forced, optforce)
         return self
-    
+
     def _really_force(self, optforce):
         assert self.source_op is not None
         if not we_are_translated():
             self.source_op.name = 'FORCE ' + self.source_op.name
-        optforce.emit_operation(self.source_op)        
+        optforce.emit_operation(self.source_op)
         self.box = box = self.source_op.result
         for index in range(len(self._items)):
             subvalue = self._items[index]
@@ -276,7 +276,7 @@
 
     def new(self):
         return OptVirtualize()
-        
+
     def make_virtual(self, known_class, box, source_op=None):
         vvalue = VirtualValue(self.optimizer.cpu, known_class, box, source_op)
         self.make_equal_to(box, vvalue)
@@ -386,7 +386,8 @@
 
     def optimize_NEW_ARRAY(self, op):
         sizebox = self.get_constant_box(op.getarg(0))
-        if sizebox is not None:
+        # For now we can't make arrays of structs virtual.
+        if sizebox is not None and not op.getdescr().is_array_of_structs():
             # if the original 'op' did not have a ConstInt as argument,
             # build a new one with the ConstInt argument
             if not isinstance(op.getarg(0), ConstInt):
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -36,6 +36,7 @@
 
 
 class MIFrame(object):
+    debug = False
 
     def __init__(self, metainterp):
         self.metainterp = metainterp
@@ -164,7 +165,7 @@
         if not we_are_translated():
             for b in registers[count:]:
                 assert not oldbox.same_box(b)
-                
+
 
     def make_result_of_lastop(self, resultbox):
         got_type = resultbox.type
@@ -198,7 +199,7 @@
                     'float_add', 'float_sub', 'float_mul', 'float_truediv',
                     'float_lt', 'float_le', 'float_eq',
                     'float_ne', 'float_gt', 'float_ge',
-                    'ptr_eq', 'ptr_ne',
+                    'ptr_eq', 'ptr_ne', 'instance_ptr_eq', 'instance_ptr_ne',
                     ]:
         exec py.code.Source('''
             @arguments("box", "box")
@@ -548,6 +549,14 @@
     opimpl_getfield_gc_r_pure = _opimpl_getfield_gc_pure_any
     opimpl_getfield_gc_f_pure = _opimpl_getfield_gc_pure_any
 
+    @arguments("box", "box", "descr")
+    def _opimpl_getinteriorfield_gc_any(self, array, index, descr):
+        return self.execute_with_descr(rop.GETINTERIORFIELD_GC, descr,
+                                       array, index)
+    opimpl_getinteriorfield_gc_i = _opimpl_getinteriorfield_gc_any
+    opimpl_getinteriorfield_gc_f = _opimpl_getinteriorfield_gc_any
+    opimpl_getinteriorfield_gc_r = _opimpl_getinteriorfield_gc_any
+
     @specialize.arg(1)
     def _opimpl_getfield_gc_any_pureornot(self, opnum, box, fielddescr):
         tobox = self.metainterp.heapcache.getfield(box, fielddescr)
@@ -588,6 +597,15 @@
     opimpl_setfield_gc_r = _opimpl_setfield_gc_any
     opimpl_setfield_gc_f = _opimpl_setfield_gc_any
 
+    @arguments("box", "box", "box", "descr")
+    def _opimpl_setinteriorfield_gc_any(self, array, index, value, descr):
+        self.execute_with_descr(rop.SETINTERIORFIELD_GC, descr,
+                                array, index, value)
+    opimpl_setinteriorfield_gc_i = _opimpl_setinteriorfield_gc_any
+    opimpl_setinteriorfield_gc_f = _opimpl_setinteriorfield_gc_any
+    opimpl_setinteriorfield_gc_r = _opimpl_setinteriorfield_gc_any
+
+
     @arguments("box", "descr")
     def _opimpl_getfield_raw_any(self, box, fielddescr):
         return self.execute_with_descr(rop.GETFIELD_RAW, fielddescr, box)
@@ -2588,17 +2606,21 @@
         self.pc = position
         #
         if not we_are_translated():
-            print '\tpyjitpl: %s(%s)' % (name, ', '.join(map(repr, args))),
+            if self.debug:
+                print '\tpyjitpl: %s(%s)' % (name, ', '.join(map(repr, args))),
             try:
                 resultbox = unboundmethod(self, *args)
             except Exception, e:
-                print '-> %s!' % e.__class__.__name__
+                if self.debug:
+                    print '-> %s!' % e.__class__.__name__
                 raise
             if num_return_args == 0:
-                print
+                if self.debug:
+                    print
                 assert resultbox is None
             else:
-                print '-> %r' % (resultbox,)
+                if self.debug:
+                    print '-> %r' % (resultbox,)
                 assert argcodes[next_argcode] == '>'
                 result_argcode = argcodes[next_argcode + 1]
                 assert resultbox.type == {'i': history.INT,
diff --git a/pypy/jit/metainterp/quasiimmut.py b/pypy/jit/metainterp/quasiimmut.py
--- a/pypy/jit/metainterp/quasiimmut.py
+++ b/pypy/jit/metainterp/quasiimmut.py
@@ -2,6 +2,7 @@
 from pypy.rpython.lltypesystem import lltype, rclass
 from pypy.rpython.annlowlevel import cast_base_ptr_to_instance
 from pypy.jit.metainterp.history import AbstractDescr
+from pypy.rlib.objectmodel import we_are_translated
 
 
 def get_mutate_field_name(fieldname):
@@ -50,13 +51,13 @@
 
 class QuasiImmut(object):
     llopaque = True
+    compress_limit = 30
     
     def __init__(self, cpu):
         self.cpu = cpu
         # list of weakrefs to the LoopTokens that must be invalidated if
         # this value ever changes
         self.looptokens_wrefs = []
-        self.compress_limit = 30
 
     def hide(self):
         qmut_ptr = self.cpu.ts.cast_instance_to_base_ref(self)
@@ -75,6 +76,8 @@
     def compress_looptokens_list(self):
         self.looptokens_wrefs = [wref for wref in self.looptokens_wrefs
                                       if wref() is not None]
+        # NB. we must keep around the looptoken_wrefs that are
+        # already invalidated; see below
         self.compress_limit = (len(self.looptokens_wrefs) + 15) * 2
 
     def invalidate(self):
@@ -86,7 +89,16 @@
         for wref in wrefs:
             looptoken = wref()
             if looptoken is not None:
+                looptoken.invalidated = True
                 self.cpu.invalidate_loop(looptoken)
+                # NB. we must call cpu.invalidate_loop() even if
+                # looptoken.invalidated was already set to True.
+                # It's possible to invalidate several times the
+                # same looptoken; see comments in jit.backend.model
+                # in invalidate_loop().
+                if not we_are_translated():
+                    self.cpu.stats.invalidated_token_numbers.add(
+                        looptoken.number)
 
 
 class QuasiImmutDescr(AbstractDescr):
diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py
--- a/pypy/jit/metainterp/resoperation.py
+++ b/pypy/jit/metainterp/resoperation.py
@@ -1,5 +1,4 @@
 from pypy.rlib.objectmodel import we_are_translated
-from pypy.rlib.debug import make_sure_not_resized
 
 def ResOperation(opnum, args, result, descr=None):
     cls = opclasses[opnum]
@@ -405,8 +404,8 @@
     'FLOAT_TRUEDIV/2',
     'FLOAT_NEG/1',
     'FLOAT_ABS/1',
-    'CAST_FLOAT_TO_INT/1',
-    'CAST_INT_TO_FLOAT/1',
+    'CAST_FLOAT_TO_INT/1',          # don't use for unsigned ints; we would
+    'CAST_INT_TO_FLOAT/1',          # need some messy code in the backend
     'CAST_FLOAT_TO_SINGLEFLOAT/1',
     'CAST_SINGLEFLOAT_TO_FLOAT/1',
     #
@@ -431,13 +430,15 @@
     'INT_IS_TRUE/1b',
     'INT_NEG/1',
     'INT_INVERT/1',
+    #
+    'SAME_AS/1',      # gets a Const or a Box, turns it into another Box
     'CAST_PTR_TO_INT/1',
     'CAST_INT_TO_PTR/1',
     #
-    'SAME_AS/1',      # gets a Const or a Box, turns it into another Box
-    #
     'PTR_EQ/2b',
     'PTR_NE/2b',
+    'INSTANCE_PTR_EQ/2b',
+    'INSTANCE_PTR_NE/2b',
     'CAST_OPAQUE_PTR/1b',
     #
     'ARRAYLEN_GC/1d',
@@ -457,6 +458,7 @@
 
     'GETARRAYITEM_GC/2d',
     'GETARRAYITEM_RAW/2d',
+    'GETINTERIORFIELD_GC/2d',
     'GETFIELD_GC/1d',
     'GETFIELD_RAW/1d',
     '_MALLOC_FIRST',
@@ -473,6 +475,7 @@
 
     'SETARRAYITEM_GC/3d',
     'SETARRAYITEM_RAW/3d',
+    'SETINTERIORFIELD_GC/3d',
     'SETFIELD_GC/2d',
     'SETFIELD_RAW/2d',
     'STRSETITEM/3',
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -3435,7 +3435,75 @@
             return sa
         res = self.meta_interp(f, [16])
         assert res == f(16)
-        
+
+    def test_ptr_eq(self):
+        myjitdriver = JitDriver(greens = [], reds = ["n", "x"])
+        class A(object):
+            def __init__(self, v):
+                self.v = v
+        def f(n, x):
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, x=x)
+                z = 0 / x
+                a1 = A("key")
+                a2 = A("\x00")
+                n -= [a1, a2][z].v is not a2.v
+            return n
+        res = self.meta_interp(f, [10, 1])
+        assert res == 0
+
+    def test_instance_ptr_eq(self):
+        myjitdriver = JitDriver(greens = [], reds = ["n", "i", "a1", "a2"])
+        class A(object):
+            pass
+        def f(n):
+            a1 = A()
+            a2 = A()
+            i = 0
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, i=i, a1=a1, a2=a2)
+                if n % 2:
+                    a = a2
+                else:
+                    a = a1
+                i += a is a1
+                n -= 1
+            return i
+        res = self.meta_interp(f, [10])
+        assert res == f(10)
+        def f(n):
+            a1 = A()
+            a2 = A()
+            i = 0
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, i=i, a1=a1, a2=a2)
+                if n % 2:
+                    a = a2
+                else:
+                    a = a1
+                if a is a2:
+                    i += 1
+                n -= 1
+            return i
+        res = self.meta_interp(f, [10])
+        assert res == f(10)
+
+    def test_virtual_array_of_structs(self):
+        myjitdriver = JitDriver(greens = [], reds=["n", "d"])
+        def f(n):
+            d = None
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, d=d)
+                d = {}
+                if n % 2:
+                    d["k"] = n
+                else:
+                    d["z"] = n
+                n -= len(d)
+            return n
+        res = self.meta_interp(f, [10])
+        assert res == 0
+
 
 
 class TestLLtype(BaseLLtypeTests, LLJitMixin):
diff --git a/pypy/jit/metainterp/test/test_dict.py b/pypy/jit/metainterp/test/test_dict.py
--- a/pypy/jit/metainterp/test/test_dict.py
+++ b/pypy/jit/metainterp/test/test_dict.py
@@ -91,7 +91,7 @@
         res1 = f(100)
         res2 = self.meta_interp(f, [100], listops=True)
         assert res1 == res2
-        self.check_loops(int_mod=1) # the hash was traced
+        self.check_loops(int_mod=1) # the hash was traced and eq, but cached
 
     def test_dict_setdefault(self):
         myjitdriver = JitDriver(greens = [], reds = ['total', 'dct'])
@@ -128,7 +128,7 @@
         assert f(100) == 50
         res = self.meta_interp(f, [100], listops=True)
         assert res == 50
-        self.check_loops(int_mod=1)
+        self.check_loops(int_mod=1) # key + eq, but cached
 
     def test_repeated_lookup(self):
         myjitdriver = JitDriver(greens = [], reds = ['n', 'd'])
@@ -153,10 +153,12 @@
 
         res = self.meta_interp(f, [100], listops=True)
         assert res == f(50)
-        self.check_loops({"call": 7, "guard_false": 1, "guard_no_exception": 6,
+        self.check_loops({"call": 5, "getfield_gc": 1, "getinteriorfield_gc": 1,
+                          "guard_false": 1, "guard_no_exception": 4,
                           "guard_true": 1, "int_and": 1, "int_gt": 1,
                           "int_is_true": 1, "int_sub": 1, "jump": 1,
-                          "new_with_vtable": 1, "setfield_gc": 1})
+                          "new_with_vtable": 1, "new": 1, "new_array": 1,
+                          "setfield_gc": 3, })
 
 
 class TestOOtype(DictTests, OOJitMixin):
diff --git a/pypy/jit/metainterp/test/test_float.py b/pypy/jit/metainterp/test/test_float.py
--- a/pypy/jit/metainterp/test/test_float.py
+++ b/pypy/jit/metainterp/test/test_float.py
@@ -1,5 +1,6 @@
-import math
+import math, sys
 from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin
+from pypy.rlib.rarithmetic import intmask, r_uint
 
 
 class FloatTests:
@@ -45,6 +46,34 @@
         res = self.interp_operations(f, [-2.0])
         assert res == -8.5
 
+    def test_cast_float_to_int(self):
+        def g(f):
+            return int(f)
+        res = self.interp_operations(g, [-12345.9])
+        assert res == -12345
+
+    def test_cast_float_to_uint(self):
+        def g(f):
+            return intmask(r_uint(f))
+        res = self.interp_operations(g, [sys.maxint*2.0])
+        assert res == intmask(long(sys.maxint*2.0))
+        res = self.interp_operations(g, [-12345.9])
+        assert res == -12345
+
+    def test_cast_int_to_float(self):
+        def g(i):
+            return float(i)
+        res = self.interp_operations(g, [-12345])
+        assert type(res) is float and res == -12345.0
+
+    def test_cast_uint_to_float(self):
+        def g(i):
+            return float(r_uint(i))
+        res = self.interp_operations(g, [intmask(sys.maxint*2)])
+        assert type(res) is float and res == float(sys.maxint*2)
+        res = self.interp_operations(g, [-12345])
+        assert type(res) is float and res == float(long(r_uint(-12345)))
+
 
 class TestOOtype(FloatTests, OOJitMixin):
     pass
diff --git a/pypy/jit/metainterp/test/test_memmgr.py b/pypy/jit/metainterp/test/test_memmgr.py
--- a/pypy/jit/metainterp/test/test_memmgr.py
+++ b/pypy/jit/metainterp/test/test_memmgr.py
@@ -18,6 +18,7 @@
 
 class FakeLoopToken:
     generation = 0
+    invalidated = False
 
 
 class _TestMemoryManager:
diff --git a/pypy/jit/metainterp/test/test_quasiimmut.py b/pypy/jit/metainterp/test/test_quasiimmut.py
--- a/pypy/jit/metainterp/test/test_quasiimmut.py
+++ b/pypy/jit/metainterp/test/test_quasiimmut.py
@@ -48,6 +48,13 @@
 
 class QuasiImmutTests(object):
 
+    def setup_method(self, meth):
+        self.prev_compress_limit = QuasiImmut.compress_limit
+        QuasiImmut.compress_limit = 1
+
+    def teardown_method(self, meth):
+        QuasiImmut.compress_limit = self.prev_compress_limit
+
     def test_simple_1(self):
         myjitdriver = JitDriver(greens=['foo'], reds=['x', 'total'])
         class Foo:
@@ -289,7 +296,7 @@
             return total
 
         res = self.meta_interp(main, [])
-        self.check_loop_count(9)
+        self.check_tree_loop_count(6)
         assert res == main()
 
     def test_change_during_running(self):
@@ -317,7 +324,7 @@
         assert f(100, 15) == 3009
         res = self.meta_interp(f, [100, 15])
         assert res == 3009
-        self.check_loops(guard_not_invalidated=2, getfield_gc=0,
+        self.check_loops(guard_not_invalidated=4, getfield_gc=0,
                          call_may_force=0, guard_not_forced=0)
 
     def test_list_simple_1(self):
@@ -453,10 +460,30 @@
         assert f(100, 15) == 3009
         res = self.meta_interp(f, [100, 15])
         assert res == 3009
-        self.check_loops(guard_not_invalidated=2, getfield_gc=0,
+        self.check_loops(guard_not_invalidated=4, getfield_gc=0,
                          getarrayitem_gc=0, getarrayitem_gc_pure=0,
                          call_may_force=0, guard_not_forced=0)
 
+    def test_invalidated_loop_is_not_used_any_more_as_target(self):
+        myjitdriver = JitDriver(greens=['foo'], reds=['x'])
+        class Foo:
+            _immutable_fields_ = ['step?']
+        @dont_look_inside
+        def residual(x, foo):
+            if x == 20:
+                foo.step = 1
+        def f(x):
+            foo = Foo()
+            foo.step = 2
+            while x > 0:
+                myjitdriver.jit_merge_point(foo=foo, x=x)
+                residual(x, foo)
+                x -= foo.step
+            return foo.step
+        res = self.meta_interp(f, [60])
+        assert res == 1
+        self.check_tree_loop_count(4)   # at least not 2 like before
+
 
 class TestLLtypeGreenFieldsTests(QuasiImmutTests, LLJitMixin):
     pass
diff --git a/pypy/jit/metainterp/test/test_resume.py b/pypy/jit/metainterp/test/test_resume.py
--- a/pypy/jit/metainterp/test/test_resume.py
+++ b/pypy/jit/metainterp/test/test_resume.py
@@ -1,3 +1,4 @@
+from __future__ import with_statement
 import py
 from pypy.rpython.lltypesystem import lltype, llmemory, rffi
 from pypy.jit.metainterp.optimizeopt.optimizer import OptValue
diff --git a/pypy/jit/metainterp/warmspot.py b/pypy/jit/metainterp/warmspot.py
--- a/pypy/jit/metainterp/warmspot.py
+++ b/pypy/jit/metainterp/warmspot.py
@@ -206,7 +206,7 @@
         self.make_enter_functions()
         self.rewrite_jit_merge_points(policy)
 
-        verbose = not self.cpu.translate_support_code
+        verbose = False # not self.cpu.translate_support_code
         self.codewriter.make_jitcodes(verbose=verbose)
         self.rewrite_can_enter_jits()
         self.rewrite_set_param()
diff --git a/pypy/jit/metainterp/warmstate.py b/pypy/jit/metainterp/warmstate.py
--- a/pypy/jit/metainterp/warmstate.py
+++ b/pypy/jit/metainterp/warmstate.py
@@ -178,7 +178,7 @@
         if self.compiled_merge_points_wref is not None:
             for wref in self.compiled_merge_points_wref:
                 looptoken = wref()
-                if looptoken is not None:
+                if looptoken is not None and not looptoken.invalidated:
                     result.append(looptoken)
         return result
 
diff --git a/pypy/module/__builtin__/__init__.py b/pypy/module/__builtin__/__init__.py
--- a/pypy/module/__builtin__/__init__.py
+++ b/pypy/module/__builtin__/__init__.py
@@ -23,6 +23,7 @@
         'map'           : 'app_functional.map',
         'reduce'        : 'app_functional.reduce',
         'filter'        : 'app_functional.filter',
+        'zip'           : 'app_functional.zip',
         'vars'          : 'app_inspect.vars',
         'dir'           : 'app_inspect.dir',
 
@@ -89,7 +90,6 @@
         'enumerate'     : 'functional.W_Enumerate',
         'min'           : 'functional.min',
         'max'           : 'functional.max',
-        'zip'           : 'functional.zip',
         'reversed'      : 'functional.reversed',
         'super'         : 'descriptor.W_Super',
         'staticmethod'  : 'descriptor.StaticMethod',
@@ -119,7 +119,7 @@
            builtin = space.interpclass_w(w_builtin)
            if isinstance(builtin, module.Module):
                return builtin
-       # no builtin! make a default one.  Given them None, at least.
+       # no builtin! make a default one.  Give them None, at least.
        builtin = module.Module(space, None)
        space.setitem(builtin.w_dict, space.wrap('None'), space.w_None)
        return builtin
diff --git a/pypy/module/__builtin__/app_functional.py b/pypy/module/__builtin__/app_functional.py
--- a/pypy/module/__builtin__/app_functional.py
+++ b/pypy/module/__builtin__/app_functional.py
@@ -162,4 +162,21 @@
         item = seq[i]
         if func(item):
             result.append(item)
-    return tuple(result)
\ No newline at end of file
+    return tuple(result)
+
+def zip(*sequences):
+    """zip(seq1 [, seq2 [...]]) -> [(seq1[0], seq2[0] ...), (...)]
+
+Return a list of tuples, where each tuple contains the i-th element
+from each of the argument sequences.  The returned list is truncated
+in length to the length of the shortest argument sequence."""
+    if not sequences:
+        return []
+    result = []
+    iterators = [iter(seq) for seq in sequences]
+    while True:
+        try:
+            items = [next(it) for it in iterators]
+        except StopIteration:
+            return result
+        result.append(tuple(items))
diff --git a/pypy/module/__builtin__/app_io.py b/pypy/module/__builtin__/app_io.py
--- a/pypy/module/__builtin__/app_io.py
+++ b/pypy/module/__builtin__/app_io.py
@@ -27,7 +27,20 @@
     co = compile(source.rstrip()+"\n", filename, 'exec')
     exec co in glob, loc
 
-def raw_input(prompt=None):
+def _write_prompt(stdout, prompt):
+    print >> stdout, prompt,
+    try:
+        flush = stdout.flush
+    except AttributeError:
+        pass
+    else:
+        flush()
+    try:
+        stdout.softspace = 0
+    except (AttributeError, TypeError):
+        pass
+
+def raw_input(prompt=''):
     """raw_input([prompt]) -> string
 
 Read a string from standard input.  The trailing newline is stripped.
@@ -47,18 +60,10 @@
     if (hasattr(sys, '__raw_input__') and
         isinstance(stdin, file)  and stdin.fileno() == 0 and stdin.isatty() and
         isinstance(stdout, file) and stdout.fileno() == 1):
-        if prompt is None:
-            prompt = ''
-        return sys.__raw_input__(prompt)
+        _write_prompt(stdout, '')
+        return sys.__raw_input__(str(prompt))
 
-    if prompt is not None:
-        stdout.write(prompt)
-        try:
-            flush = stdout.flush
-        except AttributeError:
-            pass
-        else:
-            flush()
+    _write_prompt(stdout, prompt)
     line = stdin.readline()
     if not line:    # inputting an empty line gives line == '\n'
         raise EOFError
@@ -66,7 +71,7 @@
         return line[:-1]
     return line
 
-def input(prompt=None):
+def input(prompt=''):
     """Equivalent to eval(raw_input(prompt))."""
     return eval(raw_input(prompt))
 
diff --git a/pypy/module/__builtin__/compiling.py b/pypy/module/__builtin__/compiling.py
--- a/pypy/module/__builtin__/compiling.py
+++ b/pypy/module/__builtin__/compiling.py
@@ -97,6 +97,9 @@
     elif space.is_w(w_locals, space.w_None):
         w_locals = w_globals
 
-    space.builtin.pick_builtin(w_globals)
+    # xxx removed: adding '__builtins__' to the w_globals dict, if there
+    # is none.  This logic was removed as costly (it requires to get at
+    # the gettopframe_nohidden()).  I bet no test fails, and it's a really
+    # obscure case.
 
     return codeobj.exec_code(space, w_globals, w_locals)
diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py
--- a/pypy/module/__builtin__/functional.py
+++ b/pypy/module/__builtin__/functional.py
@@ -201,27 +201,6 @@
     """
     return min_max(space, __args__, "min")
 
- at unwrap_spec(sequences_w="args_w")
-def zip(space, sequences_w):
-    """Return a list of tuples, where the nth tuple contains every nth item of
-    each collection.
-
-    If the collections have different lengths, zip returns a list as long as the
-    shortest collection, ignoring the trailing items in the other collections.
-    """
-    if not sequences_w:
-        return space.newlist([])
-    result_w = []
-    iterators_w = [space.iter(w_seq) for w_seq in sequences_w]
-    while True:
-        try:
-            items_w = [space.next(w_it) for w_it in iterators_w]
-        except OperationError, e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            return space.newlist(result_w)
-        result_w.append(space.newtuple(items_w))
-
 class W_Enumerate(Wrappable):
 
     def __init__(self, w_iter, w_start):
diff --git a/pypy/module/__builtin__/test/test_rawinput.py b/pypy/module/__builtin__/test/test_rawinput.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/__builtin__/test/test_rawinput.py
@@ -0,0 +1,80 @@
+import autopath
+
+
+class AppTestRawInput():
+
+    def test_input_and_raw_input(self):
+        import sys, StringIO
+        for prompt, expected in [("def:", "abc/ def:/ghi\n"),
+                                 ("", "abc/ /ghi\n"),
+                                 (42, "abc/ 42/ghi\n"),
+                                 (None, "abc/ None/ghi\n"),
+                                 (Ellipsis, "abc/ /ghi\n")]:
+            for inputfn, inputtext, gottext in [
+                    (raw_input, "foo\nbar\n", "foo"),
+                    (input, "40+2\n", 42)]:
+                save = sys.stdin, sys.stdout
+                try:
+                    sys.stdin = StringIO.StringIO(inputtext)
+                    out = sys.stdout = StringIO.StringIO()
+                    print "abc",    # softspace = 1
+                    out.write('/')
+                    if prompt is Ellipsis:
+                        got = inputfn()
+                    else:
+                        got = inputfn(prompt)
+                    out.write('/')
+                    print "ghi"
+                finally:
+                    sys.stdin, sys.stdout = save
+                assert out.getvalue() == expected
+                assert got == gottext
+
+    def test_softspace(self):
+        import sys
+        import StringIO
+        fin = StringIO.StringIO()
+        fout = StringIO.StringIO()
+
+        fin.write("Coconuts\n")
+        fin.seek(0)
+
+        sys_stdin_orig = sys.stdin
+        sys_stdout_orig = sys.stdout
+
+        sys.stdin = fin
+        sys.stdout = fout
+
+        print "test",
+        raw_input("test")
+
+        sys.stdin = sys_stdin_orig
+        sys.stdout = sys_stdout_orig
+
+        fout.seek(0)
+        assert fout.read() == "test test"
+
+    def test_softspace_carryover(self):
+        import sys
+        import StringIO
+        fin = StringIO.StringIO()
+        fout = StringIO.StringIO()
+
+        fin.write("Coconuts\n")
+        fin.seek(0)
+
+        sys_stdin_orig = sys.stdin
+        sys_stdout_orig = sys.stdout
+
+        sys.stdin = fin
+        sys.stdout = fout
+
+        print "test",
+        raw_input("test")
+        print "test",
+
+        sys.stdin = sys_stdin_orig
+        sys.stdout = sys_stdout_orig
+
+        fout.seek(0)
+        assert fout.read() == "test testtest"
diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -19,7 +19,7 @@
 class W_RSocket(Wrappable, RSocket):
     def __del__(self):
         self.clear_all_weakrefs()
-        self.close()
+        RSocket.__del__(self)
 
     def accept_w(self, space):
         """accept() -> (socket object, address info)
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -211,7 +211,9 @@
             return result
 
         def __del__(self):
-            self.clear_all_weakrefs()
+            # note that we don't call clear_all_weakrefs here because
+            # an array with freed buffer is ok to see - it's just empty with 0
+            # length
             self.setlen(0)
 
         def setlen(self, size):
diff --git a/pypy/module/array/test/test_array.py b/pypy/module/array/test/test_array.py
--- a/pypy/module/array/test/test_array.py
+++ b/pypy/module/array/test/test_array.py
@@ -824,6 +824,22 @@
         r = weakref.ref(a)
         assert r() is a
 
+    def test_subclass_del(self):
+        import array, gc, weakref
+        l = []
+        
+        class A(array.array):
+            pass
+
+        a = A('d')
+        a.append(3.0)
+        r = weakref.ref(a, lambda a: l.append(a()))
+        del a
+        gc.collect()
+        assert l
+        assert l[0] is None or len(l[0]) == 0
+
+
 class TestCPythonsOwnArray(BaseArrayTests):
 
     def setup_class(cls):
@@ -844,11 +860,7 @@
         cls.w_tempfile = cls.space.wrap(
             str(py.test.ensuretemp('array').join('tmpfile')))
         cls.w_maxint = cls.space.wrap(sys.maxint)
-
-
-
-
-
+    
     def test_buffer_info(self):
         a = self.array('c', 'Hi!')
         bi = a.buffer_info()
diff --git a/pypy/module/pyexpat/__init__.py b/pypy/module/pyexpat/__init__.py
--- a/pypy/module/pyexpat/__init__.py
+++ b/pypy/module/pyexpat/__init__.py
@@ -4,12 +4,8 @@
 
 class ErrorsModule(MixedModule):
     "Definition of pyexpat.errors module."
-
-    appleveldefs = {
-        }
-
-    interpleveldefs = {
-        }
+    appleveldefs = {}
+    interpleveldefs = {}
 
     def setup_after_space_initialization(self):
         from pypy.module.pyexpat import interp_pyexpat
@@ -18,6 +14,18 @@
                     interp_pyexpat.ErrorString(self.space,
                     getattr(interp_pyexpat, name)))
 
+class ModelModule(MixedModule):
+    "Definition of pyexpat.model module."
+    appleveldefs = {}
+    interpleveldefs = {}
+
+    def setup_after_space_initialization(self):
+        from pypy.module.pyexpat import interp_pyexpat
+        space = self.space
+        for name in interp_pyexpat.xml_model_list:
+            value = getattr(interp_pyexpat, name)
+            space.setattr(self, space.wrap(name), space.wrap(value))
+
 class Module(MixedModule):
     "Python wrapper for Expat parser."
 
@@ -39,6 +47,7 @@
 
     submodules = {
         'errors': ErrorsModule,
+        'model':  ModelModule,
     }
 
     for name in ['XML_PARAM_ENTITY_PARSING_NEVER',
diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py
--- a/pypy/module/pyexpat/interp_pyexpat.py
+++ b/pypy/module/pyexpat/interp_pyexpat.py
@@ -76,6 +76,18 @@
     "XML_ERROR_FINISHED",
     "XML_ERROR_SUSPEND_PE",
     ]
+xml_model_list = [
+    "XML_CTYPE_EMPTY",
+    "XML_CTYPE_ANY",
+    "XML_CTYPE_MIXED",
+    "XML_CTYPE_NAME",
+    "XML_CTYPE_CHOICE",
+    "XML_CTYPE_SEQ",
+    "XML_CQUANT_NONE",
+    "XML_CQUANT_OPT",
+    "XML_CQUANT_REP",
+    "XML_CQUANT_PLUS",
+    ]
 
 class CConfigure:
     _compilation_info_ = eci
@@ -104,6 +116,8 @@
 
     for name in xml_error_list:
         locals()[name] = rffi_platform.ConstantInteger(name)
+    for name in xml_model_list:
+        locals()[name] = rffi_platform.ConstantInteger(name)
 
 for k, v in rffi_platform.configure(CConfigure).items():
     globals()[k] = v
diff --git a/pypy/module/pyexpat/test/test_parser.py b/pypy/module/pyexpat/test/test_parser.py
--- a/pypy/module/pyexpat/test/test_parser.py
+++ b/pypy/module/pyexpat/test/test_parser.py
@@ -131,3 +131,7 @@
                'encoding specified in XML declaration is incorrect')
         assert (pyexpat.errors.XML_ERROR_XML_DECL ==
                 'XML declaration not well-formed')
+
+    def test_model(self):
+        import pyexpat
+        assert isinstance(pyexpat.model.XML_CTYPE_EMPTY, int)
diff --git a/pypy/module/pypyjit/policy.py b/pypy/module/pypyjit/policy.py
--- a/pypy/module/pypyjit/policy.py
+++ b/pypy/module/pypyjit/policy.py
@@ -16,7 +16,8 @@
         if modname in ['pypyjit', 'signal', 'micronumpy', 'math', 'exceptions',
                        'imp', 'sys', 'array', '_ffi', 'itertools', 'operator',
                        'posix', '_socket', '_sre', '_lsprof', '_weakref',
-                       '__pypy__', 'cStringIO', '_collections', 'struct']:
+                       '__pypy__', 'cStringIO', '_collections', 'struct',
+                       'mmap']:
             return True
         return False
 
diff --git a/pypy/module/pypyjit/test_pypy_c/model.py b/pypy/module/pypyjit/test_pypy_c/model.py
--- a/pypy/module/pypyjit/test_pypy_c/model.py
+++ b/pypy/module/pypyjit/test_pypy_c/model.py
@@ -387,8 +387,8 @@
                 return ''
             text = str(py.code.Source(src).deindent().indent())
             lines = text.splitlines(True)
-            if opindex is not None and 0 <= opindex < len(lines):
-                lines[opindex] = lines[opindex].rstrip() + '\t<=====\n'
+            if opindex is not None and 0 <= opindex <= len(lines):
+                lines.insert(opindex, '\n\t===== HERE =====\n')
             return ''.join(lines)
         #
         expected_src = self.preprocess_expected_src(expected_src)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_containers.py b/pypy/module/pypyjit/test_pypy_c/test_containers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_containers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_containers.py
@@ -46,7 +46,7 @@
         assert loop.match_by_id("getitem", """
             i28 = call(ConstClass(ll_dict_lookup__dicttablePtr_objectPtr_Signed), p18, p6, i25, descr=...)
             ...
-            p33 = call(ConstClass(ll_get_value__dicttablePtr_Signed), p18, i28, descr=...)
+            p33 = getinteriorfield_gc(p31, i26, descr=<InteriorFieldDescr <GcPtrFieldDescr dictentry.value .*>>)
             ...
         """)
 
diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -41,7 +41,7 @@
             guard_true(i32, descr=...)
             i34 = int_add(i6, 1)
             --TICK--
-            jump(p0, p1, p2, p3, p4, p5, i34, p7, p8, i9, i10, p11, i12, p13, descr=<Loop4>)
+            jump(p0, p1, p2, p3, p4, p5, i34, p7, p8, i9, i10, p11, i12, p13, descr=...)
         """)
 
     def test_long(self):
@@ -93,7 +93,8 @@
             i46 = call(ConstClass(ll_startswith__rpy_stringPtr_rpy_stringPtr), p28, ConstPtr(ptr45), descr=<BoolCallDescr>)
             guard_false(i46, descr=...)
             p51 = new_with_vtable(21136408)
-            setfield_gc(p51, _, descr=...)    # 6 setfields, but the order is dict-order-dependent
+            setfield_gc(p51, _, descr=...)    # 7 setfields, but the order is dict-order-dependent
+            setfield_gc(p51, _, descr=...)
             setfield_gc(p51, _, descr=...)
             setfield_gc(p51, _, descr=...)
             setfield_gc(p51, _, descr=...)
@@ -106,7 +107,7 @@
             i58 = int_add_ovf(i6, i57)
             guard_no_overflow(descr=...)
             --TICK--
-            jump(p0, p1, p2, p3, p4, p5, i58, i7, descr=<Loop4>)
+            jump(p0, p1, p2, p3, p4, p5, i58, i7, descr=...)
         """)
 
     def test_str_mod(self):
diff --git a/pypy/module/rctime/interp_time.py b/pypy/module/rctime/interp_time.py
--- a/pypy/module/rctime/interp_time.py
+++ b/pypy/module/rctime/interp_time.py
@@ -245,6 +245,9 @@
 if sys.platform != 'win32':
     @unwrap_spec(secs=float)
     def sleep(space, secs):
+        if secs < 0:
+            raise OperationError(space.w_IOError,
+                                 space.wrap("Invalid argument: negative time in sleep"))
         pytime.sleep(secs)
 else:
     from pypy.rlib import rwin32
@@ -265,6 +268,9 @@
                                    OSError(EINTR, "sleep() interrupted"))
     @unwrap_spec(secs=float)
     def sleep(space, secs):
+        if secs < 0:
+            raise OperationError(space.w_IOError,
+                                 space.wrap("Invalid argument: negative time in sleep"))
         # as decreed by Guido, only the main thread can be
         # interrupted.
         main_thread = space.fromcache(State).main_thread
diff --git a/pypy/module/rctime/test/test_rctime.py b/pypy/module/rctime/test/test_rctime.py
--- a/pypy/module/rctime/test/test_rctime.py
+++ b/pypy/module/rctime/test/test_rctime.py
@@ -20,8 +20,9 @@
         import sys
         import os
         raises(TypeError, rctime.sleep, "foo")
-        rctime.sleep(1.2345)
-        
+        rctime.sleep(0.12345)
+        raises(IOError, rctime.sleep, -1.0)
+
     def test_clock(self):
         import time as rctime
         rctime.clock()
diff --git a/pypy/objspace/std/newformat.py b/pypy/objspace/std/newformat.py
--- a/pypy/objspace/std/newformat.py
+++ b/pypy/objspace/std/newformat.py
@@ -120,6 +120,8 @@
             out.append_slice(s, last_literal, end)
             return out.build()
 
+        # This is only ever called if we're already unrolling _do_build_string
+        @jit.unroll_safe
         def _parse_field(self, start, end):
             s = self.template
             # Find ":" or "!"
@@ -149,6 +151,7 @@
                 i += 1
             return s[start:end], None, end
 
+        @jit.unroll_safe
         def _get_argument(self, name):
             # First, find the argument.
             space = self.space
@@ -207,6 +210,7 @@
                     raise OperationError(space.w_IndexError, w_msg)
             return self._resolve_lookups(w_arg, name, i, end)
 
+        @jit.unroll_safe
         def _resolve_lookups(self, w_obj, name, start, end):
             # Resolve attribute and item lookups.
             space = self.space
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -83,11 +83,12 @@
         if self.config.objspace.std.withtproxy:
             transparent.setup(self)
 
+        interplevel_classes = {}
         for type, classes in self.model.typeorder.iteritems():
-            if len(classes) >= 3:
+            if len(classes) >= 3: # XXX what does this 3 mean??!
                 # W_Root, AnyXxx and actual object
-                self.gettypefor(type).interplevel_cls = classes[0][0]
-
+                interplevel_classes[self.gettypefor(type)] = classes[0][0]
+        self._interplevel_classes = interplevel_classes
 
     def get_builtin_types(self):
         return self.builtin_types
@@ -579,7 +580,7 @@
             raise OperationError(self.w_TypeError,
                                  self.wrap("need type object"))
         if is_annotation_constant(w_type):
-            cls = w_type.interplevel_cls
+            cls = self._get_interplevel_cls(w_type)
             if cls is not None:
                 assert w_inst is not None
                 if isinstance(w_inst, cls):
@@ -589,3 +590,9 @@
     @specialize.arg_or_var(2)
     def isinstance_w(space, w_inst, w_type):
         return space._type_isinstance(w_inst, w_type)
+
+    @specialize.memo()
+    def _get_interplevel_cls(self, w_type):
+        if not hasattr(self, "_interplevel_classes"):
+            return None # before running initialize
+        return self._interplevel_classes.get(w_type, None)
diff --git a/pypy/objspace/std/smallintobject.py b/pypy/objspace/std/smallintobject.py
--- a/pypy/objspace/std/smallintobject.py
+++ b/pypy/objspace/std/smallintobject.py
@@ -12,6 +12,7 @@
 from pypy.rlib.rbigint import rbigint
 from pypy.rlib.rarithmetic import r_uint
 from pypy.tool.sourcetools import func_with_new_name
+from pypy.objspace.std.inttype import wrapint
 
 class W_SmallIntObject(W_Object, UnboxedValue):
     __slots__ = 'intval'
@@ -48,14 +49,36 @@
 def delegate_SmallInt2Complex(space, w_small):
     return space.newcomplex(float(w_small.intval), 0.0)
 
+def add__SmallInt_SmallInt(space, w_a, w_b):
+    return wrapint(space, w_a.intval + w_b.intval) # cannot overflow
+
+def sub__SmallInt_SmallInt(space, w_a, w_b):
+    return wrapint(space, w_a.intval - w_b.intval) # cannot overflow
+
+def floordiv__SmallInt_SmallInt(space, w_a, w_b):
+    return wrapint(space, w_a.intval // w_b.intval) # cannot overflow
+
+div__SmallInt_SmallInt = floordiv__SmallInt_SmallInt
+
+def mod__SmallInt_SmallInt(space, w_a, w_b):
+    return wrapint(space, w_a.intval % w_b.intval) # cannot overflow
+
+def divmod__SmallInt_SmallInt(space, w_a, w_b):
+    w = wrapint(space, w_a.intval // w_b.intval) # cannot overflow
+    z = wrapint(space, w_a.intval % w_b.intval)
+    return space.newtuple([w, z])
+
 def copy_multimethods(ns):
     """Copy integer multimethods for small int."""
     for name, func in intobject.__dict__.iteritems():
         if "__Int" in name:
             new_name = name.replace("Int", "SmallInt")
-            # Copy the function, so the annotator specializes it for
-            # W_SmallIntObject.
-            ns[new_name] = func_with_new_name(func, new_name)
+            if new_name not in ns:
+                # Copy the function, so the annotator specializes it for
+                # W_SmallIntObject.
+                ns[new_name] = func = func_with_new_name(func, new_name, globals=ns)
+        else:
+            ns[name] = func
     ns["get_integer"] = ns["pos__SmallInt"] = ns["int__SmallInt"]
     ns["get_negint"] = ns["neg__SmallInt"]
 
diff --git a/pypy/objspace/std/strutil.py b/pypy/objspace/std/strutil.py
--- a/pypy/objspace/std/strutil.py
+++ b/pypy/objspace/std/strutil.py
@@ -35,7 +35,7 @@
 
     def error(self):
         raise ParseStringError("invalid literal for %s() with base %d: '%s'" %
-                               (self.fname, self.base, self.literal))
+                               (self.fname, self.original_base, self.literal))
 
     def __init__(self, s, literal, base, fname):
         self.literal = literal
@@ -47,7 +47,8 @@
         elif s.startswith('+'):
             s = strip_spaces(s[1:])
         self.sign = sign
-        
+        self.original_base = base
+
         if base == 0:
             if s.startswith('0x') or s.startswith('0X'):
                 base = 16
diff --git a/pypy/objspace/std/test/test_obj.py b/pypy/objspace/std/test/test_obj.py
--- a/pypy/objspace/std/test/test_obj.py
+++ b/pypy/objspace/std/test/test_obj.py
@@ -102,3 +102,11 @@
             def __repr__(self):
                 return 123456
         assert A().__str__() == 123456
+
+def test_isinstance_shortcut():
+    from pypy.objspace.std import objspace
+    space = objspace.StdObjSpace()
+    w_a = space.wrap("a")
+    space.type = None
+    space.isinstance_w(w_a, space.w_str) # does not crash
+
diff --git a/pypy/objspace/std/test/test_strutil.py b/pypy/objspace/std/test/test_strutil.py
--- a/pypy/objspace/std/test/test_strutil.py
+++ b/pypy/objspace/std/test/test_strutil.py
@@ -89,6 +89,8 @@
 
         exc = raises(ParseStringError, string_to_int, '')
         assert exc.value.msg == "invalid literal for int() with base 10: ''"
+        exc = raises(ParseStringError, string_to_int, '', 0)
+        assert exc.value.msg == "invalid literal for int() with base 0: ''"
 
     def test_string_to_int_overflow(self):
         import sys
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -102,7 +102,6 @@
                           'instancetypedef',
                           'terminator',
                           '_version_tag?',
-                          'interplevel_cls',
                           ]
 
     # for config.objspace.std.getattributeshortcut
@@ -117,9 +116,6 @@
     # of the __new__ is an instance of the type
     w_bltin_new = None
 
-    interplevel_cls = None # not None for prebuilt instances of
-                           # interpreter-level types
-
     @dont_look_inside
     def __init__(w_self, space, name, bases_w, dict_w,
                  overridetypedef=None):
diff --git a/pypy/pytest.ini b/pypy/pytest.ini
--- a/pypy/pytest.ini
+++ b/pypy/pytest.ini
@@ -1,2 +1,2 @@
 [pytest]
-addopts = --assertmode=old
\ No newline at end of file
+addopts = --assertmode=old -rf
diff --git a/pypy/rlib/_rsocket_rffi.py b/pypy/rlib/_rsocket_rffi.py
--- a/pypy/rlib/_rsocket_rffi.py
+++ b/pypy/rlib/_rsocket_rffi.py
@@ -16,6 +16,7 @@
 _MINGW = target_platform.name == "mingw32"
 _SOLARIS = sys.platform == "sunos5"
 _MACOSX = sys.platform == "darwin"
+_HAS_AF_PACKET = sys.platform.startswith('linux')   # only Linux for now
 
 if _POSIX:
     includes = ('sys/types.h',
@@ -34,11 +35,12 @@
                 'stdint.h', 
                 'errno.h',
                 )
+    if _HAS_AF_PACKET:
+        includes += ('netpacket/packet.h',
+                     'sys/ioctl.h',
+                     'net/if.h')
 
-    cond_includes = [('AF_NETLINK', 'linux/netlink.h'),
-                     ('AF_PACKET', 'netpacket/packet.h'),
-                     ('AF_PACKET', 'sys/ioctl.h'),
-                     ('AF_PACKET', 'net/if.h')]
+    cond_includes = [('AF_NETLINK', 'linux/netlink.h')]
     
     libraries = ()
     calling_conv = 'c'
@@ -320,18 +322,18 @@
                                              ('events', rffi.SHORT),
                                              ('revents', rffi.SHORT)])
 
-    CConfig.sockaddr_ll = platform.Struct('struct sockaddr_ll',
+    if _HAS_AF_PACKET:
+        CConfig.sockaddr_ll = platform.Struct('struct sockaddr_ll',
                               [('sll_ifindex', rffi.INT),
                                ('sll_protocol', rffi.INT),
                                ('sll_pkttype', rffi.INT),
                                ('sll_hatype', rffi.INT),
                                ('sll_addr', rffi.CFixedArray(rffi.CHAR, 8)),
-                               ('sll_halen', rffi.INT)],
-                              ifdef='AF_PACKET')
+                               ('sll_halen', rffi.INT)])
 
-    CConfig.ifreq = platform.Struct('struct ifreq', [('ifr_ifindex', rffi.INT),
-                                 ('ifr_name', rffi.CFixedArray(rffi.CHAR, 8))],
-                                    ifdef='AF_PACKET')
+        CConfig.ifreq = platform.Struct('struct ifreq',
+                                [('ifr_ifindex', rffi.INT),
+                                 ('ifr_name', rffi.CFixedArray(rffi.CHAR, 8))])
 
 if _WIN32:
     CConfig.WSAEVENT = platform.SimpleType('WSAEVENT', rffi.VOIDP)
@@ -386,6 +388,8 @@
         constants[name] = value
     else:
         constants[name] = default
+if not _HAS_AF_PACKET and 'AF_PACKET' in constants:
+    del constants['AF_PACKET']
 
 constants['has_ipv6'] = True # This is a configuration option in CPython
 for name, value in constants.items():
@@ -439,21 +443,14 @@
 if _POSIX:
     nfds_t = cConfig.nfds_t
     pollfd = cConfig.pollfd
-    if cConfig.sockaddr_ll is not None:
+    if _HAS_AF_PACKET:
         sockaddr_ll = cConfig.sockaddr_ll
-    ifreq = cConfig.ifreq
+        ifreq = cConfig.ifreq
 if WIN32:
     WSAEVENT = cConfig.WSAEVENT
     WSANETWORKEVENTS = cConfig.WSANETWORKEVENTS
 timeval = cConfig.timeval
 
-#if _POSIX:
-#    includes = list(includes)
-#    for _name, _header in cond_includes:
-#        if getattr(cConfig, _name) is not None:
-#            includes.append(_header)
-#    eci = ExternalCompilationInfo(includes=includes, libraries=libraries,
-#                                  separate_module_sources=sources)
 
 def external(name, args, result, **kwds):
     return rffi.llexternal(name, args, result, compilation_info=eci,
@@ -544,7 +541,7 @@
     socketpair_t = rffi.CArray(socketfd_type)
     socketpair = external('socketpair', [rffi.INT, rffi.INT, rffi.INT,
                           lltype.Ptr(socketpair_t)], rffi.INT)
-    if ifreq is not None:
+    if _HAS_AF_PACKET:
         ioctl = external('ioctl', [socketfd_type, rffi.INT, lltype.Ptr(ifreq)],
                          rffi.INT)
 
diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -8,6 +8,8 @@
 from pypy.rpython.extregistry import ExtRegistryEntry
 from pypy.tool.sourcetools import func_with_new_name
 
+DEBUG_ELIDABLE_FUNCTIONS = False
+
 
 def elidable(func):
     """ Decorate a function as "trace-elidable". This means precisely that:
@@ -24,6 +26,18 @@
     If a particular call to this function ends up raising an exception, then it
     is handled like a normal function call (this decorator is ignored).
     """
+    if DEBUG_ELIDABLE_FUNCTIONS:
+        cache = {}
+        oldfunc = func
+        def func(*args):
+            result = oldfunc(*args)    # if it raises, no caching
+            try:
+                oldresult = cache.setdefault(args, result)
+            except TypeError:
+                pass           # unhashable args
+            else:
+                assert oldresult == result
+            return result
     func._elidable_function_ = True
     return func
 
diff --git a/pypy/rlib/rerased.py b/pypy/rlib/rerased.py
--- a/pypy/rlib/rerased.py
+++ b/pypy/rlib/rerased.py
@@ -135,6 +135,8 @@
     _about_ = erase_int
 
     def compute_result_annotation(self, s_obj):
+        config = self.bookkeeper.annotator.translator.config
+        assert config.translation.taggedpointers, "need to enable tagged pointers to use erase_int"
         assert annmodel.SomeInteger().contains(s_obj)
         return SomeErased()
 
@@ -228,6 +230,8 @@
 
     def convert_const(self, value):
         if value._identity is _identity_for_ints:
+            config = self.rtyper.annotator.translator.config
+            assert config.translation.taggedpointers, "need to enable tagged pointers to use erase_int"
             return lltype.cast_int_to_ptr(self.lowleveltype, value._x * 2 + 1)
         bk = self.rtyper.annotator.bookkeeper
         s_obj = value._identity.get_input_annotation(bk)
diff --git a/pypy/rlib/rgc.py b/pypy/rlib/rgc.py
--- a/pypy/rlib/rgc.py
+++ b/pypy/rlib/rgc.py
@@ -214,6 +214,10 @@
     func._gc_no_collect_ = True
     return func
 
+def is_light_finalizer(func):
+    func._is_light_finalizer_ = True
+    return func
+
 # ____________________________________________________________
 
 def get_rpy_roots():
diff --git a/pypy/rlib/rmmap.py b/pypy/rlib/rmmap.py
--- a/pypy/rlib/rmmap.py
+++ b/pypy/rlib/rmmap.py
@@ -292,6 +292,9 @@
         elif _POSIX:
             self.closed = True
             if self.fd != -1:
+                # XXX this is buggy - raising in an RPython del is not a good
+                #     idea, we should swallow the exception or ignore the
+                #     underlaying close error code
                 os.close(self.fd)
                 self.fd = -1
             if self.size > 0:
diff --git a/pypy/rlib/rsocket.py b/pypy/rlib/rsocket.py
--- a/pypy/rlib/rsocket.py
+++ b/pypy/rlib/rsocket.py
@@ -8,6 +8,7 @@
 # Known missing features:
 #
 #   - address families other than AF_INET, AF_INET6, AF_UNIX, AF_PACKET
+#   - AF_PACKET is only supported on Linux
 #   - methods makefile(),
 #   - SSL
 #
@@ -55,6 +56,7 @@
 
 
 _FAMILIES = {}
+
 class Address(object):
     """The base class for RPython-level objects representing addresses.
     Fields:  addr    - a _c.sockaddr_ptr (memory owned by the Address instance)
@@ -76,9 +78,8 @@
         self.addrlen = addrlen
 
     def __del__(self):
-        addr = self.addr_p
-        if addr:
-            lltype.free(addr, flavor='raw')
+        if self.addr_p:
+            lltype.free(self.addr_p, flavor='raw')
 
     def setdata(self, addr, addrlen):
         # initialize self.addr and self.addrlen.  'addr' can be a different
@@ -612,7 +613,10 @@
         self.timeout = defaults.timeout
         
     def __del__(self):
-        self.close()
+        fd = self.fd
+        if fd != _c.INVALID_SOCKET:
+            self.fd = _c.INVALID_SOCKET
+            _c.socketclose(fd)
 
     if hasattr(_c, 'fcntl'):
         def _setblocking(self, block):
diff --git a/pypy/rlib/test/test_rerased.py b/pypy/rlib/test/test_rerased.py
--- a/pypy/rlib/test/test_rerased.py
+++ b/pypy/rlib/test/test_rerased.py
@@ -10,6 +10,13 @@
 
 from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin
 
+def make_annotator():
+    a = RPythonAnnotator()
+    a.translator.config.translation.taggedpointers = True
+    return a
+
+
+
 class X(object):
     pass
 
@@ -55,7 +62,7 @@
 def test_annotate_1():
     def f():
         return eraseX(X())
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [])
     assert isinstance(s, SomeErased)
 
@@ -66,7 +73,7 @@
         #assert not is_integer(e)
         x2 = uneraseX(e)
         return x2
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [])
     assert isinstance(s, annmodel.SomeInstance)
     assert s.classdef == a.bookkeeper.getuniqueclassdef(X)
@@ -77,7 +84,7 @@
         #assert is_integer(e)
         x2 = unerase_int(e)
         return x2
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [])
     assert isinstance(s, annmodel.SomeInteger)
 
@@ -105,7 +112,7 @@
         x = make(n)
         return check(x, n)
     #
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [int])
     assert isinstance(s, annmodel.SomeInteger)
 
@@ -135,7 +142,7 @@
             else:
                 return inst
     #
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [])
     assert isinstance(s, annmodel.SomeInstance)
     assert s.classdef == a.bookkeeper.getuniqueclassdef(A)
@@ -155,7 +162,7 @@
             e = e2
         return unerase(e)
     #
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [int])
     assert isinstance(s, annmodel.SomeInstance)
     assert s.classdef == a.bookkeeper.getuniqueclassdef(X)
@@ -165,11 +172,14 @@
     e1 = erase_int(42)
     def f(i):
         return unerase_int(e1)
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [int])
     assert isinstance(s, annmodel.SomeInteger)
 
 class BaseTestRErased(BaseRtypingTest):
+    def interpret(self, *args, **kwargs):
+        kwargs["taggedpointers"] = True
+        return BaseRtypingTest.interpret(self, *args, **kwargs)
     def test_rtype_1(self):
         def f():
             return eraseX(X())
diff --git a/pypy/rpython/llinterp.py b/pypy/rpython/llinterp.py
--- a/pypy/rpython/llinterp.py
+++ b/pypy/rpython/llinterp.py
@@ -1095,13 +1095,6 @@
             assert y >= 0
         return self.op_int_add_ovf(x, y)
 
-    def op_cast_float_to_int(self, f):
-        assert type(f) is float
-        try:
-            return ovfcheck(int(f))
-        except OverflowError:
-            self.make_llexception()
-
     def op_int_is_true(self, x):
         # special case
         if type(x) is CDefinedIntSymbolic:
diff --git a/pypy/rpython/lltypesystem/ll2ctypes.py b/pypy/rpython/lltypesystem/ll2ctypes.py
--- a/pypy/rpython/lltypesystem/ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/ll2ctypes.py
@@ -15,12 +15,11 @@
     load_library_kwargs = {}
 
 import os
-from pypy import conftest
 from pypy.rpython.lltypesystem import lltype, llmemory
 from pypy.rpython.extfunc import ExtRegistryEntry
 from pypy.rlib.objectmodel import Symbolic, ComputedIntSymbolic
 from pypy.tool.uid import fixid
-from pypy.rlib.rarithmetic import r_uint, r_singlefloat, r_longfloat, base_int, intmask
+from pypy.rlib.rarithmetic import r_singlefloat, r_longfloat, base_int, intmask
 from pypy.annotation import model as annmodel
 from pypy.rpython.llinterp import LLInterpreter, LLException
 from pypy.rpython.lltypesystem.rclass import OBJECT, OBJECT_VTABLE
@@ -531,6 +530,10 @@
     def __str__(self):
         return repr(self)
 
+    def _setparentstructure(self, parent, parentindex):
+        super(_parentable_mixin, self)._setparentstructure(parent, parentindex)
+        self._keepparent = parent   # always keep a strong ref
+
 class _struct_mixin(_parentable_mixin):
     """Mixin added to _struct containers when they become ctypes-based."""
     __slots__ = ()
@@ -566,7 +569,10 @@
         return 0, sys.maxint
 
     def getitem(self, index, uninitialized_ok=False):
-        return self._storage.contents._getitem(index, boundscheck=False)
+        res = self._storage.contents._getitem(index, boundscheck=False)
+        if isinstance(self._TYPE.OF, lltype.ContainerType):
+            res._obj._setparentstructure(self, index)
+        return res
 
     def setitem(self, index, value):
         self._storage.contents._setitem(index, value, boundscheck=False)
@@ -1279,6 +1285,8 @@
             self.intval = intmask(void_p.value)
 
     def __eq__(self, other):
+        if not other:
+            return self.intval == 0
         if isinstance(other, _llgcopaque):
             return self.intval == other.intval
         storage = object()
diff --git a/pypy/rpython/lltypesystem/lloperation.py b/pypy/rpython/lltypesystem/lloperation.py
--- a/pypy/rpython/lltypesystem/lloperation.py
+++ b/pypy/rpython/lltypesystem/lloperation.py
@@ -343,8 +343,8 @@
     'cast_uint_to_float':   LLOp(canfold=True),
     'cast_longlong_to_float' :LLOp(canfold=True),
     'cast_ulonglong_to_float':LLOp(canfold=True),
-    'cast_float_to_int':    LLOp(canraise=(OverflowError,), tryfold=True),
-    'cast_float_to_uint':   LLOp(canfold=True),    # XXX need OverflowError?
+    'cast_float_to_int':    LLOp(canfold=True),
+    'cast_float_to_uint':   LLOp(canfold=True),
     'cast_float_to_longlong' :LLOp(canfold=True),
     'cast_float_to_ulonglong':LLOp(canfold=True),
     'truncate_longlong_to_int':LLOp(canfold=True),
diff --git a/pypy/rpython/lltypesystem/lltype.py b/pypy/rpython/lltypesystem/lltype.py
--- a/pypy/rpython/lltypesystem/lltype.py
+++ b/pypy/rpython/lltypesystem/lltype.py
@@ -1522,7 +1522,7 @@
             and parentindex in (self._parent_type._names[0], 0)
             and self._TYPE._gckind == typeOf(parent)._gckind):
             # keep strong reference to parent, we share the same allocation
-            self._keepparent = parent 
+            self._keepparent = parent
 
     def _parentstructure(self, check=True):
         if self._wrparent is not None:
@@ -1713,6 +1713,7 @@
         return v
 
     def setitem(self, index, value):
+        assert typeOf(value) == self._TYPE.OF
         self.items[index] = value
 
 assert not '__dict__' in dir(_array)
@@ -1730,7 +1731,8 @@
         # Keep the parent array alive, we share the same allocation.
         # Don't do it if we are inside a GC object, though -- it's someone
         # else's job to keep the GC object alive
-        if typeOf(top_container(parent))._gckind == 'raw':
+        if (typeOf(top_container(parent))._gckind == 'raw' or
+            hasattr(top_container(parent)._storage, 'contents')):  # ll2ctypes
             self._keepparent = parent
 
     def __str__(self):
diff --git a/pypy/rpython/lltypesystem/opimpl.py b/pypy/rpython/lltypesystem/opimpl.py
--- a/pypy/rpython/lltypesystem/opimpl.py
+++ b/pypy/rpython/lltypesystem/opimpl.py
@@ -355,6 +355,10 @@
     assert type(b) is bool
     return float(b)
 
+def op_cast_float_to_int(f):
+    assert type(f) is float
+    return intmask(int(f))
+
 def op_cast_float_to_uint(f):
     assert type(f) is float
     return r_uint(long(f))
diff --git a/pypy/rpython/lltypesystem/rbuilder.py b/pypy/rpython/lltypesystem/rbuilder.py
--- a/pypy/rpython/lltypesystem/rbuilder.py
+++ b/pypy/rpython/lltypesystem/rbuilder.py
@@ -29,7 +29,7 @@
         except OverflowError:
             raise MemoryError
         newbuf = mallocfn(new_allocated)
-        copycontentsfn(ll_builder.buf, newbuf, 0, 0, ll_builder.allocated)
+        copycontentsfn(ll_builder.buf, newbuf, 0, 0, ll_builder.used)
         ll_builder.buf = newbuf
         ll_builder.allocated = new_allocated
     return func_with_new_name(stringbuilder_grow, name)
@@ -56,7 +56,7 @@
 class BaseStringBuilderRepr(AbstractStringBuilderRepr):
     def empty(self):
         return nullptr(self.lowleveltype.TO)
-    
+
     @classmethod
     def ll_new(cls, init_size):
         if init_size < 0 or init_size > MAX:
diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py
--- a/pypy/rpython/lltypesystem/rdict.py
+++ b/pypy/rpython/lltypesystem/rdict.py
@@ -1,16 +1,14 @@
 from pypy.tool.pairtype import pairtype
-from pypy.annotation import model as annmodel
 from pypy.objspace.flow.model import Constant
-from pypy.rpython.rdict import AbstractDictRepr, AbstractDictIteratorRepr,\
-     rtype_newdict
+from pypy.rpython.rdict import (AbstractDictRepr, AbstractDictIteratorRepr,
+     rtype_newdict)
 from pypy.rpython.lltypesystem import lltype
+from pypy.rlib import objectmodel, jit
 from pypy.rlib.rarithmetic import r_uint, intmask, LONG_BIT
-from pypy.rlib.objectmodel import hlinvoke
-from pypy.rpython import robject
-from pypy.rlib import objectmodel, jit
 from pypy.rpython import rmodel
 from pypy.rpython.error import TyperError
 
+
 HIGHEST_BIT = intmask(1 << (LONG_BIT - 1))
 MASK = intmask(HIGHEST_BIT - 1)
 
@@ -417,17 +415,16 @@
     ENTRIES = lltype.typeOf(entries).TO
     return ENTRIES.fasthashfn(entries[i].key)
 
- at jit.dont_look_inside
 def ll_get_value(d, i):
     return d.entries[i].value
 
 def ll_keyhash_custom(d, key):
     DICT = lltype.typeOf(d).TO
-    return hlinvoke(DICT.r_rdict_hashfn, d.fnkeyhash, key)
+    return objectmodel.hlinvoke(DICT.r_rdict_hashfn, d.fnkeyhash, key)
 
 def ll_keyeq_custom(d, key1, key2):
     DICT = lltype.typeOf(d).TO
-    return hlinvoke(DICT.r_rdict_eqfn, d.fnkeyeq, key1, key2)
+    return objectmodel.hlinvoke(DICT.r_rdict_eqfn, d.fnkeyeq, key1, key2)
 
 def ll_dict_len(d):
     return d.num_items
@@ -448,6 +445,8 @@
     i = ll_dict_lookup(d, key, hash)
     return _ll_dict_setitem_lookup_done(d, key, value, hash, i)
 
+# Leaving as dont_look_inside ATM, it has a few branches which could lead to
+# many bridges if we don't consider their possible frequency.
 @jit.dont_look_inside
 def _ll_dict_setitem_lookup_done(d, key, value, hash, i):
     valid = (i & HIGHEST_BIT) == 0
@@ -492,6 +491,8 @@
         raise KeyError
     _ll_dict_del(d, i)
 
+# XXX: Move the size checking and resize into a single call which is opauqe to
+# the JIT to avoid extra branches.
 @jit.dont_look_inside
 def _ll_dict_del(d, i):
     d.entries.mark_deleted(i)
@@ -532,6 +533,7 @@
 # ------- a port of CPython's dictobject.c's lookdict implementation -------
 PERTURB_SHIFT = 5
 
+ at jit.dont_look_inside
 def ll_dict_lookup(d, key, hash):
     entries = d.entries
     ENTRIES = lltype.typeOf(entries).TO
@@ -621,7 +623,6 @@
     d.num_items = 0
     d.resize_counter = DICT_INITSIZE * 2
     return d
-ll_newdict.oopspec = 'newdict()'
 
 def ll_newdict_size(DICT, length_estimate):
     length_estimate = (length_estimate // 2) * 3
@@ -633,7 +634,6 @@
     d.num_items = 0
     d.resize_counter = n * 2
     return d
-ll_newdict_size.oopspec = 'newdict()'
 
 # pypy.rpython.memory.lldict uses a dict based on Struct and Array
 # instead of GcStruct and GcArray, which is done by using different
@@ -866,7 +866,6 @@
         global_popitem_index.nextindex = base + counter
     return i
 
- at jit.dont_look_inside
 def ll_popitem(ELEM, dic):
     i = _ll_getnextitem(dic)
     entry = dic.entries[i]
diff --git a/pypy/rpython/lltypesystem/test/test_ll2ctypes.py b/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
--- a/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
@@ -8,6 +8,7 @@
 from pypy.rpython.lltypesystem.ll2ctypes import uninitialized2ctypes
 from pypy.rpython.lltypesystem.ll2ctypes import ALLOCATED, force_cast
 from pypy.rpython.lltypesystem.ll2ctypes import cast_adr_to_int, get_ctypes_type
+from pypy.rpython.lltypesystem.ll2ctypes import _llgcopaque
 from pypy.rpython.annlowlevel import llhelper
 from pypy.rlib import rposix
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
@@ -1349,6 +1350,11 @@
         round = ctypes2lltype(llmemory.GCREF, lltype2ctypes(opaque.hide()))
         assert Opaque.show(round) is opaque
 
+    def test_array_of_structs(self):
+        A = lltype.GcArray(lltype.Struct('x', ('v', lltype.Signed)))
+        a = lltype.malloc(A, 5)
+        a2 = ctypes2lltype(lltype.Ptr(A), lltype2ctypes(a))
+        assert a2._obj.getitem(0)._obj._parentstructure() is a2._obj
 
 class TestPlatform(object):
     def test_lib_on_libpaths(self):
@@ -1390,3 +1396,7 @@
         f = rffi.llexternal('f', [rffi.INT, rffi.INT], rffi.INT,
                             compilation_info=eci)
         assert f(3, 4) == 7
+
+    def test_llgcopaque_eq(self):
+        assert _llgcopaque(1) != None
+        assert _llgcopaque(0) == None
diff --git a/pypy/rpython/memory/gc/base.py b/pypy/rpython/memory/gc/base.py
--- a/pypy/rpython/memory/gc/base.py
+++ b/pypy/rpython/memory/gc/base.py
@@ -1,4 +1,5 @@
-from pypy.rpython.lltypesystem import lltype, llmemory, llarena
+from pypy.rpython.lltypesystem import lltype, llmemory, llarena, rffi
+from pypy.rpython.lltypesystem.lloperation import llop
 from pypy.rlib.debug import ll_assert
 from pypy.rpython.memory.gcheader import GCHeaderBuilder
 from pypy.rpython.memory.support import DEFAULT_CHUNK_SIZE
@@ -62,6 +63,7 @@
     def set_query_functions(self, is_varsize, has_gcptr_in_varsize,
                             is_gcarrayofgcptr,
                             getfinalizer,
+                            getlightfinalizer,
                             offsets_to_gc_pointers,
                             fixed_size, varsize_item_sizes,
                             varsize_offset_to_variable_part,
@@ -74,6 +76,7 @@
                             get_custom_trace,
                             fast_path_tracing):
         self.getfinalizer = getfinalizer
+        self.getlightfinalizer = getlightfinalizer
         self.is_varsize = is_varsize
         self.has_gcptr_in_varsize = has_gcptr_in_varsize
         self.is_gcarrayofgcptr = is_gcarrayofgcptr
@@ -139,6 +142,7 @@
 
         size = self.fixed_size(typeid)
         needs_finalizer = bool(self.getfinalizer(typeid))
+        finalizer_is_light = bool(self.getlightfinalizer(typeid))
         contains_weakptr = self.weakpointer_offset(typeid) >= 0
         assert not (needs_finalizer and contains_weakptr)
         if self.is_varsize(typeid):
@@ -158,6 +162,7 @@
             else:
                 malloc_fixedsize = self.malloc_fixedsize
             ref = malloc_fixedsize(typeid, size, needs_finalizer,
+                                   finalizer_is_light,
                                    contains_weakptr)
         # lots of cast and reverse-cast around...
         return llmemory.cast_ptr_to_adr(ref)
diff --git a/pypy/rpython/memory/gc/generation.py b/pypy/rpython/memory/gc/generation.py
--- a/pypy/rpython/memory/gc/generation.py
+++ b/pypy/rpython/memory/gc/generation.py
@@ -167,7 +167,9 @@
         return self.nursery <= addr < self.nursery_top
 
     def malloc_fixedsize_clear(self, typeid, size,
-                               has_finalizer=False, contains_weakptr=False):
+                               has_finalizer=False,
+                               is_finalizer_light=False,
+                               contains_weakptr=False):
         if (has_finalizer or
             (raw_malloc_usage(size) > self.lb_young_fixedsize and
              raw_malloc_usage(size) > self.largest_young_fixedsize)):
@@ -179,6 +181,7 @@
             # "non-simple" case or object too big: don't use the nursery
             return SemiSpaceGC.malloc_fixedsize_clear(self, typeid, size,
                                                       has_finalizer,
+                                                      is_finalizer_light,
                                                       contains_weakptr)
         size_gc_header = self.gcheaderbuilder.size_gc_header
         totalsize = size_gc_header + size
diff --git a/pypy/rpython/memory/gc/marksweep.py b/pypy/rpython/memory/gc/marksweep.py
--- a/pypy/rpython/memory/gc/marksweep.py
+++ b/pypy/rpython/memory/gc/marksweep.py
@@ -93,7 +93,8 @@
         pass
 
     def malloc_fixedsize(self, typeid16, size,
-                         has_finalizer=False, contains_weakptr=False):
+                         has_finalizer=False, is_finalizer_light=False,
+                         contains_weakptr=False):
         self.maybe_collect()
         size_gc_header = self.gcheaderbuilder.size_gc_header
         try:
@@ -128,7 +129,9 @@
     malloc_fixedsize._dont_inline_ = True
 
     def malloc_fixedsize_clear(self, typeid16, size,
-                               has_finalizer=False, contains_weakptr=False):
+                               has_finalizer=False,
+                               is_finalizer_light=False,
+                               contains_weakptr=False):
         self.maybe_collect()
         size_gc_header = self.gcheaderbuilder.size_gc_header
         try:
diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -290,6 +290,8 @@
         #
         # A list of all objects with finalizers (these are never young).
         self.objects_with_finalizers = self.AddressDeque()
+        self.young_objects_with_light_finalizers = self.AddressStack()
+        self.old_objects_with_light_finalizers = self.AddressStack()
         #
         # Two lists of the objects with weakrefs.  No weakref can be an
         # old object weakly pointing to a young object: indeed, weakrefs
@@ -457,14 +459,16 @@
 
 
     def malloc_fixedsize_clear(self, typeid, size,
-                               needs_finalizer=False, contains_weakptr=False):
+                               needs_finalizer=False,
+                               is_finalizer_light=False,
+                               contains_weakptr=False):
         size_gc_header = self.gcheaderbuilder.size_gc_header
         totalsize = size_gc_header + size
         rawtotalsize = raw_malloc_usage(totalsize)
         #
         # If the object needs a finalizer, ask for a rawmalloc.
         # The following check should be constant-folded.
-        if needs_finalizer:
+        if needs_finalizer and not is_finalizer_light:
             ll_assert(not contains_weakptr,
                      "'needs_finalizer' and 'contains_weakptr' both specified")
             obj = self.external_malloc(typeid, 0, can_make_young=False)
@@ -494,13 +498,14 @@
             #
             # Build the object.
             llarena.arena_reserve(result, totalsize)
+            obj = result + size_gc_header
+            if is_finalizer_light:
+                self.young_objects_with_light_finalizers.append(obj)
             self.init_gc_object(result, typeid, flags=0)
             #
             # If it is a weakref, record it (check constant-folded).
             if contains_weakptr:
-                self.young_objects_with_weakrefs.append(result+size_gc_header)
-            #
-            obj = result + size_gc_header
+                self.young_objects_with_weakrefs.append(obj)
         #
         return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
 
@@ -1264,6 +1269,8 @@
         # weakrefs' targets.
         if self.young_objects_with_weakrefs.non_empty():
             self.invalidate_young_weakrefs()
+        if self.young_objects_with_light_finalizers.non_empty():
+            self.deal_with_young_objects_with_finalizers()
         #
         # Clear this mapping.
         if self.nursery_objects_shadows.length() > 0:
@@ -1292,10 +1299,12 @@
         # if a prebuilt GcStruct contains a pointer to a young object,
         # then the write_barrier must have ensured that the prebuilt
         # GcStruct is in the list self.old_objects_pointing_to_young.
+        debug_start("gc-minor-walkroots")
         self.root_walker.walk_roots(
             MiniMarkGC._trace_drag_out1,  # stack roots
             MiniMarkGC._trace_drag_out1,  # static in prebuilt non-gc
             None)                         # static in prebuilt gc
+        debug_stop("gc-minor-walkroots")
 
     def collect_cardrefs_to_nursery(self):
         size_gc_header = self.gcheaderbuilder.size_gc_header
@@ -1582,6 +1591,9 @@
         # Weakref support: clear the weak pointers to dying objects
         if self.old_objects_with_weakrefs.non_empty():
             self.invalidate_old_weakrefs()
+        if self.old_objects_with_light_finalizers.non_empty():
+            self.deal_with_old_objects_with_finalizers()
+
         #
         # Walk all rawmalloced objects and free the ones that don't
         # have the GCFLAG_VISITED flag.
@@ -1647,8 +1659,7 @@
         if self.header(obj).tid & GCFLAG_VISITED:
             self.header(obj).tid &= ~GCFLAG_VISITED
             return False     # survives
-        else:
-            return True      # dies
+        return True      # dies
 
     def _reset_gcflag_visited(self, obj, ignored):
         self.header(obj).tid &= ~GCFLAG_VISITED
@@ -1827,6 +1838,39 @@
     # ----------
     # Finalizers
 
+    def deal_with_young_objects_with_finalizers(self):
+        """ This is a much simpler version of dealing with finalizers
+        and an optimization - we can reasonably assume that those finalizers
+        don't do anything fancy and *just* call them. Among other things
+        they won't resurrect objects
+        """
+        while self.young_objects_with_light_finalizers.non_empty():
+            obj = self.young_objects_with_light_finalizers.pop()
+            if not self.is_forwarded(obj):
+                finalizer = self.getlightfinalizer(self.get_type_id(obj))
+                ll_assert(bool(finalizer), "no light finalizer found")
+                finalizer(obj, llmemory.NULL)
+
+    def deal_with_old_objects_with_finalizers(self):
+        """ This is a much simpler version of dealing with finalizers
+        and an optimization - we can reasonably assume that those finalizers
+        don't do anything fancy and *just* call them. Among other things
+        they won't resurrect objects
+        """
+        new_objects = self.AddressStack()
+        while self.old_objects_with_light_finalizers.non_empty():
+            obj = self.old_objects_with_light_finalizers.pop()
+            if self.header(obj).tid & GCFLAG_VISITED:
+                # surviving
+                new_objects.append(obj)
+            else:
+                # dying
+                finalizer = self.getlightfinalizer(self.get_type_id(obj))
+                ll_assert(bool(finalizer), "no light finalizer found")
+                finalizer(obj, llmemory.NULL)
+        self.old_objects_with_light_finalizers.delete()
+        self.old_objects_with_light_finalizers = new_objects
+
     def deal_with_objects_with_finalizers(self):
         # Walk over list of objects with finalizers.
         # If it is not surviving, add it to the list of to-be-called
@@ -1957,7 +2001,6 @@
             #
             self.old_objects_with_weakrefs.append(obj)
 
-
     def invalidate_old_weakrefs(self):
         """Called during a major collection."""
         # walk over list of objects that contain weakrefs
diff --git a/pypy/rpython/memory/gc/semispace.py b/pypy/rpython/memory/gc/semispace.py
--- a/pypy/rpython/memory/gc/semispace.py
+++ b/pypy/rpython/memory/gc/semispace.py
@@ -82,6 +82,7 @@
         self.free = self.tospace
         MovingGCBase.setup(self)
         self.objects_with_finalizers = self.AddressDeque()
+        self.objects_with_light_finalizers = self.AddressStack()
         self.objects_with_weakrefs = self.AddressStack()
 
     def _teardown(self):
@@ -93,7 +94,9 @@
     # because the spaces are filled with zeroes in advance.
 
     def malloc_fixedsize_clear(self, typeid16, size,
-                               has_finalizer=False, contains_weakptr=False):
+                               has_finalizer=False,
+                               is_finalizer_light=False,
+                               contains_weakptr=False):
         size_gc_header = self.gcheaderbuilder.size_gc_header
         totalsize = size_gc_header + size
         result = self.free
@@ -102,7 +105,9 @@
         llarena.arena_reserve(result, totalsize)
         self.init_gc_object(result, typeid16)
         self.free = result + totalsize
-        if has_finalizer:
+        if is_finalizer_light:
+            self.objects_with_light_finalizers.append(result + size_gc_header)
+        elif has_finalizer:
             self.objects_with_finalizers.append(result + size_gc_header)
         if contains_weakptr:
             self.objects_with_weakrefs.append(result + size_gc_header)
@@ -263,6 +268,8 @@
         if self.run_finalizers.non_empty():
             self.update_run_finalizers()
         scan = self.scan_copied(scan)
+        if self.objects_with_light_finalizers.non_empty():
+            self.deal_with_objects_with_light_finalizers()
         if self.objects_with_finalizers.non_empty():
             scan = self.deal_with_objects_with_finalizers(scan)
         if self.objects_with_weakrefs.non_empty():
@@ -471,6 +478,23 @@
         # immortal objects always have GCFLAG_FORWARDED set;
         # see get_forwarding_address().
 
+    def deal_with_objects_with_light_finalizers(self):
+        """ This is a much simpler version of dealing with finalizers
+        and an optimization - we can reasonably assume that those finalizers
+        don't do anything fancy and *just* call them. Among other things
+        they won't resurrect objects
+        """
+        new_objects = self.AddressStack()
+        while self.objects_with_light_finalizers.non_empty():
+            obj = self.objects_with_light_finalizers.pop()
+            if self.surviving(obj):
+                new_objects.append(self.get_forwarding_address(obj))
+            else:
+                finalizer = self.getfinalizer(self.get_type_id(obj))
+                finalizer(obj, llmemory.NULL)
+        self.objects_with_light_finalizers.delete()
+        self.objects_with_light_finalizers = new_objects
+
     def deal_with_objects_with_finalizers(self, scan):
         # walk over list of objects with finalizers
         # if it is not copied, add it to the list of to-be-called finalizers
diff --git a/pypy/rpython/memory/gctransform/framework.py b/pypy/rpython/memory/gctransform/framework.py
--- a/pypy/rpython/memory/gctransform/framework.py
+++ b/pypy/rpython/memory/gctransform/framework.py
@@ -12,6 +12,7 @@
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.translator.backendopt import graphanalyze
 from pypy.translator.backendopt.support import var_needsgc
+from pypy.translator.backendopt.finalizer import FinalizerAnalyzer
 from pypy.annotation import model as annmodel
 from pypy.rpython import annlowlevel
 from pypy.rpython.rbuiltin import gen_cast
@@ -258,6 +259,7 @@
             [s_gc, s_typeid16,
              annmodel.SomeInteger(nonneg=True),
              annmodel.SomeBool(),
+             annmodel.SomeBool(),
              annmodel.SomeBool()], s_gcref,
             inline = False)
         if hasattr(GCClass, 'malloc_fixedsize'):
@@ -267,6 +269,7 @@
                 [s_gc, s_typeid16,
                  annmodel.SomeInteger(nonneg=True),
                  annmodel.SomeBool(),
+                 annmodel.SomeBool(),
                  annmodel.SomeBool()], s_gcref,
                 inline = False)
         else:
@@ -319,7 +322,7 @@
             raise NotImplementedError("GC needs write barrier, but does not provide writebarrier_before_copy functionality")
 
         # in some GCs we can inline the common case of
-        # malloc_fixedsize(typeid, size, True, False, False)
+        # malloc_fixedsize(typeid, size, False, False, False)
         if getattr(GCClass, 'inline_simple_malloc', False):
             # make a copy of this function so that it gets annotated
             # independently and the constants are folded inside
@@ -337,7 +340,7 @@
                 malloc_fast,
                 [s_gc, s_typeid16,
                  annmodel.SomeInteger(nonneg=True),
-                 s_False, s_False], s_gcref,
+                 s_False, s_False, s_False], s_gcref,
                 inline = True)
         else:
             self.malloc_fast_ptr = None
@@ -668,7 +671,13 @@
         kind_and_fptr = self.special_funcptr_for_type(TYPE)
         has_finalizer = (kind_and_fptr is not None and
                          kind_and_fptr[0] == "finalizer")
+        has_light_finalizer = (kind_and_fptr is not None and
+                               kind_and_fptr[0] == "light_finalizer")
+        if has_light_finalizer:
+            has_finalizer = True
         c_has_finalizer = rmodel.inputconst(lltype.Bool, has_finalizer)
+        c_has_light_finalizer = rmodel.inputconst(lltype.Bool,
+                                                  has_light_finalizer)
 
         if not op.opname.endswith('_varsize') and not flags.get('varsize'):
             #malloc_ptr = self.malloc_fixedsize_ptr
@@ -682,7 +691,8 @@
             else:
                 malloc_ptr = self.malloc_fixedsize_ptr
             args = [self.c_const_gc, c_type_id, c_size,
-                    c_has_finalizer, rmodel.inputconst(lltype.Bool, False)]
+                    c_has_finalizer, c_has_light_finalizer,
+                    rmodel.inputconst(lltype.Bool, False)]
         else:
             assert not c_has_finalizer.value
             info_varsize = self.layoutbuilder.get_info_varsize(type_id)
@@ -847,12 +857,13 @@
         # used by the JIT (see pypy.jit.backend.llsupport.gc)
         op = hop.spaceop
         [v_typeid, v_size,
-         v_has_finalizer, v_contains_weakptr] = op.args
+         v_has_finalizer, v_has_light_finalizer, v_contains_weakptr] = op.args
         livevars = self.push_roots(hop)
         hop.genop("direct_call",
                   [self.malloc_fixedsize_clear_ptr, self.c_const_gc,
                    v_typeid, v_size,
-                   v_has_finalizer, v_contains_weakptr],
+                   v_has_finalizer, v_has_light_finalizer,
+                   v_contains_weakptr],
                   resultvar=op.result)
         self.pop_roots(hop, livevars)
 
@@ -912,10 +923,10 @@
         info = self.layoutbuilder.get_info(type_id)
         c_size = rmodel.inputconst(lltype.Signed, info.fixedsize)
         malloc_ptr = self.malloc_fixedsize_ptr
-        c_has_finalizer = rmodel.inputconst(lltype.Bool, False)
+        c_false = rmodel.inputconst(lltype.Bool, False)
         c_has_weakptr = rmodel.inputconst(lltype.Bool, True)
         args = [self.c_const_gc, c_type_id, c_size,
-                c_has_finalizer, c_has_weakptr]
+                c_false, c_false, c_has_weakptr]
 
         # push and pop the current live variables *including* the argument
         # to the weakref_create operation, which must be kept alive and
@@ -1250,6 +1261,7 @@
             lltype2vtable = translator.rtyper.lltype2vtable
         else:
             lltype2vtable = None
+        self.translator = translator
         super(TransformerLayoutBuilder, self).__init__(GCClass, lltype2vtable)
 
     def has_finalizer(self, TYPE):
@@ -1257,6 +1269,10 @@
         return rtti is not None and getattr(rtti._obj, 'destructor_funcptr',
                                             None)
 
+    def has_light_finalizer(self, TYPE):
+        special = self.special_funcptr_for_type(TYPE)
+        return special is not None and special[0] == 'light_finalizer'
+
     def has_custom_trace(self, TYPE):
         rtti = get_rtti(TYPE)
         return rtti is not None and getattr(rtti._obj, 'custom_trace_funcptr',
@@ -1264,7 +1280,7 @@
 
     def make_finalizer_funcptr_for_type(self, TYPE):
         if not self.has_finalizer(TYPE):
-            return None
+            return None, False
         rtti = get_rtti(TYPE)
         destrptr = rtti._obj.destructor_funcptr
         DESTR_ARG = lltype.typeOf(destrptr).TO.ARGS[0]
@@ -1276,7 +1292,9 @@
             return llmemory.NULL
         fptr = self.transformer.annotate_finalizer(ll_finalizer,
                 [llmemory.Address, llmemory.Address], llmemory.Address)
-        return fptr
+        g = destrptr._obj.graph
+        light = not FinalizerAnalyzer(self.translator).analyze_light_finalizer(g)
+        return fptr, light
 
     def make_custom_trace_funcptr_for_type(self, TYPE):
         if not self.has_custom_trace(TYPE):
diff --git a/pypy/rpython/memory/gctypelayout.py b/pypy/rpython/memory/gctypelayout.py
--- a/pypy/rpython/memory/gctypelayout.py
+++ b/pypy/rpython/memory/gctypelayout.py
@@ -1,7 +1,6 @@
 from pypy.rpython.lltypesystem import lltype, llmemory, llarena, llgroup
 from pypy.rpython.lltypesystem import rclass
 from pypy.rpython.lltypesystem.lloperation import llop
-from pypy.rlib.objectmodel import we_are_translated
 from pypy.rlib.debug import ll_assert
 from pypy.rlib.rarithmetic import intmask
 from pypy.tool.identity_dict import identity_dict
@@ -85,6 +84,13 @@
         else:
             return lltype.nullptr(GCData.FINALIZER_OR_CT_FUNC)
 
+    def q_light_finalizer(self, typeid):
+        typeinfo = self.get(typeid)
+        if typeinfo.infobits & T_HAS_LIGHTWEIGHT_FINALIZER:
+            return typeinfo.finalizer_or_customtrace
+        else:
+            return lltype.nullptr(GCData.FINALIZER_OR_CT_FUNC)        
+
     def q_offsets_to_gc_pointers(self, typeid):
         return self.get(typeid).ofstoptrs
 
@@ -142,6 +148,7 @@
             self.q_has_gcptr_in_varsize,
             self.q_is_gcarrayofgcptr,
             self.q_finalizer,
+            self.q_light_finalizer,
             self.q_offsets_to_gc_pointers,
             self.q_fixed_size,
             self.q_varsize_item_sizes,
@@ -157,16 +164,17 @@
 
 
 # the lowest 16bits are used to store group member index
-T_MEMBER_INDEX         =   0xffff
-T_IS_VARSIZE           = 0x010000
-T_HAS_GCPTR_IN_VARSIZE = 0x020000
-T_IS_GCARRAY_OF_GCPTR  = 0x040000
-T_IS_WEAKREF           = 0x080000
-T_IS_RPYTHON_INSTANCE  = 0x100000    # the type is a subclass of OBJECT
-T_HAS_FINALIZER        = 0x200000
-T_HAS_CUSTOM_TRACE     = 0x400000
-T_KEY_MASK             = intmask(0xFF000000)
-T_KEY_VALUE            = intmask(0x5A000000)    # bug detection only
+T_MEMBER_INDEX              =   0xffff
+T_IS_VARSIZE                = 0x010000
+T_HAS_GCPTR_IN_VARSIZE      = 0x020000
+T_IS_GCARRAY_OF_GCPTR       = 0x040000
+T_IS_WEAKREF                = 0x080000
+T_IS_RPYTHON_INSTANCE       = 0x100000 # the type is a subclass of OBJECT
+T_HAS_FINALIZER             = 0x200000
+T_HAS_CUSTOM_TRACE          = 0x400000
+T_HAS_LIGHTWEIGHT_FINALIZER = 0x800000
+T_KEY_MASK                  = intmask(0xFF000000)
+T_KEY_VALUE                 = intmask(0x5A000000) # bug detection only
 
 def _check_valid_type_info(p):
     ll_assert(p.infobits & T_KEY_MASK == T_KEY_VALUE, "invalid type_id")
@@ -194,6 +202,8 @@
         info.finalizer_or_customtrace = fptr
         if kind == "finalizer":
             infobits |= T_HAS_FINALIZER
+        elif kind == 'light_finalizer':
+            infobits |= T_HAS_FINALIZER | T_HAS_LIGHTWEIGHT_FINALIZER
         elif kind == "custom_trace":
             infobits |= T_HAS_CUSTOM_TRACE
         else:
@@ -367,12 +377,15 @@
     def special_funcptr_for_type(self, TYPE):
         if TYPE in self._special_funcptrs:
             return self._special_funcptrs[TYPE]
-        fptr1 = self.make_finalizer_funcptr_for_type(TYPE)
+        fptr1, is_lightweight = self.make_finalizer_funcptr_for_type(TYPE)
         fptr2 = self.make_custom_trace_funcptr_for_type(TYPE)
         assert not (fptr1 and fptr2), (
             "type %r needs both a finalizer and a custom tracer" % (TYPE,))
         if fptr1:
-            kind_and_fptr = "finalizer", fptr1
+            if is_lightweight:
+                kind_and_fptr = "light_finalizer", fptr1
+            else:
+                kind_and_fptr = "finalizer", fptr1
         elif fptr2:
             kind_and_fptr = "custom_trace", fptr2
         else:
@@ -382,7 +395,7 @@
 
     def make_finalizer_funcptr_for_type(self, TYPE):
         # must be overridden for proper finalizer support
-        return None
+        return None, False
 
     def make_custom_trace_funcptr_for_type(self, TYPE):
         # must be overridden for proper custom tracer support
diff --git a/pypy/rpython/memory/gcwrapper.py b/pypy/rpython/memory/gcwrapper.py
--- a/pypy/rpython/memory/gcwrapper.py
+++ b/pypy/rpython/memory/gcwrapper.py
@@ -1,3 +1,4 @@
+from pypy.translator.backendopt.finalizer import FinalizerAnalyzer
 from pypy.rpython.lltypesystem import lltype, llmemory, llheap
 from pypy.rpython import llinterp
 from pypy.rpython.annlowlevel import llhelper
@@ -196,9 +197,11 @@
             DESTR_ARG = lltype.typeOf(destrptr).TO.ARGS[0]
             destrgraph = destrptr._obj.graph
         else:
-            return None
+            return None, False
 
         assert not type_contains_pyobjs(TYPE), "not implemented"
+        t = self.llinterp.typer.annotator.translator
+        light = not FinalizerAnalyzer(t).analyze_light_finalizer(destrgraph)
         def ll_finalizer(addr, dummy):
             assert dummy == llmemory.NULL
             try:
@@ -208,7 +211,7 @@
                 raise RuntimeError(
                     "a finalizer raised an exception, shouldn't happen")
             return llmemory.NULL
-        return llhelper(gctypelayout.GCData.FINALIZER_OR_CT, ll_finalizer)
+        return llhelper(gctypelayout.GCData.FINALIZER_OR_CT, ll_finalizer), light
 
     def make_custom_trace_funcptr_for_type(self, TYPE):
         from pypy.rpython.memory.gctransform.support import get_rtti, \
diff --git a/pypy/rpython/memory/test/test_gc.py b/pypy/rpython/memory/test/test_gc.py
--- a/pypy/rpython/memory/test/test_gc.py
+++ b/pypy/rpython/memory/test/test_gc.py
@@ -5,7 +5,6 @@
 from pypy.rpython.memory.test import snippet
 from pypy.rpython.test.test_llinterp import get_interpreter
 from pypy.rpython.lltypesystem import lltype
-from pypy.rpython.lltypesystem.rstr import STR
 from pypy.rpython.lltypesystem.lloperation import llop
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.rlib.objectmodel import compute_unique_id
@@ -57,7 +56,7 @@
                 while j < 20:
                     j += 1
                     a.append(j)
-        res = self.interpret(malloc_a_lot, [])
+        self.interpret(malloc_a_lot, [])
         #assert simulator.current_size - curr < 16000 * INT_SIZE / 4
         #print "size before: %s, size after %s" % (curr, simulator.current_size)
 
@@ -73,7 +72,7 @@
                 while j < 20:
                     j += 1
                     b.append((1, j, i))
-        res = self.interpret(malloc_a_lot, [])
+        self.interpret(malloc_a_lot, [])
         #assert simulator.current_size - curr < 16000 * INT_SIZE / 4
         #print "size before: %s, size after %s" % (curr, simulator.current_size)
 
@@ -129,7 +128,7 @@
         res = self.interpret(concat, [100])
         assert res == concat(100)
         #assert simulator.current_size - curr < 16000 * INT_SIZE / 4
-        
+
     def test_finalizer(self):
         class B(object):
             pass
@@ -278,7 +277,7 @@
                                self.interpret, f, [])
 
     def test_weakref(self):
-        import weakref, gc
+        import weakref
         class A(object):
             pass
         def g():
@@ -299,7 +298,7 @@
         assert res
 
     def test_weakref_to_object_with_finalizer(self):
-        import weakref, gc
+        import weakref
         class A(object):
             count = 0
         a = A()
@@ -338,7 +337,7 @@
         assert res
 
     def test_cycle_with_weakref_and_del(self):
-        import weakref, gc
+        import weakref
         class A(object):
             count = 0
         a = A()
@@ -367,7 +366,7 @@
         assert res == 11
 
     def test_weakref_to_object_with_finalizer_ordering(self):
-        import weakref, gc
+        import weakref
         class A(object):
             count = 0
         a = A()
@@ -616,7 +615,7 @@
                     assert not rgc.can_move(a)
                     return 1
                 return 0
-            except Exception, e:
+            except Exception:
                 return 2
 
         assert self.interpret(func, []) == int(self.GC_CAN_MALLOC_NONMOVABLE)
@@ -647,8 +646,6 @@
         assert self.interpret(f, [bigsize, 0, flag]) == 0x62024241
 
     def test_tagged_simple(self):
-        from pypy.rlib.objectmodel import UnboxedValue
-
         class Unrelated(object):
             pass
 
@@ -689,8 +686,6 @@
         assert res == -897
 
     def test_tagged_id(self):
-        from pypy.rlib.objectmodel import UnboxedValue, compute_unique_id
-
         class Unrelated(object):
             pass
 
diff --git a/pypy/rpython/memory/test/test_transformed_gc.py b/pypy/rpython/memory/test/test_transformed_gc.py
--- a/pypy/rpython/memory/test/test_transformed_gc.py
+++ b/pypy/rpython/memory/test/test_transformed_gc.py
@@ -345,22 +345,22 @@
         b = B()
         b.nextid = 0
         b.num_deleted = 0
-        class A(object):
+        class AAA(object):
             def __init__(self):
                 self.id = b.nextid
                 b.nextid += 1
             def __del__(self):
                 b.num_deleted += 1
                 C()
-        class C(A):
+        class C(AAA):
             def __del__(self):
                 b.num_deleted += 1
         def f(x, y):
-            a = A()
+            a = AAA()
             i = 0
             while i < x:
                 i += 1
-                a = A()
+                a = AAA()
             llop.gc__collect(lltype.Void)
             llop.gc__collect(lltype.Void)
             return b.num_deleted
@@ -807,6 +807,7 @@
                     op.args = [Constant(type_id, llgroup.HALFWORD),
                                Constant(llmemory.sizeof(P), lltype.Signed),
                                Constant(False, lltype.Bool), # has_finalizer
+                               Constant(False, lltype.Bool), # is_finalizer_light
                                Constant(False, lltype.Bool)] # contains_weakptr
                     break
             else:
@@ -843,6 +844,7 @@
                     op.args = [Constant(type_id, llgroup.HALFWORD),
                                Constant(llmemory.sizeof(P), lltype.Signed),
                                Constant(False, lltype.Bool), # has_finalizer
+                               Constant(False, lltype.Bool), # is_finalizer_light
                                Constant(False, lltype.Bool)] # contains_weakptr
                     break
             else:
diff --git a/pypy/rpython/module/ll_os.py b/pypy/rpython/module/ll_os.py
--- a/pypy/rpython/module/ll_os.py
+++ b/pypy/rpython/module/ll_os.py
@@ -959,8 +959,6 @@
                             os_ftruncate(rffi.cast(rffi.INT, fd),
                                          rffi.cast(rffi.LONGLONG, length)))
             if res < 0:
-                # Note: for consistency we raise OSError, but CPython
-                # raises IOError here
                 raise OSError(rposix.get_errno(), "os_ftruncate failed")
 
         return extdef([int, r_longlong], s_None,
diff --git a/pypy/rpython/rtyper.py b/pypy/rpython/rtyper.py
--- a/pypy/rpython/rtyper.py
+++ b/pypy/rpython/rtyper.py
@@ -717,7 +717,7 @@
             raise TyperError("runtime type info function %r returns %r, "
                              "excepted Ptr(RuntimeTypeInfo)" % (func, s))
         funcptr = self.getcallable(graph)
-        attachRuntimeTypeInfo(GCSTRUCT, funcptr, destrptr)
+        attachRuntimeTypeInfo(GCSTRUCT, funcptr, destrptr, None)
 
 # register operations from annotation model
 RPythonTyper._registeroperations(annmodel)
diff --git a/pypy/rpython/test/test_rclass.py b/pypy/rpython/test/test_rclass.py
--- a/pypy/rpython/test/test_rclass.py
+++ b/pypy/rpython/test/test_rclass.py
@@ -3,7 +3,7 @@
 from pypy.translator.translator import TranslationContext, graphof
 from pypy.rpython.lltypesystem.lltype import *
 from pypy.rpython.ootypesystem import ootype
-from pypy.rlib.rarithmetic import intmask, r_longlong
+from pypy.rlib.rarithmetic import r_longlong
 from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin
 from pypy.rpython.rclass import IR_IMMUTABLE, IR_IMMUTABLE_ARRAY
 from pypy.rpython.rclass import IR_QUASIIMMUTABLE, IR_QUASIIMMUTABLE_ARRAY
@@ -972,10 +972,10 @@
         graph = graphof(t, f)
         TYPE = graph.startblock.operations[0].args[0].value
         RTTI = getRuntimeTypeInfo(TYPE)
-        queryptr = RTTI._obj.query_funcptr # should not raise
+        RTTI._obj.query_funcptr # should not raise
         destrptr = RTTI._obj.destructor_funcptr
         assert destrptr is not None
-    
+
     def test_del_inheritance(self):
         from pypy.rlib import rgc
         class State:
diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -56,8 +56,8 @@
     binaries = [(pypy_c, rename_pypy_c)]
     #
     if sys.platform == 'win32':
-        # Can't rename a DLL: it is always called 'libpypy_c.dll'
-        for extra in ['libpypy_c.dll',
+        # Can't rename a DLL: it is always called 'libpypy-c.dll'
+        for extra in ['libpypy-c.dll',
                       'libexpat.dll', 'sqlite3.dll', 'msvcr90.dll']:
             p = pypy_c.dirpath().join(extra)
             if not p.check():
diff --git a/pypy/tool/sourcetools.py b/pypy/tool/sourcetools.py
--- a/pypy/tool/sourcetools.py
+++ b/pypy/tool/sourcetools.py
@@ -216,9 +216,11 @@
 
 # ____________________________________________________________
 
-def func_with_new_name(func, newname):
+def func_with_new_name(func, newname, globals=None):
     """Make a renamed copy of a function."""
-    f = new.function(func.func_code, func.func_globals,
+    if globals is None:
+        globals = func.func_globals
+    f = new.function(func.func_code, globals,
                         newname, func.func_defaults,
                         func.func_closure)
     if func.func_dict:
diff --git a/pypy/translator/backendopt/finalizer.py b/pypy/translator/backendopt/finalizer.py
new file mode 100644
--- /dev/null
+++ b/pypy/translator/backendopt/finalizer.py
@@ -0,0 +1,46 @@
+
+from pypy.translator.backendopt import graphanalyze
+from pypy.rpython.lltypesystem import lltype
+
+class FinalizerError(Exception):
+    """ __del__ marked as lightweight finalizer, but the analyzer did
+    not agreed
+    """
+
+class FinalizerAnalyzer(graphanalyze.BoolGraphAnalyzer):
+    """ Analyzer that determines whether a finalizer is lightweight enough
+    so it can be called without all the complicated logic in the garbage
+    collector. The set of operations here is restrictive for a good reason
+    - it's better to be safe. Specifically disallowed operations:
+
+    * anything that escapes self
+    * anything that can allocate
+    """
+    ok_operations = ['ptr_nonzero', 'ptr_eq', 'ptr_ne', 'free', 'same_as',
+                     'direct_ptradd', 'force_cast', 'track_alloc_stop',
+                     'raw_free']
+    
+    def analyze_light_finalizer(self, graph):
+        result = self.analyze_direct_call(graph)
+        if (result is self.top_result() and
+            getattr(graph.func, '_is_light_finalizer_', False)):
+            raise FinalizerError(FinalizerError.__doc__, graph)
+        return result
+    
+    def analyze_simple_operation(self, op, graphinfo):
+        if op.opname in self.ok_operations:
+            return self.bottom_result()
+        if (op.opname.startswith('int_') or op.opname.startswith('float_')
+            or op.opname.startswith('cast_')):
+            return self.bottom_result()
+        if op.opname == 'setfield' or op.opname == 'bare_setfield':
+            TP = op.args[2].concretetype
+            if not isinstance(TP, lltype.Ptr) or TP.TO._gckind == 'raw':
+                # primitive type
+                return self.bottom_result()
+        if op.opname == 'getfield':
+            TP = op.result.concretetype
+            if not isinstance(TP, lltype.Ptr) or TP.TO._gckind == 'raw':
+                # primitive type
+                return self.bottom_result()            
+        return self.top_result()
diff --git a/pypy/translator/backendopt/test/test_finalizer.py b/pypy/translator/backendopt/test/test_finalizer.py
new file mode 100644
--- /dev/null
+++ b/pypy/translator/backendopt/test/test_finalizer.py
@@ -0,0 +1,142 @@
+
+import py
+from pypy.translator.backendopt.finalizer import FinalizerAnalyzer,\
+     FinalizerError
+from pypy.translator.translator import TranslationContext, graphof
+from pypy.translator.backendopt.all import backend_optimizations
+from pypy.translator.unsimplify import varoftype
+from pypy.rpython.lltypesystem import lltype, rffi
+from pypy.conftest import option
+from pypy.rlib import rgc
+
+
+class BaseFinalizerAnalyzerTests(object):
+    """ Below are typical destructors that we encounter in pypy
+    """
+
+    type_system = None
+    
+    def analyze(self, func, sig, func_to_analyze=None, backendopt=False):
+        if func_to_analyze is None:
+            func_to_analyze = func
+        t = TranslationContext()
+        t.buildannotator().build_types(func, sig)
+        t.buildrtyper(type_system=self.type_system).specialize()
+        if backendopt:
+            backend_optimizations(t)
+        if option.view:
+            t.view()
+        a = FinalizerAnalyzer(t)
+        fgraph = graphof(t, func_to_analyze)
+        result = a.analyze_light_finalizer(fgraph)
+        return result
+
+    def test_nothing(self):
+        def f():
+            pass
+        r = self.analyze(f, [])
+        assert not r
+
+def test_various_ops():
+    from pypy.objspace.flow.model import SpaceOperation, Constant
+
+    X = lltype.Ptr(lltype.GcStruct('X'))
+    Z = lltype.Ptr(lltype.Struct('Z'))
+    S = lltype.GcStruct('S', ('x', lltype.Signed),
+                        ('y', X),
+                        ('z', Z))
+    v1 = varoftype(lltype.Bool)
+    v2 = varoftype(lltype.Signed)
+    f = FinalizerAnalyzer(None)
+    r = f.analyze(SpaceOperation('cast_int_to_bool', [v2],
+                                                       v1))
+    assert not r
+    v1 = varoftype(lltype.Ptr(S))
+    v2 = varoftype(lltype.Signed)
+    v3 = varoftype(X)
+    v4 = varoftype(Z)
+    assert not f.analyze(SpaceOperation('bare_setfield', [v1, Constant('x'),
+                                                          v2], None))
+    assert     f.analyze(SpaceOperation('bare_setfield', [v1, Constant('y'),
+                                                          v3], None))
+    assert not f.analyze(SpaceOperation('bare_setfield', [v1, Constant('z'),
+                                                          v4], None))
+    
+        
+class TestLLType(BaseFinalizerAnalyzerTests):
+    type_system = 'lltype'
+
+    def test_malloc(self):
+        S = lltype.GcStruct('S')
+        
+        def f():
+            return lltype.malloc(S)
+
+        r = self.analyze(f, [])
+        assert r
+
+    def test_raw_free_getfield(self):
+        S = lltype.Struct('S')
+        
+        class A(object):
+            def __init__(self):
+                self.x = lltype.malloc(S, flavor='raw')
+
+            def __del__(self):
+                if self.x:
+                    self.x = lltype.nullptr(S)
+                    lltype.free(self.x, flavor='raw')
+
+        def f():
+            return A()
+
+        r = self.analyze(f, [], A.__del__.im_func)
+        assert not r
+
+    def test_c_call(self):
+        C = rffi.CArray(lltype.Signed)
+        c = rffi.llexternal('x', [lltype.Ptr(C)], lltype.Signed)
+
+        def g():
+            p = lltype.malloc(C, 3, flavor='raw')
+            f(p)
+        
+        def f(p):
+            c(rffi.ptradd(p, 0))
+            lltype.free(p, flavor='raw')
+
+        r = self.analyze(g, [], f, backendopt=True)
+        assert not r
+
+    def test_chain(self):
+        class B(object):
+            def __init__(self):
+                self.counter = 1
+        
+        class A(object):
+            def __init__(self):
+                self.x = B()
+
+            def __del__(self):
+                self.x.counter += 1
+
+        def f():
+            A()
+
+        r = self.analyze(f, [], A.__del__.im_func)
+        assert r
+
+    def test_is_light_finalizer_decorator(self):
+        S = lltype.GcStruct('S')
+
+        @rgc.is_light_finalizer
+        def f():
+            lltype.malloc(S)
+        @rgc.is_light_finalizer
+        def g():
+            pass
+        self.analyze(g, []) # did not explode
+        py.test.raises(FinalizerError, self.analyze, f, [])
+
+class TestOOType(BaseFinalizerAnalyzerTests):
+    type_system = 'ootype'
diff --git a/pypy/translator/c/database.py b/pypy/translator/c/database.py
--- a/pypy/translator/c/database.py
+++ b/pypy/translator/c/database.py
@@ -176,7 +176,7 @@
                      # introduced by the GC transformer, or the type_info_table
         return node
 
-    def get(self, obj):
+    def get(self, obj, funcgen=None):
         if isinstance(obj, CConstant):
             return obj.c_name  # without further checks
         T = typeOf(obj)
@@ -228,6 +228,8 @@
                     return '((%s) %d)' % (cdecl(self.gettype(T), ''),
                                           obj._obj)
                 node = self.getcontainernode(container)
+                if node._funccodegen_owner is None:
+                    node._funccodegen_owner = funcgen
                 return node.getptrname()
             else:
                 return '((%s) NULL)' % (cdecl(self.gettype(T), ''), )
@@ -284,14 +286,16 @@
             finish_callbacks.append(('GC transformer: finished tables',
                                      self.gctransformer.get_finish_tables()))
 
-        def add_dependencies(newdependencies):
+        def add_dependencies(newdependencies, parent=None):
             for value in newdependencies:
                 #if isinstance(value, _uninitialized):
                 #    continue
                 if isinstance(typeOf(value), ContainerType):
-                    self.getcontainernode(value)
+                    node = self.getcontainernode(value)
+                    if parent and node._funccodegen_owner is not None:
+                        node._funccodegen_owner = parent._funccodegen_owner
                 else:
-                    self.get(value)
+                    self.get(value, parent and parent._funccodegen_owner)
 
         while True:
             while True:
@@ -303,7 +307,7 @@
                 if i == len(self.containerlist):
                     break
                 node = self.containerlist[i]
-                add_dependencies(node.enum_dependencies())
+                add_dependencies(node.enum_dependencies(), node)
                 i += 1
                 self.completedcontainers = i
                 if i == show_i:
diff --git a/pypy/translator/c/gc.py b/pypy/translator/c/gc.py
--- a/pypy/translator/c/gc.py
+++ b/pypy/translator/c/gc.py
@@ -170,6 +170,7 @@
     nodekind = 'refcnt rtti'
     globalcontainer = True
     typename = 'void (@)(void *)'
+    _funccodegen_owner = None
 
     def __init__(self, db, T, obj):
         assert T == RuntimeTypeInfo
@@ -266,6 +267,7 @@
     nodekind = 'boehm rtti'
     globalcontainer = True
     typename = 'char @'
+    _funccodegen_owner = None
 
     def __init__(self, db, T, obj):
         assert T == RuntimeTypeInfo
diff --git a/pypy/translator/c/gcc/test/test_asmgcroot.py b/pypy/translator/c/gcc/test/test_asmgcroot.py
--- a/pypy/translator/c/gcc/test/test_asmgcroot.py
+++ b/pypy/translator/c/gcc/test/test_asmgcroot.py
@@ -21,6 +21,7 @@
         config = get_pypy_config(translating=True)
         config.translation.gc = cls.gcpolicy
         config.translation.gcrootfinder = "asmgcc"
+        config.translation.taggedpointers = getattr(cls, "taggedpointers", False)
         return config
 
     @classmethod
diff --git a/pypy/translator/c/genc.py b/pypy/translator/c/genc.py
--- a/pypy/translator/c/genc.py
+++ b/pypy/translator/c/genc.py
@@ -682,8 +682,7 @@
     def getbasecfilefornode(self, node, basecname):
         # For FuncNode instances, use the python source filename (relative to
         # the top directory):
-        if hasattr(node.obj, 'graph'):
-            g = node.obj.graph
+        def invent_nice_name(g):
             # Lookup the filename from the function.
             # However, not all FunctionGraph objs actually have a "func":
             if hasattr(g, 'func'):
@@ -693,6 +692,15 @@
                     if pypkgpath:
                         relpypath =  localpath.relto(pypkgpath)
                         return relpypath.replace('.py', '.c')
+            return None
+        if hasattr(node.obj, 'graph'):
+            name = invent_nice_name(node.obj.graph)
+            if name is not None:
+                return name
+        elif node._funccodegen_owner is not None:
+            name = invent_nice_name(node._funccodegen_owner.graph)
+            if name is not None:
+                return "data_" + name
         return basecname
 
     def splitnodesimpl(self, basecname, nodes, nextra, nbetween,
diff --git a/pypy/translator/c/node.py b/pypy/translator/c/node.py
--- a/pypy/translator/c/node.py
+++ b/pypy/translator/c/node.py
@@ -485,6 +485,7 @@
         __slots__ = """db obj 
                        typename implementationtypename
                         name
+                        _funccodegen_owner
                         globalcontainer""".split()
     eci_name = '_compilation_info'
 
@@ -509,6 +510,7 @@
         if self.typename != self.implementationtypename:
             if db.gettypedefnode(T).extra_union_for_varlength:
                 self.name += '.b'
+        self._funccodegen_owner = None
 
     def getptrname(self):
         return '(&%s)' % self.name
@@ -842,6 +844,9 @@
         if self.funcgens:
             argnames = self.funcgens[0].argnames()  #Assume identical for all funcgens
             self.implementationtypename = self.db.gettype(self.T, argnames=argnames)
+            self._funccodegen_owner = self.funcgens[0]
+        else:
+            self._funccodegen_owner = None
 
     def basename(self):
         return self.obj._name
@@ -1005,6 +1010,7 @@
     globalcontainer = True
     typename = 'PyObject @'
     implementationtypename = 'PyObject *@'
+    _funccodegen_owner = None
 
     def __init__(self, db, T, obj):
         # obj is a _pyobject here; obj.value is the underlying CPython object
diff --git a/pypy/translator/c/primitive.py b/pypy/translator/c/primitive.py
--- a/pypy/translator/c/primitive.py
+++ b/pypy/translator/c/primitive.py
@@ -141,12 +141,22 @@
 
 def name_gcref(value, db):
     if value:
-        realobj = value._obj.container
+        obj = value._obj
+        if isinstance(obj, int):
+            # a tagged pointer
+            return _name_tagged(obj, db)
+        realobj = obj.container
+        if isinstance(realobj, int):
+            return _name_tagged(realobj, db)
         realvalue = cast_opaque_ptr(Ptr(typeOf(realobj)), value)
         return db.get(realvalue)
     else:
         return 'NULL'
 
+def _name_tagged(obj, db):
+    assert obj & 1 == 1
+    return '((%s) %d)' % (cdecl("void*", ''), obj)
+
 def name_small_integer(value, db):
     """Works for integers of size at most INT or UINT."""
     if isinstance(value, Symbolic):
diff --git a/pypy/translator/c/test/test_newgc.py b/pypy/translator/c/test/test_newgc.py
--- a/pypy/translator/c/test/test_newgc.py
+++ b/pypy/translator/c/test/test_newgc.py
@@ -1487,6 +1487,43 @@
         res = self.run("tagged")
         assert res == expected
 
+    def define_erased(cls):
+        from pypy.rlib import rerased
+        erase, unerase = rerased.new_erasing_pair("test")
+        class Unrelated(object):
+            pass
+
+        u = Unrelated()
+        u.tagged = True
+        u.x = rerased.erase_int(41)
+        class A(object):
+            pass
+        def fn():
+            n = 1
+            while n >= 0:
+                if u.tagged:
+                    n = rerased.unerase_int(u.x)
+                    a = A()
+                    a.n = n - 1
+                    u.x = erase(a)
+                    u.tagged = False
+                else:
+                    n = unerase(u.x).n
+                    u.x = rerased.erase_int(n - 1)
+                    u.tagged = True
+        def func():
+            rgc.collect() # check that a prebuilt erased integer doesn't explode
+            u.x = rerased.erase_int(1000)
+            u.tagged = True
+            fn()
+            return 1
+        return func
+
+    def test_erased(self):
+        expected = self.run_orig("erased")
+        res = self.run("erased")
+        assert res == expected
+
 from pypy.rlib.objectmodel import UnboxedValue
 
 class TaggedBase(object):
diff --git a/pypy/translator/c/test/test_rtagged.py b/pypy/translator/c/test/test_rtagged.py
--- a/pypy/translator/c/test/test_rtagged.py
+++ b/pypy/translator/c/test/test_rtagged.py
@@ -77,3 +77,12 @@
     data = g.read()
     g.close()
     assert data.rstrip().endswith('ALL OK')
+
+def test_name_gcref():
+    from pypy.rpython.lltypesystem import lltype, llmemory, rclass
+    from pypy.translator.c import primitive
+    from pypy.translator.c.database import LowLevelDatabase
+    x = lltype.cast_int_to_ptr(rclass.OBJECTPTR, 19)
+    y = lltype.cast_opaque_ptr(llmemory.GCREF, x)
+    db = LowLevelDatabase()
+    assert primitive.name_gcref(y, db) == "((void*) 19)"
diff --git a/pypy/translator/platform/linux.py b/pypy/translator/platform/linux.py
--- a/pypy/translator/platform/linux.py
+++ b/pypy/translator/platform/linux.py
@@ -1,5 +1,6 @@
 """Support for Linux."""
 
+import sys
 from pypy.translator.platform.posix import BasePosix
 
 class BaseLinux(BasePosix):
@@ -26,7 +27,11 @@
 
     def library_dirs_for_libffi_a(self):
         # places where we need to look for libffi.a
-        return self.library_dirs_for_libffi() + ['/usr/lib']
+        # XXX obscuuure!  only look for libffi.a if run with translate.py
+        if 'translate' in sys.modules:
+            return self.library_dirs_for_libffi() + ['/usr/lib']
+        else:
+            return []
 
 
 class Linux(BaseLinux):
diff --git a/pypy/translator/platform/posix.py b/pypy/translator/platform/posix.py
--- a/pypy/translator/platform/posix.py
+++ b/pypy/translator/platform/posix.py
@@ -157,7 +157,7 @@
 
         rules = [
             ('all', '$(DEFAULT_TARGET)', []),
-            ('$(TARGET)', '$(OBJECTS)', '$(CC_LINK) $(LDFLAGS) $(LDFLAGSEXTRA) -o $@ $(OBJECTS) $(LIBDIRS) $(LIBS) $(LINKFILES)'),
+            ('$(TARGET)', '$(OBJECTS)', '$(CC_LINK) $(LDFLAGSEXTRA) -o $@ $(OBJECTS) $(LIBDIRS) $(LIBS) $(LINKFILES) $(LDFLAGS)'),
             ('%.o', '%.c', '$(CC) $(CFLAGS) $(CFLAGSEXTRA) -o $@ -c $< $(INCLUDEDIRS)'),
             ]
 

From noreply at buildbot.pypy.org  Wed Oct 26 21:29:29 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 26 Oct 2011 21:29:29 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-multidim: fix typo
Message-ID: <20111026192929.64620820C7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: numpy-multidim
Changeset: r48487:ac15793a59b2
Date: 2011-10-26 21:27 +0200
http://bitbucket.org/pypy/pypy/changeset/ac15793a59b2/

Log:	fix typo

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -310,7 +310,7 @@
 
 class Scalar(BaseArray):
     """
-    Intermediate class representing a float literal.
+    Intermediate class representing a literal.
     """
     signature = signature.BaseSignature()
 

From noreply at buildbot.pypy.org  Wed Oct 26 22:14:58 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 26 Oct 2011 22:14:58 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-multidim: for a good start - support for
	zeroes
Message-ID: <20111026201458.DC23E820C7@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: numpy-multidim
Changeset: r48488:159a0cccc4a9
Date: 2011-10-26 22:13 +0200
http://bitbucket.org/pypy/pypy/changeset/159a0cccc4a9/

Log:	for a good start - support for zeroes

diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -5,7 +5,7 @@
 
 from pypy.interpreter.baseobjspace import InternalSpaceCache, W_Root
 from pypy.module.micronumpy.interp_dtype import W_Float64Dtype
-from pypy.module.micronumpy.interp_numarray import Scalar, SingleDimArray, BaseArray
+from pypy.module.micronumpy.interp_numarray import Scalar, NDimArray, BaseArray
 from pypy.rlib.objectmodel import specialize
 
 
@@ -13,7 +13,7 @@
     pass
 
 def create_array(dtype, size):
-    a = SingleDimArray(size, dtype=dtype)
+    a = NDimArray(size, dtype=dtype)
     for i in range(size):
         dtype.setitem(a.storage, i, dtype.box(float(i % 10)))
     return a
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -46,7 +46,7 @@
         dtype = space.interp_w(interp_dtype.W_Dtype,
             space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype)
         )
-        arr = SingleDimArray(len(l), dtype=dtype)
+        arr = NDimArray(len(l), [len(l)], dtype=dtype)
         i = 0
         for w_elem in l:
             dtype.setitem_w(space, arr.storage, i, w_elem)
@@ -348,7 +348,7 @@
         i = 0
         signature = self.signature
         result_size = self.find_size()
-        result = SingleDimArray(result_size, self.find_dtype())
+        result = NDimArray(result_size, [result_size], self.find_dtype())
         while i < result_size:
             numpy_driver.jit_merge_point(signature=signature,
                                          result_size=result_size, i=i,
@@ -468,6 +468,7 @@
         raise NotImplementedError
 
     def descr_len(self, space):
+        # XXX find shape first
         return space.wrap(self.find_size())
 
     def calc_index(self, item):
@@ -510,10 +511,11 @@
         return (self.start + item * self.step)
 
 
-class SingleDimArray(BaseArray):
-    def __init__(self, size, dtype):
+class NDimArray(BaseArray):
+    def __init__(self, size, shape, dtype):
         BaseArray.__init__(self)
         self.size = size
+        self.shape = shape
         self.dtype = dtype
         self.storage = dtype.malloc(size)
         self.signature = dtype.signature
@@ -534,7 +536,7 @@
         return self.dtype.getitem(self.storage, i)
 
     def descr_len(self, space):
-        return space.wrap(self.size)
+        return space.wrap(self.shape[0])
 
     def setitem_w(self, space, item, w_value):
         self.invalidated()
@@ -550,12 +552,21 @@
     def __del__(self):
         lltype.free(self.storage, flavor='raw', track_allocation=False)
 
- at unwrap_spec(size=int)
-def zeros(space, size, w_dtype=None):
+def zeros(space, w_size, w_dtype=None):
     dtype = space.interp_w(interp_dtype.W_Dtype,
         space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype)
     )
-    return space.wrap(SingleDimArray(size, dtype=dtype))
+    if space.isinstance_w(w_size, space.w_int):
+        size = space.int_w(w_size)
+        shape = [size]
+    else:
+        size = 1
+        shape = []
+        for w_item in space.fixedview(w_size):
+            item = space.int_w(w_item)
+            size *= item
+            shape.append(item)
+    return space.wrap(NDimArray(size, shape, dtype=dtype))
 
 @unwrap_spec(size=int)
 def ones(space, size, w_dtype=None):
@@ -563,7 +574,7 @@
         space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype)
     )
 
-    arr = SingleDimArray(size, dtype=dtype)
+    arr = NDimArray(size, [size], dtype=dtype)
     one = dtype.adapt_val(1)
     for i in xrange(size):
         arr.dtype.setitem(arr.storage, i, one)
diff --git a/pypy/module/micronumpy/interp_support.py b/pypy/module/micronumpy/interp_support.py
--- a/pypy/module/micronumpy/interp_support.py
+++ b/pypy/module/micronumpy/interp_support.py
@@ -9,7 +9,7 @@
 
 @unwrap_spec(s=str)
 def fromstring(space, s):
-    from pypy.module.micronumpy.interp_numarray import SingleDimArray
+    from pypy.module.micronumpy.interp_numarray import NDimArray
     length = len(s)
 
     if length % FLOAT_SIZE == 0:
@@ -19,7 +19,7 @@
             "string length %d not divisable by %d" % (length, FLOAT_SIZE)))
 
     dtype = space.fromcache(W_Float64Dtype)
-    a = SingleDimArray(number, dtype=dtype)
+    a = NDimArray(number, dtype=dtype)
 
     start = 0
     end = FLOAT_SIZE
@@ -31,4 +31,4 @@
         start += FLOAT_SIZE
         end += FLOAT_SIZE
 
-    return space.wrap(a)
\ No newline at end of file
+    return space.wrap(a)
diff --git a/pypy/module/micronumpy/test/test_base.py b/pypy/module/micronumpy/test/test_base.py
--- a/pypy/module/micronumpy/test/test_base.py
+++ b/pypy/module/micronumpy/test/test_base.py
@@ -1,6 +1,6 @@
 from pypy.conftest import gettestobjspace
 from pypy.module.micronumpy import interp_dtype
-from pypy.module.micronumpy.interp_numarray import SingleDimArray, Scalar
+from pypy.module.micronumpy.interp_numarray import NDimArray, Scalar
 from pypy.module.micronumpy.interp_ufuncs import (find_binop_result_dtype,
         find_unaryop_result_dtype)
 
@@ -13,7 +13,7 @@
     def test_binop_signature(self, space):
         float64_dtype = space.fromcache(interp_dtype.W_Float64Dtype)
 
-        ar = SingleDimArray(10, dtype=float64_dtype)
+        ar = NDimArray(10, dtype=float64_dtype)
         v1 = ar.descr_add(space, ar)
         v2 = ar.descr_add(space, Scalar(float64_dtype, 2.0))
         assert v1.signature is not v2.signature
@@ -22,7 +22,7 @@
         v4 = ar.descr_add(space, ar)
         assert v1.signature is v4.signature
 
-        bool_ar = SingleDimArray(10, dtype=space.fromcache(interp_dtype.W_BoolDtype))
+        bool_ar = NDimArray(10, dtype=space.fromcache(interp_dtype.W_BoolDtype))
         v5 = ar.descr_add(space, bool_ar)
         assert v5.signature is not v1.signature
         assert v5.signature is not v2.signature
@@ -30,7 +30,7 @@
         assert v5.signature is v6.signature
 
     def test_slice_signature(self, space):
-        ar = SingleDimArray(10, dtype=space.fromcache(interp_dtype.W_Float64Dtype))
+        ar = NDimArray(10, dtype=space.fromcache(interp_dtype.W_Float64Dtype))
         v1 = ar.descr_getitem(space, space.wrap(slice(1, 5, 1)))
         v2 = ar.descr_getitem(space, space.wrap(slice(4, 6, 1)))
         assert v1.signature is v2.signature
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -599,6 +599,11 @@
             for i in xrange(5):
                 assert c[i] == func(b[i], 3)
 
+class AppTestMultiDim(BaseNumpyAppTest):
+    def test_init(self):
+        import numpy
+        a = numpy.zeros((2, 2))
+        assert len(a) == 2
 
 class AppTestSupport(object):
     def setup_class(cls):

From noreply at buildbot.pypy.org  Thu Oct 27 00:13:29 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Thu, 27 Oct 2011 00:13:29 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-dicts: added a test_pypy_c test for this
Message-ID: <20111026221329.B540E820C7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: virtual-dicts
Changeset: r48489:6034c17423bd
Date: 2011-10-26 18:13 -0400
http://bitbucket.org/pypy/pypy/changeset/6034c17423bd/

Log:	added a test_pypy_c test for this

diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py
--- a/pypy/module/pypyjit/test_pypy_c/test_call.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_call.py
@@ -465,3 +465,25 @@
             setfield_gc(p4, p22, descr=<GcPtrFieldDescr pypy.interpreter.nestedscope.Cell.inst_w_value .*>)
             jump(p0, p1, p2, p3, p4, p7, p22, p7, descr=<Loop0>)
         """)
+
+    def test_kwargs_virtual(self):
+        def main(n):
+            def g(**kwargs):
+                return kwargs["x"] + 1
+
+            i = 0
+            while i < n:
+                i = g(x=i)
+            return i
+
+        log = self.run(main, [500])
+        assert log.result == 500
+        loop, = log.loops_by_filename(self.filepath)
+        assert loop.match("""
+            i2 = int_lt(i0, i1)
+            guard_true(i2, descr=...)
+            i3 = force_token()
+            i4 = int_add(i0, 1)
+            --TICK--
+            jump(..., descr=...)
+        """)
\ No newline at end of file

From noreply at buildbot.pypy.org  Thu Oct 27 01:50:23 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Thu, 27 Oct 2011 01:50:23 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-dicts: finally fix this test
Message-ID: <20111026235023.CDD93820C7@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: virtual-dicts
Changeset: r48490:bc6e9f337a06
Date: 2011-10-26 19:50 -0400
http://bitbucket.org/pypy/pypy/changeset/bc6e9f337a06/

Log:	finally fix this test

diff --git a/pypy/module/pypyjit/test_pypy_c/test_containers.py b/pypy/module/pypyjit/test_pypy_c/test_containers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_containers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_containers.py
@@ -44,7 +44,7 @@
         # gc_id call is hoisted out of the loop, the id of a value obviously
         # can't change ;)
         assert loop.match_by_id("getitem", """
-            i28 = call(ConstClass(ll_dict_lookup__dicttablePtr_objectPtr_Signed), p18, p6, i25, descr=...)
+            i26 = call(ConstClass(ll_dict_lookup), p18, p6, i25, descr=...)
             ...
             p33 = getinteriorfield_gc(p31, i26, descr=<InteriorFieldDescr <GcPtrFieldDescr dictentry.value .*>>)
             ...

From noreply at buildbot.pypy.org  Thu Oct 27 10:25:09 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Oct 2011 10:25:09 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-multidim: shape
Message-ID: <20111027082509.7EE23820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: numpy-multidim
Changeset: r48491:cc8b141031c6
Date: 2011-10-26 22:34 +0200
http://bitbucket.org/pypy/pypy/changeset/cc8b141031c6/

Log:	shape

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -199,7 +199,7 @@
         return space.wrap(self.find_dtype())
 
     def descr_get_shape(self, space):
-        return space.newtuple([self.descr_len(space)])
+        return space.newtuple([space.wrap(i) for i in self.shape])
 
     def descr_copy(self, space):
         return space.call_function(space.gettypefor(BaseArray), self, self.find_dtype())
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -605,6 +605,14 @@
         a = numpy.zeros((2, 2))
         assert len(a) == 2
 
+    def test_shape(self):
+        import numpy
+        assert numpy.zeros(1).shape == (1,)
+        assert numpy.zeros((2, 2)).shape == (2,2)
+        assert numpy.zeros((3, 1, 2)).shape == (3, 1, 2)
+        assert len(numpy.zeros((3, 1, 2))) == 3
+        
+
 class AppTestSupport(object):
     def setup_class(cls):
         import struct

From noreply at buildbot.pypy.org  Thu Oct 27 10:25:10 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Oct 2011 10:25:10 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-multidim: tentative checkin
Message-ID: <20111027082510.AD3BF820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: numpy-multidim
Changeset: r48492:4dd7d695536a
Date: 2011-10-27 00:04 +0200
http://bitbucket.org/pypy/pypy/changeset/4dd7d695536a/

Log:	tentative checkin

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -223,17 +223,32 @@
         concrete = self.get_concrete()
         return space.wrap("[" + " ".join(concrete._getnums(True)) + "]")
 
+    def item_at_index(self, index, space):
+        # we assume C ordering for now
+        item = 0
+        for i in range(len(index)):
+            if i != 0:
+                item *= self.shape[i]
+            if index[i] >= self.shape[i]:
+                raise OperationError(space.w_IndexError,
+                                     space.wrap("index (%d) out of range (0<=index<%d" % (index[i], self.shape[i])))
+            item += index[i]
+        return item
+
     def descr_getitem(self, space, w_idx):
         # TODO: indexing by arrays and lists
         if space.isinstance_w(w_idx, space.w_tuple):
+            # or any other sequence actually
             length = space.len_w(w_idx)
             if length == 0:
                 return space.wrap(self)
-            if length > 1: # only one dimension for now.
+            if length > len(self.shape):
                 raise OperationError(space.w_IndexError,
                                      space.wrap("invalid index"))
-            w_idx = space.getitem(w_idx, space.wrap(0))
-        start, stop, step, slice_length = space.decode_index4(w_idx, self.find_size())
+            indices = [space.int_w(w_item) for w_item in space.fixedview(w_idx)]
+            item = self.item_at_index(indices, space)
+            return self.get_concrete().eval(item).wrap(space)
+        start, stop, step, slice_length = space.decode_index4(w_idx, self.shape[0])
         if step == 0:
             # Single index
             return self.get_concrete().eval(start).wrap(space)
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -611,7 +611,17 @@
         assert numpy.zeros((2, 2)).shape == (2,2)
         assert numpy.zeros((3, 1, 2)).shape == (3, 1, 2)
         assert len(numpy.zeros((3, 1, 2))) == 3
-        
+
+    def test_getsetitem(self):
+        import numpy
+        a = numpy.zeros((2, 3, 1))
+        raises(IndexError, a.__getitem__, (0, 0, 0, 0))
+        raises(IndexError, a.__getitem__, (3,))
+        raises(IndexError, a.__getitem__, (1, 3))
+        assert a[1, 1, 0] == 0
+        a[1, 2, 0] = 3
+        assert a[1, 2, 0] == 3
+        assert a[1, 1, 0] == 0
 
 class AppTestSupport(object):
     def setup_class(cls):

From noreply at buildbot.pypy.org  Thu Oct 27 10:25:11 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Oct 2011 10:25:11 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-multidim: a better fix for test_zll_random
Message-ID: <20111027082511.ED1A5820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: numpy-multidim
Changeset: r48493:52416083d855
Date: 2011-10-27 10:24 +0200
http://bitbucket.org/pypy/pypy/changeset/52416083d855/

Log:	a better fix for test_zll_random

diff --git a/pypy/jit/backend/llsupport/regalloc.py b/pypy/jit/backend/llsupport/regalloc.py
--- a/pypy/jit/backend/llsupport/regalloc.py
+++ b/pypy/jit/backend/llsupport/regalloc.py
@@ -178,8 +178,6 @@
         cur_max_age = -1
         candidate = None
         for next in self.reg_bindings:
-            if isinstance(next, TempBox):
-                continue
             reg = self.reg_bindings[next]
             if next in forbidden_vars:
                 continue
diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py
--- a/pypy/jit/backend/x86/regalloc.py
+++ b/pypy/jit/backend/x86/regalloc.py
@@ -1050,7 +1050,7 @@
         tempvar = TempBox()
         index_loc = self.rm.force_result_in_reg(tempvar, op.getarg(1), args)
         # we're free to modify index now
-        value_loc = self.make_sure_var_in_reg(op.getarg(2), args,
+        value_loc = self.make_sure_var_in_reg(op.getarg(2), args + [tempvar],
                                               need_lower_byte=need_lower_byte)
         self.rm.possibly_free_var(tempvar)
         self.possibly_free_vars(args)

From noreply at buildbot.pypy.org  Thu Oct 27 10:25:53 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Oct 2011 10:25:53 +0200 (CEST)
Subject: [pypy-commit] pypy default: a better fix for test_zll_random
Message-ID: <20111027082553.7EF5E820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r48494:b6e0fd170dae
Date: 2011-10-27 10:24 +0200
http://bitbucket.org/pypy/pypy/changeset/b6e0fd170dae/

Log:	a better fix for test_zll_random

diff --git a/pypy/jit/backend/llsupport/regalloc.py b/pypy/jit/backend/llsupport/regalloc.py
--- a/pypy/jit/backend/llsupport/regalloc.py
+++ b/pypy/jit/backend/llsupport/regalloc.py
@@ -178,8 +178,6 @@
         cur_max_age = -1
         candidate = None
         for next in self.reg_bindings:
-            if isinstance(next, TempBox):
-                continue
             reg = self.reg_bindings[next]
             if next in forbidden_vars:
                 continue
diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py
--- a/pypy/jit/backend/x86/regalloc.py
+++ b/pypy/jit/backend/x86/regalloc.py
@@ -1050,7 +1050,7 @@
         tempvar = TempBox()
         index_loc = self.rm.force_result_in_reg(tempvar, op.getarg(1), args)
         # we're free to modify index now
-        value_loc = self.make_sure_var_in_reg(op.getarg(2), args,
+        value_loc = self.make_sure_var_in_reg(op.getarg(2), args + [tempvar],
                                               need_lower_byte=need_lower_byte)
         self.rm.possibly_free_var(tempvar)
         self.possibly_free_vars(args)

From noreply at buildbot.pypy.org  Thu Oct 27 10:57:11 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 27 Oct 2011 10:57:11 +0200 (CEST)
Subject: [pypy-commit] pypy default: Revert the hack to llsupport/regalloc.
 Implement a version that
Message-ID: <20111027085711.DB818820C2@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48495:478d71b47de8
Date: 2011-10-27 10:51 +0200
http://bitbucket.org/pypy/pypy/changeset/478d71b47de8/

Log:	Revert the hack to llsupport/regalloc. Implement a version that
	uses the hopefully "correct and official" API.

diff --git a/pypy/jit/backend/llsupport/regalloc.py b/pypy/jit/backend/llsupport/regalloc.py
--- a/pypy/jit/backend/llsupport/regalloc.py
+++ b/pypy/jit/backend/llsupport/regalloc.py
@@ -178,8 +178,6 @@
         cur_max_age = -1
         candidate = None
         for next in self.reg_bindings:
-            if isinstance(next, TempBox):
-                continue
             reg = self.reg_bindings[next]
             if next in forbidden_vars:
                 continue
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1596,11 +1596,24 @@
     genop_getarrayitem_gc_pure = genop_getarrayitem_gc
     genop_getarrayitem_raw = genop_getarrayitem_gc
 
+    def _get_interiorfield_index(self, temp_loc, index_loc, itemsize_loc):
+        assert isinstance(itemsize_loc, ImmedLoc)
+        if isinstance(index_loc, ImmedLoc):
+            return imm(index_loc.value * itemsize_loc.value)
+        else:
+            # XXX should not use IMUL in most cases
+            assert isinstance(temp_loc, RegLoc)
+            assert isinstance(index_loc, RegLoc)
+            self.mc.IMUL_rri(temp_loc.value, index_loc.value,
+                             itemsize_loc.value)
+            return temp_loc
+
     def genop_getinteriorfield_gc(self, op, arglocs, resloc):
-        base_loc, ofs_loc, itemsize_loc, fieldsize_loc, index_loc, sign_loc = arglocs
-        # XXX should not use IMUL in most cases
-        self.mc.IMUL(index_loc, itemsize_loc)
-        src_addr = AddressLoc(base_loc, index_loc, 0, ofs_loc.value)
+        (base_loc, ofs_loc, itemsize_loc, fieldsize_loc,
+            index_loc, sign_loc) = arglocs
+        temp_loc = self._get_interiorfield_index(resloc, index_loc,
+                                                 itemsize_loc)
+        src_addr = AddressLoc(base_loc, temp_loc, 0, ofs_loc.value)
         self.load_from_mem(resloc, src_addr, fieldsize_loc, sign_loc)
 
 
@@ -1611,13 +1624,11 @@
         self.save_into_mem(dest_addr, value_loc, size_loc)
 
     def genop_discard_setinteriorfield_gc(self, op, arglocs):
-        base_loc, ofs_loc, itemsize_loc, fieldsize_loc, index_loc, value_loc = arglocs
-        # XXX should not use IMUL in most cases
-        if isinstance(index_loc, ImmedLoc):
-            index_loc = imm(index_loc.value * itemsize_loc.value)
-        else:
-            self.mc.IMUL(index_loc, itemsize_loc)
-        dest_addr = AddressLoc(base_loc, index_loc, 0, ofs_loc.value)
+        (base_loc, ofs_loc, itemsize_loc, fieldsize_loc,
+            index_loc, temp_loc, value_loc) = arglocs
+        temp_loc = self._get_interiorfield_index(temp_loc, index_loc,
+                                                 itemsize_loc)
+        dest_addr = AddressLoc(base_loc, temp_loc, 0, ofs_loc.value)
         self.save_into_mem(dest_addr, value_loc, fieldsize_loc)
 
     def genop_discard_setarrayitem_gc(self, op, arglocs):
diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py
--- a/pypy/jit/backend/x86/regalloc.py
+++ b/pypy/jit/backend/x86/regalloc.py
@@ -1046,16 +1046,26 @@
             need_lower_byte = True
         else:
             need_lower_byte = False
-        base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
-        tempvar = TempBox()
-        index_loc = self.rm.force_result_in_reg(tempvar, op.getarg(1), args)
-        # we're free to modify index now
-        value_loc = self.make_sure_var_in_reg(op.getarg(2), args,
+        box_base, box_index, box_value = args
+        base_loc = self.rm.make_sure_var_in_reg(box_base, args)
+        index_loc = self.rm.make_sure_var_in_reg(box_index, args)
+        value_loc = self.make_sure_var_in_reg(box_value, args,
                                               need_lower_byte=need_lower_byte)
-        self.rm.possibly_free_var(tempvar)
-        self.possibly_free_vars(args)
+        # If 'index_loc' is not an immediate, then we need a 'temp_loc' that
+        # is a register whose value will be destroyed.  It's fine to destroy
+        # the same register as 'index_loc', but not the other ones.
+        self.rm.possibly_free_var(box_index)
+        if not isinstance(index_loc, ImmedLoc):
+            tempvar = TempBox()
+            temp_loc = self.rm.force_allocate_reg(tempvar, [box_base,
+                                                            box_value])
+            self.rm.possibly_free_var(tempvar)
+        else:
+            temp_loc = None
+        self.rm.possibly_free_var(box_base)
+        self.possibly_free_var(box_value)
         self.PerformDiscard(op, [base_loc, ofs, itemsize, fieldsize,
-                                 index_loc, value_loc])
+                                 index_loc, temp_loc, value_loc])
 
     def consider_strsetitem(self, op):
         args = op.getarglist()
@@ -1126,13 +1136,14 @@
         else:
             sign_loc = imm0
         args = op.getarglist()
-        tmpvar = TempBox()
         base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
-        index_loc = self.rm.force_result_in_reg(tmpvar, op.getarg(1),
-                                                args)
-        self.rm.possibly_free_vars_for_op(op)
-        self.rm.possibly_free_var(tmpvar)
-        result_loc = self.force_allocate_reg(op.result)
+        index_loc = self.rm.make_sure_var_in_reg(op.getarg(1), args)
+        # 'base' and 'index' are put in two registers (or one if 'index'
+        # is an immediate).  'result' can be in the same register as
+        # 'index' but must be in a different register than 'base'.
+        self.rm.possibly_free_var(op.getarg(1))
+        result_loc = self.force_allocate_reg(op.result, [op.getarg(0)])
+        self.rm.possibly_free_var(op.getarg(0))
         self.Perform(op, [base_loc, ofs, itemsize, fieldsize,
                           index_loc, sign_loc], result_loc)
 

From noreply at buildbot.pypy.org  Thu Oct 27 10:57:13 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 27 Oct 2011 10:57:13 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20111027085713.1C18A820C2@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48496:38058e92147d
Date: 2011-10-27 10:56 +0200
http://bitbucket.org/pypy/pypy/changeset/38058e92147d/

Log:	merge heads

diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1596,11 +1596,24 @@
     genop_getarrayitem_gc_pure = genop_getarrayitem_gc
     genop_getarrayitem_raw = genop_getarrayitem_gc
 
+    def _get_interiorfield_index(self, temp_loc, index_loc, itemsize_loc):
+        assert isinstance(itemsize_loc, ImmedLoc)
+        if isinstance(index_loc, ImmedLoc):
+            return imm(index_loc.value * itemsize_loc.value)
+        else:
+            # XXX should not use IMUL in most cases
+            assert isinstance(temp_loc, RegLoc)
+            assert isinstance(index_loc, RegLoc)
+            self.mc.IMUL_rri(temp_loc.value, index_loc.value,
+                             itemsize_loc.value)
+            return temp_loc
+
     def genop_getinteriorfield_gc(self, op, arglocs, resloc):
-        base_loc, ofs_loc, itemsize_loc, fieldsize_loc, index_loc, sign_loc = arglocs
-        # XXX should not use IMUL in most cases
-        self.mc.IMUL(index_loc, itemsize_loc)
-        src_addr = AddressLoc(base_loc, index_loc, 0, ofs_loc.value)
+        (base_loc, ofs_loc, itemsize_loc, fieldsize_loc,
+            index_loc, sign_loc) = arglocs
+        temp_loc = self._get_interiorfield_index(resloc, index_loc,
+                                                 itemsize_loc)
+        src_addr = AddressLoc(base_loc, temp_loc, 0, ofs_loc.value)
         self.load_from_mem(resloc, src_addr, fieldsize_loc, sign_loc)
 
 
@@ -1611,13 +1624,11 @@
         self.save_into_mem(dest_addr, value_loc, size_loc)
 
     def genop_discard_setinteriorfield_gc(self, op, arglocs):
-        base_loc, ofs_loc, itemsize_loc, fieldsize_loc, index_loc, value_loc = arglocs
-        # XXX should not use IMUL in most cases
-        if isinstance(index_loc, ImmedLoc):
-            index_loc = imm(index_loc.value * itemsize_loc.value)
-        else:
-            self.mc.IMUL(index_loc, itemsize_loc)
-        dest_addr = AddressLoc(base_loc, index_loc, 0, ofs_loc.value)
+        (base_loc, ofs_loc, itemsize_loc, fieldsize_loc,
+            index_loc, temp_loc, value_loc) = arglocs
+        temp_loc = self._get_interiorfield_index(temp_loc, index_loc,
+                                                 itemsize_loc)
+        dest_addr = AddressLoc(base_loc, temp_loc, 0, ofs_loc.value)
         self.save_into_mem(dest_addr, value_loc, fieldsize_loc)
 
     def genop_discard_setarrayitem_gc(self, op, arglocs):
diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py
--- a/pypy/jit/backend/x86/regalloc.py
+++ b/pypy/jit/backend/x86/regalloc.py
@@ -1046,16 +1046,26 @@
             need_lower_byte = True
         else:
             need_lower_byte = False
-        base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
-        tempvar = TempBox()
-        index_loc = self.rm.force_result_in_reg(tempvar, op.getarg(1), args)
-        # we're free to modify index now
-        value_loc = self.make_sure_var_in_reg(op.getarg(2), args + [tempvar],
+        box_base, box_index, box_value = args
+        base_loc = self.rm.make_sure_var_in_reg(box_base, args)
+        index_loc = self.rm.make_sure_var_in_reg(box_index, args)
+        value_loc = self.make_sure_var_in_reg(box_value, args,
                                               need_lower_byte=need_lower_byte)
-        self.rm.possibly_free_var(tempvar)
-        self.possibly_free_vars(args)
+        # If 'index_loc' is not an immediate, then we need a 'temp_loc' that
+        # is a register whose value will be destroyed.  It's fine to destroy
+        # the same register as 'index_loc', but not the other ones.
+        self.rm.possibly_free_var(box_index)
+        if not isinstance(index_loc, ImmedLoc):
+            tempvar = TempBox()
+            temp_loc = self.rm.force_allocate_reg(tempvar, [box_base,
+                                                            box_value])
+            self.rm.possibly_free_var(tempvar)
+        else:
+            temp_loc = None
+        self.rm.possibly_free_var(box_base)
+        self.possibly_free_var(box_value)
         self.PerformDiscard(op, [base_loc, ofs, itemsize, fieldsize,
-                                 index_loc, value_loc])
+                                 index_loc, temp_loc, value_loc])
 
     def consider_strsetitem(self, op):
         args = op.getarglist()
@@ -1126,13 +1136,14 @@
         else:
             sign_loc = imm0
         args = op.getarglist()
-        tmpvar = TempBox()
         base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
-        index_loc = self.rm.force_result_in_reg(tmpvar, op.getarg(1),
-                                                args)
-        self.rm.possibly_free_vars_for_op(op)
-        self.rm.possibly_free_var(tmpvar)
-        result_loc = self.force_allocate_reg(op.result)
+        index_loc = self.rm.make_sure_var_in_reg(op.getarg(1), args)
+        # 'base' and 'index' are put in two registers (or one if 'index'
+        # is an immediate).  'result' can be in the same register as
+        # 'index' but must be in a different register than 'base'.
+        self.rm.possibly_free_var(op.getarg(1))
+        result_loc = self.force_allocate_reg(op.result, [op.getarg(0)])
+        self.rm.possibly_free_var(op.getarg(0))
         self.Perform(op, [base_loc, ofs, itemsize, fieldsize,
                           index_loc, sign_loc], result_loc)
 

From noreply at buildbot.pypy.org  Thu Oct 27 11:04:49 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 27 Oct 2011 11:04:49 +0200 (CEST)
Subject: [pypy-commit] pypy default: A better interface. Possibly a
	translation fix.
Message-ID: <20111027090449.307CF820C2@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48497:9f32a507b5be
Date: 2011-10-27 11:04 +0200
http://bitbucket.org/pypy/pypy/changeset/9f32a507b5be/

Log:	A better interface. Possibly a translation fix.

diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1596,24 +1596,26 @@
     genop_getarrayitem_gc_pure = genop_getarrayitem_gc
     genop_getarrayitem_raw = genop_getarrayitem_gc
 
-    def _get_interiorfield_index(self, temp_loc, index_loc, itemsize_loc):
+    def _get_interiorfield_addr(self, temp_loc, index_loc, itemsize_loc,
+                                base_loc, ofs_loc):
         assert isinstance(itemsize_loc, ImmedLoc)
         if isinstance(index_loc, ImmedLoc):
-            return imm(index_loc.value * itemsize_loc.value)
+            temp_loc = imm(index_loc.value * itemsize_loc.value)
         else:
             # XXX should not use IMUL in most cases
             assert isinstance(temp_loc, RegLoc)
             assert isinstance(index_loc, RegLoc)
             self.mc.IMUL_rri(temp_loc.value, index_loc.value,
                              itemsize_loc.value)
-            return temp_loc
+        assert isinstance(ofs_loc, ImmedLoc)
+        return AddressLoc(base_loc, temp_loc, 0, ofs_loc.value)
 
     def genop_getinteriorfield_gc(self, op, arglocs, resloc):
         (base_loc, ofs_loc, itemsize_loc, fieldsize_loc,
             index_loc, sign_loc) = arglocs
-        temp_loc = self._get_interiorfield_index(resloc, index_loc,
-                                                 itemsize_loc)
-        src_addr = AddressLoc(base_loc, temp_loc, 0, ofs_loc.value)
+        src_addr = self._get_interiorfield_addr(resloc, index_loc,
+                                                itemsize_loc, base_loc,
+                                                ofs_loc)
         self.load_from_mem(resloc, src_addr, fieldsize_loc, sign_loc)
 
 
@@ -1626,9 +1628,9 @@
     def genop_discard_setinteriorfield_gc(self, op, arglocs):
         (base_loc, ofs_loc, itemsize_loc, fieldsize_loc,
             index_loc, temp_loc, value_loc) = arglocs
-        temp_loc = self._get_interiorfield_index(temp_loc, index_loc,
-                                                 itemsize_loc)
-        dest_addr = AddressLoc(base_loc, temp_loc, 0, ofs_loc.value)
+        dest_addr = self._get_interiorfield_addr(temp_loc, index_loc,
+                                                 itemsize_loc, base_loc,
+                                                 ofs_loc)
         self.save_into_mem(dest_addr, value_loc, fieldsize_loc)
 
     def genop_discard_setarrayitem_gc(self, op, arglocs):

From noreply at buildbot.pypy.org  Thu Oct 27 11:53:19 2011
From: noreply at buildbot.pypy.org (cfbolz)
Date: Thu, 27 Oct 2011 11:53:19 +0200 (CEST)
Subject: [pypy-commit] pypy int-tag-untag-as-operations: make all
	optimizeopt tests pass
Message-ID: <20111027095319.34BBB820C2@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: int-tag-untag-as-operations
Changeset: r48498:5dea661f90f6
Date: 2011-10-25 13:53 +0200
http://bitbucket.org/pypy/pypy/changeset/5dea661f90f6/

Log:	make all optimizeopt tests pass

diff --git a/pypy/jit/backend/llgraph/llimpl.py b/pypy/jit/backend/llgraph/llimpl.py
--- a/pypy/jit/backend/llgraph/llimpl.py
+++ b/pypy/jit/backend/llgraph/llimpl.py
@@ -87,10 +87,13 @@
     'int_is_true'     : (('int',), 'bool'),
     'int_is_zero'     : (('int',), 'bool'),
     'int_neg'         : (('int',), 'int'),
+    'int_tag'         : (('int', ), 'int'),
+    'int_untag'       : (('int', ), 'int'),
     'int_invert'      : (('int',), 'int'),
     'int_add_ovf'     : (('int', 'int'), 'int'),
     'int_sub_ovf'     : (('int', 'int'), 'int'),
     'int_mul_ovf'     : (('int', 'int'), 'int'),
+    'int_tag_ovf'     : (('int', ), 'int'),
     'uint_add'        : (('int', 'int'), 'int'),
     'uint_sub'        : (('int', 'int'), 'int'),
     'uint_mul'        : (('int', 'int'), 'int'),
diff --git a/pypy/jit/metainterp/optimizeopt/intbounds.py b/pypy/jit/metainterp/optimizeopt/intbounds.py
--- a/pypy/jit/metainterp/optimizeopt/intbounds.py
+++ b/pypy/jit/metainterp/optimizeopt/intbounds.py
@@ -305,20 +305,20 @@
 
     def optimize_INT_TAG(self, op):
         v1 = self.getvalue(op.getarg(0))
+        self.emit_operation(op)
         r = self.getvalue(op.result)
         resbound = v1.intbound.mul(2).add(1)
         r.intbound.intersect(resbound)
         maxbounds = IntBound((-sys.maxint-1) >> 1, sys.maxint >> 1)
         v1.intbound.intersect(maxbounds)
         self.pure(rop.INT_UNTAG, [op.result], op.getarg(0))
-        self.emit_operation(op)
 
     def optimize_INT_UNTAG(self, op):
         v1 = self.getvalue(op.getarg(0))
         self.pure(rop.INT_TAG, [op.result], op.getarg(0))
+        self.emit_operation(op)
         r = self.getvalue(op.result)
         r.intbound.intersect(v1.intbound.rshift_bound(IntBound(1, 1)))
-        self.emit_operation(op)
 
     def optimize_ARRAYLEN_GC(self, op):
         self.emit_operation(op)
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -5124,6 +5124,7 @@
         [i0]
         i1 = int_lt(i0, 1000)
         guard_true(i1), []
+        i2 = int_tag(i0)
         i3 = int_add(i0, 1)
         jump(i3)
         """

From noreply at buildbot.pypy.org  Thu Oct 27 11:53:20 2011
From: noreply at buildbot.pypy.org (cfbolz)
Date: Thu, 27 Oct 2011 11:53:20 +0200 (CEST)
Subject: [pypy-commit] pypy int-tag-untag-as-operations: those are also not
	needed
Message-ID: <20111027095320.62A8D820C2@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: int-tag-untag-as-operations
Changeset: r48499:be97182b1a31
Date: 2011-10-27 11:52 +0200
http://bitbucket.org/pypy/pypy/changeset/be97182b1a31/

Log:	those are also not needed

diff --git a/pypy/jit/metainterp/test/test_immutable.py b/pypy/jit/metainterp/test/test_immutable.py
--- a/pypy/jit/metainterp/test/test_immutable.py
+++ b/pypy/jit/metainterp/test/test_immutable.py
@@ -64,7 +64,7 @@
             l[2] = 30
             a = escape(X(l))
             return a.y[index]
-        res = self.interp_operations(f, [2], listops=True)
+        res = self.interp_operations(f, [2])
         assert res == 30
         self.check_operations_history(getfield_gc=0, getfield_gc_pure=1,
                             getarrayitem_gc=0, getarrayitem_gc_pure=1)
@@ -82,7 +82,7 @@
         def f(x, index):
             y = escape(X([x], x+1))
             return y.lst[index] + y.y + 5
-        res = self.interp_operations(f, [23, 0], listops=True)
+        res = self.interp_operations(f, [23, 0])
         assert res == 23 + 24 + 5
         self.check_operations_history(getfield_gc=0, getfield_gc_pure=2,
                             getarrayitem_gc=0, getarrayitem_gc_pure=1,
diff --git a/pypy/jit/metainterp/test/test_list.py b/pypy/jit/metainterp/test/test_list.py
--- a/pypy/jit/metainterp/test/test_list.py
+++ b/pypy/jit/metainterp/test/test_list.py
@@ -123,7 +123,7 @@
             l2 = l[:]
             return l2[0] + l2[1] + l2[2] + l2[3]
 
-        res = self.interp_operations(f, [], listops=True)
+        res = self.interp_operations(f, [])
         assert res == 10
 
     def test_arraycopy_full(self):
diff --git a/pypy/jit/metainterp/test/test_slist.py b/pypy/jit/metainterp/test/test_slist.py
--- a/pypy/jit/metainterp/test/test_slist.py
+++ b/pypy/jit/metainterp/test/test_slist.py
@@ -31,7 +31,7 @@
             lst2.append(FooBar(5))
             m += lst2.pop().z     # 49
             return m
-        res = self.interp_operations(f, [11], listops=True)
+        res = self.interp_operations(f, [11])
         assert res == 49
         self.check_operations_history(call=3)
 

From noreply at buildbot.pypy.org  Thu Oct 27 11:53:23 2011
From: noreply at buildbot.pypy.org (cfbolz)
Date: Thu, 27 Oct 2011 11:53:23 +0200 (CEST)
Subject: [pypy-commit] pypy int-tag-untag-as-operations: merge default
Message-ID: <20111027095323.A3DE6820C2@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: int-tag-untag-as-operations
Changeset: r48500:76ff9be0b630
Date: 2011-10-27 11:52 +0200
http://bitbucket.org/pypy/pypy/changeset/76ff9be0b630/

Log:	merge default

diff --git a/lib-python/modified-2.7/urllib2.py b/lib-python/modified-2.7/urllib2.py
--- a/lib-python/modified-2.7/urllib2.py
+++ b/lib-python/modified-2.7/urllib2.py
@@ -395,11 +395,7 @@
         meth_name = protocol+"_response"
         for processor in self.process_response.get(protocol, []):
             meth = getattr(processor, meth_name)
-            try:
-                response = meth(req, response)
-            except:
-                response.close()
-                raise
+            response = meth(req, response)
 
         return response
 
diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -17,6 +17,12 @@
 projects, or anything else in PyPy, pop up on IRC or write to us on the
 `mailing list`_.
 
+Make big integers faster
+-------------------------
+
+PyPy's implementation of the Python ``long`` type is slower than CPython's.
+Find out why and optimize them.
+
 Numpy improvements
 ------------------
 
diff --git a/pypy/jit/backend/llgraph/llimpl.py b/pypy/jit/backend/llgraph/llimpl.py
--- a/pypy/jit/backend/llgraph/llimpl.py
+++ b/pypy/jit/backend/llgraph/llimpl.py
@@ -7,9 +7,7 @@
 import weakref
 from pypy.objspace.flow.model import Variable, Constant
 from pypy.annotation import model as annmodel
-from pypy.jit.metainterp.history import (ConstInt, ConstPtr,
-                                         BoxInt, BoxPtr, BoxObj, BoxFloat,
-                                         REF, INT, FLOAT)
+from pypy.jit.metainterp.history import REF, INT, FLOAT
 from pypy.jit.codewriter import heaptracker
 from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr, rffi
 from pypy.rpython.ootypesystem import ootype
@@ -17,7 +15,7 @@
 from pypy.rpython.llinterp import LLException
 from pypy.rpython.extregistry import ExtRegistryEntry
 
-from pypy.jit.metainterp import resoperation, executor
+from pypy.jit.metainterp import resoperation
 from pypy.jit.metainterp.resoperation import rop
 from pypy.jit.backend.llgraph import symbolic
 from pypy.jit.codewriter import longlong
@@ -337,6 +335,13 @@
     assert isinstance(type, str) and len(type) == 1
     op.descr = Descr(ofs, type, arg_types=arg_types)
 
+def compile_add_descr_arg(loop, ofs, type, arg_types):
+    from pypy.jit.backend.llgraph.runner import Descr
+    loop = _from_opaque(loop)
+    op = loop.operations[-1]
+    assert isinstance(type, str) and len(type) == 1
+    op.args.append(Descr(ofs, type, arg_types=arg_types))
+
 def compile_add_loop_token(loop, descr):
     if we_are_translated():
         raise ValueError("CALL_ASSEMBLER not supported")
@@ -441,8 +446,11 @@
         self._may_force = -1
 
     def getenv(self, v):
+        from pypy.jit.backend.llgraph.runner import Descr
         if isinstance(v, Constant):
             return v.value
+        elif isinstance(v, Descr):
+            return v
         else:
             return self.env[v]
 
@@ -821,6 +829,29 @@
         else:
             raise NotImplementedError
 
+    def op_getinteriorfield_gc(self, descr, array, index):
+        if descr.typeinfo == REF:
+            return do_getinteriorfield_gc_ptr(array, index, descr.ofs)
+        elif descr.typeinfo == INT:
+            return do_getinteriorfield_gc_int(array, index, descr.ofs)
+        elif descr.typeinfo == FLOAT:
+            return do_getinteriorfield_gc_float(array, index, descr.ofs)
+        else:
+            raise NotImplementedError
+
+    def op_setinteriorfield_gc(self, descr, array, index, newvalue):
+        if descr.typeinfo == REF:
+            return do_setinteriorfield_gc_ptr(array, index, descr.ofs,
+                                              newvalue)
+        elif descr.typeinfo == INT:
+            return do_setinteriorfield_gc_int(array, index, descr.ofs,
+                                              newvalue)
+        elif descr.typeinfo == FLOAT:
+            return do_setinteriorfield_gc_float(array, index, descr.ofs,
+                                                newvalue)
+        else:
+            raise NotImplementedError
+
     def op_setfield_gc(self, fielddescr, struct, newvalue):
         if fielddescr.typeinfo == REF:
             do_setfield_gc_ptr(struct, fielddescr.ofs, newvalue)
@@ -1368,6 +1399,22 @@
 def do_getfield_gc_ptr(struct, fieldnum):
     return cast_to_ptr(_getfield_gc(struct, fieldnum))
 
+def _getinteriorfield_gc(struct, fieldnum):
+    STRUCT, fieldname = symbolic.TokenToField[fieldnum]
+    return getattr(struct, fieldname)
+
+def do_getinteriorfield_gc_int(array, index, fieldnum):
+    struct = array._obj.container.getitem(index)
+    return cast_to_int(_getinteriorfield_gc(struct, fieldnum))
+
+def do_getinteriorfield_gc_float(array, index, fieldnum):
+    struct = array._obj.container.getitem(index)
+    return cast_to_floatstorage(_getinteriorfield_gc(struct, fieldnum))
+
+def do_getinteriorfield_gc_ptr(array, index, fieldnum):
+    struct = array._obj.container.getitem(index)
+    return cast_to_ptr(_getinteriorfield_gc(struct, fieldnum))
+
 def _getfield_raw(struct, fieldnum):
     STRUCT, fieldname = symbolic.TokenToField[fieldnum]
     ptr = cast_from_int(lltype.Ptr(STRUCT), struct)
@@ -1423,26 +1470,28 @@
     newvalue = cast_from_ptr(ITEMTYPE, newvalue)
     array.setitem(index, newvalue)
 
-def do_setfield_gc_int(struct, fieldnum, newvalue):
-    STRUCT, fieldname = symbolic.TokenToField[fieldnum]
-    ptr = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), struct)
-    FIELDTYPE = getattr(STRUCT, fieldname)
-    newvalue = cast_from_int(FIELDTYPE, newvalue)
-    setattr(ptr, fieldname, newvalue)
+def new_setfield_gc(cast_func):
+    def do_setfield_gc(struct, fieldnum, newvalue):
+        STRUCT, fieldname = symbolic.TokenToField[fieldnum]
+        ptr = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), struct)
+        FIELDTYPE = getattr(STRUCT, fieldname)
+        newvalue = cast_func(FIELDTYPE, newvalue)
+        setattr(ptr, fieldname, newvalue)
+    return do_setfield_gc
+do_setfield_gc_int = new_setfield_gc(cast_from_int)
+do_setfield_gc_float = new_setfield_gc(cast_from_floatstorage)
+do_setfield_gc_ptr = new_setfield_gc(cast_from_ptr)
 
-def do_setfield_gc_float(struct, fieldnum, newvalue):
-    STRUCT, fieldname = symbolic.TokenToField[fieldnum]
-    ptr = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), struct)
-    FIELDTYPE = getattr(STRUCT, fieldname)
-    newvalue = cast_from_floatstorage(FIELDTYPE, newvalue)
-    setattr(ptr, fieldname, newvalue)
-
-def do_setfield_gc_ptr(struct, fieldnum, newvalue):
-    STRUCT, fieldname = symbolic.TokenToField[fieldnum]
-    ptr = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), struct)
-    FIELDTYPE = getattr(STRUCT, fieldname)
-    newvalue = cast_from_ptr(FIELDTYPE, newvalue)
-    setattr(ptr, fieldname, newvalue)
+def new_setinteriorfield_gc(cast_func):
+    def do_setinteriorfield_gc(array, index, fieldnum, newvalue):
+        STRUCT, fieldname = symbolic.TokenToField[fieldnum]
+        struct = array._obj.container.getitem(index)
+        FIELDTYPE = getattr(STRUCT, fieldname)
+        setattr(struct, fieldname, cast_func(FIELDTYPE, newvalue))
+    return do_setinteriorfield_gc
+do_setinteriorfield_gc_int = new_setinteriorfield_gc(cast_from_int)
+do_setinteriorfield_gc_float = new_setinteriorfield_gc(cast_from_floatstorage)
+do_setinteriorfield_gc_ptr = new_setinteriorfield_gc(cast_from_ptr)        
 
 def do_setfield_raw_int(struct, fieldnum, newvalue):
     STRUCT, fieldname = symbolic.TokenToField[fieldnum]
@@ -1708,6 +1757,7 @@
 setannotation(compile_start_float_var, annmodel.SomeInteger())
 setannotation(compile_add, annmodel.s_None)
 setannotation(compile_add_descr, annmodel.s_None)
+setannotation(compile_add_descr_arg, annmodel.s_None)
 setannotation(compile_add_var, annmodel.s_None)
 setannotation(compile_add_int_const, annmodel.s_None)
 setannotation(compile_add_ref_const, annmodel.s_None)
@@ -1755,6 +1805,9 @@
 setannotation(do_getfield_raw_int, annmodel.SomeInteger())
 setannotation(do_getfield_raw_ptr, annmodel.SomePtr(llmemory.GCREF))
 setannotation(do_getfield_raw_float, s_FloatStorage)
+setannotation(do_getinteriorfield_gc_int, annmodel.SomeInteger())
+setannotation(do_getinteriorfield_gc_ptr, annmodel.SomePtr(llmemory.GCREF))
+setannotation(do_getinteriorfield_gc_float, s_FloatStorage)
 setannotation(do_new, annmodel.SomePtr(llmemory.GCREF))
 setannotation(do_new_array, annmodel.SomePtr(llmemory.GCREF))
 setannotation(do_setarrayitem_gc_int, annmodel.s_None)
@@ -1768,6 +1821,9 @@
 setannotation(do_setfield_raw_int, annmodel.s_None)
 setannotation(do_setfield_raw_ptr, annmodel.s_None)
 setannotation(do_setfield_raw_float, annmodel.s_None)
+setannotation(do_setinteriorfield_gc_int, annmodel.s_None)
+setannotation(do_setinteriorfield_gc_ptr, annmodel.s_None)
+setannotation(do_setinteriorfield_gc_float, annmodel.s_None)
 setannotation(do_newstr, annmodel.SomePtr(llmemory.GCREF))
 setannotation(do_strsetitem, annmodel.s_None)
 setannotation(do_newunicode, annmodel.SomePtr(llmemory.GCREF))
diff --git a/pypy/jit/backend/llgraph/runner.py b/pypy/jit/backend/llgraph/runner.py
--- a/pypy/jit/backend/llgraph/runner.py
+++ b/pypy/jit/backend/llgraph/runner.py
@@ -2,21 +2,19 @@
 Minimal-API wrapper around the llinterpreter to run operations.
 """
 
-import sys
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.rpython.lltypesystem import lltype, llmemory, rclass
 from pypy.rpython.ootypesystem import ootype
 from pypy.rpython.llinterp import LLInterpreter
 from pypy.jit.metainterp import history
-from pypy.jit.metainterp.history import REF, INT, FLOAT
+from pypy.jit.metainterp.history import REF, INT, FLOAT, STRUCT
 from pypy.jit.metainterp.warmstate import unwrap
-from pypy.jit.metainterp.resoperation import ResOperation, rop
+from pypy.jit.metainterp.resoperation import rop
 from pypy.jit.backend import model
 from pypy.jit.backend.llgraph import llimpl, symbolic
 from pypy.jit.metainterp.typesystem import llhelper, oohelper
 from pypy.jit.codewriter import heaptracker, longlong
-from pypy.rlib import rgc
 
 class MiniStats:
     pass
@@ -62,6 +60,9 @@
     def is_array_of_floats(self):
         return self.typeinfo == FLOAT
 
+    def is_array_of_structs(self):
+        return self.typeinfo == STRUCT
+
     def as_vtable_size_descr(self):
         return self
 
@@ -177,8 +178,10 @@
             llimpl.compile_add(c, op.getopnum())
             descr = op.getdescr()
             if isinstance(descr, Descr):
-                llimpl.compile_add_descr(c, descr.ofs, descr.typeinfo, descr.arg_types)
-            if isinstance(descr, history.LoopToken) and op.getopnum() != rop.JUMP:
+                llimpl.compile_add_descr(c, descr.ofs, descr.typeinfo,
+                                         descr.arg_types)
+            if (isinstance(descr, history.LoopToken) and
+                op.getopnum() != rop.JUMP):
                 llimpl.compile_add_loop_token(c, descr)
             if self.is_oo and isinstance(descr, (OODescr, MethDescr)):
                 # hack hack, not rpython
@@ -193,6 +196,9 @@
                     llimpl.compile_add_ref_const(c, x.value, self.ts.BASETYPE)
                 elif isinstance(x, history.ConstFloat):
                     llimpl.compile_add_float_const(c, x.value)
+                elif isinstance(x, Descr):
+                    llimpl.compile_add_descr_arg(c, x.ofs, x.typeinfo,
+                                                 x.arg_types)
                 else:
                     raise Exception("'%s' args contain: %r" % (op.getopname(),
                                                                x))
@@ -316,6 +322,13 @@
         token = history.getkind(getattr(S, fieldname))
         return self.getdescr(ofs, token[0], name=fieldname)
 
+    def interiorfielddescrof(self, A, fieldname):
+        S = A.OF
+        ofs2 = symbolic.get_size(A)
+        ofs, size = symbolic.get_field_token(S, fieldname)
+        token = history.getkind(getattr(S, fieldname))
+        return self.getdescr(ofs, token[0], name=fieldname, extrainfo=ofs2)
+
     def calldescrof(self, FUNC, ARGS, RESULT, extrainfo):
         arg_types = []
         for ARG in ARGS:
@@ -353,8 +366,13 @@
     def arraydescrof(self, A):
         assert A.OF != lltype.Void
         size = symbolic.get_size(A)
-        token = history.getkind(A.OF)
-        return self.getdescr(size, token[0])
+        if isinstance(A.OF, lltype.Ptr) or isinstance(A.OF, lltype.Primitive):
+            token = history.getkind(A.OF)[0]
+        elif isinstance(A.OF, lltype.Struct):
+            token = 's'
+        else:
+            token = '?'
+        return self.getdescr(size, token)
 
     # ---------- the backend-dependent operations ----------
 
@@ -406,6 +424,29 @@
         assert isinstance(fielddescr, Descr)
         return llimpl.do_getfield_raw_float(struct, fielddescr.ofs)
 
+    def bh_getinteriorfield_gc_i(self, array, index, descr):
+        assert isinstance(descr, Descr)
+        return llimpl.do_getinteriorfield_gc_int(array, index, descr.ofs)
+    def bh_getinteriorfield_gc_r(self, array, index, descr):
+        assert isinstance(descr, Descr)
+        return llimpl.do_getinteriorfield_gc_ptr(array, index, descr.ofs)
+    def bh_getinteriorfield_gc_f(self, array, index, descr):
+        assert isinstance(descr, Descr)
+        return llimpl.do_getinteriorfield_gc_float(array, index, descr.ofs)
+
+    def bh_setinteriorfield_gc_i(self, array, index, descr, value):
+        assert isinstance(descr, Descr)
+        return llimpl.do_setinteriorfield_gc_int(array, index, descr.ofs,
+                                                 value)
+    def bh_setinteriorfield_gc_r(self, array, index, descr, value):
+        assert isinstance(descr, Descr)
+        return llimpl.do_setinteriorfield_gc_ptr(array, index, descr.ofs,
+                                                 value)
+    def bh_setinteriorfield_gc_f(self, array, index, descr, value):
+        assert isinstance(descr, Descr)
+        return llimpl.do_setinteriorfield_gc_float(array, index, descr.ofs,
+                                                   value)
+
     def bh_new(self, sizedescr):
         assert isinstance(sizedescr, Descr)
         return llimpl.do_new(sizedescr.ofs)
@@ -418,7 +459,6 @@
 
     def bh_classof(self, struct):
         struct = lltype.cast_opaque_ptr(rclass.OBJECTPTR, struct)
-        result = struct.typeptr
         result_adr = llmemory.cast_ptr_to_adr(struct.typeptr)
         return heaptracker.adr2int(result_adr)
 
diff --git a/pypy/jit/backend/llsupport/descr.py b/pypy/jit/backend/llsupport/descr.py
--- a/pypy/jit/backend/llsupport/descr.py
+++ b/pypy/jit/backend/llsupport/descr.py
@@ -1,13 +1,10 @@
 import py
-from pypy.rpython.lltypesystem import lltype, rffi, llmemory, rclass
+from pypy.rpython.lltypesystem import lltype, rffi, llmemory
 from pypy.rpython.lltypesystem.lloperation import llop
 from pypy.jit.backend.llsupport import symbolic, support
-from pypy.jit.metainterp.history import AbstractDescr, getkind, BoxInt, BoxPtr
-from pypy.jit.metainterp.history import BasicFailDescr, LoopToken, BoxFloat
+from pypy.jit.metainterp.history import AbstractDescr, getkind
 from pypy.jit.metainterp import history
-from pypy.jit.metainterp.resoperation import ResOperation, rop
 from pypy.jit.codewriter import heaptracker, longlong
-from pypy.rlib.rarithmetic import r_longlong, r_ulonglong
 
 # The point of the class organization in this file is to make instances
 # as compact as possible.  This is done by not storing the field size or
@@ -23,6 +20,7 @@
         self._cache_field = {}
         self._cache_array = {}
         self._cache_call = {}
+        self._cache_interiorfield = {}
 
     def init_size_descr(self, STRUCT, sizedescr):
         assert isinstance(STRUCT, lltype.GcStruct)
@@ -142,7 +140,6 @@
         cachedict[fieldname] = fielddescr
         return fielddescr
 
-
 # ____________________________________________________________
 # ArrayDescrs
 
@@ -167,6 +164,7 @@
 
     _is_array_of_pointers = False      # unless overridden by GcPtrArrayDescr
     _is_array_of_floats   = False      # unless overridden by FloatArrayDescr
+    _is_array_of_structs  = False      # unless overridden by StructArrayDescr
     _is_item_signed       = False      # unless overridden by XxxArrayDescr
 
     def is_array_of_pointers(self):
@@ -175,6 +173,9 @@
     def is_array_of_floats(self):
         return self._is_array_of_floats
 
+    def is_array_of_structs(self):
+        return self._is_array_of_structs
+
     def is_item_signed(self):
         return self._is_item_signed
 
@@ -199,6 +200,10 @@
     def get_item_size(self, translate_support_code):
         return symbolic.get_size(lltype.Float, translate_support_code)
 
+class StructArrayDescr(BaseArrayDescr):
+    _clsname = 'StructArrayDescr'
+    _is_array_of_structs = True
+
 class BaseArrayNoLengthDescr(BaseArrayDescr):
     def get_base_size(self, translate_support_code):
         return 0
@@ -218,6 +223,13 @@
 def getArrayDescrClass(ARRAY):
     if ARRAY.OF is lltype.Float:
         return FloatArrayDescr
+    elif isinstance(ARRAY.OF, lltype.Struct):
+        class Descr(StructArrayDescr):
+            _clsname = '%sArrayDescr' % ARRAY.OF._name
+            def get_item_size(self, translate_support_code):
+                return symbolic.get_size(ARRAY.OF, translate_support_code)
+        Descr.__name__ = Descr._clsname
+        return Descr
     return getDescrClass(ARRAY.OF, BaseArrayDescr, GcPtrArrayDescr,
                          NonGcPtrArrayDescr, 'Array', 'get_item_size',
                          '_is_array_of_floats', '_is_item_signed')
@@ -252,6 +264,36 @@
         cache[ARRAY] = arraydescr
         return arraydescr
 
+# ____________________________________________________________
+# InteriorFieldDescr
+
+class InteriorFieldDescr(AbstractDescr):
+    arraydescr = BaseArrayDescr()     # workaround for the annotator
+    fielddescr = BaseFieldDescr('', 0)
+
+    def __init__(self, arraydescr, fielddescr):
+        self.arraydescr = arraydescr
+        self.fielddescr = fielddescr
+
+    def is_pointer_field(self):
+        return self.fielddescr.is_pointer_field()
+
+    def is_float_field(self):
+        return self.fielddescr.is_float_field()
+
+    def repr_of_descr(self):
+        return '<InteriorFieldDescr %s>' % self.fielddescr.repr_of_descr()
+
+def get_interiorfield_descr(gc_ll_descr, ARRAY, FIELDTP, name):
+    cache = gc_ll_descr._cache_interiorfield
+    try:
+        return cache[(ARRAY, FIELDTP, name)]
+    except KeyError:
+        arraydescr = get_array_descr(gc_ll_descr, ARRAY)
+        fielddescr = get_field_descr(gc_ll_descr, FIELDTP, name)
+        descr = InteriorFieldDescr(arraydescr, fielddescr)
+        cache[(ARRAY, FIELDTP, name)] = descr
+        return descr
 
 # ____________________________________________________________
 # CallDescrs
@@ -525,7 +567,8 @@
         #
         if TYPE is lltype.Float or is_longlong(TYPE):
             setattr(Descr, floatattrname, True)
-        elif TYPE is not lltype.Bool and rffi.cast(TYPE, -1) == -1:
+        elif (TYPE is not lltype.Bool and isinstance(TYPE, lltype.Number) and
+              rffi.cast(TYPE, -1) == -1):
             setattr(Descr, signedattrname, True)
         #
         _cache[nameprefix, TYPE] = Descr
diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py
--- a/pypy/jit/backend/llsupport/gc.py
+++ b/pypy/jit/backend/llsupport/gc.py
@@ -45,6 +45,14 @@
     def freeing_block(self, start, stop):
         pass
 
+    def get_funcptr_for_newarray(self):
+        return llhelper(self.GC_MALLOC_ARRAY, self.malloc_array)
+    def get_funcptr_for_newstr(self):
+        return llhelper(self.GC_MALLOC_STR_UNICODE, self.malloc_str)
+    def get_funcptr_for_newunicode(self):
+        return llhelper(self.GC_MALLOC_STR_UNICODE, self.malloc_unicode)
+
+
     def record_constptrs(self, op, gcrefs_output_list):
         for i in range(op.numargs()):
             v = op.getarg(i)
@@ -96,6 +104,39 @@
         malloc_fn_ptr = self.configure_boehm_once()
         self.funcptr_for_new = malloc_fn_ptr
 
+        def malloc_array(basesize, itemsize, ofs_length, num_elem):
+            try:
+                size = ovfcheck(basesize + ovfcheck(itemsize * num_elem))
+            except OverflowError:
+                return lltype.nullptr(llmemory.GCREF.TO)
+            res = self.funcptr_for_new(size)
+            if not res:
+                return res
+            rffi.cast(rffi.CArrayPtr(lltype.Signed), res)[ofs_length/WORD] = num_elem
+            return res
+        self.malloc_array = malloc_array
+        self.GC_MALLOC_ARRAY = lltype.Ptr(lltype.FuncType(
+            [lltype.Signed] * 4, llmemory.GCREF))
+
+
+        (str_basesize, str_itemsize, str_ofs_length
+         ) = symbolic.get_array_token(rstr.STR, self.translate_support_code)
+        (unicode_basesize, unicode_itemsize, unicode_ofs_length
+         ) = symbolic.get_array_token(rstr.UNICODE, self.translate_support_code)
+        def malloc_str(length):
+            return self.malloc_array(
+                str_basesize, str_itemsize, str_ofs_length, length
+            )
+        def malloc_unicode(length):
+            return self.malloc_array(
+                unicode_basesize, unicode_itemsize, unicode_ofs_length, length
+            )
+        self.malloc_str = malloc_str
+        self.malloc_unicode = malloc_unicode
+        self.GC_MALLOC_STR_UNICODE = lltype.Ptr(lltype.FuncType(
+            [lltype.Signed], llmemory.GCREF))
+
+
         # on some platform GC_init is required before any other
         # GC_* functions, call it here for the benefit of tests
         # XXX move this to tests
@@ -116,39 +157,27 @@
         ofs_length = arraydescr.get_ofs_length(self.translate_support_code)
         basesize = arraydescr.get_base_size(self.translate_support_code)
         itemsize = arraydescr.get_item_size(self.translate_support_code)
-        size = basesize + itemsize * num_elem
-        res = self.funcptr_for_new(size)
-        rffi.cast(rffi.CArrayPtr(lltype.Signed), res)[ofs_length/WORD] = num_elem
-        return res
+        return self.malloc_array(basesize, itemsize, ofs_length, num_elem)
 
     def gc_malloc_str(self, num_elem):
-        basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR,
-                                                   self.translate_support_code)
-        assert itemsize == 1
-        size = basesize + num_elem
-        res = self.funcptr_for_new(size)
-        rffi.cast(rffi.CArrayPtr(lltype.Signed), res)[ofs_length/WORD] = num_elem
-        return res
+        return self.malloc_str(num_elem)
 
     def gc_malloc_unicode(self, num_elem):
-        basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE,
-                                                   self.translate_support_code)
-        size = basesize + num_elem * itemsize
-        res = self.funcptr_for_new(size)
-        rffi.cast(rffi.CArrayPtr(lltype.Signed), res)[ofs_length/WORD] = num_elem
-        return res
+        return self.malloc_unicode(num_elem)
 
     def args_for_new(self, sizedescr):
         assert isinstance(sizedescr, BaseSizeDescr)
         return [sizedescr.size]
 
+    def args_for_new_array(self, arraydescr):
+        ofs_length = arraydescr.get_ofs_length(self.translate_support_code)
+        basesize = arraydescr.get_base_size(self.translate_support_code)
+        itemsize = arraydescr.get_item_size(self.translate_support_code)
+        return [basesize, itemsize, ofs_length]
+
     def get_funcptr_for_new(self):
         return self.funcptr_for_new
 
-    get_funcptr_for_newarray = None
-    get_funcptr_for_newstr = None
-    get_funcptr_for_newunicode = None
-
     def rewrite_assembler(self, cpu, operations, gcrefs_output_list):
         # record all GCREFs too, because Boehm cannot see them and keep them
         # alive if they end up as constants in the assembler
@@ -620,10 +649,13 @@
         def malloc_basic(size, tid):
             type_id = llop.extract_ushort(llgroup.HALFWORD, tid)
             has_finalizer = bool(tid & (1<<llgroup.HALFSHIFT))
+            has_light_finalizer = bool(tid & (1<<(llgroup.HALFSHIFT + 1)))
             check_typeid(type_id)
             res = llop1.do_malloc_fixedsize_clear(llmemory.GCREF,
                                                   type_id, size,
-                                                  has_finalizer, False)
+                                                  has_finalizer,
+                                                  has_light_finalizer,
+                                                  False)
             # In case the operation above failed, we are returning NULL
             # from this function to assembler.  There is also an RPython
             # exception set, typically MemoryError; but it's easier and
@@ -694,7 +726,7 @@
             # also use it to allocate varsized objects.  The tid
             # and possibly the length are both set afterward.
             gcref = llop1.do_malloc_fixedsize_clear(llmemory.GCREF,
-                                        0, size, False, False)
+                                        0, size, False, False, False)
             return rffi.cast(lltype.Signed, gcref)
         self.malloc_slowpath = malloc_slowpath
         self.MALLOC_SLOWPATH = lltype.FuncType([lltype.Signed], lltype.Signed)
@@ -718,7 +750,9 @@
         type_id = self.layoutbuilder.get_type_id(S)
         assert not self.layoutbuilder.is_weakref_type(S)
         has_finalizer = bool(self.layoutbuilder.has_finalizer(S))
-        flags = int(has_finalizer) << llgroup.HALFSHIFT
+        has_light_finalizer = bool(self.layoutbuilder.has_light_finalizer(S))
+        flags = (int(has_finalizer) << llgroup.HALFSHIFT |
+                 int(has_light_finalizer) << (llgroup.HALFSHIFT + 1))
         descr.tid = llop.combine_ushort(lltype.Signed, type_id, flags)
 
     def init_array_descr(self, A, descr):
@@ -752,15 +786,6 @@
     def get_funcptr_for_new(self):
         return llhelper(self.GC_MALLOC_BASIC, self.malloc_basic)
 
-    def get_funcptr_for_newarray(self):
-        return llhelper(self.GC_MALLOC_ARRAY, self.malloc_array)
-
-    def get_funcptr_for_newstr(self):
-        return llhelper(self.GC_MALLOC_STR_UNICODE, self.malloc_str)
-
-    def get_funcptr_for_newunicode(self):
-        return llhelper(self.GC_MALLOC_STR_UNICODE, self.malloc_unicode)
-
     def do_write_barrier(self, gcref_struct, gcref_newptr):
         hdr_addr = llmemory.cast_ptr_to_adr(gcref_struct)
         hdr_addr -= self.gcheaderbuilder.size_gc_header
diff --git a/pypy/jit/backend/llsupport/llmodel.py b/pypy/jit/backend/llsupport/llmodel.py
--- a/pypy/jit/backend/llsupport/llmodel.py
+++ b/pypy/jit/backend/llsupport/llmodel.py
@@ -1,24 +1,18 @@
-import sys
 from pypy.rpython.lltypesystem import lltype, llmemory, rffi, rclass, rstr
 from pypy.rpython.lltypesystem.lloperation import llop
-from pypy.rpython.llinterp import LLInterpreter, LLException
+from pypy.rpython.llinterp import LLInterpreter
 from pypy.rpython.annlowlevel import llhelper
 from pypy.rlib.objectmodel import we_are_translated, specialize
-from pypy.jit.metainterp.history import BoxInt, BoxPtr, set_future_values,\
-     BoxFloat
 from pypy.jit.metainterp import history
 from pypy.jit.codewriter import heaptracker, longlong
 from pypy.jit.backend.model import AbstractCPU
 from pypy.jit.backend.llsupport import symbolic
 from pypy.jit.backend.llsupport.symbolic import WORD, unroll_basic_sizes
-from pypy.jit.backend.llsupport.descr import get_size_descr,  BaseSizeDescr
-from pypy.jit.backend.llsupport.descr import get_field_descr, BaseFieldDescr
-from pypy.jit.backend.llsupport.descr import get_array_descr, BaseArrayDescr
-from pypy.jit.backend.llsupport.descr import get_call_descr
-from pypy.jit.backend.llsupport.descr import BaseIntCallDescr, GcPtrCallDescr
-from pypy.jit.backend.llsupport.descr import FloatCallDescr, VoidCallDescr
+from pypy.jit.backend.llsupport.descr import (get_size_descr,
+     get_field_descr, BaseFieldDescr, get_array_descr, BaseArrayDescr,
+     get_call_descr, BaseIntCallDescr, GcPtrCallDescr, FloatCallDescr,
+     VoidCallDescr, InteriorFieldDescr, get_interiorfield_descr)
 from pypy.jit.backend.llsupport.asmmemmgr import AsmMemoryManager
-from pypy.rpython.annlowlevel import cast_instance_to_base_ptr
 
 
 class AbstractLLCPU(AbstractCPU):
@@ -241,6 +235,9 @@
     def arraydescrof(self, A):
         return get_array_descr(self.gc_ll_descr, A)
 
+    def interiorfielddescrof(self, A, fieldname):
+        return get_interiorfield_descr(self.gc_ll_descr, A, A.OF, fieldname)
+
     def unpack_arraydescr(self, arraydescr):
         assert isinstance(arraydescr, BaseArrayDescr)
         return arraydescr.get_base_size(self.translate_support_code)
@@ -358,6 +355,100 @@
     bh_getarrayitem_raw_i = bh_getarrayitem_gc_i
     bh_getarrayitem_raw_f = bh_getarrayitem_gc_f
 
+    def bh_getinteriorfield_gc_i(self, gcref, itemindex, descr):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        fieldsize = descr.fielddescr.get_field_size(self.translate_support_code)
+        sign = descr.fielddescr.is_field_signed()
+        fullofs = itemindex * size + ofs
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), fullofs)
+        for STYPE, UTYPE, itemsize in unroll_basic_sizes:
+            if fieldsize == itemsize:
+                if sign:
+                    item = rffi.cast(rffi.CArrayPtr(STYPE), items)
+                    val = item[0]
+                    val = rffi.cast(lltype.Signed, val)
+                else:
+                    item = rffi.cast(rffi.CArrayPtr(UTYPE), items)
+                    val = item[0]
+                    val = rffi.cast(lltype.Signed, val)
+                # --- end of GC unsafe code ---
+                return val
+        else:
+            raise NotImplementedError("size = %d" % fieldsize)
+
+    def bh_getinteriorfield_gc_r(self, gcref, itemindex, descr):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs +
+                            size * itemindex)
+        items = rffi.cast(rffi.CArrayPtr(lltype.Signed), items)
+        pval = self._cast_int_to_gcref(items[0])
+        # --- end of GC unsafe code ---
+        return pval
+
+    def bh_getinteriorfield_gc_f(self, gcref, itemindex, descr):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs +
+                            size * itemindex)
+        items = rffi.cast(rffi.CArrayPtr(longlong.FLOATSTORAGE), items)
+        fval = items[0]
+        # --- end of GC unsafe code ---
+        return fval
+
+    def bh_setinteriorfield_gc_i(self, gcref, itemindex, descr, value):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        fieldsize = descr.fielddescr.get_field_size(self.translate_support_code)
+        ofs = itemindex * size + ofs
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs)
+        for TYPE, _, itemsize in unroll_basic_sizes:
+            if fieldsize == itemsize:
+                items = rffi.cast(rffi.CArrayPtr(TYPE), items)
+                items[0] = rffi.cast(TYPE, value)
+                # --- end of GC unsafe code ---
+                return
+        else:
+            raise NotImplementedError("size = %d" % fieldsize)
+
+    def bh_setinteriorfield_gc_r(self, gcref, itemindex, descr, newvalue):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        self.gc_ll_descr.do_write_barrier(gcref, newvalue)
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref),
+                            ofs + size * itemindex)
+        items = rffi.cast(rffi.CArrayPtr(lltype.Signed), items)
+        items[0] = self.cast_gcref_to_int(newvalue)
+        # --- end of GC unsafe code ---
+
+    def bh_setinteriorfield_gc_f(self, gcref, itemindex, descr, newvalue):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref),
+                            ofs + size * itemindex)
+        items = rffi.cast(rffi.CArrayPtr(longlong.FLOATSTORAGE), items)
+        items[0] = newvalue
+        # --- end of GC unsafe code ---
+
     def bh_strlen(self, string):
         s = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), string)
         return len(s.chars)
@@ -475,7 +566,6 @@
 
     def bh_classof(self, struct):
         struct = lltype.cast_opaque_ptr(rclass.OBJECTPTR, struct)
-        result = struct.typeptr
         result_adr = llmemory.cast_ptr_to_adr(struct.typeptr)
         return heaptracker.adr2int(result_adr)
 
diff --git a/pypy/jit/backend/llsupport/regalloc.py b/pypy/jit/backend/llsupport/regalloc.py
--- a/pypy/jit/backend/llsupport/regalloc.py
+++ b/pypy/jit/backend/llsupport/regalloc.py
@@ -287,7 +287,6 @@
         self.reg_bindings[to_v] = reg
 
     def _move_variable_away(self, v, prev_loc):
-        reg = None
         if self.free_regs:
             loc = self.free_regs.pop()
             self.reg_bindings[v] = loc
diff --git a/pypy/jit/backend/llsupport/test/test_asmmemmgr.py b/pypy/jit/backend/llsupport/test/test_asmmemmgr.py
--- a/pypy/jit/backend/llsupport/test/test_asmmemmgr.py
+++ b/pypy/jit/backend/llsupport/test/test_asmmemmgr.py
@@ -211,14 +211,14 @@
     debug._log = debug.DebugLog()
     try:
         mc._dump(addr, 'test-logname-section')
-        log = list(debug._log)
+        log = list(debug._log) 
     finally:
         debug._log = None
     encoded = ''.join(writtencode).encode('hex').upper()
     ataddr = '@%x' % addr
     assert log == [('test-logname-section',
                     [('debug_print', 'CODE_DUMP', ataddr, '+0 ', encoded)])]
-    #
+    
     lltype.free(p, flavor='raw')
 
 def test_blockbuildermixin2():
diff --git a/pypy/jit/backend/llsupport/test/test_descr.py b/pypy/jit/backend/llsupport/test/test_descr.py
--- a/pypy/jit/backend/llsupport/test/test_descr.py
+++ b/pypy/jit/backend/llsupport/test/test_descr.py
@@ -3,7 +3,6 @@
 from pypy.jit.backend.llsupport import symbolic
 from pypy.rlib.objectmodel import Symbolic
 from pypy.rpython.annlowlevel import llhelper
-from pypy.jit.metainterp.history import BoxInt, BoxFloat, BoxPtr
 from pypy.jit.metainterp import history
 from pypy.jit.codewriter import longlong
 import sys, struct, py
@@ -149,7 +148,9 @@
     A2 = lltype.GcArray(lltype.Ptr(T))
     A3 = lltype.GcArray(lltype.Ptr(U))
     A4 = lltype.GcArray(lltype.Float)
-    A5 = lltype.GcArray(lltype.SingleFloat)
+    A5 = lltype.GcArray(lltype.Struct('x', ('v', lltype.Signed),
+                                      ('k', lltype.Signed)))
+    A6 = lltype.GcArray(lltype.SingleFloat)
     assert getArrayDescrClass(A2) is GcPtrArrayDescr
     assert getArrayDescrClass(A3) is NonGcPtrArrayDescr
     cls = getArrayDescrClass(A1)
@@ -158,7 +159,7 @@
     clsf = getArrayDescrClass(A4)
     assert clsf != cls
     assert clsf == getArrayDescrClass(lltype.GcArray(lltype.Float))
-    clss = getArrayDescrClass(A5)
+    clss = getArrayDescrClass(A6)
     assert clss not in (clsf, cls)
     assert clss == getArrayDescrClass(lltype.GcArray(rffi.UINT))
     #
@@ -168,11 +169,12 @@
     descr3 = get_array_descr(c0, A3)
     descr4 = get_array_descr(c0, A4)
     descr5 = get_array_descr(c0, A5)
+    descr6 = get_array_descr(c0, A6)
     assert descr1.__class__ is cls
     assert descr2.__class__ is GcPtrArrayDescr
     assert descr3.__class__ is NonGcPtrArrayDescr
     assert descr4.__class__ is clsf
-    assert descr5.__class__ is clss
+    assert descr6.__class__ is clss
     assert descr1 == get_array_descr(c0, lltype.GcArray(lltype.Char))
     assert not descr1.is_array_of_pointers()
     assert     descr2.is_array_of_pointers()
@@ -202,7 +204,8 @@
     assert descr2.get_item_size(False) == rffi.sizeof(lltype.Ptr(T))
     assert descr3.get_item_size(False) == rffi.sizeof(lltype.Ptr(U))
     assert descr4.get_item_size(False) == rffi.sizeof(lltype.Float)
-    assert descr5.get_item_size(False) == rffi.sizeof(lltype.SingleFloat)
+    assert descr5.get_item_size(False) == rffi.sizeof(lltype.Signed) * 2
+    assert descr6.get_item_size(False) == rffi.sizeof(lltype.SingleFloat)
     #
     assert isinstance(descr1.get_base_size(True), Symbolic)
     assert isinstance(descr2.get_base_size(True), Symbolic)
@@ -348,7 +351,6 @@
                             (rffi.SHORT,      True), (rffi.USHORT, False),
                             (rffi.INT,        True), (rffi.UINT,   False),
                             (rffi.LONG,       True), (rffi.ULONG,  False)]:
-        A = lltype.GcArray(RESTYPE)
         for tsc in [False, True]:
             c2 = GcCache(tsc)
             descr1 = get_call_descr(c2, [], RESTYPE)
@@ -379,7 +381,6 @@
     descr3i = get_array_descr(c0, lltype.GcArray(lltype.Char))
     assert descr3i.repr_of_descr() == '<CharArrayDescr>'
     #
-    cache = {}
     descr4 = get_call_descr(c0, [lltype.Char, lltype.Ptr(S)], lltype.Ptr(S))
     assert 'GcPtrCallDescr' in descr4.repr_of_descr()
     #
@@ -412,10 +413,10 @@
     ARGS = [lltype.Float, lltype.Ptr(ARRAY)]
     RES = lltype.Float
 
-    def f(a, b):
+    def f2(a, b):
         return float(b[0]) + a
 
-    fnptr = llhelper(lltype.Ptr(lltype.FuncType(ARGS, RES)), f)
+    fnptr = llhelper(lltype.Ptr(lltype.FuncType(ARGS, RES)), f2)
     descr2 = get_call_descr(c0, ARGS, RES)
     a = lltype.malloc(ARRAY, 3)
     opaquea = lltype.cast_opaque_ptr(llmemory.GCREF, a)
diff --git a/pypy/jit/backend/llsupport/test/test_gc.py b/pypy/jit/backend/llsupport/test/test_gc.py
--- a/pypy/jit/backend/llsupport/test/test_gc.py
+++ b/pypy/jit/backend/llsupport/test/test_gc.py
@@ -247,12 +247,14 @@
         self.record = []
 
     def do_malloc_fixedsize_clear(self, RESTYPE, type_id, size,
-                                  has_finalizer, contains_weakptr):
+                                  has_finalizer, has_light_finalizer,
+                                  contains_weakptr):
         assert not contains_weakptr
+        assert not has_finalizer           # in these tests
+        assert not has_light_finalizer     # in these tests
         p = llmemory.raw_malloc(size)
         p = llmemory.cast_adr_to_ptr(p, RESTYPE)
-        flags = int(has_finalizer) << 16
-        tid = llop.combine_ushort(lltype.Signed, type_id, flags)
+        tid = llop.combine_ushort(lltype.Signed, type_id, 0)
         self.record.append(("fixedsize", repr(size), tid, p))
         return p
 
diff --git a/pypy/jit/backend/model.py b/pypy/jit/backend/model.py
--- a/pypy/jit/backend/model.py
+++ b/pypy/jit/backend/model.py
@@ -1,5 +1,5 @@
 from pypy.rlib.debug import debug_start, debug_print, debug_stop
-from pypy.jit.metainterp import history, compile
+from pypy.jit.metainterp import history
 
 
 class AbstractCPU(object):
@@ -213,6 +213,10 @@
     def typedescrof(TYPE):
         raise NotImplementedError
 
+    @staticmethod
+    def interiorfielddescrof(A, fieldname):
+        raise NotImplementedError
+
     # ---------- the backend-dependent operations ----------
 
     # lltype specific operations
diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -5,7 +5,7 @@
                                          BoxInt, Box, BoxPtr,
                                          LoopToken,
                                          ConstInt, ConstPtr,
-                                         BoxObj, Const,
+                                         BoxObj,
                                          ConstObj, BoxFloat, ConstFloat)
 from pypy.jit.metainterp.resoperation import ResOperation, rop
 from pypy.jit.metainterp.typesystem import deref
@@ -111,7 +111,7 @@
         self.cpu.set_future_value_int(0, 2)
         fail = self.cpu.execute_token(looptoken)
         res = self.cpu.get_latest_value_int(0)
-        assert res == 3        
+        assert res == 3
         assert fail.identifier == 1
 
     def test_compile_loop(self):
@@ -127,7 +127,7 @@
             ]
         inputargs = [i0]
         operations[2].setfailargs([i1])
-        
+
         self.cpu.compile_loop(inputargs, operations, looptoken)
         self.cpu.set_future_value_int(0, 2)
         fail = self.cpu.execute_token(looptoken)
@@ -148,7 +148,7 @@
             ]
         inputargs = [i0]
         operations[2].setfailargs([None, None, i1, None])
-        
+
         self.cpu.compile_loop(inputargs, operations, looptoken)
         self.cpu.set_future_value_int(0, 2)
         fail = self.cpu.execute_token(looptoken)
@@ -372,7 +372,7 @@
         for opnum, boxargs, retvalue in get_int_tests():
             res = self.execute_operation(opnum, boxargs, 'int')
             assert res.value == retvalue
-        
+
     def test_float_operations(self):
         from pypy.jit.metainterp.test.test_executor import get_float_tests
         for opnum, boxargs, rettype, retvalue in get_float_tests(self.cpu):
@@ -438,7 +438,7 @@
 
     def test_ovf_operations_reversed(self):
         self.test_ovf_operations(reversed=True)
-        
+
     def test_bh_call(self):
         cpu = self.cpu
         #
@@ -503,7 +503,7 @@
                                          [funcbox, BoxInt(num), BoxInt(num)],
                                          'int', descr=dyn_calldescr)
             assert res.value == 2 * num
-            
+
 
         if cpu.supports_floats:
             def func(f0, f1, f2, f3, f4, f5, f6, i0, i1, f7, f8, f9):
@@ -543,7 +543,7 @@
         funcbox = self.get_funcbox(self.cpu, func_ptr)
         res = self.execute_operation(rop.CALL, [funcbox] + map(BoxInt, args), 'int', descr=calldescr)
         assert res.value == func(*args)
-        
+
     def test_call_stack_alignment(self):
         # test stack alignment issues, notably for Mac OS/X.
         # also test the ordering of the arguments.
@@ -615,7 +615,7 @@
         res = self.execute_operation(rop.GETFIELD_GC, [t_box],
                                      'int', descr=shortdescr)
         assert res.value == 1331
-        
+
         #
         u_box, U_box = self.alloc_instance(self.U)
         fielddescr2 = self.cpu.fielddescrof(self.S, 'next')
@@ -695,7 +695,7 @@
 
     def test_failing_guard_class(self):
         t_box, T_box = self.alloc_instance(self.T)
-        u_box, U_box = self.alloc_instance(self.U)        
+        u_box, U_box = self.alloc_instance(self.U)
         null_box = self.null_instance()
         for opname, args in [(rop.GUARD_CLASS, [t_box, U_box]),
                              (rop.GUARD_CLASS, [u_box, T_box]),
@@ -787,7 +787,7 @@
         r = self.execute_operation(rop.GETARRAYITEM_GC, [a_box, BoxInt(3)],
                                    'int', descr=arraydescr)
         assert r.value == 160
-        
+
         #
         if isinstance(A, lltype.GcArray):
             A = lltype.Ptr(A)
@@ -880,6 +880,73 @@
                                    'int', descr=arraydescr)
         assert r.value == 7441
 
+    def test_array_of_structs(self):
+        TP = lltype.GcStruct('x')
+        ITEM = lltype.Struct('x',
+                             ('vs', lltype.Signed),
+                             ('vu', lltype.Unsigned),
+                             ('vsc', rffi.SIGNEDCHAR),
+                             ('vuc', rffi.UCHAR),
+                             ('vss', rffi.SHORT),
+                             ('vus', rffi.USHORT),
+                             ('vsi', rffi.INT),
+                             ('vui', rffi.UINT),
+                             ('k', lltype.Float),
+                             ('p', lltype.Ptr(TP)))
+        a_box, A = self.alloc_array_of(ITEM, 15)
+        s_box, S = self.alloc_instance(TP)
+        kdescr = self.cpu.interiorfielddescrof(A, 'k')
+        pdescr = self.cpu.interiorfielddescrof(A, 'p')
+        self.execute_operation(rop.SETINTERIORFIELD_GC, [a_box, BoxInt(3),
+                                                         boxfloat(1.5)],
+                               'void', descr=kdescr)
+        f = self.cpu.bh_getinteriorfield_gc_f(a_box.getref_base(), 3, kdescr)
+        assert longlong.getrealfloat(f) == 1.5
+        self.cpu.bh_setinteriorfield_gc_f(a_box.getref_base(), 3, kdescr, longlong.getfloatstorage(2.5))
+        r = self.execute_operation(rop.GETINTERIORFIELD_GC, [a_box, BoxInt(3)],
+                                   'float', descr=kdescr)
+        assert r.getfloat() == 2.5
+        #
+        NUMBER_FIELDS = [('vs', lltype.Signed),
+                         ('vu', lltype.Unsigned),
+                         ('vsc', rffi.SIGNEDCHAR),
+                         ('vuc', rffi.UCHAR),
+                         ('vss', rffi.SHORT),
+                         ('vus', rffi.USHORT),
+                         ('vsi', rffi.INT),
+                         ('vui', rffi.UINT)]
+        for name, TYPE in NUMBER_FIELDS[::-1]:
+            vdescr = self.cpu.interiorfielddescrof(A, name)
+            self.execute_operation(rop.SETINTERIORFIELD_GC, [a_box, BoxInt(3),
+                                                             BoxInt(-15)],
+                                   'void', descr=vdescr)
+        for name, TYPE in NUMBER_FIELDS:
+            vdescr = self.cpu.interiorfielddescrof(A, name)
+            i = self.cpu.bh_getinteriorfield_gc_i(a_box.getref_base(), 3,
+                                                  vdescr)
+            assert i == rffi.cast(lltype.Signed, rffi.cast(TYPE, -15))
+        for name, TYPE in NUMBER_FIELDS[::-1]:
+            vdescr = self.cpu.interiorfielddescrof(A, name)
+            self.cpu.bh_setinteriorfield_gc_i(a_box.getref_base(), 3,
+                                              vdescr, -25)
+        for name, TYPE in NUMBER_FIELDS:
+            vdescr = self.cpu.interiorfielddescrof(A, name)
+            r = self.execute_operation(rop.GETINTERIORFIELD_GC,
+                                       [a_box, BoxInt(3)],
+                                       'int', descr=vdescr)
+            assert r.getint() == rffi.cast(lltype.Signed, rffi.cast(TYPE, -25))
+        #
+        self.execute_operation(rop.SETINTERIORFIELD_GC, [a_box, BoxInt(4),
+                                                         s_box],
+                               'void', descr=pdescr)
+        r = self.cpu.bh_getinteriorfield_gc_r(a_box.getref_base(), 4, pdescr)
+        assert r == s_box.getref_base()
+        self.cpu.bh_setinteriorfield_gc_r(a_box.getref_base(), 3, pdescr,
+                                          s_box.getref_base())
+        r = self.execute_operation(rop.GETINTERIORFIELD_GC, [a_box, BoxInt(3)],
+                                   'ref', descr=pdescr)
+        assert r.getref_base() == s_box.getref_base()
+
     def test_string_basic(self):
         s_box = self.alloc_string("hello\xfe")
         r = self.execute_operation(rop.STRLEN, [s_box], 'int')
@@ -1402,7 +1469,7 @@
         addr = llmemory.cast_ptr_to_adr(func_ptr)
         return ConstInt(heaptracker.adr2int(addr))
 
-    
+
     MY_VTABLE = rclass.OBJECT_VTABLE    # for tests only
 
     S = lltype.GcForwardReference()
@@ -1439,7 +1506,6 @@
         return BoxPtr(lltype.nullptr(llmemory.GCREF.TO))
 
     def alloc_array_of(self, ITEM, length):
-        cpu = self.cpu
         A = lltype.GcArray(ITEM)
         a = lltype.malloc(A, length)
         a_box = BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, a))
@@ -2318,7 +2384,7 @@
         for opname, arg, res in ops:
             self.execute_operation(opname, [arg], 'void')
             assert self.guard_failed == res
-        
+
         lltype.free(x, flavor='raw')
 
     def test_assembler_call(self):
@@ -2398,7 +2464,7 @@
         FakeJitDriverSD.portal_calldescr = self.cpu.calldescrof(
             lltype.Ptr(lltype.FuncType(ARGS, RES)), ARGS, RES,
             EffectInfo.MOST_GENERAL)
-        
+
         ops = '''
         [f0, f1]
         f2 = float_add(f0, f1)
@@ -2489,7 +2555,7 @@
         FakeJitDriverSD.portal_calldescr = self.cpu.calldescrof(
             lltype.Ptr(lltype.FuncType(ARGS, RES)), ARGS, RES,
             EffectInfo.MOST_GENERAL)
-        
+
         ops = '''
         [f0, f1]
         f2 = float_add(f0, f1)
@@ -2940,4 +3006,4 @@
 
     def alloc_unicode(self, unicode):
         py.test.skip("implement me")
-    
+
diff --git a/pypy/jit/backend/test/test_ll_random.py b/pypy/jit/backend/test/test_ll_random.py
--- a/pypy/jit/backend/test/test_ll_random.py
+++ b/pypy/jit/backend/test/test_ll_random.py
@@ -28,16 +28,27 @@
         fork.structure_types_and_vtables = self.structure_types_and_vtables
         return fork
 
-    def get_structptr_var(self, r, must_have_vtable=False, type=lltype.Struct):
+    def _choose_ptr_vars(self, from_, type, array_of_structs):
+        ptrvars = []
+        for i in range(len(from_)):
+            v, S = from_[i][:2]
+            if not isinstance(S, type):
+                continue
+            if ((isinstance(S, lltype.Array) and
+                 isinstance(S.OF, lltype.Struct)) == array_of_structs):
+                ptrvars.append((v, S))
+        return ptrvars
+
+    def get_structptr_var(self, r, must_have_vtable=False, type=lltype.Struct,
+                          array_of_structs=False):
         while True:
-            ptrvars = [(v, S) for (v, S) in self.ptrvars
-                              if isinstance(S, type)]
+            ptrvars = self._choose_ptr_vars(self.ptrvars, type,
+                                            array_of_structs)
             if ptrvars and r.random() < 0.8:
                 v, S = r.choice(ptrvars)
             else:
-                prebuilt_ptr_consts = [(v, S)
-                                 for (v, S, _) in self.prebuilt_ptr_consts
-                                 if isinstance(S, type)]
+                prebuilt_ptr_consts = self._choose_ptr_vars(
+                    self.prebuilt_ptr_consts, type, array_of_structs)
                 if prebuilt_ptr_consts and r.random() < 0.7:
                     v, S = r.choice(prebuilt_ptr_consts)
                 else:
@@ -48,7 +59,8 @@
                                                 has_vtable=must_have_vtable)
                     else:
                         # create a new constant array
-                        p = self.get_random_array(r)
+                        p = self.get_random_array(r,
+                                    must_be_array_of_structs=array_of_structs)
                     S = lltype.typeOf(p).TO
                     v = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, p))
                     self.prebuilt_ptr_consts.append((v, S,
@@ -74,7 +86,8 @@
                 TYPE = lltype.Signed
         return TYPE
 
-    def get_random_structure_type(self, r, with_vtable=None, cache=True):
+    def get_random_structure_type(self, r, with_vtable=None, cache=True,
+                                  type=lltype.GcStruct):
         if cache and self.structure_types and r.random() < 0.5:
             return r.choice(self.structure_types)
         fields = []
@@ -85,7 +98,7 @@
         for i in range(r.randrange(1, 5)):
             TYPE = self.get_random_primitive_type(r)
             fields.append(('f%d' % i, TYPE))
-        S = lltype.GcStruct('S%d' % self.counter, *fields, **kwds)
+        S = type('S%d' % self.counter, *fields, **kwds)
         self.counter += 1
         if cache:
             self.structure_types.append(S)
@@ -125,17 +138,29 @@
                 setattr(p, fieldname, rffi.cast(TYPE, r.random_integer()))
         return p
 
-    def get_random_array_type(self, r):
-        TYPE = self.get_random_primitive_type(r)
+    def get_random_array_type(self, r, can_be_array_of_struct=False,
+                              must_be_array_of_structs=False):
+        if ((can_be_array_of_struct and r.random() < 0.1) or
+            must_be_array_of_structs):
+            TYPE = self.get_random_structure_type(r, cache=False,
+                                                  type=lltype.Struct)
+        else:
+            TYPE = self.get_random_primitive_type(r)
         return lltype.GcArray(TYPE)
 
-    def get_random_array(self, r):
-        A = self.get_random_array_type(r)
+    def get_random_array(self, r, must_be_array_of_structs=False):
+        A = self.get_random_array_type(r,
+                           must_be_array_of_structs=must_be_array_of_structs)
         length = (r.random_integer() // 15) % 300  # length: between 0 and 299
                                                    # likely to be small
         p = lltype.malloc(A, length)
-        for i in range(length):
-            p[i] = rffi.cast(A.OF, r.random_integer())
+        if isinstance(A.OF, lltype.Primitive):
+            for i in range(length):
+                p[i] = rffi.cast(A.OF, r.random_integer())
+        else:
+            for i in range(length):
+                for fname, TP in A.OF._flds.iteritems():
+                    setattr(p[i], fname, rffi.cast(TP, r.random_integer()))
         return p
 
     def get_index(self, length, r):
@@ -155,8 +180,16 @@
                     dic[fieldname] = getattr(p, fieldname)
         else:
             assert isinstance(S, lltype.Array)
-            for i in range(len(p)):
-                dic[i] = p[i]
+            if isinstance(S.OF, lltype.Struct):
+                for i in range(len(p)):
+                    item = p[i]
+                    s1 = {}
+                    for fieldname in S.OF._names:
+                        s1[fieldname] = getattr(item, fieldname)
+                    dic[i] = s1
+            else:
+                for i in range(len(p)):
+                    dic[i] = p[i]
         return dic
 
     def print_loop_prebuilt(self, names, writevar, s):
@@ -220,7 +253,7 @@
 
 class GetFieldOperation(test_random.AbstractOperation):
     def field_descr(self, builder, r):
-        v, S = builder.get_structptr_var(r)
+        v, S = builder.get_structptr_var(r, )
         names = S._names
         if names[0] == 'parent':
             names = names[1:]
@@ -239,6 +272,28 @@
                 continue
             break
 
+class GetInteriorFieldOperation(test_random.AbstractOperation):
+    def field_descr(self, builder, r):
+        v, A = builder.get_structptr_var(r, type=lltype.Array,
+                                         array_of_structs=True)
+        array = v.getref(lltype.Ptr(A))
+        v_index = builder.get_index(len(array), r)
+        name = r.choice(A.OF._names)
+        descr = builder.cpu.interiorfielddescrof(A, name)
+        descr._random_info = 'cpu.interiorfielddescrof(%s, %r)' % (A.OF._name,
+                                                                   name)
+        TYPE = getattr(A.OF, name)
+        return v, v_index, descr, TYPE
+
+    def produce_into(self, builder, r):
+        while True:
+            try:
+                v, v_index, descr, _ = self.field_descr(builder, r)
+                self.put(builder, [v, v_index], descr)
+            except lltype.UninitializedMemoryAccess:
+                continue
+            break
+
 class SetFieldOperation(GetFieldOperation):
     def produce_into(self, builder, r):
         v, descr, TYPE = self.field_descr(builder, r)
@@ -251,6 +306,18 @@
                 break
         builder.do(self.opnum, [v, w], descr)
 
+class SetInteriorFieldOperation(GetInteriorFieldOperation):
+    def produce_into(self, builder, r):
+        v, v_index, descr, TYPE = self.field_descr(builder, r)
+        while True:
+            if r.random() < 0.3:
+                w = ConstInt(r.random_integer())
+            else:
+                w = r.choice(builder.intvars)
+            if rffi.cast(lltype.Signed, rffi.cast(TYPE, w.value)) == w.value:
+                break
+        builder.do(self.opnum, [v, v_index, w], descr)
+
 class NewOperation(test_random.AbstractOperation):
     def size_descr(self, builder, S):
         descr = builder.cpu.sizeof(S)
@@ -306,7 +373,7 @@
 
 class NewArrayOperation(ArrayOperation):
     def produce_into(self, builder, r):
-        A = builder.get_random_array_type(r)
+        A = builder.get_random_array_type(r, can_be_array_of_struct=True)
         v_size = builder.get_index(300, r)
         v_ptr = builder.do(self.opnum, [v_size], self.array_descr(builder, A))
         builder.ptrvars.append((v_ptr, A))
@@ -586,7 +653,9 @@
 for i in range(4):      # make more common
     OPERATIONS.append(GetFieldOperation(rop.GETFIELD_GC))
     OPERATIONS.append(GetFieldOperation(rop.GETFIELD_GC))
+    OPERATIONS.append(GetInteriorFieldOperation(rop.GETINTERIORFIELD_GC))
     OPERATIONS.append(SetFieldOperation(rop.SETFIELD_GC))
+    OPERATIONS.append(SetInteriorFieldOperation(rop.SETINTERIORFIELD_GC))
     OPERATIONS.append(NewOperation(rop.NEW))
     OPERATIONS.append(NewOperation(rop.NEW_WITH_VTABLE))
 
diff --git a/pypy/jit/backend/test/test_random.py b/pypy/jit/backend/test/test_random.py
--- a/pypy/jit/backend/test/test_random.py
+++ b/pypy/jit/backend/test/test_random.py
@@ -601,6 +601,10 @@
             for name, value in fields.items():
                 if isinstance(name, str):
                     setattr(container, name, value)
+                elif isinstance(value, dict):
+                    item = container.getitem(name)
+                    for key1, value1 in value.items():
+                        setattr(item, key1, value1)
                 else:
                     container.setitem(name, value)
 
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1,7 +1,7 @@
 import sys, os
 from pypy.jit.backend.llsupport import symbolic
 from pypy.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper
-from pypy.jit.metainterp.history import Const, Box, BoxInt, BoxPtr, BoxFloat
+from pypy.jit.metainterp.history import Const, Box, BoxInt, ConstInt
 from pypy.jit.metainterp.history import (AbstractFailDescr, INT, REF, FLOAT,
                                          LoopToken)
 from pypy.rpython.lltypesystem import lltype, rffi, rstr, llmemory
@@ -36,7 +36,6 @@
 from pypy.rlib import rgc
 from pypy.rlib.clibffi import FFI_DEFAULT_ABI
 from pypy.jit.backend.x86.jump import remap_frame_layout
-from pypy.jit.metainterp.history import ConstInt, BoxInt
 from pypy.jit.codewriter.effectinfo import EffectInfo
 from pypy.jit.codewriter import longlong
 
@@ -729,8 +728,8 @@
         # Also, make sure this is consistent with FRAME_FIXED_SIZE.
         self.mc.PUSH_r(ebp.value)
         self.mc.MOV_rr(ebp.value, esp.value)
-        for regloc in self.cpu.CALLEE_SAVE_REGISTERS:
-            self.mc.PUSH_r(regloc.value)
+        for loc in self.cpu.CALLEE_SAVE_REGISTERS:
+            self.mc.PUSH_r(loc.value)
 
         gcrootmap = self.cpu.gc_ll_descr.gcrootmap
         if gcrootmap and gcrootmap.is_shadow_stack:
@@ -994,7 +993,7 @@
         effectinfo = op.getdescr().get_extra_info()
         oopspecindex = effectinfo.oopspecindex
         genop_llong_list[oopspecindex](self, op, arglocs, resloc)
-        
+
     def regalloc_perform_math(self, op, arglocs, resloc):
         effectinfo = op.getdescr().get_extra_info()
         oopspecindex = effectinfo.oopspecindex
@@ -1277,8 +1276,8 @@
     genop_int_ne = _cmpop("NE", "NE")
     genop_int_gt = _cmpop("G", "L")
     genop_int_ge = _cmpop("GE", "LE")
-    genop_ptr_eq = genop_int_eq
-    genop_ptr_ne = genop_int_ne
+    genop_ptr_eq = genop_instance_ptr_eq = genop_int_eq
+    genop_ptr_ne = genop_instance_ptr_ne = genop_int_ne
 
     genop_float_lt = _cmpop_float('B', 'A')
     genop_float_le = _cmpop_float('BE', 'AE')
@@ -1298,8 +1297,8 @@
     genop_guard_int_ne = _cmpop_guard("NE", "NE", "E", "E")
     genop_guard_int_gt = _cmpop_guard("G", "L", "LE", "GE")
     genop_guard_int_ge = _cmpop_guard("GE", "LE", "L", "G")
-    genop_guard_ptr_eq = genop_guard_int_eq
-    genop_guard_ptr_ne = genop_guard_int_ne
+    genop_guard_ptr_eq = genop_guard_instance_ptr_eq = genop_guard_int_eq
+    genop_guard_ptr_ne = genop_guard_instance_ptr_ne = genop_guard_int_ne
 
     genop_guard_uint_gt = _cmpop_guard("A", "B", "BE", "AE")
     genop_guard_uint_lt = _cmpop_guard("B", "A", "AE", "BE")
@@ -1609,12 +1608,43 @@
     genop_getarrayitem_gc_pure = genop_getarrayitem_gc
     genop_getarrayitem_raw = genop_getarrayitem_gc
 
+    def _get_interiorfield_addr(self, temp_loc, index_loc, itemsize_loc,
+                                base_loc, ofs_loc):
+        assert isinstance(itemsize_loc, ImmedLoc)
+        if isinstance(index_loc, ImmedLoc):
+            temp_loc = imm(index_loc.value * itemsize_loc.value)
+        else:
+            # XXX should not use IMUL in most cases
+            assert isinstance(temp_loc, RegLoc)
+            assert isinstance(index_loc, RegLoc)
+            self.mc.IMUL_rri(temp_loc.value, index_loc.value,
+                             itemsize_loc.value)
+        assert isinstance(ofs_loc, ImmedLoc)
+        return AddressLoc(base_loc, temp_loc, 0, ofs_loc.value)
+
+    def genop_getinteriorfield_gc(self, op, arglocs, resloc):
+        (base_loc, ofs_loc, itemsize_loc, fieldsize_loc,
+            index_loc, sign_loc) = arglocs
+        src_addr = self._get_interiorfield_addr(resloc, index_loc,
+                                                itemsize_loc, base_loc,
+                                                ofs_loc)
+        self.load_from_mem(resloc, src_addr, fieldsize_loc, sign_loc)
+
+
     def genop_discard_setfield_gc(self, op, arglocs):
         base_loc, ofs_loc, size_loc, value_loc = arglocs
         assert isinstance(size_loc, ImmedLoc)
         dest_addr = AddressLoc(base_loc, ofs_loc)
         self.save_into_mem(dest_addr, value_loc, size_loc)
 
+    def genop_discard_setinteriorfield_gc(self, op, arglocs):
+        (base_loc, ofs_loc, itemsize_loc, fieldsize_loc,
+            index_loc, temp_loc, value_loc) = arglocs
+        dest_addr = self._get_interiorfield_addr(temp_loc, index_loc,
+                                                 itemsize_loc, base_loc,
+                                                 ofs_loc)
+        self.save_into_mem(dest_addr, value_loc, fieldsize_loc)
+
     def genop_discard_setarrayitem_gc(self, op, arglocs):
         base_loc, ofs_loc, value_loc, size_loc, baseofs = arglocs
         assert isinstance(baseofs, ImmedLoc)
diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py
--- a/pypy/jit/backend/x86/regalloc.py
+++ b/pypy/jit/backend/x86/regalloc.py
@@ -7,7 +7,7 @@
                                          ResOperation, BoxPtr, ConstFloat,
                                          BoxFloat, LoopToken, INT, REF, FLOAT)
 from pypy.jit.backend.x86.regloc import *
-from pypy.rpython.lltypesystem import lltype, ll2ctypes, rffi, rstr
+from pypy.rpython.lltypesystem import lltype, rffi, rstr
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.rlib import rgc
 from pypy.jit.backend.llsupport import symbolic
@@ -17,11 +17,12 @@
 from pypy.jit.metainterp.resoperation import rop
 from pypy.jit.backend.llsupport.descr import BaseFieldDescr, BaseArrayDescr
 from pypy.jit.backend.llsupport.descr import BaseCallDescr, BaseSizeDescr
+from pypy.jit.backend.llsupport.descr import InteriorFieldDescr
 from pypy.jit.backend.llsupport.regalloc import FrameManager, RegisterManager,\
      TempBox
 from pypy.jit.backend.x86.arch import WORD, FRAME_FIXED_SIZE
 from pypy.jit.backend.x86.arch import IS_X86_32, IS_X86_64, MY_COPY_OF_REGS
-from pypy.rlib.rarithmetic import r_longlong, r_uint
+from pypy.rlib.rarithmetic import r_longlong
 
 class X86RegisterManager(RegisterManager):
 
@@ -433,7 +434,7 @@
             if self.can_merge_with_next_guard(op, i, operations):
                 oplist_with_guard[op.getopnum()](self, op, operations[i + 1])
                 i += 1
-            elif not we_are_translated() and op.getopnum() == -124: 
+            elif not we_are_translated() and op.getopnum() == -124:
                 self._consider_force_spill(op)
             else:
                 oplist[op.getopnum()](self, op)
@@ -661,8 +662,8 @@
     consider_uint_lt = _consider_compop
     consider_uint_le = _consider_compop
     consider_uint_ge = _consider_compop
-    consider_ptr_eq = _consider_compop
-    consider_ptr_ne = _consider_compop
+    consider_ptr_eq = consider_instance_ptr_eq = _consider_compop
+    consider_ptr_ne = consider_instance_ptr_ne = _consider_compop
 
     def _consider_float_op(self, op):
         loc1 = self.xrm.loc(op.getarg(1))
@@ -826,7 +827,7 @@
         save_all_regs = guard_not_forced_op is not None
         self.xrm.before_call(force_store, save_all_regs=save_all_regs)
         if not save_all_regs:
-            gcrootmap = gc_ll_descr = self.assembler.cpu.gc_ll_descr.gcrootmap
+            gcrootmap = self.assembler.cpu.gc_ll_descr.gcrootmap
             if gcrootmap and gcrootmap.is_shadow_stack:
                 save_all_regs = 2
         self.rm.before_call(force_store, save_all_regs=save_all_regs)
@@ -983,74 +984,27 @@
             return self._call(op, arglocs)
 
     def consider_newstr(self, op):
-        gc_ll_descr = self.assembler.cpu.gc_ll_descr
-        if gc_ll_descr.get_funcptr_for_newstr is not None:
-            # framework GC
-            loc = self.loc(op.getarg(0))
-            return self._call(op, [loc])
-        # boehm GC (XXX kill the following code at some point)
-        ofs_items, itemsize, ofs = symbolic.get_array_token(rstr.STR, self.translate_support_code)
-        assert itemsize == 1
-        return self._malloc_varsize(ofs_items, ofs, 0, op.getarg(0),
-                                    op.result)
+        loc = self.loc(op.getarg(0))
+        return self._call(op, [loc])
 
     def consider_newunicode(self, op):
-        gc_ll_descr = self.assembler.cpu.gc_ll_descr
-        if gc_ll_descr.get_funcptr_for_newunicode is not None:
-            # framework GC
-            loc = self.loc(op.getarg(0))
-            return self._call(op, [loc])
-        # boehm GC (XXX kill the following code at some point)
-        ofs_items, _, ofs = symbolic.get_array_token(rstr.UNICODE,
-                                                   self.translate_support_code)
-        scale = self._get_unicode_item_scale()
-        return self._malloc_varsize(ofs_items, ofs, scale, op.getarg(0),
-                                    op.result)
-
-    def _malloc_varsize(self, ofs_items, ofs_length, scale, v, res_v):
-        # XXX kill this function at some point
-        if isinstance(v, Box):
-            loc = self.rm.make_sure_var_in_reg(v, [v])
-            tempbox = TempBox()
-            other_loc = self.rm.force_allocate_reg(tempbox, [v])
-            self.assembler.load_effective_addr(loc, ofs_items,scale, other_loc)
-        else:
-            tempbox = None
-            other_loc = imm(ofs_items + (v.getint() << scale))
-        self._call(ResOperation(rop.NEW, [], res_v),
-                   [other_loc], [v])
-        loc = self.rm.make_sure_var_in_reg(v, [res_v])
-        assert self.loc(res_v) == eax
-        # now we have to reload length to some reasonable place
-        self.rm.possibly_free_var(v)
-        if tempbox is not None:
-            self.rm.possibly_free_var(tempbox)
-        self.PerformDiscard(ResOperation(rop.SETFIELD_GC, [None, None], None),
-                            [eax, imm(ofs_length), imm(WORD), loc])
+        loc = self.loc(op.getarg(0))
+        return self._call(op, [loc])
 
     def consider_new_array(self, op):
         gc_ll_descr = self.assembler.cpu.gc_ll_descr
-        if gc_ll_descr.get_funcptr_for_newarray is not None:
-            # framework GC
-            box_num_elem = op.getarg(0)
-            if isinstance(box_num_elem, ConstInt):
-                num_elem = box_num_elem.value
-                if gc_ll_descr.can_inline_malloc_varsize(op.getdescr(),
-                                                         num_elem):
-                    self.fastpath_malloc_varsize(op, op.getdescr(), num_elem)
-                    return
-            args = self.assembler.cpu.gc_ll_descr.args_for_new_array(
-                op.getdescr())
-            arglocs = [imm(x) for x in args]
-            arglocs.append(self.loc(box_num_elem))
-            self._call(op, arglocs)
-            return
-        # boehm GC (XXX kill the following code at some point)
-        itemsize, basesize, ofs_length, _, _ = (
-            self._unpack_arraydescr(op.getdescr()))
-        scale_of_field = _get_scale(itemsize)
-        self._malloc_varsize(basesize, ofs_length, scale_of_field,
-                             op.getarg(0), op.result)
+        box_num_elem = op.getarg(0)
+        if isinstance(box_num_elem, ConstInt):
+            num_elem = box_num_elem.value
+            if gc_ll_descr.can_inline_malloc_varsize(op.getdescr(),
+                                                     num_elem):
+                self.fastpath_malloc_varsize(op, op.getdescr(), num_elem)
+                return
+        args = self.assembler.cpu.gc_ll_descr.args_for_new_array(
+            op.getdescr())
+        arglocs = [imm(x) for x in args]
+        arglocs.append(self.loc(box_num_elem))
+        self._call(op, arglocs)
 
     def _unpack_arraydescr(self, arraydescr):
         assert isinstance(arraydescr, BaseArrayDescr)
@@ -1069,6 +1023,16 @@
         sign = fielddescr.is_field_signed()
         return imm(ofs), imm(size), ptr, sign
 
+    def _unpack_interiorfielddescr(self, descr):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs = arraydescr.get_base_size(self.translate_support_code)
+        itemsize = arraydescr.get_item_size(self.translate_support_code)
+        fieldsize = descr.fielddescr.get_field_size(self.translate_support_code)
+        sign = descr.fielddescr.is_field_signed()
+        ofs += descr.fielddescr.offset
+        return imm(ofs), imm(itemsize), imm(fieldsize), sign
+
     def consider_setfield_gc(self, op):
         ofs_loc, size_loc, _, _ = self._unpack_fielddescr(op.getdescr())
         assert isinstance(size_loc, ImmedLoc)
@@ -1085,6 +1049,35 @@
 
     consider_setfield_raw = consider_setfield_gc
 
+    def consider_setinteriorfield_gc(self, op):
+        t = self._unpack_interiorfielddescr(op.getdescr())
+        ofs, itemsize, fieldsize, _ = t
+        args = op.getarglist()
+        if fieldsize.value == 1:
+            need_lower_byte = True
+        else:
+            need_lower_byte = False
+        box_base, box_index, box_value = args
+        base_loc = self.rm.make_sure_var_in_reg(box_base, args)
+        index_loc = self.rm.make_sure_var_in_reg(box_index, args)
+        value_loc = self.make_sure_var_in_reg(box_value, args,
+                                              need_lower_byte=need_lower_byte)
+        # If 'index_loc' is not an immediate, then we need a 'temp_loc' that
+        # is a register whose value will be destroyed.  It's fine to destroy
+        # the same register as 'index_loc', but not the other ones.
+        self.rm.possibly_free_var(box_index)
+        if not isinstance(index_loc, ImmedLoc):
+            tempvar = TempBox()
+            temp_loc = self.rm.force_allocate_reg(tempvar, [box_base,
+                                                            box_value])
+            self.rm.possibly_free_var(tempvar)
+        else:
+            temp_loc = None
+        self.rm.possibly_free_var(box_base)
+        self.possibly_free_var(box_value)
+        self.PerformDiscard(op, [base_loc, ofs, itemsize, fieldsize,
+                                 index_loc, temp_loc, value_loc])
+
     def consider_strsetitem(self, op):
         args = op.getarglist()
         base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
@@ -1146,6 +1139,25 @@
     consider_getarrayitem_raw = consider_getarrayitem_gc
     consider_getarrayitem_gc_pure = consider_getarrayitem_gc
 
+    def consider_getinteriorfield_gc(self, op):
+        t = self._unpack_interiorfielddescr(op.getdescr())
+        ofs, itemsize, fieldsize, sign = t
+        if sign:
+            sign_loc = imm1
+        else:
+            sign_loc = imm0
+        args = op.getarglist()
+        base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
+        index_loc = self.rm.make_sure_var_in_reg(op.getarg(1), args)
+        # 'base' and 'index' are put in two registers (or one if 'index'
+        # is an immediate).  'result' can be in the same register as
+        # 'index' but must be in a different register than 'base'.
+        self.rm.possibly_free_var(op.getarg(1))
+        result_loc = self.force_allocate_reg(op.result, [op.getarg(0)])
+        self.rm.possibly_free_var(op.getarg(0))
+        self.Perform(op, [base_loc, ofs, itemsize, fieldsize,
+                          index_loc, sign_loc], result_loc)
+
     def consider_int_is_true(self, op, guard_op):
         # doesn't need arg to be in a register
         argloc = self.loc(op.getarg(0))
@@ -1252,7 +1264,6 @@
         self.rm.possibly_free_var(srcaddr_box)
 
     def _gen_address_inside_string(self, baseloc, ofsloc, resloc, is_unicode):
-        cpu = self.assembler.cpu
         if is_unicode:
             ofs_items, _, _ = symbolic.get_array_token(rstr.UNICODE,
                                                   self.translate_support_code)
@@ -1311,7 +1322,7 @@
         tmpreg = X86RegisterManager.all_regs[0]
         tmploc = self.rm.force_allocate_reg(box, selected_reg=tmpreg)
         xmmtmp = X86XMMRegisterManager.all_regs[0]
-        xmmtmploc = self.xrm.force_allocate_reg(box1, selected_reg=xmmtmp)
+        self.xrm.force_allocate_reg(box1, selected_reg=xmmtmp)
         # Part about non-floats
         # XXX we don't need a copy, we only just the original list
         src_locations1 = [self.loc(op.getarg(i)) for i in range(op.numargs())
@@ -1391,7 +1402,7 @@
     return lambda self, op: fn(self, op, None)
 
 def is_comparison_or_ovf_op(opnum):
-    from pypy.jit.metainterp.resoperation import opclasses, AbstractResOp
+    from pypy.jit.metainterp.resoperation import opclasses
     cls = opclasses[opnum]
     # hack hack: in theory they are instance method, but they don't use
     # any instance field, we can use a fake object
diff --git a/pypy/jit/backend/x86/test/test_del.py b/pypy/jit/backend/x86/test/test_del.py
--- a/pypy/jit/backend/x86/test/test_del.py
+++ b/pypy/jit/backend/x86/test/test_del.py
@@ -1,5 +1,4 @@
 
-import py
 from pypy.jit.backend.x86.test.test_basic import Jit386Mixin
 from pypy.jit.metainterp.test.test_del import DelTests
 
diff --git a/pypy/jit/backend/x86/test/test_dict.py b/pypy/jit/backend/x86/test/test_dict.py
new file mode 100644
--- /dev/null
+++ b/pypy/jit/backend/x86/test/test_dict.py
@@ -0,0 +1,9 @@
+
+from pypy.jit.backend.x86.test.test_basic import Jit386Mixin
+from pypy.jit.metainterp.test.test_dict import DictTests
+
+
+class TestDict(Jit386Mixin, DictTests):
+    # for the individual tests see
+    # ====> ../../../metainterp/test/test_dict.py
+    pass
diff --git a/pypy/jit/backend/x86/test/test_runner.py b/pypy/jit/backend/x86/test/test_runner.py
--- a/pypy/jit/backend/x86/test/test_runner.py
+++ b/pypy/jit/backend/x86/test/test_runner.py
@@ -31,7 +31,7 @@
 
     # for the individual tests see
     # ====> ../../test/runner_test.py
-    
+
     def setup_method(self, meth):
         self.cpu = CPU(rtyper=None, stats=FakeStats())
         self.cpu.setup_once()
@@ -69,22 +69,16 @@
 
     def test_allocations(self):
         from pypy.rpython.lltypesystem import rstr
-        
+
         allocs = [None]
         all = []
+        orig_new = self.cpu.gc_ll_descr.funcptr_for_new
         def f(size):
             allocs.insert(0, size)
-            buf = ctypes.create_string_buffer(size)
-            all.append(buf)
-            return ctypes.cast(buf, ctypes.c_void_p).value
-        func = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int)(f)
-        addr = ctypes.cast(func, ctypes.c_void_p).value
-        # ctypes produces an unsigned value. We need it to be signed for, eg,
-        # relative addressing to work properly.
-        addr = rffi.cast(lltype.Signed, addr)
-        
+            return orig_new(size)
+
         self.cpu.assembler.setup_once()
-        self.cpu.assembler.malloc_func_addr = addr
+        self.cpu.gc_ll_descr.funcptr_for_new = f
         ofs = symbolic.get_field_token(rstr.STR, 'chars', False)[0]
 
         res = self.execute_operation(rop.NEWSTR, [ConstInt(7)], 'ref')
@@ -108,7 +102,7 @@
         res = self.execute_operation(rop.NEW_ARRAY, [ConstInt(10)],
                                          'ref', descr)
         assert allocs[0] == 10*WORD + ofs + WORD
-        resbuf = self._resbuf(res)            
+        resbuf = self._resbuf(res)
         assert resbuf[ofs/WORD] == 10
 
         # ------------------------------------------------------------
@@ -116,7 +110,7 @@
         res = self.execute_operation(rop.NEW_ARRAY, [BoxInt(10)],
                                          'ref', descr)
         assert allocs[0] == 10*WORD + ofs + WORD
-        resbuf = self._resbuf(res)                        
+        resbuf = self._resbuf(res)
         assert resbuf[ofs/WORD] == 10
 
     def test_stringitems(self):
@@ -146,7 +140,7 @@
                                                      ConstInt(2), BoxInt(38)],
                                'void', descr)
         assert resbuf[itemsofs/WORD + 2] == 38
-        
+
         self.execute_operation(rop.SETARRAYITEM_GC, [res,
                                                      BoxInt(3), BoxInt(42)],
                                'void', descr)
@@ -167,7 +161,7 @@
                                                          BoxInt(2)],
                                    'int', descr)
         assert r.value == 38
-        
+
         r = self.execute_operation(rop.GETARRAYITEM_GC, [res, BoxInt(3)],
                                    'int', descr)
         assert r.value == 42
@@ -226,7 +220,7 @@
         self.execute_operation(rop.SETFIELD_GC, [res, BoxInt(1234)], 'void', ofs_i)
         i = self.execute_operation(rop.GETFIELD_GC, [res], 'int', ofs_i)
         assert i.value == 1234
-        
+
         #u = self.execute_operation(rop.GETFIELD_GC, [res, ofs_u], 'int')
         #assert u.value == 5
         self.execute_operation(rop.SETFIELD_GC, [res, ConstInt(1)], 'void',
@@ -299,7 +293,7 @@
                     else:
                         assert result != execute(self.cpu, None,
                                                  op, None, b).value
-                    
+
 
     def test_stuff_followed_by_guard(self):
         boxes = [(BoxInt(1), BoxInt(0)),
@@ -523,7 +517,7 @@
     def test_debugger_on(self):
         from pypy.tool.logparser import parse_log_file, extract_category
         from pypy.rlib import debug
-        
+
         loop = """
         [i0]
         debug_merge_point('xyz', 0)
diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -52,9 +52,11 @@
         newoperations = []
         #
         def do_rename(var, var_or_const):
+            if var.concretetype is lltype.Void:
+                renamings[var] = Constant(None, lltype.Void)
+                return
             renamings[var] = var_or_const
-            if (isinstance(var_or_const, Constant)
-                and var.concretetype != lltype.Void):
+            if isinstance(var_or_const, Constant):
                 value = var_or_const.value
                 value = lltype._cast_whatever(var.concretetype, value)
                 renamings_constants[var] = Constant(value, var.concretetype)
@@ -445,6 +447,8 @@
     rewrite_op_gc_identityhash = _do_builtin_call
     rewrite_op_gc_id           = _do_builtin_call
     rewrite_op_uint_mod        = _do_builtin_call
+    rewrite_op_cast_float_to_uint = _do_builtin_call
+    rewrite_op_cast_uint_to_float = _do_builtin_call
 
     # ----------
     # getfield/setfield/mallocs etc.
@@ -739,29 +743,54 @@
         return SpaceOperation(opname, [op.args[0]], op.result)
 
     def rewrite_op_getinteriorfield(self, op):
-        # only supports strings and unicodes
         assert len(op.args) == 3
-        assert op.args[1].value == 'chars'
         optype = op.args[0].concretetype
         if optype == lltype.Ptr(rstr.STR):
             opname = "strgetitem"
+            return SpaceOperation(opname, [op.args[0], op.args[2]], op.result)
+        elif optype == lltype.Ptr(rstr.UNICODE):
+            opname = "unicodegetitem"
+            return SpaceOperation(opname, [op.args[0], op.args[2]], op.result)
         else:
-            assert optype == lltype.Ptr(rstr.UNICODE)
-            opname = "unicodegetitem"
-        return SpaceOperation(opname, [op.args[0], op.args[2]], op.result)
+            v_inst, v_index, c_field = op.args
+            if op.result.concretetype is lltype.Void:
+                return
+            # only GcArray of Struct supported
+            assert isinstance(v_inst.concretetype.TO, lltype.GcArray)
+            STRUCT = v_inst.concretetype.TO.OF
+            assert isinstance(STRUCT, lltype.Struct)
+            descr = self.cpu.interiorfielddescrof(v_inst.concretetype.TO,
+                                                  c_field.value)
+            args = [v_inst, v_index, descr]
+            kind = getkind(op.result.concretetype)[0]
+            return SpaceOperation('getinteriorfield_gc_%s' % kind, args,
+                                  op.result)
 
     def rewrite_op_setinteriorfield(self, op):
-        # only supports strings and unicodes
         assert len(op.args) == 4
-        assert op.args[1].value == 'chars'
         optype = op.args[0].concretetype
         if optype == lltype.Ptr(rstr.STR):
             opname = "strsetitem"
+            return SpaceOperation(opname, [op.args[0], op.args[2], op.args[3]],
+                                  op.result)
+        elif optype == lltype.Ptr(rstr.UNICODE):
+            opname = "unicodesetitem"
+            return SpaceOperation(opname, [op.args[0], op.args[2], op.args[3]],
+                                  op.result)
         else:
-            assert optype == lltype.Ptr(rstr.UNICODE)
-            opname = "unicodesetitem"
-        return SpaceOperation(opname, [op.args[0], op.args[2], op.args[3]],
-                              op.result)
+            v_inst, v_index, c_field, v_value = op.args
+            if v_value.concretetype is lltype.Void:
+                return
+            # only GcArray of Struct supported
+            assert isinstance(v_inst.concretetype.TO, lltype.GcArray)
+            STRUCT = v_inst.concretetype.TO.OF
+            assert isinstance(STRUCT, lltype.Struct)
+            descr = self.cpu.interiorfielddescrof(v_inst.concretetype.TO,
+                                                  c_field.value)
+            kind = getkind(v_value.concretetype)[0]
+            args = [v_inst, v_index, v_value, descr]
+            return SpaceOperation('setinteriorfield_gc_%s' % kind, args,
+                                  op.result)
 
     def _rewrite_equality(self, op, opname):
         arg0, arg1 = op.args
@@ -775,6 +804,9 @@
     def _is_gc(self, v):
         return getattr(getattr(v.concretetype, "TO", None), "_gckind", "?") == 'gc'
 
+    def _is_rclass_instance(self, v):
+        return lltype._castdepth(v.concretetype.TO, rclass.OBJECT) >= 0
+
     def _rewrite_cmp_ptrs(self, op):
         if self._is_gc(op.args[0]):
             return op
@@ -792,11 +824,21 @@
         return self._rewrite_equality(op, 'int_is_true')
 
     def rewrite_op_ptr_eq(self, op):
-        op1 = self._rewrite_equality(op, 'ptr_iszero')
+        prefix = ''
+        if self._is_rclass_instance(op.args[0]):
+            assert self._is_rclass_instance(op.args[1])
+            op = SpaceOperation('instance_ptr_eq', op.args, op.result)
+            prefix = 'instance_'
+        op1 = self._rewrite_equality(op, prefix + 'ptr_iszero')
         return self._rewrite_cmp_ptrs(op1)
 
     def rewrite_op_ptr_ne(self, op):
-        op1 = self._rewrite_equality(op, 'ptr_nonzero')
+        prefix = ''
+        if self._is_rclass_instance(op.args[0]):
+            assert self._is_rclass_instance(op.args[1])
+            op = SpaceOperation('instance_ptr_ne', op.args, op.result)
+            prefix = 'instance_'
+        op1 = self._rewrite_equality(op, prefix + 'ptr_nonzero')
         return self._rewrite_cmp_ptrs(op1)
 
     rewrite_op_ptr_iszero = _rewrite_cmp_ptrs
@@ -825,26 +867,44 @@
         elif not float_arg and float_res:
             # some int -> some float
             ops = []
-            v1 = varoftype(lltype.Signed)
-            oplist = self.rewrite_operation(
-                SpaceOperation('force_cast', [v_arg], v1)
-            )
-            if oplist:
-                ops.extend(oplist)
+            v2 = varoftype(lltype.Float)
+            sizesign = rffi.size_and_sign(v_arg.concretetype)
+            if sizesign <= rffi.size_and_sign(lltype.Signed):
+                # cast from a type that fits in an int: either the size is
+                # smaller, or it is equal and it is not unsigned
+                v1 = varoftype(lltype.Signed)
+                oplist = self.rewrite_operation(
+                    SpaceOperation('force_cast', [v_arg], v1)
+                )
+                if oplist:
+                    ops.extend(oplist)
+                else:
+                    v1 = v_arg
+                op = self.rewrite_operation(
+                    SpaceOperation('cast_int_to_float', [v1], v2)
+                )
+                ops.append(op)
             else:
-                v1 = v_arg
-            v2 = varoftype(lltype.Float)
-            op = self.rewrite_operation(
-                SpaceOperation('cast_int_to_float', [v1], v2)
-            )
-            ops.append(op)
+                if sizesign == rffi.size_and_sign(lltype.Unsigned):
+                    opname = 'cast_uint_to_float'
+                elif sizesign == rffi.size_and_sign(lltype.SignedLongLong):
+                    opname = 'cast_longlong_to_float'
+                elif sizesign == rffi.size_and_sign(lltype.UnsignedLongLong):
+                    opname = 'cast_ulonglong_to_float'
+                else:
+                    raise AssertionError('cast_x_to_float: %r' % (sizesign,))
+                ops1 = self.rewrite_operation(
+                    SpaceOperation(opname, [v_arg], v2)
+                )
+                if not isinstance(ops1, list): ops1 = [ops1]
+                ops.extend(ops1)
             op2 = self.rewrite_operation(
                 SpaceOperation('force_cast', [v2], v_result)
             )
             if op2:
                 ops.append(op2)
             else:
-                op.result = v_result
+                ops[-1].result = v_result
             return ops
         elif float_arg and not float_res:
             # some float -> some int
@@ -857,18 +917,36 @@
                 ops.append(op1)
             else:
                 v1 = v_arg
-            v2 = varoftype(lltype.Signed)
-            op = self.rewrite_operation(
-                SpaceOperation('cast_float_to_int', [v1], v2)
-            )
-            ops.append(op)
-            oplist = self.rewrite_operation(
-                SpaceOperation('force_cast', [v2], v_result)
-            )
-            if oplist:
-                ops.extend(oplist)
+            sizesign = rffi.size_and_sign(v_result.concretetype)
+            if sizesign <= rffi.size_and_sign(lltype.Signed):
+                # cast to a type that fits in an int: either the size is
+                # smaller, or it is equal and it is not unsigned
+                v2 = varoftype(lltype.Signed)
+                op = self.rewrite_operation(
+                    SpaceOperation('cast_float_to_int', [v1], v2)
+                )
+                ops.append(op)
+                oplist = self.rewrite_operation(
+                    SpaceOperation('force_cast', [v2], v_result)
+                )
+                if oplist:
+                    ops.extend(oplist)
+                else:
+                    op.result = v_result
             else:
-                op.result = v_result
+                if sizesign == rffi.size_and_sign(lltype.Unsigned):
+                    opname = 'cast_float_to_uint'
+                elif sizesign == rffi.size_and_sign(lltype.SignedLongLong):
+                    opname = 'cast_float_to_longlong'
+                elif sizesign == rffi.size_and_sign(lltype.UnsignedLongLong):
+                    opname = 'cast_float_to_ulonglong'
+                else:
+                    raise AssertionError('cast_float_to_x: %r' % (sizesign,))
+                ops1 = self.rewrite_operation(
+                    SpaceOperation(opname, [v1], v_result)
+                )
+                if not isinstance(ops1, list): ops1 = [ops1]
+                ops.extend(ops1)
             return ops
         else:
             assert False
@@ -1074,8 +1152,6 @@
     # The new operation is optionally further processed by rewrite_operation().
     for _old, _new in [('bool_not', 'int_is_zero'),
                        ('cast_bool_to_float', 'cast_int_to_float'),
-                       ('cast_uint_to_float', 'cast_int_to_float'),
-                       ('cast_float_to_uint', 'cast_float_to_int'),
 
                        ('int_add_nonneg_ovf', 'int_add_ovf'),
                        ('keepalive', '-live-'),
diff --git a/pypy/jit/codewriter/support.py b/pypy/jit/codewriter/support.py
--- a/pypy/jit/codewriter/support.py
+++ b/pypy/jit/codewriter/support.py
@@ -13,7 +13,6 @@
 from pypy.translator.simplify import get_funcobj
 from pypy.translator.unsimplify import split_block
 from pypy.objspace.flow.model import Constant
-from pypy import conftest
 from pypy.translator.translator import TranslationContext
 from pypy.annotation.policy import AnnotatorPolicy
 from pypy.annotation import model as annmodel
@@ -49,15 +48,13 @@
     a.build_types(func, argtypes, main_entry_point=True)
     rtyper = t.buildrtyper(type_system = type_system)
     rtyper.specialize()
-    if inline:
-        auto_inlining(t, threshold=inline)
+    #if inline:
+    #    auto_inlining(t, threshold=inline)
     if backendoptimize:
         from pypy.translator.backendopt.all import backend_optimizations
         backend_optimizations(t, inline_threshold=inline or 0,
                 remove_asserts=True, really_remove_asserts=True)
 
-    #if conftest.option.view:
-    #    t.view()
     return rtyper
 
 def getgraph(func, values):
@@ -233,6 +230,17 @@
     else:
         return x
 
+def _ll_1_cast_uint_to_float(x):
+    # XXX on 32-bit platforms, this should be done using cast_longlong_to_float
+    # (which is a residual call right now in the x86 backend)
+    return llop.cast_uint_to_float(lltype.Float, x)
+
+def _ll_1_cast_float_to_uint(x):
+    # XXX on 32-bit platforms, this should be done using cast_float_to_longlong
+    # (which is a residual call right now in the x86 backend)
+    return llop.cast_float_to_uint(lltype.Unsigned, x)
+
+
 # math support
 # ------------
 
@@ -457,6 +465,8 @@
         return LLtypeHelpers._dictnext_items(lltype.Ptr(RES), iter)
     _ll_1_dictiter_nextitems.need_result_type = True
 
+    _ll_1_dict_resize = ll_rdict.ll_dict_resize
+
     # ---------- strings and unicode ----------
 
     _ll_1_str_str2unicode = ll_rstr.LLHelpers.ll_str2unicode
diff --git a/pypy/jit/codewriter/test/test_flatten.py b/pypy/jit/codewriter/test/test_flatten.py
--- a/pypy/jit/codewriter/test/test_flatten.py
+++ b/pypy/jit/codewriter/test/test_flatten.py
@@ -8,7 +8,7 @@
 from pypy.rpython.lltypesystem import lltype, rclass, rstr
 from pypy.objspace.flow.model import SpaceOperation, Variable, Constant
 from pypy.translator.unsimplify import varoftype
-from pypy.rlib.rarithmetic import ovfcheck, r_uint
+from pypy.rlib.rarithmetic import ovfcheck, r_uint, r_longlong, r_ulonglong
 from pypy.rlib.jit import dont_look_inside, _we_are_jitted, JitDriver
 from pypy.rlib.objectmodel import keepalive_until_here
 from pypy.rlib import jit
@@ -70,7 +70,8 @@
         return 'residual'
     def getcalldescr(self, op, oopspecindex=None, extraeffect=None):
         try:
-            if 'cannot_raise' in op.args[0].value._obj.graph.name:
+            name = op.args[0].value._obj._name
+            if 'cannot_raise' in name or name.startswith('cast_'):
                 return self._descr_cannot_raise
         except AttributeError:
             pass
@@ -900,6 +901,67 @@
             int_return %i4
         """, transform=True)
 
+        def f(dbl):
+            return rffi.cast(rffi.UCHAR, dbl)
+        self.encoding_test(f, [12.456], """
+            cast_float_to_int %f0 -> %i0
+            int_and %i0, $255 -> %i1
+            int_return %i1
+        """, transform=True)
+
+        def f(dbl):
+            return rffi.cast(lltype.Unsigned, dbl)
+        self.encoding_test(f, [12.456], """
+            residual_call_irf_i $<* fn cast_float_to_uint>, <Descr>, I[], R[], F[%f0] -> %i0
+            int_return %i0
+        """, transform=True)
+
+        def f(i):
+            return rffi.cast(lltype.Float, chr(i))    # "char -> float"
+        self.encoding_test(f, [12], """
+            cast_int_to_float %i0 -> %f0
+            float_return %f0
+        """, transform=True)
+
+        def f(i):
+            return rffi.cast(lltype.Float, r_uint(i))    # "uint -> float"
+        self.encoding_test(f, [12], """
+            residual_call_irf_f $<* fn cast_uint_to_float>, <Descr>, I[%i0], R[], F[] -> %f0
+            float_return %f0
+        """, transform=True)
+
+        if not longlong.is_64_bit:
+            def f(dbl):
+                return rffi.cast(lltype.SignedLongLong, dbl)
+            self.encoding_test(f, [12.3], """
+                residual_call_irf_f $<* fn llong_from_float>, <Descr>, I[], R[], F[%f0] -> %f1
+                float_return %f1
+            """, transform=True)
+
+            def f(dbl):
+                return rffi.cast(lltype.UnsignedLongLong, dbl)
+            self.encoding_test(f, [12.3], """
+                residual_call_irf_f $<* fn ullong_from_float>, <Descr>, I[], R[], F[%f0] -> %f1
+                float_return %f1
+            """, transform=True)
+
+            def f(x):
+                ll = r_longlong(x)
+                return rffi.cast(lltype.Float, ll)
+            self.encoding_test(f, [12], """
+                residual_call_irf_f $<* fn llong_from_int>, <Descr>, I[%i0], R[], F[] -> %f0
+                residual_call_irf_f $<* fn llong_to_float>, <Descr>, I[], R[], F[%f0] -> %f1
+                float_return %f1
+            """, transform=True)
+
+            def f(x):
+                ll = r_ulonglong(x)
+                return rffi.cast(lltype.Float, ll)
+            self.encoding_test(f, [12], """
+                residual_call_irf_f $<* fn ullong_from_int>, <Descr>, I[%i0], R[], F[] -> %f0
+                residual_call_irf_f $<* fn ullong_u_to_float>, <Descr>, I[], R[], F[%f0] -> %f1
+                float_return %f1
+            """, transform=True)
 
     def test_direct_ptradd(self):
         from pypy.rpython.lltypesystem import rffi
diff --git a/pypy/jit/codewriter/test/test_jtransform.py b/pypy/jit/codewriter/test/test_jtransform.py
--- a/pypy/jit/codewriter/test/test_jtransform.py
+++ b/pypy/jit/codewriter/test/test_jtransform.py
@@ -1,4 +1,3 @@
-import py
 import random
 try:
     from itertools import product
@@ -16,13 +15,13 @@
 
 from pypy.objspace.flow.model import FunctionGraph, Block, Link
 from pypy.objspace.flow.model import SpaceOperation, Variable, Constant
-from pypy.jit.codewriter.jtransform import Transformer
-from pypy.jit.metainterp.history import getkind
-from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr, rlist
+from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr
 from pypy.rpython.lltypesystem.module import ll_math
 from pypy.translator.unsimplify import varoftype
 from pypy.jit.codewriter import heaptracker, effectinfo
 from pypy.jit.codewriter.flatten import ListOfKind
+from pypy.jit.codewriter.jtransform import Transformer
+from pypy.jit.metainterp.history import getkind
 
 def const(x):
     return Constant(x, lltype.typeOf(x))
@@ -37,6 +36,8 @@
         return ('calldescr', FUNC, ARGS, RESULT)
     def fielddescrof(self, STRUCT, name):
         return ('fielddescr', STRUCT, name)
+    def interiorfielddescrof(self, ARRAY, name):
+        return ('interiorfielddescr', ARRAY, name)
     def arraydescrof(self, ARRAY):
         return FakeDescr(('arraydescr', ARRAY))
     def sizeof(self, STRUCT):
@@ -539,7 +540,7 @@
 
 def test_rename_on_links():
     v1 = Variable()
-    v2 = Variable()
+    v2 = Variable(); v2.concretetype = llmemory.Address
     v3 = Variable()
     block = Block([v1])
     block.operations = [SpaceOperation('cast_pointer', [v1], v2)]
@@ -575,10 +576,10 @@
         assert op1.args == [v2]
 
 def test_ptr_eq():
-    v1 = varoftype(rclass.OBJECTPTR)
-    v2 = varoftype(rclass.OBJECTPTR)
+    v1 = varoftype(lltype.Ptr(rstr.STR))
+    v2 = varoftype(lltype.Ptr(rstr.STR))
     v3 = varoftype(lltype.Bool)
-    c0 = const(lltype.nullptr(rclass.OBJECT))
+    c0 = const(lltype.nullptr(rstr.STR))
     #
     for opname, reducedname in [('ptr_eq', 'ptr_iszero'),
                                 ('ptr_ne', 'ptr_nonzero')]:
@@ -597,6 +598,31 @@
         assert op1.opname == reducedname
         assert op1.args == [v2]
 
+def test_instance_ptr_eq():
+    v1 = varoftype(rclass.OBJECTPTR)
+    v2 = varoftype(rclass.OBJECTPTR)
+    v3 = varoftype(lltype.Bool)
+    c0 = const(lltype.nullptr(rclass.OBJECT))
+
+    for opname, newopname, reducedname in [
+        ('ptr_eq', 'instance_ptr_eq', 'instance_ptr_iszero'),
+        ('ptr_ne', 'instance_ptr_ne', 'instance_ptr_nonzero')
+    ]:
+        op = SpaceOperation(opname, [v1, v2], v3)
+        op1 = Transformer().rewrite_operation(op)
+        assert op1.opname == newopname
+        assert op1.args == [v1, v2]
+
+        op = SpaceOperation(opname, [v1, c0], v3)
+        op1 = Transformer().rewrite_operation(op)
+        assert op1.opname == reducedname
+        assert op1.args == [v1]
+
+        op = SpaceOperation(opname, [c0, v1], v3)
+        op1 = Transformer().rewrite_operation(op)
+        assert op1.opname == reducedname
+        assert op1.args == [v1]
+
 def test_nongc_ptr_eq():
     v1 = varoftype(rclass.NONGCOBJECTPTR)
     v2 = varoftype(rclass.NONGCOBJECTPTR)
@@ -676,6 +702,22 @@
     assert op1.args == [v, v_index]
     assert op1.result == v_result
 
+def test_dict_getinteriorfield():
+    DICT = lltype.GcArray(lltype.Struct('ENTRY', ('v', lltype.Signed),
+                                        ('k', lltype.Signed)))
+    v = varoftype(lltype.Ptr(DICT))
+    i = varoftype(lltype.Signed)
+    v_result = varoftype(lltype.Signed)
+    op = SpaceOperation('getinteriorfield', [v, i, Constant('v', lltype.Void)],
+                        v_result)
+    op1 = Transformer(FakeCPU()).rewrite_operation(op)
+    assert op1.opname == 'getinteriorfield_gc_i'
+    assert op1.args == [v, i, ('interiorfielddescr', DICT, 'v')]
+    op = SpaceOperation('getinteriorfield', [v, i, Constant('v', lltype.Void)],
+                        Constant(None, lltype.Void))
+    op1 = Transformer(FakeCPU()).rewrite_operation(op)
+    assert op1 is None
+
 def test_str_setinteriorfield():
     v = varoftype(lltype.Ptr(rstr.STR))
     v_index = varoftype(lltype.Signed)
@@ -702,6 +744,23 @@
     assert op1.args == [v, v_index, v_newchr]
     assert op1.result == v_void
 
+def test_dict_setinteriorfield():
+    DICT = lltype.GcArray(lltype.Struct('ENTRY', ('v', lltype.Signed),
+                                        ('k', lltype.Signed)))
+    v = varoftype(lltype.Ptr(DICT))
+    i = varoftype(lltype.Signed)
+    v_void = varoftype(lltype.Void)
+    op = SpaceOperation('setinteriorfield', [v, i, Constant('v', lltype.Void),
+                                             i],
+                        v_void)
+    op1 = Transformer(FakeCPU()).rewrite_operation(op)
+    assert op1.opname == 'setinteriorfield_gc_i'
+    assert op1.args == [v, i, i, ('interiorfielddescr', DICT, 'v')]
+    op = SpaceOperation('setinteriorfield', [v, i, Constant('v', lltype.Void),
+                                             v_void], v_void)
+    op1 = Transformer(FakeCPU()).rewrite_operation(op)
+    assert not op1
+
 def test_promote_1():
     v1 = varoftype(lltype.Signed)
     v2 = varoftype(lltype.Signed)
diff --git a/pypy/jit/metainterp/blackhole.py b/pypy/jit/metainterp/blackhole.py
--- a/pypy/jit/metainterp/blackhole.py
+++ b/pypy/jit/metainterp/blackhole.py
@@ -6,7 +6,6 @@
 from pypy.rlib.debug import make_sure_not_resized
 from pypy.rpython.lltypesystem import lltype, llmemory, rclass
 from pypy.rpython.lltypesystem.lloperation import llop
-from pypy.rpython.llinterp import LLException
 from pypy.jit.codewriter.jitcode import JitCode, SwitchDictDescr
 from pypy.jit.codewriter import heaptracker, longlong
 from pypy.jit.metainterp.jitexc import JitException, get_llexception, reraise
@@ -511,6 +510,12 @@
     @arguments("r", returns="i")
     def bhimpl_ptr_nonzero(a):
         return bool(a)
+    @arguments("r", "r", returns="i")
+    def bhimpl_instance_ptr_eq(a, b):
+        return a == b
+    @arguments("r", "r", returns="i")
+    def bhimpl_instance_ptr_ne(a, b):
+        return a != b
     @arguments("r", returns="r")
     def bhimpl_cast_opaque_ptr(a):
         return a
@@ -642,6 +647,9 @@
         a = longlong.getrealfloat(a)
         # note: we need to call int() twice to care for the fact that
         # int(-2147483648.0) returns a long :-(
+        # we could also call intmask() instead of the outermost int(), but
+        # it's probably better to explicitly crash (by getting a long) if a
+        # non-translated version tries to cast a too large float to an int.
         return int(int(a))
 
     @arguments("i", returns="f")
@@ -1165,6 +1173,26 @@
         array = cpu.bh_getfield_gc_r(vable, fdescr)
         return cpu.bh_arraylen_gc(adescr, array)
 
+    @arguments("cpu", "r", "i", "d", returns="i")
+    def bhimpl_getinteriorfield_gc_i(cpu, array, index, descr):
+        return cpu.bh_getinteriorfield_gc_i(array, index, descr)
+    @arguments("cpu", "r", "i", "d", returns="r")
+    def bhimpl_getinteriorfield_gc_r(cpu, array, index, descr):
+        return cpu.bh_getinteriorfield_gc_r(array, index, descr)
+    @arguments("cpu", "r", "i", "d", returns="f")
+    def bhimpl_getinteriorfield_gc_f(cpu, array, index, descr):
+        return cpu.bh_getinteriorfield_gc_f(array, index, descr)
+
+    @arguments("cpu", "r", "i", "d", "i")
+    def bhimpl_setinteriorfield_gc_i(cpu, array, index, descr, value):
+        cpu.bh_setinteriorfield_gc_i(array, index, descr, value)
+    @arguments("cpu", "r", "i", "d", "r")
+    def bhimpl_setinteriorfield_gc_r(cpu, array, index, descr, value):
+        cpu.bh_setinteriorfield_gc_r(array, index, descr, value)
+    @arguments("cpu", "r", "i", "d", "f")
+    def bhimpl_setinteriorfield_gc_f(cpu, array, index, descr, value):
+        cpu.bh_setinteriorfield_gc_f(array, index, descr, value)
+
     @arguments("cpu", "r", "d", returns="i")
     def bhimpl_getfield_gc_i(cpu, struct, fielddescr):
         return cpu.bh_getfield_gc_i(struct, fielddescr)
diff --git a/pypy/jit/metainterp/executor.py b/pypy/jit/metainterp/executor.py
--- a/pypy/jit/metainterp/executor.py
+++ b/pypy/jit/metainterp/executor.py
@@ -1,11 +1,8 @@
 """This implements pyjitpl's execution of operations.
 """
 
-import py
-from pypy.rpython.lltypesystem import lltype, llmemory, rstr
-from pypy.rpython.ootypesystem import ootype
-from pypy.rpython.lltypesystem.lloperation import llop
-from pypy.rlib.rarithmetic import ovfcheck, r_uint, intmask, r_longlong
+from pypy.rpython.lltypesystem import lltype, rstr
+from pypy.rlib.rarithmetic import ovfcheck, r_longlong
 from pypy.rlib.rtimer import read_timestamp
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.jit.metainterp.history import BoxInt, BoxPtr, BoxFloat, check_descr
@@ -123,6 +120,29 @@
     else:
         cpu.bh_setarrayitem_raw_i(arraydescr, array, index, itembox.getint())
 
+def do_getinteriorfield_gc(cpu, _, arraybox, indexbox, descr):
+    array = arraybox.getref_base()
+    index = indexbox.getint()
+    if descr.is_pointer_field():
+        return BoxPtr(cpu.bh_getinteriorfield_gc_r(array, index, descr))
+    elif descr.is_float_field():
+        return BoxFloat(cpu.bh_getinteriorfield_gc_f(array, index, descr))
+    else:
+        return BoxInt(cpu.bh_getinteriorfield_gc_i(array, index, descr))
+
+def do_setinteriorfield_gc(cpu, _, arraybox, indexbox, valuebox, descr):
+    array = arraybox.getref_base()
+    index = indexbox.getint()
+    if descr.is_pointer_field():
+        cpu.bh_setinteriorfield_gc_r(array, index, descr,
+                                     valuebox.getref_base())
+    elif descr.is_float_field():
+        cpu.bh_setinteriorfield_gc_f(array, index, descr,
+                                     valuebox.getfloatstorage())
+    else:
+        cpu.bh_setinteriorfield_gc_i(array, index, descr,
+                                     valuebox.getint())
+
 def do_getfield_gc(cpu, _, structbox, fielddescr):
     struct = structbox.getref_base()
     if fielddescr.is_pointer_field():
diff --git a/pypy/jit/metainterp/history.py b/pypy/jit/metainterp/history.py
--- a/pypy/jit/metainterp/history.py
+++ b/pypy/jit/metainterp/history.py
@@ -16,6 +16,7 @@
 INT   = 'i'
 REF   = 'r'
 FLOAT = 'f'
+STRUCT = 's'
 HOLE  = '_'
 VOID  = 'v'
 
@@ -172,6 +173,11 @@
         """
         raise NotImplementedError
 
+    def is_array_of_structs(self):
+        """ Implement for array descr
+        """
+        raise NotImplementedError
+
     def is_pointer_field(self):
         """ Implement for field descr
         """
diff --git a/pypy/jit/metainterp/optimizeopt/rewrite.py b/pypy/jit/metainterp/optimizeopt/rewrite.py
--- a/pypy/jit/metainterp/optimizeopt/rewrite.py
+++ b/pypy/jit/metainterp/optimizeopt/rewrite.py
@@ -337,7 +337,7 @@
     def optimize_INT_IS_ZERO(self, op):
         self._optimize_nullness(op, op.getarg(0), False)
 
-    def _optimize_oois_ooisnot(self, op, expect_isnot):
+    def _optimize_oois_ooisnot(self, op, expect_isnot, instance):
         value0 = self.getvalue(op.getarg(0))
         value1 = self.getvalue(op.getarg(1))
         if value0.is_virtual():
@@ -355,21 +355,28 @@
         elif value0 is value1:
             self.make_constant_int(op.result, not expect_isnot)
         else:
-            cls0 = value0.get_constant_class(self.optimizer.cpu)
-            if cls0 is not None:
-                cls1 = value1.get_constant_class(self.optimizer.cpu)
-                if cls1 is not None and not cls0.same_constant(cls1):
-                    # cannot be the same object, as we know that their
-                    # class is different
-                    self.make_constant_int(op.result, expect_isnot)
-                    return
+            if instance:
+                cls0 = value0.get_constant_class(self.optimizer.cpu)
+                if cls0 is not None:
+                    cls1 = value1.get_constant_class(self.optimizer.cpu)
+                    if cls1 is not None and not cls0.same_constant(cls1):
+                        # cannot be the same object, as we know that their
+                        # class is different
+                        self.make_constant_int(op.result, expect_isnot)
+                        return
             self.emit_operation(op)
 
+    def optimize_PTR_EQ(self, op):
+        self._optimize_oois_ooisnot(op, False, False)
+
     def optimize_PTR_NE(self, op):
-        self._optimize_oois_ooisnot(op, True)
+        self._optimize_oois_ooisnot(op, True, False)
 
-    def optimize_PTR_EQ(self, op):
-        self._optimize_oois_ooisnot(op, False)
+    def optimize_INSTANCE_PTR_EQ(self, op):
+        self._optimize_oois_ooisnot(op, False, True)
+
+    def optimize_INSTANCE_PTR_NE(self, op):
+        self._optimize_oois_ooisnot(op, True, True)
 
 ##    def optimize_INSTANCEOF(self, op):
 ##        value = self.getvalue(op.args[0])
@@ -448,6 +455,9 @@
         if v2.is_constant() and v2.box.getint() == 1:
             self.make_equal_to(op.result, v1)
             return
+        elif v1.is_constant() and v1.box.getint() == 0:
+            self.make_constant_int(op.result, 0)
+            return
         if v1.intbound.known_ge(IntBound(0, 0)) and v2.is_constant():
             val = v2.box.getint()
             if val & (val - 1) == 0 and val > 0: # val == 2**shift
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -508,13 +508,13 @@
         ops = """
         [p0]
         guard_class(p0, ConstClass(node_vtable)) []
-        i0 = ptr_ne(p0, NULL)
+        i0 = instance_ptr_ne(p0, NULL)
         guard_true(i0) []
-        i1 = ptr_eq(p0, NULL)
+        i1 = instance_ptr_eq(p0, NULL)
         guard_false(i1) []
-        i2 = ptr_ne(NULL, p0)
+        i2 = instance_ptr_ne(NULL, p0)
         guard_true(i0) []
-        i3 = ptr_eq(NULL, p0)
+        i3 = instance_ptr_eq(NULL, p0)
         guard_false(i1) []
         jump(p0)
         """
@@ -2026,7 +2026,7 @@
         ops = """
         [p1]
         guard_class(p1, ConstClass(node_vtable2)) []
-        i = ptr_ne(ConstPtr(myptr), p1)
+        i = instance_ptr_ne(ConstPtr(myptr), p1)
         guard_true(i) []
         jump(p1)
         """
@@ -2181,6 +2181,17 @@
         """
         self.optimize_loop(ops, expected)
 
+        ops = """
+        [i0]
+        i1 = int_floordiv(0, i0)
+        jump(i1)
+        """
+        expected = """
+        [i0]
+        jump(0)
+        """
+        self.optimize_loop(ops, expected)
+
     def test_fold_partially_constant_ops_ovf(self):
         ops = """
         [i0]
@@ -4789,6 +4800,18 @@
         """
         self.optimize_strunicode_loop(ops, expected)
 
+    def test_ptr_eq_str_constant(self):
+        ops = """
+        []
+        i0 = ptr_eq(s"abc", s"\x00")
+        finish(i0)
+        """
+        expected = """
+        []
+        finish(0)
+        """
+        self.optimize_loop(ops, expected)
+
 
 class TestLLtype(BaseTestOptimizeBasic, LLtypeMixin):
     pass
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -2683,7 +2683,7 @@
         ops = """
         [p1]
         guard_class(p1, ConstClass(node_vtable2)) []
-        i = ptr_ne(ConstPtr(myptr), p1)
+        i = instance_ptr_ne(ConstPtr(myptr), p1)
         guard_true(i) []
         jump(p1)
         """
@@ -3331,7 +3331,7 @@
         jump(p1, i1, i2, i6)
         '''
         self.optimize_loop(ops, expected, preamble)
-        
+
 
     # ----------
 
@@ -7368,7 +7368,7 @@
         ops = """
         [p1, p2]
         setarrayitem_gc(p1, 2, 10, descr=arraydescr)
-        setarrayitem_gc(p2, 3, 13, descr=arraydescr)        
+        setarrayitem_gc(p2, 3, 13, descr=arraydescr)
         call(0, p1, p2, 0, 0, 10, descr=arraycopydescr)
         jump(p1, p2)
         """
diff --git a/pypy/jit/metainterp/optimizeopt/virtualize.py b/pypy/jit/metainterp/optimizeopt/virtualize.py
--- a/pypy/jit/metainterp/optimizeopt/virtualize.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualize.py
@@ -59,7 +59,7 @@
 
     def import_from(self, other, optimizer):
         raise NotImplementedError("should not be called at this level")
-    
+
 def get_fielddescrlist_cache(cpu):
     if not hasattr(cpu, '_optimizeopt_fielddescrlist_cache'):
         result = descrlist_dict()
@@ -113,7 +113,7 @@
         #
         if not we_are_translated():
             op.name = 'FORCE ' + self.source_op.name
-            
+
         if self._is_immutable_and_filled_with_constants(optforce):
             box = optforce.optimizer.constant_fold(op)
             self.make_constant(box)
@@ -239,12 +239,12 @@
         for index in range(len(self._items)):
             self._items[index] = self._items[index].force_at_end_of_preamble(already_forced, optforce)
         return self
-    
+
     def _really_force(self, optforce):
         assert self.source_op is not None
         if not we_are_translated():
             self.source_op.name = 'FORCE ' + self.source_op.name
-        optforce.emit_operation(self.source_op)        
+        optforce.emit_operation(self.source_op)
         self.box = box = self.source_op.result
         for index in range(len(self._items)):
             subvalue = self._items[index]
@@ -276,7 +276,7 @@
 
     def new(self):
         return OptVirtualize()
-        
+
     def make_virtual(self, known_class, box, source_op=None):
         vvalue = VirtualValue(self.optimizer.cpu, known_class, box, source_op)
         self.make_equal_to(box, vvalue)
@@ -386,7 +386,8 @@
 
     def optimize_NEW_ARRAY(self, op):
         sizebox = self.get_constant_box(op.getarg(0))
-        if sizebox is not None:
+        # For now we can't make arrays of structs virtual.
+        if sizebox is not None and not op.getdescr().is_array_of_structs():
             # if the original 'op' did not have a ConstInt as argument,
             # build a new one with the ConstInt argument
             if not isinstance(op.getarg(0), ConstInt):
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -36,6 +36,7 @@
 
 
 class MIFrame(object):
+    debug = False
 
     def __init__(self, metainterp):
         self.metainterp = metainterp
@@ -164,7 +165,7 @@
         if not we_are_translated():
             for b in registers[count:]:
                 assert not oldbox.same_box(b)
-                
+
 
     def make_result_of_lastop(self, resultbox):
         got_type = resultbox.type
@@ -198,7 +199,7 @@
                     'float_add', 'float_sub', 'float_mul', 'float_truediv',
                     'float_lt', 'float_le', 'float_eq',
                     'float_ne', 'float_gt', 'float_ge',
-                    'ptr_eq', 'ptr_ne',
+                    'ptr_eq', 'ptr_ne', 'instance_ptr_eq', 'instance_ptr_ne',
                     ]:
         exec py.code.Source('''
             @arguments("box", "box")
@@ -557,6 +558,14 @@
     opimpl_getfield_gc_r_pure = _opimpl_getfield_gc_pure_any
     opimpl_getfield_gc_f_pure = _opimpl_getfield_gc_pure_any
 
+    @arguments("box", "box", "descr")
+    def _opimpl_getinteriorfield_gc_any(self, array, index, descr):
+        return self.execute_with_descr(rop.GETINTERIORFIELD_GC, descr,
+                                       array, index)
+    opimpl_getinteriorfield_gc_i = _opimpl_getinteriorfield_gc_any
+    opimpl_getinteriorfield_gc_f = _opimpl_getinteriorfield_gc_any
+    opimpl_getinteriorfield_gc_r = _opimpl_getinteriorfield_gc_any
+
     @specialize.arg(1)
     def _opimpl_getfield_gc_any_pureornot(self, opnum, box, fielddescr):
         tobox = self.metainterp.heapcache.getfield(box, fielddescr)
@@ -597,6 +606,15 @@
     opimpl_setfield_gc_r = _opimpl_setfield_gc_any
     opimpl_setfield_gc_f = _opimpl_setfield_gc_any
 
+    @arguments("box", "box", "box", "descr")
+    def _opimpl_setinteriorfield_gc_any(self, array, index, value, descr):
+        self.execute_with_descr(rop.SETINTERIORFIELD_GC, descr,
+                                array, index, value)
+    opimpl_setinteriorfield_gc_i = _opimpl_setinteriorfield_gc_any
+    opimpl_setinteriorfield_gc_f = _opimpl_setinteriorfield_gc_any
+    opimpl_setinteriorfield_gc_r = _opimpl_setinteriorfield_gc_any
+
+
     @arguments("box", "descr")
     def _opimpl_getfield_raw_any(self, box, fielddescr):
         return self.execute_with_descr(rop.GETFIELD_RAW, fielddescr, box)
@@ -2597,17 +2615,21 @@
         self.pc = position
         #
         if not we_are_translated():
-            print '\tpyjitpl: %s(%s)' % (name, ', '.join(map(repr, args))),
+            if self.debug:
+                print '\tpyjitpl: %s(%s)' % (name, ', '.join(map(repr, args))),
             try:
                 resultbox = unboundmethod(self, *args)
             except Exception, e:
-                print '-> %s!' % e.__class__.__name__
+                if self.debug:
+                    print '-> %s!' % e.__class__.__name__
                 raise
             if num_return_args == 0:
-                print
+                if self.debug:
+                    print
                 assert resultbox is None
             else:
-                print '-> %r' % (resultbox,)
+                if self.debug:
+                    print '-> %r' % (resultbox,)
                 assert argcodes[next_argcode] == '>'
                 result_argcode = argcodes[next_argcode + 1]
                 assert resultbox.type == {'i': history.INT,
diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py
--- a/pypy/jit/metainterp/resoperation.py
+++ b/pypy/jit/metainterp/resoperation.py
@@ -1,5 +1,4 @@
 from pypy.rlib.objectmodel import we_are_translated
-from pypy.rlib.debug import make_sure_not_resized
 
 def ResOperation(opnum, args, result, descr=None):
     cls = opclasses[opnum]
@@ -405,8 +404,8 @@
     'FLOAT_TRUEDIV/2',
     'FLOAT_NEG/1',
     'FLOAT_ABS/1',
-    'CAST_FLOAT_TO_INT/1',
-    'CAST_INT_TO_FLOAT/1',
+    'CAST_FLOAT_TO_INT/1',          # don't use for unsigned ints; we would
+    'CAST_INT_TO_FLOAT/1',          # need some messy code in the backend
     'CAST_FLOAT_TO_SINGLEFLOAT/1',
     'CAST_SINGLEFLOAT_TO_FLOAT/1',
     #
@@ -440,6 +439,8 @@
     #
     'PTR_EQ/2b',
     'PTR_NE/2b',
+    'INSTANCE_PTR_EQ/2b',
+    'INSTANCE_PTR_NE/2b',
     'CAST_OPAQUE_PTR/1b',
     #
     'ARRAYLEN_GC/1d',
@@ -459,6 +460,7 @@
 
     'GETARRAYITEM_GC/2d',
     'GETARRAYITEM_RAW/2d',
+    'GETINTERIORFIELD_GC/2d',
     'GETFIELD_GC/1d',
     'GETFIELD_RAW/1d',
     '_MALLOC_FIRST',
@@ -475,6 +477,7 @@
 
     'SETARRAYITEM_GC/3d',
     'SETARRAYITEM_RAW/3d',
+    'SETINTERIORFIELD_GC/3d',
     'SETFIELD_GC/2d',
     'SETFIELD_RAW/2d',
     'STRSETITEM/3',
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -3436,6 +3436,74 @@
         res = self.meta_interp(f, [16])
         assert res == f(16)
 
+    def test_ptr_eq(self):
+        myjitdriver = JitDriver(greens = [], reds = ["n", "x"])
+        class A(object):
+            def __init__(self, v):
+                self.v = v
+        def f(n, x):
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, x=x)
+                z = 0 / x
+                a1 = A("key")
+                a2 = A("\x00")
+                n -= [a1, a2][z].v is not a2.v
+            return n
+        res = self.meta_interp(f, [10, 1])
+        assert res == 0
+
+    def test_instance_ptr_eq(self):
+        myjitdriver = JitDriver(greens = [], reds = ["n", "i", "a1", "a2"])
+        class A(object):
+            pass
+        def f(n):
+            a1 = A()
+            a2 = A()
+            i = 0
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, i=i, a1=a1, a2=a2)
+                if n % 2:
+                    a = a2
+                else:
+                    a = a1
+                i += a is a1
+                n -= 1
+            return i
+        res = self.meta_interp(f, [10])
+        assert res == f(10)
+        def f(n):
+            a1 = A()
+            a2 = A()
+            i = 0
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, i=i, a1=a1, a2=a2)
+                if n % 2:
+                    a = a2
+                else:
+                    a = a1
+                if a is a2:
+                    i += 1
+                n -= 1
+            return i
+        res = self.meta_interp(f, [10])
+        assert res == f(10)
+
+    def test_virtual_array_of_structs(self):
+        myjitdriver = JitDriver(greens = [], reds=["n", "d"])
+        def f(n):
+            d = None
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, d=d)
+                d = {}
+                if n % 2:
+                    d["k"] = n
+                else:
+                    d["z"] = n
+                n -= len(d)
+            return n
+        res = self.meta_interp(f, [10])
+        assert res == 0
+
     def test_tagged(self):
         from pypy.rlib.objectmodel import UnboxedValue
         class Base(object):
diff --git a/pypy/jit/metainterp/test/test_dict.py b/pypy/jit/metainterp/test/test_dict.py
--- a/pypy/jit/metainterp/test/test_dict.py
+++ b/pypy/jit/metainterp/test/test_dict.py
@@ -91,7 +91,7 @@
         res1 = f(100)
         res2 = self.meta_interp(f, [100], listops=True)
         assert res1 == res2
-        self.check_loops(int_mod=1) # the hash was traced
+        self.check_loops(int_mod=1) # the hash was traced and eq, but cached
 
     def test_dict_setdefault(self):
         myjitdriver = JitDriver(greens = [], reds = ['total', 'dct'])
@@ -128,7 +128,7 @@
         assert f(100) == 50
         res = self.meta_interp(f, [100], listops=True)
         assert res == 50
-        self.check_loops(int_mod=1)
+        self.check_loops(int_mod=1) # key + eq, but cached
 
     def test_repeated_lookup(self):
         myjitdriver = JitDriver(greens = [], reds = ['n', 'd'])
@@ -153,10 +153,12 @@
 
         res = self.meta_interp(f, [100], listops=True)
         assert res == f(50)
-        self.check_loops({"call": 7, "guard_false": 1, "guard_no_exception": 6,
+        self.check_loops({"call": 5, "getfield_gc": 1, "getinteriorfield_gc": 1,
+                          "guard_false": 1, "guard_no_exception": 4,
                           "guard_true": 1, "int_and": 1, "int_gt": 1,
                           "int_is_true": 1, "int_sub": 1, "jump": 1,
-                          "new_with_vtable": 1, "setfield_gc": 1})
+                          "new_with_vtable": 1, "new": 1, "new_array": 1,
+                          "setfield_gc": 3, })
 
 
 class TestOOtype(DictTests, OOJitMixin):
diff --git a/pypy/jit/metainterp/test/test_float.py b/pypy/jit/metainterp/test/test_float.py
--- a/pypy/jit/metainterp/test/test_float.py
+++ b/pypy/jit/metainterp/test/test_float.py
@@ -1,5 +1,6 @@
-import math
+import math, sys
 from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin
+from pypy.rlib.rarithmetic import intmask, r_uint
 
 
 class FloatTests:
@@ -45,6 +46,34 @@
         res = self.interp_operations(f, [-2.0])
         assert res == -8.5
 
+    def test_cast_float_to_int(self):
+        def g(f):
+            return int(f)
+        res = self.interp_operations(g, [-12345.9])
+        assert res == -12345
+
+    def test_cast_float_to_uint(self):
+        def g(f):
+            return intmask(r_uint(f))
+        res = self.interp_operations(g, [sys.maxint*2.0])
+        assert res == intmask(long(sys.maxint*2.0))
+        res = self.interp_operations(g, [-12345.9])
+        assert res == -12345
+
+    def test_cast_int_to_float(self):
+        def g(i):
+            return float(i)
+        res = self.interp_operations(g, [-12345])
+        assert type(res) is float and res == -12345.0
+
+    def test_cast_uint_to_float(self):
+        def g(i):
+            return float(r_uint(i))
+        res = self.interp_operations(g, [intmask(sys.maxint*2)])
+        assert type(res) is float and res == float(sys.maxint*2)
+        res = self.interp_operations(g, [-12345])
+        assert type(res) is float and res == float(long(r_uint(-12345)))
+
 
 class TestOOtype(FloatTests, OOJitMixin):
     pass
diff --git a/pypy/jit/metainterp/warmspot.py b/pypy/jit/metainterp/warmspot.py
--- a/pypy/jit/metainterp/warmspot.py
+++ b/pypy/jit/metainterp/warmspot.py
@@ -206,7 +206,7 @@
         self.make_enter_functions()
         self.rewrite_jit_merge_points(policy)
 
-        verbose = not self.cpu.translate_support_code
+        verbose = False # not self.cpu.translate_support_code
         self.codewriter.make_jitcodes(verbose=verbose)
         self.rewrite_can_enter_jits()
         self.rewrite_set_param()
diff --git a/pypy/module/__builtin__/app_io.py b/pypy/module/__builtin__/app_io.py
--- a/pypy/module/__builtin__/app_io.py
+++ b/pypy/module/__builtin__/app_io.py
@@ -71,7 +71,7 @@
         return line[:-1]
     return line
 
-def input(prompt=None):
+def input(prompt=''):
     """Equivalent to eval(raw_input(prompt))."""
     return eval(raw_input(prompt))
 
diff --git a/pypy/module/__builtin__/test/test_rawinput.py b/pypy/module/__builtin__/test/test_rawinput.py
--- a/pypy/module/__builtin__/test/test_rawinput.py
+++ b/pypy/module/__builtin__/test/test_rawinput.py
@@ -3,29 +3,32 @@
 
 class AppTestRawInput():
 
-    def test_raw_input(self):
+    def test_input_and_raw_input(self):
         import sys, StringIO
         for prompt, expected in [("def:", "abc/ def:/ghi\n"),
                                  ("", "abc/ /ghi\n"),
                                  (42, "abc/ 42/ghi\n"),
                                  (None, "abc/ None/ghi\n"),
                                  (Ellipsis, "abc/ /ghi\n")]:
-            save = sys.stdin, sys.stdout
-            try:
-                sys.stdin = StringIO.StringIO("foo\nbar\n")
-                out = sys.stdout = StringIO.StringIO()
-                print "abc",    # softspace = 1
-                out.write('/')
-                if prompt is Ellipsis:
-                    got = raw_input()
-                else:
-                    got = raw_input(prompt)
-                out.write('/')
-                print "ghi"
-            finally:
-                sys.stdin, sys.stdout = save
-            assert out.getvalue() == expected
-            assert got == "foo"
+            for inputfn, inputtext, gottext in [
+                    (raw_input, "foo\nbar\n", "foo"),
+                    (input, "40+2\n", 42)]:
+                save = sys.stdin, sys.stdout
+                try:
+                    sys.stdin = StringIO.StringIO(inputtext)
+                    out = sys.stdout = StringIO.StringIO()
+                    print "abc",    # softspace = 1
+                    out.write('/')
+                    if prompt is Ellipsis:
+                        got = inputfn()
+                    else:
+                        got = inputfn(prompt)
+                    out.write('/')
+                    print "ghi"
+                finally:
+                    sys.stdin, sys.stdout = save
+                assert out.getvalue() == expected
+                assert got == gottext
 
     def test_softspace(self):
         import sys
diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -19,7 +19,7 @@
 class W_RSocket(Wrappable, RSocket):
     def __del__(self):
         self.clear_all_weakrefs()
-        self.close()
+        RSocket.__del__(self)
 
     def accept_w(self, space):
         """accept() -> (socket object, address info)
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -211,7 +211,9 @@
             return result
 
         def __del__(self):
-            self.clear_all_weakrefs()
+            # note that we don't call clear_all_weakrefs here because
+            # an array with freed buffer is ok to see - it's just empty with 0
+            # length
             self.setlen(0)
 
         def setlen(self, size):
diff --git a/pypy/module/array/test/test_array.py b/pypy/module/array/test/test_array.py
--- a/pypy/module/array/test/test_array.py
+++ b/pypy/module/array/test/test_array.py
@@ -824,6 +824,22 @@
         r = weakref.ref(a)
         assert r() is a
 
+    def test_subclass_del(self):
+        import array, gc, weakref
+        l = []
+        
+        class A(array.array):
+            pass
+
+        a = A('d')
+        a.append(3.0)
+        r = weakref.ref(a, lambda a: l.append(a()))
+        del a
+        gc.collect()
+        assert l
+        assert l[0] is None or len(l[0]) == 0
+
+
 class TestCPythonsOwnArray(BaseArrayTests):
 
     def setup_class(cls):
@@ -844,11 +860,7 @@
         cls.w_tempfile = cls.space.wrap(
             str(py.test.ensuretemp('array').join('tmpfile')))
         cls.w_maxint = cls.space.wrap(sys.maxint)
-
-
-
-
-
+    
     def test_buffer_info(self):
         a = self.array('c', 'Hi!')
         bi = a.buffer_info()
diff --git a/pypy/module/pypyjit/policy.py b/pypy/module/pypyjit/policy.py
--- a/pypy/module/pypyjit/policy.py
+++ b/pypy/module/pypyjit/policy.py
@@ -16,7 +16,8 @@
         if modname in ['pypyjit', 'signal', 'micronumpy', 'math', 'exceptions',
                        'imp', 'sys', 'array', '_ffi', 'itertools', 'operator',
                        'posix', '_socket', '_sre', '_lsprof', '_weakref',
-                       '__pypy__', 'cStringIO', '_collections', 'struct']:
+                       '__pypy__', 'cStringIO', '_collections', 'struct',
+                       'mmap']:
             return True
         return False
 
diff --git a/pypy/module/pypyjit/test_pypy_c/test_containers.py b/pypy/module/pypyjit/test_pypy_c/test_containers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_containers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_containers.py
@@ -46,7 +46,7 @@
         assert loop.match_by_id("getitem", """
             i28 = call(ConstClass(ll_dict_lookup__dicttablePtr_objectPtr_Signed), p18, p6, i25, descr=...)
             ...
-            p33 = call(ConstClass(ll_get_value__dicttablePtr_Signed), p18, i28, descr=...)
+            p33 = getinteriorfield_gc(p31, i26, descr=<InteriorFieldDescr <GcPtrFieldDescr dictentry.value .*>>)
             ...
         """)
 
diff --git a/pypy/module/rctime/interp_time.py b/pypy/module/rctime/interp_time.py
--- a/pypy/module/rctime/interp_time.py
+++ b/pypy/module/rctime/interp_time.py
@@ -245,6 +245,9 @@
 if sys.platform != 'win32':
     @unwrap_spec(secs=float)
     def sleep(space, secs):
+        if secs < 0:
+            raise OperationError(space.w_IOError,
+                                 space.wrap("Invalid argument: negative time in sleep"))
         pytime.sleep(secs)
 else:
     from pypy.rlib import rwin32
@@ -265,6 +268,9 @@
                                    OSError(EINTR, "sleep() interrupted"))
     @unwrap_spec(secs=float)
     def sleep(space, secs):
+        if secs < 0:
+            raise OperationError(space.w_IOError,
+                                 space.wrap("Invalid argument: negative time in sleep"))
         # as decreed by Guido, only the main thread can be
         # interrupted.
         main_thread = space.fromcache(State).main_thread
diff --git a/pypy/module/rctime/test/test_rctime.py b/pypy/module/rctime/test/test_rctime.py
--- a/pypy/module/rctime/test/test_rctime.py
+++ b/pypy/module/rctime/test/test_rctime.py
@@ -20,8 +20,9 @@
         import sys
         import os
         raises(TypeError, rctime.sleep, "foo")
-        rctime.sleep(1.2345)
-        
+        rctime.sleep(0.12345)
+        raises(IOError, rctime.sleep, -1.0)
+
     def test_clock(self):
         import time as rctime
         rctime.clock()
diff --git a/pypy/objspace/std/newformat.py b/pypy/objspace/std/newformat.py
--- a/pypy/objspace/std/newformat.py
+++ b/pypy/objspace/std/newformat.py
@@ -120,6 +120,8 @@
             out.append_slice(s, last_literal, end)
             return out.build()
 
+        # This is only ever called if we're already unrolling _do_build_string
+        @jit.unroll_safe
         def _parse_field(self, start, end):
             s = self.template
             # Find ":" or "!"
@@ -149,6 +151,7 @@
                 i += 1
             return s[start:end], None, end
 
+        @jit.unroll_safe
         def _get_argument(self, name):
             # First, find the argument.
             space = self.space
@@ -207,6 +210,7 @@
                     raise OperationError(space.w_IndexError, w_msg)
             return self._resolve_lookups(w_arg, name, i, end)
 
+        @jit.unroll_safe
         def _resolve_lookups(self, w_obj, name, start, end):
             # Resolve attribute and item lookups.
             space = self.space
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -83,11 +83,12 @@
         if self.config.objspace.std.withtproxy:
             transparent.setup(self)
 
+        interplevel_classes = {}
         for type, classes in self.model.typeorder.iteritems():
-            if len(classes) >= 3:
+            if len(classes) >= 3: # XXX what does this 3 mean??!
                 # W_Root, AnyXxx and actual object
-                self.gettypefor(type).interplevel_cls = classes[0][0]
-
+                interplevel_classes[self.gettypefor(type)] = classes[0][0]
+        self._interplevel_classes = interplevel_classes
 
     def get_builtin_types(self):
         return self.builtin_types
@@ -579,7 +580,7 @@
             raise OperationError(self.w_TypeError,
                                  self.wrap("need type object"))
         if is_annotation_constant(w_type):
-            cls = w_type.interplevel_cls
+            cls = self._get_interplevel_cls(w_type)
             if cls is not None:
                 assert w_inst is not None
                 if isinstance(w_inst, cls):
@@ -589,3 +590,9 @@
     @specialize.arg_or_var(2)
     def isinstance_w(space, w_inst, w_type):
         return space._type_isinstance(w_inst, w_type)
+
+    @specialize.memo()
+    def _get_interplevel_cls(self, w_type):
+        if not hasattr(self, "_interplevel_classes"):
+            return None # before running initialize
+        return self._interplevel_classes.get(w_type, None)
diff --git a/pypy/objspace/std/smallintobject.py b/pypy/objspace/std/smallintobject.py
--- a/pypy/objspace/std/smallintobject.py
+++ b/pypy/objspace/std/smallintobject.py
@@ -12,6 +12,7 @@
 from pypy.rlib.rbigint import rbigint
 from pypy.rlib.rarithmetic import r_uint
 from pypy.tool.sourcetools import func_with_new_name
+from pypy.objspace.std.inttype import wrapint
 
 class W_SmallIntObject(W_Object, UnboxedValue):
     __slots__ = 'intval'
@@ -48,14 +49,36 @@
 def delegate_SmallInt2Complex(space, w_small):
     return space.newcomplex(float(w_small.intval), 0.0)
 
+def add__SmallInt_SmallInt(space, w_a, w_b):
+    return wrapint(space, w_a.intval + w_b.intval) # cannot overflow
+
+def sub__SmallInt_SmallInt(space, w_a, w_b):
+    return wrapint(space, w_a.intval - w_b.intval) # cannot overflow
+
+def floordiv__SmallInt_SmallInt(space, w_a, w_b):
+    return wrapint(space, w_a.intval // w_b.intval) # cannot overflow
+
+div__SmallInt_SmallInt = floordiv__SmallInt_SmallInt
+
+def mod__SmallInt_SmallInt(space, w_a, w_b):
+    return wrapint(space, w_a.intval % w_b.intval) # cannot overflow
+
+def divmod__SmallInt_SmallInt(space, w_a, w_b):
+    w = wrapint(space, w_a.intval // w_b.intval) # cannot overflow
+    z = wrapint(space, w_a.intval % w_b.intval)
+    return space.newtuple([w, z])
+
 def copy_multimethods(ns):
     """Copy integer multimethods for small int."""
     for name, func in intobject.__dict__.iteritems():
         if "__Int" in name:
             new_name = name.replace("Int", "SmallInt")
-            # Copy the function, so the annotator specializes it for
-            # W_SmallIntObject.
-            ns[new_name] = func_with_new_name(func, new_name)
+            if new_name not in ns:
+                # Copy the function, so the annotator specializes it for
+                # W_SmallIntObject.
+                ns[new_name] = func = func_with_new_name(func, new_name, globals=ns)
+        else:
+            ns[name] = func
     ns["get_integer"] = ns["pos__SmallInt"] = ns["int__SmallInt"]
     ns["get_negint"] = ns["neg__SmallInt"]
 
diff --git a/pypy/objspace/std/test/test_obj.py b/pypy/objspace/std/test/test_obj.py
--- a/pypy/objspace/std/test/test_obj.py
+++ b/pypy/objspace/std/test/test_obj.py
@@ -102,3 +102,11 @@
             def __repr__(self):
                 return 123456
         assert A().__str__() == 123456
+
+def test_isinstance_shortcut():
+    from pypy.objspace.std import objspace
+    space = objspace.StdObjSpace()
+    w_a = space.wrap("a")
+    space.type = None
+    space.isinstance_w(w_a, space.w_str) # does not crash
+
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -102,7 +102,6 @@
                           'instancetypedef',
                           'terminator',
                           '_version_tag?',
-                          'interplevel_cls',
                           ]
 
     # for config.objspace.std.getattributeshortcut
@@ -117,9 +116,6 @@
     # of the __new__ is an instance of the type
     w_bltin_new = None
 
-    interplevel_cls = None # not None for prebuilt instances of
-                           # interpreter-level types
-
     @dont_look_inside
     def __init__(w_self, space, name, bases_w, dict_w,
                  overridetypedef=None):
diff --git a/pypy/pytest.ini b/pypy/pytest.ini
--- a/pypy/pytest.ini
+++ b/pypy/pytest.ini
@@ -1,2 +1,2 @@
 [pytest]
-addopts = --assertmode=old
\ No newline at end of file
+addopts = --assertmode=old -rf
diff --git a/pypy/rlib/rgc.py b/pypy/rlib/rgc.py
--- a/pypy/rlib/rgc.py
+++ b/pypy/rlib/rgc.py
@@ -214,6 +214,10 @@
     func._gc_no_collect_ = True
     return func
 
+def is_light_finalizer(func):
+    func._is_light_finalizer_ = True
+    return func
+
 # ____________________________________________________________
 
 def get_rpy_roots():
diff --git a/pypy/rlib/rmmap.py b/pypy/rlib/rmmap.py
--- a/pypy/rlib/rmmap.py
+++ b/pypy/rlib/rmmap.py
@@ -292,6 +292,9 @@
         elif _POSIX:
             self.closed = True
             if self.fd != -1:
+                # XXX this is buggy - raising in an RPython del is not a good
+                #     idea, we should swallow the exception or ignore the
+                #     underlaying close error code
                 os.close(self.fd)
                 self.fd = -1
             if self.size > 0:
diff --git a/pypy/rlib/rsocket.py b/pypy/rlib/rsocket.py
--- a/pypy/rlib/rsocket.py
+++ b/pypy/rlib/rsocket.py
@@ -56,6 +56,7 @@
 
 
 _FAMILIES = {}
+
 class Address(object):
     """The base class for RPython-level objects representing addresses.
     Fields:  addr    - a _c.sockaddr_ptr (memory owned by the Address instance)
@@ -77,9 +78,8 @@
         self.addrlen = addrlen
 
     def __del__(self):
-        addr = self.addr_p
-        if addr:
-            lltype.free(addr, flavor='raw')
+        if self.addr_p:
+            lltype.free(self.addr_p, flavor='raw')
 
     def setdata(self, addr, addrlen):
         # initialize self.addr and self.addrlen.  'addr' can be a different
@@ -613,7 +613,10 @@
         self.timeout = defaults.timeout
         
     def __del__(self):
-        self.close()
+        fd = self.fd
+        if fd != _c.INVALID_SOCKET:
+            self.fd = _c.INVALID_SOCKET
+            _c.socketclose(fd)
 
     if hasattr(_c, 'fcntl'):
         def _setblocking(self, block):
diff --git a/pypy/rpython/llinterp.py b/pypy/rpython/llinterp.py
--- a/pypy/rpython/llinterp.py
+++ b/pypy/rpython/llinterp.py
@@ -1105,13 +1105,6 @@
         assert x & 1, "argument has to be tagged!"
         return x >> 1
 
-    def op_cast_float_to_int(self, f):
-        assert type(f) is float
-        try:
-            return ovfcheck(int(f))
-        except OverflowError:
-            self.make_llexception()
-
     def op_int_is_true(self, x):
         # special case
         if type(x) is CDefinedIntSymbolic:
diff --git a/pypy/rpython/lltypesystem/ll2ctypes.py b/pypy/rpython/lltypesystem/ll2ctypes.py
--- a/pypy/rpython/lltypesystem/ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/ll2ctypes.py
@@ -15,12 +15,11 @@
     load_library_kwargs = {}
 
 import os
-from pypy import conftest
 from pypy.rpython.lltypesystem import lltype, llmemory
 from pypy.rpython.extfunc import ExtRegistryEntry
 from pypy.rlib.objectmodel import Symbolic, ComputedIntSymbolic
 from pypy.tool.uid import fixid
-from pypy.rlib.rarithmetic import r_uint, r_singlefloat, r_longfloat, base_int, intmask
+from pypy.rlib.rarithmetic import r_singlefloat, r_longfloat, base_int, intmask
 from pypy.annotation import model as annmodel
 from pypy.rpython.llinterp import LLInterpreter, LLException
 from pypy.rpython.lltypesystem.rclass import OBJECT, OBJECT_VTABLE
@@ -531,6 +530,10 @@
     def __str__(self):
         return repr(self)
 
+    def _setparentstructure(self, parent, parentindex):
+        super(_parentable_mixin, self)._setparentstructure(parent, parentindex)
+        self._keepparent = parent   # always keep a strong ref
+
 class _struct_mixin(_parentable_mixin):
     """Mixin added to _struct containers when they become ctypes-based."""
     __slots__ = ()
@@ -566,7 +569,10 @@
         return 0, sys.maxint
 
     def getitem(self, index, uninitialized_ok=False):
-        return self._storage.contents._getitem(index, boundscheck=False)
+        res = self._storage.contents._getitem(index, boundscheck=False)
+        if isinstance(self._TYPE.OF, lltype.ContainerType):
+            res._obj._setparentstructure(self, index)
+        return res
 
     def setitem(self, index, value):
         self._storage.contents._setitem(index, value, boundscheck=False)
@@ -1279,6 +1285,8 @@
             self.intval = intmask(void_p.value)
 
     def __eq__(self, other):
+        if not other:
+            return self.intval == 0
         if isinstance(other, _llgcopaque):
             return self.intval == other.intval
         storage = object()
diff --git a/pypy/rpython/lltypesystem/lloperation.py b/pypy/rpython/lltypesystem/lloperation.py
--- a/pypy/rpython/lltypesystem/lloperation.py
+++ b/pypy/rpython/lltypesystem/lloperation.py
@@ -345,8 +345,8 @@
     'cast_uint_to_float':   LLOp(canfold=True),
     'cast_longlong_to_float' :LLOp(canfold=True),
     'cast_ulonglong_to_float':LLOp(canfold=True),
-    'cast_float_to_int':    LLOp(canraise=(OverflowError,), tryfold=True),
-    'cast_float_to_uint':   LLOp(canfold=True),    # XXX need OverflowError?
+    'cast_float_to_int':    LLOp(canfold=True),
+    'cast_float_to_uint':   LLOp(canfold=True),
     'cast_float_to_longlong' :LLOp(canfold=True),
     'cast_float_to_ulonglong':LLOp(canfold=True),
     'truncate_longlong_to_int':LLOp(canfold=True),
diff --git a/pypy/rpython/lltypesystem/lltype.py b/pypy/rpython/lltypesystem/lltype.py
--- a/pypy/rpython/lltypesystem/lltype.py
+++ b/pypy/rpython/lltypesystem/lltype.py
@@ -1522,7 +1522,7 @@
             and parentindex in (self._parent_type._names[0], 0)
             and self._TYPE._gckind == typeOf(parent)._gckind):
             # keep strong reference to parent, we share the same allocation
-            self._keepparent = parent 
+            self._keepparent = parent
 
     def _parentstructure(self, check=True):
         if self._wrparent is not None:
@@ -1713,6 +1713,7 @@
         return v
 
     def setitem(self, index, value):
+        assert typeOf(value) == self._TYPE.OF
         self.items[index] = value
 
 assert not '__dict__' in dir(_array)
@@ -1730,7 +1731,8 @@
         # Keep the parent array alive, we share the same allocation.
         # Don't do it if we are inside a GC object, though -- it's someone
         # else's job to keep the GC object alive
-        if typeOf(top_container(parent))._gckind == 'raw':
+        if (typeOf(top_container(parent))._gckind == 'raw' or
+            hasattr(top_container(parent)._storage, 'contents')):  # ll2ctypes
             self._keepparent = parent
 
     def __str__(self):
diff --git a/pypy/rpython/lltypesystem/opimpl.py b/pypy/rpython/lltypesystem/opimpl.py
--- a/pypy/rpython/lltypesystem/opimpl.py
+++ b/pypy/rpython/lltypesystem/opimpl.py
@@ -355,6 +355,10 @@
     assert type(b) is bool
     return float(b)
 
+def op_cast_float_to_int(f):
+    assert type(f) is float
+    return intmask(int(f))
+
 def op_cast_float_to_uint(f):
     assert type(f) is float
     return r_uint(long(f))
diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py
--- a/pypy/rpython/lltypesystem/rdict.py
+++ b/pypy/rpython/lltypesystem/rdict.py
@@ -1,16 +1,14 @@
 from pypy.tool.pairtype import pairtype
-from pypy.annotation import model as annmodel
 from pypy.objspace.flow.model import Constant
-from pypy.rpython.rdict import AbstractDictRepr, AbstractDictIteratorRepr,\
-     rtype_newdict
+from pypy.rpython.rdict import (AbstractDictRepr, AbstractDictIteratorRepr,
+     rtype_newdict)
 from pypy.rpython.lltypesystem import lltype
+from pypy.rlib import objectmodel, jit
 from pypy.rlib.rarithmetic import r_uint, intmask, LONG_BIT
-from pypy.rlib.objectmodel import hlinvoke
-from pypy.rpython import robject
-from pypy.rlib import objectmodel, jit
 from pypy.rpython import rmodel
 from pypy.rpython.error import TyperError
 
+
 HIGHEST_BIT = intmask(1 << (LONG_BIT - 1))
 MASK = intmask(HIGHEST_BIT - 1)
 
@@ -417,17 +415,16 @@
     ENTRIES = lltype.typeOf(entries).TO
     return ENTRIES.fasthashfn(entries[i].key)
 
- at jit.dont_look_inside
 def ll_get_value(d, i):
     return d.entries[i].value
 
 def ll_keyhash_custom(d, key):
     DICT = lltype.typeOf(d).TO
-    return hlinvoke(DICT.r_rdict_hashfn, d.fnkeyhash, key)
+    return objectmodel.hlinvoke(DICT.r_rdict_hashfn, d.fnkeyhash, key)
 
 def ll_keyeq_custom(d, key1, key2):
     DICT = lltype.typeOf(d).TO
-    return hlinvoke(DICT.r_rdict_eqfn, d.fnkeyeq, key1, key2)
+    return objectmodel.hlinvoke(DICT.r_rdict_eqfn, d.fnkeyeq, key1, key2)
 
 def ll_dict_len(d):
     return d.num_items
@@ -448,6 +445,8 @@
     i = ll_dict_lookup(d, key, hash)
     return _ll_dict_setitem_lookup_done(d, key, value, hash, i)
 
+# Leaving as dont_look_inside ATM, it has a few branches which could lead to
+# many bridges if we don't consider their possible frequency.
 @jit.dont_look_inside
 def _ll_dict_setitem_lookup_done(d, key, value, hash, i):
     valid = (i & HIGHEST_BIT) == 0
@@ -492,6 +491,8 @@
         raise KeyError
     _ll_dict_del(d, i)
 
+# XXX: Move the size checking and resize into a single call which is opauqe to
+# the JIT to avoid extra branches.
 @jit.dont_look_inside
 def _ll_dict_del(d, i):
     d.entries.mark_deleted(i)
@@ -532,6 +533,7 @@
 # ------- a port of CPython's dictobject.c's lookdict implementation -------
 PERTURB_SHIFT = 5
 
+ at jit.dont_look_inside
 def ll_dict_lookup(d, key, hash):
     entries = d.entries
     ENTRIES = lltype.typeOf(entries).TO
@@ -621,7 +623,6 @@
     d.num_items = 0
     d.resize_counter = DICT_INITSIZE * 2
     return d
-ll_newdict.oopspec = 'newdict()'
 
 def ll_newdict_size(DICT, length_estimate):
     length_estimate = (length_estimate // 2) * 3
@@ -633,7 +634,6 @@
     d.num_items = 0
     d.resize_counter = n * 2
     return d
-ll_newdict_size.oopspec = 'newdict()'
 
 # pypy.rpython.memory.lldict uses a dict based on Struct and Array
 # instead of GcStruct and GcArray, which is done by using different
@@ -866,7 +866,6 @@
         global_popitem_index.nextindex = base + counter
     return i
 
- at jit.dont_look_inside
 def ll_popitem(ELEM, dic):
     i = _ll_getnextitem(dic)
     entry = dic.entries[i]
diff --git a/pypy/rpython/lltypesystem/test/test_ll2ctypes.py b/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
--- a/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
@@ -8,6 +8,7 @@
 from pypy.rpython.lltypesystem.ll2ctypes import uninitialized2ctypes
 from pypy.rpython.lltypesystem.ll2ctypes import ALLOCATED, force_cast
 from pypy.rpython.lltypesystem.ll2ctypes import cast_adr_to_int, get_ctypes_type
+from pypy.rpython.lltypesystem.ll2ctypes import _llgcopaque
 from pypy.rpython.annlowlevel import llhelper
 from pypy.rlib import rposix
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
@@ -1349,6 +1350,11 @@
         round = ctypes2lltype(llmemory.GCREF, lltype2ctypes(opaque.hide()))
         assert Opaque.show(round) is opaque
 
+    def test_array_of_structs(self):
+        A = lltype.GcArray(lltype.Struct('x', ('v', lltype.Signed)))
+        a = lltype.malloc(A, 5)
+        a2 = ctypes2lltype(lltype.Ptr(A), lltype2ctypes(a))
+        assert a2._obj.getitem(0)._obj._parentstructure() is a2._obj
 
 class TestPlatform(object):
     def test_lib_on_libpaths(self):
@@ -1390,3 +1396,7 @@
         f = rffi.llexternal('f', [rffi.INT, rffi.INT], rffi.INT,
                             compilation_info=eci)
         assert f(3, 4) == 7
+
+    def test_llgcopaque_eq(self):
+        assert _llgcopaque(1) != None
+        assert _llgcopaque(0) == None
diff --git a/pypy/rpython/memory/gc/base.py b/pypy/rpython/memory/gc/base.py
--- a/pypy/rpython/memory/gc/base.py
+++ b/pypy/rpython/memory/gc/base.py
@@ -1,4 +1,5 @@
-from pypy.rpython.lltypesystem import lltype, llmemory, llarena
+from pypy.rpython.lltypesystem import lltype, llmemory, llarena, rffi
+from pypy.rpython.lltypesystem.lloperation import llop
 from pypy.rlib.debug import ll_assert
 from pypy.rpython.memory.gcheader import GCHeaderBuilder
 from pypy.rpython.memory.support import DEFAULT_CHUNK_SIZE
@@ -62,6 +63,7 @@
     def set_query_functions(self, is_varsize, has_gcptr_in_varsize,
                             is_gcarrayofgcptr,
                             getfinalizer,
+                            getlightfinalizer,
                             offsets_to_gc_pointers,
                             fixed_size, varsize_item_sizes,
                             varsize_offset_to_variable_part,
@@ -74,6 +76,7 @@
                             get_custom_trace,
                             fast_path_tracing):
         self.getfinalizer = getfinalizer
+        self.getlightfinalizer = getlightfinalizer
         self.is_varsize = is_varsize
         self.has_gcptr_in_varsize = has_gcptr_in_varsize
         self.is_gcarrayofgcptr = is_gcarrayofgcptr
@@ -139,6 +142,7 @@
 
         size = self.fixed_size(typeid)
         needs_finalizer = bool(self.getfinalizer(typeid))
+        finalizer_is_light = bool(self.getlightfinalizer(typeid))
         contains_weakptr = self.weakpointer_offset(typeid) >= 0
         assert not (needs_finalizer and contains_weakptr)
         if self.is_varsize(typeid):
@@ -158,6 +162,7 @@
             else:
                 malloc_fixedsize = self.malloc_fixedsize
             ref = malloc_fixedsize(typeid, size, needs_finalizer,
+                                   finalizer_is_light,
                                    contains_weakptr)
         # lots of cast and reverse-cast around...
         return llmemory.cast_ptr_to_adr(ref)
diff --git a/pypy/rpython/memory/gc/generation.py b/pypy/rpython/memory/gc/generation.py
--- a/pypy/rpython/memory/gc/generation.py
+++ b/pypy/rpython/memory/gc/generation.py
@@ -167,7 +167,9 @@
         return self.nursery <= addr < self.nursery_top
 
     def malloc_fixedsize_clear(self, typeid, size,
-                               has_finalizer=False, contains_weakptr=False):
+                               has_finalizer=False,
+                               is_finalizer_light=False,
+                               contains_weakptr=False):
         if (has_finalizer or
             (raw_malloc_usage(size) > self.lb_young_fixedsize and
              raw_malloc_usage(size) > self.largest_young_fixedsize)):
@@ -179,6 +181,7 @@
             # "non-simple" case or object too big: don't use the nursery
             return SemiSpaceGC.malloc_fixedsize_clear(self, typeid, size,
                                                       has_finalizer,
+                                                      is_finalizer_light,
                                                       contains_weakptr)
         size_gc_header = self.gcheaderbuilder.size_gc_header
         totalsize = size_gc_header + size
diff --git a/pypy/rpython/memory/gc/marksweep.py b/pypy/rpython/memory/gc/marksweep.py
--- a/pypy/rpython/memory/gc/marksweep.py
+++ b/pypy/rpython/memory/gc/marksweep.py
@@ -93,7 +93,8 @@
         pass
 
     def malloc_fixedsize(self, typeid16, size,
-                         has_finalizer=False, contains_weakptr=False):
+                         has_finalizer=False, is_finalizer_light=False,
+                         contains_weakptr=False):
         self.maybe_collect()
         size_gc_header = self.gcheaderbuilder.size_gc_header
         try:
@@ -128,7 +129,9 @@
     malloc_fixedsize._dont_inline_ = True
 
     def malloc_fixedsize_clear(self, typeid16, size,
-                               has_finalizer=False, contains_weakptr=False):
+                               has_finalizer=False,
+                               is_finalizer_light=False,
+                               contains_weakptr=False):
         self.maybe_collect()
         size_gc_header = self.gcheaderbuilder.size_gc_header
         try:
diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -290,6 +290,8 @@
         #
         # A list of all objects with finalizers (these are never young).
         self.objects_with_finalizers = self.AddressDeque()
+        self.young_objects_with_light_finalizers = self.AddressStack()
+        self.old_objects_with_light_finalizers = self.AddressStack()
         #
         # Two lists of the objects with weakrefs.  No weakref can be an
         # old object weakly pointing to a young object: indeed, weakrefs
@@ -457,14 +459,16 @@
 
 
     def malloc_fixedsize_clear(self, typeid, size,
-                               needs_finalizer=False, contains_weakptr=False):
+                               needs_finalizer=False,
+                               is_finalizer_light=False,
+                               contains_weakptr=False):
         size_gc_header = self.gcheaderbuilder.size_gc_header
         totalsize = size_gc_header + size
         rawtotalsize = raw_malloc_usage(totalsize)
         #
         # If the object needs a finalizer, ask for a rawmalloc.
         # The following check should be constant-folded.
-        if needs_finalizer:
+        if needs_finalizer and not is_finalizer_light:
             ll_assert(not contains_weakptr,
                      "'needs_finalizer' and 'contains_weakptr' both specified")
             obj = self.external_malloc(typeid, 0, can_make_young=False)
@@ -494,13 +498,14 @@
             #
             # Build the object.
             llarena.arena_reserve(result, totalsize)
+            obj = result + size_gc_header
+            if is_finalizer_light:
+                self.young_objects_with_light_finalizers.append(obj)
             self.init_gc_object(result, typeid, flags=0)
             #
             # If it is a weakref, record it (check constant-folded).
             if contains_weakptr:
-                self.young_objects_with_weakrefs.append(result+size_gc_header)
-            #
-            obj = result + size_gc_header
+                self.young_objects_with_weakrefs.append(obj)
         #
         return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
 
@@ -1264,6 +1269,8 @@
         # weakrefs' targets.
         if self.young_objects_with_weakrefs.non_empty():
             self.invalidate_young_weakrefs()
+        if self.young_objects_with_light_finalizers.non_empty():
+            self.deal_with_young_objects_with_finalizers()
         #
         # Clear this mapping.
         if self.nursery_objects_shadows.length() > 0:
@@ -1584,6 +1591,9 @@
         # Weakref support: clear the weak pointers to dying objects
         if self.old_objects_with_weakrefs.non_empty():
             self.invalidate_old_weakrefs()
+        if self.old_objects_with_light_finalizers.non_empty():
+            self.deal_with_old_objects_with_finalizers()
+
         #
         # Walk all rawmalloced objects and free the ones that don't
         # have the GCFLAG_VISITED flag.
@@ -1649,8 +1659,7 @@
         if self.header(obj).tid & GCFLAG_VISITED:
             self.header(obj).tid &= ~GCFLAG_VISITED
             return False     # survives
-        else:
-            return True      # dies
+        return True      # dies
 
     def _reset_gcflag_visited(self, obj, ignored):
         self.header(obj).tid &= ~GCFLAG_VISITED
@@ -1829,6 +1838,39 @@
     # ----------
     # Finalizers
 
+    def deal_with_young_objects_with_finalizers(self):
+        """ This is a much simpler version of dealing with finalizers
+        and an optimization - we can reasonably assume that those finalizers
+        don't do anything fancy and *just* call them. Among other things
+        they won't resurrect objects
+        """
+        while self.young_objects_with_light_finalizers.non_empty():
+            obj = self.young_objects_with_light_finalizers.pop()
+            if not self.is_forwarded(obj):
+                finalizer = self.getlightfinalizer(self.get_type_id(obj))
+                ll_assert(bool(finalizer), "no light finalizer found")
+                finalizer(obj, llmemory.NULL)
+
+    def deal_with_old_objects_with_finalizers(self):
+        """ This is a much simpler version of dealing with finalizers
+        and an optimization - we can reasonably assume that those finalizers
+        don't do anything fancy and *just* call them. Among other things
+        they won't resurrect objects
+        """
+        new_objects = self.AddressStack()
+        while self.old_objects_with_light_finalizers.non_empty():
+            obj = self.old_objects_with_light_finalizers.pop()
+            if self.header(obj).tid & GCFLAG_VISITED:
+                # surviving
+                new_objects.append(obj)
+            else:
+                # dying
+                finalizer = self.getlightfinalizer(self.get_type_id(obj))
+                ll_assert(bool(finalizer), "no light finalizer found")
+                finalizer(obj, llmemory.NULL)
+        self.old_objects_with_light_finalizers.delete()
+        self.old_objects_with_light_finalizers = new_objects
+
     def deal_with_objects_with_finalizers(self):
         # Walk over list of objects with finalizers.
         # If it is not surviving, add it to the list of to-be-called
@@ -1959,7 +2001,6 @@
             #
             self.old_objects_with_weakrefs.append(obj)
 
-
     def invalidate_old_weakrefs(self):
         """Called during a major collection."""
         # walk over list of objects that contain weakrefs
diff --git a/pypy/rpython/memory/gc/semispace.py b/pypy/rpython/memory/gc/semispace.py
--- a/pypy/rpython/memory/gc/semispace.py
+++ b/pypy/rpython/memory/gc/semispace.py
@@ -82,6 +82,7 @@
         self.free = self.tospace
         MovingGCBase.setup(self)
         self.objects_with_finalizers = self.AddressDeque()
+        self.objects_with_light_finalizers = self.AddressStack()
         self.objects_with_weakrefs = self.AddressStack()
 
     def _teardown(self):
@@ -93,7 +94,9 @@
     # because the spaces are filled with zeroes in advance.
 
     def malloc_fixedsize_clear(self, typeid16, size,
-                               has_finalizer=False, contains_weakptr=False):
+                               has_finalizer=False,
+                               is_finalizer_light=False,
+                               contains_weakptr=False):
         size_gc_header = self.gcheaderbuilder.size_gc_header
         totalsize = size_gc_header + size
         result = self.free
@@ -102,7 +105,9 @@
         llarena.arena_reserve(result, totalsize)
         self.init_gc_object(result, typeid16)
         self.free = result + totalsize
-        if has_finalizer:
+        if is_finalizer_light:
+            self.objects_with_light_finalizers.append(result + size_gc_header)
+        elif has_finalizer:
             self.objects_with_finalizers.append(result + size_gc_header)
         if contains_weakptr:
             self.objects_with_weakrefs.append(result + size_gc_header)
@@ -263,6 +268,8 @@
         if self.run_finalizers.non_empty():
             self.update_run_finalizers()
         scan = self.scan_copied(scan)
+        if self.objects_with_light_finalizers.non_empty():
+            self.deal_with_objects_with_light_finalizers()
         if self.objects_with_finalizers.non_empty():
             scan = self.deal_with_objects_with_finalizers(scan)
         if self.objects_with_weakrefs.non_empty():
@@ -471,6 +478,23 @@
         # immortal objects always have GCFLAG_FORWARDED set;
         # see get_forwarding_address().
 
+    def deal_with_objects_with_light_finalizers(self):
+        """ This is a much simpler version of dealing with finalizers
+        and an optimization - we can reasonably assume that those finalizers
+        don't do anything fancy and *just* call them. Among other things
+        they won't resurrect objects
+        """
+        new_objects = self.AddressStack()
+        while self.objects_with_light_finalizers.non_empty():
+            obj = self.objects_with_light_finalizers.pop()
+            if self.surviving(obj):
+                new_objects.append(self.get_forwarding_address(obj))
+            else:
+                finalizer = self.getfinalizer(self.get_type_id(obj))
+                finalizer(obj, llmemory.NULL)
+        self.objects_with_light_finalizers.delete()
+        self.objects_with_light_finalizers = new_objects
+
     def deal_with_objects_with_finalizers(self, scan):
         # walk over list of objects with finalizers
         # if it is not copied, add it to the list of to-be-called finalizers
diff --git a/pypy/rpython/memory/gctransform/framework.py b/pypy/rpython/memory/gctransform/framework.py
--- a/pypy/rpython/memory/gctransform/framework.py
+++ b/pypy/rpython/memory/gctransform/framework.py
@@ -12,6 +12,7 @@
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.translator.backendopt import graphanalyze
 from pypy.translator.backendopt.support import var_needsgc
+from pypy.translator.backendopt.finalizer import FinalizerAnalyzer
 from pypy.annotation import model as annmodel
 from pypy.rpython import annlowlevel
 from pypy.rpython.rbuiltin import gen_cast
@@ -258,6 +259,7 @@
             [s_gc, s_typeid16,
              annmodel.SomeInteger(nonneg=True),
              annmodel.SomeBool(),
+             annmodel.SomeBool(),
              annmodel.SomeBool()], s_gcref,
             inline = False)
         if hasattr(GCClass, 'malloc_fixedsize'):
@@ -267,6 +269,7 @@
                 [s_gc, s_typeid16,
                  annmodel.SomeInteger(nonneg=True),
                  annmodel.SomeBool(),
+                 annmodel.SomeBool(),
                  annmodel.SomeBool()], s_gcref,
                 inline = False)
         else:
@@ -319,7 +322,7 @@
             raise NotImplementedError("GC needs write barrier, but does not provide writebarrier_before_copy functionality")
 
         # in some GCs we can inline the common case of
-        # malloc_fixedsize(typeid, size, True, False, False)
+        # malloc_fixedsize(typeid, size, False, False, False)
         if getattr(GCClass, 'inline_simple_malloc', False):
             # make a copy of this function so that it gets annotated
             # independently and the constants are folded inside
@@ -337,7 +340,7 @@
                 malloc_fast,
                 [s_gc, s_typeid16,
                  annmodel.SomeInteger(nonneg=True),
-                 s_False, s_False], s_gcref,
+                 s_False, s_False, s_False], s_gcref,
                 inline = True)
         else:
             self.malloc_fast_ptr = None
@@ -668,7 +671,13 @@
         kind_and_fptr = self.special_funcptr_for_type(TYPE)
         has_finalizer = (kind_and_fptr is not None and
                          kind_and_fptr[0] == "finalizer")
+        has_light_finalizer = (kind_and_fptr is not None and
+                               kind_and_fptr[0] == "light_finalizer")
+        if has_light_finalizer:
+            has_finalizer = True
         c_has_finalizer = rmodel.inputconst(lltype.Bool, has_finalizer)
+        c_has_light_finalizer = rmodel.inputconst(lltype.Bool,
+                                                  has_light_finalizer)
 
         if not op.opname.endswith('_varsize') and not flags.get('varsize'):
             #malloc_ptr = self.malloc_fixedsize_ptr
@@ -682,7 +691,8 @@
             else:
                 malloc_ptr = self.malloc_fixedsize_ptr
             args = [self.c_const_gc, c_type_id, c_size,
-                    c_has_finalizer, rmodel.inputconst(lltype.Bool, False)]
+                    c_has_finalizer, c_has_light_finalizer,
+                    rmodel.inputconst(lltype.Bool, False)]
         else:
             assert not c_has_finalizer.value
             info_varsize = self.layoutbuilder.get_info_varsize(type_id)
@@ -847,12 +857,13 @@
         # used by the JIT (see pypy.jit.backend.llsupport.gc)
         op = hop.spaceop
         [v_typeid, v_size,
-         v_has_finalizer, v_contains_weakptr] = op.args
+         v_has_finalizer, v_has_light_finalizer, v_contains_weakptr] = op.args
         livevars = self.push_roots(hop)
         hop.genop("direct_call",
                   [self.malloc_fixedsize_clear_ptr, self.c_const_gc,
                    v_typeid, v_size,
-                   v_has_finalizer, v_contains_weakptr],
+                   v_has_finalizer, v_has_light_finalizer,
+                   v_contains_weakptr],
                   resultvar=op.result)
         self.pop_roots(hop, livevars)
 
@@ -912,10 +923,10 @@
         info = self.layoutbuilder.get_info(type_id)
         c_size = rmodel.inputconst(lltype.Signed, info.fixedsize)
         malloc_ptr = self.malloc_fixedsize_ptr
-        c_has_finalizer = rmodel.inputconst(lltype.Bool, False)
+        c_false = rmodel.inputconst(lltype.Bool, False)
         c_has_weakptr = rmodel.inputconst(lltype.Bool, True)
         args = [self.c_const_gc, c_type_id, c_size,
-                c_has_finalizer, c_has_weakptr]
+                c_false, c_false, c_has_weakptr]
 
         # push and pop the current live variables *including* the argument
         # to the weakref_create operation, which must be kept alive and
@@ -1250,6 +1261,7 @@
             lltype2vtable = translator.rtyper.lltype2vtable
         else:
             lltype2vtable = None
+        self.translator = translator
         super(TransformerLayoutBuilder, self).__init__(GCClass, lltype2vtable)
 
     def has_finalizer(self, TYPE):
@@ -1257,6 +1269,10 @@
         return rtti is not None and getattr(rtti._obj, 'destructor_funcptr',
                                             None)
 
+    def has_light_finalizer(self, TYPE):
+        special = self.special_funcptr_for_type(TYPE)
+        return special is not None and special[0] == 'light_finalizer'
+
     def has_custom_trace(self, TYPE):
         rtti = get_rtti(TYPE)
         return rtti is not None and getattr(rtti._obj, 'custom_trace_funcptr',
@@ -1264,7 +1280,7 @@
 
     def make_finalizer_funcptr_for_type(self, TYPE):
         if not self.has_finalizer(TYPE):
-            return None
+            return None, False
         rtti = get_rtti(TYPE)
         destrptr = rtti._obj.destructor_funcptr
         DESTR_ARG = lltype.typeOf(destrptr).TO.ARGS[0]
@@ -1276,7 +1292,9 @@
             return llmemory.NULL
         fptr = self.transformer.annotate_finalizer(ll_finalizer,
                 [llmemory.Address, llmemory.Address], llmemory.Address)
-        return fptr
+        g = destrptr._obj.graph
+        light = not FinalizerAnalyzer(self.translator).analyze_light_finalizer(g)
+        return fptr, light
 
     def make_custom_trace_funcptr_for_type(self, TYPE):
         if not self.has_custom_trace(TYPE):
diff --git a/pypy/rpython/memory/gctypelayout.py b/pypy/rpython/memory/gctypelayout.py
--- a/pypy/rpython/memory/gctypelayout.py
+++ b/pypy/rpython/memory/gctypelayout.py
@@ -1,7 +1,6 @@
 from pypy.rpython.lltypesystem import lltype, llmemory, llarena, llgroup
 from pypy.rpython.lltypesystem import rclass
 from pypy.rpython.lltypesystem.lloperation import llop
-from pypy.rlib.objectmodel import we_are_translated
 from pypy.rlib.debug import ll_assert
 from pypy.rlib.rarithmetic import intmask
 from pypy.tool.identity_dict import identity_dict
@@ -85,6 +84,13 @@
         else:
             return lltype.nullptr(GCData.FINALIZER_OR_CT_FUNC)
 
+    def q_light_finalizer(self, typeid):
+        typeinfo = self.get(typeid)
+        if typeinfo.infobits & T_HAS_LIGHTWEIGHT_FINALIZER:
+            return typeinfo.finalizer_or_customtrace
+        else:
+            return lltype.nullptr(GCData.FINALIZER_OR_CT_FUNC)        
+
     def q_offsets_to_gc_pointers(self, typeid):
         return self.get(typeid).ofstoptrs
 
@@ -142,6 +148,7 @@
             self.q_has_gcptr_in_varsize,
             self.q_is_gcarrayofgcptr,
             self.q_finalizer,
+            self.q_light_finalizer,
             self.q_offsets_to_gc_pointers,
             self.q_fixed_size,
             self.q_varsize_item_sizes,
@@ -157,16 +164,17 @@
 
 
 # the lowest 16bits are used to store group member index
-T_MEMBER_INDEX         =   0xffff
-T_IS_VARSIZE           = 0x010000
-T_HAS_GCPTR_IN_VARSIZE = 0x020000
-T_IS_GCARRAY_OF_GCPTR  = 0x040000
-T_IS_WEAKREF           = 0x080000
-T_IS_RPYTHON_INSTANCE  = 0x100000    # the type is a subclass of OBJECT
-T_HAS_FINALIZER        = 0x200000
-T_HAS_CUSTOM_TRACE     = 0x400000
-T_KEY_MASK             = intmask(0xFF000000)
-T_KEY_VALUE            = intmask(0x5A000000)    # bug detection only
+T_MEMBER_INDEX              =   0xffff
+T_IS_VARSIZE                = 0x010000
+T_HAS_GCPTR_IN_VARSIZE      = 0x020000
+T_IS_GCARRAY_OF_GCPTR       = 0x040000
+T_IS_WEAKREF                = 0x080000
+T_IS_RPYTHON_INSTANCE       = 0x100000 # the type is a subclass of OBJECT
+T_HAS_FINALIZER             = 0x200000
+T_HAS_CUSTOM_TRACE          = 0x400000
+T_HAS_LIGHTWEIGHT_FINALIZER = 0x800000
+T_KEY_MASK                  = intmask(0xFF000000)
+T_KEY_VALUE                 = intmask(0x5A000000) # bug detection only
 
 def _check_valid_type_info(p):
     ll_assert(p.infobits & T_KEY_MASK == T_KEY_VALUE, "invalid type_id")
@@ -194,6 +202,8 @@
         info.finalizer_or_customtrace = fptr
         if kind == "finalizer":
             infobits |= T_HAS_FINALIZER
+        elif kind == 'light_finalizer':
+            infobits |= T_HAS_FINALIZER | T_HAS_LIGHTWEIGHT_FINALIZER
         elif kind == "custom_trace":
             infobits |= T_HAS_CUSTOM_TRACE
         else:
@@ -367,12 +377,15 @@
     def special_funcptr_for_type(self, TYPE):
         if TYPE in self._special_funcptrs:
             return self._special_funcptrs[TYPE]
-        fptr1 = self.make_finalizer_funcptr_for_type(TYPE)
+        fptr1, is_lightweight = self.make_finalizer_funcptr_for_type(TYPE)
         fptr2 = self.make_custom_trace_funcptr_for_type(TYPE)
         assert not (fptr1 and fptr2), (
             "type %r needs both a finalizer and a custom tracer" % (TYPE,))
         if fptr1:
-            kind_and_fptr = "finalizer", fptr1
+            if is_lightweight:
+                kind_and_fptr = "light_finalizer", fptr1
+            else:
+                kind_and_fptr = "finalizer", fptr1
         elif fptr2:
             kind_and_fptr = "custom_trace", fptr2
         else:
@@ -382,7 +395,7 @@
 
     def make_finalizer_funcptr_for_type(self, TYPE):
         # must be overridden for proper finalizer support
-        return None
+        return None, False
 
     def make_custom_trace_funcptr_for_type(self, TYPE):
         # must be overridden for proper custom tracer support
diff --git a/pypy/rpython/memory/gcwrapper.py b/pypy/rpython/memory/gcwrapper.py
--- a/pypy/rpython/memory/gcwrapper.py
+++ b/pypy/rpython/memory/gcwrapper.py
@@ -1,3 +1,4 @@
+from pypy.translator.backendopt.finalizer import FinalizerAnalyzer
 from pypy.rpython.lltypesystem import lltype, llmemory, llheap
 from pypy.rpython import llinterp
 from pypy.rpython.annlowlevel import llhelper
@@ -196,9 +197,11 @@
             DESTR_ARG = lltype.typeOf(destrptr).TO.ARGS[0]
             destrgraph = destrptr._obj.graph
         else:
-            return None
+            return None, False
 
         assert not type_contains_pyobjs(TYPE), "not implemented"
+        t = self.llinterp.typer.annotator.translator
+        light = not FinalizerAnalyzer(t).analyze_light_finalizer(destrgraph)
         def ll_finalizer(addr, dummy):
             assert dummy == llmemory.NULL
             try:
@@ -208,7 +211,7 @@
                 raise RuntimeError(
                     "a finalizer raised an exception, shouldn't happen")
             return llmemory.NULL
-        return llhelper(gctypelayout.GCData.FINALIZER_OR_CT, ll_finalizer)
+        return llhelper(gctypelayout.GCData.FINALIZER_OR_CT, ll_finalizer), light
 
     def make_custom_trace_funcptr_for_type(self, TYPE):
         from pypy.rpython.memory.gctransform.support import get_rtti, \
diff --git a/pypy/rpython/memory/test/test_gc.py b/pypy/rpython/memory/test/test_gc.py
--- a/pypy/rpython/memory/test/test_gc.py
+++ b/pypy/rpython/memory/test/test_gc.py
@@ -5,7 +5,6 @@
 from pypy.rpython.memory.test import snippet
 from pypy.rpython.test.test_llinterp import get_interpreter
 from pypy.rpython.lltypesystem import lltype
-from pypy.rpython.lltypesystem.rstr import STR
 from pypy.rpython.lltypesystem.lloperation import llop
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.rlib.objectmodel import compute_unique_id
@@ -57,7 +56,7 @@
                 while j < 20:
                     j += 1
                     a.append(j)
-        res = self.interpret(malloc_a_lot, [])
+        self.interpret(malloc_a_lot, [])
         #assert simulator.current_size - curr < 16000 * INT_SIZE / 4
         #print "size before: %s, size after %s" % (curr, simulator.current_size)
 
@@ -73,7 +72,7 @@
                 while j < 20:
                     j += 1
                     b.append((1, j, i))
-        res = self.interpret(malloc_a_lot, [])
+        self.interpret(malloc_a_lot, [])
         #assert simulator.current_size - curr < 16000 * INT_SIZE / 4
         #print "size before: %s, size after %s" % (curr, simulator.current_size)
 
@@ -129,7 +128,7 @@
         res = self.interpret(concat, [100])
         assert res == concat(100)
         #assert simulator.current_size - curr < 16000 * INT_SIZE / 4
-        
+
     def test_finalizer(self):
         class B(object):
             pass
@@ -278,7 +277,7 @@
                                self.interpret, f, [])
 
     def test_weakref(self):
-        import weakref, gc
+        import weakref
         class A(object):
             pass
         def g():
@@ -299,7 +298,7 @@
         assert res
 
     def test_weakref_to_object_with_finalizer(self):
-        import weakref, gc
+        import weakref
         class A(object):
             count = 0
         a = A()
@@ -338,7 +337,7 @@
         assert res
 
     def test_cycle_with_weakref_and_del(self):
-        import weakref, gc
+        import weakref
         class A(object):
             count = 0
         a = A()
@@ -367,7 +366,7 @@
         assert res == 11
 
     def test_weakref_to_object_with_finalizer_ordering(self):
-        import weakref, gc
+        import weakref
         class A(object):
             count = 0
         a = A()
@@ -616,7 +615,7 @@
                     assert not rgc.can_move(a)
                     return 1
                 return 0
-            except Exception, e:
+            except Exception:
                 return 2
 
         assert self.interpret(func, []) == int(self.GC_CAN_MALLOC_NONMOVABLE)
@@ -647,8 +646,6 @@
         assert self.interpret(f, [bigsize, 0, flag]) == 0x62024241
 
     def test_tagged_simple(self):
-        from pypy.rlib.objectmodel import UnboxedValue
-
         class Unrelated(object):
             pass
 
@@ -689,8 +686,6 @@
         assert res == -897
 
     def test_tagged_id(self):
-        from pypy.rlib.objectmodel import UnboxedValue, compute_unique_id
-
         class Unrelated(object):
             pass
 
diff --git a/pypy/rpython/memory/test/test_transformed_gc.py b/pypy/rpython/memory/test/test_transformed_gc.py
--- a/pypy/rpython/memory/test/test_transformed_gc.py
+++ b/pypy/rpython/memory/test/test_transformed_gc.py
@@ -345,22 +345,22 @@
         b = B()
         b.nextid = 0
         b.num_deleted = 0
-        class A(object):
+        class AAA(object):
             def __init__(self):
                 self.id = b.nextid
                 b.nextid += 1
             def __del__(self):
                 b.num_deleted += 1
                 C()
-        class C(A):
+        class C(AAA):
             def __del__(self):
                 b.num_deleted += 1
         def f(x, y):
-            a = A()
+            a = AAA()
             i = 0
             while i < x:
                 i += 1
-                a = A()
+                a = AAA()
             llop.gc__collect(lltype.Void)
             llop.gc__collect(lltype.Void)
             return b.num_deleted
@@ -807,6 +807,7 @@
                     op.args = [Constant(type_id, llgroup.HALFWORD),
                                Constant(llmemory.sizeof(P), lltype.Signed),
                                Constant(False, lltype.Bool), # has_finalizer
+                               Constant(False, lltype.Bool), # is_finalizer_light
                                Constant(False, lltype.Bool)] # contains_weakptr
                     break
             else:
@@ -843,6 +844,7 @@
                     op.args = [Constant(type_id, llgroup.HALFWORD),
                                Constant(llmemory.sizeof(P), lltype.Signed),
                                Constant(False, lltype.Bool), # has_finalizer
+                               Constant(False, lltype.Bool), # is_finalizer_light
                                Constant(False, lltype.Bool)] # contains_weakptr
                     break
             else:
diff --git a/pypy/rpython/module/ll_os.py b/pypy/rpython/module/ll_os.py
--- a/pypy/rpython/module/ll_os.py
+++ b/pypy/rpython/module/ll_os.py
@@ -959,8 +959,6 @@
                             os_ftruncate(rffi.cast(rffi.INT, fd),
                                          rffi.cast(rffi.LONGLONG, length)))
             if res < 0:
-                # Note: for consistency we raise OSError, but CPython
-                # raises IOError here
                 raise OSError(rposix.get_errno(), "os_ftruncate failed")
 
         return extdef([int, r_longlong], s_None,
diff --git a/pypy/rpython/rtyper.py b/pypy/rpython/rtyper.py
--- a/pypy/rpython/rtyper.py
+++ b/pypy/rpython/rtyper.py
@@ -717,7 +717,7 @@
             raise TyperError("runtime type info function %r returns %r, "
                              "excepted Ptr(RuntimeTypeInfo)" % (func, s))
         funcptr = self.getcallable(graph)
-        attachRuntimeTypeInfo(GCSTRUCT, funcptr, destrptr)
+        attachRuntimeTypeInfo(GCSTRUCT, funcptr, destrptr, None)
 
 # register operations from annotation model
 RPythonTyper._registeroperations(annmodel)
diff --git a/pypy/rpython/test/test_rclass.py b/pypy/rpython/test/test_rclass.py
--- a/pypy/rpython/test/test_rclass.py
+++ b/pypy/rpython/test/test_rclass.py
@@ -3,7 +3,7 @@
 from pypy.translator.translator import TranslationContext, graphof
 from pypy.rpython.lltypesystem.lltype import *
 from pypy.rpython.ootypesystem import ootype
-from pypy.rlib.rarithmetic import intmask, r_longlong
+from pypy.rlib.rarithmetic import r_longlong
 from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin
 from pypy.rpython.rclass import IR_IMMUTABLE, IR_IMMUTABLE_ARRAY
 from pypy.rpython.rclass import IR_QUASIIMMUTABLE, IR_QUASIIMMUTABLE_ARRAY
@@ -972,10 +972,10 @@
         graph = graphof(t, f)
         TYPE = graph.startblock.operations[0].args[0].value
         RTTI = getRuntimeTypeInfo(TYPE)
-        queryptr = RTTI._obj.query_funcptr # should not raise
+        RTTI._obj.query_funcptr # should not raise
         destrptr = RTTI._obj.destructor_funcptr
         assert destrptr is not None
-    
+
     def test_del_inheritance(self):
         from pypy.rlib import rgc
         class State:
diff --git a/pypy/tool/sourcetools.py b/pypy/tool/sourcetools.py
--- a/pypy/tool/sourcetools.py
+++ b/pypy/tool/sourcetools.py
@@ -216,9 +216,11 @@
 
 # ____________________________________________________________
 
-def func_with_new_name(func, newname):
+def func_with_new_name(func, newname, globals=None):
     """Make a renamed copy of a function."""
-    f = new.function(func.func_code, func.func_globals,
+    if globals is None:
+        globals = func.func_globals
+    f = new.function(func.func_code, globals,
                         newname, func.func_defaults,
                         func.func_closure)
     if func.func_dict:
diff --git a/pypy/translator/backendopt/finalizer.py b/pypy/translator/backendopt/finalizer.py
new file mode 100644
--- /dev/null
+++ b/pypy/translator/backendopt/finalizer.py
@@ -0,0 +1,46 @@
+
+from pypy.translator.backendopt import graphanalyze
+from pypy.rpython.lltypesystem import lltype
+
+class FinalizerError(Exception):
+    """ __del__ marked as lightweight finalizer, but the analyzer did
+    not agreed
+    """
+
+class FinalizerAnalyzer(graphanalyze.BoolGraphAnalyzer):
+    """ Analyzer that determines whether a finalizer is lightweight enough
+    so it can be called without all the complicated logic in the garbage
+    collector. The set of operations here is restrictive for a good reason
+    - it's better to be safe. Specifically disallowed operations:
+
+    * anything that escapes self
+    * anything that can allocate
+    """
+    ok_operations = ['ptr_nonzero', 'ptr_eq', 'ptr_ne', 'free', 'same_as',
+                     'direct_ptradd', 'force_cast', 'track_alloc_stop',
+                     'raw_free']
+    
+    def analyze_light_finalizer(self, graph):
+        result = self.analyze_direct_call(graph)
+        if (result is self.top_result() and
+            getattr(graph.func, '_is_light_finalizer_', False)):
+            raise FinalizerError(FinalizerError.__doc__, graph)
+        return result
+    
+    def analyze_simple_operation(self, op, graphinfo):
+        if op.opname in self.ok_operations:
+            return self.bottom_result()
+        if (op.opname.startswith('int_') or op.opname.startswith('float_')
+            or op.opname.startswith('cast_')):
+            return self.bottom_result()
+        if op.opname == 'setfield' or op.opname == 'bare_setfield':
+            TP = op.args[2].concretetype
+            if not isinstance(TP, lltype.Ptr) or TP.TO._gckind == 'raw':
+                # primitive type
+                return self.bottom_result()
+        if op.opname == 'getfield':
+            TP = op.result.concretetype
+            if not isinstance(TP, lltype.Ptr) or TP.TO._gckind == 'raw':
+                # primitive type
+                return self.bottom_result()            
+        return self.top_result()
diff --git a/pypy/translator/backendopt/test/test_finalizer.py b/pypy/translator/backendopt/test/test_finalizer.py
new file mode 100644
--- /dev/null
+++ b/pypy/translator/backendopt/test/test_finalizer.py
@@ -0,0 +1,142 @@
+
+import py
+from pypy.translator.backendopt.finalizer import FinalizerAnalyzer,\
+     FinalizerError
+from pypy.translator.translator import TranslationContext, graphof
+from pypy.translator.backendopt.all import backend_optimizations
+from pypy.translator.unsimplify import varoftype
+from pypy.rpython.lltypesystem import lltype, rffi
+from pypy.conftest import option
+from pypy.rlib import rgc
+
+
+class BaseFinalizerAnalyzerTests(object):
+    """ Below are typical destructors that we encounter in pypy
+    """
+
+    type_system = None
+    
+    def analyze(self, func, sig, func_to_analyze=None, backendopt=False):
+        if func_to_analyze is None:
+            func_to_analyze = func
+        t = TranslationContext()
+        t.buildannotator().build_types(func, sig)
+        t.buildrtyper(type_system=self.type_system).specialize()
+        if backendopt:
+            backend_optimizations(t)
+        if option.view:
+            t.view()
+        a = FinalizerAnalyzer(t)
+        fgraph = graphof(t, func_to_analyze)
+        result = a.analyze_light_finalizer(fgraph)
+        return result
+
+    def test_nothing(self):
+        def f():
+            pass
+        r = self.analyze(f, [])
+        assert not r
+
+def test_various_ops():
+    from pypy.objspace.flow.model import SpaceOperation, Constant
+
+    X = lltype.Ptr(lltype.GcStruct('X'))
+    Z = lltype.Ptr(lltype.Struct('Z'))
+    S = lltype.GcStruct('S', ('x', lltype.Signed),
+                        ('y', X),
+                        ('z', Z))
+    v1 = varoftype(lltype.Bool)
+    v2 = varoftype(lltype.Signed)
+    f = FinalizerAnalyzer(None)
+    r = f.analyze(SpaceOperation('cast_int_to_bool', [v2],
+                                                       v1))
+    assert not r
+    v1 = varoftype(lltype.Ptr(S))
+    v2 = varoftype(lltype.Signed)
+    v3 = varoftype(X)
+    v4 = varoftype(Z)
+    assert not f.analyze(SpaceOperation('bare_setfield', [v1, Constant('x'),
+                                                          v2], None))
+    assert     f.analyze(SpaceOperation('bare_setfield', [v1, Constant('y'),
+                                                          v3], None))
+    assert not f.analyze(SpaceOperation('bare_setfield', [v1, Constant('z'),
+                                                          v4], None))
+    
+        
+class TestLLType(BaseFinalizerAnalyzerTests):
+    type_system = 'lltype'
+
+    def test_malloc(self):
+        S = lltype.GcStruct('S')
+        
+        def f():
+            return lltype.malloc(S)
+
+        r = self.analyze(f, [])
+        assert r
+
+    def test_raw_free_getfield(self):
+        S = lltype.Struct('S')
+        
+        class A(object):
+            def __init__(self):
+                self.x = lltype.malloc(S, flavor='raw')
+
+            def __del__(self):
+                if self.x:
+                    self.x = lltype.nullptr(S)
+                    lltype.free(self.x, flavor='raw')
+
+        def f():
+            return A()
+
+        r = self.analyze(f, [], A.__del__.im_func)
+        assert not r
+
+    def test_c_call(self):
+        C = rffi.CArray(lltype.Signed)
+        c = rffi.llexternal('x', [lltype.Ptr(C)], lltype.Signed)
+
+        def g():
+            p = lltype.malloc(C, 3, flavor='raw')
+            f(p)
+        
+        def f(p):
+            c(rffi.ptradd(p, 0))
+            lltype.free(p, flavor='raw')
+
+        r = self.analyze(g, [], f, backendopt=True)
+        assert not r
+
+    def test_chain(self):
+        class B(object):
+            def __init__(self):
+                self.counter = 1
+        
+        class A(object):
+            def __init__(self):
+                self.x = B()
+
+            def __del__(self):
+                self.x.counter += 1
+
+        def f():
+            A()
+
+        r = self.analyze(f, [], A.__del__.im_func)
+        assert r
+
+    def test_is_light_finalizer_decorator(self):
+        S = lltype.GcStruct('S')
+
+        @rgc.is_light_finalizer
+        def f():
+            lltype.malloc(S)
+        @rgc.is_light_finalizer
+        def g():
+            pass
+        self.analyze(g, []) # did not explode
+        py.test.raises(FinalizerError, self.analyze, f, [])
+
+class TestOOType(BaseFinalizerAnalyzerTests):
+    type_system = 'ootype'
diff --git a/pypy/translator/c/primitive.py b/pypy/translator/c/primitive.py
--- a/pypy/translator/c/primitive.py
+++ b/pypy/translator/c/primitive.py
@@ -144,14 +144,19 @@
         obj = value._obj
         if isinstance(obj, int):
             # a tagged pointer
-            assert obj & 1 == 1
-            return '((%s) %d)' % (cdecl("void*", ''), obj)
+            return _name_tagged(obj, db)
         realobj = obj.container
+        if isinstance(realobj, int):
+            return _name_tagged(realobj, db)
         realvalue = cast_opaque_ptr(Ptr(typeOf(realobj)), value)
         return db.get(realvalue)
     else:
         return 'NULL'
 
+def _name_tagged(obj, db):
+    assert obj & 1 == 1
+    return '((%s) %d)' % (cdecl("void*", ''), obj)
+
 def name_small_integer(value, db):
     """Works for integers of size at most INT or UINT."""
     if isinstance(value, Symbolic):
diff --git a/pypy/translator/c/test/test_rtagged.py b/pypy/translator/c/test/test_rtagged.py
--- a/pypy/translator/c/test/test_rtagged.py
+++ b/pypy/translator/c/test/test_rtagged.py
@@ -77,3 +77,12 @@
     data = g.read()
     g.close()
     assert data.rstrip().endswith('ALL OK')
+
+def test_name_gcref():
+    from pypy.rpython.lltypesystem import lltype, llmemory, rclass
+    from pypy.translator.c import primitive
+    from pypy.translator.c.database import LowLevelDatabase
+    x = lltype.cast_int_to_ptr(rclass.OBJECTPTR, 19)
+    y = lltype.cast_opaque_ptr(llmemory.GCREF, x)
+    db = LowLevelDatabase()
+    assert primitive.name_gcref(y, db) == "((void*) 19)"

From noreply at buildbot.pypy.org  Thu Oct 27 11:54:20 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Oct 2011 11:54:20 +0200 (CEST)
Subject: [pypy-commit] benchmarks default: don't report stackcheck numbers
Message-ID: <20111027095420.3FE79820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r151:39ae6234d7ac
Date: 2011-10-27 11:53 +0200
http://bitbucket.org/pypy/benchmarks/changeset/39ae6234d7ac/

Log:	don't report stackcheck numbers

diff --git a/benchmarks.py b/benchmarks.py
--- a/benchmarks.py
+++ b/benchmarks.py
@@ -81,12 +81,12 @@
     for line in lines:
         if (line == 'Timings:' or
             line.startswith('============') or
-            line.startswith('Total:')):
+            line.startswith('Total:') or
+            'stackcheck' in line):
             continue
         name, _, time = map(str.strip, line.partition('---'))
         name = name.replace('_lltype', '')
         name = name.replace('_c', '')
-        name = name.replace('stackcheckinsertion', 'stackcheck')
         assert time.endswith(' s')
         time = float(time[:-2])
         timings.append((name, time))

From noreply at buildbot.pypy.org  Thu Oct 27 11:57:51 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Oct 2011 11:57:51 +0200 (CEST)
Subject: [pypy-commit] benchmarks default: fix a test
Message-ID: <20111027095751.39818820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r152:f04d6d63ba60
Date: 2011-10-27 11:57 +0200
http://bitbucket.org/pypy/benchmarks/changeset/f04d6d63ba60/

Log:	fix a test

diff --git a/benchmarks.py b/benchmarks.py
--- a/benchmarks.py
+++ b/benchmarks.py
@@ -110,7 +110,6 @@
     assert timings == [
         ('annotate', 1.3),
         ('rtype', 4.6),
-        ('stackcheck', 2.3),
         ('database', 0.4)
         ]
 

From noreply at buildbot.pypy.org  Thu Oct 27 12:03:31 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Oct 2011 12:03:31 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-multidim: getsetitem for single items
Message-ID: <20111027100331.03C1E820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: numpy-multidim
Changeset: r48501:345d2c256ce7
Date: 2011-10-27 12:02 +0200
http://bitbucket.org/pypy/pypy/changeset/345d2c256ce7/

Log:	getsetitem for single items

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -223,8 +223,12 @@
         concrete = self.get_concrete()
         return space.wrap("[" + " ".join(concrete._getnums(True)) + "]")
 
-    def item_at_index(self, index, space):
+    def _single_item_at_index(self, space, w_idx):
         # we assume C ordering for now
+        if len(self.shape) == 1:
+            return space.int_w(w_idx)
+        index = [space.int_w(w_item)
+                 for w_item in space.fixedview(w_idx)]
         item = 0
         for i in range(len(index)):
             if i != 0:
@@ -235,19 +239,31 @@
             item += index[i]
         return item
 
+    def _single_item_result(self, space, w_idx):
+        """ The result of getitem/setitem is a single item if w_idx
+        is a list of scalars that match the size of shape
+        """
+        if len(self.shape) == 1:
+            if (space.isinstance_w(w_idx, space.w_slice) or
+                space.isinstance_w(w_idx, space.w_int)):
+                return True
+            return False
+        lgt = space.len_w(w_idx)
+        if lgt > len(self.shape):
+            raise OperationError(space.w_IndexError,
+                                 space.wrap("invalid index"))
+        if lgt < len(self.shape):
+            return False
+        for w_item in space.fixedview(w_idx):
+            if space.isinstance_w(w_item, space.w_slice):
+                return False
+        return True
+
     def descr_getitem(self, space, w_idx):
-        # TODO: indexing by arrays and lists
-        if space.isinstance_w(w_idx, space.w_tuple):
-            # or any other sequence actually
-            length = space.len_w(w_idx)
-            if length == 0:
-                return space.wrap(self)
-            if length > len(self.shape):
-                raise OperationError(space.w_IndexError,
-                                     space.wrap("invalid index"))
-            indices = [space.int_w(w_item) for w_item in space.fixedview(w_idx)]
-            item = self.item_at_index(indices, space)
+        if self._single_item_result(space, w_idx):
+            item = self._single_item_at_index(space, w_idx)
             return self.get_concrete().eval(item).wrap(space)
+        xxx
         start, stop, step, slice_length = space.decode_index4(w_idx, self.shape[0])
         if step == 0:
             # Single index
@@ -263,6 +279,11 @@
     def descr_setitem(self, space, w_idx, w_value):
         # TODO: indexing by arrays and lists
         self.invalidated()
+        if self._single_item_at_index(space, w_idx):
+            item = self._single_item_at_index(space, w_idx)
+            self.get_concrete().setitem_w(space, item, w_value)
+            return
+        xxx
         if space.isinstance_w(w_idx, space.w_tuple):
             length = space.len_w(w_idx)
             if length > 1: # only one dimension for now.
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -615,9 +615,9 @@
     def test_getsetitem(self):
         import numpy
         a = numpy.zeros((2, 3, 1))
-        raises(IndexError, a.__getitem__, (0, 0, 0, 0))
-        raises(IndexError, a.__getitem__, (3,))
-        raises(IndexError, a.__getitem__, (1, 3))
+        #raises(IndexError, a.__getitem__, (0, 0, 0, 0))
+        #raises(IndexError, a.__getitem__, (3,))
+        #raises(IndexError, a.__getitem__, (1, 3))
         assert a[1, 1, 0] == 0
         a[1, 2, 0] = 3
         assert a[1, 2, 0] == 3

From noreply at buildbot.pypy.org  Thu Oct 27 12:41:06 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Oct 2011 12:41:06 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-multidim: few small fixes,
	start working on NDimSlice
Message-ID: <20111027104106.AC585820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: numpy-multidim
Changeset: r48502:21fa3eefa93b
Date: 2011-10-27 12:40 +0200
http://bitbucket.org/pypy/pypy/changeset/21fa3eefa93b/

Log:	few small fixes, start working on NDimSlice

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -17,8 +17,9 @@
 class BaseArray(Wrappable):
     _attrs_ = ["invalidates", "signature"]
 
-    def __init__(self):
+    def __init__(self, shape):
         self.invalidates = []
+        self.shape = shape
 
     def invalidated(self):
         if self.invalidates:
@@ -277,7 +278,6 @@
             return space.wrap(res)
 
     def descr_setitem(self, space, w_idx, w_value):
-        # TODO: indexing by arrays and lists
         self.invalidated()
         if self._single_item_at_index(space, w_idx):
             item = self._single_item_at_index(space, w_idx)
@@ -353,7 +353,7 @@
     _attrs_ = ["dtype", "value"]
 
     def __init__(self, dtype, value):
-        BaseArray.__init__(self)
+        BaseArray.__init__(self, [])
         self.dtype = dtype
         self.value = value
 
@@ -504,18 +504,19 @@
         raise NotImplementedError
 
     def descr_len(self, space):
+        xxx
         # XXX find shape first
         return space.wrap(self.find_size())
 
     def calc_index(self, item):
         raise NotImplementedError
 
-class SingleDimSlice(ViewArray):
+class NDimSlice(ViewArray):
     signature = signature.BaseSignature()
 
     def __init__(self, start, stop, step, slice_length, parent, signature):
         ViewArray.__init__(self, parent, signature)
-        if isinstance(parent, SingleDimSlice):
+        if isinstance(parent, NDimSlice):
             self.start = parent.calc_index(start)
             self.stop = parent.calc_index(stop)
             self.step = parent.step * step
@@ -549,9 +550,8 @@
 
 class NDimArray(BaseArray):
     def __init__(self, size, shape, dtype):
-        BaseArray.__init__(self)
+        BaseArray.__init__(self, shape)
         self.size = size
-        self.shape = shape
         self.dtype = dtype
         self.storage = dtype.malloc(size)
         self.signature = dtype.signature
@@ -572,7 +572,10 @@
         return self.dtype.getitem(self.storage, i)
 
     def descr_len(self, space):
-        return space.wrap(self.shape[0])
+        if len(self.shape):
+            return space.wrap(self.shape[0])
+        raise OperationError(space.w_TypeError, space.wrap(
+            "len() of unsized object"))
 
     def setitem_w(self, space, item, w_value):
         self.invalidated()
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -611,13 +611,14 @@
         assert numpy.zeros((2, 2)).shape == (2,2)
         assert numpy.zeros((3, 1, 2)).shape == (3, 1, 2)
         assert len(numpy.zeros((3, 1, 2))) == 3
+        raises(TypeError, len, numpy.zeros(()))
 
     def test_getsetitem(self):
         import numpy
         a = numpy.zeros((2, 3, 1))
-        #raises(IndexError, a.__getitem__, (0, 0, 0, 0))
-        #raises(IndexError, a.__getitem__, (3,))
-        #raises(IndexError, a.__getitem__, (1, 3))
+        raises(IndexError, a.__getitem__, (2, 0, 0))
+        raises(IndexError, a.__getitem__, (0, 3, 0))
+        raises(IndexError, a.__getitem__, (0, 0, 1))
         assert a[1, 1, 0] == 0
         a[1, 2, 0] = 3
         assert a[1, 2, 0] == 3

From noreply at buildbot.pypy.org  Thu Oct 27 14:59:44 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Oct 2011 14:59:44 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: Add a blog post
Message-ID: <20111027125944.AB141820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: extradoc
Changeset: r3939:bb06f91061f3
Date: 2011-10-27 14:58 +0200
http://bitbucket.org/pypy/extradoc/changeset/bb06f91061f3/

Log:	Add a blog post

diff --git a/blog/draft/faster-json.rst b/blog/draft/faster-json.rst
new file mode 100644
--- /dev/null
+++ b/blog/draft/faster-json.rst
@@ -0,0 +1,107 @@
+Speeding up JSON encoding in PyPy
+=================================
+
+Hi
+
+Recently I spent a bit of effort into speeding up JSON in PyPy. I started with
+writing a `benchmark`_, which is admiteddly not very good, but it's better
+than nothing (suggestions to improve welcomed!).
+
+For this particular benchmark, the numbers are as follow. Note that CPython
+uses hand-optimized C extension and PyPy uses a pure python version,
+hand-optimized in trunk, default in older versions. I'm taking the third run,
+when things are warmed up, full session `here`_.
+
++----------------------------+-------------+
+| CPython 2.6                | 22s         |
++----------------------------+-------------+
+| CPython 2.7                | **3.7s**    |
++----------------------------+-------------+
+| CPython 2.7 no C extension | 44s         |
++----------------------------+-------------+
+| PyPy 1.5                   | 34s         |
++----------------------------+-------------+
+| PyPy 1.6                   | 22s         |
++----------------------------+-------------+
+| PyPy trunk                 | **3.3s**    |
++----------------------------+-------------+
+
+.. _`benchmark`: https://bitbucket.org/pypy/benchmarks/src/f04d6d63ba60/own/json_bench.py
+.. _`here`: http://paste.pocoo.org/show/498988/
+
+Lessons learned:
+
+Expectations are high
+---------------------
+
+A lot of performance critical stuff in Python world is already written in a hand
+optimized C. Writing C (especially when you interface with CPython C API) is
+ugly and takes significant effort so it's only true for places which are
+well separated enough, but still. People would expect PyPy to outperform
+C extensions. Fortunately it's possible, but requires a bit of effort on
+the programmer side as well.
+
+Often interface between the C and Python part is ugly
+-----------------------------------------------------
+
+This is very clear if you look at json module as implemented in CPython's
+standard library. Not everything is in C (it would probably be just too
+much effort) and the interface to what is in C is guided via profiling not
+via what kind of interface makes sense. It's clear from CPython 2.6 to 2.7.
+Just adapting the code to interface with C made the Python version slower.
+Removing this clutter improves the readability a lot and improves PyPy's version
+a bit, although I don't have hard numbers.
+
+JitViewer is crucial
+--------------------
+
+In case you're fighting with PyPy's performance, `jitviewer`_ is worth a shot.
+While it's not completely trivial to understand what's going on, it'll
+definitely show you what kind of loops got compiled and how.
+
+.. _`jitviewer`: https://bitbucket.org/pypy/jitviewer
+
+No nice and fast way to build strings in Python
+-----------------------------------------------
+
+PyPy has a custom thing called ``__pypy__.builders.StringBuilder``. It has
+few features that make it much easier to optimize than other ways like
+``str.join()`` or ``cStringIO``.
+
+* You can specify the start size. Helps a lot if you can even provide a rough
+  estimate on the size of the string (less copying)
+* Only append and build allowed. While string is built you can't seek or
+  do anything else. Once it's built you can never append any more.
+* Unicode version available as well as ``__pypy__.builders.UnicodeBuilder``.
+
+Method calls are ok, immutable globals are ok
+---------------------------------------------
+
+PyPy's JIT seem to be good enough than at least in simple cases, calling
+methods for common infrastructure or loading globals (instead of rebinding as
+locals) is fast enough and improves code readability.
+
+I must admit I worked around PyPy's performance bug
+---------------------------------------------------
+
+For reasons obscure (although fixable), this::
+
+  for c in s: # s is string
+    del c
+
+is faster than::
+
+  for c in s:
+    pass
+
+This is a bug and should be fixed, but on different branch ;-)
+
+PyPy's JIT is kind of good
+--------------------------
+
+I was pretty surprised, but the JIT actually did make stuff work nicely. Seems
+you can write code in Python if you want to make it run fast, but you have
+to be a bit careful. Again, jitviewer is your friend
+
+Cheers,
+fijal

From noreply at buildbot.pypy.org  Thu Oct 27 15:16:25 2011
From: noreply at buildbot.pypy.org (antocuni)
Date: Thu, 27 Oct 2011 15:16:25 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: some english fixes,
 and rephrase a couple of sentences
Message-ID: <20111027131625.15F99820C2@wyvern.cs.uni-duesseldorf.de>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: extradoc
Changeset: r3940:4e7631f04f96
Date: 2011-10-27 15:16 +0200
http://bitbucket.org/pypy/extradoc/changeset/4e7631f04f96/

Log:	some english fixes, and rephrase a couple of sentences

diff --git a/blog/draft/faster-json.rst b/blog/draft/faster-json.rst
--- a/blog/draft/faster-json.rst
+++ b/blog/draft/faster-json.rst
@@ -4,13 +4,17 @@
 Hi
 
 Recently I spent a bit of effort into speeding up JSON in PyPy. I started with
-writing a `benchmark`_, which is admiteddly not very good, but it's better
-than nothing (suggestions to improve welcomed!).
+writing a `benchmark`_, which is admittedly not a very good one, but it's
+better than nothing (suggestions on how to improve it are welcome!). XXX:
+explain in one line what the benchmark does?
 
-For this particular benchmark, the numbers are as follow. Note that CPython
-uses hand-optimized C extension and PyPy uses a pure python version,
-hand-optimized in trunk, default in older versions. I'm taking the third run,
-when things are warmed up, full session `here`_.
+For this particular benchmark, the numbers are as follow. Note that CPython by
+default uses the optimized C extension, while PyPy uses the pure Python one.
+PyPy trunk contains another pure Python version which has been optimized
+specifically for the PyPy JIT, which is the subject of this post.
+
+The number reported is the time taken for the third run, when things are
+warmed up. Full session `here`_.
 
 +----------------------------+-------------+
 | CPython 2.6                | 22s         |
@@ -84,7 +88,7 @@
 I must admit I worked around PyPy's performance bug
 ---------------------------------------------------
 
-For reasons obscure (although fixable), this::
+For obscure (although eventually fixable) reasons, this::
 
   for c in s: # s is string
     del c
@@ -94,14 +98,14 @@
   for c in s:
     pass
 
-This is a bug and should be fixed, but on different branch ;-)
+This is a PyPy performance bug and should be fixed, but on different branch ;-)
 
-PyPy's JIT is kind of good
+PyPy's JIT is good
 --------------------------
 
-I was pretty surprised, but the JIT actually did make stuff work nicely. Seems
-you can write code in Python if you want to make it run fast, but you have
-to be a bit careful. Again, jitviewer is your friend
+I was pretty surprised, but the JIT actually did make stuff work nicely.
+It is possible to write code in Python and make it run fast, but you have
+to be a bit careful. Again, jitviewer is your friend.
 
 Cheers,
 fijal

From noreply at buildbot.pypy.org  Thu Oct 27 15:20:05 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Oct 2011 15:20:05 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: make some stuff bold
Message-ID: <20111027132005.67E98820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: extradoc
Changeset: r3941:16b98b486879
Date: 2011-10-27 15:17 +0200
http://bitbucket.org/pypy/extradoc/changeset/16b98b486879/

Log:	make some stuff bold

diff --git a/blog/draft/faster-json.rst b/blog/draft/faster-json.rst
--- a/blog/draft/faster-json.rst
+++ b/blog/draft/faster-json.rst
@@ -7,9 +7,9 @@
 writing a `benchmark`_, which is admiteddly not very good, but it's better
 than nothing (suggestions to improve welcomed!).
 
-For this particular benchmark, the numbers are as follow. Note that CPython
+For this particular benchmark, the numbers are as follow. **Note that CPython
 uses hand-optimized C extension and PyPy uses a pure python version,
-hand-optimized in trunk, default in older versions. I'm taking the third run,
+hand-optimized in trunk, default in older versions**. I'm taking the third run,
 when things are warmed up, full session `here`_.
 
 +----------------------------+-------------+

From noreply at buildbot.pypy.org  Thu Oct 27 15:20:06 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Oct 2011 15:20:06 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: merge
Message-ID: <20111027132006.86735820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: extradoc
Changeset: r3942:300ff59a5bbd
Date: 2011-10-27 15:19 +0200
http://bitbucket.org/pypy/extradoc/changeset/300ff59a5bbd/

Log:	merge

diff --git a/blog/draft/faster-json.rst b/blog/draft/faster-json.rst
--- a/blog/draft/faster-json.rst
+++ b/blog/draft/faster-json.rst
@@ -4,13 +4,17 @@
 Hi
 
 Recently I spent a bit of effort into speeding up JSON in PyPy. I started with
-writing a `benchmark`_, which is admiteddly not very good, but it's better
-than nothing (suggestions to improve welcomed!).
+writing a `benchmark`_, which is admittedly not a very good one, but it's
+better than nothing (suggestions on how to improve it are welcome!). XXX:
+explain in one line what the benchmark does?
 
-For this particular benchmark, the numbers are as follow. **Note that CPython
-uses hand-optimized C extension and PyPy uses a pure python version,
-hand-optimized in trunk, default in older versions**. I'm taking the third run,
-when things are warmed up, full session `here`_.
+For this particular benchmark, the numbers are as follow. **Note that CPython by
+default uses the optimized C extension, while PyPy uses the pure Python one**.
+PyPy trunk contains another pure Python version which has been optimized
+specifically for the PyPy JIT, which is the subject of this post.
+
+The number reported is the time taken for the third run, when things are
+warmed up. Full session `here`_.
 
 +----------------------------+-------------+
 | CPython 2.6                | 22s         |
@@ -84,7 +88,7 @@
 I must admit I worked around PyPy's performance bug
 ---------------------------------------------------
 
-For reasons obscure (although fixable), this::
+For obscure (although eventually fixable) reasons, this::
 
   for c in s: # s is string
     del c
@@ -94,14 +98,14 @@
   for c in s:
     pass
 
-This is a bug and should be fixed, but on different branch ;-)
+This is a PyPy performance bug and should be fixed, but on different branch ;-)
 
-PyPy's JIT is kind of good
+PyPy's JIT is good
 --------------------------
 
-I was pretty surprised, but the JIT actually did make stuff work nicely. Seems
-you can write code in Python if you want to make it run fast, but you have
-to be a bit careful. Again, jitviewer is your friend
+I was pretty surprised, but the JIT actually did make stuff work nicely.
+It is possible to write code in Python and make it run fast, but you have
+to be a bit careful. Again, jitviewer is your friend.
 
 Cheers,
 fijal

From noreply at buildbot.pypy.org  Thu Oct 27 16:14:40 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Oct 2011 16:14:40 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-multidim: few small fixes,
	start working on NDimSlice
Message-ID: <20111027141440.87459820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: numpy-multidim
Changeset: r48503:6fd1b6212a20
Date: 2011-10-27 12:50 +0200
http://bitbucket.org/pypy/pypy/changeset/6fd1b6212a20/

Log:	few small fixes, start working on NDimSlice

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -260,22 +260,24 @@
                 return False
         return True
 
+    def _create_slice(self, space, w_idx):
+        new_sig = signature.Signature.find_sig([
+            NDimSlice.signature, self.signature
+        ])
+        if (space.isinstance_w(w_idx, space.w_int) or
+            space.isinstance_w(w_idx, space.w_slice)):
+            chunks = [space.decode_index4(w_idx, self.shape[0])]
+        else:
+            chunks = []
+            for i, w_item in enumerate(space.fixedview(w_idx)):
+                chunks.append(space.decode_index4(w_item, self.shape[i]))
+        return NDimSlice(self, new_sig, chunks)
+
     def descr_getitem(self, space, w_idx):
         if self._single_item_result(space, w_idx):
             item = self._single_item_at_index(space, w_idx)
             return self.get_concrete().eval(item).wrap(space)
-        xxx
-        start, stop, step, slice_length = space.decode_index4(w_idx, self.shape[0])
-        if step == 0:
-            # Single index
-            return self.get_concrete().eval(start).wrap(space)
-        else:
-            # Slice
-            new_sig = signature.Signature.find_sig([
-                SingleDimSlice.signature, self.signature
-            ])
-            res = SingleDimSlice(start, stop, step, slice_length, self, new_sig)
-            return space.wrap(res)
+        return space.wrap(self._create_slice(space, w_idx))
 
     def descr_setitem(self, space, w_idx, w_value):
         self.invalidated()
@@ -514,18 +516,10 @@
 class NDimSlice(ViewArray):
     signature = signature.BaseSignature()
 
-    def __init__(self, start, stop, step, slice_length, parent, signature):
+    def __init__(self, parent, signature, chunks):
         ViewArray.__init__(self, parent, signature)
-        if isinstance(parent, NDimSlice):
-            self.start = parent.calc_index(start)
-            self.stop = parent.calc_index(stop)
-            self.step = parent.step * step
-            self.parent = parent.parent
-        else:
-            self.start = start
-            self.stop = stop
-            self.step = step
-            self.parent = parent
+        self.chunks = chunks
+        xxxx
         self.size = slice_length
 
     def get_root_storage(self):
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -624,6 +624,14 @@
         assert a[1, 2, 0] == 3
         assert a[1, 1, 0] == 0
 
+    def test_slices(self):
+        import numpy
+        a = numpy.zeros((4, 3, 2))
+        raises(IndexError, a.__getitem__, (4,))
+        raises(IndexError, a.__getitem__, (3, 3))
+        raises(IndexError, a.__getitem__, (:, 3))
+        
+
 class AppTestSupport(object):
     def setup_class(cls):
         import struct

From noreply at buildbot.pypy.org  Thu Oct 27 16:14:41 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Oct 2011 16:14:41 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-multidim: fix the test. previous commit by
 accident - start working on slices
Message-ID: <20111027141441.B5896820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: numpy-multidim
Changeset: r48504:c50cad7db70b
Date: 2011-10-27 12:52 +0200
http://bitbucket.org/pypy/pypy/changeset/c50cad7db70b/

Log:	fix the test. previous commit by accident - start working on slices

diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -629,7 +629,7 @@
         a = numpy.zeros((4, 3, 2))
         raises(IndexError, a.__getitem__, (4,))
         raises(IndexError, a.__getitem__, (3, 3))
-        raises(IndexError, a.__getitem__, (:, 3))
+        raises(IndexError, a.__getitem__, (slice(None), 3))
         
 
 class AppTestSupport(object):

From noreply at buildbot.pypy.org  Thu Oct 27 16:14:42 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Oct 2011 16:14:42 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-multidim: Work on casting. Seems to fly
Message-ID: <20111027141442.E35CF820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: numpy-multidim
Changeset: r48505:301f1937556a
Date: 2011-10-27 16:13 +0200
http://bitbucket.org/pypy/pypy/changeset/301f1937556a/

Log:	Work on casting. Seems to fly

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -211,7 +211,8 @@
     def descr_repr(self, space):
         # Simple implementation so that we can see the array. Needs work.
         concrete = self.get_concrete()
-        res = "array([" + ", ".join(concrete._getnums(False)) + "]"
+        #res = "array([" + ", ".join(concrete._getnums(False)) + "]"
+        res = 'array()'
         dtype = concrete.find_dtype()
         if (dtype is not space.fromcache(interp_dtype.W_Float64Dtype) and
             dtype is not space.fromcache(interp_dtype.W_Int64Dtype)) or not self.find_size():
@@ -240,20 +241,29 @@
             item += index[i]
         return item
 
+    def len_of_shape(self):
+        return len(self.shape)
+
+    def get_root_shape(self):
+        return self.shape
+
     def _single_item_result(self, space, w_idx):
         """ The result of getitem/setitem is a single item if w_idx
         is a list of scalars that match the size of shape
         """
-        if len(self.shape) == 1:
-            if (space.isinstance_w(w_idx, space.w_slice) or
-                space.isinstance_w(w_idx, space.w_int)):
+        shape_len = self.len_of_shape()
+        if shape_len == 1:
+            if space.isinstance_w(w_idx, space.w_int):
                 return True
             return False
+        if (space.isinstance_w(w_idx, space.w_slice) or
+            space.isinstance_w(w_idx, space.w_int)):
+            return False
         lgt = space.len_w(w_idx)
-        if lgt > len(self.shape):
+        if lgt > shape_len:
             raise OperationError(space.w_IndexError,
                                  space.wrap("invalid index"))
-        if lgt < len(self.shape):
+        if lgt < shape_len:
             return False
         for w_item in space.fixedview(w_idx):
             if space.isinstance_w(w_item, space.w_slice):
@@ -266,12 +276,25 @@
         ])
         if (space.isinstance_w(w_idx, space.w_int) or
             space.isinstance_w(w_idx, space.w_slice)):
-            chunks = [space.decode_index4(w_idx, self.shape[0])]
+            start, stop, step, lgt = space.decode_index4(w_idx, self.shape[0])
+            if lgt == 1:
+                shape = self.shape[1:]
+            else:
+                shape = self.shape
+            chunks = [(start, stop, step, lgt)]
         else:
             chunks = []
+            shape = self.shape[:]
             for i, w_item in enumerate(space.fixedview(w_idx)):
-                chunks.append(space.decode_index4(w_item, self.shape[i]))
-        return NDimSlice(self, new_sig, chunks)
+                start, stop, step, lgt = space.decode_index4(w_item,
+                                                             self.shape[i])
+                chunks.append((start, stop, step, lgt))
+                if lgt == 1:
+                    shape[i] = -1
+                else:
+                    shape[i] = lgt
+            shape = [i for i in shape if i != -1]
+        return NDimSlice(self, new_sig, chunks, shape)
 
     def descr_getitem(self, space, w_idx):
         if self._single_item_result(space, w_idx):
@@ -481,10 +504,13 @@
     Class for representing views of arrays, they will reflect changes of parent
     arrays. Example: slices
     """
-    def __init__(self, parent, signature):
-        BaseArray.__init__(self)
+    def __init__(self, parent, signature, shape):
+        BaseArray.__init__(self, shape)
         self.signature = signature
         self.parent = parent
+        self.size = 1
+        for elem in shape:
+            self.size *= elem
         self.invalidates = parent.invalidates
 
     def get_concrete(self):
@@ -506,9 +532,7 @@
         raise NotImplementedError
 
     def descr_len(self, space):
-        xxx
-        # XXX find shape first
-        return space.wrap(self.find_size())
+        return space.wrap(self.shape[0])
 
     def calc_index(self, item):
         raise NotImplementedError
@@ -516,11 +540,13 @@
 class NDimSlice(ViewArray):
     signature = signature.BaseSignature()
 
-    def __init__(self, parent, signature, chunks):
-        ViewArray.__init__(self, parent, signature)
+    def __init__(self, parent, signature, chunks, shape):
+        ViewArray.__init__(self, parent, signature, shape)
         self.chunks = chunks
-        xxxx
-        self.size = slice_length
+        self.shape_reduction = 0
+        for chunk in chunks:
+            if chunk[-1] == 1:
+                self.shape_reduction += 1
 
     def get_root_storage(self):
         return self.parent.get_concrete().get_root_storage()
@@ -532,14 +558,53 @@
         return self.parent.find_dtype()
 
     def setslice(self, space, start, stop, step, slice_length, arr):
+        xxx
         start = self.calc_index(start)
         if stop != -1:
             stop = self.calc_index(stop)
         step = self.step * step
         self._sliceloop(start, stop, step, arr, self.parent)
 
+    def len_of_shape(self):
+        return self.parent.len_of_shape() - self.shape_reduction
+
+    def get_root_shape(self):
+        return self.parent.get_root_shape()
+
+    # XXX we might want to provide a custom finder of where we look for
+    #     a particular item, right now we'll do the calculations again
+
     def calc_index(self, item):
-        return (self.start + item * self.step)
+        index = []
+        _item = item
+        for i in range(len(self.shape) -1, 0, -1):
+            s = self.shape[i]
+            index.append(_item % s)
+            _item //= s
+        index.append(_item)
+        index.reverse()
+        i = 0
+        item = 0
+        k = 0
+        shape = self.parent.shape
+        for chunk in self.chunks:
+            if k != 0:
+                item *= shape[k]
+            k += 1
+            start, stop, step, lgt = chunk
+            if lgt == 1:
+                # we don't consume an index
+                item += start
+            else:
+                item += start + step * index[i]
+                i += 1
+        while k < len(shape):
+            if k != 0:
+                item *= shape[k]
+            k += 1
+            item += index[i]
+            i += 1
+        return item
 
 
 class NDimArray(BaseArray):
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -630,7 +630,30 @@
         raises(IndexError, a.__getitem__, (4,))
         raises(IndexError, a.__getitem__, (3, 3))
         raises(IndexError, a.__getitem__, (slice(None), 3))
-        
+        a[0,1,1] = 13
+        a[1,2,1] = 15
+        b = a[0]
+        assert len(b) == 3
+        assert b.shape == (3, 2)
+        assert b[1,1] == 13
+        b = a[1]
+        assert b.shape == (3, 2)
+        assert b[2,1] == 15
+        b = a[:,1]
+        assert b.shape == (4, 2)
+        assert b[0,1] == 13
+        b = a[:,1,:]
+        assert b.shape == (4, 2)
+        assert b[0,1] == 13
+        b = a[1, 2]
+        assert b[1] == 15
+        b = a[:]
+        assert b.shape == (4, 3, 2)
+        assert b[1,2,1] == 15
+        assert b[0,1,1] == 13
+        b = a[:][:,1][:]
+        assert b[2,1] == 0.0
+        assert b[0,1] == 13
 
 class AppTestSupport(object):
     def setup_class(cls):

From noreply at buildbot.pypy.org  Thu Oct 27 16:19:28 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Oct 2011 16:19:28 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-multidim: more tests
Message-ID: <20111027141928.085F3820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: numpy-multidim
Changeset: r48506:4fb18345b1ab
Date: 2011-10-27 16:18 +0200
http://bitbucket.org/pypy/pypy/changeset/4fb18345b1ab/

Log:	more tests

diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -654,6 +654,7 @@
         b = a[:][:,1][:]
         assert b[2,1] == 0.0
         assert b[0,1] == 13
+        raises(IndexError, b.__getitem__, (4, 1))
 
 class AppTestSupport(object):
     def setup_class(cls):

From noreply at buildbot.pypy.org  Thu Oct 27 16:21:06 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Oct 2011 16:21:06 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-multidim: even more tests,
	you can never be sure
Message-ID: <20111027142106.B6364820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: numpy-multidim
Changeset: r48507:e6db0d0d0445
Date: 2011-10-27 16:19 +0200
http://bitbucket.org/pypy/pypy/changeset/e6db0d0d0445/

Log:	even more tests, you can never be sure

diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -655,6 +655,8 @@
         assert b[2,1] == 0.0
         assert b[0,1] == 13
         raises(IndexError, b.__getitem__, (4, 1))
+        assert a[0][1][1] == 13
+        assert a[1][2][1] == 15
 
 class AppTestSupport(object):
     def setup_class(cls):

From noreply at buildbot.pypy.org  Thu Oct 27 16:35:40 2011
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Thu, 27 Oct 2011 16:35:40 +0200 (CEST)
Subject: [pypy-commit] pypy default: copy heapq to modified-2.7 and
 correctly deal with nlargest/nsmallest getting n < 0, fixes issue924
Message-ID: <20111027143540.B6CDD820C2@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: 
Changeset: r48508:3fec6b7c733c
Date: 2011-10-27 16:35 +0200
http://bitbucket.org/pypy/pypy/changeset/3fec6b7c733c/

Log:	copy heapq to modified-2.7 and correctly deal with
	nlargest/nsmallest getting n < 0, fixes issue924

diff --git a/lib-python/modified-2.7/heapq.py b/lib-python/modified-2.7/heapq.py
new file mode 100644
--- /dev/null
+++ b/lib-python/modified-2.7/heapq.py
@@ -0,0 +1,442 @@
+# -*- coding: latin-1 -*-
+
+"""Heap queue algorithm (a.k.a. priority queue).
+
+Heaps are arrays for which a[k] <= a[2*k+1] and a[k] <= a[2*k+2] for
+all k, counting elements from 0.  For the sake of comparison,
+non-existing elements are considered to be infinite.  The interesting
+property of a heap is that a[0] is always its smallest element.
+
+Usage:
+
+heap = []            # creates an empty heap
+heappush(heap, item) # pushes a new item on the heap
+item = heappop(heap) # pops the smallest item from the heap
+item = heap[0]       # smallest item on the heap without popping it
+heapify(x)           # transforms list into a heap, in-place, in linear time
+item = heapreplace(heap, item) # pops and returns smallest item, and adds
+                               # new item; the heap size is unchanged
+
+Our API differs from textbook heap algorithms as follows:
+
+- We use 0-based indexing.  This makes the relationship between the
+  index for a node and the indexes for its children slightly less
+  obvious, but is more suitable since Python uses 0-based indexing.
+
+- Our heappop() method returns the smallest item, not the largest.
+
+These two make it possible to view the heap as a regular Python list
+without surprises: heap[0] is the smallest item, and heap.sort()
+maintains the heap invariant!
+"""
+
+# Original code by Kevin O'Connor, augmented by Tim Peters and Raymond Hettinger
+
+__about__ = """Heap queues
+
+[explanation by Fran&#65533;ois Pinard]
+
+Heaps are arrays for which a[k] <= a[2*k+1] and a[k] <= a[2*k+2] for
+all k, counting elements from 0.  For the sake of comparison,
+non-existing elements are considered to be infinite.  The interesting
+property of a heap is that a[0] is always its smallest element.
+
+The strange invariant above is meant to be an efficient memory
+representation for a tournament.  The numbers below are `k', not a[k]:
+
+                                   0
+
+                  1                                 2
+
+          3               4                5               6
+
+      7       8       9       10      11      12      13      14
+
+    15 16   17 18   19 20   21 22   23 24   25 26   27 28   29 30
+
+
+In the tree above, each cell `k' is topping `2*k+1' and `2*k+2'.  In
+an usual binary tournament we see in sports, each cell is the winner
+over the two cells it tops, and we can trace the winner down the tree
+to see all opponents s/he had.  However, in many computer applications
+of such tournaments, we do not need to trace the history of a winner.
+To be more memory efficient, when a winner is promoted, we try to
+replace it by something else at a lower level, and the rule becomes
+that a cell and the two cells it tops contain three different items,
+but the top cell "wins" over the two topped cells.
+
+If this heap invariant is protected at all time, index 0 is clearly
+the overall winner.  The simplest algorithmic way to remove it and
+find the "next" winner is to move some loser (let's say cell 30 in the
+diagram above) into the 0 position, and then percolate this new 0 down
+the tree, exchanging values, until the invariant is re-established.
+This is clearly logarithmic on the total number of items in the tree.
+By iterating over all items, you get an O(n ln n) sort.
+
+A nice feature of this sort is that you can efficiently insert new
+items while the sort is going on, provided that the inserted items are
+not "better" than the last 0'th element you extracted.  This is
+especially useful in simulation contexts, where the tree holds all
+incoming events, and the "win" condition means the smallest scheduled
+time.  When an event schedule other events for execution, they are
+scheduled into the future, so they can easily go into the heap.  So, a
+heap is a good structure for implementing schedulers (this is what I
+used for my MIDI sequencer :-).
+
+Various structures for implementing schedulers have been extensively
+studied, and heaps are good for this, as they are reasonably speedy,
+the speed is almost constant, and the worst case is not much different
+than the average case.  However, there are other representations which
+are more efficient overall, yet the worst cases might be terrible.
+
+Heaps are also very useful in big disk sorts.  You most probably all
+know that a big sort implies producing "runs" (which are pre-sorted
+sequences, which size is usually related to the amount of CPU memory),
+followed by a merging passes for these runs, which merging is often
+very cleverly organised[1].  It is very important that the initial
+sort produces the longest runs possible.  Tournaments are a good way
+to that.  If, using all the memory available to hold a tournament, you
+replace and percolate items that happen to fit the current run, you'll
+produce runs which are twice the size of the memory for random input,
+and much better for input fuzzily ordered.
+
+Moreover, if you output the 0'th item on disk and get an input which
+may not fit in the current tournament (because the value "wins" over
+the last output value), it cannot fit in the heap, so the size of the
+heap decreases.  The freed memory could be cleverly reused immediately
+for progressively building a second heap, which grows at exactly the
+same rate the first heap is melting.  When the first heap completely
+vanishes, you switch heaps and start a new run.  Clever and quite
+effective!
+
+In a word, heaps are useful memory structures to know.  I use them in
+a few applications, and I think it is good to keep a `heap' module
+around. :-)
+
+--------------------
+[1] The disk balancing algorithms which are current, nowadays, are
+more annoying than clever, and this is a consequence of the seeking
+capabilities of the disks.  On devices which cannot seek, like big
+tape drives, the story was quite different, and one had to be very
+clever to ensure (far in advance) that each tape movement will be the
+most effective possible (that is, will best participate at
+"progressing" the merge).  Some tapes were even able to read
+backwards, and this was also used to avoid the rewinding time.
+Believe me, real good tape sorts were quite spectacular to watch!
+From all times, sorting has always been a Great Art! :-)
+"""
+
+__all__ = ['heappush', 'heappop', 'heapify', 'heapreplace', 'merge',
+           'nlargest', 'nsmallest', 'heappushpop']
+
+from itertools import islice, repeat, count, imap, izip, tee, chain
+from operator import itemgetter
+import bisect
+
+def heappush(heap, item):
+    """Push item onto heap, maintaining the heap invariant."""
+    heap.append(item)
+    _siftdown(heap, 0, len(heap)-1)
+
+def heappop(heap):
+    """Pop the smallest item off the heap, maintaining the heap invariant."""
+    lastelt = heap.pop()    # raises appropriate IndexError if heap is empty
+    if heap:
+        returnitem = heap[0]
+        heap[0] = lastelt
+        _siftup(heap, 0)
+    else:
+        returnitem = lastelt
+    return returnitem
+
+def heapreplace(heap, item):
+    """Pop and return the current smallest value, and add the new item.
+
+    This is more efficient than heappop() followed by heappush(), and can be
+    more appropriate when using a fixed-size heap.  Note that the value
+    returned may be larger than item!  That constrains reasonable uses of
+    this routine unless written as part of a conditional replacement:
+
+        if item > heap[0]:
+            item = heapreplace(heap, item)
+    """
+    returnitem = heap[0]    # raises appropriate IndexError if heap is empty
+    heap[0] = item
+    _siftup(heap, 0)
+    return returnitem
+
+def heappushpop(heap, item):
+    """Fast version of a heappush followed by a heappop."""
+    if heap and heap[0] < item:
+        item, heap[0] = heap[0], item
+        _siftup(heap, 0)
+    return item
+
+def heapify(x):
+    """Transform list into a heap, in-place, in O(len(heap)) time."""
+    n = len(x)
+    # Transform bottom-up.  The largest index there's any point to looking at
+    # is the largest with a child index in-range, so must have 2*i + 1 < n,
+    # or i < (n-1)/2.  If n is even = 2*j, this is (2*j-1)/2 = j-1/2 so
+    # j-1 is the largest, which is n//2 - 1.  If n is odd = 2*j+1, this is
+    # (2*j+1-1)/2 = j so j-1 is the largest, and that's again n//2-1.
+    for i in reversed(xrange(n//2)):
+        _siftup(x, i)
+
+def nlargest(n, iterable):
+    """Find the n largest elements in a dataset.
+
+    Equivalent to:  sorted(iterable, reverse=True)[:n]
+    """
+    if n < 0: # for consistency with the c impl
+        return []
+    it = iter(iterable)
+    result = list(islice(it, n))
+    if not result:
+        return result
+    heapify(result)
+    _heappushpop = heappushpop
+    for elem in it:
+        _heappushpop(result, elem)
+    result.sort(reverse=True)
+    return result
+
+def nsmallest(n, iterable):
+    """Find the n smallest elements in a dataset.
+
+    Equivalent to:  sorted(iterable)[:n]
+    """
+    if n < 0: # for consistency with the c impl
+        return []
+    if hasattr(iterable, '__len__') and n * 10 <= len(iterable):
+        # For smaller values of n, the bisect method is faster than a minheap.
+        # It is also memory efficient, consuming only n elements of space.
+        it = iter(iterable)
+        result = sorted(islice(it, 0, n))
+        if not result:
+            return result
+        insort = bisect.insort
+        pop = result.pop
+        los = result[-1]    # los --> Largest of the nsmallest
+        for elem in it:
+            if los <= elem:
+                continue
+            insort(result, elem)
+            pop()
+            los = result[-1]
+        return result
+    # An alternative approach manifests the whole iterable in memory but
+    # saves comparisons by heapifying all at once.  Also, saves time
+    # over bisect.insort() which has O(n) data movement time for every
+    # insertion.  Finding the n smallest of an m length iterable requires
+    #    O(m) + O(n log m) comparisons.
+    h = list(iterable)
+    heapify(h)
+    return map(heappop, repeat(h, min(n, len(h))))
+
+# 'heap' is a heap at all indices >= startpos, except possibly for pos.  pos
+# is the index of a leaf with a possibly out-of-order value.  Restore the
+# heap invariant.
+def _siftdown(heap, startpos, pos):
+    newitem = heap[pos]
+    # Follow the path to the root, moving parents down until finding a place
+    # newitem fits.
+    while pos > startpos:
+        parentpos = (pos - 1) >> 1
+        parent = heap[parentpos]
+        if newitem < parent:
+            heap[pos] = parent
+            pos = parentpos
+            continue
+        break
+    heap[pos] = newitem
+
+# The child indices of heap index pos are already heaps, and we want to make
+# a heap at index pos too.  We do this by bubbling the smaller child of
+# pos up (and so on with that child's children, etc) until hitting a leaf,
+# then using _siftdown to move the oddball originally at index pos into place.
+#
+# We *could* break out of the loop as soon as we find a pos where newitem <=
+# both its children, but turns out that's not a good idea, and despite that
+# many books write the algorithm that way.  During a heap pop, the last array
+# element is sifted in, and that tends to be large, so that comparing it
+# against values starting from the root usually doesn't pay (= usually doesn't
+# get us out of the loop early).  See Knuth, Volume 3, where this is
+# explained and quantified in an exercise.
+#
+# Cutting the # of comparisons is important, since these routines have no
+# way to extract "the priority" from an array element, so that intelligence
+# is likely to be hiding in custom __cmp__ methods, or in array elements
+# storing (priority, record) tuples.  Comparisons are thus potentially
+# expensive.
+#
+# On random arrays of length 1000, making this change cut the number of
+# comparisons made by heapify() a little, and those made by exhaustive
+# heappop() a lot, in accord with theory.  Here are typical results from 3
+# runs (3 just to demonstrate how small the variance is):
+#
+# Compares needed by heapify     Compares needed by 1000 heappops
+# --------------------------     --------------------------------
+# 1837 cut to 1663               14996 cut to 8680
+# 1855 cut to 1659               14966 cut to 8678
+# 1847 cut to 1660               15024 cut to 8703
+#
+# Building the heap by using heappush() 1000 times instead required
+# 2198, 2148, and 2219 compares:  heapify() is more efficient, when
+# you can use it.
+#
+# The total compares needed by list.sort() on the same lists were 8627,
+# 8627, and 8632 (this should be compared to the sum of heapify() and
+# heappop() compares):  list.sort() is (unsurprisingly!) more efficient
+# for sorting.
+
+def _siftup(heap, pos):
+    endpos = len(heap)
+    startpos = pos
+    newitem = heap[pos]
+    # Bubble up the smaller child until hitting a leaf.
+    childpos = 2*pos + 1    # leftmost child position
+    while childpos < endpos:
+        # Set childpos to index of smaller child.
+        rightpos = childpos + 1
+        if rightpos < endpos and not heap[childpos] < heap[rightpos]:
+            childpos = rightpos
+        # Move the smaller child up.
+        heap[pos] = heap[childpos]
+        pos = childpos
+        childpos = 2*pos + 1
+    # The leaf at pos is empty now.  Put newitem there, and bubble it up
+    # to its final resting place (by sifting its parents down).
+    heap[pos] = newitem
+    _siftdown(heap, startpos, pos)
+
+# If available, use C implementation
+try:
+    from _heapq import *
+except ImportError:
+    pass
+
+def merge(*iterables):
+    '''Merge multiple sorted inputs into a single sorted output.
+
+    Similar to sorted(itertools.chain(*iterables)) but returns a generator,
+    does not pull the data into memory all at once, and assumes that each of
+    the input streams is already sorted (smallest to largest).
+
+    >>> list(merge([1,3,5,7], [0,2,4,8], [5,10,15,20], [], [25]))
+    [0, 1, 2, 3, 4, 5, 5, 7, 8, 10, 15, 20, 25]
+
+    '''
+    _heappop, _heapreplace, _StopIteration = heappop, heapreplace, StopIteration
+
+    h = []
+    h_append = h.append
+    for itnum, it in enumerate(map(iter, iterables)):
+        try:
+            next = it.next
+            h_append([next(), itnum, next])
+        except _StopIteration:
+            pass
+    heapify(h)
+
+    while 1:
+        try:
+            while 1:
+                v, itnum, next = s = h[0]   # raises IndexError when h is empty
+                yield v
+                s[0] = next()               # raises StopIteration when exhausted
+                _heapreplace(h, s)          # restore heap condition
+        except _StopIteration:
+            _heappop(h)                     # remove empty iterator
+        except IndexError:
+            return
+
+# Extend the implementations of nsmallest and nlargest to use a key= argument
+_nsmallest = nsmallest
+def nsmallest(n, iterable, key=None):
+    """Find the n smallest elements in a dataset.
+
+    Equivalent to:  sorted(iterable, key=key)[:n]
+    """
+    # Short-cut for n==1 is to use min() when len(iterable)>0
+    if n == 1:
+        it = iter(iterable)
+        head = list(islice(it, 1))
+        if not head:
+            return []
+        if key is None:
+            return [min(chain(head, it))]
+        return [min(chain(head, it), key=key)]
+
+    # When n>=size, it's faster to use sort()
+    try:
+        size = len(iterable)
+    except (TypeError, AttributeError):
+        pass
+    else:
+        if n >= size:
+            return sorted(iterable, key=key)[:n]
+
+    # When key is none, use simpler decoration
+    if key is None:
+        it = izip(iterable, count())                        # decorate
+        result = _nsmallest(n, it)
+        return map(itemgetter(0), result)                   # undecorate
+
+    # General case, slowest method
+    in1, in2 = tee(iterable)
+    it = izip(imap(key, in1), count(), in2)                 # decorate
+    result = _nsmallest(n, it)
+    return map(itemgetter(2), result)                       # undecorate
+
+_nlargest = nlargest
+def nlargest(n, iterable, key=None):
+    """Find the n largest elements in a dataset.
+
+    Equivalent to:  sorted(iterable, key=key, reverse=True)[:n]
+    """
+
+    # Short-cut for n==1 is to use max() when len(iterable)>0
+    if n == 1:
+        it = iter(iterable)
+        head = list(islice(it, 1))
+        if not head:
+            return []
+        if key is None:
+            return [max(chain(head, it))]
+        return [max(chain(head, it), key=key)]
+
+    # When n>=size, it's faster to use sort()
+    try:
+        size = len(iterable)
+    except (TypeError, AttributeError):
+        pass
+    else:
+        if n >= size:
+            return sorted(iterable, key=key, reverse=True)[:n]
+
+    # When key is none, use simpler decoration
+    if key is None:
+        it = izip(iterable, count(0,-1))                    # decorate
+        result = _nlargest(n, it)
+        return map(itemgetter(0), result)                   # undecorate
+
+    # General case, slowest method
+    in1, in2 = tee(iterable)
+    it = izip(imap(key, in1), count(0,-1), in2)             # decorate
+    result = _nlargest(n, it)
+    return map(itemgetter(2), result)                       # undecorate
+
+if __name__ == "__main__":
+    # Simple sanity test
+    heap = []
+    data = [1, 3, 5, 7, 9, 2, 4, 6, 8, 0]
+    for item in data:
+        heappush(heap, item)
+    sort = []
+    while heap:
+        sort.append(heappop(heap))
+    print sort
+
+    import doctest
+    doctest.testmod()
diff --git a/lib-python/modified-2.7/test/test_heapq.py b/lib-python/modified-2.7/test/test_heapq.py
--- a/lib-python/modified-2.7/test/test_heapq.py
+++ b/lib-python/modified-2.7/test/test_heapq.py
@@ -186,6 +186,11 @@
         self.assertFalse(sys.modules['heapq'] is self.module)
         self.assertTrue(hasattr(self.module.heapify, 'func_code'))
 
+    def test_islice_protection(self):
+        m = self.module
+        self.assertFalse(m.nsmallest(-1, [1]))
+        self.assertFalse(m.nlargest(-1, [1]))
+
 
 class TestHeapC(TestHeap):
     module = c_heapq

From noreply at buildbot.pypy.org  Thu Oct 27 17:09:32 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Oct 2011 17:09:32 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: fixes by rhyolite
Message-ID: <20111027150932.993AA820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: extradoc
Changeset: r3943:5ce3464a789c
Date: 2011-10-27 17:09 +0200
http://bitbucket.org/pypy/extradoc/changeset/5ce3464a789c/

Log:	fixes by rhyolite

diff --git a/blog/draft/faster-json.rst b/blog/draft/faster-json.rst
--- a/blog/draft/faster-json.rst
+++ b/blog/draft/faster-json.rst
@@ -11,7 +11,8 @@
 For this particular benchmark, the numbers are as follow. **Note that CPython by
 default uses the optimized C extension, while PyPy uses the pure Python one**.
 PyPy trunk contains another pure Python version which has been optimized
-specifically for the PyPy JIT, which is the subject of this post.
+specifically for the PyPy JIT. Detailed optimizations are described later in
+this post.
 
 The number reported is the time taken for the third run, when things are
 warmed up. Full session `here`_.
@@ -40,10 +41,13 @@
 
 A lot of performance critical stuff in Python world is already written in a hand
 optimized C. Writing C (especially when you interface with CPython C API) is
-ugly and takes significant effort so it's only true for places which are
-well separated enough, but still. People would expect PyPy to outperform
-C extensions. Fortunately it's possible, but requires a bit of effort on
-the programmer side as well.
+ugly and takes significant effort. This approach does not scale well when
+there is a lot of code to be written or when there is a very tight coupling
+between the part to be rewritten and the rest of the code. Still, people would
+expect PyPy to be better at "tasks" and not precisely at running equivalent
+code, hence a comparison between the C extension and the pure python version
+is sound. Fortunately it's possible to outperform the C extension, but requires
+a bit of effort on the programmer side as well.
 
 Often interface between the C and Python part is ugly
 -----------------------------------------------------
@@ -69,13 +73,13 @@
 -----------------------------------------------
 
 PyPy has a custom thing called ``__pypy__.builders.StringBuilder``. It has
-few features that make it much easier to optimize than other ways like
+a few features that make it much easier to optimize than other ways like
 ``str.join()`` or ``cStringIO``.
 
-* You can specify the start size. Helps a lot if you can even provide a rough
-  estimate on the size of the string (less copying)
-* Only append and build allowed. While string is built you can't seek or
-  do anything else. Once it's built you can never append any more.
+* You can specify the start size, which helps a lot if you can even provide
+  a rough estimate on the size of the string (less copying)
+* Only append and build are allowed. After the string is built you
+  can't seek or do anything else. Once it's built you can never append any more.
 * Unicode version available as well as ``__pypy__.builders.UnicodeBuilder``.
 
 Method calls are ok, immutable globals are ok

From noreply at buildbot.pypy.org  Thu Oct 27 17:13:25 2011
From: noreply at buildbot.pypy.org (edelsohn)
Date: Thu, 27 Oct 2011 17:13:25 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: More native English speaker
	phrases.
Message-ID: <20111027151325.EB767820C2@wyvern.cs.uni-duesseldorf.de>

Author: edelsohn
Branch: extradoc
Changeset: r3944:ae2dbf57dd1b
Date: 2011-10-27 11:13 -0400
http://bitbucket.org/pypy/extradoc/changeset/ae2dbf57dd1b/

Log:	More native English speaker phrases.

diff --git a/blog/draft/faster-json.rst b/blog/draft/faster-json.rst
--- a/blog/draft/faster-json.rst
+++ b/blog/draft/faster-json.rst
@@ -73,19 +73,19 @@
 -----------------------------------------------
 
 PyPy has a custom thing called ``__pypy__.builders.StringBuilder``. It has
-a few features that make it much easier to optimize than other ways like
+a few a features that make it much easier to optimize than other ways like
 ``str.join()`` or ``cStringIO``.
 
 * You can specify the start size, which helps a lot if you can even provide
   a rough estimate on the size of the string (less copying)
-* Only append and build are allowed. After the string is built you
-  can't seek or do anything else. Once it's built you can never append any more.
+* Only append and build are allowed. While  the string is being built you
+  can't seek or do anything else. After it's built you can never append any more.
 * Unicode version available as well as ``__pypy__.builders.UnicodeBuilder``.
 
 Method calls are ok, immutable globals are ok
 ---------------------------------------------
 
-PyPy's JIT seem to be good enough than at least in simple cases, calling
+PyPy's JIT seems to be good enough for at least the simple cases. Calling
 methods for common infrastructure or loading globals (instead of rebinding as
 locals) is fast enough and improves code readability.
 

From noreply at buildbot.pypy.org  Thu Oct 27 17:15:26 2011
From: noreply at buildbot.pypy.org (edelsohn)
Date: Thu, 27 Oct 2011 17:15:26 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: Another typo.
Message-ID: <20111027151526.4F78E820C2@wyvern.cs.uni-duesseldorf.de>

Author: edelsohn
Branch: extradoc
Changeset: r3945:b4c00aeae648
Date: 2011-10-27 11:15 -0400
http://bitbucket.org/pypy/extradoc/changeset/b4c00aeae648/

Log:	Another typo.

diff --git a/blog/draft/faster-json.rst b/blog/draft/faster-json.rst
--- a/blog/draft/faster-json.rst
+++ b/blog/draft/faster-json.rst
@@ -102,7 +102,7 @@
   for c in s:
     pass
 
-This is a PyPy performance bug and should be fixed, but on different branch ;-)
+This is a PyPy performance bug and should be fixed, but on a different branch ;-)
 
 PyPy's JIT is good
 --------------------------

From noreply at buildbot.pypy.org  Thu Oct 27 17:18:03 2011
From: noreply at buildbot.pypy.org (edelsohn)
Date: Thu, 27 Oct 2011 17:18:03 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: I'm on a roll.
Message-ID: <20111027151803.30C71820C2@wyvern.cs.uni-duesseldorf.de>

Author: edelsohn
Branch: extradoc
Changeset: r3946:a9917028d281
Date: 2011-10-27 11:17 -0400
http://bitbucket.org/pypy/extradoc/changeset/a9917028d281/

Log:	I'm on a roll.

diff --git a/blog/draft/faster-json.rst b/blog/draft/faster-json.rst
--- a/blog/draft/faster-json.rst
+++ b/blog/draft/faster-json.rst
@@ -55,8 +55,8 @@
 This is very clear if you look at json module as implemented in CPython's
 standard library. Not everything is in C (it would probably be just too
 much effort) and the interface to what is in C is guided via profiling not
-via what kind of interface makes sense. It's clear from CPython 2.6 to 2.7.
-Just adapting the code to interface with C made the Python version slower.
+by what kind of interface makes sense. This especially is evident comparing CPython 2.6 to 2.7.
+Just adapting the code to an interface with C made the Python version slower.
 Removing this clutter improves the readability a lot and improves PyPy's version
 a bit, although I don't have hard numbers.
 

From noreply at buildbot.pypy.org  Thu Oct 27 17:27:26 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Oct 2011 17:27:26 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: mention generators
Message-ID: <20111027152726.74036820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: extradoc
Changeset: r3947:f79ab024b2c1
Date: 2011-10-27 17:25 +0200
http://bitbucket.org/pypy/extradoc/changeset/f79ab024b2c1/

Log:	mention generators

diff --git a/blog/draft/faster-json.rst b/blog/draft/faster-json.rst
--- a/blog/draft/faster-json.rst
+++ b/blog/draft/faster-json.rst
@@ -89,6 +89,17 @@
 methods for common infrastructure or loading globals (instead of rebinding as
 locals) is fast enough and improves code readability.
 
+Generators are slower than they should be
+-----------------------------------------
+
+I changed the entire thing to simply call ``builder.append`` instead of
+yielding to the main loop where it would be gathered. This is kind of a PyPy
+bug that using generators extensively is slower, but a bit hard to fix.
+Especially in cases where there is relatively little data being passed around
+(few bytes), it makes sense to gather it first. If I were to implement an
+efficient version of ``iterencode``, I would probably handle chunks of
+predetermined size, about 1000 bytes instead of yielding data every few bytes.
+
 I must admit I worked around PyPy's performance bug
 ---------------------------------------------------
 

From noreply at buildbot.pypy.org  Thu Oct 27 17:27:27 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Oct 2011 17:27:27 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: expand jit is good section
Message-ID: <20111027152727.92E48820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: extradoc
Changeset: r3948:7e2623ff21cf
Date: 2011-10-27 17:27 +0200
http://bitbucket.org/pypy/extradoc/changeset/7e2623ff21cf/

Log:	expand jit is good section

diff --git a/blog/draft/faster-json.rst b/blog/draft/faster-json.rst
--- a/blog/draft/faster-json.rst
+++ b/blog/draft/faster-json.rst
@@ -119,8 +119,13 @@
 --------------------------
 
 I was pretty surprised, but the JIT actually did make stuff work nicely.
-It is possible to write code in Python and make it run fast, but you have
-to be a bit careful. Again, jitviewer is your friend.
+The changes that were done were relatively minor and straightforward, once
+the module was cleaned to the normal "pythonic" state.
+It is worth noting that it's possible to write code in Python and make it
+run really fast, but you have to be a bit careful. Again, jitviewer is your
+friend when determining why things are slow. I hope we can write more tools
+in the future that would more automatically guide people through potential
+performance pitfals.
 
 Cheers,
 fijal

From noreply at buildbot.pypy.org  Thu Oct 27 17:28:03 2011
From: noreply at buildbot.pypy.org (cfbolz)
Date: Thu, 27 Oct 2011 17:28:03 +0200 (CEST)
Subject: [pypy-commit] pypy int-tag-untag-as-operations: make tagged
 instances use the new operations as well
Message-ID: <20111027152803.59CAB820C2@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: int-tag-untag-as-operations
Changeset: r48509:d3b9577c4c08
Date: 2011-10-27 17:27 +0200
http://bitbucket.org/pypy/pypy/changeset/d3b9577c4c08/

Log:	make tagged instances use the new operations as well

diff --git a/pypy/rpython/lltypesystem/rtagged.py b/pypy/rpython/lltypesystem/rtagged.py
--- a/pypy/rpython/lltypesystem/rtagged.py
+++ b/pypy/rpython/lltypesystem/rtagged.py
@@ -4,6 +4,7 @@
 from pypy.rpython.lltypesystem.rclass import InstanceRepr, CLASSTYPE, ll_inst_type
 from pypy.rpython.lltypesystem.rclass import MissingRTypeAttribute
 from pypy.rpython.lltypesystem.rclass import ll_issubclass_const
+from pypy.rpython.lltypesystem.lloperation import llop
 from pypy.rpython.rmodel import TyperError, inputconst
 
 
@@ -43,18 +44,17 @@
         v_value = hop.inputarg(lltype.Signed, arg=1)
         c_one = hop.inputconst(lltype.Signed, 1)
         hop.exception_is_here()
-        v2 = hop.genop('int_add_ovf', [v_value, v_value],
+        v2 = hop.genop('int_tag_ovf', [v_value],
                        resulttype = lltype.Signed)
-        v2p1 = hop.genop('int_add', [v2, c_one],
-                         resulttype = lltype.Signed)
-        v_instance =  hop.genop('cast_int_to_ptr', [v2p1],
+        v_instance =  hop.genop('cast_int_to_ptr', [v2],
                                 resulttype = self.lowleveltype)
         return v_instance, False   # don't call __init__
 
     def convert_const_exact(self, value):
         self.setup()
         number = value.get_untagged_value()
-        return ll_int_to_unboxed(self.lowleveltype, number)
+        tagged = number * 2 + 1
+        return lltype.cast_int_to_ptr(self.lowleveltype, tagged)
 
     def getvalue_from_unboxed(self, llops, vinst):
         assert not self.is_parent
@@ -142,11 +142,9 @@
                                  minid, maxid, c_answer_if_unboxed)
 
 
-def ll_int_to_unboxed(PTRTYPE, value):
-    return lltype.cast_int_to_ptr(PTRTYPE, value*2+1)
 
 def ll_unboxed_to_int(p):
-    return lltype.cast_ptr_to_int(p) >> 1
+    return llop.int_untag(lltype.Signed, lltype.cast_ptr_to_int(p))
 
 def ll_unboxed_getclass_canbenone(instance, class_if_unboxed):
     if instance:

From noreply at buildbot.pypy.org  Thu Oct 27 17:36:40 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Oct 2011 17:36:40 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: add another para
Message-ID: <20111027153640.954BD820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: extradoc
Changeset: r3949:5c778250e982
Date: 2011-10-27 17:36 +0200
http://bitbucket.org/pypy/extradoc/changeset/5c778250e982/

Log:	add another para

diff --git a/blog/draft/faster-json.rst b/blog/draft/faster-json.rst
--- a/blog/draft/faster-json.rst
+++ b/blog/draft/faster-json.rst
@@ -89,6 +89,17 @@
 methods for common infrastructure or loading globals (instead of rebinding as
 locals) is fast enough and improves code readability.
 
+Copying is expensive
+--------------------
+
+If you use regular expressions replace, this would always copy a string as of
+now. If you know your regexp is simple, first try to match it if there is
+anything to replace in the first place. This is a pretty hard optimization to
+do automatically -- simply matching the regular expression can be too costly
+for it to make sense. In our particular example however, the regexp is really
+simple, checking ranges of characters. It also seems that this is by far the
+fastest way to escape characters as of now.
+
 Generators are slower than they should be
 -----------------------------------------
 

From noreply at buildbot.pypy.org  Thu Oct 27 17:37:47 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Oct 2011 17:37:47 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-multidim: setitem with slice - part one
Message-ID: <20111027153747.79B21820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: numpy-multidim
Changeset: r48510:61f36db28f06
Date: 2011-10-27 17:37 +0200
http://bitbucket.org/pypy/pypy/changeset/61f36db28f06/

Log:	setitem with slice - part one

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -10,9 +10,11 @@
 
 numpy_driver = jit.JitDriver(greens = ['signature'],
                              reds = ['result_size', 'i', 'self', 'result'])
-all_driver = jit.JitDriver(greens=['signature'], reds=['i', 'size', 'self', 'dtype'])
-any_driver = jit.JitDriver(greens=['signature'], reds=['i', 'size', 'self', 'dtype'])
-slice_driver = jit.JitDriver(greens=['signature'], reds=['i', 'j', 'step', 'stop', 'source', 'dest'])
+all_driver = jit.JitDriver(greens=['signature'], reds=['i', 'size', 'self',
+                                                       'dtype'])
+any_driver = jit.JitDriver(greens=['signature'], reds=['i', 'size', 'self',
+                                                       'dtype'])
+slice_driver = jit.JitDriver(greens=['signature'], reds=['i', 'self', 'source'])
 
 class BaseArray(Wrappable):
     _attrs_ = ["invalidates", "signature"]
@@ -304,55 +306,26 @@
 
     def descr_setitem(self, space, w_idx, w_value):
         self.invalidated()
-        if self._single_item_at_index(space, w_idx):
+        if self._single_item_result(space, w_idx):
             item = self._single_item_at_index(space, w_idx)
             self.get_concrete().setitem_w(space, item, w_value)
             return
-        xxx
-        if space.isinstance_w(w_idx, space.w_tuple):
-            length = space.len_w(w_idx)
-            if length > 1: # only one dimension for now.
-                raise OperationError(space.w_IndexError,
-                                     space.wrap("invalid index"))
-            if length == 0:
-                w_idx = space.newslice(space.wrap(0),
-                                      space.wrap(self.find_size()),
-                                      space.wrap(1))
-            else:
-                w_idx = space.getitem(w_idx, space.wrap(0))
-        start, stop, step, slice_length = space.decode_index4(w_idx,
-                                                              self.find_size())
-        if step == 0:
-            # Single index
-            self.get_concrete().setitem_w(space, start, w_value)
+        concrete = self.get_concrete()
+        if isinstance(w_value, BaseArray):
+            # for now we just copy if setting part of an array from
+            # part of itself. can be improved.
+            if (concrete.get_root_storage() ==
+                w_value.get_concrete().get_root_storage()):
+                w_value = space.call_function(space.gettypefor(BaseArray), w_value)
+                assert isinstance(w_value, BaseArray)
         else:
-            concrete = self.get_concrete()
-            if isinstance(w_value, BaseArray):
-                # for now we just copy if setting part of an array from
-                # part of itself. can be improved.
-                if (concrete.get_root_storage() ==
-                    w_value.get_concrete().get_root_storage()):
-                    w_value = space.call_function(space.gettypefor(BaseArray), w_value)
-                    assert isinstance(w_value, BaseArray)
-            else:
-                w_value = convert_to_array(space, w_value)
-            concrete.setslice(space, start, stop, step,
-                                               slice_length, w_value)
+            w_value = convert_to_array(space, w_value)
+        view = self._create_slice(space, w_idx)
+        view.setslice(space, w_value)
 
     def descr_mean(self, space):
         return space.wrap(space.float_w(self.descr_sum(space))/self.find_size())
 
-    def _sliceloop(self, start, stop, step, source, dest):
-        i = start
-        j = 0
-        while (step > 0 and i < stop) or (step < 0 and i > stop):
-            slice_driver.jit_merge_point(signature=source.signature, step=step,
-                                         stop=stop, i=i, j=j, source=source,
-                                         dest=dest)
-            dest.setitem(i, source.eval(j).convert_to(dest.find_dtype()))
-            j += 1
-            i += step
-
 def convert_to_array(space, w_obj):
     if isinstance(w_obj, BaseArray):
         return w_obj
@@ -557,13 +530,23 @@
     def find_dtype(self):
         return self.parent.find_dtype()
 
-    def setslice(self, space, start, stop, step, slice_length, arr):
-        xxx
-        start = self.calc_index(start)
-        if stop != -1:
-            stop = self.calc_index(stop)
-        step = self.step * step
-        self._sliceloop(start, stop, step, arr, self.parent)
+    def setslice(self, space, w_value):
+        assert isinstance(w_value, NDimArray)
+        if self.shape != w_value.shape:
+            raise OperationError(space.w_TypeError, space.wrap(
+                "wrong assignment"))
+        self._sliceloop(w_value)
+
+    def _sliceloop(self, source):
+        i = 0
+        while i < self.size:
+            slice_driver.jit_merge_point(signature=source.signature, i=i,
+                                         self=self, source=source)
+            self.setitem(i, source.eval(i).convert_to(self.find_dtype()))
+            i += 1
+
+    def setitem(self, item, value):
+        self.parent.setitem(self.calc_index(item), value)
 
     def len_of_shape(self):
         return self.parent.len_of_shape() - self.shape_reduction
@@ -644,9 +627,6 @@
         self.invalidated()
         self.dtype.setitem(self.storage, item, value)
 
-    def setslice(self, space, start, stop, step, slice_length, arr):
-        self._sliceloop(start, stop, step, arr, self)
-
     def __del__(self):
         lltype.free(self.storage, flavor='raw', track_allocation=False)
 
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -658,6 +658,12 @@
         assert a[0][1][1] == 13
         assert a[1][2][1] == 15
 
+    def test_setitem_slice(self):
+        import numpy
+        a = numpy.zeros((3, 4))
+        a[1] = [1, 2, 3, 4]
+        assert a[1, 2] == 3
+
 class AppTestSupport(object):
     def setup_class(cls):
         import struct

From noreply at buildbot.pypy.org  Thu Oct 27 17:47:28 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Oct 2011 17:47:28 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: some more fixes from rhyolite
Message-ID: <20111027154728.79ED0820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: extradoc
Changeset: r3950:f55956fe1956
Date: 2011-10-27 17:47 +0200
http://bitbucket.org/pypy/extradoc/changeset/f55956fe1956/

Log:	some more fixes from rhyolite

diff --git a/blog/draft/faster-json.rst b/blog/draft/faster-json.rst
--- a/blog/draft/faster-json.rst
+++ b/blog/draft/faster-json.rst
@@ -89,12 +89,13 @@
 methods for common infrastructure or loading globals (instead of rebinding as
 locals) is fast enough and improves code readability.
 
-Copying is expensive
---------------------
+String copying is expensive
+---------------------------
 
-If you use regular expressions replace, this would always copy a string as of
-now. If you know your regexp is simple, first try to match it if there is
-anything to replace in the first place. This is a pretty hard optimization to
+If you use ``re.sub``, the current implementation will always create a copy
+of the string even if there was no match to replace.
+If you know your regexp is simple, first try to check if there is
+anything to replace. This is a pretty hard optimization to
 do automatically -- simply matching the regular expression can be too costly
 for it to make sense. In our particular example however, the regexp is really
 simple, checking ranges of characters. It also seems that this is by far the

From noreply at buildbot.pypy.org  Thu Oct 27 18:40:34 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Thu, 27 Oct 2011 18:40:34 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-dicts: merged default
Message-ID: <20111027164034.185F9820C2@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: virtual-dicts
Changeset: r48511:5e267e0a16b0
Date: 2011-10-27 12:31 -0400
http://bitbucket.org/pypy/pypy/changeset/5e267e0a16b0/

Log:	merged default

diff --git a/lib-python/modified-2.7/heapq.py b/lib-python/modified-2.7/heapq.py
new file mode 100644
--- /dev/null
+++ b/lib-python/modified-2.7/heapq.py
@@ -0,0 +1,442 @@
+# -*- coding: latin-1 -*-
+
+"""Heap queue algorithm (a.k.a. priority queue).
+
+Heaps are arrays for which a[k] <= a[2*k+1] and a[k] <= a[2*k+2] for
+all k, counting elements from 0.  For the sake of comparison,
+non-existing elements are considered to be infinite.  The interesting
+property of a heap is that a[0] is always its smallest element.
+
+Usage:
+
+heap = []            # creates an empty heap
+heappush(heap, item) # pushes a new item on the heap
+item = heappop(heap) # pops the smallest item from the heap
+item = heap[0]       # smallest item on the heap without popping it
+heapify(x)           # transforms list into a heap, in-place, in linear time
+item = heapreplace(heap, item) # pops and returns smallest item, and adds
+                               # new item; the heap size is unchanged
+
+Our API differs from textbook heap algorithms as follows:
+
+- We use 0-based indexing.  This makes the relationship between the
+  index for a node and the indexes for its children slightly less
+  obvious, but is more suitable since Python uses 0-based indexing.
+
+- Our heappop() method returns the smallest item, not the largest.
+
+These two make it possible to view the heap as a regular Python list
+without surprises: heap[0] is the smallest item, and heap.sort()
+maintains the heap invariant!
+"""
+
+# Original code by Kevin O'Connor, augmented by Tim Peters and Raymond Hettinger
+
+__about__ = """Heap queues
+
+[explanation by Fran&#65533;ois Pinard]
+
+Heaps are arrays for which a[k] <= a[2*k+1] and a[k] <= a[2*k+2] for
+all k, counting elements from 0.  For the sake of comparison,
+non-existing elements are considered to be infinite.  The interesting
+property of a heap is that a[0] is always its smallest element.
+
+The strange invariant above is meant to be an efficient memory
+representation for a tournament.  The numbers below are `k', not a[k]:
+
+                                   0
+
+                  1                                 2
+
+          3               4                5               6
+
+      7       8       9       10      11      12      13      14
+
+    15 16   17 18   19 20   21 22   23 24   25 26   27 28   29 30
+
+
+In the tree above, each cell `k' is topping `2*k+1' and `2*k+2'.  In
+an usual binary tournament we see in sports, each cell is the winner
+over the two cells it tops, and we can trace the winner down the tree
+to see all opponents s/he had.  However, in many computer applications
+of such tournaments, we do not need to trace the history of a winner.
+To be more memory efficient, when a winner is promoted, we try to
+replace it by something else at a lower level, and the rule becomes
+that a cell and the two cells it tops contain three different items,
+but the top cell "wins" over the two topped cells.
+
+If this heap invariant is protected at all time, index 0 is clearly
+the overall winner.  The simplest algorithmic way to remove it and
+find the "next" winner is to move some loser (let's say cell 30 in the
+diagram above) into the 0 position, and then percolate this new 0 down
+the tree, exchanging values, until the invariant is re-established.
+This is clearly logarithmic on the total number of items in the tree.
+By iterating over all items, you get an O(n ln n) sort.
+
+A nice feature of this sort is that you can efficiently insert new
+items while the sort is going on, provided that the inserted items are
+not "better" than the last 0'th element you extracted.  This is
+especially useful in simulation contexts, where the tree holds all
+incoming events, and the "win" condition means the smallest scheduled
+time.  When an event schedule other events for execution, they are
+scheduled into the future, so they can easily go into the heap.  So, a
+heap is a good structure for implementing schedulers (this is what I
+used for my MIDI sequencer :-).
+
+Various structures for implementing schedulers have been extensively
+studied, and heaps are good for this, as they are reasonably speedy,
+the speed is almost constant, and the worst case is not much different
+than the average case.  However, there are other representations which
+are more efficient overall, yet the worst cases might be terrible.
+
+Heaps are also very useful in big disk sorts.  You most probably all
+know that a big sort implies producing "runs" (which are pre-sorted
+sequences, which size is usually related to the amount of CPU memory),
+followed by a merging passes for these runs, which merging is often
+very cleverly organised[1].  It is very important that the initial
+sort produces the longest runs possible.  Tournaments are a good way
+to that.  If, using all the memory available to hold a tournament, you
+replace and percolate items that happen to fit the current run, you'll
+produce runs which are twice the size of the memory for random input,
+and much better for input fuzzily ordered.
+
+Moreover, if you output the 0'th item on disk and get an input which
+may not fit in the current tournament (because the value "wins" over
+the last output value), it cannot fit in the heap, so the size of the
+heap decreases.  The freed memory could be cleverly reused immediately
+for progressively building a second heap, which grows at exactly the
+same rate the first heap is melting.  When the first heap completely
+vanishes, you switch heaps and start a new run.  Clever and quite
+effective!
+
+In a word, heaps are useful memory structures to know.  I use them in
+a few applications, and I think it is good to keep a `heap' module
+around. :-)
+
+--------------------
+[1] The disk balancing algorithms which are current, nowadays, are
+more annoying than clever, and this is a consequence of the seeking
+capabilities of the disks.  On devices which cannot seek, like big
+tape drives, the story was quite different, and one had to be very
+clever to ensure (far in advance) that each tape movement will be the
+most effective possible (that is, will best participate at
+"progressing" the merge).  Some tapes were even able to read
+backwards, and this was also used to avoid the rewinding time.
+Believe me, real good tape sorts were quite spectacular to watch!
+From all times, sorting has always been a Great Art! :-)
+"""
+
+__all__ = ['heappush', 'heappop', 'heapify', 'heapreplace', 'merge',
+           'nlargest', 'nsmallest', 'heappushpop']
+
+from itertools import islice, repeat, count, imap, izip, tee, chain
+from operator import itemgetter
+import bisect
+
+def heappush(heap, item):
+    """Push item onto heap, maintaining the heap invariant."""
+    heap.append(item)
+    _siftdown(heap, 0, len(heap)-1)
+
+def heappop(heap):
+    """Pop the smallest item off the heap, maintaining the heap invariant."""
+    lastelt = heap.pop()    # raises appropriate IndexError if heap is empty
+    if heap:
+        returnitem = heap[0]
+        heap[0] = lastelt
+        _siftup(heap, 0)
+    else:
+        returnitem = lastelt
+    return returnitem
+
+def heapreplace(heap, item):
+    """Pop and return the current smallest value, and add the new item.
+
+    This is more efficient than heappop() followed by heappush(), and can be
+    more appropriate when using a fixed-size heap.  Note that the value
+    returned may be larger than item!  That constrains reasonable uses of
+    this routine unless written as part of a conditional replacement:
+
+        if item > heap[0]:
+            item = heapreplace(heap, item)
+    """
+    returnitem = heap[0]    # raises appropriate IndexError if heap is empty
+    heap[0] = item
+    _siftup(heap, 0)
+    return returnitem
+
+def heappushpop(heap, item):
+    """Fast version of a heappush followed by a heappop."""
+    if heap and heap[0] < item:
+        item, heap[0] = heap[0], item
+        _siftup(heap, 0)
+    return item
+
+def heapify(x):
+    """Transform list into a heap, in-place, in O(len(heap)) time."""
+    n = len(x)
+    # Transform bottom-up.  The largest index there's any point to looking at
+    # is the largest with a child index in-range, so must have 2*i + 1 < n,
+    # or i < (n-1)/2.  If n is even = 2*j, this is (2*j-1)/2 = j-1/2 so
+    # j-1 is the largest, which is n//2 - 1.  If n is odd = 2*j+1, this is
+    # (2*j+1-1)/2 = j so j-1 is the largest, and that's again n//2-1.
+    for i in reversed(xrange(n//2)):
+        _siftup(x, i)
+
+def nlargest(n, iterable):
+    """Find the n largest elements in a dataset.
+
+    Equivalent to:  sorted(iterable, reverse=True)[:n]
+    """
+    if n < 0: # for consistency with the c impl
+        return []
+    it = iter(iterable)
+    result = list(islice(it, n))
+    if not result:
+        return result
+    heapify(result)
+    _heappushpop = heappushpop
+    for elem in it:
+        _heappushpop(result, elem)
+    result.sort(reverse=True)
+    return result
+
+def nsmallest(n, iterable):
+    """Find the n smallest elements in a dataset.
+
+    Equivalent to:  sorted(iterable)[:n]
+    """
+    if n < 0: # for consistency with the c impl
+        return []
+    if hasattr(iterable, '__len__') and n * 10 <= len(iterable):
+        # For smaller values of n, the bisect method is faster than a minheap.
+        # It is also memory efficient, consuming only n elements of space.
+        it = iter(iterable)
+        result = sorted(islice(it, 0, n))
+        if not result:
+            return result
+        insort = bisect.insort
+        pop = result.pop
+        los = result[-1]    # los --> Largest of the nsmallest
+        for elem in it:
+            if los <= elem:
+                continue
+            insort(result, elem)
+            pop()
+            los = result[-1]
+        return result
+    # An alternative approach manifests the whole iterable in memory but
+    # saves comparisons by heapifying all at once.  Also, saves time
+    # over bisect.insort() which has O(n) data movement time for every
+    # insertion.  Finding the n smallest of an m length iterable requires
+    #    O(m) + O(n log m) comparisons.
+    h = list(iterable)
+    heapify(h)
+    return map(heappop, repeat(h, min(n, len(h))))
+
+# 'heap' is a heap at all indices >= startpos, except possibly for pos.  pos
+# is the index of a leaf with a possibly out-of-order value.  Restore the
+# heap invariant.
+def _siftdown(heap, startpos, pos):
+    newitem = heap[pos]
+    # Follow the path to the root, moving parents down until finding a place
+    # newitem fits.
+    while pos > startpos:
+        parentpos = (pos - 1) >> 1
+        parent = heap[parentpos]
+        if newitem < parent:
+            heap[pos] = parent
+            pos = parentpos
+            continue
+        break
+    heap[pos] = newitem
+
+# The child indices of heap index pos are already heaps, and we want to make
+# a heap at index pos too.  We do this by bubbling the smaller child of
+# pos up (and so on with that child's children, etc) until hitting a leaf,
+# then using _siftdown to move the oddball originally at index pos into place.
+#
+# We *could* break out of the loop as soon as we find a pos where newitem <=
+# both its children, but turns out that's not a good idea, and despite that
+# many books write the algorithm that way.  During a heap pop, the last array
+# element is sifted in, and that tends to be large, so that comparing it
+# against values starting from the root usually doesn't pay (= usually doesn't
+# get us out of the loop early).  See Knuth, Volume 3, where this is
+# explained and quantified in an exercise.
+#
+# Cutting the # of comparisons is important, since these routines have no
+# way to extract "the priority" from an array element, so that intelligence
+# is likely to be hiding in custom __cmp__ methods, or in array elements
+# storing (priority, record) tuples.  Comparisons are thus potentially
+# expensive.
+#
+# On random arrays of length 1000, making this change cut the number of
+# comparisons made by heapify() a little, and those made by exhaustive
+# heappop() a lot, in accord with theory.  Here are typical results from 3
+# runs (3 just to demonstrate how small the variance is):
+#
+# Compares needed by heapify     Compares needed by 1000 heappops
+# --------------------------     --------------------------------
+# 1837 cut to 1663               14996 cut to 8680
+# 1855 cut to 1659               14966 cut to 8678
+# 1847 cut to 1660               15024 cut to 8703
+#
+# Building the heap by using heappush() 1000 times instead required
+# 2198, 2148, and 2219 compares:  heapify() is more efficient, when
+# you can use it.
+#
+# The total compares needed by list.sort() on the same lists were 8627,
+# 8627, and 8632 (this should be compared to the sum of heapify() and
+# heappop() compares):  list.sort() is (unsurprisingly!) more efficient
+# for sorting.
+
+def _siftup(heap, pos):
+    endpos = len(heap)
+    startpos = pos
+    newitem = heap[pos]
+    # Bubble up the smaller child until hitting a leaf.
+    childpos = 2*pos + 1    # leftmost child position
+    while childpos < endpos:
+        # Set childpos to index of smaller child.
+        rightpos = childpos + 1
+        if rightpos < endpos and not heap[childpos] < heap[rightpos]:
+            childpos = rightpos
+        # Move the smaller child up.
+        heap[pos] = heap[childpos]
+        pos = childpos
+        childpos = 2*pos + 1
+    # The leaf at pos is empty now.  Put newitem there, and bubble it up
+    # to its final resting place (by sifting its parents down).
+    heap[pos] = newitem
+    _siftdown(heap, startpos, pos)
+
+# If available, use C implementation
+try:
+    from _heapq import *
+except ImportError:
+    pass
+
+def merge(*iterables):
+    '''Merge multiple sorted inputs into a single sorted output.
+
+    Similar to sorted(itertools.chain(*iterables)) but returns a generator,
+    does not pull the data into memory all at once, and assumes that each of
+    the input streams is already sorted (smallest to largest).
+
+    >>> list(merge([1,3,5,7], [0,2,4,8], [5,10,15,20], [], [25]))
+    [0, 1, 2, 3, 4, 5, 5, 7, 8, 10, 15, 20, 25]
+
+    '''
+    _heappop, _heapreplace, _StopIteration = heappop, heapreplace, StopIteration
+
+    h = []
+    h_append = h.append
+    for itnum, it in enumerate(map(iter, iterables)):
+        try:
+            next = it.next
+            h_append([next(), itnum, next])
+        except _StopIteration:
+            pass
+    heapify(h)
+
+    while 1:
+        try:
+            while 1:
+                v, itnum, next = s = h[0]   # raises IndexError when h is empty
+                yield v
+                s[0] = next()               # raises StopIteration when exhausted
+                _heapreplace(h, s)          # restore heap condition
+        except _StopIteration:
+            _heappop(h)                     # remove empty iterator
+        except IndexError:
+            return
+
+# Extend the implementations of nsmallest and nlargest to use a key= argument
+_nsmallest = nsmallest
+def nsmallest(n, iterable, key=None):
+    """Find the n smallest elements in a dataset.
+
+    Equivalent to:  sorted(iterable, key=key)[:n]
+    """
+    # Short-cut for n==1 is to use min() when len(iterable)>0
+    if n == 1:
+        it = iter(iterable)
+        head = list(islice(it, 1))
+        if not head:
+            return []
+        if key is None:
+            return [min(chain(head, it))]
+        return [min(chain(head, it), key=key)]
+
+    # When n>=size, it's faster to use sort()
+    try:
+        size = len(iterable)
+    except (TypeError, AttributeError):
+        pass
+    else:
+        if n >= size:
+            return sorted(iterable, key=key)[:n]
+
+    # When key is none, use simpler decoration
+    if key is None:
+        it = izip(iterable, count())                        # decorate
+        result = _nsmallest(n, it)
+        return map(itemgetter(0), result)                   # undecorate
+
+    # General case, slowest method
+    in1, in2 = tee(iterable)
+    it = izip(imap(key, in1), count(), in2)                 # decorate
+    result = _nsmallest(n, it)
+    return map(itemgetter(2), result)                       # undecorate
+
+_nlargest = nlargest
+def nlargest(n, iterable, key=None):
+    """Find the n largest elements in a dataset.
+
+    Equivalent to:  sorted(iterable, key=key, reverse=True)[:n]
+    """
+
+    # Short-cut for n==1 is to use max() when len(iterable)>0
+    if n == 1:
+        it = iter(iterable)
+        head = list(islice(it, 1))
+        if not head:
+            return []
+        if key is None:
+            return [max(chain(head, it))]
+        return [max(chain(head, it), key=key)]
+
+    # When n>=size, it's faster to use sort()
+    try:
+        size = len(iterable)
+    except (TypeError, AttributeError):
+        pass
+    else:
+        if n >= size:
+            return sorted(iterable, key=key, reverse=True)[:n]
+
+    # When key is none, use simpler decoration
+    if key is None:
+        it = izip(iterable, count(0,-1))                    # decorate
+        result = _nlargest(n, it)
+        return map(itemgetter(0), result)                   # undecorate
+
+    # General case, slowest method
+    in1, in2 = tee(iterable)
+    it = izip(imap(key, in1), count(0,-1), in2)             # decorate
+    result = _nlargest(n, it)
+    return map(itemgetter(2), result)                       # undecorate
+
+if __name__ == "__main__":
+    # Simple sanity test
+    heap = []
+    data = [1, 3, 5, 7, 9, 2, 4, 6, 8, 0]
+    for item in data:
+        heappush(heap, item)
+    sort = []
+    while heap:
+        sort.append(heappop(heap))
+    print sort
+
+    import doctest
+    doctest.testmod()
diff --git a/lib-python/modified-2.7/test/test_heapq.py b/lib-python/modified-2.7/test/test_heapq.py
--- a/lib-python/modified-2.7/test/test_heapq.py
+++ b/lib-python/modified-2.7/test/test_heapq.py
@@ -186,6 +186,11 @@
         self.assertFalse(sys.modules['heapq'] is self.module)
         self.assertTrue(hasattr(self.module.heapify, 'func_code'))
 
+    def test_islice_protection(self):
+        m = self.module
+        self.assertFalse(m.nsmallest(-1, [1]))
+        self.assertFalse(m.nlargest(-1, [1]))
+
 
 class TestHeapC(TestHeap):
     module = c_heapq
diff --git a/pypy/jit/backend/llsupport/regalloc.py b/pypy/jit/backend/llsupport/regalloc.py
--- a/pypy/jit/backend/llsupport/regalloc.py
+++ b/pypy/jit/backend/llsupport/regalloc.py
@@ -178,8 +178,6 @@
         cur_max_age = -1
         candidate = None
         for next in self.reg_bindings:
-            if isinstance(next, TempBox):
-                continue
             reg = self.reg_bindings[next]
             if next in forbidden_vars:
                 continue
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1596,11 +1596,26 @@
     genop_getarrayitem_gc_pure = genop_getarrayitem_gc
     genop_getarrayitem_raw = genop_getarrayitem_gc
 
+    def _get_interiorfield_addr(self, temp_loc, index_loc, itemsize_loc,
+                                base_loc, ofs_loc):
+        assert isinstance(itemsize_loc, ImmedLoc)
+        if isinstance(index_loc, ImmedLoc):
+            temp_loc = imm(index_loc.value * itemsize_loc.value)
+        else:
+            # XXX should not use IMUL in most cases
+            assert isinstance(temp_loc, RegLoc)
+            assert isinstance(index_loc, RegLoc)
+            self.mc.IMUL_rri(temp_loc.value, index_loc.value,
+                             itemsize_loc.value)
+        assert isinstance(ofs_loc, ImmedLoc)
+        return AddressLoc(base_loc, temp_loc, 0, ofs_loc.value)
+
     def genop_getinteriorfield_gc(self, op, arglocs, resloc):
-        base_loc, ofs_loc, itemsize_loc, fieldsize_loc, index_loc, sign_loc = arglocs
-        # XXX should not use IMUL in most cases
-        self.mc.IMUL(index_loc, itemsize_loc)
-        src_addr = AddressLoc(base_loc, index_loc, 0, ofs_loc.value)
+        (base_loc, ofs_loc, itemsize_loc, fieldsize_loc,
+            index_loc, sign_loc) = arglocs
+        src_addr = self._get_interiorfield_addr(resloc, index_loc,
+                                                itemsize_loc, base_loc,
+                                                ofs_loc)
         self.load_from_mem(resloc, src_addr, fieldsize_loc, sign_loc)
 
 
@@ -1611,13 +1626,11 @@
         self.save_into_mem(dest_addr, value_loc, size_loc)
 
     def genop_discard_setinteriorfield_gc(self, op, arglocs):
-        base_loc, ofs_loc, itemsize_loc, fieldsize_loc, index_loc, value_loc = arglocs
-        # XXX should not use IMUL in most cases
-        if isinstance(index_loc, ImmedLoc):
-            index_loc = imm(index_loc.value * itemsize_loc.value)
-        else:
-            self.mc.IMUL(index_loc, itemsize_loc)
-        dest_addr = AddressLoc(base_loc, index_loc, 0, ofs_loc.value)
+        (base_loc, ofs_loc, itemsize_loc, fieldsize_loc,
+            index_loc, temp_loc, value_loc) = arglocs
+        dest_addr = self._get_interiorfield_addr(temp_loc, index_loc,
+                                                 itemsize_loc, base_loc,
+                                                 ofs_loc)
         self.save_into_mem(dest_addr, value_loc, fieldsize_loc)
 
     def genop_discard_setarrayitem_gc(self, op, arglocs):
diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py
--- a/pypy/jit/backend/x86/regalloc.py
+++ b/pypy/jit/backend/x86/regalloc.py
@@ -1046,16 +1046,26 @@
             need_lower_byte = True
         else:
             need_lower_byte = False
-        base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
-        tempvar = TempBox()
-        index_loc = self.rm.force_result_in_reg(tempvar, op.getarg(1), args)
-        # we're free to modify index now
-        value_loc = self.make_sure_var_in_reg(op.getarg(2), args,
+        box_base, box_index, box_value = args
+        base_loc = self.rm.make_sure_var_in_reg(box_base, args)
+        index_loc = self.rm.make_sure_var_in_reg(box_index, args)
+        value_loc = self.make_sure_var_in_reg(box_value, args,
                                               need_lower_byte=need_lower_byte)
-        self.rm.possibly_free_var(tempvar)
-        self.possibly_free_vars(args)
+        # If 'index_loc' is not an immediate, then we need a 'temp_loc' that
+        # is a register whose value will be destroyed.  It's fine to destroy
+        # the same register as 'index_loc', but not the other ones.
+        self.rm.possibly_free_var(box_index)
+        if not isinstance(index_loc, ImmedLoc):
+            tempvar = TempBox()
+            temp_loc = self.rm.force_allocate_reg(tempvar, [box_base,
+                                                            box_value])
+            self.rm.possibly_free_var(tempvar)
+        else:
+            temp_loc = None
+        self.rm.possibly_free_var(box_base)
+        self.possibly_free_var(box_value)
         self.PerformDiscard(op, [base_loc, ofs, itemsize, fieldsize,
-                                 index_loc, value_loc])
+                                 index_loc, temp_loc, value_loc])
 
     def consider_strsetitem(self, op):
         args = op.getarglist()
@@ -1126,13 +1136,14 @@
         else:
             sign_loc = imm0
         args = op.getarglist()
-        tmpvar = TempBox()
         base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
-        index_loc = self.rm.force_result_in_reg(tmpvar, op.getarg(1),
-                                                args)
-        self.rm.possibly_free_vars_for_op(op)
-        self.rm.possibly_free_var(tmpvar)
-        result_loc = self.force_allocate_reg(op.result)
+        index_loc = self.rm.make_sure_var_in_reg(op.getarg(1), args)
+        # 'base' and 'index' are put in two registers (or one if 'index'
+        # is an immediate).  'result' can be in the same register as
+        # 'index' but must be in a different register than 'base'.
+        self.rm.possibly_free_var(op.getarg(1))
+        result_loc = self.force_allocate_reg(op.result, [op.getarg(0)])
+        self.rm.possibly_free_var(op.getarg(0))
         self.Perform(op, [base_loc, ofs, itemsize, fieldsize,
                           index_loc, sign_loc], result_loc)
 

From noreply at buildbot.pypy.org  Thu Oct 27 18:40:35 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Thu, 27 Oct 2011 18:40:35 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-dicts: optimizeopt test for this
Message-ID: <20111027164035.4AC58820C2@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: virtual-dicts
Changeset: r48512:4aa0b3d16e34
Date: 2011-10-27 12:40 -0400
http://bitbucket.org/pypy/pypy/changeset/4aa0b3d16e34/

Log:	optimizeopt test for this

diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -935,7 +935,6 @@
         """
         self.optimize_loop(ops, expected)
 
-
     def test_virtual_constant_isnonnull(self):
         ops = """
         [i0]
@@ -951,6 +950,32 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_virtual_array_of_struct(self):
+        ops = """
+        [f0, f1, f2, f3]
+        p0 = new_array(2, descr=complexarraydescr)
+        setinteriorfield_gc(p0, 0, f0, descr=complexrealdescr)
+        setinteriorfield_gc(p0, 0, f1, descr=compleximagdescr)
+        setinteriorfield_gc(p0, 1, f2, descr=complexrealdescr)
+        setinteriorfield_gc(p0, 1, f3, descr=compleximagdescr)
+        f4 = getinteriorfield_gc(p0, 0, descr=complexrealdescr)
+        f5 = getinteriorfield_gc(p0, 1, descr=complexrealdescr)
+        f6 = float_mul(f4, f5)
+        f7 = getinteriorfield_gc(p0, 0, descr=compleximagdescr)
+        f8 = getinteriorfield_gc(p0, 1, descr=compleximagdescr)
+        f9 = float_mul(f7, f8)
+        f10 = float_add(f6, f9)
+        finish(f10)
+        """
+        expected = """
+        [f0, f1, f2, f3]
+        f4 = float_mul(f0, f2)
+        f5 = float_mul(f1, f3)
+        f6 = float_add(f4, f5)
+        finish(f6)
+        """
+        self.optimize_loop(ops, expected)
+
     def test_nonvirtual_1(self):
         ops = """
         [i]
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_util.py b/pypy/jit/metainterp/optimizeopt/test/test_util.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_util.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_util.py
@@ -185,6 +185,18 @@
              EffectInfo([], [arraydescr], [], [arraydescr],
                         oopspecindex=EffectInfo.OS_ARRAYCOPY))
 
+
+    # array of structs (complex data)
+    complexarray = lltype.GcArray(
+        lltype.Struct("complex",
+            ("real", lltype.Float),
+            ("imag", lltype.Float),
+        )
+    )
+    complexarraydescr = cpu.arraydescrof(complexarray)
+    complexrealdescr = cpu.interiorfielddescrof(complexarray, "real")
+    compleximagdescr = cpu.interiorfielddescrof(complexarray, "imag")
+
     for _name, _os in [
         ('strconcatdescr',               'OS_STR_CONCAT'),
         ('strslicedescr',                'OS_STR_SLICE'),
@@ -240,7 +252,7 @@
 ##    def get_class_of_box(self, box):
 ##        root = box.getref(ootype.ROOT)
 ##        return ootype.classof(root)
-    
+
 ##    cpu = runner.OOtypeCPU(None)
 ##    NODE = ootype.Instance('NODE', ootype.ROOT, {})
 ##    NODE._add_fields({'value': ootype.Signed,

From noreply at buildbot.pypy.org  Thu Oct 27 18:43:14 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Thu, 27 Oct 2011 18:43:14 +0200 (CEST)
Subject: [pypy-commit] pypy virtual-dicts: close for merge
Message-ID: <20111027164314.61C6E820C2@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: virtual-dicts
Changeset: r48513:9e7581b504d9
Date: 2011-10-27 12:41 -0400
http://bitbucket.org/pypy/pypy/changeset/9e7581b504d9/

Log:	close for merge


From noreply at buildbot.pypy.org  Thu Oct 27 18:43:15 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Thu, 27 Oct 2011 18:43:15 +0200 (CEST)
Subject: [pypy-commit] pypy default: merged virtual-dicts branch,
 this branch makes it possible for array of structs to be
 virtualized, and allows inlining into a few rdict functions,
 which in practice allows dicts where all the operations on them are with
 constant keys to remain virtual. in python code this is useful for things
 like "some str" % {"dict": "value"},
 or g(x=x) with g(**kwargs): return kwargs['x']
Message-ID: <20111027164315.D4E4F820C2@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r48514:96310036438c
Date: 2011-10-27 12:42 -0400
http://bitbucket.org/pypy/pypy/changeset/96310036438c/

Log:	merged virtual-dicts branch, this branch makes it possible for array
	of structs to be virtualized, and allows inlining into a few rdict
	functions, which in practice allows dicts where all the operations
	on them are with constant keys to remain virtual. in python code
	this is useful for things like "some str" % {"dict": "value"}, or
	g(x=x) with g(**kwargs): return kwargs['x']

diff --git a/pypy/jit/backend/llsupport/descr.py b/pypy/jit/backend/llsupport/descr.py
--- a/pypy/jit/backend/llsupport/descr.py
+++ b/pypy/jit/backend/llsupport/descr.py
@@ -281,6 +281,9 @@
     def is_float_field(self):
         return self.fielddescr.is_float_field()
 
+    def sort_key(self):
+        return self.fielddescr.sort_key()
+
     def repr_of_descr(self):
         return '<InteriorFieldDescr %s>' % self.fielddescr.repr_of_descr()
 
diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -844,6 +844,10 @@
         if self._is_gc(op.args[0]):
             return op
 
+    def rewrite_op_cast_opaque_ptr(self, op):
+        # None causes the result of this op to get aliased to op.args[0]
+        return [SpaceOperation('mark_opaque_ptr', op.args, None), None]
+
     def rewrite_op_force_cast(self, op):
         v_arg = op.args[0]
         v_result = op.result
diff --git a/pypy/jit/codewriter/test/test_jtransform.py b/pypy/jit/codewriter/test/test_jtransform.py
--- a/pypy/jit/codewriter/test/test_jtransform.py
+++ b/pypy/jit/codewriter/test/test_jtransform.py
@@ -1128,3 +1128,16 @@
                         varoftype(lltype.Signed))
     tr = Transformer(None, None)
     raises(NotImplementedError, tr.rewrite_operation, op)
+
+def test_cast_opaque_ptr():
+    S = lltype.GcStruct("S", ("x", lltype.Signed))
+    v1 = varoftype(lltype.Ptr(S))
+    v2 = varoftype(lltype.Ptr(rclass.OBJECT))
+
+    op = SpaceOperation('cast_opaque_ptr', [v1], v2)
+    tr = Transformer()
+    [op1, op2] = tr.rewrite_operation(op)
+    assert op1.opname == 'mark_opaque_ptr'
+    assert op1.args == [v1]
+    assert op1.result is None
+    assert op2 is None
\ No newline at end of file
diff --git a/pypy/jit/metainterp/blackhole.py b/pypy/jit/metainterp/blackhole.py
--- a/pypy/jit/metainterp/blackhole.py
+++ b/pypy/jit/metainterp/blackhole.py
@@ -505,9 +505,6 @@
     @arguments("r", "r", returns="i")
     def bhimpl_instance_ptr_ne(a, b):
         return a != b
-    @arguments("r", returns="r")
-    def bhimpl_cast_opaque_ptr(a):
-        return a
     @arguments("r", returns="i")
     def bhimpl_cast_ptr_to_int(a):
         i = lltype.cast_ptr_to_int(a)
@@ -518,6 +515,10 @@
         ll_assert((i & 1) == 1, "bhimpl_cast_int_to_ptr: not an odd int")
         return lltype.cast_int_to_ptr(llmemory.GCREF, i)
 
+    @arguments("r")
+    def bhimpl_mark_opaque_ptr(a):
+        pass
+
     @arguments("i", returns="i")
     def bhimpl_int_copy(a):
         return a
diff --git a/pypy/jit/metainterp/heapcache.py b/pypy/jit/metainterp/heapcache.py
--- a/pypy/jit/metainterp/heapcache.py
+++ b/pypy/jit/metainterp/heapcache.py
@@ -34,7 +34,6 @@
         self.clear_caches(opnum, descr, argboxes)
 
     def mark_escaped(self, opnum, argboxes):
-        idx = 0
         if opnum == rop.SETFIELD_GC:
             assert len(argboxes) == 2
             box, valuebox = argboxes
@@ -42,8 +41,20 @@
                 self.dependencies.setdefault(box, []).append(valuebox)
             else:
                 self._escape(valuebox)
-        # GETFIELD_GC doesn't escape it's argument
-        elif opnum != rop.GETFIELD_GC:
+        elif opnum == rop.SETARRAYITEM_GC:
+            assert len(argboxes) == 3
+            box, indexbox, valuebox = argboxes
+            if self.is_unescaped(box) and self.is_unescaped(valuebox):
+                self.dependencies.setdefault(box, []).append(valuebox)
+            else:
+                self._escape(valuebox)
+        # GETFIELD_GC, MARK_OPAQUE_PTR, PTR_EQ, and PTR_NE don't escape their
+        # arguments
+        elif (opnum != rop.GETFIELD_GC and
+              opnum != rop.MARK_OPAQUE_PTR and
+              opnum != rop.PTR_EQ and
+              opnum != rop.PTR_NE):
+            idx = 0
             for box in argboxes:
                 # setarrayitem_gc don't escape its first argument
                 if not (idx == 0 and opnum in [rop.SETARRAYITEM_GC]):
@@ -60,13 +71,13 @@
                 self._escape(dep)
 
     def clear_caches(self, opnum, descr, argboxes):
-        if opnum == rop.SETFIELD_GC:
-            return
-        if opnum == rop.SETARRAYITEM_GC:
-            return
-        if opnum == rop.SETFIELD_RAW:
-            return
-        if opnum == rop.SETARRAYITEM_RAW:
+        if (opnum == rop.SETFIELD_GC or
+            opnum == rop.SETARRAYITEM_GC or
+            opnum == rop.SETFIELD_RAW or
+            opnum == rop.SETARRAYITEM_RAW or
+            opnum == rop.SETINTERIORFIELD_GC or
+            opnum == rop.COPYSTRCONTENT or
+            opnum == rop.COPYUNICODECONTENT):
             return
         if rop._OVF_FIRST <= opnum <= rop._OVF_LAST:
             return
@@ -75,9 +86,9 @@
         if opnum == rop.CALL or opnum == rop.CALL_LOOPINVARIANT:
             effectinfo = descr.get_extra_info()
             ef = effectinfo.extraeffect
-            if ef == effectinfo.EF_LOOPINVARIANT or \
-               ef == effectinfo.EF_ELIDABLE_CANNOT_RAISE or \
-               ef == effectinfo.EF_ELIDABLE_CAN_RAISE:
+            if (ef == effectinfo.EF_LOOPINVARIANT or
+                ef == effectinfo.EF_ELIDABLE_CANNOT_RAISE or
+                ef == effectinfo.EF_ELIDABLE_CAN_RAISE):
                 return
             # A special case for ll_arraycopy, because it is so common, and its
             # effects are so well defined.
diff --git a/pypy/jit/metainterp/optimizeopt/optimizer.py b/pypy/jit/metainterp/optimizeopt/optimizer.py
--- a/pypy/jit/metainterp/optimizeopt/optimizer.py
+++ b/pypy/jit/metainterp/optimizeopt/optimizer.py
@@ -209,13 +209,19 @@
     def setfield(self, ofs, value):
         raise NotImplementedError
 
+    def getlength(self):
+        raise NotImplementedError
+
     def getitem(self, index):
         raise NotImplementedError
 
-    def getlength(self):
+    def setitem(self, index, value):
         raise NotImplementedError
 
-    def setitem(self, index, value):
+    def getinteriorfield(self, index, ofs, default):
+        raise NotImplementedError
+
+    def setinteriorfield(self, index, ofs, value):
         raise NotImplementedError
 
 
@@ -283,11 +289,11 @@
             return self.optimizer.optpure.has_pure_result(opnum, args, descr)
         return False
 
-    def get_pure_result(self, key):    
+    def get_pure_result(self, key):
         if self.optimizer.optpure:
             return self.optimizer.optpure.get_pure_result(key)
         return None
-    
+
     def setup(self):
         pass
 
@@ -524,7 +530,7 @@
 
     def replace_op(self, old_op, new_op):
         # XXX: Do we want to cache indexes to prevent search?
-        i = len(self._newoperations) 
+        i = len(self._newoperations)
         while i > 0:
             i -= 1
             if self._newoperations[i] is old_op:
diff --git a/pypy/jit/metainterp/optimizeopt/rewrite.py b/pypy/jit/metainterp/optimizeopt/rewrite.py
--- a/pypy/jit/metainterp/optimizeopt/rewrite.py
+++ b/pypy/jit/metainterp/optimizeopt/rewrite.py
@@ -465,10 +465,9 @@
                                         args = [op.getarg(0), ConstInt(highest_bit(val))])
         self.emit_operation(op)
 
-    def optimize_CAST_OPAQUE_PTR(self, op):
+    def optimize_MARK_OPAQUE_PTR(self, op):
         value = self.getvalue(op.getarg(0))
         self.optimizer.opaque_pointers[value] = True
-        self.make_equal_to(op.result, value)
 
     def optimize_CAST_PTR_TO_INT(self, op):
         self.pure(rop.CAST_INT_TO_PTR, [op.result], op.getarg(0))
diff --git a/pypy/jit/metainterp/optimizeopt/simplify.py b/pypy/jit/metainterp/optimizeopt/simplify.py
--- a/pypy/jit/metainterp/optimizeopt/simplify.py
+++ b/pypy/jit/metainterp/optimizeopt/simplify.py
@@ -25,7 +25,8 @@
         #     but it's a bit hard to implement robustly if heap.py is also run
         pass
 
-    optimize_CAST_OPAQUE_PTR = optimize_VIRTUAL_REF
+    def optimize_MARK_OPAQUE_PTR(self, op):
+        pass
 
 
 dispatch_opt = make_dispatcher_method(OptSimplify, 'optimize_',
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -935,7 +935,6 @@
         """
         self.optimize_loop(ops, expected)
 
-
     def test_virtual_constant_isnonnull(self):
         ops = """
         [i0]
@@ -951,6 +950,32 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_virtual_array_of_struct(self):
+        ops = """
+        [f0, f1, f2, f3]
+        p0 = new_array(2, descr=complexarraydescr)
+        setinteriorfield_gc(p0, 0, f0, descr=complexrealdescr)
+        setinteriorfield_gc(p0, 0, f1, descr=compleximagdescr)
+        setinteriorfield_gc(p0, 1, f2, descr=complexrealdescr)
+        setinteriorfield_gc(p0, 1, f3, descr=compleximagdescr)
+        f4 = getinteriorfield_gc(p0, 0, descr=complexrealdescr)
+        f5 = getinteriorfield_gc(p0, 1, descr=complexrealdescr)
+        f6 = float_mul(f4, f5)
+        f7 = getinteriorfield_gc(p0, 0, descr=compleximagdescr)
+        f8 = getinteriorfield_gc(p0, 1, descr=compleximagdescr)
+        f9 = float_mul(f7, f8)
+        f10 = float_add(f6, f9)
+        finish(f10)
+        """
+        expected = """
+        [f0, f1, f2, f3]
+        f4 = float_mul(f0, f2)
+        f5 = float_mul(f1, f3)
+        f6 = float_add(f4, f5)
+        finish(f6)
+        """
+        self.optimize_loop(ops, expected)
+
     def test_nonvirtual_1(self):
         ops = """
         [i]
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_util.py b/pypy/jit/metainterp/optimizeopt/test/test_util.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_util.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_util.py
@@ -185,6 +185,18 @@
              EffectInfo([], [arraydescr], [], [arraydescr],
                         oopspecindex=EffectInfo.OS_ARRAYCOPY))
 
+
+    # array of structs (complex data)
+    complexarray = lltype.GcArray(
+        lltype.Struct("complex",
+            ("real", lltype.Float),
+            ("imag", lltype.Float),
+        )
+    )
+    complexarraydescr = cpu.arraydescrof(complexarray)
+    complexrealdescr = cpu.interiorfielddescrof(complexarray, "real")
+    compleximagdescr = cpu.interiorfielddescrof(complexarray, "imag")
+
     for _name, _os in [
         ('strconcatdescr',               'OS_STR_CONCAT'),
         ('strslicedescr',                'OS_STR_SLICE'),
@@ -240,7 +252,7 @@
 ##    def get_class_of_box(self, box):
 ##        root = box.getref(ootype.ROOT)
 ##        return ootype.classof(root)
-    
+
 ##    cpu = runner.OOtypeCPU(None)
 ##    NODE = ootype.Instance('NODE', ootype.ROOT, {})
 ##    NODE._add_fields({'value': ootype.Signed,
diff --git a/pypy/jit/metainterp/optimizeopt/virtualize.py b/pypy/jit/metainterp/optimizeopt/virtualize.py
--- a/pypy/jit/metainterp/optimizeopt/virtualize.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualize.py
@@ -271,6 +271,69 @@
     def _make_virtual(self, modifier):
         return modifier.make_varray(self.arraydescr)
 
+class VArrayStructValue(AbstractVirtualValue):
+    def __init__(self, arraydescr, size, keybox, source_op=None):
+        AbstractVirtualValue.__init__(self, keybox, source_op)
+        self.arraydescr = arraydescr
+        self._items = [{} for _ in xrange(size)]
+
+    def getlength(self):
+        return len(self._items)
+
+    def getinteriorfield(self, index, ofs, default):
+        return self._items[index].get(ofs, default)
+
+    def setinteriorfield(self, index, ofs, itemvalue):
+        assert isinstance(itemvalue, optimizer.OptValue)
+        self._items[index][ofs] = itemvalue
+
+    def _really_force(self, optforce):
+        assert self.source_op is not None
+        if not we_are_translated():
+            self.source_op.name = 'FORCE ' + self.source_op.name
+        optforce.emit_operation(self.source_op)
+        self.box = box = self.source_op.result
+        for index in range(len(self._items)):
+            for descr, value in self._items[index].iteritems():
+                subbox = value.force_box(optforce)
+                op = ResOperation(rop.SETINTERIORFIELD_GC,
+                    [box, ConstInt(index), subbox], None, descr=descr
+                )
+                optforce.emit_operation(op)
+
+    def _get_list_of_descrs(self):
+        descrs = []
+        for item in self._items:
+            item_descrs = item.keys()
+            sort_descrs(item_descrs)
+            descrs.append(item_descrs)
+        return descrs
+
+    def get_args_for_fail(self, modifier):
+        if self.box is None and not modifier.already_seen_virtual(self.keybox):
+            itemdescrs = self._get_list_of_descrs()
+            itemboxes = []
+            for i in range(len(self._items)):
+                for descr in itemdescrs[i]:
+                    itemboxes.append(self._items[i][descr].get_key_box())
+            modifier.register_virtual_fields(self.keybox, itemboxes)
+            for i in range(len(self._items)):
+                for descr in itemdescrs[i]:
+                    self._items[i][descr].get_args_for_fail(modifier)
+
+    def force_at_end_of_preamble(self, already_forced, optforce):
+        if self in already_forced:
+            return self
+        already_forced[self] = self
+        for index in range(len(self._items)):
+            for descr in self._items[index].keys():
+                self._items[index][descr] = self._items[index][descr].force_at_end_of_preamble(already_forced, optforce)
+        return self
+
+    def _make_virtual(self, modifier):
+        return modifier.make_varraystruct(self.arraydescr, self._get_list_of_descrs())
+
+
 class OptVirtualize(optimizer.Optimization):
     "Virtualize objects until they escape."
 
@@ -283,8 +346,11 @@
         return vvalue
 
     def make_varray(self, arraydescr, size, box, source_op=None):
-        constvalue = self.new_const_item(arraydescr)
-        vvalue = VArrayValue(arraydescr, constvalue, size, box, source_op)
+        if arraydescr.is_array_of_structs():
+            vvalue = VArrayStructValue(arraydescr, size, box, source_op)
+        else:
+            constvalue = self.new_const_item(arraydescr)
+            vvalue = VArrayValue(arraydescr, constvalue, size, box, source_op)
         self.make_equal_to(box, vvalue)
         return vvalue
 
@@ -386,8 +452,7 @@
 
     def optimize_NEW_ARRAY(self, op):
         sizebox = self.get_constant_box(op.getarg(0))
-        # For now we can't make arrays of structs virtual.
-        if sizebox is not None and not op.getdescr().is_array_of_structs():
+        if sizebox is not None:
             # if the original 'op' did not have a ConstInt as argument,
             # build a new one with the ConstInt argument
             if not isinstance(op.getarg(0), ConstInt):
@@ -432,6 +497,34 @@
         value.ensure_nonnull()
         self.emit_operation(op)
 
+    def optimize_GETINTERIORFIELD_GC(self, op):
+        value = self.getvalue(op.getarg(0))
+        if value.is_virtual():
+            indexbox = self.get_constant_box(op.getarg(1))
+            if indexbox is not None:
+                descr = op.getdescr()
+                fieldvalue = value.getinteriorfield(
+                    indexbox.getint(), descr, None
+                )
+                if fieldvalue is None:
+                    fieldvalue = self.new_const(descr)
+                self.make_equal_to(op.result, fieldvalue)
+                return
+        value.ensure_nonnull()
+        self.emit_operation(op)
+
+    def optimize_SETINTERIORFIELD_GC(self, op):
+        value = self.getvalue(op.getarg(0))
+        if value.is_virtual():
+            indexbox = self.get_constant_box(op.getarg(1))
+            if indexbox is not None:
+                value.setinteriorfield(
+                    indexbox.getint(), op.getdescr(), self.getvalue(op.getarg(2))
+                )
+                return
+        value.ensure_nonnull()
+        self.emit_operation(op)
+
 
 dispatch_opt = make_dispatcher_method(OptVirtualize, 'optimize_',
         default=OptVirtualize.emit_operation)
diff --git a/pypy/jit/metainterp/optimizeopt/virtualstate.py b/pypy/jit/metainterp/optimizeopt/virtualstate.py
--- a/pypy/jit/metainterp/optimizeopt/virtualstate.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualstate.py
@@ -16,7 +16,7 @@
 
 class AbstractVirtualStateInfo(resume.AbstractVirtualInfo):
     position = -1
-    
+
     def generalization_of(self, other, renum, bad):
         raise NotImplementedError
 
@@ -54,7 +54,7 @@
                 s.debug_print(indent + "    ", seen, bad)
         else:
             debug_print(indent + "    ...")
-                
+
 
     def debug_header(self, indent):
         raise NotImplementedError
@@ -77,13 +77,15 @@
             bad[self] = True
             bad[other] = True
             return False
+
+        assert isinstance(other, AbstractVirtualStructStateInfo)
         assert len(self.fielddescrs) == len(self.fieldstate)
         assert len(other.fielddescrs) == len(other.fieldstate)
         if len(self.fielddescrs) != len(other.fielddescrs):
             bad[self] = True
             bad[other] = True
             return False
-        
+
         for i in range(len(self.fielddescrs)):
             if other.fielddescrs[i] is not self.fielddescrs[i]:
                 bad[self] = True
@@ -112,8 +114,8 @@
     def _enum(self, virtual_state):
         for s in self.fieldstate:
             s.enum(virtual_state)
-        
-        
+
+
 class VirtualStateInfo(AbstractVirtualStructStateInfo):
     def __init__(self, known_class, fielddescrs):
         AbstractVirtualStructStateInfo.__init__(self, fielddescrs)
@@ -128,13 +130,13 @@
 
     def debug_header(self, indent):
         debug_print(indent + 'VirtualStateInfo(%d):' % self.position)
-        
+
 class VStructStateInfo(AbstractVirtualStructStateInfo):
     def __init__(self, typedescr, fielddescrs):
         AbstractVirtualStructStateInfo.__init__(self, fielddescrs)
         self.typedescr = typedescr
 
-    def _generalization_of(self, other):        
+    def _generalization_of(self, other):
         if not isinstance(other, VStructStateInfo):
             return False
         if self.typedescr is not other.typedescr:
@@ -143,7 +145,7 @@
 
     def debug_header(self, indent):
         debug_print(indent + 'VStructStateInfo(%d):' % self.position)
-        
+
 class VArrayStateInfo(AbstractVirtualStateInfo):
     def __init__(self, arraydescr):
         self.arraydescr = arraydescr
@@ -157,11 +159,7 @@
             bad[other] = True
             return False
         renum[self.position] = other.position
-        if not isinstance(other, VArrayStateInfo):
-            bad[self] = True
-            bad[other] = True
-            return False
-        if self.arraydescr is not other.arraydescr:
+        if not self._generalization_of(other):
             bad[self] = True
             bad[other] = True
             return False
@@ -177,6 +175,10 @@
                 return False
         return True
 
+    def _generalization_of(self, other):
+        return (isinstance(other, VArrayStateInfo) and
+            self.arraydescr is other.arraydescr)
+
     def enum_forced_boxes(self, boxes, value, optimizer):
         assert isinstance(value, virtualize.VArrayValue)
         assert value.is_virtual()
@@ -192,8 +194,75 @@
 
     def debug_header(self, indent):
         debug_print(indent + 'VArrayStateInfo(%d):' % self.position)
-            
-        
+
+class VArrayStructStateInfo(AbstractVirtualStateInfo):
+    def __init__(self, arraydescr, fielddescrs):
+        self.arraydescr = arraydescr
+        self.fielddescrs = fielddescrs
+
+    def generalization_of(self, other, renum, bad):
+        assert self.position != -1
+        if self.position in renum:
+            if renum[self.position] == other.position:
+                return True
+            bad[self] = True
+            bad[other] = True
+            return False
+        renum[self.position] = other.position
+        if not self._generalization_of(other):
+            bad[self] = True
+            bad[other] = True
+            return False
+
+        assert isinstance(other, VArrayStructStateInfo)
+        if len(self.fielddescrs) != len(other.fielddescrs):
+            bad[self] = True
+            bad[other] = True
+            return False
+
+        p = 0
+        for i in range(len(self.fielddescrs)):
+            if len(self.fielddescrs[i]) != len(other.fielddescrs[i]):
+                bad[self] = True
+                bad[other] = True
+                return False
+            for j in range(len(self.fielddescrs[i])):
+                if self.fielddescrs[i][j] is not other.fielddescrs[i][j]:
+                    bad[self] = True
+                    bad[other] = True
+                    return False
+                if not self.fieldstate[p].generalization_of(other.fieldstate[p],
+                                                            renum, bad):
+                    bad[self] = True
+                    bad[other] = True
+                    return False
+                p += 1
+        return True
+
+    def _generalization_of(self, other):
+        return (isinstance(other, VArrayStructStateInfo) and
+            self.arraydescr is other.arraydescr)
+
+    def _enum(self, virtual_state):
+        for s in self.fieldstate:
+            s.enum(virtual_state)
+
+    def enum_forced_boxes(self, boxes, value, optimizer):
+        assert isinstance(value, virtualize.VArrayStructValue)
+        assert value.is_virtual()
+        p = 0
+        for i in range(len(self.fielddescrs)):
+            for j in range(len(self.fielddescrs[i])):
+                v = value._items[i][self.fielddescrs[i][j]]
+                s = self.fieldstate[p]
+                if s.position > self.position:
+                    s.enum_forced_boxes(boxes, v, optimizer)
+                p += 1
+
+    def debug_header(self, indent):
+        debug_print(indent + 'VArrayStructStateInfo(%d):' % self.position)
+
+
 class NotVirtualStateInfo(AbstractVirtualStateInfo):
     def __init__(self, value):
         self.known_class = value.known_class
@@ -277,7 +346,7 @@
             op = ResOperation(rop.GUARD_CLASS, [box, self.known_class], None)
             extra_guards.append(op)
             return
-        
+
         if self.level == LEVEL_NONNULL and \
                other.level == LEVEL_UNKNOWN and \
                isinstance(box, BoxPtr) and \
@@ -285,7 +354,7 @@
             op = ResOperation(rop.GUARD_NONNULL, [box], None)
             extra_guards.append(op)
             return
-        
+
         if self.level == LEVEL_UNKNOWN and \
                other.level == LEVEL_UNKNOWN and \
                isinstance(box, BoxInt) and \
@@ -309,7 +378,7 @@
                     op = ResOperation(rop.GUARD_TRUE, [res], None)
                     extra_guards.append(op)
             return
-        
+
         # Remaining cases are probably not interesting
         raise InvalidLoop
         if self.level == LEVEL_CONSTANT:
@@ -319,7 +388,7 @@
     def enum_forced_boxes(self, boxes, value, optimizer):
         if self.level == LEVEL_CONSTANT:
             return
-        assert 0 <= self.position_in_notvirtuals 
+        assert 0 <= self.position_in_notvirtuals
         boxes[self.position_in_notvirtuals] = value.force_box(optimizer)
 
     def _enum(self, virtual_state):
@@ -348,7 +417,7 @@
         lb = ''
         if self.lenbound:
             lb = ', ' + self.lenbound.bound.__repr__()
-        
+
         debug_print(indent + mark + 'NotVirtualInfo(%d' % self.position +
                     ', ' + l + ', ' + self.intbound.__repr__() + lb + ')')
 
@@ -370,7 +439,7 @@
                 return False
         return True
 
-    def generate_guards(self, other, args, cpu, extra_guards):        
+    def generate_guards(self, other, args, cpu, extra_guards):
         assert len(self.state) == len(other.state) == len(args)
         renum = {}
         for i in range(len(self.state)):
@@ -393,7 +462,7 @@
                     inputargs.append(box)
 
         assert None not in inputargs
-            
+
         return inputargs
 
     def debug_print(self, hdr='', bad=None):
@@ -412,7 +481,7 @@
 
     def register_virtual_fields(self, keybox, fieldboxes):
         self.fieldboxes[keybox] = fieldboxes
-        
+
     def already_seen_virtual(self, keybox):
         return keybox in self.fieldboxes
 
@@ -463,6 +532,9 @@
     def make_varray(self, arraydescr):
         return VArrayStateInfo(arraydescr)
 
+    def make_varraystruct(self, arraydescr, fielddescrs):
+        return VArrayStructStateInfo(arraydescr, fielddescrs)
+
 class BoxNotProducable(Exception):
     pass
 
@@ -501,12 +573,12 @@
             else: # Low priority
                 lo -= 1
         return alts
-            
+
     def renamed(self, box):
         if box in self.rename:
             return self.rename[box]
         return box
-    
+
     def add_to_short(self, box, op):
         if op:
             op = op.clone()
@@ -528,12 +600,12 @@
             self.optimizer.make_equal_to(newbox, value)
         else:
             self.short_boxes[box] = op
-        
+
     def produce_short_preamble_box(self, box):
         if box in self.short_boxes:
-            return 
+            return
         if isinstance(box, Const):
-            return 
+            return
         if box in self.potential_ops:
             ops = self.prioritized_alternatives(box)
             produced_one = False
@@ -570,7 +642,7 @@
             else:
                 debug_print(logops.repr_of_arg(box) + ': None')
         debug_stop('jit-short-boxes')
-        
+
     def operations(self):
         if not we_are_translated(): # For tests
             ops = self.short_boxes.values()
@@ -588,7 +660,7 @@
         if not isinstance(oldbox, Const) and newbox not in self.short_boxes:
             self.short_boxes[newbox] = self.short_boxes[oldbox]
         self.aliases[newbox] = oldbox
-        
+
     def original(self, box):
         while box in self.aliases:
             box = self.aliases[box]
diff --git a/pypy/jit/metainterp/optimizeopt/vstring.py b/pypy/jit/metainterp/optimizeopt/vstring.py
--- a/pypy/jit/metainterp/optimizeopt/vstring.py
+++ b/pypy/jit/metainterp/optimizeopt/vstring.py
@@ -163,17 +163,6 @@
             for value in self._chars:
                 value.get_args_for_fail(modifier)
 
-    def FIXME_enum_forced_boxes(self, boxes, already_seen):
-        key = self.get_key_box()
-        if key in already_seen:
-            return
-        already_seen[key] = None
-        if self.box is None:
-            for box in self._chars:
-                box.enum_forced_boxes(boxes, already_seen)
-        else:
-            boxes.append(self.box)
-
     def _make_virtual(self, modifier):
         return modifier.make_vstrplain(self.mode is mode_unicode)
 
@@ -226,18 +215,6 @@
             self.left.get_args_for_fail(modifier)
             self.right.get_args_for_fail(modifier)
 
-    def FIXME_enum_forced_boxes(self, boxes, already_seen):
-        key = self.get_key_box()
-        if key in already_seen:
-            return
-        already_seen[key] = None
-        if self.box is None:
-            self.left.enum_forced_boxes(boxes, already_seen)
-            self.right.enum_forced_boxes(boxes, already_seen)
-            self.lengthbox = None
-        else:
-            boxes.append(self.box)
-
     def _make_virtual(self, modifier):
         return modifier.make_vstrconcat(self.mode is mode_unicode)
 
@@ -284,18 +261,6 @@
             self.vstart.get_args_for_fail(modifier)
             self.vlength.get_args_for_fail(modifier)
 
-    def FIXME_enum_forced_boxes(self, boxes, already_seen):
-        key = self.get_key_box()
-        if key in already_seen:
-            return
-        already_seen[key] = None
-        if self.box is None:
-            self.vstr.enum_forced_boxes(boxes, already_seen)
-            self.vstart.enum_forced_boxes(boxes, already_seen)
-            self.vlength.enum_forced_boxes(boxes, already_seen)
-        else:
-            boxes.append(self.box)
-
     def _make_virtual(self, modifier):
         return modifier.make_vstrslice(self.mode is mode_unicode)
 
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -240,8 +240,8 @@
         return self.execute(rop.PTR_EQ, box, history.CONST_NULL)
 
     @arguments("box")
-    def opimpl_cast_opaque_ptr(self, box):
-        return self.execute(rop.CAST_OPAQUE_PTR, box)
+    def opimpl_mark_opaque_ptr(self, box):
+        return self.execute(rop.MARK_OPAQUE_PTR, box)
 
     @arguments("box")
     def _opimpl_any_return(self, box):
diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py
--- a/pypy/jit/metainterp/resoperation.py
+++ b/pypy/jit/metainterp/resoperation.py
@@ -439,7 +439,6 @@
     'PTR_NE/2b',
     'INSTANCE_PTR_EQ/2b',
     'INSTANCE_PTR_NE/2b',
-    'CAST_OPAQUE_PTR/1b',
     #
     'ARRAYLEN_GC/1d',
     'STRLEN/1',
@@ -471,6 +470,7 @@
     'FORCE_TOKEN/0',
     'VIRTUAL_REF/2',         # removed before it's passed to the backend
     'READ_TIMESTAMP/0',
+    'MARK_OPAQUE_PTR/1b',
     '_NOSIDEEFFECT_LAST', # ----- end of no_side_effect operations -----
 
     'SETARRAYITEM_GC/3d',
diff --git a/pypy/jit/metainterp/resume.py b/pypy/jit/metainterp/resume.py
--- a/pypy/jit/metainterp/resume.py
+++ b/pypy/jit/metainterp/resume.py
@@ -139,7 +139,7 @@
         self.numberings = {}
         self.cached_boxes = {}
         self.cached_virtuals = {}
-    
+
         self.nvirtuals = 0
         self.nvholes = 0
         self.nvreused = 0
@@ -273,6 +273,9 @@
     def make_varray(self, arraydescr):
         return VArrayInfo(arraydescr)
 
+    def make_varraystruct(self, arraydescr, fielddescrs):
+        return VArrayStructInfo(arraydescr, fielddescrs)
+
     def make_vstrplain(self, is_unicode=False):
         if is_unicode:
             return VUniPlainInfo()
@@ -402,7 +405,7 @@
                 virtuals[num] = vinfo
 
         if self._invalidation_needed(len(liveboxes), nholes):
-            memo.clear_box_virtual_numbers()           
+            memo.clear_box_virtual_numbers()
 
     def _invalidation_needed(self, nliveboxes, nholes):
         memo = self.memo
@@ -455,7 +458,7 @@
 
     def debug_prints(self):
         raise NotImplementedError
-        
+
 class AbstractVirtualStructInfo(AbstractVirtualInfo):
     def __init__(self, fielddescrs):
         self.fielddescrs = fielddescrs
@@ -537,6 +540,29 @@
         for i in self.fieldnums:
             debug_print("\t\t", str(untag(i)))
 
+
+class VArrayStructInfo(AbstractVirtualInfo):
+    def __init__(self, arraydescr, fielddescrs):
+        self.arraydescr = arraydescr
+        self.fielddescrs = fielddescrs
+
+    def debug_prints(self):
+        debug_print("\tvarraystructinfo", self.arraydescr)
+        for i in self.fieldnums:
+            debug_print("\t\t", str(untag(i)))
+
+    @specialize.argtype(1)
+    def allocate(self, decoder, index):
+        array = decoder.allocate_array(self.arraydescr, len(self.fielddescrs))
+        decoder.virtuals_cache[index] = array
+        p = 0
+        for i in range(len(self.fielddescrs)):
+            for j in range(len(self.fielddescrs[i])):
+                decoder.setinteriorfield(i, self.fielddescrs[i][j], array, self.fieldnums[p])
+                p += 1
+        return array
+
+
 class VStrPlainInfo(AbstractVirtualInfo):
     """Stands for the string made out of the characters of all fieldnums."""
 
@@ -884,6 +910,17 @@
         self.metainterp.execute_and_record(rop.SETFIELD_GC, descr,
                                            structbox, fieldbox)
 
+    def setinteriorfield(self, index, descr, array, fieldnum):
+        if descr.is_pointer_field():
+            kind = REF
+        elif descr.is_float_field():
+            kind = FLOAT
+        else:
+            kind = INT
+        fieldbox = self.decode_box(fieldnum, kind)
+        self.metainterp.execute_and_record(rop.SETINTERIORFIELD_GC, descr,
+                                           array, ConstInt(index), fieldbox)
+
     def setarrayitem_int(self, arraydescr, arraybox, index, fieldnum):
         self._setarrayitem(arraydescr, arraybox, index, fieldnum, INT)
 
@@ -1164,6 +1201,17 @@
             newvalue = self.decode_int(fieldnum)
             self.cpu.bh_setfield_gc_i(struct, descr, newvalue)
 
+    def setinteriorfield(self, index, descr, array, fieldnum):
+        if descr.is_pointer_field():
+            newvalue = self.decode_ref(fieldnum)
+            self.cpu.bh_setinteriorfield_gc_r(array, index, descr, newvalue)
+        elif descr.is_float_field():
+            newvalue = self.decode_float(fieldnum)
+            self.cpu.bh_setinteriorfield_gc_f(array, index, descr, newvalue)
+        else:
+            newvalue = self.decode_int(fieldnum)
+            self.cpu.bh_setinteriorfield_gc_i(array, index, descr, newvalue)
+
     def setarrayitem_int(self, arraydescr, array, index, fieldnum):
         newvalue = self.decode_int(fieldnum)
         self.cpu.bh_setarrayitem_gc_i(arraydescr, array, index, newvalue)
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -10,6 +10,7 @@
 from pypy.jit.metainterp.typesystem import LLTypeHelper, OOTypeHelper
 from pypy.jit.metainterp.warmspot import get_stats
 from pypy.jit.metainterp.warmstate import set_future_value
+from pypy.rlib import rerased
 from pypy.rlib.jit import (JitDriver, we_are_jitted, hint, dont_look_inside,
     loop_invariant, elidable, promote, jit_debug, assert_green,
     AssertGreenFailed, unroll_safe, current_trace_length, look_inside_iff,
@@ -3494,16 +3495,70 @@
             d = None
             while n > 0:
                 myjitdriver.jit_merge_point(n=n, d=d)
-                d = {}
+                d = {"q": 1}
                 if n % 2:
                     d["k"] = n
                 else:
                     d["z"] = n
-                n -= len(d)
+                n -= len(d) - d["q"]
             return n
         res = self.meta_interp(f, [10])
         assert res == 0
 
+    def test_virtual_dict_constant_keys(self):
+        myjitdriver = JitDriver(greens = [], reds = ["n"])
+        def g(d):
+            return d["key"] - 1
+
+        def f(n):
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n)
+                n = g({"key": n})
+            return n
+
+        res = self.meta_interp(f, [10])
+        assert res == 0
+        self.check_loops({"int_sub": 1, "int_gt": 1, "guard_true": 1, "jump": 1})
+
+    def test_virtual_opaque_ptr(self):
+        myjitdriver = JitDriver(greens = [], reds = ["n"])
+        erase, unerase = rerased.new_erasing_pair("x")
+        @look_inside_iff(lambda x: isvirtual(x))
+        def g(x):
+            return x[0]
+        def f(n):
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n)
+                x = []
+                y = erase(x)
+                z = unerase(y)
+                z.append(1)
+                n -= g(z)
+            return n
+        res = self.meta_interp(f, [10])
+        assert res == 0
+        self.check_loops({"int_sub": 1, "int_gt": 1, "guard_true": 1, "jump": 1})
+
+    def test_virtual_opaque_dict(self):
+        myjitdriver = JitDriver(greens = [], reds = ["n"])
+        erase, unerase = rerased.new_erasing_pair("x")
+        @look_inside_iff(lambda x: isvirtual(x))
+        def g(x):
+            return x[0]["key"] - 1
+        def f(n):
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n)
+                x = [{}]
+                x[0]["key"] = n
+                x[0]["other key"] = n
+                y = erase(x)
+                z = unerase(y)
+                n = g(x)
+            return n
+        res = self.meta_interp(f, [10])
+        assert res == 0
+        self.check_loops({"int_sub": 1, "int_gt": 1, "guard_true": 1, "jump": 1})
+
 
 
 class TestLLtype(BaseLLtypeTests, LLJitMixin):
@@ -3561,8 +3616,7 @@
         res = self.meta_interp(main, [False, 100, True], taggedpointers=True)
 
     def test_rerased(self):
-        from pypy.rlib.rerased import erase_int, unerase_int, new_erasing_pair
-        eraseX, uneraseX = new_erasing_pair("X")
+        eraseX, uneraseX = rerased.new_erasing_pair("X")
         #
         class X:
             def __init__(self, a, b):
@@ -3575,14 +3629,14 @@
                 e = eraseX(X(i, j))
             else:
                 try:
-                    e = erase_int(i)
+                    e = rerased.erase_int(i)
                 except OverflowError:
                     return -42
             if j & 1:
                 x = uneraseX(e)
                 return x.a - x.b
             else:
-                return unerase_int(e)
+                return rerased.unerase_int(e)
         #
         x = self.interp_operations(f, [-128, 0], taggedpointers=True)
         assert x == -128
diff --git a/pypy/jit/metainterp/test/test_heapcache.py b/pypy/jit/metainterp/test/test_heapcache.py
--- a/pypy/jit/metainterp/test/test_heapcache.py
+++ b/pypy/jit/metainterp/test/test_heapcache.py
@@ -371,3 +371,17 @@
         assert h.is_unescaped(box1)
         h.invalidate_caches(rop.SETARRAYITEM_GC, None, [box2, index1, box1])
         assert not h.is_unescaped(box1)
+
+        h = HeapCache()
+        h.new_array(box1, lengthbox1)
+        h.new(box2)
+        assert h.is_unescaped(box1)
+        assert h.is_unescaped(box2)
+        h.invalidate_caches(rop.SETARRAYITEM_GC, None, [box1, lengthbox2, box2])
+        assert h.is_unescaped(box1)
+        assert h.is_unescaped(box2)
+        h.invalidate_caches(
+            rop.CALL, FakeCallDescr(FakeEffektinfo.EF_RANDOM_EFFECTS), [box1]
+        )
+        assert not h.is_unescaped(box1)
+        assert not h.is_unescaped(box2)
diff --git a/pypy/jit/metainterp/test/test_tracingopts.py b/pypy/jit/metainterp/test/test_tracingopts.py
--- a/pypy/jit/metainterp/test/test_tracingopts.py
+++ b/pypy/jit/metainterp/test/test_tracingopts.py
@@ -3,6 +3,7 @@
 from pypy.jit.metainterp.test.support import LLJitMixin
 from pypy.rlib import jit
 from pypy.rlib.rarithmetic import ovfcheck
+from pypy.rlib.rstring import StringBuilder
 
 import py
 
@@ -590,4 +591,14 @@
         assert res == 4
         self.check_operations_history(int_add_ovf=0)
         res = self.interp_operations(fn, [sys.maxint])
-        assert res == 12
\ No newline at end of file
+        assert res == 12
+
+    def test_copy_str_content(self):
+        def fn(n):
+            a = StringBuilder()
+            x = [1]
+            a.append("hello world")
+            return x[0]
+        res = self.interp_operations(fn, [0])
+        assert res == 1
+        self.check_operations_history(getarrayitem_gc=0, getarrayitem_gc_pure=0 )
\ No newline at end of file
diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py
--- a/pypy/module/pypyjit/test_pypy_c/test_call.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_call.py
@@ -465,3 +465,25 @@
             setfield_gc(p4, p22, descr=<GcPtrFieldDescr pypy.interpreter.nestedscope.Cell.inst_w_value .*>)
             jump(p0, p1, p2, p3, p4, p7, p22, p7, descr=<Loop0>)
         """)
+
+    def test_kwargs_virtual(self):
+        def main(n):
+            def g(**kwargs):
+                return kwargs["x"] + 1
+
+            i = 0
+            while i < n:
+                i = g(x=i)
+            return i
+
+        log = self.run(main, [500])
+        assert log.result == 500
+        loop, = log.loops_by_filename(self.filepath)
+        assert loop.match("""
+            i2 = int_lt(i0, i1)
+            guard_true(i2, descr=...)
+            i3 = force_token()
+            i4 = int_add(i0, 1)
+            --TICK--
+            jump(..., descr=...)
+        """)
\ No newline at end of file
diff --git a/pypy/module/pypyjit/test_pypy_c/test_containers.py b/pypy/module/pypyjit/test_pypy_c/test_containers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_containers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_containers.py
@@ -44,7 +44,7 @@
         # gc_id call is hoisted out of the loop, the id of a value obviously
         # can't change ;)
         assert loop.match_by_id("getitem", """
-            i28 = call(ConstClass(ll_dict_lookup__dicttablePtr_objectPtr_Signed), p18, p6, i25, descr=...)
+            i26 = call(ConstClass(ll_dict_lookup), p18, p6, i25, descr=...)
             ...
             p33 = getinteriorfield_gc(p31, i26, descr=<InteriorFieldDescr <GcPtrFieldDescr dictentry.value .*>>)
             ...
diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py
--- a/pypy/rpython/lltypesystem/rdict.py
+++ b/pypy/rpython/lltypesystem/rdict.py
@@ -445,9 +445,9 @@
     i = ll_dict_lookup(d, key, hash)
     return _ll_dict_setitem_lookup_done(d, key, value, hash, i)
 
-# Leaving as dont_look_inside ATM, it has a few branches which could lead to
-# many bridges if we don't consider their possible frequency.
- at jit.dont_look_inside
+# It may be safe to look inside always, it has a few branches though, and their
+# frequencies needs to be investigated.
+ at jit.look_inside_iff(lambda d, key, value, hash, i: jit.isvirtual(d) and jit.isconstant(key))
 def _ll_dict_setitem_lookup_done(d, key, value, hash, i):
     valid = (i & HIGHEST_BIT) == 0
     i = i & MASK
@@ -533,7 +533,7 @@
 # ------- a port of CPython's dictobject.c's lookdict implementation -------
 PERTURB_SHIFT = 5
 
- at jit.dont_look_inside
+ at jit.look_inside_iff(lambda d, key, hash: jit.isvirtual(d) and jit.isconstant(key))
 def ll_dict_lookup(d, key, hash):
     entries = d.entries
     ENTRIES = lltype.typeOf(entries).TO

From noreply at buildbot.pypy.org  Thu Oct 27 18:56:56 2011
From: noreply at buildbot.pypy.org (hager)
Date: Thu, 27 Oct 2011 18:56:56 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: Ok,
	there actually IS an SLWI instr. on PPC.
Message-ID: <20111027165656.124E2820C2@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r48515:064b64295d89
Date: 2011-10-26 10:38 +0200
http://bitbucket.org/pypy/pypy/changeset/064b64295d89/

Log:	Ok, there actually IS an SLWI instr. on PPC. Improved generated code
	in emit_unicodegetitem and emit_unicodesetitem.

diff --git a/pypy/jit/backend/ppc/ppcgen/opassembler.py b/pypy/jit/backend/ppc/ppcgen/opassembler.py
--- a/pypy/jit/backend/ppc/ppcgen/opassembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/opassembler.py
@@ -401,9 +401,7 @@
     def emit_unicodegetitem(self, op, arglocs, regalloc):
         res, base_loc, ofs_loc, scale, basesize, itemsize = arglocs
 
-        # XXX arrrrgh, why does PPC not have an SLWI instruction ?
-        self.mc.li(r.r0.value, scale.value)
-        self.mc.slw(ofs_loc.value, ofs_loc.value, r.r0.value)
+        self.mc.slwi(ofs_loc.value, ofs_loc.value, scale.value)
         self.mc.add(res.value, base_loc.value, ofs_loc.value)
 
         if scale.value == 2:
@@ -417,8 +415,7 @@
     def emit_unicodesetitem(self, op, arglocs, regalloc):
         value_loc, base_loc, ofs_loc, scale, basesize, itemsize = arglocs
 
-        self.mc.li(r.r0.value, scale.value)
-        self.mc.slw(ofs_loc.value, ofs_loc.value, r.r0.value)
+        self.mc.slwi(ofs_loc.value, ofs_loc.value, scale.value)
         self.mc.add(base_loc.value, base_loc.value, ofs_loc.value)
 
         if scale.value == 2:

From noreply at buildbot.pypy.org  Thu Oct 27 18:56:57 2011
From: noreply at buildbot.pypy.org (hager)
Date: Thu, 27 Oct 2011 18:56:57 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: merge
Message-ID: <20111027165657.4007A820C2@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r48516:59fae2045cad
Date: 2011-10-26 10:42 +0200
http://bitbucket.org/pypy/pypy/changeset/59fae2045cad/

Log:	merge

diff --git a/pypy/jit/backend/ppc/ppcgen/opassembler.py b/pypy/jit/backend/ppc/ppcgen/opassembler.py
--- a/pypy/jit/backend/ppc/ppcgen/opassembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/opassembler.py
@@ -222,9 +222,15 @@
         offset = locs[2]
         if offset is not None:
             if offset.is_imm():
-                self.mc.lwz(r.r0.value, locs[0].value, offset.value)
+                if IS_PPC_32:
+                    self.mc.lwz(r.r0.value, locs[0].value, offset.value)
+                else:
+                    self.mc.ld(r.r0.value, locs[0].value, offset.value)
             else:
-                self.mc.lwzx(r.r0.value, locs[0].value, offset.value)
+                if IS_PPC_32:
+                    self.mc.lwzx(r.r0.value, locs[0].value, offset.value)
+                else:
+                    self.mc.ldx(r.r0.value, locs[0].value, offset.value)
             self.mc.cmp(r.r0.value, locs[1].value)
         else:
             assert 0, "not implemented yet"
@@ -310,12 +316,13 @@
         else:
             assert 0, "size not supported"
 
-    # XXX 64 bit adjustment
     def emit_arraylen_gc(self, op, arglocs, regalloc):
         res, base_loc, ofs = arglocs
-        self.mc.lwz(res.value, base_loc.value, ofs.value)
+        if IS_PPC_32:
+            self.mc.lwz(res.value, base_loc.value, ofs.value)
+        else:
+            self.mc.ld(res.value, base_loc.value, ofs.value)
 
-    # XXX 64 bit adjustment
     def emit_setarrayitem_gc(self, op, arglocs, regalloc):
         value_loc, base_loc, ofs_loc, scale, ofs = arglocs
         if scale.value > 0:
@@ -330,7 +337,7 @@
             scale_loc = r.r0
 
         if scale.value == 3:
-            assert 0, "not implemented yet"
+            self.mc.stdx(value_loc.value, base_loc.value, scale_loc.value)
         elif scale.value == 2:
             self.mc.stwx(value_loc.value, base_loc.value, scale_loc.value)
         elif scale.value == 1:
@@ -340,7 +347,6 @@
         else:
             assert 0, "scale %s not supported" % (scale.value)
 
-    # XXX 64 bit adjustment
     def emit_getarrayitem_gc(self, op, arglocs, regalloc):
         res, base_loc, ofs_loc, scale, ofs = arglocs
         if scale.value > 0:
@@ -354,7 +360,7 @@
             scale_loc = r.r0
 
         if scale.value == 3:
-            assert 0, "not implemented yet"
+            self.mc.ldx(res.value, base_loc.value, scale_loc.value)
         elif scale.value == 2:
             self.mc.lwzx(res.value, base_loc.value, scale_loc.value)
         elif scale.value == 1:
@@ -371,13 +377,18 @@
             signed = descr.is_item_signed()
             self._ensure_result_bit_extension(res, size, signed)
 
-    # XXX 64 bit adjustment needed
     def emit_strlen(self, op, arglocs, regalloc):
         l0, l1, res = arglocs
         if l1.is_imm():
-            self.mc.lwz(res.value, l0.value, l1.getint())
+            if IS_PPC_32:
+                self.mc.lwz(res.value, l0.value, l1.getint())
+            else:
+                self.mc.ld(res.value, l0.value, l1.getint())
         else:
-            self.mc.lwzx(res.value, l0.value, l1.value)
+            if IS_PPC_32:
+                self.mc.lwzx(res.value, l0.value, l1.value)
+            else:
+                self.mc.ldx(res.value, l0.value, l1.value)
 
     def emit_strgetitem(self, op, arglocs, regalloc):
         res, base_loc, ofs_loc, basesize = arglocs

From noreply at buildbot.pypy.org  Thu Oct 27 18:56:58 2011
From: noreply at buildbot.pypy.org (hager)
Date: Thu, 27 Oct 2011 18:56:58 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: Skip test_array_of_structs if
 floats are not supported.
Message-ID: <20111027165658.7348F820C2@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r48517:b6e8bd8c4973
Date: 2011-10-27 12:08 +0200
http://bitbucket.org/pypy/pypy/changeset/b6e8bd8c4973/

Log:	Skip test_array_of_structs if floats are not supported.

diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -991,6 +991,8 @@
         assert r.value == 7441
 
     def test_array_of_structs(self):
+        if not self.cpu.supports_floats:
+            py.test.skip("floats are not supported")
         TP = lltype.GcStruct('x')
         ITEM = lltype.Struct('x',
                              ('vs', lltype.Signed),

From noreply at buildbot.pypy.org  Thu Oct 27 18:56:59 2011
From: noreply at buildbot.pypy.org (hager)
Date: Thu, 27 Oct 2011 18:56:59 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: Implemented DEBUG_MERGE_POINT
	and JIT_DEBUG.
Message-ID: <20111027165659.A157C820C2@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r48518:0c1ba83db460
Date: 2011-10-27 18:56 +0200
http://bitbucket.org/pypy/pypy/changeset/0c1ba83db460/

Log:	Implemented DEBUG_MERGE_POINT and JIT_DEBUG.

diff --git a/pypy/jit/backend/ppc/ppcgen/opassembler.py b/pypy/jit/backend/ppc/ppcgen/opassembler.py
--- a/pypy/jit/backend/ppc/ppcgen/opassembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/opassembler.py
@@ -440,5 +440,10 @@
         argloc, resloc = arglocs
         self.regalloc_mov(argloc, resloc)
 
+    def emit_debug_merge_point(self, op, arglocs, regalloc):
+        pass
+
+    emit_jit_debug = emit_debug_merge_point
+
     def nop(self):
         self.mc.ori(0, 0, 0)
diff --git a/pypy/jit/backend/ppc/ppcgen/regalloc.py b/pypy/jit/backend/ppc/ppcgen/regalloc.py
--- a/pypy/jit/backend/ppc/ppcgen/regalloc.py
+++ b/pypy/jit/backend/ppc/ppcgen/regalloc.py
@@ -539,6 +539,12 @@
         self.possibly_free_var(op.result)
         return [argloc, resloc]
 
+    def void(self, op):
+        return []
+
+    prepare_debug_merge_point = void
+    prepare_jit_debug = void
+
     # from ../x86/regalloc.py:791
     def _unpack_fielddescr(self, fielddescr):
         assert isinstance(fielddescr, BaseFieldDescr)

From noreply at buildbot.pypy.org  Thu Oct 27 19:06:19 2011
From: noreply at buildbot.pypy.org (hager)
Date: Thu, 27 Oct 2011 19:06:19 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: CAST_INT_TO_PTR,
	CAST_PTR_TO_INT.
Message-ID: <20111027170619.D3953820C2@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r48519:cffb9538d307
Date: 2011-10-27 19:05 +0200
http://bitbucket.org/pypy/pypy/changeset/cffb9538d307/

Log:	CAST_INT_TO_PTR, CAST_PTR_TO_INT.

diff --git a/pypy/jit/backend/ppc/ppcgen/opassembler.py b/pypy/jit/backend/ppc/ppcgen/opassembler.py
--- a/pypy/jit/backend/ppc/ppcgen/opassembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/opassembler.py
@@ -440,6 +440,9 @@
         argloc, resloc = arglocs
         self.regalloc_mov(argloc, resloc)
 
+    emit_cast_ptr_to_int = emit_same_as
+    emit_cast_int_to_ptr = emit_same_as
+
     def emit_debug_merge_point(self, op, arglocs, regalloc):
         pass
 
diff --git a/pypy/jit/backend/ppc/ppcgen/regalloc.py b/pypy/jit/backend/ppc/ppcgen/regalloc.py
--- a/pypy/jit/backend/ppc/ppcgen/regalloc.py
+++ b/pypy/jit/backend/ppc/ppcgen/regalloc.py
@@ -539,6 +539,9 @@
         self.possibly_free_var(op.result)
         return [argloc, resloc]
 
+    prepare_cast_ptr_to_int = prepare_same_as
+    prepare_cast_int_to_ptr = prepare_same_as
+
     def void(self, op):
         return []
 

From noreply at buildbot.pypy.org  Thu Oct 27 19:09:05 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Oct 2011 19:09:05 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-multidim: another test
Message-ID: <20111027170905.63EDA820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: numpy-multidim
Changeset: r48520:1065599494c0
Date: 2011-10-27 17:38 +0200
http://bitbucket.org/pypy/pypy/changeset/1065599494c0/

Log:	another test

diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -663,6 +663,7 @@
         a = numpy.zeros((3, 4))
         a[1] = [1, 2, 3, 4]
         assert a[1, 2] == 3
+        raises(TypeError, a[1].__setitem__, [1, 2, 3])
 
 class AppTestSupport(object):
     def setup_class(cls):

From noreply at buildbot.pypy.org  Thu Oct 27 19:09:06 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Oct 2011 19:09:06 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-multidim: skip test_zjit for now
Message-ID: <20111027170906.91586820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: numpy-multidim
Changeset: r48521:647e3d21832c
Date: 2011-10-27 17:47 +0200
http://bitbucket.org/pypy/pypy/changeset/647e3d21832c/

Log:	skip test_zjit for now

diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -3,14 +3,14 @@
 from pypy.module.micronumpy.compile import (numpy_compile, FakeSpace,
     FloatObject, IntObject)
 from pypy.module.micronumpy.interp_dtype import W_Int32Dtype, W_Float64Dtype, W_Int64Dtype, W_UInt64Dtype
-from pypy.module.micronumpy.interp_numarray import (BaseArray, SingleDimArray,
-    SingleDimSlice, scalar_w)
+from pypy.module.micronumpy.interp_numarray import (BaseArray, NDimArray,
+    NDimSlice, scalar_w)
 from pypy.rlib.nonconst import NonConstant
 from pypy.rpython.annlowlevel import llstr
 from pypy.rpython.test.test_llinterp import interpret
 
 import py
-
+py.test.skip("XXX")
 
 class TestNumpyJIt(LLJitMixin):
     def setup_class(cls):

From noreply at buildbot.pypy.org  Thu Oct 27 19:09:07 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Oct 2011 19:09:07 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-multidim: fix until all tests pass,
	except test_zjit
Message-ID: <20111027170907.C2DC6820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: numpy-multidim
Changeset: r48522:79e8268eceeb
Date: 2011-10-27 19:08 +0200
http://bitbucket.org/pypy/pypy/changeset/79e8268eceeb/

Log:	fix until all tests pass, except test_zjit

diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -5,7 +5,7 @@
     applevel_name = 'numpy'
 
     interpleveldefs = {
-        'array': 'interp_numarray.SingleDimArray',
+        'array': 'interp_numarray.NDimArray',
         'dtype': 'interp_dtype.W_Dtype',
         'ufunc': 'interp_ufuncs.W_Ufunc',
 
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -213,8 +213,7 @@
     def descr_repr(self, space):
         # Simple implementation so that we can see the array. Needs work.
         concrete = self.get_concrete()
-        #res = "array([" + ", ".join(concrete._getnums(False)) + "]"
-        res = 'array()'
+        res = "array([" + ", ".join(concrete._getnums(False)) + "]"
         dtype = concrete.find_dtype()
         if (dtype is not space.fromcache(interp_dtype.W_Float64Dtype) and
             dtype is not space.fromcache(interp_dtype.W_Int64Dtype)) or not self.find_size():
@@ -229,18 +228,27 @@
 
     def _single_item_at_index(self, space, w_idx):
         # we assume C ordering for now
-        if len(self.shape) == 1:
-            return space.int_w(w_idx)
+        if space.isinstance_w(w_idx, space.w_int):
+            idx = space.int_w(w_idx)
+            if idx < 0:
+                idx = self.shape[0] + idx
+            if idx < 0 or idx >= self.shape[0]:
+                raise OperationError(space.w_IndexError,
+                                     space.wrap("index out of range"))
+            return idx
         index = [space.int_w(w_item)
                  for w_item in space.fixedview(w_idx)]
         item = 0
         for i in range(len(index)):
+            v = index[i]
+            if v < 0:
+                v += self.shape[i]
+            if v < 0 or v >= self.shape[i]:
+                raise OperationError(space.w_IndexError,
+                                     space.wrap("index (%d) out of range (0<=index<%d" % (index[i], self.shape[i])))
             if i != 0:
                 item *= self.shape[i]
-            if index[i] >= self.shape[i]:
-                raise OperationError(space.w_IndexError,
-                                     space.wrap("index (%d) out of range (0<=index<%d" % (index[i], self.shape[i])))
-            item += index[i]
+            item += v
         return item
 
     def len_of_shape(self):
@@ -257,9 +265,10 @@
         if shape_len == 1:
             if space.isinstance_w(w_idx, space.w_int):
                 return True
-            return False
-        if (space.isinstance_w(w_idx, space.w_slice) or
-            space.isinstance_w(w_idx, space.w_int)):
+            if space.isinstance_w(w_idx, space.w_slice):
+                return False
+        elif (space.isinstance_w(w_idx, space.w_slice) or
+              space.isinstance_w(w_idx, space.w_int)):
             return False
         lgt = space.len_w(w_idx)
         if lgt > shape_len:
@@ -279,10 +288,10 @@
         if (space.isinstance_w(w_idx, space.w_int) or
             space.isinstance_w(w_idx, space.w_slice)):
             start, stop, step, lgt = space.decode_index4(w_idx, self.shape[0])
-            if lgt == 1:
+            if step == 0:
                 shape = self.shape[1:]
             else:
-                shape = self.shape
+                shape = [lgt] + self.shape[1:]
             chunks = [(start, stop, step, lgt)]
         else:
             chunks = []
@@ -291,7 +300,7 @@
                 start, stop, step, lgt = space.decode_index4(w_item,
                                                              self.shape[i])
                 chunks.append((start, stop, step, lgt))
-                if lgt == 1:
+                if step == 0:
                     shape[i] = -1
                 else:
                     shape[i] = lgt
@@ -368,8 +377,8 @@
     """
     Class for representing virtual arrays, such as binary ops or ufuncs
     """
-    def __init__(self, signature, res_dtype):
-        BaseArray.__init__(self)
+    def __init__(self, signature, shape, res_dtype):
+        BaseArray.__init__(self, shape)
         self.forced_result = None
         self.signature = signature
         self.res_dtype = res_dtype
@@ -419,8 +428,8 @@
 
 
 class Call1(VirtualArray):
-    def __init__(self, signature, res_dtype, values):
-        VirtualArray.__init__(self, signature, res_dtype)
+    def __init__(self, signature, shape, res_dtype, values):
+        VirtualArray.__init__(self, signature, shape, res_dtype)
         self.values = values
 
     def _del_sources(self):
@@ -445,8 +454,8 @@
     """
     Intermediate class for performing binary operations.
     """
-    def __init__(self, signature, calc_dtype, res_dtype, left, right):
-        VirtualArray.__init__(self, signature, res_dtype)
+    def __init__(self, signature, shape, calc_dtype, res_dtype, left, right):
+        VirtualArray.__init__(self, signature, shape, res_dtype)
         self.left = left
         self.right = right
         self.calc_dtype = calc_dtype
@@ -505,7 +514,9 @@
         raise NotImplementedError
 
     def descr_len(self, space):
-        return space.wrap(self.shape[0])
+        if self.shape:
+            return space.wrap(self.shape[0])
+        return space.wrap(1)
 
     def calc_index(self, item):
         raise NotImplementedError
@@ -531,10 +542,10 @@
         return self.parent.find_dtype()
 
     def setslice(self, space, w_value):
-        assert isinstance(w_value, NDimArray)
-        if self.shape != w_value.shape:
-            raise OperationError(space.w_TypeError, space.wrap(
-                "wrong assignment"))
+        if isinstance(w_value, NDimArray):
+            if self.shape != w_value.shape:
+                raise OperationError(space.w_TypeError, space.wrap(
+                    "wrong assignment"))
         self._sliceloop(w_value)
 
     def _sliceloop(self, source):
@@ -575,7 +586,7 @@
                 item *= shape[k]
             k += 1
             start, stop, step, lgt = chunk
-            if lgt == 1:
+            if step == 0:
                 # we don't consume an index
                 item += start
             else:
diff --git a/pypy/module/micronumpy/interp_support.py b/pypy/module/micronumpy/interp_support.py
--- a/pypy/module/micronumpy/interp_support.py
+++ b/pypy/module/micronumpy/interp_support.py
@@ -19,7 +19,7 @@
             "string length %d not divisable by %d" % (length, FLOAT_SIZE)))
 
     dtype = space.fromcache(W_Float64Dtype)
-    a = NDimArray(number, dtype=dtype)
+    a = NDimArray(number, [number], dtype=dtype)
 
     start = 0
     end = FLOAT_SIZE
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -105,7 +105,7 @@
             return self.func(res_dtype, w_obj.value.convert_to(res_dtype)).wrap(space)
 
         new_sig = signature.Signature.find_sig([self.signature, w_obj.signature])
-        w_res = Call1(new_sig, res_dtype, w_obj)
+        w_res = Call1(new_sig, w_obj.shape, res_dtype, w_obj)
         w_obj.add_invalidates(w_res)
         return w_res
 
@@ -147,7 +147,8 @@
         new_sig = signature.Signature.find_sig([
             self.signature, w_lhs.signature, w_rhs.signature
         ])
-        w_res = Call2(new_sig, calc_dtype, res_dtype, w_lhs, w_rhs)
+        w_res = Call2(new_sig, w_lhs.shape or w_rhs.shape, calc_dtype,
+                      res_dtype, w_lhs, w_rhs)
         w_lhs.add_invalidates(w_res)
         w_rhs.add_invalidates(w_res)
         return w_res
diff --git a/pypy/module/micronumpy/test/test_base.py b/pypy/module/micronumpy/test/test_base.py
--- a/pypy/module/micronumpy/test/test_base.py
+++ b/pypy/module/micronumpy/test/test_base.py
@@ -13,7 +13,7 @@
     def test_binop_signature(self, space):
         float64_dtype = space.fromcache(interp_dtype.W_Float64Dtype)
 
-        ar = NDimArray(10, dtype=float64_dtype)
+        ar = NDimArray(10, [10], dtype=float64_dtype)
         v1 = ar.descr_add(space, ar)
         v2 = ar.descr_add(space, Scalar(float64_dtype, 2.0))
         assert v1.signature is not v2.signature
@@ -22,7 +22,7 @@
         v4 = ar.descr_add(space, ar)
         assert v1.signature is v4.signature
 
-        bool_ar = NDimArray(10, dtype=space.fromcache(interp_dtype.W_BoolDtype))
+        bool_ar = NDimArray(10, [10], dtype=space.fromcache(interp_dtype.W_BoolDtype))
         v5 = ar.descr_add(space, bool_ar)
         assert v5.signature is not v1.signature
         assert v5.signature is not v2.signature
@@ -30,7 +30,7 @@
         assert v5.signature is v6.signature
 
     def test_slice_signature(self, space):
-        ar = NDimArray(10, dtype=space.fromcache(interp_dtype.W_Float64Dtype))
+        ar = NDimArray(10, [10], dtype=space.fromcache(interp_dtype.W_Float64Dtype))
         v1 = ar.descr_getitem(space, space.wrap(slice(1, 5, 1)))
         v2 = ar.descr_getitem(space, space.wrap(slice(4, 6, 1)))
         assert v1.signature is v2.signature
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -42,6 +42,8 @@
         b = a.copy()
         for i in xrange(5):
             assert b[i] == a[i]
+        a[3] = 22
+        assert b[3] == 3
 
     def test_iterator_init(self):
         from numpy import array

From noreply at buildbot.pypy.org  Thu Oct 27 19:14:00 2011
From: noreply at buildbot.pypy.org (hager)
Date: Thu, 27 Oct 2011 19:14:00 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: GETFIELD_RAW, SETFIELD_RAW,
 GETFIELD_RAW_PURE, GETFIELD_GC_PURE.
Message-ID: <20111027171400.B97CF820C2@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r48523:00a70b5f77a9
Date: 2011-10-27 19:13 +0200
http://bitbucket.org/pypy/pypy/changeset/00a70b5f77a9/

Log:	GETFIELD_RAW, SETFIELD_RAW, GETFIELD_RAW_PURE, GETFIELD_GC_PURE.

diff --git a/pypy/jit/backend/ppc/ppcgen/opassembler.py b/pypy/jit/backend/ppc/ppcgen/opassembler.py
--- a/pypy/jit/backend/ppc/ppcgen/opassembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/opassembler.py
@@ -264,7 +264,6 @@
                          descr._ppc_frame_manager_depth)
             regalloc.frame_manager.frame_depth = new_fd
 
-    # XXX adjust 64 bit
     def emit_setfield_gc(self, op, arglocs, regalloc):
         value_loc, base_loc, ofs, size = arglocs
         if size.value == 8:
@@ -289,8 +288,8 @@
                 self.mc.stbx(value_loc.value, base_loc.value, ofs.value)
         else:
             assert 0, "size not supported"
+    emit_setfield_raw = emit_setfield_gc
 
-    # XXX adjust 64 bit
     def emit_getfield_gc(self, op, arglocs, regalloc):
         base_loc, ofs, res, size = arglocs
         if size.value == 8:
@@ -315,6 +314,9 @@
                 self.mc.lbzx(res.value, base_loc.value, ofs.value)
         else:
             assert 0, "size not supported"
+    emit_getfield_raw = emit_getfield_gc
+    emit_getfield_raw_pure = emit_getfield_gc
+    emit_getfield_gc_pure = emit_getfield_gc
 
     def emit_arraylen_gc(self, op, arglocs, regalloc):
         res, base_loc, ofs = arglocs
diff --git a/pypy/jit/backend/ppc/ppcgen/regalloc.py b/pypy/jit/backend/ppc/ppcgen/regalloc.py
--- a/pypy/jit/backend/ppc/ppcgen/regalloc.py
+++ b/pypy/jit/backend/ppc/ppcgen/regalloc.py
@@ -359,6 +359,8 @@
         self.possibly_free_vars(boxes)
         return [value_loc, base_loc, ofs_loc, imm(size)]
 
+    prepare_setfield_raw = prepare_setfield_gc
+
     def prepare_getfield_gc(self, op):
         a0 = op.getarg(0)
         ofs, size, ptr = self._unpack_fielddescr(op.getdescr())
@@ -375,6 +377,10 @@
         self.possibly_free_var(op.result)
         return [base_loc, ofs_loc, res, imm(size)]
 
+    prepare_getfield_raw = prepare_getfield_gc
+    prepare_getfield_raw_pure = prepare_getfield_gc
+    prepare_getfield_gc_pure = prepare_getfield_gc
+
     def prepare_arraylen_gc(self, op):
         arraydescr = op.getdescr()
         assert isinstance(arraydescr, BaseArrayDescr)

From noreply at buildbot.pypy.org  Thu Oct 27 19:36:41 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 27 Oct 2011 19:36:41 +0200 (CEST)
Subject: [pypy-commit] pypy stm: A subclass of llinterp that detects
 non-STM-friendly instructions.
Message-ID: <20111027173641.B1B1A820C2@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm
Changeset: r48524:6ed608d0af8e
Date: 2011-10-27 15:39 +0200
http://bitbucket.org/pypy/pypy/changeset/6ed608d0af8e/

Log:	A subclass of llinterp that detects non-STM-friendly instructions.

diff --git a/pypy/translator/stm/llstminterp.py b/pypy/translator/stm/llstminterp.py
new file mode 100644
--- /dev/null
+++ b/pypy/translator/stm/llstminterp.py
@@ -0,0 +1,37 @@
+from pypy.rpython.llinterp import LLFrame
+
+
+class ForbiddenInstructionInSTMMode(Exception):
+    pass
+
+
+def eval_stm_graph(llinterp, graph, values):
+    llinterp.frame_class = LLSTMFrame
+    try:
+        return llinterp.eval_graph(graph, values)
+    finally:
+        llinterp.frame_class = LLFrame
+
+
+class LLSTMFrame(LLFrame):
+
+    ALLOW_OPERATIONS = set([
+        'int_*',
+        ])
+
+    def getoperationhandler(self, opname):
+        ophandler = getattr(self, 'opstm_' + opname, None)
+        if ophandler is None:
+            self._validate_stmoperation_handler(opname)
+            ophandler = LLFrame.getoperationhandler(self, opname)
+            setattr(self, 'opstm_' + opname, ophandler)
+        return ophandler
+
+    def _validate_stmoperation_handler(self, opname):
+        OK = self.ALLOW_OPERATIONS
+        if opname in OK:
+            return
+        for i in range(len(opname)-1, -1, -1):
+            if (opname[:i] + '*') in OK:
+                return
+        raise ForbiddenInstructionInSTMMode(opname, self.graph)
diff --git a/pypy/translator/stm/test/test_llstminterp.py b/pypy/translator/stm/test/test_llstminterp.py
new file mode 100644
--- /dev/null
+++ b/pypy/translator/stm/test/test_llstminterp.py
@@ -0,0 +1,23 @@
+import py
+from pypy.rpython.lltypesystem import lltype
+from pypy.rpython.test.test_llinterp import get_interpreter
+from pypy.translator.stm.llstminterp import eval_stm_graph
+from pypy.translator.stm.llstminterp import ForbiddenInstructionInSTMMode
+
+
+def test_simple():
+    def func(n):
+        return (n+1) * (n+2)
+    interp, graph = get_interpreter(func, [5])
+    res = eval_stm_graph(interp, graph, [5])
+    assert res == 42
+
+def test_forbidden():
+    S = lltype.GcStruct('S', ('x', lltype.Signed))
+    p = lltype.malloc(S, immortal=True)
+    p.x = 42
+    def func(p):
+        return p.x
+    interp, graph = get_interpreter(func, [p])
+    py.test.raises(ForbiddenInstructionInSTMMode,
+                   eval_stm_graph, interp, graph, [p])

From noreply at buildbot.pypy.org  Thu Oct 27 19:36:42 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 27 Oct 2011 19:36:42 +0200 (CEST)
Subject: [pypy-commit] pypy stm: Keep track of the stm mode in which the
	LLSTMFrame is.
Message-ID: <20111027173642.E1562820C2@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm
Changeset: r48525:acb04280b949
Date: 2011-10-27 16:10 +0200
http://bitbucket.org/pypy/pypy/changeset/acb04280b949/

Log:	Keep track of the stm mode in which the LLSTMFrame is. Only allow
	operations that make sense in the current mode.

diff --git a/pypy/translator/stm/_rffi_stm.py b/pypy/translator/stm/_rffi_stm.py
--- a/pypy/translator/stm/_rffi_stm.py
+++ b/pypy/translator/stm/_rffi_stm.py
@@ -24,7 +24,7 @@
 descriptor_init = llexternal('stm_descriptor_init', [], lltype.Void)
 descriptor_done = llexternal('stm_descriptor_done', [], lltype.Void)
 
-begin_transaction = llexternal('STM_begin_transaction',[], lltype.Void)
+begin_transaction = llexternal('STM_begin_transaction', [], lltype.Void)
 commit_transaction = llexternal('stm_commit_transaction', [], lltype.Signed)
 
 stm_read_word = llexternal('stm_read_word', [SignedP], lltype.Signed)
diff --git a/pypy/translator/stm/llstminterp.py b/pypy/translator/stm/llstminterp.py
--- a/pypy/translator/stm/llstminterp.py
+++ b/pypy/translator/stm/llstminterp.py
@@ -1,13 +1,15 @@
 from pypy.rpython.llinterp import LLFrame
+from pypy.translator.stm import rstm
 
 
 class ForbiddenInstructionInSTMMode(Exception):
     pass
 
 
-def eval_stm_graph(llinterp, graph, values):
+def eval_stm_graph(llinterp, graph, values, stm_mode="not_in_transaction"):
     llinterp.frame_class = LLSTMFrame
     try:
+        llinterp.stm_mode = stm_mode
         return llinterp.eval_graph(graph, values)
     finally:
         llinterp.frame_class = LLFrame
@@ -15,23 +17,60 @@
 
 class LLSTMFrame(LLFrame):
 
-    ALLOW_OPERATIONS = set([
+    ALWAYS_ALLOW_OPERATIONS = set([
         'int_*',
         ])
+    ALLOW_WHEN_NOT_IN_TRANSACTION = set([
+        'stm_begin_transaction',
+        ])
+    ALLOW_WHEN_REGULAR_TRANSACTION = set([
+        'stm_getfield', 'stm_setfield',
+        'stm_commit_transaction',
+        ])
+    ALLOW_WHEN_INEVITABLE_TRANSACTION = ALLOW_WHEN_REGULAR_TRANSACTION.union(
+        set([
+        ]))
 
     def getoperationhandler(self, opname):
-        ophandler = getattr(self, 'opstm_' + opname, None)
+        stm_mode = self.llinterpreter.stm_mode
+        attrname = '_opstm_%s__%s' % (stm_mode, opname)
+        ophandler = getattr(self, attrname, None)
         if ophandler is None:
-            self._validate_stmoperation_handler(opname)
+            self._validate_stmoperation_handler(stm_mode, opname)
             ophandler = LLFrame.getoperationhandler(self, opname)
-            setattr(self, 'opstm_' + opname, ophandler)
+            setattr(self, attrname, ophandler)
         return ophandler
 
-    def _validate_stmoperation_handler(self, opname):
-        OK = self.ALLOW_OPERATIONS
-        if opname in OK:
+    def _op_in_set(self, opname, set):
+        if opname in set:
+            return True
+        for i in range(len(opname)-1, -1, -1):
+            if (opname[:i] + '*') in set:
+                return True
+        return False
+
+    def _validate_stmoperation_handler(self, stm_mode, opname):
+        if self._op_in_set(opname, self.ALWAYS_ALLOW_OPERATIONS):
             return
-        for i in range(len(opname)-1, -1, -1):
-            if (opname[:i] + '*') in OK:
-                return
-        raise ForbiddenInstructionInSTMMode(opname, self.graph)
+        allow = getattr(self, 'ALLOW_WHEN_' + stm_mode.upper())
+        if self._op_in_set(opname, allow):
+            return
+        raise ForbiddenInstructionInSTMMode(stm_mode, opname, self.graph)
+
+    # ---------- stm-only operations ----------
+    # Note that for these tests we assume no real multithreading,
+    # so that we just emulate the operations the easy way
+
+    def op_stm_getfield(self, struct, fieldname):
+        return self.op_getfield(struct, fieldname)
+
+    def op_stm_setfield(self, struct, fieldname, value):
+        self.op_setfield(struct, fieldname, value)
+
+    def op_stm_begin_transaction(self):
+        assert self.llinterpreter.stm_mode == "not_in_transaction"
+        self.llinterpreter.stm_mode = "regular_transaction"
+
+    def op_stm_commit_transaction(self):
+        assert self.llinterpreter.stm_mode != "not_in_transaction"
+        self.llinterpreter.stm_mode = "not_in_transaction"
diff --git a/pypy/translator/stm/rstm.py b/pypy/translator/stm/rstm.py
--- a/pypy/translator/stm/rstm.py
+++ b/pypy/translator/stm/rstm.py
@@ -79,6 +79,14 @@
         #print 'getting %x, mask=%x, replacing with %x' % (word, mask, val)
         _rffi_stm.stm_write_word(p, val)
 
+def begin_transaction():
+    "NOT_RPYTHON"
+    raise NotImplementedError("hard to really emulate")
+
+def commit_transaction():
+    "NOT_RPYTHON"
+    raise NotImplementedError("hard to really emulate")
+
 # ____________________________________________________________
 
 
@@ -93,6 +101,7 @@
         v_structptr = hop.inputarg(r_structptr, arg=0)
         fieldname = hop.args_v[1].value
         c_fieldname = hop.inputconst(lltype.Void, fieldname)
+        hop.exception_cannot_occur()
         return hop.genop('stm_getfield', [v_structptr, c_fieldname],
                          resulttype = hop.r_result)
 
@@ -109,4 +118,16 @@
         fieldname = hop.args_v[1].value
         v_newvalue = hop.inputarg(hop.args_r[2], arg=2)
         c_fieldname = hop.inputconst(lltype.Void, fieldname)
+        hop.exception_cannot_occur()
         hop.genop('stm_setfield', [v_structptr, c_fieldname, v_newvalue])
+
+
+class ExtEntry(ExtRegistryEntry):
+    _about_ = (begin_transaction, commit_transaction)
+
+    def compute_result_annotation(self):
+        return None
+
+    def specialize_call(self, hop):
+        hop.exception_cannot_occur()
+        hop.genop("stm_" + self.instance.__name__, [])
diff --git a/pypy/translator/stm/test/test_llstminterp.py b/pypy/translator/stm/test/test_llstminterp.py
--- a/pypy/translator/stm/test/test_llstminterp.py
+++ b/pypy/translator/stm/test/test_llstminterp.py
@@ -3,6 +3,7 @@
 from pypy.rpython.test.test_llinterp import get_interpreter
 from pypy.translator.stm.llstminterp import eval_stm_graph
 from pypy.translator.stm.llstminterp import ForbiddenInstructionInSTMMode
+from pypy.translator.stm import rstm
 
 
 def test_simple():
@@ -20,4 +21,35 @@
         return p.x
     interp, graph = get_interpreter(func, [p])
     py.test.raises(ForbiddenInstructionInSTMMode,
+                   eval_stm_graph, interp, graph, [p],
+                   stm_mode="regular_transaction")
+
+def test_stm_getfield():
+    S = lltype.GcStruct('S', ('x', lltype.Signed))
+    p = lltype.malloc(S, immortal=True)
+    p.x = 42
+    def func(p):
+        return rstm.stm_getfield(p, 'x')
+    interp, graph = get_interpreter(func, [p])
+    # forbidden in "not_in_transaction" mode
+    py.test.raises(ForbiddenInstructionInSTMMode,
                    eval_stm_graph, interp, graph, [p])
+    # works in "regular_transaction" mode
+    res = eval_stm_graph(interp, graph, [p], stm_mode="regular_transaction")
+    assert res == 42
+    # works in "inevitable_transaction" mode
+    res = eval_stm_graph(interp, graph, [p], stm_mode="inevitable_transaction")
+    assert res == 42
+
+def test_begin_commit_transaction():
+    S = lltype.GcStruct('S', ('x', lltype.Signed))
+    p = lltype.malloc(S, immortal=True)
+    p.x = 42
+    def func(p):
+        rstm.begin_transaction()
+        res = rstm.stm_getfield(p, 'x')
+        rstm.commit_transaction()
+        return res
+    interp, graph = get_interpreter(func, [p])
+    res = eval_stm_graph(interp, graph, [p])
+    assert res == 42

From noreply at buildbot.pypy.org  Thu Oct 27 19:36:44 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 27 Oct 2011 19:36:44 +0200 (CEST)
Subject: [pypy-commit] pypy stm: A graph transformer,
	so far just renaming '{get, set}field'
Message-ID: <20111027173644.1D04F820C2@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm
Changeset: r48526:3de5c2a4796d
Date: 2011-10-27 16:34 +0200
http://bitbucket.org/pypy/pypy/changeset/3de5c2a4796d/

Log:	A graph transformer, so far just renaming '{get,set}field' into
	'stm_{get,set}field'.

diff --git a/pypy/translator/stm/llstminterp.py b/pypy/translator/stm/llstminterp.py
--- a/pypy/translator/stm/llstminterp.py
+++ b/pypy/translator/stm/llstminterp.py
@@ -27,9 +27,8 @@
         'stm_getfield', 'stm_setfield',
         'stm_commit_transaction',
         ])
-    ALLOW_WHEN_INEVITABLE_TRANSACTION = ALLOW_WHEN_REGULAR_TRANSACTION.union(
-        set([
-        ]))
+    ALLOW_WHEN_INEVITABLE_TRANSACTION = ALLOW_WHEN_REGULAR_TRANSACTION.union([
+        ])
 
     def getoperationhandler(self, opname):
         stm_mode = self.llinterpreter.stm_mode
diff --git a/pypy/translator/stm/test/test_transform.py b/pypy/translator/stm/test/test_transform.py
new file mode 100644
--- /dev/null
+++ b/pypy/translator/stm/test/test_transform.py
@@ -0,0 +1,16 @@
+from pypy.rpython.lltypesystem import lltype
+from pypy.rpython.test.test_llinterp import get_interpreter
+from pypy.translator.stm.llstminterp import eval_stm_graph
+from pypy.translator.stm.transform import transform_graph
+
+
+def test_simple():
+    S = lltype.GcStruct('S', ('x', lltype.Signed))
+    p = lltype.malloc(S, immortal=True)
+    p.x = 42
+    def func(p):
+        return p.x
+    interp, graph = get_interpreter(func, [p])
+    transform_graph(graph)
+    res = eval_stm_graph(interp, graph, [p], stm_mode="regular_transaction")
+    assert res == 42
diff --git a/pypy/translator/stm/transform.py b/pypy/translator/stm/transform.py
new file mode 100644
--- /dev/null
+++ b/pypy/translator/stm/transform.py
@@ -0,0 +1,29 @@
+from pypy.objspace.flow.model import SpaceOperation
+
+
+class STMTransformer(object):
+
+    def transform_block(self, block):
+        if block.operations == ():
+            return
+        newoperations = []
+        for op in block.operations:
+            meth = getattr(self, 'stt_' + op.opname, list.append)
+            meth(newoperations, op)
+        block.operations = newoperations
+
+    def transform_graph(self, graph):
+        for block in graph.iterblocks():
+            self.transform_block(block)
+
+    def stt_getfield(self, newoperations, op):
+        op1 = SpaceOperation('stm_getfield', op.args, op.result)
+        newoperations.append(op1)
+
+    def stt_setfield(self, newoperations, op):
+        op1 = SpaceOperation('stm_setfield', op.args, op.result)
+        newoperations.append(op1)
+
+
+def transform_graph(graph):
+    STMTransformer().transform_graph(graph)

From noreply at buildbot.pypy.org  Thu Oct 27 19:36:45 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 27 Oct 2011 19:36:45 +0200 (CEST)
Subject: [pypy-commit] pypy stm: We cannot return out of the frame that
 started a regular transaction.
Message-ID: <20111027173645.49E6A820C2@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm
Changeset: r48527:070bf17d0fbb
Date: 2011-10-27 16:47 +0200
http://bitbucket.org/pypy/pypy/changeset/070bf17d0fbb/

Log:	We cannot return out of the frame that started a regular
	transaction.

diff --git a/pypy/translator/stm/llstminterp.py b/pypy/translator/stm/llstminterp.py
--- a/pypy/translator/stm/llstminterp.py
+++ b/pypy/translator/stm/llstminterp.py
@@ -5,12 +5,20 @@
 class ForbiddenInstructionInSTMMode(Exception):
     pass
 
+class ReturnWithTransactionActive(Exception):
+    pass
+
 
 def eval_stm_graph(llinterp, graph, values, stm_mode="not_in_transaction"):
     llinterp.frame_class = LLSTMFrame
     try:
         llinterp.stm_mode = stm_mode
-        return llinterp.eval_graph(graph, values)
+        llinterp.last_transaction_started_in_frame = None
+        res = llinterp.eval_graph(graph, values)
+        assert llinterp.stm_mode == stm_mode, (
+            "llinterp.stm_mode is %r after eval_graph, but should be %r" % (
+            llinterp.stm_mode, stm_mode))
+        return res
     finally:
         llinterp.frame_class = LLFrame
 
@@ -19,6 +27,7 @@
 
     ALWAYS_ALLOW_OPERATIONS = set([
         'int_*',
+        'direct_call',
         ])
     ALLOW_WHEN_NOT_IN_TRANSACTION = set([
         'stm_begin_transaction',
@@ -30,6 +39,13 @@
     ALLOW_WHEN_INEVITABLE_TRANSACTION = ALLOW_WHEN_REGULAR_TRANSACTION.union([
         ])
 
+    def eval(self):
+        res = LLFrame.eval(self)
+        if (self.llinterpreter.stm_mode == "regular_transaction" and
+            self.llinterpreter.last_transaction_started_in_frame is self):
+            raise ReturnWithTransactionActive(self.graph)
+        return res
+
     def getoperationhandler(self, opname):
         stm_mode = self.llinterpreter.stm_mode
         attrname = '_opstm_%s__%s' % (stm_mode, opname)
@@ -69,6 +85,7 @@
     def op_stm_begin_transaction(self):
         assert self.llinterpreter.stm_mode == "not_in_transaction"
         self.llinterpreter.stm_mode = "regular_transaction"
+        self.llinterpreter.last_transaction_started_in_frame = self
 
     def op_stm_commit_transaction(self):
         assert self.llinterpreter.stm_mode != "not_in_transaction"
diff --git a/pypy/translator/stm/test/test_llstminterp.py b/pypy/translator/stm/test/test_llstminterp.py
--- a/pypy/translator/stm/test/test_llstminterp.py
+++ b/pypy/translator/stm/test/test_llstminterp.py
@@ -3,6 +3,7 @@
 from pypy.rpython.test.test_llinterp import get_interpreter
 from pypy.translator.stm.llstminterp import eval_stm_graph
 from pypy.translator.stm.llstminterp import ForbiddenInstructionInSTMMode
+from pypy.translator.stm.llstminterp import ReturnWithTransactionActive
 from pypy.translator.stm import rstm
 
 
@@ -53,3 +54,25 @@
     interp, graph = get_interpreter(func, [p])
     res = eval_stm_graph(interp, graph, [p])
     assert res == 42
+
+def test_call_and_return_with_regular_transaction():
+    def g():
+        pass
+    g._dont_inline_ = True
+    def func():
+        rstm.begin_transaction()
+        g()
+        rstm.commit_transaction()
+    interp, graph = get_interpreter(func, [])
+    eval_stm_graph(interp, graph, [])
+
+def test_cannot_return_with_regular_transaction():
+    def g():
+        rstm.begin_transaction()
+    g._dont_inline_ = True
+    def func():
+        g()
+        rstm.commit_transaction()
+    interp, graph = get_interpreter(func, [])
+    py.test.raises(ReturnWithTransactionActive,
+                   eval_stm_graph, interp, graph, [])

From noreply at buildbot.pypy.org  Thu Oct 27 19:36:46 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 27 Oct 2011 19:36:46 +0200 (CEST)
Subject: [pypy-commit] pypy stm: - Refactor to support getfield of
	immutables.
Message-ID: <20111027173646.782C6820C2@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm
Changeset: r48528:caa7e690cfc7
Date: 2011-10-27 17:05 +0200
http://bitbucket.org/pypy/pypy/changeset/caa7e690cfc7/

Log:	- Refactor to support getfield of immutables. That's enough to
	raise simple exceptions.

	- Complain when an exception leaves the frame in which we did
	begin_transaction().

diff --git a/pypy/translator/stm/llstminterp.py b/pypy/translator/stm/llstminterp.py
--- a/pypy/translator/stm/llstminterp.py
+++ b/pypy/translator/stm/llstminterp.py
@@ -1,4 +1,5 @@
-from pypy.rpython.llinterp import LLFrame
+from pypy.rpython.lltypesystem import lltype
+from pypy.rpython.llinterp import LLFrame, LLException
 from pypy.translator.stm import rstm
 
 
@@ -26,35 +27,36 @@
 class LLSTMFrame(LLFrame):
 
     ALWAYS_ALLOW_OPERATIONS = set([
-        'int_*',
+        'int_*', 'same_as', 'cast_*',
         'direct_call',
         ])
-    ALLOW_WHEN_NOT_IN_TRANSACTION = set([
-        'stm_begin_transaction',
-        ])
-    ALLOW_WHEN_REGULAR_TRANSACTION = set([
-        'stm_getfield', 'stm_setfield',
-        'stm_commit_transaction',
-        ])
-    ALLOW_WHEN_INEVITABLE_TRANSACTION = ALLOW_WHEN_REGULAR_TRANSACTION.union([
-        ])
 
     def eval(self):
-        res = LLFrame.eval(self)
-        if (self.llinterpreter.stm_mode == "regular_transaction" and
-            self.llinterpreter.last_transaction_started_in_frame is self):
-            raise ReturnWithTransactionActive(self.graph)
+        try:
+            res = LLFrame.eval(self)
+        except LLException, e:
+            self.returning_from_frame_now()
+            raise e
+        self.returning_from_frame_now()
         return res
 
+    def returning_from_frame_now(self):
+        if (self.llinterpreter.stm_mode == "regular_transaction" and
+                self.llinterpreter.last_transaction_started_in_frame is self):
+            raise ReturnWithTransactionActive(self.graph)
+
     def getoperationhandler(self, opname):
-        stm_mode = self.llinterpreter.stm_mode
-        attrname = '_opstm_%s__%s' % (stm_mode, opname)
-        ophandler = getattr(self, attrname, None)
-        if ophandler is None:
-            self._validate_stmoperation_handler(stm_mode, opname)
-            ophandler = LLFrame.getoperationhandler(self, opname)
-            setattr(self, attrname, ophandler)
-        return ophandler
+        try:
+            return getattr(self, 'opstm_' + opname)
+        except AttributeError:
+            stm_mode = self.llinterpreter.stm_mode
+            attrname = '_opstm_%s__%s' % (stm_mode, opname)
+            ophandler = getattr(self, attrname, None)
+            if ophandler is None:
+                self._validate_stmoperation_handler(stm_mode, opname)
+                ophandler = LLFrame.getoperationhandler(self, opname)
+                setattr(self, attrname, ophandler)
+            return ophandler
 
     def _op_in_set(self, opname, set):
         if opname in set:
@@ -67,26 +69,42 @@
     def _validate_stmoperation_handler(self, stm_mode, opname):
         if self._op_in_set(opname, self.ALWAYS_ALLOW_OPERATIONS):
             return
-        allow = getattr(self, 'ALLOW_WHEN_' + stm_mode.upper())
-        if self._op_in_set(opname, allow):
-            return
         raise ForbiddenInstructionInSTMMode(stm_mode, opname, self.graph)
 
+    # ---------- operations that are sometimes safe ----------
+
+    def opstm_getfield(self, struct, fieldname):
+        STRUCT = lltype.typeOf(struct).TO
+        if STRUCT._immutable_field(fieldname):
+            # immutable field reads are always allowed
+            return LLFrame.op_getfield(self, struct, fieldname)
+        else:
+            # mutable 'getfields' are always forbidden for now
+            self.check_stm_mode(lambda m: False)
+            xxx
+
     # ---------- stm-only operations ----------
     # Note that for these tests we assume no real multithreading,
     # so that we just emulate the operations the easy way
 
-    def op_stm_getfield(self, struct, fieldname):
-        return self.op_getfield(struct, fieldname)
+    def check_stm_mode(self, checker):
+        stm_mode = self.llinterpreter.stm_mode
+        if not checker(stm_mode):
+            raise ForbiddenInstructionInSTMMode(stm_mode, self.graph)
 
-    def op_stm_setfield(self, struct, fieldname, value):
-        self.op_setfield(struct, fieldname, value)
+    def opstm_stm_getfield(self, struct, fieldname):
+        self.check_stm_mode(lambda m: m != "not_in_transaction")
+        return LLFrame.op_getfield(self, struct, fieldname)
 
-    def op_stm_begin_transaction(self):
-        assert self.llinterpreter.stm_mode == "not_in_transaction"
+    def opstm_stm_setfield(self, struct, fieldname, value):
+        self.check_stm_mode(lambda m: m != "not_in_transaction")
+        LLFrame.op_setfield(self, struct, fieldname, value)
+
+    def opstm_stm_begin_transaction(self):
+        self.check_stm_mode(lambda m: m == "not_in_transaction")
         self.llinterpreter.stm_mode = "regular_transaction"
         self.llinterpreter.last_transaction_started_in_frame = self
 
-    def op_stm_commit_transaction(self):
-        assert self.llinterpreter.stm_mode != "not_in_transaction"
+    def opstm_stm_commit_transaction(self):
+        self.check_stm_mode(lambda m: m != "not_in_transaction")
         self.llinterpreter.stm_mode = "not_in_transaction"
diff --git a/pypy/translator/stm/test/test_llstminterp.py b/pypy/translator/stm/test/test_llstminterp.py
--- a/pypy/translator/stm/test/test_llstminterp.py
+++ b/pypy/translator/stm/test/test_llstminterp.py
@@ -6,6 +6,9 @@
 from pypy.translator.stm.llstminterp import ReturnWithTransactionActive
 from pypy.translator.stm import rstm
 
+ALL_STM_MODES = ["not_in_transaction",
+                 "regular_transaction",
+                 "inevitable_transaction"]
 
 def test_simple():
     def func(n):
@@ -42,6 +45,18 @@
     res = eval_stm_graph(interp, graph, [p], stm_mode="inevitable_transaction")
     assert res == 42
 
+def test_getfield_immutable():
+    S = lltype.GcStruct('S', ('x', lltype.Signed), hints = {'immutable': True})
+    p = lltype.malloc(S, immortal=True)
+    p.x = 42
+    def func(p):
+        return p.x
+    interp, graph = get_interpreter(func, [p])
+    # a plain 'getfield' of an immutable field works in all modes
+    for mode in ALL_STM_MODES:
+        res = eval_stm_graph(interp, graph, [p], stm_mode=mode)
+        assert res == 42
+
 def test_begin_commit_transaction():
     S = lltype.GcStruct('S', ('x', lltype.Signed))
     p = lltype.malloc(S, immortal=True)
@@ -76,3 +91,18 @@
     interp, graph = get_interpreter(func, [])
     py.test.raises(ReturnWithTransactionActive,
                    eval_stm_graph, interp, graph, [])
+
+def test_cannot_raise_with_regular_transaction():
+    def g():
+        rstm.begin_transaction()
+        raise ValueError
+    g._dont_inline_ = True
+    def func():
+        try:
+            g()
+        except ValueError:
+            pass
+        rstm.commit_transaction()
+    interp, graph = get_interpreter(func, [])
+    py.test.raises(ReturnWithTransactionActive,
+                   eval_stm_graph, interp, graph, [])

From noreply at buildbot.pypy.org  Thu Oct 27 19:36:47 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 27 Oct 2011 19:36:47 +0200 (CEST)
Subject: [pypy-commit] pypy stm: Simplify.
Message-ID: <20111027173647.A2EAE820C2@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm
Changeset: r48529:d382548b28e1
Date: 2011-10-27 17:10 +0200
http://bitbucket.org/pypy/pypy/changeset/d382548b28e1/

Log:	Simplify.

diff --git a/pypy/translator/stm/llstminterp.py b/pypy/translator/stm/llstminterp.py
--- a/pypy/translator/stm/llstminterp.py
+++ b/pypy/translator/stm/llstminterp.py
@@ -46,17 +46,12 @@
             raise ReturnWithTransactionActive(self.graph)
 
     def getoperationhandler(self, opname):
-        try:
-            return getattr(self, 'opstm_' + opname)
-        except AttributeError:
-            stm_mode = self.llinterpreter.stm_mode
-            attrname = '_opstm_%s__%s' % (stm_mode, opname)
-            ophandler = getattr(self, attrname, None)
-            if ophandler is None:
-                self._validate_stmoperation_handler(stm_mode, opname)
-                ophandler = LLFrame.getoperationhandler(self, opname)
-                setattr(self, attrname, ophandler)
-            return ophandler
+        ophandler = getattr(self, 'opstm_' + opname, None)
+        if ophandler is None:
+            self._validate_stmoperation_handler(opname)
+            ophandler = LLFrame.getoperationhandler(self, opname)
+            setattr(self, 'opstm_' + opname, ophandler)
+        return ophandler
 
     def _op_in_set(self, opname, set):
         if opname in set:
@@ -66,10 +61,10 @@
                 return True
         return False
 
-    def _validate_stmoperation_handler(self, stm_mode, opname):
+    def _validate_stmoperation_handler(self, opname):
         if self._op_in_set(opname, self.ALWAYS_ALLOW_OPERATIONS):
             return
-        raise ForbiddenInstructionInSTMMode(stm_mode, opname, self.graph)
+        raise ForbiddenInstructionInSTMMode(opname, self.graph)
 
     # ---------- operations that are sometimes safe ----------
 

From noreply at buildbot.pypy.org  Thu Oct 27 19:36:48 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 27 Oct 2011 19:36:48 +0200 (CEST)
Subject: [pypy-commit] pypy stm: transform.py should not touch immutable
	getfields
Message-ID: <20111027173648.CF008820C2@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm
Changeset: r48530:df4fdd533f50
Date: 2011-10-27 17:14 +0200
http://bitbucket.org/pypy/pypy/changeset/df4fdd533f50/

Log:	transform.py should not touch immutable getfields

diff --git a/pypy/translator/stm/test/test_transform.py b/pypy/translator/stm/test/test_transform.py
--- a/pypy/translator/stm/test/test_transform.py
+++ b/pypy/translator/stm/test/test_transform.py
@@ -1,5 +1,6 @@
 from pypy.rpython.lltypesystem import lltype
 from pypy.rpython.test.test_llinterp import get_interpreter
+from pypy.objspace.flow.model import summary
 from pypy.translator.stm.llstminterp import eval_stm_graph
 from pypy.translator.stm.transform import transform_graph
 
@@ -12,5 +13,18 @@
         return p.x
     interp, graph = get_interpreter(func, [p])
     transform_graph(graph)
+    assert summary(graph) == {'stm_getfield': 1}
     res = eval_stm_graph(interp, graph, [p], stm_mode="regular_transaction")
     assert res == 42
+
+def test_immutable_field():
+    S = lltype.GcStruct('S', ('x', lltype.Signed), hints = {'immutable': True})
+    p = lltype.malloc(S, immortal=True)
+    p.x = 42
+    def func(p):
+        return p.x
+    interp, graph = get_interpreter(func, [p])
+    transform_graph(graph)
+    assert summary(graph) == {'getfield': 1}
+    res = eval_stm_graph(interp, graph, [p], stm_mode="regular_transaction")
+    assert res == 42
diff --git a/pypy/translator/stm/transform.py b/pypy/translator/stm/transform.py
--- a/pypy/translator/stm/transform.py
+++ b/pypy/translator/stm/transform.py
@@ -17,7 +17,11 @@
             self.transform_block(block)
 
     def stt_getfield(self, newoperations, op):
-        op1 = SpaceOperation('stm_getfield', op.args, op.result)
+        STRUCT = op.args[0].concretetype.TO
+        if STRUCT._immutable_field(op.args[1].value):
+            op1 = op
+        else:
+            op1 = SpaceOperation('stm_getfield', op.args, op.result)
         newoperations.append(op1)
 
     def stt_setfield(self, newoperations, op):

From noreply at buildbot.pypy.org  Thu Oct 27 19:39:30 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Oct 2011 19:39:30 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-multidim: creation from sequences
Message-ID: <20111027173930.389A2820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: numpy-multidim
Changeset: r48531:d73fb983a4bb
Date: 2011-10-27 19:38 +0200
http://bitbucket.org/pypy/pypy/changeset/d73fb983a4bb/

Log:	creation from sequences

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -16,6 +16,42 @@
                                                        'dtype'])
 slice_driver = jit.JitDriver(greens=['signature'], reds=['i', 'self', 'source'])
 
+def _find_dtype(space, w_iterable):
+    stack = [w_iterable]
+    w_dtype = None
+    while stack:
+        w_next = stack.pop()
+        if space.issequence_w(w_next):
+            for w_item in space.listview(w_next):
+                stack.append(w_item)
+        else:
+            w_dtype = interp_ufuncs.find_dtype_for_scalar(space, w_item, w_dtype)
+            if w_dtype is space.fromcache(interp_dtype.W_Float64Dtype):
+                return w_dtype
+    if w_dtype is None:
+        return space.w_None
+    return w_dtype
+
+def _find_shape_and_elems(space, w_iterable):
+    shape = [space.len_w(w_iterable)]
+    batch = space.listview(w_iterable)
+    while True:
+        new_batch = []
+        if not space.issequence_w(batch[0]):
+            for elem in batch:
+                if space.issequence_w(elem):
+                    raise OperationError(space.w_ValueError, space.wrap(
+                        "setting an array element with a sequence"))
+            return shape, batch
+        size = space.len_w(batch[0])
+        for w_elem in batch:
+            if not space.issequence_w(w_elem) or space.len_w(w_elem) != size:
+                raise OperationError(space.w_ValueError, space.wrap(
+                    "setting an array element with a sequence"))
+            new_batch += space.listview(w_elem)
+        shape.append(size)
+        batch = new_batch
+
 class BaseArray(Wrappable):
     _attrs_ = ["invalidates", "signature"]
 
@@ -36,24 +72,18 @@
         self.invalidates.append(other)
 
     def descr__new__(space, w_subtype, w_size_or_iterable, w_dtype=None):
-        l = space.listview(w_size_or_iterable)
+        # find scalar
         if space.is_w(w_dtype, space.w_None):
-            w_dtype = None
-            for w_item in l:
-                w_dtype = interp_ufuncs.find_dtype_for_scalar(space, w_item, w_dtype)
-                if w_dtype is space.fromcache(interp_dtype.W_Float64Dtype):
-                    break
-            if w_dtype is None:
-                w_dtype = space.w_None
-
+            w_dtype = _find_dtype(space, w_size_or_iterable)
         dtype = space.interp_w(interp_dtype.W_Dtype,
             space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype)
         )
-        arr = NDimArray(len(l), [len(l)], dtype=dtype)
+        shape, elems_w = _find_shape_and_elems(space, w_size_or_iterable)
+        size = len(elems_w)
+        arr = NDimArray(size, shape, dtype=dtype)
         i = 0
-        for w_elem in l:
+        for i, w_elem in enumerate(elems_w):
             dtype.setitem_w(space, arr.storage, i, w_elem)
-            i += 1
         return arr
 
     def _unaryop_impl(ufunc_name):
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -49,6 +49,9 @@
         from numpy import array
         a = array(range(5))
         assert a[3] == 3
+        a = array(1)
+        assert a[0] == 1
+        assert a.shape == ()
 
     def test_repr(self):
         from numpy import array, zeros
@@ -660,6 +663,17 @@
         assert a[0][1][1] == 13
         assert a[1][2][1] == 15
 
+    def test_init_2(self):
+        import numpy
+        raises(ValueError, numpy.array, [[1], 2])
+        raises(ValueError, numpy.array, [[1, 2], [3]])
+        raises(ValueError, numpy.array, [[[1, 2], [3, 4], 5]])
+        raises(ValueError, numpy.array, [[[1, 2], [3, 4], [5]]])
+        a = numpy.array([[1, 2], [4, 5]])
+        assert a[0, 1] == a[0][1] == 2
+        a = numpy.array(([[[1, 2], [3, 4], [5, 6]]]))
+        assert a[0, 1] == [3, 4]
+
     def test_setitem_slice(self):
         import numpy
         a = numpy.zeros((3, 4))

From noreply at buildbot.pypy.org  Thu Oct 27 19:41:58 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Oct 2011 19:41:58 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-multidim: creation from sequences
Message-ID: <20111027174158.AB71A820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: numpy-multidim
Changeset: r48532:57661ae58454
Date: 2011-10-27 19:40 +0200
http://bitbucket.org/pypy/pypy/changeset/57661ae58454/

Log:	creation from sequences

diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -680,6 +680,12 @@
         a[1] = [1, 2, 3, 4]
         assert a[1, 2] == 3
         raises(TypeError, a[1].__setitem__, [1, 2, 3])
+        a = numpy.array([[1, 2], [3, 4]])
+        assert a == [[1, 2], [3, 4]]
+        a[1] = numpy.array([5, 6])
+        assert a == [[1, 2], [5, 6]]
+        a[:,1] = numpy.array([8, 10])
+        assert a == [[1, 8], [5, 10]]
 
 class AppTestSupport(object):
     def setup_class(cls):

From noreply at buildbot.pypy.org  Thu Oct 27 19:49:51 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Thu, 27 Oct 2011 19:49:51 +0200 (CEST)
Subject: [pypy-commit] pypy default: optimize numpy.ones a bit
Message-ID: <20111027174951.15A55820C2@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r48533:e54c4fc626c7
Date: 2011-10-27 13:49 -0400
http://bitbucket.org/pypy/pypy/changeset/e54c4fc626c7/

Log:	optimize numpy.ones a bit

diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -108,6 +108,12 @@
         def setitem_w(self, space, storage, i, w_item):
             self.setitem(storage, i, self.unwrap(space, w_item))
 
+        def fill(self, storage, item, start, stop):
+            storage = self.unerase(storage)
+            item = self.unbox(item)
+            for i in xrange(start, stop):
+                storage[i] = item
+
         @specialize.argtype(1)
         def adapt_val(self, val):
             return self.box(rffi.cast(TP.TO.OF, val))
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -565,8 +565,7 @@
 
     arr = SingleDimArray(size, dtype=dtype)
     one = dtype.adapt_val(1)
-    for i in xrange(size):
-        arr.dtype.setitem(arr.storage, i, one)
+    arr.dtype.fill(arr.storage, one, 0, size)
     return space.wrap(arr)
 
 BaseArray.typedef = TypeDef(

From noreply at buildbot.pypy.org  Thu Oct 27 19:58:36 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Thu, 27 Oct 2011 19:58:36 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix this test
Message-ID: <20111027175836.1937A820C2@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r48534:de9715d6219f
Date: 2011-10-27 13:58 -0400
http://bitbucket.org/pypy/pypy/changeset/de9715d6219f/

Log:	fix this test

diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -4206,10 +4206,12 @@
         class FakeCallInfoCollection:
             def callinfo_for_oopspec(self, oopspecindex):
                 calldescrtype = type(LLtypeMixin.strequaldescr)
+                effectinfotype = type(LLtypeMixin.strequaldescr.get_extra_info())
                 for value in LLtypeMixin.__dict__.values():
                     if isinstance(value, calldescrtype):
                         extra = value.get_extra_info()
-                        if extra and extra.oopspecindex == oopspecindex:
+                        if (extra and isinstance(extra, effectinfotype) and
+                            extra.oopspecindex == oopspecindex):
                             # returns 0 for 'func' in this test
                             return value, 0
                 raise AssertionError("not found: oopspecindex=%d" %
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -5800,10 +5800,12 @@
         class FakeCallInfoCollection:
             def callinfo_for_oopspec(self, oopspecindex):
                 calldescrtype = type(LLtypeMixin.strequaldescr)
+                effectinfotype = type(LLtypeMixin.strequaldescr.get_extra_info())
                 for value in LLtypeMixin.__dict__.values():
                     if isinstance(value, calldescrtype):
                         extra = value.get_extra_info()
-                        if extra and extra.oopspecindex == oopspecindex:
+                        if (extra and isinstance(extra, effectinfotype) and
+                            extra.oopspecindex == oopspecindex):
                             # returns 0 for 'func' in this test
                             return value, 0
                 raise AssertionError("not found: oopspecindex=%d" %

From noreply at buildbot.pypy.org  Thu Oct 27 20:01:40 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Oct 2011 20:01:40 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-multidim: nonzero support and more tests
	for slices
Message-ID: <20111027180140.B3491820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: numpy-multidim
Changeset: r48535:e7d27a90d510
Date: 2011-10-27 20:01 +0200
http://bitbucket.org/pypy/pypy/changeset/e7d27a90d510/

Log:	nonzero support and more tests for slices

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -25,7 +25,7 @@
             for w_item in space.listview(w_next):
                 stack.append(w_item)
         else:
-            w_dtype = interp_ufuncs.find_dtype_for_scalar(space, w_item, w_dtype)
+            w_dtype = interp_ufuncs.find_dtype_for_scalar(space, w_next, w_dtype)
             if w_dtype is space.fromcache(interp_dtype.W_Float64Dtype):
                 return w_dtype
     if w_dtype is None:
@@ -71,14 +71,16 @@
     def add_invalidates(self, other):
         self.invalidates.append(other)
 
-    def descr__new__(space, w_subtype, w_size_or_iterable, w_dtype=None):
+    def descr__new__(space, w_subtype, w_item_or_iterable, w_dtype=None):
         # find scalar
         if space.is_w(w_dtype, space.w_None):
-            w_dtype = _find_dtype(space, w_size_or_iterable)
+            w_dtype = _find_dtype(space, w_item_or_iterable)
         dtype = space.interp_w(interp_dtype.W_Dtype,
             space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype)
         )
-        shape, elems_w = _find_shape_and_elems(space, w_size_or_iterable)
+        if not space.issequence_w(w_item_or_iterable):
+            return scalar_w(space, dtype, w_item_or_iterable)
+        shape, elems_w = _find_shape_and_elems(space, w_item_or_iterable)
         size = len(elems_w)
         arr = NDimArray(size, shape, dtype=dtype)
         i = 0
@@ -365,6 +367,12 @@
     def descr_mean(self, space):
         return space.wrap(space.float_w(self.descr_sum(space))/self.find_size())
 
+    def descr_nonzero(self, space):
+        if self.find_size() > 1:
+            raise OperationError(space.w_ValueError, space.wrap(
+                "The truth value of an array with more than one element is ambiguous. Use a.any() or a.all()"))
+        return self.get_concrete().eval(0).wrap(space)
+
 def convert_to_array(space, w_obj):
     if isinstance(w_obj, BaseArray):
         return w_obj
@@ -395,7 +403,10 @@
         self.value = value
 
     def find_size(self):
-        raise ValueError
+        return 1
+
+    def get_concrete(self):
+        return self
 
     def find_dtype(self):
         return self.dtype
@@ -711,6 +722,7 @@
     __pos__ = interp2app(BaseArray.descr_pos),
     __neg__ = interp2app(BaseArray.descr_neg),
     __abs__ = interp2app(BaseArray.descr_abs),
+    __nonzero__ = interp2app(BaseArray.descr_nonzero),
 
     __add__ = interp2app(BaseArray.descr_add),
     __sub__ = interp2app(BaseArray.descr_sub),
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -604,6 +604,16 @@
             for i in xrange(5):
                 assert c[i] == func(b[i], 3)
 
+    def test_nonzero(self):
+        from numpy import array
+        a = array([1, 2])
+        raises(ValueError, bool, a)
+        raises(ValueError, bool, a == a)
+        assert bool(array(1))
+        assert not bool(array(0))
+        assert bool(array([1]))
+        assert not bool(array([0]))
+
 class AppTestMultiDim(BaseNumpyAppTest):
     def test_init(self):
         import numpy
@@ -672,7 +682,7 @@
         a = numpy.array([[1, 2], [4, 5]])
         assert a[0, 1] == a[0][1] == 2
         a = numpy.array(([[[1, 2], [3, 4], [5, 6]]]))
-        assert a[0, 1] == [3, 4]
+        assert (a[0, 1] == [3, 4]).all()
 
     def test_setitem_slice(self):
         import numpy
@@ -681,11 +691,13 @@
         assert a[1, 2] == 3
         raises(TypeError, a[1].__setitem__, [1, 2, 3])
         a = numpy.array([[1, 2], [3, 4]])
-        assert a == [[1, 2], [3, 4]]
+        assert (a == [[1, 2], [3, 4]]).all()
         a[1] = numpy.array([5, 6])
-        assert a == [[1, 2], [5, 6]]
+        assert (a == [[1, 2], [5, 6]]).all()
         a[:,1] = numpy.array([8, 10])
-        assert a == [[1, 8], [5, 10]]
+        assert (a == [[1, 8], [5, 10]]).all()
+        a[:,::-1] = numpy.array([11, 12])
+        assert (a == [[12, 11], [12, 11]]).all()
 
 class AppTestSupport(object):
     def setup_class(cls):

From noreply at buildbot.pypy.org  Thu Oct 27 20:05:30 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Oct 2011 20:05:30 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-multidim: make the test test what I wanted
	to test
Message-ID: <20111027180530.4220D820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: numpy-multidim
Changeset: r48536:fbb3d36afef8
Date: 2011-10-27 20:04 +0200
http://bitbucket.org/pypy/pypy/changeset/fbb3d36afef8/

Log:	make the test test what I wanted to test

diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -696,8 +696,8 @@
         assert (a == [[1, 2], [5, 6]]).all()
         a[:,1] = numpy.array([8, 10])
         assert (a == [[1, 8], [5, 10]]).all()
-        a[:,::-1] = numpy.array([11, 12])
-        assert (a == [[12, 11], [12, 11]]).all()
+        a[0,::-1] = numpy.array([11, 12])
+        assert (a == [[12, 11], [5, 10]]).all()
 
 class AppTestSupport(object):
     def setup_class(cls):

From noreply at buildbot.pypy.org  Thu Oct 27 20:23:28 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Oct 2011 20:23:28 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-multidim: more tests
Message-ID: <20111027182328.98F0D820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: numpy-multidim
Changeset: r48537:2944b13d49dd
Date: 2011-10-27 20:22 +0200
http://bitbucket.org/pypy/pypy/changeset/2944b13d49dd/

Log:	more tests

diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -699,6 +699,11 @@
         a[0,::-1] = numpy.array([11, 12])
         assert (a == [[12, 11], [5, 10]]).all()
 
+    def test_ufunc(self):
+        from numpy import array
+        a = array([[1, 2], [3, 4], [5, 6]])
+        assert ((a + a) == array([[1+1, 2+2], [3+3, 4+4], [5+5, 6+6]])).all()
+
 class AppTestSupport(object):
     def setup_class(cls):
         import struct

From noreply at buildbot.pypy.org  Thu Oct 27 20:30:21 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Thu, 27 Oct 2011 20:30:21 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: this is done
Message-ID: <20111027183021.4C85B820C2@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: extradoc
Changeset: r3951:386468696bbc
Date: 2011-10-27 14:30 -0400
http://bitbucket.org/pypy/extradoc/changeset/386468696bbc/

Log:	this is done

diff --git a/planning/jit.txt b/planning/jit.txt
--- a/planning/jit.txt
+++ b/planning/jit.txt
@@ -190,7 +190,5 @@
   - Better pointer aliasing analyzer that will emit guards that pointers are
 	different when needed.
 
-  - Make heap optimizer aware of setitems produced by forcing virtuals.
-
   - Movinging loop-invariant setitems out of the loops entierly.
 

From notifications-noreply at bitbucket.org  Thu Oct 27 20:46:11 2011
From: notifications-noreply at bitbucket.org (Bitbucket)
Date: Thu, 27 Oct 2011 18:46:11 -0000
Subject: [pypy-commit] Notification: pypy
Message-ID: <20111027184611.1915.64418@bitbucket01.managed.contegix.com>

You have received a notification from Bruno Gola.

Hi, I forked pypy. My fork is at https://bitbucket.org/brunogola/pypy.

--
Disable notifications at https://bitbucket.org/account/notifications/

From noreply at buildbot.pypy.org  Thu Oct 27 21:42:53 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Thu, 27 Oct 2011 21:42:53 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix a test that was still trying to use
 W_TypeObject.interplevel_cls
Message-ID: <20111027194253.AA915820C2@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r48538:993b01fd53d4
Date: 2011-10-27 15:42 -0400
http://bitbucket.org/pypy/pypy/changeset/993b01fd53d4/

Log:	fix a test that was still trying to use W_TypeObject.interplevel_cls

diff --git a/pypy/objspace/std/test/test_stdobjspace.py b/pypy/objspace/std/test/test_stdobjspace.py
--- a/pypy/objspace/std/test/test_stdobjspace.py
+++ b/pypy/objspace/std/test/test_stdobjspace.py
@@ -14,11 +14,11 @@
 
     def test_int_w_non_int(self):
         raises(OperationError,self.space.int_w,self.space.wrap(None))
-        raises(OperationError,self.space.int_w,self.space.wrap(""))        
+        raises(OperationError,self.space.int_w,self.space.wrap(""))
 
     def test_uint_w_non_int(self):
         raises(OperationError,self.space.uint_w,self.space.wrap(None))
-        raises(OperationError,self.space.uint_w,self.space.wrap(""))        
+        raises(OperationError,self.space.uint_w,self.space.wrap(""))
 
     def test_multimethods_defined_on(self):
         from pypy.objspace.std.stdtypedef import multimethods_defined_on
@@ -49,14 +49,14 @@
     def test_fastpath_isinstance(self):
         from pypy.objspace.std.stringobject import W_StringObject
         from pypy.objspace.std.intobject import W_IntObject
-        
+
         space = self.space
-        assert space.w_str.interplevel_cls is W_StringObject
-        assert space.w_int.interplevel_cls is W_IntObject
+        assert space._get_interplevel_cls(space.w_str) is W_StringObject
+        assert space._get_interplevel_cls(space.w_int) is W_IntObject
         class X(W_StringObject):
             def __init__(self):
                 pass
-            
+
             typedef = None
 
         assert space.isinstance_w(X(), space.w_str)

From noreply at buildbot.pypy.org  Thu Oct 27 22:31:30 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Oct 2011 22:31:30 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-minilang: progress on supporting strange
 language for test_zjit
Message-ID: <20111027203130.CD9AB820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: numpy-minilang
Changeset: r48539:ced3b41aaacb
Date: 2011-10-27 22:07 +0200
http://bitbucket.org/pypy/pypy/changeset/ced3b41aaacb/

Log:	progress on supporting strange language for test_zjit

diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -4,8 +4,8 @@
 """
 
 from pypy.interpreter.baseobjspace import InternalSpaceCache, W_Root
-from pypy.module.micronumpy.interp_dtype import W_Float64Dtype
-from pypy.module.micronumpy.interp_numarray import Scalar, SingleDimArray, BaseArray
+from pypy.module.micronumpy.interp_dtype import W_Float64Dtype, W_Int32Dtype
+from pypy.module.micronumpy.interp_numarray import Scalar, BaseArray, descr_new_array
 from pypy.rlib.objectmodel import specialize
 
 
@@ -21,13 +21,20 @@
 class FakeSpace(object):
     w_ValueError = None
     w_TypeError = None
+    w_None = None
+
+    w_bool = "bool"
+    w_int = "int"
+    w_float = "float"
+    w_list = "list"
+    w_long = "long"
 
     def __init__(self):
         """NOT_RPYTHON"""
         self.fromcache = InternalSpaceCache(self).getorbuild
 
     def issequence_w(self, w_obj):
-        return True
+        return w_obj.seq
 
     @specialize.argtype(1)
     def wrap(self, obj):
@@ -39,6 +46,13 @@
             return IntObject(obj)
         raise Exception
 
+    def newlist(self, items):
+        return ListObject(items)
+
+    def listview(self, obj):
+        assert isinstance(obj, ListObject)
+        return obj.items
+
     def float(self, w_obj):
         assert isinstance(w_obj, FloatObject)
         return w_obj
@@ -46,23 +60,243 @@
     def float_w(self, w_obj):
         return w_obj.floatval
 
+    def is_w(self, w_obj, w_what):
+        return w_obj is w_what
+
+    def type(self, w_obj):
+        return w_obj.tp
+
+    def gettypefor(self, w_obj):
+        return None
+
+    def call_function(self, tp, w_dtype):
+        return w_dtype
+
+    @specialize.arg(1)
+    def interp_w(self, tp, what):
+        return what
 
 class FloatObject(W_Root):
+    seq = False
+    tp = FakeSpace.w_float
     def __init__(self, floatval):
         self.floatval = floatval
 
 class BoolObject(W_Root):
+    seq = False
+    tp = FakeSpace.w_bool
     def __init__(self, boolval):
         self.boolval = boolval
 
 class IntObject(W_Root):
+    seq = False
+    tp = FakeSpace.w_int
     def __init__(self, intval):
         self.intval = intval
 
+class ListObject(W_Root):
+    seq = True
+    tp = FakeSpace.w_list
+    def __init__(self, items):
+        self.items = items
+
 
 space = FakeSpace()
 
-def numpy_compile(bytecode, array_size):
+class InterpreterState(object):
+    def __init__(self, code):
+        self.code = code
+        self.variables = {}
+        self.results = []
+
+    def run(self, space):
+        self.space = space
+        for stmt in self.code.statements:
+            stmt.execute(self)
+
+class Node(object):
+    def __eq__(self, other):
+        return (self.__class__ == other.__class__ and
+                self.__dict__ == other.__dict__)
+
+    def __ne__(self, other):
+        return not self == other
+
+    def wrap(self, space):
+        raise NotImplementedError
+
+    def execute(self, interp):
+        raise NotImplementedError
+
+class Assignment(Node):
+    def __init__(self, id, expr):
+        self.id = id
+        self.expr = expr
+
+    def execute(self, interp):
+        interp.variables[self.id.name] = self.expr.execute(interp)
+
+    def __repr__(self):
+        return "%r = %r" % (self.id, self.expr)
+
+class Variable(Node):
+    def __init__(self, name):
+        self.name = name
+
+    def execute(self, interp):
+        return interp.variables[self.name]
+
+    def __repr__(self):
+        return 'v(%s)' % self.name
+
+class Operator(Node):
+    def __init__(self, lhs, name, rhs):
+        self.name = name
+        self.lhs = lhs
+        self.rhs = rhs
+
+    def execute(self, interp):
+        if self.name == '+':
+            w_lhs = self.lhs.execute(interp)
+            w_rhs = self.rhs.execute(interp)
+            return w_lhs.descr_add(interp.space, w_rhs)
+        elif self.name == '->':
+            w_lhs = self.lhs.execute(interp)
+            w_rhs = self.rhs.execute(interp)
+            if isinstance(w_rhs, Scalar):
+                index = int(space.float_w(w_rhs.value.wrap(interp.space)))
+                return w_lhs.get_concrete().eval(index)
+            else:
+                xxx
+
+    def __repr__(self):
+        return '(%r %s %r)' % (self.lhs, self.name, self.rhs)
+
+class FloatConstant(Node):
+    def __init__(self, v):
+        self.v = float(v)
+
+    def __repr__(self):
+        return "Const(%s)" % self.v
+
+    def wrap(self, space):
+        return space.wrap(self.v)
+
+    def execute(self, interp):
+        dtype = interp.space.fromcache(W_Float64Dtype)
+        return Scalar(dtype, dtype.box(self.v))
+
+class RangeConstant(Node):
+    def __init__(self, v):
+        self.v = int(v)
+
+    def __repr__(self):
+        return 'Range(%s)' % self.v
+
+class Code(Node):
+    def __init__(self, statements):
+        self.statements = statements
+
+    def __repr__(self):
+        return "\n".join([repr(i) for i in self.statements])
+
+class ArrayConstant(Node):
+    def __init__(self, items):
+        self.items = items
+
+    def wrap(self, space):
+        return space.newlist([item.wrap(space) for item in self.items])
+
+    def execute(self, interp):
+        w_list = self.wrap(interp.space)
+        return descr_new_array(interp.space, None, w_list)
+
+    def __repr__(self):
+        return "[" + ", ".join([repr(item) for item in self.items]) + "]"
+
+class Execute(Node):
+    def __init__(self, expr):
+        self.expr = expr
+
+    def __repr__(self):
+        return repr(self.expr)
+
+    def execute(self, interp):
+        interp.results.append(self.expr.execute(interp))
+
+class Parser(object):
+    def parse_identifier(self, id):
+        id = id.strip()
+        assert id.isalpha()
+        return Variable(id)
+
+    def parse_expression(self, expr):
+        tokens = [i for i in expr.split(" ") if i]
+        if len(tokens) == 1:
+            return self.parse_constant_or_identifier(tokens[0])
+        stack = []
+        tokens.reverse()
+        while tokens:
+            token = tokens.pop()
+            if token == ')':
+                xxx
+            elif self.is_identifier_or_const(token):
+                if stack:
+                    name = stack.pop()
+                    lhs = stack.pop()
+                    rhs = self.parse_constant_or_identifier(token)
+                    stack.append(Operator(lhs, name, rhs))
+                else:
+                    stack.append(self.parse_constant_or_identifier(token))
+            else:
+                stack.append(token)
+        assert len(stack) == 1
+        return stack[-1]
+
+    def parse_constant(self, v):
+        if v[0] == '[':
+            return ArrayConstant([self.parse_constant(elem)
+                                  for elem in v[1:-1].split(",")])
+        if v[0] == '|':
+            return RangeConstant(v[1:-1])
+        return FloatConstant(v)
+
+    def is_identifier_or_const(self, v):
+        c = v[0]
+        if ((c >= 'a' and c <= 'z') or (c >= 'A' and c <= 'Z') or
+            (c >= '0' and c <= '9') or c in '-.['):
+            if v == '-' or v == "->":
+                return False
+            return True
+        return False
+
+    def parse_constant_or_identifier(self, v):
+        c = v[0]
+        if (c >= 'a' and c <= 'z') or (c >= 'A' and c <= 'Z'):
+            return self.parse_identifier(v)
+        return self.parse_constant(v)
+        
+    def parse_statement(self, line):
+        if '=' in line:
+            lhs, rhs = line.split("=")
+            return Assignment(self.parse_identifier(lhs),
+                              self.parse_expression(rhs))
+        else:
+            return Execute(self.parse_expression(line))
+
+    def parse(self, code):
+        statements = []
+        for line in code.split("\n"):
+            line = line.strip(" ")
+            if line:
+                statements.append(self.parse_statement(line))
+        return Code(statements)
+
+def numpy_compile(code):
+    parser = Parser()
+    return InterpreterState(parser.parse(code))
+
+def xxx_numpy_compile(bytecode, array_size):
     stack = []
     i = 0
     dtype = space.fromcache(W_Float64Dtype)
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -14,6 +14,27 @@
 any_driver = jit.JitDriver(greens=['signature'], reds=['i', 'size', 'self', 'dtype'])
 slice_driver = jit.JitDriver(greens=['signature'], reds=['i', 'j', 'step', 'stop', 'source', 'dest'])
 
+def descr_new_array(space, w_subtype, w_size_or_iterable, w_dtype=None):
+    l = space.listview(w_size_or_iterable)
+    if space.is_w(w_dtype, space.w_None):
+        w_dtype = None
+        for w_item in l:
+            w_dtype = interp_ufuncs.find_dtype_for_scalar(space, w_item, w_dtype)
+            if w_dtype is space.fromcache(interp_dtype.W_Float64Dtype):
+                break
+        if w_dtype is None:
+            w_dtype = space.w_None
+
+    dtype = space.interp_w(interp_dtype.W_Dtype,
+        space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype)
+    )
+    arr = SingleDimArray(len(l), dtype=dtype)
+    i = 0
+    for w_elem in l:
+        dtype.setitem_w(space, arr.storage, i, w_elem)
+        i += 1
+    return arr
+
 class BaseArray(Wrappable):
     _attrs_ = ["invalidates", "signature"]
 
@@ -32,27 +53,6 @@
     def add_invalidates(self, other):
         self.invalidates.append(other)
 
-    def descr__new__(space, w_subtype, w_size_or_iterable, w_dtype=None):
-        l = space.listview(w_size_or_iterable)
-        if space.is_w(w_dtype, space.w_None):
-            w_dtype = None
-            for w_item in l:
-                w_dtype = interp_ufuncs.find_dtype_for_scalar(space, w_item, w_dtype)
-                if w_dtype is space.fromcache(interp_dtype.W_Float64Dtype):
-                    break
-            if w_dtype is None:
-                w_dtype = space.w_None
-
-        dtype = space.interp_w(interp_dtype.W_Dtype,
-            space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype)
-        )
-        arr = SingleDimArray(len(l), dtype=dtype)
-        i = 0
-        for w_elem in l:
-            dtype.setitem_w(space, arr.storage, i, w_elem)
-            i += 1
-        return arr
-
     def _unaryop_impl(ufunc_name):
         def impl(self, space):
             return getattr(interp_ufuncs.get(space), ufunc_name).call(space, [self])
@@ -571,7 +571,7 @@
 
 BaseArray.typedef = TypeDef(
     'numarray',
-    __new__ = interp2app(BaseArray.descr__new__.im_func),
+    __new__ = interp2app(descr_new_array),
 
 
     __len__ = interp2app(BaseArray.descr_len),
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -20,6 +20,13 @@
         cls.uint64_dtype = cls.space.fromcache(W_UInt64Dtype)
         cls.int32_dtype = cls.space.fromcache(W_Int32Dtype)
 
+    def run(self, code):
+        space = FakeSpace()
+        interp = numpy_compile(code)
+        def f():
+            interp.run(space)
+        self.meta_interpxxx
+
     def test_add(self):
         def f(i):
             ar = SingleDimArray(i, dtype=self.float64_dtype)

From noreply at buildbot.pypy.org  Thu Oct 27 22:31:32 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Oct 2011 22:31:32 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-minilang: a bit of rpythonization
Message-ID: <20111027203132.0972E820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: numpy-minilang
Changeset: r48540:263d430ab0d3
Date: 2011-10-27 22:30 +0200
http://bitbucket.org/pypy/pypy/changeset/263d430ab0d3/

Log:	a bit of rpythonization

diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -32,6 +32,7 @@
     def __init__(self):
         """NOT_RPYTHON"""
         self.fromcache = InternalSpaceCache(self).getorbuild
+        self.w_float64dtype = W_Float64Dtype(self)
 
     def issequence_w(self, w_obj):
         return w_obj.seq
@@ -58,8 +59,20 @@
         return w_obj
 
     def float_w(self, w_obj):
+        assert isinstance(w_obj, FloatObject)        
         return w_obj.floatval
 
+    def int_w(self, w_obj):
+        assert isinstance(w_obj, IntObject)
+        return w_obj.intval
+
+    def int(self, w_obj):
+        return w_obj
+
+    def is_true(self, w_obj):
+        assert isinstance(w_obj, BoolObject)
+        return w_obj.boolval
+
     def is_w(self, w_obj, w_what):
         return w_obj is w_what
 
@@ -156,18 +169,19 @@
         self.rhs = rhs
 
     def execute(self, interp):
+        w_lhs = self.lhs.execute(interp)
+        w_rhs = self.rhs.execute(interp)
+        assert isinstance(w_lhs, BaseArray)
         if self.name == '+':
-            w_lhs = self.lhs.execute(interp)
-            w_rhs = self.rhs.execute(interp)
             return w_lhs.descr_add(interp.space, w_rhs)
         elif self.name == '->':
-            w_lhs = self.lhs.execute(interp)
-            w_rhs = self.rhs.execute(interp)
             if isinstance(w_rhs, Scalar):
                 index = int(space.float_w(w_rhs.value.wrap(interp.space)))
                 return w_lhs.get_concrete().eval(index)
             else:
-                xxx
+                raise NotImplementedError
+        else:
+            raise NotImplementedError
 
     def __repr__(self):
         return '(%r %s %r)' % (self.lhs, self.name, self.rhs)
@@ -183,7 +197,7 @@
         return space.wrap(self.v)
 
     def execute(self, interp):
-        dtype = interp.space.fromcache(W_Float64Dtype)
+        dtype = space.w_float64dtype
         return Scalar(dtype, dtype.box(self.v))
 
 class RangeConstant(Node):
@@ -226,8 +240,8 @@
 
 class Parser(object):
     def parse_identifier(self, id):
-        id = id.strip()
-        assert id.isalpha()
+        id = id.strip(" ")
+        #assert id.isalpha()
         return Variable(id)
 
     def parse_expression(self, expr):
@@ -239,17 +253,17 @@
         while tokens:
             token = tokens.pop()
             if token == ')':
-                xxx
+                raise NotImplementedError
             elif self.is_identifier_or_const(token):
                 if stack:
-                    name = stack.pop()
+                    name = stack.pop().name
                     lhs = stack.pop()
                     rhs = self.parse_constant_or_identifier(token)
                     stack.append(Operator(lhs, name, rhs))
                 else:
                     stack.append(self.parse_constant_or_identifier(token))
             else:
-                stack.append(token)
+                stack.append(Variable(token))
         assert len(stack) == 1
         return stack[-1]
 
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -1,7 +1,7 @@
 from pypy.jit.metainterp.test.support import LLJitMixin
 from pypy.module.micronumpy import interp_ufuncs, signature
-from pypy.module.micronumpy.compile import (numpy_compile, FakeSpace,
-    FloatObject, IntObject)
+from pypy.module.micronumpy.compile import (InterpreterState, FakeSpace,
+    FloatObject, IntObject, Parser)
 from pypy.module.micronumpy.interp_dtype import W_Int32Dtype, W_Float64Dtype, W_Int64Dtype, W_UInt64Dtype
 from pypy.module.micronumpy.interp_numarray import (BaseArray, SingleDimArray,
     SingleDimSlice, scalar_w)
@@ -21,23 +21,31 @@
         cls.int32_dtype = cls.space.fromcache(W_Int32Dtype)
 
     def run(self, code):
+        # trick annotator
+        c = """
+        a = 3
+        """
+        
         space = FakeSpace()
-        interp = numpy_compile(code)
-        def f():
+        parser = Parser()
+        codes = [parser.parse(code), parser.parse(c)]
+        
+        def f(i):
+            interp = InterpreterState(codes[i])
             interp.run(space)
-        self.meta_interpxxx
+            return interp.results[0]
+        return self.meta_interp(f, [0], listops=True, backendopt=True)
 
     def test_add(self):
-        def f(i):
-            ar = SingleDimArray(i, dtype=self.float64_dtype)
-            v = interp_ufuncs.get(self.space).add.call(self.space, [ar, ar])
-            return v.get_concrete().eval(3).val
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
+        result = self.run("""
+        a = |30|
+        b = a + a
+        b -> 3
+        """)
         self.check_loops({'getarrayitem_raw': 2, 'float_add': 1,
                           'setarrayitem_raw': 1, 'int_add': 1,
                           'int_lt': 1, 'guard_true': 1, 'jump': 1})
-        assert result == f(5)
+        assert result == 3 + 3
 
     def test_floatadd(self):
         def f(i):

From noreply at buildbot.pypy.org  Thu Oct 27 22:38:36 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Oct 2011 22:38:36 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-minilang: oops, forgot to add a test
Message-ID: <20111027203836.50BFD820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: numpy-minilang
Changeset: r48541:349ca005d2ef
Date: 2011-10-27 22:37 +0200
http://bitbucket.org/pypy/pypy/changeset/349ca005d2ef/

Log:	oops, forgot to add a test

diff --git a/pypy/module/micronumpy/test/test_compile.py b/pypy/module/micronumpy/test/test_compile.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/test/test_compile.py
@@ -0,0 +1,116 @@
+
+from pypy.module.micronumpy.compile import *
+
+class TestCompiler(object):
+    def compile(self, code):
+        return numpy_compile(code)
+    
+    def test_vars(self):
+        code = """
+        a = 2
+        b = 3
+        """
+        interp = self.compile(code)
+        assert isinstance(interp.code.statements[0], Assignment)
+        assert interp.code.statements[0].id.name == 'a'
+        assert interp.code.statements[0].expr.v == 2
+        assert interp.code.statements[1].id.name == 'b'
+        assert interp.code.statements[1].expr.v == 3
+
+    def test_array_literal(self):
+        code = """
+        a = [1,2,3]
+        """
+        interp = self.compile(code)
+        assert isinstance(interp.code.statements[0].expr, ArrayConstant)
+        st = interp.code.statements[0]
+        assert st.expr.items == [FloatConstant(1), FloatConstant(2),
+                                 FloatConstant(3)]
+    
+    def test_array_literal2(self):
+        code = """
+        a = [[1],[2],[3]]
+        """
+        interp = self.compile(code)
+        assert isinstance(interp.code.statements[0].expr, ArrayConstant)
+        st = interp.code.statements[0]
+        assert st.expr.items == [ArrayConstant([FloatConstant(1)]),
+                                 ArrayConstant([FloatConstant(2)]),
+                                 ArrayConstant([FloatConstant(3)])]
+
+    def test_expr_1(self):
+        code = """
+        b = a + 1
+        """
+        interp = self.compile(code)
+        assert (interp.code.statements[0].expr ==
+                Operator(Variable("a"), "+", FloatConstant(1)))
+
+    def test_expr_2(self):
+        code = """
+        b = a + b - 3
+        """
+        interp = self.compile(code)
+        assert (interp.code.statements[0].expr ==
+                Operator(Operator(Variable("a"), "+", Variable("b")), "-",
+                         FloatConstant(3)))
+
+    def test_expr_3(self):
+        # an equivalent of range
+        code = """
+        a = |20|
+        """
+        interp = self.compile(code)
+        assert interp.code.statements[0].expr == RangeConstant(20)
+
+    def test_expr_only(self):
+        code = """
+        3 + a
+        """
+        interp = self.compile(code)
+        assert interp.code.statements[0] == Execute(
+            Operator(FloatConstant(3), "+", Variable("a")))
+
+    def test_array_access(self):
+        code = """
+        a -> 3
+        """
+        interp = self.compile(code)
+        assert interp.code.statements[0] == Execute(
+            Operator(Variable("a"), "->", FloatConstant(3)))
+
+class TestRunner(object):
+    def run(self, code):
+        interp = numpy_compile(code)
+        space = FakeSpace()
+        interp.run(space)
+        return interp
+
+    def test_one(self):
+        code = """
+        a = 3
+        b = 4
+        a + b
+        """
+        interp = self.run(code)
+        assert sorted(interp.variables.keys()) == ['a', 'b']
+        assert interp.results[0]
+
+    def test_array_add(self):
+        code = """
+        a = [1,2,3,4]
+        b = [4,5,6,5]
+        a + b
+        """
+        interp = self.run(code)
+        assert interp.results[0]._getnums(False) == ["5.0", "7.0", "9.0", "9.0"]
+
+    def test_array_getitem(self):
+        code = """
+        a = [1,2,3,4]
+        b = [4,5,6,5]
+        a + b -> 3
+        """
+        interp = self.run(code)
+        assert interp.results[0].val == 3 + 6
+        

From noreply at buildbot.pypy.org  Thu Oct 27 22:52:46 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Oct 2011 22:52:46 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-minilang: ok, not mix two spaces,
	leads to confusing errors
Message-ID: <20111027205246.4D44A820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: numpy-minilang
Changeset: r48542:c729d657e525
Date: 2011-10-27 22:52 +0200
http://bitbucket.org/pypy/pypy/changeset/c729d657e525/

Log:	ok, not mix two spaces, leads to confusing errors

diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -114,8 +114,6 @@
         self.items = items
 
 
-space = FakeSpace()
-
 class InterpreterState(object):
     def __init__(self, code):
         self.code = code
@@ -176,7 +174,8 @@
             return w_lhs.descr_add(interp.space, w_rhs)
         elif self.name == '->':
             if isinstance(w_rhs, Scalar):
-                index = int(space.float_w(w_rhs.value.wrap(interp.space)))
+                index = int(interp.space.float_w(
+                    w_rhs.value.wrap(interp.space)))
                 return w_lhs.get_concrete().eval(index)
             else:
                 raise NotImplementedError
@@ -197,7 +196,8 @@
         return space.wrap(self.v)
 
     def execute(self, interp):
-        dtype = space.w_float64dtype
+        dtype = interp.space.fromcache(W_Float64Dtype)
+        assert isinstance(dtype, W_Float64Dtype)
         return Scalar(dtype, dtype.box(self.v))
 
 class RangeConstant(Node):
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -13,13 +13,6 @@
 
 
 class TestNumpyJIt(LLJitMixin):
-    def setup_class(cls):
-        cls.space = FakeSpace()
-        cls.float64_dtype = cls.space.fromcache(W_Float64Dtype)
-        cls.int64_dtype = cls.space.fromcache(W_Int64Dtype)
-        cls.uint64_dtype = cls.space.fromcache(W_UInt64Dtype)
-        cls.int32_dtype = cls.space.fromcache(W_Int32Dtype)
-
     def run(self, code):
         # trick annotator
         c = """

From noreply at buildbot.pypy.org  Thu Oct 27 23:32:12 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Oct 2011 23:32:12 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-minilang: pass the test_zjit finally
Message-ID: <20111027213212.7AC5B820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: numpy-minilang
Changeset: r48543:f1d8e024a3ed
Date: 2011-10-27 23:31 +0200
http://bitbucket.org/pypy/pypy/changeset/f1d8e024a3ed/

Log:	pass the test_zjit finally

diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -37,6 +37,9 @@
     def issequence_w(self, w_obj):
         return w_obj.seq
 
+    def isinstance_w(self, w_obj, w_tp):
+        return False
+
     @specialize.argtype(1)
     def wrap(self, obj):
         if isinstance(obj, float):
@@ -87,6 +90,7 @@
 
     @specialize.arg(1)
     def interp_w(self, tp, what):
+        assert isinstance(what, tp)
         return what
 
 class FloatObject(W_Root):
@@ -176,7 +180,8 @@
             if isinstance(w_rhs, Scalar):
                 index = int(interp.space.float_w(
                     w_rhs.value.wrap(interp.space)))
-                return w_lhs.get_concrete().eval(index)
+                dtype = interp.space.fromcache(W_Float64Dtype)
+                return Scalar(dtype, w_lhs.get_concrete().eval(index))
             else:
                 raise NotImplementedError
         else:
@@ -204,6 +209,12 @@
     def __init__(self, v):
         self.v = int(v)
 
+    def execute(self, interp):
+        w_list = interp.space.newlist(
+            [interp.space.wrap(float(i)) for i in range(self.v)])
+        dtype = interp.space.fromcache(W_Float64Dtype)
+        return descr_new_array(interp.space, None, w_list, w_dtype=dtype)
+
     def __repr__(self):
         return 'Range(%s)' % self.v
 
@@ -223,7 +234,8 @@
 
     def execute(self, interp):
         w_list = self.wrap(interp.space)
-        return descr_new_array(interp.space, None, w_list)
+        dtype = interp.space.fromcache(W_Float64Dtype)
+        return descr_new_array(interp.space, None, w_list, w_dtype=dtype)
 
     def __repr__(self):
         return "[" + ", ".join([repr(item) for item in self.items]) + "]"
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -236,22 +236,20 @@
     return dt
 
 def find_dtype_for_scalar(space, w_obj, current_guess=None):
-    w_type = space.type(w_obj)
-
     bool_dtype = space.fromcache(interp_dtype.W_BoolDtype)
     long_dtype = space.fromcache(interp_dtype.W_LongDtype)
     int64_dtype = space.fromcache(interp_dtype.W_Int64Dtype)
 
-    if space.is_w(w_type, space.w_bool):
+    if space.isinstance_w(w_obj, space.w_bool):
         if current_guess is None or current_guess is bool_dtype:
             return bool_dtype
         return current_guess
-    elif space.is_w(w_type, space.w_int):
+    elif space.isinstance_w(w_obj, space.w_int):
         if (current_guess is None or current_guess is bool_dtype or
             current_guess is long_dtype):
             return long_dtype
         return current_guess
-    elif space.is_w(w_type, space.w_long):
+    elif space.isinstance_w(w_obj, space.w_long):
         if (current_guess is None or current_guess is bool_dtype or
             current_guess is long_dtype or current_guess is int64_dtype):
             return int64_dtype
diff --git a/pypy/module/micronumpy/test/test_compile.py b/pypy/module/micronumpy/test/test_compile.py
--- a/pypy/module/micronumpy/test/test_compile.py
+++ b/pypy/module/micronumpy/test/test_compile.py
@@ -112,5 +112,12 @@
         a + b -> 3
         """
         interp = self.run(code)
-        assert interp.results[0].val == 3 + 6
+        assert interp.results[0].value.val == 3 + 6
         
+    def test_range_getitem(self):
+        code = """
+        r = |20|
+        r -> 3
+        """
+        interp = self.run(code)
+        assert interp.results[0].value.val == 3
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -4,7 +4,7 @@
     FloatObject, IntObject, Parser)
 from pypy.module.micronumpy.interp_dtype import W_Int32Dtype, W_Float64Dtype, W_Int64Dtype, W_UInt64Dtype
 from pypy.module.micronumpy.interp_numarray import (BaseArray, SingleDimArray,
-    SingleDimSlice, scalar_w)
+    SingleDimSlice, scalar_w, Scalar)
 from pypy.rlib.nonconst import NonConstant
 from pypy.rpython.annlowlevel import llstr
 from pypy.rpython.test.test_llinterp import interpret
@@ -17,6 +17,8 @@
         # trick annotator
         c = """
         a = 3
+        b = [1,2] + [3,4]
+        c = a
         """
         
         space = FakeSpace()
@@ -26,7 +28,9 @@
         def f(i):
             interp = InterpreterState(codes[i])
             interp.run(space)
-            return interp.results[0]
+            res = interp.results[0]
+            assert isinstance(res, BaseArray)
+            return interp.space.float_w(res.eval(0).wrap(interp.space))
         return self.meta_interp(f, [0], listops=True, backendopt=True)
 
     def test_add(self):

From noreply at buildbot.pypy.org  Fri Oct 28 00:01:10 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 28 Oct 2011 00:01:10 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-minilang: a test and a fix. took a while :/
Message-ID: <20111027220110.2C929820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: numpy-minilang
Changeset: r48544:41172dd9df6f
Date: 2011-10-27 23:57 +0200
http://bitbucket.org/pypy/pypy/changeset/41172dd9df6f/

Log:	a test and a fix. took a while :/

diff --git a/pypy/rpython/lltypesystem/rstr.py b/pypy/rpython/lltypesystem/rstr.py
--- a/pypy/rpython/lltypesystem/rstr.py
+++ b/pypy/rpython/lltypesystem/rstr.py
@@ -20,6 +20,7 @@
 from pypy.rpython.rmodel import Repr
 from pypy.rpython.lltypesystem import llmemory
 from pypy.tool.sourcetools import func_with_new_name
+from pypy.rpython.lltypesystem.lloperation import llop
 
 # ____________________________________________________________
 #
@@ -366,6 +367,8 @@
         if right:
             while lpos < rpos and s.chars[rpos] == ch:
                 rpos -= 1
+        if rpos <= lpos:
+            return s.empty()
         r_len = rpos - lpos + 1
         result = s.malloc(r_len)
         s.copy_contents(s, result, lpos, 0, r_len)
diff --git a/pypy/rpython/test/test_rstr.py b/pypy/rpython/test/test_rstr.py
--- a/pypy/rpython/test/test_rstr.py
+++ b/pypy/rpython/test/test_rstr.py
@@ -372,12 +372,16 @@
             return const('!ab!').lstrip(const('!'))
         def right():
             return const('!ab!').rstrip(const('!'))
+        def empty():
+            return const('    ').strip(' ')
         res = self.interpret(both, [])
         assert self.ll_to_string(res) == const('ab')
         res = self.interpret(left, [])
         assert self.ll_to_string(res) == const('ab!')
         res = self.interpret(right, [])
         assert self.ll_to_string(res) == const('!ab')
+        res = self.interpret(empty, [])
+        assert self.ll_to_string(res) == const('')
 
     def test_upper(self):
         const = self.const

From noreply at buildbot.pypy.org  Fri Oct 28 00:01:11 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 28 Oct 2011 00:01:11 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-minilang: convert tests and some rpython
	fixes
Message-ID: <20111027220111.59E02820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: numpy-minilang
Changeset: r48545:c75ffa281f05
Date: 2011-10-28 00:00 +0200
http://bitbucket.org/pypy/pypy/changeset/c75ffa281f05/

Log:	convert tests and some rpython fixes

diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -280,17 +280,19 @@
         return stack[-1]
 
     def parse_constant(self, v):
+        lgt = len(v)-1
+        assert lgt >= 0
         if v[0] == '[':
             return ArrayConstant([self.parse_constant(elem)
-                                  for elem in v[1:-1].split(",")])
+                                  for elem in v[1:lgt].split(",")])
         if v[0] == '|':
-            return RangeConstant(v[1:-1])
+            return RangeConstant(v[1:lgt])
         return FloatConstant(v)
 
     def is_identifier_or_const(self, v):
         c = v[0]
         if ((c >= 'a' and c <= 'z') or (c >= 'A' and c <= 'Z') or
-            (c >= '0' and c <= '9') or c in '-.['):
+            (c >= '0' and c <= '9') or c in '-.[|'):
             if v == '-' or v == "->":
                 return False
             return True
diff --git a/pypy/module/micronumpy/test/test_compile.py b/pypy/module/micronumpy/test/test_compile.py
--- a/pypy/module/micronumpy/test/test_compile.py
+++ b/pypy/module/micronumpy/test/test_compile.py
@@ -116,8 +116,8 @@
         
     def test_range_getitem(self):
         code = """
-        r = |20|
+        r = |20| + 3
         r -> 3
         """
         interp = self.run(code)
-        assert interp.results[0].value.val == 3
+        assert interp.results[0].value.val == 6
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -1,12 +1,11 @@
 from pypy.jit.metainterp.test.support import LLJitMixin
 from pypy.module.micronumpy import interp_ufuncs, signature
-from pypy.module.micronumpy.compile import (InterpreterState, FakeSpace,
-    FloatObject, IntObject, Parser)
-from pypy.module.micronumpy.interp_dtype import W_Int32Dtype, W_Float64Dtype, W_Int64Dtype, W_UInt64Dtype
+from pypy.module.micronumpy.compile import (FakeSpace,
+    FloatObject, IntObject, numpy_compile)
 from pypy.module.micronumpy.interp_numarray import (BaseArray, SingleDimArray,
-    SingleDimSlice, scalar_w, Scalar)
+    SingleDimSlice, scalar_w)
 from pypy.rlib.nonconst import NonConstant
-from pypy.rpython.annlowlevel import llstr
+from pypy.rpython.annlowlevel import llstr, hlstr
 from pypy.rpython.test.test_llinterp import interpret
 
 import py
@@ -14,24 +13,16 @@
 
 class TestNumpyJIt(LLJitMixin):
     def run(self, code):
-        # trick annotator
-        c = """
-        a = 3
-        b = [1,2] + [3,4]
-        c = a
-        """
         
         space = FakeSpace()
-        parser = Parser()
-        codes = [parser.parse(code), parser.parse(c)]
         
-        def f(i):
-            interp = InterpreterState(codes[i])
+        def f(code):
+            interp = numpy_compile(hlstr(code))
             interp.run(space)
             res = interp.results[0]
             assert isinstance(res, BaseArray)
             return interp.space.float_w(res.eval(0).wrap(interp.space))
-        return self.meta_interp(f, [0], listops=True, backendopt=True)
+        return self.meta_interp(f, [llstr(code)], listops=True, backendopt=True)
 
     def test_add(self):
         result = self.run("""
@@ -45,21 +36,14 @@
         assert result == 3 + 3
 
     def test_floatadd(self):
-        def f(i):
-            ar = SingleDimArray(i, dtype=self.float64_dtype)
-            v = interp_ufuncs.get(self.space).add.call(self.space, [
-                    ar,
-                    scalar_w(self.space, self.float64_dtype, self.space.wrap(4.5))
-                ],
-            )
-            assert isinstance(v, BaseArray)
-            return v.get_concrete().eval(3).val
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
+        result = self.run("""
+        a = |30| + 3
+        a -> 3
+        """)
+        assert result == 3 + 3
         self.check_loops({"getarrayitem_raw": 1, "float_add": 1,
                           "setarrayitem_raw": 1, "int_add": 1,
                           "int_lt": 1, "guard_true": 1, "jump": 1})
-        assert result == f(5)
 
     def test_sum(self):
         space = self.space

From noreply at buildbot.pypy.org  Fri Oct 28 00:21:02 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 28 Oct 2011 00:21:02 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-minilang: reuse the meta interped graph
Message-ID: <20111027222102.CACAA820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: numpy-minilang
Changeset: r48546:a9aa7bc83108
Date: 2011-10-28 00:20 +0200
http://bitbucket.org/pypy/pypy/changeset/a9aa7bc83108/

Log:	reuse the meta interped graph

diff --git a/pypy/jit/metainterp/history.py b/pypy/jit/metainterp/history.py
--- a/pypy/jit/metainterp/history.py
+++ b/pypy/jit/metainterp/history.py
@@ -929,6 +929,9 @@
     def view(self, **kwds):
         pass
 
+    def clear(self):
+        pass
+
 class Stats(object):
     """For tests."""
 
@@ -943,6 +946,12 @@
         self.aborted_keys = []
         self.invalidated_token_numbers = set()
 
+    def clear(self):
+        del self.loops[:]
+        del self.locations[:]
+        del self.aborted_keys[:]
+        self.invalidated_token_numbers.clear()
+
     def set_history(self, history):
         self.operations = history.operations
 
diff --git a/pypy/jit/metainterp/warmspot.py b/pypy/jit/metainterp/warmspot.py
--- a/pypy/jit/metainterp/warmspot.py
+++ b/pypy/jit/metainterp/warmspot.py
@@ -62,7 +62,7 @@
     clear_tcache()
     return jittify_and_run(interp, graph, args, backendopt=backendopt, **kwds)
 
-def jittify_and_run(interp, graph, args, repeat=1,
+def jittify_and_run(interp, graph, args, repeat=1, graph_and_interp_only=False,
                     backendopt=False, trace_limit=sys.maxint,
                     inline=False, loop_longevity=0, retrace_limit=5,
                     function_threshold=4,
@@ -93,6 +93,8 @@
         jd.warmstate.set_param_max_retrace_guards(max_retrace_guards)
         jd.warmstate.set_param_enable_opts(enable_opts)
     warmrunnerdesc.finish()
+    if graph_and_interp_only:
+        return interp, graph
     res = interp.eval_graph(graph, args)
     if not kwds.get('translate_support_code', False):
         warmrunnerdesc.metainterp_sd.profiler.finish()
@@ -157,6 +159,9 @@
 def get_stats():
     return pyjitpl._warmrunnerdesc.stats
 
+def reset_stats():
+    pyjitpl._warmrunnerdesc.stats.clear()
+
 def get_translator():
     return pyjitpl._warmrunnerdesc.translator
 
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -7,13 +7,16 @@
 from pypy.rlib.nonconst import NonConstant
 from pypy.rpython.annlowlevel import llstr, hlstr
 from pypy.rpython.test.test_llinterp import interpret
+from pypy.jit.metainterp.warmspot import reset_stats
 
 import py
 
 
 class TestNumpyJIt(LLJitMixin):
+    graph = None
+    interp = None
+        
     def run(self, code):
-        
         space = FakeSpace()
         
         def f(code):
@@ -22,7 +25,17 @@
             res = interp.results[0]
             assert isinstance(res, BaseArray)
             return interp.space.float_w(res.eval(0).wrap(interp.space))
-        return self.meta_interp(f, [llstr(code)], listops=True, backendopt=True)
+
+        if self.graph is None:
+            interp, graph = self.meta_interp(f, [llstr(code)],
+                                             listops=True,
+                                             backendopt=True,
+                                             graph_and_interp_only=True)
+            self.__class__.interp = interp
+            self.__class__.graph = graph
+
+        reset_stats()
+        return self.interp.eval_graph(self.graph, [llstr(code)])
 
     def test_add(self):
         result = self.run("""
@@ -45,6 +58,7 @@
                           "setarrayitem_raw": 1, "int_add": 1,
                           "int_lt": 1, "guard_true": 1, "jump": 1})
 
+class TstXyz(object):
     def test_sum(self):
         space = self.space
         float64_dtype = self.float64_dtype

From noreply at buildbot.pypy.org  Fri Oct 28 00:22:04 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 28 Oct 2011 00:22:04 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-minilang: cleanup
Message-ID: <20111027222204.1C414820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: numpy-minilang
Changeset: r48547:7861ff043f50
Date: 2011-10-28 00:21 +0200
http://bitbucket.org/pypy/pypy/changeset/7861ff043f50/

Log:	cleanup

diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -4,7 +4,7 @@
 """
 
 from pypy.interpreter.baseobjspace import InternalSpaceCache, W_Root
-from pypy.module.micronumpy.interp_dtype import W_Float64Dtype, W_Int32Dtype
+from pypy.module.micronumpy.interp_dtype import W_Float64Dtype
 from pypy.module.micronumpy.interp_numarray import Scalar, BaseArray, descr_new_array
 from pypy.rlib.objectmodel import specialize
 
@@ -12,12 +12,6 @@
 class BogusBytecode(Exception):
     pass
 
-def create_array(dtype, size):
-    a = SingleDimArray(size, dtype=dtype)
-    for i in range(size):
-        dtype.setitem(a.storage, i, dtype.box(float(i % 10)))
-    return a
-
 class FakeSpace(object):
     w_ValueError = None
     w_TypeError = None
@@ -323,50 +317,3 @@
 def numpy_compile(code):
     parser = Parser()
     return InterpreterState(parser.parse(code))
-
-def xxx_numpy_compile(bytecode, array_size):
-    stack = []
-    i = 0
-    dtype = space.fromcache(W_Float64Dtype)
-    for b in bytecode:
-        if b == 'a':
-            stack.append(create_array(dtype, array_size))
-            i += 1
-        elif b == 'f':
-            stack.append(Scalar(dtype, dtype.box(1.2)))
-        elif b == '+':
-            right = stack.pop()
-            res = stack.pop().descr_add(space, right)
-            assert isinstance(res, BaseArray)
-            stack.append(res)
-        elif b == '-':
-            right = stack.pop()
-            res = stack.pop().descr_sub(space, right)
-            assert isinstance(res, BaseArray)
-            stack.append(res)
-        elif b == '*':
-            right = stack.pop()
-            res = stack.pop().descr_mul(space, right)
-            assert isinstance(res, BaseArray)
-            stack.append(res)
-        elif b == '/':
-            right = stack.pop()
-            res = stack.pop().descr_div(space, right)
-            assert isinstance(res, BaseArray)
-            stack.append(res)
-        elif b == '%':
-            right = stack.pop()
-            res = stack.pop().descr_mod(space, right)
-            assert isinstance(res, BaseArray)
-            stack.append(res)
-        elif b == '|':
-            res = stack.pop().descr_abs(space)
-            assert isinstance(res, BaseArray)
-            stack.append(res)
-        else:
-            print "Unknown opcode: %s" % b
-            raise BogusBytecode()
-    if len(stack) != 1:
-        print "Bogus bytecode, uneven stack length"
-        raise BogusBytecode()
-    return stack[0]
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -58,7 +58,7 @@
                           "setarrayitem_raw": 1, "int_add": 1,
                           "int_lt": 1, "guard_true": 1, "jump": 1})
 
-class TstXyz(object):
+class DisabledTestNumpy(object):
     def test_sum(self):
         space = self.space
         float64_dtype = self.float64_dtype
@@ -342,17 +342,3 @@
         result = self.meta_interp(f, [5], listops=True, backendopt=True)
         assert result == f(5)
 
-class TestTranslation(object):
-    def test_compile(self):
-        x = numpy_compile('aa+f*f/a-', 10)
-        x = x.compute()
-        assert isinstance(x, SingleDimArray)
-        assert x.size == 10
-        assert x.eval(0).val == 0
-        assert x.eval(1).val == ((1 + 1) * 1.2) / 1.2 - 1
-
-    def test_translation(self):
-        # we import main to check if the target compiles
-        from pypy.translator.goal.targetnumpystandalone import main
-
-        interpret(main, [llstr('af+'), 100])

From pullrequests-noreply at bitbucket.org  Fri Oct 28 01:46:19 2011
From: pullrequests-noreply at bitbucket.org (Bitbucket)
Date: Thu, 27 Oct 2011 23:46:19 -0000
Subject: [pypy-commit] [OPEN] Pull request #13 for pypy/pypy: fixes issue 923
Message-ID: <pullreq-caf9fa132b86@bitbucket.org>

A new pull request has been opened by Bruno Gola.

brunogola/pypy has changes to be pulled into pypy/pypy.

https://bitbucket.org/pypy/pypy/pull-request/13/fixes-issue-923

Title: fixes issue 923

more info: https://bugs.pypy.org/issue923

Changes to be pulled:

caf9fa132b86 by Bruno Gola: "merged with default branch"
d4c7fe2ac048 by Bruno Gola: "test_search.py and test_zinternal.py passing"
4c56b2a60b5a by Bruno Gola: "[fixes issue 923] matching RegExp with optional zero-width assertion groups"


--
This is an issue notification from bitbucket.org.
You are receiving this either because you are the participating
in a pull request, or you are following it.


From notifications-noreply at bitbucket.org  Fri Oct 28 04:09:48 2011
From: notifications-noreply at bitbucket.org (Bitbucket)
Date: Fri, 28 Oct 2011 02:09:48 -0000
Subject: [pypy-commit] [COMMENT] Pull request #13 for pypy/pypy: fixes issue
	923
Message-ID: <20111028020948.2570.91210@bitbucket05.managed.contegix.com>

New comment on pull request:

https://bitbucket.org/pypy/pypy/pull-request/13/fixes-issue-923#comment-673

Alex Gaynor (alex_gaynor) said:

This looks basically correct, thanks for tracking this down!  I'm not going to merge it though, because I'm not sufficiently confident in my knowledge of this code, hopefully armin or someone else can take a look and merge this.

--
This is a pull request comment notification from bitbucket.org.
You are receiving this either because you are participating
in a pull request, or you are following it.

From noreply at buildbot.pypy.org  Fri Oct 28 09:34:12 2011
From: noreply at buildbot.pypy.org (brunogola)
Date: Fri, 28 Oct 2011 09:34:12 +0200 (CEST)
Subject: [pypy-commit] pypy default: test_search.py and test_zinternal.py
	passing
Message-ID: <20111028073412.3F6BB82A87@wyvern.cs.uni-duesseldorf.de>

Author: Bruno Gola <brunogola at gmail.com>
Branch: 
Changeset: r48549:d4c7fe2ac048
Date: 2011-10-27 21:01 -0200
http://bitbucket.org/pypy/pypy/changeset/d4c7fe2ac048/

Log:	test_search.py and test_zinternal.py passing

diff --git a/pypy/rlib/rsre/rsre_core.py b/pypy/rlib/rsre/rsre_core.py
--- a/pypy/rlib/rsre/rsre_core.py
+++ b/pypy/rlib/rsre/rsre_core.py
@@ -389,11 +389,10 @@
             # zero-width match protection
             min = ctx.pat(ppos+1)
             if self.num_pending >= min:
-                if ptr == ctx.match_end and ctx.match_marks:
-                    # matched marks inside a zero-width assertion
-                    marks = ctx.match_marks
                 while enum is not None and ptr == ctx.match_end:
                     enum = enum.move_to_next_result(ctx)
+                    # matched marks for zero-width assertions
+                    marks = ctx.match_marks
             #
             if enum is not None:
                 # matched one more 'item'.  record it and continue.

From noreply at buildbot.pypy.org  Fri Oct 28 09:34:11 2011
From: noreply at buildbot.pypy.org (brunogola)
Date: Fri, 28 Oct 2011 09:34:11 +0200 (CEST)
Subject: [pypy-commit] pypy default: [fixes issue 923] matching RegExp with
 optional zero-width assertion groups
Message-ID: <20111028073411.0F608820C2@wyvern.cs.uni-duesseldorf.de>

Author: Bruno Gola <brunogola at gmail.com>
Branch: 
Changeset: r48548:4c56b2a60b5a
Date: 2011-10-27 19:52 -0200
http://bitbucket.org/pypy/pypy/changeset/4c56b2a60b5a/

Log:	[fixes issue 923] matching RegExp with optional zero-width assertion
	groups

diff --git a/pypy/rlib/rsre/rsre_core.py b/pypy/rlib/rsre/rsre_core.py
--- a/pypy/rlib/rsre/rsre_core.py
+++ b/pypy/rlib/rsre/rsre_core.py
@@ -389,6 +389,9 @@
             # zero-width match protection
             min = ctx.pat(ppos+1)
             if self.num_pending >= min:
+                if ptr == ctx.match_end and ctx.match_marks:
+                    # matched marks inside a zero-width assertion
+                    marks = ctx.match_marks
                 while enum is not None and ptr == ctx.match_end:
                     enum = enum.move_to_next_result(ctx)
             #
diff --git a/pypy/rlib/rsre/test/test_re.py b/pypy/rlib/rsre/test/test_re.py
--- a/pypy/rlib/rsre/test/test_re.py
+++ b/pypy/rlib/rsre/test/test_re.py
@@ -226,6 +226,13 @@
                          (None, 'b', None))
         assert pat.match('ac').group(1, 'b2', 3) == ('a', None, 'c')
 
+    def test_bug_923(self):
+        # Issue923: grouping inside optional lookahead problem
+        assert re.match(r'a(?=(b))?', "ab").groups() == ("b",)
+        assert re.match(r'(a(?=(b))?)', "ab").groups() == ('a', 'b')
+        assert re.match(r'(a)(?=(b))?', "ab").groups() == ('a', 'b')
+        assert re.match(r'(?P<g1>a)(?=(?P<g2>b))?', "ab").groupdict() == {'g1': 'a', 'g2': 'b'}
+
     def test_re_groupref_exists(self):
         assert re.match('^(\()?([^()]+)(?(1)\))$', '(a)').groups() == (
                          ('(', 'a'))

From noreply at buildbot.pypy.org  Fri Oct 28 09:34:17 2011
From: noreply at buildbot.pypy.org (brunogola)
Date: Fri, 28 Oct 2011 09:34:17 +0200 (CEST)
Subject: [pypy-commit] pypy default: merged with default branch
Message-ID: <20111028073417.CA800820C2@wyvern.cs.uni-duesseldorf.de>

Author: Bruno Gola <brunogola at gmail.com>
Branch: 
Changeset: r48550:caf9fa132b86
Date: 2011-10-27 21:12 -0200
http://bitbucket.org/pypy/pypy/changeset/caf9fa132b86/

Log:	merged with default branch

diff too long, truncating to 10000 out of 16297 lines

diff --git a/lib-python/modified-2.7/heapq.py b/lib-python/modified-2.7/heapq.py
new file mode 100644
--- /dev/null
+++ b/lib-python/modified-2.7/heapq.py
@@ -0,0 +1,442 @@
+# -*- coding: latin-1 -*-
+
+"""Heap queue algorithm (a.k.a. priority queue).
+
+Heaps are arrays for which a[k] <= a[2*k+1] and a[k] <= a[2*k+2] for
+all k, counting elements from 0.  For the sake of comparison,
+non-existing elements are considered to be infinite.  The interesting
+property of a heap is that a[0] is always its smallest element.
+
+Usage:
+
+heap = []            # creates an empty heap
+heappush(heap, item) # pushes a new item on the heap
+item = heappop(heap) # pops the smallest item from the heap
+item = heap[0]       # smallest item on the heap without popping it
+heapify(x)           # transforms list into a heap, in-place, in linear time
+item = heapreplace(heap, item) # pops and returns smallest item, and adds
+                               # new item; the heap size is unchanged
+
+Our API differs from textbook heap algorithms as follows:
+
+- We use 0-based indexing.  This makes the relationship between the
+  index for a node and the indexes for its children slightly less
+  obvious, but is more suitable since Python uses 0-based indexing.
+
+- Our heappop() method returns the smallest item, not the largest.
+
+These two make it possible to view the heap as a regular Python list
+without surprises: heap[0] is the smallest item, and heap.sort()
+maintains the heap invariant!
+"""
+
+# Original code by Kevin O'Connor, augmented by Tim Peters and Raymond Hettinger
+
+__about__ = """Heap queues
+
+[explanation by Fran&#65533;ois Pinard]
+
+Heaps are arrays for which a[k] <= a[2*k+1] and a[k] <= a[2*k+2] for
+all k, counting elements from 0.  For the sake of comparison,
+non-existing elements are considered to be infinite.  The interesting
+property of a heap is that a[0] is always its smallest element.
+
+The strange invariant above is meant to be an efficient memory
+representation for a tournament.  The numbers below are `k', not a[k]:
+
+                                   0
+
+                  1                                 2
+
+          3               4                5               6
+
+      7       8       9       10      11      12      13      14
+
+    15 16   17 18   19 20   21 22   23 24   25 26   27 28   29 30
+
+
+In the tree above, each cell `k' is topping `2*k+1' and `2*k+2'.  In
+an usual binary tournament we see in sports, each cell is the winner
+over the two cells it tops, and we can trace the winner down the tree
+to see all opponents s/he had.  However, in many computer applications
+of such tournaments, we do not need to trace the history of a winner.
+To be more memory efficient, when a winner is promoted, we try to
+replace it by something else at a lower level, and the rule becomes
+that a cell and the two cells it tops contain three different items,
+but the top cell "wins" over the two topped cells.
+
+If this heap invariant is protected at all time, index 0 is clearly
+the overall winner.  The simplest algorithmic way to remove it and
+find the "next" winner is to move some loser (let's say cell 30 in the
+diagram above) into the 0 position, and then percolate this new 0 down
+the tree, exchanging values, until the invariant is re-established.
+This is clearly logarithmic on the total number of items in the tree.
+By iterating over all items, you get an O(n ln n) sort.
+
+A nice feature of this sort is that you can efficiently insert new
+items while the sort is going on, provided that the inserted items are
+not "better" than the last 0'th element you extracted.  This is
+especially useful in simulation contexts, where the tree holds all
+incoming events, and the "win" condition means the smallest scheduled
+time.  When an event schedule other events for execution, they are
+scheduled into the future, so they can easily go into the heap.  So, a
+heap is a good structure for implementing schedulers (this is what I
+used for my MIDI sequencer :-).
+
+Various structures for implementing schedulers have been extensively
+studied, and heaps are good for this, as they are reasonably speedy,
+the speed is almost constant, and the worst case is not much different
+than the average case.  However, there are other representations which
+are more efficient overall, yet the worst cases might be terrible.
+
+Heaps are also very useful in big disk sorts.  You most probably all
+know that a big sort implies producing "runs" (which are pre-sorted
+sequences, which size is usually related to the amount of CPU memory),
+followed by a merging passes for these runs, which merging is often
+very cleverly organised[1].  It is very important that the initial
+sort produces the longest runs possible.  Tournaments are a good way
+to that.  If, using all the memory available to hold a tournament, you
+replace and percolate items that happen to fit the current run, you'll
+produce runs which are twice the size of the memory for random input,
+and much better for input fuzzily ordered.
+
+Moreover, if you output the 0'th item on disk and get an input which
+may not fit in the current tournament (because the value "wins" over
+the last output value), it cannot fit in the heap, so the size of the
+heap decreases.  The freed memory could be cleverly reused immediately
+for progressively building a second heap, which grows at exactly the
+same rate the first heap is melting.  When the first heap completely
+vanishes, you switch heaps and start a new run.  Clever and quite
+effective!
+
+In a word, heaps are useful memory structures to know.  I use them in
+a few applications, and I think it is good to keep a `heap' module
+around. :-)
+
+--------------------
+[1] The disk balancing algorithms which are current, nowadays, are
+more annoying than clever, and this is a consequence of the seeking
+capabilities of the disks.  On devices which cannot seek, like big
+tape drives, the story was quite different, and one had to be very
+clever to ensure (far in advance) that each tape movement will be the
+most effective possible (that is, will best participate at
+"progressing" the merge).  Some tapes were even able to read
+backwards, and this was also used to avoid the rewinding time.
+Believe me, real good tape sorts were quite spectacular to watch!
+From all times, sorting has always been a Great Art! :-)
+"""
+
+__all__ = ['heappush', 'heappop', 'heapify', 'heapreplace', 'merge',
+           'nlargest', 'nsmallest', 'heappushpop']
+
+from itertools import islice, repeat, count, imap, izip, tee, chain
+from operator import itemgetter
+import bisect
+
+def heappush(heap, item):
+    """Push item onto heap, maintaining the heap invariant."""
+    heap.append(item)
+    _siftdown(heap, 0, len(heap)-1)
+
+def heappop(heap):
+    """Pop the smallest item off the heap, maintaining the heap invariant."""
+    lastelt = heap.pop()    # raises appropriate IndexError if heap is empty
+    if heap:
+        returnitem = heap[0]
+        heap[0] = lastelt
+        _siftup(heap, 0)
+    else:
+        returnitem = lastelt
+    return returnitem
+
+def heapreplace(heap, item):
+    """Pop and return the current smallest value, and add the new item.
+
+    This is more efficient than heappop() followed by heappush(), and can be
+    more appropriate when using a fixed-size heap.  Note that the value
+    returned may be larger than item!  That constrains reasonable uses of
+    this routine unless written as part of a conditional replacement:
+
+        if item > heap[0]:
+            item = heapreplace(heap, item)
+    """
+    returnitem = heap[0]    # raises appropriate IndexError if heap is empty
+    heap[0] = item
+    _siftup(heap, 0)
+    return returnitem
+
+def heappushpop(heap, item):
+    """Fast version of a heappush followed by a heappop."""
+    if heap and heap[0] < item:
+        item, heap[0] = heap[0], item
+        _siftup(heap, 0)
+    return item
+
+def heapify(x):
+    """Transform list into a heap, in-place, in O(len(heap)) time."""
+    n = len(x)
+    # Transform bottom-up.  The largest index there's any point to looking at
+    # is the largest with a child index in-range, so must have 2*i + 1 < n,
+    # or i < (n-1)/2.  If n is even = 2*j, this is (2*j-1)/2 = j-1/2 so
+    # j-1 is the largest, which is n//2 - 1.  If n is odd = 2*j+1, this is
+    # (2*j+1-1)/2 = j so j-1 is the largest, and that's again n//2-1.
+    for i in reversed(xrange(n//2)):
+        _siftup(x, i)
+
+def nlargest(n, iterable):
+    """Find the n largest elements in a dataset.
+
+    Equivalent to:  sorted(iterable, reverse=True)[:n]
+    """
+    if n < 0: # for consistency with the c impl
+        return []
+    it = iter(iterable)
+    result = list(islice(it, n))
+    if not result:
+        return result
+    heapify(result)
+    _heappushpop = heappushpop
+    for elem in it:
+        _heappushpop(result, elem)
+    result.sort(reverse=True)
+    return result
+
+def nsmallest(n, iterable):
+    """Find the n smallest elements in a dataset.
+
+    Equivalent to:  sorted(iterable)[:n]
+    """
+    if n < 0: # for consistency with the c impl
+        return []
+    if hasattr(iterable, '__len__') and n * 10 <= len(iterable):
+        # For smaller values of n, the bisect method is faster than a minheap.
+        # It is also memory efficient, consuming only n elements of space.
+        it = iter(iterable)
+        result = sorted(islice(it, 0, n))
+        if not result:
+            return result
+        insort = bisect.insort
+        pop = result.pop
+        los = result[-1]    # los --> Largest of the nsmallest
+        for elem in it:
+            if los <= elem:
+                continue
+            insort(result, elem)
+            pop()
+            los = result[-1]
+        return result
+    # An alternative approach manifests the whole iterable in memory but
+    # saves comparisons by heapifying all at once.  Also, saves time
+    # over bisect.insort() which has O(n) data movement time for every
+    # insertion.  Finding the n smallest of an m length iterable requires
+    #    O(m) + O(n log m) comparisons.
+    h = list(iterable)
+    heapify(h)
+    return map(heappop, repeat(h, min(n, len(h))))
+
+# 'heap' is a heap at all indices >= startpos, except possibly for pos.  pos
+# is the index of a leaf with a possibly out-of-order value.  Restore the
+# heap invariant.
+def _siftdown(heap, startpos, pos):
+    newitem = heap[pos]
+    # Follow the path to the root, moving parents down until finding a place
+    # newitem fits.
+    while pos > startpos:
+        parentpos = (pos - 1) >> 1
+        parent = heap[parentpos]
+        if newitem < parent:
+            heap[pos] = parent
+            pos = parentpos
+            continue
+        break
+    heap[pos] = newitem
+
+# The child indices of heap index pos are already heaps, and we want to make
+# a heap at index pos too.  We do this by bubbling the smaller child of
+# pos up (and so on with that child's children, etc) until hitting a leaf,
+# then using _siftdown to move the oddball originally at index pos into place.
+#
+# We *could* break out of the loop as soon as we find a pos where newitem <=
+# both its children, but turns out that's not a good idea, and despite that
+# many books write the algorithm that way.  During a heap pop, the last array
+# element is sifted in, and that tends to be large, so that comparing it
+# against values starting from the root usually doesn't pay (= usually doesn't
+# get us out of the loop early).  See Knuth, Volume 3, where this is
+# explained and quantified in an exercise.
+#
+# Cutting the # of comparisons is important, since these routines have no
+# way to extract "the priority" from an array element, so that intelligence
+# is likely to be hiding in custom __cmp__ methods, or in array elements
+# storing (priority, record) tuples.  Comparisons are thus potentially
+# expensive.
+#
+# On random arrays of length 1000, making this change cut the number of
+# comparisons made by heapify() a little, and those made by exhaustive
+# heappop() a lot, in accord with theory.  Here are typical results from 3
+# runs (3 just to demonstrate how small the variance is):
+#
+# Compares needed by heapify     Compares needed by 1000 heappops
+# --------------------------     --------------------------------
+# 1837 cut to 1663               14996 cut to 8680
+# 1855 cut to 1659               14966 cut to 8678
+# 1847 cut to 1660               15024 cut to 8703
+#
+# Building the heap by using heappush() 1000 times instead required
+# 2198, 2148, and 2219 compares:  heapify() is more efficient, when
+# you can use it.
+#
+# The total compares needed by list.sort() on the same lists were 8627,
+# 8627, and 8632 (this should be compared to the sum of heapify() and
+# heappop() compares):  list.sort() is (unsurprisingly!) more efficient
+# for sorting.
+
+def _siftup(heap, pos):
+    endpos = len(heap)
+    startpos = pos
+    newitem = heap[pos]
+    # Bubble up the smaller child until hitting a leaf.
+    childpos = 2*pos + 1    # leftmost child position
+    while childpos < endpos:
+        # Set childpos to index of smaller child.
+        rightpos = childpos + 1
+        if rightpos < endpos and not heap[childpos] < heap[rightpos]:
+            childpos = rightpos
+        # Move the smaller child up.
+        heap[pos] = heap[childpos]
+        pos = childpos
+        childpos = 2*pos + 1
+    # The leaf at pos is empty now.  Put newitem there, and bubble it up
+    # to its final resting place (by sifting its parents down).
+    heap[pos] = newitem
+    _siftdown(heap, startpos, pos)
+
+# If available, use C implementation
+try:
+    from _heapq import *
+except ImportError:
+    pass
+
+def merge(*iterables):
+    '''Merge multiple sorted inputs into a single sorted output.
+
+    Similar to sorted(itertools.chain(*iterables)) but returns a generator,
+    does not pull the data into memory all at once, and assumes that each of
+    the input streams is already sorted (smallest to largest).
+
+    >>> list(merge([1,3,5,7], [0,2,4,8], [5,10,15,20], [], [25]))
+    [0, 1, 2, 3, 4, 5, 5, 7, 8, 10, 15, 20, 25]
+
+    '''
+    _heappop, _heapreplace, _StopIteration = heappop, heapreplace, StopIteration
+
+    h = []
+    h_append = h.append
+    for itnum, it in enumerate(map(iter, iterables)):
+        try:
+            next = it.next
+            h_append([next(), itnum, next])
+        except _StopIteration:
+            pass
+    heapify(h)
+
+    while 1:
+        try:
+            while 1:
+                v, itnum, next = s = h[0]   # raises IndexError when h is empty
+                yield v
+                s[0] = next()               # raises StopIteration when exhausted
+                _heapreplace(h, s)          # restore heap condition
+        except _StopIteration:
+            _heappop(h)                     # remove empty iterator
+        except IndexError:
+            return
+
+# Extend the implementations of nsmallest and nlargest to use a key= argument
+_nsmallest = nsmallest
+def nsmallest(n, iterable, key=None):
+    """Find the n smallest elements in a dataset.
+
+    Equivalent to:  sorted(iterable, key=key)[:n]
+    """
+    # Short-cut for n==1 is to use min() when len(iterable)>0
+    if n == 1:
+        it = iter(iterable)
+        head = list(islice(it, 1))
+        if not head:
+            return []
+        if key is None:
+            return [min(chain(head, it))]
+        return [min(chain(head, it), key=key)]
+
+    # When n>=size, it's faster to use sort()
+    try:
+        size = len(iterable)
+    except (TypeError, AttributeError):
+        pass
+    else:
+        if n >= size:
+            return sorted(iterable, key=key)[:n]
+
+    # When key is none, use simpler decoration
+    if key is None:
+        it = izip(iterable, count())                        # decorate
+        result = _nsmallest(n, it)
+        return map(itemgetter(0), result)                   # undecorate
+
+    # General case, slowest method
+    in1, in2 = tee(iterable)
+    it = izip(imap(key, in1), count(), in2)                 # decorate
+    result = _nsmallest(n, it)
+    return map(itemgetter(2), result)                       # undecorate
+
+_nlargest = nlargest
+def nlargest(n, iterable, key=None):
+    """Find the n largest elements in a dataset.
+
+    Equivalent to:  sorted(iterable, key=key, reverse=True)[:n]
+    """
+
+    # Short-cut for n==1 is to use max() when len(iterable)>0
+    if n == 1:
+        it = iter(iterable)
+        head = list(islice(it, 1))
+        if not head:
+            return []
+        if key is None:
+            return [max(chain(head, it))]
+        return [max(chain(head, it), key=key)]
+
+    # When n>=size, it's faster to use sort()
+    try:
+        size = len(iterable)
+    except (TypeError, AttributeError):
+        pass
+    else:
+        if n >= size:
+            return sorted(iterable, key=key, reverse=True)[:n]
+
+    # When key is none, use simpler decoration
+    if key is None:
+        it = izip(iterable, count(0,-1))                    # decorate
+        result = _nlargest(n, it)
+        return map(itemgetter(0), result)                   # undecorate
+
+    # General case, slowest method
+    in1, in2 = tee(iterable)
+    it = izip(imap(key, in1), count(0,-1), in2)             # decorate
+    result = _nlargest(n, it)
+    return map(itemgetter(2), result)                       # undecorate
+
+if __name__ == "__main__":
+    # Simple sanity test
+    heap = []
+    data = [1, 3, 5, 7, 9, 2, 4, 6, 8, 0]
+    for item in data:
+        heappush(heap, item)
+    sort = []
+    while heap:
+        sort.append(heappop(heap))
+    print sort
+
+    import doctest
+    doctest.testmod()
diff --git a/lib-python/modified-2.7/httplib.py b/lib-python/modified-2.7/httplib.py
new file mode 100644
--- /dev/null
+++ b/lib-python/modified-2.7/httplib.py
@@ -0,0 +1,1377 @@
+"""HTTP/1.1 client library
+
+<intro stuff goes here>
+<other stuff, too>
+
+HTTPConnection goes through a number of "states", which define when a client
+may legally make another request or fetch the response for a particular
+request. This diagram details these state transitions:
+
+    (null)
+      |
+      | HTTPConnection()
+      v
+    Idle
+      |
+      | putrequest()
+      v
+    Request-started
+      |
+      | ( putheader() )*  endheaders()
+      v
+    Request-sent
+      |
+      | response = getresponse()
+      v
+    Unread-response   [Response-headers-read]
+      |\____________________
+      |                     |
+      | response.read()     | putrequest()
+      v                     v
+    Idle                  Req-started-unread-response
+                     ______/|
+                   /        |
+   response.read() |        | ( putheader() )*  endheaders()
+                   v        v
+       Request-started    Req-sent-unread-response
+                            |
+                            | response.read()
+                            v
+                          Request-sent
+
+This diagram presents the following rules:
+  -- a second request may not be started until {response-headers-read}
+  -- a response [object] cannot be retrieved until {request-sent}
+  -- there is no differentiation between an unread response body and a
+     partially read response body
+
+Note: this enforcement is applied by the HTTPConnection class. The
+      HTTPResponse class does not enforce this state machine, which
+      implies sophisticated clients may accelerate the request/response
+      pipeline. Caution should be taken, though: accelerating the states
+      beyond the above pattern may imply knowledge of the server's
+      connection-close behavior for certain requests. For example, it
+      is impossible to tell whether the server will close the connection
+      UNTIL the response headers have been read; this means that further
+      requests cannot be placed into the pipeline until it is known that
+      the server will NOT be closing the connection.
+
+Logical State                  __state            __response
+-------------                  -------            ----------
+Idle                           _CS_IDLE           None
+Request-started                _CS_REQ_STARTED    None
+Request-sent                   _CS_REQ_SENT       None
+Unread-response                _CS_IDLE           <response_class>
+Req-started-unread-response    _CS_REQ_STARTED    <response_class>
+Req-sent-unread-response       _CS_REQ_SENT       <response_class>
+"""
+
+from array import array
+import os
+import socket
+from sys import py3kwarning
+from urlparse import urlsplit
+import warnings
+with warnings.catch_warnings():
+    if py3kwarning:
+        warnings.filterwarnings("ignore", ".*mimetools has been removed",
+                                DeprecationWarning)
+    import mimetools
+
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from StringIO import StringIO
+
+__all__ = ["HTTP", "HTTPResponse", "HTTPConnection",
+           "HTTPException", "NotConnected", "UnknownProtocol",
+           "UnknownTransferEncoding", "UnimplementedFileMode",
+           "IncompleteRead", "InvalidURL", "ImproperConnectionState",
+           "CannotSendRequest", "CannotSendHeader", "ResponseNotReady",
+           "BadStatusLine", "error", "responses"]
+
+HTTP_PORT = 80
+HTTPS_PORT = 443
+
+_UNKNOWN = 'UNKNOWN'
+
+# connection states
+_CS_IDLE = 'Idle'
+_CS_REQ_STARTED = 'Request-started'
+_CS_REQ_SENT = 'Request-sent'
+
+# status codes
+# informational
+CONTINUE = 100
+SWITCHING_PROTOCOLS = 101
+PROCESSING = 102
+
+# successful
+OK = 200
+CREATED = 201
+ACCEPTED = 202
+NON_AUTHORITATIVE_INFORMATION = 203
+NO_CONTENT = 204
+RESET_CONTENT = 205
+PARTIAL_CONTENT = 206
+MULTI_STATUS = 207
+IM_USED = 226
+
+# redirection
+MULTIPLE_CHOICES = 300
+MOVED_PERMANENTLY = 301
+FOUND = 302
+SEE_OTHER = 303
+NOT_MODIFIED = 304
+USE_PROXY = 305
+TEMPORARY_REDIRECT = 307
+
+# client error
+BAD_REQUEST = 400
+UNAUTHORIZED = 401
+PAYMENT_REQUIRED = 402
+FORBIDDEN = 403
+NOT_FOUND = 404
+METHOD_NOT_ALLOWED = 405
+NOT_ACCEPTABLE = 406
+PROXY_AUTHENTICATION_REQUIRED = 407
+REQUEST_TIMEOUT = 408
+CONFLICT = 409
+GONE = 410
+LENGTH_REQUIRED = 411
+PRECONDITION_FAILED = 412
+REQUEST_ENTITY_TOO_LARGE = 413
+REQUEST_URI_TOO_LONG = 414
+UNSUPPORTED_MEDIA_TYPE = 415
+REQUESTED_RANGE_NOT_SATISFIABLE = 416
+EXPECTATION_FAILED = 417
+UNPROCESSABLE_ENTITY = 422
+LOCKED = 423
+FAILED_DEPENDENCY = 424
+UPGRADE_REQUIRED = 426
+
+# server error
+INTERNAL_SERVER_ERROR = 500
+NOT_IMPLEMENTED = 501
+BAD_GATEWAY = 502
+SERVICE_UNAVAILABLE = 503
+GATEWAY_TIMEOUT = 504
+HTTP_VERSION_NOT_SUPPORTED = 505
+INSUFFICIENT_STORAGE = 507
+NOT_EXTENDED = 510
+
+# Mapping status codes to official W3C names
+responses = {
+    100: 'Continue',
+    101: 'Switching Protocols',
+
+    200: 'OK',
+    201: 'Created',
+    202: 'Accepted',
+    203: 'Non-Authoritative Information',
+    204: 'No Content',
+    205: 'Reset Content',
+    206: 'Partial Content',
+
+    300: 'Multiple Choices',
+    301: 'Moved Permanently',
+    302: 'Found',
+    303: 'See Other',
+    304: 'Not Modified',
+    305: 'Use Proxy',
+    306: '(Unused)',
+    307: 'Temporary Redirect',
+
+    400: 'Bad Request',
+    401: 'Unauthorized',
+    402: 'Payment Required',
+    403: 'Forbidden',
+    404: 'Not Found',
+    405: 'Method Not Allowed',
+    406: 'Not Acceptable',
+    407: 'Proxy Authentication Required',
+    408: 'Request Timeout',
+    409: 'Conflict',
+    410: 'Gone',
+    411: 'Length Required',
+    412: 'Precondition Failed',
+    413: 'Request Entity Too Large',
+    414: 'Request-URI Too Long',
+    415: 'Unsupported Media Type',
+    416: 'Requested Range Not Satisfiable',
+    417: 'Expectation Failed',
+
+    500: 'Internal Server Error',
+    501: 'Not Implemented',
+    502: 'Bad Gateway',
+    503: 'Service Unavailable',
+    504: 'Gateway Timeout',
+    505: 'HTTP Version Not Supported',
+}
+
+# maximal amount of data to read at one time in _safe_read
+MAXAMOUNT = 1048576
+
+class HTTPMessage(mimetools.Message):
+
+    def addheader(self, key, value):
+        """Add header for field key handling repeats."""
+        prev = self.dict.get(key)
+        if prev is None:
+            self.dict[key] = value
+        else:
+            combined = ", ".join((prev, value))
+            self.dict[key] = combined
+
+    def addcontinue(self, key, more):
+        """Add more field data from a continuation line."""
+        prev = self.dict[key]
+        self.dict[key] = prev + "\n " + more
+
+    def readheaders(self):
+        """Read header lines.
+
+        Read header lines up to the entirely blank line that terminates them.
+        The (normally blank) line that ends the headers is skipped, but not
+        included in the returned list.  If a non-header line ends the headers,
+        (which is an error), an attempt is made to backspace over it; it is
+        never included in the returned list.
+
+        The variable self.status is set to the empty string if all went well,
+        otherwise it is an error message.  The variable self.headers is a
+        completely uninterpreted list of lines contained in the header (so
+        printing them will reproduce the header exactly as it appears in the
+        file).
+
+        If multiple header fields with the same name occur, they are combined
+        according to the rules in RFC 2616 sec 4.2:
+
+        Appending each subsequent field-value to the first, each separated
+        by a comma. The order in which header fields with the same field-name
+        are received is significant to the interpretation of the combined
+        field value.
+        """
+        # XXX The implementation overrides the readheaders() method of
+        # rfc822.Message.  The base class design isn't amenable to
+        # customized behavior here so the method here is a copy of the
+        # base class code with a few small changes.
+
+        self.dict = {}
+        self.unixfrom = ''
+        self.headers = hlist = []
+        self.status = ''
+        headerseen = ""
+        firstline = 1
+        startofline = unread = tell = None
+        if hasattr(self.fp, 'unread'):
+            unread = self.fp.unread
+        elif self.seekable:
+            tell = self.fp.tell
+        while True:
+            if tell:
+                try:
+                    startofline = tell()
+                except IOError:
+                    startofline = tell = None
+                    self.seekable = 0
+            line = self.fp.readline()
+            if not line:
+                self.status = 'EOF in headers'
+                break
+            # Skip unix From name time lines
+            if firstline and line.startswith('From '):
+                self.unixfrom = self.unixfrom + line
+                continue
+            firstline = 0
+            if headerseen and line[0] in ' \t':
+                # XXX Not sure if continuation lines are handled properly
+                # for http and/or for repeating headers
+                # It's a continuation line.
+                hlist.append(line)
+                self.addcontinue(headerseen, line.strip())
+                continue
+            elif self.iscomment(line):
+                # It's a comment.  Ignore it.
+                continue
+            elif self.islast(line):
+                # Note! No pushback here!  The delimiter line gets eaten.
+                break
+            headerseen = self.isheader(line)
+            if headerseen:
+                # It's a legal header line, save it.
+                hlist.append(line)
+                self.addheader(headerseen, line[len(headerseen)+1:].strip())
+                continue
+            else:
+                # It's not a header line; throw it back and stop here.
+                if not self.dict:
+                    self.status = 'No headers'
+                else:
+                    self.status = 'Non-header line where header expected'
+                # Try to undo the read.
+                if unread:
+                    unread(line)
+                elif tell:
+                    self.fp.seek(startofline)
+                else:
+                    self.status = self.status + '; bad seek'
+                break
+
+class HTTPResponse:
+
+    # strict: If true, raise BadStatusLine if the status line can't be
+    # parsed as a valid HTTP/1.0 or 1.1 status line.  By default it is
+    # false because it prevents clients from talking to HTTP/0.9
+    # servers.  Note that a response with a sufficiently corrupted
+    # status line will look like an HTTP/0.9 response.
+
+    # See RFC 2616 sec 19.6 and RFC 1945 sec 6 for details.
+
+    def __init__(self, sock, debuglevel=0, strict=0, method=None, buffering=False):
+        if buffering:
+            # The caller won't be using any sock.recv() calls, so buffering
+            # is fine and recommended for performance.
+            self.fp = sock.makefile('rb')
+        else:
+            # The buffer size is specified as zero, because the headers of
+            # the response are read with readline().  If the reads were
+            # buffered the readline() calls could consume some of the
+            # response, which make be read via a recv() on the underlying
+            # socket.
+            self.fp = sock.makefile('rb', 0)
+        self.debuglevel = debuglevel
+        self.strict = strict
+        self._method = method
+
+        self.msg = None
+
+        # from the Status-Line of the response
+        self.version = _UNKNOWN # HTTP-Version
+        self.status = _UNKNOWN  # Status-Code
+        self.reason = _UNKNOWN  # Reason-Phrase
+
+        self.chunked = _UNKNOWN         # is "chunked" being used?
+        self.chunk_left = _UNKNOWN      # bytes left to read in current chunk
+        self.length = _UNKNOWN          # number of bytes left in response
+        self.will_close = _UNKNOWN      # conn will close at end of response
+
+    def _read_status(self):
+        # Initialize with Simple-Response defaults
+        line = self.fp.readline()
+        if self.debuglevel > 0:
+            print "reply:", repr(line)
+        if not line:
+            # Presumably, the server closed the connection before
+            # sending a valid response.
+            raise BadStatusLine(line)
+        try:
+            [version, status, reason] = line.split(None, 2)
+        except ValueError:
+            try:
+                [version, status] = line.split(None, 1)
+                reason = ""
+            except ValueError:
+                # empty version will cause next test to fail and status
+                # will be treated as 0.9 response.
+                version = ""
+        if not version.startswith('HTTP/'):
+            if self.strict:
+                self.close()
+                raise BadStatusLine(line)
+            else:
+                # assume it's a Simple-Response from an 0.9 server
+                self.fp = LineAndFileWrapper(line, self.fp)
+                return "HTTP/0.9", 200, ""
+
+        # The status code is a three-digit number
+        try:
+            status = int(status)
+            if status < 100 or status > 999:
+                raise BadStatusLine(line)
+        except ValueError:
+            raise BadStatusLine(line)
+        return version, status, reason
+
+    def begin(self):
+        if self.msg is not None:
+            # we've already started reading the response
+            return
+
+        # read until we get a non-100 response
+        while True:
+            version, status, reason = self._read_status()
+            if status != CONTINUE:
+                break
+            # skip the header from the 100 response
+            while True:
+                skip = self.fp.readline().strip()
+                if not skip:
+                    break
+                if self.debuglevel > 0:
+                    print "header:", skip
+
+        self.status = status
+        self.reason = reason.strip()
+        if version == 'HTTP/1.0':
+            self.version = 10
+        elif version.startswith('HTTP/1.'):
+            self.version = 11   # use HTTP/1.1 code for HTTP/1.x where x>=1
+        elif version == 'HTTP/0.9':
+            self.version = 9
+        else:
+            raise UnknownProtocol(version)
+
+        if self.version == 9:
+            self.length = None
+            self.chunked = 0
+            self.will_close = 1
+            self.msg = HTTPMessage(StringIO())
+            return
+
+        self.msg = HTTPMessage(self.fp, 0)
+        if self.debuglevel > 0:
+            for hdr in self.msg.headers:
+                print "header:", hdr,
+
+        # don't let the msg keep an fp
+        self.msg.fp = None
+
+        # are we using the chunked-style of transfer encoding?
+        tr_enc = self.msg.getheader('transfer-encoding')
+        if tr_enc and tr_enc.lower() == "chunked":
+            self.chunked = 1
+            self.chunk_left = None
+        else:
+            self.chunked = 0
+
+        # will the connection close at the end of the response?
+        self.will_close = self._check_close()
+
+        # do we have a Content-Length?
+        # NOTE: RFC 2616, S4.4, #3 says we ignore this if tr_enc is "chunked"
+        length = self.msg.getheader('content-length')
+        if length and not self.chunked:
+            try:
+                self.length = int(length)
+            except ValueError:
+                self.length = None
+            else:
+                if self.length < 0:  # ignore nonsensical negative lengths
+                    self.length = None
+        else:
+            self.length = None
+
+        # does the body have a fixed length? (of zero)
+        if (status == NO_CONTENT or status == NOT_MODIFIED or
+            100 <= status < 200 or      # 1xx codes
+            self._method == 'HEAD'):
+            self.length = 0
+
+        # if the connection remains open, and we aren't using chunked, and
+        # a content-length was not provided, then assume that the connection
+        # WILL close.
+        if not self.will_close and \
+           not self.chunked and \
+           self.length is None:
+            self.will_close = 1
+
+    def _check_close(self):
+        conn = self.msg.getheader('connection')
+        if self.version == 11:
+            # An HTTP/1.1 proxy is assumed to stay open unless
+            # explicitly closed.
+            conn = self.msg.getheader('connection')
+            if conn and "close" in conn.lower():
+                return True
+            return False
+
+        # Some HTTP/1.0 implementations have support for persistent
+        # connections, using rules different than HTTP/1.1.
+
+        # For older HTTP, Keep-Alive indicates persistent connection.
+        if self.msg.getheader('keep-alive'):
+            return False
+
+        # At least Akamai returns a "Connection: Keep-Alive" header,
+        # which was supposed to be sent by the client.
+        if conn and "keep-alive" in conn.lower():
+            return False
+
+        # Proxy-Connection is a netscape hack.
+        pconn = self.msg.getheader('proxy-connection')
+        if pconn and "keep-alive" in pconn.lower():
+            return False
+
+        # otherwise, assume it will close
+        return True
+
+    def close(self):
+        if self.fp:
+            self.fp.close()
+            self.fp = None
+
+    def isclosed(self):
+        # NOTE: it is possible that we will not ever call self.close(). This
+        #       case occurs when will_close is TRUE, length is None, and we
+        #       read up to the last byte, but NOT past it.
+        #
+        # IMPLIES: if will_close is FALSE, then self.close() will ALWAYS be
+        #          called, meaning self.isclosed() is meaningful.
+        return self.fp is None
+
+    # XXX It would be nice to have readline and __iter__ for this, too.
+
+    def read(self, amt=None):
+        if self.fp is None:
+            return ''
+
+        if self._method == 'HEAD':
+            self.close()
+            return ''
+
+        if self.chunked:
+            return self._read_chunked(amt)
+
+        if amt is None:
+            # unbounded read
+            if self.length is None:
+                s = self.fp.read()
+            else:
+                s = self._safe_read(self.length)
+                self.length = 0
+            self.close()        # we read everything
+            return s
+
+        if self.length is not None:
+            if amt > self.length:
+                # clip the read to the "end of response"
+                amt = self.length
+
+        # we do not use _safe_read() here because this may be a .will_close
+        # connection, and the user is reading more bytes than will be provided
+        # (for example, reading in 1k chunks)
+        s = self.fp.read(amt)
+        if self.length is not None:
+            self.length -= len(s)
+            if not self.length:
+                self.close()
+        return s
+
+    def _read_chunked(self, amt):
+        assert self.chunked != _UNKNOWN
+        chunk_left = self.chunk_left
+        value = []
+        while True:
+            if chunk_left is None:
+                line = self.fp.readline()
+                i = line.find(';')
+                if i >= 0:
+                    line = line[:i] # strip chunk-extensions
+                try:
+                    chunk_left = int(line, 16)
+                except ValueError:
+                    # close the connection as protocol synchronisation is
+                    # probably lost
+                    self.close()
+                    raise IncompleteRead(''.join(value))
+                if chunk_left == 0:
+                    break
+            if amt is None:
+                value.append(self._safe_read(chunk_left))
+            elif amt < chunk_left:
+                value.append(self._safe_read(amt))
+                self.chunk_left = chunk_left - amt
+                return ''.join(value)
+            elif amt == chunk_left:
+                value.append(self._safe_read(amt))
+                self._safe_read(2)  # toss the CRLF at the end of the chunk
+                self.chunk_left = None
+                return ''.join(value)
+            else:
+                value.append(self._safe_read(chunk_left))
+                amt -= chunk_left
+
+            # we read the whole chunk, get another
+            self._safe_read(2)      # toss the CRLF at the end of the chunk
+            chunk_left = None
+
+        # read and discard trailer up to the CRLF terminator
+        ### note: we shouldn't have any trailers!
+        while True:
+            line = self.fp.readline()
+            if not line:
+                # a vanishingly small number of sites EOF without
+                # sending the trailer
+                break
+            if line == '\r\n':
+                break
+
+        # we read everything; close the "file"
+        self.close()
+
+        return ''.join(value)
+
+    def _safe_read(self, amt):
+        """Read the number of bytes requested, compensating for partial reads.
+
+        Normally, we have a blocking socket, but a read() can be interrupted
+        by a signal (resulting in a partial read).
+
+        Note that we cannot distinguish between EOF and an interrupt when zero
+        bytes have been read. IncompleteRead() will be raised in this
+        situation.
+
+        This function should be used when <amt> bytes "should" be present for
+        reading. If the bytes are truly not available (due to EOF), then the
+        IncompleteRead exception can be used to detect the problem.
+        """
+        # NOTE(gps): As of svn r74426 socket._fileobject.read(x) will never
+        # return less than x bytes unless EOF is encountered.  It now handles
+        # signal interruptions (socket.error EINTR) internally.  This code
+        # never caught that exception anyways.  It seems largely pointless.
+        # self.fp.read(amt) will work fine.
+        s = []
+        while amt > 0:
+            chunk = self.fp.read(min(amt, MAXAMOUNT))
+            if not chunk:
+                raise IncompleteRead(''.join(s), amt)
+            s.append(chunk)
+            amt -= len(chunk)
+        return ''.join(s)
+
+    def fileno(self):
+        return self.fp.fileno()
+
+    def getheader(self, name, default=None):
+        if self.msg is None:
+            raise ResponseNotReady()
+        return self.msg.getheader(name, default)
+
+    def getheaders(self):
+        """Return list of (header, value) tuples."""
+        if self.msg is None:
+            raise ResponseNotReady()
+        return self.msg.items()
+
+
+class HTTPConnection:
+
+    _http_vsn = 11
+    _http_vsn_str = 'HTTP/1.1'
+
+    response_class = HTTPResponse
+    default_port = HTTP_PORT
+    auto_open = 1
+    debuglevel = 0
+    strict = 0
+
+    def __init__(self, host, port=None, strict=None,
+                 timeout=socket._GLOBAL_DEFAULT_TIMEOUT, source_address=None):
+        self.timeout = timeout
+        self.source_address = source_address
+        self.sock = None
+        self._buffer = []
+        self.__response = None
+        self.__state = _CS_IDLE
+        self._method = None
+        self._tunnel_host = None
+        self._tunnel_port = None
+        self._tunnel_headers = {}
+
+        self._set_hostport(host, port)
+        if strict is not None:
+            self.strict = strict
+
+    def set_tunnel(self, host, port=None, headers=None):
+        """ Sets up the host and the port for the HTTP CONNECT Tunnelling.
+
+        The headers argument should be a mapping of extra HTTP headers
+        to send with the CONNECT request.
+        """
+        self._tunnel_host = host
+        self._tunnel_port = port
+        if headers:
+            self._tunnel_headers = headers
+        else:
+            self._tunnel_headers.clear()
+
+    def _set_hostport(self, host, port):
+        if port is None:
+            i = host.rfind(':')
+            j = host.rfind(']')         # ipv6 addresses have [...]
+            if i > j:
+                try:
+                    port = int(host[i+1:])
+                except ValueError:
+                    raise InvalidURL("nonnumeric port: '%s'" % host[i+1:])
+                host = host[:i]
+            else:
+                port = self.default_port
+            if host and host[0] == '[' and host[-1] == ']':
+                host = host[1:-1]
+        self.host = host
+        self.port = port
+
+    def set_debuglevel(self, level):
+        self.debuglevel = level
+
+    def _tunnel(self):
+        self._set_hostport(self._tunnel_host, self._tunnel_port)
+        self.send("CONNECT %s:%d HTTP/1.0\r\n" % (self.host, self.port))
+        for header, value in self._tunnel_headers.iteritems():
+            self.send("%s: %s\r\n" % (header, value))
+        self.send("\r\n")
+        response = self.response_class(self.sock, strict = self.strict,
+                                       method = self._method)
+        (version, code, message) = response._read_status()
+
+        if code != 200:
+            self.close()
+            raise socket.error("Tunnel connection failed: %d %s" % (code,
+                                                                    message.strip()))
+        while True:
+            line = response.fp.readline()
+            if line == '\r\n': break
+
+
+    def connect(self):
+        """Connect to the host and port specified in __init__."""
+        self.sock = socket.create_connection((self.host,self.port),
+                                             self.timeout, self.source_address)
+
+        if self._tunnel_host:
+            self._tunnel()
+
+    def close(self):
+        """Close the connection to the HTTP server."""
+        if self.sock:
+            self.sock.close()   # close it manually... there may be other refs
+            self.sock = None
+        if self.__response:
+            self.__response.close()
+            self.__response = None
+        self.__state = _CS_IDLE
+
+    def send(self, data):
+        """Send `data' to the server."""
+        if self.sock is None:
+            if self.auto_open:
+                self.connect()
+            else:
+                raise NotConnected()
+
+        if self.debuglevel > 0:
+            print "send:", repr(data)
+        blocksize = 8192
+        if hasattr(data,'read') and not isinstance(data, array):
+            if self.debuglevel > 0: print "sendIng a read()able"
+            datablock = data.read(blocksize)
+            while datablock:
+                self.sock.sendall(datablock)
+                datablock = data.read(blocksize)
+        else:
+            self.sock.sendall(data)
+
+    def _output(self, s):
+        """Add a line of output to the current request buffer.
+
+        Assumes that the line does *not* end with \\r\\n.
+        """
+        self._buffer.append(s)
+
+    def _send_output(self, message_body=None):
+        """Send the currently buffered request and clear the buffer.
+
+        Appends an extra \\r\\n to the buffer.
+        A message_body may be specified, to be appended to the request.
+        """
+        self._buffer.extend(("", ""))
+        msg = "\r\n".join(self._buffer)
+        del self._buffer[:]
+        # If msg and message_body are sent in a single send() call,
+        # it will avoid performance problems caused by the interaction
+        # between delayed ack and the Nagle algorithim.
+        if isinstance(message_body, str):
+            msg += message_body
+            message_body = None
+        self.send(msg)
+        if message_body is not None:
+            #message_body was not a string (i.e. it is a file) and
+            #we must run the risk of Nagle
+            self.send(message_body)
+
+    def putrequest(self, method, url, skip_host=0, skip_accept_encoding=0):
+        """Send a request to the server.
+
+        `method' specifies an HTTP request method, e.g. 'GET'.
+        `url' specifies the object being requested, e.g. '/index.html'.
+        `skip_host' if True does not add automatically a 'Host:' header
+        `skip_accept_encoding' if True does not add automatically an
+           'Accept-Encoding:' header
+        """
+
+        # if a prior response has been completed, then forget about it.
+        if self.__response and self.__response.isclosed():
+            self.__response = None
+
+
+        # in certain cases, we cannot issue another request on this connection.
+        # this occurs when:
+        #   1) we are in the process of sending a request.   (_CS_REQ_STARTED)
+        #   2) a response to a previous request has signalled that it is going
+        #      to close the connection upon completion.
+        #   3) the headers for the previous response have not been read, thus
+        #      we cannot determine whether point (2) is true.   (_CS_REQ_SENT)
+        #
+        # if there is no prior response, then we can request at will.
+        #
+        # if point (2) is true, then we will have passed the socket to the
+        # response (effectively meaning, "there is no prior response"), and
+        # will open a new one when a new request is made.
+        #
+        # Note: if a prior response exists, then we *can* start a new request.
+        #       We are not allowed to begin fetching the response to this new
+        #       request, however, until that prior response is complete.
+        #
+        if self.__state == _CS_IDLE:
+            self.__state = _CS_REQ_STARTED
+        else:
+            raise CannotSendRequest()
+
+        # Save the method we use, we need it later in the response phase
+        self._method = method
+        if not url:
+            url = '/'
+        hdr = '%s %s %s' % (method, url, self._http_vsn_str)
+
+        self._output(hdr)
+
+        if self._http_vsn == 11:
+            # Issue some standard headers for better HTTP/1.1 compliance
+
+            if not skip_host:
+                # this header is issued *only* for HTTP/1.1
+                # connections. more specifically, this means it is
+                # only issued when the client uses the new
+                # HTTPConnection() class. backwards-compat clients
+                # will be using HTTP/1.0 and those clients may be
+                # issuing this header themselves. we should NOT issue
+                # it twice; some web servers (such as Apache) barf
+                # when they see two Host: headers
+
+                # If we need a non-standard port,include it in the
+                # header.  If the request is going through a proxy,
+                # but the host of the actual URL, not the host of the
+                # proxy.
+
+                netloc = ''
+                if url.startswith('http'):
+                    nil, netloc, nil, nil, nil = urlsplit(url)
+
+                if netloc:
+                    try:
+                        netloc_enc = netloc.encode("ascii")
+                    except UnicodeEncodeError:
+                        netloc_enc = netloc.encode("idna")
+                    self.putheader('Host', netloc_enc)
+                else:
+                    try:
+                        host_enc = self.host.encode("ascii")
+                    except UnicodeEncodeError:
+                        host_enc = self.host.encode("idna")
+                    # Wrap the IPv6 Host Header with [] (RFC 2732)
+                    if host_enc.find(':') >= 0:
+                        host_enc = "[" + host_enc + "]"
+                    if self.port == self.default_port:
+                        self.putheader('Host', host_enc)
+                    else:
+                        self.putheader('Host', "%s:%s" % (host_enc, self.port))
+
+            # note: we are assuming that clients will not attempt to set these
+            #       headers since *this* library must deal with the
+            #       consequences. this also means that when the supporting
+            #       libraries are updated to recognize other forms, then this
+            #       code should be changed (removed or updated).
+
+            # we only want a Content-Encoding of "identity" since we don't
+            # support encodings such as x-gzip or x-deflate.
+            if not skip_accept_encoding:
+                self.putheader('Accept-Encoding', 'identity')
+
+            # we can accept "chunked" Transfer-Encodings, but no others
+            # NOTE: no TE header implies *only* "chunked"
+            #self.putheader('TE', 'chunked')
+
+            # if TE is supplied in the header, then it must appear in a
+            # Connection header.
+            #self.putheader('Connection', 'TE')
+
+        else:
+            # For HTTP/1.0, the server will assume "not chunked"
+            pass
+
+    def putheader(self, header, *values):
+        """Send a request header line to the server.
+
+        For example: h.putheader('Accept', 'text/html')
+        """
+        if self.__state != _CS_REQ_STARTED:
+            raise CannotSendHeader()
+
+        hdr = '%s: %s' % (header, '\r\n\t'.join([str(v) for v in values]))
+        self._output(hdr)
+
+    def endheaders(self, message_body=None):
+        """Indicate that the last header line has been sent to the server.
+
+        This method sends the request to the server.  The optional
+        message_body argument can be used to pass message body
+        associated with the request.  The message body will be sent in
+        the same packet as the message headers if possible.  The
+        message_body should be a string.
+        """
+        if self.__state == _CS_REQ_STARTED:
+            self.__state = _CS_REQ_SENT
+        else:
+            raise CannotSendHeader()
+        self._send_output(message_body)
+
+    def request(self, method, url, body=None, headers={}):
+        """Send a complete request to the server."""
+        self._send_request(method, url, body, headers)
+
+    def _set_content_length(self, body):
+        # Set the content-length based on the body.
+        thelen = None
+        try:
+            thelen = str(len(body))
+        except TypeError, te:
+            # If this is a file-like object, try to
+            # fstat its file descriptor
+            try:
+                thelen = str(os.fstat(body.fileno()).st_size)
+            except (AttributeError, OSError):
+                # Don't send a length if this failed
+                if self.debuglevel > 0: print "Cannot stat!!"
+
+        if thelen is not None:
+            self.putheader('Content-Length', thelen)
+
+    def _send_request(self, method, url, body, headers):
+        # Honor explicitly requested Host: and Accept-Encoding: headers.
+        header_names = dict.fromkeys([k.lower() for k in headers])
+        skips = {}
+        if 'host' in header_names:
+            skips['skip_host'] = 1
+        if 'accept-encoding' in header_names:
+            skips['skip_accept_encoding'] = 1
+
+        self.putrequest(method, url, **skips)
+
+        if body and ('content-length' not in header_names):
+            self._set_content_length(body)
+        for hdr, value in headers.iteritems():
+            self.putheader(hdr, value)
+        self.endheaders(body)
+
+    def getresponse(self, buffering=False):
+        "Get the response from the server."
+
+        # if a prior response has been completed, then forget about it.
+        if self.__response and self.__response.isclosed():
+            self.__response = None
+
+        #
+        # if a prior response exists, then it must be completed (otherwise, we
+        # cannot read this response's header to determine the connection-close
+        # behavior)
+        #
+        # note: if a prior response existed, but was connection-close, then the
+        # socket and response were made independent of this HTTPConnection
+        # object since a new request requires that we open a whole new
+        # connection
+        #
+        # this means the prior response had one of two states:
+        #   1) will_close: this connection was reset and the prior socket and
+        #                  response operate independently
+        #   2) persistent: the response was retained and we await its
+        #                  isclosed() status to become true.
+        #
+        if self.__state != _CS_REQ_SENT or self.__response:
+            raise ResponseNotReady()
+
+        args = (self.sock,)
+        kwds = {"strict":self.strict, "method":self._method}
+        if self.debuglevel > 0:
+            args += (self.debuglevel,)
+        if buffering:
+            #only add this keyword if non-default, for compatibility with
+            #other response_classes.
+            kwds["buffering"] = True;
+        response = self.response_class(*args, **kwds)
+
+        try:
+            response.begin()
+        except:
+            response.close()
+            raise
+        assert response.will_close != _UNKNOWN
+        self.__state = _CS_IDLE
+
+        if response.will_close:
+            # this effectively passes the connection to the response
+            self.close()
+        else:
+            # remember this, so we can tell when it is complete
+            self.__response = response
+
+        return response
+
+
+class HTTP:
+    "Compatibility class with httplib.py from 1.5."
+
+    _http_vsn = 10
+    _http_vsn_str = 'HTTP/1.0'
+
+    debuglevel = 0
+
+    _connection_class = HTTPConnection
+
+    def __init__(self, host='', port=None, strict=None):
+        "Provide a default host, since the superclass requires one."
+
+        # some joker passed 0 explicitly, meaning default port
+        if port == 0:
+            port = None
+
+        # Note that we may pass an empty string as the host; this will throw
+        # an error when we attempt to connect. Presumably, the client code
+        # will call connect before then, with a proper host.
+        self._setup(self._connection_class(host, port, strict))
+
+    def _setup(self, conn):
+        self._conn = conn
+
+        # set up delegation to flesh out interface
+        self.send = conn.send
+        self.putrequest = conn.putrequest
+        self.putheader = conn.putheader
+        self.endheaders = conn.endheaders
+        self.set_debuglevel = conn.set_debuglevel
+
+        conn._http_vsn = self._http_vsn
+        conn._http_vsn_str = self._http_vsn_str
+
+        self.file = None
+
+    def connect(self, host=None, port=None):
+        "Accept arguments to set the host/port, since the superclass doesn't."
+
+        if host is not None:
+            self._conn._set_hostport(host, port)
+        self._conn.connect()
+
+    def getfile(self):
+        "Provide a getfile, since the superclass' does not use this concept."
+        return self.file
+
+    def getreply(self, buffering=False):
+        """Compat definition since superclass does not define it.
+
+        Returns a tuple consisting of:
+        - server status code (e.g. '200' if all goes well)
+        - server "reason" corresponding to status code
+        - any RFC822 headers in the response from the server
+        """
+        try:
+            if not buffering:
+                response = self._conn.getresponse()
+            else:
+                #only add this keyword if non-default for compatibility
+                #with other connection classes
+                response = self._conn.getresponse(buffering)
+        except BadStatusLine, e:
+            ### hmm. if getresponse() ever closes the socket on a bad request,
+            ### then we are going to have problems with self.sock
+
+            ### should we keep this behavior? do people use it?
+            # keep the socket open (as a file), and return it
+            self.file = self._conn.sock.makefile('rb', 0)
+
+            # close our socket -- we want to restart after any protocol error
+            self.close()
+
+            self.headers = None
+            return -1, e.line, None
+
+        self.headers = response.msg
+        self.file = response.fp
+        return response.status, response.reason, response.msg
+
+    def close(self):
+        self._conn.close()
+
+        # note that self.file == response.fp, which gets closed by the
+        # superclass. just clear the object ref here.
+        ### hmm. messy. if status==-1, then self.file is owned by us.
+        ### well... we aren't explicitly closing, but losing this ref will
+        ### do it
+        self.file = None
+
+try:
+    import ssl
+except ImportError:
+    pass
+else:
+    class HTTPSConnection(HTTPConnection):
+        "This class allows communication via SSL."
+
+        default_port = HTTPS_PORT
+
+        def __init__(self, host, port=None, key_file=None, cert_file=None,
+                     strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
+                     source_address=None):
+            HTTPConnection.__init__(self, host, port, strict, timeout,
+                                    source_address)
+            self.key_file = key_file
+            self.cert_file = cert_file
+
+        def connect(self):
+            "Connect to a host on a given (SSL) port."
+
+            sock = socket.create_connection((self.host, self.port),
+                                            self.timeout, self.source_address)
+            if self._tunnel_host:
+                self.sock = sock
+                self._tunnel()
+            self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file)
+
+    __all__.append("HTTPSConnection")
+
+    class HTTPS(HTTP):
+        """Compatibility with 1.5 httplib interface
+
+        Python 1.5.2 did not have an HTTPS class, but it defined an
+        interface for sending http requests that is also useful for
+        https.
+        """
+
+        _connection_class = HTTPSConnection
+
+        def __init__(self, host='', port=None, key_file=None, cert_file=None,
+                     strict=None):
+            # provide a default host, pass the X509 cert info
+
+            # urf. compensate for bad input.
+            if port == 0:
+                port = None
+            self._setup(self._connection_class(host, port, key_file,
+                                               cert_file, strict))
+
+            # we never actually use these for anything, but we keep them
+            # here for compatibility with post-1.5.2 CVS.
+            self.key_file = key_file
+            self.cert_file = cert_file
+
+
+    def FakeSocket (sock, sslobj):
+        warnings.warn("FakeSocket is deprecated, and won't be in 3.x.  " +
+                      "Use the result of ssl.wrap_socket() directly instead.",
+                      DeprecationWarning, stacklevel=2)
+        return sslobj
+
+
+class HTTPException(Exception):
+    # Subclasses that define an __init__ must call Exception.__init__
+    # or define self.args.  Otherwise, str() will fail.
+    pass
+
+class NotConnected(HTTPException):
+    pass
+
+class InvalidURL(HTTPException):
+    pass
+
+class UnknownProtocol(HTTPException):
+    def __init__(self, version):
+        self.args = version,
+        self.version = version
+
+class UnknownTransferEncoding(HTTPException):
+    pass
+
+class UnimplementedFileMode(HTTPException):
+    pass
+
+class IncompleteRead(HTTPException):
+    def __init__(self, partial, expected=None):
+        self.args = partial,
+        self.partial = partial
+        self.expected = expected
+    def __repr__(self):
+        if self.expected is not None:
+            e = ', %i more expected' % self.expected
+        else:
+            e = ''
+        return 'IncompleteRead(%i bytes read%s)' % (len(self.partial), e)
+    def __str__(self):
+        return repr(self)
+
+class ImproperConnectionState(HTTPException):
+    pass
+
+class CannotSendRequest(ImproperConnectionState):
+    pass
+
+class CannotSendHeader(ImproperConnectionState):
+    pass
+
+class ResponseNotReady(ImproperConnectionState):
+    pass
+
+class BadStatusLine(HTTPException):
+    def __init__(self, line):
+        if not line:
+            line = repr(line)
+        self.args = line,
+        self.line = line
+
+# for backwards compatibility
+error = HTTPException
+
+class LineAndFileWrapper:
+    """A limited file-like object for HTTP/0.9 responses."""
+
+    # The status-line parsing code calls readline(), which normally
+    # get the HTTP status line.  For a 0.9 response, however, this is
+    # actually the first line of the body!  Clients need to get a
+    # readable file object that contains that line.
+
+    def __init__(self, line, file):
+        self._line = line
+        self._file = file
+        self._line_consumed = 0
+        self._line_offset = 0
+        self._line_left = len(line)
+
+    def __getattr__(self, attr):
+        return getattr(self._file, attr)
+
+    def _done(self):
+        # called when the last byte is read from the line.  After the
+        # call, all read methods are delegated to the underlying file
+        # object.
+        self._line_consumed = 1
+        self.read = self._file.read
+        self.readline = self._file.readline
+        self.readlines = self._file.readlines
+
+    def read(self, amt=None):
+        if self._line_consumed:
+            return self._file.read(amt)
+        assert self._line_left
+        if amt is None or amt > self._line_left:
+            s = self._line[self._line_offset:]
+            self._done()
+            if amt is None:
+                return s + self._file.read()
+            else:
+                return s + self._file.read(amt - len(s))
+        else:
+            assert amt <= self._line_left
+            i = self._line_offset
+            j = i + amt
+            s = self._line[i:j]
+            self._line_offset = j
+            self._line_left -= amt
+            if self._line_left == 0:
+                self._done()
+            return s
+
+    def readline(self):
+        if self._line_consumed:
+            return self._file.readline()
+        assert self._line_left
+        s = self._line[self._line_offset:]
+        self._done()
+        return s
+
+    def readlines(self, size=None):
+        if self._line_consumed:
+            return self._file.readlines(size)
+        assert self._line_left
+        L = [self._line[self._line_offset:]]
+        self._done()
+        if size is None:
+            return L + self._file.readlines()
+        else:
+            return L + self._file.readlines(size)
+
+def test():
+    """Test this module.
+
+    A hodge podge of tests collected here, because they have too many
+    external dependencies for the regular test suite.
+    """
+
+    import sys
+    import getopt
+    opts, args = getopt.getopt(sys.argv[1:], 'd')
+    dl = 0
+    for o, a in opts:
+        if o == '-d': dl = dl + 1
+    host = 'www.python.org'
+    selector = '/'
+    if args[0:]: host = args[0]
+    if args[1:]: selector = args[1]
+    h = HTTP()
+    h.set_debuglevel(dl)
+    h.connect(host)
+    h.putrequest('GET', selector)
+    h.endheaders()
+    status, reason, headers = h.getreply()
+    print 'status =', status
+    print 'reason =', reason
+    print "read", len(h.getfile().read())
+    print
+    if headers:
+        for header in headers.headers: print header.strip()
+    print
+
+    # minimal test that code to extract host from url works
+    class HTTP11(HTTP):
+        _http_vsn = 11
+        _http_vsn_str = 'HTTP/1.1'
+
+    h = HTTP11('www.python.org')
+    h.putrequest('GET', 'http://www.python.org/~jeremy/')
+    h.endheaders()
+    h.getreply()
+    h.close()
+
+    try:
+        import ssl
+    except ImportError:
+        pass
+    else:
+
+        for host, selector in (('sourceforge.net', '/projects/python'),
+                               ):
+            print "https://%s%s" % (host, selector)
+            hs = HTTPS()
+            hs.set_debuglevel(dl)
+            hs.connect(host)
+            hs.putrequest('GET', selector)
+            hs.endheaders()
+            status, reason, headers = hs.getreply()
+            print 'status =', status
+            print 'reason =', reason
+            print "read", len(hs.getfile().read())
+            print
+            if headers:
+                for header in headers.headers: print header.strip()
+            print
+
+if __name__ == '__main__':
+    test()
diff --git a/lib-python/modified-2.7/json/encoder.py b/lib-python/modified-2.7/json/encoder.py
--- a/lib-python/modified-2.7/json/encoder.py
+++ b/lib-python/modified-2.7/json/encoder.py
@@ -2,14 +2,7 @@
 """
 import re
 
-try:
-    from _json import encode_basestring_ascii as c_encode_basestring_ascii
-except ImportError:
-    c_encode_basestring_ascii = None
-try:
-    from _json import make_encoder as c_make_encoder
-except ImportError:
-    c_make_encoder = None
+from __pypy__.builders import StringBuilder, UnicodeBuilder
 
 ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]')
 ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])')
@@ -24,8 +17,7 @@
     '\t': '\\t',
 }
 for i in range(0x20):
-    ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i))
-    #ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,))
+    ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,))
 
 # Assume this produces an infinity on all machines (probably not guaranteed)
 INFINITY = float('1e66666')
@@ -37,10 +29,9 @@
     """
     def replace(match):
         return ESCAPE_DCT[match.group(0)]
-    return '"' + ESCAPE.sub(replace, s) + '"'
+    return ESCAPE.sub(replace, s)
 
-
-def py_encode_basestring_ascii(s):
+def encode_basestring_ascii(s):
     """Return an ASCII-only JSON representation of a Python string
 
     """
@@ -53,20 +44,18 @@
         except KeyError:
             n = ord(s)
             if n < 0x10000:
-                return '\\u{0:04x}'.format(n)
-                #return '\\u%04x' % (n,)
+                return '\\u%04x' % (n,)
             else:
                 # surrogate pair
                 n -= 0x10000
                 s1 = 0xd800 | ((n >> 10) & 0x3ff)
                 s2 = 0xdc00 | (n & 0x3ff)
-                return '\\u{0:04x}\\u{1:04x}'.format(s1, s2)
-                #return '\\u%04x\\u%04x' % (s1, s2)
-    return '"' + str(ESCAPE_ASCII.sub(replace, s)) + '"'
-
-
-encode_basestring_ascii = (
-    c_encode_basestring_ascii or py_encode_basestring_ascii)
+                return '\\u%04x\\u%04x' % (s1, s2)
+    if ESCAPE_ASCII.search(s):
+        return str(ESCAPE_ASCII.sub(replace, s))
+    return s
+py_encode_basestring_ascii = lambda s: '"' + encode_basestring_ascii(s) + '"'
+c_encode_basestring_ascii = None
 
 class JSONEncoder(object):
     """Extensible JSON <http://json.org> encoder for Python data structures.
@@ -147,6 +136,17 @@
 
         self.skipkeys = skipkeys
         self.ensure_ascii = ensure_ascii
+        if ensure_ascii:
+            self.encoder = encode_basestring_ascii
+        else:
+            self.encoder = encode_basestring
+        if encoding != 'utf-8':
+            orig_encoder = self.encoder
+            def encoder(o):
+                if isinstance(o, str):
+                    o = o.decode(encoding)
+                return orig_encoder(o)
+            self.encoder = encoder
         self.check_circular = check_circular
         self.allow_nan = allow_nan
         self.sort_keys = sort_keys
@@ -184,24 +184,126 @@
         '{"foo": ["bar", "baz"]}'
 
         """
-        # This is for extremely simple cases and benchmarks.
+        if self.check_circular:
+            markers = {}
+        else:
+            markers = None
+        if self.ensure_ascii:
+            builder = StringBuilder()
+        else:
+            builder = UnicodeBuilder()
+        self._encode(o, markers, builder, 0)
+        return builder.build()
+
+    def _emit_indent(self, builder, _current_indent_level):
+        if self.indent is not None:
+            _current_indent_level += 1
+            newline_indent = '\n' + (' ' * (self.indent *
+                                            _current_indent_level))
+            separator = self.item_separator + newline_indent
+            builder.append(newline_indent)
+        else:
+            separator = self.item_separator
+        return separator, _current_indent_level
+
+    def _emit_unindent(self, builder, _current_indent_level):
+        if self.indent is not None:
+            builder.append('\n')
+            builder.append(' ' * (self.indent * (_current_indent_level - 1)))
+
+    def _encode(self, o, markers, builder, _current_indent_level):
         if isinstance(o, basestring):
-            if isinstance(o, str):
-                _encoding = self.encoding
-                if (_encoding is not None
-                        and not (_encoding == 'utf-8')):
-                    o = o.decode(_encoding)
-            if self.ensure_ascii:
-                return encode_basestring_ascii(o)
+            builder.append('"')
+            builder.append(self.encoder(o))
+            builder.append('"')
+        elif o is None:
+            builder.append('null')
+        elif o is True:
+            builder.append('true')
+        elif o is False:
+            builder.append('false')
+        elif isinstance(o, (int, long)):
+            builder.append(str(o))
+        elif isinstance(o, float):
+            builder.append(self._floatstr(o))
+        elif isinstance(o, (list, tuple)):
+            if not o:
+                builder.append('[]')
+                return
+            self._encode_list(o, markers, builder, _current_indent_level)
+        elif isinstance(o, dict):
+            if not o:
+                builder.append('{}')
+                return
+            self._encode_dict(o, markers, builder, _current_indent_level)
+        else:
+            self._mark_markers(markers, o)
+            res = self.default(o)
+            self._encode(res, markers, builder, _current_indent_level)
+            self._remove_markers(markers, o)
+            return res
+
+    def _encode_list(self, l, markers, builder, _current_indent_level):
+        self._mark_markers(markers, l)
+        builder.append('[')
+        first = True
+        separator, _current_indent_level = self._emit_indent(builder,
+                                                      _current_indent_level)
+        for elem in l:
+            if first:
+                first = False
             else:
-                return encode_basestring(o)
-        # This doesn't pass the iterator directly to ''.join() because the
-        # exceptions aren't as detailed.  The list call should be roughly
-        # equivalent to the PySequence_Fast that ''.join() would do.
-        chunks = self.iterencode(o, _one_shot=True)
-        if not isinstance(chunks, (list, tuple)):
-            chunks = list(chunks)
-        return ''.join(chunks)
+                builder.append(separator)
+            self._encode(elem, markers, builder, _current_indent_level)
+            del elem # XXX grumble
+        self._emit_unindent(builder, _current_indent_level)
+        builder.append(']')
+        self._remove_markers(markers, l)
+
+    def _encode_dict(self, d, markers, builder, _current_indent_level):
+        self._mark_markers(markers, d)
+        first = True
+        builder.append('{')
+        separator, _current_indent_level = self._emit_indent(builder,
+                                                         _current_indent_level)
+        if self.sort_keys:
+            items = sorted(d.items(), key=lambda kv: kv[0])
+        else:
+            items = d.iteritems()
+
+        for key, v in items:
+            if first:
+                first = False
+            else:
+                builder.append(separator)
+            if isinstance(key, basestring):
+                pass
+            # JavaScript is weakly typed for these, so it makes sense to
+            # also allow them.  Many encoders seem to do something like this.
+            elif isinstance(key, float):
+                key = self._floatstr(key)
+            elif key is True:
+                key = 'true'
+            elif key is False:
+                key = 'false'
+            elif key is None:
+                key = 'null'
+            elif isinstance(key, (int, long)):
+                key = str(key)
+            elif self.skipkeys:
+                continue
+            else:
+                raise TypeError("key " + repr(key) + " is not a string")
+            builder.append('"')
+            builder.append(self.encoder(key))
+            builder.append('"')
+            builder.append(self.key_separator)
+            self._encode(v, markers, builder, _current_indent_level)
+            del key
+            del v # XXX grumble
+        self._emit_unindent(builder, _current_indent_level)
+        builder.append('}')
+        self._remove_markers(markers, d)
 
     def iterencode(self, o, _one_shot=False):
         """Encode the given object and yield each string
@@ -217,86 +319,54 @@
             markers = {}
         else:
             markers = None
-        if self.ensure_ascii:
-            _encoder = encode_basestring_ascii
+        return self._iterencode(o, markers, 0)
+
+    def _floatstr(self, o):
+        # Check for specials.  Note that this type of test is processor
+        # and/or platform-specific, so do tests which don't depend on the
+        # internals.
+
+        if o != o:
+            text = 'NaN'
+        elif o == INFINITY:
+            text = 'Infinity'
+        elif o == -INFINITY:
+            text = '-Infinity'
         else:
-            _encoder = encode_basestring
-        if self.encoding != 'utf-8':
-            def _encoder(o, _orig_encoder=_encoder, _encoding=self.encoding):
-                if isinstance(o, str):
-                    o = o.decode(_encoding)
-                return _orig_encoder(o)
+            return FLOAT_REPR(o)
 
-        def floatstr(o, allow_nan=self.allow_nan,
-                _repr=FLOAT_REPR, _inf=INFINITY, _neginf=-INFINITY):
-            # Check for specials.  Note that this type of test is processor
-            # and/or platform-specific, so do tests which don't depend on the
-            # internals.
+        if not self.allow_nan:
+            raise ValueError(
+                "Out of range float values are not JSON compliant: " +
+                repr(o))
 
-            if o != o:
-                text = 'NaN'
-            elif o == _inf:
-                text = 'Infinity'
-            elif o == _neginf:
-                text = '-Infinity'
-            else:
-                return _repr(o)
+        return text
 
-            if not allow_nan:
-                raise ValueError(
-                    "Out of range float values are not JSON compliant: " +
-                    repr(o))
+    def _mark_markers(self, markers, o):
+        if markers is not None:
+            if id(o) in markers:
+                raise ValueError("Circular reference detected")
+            markers[id(o)] = None
 
-            return text
+    def _remove_markers(self, markers, o):
+        if markers is not None:
+            del markers[id(o)]
 
-
-        if (_one_shot and c_make_encoder is not None
-                and not self.indent and not self.sort_keys):
-            _iterencode = c_make_encoder(
-                markers, self.default, _encoder, self.indent,
-                self.key_separator, self.item_separator, self.sort_keys,
-                self.skipkeys, self.allow_nan)
-        else:
-            _iterencode = _make_iterencode(
-                markers, self.default, _encoder, self.indent, floatstr,
-                self.key_separator, self.item_separator, self.sort_keys,
-                self.skipkeys, _one_shot)
-        return _iterencode(o, 0)
-
-def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,
-        _key_separator, _item_separator, _sort_keys, _skipkeys, _one_shot,
-        ## HACK: hand-optimized bytecode; turn globals into locals
-        ValueError=ValueError,
-        basestring=basestring,
-        dict=dict,
-        float=float,
-        id=id,
-        int=int,
-        isinstance=isinstance,
-        list=list,
-        long=long,
-        str=str,
-        tuple=tuple,
-    ):
-
-    def _iterencode_list(lst, _current_indent_level):
+    def _iterencode_list(self, lst, markers, _current_indent_level):
         if not lst:
             yield '[]'
             return
-        if markers is not None:
-            markerid = id(lst)
-            if markerid in markers:
-                raise ValueError("Circular reference detected")
-            markers[markerid] = lst
+        self._mark_markers(markers, lst)
         buf = '['
-        if _indent is not None:
+        if self.indent is not None:
             _current_indent_level += 1
-            newline_indent = '\n' + (' ' * (_indent * _current_indent_level))
-            separator = _item_separator + newline_indent
+            newline_indent = '\n' + (' ' * (self.indent *
+                                            _current_indent_level))
+            separator = self.item_separator + newline_indent
             buf += newline_indent
         else:
             newline_indent = None
-            separator = _item_separator
+            separator = self.item_separator
         first = True
         for value in lst:
             if first:
@@ -304,7 +374,7 @@
             else:
                 buf = separator
             if isinstance(value, basestring):
-                yield buf + _encoder(value)
+                yield buf + '"' + self.encoder(value) + '"'
             elif value is None:
                 yield buf + 'null'
             elif value is True:
@@ -314,44 +384,43 @@
             elif isinstance(value, (int, long)):
                 yield buf + str(value)
             elif isinstance(value, float):
-                yield buf + _floatstr(value)
+                yield buf + self._floatstr(value)
             else:
                 yield buf
                 if isinstance(value, (list, tuple)):
-                    chunks = _iterencode_list(value, _current_indent_level)
+                    chunks = self._iterencode_list(value, markers,
+                                                   _current_indent_level)
                 elif isinstance(value, dict):
-                    chunks = _iterencode_dict(value, _current_indent_level)
+                    chunks = self._iterencode_dict(value, markers,
+                                                   _current_indent_level)
                 else:
-                    chunks = _iterencode(value, _current_indent_level)
+                    chunks = self._iterencode(value, markers,
+                                              _current_indent_level)
                 for chunk in chunks:
                     yield chunk
         if newline_indent is not None:
             _current_indent_level -= 1
-            yield '\n' + (' ' * (_indent * _current_indent_level))
+            yield '\n' + (' ' * (self.indent * _current_indent_level))
         yield ']'
-        if markers is not None:
-            del markers[markerid]
+        self._remove_markers(markers, lst)
 
-    def _iterencode_dict(dct, _current_indent_level):
+    def _iterencode_dict(self, dct, markers, _current_indent_level):
         if not dct:
             yield '{}'
             return
-        if markers is not None:
-            markerid = id(dct)
-            if markerid in markers:
-                raise ValueError("Circular reference detected")
-            markers[markerid] = dct
+        self._mark_markers(markers, dct)
         yield '{'
-        if _indent is not None:
+        if self.indent is not None:
             _current_indent_level += 1
-            newline_indent = '\n' + (' ' * (_indent * _current_indent_level))
-            item_separator = _item_separator + newline_indent
+            newline_indent = '\n' + (' ' * (self.indent *
+                                            _current_indent_level))
+            item_separator = self.item_separator + newline_indent
             yield newline_indent
         else:
             newline_indent = None
-            item_separator = _item_separator
+            item_separator = self.item_separator
         first = True
-        if _sort_keys:
+        if self.sort_keys:
             items = sorted(dct.items(), key=lambda kv: kv[0])
         else:
             items = dct.iteritems()
@@ -361,7 +430,7 @@
             # JavaScript is weakly typed for these, so it makes sense to
             # also allow them.  Many encoders seem to do something like this.
             elif isinstance(key, float):
-                key = _floatstr(key)
+                key = self._floatstr(key)
             elif key is True:
                 key = 'true'
             elif key is False:
@@ -370,7 +439,7 @@
                 key = 'null'
             elif isinstance(key, (int, long)):
                 key = str(key)
-            elif _skipkeys:
+            elif self.skipkeys:
                 continue
             else:
                 raise TypeError("key " + repr(key) + " is not a string")
@@ -378,10 +447,10 @@
                 first = False
             else:
                 yield item_separator
-            yield _encoder(key)
-            yield _key_separator
+            yield '"' + self.encoder(key) + '"'
+            yield self.key_separator
             if isinstance(value, basestring):
-                yield _encoder(value)
+                yield '"' + self.encoder(value) + '"'
             elif value is None:
                 yield 'null'
             elif value is True:
@@ -391,26 +460,28 @@
             elif isinstance(value, (int, long)):
                 yield str(value)
             elif isinstance(value, float):
-                yield _floatstr(value)
+                yield self._floatstr(value)
             else:
                 if isinstance(value, (list, tuple)):
-                    chunks = _iterencode_list(value, _current_indent_level)
+                    chunks = self._iterencode_list(value, markers,
+                                                   _current_indent_level)
                 elif isinstance(value, dict):
-                    chunks = _iterencode_dict(value, _current_indent_level)
+                    chunks = self._iterencode_dict(value, markers,
+                                                   _current_indent_level)
                 else:
-                    chunks = _iterencode(value, _current_indent_level)
+                    chunks = self._iterencode(value, markers,
+                                              _current_indent_level)
                 for chunk in chunks:
                     yield chunk
         if newline_indent is not None:
             _current_indent_level -= 1
-            yield '\n' + (' ' * (_indent * _current_indent_level))
+            yield '\n' + (' ' * (self.indent * _current_indent_level))
         yield '}'
-        if markers is not None:
-            del markers[markerid]
+        self._remove_markers(markers, dct)
 
-    def _iterencode(o, _current_indent_level):
+    def _iterencode(self, o, markers, _current_indent_level):
         if isinstance(o, basestring):
-            yield _encoder(o)
+            yield '"' + self.encoder(o) + '"'
         elif o is None:
             yield 'null'
         elif o is True:
@@ -420,23 +491,19 @@
         elif isinstance(o, (int, long)):
             yield str(o)
         elif isinstance(o, float):
-            yield _floatstr(o)
+            yield self._floatstr(o)
         elif isinstance(o, (list, tuple)):
-            for chunk in _iterencode_list(o, _current_indent_level):
+            for chunk in self._iterencode_list(o, markers,
+                                               _current_indent_level):
                 yield chunk
         elif isinstance(o, dict):
-            for chunk in _iterencode_dict(o, _current_indent_level):
+            for chunk in self._iterencode_dict(o, markers,
+                                               _current_indent_level):
                 yield chunk
         else:
-            if markers is not None:
-                markerid = id(o)
-                if markerid in markers:
-                    raise ValueError("Circular reference detected")
-                markers[markerid] = o
-            o = _default(o)
-            for chunk in _iterencode(o, _current_indent_level):
+            self._mark_markers(markers, o)
+            obj = self.default(o)
+            for chunk in self._iterencode(obj, markers,
+                                          _current_indent_level):
                 yield chunk
-            if markers is not None:
-                del markers[markerid]
-
-    return _iterencode
+            self._remove_markers(markers, o)
diff --git a/lib-python/modified-2.7/json/tests/test_unicode.py b/lib-python/modified-2.7/json/tests/test_unicode.py
--- a/lib-python/modified-2.7/json/tests/test_unicode.py
+++ b/lib-python/modified-2.7/json/tests/test_unicode.py
@@ -80,3 +80,9 @@
         self.assertEqual(type(json.loads(u'["a"]')[0]), unicode)
         # Issue 10038.
         self.assertEqual(type(json.loads('"foo"')), unicode)
+
+    def test_encode_not_utf_8(self):
+        self.assertEqual(json.dumps('\xb1\xe6', encoding='iso8859-2'),
+                         '"\\u0105\\u0107"')
+        self.assertEqual(json.dumps(['\xb1\xe6'], encoding='iso8859-2'),
+                         '["\\u0105\\u0107"]')
diff --git a/lib-python/modified-2.7/test/test_array.py b/lib-python/modified-2.7/test/test_array.py
--- a/lib-python/modified-2.7/test/test_array.py
+++ b/lib-python/modified-2.7/test/test_array.py
@@ -295,9 +295,10 @@
         )
 
         b = array.array(self.badtypecode())
-        self.assertRaises(TypeError, "a + b")
-
-        self.assertRaises(TypeError, "a + 'bad'")
+        with self.assertRaises(TypeError):
+            a + b
+        with self.assertRaises(TypeError):
+            a + 'bad'
 
     def test_iadd(self):
         a = array.array(self.typecode, self.example[::-1])
@@ -316,9 +317,10 @@
         )
 
         b = array.array(self.badtypecode())
-        self.assertRaises(TypeError, "a += b")
-
-        self.assertRaises(TypeError, "a += 'bad'")
+        with self.assertRaises(TypeError):
+            a += b
+        with self.assertRaises(TypeError):
+            a += 'bad'
 
     def test_mul(self):
         a = 5*array.array(self.typecode, self.example)
@@ -345,7 +347,8 @@
             array.array(self.typecode)
         )
 
-        self.assertRaises(TypeError, "a * 'bad'")
+        with self.assertRaises(TypeError):
+            a * 'bad'
 
     def test_imul(self):
         a = array.array(self.typecode, self.example)
@@ -374,7 +377,8 @@
         a *= -1
         self.assertEqual(a, array.array(self.typecode))
 
-        self.assertRaises(TypeError, "a *= 'bad'")
+        with self.assertRaises(TypeError):
+            a *= 'bad'
 
     def test_getitem(self):
         a = array.array(self.typecode, self.example)
diff --git a/lib-python/modified-2.7/test/test_heapq.py b/lib-python/modified-2.7/test/test_heapq.py
--- a/lib-python/modified-2.7/test/test_heapq.py
+++ b/lib-python/modified-2.7/test/test_heapq.py
@@ -186,6 +186,11 @@
         self.assertFalse(sys.modules['heapq'] is self.module)
         self.assertTrue(hasattr(self.module.heapify, 'func_code'))
 
+    def test_islice_protection(self):
+        m = self.module
+        self.assertFalse(m.nsmallest(-1, [1]))
+        self.assertFalse(m.nlargest(-1, [1]))
+
 
 class TestHeapC(TestHeap):
     module = c_heapq
diff --git a/lib-python/modified-2.7/test/test_urllib2.py b/lib-python/modified-2.7/test/test_urllib2.py
--- a/lib-python/modified-2.7/test/test_urllib2.py
+++ b/lib-python/modified-2.7/test/test_urllib2.py
@@ -307,6 +307,9 @@
     def getresponse(self):
         return MockHTTPResponse(MockFile(), {}, 200, "OK")
 
+    def close(self):
+        pass
+
 class MockHandler:
     # useful for testing handler machinery
     # see add_ordered_mock_handlers() docstring
diff --git a/lib-python/modified-2.7/urllib2.py b/lib-python/modified-2.7/urllib2.py
new file mode 100644
--- /dev/null
+++ b/lib-python/modified-2.7/urllib2.py
@@ -0,0 +1,1436 @@
+"""An extensible library for opening URLs using a variety of protocols
+
+The simplest way to use this module is to call the urlopen function,
+which accepts a string containing a URL or a Request object (described
+below).  It opens the URL and returns the results as file-like
+object; the returned object has some extra methods described below.
+
+The OpenerDirector manages a collection of Handler objects that do
+all the actual work.  Each Handler implements a particular protocol or
+option.  The OpenerDirector is a composite object that invokes the
+Handlers needed to open the requested URL.  For example, the
+HTTPHandler performs HTTP GET and POST requests and deals with
+non-error returns.  The HTTPRedirectHandler automatically deals with
+HTTP 301, 302, 303 and 307 redirect errors, and the HTTPDigestAuthHandler
+deals with digest authentication.
+
+urlopen(url, data=None) -- Basic usage is the same as original
+urllib.  pass the url and optionally data to post to an HTTP URL, and
+get a file-like object back.  One difference is that you can also pass
+a Request instance instead of URL.  Raises a URLError (subclass of
+IOError); for HTTP errors, raises an HTTPError, which can also be
+treated as a valid response.
+
+build_opener -- Function that creates a new OpenerDirector instance.
+Will install the default handlers.  Accepts one or more Handlers as
+arguments, either instances or Handler classes that it will
+instantiate.  If one of the argument is a subclass of the default
+handler, the argument will be installed instead of the default.
+
+install_opener -- Installs a new opener as the default opener.
+
+objects of interest:
+
+OpenerDirector -- Sets up the User Agent as the Python-urllib client and manages
+the Handler classes, while dealing with requests and responses.
+
+Request -- An object that encapsulates the state of a request.  The
+state can be as simple as the URL.  It can also include extra HTTP
+headers, e.g. a User-Agent.
+
+BaseHandler --
+
+exceptions:
+URLError -- A subclass of IOError, individual protocols have their own
+specific subclass.
+
+HTTPError -- Also a valid HTTP response, so you can treat an HTTP error
+as an exceptional event or valid response.
+
+internals:
+BaseHandler and parent
+_call_chain conventions
+
+Example usage:
+
+import urllib2
+
+# set up authentication info
+authinfo = urllib2.HTTPBasicAuthHandler()
+authinfo.add_password(realm='PDQ Application',
+                      uri='https://mahler:8092/site-updates.py',
+                      user='klem',
+                      passwd='geheim$parole')
+
+proxy_support = urllib2.ProxyHandler({"http" : "http://ahad-haam:3128"})
+
+# build a new opener that adds authentication and caching FTP handlers
+opener = urllib2.build_opener(proxy_support, authinfo, urllib2.CacheFTPHandler)
+
+# install it
+urllib2.install_opener(opener)
+
+f = urllib2.urlopen('http://www.python.org/')
+
+
+"""
+
+# XXX issues:
+# If an authentication error handler that tries to perform
+# authentication for some reason but fails, how should the error be
+# signalled?  The client needs to know the HTTP error code.  But if
+# the handler knows that the problem was, e.g., that it didn't know
+# that hash algo that requested in the challenge, it would be good to
+# pass that information along to the client, too.
+# ftp errors aren't handled cleanly
+# check digest against correct (i.e. non-apache) implementation
+
+# Possible extensions:
+# complex proxies  XXX not sure what exactly was meant by this
+# abstract factory for opener
+
+import base64
+import hashlib
+import httplib
+import mimetools
+import os
+import posixpath
+import random
+import re
+import socket
+import sys
+import time
+import urlparse
+import bisect
+
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from StringIO import StringIO
+
+from urllib import (unwrap, unquote, splittype, splithost, quote,
+     addinfourl, splitport, splittag,
+     splitattr, ftpwrapper, splituser, splitpasswd, splitvalue)
+
+# support for FileHandler, proxies via environment variables
+from urllib import localhost, url2pathname, getproxies, proxy_bypass
+
+# used in User-Agent header sent
+__version__ = sys.version[:3]
+
+_opener = None
+def urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
+    global _opener
+    if _opener is None:
+        _opener = build_opener()
+    return _opener.open(url, data, timeout)
+
+def install_opener(opener):
+    global _opener
+    _opener = opener
+
+# do these error classes make sense?
+# make sure all of the IOError stuff is overridden.  we just want to be
+# subtypes.
+
+class URLError(IOError):
+    # URLError is a sub-type of IOError, but it doesn't share any of
+    # the implementation.  need to override __init__ and __str__.
+    # It sets self.args for compatibility with other EnvironmentError
+    # subclasses, but args doesn't have the typical format with errno in
+    # slot 0 and strerror in slot 1.  This may be better than nothing.
+    def __init__(self, reason):
+        self.args = reason,
+        self.reason = reason
+
+    def __str__(self):
+        return '<urlopen error %s>' % self.reason
+
+class HTTPError(URLError, addinfourl):
+    """Raised when HTTP error occurs, but also acts like non-error return"""
+    __super_init = addinfourl.__init__
+
+    def __init__(self, url, code, msg, hdrs, fp):
+        self.code = code
+        self.msg = msg
+        self.hdrs = hdrs
+        self.fp = fp
+        self.filename = url
+        # The addinfourl classes depend on fp being a valid file
+        # object.  In some cases, the HTTPError may not have a valid
+        # file object.  If this happens, the simplest workaround is to
+        # not initialize the base classes.
+        if fp is not None:
+            self.__super_init(fp, hdrs, url, code)
+
+    def __str__(self):
+        return 'HTTP Error %s: %s' % (self.code, self.msg)
+
+# copied from cookielib.py
+_cut_port_re = re.compile(r":\d+$")
+def request_host(request):
+    """Return request-host, as defined by RFC 2965.
+
+    Variation from RFC: returned value is lowercased, for convenient
+    comparison.
+
+    """
+    url = request.get_full_url()
+    host = urlparse.urlparse(url)[1]
+    if host == "":
+        host = request.get_header("Host", "")
+
+    # remove port, if present
+    host = _cut_port_re.sub("", host, 1)
+    return host.lower()
+
+class Request:
+
+    def __init__(self, url, data=None, headers={},
+                 origin_req_host=None, unverifiable=False):
+        # unwrap('<URL:type://host/path>') --> 'type://host/path'
+        self.__original = unwrap(url)
+        self.__original, fragment = splittag(self.__original)
+        self.type = None
+        # self.__r_type is what's left after doing the splittype
+        self.host = None
+        self.port = None
+        self._tunnel_host = None
+        self.data = data
+        self.headers = {}
+        for key, value in headers.items():
+            self.add_header(key, value)
+        self.unredirected_hdrs = {}
+        if origin_req_host is None:
+            origin_req_host = request_host(self)
+        self.origin_req_host = origin_req_host
+        self.unverifiable = unverifiable
+
+    def __getattr__(self, attr):
+        # XXX this is a fallback mechanism to guard against these
+        # methods getting called in a non-standard order.  this may be
+        # too complicated and/or unnecessary.
+        # XXX should the __r_XXX attributes be public?
+        if attr[:12] == '_Request__r_':
+            name = attr[12:]
+            if hasattr(Request, 'get_' + name):
+                getattr(self, 'get_' + name)()
+                return getattr(self, attr)
+        raise AttributeError, attr
+
+    def get_method(self):
+        if self.has_data():
+            return "POST"
+        else:
+            return "GET"
+
+    # XXX these helper methods are lame
+
+    def add_data(self, data):
+        self.data = data
+
+    def has_data(self):
+        return self.data is not None
+
+    def get_data(self):
+        return self.data
+
+    def get_full_url(self):
+        return self.__original
+
+    def get_type(self):
+        if self.type is None:
+            self.type, self.__r_type = splittype(self.__original)
+            if self.type is None:
+                raise ValueError, "unknown url type: %s" % self.__original
+        return self.type
+
+    def get_host(self):
+        if self.host is None:
+            self.host, self.__r_host = splithost(self.__r_type)
+            if self.host:
+                self.host = unquote(self.host)
+        return self.host
+
+    def get_selector(self):
+        return self.__r_host
+
+    def set_proxy(self, host, type):
+        if self.type == 'https' and not self._tunnel_host:
+            self._tunnel_host = self.host
+        else:
+            self.type = type
+            self.__r_host = self.__original
+
+        self.host = host
+
+    def has_proxy(self):
+        return self.__r_host == self.__original
+
+    def get_origin_req_host(self):
+        return self.origin_req_host
+
+    def is_unverifiable(self):
+        return self.unverifiable
+
+    def add_header(self, key, val):
+        # useful for something like authentication
+        self.headers[key.capitalize()] = val
+
+    def add_unredirected_header(self, key, val):
+        # will not be added to a redirected request
+        self.unredirected_hdrs[key.capitalize()] = val
+
+    def has_header(self, header_name):
+        return (header_name in self.headers or
+                header_name in self.unredirected_hdrs)
+
+    def get_header(self, header_name, default=None):
+        return self.headers.get(
+            header_name,
+            self.unredirected_hdrs.get(header_name, default))
+
+    def header_items(self):
+        hdrs = self.unredirected_hdrs.copy()
+        hdrs.update(self.headers)
+        return hdrs.items()
+
+class OpenerDirector:
+    def __init__(self):
+        client_version = "Python-urllib/%s" % __version__
+        self.addheaders = [('User-agent', client_version)]
+        # manage the individual handlers
+        self.handlers = []
+        self.handle_open = {}
+        self.handle_error = {}
+        self.process_response = {}
+        self.process_request = {}
+
+    def add_handler(self, handler):
+        if not hasattr(handler, "add_parent"):
+            raise TypeError("expected BaseHandler instance, got %r" %
+                            type(handler))
+
+        added = False
+        for meth in dir(handler):
+            if meth in ["redirect_request", "do_open", "proxy_open"]:
+                # oops, coincidental match
+                continue
+
+            i = meth.find("_")
+            protocol = meth[:i]
+            condition = meth[i+1:]
+
+            if condition.startswith("error"):
+                j = condition.find("_") + i + 1
+                kind = meth[j+1:]
+                try:
+                    kind = int(kind)
+                except ValueError:
+                    pass
+                lookup = self.handle_error.get(protocol, {})
+                self.handle_error[protocol] = lookup
+            elif condition == "open":
+                kind = protocol
+                lookup = self.handle_open
+            elif condition == "response":
+                kind = protocol
+                lookup = self.process_response
+            elif condition == "request":
+                kind = protocol
+                lookup = self.process_request
+            else:
+                continue
+
+            handlers = lookup.setdefault(kind, [])
+            if handlers:
+                bisect.insort(handlers, handler)
+            else:
+                handlers.append(handler)
+            added = True
+
+        if added:
+            # the handlers must work in an specific order, the order
+            # is specified in a Handler attribute
+            bisect.insort(self.handlers, handler)
+            handler.add_parent(self)
+
+    def close(self):
+        # Only exists for backwards compatibility.
+        pass
+
+    def _call_chain(self, chain, kind, meth_name, *args):
+        # Handlers raise an exception if no one else should try to handle
+        # the request, or return None if they can't but another handler
+        # could.  Otherwise, they return the response.
+        handlers = chain.get(kind, ())
+        for handler in handlers:
+            func = getattr(handler, meth_name)
+
+            result = func(*args)
+            if result is not None:
+                return result
+
+    def open(self, fullurl, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
+        # accept a URL or a Request object
+        if isinstance(fullurl, basestring):
+            req = Request(fullurl, data)
+        else:
+            req = fullurl
+            if data is not None:
+                req.add_data(data)
+
+        req.timeout = timeout
+        protocol = req.get_type()
+
+        # pre-process request
+        meth_name = protocol+"_request"
+        for processor in self.process_request.get(protocol, []):
+            meth = getattr(processor, meth_name)
+            req = meth(req)
+
+        response = self._open(req, data)
+
+        # post-process response
+        meth_name = protocol+"_response"
+        for processor in self.process_response.get(protocol, []):
+            meth = getattr(processor, meth_name)
+            response = meth(req, response)
+
+        return response
+
+    def _open(self, req, data=None):
+        result = self._call_chain(self.handle_open, 'default',
+                                  'default_open', req)
+        if result:
+            return result
+
+        protocol = req.get_type()
+        result = self._call_chain(self.handle_open, protocol, protocol +
+                                  '_open', req)
+        if result:
+            return result
+
+        return self._call_chain(self.handle_open, 'unknown',
+                                'unknown_open', req)
+
+    def error(self, proto, *args):
+        if proto in ('http', 'https'):
+            # XXX http[s] protocols are special-cased
+            dict = self.handle_error['http'] # https is not different than http
+            proto = args[2]  # YUCK!
+            meth_name = 'http_error_%s' % proto
+            http_err = 1
+            orig_args = args
+        else:
+            dict = self.handle_error
+            meth_name = proto + '_error'
+            http_err = 0
+        args = (dict, proto, meth_name) + args
+        result = self._call_chain(*args)
+        if result:
+            return result
+
+        if http_err:
+            args = (dict, 'default', 'http_error_default') + orig_args
+            return self._call_chain(*args)
+
+# XXX probably also want an abstract factory that knows when it makes
+# sense to skip a superclass in favor of a subclass and when it might
+# make sense to include both
+
+def build_opener(*handlers):
+    """Create an opener object from a list of handlers.
+
+    The opener will use several default handlers, including support
+    for HTTP, FTP and when applicable, HTTPS.
+
+    If any of the handlers passed as arguments are subclasses of the
+    default handlers, the default handlers will not be used.
+    """
+    import types
+    def isclass(obj):
+        return isinstance(obj, (types.ClassType, type))
+
+    opener = OpenerDirector()
+    default_classes = [ProxyHandler, UnknownHandler, HTTPHandler,
+                       HTTPDefaultErrorHandler, HTTPRedirectHandler,
+                       FTPHandler, FileHandler, HTTPErrorProcessor]
+    if hasattr(httplib, 'HTTPS'):
+        default_classes.append(HTTPSHandler)
+    skip = set()
+    for klass in default_classes:
+        for check in handlers:
+            if isclass(check):
+                if issubclass(check, klass):
+                    skip.add(klass)
+            elif isinstance(check, klass):
+                skip.add(klass)
+    for klass in skip:
+        default_classes.remove(klass)
+
+    for klass in default_classes:
+        opener.add_handler(klass())
+
+    for h in handlers:
+        if isclass(h):
+            h = h()
+        opener.add_handler(h)
+    return opener
+
+class BaseHandler:
+    handler_order = 500
+
+    def add_parent(self, parent):
+        self.parent = parent
+
+    def close(self):
+        # Only exists for backwards compatibility
+        pass
+
+    def __lt__(self, other):
+        if not hasattr(other, "handler_order"):
+            # Try to preserve the old behavior of having custom classes
+            # inserted after default ones (works only for custom user
+            # classes which are not aware of handler_order).
+            return True
+        return self.handler_order < other.handler_order
+
+
+class HTTPErrorProcessor(BaseHandler):
+    """Process HTTP error responses."""
+    handler_order = 1000  # after all other processing
+
+    def http_response(self, request, response):
+        code, msg, hdrs = response.code, response.msg, response.info()
+
+        # According to RFC 2616, "2xx" code indicates that the client's
+        # request was successfully received, understood, and accepted.
+        if not (200 <= code < 300):
+            response = self.parent.error(
+                'http', request, response, code, msg, hdrs)
+
+        return response
+
+    https_response = http_response
+
+class HTTPDefaultErrorHandler(BaseHandler):
+    def http_error_default(self, req, fp, code, msg, hdrs):
+        raise HTTPError(req.get_full_url(), code, msg, hdrs, fp)
+
+class HTTPRedirectHandler(BaseHandler):
+    # maximum number of redirections to any single URL
+    # this is needed because of the state that cookies introduce
+    max_repeats = 4
+    # maximum total number of redirections (regardless of URL) before
+    # assuming we're in a loop
+    max_redirections = 10
+
+    def redirect_request(self, req, fp, code, msg, headers, newurl):
+        """Return a Request or None in response to a redirect.
+
+        This is called by the http_error_30x methods when a
+        redirection response is received.  If a redirection should
+        take place, return a new Request to allow http_error_30x to
+        perform the redirect.  Otherwise, raise HTTPError if no-one
+        else should try to handle this url.  Return None if you can't
+        but another Handler might.
+        """
+        m = req.get_method()
+        if (code in (301, 302, 303, 307) and m in ("GET", "HEAD")
+            or code in (301, 302, 303) and m == "POST"):
+            # Strictly (according to RFC 2616), 301 or 302 in response
+            # to a POST MUST NOT cause a redirection without confirmation
+            # from the user (of urllib2, in this case).  In practice,
+            # essentially all clients do redirect in this case, so we
+            # do the same.
+            # be conciliant with URIs containing a space
+            newurl = newurl.replace(' ', '%20')
+            newheaders = dict((k,v) for k,v in req.headers.items()
+                              if k.lower() not in ("content-length", "content-type")
+                             )
+            return Request(newurl,
+                           headers=newheaders,
+                           origin_req_host=req.get_origin_req_host(),
+                           unverifiable=True)
+        else:
+            raise HTTPError(req.get_full_url(), code, msg, headers, fp)
+
+    # Implementation note: To avoid the server sending us into an
+    # infinite loop, the request object needs to track what URLs we
+    # have already seen.  Do this by adding a handler-specific
+    # attribute to the Request object.
+    def http_error_302(self, req, fp, code, msg, headers):
+        # Some servers (incorrectly) return multiple Location headers
+        # (so probably same goes for URI).  Use first header.
+        if 'location' in headers:
+            newurl = headers.getheaders('location')[0]
+        elif 'uri' in headers:
+            newurl = headers.getheaders('uri')[0]
+        else:
+            return
+
+        # fix a possible malformed URL
+        urlparts = urlparse.urlparse(newurl)
+        if not urlparts.path:
+            urlparts = list(urlparts)
+            urlparts[2] = "/"
+        newurl = urlparse.urlunparse(urlparts)
+
+        newurl = urlparse.urljoin(req.get_full_url(), newurl)
+
+        # XXX Probably want to forget about the state of the current
+        # request, although that might interact poorly with other
+        # handlers that also use handler-specific request attributes
+        new = self.redirect_request(req, fp, code, msg, headers, newurl)
+        if new is None:
+            return
+
+        # loop detection
+        # .redirect_dict has a key url if url was previously visited.
+        if hasattr(req, 'redirect_dict'):
+            visited = new.redirect_dict = req.redirect_dict
+            if (visited.get(newurl, 0) >= self.max_repeats or
+                len(visited) >= self.max_redirections):
+                raise HTTPError(req.get_full_url(), code,
+                                self.inf_msg + msg, headers, fp)
+        else:
+            visited = new.redirect_dict = req.redirect_dict = {}
+        visited[newurl] = visited.get(newurl, 0) + 1
+
+        # Don't close the fp until we are sure that we won't use it
+        # with HTTPError.
+        fp.read()
+        fp.close()
+
+        return self.parent.open(new, timeout=req.timeout)
+
+    http_error_301 = http_error_303 = http_error_307 = http_error_302
+
+    inf_msg = "The HTTP server returned a redirect error that would " \
+              "lead to an infinite loop.\n" \
+              "The last 30x error message was:\n"
+
+
+def _parse_proxy(proxy):
+    """Return (scheme, user, password, host/port) given a URL or an authority.
+
+    If a URL is supplied, it must have an authority (host:port) component.
+    According to RFC 3986, having an authority component means the URL must
+    have two slashes after the scheme:
+
+    >>> _parse_proxy('file:/ftp.example.com/')
+    Traceback (most recent call last):
+    ValueError: proxy URL with no authority: 'file:/ftp.example.com/'
+
+    The first three items of the returned tuple may be None.
+
+    Examples of authority parsing:
+
+    >>> _parse_proxy('proxy.example.com')
+    (None, None, None, 'proxy.example.com')
+    >>> _parse_proxy('proxy.example.com:3128')
+    (None, None, None, 'proxy.example.com:3128')
+
+    The authority component may optionally include userinfo (assumed to be
+    username:password):
+
+    >>> _parse_proxy('joe:password at proxy.example.com')
+    (None, 'joe', 'password', 'proxy.example.com')
+    >>> _parse_proxy('joe:password at proxy.example.com:3128')
+    (None, 'joe', 'password', 'proxy.example.com:3128')
+
+    Same examples, but with URLs instead:
+
+    >>> _parse_proxy('http://proxy.example.com/')
+    ('http', None, None, 'proxy.example.com')
+    >>> _parse_proxy('http://proxy.example.com:3128/')
+    ('http', None, None, 'proxy.example.com:3128')
+    >>> _parse_proxy('http://joe:password at proxy.example.com/')
+    ('http', 'joe', 'password', 'proxy.example.com')
+    >>> _parse_proxy('http://joe:password at proxy.example.com:3128')
+    ('http', 'joe', 'password', 'proxy.example.com:3128')
+
+    Everything after the authority is ignored:
+
+    >>> _parse_proxy('ftp://joe:password at proxy.example.com/rubbish:3128')
+    ('ftp', 'joe', 'password', 'proxy.example.com')
+
+    Test for no trailing '/' case:
+
+    >>> _parse_proxy('http://joe:password at proxy.example.com')
+    ('http', 'joe', 'password', 'proxy.example.com')
+
+    """
+    scheme, r_scheme = splittype(proxy)
+    if not r_scheme.startswith("/"):
+        # authority
+        scheme = None
+        authority = proxy
+    else:
+        # URL
+        if not r_scheme.startswith("//"):
+            raise ValueError("proxy URL with no authority: %r" % proxy)
+        # We have an authority, so for RFC 3986-compliant URLs (by ss 3.
+        # and 3.3.), path is empty or starts with '/'
+        end = r_scheme.find("/", 2)
+        if end == -1:
+            end = None
+        authority = r_scheme[2:end]
+    userinfo, hostport = splituser(authority)
+    if userinfo is not None:
+        user, password = splitpasswd(userinfo)
+    else:
+        user = password = None
+    return scheme, user, password, hostport
+
+class ProxyHandler(BaseHandler):
+    # Proxies must be in front
+    handler_order = 100
+
+    def __init__(self, proxies=None):
+        if proxies is None:
+            proxies = getproxies()
+        assert hasattr(proxies, 'has_key'), "proxies must be a mapping"
+        self.proxies = proxies
+        for type, url in proxies.items():
+            setattr(self, '%s_open' % type,
+                    lambda r, proxy=url, type=type, meth=self.proxy_open: \
+                    meth(r, proxy, type))
+
+    def proxy_open(self, req, proxy, type):
+        orig_type = req.get_type()
+        proxy_type, user, password, hostport = _parse_proxy(proxy)
+
+        if proxy_type is None:
+            proxy_type = orig_type
+
+        if req.host and proxy_bypass(req.host):
+            return None
+
+        if user and password:
+            user_pass = '%s:%s' % (unquote(user), unquote(password))
+            creds = base64.b64encode(user_pass).strip()
+            req.add_header('Proxy-authorization', 'Basic ' + creds)
+        hostport = unquote(hostport)
+        req.set_proxy(hostport, proxy_type)
+
+        if orig_type == proxy_type or orig_type == 'https':
+            # let other handlers take care of it
+            return None
+        else:
+            # need to start over, because the other handlers don't
+            # grok the proxy's URL type
+            # e.g. if we have a constructor arg proxies like so:
+            # {'http': 'ftp://proxy.example.com'}, we may end up turning
+            # a request for http://acme.example.com/a into one for
+            # ftp://proxy.example.com/a
+            return self.parent.open(req, timeout=req.timeout)
+
+class HTTPPasswordMgr:
+
+    def __init__(self):
+        self.passwd = {}
+
+    def add_password(self, realm, uri, user, passwd):
+        # uri could be a single URI or a sequence
+        if isinstance(uri, basestring):
+            uri = [uri]
+        if not realm in self.passwd:
+            self.passwd[realm] = {}
+        for default_port in True, False:
+            reduced_uri = tuple(
+                [self.reduce_uri(u, default_port) for u in uri])
+            self.passwd[realm][reduced_uri] = (user, passwd)
+
+    def find_user_password(self, realm, authuri):
+        domains = self.passwd.get(realm, {})
+        for default_port in True, False:
+            reduced_authuri = self.reduce_uri(authuri, default_port)
+            for uris, authinfo in domains.iteritems():
+                for uri in uris:
+                    if self.is_suburi(uri, reduced_authuri):
+                        return authinfo
+        return None, None
+
+    def reduce_uri(self, uri, default_port=True):
+        """Accept authority or URI and extract only the authority and path."""
+        # note HTTP URLs do not have a userinfo component
+        parts = urlparse.urlsplit(uri)
+        if parts[1]:
+            # URI
+            scheme = parts[0]
+            authority = parts[1]
+            path = parts[2] or '/'
+        else:
+            # host or host:port
+            scheme = None
+            authority = uri
+            path = '/'
+        host, port = splitport(authority)
+        if default_port and port is None and scheme is not None:
+            dport = {"http": 80,
+                     "https": 443,
+                     }.get(scheme)
+            if dport is not None:
+                authority = "%s:%d" % (host, dport)
+        return authority, path
+
+    def is_suburi(self, base, test):
+        """Check if test is below base in a URI tree
+
+        Both args must be URIs in reduced form.
+        """
+        if base == test:
+            return True
+        if base[0] != test[0]:
+            return False
+        common = posixpath.commonprefix((base[1], test[1]))
+        if len(common) == len(base[1]):
+            return True
+        return False
+
+
+class HTTPPasswordMgrWithDefaultRealm(HTTPPasswordMgr):
+
+    def find_user_password(self, realm, authuri):
+        user, password = HTTPPasswordMgr.find_user_password(self, realm,
+                                                            authuri)
+        if user is not None:
+            return user, password
+        return HTTPPasswordMgr.find_user_password(self, None, authuri)
+
+
+class AbstractBasicAuthHandler:
+
+    # XXX this allows for multiple auth-schemes, but will stupidly pick
+    # the last one with a realm specified.
+
+    # allow for double- and single-quoted realm values
+    # (single quotes are a violation of the RFC, but appear in the wild)
+    rx = re.compile('(?:.*,)*[ \t]*([^ \t]+)[ \t]+'
+                    'realm=(["\'])(.*?)\\2', re.I)
+
+    # XXX could pre-emptively send auth info already accepted (RFC 2617,
+    # end of section 2, and section 1.2 immediately after "credentials"
+    # production).
+
+    def __init__(self, password_mgr=None):
+        if password_mgr is None:
+            password_mgr = HTTPPasswordMgr()
+        self.passwd = password_mgr
+        self.add_password = self.passwd.add_password
+        self.retried = 0
+
+    def reset_retry_count(self):
+        self.retried = 0
+
+    def http_error_auth_reqed(self, authreq, host, req, headers):
+        # host may be an authority (without userinfo) or a URL with an
+        # authority
+        # XXX could be multiple headers
+        authreq = headers.get(authreq, None)
+
+        if self.retried > 5:
+            # retry sending the username:password 5 times before failing.
+            raise HTTPError(req.get_full_url(), 401, "basic auth failed",
+                            headers, None)
+        else:
+            self.retried += 1
+
+        if authreq:
+            mo = AbstractBasicAuthHandler.rx.search(authreq)
+            if mo:
+                scheme, quote, realm = mo.groups()
+                if scheme.lower() == 'basic':
+                    response = self.retry_http_basic_auth(host, req, realm)
+                    if response and response.code != 401:
+                        self.retried = 0
+                    return response
+
+    def retry_http_basic_auth(self, host, req, realm):
+        user, pw = self.passwd.find_user_password(realm, host)
+        if pw is not None:
+            raw = "%s:%s" % (user, pw)
+            auth = 'Basic %s' % base64.b64encode(raw).strip()
+            if req.headers.get(self.auth_header, None) == auth:
+                return None
+            req.add_unredirected_header(self.auth_header, auth)
+            return self.parent.open(req, timeout=req.timeout)
+        else:
+            return None
+
+
+class HTTPBasicAuthHandler(AbstractBasicAuthHandler, BaseHandler):
+
+    auth_header = 'Authorization'
+
+    def http_error_401(self, req, fp, code, msg, headers):
+        url = req.get_full_url()
+        response = self.http_error_auth_reqed('www-authenticate',
+                                              url, req, headers)
+        self.reset_retry_count()
+        return response
+
+
+class ProxyBasicAuthHandler(AbstractBasicAuthHandler, BaseHandler):
+
+    auth_header = 'Proxy-authorization'
+
+    def http_error_407(self, req, fp, code, msg, headers):
+        # http_error_auth_reqed requires that there is no userinfo component in
+        # authority.  Assume there isn't one, since urllib2 does not (and
+        # should not, RFC 3986 s. 3.2.1) support requests for URLs containing
+        # userinfo.
+        authority = req.get_host()
+        response = self.http_error_auth_reqed('proxy-authenticate',
+                                          authority, req, headers)
+        self.reset_retry_count()
+        return response
+
+
+def randombytes(n):
+    """Return n random bytes."""
+    # Use /dev/urandom if it is available.  Fall back to random module
+    # if not.  It might be worthwhile to extend this function to use
+    # other platform-specific mechanisms for getting random bytes.
+    if os.path.exists("/dev/urandom"):
+        f = open("/dev/urandom")
+        s = f.read(n)
+        f.close()
+        return s
+    else:
+        L = [chr(random.randrange(0, 256)) for i in range(n)]
+        return "".join(L)
+
+class AbstractDigestAuthHandler:
+    # Digest authentication is specified in RFC 2617.
+
+    # XXX The client does not inspect the Authentication-Info header
+    # in a successful response.
+
+    # XXX It should be possible to test this implementation against
+    # a mock server that just generates a static set of challenges.
+
+    # XXX qop="auth-int" supports is shaky
+
+    def __init__(self, passwd=None):
+        if passwd is None:
+            passwd = HTTPPasswordMgr()
+        self.passwd = passwd
+        self.add_password = self.passwd.add_password
+        self.retried = 0
+        self.nonce_count = 0
+        self.last_nonce = None
+
+    def reset_retry_count(self):
+        self.retried = 0
+
+    def http_error_auth_reqed(self, auth_header, host, req, headers):
+        authreq = headers.get(auth_header, None)
+        if self.retried > 5:
+            # Don't fail endlessly - if we failed once, we'll probably
+            # fail a second time. Hm. Unless the Password Manager is
+            # prompting for the information. Crap. This isn't great
+            # but it's better than the current 'repeat until recursion
+            # depth exceeded' approach <wink>
+            raise HTTPError(req.get_full_url(), 401, "digest auth failed",
+                            headers, None)
+        else:
+            self.retried += 1
+        if authreq:
+            scheme = authreq.split()[0]
+            if scheme.lower() == 'digest':
+                return self.retry_http_digest_auth(req, authreq)
+
+    def retry_http_digest_auth(self, req, auth):
+        token, challenge = auth.split(' ', 1)
+        chal = parse_keqv_list(parse_http_list(challenge))
+        auth = self.get_authorization(req, chal)
+        if auth:
+            auth_val = 'Digest %s' % auth
+            if req.headers.get(self.auth_header, None) == auth_val:
+                return None
+            req.add_unredirected_header(self.auth_header, auth_val)
+            resp = self.parent.open(req, timeout=req.timeout)
+            return resp
+
+    def get_cnonce(self, nonce):
+        # The cnonce-value is an opaque
+        # quoted string value provided by the client and used by both client
+        # and server to avoid chosen plaintext attacks, to provide mutual
+        # authentication, and to provide some message integrity protection.
+        # This isn't a fabulous effort, but it's probably Good Enough.
+        dig = hashlib.sha1("%s:%s:%s:%s" % (self.nonce_count, nonce, time.ctime(),
+                                            randombytes(8))).hexdigest()
+        return dig[:16]
+
+    def get_authorization(self, req, chal):
+        try:
+            realm = chal['realm']
+            nonce = chal['nonce']
+            qop = chal.get('qop')
+            algorithm = chal.get('algorithm', 'MD5')
+            # mod_digest doesn't send an opaque, even though it isn't
+            # supposed to be optional
+            opaque = chal.get('opaque', None)
+        except KeyError:
+            return None
+
+        H, KD = self.get_algorithm_impls(algorithm)
+        if H is None:
+            return None
+
+        user, pw = self.passwd.find_user_password(realm, req.get_full_url())
+        if user is None:
+            return None
+
+        # XXX not implemented yet
+        if req.has_data():
+            entdig = self.get_entity_digest(req.get_data(), chal)
+        else:
+            entdig = None
+
+        A1 = "%s:%s:%s" % (user, realm, pw)
+        A2 = "%s:%s" % (req.get_method(),
+                        # XXX selector: what about proxies and full urls
+                        req.get_selector())
+        if qop == 'auth':
+            if nonce == self.last_nonce:
+                self.nonce_count += 1
+            else:
+                self.nonce_count = 1
+                self.last_nonce = nonce
+
+            ncvalue = '%08x' % self.nonce_count
+            cnonce = self.get_cnonce(nonce)
+            noncebit = "%s:%s:%s:%s:%s" % (nonce, ncvalue, cnonce, qop, H(A2))
+            respdig = KD(H(A1), noncebit)
+        elif qop is None:
+            respdig = KD(H(A1), "%s:%s" % (nonce, H(A2)))
+        else:
+            # XXX handle auth-int.
+            raise URLError("qop '%s' is not supported." % qop)
+
+        # XXX should the partial digests be encoded too?
+
+        base = 'username="%s", realm="%s", nonce="%s", uri="%s", ' \
+               'response="%s"' % (user, realm, nonce, req.get_selector(),
+                                  respdig)
+        if opaque:
+            base += ', opaque="%s"' % opaque
+        if entdig:
+            base += ', digest="%s"' % entdig
+        base += ', algorithm="%s"' % algorithm
+        if qop:
+            base += ', qop=auth, nc=%s, cnonce="%s"' % (ncvalue, cnonce)
+        return base
+
+    def get_algorithm_impls(self, algorithm):
+        # algorithm should be case-insensitive according to RFC2617
+        algorithm = algorithm.upper()
+        # lambdas assume digest modules are imported at the top level
+        if algorithm == 'MD5':
+            H = lambda x: hashlib.md5(x).hexdigest()
+        elif algorithm == 'SHA':
+            H = lambda x: hashlib.sha1(x).hexdigest()
+        # XXX MD5-sess
+        KD = lambda s, d: H("%s:%s" % (s, d))
+        return H, KD
+
+    def get_entity_digest(self, data, chal):
+        # XXX not implemented yet
+        return None
+
+
+class HTTPDigestAuthHandler(BaseHandler, AbstractDigestAuthHandler):
+    """An authentication protocol defined by RFC 2069
+
+    Digest authentication improves on basic authentication because it
+    does not transmit passwords in the clear.
+    """
+
+    auth_header = 'Authorization'
+    handler_order = 490  # before Basic auth
+
+    def http_error_401(self, req, fp, code, msg, headers):
+        host = urlparse.urlparse(req.get_full_url())[1]
+        retry = self.http_error_auth_reqed('www-authenticate',
+                                           host, req, headers)
+        self.reset_retry_count()
+        return retry
+
+
+class ProxyDigestAuthHandler(BaseHandler, AbstractDigestAuthHandler):
+
+    auth_header = 'Proxy-Authorization'
+    handler_order = 490  # before Basic auth
+
+    def http_error_407(self, req, fp, code, msg, headers):
+        host = req.get_host()
+        retry = self.http_error_auth_reqed('proxy-authenticate',
+                                           host, req, headers)
+        self.reset_retry_count()
+        return retry
+
+class AbstractHTTPHandler(BaseHandler):
+
+    def __init__(self, debuglevel=0):
+        self._debuglevel = debuglevel
+
+    def set_http_debuglevel(self, level):
+        self._debuglevel = level
+
+    def do_request_(self, request):
+        host = request.get_host()
+        if not host:
+            raise URLError('no host given')
+
+        if request.has_data():  # POST
+            data = request.get_data()
+            if not request.has_header('Content-type'):
+                request.add_unredirected_header(
+                    'Content-type',
+                    'application/x-www-form-urlencoded')
+            if not request.has_header('Content-length'):
+                request.add_unredirected_header(
+                    'Content-length', '%d' % len(data))
+
+        sel_host = host
+        if request.has_proxy():
+            scheme, sel = splittype(request.get_selector())
+            sel_host, sel_path = splithost(sel)
+
+        if not request.has_header('Host'):
+            request.add_unredirected_header('Host', sel_host)
+        for name, value in self.parent.addheaders:
+            name = name.capitalize()
+            if not request.has_header(name):
+                request.add_unredirected_header(name, value)
+
+        return request
+
+    def do_open(self, http_class, req):
+        """Return an addinfourl object for the request, using http_class.
+
+        http_class must implement the HTTPConnection API from httplib.
+        The addinfourl return value is a file-like object.  It also
+        has methods and attributes including:
+            - info(): return a mimetools.Message object for the headers
+            - geturl(): return the original request URL
+            - code: HTTP status code
+        """
+        host = req.get_host()
+        if not host:
+            raise URLError('no host given')
+
+        h = http_class(host, timeout=req.timeout) # will parse host:port
+        h.set_debuglevel(self._debuglevel)
+
+        headers = dict(req.unredirected_hdrs)
+        headers.update(dict((k, v) for k, v in req.headers.items()
+                            if k not in headers))
+
+        # We want to make an HTTP/1.1 request, but the addinfourl
+        # class isn't prepared to deal with a persistent connection.
+        # It will try to read all remaining data from the socket,
+        # which will block while the server waits for the next request.
+        # So make sure the connection gets closed after the (only)
+        # request.
+        headers["Connection"] = "close"
+        headers = dict(
+            (name.title(), val) for name, val in headers.items())
+
+        if req._tunnel_host:
+            tunnel_headers = {}
+            proxy_auth_hdr = "Proxy-Authorization"
+            if proxy_auth_hdr in headers:
+                tunnel_headers[proxy_auth_hdr] = headers[proxy_auth_hdr]
+                # Proxy-Authorization should not be sent to origin
+                # server.
+                del headers[proxy_auth_hdr]
+            h.set_tunnel(req._tunnel_host, headers=tunnel_headers)
+
+        try:
+            h.request(req.get_method(), req.get_selector(), req.data, headers)
+            try:
+                r = h.getresponse(buffering=True)
+            except TypeError: #buffering kw not supported
+                r = h.getresponse()
+        except socket.error, err: # XXX what error?
+            h.close()
+            raise URLError(err)
+
+        # Pick apart the HTTPResponse object to get the addinfourl
+        # object initialized properly.
+
+        # Wrap the HTTPResponse object in socket's file object adapter
+        # for Windows.  That adapter calls recv(), so delegate recv()
+        # to read().  This weird wrapping allows the returned object to
+        # have readline() and readlines() methods.
+
+        # XXX It might be better to extract the read buffering code
+        # out of socket._fileobject() and into a base class.
+
+        r.recv = r.read
+        fp = socket._fileobject(r, close=True)
+
+        resp = addinfourl(fp, r.msg, req.get_full_url())
+        resp.code = r.status
+        resp.msg = r.reason
+        return resp
+
+
+class HTTPHandler(AbstractHTTPHandler):
+
+    def http_open(self, req):
+        return self.do_open(httplib.HTTPConnection, req)
+
+    http_request = AbstractHTTPHandler.do_request_
+
+if hasattr(httplib, 'HTTPS'):
+    class HTTPSHandler(AbstractHTTPHandler):
+
+        def https_open(self, req):
+            return self.do_open(httplib.HTTPSConnection, req)
+
+        https_request = AbstractHTTPHandler.do_request_
+
+class HTTPCookieProcessor(BaseHandler):
+    def __init__(self, cookiejar=None):
+        import cookielib
+        if cookiejar is None:
+            cookiejar = cookielib.CookieJar()
+        self.cookiejar = cookiejar
+
+    def http_request(self, request):
+        self.cookiejar.add_cookie_header(request)
+        return request
+
+    def http_response(self, request, response):
+        self.cookiejar.extract_cookies(response, request)
+        return response
+
+    https_request = http_request
+    https_response = http_response
+
+class UnknownHandler(BaseHandler):
+    def unknown_open(self, req):
+        type = req.get_type()
+        raise URLError('unknown url type: %s' % type)
+
+def parse_keqv_list(l):
+    """Parse list of key=value strings where keys are not duplicated."""
+    parsed = {}
+    for elt in l:
+        k, v = elt.split('=', 1)
+        if v[0] == '"' and v[-1] == '"':
+            v = v[1:-1]
+        parsed[k] = v
+    return parsed
+
+def parse_http_list(s):
+    """Parse lists as described by RFC 2068 Section 2.
+
+    In particular, parse comma-separated lists where the elements of
+    the list may include quoted-strings.  A quoted-string could
+    contain a comma.  A non-quoted string could have quotes in the
+    middle.  Neither commas nor quotes count if they are escaped.
+    Only double-quotes count, not single-quotes.
+    """
+    res = []
+    part = ''
+
+    escape = quote = False
+    for cur in s:
+        if escape:
+            part += cur
+            escape = False
+            continue
+        if quote:
+            if cur == '\\':
+                escape = True
+                continue
+            elif cur == '"':
+                quote = False
+            part += cur
+            continue
+
+        if cur == ',':
+            res.append(part)
+            part = ''
+            continue
+
+        if cur == '"':
+            quote = True
+
+        part += cur
+
+    # append last part
+    if part:
+        res.append(part)
+
+    return [part.strip() for part in res]
+
+def _safe_gethostbyname(host):
+    try:
+        return socket.gethostbyname(host)
+    except socket.gaierror:
+        return None
+
+class FileHandler(BaseHandler):
+    # Use local file or FTP depending on form of URL
+    def file_open(self, req):
+        url = req.get_selector()
+        if url[:2] == '//' and url[2:3] != '/' and (req.host and
+                req.host != 'localhost'):
+            req.type = 'ftp'
+            return self.parent.open(req)
+        else:
+            return self.open_local_file(req)
+
+    # names for the localhost
+    names = None
+    def get_names(self):
+        if FileHandler.names is None:
+            try:
+                FileHandler.names = tuple(
+                    socket.gethostbyname_ex('localhost')[2] +
+                    socket.gethostbyname_ex(socket.gethostname())[2])
+            except socket.gaierror:
+                FileHandler.names = (socket.gethostbyname('localhost'),)
+        return FileHandler.names
+
+    # not entirely sure what the rules are here
+    def open_local_file(self, req):
+        import email.utils
+        import mimetypes
+        host = req.get_host()
+        filename = req.get_selector()
+        localfile = url2pathname(filename)
+        try:
+            stats = os.stat(localfile)
+            size = stats.st_size
+            modified = email.utils.formatdate(stats.st_mtime, usegmt=True)
+            mtype = mimetypes.guess_type(filename)[0]
+            headers = mimetools.Message(StringIO(
+                'Content-type: %s\nContent-length: %d\nLast-modified: %s\n' %
+                (mtype or 'text/plain', size, modified)))
+            if host:
+                host, port = splitport(host)
+            if not host or \
+                (not port and _safe_gethostbyname(host) in self.get_names()):
+                if host:
+                    origurl = 'file://' + host + filename
+                else:
+                    origurl = 'file://' + filename
+                return addinfourl(open(localfile, 'rb'), headers, origurl)
+        except OSError, msg:
+            # urllib2 users shouldn't expect OSErrors coming from urlopen()
+            raise URLError(msg)
+        raise URLError('file not on local host')
+
+class FTPHandler(BaseHandler):
+    def ftp_open(self, req):
+        import ftplib
+        import mimetypes
+        host = req.get_host()
+        if not host:
+            raise URLError('ftp error: no host given')
+        host, port = splitport(host)
+        if port is None:
+            port = ftplib.FTP_PORT
+        else:
+            port = int(port)
+
+        # username/password handling
+        user, host = splituser(host)
+        if user:
+            user, passwd = splitpasswd(user)
+        else:
+            passwd = None
+        host = unquote(host)
+        user = user or ''
+        passwd = passwd or ''
+
+        try:
+            host = socket.gethostbyname(host)
+        except socket.error, msg:
+            raise URLError(msg)
+        path, attrs = splitattr(req.get_selector())
+        dirs = path.split('/')
+        dirs = map(unquote, dirs)
+        dirs, file = dirs[:-1], dirs[-1]
+        if dirs and not dirs[0]:
+            dirs = dirs[1:]
+        try:
+            fw = self.connect_ftp(user, passwd, host, port, dirs, req.timeout)
+            type = file and 'I' or 'D'
+            for attr in attrs:
+                attr, value = splitvalue(attr)
+                if attr.lower() == 'type' and \
+                   value in ('a', 'A', 'i', 'I', 'd', 'D'):
+                    type = value.upper()
+            fp, retrlen = fw.retrfile(file, type)
+            headers = ""
+            mtype = mimetypes.guess_type(req.get_full_url())[0]
+            if mtype:
+                headers += "Content-type: %s\n" % mtype
+            if retrlen is not None and retrlen >= 0:
+                headers += "Content-length: %d\n" % retrlen
+            sf = StringIO(headers)
+            headers = mimetools.Message(sf)
+            return addinfourl(fp, headers, req.get_full_url())
+        except ftplib.all_errors, msg:
+            raise URLError, ('ftp error: %s' % msg), sys.exc_info()[2]
+
+    def connect_ftp(self, user, passwd, host, port, dirs, timeout):
+        fw = ftpwrapper(user, passwd, host, port, dirs, timeout)
+##        fw.ftp.set_debuglevel(1)
+        return fw
+
+class CacheFTPHandler(FTPHandler):
+    # XXX would be nice to have pluggable cache strategies
+    # XXX this stuff is definitely not thread safe
+    def __init__(self):
+        self.cache = {}
+        self.timeout = {}
+        self.soonest = 0
+        self.delay = 60
+        self.max_conns = 16
+
+    def setTimeout(self, t):
+        self.delay = t
+
+    def setMaxConns(self, m):
+        self.max_conns = m
+
+    def connect_ftp(self, user, passwd, host, port, dirs, timeout):
+        key = user, host, port, '/'.join(dirs), timeout
+        if key in self.cache:
+            self.timeout[key] = time.time() + self.delay
+        else:
+            self.cache[key] = ftpwrapper(user, passwd, host, port, dirs, timeout)
+            self.timeout[key] = time.time() + self.delay
+        self.check_cache()
+        return self.cache[key]
+
+    def check_cache(self):
+        # first check for old ones
+        t = time.time()
+        if self.soonest <= t:
+            for k, v in self.timeout.items():
+                if v < t:
+                    self.cache[k].close()
+                    del self.cache[k]
+                    del self.timeout[k]
+        self.soonest = min(self.timeout.values())
+
+        # then check the size
+        if len(self.cache) == self.max_conns:
+            for k, v in self.timeout.items():
+                if v == self.soonest:
+                    del self.cache[k]
+                    del self.timeout[k]
+                    break
+            self.soonest = min(self.timeout.values())
diff --git a/lib_pypy/pyrepl/readline.py b/lib_pypy/pyrepl/readline.py
--- a/lib_pypy/pyrepl/readline.py
+++ b/lib_pypy/pyrepl/readline.py
@@ -395,9 +395,21 @@
     _wrapper.f_in = f_in
     _wrapper.f_out = f_out
 
-    if hasattr(sys, '__raw_input__'):    # PyPy
-        _old_raw_input = sys.__raw_input__
+    if '__pypy__' in sys.builtin_module_names:    # PyPy
+
+        def _old_raw_input(prompt=''):
+            # sys.__raw_input__() is only called when stdin and stdout are
+            # as expected and are ttys.  If it is the case, then get_reader()
+            # should not really fail in _wrapper.raw_input().  If it still
+            # does, then we will just cancel the redirection and call again
+            # the built-in raw_input().
+            try:
+                del sys.__raw_input__
+            except AttributeError:
+                pass
+            return raw_input(prompt)
         sys.__raw_input__ = _wrapper.raw_input
+
     else:
         # this is not really what readline.c does.  Better than nothing I guess
         import __builtin__
diff --git a/lib_pypy/resource.py b/lib_pypy/resource.py
--- a/lib_pypy/resource.py
+++ b/lib_pypy/resource.py
@@ -7,7 +7,7 @@
 
 from ctypes_support import standard_c_lib as libc
 from ctypes_support import get_errno
-from ctypes import Structure, c_int, c_long, byref, sizeof, POINTER
+from ctypes import Structure, c_int, c_long, byref, POINTER
 from errno import EINVAL, EPERM
 import _structseq
 
@@ -165,7 +165,6 @@
 
 @builtinify
 def getpagesize():
-    pagesize = 0
     if _getpagesize:
         return _getpagesize()
     else:
diff --git a/pypy/annotation/classdef.py b/pypy/annotation/classdef.py
--- a/pypy/annotation/classdef.py
+++ b/pypy/annotation/classdef.py
@@ -276,8 +276,8 @@
         # create the Attribute and do the generalization asked for
         newattr = Attribute(attr, self.bookkeeper)
         if s_value:
-            if newattr.name == 'intval' and getattr(s_value, 'unsigned', False):
-                import pdb; pdb.set_trace()
+            #if newattr.name == 'intval' and getattr(s_value, 'unsigned', False):
+            #    import pdb; pdb.set_trace()
             newattr.s_value = s_value
 
         # keep all subattributes' values
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -72,6 +72,7 @@
     del working_modules['fcntl']  # LOCK_NB not defined
     del working_modules["_minimal_curses"]
     del working_modules["termios"]
+    del working_modules["_multiprocessing"]   # depends on rctime
 
 
 
@@ -127,7 +128,7 @@
 
 pypy_optiondescription = OptionDescription("objspace", "Object Space Options", [
     ChoiceOption("name", "Object Space name",
-                 ["std", "flow", "thunk", "dump", "taint"],
+                 ["std", "flow", "thunk", "dump"],
                  "std",
                  cmdline='--objspace -o'),
 
diff --git a/pypy/doc/__pypy__-module.rst b/pypy/doc/__pypy__-module.rst
--- a/pypy/doc/__pypy__-module.rst
+++ b/pypy/doc/__pypy__-module.rst
@@ -37,29 +37,6 @@
 .. _`thunk object space docs`: objspace-proxies.html#thunk
 .. _`interface section of the thunk object space docs`: objspace-proxies.html#thunk-interface
 
-.. broken:
-
-    Taint Object Space Functionality
-    ================================
-
-    When the taint object space is used (choose with :config:`objspace.name`),
-    the following names are put into ``__pypy__``:
-
-     - ``taint``
-     - ``is_tainted``
-     - ``untaint``
-     - ``taint_atomic``
-     - ``_taint_debug``
-     - ``_taint_look``
-     - ``TaintError``
-
-    Those are all described in the `interface section of the taint object space
-    docs`_.
-
-    For more detailed explanations and examples see the `taint object space docs`_.
-
-    .. _`taint object space docs`: objspace-proxies.html#taint
-    .. _`interface section of the taint object space docs`: objspace-proxies.html#taint-interface
 
 Transparent Proxy Functionality
 ===============================
diff --git a/pypy/doc/config/objspace.name.txt b/pypy/doc/config/objspace.name.txt
--- a/pypy/doc/config/objspace.name.txt
+++ b/pypy/doc/config/objspace.name.txt
@@ -4,7 +4,6 @@
 for normal usage):
 
   * thunk_: The thunk object space adds lazy evaluation to PyPy.
-  * taint_: The taint object space adds soft security features.
   * dump_:  Using this object spaces results in the dumpimp of all operations
     to a log.
 
@@ -12,5 +11,4 @@
 .. _`Object Space Proxies`: ../objspace-proxies.html
 .. _`Standard Object Space`: ../objspace.html#standard-object-space
 .. _thunk: ../objspace-proxies.html#thunk
-.. _taint: ../objspace-proxies.html#taint
 .. _dump: ../objspace-proxies.html#dump
diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst
--- a/pypy/doc/index.rst
+++ b/pypy/doc/index.rst
@@ -309,7 +309,6 @@
 .. _`object space`: objspace.html
 .. _FlowObjSpace: objspace.html#the-flow-object-space 
 .. _`trace object space`: objspace.html#the-trace-object-space 
-.. _`taint object space`: objspace-proxies.html#taint
 .. _`thunk object space`: objspace-proxies.html#thunk
 .. _`transparent proxies`: objspace-proxies.html#tproxy
 .. _`Differences between PyPy and CPython`: cpython_differences.html
diff --git a/pypy/doc/objspace-proxies.rst b/pypy/doc/objspace-proxies.rst
--- a/pypy/doc/objspace-proxies.rst
+++ b/pypy/doc/objspace-proxies.rst
@@ -129,297 +129,6 @@
    function behaves lazily: all calls to it return a thunk object.
 
 
-.. broken right now:
-
-    .. _taint:
-
-    The Taint Object Space
-    ======================
-
-    Motivation
-    ----------
-
-    The Taint Object Space provides a form of security: "tainted objects",
-    inspired by various sources, see [D12.1]_ for a more detailed discussion. 
-
-    The basic idea of this kind of security is not to protect against
-    malicious code but to help with handling and boxing sensitive data. 
-    It covers two kinds of sensitive data: secret data which should not leak, 
-    and untrusted data coming from an external source and that must be 
-    validated before it is used.
-
-    The idea is that, considering a large application that handles these
-    kinds of sensitive data, there are typically only a small number of
-    places that need to explicitly manipulate that sensitive data; all the
-    other places merely pass it around, or do entirely unrelated things.
-
-    Nevertheless, if a large application needs to be reviewed for security,
-    it must be entirely carefully checked, because it is possible that a
-    bug at some apparently unrelated place could lead to a leak of sensitive
-    information in a way that an external attacker could exploit.  For
-    example, if any part of the application provides web services, an
-    attacker might be able to issue unexpected requests with a regular web
-    browser and deduce secret information from the details of the answers he
-    gets.  Another example is the common CGI attack where an attacker sends
-    malformed inputs and causes the CGI script to do unintended things.
-
-    An approach like that of the Taint Object Space allows the small parts
-    of the program that manipulate sensitive data to be explicitly marked.
-    The effect of this is that although these small parts still need a
-    careful security review, the rest of the application no longer does,
-    because even a bug would be unable to leak the information.
-
-    We have implemented a simple two-level model: objects are either
-    regular (untainted), or sensitive (tainted).  Objects are marked as
-    sensitive if they are secret or untrusted, and only declassified at
-    carefully-checked positions (e.g. where the secret data is needed, or
-    after the untrusted data has been fully validated).
-
-    It would be simple to extend the code for more fine-grained scales of
-    secrecy.  For example it is typical in the literature to consider
-    user-specified lattices of secrecy levels, corresponding to multiple
-    "owners" that cannot access data belonging to another "owner" unless
-    explicitly authorized to do so.
-
-    Tainting and untainting
-    -----------------------
-
-    Start a py.py with the Taint Object Space and try the following example::
-
-        $ py.py -o taint
-        >>>> from __pypy__ import taint
-        >>>> x = taint(6)
-
-        # x is hidden from now on.  We can pass it around and
-        # even operate on it, but not inspect it.  Taintness
-        # is propagated to operation results.
-
-        >>>> x
-        TaintError
-
-        >>>> if x > 5: y = 2   # see below
-        TaintError
-
-        >>>> y = x + 5         # ok
-        >>>> lst = [x, y]
-        >>>> z = lst.pop()
-        >>>> t = type(z)       # type() works too, tainted answer
-        >>>> t
-        TaintError
-        >>>> u = t is int      # even 'is' works
-        >>>> u
-        TaintError
-
-    Notice that using a tainted boolean like ``x > 5`` in an ``if``
-    statement is forbidden.  This is because knowing which path is followed
-    would give away a hint about ``x``; in the example above, if the
-    statement ``if x > 5: y = 2`` was allowed to run, we would know
-    something about the value of ``x`` by looking at the (untainted) value
-    in the variable ``y``.
-
-    Of course, there is a way to inspect tainted objects.  The basic way is
-    to explicitly "declassify" it with the ``untaint()`` function.  In an
-    application, the places that use ``untaint()`` are the places that need
-    careful security review.  To avoid unexpected objects showing up, the
-    ``untaint()`` function must be called with the exact type of the object
-    to declassify.  It will raise ``TaintError`` if the type doesn't match::
-
-        >>>> from __pypy__ import taint
-        >>>> untaint(int, x)
-        6
-        >>>> untaint(int, z)
-        11
-        >>>> untaint(bool, x > 5)
-        True
-        >>>> untaint(int, x > 5)
-        TaintError
-
-
-    Taint Bombs
-    -----------
-
-    In this area, a common problem is what to do about failing operations.
-    If an operation raises an exception when manipulating a tainted object,
-    then the very presence of the exception can leak information about the
-    tainted object itself.  Consider::
-
-        >>>> 5 / (x-6)
-
-    By checking if this raises ``ZeroDivisionError`` or not, we would know
-    if ``x`` was equal to 6 or not.  The solution to this problem in the
-    Taint Object Space is to introduce *Taint Bombs*.  They are a kind of
-    tainted object that doesn't contain a real object, but a pending
-    exception.  Taint Bombs are indistinguishable from normal tainted
-    objects to unprivileged code. See::
-
-        >>>> x = taint(6)
-        >>>> i = 5 / (x-6)     # no exception here
-        >>>> j = i + 1         # nor here
-        >>>> k = j + 5         # nor here
-        >>>> untaint(int, k)
-        TaintError
-
-    In the above example, all of ``i``, ``j`` and ``k`` contain a Taint
-    Bomb.  Trying to untaint it raises an exception - a generic
-    ``TaintError``.  What we win is that the exception gives little away,
-    and most importantly it occurs at the point where ``untaint()`` is
-    called, not where the operation failed.  This means that all calls to
-    ``untaint()`` - but not the rest of the code - must be carefully
-    reviewed for what occurs if they receive a Taint Bomb; they might catch
-    the ``TaintError`` and give the user a generic message that something
-    went wrong, if we are reasonably careful that the message or even its
-    presence doesn't give information away.  This might be a
-    problem by itself, but there is no satisfying general solution here:
-    it must be considered on a case-by-case basis.  Again, what the
-    Taint Object Space approach achieves is not solving these problems, but
-    localizing them to well-defined small parts of the application - namely,
-    around calls to ``untaint()``.
-
-    The ``TaintError`` exception deliberately does not include any
-    useful error messages, because they might give information away.
-    Of course, this makes debugging quite a bit harder; a difficult
-    problem to solve properly.  So far we have implemented a way to peek in a Taint
-    Box or Bomb, ``__pypy__._taint_look(x)``, and a "debug mode" that
-    prints the exception as soon as a Bomb is created - both write
-    information to the low-level stderr of the application, where we hope
-    that it is unlikely to be seen by anyone but the application
-    developer.
-
-
-    Taint Atomic functions
-    ----------------------
-
-    Occasionally, a more complicated computation must be performed on a
-    tainted object.  This requires first untainting the object, performing the
-    computations, and then carefully tainting the result again (including
-    hiding all exceptions into Bombs).
-
-    There is a built-in decorator that does this for you::
-
-        >>>> @__pypy__.taint_atomic
-        >>>> def myop(x, y):
-        ....     while x > 0:
-        ....         x -= y
-        ....     return x
-        ....
-        >>>> myop(42, 10)
-        -8
-        >>>> z = myop(taint(42), 10)
-        >>>> z
-        TaintError
-        >>>> untaint(int, z)
-        -8
-
-    The decorator makes a whole function behave like a built-in operation.
-    If no tainted argument is passed in, the function behaves normally.  But
-    if any of the arguments is tainted, it is automatically untainted - so
-    the function body always sees untainted arguments - and the eventual
-    result is tainted again (possibly in a Taint Bomb).
-
-    It is important for the function marked as ``taint_atomic`` to have no
-    visible side effects, as these could cause information leakage.
-    This is currently not enforced, which means that all ``taint_atomic``
-    functions have to be carefully reviewed for security (but not the
-    callers of ``taint_atomic`` functions).
-
-    A possible future extension would be to forbid side-effects on
-    non-tainted objects from all ``taint_atomic`` functions.
-
-    An example of usage: given a tainted object ``passwords_db`` that
-    references a database of passwords, we can write a function
-    that checks if a password is valid as follows::
-
-        @taint_atomic
-        def validate(passwords_db, username, password):
-            assert type(passwords_db) is PasswordDatabase
-            assert type(username) is str
-            assert type(password) is str
-            ...load username entry from passwords_db...
-            return expected_password == password
-
-    It returns a tainted boolean answer, or a Taint Bomb if something
-    went wrong.  A caller can do::
-
-        ok = validate(passwords_db, 'john', '1234')
-        ok = untaint(bool, ok)
-
-    This can give three outcomes: ``True``, ``False``, or a ``TaintError``
-    exception (with no information on it) if anything went wrong.  If even
-    this is considered giving too much information away, the ``False`` case
-    can be made indistinguishable from the ``TaintError`` case (simply by
-    raising an exception in ``validate()`` if the password is wrong).
-
-    In the above example, the security results achieved are the following:
-    as long as ``validate()`` does not leak information, no other part of
-    the code can obtain more information about a passwords database than a
-    Yes/No answer to a precise query.
-
-    A possible extension of the ``taint_atomic`` decorator would be to check
-    the argument types, as ``untaint()`` does, for the same reason: to
-    prevent bugs where a function like ``validate()`` above is accidentally
-    called with the wrong kind of tainted object, which would make it
-    misbehave.  For now, all ``taint_atomic`` functions should be
-    conservative and carefully check all assumptions on their input
-    arguments.
-
-
-    .. _`taint-interface`:
-
-    Interface
-    ---------
-
-    .. _`like a built-in operation`:
-
-    The basic rule of the Tainted Object Space is that it introduces two new
-    kinds of objects, Tainted Boxes and Tainted Bombs (which are not types
-    in the Python sense).  Each box internally contains a regular object;
-    each bomb internally contains an exception object.  An operation
-    involving Tainted Boxes is performed on the objects contained in the
-    boxes, and gives a Tainted Box or a Tainted Bomb as a result (such an
-    operation does not let an exception be raised).  An operation called
-    with a Tainted Bomb argument immediately returns the same Tainted Bomb.
-
-    In a PyPy running with (or translated with) the Taint Object Space,
-    the ``__pypy__`` module exposes the following interface:
-
-    * ``taint(obj)``
-
-        Return a new Tainted Box wrapping ``obj``.  Return ``obj`` itself
-        if it is already tainted (a Box or a Bomb).
-
-    * ``is_tainted(obj)``
-
-        Check if ``obj`` is tainted (a Box or a Bomb).
-
-    * ``untaint(type, obj)``
-
-        Untaints ``obj`` if it is tainted.  Raise ``TaintError`` if the type
-        of the untainted object is not exactly ``type``, or if ``obj`` is a
-        Bomb.
-
-    * ``taint_atomic(func)``
-
-        Return a wrapper function around the callable ``func``.  The wrapper
-        behaves `like a built-in operation`_ with respect to untainting the
-        arguments, tainting the result, and returning a Bomb.
-
-    * ``TaintError``
-
-        Exception.  On purpose, it provides no attribute or error message.
-
-    * ``_taint_debug(level)``
-
-        Set the debugging level to ``level`` (0=off).  At level 1 or above,
-        all Taint Bombs print a diagnostic message to stderr when they are
-        created.
-
-    * ``_taint_look(obj)``
-
-        For debugging purposes: prints (to stderr) the type and address of
-        the object in a Tainted Box, or prints the exception if ``obj`` is
-        a Taint Bomb.
-
-
 .. _dump:
 
 The Dump Object Space
diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -17,6 +17,12 @@
 projects, or anything else in PyPy, pop up on IRC or write to us on the
 `mailing list`_.
 
+Make big integers faster
+-------------------------
+
+PyPy's implementation of the Python ``long`` type is slower than CPython's.
+Find out why and optimize them.
+
 Numpy improvements
 ------------------
 
diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -2,7 +2,7 @@
 from pypy.interpreter.baseobjspace import Wrappable
 from pypy.interpreter import typedef
 from pypy.interpreter.gateway import interp2app
-from pypy.interpreter.error import OperationError
+from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.tool.pairtype import extendabletype
 from pypy.tool.sourcetools import func_with_new_name
@@ -2925,14 +2925,13 @@
 def Module_get_body(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -2968,14 +2967,13 @@
 def Interactive_get_body(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3015,8 +3013,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     return space.wrap(w_self.body)
 
 def Expression_set_body(space, w_self, w_new_value):
@@ -3057,14 +3054,13 @@
 def Suite_get_body(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3104,8 +3100,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._lineno_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'lineno'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lineno')
     return space.wrap(w_self.lineno)
 
 def stmt_set_lineno(space, w_self, w_new_value):
@@ -3126,8 +3121,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._col_offset_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'col_offset'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'col_offset')
     return space.wrap(w_self.col_offset)
 
 def stmt_set_col_offset(space, w_self, w_new_value):
@@ -3157,8 +3151,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'name'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name')
     return space.wrap(w_self.name)
 
 def FunctionDef_set_name(space, w_self, w_new_value):
@@ -3179,8 +3172,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'args'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args')
     return space.wrap(w_self.args)
 
 def FunctionDef_set_args(space, w_self, w_new_value):
@@ -3197,14 +3189,13 @@
 def FunctionDef_get_body(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3215,14 +3206,13 @@
 def FunctionDef_get_decorator_list(space, w_self):
     if not w_self.initialization_state & 8:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'decorator_list'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'decorator_list')
     if w_self.w_decorator_list is None:
         if w_self.decorator_list is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.decorator_list]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_decorator_list = w_list
     return w_self.w_decorator_list
 
@@ -3266,8 +3256,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'name'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name')
     return space.wrap(w_self.name)
 
 def ClassDef_set_name(space, w_self, w_new_value):
@@ -3284,14 +3273,13 @@
 def ClassDef_get_bases(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'bases'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'bases')
     if w_self.w_bases is None:
         if w_self.bases is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.bases]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_bases = w_list
     return w_self.w_bases
 
@@ -3302,14 +3290,13 @@
 def ClassDef_get_body(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3320,14 +3307,13 @@
 def ClassDef_get_decorator_list(space, w_self):
     if not w_self.initialization_state & 8:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'decorator_list'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'decorator_list')
     if w_self.w_decorator_list is None:
         if w_self.decorator_list is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.decorator_list]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_decorator_list = w_list
     return w_self.w_decorator_list
 
@@ -3372,8 +3358,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Return_set_value(space, w_self, w_new_value):
@@ -3414,14 +3399,13 @@
 def Delete_get_targets(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'targets'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'targets')
     if w_self.w_targets is None:
         if w_self.targets is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.targets]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_targets = w_list
     return w_self.w_targets
 
@@ -3457,14 +3441,13 @@
 def Assign_get_targets(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'targets'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'targets')
     if w_self.w_targets is None:
         if w_self.targets is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.targets]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_targets = w_list
     return w_self.w_targets
 
@@ -3479,8 +3462,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Assign_set_value(space, w_self, w_new_value):
@@ -3527,8 +3509,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'target'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'target')
     return space.wrap(w_self.target)
 
 def AugAssign_set_target(space, w_self, w_new_value):
@@ -3549,8 +3530,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'op'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op')
     return operator_to_class[w_self.op - 1]()
 
 def AugAssign_set_op(space, w_self, w_new_value):
@@ -3573,8 +3553,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def AugAssign_set_value(space, w_self, w_new_value):
@@ -3621,8 +3600,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'dest'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'dest')
     return space.wrap(w_self.dest)
 
 def Print_set_dest(space, w_self, w_new_value):
@@ -3639,14 +3617,13 @@
 def Print_get_values(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'values'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'values')
     if w_self.w_values is None:
         if w_self.values is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.values]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_values = w_list
     return w_self.w_values
 
@@ -3661,8 +3638,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'nl'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'nl')
     return space.wrap(w_self.nl)
 
 def Print_set_nl(space, w_self, w_new_value):
@@ -3710,8 +3686,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'target'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'target')
     return space.wrap(w_self.target)
 
 def For_set_target(space, w_self, w_new_value):
@@ -3732,8 +3707,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'iter'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'iter')
     return space.wrap(w_self.iter)
 
 def For_set_iter(space, w_self, w_new_value):
@@ -3750,14 +3724,13 @@
 def For_get_body(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3768,14 +3741,13 @@
 def For_get_orelse(space, w_self):
     if not w_self.initialization_state & 8:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'orelse'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
     if w_self.w_orelse is None:
         if w_self.orelse is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.orelse]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_orelse = w_list
     return w_self.w_orelse
 
@@ -3819,8 +3791,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'test'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test')
     return space.wrap(w_self.test)
 
 def While_set_test(space, w_self, w_new_value):
@@ -3837,14 +3808,13 @@
 def While_get_body(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3855,14 +3825,13 @@
 def While_get_orelse(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'orelse'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
     if w_self.w_orelse is None:
         if w_self.orelse is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.orelse]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_orelse = w_list
     return w_self.w_orelse
 
@@ -3905,8 +3874,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'test'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test')
     return space.wrap(w_self.test)
 
 def If_set_test(space, w_self, w_new_value):
@@ -3923,14 +3891,13 @@
 def If_get_body(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3941,14 +3908,13 @@
 def If_get_orelse(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'orelse'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
     if w_self.w_orelse is None:
         if w_self.orelse is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.orelse]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_orelse = w_list
     return w_self.w_orelse
 
@@ -3991,8 +3957,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'context_expr'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'context_expr')
     return space.wrap(w_self.context_expr)
 
 def With_set_context_expr(space, w_self, w_new_value):
@@ -4013,8 +3978,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'optional_vars'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'optional_vars')
     return space.wrap(w_self.optional_vars)
 
 def With_set_optional_vars(space, w_self, w_new_value):
@@ -4031,14 +3995,13 @@
 def With_get_body(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -4080,8 +4043,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'type'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'type')
     return space.wrap(w_self.type)
 
 def Raise_set_type(space, w_self, w_new_value):
@@ -4102,8 +4064,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'inst'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'inst')
     return space.wrap(w_self.inst)
 
 def Raise_set_inst(space, w_self, w_new_value):
@@ -4124,8 +4085,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'tback'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'tback')
     return space.wrap(w_self.tback)
 
 def Raise_set_tback(space, w_self, w_new_value):
@@ -4168,14 +4128,13 @@
 def TryExcept_get_body(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -4186,14 +4145,13 @@
 def TryExcept_get_handlers(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'handlers'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'handlers')
     if w_self.w_handlers is None:
         if w_self.handlers is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.handlers]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_handlers = w_list
     return w_self.w_handlers
 
@@ -4204,14 +4162,13 @@
 def TryExcept_get_orelse(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'orelse'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
     if w_self.w_orelse is None:
         if w_self.orelse is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.orelse]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_orelse = w_list
     return w_self.w_orelse
 
@@ -4251,14 +4208,13 @@
 def TryFinally_get_body(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -4269,14 +4225,13 @@
 def TryFinally_get_finalbody(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'finalbody'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'finalbody')
     if w_self.w_finalbody is None:
         if w_self.finalbody is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.finalbody]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_finalbody = w_list
     return w_self.w_finalbody
 
@@ -4318,8 +4273,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'test'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test')
     return space.wrap(w_self.test)
 
 def Assert_set_test(space, w_self, w_new_value):
@@ -4340,8 +4294,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'msg'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'msg')
     return space.wrap(w_self.msg)
 
 def Assert_set_msg(space, w_self, w_new_value):
@@ -4383,14 +4336,13 @@
 def Import_get_names(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'names'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'names')
     if w_self.w_names is None:
         if w_self.names is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.names]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_names = w_list
     return w_self.w_names
 
@@ -4430,8 +4382,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'module'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'module')
     return space.wrap(w_self.module)
 
 def ImportFrom_set_module(space, w_self, w_new_value):
@@ -4451,14 +4402,13 @@
 def ImportFrom_get_names(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'names'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'names')
     if w_self.w_names is None:
         if w_self.names is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.names]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_names = w_list
     return w_self.w_names
 
@@ -4473,8 +4423,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'level'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'level')
     return space.wrap(w_self.level)
 
 def ImportFrom_set_level(space, w_self, w_new_value):
@@ -4522,8 +4471,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     return space.wrap(w_self.body)
 
 def Exec_set_body(space, w_self, w_new_value):
@@ -4544,8 +4492,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'globals'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'globals')
     return space.wrap(w_self.globals)
 
 def Exec_set_globals(space, w_self, w_new_value):
@@ -4566,8 +4513,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'locals'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'locals')
     return space.wrap(w_self.locals)
 
 def Exec_set_locals(space, w_self, w_new_value):
@@ -4610,14 +4556,13 @@
 def Global_get_names(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'names'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'names')
     if w_self.w_names is None:
         if w_self.names is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.names]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_names = w_list
     return w_self.w_names
 
@@ -4657,8 +4602,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Expr_set_value(space, w_self, w_new_value):
@@ -4754,8 +4698,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._lineno_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'lineno'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lineno')
     return space.wrap(w_self.lineno)
 
 def expr_set_lineno(space, w_self, w_new_value):
@@ -4776,8 +4719,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._col_offset_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'col_offset'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'col_offset')
     return space.wrap(w_self.col_offset)
 
 def expr_set_col_offset(space, w_self, w_new_value):
@@ -4807,8 +4749,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'op'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op')
     return boolop_to_class[w_self.op - 1]()
 
 def BoolOp_set_op(space, w_self, w_new_value):
@@ -4827,14 +4768,13 @@
 def BoolOp_get_values(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'values'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'values')
     if w_self.w_values is None:
         if w_self.values is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.values]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_values = w_list
     return w_self.w_values
 
@@ -4875,8 +4815,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'left'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'left')
     return space.wrap(w_self.left)
 
 def BinOp_set_left(space, w_self, w_new_value):
@@ -4897,8 +4836,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'op'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op')
     return operator_to_class[w_self.op - 1]()
 
 def BinOp_set_op(space, w_self, w_new_value):
@@ -4921,8 +4859,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'right'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'right')
     return space.wrap(w_self.right)
 
 def BinOp_set_right(space, w_self, w_new_value):
@@ -4969,8 +4906,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'op'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op')
     return unaryop_to_class[w_self.op - 1]()
 
 def UnaryOp_set_op(space, w_self, w_new_value):
@@ -4993,8 +4929,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'operand'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'operand')
     return space.wrap(w_self.operand)
 
 def UnaryOp_set_operand(space, w_self, w_new_value):
@@ -5040,8 +4975,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'args'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args')
     return space.wrap(w_self.args)
 
 def Lambda_set_args(space, w_self, w_new_value):
@@ -5062,8 +4996,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     return space.wrap(w_self.body)
 
 def Lambda_set_body(space, w_self, w_new_value):
@@ -5109,8 +5042,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'test'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test')
     return space.wrap(w_self.test)
 
 def IfExp_set_test(space, w_self, w_new_value):
@@ -5131,8 +5063,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     return space.wrap(w_self.body)
 
 def IfExp_set_body(space, w_self, w_new_value):
@@ -5153,8 +5084,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'orelse'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
     return space.wrap(w_self.orelse)
 
 def IfExp_set_orelse(space, w_self, w_new_value):
@@ -5197,14 +5127,13 @@
 def Dict_get_keys(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'keys'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'keys')
     if w_self.w_keys is None:
         if w_self.keys is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.keys]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_keys = w_list
     return w_self.w_keys
 
@@ -5215,14 +5144,13 @@
 def Dict_get_values(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'values'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'values')
     if w_self.w_values is None:
         if w_self.values is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.values]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_values = w_list
     return w_self.w_values
 
@@ -5260,14 +5188,13 @@
 def Set_get_elts(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elts'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elts')
     if w_self.w_elts is None:
         if w_self.elts is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.elts]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_elts = w_list
     return w_self.w_elts
 
@@ -5307,8 +5234,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elt'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elt')
     return space.wrap(w_self.elt)
 
 def ListComp_set_elt(space, w_self, w_new_value):
@@ -5325,14 +5251,13 @@
 def ListComp_get_generators(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'generators'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators')
     if w_self.w_generators is None:
         if w_self.generators is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.generators]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_generators = w_list
     return w_self.w_generators
 
@@ -5373,8 +5298,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elt'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elt')
     return space.wrap(w_self.elt)
 
 def SetComp_set_elt(space, w_self, w_new_value):
@@ -5391,14 +5315,13 @@
 def SetComp_get_generators(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'generators'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators')
     if w_self.w_generators is None:
         if w_self.generators is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.generators]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_generators = w_list
     return w_self.w_generators
 
@@ -5439,8 +5362,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'key'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'key')
     return space.wrap(w_self.key)
 
 def DictComp_set_key(space, w_self, w_new_value):
@@ -5461,8 +5383,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def DictComp_set_value(space, w_self, w_new_value):
@@ -5479,14 +5400,13 @@
 def DictComp_get_generators(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'generators'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators')
     if w_self.w_generators is None:
         if w_self.generators is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.generators]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_generators = w_list
     return w_self.w_generators
 
@@ -5528,8 +5448,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elt'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elt')
     return space.wrap(w_self.elt)
 
 def GeneratorExp_set_elt(space, w_self, w_new_value):
@@ -5546,14 +5465,13 @@
 def GeneratorExp_get_generators(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'generators'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators')
     if w_self.w_generators is None:
         if w_self.generators is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.generators]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_generators = w_list
     return w_self.w_generators
 
@@ -5594,8 +5512,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Yield_set_value(space, w_self, w_new_value):
@@ -5640,8 +5557,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'left'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'left')
     return space.wrap(w_self.left)
 
 def Compare_set_left(space, w_self, w_new_value):
@@ -5658,14 +5574,13 @@
 def Compare_get_ops(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ops'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ops')
     if w_self.w_ops is None:
         if w_self.ops is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [cmpop_to_class[node - 1]() for node in w_self.ops]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_ops = w_list
     return w_self.w_ops
 
@@ -5676,14 +5591,13 @@
 def Compare_get_comparators(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'comparators'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'comparators')
     if w_self.w_comparators is None:
         if w_self.comparators is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.comparators]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_comparators = w_list
     return w_self.w_comparators
 
@@ -5726,8 +5640,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'func'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'func')
     return space.wrap(w_self.func)
 
 def Call_set_func(space, w_self, w_new_value):
@@ -5744,14 +5657,13 @@
 def Call_get_args(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'args'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args')
     if w_self.w_args is None:
         if w_self.args is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.args]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_args = w_list
     return w_self.w_args
 
@@ -5762,14 +5674,13 @@
 def Call_get_keywords(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'keywords'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'keywords')
     if w_self.w_keywords is None:
         if w_self.keywords is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.keywords]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_keywords = w_list
     return w_self.w_keywords
 
@@ -5784,8 +5695,7 @@
             return w_obj
     if not w_self.initialization_state & 8:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'starargs'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'starargs')
     return space.wrap(w_self.starargs)
 
 def Call_set_starargs(space, w_self, w_new_value):
@@ -5806,8 +5716,7 @@
             return w_obj
     if not w_self.initialization_state & 16:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'kwargs'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'kwargs')
     return space.wrap(w_self.kwargs)
 
 def Call_set_kwargs(space, w_self, w_new_value):
@@ -5858,8 +5767,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Repr_set_value(space, w_self, w_new_value):
@@ -5904,8 +5812,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'n'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'n')
     return w_self.n
 
 def Num_set_n(space, w_self, w_new_value):
@@ -5950,8 +5857,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 's'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 's')
     return w_self.s
 
 def Str_set_s(space, w_self, w_new_value):
@@ -5996,8 +5902,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Attribute_set_value(space, w_self, w_new_value):
@@ -6018,8 +5923,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'attr'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'attr')
     return space.wrap(w_self.attr)
 
 def Attribute_set_attr(space, w_self, w_new_value):
@@ -6040,8 +5944,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ctx'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def Attribute_set_ctx(space, w_self, w_new_value):
@@ -6090,8 +5993,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Subscript_set_value(space, w_self, w_new_value):
@@ -6112,8 +6014,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'slice'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'slice')
     return space.wrap(w_self.slice)
 
 def Subscript_set_slice(space, w_self, w_new_value):
@@ -6134,8 +6035,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ctx'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def Subscript_set_ctx(space, w_self, w_new_value):
@@ -6184,8 +6084,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'id'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'id')
     return space.wrap(w_self.id)
 
 def Name_set_id(space, w_self, w_new_value):
@@ -6206,8 +6105,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ctx'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def Name_set_ctx(space, w_self, w_new_value):
@@ -6251,14 +6149,13 @@
 def List_get_elts(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elts'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elts')
     if w_self.w_elts is None:
         if w_self.elts is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.elts]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_elts = w_list
     return w_self.w_elts
 
@@ -6273,8 +6170,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ctx'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def List_set_ctx(space, w_self, w_new_value):
@@ -6319,14 +6215,13 @@
 def Tuple_get_elts(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elts'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elts')
     if w_self.w_elts is None:
         if w_self.elts is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.elts]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_elts = w_list
     return w_self.w_elts
 
@@ -6341,8 +6236,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ctx'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def Tuple_set_ctx(space, w_self, w_new_value):
@@ -6391,8 +6285,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return w_self.value
 
 def Const_set_value(space, w_self, w_new_value):
@@ -6510,8 +6403,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'lower'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lower')
     return space.wrap(w_self.lower)
 
 def Slice_set_lower(space, w_self, w_new_value):
@@ -6532,8 +6424,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'upper'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'upper')
     return space.wrap(w_self.upper)
 
 def Slice_set_upper(space, w_self, w_new_value):
@@ -6554,8 +6445,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'step'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'step')
     return space.wrap(w_self.step)
 
 def Slice_set_step(space, w_self, w_new_value):
@@ -6598,14 +6488,13 @@
 def ExtSlice_get_dims(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'dims'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'dims')
     if w_self.w_dims is None:
         if w_self.dims is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.dims]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_dims = w_list
     return w_self.w_dims
 
@@ -6645,8 +6534,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Index_set_value(space, w_self, w_new_value):
@@ -6915,8 +6803,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'target'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'target')
     return space.wrap(w_self.target)
 
 def comprehension_set_target(space, w_self, w_new_value):
@@ -6937,8 +6824,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'iter'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'iter')
     return space.wrap(w_self.iter)
 
 def comprehension_set_iter(space, w_self, w_new_value):
@@ -6955,14 +6841,13 @@
 def comprehension_get_ifs(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ifs'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ifs')
     if w_self.w_ifs is None:
         if w_self.ifs is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.ifs]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_ifs = w_list
     return w_self.w_ifs
 
@@ -7004,8 +6889,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._lineno_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'lineno'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lineno')
     return space.wrap(w_self.lineno)
 
 def excepthandler_set_lineno(space, w_self, w_new_value):
@@ -7026,8 +6910,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._col_offset_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'col_offset'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'col_offset')
     return space.wrap(w_self.col_offset)
 
 def excepthandler_set_col_offset(space, w_self, w_new_value):
@@ -7057,8 +6940,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'type'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'type')
     return space.wrap(w_self.type)
 
 def ExceptHandler_set_type(space, w_self, w_new_value):
@@ -7079,8 +6961,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'name'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name')
     return space.wrap(w_self.name)
 
 def ExceptHandler_set_name(space, w_self, w_new_value):
@@ -7097,14 +6978,13 @@
 def ExceptHandler_get_body(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -7142,14 +7022,13 @@
 def arguments_get_args(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'args'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args')
     if w_self.w_args is None:
         if w_self.args is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.args]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_args = w_list
     return w_self.w_args
 
@@ -7164,8 +7043,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'vararg'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'vararg')
     return space.wrap(w_self.vararg)
 
 def arguments_set_vararg(space, w_self, w_new_value):
@@ -7189,8 +7067,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'kwarg'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'kwarg')
     return space.wrap(w_self.kwarg)
 
 def arguments_set_kwarg(space, w_self, w_new_value):
@@ -7210,14 +7087,13 @@
 def arguments_get_defaults(space, w_self):
     if not w_self.initialization_state & 8:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'defaults'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'defaults')
     if w_self.w_defaults is None:
         if w_self.defaults is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.defaults]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_defaults = w_list
     return w_self.w_defaults
 
@@ -7261,8 +7137,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'arg'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'arg')
     return space.wrap(w_self.arg)
 
 def keyword_set_arg(space, w_self, w_new_value):
@@ -7283,8 +7158,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def keyword_set_value(space, w_self, w_new_value):
@@ -7330,8 +7204,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'name'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name')
     return space.wrap(w_self.name)
 
 def alias_set_name(space, w_self, w_new_value):
@@ -7352,8 +7225,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'asname'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'asname')
     return space.wrap(w_self.asname)
 
 def alias_set_asname(space, w_self, w_new_value):
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -414,13 +414,12 @@
             self.emit("        return w_obj", 1)
         self.emit("if not w_self.initialization_state & %s:" % (flag,), 1)
         self.emit("typename = space.type(w_self).getname(space)", 2)
-        self.emit("w_err = space.wrap(\"'%%s' object has no attribute '%s'\" %% typename)" %
+        self.emit("raise operationerrfmt(space.w_AttributeError, \"'%%s' object has no attribute '%%s'\", typename, '%s')" %
                   (field.name,), 2)
-        self.emit("raise OperationError(space.w_AttributeError, w_err)", 2)
         if field.seq:
             self.emit("if w_self.w_%s is None:" % (field.name,), 1)
             self.emit("if w_self.%s is None:" % (field.name,), 2)
-            self.emit("w_list = space.newlist([])", 3)
+            self.emit("list_w = []", 3)
             self.emit("else:", 2)
             if field.type.value in self.data.simple_types:
                 wrapper = "%s_to_class[node - 1]()" % (field.type,)
@@ -428,7 +427,7 @@
                 wrapper = "space.wrap(node)"
             self.emit("list_w = [%s for node in w_self.%s]" %
                       (wrapper, field.name), 3)
-            self.emit("w_list = space.newlist(list_w)", 3)
+            self.emit("w_list = space.newlist(list_w)", 2)
             self.emit("w_self.w_%s = w_list" % (field.name,), 2)
             self.emit("return w_self.w_%s" % (field.name,), 1)
         elif field.type.value in self.data.simple_types:
@@ -540,7 +539,7 @@
 from pypy.interpreter.baseobjspace import Wrappable
 from pypy.interpreter import typedef
 from pypy.interpreter.gateway import interp2app
-from pypy.interpreter.error import OperationError
+from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.tool.pairtype import extendabletype
 from pypy.tool.sourcetools import func_with_new_name
@@ -639,9 +638,7 @@
             missing = required[i]
             if missing is not None:
                  err = "required field \\"%s\\" missing from %s"
-                 err = err % (missing, host)
-                 w_err = space.wrap(err)
-                 raise OperationError(space.w_TypeError, w_err)
+                 raise operationerrfmt(space.w_TypeError, err, missing, host)
     raise AssertionError("should not reach here")
 
 
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -391,8 +391,11 @@
     def decrement_ticker(self, by):
         value = self._ticker
         if self.has_bytecode_counter:    # this 'if' is constant-folded
-            value -= by
-            self._ticker = value
+            if jit.isconstant(by) and by == 0:
+                pass     # normally constant-folded too
+            else:
+                value -= by
+                self._ticker = value
         return value
 
 
diff --git a/pypy/interpreter/pyparser/pytokenizer.py b/pypy/interpreter/pyparser/pytokenizer.py
--- a/pypy/interpreter/pyparser/pytokenizer.py
+++ b/pypy/interpreter/pyparser/pytokenizer.py
@@ -226,7 +226,7 @@
                         parenlev = parenlev - 1
                         if parenlev < 0:
                             raise TokenError("unmatched '%s'" % initial, line,
-                                             lnum-1, 0, token_list)
+                                             lnum, start + 1, token_list)
                     if token in python_opmap:
                         punct = python_opmap[token]
                     else:
diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py
--- a/pypy/interpreter/pyparser/test/test_pyparse.py
+++ b/pypy/interpreter/pyparser/test/test_pyparse.py
@@ -87,6 +87,10 @@
         assert exc.lineno == 1
         assert exc.offset == 5
         assert exc.lastlineno == 5
+        exc = py.test.raises(SyntaxError, parse, "abc)").value
+        assert exc.msg == "unmatched ')'"
+        assert exc.lineno == 1
+        assert exc.offset == 4
 
     def test_is(self):
         self.parse("x is y")
diff --git a/pypy/jit/backend/llgraph/llimpl.py b/pypy/jit/backend/llgraph/llimpl.py
--- a/pypy/jit/backend/llgraph/llimpl.py
+++ b/pypy/jit/backend/llgraph/llimpl.py
@@ -7,9 +7,7 @@
 import weakref
 from pypy.objspace.flow.model import Variable, Constant
 from pypy.annotation import model as annmodel
-from pypy.jit.metainterp.history import (ConstInt, ConstPtr,
-                                         BoxInt, BoxPtr, BoxObj, BoxFloat,
-                                         REF, INT, FLOAT)
+from pypy.jit.metainterp.history import REF, INT, FLOAT
 from pypy.jit.codewriter import heaptracker
 from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr, rffi
 from pypy.rpython.ootypesystem import ootype
@@ -17,7 +15,7 @@
 from pypy.rpython.llinterp import LLException
 from pypy.rpython.extregistry import ExtRegistryEntry
 
-from pypy.jit.metainterp import resoperation, executor
+from pypy.jit.metainterp import resoperation
 from pypy.jit.metainterp.resoperation import rop
 from pypy.jit.backend.llgraph import symbolic
 from pypy.jit.codewriter import longlong
@@ -165,6 +163,7 @@
     'unicodegetitem'  : (('ref', 'int'), 'int'),
     'unicodesetitem'  : (('ref', 'int', 'int'), 'int'),
     'cast_ptr_to_int' : (('ref',), 'int'),
+    'cast_int_to_ptr' : (('int',), 'ref'),
     'debug_merge_point': (('ref', 'int'), None),
     'force_token'     : ((), 'int'),
     'call_may_force'  : (('int', 'varargs'), 'intorptr'),
@@ -333,6 +332,13 @@
     assert isinstance(type, str) and len(type) == 1
     op.descr = Descr(ofs, type, arg_types=arg_types)
 
+def compile_add_descr_arg(loop, ofs, type, arg_types):
+    from pypy.jit.backend.llgraph.runner import Descr
+    loop = _from_opaque(loop)
+    op = loop.operations[-1]
+    assert isinstance(type, str) and len(type) == 1
+    op.args.append(Descr(ofs, type, arg_types=arg_types))
+
 def compile_add_loop_token(loop, descr):
     if we_are_translated():
         raise ValueError("CALL_ASSEMBLER not supported")
@@ -437,8 +443,11 @@
         self._may_force = -1
 
     def getenv(self, v):
+        from pypy.jit.backend.llgraph.runner import Descr
         if isinstance(v, Constant):
             return v.value
+        elif isinstance(v, Descr):
+            return v
         else:
             return self.env[v]
 
@@ -806,6 +815,29 @@
         else:
             raise NotImplementedError
 
+    def op_getinteriorfield_gc(self, descr, array, index):
+        if descr.typeinfo == REF:
+            return do_getinteriorfield_gc_ptr(array, index, descr.ofs)
+        elif descr.typeinfo == INT:
+            return do_getinteriorfield_gc_int(array, index, descr.ofs)
+        elif descr.typeinfo == FLOAT:
+            return do_getinteriorfield_gc_float(array, index, descr.ofs)
+        else:
+            raise NotImplementedError
+
+    def op_setinteriorfield_gc(self, descr, array, index, newvalue):
+        if descr.typeinfo == REF:
+            return do_setinteriorfield_gc_ptr(array, index, descr.ofs,
+                                              newvalue)
+        elif descr.typeinfo == INT:
+            return do_setinteriorfield_gc_int(array, index, descr.ofs,
+                                              newvalue)
+        elif descr.typeinfo == FLOAT:
+            return do_setinteriorfield_gc_float(array, index, descr.ofs,
+                                                newvalue)
+        else:
+            raise NotImplementedError
+
     def op_setfield_gc(self, fielddescr, struct, newvalue):
         if fielddescr.typeinfo == REF:
             do_setfield_gc_ptr(struct, fielddescr.ofs, newvalue)
@@ -875,9 +907,6 @@
     def op_new_array(self, arraydescr, count):
         return do_new_array(arraydescr.ofs, count)
 
-    def op_cast_ptr_to_int(self, descr, ptr):
-        return cast_to_int(ptr)
-
     def op_force_token(self, descr):
         opaque_frame = _to_opaque(self)
         return llmemory.cast_ptr_to_adr(opaque_frame)
@@ -1356,6 +1385,22 @@
 def do_getfield_gc_ptr(struct, fieldnum):
     return cast_to_ptr(_getfield_gc(struct, fieldnum))
 
+def _getinteriorfield_gc(struct, fieldnum):
+    STRUCT, fieldname = symbolic.TokenToField[fieldnum]
+    return getattr(struct, fieldname)
+
+def do_getinteriorfield_gc_int(array, index, fieldnum):
+    struct = array._obj.container.getitem(index)
+    return cast_to_int(_getinteriorfield_gc(struct, fieldnum))
+
+def do_getinteriorfield_gc_float(array, index, fieldnum):
+    struct = array._obj.container.getitem(index)
+    return cast_to_floatstorage(_getinteriorfield_gc(struct, fieldnum))
+
+def do_getinteriorfield_gc_ptr(array, index, fieldnum):
+    struct = array._obj.container.getitem(index)
+    return cast_to_ptr(_getinteriorfield_gc(struct, fieldnum))
+
 def _getfield_raw(struct, fieldnum):
     STRUCT, fieldname = symbolic.TokenToField[fieldnum]
     ptr = cast_from_int(lltype.Ptr(STRUCT), struct)
@@ -1411,26 +1456,28 @@
     newvalue = cast_from_ptr(ITEMTYPE, newvalue)
     array.setitem(index, newvalue)
 
-def do_setfield_gc_int(struct, fieldnum, newvalue):
-    STRUCT, fieldname = symbolic.TokenToField[fieldnum]
-    ptr = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), struct)
-    FIELDTYPE = getattr(STRUCT, fieldname)
-    newvalue = cast_from_int(FIELDTYPE, newvalue)
-    setattr(ptr, fieldname, newvalue)
+def new_setfield_gc(cast_func):
+    def do_setfield_gc(struct, fieldnum, newvalue):
+        STRUCT, fieldname = symbolic.TokenToField[fieldnum]
+        ptr = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), struct)
+        FIELDTYPE = getattr(STRUCT, fieldname)
+        newvalue = cast_func(FIELDTYPE, newvalue)
+        setattr(ptr, fieldname, newvalue)
+    return do_setfield_gc
+do_setfield_gc_int = new_setfield_gc(cast_from_int)
+do_setfield_gc_float = new_setfield_gc(cast_from_floatstorage)
+do_setfield_gc_ptr = new_setfield_gc(cast_from_ptr)
 
-def do_setfield_gc_float(struct, fieldnum, newvalue):
-    STRUCT, fieldname = symbolic.TokenToField[fieldnum]
-    ptr = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), struct)
-    FIELDTYPE = getattr(STRUCT, fieldname)
-    newvalue = cast_from_floatstorage(FIELDTYPE, newvalue)
-    setattr(ptr, fieldname, newvalue)
-
-def do_setfield_gc_ptr(struct, fieldnum, newvalue):
-    STRUCT, fieldname = symbolic.TokenToField[fieldnum]
-    ptr = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), struct)
-    FIELDTYPE = getattr(STRUCT, fieldname)
-    newvalue = cast_from_ptr(FIELDTYPE, newvalue)
-    setattr(ptr, fieldname, newvalue)
+def new_setinteriorfield_gc(cast_func):
+    def do_setinteriorfield_gc(array, index, fieldnum, newvalue):
+        STRUCT, fieldname = symbolic.TokenToField[fieldnum]
+        struct = array._obj.container.getitem(index)
+        FIELDTYPE = getattr(STRUCT, fieldname)
+        setattr(struct, fieldname, cast_func(FIELDTYPE, newvalue))
+    return do_setinteriorfield_gc
+do_setinteriorfield_gc_int = new_setinteriorfield_gc(cast_from_int)
+do_setinteriorfield_gc_float = new_setinteriorfield_gc(cast_from_floatstorage)
+do_setinteriorfield_gc_ptr = new_setinteriorfield_gc(cast_from_ptr)        
 
 def do_setfield_raw_int(struct, fieldnum, newvalue):
     STRUCT, fieldname = symbolic.TokenToField[fieldnum]
@@ -1696,6 +1743,7 @@
 setannotation(compile_start_float_var, annmodel.SomeInteger())
 setannotation(compile_add, annmodel.s_None)
 setannotation(compile_add_descr, annmodel.s_None)
+setannotation(compile_add_descr_arg, annmodel.s_None)
 setannotation(compile_add_var, annmodel.s_None)
 setannotation(compile_add_int_const, annmodel.s_None)
 setannotation(compile_add_ref_const, annmodel.s_None)
@@ -1743,6 +1791,9 @@
 setannotation(do_getfield_raw_int, annmodel.SomeInteger())
 setannotation(do_getfield_raw_ptr, annmodel.SomePtr(llmemory.GCREF))
 setannotation(do_getfield_raw_float, s_FloatStorage)
+setannotation(do_getinteriorfield_gc_int, annmodel.SomeInteger())
+setannotation(do_getinteriorfield_gc_ptr, annmodel.SomePtr(llmemory.GCREF))
+setannotation(do_getinteriorfield_gc_float, s_FloatStorage)
 setannotation(do_new, annmodel.SomePtr(llmemory.GCREF))
 setannotation(do_new_array, annmodel.SomePtr(llmemory.GCREF))
 setannotation(do_setarrayitem_gc_int, annmodel.s_None)
@@ -1756,6 +1807,9 @@
 setannotation(do_setfield_raw_int, annmodel.s_None)
 setannotation(do_setfield_raw_ptr, annmodel.s_None)
 setannotation(do_setfield_raw_float, annmodel.s_None)
+setannotation(do_setinteriorfield_gc_int, annmodel.s_None)
+setannotation(do_setinteriorfield_gc_ptr, annmodel.s_None)
+setannotation(do_setinteriorfield_gc_float, annmodel.s_None)
 setannotation(do_newstr, annmodel.SomePtr(llmemory.GCREF))
 setannotation(do_strsetitem, annmodel.s_None)
 setannotation(do_newunicode, annmodel.SomePtr(llmemory.GCREF))
diff --git a/pypy/jit/backend/llgraph/runner.py b/pypy/jit/backend/llgraph/runner.py
--- a/pypy/jit/backend/llgraph/runner.py
+++ b/pypy/jit/backend/llgraph/runner.py
@@ -2,21 +2,19 @@
 Minimal-API wrapper around the llinterpreter to run operations.
 """
 
-import sys
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.rpython.lltypesystem import lltype, llmemory, rclass
 from pypy.rpython.ootypesystem import ootype
 from pypy.rpython.llinterp import LLInterpreter
 from pypy.jit.metainterp import history
-from pypy.jit.metainterp.history import REF, INT, FLOAT
+from pypy.jit.metainterp.history import REF, INT, FLOAT, STRUCT
 from pypy.jit.metainterp.warmstate import unwrap
-from pypy.jit.metainterp.resoperation import ResOperation, rop
+from pypy.jit.metainterp.resoperation import rop
 from pypy.jit.backend import model
 from pypy.jit.backend.llgraph import llimpl, symbolic
 from pypy.jit.metainterp.typesystem import llhelper, oohelper
 from pypy.jit.codewriter import heaptracker, longlong
-from pypy.rlib import rgc
 
 class MiniStats:
     pass
@@ -62,6 +60,9 @@
     def is_array_of_floats(self):
         return self.typeinfo == FLOAT
 
+    def is_array_of_structs(self):
+        return self.typeinfo == STRUCT
+
     def as_vtable_size_descr(self):
         return self
 
@@ -177,8 +178,10 @@
             llimpl.compile_add(c, op.getopnum())
             descr = op.getdescr()
             if isinstance(descr, Descr):
-                llimpl.compile_add_descr(c, descr.ofs, descr.typeinfo, descr.arg_types)
-            if isinstance(descr, history.LoopToken) and op.getopnum() != rop.JUMP:
+                llimpl.compile_add_descr(c, descr.ofs, descr.typeinfo,
+                                         descr.arg_types)
+            if (isinstance(descr, history.LoopToken) and
+                op.getopnum() != rop.JUMP):
                 llimpl.compile_add_loop_token(c, descr)
             if self.is_oo and isinstance(descr, (OODescr, MethDescr)):
                 # hack hack, not rpython
@@ -193,6 +196,9 @@
                     llimpl.compile_add_ref_const(c, x.value, self.ts.BASETYPE)
                 elif isinstance(x, history.ConstFloat):
                     llimpl.compile_add_float_const(c, x.value)
+                elif isinstance(x, Descr):
+                    llimpl.compile_add_descr_arg(c, x.ofs, x.typeinfo,
+                                                 x.arg_types)
                 else:
                     raise Exception("'%s' args contain: %r" % (op.getopname(),
                                                                x))
@@ -316,6 +322,13 @@
         token = history.getkind(getattr(S, fieldname))
         return self.getdescr(ofs, token[0], name=fieldname)
 
+    def interiorfielddescrof(self, A, fieldname):
+        S = A.OF
+        ofs2 = symbolic.get_size(A)
+        ofs, size = symbolic.get_field_token(S, fieldname)
+        token = history.getkind(getattr(S, fieldname))
+        return self.getdescr(ofs, token[0], name=fieldname, extrainfo=ofs2)
+
     def calldescrof(self, FUNC, ARGS, RESULT, extrainfo):
         arg_types = []
         for ARG in ARGS:
@@ -353,8 +366,13 @@
     def arraydescrof(self, A):
         assert A.OF != lltype.Void
         size = symbolic.get_size(A)
-        token = history.getkind(A.OF)
-        return self.getdescr(size, token[0])
+        if isinstance(A.OF, lltype.Ptr) or isinstance(A.OF, lltype.Primitive):
+            token = history.getkind(A.OF)[0]
+        elif isinstance(A.OF, lltype.Struct):
+            token = 's'
+        else:
+            token = '?'
+        return self.getdescr(size, token)
 
     # ---------- the backend-dependent operations ----------
 
@@ -406,6 +424,29 @@
         assert isinstance(fielddescr, Descr)
         return llimpl.do_getfield_raw_float(struct, fielddescr.ofs)
 
+    def bh_getinteriorfield_gc_i(self, array, index, descr):
+        assert isinstance(descr, Descr)
+        return llimpl.do_getinteriorfield_gc_int(array, index, descr.ofs)
+    def bh_getinteriorfield_gc_r(self, array, index, descr):
+        assert isinstance(descr, Descr)
+        return llimpl.do_getinteriorfield_gc_ptr(array, index, descr.ofs)
+    def bh_getinteriorfield_gc_f(self, array, index, descr):
+        assert isinstance(descr, Descr)
+        return llimpl.do_getinteriorfield_gc_float(array, index, descr.ofs)
+
+    def bh_setinteriorfield_gc_i(self, array, index, descr, value):
+        assert isinstance(descr, Descr)
+        return llimpl.do_setinteriorfield_gc_int(array, index, descr.ofs,
+                                                 value)
+    def bh_setinteriorfield_gc_r(self, array, index, descr, value):
+        assert isinstance(descr, Descr)
+        return llimpl.do_setinteriorfield_gc_ptr(array, index, descr.ofs,
+                                                 value)
+    def bh_setinteriorfield_gc_f(self, array, index, descr, value):
+        assert isinstance(descr, Descr)
+        return llimpl.do_setinteriorfield_gc_float(array, index, descr.ofs,
+                                                   value)
+
     def bh_new(self, sizedescr):
         assert isinstance(sizedescr, Descr)
         return llimpl.do_new(sizedescr.ofs)
@@ -418,7 +459,6 @@
 
     def bh_classof(self, struct):
         struct = lltype.cast_opaque_ptr(rclass.OBJECTPTR, struct)
-        result = struct.typeptr
         result_adr = llmemory.cast_ptr_to_adr(struct.typeptr)
         return heaptracker.adr2int(result_adr)
 
diff --git a/pypy/jit/backend/llsupport/descr.py b/pypy/jit/backend/llsupport/descr.py
--- a/pypy/jit/backend/llsupport/descr.py
+++ b/pypy/jit/backend/llsupport/descr.py
@@ -1,13 +1,10 @@
 import py
-from pypy.rpython.lltypesystem import lltype, rffi, llmemory, rclass
+from pypy.rpython.lltypesystem import lltype, rffi, llmemory
 from pypy.rpython.lltypesystem.lloperation import llop
 from pypy.jit.backend.llsupport import symbolic, support
-from pypy.jit.metainterp.history import AbstractDescr, getkind, BoxInt, BoxPtr
-from pypy.jit.metainterp.history import BasicFailDescr, LoopToken, BoxFloat
+from pypy.jit.metainterp.history import AbstractDescr, getkind
 from pypy.jit.metainterp import history
-from pypy.jit.metainterp.resoperation import ResOperation, rop
 from pypy.jit.codewriter import heaptracker, longlong
-from pypy.rlib.rarithmetic import r_longlong, r_ulonglong
 
 # The point of the class organization in this file is to make instances
 # as compact as possible.  This is done by not storing the field size or
@@ -23,6 +20,7 @@
         self._cache_field = {}
         self._cache_array = {}
         self._cache_call = {}
+        self._cache_interiorfield = {}
 
     def init_size_descr(self, STRUCT, sizedescr):
         assert isinstance(STRUCT, lltype.GcStruct)
@@ -142,7 +140,6 @@
         cachedict[fieldname] = fielddescr
         return fielddescr
 
-
 # ____________________________________________________________
 # ArrayDescrs
 
@@ -167,6 +164,7 @@
 
     _is_array_of_pointers = False      # unless overridden by GcPtrArrayDescr
     _is_array_of_floats   = False      # unless overridden by FloatArrayDescr
+    _is_array_of_structs  = False      # unless overridden by StructArrayDescr
     _is_item_signed       = False      # unless overridden by XxxArrayDescr
 
     def is_array_of_pointers(self):
@@ -175,6 +173,9 @@
     def is_array_of_floats(self):
         return self._is_array_of_floats
 
+    def is_array_of_structs(self):
+        return self._is_array_of_structs
+
     def is_item_signed(self):
         return self._is_item_signed
 
@@ -199,6 +200,10 @@
     def get_item_size(self, translate_support_code):
         return symbolic.get_size(lltype.Float, translate_support_code)
 
+class StructArrayDescr(BaseArrayDescr):
+    _clsname = 'StructArrayDescr'
+    _is_array_of_structs = True
+
 class BaseArrayNoLengthDescr(BaseArrayDescr):
     def get_base_size(self, translate_support_code):
         return 0
@@ -218,6 +223,13 @@
 def getArrayDescrClass(ARRAY):
     if ARRAY.OF is lltype.Float:
         return FloatArrayDescr
+    elif isinstance(ARRAY.OF, lltype.Struct):
+        class Descr(StructArrayDescr):
+            _clsname = '%sArrayDescr' % ARRAY.OF._name
+            def get_item_size(self, translate_support_code):
+                return symbolic.get_size(ARRAY.OF, translate_support_code)
+        Descr.__name__ = Descr._clsname
+        return Descr
     return getDescrClass(ARRAY.OF, BaseArrayDescr, GcPtrArrayDescr,
                          NonGcPtrArrayDescr, 'Array', 'get_item_size',
                          '_is_array_of_floats', '_is_item_signed')
@@ -252,6 +264,39 @@
         cache[ARRAY] = arraydescr
         return arraydescr
 
+# ____________________________________________________________
+# InteriorFieldDescr
+
+class InteriorFieldDescr(AbstractDescr):
+    arraydescr = BaseArrayDescr()     # workaround for the annotator
+    fielddescr = BaseFieldDescr('', 0)
+
+    def __init__(self, arraydescr, fielddescr):
+        self.arraydescr = arraydescr
+        self.fielddescr = fielddescr
+
+    def is_pointer_field(self):
+        return self.fielddescr.is_pointer_field()
+
+    def is_float_field(self):
+        return self.fielddescr.is_float_field()
+
+    def sort_key(self):
+        return self.fielddescr.sort_key()
+
+    def repr_of_descr(self):
+        return '<InteriorFieldDescr %s>' % self.fielddescr.repr_of_descr()
+
+def get_interiorfield_descr(gc_ll_descr, ARRAY, FIELDTP, name):
+    cache = gc_ll_descr._cache_interiorfield
+    try:
+        return cache[(ARRAY, FIELDTP, name)]
+    except KeyError:
+        arraydescr = get_array_descr(gc_ll_descr, ARRAY)
+        fielddescr = get_field_descr(gc_ll_descr, FIELDTP, name)
+        descr = InteriorFieldDescr(arraydescr, fielddescr)
+        cache[(ARRAY, FIELDTP, name)] = descr
+        return descr
 
 # ____________________________________________________________
 # CallDescrs
@@ -525,7 +570,8 @@
         #
         if TYPE is lltype.Float or is_longlong(TYPE):
             setattr(Descr, floatattrname, True)
-        elif TYPE is not lltype.Bool and rffi.cast(TYPE, -1) == -1:
+        elif (TYPE is not lltype.Bool and isinstance(TYPE, lltype.Number) and
+              rffi.cast(TYPE, -1) == -1):
             setattr(Descr, signedattrname, True)
         #
         _cache[nameprefix, TYPE] = Descr
diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py
--- a/pypy/jit/backend/llsupport/gc.py
+++ b/pypy/jit/backend/llsupport/gc.py
@@ -45,6 +45,22 @@
     def freeing_block(self, start, stop):
         pass
 
+    def get_funcptr_for_newarray(self):
+        return llhelper(self.GC_MALLOC_ARRAY, self.malloc_array)
+    def get_funcptr_for_newstr(self):
+        return llhelper(self.GC_MALLOC_STR_UNICODE, self.malloc_str)
+    def get_funcptr_for_newunicode(self):
+        return llhelper(self.GC_MALLOC_STR_UNICODE, self.malloc_unicode)
+
+
+    def record_constptrs(self, op, gcrefs_output_list):
+        for i in range(op.numargs()):
+            v = op.getarg(i)
+            if isinstance(v, ConstPtr) and bool(v.value):
+                p = v.value
+                rgc._make_sure_does_not_move(p)
+                gcrefs_output_list.append(p)
+
 # ____________________________________________________________
 
 class GcLLDescr_boehm(GcLLDescription):
@@ -88,6 +104,39 @@
         malloc_fn_ptr = self.configure_boehm_once()
         self.funcptr_for_new = malloc_fn_ptr
 
+        def malloc_array(basesize, itemsize, ofs_length, num_elem):
+            try:
+                size = ovfcheck(basesize + ovfcheck(itemsize * num_elem))
+            except OverflowError:
+                return lltype.nullptr(llmemory.GCREF.TO)
+            res = self.funcptr_for_new(size)
+            if not res:
+                return res
+            rffi.cast(rffi.CArrayPtr(lltype.Signed), res)[ofs_length/WORD] = num_elem
+            return res
+        self.malloc_array = malloc_array
+        self.GC_MALLOC_ARRAY = lltype.Ptr(lltype.FuncType(
+            [lltype.Signed] * 4, llmemory.GCREF))
+
+
+        (str_basesize, str_itemsize, str_ofs_length
+         ) = symbolic.get_array_token(rstr.STR, self.translate_support_code)
+        (unicode_basesize, unicode_itemsize, unicode_ofs_length
+         ) = symbolic.get_array_token(rstr.UNICODE, self.translate_support_code)
+        def malloc_str(length):
+            return self.malloc_array(
+                str_basesize, str_itemsize, str_ofs_length, length
+            )
+        def malloc_unicode(length):
+            return self.malloc_array(
+                unicode_basesize, unicode_itemsize, unicode_ofs_length, length
+            )
+        self.malloc_str = malloc_str
+        self.malloc_unicode = malloc_unicode
+        self.GC_MALLOC_STR_UNICODE = lltype.Ptr(lltype.FuncType(
+            [lltype.Signed], llmemory.GCREF))
+
+
         # on some platform GC_init is required before any other
         # GC_* functions, call it here for the benefit of tests
         # XXX move this to tests
@@ -108,38 +157,34 @@
         ofs_length = arraydescr.get_ofs_length(self.translate_support_code)
         basesize = arraydescr.get_base_size(self.translate_support_code)
         itemsize = arraydescr.get_item_size(self.translate_support_code)
-        size = basesize + itemsize * num_elem
-        res = self.funcptr_for_new(size)
-        rffi.cast(rffi.CArrayPtr(lltype.Signed), res)[ofs_length/WORD] = num_elem
-        return res
+        return self.malloc_array(basesize, itemsize, ofs_length, num_elem)
 
     def gc_malloc_str(self, num_elem):
-        basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR,
-                                                   self.translate_support_code)
-        assert itemsize == 1
-        size = basesize + num_elem
-        res = self.funcptr_for_new(size)
-        rffi.cast(rffi.CArrayPtr(lltype.Signed), res)[ofs_length/WORD] = num_elem
-        return res
+        return self.malloc_str(num_elem)
 
     def gc_malloc_unicode(self, num_elem):
-        basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE,
-                                                   self.translate_support_code)
-        size = basesize + num_elem * itemsize
-        res = self.funcptr_for_new(size)
-        rffi.cast(rffi.CArrayPtr(lltype.Signed), res)[ofs_length/WORD] = num_elem
-        return res
+        return self.malloc_unicode(num_elem)
 
     def args_for_new(self, sizedescr):
         assert isinstance(sizedescr, BaseSizeDescr)
         return [sizedescr.size]
 
+    def args_for_new_array(self, arraydescr):
+        ofs_length = arraydescr.get_ofs_length(self.translate_support_code)
+        basesize = arraydescr.get_base_size(self.translate_support_code)
+        itemsize = arraydescr.get_item_size(self.translate_support_code)
+        return [basesize, itemsize, ofs_length]
+
     def get_funcptr_for_new(self):
         return self.funcptr_for_new
 
-    get_funcptr_for_newarray = None
-    get_funcptr_for_newstr = None
-    get_funcptr_for_newunicode = None
+    def rewrite_assembler(self, cpu, operations, gcrefs_output_list):
+        # record all GCREFs too, because Boehm cannot see them and keep them
+        # alive if they end up as constants in the assembler
+        for op in operations:
+            self.record_constptrs(op, gcrefs_output_list)
+        return GcLLDescription.rewrite_assembler(self, cpu, operations,
+                                                 gcrefs_output_list)
 
 
 # ____________________________________________________________
@@ -604,10 +649,13 @@
         def malloc_basic(size, tid):
             type_id = llop.extract_ushort(llgroup.HALFWORD, tid)
             has_finalizer = bool(tid & (1<<llgroup.HALFSHIFT))
+            has_light_finalizer = bool(tid & (1<<(llgroup.HALFSHIFT + 1)))
             check_typeid(type_id)
             res = llop1.do_malloc_fixedsize_clear(llmemory.GCREF,
                                                   type_id, size,
-                                                  has_finalizer, False)
+                                                  has_finalizer,
+                                                  has_light_finalizer,
+                                                  False)
             # In case the operation above failed, we are returning NULL
             # from this function to assembler.  There is also an RPython
             # exception set, typically MemoryError; but it's easier and
@@ -678,7 +726,7 @@
             # also use it to allocate varsized objects.  The tid
             # and possibly the length are both set afterward.
             gcref = llop1.do_malloc_fixedsize_clear(llmemory.GCREF,
-                                        0, size, False, False)
+                                        0, size, False, False, False)
             return rffi.cast(lltype.Signed, gcref)
         self.malloc_slowpath = malloc_slowpath
         self.MALLOC_SLOWPATH = lltype.FuncType([lltype.Signed], lltype.Signed)
@@ -702,7 +750,9 @@
         type_id = self.layoutbuilder.get_type_id(S)
         assert not self.layoutbuilder.is_weakref_type(S)
         has_finalizer = bool(self.layoutbuilder.has_finalizer(S))
-        flags = int(has_finalizer) << llgroup.HALFSHIFT
+        has_light_finalizer = bool(self.layoutbuilder.has_light_finalizer(S))
+        flags = (int(has_finalizer) << llgroup.HALFSHIFT |
+                 int(has_light_finalizer) << (llgroup.HALFSHIFT + 1))
         descr.tid = llop.combine_ushort(lltype.Signed, type_id, flags)
 
     def init_array_descr(self, A, descr):
@@ -736,15 +786,6 @@
     def get_funcptr_for_new(self):
         return llhelper(self.GC_MALLOC_BASIC, self.malloc_basic)
 
-    def get_funcptr_for_newarray(self):
-        return llhelper(self.GC_MALLOC_ARRAY, self.malloc_array)
-
-    def get_funcptr_for_newstr(self):
-        return llhelper(self.GC_MALLOC_STR_UNICODE, self.malloc_str)
-
-    def get_funcptr_for_newunicode(self):
-        return llhelper(self.GC_MALLOC_STR_UNICODE, self.malloc_unicode)
-
     def do_write_barrier(self, gcref_struct, gcref_newptr):
         hdr_addr = llmemory.cast_ptr_to_adr(gcref_struct)
         hdr_addr -= self.gcheaderbuilder.size_gc_header
@@ -757,14 +798,6 @@
             funcptr(llmemory.cast_ptr_to_adr(gcref_struct),
                     llmemory.cast_ptr_to_adr(gcref_newptr))
 
-    def record_constptrs(self, op, gcrefs_output_list):
-        for i in range(op.numargs()):
-            v = op.getarg(i)
-            if isinstance(v, ConstPtr) and bool(v.value):
-                p = v.value
-                rgc._make_sure_does_not_move(p)
-                gcrefs_output_list.append(p)
-
     def rewrite_assembler(self, cpu, operations, gcrefs_output_list):
         # Perform two kinds of rewrites in parallel:
         #
diff --git a/pypy/jit/backend/llsupport/llmodel.py b/pypy/jit/backend/llsupport/llmodel.py
--- a/pypy/jit/backend/llsupport/llmodel.py
+++ b/pypy/jit/backend/llsupport/llmodel.py
@@ -1,24 +1,18 @@
-import sys
 from pypy.rpython.lltypesystem import lltype, llmemory, rffi, rclass, rstr
 from pypy.rpython.lltypesystem.lloperation import llop
-from pypy.rpython.llinterp import LLInterpreter, LLException
+from pypy.rpython.llinterp import LLInterpreter
 from pypy.rpython.annlowlevel import llhelper
 from pypy.rlib.objectmodel import we_are_translated, specialize
-from pypy.jit.metainterp.history import BoxInt, BoxPtr, set_future_values,\
-     BoxFloat
 from pypy.jit.metainterp import history
 from pypy.jit.codewriter import heaptracker, longlong
 from pypy.jit.backend.model import AbstractCPU
 from pypy.jit.backend.llsupport import symbolic
 from pypy.jit.backend.llsupport.symbolic import WORD, unroll_basic_sizes
-from pypy.jit.backend.llsupport.descr import get_size_descr,  BaseSizeDescr
-from pypy.jit.backend.llsupport.descr import get_field_descr, BaseFieldDescr
-from pypy.jit.backend.llsupport.descr import get_array_descr, BaseArrayDescr
-from pypy.jit.backend.llsupport.descr import get_call_descr
-from pypy.jit.backend.llsupport.descr import BaseIntCallDescr, GcPtrCallDescr
-from pypy.jit.backend.llsupport.descr import FloatCallDescr, VoidCallDescr
+from pypy.jit.backend.llsupport.descr import (get_size_descr,
+     get_field_descr, BaseFieldDescr, get_array_descr, BaseArrayDescr,
+     get_call_descr, BaseIntCallDescr, GcPtrCallDescr, FloatCallDescr,
+     VoidCallDescr, InteriorFieldDescr, get_interiorfield_descr)
 from pypy.jit.backend.llsupport.asmmemmgr import AsmMemoryManager
-from pypy.rpython.annlowlevel import cast_instance_to_base_ptr
 
 
 class AbstractLLCPU(AbstractCPU):
@@ -241,6 +235,9 @@
     def arraydescrof(self, A):
         return get_array_descr(self.gc_ll_descr, A)
 
+    def interiorfielddescrof(self, A, fieldname):
+        return get_interiorfield_descr(self.gc_ll_descr, A, A.OF, fieldname)
+
     def unpack_arraydescr(self, arraydescr):
         assert isinstance(arraydescr, BaseArrayDescr)
         return arraydescr.get_base_size(self.translate_support_code)
@@ -358,6 +355,100 @@
     bh_getarrayitem_raw_i = bh_getarrayitem_gc_i
     bh_getarrayitem_raw_f = bh_getarrayitem_gc_f
 
+    def bh_getinteriorfield_gc_i(self, gcref, itemindex, descr):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        fieldsize = descr.fielddescr.get_field_size(self.translate_support_code)
+        sign = descr.fielddescr.is_field_signed()
+        fullofs = itemindex * size + ofs
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), fullofs)
+        for STYPE, UTYPE, itemsize in unroll_basic_sizes:
+            if fieldsize == itemsize:
+                if sign:
+                    item = rffi.cast(rffi.CArrayPtr(STYPE), items)
+                    val = item[0]
+                    val = rffi.cast(lltype.Signed, val)
+                else:
+                    item = rffi.cast(rffi.CArrayPtr(UTYPE), items)
+                    val = item[0]
+                    val = rffi.cast(lltype.Signed, val)
+                # --- end of GC unsafe code ---
+                return val
+        else:
+            raise NotImplementedError("size = %d" % fieldsize)
+
+    def bh_getinteriorfield_gc_r(self, gcref, itemindex, descr):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs +
+                            size * itemindex)
+        items = rffi.cast(rffi.CArrayPtr(lltype.Signed), items)
+        pval = self._cast_int_to_gcref(items[0])
+        # --- end of GC unsafe code ---
+        return pval
+
+    def bh_getinteriorfield_gc_f(self, gcref, itemindex, descr):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs +
+                            size * itemindex)
+        items = rffi.cast(rffi.CArrayPtr(longlong.FLOATSTORAGE), items)
+        fval = items[0]
+        # --- end of GC unsafe code ---
+        return fval
+
+    def bh_setinteriorfield_gc_i(self, gcref, itemindex, descr, value):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        fieldsize = descr.fielddescr.get_field_size(self.translate_support_code)
+        ofs = itemindex * size + ofs
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs)
+        for TYPE, _, itemsize in unroll_basic_sizes:
+            if fieldsize == itemsize:
+                items = rffi.cast(rffi.CArrayPtr(TYPE), items)
+                items[0] = rffi.cast(TYPE, value)
+                # --- end of GC unsafe code ---
+                return
+        else:
+            raise NotImplementedError("size = %d" % fieldsize)
+
+    def bh_setinteriorfield_gc_r(self, gcref, itemindex, descr, newvalue):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        self.gc_ll_descr.do_write_barrier(gcref, newvalue)
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref),
+                            ofs + size * itemindex)
+        items = rffi.cast(rffi.CArrayPtr(lltype.Signed), items)
+        items[0] = self.cast_gcref_to_int(newvalue)
+        # --- end of GC unsafe code ---
+
+    def bh_setinteriorfield_gc_f(self, gcref, itemindex, descr, newvalue):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref),
+                            ofs + size * itemindex)
+        items = rffi.cast(rffi.CArrayPtr(longlong.FLOATSTORAGE), items)
+        items[0] = newvalue
+        # --- end of GC unsafe code ---
+
     def bh_strlen(self, string):
         s = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), string)
         return len(s.chars)
@@ -475,7 +566,6 @@
 
     def bh_classof(self, struct):
         struct = lltype.cast_opaque_ptr(rclass.OBJECTPTR, struct)
-        result = struct.typeptr
         result_adr = llmemory.cast_ptr_to_adr(struct.typeptr)
         return heaptracker.adr2int(result_adr)
 
diff --git a/pypy/jit/backend/llsupport/regalloc.py b/pypy/jit/backend/llsupport/regalloc.py
--- a/pypy/jit/backend/llsupport/regalloc.py
+++ b/pypy/jit/backend/llsupport/regalloc.py
@@ -287,7 +287,6 @@
         self.reg_bindings[to_v] = reg
 
     def _move_variable_away(self, v, prev_loc):
-        reg = None
         if self.free_regs:
             loc = self.free_regs.pop()
             self.reg_bindings[v] = loc
diff --git a/pypy/jit/backend/llsupport/test/test_asmmemmgr.py b/pypy/jit/backend/llsupport/test/test_asmmemmgr.py
--- a/pypy/jit/backend/llsupport/test/test_asmmemmgr.py
+++ b/pypy/jit/backend/llsupport/test/test_asmmemmgr.py
@@ -211,14 +211,14 @@
     debug._log = debug.DebugLog()
     try:
         mc._dump(addr, 'test-logname-section')
-        log = list(debug._log)
+        log = list(debug._log) 
     finally:
         debug._log = None
     encoded = ''.join(writtencode).encode('hex').upper()
     ataddr = '@%x' % addr
     assert log == [('test-logname-section',
                     [('debug_print', 'CODE_DUMP', ataddr, '+0 ', encoded)])]
-    #
+    
     lltype.free(p, flavor='raw')
 
 def test_blockbuildermixin2():
diff --git a/pypy/jit/backend/llsupport/test/test_descr.py b/pypy/jit/backend/llsupport/test/test_descr.py
--- a/pypy/jit/backend/llsupport/test/test_descr.py
+++ b/pypy/jit/backend/llsupport/test/test_descr.py
@@ -3,7 +3,6 @@
 from pypy.jit.backend.llsupport import symbolic
 from pypy.rlib.objectmodel import Symbolic
 from pypy.rpython.annlowlevel import llhelper
-from pypy.jit.metainterp.history import BoxInt, BoxFloat, BoxPtr
 from pypy.jit.metainterp import history
 from pypy.jit.codewriter import longlong
 import sys, struct, py
@@ -149,7 +148,9 @@
     A2 = lltype.GcArray(lltype.Ptr(T))
     A3 = lltype.GcArray(lltype.Ptr(U))
     A4 = lltype.GcArray(lltype.Float)
-    A5 = lltype.GcArray(lltype.SingleFloat)
+    A5 = lltype.GcArray(lltype.Struct('x', ('v', lltype.Signed),
+                                      ('k', lltype.Signed)))
+    A6 = lltype.GcArray(lltype.SingleFloat)
     assert getArrayDescrClass(A2) is GcPtrArrayDescr
     assert getArrayDescrClass(A3) is NonGcPtrArrayDescr
     cls = getArrayDescrClass(A1)
@@ -158,7 +159,7 @@
     clsf = getArrayDescrClass(A4)
     assert clsf != cls
     assert clsf == getArrayDescrClass(lltype.GcArray(lltype.Float))
-    clss = getArrayDescrClass(A5)
+    clss = getArrayDescrClass(A6)
     assert clss not in (clsf, cls)
     assert clss == getArrayDescrClass(lltype.GcArray(rffi.UINT))
     #
@@ -168,11 +169,12 @@
     descr3 = get_array_descr(c0, A3)
     descr4 = get_array_descr(c0, A4)
     descr5 = get_array_descr(c0, A5)
+    descr6 = get_array_descr(c0, A6)
     assert descr1.__class__ is cls
     assert descr2.__class__ is GcPtrArrayDescr
     assert descr3.__class__ is NonGcPtrArrayDescr
     assert descr4.__class__ is clsf
-    assert descr5.__class__ is clss
+    assert descr6.__class__ is clss
     assert descr1 == get_array_descr(c0, lltype.GcArray(lltype.Char))
     assert not descr1.is_array_of_pointers()
     assert     descr2.is_array_of_pointers()
@@ -202,7 +204,8 @@
     assert descr2.get_item_size(False) == rffi.sizeof(lltype.Ptr(T))
     assert descr3.get_item_size(False) == rffi.sizeof(lltype.Ptr(U))
     assert descr4.get_item_size(False) == rffi.sizeof(lltype.Float)
-    assert descr5.get_item_size(False) == rffi.sizeof(lltype.SingleFloat)
+    assert descr5.get_item_size(False) == rffi.sizeof(lltype.Signed) * 2
+    assert descr6.get_item_size(False) == rffi.sizeof(lltype.SingleFloat)
     #
     assert isinstance(descr1.get_base_size(True), Symbolic)
     assert isinstance(descr2.get_base_size(True), Symbolic)
@@ -348,7 +351,6 @@
                             (rffi.SHORT,      True), (rffi.USHORT, False),
                             (rffi.INT,        True), (rffi.UINT,   False),
                             (rffi.LONG,       True), (rffi.ULONG,  False)]:
-        A = lltype.GcArray(RESTYPE)
         for tsc in [False, True]:
             c2 = GcCache(tsc)
             descr1 = get_call_descr(c2, [], RESTYPE)
@@ -379,7 +381,6 @@
     descr3i = get_array_descr(c0, lltype.GcArray(lltype.Char))
     assert descr3i.repr_of_descr() == '<CharArrayDescr>'
     #
-    cache = {}
     descr4 = get_call_descr(c0, [lltype.Char, lltype.Ptr(S)], lltype.Ptr(S))
     assert 'GcPtrCallDescr' in descr4.repr_of_descr()
     #
@@ -412,10 +413,10 @@
     ARGS = [lltype.Float, lltype.Ptr(ARRAY)]
     RES = lltype.Float
 
-    def f(a, b):
+    def f2(a, b):
         return float(b[0]) + a
 
-    fnptr = llhelper(lltype.Ptr(lltype.FuncType(ARGS, RES)), f)
+    fnptr = llhelper(lltype.Ptr(lltype.FuncType(ARGS, RES)), f2)
     descr2 = get_call_descr(c0, ARGS, RES)
     a = lltype.malloc(ARRAY, 3)
     opaquea = lltype.cast_opaque_ptr(llmemory.GCREF, a)
diff --git a/pypy/jit/backend/llsupport/test/test_gc.py b/pypy/jit/backend/llsupport/test/test_gc.py
--- a/pypy/jit/backend/llsupport/test/test_gc.py
+++ b/pypy/jit/backend/llsupport/test/test_gc.py
@@ -247,12 +247,14 @@
         self.record = []
 
     def do_malloc_fixedsize_clear(self, RESTYPE, type_id, size,
-                                  has_finalizer, contains_weakptr):
+                                  has_finalizer, has_light_finalizer,
+                                  contains_weakptr):
         assert not contains_weakptr
+        assert not has_finalizer           # in these tests
+        assert not has_light_finalizer     # in these tests
         p = llmemory.raw_malloc(size)
         p = llmemory.cast_adr_to_ptr(p, RESTYPE)
-        flags = int(has_finalizer) << 16
-        tid = llop.combine_ushort(lltype.Signed, type_id, flags)
+        tid = llop.combine_ushort(lltype.Signed, type_id, 0)
         self.record.append(("fixedsize", repr(size), tid, p))
         return p
 
diff --git a/pypy/jit/backend/model.py b/pypy/jit/backend/model.py
--- a/pypy/jit/backend/model.py
+++ b/pypy/jit/backend/model.py
@@ -1,5 +1,5 @@
 from pypy.rlib.debug import debug_start, debug_print, debug_stop
-from pypy.jit.metainterp import history, compile
+from pypy.jit.metainterp import history
 
 
 class AbstractCPU(object):
@@ -213,6 +213,10 @@
     def typedescrof(TYPE):
         raise NotImplementedError
 
+    @staticmethod
+    def interiorfielddescrof(A, fieldname):
+        raise NotImplementedError
+
     # ---------- the backend-dependent operations ----------
 
     # lltype specific operations
diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -5,7 +5,7 @@
                                          BoxInt, Box, BoxPtr,
                                          LoopToken,
                                          ConstInt, ConstPtr,
-                                         BoxObj, Const,
+                                         BoxObj,
                                          ConstObj, BoxFloat, ConstFloat)
 from pypy.jit.metainterp.resoperation import ResOperation, rop
 from pypy.jit.metainterp.typesystem import deref
@@ -111,7 +111,7 @@
         self.cpu.set_future_value_int(0, 2)
         fail = self.cpu.execute_token(looptoken)
         res = self.cpu.get_latest_value_int(0)
-        assert res == 3        
+        assert res == 3
         assert fail.identifier == 1
 
     def test_compile_loop(self):
@@ -127,7 +127,7 @@
             ]
         inputargs = [i0]
         operations[2].setfailargs([i1])
-        
+
         self.cpu.compile_loop(inputargs, operations, looptoken)
         self.cpu.set_future_value_int(0, 2)
         fail = self.cpu.execute_token(looptoken)
@@ -148,7 +148,7 @@
             ]
         inputargs = [i0]
         operations[2].setfailargs([None, None, i1, None])
-        
+
         self.cpu.compile_loop(inputargs, operations, looptoken)
         self.cpu.set_future_value_int(0, 2)
         fail = self.cpu.execute_token(looptoken)
@@ -372,7 +372,7 @@
         for opnum, boxargs, retvalue in get_int_tests():
             res = self.execute_operation(opnum, boxargs, 'int')
             assert res.value == retvalue
-        
+
     def test_float_operations(self):
         from pypy.jit.metainterp.test.test_executor import get_float_tests
         for opnum, boxargs, rettype, retvalue in get_float_tests(self.cpu):
@@ -438,7 +438,7 @@
 
     def test_ovf_operations_reversed(self):
         self.test_ovf_operations(reversed=True)
-        
+
     def test_bh_call(self):
         cpu = self.cpu
         #
@@ -503,7 +503,7 @@
                                          [funcbox, BoxInt(num), BoxInt(num)],
                                          'int', descr=dyn_calldescr)
             assert res.value == 2 * num
-            
+
 
         if cpu.supports_floats:
             def func(f0, f1, f2, f3, f4, f5, f6, i0, i1, f7, f8, f9):
@@ -543,7 +543,7 @@
         funcbox = self.get_funcbox(self.cpu, func_ptr)
         res = self.execute_operation(rop.CALL, [funcbox] + map(BoxInt, args), 'int', descr=calldescr)
         assert res.value == func(*args)
-        
+
     def test_call_stack_alignment(self):
         # test stack alignment issues, notably for Mac OS/X.
         # also test the ordering of the arguments.
@@ -615,7 +615,7 @@
         res = self.execute_operation(rop.GETFIELD_GC, [t_box],
                                      'int', descr=shortdescr)
         assert res.value == 1331
-        
+
         #
         u_box, U_box = self.alloc_instance(self.U)
         fielddescr2 = self.cpu.fielddescrof(self.S, 'next')
@@ -695,7 +695,7 @@
 
     def test_failing_guard_class(self):
         t_box, T_box = self.alloc_instance(self.T)
-        u_box, U_box = self.alloc_instance(self.U)        
+        u_box, U_box = self.alloc_instance(self.U)
         null_box = self.null_instance()
         for opname, args in [(rop.GUARD_CLASS, [t_box, U_box]),
                              (rop.GUARD_CLASS, [u_box, T_box]),
@@ -787,7 +787,7 @@
         r = self.execute_operation(rop.GETARRAYITEM_GC, [a_box, BoxInt(3)],
                                    'int', descr=arraydescr)
         assert r.value == 160
-        
+
         #
         if isinstance(A, lltype.GcArray):
             A = lltype.Ptr(A)
@@ -880,6 +880,73 @@
                                    'int', descr=arraydescr)
         assert r.value == 7441
 
+    def test_array_of_structs(self):
+        TP = lltype.GcStruct('x')
+        ITEM = lltype.Struct('x',
+                             ('vs', lltype.Signed),
+                             ('vu', lltype.Unsigned),
+                             ('vsc', rffi.SIGNEDCHAR),
+                             ('vuc', rffi.UCHAR),
+                             ('vss', rffi.SHORT),
+                             ('vus', rffi.USHORT),
+                             ('vsi', rffi.INT),
+                             ('vui', rffi.UINT),
+                             ('k', lltype.Float),
+                             ('p', lltype.Ptr(TP)))
+        a_box, A = self.alloc_array_of(ITEM, 15)
+        s_box, S = self.alloc_instance(TP)
+        kdescr = self.cpu.interiorfielddescrof(A, 'k')
+        pdescr = self.cpu.interiorfielddescrof(A, 'p')
+        self.execute_operation(rop.SETINTERIORFIELD_GC, [a_box, BoxInt(3),
+                                                         boxfloat(1.5)],
+                               'void', descr=kdescr)
+        f = self.cpu.bh_getinteriorfield_gc_f(a_box.getref_base(), 3, kdescr)
+        assert longlong.getrealfloat(f) == 1.5
+        self.cpu.bh_setinteriorfield_gc_f(a_box.getref_base(), 3, kdescr, longlong.getfloatstorage(2.5))
+        r = self.execute_operation(rop.GETINTERIORFIELD_GC, [a_box, BoxInt(3)],
+                                   'float', descr=kdescr)
+        assert r.getfloat() == 2.5
+        #
+        NUMBER_FIELDS = [('vs', lltype.Signed),
+                         ('vu', lltype.Unsigned),
+                         ('vsc', rffi.SIGNEDCHAR),
+                         ('vuc', rffi.UCHAR),
+                         ('vss', rffi.SHORT),
+                         ('vus', rffi.USHORT),
+                         ('vsi', rffi.INT),
+                         ('vui', rffi.UINT)]
+        for name, TYPE in NUMBER_FIELDS[::-1]:
+            vdescr = self.cpu.interiorfielddescrof(A, name)
+            self.execute_operation(rop.SETINTERIORFIELD_GC, [a_box, BoxInt(3),
+                                                             BoxInt(-15)],
+                                   'void', descr=vdescr)
+        for name, TYPE in NUMBER_FIELDS:
+            vdescr = self.cpu.interiorfielddescrof(A, name)
+            i = self.cpu.bh_getinteriorfield_gc_i(a_box.getref_base(), 3,
+                                                  vdescr)
+            assert i == rffi.cast(lltype.Signed, rffi.cast(TYPE, -15))
+        for name, TYPE in NUMBER_FIELDS[::-1]:
+            vdescr = self.cpu.interiorfielddescrof(A, name)
+            self.cpu.bh_setinteriorfield_gc_i(a_box.getref_base(), 3,
+                                              vdescr, -25)
+        for name, TYPE in NUMBER_FIELDS:
+            vdescr = self.cpu.interiorfielddescrof(A, name)
+            r = self.execute_operation(rop.GETINTERIORFIELD_GC,
+                                       [a_box, BoxInt(3)],
+                                       'int', descr=vdescr)
+            assert r.getint() == rffi.cast(lltype.Signed, rffi.cast(TYPE, -25))
+        #
+        self.execute_operation(rop.SETINTERIORFIELD_GC, [a_box, BoxInt(4),
+                                                         s_box],
+                               'void', descr=pdescr)
+        r = self.cpu.bh_getinteriorfield_gc_r(a_box.getref_base(), 4, pdescr)
+        assert r == s_box.getref_base()
+        self.cpu.bh_setinteriorfield_gc_r(a_box.getref_base(), 3, pdescr,
+                                          s_box.getref_base())
+        r = self.execute_operation(rop.GETINTERIORFIELD_GC, [a_box, BoxInt(3)],
+                                   'ref', descr=pdescr)
+        assert r.getref_base() == s_box.getref_base()
+
     def test_string_basic(self):
         s_box = self.alloc_string("hello\xfe")
         r = self.execute_operation(rop.STRLEN, [s_box], 'int')
@@ -1402,7 +1469,7 @@
         addr = llmemory.cast_ptr_to_adr(func_ptr)
         return ConstInt(heaptracker.adr2int(addr))
 
-    
+
     MY_VTABLE = rclass.OBJECT_VTABLE    # for tests only
 
     S = lltype.GcForwardReference()
@@ -1439,7 +1506,6 @@
         return BoxPtr(lltype.nullptr(llmemory.GCREF.TO))
 
     def alloc_array_of(self, ITEM, length):
-        cpu = self.cpu
         A = lltype.GcArray(ITEM)
         a = lltype.malloc(A, length)
         a_box = BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, a))
@@ -1468,20 +1534,16 @@
         return u''.join(u.chars)
 
 
-    def test_casts(self):
-        py.test.skip("xxx fix or kill")
-        from pypy.rpython.lltypesystem import lltype, llmemory
-        TP = lltype.GcStruct('x')
-        x = lltype.malloc(TP)        
-        x = lltype.cast_opaque_ptr(llmemory.GCREF, x)
+    def test_cast_int_to_ptr(self):
+        res = self.execute_operation(rop.CAST_INT_TO_PTR,
+                                     [BoxInt(-17)],  'ref').value
+        assert lltype.cast_ptr_to_int(res) == -17
+
+    def test_cast_ptr_to_int(self):
+        x = lltype.cast_int_to_ptr(llmemory.GCREF, -19)
         res = self.execute_operation(rop.CAST_PTR_TO_INT,
-                                     [BoxPtr(x)],  'int').value
-        expected = self.cpu.cast_adr_to_int(llmemory.cast_ptr_to_adr(x))
-        assert rffi.get_real_int(res) == rffi.get_real_int(expected)
-        res = self.execute_operation(rop.CAST_PTR_TO_INT,
-                                     [ConstPtr(x)],  'int').value
-        expected = self.cpu.cast_adr_to_int(llmemory.cast_ptr_to_adr(x))
-        assert rffi.get_real_int(res) == rffi.get_real_int(expected)
+                                     [BoxPtr(x)], 'int').value
+        assert res == -19
 
     def test_ooops_non_gc(self):
         x = lltype.malloc(lltype.Struct('x'), flavor='raw')
@@ -2299,13 +2361,6 @@
         #
         cpu.bh_strsetitem(x, 4, ord('/'))
         assert str.chars[4] == '/'
-        #
-##        x = cpu.bh_newstr(5)
-##        y = cpu.bh_cast_ptr_to_int(x)
-##        z = cpu.bh_cast_ptr_to_int(x)
-##        y = rffi.get_real_int(y)
-##        z = rffi.get_real_int(z)
-##        assert type(y) == type(z) == int and y == z
 
     def test_sorting_of_fields(self):
         S = self.S
@@ -2329,7 +2384,7 @@
         for opname, arg, res in ops:
             self.execute_operation(opname, [arg], 'void')
             assert self.guard_failed == res
-        
+
         lltype.free(x, flavor='raw')
 
     def test_assembler_call(self):
@@ -2409,7 +2464,7 @@
         FakeJitDriverSD.portal_calldescr = self.cpu.calldescrof(
             lltype.Ptr(lltype.FuncType(ARGS, RES)), ARGS, RES,
             EffectInfo.MOST_GENERAL)
-        
+
         ops = '''
         [f0, f1]
         f2 = float_add(f0, f1)
@@ -2500,7 +2555,7 @@
         FakeJitDriverSD.portal_calldescr = self.cpu.calldescrof(
             lltype.Ptr(lltype.FuncType(ARGS, RES)), ARGS, RES,
             EffectInfo.MOST_GENERAL)
-        
+
         ops = '''
         [f0, f1]
         f2 = float_add(f0, f1)
@@ -2951,4 +3006,4 @@
 
     def alloc_unicode(self, unicode):
         py.test.skip("implement me")
-    
+
diff --git a/pypy/jit/backend/test/test_ll_random.py b/pypy/jit/backend/test/test_ll_random.py
--- a/pypy/jit/backend/test/test_ll_random.py
+++ b/pypy/jit/backend/test/test_ll_random.py
@@ -28,16 +28,27 @@
         fork.structure_types_and_vtables = self.structure_types_and_vtables
         return fork
 
-    def get_structptr_var(self, r, must_have_vtable=False, type=lltype.Struct):
+    def _choose_ptr_vars(self, from_, type, array_of_structs):
+        ptrvars = []
+        for i in range(len(from_)):
+            v, S = from_[i][:2]
+            if not isinstance(S, type):
+                continue
+            if ((isinstance(S, lltype.Array) and
+                 isinstance(S.OF, lltype.Struct)) == array_of_structs):
+                ptrvars.append((v, S))
+        return ptrvars
+
+    def get_structptr_var(self, r, must_have_vtable=False, type=lltype.Struct,
+                          array_of_structs=False):
         while True:
-            ptrvars = [(v, S) for (v, S) in self.ptrvars
-                              if isinstance(S, type)]
+            ptrvars = self._choose_ptr_vars(self.ptrvars, type,
+                                            array_of_structs)
             if ptrvars and r.random() < 0.8:
                 v, S = r.choice(ptrvars)
             else:
-                prebuilt_ptr_consts = [(v, S)
-                                 for (v, S, _) in self.prebuilt_ptr_consts
-                                 if isinstance(S, type)]
+                prebuilt_ptr_consts = self._choose_ptr_vars(
+                    self.prebuilt_ptr_consts, type, array_of_structs)
                 if prebuilt_ptr_consts and r.random() < 0.7:
                     v, S = r.choice(prebuilt_ptr_consts)
                 else:
@@ -48,7 +59,8 @@
                                                 has_vtable=must_have_vtable)
                     else:
                         # create a new constant array
-                        p = self.get_random_array(r)
+                        p = self.get_random_array(r,
+                                    must_be_array_of_structs=array_of_structs)
                     S = lltype.typeOf(p).TO
                     v = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, p))
                     self.prebuilt_ptr_consts.append((v, S,
@@ -74,7 +86,8 @@
                 TYPE = lltype.Signed
         return TYPE
 
-    def get_random_structure_type(self, r, with_vtable=None, cache=True):
+    def get_random_structure_type(self, r, with_vtable=None, cache=True,
+                                  type=lltype.GcStruct):
         if cache and self.structure_types and r.random() < 0.5:
             return r.choice(self.structure_types)
         fields = []
@@ -85,7 +98,7 @@
         for i in range(r.randrange(1, 5)):
             TYPE = self.get_random_primitive_type(r)
             fields.append(('f%d' % i, TYPE))
-        S = lltype.GcStruct('S%d' % self.counter, *fields, **kwds)
+        S = type('S%d' % self.counter, *fields, **kwds)
         self.counter += 1
         if cache:
             self.structure_types.append(S)
@@ -125,17 +138,29 @@
                 setattr(p, fieldname, rffi.cast(TYPE, r.random_integer()))
         return p
 
-    def get_random_array_type(self, r):
-        TYPE = self.get_random_primitive_type(r)
+    def get_random_array_type(self, r, can_be_array_of_struct=False,
+                              must_be_array_of_structs=False):
+        if ((can_be_array_of_struct and r.random() < 0.1) or
+            must_be_array_of_structs):
+            TYPE = self.get_random_structure_type(r, cache=False,
+                                                  type=lltype.Struct)
+        else:
+            TYPE = self.get_random_primitive_type(r)
         return lltype.GcArray(TYPE)
 
-    def get_random_array(self, r):
-        A = self.get_random_array_type(r)
+    def get_random_array(self, r, must_be_array_of_structs=False):
+        A = self.get_random_array_type(r,
+                           must_be_array_of_structs=must_be_array_of_structs)
         length = (r.random_integer() // 15) % 300  # length: between 0 and 299
                                                    # likely to be small
         p = lltype.malloc(A, length)
-        for i in range(length):
-            p[i] = rffi.cast(A.OF, r.random_integer())
+        if isinstance(A.OF, lltype.Primitive):
+            for i in range(length):
+                p[i] = rffi.cast(A.OF, r.random_integer())
+        else:
+            for i in range(length):
+                for fname, TP in A.OF._flds.iteritems():
+                    setattr(p[i], fname, rffi.cast(TP, r.random_integer()))
         return p
 
     def get_index(self, length, r):
@@ -155,8 +180,16 @@
                     dic[fieldname] = getattr(p, fieldname)
         else:
             assert isinstance(S, lltype.Array)
-            for i in range(len(p)):
-                dic[i] = p[i]
+            if isinstance(S.OF, lltype.Struct):
+                for i in range(len(p)):
+                    item = p[i]
+                    s1 = {}
+                    for fieldname in S.OF._names:
+                        s1[fieldname] = getattr(item, fieldname)
+                    dic[i] = s1
+            else:
+                for i in range(len(p)):
+                    dic[i] = p[i]
         return dic
 
     def print_loop_prebuilt(self, names, writevar, s):
@@ -220,7 +253,7 @@
 
 class GetFieldOperation(test_random.AbstractOperation):
     def field_descr(self, builder, r):
-        v, S = builder.get_structptr_var(r)
+        v, S = builder.get_structptr_var(r, )
         names = S._names
         if names[0] == 'parent':
             names = names[1:]
@@ -239,6 +272,28 @@
                 continue
             break
 
+class GetInteriorFieldOperation(test_random.AbstractOperation):
+    def field_descr(self, builder, r):
+        v, A = builder.get_structptr_var(r, type=lltype.Array,
+                                         array_of_structs=True)
+        array = v.getref(lltype.Ptr(A))
+        v_index = builder.get_index(len(array), r)
+        name = r.choice(A.OF._names)
+        descr = builder.cpu.interiorfielddescrof(A, name)
+        descr._random_info = 'cpu.interiorfielddescrof(%s, %r)' % (A.OF._name,
+                                                                   name)
+        TYPE = getattr(A.OF, name)
+        return v, v_index, descr, TYPE
+
+    def produce_into(self, builder, r):
+        while True:
+            try:
+                v, v_index, descr, _ = self.field_descr(builder, r)
+                self.put(builder, [v, v_index], descr)
+            except lltype.UninitializedMemoryAccess:
+                continue
+            break
+
 class SetFieldOperation(GetFieldOperation):
     def produce_into(self, builder, r):
         v, descr, TYPE = self.field_descr(builder, r)
@@ -251,6 +306,18 @@
                 break
         builder.do(self.opnum, [v, w], descr)
 
+class SetInteriorFieldOperation(GetInteriorFieldOperation):
+    def produce_into(self, builder, r):
+        v, v_index, descr, TYPE = self.field_descr(builder, r)
+        while True:
+            if r.random() < 0.3:
+                w = ConstInt(r.random_integer())
+            else:
+                w = r.choice(builder.intvars)
+            if rffi.cast(lltype.Signed, rffi.cast(TYPE, w.value)) == w.value:
+                break
+        builder.do(self.opnum, [v, v_index, w], descr)
+
 class NewOperation(test_random.AbstractOperation):
     def size_descr(self, builder, S):
         descr = builder.cpu.sizeof(S)
@@ -306,7 +373,7 @@
 
 class NewArrayOperation(ArrayOperation):
     def produce_into(self, builder, r):
-        A = builder.get_random_array_type(r)
+        A = builder.get_random_array_type(r, can_be_array_of_struct=True)
         v_size = builder.get_index(300, r)
         v_ptr = builder.do(self.opnum, [v_size], self.array_descr(builder, A))
         builder.ptrvars.append((v_ptr, A))
@@ -586,7 +653,9 @@
 for i in range(4):      # make more common
     OPERATIONS.append(GetFieldOperation(rop.GETFIELD_GC))
     OPERATIONS.append(GetFieldOperation(rop.GETFIELD_GC))
+    OPERATIONS.append(GetInteriorFieldOperation(rop.GETINTERIORFIELD_GC))
     OPERATIONS.append(SetFieldOperation(rop.SETFIELD_GC))
+    OPERATIONS.append(SetInteriorFieldOperation(rop.SETINTERIORFIELD_GC))
     OPERATIONS.append(NewOperation(rop.NEW))
     OPERATIONS.append(NewOperation(rop.NEW_WITH_VTABLE))
 
diff --git a/pypy/jit/backend/test/test_random.py b/pypy/jit/backend/test/test_random.py
--- a/pypy/jit/backend/test/test_random.py
+++ b/pypy/jit/backend/test/test_random.py
@@ -595,6 +595,10 @@
             for name, value in fields.items():
                 if isinstance(name, str):
                     setattr(container, name, value)
+                elif isinstance(value, dict):
+                    item = container.getitem(name)
+                    for key1, value1 in value.items():
+                        setattr(item, key1, value1)
                 else:
                     container.setitem(name, value)
 
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1,7 +1,7 @@
 import sys, os
 from pypy.jit.backend.llsupport import symbolic
 from pypy.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper
-from pypy.jit.metainterp.history import Const, Box, BoxInt, BoxPtr, BoxFloat
+from pypy.jit.metainterp.history import Const, Box, BoxInt, ConstInt
 from pypy.jit.metainterp.history import (AbstractFailDescr, INT, REF, FLOAT,
                                          LoopToken)
 from pypy.rpython.lltypesystem import lltype, rffi, rstr, llmemory
@@ -36,7 +36,6 @@
 from pypy.rlib import rgc
 from pypy.rlib.clibffi import FFI_DEFAULT_ABI
 from pypy.jit.backend.x86.jump import remap_frame_layout
-from pypy.jit.metainterp.history import ConstInt, BoxInt
 from pypy.jit.codewriter.effectinfo import EffectInfo
 from pypy.jit.codewriter import longlong
 
@@ -729,8 +728,8 @@
         # Also, make sure this is consistent with FRAME_FIXED_SIZE.
         self.mc.PUSH_r(ebp.value)
         self.mc.MOV_rr(ebp.value, esp.value)
-        for regloc in self.cpu.CALLEE_SAVE_REGISTERS:
-            self.mc.PUSH_r(regloc.value)
+        for loc in self.cpu.CALLEE_SAVE_REGISTERS:
+            self.mc.PUSH_r(loc.value)
 
         gcrootmap = self.cpu.gc_ll_descr.gcrootmap
         if gcrootmap and gcrootmap.is_shadow_stack:
@@ -994,7 +993,7 @@
         effectinfo = op.getdescr().get_extra_info()
         oopspecindex = effectinfo.oopspecindex
         genop_llong_list[oopspecindex](self, op, arglocs, resloc)
-        
+
     def regalloc_perform_math(self, op, arglocs, resloc):
         effectinfo = op.getdescr().get_extra_info()
         oopspecindex = effectinfo.oopspecindex
@@ -1277,8 +1276,8 @@
     genop_int_ne = _cmpop("NE", "NE")
     genop_int_gt = _cmpop("G", "L")
     genop_int_ge = _cmpop("GE", "LE")
-    genop_ptr_eq = genop_int_eq
-    genop_ptr_ne = genop_int_ne
+    genop_ptr_eq = genop_instance_ptr_eq = genop_int_eq
+    genop_ptr_ne = genop_instance_ptr_ne = genop_int_ne
 
     genop_float_lt = _cmpop_float('B', 'A')
     genop_float_le = _cmpop_float('BE', 'AE')
@@ -1298,8 +1297,8 @@
     genop_guard_int_ne = _cmpop_guard("NE", "NE", "E", "E")
     genop_guard_int_gt = _cmpop_guard("G", "L", "LE", "GE")
     genop_guard_int_ge = _cmpop_guard("GE", "LE", "L", "G")
-    genop_guard_ptr_eq = genop_guard_int_eq
-    genop_guard_ptr_ne = genop_guard_int_ne
+    genop_guard_ptr_eq = genop_guard_instance_ptr_eq = genop_guard_int_eq
+    genop_guard_ptr_ne = genop_guard_instance_ptr_ne = genop_guard_int_ne
 
     genop_guard_uint_gt = _cmpop_guard("A", "B", "BE", "AE")
     genop_guard_uint_lt = _cmpop_guard("B", "A", "AE", "BE")
@@ -1311,7 +1310,7 @@
     genop_guard_float_eq = _cmpop_guard_float("E", "E", "NE","NE")
     genop_guard_float_gt = _cmpop_guard_float("A", "B", "BE","AE")
     genop_guard_float_ge = _cmpop_guard_float("AE","BE", "B", "A")
-    
+
     def genop_math_sqrt(self, op, arglocs, resloc):
         self.mc.SQRTSD(arglocs[0], resloc)
 
@@ -1387,7 +1386,8 @@
 
     def genop_same_as(self, op, arglocs, resloc):
         self.mov(arglocs[0], resloc)
-    #genop_cast_ptr_to_int = genop_same_as
+    genop_cast_ptr_to_int = genop_same_as
+    genop_cast_int_to_ptr = genop_same_as
 
     def genop_int_mod(self, op, arglocs, resloc):
         if IS_X86_32:
@@ -1596,12 +1596,43 @@
     genop_getarrayitem_gc_pure = genop_getarrayitem_gc
     genop_getarrayitem_raw = genop_getarrayitem_gc
 
+    def _get_interiorfield_addr(self, temp_loc, index_loc, itemsize_loc,
+                                base_loc, ofs_loc):
+        assert isinstance(itemsize_loc, ImmedLoc)
+        if isinstance(index_loc, ImmedLoc):
+            temp_loc = imm(index_loc.value * itemsize_loc.value)
+        else:
+            # XXX should not use IMUL in most cases
+            assert isinstance(temp_loc, RegLoc)
+            assert isinstance(index_loc, RegLoc)
+            self.mc.IMUL_rri(temp_loc.value, index_loc.value,
+                             itemsize_loc.value)
+        assert isinstance(ofs_loc, ImmedLoc)
+        return AddressLoc(base_loc, temp_loc, 0, ofs_loc.value)
+
+    def genop_getinteriorfield_gc(self, op, arglocs, resloc):
+        (base_loc, ofs_loc, itemsize_loc, fieldsize_loc,
+            index_loc, sign_loc) = arglocs
+        src_addr = self._get_interiorfield_addr(resloc, index_loc,
+                                                itemsize_loc, base_loc,
+                                                ofs_loc)
+        self.load_from_mem(resloc, src_addr, fieldsize_loc, sign_loc)
+
+
     def genop_discard_setfield_gc(self, op, arglocs):
         base_loc, ofs_loc, size_loc, value_loc = arglocs
         assert isinstance(size_loc, ImmedLoc)
         dest_addr = AddressLoc(base_loc, ofs_loc)
         self.save_into_mem(dest_addr, value_loc, size_loc)
 
+    def genop_discard_setinteriorfield_gc(self, op, arglocs):
+        (base_loc, ofs_loc, itemsize_loc, fieldsize_loc,
+            index_loc, temp_loc, value_loc) = arglocs
+        dest_addr = self._get_interiorfield_addr(temp_loc, index_loc,
+                                                 itemsize_loc, base_loc,
+                                                 ofs_loc)
+        self.save_into_mem(dest_addr, value_loc, fieldsize_loc)
+
     def genop_discard_setarrayitem_gc(self, op, arglocs):
         base_loc, ofs_loc, value_loc, size_loc, baseofs = arglocs
         assert isinstance(baseofs, ImmedLoc)
diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py
--- a/pypy/jit/backend/x86/regalloc.py
+++ b/pypy/jit/backend/x86/regalloc.py
@@ -7,7 +7,7 @@
                                          ResOperation, BoxPtr, ConstFloat,
                                          BoxFloat, LoopToken, INT, REF, FLOAT)
 from pypy.jit.backend.x86.regloc import *
-from pypy.rpython.lltypesystem import lltype, ll2ctypes, rffi, rstr
+from pypy.rpython.lltypesystem import lltype, rffi, rstr
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.rlib import rgc
 from pypy.jit.backend.llsupport import symbolic
@@ -17,11 +17,12 @@
 from pypy.jit.metainterp.resoperation import rop
 from pypy.jit.backend.llsupport.descr import BaseFieldDescr, BaseArrayDescr
 from pypy.jit.backend.llsupport.descr import BaseCallDescr, BaseSizeDescr
+from pypy.jit.backend.llsupport.descr import InteriorFieldDescr
 from pypy.jit.backend.llsupport.regalloc import FrameManager, RegisterManager,\
      TempBox
 from pypy.jit.backend.x86.arch import WORD, FRAME_FIXED_SIZE
 from pypy.jit.backend.x86.arch import IS_X86_32, IS_X86_64, MY_COPY_OF_REGS
-from pypy.rlib.rarithmetic import r_longlong, r_uint
+from pypy.rlib.rarithmetic import r_longlong
 
 class X86RegisterManager(RegisterManager):
 
@@ -433,7 +434,7 @@
             if self.can_merge_with_next_guard(op, i, operations):
                 oplist_with_guard[op.getopnum()](self, op, operations[i + 1])
                 i += 1
-            elif not we_are_translated() and op.getopnum() == -124: 
+            elif not we_are_translated() and op.getopnum() == -124:
                 self._consider_force_spill(op)
             else:
                 oplist[op.getopnum()](self, op)
@@ -650,8 +651,8 @@
     consider_uint_lt = _consider_compop
     consider_uint_le = _consider_compop
     consider_uint_ge = _consider_compop
-    consider_ptr_eq = _consider_compop
-    consider_ptr_ne = _consider_compop
+    consider_ptr_eq = consider_instance_ptr_eq = _consider_compop
+    consider_ptr_ne = consider_instance_ptr_ne = _consider_compop
 
     def _consider_float_op(self, op):
         loc1 = self.xrm.loc(op.getarg(1))
@@ -815,7 +816,7 @@
         save_all_regs = guard_not_forced_op is not None
         self.xrm.before_call(force_store, save_all_regs=save_all_regs)
         if not save_all_regs:
-            gcrootmap = gc_ll_descr = self.assembler.cpu.gc_ll_descr.gcrootmap
+            gcrootmap = self.assembler.cpu.gc_ll_descr.gcrootmap
             if gcrootmap and gcrootmap.is_shadow_stack:
                 save_all_regs = 2
         self.rm.before_call(force_store, save_all_regs=save_all_regs)
@@ -972,74 +973,27 @@
             return self._call(op, arglocs)
 
     def consider_newstr(self, op):
-        gc_ll_descr = self.assembler.cpu.gc_ll_descr
-        if gc_ll_descr.get_funcptr_for_newstr is not None:
-            # framework GC
-            loc = self.loc(op.getarg(0))
-            return self._call(op, [loc])
-        # boehm GC (XXX kill the following code at some point)
-        ofs_items, itemsize, ofs = symbolic.get_array_token(rstr.STR, self.translate_support_code)
-        assert itemsize == 1
-        return self._malloc_varsize(ofs_items, ofs, 0, op.getarg(0),
-                                    op.result)
+        loc = self.loc(op.getarg(0))
+        return self._call(op, [loc])
 
     def consider_newunicode(self, op):
-        gc_ll_descr = self.assembler.cpu.gc_ll_descr
-        if gc_ll_descr.get_funcptr_for_newunicode is not None:
-            # framework GC
-            loc = self.loc(op.getarg(0))
-            return self._call(op, [loc])
-        # boehm GC (XXX kill the following code at some point)
-        ofs_items, _, ofs = symbolic.get_array_token(rstr.UNICODE,
-                                                   self.translate_support_code)
-        scale = self._get_unicode_item_scale()
-        return self._malloc_varsize(ofs_items, ofs, scale, op.getarg(0),
-                                    op.result)
-
-    def _malloc_varsize(self, ofs_items, ofs_length, scale, v, res_v):
-        # XXX kill this function at some point
-        if isinstance(v, Box):
-            loc = self.rm.make_sure_var_in_reg(v, [v])
-            tempbox = TempBox()
-            other_loc = self.rm.force_allocate_reg(tempbox, [v])
-            self.assembler.load_effective_addr(loc, ofs_items,scale, other_loc)
-        else:
-            tempbox = None
-            other_loc = imm(ofs_items + (v.getint() << scale))
-        self._call(ResOperation(rop.NEW, [], res_v),
-                   [other_loc], [v])
-        loc = self.rm.make_sure_var_in_reg(v, [res_v])
-        assert self.loc(res_v) == eax
-        # now we have to reload length to some reasonable place
-        self.rm.possibly_free_var(v)
-        if tempbox is not None:
-            self.rm.possibly_free_var(tempbox)
-        self.PerformDiscard(ResOperation(rop.SETFIELD_GC, [None, None], None),
-                            [eax, imm(ofs_length), imm(WORD), loc])
+        loc = self.loc(op.getarg(0))
+        return self._call(op, [loc])
 
     def consider_new_array(self, op):
         gc_ll_descr = self.assembler.cpu.gc_ll_descr
-        if gc_ll_descr.get_funcptr_for_newarray is not None:
-            # framework GC
-            box_num_elem = op.getarg(0)
-            if isinstance(box_num_elem, ConstInt):
-                num_elem = box_num_elem.value
-                if gc_ll_descr.can_inline_malloc_varsize(op.getdescr(),
-                                                         num_elem):
-                    self.fastpath_malloc_varsize(op, op.getdescr(), num_elem)
-                    return
-            args = self.assembler.cpu.gc_ll_descr.args_for_new_array(
-                op.getdescr())
-            arglocs = [imm(x) for x in args]
-            arglocs.append(self.loc(box_num_elem))
-            self._call(op, arglocs)
-            return
-        # boehm GC (XXX kill the following code at some point)
-        itemsize, basesize, ofs_length, _, _ = (
-            self._unpack_arraydescr(op.getdescr()))
-        scale_of_field = _get_scale(itemsize)
-        self._malloc_varsize(basesize, ofs_length, scale_of_field,
-                             op.getarg(0), op.result)
+        box_num_elem = op.getarg(0)
+        if isinstance(box_num_elem, ConstInt):
+            num_elem = box_num_elem.value
+            if gc_ll_descr.can_inline_malloc_varsize(op.getdescr(),
+                                                     num_elem):
+                self.fastpath_malloc_varsize(op, op.getdescr(), num_elem)
+                return
+        args = self.assembler.cpu.gc_ll_descr.args_for_new_array(
+            op.getdescr())
+        arglocs = [imm(x) for x in args]
+        arglocs.append(self.loc(box_num_elem))
+        self._call(op, arglocs)
 
     def _unpack_arraydescr(self, arraydescr):
         assert isinstance(arraydescr, BaseArrayDescr)
@@ -1058,6 +1012,16 @@
         sign = fielddescr.is_field_signed()
         return imm(ofs), imm(size), ptr, sign
 
+    def _unpack_interiorfielddescr(self, descr):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs = arraydescr.get_base_size(self.translate_support_code)
+        itemsize = arraydescr.get_item_size(self.translate_support_code)
+        fieldsize = descr.fielddescr.get_field_size(self.translate_support_code)
+        sign = descr.fielddescr.is_field_signed()
+        ofs += descr.fielddescr.offset
+        return imm(ofs), imm(itemsize), imm(fieldsize), sign
+
     def consider_setfield_gc(self, op):
         ofs_loc, size_loc, _, _ = self._unpack_fielddescr(op.getdescr())
         assert isinstance(size_loc, ImmedLoc)
@@ -1074,6 +1038,35 @@
 
     consider_setfield_raw = consider_setfield_gc
 
+    def consider_setinteriorfield_gc(self, op):
+        t = self._unpack_interiorfielddescr(op.getdescr())
+        ofs, itemsize, fieldsize, _ = t
+        args = op.getarglist()
+        if fieldsize.value == 1:
+            need_lower_byte = True
+        else:
+            need_lower_byte = False
+        box_base, box_index, box_value = args
+        base_loc = self.rm.make_sure_var_in_reg(box_base, args)
+        index_loc = self.rm.make_sure_var_in_reg(box_index, args)
+        value_loc = self.make_sure_var_in_reg(box_value, args,
+                                              need_lower_byte=need_lower_byte)
+        # If 'index_loc' is not an immediate, then we need a 'temp_loc' that
+        # is a register whose value will be destroyed.  It's fine to destroy
+        # the same register as 'index_loc', but not the other ones.
+        self.rm.possibly_free_var(box_index)
+        if not isinstance(index_loc, ImmedLoc):
+            tempvar = TempBox()
+            temp_loc = self.rm.force_allocate_reg(tempvar, [box_base,
+                                                            box_value])
+            self.rm.possibly_free_var(tempvar)
+        else:
+            temp_loc = None
+        self.rm.possibly_free_var(box_base)
+        self.possibly_free_var(box_value)
+        self.PerformDiscard(op, [base_loc, ofs, itemsize, fieldsize,
+                                 index_loc, temp_loc, value_loc])
+
     def consider_strsetitem(self, op):
         args = op.getarglist()
         base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
@@ -1135,6 +1128,25 @@
     consider_getarrayitem_raw = consider_getarrayitem_gc
     consider_getarrayitem_gc_pure = consider_getarrayitem_gc
 
+    def consider_getinteriorfield_gc(self, op):
+        t = self._unpack_interiorfielddescr(op.getdescr())
+        ofs, itemsize, fieldsize, sign = t
+        if sign:
+            sign_loc = imm1
+        else:
+            sign_loc = imm0
+        args = op.getarglist()
+        base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
+        index_loc = self.rm.make_sure_var_in_reg(op.getarg(1), args)
+        # 'base' and 'index' are put in two registers (or one if 'index'
+        # is an immediate).  'result' can be in the same register as
+        # 'index' but must be in a different register than 'base'.
+        self.rm.possibly_free_var(op.getarg(1))
+        result_loc = self.force_allocate_reg(op.result, [op.getarg(0)])
+        self.rm.possibly_free_var(op.getarg(0))
+        self.Perform(op, [base_loc, ofs, itemsize, fieldsize,
+                          index_loc, sign_loc], result_loc)
+
     def consider_int_is_true(self, op, guard_op):
         # doesn't need arg to be in a register
         argloc = self.loc(op.getarg(0))
@@ -1152,7 +1164,8 @@
         self.possibly_free_var(op.getarg(0))
         resloc = self.force_allocate_reg(op.result)
         self.Perform(op, [argloc], resloc)
-    #consider_cast_ptr_to_int = consider_same_as
+    consider_cast_ptr_to_int = consider_same_as
+    consider_cast_int_to_ptr = consider_same_as
 
     def consider_strlen(self, op):
         args = op.getarglist()
@@ -1240,7 +1253,6 @@
         self.rm.possibly_free_var(srcaddr_box)
 
     def _gen_address_inside_string(self, baseloc, ofsloc, resloc, is_unicode):
-        cpu = self.assembler.cpu
         if is_unicode:
             ofs_items, _, _ = symbolic.get_array_token(rstr.UNICODE,
                                                   self.translate_support_code)
@@ -1299,7 +1311,7 @@
         tmpreg = X86RegisterManager.all_regs[0]
         tmploc = self.rm.force_allocate_reg(box, selected_reg=tmpreg)
         xmmtmp = X86XMMRegisterManager.all_regs[0]
-        xmmtmploc = self.xrm.force_allocate_reg(box1, selected_reg=xmmtmp)
+        self.xrm.force_allocate_reg(box1, selected_reg=xmmtmp)
         # Part about non-floats
         # XXX we don't need a copy, we only just the original list
         src_locations1 = [self.loc(op.getarg(i)) for i in range(op.numargs())
@@ -1379,7 +1391,7 @@
     return lambda self, op: fn(self, op, None)
 
 def is_comparison_or_ovf_op(opnum):
-    from pypy.jit.metainterp.resoperation import opclasses, AbstractResOp
+    from pypy.jit.metainterp.resoperation import opclasses
     cls = opclasses[opnum]
     # hack hack: in theory they are instance method, but they don't use
     # any instance field, we can use a fake object
diff --git a/pypy/jit/backend/x86/test/test_del.py b/pypy/jit/backend/x86/test/test_del.py
--- a/pypy/jit/backend/x86/test/test_del.py
+++ b/pypy/jit/backend/x86/test/test_del.py
@@ -1,5 +1,4 @@
 
-import py
 from pypy.jit.backend.x86.test.test_basic import Jit386Mixin
 from pypy.jit.metainterp.test.test_del import DelTests
 
diff --git a/pypy/jit/backend/x86/test/test_dict.py b/pypy/jit/backend/x86/test/test_dict.py
new file mode 100644
--- /dev/null
+++ b/pypy/jit/backend/x86/test/test_dict.py
@@ -0,0 +1,9 @@
+
+from pypy.jit.backend.x86.test.test_basic import Jit386Mixin
+from pypy.jit.metainterp.test.test_dict import DictTests
+
+
+class TestDict(Jit386Mixin, DictTests):
+    # for the individual tests see
+    # ====> ../../../metainterp/test/test_dict.py
+    pass
diff --git a/pypy/jit/backend/x86/test/test_runner.py b/pypy/jit/backend/x86/test/test_runner.py
--- a/pypy/jit/backend/x86/test/test_runner.py
+++ b/pypy/jit/backend/x86/test/test_runner.py
@@ -31,7 +31,7 @@
 
     # for the individual tests see
     # ====> ../../test/runner_test.py
-    
+
     def setup_method(self, meth):
         self.cpu = CPU(rtyper=None, stats=FakeStats())
         self.cpu.setup_once()
@@ -69,22 +69,16 @@
 
     def test_allocations(self):
         from pypy.rpython.lltypesystem import rstr
-        
+
         allocs = [None]
         all = []
+        orig_new = self.cpu.gc_ll_descr.funcptr_for_new
         def f(size):
             allocs.insert(0, size)
-            buf = ctypes.create_string_buffer(size)
-            all.append(buf)
-            return ctypes.cast(buf, ctypes.c_void_p).value
-        func = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int)(f)
-        addr = ctypes.cast(func, ctypes.c_void_p).value
-        # ctypes produces an unsigned value. We need it to be signed for, eg,
-        # relative addressing to work properly.
-        addr = rffi.cast(lltype.Signed, addr)
-        
+            return orig_new(size)
+
         self.cpu.assembler.setup_once()
-        self.cpu.assembler.malloc_func_addr = addr
+        self.cpu.gc_ll_descr.funcptr_for_new = f
         ofs = symbolic.get_field_token(rstr.STR, 'chars', False)[0]
 
         res = self.execute_operation(rop.NEWSTR, [ConstInt(7)], 'ref')
@@ -108,7 +102,7 @@
         res = self.execute_operation(rop.NEW_ARRAY, [ConstInt(10)],
                                          'ref', descr)
         assert allocs[0] == 10*WORD + ofs + WORD
-        resbuf = self._resbuf(res)            
+        resbuf = self._resbuf(res)
         assert resbuf[ofs/WORD] == 10
 
         # ------------------------------------------------------------
@@ -116,7 +110,7 @@
         res = self.execute_operation(rop.NEW_ARRAY, [BoxInt(10)],
                                          'ref', descr)
         assert allocs[0] == 10*WORD + ofs + WORD
-        resbuf = self._resbuf(res)                        
+        resbuf = self._resbuf(res)
         assert resbuf[ofs/WORD] == 10
 
     def test_stringitems(self):
@@ -146,7 +140,7 @@
                                                      ConstInt(2), BoxInt(38)],
                                'void', descr)
         assert resbuf[itemsofs/WORD + 2] == 38
-        
+
         self.execute_operation(rop.SETARRAYITEM_GC, [res,
                                                      BoxInt(3), BoxInt(42)],
                                'void', descr)
@@ -167,7 +161,7 @@
                                                          BoxInt(2)],
                                    'int', descr)
         assert r.value == 38
-        
+
         r = self.execute_operation(rop.GETARRAYITEM_GC, [res, BoxInt(3)],
                                    'int', descr)
         assert r.value == 42
@@ -226,7 +220,7 @@
         self.execute_operation(rop.SETFIELD_GC, [res, BoxInt(1234)], 'void', ofs_i)
         i = self.execute_operation(rop.GETFIELD_GC, [res], 'int', ofs_i)
         assert i.value == 1234
-        
+
         #u = self.execute_operation(rop.GETFIELD_GC, [res, ofs_u], 'int')
         #assert u.value == 5
         self.execute_operation(rop.SETFIELD_GC, [res, ConstInt(1)], 'void',
@@ -299,7 +293,7 @@
                     else:
                         assert result != execute(self.cpu, None,
                                                  op, None, b).value
-                    
+
 
     def test_stuff_followed_by_guard(self):
         boxes = [(BoxInt(1), BoxInt(0)),
@@ -523,7 +517,7 @@
     def test_debugger_on(self):
         from pypy.tool.logparser import parse_log_file, extract_category
         from pypy.rlib import debug
-        
+
         loop = """
         [i0]
         debug_merge_point('xyz', 0)
diff --git a/pypy/jit/backend/x86/tool/viewcode.py b/pypy/jit/backend/x86/tool/viewcode.py
--- a/pypy/jit/backend/x86/tool/viewcode.py
+++ b/pypy/jit/backend/x86/tool/viewcode.py
@@ -9,7 +9,12 @@
 """
 
 import autopath
-import operator, sys, os, re, py, new
+import new
+import operator
+import py
+import re
+import sys
+import subprocess
 from bisect import bisect_left
 
 # don't use pypy.tool.udir here to avoid removing old usessions which
@@ -44,14 +49,16 @@
     f = open(tmpfile, 'wb')
     f.write(data)
     f.close()
-    g = os.popen(objdump % {
+    p = subprocess.Popen(objdump % {
         'file': tmpfile,
         'origin': originaddr,
         'backend': objdump_backend_option[backend_name],
-    }, 'r')
-    result = g.readlines()
-    g.close()
-    lines = result[6:]   # drop some objdump cruft
+    }, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    stdout, stderr = p.communicate()
+    assert not p.returncode, ('Encountered an error running objdump: %s' %
+                              stderr)
+    # drop some objdump cruft
+    lines = stdout.splitlines()[6:]
     return format_code_dump_with_labels(originaddr, lines, label_list)
 
 def format_code_dump_with_labels(originaddr, lines, label_list):
@@ -85,8 +92,12 @@
     #
     print 'loading symbols from %s...' % (filename,)
     symbols = {}
-    g = os.popen(symbollister % filename, "r")
-    for line in g:
+    p = subprocess.Popen(symbollister % filename, shell=True,
+                         stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    stdout, stderr = p.communicate()
+    assert not p.returncode, ('Encountered an error running nm: %s' %
+                              stderr)
+    for line in stdout.splitlines():
         match = re_symbolentry.match(line)
         if match:
             addr = long(match.group(1), 16)
@@ -94,7 +105,6 @@
             if name.startswith('pypy_g_'):
                 name = '\xb7' + name[7:]
             symbols[addr] = name
-    g.close()
     print '%d symbols found' % (len(symbols),)
     return symbols
 
diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -52,9 +52,11 @@
         newoperations = []
         #
         def do_rename(var, var_or_const):
+            if var.concretetype is lltype.Void:
+                renamings[var] = Constant(None, lltype.Void)
+                return
             renamings[var] = var_or_const
-            if (isinstance(var_or_const, Constant)
-                and var.concretetype != lltype.Void):
+            if isinstance(var_or_const, Constant):
                 value = var_or_const.value
                 value = lltype._cast_whatever(var.concretetype, value)
                 renamings_constants[var] = Constant(value, var.concretetype)
@@ -441,6 +443,8 @@
     rewrite_op_gc_identityhash = _do_builtin_call
     rewrite_op_gc_id           = _do_builtin_call
     rewrite_op_uint_mod        = _do_builtin_call
+    rewrite_op_cast_float_to_uint = _do_builtin_call
+    rewrite_op_cast_uint_to_float = _do_builtin_call
 
     # ----------
     # getfield/setfield/mallocs etc.
@@ -455,6 +459,23 @@
             # the special return value None forces op.result to be considered
             # equal to op.args[0]
             return [op0, op1, None]
+        if (hints.get('promote_string') and
+            op.args[0].concretetype is not lltype.Void):
+            S = lltype.Ptr(rstr.STR)
+            assert op.args[0].concretetype == S
+            self._register_extra_helper(EffectInfo.OS_STREQ_NONNULL,
+                                        "str.eq_nonnull",
+                                        [S, S],
+                                        lltype.Signed,
+                                        EffectInfo.EF_ELIDABLE_CANNOT_RAISE)
+            descr, p = self.callcontrol.callinfocollection.callinfo_for_oopspec(
+                EffectInfo.OS_STREQ_NONNULL)
+            # XXX this is fairly ugly way of creating a constant,
+            #     however, callinfocollection has no better interface
+            c = Constant(p.adr.ptr, lltype.typeOf(p.adr.ptr))
+            op1 = SpaceOperation('str_guard_value', [op.args[0], c, descr],
+                                 op.result)
+            return [SpaceOperation('-live-', [], None), op1, None]
         else:
             log.WARNING('ignoring hint %r at %r' % (hints, self.graph))
 
@@ -718,29 +739,54 @@
         return SpaceOperation(opname, [op.args[0]], op.result)
 
     def rewrite_op_getinteriorfield(self, op):
-        # only supports strings and unicodes
         assert len(op.args) == 3
-        assert op.args[1].value == 'chars'
         optype = op.args[0].concretetype
         if optype == lltype.Ptr(rstr.STR):
             opname = "strgetitem"
+            return SpaceOperation(opname, [op.args[0], op.args[2]], op.result)
+        elif optype == lltype.Ptr(rstr.UNICODE):
+            opname = "unicodegetitem"
+            return SpaceOperation(opname, [op.args[0], op.args[2]], op.result)
         else:
-            assert optype == lltype.Ptr(rstr.UNICODE)
-            opname = "unicodegetitem"
-        return SpaceOperation(opname, [op.args[0], op.args[2]], op.result)
+            v_inst, v_index, c_field = op.args
+            if op.result.concretetype is lltype.Void:
+                return
+            # only GcArray of Struct supported
+            assert isinstance(v_inst.concretetype.TO, lltype.GcArray)
+            STRUCT = v_inst.concretetype.TO.OF
+            assert isinstance(STRUCT, lltype.Struct)
+            descr = self.cpu.interiorfielddescrof(v_inst.concretetype.TO,
+                                                  c_field.value)
+            args = [v_inst, v_index, descr]
+            kind = getkind(op.result.concretetype)[0]
+            return SpaceOperation('getinteriorfield_gc_%s' % kind, args,
+                                  op.result)
 
     def rewrite_op_setinteriorfield(self, op):
-        # only supports strings and unicodes
         assert len(op.args) == 4
-        assert op.args[1].value == 'chars'
         optype = op.args[0].concretetype
         if optype == lltype.Ptr(rstr.STR):
             opname = "strsetitem"
+            return SpaceOperation(opname, [op.args[0], op.args[2], op.args[3]],
+                                  op.result)
+        elif optype == lltype.Ptr(rstr.UNICODE):
+            opname = "unicodesetitem"
+            return SpaceOperation(opname, [op.args[0], op.args[2], op.args[3]],
+                                  op.result)
         else:
-            assert optype == lltype.Ptr(rstr.UNICODE)
-            opname = "unicodesetitem"
-        return SpaceOperation(opname, [op.args[0], op.args[2], op.args[3]],
-                              op.result)
+            v_inst, v_index, c_field, v_value = op.args
+            if v_value.concretetype is lltype.Void:
+                return
+            # only GcArray of Struct supported
+            assert isinstance(v_inst.concretetype.TO, lltype.GcArray)
+            STRUCT = v_inst.concretetype.TO.OF
+            assert isinstance(STRUCT, lltype.Struct)
+            descr = self.cpu.interiorfielddescrof(v_inst.concretetype.TO,
+                                                  c_field.value)
+            kind = getkind(v_value.concretetype)[0]
+            args = [v_inst, v_index, v_value, descr]
+            return SpaceOperation('setinteriorfield_gc_%s' % kind, args,
+                                  op.result)
 
     def _rewrite_equality(self, op, opname):
         arg0, arg1 = op.args
@@ -754,6 +800,9 @@
     def _is_gc(self, v):
         return getattr(getattr(v.concretetype, "TO", None), "_gckind", "?") == 'gc'
 
+    def _is_rclass_instance(self, v):
+        return lltype._castdepth(v.concretetype.TO, rclass.OBJECT) >= 0
+
     def _rewrite_cmp_ptrs(self, op):
         if self._is_gc(op.args[0]):
             return op
@@ -771,11 +820,21 @@
         return self._rewrite_equality(op, 'int_is_true')
 
     def rewrite_op_ptr_eq(self, op):
-        op1 = self._rewrite_equality(op, 'ptr_iszero')
+        prefix = ''
+        if self._is_rclass_instance(op.args[0]):
+            assert self._is_rclass_instance(op.args[1])
+            op = SpaceOperation('instance_ptr_eq', op.args, op.result)
+            prefix = 'instance_'
+        op1 = self._rewrite_equality(op, prefix + 'ptr_iszero')
         return self._rewrite_cmp_ptrs(op1)
 
     def rewrite_op_ptr_ne(self, op):
-        op1 = self._rewrite_equality(op, 'ptr_nonzero')
+        prefix = ''
+        if self._is_rclass_instance(op.args[0]):
+            assert self._is_rclass_instance(op.args[1])
+            op = SpaceOperation('instance_ptr_ne', op.args, op.result)
+            prefix = 'instance_'
+        op1 = self._rewrite_equality(op, prefix + 'ptr_nonzero')
         return self._rewrite_cmp_ptrs(op1)
 
     rewrite_op_ptr_iszero = _rewrite_cmp_ptrs
@@ -783,8 +842,11 @@
 
     def rewrite_op_cast_ptr_to_int(self, op):
         if self._is_gc(op.args[0]):
-            #return op
-            raise NotImplementedError("cast_ptr_to_int")
+            return op
+
+    def rewrite_op_cast_opaque_ptr(self, op):
+        # None causes the result of this op to get aliased to op.args[0]
+        return [SpaceOperation('mark_opaque_ptr', op.args, None), None]
 
     def rewrite_op_force_cast(self, op):
         v_arg = op.args[0]
@@ -805,26 +867,44 @@
         elif not float_arg and float_res:
             # some int -> some float
             ops = []
-            v1 = varoftype(lltype.Signed)
-            oplist = self.rewrite_operation(
-                SpaceOperation('force_cast', [v_arg], v1)
-            )
-            if oplist:
-                ops.extend(oplist)
+            v2 = varoftype(lltype.Float)
+            sizesign = rffi.size_and_sign(v_arg.concretetype)
+            if sizesign <= rffi.size_and_sign(lltype.Signed):
+                # cast from a type that fits in an int: either the size is
+                # smaller, or it is equal and it is not unsigned
+                v1 = varoftype(lltype.Signed)
+                oplist = self.rewrite_operation(
+                    SpaceOperation('force_cast', [v_arg], v1)
+                )
+                if oplist:
+                    ops.extend(oplist)
+                else:
+                    v1 = v_arg
+                op = self.rewrite_operation(
+                    SpaceOperation('cast_int_to_float', [v1], v2)
+                )
+                ops.append(op)
             else:
-                v1 = v_arg
-            v2 = varoftype(lltype.Float)
-            op = self.rewrite_operation(
-                SpaceOperation('cast_int_to_float', [v1], v2)
-            )
-            ops.append(op)
+                if sizesign == rffi.size_and_sign(lltype.Unsigned):
+                    opname = 'cast_uint_to_float'
+                elif sizesign == rffi.size_and_sign(lltype.SignedLongLong):
+                    opname = 'cast_longlong_to_float'
+                elif sizesign == rffi.size_and_sign(lltype.UnsignedLongLong):
+                    opname = 'cast_ulonglong_to_float'
+                else:
+                    raise AssertionError('cast_x_to_float: %r' % (sizesign,))
+                ops1 = self.rewrite_operation(
+                    SpaceOperation(opname, [v_arg], v2)
+                )
+                if not isinstance(ops1, list): ops1 = [ops1]
+                ops.extend(ops1)
             op2 = self.rewrite_operation(
                 SpaceOperation('force_cast', [v2], v_result)
             )
             if op2:
                 ops.append(op2)
             else:
-                op.result = v_result
+                ops[-1].result = v_result
             return ops
         elif float_arg and not float_res:
             # some float -> some int
@@ -837,18 +917,36 @@
                 ops.append(op1)
             else:
                 v1 = v_arg
-            v2 = varoftype(lltype.Signed)
-            op = self.rewrite_operation(
-                SpaceOperation('cast_float_to_int', [v1], v2)
-            )
-            ops.append(op)
-            oplist = self.rewrite_operation(
-                SpaceOperation('force_cast', [v2], v_result)
-            )
-            if oplist:
-                ops.extend(oplist)
+            sizesign = rffi.size_and_sign(v_result.concretetype)
+            if sizesign <= rffi.size_and_sign(lltype.Signed):
+                # cast to a type that fits in an int: either the size is
+                # smaller, or it is equal and it is not unsigned
+                v2 = varoftype(lltype.Signed)
+                op = self.rewrite_operation(
+                    SpaceOperation('cast_float_to_int', [v1], v2)
+                )
+                ops.append(op)
+                oplist = self.rewrite_operation(
+                    SpaceOperation('force_cast', [v2], v_result)
+                )
+                if oplist:
+                    ops.extend(oplist)
+                else:
+                    op.result = v_result
             else:
-                op.result = v_result
+                if sizesign == rffi.size_and_sign(lltype.Unsigned):
+                    opname = 'cast_float_to_uint'
+                elif sizesign == rffi.size_and_sign(lltype.SignedLongLong):
+                    opname = 'cast_float_to_longlong'
+                elif sizesign == rffi.size_and_sign(lltype.UnsignedLongLong):
+                    opname = 'cast_float_to_ulonglong'
+                else:
+                    raise AssertionError('cast_float_to_x: %r' % (sizesign,))
+                ops1 = self.rewrite_operation(
+                    SpaceOperation(opname, [v1], v_result)
+                )
+                if not isinstance(ops1, list): ops1 = [ops1]
+                ops.extend(ops1)
             return ops
         else:
             assert False
@@ -1054,8 +1152,6 @@
     # The new operation is optionally further processed by rewrite_operation().
     for _old, _new in [('bool_not', 'int_is_zero'),
                        ('cast_bool_to_float', 'cast_int_to_float'),
-                       ('cast_uint_to_float', 'cast_int_to_float'),
-                       ('cast_float_to_uint', 'cast_float_to_int'),
 
                        ('int_add_nonneg_ovf', 'int_add_ovf'),
                        ('keepalive', '-live-'),
@@ -1526,6 +1622,10 @@
     def rewrite_op_jit_force_virtual(self, op):
         return self._do_builtin_call(op)
 
+    def rewrite_op_jit_is_virtual(self, op):
+        raise Exception, (
+            "'vref.virtual' should not be used from jit-visible code")
+
     def rewrite_op_jit_force_virtualizable(self, op):
         # this one is for virtualizables
         vinfo = self.get_vinfo(op.args[0])
diff --git a/pypy/jit/codewriter/support.py b/pypy/jit/codewriter/support.py
--- a/pypy/jit/codewriter/support.py
+++ b/pypy/jit/codewriter/support.py
@@ -13,7 +13,6 @@
 from pypy.translator.simplify import get_funcobj
 from pypy.translator.unsimplify import split_block
 from pypy.objspace.flow.model import Constant
-from pypy import conftest
 from pypy.translator.translator import TranslationContext
 from pypy.annotation.policy import AnnotatorPolicy
 from pypy.annotation import model as annmodel
@@ -48,15 +47,13 @@
     a.build_types(func, argtypes, main_entry_point=True)
     rtyper = t.buildrtyper(type_system = type_system)
     rtyper.specialize()
-    if inline:
-        auto_inlining(t, threshold=inline)
+    #if inline:
+    #    auto_inlining(t, threshold=inline)
     if backendoptimize:
         from pypy.translator.backendopt.all import backend_optimizations
         backend_optimizations(t, inline_threshold=inline or 0,
                 remove_asserts=True, really_remove_asserts=True)
 
-    #if conftest.option.view:
-    #    t.view()
     return rtyper
 
 def getgraph(func, values):
@@ -232,6 +229,17 @@
     else:
         return x
 
+def _ll_1_cast_uint_to_float(x):
+    # XXX on 32-bit platforms, this should be done using cast_longlong_to_float
+    # (which is a residual call right now in the x86 backend)
+    return llop.cast_uint_to_float(lltype.Float, x)
+
+def _ll_1_cast_float_to_uint(x):
+    # XXX on 32-bit platforms, this should be done using cast_float_to_longlong
+    # (which is a residual call right now in the x86 backend)
+    return llop.cast_float_to_uint(lltype.Unsigned, x)
+
+
 # math support
 # ------------
 
@@ -456,6 +464,8 @@
         return LLtypeHelpers._dictnext_items(lltype.Ptr(RES), iter)
     _ll_1_dictiter_nextitems.need_result_type = True
 
+    _ll_1_dict_resize = ll_rdict.ll_dict_resize
+
     # ---------- strings and unicode ----------
 
     _ll_1_str_str2unicode = ll_rstr.LLHelpers.ll_str2unicode
diff --git a/pypy/jit/codewriter/test/test_flatten.py b/pypy/jit/codewriter/test/test_flatten.py
--- a/pypy/jit/codewriter/test/test_flatten.py
+++ b/pypy/jit/codewriter/test/test_flatten.py
@@ -8,7 +8,7 @@
 from pypy.rpython.lltypesystem import lltype, rclass, rstr
 from pypy.objspace.flow.model import SpaceOperation, Variable, Constant
 from pypy.translator.unsimplify import varoftype
-from pypy.rlib.rarithmetic import ovfcheck, r_uint
+from pypy.rlib.rarithmetic import ovfcheck, r_uint, r_longlong, r_ulonglong
 from pypy.rlib.jit import dont_look_inside, _we_are_jitted, JitDriver
 from pypy.rlib.objectmodel import keepalive_until_here
 from pypy.rlib import jit
@@ -70,7 +70,8 @@
         return 'residual'
     def getcalldescr(self, op, oopspecindex=None, extraeffect=None):
         try:
-            if 'cannot_raise' in op.args[0].value._obj.graph.name:
+            name = op.args[0].value._obj._name
+            if 'cannot_raise' in name or name.startswith('cast_'):
                 return self._descr_cannot_raise
         except AttributeError:
             pass
@@ -900,6 +901,67 @@
             int_return %i4
         """, transform=True)
 
+        def f(dbl):
+            return rffi.cast(rffi.UCHAR, dbl)
+        self.encoding_test(f, [12.456], """
+            cast_float_to_int %f0 -> %i0
+            int_and %i0, $255 -> %i1
+            int_return %i1
+        """, transform=True)
+
+        def f(dbl):
+            return rffi.cast(lltype.Unsigned, dbl)
+        self.encoding_test(f, [12.456], """
+            residual_call_irf_i $<* fn cast_float_to_uint>, <Descr>, I[], R[], F[%f0] -> %i0
+            int_return %i0
+        """, transform=True)
+
+        def f(i):
+            return rffi.cast(lltype.Float, chr(i))    # "char -> float"
+        self.encoding_test(f, [12], """
+            cast_int_to_float %i0 -> %f0
+            float_return %f0
+        """, transform=True)
+
+        def f(i):
+            return rffi.cast(lltype.Float, r_uint(i))    # "uint -> float"
+        self.encoding_test(f, [12], """
+            residual_call_irf_f $<* fn cast_uint_to_float>, <Descr>, I[%i0], R[], F[] -> %f0
+            float_return %f0
+        """, transform=True)
+
+        if not longlong.is_64_bit:
+            def f(dbl):
+                return rffi.cast(lltype.SignedLongLong, dbl)
+            self.encoding_test(f, [12.3], """
+                residual_call_irf_f $<* fn llong_from_float>, <Descr>, I[], R[], F[%f0] -> %f1
+                float_return %f1
+            """, transform=True)
+
+            def f(dbl):
+                return rffi.cast(lltype.UnsignedLongLong, dbl)
+            self.encoding_test(f, [12.3], """
+                residual_call_irf_f $<* fn ullong_from_float>, <Descr>, I[], R[], F[%f0] -> %f1
+                float_return %f1
+            """, transform=True)
+
+            def f(x):
+                ll = r_longlong(x)
+                return rffi.cast(lltype.Float, ll)
+            self.encoding_test(f, [12], """
+                residual_call_irf_f $<* fn llong_from_int>, <Descr>, I[%i0], R[], F[] -> %f0
+                residual_call_irf_f $<* fn llong_to_float>, <Descr>, I[], R[], F[%f0] -> %f1
+                float_return %f1
+            """, transform=True)
+
+            def f(x):
+                ll = r_ulonglong(x)
+                return rffi.cast(lltype.Float, ll)
+            self.encoding_test(f, [12], """
+                residual_call_irf_f $<* fn ullong_from_int>, <Descr>, I[%i0], R[], F[] -> %f0
+                residual_call_irf_f $<* fn ullong_u_to_float>, <Descr>, I[], R[], F[%f0] -> %f1
+                float_return %f1
+            """, transform=True)
 
     def test_direct_ptradd(self):
         from pypy.rpython.lltypesystem import rffi
diff --git a/pypy/jit/codewriter/test/test_jtransform.py b/pypy/jit/codewriter/test/test_jtransform.py
--- a/pypy/jit/codewriter/test/test_jtransform.py
+++ b/pypy/jit/codewriter/test/test_jtransform.py
@@ -1,14 +1,27 @@
-import py
 import random
+try:
+    from itertools import product
+except ImportError:
+    # Python 2.5, this is taken from the CPython docs, but simplified.
+    def product(*args):
+        # product('ABCD', 'xy') --> Ax Ay Bx By Cx Cy Dx Dy
+        # product(range(2), repeat=3) --> 000 001 010 011 100 101 110 111
+        pools = map(tuple, args)
+        result = [[]]
+        for pool in pools:
+            result = [x+[y] for x in result for y in pool]
+        for prod in result:
+            yield tuple(prod)
+
 from pypy.objspace.flow.model import FunctionGraph, Block, Link
 from pypy.objspace.flow.model import SpaceOperation, Variable, Constant
-from pypy.jit.codewriter.jtransform import Transformer
-from pypy.jit.metainterp.history import getkind
-from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr, rlist
+from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr
 from pypy.rpython.lltypesystem.module import ll_math
 from pypy.translator.unsimplify import varoftype
 from pypy.jit.codewriter import heaptracker, effectinfo
 from pypy.jit.codewriter.flatten import ListOfKind
+from pypy.jit.codewriter.jtransform import Transformer
+from pypy.jit.metainterp.history import getkind
 
 def const(x):
     return Constant(x, lltype.typeOf(x))
@@ -23,6 +36,8 @@
         return ('calldescr', FUNC, ARGS, RESULT)
     def fielddescrof(self, STRUCT, name):
         return ('fielddescr', STRUCT, name)
+    def interiorfielddescrof(self, ARRAY, name):
+        return ('interiorfielddescr', ARRAY, name)
     def arraydescrof(self, ARRAY):
         return FakeDescr(('arraydescr', ARRAY))
     def sizeof(self, STRUCT):
@@ -85,6 +100,12 @@
             if i == oopspecindex:
                 return True
         return False
+    def callinfo_for_oopspec(self, oopspecindex):
+        assert oopspecindex == effectinfo.EffectInfo.OS_STREQ_NONNULL
+        class c:
+            class adr:
+                ptr = 1
+        return ('calldescr', c)
 
 class FakeBuiltinCallControl:
     def __init__(self):
@@ -105,6 +126,7 @@
              EI.OS_STR2UNICODE:([PSTR], PUNICODE),
              EI.OS_STR_CONCAT: ([PSTR, PSTR], PSTR),
              EI.OS_STR_SLICE:  ([PSTR, INT, INT], PSTR),
+             EI.OS_STREQ_NONNULL:  ([PSTR, PSTR], INT),
              EI.OS_UNI_CONCAT: ([PUNICODE, PUNICODE], PUNICODE),
              EI.OS_UNI_SLICE:  ([PUNICODE, INT, INT], PUNICODE),
              EI.OS_UNI_EQUAL:  ([PUNICODE, PUNICODE], lltype.Bool),
@@ -254,26 +276,35 @@
             assert op1.result is None
 
 def test_calls():
-    for RESTYPE in [lltype.Signed, rclass.OBJECTPTR,
-                    lltype.Float, lltype.Void]:
-      for with_void in [False, True]:
-        for with_i in [False, True]:
-          for with_r in [False, True]:
-            for with_f in [False, True]:
-              ARGS = []
-              if with_void: ARGS += [lltype.Void, lltype.Void]
-              if with_i: ARGS += [lltype.Signed, lltype.Char]
-              if with_r: ARGS += [rclass.OBJECTPTR, lltype.Ptr(rstr.STR)]
-              if with_f: ARGS += [lltype.Float, lltype.Float]
-              random.shuffle(ARGS)
-              if RESTYPE == lltype.Float: with_f = True
-              if with_f: expectedkind = 'irf'   # all kinds
-              elif with_i: expectedkind = 'ir'  # integers and references
-              else: expectedkind = 'r'          # only references
-              yield residual_call_test, ARGS, RESTYPE, expectedkind
-              yield direct_call_test, ARGS, RESTYPE, expectedkind
-              yield indirect_residual_call_test, ARGS, RESTYPE, expectedkind
-              yield indirect_regular_call_test, ARGS, RESTYPE, expectedkind
+    for RESTYPE, with_void, with_i, with_r, with_f in product(
+        [lltype.Signed, rclass.OBJECTPTR, lltype.Float, lltype.Void],
+        [False, True],
+        [False, True],
+        [False, True],
+        [False, True],
+    ):
+        ARGS = []
+        if with_void:
+            ARGS += [lltype.Void, lltype.Void]
+        if with_i:
+            ARGS += [lltype.Signed, lltype.Char]
+        if with_r:
+            ARGS += [rclass.OBJECTPTR, lltype.Ptr(rstr.STR)]
+        if with_f:
+            ARGS += [lltype.Float, lltype.Float]
+        random.shuffle(ARGS)
+        if RESTYPE == lltype.Float:
+            with_f = True
+        if with_f:
+            expectedkind = 'irf'   # all kinds
+        elif with_i:
+            expectedkind = 'ir'  # integers and references
+        else:
+            expectedkind = 'r'          # only references
+        yield residual_call_test, ARGS, RESTYPE, expectedkind
+        yield direct_call_test, ARGS, RESTYPE, expectedkind
+        yield indirect_residual_call_test, ARGS, RESTYPE, expectedkind
+        yield indirect_regular_call_test, ARGS, RESTYPE, expectedkind
 
 def get_direct_call_op(argtypes, restype):
     FUNC = lltype.FuncType(argtypes, restype)
@@ -509,7 +540,7 @@
 
 def test_rename_on_links():
     v1 = Variable()
-    v2 = Variable()
+    v2 = Variable(); v2.concretetype = llmemory.Address
     v3 = Variable()
     block = Block([v1])
     block.operations = [SpaceOperation('cast_pointer', [v1], v2)]
@@ -545,10 +576,10 @@
         assert op1.args == [v2]
 
 def test_ptr_eq():
-    v1 = varoftype(rclass.OBJECTPTR)
-    v2 = varoftype(rclass.OBJECTPTR)
+    v1 = varoftype(lltype.Ptr(rstr.STR))
+    v2 = varoftype(lltype.Ptr(rstr.STR))
     v3 = varoftype(lltype.Bool)
-    c0 = const(lltype.nullptr(rclass.OBJECT))
+    c0 = const(lltype.nullptr(rstr.STR))
     #
     for opname, reducedname in [('ptr_eq', 'ptr_iszero'),
                                 ('ptr_ne', 'ptr_nonzero')]:
@@ -567,6 +598,31 @@
         assert op1.opname == reducedname
         assert op1.args == [v2]
 
+def test_instance_ptr_eq():
+    v1 = varoftype(rclass.OBJECTPTR)
+    v2 = varoftype(rclass.OBJECTPTR)
+    v3 = varoftype(lltype.Bool)
+    c0 = const(lltype.nullptr(rclass.OBJECT))
+
+    for opname, newopname, reducedname in [
+        ('ptr_eq', 'instance_ptr_eq', 'instance_ptr_iszero'),
+        ('ptr_ne', 'instance_ptr_ne', 'instance_ptr_nonzero')
+    ]:
+        op = SpaceOperation(opname, [v1, v2], v3)
+        op1 = Transformer().rewrite_operation(op)
+        assert op1.opname == newopname
+        assert op1.args == [v1, v2]
+
+        op = SpaceOperation(opname, [v1, c0], v3)
+        op1 = Transformer().rewrite_operation(op)
+        assert op1.opname == reducedname
+        assert op1.args == [v1]
+
+        op = SpaceOperation(opname, [c0, v1], v3)
+        op1 = Transformer().rewrite_operation(op)
+        assert op1.opname == reducedname
+        assert op1.args == [v1]
+
 def test_nongc_ptr_eq():
     v1 = varoftype(rclass.NONGCOBJECTPTR)
     v2 = varoftype(rclass.NONGCOBJECTPTR)
@@ -646,6 +702,22 @@
     assert op1.args == [v, v_index]
     assert op1.result == v_result
 
+def test_dict_getinteriorfield():
+    DICT = lltype.GcArray(lltype.Struct('ENTRY', ('v', lltype.Signed),
+                                        ('k', lltype.Signed)))
+    v = varoftype(lltype.Ptr(DICT))
+    i = varoftype(lltype.Signed)
+    v_result = varoftype(lltype.Signed)
+    op = SpaceOperation('getinteriorfield', [v, i, Constant('v', lltype.Void)],
+                        v_result)
+    op1 = Transformer(FakeCPU()).rewrite_operation(op)
+    assert op1.opname == 'getinteriorfield_gc_i'
+    assert op1.args == [v, i, ('interiorfielddescr', DICT, 'v')]
+    op = SpaceOperation('getinteriorfield', [v, i, Constant('v', lltype.Void)],
+                        Constant(None, lltype.Void))
+    op1 = Transformer(FakeCPU()).rewrite_operation(op)
+    assert op1 is None
+
 def test_str_setinteriorfield():
     v = varoftype(lltype.Ptr(rstr.STR))
     v_index = varoftype(lltype.Signed)
@@ -672,6 +744,23 @@
     assert op1.args == [v, v_index, v_newchr]
     assert op1.result == v_void
 
+def test_dict_setinteriorfield():
+    DICT = lltype.GcArray(lltype.Struct('ENTRY', ('v', lltype.Signed),
+                                        ('k', lltype.Signed)))
+    v = varoftype(lltype.Ptr(DICT))
+    i = varoftype(lltype.Signed)
+    v_void = varoftype(lltype.Void)
+    op = SpaceOperation('setinteriorfield', [v, i, Constant('v', lltype.Void),
+                                             i],
+                        v_void)
+    op1 = Transformer(FakeCPU()).rewrite_operation(op)
+    assert op1.opname == 'setinteriorfield_gc_i'
+    assert op1.args == [v, i, i, ('interiorfielddescr', DICT, 'v')]
+    op = SpaceOperation('setinteriorfield', [v, i, Constant('v', lltype.Void),
+                                             v_void], v_void)
+    op1 = Transformer(FakeCPU()).rewrite_operation(op)
+    assert not op1
+
 def test_promote_1():
     v1 = varoftype(lltype.Signed)
     v2 = varoftype(lltype.Signed)
@@ -821,6 +910,21 @@
     assert op1.args[2] == ListOfKind('ref', [v1, v2])
     assert op1.result == v3
 
+def test_str_promote():
+    PSTR = lltype.Ptr(rstr.STR)
+    v1 = varoftype(PSTR)
+    v2 = varoftype(PSTR)
+    op = SpaceOperation('hint',
+                        [v1, Constant({'promote_string': True}, lltype.Void)],
+                        v2)
+    tr = Transformer(FakeCPU(), FakeBuiltinCallControl())
+    op0, op1, _ = tr.rewrite_operation(op)
+    assert op1.opname == 'str_guard_value'
+    assert op1.args[0] == v1
+    assert op1.args[2] == 'calldescr'
+    assert op1.result == v2
+    assert op0.opname == '-live-'
+
 def test_unicode_concat():
     # test that the oopspec is present and correctly transformed
     PSTR = lltype.Ptr(rstr.UNICODE)
@@ -1024,3 +1128,16 @@
                         varoftype(lltype.Signed))
     tr = Transformer(None, None)
     raises(NotImplementedError, tr.rewrite_operation, op)
+
+def test_cast_opaque_ptr():
+    S = lltype.GcStruct("S", ("x", lltype.Signed))
+    v1 = varoftype(lltype.Ptr(S))
+    v2 = varoftype(lltype.Ptr(rclass.OBJECT))
+
+    op = SpaceOperation('cast_opaque_ptr', [v1], v2)
+    tr = Transformer()
+    [op1, op2] = tr.rewrite_operation(op)
+    assert op1.opname == 'mark_opaque_ptr'
+    assert op1.args == [v1]
+    assert op1.result is None
+    assert op2 is None
\ No newline at end of file
diff --git a/pypy/jit/metainterp/blackhole.py b/pypy/jit/metainterp/blackhole.py
--- a/pypy/jit/metainterp/blackhole.py
+++ b/pypy/jit/metainterp/blackhole.py
@@ -2,11 +2,10 @@
 from pypy.rlib.rtimer import read_timestamp
 from pypy.rlib.rarithmetic import intmask, LONG_BIT, r_uint, ovfcheck
 from pypy.rlib.objectmodel import we_are_translated
-from pypy.rlib.debug import debug_start, debug_stop
+from pypy.rlib.debug import debug_start, debug_stop, ll_assert
 from pypy.rlib.debug import make_sure_not_resized
 from pypy.rpython.lltypesystem import lltype, llmemory, rclass
 from pypy.rpython.lltypesystem.lloperation import llop
-from pypy.rpython.llinterp import LLException
 from pypy.jit.codewriter.jitcode import JitCode, SwitchDictDescr
 from pypy.jit.codewriter import heaptracker, longlong
 from pypy.jit.metainterp.jitexc import JitException, get_llexception, reraise
@@ -500,9 +499,25 @@
     @arguments("r", returns="i")
     def bhimpl_ptr_nonzero(a):
         return bool(a)
-    @arguments("r", returns="r")
-    def bhimpl_cast_opaque_ptr(a):
-        return a
+    @arguments("r", "r", returns="i")
+    def bhimpl_instance_ptr_eq(a, b):
+        return a == b
+    @arguments("r", "r", returns="i")
+    def bhimpl_instance_ptr_ne(a, b):
+        return a != b
+    @arguments("r", returns="i")
+    def bhimpl_cast_ptr_to_int(a):
+        i = lltype.cast_ptr_to_int(a)
+        ll_assert((i & 1) == 1, "bhimpl_cast_ptr_to_int: not an odd int")
+        return i
+    @arguments("i", returns="r")
+    def bhimpl_cast_int_to_ptr(i):
+        ll_assert((i & 1) == 1, "bhimpl_cast_int_to_ptr: not an odd int")
+        return lltype.cast_int_to_ptr(llmemory.GCREF, i)
+
+    @arguments("r")
+    def bhimpl_mark_opaque_ptr(a):
+        pass
 
     @arguments("i", returns="i")
     def bhimpl_int_copy(a):
@@ -523,6 +538,9 @@
     @arguments("f")
     def bhimpl_float_guard_value(a):
         pass
+    @arguments("r", "i", "d", returns="r")
+    def bhimpl_str_guard_value(a, i, d):
+        return a
 
     @arguments("self", "i")
     def bhimpl_int_push(self, a):
@@ -619,6 +637,9 @@
         a = longlong.getrealfloat(a)
         # note: we need to call int() twice to care for the fact that
         # int(-2147483648.0) returns a long :-(
+        # we could also call intmask() instead of the outermost int(), but
+        # it's probably better to explicitly crash (by getting a long) if a
+        # non-translated version tries to cast a too large float to an int.
         return int(int(a))
 
     @arguments("i", returns="f")
@@ -1142,6 +1163,26 @@
         array = cpu.bh_getfield_gc_r(vable, fdescr)
         return cpu.bh_arraylen_gc(adescr, array)
 
+    @arguments("cpu", "r", "i", "d", returns="i")
+    def bhimpl_getinteriorfield_gc_i(cpu, array, index, descr):
+        return cpu.bh_getinteriorfield_gc_i(array, index, descr)
+    @arguments("cpu", "r", "i", "d", returns="r")
+    def bhimpl_getinteriorfield_gc_r(cpu, array, index, descr):
+        return cpu.bh_getinteriorfield_gc_r(array, index, descr)
+    @arguments("cpu", "r", "i", "d", returns="f")
+    def bhimpl_getinteriorfield_gc_f(cpu, array, index, descr):
+        return cpu.bh_getinteriorfield_gc_f(array, index, descr)
+
+    @arguments("cpu", "r", "i", "d", "i")
+    def bhimpl_setinteriorfield_gc_i(cpu, array, index, descr, value):
+        cpu.bh_setinteriorfield_gc_i(array, index, descr, value)
+    @arguments("cpu", "r", "i", "d", "r")
+    def bhimpl_setinteriorfield_gc_r(cpu, array, index, descr, value):
+        cpu.bh_setinteriorfield_gc_r(array, index, descr, value)
+    @arguments("cpu", "r", "i", "d", "f")
+    def bhimpl_setinteriorfield_gc_f(cpu, array, index, descr, value):
+        cpu.bh_setinteriorfield_gc_f(array, index, descr, value)
+
     @arguments("cpu", "r", "d", returns="i")
     def bhimpl_getfield_gc_i(cpu, struct, fielddescr):
         return cpu.bh_getfield_gc_i(struct, fielddescr)
diff --git a/pypy/jit/metainterp/executor.py b/pypy/jit/metainterp/executor.py
--- a/pypy/jit/metainterp/executor.py
+++ b/pypy/jit/metainterp/executor.py
@@ -1,11 +1,8 @@
 """This implements pyjitpl's execution of operations.
 """
 
-import py
-from pypy.rpython.lltypesystem import lltype, llmemory, rstr
-from pypy.rpython.ootypesystem import ootype
-from pypy.rpython.lltypesystem.lloperation import llop
-from pypy.rlib.rarithmetic import ovfcheck, r_uint, intmask, r_longlong
+from pypy.rpython.lltypesystem import lltype, rstr
+from pypy.rlib.rarithmetic import ovfcheck, r_longlong
 from pypy.rlib.rtimer import read_timestamp
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.jit.metainterp.history import BoxInt, BoxPtr, BoxFloat, check_descr
@@ -123,6 +120,29 @@
     else:
         cpu.bh_setarrayitem_raw_i(arraydescr, array, index, itembox.getint())
 
+def do_getinteriorfield_gc(cpu, _, arraybox, indexbox, descr):
+    array = arraybox.getref_base()
+    index = indexbox.getint()
+    if descr.is_pointer_field():
+        return BoxPtr(cpu.bh_getinteriorfield_gc_r(array, index, descr))
+    elif descr.is_float_field():
+        return BoxFloat(cpu.bh_getinteriorfield_gc_f(array, index, descr))
+    else:
+        return BoxInt(cpu.bh_getinteriorfield_gc_i(array, index, descr))
+
+def do_setinteriorfield_gc(cpu, _, arraybox, indexbox, valuebox, descr):
+    array = arraybox.getref_base()
+    index = indexbox.getint()
+    if descr.is_pointer_field():
+        cpu.bh_setinteriorfield_gc_r(array, index, descr,
+                                     valuebox.getref_base())
+    elif descr.is_float_field():
+        cpu.bh_setinteriorfield_gc_f(array, index, descr,
+                                     valuebox.getfloatstorage())
+    else:
+        cpu.bh_setinteriorfield_gc_i(array, index, descr,
+                                     valuebox.getint())
+
 def do_getfield_gc(cpu, _, structbox, fielddescr):
     struct = structbox.getref_base()
     if fielddescr.is_pointer_field():
diff --git a/pypy/jit/metainterp/graphpage.py b/pypy/jit/metainterp/graphpage.py
--- a/pypy/jit/metainterp/graphpage.py
+++ b/pypy/jit/metainterp/graphpage.py
@@ -12,8 +12,8 @@
     def get_display_text(self):
         return None
 
-def display_loops(loops, errmsg=None, highlight_loops=()):
-    graphs = [(loop, loop in highlight_loops) for loop in loops]    
+def display_loops(loops, errmsg=None, highlight_loops={}):
+    graphs = [(loop, highlight_loops.get(loop, 0)) for loop in loops]    
     for graph, highlight in graphs:
         for op in graph.get_operations():
             if is_interesting_guard(op):
@@ -65,8 +65,7 @@
     def add_graph(self, graph, highlight=False):
         graphindex = len(self.graphs)
         self.graphs.append(graph)
-        if highlight:
-            self.highlight_graphs[graph] = True
+        self.highlight_graphs[graph] = highlight
         for i, op in enumerate(graph.get_operations()):
             self.all_operations[op] = graphindex, i
 
@@ -126,10 +125,13 @@
             self.dotgen.emit('subgraph cluster%d {' % graphindex)
         label = graph.get_display_text()
         if label is not None:
-            if self.highlight_graphs.get(graph):
-                fillcolor = '#f084c2'
+            colorindex = self.highlight_graphs.get(graph, 0)
+            if colorindex == 1:
+                fillcolor = '#f084c2'    # highlighted graph
+            elif colorindex == 2:
+                fillcolor = '#808080'    # invalidated graph
             else:
-                fillcolor = '#84f0c2'
+                fillcolor = '#84f0c2'    # normal color
             self.dotgen.emit_node(graphname, shape="octagon",
                                   label=label, fillcolor=fillcolor)
             self.pendingedges.append((graphname,
diff --git a/pypy/jit/metainterp/heapcache.py b/pypy/jit/metainterp/heapcache.py
--- a/pypy/jit/metainterp/heapcache.py
+++ b/pypy/jit/metainterp/heapcache.py
@@ -34,7 +34,6 @@
         self.clear_caches(opnum, descr, argboxes)
 
     def mark_escaped(self, opnum, argboxes):
-        idx = 0
         if opnum == rop.SETFIELD_GC:
             assert len(argboxes) == 2
             box, valuebox = argboxes
@@ -42,8 +41,20 @@
                 self.dependencies.setdefault(box, []).append(valuebox)
             else:
                 self._escape(valuebox)
-        # GETFIELD_GC doesn't escape it's argument
-        elif opnum != rop.GETFIELD_GC:
+        elif opnum == rop.SETARRAYITEM_GC:
+            assert len(argboxes) == 3
+            box, indexbox, valuebox = argboxes
+            if self.is_unescaped(box) and self.is_unescaped(valuebox):
+                self.dependencies.setdefault(box, []).append(valuebox)
+            else:
+                self._escape(valuebox)
+        # GETFIELD_GC, MARK_OPAQUE_PTR, PTR_EQ, and PTR_NE don't escape their
+        # arguments
+        elif (opnum != rop.GETFIELD_GC and
+              opnum != rop.MARK_OPAQUE_PTR and
+              opnum != rop.PTR_EQ and
+              opnum != rop.PTR_NE):
+            idx = 0
             for box in argboxes:
                 # setarrayitem_gc don't escape its first argument
                 if not (idx == 0 and opnum in [rop.SETARRAYITEM_GC]):
@@ -54,18 +65,19 @@
         if box in self.new_boxes:
             self.new_boxes[box] = False
         if box in self.dependencies:
-            for dep in self.dependencies[box]:
+            deps = self.dependencies[box]
+            del self.dependencies[box]
+            for dep in deps:
                 self._escape(dep)
-            del self.dependencies[box]
 
     def clear_caches(self, opnum, descr, argboxes):
-        if opnum == rop.SETFIELD_GC:
-            return
-        if opnum == rop.SETARRAYITEM_GC:
-            return
-        if opnum == rop.SETFIELD_RAW:
-            return
-        if opnum == rop.SETARRAYITEM_RAW:
+        if (opnum == rop.SETFIELD_GC or
+            opnum == rop.SETARRAYITEM_GC or
+            opnum == rop.SETFIELD_RAW or
+            opnum == rop.SETARRAYITEM_RAW or
+            opnum == rop.SETINTERIORFIELD_GC or
+            opnum == rop.COPYSTRCONTENT or
+            opnum == rop.COPYUNICODECONTENT):
             return
         if rop._OVF_FIRST <= opnum <= rop._OVF_LAST:
             return
@@ -74,9 +86,9 @@
         if opnum == rop.CALL or opnum == rop.CALL_LOOPINVARIANT:
             effectinfo = descr.get_extra_info()
             ef = effectinfo.extraeffect
-            if ef == effectinfo.EF_LOOPINVARIANT or \
-               ef == effectinfo.EF_ELIDABLE_CANNOT_RAISE or \
-               ef == effectinfo.EF_ELIDABLE_CAN_RAISE:
+            if (ef == effectinfo.EF_LOOPINVARIANT or
+                ef == effectinfo.EF_ELIDABLE_CANNOT_RAISE or
+                ef == effectinfo.EF_ELIDABLE_CAN_RAISE):
                 return
             # A special case for ll_arraycopy, because it is so common, and its
             # effects are so well defined.
diff --git a/pypy/jit/metainterp/history.py b/pypy/jit/metainterp/history.py
--- a/pypy/jit/metainterp/history.py
+++ b/pypy/jit/metainterp/history.py
@@ -9,12 +9,14 @@
 
 from pypy.jit.metainterp.resoperation import ResOperation, rop
 from pypy.jit.codewriter import heaptracker, longlong
+from pypy.rlib.objectmodel import compute_identity_hash
 
 # ____________________________________________________________
 
 INT   = 'i'
 REF   = 'r'
 FLOAT = 'f'
+STRUCT = 's'
 HOLE  = '_'
 VOID  = 'v'
 
@@ -104,7 +106,7 @@
     getref._annspecialcase_ = 'specialize:arg(1)'
 
     def _get_hash_(self):
-        raise NotImplementedError
+        return compute_identity_hash(self)
 
     def clonebox(self):
         raise NotImplementedError
@@ -133,6 +135,9 @@
     def _get_str(self):
         raise NotImplementedError
 
+    def same_box(self, other):
+        return self is other
+
 class AbstractDescr(AbstractValue):
     __slots__ = ()
 
@@ -168,6 +173,11 @@
         """
         raise NotImplementedError
 
+    def is_array_of_structs(self):
+        """ Implement for array descr
+        """
+        raise NotImplementedError
+
     def is_pointer_field(self):
         """ Implement for field descr
         """
@@ -241,32 +251,15 @@
     def constbox(self):
         return self
 
+    def same_box(self, other):
+        return self.same_constant(other)
+
     def same_constant(self, other):
         raise NotImplementedError
 
     def __repr__(self):
         return 'Const(%s)' % self._getrepr_()
 
-    def __eq__(self, other):
-        "NOT_RPYTHON"
-        # Remember that you should not compare Consts with '==' in RPython.
-        # Consts have no special __hash__, in order to force different Consts
-        # from being considered as different keys when stored in dicts
-        # (as they always are after translation).  Use a dict_equal_consts()
-        # to get the other behavior (i.e. using this __eq__).
-        if self.__class__ is not other.__class__:
-            return False
-        try:
-            return self.value == other.value
-        except TypeError:
-            if (isinstance(self.value, Symbolic) and
-                isinstance(other.value, Symbolic)):
-                return self.value is other.value
-            raise
-
-    def __ne__(self, other):
-        return not (self == other)
-
 
 class ConstInt(Const):
     type = INT
@@ -688,33 +681,6 @@
 
 # ____________________________________________________________
 
-def dict_equal_consts():
-    "NOT_RPYTHON"
-    # Returns a dict in which Consts that compare as equal
-    # are identified when used as keys.
-    return r_dict(dc_eq, dc_hash)
-
-def dc_eq(c1, c2):
-    return c1 == c2
-
-def dc_hash(c):
-    "NOT_RPYTHON"
-    # This is called during translation only.  Avoid using identityhash(),
-    # to avoid forcing a hash, at least on lltype objects.
-    if not isinstance(c, Const):
-        return hash(c)
-    if isinstance(c.value, Symbolic):
-        return id(c.value)
-    try:
-        if isinstance(c, ConstPtr):
-            p = lltype.normalizeptr(c.value)
-            if p is not None:
-                return hash(p._obj)
-            else:
-                return 0
-        return c._get_hash_()
-    except lltype.DelayedPointer:
-        return -2      # xxx risk of changing hash...
 
 def make_hashable_int(i):
     from pypy.rpython.lltypesystem.ll2ctypes import NotCtypesAllocatedStructure
@@ -772,6 +738,7 @@
     failed_states = None
     retraced_count = 0
     terminating = False # see TerminatingLoopToken in compile.py
+    invalidated = False
     outermost_jitdriver_sd = None
     # and more data specified by the backend when the loop is compiled
     number = -1
@@ -974,6 +941,7 @@
         self.loops = []
         self.locations = []
         self.aborted_keys = []
+        self.invalidated_token_numbers = set()
 
     def set_history(self, history):
         self.operations = history.operations
@@ -1052,7 +1020,12 @@
             if loop in loops:
                 loops.remove(loop)
             loops.append(loop)
-        display_loops(loops, errmsg, extraloops)
+        highlight_loops = dict.fromkeys(extraloops, 1)
+        for loop in loops:
+            if hasattr(loop, '_looptoken_number') and (
+                    loop._looptoken_number in self.invalidated_token_numbers):
+                highlight_loops.setdefault(loop, 2)
+        display_loops(loops, errmsg, highlight_loops)
 
 # ----------------------------------------------------------------
 
diff --git a/pypy/jit/metainterp/memmgr.py b/pypy/jit/metainterp/memmgr.py
--- a/pypy/jit/metainterp/memmgr.py
+++ b/pypy/jit/metainterp/memmgr.py
@@ -68,7 +68,8 @@
         debug_print("Loop tokens before:", oldtotal)
         max_generation = self.current_generation - (self.max_age-1)
         for looptoken in self.alive_loops.keys():
-            if 0 <= looptoken.generation < max_generation:
+            if (0 <= looptoken.generation < max_generation or
+                looptoken.invalidated):
                 del self.alive_loops[looptoken]
         newtotal = len(self.alive_loops)
         debug_print("Loop tokens freed: ", oldtotal - newtotal)
diff --git a/pypy/jit/metainterp/optimizeopt/optimizer.py b/pypy/jit/metainterp/optimizeopt/optimizer.py
--- a/pypy/jit/metainterp/optimizeopt/optimizer.py
+++ b/pypy/jit/metainterp/optimizeopt/optimizer.py
@@ -209,13 +209,19 @@
     def setfield(self, ofs, value):
         raise NotImplementedError
 
+    def getlength(self):
+        raise NotImplementedError
+
     def getitem(self, index):
         raise NotImplementedError
 
-    def getlength(self):
+    def setitem(self, index, value):
         raise NotImplementedError
 
-    def setitem(self, index, value):
+    def getinteriorfield(self, index, ofs, default):
+        raise NotImplementedError
+
+    def setinteriorfield(self, index, ofs, value):
         raise NotImplementedError
 
 
@@ -283,11 +289,11 @@
             return self.optimizer.optpure.has_pure_result(opnum, args, descr)
         return False
 
-    def get_pure_result(self, key):    
+    def get_pure_result(self, key):
         if self.optimizer.optpure:
             return self.optimizer.optpure.get_pure_result(key)
         return None
-    
+
     def setup(self):
         pass
 
@@ -524,7 +530,7 @@
 
     def replace_op(self, old_op, new_op):
         # XXX: Do we want to cache indexes to prevent search?
-        i = len(self._newoperations) 
+        i = len(self._newoperations)
         while i > 0:
             i -= 1
             if self._newoperations[i] is old_op:
diff --git a/pypy/jit/metainterp/optimizeopt/rewrite.py b/pypy/jit/metainterp/optimizeopt/rewrite.py
--- a/pypy/jit/metainterp/optimizeopt/rewrite.py
+++ b/pypy/jit/metainterp/optimizeopt/rewrite.py
@@ -106,10 +106,9 @@
             self.make_equal_to(op.result, v1)
         else:
             self.emit_operation(op)
-
-        # Synthesize the reverse ops for optimize_default to reuse
-        self.pure(rop.INT_ADD, [op.result, op.getarg(1)], op.getarg(0))
-        self.pure(rop.INT_SUB, [op.getarg(0), op.result], op.getarg(1))
+            # Synthesize the reverse ops for optimize_default to reuse
+            self.pure(rop.INT_ADD, [op.result, op.getarg(1)], op.getarg(0))
+            self.pure(rop.INT_SUB, [op.getarg(0), op.result], op.getarg(1))
 
     def optimize_INT_ADD(self, op):
         v1 = self.getvalue(op.getarg(0))
@@ -122,10 +121,9 @@
             self.make_equal_to(op.result, v1)
         else:
             self.emit_operation(op)
-
-        # Synthesize the reverse op for optimize_default to reuse
-        self.pure(rop.INT_SUB, [op.result, op.getarg(1)], op.getarg(0))
-        self.pure(rop.INT_SUB, [op.result, op.getarg(0)], op.getarg(1))
+            # Synthesize the reverse op for optimize_default to reuse
+            self.pure(rop.INT_SUB, [op.result, op.getarg(1)], op.getarg(0))
+            self.pure(rop.INT_SUB, [op.result, op.getarg(0)], op.getarg(1))
 
     def optimize_INT_MUL(self, op):
         v1 = self.getvalue(op.getarg(0))
@@ -141,13 +139,13 @@
             self.make_constant_int(op.result, 0)
         else:
             for lhs, rhs in [(v1, v2), (v2, v1)]:
-                # x & (x -1) == 0 is a quick test for power of 2
-                if (lhs.is_constant() and
-                    (lhs.box.getint() & (lhs.box.getint() - 1)) == 0):
-                    new_rhs = ConstInt(highest_bit(lhs.box.getint()))
-                    op = op.copy_and_change(rop.INT_LSHIFT, args=[rhs.box, new_rhs])
-                    break
-
+                if lhs.is_constant():
+                    x = lhs.box.getint()
+                    # x & (x - 1) == 0 is a quick test for power of 2
+                    if x & (x - 1) == 0:
+                        new_rhs = ConstInt(highest_bit(lhs.box.getint()))
+                        op = op.copy_and_change(rop.INT_LSHIFT, args=[rhs.box, new_rhs])
+                        break
             self.emit_operation(op)
 
     def optimize_UINT_FLOORDIV(self, op):
@@ -339,7 +337,7 @@
     def optimize_INT_IS_ZERO(self, op):
         self._optimize_nullness(op, op.getarg(0), False)
 
-    def _optimize_oois_ooisnot(self, op, expect_isnot):
+    def _optimize_oois_ooisnot(self, op, expect_isnot, instance):
         value0 = self.getvalue(op.getarg(0))
         value1 = self.getvalue(op.getarg(1))
         if value0.is_virtual():
@@ -357,21 +355,28 @@
         elif value0 is value1:
             self.make_constant_int(op.result, not expect_isnot)
         else:
-            cls0 = value0.get_constant_class(self.optimizer.cpu)
-            if cls0 is not None:
-                cls1 = value1.get_constant_class(self.optimizer.cpu)
-                if cls1 is not None and not cls0.same_constant(cls1):
-                    # cannot be the same object, as we know that their
-                    # class is different
-                    self.make_constant_int(op.result, expect_isnot)
-                    return
+            if instance:
+                cls0 = value0.get_constant_class(self.optimizer.cpu)
+                if cls0 is not None:
+                    cls1 = value1.get_constant_class(self.optimizer.cpu)
+                    if cls1 is not None and not cls0.same_constant(cls1):
+                        # cannot be the same object, as we know that their
+                        # class is different
+                        self.make_constant_int(op.result, expect_isnot)
+                        return
             self.emit_operation(op)
 
+    def optimize_PTR_EQ(self, op):
+        self._optimize_oois_ooisnot(op, False, False)
+
     def optimize_PTR_NE(self, op):
-        self._optimize_oois_ooisnot(op, True)
+        self._optimize_oois_ooisnot(op, True, False)
 
-    def optimize_PTR_EQ(self, op):
-        self._optimize_oois_ooisnot(op, False)
+    def optimize_INSTANCE_PTR_EQ(self, op):
+        self._optimize_oois_ooisnot(op, False, True)
+
+    def optimize_INSTANCE_PTR_NE(self, op):
+        self._optimize_oois_ooisnot(op, True, True)
 
 ##    def optimize_INSTANCEOF(self, op):
 ##        value = self.getvalue(op.args[0])
@@ -450,6 +455,9 @@
         if v2.is_constant() and v2.box.getint() == 1:
             self.make_equal_to(op.result, v1)
             return
+        elif v1.is_constant() and v1.box.getint() == 0:
+            self.make_constant_int(op.result, 0)
+            return
         if v1.intbound.known_ge(IntBound(0, 0)) and v2.is_constant():
             val = v2.box.getint()
             if val & (val - 1) == 0 and val > 0: # val == 2**shift
@@ -457,10 +465,17 @@
                                         args = [op.getarg(0), ConstInt(highest_bit(val))])
         self.emit_operation(op)
 
-    def optimize_CAST_OPAQUE_PTR(self, op):
+    def optimize_MARK_OPAQUE_PTR(self, op):
         value = self.getvalue(op.getarg(0))
         self.optimizer.opaque_pointers[value] = True
-        self.make_equal_to(op.result, value)
+
+    def optimize_CAST_PTR_TO_INT(self, op):
+        self.pure(rop.CAST_INT_TO_PTR, [op.result], op.getarg(0))
+        self.emit_operation(op)
+
+    def optimize_CAST_INT_TO_PTR(self, op):
+        self.pure(rop.CAST_PTR_TO_INT, [op.result], op.getarg(0))
+        self.emit_operation(op)
 
 dispatch_opt = make_dispatcher_method(OptRewrite, 'optimize_',
         default=OptRewrite.emit_operation)
diff --git a/pypy/jit/metainterp/optimizeopt/simplify.py b/pypy/jit/metainterp/optimizeopt/simplify.py
--- a/pypy/jit/metainterp/optimizeopt/simplify.py
+++ b/pypy/jit/metainterp/optimizeopt/simplify.py
@@ -25,7 +25,8 @@
         #     but it's a bit hard to implement robustly if heap.py is also run
         pass
 
-    optimize_CAST_OPAQUE_PTR = optimize_VIRTUAL_REF
+    def optimize_MARK_OPAQUE_PTR(self, op):
+        pass
 
 
 dispatch_opt = make_dispatcher_method(OptSimplify, 'optimize_',
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -508,13 +508,13 @@
         ops = """
         [p0]
         guard_class(p0, ConstClass(node_vtable)) []
-        i0 = ptr_ne(p0, NULL)
+        i0 = instance_ptr_ne(p0, NULL)
         guard_true(i0) []
-        i1 = ptr_eq(p0, NULL)
+        i1 = instance_ptr_eq(p0, NULL)
         guard_false(i1) []
-        i2 = ptr_ne(NULL, p0)
+        i2 = instance_ptr_ne(NULL, p0)
         guard_true(i0) []
-        i3 = ptr_eq(NULL, p0)
+        i3 = instance_ptr_eq(NULL, p0)
         guard_false(i1) []
         jump(p0)
         """
@@ -935,7 +935,6 @@
         """
         self.optimize_loop(ops, expected)
 
-
     def test_virtual_constant_isnonnull(self):
         ops = """
         [i0]
@@ -951,6 +950,32 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_virtual_array_of_struct(self):
+        ops = """
+        [f0, f1, f2, f3]
+        p0 = new_array(2, descr=complexarraydescr)
+        setinteriorfield_gc(p0, 0, f0, descr=complexrealdescr)
+        setinteriorfield_gc(p0, 0, f1, descr=compleximagdescr)
+        setinteriorfield_gc(p0, 1, f2, descr=complexrealdescr)
+        setinteriorfield_gc(p0, 1, f3, descr=compleximagdescr)
+        f4 = getinteriorfield_gc(p0, 0, descr=complexrealdescr)
+        f5 = getinteriorfield_gc(p0, 1, descr=complexrealdescr)
+        f6 = float_mul(f4, f5)
+        f7 = getinteriorfield_gc(p0, 0, descr=compleximagdescr)
+        f8 = getinteriorfield_gc(p0, 1, descr=compleximagdescr)
+        f9 = float_mul(f7, f8)
+        f10 = float_add(f6, f9)
+        finish(f10)
+        """
+        expected = """
+        [f0, f1, f2, f3]
+        f4 = float_mul(f0, f2)
+        f5 = float_mul(f1, f3)
+        f6 = float_add(f4, f5)
+        finish(f6)
+        """
+        self.optimize_loop(ops, expected)
+
     def test_nonvirtual_1(self):
         ops = """
         [i]
@@ -2026,7 +2051,7 @@
         ops = """
         [p1]
         guard_class(p1, ConstClass(node_vtable2)) []
-        i = ptr_ne(ConstPtr(myptr), p1)
+        i = instance_ptr_ne(ConstPtr(myptr), p1)
         guard_true(i) []
         jump(p1)
         """
@@ -2181,6 +2206,17 @@
         """
         self.optimize_loop(ops, expected)
 
+        ops = """
+        [i0]
+        i1 = int_floordiv(0, i0)
+        jump(i1)
+        """
+        expected = """
+        [i0]
+        jump(0)
+        """
+        self.optimize_loop(ops, expected)
+
     def test_fold_partially_constant_ops_ovf(self):
         ops = """
         [i0]
@@ -2329,7 +2365,7 @@
         def _variables_equal(box, varname, strict):
             if varname not in virtuals:
                 if strict:
-                    assert box == oparse.getvar(varname)
+                    assert box.same_box(oparse.getvar(varname))
                 else:
                     assert box.value == oparse.getvar(varname).value
             else:
@@ -4170,10 +4206,12 @@
         class FakeCallInfoCollection:
             def callinfo_for_oopspec(self, oopspecindex):
                 calldescrtype = type(LLtypeMixin.strequaldescr)
+                effectinfotype = type(LLtypeMixin.strequaldescr.get_extra_info())
                 for value in LLtypeMixin.__dict__.values():
                     if isinstance(value, calldescrtype):
                         extra = value.get_extra_info()
-                        if extra and extra.oopspecindex == oopspecindex:
+                        if (extra and isinstance(extra, effectinfotype) and
+                            extra.oopspecindex == oopspecindex):
                             # returns 0 for 'func' in this test
                             return value, 0
                 raise AssertionError("not found: oopspecindex=%d" %
@@ -4789,6 +4827,18 @@
         """
         self.optimize_strunicode_loop(ops, expected)
 
+    def test_ptr_eq_str_constant(self):
+        ops = """
+        []
+        i0 = ptr_eq(s"abc", s"\x00")
+        finish(i0)
+        """
+        expected = """
+        []
+        finish(0)
+        """
+        self.optimize_loop(ops, expected)
+
 
 class TestLLtype(BaseTestOptimizeBasic, LLtypeMixin):
     pass
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -234,6 +234,30 @@
             """ % expected_value
             self.optimize_loop(ops, expected)
 
+    def test_reverse_of_cast(self):
+        ops = """
+        [i0]
+        p0 = cast_int_to_ptr(i0)
+        i1 = cast_ptr_to_int(p0)
+        jump(i1)
+        """
+        expected = """
+        [i0]
+        jump(i0)
+        """
+        self.optimize_loop(ops, expected)
+        ops = """
+        [p0]
+        i1 = cast_ptr_to_int(p0)
+        p1 = cast_int_to_ptr(i1)
+        jump(p1)
+        """
+        expected = """
+        [p0]
+        jump(p0)
+        """
+        self.optimize_loop(ops, expected)
+
     # ----------
 
     def test_remove_guard_class_1(self):
@@ -2659,7 +2683,7 @@
         ops = """
         [p1]
         guard_class(p1, ConstClass(node_vtable2)) []
-        i = ptr_ne(ConstPtr(myptr), p1)
+        i = instance_ptr_ne(ConstPtr(myptr), p1)
         guard_true(i) []
         jump(p1)
         """
@@ -3307,7 +3331,7 @@
         jump(p1, i1, i2, i6)
         '''
         self.optimize_loop(ops, expected, preamble)
-        
+
 
     # ----------
 
@@ -5776,10 +5800,12 @@
         class FakeCallInfoCollection:
             def callinfo_for_oopspec(self, oopspecindex):
                 calldescrtype = type(LLtypeMixin.strequaldescr)
+                effectinfotype = type(LLtypeMixin.strequaldescr.get_extra_info())
                 for value in LLtypeMixin.__dict__.values():
                     if isinstance(value, calldescrtype):
                         extra = value.get_extra_info()
-                        if extra and extra.oopspecindex == oopspecindex:
+                        if (extra and isinstance(extra, effectinfotype) and
+                            extra.oopspecindex == oopspecindex):
                             # returns 0 for 'func' in this test
                             return value, 0
                 raise AssertionError("not found: oopspecindex=%d" %
@@ -7256,7 +7282,7 @@
         ops = """
         [p1, p2]
         setarrayitem_gc(p1, 2, 10, descr=arraydescr)
-        setarrayitem_gc(p2, 3, 13, descr=arraydescr)        
+        setarrayitem_gc(p2, 3, 13, descr=arraydescr)
         call(0, p1, p2, 0, 0, 10, descr=arraycopydescr)
         jump(p1, p2)
         """
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_util.py b/pypy/jit/metainterp/optimizeopt/test/test_util.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_util.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_util.py
@@ -185,6 +185,18 @@
              EffectInfo([], [arraydescr], [], [arraydescr],
                         oopspecindex=EffectInfo.OS_ARRAYCOPY))
 
+
+    # array of structs (complex data)
+    complexarray = lltype.GcArray(
+        lltype.Struct("complex",
+            ("real", lltype.Float),
+            ("imag", lltype.Float),
+        )
+    )
+    complexarraydescr = cpu.arraydescrof(complexarray)
+    complexrealdescr = cpu.interiorfielddescrof(complexarray, "real")
+    compleximagdescr = cpu.interiorfielddescrof(complexarray, "imag")
+
     for _name, _os in [
         ('strconcatdescr',               'OS_STR_CONCAT'),
         ('strslicedescr',                'OS_STR_SLICE'),
@@ -240,7 +252,7 @@
 ##    def get_class_of_box(self, box):
 ##        root = box.getref(ootype.ROOT)
 ##        return ootype.classof(root)
-    
+
 ##    cpu = runner.OOtypeCPU(None)
 ##    NODE = ootype.Instance('NODE', ootype.ROOT, {})
 ##    NODE._add_fields({'value': ootype.Signed,
diff --git a/pypy/jit/metainterp/optimizeopt/util.py b/pypy/jit/metainterp/optimizeopt/util.py
--- a/pypy/jit/metainterp/optimizeopt/util.py
+++ b/pypy/jit/metainterp/optimizeopt/util.py
@@ -90,14 +90,11 @@
     for i in range(len(args1)):
         arg1 = args1[i]
         arg2 = args2[i]
-        if isinstance(arg1, history.Const):
-            if arg1.__class__ is not arg2.__class__:
+        if arg1 is None:
+            if arg2 is not None:
                 return False
-            if not arg1.same_constant(arg2):
-                return False
-        else:
-            if not arg1 is arg2:
-                return False
+        elif not arg1.same_box(arg2):
+            return False
     return True
 
 def args_hash(args):
@@ -106,10 +103,8 @@
     for arg in args:
         if arg is None:
             y = 17
-        elif isinstance(arg, history.Const):
+        else:
             y = arg._get_hash_()
-        else:
-            y = compute_identity_hash(arg)
         res = intmask((1000003 * res) ^ y)
     return res
 
@@ -145,9 +140,12 @@
         for i in range(op1.numargs()):
             x = op1.getarg(i)
             y = op2.getarg(i)
-            assert x == remap.get(y, y)
+            assert x.same_box(remap.get(y, y))
         if op2.result in remap:
-            assert op1.result == remap[op2.result]
+            if op2.result is None:
+                assert op1.result == remap[op2.result]
+            else:
+                assert op1.result.same_box(remap[op2.result])
         else:
             remap[op2.result] = op1.result
         if op1.getopnum() != rop.JUMP:      # xxx obscure
@@ -156,11 +154,20 @@
             assert len(op1.getfailargs()) == len(op2.getfailargs())
             if strict_fail_args:
                 for x, y in zip(op1.getfailargs(), op2.getfailargs()):
-                    assert x == remap.get(y, y)
+                    if x is None:
+                        assert remap.get(y, y) is None
+                    else:
+                        assert x.same_box(remap.get(y, y))
             else:
                 fail_args1 = set(op1.getfailargs())
                 fail_args2 = set([remap.get(y, y) for y in op2.getfailargs()])
-                assert fail_args1 == fail_args2
+                for x in fail_args1:
+                    for y in fail_args2:
+                        if x.same_box(y):
+                            fail_args2.remove(y)
+                            break
+                    else:
+                        assert False
     assert len(oplist1) == len(oplist2)
     print '-'*totwidth
     return True
diff --git a/pypy/jit/metainterp/optimizeopt/virtualize.py b/pypy/jit/metainterp/optimizeopt/virtualize.py
--- a/pypy/jit/metainterp/optimizeopt/virtualize.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualize.py
@@ -59,7 +59,7 @@
 
     def import_from(self, other, optimizer):
         raise NotImplementedError("should not be called at this level")
-    
+
 def get_fielddescrlist_cache(cpu):
     if not hasattr(cpu, '_optimizeopt_fielddescrlist_cache'):
         result = descrlist_dict()
@@ -113,7 +113,7 @@
         #
         if not we_are_translated():
             op.name = 'FORCE ' + self.source_op.name
-            
+
         if self._is_immutable_and_filled_with_constants(optforce):
             box = optforce.optimizer.constant_fold(op)
             self.make_constant(box)
@@ -239,12 +239,12 @@
         for index in range(len(self._items)):
             self._items[index] = self._items[index].force_at_end_of_preamble(already_forced, optforce)
         return self
-    
+
     def _really_force(self, optforce):
         assert self.source_op is not None
         if not we_are_translated():
             self.source_op.name = 'FORCE ' + self.source_op.name
-        optforce.emit_operation(self.source_op)        
+        optforce.emit_operation(self.source_op)
         self.box = box = self.source_op.result
         for index in range(len(self._items)):
             subvalue = self._items[index]
@@ -271,20 +271,86 @@
     def _make_virtual(self, modifier):
         return modifier.make_varray(self.arraydescr)
 
+class VArrayStructValue(AbstractVirtualValue):
+    def __init__(self, arraydescr, size, keybox, source_op=None):
+        AbstractVirtualValue.__init__(self, keybox, source_op)
+        self.arraydescr = arraydescr
+        self._items = [{} for _ in xrange(size)]
+
+    def getlength(self):
+        return len(self._items)
+
+    def getinteriorfield(self, index, ofs, default):
+        return self._items[index].get(ofs, default)
+
+    def setinteriorfield(self, index, ofs, itemvalue):
+        assert isinstance(itemvalue, optimizer.OptValue)
+        self._items[index][ofs] = itemvalue
+
+    def _really_force(self, optforce):
+        assert self.source_op is not None
+        if not we_are_translated():
+            self.source_op.name = 'FORCE ' + self.source_op.name
+        optforce.emit_operation(self.source_op)
+        self.box = box = self.source_op.result
+        for index in range(len(self._items)):
+            for descr, value in self._items[index].iteritems():
+                subbox = value.force_box(optforce)
+                op = ResOperation(rop.SETINTERIORFIELD_GC,
+                    [box, ConstInt(index), subbox], None, descr=descr
+                )
+                optforce.emit_operation(op)
+
+    def _get_list_of_descrs(self):
+        descrs = []
+        for item in self._items:
+            item_descrs = item.keys()
+            sort_descrs(item_descrs)
+            descrs.append(item_descrs)
+        return descrs
+
+    def get_args_for_fail(self, modifier):
+        if self.box is None and not modifier.already_seen_virtual(self.keybox):
+            itemdescrs = self._get_list_of_descrs()
+            itemboxes = []
+            for i in range(len(self._items)):
+                for descr in itemdescrs[i]:
+                    itemboxes.append(self._items[i][descr].get_key_box())
+            modifier.register_virtual_fields(self.keybox, itemboxes)
+            for i in range(len(self._items)):
+                for descr in itemdescrs[i]:
+                    self._items[i][descr].get_args_for_fail(modifier)
+
+    def force_at_end_of_preamble(self, already_forced, optforce):
+        if self in already_forced:
+            return self
+        already_forced[self] = self
+        for index in range(len(self._items)):
+            for descr in self._items[index].keys():
+                self._items[index][descr] = self._items[index][descr].force_at_end_of_preamble(already_forced, optforce)
+        return self
+
+    def _make_virtual(self, modifier):
+        return modifier.make_varraystruct(self.arraydescr, self._get_list_of_descrs())
+
+
 class OptVirtualize(optimizer.Optimization):
     "Virtualize objects until they escape."
 
     def new(self):
         return OptVirtualize()
-        
+
     def make_virtual(self, known_class, box, source_op=None):
         vvalue = VirtualValue(self.optimizer.cpu, known_class, box, source_op)
         self.make_equal_to(box, vvalue)
         return vvalue
 
     def make_varray(self, arraydescr, size, box, source_op=None):
-        constvalue = self.new_const_item(arraydescr)
-        vvalue = VArrayValue(arraydescr, constvalue, size, box, source_op)
+        if arraydescr.is_array_of_structs():
+            vvalue = VArrayStructValue(arraydescr, size, box, source_op)
+        else:
+            constvalue = self.new_const_item(arraydescr)
+            vvalue = VArrayValue(arraydescr, constvalue, size, box, source_op)
         self.make_equal_to(box, vvalue)
         return vvalue
 
@@ -431,6 +497,34 @@
         value.ensure_nonnull()
         self.emit_operation(op)
 
+    def optimize_GETINTERIORFIELD_GC(self, op):
+        value = self.getvalue(op.getarg(0))
+        if value.is_virtual():
+            indexbox = self.get_constant_box(op.getarg(1))
+            if indexbox is not None:
+                descr = op.getdescr()
+                fieldvalue = value.getinteriorfield(
+                    indexbox.getint(), descr, None
+                )
+                if fieldvalue is None:
+                    fieldvalue = self.new_const(descr)
+                self.make_equal_to(op.result, fieldvalue)
+                return
+        value.ensure_nonnull()
+        self.emit_operation(op)
+
+    def optimize_SETINTERIORFIELD_GC(self, op):
+        value = self.getvalue(op.getarg(0))
+        if value.is_virtual():
+            indexbox = self.get_constant_box(op.getarg(1))
+            if indexbox is not None:
+                value.setinteriorfield(
+                    indexbox.getint(), op.getdescr(), self.getvalue(op.getarg(2))
+                )
+                return
+        value.ensure_nonnull()
+        self.emit_operation(op)
+
 
 dispatch_opt = make_dispatcher_method(OptVirtualize, 'optimize_',
         default=OptVirtualize.emit_operation)
diff --git a/pypy/jit/metainterp/optimizeopt/virtualstate.py b/pypy/jit/metainterp/optimizeopt/virtualstate.py
--- a/pypy/jit/metainterp/optimizeopt/virtualstate.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualstate.py
@@ -16,7 +16,7 @@
 
 class AbstractVirtualStateInfo(resume.AbstractVirtualInfo):
     position = -1
-    
+
     def generalization_of(self, other, renum, bad):
         raise NotImplementedError
 
@@ -54,7 +54,7 @@
                 s.debug_print(indent + "    ", seen, bad)
         else:
             debug_print(indent + "    ...")
-                
+
 
     def debug_header(self, indent):
         raise NotImplementedError
@@ -77,13 +77,15 @@
             bad[self] = True
             bad[other] = True
             return False
+
+        assert isinstance(other, AbstractVirtualStructStateInfo)
         assert len(self.fielddescrs) == len(self.fieldstate)
         assert len(other.fielddescrs) == len(other.fieldstate)
         if len(self.fielddescrs) != len(other.fielddescrs):
             bad[self] = True
             bad[other] = True
             return False
-        
+
         for i in range(len(self.fielddescrs)):
             if other.fielddescrs[i] is not self.fielddescrs[i]:
                 bad[self] = True
@@ -112,8 +114,8 @@
     def _enum(self, virtual_state):
         for s in self.fieldstate:
             s.enum(virtual_state)
-        
-        
+
+
 class VirtualStateInfo(AbstractVirtualStructStateInfo):
     def __init__(self, known_class, fielddescrs):
         AbstractVirtualStructStateInfo.__init__(self, fielddescrs)
@@ -128,13 +130,13 @@
 
     def debug_header(self, indent):
         debug_print(indent + 'VirtualStateInfo(%d):' % self.position)
-        
+
 class VStructStateInfo(AbstractVirtualStructStateInfo):
     def __init__(self, typedescr, fielddescrs):
         AbstractVirtualStructStateInfo.__init__(self, fielddescrs)
         self.typedescr = typedescr
 
-    def _generalization_of(self, other):        
+    def _generalization_of(self, other):
         if not isinstance(other, VStructStateInfo):
             return False
         if self.typedescr is not other.typedescr:
@@ -143,7 +145,7 @@
 
     def debug_header(self, indent):
         debug_print(indent + 'VStructStateInfo(%d):' % self.position)
-        
+
 class VArrayStateInfo(AbstractVirtualStateInfo):
     def __init__(self, arraydescr):
         self.arraydescr = arraydescr
@@ -157,11 +159,7 @@
             bad[other] = True
             return False
         renum[self.position] = other.position
-        if not isinstance(other, VArrayStateInfo):
-            bad[self] = True
-            bad[other] = True
-            return False
-        if self.arraydescr is not other.arraydescr:
+        if not self._generalization_of(other):
             bad[self] = True
             bad[other] = True
             return False
@@ -177,6 +175,10 @@
                 return False
         return True
 
+    def _generalization_of(self, other):
+        return (isinstance(other, VArrayStateInfo) and
+            self.arraydescr is other.arraydescr)
+
     def enum_forced_boxes(self, boxes, value, optimizer):
         assert isinstance(value, virtualize.VArrayValue)
         assert value.is_virtual()
@@ -192,8 +194,75 @@
 
     def debug_header(self, indent):
         debug_print(indent + 'VArrayStateInfo(%d):' % self.position)
-            
-        
+
+class VArrayStructStateInfo(AbstractVirtualStateInfo):
+    def __init__(self, arraydescr, fielddescrs):
+        self.arraydescr = arraydescr
+        self.fielddescrs = fielddescrs
+
+    def generalization_of(self, other, renum, bad):
+        assert self.position != -1
+        if self.position in renum:
+            if renum[self.position] == other.position:
+                return True
+            bad[self] = True
+            bad[other] = True
+            return False
+        renum[self.position] = other.position
+        if not self._generalization_of(other):
+            bad[self] = True
+            bad[other] = True
+            return False
+
+        assert isinstance(other, VArrayStructStateInfo)
+        if len(self.fielddescrs) != len(other.fielddescrs):
+            bad[self] = True

From pullrequests-noreply at bitbucket.org  Fri Oct 28 09:34:23 2011
From: pullrequests-noreply at bitbucket.org (Bitbucket)
Date: Fri, 28 Oct 2011 07:34:23 -0000
Subject: [pypy-commit] [ACCEPTED] Pull request #13 for pypy/pypy: fixes
	issue 923
In-Reply-To: <pullreq-caf9fa132b86@bitbucket.org>
References: <pullreq-caf9fa132b86@bitbucket.org>
Message-ID: <20111028073423.32365.5232@bitbucket01.managed.contegix.com>

Pull request #13 has been accepted by fijal.

Changes in brunogola/pypy have been pulled into pypy/pypy.

https://bitbucket.org/pypy/pypy/pull-request/13/fixes-issue-923
--
This is an issue notification from bitbucket.org.
You are receiving this either because you are the participating
in a pull request, or you are following it.


From noreply at buildbot.pypy.org  Fri Oct 28 10:43:39 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 28 Oct 2011 10:43:39 +0200 (CEST)
Subject: [pypy-commit] pypy stm: Transform and compile to C code: first
	early version.
Message-ID: <20111028084339.775C9820C2@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm
Changeset: r48551:3a174d94e902
Date: 2011-10-27 22:36 +0200
http://bitbucket.org/pypy/pypy/changeset/3a174d94e902/

Log:	Transform and compile to C code: first early version.

diff --git a/pypy/config/translationoption.py b/pypy/config/translationoption.py
--- a/pypy/config/translationoption.py
+++ b/pypy/config/translationoption.py
@@ -102,6 +102,8 @@
     # other noticeable options
     BoolOption("thread", "enable use of threading primitives",
                default=False, cmdline="--thread"),
+    BoolOption("stm", "enable use of Software Transactional Memory",
+               default=False, cmdline="--stm"),
     BoolOption("sandbox", "Produce a fully-sandboxed executable",
                default=False, cmdline="--sandbox",
                requires=[("translation.thread", False)],
diff --git a/pypy/translator/c/funcgen.py b/pypy/translator/c/funcgen.py
--- a/pypy/translator/c/funcgen.py
+++ b/pypy/translator/c/funcgen.py
@@ -596,6 +596,8 @@
         return self.op_stm(op)
     OP_STM_GETFIELD = _OP_STM
     OP_STM_SETFIELD = _OP_STM
+    OP_STM_BEGIN_TRANSACTION = _OP_STM
+    OP_STM_COMMIT_TRANSACTION = _OP_STM
 
 
     def OP_PTR_NONZERO(self, op):
diff --git a/pypy/translator/c/genc.py b/pypy/translator/c/genc.py
--- a/pypy/translator/c/genc.py
+++ b/pypy/translator/c/genc.py
@@ -131,6 +131,12 @@
     def build_database(self):
         translator = self.translator
 
+        if self.config.translation.stm:
+            from pypy.translator.stm import transform
+            transformer = transform.STMTransformer(self.translator)
+            transformer.transform()
+            log.info("Software Transactional Memory transformation applied")
+
         gcpolicyclass = self.get_gcpolicyclass()
 
         if self.config.translation.gcrootfinder == "asmgcc":
@@ -179,6 +185,10 @@
         # we need a concrete gcpolicy to do this
         self.merge_eci(db.gcpolicy.compilation_info())
 
+        if self.config.translation.stm:
+            from pypy.translator.stm._rffi_stm import eci
+            self.merge_eci(eci)
+
         all = []
         for node in self.db.globalcontainers():
             eci = node.compilation_info()
diff --git a/pypy/translator/stm/_rffi_stm.py b/pypy/translator/stm/_rffi_stm.py
--- a/pypy/translator/stm/_rffi_stm.py
+++ b/pypy/translator/stm/_rffi_stm.py
@@ -3,14 +3,15 @@
 from pypy.tool.autopath import pypydir
 from pypy.rpython.lltypesystem import lltype, rffi
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
+from pypy.rlib.rarithmetic import LONG_BIT
 
 
 cdir = py.path.local(pypydir) / 'translator' / 'stm'
 
-
 eci = ExternalCompilationInfo(
     include_dirs = [cdir],
     includes = ['src_stm/et.h'],
+    pre_include_bits = ['#define PYPY_LONG_BIT %d' % LONG_BIT],
     separate_module_sources = ['#include "src_stm/et.c"\n'],
 )
 
@@ -24,8 +25,8 @@
 descriptor_init = llexternal('stm_descriptor_init', [], lltype.Void)
 descriptor_done = llexternal('stm_descriptor_done', [], lltype.Void)
 
-begin_transaction = llexternal('STM_begin_transaction', [], lltype.Void)
-commit_transaction = llexternal('stm_commit_transaction', [], lltype.Signed)
+#begin_transaction = llexternal('STM_begin_transaction', [], lltype.Void)
+#commit_transaction = llexternal('stm_commit_transaction', [], lltype.Signed)
 
 stm_read_word = llexternal('stm_read_word', [SignedP], lltype.Signed)
 stm_write_word = llexternal('stm_write_word', [SignedP, lltype.Signed],
diff --git a/pypy/translator/stm/funcgen.py b/pypy/translator/stm/funcgen.py
--- a/pypy/translator/stm/funcgen.py
+++ b/pypy/translator/stm/funcgen.py
@@ -84,7 +84,14 @@
             cdecl(funcgen.db.gettype(STRUCT), ''),
             structdef.c_struct_field_name(fieldname), newvalue))
 
+def stm_begin_transaction(funcgen, op):
+    return 'STM_begin_transaction();'
+
+def stm_commit_transaction(funcgen, op):
+    return 'stm_commit_transaction();'
+
 
 def op_stm(funcgen, op):
+    assert funcgen.db.translator.stm_transformation_applied
     func = globals()[op.opname]
     return func(funcgen, op)
diff --git a/pypy/translator/stm/src_stm/et.c b/pypy/translator/stm/src_stm/et.c
--- a/pypy/translator/stm/src_stm/et.c
+++ b/pypy/translator/stm/src_stm/et.c
@@ -775,6 +775,7 @@
   stm_write_word(p, val);
 }
 
+#if PYPY_LONG_BIT == 32
 long long stm_read_doubleword(long *addr)
 {
   /* 32-bit only */
@@ -789,15 +790,17 @@
   stm_write_word(addr, (long)val);
   stm_write_word(addr + 1, (long)(val >> 32));
 }
+#endif
 
 double stm_read_double(long *addr)
 {
   long long x;
   double dd;
-  if (sizeof(double) > sizeof(long))
-    x = stm_read_doubleword(addr);   /* 32 bits */
-  else
-    x = stm_read_word(addr);         /* 64 bits */
+#if PYPY_LONG_BIT == 32
+  x = stm_read_doubleword(addr);   /* 32 bits */
+#else
+  x = stm_read_word(addr);         /* 64 bits */
+#endif
   assert(sizeof(double) == 8 && sizeof(long long) == 8);
   memcpy(&dd, &x, 8);
   return dd;
@@ -808,24 +811,27 @@
   long long ll;
   assert(sizeof(double) == 8 && sizeof(long long) == 8);
   memcpy(&ll, &val, 8);
-  if (sizeof(double) > sizeof(long))
-    stm_write_doubleword(addr, ll);   /* 32 bits */
-  else
-    stm_write_word(addr, ll);         /* 64 bits */
+#if PYPY_LONG_BIT == 32
+  stm_write_doubleword(addr, ll);   /* 32 bits */
+#else
+  stm_write_word(addr, ll);         /* 64 bits */
+#endif
 }
 
 float stm_read_float(long *addr)
 {
   unsigned int x;
   float ff;
-  if (sizeof(float) == sizeof(long))
-    x = stm_read_word(addr);         /* 32 bits */
-  else if (((long)(char*)addr) & 7) {
+#if PYPY_LONG_BIT == 32
+  x = stm_read_word(addr);         /* 32 bits */
+#else
+  if (((long)(char*)addr) & 7) {
     addr = (long *)(((char *)addr) - 4);
     x = (unsigned int)(stm_read_word(addr) >> 32);   /* 64 bits, unaligned */
   }
   else
     x = (unsigned int)stm_read_word(addr);           /* 64 bits, aligned */
+#endif
   assert(sizeof(float) == 4 && sizeof(unsigned int) == 4);
   memcpy(&ff, &x, 4);
   return ff;
@@ -836,11 +842,13 @@
   unsigned int ii;
   assert(sizeof(float) == 4 && sizeof(unsigned int) == 4);
   memcpy(&ii, &val, 4);
-  if (sizeof(float) == sizeof(long))
-    stm_write_word(addr, ii);         /* 32 bits */
-  else if (((long)(char*)addr) & 7)
+#if PYPY_LONG_BIT == 32
+  stm_write_word(addr, ii);         /* 32 bits */
+#else
+  if (((long)(char*)addr) & 7)
     stm_write_partial_word(4, (((char *)addr) - 4),
                            4, ii);                   /* 64 bits, unaligned */
   else
     stm_write_partial_word(4, (char *)addr, 0, ii);    /* 64 bits, aligned */
+#endif
 }
diff --git a/pypy/translator/stm/src_stm/et.h b/pypy/translator/stm/src_stm/et.h
--- a/pypy/translator/stm/src_stm/et.h
+++ b/pypy/translator/stm/src_stm/et.h
@@ -9,6 +9,7 @@
 #define _ET_H
 
 #include <setjmp.h>
+#include "src/commondefs.h"
 
 
 void stm_descriptor_init(void);
@@ -40,8 +41,10 @@
 void stm_write_double(long *addr, double val);
 float stm_read_float(long *addr);
 void stm_write_float(long *addr, float val);
+#if PYPY_LONG_BIT == 32
 long long stm_read_doubleword(long *addr);
 void stm_write_doubleword(long *addr, long long val);
+#endif
 
 
 #endif  /* _ET_H */
diff --git a/pypy/translator/stm/test/test_rstm.py b/pypy/translator/stm/test/test_rstm.py
--- a/pypy/translator/stm/test/test_rstm.py
+++ b/pypy/translator/stm/test/test_rstm.py
@@ -161,6 +161,7 @@
         t.config.translation.gc = 'boehm'
         t.buildannotator().build_types(entry_point, [s_list_of_strings])
         t.buildrtyper().specialize()
+        t.stm_transformation_applied = True   # not really, but for these tests
         cbuilder = CStandaloneBuilder(t, entry_point, t.config)
         force_debug = ExternalCompilationInfo(pre_include_bits=[
             "#define RPY_ASSERT 1\n"
diff --git a/pypy/translator/stm/test/test_transform.py b/pypy/translator/stm/test/test_transform.py
--- a/pypy/translator/stm/test/test_transform.py
+++ b/pypy/translator/stm/test/test_transform.py
@@ -3,6 +3,9 @@
 from pypy.objspace.flow.model import summary
 from pypy.translator.stm.llstminterp import eval_stm_graph
 from pypy.translator.stm.transform import transform_graph
+from pypy.translator.stm import rstm
+from pypy.translator.c.test.test_standalone import StandaloneTests
+from pypy.rlib.debug import debug_print
 
 
 def test_simple():
@@ -28,3 +31,26 @@
     assert summary(graph) == {'getfield': 1}
     res = eval_stm_graph(interp, graph, [p], stm_mode="regular_transaction")
     assert res == 42
+
+
+class TestTransformSingleThread(StandaloneTests):
+
+    def compile(self, entry_point):
+        from pypy.config.pypyoption import get_pypy_config
+        self.config = get_pypy_config(translating=True)
+        self.config.translation.stm = True
+        return StandaloneTests.compile(self, entry_point)
+
+    def test_no_pointer_operations(self):
+        def simplefunc(argv):
+            rstm.begin_transaction()
+            i = 0
+            while i < 100:
+                i += 3
+            rstm.commit_transaction()
+            debug_print(i)
+            return 0
+        t, cbuilder = self.compile(simplefunc)
+        dataout, dataerr = cbuilder.cmdexec('', err=True)
+        assert dataout == ''
+        assert '102' in dataerr.splitlines()
diff --git a/pypy/translator/stm/transform.py b/pypy/translator/stm/transform.py
--- a/pypy/translator/stm/transform.py
+++ b/pypy/translator/stm/transform.py
@@ -1,8 +1,18 @@
 from pypy.objspace.flow.model import SpaceOperation
+from pypy.translator.stm import _rffi_stm
 
 
 class STMTransformer(object):
 
+    def __init__(self, translator=None):
+        self.translator = translator
+
+    def transform(self):
+        self.add_descriptor_init_stuff()
+        for graph in self.translator.graphs:
+            self.transform_graph(graph)
+        self.translator.stm_transformation_applied = True
+
     def transform_block(self, block):
         if block.operations == ():
             return
@@ -16,6 +26,18 @@
         for block in graph.iterblocks():
             self.transform_block(block)
 
+    def add_descriptor_init_stuff(self):
+        from pypy.translator.unsimplify import call_initial_function
+        from pypy.translator.unsimplify import call_final_function
+        def descriptor_init():
+            _rffi_stm.descriptor_init()
+        def descriptor_done():
+            _rffi_stm.descriptor_done()
+        call_initial_function(self.translator, descriptor_init)
+        call_final_function(self.translator, descriptor_done)
+
+    # ----------
+
     def stt_getfield(self, newoperations, op):
         STRUCT = op.args[0].concretetype.TO
         if STRUCT._immutable_field(op.args[1].value):
@@ -30,4 +52,5 @@
 
 
 def transform_graph(graph):
+    # for tests: only transforms one graph
     STMTransformer().transform_graph(graph)
diff --git a/pypy/translator/unsimplify.py b/pypy/translator/unsimplify.py
--- a/pypy/translator/unsimplify.py
+++ b/pypy/translator/unsimplify.py
@@ -144,7 +144,8 @@
     if own_annhelper:
         annhelper.finish()
 
-    entry_point = translator.entry_point_graph
+    entry_point = getattr(translator, 'entry_point_graph',
+                          translator.graphs[0])
     args = [copyvar(translator.annotator, v) for v in entry_point.getargs()]
     extrablock = Block(args)
     v_none = varoftype(lltype.Void)
@@ -169,7 +170,8 @@
     if own_annhelper:
         annhelper.finish()
 
-    entry_point = translator.entry_point_graph
+    entry_point = getattr(translator, 'entry_point_graph',
+                          translator.graphs[0])
     v = copyvar(translator.annotator, entry_point.getreturnvar())
     extrablock = Block([v])
     v_none = varoftype(lltype.Void)

From noreply at buildbot.pypy.org  Fri Oct 28 10:43:40 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 28 Oct 2011 10:43:40 +0200 (CEST)
Subject: [pypy-commit] pypy stm: Fixes.
Message-ID: <20111028084340.A744D820C2@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm
Changeset: r48552:c00c946fb09b
Date: 2011-10-28 03:02 +0200
http://bitbucket.org/pypy/pypy/changeset/c00c946fb09b/

Log:	Fixes.

diff --git a/pypy/translator/c/src/commondefs.h b/pypy/translator/c/src/commondefs.h
--- a/pypy/translator/c/src/commondefs.h
+++ b/pypy/translator/c/src/commondefs.h
@@ -36,6 +36,10 @@
 #  error "unsupported value for LLONG_MIN"
 #endif
 
+#ifndef PYPY_LONG_BIT
+#  error "PYPY_LONG_BIT must be defined before this file can be imported"
+#endif
+
 
 /******************** 32-bit support ********************/
 #if PYPY_LONG_BIT == 32
diff --git a/pypy/translator/stm/_rffi_stm.py b/pypy/translator/stm/_rffi_stm.py
--- a/pypy/translator/stm/_rffi_stm.py
+++ b/pypy/translator/stm/_rffi_stm.py
@@ -7,9 +7,10 @@
 
 
 cdir = py.path.local(pypydir) / 'translator' / 'stm'
+cdir2 = py.path.local(pypydir) / 'translator' / 'c'
 
 eci = ExternalCompilationInfo(
-    include_dirs = [cdir],
+    include_dirs = [cdir, cdir2],
     includes = ['src_stm/et.h'],
     pre_include_bits = ['#define PYPY_LONG_BIT %d' % LONG_BIT],
     separate_module_sources = ['#include "src_stm/et.c"\n'],
@@ -19,7 +20,7 @@
     return rffi.llexternal(name, args, result, compilation_info=eci,
                            _nowrapper=True, **kwds)
 
-SignedP = lltype.Ptr(lltype.Array(lltype.Signed, hints={'nolength': True}))
+SignedP = rffi.CArrayPtr(lltype.Signed)
 
 
 descriptor_init = llexternal('stm_descriptor_init', [], lltype.Void)

From pullrequests-noreply at bitbucket.org  Fri Oct 28 10:43:52 2011
From: pullrequests-noreply at bitbucket.org (Bitbucket)
Date: Fri, 28 Oct 2011 08:43:52 -0000
Subject: [pypy-commit] [ACCEPTED] Pull request #13 for pypy/pypy: fixes
	issue 923
In-Reply-To: <pullreq-caf9fa132b86@bitbucket.org>
References: <pullreq-caf9fa132b86@bitbucket.org>
Message-ID: <20111028084352.11691.59713@bitbucket12.managed.contegix.com>

Pull request #13 has been accepted by arigo.

Changes in brunogola/pypy have been pulled into pypy/pypy.

https://bitbucket.org/pypy/pypy/pull-request/13/fixes-issue-923
--
This is an issue notification from bitbucket.org.
You are receiving this either because you are the participating
in a pull request, or you are following it.


From notifications-noreply at bitbucket.org  Fri Oct 28 10:56:24 2011
From: notifications-noreply at bitbucket.org (Bitbucket)
Date: Fri, 28 Oct 2011 08:56:24 -0000
Subject: [pypy-commit] [COMMENT] Pull request #13 for pypy/pypy: fixes issue
	923
Message-ID: <20111028085624.31488.68001@bitbucket01.managed.contegix.com>

New comment on pull request:

https://bitbucket.org/pypy/pypy/pull-request/13/fixes-issue-923#comment-675

arigo said:

Uh??  It says "Accepted by arigo 11 minutes ago".  I never did that.  I only opened this page 3 minutes ago.

--
This is a pull request comment notification from bitbucket.org.
You are receiving this either because you are participating
in a pull request, or you are following it.

From noreply at buildbot.pypy.org  Fri Oct 28 11:01:24 2011
From: noreply at buildbot.pypy.org (cfbolz)
Date: Fri, 28 Oct 2011 11:01:24 +0200 (CEST)
Subject: [pypy-commit] pypy int-tag-untag-as-operations: use int_untag for
 untagging too (duh)
Message-ID: <20111028090124.8B443820C2@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: int-tag-untag-as-operations
Changeset: r48553:c0429bb24dd1
Date: 2011-10-28 10:13 +0200
http://bitbucket.org/pypy/pypy/changeset/c0429bb24dd1/

Log:	use int_untag for untagging too (duh)

diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -3556,6 +3556,7 @@
                 pc += 1
             return pc
         res = self.meta_interp(main, [False, 100, True], taggedpointers=True)
+        self.check_loops(int_untag=1, int_tag_ovf=2)
 
     def test_rerased(self):
         from pypy.rlib.rerased import erase_int, unerase_int, new_erasing_pair
diff --git a/pypy/rpython/lltypesystem/rtagged.py b/pypy/rpython/lltypesystem/rtagged.py
--- a/pypy/rpython/lltypesystem/rtagged.py
+++ b/pypy/rpython/lltypesystem/rtagged.py
@@ -59,8 +59,7 @@
     def getvalue_from_unboxed(self, llops, vinst):
         assert not self.is_parent
         v2 = llops.genop('cast_ptr_to_int', [vinst],  resulttype=lltype.Signed)
-        c_one = inputconst(lltype.Signed, 1)
-        return llops.genop('int_rshift', [v2, c_one], resulttype=lltype.Signed)
+        return llops.genop('int_untag', [v2], resulttype=lltype.Signed)
 
     def gettype_from_unboxed(self, llops, vinst, can_be_none=False):
         unboxedclass_repr = getclassrepr(self.rtyper, self.unboxedclassdef)

From noreply at buildbot.pypy.org  Fri Oct 28 11:01:25 2011
From: noreply at buildbot.pypy.org (cfbolz)
Date: Fri, 28 Oct 2011 11:01:25 +0200 (CEST)
Subject: [pypy-commit] pypy int-tag-untag-as-operations: when the optimizer
 sees an int_tag operation, it can later remove the
Message-ID: <20111028090125.BEFD7820C2@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: int-tag-untag-as-operations
Changeset: r48554:1bab583b7cdb
Date: 2011-10-28 11:01 +0200
http://bitbucket.org/pypy/pypy/changeset/1bab583b7cdb/

Log:	when the optimizer sees an int_tag operation, it can later remove
	the int_and(x, 1) operation because it knows the int is tagged.

diff --git a/pypy/jit/metainterp/optimizeopt/intbounds.py b/pypy/jit/metainterp/optimizeopt/intbounds.py
--- a/pypy/jit/metainterp/optimizeopt/intbounds.py
+++ b/pypy/jit/metainterp/optimizeopt/intbounds.py
@@ -80,6 +80,14 @@
     def optimize_INT_AND(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
+        if (self.optimizer.metainterp_sd.config.translation.taggedpointers and
+                v2.is_constant() and v2.box.getint() == 1):
+            if self.has_pure_result(rop.INT_UNTAG, [v1.box], None):
+                # the result of untagging the int is known, so the box must be
+                # tagged, so int_and(x, 1) == 1
+                value = self.getvalue(ConstInt(1))
+                self.optimizer.make_equal_to(op.result, value)
+                return
         self.emit_operation(op)
 
         r = self.getvalue(op.result)
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -5163,6 +5163,28 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_int_tag_removes_int_and(self):
+        ops = """
+        [i0]
+        i1 = int_tag_ovf(i0)
+        guard_no_overflow() []
+        i2 = int_and(i1, 1)
+        i3 = int_is_true(i2)
+        guard_true(i3) []
+        i4 = int_untag(i1)
+        i5 = int_add(i4, 1)
+        jump(i5)
+        """
+        expected = """
+        [i0]
+        i1 = int_tag_ovf(i0)
+        guard_no_overflow() []
+        i2 = int_add(i0, 1)
+        jump(i2)
+        """
+        self.optimize_loop(ops, expected)
+
+
     def test_mul_ovf(self):
         ops = """
         [i0, i1]
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_util.py b/pypy/jit/metainterp/optimizeopt/test/test_util.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_util.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_util.py
@@ -321,6 +321,7 @@
         self.globaldata = Fake()
         self.config = get_pypy_config(translating=True)
         self.config.translation.jit_ffi = True
+        self.config.translation.taggedpointers = True
 
     class warmrunnerdesc:
         class memory_manager:

From noreply at buildbot.pypy.org  Fri Oct 28 11:13:52 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 28 Oct 2011 11:13:52 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-minilang: function call and move one more
	test
Message-ID: <20111028091352.23E49820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: numpy-minilang
Changeset: r48555:7d18eb124a06
Date: 2011-10-28 11:13 +0200
http://bitbucket.org/pypy/pypy/changeset/7d18eb124a06/

Log:	function call and move one more test

diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -12,6 +12,15 @@
 class BogusBytecode(Exception):
     pass
 
+class ArgumentMismatch(Exception):
+    pass
+
+class ArgumentNotAnArray(Exception):
+    pass
+
+class WrongFunctionName(Exception):
+    pass
+
 class FakeSpace(object):
     w_ValueError = None
     w_TypeError = None
@@ -174,8 +183,7 @@
             if isinstance(w_rhs, Scalar):
                 index = int(interp.space.float_w(
                     w_rhs.value.wrap(interp.space)))
-                dtype = interp.space.fromcache(W_Float64Dtype)
-                return Scalar(dtype, w_lhs.get_concrete().eval(index))
+                return w_lhs.get_concrete().eval(index).wrap(interp.space)
             else:
                 raise NotImplementedError
         else:
@@ -244,6 +252,26 @@
     def execute(self, interp):
         interp.results.append(self.expr.execute(interp))
 
+class FunctionCall(Node):
+    def __init__(self, name, args):
+        self.name = name
+        self.args = args
+
+    def __repr__(self):
+        return "%s(%s)" % (self.name, ", ".join([repr(arg)
+                                                 for arg in self.args]))
+
+    def execute(self, interp):
+        if self.name == 'sum':
+            if len(self.args) != 1:
+                raise ArgumentMismatch
+            arr = self.args[0].execute(interp)
+            if not isinstance(arr, BaseArray):
+                raise ArgumentNotAnArray
+            return arr.descr_sum(interp.space)
+        else:
+            raise WrongFunctionName
+
 class Parser(object):
     def parse_identifier(self, id):
         id = id.strip(" ")
@@ -292,9 +320,21 @@
             return True
         return False
 
+    def parse_function_call(self, v):
+        l = v.split('(')
+        assert len(l) == 2
+        name = l[0]
+        cut = len(l[1]) - 1
+        assert cut >= 0
+        args = [self.parse_constant_or_identifier(id)
+                for id in l[1][:cut].split(",")]
+        return FunctionCall(name, args)
+
     def parse_constant_or_identifier(self, v):
         c = v[0]
         if (c >= 'a' and c <= 'z') or (c >= 'A' and c <= 'Z'):
+            if '(' in v:
+                return self.parse_function_call(v)
             return self.parse_identifier(v)
         return self.parse_constant(v)
         
diff --git a/pypy/module/micronumpy/test/test_compile.py b/pypy/module/micronumpy/test/test_compile.py
--- a/pypy/module/micronumpy/test/test_compile.py
+++ b/pypy/module/micronumpy/test/test_compile.py
@@ -18,9 +18,7 @@
         assert interp.code.statements[1].expr.v == 3
 
     def test_array_literal(self):
-        code = """
-        a = [1,2,3]
-        """
+        code = "a = [1,2,3]"
         interp = self.compile(code)
         assert isinstance(interp.code.statements[0].expr, ArrayConstant)
         st = interp.code.statements[0]
@@ -28,9 +26,7 @@
                                  FloatConstant(3)]
     
     def test_array_literal2(self):
-        code = """
-        a = [[1],[2],[3]]
-        """
+        code = "a = [[1],[2],[3]]"
         interp = self.compile(code)
         assert isinstance(interp.code.statements[0].expr, ArrayConstant)
         st = interp.code.statements[0]
@@ -39,17 +35,13 @@
                                  ArrayConstant([FloatConstant(3)])]
 
     def test_expr_1(self):
-        code = """
-        b = a + 1
-        """
+        code = "b = a + 1"
         interp = self.compile(code)
         assert (interp.code.statements[0].expr ==
                 Operator(Variable("a"), "+", FloatConstant(1)))
 
     def test_expr_2(self):
-        code = """
-        b = a + b - 3
-        """
+        code = "b = a + b - 3"
         interp = self.compile(code)
         assert (interp.code.statements[0].expr ==
                 Operator(Operator(Variable("a"), "+", Variable("b")), "-",
@@ -57,28 +49,28 @@
 
     def test_expr_3(self):
         # an equivalent of range
-        code = """
-        a = |20|
-        """
+        code = "a = |20|"
         interp = self.compile(code)
         assert interp.code.statements[0].expr == RangeConstant(20)
 
     def test_expr_only(self):
-        code = """
-        3 + a
-        """
+        code = "3 + a"
         interp = self.compile(code)
         assert interp.code.statements[0] == Execute(
             Operator(FloatConstant(3), "+", Variable("a")))
 
     def test_array_access(self):
-        code = """
-        a -> 3
-        """
+        code = "a -> 3"
         interp = self.compile(code)
         assert interp.code.statements[0] == Execute(
             Operator(Variable("a"), "->", FloatConstant(3)))
 
+    def test_function_call(self):
+        code = "sum(a)"
+        interp = self.compile(code)
+        assert interp.code.statements[0] == Execute(
+            FunctionCall("sum", [Variable("a")]))
+
 class TestRunner(object):
     def run(self, code):
         interp = numpy_compile(code)
@@ -112,7 +104,7 @@
         a + b -> 3
         """
         interp = self.run(code)
-        assert interp.results[0].value.val == 3 + 6
+        assert interp.results[0].floatval == 3 + 6
         
     def test_range_getitem(self):
         code = """
@@ -120,4 +112,13 @@
         r -> 3
         """
         interp = self.run(code)
-        assert interp.results[0].value.val == 6
+        assert interp.results[0].floatval == 6
+
+    def test_sum(self):
+        code = """
+        a = [1,2,3,4,5]
+        r = sum(a)
+        r
+        """
+        interp = self.run(code)
+        assert interp.results[0].floatval == 15
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -23,8 +23,7 @@
             interp = numpy_compile(hlstr(code))
             interp.run(space)
             res = interp.results[0]
-            assert isinstance(res, BaseArray)
-            return interp.space.float_w(res.eval(0).wrap(interp.space))
+            return interp.space.float_w(res)
 
         if self.graph is None:
             interp, graph = self.meta_interp(f, [llstr(code)],
@@ -58,6 +57,17 @@
                           "setarrayitem_raw": 1, "int_add": 1,
                           "int_lt": 1, "guard_true": 1, "jump": 1})
 
+    def test_sum(self):
+        result = self.run("""
+        a = |30|
+        b = a + a
+        sum(b)
+        """)
+        assert result == 2 * sum(range(30))
+        self.check_loops({"getarrayitem_raw": 2, "float_add": 2,
+                          "int_add": 1,
+                          "int_lt": 1, "guard_true": 1, "jump": 1})
+
 class DisabledTestNumpy(object):
     def test_sum(self):
         space = self.space

From noreply at buildbot.pypy.org  Fri Oct 28 11:22:44 2011
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Fri, 28 Oct 2011 11:22:44 +0200 (CEST)
Subject: [pypy-commit] pyrepl py3ksupport: add a modified version of
 _minimal_curses, get to the point of kind of running pythoni on python3
Message-ID: <20111028092244.DE7B8820C2@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: py3ksupport
Changeset: r152:19d9ded8abdc
Date: 2011-10-28 11:22 +0200
http://bitbucket.org/pypy/pyrepl/changeset/19d9ded8abdc/

Log:	add a modified version of _minimal_curses, get to the point of kind
	of running pythoni on python3

diff --git a/pyrepl/_minimal_curses.py b/pyrepl/_minimal_curses.py
new file mode 100644
--- /dev/null
+++ b/pyrepl/_minimal_curses.py
@@ -0,0 +1,69 @@
+"""Minimal '_curses' module, the low-level interface for curses module
+which is not meant to be used directly.
+
+Based on ctypes.  It's too incomplete to be really called '_curses', so
+to use it, you have to import it and stick it in sys.modules['_curses']
+manually.
+
+Note that there is also a built-in module _minimal_curses which will
+hide this one if compiled in.
+"""
+
+import ctypes, ctypes.util
+
+class error(Exception):
+    pass
+
+
+def _find_clib():
+    trylibs = ['ncurses', 'curses']
+
+    for lib in trylibs:
+        path = ctypes.util.find_library(lib)
+        if path:
+            return path
+    raise ImportError("curses library not found")
+
+_clibpath = _find_clib()
+clib = ctypes.cdll.LoadLibrary(_clibpath)
+
+clib.setupterm.argtypes = [ctypes.c_char_p, ctypes.c_int,
+                           ctypes.POINTER(ctypes.c_int)]
+clib.setupterm.restype = ctypes.c_int
+
+clib.tigetstr.argtypes = [ctypes.c_char_p]
+clib.tigetstr.restype = ctypes.POINTER(ctypes.c_char)
+
+clib.tparm.argtypes = [ctypes.c_char_p] + 9 * [ctypes.c_int]
+clib.tparm.restype = ctypes.c_char_p
+
+OK = 0
+ERR = -1
+
+# ____________________________________________________________
+
+try: from __pypy__ import builtinify
+except ImportError: builtinify = lambda f: f
+
+ at builtinify
+def setupterm(termstr, fd):
+    err = ctypes.c_int(0)
+    result = clib.setupterm(termstr, fd, ctypes.byref(err))
+    if result == ERR:
+        raise error("setupterm() failed (err=%d)" % err.value)
+
+ at builtinify
+def tigetstr(cap):
+    if not isinstance(cap, bytes):
+        cap = cap.encode('ascii')
+    result = clib.tigetstr(cap)
+    if ctypes.cast(result, ctypes.c_void_p).value == ERR:
+        return None
+    return ctypes.cast(result, ctypes.c_char_p).value
+
+ at builtinify
+def tparm(str, i1=0, i2=0, i3=0, i4=0, i5=0, i6=0, i7=0, i8=0, i9=0):
+    result = clib.tparm(str, i1, i2, i3, i4, i5, i6, i7, i8, i9)
+    if result is None:
+        raise error("tparm() returned NULL")
+    return result
diff --git a/pyrepl/curses.py b/pyrepl/curses.py
--- a/pyrepl/curses.py
+++ b/pyrepl/curses.py
@@ -19,21 +19,5 @@
 # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
 # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 
-# Some try-import logic for two purposes: avoiding to bring in the whole
-# pure Python curses package if possible; and, in _curses is not actually
-# present, falling back to _minimal_curses (which is either a ctypes-based
-# pure Python module or a PyPy built-in module).
-try:
-    import _curses
-except ImportError:
-    try:
-        import _minimal_curses as _curses
-    except ImportError:
-        # Who knows, maybe some environment has "curses" but not "_curses".
-        # If not, at least the following import gives a clean ImportError.
-        import _curses
 
-setupterm = _curses.setupterm
-tigetstr = _curses.tigetstr
-tparm = _curses.tparm
-error = _curses.error
+from ._minimal_curses import setupterm, tigetstr, tparm, error
diff --git a/pyrepl/keymap.py b/pyrepl/keymap.py
--- a/pyrepl/keymap.py
+++ b/pyrepl/keymap.py
@@ -170,8 +170,9 @@
         r.extend(k)
     return r
 
-def compile_keymap(keymap, empty=''):
+def compile_keymap(keymap, empty=b''):
     r = {}
+    import pprint
     for key, value in keymap.items():
         r.setdefault(key[0], {})[key[1:]] = value
     for key, value in r.items():
diff --git a/pyrepl/python_reader.py b/pyrepl/python_reader.py
--- a/pyrepl/python_reader.py
+++ b/pyrepl/python_reader.py
@@ -32,6 +32,12 @@
     import cPickle as pickle
 except ImportError:
     import pickle
+
+try:
+    unicode
+except:
+    unicode = str
+
 try:
     import imp
     imp.find_module("twisted")
@@ -69,7 +75,7 @@
         text = r.get_unicode()
         try:
             # ooh, look at the hack:
-            code = r.compiler("#coding:utf-8\n"+text.encode('utf-8'))
+            code = r.compiler("#coding:utf-8\n"+text)
         except (OverflowError, SyntaxError, ValueError):
             self.finish = 1
         else:
@@ -188,8 +194,7 @@
     def execute(self, text):
         try:
             # ooh, look at the hack:            
-            code = self.compile("# coding:utf8\n"+text.encode('utf-8'),
-                                '<input>', 'single')
+            code = self.compile(text, '<input>', 'single')
         except (OverflowError, SyntaxError, ValueError):
             self.showsyntaxerror("<input>")
         else:
diff --git a/pyrepl/unix_console.py b/pyrepl/unix_console.py
--- a/pyrepl/unix_console.py
+++ b/pyrepl/unix_console.py
@@ -30,6 +30,7 @@
 class InvalidTerminal(RuntimeError):
     pass
 
+
 _error = (termios.error, curses.error, InvalidTerminal)
 
 # there are arguments for changing this to "refresh"
@@ -58,7 +59,7 @@
 
 del r, maybe_add_baudrate
 
-delayprog = re.compile("\\$<([0-9]+)((?:/|\\*){0,2})>")
+delayprog = re.compile(b"\\$<([0-9]+)((?:/|\\*){0,2})>")
 
 try:
     poll = select.poll
@@ -157,7 +158,7 @@
         self.__move = self.__move_short
 
         self.event_queue = unix_eventqueue.EventQueue(self.input_fd)
-        self.partial_char = ''
+        self.partial_char = b''
         self.cursor_visible = 1
 
     def change_encoding(self, encoding):
@@ -303,6 +304,7 @@
         self.__buffer.append((text, 0))
 
     def __write_code(self, fmt, *args):
+
         self.__buffer.append((curses.tparm(fmt, *args), 1))
 
     def __maybe_write_code(self, fmt, *args):
@@ -405,7 +407,7 @@
     def push_char(self, char):
         self.partial_char += char
         try:
-            c = unicode(self.partial_char, self.encoding)
+            c = self.partial_char.decode(self.encoding)
         except UnicodeError as e:
             if len(e.args) > 4 and \
                    e.args[4] == 'unexpected end of data':
@@ -413,7 +415,7 @@
             else:
                 raise
         else:
-            self.partial_char = ''
+            self.partial_char = b''
             self.event_queue.push(c)
         
     def get_event(self, block=1):
diff --git a/pyrepl/unix_eventqueue.py b/pyrepl/unix_eventqueue.py
--- a/pyrepl/unix_eventqueue.py
+++ b/pyrepl/unix_eventqueue.py
@@ -26,6 +26,11 @@
 from pyrepl import curses
 from termios import tcgetattr, VERASE
 import os
+try:
+    unicode
+except NameError:
+    unicode = str
+
 
 _keynames = {
     "delete" : "kdch1",
@@ -54,7 +59,7 @@
             if keycode:
                 our_keycodes[keycode] = unicode(key)
         if os.isatty(fd):
-            our_keycodes[tcgetattr(fd)[6][VERASE]] = u'backspace'
+            our_keycodes[tcgetattr(fd)[6][VERASE]] = unicode('backspace')
         self.k = self.ck = keymap.compile_keymap(our_keycodes)
         self.events = []
         self.buf = []

From noreply at buildbot.pypy.org  Fri Oct 28 11:57:42 2011
From: noreply at buildbot.pypy.org (RonnyPfannschmidt)
Date: Fri, 28 Oct 2011 11:57:42 +0200 (CEST)
Subject: [pypy-commit] pyrepl py3ksupport: use lines of unicode_escaped
 strings for history - its multi-python safe and more readable
Message-ID: <20111028095742.3678D820C2@wyvern.cs.uni-duesseldorf.de>

Author: Ronny Pfannschmidt <Ronny.Pfannschmidt at gmx.de>
Branch: py3ksupport
Changeset: r153:61d643893b64
Date: 2011-10-28 11:57 +0200
http://bitbucket.org/pypy/pyrepl/changeset/61d643893b64/

Log:	use lines of unicode_escaped strings for history - its multi-python
	safe and more readable

diff --git a/pyrepl/python_reader.py b/pyrepl/python_reader.py
--- a/pyrepl/python_reader.py
+++ b/pyrepl/python_reader.py
@@ -21,6 +21,7 @@
 
 # one impressive collections of imports:
 from __future__ import print_function
+from __future__ import unicode_literals
 from pyrepl.completing_reader import CompletingReader
 from pyrepl.historical_reader import HistoricalReader
 from pyrepl import completing_reader, reader
@@ -75,7 +76,7 @@
         text = r.get_unicode()
         try:
             # ooh, look at the hack:
-            code = r.compiler("#coding:utf-8\n"+text)
+            code = r.compiler(text)
         except (OverflowError, SyntaxError, ValueError):
             self.finish = 1
         else:
@@ -89,16 +90,14 @@
 import_line_prog = re.compile(
     "^(?:import|from)\s+(?P<mod>[A-Za-z_.0-9]*)\s*$")
 
-def mk_saver(reader):
-    def saver(reader=reader):
-        try:
-            file = open(os.path.expanduser("~/.pythoni.hist"), "w")
-        except IOError:
-            pass
-        else:
-            pickle.dump(reader.history, file)
-            file.close()
-    return saver
+def saver(reader=reader):
+    try:
+        with open(os.path.expanduser("~/.pythoni.hist"), "wb") as fp:
+            fp.write(b'\n'.join(item.encode('unicode_escape')
+                                for item in reader.history))
+    except IOError as e:
+        print(e)
+        pass
 
 class PythonicReader(CompletingReader, HistoricalReader):
     def collect_keymap(self):
@@ -119,17 +118,18 @@
         else:
             self.compiler = compiler
         try:
-            file = open(os.path.expanduser("~/.pythoni.hist"))
+            file = open(os.path.expanduser("~/.pythoni.hist"), 'rb')
         except IOError:
-            pass
+            self.history = []
         else:
             try:
-                self.history = pickle.load(file)
+                lines = file.readlines()
+                self.history = [ x.rstrip(b'\n').decode('unicode_escape') for x in lines]
             except:
                 self.history = []
             self.historyi = len(self.history)
             file.close()
-        atexit.register(mk_saver(self))
+        atexit.register(lambda: saver(self))
         for c in [maybe_accept]:
             self.commands[c.__name__] = c
             self.commands[c.__name__.replace('_', '-')] = c        

From noreply at buildbot.pypy.org  Fri Oct 28 12:14:33 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 28 Oct 2011 12:14:33 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-minilang: more bugs in strip
Message-ID: <20111028101433.CD918820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: numpy-minilang
Changeset: r48556:75b8ab99578c
Date: 2011-10-28 12:11 +0200
http://bitbucket.org/pypy/pypy/changeset/75b8ab99578c/

Log:	more bugs in strip

diff --git a/pypy/rpython/lltypesystem/rstr.py b/pypy/rpython/lltypesystem/rstr.py
--- a/pypy/rpython/lltypesystem/rstr.py
+++ b/pypy/rpython/lltypesystem/rstr.py
@@ -365,9 +365,9 @@
             while lpos < rpos and s.chars[lpos] == ch:
                 lpos += 1
         if right:
-            while lpos < rpos and s.chars[rpos] == ch:
+            while lpos < rpos + 1 and s.chars[rpos] == ch:
                 rpos -= 1
-        if rpos <= lpos:
+        if rpos < lpos:
             return s.empty()
         r_len = rpos - lpos + 1
         result = s.malloc(r_len)
diff --git a/pypy/rpython/test/test_rstr.py b/pypy/rpython/test/test_rstr.py
--- a/pypy/rpython/test/test_rstr.py
+++ b/pypy/rpython/test/test_rstr.py
@@ -374,6 +374,8 @@
             return const('!ab!').rstrip(const('!'))
         def empty():
             return const('    ').strip(' ')
+        def left2():
+            return const('a  ').strip(' ')
         res = self.interpret(both, [])
         assert self.ll_to_string(res) == const('ab')
         res = self.interpret(left, [])
@@ -382,6 +384,8 @@
         assert self.ll_to_string(res) == const('!ab')
         res = self.interpret(empty, [])
         assert self.ll_to_string(res) == const('')
+        res = self.interpret(left2, [])
+        assert self.ll_to_string(res) == const('a')
 
     def test_upper(self):
         const = self.const

From noreply at buildbot.pypy.org  Fri Oct 28 12:14:35 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 28 Oct 2011 12:14:35 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-minilang: work more on test_zjit. now it
 shows a problem in our handling!!! (wow)
Message-ID: <20111028101435.0B17F820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: numpy-minilang
Changeset: r48557:312b01650172
Date: 2011-10-28 12:13 +0200
http://bitbucket.org/pypy/pypy/changeset/312b01650172/

Log:	work more on test_zjit. now it shows a problem in our handling!!!
	(wow)

diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -5,7 +5,8 @@
 
 from pypy.interpreter.baseobjspace import InternalSpaceCache, W_Root
 from pypy.module.micronumpy.interp_dtype import W_Float64Dtype
-from pypy.module.micronumpy.interp_numarray import Scalar, BaseArray, descr_new_array
+from pypy.module.micronumpy.interp_numarray import (Scalar, BaseArray,
+     descr_new_array, scalar_w, SingleDimArray)
 from pypy.rlib.objectmodel import specialize
 
 
@@ -21,6 +22,8 @@
 class WrongFunctionName(Exception):
     pass
 
+SINGLE_ARG_FUNCTIONS = ["sum", "prod", "max", "min"]
+
 class FakeSpace(object):
     w_ValueError = None
     w_TypeError = None
@@ -31,6 +34,7 @@
     w_float = "float"
     w_list = "list"
     w_long = "long"
+    w_tuple = 'tuple'
 
     def __init__(self):
         """NOT_RPYTHON"""
@@ -38,11 +42,14 @@
         self.w_float64dtype = W_Float64Dtype(self)
 
     def issequence_w(self, w_obj):
-        return w_obj.seq
+        return isinstance(w_obj, ListObject) or isinstance(w_obj, SingleDimArray)
 
     def isinstance_w(self, w_obj, w_tp):
         return False
 
+    def decode_index4(self, w_idx, size):
+        return (self.int_w(w_idx), 0, 0, 1)
+
     @specialize.argtype(1)
     def wrap(self, obj):
         if isinstance(obj, float):
@@ -69,8 +76,11 @@
         return w_obj.floatval
 
     def int_w(self, w_obj):
-        assert isinstance(w_obj, IntObject)
-        return w_obj.intval
+        if isinstance(w_obj, IntObject):
+            return w_obj.intval
+        elif isinstance(w_obj, FloatObject):
+            return int(w_obj.floatval)
+        raise NotImplementedError
 
     def int(self, w_obj):
         return w_obj
@@ -97,25 +107,21 @@
         return what
 
 class FloatObject(W_Root):
-    seq = False
     tp = FakeSpace.w_float
     def __init__(self, floatval):
         self.floatval = floatval
 
 class BoolObject(W_Root):
-    seq = False
     tp = FakeSpace.w_bool
     def __init__(self, boolval):
         self.boolval = boolval
 
 class IntObject(W_Root):
-    seq = False
     tp = FakeSpace.w_int
     def __init__(self, intval):
         self.intval = intval
 
 class ListObject(W_Root):
-    seq = True
     tp = FakeSpace.w_list
     def __init__(self, items):
         self.items = items
@@ -147,15 +153,30 @@
         raise NotImplementedError
 
 class Assignment(Node):
-    def __init__(self, id, expr):
-        self.id = id
+    def __init__(self, name, expr):
+        self.name = name
         self.expr = expr
 
     def execute(self, interp):
-        interp.variables[self.id.name] = self.expr.execute(interp)
+        interp.variables[self.name] = self.expr.execute(interp)
 
     def __repr__(self):
-        return "%r = %r" % (self.id, self.expr)
+        return "%% = %r" % (self.name, self.expr)
+
+class ArrayAssignment(Node):
+    def __init__(self, name, index, expr):
+        self.name = name
+        self.index = index
+        self.expr = expr
+
+    def execute(self, interp):
+        arr = interp.variables[self.name]
+        w_index = self.index.execute(interp).eval(0).wrap(interp.space)
+        w_val = self.expr.execute(interp).eval(0).wrap(interp.space)
+        arr.descr_setitem(interp.space, w_index, w_val)
+
+    def __repr__(self):
+        return "%s[%r] = %r" % (self.name, self.index, self.expr)
 
 class Variable(Node):
     def __init__(self, name):
@@ -178,12 +199,17 @@
         w_rhs = self.rhs.execute(interp)
         assert isinstance(w_lhs, BaseArray)
         if self.name == '+':
-            return w_lhs.descr_add(interp.space, w_rhs)
+            w_res = w_lhs.descr_add(interp.space, w_rhs)
+            if not isinstance(w_res, BaseArray):
+                dtype = interp.space.fromcache(W_Float64Dtype)
+                w_res = scalar_w(interp.space, dtype, w_res)
+            return w_res
         elif self.name == '->':
             if isinstance(w_rhs, Scalar):
                 index = int(interp.space.float_w(
                     w_rhs.value.wrap(interp.space)))
-                return w_lhs.get_concrete().eval(index).wrap(interp.space)
+                dtype = interp.space.fromcache(W_Float64Dtype)
+                return Scalar(dtype, w_lhs.get_concrete().eval(index))
             else:
                 raise NotImplementedError
         else:
@@ -262,13 +288,24 @@
                                                  for arg in self.args]))
 
     def execute(self, interp):
-        if self.name == 'sum':
+        if self.name in SINGLE_ARG_FUNCTIONS:
             if len(self.args) != 1:
                 raise ArgumentMismatch
             arr = self.args[0].execute(interp)
             if not isinstance(arr, BaseArray):
                 raise ArgumentNotAnArray
-            return arr.descr_sum(interp.space)
+            if self.name == "sum":
+                w_res = arr.descr_sum(interp.space)
+            elif self.name == "prod":
+                w_res = arr.descr_prod(interp.space)
+            elif self.name == "max":
+                w_res = arr.descr_max(interp.space)
+            elif self.name == "min":
+                w_res = arr.descr_min(interp.space)
+            else:
+                assert False # unreachable code
+            dtype = interp.space.fromcache(W_Float64Dtype)
+            return scalar_w(interp.space, dtype, w_res)
         else:
             raise WrongFunctionName
 
@@ -337,12 +374,24 @@
                 return self.parse_function_call(v)
             return self.parse_identifier(v)
         return self.parse_constant(v)
+
+    def parse_array_subscript(self, v):
+        v = v.strip(" ")
+        l = v.split("[")
+        lgt = len(l[1]) - 1
+        assert lgt >= 0
+        rhs = self.parse_constant_or_identifier(l[1][:lgt])
+        return l[0], rhs
         
     def parse_statement(self, line):
         if '=' in line:
             lhs, rhs = line.split("=")
-            return Assignment(self.parse_identifier(lhs),
-                              self.parse_expression(rhs))
+            lhs = lhs.strip(" ")
+            if '[' in lhs:
+                name, index = self.parse_array_subscript(lhs)
+                return ArrayAssignment(name, index, self.parse_expression(rhs))
+            else: 
+                return Assignment(lhs, self.parse_expression(rhs))
         else:
             return Execute(self.parse_expression(line))
 
diff --git a/pypy/module/micronumpy/test/test_compile.py b/pypy/module/micronumpy/test/test_compile.py
--- a/pypy/module/micronumpy/test/test_compile.py
+++ b/pypy/module/micronumpy/test/test_compile.py
@@ -12,9 +12,9 @@
         """
         interp = self.compile(code)
         assert isinstance(interp.code.statements[0], Assignment)
-        assert interp.code.statements[0].id.name == 'a'
+        assert interp.code.statements[0].name == 'a'
         assert interp.code.statements[0].expr.v == 2
-        assert interp.code.statements[1].id.name == 'b'
+        assert interp.code.statements[1].name == 'b'
         assert interp.code.statements[1].expr.v == 3
 
     def test_array_literal(self):
@@ -104,7 +104,7 @@
         a + b -> 3
         """
         interp = self.run(code)
-        assert interp.results[0].floatval == 3 + 6
+        assert interp.results[0].value.val == 3 + 6
         
     def test_range_getitem(self):
         code = """
@@ -112,7 +112,7 @@
         r -> 3
         """
         interp = self.run(code)
-        assert interp.results[0].floatval == 6
+        assert interp.results[0].value.val == 6
 
     def test_sum(self):
         code = """
@@ -121,4 +121,31 @@
         r
         """
         interp = self.run(code)
-        assert interp.results[0].floatval == 15
+        assert interp.results[0].value.val == 15
+
+    def test_array_write(self):
+        code = """
+        a = [1,2,3,4,5]
+        a[3] = 15
+        a -> 3
+        """
+        interp = self.run(code)
+        assert interp.results[0].value.val == 15
+
+    def test_min(self):
+        interp = self.run("""
+        a = |30|
+        a[15] = -12
+        b = a + a
+        min(b)
+        """)
+        assert interp.results[0].value.val == -24
+
+    def test_max(self):
+        interp = self.run("""
+        a = |30|
+        a[13] = 128
+        b = a + a
+        max(b)
+        """)
+        assert interp.results[0].value.val == 256
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -23,7 +23,7 @@
             interp = numpy_compile(hlstr(code))
             interp.run(space)
             res = interp.results[0]
-            return interp.space.float_w(res)
+            return interp.space.float_w(res.eval(0).wrap(interp.space))
 
         if self.graph is None:
             interp, graph = self.meta_interp(f, [llstr(code)],
@@ -68,48 +68,45 @@
                           "int_add": 1,
                           "int_lt": 1, "guard_true": 1, "jump": 1})
 
-class DisabledTestNumpy(object):
-    def test_sum(self):
-        space = self.space
-        float64_dtype = self.float64_dtype
-        int64_dtype = self.int64_dtype
-
-        def f(i):
-            if NonConstant(False):
-                dtype = int64_dtype
-            else:
-                dtype = float64_dtype
-            ar = SingleDimArray(i, dtype=dtype)
-            v = ar.descr_add(space, ar).descr_sum(space)
-            assert isinstance(v, FloatObject)
-            return v.floatval
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
-        self.check_loops({"getarrayitem_raw": 2, "float_add": 2,
-                          "int_add": 1,
-                          "int_lt": 1, "guard_true": 1, "jump": 1})
-        assert result == f(5)
-
     def test_prod(self):
-        space = self.space
-        float64_dtype = self.float64_dtype
-        int64_dtype = self.int64_dtype
-
-        def f(i):
-            if NonConstant(False):
-                dtype = int64_dtype
-            else:
-                dtype = float64_dtype
-            ar = SingleDimArray(i, dtype=dtype)
-            v = ar.descr_add(space, ar).descr_prod(space)
-            assert isinstance(v, FloatObject)
-            return v.floatval
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
+        result = self.run("""
+        a = |30|
+        b = a + a
+        prod(b)
+        """)
+        expected = 1
+        for i in range(30):
+            expected *= i * 2
+        assert result == expected
         self.check_loops({"getarrayitem_raw": 2, "float_add": 1,
                           "float_mul": 1, "int_add": 1,
                           "int_lt": 1, "guard_true": 1, "jump": 1})
-        assert result == f(5)
+
+    def test_max(self):
+        result = self.run("""
+        a = |30|
+        a[13] = 128
+        b = a + a
+        max(b)
+        """)
+        assert result == 256
+        self.check_loops({"getarrayitem_raw": 2, "float_add": 1,
+                          "float_mul": 1, "int_add": 1,
+                          "int_lt": 1, "guard_true": 1, "jump": 1})
+
+    def test_min(self):
+        result = self.run("""
+        a = |30|
+        a[15] = -12
+        b = a + a
+        min(b)
+        """)
+        assert result == -24
+        self.check_loops({"getarrayitem_raw": 2, "float_add": 1,
+                          "float_mul": 1, "int_add": 1,
+                          "int_lt": 1, "guard_true": 1, "jump": 1})
+
+class DisabledTestNumpy(object):
 
     def test_max(self):
         space = self.space

From noreply at buildbot.pypy.org  Fri Oct 28 12:16:24 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 28 Oct 2011 12:16:24 +0200 (CEST)
Subject: [pypy-commit] pypy stm: 'transaction_boundary' is better for the
	transformer.
Message-ID: <20111028101624.F3C4B820C2@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm
Changeset: r48558:cf109f42d6ab
Date: 2011-10-28 11:54 +0200
http://bitbucket.org/pypy/pypy/changeset/cf109f42d6ab/

Log:	'transaction_boundary' is better for the transformer.

diff --git a/pypy/translator/stm/llstminterp.py b/pypy/translator/stm/llstminterp.py
--- a/pypy/translator/stm/llstminterp.py
+++ b/pypy/translator/stm/llstminterp.py
@@ -10,13 +10,17 @@
     pass
 
 
-def eval_stm_graph(llinterp, graph, values, stm_mode="not_in_transaction"):
+def eval_stm_graph(llinterp, graph, values, stm_mode="not_in_transaction",
+                   final_stm_mode=Ellipsis, automatic_promotion=False):
     llinterp.frame_class = LLSTMFrame
     try:
         llinterp.stm_mode = stm_mode
+        llinterp.stm_automatic_promotion = automatic_promotion
         llinterp.last_transaction_started_in_frame = None
         res = llinterp.eval_graph(graph, values)
-        assert llinterp.stm_mode == stm_mode, (
+        if final_stm_mode is Ellipsis:
+            final_stm_mode = stm_mode
+        assert llinterp.stm_mode == final_stm_mode, (
             "llinterp.stm_mode is %r after eval_graph, but should be %r" % (
             llinterp.stm_mode, stm_mode))
         return res
@@ -43,7 +47,10 @@
     def returning_from_frame_now(self):
         if (self.llinterpreter.stm_mode == "regular_transaction" and
                 self.llinterpreter.last_transaction_started_in_frame is self):
-            raise ReturnWithTransactionActive(self.graph)
+            if self.llinterpreter.stm_automatic_promotion:
+                self.llinterpreter.stm_mode = "inevitable_transaction"
+            else:
+                raise ReturnWithTransactionActive(self.graph)
 
     def getoperationhandler(self, opname):
         ophandler = getattr(self, 'opstm_' + opname, None)
@@ -103,3 +110,8 @@
     def opstm_stm_commit_transaction(self):
         self.check_stm_mode(lambda m: m != "not_in_transaction")
         self.llinterpreter.stm_mode = "not_in_transaction"
+
+    def opstm_stm_transaction_boundary(self):
+        self.check_stm_mode(lambda m: m != "not_in_transaction")
+        self.llinterpreter.stm_mode = "regular_transaction"
+        self.llinterpreter.last_transaction_started_in_frame = self
diff --git a/pypy/translator/stm/rstm.py b/pypy/translator/stm/rstm.py
--- a/pypy/translator/stm/rstm.py
+++ b/pypy/translator/stm/rstm.py
@@ -80,11 +80,15 @@
         _rffi_stm.stm_write_word(p, val)
 
 def begin_transaction():
-    "NOT_RPYTHON"
+    "NOT_RPYTHON.  For tests only"
     raise NotImplementedError("hard to really emulate")
 
 def commit_transaction():
-    "NOT_RPYTHON"
+    "NOT_RPYTHON.  For tests only"
+    raise NotImplementedError("hard to really emulate")
+
+def transaction_boundary():
+    "NOT_RPYTHON.  This is the one normally used"
     raise NotImplementedError("hard to really emulate")
 
 # ____________________________________________________________
@@ -123,7 +127,7 @@
 
 
 class ExtEntry(ExtRegistryEntry):
-    _about_ = (begin_transaction, commit_transaction)
+    _about_ = (begin_transaction, commit_transaction, transaction_boundary)
 
     def compute_result_annotation(self):
         return None
diff --git a/pypy/translator/stm/test/test_llstminterp.py b/pypy/translator/stm/test/test_llstminterp.py
--- a/pypy/translator/stm/test/test_llstminterp.py
+++ b/pypy/translator/stm/test/test_llstminterp.py
@@ -37,7 +37,8 @@
     interp, graph = get_interpreter(func, [p])
     # forbidden in "not_in_transaction" mode
     py.test.raises(ForbiddenInstructionInSTMMode,
-                   eval_stm_graph, interp, graph, [p])
+                   eval_stm_graph, interp, graph, [p],
+                   stm_mode="not_in_transaction")
     # works in "regular_transaction" mode
     res = eval_stm_graph(interp, graph, [p], stm_mode="regular_transaction")
     assert res == 42
@@ -106,3 +107,17 @@
     interp, graph = get_interpreter(func, [])
     py.test.raises(ReturnWithTransactionActive,
                    eval_stm_graph, interp, graph, [])
+
+def test_transaction_boundary():
+    def func(n):
+        if n > 5:
+            rstm.transaction_boundary()
+    interp, graph = get_interpreter(func, [2])
+    eval_stm_graph(interp, graph, [10],
+                   stm_mode="regular_transaction",
+                   final_stm_mode="inevitable_transaction",
+                   automatic_promotion=True)
+    eval_stm_graph(interp, graph, [1],
+                   stm_mode="regular_transaction",
+                   final_stm_mode="regular_transaction",
+                   automatic_promotion=True)

From noreply at buildbot.pypy.org  Fri Oct 28 12:16:26 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 28 Oct 2011 12:16:26 +0200 (CEST)
Subject: [pypy-commit] pypy stm: Progress
Message-ID: <20111028101626.2CA38820C2@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm
Changeset: r48559:346f0c8a0ed7
Date: 2011-10-28 12:16 +0200
http://bitbucket.org/pypy/pypy/changeset/346f0c8a0ed7/

Log:	Progress

diff --git a/pypy/translator/stm/_rffi_stm.py b/pypy/translator/stm/_rffi_stm.py
--- a/pypy/translator/stm/_rffi_stm.py
+++ b/pypy/translator/stm/_rffi_stm.py
@@ -11,9 +11,9 @@
 
 eci = ExternalCompilationInfo(
     include_dirs = [cdir, cdir2],
-    includes = ['src_stm/et.h'],
+    includes = ['src_stm/et.h', 'src_stm/et.c'],
     pre_include_bits = ['#define PYPY_LONG_BIT %d' % LONG_BIT],
-    separate_module_sources = ['#include "src_stm/et.c"\n'],
+    separate_module_sources = ['\n'],    # hack for test_rffi_stm
 )
 
 def llexternal(name, args, result, **kwds):
diff --git a/pypy/translator/stm/src_stm/et.c b/pypy/translator/stm/src_stm/et.c
--- a/pypy/translator/stm/src_stm/et.c
+++ b/pypy/translator/stm/src_stm/et.c
@@ -1,11 +1,14 @@
 /* -*- c-basic-offset: 2 -*- */
 
+#ifndef PYPY_NOT_MAIN_FILE
+
 /* XXX assumes that time never wraps around (in a 'long'), which may be
  * correct on 64-bit machines but not on 32-bit machines if the process
  * runs for long enough.
  *
  * XXX measure the overhead of the global_timestamp
  */
+
 #include <stdlib.h>
 #include <stdio.h>
 #include <assert.h>
@@ -74,6 +77,9 @@
   owner_version_t my_lock_word;
   unsigned init_counter;
   struct RedoLog redolog;   /* last item, because it's the biggest one */
+#ifdef RPY_ASSERT
+  int transaction_active;
+#endif
 };
 
 /* global_timestamp contains in its lowest bit a flag equal to 1
@@ -235,6 +241,10 @@
 {
   d->reads.size = 0;
   redolog_clear(&d->redolog);
+#ifdef RPY_ASSERT
+  assert(d->transaction_active);
+  d->transaction_active = 0;
+#endif
 }
 
 static void tx_cleanup(struct tx_descriptor *d)
@@ -597,6 +607,10 @@
 void stm_begin_transaction(jmp_buf* buf)
 {
   struct tx_descriptor *d = thread_descriptor;
+#ifdef RPY_ASSERT
+  assert(!d->transaction_active);
+  d->transaction_active = 1;
+#endif
   d->setjmp_buf = buf;
   d->start_time = d->last_known_global_timestamp & ~1;
 }
@@ -852,3 +866,5 @@
     stm_write_partial_word(4, (char *)addr, 0, ii);    /* 64 bits, aligned */
 #endif
 }
+
+#endif  /* PYPY_NOT_MAIN_FILE */
diff --git a/pypy/translator/stm/test/test_transform.py b/pypy/translator/stm/test/test_transform.py
--- a/pypy/translator/stm/test/test_transform.py
+++ b/pypy/translator/stm/test/test_transform.py
@@ -39,7 +39,7 @@
         from pypy.config.pypyoption import get_pypy_config
         self.config = get_pypy_config(translating=True)
         self.config.translation.stm = True
-        return StandaloneTests.compile(self, entry_point)
+        return StandaloneTests.compile(self, entry_point, debug=True)
 
     def test_no_pointer_operations(self):
         def simplefunc(argv):
@@ -54,3 +54,21 @@
         dataout, dataerr = cbuilder.cmdexec('', err=True)
         assert dataout == ''
         assert '102' in dataerr.splitlines()
+
+    def test_fails_when_nonbalanced_begin(self):
+        def g():
+            rstm.begin_transaction()
+        g._dont_inline_ = True
+        def simplefunc(argv):
+            rstm.begin_transaction()
+            g()
+            return 0
+        t, cbuilder = self.compile(simplefunc)
+        cbuilder.cmdexec('', expect_crash=True)
+
+    def test_fails_when_nonbalanced_commit(self):
+        def simplefunc(argv):
+            rstm.commit_transaction()
+            return 0
+        t, cbuilder = self.compile(simplefunc)
+        cbuilder.cmdexec('', expect_crash=True)

From noreply at buildbot.pypy.org  Fri Oct 28 12:16:47 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 28 Oct 2011 12:16:47 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-minilang: remove ported tests
Message-ID: <20111028101647.8188F820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: numpy-minilang
Changeset: r48560:2518fc31f52d
Date: 2011-10-28 12:16 +0200
http://bitbucket.org/pypy/pypy/changeset/2518fc31f52d/

Log:	remove ported tests

diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -108,58 +108,6 @@
 
 class DisabledTestNumpy(object):
 
-    def test_max(self):
-        space = self.space
-        float64_dtype = self.float64_dtype
-        int64_dtype = self.int64_dtype
-
-        def f(i):
-            if NonConstant(False):
-                dtype = int64_dtype
-            else:
-                dtype = float64_dtype
-            ar = SingleDimArray(i, dtype=dtype)
-            j = 0
-            while j < i:
-                ar.get_concrete().setitem(j, float64_dtype.box(float(j)))
-                j += 1
-            v = ar.descr_add(space, ar).descr_max(space)
-            assert isinstance(v, FloatObject)
-            return v.floatval
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
-        self.check_loops({"getarrayitem_raw": 2, "float_add": 1,
-                          "float_gt": 1, "int_add": 1,
-                          "int_lt": 1, "guard_true": 1,
-                          "guard_false": 1, "jump": 1})
-        assert result == f(5)
-
-    def test_min(self):
-        space = self.space
-        float64_dtype = self.float64_dtype
-        int64_dtype = self.int64_dtype
-
-        def f(i):
-            if NonConstant(False):
-                dtype = int64_dtype
-            else:
-                dtype = float64_dtype
-            ar = SingleDimArray(i, dtype=dtype)
-            j = 0
-            while j < i:
-                ar.get_concrete().setitem(j, float64_dtype.box(float(j)))
-                j += 1
-            v = ar.descr_add(space, ar).descr_min(space)
-            assert isinstance(v, FloatObject)
-            return v.floatval
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
-        self.check_loops({"getarrayitem_raw": 2, "float_add": 1,
-                           "float_lt": 1, "int_add": 1,
-                           "int_lt": 1, "guard_true": 2,
-                           "jump": 1})
-        assert result == f(5)
-
     def test_argmin(self):
         space = self.space
         float64_dtype = self.float64_dtype

From noreply at buildbot.pypy.org  Fri Oct 28 13:34:17 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 28 Oct 2011 13:34:17 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-minilang: minor tweaks and port more tests
Message-ID: <20111028113417.9DDF2820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: numpy-minilang
Changeset: r48561:409b83704acb
Date: 2011-10-28 13:33 +0200
http://bitbucket.org/pypy/pypy/changeset/409b83704acb/

Log:	minor tweaks and port more tests

diff --git a/pypy/jit/metainterp/history.py b/pypy/jit/metainterp/history.py
--- a/pypy/jit/metainterp/history.py
+++ b/pypy/jit/metainterp/history.py
@@ -951,6 +951,9 @@
         del self.locations[:]
         del self.aborted_keys[:]
         self.invalidated_token_numbers.clear()
+        self.compiled_count = 0
+        self.enter_count = 0
+        self.aborted_count = 0
 
     def set_history(self, history):
         self.operations = history.operations
diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -4,9 +4,10 @@
 """
 
 from pypy.interpreter.baseobjspace import InternalSpaceCache, W_Root
-from pypy.module.micronumpy.interp_dtype import W_Float64Dtype
+from pypy.module.micronumpy.interp_dtype import W_Float64Dtype, W_BoolDtype
 from pypy.module.micronumpy.interp_numarray import (Scalar, BaseArray,
      descr_new_array, scalar_w, SingleDimArray)
+from pypy.module.micronumpy import interp_ufuncs
 from pypy.rlib.objectmodel import specialize
 
 
@@ -22,7 +23,7 @@
 class WrongFunctionName(Exception):
     pass
 
-SINGLE_ARG_FUNCTIONS = ["sum", "prod", "max", "min"]
+SINGLE_ARG_FUNCTIONS = ["sum", "prod", "max", "min", "all", "any", "unegative"]
 
 class FakeSpace(object):
     w_ValueError = None
@@ -200,10 +201,10 @@
         assert isinstance(w_lhs, BaseArray)
         if self.name == '+':
             w_res = w_lhs.descr_add(interp.space, w_rhs)
-            if not isinstance(w_res, BaseArray):
-                dtype = interp.space.fromcache(W_Float64Dtype)
-                w_res = scalar_w(interp.space, dtype, w_res)
-            return w_res
+        elif self.name == '*':
+            w_res = w_lhs.descr_mul(interp.space, w_rhs)
+        elif self.name == '-':
+            w_res = w_lhs.descr_sub(interp.space, w_rhs)            
         elif self.name == '->':
             if isinstance(w_rhs, Scalar):
                 index = int(interp.space.float_w(
@@ -214,6 +215,10 @@
                 raise NotImplementedError
         else:
             raise NotImplementedError
+        if not isinstance(w_res, BaseArray):
+            dtype = interp.space.fromcache(W_Float64Dtype)
+            w_res = scalar_w(interp.space, dtype, w_res)
+        return w_res
 
     def __repr__(self):
         return '(%r %s %r)' % (self.lhs, self.name, self.rhs)
@@ -302,9 +307,23 @@
                 w_res = arr.descr_max(interp.space)
             elif self.name == "min":
                 w_res = arr.descr_min(interp.space)
+            elif self.name == "any":
+                w_res = arr.descr_any(interp.space)
+            elif self.name == "all":
+                w_res = arr.descr_all(interp.space)
+            elif self.name == "unegative":
+                neg = interp_ufuncs.get(interp.space).negative
+                w_res = neg.call(interp.space, [arr])
             else:
                 assert False # unreachable code
-            dtype = interp.space.fromcache(W_Float64Dtype)
+            if isinstance(w_res, BaseArray):
+                return w_res
+            if isinstance(w_res, FloatObject):
+                dtype = interp.space.fromcache(W_Float64Dtype)
+            elif isinstance(w_res, BoolObject):
+                dtype = interp.space.fromcache(W_BoolDtype)
+            else:
+                dtype = None
             return scalar_w(interp.space, dtype, w_res)
         else:
             raise WrongFunctionName
@@ -398,6 +417,8 @@
     def parse(self, code):
         statements = []
         for line in code.split("\n"):
+            if '#' in line:
+                line = line.split('#', 1)[0]
             line = line.strip(" ")
             if line:
                 statements.append(self.parse_statement(line))
diff --git a/pypy/module/micronumpy/test/test_compile.py b/pypy/module/micronumpy/test/test_compile.py
--- a/pypy/module/micronumpy/test/test_compile.py
+++ b/pypy/module/micronumpy/test/test_compile.py
@@ -71,6 +71,15 @@
         assert interp.code.statements[0] == Execute(
             FunctionCall("sum", [Variable("a")]))
 
+    def test_comment(self):
+        code = """
+        # some comment
+        a = b + 3  # another comment
+        """
+        interp = self.compile(code)
+        assert interp.code.statements[0] == Assignment(
+            'a', Operator(Variable('b'), "+", FloatConstant(3)))
+
 class TestRunner(object):
     def run(self, code):
         interp = numpy_compile(code)
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -82,6 +82,8 @@
         b = negative(a)
         a[0] = 5.0
         assert b[0] == 5.0
+        a = array(range(30))
+        assert negative(a + a)[3] == -6
 
     def test_abs(self):
         from numpy import array, absolute
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -1,13 +1,13 @@
 from pypy.jit.metainterp.test.support import LLJitMixin
 from pypy.module.micronumpy import interp_ufuncs, signature
 from pypy.module.micronumpy.compile import (FakeSpace,
-    FloatObject, IntObject, numpy_compile)
+    FloatObject, IntObject, numpy_compile, BoolObject)
 from pypy.module.micronumpy.interp_numarray import (BaseArray, SingleDimArray,
-    SingleDimSlice, scalar_w)
+    SingleDimSlice)
 from pypy.rlib.nonconst import NonConstant
 from pypy.rpython.annlowlevel import llstr, hlstr
-from pypy.rpython.test.test_llinterp import interpret
 from pypy.jit.metainterp.warmspot import reset_stats
+from pypy.jit.metainterp import pyjitpl
 
 import py
 
@@ -22,8 +22,16 @@
         def f(code):
             interp = numpy_compile(hlstr(code))
             interp.run(space)
-            res = interp.results[0]
-            return interp.space.float_w(res.eval(0).wrap(interp.space))
+            res = interp.results[-1]
+            w_res = res.eval(0).wrap(interp.space)
+            if isinstance(w_res, BoolObject):
+                return float(w_res.boolval)
+            elif isinstance(w_res, FloatObject):
+                return w_res.floatval
+            elif isinstance(w_res, IntObject):
+                return w_res.intval
+            else:
+                return -42.
 
         if self.graph is None:
             interp, graph = self.meta_interp(f, [llstr(code)],
@@ -34,6 +42,7 @@
             self.__class__.graph = graph
 
         reset_stats()
+        pyjitpl._warmrunnerdesc.memory_manager.alive_loops.clear()
         return self.interp.eval_graph(self.graph, [llstr(code)])
 
     def test_add(self):
@@ -83,6 +92,7 @@
                           "int_lt": 1, "guard_true": 1, "jump": 1})
 
     def test_max(self):
+        py.test.skip("broken, investigate")
         result = self.run("""
         a = |30|
         a[13] = 128
@@ -95,6 +105,7 @@
                           "int_lt": 1, "guard_true": 1, "jump": 1})
 
     def test_min(self):
+        py.test.skip("broken, investigate")
         result = self.run("""
         a = |30|
         a[15] = -12
@@ -106,113 +117,72 @@
                           "float_mul": 1, "int_add": 1,
                           "int_lt": 1, "guard_true": 1, "jump": 1})
 
-class DisabledTestNumpy(object):
-
-    def test_argmin(self):
-        space = self.space
-        float64_dtype = self.float64_dtype
-
-        def f(i):
-            ar = SingleDimArray(i, dtype=NonConstant(float64_dtype))
-            j = 0
-            while j < i:
-                ar.get_concrete().setitem(j, float64_dtype.box(float(j)))
-                j += 1
-            return ar.descr_add(space, ar).descr_argmin(space).intval
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
+    def test_any(self):
+        result = self.run("""
+        a = [0,0,0,0,0,0,0,0,0,0,0]
+        a[8] = -12
+        b = a + a
+        any(b)
+        """)
+        assert result == 1
         self.check_loops({"getarrayitem_raw": 2, "float_add": 1,
-                           "float_lt": 1, "int_add": 1,
-                           "int_lt": 1, "guard_true": 2,
-                           "jump": 1})
-        assert result == f(5)
-
-    def test_all(self):
-        space = self.space
-        float64_dtype = self.float64_dtype
-
-        def f(i):
-            ar = SingleDimArray(i, dtype=NonConstant(float64_dtype))
-            j = 0
-            while j < i:
-                ar.get_concrete().setitem(j, float64_dtype.box(1.0))
-                j += 1
-            return ar.descr_add(space, ar).descr_all(space).boolval
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
-        self.check_loops({"getarrayitem_raw": 2, "float_add": 1,
-                          "int_add": 1, "float_ne": 1,
-                          "int_lt": 1, "guard_true": 2, "jump": 1})
-        assert result == f(5)
-
-    def test_any(self):
-        space = self.space
-        float64_dtype = self.float64_dtype
-
-        def f(i):
-            ar = SingleDimArray(i, dtype=NonConstant(float64_dtype))
-            return ar.descr_add(space, ar).descr_any(space).boolval
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
-        self.check_loops({"getarrayitem_raw": 2, "float_add": 1,
-                          "int_add": 1, "float_ne": 1, "guard_false": 1,
-                          "int_lt": 1, "guard_true": 1, "jump": 1})
-        assert result == f(5)
+                          "float_ne": 1, "int_add": 1,
+                          "int_lt": 1, "guard_true": 1, "jump": 1,
+                          "guard_false": 1})
 
     def test_already_forced(self):
-        space = self.space
-
-        def f(i):
-            ar = SingleDimArray(i, dtype=self.float64_dtype)
-            v1 = interp_ufuncs.get(self.space).add.call(space, [ar, scalar_w(space, self.float64_dtype, space.wrap(4.5))])
-            assert isinstance(v1, BaseArray)
-            v2 = interp_ufuncs.get(self.space).multiply.call(space, [v1, scalar_w(space, self.float64_dtype, space.wrap(4.5))])
-            v1.force_if_needed()
-            assert isinstance(v2, BaseArray)
-            return v2.get_concrete().eval(3).val
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
+        result = self.run("""
+        a = |30|
+        b = a + 4.5
+        b -> 5 # forces
+        c = b * 8
+        c -> 5
+        """)
+        assert result == (5 + 4.5) * 8
         # This is the sum of the ops for both loops, however if you remove the
         # optimization then you end up with 2 float_adds, so we can still be
         # sure it was optimized correctly.
         self.check_loops({"getarrayitem_raw": 2, "float_mul": 1, "float_add": 1,
                            "setarrayitem_raw": 2, "int_add": 2,
                            "int_lt": 2, "guard_true": 2, "jump": 2})
-        assert result == f(5)
 
     def test_ufunc(self):
-        space = self.space
-        def f(i):
-            ar = SingleDimArray(i, dtype=self.float64_dtype)
-            v1 = interp_ufuncs.get(self.space).add.call(space, [ar, ar])
-            v2 = interp_ufuncs.get(self.space).negative.call(space, [v1])
-            return v2.get_concrete().eval(3).val
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
+        result = self.run("""
+        a = |30|
+        b = a + a
+        c = unegative(b)
+        c -> 3
+        """)
+        assert result == -6
         self.check_loops({"getarrayitem_raw": 2, "float_add": 1, "float_neg": 1,
                           "setarrayitem_raw": 1, "int_add": 1,
                           "int_lt": 1, "guard_true": 1, "jump": 1,
         })
-        assert result == f(5)
 
-    def test_appropriate_specialization(self):
-        space = self.space
-        def f(i):
-            ar = SingleDimArray(i, dtype=self.float64_dtype)
-
-            v1 = interp_ufuncs.get(self.space).add.call(space, [ar, ar])
-            v2 = interp_ufuncs.get(self.space).negative.call(space, [v1])
-            v2.get_concrete()
-
-            for i in xrange(5):
-                v1 = interp_ufuncs.get(self.space).multiply.call(space, [ar, ar])
-                v2 = interp_ufuncs.get(self.space).negative.call(space, [v1])
-                v2.get_concrete()
-
-        self.meta_interp(f, [5], listops=True, backendopt=True)
+    def test_specialization(self):
+        self.run("""
+        a = |30|
+        b = a + a
+        c = unegative(b)
+        c -> 3
+        d = a * a
+        unegative(d)
+        d -> 3
+        d = a * a
+        unegative(d)
+        d -> 3
+        d = a * a
+        unegative(d)
+        d -> 3
+        d = a * a
+        unegative(d)
+        d -> 3
+        """)
         # This is 3, not 2 because there is a bridge for the exit.
         self.check_loop_count(3)
 
+
+class DisabledTestNumpy(object):
     def test_slice(self):
         def f(i):
             step = 3

From noreply at buildbot.pypy.org  Fri Oct 28 14:38:22 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 28 Oct 2011 14:38:22 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-minilang: make old tests pass (just
	because).
Message-ID: <20111028123822.54D58820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: numpy-minilang
Changeset: r48562:c1f7d371f290
Date: 2011-10-28 14:37 +0200
http://bitbucket.org/pypy/pypy/changeset/c1f7d371f290/

Log:	make old tests pass (just because).

diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -127,7 +127,6 @@
     def __init__(self, items):
         self.items = items
 
-
 class InterpreterState(object):
     def __init__(self, code):
         self.code = code
@@ -197,8 +196,11 @@
 
     def execute(self, interp):
         w_lhs = self.lhs.execute(interp)
+        assert isinstance(w_lhs, BaseArray)
+        if isinstance(self.rhs, SliceConstant):
+            # XXX interface has changed on multidim branch
+            raise NotImplementedError
         w_rhs = self.rhs.execute(interp)
-        assert isinstance(w_lhs, BaseArray)
         if self.name == '+':
             w_res = w_lhs.descr_add(interp.space, w_rhs)
         elif self.name == '*':
@@ -273,6 +275,13 @@
     def __repr__(self):
         return "[" + ", ".join([repr(item) for item in self.items]) + "]"
 
+class SliceConstant(Node):
+    def __init__(self):
+        pass
+
+    def __repr__(self):
+        return 'slice()'
+
 class Execute(Node):
     def __init__(self, expr):
         self.expr = expr
@@ -360,6 +369,10 @@
     def parse_constant(self, v):
         lgt = len(v)-1
         assert lgt >= 0
+        if ':' in v:
+            # a slice
+            assert v == ':'
+            return SliceConstant()
         if v[0] == '[':
             return ArrayConstant([self.parse_constant(elem)
                                   for elem in v[1:lgt].split(",")])
@@ -370,7 +383,7 @@
     def is_identifier_or_const(self, v):
         c = v[0]
         if ((c >= 'a' and c <= 'z') or (c >= 'A' and c <= 'Z') or
-            (c >= '0' and c <= '9') or c in '-.[|'):
+            (c >= '0' and c <= '9') or c in '-.[|:'):
             if v == '-' or v == "->":
                 return False
             return True
diff --git a/pypy/module/micronumpy/test/test_compile.py b/pypy/module/micronumpy/test/test_compile.py
--- a/pypy/module/micronumpy/test/test_compile.py
+++ b/pypy/module/micronumpy/test/test_compile.py
@@ -158,3 +158,12 @@
         max(b)
         """)
         assert interp.results[0].value.val == 256
+
+    def test_slice(self):
+        py.test.skip("in progress")
+        interp = self.run("""
+        a = [1,2,3,4]
+        b = a -> :
+        b -> 3
+        """)
+        assert interp.results[0].value.val == 3
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -2,7 +2,7 @@
 from pypy.module.micronumpy import interp_ufuncs, signature
 from pypy.module.micronumpy.compile import (FakeSpace,
     FloatObject, IntObject, numpy_compile, BoolObject)
-from pypy.module.micronumpy.interp_numarray import (BaseArray, SingleDimArray,
+from pypy.module.micronumpy.interp_numarray import (SingleDimArray,
     SingleDimSlice)
 from pypy.rlib.nonconst import NonConstant
 from pypy.rpython.annlowlevel import llstr, hlstr
@@ -182,7 +182,14 @@
         self.check_loop_count(3)
 
 
-class DisabledTestNumpy(object):
+class TestNumpyOld(LLJitMixin):
+    def setup_class(cls):
+        from pypy.module.micronumpy.compile import FakeSpace
+        from pypy.module.micronumpy.interp_dtype import W_Float64Dtype
+        
+        cls.space = FakeSpace()
+        cls.float64_dtype = cls.space.fromcache(W_Float64Dtype)
+    
     def test_slice(self):
         def f(i):
             step = 3

From noreply at buildbot.pypy.org  Fri Oct 28 15:01:44 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 28 Oct 2011 15:01:44 +0200 (CEST)
Subject: [pypy-commit] pypy default: Merge numpy-minilang,
 this refactors test_zjit to use new cool mini-language
Message-ID: <20111028130144.BBE60820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r48563:c91b4f3c204f
Date: 2011-10-28 15:00 +0200
http://bitbucket.org/pypy/pypy/changeset/c91b4f3c204f/

Log:	Merge numpy-minilang, this refactors test_zjit to use new cool mini-
	language instead of hardcoding things.

diff --git a/pypy/jit/metainterp/history.py b/pypy/jit/metainterp/history.py
--- a/pypy/jit/metainterp/history.py
+++ b/pypy/jit/metainterp/history.py
@@ -929,6 +929,9 @@
     def view(self, **kwds):
         pass
 
+    def clear(self):
+        pass
+
 class Stats(object):
     """For tests."""
 
@@ -943,6 +946,15 @@
         self.aborted_keys = []
         self.invalidated_token_numbers = set()
 
+    def clear(self):
+        del self.loops[:]
+        del self.locations[:]
+        del self.aborted_keys[:]
+        self.invalidated_token_numbers.clear()
+        self.compiled_count = 0
+        self.enter_count = 0
+        self.aborted_count = 0
+
     def set_history(self, history):
         self.operations = history.operations
 
diff --git a/pypy/jit/metainterp/warmspot.py b/pypy/jit/metainterp/warmspot.py
--- a/pypy/jit/metainterp/warmspot.py
+++ b/pypy/jit/metainterp/warmspot.py
@@ -62,7 +62,7 @@
     clear_tcache()
     return jittify_and_run(interp, graph, args, backendopt=backendopt, **kwds)
 
-def jittify_and_run(interp, graph, args, repeat=1,
+def jittify_and_run(interp, graph, args, repeat=1, graph_and_interp_only=False,
                     backendopt=False, trace_limit=sys.maxint,
                     inline=False, loop_longevity=0, retrace_limit=5,
                     function_threshold=4,
@@ -93,6 +93,8 @@
         jd.warmstate.set_param_max_retrace_guards(max_retrace_guards)
         jd.warmstate.set_param_enable_opts(enable_opts)
     warmrunnerdesc.finish()
+    if graph_and_interp_only:
+        return interp, graph
     res = interp.eval_graph(graph, args)
     if not kwds.get('translate_support_code', False):
         warmrunnerdesc.metainterp_sd.profiler.finish()
@@ -157,6 +159,9 @@
 def get_stats():
     return pyjitpl._warmrunnerdesc.stats
 
+def reset_stats():
+    pyjitpl._warmrunnerdesc.stats.clear()
+
 def get_translator():
     return pyjitpl._warmrunnerdesc.translator
 
diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -4,30 +4,52 @@
 """
 
 from pypy.interpreter.baseobjspace import InternalSpaceCache, W_Root
-from pypy.module.micronumpy.interp_dtype import W_Float64Dtype
-from pypy.module.micronumpy.interp_numarray import Scalar, SingleDimArray, BaseArray
+from pypy.module.micronumpy.interp_dtype import W_Float64Dtype, W_BoolDtype
+from pypy.module.micronumpy.interp_numarray import (Scalar, BaseArray,
+     descr_new_array, scalar_w, SingleDimArray)
+from pypy.module.micronumpy import interp_ufuncs
 from pypy.rlib.objectmodel import specialize
 
 
 class BogusBytecode(Exception):
     pass
 
-def create_array(dtype, size):
-    a = SingleDimArray(size, dtype=dtype)
-    for i in range(size):
-        dtype.setitem(a.storage, i, dtype.box(float(i % 10)))
-    return a
+class ArgumentMismatch(Exception):
+    pass
+
+class ArgumentNotAnArray(Exception):
+    pass
+
+class WrongFunctionName(Exception):
+    pass
+
+SINGLE_ARG_FUNCTIONS = ["sum", "prod", "max", "min", "all", "any", "unegative"]
 
 class FakeSpace(object):
     w_ValueError = None
     w_TypeError = None
+    w_None = None
+
+    w_bool = "bool"
+    w_int = "int"
+    w_float = "float"
+    w_list = "list"
+    w_long = "long"
+    w_tuple = 'tuple'
 
     def __init__(self):
         """NOT_RPYTHON"""
         self.fromcache = InternalSpaceCache(self).getorbuild
+        self.w_float64dtype = W_Float64Dtype(self)
 
     def issequence_w(self, w_obj):
-        return True
+        return isinstance(w_obj, ListObject) or isinstance(w_obj, SingleDimArray)
+
+    def isinstance_w(self, w_obj, w_tp):
+        return False
+
+    def decode_index4(self, w_idx, size):
+        return (self.int_w(w_idx), 0, 0, 1)
 
     @specialize.argtype(1)
     def wrap(self, obj):
@@ -39,72 +61,382 @@
             return IntObject(obj)
         raise Exception
 
+    def newlist(self, items):
+        return ListObject(items)
+
+    def listview(self, obj):
+        assert isinstance(obj, ListObject)
+        return obj.items
+
     def float(self, w_obj):
         assert isinstance(w_obj, FloatObject)
         return w_obj
 
     def float_w(self, w_obj):
+        assert isinstance(w_obj, FloatObject)        
         return w_obj.floatval
 
+    def int_w(self, w_obj):
+        if isinstance(w_obj, IntObject):
+            return w_obj.intval
+        elif isinstance(w_obj, FloatObject):
+            return int(w_obj.floatval)
+        raise NotImplementedError
+
+    def int(self, w_obj):
+        return w_obj
+
+    def is_true(self, w_obj):
+        assert isinstance(w_obj, BoolObject)
+        return w_obj.boolval
+
+    def is_w(self, w_obj, w_what):
+        return w_obj is w_what
+
+    def type(self, w_obj):
+        return w_obj.tp
+
+    def gettypefor(self, w_obj):
+        return None
+
+    def call_function(self, tp, w_dtype):
+        return w_dtype
+
+    @specialize.arg(1)
+    def interp_w(self, tp, what):
+        assert isinstance(what, tp)
+        return what
 
 class FloatObject(W_Root):
+    tp = FakeSpace.w_float
     def __init__(self, floatval):
         self.floatval = floatval
 
 class BoolObject(W_Root):
+    tp = FakeSpace.w_bool
     def __init__(self, boolval):
         self.boolval = boolval
 
 class IntObject(W_Root):
+    tp = FakeSpace.w_int
     def __init__(self, intval):
         self.intval = intval
 
+class ListObject(W_Root):
+    tp = FakeSpace.w_list
+    def __init__(self, items):
+        self.items = items
 
-space = FakeSpace()
+class InterpreterState(object):
+    def __init__(self, code):
+        self.code = code
+        self.variables = {}
+        self.results = []
 
-def numpy_compile(bytecode, array_size):
-    stack = []
-    i = 0
-    dtype = space.fromcache(W_Float64Dtype)
-    for b in bytecode:
-        if b == 'a':
-            stack.append(create_array(dtype, array_size))
-            i += 1
-        elif b == 'f':
-            stack.append(Scalar(dtype, dtype.box(1.2)))
-        elif b == '+':
-            right = stack.pop()
-            res = stack.pop().descr_add(space, right)
-            assert isinstance(res, BaseArray)
-            stack.append(res)
-        elif b == '-':
-            right = stack.pop()
-            res = stack.pop().descr_sub(space, right)
-            assert isinstance(res, BaseArray)
-            stack.append(res)
-        elif b == '*':
-            right = stack.pop()
-            res = stack.pop().descr_mul(space, right)
-            assert isinstance(res, BaseArray)
-            stack.append(res)
-        elif b == '/':
-            right = stack.pop()
-            res = stack.pop().descr_div(space, right)
-            assert isinstance(res, BaseArray)
-            stack.append(res)
-        elif b == '%':
-            right = stack.pop()
-            res = stack.pop().descr_mod(space, right)
-            assert isinstance(res, BaseArray)
-            stack.append(res)
-        elif b == '|':
-            res = stack.pop().descr_abs(space)
-            assert isinstance(res, BaseArray)
-            stack.append(res)
+    def run(self, space):
+        self.space = space
+        for stmt in self.code.statements:
+            stmt.execute(self)
+
+class Node(object):
+    def __eq__(self, other):
+        return (self.__class__ == other.__class__ and
+                self.__dict__ == other.__dict__)
+
+    def __ne__(self, other):
+        return not self == other
+
+    def wrap(self, space):
+        raise NotImplementedError
+
+    def execute(self, interp):
+        raise NotImplementedError
+
+class Assignment(Node):
+    def __init__(self, name, expr):
+        self.name = name
+        self.expr = expr
+
+    def execute(self, interp):
+        interp.variables[self.name] = self.expr.execute(interp)
+
+    def __repr__(self):
+        return "%% = %r" % (self.name, self.expr)
+
+class ArrayAssignment(Node):
+    def __init__(self, name, index, expr):
+        self.name = name
+        self.index = index
+        self.expr = expr
+
+    def execute(self, interp):
+        arr = interp.variables[self.name]
+        w_index = self.index.execute(interp).eval(0).wrap(interp.space)
+        w_val = self.expr.execute(interp).eval(0).wrap(interp.space)
+        arr.descr_setitem(interp.space, w_index, w_val)
+
+    def __repr__(self):
+        return "%s[%r] = %r" % (self.name, self.index, self.expr)
+
+class Variable(Node):
+    def __init__(self, name):
+        self.name = name
+
+    def execute(self, interp):
+        return interp.variables[self.name]
+
+    def __repr__(self):
+        return 'v(%s)' % self.name
+
+class Operator(Node):
+    def __init__(self, lhs, name, rhs):
+        self.name = name
+        self.lhs = lhs
+        self.rhs = rhs
+
+    def execute(self, interp):
+        w_lhs = self.lhs.execute(interp)
+        assert isinstance(w_lhs, BaseArray)
+        if isinstance(self.rhs, SliceConstant):
+            # XXX interface has changed on multidim branch
+            raise NotImplementedError
+        w_rhs = self.rhs.execute(interp)
+        if self.name == '+':
+            w_res = w_lhs.descr_add(interp.space, w_rhs)
+        elif self.name == '*':
+            w_res = w_lhs.descr_mul(interp.space, w_rhs)
+        elif self.name == '-':
+            w_res = w_lhs.descr_sub(interp.space, w_rhs)            
+        elif self.name == '->':
+            if isinstance(w_rhs, Scalar):
+                index = int(interp.space.float_w(
+                    w_rhs.value.wrap(interp.space)))
+                dtype = interp.space.fromcache(W_Float64Dtype)
+                return Scalar(dtype, w_lhs.get_concrete().eval(index))
+            else:
+                raise NotImplementedError
         else:
-            print "Unknown opcode: %s" % b
-            raise BogusBytecode()
-    if len(stack) != 1:
-        print "Bogus bytecode, uneven stack length"
-        raise BogusBytecode()
-    return stack[0]
+            raise NotImplementedError
+        if not isinstance(w_res, BaseArray):
+            dtype = interp.space.fromcache(W_Float64Dtype)
+            w_res = scalar_w(interp.space, dtype, w_res)
+        return w_res
+
+    def __repr__(self):
+        return '(%r %s %r)' % (self.lhs, self.name, self.rhs)
+
+class FloatConstant(Node):
+    def __init__(self, v):
+        self.v = float(v)
+
+    def __repr__(self):
+        return "Const(%s)" % self.v
+
+    def wrap(self, space):
+        return space.wrap(self.v)
+
+    def execute(self, interp):
+        dtype = interp.space.fromcache(W_Float64Dtype)
+        assert isinstance(dtype, W_Float64Dtype)
+        return Scalar(dtype, dtype.box(self.v))
+
+class RangeConstant(Node):
+    def __init__(self, v):
+        self.v = int(v)
+
+    def execute(self, interp):
+        w_list = interp.space.newlist(
+            [interp.space.wrap(float(i)) for i in range(self.v)])
+        dtype = interp.space.fromcache(W_Float64Dtype)
+        return descr_new_array(interp.space, None, w_list, w_dtype=dtype)
+
+    def __repr__(self):
+        return 'Range(%s)' % self.v
+
+class Code(Node):
+    def __init__(self, statements):
+        self.statements = statements
+
+    def __repr__(self):
+        return "\n".join([repr(i) for i in self.statements])
+
+class ArrayConstant(Node):
+    def __init__(self, items):
+        self.items = items
+
+    def wrap(self, space):
+        return space.newlist([item.wrap(space) for item in self.items])
+
+    def execute(self, interp):
+        w_list = self.wrap(interp.space)
+        dtype = interp.space.fromcache(W_Float64Dtype)
+        return descr_new_array(interp.space, None, w_list, w_dtype=dtype)
+
+    def __repr__(self):
+        return "[" + ", ".join([repr(item) for item in self.items]) + "]"
+
+class SliceConstant(Node):
+    def __init__(self):
+        pass
+
+    def __repr__(self):
+        return 'slice()'
+
+class Execute(Node):
+    def __init__(self, expr):
+        self.expr = expr
+
+    def __repr__(self):
+        return repr(self.expr)
+
+    def execute(self, interp):
+        interp.results.append(self.expr.execute(interp))
+
+class FunctionCall(Node):
+    def __init__(self, name, args):
+        self.name = name
+        self.args = args
+
+    def __repr__(self):
+        return "%s(%s)" % (self.name, ", ".join([repr(arg)
+                                                 for arg in self.args]))
+
+    def execute(self, interp):
+        if self.name in SINGLE_ARG_FUNCTIONS:
+            if len(self.args) != 1:
+                raise ArgumentMismatch
+            arr = self.args[0].execute(interp)
+            if not isinstance(arr, BaseArray):
+                raise ArgumentNotAnArray
+            if self.name == "sum":
+                w_res = arr.descr_sum(interp.space)
+            elif self.name == "prod":
+                w_res = arr.descr_prod(interp.space)
+            elif self.name == "max":
+                w_res = arr.descr_max(interp.space)
+            elif self.name == "min":
+                w_res = arr.descr_min(interp.space)
+            elif self.name == "any":
+                w_res = arr.descr_any(interp.space)
+            elif self.name == "all":
+                w_res = arr.descr_all(interp.space)
+            elif self.name == "unegative":
+                neg = interp_ufuncs.get(interp.space).negative
+                w_res = neg.call(interp.space, [arr])
+            else:
+                assert False # unreachable code
+            if isinstance(w_res, BaseArray):
+                return w_res
+            if isinstance(w_res, FloatObject):
+                dtype = interp.space.fromcache(W_Float64Dtype)
+            elif isinstance(w_res, BoolObject):
+                dtype = interp.space.fromcache(W_BoolDtype)
+            else:
+                dtype = None
+            return scalar_w(interp.space, dtype, w_res)
+        else:
+            raise WrongFunctionName
+
+class Parser(object):
+    def parse_identifier(self, id):
+        id = id.strip(" ")
+        #assert id.isalpha()
+        return Variable(id)
+
+    def parse_expression(self, expr):
+        tokens = [i for i in expr.split(" ") if i]
+        if len(tokens) == 1:
+            return self.parse_constant_or_identifier(tokens[0])
+        stack = []
+        tokens.reverse()
+        while tokens:
+            token = tokens.pop()
+            if token == ')':
+                raise NotImplementedError
+            elif self.is_identifier_or_const(token):
+                if stack:
+                    name = stack.pop().name
+                    lhs = stack.pop()
+                    rhs = self.parse_constant_or_identifier(token)
+                    stack.append(Operator(lhs, name, rhs))
+                else:
+                    stack.append(self.parse_constant_or_identifier(token))
+            else:
+                stack.append(Variable(token))
+        assert len(stack) == 1
+        return stack[-1]
+
+    def parse_constant(self, v):
+        lgt = len(v)-1
+        assert lgt >= 0
+        if ':' in v:
+            # a slice
+            assert v == ':'
+            return SliceConstant()
+        if v[0] == '[':
+            return ArrayConstant([self.parse_constant(elem)
+                                  for elem in v[1:lgt].split(",")])
+        if v[0] == '|':
+            return RangeConstant(v[1:lgt])
+        return FloatConstant(v)
+
+    def is_identifier_or_const(self, v):
+        c = v[0]
+        if ((c >= 'a' and c <= 'z') or (c >= 'A' and c <= 'Z') or
+            (c >= '0' and c <= '9') or c in '-.[|:'):
+            if v == '-' or v == "->":
+                return False
+            return True
+        return False
+
+    def parse_function_call(self, v):
+        l = v.split('(')
+        assert len(l) == 2
+        name = l[0]
+        cut = len(l[1]) - 1
+        assert cut >= 0
+        args = [self.parse_constant_or_identifier(id)
+                for id in l[1][:cut].split(",")]
+        return FunctionCall(name, args)
+
+    def parse_constant_or_identifier(self, v):
+        c = v[0]
+        if (c >= 'a' and c <= 'z') or (c >= 'A' and c <= 'Z'):
+            if '(' in v:
+                return self.parse_function_call(v)
+            return self.parse_identifier(v)
+        return self.parse_constant(v)
+
+    def parse_array_subscript(self, v):
+        v = v.strip(" ")
+        l = v.split("[")
+        lgt = len(l[1]) - 1
+        assert lgt >= 0
+        rhs = self.parse_constant_or_identifier(l[1][:lgt])
+        return l[0], rhs
+        
+    def parse_statement(self, line):
+        if '=' in line:
+            lhs, rhs = line.split("=")
+            lhs = lhs.strip(" ")
+            if '[' in lhs:
+                name, index = self.parse_array_subscript(lhs)
+                return ArrayAssignment(name, index, self.parse_expression(rhs))
+            else: 
+                return Assignment(lhs, self.parse_expression(rhs))
+        else:
+            return Execute(self.parse_expression(line))
+
+    def parse(self, code):
+        statements = []
+        for line in code.split("\n"):
+            if '#' in line:
+                line = line.split('#', 1)[0]
+            line = line.strip(" ")
+            if line:
+                statements.append(self.parse_statement(line))
+        return Code(statements)
+
+def numpy_compile(code):
+    parser = Parser()
+    return InterpreterState(parser.parse(code))
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -14,6 +14,27 @@
 any_driver = jit.JitDriver(greens=['signature'], reds=['i', 'size', 'self', 'dtype'])
 slice_driver = jit.JitDriver(greens=['signature'], reds=['i', 'j', 'step', 'stop', 'source', 'dest'])
 
+def descr_new_array(space, w_subtype, w_size_or_iterable, w_dtype=None):
+    l = space.listview(w_size_or_iterable)
+    if space.is_w(w_dtype, space.w_None):
+        w_dtype = None
+        for w_item in l:
+            w_dtype = interp_ufuncs.find_dtype_for_scalar(space, w_item, w_dtype)
+            if w_dtype is space.fromcache(interp_dtype.W_Float64Dtype):
+                break
+        if w_dtype is None:
+            w_dtype = space.w_None
+
+    dtype = space.interp_w(interp_dtype.W_Dtype,
+        space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype)
+    )
+    arr = SingleDimArray(len(l), dtype=dtype)
+    i = 0
+    for w_elem in l:
+        dtype.setitem_w(space, arr.storage, i, w_elem)
+        i += 1
+    return arr
+
 class BaseArray(Wrappable):
     _attrs_ = ["invalidates", "signature"]
 
@@ -32,27 +53,6 @@
     def add_invalidates(self, other):
         self.invalidates.append(other)
 
-    def descr__new__(space, w_subtype, w_size_or_iterable, w_dtype=None):
-        l = space.listview(w_size_or_iterable)
-        if space.is_w(w_dtype, space.w_None):
-            w_dtype = None
-            for w_item in l:
-                w_dtype = interp_ufuncs.find_dtype_for_scalar(space, w_item, w_dtype)
-                if w_dtype is space.fromcache(interp_dtype.W_Float64Dtype):
-                    break
-            if w_dtype is None:
-                w_dtype = space.w_None
-
-        dtype = space.interp_w(interp_dtype.W_Dtype,
-            space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype)
-        )
-        arr = SingleDimArray(len(l), dtype=dtype)
-        i = 0
-        for w_elem in l:
-            dtype.setitem_w(space, arr.storage, i, w_elem)
-            i += 1
-        return arr
-
     def _unaryop_impl(ufunc_name):
         def impl(self, space):
             return getattr(interp_ufuncs.get(space), ufunc_name).call(space, [self])
@@ -570,7 +570,7 @@
 
 BaseArray.typedef = TypeDef(
     'numarray',
-    __new__ = interp2app(BaseArray.descr__new__.im_func),
+    __new__ = interp2app(descr_new_array),
 
 
     __len__ = interp2app(BaseArray.descr_len),
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -236,22 +236,20 @@
     return dt
 
 def find_dtype_for_scalar(space, w_obj, current_guess=None):
-    w_type = space.type(w_obj)
-
     bool_dtype = space.fromcache(interp_dtype.W_BoolDtype)
     long_dtype = space.fromcache(interp_dtype.W_LongDtype)
     int64_dtype = space.fromcache(interp_dtype.W_Int64Dtype)
 
-    if space.is_w(w_type, space.w_bool):
+    if space.isinstance_w(w_obj, space.w_bool):
         if current_guess is None or current_guess is bool_dtype:
             return bool_dtype
         return current_guess
-    elif space.is_w(w_type, space.w_int):
+    elif space.isinstance_w(w_obj, space.w_int):
         if (current_guess is None or current_guess is bool_dtype or
             current_guess is long_dtype):
             return long_dtype
         return current_guess
-    elif space.is_w(w_type, space.w_long):
+    elif space.isinstance_w(w_obj, space.w_long):
         if (current_guess is None or current_guess is bool_dtype or
             current_guess is long_dtype or current_guess is int64_dtype):
             return int64_dtype
diff --git a/pypy/module/micronumpy/test/test_compile.py b/pypy/module/micronumpy/test/test_compile.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/test/test_compile.py
@@ -0,0 +1,169 @@
+
+from pypy.module.micronumpy.compile import *
+
+class TestCompiler(object):
+    def compile(self, code):
+        return numpy_compile(code)
+    
+    def test_vars(self):
+        code = """
+        a = 2
+        b = 3
+        """
+        interp = self.compile(code)
+        assert isinstance(interp.code.statements[0], Assignment)
+        assert interp.code.statements[0].name == 'a'
+        assert interp.code.statements[0].expr.v == 2
+        assert interp.code.statements[1].name == 'b'
+        assert interp.code.statements[1].expr.v == 3
+
+    def test_array_literal(self):
+        code = "a = [1,2,3]"
+        interp = self.compile(code)
+        assert isinstance(interp.code.statements[0].expr, ArrayConstant)
+        st = interp.code.statements[0]
+        assert st.expr.items == [FloatConstant(1), FloatConstant(2),
+                                 FloatConstant(3)]
+    
+    def test_array_literal2(self):
+        code = "a = [[1],[2],[3]]"
+        interp = self.compile(code)
+        assert isinstance(interp.code.statements[0].expr, ArrayConstant)
+        st = interp.code.statements[0]
+        assert st.expr.items == [ArrayConstant([FloatConstant(1)]),
+                                 ArrayConstant([FloatConstant(2)]),
+                                 ArrayConstant([FloatConstant(3)])]
+
+    def test_expr_1(self):
+        code = "b = a + 1"
+        interp = self.compile(code)
+        assert (interp.code.statements[0].expr ==
+                Operator(Variable("a"), "+", FloatConstant(1)))
+
+    def test_expr_2(self):
+        code = "b = a + b - 3"
+        interp = self.compile(code)
+        assert (interp.code.statements[0].expr ==
+                Operator(Operator(Variable("a"), "+", Variable("b")), "-",
+                         FloatConstant(3)))
+
+    def test_expr_3(self):
+        # an equivalent of range
+        code = "a = |20|"
+        interp = self.compile(code)
+        assert interp.code.statements[0].expr == RangeConstant(20)
+
+    def test_expr_only(self):
+        code = "3 + a"
+        interp = self.compile(code)
+        assert interp.code.statements[0] == Execute(
+            Operator(FloatConstant(3), "+", Variable("a")))
+
+    def test_array_access(self):
+        code = "a -> 3"
+        interp = self.compile(code)
+        assert interp.code.statements[0] == Execute(
+            Operator(Variable("a"), "->", FloatConstant(3)))
+
+    def test_function_call(self):
+        code = "sum(a)"
+        interp = self.compile(code)
+        assert interp.code.statements[0] == Execute(
+            FunctionCall("sum", [Variable("a")]))
+
+    def test_comment(self):
+        code = """
+        # some comment
+        a = b + 3  # another comment
+        """
+        interp = self.compile(code)
+        assert interp.code.statements[0] == Assignment(
+            'a', Operator(Variable('b'), "+", FloatConstant(3)))
+
+class TestRunner(object):
+    def run(self, code):
+        interp = numpy_compile(code)
+        space = FakeSpace()
+        interp.run(space)
+        return interp
+
+    def test_one(self):
+        code = """
+        a = 3
+        b = 4
+        a + b
+        """
+        interp = self.run(code)
+        assert sorted(interp.variables.keys()) == ['a', 'b']
+        assert interp.results[0]
+
+    def test_array_add(self):
+        code = """
+        a = [1,2,3,4]
+        b = [4,5,6,5]
+        a + b
+        """
+        interp = self.run(code)
+        assert interp.results[0]._getnums(False) == ["5.0", "7.0", "9.0", "9.0"]
+
+    def test_array_getitem(self):
+        code = """
+        a = [1,2,3,4]
+        b = [4,5,6,5]
+        a + b -> 3
+        """
+        interp = self.run(code)
+        assert interp.results[0].value.val == 3 + 6
+        
+    def test_range_getitem(self):
+        code = """
+        r = |20| + 3
+        r -> 3
+        """
+        interp = self.run(code)
+        assert interp.results[0].value.val == 6
+
+    def test_sum(self):
+        code = """
+        a = [1,2,3,4,5]
+        r = sum(a)
+        r
+        """
+        interp = self.run(code)
+        assert interp.results[0].value.val == 15
+
+    def test_array_write(self):
+        code = """
+        a = [1,2,3,4,5]
+        a[3] = 15
+        a -> 3
+        """
+        interp = self.run(code)
+        assert interp.results[0].value.val == 15
+
+    def test_min(self):
+        interp = self.run("""
+        a = |30|
+        a[15] = -12
+        b = a + a
+        min(b)
+        """)
+        assert interp.results[0].value.val == -24
+
+    def test_max(self):
+        interp = self.run("""
+        a = |30|
+        a[13] = 128
+        b = a + a
+        max(b)
+        """)
+        assert interp.results[0].value.val == 256
+
+    def test_slice(self):
+        py.test.skip("in progress")
+        interp = self.run("""
+        a = [1,2,3,4]
+        b = a -> :
+        b -> 3
+        """)
+        assert interp.results[0].value.val == 3
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -82,6 +82,8 @@
         b = negative(a)
         a[0] = 5.0
         assert b[0] == 5.0
+        a = array(range(30))
+        assert negative(a + a)[3] == -6
 
     def test_abs(self):
         from numpy import array, absolute
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -1,253 +1,195 @@
 from pypy.jit.metainterp.test.support import LLJitMixin
 from pypy.module.micronumpy import interp_ufuncs, signature
-from pypy.module.micronumpy.compile import (numpy_compile, FakeSpace,
-    FloatObject, IntObject)
-from pypy.module.micronumpy.interp_dtype import W_Int32Dtype, W_Float64Dtype, W_Int64Dtype, W_UInt64Dtype
-from pypy.module.micronumpy.interp_numarray import (BaseArray, SingleDimArray,
-    SingleDimSlice, scalar_w)
+from pypy.module.micronumpy.compile import (FakeSpace,
+    FloatObject, IntObject, numpy_compile, BoolObject)
+from pypy.module.micronumpy.interp_numarray import (SingleDimArray,
+    SingleDimSlice)
 from pypy.rlib.nonconst import NonConstant
-from pypy.rpython.annlowlevel import llstr
-from pypy.rpython.test.test_llinterp import interpret
+from pypy.rpython.annlowlevel import llstr, hlstr
+from pypy.jit.metainterp.warmspot import reset_stats
+from pypy.jit.metainterp import pyjitpl
 
 import py
 
 
 class TestNumpyJIt(LLJitMixin):
-    def setup_class(cls):
-        cls.space = FakeSpace()
-        cls.float64_dtype = cls.space.fromcache(W_Float64Dtype)
-        cls.int64_dtype = cls.space.fromcache(W_Int64Dtype)
-        cls.uint64_dtype = cls.space.fromcache(W_UInt64Dtype)
-        cls.int32_dtype = cls.space.fromcache(W_Int32Dtype)
+    graph = None
+    interp = None
+        
+    def run(self, code):
+        space = FakeSpace()
+        
+        def f(code):
+            interp = numpy_compile(hlstr(code))
+            interp.run(space)
+            res = interp.results[-1]
+            w_res = res.eval(0).wrap(interp.space)
+            if isinstance(w_res, BoolObject):
+                return float(w_res.boolval)
+            elif isinstance(w_res, FloatObject):
+                return w_res.floatval
+            elif isinstance(w_res, IntObject):
+                return w_res.intval
+            else:
+                return -42.
+
+        if self.graph is None:
+            interp, graph = self.meta_interp(f, [llstr(code)],
+                                             listops=True,
+                                             backendopt=True,
+                                             graph_and_interp_only=True)
+            self.__class__.interp = interp
+            self.__class__.graph = graph
+
+        reset_stats()
+        pyjitpl._warmrunnerdesc.memory_manager.alive_loops.clear()
+        return self.interp.eval_graph(self.graph, [llstr(code)])
 
     def test_add(self):
-        def f(i):
-            ar = SingleDimArray(i, dtype=self.float64_dtype)
-            v = interp_ufuncs.get(self.space).add.call(self.space, [ar, ar])
-            return v.get_concrete().eval(3).val
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
+        result = self.run("""
+        a = |30|
+        b = a + a
+        b -> 3
+        """)
         self.check_loops({'getarrayitem_raw': 2, 'float_add': 1,
                           'setarrayitem_raw': 1, 'int_add': 1,
                           'int_lt': 1, 'guard_true': 1, 'jump': 1})
-        assert result == f(5)
+        assert result == 3 + 3
 
     def test_floatadd(self):
-        def f(i):
-            ar = SingleDimArray(i, dtype=self.float64_dtype)
-            v = interp_ufuncs.get(self.space).add.call(self.space, [
-                    ar,
-                    scalar_w(self.space, self.float64_dtype, self.space.wrap(4.5))
-                ],
-            )
-            assert isinstance(v, BaseArray)
-            return v.get_concrete().eval(3).val
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
+        result = self.run("""
+        a = |30| + 3
+        a -> 3
+        """)
+        assert result == 3 + 3
         self.check_loops({"getarrayitem_raw": 1, "float_add": 1,
                           "setarrayitem_raw": 1, "int_add": 1,
                           "int_lt": 1, "guard_true": 1, "jump": 1})
-        assert result == f(5)
 
     def test_sum(self):
-        space = self.space
-        float64_dtype = self.float64_dtype
-        int64_dtype = self.int64_dtype
-
-        def f(i):
-            if NonConstant(False):
-                dtype = int64_dtype
-            else:
-                dtype = float64_dtype
-            ar = SingleDimArray(i, dtype=dtype)
-            v = ar.descr_add(space, ar).descr_sum(space)
-            assert isinstance(v, FloatObject)
-            return v.floatval
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
+        result = self.run("""
+        a = |30|
+        b = a + a
+        sum(b)
+        """)
+        assert result == 2 * sum(range(30))
         self.check_loops({"getarrayitem_raw": 2, "float_add": 2,
                           "int_add": 1,
                           "int_lt": 1, "guard_true": 1, "jump": 1})
-        assert result == f(5)
 
     def test_prod(self):
-        space = self.space
-        float64_dtype = self.float64_dtype
-        int64_dtype = self.int64_dtype
-
-        def f(i):
-            if NonConstant(False):
-                dtype = int64_dtype
-            else:
-                dtype = float64_dtype
-            ar = SingleDimArray(i, dtype=dtype)
-            v = ar.descr_add(space, ar).descr_prod(space)
-            assert isinstance(v, FloatObject)
-            return v.floatval
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
+        result = self.run("""
+        a = |30|
+        b = a + a
+        prod(b)
+        """)
+        expected = 1
+        for i in range(30):
+            expected *= i * 2
+        assert result == expected
         self.check_loops({"getarrayitem_raw": 2, "float_add": 1,
                           "float_mul": 1, "int_add": 1,
                           "int_lt": 1, "guard_true": 1, "jump": 1})
-        assert result == f(5)
 
     def test_max(self):
-        space = self.space
-        float64_dtype = self.float64_dtype
-        int64_dtype = self.int64_dtype
-
-        def f(i):
-            if NonConstant(False):
-                dtype = int64_dtype
-            else:
-                dtype = float64_dtype
-            ar = SingleDimArray(i, dtype=dtype)
-            j = 0
-            while j < i:
-                ar.get_concrete().setitem(j, float64_dtype.box(float(j)))
-                j += 1
-            v = ar.descr_add(space, ar).descr_max(space)
-            assert isinstance(v, FloatObject)
-            return v.floatval
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
+        py.test.skip("broken, investigate")
+        result = self.run("""
+        a = |30|
+        a[13] = 128
+        b = a + a
+        max(b)
+        """)
+        assert result == 256
         self.check_loops({"getarrayitem_raw": 2, "float_add": 1,
-                          "float_gt": 1, "int_add": 1,
-                          "int_lt": 1, "guard_true": 1,
-                          "guard_false": 1, "jump": 1})
-        assert result == f(5)
+                          "float_mul": 1, "int_add": 1,
+                          "int_lt": 1, "guard_true": 1, "jump": 1})
 
     def test_min(self):
-        space = self.space
-        float64_dtype = self.float64_dtype
-        int64_dtype = self.int64_dtype
-
-        def f(i):
-            if NonConstant(False):
-                dtype = int64_dtype
-            else:
-                dtype = float64_dtype
-            ar = SingleDimArray(i, dtype=dtype)
-            j = 0
-            while j < i:
-                ar.get_concrete().setitem(j, float64_dtype.box(float(j)))
-                j += 1
-            v = ar.descr_add(space, ar).descr_min(space)
-            assert isinstance(v, FloatObject)
-            return v.floatval
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
+        py.test.skip("broken, investigate")
+        result = self.run("""
+        a = |30|
+        a[15] = -12
+        b = a + a
+        min(b)
+        """)
+        assert result == -24
         self.check_loops({"getarrayitem_raw": 2, "float_add": 1,
-                           "float_lt": 1, "int_add": 1,
-                           "int_lt": 1, "guard_true": 2,
-                           "jump": 1})
-        assert result == f(5)
-
-    def test_argmin(self):
-        space = self.space
-        float64_dtype = self.float64_dtype
-
-        def f(i):
-            ar = SingleDimArray(i, dtype=NonConstant(float64_dtype))
-            j = 0
-            while j < i:
-                ar.get_concrete().setitem(j, float64_dtype.box(float(j)))
-                j += 1
-            return ar.descr_add(space, ar).descr_argmin(space).intval
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
-        self.check_loops({"getarrayitem_raw": 2, "float_add": 1,
-                           "float_lt": 1, "int_add": 1,
-                           "int_lt": 1, "guard_true": 2,
-                           "jump": 1})
-        assert result == f(5)
-
-    def test_all(self):
-        space = self.space
-        float64_dtype = self.float64_dtype
-
-        def f(i):
-            ar = SingleDimArray(i, dtype=NonConstant(float64_dtype))
-            j = 0
-            while j < i:
-                ar.get_concrete().setitem(j, float64_dtype.box(1.0))
-                j += 1
-            return ar.descr_add(space, ar).descr_all(space).boolval
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
-        self.check_loops({"getarrayitem_raw": 2, "float_add": 1,
-                          "int_add": 1, "float_ne": 1,
-                          "int_lt": 1, "guard_true": 2, "jump": 1})
-        assert result == f(5)
+                          "float_mul": 1, "int_add": 1,
+                          "int_lt": 1, "guard_true": 1, "jump": 1})
 
     def test_any(self):
-        space = self.space
-        float64_dtype = self.float64_dtype
-
-        def f(i):
-            ar = SingleDimArray(i, dtype=NonConstant(float64_dtype))
-            return ar.descr_add(space, ar).descr_any(space).boolval
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
+        result = self.run("""
+        a = [0,0,0,0,0,0,0,0,0,0,0]
+        a[8] = -12
+        b = a + a
+        any(b)
+        """)
+        assert result == 1
         self.check_loops({"getarrayitem_raw": 2, "float_add": 1,
-                          "int_add": 1, "float_ne": 1, "guard_false": 1,
-                          "int_lt": 1, "guard_true": 1, "jump": 1})
-        assert result == f(5)
+                          "float_ne": 1, "int_add": 1,
+                          "int_lt": 1, "guard_true": 1, "jump": 1,
+                          "guard_false": 1})
 
     def test_already_forced(self):
-        space = self.space
-
-        def f(i):
-            ar = SingleDimArray(i, dtype=self.float64_dtype)
-            v1 = interp_ufuncs.get(self.space).add.call(space, [ar, scalar_w(space, self.float64_dtype, space.wrap(4.5))])
-            assert isinstance(v1, BaseArray)
-            v2 = interp_ufuncs.get(self.space).multiply.call(space, [v1, scalar_w(space, self.float64_dtype, space.wrap(4.5))])
-            v1.force_if_needed()
-            assert isinstance(v2, BaseArray)
-            return v2.get_concrete().eval(3).val
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
+        result = self.run("""
+        a = |30|
+        b = a + 4.5
+        b -> 5 # forces
+        c = b * 8
+        c -> 5
+        """)
+        assert result == (5 + 4.5) * 8
         # This is the sum of the ops for both loops, however if you remove the
         # optimization then you end up with 2 float_adds, so we can still be
         # sure it was optimized correctly.
         self.check_loops({"getarrayitem_raw": 2, "float_mul": 1, "float_add": 1,
                            "setarrayitem_raw": 2, "int_add": 2,
                            "int_lt": 2, "guard_true": 2, "jump": 2})
-        assert result == f(5)
 
     def test_ufunc(self):
-        space = self.space
-        def f(i):
-            ar = SingleDimArray(i, dtype=self.float64_dtype)
-            v1 = interp_ufuncs.get(self.space).add.call(space, [ar, ar])
-            v2 = interp_ufuncs.get(self.space).negative.call(space, [v1])
-            return v2.get_concrete().eval(3).val
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
+        result = self.run("""
+        a = |30|
+        b = a + a
+        c = unegative(b)
+        c -> 3
+        """)
+        assert result == -6
         self.check_loops({"getarrayitem_raw": 2, "float_add": 1, "float_neg": 1,
                           "setarrayitem_raw": 1, "int_add": 1,
                           "int_lt": 1, "guard_true": 1, "jump": 1,
         })
-        assert result == f(5)
 
-    def test_appropriate_specialization(self):
-        space = self.space
-        def f(i):
-            ar = SingleDimArray(i, dtype=self.float64_dtype)
-
-            v1 = interp_ufuncs.get(self.space).add.call(space, [ar, ar])
-            v2 = interp_ufuncs.get(self.space).negative.call(space, [v1])
-            v2.get_concrete()
-
-            for i in xrange(5):
-                v1 = interp_ufuncs.get(self.space).multiply.call(space, [ar, ar])
-                v2 = interp_ufuncs.get(self.space).negative.call(space, [v1])
-                v2.get_concrete()
-
-        self.meta_interp(f, [5], listops=True, backendopt=True)
+    def test_specialization(self):
+        self.run("""
+        a = |30|
+        b = a + a
+        c = unegative(b)
+        c -> 3
+        d = a * a
+        unegative(d)
+        d -> 3
+        d = a * a
+        unegative(d)
+        d -> 3
+        d = a * a
+        unegative(d)
+        d -> 3
+        d = a * a
+        unegative(d)
+        d -> 3
+        """)
         # This is 3, not 2 because there is a bridge for the exit.
         self.check_loop_count(3)
 
+
+class TestNumpyOld(LLJitMixin):
+    def setup_class(cls):
+        from pypy.module.micronumpy.compile import FakeSpace
+        from pypy.module.micronumpy.interp_dtype import W_Float64Dtype
+        
+        cls.space = FakeSpace()
+        cls.float64_dtype = cls.space.fromcache(W_Float64Dtype)
+    
     def test_slice(self):
         def f(i):
             step = 3
@@ -332,17 +274,3 @@
         result = self.meta_interp(f, [5], listops=True, backendopt=True)
         assert result == f(5)
 
-class TestTranslation(object):
-    def test_compile(self):
-        x = numpy_compile('aa+f*f/a-', 10)
-        x = x.compute()
-        assert isinstance(x, SingleDimArray)
-        assert x.size == 10
-        assert x.eval(0).val == 0
-        assert x.eval(1).val == ((1 + 1) * 1.2) / 1.2 - 1
-
-    def test_translation(self):
-        # we import main to check if the target compiles
-        from pypy.translator.goal.targetnumpystandalone import main
-
-        interpret(main, [llstr('af+'), 100])
diff --git a/pypy/rpython/lltypesystem/rstr.py b/pypy/rpython/lltypesystem/rstr.py
--- a/pypy/rpython/lltypesystem/rstr.py
+++ b/pypy/rpython/lltypesystem/rstr.py
@@ -20,6 +20,7 @@
 from pypy.rpython.rmodel import Repr
 from pypy.rpython.lltypesystem import llmemory
 from pypy.tool.sourcetools import func_with_new_name
+from pypy.rpython.lltypesystem.lloperation import llop
 
 # ____________________________________________________________
 #
@@ -364,8 +365,10 @@
             while lpos < rpos and s.chars[lpos] == ch:
                 lpos += 1
         if right:
-            while lpos < rpos and s.chars[rpos] == ch:
+            while lpos < rpos + 1 and s.chars[rpos] == ch:
                 rpos -= 1
+        if rpos < lpos:
+            return s.empty()
         r_len = rpos - lpos + 1
         result = s.malloc(r_len)
         s.copy_contents(s, result, lpos, 0, r_len)
diff --git a/pypy/rpython/test/test_rstr.py b/pypy/rpython/test/test_rstr.py
--- a/pypy/rpython/test/test_rstr.py
+++ b/pypy/rpython/test/test_rstr.py
@@ -372,12 +372,20 @@
             return const('!ab!').lstrip(const('!'))
         def right():
             return const('!ab!').rstrip(const('!'))
+        def empty():
+            return const('    ').strip(' ')
+        def left2():
+            return const('a  ').strip(' ')
         res = self.interpret(both, [])
         assert self.ll_to_string(res) == const('ab')
         res = self.interpret(left, [])
         assert self.ll_to_string(res) == const('ab!')
         res = self.interpret(right, [])
         assert self.ll_to_string(res) == const('!ab')
+        res = self.interpret(empty, [])
+        assert self.ll_to_string(res) == const('')
+        res = self.interpret(left2, [])
+        assert self.ll_to_string(res) == const('a')
 
     def test_upper(self):
         const = self.const

From noreply at buildbot.pypy.org  Fri Oct 28 15:01:45 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 28 Oct 2011 15:01:45 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-minilang: merged
Message-ID: <20111028130145.E505F820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: numpy-minilang
Changeset: r48564:7a58d06641f3
Date: 2011-10-28 15:01 +0200
http://bitbucket.org/pypy/pypy/changeset/7a58d06641f3/

Log:	merged


From noreply at buildbot.pypy.org  Fri Oct 28 16:58:35 2011
From: noreply at buildbot.pypy.org (cfbolz)
Date: Fri, 28 Oct 2011 16:58:35 +0200 (CEST)
Subject: [pypy-commit] pypy int-tag-untag-as-operations: move an ll_assert
 around: cast_ptr_to_int *can* be used on an even pointer (to
Message-ID: <20111028145835.526AB820C2@wyvern.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: int-tag-untag-as-operations
Changeset: r48565:4c4642283aff
Date: 2011-10-28 16:58 +0200
http://bitbucket.org/pypy/pypy/changeset/4c4642283aff/

Log:	move an ll_assert around: cast_ptr_to_int *can* be used on an even
	pointer (to check whether it is tagged). however, int_untag should
	really only be used on odd numbers.

diff --git a/pypy/jit/metainterp/blackhole.py b/pypy/jit/metainterp/blackhole.py
--- a/pypy/jit/metainterp/blackhole.py
+++ b/pypy/jit/metainterp/blackhole.py
@@ -448,6 +448,7 @@
 
     @arguments("i", returns="i")
     def bhimpl_int_untag(a):
+        ll_assert((a & 1) == 1, "bhimpl_int_untag: not an odd int")
         return a >> 1
     @arguments("i", returns="i")
     def bhimpl_int_tag(a):
@@ -522,7 +523,6 @@
     @arguments("r", returns="i")
     def bhimpl_cast_ptr_to_int(a):
         i = lltype.cast_ptr_to_int(a)
-        ll_assert((i & 1) == 1, "bhimpl_cast_ptr_to_int: not an odd int")
         return i
     @arguments("i", returns="r")
     def bhimpl_cast_int_to_ptr(i):

From noreply at buildbot.pypy.org  Fri Oct 28 17:20:51 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 28 Oct 2011 17:20:51 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-multidim: merge default - tests still
	broken, working on it
Message-ID: <20111028152051.11E94820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: numpy-multidim
Changeset: r48566:64aadf44e926
Date: 2011-10-28 17:18 +0200
http://bitbucket.org/pypy/pypy/changeset/64aadf44e926/

Log:	merge default - tests still broken, working on it

diff --git a/lib-python/modified-2.7/heapq.py b/lib-python/modified-2.7/heapq.py
new file mode 100644
--- /dev/null
+++ b/lib-python/modified-2.7/heapq.py
@@ -0,0 +1,442 @@
+# -*- coding: latin-1 -*-
+
+"""Heap queue algorithm (a.k.a. priority queue).
+
+Heaps are arrays for which a[k] <= a[2*k+1] and a[k] <= a[2*k+2] for
+all k, counting elements from 0.  For the sake of comparison,
+non-existing elements are considered to be infinite.  The interesting
+property of a heap is that a[0] is always its smallest element.
+
+Usage:
+
+heap = []            # creates an empty heap
+heappush(heap, item) # pushes a new item on the heap
+item = heappop(heap) # pops the smallest item from the heap
+item = heap[0]       # smallest item on the heap without popping it
+heapify(x)           # transforms list into a heap, in-place, in linear time
+item = heapreplace(heap, item) # pops and returns smallest item, and adds
+                               # new item; the heap size is unchanged
+
+Our API differs from textbook heap algorithms as follows:
+
+- We use 0-based indexing.  This makes the relationship between the
+  index for a node and the indexes for its children slightly less
+  obvious, but is more suitable since Python uses 0-based indexing.
+
+- Our heappop() method returns the smallest item, not the largest.
+
+These two make it possible to view the heap as a regular Python list
+without surprises: heap[0] is the smallest item, and heap.sort()
+maintains the heap invariant!
+"""
+
+# Original code by Kevin O'Connor, augmented by Tim Peters and Raymond Hettinger
+
+__about__ = """Heap queues
+
+[explanation by Fran&#65533;ois Pinard]
+
+Heaps are arrays for which a[k] <= a[2*k+1] and a[k] <= a[2*k+2] for
+all k, counting elements from 0.  For the sake of comparison,
+non-existing elements are considered to be infinite.  The interesting
+property of a heap is that a[0] is always its smallest element.
+
+The strange invariant above is meant to be an efficient memory
+representation for a tournament.  The numbers below are `k', not a[k]:
+
+                                   0
+
+                  1                                 2
+
+          3               4                5               6
+
+      7       8       9       10      11      12      13      14
+
+    15 16   17 18   19 20   21 22   23 24   25 26   27 28   29 30
+
+
+In the tree above, each cell `k' is topping `2*k+1' and `2*k+2'.  In
+an usual binary tournament we see in sports, each cell is the winner
+over the two cells it tops, and we can trace the winner down the tree
+to see all opponents s/he had.  However, in many computer applications
+of such tournaments, we do not need to trace the history of a winner.
+To be more memory efficient, when a winner is promoted, we try to
+replace it by something else at a lower level, and the rule becomes
+that a cell and the two cells it tops contain three different items,
+but the top cell "wins" over the two topped cells.
+
+If this heap invariant is protected at all time, index 0 is clearly
+the overall winner.  The simplest algorithmic way to remove it and
+find the "next" winner is to move some loser (let's say cell 30 in the
+diagram above) into the 0 position, and then percolate this new 0 down
+the tree, exchanging values, until the invariant is re-established.
+This is clearly logarithmic on the total number of items in the tree.
+By iterating over all items, you get an O(n ln n) sort.
+
+A nice feature of this sort is that you can efficiently insert new
+items while the sort is going on, provided that the inserted items are
+not "better" than the last 0'th element you extracted.  This is
+especially useful in simulation contexts, where the tree holds all
+incoming events, and the "win" condition means the smallest scheduled
+time.  When an event schedule other events for execution, they are
+scheduled into the future, so they can easily go into the heap.  So, a
+heap is a good structure for implementing schedulers (this is what I
+used for my MIDI sequencer :-).
+
+Various structures for implementing schedulers have been extensively
+studied, and heaps are good for this, as they are reasonably speedy,
+the speed is almost constant, and the worst case is not much different
+than the average case.  However, there are other representations which
+are more efficient overall, yet the worst cases might be terrible.
+
+Heaps are also very useful in big disk sorts.  You most probably all
+know that a big sort implies producing "runs" (which are pre-sorted
+sequences, which size is usually related to the amount of CPU memory),
+followed by a merging passes for these runs, which merging is often
+very cleverly organised[1].  It is very important that the initial
+sort produces the longest runs possible.  Tournaments are a good way
+to that.  If, using all the memory available to hold a tournament, you
+replace and percolate items that happen to fit the current run, you'll
+produce runs which are twice the size of the memory for random input,
+and much better for input fuzzily ordered.
+
+Moreover, if you output the 0'th item on disk and get an input which
+may not fit in the current tournament (because the value "wins" over
+the last output value), it cannot fit in the heap, so the size of the
+heap decreases.  The freed memory could be cleverly reused immediately
+for progressively building a second heap, which grows at exactly the
+same rate the first heap is melting.  When the first heap completely
+vanishes, you switch heaps and start a new run.  Clever and quite
+effective!
+
+In a word, heaps are useful memory structures to know.  I use them in
+a few applications, and I think it is good to keep a `heap' module
+around. :-)
+
+--------------------
+[1] The disk balancing algorithms which are current, nowadays, are
+more annoying than clever, and this is a consequence of the seeking
+capabilities of the disks.  On devices which cannot seek, like big
+tape drives, the story was quite different, and one had to be very
+clever to ensure (far in advance) that each tape movement will be the
+most effective possible (that is, will best participate at
+"progressing" the merge).  Some tapes were even able to read
+backwards, and this was also used to avoid the rewinding time.
+Believe me, real good tape sorts were quite spectacular to watch!
+From all times, sorting has always been a Great Art! :-)
+"""
+
+__all__ = ['heappush', 'heappop', 'heapify', 'heapreplace', 'merge',
+           'nlargest', 'nsmallest', 'heappushpop']
+
+from itertools import islice, repeat, count, imap, izip, tee, chain
+from operator import itemgetter
+import bisect
+
+def heappush(heap, item):
+    """Push item onto heap, maintaining the heap invariant."""
+    heap.append(item)
+    _siftdown(heap, 0, len(heap)-1)
+
+def heappop(heap):
+    """Pop the smallest item off the heap, maintaining the heap invariant."""
+    lastelt = heap.pop()    # raises appropriate IndexError if heap is empty
+    if heap:
+        returnitem = heap[0]
+        heap[0] = lastelt
+        _siftup(heap, 0)
+    else:
+        returnitem = lastelt
+    return returnitem
+
+def heapreplace(heap, item):
+    """Pop and return the current smallest value, and add the new item.
+
+    This is more efficient than heappop() followed by heappush(), and can be
+    more appropriate when using a fixed-size heap.  Note that the value
+    returned may be larger than item!  That constrains reasonable uses of
+    this routine unless written as part of a conditional replacement:
+
+        if item > heap[0]:
+            item = heapreplace(heap, item)
+    """
+    returnitem = heap[0]    # raises appropriate IndexError if heap is empty
+    heap[0] = item
+    _siftup(heap, 0)
+    return returnitem
+
+def heappushpop(heap, item):
+    """Fast version of a heappush followed by a heappop."""
+    if heap and heap[0] < item:
+        item, heap[0] = heap[0], item
+        _siftup(heap, 0)
+    return item
+
+def heapify(x):
+    """Transform list into a heap, in-place, in O(len(heap)) time."""
+    n = len(x)
+    # Transform bottom-up.  The largest index there's any point to looking at
+    # is the largest with a child index in-range, so must have 2*i + 1 < n,
+    # or i < (n-1)/2.  If n is even = 2*j, this is (2*j-1)/2 = j-1/2 so
+    # j-1 is the largest, which is n//2 - 1.  If n is odd = 2*j+1, this is
+    # (2*j+1-1)/2 = j so j-1 is the largest, and that's again n//2-1.
+    for i in reversed(xrange(n//2)):
+        _siftup(x, i)
+
+def nlargest(n, iterable):
+    """Find the n largest elements in a dataset.
+
+    Equivalent to:  sorted(iterable, reverse=True)[:n]
+    """
+    if n < 0: # for consistency with the c impl
+        return []
+    it = iter(iterable)
+    result = list(islice(it, n))
+    if not result:
+        return result
+    heapify(result)
+    _heappushpop = heappushpop
+    for elem in it:
+        _heappushpop(result, elem)
+    result.sort(reverse=True)
+    return result
+
+def nsmallest(n, iterable):
+    """Find the n smallest elements in a dataset.
+
+    Equivalent to:  sorted(iterable)[:n]
+    """
+    if n < 0: # for consistency with the c impl
+        return []
+    if hasattr(iterable, '__len__') and n * 10 <= len(iterable):
+        # For smaller values of n, the bisect method is faster than a minheap.
+        # It is also memory efficient, consuming only n elements of space.
+        it = iter(iterable)
+        result = sorted(islice(it, 0, n))
+        if not result:
+            return result
+        insort = bisect.insort
+        pop = result.pop
+        los = result[-1]    # los --> Largest of the nsmallest
+        for elem in it:
+            if los <= elem:
+                continue
+            insort(result, elem)
+            pop()
+            los = result[-1]
+        return result
+    # An alternative approach manifests the whole iterable in memory but
+    # saves comparisons by heapifying all at once.  Also, saves time
+    # over bisect.insort() which has O(n) data movement time for every
+    # insertion.  Finding the n smallest of an m length iterable requires
+    #    O(m) + O(n log m) comparisons.
+    h = list(iterable)
+    heapify(h)
+    return map(heappop, repeat(h, min(n, len(h))))
+
+# 'heap' is a heap at all indices >= startpos, except possibly for pos.  pos
+# is the index of a leaf with a possibly out-of-order value.  Restore the
+# heap invariant.
+def _siftdown(heap, startpos, pos):
+    newitem = heap[pos]
+    # Follow the path to the root, moving parents down until finding a place
+    # newitem fits.
+    while pos > startpos:
+        parentpos = (pos - 1) >> 1
+        parent = heap[parentpos]
+        if newitem < parent:
+            heap[pos] = parent
+            pos = parentpos
+            continue
+        break
+    heap[pos] = newitem
+
+# The child indices of heap index pos are already heaps, and we want to make
+# a heap at index pos too.  We do this by bubbling the smaller child of
+# pos up (and so on with that child's children, etc) until hitting a leaf,
+# then using _siftdown to move the oddball originally at index pos into place.
+#
+# We *could* break out of the loop as soon as we find a pos where newitem <=
+# both its children, but turns out that's not a good idea, and despite that
+# many books write the algorithm that way.  During a heap pop, the last array
+# element is sifted in, and that tends to be large, so that comparing it
+# against values starting from the root usually doesn't pay (= usually doesn't
+# get us out of the loop early).  See Knuth, Volume 3, where this is
+# explained and quantified in an exercise.
+#
+# Cutting the # of comparisons is important, since these routines have no
+# way to extract "the priority" from an array element, so that intelligence
+# is likely to be hiding in custom __cmp__ methods, or in array elements
+# storing (priority, record) tuples.  Comparisons are thus potentially
+# expensive.
+#
+# On random arrays of length 1000, making this change cut the number of
+# comparisons made by heapify() a little, and those made by exhaustive
+# heappop() a lot, in accord with theory.  Here are typical results from 3
+# runs (3 just to demonstrate how small the variance is):
+#
+# Compares needed by heapify     Compares needed by 1000 heappops
+# --------------------------     --------------------------------
+# 1837 cut to 1663               14996 cut to 8680
+# 1855 cut to 1659               14966 cut to 8678
+# 1847 cut to 1660               15024 cut to 8703
+#
+# Building the heap by using heappush() 1000 times instead required
+# 2198, 2148, and 2219 compares:  heapify() is more efficient, when
+# you can use it.
+#
+# The total compares needed by list.sort() on the same lists were 8627,
+# 8627, and 8632 (this should be compared to the sum of heapify() and
+# heappop() compares):  list.sort() is (unsurprisingly!) more efficient
+# for sorting.
+
+def _siftup(heap, pos):
+    endpos = len(heap)
+    startpos = pos
+    newitem = heap[pos]
+    # Bubble up the smaller child until hitting a leaf.
+    childpos = 2*pos + 1    # leftmost child position
+    while childpos < endpos:
+        # Set childpos to index of smaller child.
+        rightpos = childpos + 1
+        if rightpos < endpos and not heap[childpos] < heap[rightpos]:
+            childpos = rightpos
+        # Move the smaller child up.
+        heap[pos] = heap[childpos]
+        pos = childpos
+        childpos = 2*pos + 1
+    # The leaf at pos is empty now.  Put newitem there, and bubble it up
+    # to its final resting place (by sifting its parents down).
+    heap[pos] = newitem
+    _siftdown(heap, startpos, pos)
+
+# If available, use C implementation
+try:
+    from _heapq import *
+except ImportError:
+    pass
+
+def merge(*iterables):
+    '''Merge multiple sorted inputs into a single sorted output.
+
+    Similar to sorted(itertools.chain(*iterables)) but returns a generator,
+    does not pull the data into memory all at once, and assumes that each of
+    the input streams is already sorted (smallest to largest).
+
+    >>> list(merge([1,3,5,7], [0,2,4,8], [5,10,15,20], [], [25]))
+    [0, 1, 2, 3, 4, 5, 5, 7, 8, 10, 15, 20, 25]
+
+    '''
+    _heappop, _heapreplace, _StopIteration = heappop, heapreplace, StopIteration
+
+    h = []
+    h_append = h.append
+    for itnum, it in enumerate(map(iter, iterables)):
+        try:
+            next = it.next
+            h_append([next(), itnum, next])
+        except _StopIteration:
+            pass
+    heapify(h)
+
+    while 1:
+        try:
+            while 1:
+                v, itnum, next = s = h[0]   # raises IndexError when h is empty
+                yield v
+                s[0] = next()               # raises StopIteration when exhausted
+                _heapreplace(h, s)          # restore heap condition
+        except _StopIteration:
+            _heappop(h)                     # remove empty iterator
+        except IndexError:
+            return
+
+# Extend the implementations of nsmallest and nlargest to use a key= argument
+_nsmallest = nsmallest
+def nsmallest(n, iterable, key=None):
+    """Find the n smallest elements in a dataset.
+
+    Equivalent to:  sorted(iterable, key=key)[:n]
+    """
+    # Short-cut for n==1 is to use min() when len(iterable)>0
+    if n == 1:
+        it = iter(iterable)
+        head = list(islice(it, 1))
+        if not head:
+            return []
+        if key is None:
+            return [min(chain(head, it))]
+        return [min(chain(head, it), key=key)]
+
+    # When n>=size, it's faster to use sort()
+    try:
+        size = len(iterable)
+    except (TypeError, AttributeError):
+        pass
+    else:
+        if n >= size:
+            return sorted(iterable, key=key)[:n]
+
+    # When key is none, use simpler decoration
+    if key is None:
+        it = izip(iterable, count())                        # decorate
+        result = _nsmallest(n, it)
+        return map(itemgetter(0), result)                   # undecorate
+
+    # General case, slowest method
+    in1, in2 = tee(iterable)
+    it = izip(imap(key, in1), count(), in2)                 # decorate
+    result = _nsmallest(n, it)
+    return map(itemgetter(2), result)                       # undecorate
+
+_nlargest = nlargest
+def nlargest(n, iterable, key=None):
+    """Find the n largest elements in a dataset.
+
+    Equivalent to:  sorted(iterable, key=key, reverse=True)[:n]
+    """
+
+    # Short-cut for n==1 is to use max() when len(iterable)>0
+    if n == 1:
+        it = iter(iterable)
+        head = list(islice(it, 1))
+        if not head:
+            return []
+        if key is None:
+            return [max(chain(head, it))]
+        return [max(chain(head, it), key=key)]
+
+    # When n>=size, it's faster to use sort()
+    try:
+        size = len(iterable)
+    except (TypeError, AttributeError):
+        pass
+    else:
+        if n >= size:
+            return sorted(iterable, key=key, reverse=True)[:n]
+
+    # When key is none, use simpler decoration
+    if key is None:
+        it = izip(iterable, count(0,-1))                    # decorate
+        result = _nlargest(n, it)
+        return map(itemgetter(0), result)                   # undecorate
+
+    # General case, slowest method
+    in1, in2 = tee(iterable)
+    it = izip(imap(key, in1), count(0,-1), in2)             # decorate
+    result = _nlargest(n, it)
+    return map(itemgetter(2), result)                       # undecorate
+
+if __name__ == "__main__":
+    # Simple sanity test
+    heap = []
+    data = [1, 3, 5, 7, 9, 2, 4, 6, 8, 0]
+    for item in data:
+        heappush(heap, item)
+    sort = []
+    while heap:
+        sort.append(heappop(heap))
+    print sort
+
+    import doctest
+    doctest.testmod()
diff --git a/lib-python/modified-2.7/test/test_heapq.py b/lib-python/modified-2.7/test/test_heapq.py
--- a/lib-python/modified-2.7/test/test_heapq.py
+++ b/lib-python/modified-2.7/test/test_heapq.py
@@ -186,6 +186,11 @@
         self.assertFalse(sys.modules['heapq'] is self.module)
         self.assertTrue(hasattr(self.module.heapify, 'func_code'))
 
+    def test_islice_protection(self):
+        m = self.module
+        self.assertFalse(m.nsmallest(-1, [1]))
+        self.assertFalse(m.nlargest(-1, [1]))
+
 
 class TestHeapC(TestHeap):
     module = c_heapq
diff --git a/pypy/jit/backend/llsupport/descr.py b/pypy/jit/backend/llsupport/descr.py
--- a/pypy/jit/backend/llsupport/descr.py
+++ b/pypy/jit/backend/llsupport/descr.py
@@ -281,6 +281,9 @@
     def is_float_field(self):
         return self.fielddescr.is_float_field()
 
+    def sort_key(self):
+        return self.fielddescr.sort_key()
+
     def repr_of_descr(self):
         return '<InteriorFieldDescr %s>' % self.fielddescr.repr_of_descr()
 
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1596,11 +1596,26 @@
     genop_getarrayitem_gc_pure = genop_getarrayitem_gc
     genop_getarrayitem_raw = genop_getarrayitem_gc
 
+    def _get_interiorfield_addr(self, temp_loc, index_loc, itemsize_loc,
+                                base_loc, ofs_loc):
+        assert isinstance(itemsize_loc, ImmedLoc)
+        if isinstance(index_loc, ImmedLoc):
+            temp_loc = imm(index_loc.value * itemsize_loc.value)
+        else:
+            # XXX should not use IMUL in most cases
+            assert isinstance(temp_loc, RegLoc)
+            assert isinstance(index_loc, RegLoc)
+            self.mc.IMUL_rri(temp_loc.value, index_loc.value,
+                             itemsize_loc.value)
+        assert isinstance(ofs_loc, ImmedLoc)
+        return AddressLoc(base_loc, temp_loc, 0, ofs_loc.value)
+
     def genop_getinteriorfield_gc(self, op, arglocs, resloc):
-        base_loc, ofs_loc, itemsize_loc, fieldsize_loc, index_loc, sign_loc = arglocs
-        # XXX should not use IMUL in most cases
-        self.mc.IMUL(index_loc, itemsize_loc)
-        src_addr = AddressLoc(base_loc, index_loc, 0, ofs_loc.value)
+        (base_loc, ofs_loc, itemsize_loc, fieldsize_loc,
+            index_loc, sign_loc) = arglocs
+        src_addr = self._get_interiorfield_addr(resloc, index_loc,
+                                                itemsize_loc, base_loc,
+                                                ofs_loc)
         self.load_from_mem(resloc, src_addr, fieldsize_loc, sign_loc)
 
 
@@ -1611,13 +1626,11 @@
         self.save_into_mem(dest_addr, value_loc, size_loc)
 
     def genop_discard_setinteriorfield_gc(self, op, arglocs):
-        base_loc, ofs_loc, itemsize_loc, fieldsize_loc, index_loc, value_loc = arglocs
-        # XXX should not use IMUL in most cases
-        if isinstance(index_loc, ImmedLoc):
-            index_loc = imm(index_loc.value * itemsize_loc.value)
-        else:
-            self.mc.IMUL(index_loc, itemsize_loc)
-        dest_addr = AddressLoc(base_loc, index_loc, 0, ofs_loc.value)
+        (base_loc, ofs_loc, itemsize_loc, fieldsize_loc,
+            index_loc, temp_loc, value_loc) = arglocs
+        dest_addr = self._get_interiorfield_addr(temp_loc, index_loc,
+                                                 itemsize_loc, base_loc,
+                                                 ofs_loc)
         self.save_into_mem(dest_addr, value_loc, fieldsize_loc)
 
     def genop_discard_setarrayitem_gc(self, op, arglocs):
diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py
--- a/pypy/jit/backend/x86/regalloc.py
+++ b/pypy/jit/backend/x86/regalloc.py
@@ -1046,16 +1046,26 @@
             need_lower_byte = True
         else:
             need_lower_byte = False
-        base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
-        tempvar = TempBox()
-        index_loc = self.rm.force_result_in_reg(tempvar, op.getarg(1), args)
-        # we're free to modify index now
-        value_loc = self.make_sure_var_in_reg(op.getarg(2), args + [tempvar],
+        box_base, box_index, box_value = args
+        base_loc = self.rm.make_sure_var_in_reg(box_base, args)
+        index_loc = self.rm.make_sure_var_in_reg(box_index, args)
+        value_loc = self.make_sure_var_in_reg(box_value, args,
                                               need_lower_byte=need_lower_byte)
-        self.rm.possibly_free_var(tempvar)
-        self.possibly_free_vars(args)
+        # If 'index_loc' is not an immediate, then we need a 'temp_loc' that
+        # is a register whose value will be destroyed.  It's fine to destroy
+        # the same register as 'index_loc', but not the other ones.
+        self.rm.possibly_free_var(box_index)
+        if not isinstance(index_loc, ImmedLoc):
+            tempvar = TempBox()
+            temp_loc = self.rm.force_allocate_reg(tempvar, [box_base,
+                                                            box_value])
+            self.rm.possibly_free_var(tempvar)
+        else:
+            temp_loc = None
+        self.rm.possibly_free_var(box_base)
+        self.possibly_free_var(box_value)
         self.PerformDiscard(op, [base_loc, ofs, itemsize, fieldsize,
-                                 index_loc, value_loc])
+                                 index_loc, temp_loc, value_loc])
 
     def consider_strsetitem(self, op):
         args = op.getarglist()
@@ -1126,13 +1136,14 @@
         else:
             sign_loc = imm0
         args = op.getarglist()
-        tmpvar = TempBox()
         base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
-        index_loc = self.rm.force_result_in_reg(tmpvar, op.getarg(1),
-                                                args)
-        self.rm.possibly_free_vars_for_op(op)
-        self.rm.possibly_free_var(tmpvar)
-        result_loc = self.force_allocate_reg(op.result)
+        index_loc = self.rm.make_sure_var_in_reg(op.getarg(1), args)
+        # 'base' and 'index' are put in two registers (or one if 'index'
+        # is an immediate).  'result' can be in the same register as
+        # 'index' but must be in a different register than 'base'.
+        self.rm.possibly_free_var(op.getarg(1))
+        result_loc = self.force_allocate_reg(op.result, [op.getarg(0)])
+        self.rm.possibly_free_var(op.getarg(0))
         self.Perform(op, [base_loc, ofs, itemsize, fieldsize,
                           index_loc, sign_loc], result_loc)
 
diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -844,6 +844,10 @@
         if self._is_gc(op.args[0]):
             return op
 
+    def rewrite_op_cast_opaque_ptr(self, op):
+        # None causes the result of this op to get aliased to op.args[0]
+        return [SpaceOperation('mark_opaque_ptr', op.args, None), None]
+
     def rewrite_op_force_cast(self, op):
         v_arg = op.args[0]
         v_result = op.result
diff --git a/pypy/jit/codewriter/test/test_jtransform.py b/pypy/jit/codewriter/test/test_jtransform.py
--- a/pypy/jit/codewriter/test/test_jtransform.py
+++ b/pypy/jit/codewriter/test/test_jtransform.py
@@ -1128,3 +1128,16 @@
                         varoftype(lltype.Signed))
     tr = Transformer(None, None)
     raises(NotImplementedError, tr.rewrite_operation, op)
+
+def test_cast_opaque_ptr():
+    S = lltype.GcStruct("S", ("x", lltype.Signed))
+    v1 = varoftype(lltype.Ptr(S))
+    v2 = varoftype(lltype.Ptr(rclass.OBJECT))
+
+    op = SpaceOperation('cast_opaque_ptr', [v1], v2)
+    tr = Transformer()
+    [op1, op2] = tr.rewrite_operation(op)
+    assert op1.opname == 'mark_opaque_ptr'
+    assert op1.args == [v1]
+    assert op1.result is None
+    assert op2 is None
\ No newline at end of file
diff --git a/pypy/jit/metainterp/blackhole.py b/pypy/jit/metainterp/blackhole.py
--- a/pypy/jit/metainterp/blackhole.py
+++ b/pypy/jit/metainterp/blackhole.py
@@ -505,9 +505,6 @@
     @arguments("r", "r", returns="i")
     def bhimpl_instance_ptr_ne(a, b):
         return a != b
-    @arguments("r", returns="r")
-    def bhimpl_cast_opaque_ptr(a):
-        return a
     @arguments("r", returns="i")
     def bhimpl_cast_ptr_to_int(a):
         i = lltype.cast_ptr_to_int(a)
@@ -518,6 +515,10 @@
         ll_assert((i & 1) == 1, "bhimpl_cast_int_to_ptr: not an odd int")
         return lltype.cast_int_to_ptr(llmemory.GCREF, i)
 
+    @arguments("r")
+    def bhimpl_mark_opaque_ptr(a):
+        pass
+
     @arguments("i", returns="i")
     def bhimpl_int_copy(a):
         return a
diff --git a/pypy/jit/metainterp/heapcache.py b/pypy/jit/metainterp/heapcache.py
--- a/pypy/jit/metainterp/heapcache.py
+++ b/pypy/jit/metainterp/heapcache.py
@@ -34,7 +34,6 @@
         self.clear_caches(opnum, descr, argboxes)
 
     def mark_escaped(self, opnum, argboxes):
-        idx = 0
         if opnum == rop.SETFIELD_GC:
             assert len(argboxes) == 2
             box, valuebox = argboxes
@@ -42,8 +41,20 @@
                 self.dependencies.setdefault(box, []).append(valuebox)
             else:
                 self._escape(valuebox)
-        # GETFIELD_GC doesn't escape it's argument
-        elif opnum != rop.GETFIELD_GC:
+        elif opnum == rop.SETARRAYITEM_GC:
+            assert len(argboxes) == 3
+            box, indexbox, valuebox = argboxes
+            if self.is_unescaped(box) and self.is_unescaped(valuebox):
+                self.dependencies.setdefault(box, []).append(valuebox)
+            else:
+                self._escape(valuebox)
+        # GETFIELD_GC, MARK_OPAQUE_PTR, PTR_EQ, and PTR_NE don't escape their
+        # arguments
+        elif (opnum != rop.GETFIELD_GC and
+              opnum != rop.MARK_OPAQUE_PTR and
+              opnum != rop.PTR_EQ and
+              opnum != rop.PTR_NE):
+            idx = 0
             for box in argboxes:
                 # setarrayitem_gc don't escape its first argument
                 if not (idx == 0 and opnum in [rop.SETARRAYITEM_GC]):
@@ -60,13 +71,13 @@
                 self._escape(dep)
 
     def clear_caches(self, opnum, descr, argboxes):
-        if opnum == rop.SETFIELD_GC:
-            return
-        if opnum == rop.SETARRAYITEM_GC:
-            return
-        if opnum == rop.SETFIELD_RAW:
-            return
-        if opnum == rop.SETARRAYITEM_RAW:
+        if (opnum == rop.SETFIELD_GC or
+            opnum == rop.SETARRAYITEM_GC or
+            opnum == rop.SETFIELD_RAW or
+            opnum == rop.SETARRAYITEM_RAW or
+            opnum == rop.SETINTERIORFIELD_GC or
+            opnum == rop.COPYSTRCONTENT or
+            opnum == rop.COPYUNICODECONTENT):
             return
         if rop._OVF_FIRST <= opnum <= rop._OVF_LAST:
             return
@@ -75,9 +86,9 @@
         if opnum == rop.CALL or opnum == rop.CALL_LOOPINVARIANT:
             effectinfo = descr.get_extra_info()
             ef = effectinfo.extraeffect
-            if ef == effectinfo.EF_LOOPINVARIANT or \
-               ef == effectinfo.EF_ELIDABLE_CANNOT_RAISE or \
-               ef == effectinfo.EF_ELIDABLE_CAN_RAISE:
+            if (ef == effectinfo.EF_LOOPINVARIANT or
+                ef == effectinfo.EF_ELIDABLE_CANNOT_RAISE or
+                ef == effectinfo.EF_ELIDABLE_CAN_RAISE):
                 return
             # A special case for ll_arraycopy, because it is so common, and its
             # effects are so well defined.
diff --git a/pypy/jit/metainterp/history.py b/pypy/jit/metainterp/history.py
--- a/pypy/jit/metainterp/history.py
+++ b/pypy/jit/metainterp/history.py
@@ -929,6 +929,9 @@
     def view(self, **kwds):
         pass
 
+    def clear(self):
+        pass
+
 class Stats(object):
     """For tests."""
 
@@ -943,6 +946,15 @@
         self.aborted_keys = []
         self.invalidated_token_numbers = set()
 
+    def clear(self):
+        del self.loops[:]
+        del self.locations[:]
+        del self.aborted_keys[:]
+        self.invalidated_token_numbers.clear()
+        self.compiled_count = 0
+        self.enter_count = 0
+        self.aborted_count = 0
+
     def set_history(self, history):
         self.operations = history.operations
 
diff --git a/pypy/jit/metainterp/optimizeopt/optimizer.py b/pypy/jit/metainterp/optimizeopt/optimizer.py
--- a/pypy/jit/metainterp/optimizeopt/optimizer.py
+++ b/pypy/jit/metainterp/optimizeopt/optimizer.py
@@ -209,13 +209,19 @@
     def setfield(self, ofs, value):
         raise NotImplementedError
 
+    def getlength(self):
+        raise NotImplementedError
+
     def getitem(self, index):
         raise NotImplementedError
 
-    def getlength(self):
+    def setitem(self, index, value):
         raise NotImplementedError
 
-    def setitem(self, index, value):
+    def getinteriorfield(self, index, ofs, default):
+        raise NotImplementedError
+
+    def setinteriorfield(self, index, ofs, value):
         raise NotImplementedError
 
 
@@ -283,11 +289,11 @@
             return self.optimizer.optpure.has_pure_result(opnum, args, descr)
         return False
 
-    def get_pure_result(self, key):    
+    def get_pure_result(self, key):
         if self.optimizer.optpure:
             return self.optimizer.optpure.get_pure_result(key)
         return None
-    
+
     def setup(self):
         pass
 
@@ -524,7 +530,7 @@
 
     def replace_op(self, old_op, new_op):
         # XXX: Do we want to cache indexes to prevent search?
-        i = len(self._newoperations) 
+        i = len(self._newoperations)
         while i > 0:
             i -= 1
             if self._newoperations[i] is old_op:
diff --git a/pypy/jit/metainterp/optimizeopt/rewrite.py b/pypy/jit/metainterp/optimizeopt/rewrite.py
--- a/pypy/jit/metainterp/optimizeopt/rewrite.py
+++ b/pypy/jit/metainterp/optimizeopt/rewrite.py
@@ -465,10 +465,9 @@
                                         args = [op.getarg(0), ConstInt(highest_bit(val))])
         self.emit_operation(op)
 
-    def optimize_CAST_OPAQUE_PTR(self, op):
+    def optimize_MARK_OPAQUE_PTR(self, op):
         value = self.getvalue(op.getarg(0))
         self.optimizer.opaque_pointers[value] = True
-        self.make_equal_to(op.result, value)
 
     def optimize_CAST_PTR_TO_INT(self, op):
         self.pure(rop.CAST_INT_TO_PTR, [op.result], op.getarg(0))
diff --git a/pypy/jit/metainterp/optimizeopt/simplify.py b/pypy/jit/metainterp/optimizeopt/simplify.py
--- a/pypy/jit/metainterp/optimizeopt/simplify.py
+++ b/pypy/jit/metainterp/optimizeopt/simplify.py
@@ -25,7 +25,8 @@
         #     but it's a bit hard to implement robustly if heap.py is also run
         pass
 
-    optimize_CAST_OPAQUE_PTR = optimize_VIRTUAL_REF
+    def optimize_MARK_OPAQUE_PTR(self, op):
+        pass
 
 
 dispatch_opt = make_dispatcher_method(OptSimplify, 'optimize_',
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -935,7 +935,6 @@
         """
         self.optimize_loop(ops, expected)
 
-
     def test_virtual_constant_isnonnull(self):
         ops = """
         [i0]
@@ -951,6 +950,32 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_virtual_array_of_struct(self):
+        ops = """
+        [f0, f1, f2, f3]
+        p0 = new_array(2, descr=complexarraydescr)
+        setinteriorfield_gc(p0, 0, f0, descr=complexrealdescr)
+        setinteriorfield_gc(p0, 0, f1, descr=compleximagdescr)
+        setinteriorfield_gc(p0, 1, f2, descr=complexrealdescr)
+        setinteriorfield_gc(p0, 1, f3, descr=compleximagdescr)
+        f4 = getinteriorfield_gc(p0, 0, descr=complexrealdescr)
+        f5 = getinteriorfield_gc(p0, 1, descr=complexrealdescr)
+        f6 = float_mul(f4, f5)
+        f7 = getinteriorfield_gc(p0, 0, descr=compleximagdescr)
+        f8 = getinteriorfield_gc(p0, 1, descr=compleximagdescr)
+        f9 = float_mul(f7, f8)
+        f10 = float_add(f6, f9)
+        finish(f10)
+        """
+        expected = """
+        [f0, f1, f2, f3]
+        f4 = float_mul(f0, f2)
+        f5 = float_mul(f1, f3)
+        f6 = float_add(f4, f5)
+        finish(f6)
+        """
+        self.optimize_loop(ops, expected)
+
     def test_nonvirtual_1(self):
         ops = """
         [i]
@@ -4181,10 +4206,12 @@
         class FakeCallInfoCollection:
             def callinfo_for_oopspec(self, oopspecindex):
                 calldescrtype = type(LLtypeMixin.strequaldescr)
+                effectinfotype = type(LLtypeMixin.strequaldescr.get_extra_info())
                 for value in LLtypeMixin.__dict__.values():
                     if isinstance(value, calldescrtype):
                         extra = value.get_extra_info()
-                        if extra and extra.oopspecindex == oopspecindex:
+                        if (extra and isinstance(extra, effectinfotype) and
+                            extra.oopspecindex == oopspecindex):
                             # returns 0 for 'func' in this test
                             return value, 0
                 raise AssertionError("not found: oopspecindex=%d" %
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -5800,10 +5800,12 @@
         class FakeCallInfoCollection:
             def callinfo_for_oopspec(self, oopspecindex):
                 calldescrtype = type(LLtypeMixin.strequaldescr)
+                effectinfotype = type(LLtypeMixin.strequaldescr.get_extra_info())
                 for value in LLtypeMixin.__dict__.values():
                     if isinstance(value, calldescrtype):
                         extra = value.get_extra_info()
-                        if extra and extra.oopspecindex == oopspecindex:
+                        if (extra and isinstance(extra, effectinfotype) and
+                            extra.oopspecindex == oopspecindex):
                             # returns 0 for 'func' in this test
                             return value, 0
                 raise AssertionError("not found: oopspecindex=%d" %
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_util.py b/pypy/jit/metainterp/optimizeopt/test/test_util.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_util.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_util.py
@@ -185,6 +185,18 @@
              EffectInfo([], [arraydescr], [], [arraydescr],
                         oopspecindex=EffectInfo.OS_ARRAYCOPY))
 
+
+    # array of structs (complex data)
+    complexarray = lltype.GcArray(
+        lltype.Struct("complex",
+            ("real", lltype.Float),
+            ("imag", lltype.Float),
+        )
+    )
+    complexarraydescr = cpu.arraydescrof(complexarray)
+    complexrealdescr = cpu.interiorfielddescrof(complexarray, "real")
+    compleximagdescr = cpu.interiorfielddescrof(complexarray, "imag")
+
     for _name, _os in [
         ('strconcatdescr',               'OS_STR_CONCAT'),
         ('strslicedescr',                'OS_STR_SLICE'),
@@ -240,7 +252,7 @@
 ##    def get_class_of_box(self, box):
 ##        root = box.getref(ootype.ROOT)
 ##        return ootype.classof(root)
-    
+
 ##    cpu = runner.OOtypeCPU(None)
 ##    NODE = ootype.Instance('NODE', ootype.ROOT, {})
 ##    NODE._add_fields({'value': ootype.Signed,
diff --git a/pypy/jit/metainterp/optimizeopt/virtualize.py b/pypy/jit/metainterp/optimizeopt/virtualize.py
--- a/pypy/jit/metainterp/optimizeopt/virtualize.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualize.py
@@ -271,6 +271,69 @@
     def _make_virtual(self, modifier):
         return modifier.make_varray(self.arraydescr)
 
+class VArrayStructValue(AbstractVirtualValue):
+    def __init__(self, arraydescr, size, keybox, source_op=None):
+        AbstractVirtualValue.__init__(self, keybox, source_op)
+        self.arraydescr = arraydescr
+        self._items = [{} for _ in xrange(size)]
+
+    def getlength(self):
+        return len(self._items)
+
+    def getinteriorfield(self, index, ofs, default):
+        return self._items[index].get(ofs, default)
+
+    def setinteriorfield(self, index, ofs, itemvalue):
+        assert isinstance(itemvalue, optimizer.OptValue)
+        self._items[index][ofs] = itemvalue
+
+    def _really_force(self, optforce):
+        assert self.source_op is not None
+        if not we_are_translated():
+            self.source_op.name = 'FORCE ' + self.source_op.name
+        optforce.emit_operation(self.source_op)
+        self.box = box = self.source_op.result
+        for index in range(len(self._items)):
+            for descr, value in self._items[index].iteritems():
+                subbox = value.force_box(optforce)
+                op = ResOperation(rop.SETINTERIORFIELD_GC,
+                    [box, ConstInt(index), subbox], None, descr=descr
+                )
+                optforce.emit_operation(op)
+
+    def _get_list_of_descrs(self):
+        descrs = []
+        for item in self._items:
+            item_descrs = item.keys()
+            sort_descrs(item_descrs)
+            descrs.append(item_descrs)
+        return descrs
+
+    def get_args_for_fail(self, modifier):
+        if self.box is None and not modifier.already_seen_virtual(self.keybox):
+            itemdescrs = self._get_list_of_descrs()
+            itemboxes = []
+            for i in range(len(self._items)):
+                for descr in itemdescrs[i]:
+                    itemboxes.append(self._items[i][descr].get_key_box())
+            modifier.register_virtual_fields(self.keybox, itemboxes)
+            for i in range(len(self._items)):
+                for descr in itemdescrs[i]:
+                    self._items[i][descr].get_args_for_fail(modifier)
+
+    def force_at_end_of_preamble(self, already_forced, optforce):
+        if self in already_forced:
+            return self
+        already_forced[self] = self
+        for index in range(len(self._items)):
+            for descr in self._items[index].keys():
+                self._items[index][descr] = self._items[index][descr].force_at_end_of_preamble(already_forced, optforce)
+        return self
+
+    def _make_virtual(self, modifier):
+        return modifier.make_varraystruct(self.arraydescr, self._get_list_of_descrs())
+
+
 class OptVirtualize(optimizer.Optimization):
     "Virtualize objects until they escape."
 
@@ -283,8 +346,11 @@
         return vvalue
 
     def make_varray(self, arraydescr, size, box, source_op=None):
-        constvalue = self.new_const_item(arraydescr)
-        vvalue = VArrayValue(arraydescr, constvalue, size, box, source_op)
+        if arraydescr.is_array_of_structs():
+            vvalue = VArrayStructValue(arraydescr, size, box, source_op)
+        else:
+            constvalue = self.new_const_item(arraydescr)
+            vvalue = VArrayValue(arraydescr, constvalue, size, box, source_op)
         self.make_equal_to(box, vvalue)
         return vvalue
 
@@ -386,8 +452,7 @@
 
     def optimize_NEW_ARRAY(self, op):
         sizebox = self.get_constant_box(op.getarg(0))
-        # For now we can't make arrays of structs virtual.
-        if sizebox is not None and not op.getdescr().is_array_of_structs():
+        if sizebox is not None:
             # if the original 'op' did not have a ConstInt as argument,
             # build a new one with the ConstInt argument
             if not isinstance(op.getarg(0), ConstInt):
@@ -432,6 +497,34 @@
         value.ensure_nonnull()
         self.emit_operation(op)
 
+    def optimize_GETINTERIORFIELD_GC(self, op):
+        value = self.getvalue(op.getarg(0))
+        if value.is_virtual():
+            indexbox = self.get_constant_box(op.getarg(1))
+            if indexbox is not None:
+                descr = op.getdescr()
+                fieldvalue = value.getinteriorfield(
+                    indexbox.getint(), descr, None
+                )
+                if fieldvalue is None:
+                    fieldvalue = self.new_const(descr)
+                self.make_equal_to(op.result, fieldvalue)
+                return
+        value.ensure_nonnull()
+        self.emit_operation(op)
+
+    def optimize_SETINTERIORFIELD_GC(self, op):
+        value = self.getvalue(op.getarg(0))
+        if value.is_virtual():
+            indexbox = self.get_constant_box(op.getarg(1))
+            if indexbox is not None:
+                value.setinteriorfield(
+                    indexbox.getint(), op.getdescr(), self.getvalue(op.getarg(2))
+                )
+                return
+        value.ensure_nonnull()
+        self.emit_operation(op)
+
 
 dispatch_opt = make_dispatcher_method(OptVirtualize, 'optimize_',
         default=OptVirtualize.emit_operation)
diff --git a/pypy/jit/metainterp/optimizeopt/virtualstate.py b/pypy/jit/metainterp/optimizeopt/virtualstate.py
--- a/pypy/jit/metainterp/optimizeopt/virtualstate.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualstate.py
@@ -16,7 +16,7 @@
 
 class AbstractVirtualStateInfo(resume.AbstractVirtualInfo):
     position = -1
-    
+
     def generalization_of(self, other, renum, bad):
         raise NotImplementedError
 
@@ -54,7 +54,7 @@
                 s.debug_print(indent + "    ", seen, bad)
         else:
             debug_print(indent + "    ...")
-                
+
 
     def debug_header(self, indent):
         raise NotImplementedError
@@ -77,13 +77,15 @@
             bad[self] = True
             bad[other] = True
             return False
+
+        assert isinstance(other, AbstractVirtualStructStateInfo)
         assert len(self.fielddescrs) == len(self.fieldstate)
         assert len(other.fielddescrs) == len(other.fieldstate)
         if len(self.fielddescrs) != len(other.fielddescrs):
             bad[self] = True
             bad[other] = True
             return False
-        
+
         for i in range(len(self.fielddescrs)):
             if other.fielddescrs[i] is not self.fielddescrs[i]:
                 bad[self] = True
@@ -112,8 +114,8 @@
     def _enum(self, virtual_state):
         for s in self.fieldstate:
             s.enum(virtual_state)
-        
-        
+
+
 class VirtualStateInfo(AbstractVirtualStructStateInfo):
     def __init__(self, known_class, fielddescrs):
         AbstractVirtualStructStateInfo.__init__(self, fielddescrs)
@@ -128,13 +130,13 @@
 
     def debug_header(self, indent):
         debug_print(indent + 'VirtualStateInfo(%d):' % self.position)
-        
+
 class VStructStateInfo(AbstractVirtualStructStateInfo):
     def __init__(self, typedescr, fielddescrs):
         AbstractVirtualStructStateInfo.__init__(self, fielddescrs)
         self.typedescr = typedescr
 
-    def _generalization_of(self, other):        
+    def _generalization_of(self, other):
         if not isinstance(other, VStructStateInfo):
             return False
         if self.typedescr is not other.typedescr:
@@ -143,7 +145,7 @@
 
     def debug_header(self, indent):
         debug_print(indent + 'VStructStateInfo(%d):' % self.position)
-        
+
 class VArrayStateInfo(AbstractVirtualStateInfo):
     def __init__(self, arraydescr):
         self.arraydescr = arraydescr
@@ -157,11 +159,7 @@
             bad[other] = True
             return False
         renum[self.position] = other.position
-        if not isinstance(other, VArrayStateInfo):
-            bad[self] = True
-            bad[other] = True
-            return False
-        if self.arraydescr is not other.arraydescr:
+        if not self._generalization_of(other):
             bad[self] = True
             bad[other] = True
             return False
@@ -177,6 +175,10 @@
                 return False
         return True
 
+    def _generalization_of(self, other):
+        return (isinstance(other, VArrayStateInfo) and
+            self.arraydescr is other.arraydescr)
+
     def enum_forced_boxes(self, boxes, value, optimizer):
         assert isinstance(value, virtualize.VArrayValue)
         assert value.is_virtual()
@@ -192,8 +194,75 @@
 
     def debug_header(self, indent):
         debug_print(indent + 'VArrayStateInfo(%d):' % self.position)
-            
-        
+
+class VArrayStructStateInfo(AbstractVirtualStateInfo):
+    def __init__(self, arraydescr, fielddescrs):
+        self.arraydescr = arraydescr
+        self.fielddescrs = fielddescrs
+
+    def generalization_of(self, other, renum, bad):
+        assert self.position != -1
+        if self.position in renum:
+            if renum[self.position] == other.position:
+                return True
+            bad[self] = True
+            bad[other] = True
+            return False
+        renum[self.position] = other.position
+        if not self._generalization_of(other):
+            bad[self] = True
+            bad[other] = True
+            return False
+
+        assert isinstance(other, VArrayStructStateInfo)
+        if len(self.fielddescrs) != len(other.fielddescrs):
+            bad[self] = True
+            bad[other] = True
+            return False
+
+        p = 0
+        for i in range(len(self.fielddescrs)):
+            if len(self.fielddescrs[i]) != len(other.fielddescrs[i]):
+                bad[self] = True
+                bad[other] = True
+                return False
+            for j in range(len(self.fielddescrs[i])):
+                if self.fielddescrs[i][j] is not other.fielddescrs[i][j]:
+                    bad[self] = True
+                    bad[other] = True
+                    return False
+                if not self.fieldstate[p].generalization_of(other.fieldstate[p],
+                                                            renum, bad):
+                    bad[self] = True
+                    bad[other] = True
+                    return False
+                p += 1
+        return True
+
+    def _generalization_of(self, other):
+        return (isinstance(other, VArrayStructStateInfo) and
+            self.arraydescr is other.arraydescr)
+
+    def _enum(self, virtual_state):
+        for s in self.fieldstate:
+            s.enum(virtual_state)
+
+    def enum_forced_boxes(self, boxes, value, optimizer):
+        assert isinstance(value, virtualize.VArrayStructValue)
+        assert value.is_virtual()
+        p = 0
+        for i in range(len(self.fielddescrs)):
+            for j in range(len(self.fielddescrs[i])):
+                v = value._items[i][self.fielddescrs[i][j]]
+                s = self.fieldstate[p]
+                if s.position > self.position:
+                    s.enum_forced_boxes(boxes, v, optimizer)
+                p += 1
+
+    def debug_header(self, indent):
+        debug_print(indent + 'VArrayStructStateInfo(%d):' % self.position)
+
+
 class NotVirtualStateInfo(AbstractVirtualStateInfo):
     def __init__(self, value):
         self.known_class = value.known_class
@@ -277,7 +346,7 @@
             op = ResOperation(rop.GUARD_CLASS, [box, self.known_class], None)
             extra_guards.append(op)
             return
-        
+
         if self.level == LEVEL_NONNULL and \
                other.level == LEVEL_UNKNOWN and \
                isinstance(box, BoxPtr) and \
@@ -285,7 +354,7 @@
             op = ResOperation(rop.GUARD_NONNULL, [box], None)
             extra_guards.append(op)
             return
-        
+
         if self.level == LEVEL_UNKNOWN and \
                other.level == LEVEL_UNKNOWN and \
                isinstance(box, BoxInt) and \
@@ -309,7 +378,7 @@
                     op = ResOperation(rop.GUARD_TRUE, [res], None)
                     extra_guards.append(op)
             return
-        
+
         # Remaining cases are probably not interesting
         raise InvalidLoop
         if self.level == LEVEL_CONSTANT:
@@ -319,7 +388,7 @@
     def enum_forced_boxes(self, boxes, value, optimizer):
         if self.level == LEVEL_CONSTANT:
             return
-        assert 0 <= self.position_in_notvirtuals 
+        assert 0 <= self.position_in_notvirtuals
         boxes[self.position_in_notvirtuals] = value.force_box(optimizer)
 
     def _enum(self, virtual_state):
@@ -348,7 +417,7 @@
         lb = ''
         if self.lenbound:
             lb = ', ' + self.lenbound.bound.__repr__()
-        
+
         debug_print(indent + mark + 'NotVirtualInfo(%d' % self.position +
                     ', ' + l + ', ' + self.intbound.__repr__() + lb + ')')
 
@@ -370,7 +439,7 @@
                 return False
         return True
 
-    def generate_guards(self, other, args, cpu, extra_guards):        
+    def generate_guards(self, other, args, cpu, extra_guards):
         assert len(self.state) == len(other.state) == len(args)
         renum = {}
         for i in range(len(self.state)):
@@ -393,7 +462,7 @@
                     inputargs.append(box)
 
         assert None not in inputargs
-            
+
         return inputargs
 
     def debug_print(self, hdr='', bad=None):
@@ -412,7 +481,7 @@
 
     def register_virtual_fields(self, keybox, fieldboxes):
         self.fieldboxes[keybox] = fieldboxes
-        
+
     def already_seen_virtual(self, keybox):
         return keybox in self.fieldboxes
 
@@ -463,6 +532,9 @@
     def make_varray(self, arraydescr):
         return VArrayStateInfo(arraydescr)
 
+    def make_varraystruct(self, arraydescr, fielddescrs):
+        return VArrayStructStateInfo(arraydescr, fielddescrs)
+
 class BoxNotProducable(Exception):
     pass
 
@@ -501,12 +573,12 @@
             else: # Low priority
                 lo -= 1
         return alts
-            
+
     def renamed(self, box):
         if box in self.rename:
             return self.rename[box]
         return box
-    
+
     def add_to_short(self, box, op):
         if op:
             op = op.clone()
@@ -528,12 +600,12 @@
             self.optimizer.make_equal_to(newbox, value)
         else:
             self.short_boxes[box] = op
-        
+
     def produce_short_preamble_box(self, box):
         if box in self.short_boxes:
-            return 
+            return
         if isinstance(box, Const):
-            return 
+            return
         if box in self.potential_ops:
             ops = self.prioritized_alternatives(box)
             produced_one = False
@@ -570,7 +642,7 @@
             else:
                 debug_print(logops.repr_of_arg(box) + ': None')
         debug_stop('jit-short-boxes')
-        
+
     def operations(self):
         if not we_are_translated(): # For tests
             ops = self.short_boxes.values()
@@ -588,7 +660,7 @@
         if not isinstance(oldbox, Const) and newbox not in self.short_boxes:
             self.short_boxes[newbox] = self.short_boxes[oldbox]
         self.aliases[newbox] = oldbox
-        
+
     def original(self, box):
         while box in self.aliases:
             box = self.aliases[box]
diff --git a/pypy/jit/metainterp/optimizeopt/vstring.py b/pypy/jit/metainterp/optimizeopt/vstring.py
--- a/pypy/jit/metainterp/optimizeopt/vstring.py
+++ b/pypy/jit/metainterp/optimizeopt/vstring.py
@@ -163,17 +163,6 @@
             for value in self._chars:
                 value.get_args_for_fail(modifier)
 
-    def FIXME_enum_forced_boxes(self, boxes, already_seen):
-        key = self.get_key_box()
-        if key in already_seen:
-            return
-        already_seen[key] = None
-        if self.box is None:
-            for box in self._chars:
-                box.enum_forced_boxes(boxes, already_seen)
-        else:
-            boxes.append(self.box)
-
     def _make_virtual(self, modifier):
         return modifier.make_vstrplain(self.mode is mode_unicode)
 
@@ -226,18 +215,6 @@
             self.left.get_args_for_fail(modifier)
             self.right.get_args_for_fail(modifier)
 
-    def FIXME_enum_forced_boxes(self, boxes, already_seen):
-        key = self.get_key_box()
-        if key in already_seen:
-            return
-        already_seen[key] = None
-        if self.box is None:
-            self.left.enum_forced_boxes(boxes, already_seen)
-            self.right.enum_forced_boxes(boxes, already_seen)
-            self.lengthbox = None
-        else:
-            boxes.append(self.box)
-
     def _make_virtual(self, modifier):
         return modifier.make_vstrconcat(self.mode is mode_unicode)
 
@@ -284,18 +261,6 @@
             self.vstart.get_args_for_fail(modifier)
             self.vlength.get_args_for_fail(modifier)
 
-    def FIXME_enum_forced_boxes(self, boxes, already_seen):
-        key = self.get_key_box()
-        if key in already_seen:
-            return
-        already_seen[key] = None
-        if self.box is None:
-            self.vstr.enum_forced_boxes(boxes, already_seen)
-            self.vstart.enum_forced_boxes(boxes, already_seen)
-            self.vlength.enum_forced_boxes(boxes, already_seen)
-        else:
-            boxes.append(self.box)
-
     def _make_virtual(self, modifier):
         return modifier.make_vstrslice(self.mode is mode_unicode)
 
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -240,8 +240,8 @@
         return self.execute(rop.PTR_EQ, box, history.CONST_NULL)
 
     @arguments("box")
-    def opimpl_cast_opaque_ptr(self, box):
-        return self.execute(rop.CAST_OPAQUE_PTR, box)
+    def opimpl_mark_opaque_ptr(self, box):
+        return self.execute(rop.MARK_OPAQUE_PTR, box)
 
     @arguments("box")
     def _opimpl_any_return(self, box):
diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py
--- a/pypy/jit/metainterp/resoperation.py
+++ b/pypy/jit/metainterp/resoperation.py
@@ -439,7 +439,6 @@
     'PTR_NE/2b',
     'INSTANCE_PTR_EQ/2b',
     'INSTANCE_PTR_NE/2b',
-    'CAST_OPAQUE_PTR/1b',
     #
     'ARRAYLEN_GC/1d',
     'STRLEN/1',
@@ -471,6 +470,7 @@
     'FORCE_TOKEN/0',
     'VIRTUAL_REF/2',         # removed before it's passed to the backend
     'READ_TIMESTAMP/0',
+    'MARK_OPAQUE_PTR/1b',
     '_NOSIDEEFFECT_LAST', # ----- end of no_side_effect operations -----
 
     'SETARRAYITEM_GC/3d',
diff --git a/pypy/jit/metainterp/resume.py b/pypy/jit/metainterp/resume.py
--- a/pypy/jit/metainterp/resume.py
+++ b/pypy/jit/metainterp/resume.py
@@ -139,7 +139,7 @@
         self.numberings = {}
         self.cached_boxes = {}
         self.cached_virtuals = {}
-    
+
         self.nvirtuals = 0
         self.nvholes = 0
         self.nvreused = 0
@@ -273,6 +273,9 @@
     def make_varray(self, arraydescr):
         return VArrayInfo(arraydescr)
 
+    def make_varraystruct(self, arraydescr, fielddescrs):
+        return VArrayStructInfo(arraydescr, fielddescrs)
+
     def make_vstrplain(self, is_unicode=False):
         if is_unicode:
             return VUniPlainInfo()
@@ -402,7 +405,7 @@
                 virtuals[num] = vinfo
 
         if self._invalidation_needed(len(liveboxes), nholes):
-            memo.clear_box_virtual_numbers()           
+            memo.clear_box_virtual_numbers()
 
     def _invalidation_needed(self, nliveboxes, nholes):
         memo = self.memo
@@ -455,7 +458,7 @@
 
     def debug_prints(self):
         raise NotImplementedError
-        
+
 class AbstractVirtualStructInfo(AbstractVirtualInfo):
     def __init__(self, fielddescrs):
         self.fielddescrs = fielddescrs
@@ -537,6 +540,29 @@
         for i in self.fieldnums:
             debug_print("\t\t", str(untag(i)))
 
+
+class VArrayStructInfo(AbstractVirtualInfo):
+    def __init__(self, arraydescr, fielddescrs):
+        self.arraydescr = arraydescr
+        self.fielddescrs = fielddescrs
+
+    def debug_prints(self):
+        debug_print("\tvarraystructinfo", self.arraydescr)
+        for i in self.fieldnums:
+            debug_print("\t\t", str(untag(i)))
+
+    @specialize.argtype(1)
+    def allocate(self, decoder, index):
+        array = decoder.allocate_array(self.arraydescr, len(self.fielddescrs))
+        decoder.virtuals_cache[index] = array
+        p = 0
+        for i in range(len(self.fielddescrs)):
+            for j in range(len(self.fielddescrs[i])):
+                decoder.setinteriorfield(i, self.fielddescrs[i][j], array, self.fieldnums[p])
+                p += 1
+        return array
+
+
 class VStrPlainInfo(AbstractVirtualInfo):
     """Stands for the string made out of the characters of all fieldnums."""
 
@@ -884,6 +910,17 @@
         self.metainterp.execute_and_record(rop.SETFIELD_GC, descr,
                                            structbox, fieldbox)
 
+    def setinteriorfield(self, index, descr, array, fieldnum):
+        if descr.is_pointer_field():
+            kind = REF
+        elif descr.is_float_field():
+            kind = FLOAT
+        else:
+            kind = INT
+        fieldbox = self.decode_box(fieldnum, kind)
+        self.metainterp.execute_and_record(rop.SETINTERIORFIELD_GC, descr,
+                                           array, ConstInt(index), fieldbox)
+
     def setarrayitem_int(self, arraydescr, arraybox, index, fieldnum):
         self._setarrayitem(arraydescr, arraybox, index, fieldnum, INT)
 
@@ -1164,6 +1201,17 @@
             newvalue = self.decode_int(fieldnum)
             self.cpu.bh_setfield_gc_i(struct, descr, newvalue)
 
+    def setinteriorfield(self, index, descr, array, fieldnum):
+        if descr.is_pointer_field():
+            newvalue = self.decode_ref(fieldnum)
+            self.cpu.bh_setinteriorfield_gc_r(array, index, descr, newvalue)
+        elif descr.is_float_field():
+            newvalue = self.decode_float(fieldnum)
+            self.cpu.bh_setinteriorfield_gc_f(array, index, descr, newvalue)
+        else:
+            newvalue = self.decode_int(fieldnum)
+            self.cpu.bh_setinteriorfield_gc_i(array, index, descr, newvalue)
+
     def setarrayitem_int(self, arraydescr, array, index, fieldnum):
         newvalue = self.decode_int(fieldnum)
         self.cpu.bh_setarrayitem_gc_i(arraydescr, array, index, newvalue)
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -10,6 +10,7 @@
 from pypy.jit.metainterp.typesystem import LLTypeHelper, OOTypeHelper
 from pypy.jit.metainterp.warmspot import get_stats
 from pypy.jit.metainterp.warmstate import set_future_value
+from pypy.rlib import rerased
 from pypy.rlib.jit import (JitDriver, we_are_jitted, hint, dont_look_inside,
     loop_invariant, elidable, promote, jit_debug, assert_green,
     AssertGreenFailed, unroll_safe, current_trace_length, look_inside_iff,
@@ -3494,16 +3495,70 @@
             d = None
             while n > 0:
                 myjitdriver.jit_merge_point(n=n, d=d)
-                d = {}
+                d = {"q": 1}
                 if n % 2:
                     d["k"] = n
                 else:
                     d["z"] = n
-                n -= len(d)
+                n -= len(d) - d["q"]
             return n
         res = self.meta_interp(f, [10])
         assert res == 0
 
+    def test_virtual_dict_constant_keys(self):
+        myjitdriver = JitDriver(greens = [], reds = ["n"])
+        def g(d):
+            return d["key"] - 1
+
+        def f(n):
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n)
+                n = g({"key": n})
+            return n
+
+        res = self.meta_interp(f, [10])
+        assert res == 0
+        self.check_loops({"int_sub": 1, "int_gt": 1, "guard_true": 1, "jump": 1})
+
+    def test_virtual_opaque_ptr(self):
+        myjitdriver = JitDriver(greens = [], reds = ["n"])
+        erase, unerase = rerased.new_erasing_pair("x")
+        @look_inside_iff(lambda x: isvirtual(x))
+        def g(x):
+            return x[0]
+        def f(n):
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n)
+                x = []
+                y = erase(x)
+                z = unerase(y)
+                z.append(1)
+                n -= g(z)
+            return n
+        res = self.meta_interp(f, [10])
+        assert res == 0
+        self.check_loops({"int_sub": 1, "int_gt": 1, "guard_true": 1, "jump": 1})
+
+    def test_virtual_opaque_dict(self):
+        myjitdriver = JitDriver(greens = [], reds = ["n"])
+        erase, unerase = rerased.new_erasing_pair("x")
+        @look_inside_iff(lambda x: isvirtual(x))
+        def g(x):
+            return x[0]["key"] - 1
+        def f(n):
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n)
+                x = [{}]
+                x[0]["key"] = n
+                x[0]["other key"] = n
+                y = erase(x)
+                z = unerase(y)
+                n = g(x)
+            return n
+        res = self.meta_interp(f, [10])
+        assert res == 0
+        self.check_loops({"int_sub": 1, "int_gt": 1, "guard_true": 1, "jump": 1})
+
 
 
 class TestLLtype(BaseLLtypeTests, LLJitMixin):
@@ -3561,8 +3616,7 @@
         res = self.meta_interp(main, [False, 100, True], taggedpointers=True)
 
     def test_rerased(self):
-        from pypy.rlib.rerased import erase_int, unerase_int, new_erasing_pair
-        eraseX, uneraseX = new_erasing_pair("X")
+        eraseX, uneraseX = rerased.new_erasing_pair("X")
         #
         class X:
             def __init__(self, a, b):
@@ -3575,14 +3629,14 @@
                 e = eraseX(X(i, j))
             else:
                 try:
-                    e = erase_int(i)
+                    e = rerased.erase_int(i)
                 except OverflowError:
                     return -42
             if j & 1:
                 x = uneraseX(e)
                 return x.a - x.b
             else:
-                return unerase_int(e)
+                return rerased.unerase_int(e)
         #
         x = self.interp_operations(f, [-128, 0], taggedpointers=True)
         assert x == -128
diff --git a/pypy/jit/metainterp/test/test_heapcache.py b/pypy/jit/metainterp/test/test_heapcache.py
--- a/pypy/jit/metainterp/test/test_heapcache.py
+++ b/pypy/jit/metainterp/test/test_heapcache.py
@@ -371,3 +371,17 @@
         assert h.is_unescaped(box1)
         h.invalidate_caches(rop.SETARRAYITEM_GC, None, [box2, index1, box1])
         assert not h.is_unescaped(box1)
+
+        h = HeapCache()
+        h.new_array(box1, lengthbox1)
+        h.new(box2)
+        assert h.is_unescaped(box1)
+        assert h.is_unescaped(box2)
+        h.invalidate_caches(rop.SETARRAYITEM_GC, None, [box1, lengthbox2, box2])
+        assert h.is_unescaped(box1)
+        assert h.is_unescaped(box2)
+        h.invalidate_caches(
+            rop.CALL, FakeCallDescr(FakeEffektinfo.EF_RANDOM_EFFECTS), [box1]
+        )
+        assert not h.is_unescaped(box1)
+        assert not h.is_unescaped(box2)
diff --git a/pypy/jit/metainterp/test/test_tracingopts.py b/pypy/jit/metainterp/test/test_tracingopts.py
--- a/pypy/jit/metainterp/test/test_tracingopts.py
+++ b/pypy/jit/metainterp/test/test_tracingopts.py
@@ -3,6 +3,7 @@
 from pypy.jit.metainterp.test.support import LLJitMixin
 from pypy.rlib import jit
 from pypy.rlib.rarithmetic import ovfcheck
+from pypy.rlib.rstring import StringBuilder
 
 import py
 
@@ -590,4 +591,14 @@
         assert res == 4
         self.check_operations_history(int_add_ovf=0)
         res = self.interp_operations(fn, [sys.maxint])
-        assert res == 12
\ No newline at end of file
+        assert res == 12
+
+    def test_copy_str_content(self):
+        def fn(n):
+            a = StringBuilder()
+            x = [1]
+            a.append("hello world")
+            return x[0]
+        res = self.interp_operations(fn, [0])
+        assert res == 1
+        self.check_operations_history(getarrayitem_gc=0, getarrayitem_gc_pure=0 )
\ No newline at end of file
diff --git a/pypy/jit/metainterp/warmspot.py b/pypy/jit/metainterp/warmspot.py
--- a/pypy/jit/metainterp/warmspot.py
+++ b/pypy/jit/metainterp/warmspot.py
@@ -62,7 +62,7 @@
     clear_tcache()
     return jittify_and_run(interp, graph, args, backendopt=backendopt, **kwds)
 
-def jittify_and_run(interp, graph, args, repeat=1,
+def jittify_and_run(interp, graph, args, repeat=1, graph_and_interp_only=False,
                     backendopt=False, trace_limit=sys.maxint,
                     inline=False, loop_longevity=0, retrace_limit=5,
                     function_threshold=4,
@@ -93,6 +93,8 @@
         jd.warmstate.set_param_max_retrace_guards(max_retrace_guards)
         jd.warmstate.set_param_enable_opts(enable_opts)
     warmrunnerdesc.finish()
+    if graph_and_interp_only:
+        return interp, graph
     res = interp.eval_graph(graph, args)
     if not kwds.get('translate_support_code', False):
         warmrunnerdesc.metainterp_sd.profiler.finish()
@@ -157,6 +159,9 @@
 def get_stats():
     return pyjitpl._warmrunnerdesc.stats
 
+def reset_stats():
+    pyjitpl._warmrunnerdesc.stats.clear()
+
 def get_translator():
     return pyjitpl._warmrunnerdesc.translator
 
diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -4,30 +4,55 @@
 """
 
 from pypy.interpreter.baseobjspace import InternalSpaceCache, W_Root
-from pypy.module.micronumpy.interp_dtype import W_Float64Dtype
-from pypy.module.micronumpy.interp_numarray import Scalar, NDimArray, BaseArray
+from pypy.module.micronumpy.interp_dtype import W_Float64Dtype, W_BoolDtype
+from pypy.module.micronumpy.interp_numarray import (Scalar, BaseArray,
+     descr_new_array, scalar_w, NDimArray)
+from pypy.module.micronumpy import interp_ufuncs
 from pypy.rlib.objectmodel import specialize
 
 
 class BogusBytecode(Exception):
     pass
 
-def create_array(dtype, size):
-    a = NDimArray(size, dtype=dtype)
-    for i in range(size):
-        dtype.setitem(a.storage, i, dtype.box(float(i % 10)))
-    return a
+class ArgumentMismatch(Exception):
+    pass
+
+class ArgumentNotAnArray(Exception):
+    pass
+
+class WrongFunctionName(Exception):
+    pass
+
+SINGLE_ARG_FUNCTIONS = ["sum", "prod", "max", "min", "all", "any", "unegative"]
 
 class FakeSpace(object):
     w_ValueError = None
     w_TypeError = None
+    w_None = None
+
+    w_bool = "bool"
+    w_int = "int"
+    w_float = "float"
+    w_list = "list"
+    w_long = "long"
+    w_tuple = 'tuple'
+    w_slice = "slice"
 
     def __init__(self):
         """NOT_RPYTHON"""
         self.fromcache = InternalSpaceCache(self).getorbuild
+        self.w_float64dtype = W_Float64Dtype(self)
 
     def issequence_w(self, w_obj):
-        return True
+        return isinstance(w_obj, ListObject) or isinstance(w_obj, NDimArray)
+
+    def isinstance_w(self, w_obj, w_tp):
+        if w_obj.tp == w_tp:
+            return True
+        return False
+
+    def decode_index4(self, w_idx, size):
+        return (self.int_w(w_idx), 0, 0, 1)
 
     @specialize.argtype(1)
     def wrap(self, obj):
@@ -39,72 +64,391 @@
             return IntObject(obj)
         raise Exception
 
+    def newlist(self, items):
+        return ListObject(items)
+
+    def listview(self, obj):
+        assert isinstance(obj, ListObject)
+        return obj.items
+
     def float(self, w_obj):
         assert isinstance(w_obj, FloatObject)
         return w_obj
 
     def float_w(self, w_obj):
+        assert isinstance(w_obj, FloatObject)        
         return w_obj.floatval
 
+    def int_w(self, w_obj):
+        if isinstance(w_obj, IntObject):
+            return w_obj.intval
+        elif isinstance(w_obj, FloatObject):
+            return int(w_obj.floatval)
+        raise NotImplementedError
+
+    def int(self, w_obj):
+        return w_obj
+
+    def is_true(self, w_obj):
+        assert isinstance(w_obj, BoolObject)
+        return w_obj.boolval
+
+    def is_w(self, w_obj, w_what):
+        return w_obj is w_what
+
+    def type(self, w_obj):
+        return w_obj.tp
+
+    def gettypefor(self, w_obj):
+        return None
+
+    def call_function(self, tp, w_dtype):
+        return w_dtype
+
+    @specialize.arg(1)
+    def interp_w(self, tp, what):
+        assert isinstance(what, tp)
+        return what
+
+    def len_w(self, w_obj):
+        if isinstance(w_obj, ListObject):
+            return len(w_obj.items)
+        # XXX array probably
+        assert False
 
 class FloatObject(W_Root):
+    tp = FakeSpace.w_float
     def __init__(self, floatval):
         self.floatval = floatval
 
 class BoolObject(W_Root):
+    tp = FakeSpace.w_bool
     def __init__(self, boolval):
         self.boolval = boolval
 
 class IntObject(W_Root):
+    tp = FakeSpace.w_int
     def __init__(self, intval):
         self.intval = intval
 
+class ListObject(W_Root):
+    tp = FakeSpace.w_list
+    def __init__(self, items):
+        self.items = items
 
-space = FakeSpace()
+class InterpreterState(object):
+    def __init__(self, code):
+        self.code = code
+        self.variables = {}
+        self.results = []
 
-def numpy_compile(bytecode, array_size):
-    stack = []
-    i = 0
-    dtype = space.fromcache(W_Float64Dtype)
-    for b in bytecode:
-        if b == 'a':
-            stack.append(create_array(dtype, array_size))
-            i += 1
-        elif b == 'f':
-            stack.append(Scalar(dtype, dtype.box(1.2)))
-        elif b == '+':
-            right = stack.pop()
-            res = stack.pop().descr_add(space, right)
-            assert isinstance(res, BaseArray)
-            stack.append(res)
-        elif b == '-':
-            right = stack.pop()
-            res = stack.pop().descr_sub(space, right)
-            assert isinstance(res, BaseArray)
-            stack.append(res)
-        elif b == '*':
-            right = stack.pop()
-            res = stack.pop().descr_mul(space, right)
-            assert isinstance(res, BaseArray)
-            stack.append(res)
-        elif b == '/':
-            right = stack.pop()
-            res = stack.pop().descr_div(space, right)
-            assert isinstance(res, BaseArray)
-            stack.append(res)
-        elif b == '%':
-            right = stack.pop()
-            res = stack.pop().descr_mod(space, right)
-            assert isinstance(res, BaseArray)
-            stack.append(res)
-        elif b == '|':
-            res = stack.pop().descr_abs(space)
-            assert isinstance(res, BaseArray)
-            stack.append(res)
+    def run(self, space):
+        self.space = space
+        for stmt in self.code.statements:
+            stmt.execute(self)
+
+class Node(object):
+    def __eq__(self, other):
+        return (self.__class__ == other.__class__ and
+                self.__dict__ == other.__dict__)
+
+    def __ne__(self, other):
+        return not self == other
+
+    def wrap(self, space):
+        raise NotImplementedError
+
+    def execute(self, interp):
+        raise NotImplementedError
+
+class Assignment(Node):
+    def __init__(self, name, expr):
+        self.name = name
+        self.expr = expr
+
+    def execute(self, interp):
+        interp.variables[self.name] = self.expr.execute(interp)
+
+    def __repr__(self):
+        return "%% = %r" % (self.name, self.expr)
+
+class ArrayAssignment(Node):
+    def __init__(self, name, index, expr):
+        self.name = name
+        self.index = index
+        self.expr = expr
+
+    def execute(self, interp):
+        arr = interp.variables[self.name]
+        w_index = self.index.execute(interp).eval(0).wrap(interp.space)
+        # cast to int
+        if isinstance(w_index, FloatObject):
+            w_index = IntObject(int(w_index.floatval))
+        w_val = self.expr.execute(interp).eval(0).wrap(interp.space)
+        arr.descr_setitem(interp.space, w_index, w_val)
+
+    def __repr__(self):
+        return "%s[%r] = %r" % (self.name, self.index, self.expr)
+
+class Variable(Node):
+    def __init__(self, name):
+        self.name = name
+
+    def execute(self, interp):
+        return interp.variables[self.name]
+
+    def __repr__(self):
+        return 'v(%s)' % self.name
+
+class Operator(Node):
+    def __init__(self, lhs, name, rhs):
+        self.name = name
+        self.lhs = lhs
+        self.rhs = rhs
+
+    def execute(self, interp):
+        w_lhs = self.lhs.execute(interp)
+        assert isinstance(w_lhs, BaseArray)
+        if isinstance(self.rhs, SliceConstant):
+            # XXX interface has changed on multidim branch
+            raise NotImplementedError
+        w_rhs = self.rhs.execute(interp)
+        if self.name == '+':
+            w_res = w_lhs.descr_add(interp.space, w_rhs)
+        elif self.name == '*':
+            w_res = w_lhs.descr_mul(interp.space, w_rhs)
+        elif self.name == '-':
+            w_res = w_lhs.descr_sub(interp.space, w_rhs)            
+        elif self.name == '->':
+            if isinstance(w_rhs, Scalar):
+                index = int(interp.space.float_w(
+                    w_rhs.value.wrap(interp.space)))
+                dtype = interp.space.fromcache(W_Float64Dtype)
+                return Scalar(dtype, w_lhs.get_concrete().eval(index))
+            else:
+                raise NotImplementedError
         else:
-            print "Unknown opcode: %s" % b
-            raise BogusBytecode()
-    if len(stack) != 1:
-        print "Bogus bytecode, uneven stack length"
-        raise BogusBytecode()
-    return stack[0]
+            raise NotImplementedError
+        if not isinstance(w_res, BaseArray):
+            dtype = interp.space.fromcache(W_Float64Dtype)
+            w_res = scalar_w(interp.space, dtype, w_res)
+        return w_res
+
+    def __repr__(self):
+        return '(%r %s %r)' % (self.lhs, self.name, self.rhs)
+
+class FloatConstant(Node):
+    def __init__(self, v):
+        self.v = float(v)
+
+    def __repr__(self):
+        return "Const(%s)" % self.v
+
+    def wrap(self, space):
+        return space.wrap(self.v)
+
+    def execute(self, interp):
+        dtype = interp.space.fromcache(W_Float64Dtype)
+        assert isinstance(dtype, W_Float64Dtype)
+        return Scalar(dtype, dtype.box(self.v))
+
+class RangeConstant(Node):
+    def __init__(self, v):
+        self.v = int(v)
+
+    def execute(self, interp):
+        w_list = interp.space.newlist(
+            [interp.space.wrap(float(i)) for i in range(self.v)])
+        dtype = interp.space.fromcache(W_Float64Dtype)
+        return descr_new_array(interp.space, None, w_list, w_dtype=dtype)
+
+    def __repr__(self):
+        return 'Range(%s)' % self.v
+
+class Code(Node):
+    def __init__(self, statements):
+        self.statements = statements
+
+    def __repr__(self):
+        return "\n".join([repr(i) for i in self.statements])
+
+class ArrayConstant(Node):
+    def __init__(self, items):
+        self.items = items
+
+    def wrap(self, space):
+        return space.newlist([item.wrap(space) for item in self.items])
+
+    def execute(self, interp):
+        w_list = self.wrap(interp.space)
+        dtype = interp.space.fromcache(W_Float64Dtype)
+        return descr_new_array(interp.space, None, w_list, w_dtype=dtype)
+
+    def __repr__(self):
+        return "[" + ", ".join([repr(item) for item in self.items]) + "]"
+
+class SliceConstant(Node):
+    def __init__(self):
+        pass
+
+    def __repr__(self):
+        return 'slice()'
+
+class Execute(Node):
+    def __init__(self, expr):
+        self.expr = expr
+
+    def __repr__(self):
+        return repr(self.expr)
+
+    def execute(self, interp):
+        interp.results.append(self.expr.execute(interp))
+
+class FunctionCall(Node):
+    def __init__(self, name, args):
+        self.name = name
+        self.args = args
+
+    def __repr__(self):
+        return "%s(%s)" % (self.name, ", ".join([repr(arg)
+                                                 for arg in self.args]))
+
+    def execute(self, interp):
+        if self.name in SINGLE_ARG_FUNCTIONS:
+            if len(self.args) != 1:
+                raise ArgumentMismatch
+            arr = self.args[0].execute(interp)
+            if not isinstance(arr, BaseArray):
+                raise ArgumentNotAnArray
+            if self.name == "sum":
+                w_res = arr.descr_sum(interp.space)
+            elif self.name == "prod":
+                w_res = arr.descr_prod(interp.space)
+            elif self.name == "max":
+                w_res = arr.descr_max(interp.space)
+            elif self.name == "min":
+                w_res = arr.descr_min(interp.space)
+            elif self.name == "any":
+                w_res = arr.descr_any(interp.space)
+            elif self.name == "all":
+                w_res = arr.descr_all(interp.space)
+            elif self.name == "unegative":
+                neg = interp_ufuncs.get(interp.space).negative
+                w_res = neg.call(interp.space, [arr])
+            else:
+                assert False # unreachable code
+            if isinstance(w_res, BaseArray):
+                return w_res
+            if isinstance(w_res, FloatObject):
+                dtype = interp.space.fromcache(W_Float64Dtype)
+            elif isinstance(w_res, BoolObject):
+                dtype = interp.space.fromcache(W_BoolDtype)
+            else:
+                dtype = None
+            return scalar_w(interp.space, dtype, w_res)
+        else:
+            raise WrongFunctionName
+
+class Parser(object):
+    def parse_identifier(self, id):
+        id = id.strip(" ")
+        #assert id.isalpha()
+        return Variable(id)
+
+    def parse_expression(self, expr):
+        tokens = [i for i in expr.split(" ") if i]
+        if len(tokens) == 1:
+            return self.parse_constant_or_identifier(tokens[0])
+        stack = []
+        tokens.reverse()
+        while tokens:
+            token = tokens.pop()
+            if token == ')':
+                raise NotImplementedError
+            elif self.is_identifier_or_const(token):
+                if stack:
+                    name = stack.pop().name
+                    lhs = stack.pop()
+                    rhs = self.parse_constant_or_identifier(token)
+                    stack.append(Operator(lhs, name, rhs))
+                else:
+                    stack.append(self.parse_constant_or_identifier(token))
+            else:
+                stack.append(Variable(token))
+        assert len(stack) == 1
+        return stack[-1]
+
+    def parse_constant(self, v):
+        lgt = len(v)-1
+        assert lgt >= 0
+        if ':' in v:
+            # a slice
+            assert v == ':'
+            return SliceConstant()
+        if v[0] == '[':
+            return ArrayConstant([self.parse_constant(elem)
+                                  for elem in v[1:lgt].split(",")])
+        if v[0] == '|':
+            return RangeConstant(v[1:lgt])
+        return FloatConstant(v)
+
+    def is_identifier_or_const(self, v):
+        c = v[0]
+        if ((c >= 'a' and c <= 'z') or (c >= 'A' and c <= 'Z') or
+            (c >= '0' and c <= '9') or c in '-.[|:'):
+            if v == '-' or v == "->":
+                return False
+            return True
+        return False
+
+    def parse_function_call(self, v):
+        l = v.split('(')
+        assert len(l) == 2
+        name = l[0]
+        cut = len(l[1]) - 1
+        assert cut >= 0
+        args = [self.parse_constant_or_identifier(id)
+                for id in l[1][:cut].split(",")]
+        return FunctionCall(name, args)
+
+    def parse_constant_or_identifier(self, v):
+        c = v[0]
+        if (c >= 'a' and c <= 'z') or (c >= 'A' and c <= 'Z'):
+            if '(' in v:
+                return self.parse_function_call(v)
+            return self.parse_identifier(v)
+        return self.parse_constant(v)
+
+    def parse_array_subscript(self, v):
+        v = v.strip(" ")
+        l = v.split("[")
+        lgt = len(l[1]) - 1
+        assert lgt >= 0
+        rhs = self.parse_constant_or_identifier(l[1][:lgt])
+        return l[0], rhs
+        
+    def parse_statement(self, line):
+        if '=' in line:
+            lhs, rhs = line.split("=")
+            lhs = lhs.strip(" ")
+            if '[' in lhs:
+                name, index = self.parse_array_subscript(lhs)
+                return ArrayAssignment(name, index, self.parse_expression(rhs))
+            else: 
+                return Assignment(lhs, self.parse_expression(rhs))
+        else:
+            return Execute(self.parse_expression(line))
+
+    def parse(self, code):
+        statements = []
+        for line in code.split("\n"):
+            if '#' in line:
+                line = line.split('#', 1)[0]
+            line = line.strip(" ")
+            if line:
+                statements.append(self.parse_statement(line))
+        return Code(statements)
+
+def numpy_compile(code):
+    parser = Parser()
+    return InterpreterState(parser.parse(code))
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -108,6 +108,12 @@
         def setitem_w(self, space, storage, i, w_item):
             self.setitem(storage, i, self.unwrap(space, w_item))
 
+        def fill(self, storage, item, start, stop):
+            storage = self.unerase(storage)
+            item = self.unbox(item)
+            for i in xrange(start, stop):
+                storage[i] = item
+
         @specialize.argtype(1)
         def adapt_val(self, val):
             return self.box(rffi.cast(TP.TO.OF, val))
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -37,6 +37,8 @@
     batch = space.listview(w_iterable)
     while True:
         new_batch = []
+        if not batch:
+            return shape, []
         if not space.issequence_w(batch[0]):
             for elem in batch:
                 if space.issequence_w(elem):
@@ -52,6 +54,23 @@
         shape.append(size)
         batch = new_batch
 
+def descr_new_array(space, w_subtype, w_item_or_iterable, w_dtype=None):
+    # find scalar
+    if space.is_w(w_dtype, space.w_None):
+        w_dtype = _find_dtype(space, w_item_or_iterable)
+    dtype = space.interp_w(interp_dtype.W_Dtype,
+        space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype)
+    )
+    if not space.issequence_w(w_item_or_iterable):
+        return scalar_w(space, dtype, w_item_or_iterable)
+    shape, elems_w = _find_shape_and_elems(space, w_item_or_iterable)
+    size = len(elems_w)
+    arr = NDimArray(size, shape, dtype=dtype)
+    i = 0
+    for i, w_elem in enumerate(elems_w):
+        dtype.setitem_w(space, arr.storage, i, w_elem)
+    return arr
+
 class BaseArray(Wrappable):
     _attrs_ = ["invalidates", "signature"]
 
@@ -71,23 +90,6 @@
     def add_invalidates(self, other):
         self.invalidates.append(other)
 
-    def descr__new__(space, w_subtype, w_item_or_iterable, w_dtype=None):
-        # find scalar
-        if space.is_w(w_dtype, space.w_None):
-            w_dtype = _find_dtype(space, w_item_or_iterable)
-        dtype = space.interp_w(interp_dtype.W_Dtype,
-            space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype)
-        )
-        if not space.issequence_w(w_item_or_iterable):
-            return scalar_w(space, dtype, w_item_or_iterable)
-        shape, elems_w = _find_shape_and_elems(space, w_item_or_iterable)
-        size = len(elems_w)
-        arr = NDimArray(size, shape, dtype=dtype)
-        i = 0
-        for i, w_elem in enumerate(elems_w):
-            dtype.setitem_w(space, arr.storage, i, w_elem)
-        return arr
-
     def _unaryop_impl(ufunc_name):
         def impl(self, space):
             return getattr(interp_ufuncs.get(space), ufunc_name).call(space, [self])
@@ -262,6 +264,11 @@
         # we assume C ordering for now
         if space.isinstance_w(w_idx, space.w_int):
             idx = space.int_w(w_idx)
+            if not self.shape:
+                if idx != 0:
+                    raise OperationError(space.w_IndexError,
+                                         space.wrap("index out of range"))
+                return 0
             if idx < 0:
                 idx = self.shape[0] + idx
             if idx < 0 or idx >= self.shape[0]:
@@ -294,6 +301,11 @@
         is a list of scalars that match the size of shape
         """
         shape_len = self.len_of_shape()
+        if shape_len == 0:
+            if not space.isinstance_w(w_idx, space.w_int):
+                raise OperationError(space.w_IndexError, space.wrap(
+                    "wrong index"))
+            return True
         if shape_len == 1:
             if space.isinstance_w(w_idx, space.w_int):
                 return True
@@ -403,7 +415,7 @@
         self.value = value
 
     def find_size(self):
-        return 1
+        raise ValueError
 
     def get_concrete(self):
         return self
@@ -432,7 +444,7 @@
         i = 0
         signature = self.signature
         result_size = self.find_size()
-        result = NDimArray(result_size, [result_size], self.find_dtype())
+        result = NDimArray(result_size, self.shape, self.find_dtype())
         while i < result_size:
             numpy_driver.jit_merge_point(signature=signature,
                                          result_size=result_size, i=i,
@@ -706,13 +718,12 @@
 
     arr = NDimArray(size, [size], dtype=dtype)
     one = dtype.adapt_val(1)
-    for i in xrange(size):
-        arr.dtype.setitem(arr.storage, i, one)
+    arr.dtype.fill(arr.storage, one, 0, size)
     return space.wrap(arr)
 
 BaseArray.typedef = TypeDef(
     'numarray',
-    __new__ = interp2app(BaseArray.descr__new__.im_func),
+    __new__ = interp2app(descr_new_array),
 
 
     __len__ = interp2app(BaseArray.descr_len),
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -237,22 +237,20 @@
     return dt
 
 def find_dtype_for_scalar(space, w_obj, current_guess=None):
-    w_type = space.type(w_obj)
-
     bool_dtype = space.fromcache(interp_dtype.W_BoolDtype)
     long_dtype = space.fromcache(interp_dtype.W_LongDtype)
     int64_dtype = space.fromcache(interp_dtype.W_Int64Dtype)
 
-    if space.is_w(w_type, space.w_bool):
+    if space.isinstance_w(w_obj, space.w_bool):
         if current_guess is None or current_guess is bool_dtype:
             return bool_dtype
         return current_guess
-    elif space.is_w(w_type, space.w_int):
+    elif space.isinstance_w(w_obj, space.w_int):
         if (current_guess is None or current_guess is bool_dtype or
             current_guess is long_dtype):
             return long_dtype
         return current_guess
-    elif space.is_w(w_type, space.w_long):
+    elif space.isinstance_w(w_obj, space.w_long):
         if (current_guess is None or current_guess is bool_dtype or
             current_guess is long_dtype or current_guess is int64_dtype):
             return int64_dtype
diff --git a/pypy/module/micronumpy/test/test_compile.py b/pypy/module/micronumpy/test/test_compile.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/test/test_compile.py
@@ -0,0 +1,170 @@
+
+import py
+from pypy.module.micronumpy.compile import *
+
+class TestCompiler(object):
+    def compile(self, code):
+        return numpy_compile(code)
+    
+    def test_vars(self):
+        code = """
+        a = 2
+        b = 3
+        """
+        interp = self.compile(code)
+        assert isinstance(interp.code.statements[0], Assignment)
+        assert interp.code.statements[0].name == 'a'
+        assert interp.code.statements[0].expr.v == 2
+        assert interp.code.statements[1].name == 'b'
+        assert interp.code.statements[1].expr.v == 3
+
+    def test_array_literal(self):
+        code = "a = [1,2,3]"
+        interp = self.compile(code)
+        assert isinstance(interp.code.statements[0].expr, ArrayConstant)
+        st = interp.code.statements[0]
+        assert st.expr.items == [FloatConstant(1), FloatConstant(2),
+                                 FloatConstant(3)]
+    
+    def test_array_literal2(self):
+        code = "a = [[1],[2],[3]]"
+        interp = self.compile(code)
+        assert isinstance(interp.code.statements[0].expr, ArrayConstant)
+        st = interp.code.statements[0]
+        assert st.expr.items == [ArrayConstant([FloatConstant(1)]),
+                                 ArrayConstant([FloatConstant(2)]),
+                                 ArrayConstant([FloatConstant(3)])]
+
+    def test_expr_1(self):
+        code = "b = a + 1"
+        interp = self.compile(code)
+        assert (interp.code.statements[0].expr ==
+                Operator(Variable("a"), "+", FloatConstant(1)))
+
+    def test_expr_2(self):
+        code = "b = a + b - 3"
+        interp = self.compile(code)
+        assert (interp.code.statements[0].expr ==
+                Operator(Operator(Variable("a"), "+", Variable("b")), "-",
+                         FloatConstant(3)))
+
+    def test_expr_3(self):
+        # an equivalent of range
+        code = "a = |20|"
+        interp = self.compile(code)
+        assert interp.code.statements[0].expr == RangeConstant(20)
+
+    def test_expr_only(self):
+        code = "3 + a"
+        interp = self.compile(code)
+        assert interp.code.statements[0] == Execute(
+            Operator(FloatConstant(3), "+", Variable("a")))
+
+    def test_array_access(self):
+        code = "a -> 3"
+        interp = self.compile(code)
+        assert interp.code.statements[0] == Execute(
+            Operator(Variable("a"), "->", FloatConstant(3)))
+
+    def test_function_call(self):
+        code = "sum(a)"
+        interp = self.compile(code)
+        assert interp.code.statements[0] == Execute(
+            FunctionCall("sum", [Variable("a")]))
+
+    def test_comment(self):
+        code = """
+        # some comment
+        a = b + 3  # another comment
+        """
+        interp = self.compile(code)
+        assert interp.code.statements[0] == Assignment(
+            'a', Operator(Variable('b'), "+", FloatConstant(3)))
+
+class TestRunner(object):
+    def run(self, code):
+        interp = numpy_compile(code)
+        space = FakeSpace()
+        interp.run(space)
+        return interp
+
+    def test_one(self):
+        code = """
+        a = 3
+        b = 4
+        a + b
+        """
+        interp = self.run(code)
+        assert sorted(interp.variables.keys()) == ['a', 'b']
+        assert interp.results[0]
+
+    def test_array_add(self):
+        code = """
+        a = [1,2,3,4]
+        b = [4,5,6,5]
+        a + b
+        """
+        interp = self.run(code)
+        assert interp.results[0]._getnums(False) == ["5.0", "7.0", "9.0", "9.0"]
+
+    def test_array_getitem(self):
+        code = """
+        a = [1,2,3,4]
+        b = [4,5,6,5]
+        a + b -> 3
+        """
+        interp = self.run(code)
+        assert interp.results[0].value.val == 3 + 6
+        
+    def test_range_getitem(self):
+        code = """
+        r = |20| + 3
+        r -> 3
+        """
+        interp = self.run(code)
+        assert interp.results[0].value.val == 6
+
+    def test_sum(self):
+        code = """
+        a = [1,2,3,4,5]
+        r = sum(a)
+        r
+        """
+        interp = self.run(code)
+        assert interp.results[0].value.val == 15
+
+    def test_array_write(self):
+        code = """
+        a = [1,2,3,4,5]
+        a[3] = 15
+        a -> 3
+        """
+        interp = self.run(code)
+        assert interp.results[0].value.val == 15
+
+    def test_min(self):
+        interp = self.run("""
+        a = |30|
+        a[15] = -12
+        b = a + a
+        min(b)
+        """)
+        assert interp.results[0].value.val == -24
+
+    def test_max(self):
+        interp = self.run("""
+        a = |30|
+        a[13] = 128
+        b = a + a
+        max(b)
+        """)
+        assert interp.results[0].value.val == 256
+
+    def test_slice(self):
+        py.test.skip("in progress")
+        interp = self.run("""
+        a = [1,2,3,4]
+        b = a -> :
+        b -> 3
+        """)
+        assert interp.results[0].value.val == 3
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -82,6 +82,8 @@
         b = negative(a)
         a[0] = 5.0
         assert b[0] == 5.0
+        a = array(range(30))
+        assert negative(a + a)[3] == -6
 
     def test_abs(self):
         from numpy import array, absolute
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -1,261 +1,206 @@
+
+""" Tests that check if JIT-compiled numpy operations produce reasonably
+good assembler
+"""
+
+
+import py
 from pypy.jit.metainterp.test.support import LLJitMixin
 from pypy.module.micronumpy import interp_ufuncs, signature
 from pypy.module.micronumpy.compile import (numpy_compile, FakeSpace,
-    FloatObject, IntObject)
-from pypy.module.micronumpy.interp_dtype import W_Int32Dtype, W_Float64Dtype, W_Int64Dtype, W_UInt64Dtype
-from pypy.module.micronumpy.interp_numarray import (BaseArray, NDimArray,
-    NDimSlice, scalar_w)
+    FloatObject, IntObject, BoolObject)
+from pypy.module.micronumpy.interp_numarray import NDimArray, NDimSlice
 from pypy.rlib.nonconst import NonConstant
-from pypy.rpython.annlowlevel import llstr
-from pypy.rpython.test.test_llinterp import interpret
-
-import py
-py.test.skip("XXX")
+from pypy.rpython.annlowlevel import llstr, hlstr
+from pypy.jit.metainterp.warmspot import reset_stats
+from pypy.jit.metainterp import pyjitpl
 
 class TestNumpyJIt(LLJitMixin):
-    def setup_class(cls):
-        cls.space = FakeSpace()
-        cls.float64_dtype = cls.space.fromcache(W_Float64Dtype)
-        cls.int64_dtype = cls.space.fromcache(W_Int64Dtype)
-        cls.uint64_dtype = cls.space.fromcache(W_UInt64Dtype)
-        cls.int32_dtype = cls.space.fromcache(W_Int32Dtype)
+    graph = None
+    interp = None
+        
+    def run(self, code):
+        space = FakeSpace()
+        
+        def f(code):
+            interp = numpy_compile(hlstr(code))
+            interp.run(space)
+            res = interp.results[-1]
+            w_res = res.eval(0).wrap(interp.space)
+            if isinstance(w_res, BoolObject):
+                return float(w_res.boolval)
+            elif isinstance(w_res, FloatObject):
+                return w_res.floatval
+            elif isinstance(w_res, IntObject):
+                return w_res.intval
+            else:
+                return -42.
+
+        if self.graph is None:
+            interp, graph = self.meta_interp(f, [llstr(code)],
+                                             listops=True,
+                                             backendopt=True,
+                                             graph_and_interp_only=True)
+            self.__class__.interp = interp
+            self.__class__.graph = graph
+
+        reset_stats()
+        pyjitpl._warmrunnerdesc.memory_manager.alive_loops.clear()
+        return self.interp.eval_graph(self.graph, [llstr(code)])
 
     def test_add(self):
-        def f(i):
-            ar = SingleDimArray(i, dtype=self.float64_dtype)
-            v = interp_ufuncs.get(self.space).add.call(self.space, [ar, ar])
-            return v.get_concrete().eval(3).val
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
+        result = self.run("""
+        a = |30|
+        b = a + a
+        b -> 3
+        """)
         self.check_loops({'getarrayitem_raw': 2, 'float_add': 1,
                           'setarrayitem_raw': 1, 'int_add': 1,
                           'int_lt': 1, 'guard_true': 1, 'jump': 1})
-        assert result == f(5)
+        assert result == 3 + 3
 
     def test_floatadd(self):
-        def f(i):
-            ar = SingleDimArray(i, dtype=self.float64_dtype)
-            v = interp_ufuncs.get(self.space).add.call(self.space, [
-                    ar,
-                    scalar_w(self.space, self.float64_dtype, self.space.wrap(4.5))
-                ],
-            )
-            assert isinstance(v, BaseArray)
-            return v.get_concrete().eval(3).val
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
+        result = self.run("""
+        a = |30| + 3
+        a -> 3
+        """)
+        assert result == 3 + 3
         self.check_loops({"getarrayitem_raw": 1, "float_add": 1,
                           "setarrayitem_raw": 1, "int_add": 1,
                           "int_lt": 1, "guard_true": 1, "jump": 1})
-        assert result == f(5)
 
     def test_sum(self):
-        space = self.space
-        float64_dtype = self.float64_dtype
-        int64_dtype = self.int64_dtype
-
-        def f(i):
-            if NonConstant(False):
-                dtype = int64_dtype
-            else:
-                dtype = float64_dtype
-            ar = SingleDimArray(i, dtype=dtype)
-            v = ar.descr_add(space, ar).descr_sum(space)
-            assert isinstance(v, FloatObject)
-            return v.floatval
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
+        result = self.run("""
+        a = |30|
+        b = a + a
+        sum(b)
+        """)
+        assert result == 2 * sum(range(30))
         self.check_loops({"getarrayitem_raw": 2, "float_add": 2,
                           "int_add": 1,
                           "int_lt": 1, "guard_true": 1, "jump": 1})
-        assert result == f(5)
 
     def test_prod(self):
-        space = self.space
-        float64_dtype = self.float64_dtype
-        int64_dtype = self.int64_dtype
-
-        def f(i):
-            if NonConstant(False):
-                dtype = int64_dtype
-            else:
-                dtype = float64_dtype
-            ar = SingleDimArray(i, dtype=dtype)
-            v = ar.descr_add(space, ar).descr_prod(space)
-            assert isinstance(v, FloatObject)
-            return v.floatval
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
+        result = self.run("""
+        a = |30|
+        b = a + a
+        prod(b)
+        """)
+        expected = 1
+        for i in range(30):
+            expected *= i * 2
+        assert result == expected
         self.check_loops({"getarrayitem_raw": 2, "float_add": 1,
                           "float_mul": 1, "int_add": 1,
                           "int_lt": 1, "guard_true": 1, "jump": 1})
-        assert result == f(5)
 
     def test_max(self):
-        space = self.space
-        float64_dtype = self.float64_dtype
-        int64_dtype = self.int64_dtype
-
-        def f(i):
-            if NonConstant(False):
-                dtype = int64_dtype
-            else:
-                dtype = float64_dtype
-            ar = SingleDimArray(i, dtype=dtype)
-            j = 0
-            while j < i:
-                ar.get_concrete().setitem(j, float64_dtype.box(float(j)))
-                j += 1
-            v = ar.descr_add(space, ar).descr_max(space)
-            assert isinstance(v, FloatObject)
-            return v.floatval
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
+        py.test.skip("broken, investigate")
+        result = self.run("""
+        a = |30|
+        a[13] = 128
+        b = a + a
+        max(b)
+        """)
+        assert result == 256
         self.check_loops({"getarrayitem_raw": 2, "float_add": 1,
-                          "float_gt": 1, "int_add": 1,
-                          "int_lt": 1, "guard_true": 1,
-                          "guard_false": 1, "jump": 1})
-        assert result == f(5)
+                          "float_mul": 1, "int_add": 1,
+                          "int_lt": 1, "guard_true": 1, "jump": 1})
 
     def test_min(self):
-        space = self.space
-        float64_dtype = self.float64_dtype
-        int64_dtype = self.int64_dtype
-
-        def f(i):
-            if NonConstant(False):
-                dtype = int64_dtype
-            else:
-                dtype = float64_dtype
-            ar = SingleDimArray(i, dtype=dtype)
-            j = 0
-            while j < i:
-                ar.get_concrete().setitem(j, float64_dtype.box(float(j)))
-                j += 1
-            v = ar.descr_add(space, ar).descr_min(space)
-            assert isinstance(v, FloatObject)
-            return v.floatval
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
+        py.test.skip("broken, investigate")
+        result = self.run("""
+        a = |30|
+        a[15] = -12
+        b = a + a
+        min(b)
+        """)
+        assert result == -24
         self.check_loops({"getarrayitem_raw": 2, "float_add": 1,
-                           "float_lt": 1, "int_add": 1,
-                           "int_lt": 1, "guard_true": 2,
-                           "jump": 1})
-        assert result == f(5)
-
-    def test_argmin(self):
-        space = self.space
-        float64_dtype = self.float64_dtype
-
-        def f(i):
-            ar = SingleDimArray(i, dtype=NonConstant(float64_dtype))
-            j = 0
-            while j < i:
-                ar.get_concrete().setitem(j, float64_dtype.box(float(j)))
-                j += 1
-            return ar.descr_add(space, ar).descr_argmin(space).intval
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
-        self.check_loops({"getarrayitem_raw": 2, "float_add": 1,
-                           "float_lt": 1, "int_add": 1,
-                           "int_lt": 1, "guard_true": 2,
-                           "jump": 1})
-        assert result == f(5)
-
-    def test_all(self):
-        space = self.space
-        float64_dtype = self.float64_dtype
-
-        def f(i):
-            ar = SingleDimArray(i, dtype=NonConstant(float64_dtype))
-            j = 0
-            while j < i:
-                ar.get_concrete().setitem(j, float64_dtype.box(1.0))
-                j += 1
-            return ar.descr_add(space, ar).descr_all(space).boolval
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
-        self.check_loops({"getarrayitem_raw": 2, "float_add": 1,
-                          "int_add": 1, "float_ne": 1,
-                          "int_lt": 1, "guard_true": 2, "jump": 1})
-        assert result == f(5)
+                          "float_mul": 1, "int_add": 1,
+                          "int_lt": 1, "guard_true": 1, "jump": 1})
 
     def test_any(self):
-        space = self.space
-        float64_dtype = self.float64_dtype
-
-        def f(i):
-            ar = SingleDimArray(i, dtype=NonConstant(float64_dtype))
-            return ar.descr_add(space, ar).descr_any(space).boolval
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
+        result = self.run("""
+        a = [0,0,0,0,0,0,0,0,0,0,0]
+        a[8] = -12
+        b = a + a
+        any(b)
+        """)
+        assert result == 1
         self.check_loops({"getarrayitem_raw": 2, "float_add": 1,
-                          "int_add": 1, "float_ne": 1, "guard_false": 1,
-                          "int_lt": 1, "guard_true": 1, "jump": 1})
-        assert result == f(5)
+                          "float_ne": 1, "int_add": 1,
+                          "int_lt": 1, "guard_true": 1, "jump": 1,
+                          "guard_false": 1})
 
     def test_already_forced(self):
-        space = self.space
-
-        def f(i):
-            ar = SingleDimArray(i, dtype=self.float64_dtype)
-            v1 = interp_ufuncs.get(self.space).add.call(space, [ar, scalar_w(space, self.float64_dtype, space.wrap(4.5))])
-            assert isinstance(v1, BaseArray)
-            v2 = interp_ufuncs.get(self.space).multiply.call(space, [v1, scalar_w(space, self.float64_dtype, space.wrap(4.5))])
-            v1.force_if_needed()
-            assert isinstance(v2, BaseArray)
-            return v2.get_concrete().eval(3).val
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
+        result = self.run("""
+        a = |30|
+        b = a + 4.5
+        b -> 5 # forces
+        c = b * 8
+        c -> 5
+        """)
+        assert result == (5 + 4.5) * 8
         # This is the sum of the ops for both loops, however if you remove the
         # optimization then you end up with 2 float_adds, so we can still be
         # sure it was optimized correctly.
         self.check_loops({"getarrayitem_raw": 2, "float_mul": 1, "float_add": 1,
                            "setarrayitem_raw": 2, "int_add": 2,
                            "int_lt": 2, "guard_true": 2, "jump": 2})
-        assert result == f(5)
 
     def test_ufunc(self):
-        space = self.space
-        def f(i):
-            ar = SingleDimArray(i, dtype=self.float64_dtype)
-            v1 = interp_ufuncs.get(self.space).add.call(space, [ar, ar])
-            v2 = interp_ufuncs.get(self.space).negative.call(space, [v1])
-            return v2.get_concrete().eval(3).val
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
+        result = self.run("""
+        a = |30|
+        b = a + a
+        c = unegative(b)
+        c -> 3
+        """)
+        assert result == -6
         self.check_loops({"getarrayitem_raw": 2, "float_add": 1, "float_neg": 1,
                           "setarrayitem_raw": 1, "int_add": 1,
                           "int_lt": 1, "guard_true": 1, "jump": 1,
         })
-        assert result == f(5)
 
-    def test_appropriate_specialization(self):
-        space = self.space
-        def f(i):
-            ar = SingleDimArray(i, dtype=self.float64_dtype)
-
-            v1 = interp_ufuncs.get(self.space).add.call(space, [ar, ar])
-            v2 = interp_ufuncs.get(self.space).negative.call(space, [v1])
-            v2.get_concrete()
-
-            for i in xrange(5):
-                v1 = interp_ufuncs.get(self.space).multiply.call(space, [ar, ar])
-                v2 = interp_ufuncs.get(self.space).negative.call(space, [v1])
-                v2.get_concrete()
-
-        self.meta_interp(f, [5], listops=True, backendopt=True)
+    def test_specialization(self):
+        self.run("""
+        a = |30|
+        b = a + a
+        c = unegative(b)
+        c -> 3
+        d = a * a
+        unegative(d)
+        d -> 3
+        d = a * a
+        unegative(d)
+        d -> 3
+        d = a * a
+        unegative(d)
+        d -> 3
+        d = a * a
+        unegative(d)
+        d -> 3
+        """)
         # This is 3, not 2 because there is a bridge for the exit.
         self.check_loop_count(3)
 
+
+class TestNumpyOld(LLJitMixin):
+    def setup_class(cls):
+        from pypy.module.micronumpy.compile import FakeSpace
+        from pypy.module.micronumpy.interp_dtype import W_Float64Dtype
+        
+        cls.space = FakeSpace()
+        cls.float64_dtype = cls.space.fromcache(W_Float64Dtype)
+    
     def test_slice(self):
         def f(i):
             step = 3
-            ar = SingleDimArray(step*i, dtype=self.float64_dtype)
+            ar = NDimArray(step*i, dtype=self.float64_dtype)
             new_sig = signature.Signature.find_sig([
-                SingleDimSlice.signature, ar.signature
+                NDimSlice.signature, ar.signature
             ])
-            s = SingleDimSlice(0, step*i, step, i, ar, new_sig)
+            s = NDimSlice(0, step*i, step, i, ar, new_sig)
             v = interp_ufuncs.get(self.space).add.call(self.space, [s, s])
             return v.get_concrete().eval(3).val
 
@@ -269,15 +214,15 @@
         def f(i):
             step1 = 2
             step2 = 3
-            ar = SingleDimArray(step2*i, dtype=self.float64_dtype)
+            ar = NDimArray(step2*i, dtype=self.float64_dtype)
             new_sig = signature.Signature.find_sig([
-                SingleDimSlice.signature, ar.signature
+                NDimSlice.signature, ar.signature
             ])
-            s1 = SingleDimSlice(0, step1*i, step1, i, ar, new_sig)
+            s1 = NDimSlice(0, step1*i, step1, i, ar, new_sig)
             new_sig = signature.Signature.find_sig([
-                SingleDimSlice.signature, s1.signature
+                NDimSlice.signature, s1.signature
             ])
-            s2 = SingleDimSlice(0, step2*i, step2, i, ar, new_sig)
+            s2 = NDimSlice(0, step2*i, step2, i, ar, new_sig)
             v = interp_ufuncs.get(self.space).add.call(self.space, [s1, s2])
             return v.get_concrete().eval(3).val
 
@@ -293,8 +238,8 @@
 
         def f(i):
             step = NonConstant(3)
-            ar = SingleDimArray(step*i, dtype=float64_dtype)
-            ar2 = SingleDimArray(i, dtype=float64_dtype)
+            ar = NDimArray(step*i, dtype=float64_dtype)
+            ar2 = NDimArray(i, dtype=float64_dtype)
             ar2.get_concrete().setitem(1, float64_dtype.box(5.5))
             arg = ar2.descr_add(space, ar2)
             ar.setslice(space, 0, step*i, step, i, arg)
@@ -332,17 +277,3 @@
         result = self.meta_interp(f, [5], listops=True, backendopt=True)
         assert result == f(5)
 
-class TestTranslation(object):
-    def test_compile(self):
-        x = numpy_compile('aa+f*f/a-', 10)
-        x = x.compute()
-        assert isinstance(x, SingleDimArray)
-        assert x.size == 10
-        assert x.eval(0).val == 0
-        assert x.eval(1).val == ((1 + 1) * 1.2) / 1.2 - 1
-
-    def test_translation(self):
-        # we import main to check if the target compiles
-        from pypy.translator.goal.targetnumpystandalone import main
-
-        interpret(main, [llstr('af+'), 100])
diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py
--- a/pypy/module/pypyjit/test_pypy_c/test_call.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_call.py
@@ -465,3 +465,25 @@
             setfield_gc(p4, p22, descr=<GcPtrFieldDescr pypy.interpreter.nestedscope.Cell.inst_w_value .*>)
             jump(p0, p1, p2, p3, p4, p7, p22, p7, descr=<Loop0>)
         """)
+
+    def test_kwargs_virtual(self):
+        def main(n):
+            def g(**kwargs):
+                return kwargs["x"] + 1
+
+            i = 0
+            while i < n:
+                i = g(x=i)
+            return i
+
+        log = self.run(main, [500])
+        assert log.result == 500
+        loop, = log.loops_by_filename(self.filepath)
+        assert loop.match("""
+            i2 = int_lt(i0, i1)
+            guard_true(i2, descr=...)
+            i3 = force_token()
+            i4 = int_add(i0, 1)
+            --TICK--
+            jump(..., descr=...)
+        """)
\ No newline at end of file
diff --git a/pypy/module/pypyjit/test_pypy_c/test_containers.py b/pypy/module/pypyjit/test_pypy_c/test_containers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_containers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_containers.py
@@ -44,7 +44,7 @@
         # gc_id call is hoisted out of the loop, the id of a value obviously
         # can't change ;)
         assert loop.match_by_id("getitem", """
-            i28 = call(ConstClass(ll_dict_lookup__dicttablePtr_objectPtr_Signed), p18, p6, i25, descr=...)
+            i26 = call(ConstClass(ll_dict_lookup), p18, p6, i25, descr=...)
             ...
             p33 = getinteriorfield_gc(p31, i26, descr=<InteriorFieldDescr <GcPtrFieldDescr dictentry.value .*>>)
             ...
diff --git a/pypy/objspace/std/test/test_stdobjspace.py b/pypy/objspace/std/test/test_stdobjspace.py
--- a/pypy/objspace/std/test/test_stdobjspace.py
+++ b/pypy/objspace/std/test/test_stdobjspace.py
@@ -14,11 +14,11 @@
 
     def test_int_w_non_int(self):
         raises(OperationError,self.space.int_w,self.space.wrap(None))
-        raises(OperationError,self.space.int_w,self.space.wrap(""))        
+        raises(OperationError,self.space.int_w,self.space.wrap(""))
 
     def test_uint_w_non_int(self):
         raises(OperationError,self.space.uint_w,self.space.wrap(None))
-        raises(OperationError,self.space.uint_w,self.space.wrap(""))        
+        raises(OperationError,self.space.uint_w,self.space.wrap(""))
 
     def test_multimethods_defined_on(self):
         from pypy.objspace.std.stdtypedef import multimethods_defined_on
@@ -49,14 +49,14 @@
     def test_fastpath_isinstance(self):
         from pypy.objspace.std.stringobject import W_StringObject
         from pypy.objspace.std.intobject import W_IntObject
-        
+
         space = self.space
-        assert space.w_str.interplevel_cls is W_StringObject
-        assert space.w_int.interplevel_cls is W_IntObject
+        assert space._get_interplevel_cls(space.w_str) is W_StringObject
+        assert space._get_interplevel_cls(space.w_int) is W_IntObject
         class X(W_StringObject):
             def __init__(self):
                 pass
-            
+
             typedef = None
 
         assert space.isinstance_w(X(), space.w_str)
diff --git a/pypy/rlib/rsre/rsre_core.py b/pypy/rlib/rsre/rsre_core.py
--- a/pypy/rlib/rsre/rsre_core.py
+++ b/pypy/rlib/rsre/rsre_core.py
@@ -391,6 +391,8 @@
             if self.num_pending >= min:
                 while enum is not None and ptr == ctx.match_end:
                     enum = enum.move_to_next_result(ctx)
+                    # matched marks for zero-width assertions
+                    marks = ctx.match_marks
             #
             if enum is not None:
                 # matched one more 'item'.  record it and continue.
diff --git a/pypy/rlib/rsre/test/test_re.py b/pypy/rlib/rsre/test/test_re.py
--- a/pypy/rlib/rsre/test/test_re.py
+++ b/pypy/rlib/rsre/test/test_re.py
@@ -226,6 +226,13 @@
                          (None, 'b', None))
         assert pat.match('ac').group(1, 'b2', 3) == ('a', None, 'c')
 
+    def test_bug_923(self):
+        # Issue923: grouping inside optional lookahead problem
+        assert re.match(r'a(?=(b))?', "ab").groups() == ("b",)
+        assert re.match(r'(a(?=(b))?)', "ab").groups() == ('a', 'b')
+        assert re.match(r'(a)(?=(b))?', "ab").groups() == ('a', 'b')
+        assert re.match(r'(?P<g1>a)(?=(?P<g2>b))?', "ab").groupdict() == {'g1': 'a', 'g2': 'b'}
+
     def test_re_groupref_exists(self):
         assert re.match('^(\()?([^()]+)(?(1)\))$', '(a)').groups() == (
                          ('(', 'a'))
diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py
--- a/pypy/rpython/lltypesystem/rdict.py
+++ b/pypy/rpython/lltypesystem/rdict.py
@@ -445,9 +445,9 @@
     i = ll_dict_lookup(d, key, hash)
     return _ll_dict_setitem_lookup_done(d, key, value, hash, i)
 
-# Leaving as dont_look_inside ATM, it has a few branches which could lead to
-# many bridges if we don't consider their possible frequency.
- at jit.dont_look_inside
+# It may be safe to look inside always, it has a few branches though, and their
+# frequencies needs to be investigated.
+ at jit.look_inside_iff(lambda d, key, value, hash, i: jit.isvirtual(d) and jit.isconstant(key))
 def _ll_dict_setitem_lookup_done(d, key, value, hash, i):
     valid = (i & HIGHEST_BIT) == 0
     i = i & MASK
@@ -533,7 +533,7 @@
 # ------- a port of CPython's dictobject.c's lookdict implementation -------
 PERTURB_SHIFT = 5
 
- at jit.dont_look_inside
+ at jit.look_inside_iff(lambda d, key, hash: jit.isvirtual(d) and jit.isconstant(key))
 def ll_dict_lookup(d, key, hash):
     entries = d.entries
     ENTRIES = lltype.typeOf(entries).TO
diff --git a/pypy/rpython/lltypesystem/rstr.py b/pypy/rpython/lltypesystem/rstr.py
--- a/pypy/rpython/lltypesystem/rstr.py
+++ b/pypy/rpython/lltypesystem/rstr.py
@@ -20,6 +20,7 @@
 from pypy.rpython.rmodel import Repr
 from pypy.rpython.lltypesystem import llmemory
 from pypy.tool.sourcetools import func_with_new_name
+from pypy.rpython.lltypesystem.lloperation import llop
 
 # ____________________________________________________________
 #
@@ -364,8 +365,10 @@
             while lpos < rpos and s.chars[lpos] == ch:
                 lpos += 1
         if right:
-            while lpos < rpos and s.chars[rpos] == ch:
+            while lpos < rpos + 1 and s.chars[rpos] == ch:
                 rpos -= 1
+        if rpos < lpos:
+            return s.empty()
         r_len = rpos - lpos + 1
         result = s.malloc(r_len)
         s.copy_contents(s, result, lpos, 0, r_len)
diff --git a/pypy/rpython/test/test_rstr.py b/pypy/rpython/test/test_rstr.py
--- a/pypy/rpython/test/test_rstr.py
+++ b/pypy/rpython/test/test_rstr.py
@@ -372,12 +372,20 @@
             return const('!ab!').lstrip(const('!'))
         def right():
             return const('!ab!').rstrip(const('!'))
+        def empty():
+            return const('    ').strip(' ')
+        def left2():
+            return const('a  ').strip(' ')
         res = self.interpret(both, [])
         assert self.ll_to_string(res) == const('ab')
         res = self.interpret(left, [])
         assert self.ll_to_string(res) == const('ab!')
         res = self.interpret(right, [])
         assert self.ll_to_string(res) == const('!ab')
+        res = self.interpret(empty, [])
+        assert self.ll_to_string(res) == const('')
+        res = self.interpret(left2, [])
+        assert self.ll_to_string(res) == const('a')
 
     def test_upper(self):
         const = self.const

From noreply at buildbot.pypy.org  Fri Oct 28 17:20:52 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 28 Oct 2011 17:20:52 +0200 (CEST)
Subject: [pypy-commit] pypy default: forgotten import
Message-ID: <20111028152052.48937820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: 
Changeset: r48567:8b3dc9ba02ca
Date: 2011-10-28 17:20 +0200
http://bitbucket.org/pypy/pypy/changeset/8b3dc9ba02ca/

Log:	forgotten import

diff --git a/pypy/module/micronumpy/test/test_compile.py b/pypy/module/micronumpy/test/test_compile.py
--- a/pypy/module/micronumpy/test/test_compile.py
+++ b/pypy/module/micronumpy/test/test_compile.py
@@ -1,4 +1,5 @@
 
+import py
 from pypy.module.micronumpy.compile import *
 
 class TestCompiler(object):

From noreply at buildbot.pypy.org  Fri Oct 28 17:44:43 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Fri, 28 Oct 2011 17:44:43 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-complex: merged default
Message-ID: <20111028154443.B5CE9820C2@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: numpy-complex
Changeset: r48568:8f899b94cb87
Date: 2011-10-28 00:11 -0400
http://bitbucket.org/pypy/pypy/changeset/8f899b94cb87/

Log:	merged default

diff too long, truncating to 10000 out of 10937 lines

diff --git a/lib-python/modified-2.7/heapq.py b/lib-python/modified-2.7/heapq.py
new file mode 100644
--- /dev/null
+++ b/lib-python/modified-2.7/heapq.py
@@ -0,0 +1,442 @@
+# -*- coding: latin-1 -*-
+
+"""Heap queue algorithm (a.k.a. priority queue).
+
+Heaps are arrays for which a[k] <= a[2*k+1] and a[k] <= a[2*k+2] for
+all k, counting elements from 0.  For the sake of comparison,
+non-existing elements are considered to be infinite.  The interesting
+property of a heap is that a[0] is always its smallest element.
+
+Usage:
+
+heap = []            # creates an empty heap
+heappush(heap, item) # pushes a new item on the heap
+item = heappop(heap) # pops the smallest item from the heap
+item = heap[0]       # smallest item on the heap without popping it
+heapify(x)           # transforms list into a heap, in-place, in linear time
+item = heapreplace(heap, item) # pops and returns smallest item, and adds
+                               # new item; the heap size is unchanged
+
+Our API differs from textbook heap algorithms as follows:
+
+- We use 0-based indexing.  This makes the relationship between the
+  index for a node and the indexes for its children slightly less
+  obvious, but is more suitable since Python uses 0-based indexing.
+
+- Our heappop() method returns the smallest item, not the largest.
+
+These two make it possible to view the heap as a regular Python list
+without surprises: heap[0] is the smallest item, and heap.sort()
+maintains the heap invariant!
+"""
+
+# Original code by Kevin O'Connor, augmented by Tim Peters and Raymond Hettinger
+
+__about__ = """Heap queues
+
+[explanation by Fran&#65533;ois Pinard]
+
+Heaps are arrays for which a[k] <= a[2*k+1] and a[k] <= a[2*k+2] for
+all k, counting elements from 0.  For the sake of comparison,
+non-existing elements are considered to be infinite.  The interesting
+property of a heap is that a[0] is always its smallest element.
+
+The strange invariant above is meant to be an efficient memory
+representation for a tournament.  The numbers below are `k', not a[k]:
+
+                                   0
+
+                  1                                 2
+
+          3               4                5               6
+
+      7       8       9       10      11      12      13      14
+
+    15 16   17 18   19 20   21 22   23 24   25 26   27 28   29 30
+
+
+In the tree above, each cell `k' is topping `2*k+1' and `2*k+2'.  In
+an usual binary tournament we see in sports, each cell is the winner
+over the two cells it tops, and we can trace the winner down the tree
+to see all opponents s/he had.  However, in many computer applications
+of such tournaments, we do not need to trace the history of a winner.
+To be more memory efficient, when a winner is promoted, we try to
+replace it by something else at a lower level, and the rule becomes
+that a cell and the two cells it tops contain three different items,
+but the top cell "wins" over the two topped cells.
+
+If this heap invariant is protected at all time, index 0 is clearly
+the overall winner.  The simplest algorithmic way to remove it and
+find the "next" winner is to move some loser (let's say cell 30 in the
+diagram above) into the 0 position, and then percolate this new 0 down
+the tree, exchanging values, until the invariant is re-established.
+This is clearly logarithmic on the total number of items in the tree.
+By iterating over all items, you get an O(n ln n) sort.
+
+A nice feature of this sort is that you can efficiently insert new
+items while the sort is going on, provided that the inserted items are
+not "better" than the last 0'th element you extracted.  This is
+especially useful in simulation contexts, where the tree holds all
+incoming events, and the "win" condition means the smallest scheduled
+time.  When an event schedule other events for execution, they are
+scheduled into the future, so they can easily go into the heap.  So, a
+heap is a good structure for implementing schedulers (this is what I
+used for my MIDI sequencer :-).
+
+Various structures for implementing schedulers have been extensively
+studied, and heaps are good for this, as they are reasonably speedy,
+the speed is almost constant, and the worst case is not much different
+than the average case.  However, there are other representations which
+are more efficient overall, yet the worst cases might be terrible.
+
+Heaps are also very useful in big disk sorts.  You most probably all
+know that a big sort implies producing "runs" (which are pre-sorted
+sequences, which size is usually related to the amount of CPU memory),
+followed by a merging passes for these runs, which merging is often
+very cleverly organised[1].  It is very important that the initial
+sort produces the longest runs possible.  Tournaments are a good way
+to that.  If, using all the memory available to hold a tournament, you
+replace and percolate items that happen to fit the current run, you'll
+produce runs which are twice the size of the memory for random input,
+and much better for input fuzzily ordered.
+
+Moreover, if you output the 0'th item on disk and get an input which
+may not fit in the current tournament (because the value "wins" over
+the last output value), it cannot fit in the heap, so the size of the
+heap decreases.  The freed memory could be cleverly reused immediately
+for progressively building a second heap, which grows at exactly the
+same rate the first heap is melting.  When the first heap completely
+vanishes, you switch heaps and start a new run.  Clever and quite
+effective!
+
+In a word, heaps are useful memory structures to know.  I use them in
+a few applications, and I think it is good to keep a `heap' module
+around. :-)
+
+--------------------
+[1] The disk balancing algorithms which are current, nowadays, are
+more annoying than clever, and this is a consequence of the seeking
+capabilities of the disks.  On devices which cannot seek, like big
+tape drives, the story was quite different, and one had to be very
+clever to ensure (far in advance) that each tape movement will be the
+most effective possible (that is, will best participate at
+"progressing" the merge).  Some tapes were even able to read
+backwards, and this was also used to avoid the rewinding time.
+Believe me, real good tape sorts were quite spectacular to watch!
+From all times, sorting has always been a Great Art! :-)
+"""
+
+__all__ = ['heappush', 'heappop', 'heapify', 'heapreplace', 'merge',
+           'nlargest', 'nsmallest', 'heappushpop']
+
+from itertools import islice, repeat, count, imap, izip, tee, chain
+from operator import itemgetter
+import bisect
+
+def heappush(heap, item):
+    """Push item onto heap, maintaining the heap invariant."""
+    heap.append(item)
+    _siftdown(heap, 0, len(heap)-1)
+
+def heappop(heap):
+    """Pop the smallest item off the heap, maintaining the heap invariant."""
+    lastelt = heap.pop()    # raises appropriate IndexError if heap is empty
+    if heap:
+        returnitem = heap[0]
+        heap[0] = lastelt
+        _siftup(heap, 0)
+    else:
+        returnitem = lastelt
+    return returnitem
+
+def heapreplace(heap, item):
+    """Pop and return the current smallest value, and add the new item.
+
+    This is more efficient than heappop() followed by heappush(), and can be
+    more appropriate when using a fixed-size heap.  Note that the value
+    returned may be larger than item!  That constrains reasonable uses of
+    this routine unless written as part of a conditional replacement:
+
+        if item > heap[0]:
+            item = heapreplace(heap, item)
+    """
+    returnitem = heap[0]    # raises appropriate IndexError if heap is empty
+    heap[0] = item
+    _siftup(heap, 0)
+    return returnitem
+
+def heappushpop(heap, item):
+    """Fast version of a heappush followed by a heappop."""
+    if heap and heap[0] < item:
+        item, heap[0] = heap[0], item
+        _siftup(heap, 0)
+    return item
+
+def heapify(x):
+    """Transform list into a heap, in-place, in O(len(heap)) time."""
+    n = len(x)
+    # Transform bottom-up.  The largest index there's any point to looking at
+    # is the largest with a child index in-range, so must have 2*i + 1 < n,
+    # or i < (n-1)/2.  If n is even = 2*j, this is (2*j-1)/2 = j-1/2 so
+    # j-1 is the largest, which is n//2 - 1.  If n is odd = 2*j+1, this is
+    # (2*j+1-1)/2 = j so j-1 is the largest, and that's again n//2-1.
+    for i in reversed(xrange(n//2)):
+        _siftup(x, i)
+
+def nlargest(n, iterable):
+    """Find the n largest elements in a dataset.
+
+    Equivalent to:  sorted(iterable, reverse=True)[:n]
+    """
+    if n < 0: # for consistency with the c impl
+        return []
+    it = iter(iterable)
+    result = list(islice(it, n))
+    if not result:
+        return result
+    heapify(result)
+    _heappushpop = heappushpop
+    for elem in it:
+        _heappushpop(result, elem)
+    result.sort(reverse=True)
+    return result
+
+def nsmallest(n, iterable):
+    """Find the n smallest elements in a dataset.
+
+    Equivalent to:  sorted(iterable)[:n]
+    """
+    if n < 0: # for consistency with the c impl
+        return []
+    if hasattr(iterable, '__len__') and n * 10 <= len(iterable):
+        # For smaller values of n, the bisect method is faster than a minheap.
+        # It is also memory efficient, consuming only n elements of space.
+        it = iter(iterable)
+        result = sorted(islice(it, 0, n))
+        if not result:
+            return result
+        insort = bisect.insort
+        pop = result.pop
+        los = result[-1]    # los --> Largest of the nsmallest
+        for elem in it:
+            if los <= elem:
+                continue
+            insort(result, elem)
+            pop()
+            los = result[-1]
+        return result
+    # An alternative approach manifests the whole iterable in memory but
+    # saves comparisons by heapifying all at once.  Also, saves time
+    # over bisect.insort() which has O(n) data movement time for every
+    # insertion.  Finding the n smallest of an m length iterable requires
+    #    O(m) + O(n log m) comparisons.
+    h = list(iterable)
+    heapify(h)
+    return map(heappop, repeat(h, min(n, len(h))))
+
+# 'heap' is a heap at all indices >= startpos, except possibly for pos.  pos
+# is the index of a leaf with a possibly out-of-order value.  Restore the
+# heap invariant.
+def _siftdown(heap, startpos, pos):
+    newitem = heap[pos]
+    # Follow the path to the root, moving parents down until finding a place
+    # newitem fits.
+    while pos > startpos:
+        parentpos = (pos - 1) >> 1
+        parent = heap[parentpos]
+        if newitem < parent:
+            heap[pos] = parent
+            pos = parentpos
+            continue
+        break
+    heap[pos] = newitem
+
+# The child indices of heap index pos are already heaps, and we want to make
+# a heap at index pos too.  We do this by bubbling the smaller child of
+# pos up (and so on with that child's children, etc) until hitting a leaf,
+# then using _siftdown to move the oddball originally at index pos into place.
+#
+# We *could* break out of the loop as soon as we find a pos where newitem <=
+# both its children, but turns out that's not a good idea, and despite that
+# many books write the algorithm that way.  During a heap pop, the last array
+# element is sifted in, and that tends to be large, so that comparing it
+# against values starting from the root usually doesn't pay (= usually doesn't
+# get us out of the loop early).  See Knuth, Volume 3, where this is
+# explained and quantified in an exercise.
+#
+# Cutting the # of comparisons is important, since these routines have no
+# way to extract "the priority" from an array element, so that intelligence
+# is likely to be hiding in custom __cmp__ methods, or in array elements
+# storing (priority, record) tuples.  Comparisons are thus potentially
+# expensive.
+#
+# On random arrays of length 1000, making this change cut the number of
+# comparisons made by heapify() a little, and those made by exhaustive
+# heappop() a lot, in accord with theory.  Here are typical results from 3
+# runs (3 just to demonstrate how small the variance is):
+#
+# Compares needed by heapify     Compares needed by 1000 heappops
+# --------------------------     --------------------------------
+# 1837 cut to 1663               14996 cut to 8680
+# 1855 cut to 1659               14966 cut to 8678
+# 1847 cut to 1660               15024 cut to 8703
+#
+# Building the heap by using heappush() 1000 times instead required
+# 2198, 2148, and 2219 compares:  heapify() is more efficient, when
+# you can use it.
+#
+# The total compares needed by list.sort() on the same lists were 8627,
+# 8627, and 8632 (this should be compared to the sum of heapify() and
+# heappop() compares):  list.sort() is (unsurprisingly!) more efficient
+# for sorting.
+
+def _siftup(heap, pos):
+    endpos = len(heap)
+    startpos = pos
+    newitem = heap[pos]
+    # Bubble up the smaller child until hitting a leaf.
+    childpos = 2*pos + 1    # leftmost child position
+    while childpos < endpos:
+        # Set childpos to index of smaller child.
+        rightpos = childpos + 1
+        if rightpos < endpos and not heap[childpos] < heap[rightpos]:
+            childpos = rightpos
+        # Move the smaller child up.
+        heap[pos] = heap[childpos]
+        pos = childpos
+        childpos = 2*pos + 1
+    # The leaf at pos is empty now.  Put newitem there, and bubble it up
+    # to its final resting place (by sifting its parents down).
+    heap[pos] = newitem
+    _siftdown(heap, startpos, pos)
+
+# If available, use C implementation
+try:
+    from _heapq import *
+except ImportError:
+    pass
+
+def merge(*iterables):
+    '''Merge multiple sorted inputs into a single sorted output.
+
+    Similar to sorted(itertools.chain(*iterables)) but returns a generator,
+    does not pull the data into memory all at once, and assumes that each of
+    the input streams is already sorted (smallest to largest).
+
+    >>> list(merge([1,3,5,7], [0,2,4,8], [5,10,15,20], [], [25]))
+    [0, 1, 2, 3, 4, 5, 5, 7, 8, 10, 15, 20, 25]
+
+    '''
+    _heappop, _heapreplace, _StopIteration = heappop, heapreplace, StopIteration
+
+    h = []
+    h_append = h.append
+    for itnum, it in enumerate(map(iter, iterables)):
+        try:
+            next = it.next
+            h_append([next(), itnum, next])
+        except _StopIteration:
+            pass
+    heapify(h)
+
+    while 1:
+        try:
+            while 1:
+                v, itnum, next = s = h[0]   # raises IndexError when h is empty
+                yield v
+                s[0] = next()               # raises StopIteration when exhausted
+                _heapreplace(h, s)          # restore heap condition
+        except _StopIteration:
+            _heappop(h)                     # remove empty iterator
+        except IndexError:
+            return
+
+# Extend the implementations of nsmallest and nlargest to use a key= argument
+_nsmallest = nsmallest
+def nsmallest(n, iterable, key=None):
+    """Find the n smallest elements in a dataset.
+
+    Equivalent to:  sorted(iterable, key=key)[:n]
+    """
+    # Short-cut for n==1 is to use min() when len(iterable)>0
+    if n == 1:
+        it = iter(iterable)
+        head = list(islice(it, 1))
+        if not head:
+            return []
+        if key is None:
+            return [min(chain(head, it))]
+        return [min(chain(head, it), key=key)]
+
+    # When n>=size, it's faster to use sort()
+    try:
+        size = len(iterable)
+    except (TypeError, AttributeError):
+        pass
+    else:
+        if n >= size:
+            return sorted(iterable, key=key)[:n]
+
+    # When key is none, use simpler decoration
+    if key is None:
+        it = izip(iterable, count())                        # decorate
+        result = _nsmallest(n, it)
+        return map(itemgetter(0), result)                   # undecorate
+
+    # General case, slowest method
+    in1, in2 = tee(iterable)
+    it = izip(imap(key, in1), count(), in2)                 # decorate
+    result = _nsmallest(n, it)
+    return map(itemgetter(2), result)                       # undecorate
+
+_nlargest = nlargest
+def nlargest(n, iterable, key=None):
+    """Find the n largest elements in a dataset.
+
+    Equivalent to:  sorted(iterable, key=key, reverse=True)[:n]
+    """
+
+    # Short-cut for n==1 is to use max() when len(iterable)>0
+    if n == 1:
+        it = iter(iterable)
+        head = list(islice(it, 1))
+        if not head:
+            return []
+        if key is None:
+            return [max(chain(head, it))]
+        return [max(chain(head, it), key=key)]
+
+    # When n>=size, it's faster to use sort()
+    try:
+        size = len(iterable)
+    except (TypeError, AttributeError):
+        pass
+    else:
+        if n >= size:
+            return sorted(iterable, key=key, reverse=True)[:n]
+
+    # When key is none, use simpler decoration
+    if key is None:
+        it = izip(iterable, count(0,-1))                    # decorate
+        result = _nlargest(n, it)
+        return map(itemgetter(0), result)                   # undecorate
+
+    # General case, slowest method
+    in1, in2 = tee(iterable)
+    it = izip(imap(key, in1), count(0,-1), in2)             # decorate
+    result = _nlargest(n, it)
+    return map(itemgetter(2), result)                       # undecorate
+
+if __name__ == "__main__":
+    # Simple sanity test
+    heap = []
+    data = [1, 3, 5, 7, 9, 2, 4, 6, 8, 0]
+    for item in data:
+        heappush(heap, item)
+    sort = []
+    while heap:
+        sort.append(heappop(heap))
+    print sort
+
+    import doctest
+    doctest.testmod()
diff --git a/lib-python/modified-2.7/json/encoder.py b/lib-python/modified-2.7/json/encoder.py
--- a/lib-python/modified-2.7/json/encoder.py
+++ b/lib-python/modified-2.7/json/encoder.py
@@ -2,14 +2,7 @@
 """
 import re
 
-try:
-    from _json import encode_basestring_ascii as c_encode_basestring_ascii
-except ImportError:
-    c_encode_basestring_ascii = None
-try:
-    from _json import make_encoder as c_make_encoder
-except ImportError:
-    c_make_encoder = None
+from __pypy__.builders import StringBuilder, UnicodeBuilder
 
 ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]')
 ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])')
@@ -24,8 +17,7 @@
     '\t': '\\t',
 }
 for i in range(0x20):
-    ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i))
-    #ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,))
+    ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,))
 
 # Assume this produces an infinity on all machines (probably not guaranteed)
 INFINITY = float('1e66666')
@@ -37,10 +29,9 @@
     """
     def replace(match):
         return ESCAPE_DCT[match.group(0)]
-    return '"' + ESCAPE.sub(replace, s) + '"'
+    return ESCAPE.sub(replace, s)
 
-
-def py_encode_basestring_ascii(s):
+def encode_basestring_ascii(s):
     """Return an ASCII-only JSON representation of a Python string
 
     """
@@ -53,20 +44,18 @@
         except KeyError:
             n = ord(s)
             if n < 0x10000:
-                return '\\u{0:04x}'.format(n)
-                #return '\\u%04x' % (n,)
+                return '\\u%04x' % (n,)
             else:
                 # surrogate pair
                 n -= 0x10000
                 s1 = 0xd800 | ((n >> 10) & 0x3ff)
                 s2 = 0xdc00 | (n & 0x3ff)
-                return '\\u{0:04x}\\u{1:04x}'.format(s1, s2)
-                #return '\\u%04x\\u%04x' % (s1, s2)
-    return '"' + str(ESCAPE_ASCII.sub(replace, s)) + '"'
-
-
-encode_basestring_ascii = (
-    c_encode_basestring_ascii or py_encode_basestring_ascii)
+                return '\\u%04x\\u%04x' % (s1, s2)
+    if ESCAPE_ASCII.search(s):
+        return str(ESCAPE_ASCII.sub(replace, s))
+    return s
+py_encode_basestring_ascii = lambda s: '"' + encode_basestring_ascii(s) + '"'
+c_encode_basestring_ascii = None
 
 class JSONEncoder(object):
     """Extensible JSON <http://json.org> encoder for Python data structures.
@@ -147,6 +136,17 @@
 
         self.skipkeys = skipkeys
         self.ensure_ascii = ensure_ascii
+        if ensure_ascii:
+            self.encoder = encode_basestring_ascii
+        else:
+            self.encoder = encode_basestring
+        if encoding != 'utf-8':
+            orig_encoder = self.encoder
+            def encoder(o):
+                if isinstance(o, str):
+                    o = o.decode(encoding)
+                return orig_encoder(o)
+            self.encoder = encoder
         self.check_circular = check_circular
         self.allow_nan = allow_nan
         self.sort_keys = sort_keys
@@ -184,24 +184,126 @@
         '{"foo": ["bar", "baz"]}'
 
         """
-        # This is for extremely simple cases and benchmarks.
+        if self.check_circular:
+            markers = {}
+        else:
+            markers = None
+        if self.ensure_ascii:
+            builder = StringBuilder()
+        else:
+            builder = UnicodeBuilder()
+        self._encode(o, markers, builder, 0)
+        return builder.build()
+
+    def _emit_indent(self, builder, _current_indent_level):
+        if self.indent is not None:
+            _current_indent_level += 1
+            newline_indent = '\n' + (' ' * (self.indent *
+                                            _current_indent_level))
+            separator = self.item_separator + newline_indent
+            builder.append(newline_indent)
+        else:
+            separator = self.item_separator
+        return separator, _current_indent_level
+
+    def _emit_unindent(self, builder, _current_indent_level):
+        if self.indent is not None:
+            builder.append('\n')
+            builder.append(' ' * (self.indent * (_current_indent_level - 1)))
+
+    def _encode(self, o, markers, builder, _current_indent_level):
         if isinstance(o, basestring):
-            if isinstance(o, str):
-                _encoding = self.encoding
-                if (_encoding is not None
-                        and not (_encoding == 'utf-8')):
-                    o = o.decode(_encoding)
-            if self.ensure_ascii:
-                return encode_basestring_ascii(o)
+            builder.append('"')
+            builder.append(self.encoder(o))
+            builder.append('"')
+        elif o is None:
+            builder.append('null')
+        elif o is True:
+            builder.append('true')
+        elif o is False:
+            builder.append('false')
+        elif isinstance(o, (int, long)):
+            builder.append(str(o))
+        elif isinstance(o, float):
+            builder.append(self._floatstr(o))
+        elif isinstance(o, (list, tuple)):
+            if not o:
+                builder.append('[]')
+                return
+            self._encode_list(o, markers, builder, _current_indent_level)
+        elif isinstance(o, dict):
+            if not o:
+                builder.append('{}')
+                return
+            self._encode_dict(o, markers, builder, _current_indent_level)
+        else:
+            self._mark_markers(markers, o)
+            res = self.default(o)
+            self._encode(res, markers, builder, _current_indent_level)
+            self._remove_markers(markers, o)
+            return res
+
+    def _encode_list(self, l, markers, builder, _current_indent_level):
+        self._mark_markers(markers, l)
+        builder.append('[')
+        first = True
+        separator, _current_indent_level = self._emit_indent(builder,
+                                                      _current_indent_level)
+        for elem in l:
+            if first:
+                first = False
             else:
-                return encode_basestring(o)
-        # This doesn't pass the iterator directly to ''.join() because the
-        # exceptions aren't as detailed.  The list call should be roughly
-        # equivalent to the PySequence_Fast that ''.join() would do.
-        chunks = self.iterencode(o, _one_shot=True)
-        if not isinstance(chunks, (list, tuple)):
-            chunks = list(chunks)
-        return ''.join(chunks)
+                builder.append(separator)
+            self._encode(elem, markers, builder, _current_indent_level)
+            del elem # XXX grumble
+        self._emit_unindent(builder, _current_indent_level)
+        builder.append(']')
+        self._remove_markers(markers, l)
+
+    def _encode_dict(self, d, markers, builder, _current_indent_level):
+        self._mark_markers(markers, d)
+        first = True
+        builder.append('{')
+        separator, _current_indent_level = self._emit_indent(builder,
+                                                         _current_indent_level)
+        if self.sort_keys:
+            items = sorted(d.items(), key=lambda kv: kv[0])
+        else:
+            items = d.iteritems()
+
+        for key, v in items:
+            if first:
+                first = False
+            else:
+                builder.append(separator)
+            if isinstance(key, basestring):
+                pass
+            # JavaScript is weakly typed for these, so it makes sense to
+            # also allow them.  Many encoders seem to do something like this.
+            elif isinstance(key, float):
+                key = self._floatstr(key)
+            elif key is True:
+                key = 'true'
+            elif key is False:
+                key = 'false'
+            elif key is None:
+                key = 'null'
+            elif isinstance(key, (int, long)):
+                key = str(key)
+            elif self.skipkeys:
+                continue
+            else:
+                raise TypeError("key " + repr(key) + " is not a string")
+            builder.append('"')
+            builder.append(self.encoder(key))
+            builder.append('"')
+            builder.append(self.key_separator)
+            self._encode(v, markers, builder, _current_indent_level)
+            del key
+            del v # XXX grumble
+        self._emit_unindent(builder, _current_indent_level)
+        builder.append('}')
+        self._remove_markers(markers, d)
 
     def iterencode(self, o, _one_shot=False):
         """Encode the given object and yield each string
@@ -217,86 +319,54 @@
             markers = {}
         else:
             markers = None
-        if self.ensure_ascii:
-            _encoder = encode_basestring_ascii
+        return self._iterencode(o, markers, 0)
+
+    def _floatstr(self, o):
+        # Check for specials.  Note that this type of test is processor
+        # and/or platform-specific, so do tests which don't depend on the
+        # internals.
+
+        if o != o:
+            text = 'NaN'
+        elif o == INFINITY:
+            text = 'Infinity'
+        elif o == -INFINITY:
+            text = '-Infinity'
         else:
-            _encoder = encode_basestring
-        if self.encoding != 'utf-8':
-            def _encoder(o, _orig_encoder=_encoder, _encoding=self.encoding):
-                if isinstance(o, str):
-                    o = o.decode(_encoding)
-                return _orig_encoder(o)
+            return FLOAT_REPR(o)
 
-        def floatstr(o, allow_nan=self.allow_nan,
-                _repr=FLOAT_REPR, _inf=INFINITY, _neginf=-INFINITY):
-            # Check for specials.  Note that this type of test is processor
-            # and/or platform-specific, so do tests which don't depend on the
-            # internals.
+        if not self.allow_nan:
+            raise ValueError(
+                "Out of range float values are not JSON compliant: " +
+                repr(o))
 
-            if o != o:
-                text = 'NaN'
-            elif o == _inf:
-                text = 'Infinity'
-            elif o == _neginf:
-                text = '-Infinity'
-            else:
-                return _repr(o)
+        return text
 
-            if not allow_nan:
-                raise ValueError(
-                    "Out of range float values are not JSON compliant: " +
-                    repr(o))
+    def _mark_markers(self, markers, o):
+        if markers is not None:
+            if id(o) in markers:
+                raise ValueError("Circular reference detected")
+            markers[id(o)] = None
 
-            return text
+    def _remove_markers(self, markers, o):
+        if markers is not None:
+            del markers[id(o)]
 
-
-        if (_one_shot and c_make_encoder is not None
-                and not self.indent and not self.sort_keys):
-            _iterencode = c_make_encoder(
-                markers, self.default, _encoder, self.indent,
-                self.key_separator, self.item_separator, self.sort_keys,
-                self.skipkeys, self.allow_nan)
-        else:
-            _iterencode = _make_iterencode(
-                markers, self.default, _encoder, self.indent, floatstr,
-                self.key_separator, self.item_separator, self.sort_keys,
-                self.skipkeys, _one_shot)
-        return _iterencode(o, 0)
-
-def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,
-        _key_separator, _item_separator, _sort_keys, _skipkeys, _one_shot,
-        ## HACK: hand-optimized bytecode; turn globals into locals
-        ValueError=ValueError,
-        basestring=basestring,
-        dict=dict,
-        float=float,
-        id=id,
-        int=int,
-        isinstance=isinstance,
-        list=list,
-        long=long,
-        str=str,
-        tuple=tuple,
-    ):
-
-    def _iterencode_list(lst, _current_indent_level):
+    def _iterencode_list(self, lst, markers, _current_indent_level):
         if not lst:
             yield '[]'
             return
-        if markers is not None:
-            markerid = id(lst)
-            if markerid in markers:
-                raise ValueError("Circular reference detected")
-            markers[markerid] = lst
+        self._mark_markers(markers, lst)
         buf = '['
-        if _indent is not None:
+        if self.indent is not None:
             _current_indent_level += 1
-            newline_indent = '\n' + (' ' * (_indent * _current_indent_level))
-            separator = _item_separator + newline_indent
+            newline_indent = '\n' + (' ' * (self.indent *
+                                            _current_indent_level))
+            separator = self.item_separator + newline_indent
             buf += newline_indent
         else:
             newline_indent = None
-            separator = _item_separator
+            separator = self.item_separator
         first = True
         for value in lst:
             if first:
@@ -304,7 +374,7 @@
             else:
                 buf = separator
             if isinstance(value, basestring):
-                yield buf + _encoder(value)
+                yield buf + '"' + self.encoder(value) + '"'
             elif value is None:
                 yield buf + 'null'
             elif value is True:
@@ -314,44 +384,43 @@
             elif isinstance(value, (int, long)):
                 yield buf + str(value)
             elif isinstance(value, float):
-                yield buf + _floatstr(value)
+                yield buf + self._floatstr(value)
             else:
                 yield buf
                 if isinstance(value, (list, tuple)):
-                    chunks = _iterencode_list(value, _current_indent_level)
+                    chunks = self._iterencode_list(value, markers,
+                                                   _current_indent_level)
                 elif isinstance(value, dict):
-                    chunks = _iterencode_dict(value, _current_indent_level)
+                    chunks = self._iterencode_dict(value, markers,
+                                                   _current_indent_level)
                 else:
-                    chunks = _iterencode(value, _current_indent_level)
+                    chunks = self._iterencode(value, markers,
+                                              _current_indent_level)
                 for chunk in chunks:
                     yield chunk
         if newline_indent is not None:
             _current_indent_level -= 1
-            yield '\n' + (' ' * (_indent * _current_indent_level))
+            yield '\n' + (' ' * (self.indent * _current_indent_level))
         yield ']'
-        if markers is not None:
-            del markers[markerid]
+        self._remove_markers(markers, lst)
 
-    def _iterencode_dict(dct, _current_indent_level):
+    def _iterencode_dict(self, dct, markers, _current_indent_level):
         if not dct:
             yield '{}'
             return
-        if markers is not None:
-            markerid = id(dct)
-            if markerid in markers:
-                raise ValueError("Circular reference detected")
-            markers[markerid] = dct
+        self._mark_markers(markers, dct)
         yield '{'
-        if _indent is not None:
+        if self.indent is not None:
             _current_indent_level += 1
-            newline_indent = '\n' + (' ' * (_indent * _current_indent_level))
-            item_separator = _item_separator + newline_indent
+            newline_indent = '\n' + (' ' * (self.indent *
+                                            _current_indent_level))
+            item_separator = self.item_separator + newline_indent
             yield newline_indent
         else:
             newline_indent = None
-            item_separator = _item_separator
+            item_separator = self.item_separator
         first = True
-        if _sort_keys:
+        if self.sort_keys:
             items = sorted(dct.items(), key=lambda kv: kv[0])
         else:
             items = dct.iteritems()
@@ -361,7 +430,7 @@
             # JavaScript is weakly typed for these, so it makes sense to
             # also allow them.  Many encoders seem to do something like this.
             elif isinstance(key, float):
-                key = _floatstr(key)
+                key = self._floatstr(key)
             elif key is True:
                 key = 'true'
             elif key is False:
@@ -370,7 +439,7 @@
                 key = 'null'
             elif isinstance(key, (int, long)):
                 key = str(key)
-            elif _skipkeys:
+            elif self.skipkeys:
                 continue
             else:
                 raise TypeError("key " + repr(key) + " is not a string")
@@ -378,10 +447,10 @@
                 first = False
             else:
                 yield item_separator
-            yield _encoder(key)
-            yield _key_separator
+            yield '"' + self.encoder(key) + '"'
+            yield self.key_separator
             if isinstance(value, basestring):
-                yield _encoder(value)
+                yield '"' + self.encoder(value) + '"'
             elif value is None:
                 yield 'null'
             elif value is True:
@@ -391,26 +460,28 @@
             elif isinstance(value, (int, long)):
                 yield str(value)
             elif isinstance(value, float):
-                yield _floatstr(value)
+                yield self._floatstr(value)
             else:
                 if isinstance(value, (list, tuple)):
-                    chunks = _iterencode_list(value, _current_indent_level)
+                    chunks = self._iterencode_list(value, markers,
+                                                   _current_indent_level)
                 elif isinstance(value, dict):
-                    chunks = _iterencode_dict(value, _current_indent_level)
+                    chunks = self._iterencode_dict(value, markers,
+                                                   _current_indent_level)
                 else:
-                    chunks = _iterencode(value, _current_indent_level)
+                    chunks = self._iterencode(value, markers,
+                                              _current_indent_level)
                 for chunk in chunks:
                     yield chunk
         if newline_indent is not None:
             _current_indent_level -= 1
-            yield '\n' + (' ' * (_indent * _current_indent_level))
+            yield '\n' + (' ' * (self.indent * _current_indent_level))
         yield '}'
-        if markers is not None:
-            del markers[markerid]
+        self._remove_markers(markers, dct)
 
-    def _iterencode(o, _current_indent_level):
+    def _iterencode(self, o, markers, _current_indent_level):
         if isinstance(o, basestring):
-            yield _encoder(o)
+            yield '"' + self.encoder(o) + '"'
         elif o is None:
             yield 'null'
         elif o is True:
@@ -420,23 +491,19 @@
         elif isinstance(o, (int, long)):
             yield str(o)
         elif isinstance(o, float):
-            yield _floatstr(o)
+            yield self._floatstr(o)
         elif isinstance(o, (list, tuple)):
-            for chunk in _iterencode_list(o, _current_indent_level):
+            for chunk in self._iterencode_list(o, markers,
+                                               _current_indent_level):
                 yield chunk
         elif isinstance(o, dict):
-            for chunk in _iterencode_dict(o, _current_indent_level):
+            for chunk in self._iterencode_dict(o, markers,
+                                               _current_indent_level):
                 yield chunk
         else:
-            if markers is not None:
-                markerid = id(o)
-                if markerid in markers:
-                    raise ValueError("Circular reference detected")
-                markers[markerid] = o
-            o = _default(o)
-            for chunk in _iterencode(o, _current_indent_level):
+            self._mark_markers(markers, o)
+            obj = self.default(o)
+            for chunk in self._iterencode(obj, markers,
+                                          _current_indent_level):
                 yield chunk
-            if markers is not None:
-                del markers[markerid]
-
-    return _iterencode
+            self._remove_markers(markers, o)
diff --git a/lib-python/modified-2.7/json/tests/test_unicode.py b/lib-python/modified-2.7/json/tests/test_unicode.py
--- a/lib-python/modified-2.7/json/tests/test_unicode.py
+++ b/lib-python/modified-2.7/json/tests/test_unicode.py
@@ -80,3 +80,9 @@
         self.assertEqual(type(json.loads(u'["a"]')[0]), unicode)
         # Issue 10038.
         self.assertEqual(type(json.loads('"foo"')), unicode)
+
+    def test_encode_not_utf_8(self):
+        self.assertEqual(json.dumps('\xb1\xe6', encoding='iso8859-2'),
+                         '"\\u0105\\u0107"')
+        self.assertEqual(json.dumps(['\xb1\xe6'], encoding='iso8859-2'),
+                         '["\\u0105\\u0107"]')
diff --git a/lib-python/modified-2.7/test/test_heapq.py b/lib-python/modified-2.7/test/test_heapq.py
--- a/lib-python/modified-2.7/test/test_heapq.py
+++ b/lib-python/modified-2.7/test/test_heapq.py
@@ -186,6 +186,11 @@
         self.assertFalse(sys.modules['heapq'] is self.module)
         self.assertTrue(hasattr(self.module.heapify, 'func_code'))
 
+    def test_islice_protection(self):
+        m = self.module
+        self.assertFalse(m.nsmallest(-1, [1]))
+        self.assertFalse(m.nlargest(-1, [1]))
+
 
 class TestHeapC(TestHeap):
     module = c_heapq
diff --git a/lib-python/modified-2.7/urllib2.py b/lib-python/modified-2.7/urllib2.py
--- a/lib-python/modified-2.7/urllib2.py
+++ b/lib-python/modified-2.7/urllib2.py
@@ -395,11 +395,7 @@
         meth_name = protocol+"_response"
         for processor in self.process_response.get(protocol, []):
             meth = getattr(processor, meth_name)
-            try:
-                response = meth(req, response)
-            except:
-                response.close()
-                raise
+            response = meth(req, response)
 
         return response
 
diff --git a/lib_pypy/pyrepl/readline.py b/lib_pypy/pyrepl/readline.py
--- a/lib_pypy/pyrepl/readline.py
+++ b/lib_pypy/pyrepl/readline.py
@@ -395,9 +395,21 @@
     _wrapper.f_in = f_in
     _wrapper.f_out = f_out
 
-    if hasattr(sys, '__raw_input__'):    # PyPy
-        _old_raw_input = sys.__raw_input__
+    if '__pypy__' in sys.builtin_module_names:    # PyPy
+
+        def _old_raw_input(prompt=''):
+            # sys.__raw_input__() is only called when stdin and stdout are
+            # as expected and are ttys.  If it is the case, then get_reader()
+            # should not really fail in _wrapper.raw_input().  If it still
+            # does, then we will just cancel the redirection and call again
+            # the built-in raw_input().
+            try:
+                del sys.__raw_input__
+            except AttributeError:
+                pass
+            return raw_input(prompt)
         sys.__raw_input__ = _wrapper.raw_input
+
     else:
         # this is not really what readline.c does.  Better than nothing I guess
         import __builtin__
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -72,6 +72,7 @@
     del working_modules['fcntl']  # LOCK_NB not defined
     del working_modules["_minimal_curses"]
     del working_modules["termios"]
+    del working_modules["_multiprocessing"]   # depends on rctime
 
 
 
diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -17,6 +17,12 @@
 projects, or anything else in PyPy, pop up on IRC or write to us on the
 `mailing list`_.
 
+Make big integers faster
+-------------------------
+
+PyPy's implementation of the Python ``long`` type is slower than CPython's.
+Find out why and optimize them.
+
 Numpy improvements
 ------------------
 
diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -2,7 +2,7 @@
 from pypy.interpreter.baseobjspace import Wrappable
 from pypy.interpreter import typedef
 from pypy.interpreter.gateway import interp2app
-from pypy.interpreter.error import OperationError
+from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.tool.pairtype import extendabletype
 from pypy.tool.sourcetools import func_with_new_name
@@ -2925,14 +2925,13 @@
 def Module_get_body(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -2968,14 +2967,13 @@
 def Interactive_get_body(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3015,8 +3013,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     return space.wrap(w_self.body)
 
 def Expression_set_body(space, w_self, w_new_value):
@@ -3057,14 +3054,13 @@
 def Suite_get_body(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3104,8 +3100,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._lineno_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'lineno'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lineno')
     return space.wrap(w_self.lineno)
 
 def stmt_set_lineno(space, w_self, w_new_value):
@@ -3126,8 +3121,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._col_offset_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'col_offset'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'col_offset')
     return space.wrap(w_self.col_offset)
 
 def stmt_set_col_offset(space, w_self, w_new_value):
@@ -3157,8 +3151,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'name'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name')
     return space.wrap(w_self.name)
 
 def FunctionDef_set_name(space, w_self, w_new_value):
@@ -3179,8 +3172,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'args'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args')
     return space.wrap(w_self.args)
 
 def FunctionDef_set_args(space, w_self, w_new_value):
@@ -3197,14 +3189,13 @@
 def FunctionDef_get_body(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3215,14 +3206,13 @@
 def FunctionDef_get_decorator_list(space, w_self):
     if not w_self.initialization_state & 8:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'decorator_list'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'decorator_list')
     if w_self.w_decorator_list is None:
         if w_self.decorator_list is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.decorator_list]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_decorator_list = w_list
     return w_self.w_decorator_list
 
@@ -3266,8 +3256,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'name'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name')
     return space.wrap(w_self.name)
 
 def ClassDef_set_name(space, w_self, w_new_value):
@@ -3284,14 +3273,13 @@
 def ClassDef_get_bases(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'bases'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'bases')
     if w_self.w_bases is None:
         if w_self.bases is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.bases]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_bases = w_list
     return w_self.w_bases
 
@@ -3302,14 +3290,13 @@
 def ClassDef_get_body(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3320,14 +3307,13 @@
 def ClassDef_get_decorator_list(space, w_self):
     if not w_self.initialization_state & 8:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'decorator_list'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'decorator_list')
     if w_self.w_decorator_list is None:
         if w_self.decorator_list is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.decorator_list]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_decorator_list = w_list
     return w_self.w_decorator_list
 
@@ -3372,8 +3358,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Return_set_value(space, w_self, w_new_value):
@@ -3414,14 +3399,13 @@
 def Delete_get_targets(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'targets'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'targets')
     if w_self.w_targets is None:
         if w_self.targets is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.targets]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_targets = w_list
     return w_self.w_targets
 
@@ -3457,14 +3441,13 @@
 def Assign_get_targets(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'targets'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'targets')
     if w_self.w_targets is None:
         if w_self.targets is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.targets]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_targets = w_list
     return w_self.w_targets
 
@@ -3479,8 +3462,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Assign_set_value(space, w_self, w_new_value):
@@ -3527,8 +3509,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'target'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'target')
     return space.wrap(w_self.target)
 
 def AugAssign_set_target(space, w_self, w_new_value):
@@ -3549,8 +3530,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'op'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op')
     return operator_to_class[w_self.op - 1]()
 
 def AugAssign_set_op(space, w_self, w_new_value):
@@ -3573,8 +3553,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def AugAssign_set_value(space, w_self, w_new_value):
@@ -3621,8 +3600,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'dest'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'dest')
     return space.wrap(w_self.dest)
 
 def Print_set_dest(space, w_self, w_new_value):
@@ -3639,14 +3617,13 @@
 def Print_get_values(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'values'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'values')
     if w_self.w_values is None:
         if w_self.values is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.values]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_values = w_list
     return w_self.w_values
 
@@ -3661,8 +3638,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'nl'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'nl')
     return space.wrap(w_self.nl)
 
 def Print_set_nl(space, w_self, w_new_value):
@@ -3710,8 +3686,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'target'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'target')
     return space.wrap(w_self.target)
 
 def For_set_target(space, w_self, w_new_value):
@@ -3732,8 +3707,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'iter'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'iter')
     return space.wrap(w_self.iter)
 
 def For_set_iter(space, w_self, w_new_value):
@@ -3750,14 +3724,13 @@
 def For_get_body(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3768,14 +3741,13 @@
 def For_get_orelse(space, w_self):
     if not w_self.initialization_state & 8:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'orelse'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
     if w_self.w_orelse is None:
         if w_self.orelse is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.orelse]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_orelse = w_list
     return w_self.w_orelse
 
@@ -3819,8 +3791,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'test'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test')
     return space.wrap(w_self.test)
 
 def While_set_test(space, w_self, w_new_value):
@@ -3837,14 +3808,13 @@
 def While_get_body(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3855,14 +3825,13 @@
 def While_get_orelse(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'orelse'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
     if w_self.w_orelse is None:
         if w_self.orelse is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.orelse]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_orelse = w_list
     return w_self.w_orelse
 
@@ -3905,8 +3874,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'test'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test')
     return space.wrap(w_self.test)
 
 def If_set_test(space, w_self, w_new_value):
@@ -3923,14 +3891,13 @@
 def If_get_body(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3941,14 +3908,13 @@
 def If_get_orelse(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'orelse'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
     if w_self.w_orelse is None:
         if w_self.orelse is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.orelse]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_orelse = w_list
     return w_self.w_orelse
 
@@ -3991,8 +3957,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'context_expr'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'context_expr')
     return space.wrap(w_self.context_expr)
 
 def With_set_context_expr(space, w_self, w_new_value):
@@ -4013,8 +3978,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'optional_vars'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'optional_vars')
     return space.wrap(w_self.optional_vars)
 
 def With_set_optional_vars(space, w_self, w_new_value):
@@ -4031,14 +3995,13 @@
 def With_get_body(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -4080,8 +4043,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'type'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'type')
     return space.wrap(w_self.type)
 
 def Raise_set_type(space, w_self, w_new_value):
@@ -4102,8 +4064,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'inst'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'inst')
     return space.wrap(w_self.inst)
 
 def Raise_set_inst(space, w_self, w_new_value):
@@ -4124,8 +4085,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'tback'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'tback')
     return space.wrap(w_self.tback)
 
 def Raise_set_tback(space, w_self, w_new_value):
@@ -4168,14 +4128,13 @@
 def TryExcept_get_body(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -4186,14 +4145,13 @@
 def TryExcept_get_handlers(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'handlers'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'handlers')
     if w_self.w_handlers is None:
         if w_self.handlers is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.handlers]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_handlers = w_list
     return w_self.w_handlers
 
@@ -4204,14 +4162,13 @@
 def TryExcept_get_orelse(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'orelse'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
     if w_self.w_orelse is None:
         if w_self.orelse is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.orelse]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_orelse = w_list
     return w_self.w_orelse
 
@@ -4251,14 +4208,13 @@
 def TryFinally_get_body(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -4269,14 +4225,13 @@
 def TryFinally_get_finalbody(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'finalbody'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'finalbody')
     if w_self.w_finalbody is None:
         if w_self.finalbody is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.finalbody]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_finalbody = w_list
     return w_self.w_finalbody
 
@@ -4318,8 +4273,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'test'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test')
     return space.wrap(w_self.test)
 
 def Assert_set_test(space, w_self, w_new_value):
@@ -4340,8 +4294,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'msg'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'msg')
     return space.wrap(w_self.msg)
 
 def Assert_set_msg(space, w_self, w_new_value):
@@ -4383,14 +4336,13 @@
 def Import_get_names(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'names'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'names')
     if w_self.w_names is None:
         if w_self.names is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.names]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_names = w_list
     return w_self.w_names
 
@@ -4430,8 +4382,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'module'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'module')
     return space.wrap(w_self.module)
 
 def ImportFrom_set_module(space, w_self, w_new_value):
@@ -4451,14 +4402,13 @@
 def ImportFrom_get_names(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'names'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'names')
     if w_self.w_names is None:
         if w_self.names is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.names]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_names = w_list
     return w_self.w_names
 
@@ -4473,8 +4423,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'level'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'level')
     return space.wrap(w_self.level)
 
 def ImportFrom_set_level(space, w_self, w_new_value):
@@ -4522,8 +4471,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     return space.wrap(w_self.body)
 
 def Exec_set_body(space, w_self, w_new_value):
@@ -4544,8 +4492,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'globals'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'globals')
     return space.wrap(w_self.globals)
 
 def Exec_set_globals(space, w_self, w_new_value):
@@ -4566,8 +4513,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'locals'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'locals')
     return space.wrap(w_self.locals)
 
 def Exec_set_locals(space, w_self, w_new_value):
@@ -4610,14 +4556,13 @@
 def Global_get_names(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'names'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'names')
     if w_self.w_names is None:
         if w_self.names is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.names]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_names = w_list
     return w_self.w_names
 
@@ -4657,8 +4602,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Expr_set_value(space, w_self, w_new_value):
@@ -4754,8 +4698,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._lineno_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'lineno'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lineno')
     return space.wrap(w_self.lineno)
 
 def expr_set_lineno(space, w_self, w_new_value):
@@ -4776,8 +4719,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._col_offset_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'col_offset'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'col_offset')
     return space.wrap(w_self.col_offset)
 
 def expr_set_col_offset(space, w_self, w_new_value):
@@ -4807,8 +4749,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'op'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op')
     return boolop_to_class[w_self.op - 1]()
 
 def BoolOp_set_op(space, w_self, w_new_value):
@@ -4827,14 +4768,13 @@
 def BoolOp_get_values(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'values'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'values')
     if w_self.w_values is None:
         if w_self.values is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.values]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_values = w_list
     return w_self.w_values
 
@@ -4875,8 +4815,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'left'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'left')
     return space.wrap(w_self.left)
 
 def BinOp_set_left(space, w_self, w_new_value):
@@ -4897,8 +4836,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'op'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op')
     return operator_to_class[w_self.op - 1]()
 
 def BinOp_set_op(space, w_self, w_new_value):
@@ -4921,8 +4859,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'right'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'right')
     return space.wrap(w_self.right)
 
 def BinOp_set_right(space, w_self, w_new_value):
@@ -4969,8 +4906,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'op'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op')
     return unaryop_to_class[w_self.op - 1]()
 
 def UnaryOp_set_op(space, w_self, w_new_value):
@@ -4993,8 +4929,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'operand'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'operand')
     return space.wrap(w_self.operand)
 
 def UnaryOp_set_operand(space, w_self, w_new_value):
@@ -5040,8 +4975,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'args'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args')
     return space.wrap(w_self.args)
 
 def Lambda_set_args(space, w_self, w_new_value):
@@ -5062,8 +4996,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     return space.wrap(w_self.body)
 
 def Lambda_set_body(space, w_self, w_new_value):
@@ -5109,8 +5042,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'test'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test')
     return space.wrap(w_self.test)
 
 def IfExp_set_test(space, w_self, w_new_value):
@@ -5131,8 +5063,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     return space.wrap(w_self.body)
 
 def IfExp_set_body(space, w_self, w_new_value):
@@ -5153,8 +5084,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'orelse'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
     return space.wrap(w_self.orelse)
 
 def IfExp_set_orelse(space, w_self, w_new_value):
@@ -5197,14 +5127,13 @@
 def Dict_get_keys(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'keys'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'keys')
     if w_self.w_keys is None:
         if w_self.keys is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.keys]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_keys = w_list
     return w_self.w_keys
 
@@ -5215,14 +5144,13 @@
 def Dict_get_values(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'values'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'values')
     if w_self.w_values is None:
         if w_self.values is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.values]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_values = w_list
     return w_self.w_values
 
@@ -5260,14 +5188,13 @@
 def Set_get_elts(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elts'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elts')
     if w_self.w_elts is None:
         if w_self.elts is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.elts]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_elts = w_list
     return w_self.w_elts
 
@@ -5307,8 +5234,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elt'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elt')
     return space.wrap(w_self.elt)
 
 def ListComp_set_elt(space, w_self, w_new_value):
@@ -5325,14 +5251,13 @@
 def ListComp_get_generators(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'generators'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators')
     if w_self.w_generators is None:
         if w_self.generators is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.generators]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_generators = w_list
     return w_self.w_generators
 
@@ -5373,8 +5298,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elt'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elt')
     return space.wrap(w_self.elt)
 
 def SetComp_set_elt(space, w_self, w_new_value):
@@ -5391,14 +5315,13 @@
 def SetComp_get_generators(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'generators'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators')
     if w_self.w_generators is None:
         if w_self.generators is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.generators]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_generators = w_list
     return w_self.w_generators
 
@@ -5439,8 +5362,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'key'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'key')
     return space.wrap(w_self.key)
 
 def DictComp_set_key(space, w_self, w_new_value):
@@ -5461,8 +5383,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def DictComp_set_value(space, w_self, w_new_value):
@@ -5479,14 +5400,13 @@
 def DictComp_get_generators(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'generators'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators')
     if w_self.w_generators is None:
         if w_self.generators is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.generators]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_generators = w_list
     return w_self.w_generators
 
@@ -5528,8 +5448,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elt'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elt')
     return space.wrap(w_self.elt)
 
 def GeneratorExp_set_elt(space, w_self, w_new_value):
@@ -5546,14 +5465,13 @@
 def GeneratorExp_get_generators(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'generators'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators')
     if w_self.w_generators is None:
         if w_self.generators is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.generators]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_generators = w_list
     return w_self.w_generators
 
@@ -5594,8 +5512,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Yield_set_value(space, w_self, w_new_value):
@@ -5640,8 +5557,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'left'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'left')
     return space.wrap(w_self.left)
 
 def Compare_set_left(space, w_self, w_new_value):
@@ -5658,14 +5574,13 @@
 def Compare_get_ops(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ops'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ops')
     if w_self.w_ops is None:
         if w_self.ops is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [cmpop_to_class[node - 1]() for node in w_self.ops]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_ops = w_list
     return w_self.w_ops
 
@@ -5676,14 +5591,13 @@
 def Compare_get_comparators(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'comparators'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'comparators')
     if w_self.w_comparators is None:
         if w_self.comparators is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.comparators]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_comparators = w_list
     return w_self.w_comparators
 
@@ -5726,8 +5640,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'func'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'func')
     return space.wrap(w_self.func)
 
 def Call_set_func(space, w_self, w_new_value):
@@ -5744,14 +5657,13 @@
 def Call_get_args(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'args'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args')
     if w_self.w_args is None:
         if w_self.args is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.args]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_args = w_list
     return w_self.w_args
 
@@ -5762,14 +5674,13 @@
 def Call_get_keywords(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'keywords'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'keywords')
     if w_self.w_keywords is None:
         if w_self.keywords is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.keywords]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_keywords = w_list
     return w_self.w_keywords
 
@@ -5784,8 +5695,7 @@
             return w_obj
     if not w_self.initialization_state & 8:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'starargs'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'starargs')
     return space.wrap(w_self.starargs)
 
 def Call_set_starargs(space, w_self, w_new_value):
@@ -5806,8 +5716,7 @@
             return w_obj
     if not w_self.initialization_state & 16:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'kwargs'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'kwargs')
     return space.wrap(w_self.kwargs)
 
 def Call_set_kwargs(space, w_self, w_new_value):
@@ -5858,8 +5767,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Repr_set_value(space, w_self, w_new_value):
@@ -5904,8 +5812,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'n'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'n')
     return w_self.n
 
 def Num_set_n(space, w_self, w_new_value):
@@ -5950,8 +5857,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 's'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 's')
     return w_self.s
 
 def Str_set_s(space, w_self, w_new_value):
@@ -5996,8 +5902,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Attribute_set_value(space, w_self, w_new_value):
@@ -6018,8 +5923,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'attr'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'attr')
     return space.wrap(w_self.attr)
 
 def Attribute_set_attr(space, w_self, w_new_value):
@@ -6040,8 +5944,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ctx'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def Attribute_set_ctx(space, w_self, w_new_value):
@@ -6090,8 +5993,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Subscript_set_value(space, w_self, w_new_value):
@@ -6112,8 +6014,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'slice'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'slice')
     return space.wrap(w_self.slice)
 
 def Subscript_set_slice(space, w_self, w_new_value):
@@ -6134,8 +6035,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ctx'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def Subscript_set_ctx(space, w_self, w_new_value):
@@ -6184,8 +6084,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'id'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'id')
     return space.wrap(w_self.id)
 
 def Name_set_id(space, w_self, w_new_value):
@@ -6206,8 +6105,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ctx'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def Name_set_ctx(space, w_self, w_new_value):
@@ -6251,14 +6149,13 @@
 def List_get_elts(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elts'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elts')
     if w_self.w_elts is None:
         if w_self.elts is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.elts]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_elts = w_list
     return w_self.w_elts
 
@@ -6273,8 +6170,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ctx'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def List_set_ctx(space, w_self, w_new_value):
@@ -6319,14 +6215,13 @@
 def Tuple_get_elts(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elts'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elts')
     if w_self.w_elts is None:
         if w_self.elts is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.elts]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_elts = w_list
     return w_self.w_elts
 
@@ -6341,8 +6236,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ctx'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def Tuple_set_ctx(space, w_self, w_new_value):
@@ -6391,8 +6285,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return w_self.value
 
 def Const_set_value(space, w_self, w_new_value):
@@ -6510,8 +6403,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'lower'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lower')
     return space.wrap(w_self.lower)
 
 def Slice_set_lower(space, w_self, w_new_value):
@@ -6532,8 +6424,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'upper'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'upper')
     return space.wrap(w_self.upper)
 
 def Slice_set_upper(space, w_self, w_new_value):
@@ -6554,8 +6445,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'step'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'step')
     return space.wrap(w_self.step)
 
 def Slice_set_step(space, w_self, w_new_value):
@@ -6598,14 +6488,13 @@
 def ExtSlice_get_dims(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'dims'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'dims')
     if w_self.w_dims is None:
         if w_self.dims is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.dims]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_dims = w_list
     return w_self.w_dims
 
@@ -6645,8 +6534,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Index_set_value(space, w_self, w_new_value):
@@ -6915,8 +6803,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'target'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'target')
     return space.wrap(w_self.target)
 
 def comprehension_set_target(space, w_self, w_new_value):
@@ -6937,8 +6824,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'iter'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'iter')
     return space.wrap(w_self.iter)
 
 def comprehension_set_iter(space, w_self, w_new_value):
@@ -6955,14 +6841,13 @@
 def comprehension_get_ifs(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ifs'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ifs')
     if w_self.w_ifs is None:
         if w_self.ifs is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.ifs]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_ifs = w_list
     return w_self.w_ifs
 
@@ -7004,8 +6889,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._lineno_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'lineno'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lineno')
     return space.wrap(w_self.lineno)
 
 def excepthandler_set_lineno(space, w_self, w_new_value):
@@ -7026,8 +6910,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._col_offset_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'col_offset'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'col_offset')
     return space.wrap(w_self.col_offset)
 
 def excepthandler_set_col_offset(space, w_self, w_new_value):
@@ -7057,8 +6940,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'type'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'type')
     return space.wrap(w_self.type)
 
 def ExceptHandler_set_type(space, w_self, w_new_value):
@@ -7079,8 +6961,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'name'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name')
     return space.wrap(w_self.name)
 
 def ExceptHandler_set_name(space, w_self, w_new_value):
@@ -7097,14 +6978,13 @@
 def ExceptHandler_get_body(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -7142,14 +7022,13 @@
 def arguments_get_args(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'args'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args')
     if w_self.w_args is None:
         if w_self.args is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.args]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_args = w_list
     return w_self.w_args
 
@@ -7164,8 +7043,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'vararg'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'vararg')
     return space.wrap(w_self.vararg)
 
 def arguments_set_vararg(space, w_self, w_new_value):
@@ -7189,8 +7067,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'kwarg'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'kwarg')
     return space.wrap(w_self.kwarg)
 
 def arguments_set_kwarg(space, w_self, w_new_value):
@@ -7210,14 +7087,13 @@
 def arguments_get_defaults(space, w_self):
     if not w_self.initialization_state & 8:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'defaults'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'defaults')
     if w_self.w_defaults is None:
         if w_self.defaults is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.defaults]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_defaults = w_list
     return w_self.w_defaults
 
@@ -7261,8 +7137,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'arg'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'arg')
     return space.wrap(w_self.arg)
 
 def keyword_set_arg(space, w_self, w_new_value):
@@ -7283,8 +7158,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def keyword_set_value(space, w_self, w_new_value):
@@ -7330,8 +7204,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'name'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name')
     return space.wrap(w_self.name)
 
 def alias_set_name(space, w_self, w_new_value):
@@ -7352,8 +7225,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'asname'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'asname')
     return space.wrap(w_self.asname)
 
 def alias_set_asname(space, w_self, w_new_value):
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -414,13 +414,12 @@
             self.emit("        return w_obj", 1)
         self.emit("if not w_self.initialization_state & %s:" % (flag,), 1)
         self.emit("typename = space.type(w_self).getname(space)", 2)
-        self.emit("w_err = space.wrap(\"'%%s' object has no attribute '%s'\" %% typename)" %
+        self.emit("raise operationerrfmt(space.w_AttributeError, \"'%%s' object has no attribute '%%s'\", typename, '%s')" %
                   (field.name,), 2)
-        self.emit("raise OperationError(space.w_AttributeError, w_err)", 2)
         if field.seq:
             self.emit("if w_self.w_%s is None:" % (field.name,), 1)
             self.emit("if w_self.%s is None:" % (field.name,), 2)
-            self.emit("w_list = space.newlist([])", 3)
+            self.emit("list_w = []", 3)
             self.emit("else:", 2)
             if field.type.value in self.data.simple_types:
                 wrapper = "%s_to_class[node - 1]()" % (field.type,)
@@ -428,7 +427,7 @@
                 wrapper = "space.wrap(node)"
             self.emit("list_w = [%s for node in w_self.%s]" %
                       (wrapper, field.name), 3)
-            self.emit("w_list = space.newlist(list_w)", 3)
+            self.emit("w_list = space.newlist(list_w)", 2)
             self.emit("w_self.w_%s = w_list" % (field.name,), 2)
             self.emit("return w_self.w_%s" % (field.name,), 1)
         elif field.type.value in self.data.simple_types:
@@ -540,7 +539,7 @@
 from pypy.interpreter.baseobjspace import Wrappable
 from pypy.interpreter import typedef
 from pypy.interpreter.gateway import interp2app
-from pypy.interpreter.error import OperationError
+from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.tool.pairtype import extendabletype
 from pypy.tool.sourcetools import func_with_new_name
@@ -639,9 +638,7 @@
             missing = required[i]
             if missing is not None:
                  err = "required field \\"%s\\" missing from %s"
-                 err = err % (missing, host)
-                 w_err = space.wrap(err)
-                 raise OperationError(space.w_TypeError, w_err)
+                 raise operationerrfmt(space.w_TypeError, err, missing, host)
     raise AssertionError("should not reach here")
 
 
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -391,8 +391,11 @@
     def decrement_ticker(self, by):
         value = self._ticker
         if self.has_bytecode_counter:    # this 'if' is constant-folded
-            value -= by
-            self._ticker = value
+            if jit.isconstant(by) and by == 0:
+                pass     # normally constant-folded too
+            else:
+                value -= by
+                self._ticker = value
         return value
 
 
diff --git a/pypy/interpreter/pyparser/pytokenizer.py b/pypy/interpreter/pyparser/pytokenizer.py
--- a/pypy/interpreter/pyparser/pytokenizer.py
+++ b/pypy/interpreter/pyparser/pytokenizer.py
@@ -226,7 +226,7 @@
                         parenlev = parenlev - 1
                         if parenlev < 0:
                             raise TokenError("unmatched '%s'" % initial, line,
-                                             lnum-1, 0, token_list)
+                                             lnum, start + 1, token_list)
                     if token in python_opmap:
                         punct = python_opmap[token]
                     else:
diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py
--- a/pypy/interpreter/pyparser/test/test_pyparse.py
+++ b/pypy/interpreter/pyparser/test/test_pyparse.py
@@ -87,6 +87,10 @@
         assert exc.lineno == 1
         assert exc.offset == 5
         assert exc.lastlineno == 5
+        exc = py.test.raises(SyntaxError, parse, "abc)").value
+        assert exc.msg == "unmatched ')'"
+        assert exc.lineno == 1
+        assert exc.offset == 4
 
     def test_is(self):
         self.parse("x is y")
diff --git a/pypy/jit/backend/llgraph/llimpl.py b/pypy/jit/backend/llgraph/llimpl.py
--- a/pypy/jit/backend/llgraph/llimpl.py
+++ b/pypy/jit/backend/llgraph/llimpl.py
@@ -7,9 +7,7 @@
 import weakref
 from pypy.objspace.flow.model import Variable, Constant
 from pypy.annotation import model as annmodel
-from pypy.jit.metainterp.history import (ConstInt, ConstPtr,
-                                         BoxInt, BoxPtr, BoxObj, BoxFloat,
-                                         REF, INT, FLOAT)
+from pypy.jit.metainterp.history import REF, INT, FLOAT
 from pypy.jit.codewriter import heaptracker
 from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr, rffi
 from pypy.rpython.ootypesystem import ootype
@@ -17,7 +15,7 @@
 from pypy.rpython.llinterp import LLException
 from pypy.rpython.extregistry import ExtRegistryEntry
 
-from pypy.jit.metainterp import resoperation, executor
+from pypy.jit.metainterp import resoperation
 from pypy.jit.metainterp.resoperation import rop
 from pypy.jit.backend.llgraph import symbolic
 from pypy.jit.codewriter import longlong
@@ -165,6 +163,7 @@
     'unicodegetitem'  : (('ref', 'int'), 'int'),
     'unicodesetitem'  : (('ref', 'int', 'int'), 'int'),
     'cast_ptr_to_int' : (('ref',), 'int'),
+    'cast_int_to_ptr' : (('int',), 'ref'),
     'debug_merge_point': (('ref', 'int'), None),
     'force_token'     : ((), 'int'),
     'call_may_force'  : (('int', 'varargs'), 'intorptr'),
@@ -333,6 +332,13 @@
     assert isinstance(type, str) and len(type) == 1
     op.descr = Descr(ofs, type, arg_types=arg_types)
 
+def compile_add_descr_arg(loop, ofs, type, arg_types):
+    from pypy.jit.backend.llgraph.runner import Descr
+    loop = _from_opaque(loop)
+    op = loop.operations[-1]
+    assert isinstance(type, str) and len(type) == 1
+    op.args.append(Descr(ofs, type, arg_types=arg_types))
+
 def compile_add_loop_token(loop, descr):
     if we_are_translated():
         raise ValueError("CALL_ASSEMBLER not supported")
@@ -437,8 +443,11 @@
         self._may_force = -1
 
     def getenv(self, v):
+        from pypy.jit.backend.llgraph.runner import Descr
         if isinstance(v, Constant):
             return v.value
+        elif isinstance(v, Descr):
+            return v
         else:
             return self.env[v]
 
@@ -806,6 +815,29 @@
         else:
             raise NotImplementedError
 
+    def op_getinteriorfield_gc(self, descr, array, index):
+        if descr.typeinfo == REF:
+            return do_getinteriorfield_gc_ptr(array, index, descr.ofs)
+        elif descr.typeinfo == INT:
+            return do_getinteriorfield_gc_int(array, index, descr.ofs)
+        elif descr.typeinfo == FLOAT:
+            return do_getinteriorfield_gc_float(array, index, descr.ofs)
+        else:
+            raise NotImplementedError
+
+    def op_setinteriorfield_gc(self, descr, array, index, newvalue):
+        if descr.typeinfo == REF:
+            return do_setinteriorfield_gc_ptr(array, index, descr.ofs,
+                                              newvalue)
+        elif descr.typeinfo == INT:
+            return do_setinteriorfield_gc_int(array, index, descr.ofs,
+                                              newvalue)
+        elif descr.typeinfo == FLOAT:
+            return do_setinteriorfield_gc_float(array, index, descr.ofs,
+                                                newvalue)
+        else:
+            raise NotImplementedError
+
     def op_setfield_gc(self, fielddescr, struct, newvalue):
         if fielddescr.typeinfo == REF:
             do_setfield_gc_ptr(struct, fielddescr.ofs, newvalue)
@@ -875,9 +907,6 @@
     def op_new_array(self, arraydescr, count):
         return do_new_array(arraydescr.ofs, count)
 
-    def op_cast_ptr_to_int(self, descr, ptr):
-        return cast_to_int(ptr)
-
     def op_force_token(self, descr):
         opaque_frame = _to_opaque(self)
         return llmemory.cast_ptr_to_adr(opaque_frame)
@@ -1356,6 +1385,22 @@
 def do_getfield_gc_ptr(struct, fieldnum):
     return cast_to_ptr(_getfield_gc(struct, fieldnum))
 
+def _getinteriorfield_gc(struct, fieldnum):
+    STRUCT, fieldname = symbolic.TokenToField[fieldnum]
+    return getattr(struct, fieldname)
+
+def do_getinteriorfield_gc_int(array, index, fieldnum):
+    struct = array._obj.container.getitem(index)
+    return cast_to_int(_getinteriorfield_gc(struct, fieldnum))
+
+def do_getinteriorfield_gc_float(array, index, fieldnum):
+    struct = array._obj.container.getitem(index)
+    return cast_to_floatstorage(_getinteriorfield_gc(struct, fieldnum))
+
+def do_getinteriorfield_gc_ptr(array, index, fieldnum):
+    struct = array._obj.container.getitem(index)
+    return cast_to_ptr(_getinteriorfield_gc(struct, fieldnum))
+
 def _getfield_raw(struct, fieldnum):
     STRUCT, fieldname = symbolic.TokenToField[fieldnum]
     ptr = cast_from_int(lltype.Ptr(STRUCT), struct)
@@ -1411,26 +1456,28 @@
     newvalue = cast_from_ptr(ITEMTYPE, newvalue)
     array.setitem(index, newvalue)
 
-def do_setfield_gc_int(struct, fieldnum, newvalue):
-    STRUCT, fieldname = symbolic.TokenToField[fieldnum]
-    ptr = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), struct)
-    FIELDTYPE = getattr(STRUCT, fieldname)
-    newvalue = cast_from_int(FIELDTYPE, newvalue)
-    setattr(ptr, fieldname, newvalue)
+def new_setfield_gc(cast_func):
+    def do_setfield_gc(struct, fieldnum, newvalue):
+        STRUCT, fieldname = symbolic.TokenToField[fieldnum]
+        ptr = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), struct)
+        FIELDTYPE = getattr(STRUCT, fieldname)
+        newvalue = cast_func(FIELDTYPE, newvalue)
+        setattr(ptr, fieldname, newvalue)
+    return do_setfield_gc
+do_setfield_gc_int = new_setfield_gc(cast_from_int)
+do_setfield_gc_float = new_setfield_gc(cast_from_floatstorage)
+do_setfield_gc_ptr = new_setfield_gc(cast_from_ptr)
 
-def do_setfield_gc_float(struct, fieldnum, newvalue):
-    STRUCT, fieldname = symbolic.TokenToField[fieldnum]
-    ptr = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), struct)
-    FIELDTYPE = getattr(STRUCT, fieldname)
-    newvalue = cast_from_floatstorage(FIELDTYPE, newvalue)
-    setattr(ptr, fieldname, newvalue)
-
-def do_setfield_gc_ptr(struct, fieldnum, newvalue):
-    STRUCT, fieldname = symbolic.TokenToField[fieldnum]
-    ptr = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), struct)
-    FIELDTYPE = getattr(STRUCT, fieldname)
-    newvalue = cast_from_ptr(FIELDTYPE, newvalue)
-    setattr(ptr, fieldname, newvalue)
+def new_setinteriorfield_gc(cast_func):
+    def do_setinteriorfield_gc(array, index, fieldnum, newvalue):
+        STRUCT, fieldname = symbolic.TokenToField[fieldnum]
+        struct = array._obj.container.getitem(index)
+        FIELDTYPE = getattr(STRUCT, fieldname)
+        setattr(struct, fieldname, cast_func(FIELDTYPE, newvalue))
+    return do_setinteriorfield_gc
+do_setinteriorfield_gc_int = new_setinteriorfield_gc(cast_from_int)
+do_setinteriorfield_gc_float = new_setinteriorfield_gc(cast_from_floatstorage)
+do_setinteriorfield_gc_ptr = new_setinteriorfield_gc(cast_from_ptr)        
 
 def do_setfield_raw_int(struct, fieldnum, newvalue):
     STRUCT, fieldname = symbolic.TokenToField[fieldnum]
@@ -1696,6 +1743,7 @@
 setannotation(compile_start_float_var, annmodel.SomeInteger())
 setannotation(compile_add, annmodel.s_None)
 setannotation(compile_add_descr, annmodel.s_None)
+setannotation(compile_add_descr_arg, annmodel.s_None)
 setannotation(compile_add_var, annmodel.s_None)
 setannotation(compile_add_int_const, annmodel.s_None)
 setannotation(compile_add_ref_const, annmodel.s_None)
@@ -1743,6 +1791,9 @@
 setannotation(do_getfield_raw_int, annmodel.SomeInteger())
 setannotation(do_getfield_raw_ptr, annmodel.SomePtr(llmemory.GCREF))
 setannotation(do_getfield_raw_float, s_FloatStorage)
+setannotation(do_getinteriorfield_gc_int, annmodel.SomeInteger())
+setannotation(do_getinteriorfield_gc_ptr, annmodel.SomePtr(llmemory.GCREF))
+setannotation(do_getinteriorfield_gc_float, s_FloatStorage)
 setannotation(do_new, annmodel.SomePtr(llmemory.GCREF))
 setannotation(do_new_array, annmodel.SomePtr(llmemory.GCREF))
 setannotation(do_setarrayitem_gc_int, annmodel.s_None)
@@ -1756,6 +1807,9 @@
 setannotation(do_setfield_raw_int, annmodel.s_None)
 setannotation(do_setfield_raw_ptr, annmodel.s_None)
 setannotation(do_setfield_raw_float, annmodel.s_None)
+setannotation(do_setinteriorfield_gc_int, annmodel.s_None)
+setannotation(do_setinteriorfield_gc_ptr, annmodel.s_None)
+setannotation(do_setinteriorfield_gc_float, annmodel.s_None)
 setannotation(do_newstr, annmodel.SomePtr(llmemory.GCREF))
 setannotation(do_strsetitem, annmodel.s_None)
 setannotation(do_newunicode, annmodel.SomePtr(llmemory.GCREF))
diff --git a/pypy/jit/backend/llgraph/runner.py b/pypy/jit/backend/llgraph/runner.py
--- a/pypy/jit/backend/llgraph/runner.py
+++ b/pypy/jit/backend/llgraph/runner.py
@@ -2,21 +2,19 @@
 Minimal-API wrapper around the llinterpreter to run operations.
 """
 
-import sys
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.rpython.lltypesystem import lltype, llmemory, rclass
 from pypy.rpython.ootypesystem import ootype
 from pypy.rpython.llinterp import LLInterpreter
 from pypy.jit.metainterp import history
-from pypy.jit.metainterp.history import REF, INT, FLOAT
+from pypy.jit.metainterp.history import REF, INT, FLOAT, STRUCT
 from pypy.jit.metainterp.warmstate import unwrap
-from pypy.jit.metainterp.resoperation import ResOperation, rop
+from pypy.jit.metainterp.resoperation import rop
 from pypy.jit.backend import model
 from pypy.jit.backend.llgraph import llimpl, symbolic
 from pypy.jit.metainterp.typesystem import llhelper, oohelper
 from pypy.jit.codewriter import heaptracker, longlong
-from pypy.rlib import rgc
 
 class MiniStats:
     pass
@@ -62,6 +60,9 @@
     def is_array_of_floats(self):
         return self.typeinfo == FLOAT
 
+    def is_array_of_structs(self):
+        return self.typeinfo == STRUCT
+
     def as_vtable_size_descr(self):
         return self
 
@@ -177,8 +178,10 @@
             llimpl.compile_add(c, op.getopnum())
             descr = op.getdescr()
             if isinstance(descr, Descr):
-                llimpl.compile_add_descr(c, descr.ofs, descr.typeinfo, descr.arg_types)
-            if isinstance(descr, history.LoopToken) and op.getopnum() != rop.JUMP:
+                llimpl.compile_add_descr(c, descr.ofs, descr.typeinfo,
+                                         descr.arg_types)
+            if (isinstance(descr, history.LoopToken) and
+                op.getopnum() != rop.JUMP):
                 llimpl.compile_add_loop_token(c, descr)
             if self.is_oo and isinstance(descr, (OODescr, MethDescr)):
                 # hack hack, not rpython
@@ -193,6 +196,9 @@
                     llimpl.compile_add_ref_const(c, x.value, self.ts.BASETYPE)
                 elif isinstance(x, history.ConstFloat):
                     llimpl.compile_add_float_const(c, x.value)
+                elif isinstance(x, Descr):
+                    llimpl.compile_add_descr_arg(c, x.ofs, x.typeinfo,
+                                                 x.arg_types)
                 else:
                     raise Exception("'%s' args contain: %r" % (op.getopname(),
                                                                x))
@@ -316,6 +322,13 @@
         token = history.getkind(getattr(S, fieldname))
         return self.getdescr(ofs, token[0], name=fieldname)
 
+    def interiorfielddescrof(self, A, fieldname):
+        S = A.OF
+        ofs2 = symbolic.get_size(A)
+        ofs, size = symbolic.get_field_token(S, fieldname)
+        token = history.getkind(getattr(S, fieldname))
+        return self.getdescr(ofs, token[0], name=fieldname, extrainfo=ofs2)
+
     def calldescrof(self, FUNC, ARGS, RESULT, extrainfo):
         arg_types = []
         for ARG in ARGS:
@@ -353,8 +366,13 @@
     def arraydescrof(self, A):
         assert A.OF != lltype.Void
         size = symbolic.get_size(A)
-        token = history.getkind(A.OF)
-        return self.getdescr(size, token[0])
+        if isinstance(A.OF, lltype.Ptr) or isinstance(A.OF, lltype.Primitive):
+            token = history.getkind(A.OF)[0]
+        elif isinstance(A.OF, lltype.Struct):
+            token = 's'
+        else:
+            token = '?'
+        return self.getdescr(size, token)
 
     # ---------- the backend-dependent operations ----------
 
@@ -406,6 +424,29 @@
         assert isinstance(fielddescr, Descr)
         return llimpl.do_getfield_raw_float(struct, fielddescr.ofs)
 
+    def bh_getinteriorfield_gc_i(self, array, index, descr):
+        assert isinstance(descr, Descr)
+        return llimpl.do_getinteriorfield_gc_int(array, index, descr.ofs)
+    def bh_getinteriorfield_gc_r(self, array, index, descr):
+        assert isinstance(descr, Descr)
+        return llimpl.do_getinteriorfield_gc_ptr(array, index, descr.ofs)
+    def bh_getinteriorfield_gc_f(self, array, index, descr):
+        assert isinstance(descr, Descr)
+        return llimpl.do_getinteriorfield_gc_float(array, index, descr.ofs)
+
+    def bh_setinteriorfield_gc_i(self, array, index, descr, value):
+        assert isinstance(descr, Descr)
+        return llimpl.do_setinteriorfield_gc_int(array, index, descr.ofs,
+                                                 value)
+    def bh_setinteriorfield_gc_r(self, array, index, descr, value):
+        assert isinstance(descr, Descr)
+        return llimpl.do_setinteriorfield_gc_ptr(array, index, descr.ofs,
+                                                 value)
+    def bh_setinteriorfield_gc_f(self, array, index, descr, value):
+        assert isinstance(descr, Descr)
+        return llimpl.do_setinteriorfield_gc_float(array, index, descr.ofs,
+                                                   value)
+
     def bh_new(self, sizedescr):
         assert isinstance(sizedescr, Descr)
         return llimpl.do_new(sizedescr.ofs)
@@ -418,7 +459,6 @@
 
     def bh_classof(self, struct):
         struct = lltype.cast_opaque_ptr(rclass.OBJECTPTR, struct)
-        result = struct.typeptr
         result_adr = llmemory.cast_ptr_to_adr(struct.typeptr)
         return heaptracker.adr2int(result_adr)
 
diff --git a/pypy/jit/backend/llsupport/descr.py b/pypy/jit/backend/llsupport/descr.py
--- a/pypy/jit/backend/llsupport/descr.py
+++ b/pypy/jit/backend/llsupport/descr.py
@@ -1,13 +1,10 @@
 import py
-from pypy.rpython.lltypesystem import lltype, rffi, llmemory, rclass
+from pypy.rpython.lltypesystem import lltype, rffi, llmemory
 from pypy.rpython.lltypesystem.lloperation import llop
 from pypy.jit.backend.llsupport import symbolic, support
-from pypy.jit.metainterp.history import AbstractDescr, getkind, BoxInt, BoxPtr
-from pypy.jit.metainterp.history import BasicFailDescr, LoopToken, BoxFloat
+from pypy.jit.metainterp.history import AbstractDescr, getkind
 from pypy.jit.metainterp import history
-from pypy.jit.metainterp.resoperation import ResOperation, rop
 from pypy.jit.codewriter import heaptracker, longlong
-from pypy.rlib.rarithmetic import r_longlong, r_ulonglong
 
 # The point of the class organization in this file is to make instances
 # as compact as possible.  This is done by not storing the field size or
@@ -23,6 +20,7 @@
         self._cache_field = {}
         self._cache_array = {}
         self._cache_call = {}
+        self._cache_interiorfield = {}
 
     def init_size_descr(self, STRUCT, sizedescr):
         assert isinstance(STRUCT, lltype.GcStruct)
@@ -142,7 +140,6 @@
         cachedict[fieldname] = fielddescr
         return fielddescr
 
-
 # ____________________________________________________________
 # ArrayDescrs
 
@@ -167,6 +164,7 @@
 
     _is_array_of_pointers = False      # unless overridden by GcPtrArrayDescr
     _is_array_of_floats   = False      # unless overridden by FloatArrayDescr
+    _is_array_of_structs  = False      # unless overridden by StructArrayDescr
     _is_item_signed       = False      # unless overridden by XxxArrayDescr
 
     def is_array_of_pointers(self):
@@ -175,6 +173,9 @@
     def is_array_of_floats(self):
         return self._is_array_of_floats
 
+    def is_array_of_structs(self):
+        return self._is_array_of_structs
+
     def is_item_signed(self):
         return self._is_item_signed
 
@@ -199,6 +200,10 @@
     def get_item_size(self, translate_support_code):
         return symbolic.get_size(lltype.Float, translate_support_code)
 
+class StructArrayDescr(BaseArrayDescr):
+    _clsname = 'StructArrayDescr'
+    _is_array_of_structs = True
+
 class BaseArrayNoLengthDescr(BaseArrayDescr):
     def get_base_size(self, translate_support_code):
         return 0
@@ -218,6 +223,13 @@
 def getArrayDescrClass(ARRAY):
     if ARRAY.OF is lltype.Float:
         return FloatArrayDescr
+    elif isinstance(ARRAY.OF, lltype.Struct):
+        class Descr(StructArrayDescr):
+            _clsname = '%sArrayDescr' % ARRAY.OF._name
+            def get_item_size(self, translate_support_code):
+                return symbolic.get_size(ARRAY.OF, translate_support_code)
+        Descr.__name__ = Descr._clsname
+        return Descr
     return getDescrClass(ARRAY.OF, BaseArrayDescr, GcPtrArrayDescr,
                          NonGcPtrArrayDescr, 'Array', 'get_item_size',
                          '_is_array_of_floats', '_is_item_signed')
@@ -252,6 +264,39 @@
         cache[ARRAY] = arraydescr
         return arraydescr
 
+# ____________________________________________________________
+# InteriorFieldDescr
+
+class InteriorFieldDescr(AbstractDescr):
+    arraydescr = BaseArrayDescr()     # workaround for the annotator
+    fielddescr = BaseFieldDescr('', 0)
+
+    def __init__(self, arraydescr, fielddescr):
+        self.arraydescr = arraydescr
+        self.fielddescr = fielddescr
+
+    def is_pointer_field(self):
+        return self.fielddescr.is_pointer_field()
+
+    def is_float_field(self):
+        return self.fielddescr.is_float_field()
+
+    def sort_key(self):
+        return self.fielddescr.sort_key()
+
+    def repr_of_descr(self):
+        return '<InteriorFieldDescr %s>' % self.fielddescr.repr_of_descr()
+
+def get_interiorfield_descr(gc_ll_descr, ARRAY, FIELDTP, name):
+    cache = gc_ll_descr._cache_interiorfield
+    try:
+        return cache[(ARRAY, FIELDTP, name)]
+    except KeyError:
+        arraydescr = get_array_descr(gc_ll_descr, ARRAY)
+        fielddescr = get_field_descr(gc_ll_descr, FIELDTP, name)
+        descr = InteriorFieldDescr(arraydescr, fielddescr)
+        cache[(ARRAY, FIELDTP, name)] = descr
+        return descr
 
 # ____________________________________________________________
 # CallDescrs
@@ -525,7 +570,8 @@
         #
         if TYPE is lltype.Float or is_longlong(TYPE):
             setattr(Descr, floatattrname, True)
-        elif TYPE is not lltype.Bool and rffi.cast(TYPE, -1) == -1:
+        elif (TYPE is not lltype.Bool and isinstance(TYPE, lltype.Number) and
+              rffi.cast(TYPE, -1) == -1):
             setattr(Descr, signedattrname, True)
         #
         _cache[nameprefix, TYPE] = Descr
diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py
--- a/pypy/jit/backend/llsupport/gc.py
+++ b/pypy/jit/backend/llsupport/gc.py
@@ -45,6 +45,14 @@
     def freeing_block(self, start, stop):
         pass
 
+    def get_funcptr_for_newarray(self):
+        return llhelper(self.GC_MALLOC_ARRAY, self.malloc_array)
+    def get_funcptr_for_newstr(self):
+        return llhelper(self.GC_MALLOC_STR_UNICODE, self.malloc_str)
+    def get_funcptr_for_newunicode(self):
+        return llhelper(self.GC_MALLOC_STR_UNICODE, self.malloc_unicode)
+
+
     def record_constptrs(self, op, gcrefs_output_list):
         for i in range(op.numargs()):
             v = op.getarg(i)
@@ -96,6 +104,39 @@
         malloc_fn_ptr = self.configure_boehm_once()
         self.funcptr_for_new = malloc_fn_ptr
 
+        def malloc_array(basesize, itemsize, ofs_length, num_elem):
+            try:
+                size = ovfcheck(basesize + ovfcheck(itemsize * num_elem))
+            except OverflowError:
+                return lltype.nullptr(llmemory.GCREF.TO)
+            res = self.funcptr_for_new(size)
+            if not res:
+                return res
+            rffi.cast(rffi.CArrayPtr(lltype.Signed), res)[ofs_length/WORD] = num_elem
+            return res
+        self.malloc_array = malloc_array
+        self.GC_MALLOC_ARRAY = lltype.Ptr(lltype.FuncType(
+            [lltype.Signed] * 4, llmemory.GCREF))
+
+
+        (str_basesize, str_itemsize, str_ofs_length
+         ) = symbolic.get_array_token(rstr.STR, self.translate_support_code)
+        (unicode_basesize, unicode_itemsize, unicode_ofs_length
+         ) = symbolic.get_array_token(rstr.UNICODE, self.translate_support_code)
+        def malloc_str(length):
+            return self.malloc_array(
+                str_basesize, str_itemsize, str_ofs_length, length
+            )
+        def malloc_unicode(length):
+            return self.malloc_array(
+                unicode_basesize, unicode_itemsize, unicode_ofs_length, length
+            )
+        self.malloc_str = malloc_str
+        self.malloc_unicode = malloc_unicode
+        self.GC_MALLOC_STR_UNICODE = lltype.Ptr(lltype.FuncType(
+            [lltype.Signed], llmemory.GCREF))
+
+
         # on some platform GC_init is required before any other
         # GC_* functions, call it here for the benefit of tests
         # XXX move this to tests
@@ -116,39 +157,27 @@
         ofs_length = arraydescr.get_ofs_length(self.translate_support_code)
         basesize = arraydescr.get_base_size(self.translate_support_code)
         itemsize = arraydescr.get_item_size(self.translate_support_code)
-        size = basesize + itemsize * num_elem
-        res = self.funcptr_for_new(size)
-        rffi.cast(rffi.CArrayPtr(lltype.Signed), res)[ofs_length/WORD] = num_elem
-        return res
+        return self.malloc_array(basesize, itemsize, ofs_length, num_elem)
 
     def gc_malloc_str(self, num_elem):
-        basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR,
-                                                   self.translate_support_code)
-        assert itemsize == 1
-        size = basesize + num_elem
-        res = self.funcptr_for_new(size)
-        rffi.cast(rffi.CArrayPtr(lltype.Signed), res)[ofs_length/WORD] = num_elem
-        return res
+        return self.malloc_str(num_elem)
 
     def gc_malloc_unicode(self, num_elem):
-        basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE,
-                                                   self.translate_support_code)
-        size = basesize + num_elem * itemsize
-        res = self.funcptr_for_new(size)
-        rffi.cast(rffi.CArrayPtr(lltype.Signed), res)[ofs_length/WORD] = num_elem
-        return res
+        return self.malloc_unicode(num_elem)
 
     def args_for_new(self, sizedescr):
         assert isinstance(sizedescr, BaseSizeDescr)
         return [sizedescr.size]
 
+    def args_for_new_array(self, arraydescr):
+        ofs_length = arraydescr.get_ofs_length(self.translate_support_code)
+        basesize = arraydescr.get_base_size(self.translate_support_code)
+        itemsize = arraydescr.get_item_size(self.translate_support_code)
+        return [basesize, itemsize, ofs_length]
+
     def get_funcptr_for_new(self):
         return self.funcptr_for_new
 
-    get_funcptr_for_newarray = None
-    get_funcptr_for_newstr = None
-    get_funcptr_for_newunicode = None
-
     def rewrite_assembler(self, cpu, operations, gcrefs_output_list):
         # record all GCREFs too, because Boehm cannot see them and keep them
         # alive if they end up as constants in the assembler
@@ -620,10 +649,13 @@
         def malloc_basic(size, tid):
             type_id = llop.extract_ushort(llgroup.HALFWORD, tid)
             has_finalizer = bool(tid & (1<<llgroup.HALFSHIFT))
+            has_light_finalizer = bool(tid & (1<<(llgroup.HALFSHIFT + 1)))
             check_typeid(type_id)
             res = llop1.do_malloc_fixedsize_clear(llmemory.GCREF,
                                                   type_id, size,
-                                                  has_finalizer, False)
+                                                  has_finalizer,
+                                                  has_light_finalizer,
+                                                  False)
             # In case the operation above failed, we are returning NULL
             # from this function to assembler.  There is also an RPython
             # exception set, typically MemoryError; but it's easier and
@@ -694,7 +726,7 @@
             # also use it to allocate varsized objects.  The tid
             # and possibly the length are both set afterward.
             gcref = llop1.do_malloc_fixedsize_clear(llmemory.GCREF,
-                                        0, size, False, False)
+                                        0, size, False, False, False)
             return rffi.cast(lltype.Signed, gcref)
         self.malloc_slowpath = malloc_slowpath
         self.MALLOC_SLOWPATH = lltype.FuncType([lltype.Signed], lltype.Signed)
@@ -718,7 +750,9 @@
         type_id = self.layoutbuilder.get_type_id(S)
         assert not self.layoutbuilder.is_weakref_type(S)
         has_finalizer = bool(self.layoutbuilder.has_finalizer(S))
-        flags = int(has_finalizer) << llgroup.HALFSHIFT
+        has_light_finalizer = bool(self.layoutbuilder.has_light_finalizer(S))
+        flags = (int(has_finalizer) << llgroup.HALFSHIFT |
+                 int(has_light_finalizer) << (llgroup.HALFSHIFT + 1))
         descr.tid = llop.combine_ushort(lltype.Signed, type_id, flags)
 
     def init_array_descr(self, A, descr):
@@ -752,15 +786,6 @@
     def get_funcptr_for_new(self):
         return llhelper(self.GC_MALLOC_BASIC, self.malloc_basic)
 
-    def get_funcptr_for_newarray(self):
-        return llhelper(self.GC_MALLOC_ARRAY, self.malloc_array)
-
-    def get_funcptr_for_newstr(self):
-        return llhelper(self.GC_MALLOC_STR_UNICODE, self.malloc_str)
-
-    def get_funcptr_for_newunicode(self):
-        return llhelper(self.GC_MALLOC_STR_UNICODE, self.malloc_unicode)
-
     def do_write_barrier(self, gcref_struct, gcref_newptr):
         hdr_addr = llmemory.cast_ptr_to_adr(gcref_struct)
         hdr_addr -= self.gcheaderbuilder.size_gc_header
diff --git a/pypy/jit/backend/llsupport/llmodel.py b/pypy/jit/backend/llsupport/llmodel.py
--- a/pypy/jit/backend/llsupport/llmodel.py
+++ b/pypy/jit/backend/llsupport/llmodel.py
@@ -1,24 +1,18 @@
-import sys
 from pypy.rpython.lltypesystem import lltype, llmemory, rffi, rclass, rstr
 from pypy.rpython.lltypesystem.lloperation import llop
-from pypy.rpython.llinterp import LLInterpreter, LLException
+from pypy.rpython.llinterp import LLInterpreter
 from pypy.rpython.annlowlevel import llhelper
 from pypy.rlib.objectmodel import we_are_translated, specialize
-from pypy.jit.metainterp.history import BoxInt, BoxPtr, set_future_values,\
-     BoxFloat
 from pypy.jit.metainterp import history
 from pypy.jit.codewriter import heaptracker, longlong
 from pypy.jit.backend.model import AbstractCPU
 from pypy.jit.backend.llsupport import symbolic
 from pypy.jit.backend.llsupport.symbolic import WORD, unroll_basic_sizes
-from pypy.jit.backend.llsupport.descr import get_size_descr,  BaseSizeDescr
-from pypy.jit.backend.llsupport.descr import get_field_descr, BaseFieldDescr
-from pypy.jit.backend.llsupport.descr import get_array_descr, BaseArrayDescr
-from pypy.jit.backend.llsupport.descr import get_call_descr
-from pypy.jit.backend.llsupport.descr import BaseIntCallDescr, GcPtrCallDescr
-from pypy.jit.backend.llsupport.descr import FloatCallDescr, VoidCallDescr
+from pypy.jit.backend.llsupport.descr import (get_size_descr,
+     get_field_descr, BaseFieldDescr, get_array_descr, BaseArrayDescr,
+     get_call_descr, BaseIntCallDescr, GcPtrCallDescr, FloatCallDescr,
+     VoidCallDescr, InteriorFieldDescr, get_interiorfield_descr)
 from pypy.jit.backend.llsupport.asmmemmgr import AsmMemoryManager
-from pypy.rpython.annlowlevel import cast_instance_to_base_ptr
 
 
 class AbstractLLCPU(AbstractCPU):
@@ -241,6 +235,9 @@
     def arraydescrof(self, A):
         return get_array_descr(self.gc_ll_descr, A)
 
+    def interiorfielddescrof(self, A, fieldname):
+        return get_interiorfield_descr(self.gc_ll_descr, A, A.OF, fieldname)
+
     def unpack_arraydescr(self, arraydescr):
         assert isinstance(arraydescr, BaseArrayDescr)
         return arraydescr.get_base_size(self.translate_support_code)
@@ -358,6 +355,100 @@
     bh_getarrayitem_raw_i = bh_getarrayitem_gc_i
     bh_getarrayitem_raw_f = bh_getarrayitem_gc_f
 
+    def bh_getinteriorfield_gc_i(self, gcref, itemindex, descr):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        fieldsize = descr.fielddescr.get_field_size(self.translate_support_code)
+        sign = descr.fielddescr.is_field_signed()
+        fullofs = itemindex * size + ofs
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), fullofs)
+        for STYPE, UTYPE, itemsize in unroll_basic_sizes:
+            if fieldsize == itemsize:
+                if sign:
+                    item = rffi.cast(rffi.CArrayPtr(STYPE), items)
+                    val = item[0]
+                    val = rffi.cast(lltype.Signed, val)
+                else:
+                    item = rffi.cast(rffi.CArrayPtr(UTYPE), items)
+                    val = item[0]
+                    val = rffi.cast(lltype.Signed, val)
+                # --- end of GC unsafe code ---
+                return val
+        else:
+            raise NotImplementedError("size = %d" % fieldsize)
+
+    def bh_getinteriorfield_gc_r(self, gcref, itemindex, descr):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs +
+                            size * itemindex)
+        items = rffi.cast(rffi.CArrayPtr(lltype.Signed), items)
+        pval = self._cast_int_to_gcref(items[0])
+        # --- end of GC unsafe code ---
+        return pval
+
+    def bh_getinteriorfield_gc_f(self, gcref, itemindex, descr):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs +
+                            size * itemindex)
+        items = rffi.cast(rffi.CArrayPtr(longlong.FLOATSTORAGE), items)
+        fval = items[0]
+        # --- end of GC unsafe code ---
+        return fval
+
+    def bh_setinteriorfield_gc_i(self, gcref, itemindex, descr, value):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        fieldsize = descr.fielddescr.get_field_size(self.translate_support_code)
+        ofs = itemindex * size + ofs
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs)
+        for TYPE, _, itemsize in unroll_basic_sizes:
+            if fieldsize == itemsize:
+                items = rffi.cast(rffi.CArrayPtr(TYPE), items)
+                items[0] = rffi.cast(TYPE, value)
+                # --- end of GC unsafe code ---
+                return
+        else:
+            raise NotImplementedError("size = %d" % fieldsize)
+
+    def bh_setinteriorfield_gc_r(self, gcref, itemindex, descr, newvalue):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        self.gc_ll_descr.do_write_barrier(gcref, newvalue)
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref),
+                            ofs + size * itemindex)
+        items = rffi.cast(rffi.CArrayPtr(lltype.Signed), items)
+        items[0] = self.cast_gcref_to_int(newvalue)
+        # --- end of GC unsafe code ---
+
+    def bh_setinteriorfield_gc_f(self, gcref, itemindex, descr, newvalue):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref),
+                            ofs + size * itemindex)
+        items = rffi.cast(rffi.CArrayPtr(longlong.FLOATSTORAGE), items)
+        items[0] = newvalue
+        # --- end of GC unsafe code ---
+
     def bh_strlen(self, string):
         s = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), string)
         return len(s.chars)
@@ -475,7 +566,6 @@
 
     def bh_classof(self, struct):
         struct = lltype.cast_opaque_ptr(rclass.OBJECTPTR, struct)
-        result = struct.typeptr
         result_adr = llmemory.cast_ptr_to_adr(struct.typeptr)
         return heaptracker.adr2int(result_adr)
 
diff --git a/pypy/jit/backend/llsupport/regalloc.py b/pypy/jit/backend/llsupport/regalloc.py
--- a/pypy/jit/backend/llsupport/regalloc.py
+++ b/pypy/jit/backend/llsupport/regalloc.py
@@ -287,7 +287,6 @@
         self.reg_bindings[to_v] = reg
 
     def _move_variable_away(self, v, prev_loc):
-        reg = None
         if self.free_regs:
             loc = self.free_regs.pop()
             self.reg_bindings[v] = loc
diff --git a/pypy/jit/backend/llsupport/test/test_asmmemmgr.py b/pypy/jit/backend/llsupport/test/test_asmmemmgr.py
--- a/pypy/jit/backend/llsupport/test/test_asmmemmgr.py
+++ b/pypy/jit/backend/llsupport/test/test_asmmemmgr.py
@@ -211,14 +211,14 @@
     debug._log = debug.DebugLog()
     try:
         mc._dump(addr, 'test-logname-section')
-        log = list(debug._log)
+        log = list(debug._log) 
     finally:
         debug._log = None
     encoded = ''.join(writtencode).encode('hex').upper()
     ataddr = '@%x' % addr
     assert log == [('test-logname-section',
                     [('debug_print', 'CODE_DUMP', ataddr, '+0 ', encoded)])]
-    #
+    
     lltype.free(p, flavor='raw')
 
 def test_blockbuildermixin2():
diff --git a/pypy/jit/backend/llsupport/test/test_descr.py b/pypy/jit/backend/llsupport/test/test_descr.py
--- a/pypy/jit/backend/llsupport/test/test_descr.py
+++ b/pypy/jit/backend/llsupport/test/test_descr.py
@@ -3,7 +3,6 @@
 from pypy.jit.backend.llsupport import symbolic
 from pypy.rlib.objectmodel import Symbolic
 from pypy.rpython.annlowlevel import llhelper
-from pypy.jit.metainterp.history import BoxInt, BoxFloat, BoxPtr
 from pypy.jit.metainterp import history
 from pypy.jit.codewriter import longlong
 import sys, struct, py
@@ -149,7 +148,9 @@
     A2 = lltype.GcArray(lltype.Ptr(T))
     A3 = lltype.GcArray(lltype.Ptr(U))
     A4 = lltype.GcArray(lltype.Float)
-    A5 = lltype.GcArray(lltype.SingleFloat)
+    A5 = lltype.GcArray(lltype.Struct('x', ('v', lltype.Signed),
+                                      ('k', lltype.Signed)))
+    A6 = lltype.GcArray(lltype.SingleFloat)
     assert getArrayDescrClass(A2) is GcPtrArrayDescr
     assert getArrayDescrClass(A3) is NonGcPtrArrayDescr
     cls = getArrayDescrClass(A1)
@@ -158,7 +159,7 @@
     clsf = getArrayDescrClass(A4)
     assert clsf != cls
     assert clsf == getArrayDescrClass(lltype.GcArray(lltype.Float))
-    clss = getArrayDescrClass(A5)
+    clss = getArrayDescrClass(A6)
     assert clss not in (clsf, cls)
     assert clss == getArrayDescrClass(lltype.GcArray(rffi.UINT))
     #
@@ -168,11 +169,12 @@
     descr3 = get_array_descr(c0, A3)
     descr4 = get_array_descr(c0, A4)
     descr5 = get_array_descr(c0, A5)
+    descr6 = get_array_descr(c0, A6)
     assert descr1.__class__ is cls
     assert descr2.__class__ is GcPtrArrayDescr
     assert descr3.__class__ is NonGcPtrArrayDescr
     assert descr4.__class__ is clsf
-    assert descr5.__class__ is clss
+    assert descr6.__class__ is clss
     assert descr1 == get_array_descr(c0, lltype.GcArray(lltype.Char))
     assert not descr1.is_array_of_pointers()
     assert     descr2.is_array_of_pointers()
@@ -202,7 +204,8 @@
     assert descr2.get_item_size(False) == rffi.sizeof(lltype.Ptr(T))
     assert descr3.get_item_size(False) == rffi.sizeof(lltype.Ptr(U))
     assert descr4.get_item_size(False) == rffi.sizeof(lltype.Float)
-    assert descr5.get_item_size(False) == rffi.sizeof(lltype.SingleFloat)
+    assert descr5.get_item_size(False) == rffi.sizeof(lltype.Signed) * 2
+    assert descr6.get_item_size(False) == rffi.sizeof(lltype.SingleFloat)
     #
     assert isinstance(descr1.get_base_size(True), Symbolic)
     assert isinstance(descr2.get_base_size(True), Symbolic)
@@ -348,7 +351,6 @@
                             (rffi.SHORT,      True), (rffi.USHORT, False),
                             (rffi.INT,        True), (rffi.UINT,   False),
                             (rffi.LONG,       True), (rffi.ULONG,  False)]:
-        A = lltype.GcArray(RESTYPE)
         for tsc in [False, True]:
             c2 = GcCache(tsc)
             descr1 = get_call_descr(c2, [], RESTYPE)
@@ -379,7 +381,6 @@
     descr3i = get_array_descr(c0, lltype.GcArray(lltype.Char))
     assert descr3i.repr_of_descr() == '<CharArrayDescr>'
     #
-    cache = {}
     descr4 = get_call_descr(c0, [lltype.Char, lltype.Ptr(S)], lltype.Ptr(S))
     assert 'GcPtrCallDescr' in descr4.repr_of_descr()
     #
@@ -412,10 +413,10 @@
     ARGS = [lltype.Float, lltype.Ptr(ARRAY)]
     RES = lltype.Float
 
-    def f(a, b):
+    def f2(a, b):
         return float(b[0]) + a
 
-    fnptr = llhelper(lltype.Ptr(lltype.FuncType(ARGS, RES)), f)
+    fnptr = llhelper(lltype.Ptr(lltype.FuncType(ARGS, RES)), f2)
     descr2 = get_call_descr(c0, ARGS, RES)
     a = lltype.malloc(ARRAY, 3)
     opaquea = lltype.cast_opaque_ptr(llmemory.GCREF, a)
diff --git a/pypy/jit/backend/llsupport/test/test_gc.py b/pypy/jit/backend/llsupport/test/test_gc.py
--- a/pypy/jit/backend/llsupport/test/test_gc.py
+++ b/pypy/jit/backend/llsupport/test/test_gc.py
@@ -247,12 +247,14 @@
         self.record = []
 
     def do_malloc_fixedsize_clear(self, RESTYPE, type_id, size,
-                                  has_finalizer, contains_weakptr):
+                                  has_finalizer, has_light_finalizer,
+                                  contains_weakptr):
         assert not contains_weakptr
+        assert not has_finalizer           # in these tests
+        assert not has_light_finalizer     # in these tests
         p = llmemory.raw_malloc(size)
         p = llmemory.cast_adr_to_ptr(p, RESTYPE)
-        flags = int(has_finalizer) << 16
-        tid = llop.combine_ushort(lltype.Signed, type_id, flags)
+        tid = llop.combine_ushort(lltype.Signed, type_id, 0)
         self.record.append(("fixedsize", repr(size), tid, p))
         return p
 
diff --git a/pypy/jit/backend/model.py b/pypy/jit/backend/model.py
--- a/pypy/jit/backend/model.py
+++ b/pypy/jit/backend/model.py
@@ -1,5 +1,5 @@
 from pypy.rlib.debug import debug_start, debug_print, debug_stop
-from pypy.jit.metainterp import history, compile
+from pypy.jit.metainterp import history
 
 
 class AbstractCPU(object):
@@ -213,6 +213,10 @@
     def typedescrof(TYPE):
         raise NotImplementedError
 
+    @staticmethod
+    def interiorfielddescrof(A, fieldname):
+        raise NotImplementedError
+
     # ---------- the backend-dependent operations ----------
 
     # lltype specific operations
diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -5,7 +5,7 @@
                                          BoxInt, Box, BoxPtr,
                                          LoopToken,
                                          ConstInt, ConstPtr,
-                                         BoxObj, Const,
+                                         BoxObj,
                                          ConstObj, BoxFloat, ConstFloat)
 from pypy.jit.metainterp.resoperation import ResOperation, rop
 from pypy.jit.metainterp.typesystem import deref
@@ -111,7 +111,7 @@
         self.cpu.set_future_value_int(0, 2)
         fail = self.cpu.execute_token(looptoken)
         res = self.cpu.get_latest_value_int(0)
-        assert res == 3        
+        assert res == 3
         assert fail.identifier == 1
 
     def test_compile_loop(self):
@@ -127,7 +127,7 @@
             ]
         inputargs = [i0]
         operations[2].setfailargs([i1])
-        
+
         self.cpu.compile_loop(inputargs, operations, looptoken)
         self.cpu.set_future_value_int(0, 2)
         fail = self.cpu.execute_token(looptoken)
@@ -148,7 +148,7 @@
             ]
         inputargs = [i0]
         operations[2].setfailargs([None, None, i1, None])
-        
+
         self.cpu.compile_loop(inputargs, operations, looptoken)
         self.cpu.set_future_value_int(0, 2)
         fail = self.cpu.execute_token(looptoken)
@@ -372,7 +372,7 @@
         for opnum, boxargs, retvalue in get_int_tests():
             res = self.execute_operation(opnum, boxargs, 'int')
             assert res.value == retvalue
-        
+
     def test_float_operations(self):
         from pypy.jit.metainterp.test.test_executor import get_float_tests
         for opnum, boxargs, rettype, retvalue in get_float_tests(self.cpu):
@@ -438,7 +438,7 @@
 
     def test_ovf_operations_reversed(self):
         self.test_ovf_operations(reversed=True)
-        
+
     def test_bh_call(self):
         cpu = self.cpu
         #
@@ -503,7 +503,7 @@
                                          [funcbox, BoxInt(num), BoxInt(num)],
                                          'int', descr=dyn_calldescr)
             assert res.value == 2 * num
-            
+
 
         if cpu.supports_floats:
             def func(f0, f1, f2, f3, f4, f5, f6, i0, i1, f7, f8, f9):
@@ -543,7 +543,7 @@
         funcbox = self.get_funcbox(self.cpu, func_ptr)
         res = self.execute_operation(rop.CALL, [funcbox] + map(BoxInt, args), 'int', descr=calldescr)
         assert res.value == func(*args)
-        
+
     def test_call_stack_alignment(self):
         # test stack alignment issues, notably for Mac OS/X.
         # also test the ordering of the arguments.
@@ -615,7 +615,7 @@
         res = self.execute_operation(rop.GETFIELD_GC, [t_box],
                                      'int', descr=shortdescr)
         assert res.value == 1331
-        
+
         #
         u_box, U_box = self.alloc_instance(self.U)
         fielddescr2 = self.cpu.fielddescrof(self.S, 'next')
@@ -695,7 +695,7 @@
 
     def test_failing_guard_class(self):
         t_box, T_box = self.alloc_instance(self.T)
-        u_box, U_box = self.alloc_instance(self.U)        
+        u_box, U_box = self.alloc_instance(self.U)
         null_box = self.null_instance()
         for opname, args in [(rop.GUARD_CLASS, [t_box, U_box]),
                              (rop.GUARD_CLASS, [u_box, T_box]),
@@ -787,7 +787,7 @@
         r = self.execute_operation(rop.GETARRAYITEM_GC, [a_box, BoxInt(3)],
                                    'int', descr=arraydescr)
         assert r.value == 160
-        
+
         #
         if isinstance(A, lltype.GcArray):
             A = lltype.Ptr(A)
@@ -880,6 +880,73 @@
                                    'int', descr=arraydescr)
         assert r.value == 7441
 
+    def test_array_of_structs(self):
+        TP = lltype.GcStruct('x')
+        ITEM = lltype.Struct('x',
+                             ('vs', lltype.Signed),
+                             ('vu', lltype.Unsigned),
+                             ('vsc', rffi.SIGNEDCHAR),
+                             ('vuc', rffi.UCHAR),
+                             ('vss', rffi.SHORT),
+                             ('vus', rffi.USHORT),
+                             ('vsi', rffi.INT),
+                             ('vui', rffi.UINT),
+                             ('k', lltype.Float),
+                             ('p', lltype.Ptr(TP)))
+        a_box, A = self.alloc_array_of(ITEM, 15)
+        s_box, S = self.alloc_instance(TP)
+        kdescr = self.cpu.interiorfielddescrof(A, 'k')
+        pdescr = self.cpu.interiorfielddescrof(A, 'p')
+        self.execute_operation(rop.SETINTERIORFIELD_GC, [a_box, BoxInt(3),
+                                                         boxfloat(1.5)],
+                               'void', descr=kdescr)
+        f = self.cpu.bh_getinteriorfield_gc_f(a_box.getref_base(), 3, kdescr)
+        assert longlong.getrealfloat(f) == 1.5
+        self.cpu.bh_setinteriorfield_gc_f(a_box.getref_base(), 3, kdescr, longlong.getfloatstorage(2.5))
+        r = self.execute_operation(rop.GETINTERIORFIELD_GC, [a_box, BoxInt(3)],
+                                   'float', descr=kdescr)
+        assert r.getfloat() == 2.5
+        #
+        NUMBER_FIELDS = [('vs', lltype.Signed),
+                         ('vu', lltype.Unsigned),
+                         ('vsc', rffi.SIGNEDCHAR),
+                         ('vuc', rffi.UCHAR),
+                         ('vss', rffi.SHORT),
+                         ('vus', rffi.USHORT),
+                         ('vsi', rffi.INT),
+                         ('vui', rffi.UINT)]
+        for name, TYPE in NUMBER_FIELDS[::-1]:
+            vdescr = self.cpu.interiorfielddescrof(A, name)
+            self.execute_operation(rop.SETINTERIORFIELD_GC, [a_box, BoxInt(3),
+                                                             BoxInt(-15)],
+                                   'void', descr=vdescr)
+        for name, TYPE in NUMBER_FIELDS:
+            vdescr = self.cpu.interiorfielddescrof(A, name)
+            i = self.cpu.bh_getinteriorfield_gc_i(a_box.getref_base(), 3,
+                                                  vdescr)
+            assert i == rffi.cast(lltype.Signed, rffi.cast(TYPE, -15))
+        for name, TYPE in NUMBER_FIELDS[::-1]:
+            vdescr = self.cpu.interiorfielddescrof(A, name)
+            self.cpu.bh_setinteriorfield_gc_i(a_box.getref_base(), 3,
+                                              vdescr, -25)
+        for name, TYPE in NUMBER_FIELDS:
+            vdescr = self.cpu.interiorfielddescrof(A, name)
+            r = self.execute_operation(rop.GETINTERIORFIELD_GC,
+                                       [a_box, BoxInt(3)],
+                                       'int', descr=vdescr)
+            assert r.getint() == rffi.cast(lltype.Signed, rffi.cast(TYPE, -25))
+        #
+        self.execute_operation(rop.SETINTERIORFIELD_GC, [a_box, BoxInt(4),
+                                                         s_box],
+                               'void', descr=pdescr)
+        r = self.cpu.bh_getinteriorfield_gc_r(a_box.getref_base(), 4, pdescr)
+        assert r == s_box.getref_base()
+        self.cpu.bh_setinteriorfield_gc_r(a_box.getref_base(), 3, pdescr,
+                                          s_box.getref_base())
+        r = self.execute_operation(rop.GETINTERIORFIELD_GC, [a_box, BoxInt(3)],
+                                   'ref', descr=pdescr)
+        assert r.getref_base() == s_box.getref_base()
+
     def test_string_basic(self):
         s_box = self.alloc_string("hello\xfe")
         r = self.execute_operation(rop.STRLEN, [s_box], 'int')
@@ -1402,7 +1469,7 @@
         addr = llmemory.cast_ptr_to_adr(func_ptr)
         return ConstInt(heaptracker.adr2int(addr))
 
-    
+
     MY_VTABLE = rclass.OBJECT_VTABLE    # for tests only
 
     S = lltype.GcForwardReference()
@@ -1439,7 +1506,6 @@
         return BoxPtr(lltype.nullptr(llmemory.GCREF.TO))
 
     def alloc_array_of(self, ITEM, length):
-        cpu = self.cpu
         A = lltype.GcArray(ITEM)
         a = lltype.malloc(A, length)
         a_box = BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, a))
@@ -1468,20 +1534,16 @@
         return u''.join(u.chars)
 
 
-    def test_casts(self):
-        py.test.skip("xxx fix or kill")
-        from pypy.rpython.lltypesystem import lltype, llmemory
-        TP = lltype.GcStruct('x')
-        x = lltype.malloc(TP)        
-        x = lltype.cast_opaque_ptr(llmemory.GCREF, x)
+    def test_cast_int_to_ptr(self):
+        res = self.execute_operation(rop.CAST_INT_TO_PTR,
+                                     [BoxInt(-17)],  'ref').value
+        assert lltype.cast_ptr_to_int(res) == -17
+
+    def test_cast_ptr_to_int(self):
+        x = lltype.cast_int_to_ptr(llmemory.GCREF, -19)
         res = self.execute_operation(rop.CAST_PTR_TO_INT,
-                                     [BoxPtr(x)],  'int').value
-        expected = self.cpu.cast_adr_to_int(llmemory.cast_ptr_to_adr(x))
-        assert rffi.get_real_int(res) == rffi.get_real_int(expected)
-        res = self.execute_operation(rop.CAST_PTR_TO_INT,
-                                     [ConstPtr(x)],  'int').value
-        expected = self.cpu.cast_adr_to_int(llmemory.cast_ptr_to_adr(x))
-        assert rffi.get_real_int(res) == rffi.get_real_int(expected)
+                                     [BoxPtr(x)], 'int').value
+        assert res == -19
 
     def test_ooops_non_gc(self):
         x = lltype.malloc(lltype.Struct('x'), flavor='raw')
@@ -2299,13 +2361,6 @@
         #
         cpu.bh_strsetitem(x, 4, ord('/'))
         assert str.chars[4] == '/'
-        #
-##        x = cpu.bh_newstr(5)
-##        y = cpu.bh_cast_ptr_to_int(x)
-##        z = cpu.bh_cast_ptr_to_int(x)
-##        y = rffi.get_real_int(y)
-##        z = rffi.get_real_int(z)
-##        assert type(y) == type(z) == int and y == z
 
     def test_sorting_of_fields(self):
         S = self.S
@@ -2329,7 +2384,7 @@
         for opname, arg, res in ops:
             self.execute_operation(opname, [arg], 'void')
             assert self.guard_failed == res
-        
+
         lltype.free(x, flavor='raw')
 
     def test_assembler_call(self):
@@ -2409,7 +2464,7 @@
         FakeJitDriverSD.portal_calldescr = self.cpu.calldescrof(
             lltype.Ptr(lltype.FuncType(ARGS, RES)), ARGS, RES,
             EffectInfo.MOST_GENERAL)
-        
+
         ops = '''
         [f0, f1]
         f2 = float_add(f0, f1)
@@ -2500,7 +2555,7 @@
         FakeJitDriverSD.portal_calldescr = self.cpu.calldescrof(
             lltype.Ptr(lltype.FuncType(ARGS, RES)), ARGS, RES,
             EffectInfo.MOST_GENERAL)
-        
+
         ops = '''
         [f0, f1]
         f2 = float_add(f0, f1)
@@ -2951,4 +3006,4 @@
 
     def alloc_unicode(self, unicode):
         py.test.skip("implement me")
-    
+
diff --git a/pypy/jit/backend/test/test_ll_random.py b/pypy/jit/backend/test/test_ll_random.py
--- a/pypy/jit/backend/test/test_ll_random.py
+++ b/pypy/jit/backend/test/test_ll_random.py
@@ -28,16 +28,27 @@
         fork.structure_types_and_vtables = self.structure_types_and_vtables
         return fork
 
-    def get_structptr_var(self, r, must_have_vtable=False, type=lltype.Struct):
+    def _choose_ptr_vars(self, from_, type, array_of_structs):
+        ptrvars = []
+        for i in range(len(from_)):
+            v, S = from_[i][:2]
+            if not isinstance(S, type):
+                continue
+            if ((isinstance(S, lltype.Array) and
+                 isinstance(S.OF, lltype.Struct)) == array_of_structs):
+                ptrvars.append((v, S))
+        return ptrvars
+
+    def get_structptr_var(self, r, must_have_vtable=False, type=lltype.Struct,
+                          array_of_structs=False):
         while True:
-            ptrvars = [(v, S) for (v, S) in self.ptrvars
-                              if isinstance(S, type)]
+            ptrvars = self._choose_ptr_vars(self.ptrvars, type,
+                                            array_of_structs)
             if ptrvars and r.random() < 0.8:
                 v, S = r.choice(ptrvars)
             else:
-                prebuilt_ptr_consts = [(v, S)
-                                 for (v, S, _) in self.prebuilt_ptr_consts
-                                 if isinstance(S, type)]
+                prebuilt_ptr_consts = self._choose_ptr_vars(
+                    self.prebuilt_ptr_consts, type, array_of_structs)
                 if prebuilt_ptr_consts and r.random() < 0.7:
                     v, S = r.choice(prebuilt_ptr_consts)
                 else:
@@ -48,7 +59,8 @@
                                                 has_vtable=must_have_vtable)
                     else:
                         # create a new constant array
-                        p = self.get_random_array(r)
+                        p = self.get_random_array(r,
+                                    must_be_array_of_structs=array_of_structs)
                     S = lltype.typeOf(p).TO
                     v = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, p))
                     self.prebuilt_ptr_consts.append((v, S,
@@ -74,7 +86,8 @@
                 TYPE = lltype.Signed
         return TYPE
 
-    def get_random_structure_type(self, r, with_vtable=None, cache=True):
+    def get_random_structure_type(self, r, with_vtable=None, cache=True,
+                                  type=lltype.GcStruct):
         if cache and self.structure_types and r.random() < 0.5:
             return r.choice(self.structure_types)
         fields = []
@@ -85,7 +98,7 @@
         for i in range(r.randrange(1, 5)):
             TYPE = self.get_random_primitive_type(r)
             fields.append(('f%d' % i, TYPE))
-        S = lltype.GcStruct('S%d' % self.counter, *fields, **kwds)
+        S = type('S%d' % self.counter, *fields, **kwds)
         self.counter += 1
         if cache:
             self.structure_types.append(S)
@@ -125,17 +138,29 @@
                 setattr(p, fieldname, rffi.cast(TYPE, r.random_integer()))
         return p
 
-    def get_random_array_type(self, r):
-        TYPE = self.get_random_primitive_type(r)
+    def get_random_array_type(self, r, can_be_array_of_struct=False,
+                              must_be_array_of_structs=False):
+        if ((can_be_array_of_struct and r.random() < 0.1) or
+            must_be_array_of_structs):
+            TYPE = self.get_random_structure_type(r, cache=False,
+                                                  type=lltype.Struct)
+        else:
+            TYPE = self.get_random_primitive_type(r)
         return lltype.GcArray(TYPE)
 
-    def get_random_array(self, r):
-        A = self.get_random_array_type(r)
+    def get_random_array(self, r, must_be_array_of_structs=False):
+        A = self.get_random_array_type(r,
+                           must_be_array_of_structs=must_be_array_of_structs)
         length = (r.random_integer() // 15) % 300  # length: between 0 and 299
                                                    # likely to be small
         p = lltype.malloc(A, length)
-        for i in range(length):
-            p[i] = rffi.cast(A.OF, r.random_integer())
+        if isinstance(A.OF, lltype.Primitive):
+            for i in range(length):
+                p[i] = rffi.cast(A.OF, r.random_integer())
+        else:
+            for i in range(length):
+                for fname, TP in A.OF._flds.iteritems():
+                    setattr(p[i], fname, rffi.cast(TP, r.random_integer()))
         return p
 
     def get_index(self, length, r):
@@ -155,8 +180,16 @@
                     dic[fieldname] = getattr(p, fieldname)
         else:
             assert isinstance(S, lltype.Array)
-            for i in range(len(p)):
-                dic[i] = p[i]
+            if isinstance(S.OF, lltype.Struct):
+                for i in range(len(p)):
+                    item = p[i]
+                    s1 = {}
+                    for fieldname in S.OF._names:
+                        s1[fieldname] = getattr(item, fieldname)
+                    dic[i] = s1
+            else:
+                for i in range(len(p)):
+                    dic[i] = p[i]
         return dic
 
     def print_loop_prebuilt(self, names, writevar, s):
@@ -220,7 +253,7 @@
 
 class GetFieldOperation(test_random.AbstractOperation):
     def field_descr(self, builder, r):
-        v, S = builder.get_structptr_var(r)
+        v, S = builder.get_structptr_var(r, )
         names = S._names
         if names[0] == 'parent':
             names = names[1:]
@@ -239,6 +272,28 @@
                 continue
             break
 
+class GetInteriorFieldOperation(test_random.AbstractOperation):
+    def field_descr(self, builder, r):
+        v, A = builder.get_structptr_var(r, type=lltype.Array,
+                                         array_of_structs=True)
+        array = v.getref(lltype.Ptr(A))
+        v_index = builder.get_index(len(array), r)
+        name = r.choice(A.OF._names)
+        descr = builder.cpu.interiorfielddescrof(A, name)
+        descr._random_info = 'cpu.interiorfielddescrof(%s, %r)' % (A.OF._name,
+                                                                   name)
+        TYPE = getattr(A.OF, name)
+        return v, v_index, descr, TYPE
+
+    def produce_into(self, builder, r):
+        while True:
+            try:
+                v, v_index, descr, _ = self.field_descr(builder, r)
+                self.put(builder, [v, v_index], descr)
+            except lltype.UninitializedMemoryAccess:
+                continue
+            break
+
 class SetFieldOperation(GetFieldOperation):
     def produce_into(self, builder, r):
         v, descr, TYPE = self.field_descr(builder, r)
@@ -251,6 +306,18 @@
                 break
         builder.do(self.opnum, [v, w], descr)
 
+class SetInteriorFieldOperation(GetInteriorFieldOperation):
+    def produce_into(self, builder, r):
+        v, v_index, descr, TYPE = self.field_descr(builder, r)
+        while True:
+            if r.random() < 0.3:
+                w = ConstInt(r.random_integer())
+            else:
+                w = r.choice(builder.intvars)
+            if rffi.cast(lltype.Signed, rffi.cast(TYPE, w.value)) == w.value:
+                break
+        builder.do(self.opnum, [v, v_index, w], descr)
+
 class NewOperation(test_random.AbstractOperation):
     def size_descr(self, builder, S):
         descr = builder.cpu.sizeof(S)
@@ -306,7 +373,7 @@
 
 class NewArrayOperation(ArrayOperation):
     def produce_into(self, builder, r):
-        A = builder.get_random_array_type(r)
+        A = builder.get_random_array_type(r, can_be_array_of_struct=True)
         v_size = builder.get_index(300, r)
         v_ptr = builder.do(self.opnum, [v_size], self.array_descr(builder, A))
         builder.ptrvars.append((v_ptr, A))
@@ -586,7 +653,9 @@
 for i in range(4):      # make more common
     OPERATIONS.append(GetFieldOperation(rop.GETFIELD_GC))
     OPERATIONS.append(GetFieldOperation(rop.GETFIELD_GC))
+    OPERATIONS.append(GetInteriorFieldOperation(rop.GETINTERIORFIELD_GC))
     OPERATIONS.append(SetFieldOperation(rop.SETFIELD_GC))
+    OPERATIONS.append(SetInteriorFieldOperation(rop.SETINTERIORFIELD_GC))
     OPERATIONS.append(NewOperation(rop.NEW))
     OPERATIONS.append(NewOperation(rop.NEW_WITH_VTABLE))
 
diff --git a/pypy/jit/backend/test/test_random.py b/pypy/jit/backend/test/test_random.py
--- a/pypy/jit/backend/test/test_random.py
+++ b/pypy/jit/backend/test/test_random.py
@@ -595,6 +595,10 @@
             for name, value in fields.items():
                 if isinstance(name, str):
                     setattr(container, name, value)
+                elif isinstance(value, dict):
+                    item = container.getitem(name)
+                    for key1, value1 in value.items():
+                        setattr(item, key1, value1)
                 else:
                     container.setitem(name, value)
 
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1,7 +1,7 @@
 import sys, os
 from pypy.jit.backend.llsupport import symbolic
 from pypy.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper
-from pypy.jit.metainterp.history import Const, Box, BoxInt, BoxPtr, BoxFloat
+from pypy.jit.metainterp.history import Const, Box, BoxInt, ConstInt
 from pypy.jit.metainterp.history import (AbstractFailDescr, INT, REF, FLOAT,
                                          LoopToken)
 from pypy.rpython.lltypesystem import lltype, rffi, rstr, llmemory
@@ -36,7 +36,6 @@
 from pypy.rlib import rgc
 from pypy.rlib.clibffi import FFI_DEFAULT_ABI
 from pypy.jit.backend.x86.jump import remap_frame_layout
-from pypy.jit.metainterp.history import ConstInt, BoxInt
 from pypy.jit.codewriter.effectinfo import EffectInfo
 from pypy.jit.codewriter import longlong
 
@@ -729,8 +728,8 @@
         # Also, make sure this is consistent with FRAME_FIXED_SIZE.
         self.mc.PUSH_r(ebp.value)
         self.mc.MOV_rr(ebp.value, esp.value)
-        for regloc in self.cpu.CALLEE_SAVE_REGISTERS:
-            self.mc.PUSH_r(regloc.value)
+        for loc in self.cpu.CALLEE_SAVE_REGISTERS:
+            self.mc.PUSH_r(loc.value)
 
         gcrootmap = self.cpu.gc_ll_descr.gcrootmap
         if gcrootmap and gcrootmap.is_shadow_stack:
@@ -994,7 +993,7 @@
         effectinfo = op.getdescr().get_extra_info()
         oopspecindex = effectinfo.oopspecindex
         genop_llong_list[oopspecindex](self, op, arglocs, resloc)
-        
+
     def regalloc_perform_math(self, op, arglocs, resloc):
         effectinfo = op.getdescr().get_extra_info()
         oopspecindex = effectinfo.oopspecindex
@@ -1277,8 +1276,8 @@
     genop_int_ne = _cmpop("NE", "NE")
     genop_int_gt = _cmpop("G", "L")
     genop_int_ge = _cmpop("GE", "LE")
-    genop_ptr_eq = genop_int_eq
-    genop_ptr_ne = genop_int_ne
+    genop_ptr_eq = genop_instance_ptr_eq = genop_int_eq
+    genop_ptr_ne = genop_instance_ptr_ne = genop_int_ne
 
     genop_float_lt = _cmpop_float('B', 'A')
     genop_float_le = _cmpop_float('BE', 'AE')
@@ -1298,8 +1297,8 @@
     genop_guard_int_ne = _cmpop_guard("NE", "NE", "E", "E")
     genop_guard_int_gt = _cmpop_guard("G", "L", "LE", "GE")
     genop_guard_int_ge = _cmpop_guard("GE", "LE", "L", "G")
-    genop_guard_ptr_eq = genop_guard_int_eq
-    genop_guard_ptr_ne = genop_guard_int_ne
+    genop_guard_ptr_eq = genop_guard_instance_ptr_eq = genop_guard_int_eq
+    genop_guard_ptr_ne = genop_guard_instance_ptr_ne = genop_guard_int_ne
 
     genop_guard_uint_gt = _cmpop_guard("A", "B", "BE", "AE")
     genop_guard_uint_lt = _cmpop_guard("B", "A", "AE", "BE")
@@ -1311,7 +1310,7 @@
     genop_guard_float_eq = _cmpop_guard_float("E", "E", "NE","NE")
     genop_guard_float_gt = _cmpop_guard_float("A", "B", "BE","AE")
     genop_guard_float_ge = _cmpop_guard_float("AE","BE", "B", "A")
-    
+
     def genop_math_sqrt(self, op, arglocs, resloc):
         self.mc.SQRTSD(arglocs[0], resloc)
 
@@ -1387,7 +1386,8 @@
 
     def genop_same_as(self, op, arglocs, resloc):
         self.mov(arglocs[0], resloc)
-    #genop_cast_ptr_to_int = genop_same_as
+    genop_cast_ptr_to_int = genop_same_as
+    genop_cast_int_to_ptr = genop_same_as
 
     def genop_int_mod(self, op, arglocs, resloc):
         if IS_X86_32:
@@ -1596,12 +1596,43 @@
     genop_getarrayitem_gc_pure = genop_getarrayitem_gc
     genop_getarrayitem_raw = genop_getarrayitem_gc
 
+    def _get_interiorfield_addr(self, temp_loc, index_loc, itemsize_loc,
+                                base_loc, ofs_loc):
+        assert isinstance(itemsize_loc, ImmedLoc)
+        if isinstance(index_loc, ImmedLoc):
+            temp_loc = imm(index_loc.value * itemsize_loc.value)
+        else:
+            # XXX should not use IMUL in most cases
+            assert isinstance(temp_loc, RegLoc)
+            assert isinstance(index_loc, RegLoc)
+            self.mc.IMUL_rri(temp_loc.value, index_loc.value,
+                             itemsize_loc.value)
+        assert isinstance(ofs_loc, ImmedLoc)
+        return AddressLoc(base_loc, temp_loc, 0, ofs_loc.value)
+
+    def genop_getinteriorfield_gc(self, op, arglocs, resloc):
+        (base_loc, ofs_loc, itemsize_loc, fieldsize_loc,
+            index_loc, sign_loc) = arglocs
+        src_addr = self._get_interiorfield_addr(resloc, index_loc,
+                                                itemsize_loc, base_loc,
+                                                ofs_loc)
+        self.load_from_mem(resloc, src_addr, fieldsize_loc, sign_loc)
+
+
     def genop_discard_setfield_gc(self, op, arglocs):
         base_loc, ofs_loc, size_loc, value_loc = arglocs
         assert isinstance(size_loc, ImmedLoc)
         dest_addr = AddressLoc(base_loc, ofs_loc)
         self.save_into_mem(dest_addr, value_loc, size_loc)
 
+    def genop_discard_setinteriorfield_gc(self, op, arglocs):
+        (base_loc, ofs_loc, itemsize_loc, fieldsize_loc,
+            index_loc, temp_loc, value_loc) = arglocs
+        dest_addr = self._get_interiorfield_addr(temp_loc, index_loc,
+                                                 itemsize_loc, base_loc,
+                                                 ofs_loc)
+        self.save_into_mem(dest_addr, value_loc, fieldsize_loc)
+
     def genop_discard_setarrayitem_gc(self, op, arglocs):
         base_loc, ofs_loc, value_loc, size_loc, baseofs = arglocs
         assert isinstance(baseofs, ImmedLoc)
diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py
--- a/pypy/jit/backend/x86/regalloc.py
+++ b/pypy/jit/backend/x86/regalloc.py
@@ -7,7 +7,7 @@
                                          ResOperation, BoxPtr, ConstFloat,
                                          BoxFloat, LoopToken, INT, REF, FLOAT)
 from pypy.jit.backend.x86.regloc import *
-from pypy.rpython.lltypesystem import lltype, ll2ctypes, rffi, rstr
+from pypy.rpython.lltypesystem import lltype, rffi, rstr
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.rlib import rgc
 from pypy.jit.backend.llsupport import symbolic
@@ -17,11 +17,12 @@
 from pypy.jit.metainterp.resoperation import rop
 from pypy.jit.backend.llsupport.descr import BaseFieldDescr, BaseArrayDescr
 from pypy.jit.backend.llsupport.descr import BaseCallDescr, BaseSizeDescr
+from pypy.jit.backend.llsupport.descr import InteriorFieldDescr
 from pypy.jit.backend.llsupport.regalloc import FrameManager, RegisterManager,\
      TempBox
 from pypy.jit.backend.x86.arch import WORD, FRAME_FIXED_SIZE
 from pypy.jit.backend.x86.arch import IS_X86_32, IS_X86_64, MY_COPY_OF_REGS
-from pypy.rlib.rarithmetic import r_longlong, r_uint
+from pypy.rlib.rarithmetic import r_longlong
 
 class X86RegisterManager(RegisterManager):
 
@@ -433,7 +434,7 @@
             if self.can_merge_with_next_guard(op, i, operations):
                 oplist_with_guard[op.getopnum()](self, op, operations[i + 1])
                 i += 1
-            elif not we_are_translated() and op.getopnum() == -124: 
+            elif not we_are_translated() and op.getopnum() == -124:
                 self._consider_force_spill(op)
             else:
                 oplist[op.getopnum()](self, op)
@@ -650,8 +651,8 @@
     consider_uint_lt = _consider_compop
     consider_uint_le = _consider_compop
     consider_uint_ge = _consider_compop
-    consider_ptr_eq = _consider_compop
-    consider_ptr_ne = _consider_compop
+    consider_ptr_eq = consider_instance_ptr_eq = _consider_compop
+    consider_ptr_ne = consider_instance_ptr_ne = _consider_compop
 
     def _consider_float_op(self, op):
         loc1 = self.xrm.loc(op.getarg(1))
@@ -815,7 +816,7 @@
         save_all_regs = guard_not_forced_op is not None
         self.xrm.before_call(force_store, save_all_regs=save_all_regs)
         if not save_all_regs:
-            gcrootmap = gc_ll_descr = self.assembler.cpu.gc_ll_descr.gcrootmap
+            gcrootmap = self.assembler.cpu.gc_ll_descr.gcrootmap
             if gcrootmap and gcrootmap.is_shadow_stack:
                 save_all_regs = 2
         self.rm.before_call(force_store, save_all_regs=save_all_regs)
@@ -972,74 +973,27 @@
             return self._call(op, arglocs)
 
     def consider_newstr(self, op):
-        gc_ll_descr = self.assembler.cpu.gc_ll_descr
-        if gc_ll_descr.get_funcptr_for_newstr is not None:
-            # framework GC
-            loc = self.loc(op.getarg(0))
-            return self._call(op, [loc])
-        # boehm GC (XXX kill the following code at some point)
-        ofs_items, itemsize, ofs = symbolic.get_array_token(rstr.STR, self.translate_support_code)
-        assert itemsize == 1
-        return self._malloc_varsize(ofs_items, ofs, 0, op.getarg(0),
-                                    op.result)
+        loc = self.loc(op.getarg(0))
+        return self._call(op, [loc])
 
     def consider_newunicode(self, op):
-        gc_ll_descr = self.assembler.cpu.gc_ll_descr
-        if gc_ll_descr.get_funcptr_for_newunicode is not None:
-            # framework GC
-            loc = self.loc(op.getarg(0))
-            return self._call(op, [loc])
-        # boehm GC (XXX kill the following code at some point)
-        ofs_items, _, ofs = symbolic.get_array_token(rstr.UNICODE,
-                                                   self.translate_support_code)
-        scale = self._get_unicode_item_scale()
-        return self._malloc_varsize(ofs_items, ofs, scale, op.getarg(0),
-                                    op.result)
-
-    def _malloc_varsize(self, ofs_items, ofs_length, scale, v, res_v):
-        # XXX kill this function at some point
-        if isinstance(v, Box):
-            loc = self.rm.make_sure_var_in_reg(v, [v])
-            tempbox = TempBox()
-            other_loc = self.rm.force_allocate_reg(tempbox, [v])
-            self.assembler.load_effective_addr(loc, ofs_items,scale, other_loc)
-        else:
-            tempbox = None
-            other_loc = imm(ofs_items + (v.getint() << scale))
-        self._call(ResOperation(rop.NEW, [], res_v),
-                   [other_loc], [v])
-        loc = self.rm.make_sure_var_in_reg(v, [res_v])
-        assert self.loc(res_v) == eax
-        # now we have to reload length to some reasonable place
-        self.rm.possibly_free_var(v)
-        if tempbox is not None:
-            self.rm.possibly_free_var(tempbox)
-        self.PerformDiscard(ResOperation(rop.SETFIELD_GC, [None, None], None),
-                            [eax, imm(ofs_length), imm(WORD), loc])
+        loc = self.loc(op.getarg(0))
+        return self._call(op, [loc])
 
     def consider_new_array(self, op):
         gc_ll_descr = self.assembler.cpu.gc_ll_descr
-        if gc_ll_descr.get_funcptr_for_newarray is not None:
-            # framework GC
-            box_num_elem = op.getarg(0)
-            if isinstance(box_num_elem, ConstInt):
-                num_elem = box_num_elem.value
-                if gc_ll_descr.can_inline_malloc_varsize(op.getdescr(),
-                                                         num_elem):
-                    self.fastpath_malloc_varsize(op, op.getdescr(), num_elem)
-                    return
-            args = self.assembler.cpu.gc_ll_descr.args_for_new_array(
-                op.getdescr())
-            arglocs = [imm(x) for x in args]
-            arglocs.append(self.loc(box_num_elem))
-            self._call(op, arglocs)
-            return
-        # boehm GC (XXX kill the following code at some point)
-        itemsize, basesize, ofs_length, _, _ = (
-            self._unpack_arraydescr(op.getdescr()))
-        scale_of_field = _get_scale(itemsize)
-        self._malloc_varsize(basesize, ofs_length, scale_of_field,
-                             op.getarg(0), op.result)
+        box_num_elem = op.getarg(0)
+        if isinstance(box_num_elem, ConstInt):
+            num_elem = box_num_elem.value
+            if gc_ll_descr.can_inline_malloc_varsize(op.getdescr(),
+                                                     num_elem):
+                self.fastpath_malloc_varsize(op, op.getdescr(), num_elem)
+                return
+        args = self.assembler.cpu.gc_ll_descr.args_for_new_array(
+            op.getdescr())
+        arglocs = [imm(x) for x in args]
+        arglocs.append(self.loc(box_num_elem))
+        self._call(op, arglocs)
 
     def _unpack_arraydescr(self, arraydescr):
         assert isinstance(arraydescr, BaseArrayDescr)
@@ -1058,6 +1012,16 @@
         sign = fielddescr.is_field_signed()
         return imm(ofs), imm(size), ptr, sign
 
+    def _unpack_interiorfielddescr(self, descr):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs = arraydescr.get_base_size(self.translate_support_code)
+        itemsize = arraydescr.get_item_size(self.translate_support_code)
+        fieldsize = descr.fielddescr.get_field_size(self.translate_support_code)
+        sign = descr.fielddescr.is_field_signed()
+        ofs += descr.fielddescr.offset
+        return imm(ofs), imm(itemsize), imm(fieldsize), sign
+
     def consider_setfield_gc(self, op):
         ofs_loc, size_loc, _, _ = self._unpack_fielddescr(op.getdescr())
         assert isinstance(size_loc, ImmedLoc)
@@ -1074,6 +1038,35 @@
 
     consider_setfield_raw = consider_setfield_gc
 
+    def consider_setinteriorfield_gc(self, op):
+        t = self._unpack_interiorfielddescr(op.getdescr())
+        ofs, itemsize, fieldsize, _ = t
+        args = op.getarglist()
+        if fieldsize.value == 1:
+            need_lower_byte = True
+        else:
+            need_lower_byte = False
+        box_base, box_index, box_value = args
+        base_loc = self.rm.make_sure_var_in_reg(box_base, args)
+        index_loc = self.rm.make_sure_var_in_reg(box_index, args)
+        value_loc = self.make_sure_var_in_reg(box_value, args,
+                                              need_lower_byte=need_lower_byte)
+        # If 'index_loc' is not an immediate, then we need a 'temp_loc' that
+        # is a register whose value will be destroyed.  It's fine to destroy
+        # the same register as 'index_loc', but not the other ones.
+        self.rm.possibly_free_var(box_index)
+        if not isinstance(index_loc, ImmedLoc):
+            tempvar = TempBox()
+            temp_loc = self.rm.force_allocate_reg(tempvar, [box_base,
+                                                            box_value])
+            self.rm.possibly_free_var(tempvar)
+        else:
+            temp_loc = None
+        self.rm.possibly_free_var(box_base)
+        self.possibly_free_var(box_value)
+        self.PerformDiscard(op, [base_loc, ofs, itemsize, fieldsize,
+                                 index_loc, temp_loc, value_loc])
+
     def consider_strsetitem(self, op):
         args = op.getarglist()
         base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
@@ -1135,6 +1128,25 @@
     consider_getarrayitem_raw = consider_getarrayitem_gc
     consider_getarrayitem_gc_pure = consider_getarrayitem_gc
 
+    def consider_getinteriorfield_gc(self, op):
+        t = self._unpack_interiorfielddescr(op.getdescr())
+        ofs, itemsize, fieldsize, sign = t
+        if sign:
+            sign_loc = imm1
+        else:
+            sign_loc = imm0
+        args = op.getarglist()
+        base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
+        index_loc = self.rm.make_sure_var_in_reg(op.getarg(1), args)
+        # 'base' and 'index' are put in two registers (or one if 'index'
+        # is an immediate).  'result' can be in the same register as
+        # 'index' but must be in a different register than 'base'.
+        self.rm.possibly_free_var(op.getarg(1))
+        result_loc = self.force_allocate_reg(op.result, [op.getarg(0)])
+        self.rm.possibly_free_var(op.getarg(0))
+        self.Perform(op, [base_loc, ofs, itemsize, fieldsize,
+                          index_loc, sign_loc], result_loc)
+
     def consider_int_is_true(self, op, guard_op):
         # doesn't need arg to be in a register
         argloc = self.loc(op.getarg(0))
@@ -1152,7 +1164,8 @@
         self.possibly_free_var(op.getarg(0))
         resloc = self.force_allocate_reg(op.result)
         self.Perform(op, [argloc], resloc)
-    #consider_cast_ptr_to_int = consider_same_as
+    consider_cast_ptr_to_int = consider_same_as
+    consider_cast_int_to_ptr = consider_same_as
 
     def consider_strlen(self, op):
         args = op.getarglist()
@@ -1240,7 +1253,6 @@
         self.rm.possibly_free_var(srcaddr_box)
 
     def _gen_address_inside_string(self, baseloc, ofsloc, resloc, is_unicode):
-        cpu = self.assembler.cpu
         if is_unicode:
             ofs_items, _, _ = symbolic.get_array_token(rstr.UNICODE,
                                                   self.translate_support_code)
@@ -1299,7 +1311,7 @@
         tmpreg = X86RegisterManager.all_regs[0]
         tmploc = self.rm.force_allocate_reg(box, selected_reg=tmpreg)
         xmmtmp = X86XMMRegisterManager.all_regs[0]
-        xmmtmploc = self.xrm.force_allocate_reg(box1, selected_reg=xmmtmp)
+        self.xrm.force_allocate_reg(box1, selected_reg=xmmtmp)
         # Part about non-floats
         # XXX we don't need a copy, we only just the original list
         src_locations1 = [self.loc(op.getarg(i)) for i in range(op.numargs())
@@ -1379,7 +1391,7 @@
     return lambda self, op: fn(self, op, None)
 
 def is_comparison_or_ovf_op(opnum):
-    from pypy.jit.metainterp.resoperation import opclasses, AbstractResOp
+    from pypy.jit.metainterp.resoperation import opclasses
     cls = opclasses[opnum]
     # hack hack: in theory they are instance method, but they don't use
     # any instance field, we can use a fake object
diff --git a/pypy/jit/backend/x86/test/test_del.py b/pypy/jit/backend/x86/test/test_del.py
--- a/pypy/jit/backend/x86/test/test_del.py
+++ b/pypy/jit/backend/x86/test/test_del.py
@@ -1,5 +1,4 @@
 
-import py
 from pypy.jit.backend.x86.test.test_basic import Jit386Mixin
 from pypy.jit.metainterp.test.test_del import DelTests
 
diff --git a/pypy/jit/backend/x86/test/test_dict.py b/pypy/jit/backend/x86/test/test_dict.py
new file mode 100644
--- /dev/null
+++ b/pypy/jit/backend/x86/test/test_dict.py
@@ -0,0 +1,9 @@
+
+from pypy.jit.backend.x86.test.test_basic import Jit386Mixin
+from pypy.jit.metainterp.test.test_dict import DictTests
+
+
+class TestDict(Jit386Mixin, DictTests):
+    # for the individual tests see
+    # ====> ../../../metainterp/test/test_dict.py
+    pass
diff --git a/pypy/jit/backend/x86/test/test_runner.py b/pypy/jit/backend/x86/test/test_runner.py
--- a/pypy/jit/backend/x86/test/test_runner.py
+++ b/pypy/jit/backend/x86/test/test_runner.py
@@ -31,7 +31,7 @@
 
     # for the individual tests see
     # ====> ../../test/runner_test.py
-    
+
     def setup_method(self, meth):
         self.cpu = CPU(rtyper=None, stats=FakeStats())
         self.cpu.setup_once()
@@ -69,22 +69,16 @@
 
     def test_allocations(self):
         from pypy.rpython.lltypesystem import rstr
-        
+
         allocs = [None]
         all = []
+        orig_new = self.cpu.gc_ll_descr.funcptr_for_new
         def f(size):
             allocs.insert(0, size)
-            buf = ctypes.create_string_buffer(size)
-            all.append(buf)
-            return ctypes.cast(buf, ctypes.c_void_p).value
-        func = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int)(f)
-        addr = ctypes.cast(func, ctypes.c_void_p).value
-        # ctypes produces an unsigned value. We need it to be signed for, eg,
-        # relative addressing to work properly.
-        addr = rffi.cast(lltype.Signed, addr)
-        
+            return orig_new(size)
+
         self.cpu.assembler.setup_once()
-        self.cpu.assembler.malloc_func_addr = addr
+        self.cpu.gc_ll_descr.funcptr_for_new = f
         ofs = symbolic.get_field_token(rstr.STR, 'chars', False)[0]
 
         res = self.execute_operation(rop.NEWSTR, [ConstInt(7)], 'ref')
@@ -108,7 +102,7 @@
         res = self.execute_operation(rop.NEW_ARRAY, [ConstInt(10)],
                                          'ref', descr)
         assert allocs[0] == 10*WORD + ofs + WORD
-        resbuf = self._resbuf(res)            
+        resbuf = self._resbuf(res)
         assert resbuf[ofs/WORD] == 10
 
         # ------------------------------------------------------------
@@ -116,7 +110,7 @@
         res = self.execute_operation(rop.NEW_ARRAY, [BoxInt(10)],
                                          'ref', descr)
         assert allocs[0] == 10*WORD + ofs + WORD
-        resbuf = self._resbuf(res)                        
+        resbuf = self._resbuf(res)
         assert resbuf[ofs/WORD] == 10
 
     def test_stringitems(self):
@@ -146,7 +140,7 @@
                                                      ConstInt(2), BoxInt(38)],
                                'void', descr)
         assert resbuf[itemsofs/WORD + 2] == 38
-        
+
         self.execute_operation(rop.SETARRAYITEM_GC, [res,
                                                      BoxInt(3), BoxInt(42)],
                                'void', descr)
@@ -167,7 +161,7 @@
                                                          BoxInt(2)],
                                    'int', descr)
         assert r.value == 38
-        
+
         r = self.execute_operation(rop.GETARRAYITEM_GC, [res, BoxInt(3)],
                                    'int', descr)
         assert r.value == 42
@@ -226,7 +220,7 @@
         self.execute_operation(rop.SETFIELD_GC, [res, BoxInt(1234)], 'void', ofs_i)
         i = self.execute_operation(rop.GETFIELD_GC, [res], 'int', ofs_i)
         assert i.value == 1234
-        
+
         #u = self.execute_operation(rop.GETFIELD_GC, [res, ofs_u], 'int')
         #assert u.value == 5
         self.execute_operation(rop.SETFIELD_GC, [res, ConstInt(1)], 'void',
@@ -299,7 +293,7 @@
                     else:
                         assert result != execute(self.cpu, None,
                                                  op, None, b).value
-                    
+
 
     def test_stuff_followed_by_guard(self):
         boxes = [(BoxInt(1), BoxInt(0)),
@@ -523,7 +517,7 @@
     def test_debugger_on(self):
         from pypy.tool.logparser import parse_log_file, extract_category
         from pypy.rlib import debug
-        
+
         loop = """
         [i0]
         debug_merge_point('xyz', 0)
diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -52,9 +52,11 @@
         newoperations = []
         #
         def do_rename(var, var_or_const):
+            if var.concretetype is lltype.Void:
+                renamings[var] = Constant(None, lltype.Void)
+                return
             renamings[var] = var_or_const
-            if (isinstance(var_or_const, Constant)
-                and var.concretetype != lltype.Void):
+            if isinstance(var_or_const, Constant):
                 value = var_or_const.value
                 value = lltype._cast_whatever(var.concretetype, value)
                 renamings_constants[var] = Constant(value, var.concretetype)
@@ -441,6 +443,8 @@
     rewrite_op_gc_identityhash = _do_builtin_call
     rewrite_op_gc_id           = _do_builtin_call
     rewrite_op_uint_mod        = _do_builtin_call
+    rewrite_op_cast_float_to_uint = _do_builtin_call
+    rewrite_op_cast_uint_to_float = _do_builtin_call
 
     # ----------
     # getfield/setfield/mallocs etc.
@@ -735,29 +739,54 @@
         return SpaceOperation(opname, [op.args[0]], op.result)
 
     def rewrite_op_getinteriorfield(self, op):
-        # only supports strings and unicodes
         assert len(op.args) == 3
-        assert op.args[1].value == 'chars'
         optype = op.args[0].concretetype
         if optype == lltype.Ptr(rstr.STR):
             opname = "strgetitem"
+            return SpaceOperation(opname, [op.args[0], op.args[2]], op.result)
+        elif optype == lltype.Ptr(rstr.UNICODE):
+            opname = "unicodegetitem"
+            return SpaceOperation(opname, [op.args[0], op.args[2]], op.result)
         else:
-            assert optype == lltype.Ptr(rstr.UNICODE)
-            opname = "unicodegetitem"
-        return SpaceOperation(opname, [op.args[0], op.args[2]], op.result)
+            v_inst, v_index, c_field = op.args
+            if op.result.concretetype is lltype.Void:
+                return
+            # only GcArray of Struct supported
+            assert isinstance(v_inst.concretetype.TO, lltype.GcArray)
+            STRUCT = v_inst.concretetype.TO.OF
+            assert isinstance(STRUCT, lltype.Struct)
+            descr = self.cpu.interiorfielddescrof(v_inst.concretetype.TO,
+                                                  c_field.value)
+            args = [v_inst, v_index, descr]
+            kind = getkind(op.result.concretetype)[0]
+            return SpaceOperation('getinteriorfield_gc_%s' % kind, args,
+                                  op.result)
 
     def rewrite_op_setinteriorfield(self, op):
-        # only supports strings and unicodes
         assert len(op.args) == 4
-        assert op.args[1].value == 'chars'
         optype = op.args[0].concretetype
         if optype == lltype.Ptr(rstr.STR):
             opname = "strsetitem"
+            return SpaceOperation(opname, [op.args[0], op.args[2], op.args[3]],
+                                  op.result)
+        elif optype == lltype.Ptr(rstr.UNICODE):
+            opname = "unicodesetitem"
+            return SpaceOperation(opname, [op.args[0], op.args[2], op.args[3]],
+                                  op.result)
         else:
-            assert optype == lltype.Ptr(rstr.UNICODE)
-            opname = "unicodesetitem"
-        return SpaceOperation(opname, [op.args[0], op.args[2], op.args[3]],
-                              op.result)
+            v_inst, v_index, c_field, v_value = op.args
+            if v_value.concretetype is lltype.Void:
+                return
+            # only GcArray of Struct supported
+            assert isinstance(v_inst.concretetype.TO, lltype.GcArray)
+            STRUCT = v_inst.concretetype.TO.OF
+            assert isinstance(STRUCT, lltype.Struct)
+            descr = self.cpu.interiorfielddescrof(v_inst.concretetype.TO,
+                                                  c_field.value)
+            kind = getkind(v_value.concretetype)[0]
+            args = [v_inst, v_index, v_value, descr]
+            return SpaceOperation('setinteriorfield_gc_%s' % kind, args,
+                                  op.result)
 
     def _rewrite_equality(self, op, opname):
         arg0, arg1 = op.args
@@ -771,6 +800,9 @@
     def _is_gc(self, v):
         return getattr(getattr(v.concretetype, "TO", None), "_gckind", "?") == 'gc'
 
+    def _is_rclass_instance(self, v):
+        return lltype._castdepth(v.concretetype.TO, rclass.OBJECT) >= 0
+
     def _rewrite_cmp_ptrs(self, op):
         if self._is_gc(op.args[0]):
             return op
@@ -788,11 +820,21 @@
         return self._rewrite_equality(op, 'int_is_true')
 
     def rewrite_op_ptr_eq(self, op):
-        op1 = self._rewrite_equality(op, 'ptr_iszero')
+        prefix = ''
+        if self._is_rclass_instance(op.args[0]):
+            assert self._is_rclass_instance(op.args[1])
+            op = SpaceOperation('instance_ptr_eq', op.args, op.result)
+            prefix = 'instance_'
+        op1 = self._rewrite_equality(op, prefix + 'ptr_iszero')
         return self._rewrite_cmp_ptrs(op1)
 
     def rewrite_op_ptr_ne(self, op):
-        op1 = self._rewrite_equality(op, 'ptr_nonzero')
+        prefix = ''
+        if self._is_rclass_instance(op.args[0]):
+            assert self._is_rclass_instance(op.args[1])
+            op = SpaceOperation('instance_ptr_ne', op.args, op.result)
+            prefix = 'instance_'
+        op1 = self._rewrite_equality(op, prefix + 'ptr_nonzero')
         return self._rewrite_cmp_ptrs(op1)
 
     rewrite_op_ptr_iszero = _rewrite_cmp_ptrs
@@ -800,8 +842,11 @@
 
     def rewrite_op_cast_ptr_to_int(self, op):
         if self._is_gc(op.args[0]):
-            #return op
-            raise NotImplementedError("cast_ptr_to_int")
+            return op
+
+    def rewrite_op_cast_opaque_ptr(self, op):
+        # None causes the result of this op to get aliased to op.args[0]
+        return [SpaceOperation('mark_opaque_ptr', op.args, None), None]
 
     def rewrite_op_force_cast(self, op):
         v_arg = op.args[0]
@@ -822,26 +867,44 @@
         elif not float_arg and float_res:
             # some int -> some float
             ops = []
-            v1 = varoftype(lltype.Signed)
-            oplist = self.rewrite_operation(
-                SpaceOperation('force_cast', [v_arg], v1)
-            )
-            if oplist:
-                ops.extend(oplist)
+            v2 = varoftype(lltype.Float)
+            sizesign = rffi.size_and_sign(v_arg.concretetype)
+            if sizesign <= rffi.size_and_sign(lltype.Signed):
+                # cast from a type that fits in an int: either the size is
+                # smaller, or it is equal and it is not unsigned
+                v1 = varoftype(lltype.Signed)
+                oplist = self.rewrite_operation(
+                    SpaceOperation('force_cast', [v_arg], v1)
+                )
+                if oplist:
+                    ops.extend(oplist)
+                else:
+                    v1 = v_arg
+                op = self.rewrite_operation(
+                    SpaceOperation('cast_int_to_float', [v1], v2)
+                )
+                ops.append(op)
             else:
-                v1 = v_arg
-            v2 = varoftype(lltype.Float)
-            op = self.rewrite_operation(
-                SpaceOperation('cast_int_to_float', [v1], v2)
-            )
-            ops.append(op)
+                if sizesign == rffi.size_and_sign(lltype.Unsigned):
+                    opname = 'cast_uint_to_float'
+                elif sizesign == rffi.size_and_sign(lltype.SignedLongLong):
+                    opname = 'cast_longlong_to_float'
+                elif sizesign == rffi.size_and_sign(lltype.UnsignedLongLong):
+                    opname = 'cast_ulonglong_to_float'
+                else:
+                    raise AssertionError('cast_x_to_float: %r' % (sizesign,))
+                ops1 = self.rewrite_operation(
+                    SpaceOperation(opname, [v_arg], v2)
+                )
+                if not isinstance(ops1, list): ops1 = [ops1]
+                ops.extend(ops1)
             op2 = self.rewrite_operation(
                 SpaceOperation('force_cast', [v2], v_result)
             )
             if op2:
                 ops.append(op2)
             else:
-                op.result = v_result
+                ops[-1].result = v_result
             return ops
         elif float_arg and not float_res:
             # some float -> some int
@@ -854,18 +917,36 @@
                 ops.append(op1)
             else:
                 v1 = v_arg
-            v2 = varoftype(lltype.Signed)
-            op = self.rewrite_operation(
-                SpaceOperation('cast_float_to_int', [v1], v2)
-            )
-            ops.append(op)
-            oplist = self.rewrite_operation(
-                SpaceOperation('force_cast', [v2], v_result)
-            )
-            if oplist:
-                ops.extend(oplist)
+            sizesign = rffi.size_and_sign(v_result.concretetype)
+            if sizesign <= rffi.size_and_sign(lltype.Signed):
+                # cast to a type that fits in an int: either the size is
+                # smaller, or it is equal and it is not unsigned
+                v2 = varoftype(lltype.Signed)
+                op = self.rewrite_operation(
+                    SpaceOperation('cast_float_to_int', [v1], v2)
+                )
+                ops.append(op)
+                oplist = self.rewrite_operation(
+                    SpaceOperation('force_cast', [v2], v_result)
+                )
+                if oplist:
+                    ops.extend(oplist)
+                else:
+                    op.result = v_result
             else:
-                op.result = v_result
+                if sizesign == rffi.size_and_sign(lltype.Unsigned):
+                    opname = 'cast_float_to_uint'
+                elif sizesign == rffi.size_and_sign(lltype.SignedLongLong):
+                    opname = 'cast_float_to_longlong'
+                elif sizesign == rffi.size_and_sign(lltype.UnsignedLongLong):
+                    opname = 'cast_float_to_ulonglong'
+                else:
+                    raise AssertionError('cast_float_to_x: %r' % (sizesign,))
+                ops1 = self.rewrite_operation(
+                    SpaceOperation(opname, [v1], v_result)
+                )
+                if not isinstance(ops1, list): ops1 = [ops1]
+                ops.extend(ops1)
             return ops
         else:
             assert False
@@ -1071,8 +1152,6 @@
     # The new operation is optionally further processed by rewrite_operation().
     for _old, _new in [('bool_not', 'int_is_zero'),
                        ('cast_bool_to_float', 'cast_int_to_float'),
-                       ('cast_uint_to_float', 'cast_int_to_float'),
-                       ('cast_float_to_uint', 'cast_float_to_int'),
 
                        ('int_add_nonneg_ovf', 'int_add_ovf'),
                        ('keepalive', '-live-'),
@@ -1543,6 +1622,10 @@
     def rewrite_op_jit_force_virtual(self, op):
         return self._do_builtin_call(op)
 
+    def rewrite_op_jit_is_virtual(self, op):
+        raise Exception, (
+            "'vref.virtual' should not be used from jit-visible code")
+
     def rewrite_op_jit_force_virtualizable(self, op):
         # this one is for virtualizables
         vinfo = self.get_vinfo(op.args[0])
diff --git a/pypy/jit/codewriter/support.py b/pypy/jit/codewriter/support.py
--- a/pypy/jit/codewriter/support.py
+++ b/pypy/jit/codewriter/support.py
@@ -13,7 +13,6 @@
 from pypy.translator.simplify import get_funcobj
 from pypy.translator.unsimplify import split_block
 from pypy.objspace.flow.model import Constant
-from pypy import conftest
 from pypy.translator.translator import TranslationContext
 from pypy.annotation.policy import AnnotatorPolicy
 from pypy.annotation import model as annmodel
@@ -48,15 +47,13 @@
     a.build_types(func, argtypes, main_entry_point=True)
     rtyper = t.buildrtyper(type_system = type_system)
     rtyper.specialize()
-    if inline:
-        auto_inlining(t, threshold=inline)
+    #if inline:
+    #    auto_inlining(t, threshold=inline)
     if backendoptimize:
         from pypy.translator.backendopt.all import backend_optimizations
         backend_optimizations(t, inline_threshold=inline or 0,
                 remove_asserts=True, really_remove_asserts=True)
 
-    #if conftest.option.view:
-    #    t.view()
     return rtyper
 
 def getgraph(func, values):
@@ -232,6 +229,17 @@
     else:
         return x
 
+def _ll_1_cast_uint_to_float(x):
+    # XXX on 32-bit platforms, this should be done using cast_longlong_to_float
+    # (which is a residual call right now in the x86 backend)
+    return llop.cast_uint_to_float(lltype.Float, x)
+
+def _ll_1_cast_float_to_uint(x):
+    # XXX on 32-bit platforms, this should be done using cast_float_to_longlong
+    # (which is a residual call right now in the x86 backend)
+    return llop.cast_float_to_uint(lltype.Unsigned, x)
+
+
 # math support
 # ------------
 
@@ -456,6 +464,8 @@
         return LLtypeHelpers._dictnext_items(lltype.Ptr(RES), iter)
     _ll_1_dictiter_nextitems.need_result_type = True
 
+    _ll_1_dict_resize = ll_rdict.ll_dict_resize
+
     # ---------- strings and unicode ----------
 
     _ll_1_str_str2unicode = ll_rstr.LLHelpers.ll_str2unicode
diff --git a/pypy/jit/codewriter/test/test_flatten.py b/pypy/jit/codewriter/test/test_flatten.py
--- a/pypy/jit/codewriter/test/test_flatten.py
+++ b/pypy/jit/codewriter/test/test_flatten.py
@@ -8,7 +8,7 @@
 from pypy.rpython.lltypesystem import lltype, rclass, rstr
 from pypy.objspace.flow.model import SpaceOperation, Variable, Constant
 from pypy.translator.unsimplify import varoftype
-from pypy.rlib.rarithmetic import ovfcheck, r_uint
+from pypy.rlib.rarithmetic import ovfcheck, r_uint, r_longlong, r_ulonglong
 from pypy.rlib.jit import dont_look_inside, _we_are_jitted, JitDriver
 from pypy.rlib.objectmodel import keepalive_until_here
 from pypy.rlib import jit
@@ -70,7 +70,8 @@
         return 'residual'
     def getcalldescr(self, op, oopspecindex=None, extraeffect=None):
         try:
-            if 'cannot_raise' in op.args[0].value._obj.graph.name:
+            name = op.args[0].value._obj._name
+            if 'cannot_raise' in name or name.startswith('cast_'):
                 return self._descr_cannot_raise
         except AttributeError:
             pass
@@ -900,6 +901,67 @@
             int_return %i4
         """, transform=True)
 
+        def f(dbl):
+            return rffi.cast(rffi.UCHAR, dbl)
+        self.encoding_test(f, [12.456], """
+            cast_float_to_int %f0 -> %i0
+            int_and %i0, $255 -> %i1
+            int_return %i1
+        """, transform=True)
+
+        def f(dbl):
+            return rffi.cast(lltype.Unsigned, dbl)
+        self.encoding_test(f, [12.456], """
+            residual_call_irf_i $<* fn cast_float_to_uint>, <Descr>, I[], R[], F[%f0] -> %i0
+            int_return %i0
+        """, transform=True)
+
+        def f(i):
+            return rffi.cast(lltype.Float, chr(i))    # "char -> float"
+        self.encoding_test(f, [12], """
+            cast_int_to_float %i0 -> %f0
+            float_return %f0
+        """, transform=True)
+
+        def f(i):
+            return rffi.cast(lltype.Float, r_uint(i))    # "uint -> float"
+        self.encoding_test(f, [12], """
+            residual_call_irf_f $<* fn cast_uint_to_float>, <Descr>, I[%i0], R[], F[] -> %f0
+            float_return %f0
+        """, transform=True)
+
+        if not longlong.is_64_bit:
+            def f(dbl):
+                return rffi.cast(lltype.SignedLongLong, dbl)
+            self.encoding_test(f, [12.3], """
+                residual_call_irf_f $<* fn llong_from_float>, <Descr>, I[], R[], F[%f0] -> %f1
+                float_return %f1
+            """, transform=True)
+
+            def f(dbl):
+                return rffi.cast(lltype.UnsignedLongLong, dbl)
+            self.encoding_test(f, [12.3], """
+                residual_call_irf_f $<* fn ullong_from_float>, <Descr>, I[], R[], F[%f0] -> %f1
+                float_return %f1
+            """, transform=True)
+
+            def f(x):
+                ll = r_longlong(x)
+                return rffi.cast(lltype.Float, ll)
+            self.encoding_test(f, [12], """
+                residual_call_irf_f $<* fn llong_from_int>, <Descr>, I[%i0], R[], F[] -> %f0
+                residual_call_irf_f $<* fn llong_to_float>, <Descr>, I[], R[], F[%f0] -> %f1
+                float_return %f1
+            """, transform=True)
+
+            def f(x):
+                ll = r_ulonglong(x)
+                return rffi.cast(lltype.Float, ll)
+            self.encoding_test(f, [12], """
+                residual_call_irf_f $<* fn ullong_from_int>, <Descr>, I[%i0], R[], F[] -> %f0
+                residual_call_irf_f $<* fn ullong_u_to_float>, <Descr>, I[], R[], F[%f0] -> %f1
+                float_return %f1
+            """, transform=True)
 
     def test_direct_ptradd(self):
         from pypy.rpython.lltypesystem import rffi
diff --git a/pypy/jit/codewriter/test/test_jtransform.py b/pypy/jit/codewriter/test/test_jtransform.py
--- a/pypy/jit/codewriter/test/test_jtransform.py
+++ b/pypy/jit/codewriter/test/test_jtransform.py
@@ -1,4 +1,3 @@
-import py
 import random
 try:
     from itertools import product
@@ -16,13 +15,13 @@
 
 from pypy.objspace.flow.model import FunctionGraph, Block, Link
 from pypy.objspace.flow.model import SpaceOperation, Variable, Constant
-from pypy.jit.codewriter.jtransform import Transformer
-from pypy.jit.metainterp.history import getkind
-from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr, rlist
+from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr
 from pypy.rpython.lltypesystem.module import ll_math
 from pypy.translator.unsimplify import varoftype
 from pypy.jit.codewriter import heaptracker, effectinfo
 from pypy.jit.codewriter.flatten import ListOfKind
+from pypy.jit.codewriter.jtransform import Transformer
+from pypy.jit.metainterp.history import getkind
 
 def const(x):
     return Constant(x, lltype.typeOf(x))
@@ -37,6 +36,8 @@
         return ('calldescr', FUNC, ARGS, RESULT)
     def fielddescrof(self, STRUCT, name):
         return ('fielddescr', STRUCT, name)
+    def interiorfielddescrof(self, ARRAY, name):
+        return ('interiorfielddescr', ARRAY, name)
     def arraydescrof(self, ARRAY):
         return FakeDescr(('arraydescr', ARRAY))
     def sizeof(self, STRUCT):
@@ -539,7 +540,7 @@
 
 def test_rename_on_links():
     v1 = Variable()
-    v2 = Variable()
+    v2 = Variable(); v2.concretetype = llmemory.Address
     v3 = Variable()
     block = Block([v1])
     block.operations = [SpaceOperation('cast_pointer', [v1], v2)]
@@ -575,10 +576,10 @@
         assert op1.args == [v2]
 
 def test_ptr_eq():
-    v1 = varoftype(rclass.OBJECTPTR)
-    v2 = varoftype(rclass.OBJECTPTR)
+    v1 = varoftype(lltype.Ptr(rstr.STR))
+    v2 = varoftype(lltype.Ptr(rstr.STR))
     v3 = varoftype(lltype.Bool)
-    c0 = const(lltype.nullptr(rclass.OBJECT))
+    c0 = const(lltype.nullptr(rstr.STR))
     #
     for opname, reducedname in [('ptr_eq', 'ptr_iszero'),
                                 ('ptr_ne', 'ptr_nonzero')]:
@@ -597,6 +598,31 @@
         assert op1.opname == reducedname
         assert op1.args == [v2]
 
+def test_instance_ptr_eq():
+    v1 = varoftype(rclass.OBJECTPTR)
+    v2 = varoftype(rclass.OBJECTPTR)
+    v3 = varoftype(lltype.Bool)
+    c0 = const(lltype.nullptr(rclass.OBJECT))
+
+    for opname, newopname, reducedname in [
+        ('ptr_eq', 'instance_ptr_eq', 'instance_ptr_iszero'),
+        ('ptr_ne', 'instance_ptr_ne', 'instance_ptr_nonzero')
+    ]:
+        op = SpaceOperation(opname, [v1, v2], v3)
+        op1 = Transformer().rewrite_operation(op)
+        assert op1.opname == newopname
+        assert op1.args == [v1, v2]
+
+        op = SpaceOperation(opname, [v1, c0], v3)
+        op1 = Transformer().rewrite_operation(op)
+        assert op1.opname == reducedname
+        assert op1.args == [v1]
+
+        op = SpaceOperation(opname, [c0, v1], v3)
+        op1 = Transformer().rewrite_operation(op)
+        assert op1.opname == reducedname
+        assert op1.args == [v1]
+
 def test_nongc_ptr_eq():
     v1 = varoftype(rclass.NONGCOBJECTPTR)
     v2 = varoftype(rclass.NONGCOBJECTPTR)
@@ -676,6 +702,22 @@
     assert op1.args == [v, v_index]
     assert op1.result == v_result
 
+def test_dict_getinteriorfield():
+    DICT = lltype.GcArray(lltype.Struct('ENTRY', ('v', lltype.Signed),
+                                        ('k', lltype.Signed)))
+    v = varoftype(lltype.Ptr(DICT))
+    i = varoftype(lltype.Signed)
+    v_result = varoftype(lltype.Signed)
+    op = SpaceOperation('getinteriorfield', [v, i, Constant('v', lltype.Void)],
+                        v_result)
+    op1 = Transformer(FakeCPU()).rewrite_operation(op)
+    assert op1.opname == 'getinteriorfield_gc_i'
+    assert op1.args == [v, i, ('interiorfielddescr', DICT, 'v')]
+    op = SpaceOperation('getinteriorfield', [v, i, Constant('v', lltype.Void)],
+                        Constant(None, lltype.Void))
+    op1 = Transformer(FakeCPU()).rewrite_operation(op)
+    assert op1 is None
+
 def test_str_setinteriorfield():
     v = varoftype(lltype.Ptr(rstr.STR))
     v_index = varoftype(lltype.Signed)
@@ -702,6 +744,23 @@
     assert op1.args == [v, v_index, v_newchr]
     assert op1.result == v_void
 
+def test_dict_setinteriorfield():
+    DICT = lltype.GcArray(lltype.Struct('ENTRY', ('v', lltype.Signed),
+                                        ('k', lltype.Signed)))
+    v = varoftype(lltype.Ptr(DICT))
+    i = varoftype(lltype.Signed)
+    v_void = varoftype(lltype.Void)
+    op = SpaceOperation('setinteriorfield', [v, i, Constant('v', lltype.Void),
+                                             i],
+                        v_void)
+    op1 = Transformer(FakeCPU()).rewrite_operation(op)
+    assert op1.opname == 'setinteriorfield_gc_i'
+    assert op1.args == [v, i, i, ('interiorfielddescr', DICT, 'v')]
+    op = SpaceOperation('setinteriorfield', [v, i, Constant('v', lltype.Void),
+                                             v_void], v_void)
+    op1 = Transformer(FakeCPU()).rewrite_operation(op)
+    assert not op1
+
 def test_promote_1():
     v1 = varoftype(lltype.Signed)
     v2 = varoftype(lltype.Signed)
@@ -1069,3 +1128,16 @@
                         varoftype(lltype.Signed))
     tr = Transformer(None, None)
     raises(NotImplementedError, tr.rewrite_operation, op)
+
+def test_cast_opaque_ptr():
+    S = lltype.GcStruct("S", ("x", lltype.Signed))
+    v1 = varoftype(lltype.Ptr(S))
+    v2 = varoftype(lltype.Ptr(rclass.OBJECT))
+
+    op = SpaceOperation('cast_opaque_ptr', [v1], v2)
+    tr = Transformer()
+    [op1, op2] = tr.rewrite_operation(op)
+    assert op1.opname == 'mark_opaque_ptr'
+    assert op1.args == [v1]
+    assert op1.result is None
+    assert op2 is None
\ No newline at end of file
diff --git a/pypy/jit/metainterp/blackhole.py b/pypy/jit/metainterp/blackhole.py
--- a/pypy/jit/metainterp/blackhole.py
+++ b/pypy/jit/metainterp/blackhole.py
@@ -2,11 +2,10 @@
 from pypy.rlib.rtimer import read_timestamp
 from pypy.rlib.rarithmetic import intmask, LONG_BIT, r_uint, ovfcheck
 from pypy.rlib.objectmodel import we_are_translated
-from pypy.rlib.debug import debug_start, debug_stop
+from pypy.rlib.debug import debug_start, debug_stop, ll_assert
 from pypy.rlib.debug import make_sure_not_resized
 from pypy.rpython.lltypesystem import lltype, llmemory, rclass
 from pypy.rpython.lltypesystem.lloperation import llop
-from pypy.rpython.llinterp import LLException
 from pypy.jit.codewriter.jitcode import JitCode, SwitchDictDescr
 from pypy.jit.codewriter import heaptracker, longlong
 from pypy.jit.metainterp.jitexc import JitException, get_llexception, reraise
@@ -500,9 +499,25 @@
     @arguments("r", returns="i")
     def bhimpl_ptr_nonzero(a):
         return bool(a)
-    @arguments("r", returns="r")
-    def bhimpl_cast_opaque_ptr(a):
-        return a
+    @arguments("r", "r", returns="i")
+    def bhimpl_instance_ptr_eq(a, b):
+        return a == b
+    @arguments("r", "r", returns="i")
+    def bhimpl_instance_ptr_ne(a, b):
+        return a != b
+    @arguments("r", returns="i")
+    def bhimpl_cast_ptr_to_int(a):
+        i = lltype.cast_ptr_to_int(a)
+        ll_assert((i & 1) == 1, "bhimpl_cast_ptr_to_int: not an odd int")
+        return i
+    @arguments("i", returns="r")
+    def bhimpl_cast_int_to_ptr(i):
+        ll_assert((i & 1) == 1, "bhimpl_cast_int_to_ptr: not an odd int")
+        return lltype.cast_int_to_ptr(llmemory.GCREF, i)
+
+    @arguments("r")
+    def bhimpl_mark_opaque_ptr(a):
+        pass
 
     @arguments("i", returns="i")
     def bhimpl_int_copy(a):
@@ -622,6 +637,9 @@
         a = longlong.getrealfloat(a)
         # note: we need to call int() twice to care for the fact that
         # int(-2147483648.0) returns a long :-(
+        # we could also call intmask() instead of the outermost int(), but
+        # it's probably better to explicitly crash (by getting a long) if a
+        # non-translated version tries to cast a too large float to an int.
         return int(int(a))
 
     @arguments("i", returns="f")
@@ -1145,6 +1163,26 @@
         array = cpu.bh_getfield_gc_r(vable, fdescr)
         return cpu.bh_arraylen_gc(adescr, array)
 
+    @arguments("cpu", "r", "i", "d", returns="i")
+    def bhimpl_getinteriorfield_gc_i(cpu, array, index, descr):
+        return cpu.bh_getinteriorfield_gc_i(array, index, descr)
+    @arguments("cpu", "r", "i", "d", returns="r")
+    def bhimpl_getinteriorfield_gc_r(cpu, array, index, descr):
+        return cpu.bh_getinteriorfield_gc_r(array, index, descr)
+    @arguments("cpu", "r", "i", "d", returns="f")
+    def bhimpl_getinteriorfield_gc_f(cpu, array, index, descr):
+        return cpu.bh_getinteriorfield_gc_f(array, index, descr)
+
+    @arguments("cpu", "r", "i", "d", "i")
+    def bhimpl_setinteriorfield_gc_i(cpu, array, index, descr, value):
+        cpu.bh_setinteriorfield_gc_i(array, index, descr, value)
+    @arguments("cpu", "r", "i", "d", "r")
+    def bhimpl_setinteriorfield_gc_r(cpu, array, index, descr, value):
+        cpu.bh_setinteriorfield_gc_r(array, index, descr, value)
+    @arguments("cpu", "r", "i", "d", "f")
+    def bhimpl_setinteriorfield_gc_f(cpu, array, index, descr, value):
+        cpu.bh_setinteriorfield_gc_f(array, index, descr, value)
+
     @arguments("cpu", "r", "d", returns="i")
     def bhimpl_getfield_gc_i(cpu, struct, fielddescr):
         return cpu.bh_getfield_gc_i(struct, fielddescr)
diff --git a/pypy/jit/metainterp/executor.py b/pypy/jit/metainterp/executor.py
--- a/pypy/jit/metainterp/executor.py
+++ b/pypy/jit/metainterp/executor.py
@@ -1,11 +1,8 @@
 """This implements pyjitpl's execution of operations.
 """
 
-import py
-from pypy.rpython.lltypesystem import lltype, llmemory, rstr
-from pypy.rpython.ootypesystem import ootype
-from pypy.rpython.lltypesystem.lloperation import llop
-from pypy.rlib.rarithmetic import ovfcheck, r_uint, intmask, r_longlong
+from pypy.rpython.lltypesystem import lltype, rstr
+from pypy.rlib.rarithmetic import ovfcheck, r_longlong
 from pypy.rlib.rtimer import read_timestamp
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.jit.metainterp.history import BoxInt, BoxPtr, BoxFloat, check_descr
@@ -123,6 +120,29 @@
     else:
         cpu.bh_setarrayitem_raw_i(arraydescr, array, index, itembox.getint())
 
+def do_getinteriorfield_gc(cpu, _, arraybox, indexbox, descr):
+    array = arraybox.getref_base()
+    index = indexbox.getint()
+    if descr.is_pointer_field():
+        return BoxPtr(cpu.bh_getinteriorfield_gc_r(array, index, descr))
+    elif descr.is_float_field():
+        return BoxFloat(cpu.bh_getinteriorfield_gc_f(array, index, descr))
+    else:
+        return BoxInt(cpu.bh_getinteriorfield_gc_i(array, index, descr))
+
+def do_setinteriorfield_gc(cpu, _, arraybox, indexbox, valuebox, descr):
+    array = arraybox.getref_base()
+    index = indexbox.getint()
+    if descr.is_pointer_field():
+        cpu.bh_setinteriorfield_gc_r(array, index, descr,
+                                     valuebox.getref_base())
+    elif descr.is_float_field():
+        cpu.bh_setinteriorfield_gc_f(array, index, descr,
+                                     valuebox.getfloatstorage())
+    else:
+        cpu.bh_setinteriorfield_gc_i(array, index, descr,
+                                     valuebox.getint())
+
 def do_getfield_gc(cpu, _, structbox, fielddescr):
     struct = structbox.getref_base()
     if fielddescr.is_pointer_field():
diff --git a/pypy/jit/metainterp/graphpage.py b/pypy/jit/metainterp/graphpage.py
--- a/pypy/jit/metainterp/graphpage.py
+++ b/pypy/jit/metainterp/graphpage.py
@@ -12,8 +12,8 @@
     def get_display_text(self):
         return None
 
-def display_loops(loops, errmsg=None, highlight_loops=()):
-    graphs = [(loop, loop in highlight_loops) for loop in loops]    
+def display_loops(loops, errmsg=None, highlight_loops={}):
+    graphs = [(loop, highlight_loops.get(loop, 0)) for loop in loops]    
     for graph, highlight in graphs:
         for op in graph.get_operations():
             if is_interesting_guard(op):
@@ -65,8 +65,7 @@
     def add_graph(self, graph, highlight=False):
         graphindex = len(self.graphs)
         self.graphs.append(graph)
-        if highlight:
-            self.highlight_graphs[graph] = True
+        self.highlight_graphs[graph] = highlight
         for i, op in enumerate(graph.get_operations()):
             self.all_operations[op] = graphindex, i
 
@@ -126,10 +125,13 @@
             self.dotgen.emit('subgraph cluster%d {' % graphindex)
         label = graph.get_display_text()
         if label is not None:
-            if self.highlight_graphs.get(graph):
-                fillcolor = '#f084c2'
+            colorindex = self.highlight_graphs.get(graph, 0)
+            if colorindex == 1:
+                fillcolor = '#f084c2'    # highlighted graph
+            elif colorindex == 2:
+                fillcolor = '#808080'    # invalidated graph
             else:
-                fillcolor = '#84f0c2'
+                fillcolor = '#84f0c2'    # normal color
             self.dotgen.emit_node(graphname, shape="octagon",
                                   label=label, fillcolor=fillcolor)
             self.pendingedges.append((graphname,
diff --git a/pypy/jit/metainterp/heapcache.py b/pypy/jit/metainterp/heapcache.py
--- a/pypy/jit/metainterp/heapcache.py
+++ b/pypy/jit/metainterp/heapcache.py
@@ -34,7 +34,6 @@
         self.clear_caches(opnum, descr, argboxes)
 
     def mark_escaped(self, opnum, argboxes):
-        idx = 0
         if opnum == rop.SETFIELD_GC:
             assert len(argboxes) == 2
             box, valuebox = argboxes
@@ -42,8 +41,20 @@
                 self.dependencies.setdefault(box, []).append(valuebox)
             else:
                 self._escape(valuebox)
-        # GETFIELD_GC doesn't escape it's argument
-        elif opnum != rop.GETFIELD_GC:
+        elif opnum == rop.SETARRAYITEM_GC:
+            assert len(argboxes) == 3
+            box, indexbox, valuebox = argboxes
+            if self.is_unescaped(box) and self.is_unescaped(valuebox):
+                self.dependencies.setdefault(box, []).append(valuebox)
+            else:
+                self._escape(valuebox)
+        # GETFIELD_GC, MARK_OPAQUE_PTR, PTR_EQ, and PTR_NE don't escape their
+        # arguments
+        elif (opnum != rop.GETFIELD_GC and
+              opnum != rop.MARK_OPAQUE_PTR and
+              opnum != rop.PTR_EQ and
+              opnum != rop.PTR_NE):
+            idx = 0
             for box in argboxes:
                 # setarrayitem_gc don't escape its first argument
                 if not (idx == 0 and opnum in [rop.SETARRAYITEM_GC]):
@@ -60,13 +71,13 @@
                 self._escape(dep)
 
     def clear_caches(self, opnum, descr, argboxes):
-        if opnum == rop.SETFIELD_GC:
-            return
-        if opnum == rop.SETARRAYITEM_GC:
-            return
-        if opnum == rop.SETFIELD_RAW:
-            return
-        if opnum == rop.SETARRAYITEM_RAW:
+        if (opnum == rop.SETFIELD_GC or
+            opnum == rop.SETARRAYITEM_GC or
+            opnum == rop.SETFIELD_RAW or
+            opnum == rop.SETARRAYITEM_RAW or
+            opnum == rop.SETINTERIORFIELD_GC or
+            opnum == rop.COPYSTRCONTENT or
+            opnum == rop.COPYUNICODECONTENT):
             return
         if rop._OVF_FIRST <= opnum <= rop._OVF_LAST:
             return
@@ -75,9 +86,9 @@
         if opnum == rop.CALL or opnum == rop.CALL_LOOPINVARIANT:
             effectinfo = descr.get_extra_info()
             ef = effectinfo.extraeffect
-            if ef == effectinfo.EF_LOOPINVARIANT or \
-               ef == effectinfo.EF_ELIDABLE_CANNOT_RAISE or \
-               ef == effectinfo.EF_ELIDABLE_CAN_RAISE:
+            if (ef == effectinfo.EF_LOOPINVARIANT or
+                ef == effectinfo.EF_ELIDABLE_CANNOT_RAISE or
+                ef == effectinfo.EF_ELIDABLE_CAN_RAISE):
                 return
             # A special case for ll_arraycopy, because it is so common, and its
             # effects are so well defined.
diff --git a/pypy/jit/metainterp/history.py b/pypy/jit/metainterp/history.py
--- a/pypy/jit/metainterp/history.py
+++ b/pypy/jit/metainterp/history.py
@@ -16,6 +16,7 @@
 INT   = 'i'
 REF   = 'r'
 FLOAT = 'f'
+STRUCT = 's'
 HOLE  = '_'
 VOID  = 'v'
 
@@ -172,6 +173,11 @@
         """
         raise NotImplementedError
 
+    def is_array_of_structs(self):
+        """ Implement for array descr
+        """
+        raise NotImplementedError
+
     def is_pointer_field(self):
         """ Implement for field descr
         """
@@ -732,6 +738,7 @@
     failed_states = None
     retraced_count = 0
     terminating = False # see TerminatingLoopToken in compile.py
+    invalidated = False
     outermost_jitdriver_sd = None
     # and more data specified by the backend when the loop is compiled
     number = -1
@@ -934,6 +941,7 @@
         self.loops = []
         self.locations = []
         self.aborted_keys = []
+        self.invalidated_token_numbers = set()
 
     def set_history(self, history):
         self.operations = history.operations
@@ -1012,7 +1020,12 @@
             if loop in loops:
                 loops.remove(loop)
             loops.append(loop)
-        display_loops(loops, errmsg, extraloops)
+        highlight_loops = dict.fromkeys(extraloops, 1)
+        for loop in loops:
+            if hasattr(loop, '_looptoken_number') and (
+                    loop._looptoken_number in self.invalidated_token_numbers):
+                highlight_loops.setdefault(loop, 2)
+        display_loops(loops, errmsg, highlight_loops)
 
 # ----------------------------------------------------------------
 
diff --git a/pypy/jit/metainterp/memmgr.py b/pypy/jit/metainterp/memmgr.py
--- a/pypy/jit/metainterp/memmgr.py
+++ b/pypy/jit/metainterp/memmgr.py
@@ -68,7 +68,8 @@
         debug_print("Loop tokens before:", oldtotal)
         max_generation = self.current_generation - (self.max_age-1)
         for looptoken in self.alive_loops.keys():
-            if 0 <= looptoken.generation < max_generation:
+            if (0 <= looptoken.generation < max_generation or
+                looptoken.invalidated):
                 del self.alive_loops[looptoken]
         newtotal = len(self.alive_loops)
         debug_print("Loop tokens freed: ", oldtotal - newtotal)
diff --git a/pypy/jit/metainterp/optimizeopt/optimizer.py b/pypy/jit/metainterp/optimizeopt/optimizer.py
--- a/pypy/jit/metainterp/optimizeopt/optimizer.py
+++ b/pypy/jit/metainterp/optimizeopt/optimizer.py
@@ -209,13 +209,19 @@
     def setfield(self, ofs, value):
         raise NotImplementedError
 
+    def getlength(self):
+        raise NotImplementedError
+
     def getitem(self, index):
         raise NotImplementedError
 
-    def getlength(self):
+    def setitem(self, index, value):
         raise NotImplementedError
 
-    def setitem(self, index, value):
+    def getinteriorfield(self, index, ofs, default):
+        raise NotImplementedError
+
+    def setinteriorfield(self, index, ofs, value):
         raise NotImplementedError
 
 
@@ -283,11 +289,11 @@
             return self.optimizer.optpure.has_pure_result(opnum, args, descr)
         return False
 
-    def get_pure_result(self, key):    
+    def get_pure_result(self, key):
         if self.optimizer.optpure:
             return self.optimizer.optpure.get_pure_result(key)
         return None
-    
+
     def setup(self):
         pass
 
@@ -524,7 +530,7 @@
 
     def replace_op(self, old_op, new_op):
         # XXX: Do we want to cache indexes to prevent search?
-        i = len(self._newoperations) 
+        i = len(self._newoperations)
         while i > 0:
             i -= 1
             if self._newoperations[i] is old_op:
diff --git a/pypy/jit/metainterp/optimizeopt/rewrite.py b/pypy/jit/metainterp/optimizeopt/rewrite.py
--- a/pypy/jit/metainterp/optimizeopt/rewrite.py
+++ b/pypy/jit/metainterp/optimizeopt/rewrite.py
@@ -106,10 +106,9 @@
             self.make_equal_to(op.result, v1)
         else:
             self.emit_operation(op)
-
-        # Synthesize the reverse ops for optimize_default to reuse
-        self.pure(rop.INT_ADD, [op.result, op.getarg(1)], op.getarg(0))
-        self.pure(rop.INT_SUB, [op.getarg(0), op.result], op.getarg(1))
+            # Synthesize the reverse ops for optimize_default to reuse
+            self.pure(rop.INT_ADD, [op.result, op.getarg(1)], op.getarg(0))
+            self.pure(rop.INT_SUB, [op.getarg(0), op.result], op.getarg(1))
 
     def optimize_INT_ADD(self, op):
         v1 = self.getvalue(op.getarg(0))
@@ -122,10 +121,9 @@
             self.make_equal_to(op.result, v1)
         else:
             self.emit_operation(op)
-
-        # Synthesize the reverse op for optimize_default to reuse
-        self.pure(rop.INT_SUB, [op.result, op.getarg(1)], op.getarg(0))
-        self.pure(rop.INT_SUB, [op.result, op.getarg(0)], op.getarg(1))
+            # Synthesize the reverse op for optimize_default to reuse
+            self.pure(rop.INT_SUB, [op.result, op.getarg(1)], op.getarg(0))
+            self.pure(rop.INT_SUB, [op.result, op.getarg(0)], op.getarg(1))
 
     def optimize_INT_MUL(self, op):
         v1 = self.getvalue(op.getarg(0))
@@ -141,13 +139,13 @@
             self.make_constant_int(op.result, 0)
         else:
             for lhs, rhs in [(v1, v2), (v2, v1)]:
-                # x & (x -1) == 0 is a quick test for power of 2
-                if (lhs.is_constant() and
-                    (lhs.box.getint() & (lhs.box.getint() - 1)) == 0):
-                    new_rhs = ConstInt(highest_bit(lhs.box.getint()))
-                    op = op.copy_and_change(rop.INT_LSHIFT, args=[rhs.box, new_rhs])
-                    break
-
+                if lhs.is_constant():
+                    x = lhs.box.getint()
+                    # x & (x - 1) == 0 is a quick test for power of 2
+                    if x & (x - 1) == 0:
+                        new_rhs = ConstInt(highest_bit(lhs.box.getint()))
+                        op = op.copy_and_change(rop.INT_LSHIFT, args=[rhs.box, new_rhs])
+                        break
             self.emit_operation(op)
 
     def optimize_UINT_FLOORDIV(self, op):
@@ -339,7 +337,7 @@
     def optimize_INT_IS_ZERO(self, op):
         self._optimize_nullness(op, op.getarg(0), False)
 
-    def _optimize_oois_ooisnot(self, op, expect_isnot):
+    def _optimize_oois_ooisnot(self, op, expect_isnot, instance):
         value0 = self.getvalue(op.getarg(0))
         value1 = self.getvalue(op.getarg(1))
         if value0.is_virtual():
@@ -357,21 +355,28 @@
         elif value0 is value1:
             self.make_constant_int(op.result, not expect_isnot)
         else:
-            cls0 = value0.get_constant_class(self.optimizer.cpu)
-            if cls0 is not None:
-                cls1 = value1.get_constant_class(self.optimizer.cpu)
-                if cls1 is not None and not cls0.same_constant(cls1):
-                    # cannot be the same object, as we know that their
-                    # class is different
-                    self.make_constant_int(op.result, expect_isnot)
-                    return
+            if instance:
+                cls0 = value0.get_constant_class(self.optimizer.cpu)
+                if cls0 is not None:
+                    cls1 = value1.get_constant_class(self.optimizer.cpu)
+                    if cls1 is not None and not cls0.same_constant(cls1):
+                        # cannot be the same object, as we know that their
+                        # class is different
+                        self.make_constant_int(op.result, expect_isnot)
+                        return
             self.emit_operation(op)
 
+    def optimize_PTR_EQ(self, op):
+        self._optimize_oois_ooisnot(op, False, False)
+
     def optimize_PTR_NE(self, op):
-        self._optimize_oois_ooisnot(op, True)
+        self._optimize_oois_ooisnot(op, True, False)
 
-    def optimize_PTR_EQ(self, op):
-        self._optimize_oois_ooisnot(op, False)
+    def optimize_INSTANCE_PTR_EQ(self, op):
+        self._optimize_oois_ooisnot(op, False, True)
+
+    def optimize_INSTANCE_PTR_NE(self, op):
+        self._optimize_oois_ooisnot(op, True, True)
 
 ##    def optimize_INSTANCEOF(self, op):
 ##        value = self.getvalue(op.args[0])
@@ -450,6 +455,9 @@
         if v2.is_constant() and v2.box.getint() == 1:
             self.make_equal_to(op.result, v1)
             return
+        elif v1.is_constant() and v1.box.getint() == 0:
+            self.make_constant_int(op.result, 0)
+            return
         if v1.intbound.known_ge(IntBound(0, 0)) and v2.is_constant():
             val = v2.box.getint()
             if val & (val - 1) == 0 and val > 0: # val == 2**shift
@@ -457,10 +465,17 @@
                                         args = [op.getarg(0), ConstInt(highest_bit(val))])
         self.emit_operation(op)
 
-    def optimize_CAST_OPAQUE_PTR(self, op):
+    def optimize_MARK_OPAQUE_PTR(self, op):
         value = self.getvalue(op.getarg(0))
         self.optimizer.opaque_pointers[value] = True
-        self.make_equal_to(op.result, value)
+
+    def optimize_CAST_PTR_TO_INT(self, op):
+        self.pure(rop.CAST_INT_TO_PTR, [op.result], op.getarg(0))
+        self.emit_operation(op)
+
+    def optimize_CAST_INT_TO_PTR(self, op):
+        self.pure(rop.CAST_PTR_TO_INT, [op.result], op.getarg(0))
+        self.emit_operation(op)
 
 dispatch_opt = make_dispatcher_method(OptRewrite, 'optimize_',
         default=OptRewrite.emit_operation)
diff --git a/pypy/jit/metainterp/optimizeopt/simplify.py b/pypy/jit/metainterp/optimizeopt/simplify.py
--- a/pypy/jit/metainterp/optimizeopt/simplify.py
+++ b/pypy/jit/metainterp/optimizeopt/simplify.py
@@ -25,7 +25,8 @@
         #     but it's a bit hard to implement robustly if heap.py is also run
         pass
 
-    optimize_CAST_OPAQUE_PTR = optimize_VIRTUAL_REF
+    def optimize_MARK_OPAQUE_PTR(self, op):
+        pass
 
 
 dispatch_opt = make_dispatcher_method(OptSimplify, 'optimize_',
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -508,13 +508,13 @@
         ops = """
         [p0]
         guard_class(p0, ConstClass(node_vtable)) []
-        i0 = ptr_ne(p0, NULL)
+        i0 = instance_ptr_ne(p0, NULL)
         guard_true(i0) []
-        i1 = ptr_eq(p0, NULL)
+        i1 = instance_ptr_eq(p0, NULL)
         guard_false(i1) []
-        i2 = ptr_ne(NULL, p0)
+        i2 = instance_ptr_ne(NULL, p0)
         guard_true(i0) []
-        i3 = ptr_eq(NULL, p0)
+        i3 = instance_ptr_eq(NULL, p0)
         guard_false(i1) []
         jump(p0)
         """
@@ -935,7 +935,6 @@
         """
         self.optimize_loop(ops, expected)
 
-
     def test_virtual_constant_isnonnull(self):
         ops = """
         [i0]
@@ -951,6 +950,32 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_virtual_array_of_struct(self):
+        ops = """
+        [f0, f1, f2, f3]
+        p0 = new_array(2, descr=complexarraydescr)
+        setinteriorfield_gc(p0, 0, f0, descr=complexrealdescr)
+        setinteriorfield_gc(p0, 0, f1, descr=compleximagdescr)
+        setinteriorfield_gc(p0, 1, f2, descr=complexrealdescr)
+        setinteriorfield_gc(p0, 1, f3, descr=compleximagdescr)
+        f4 = getinteriorfield_gc(p0, 0, descr=complexrealdescr)
+        f5 = getinteriorfield_gc(p0, 1, descr=complexrealdescr)
+        f6 = float_mul(f4, f5)
+        f7 = getinteriorfield_gc(p0, 0, descr=compleximagdescr)
+        f8 = getinteriorfield_gc(p0, 1, descr=compleximagdescr)
+        f9 = float_mul(f7, f8)
+        f10 = float_add(f6, f9)
+        finish(f10)
+        """
+        expected = """
+        [f0, f1, f2, f3]
+        f4 = float_mul(f0, f2)
+        f5 = float_mul(f1, f3)
+        f6 = float_add(f4, f5)
+        finish(f6)
+        """
+        self.optimize_loop(ops, expected)
+
     def test_nonvirtual_1(self):
         ops = """
         [i]
@@ -2026,7 +2051,7 @@
         ops = """
         [p1]
         guard_class(p1, ConstClass(node_vtable2)) []
-        i = ptr_ne(ConstPtr(myptr), p1)
+        i = instance_ptr_ne(ConstPtr(myptr), p1)
         guard_true(i) []
         jump(p1)
         """
@@ -2181,6 +2206,17 @@
         """
         self.optimize_loop(ops, expected)
 
+        ops = """
+        [i0]
+        i1 = int_floordiv(0, i0)
+        jump(i1)
+        """
+        expected = """
+        [i0]
+        jump(0)
+        """
+        self.optimize_loop(ops, expected)
+
     def test_fold_partially_constant_ops_ovf(self):
         ops = """
         [i0]
@@ -4170,10 +4206,12 @@
         class FakeCallInfoCollection:
             def callinfo_for_oopspec(self, oopspecindex):
                 calldescrtype = type(LLtypeMixin.strequaldescr)
+                effectinfotype = type(LLtypeMixin.strequaldescr.get_extra_info())
                 for value in LLtypeMixin.__dict__.values():
                     if isinstance(value, calldescrtype):
                         extra = value.get_extra_info()
-                        if extra and extra.oopspecindex == oopspecindex:
+                        if (extra and isinstance(extra, effectinfotype) and
+                            extra.oopspecindex == oopspecindex):
                             # returns 0 for 'func' in this test
                             return value, 0
                 raise AssertionError("not found: oopspecindex=%d" %
@@ -4789,6 +4827,18 @@
         """
         self.optimize_strunicode_loop(ops, expected)
 
+    def test_ptr_eq_str_constant(self):
+        ops = """
+        []
+        i0 = ptr_eq(s"abc", s"\x00")
+        finish(i0)
+        """
+        expected = """
+        []
+        finish(0)
+        """
+        self.optimize_loop(ops, expected)
+
 
 class TestLLtype(BaseTestOptimizeBasic, LLtypeMixin):
     pass
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -234,6 +234,30 @@
             """ % expected_value
             self.optimize_loop(ops, expected)
 
+    def test_reverse_of_cast(self):
+        ops = """
+        [i0]
+        p0 = cast_int_to_ptr(i0)
+        i1 = cast_ptr_to_int(p0)
+        jump(i1)
+        """
+        expected = """
+        [i0]
+        jump(i0)
+        """
+        self.optimize_loop(ops, expected)
+        ops = """
+        [p0]
+        i1 = cast_ptr_to_int(p0)
+        p1 = cast_int_to_ptr(i1)
+        jump(p1)
+        """
+        expected = """
+        [p0]
+        jump(p0)
+        """
+        self.optimize_loop(ops, expected)
+
     # ----------
 
     def test_remove_guard_class_1(self):
@@ -2659,7 +2683,7 @@
         ops = """
         [p1]
         guard_class(p1, ConstClass(node_vtable2)) []
-        i = ptr_ne(ConstPtr(myptr), p1)
+        i = instance_ptr_ne(ConstPtr(myptr), p1)
         guard_true(i) []
         jump(p1)
         """
@@ -3307,7 +3331,7 @@
         jump(p1, i1, i2, i6)
         '''
         self.optimize_loop(ops, expected, preamble)
-        
+
 
     # ----------
 
@@ -5776,10 +5800,12 @@
         class FakeCallInfoCollection:
             def callinfo_for_oopspec(self, oopspecindex):
                 calldescrtype = type(LLtypeMixin.strequaldescr)
+                effectinfotype = type(LLtypeMixin.strequaldescr.get_extra_info())
                 for value in LLtypeMixin.__dict__.values():
                     if isinstance(value, calldescrtype):
                         extra = value.get_extra_info()
-                        if extra and extra.oopspecindex == oopspecindex:
+                        if (extra and isinstance(extra, effectinfotype) and
+                            extra.oopspecindex == oopspecindex):
                             # returns 0 for 'func' in this test
                             return value, 0
                 raise AssertionError("not found: oopspecindex=%d" %
@@ -7256,7 +7282,7 @@
         ops = """
         [p1, p2]
         setarrayitem_gc(p1, 2, 10, descr=arraydescr)
-        setarrayitem_gc(p2, 3, 13, descr=arraydescr)        
+        setarrayitem_gc(p2, 3, 13, descr=arraydescr)
         call(0, p1, p2, 0, 0, 10, descr=arraycopydescr)
         jump(p1, p2)
         """
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_util.py b/pypy/jit/metainterp/optimizeopt/test/test_util.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_util.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_util.py
@@ -185,6 +185,18 @@
              EffectInfo([], [arraydescr], [], [arraydescr],
                         oopspecindex=EffectInfo.OS_ARRAYCOPY))
 
+
+    # array of structs (complex data)
+    complexarray = lltype.GcArray(
+        lltype.Struct("complex",
+            ("real", lltype.Float),
+            ("imag", lltype.Float),
+        )
+    )
+    complexarraydescr = cpu.arraydescrof(complexarray)
+    complexrealdescr = cpu.interiorfielddescrof(complexarray, "real")
+    compleximagdescr = cpu.interiorfielddescrof(complexarray, "imag")
+
     for _name, _os in [
         ('strconcatdescr',               'OS_STR_CONCAT'),
         ('strslicedescr',                'OS_STR_SLICE'),
@@ -240,7 +252,7 @@
 ##    def get_class_of_box(self, box):
 ##        root = box.getref(ootype.ROOT)
 ##        return ootype.classof(root)
-    
+
 ##    cpu = runner.OOtypeCPU(None)
 ##    NODE = ootype.Instance('NODE', ootype.ROOT, {})
 ##    NODE._add_fields({'value': ootype.Signed,
diff --git a/pypy/jit/metainterp/optimizeopt/virtualize.py b/pypy/jit/metainterp/optimizeopt/virtualize.py
--- a/pypy/jit/metainterp/optimizeopt/virtualize.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualize.py
@@ -59,7 +59,7 @@
 
     def import_from(self, other, optimizer):
         raise NotImplementedError("should not be called at this level")
-    
+
 def get_fielddescrlist_cache(cpu):
     if not hasattr(cpu, '_optimizeopt_fielddescrlist_cache'):
         result = descrlist_dict()
@@ -113,7 +113,7 @@
         #
         if not we_are_translated():
             op.name = 'FORCE ' + self.source_op.name
-            
+
         if self._is_immutable_and_filled_with_constants(optforce):
             box = optforce.optimizer.constant_fold(op)
             self.make_constant(box)
@@ -239,12 +239,12 @@
         for index in range(len(self._items)):
             self._items[index] = self._items[index].force_at_end_of_preamble(already_forced, optforce)
         return self
-    
+
     def _really_force(self, optforce):
         assert self.source_op is not None
         if not we_are_translated():
             self.source_op.name = 'FORCE ' + self.source_op.name
-        optforce.emit_operation(self.source_op)        
+        optforce.emit_operation(self.source_op)
         self.box = box = self.source_op.result
         for index in range(len(self._items)):
             subvalue = self._items[index]
@@ -271,20 +271,86 @@
     def _make_virtual(self, modifier):
         return modifier.make_varray(self.arraydescr)
 
+class VArrayStructValue(AbstractVirtualValue):
+    def __init__(self, arraydescr, size, keybox, source_op=None):
+        AbstractVirtualValue.__init__(self, keybox, source_op)
+        self.arraydescr = arraydescr
+        self._items = [{} for _ in xrange(size)]
+
+    def getlength(self):
+        return len(self._items)
+
+    def getinteriorfield(self, index, ofs, default):
+        return self._items[index].get(ofs, default)
+
+    def setinteriorfield(self, index, ofs, itemvalue):
+        assert isinstance(itemvalue, optimizer.OptValue)
+        self._items[index][ofs] = itemvalue
+
+    def _really_force(self, optforce):
+        assert self.source_op is not None
+        if not we_are_translated():
+            self.source_op.name = 'FORCE ' + self.source_op.name
+        optforce.emit_operation(self.source_op)
+        self.box = box = self.source_op.result
+        for index in range(len(self._items)):
+            for descr, value in self._items[index].iteritems():
+                subbox = value.force_box(optforce)
+                op = ResOperation(rop.SETINTERIORFIELD_GC,
+                    [box, ConstInt(index), subbox], None, descr=descr
+                )
+                optforce.emit_operation(op)
+
+    def _get_list_of_descrs(self):
+        descrs = []
+        for item in self._items:
+            item_descrs = item.keys()
+            sort_descrs(item_descrs)
+            descrs.append(item_descrs)
+        return descrs
+
+    def get_args_for_fail(self, modifier):
+        if self.box is None and not modifier.already_seen_virtual(self.keybox):
+            itemdescrs = self._get_list_of_descrs()
+            itemboxes = []
+            for i in range(len(self._items)):
+                for descr in itemdescrs[i]:
+                    itemboxes.append(self._items[i][descr].get_key_box())
+            modifier.register_virtual_fields(self.keybox, itemboxes)
+            for i in range(len(self._items)):
+                for descr in itemdescrs[i]:
+                    self._items[i][descr].get_args_for_fail(modifier)
+
+    def force_at_end_of_preamble(self, already_forced, optforce):
+        if self in already_forced:
+            return self
+        already_forced[self] = self
+        for index in range(len(self._items)):
+            for descr in self._items[index].keys():
+                self._items[index][descr] = self._items[index][descr].force_at_end_of_preamble(already_forced, optforce)
+        return self
+
+    def _make_virtual(self, modifier):
+        return modifier.make_varraystruct(self.arraydescr, self._get_list_of_descrs())
+
+
 class OptVirtualize(optimizer.Optimization):
     "Virtualize objects until they escape."
 
     def new(self):
         return OptVirtualize()
-        
+
     def make_virtual(self, known_class, box, source_op=None):
         vvalue = VirtualValue(self.optimizer.cpu, known_class, box, source_op)
         self.make_equal_to(box, vvalue)
         return vvalue
 
     def make_varray(self, arraydescr, size, box, source_op=None):
-        constvalue = self.new_const_item(arraydescr)
-        vvalue = VArrayValue(arraydescr, constvalue, size, box, source_op)
+        if arraydescr.is_array_of_structs():
+            vvalue = VArrayStructValue(arraydescr, size, box, source_op)
+        else:
+            constvalue = self.new_const_item(arraydescr)
+            vvalue = VArrayValue(arraydescr, constvalue, size, box, source_op)
         self.make_equal_to(box, vvalue)
         return vvalue
 
@@ -431,6 +497,34 @@
         value.ensure_nonnull()
         self.emit_operation(op)
 
+    def optimize_GETINTERIORFIELD_GC(self, op):
+        value = self.getvalue(op.getarg(0))
+        if value.is_virtual():
+            indexbox = self.get_constant_box(op.getarg(1))
+            if indexbox is not None:
+                descr = op.getdescr()
+                fieldvalue = value.getinteriorfield(
+                    indexbox.getint(), descr, None
+                )
+                if fieldvalue is None:
+                    fieldvalue = self.new_const(descr)
+                self.make_equal_to(op.result, fieldvalue)
+                return
+        value.ensure_nonnull()
+        self.emit_operation(op)
+
+    def optimize_SETINTERIORFIELD_GC(self, op):
+        value = self.getvalue(op.getarg(0))
+        if value.is_virtual():
+            indexbox = self.get_constant_box(op.getarg(1))
+            if indexbox is not None:
+                value.setinteriorfield(
+                    indexbox.getint(), op.getdescr(), self.getvalue(op.getarg(2))
+                )
+                return
+        value.ensure_nonnull()
+        self.emit_operation(op)
+
 
 dispatch_opt = make_dispatcher_method(OptVirtualize, 'optimize_',
         default=OptVirtualize.emit_operation)
diff --git a/pypy/jit/metainterp/optimizeopt/virtualstate.py b/pypy/jit/metainterp/optimizeopt/virtualstate.py
--- a/pypy/jit/metainterp/optimizeopt/virtualstate.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualstate.py
@@ -16,7 +16,7 @@
 
 class AbstractVirtualStateInfo(resume.AbstractVirtualInfo):
     position = -1
-    
+
     def generalization_of(self, other, renum, bad):
         raise NotImplementedError
 
@@ -54,7 +54,7 @@
                 s.debug_print(indent + "    ", seen, bad)
         else:
             debug_print(indent + "    ...")
-                
+
 
     def debug_header(self, indent):
         raise NotImplementedError
@@ -77,13 +77,15 @@
             bad[self] = True
             bad[other] = True
             return False
+
+        assert isinstance(other, AbstractVirtualStructStateInfo)
         assert len(self.fielddescrs) == len(self.fieldstate)
         assert len(other.fielddescrs) == len(other.fieldstate)
         if len(self.fielddescrs) != len(other.fielddescrs):
             bad[self] = True
             bad[other] = True
             return False
-        
+
         for i in range(len(self.fielddescrs)):
             if other.fielddescrs[i] is not self.fielddescrs[i]:
                 bad[self] = True
@@ -112,8 +114,8 @@
     def _enum(self, virtual_state):
         for s in self.fieldstate:
             s.enum(virtual_state)
-        
-        
+
+
 class VirtualStateInfo(AbstractVirtualStructStateInfo):
     def __init__(self, known_class, fielddescrs):
         AbstractVirtualStructStateInfo.__init__(self, fielddescrs)
@@ -128,13 +130,13 @@
 
     def debug_header(self, indent):
         debug_print(indent + 'VirtualStateInfo(%d):' % self.position)
-        
+
 class VStructStateInfo(AbstractVirtualStructStateInfo):
     def __init__(self, typedescr, fielddescrs):
         AbstractVirtualStructStateInfo.__init__(self, fielddescrs)
         self.typedescr = typedescr
 
-    def _generalization_of(self, other):        
+    def _generalization_of(self, other):
         if not isinstance(other, VStructStateInfo):
             return False
         if self.typedescr is not other.typedescr:
@@ -143,7 +145,7 @@
 
     def debug_header(self, indent):
         debug_print(indent + 'VStructStateInfo(%d):' % self.position)
-        
+
 class VArrayStateInfo(AbstractVirtualStateInfo):
     def __init__(self, arraydescr):
         self.arraydescr = arraydescr
@@ -157,11 +159,7 @@
             bad[other] = True
             return False
         renum[self.position] = other.position
-        if not isinstance(other, VArrayStateInfo):
-            bad[self] = True
-            bad[other] = True
-            return False
-        if self.arraydescr is not other.arraydescr:
+        if not self._generalization_of(other):
             bad[self] = True
             bad[other] = True
             return False
@@ -177,6 +175,10 @@
                 return False
         return True
 
+    def _generalization_of(self, other):
+        return (isinstance(other, VArrayStateInfo) and
+            self.arraydescr is other.arraydescr)
+
     def enum_forced_boxes(self, boxes, value, optimizer):
         assert isinstance(value, virtualize.VArrayValue)
         assert value.is_virtual()
@@ -192,8 +194,75 @@
 
     def debug_header(self, indent):
         debug_print(indent + 'VArrayStateInfo(%d):' % self.position)
-            
-        
+
+class VArrayStructStateInfo(AbstractVirtualStateInfo):
+    def __init__(self, arraydescr, fielddescrs):
+        self.arraydescr = arraydescr
+        self.fielddescrs = fielddescrs
+
+    def generalization_of(self, other, renum, bad):
+        assert self.position != -1
+        if self.position in renum:
+            if renum[self.position] == other.position:
+                return True
+            bad[self] = True
+            bad[other] = True
+            return False
+        renum[self.position] = other.position
+        if not self._generalization_of(other):
+            bad[self] = True
+            bad[other] = True
+            return False
+
+        assert isinstance(other, VArrayStructStateInfo)
+        if len(self.fielddescrs) != len(other.fielddescrs):
+            bad[self] = True
+            bad[other] = True
+            return False
+
+        p = 0
+        for i in range(len(self.fielddescrs)):
+            if len(self.fielddescrs[i]) != len(other.fielddescrs[i]):
+                bad[self] = True
+                bad[other] = True
+                return False
+            for j in range(len(self.fielddescrs[i])):
+                if self.fielddescrs[i][j] is not other.fielddescrs[i][j]:
+                    bad[self] = True
+                    bad[other] = True
+                    return False
+                if not self.fieldstate[p].generalization_of(other.fieldstate[p],
+                                                            renum, bad):
+                    bad[self] = True
+                    bad[other] = True
+                    return False
+                p += 1
+        return True
+
+    def _generalization_of(self, other):
+        return (isinstance(other, VArrayStructStateInfo) and
+            self.arraydescr is other.arraydescr)
+
+    def _enum(self, virtual_state):
+        for s in self.fieldstate:
+            s.enum(virtual_state)
+
+    def enum_forced_boxes(self, boxes, value, optimizer):
+        assert isinstance(value, virtualize.VArrayStructValue)
+        assert value.is_virtual()
+        p = 0
+        for i in range(len(self.fielddescrs)):
+            for j in range(len(self.fielddescrs[i])):
+                v = value._items[i][self.fielddescrs[i][j]]
+                s = self.fieldstate[p]
+                if s.position > self.position:
+                    s.enum_forced_boxes(boxes, v, optimizer)
+                p += 1
+
+    def debug_header(self, indent):
+        debug_print(indent + 'VArrayStructStateInfo(%d):' % self.position)
+
+
 class NotVirtualStateInfo(AbstractVirtualStateInfo):
     def __init__(self, value):
         self.known_class = value.known_class
@@ -277,7 +346,7 @@
             op = ResOperation(rop.GUARD_CLASS, [box, self.known_class], None)
             extra_guards.append(op)
             return
-        
+
         if self.level == LEVEL_NONNULL and \
                other.level == LEVEL_UNKNOWN and \
                isinstance(box, BoxPtr) and \
@@ -285,7 +354,7 @@
             op = ResOperation(rop.GUARD_NONNULL, [box], None)
             extra_guards.append(op)
             return
-        
+
         if self.level == LEVEL_UNKNOWN and \
                other.level == LEVEL_UNKNOWN and \
                isinstance(box, BoxInt) and \
@@ -309,7 +378,7 @@
                     op = ResOperation(rop.GUARD_TRUE, [res], None)
                     extra_guards.append(op)
             return
-        
+
         # Remaining cases are probably not interesting
         raise InvalidLoop
         if self.level == LEVEL_CONSTANT:
@@ -319,7 +388,7 @@
     def enum_forced_boxes(self, boxes, value, optimizer):
         if self.level == LEVEL_CONSTANT:
             return
-        assert 0 <= self.position_in_notvirtuals 
+        assert 0 <= self.position_in_notvirtuals
         boxes[self.position_in_notvirtuals] = value.force_box(optimizer)
 
     def _enum(self, virtual_state):
@@ -348,7 +417,7 @@
         lb = ''
         if self.lenbound:
             lb = ', ' + self.lenbound.bound.__repr__()
-        
+
         debug_print(indent + mark + 'NotVirtualInfo(%d' % self.position +
                     ', ' + l + ', ' + self.intbound.__repr__() + lb + ')')
 
@@ -370,7 +439,7 @@
                 return False
         return True
 
-    def generate_guards(self, other, args, cpu, extra_guards):        
+    def generate_guards(self, other, args, cpu, extra_guards):
         assert len(self.state) == len(other.state) == len(args)
         renum = {}
         for i in range(len(self.state)):
@@ -393,7 +462,7 @@
                     inputargs.append(box)
 
         assert None not in inputargs
-            
+
         return inputargs
 
     def debug_print(self, hdr='', bad=None):
@@ -412,7 +481,7 @@
 
     def register_virtual_fields(self, keybox, fieldboxes):
         self.fieldboxes[keybox] = fieldboxes
-        
+
     def already_seen_virtual(self, keybox):
         return keybox in self.fieldboxes
 
@@ -463,6 +532,9 @@
     def make_varray(self, arraydescr):
         return VArrayStateInfo(arraydescr)
 
+    def make_varraystruct(self, arraydescr, fielddescrs):
+        return VArrayStructStateInfo(arraydescr, fielddescrs)
+
 class BoxNotProducable(Exception):
     pass
 
@@ -501,12 +573,12 @@
             else: # Low priority
                 lo -= 1
         return alts
-            
+
     def renamed(self, box):
         if box in self.rename:
             return self.rename[box]
         return box
-    
+
     def add_to_short(self, box, op):
         if op:
             op = op.clone()
@@ -528,12 +600,12 @@
             self.optimizer.make_equal_to(newbox, value)
         else:
             self.short_boxes[box] = op
-        
+
     def produce_short_preamble_box(self, box):
         if box in self.short_boxes:
-            return 
+            return
         if isinstance(box, Const):
-            return 
+            return
         if box in self.potential_ops:
             ops = self.prioritized_alternatives(box)
             produced_one = False
@@ -570,7 +642,7 @@
             else:
                 debug_print(logops.repr_of_arg(box) + ': None')
         debug_stop('jit-short-boxes')
-        
+
     def operations(self):
         if not we_are_translated(): # For tests
             ops = self.short_boxes.values()
@@ -588,7 +660,7 @@
         if not isinstance(oldbox, Const) and newbox not in self.short_boxes:
             self.short_boxes[newbox] = self.short_boxes[oldbox]
         self.aliases[newbox] = oldbox
-        
+
     def original(self, box):
         while box in self.aliases:
             box = self.aliases[box]
diff --git a/pypy/jit/metainterp/optimizeopt/vstring.py b/pypy/jit/metainterp/optimizeopt/vstring.py
--- a/pypy/jit/metainterp/optimizeopt/vstring.py
+++ b/pypy/jit/metainterp/optimizeopt/vstring.py
@@ -163,17 +163,6 @@
             for value in self._chars:
                 value.get_args_for_fail(modifier)
 
-    def FIXME_enum_forced_boxes(self, boxes, already_seen):
-        key = self.get_key_box()
-        if key in already_seen:
-            return
-        already_seen[key] = None
-        if self.box is None:
-            for box in self._chars:
-                box.enum_forced_boxes(boxes, already_seen)
-        else:
-            boxes.append(self.box)
-
     def _make_virtual(self, modifier):
         return modifier.make_vstrplain(self.mode is mode_unicode)
 
@@ -226,18 +215,6 @@
             self.left.get_args_for_fail(modifier)
             self.right.get_args_for_fail(modifier)
 
-    def FIXME_enum_forced_boxes(self, boxes, already_seen):
-        key = self.get_key_box()
-        if key in already_seen:
-            return
-        already_seen[key] = None
-        if self.box is None:
-            self.left.enum_forced_boxes(boxes, already_seen)
-            self.right.enum_forced_boxes(boxes, already_seen)
-            self.lengthbox = None
-        else:
-            boxes.append(self.box)
-
     def _make_virtual(self, modifier):
         return modifier.make_vstrconcat(self.mode is mode_unicode)
 
@@ -284,18 +261,6 @@
             self.vstart.get_args_for_fail(modifier)
             self.vlength.get_args_for_fail(modifier)
 
-    def FIXME_enum_forced_boxes(self, boxes, already_seen):
-        key = self.get_key_box()
-        if key in already_seen:
-            return
-        already_seen[key] = None
-        if self.box is None:
-            self.vstr.enum_forced_boxes(boxes, already_seen)
-            self.vstart.enum_forced_boxes(boxes, already_seen)
-            self.vlength.enum_forced_boxes(boxes, already_seen)
-        else:
-            boxes.append(self.box)
-
     def _make_virtual(self, modifier):
         return modifier.make_vstrslice(self.mode is mode_unicode)
 
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -36,6 +36,7 @@
 
 
 class MIFrame(object):
+    debug = False
 
     def __init__(self, metainterp):
         self.metainterp = metainterp
@@ -164,7 +165,7 @@
         if not we_are_translated():
             for b in registers[count:]:
                 assert not oldbox.same_box(b)
-                
+
 
     def make_result_of_lastop(self, resultbox):
         got_type = resultbox.type
@@ -198,7 +199,7 @@
                     'float_add', 'float_sub', 'float_mul', 'float_truediv',
                     'float_lt', 'float_le', 'float_eq',
                     'float_ne', 'float_gt', 'float_ge',
-                    'ptr_eq', 'ptr_ne',
+                    'ptr_eq', 'ptr_ne', 'instance_ptr_eq', 'instance_ptr_ne',
                     ]:
         exec py.code.Source('''
             @arguments("box", "box")
@@ -222,6 +223,7 @@
                     'cast_float_to_int', 'cast_int_to_float',
                     'cast_float_to_singlefloat', 'cast_singlefloat_to_float',
                     'float_neg', 'float_abs',
+                    'cast_ptr_to_int', 'cast_int_to_ptr',
                     ]:
         exec py.code.Source('''
             @arguments("box")
@@ -238,8 +240,8 @@
         return self.execute(rop.PTR_EQ, box, history.CONST_NULL)
 
     @arguments("box")
-    def opimpl_cast_opaque_ptr(self, box):
-        return self.execute(rop.CAST_OPAQUE_PTR, box)
+    def opimpl_mark_opaque_ptr(self, box):
+        return self.execute(rop.MARK_OPAQUE_PTR, box)
 
     @arguments("box")
     def _opimpl_any_return(self, box):
@@ -547,6 +549,14 @@
     opimpl_getfield_gc_r_pure = _opimpl_getfield_gc_pure_any
     opimpl_getfield_gc_f_pure = _opimpl_getfield_gc_pure_any
 
+    @arguments("box", "box", "descr")
+    def _opimpl_getinteriorfield_gc_any(self, array, index, descr):
+        return self.execute_with_descr(rop.GETINTERIORFIELD_GC, descr,
+                                       array, index)
+    opimpl_getinteriorfield_gc_i = _opimpl_getinteriorfield_gc_any
+    opimpl_getinteriorfield_gc_f = _opimpl_getinteriorfield_gc_any
+    opimpl_getinteriorfield_gc_r = _opimpl_getinteriorfield_gc_any
+
     @specialize.arg(1)
     def _opimpl_getfield_gc_any_pureornot(self, opnum, box, fielddescr):
         tobox = self.metainterp.heapcache.getfield(box, fielddescr)
@@ -587,6 +597,15 @@
     opimpl_setfield_gc_r = _opimpl_setfield_gc_any
     opimpl_setfield_gc_f = _opimpl_setfield_gc_any
 
+    @arguments("box", "box", "box", "descr")
+    def _opimpl_setinteriorfield_gc_any(self, array, index, value, descr):
+        self.execute_with_descr(rop.SETINTERIORFIELD_GC, descr,
+                                array, index, value)
+    opimpl_setinteriorfield_gc_i = _opimpl_setinteriorfield_gc_any
+    opimpl_setinteriorfield_gc_f = _opimpl_setinteriorfield_gc_any
+    opimpl_setinteriorfield_gc_r = _opimpl_setinteriorfield_gc_any
+
+
     @arguments("box", "descr")
     def _opimpl_getfield_raw_any(self, box, fielddescr):
         return self.execute_with_descr(rop.GETFIELD_RAW, fielddescr, box)
@@ -2587,17 +2606,21 @@
         self.pc = position
         #
         if not we_are_translated():
-            print '\tpyjitpl: %s(%s)' % (name, ', '.join(map(repr, args))),
+            if self.debug:
+                print '\tpyjitpl: %s(%s)' % (name, ', '.join(map(repr, args))),
             try:
                 resultbox = unboundmethod(self, *args)
             except Exception, e:
-                print '-> %s!' % e.__class__.__name__
+                if self.debug:
+                    print '-> %s!' % e.__class__.__name__
                 raise
             if num_return_args == 0:
-                print
+                if self.debug:
+                    print
                 assert resultbox is None
             else:
-                print '-> %r' % (resultbox,)
+                if self.debug:
+                    print '-> %r' % (resultbox,)
                 assert argcodes[next_argcode] == '>'
                 result_argcode = argcodes[next_argcode + 1]
                 assert resultbox.type == {'i': history.INT,
diff --git a/pypy/jit/metainterp/quasiimmut.py b/pypy/jit/metainterp/quasiimmut.py
--- a/pypy/jit/metainterp/quasiimmut.py
+++ b/pypy/jit/metainterp/quasiimmut.py
@@ -2,6 +2,7 @@
 from pypy.rpython.lltypesystem import lltype, rclass
 from pypy.rpython.annlowlevel import cast_base_ptr_to_instance
 from pypy.jit.metainterp.history import AbstractDescr
+from pypy.rlib.objectmodel import we_are_translated
 
 
 def get_mutate_field_name(fieldname):
@@ -50,13 +51,13 @@
 
 class QuasiImmut(object):
     llopaque = True
+    compress_limit = 30
     
     def __init__(self, cpu):
         self.cpu = cpu
         # list of weakrefs to the LoopTokens that must be invalidated if
         # this value ever changes
         self.looptokens_wrefs = []
-        self.compress_limit = 30
 
     def hide(self):
         qmut_ptr = self.cpu.ts.cast_instance_to_base_ref(self)
@@ -75,6 +76,8 @@
     def compress_looptokens_list(self):
         self.looptokens_wrefs = [wref for wref in self.looptokens_wrefs
                                       if wref() is not None]
+        # NB. we must keep around the looptoken_wrefs that are
+        # already invalidated; see below
         self.compress_limit = (len(self.looptokens_wrefs) + 15) * 2
 
     def invalidate(self):
@@ -86,7 +89,16 @@
         for wref in wrefs:
             looptoken = wref()
             if looptoken is not None:
+                looptoken.invalidated = True
                 self.cpu.invalidate_loop(looptoken)
+                # NB. we must call cpu.invalidate_loop() even if
+                # looptoken.invalidated was already set to True.
+                # It's possible to invalidate several times the
+                # same looptoken; see comments in jit.backend.model
+                # in invalidate_loop().
+                if not we_are_translated():
+                    self.cpu.stats.invalidated_token_numbers.add(
+                        looptoken.number)
 
 
 class QuasiImmutDescr(AbstractDescr):
diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py
--- a/pypy/jit/metainterp/resoperation.py
+++ b/pypy/jit/metainterp/resoperation.py
@@ -1,5 +1,4 @@
 from pypy.rlib.objectmodel import we_are_translated
-from pypy.rlib.debug import make_sure_not_resized
 
 def ResOperation(opnum, args, result, descr=None):
     cls = opclasses[opnum]
@@ -405,8 +404,8 @@
     'FLOAT_TRUEDIV/2',
     'FLOAT_NEG/1',
     'FLOAT_ABS/1',
-    'CAST_FLOAT_TO_INT/1',
-    'CAST_INT_TO_FLOAT/1',
+    'CAST_FLOAT_TO_INT/1',          # don't use for unsigned ints; we would
+    'CAST_INT_TO_FLOAT/1',          # need some messy code in the backend
     'CAST_FLOAT_TO_SINGLEFLOAT/1',
     'CAST_SINGLEFLOAT_TO_FLOAT/1',
     #
@@ -433,10 +432,13 @@
     'INT_INVERT/1',
     #
     'SAME_AS/1',      # gets a Const or a Box, turns it into another Box
+    'CAST_PTR_TO_INT/1',
+    'CAST_INT_TO_PTR/1',
     #
     'PTR_EQ/2b',
     'PTR_NE/2b',
-    'CAST_OPAQUE_PTR/1b',
+    'INSTANCE_PTR_EQ/2b',
+    'INSTANCE_PTR_NE/2b',
     #
     'ARRAYLEN_GC/1d',
     'STRLEN/1',
@@ -455,6 +457,7 @@
 
     'GETARRAYITEM_GC/2d',
     'GETARRAYITEM_RAW/2d',
+    'GETINTERIORFIELD_GC/2d',
     'GETFIELD_GC/1d',
     'GETFIELD_RAW/1d',
     '_MALLOC_FIRST',
@@ -467,10 +470,12 @@
     'FORCE_TOKEN/0',
     'VIRTUAL_REF/2',         # removed before it's passed to the backend
     'READ_TIMESTAMP/0',
+    'MARK_OPAQUE_PTR/1b',
     '_NOSIDEEFFECT_LAST', # ----- end of no_side_effect operations -----
 
     'SETARRAYITEM_GC/3d',
     'SETARRAYITEM_RAW/3d',
+    'SETINTERIORFIELD_GC/3d',
     'SETFIELD_GC/2d',
     'SETFIELD_RAW/2d',
     'STRSETITEM/3',
diff --git a/pypy/jit/metainterp/resume.py b/pypy/jit/metainterp/resume.py
--- a/pypy/jit/metainterp/resume.py
+++ b/pypy/jit/metainterp/resume.py
@@ -139,7 +139,7 @@
         self.numberings = {}
         self.cached_boxes = {}
         self.cached_virtuals = {}
-    
+
         self.nvirtuals = 0
         self.nvholes = 0
         self.nvreused = 0
@@ -273,6 +273,9 @@
     def make_varray(self, arraydescr):
         return VArrayInfo(arraydescr)
 
+    def make_varraystruct(self, arraydescr, fielddescrs):
+        return VArrayStructInfo(arraydescr, fielddescrs)
+
     def make_vstrplain(self, is_unicode=False):
         if is_unicode:
             return VUniPlainInfo()
@@ -402,7 +405,7 @@
                 virtuals[num] = vinfo
 
         if self._invalidation_needed(len(liveboxes), nholes):
-            memo.clear_box_virtual_numbers()           
+            memo.clear_box_virtual_numbers()
 
     def _invalidation_needed(self, nliveboxes, nholes):
         memo = self.memo
@@ -455,7 +458,7 @@
 
     def debug_prints(self):
         raise NotImplementedError
-        
+
 class AbstractVirtualStructInfo(AbstractVirtualInfo):
     def __init__(self, fielddescrs):
         self.fielddescrs = fielddescrs
@@ -537,6 +540,29 @@
         for i in self.fieldnums:
             debug_print("\t\t", str(untag(i)))
 
+
+class VArrayStructInfo(AbstractVirtualInfo):
+    def __init__(self, arraydescr, fielddescrs):
+        self.arraydescr = arraydescr
+        self.fielddescrs = fielddescrs
+
+    def debug_prints(self):
+        debug_print("\tvarraystructinfo", self.arraydescr)
+        for i in self.fieldnums:
+            debug_print("\t\t", str(untag(i)))
+
+    @specialize.argtype(1)
+    def allocate(self, decoder, index):
+        array = decoder.allocate_array(self.arraydescr, len(self.fielddescrs))
+        decoder.virtuals_cache[index] = array
+        p = 0
+        for i in range(len(self.fielddescrs)):
+            for j in range(len(self.fielddescrs[i])):
+                decoder.setinteriorfield(i, self.fielddescrs[i][j], array, self.fieldnums[p])
+                p += 1
+        return array
+
+
 class VStrPlainInfo(AbstractVirtualInfo):
     """Stands for the string made out of the characters of all fieldnums."""
 
@@ -884,6 +910,17 @@
         self.metainterp.execute_and_record(rop.SETFIELD_GC, descr,
                                            structbox, fieldbox)
 
+    def setinteriorfield(self, index, descr, array, fieldnum):
+        if descr.is_pointer_field():
+            kind = REF
+        elif descr.is_float_field():
+            kind = FLOAT
+        else:
+            kind = INT
+        fieldbox = self.decode_box(fieldnum, kind)
+        self.metainterp.execute_and_record(rop.SETINTERIORFIELD_GC, descr,
+                                           array, ConstInt(index), fieldbox)
+
     def setarrayitem_int(self, arraydescr, arraybox, index, fieldnum):
         self._setarrayitem(arraydescr, arraybox, index, fieldnum, INT)
 
@@ -1164,6 +1201,17 @@
             newvalue = self.decode_int(fieldnum)
             self.cpu.bh_setfield_gc_i(struct, descr, newvalue)
 
+    def setinteriorfield(self, index, descr, array, fieldnum):
+        if descr.is_pointer_field():
+            newvalue = self.decode_ref(fieldnum)
+            self.cpu.bh_setinteriorfield_gc_r(array, index, descr, newvalue)
+        elif descr.is_float_field():
+            newvalue = self.decode_float(fieldnum)
+            self.cpu.bh_setinteriorfield_gc_f(array, index, descr, newvalue)
+        else:
+            newvalue = self.decode_int(fieldnum)
+            self.cpu.bh_setinteriorfield_gc_i(array, index, descr, newvalue)
+
     def setarrayitem_int(self, arraydescr, array, index, fieldnum):
         newvalue = self.decode_int(fieldnum)
         self.cpu.bh_setarrayitem_gc_i(arraydescr, array, index, newvalue)
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -10,6 +10,7 @@
 from pypy.jit.metainterp.typesystem import LLTypeHelper, OOTypeHelper
 from pypy.jit.metainterp.warmspot import get_stats
 from pypy.jit.metainterp.warmstate import set_future_value
+from pypy.rlib import rerased
 from pypy.rlib.jit import (JitDriver, we_are_jitted, hint, dont_look_inside,
     loop_invariant, elidable, promote, jit_debug, assert_green,
     AssertGreenFailed, unroll_safe, current_trace_length, look_inside_iff,
@@ -3435,12 +3436,133 @@
             return sa
         res = self.meta_interp(f, [16])
         assert res == f(16)
-        
+
+    def test_ptr_eq(self):
+        myjitdriver = JitDriver(greens = [], reds = ["n", "x"])
+        class A(object):
+            def __init__(self, v):
+                self.v = v
+        def f(n, x):
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, x=x)
+                z = 0 / x
+                a1 = A("key")
+                a2 = A("\x00")
+                n -= [a1, a2][z].v is not a2.v
+            return n
+        res = self.meta_interp(f, [10, 1])
+        assert res == 0
+
+    def test_instance_ptr_eq(self):
+        myjitdriver = JitDriver(greens = [], reds = ["n", "i", "a1", "a2"])
+        class A(object):
+            pass
+        def f(n):
+            a1 = A()
+            a2 = A()
+            i = 0
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, i=i, a1=a1, a2=a2)
+                if n % 2:
+                    a = a2
+                else:
+                    a = a1
+                i += a is a1
+                n -= 1
+            return i
+        res = self.meta_interp(f, [10])
+        assert res == f(10)
+        def f(n):
+            a1 = A()
+            a2 = A()
+            i = 0
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, i=i, a1=a1, a2=a2)
+                if n % 2:
+                    a = a2
+                else:
+                    a = a1
+                if a is a2:
+                    i += 1
+                n -= 1
+            return i
+        res = self.meta_interp(f, [10])
+        assert res == f(10)
+
+    def test_virtual_array_of_structs(self):
+        myjitdriver = JitDriver(greens = [], reds=["n", "d"])
+        def f(n):
+            d = None
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, d=d)
+                d = {"q": 1}
+                if n % 2:
+                    d["k"] = n
+                else:
+                    d["z"] = n
+                n -= len(d) - d["q"]
+            return n
+        res = self.meta_interp(f, [10])
+        assert res == 0
+
+    def test_virtual_dict_constant_keys(self):
+        myjitdriver = JitDriver(greens = [], reds = ["n"])
+        def g(d):
+            return d["key"] - 1
+
+        def f(n):
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n)
+                n = g({"key": n})
+            return n
+
+        res = self.meta_interp(f, [10])
+        assert res == 0
+        self.check_loops({"int_sub": 1, "int_gt": 1, "guard_true": 1, "jump": 1})
+
+    def test_virtual_opaque_ptr(self):
+        myjitdriver = JitDriver(greens = [], reds = ["n"])
+        erase, unerase = rerased.new_erasing_pair("x")
+        @look_inside_iff(lambda x: isvirtual(x))
+        def g(x):
+            return x[0]
+        def f(n):
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n)
+                x = []
+                y = erase(x)
+                z = unerase(y)
+                z.append(1)
+                n -= g(z)
+            return n
+        res = self.meta_interp(f, [10])
+        assert res == 0
+        self.check_loops({"int_sub": 1, "int_gt": 1, "guard_true": 1, "jump": 1})
+
+    def test_virtual_opaque_dict(self):
+        myjitdriver = JitDriver(greens = [], reds = ["n"])
+        erase, unerase = rerased.new_erasing_pair("x")
+        @look_inside_iff(lambda x: isvirtual(x))
+        def g(x):
+            return x[0]["key"] - 1
+        def f(n):
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n)
+                x = [{}]
+                x[0]["key"] = n
+                x[0]["other key"] = n
+                y = erase(x)
+                z = unerase(y)
+                n = g(x)
+            return n
+        res = self.meta_interp(f, [10])
+        assert res == 0
+        self.check_loops({"int_sub": 1, "int_gt": 1, "guard_true": 1, "jump": 1})
+
 
 
 class TestLLtype(BaseLLtypeTests, LLJitMixin):
     def test_tagged(self):
-        py.test.skip("implement me")
         from pypy.rlib.objectmodel import UnboxedValue
         class Base(object):
             __slots__ = ()
@@ -3492,3 +3614,34 @@
                 pc += 1
             return pc
         res = self.meta_interp(main, [False, 100, True], taggedpointers=True)
+
+    def test_rerased(self):
+        eraseX, uneraseX = rerased.new_erasing_pair("X")
+        #
+        class X:
+            def __init__(self, a, b):
+                self.a = a
+                self.b = b
+        #
+        def f(i, j):
+            # 'j' should be 0 or 1, not other values
+            if j > 0:
+                e = eraseX(X(i, j))
+            else:
+                try:
+                    e = rerased.erase_int(i)
+                except OverflowError:
+                    return -42
+            if j & 1:
+                x = uneraseX(e)
+                return x.a - x.b
+            else:
+                return rerased.unerase_int(e)
+        #
+        x = self.interp_operations(f, [-128, 0], taggedpointers=True)
+        assert x == -128
+        bigint = sys.maxint//2 + 1
+        x = self.interp_operations(f, [bigint, 0], taggedpointers=True)
+        assert x == -42
+        x = self.interp_operations(f, [1000, 1], taggedpointers=True)
+        assert x == 999
diff --git a/pypy/jit/metainterp/test/test_dict.py b/pypy/jit/metainterp/test/test_dict.py
--- a/pypy/jit/metainterp/test/test_dict.py
+++ b/pypy/jit/metainterp/test/test_dict.py
@@ -91,7 +91,7 @@
         res1 = f(100)
         res2 = self.meta_interp(f, [100], listops=True)
         assert res1 == res2
-        self.check_loops(int_mod=1) # the hash was traced
+        self.check_loops(int_mod=1) # the hash was traced and eq, but cached
 
     def test_dict_setdefault(self):
         myjitdriver = JitDriver(greens = [], reds = ['total', 'dct'])
@@ -128,7 +128,7 @@
         assert f(100) == 50
         res = self.meta_interp(f, [100], listops=True)
         assert res == 50
-        self.check_loops(int_mod=1)
+        self.check_loops(int_mod=1) # key + eq, but cached
 
     def test_repeated_lookup(self):
         myjitdriver = JitDriver(greens = [], reds = ['n', 'd'])
@@ -153,10 +153,12 @@
 
         res = self.meta_interp(f, [100], listops=True)
         assert res == f(50)
-        self.check_loops({"call": 7, "guard_false": 1, "guard_no_exception": 6,
+        self.check_loops({"call": 5, "getfield_gc": 1, "getinteriorfield_gc": 1,
+                          "guard_false": 1, "guard_no_exception": 4,
                           "guard_true": 1, "int_and": 1, "int_gt": 1,
                           "int_is_true": 1, "int_sub": 1, "jump": 1,
-                          "new_with_vtable": 1, "setfield_gc": 1})
+                          "new_with_vtable": 1, "new": 1, "new_array": 1,
+                          "setfield_gc": 3, })
 
 
 class TestOOtype(DictTests, OOJitMixin):
diff --git a/pypy/jit/metainterp/test/test_float.py b/pypy/jit/metainterp/test/test_float.py
--- a/pypy/jit/metainterp/test/test_float.py
+++ b/pypy/jit/metainterp/test/test_float.py
@@ -1,5 +1,6 @@
-import math
+import math, sys
 from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin
+from pypy.rlib.rarithmetic import intmask, r_uint
 
 
 class FloatTests:
@@ -45,6 +46,34 @@
         res = self.interp_operations(f, [-2.0])
         assert res == -8.5
 
+    def test_cast_float_to_int(self):
+        def g(f):
+            return int(f)
+        res = self.interp_operations(g, [-12345.9])
+        assert res == -12345
+
+    def test_cast_float_to_uint(self):
+        def g(f):
+            return intmask(r_uint(f))
+        res = self.interp_operations(g, [sys.maxint*2.0])
+        assert res == intmask(long(sys.maxint*2.0))
+        res = self.interp_operations(g, [-12345.9])
+        assert res == -12345
+
+    def test_cast_int_to_float(self):
+        def g(i):
+            return float(i)
+        res = self.interp_operations(g, [-12345])
+        assert type(res) is float and res == -12345.0
+
+    def test_cast_uint_to_float(self):
+        def g(i):
+            return float(r_uint(i))
+        res = self.interp_operations(g, [intmask(sys.maxint*2)])
+        assert type(res) is float and res == float(sys.maxint*2)
+        res = self.interp_operations(g, [-12345])
+        assert type(res) is float and res == float(long(r_uint(-12345)))
+
 
 class TestOOtype(FloatTests, OOJitMixin):
     pass
diff --git a/pypy/jit/metainterp/test/test_heapcache.py b/pypy/jit/metainterp/test/test_heapcache.py
--- a/pypy/jit/metainterp/test/test_heapcache.py
+++ b/pypy/jit/metainterp/test/test_heapcache.py
@@ -371,3 +371,17 @@
         assert h.is_unescaped(box1)
         h.invalidate_caches(rop.SETARRAYITEM_GC, None, [box2, index1, box1])
         assert not h.is_unescaped(box1)
+
+        h = HeapCache()
+        h.new_array(box1, lengthbox1)
+        h.new(box2)
+        assert h.is_unescaped(box1)
+        assert h.is_unescaped(box2)
+        h.invalidate_caches(rop.SETARRAYITEM_GC, None, [box1, lengthbox2, box2])
+        assert h.is_unescaped(box1)
+        assert h.is_unescaped(box2)
+        h.invalidate_caches(
+            rop.CALL, FakeCallDescr(FakeEffektinfo.EF_RANDOM_EFFECTS), [box1]
+        )
+        assert not h.is_unescaped(box1)
+        assert not h.is_unescaped(box2)
diff --git a/pypy/jit/metainterp/test/test_memmgr.py b/pypy/jit/metainterp/test/test_memmgr.py
--- a/pypy/jit/metainterp/test/test_memmgr.py
+++ b/pypy/jit/metainterp/test/test_memmgr.py
@@ -18,6 +18,7 @@
 
 class FakeLoopToken:
     generation = 0
+    invalidated = False
 
 
 class _TestMemoryManager:
diff --git a/pypy/jit/metainterp/test/test_quasiimmut.py b/pypy/jit/metainterp/test/test_quasiimmut.py
--- a/pypy/jit/metainterp/test/test_quasiimmut.py
+++ b/pypy/jit/metainterp/test/test_quasiimmut.py
@@ -48,6 +48,13 @@
 
 class QuasiImmutTests(object):
 
+    def setup_method(self, meth):
+        self.prev_compress_limit = QuasiImmut.compress_limit
+        QuasiImmut.compress_limit = 1
+
+    def teardown_method(self, meth):
+        QuasiImmut.compress_limit = self.prev_compress_limit
+
     def test_simple_1(self):
         myjitdriver = JitDriver(greens=['foo'], reds=['x', 'total'])
         class Foo:
@@ -289,7 +296,7 @@
             return total
 
         res = self.meta_interp(main, [])
-        self.check_loop_count(9)
+        self.check_tree_loop_count(6)
         assert res == main()
 
     def test_change_during_running(self):
@@ -317,7 +324,7 @@
         assert f(100, 15) == 3009
         res = self.meta_interp(f, [100, 15])
         assert res == 3009
-        self.check_loops(guard_not_invalidated=2, getfield_gc=0,
+        self.check_loops(guard_not_invalidated=4, getfield_gc=0,
                          call_may_force=0, guard_not_forced=0)
 
     def test_list_simple_1(self):
@@ -453,10 +460,30 @@
         assert f(100, 15) == 3009
         res = self.meta_interp(f, [100, 15])
         assert res == 3009
-        self.check_loops(guard_not_invalidated=2, getfield_gc=0,
+        self.check_loops(guard_not_invalidated=4, getfield_gc=0,
                          getarrayitem_gc=0, getarrayitem_gc_pure=0,
                          call_may_force=0, guard_not_forced=0)
 
+    def test_invalidated_loop_is_not_used_any_more_as_target(self):
+        myjitdriver = JitDriver(greens=['foo'], reds=['x'])
+        class Foo:
+            _immutable_fields_ = ['step?']
+        @dont_look_inside
+        def residual(x, foo):
+            if x == 20:
+                foo.step = 1
+        def f(x):
+            foo = Foo()
+            foo.step = 2
+            while x > 0:
+                myjitdriver.jit_merge_point(foo=foo, x=x)
+                residual(x, foo)
+                x -= foo.step
+            return foo.step
+        res = self.meta_interp(f, [60])
+        assert res == 1
+        self.check_tree_loop_count(4)   # at least not 2 like before
+
 
 class TestLLtypeGreenFieldsTests(QuasiImmutTests, LLJitMixin):
     pass
diff --git a/pypy/jit/metainterp/test/test_resume.py b/pypy/jit/metainterp/test/test_resume.py
--- a/pypy/jit/metainterp/test/test_resume.py
+++ b/pypy/jit/metainterp/test/test_resume.py
@@ -1,3 +1,4 @@
+from __future__ import with_statement
 import py
 from pypy.rpython.lltypesystem import lltype, llmemory, rffi
 from pypy.jit.metainterp.optimizeopt.optimizer import OptValue
diff --git a/pypy/jit/metainterp/test/test_tracingopts.py b/pypy/jit/metainterp/test/test_tracingopts.py
--- a/pypy/jit/metainterp/test/test_tracingopts.py
+++ b/pypy/jit/metainterp/test/test_tracingopts.py
@@ -3,6 +3,7 @@
 from pypy.jit.metainterp.test.support import LLJitMixin
 from pypy.rlib import jit
 from pypy.rlib.rarithmetic import ovfcheck
+from pypy.rlib.rstring import StringBuilder
 
 import py
 
@@ -590,4 +591,14 @@
         assert res == 4
         self.check_operations_history(int_add_ovf=0)
         res = self.interp_operations(fn, [sys.maxint])
-        assert res == 12
\ No newline at end of file
+        assert res == 12
+
+    def test_copy_str_content(self):
+        def fn(n):
+            a = StringBuilder()
+            x = [1]
+            a.append("hello world")
+            return x[0]
+        res = self.interp_operations(fn, [0])
+        assert res == 1
+        self.check_operations_history(getarrayitem_gc=0, getarrayitem_gc_pure=0 )
\ No newline at end of file
diff --git a/pypy/jit/metainterp/test/test_virtualref.py b/pypy/jit/metainterp/test/test_virtualref.py
--- a/pypy/jit/metainterp/test/test_virtualref.py
+++ b/pypy/jit/metainterp/test/test_virtualref.py
@@ -3,6 +3,7 @@
 from pypy.rpython.llinterp import LLException
 from pypy.rlib.jit import JitDriver, dont_look_inside, vref_None
 from pypy.rlib.jit import virtual_ref, virtual_ref_finish, InvalidVirtualRef
+from pypy.rlib.jit import non_virtual_ref
 from pypy.rlib.objectmodel import compute_unique_id
 from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin, _get_jitcodes
 from pypy.jit.metainterp.resoperation import rop
@@ -595,6 +596,65 @@
         res = self.meta_interp(fn, [10])
         assert res == 6
 
+    def test_is_virtual(self):
+        myjitdriver = JitDriver(greens=[], reds=['n', 'res1'])
+        class X:
+            pass
+        @dont_look_inside
+        def residual(vref):
+            return vref.virtual
+        #
+        def f(n):
+            res1 = -42
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, res1=res1)
+                x = X()
+                vref = virtual_ref(x)
+                res1 = residual(vref)
+                virtual_ref_finish(vref, x)
+                n -= 1
+            return res1
+        #
+        res = self.meta_interp(f, [10])
+        assert res == 1
+
+    def test_is_not_virtual_none(self):
+        myjitdriver = JitDriver(greens=[], reds=['n', 'res1'])
+        @dont_look_inside
+        def residual(vref):
+            return vref.virtual
+        #
+        def f(n):
+            res1 = -42
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, res1=res1)
+                res1 = residual(vref_None)
+                n -= 1
+            return res1
+        #
+        res = self.meta_interp(f, [10])
+        assert res == 0
+
+    def test_is_not_virtual_non_none(self):
+        myjitdriver = JitDriver(greens=[], reds=['n', 'res1'])
+        class X:
+            pass
+        @dont_look_inside
+        def residual(vref):
+            return vref.virtual
+        #
+        def f(n):
+            res1 = -42
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, res1=res1)
+                x = X()
+                res1 = residual(non_virtual_ref(x))
+                n -= 1
+            return res1
+        #
+        res = self.meta_interp(f, [10])
+        assert res == 0
+
 
 class TestLLtype(VRefTests, LLJitMixin):
     pass
diff --git a/pypy/jit/metainterp/virtualref.py b/pypy/jit/metainterp/virtualref.py
--- a/pypy/jit/metainterp/virtualref.py
+++ b/pypy/jit/metainterp/virtualref.py
@@ -39,6 +39,7 @@
     def replace_force_virtual_with_call(self, graphs):
         # similar to rvirtualizable2.replace_force_virtualizable_with_call().
         c_force_virtual_ptr = None
+        c_is_virtual_ptr = None
         force_virtual_count = 0
         for graph in graphs:
             for block in graph.iterblocks():
@@ -52,6 +53,13 @@
                         op.opname = 'direct_call'
                         op.args = [c_force_virtual_ptr, op.args[0]]
                         force_virtual_count += 1
+                    #
+                    if op.opname == 'jit_is_virtual':
+                        if c_is_virtual_ptr is None:
+                            c_is_virtual_ptr = self.get_is_virtual_fnptr()
+                        #
+                        op.opname = 'direct_call'
+                        op.args = [c_is_virtual_ptr, op.args[0]]
         #
         if c_force_virtual_ptr is not None:
             log("replaced %d 'jit_force_virtual' with %r" % (force_virtual_count,
@@ -129,6 +137,17 @@
             force_virtual_if_necessary)
         return inputconst(lltype.typeOf(funcptr), funcptr)
 
+    def get_is_virtual_fnptr(self):
+        #
+        def is_virtual(inst):
+            if not inst:
+                return False
+            return inst.typeptr == self.jit_virtual_ref_vtable
+        #
+        FUNC = lltype.FuncType([rclass.OBJECTPTR], lltype.Bool)
+        funcptr = self.warmrunnerdesc.helper_func(lltype.Ptr(FUNC), is_virtual)
+        return inputconst(lltype.typeOf(funcptr), funcptr)
+
     def force_virtual(self, inst):
         vref = lltype.cast_pointer(lltype.Ptr(self.JIT_VIRTUAL_REF), inst)
         token = vref.virtual_token
diff --git a/pypy/jit/metainterp/warmspot.py b/pypy/jit/metainterp/warmspot.py
--- a/pypy/jit/metainterp/warmspot.py
+++ b/pypy/jit/metainterp/warmspot.py
@@ -206,7 +206,7 @@
         self.make_enter_functions()
         self.rewrite_jit_merge_points(policy)
 
-        verbose = not self.cpu.translate_support_code
+        verbose = False # not self.cpu.translate_support_code
         self.codewriter.make_jitcodes(verbose=verbose)
         self.rewrite_can_enter_jits()
         self.rewrite_set_param()
diff --git a/pypy/jit/metainterp/warmstate.py b/pypy/jit/metainterp/warmstate.py
--- a/pypy/jit/metainterp/warmstate.py
+++ b/pypy/jit/metainterp/warmstate.py
@@ -178,7 +178,7 @@
         if self.compiled_merge_points_wref is not None:
             for wref in self.compiled_merge_points_wref:
                 looptoken = wref()
-                if looptoken is not None:
+                if looptoken is not None and not looptoken.invalidated:
                     result.append(looptoken)
         return result
 
diff --git a/pypy/module/__builtin__/__init__.py b/pypy/module/__builtin__/__init__.py
--- a/pypy/module/__builtin__/__init__.py
+++ b/pypy/module/__builtin__/__init__.py
@@ -23,6 +23,7 @@
         'map'           : 'app_functional.map',
         'reduce'        : 'app_functional.reduce',
         'filter'        : 'app_functional.filter',
+        'zip'           : 'app_functional.zip',
         'vars'          : 'app_inspect.vars',
         'dir'           : 'app_inspect.dir',
 
@@ -89,7 +90,6 @@
         'enumerate'     : 'functional.W_Enumerate',
         'min'           : 'functional.min',
         'max'           : 'functional.max',
-        'zip'           : 'functional.zip',
         'reversed'      : 'functional.reversed',
         'super'         : 'descriptor.W_Super',
         'staticmethod'  : 'descriptor.StaticMethod',
@@ -119,7 +119,7 @@
            builtin = space.interpclass_w(w_builtin)
            if isinstance(builtin, module.Module):
                return builtin
-       # no builtin! make a default one.  Given them None, at least.
+       # no builtin! make a default one.  Give them None, at least.
        builtin = module.Module(space, None)
        space.setitem(builtin.w_dict, space.wrap('None'), space.w_None)
        return builtin
diff --git a/pypy/module/__builtin__/app_functional.py b/pypy/module/__builtin__/app_functional.py
--- a/pypy/module/__builtin__/app_functional.py
+++ b/pypy/module/__builtin__/app_functional.py
@@ -162,4 +162,21 @@
         item = seq[i]
         if func(item):
             result.append(item)
-    return tuple(result)
\ No newline at end of file
+    return tuple(result)
+
+def zip(*sequences):
+    """zip(seq1 [, seq2 [...]]) -> [(seq1[0], seq2[0] ...), (...)]
+
+Return a list of tuples, where each tuple contains the i-th element
+from each of the argument sequences.  The returned list is truncated
+in length to the length of the shortest argument sequence."""
+    if not sequences:
+        return []
+    result = []
+    iterators = [iter(seq) for seq in sequences]
+    while True:
+        try:
+            items = [next(it) for it in iterators]
+        except StopIteration:
+            return result
+        result.append(tuple(items))
diff --git a/pypy/module/__builtin__/app_io.py b/pypy/module/__builtin__/app_io.py
--- a/pypy/module/__builtin__/app_io.py
+++ b/pypy/module/__builtin__/app_io.py
@@ -27,7 +27,20 @@
     co = compile(source.rstrip()+"\n", filename, 'exec')
     exec co in glob, loc
 
-def raw_input(prompt=None):
+def _write_prompt(stdout, prompt):
+    print >> stdout, prompt,
+    try:
+        flush = stdout.flush
+    except AttributeError:
+        pass
+    else:
+        flush()
+    try:
+        stdout.softspace = 0
+    except (AttributeError, TypeError):
+        pass
+
+def raw_input(prompt=''):
     """raw_input([prompt]) -> string
 
 Read a string from standard input.  The trailing newline is stripped.
@@ -47,18 +60,10 @@
     if (hasattr(sys, '__raw_input__') and
         isinstance(stdin, file)  and stdin.fileno() == 0 and stdin.isatty() and
         isinstance(stdout, file) and stdout.fileno() == 1):
-        if prompt is None:
-            prompt = ''
-        return sys.__raw_input__(prompt)
+        _write_prompt(stdout, '')
+        return sys.__raw_input__(str(prompt))
 
-    if prompt is not None:
-        stdout.write(prompt)
-        try:
-            flush = stdout.flush
-        except AttributeError:
-            pass
-        else:
-            flush()
+    _write_prompt(stdout, prompt)
     line = stdin.readline()
     if not line:    # inputting an empty line gives line == '\n'
         raise EOFError
@@ -66,7 +71,7 @@
         return line[:-1]
     return line
 
-def input(prompt=None):
+def input(prompt=''):
     """Equivalent to eval(raw_input(prompt))."""
     return eval(raw_input(prompt))
 
diff --git a/pypy/module/__builtin__/compiling.py b/pypy/module/__builtin__/compiling.py
--- a/pypy/module/__builtin__/compiling.py
+++ b/pypy/module/__builtin__/compiling.py
@@ -84,8 +84,8 @@
         raise OperationError(space.w_TypeError,
               w('eval() arg 1 must be a string or code object'))
 
-    caller = space.getexecutioncontext().gettopframe_nohidden()
     if space.is_w(w_globals, space.w_None):
+        caller = space.getexecutioncontext().gettopframe_nohidden()
         if caller is None:
             w_globals = space.newdict()
             if space.is_w(w_locals, space.w_None):
@@ -97,13 +97,9 @@
     elif space.is_w(w_locals, space.w_None):
         w_locals = w_globals
 
-    try:
-        space.getitem(w_globals, space.wrap('__builtins__'))
-    except OperationError, e:
-        if not e.match(space, space.w_KeyError):
-            raise
-        if caller is not None:
-            w_builtin = space.builtin.pick_builtin(caller.w_globals)
-            space.setitem(w_globals, space.wrap('__builtins__'), w_builtin)
+    # xxx removed: adding '__builtins__' to the w_globals dict, if there
+    # is none.  This logic was removed as costly (it requires to get at
+    # the gettopframe_nohidden()).  I bet no test fails, and it's a really
+    # obscure case.
 
     return codeobj.exec_code(space, w_globals, w_locals)
diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py
--- a/pypy/module/__builtin__/functional.py
+++ b/pypy/module/__builtin__/functional.py
@@ -201,27 +201,6 @@
     """
     return min_max(space, __args__, "min")
 
- at unwrap_spec(sequences_w="args_w")
-def zip(space, sequences_w):
-    """Return a list of tuples, where the nth tuple contains every nth item of
-    each collection.
-
-    If the collections have different lengths, zip returns a list as long as the
-    shortest collection, ignoring the trailing items in the other collections.
-    """
-    if not sequences_w:
-        return space.newlist([])
-    result_w = []
-    iterators_w = [space.iter(w_seq) for w_seq in sequences_w]
-    while True:
-        try:
-            items_w = [space.next(w_it) for w_it in iterators_w]
-        except OperationError, e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            return space.newlist(result_w)
-        result_w.append(space.newtuple(items_w))
-
 class W_Enumerate(Wrappable):
 
     def __init__(self, w_iter, w_start):
diff --git a/pypy/module/__builtin__/test/test_rawinput.py b/pypy/module/__builtin__/test/test_rawinput.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/__builtin__/test/test_rawinput.py
@@ -0,0 +1,80 @@
+import autopath
+
+
+class AppTestRawInput():
+
+    def test_input_and_raw_input(self):
+        import sys, StringIO
+        for prompt, expected in [("def:", "abc/ def:/ghi\n"),
+                                 ("", "abc/ /ghi\n"),
+                                 (42, "abc/ 42/ghi\n"),
+                                 (None, "abc/ None/ghi\n"),
+                                 (Ellipsis, "abc/ /ghi\n")]:
+            for inputfn, inputtext, gottext in [
+                    (raw_input, "foo\nbar\n", "foo"),
+                    (input, "40+2\n", 42)]:
+                save = sys.stdin, sys.stdout
+                try:
+                    sys.stdin = StringIO.StringIO(inputtext)
+                    out = sys.stdout = StringIO.StringIO()
+                    print "abc",    # softspace = 1
+                    out.write('/')
+                    if prompt is Ellipsis:
+                        got = inputfn()
+                    else:
+                        got = inputfn(prompt)
+                    out.write('/')
+                    print "ghi"
+                finally:
+                    sys.stdin, sys.stdout = save
+                assert out.getvalue() == expected
+                assert got == gottext
+
+    def test_softspace(self):
+        import sys
+        import StringIO
+        fin = StringIO.StringIO()
+        fout = StringIO.StringIO()
+
+        fin.write("Coconuts\n")
+        fin.seek(0)
+
+        sys_stdin_orig = sys.stdin
+        sys_stdout_orig = sys.stdout
+
+        sys.stdin = fin
+        sys.stdout = fout
+
+        print "test",
+        raw_input("test")
+
+        sys.stdin = sys_stdin_orig
+        sys.stdout = sys_stdout_orig
+
+        fout.seek(0)
+        assert fout.read() == "test test"
+
+    def test_softspace_carryover(self):
+        import sys
+        import StringIO
+        fin = StringIO.StringIO()
+        fout = StringIO.StringIO()
+
+        fin.write("Coconuts\n")
+        fin.seek(0)
+
+        sys_stdin_orig = sys.stdin
+        sys_stdout_orig = sys.stdout
+
+        sys.stdin = fin
+        sys.stdout = fout
+
+        print "test",
+        raw_input("test")
+        print "test",
+
+        sys.stdin = sys_stdin_orig
+        sys.stdout = sys_stdout_orig
+
+        fout.seek(0)
+        assert fout.read() == "test testtest"
diff --git a/pypy/module/_continuation/interp_continuation.py b/pypy/module/_continuation/interp_continuation.py
--- a/pypy/module/_continuation/interp_continuation.py
+++ b/pypy/module/_continuation/interp_continuation.py
@@ -19,6 +19,11 @@
         #  - normal:      self.sthread != None, not is_empty_handle(self.h)
         #  - finished:    self.sthread != None, is_empty_handle(self.h)
 
+    def __del__(self):
+        sthread = self.sthread
+        if sthread is not None and not sthread.is_empty_handle(self.h):
+            sthread.destroy(self.h)
+
     def check_sthread(self):
         ec = self.space.getexecutioncontext()
         if ec.stacklet_thread is not self.sthread:
@@ -28,6 +33,8 @@
     def descr_init(self, w_callable, __args__):
         if self.sthread is not None:
             raise geterror(self.space, "continulet already __init__ialized")
+        sthread = build_sthread(self.space)
+        workaround_disable_jit(sthread)
         #
         # hackish: build the frame "by hand", passing it the correct arguments
         space = self.space
@@ -41,7 +48,6 @@
         self.bottomframe = bottomframe
         #
         global_state.origin = self
-        sthread = build_sthread(self.space)
         self.sthread = sthread
         h = sthread.new(new_stacklet_callback)
         post_switch(sthread, h)
@@ -71,6 +77,7 @@
                 global_state.clear()
                 raise geterror(self.space, "continulet already finished")
         self.check_sthread()
+        workaround_disable_jit(self.sthread)
         #
         global_state.origin = self
         if to is None:
@@ -259,6 +266,16 @@
         sthread = ec.stacklet_thread = SThread(space, ec)
     return sthread
 
+def workaround_disable_jit(sthread):
+    # A bad workaround to kill the JIT anywhere in this thread.
+    # This forces all the frames.  It's a bad workaround because
+    # it takes O(depth) time, and it will cause some "abort:
+    # vable escape" in the JIT.  The goal is to prevent any frame
+    # from being still virtuals, because the JIT generates code
+    # to un-virtualizable them "on demand" by loading values based
+    # on FORCE_TOKEN, which is an address in the stack.
+    sthread.ec.force_all_frames()
+
 # ____________________________________________________________
 
 def permute(space, args_w):
diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -19,7 +19,7 @@
 class W_RSocket(Wrappable, RSocket):
     def __del__(self):
         self.clear_all_weakrefs()
-        self.close()
+        RSocket.__del__(self)
 
     def accept_w(self, space):
         """accept() -> (socket object, address info)
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -211,7 +211,9 @@
             return result
 
         def __del__(self):
-            self.clear_all_weakrefs()
+            # note that we don't call clear_all_weakrefs here because
+            # an array with freed buffer is ok to see - it's just empty with 0
+            # length
             self.setlen(0)
 
         def setlen(self, size):
diff --git a/pypy/module/array/test/test_array.py b/pypy/module/array/test/test_array.py
--- a/pypy/module/array/test/test_array.py
+++ b/pypy/module/array/test/test_array.py
@@ -824,6 +824,22 @@
         r = weakref.ref(a)
         assert r() is a
 
+    def test_subclass_del(self):
+        import array, gc, weakref
+        l = []
+        
+        class A(array.array):
+            pass
+
+        a = A('d')
+        a.append(3.0)
+        r = weakref.ref(a, lambda a: l.append(a()))
+        del a
+        gc.collect()
+        assert l
+        assert l[0] is None or len(l[0]) == 0
+
+
 class TestCPythonsOwnArray(BaseArrayTests):
 
     def setup_class(cls):
@@ -844,11 +860,7 @@
         cls.w_tempfile = cls.space.wrap(
             str(py.test.ensuretemp('array').join('tmpfile')))
         cls.w_maxint = cls.space.wrap(sys.maxint)
-
-
-
-
-
+    
     def test_buffer_info(self):
         a = self.array('c', 'Hi!')
         bi = a.buffer_info()
diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py
--- a/pypy/module/micronumpy/interp_dtype.py
+++ b/pypy/module/micronumpy/interp_dtype.py
@@ -116,6 +116,12 @@
         def setitem_w(self, space, storage, i, w_item):
             self.setitem(storage, i, self.unwrap(space, w_item))
 
+        def fill(self, storage, item, start, stop):
+            storage = self.unerase(storage)
+            item = self.unbox(item)
+            for i in xrange(start, stop):
+                storage[i] = item
+
         if "adapt_val" not in exclude_methods:
             @specialize.argtype(1)
             def adapt_val(self, val):
@@ -182,9 +188,6 @@
     @binop
     def mul(self, v1, v2):
         return v1 * v2
-    @binop
-    def div(self, v1, v2):
-        return v1 / v2
 
     @unaryop
     def pos(self, v):
@@ -238,6 +241,14 @@
         return float2string(self.for_computation(self.unbox(item)), 'g', rfloat.DTSF_STR_PRECISION)
 
     @binop
+    def div(self, v1, v2):
+        try:
+            return v1 / v2
+        except ZeroDivisionError:
+            if v1 == v2 == 0.0:
+                return rfloat.NAN
+            return rfloat.copysign(rfloat.INFINITY, v1 * v2)
+    @binop
     def mod(self, v1, v2):
         return math.fmod(v1, v2)
     @binop
@@ -316,6 +327,11 @@
         return str(widen(self.unbox(item)))
 
     @binop
+    def div(self, v1, v2):
+        if v2 == 0:
+            return 0
+        return v1 / v2
+    @binop
     def mod(self, v1, v2):
         return v1 % v2
 
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -563,8 +563,7 @@
 
     arr = SingleDimArray(size, dtype=dtype)
     one = dtype.adapt_val(1)
-    for i in xrange(size):
-        arr.dtype.setitem(arr.storage, i, one)
+    arr.dtype.fill(arr.storage, one, 0, size)
     return space.wrap(arr)
 
 BaseArray.typedef = TypeDef(
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -285,7 +285,9 @@
             assert b[i] == i * 5
 
     def test_div(self):
-        from numpy import array, dtype
+        from math import isnan
+        from numpy import array, dtype, inf
+
         a = array(range(1, 6))
         b = a / a
         for i in range(5):
@@ -297,6 +299,24 @@
         for i in range(5):
             assert b[i] == 1
 
+        a = array([-1, 0, 1])
+        b = array([0, 0, 0])
+        c = a / b
+        assert (c == [0, 0, 0]).all()
+
+        a = array([-1.0, 0.0, 1.0])
+        b = array([0.0, 0.0, 0.0])
+        c = a / b
+        assert c[0] == -inf
+        assert isnan(c[1])
+        assert c[2] == inf
+
+        b = array([-0.0, -0.0, -0.0])
+        c = a / b
+        assert c[0] == inf
+        assert isnan(c[1])
+        assert c[2] == -inf
+
     def test_div_other(self):
         from numpy import array
         a = array(range(5))
diff --git a/pypy/module/pyexpat/__init__.py b/pypy/module/pyexpat/__init__.py
--- a/pypy/module/pyexpat/__init__.py
+++ b/pypy/module/pyexpat/__init__.py
@@ -4,12 +4,8 @@
 
 class ErrorsModule(MixedModule):
     "Definition of pyexpat.errors module."
-
-    appleveldefs = {
-        }
-
-    interpleveldefs = {
-        }
+    appleveldefs = {}
+    interpleveldefs = {}
 
     def setup_after_space_initialization(self):
         from pypy.module.pyexpat import interp_pyexpat
@@ -18,6 +14,18 @@
                     interp_pyexpat.ErrorString(self.space,
                     getattr(interp_pyexpat, name)))
 
+class ModelModule(MixedModule):
+    "Definition of pyexpat.model module."
+    appleveldefs = {}
+    interpleveldefs = {}
+
+    def setup_after_space_initialization(self):
+        from pypy.module.pyexpat import interp_pyexpat
+        space = self.space
+        for name in interp_pyexpat.xml_model_list:
+            value = getattr(interp_pyexpat, name)
+            space.setattr(self, space.wrap(name), space.wrap(value))
+
 class Module(MixedModule):
     "Python wrapper for Expat parser."
 
@@ -39,6 +47,7 @@
 
     submodules = {
         'errors': ErrorsModule,
+        'model':  ModelModule,
     }
 
     for name in ['XML_PARAM_ENTITY_PARSING_NEVER',
diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py
--- a/pypy/module/pyexpat/interp_pyexpat.py
+++ b/pypy/module/pyexpat/interp_pyexpat.py
@@ -76,6 +76,18 @@
     "XML_ERROR_FINISHED",
     "XML_ERROR_SUSPEND_PE",
     ]
+xml_model_list = [
+    "XML_CTYPE_EMPTY",
+    "XML_CTYPE_ANY",
+    "XML_CTYPE_MIXED",
+    "XML_CTYPE_NAME",
+    "XML_CTYPE_CHOICE",
+    "XML_CTYPE_SEQ",
+    "XML_CQUANT_NONE",
+    "XML_CQUANT_OPT",
+    "XML_CQUANT_REP",
+    "XML_CQUANT_PLUS",
+    ]
 
 class CConfigure:
     _compilation_info_ = eci
@@ -104,6 +116,8 @@
 
     for name in xml_error_list:
         locals()[name] = rffi_platform.ConstantInteger(name)
+    for name in xml_model_list:
+        locals()[name] = rffi_platform.ConstantInteger(name)
 
 for k, v in rffi_platform.configure(CConfigure).items():
     globals()[k] = v
diff --git a/pypy/module/pyexpat/test/test_parser.py b/pypy/module/pyexpat/test/test_parser.py
--- a/pypy/module/pyexpat/test/test_parser.py
+++ b/pypy/module/pyexpat/test/test_parser.py
@@ -131,3 +131,7 @@
                'encoding specified in XML declaration is incorrect')
         assert (pyexpat.errors.XML_ERROR_XML_DECL ==
                 'XML declaration not well-formed')
+
+    def test_model(self):
+        import pyexpat
+        assert isinstance(pyexpat.model.XML_CTYPE_EMPTY, int)
diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py
--- a/pypy/module/pypyjit/interp_jit.py
+++ b/pypy/module/pypyjit/interp_jit.py
@@ -137,20 +137,15 @@
 
     def jump_absolute(self, jumpto, _, ec=None):
         if we_are_jitted():
-            # Normally, the tick counter is decremented by 100 for every
-            # Python opcode.  Here, to better support JIT compilation of
-            # small loops, we decrement it by a possibly smaller constant.
-            # We get the maximum 100 when the (unoptimized) trace length
-            # is at least 3200 (a bit randomly).
-            trace_length = r_uint(current_trace_length())
-            decr_by = trace_length // 32
-            if decr_by < 1:
-                decr_by = 1
-            elif decr_by > 100:    # also if current_trace_length() returned -1
-                decr_by = 100
+            #
+            # assume that only threads are using the bytecode counter
+            decr_by = 0
+            if self.space.actionflag.has_bytecode_counter:   # constant-folded
+                if self.space.threadlocals.gil_ready:   # quasi-immutable field
+                    decr_by = _get_adapted_tick_counter()
             #
             self.last_instr = intmask(jumpto)
-            ec.bytecode_trace(self, intmask(decr_by))
+            ec.bytecode_trace(self, decr_by)
             jumpto = r_uint(self.last_instr)
         #
         pypyjitdriver.can_enter_jit(frame=self, ec=ec, next_instr=jumpto,
@@ -158,6 +153,20 @@
                                     is_being_profiled=self.is_being_profiled)
         return jumpto
 
+def _get_adapted_tick_counter():
+    # Normally, the tick counter is decremented by 100 for every
+    # Python opcode.  Here, to better support JIT compilation of
+    # small loops, we decrement it by a possibly smaller constant.
+    # We get the maximum 100 when the (unoptimized) trace length
+    # is at least 3200 (a bit randomly).
+    trace_length = r_uint(current_trace_length())
+    decr_by = trace_length // 32
+    if decr_by < 1:
+        decr_by = 1
+    elif decr_by > 100:    # also if current_trace_length() returned -1
+        decr_by = 100
+    return intmask(decr_by)
+
 
 PyCode__initialize = PyCode._initialize
 
diff --git a/pypy/module/pypyjit/policy.py b/pypy/module/pypyjit/policy.py
--- a/pypy/module/pypyjit/policy.py
+++ b/pypy/module/pypyjit/policy.py
@@ -16,7 +16,8 @@
         if modname in ['pypyjit', 'signal', 'micronumpy', 'math', 'exceptions',
                        'imp', 'sys', 'array', '_ffi', 'itertools', 'operator',
                        'posix', '_socket', '_sre', '_lsprof', '_weakref',
-                       '__pypy__', 'cStringIO', '_collections', 'struct']:
+                       '__pypy__', 'cStringIO', '_collections', 'struct',
+                       'mmap']:
             return True
         return False
 
diff --git a/pypy/module/pypyjit/test_pypy_c/model.py b/pypy/module/pypyjit/test_pypy_c/model.py
--- a/pypy/module/pypyjit/test_pypy_c/model.py
+++ b/pypy/module/pypyjit/test_pypy_c/model.py
@@ -225,6 +225,8 @@
         # strip comment
         if '#' in line:
             line = line[:line.index('#')]
+        if line.strip() == 'guard_not_invalidated?':
+            return 'guard_not_invalidated', None, [], '...', False
         # find the resvar, if any
         if ' = ' in line:
             resvar, _, line = line.partition(' = ')
@@ -249,7 +251,7 @@
             descr = descr[len('descr='):]
         else:
             descr = None
-        return opname, resvar, args, descr
+        return opname, resvar, args, descr, True
 
     @classmethod
     def preprocess_expected_src(cls, src):
@@ -258,13 +260,23 @@
         # replaced with the corresponding operations, so that tests don't have
         # to repeat it every time
         ticker_check = """
+            guard_not_invalidated?
+            ticker0 = getfield_raw(ticker_address, descr=<SignedFieldDescr pypysig_long_struct.c_value .*>)
+            ticker_cond0 = int_lt(ticker0, 0)
+            guard_false(ticker_cond0, descr=...)
+        """
+        src = src.replace('--TICK--', ticker_check)
+        #
+        # this is the ticker check generated if we have threads
+        thread_ticker_check = """
+            guard_not_invalidated?
             ticker0 = getfield_raw(ticker_address, descr=<SignedFieldDescr pypysig_long_struct.c_value .*>)
             ticker1 = int_sub(ticker0, 1)
             setfield_raw(ticker_address, ticker1, descr=<SignedFieldDescr pypysig_long_struct.c_value .*>)
             ticker_cond0 = int_lt(ticker1, 0)
             guard_false(ticker_cond0, descr=...)
         """
-        src = src.replace('--TICK--', ticker_check)
+        src = src.replace('--THREAD-TICK--', thread_ticker_check)
         #
         # this is the ticker check generated in PyFrame.handle_operation_error
         exc_ticker_check = """
@@ -298,7 +310,7 @@
         if not cond:
             raise InvalidMatch(message, frame=sys._getframe(1))
 
-    def match_op(self, op, (exp_opname, exp_res, exp_args, exp_descr)):
+    def match_op(self, op, (exp_opname, exp_res, exp_args, exp_descr, _)):
         self._assert(op.name == exp_opname, "operation mismatch")
         self.match_var(op.res, exp_res)
         if exp_args != ['...']:
@@ -341,7 +353,7 @@
         what is after the '...'
         """
         iter_exp_ops = iter(expected_ops)
-        iter_ops = iter(self.ops)
+        iter_ops = RevertableIterator(self.ops)
         for opindex, exp_op in enumerate(iter_exp_ops):
             try:
                 if exp_op == '...':
@@ -360,6 +372,9 @@
                             break
                 self.match_op(op, exp_op)
             except InvalidMatch, e:
+                if exp_op[4] is False:    # optional operation
+                    iter_ops.revert_one()
+                    continue       # try to match with the next exp_op
                 e.opindex = opindex
                 raise
         #
@@ -372,8 +387,8 @@
                 return ''
             text = str(py.code.Source(src).deindent().indent())
             lines = text.splitlines(True)
-            if opindex is not None and 0 <= opindex < len(lines):
-                lines[opindex] = lines[opindex].rstrip() + '\t<=====\n'
+            if opindex is not None and 0 <= opindex <= len(lines):
+                lines.insert(opindex, '\n\t===== HERE =====\n')
             return ''.join(lines)
         #
         expected_src = self.preprocess_expected_src(expected_src)
@@ -398,3 +413,18 @@
         else:
             return True
 
+
+class RevertableIterator(object):
+    def __init__(self, sequence):
+        self.sequence = sequence
+        self.index = 0
+    def __iter__(self):
+        return self
+    def next(self):
+        index = self.index
+        if index == len(self.sequence):
+            raise StopIteration
+        self.index = index + 1
+        return self.sequence[index]
+    def revert_one(self):
+        self.index -= 1
diff --git a/pypy/module/pypyjit/test_pypy_c/test_00_model.py b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
--- a/pypy/module/pypyjit/test_pypy_c/test_00_model.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_00_model.py
@@ -145,15 +145,17 @@
 
     def test_parse_op(self):
         res = OpMatcher.parse_op("  a =   int_add(  b,  3 ) # foo")
-        assert res == ("int_add", "a", ["b", "3"], None)
+        assert res == ("int_add", "a", ["b", "3"], None, True)
         res = OpMatcher.parse_op("guard_true(a)")
-        assert res == ("guard_true", None, ["a"], None)
+        assert res == ("guard_true", None, ["a"], None, True)
         res = OpMatcher.parse_op("setfield_gc(p0, i0, descr=<foobar>)")
-        assert res == ("setfield_gc", None, ["p0", "i0"], "<foobar>")
+        assert res == ("setfield_gc", None, ["p0", "i0"], "<foobar>", True)
         res = OpMatcher.parse_op("i1 = getfield_gc(p0, descr=<foobar>)")
-        assert res == ("getfield_gc", "i1", ["p0"], "<foobar>")
+        assert res == ("getfield_gc", "i1", ["p0"], "<foobar>", True)
         res = OpMatcher.parse_op("p0 = force_token()")
-        assert res == ("force_token", "p0", [], None)
+        assert res == ("force_token", "p0", [], None, True)
+        res = OpMatcher.parse_op("guard_not_invalidated?")
+        assert res == ("guard_not_invalidated", None, [], '...', False)
 
     def test_exact_match(self):
         loop = """
@@ -341,7 +343,7 @@
             # this is the actual loop
             'int_lt', 'guard_true', 'int_add',
             # this is the signal checking stuff
-            'getfield_raw', 'int_sub', 'setfield_raw', 'int_lt', 'guard_false',
+            'guard_not_invalidated', 'getfield_raw', 'int_lt', 'guard_false',
             'jump'
             ]
 
@@ -407,7 +409,7 @@
             # this is the actual loop
             'int_lt', 'guard_true', 'force_token', 'int_add',
             # this is the signal checking stuff
-            'getfield_raw', 'int_sub', 'setfield_raw', 'int_lt', 'guard_false',
+            'guard_not_invalidated', 'getfield_raw', 'int_lt', 'guard_false',
             'jump'
             ]
 
@@ -425,10 +427,9 @@
             guard_true(i6, descr=...)
             i8 = int_add(i4, 1)
             # signal checking stuff
+            guard_not_invalidated(descr=...)
             i10 = getfield_raw(37212896, descr=<.* pypysig_long_struct.c_value .*>)
-            i12 = int_sub(i10, 1)
-            setfield_raw(37212896, i12, descr=<.* pypysig_long_struct.c_value .*>)
-            i14 = int_lt(i12, 0)
+            i14 = int_lt(i10, 0)
             guard_false(i14, descr=...)
             jump(p0, p1, p2, p3, i8, descr=...)
         """)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py
--- a/pypy/module/pypyjit/test_pypy_c/test_call.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_call.py
@@ -465,3 +465,25 @@
             setfield_gc(p4, p22, descr=<GcPtrFieldDescr pypy.interpreter.nestedscope.Cell.inst_w_value .*>)
             jump(p0, p1, p2, p3, p4, p7, p22, p7, descr=<Loop0>)
         """)
+
+    def test_kwargs_virtual(self):
+        def main(n):
+            def g(**kwargs):
+                return kwargs["x"] + 1
+
+            i = 0
+            while i < n:
+                i = g(x=i)
+            return i
+
+        log = self.run(main, [500])
+        assert log.result == 500
+        loop, = log.loops_by_filename(self.filepath)
+        assert loop.match("""
+            i2 = int_lt(i0, i1)
+            guard_true(i2, descr=...)
+            i3 = force_token()
+            i4 = int_add(i0, 1)
+            --TICK--
+            jump(..., descr=...)
+        """)
\ No newline at end of file
diff --git a/pypy/module/pypyjit/test_pypy_c/test_containers.py b/pypy/module/pypyjit/test_pypy_c/test_containers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_containers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_containers.py
@@ -44,9 +44,9 @@
         # gc_id call is hoisted out of the loop, the id of a value obviously
         # can't change ;)
         assert loop.match_by_id("getitem", """
-            i28 = call(ConstClass(ll_dict_lookup__dicttablePtr_objectPtr_Signed), p18, p6, i25, descr=...)
+            i26 = call(ConstClass(ll_dict_lookup), p18, p6, i25, descr=...)
             ...
-            p33 = call(ConstClass(ll_get_value__dicttablePtr_Signed), p18, i28, descr=...)
+            p33 = getinteriorfield_gc(p31, i26, descr=<InteriorFieldDescr <GcPtrFieldDescr dictentry.value .*>>)
             ...
         """)
 
diff --git a/pypy/module/pypyjit/test_pypy_c/test_misc.py b/pypy/module/pypyjit/test_pypy_c/test_misc.py
--- a/pypy/module/pypyjit/test_pypy_c/test_misc.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_misc.py
@@ -329,4 +329,20 @@
             guard_false(i28, descr=...)
             i30 = int_lshift(i20, 24)
             i31 = int_or(i26, i30)
-        """ % {"32_bit_only": extra})
\ No newline at end of file
+        """ % {"32_bit_only": extra})
+
+    def test_eval(self):
+        def main():
+            i = 1
+            a = compile('x+x+x+x+x+x', 'eval', 'eval')
+            b = {'x': 7}
+            while i < 1000:
+                y = eval(a,b,b)  # ID: eval
+                i += 1
+            return y
+
+        log = self.run(main)
+        assert log.result == 42
+        # the following assertion fails if the loop was cancelled due
+        # to "abort: vable escape"
+        assert len(log.loops_by_id("eval")) == 1
diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -41,7 +41,7 @@
             guard_true(i32, descr=...)
             i34 = int_add(i6, 1)
             --TICK--
-            jump(p0, p1, p2, p3, p4, p5, i34, p7, p8, i9, i10, p11, i12, p13, descr=<Loop4>)
+            jump(p0, p1, p2, p3, p4, p5, i34, p7, p8, i9, i10, p11, i12, p13, descr=...)
         """)
 
     def test_long(self):
@@ -93,7 +93,8 @@
             i46 = call(ConstClass(ll_startswith__rpy_stringPtr_rpy_stringPtr), p28, ConstPtr(ptr45), descr=<BoolCallDescr>)
             guard_false(i46, descr=...)
             p51 = new_with_vtable(21136408)
-            setfield_gc(p51, _, descr=...)    # 6 setfields, but the order is dict-order-dependent
+            setfield_gc(p51, _, descr=...)    # 7 setfields, but the order is dict-order-dependent
+            setfield_gc(p51, _, descr=...)
             setfield_gc(p51, _, descr=...)
             setfield_gc(p51, _, descr=...)
             setfield_gc(p51, _, descr=...)
@@ -106,7 +107,7 @@
             i58 = int_add_ovf(i6, i57)
             guard_no_overflow(descr=...)
             --TICK--
-            jump(p0, p1, p2, p3, p4, p5, i58, i7, descr=<Loop4>)
+            jump(p0, p1, p2, p3, p4, p5, i58, i7, descr=...)
         """)
 
     def test_str_mod(self):
diff --git a/pypy/module/pypyjit/test_pypy_c/test_thread.py b/pypy/module/pypyjit/test_pypy_c/test_thread.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/pypyjit/test_pypy_c/test_thread.py
@@ -0,0 +1,28 @@
+from pypy.module.pypyjit.test_pypy_c.test_00_model import BaseTestPyPyC
+
+
+class TestThread(BaseTestPyPyC):
+    def test_simple(self):
+        def main(n):
+            import thread
+            def f():
+                i = 0
+                while i < n:
+                    i += 1
+                done.release()
+
+            done = thread.allocate_lock()
+            done.acquire()
+            thread.start_new_thread(f, ())
+            done.acquire()
+            return 0
+        log = self.run(main, [500])
+        assert round(log.result, 6) == round(main(500), 6)
+        loop, = log.loops_by_filename(self.filepath)
+        assert loop.match("""
+            i2 = int_lt(i0, i1)
+            guard_true(i2, descr=...)
+            i3 = int_add(i0, 1)
+            --THREAD-TICK--
+            jump(..., descr=<Loop0>)
+        """)
diff --git a/pypy/module/rctime/interp_time.py b/pypy/module/rctime/interp_time.py
--- a/pypy/module/rctime/interp_time.py
+++ b/pypy/module/rctime/interp_time.py
@@ -245,6 +245,9 @@
 if sys.platform != 'win32':
     @unwrap_spec(secs=float)
     def sleep(space, secs):
+        if secs < 0:
+            raise OperationError(space.w_IOError,
+                                 space.wrap("Invalid argument: negative time in sleep"))
         pytime.sleep(secs)
 else:
     from pypy.rlib import rwin32
@@ -265,6 +268,9 @@
                                    OSError(EINTR, "sleep() interrupted"))
     @unwrap_spec(secs=float)
     def sleep(space, secs):
+        if secs < 0:
+            raise OperationError(space.w_IOError,
+                                 space.wrap("Invalid argument: negative time in sleep"))
         # as decreed by Guido, only the main thread can be
         # interrupted.
         main_thread = space.fromcache(State).main_thread
diff --git a/pypy/module/rctime/test/test_rctime.py b/pypy/module/rctime/test/test_rctime.py
--- a/pypy/module/rctime/test/test_rctime.py
+++ b/pypy/module/rctime/test/test_rctime.py
@@ -20,8 +20,9 @@
         import sys
         import os
         raises(TypeError, rctime.sleep, "foo")
-        rctime.sleep(1.2345)
-        
+        rctime.sleep(0.12345)
+        raises(IOError, rctime.sleep, -1.0)
+
     def test_clock(self):
         import time as rctime
         rctime.clock()
diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py
--- a/pypy/module/signal/interp_signal.py
+++ b/pypy/module/signal/interp_signal.py
@@ -109,8 +109,11 @@
         p = pypysig_getaddr_occurred()
         value = p.c_value
         if self.has_bytecode_counter:    # this 'if' is constant-folded
-            value -= by
-            p.c_value = value
+            if jit.isconstant(by) and by == 0:
+                pass     # normally constant-folded too
+            else:
+                value -= by
+                p.c_value = value
         return value
 
 
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -47,7 +47,7 @@
         'pypy_initial_path'     : 'state.pypy_initial_path',
 
         '_getframe'             : 'vm._getframe', 
-        '_current_frames'       : 'vm._current_frames', 
+        '_current_frames'       : 'currentframes._current_frames', 
         'setrecursionlimit'     : 'vm.setrecursionlimit', 
         'getrecursionlimit'     : 'vm.getrecursionlimit', 
         'setcheckinterval'      : 'vm.setcheckinterval', 
diff --git a/pypy/module/sys/currentframes.py b/pypy/module/sys/currentframes.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/sys/currentframes.py
@@ -0,0 +1,78 @@
+"""
+Implementation of the 'sys._current_frames()' routine.
+"""
+from pypy.interpreter import gateway
+
+app = gateway.applevel('''
+"NOT_RPYTHON"
+import __builtin__
+
+class fake_code(object):
+    co_name = "?"
+    co_filename = "?"
+    co_firstlineno = 0
+
+class fake_frame(object):
+    f_back = None
+    f_builtins = __builtin__.__dict__
+    f_code = fake_code()
+    f_exc_traceback = None
+    f_exc_type = None
+    f_exc_value = None
+    f_globals = {}
+    f_lasti = -1
+    f_lineno = 0
+    f_locals = {}
+    f_restricted = False
+    f_trace = None
+
+    def __init__(self, f):
+        if f is not None:
+            for name in ["f_builtins", "f_code", "f_globals", "f_lasti",
+                         "f_lineno"]:
+                setattr(self, name, getattr(f, name))
+''')
+
+def _current_frames(space):
+    """_current_frames() -> dictionary
+
+    Return a dictionary mapping each current thread T's thread id to T's
+    current stack "frame".  Functions in the traceback module can build the
+    call stack given such a frame.
+
+    Note that in PyPy this returns fake frame objects, to avoid a runtime
+    penalty everywhere with the JIT.  (So far these fake frames can be
+    completely uninformative depending on the JIT state; we could return
+    more with more efforts.)
+
+    This function should be used for specialized purposes only."""
+    w_result = space.newdict()
+    w_fake_frame = app.wget(space, "fake_frame")
+    w_fake_code  = app.wget(space, "fake_code")
+    ecs = space.threadlocals.getallvalues()
+    for thread_ident, ec in ecs.items():
+        vref = ec.topframeref
+        frames = []
+        while not vref.virtual:
+            f = vref()
+            if f is None:
+                break
+            frames.append(f)
+            vref = f.f_backref
+        else:
+            frames.append(None)
+        #
+        w_topframe = space.wrap(None)
+        w_prevframe = None
+        for f in frames:
+            w_nextframe = space.call_function(w_fake_frame, space.wrap(f))
+            if w_prevframe is None:
+                w_topframe = w_nextframe
+            else:
+                space.setattr(w_prevframe, space.wrap('f_back'), w_nextframe)
+            w_prevframe = w_nextframe
+        #
+        space.setitem(w_result,
+                      space.wrap(thread_ident),
+                      w_topframe)
+    return w_result
diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -556,7 +556,7 @@
             return sys._current_frames()
         frames = f()
         assert frames.keys() == [0]
-        assert frames[0].f_code.co_name == 'f'
+        assert frames[0].f_code.co_name in ('f', '?')
 
 class AppTestCurrentFramesWithThread(AppTestCurrentFrames):
     def setup_class(cls):
@@ -568,23 +568,25 @@
         import thread
 
         thread_id = thread.get_ident()
-        self.ready = False
         def other_thread():
-            self.ready = True
             print "thread started"
-            time.sleep(5)
+            lock2.release()
+            lock1.acquire()
+        lock1 = thread.allocate_lock()
+        lock2 = thread.allocate_lock()
+        lock1.acquire()
+        lock2.acquire()
         thread.start_new_thread(other_thread, ())
 
         def f():
-            for i in range(100):
-                if self.ready: break
-                time.sleep(0.1)
+            lock2.acquire()
             return sys._current_frames()
 
         frames = f()
+        lock1.release()
         thisframe = frames.pop(thread_id)
-        assert thisframe.f_code.co_name == 'f'
+        assert thisframe.f_code.co_name in ('f', '?')
 
         assert len(frames) == 1
         _, other_frame = frames.popitem()
-        assert other_frame.f_code.co_name == 'other_thread'
+        assert other_frame.f_code.co_name in ('other_thread', '?')
diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -45,25 +45,6 @@
     f.mark_as_escaped()
     return space.wrap(f)
 
-def _current_frames(space):
-    """_current_frames() -> dictionary
-
-    Return a dictionary mapping each current thread T's thread id to T's
-    current stack frame.
-
-    This function should be used for specialized purposes only."""
-    raise OperationError(space.w_NotImplementedError,
-        space.wrap("XXX sys._current_frames() incompatible with the JIT"))
-    w_result = space.newdict()
-    ecs = space.threadlocals.getallvalues()
-    for thread_ident, ec in ecs.items():
-        f = ec.gettopframe_nohidden()
-        f.mark_as_escaped()
-        space.setitem(w_result,
-                      space.wrap(thread_ident),
-                      space.wrap(f))
-    return w_result
-
 def setrecursionlimit(space, w_new_limit):
     """setrecursionlimit() sets the maximum number of nested calls that
 can occur before a RuntimeError is raised.  On PyPy the limit is
diff --git a/pypy/module/thread/gil.py b/pypy/module/thread/gil.py
--- a/pypy/module/thread/gil.py
+++ b/pypy/module/thread/gil.py
@@ -17,6 +17,7 @@
 class GILThreadLocals(OSThreadLocals):
     """A version of OSThreadLocals that enforces a GIL."""
     gil_ready = False
+    _immutable_fields_ = ['gil_ready?']
 
     def initialize(self, space):
         # add the GIL-releasing callback as an action on the space
diff --git a/pypy/module/thread/threadlocals.py b/pypy/module/thread/threadlocals.py
--- a/pypy/module/thread/threadlocals.py
+++ b/pypy/module/thread/threadlocals.py
@@ -8,9 +8,14 @@
 
     def __init__(self):
         self._valuedict = {}   # {thread_ident: ExecutionContext()}
+        self._freeze_()
+
+    def _freeze_(self):
+        self._valuedict.clear()
         self._mainthreadident = 0
         self._mostrecentkey = 0        # fast minicaching for the common case
         self._mostrecentvalue = None   # fast minicaching for the common case
+        return False
 
     def getvalue(self):
         ident = thread.get_ident()
diff --git a/pypy/objspace/std/newformat.py b/pypy/objspace/std/newformat.py
--- a/pypy/objspace/std/newformat.py
+++ b/pypy/objspace/std/newformat.py
@@ -120,6 +120,8 @@
             out.append_slice(s, last_literal, end)
             return out.build()
 
+        # This is only ever called if we're already unrolling _do_build_string
+        @jit.unroll_safe
         def _parse_field(self, start, end):
             s = self.template
             # Find ":" or "!"
@@ -149,6 +151,7 @@
                 i += 1
             return s[start:end], None, end
 
+        @jit.unroll_safe
         def _get_argument(self, name):
             # First, find the argument.
             space = self.space
@@ -207,6 +210,7 @@
                     raise OperationError(space.w_IndexError, w_msg)
             return self._resolve_lookups(w_arg, name, i, end)
 
+        @jit.unroll_safe
         def _resolve_lookups(self, w_obj, name, start, end):
             # Resolve attribute and item lookups.
             space = self.space
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -83,11 +83,12 @@
         if self.config.objspace.std.withtproxy:
             transparent.setup(self)
 
+        interplevel_classes = {}
         for type, classes in self.model.typeorder.iteritems():
-            if len(classes) >= 3:
+            if len(classes) >= 3: # XXX what does this 3 mean??!
                 # W_Root, AnyXxx and actual object
-                self.gettypefor(type).interplevel_cls = classes[0][0]
-
+                interplevel_classes[self.gettypefor(type)] = classes[0][0]
+        self._interplevel_classes = interplevel_classes
 
     def get_builtin_types(self):
         return self.builtin_types
@@ -579,7 +580,7 @@
             raise OperationError(self.w_TypeError,
                                  self.wrap("need type object"))
         if is_annotation_constant(w_type):
-            cls = w_type.interplevel_cls
+            cls = self._get_interplevel_cls(w_type)
             if cls is not None:
                 assert w_inst is not None
                 if isinstance(w_inst, cls):
@@ -589,3 +590,9 @@
     @specialize.arg_or_var(2)
     def isinstance_w(space, w_inst, w_type):
         return space._type_isinstance(w_inst, w_type)
+
+    @specialize.memo()
+    def _get_interplevel_cls(self, w_type):
+        if not hasattr(self, "_interplevel_classes"):
+            return None # before running initialize
+        return self._interplevel_classes.get(w_type, None)
diff --git a/pypy/objspace/std/smallintobject.py b/pypy/objspace/std/smallintobject.py
--- a/pypy/objspace/std/smallintobject.py
+++ b/pypy/objspace/std/smallintobject.py
@@ -12,6 +12,7 @@
 from pypy.rlib.rbigint import rbigint
 from pypy.rlib.rarithmetic import r_uint
 from pypy.tool.sourcetools import func_with_new_name
+from pypy.objspace.std.inttype import wrapint
 
 class W_SmallIntObject(W_Object, UnboxedValue):
     __slots__ = 'intval'
@@ -48,14 +49,36 @@
 def delegate_SmallInt2Complex(space, w_small):
     return space.newcomplex(float(w_small.intval), 0.0)
 
+def add__SmallInt_SmallInt(space, w_a, w_b):
+    return wrapint(space, w_a.intval + w_b.intval) # cannot overflow
+
+def sub__SmallInt_SmallInt(space, w_a, w_b):
+    return wrapint(space, w_a.intval - w_b.intval) # cannot overflow
+
+def floordiv__SmallInt_SmallInt(space, w_a, w_b):
+    return wrapint(space, w_a.intval // w_b.intval) # cannot overflow
+
+div__SmallInt_SmallInt = floordiv__SmallInt_SmallInt
+
+def mod__SmallInt_SmallInt(space, w_a, w_b):
+    return wrapint(space, w_a.intval % w_b.intval) # cannot overflow
+
+def divmod__SmallInt_SmallInt(space, w_a, w_b):
+    w = wrapint(space, w_a.intval // w_b.intval) # cannot overflow
+    z = wrapint(space, w_a.intval % w_b.intval)
+    return space.newtuple([w, z])
+
 def copy_multimethods(ns):
     """Copy integer multimethods for small int."""
     for name, func in intobject.__dict__.iteritems():
         if "__Int" in name:
             new_name = name.replace("Int", "SmallInt")
-            # Copy the function, so the annotator specializes it for
-            # W_SmallIntObject.
-            ns[new_name] = func_with_new_name(func, new_name)
+            if new_name not in ns:
+                # Copy the function, so the annotator specializes it for
+                # W_SmallIntObject.
+                ns[new_name] = func = func_with_new_name(func, new_name, globals=ns)
+        else:
+            ns[name] = func
     ns["get_integer"] = ns["pos__SmallInt"] = ns["int__SmallInt"]
     ns["get_negint"] = ns["neg__SmallInt"]
 
diff --git a/pypy/objspace/std/strutil.py b/pypy/objspace/std/strutil.py
--- a/pypy/objspace/std/strutil.py
+++ b/pypy/objspace/std/strutil.py
@@ -35,7 +35,7 @@
 
     def error(self):
         raise ParseStringError("invalid literal for %s() with base %d: '%s'" %
-                               (self.fname, self.base, self.literal))
+                               (self.fname, self.original_base, self.literal))
 
     def __init__(self, s, literal, base, fname):
         self.literal = literal
@@ -47,7 +47,8 @@
         elif s.startswith('+'):
             s = strip_spaces(s[1:])
         self.sign = sign
-        
+        self.original_base = base
+
         if base == 0:
             if s.startswith('0x') or s.startswith('0X'):
                 base = 16
diff --git a/pypy/objspace/std/test/test_obj.py b/pypy/objspace/std/test/test_obj.py
--- a/pypy/objspace/std/test/test_obj.py
+++ b/pypy/objspace/std/test/test_obj.py
@@ -102,3 +102,11 @@
             def __repr__(self):
                 return 123456
         assert A().__str__() == 123456
+
+def test_isinstance_shortcut():
+    from pypy.objspace.std import objspace
+    space = objspace.StdObjSpace()
+    w_a = space.wrap("a")
+    space.type = None
+    space.isinstance_w(w_a, space.w_str) # does not crash
+
diff --git a/pypy/objspace/std/test/test_stdobjspace.py b/pypy/objspace/std/test/test_stdobjspace.py
--- a/pypy/objspace/std/test/test_stdobjspace.py
+++ b/pypy/objspace/std/test/test_stdobjspace.py
@@ -14,11 +14,11 @@
 
     def test_int_w_non_int(self):
         raises(OperationError,self.space.int_w,self.space.wrap(None))
-        raises(OperationError,self.space.int_w,self.space.wrap(""))        
+        raises(OperationError,self.space.int_w,self.space.wrap(""))
 
     def test_uint_w_non_int(self):
         raises(OperationError,self.space.uint_w,self.space.wrap(None))
-        raises(OperationError,self.space.uint_w,self.space.wrap(""))        
+        raises(OperationError,self.space.uint_w,self.space.wrap(""))
 
     def test_multimethods_defined_on(self):
         from pypy.objspace.std.stdtypedef import multimethods_defined_on
@@ -49,14 +49,14 @@
     def test_fastpath_isinstance(self):
         from pypy.objspace.std.stringobject import W_StringObject
         from pypy.objspace.std.intobject import W_IntObject
-        
+
         space = self.space
-        assert space.w_str.interplevel_cls is W_StringObject
-        assert space.w_int.interplevel_cls is W_IntObject
+        assert space._get_interplevel_cls(space.w_str) is W_StringObject
+        assert space._get_interplevel_cls(space.w_int) is W_IntObject
         class X(W_StringObject):
             def __init__(self):
                 pass
-            
+
             typedef = None
 
         assert space.isinstance_w(X(), space.w_str)
diff --git a/pypy/objspace/std/test/test_strutil.py b/pypy/objspace/std/test/test_strutil.py
--- a/pypy/objspace/std/test/test_strutil.py
+++ b/pypy/objspace/std/test/test_strutil.py
@@ -89,6 +89,8 @@
 
         exc = raises(ParseStringError, string_to_int, '')
         assert exc.value.msg == "invalid literal for int() with base 10: ''"
+        exc = raises(ParseStringError, string_to_int, '', 0)
+        assert exc.value.msg == "invalid literal for int() with base 0: ''"
 
     def test_string_to_int_overflow(self):
         import sys
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -102,7 +102,6 @@
                           'instancetypedef',
                           'terminator',
                           '_version_tag?',
-                          'interplevel_cls',
                           ]
 
     # for config.objspace.std.getattributeshortcut
@@ -117,9 +116,6 @@
     # of the __new__ is an instance of the type
     w_bltin_new = None
 
-    interplevel_cls = None # not None for prebuilt instances of
-                           # interpreter-level types
-
     @dont_look_inside
     def __init__(w_self, space, name, bases_w, dict_w,
                  overridetypedef=None):
diff --git a/pypy/pytest.ini b/pypy/pytest.ini
--- a/pypy/pytest.ini
+++ b/pypy/pytest.ini
@@ -1,2 +1,2 @@
 [pytest]
-addopts = --assertmode=old
\ No newline at end of file
+addopts = --assertmode=old -rf
diff --git a/pypy/rlib/_jit_vref.py b/pypy/rlib/_jit_vref.py
--- a/pypy/rlib/_jit_vref.py
+++ b/pypy/rlib/_jit_vref.py
@@ -25,6 +25,10 @@
     def simple_call(self):
         return self.s_instance
 
+    def getattr(self, s_attr):
+        assert s_attr.const == 'virtual'
+        return annmodel.s_Bool
+
     def rtyper_makerepr(self, rtyper):
         if rtyper.type_system.name == 'lltypesystem':
             return vrefrepr
@@ -61,6 +65,13 @@
                              " prebuilt virtual_ref")
         return lltype.nullptr(OBJECTPTR.TO)
 
+    def rtype_getattr(self, hop):
+        s_attr = hop.args_s[1]
+        assert s_attr.const == 'virtual'
+        v = hop.inputarg(self, arg=0)
+        hop.exception_cannot_occur()
+        return hop.genop('jit_is_virtual', [v], resulttype = lltype.Bool)
+
 from pypy.rpython.ootypesystem.rclass import OBJECT
 
 class OOVRefRepr(VRefRepr):
diff --git a/pypy/rlib/_rsocket_rffi.py b/pypy/rlib/_rsocket_rffi.py
--- a/pypy/rlib/_rsocket_rffi.py
+++ b/pypy/rlib/_rsocket_rffi.py
@@ -16,6 +16,7 @@
 _MINGW = target_platform.name == "mingw32"
 _SOLARIS = sys.platform == "sunos5"
 _MACOSX = sys.platform == "darwin"
+_HAS_AF_PACKET = sys.platform.startswith('linux')   # only Linux for now
 
 if _POSIX:
     includes = ('sys/types.h',
@@ -34,11 +35,12 @@
                 'stdint.h', 
                 'errno.h',
                 )
+    if _HAS_AF_PACKET:
+        includes += ('netpacket/packet.h',
+                     'sys/ioctl.h',
+                     'net/if.h')
 
-    cond_includes = [('AF_NETLINK', 'linux/netlink.h'),
-                     ('AF_PACKET', 'netpacket/packet.h'),
-                     ('AF_PACKET', 'sys/ioctl.h'),
-                     ('AF_PACKET', 'net/if.h')]
+    cond_includes = [('AF_NETLINK', 'linux/netlink.h')]
     
     libraries = ()
     calling_conv = 'c'
@@ -320,18 +322,18 @@
                                              ('events', rffi.SHORT),
                                              ('revents', rffi.SHORT)])
 
-    CConfig.sockaddr_ll = platform.Struct('struct sockaddr_ll',
+    if _HAS_AF_PACKET:
+        CConfig.sockaddr_ll = platform.Struct('struct sockaddr_ll',
                               [('sll_ifindex', rffi.INT),
                                ('sll_protocol', rffi.INT),
                                ('sll_pkttype', rffi.INT),
                                ('sll_hatype', rffi.INT),
                                ('sll_addr', rffi.CFixedArray(rffi.CHAR, 8)),
-                               ('sll_halen', rffi.INT)],
-                              ifdef='AF_PACKET')
+                               ('sll_halen', rffi.INT)])
 
-    CConfig.ifreq = platform.Struct('struct ifreq', [('ifr_ifindex', rffi.INT),
-                                 ('ifr_name', rffi.CFixedArray(rffi.CHAR, 8))],
-                                    ifdef='AF_PACKET')
+        CConfig.ifreq = platform.Struct('struct ifreq',
+                                [('ifr_ifindex', rffi.INT),
+                                 ('ifr_name', rffi.CFixedArray(rffi.CHAR, 8))])
 
 if _WIN32:
     CConfig.WSAEVENT = platform.SimpleType('WSAEVENT', rffi.VOIDP)
@@ -386,6 +388,8 @@
         constants[name] = value
     else:
         constants[name] = default
+if not _HAS_AF_PACKET and 'AF_PACKET' in constants:
+    del constants['AF_PACKET']
 
 constants['has_ipv6'] = True # This is a configuration option in CPython
 for name, value in constants.items():
@@ -439,21 +443,14 @@
 if _POSIX:
     nfds_t = cConfig.nfds_t
     pollfd = cConfig.pollfd
-    if cConfig.sockaddr_ll is not None:
+    if _HAS_AF_PACKET:
         sockaddr_ll = cConfig.sockaddr_ll
-    ifreq = cConfig.ifreq
+        ifreq = cConfig.ifreq
 if WIN32:
     WSAEVENT = cConfig.WSAEVENT
     WSANETWORKEVENTS = cConfig.WSANETWORKEVENTS
 timeval = cConfig.timeval
 
-#if _POSIX:
-#    includes = list(includes)
-#    for _name, _header in cond_includes:
-#        if getattr(cConfig, _name) is not None:
-#            includes.append(_header)
-#    eci = ExternalCompilationInfo(includes=includes, libraries=libraries,
-#                                  separate_module_sources=sources)
 
 def external(name, args, result, **kwds):
     return rffi.llexternal(name, args, result, compilation_info=eci,
@@ -544,7 +541,7 @@
     socketpair_t = rffi.CArray(socketfd_type)
     socketpair = external('socketpair', [rffi.INT, rffi.INT, rffi.INT,
                           lltype.Ptr(socketpair_t)], rffi.INT)
-    if ifreq is not None:
+    if _HAS_AF_PACKET:
         ioctl = external('ioctl', [socketfd_type, rffi.INT, lltype.Ptr(ifreq)],
                          rffi.INT)
 
diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -8,6 +8,8 @@
 from pypy.rpython.extregistry import ExtRegistryEntry
 from pypy.tool.sourcetools import func_with_new_name
 
+DEBUG_ELIDABLE_FUNCTIONS = False
+
 
 def elidable(func):
     """ Decorate a function as "trace-elidable". This means precisely that:
@@ -24,6 +26,18 @@
     If a particular call to this function ends up raising an exception, then it
     is handled like a normal function call (this decorator is ignored).
     """
+    if DEBUG_ELIDABLE_FUNCTIONS:
+        cache = {}
+        oldfunc = func
+        def func(*args):
+            result = oldfunc(*args)    # if it raises, no caching
+            try:
+                oldresult = cache.setdefault(args, result)
+            except TypeError:
+                pass           # unhashable args
+            else:
+                assert oldresult == result
+            return result
     func._elidable_function_ = True
     return func
 
@@ -317,6 +331,12 @@
             raise InvalidVirtualRef
         return self._x
 
+    @property
+    def virtual(self):
+        """A property that is True if the vref contains a virtual that would
+        be forced by the '()' operator."""
+        return self._state == 'non-forced'
+
     def _finish(self):
         if self._state == 'non-forced':
             self._state = 'invalid'
diff --git a/pypy/rlib/rerased.py b/pypy/rlib/rerased.py
--- a/pypy/rlib/rerased.py
+++ b/pypy/rlib/rerased.py
@@ -135,6 +135,8 @@
     _about_ = erase_int
 
     def compute_result_annotation(self, s_obj):
+        config = self.bookkeeper.annotator.translator.config
+        assert config.translation.taggedpointers, "need to enable tagged pointers to use erase_int"
         assert annmodel.SomeInteger().contains(s_obj)
         return SomeErased()
 
@@ -218,18 +220,18 @@
         [v_value] = hop.inputargs(lltype.Signed)
         c_one = hop.inputconst(lltype.Signed, 1)
         hop.exception_is_here()
-        v2 = hop.genop('int_lshift_ovf', [v_value, c_one],
+        v2 = hop.genop('int_add_ovf', [v_value, v_value],
                        resulttype = lltype.Signed)
         v2p1 = hop.genop('int_add', [v2, c_one],
                          resulttype = lltype.Signed)
         v_instance = hop.genop('cast_int_to_ptr', [v2p1],
                                resulttype=self.lowleveltype)
-        v = hop.genop('cast_opaque_ptr', [v_instance],
-                      resulttype=self.lowleveltype)
-        return v
+        return v_instance
 
     def convert_const(self, value):
         if value._identity is _identity_for_ints:
+            config = self.rtyper.annotator.translator.config
+            assert config.translation.taggedpointers, "need to enable tagged pointers to use erase_int"
             return lltype.cast_int_to_ptr(self.lowleveltype, value._x * 2 + 1)
         bk = self.rtyper.annotator.bookkeeper
         s_obj = value._identity.get_input_annotation(bk)
@@ -266,10 +268,10 @@
         return hop.genop('int_rshift', [v2, c_one], resulttype=lltype.Signed)
 
     def rtype_erase_int(self, hop):
-        hop.exception_is_here()
         [v_value] = hop.inputargs(lltype.Signed)
         c_one = hop.inputconst(lltype.Signed, 1)
-        v2 = hop.genop('int_lshift_ovf', [v_value, c_one],
+        hop.exception_is_here()
+        v2 = hop.genop('int_add_ovf', [v_value, v_value],
                        resulttype = lltype.Signed)
         v2p1 = hop.genop('int_add', [v2, c_one],
                          resulttype = lltype.Signed)
diff --git a/pypy/rlib/rgc.py b/pypy/rlib/rgc.py
--- a/pypy/rlib/rgc.py
+++ b/pypy/rlib/rgc.py
@@ -214,6 +214,10 @@
     func._gc_no_collect_ = True
     return func
 
+def is_light_finalizer(func):
+    func._is_light_finalizer_ = True
+    return func
+
 # ____________________________________________________________
 
 def get_rpy_roots():
diff --git a/pypy/rlib/rmmap.py b/pypy/rlib/rmmap.py
--- a/pypy/rlib/rmmap.py
+++ b/pypy/rlib/rmmap.py
@@ -292,6 +292,9 @@
         elif _POSIX:
             self.closed = True
             if self.fd != -1:
+                # XXX this is buggy - raising in an RPython del is not a good
+                #     idea, we should swallow the exception or ignore the
+                #     underlaying close error code
                 os.close(self.fd)
                 self.fd = -1
             if self.size > 0:
diff --git a/pypy/rlib/rsocket.py b/pypy/rlib/rsocket.py
--- a/pypy/rlib/rsocket.py
+++ b/pypy/rlib/rsocket.py
@@ -8,6 +8,7 @@
 # Known missing features:
 #
 #   - address families other than AF_INET, AF_INET6, AF_UNIX, AF_PACKET
+#   - AF_PACKET is only supported on Linux
 #   - methods makefile(),
 #   - SSL
 #
@@ -55,6 +56,7 @@
 
 
 _FAMILIES = {}
+
 class Address(object):
     """The base class for RPython-level objects representing addresses.
     Fields:  addr    - a _c.sockaddr_ptr (memory owned by the Address instance)
@@ -76,9 +78,8 @@
         self.addrlen = addrlen
 
     def __del__(self):
-        addr = self.addr_p
-        if addr:
-            lltype.free(addr, flavor='raw')
+        if self.addr_p:
+            lltype.free(self.addr_p, flavor='raw')
 
     def setdata(self, addr, addrlen):
         # initialize self.addr and self.addrlen.  'addr' can be a different
@@ -612,7 +613,10 @@
         self.timeout = defaults.timeout
         
     def __del__(self):
-        self.close()
+        fd = self.fd
+        if fd != _c.INVALID_SOCKET:
+            self.fd = _c.INVALID_SOCKET
+            _c.socketclose(fd)
 
     if hasattr(_c, 'fcntl'):
         def _setblocking(self, block):
diff --git a/pypy/rlib/test/test__jit_vref.py b/pypy/rlib/test/test__jit_vref.py
--- a/pypy/rlib/test/test__jit_vref.py
+++ b/pypy/rlib/test/test__jit_vref.py
@@ -27,10 +27,13 @@
     x1 = X()
     vref = virtual_ref(x1)
     assert vref._state == 'non-forced'
+    assert vref.virtual is True
     assert vref() is x1
     assert vref._state == 'forced'
+    assert vref.virtual is False
     virtual_ref_finish(vref, x1)
     assert vref._state == 'forced'
+    assert vref.virtual is False
     assert vref() is x1
 
 def test_direct_invalid():
@@ -135,6 +138,13 @@
         x = self.interpret(f, [])
         assert x == 42
 
+    def test_rtype_virtualattr(self):
+        def f():
+            vref = virtual_ref(X())
+            return vref.virtual
+        x = self.interpret(f, [])
+        assert x is False
+
 
 class TestLLtype(BaseTestVRef, LLRtypeMixin):
     OBJECTTYPE = OBJECTPTR
diff --git a/pypy/rlib/test/test_jit.py b/pypy/rlib/test/test_jit.py
--- a/pypy/rlib/test/test_jit.py
+++ b/pypy/rlib/test/test_jit.py
@@ -139,12 +139,11 @@
 
     def test_isconstant(self):
         def f(n):
-            assert n >= 0
             assert isconstant(n) is False
             l = []
             l.append(n)
             return len(l)
-        res = self.interpret(f, [234])
+        res = self.interpret(f, [-234])
         assert res == 1
 
 
diff --git a/pypy/rlib/test/test_rerased.py b/pypy/rlib/test/test_rerased.py
--- a/pypy/rlib/test/test_rerased.py
+++ b/pypy/rlib/test/test_rerased.py
@@ -10,6 +10,13 @@
 
 from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin
 
+def make_annotator():
+    a = RPythonAnnotator()
+    a.translator.config.translation.taggedpointers = True
+    return a
+
+
+
 class X(object):
     pass
 
@@ -55,7 +62,7 @@
 def test_annotate_1():
     def f():
         return eraseX(X())
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [])
     assert isinstance(s, SomeErased)
 
@@ -66,7 +73,7 @@
         #assert not is_integer(e)
         x2 = uneraseX(e)
         return x2
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [])
     assert isinstance(s, annmodel.SomeInstance)
     assert s.classdef == a.bookkeeper.getuniqueclassdef(X)
@@ -77,7 +84,7 @@
         #assert is_integer(e)
         x2 = unerase_int(e)
         return x2
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [])
     assert isinstance(s, annmodel.SomeInteger)
 
@@ -105,7 +112,7 @@
         x = make(n)
         return check(x, n)
     #
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [int])
     assert isinstance(s, annmodel.SomeInteger)
 
@@ -135,7 +142,7 @@
             else:
                 return inst
     #
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [])
     assert isinstance(s, annmodel.SomeInstance)
     assert s.classdef == a.bookkeeper.getuniqueclassdef(A)
@@ -155,7 +162,7 @@
             e = e2
         return unerase(e)
     #
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [int])
     assert isinstance(s, annmodel.SomeInstance)
     assert s.classdef == a.bookkeeper.getuniqueclassdef(X)
@@ -165,11 +172,14 @@
     e1 = erase_int(42)
     def f(i):
         return unerase_int(e1)
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [int])
     assert isinstance(s, annmodel.SomeInteger)
 
 class BaseTestRErased(BaseRtypingTest):
+    def interpret(self, *args, **kwargs):
+        kwargs["taggedpointers"] = True
+        return BaseRtypingTest.interpret(self, *args, **kwargs)
     def test_rtype_1(self):
         def f():
             return eraseX(X())
diff --git a/pypy/rpython/llinterp.py b/pypy/rpython/llinterp.py
--- a/pypy/rpython/llinterp.py
+++ b/pypy/rpython/llinterp.py
@@ -1095,13 +1095,6 @@
             assert y >= 0
         return self.op_int_add_ovf(x, y)
 
-    def op_cast_float_to_int(self, f):
-        assert type(f) is float
-        try:
-            return ovfcheck(int(f))
-        except OverflowError:
-            self.make_llexception()
-
     def op_int_is_true(self, x):
         # special case
         if type(x) is CDefinedIntSymbolic:
diff --git a/pypy/rpython/lltypesystem/ll2ctypes.py b/pypy/rpython/lltypesystem/ll2ctypes.py
--- a/pypy/rpython/lltypesystem/ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/ll2ctypes.py
@@ -15,12 +15,11 @@
     load_library_kwargs = {}
 
 import os
-from pypy import conftest
 from pypy.rpython.lltypesystem import lltype, llmemory
 from pypy.rpython.extfunc import ExtRegistryEntry
 from pypy.rlib.objectmodel import Symbolic, ComputedIntSymbolic
 from pypy.tool.uid import fixid
-from pypy.rlib.rarithmetic import r_uint, r_singlefloat, r_longfloat, base_int, intmask
+from pypy.rlib.rarithmetic import r_singlefloat, r_longfloat, base_int, intmask
 from pypy.annotation import model as annmodel
 from pypy.rpython.llinterp import LLInterpreter, LLException
 from pypy.rpython.lltypesystem.rclass import OBJECT, OBJECT_VTABLE
@@ -531,6 +530,10 @@
     def __str__(self):
         return repr(self)
 
+    def _setparentstructure(self, parent, parentindex):
+        super(_parentable_mixin, self)._setparentstructure(parent, parentindex)
+        self._keepparent = parent   # always keep a strong ref
+
 class _struct_mixin(_parentable_mixin):
     """Mixin added to _struct containers when they become ctypes-based."""
     __slots__ = ()
@@ -566,7 +569,10 @@
         return 0, sys.maxint
 
     def getitem(self, index, uninitialized_ok=False):
-        return self._storage.contents._getitem(index, boundscheck=False)
+        res = self._storage.contents._getitem(index, boundscheck=False)
+        if isinstance(self._TYPE.OF, lltype.ContainerType):
+            res._obj._setparentstructure(self, index)
+        return res
 
     def setitem(self, index, value):
         self._storage.contents._setitem(index, value, boundscheck=False)
@@ -658,6 +664,8 @@
         if T == llmemory.GCREF:
             if isinstance(llobj._obj, _llgcopaque):
                 return ctypes.c_void_p(llobj._obj.intval)
+            if isinstance(llobj._obj, int):    # tagged pointer
+                return ctypes.c_void_p(llobj._obj)
             container = llobj._obj.container
             T = lltype.Ptr(lltype.typeOf(container))
             # otherwise it came from integer and we want a c_void_p with
@@ -1268,6 +1276,7 @@
 class _llgcopaque(lltype._container):
     _TYPE = llmemory.GCREF.TO
     _name = "_llgcopaque"
+    _read_directly_intval = True     # for _ptr._cast_to_int()
 
     def __init__(self, void_p):
         if isinstance(void_p, (int, long)):
@@ -1276,6 +1285,8 @@
             self.intval = intmask(void_p.value)
 
     def __eq__(self, other):
+        if not other:
+            return self.intval == 0
         if isinstance(other, _llgcopaque):
             return self.intval == other.intval
         storage = object()
@@ -1299,11 +1310,6 @@
             return _opaque_objs[self.intval // 2]
         return force_cast(PTRTYPE, self.intval)
 
-##     def _cast_to_int(self):
-##         return self.intval
-
-##     def _cast_to_adr(self):
-##         return _lladdress(self.intval)
 
 def cast_adr_to_int(addr):
     if isinstance(addr, llmemory.fakeaddress):
diff --git a/pypy/rpython/lltypesystem/llmemory.py b/pypy/rpython/lltypesystem/llmemory.py
--- a/pypy/rpython/lltypesystem/llmemory.py
+++ b/pypy/rpython/lltypesystem/llmemory.py
@@ -498,6 +498,8 @@
 
     def _cast_to_int(self, symbolic=False):
         if self:
+            if isinstance(self.ptr._obj0, int):    # tagged integer
+                return self.ptr._obj0
             if symbolic:
                 return AddressAsInt(self)
             else:
diff --git a/pypy/rpython/lltypesystem/lloperation.py b/pypy/rpython/lltypesystem/lloperation.py
--- a/pypy/rpython/lltypesystem/lloperation.py
+++ b/pypy/rpython/lltypesystem/lloperation.py
@@ -343,8 +343,8 @@
     'cast_uint_to_float':   LLOp(canfold=True),
     'cast_longlong_to_float' :LLOp(canfold=True),
     'cast_ulonglong_to_float':LLOp(canfold=True),
-    'cast_float_to_int':    LLOp(canraise=(OverflowError,), tryfold=True),
-    'cast_float_to_uint':   LLOp(canfold=True),    # XXX need OverflowError?
+    'cast_float_to_int':    LLOp(canfold=True),
+    'cast_float_to_uint':   LLOp(canfold=True),
     'cast_float_to_longlong' :LLOp(canfold=True),
     'cast_float_to_ulonglong':LLOp(canfold=True),
     'truncate_longlong_to_int':LLOp(canfold=True),
@@ -428,6 +428,7 @@
     'jit_marker':           LLOp(),
     'jit_force_virtualizable':LLOp(canrun=True),
     'jit_force_virtual':    LLOp(canrun=True),
+    'jit_is_virtual':       LLOp(canrun=True),
     'jit_force_quasi_immutable': LLOp(canrun=True),
     'get_exception_addr':   LLOp(),
     'get_exc_value_addr':   LLOp(),
diff --git a/pypy/rpython/lltypesystem/lltype.py b/pypy/rpython/lltypesystem/lltype.py
--- a/pypy/rpython/lltypesystem/lltype.py
+++ b/pypy/rpython/lltypesystem/lltype.py
@@ -1360,6 +1360,8 @@
         obj = normalizeptr(self, check)._getobj(check)
         if isinstance(obj, int):
             return obj     # special case for cast_int_to_ptr() results put into opaques
+        if getattr(obj, '_read_directly_intval', False):
+            return obj.intval   # special case for _llgcopaque
         result = intmask(obj._getid())
         # assume that id() returns an addressish value which is
         # not zero and aligned to at least a multiple of 4
@@ -1520,7 +1522,7 @@
             and parentindex in (self._parent_type._names[0], 0)
             and self._TYPE._gckind == typeOf(parent)._gckind):
             # keep strong reference to parent, we share the same allocation
-            self._keepparent = parent 
+            self._keepparent = parent
 
     def _parentstructure(self, check=True):
         if self._wrparent is not None:
@@ -1711,6 +1713,7 @@
         return v
 
     def setitem(self, index, value):
+        assert typeOf(value) == self._TYPE.OF
         self.items[index] = value
 
 assert not '__dict__' in dir(_array)
@@ -1728,7 +1731,8 @@
         # Keep the parent array alive, we share the same allocation.
         # Don't do it if we are inside a GC object, though -- it's someone
         # else's job to keep the GC object alive
-        if typeOf(top_container(parent))._gckind == 'raw':
+        if (typeOf(top_container(parent))._gckind == 'raw' or
+            hasattr(top_container(parent)._storage, 'contents')):  # ll2ctypes
             self._keepparent = parent
 
     def __str__(self):
diff --git a/pypy/rpython/lltypesystem/opimpl.py b/pypy/rpython/lltypesystem/opimpl.py
--- a/pypy/rpython/lltypesystem/opimpl.py
+++ b/pypy/rpython/lltypesystem/opimpl.py
@@ -355,6 +355,10 @@
     assert type(b) is bool
     return float(b)
 
+def op_cast_float_to_int(f):
+    assert type(f) is float
+    return intmask(int(f))
+
 def op_cast_float_to_uint(f):
     assert type(f) is float
     return r_uint(long(f))
@@ -538,6 +542,9 @@
 def op_jit_force_virtual(x):
     return x
 
+def op_jit_is_virtual(x):
+    return False
+
 def op_jit_force_quasi_immutable(*args):
     pass
 
diff --git a/pypy/rpython/lltypesystem/rbuilder.py b/pypy/rpython/lltypesystem/rbuilder.py
--- a/pypy/rpython/lltypesystem/rbuilder.py
+++ b/pypy/rpython/lltypesystem/rbuilder.py
@@ -29,7 +29,7 @@
         except OverflowError:
             raise MemoryError
         newbuf = mallocfn(new_allocated)
-        copycontentsfn(ll_builder.buf, newbuf, 0, 0, ll_builder.allocated)
+        copycontentsfn(ll_builder.buf, newbuf, 0, 0, ll_builder.used)
         ll_builder.buf = newbuf
         ll_builder.allocated = new_allocated
     return func_with_new_name(stringbuilder_grow, name)
@@ -56,7 +56,7 @@
 class BaseStringBuilderRepr(AbstractStringBuilderRepr):
     def empty(self):
         return nullptr(self.lowleveltype.TO)
-    
+
     @classmethod
     def ll_new(cls, init_size):
         if init_size < 0 or init_size > MAX:
diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py
--- a/pypy/rpython/lltypesystem/rdict.py
+++ b/pypy/rpython/lltypesystem/rdict.py
@@ -1,16 +1,14 @@
 from pypy.tool.pairtype import pairtype
-from pypy.annotation import model as annmodel
 from pypy.objspace.flow.model import Constant
-from pypy.rpython.rdict import AbstractDictRepr, AbstractDictIteratorRepr,\
-     rtype_newdict
+from pypy.rpython.rdict import (AbstractDictRepr, AbstractDictIteratorRepr,
+     rtype_newdict)
 from pypy.rpython.lltypesystem import lltype
+from pypy.rlib import objectmodel, jit
 from pypy.rlib.rarithmetic import r_uint, intmask, LONG_BIT
-from pypy.rlib.objectmodel import hlinvoke
-from pypy.rpython import robject
-from pypy.rlib import objectmodel, jit
 from pypy.rpython import rmodel
 from pypy.rpython.error import TyperError
 
+
 HIGHEST_BIT = intmask(1 << (LONG_BIT - 1))
 MASK = intmask(HIGHEST_BIT - 1)
 
@@ -417,17 +415,16 @@
     ENTRIES = lltype.typeOf(entries).TO
     return ENTRIES.fasthashfn(entries[i].key)
 
- at jit.dont_look_inside
 def ll_get_value(d, i):
     return d.entries[i].value
 
 def ll_keyhash_custom(d, key):
     DICT = lltype.typeOf(d).TO
-    return hlinvoke(DICT.r_rdict_hashfn, d.fnkeyhash, key)
+    return objectmodel.hlinvoke(DICT.r_rdict_hashfn, d.fnkeyhash, key)
 
 def ll_keyeq_custom(d, key1, key2):
     DICT = lltype.typeOf(d).TO
-    return hlinvoke(DICT.r_rdict_eqfn, d.fnkeyeq, key1, key2)
+    return objectmodel.hlinvoke(DICT.r_rdict_eqfn, d.fnkeyeq, key1, key2)
 
 def ll_dict_len(d):
     return d.num_items
@@ -448,7 +445,9 @@
     i = ll_dict_lookup(d, key, hash)
     return _ll_dict_setitem_lookup_done(d, key, value, hash, i)
 
- at jit.dont_look_inside
+# It may be safe to look inside always, it has a few branches though, and their
+# frequencies needs to be investigated.
+ at jit.look_inside_iff(lambda d, key, value, hash, i: jit.isvirtual(d) and jit.isconstant(key))
 def _ll_dict_setitem_lookup_done(d, key, value, hash, i):
     valid = (i & HIGHEST_BIT) == 0
     i = i & MASK
@@ -492,6 +491,8 @@
         raise KeyError
     _ll_dict_del(d, i)
 
+# XXX: Move the size checking and resize into a single call which is opauqe to
+# the JIT to avoid extra branches.
 @jit.dont_look_inside
 def _ll_dict_del(d, i):
     d.entries.mark_deleted(i)
@@ -532,6 +533,7 @@
 # ------- a port of CPython's dictobject.c's lookdict implementation -------
 PERTURB_SHIFT = 5
 
+ at jit.look_inside_iff(lambda d, key, hash: jit.isvirtual(d) and jit.isconstant(key))
 def ll_dict_lookup(d, key, hash):
     entries = d.entries
     ENTRIES = lltype.typeOf(entries).TO
@@ -621,7 +623,6 @@
     d.num_items = 0
     d.resize_counter = DICT_INITSIZE * 2
     return d
-ll_newdict.oopspec = 'newdict()'
 
 def ll_newdict_size(DICT, length_estimate):
     length_estimate = (length_estimate // 2) * 3
@@ -633,7 +634,6 @@
     d.num_items = 0
     d.resize_counter = n * 2
     return d
-ll_newdict_size.oopspec = 'newdict()'
 
 # pypy.rpython.memory.lldict uses a dict based on Struct and Array
 # instead of GcStruct and GcArray, which is done by using different
@@ -866,7 +866,6 @@
         global_popitem_index.nextindex = base + counter
     return i
 
- at jit.dont_look_inside
 def ll_popitem(ELEM, dic):
     i = _ll_getnextitem(dic)
     entry = dic.entries[i]
diff --git a/pypy/rpython/lltypesystem/rtagged.py b/pypy/rpython/lltypesystem/rtagged.py
--- a/pypy/rpython/lltypesystem/rtagged.py
+++ b/pypy/rpython/lltypesystem/rtagged.py
@@ -43,7 +43,7 @@
         v_value = hop.inputarg(lltype.Signed, arg=1)
         c_one = hop.inputconst(lltype.Signed, 1)
         hop.exception_is_here()
-        v2 = hop.genop('int_lshift_ovf', [v_value, c_one],
+        v2 = hop.genop('int_add_ovf', [v_value, v_value],
                        resulttype = lltype.Signed)
         v2p1 = hop.genop('int_add', [v2, c_one],
                          resulttype = lltype.Signed)
diff --git a/pypy/rpython/lltypesystem/test/test_ll2ctypes.py b/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
--- a/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
@@ -8,6 +8,7 @@
 from pypy.rpython.lltypesystem.ll2ctypes import uninitialized2ctypes
 from pypy.rpython.lltypesystem.ll2ctypes import ALLOCATED, force_cast
 from pypy.rpython.lltypesystem.ll2ctypes import cast_adr_to_int, get_ctypes_type
+from pypy.rpython.lltypesystem.ll2ctypes import _llgcopaque
 from pypy.rpython.annlowlevel import llhelper
 from pypy.rlib import rposix
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
@@ -1123,6 +1124,9 @@
 
         #assert lltype.cast_ptr_to_int(ref1) == intval
 
+        x = rffi.cast(llmemory.GCREF, -17)
+        assert lltype.cast_ptr_to_int(x) == -17
+
     def test_ptr_truth(self):
         abc = rffi.cast(lltype.Ptr(lltype.FuncType([], lltype.Void)), 0)
         assert not abc
@@ -1346,6 +1350,11 @@
         round = ctypes2lltype(llmemory.GCREF, lltype2ctypes(opaque.hide()))
         assert Opaque.show(round) is opaque
 
+    def test_array_of_structs(self):
+        A = lltype.GcArray(lltype.Struct('x', ('v', lltype.Signed)))
+        a = lltype.malloc(A, 5)
+        a2 = ctypes2lltype(lltype.Ptr(A), lltype2ctypes(a))
+        assert a2._obj.getitem(0)._obj._parentstructure() is a2._obj
 
 class TestPlatform(object):
     def test_lib_on_libpaths(self):
@@ -1387,3 +1396,7 @@
         f = rffi.llexternal('f', [rffi.INT, rffi.INT], rffi.INT,
                             compilation_info=eci)
         assert f(3, 4) == 7
+
+    def test_llgcopaque_eq(self):
+        assert _llgcopaque(1) != None
+        assert _llgcopaque(0) == None
diff --git a/pypy/rpython/memory/gc/base.py b/pypy/rpython/memory/gc/base.py
--- a/pypy/rpython/memory/gc/base.py
+++ b/pypy/rpython/memory/gc/base.py
@@ -1,4 +1,5 @@
-from pypy.rpython.lltypesystem import lltype, llmemory, llarena
+from pypy.rpython.lltypesystem import lltype, llmemory, llarena, rffi
+from pypy.rpython.lltypesystem.lloperation import llop
 from pypy.rlib.debug import ll_assert
 from pypy.rpython.memory.gcheader import GCHeaderBuilder
 from pypy.rpython.memory.support import DEFAULT_CHUNK_SIZE
@@ -62,6 +63,7 @@
     def set_query_functions(self, is_varsize, has_gcptr_in_varsize,
                             is_gcarrayofgcptr,
                             getfinalizer,
+                            getlightfinalizer,
                             offsets_to_gc_pointers,
                             fixed_size, varsize_item_sizes,
                             varsize_offset_to_variable_part,
@@ -74,6 +76,7 @@
                             get_custom_trace,
                             fast_path_tracing):
         self.getfinalizer = getfinalizer
+        self.getlightfinalizer = getlightfinalizer
         self.is_varsize = is_varsize
         self.has_gcptr_in_varsize = has_gcptr_in_varsize
         self.is_gcarrayofgcptr = is_gcarrayofgcptr
@@ -139,6 +142,7 @@
 
         size = self.fixed_size(typeid)
         needs_finalizer = bool(self.getfinalizer(typeid))
+        finalizer_is_light = bool(self.getlightfinalizer(typeid))
         contains_weakptr = self.weakpointer_offset(typeid) >= 0
         assert not (needs_finalizer and contains_weakptr)
         if self.is_varsize(typeid):
@@ -158,6 +162,7 @@
             else:
                 malloc_fixedsize = self.malloc_fixedsize
             ref = malloc_fixedsize(typeid, size, needs_finalizer,
+                                   finalizer_is_light,
                                    contains_weakptr)
         # lots of cast and reverse-cast around...
         return llmemory.cast_ptr_to_adr(ref)
diff --git a/pypy/rpython/memory/gc/generation.py b/pypy/rpython/memory/gc/generation.py
--- a/pypy/rpython/memory/gc/generation.py
+++ b/pypy/rpython/memory/gc/generation.py
@@ -167,7 +167,9 @@
         return self.nursery <= addr < self.nursery_top
 
     def malloc_fixedsize_clear(self, typeid, size,
-                               has_finalizer=False, contains_weakptr=False):
+                               has_finalizer=False,
+                               is_finalizer_light=False,
+                               contains_weakptr=False):
         if (has_finalizer or
             (raw_malloc_usage(size) > self.lb_young_fixedsize and
              raw_malloc_usage(size) > self.largest_young_fixedsize)):
@@ -179,6 +181,7 @@
             # "non-simple" case or object too big: don't use the nursery
             return SemiSpaceGC.malloc_fixedsize_clear(self, typeid, size,
                                                       has_finalizer,
+                                                      is_finalizer_light,
                                                       contains_weakptr)
         size_gc_header = self.gcheaderbuilder.size_gc_header
         totalsize = size_gc_header + size
diff --git a/pypy/rpython/memory/gc/marksweep.py b/pypy/rpython/memory/gc/marksweep.py
--- a/pypy/rpython/memory/gc/marksweep.py
+++ b/pypy/rpython/memory/gc/marksweep.py
@@ -93,7 +93,8 @@
         pass
 
     def malloc_fixedsize(self, typeid16, size,
-                         has_finalizer=False, contains_weakptr=False):
+                         has_finalizer=False, is_finalizer_light=False,
+                         contains_weakptr=False):
         self.maybe_collect()
         size_gc_header = self.gcheaderbuilder.size_gc_header
         try:
@@ -128,7 +129,9 @@
     malloc_fixedsize._dont_inline_ = True
 
     def malloc_fixedsize_clear(self, typeid16, size,
-                               has_finalizer=False, contains_weakptr=False):
+                               has_finalizer=False,
+                               is_finalizer_light=False,
+                               contains_weakptr=False):
         self.maybe_collect()
         size_gc_header = self.gcheaderbuilder.size_gc_header
         try:
diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -290,6 +290,8 @@
         #
         # A list of all objects with finalizers (these are never young).
         self.objects_with_finalizers = self.AddressDeque()
+        self.young_objects_with_light_finalizers = self.AddressStack()
+        self.old_objects_with_light_finalizers = self.AddressStack()
         #
         # Two lists of the objects with weakrefs.  No weakref can be an
         # old object weakly pointing to a young object: indeed, weakrefs
@@ -457,14 +459,16 @@
 
 
     def malloc_fixedsize_clear(self, typeid, size,
-                               needs_finalizer=False, contains_weakptr=False):
+                               needs_finalizer=False,
+                               is_finalizer_light=False,
+                               contains_weakptr=False):
         size_gc_header = self.gcheaderbuilder.size_gc_header
         totalsize = size_gc_header + size
         rawtotalsize = raw_malloc_usage(totalsize)
         #
         # If the object needs a finalizer, ask for a rawmalloc.
         # The following check should be constant-folded.
-        if needs_finalizer:
+        if needs_finalizer and not is_finalizer_light:
             ll_assert(not contains_weakptr,
                      "'needs_finalizer' and 'contains_weakptr' both specified")
             obj = self.external_malloc(typeid, 0, can_make_young=False)
@@ -494,13 +498,14 @@
             #
             # Build the object.
             llarena.arena_reserve(result, totalsize)
+            obj = result + size_gc_header
+            if is_finalizer_light:
+                self.young_objects_with_light_finalizers.append(obj)
             self.init_gc_object(result, typeid, flags=0)
             #
             # If it is a weakref, record it (check constant-folded).
             if contains_weakptr:
-                self.young_objects_with_weakrefs.append(result+size_gc_header)
-            #
-            obj = result + size_gc_header
+                self.young_objects_with_weakrefs.append(obj)
         #
         return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
 
@@ -1264,6 +1269,8 @@
         # weakrefs' targets.
         if self.young_objects_with_weakrefs.non_empty():
             self.invalidate_young_weakrefs()
+        if self.young_objects_with_light_finalizers.non_empty():
+            self.deal_with_young_objects_with_finalizers()
         #
         # Clear this mapping.
         if self.nursery_objects_shadows.length() > 0:
@@ -1292,10 +1299,12 @@
         # if a prebuilt GcStruct contains a pointer to a young object,
         # then the write_barrier must have ensured that the prebuilt
         # GcStruct is in the list self.old_objects_pointing_to_young.
+        debug_start("gc-minor-walkroots")
         self.root_walker.walk_roots(
             MiniMarkGC._trace_drag_out1,  # stack roots
             MiniMarkGC._trace_drag_out1,  # static in prebuilt non-gc
             None)                         # static in prebuilt gc
+        debug_stop("gc-minor-walkroots")
 
     def collect_cardrefs_to_nursery(self):
         size_gc_header = self.gcheaderbuilder.size_gc_header
@@ -1582,6 +1591,9 @@
         # Weakref support: clear the weak pointers to dying objects
         if self.old_objects_with_weakrefs.non_empty():
             self.invalidate_old_weakrefs()
+        if self.old_objects_with_light_finalizers.non_empty():
+            self.deal_with_old_objects_with_finalizers()
+
         #
         # Walk all rawmalloced objects and free the ones that don't
         # have the GCFLAG_VISITED flag.
@@ -1647,8 +1659,7 @@
         if self.header(obj).tid & GCFLAG_VISITED:
             self.header(obj).tid &= ~GCFLAG_VISITED
             return False     # survives
-        else:
-            return True      # dies
+        return True      # dies
 
     def _reset_gcflag_visited(self, obj, ignored):
         self.header(obj).tid &= ~GCFLAG_VISITED
@@ -1827,6 +1838,39 @@
     # ----------
     # Finalizers
 
+    def deal_with_young_objects_with_finalizers(self):
+        """ This is a much simpler version of dealing with finalizers
+        and an optimization - we can reasonably assume that those finalizers
+        don't do anything fancy and *just* call them. Among other things
+        they won't resurrect objects
+        """
+        while self.young_objects_with_light_finalizers.non_empty():
+            obj = self.young_objects_with_light_finalizers.pop()
+            if not self.is_forwarded(obj):
+                finalizer = self.getlightfinalizer(self.get_type_id(obj))
+                ll_assert(bool(finalizer), "no light finalizer found")
+                finalizer(obj, llmemory.NULL)
+
+    def deal_with_old_objects_with_finalizers(self):
+        """ This is a much simpler version of dealing with finalizers
+        and an optimization - we can reasonably assume that those finalizers
+        don't do anything fancy and *just* call them. Among other things
+        they won't resurrect objects
+        """
+        new_objects = self.AddressStack()
+        while self.old_objects_with_light_finalizers.non_empty():
+            obj = self.old_objects_with_light_finalizers.pop()
+            if self.header(obj).tid & GCFLAG_VISITED:
+                # surviving
+                new_objects.append(obj)
+            else:
+                # dying
+                finalizer = self.getlightfinalizer(self.get_type_id(obj))
+                ll_assert(bool(finalizer), "no light finalizer found")
+                finalizer(obj, llmemory.NULL)
+        self.old_objects_with_light_finalizers.delete()
+        self.old_objects_with_light_finalizers = new_objects
+
     def deal_with_objects_with_finalizers(self):
         # Walk over list of objects with finalizers.
         # If it is not surviving, add it to the list of to-be-called
@@ -1957,7 +2001,6 @@
             #
             self.old_objects_with_weakrefs.append(obj)
 
-
     def invalidate_old_weakrefs(self):
         """Called during a major collection."""
         # walk over list of objects that contain weakrefs
diff --git a/pypy/rpython/memory/gc/semispace.py b/pypy/rpython/memory/gc/semispace.py
--- a/pypy/rpython/memory/gc/semispace.py
+++ b/pypy/rpython/memory/gc/semispace.py
@@ -82,6 +82,7 @@
         self.free = self.tospace
         MovingGCBase.setup(self)
         self.objects_with_finalizers = self.AddressDeque()
+        self.objects_with_light_finalizers = self.AddressStack()
         self.objects_with_weakrefs = self.AddressStack()
 
     def _teardown(self):
@@ -93,7 +94,9 @@
     # because the spaces are filled with zeroes in advance.
 
     def malloc_fixedsize_clear(self, typeid16, size,
-                               has_finalizer=False, contains_weakptr=False):
+                               has_finalizer=False,
+                               is_finalizer_light=False,
+                               contains_weakptr=False):
         size_gc_header = self.gcheaderbuilder.size_gc_header
         totalsize = size_gc_header + size
         result = self.free
@@ -102,7 +105,9 @@
         llarena.arena_reserve(result, totalsize)
         self.init_gc_object(result, typeid16)
         self.free = result + totalsize
-        if has_finalizer:
+        if is_finalizer_light:
+            self.objects_with_light_finalizers.append(result + size_gc_header)
+        elif has_finalizer:
             self.objects_with_finalizers.append(result + size_gc_header)
         if contains_weakptr:
             self.objects_with_weakrefs.append(result + size_gc_header)
@@ -263,6 +268,8 @@
         if self.run_finalizers.non_empty():
             self.update_run_finalizers()
         scan = self.scan_copied(scan)
+        if self.objects_with_light_finalizers.non_empty():
+            self.deal_with_objects_with_light_finalizers()
         if self.objects_with_finalizers.non_empty():
             scan = self.deal_with_objects_with_finalizers(scan)
         if self.objects_with_weakrefs.non_empty():
@@ -471,6 +478,23 @@
         # immortal objects always have GCFLAG_FORWARDED set;
         # see get_forwarding_address().
 
+    def deal_with_objects_with_light_finalizers(self):
+        """ This is a much simpler version of dealing with finalizers
+        and an optimization - we can reasonably assume that those finalizers
+        don't do anything fancy and *just* call them. Among other things
+        they won't resurrect objects
+        """
+        new_objects = self.AddressStack()
+        while self.objects_with_light_finalizers.non_empty():
+            obj = self.objects_with_light_finalizers.pop()
+            if self.surviving(obj):
+                new_objects.append(self.get_forwarding_address(obj))
+            else:
+                finalizer = self.getfinalizer(self.get_type_id(obj))
+                finalizer(obj, llmemory.NULL)
+        self.objects_with_light_finalizers.delete()
+        self.objects_with_light_finalizers = new_objects
+
     def deal_with_objects_with_finalizers(self, scan):
         # walk over list of objects with finalizers
         # if it is not copied, add it to the list of to-be-called finalizers
diff --git a/pypy/rpython/memory/gctransform/framework.py b/pypy/rpython/memory/gctransform/framework.py
--- a/pypy/rpython/memory/gctransform/framework.py
+++ b/pypy/rpython/memory/gctransform/framework.py
@@ -12,6 +12,7 @@
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.translator.backendopt import graphanalyze
 from pypy.translator.backendopt.support import var_needsgc
+from pypy.translator.backendopt.finalizer import FinalizerAnalyzer
 from pypy.annotation import model as annmodel
 from pypy.rpython import annlowlevel
 from pypy.rpython.rbuiltin import gen_cast
@@ -258,6 +259,7 @@
             [s_gc, s_typeid16,
              annmodel.SomeInteger(nonneg=True),
              annmodel.SomeBool(),
+             annmodel.SomeBool(),
              annmodel.SomeBool()], s_gcref,
             inline = False)
         if hasattr(GCClass, 'malloc_fixedsize'):
@@ -267,6 +269,7 @@
                 [s_gc, s_typeid16,
                  annmodel.SomeInteger(nonneg=True),
                  annmodel.SomeBool(),
+                 annmodel.SomeBool(),
                  annmodel.SomeBool()], s_gcref,
                 inline = False)
         else:
@@ -319,7 +322,7 @@
             raise NotImplementedError("GC needs write barrier, but does not provide writebarrier_before_copy functionality")
 
         # in some GCs we can inline the common case of
-        # malloc_fixedsize(typeid, size, True, False, False)
+        # malloc_fixedsize(typeid, size, False, False, False)
         if getattr(GCClass, 'inline_simple_malloc', False):
             # make a copy of this function so that it gets annotated
             # independently and the constants are folded inside
@@ -337,7 +340,7 @@
                 malloc_fast,
                 [s_gc, s_typeid16,
                  annmodel.SomeInteger(nonneg=True),
-                 s_False, s_False], s_gcref,
+                 s_False, s_False, s_False], s_gcref,
                 inline = True)
         else:
             self.malloc_fast_ptr = None
@@ -668,7 +671,13 @@
         kind_and_fptr = self.special_funcptr_for_type(TYPE)
         has_finalizer = (kind_and_fptr is not None and
                          kind_and_fptr[0] == "finalizer")
+        has_light_finalizer = (kind_and_fptr is not None and
+                               kind_and_fptr[0] == "light_finalizer")
+        if has_light_finalizer:
+            has_finalizer = True
         c_has_finalizer = rmodel.inputconst(lltype.Bool, has_finalizer)
+        c_has_light_finalizer = rmodel.inputconst(lltype.Bool,
+                                                  has_light_finalizer)
 
         if not op.opname.endswith('_varsize') and not flags.get('varsize'):
             #malloc_ptr = self.malloc_fixedsize_ptr
@@ -682,7 +691,8 @@
             else:
                 malloc_ptr = self.malloc_fixedsize_ptr
             args = [self.c_const_gc, c_type_id, c_size,
-                    c_has_finalizer, rmodel.inputconst(lltype.Bool, False)]
+                    c_has_finalizer, c_has_light_finalizer,
+                    rmodel.inputconst(lltype.Bool, False)]
         else:
             assert not c_has_finalizer.value
             info_varsize = self.layoutbuilder.get_info_varsize(type_id)
@@ -847,12 +857,13 @@
         # used by the JIT (see pypy.jit.backend.llsupport.gc)
         op = hop.spaceop
         [v_typeid, v_size,
-         v_has_finalizer, v_contains_weakptr] = op.args
+         v_has_finalizer, v_has_light_finalizer, v_contains_weakptr] = op.args
         livevars = self.push_roots(hop)
         hop.genop("direct_call",
                   [self.malloc_fixedsize_clear_ptr, self.c_const_gc,
                    v_typeid, v_size,
-                   v_has_finalizer, v_contains_weakptr],
+                   v_has_finalizer, v_has_light_finalizer,
+                   v_contains_weakptr],
                   resultvar=op.result)
         self.pop_roots(hop, livevars)
 
@@ -912,10 +923,10 @@
         info = self.layoutbuilder.get_info(type_id)
         c_size = rmodel.inputconst(lltype.Signed, info.fixedsize)
         malloc_ptr = self.malloc_fixedsize_ptr
-        c_has_finalizer = rmodel.inputconst(lltype.Bool, False)
+        c_false = rmodel.inputconst(lltype.Bool, False)
         c_has_weakptr = rmodel.inputconst(lltype.Bool, True)
         args = [self.c_const_gc, c_type_id, c_size,
-                c_has_finalizer, c_has_weakptr]
+                c_false, c_false, c_has_weakptr]
 
         # push and pop the current live variables *including* the argument
         # to the weakref_create operation, which must be kept alive and
@@ -1250,6 +1261,7 @@
             lltype2vtable = translator.rtyper.lltype2vtable

From noreply at buildbot.pypy.org  Fri Oct 28 17:44:44 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Fri, 28 Oct 2011 17:44:44 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-complex: more tests for array of structs
Message-ID: <20111028154444.E88DF820C2@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: numpy-complex
Changeset: r48569:0c5959cc2de4
Date: 2011-10-28 11:44 -0400
http://bitbucket.org/pypy/pypy/changeset/0c5959cc2de4/

Log:	more tests for array of structs

diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -976,6 +976,29 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_virtual_array_of_struct_forced(self):
+        ops = """
+        [f0, f1]
+        p0 = new_array(1, descr=complexarraydescr)
+        setinteriorfield_gc(p0, 0, f0, descr=complexrealdescr)
+        setinteriorfield_gc(p0, 0, f1, descr=compleximagdescr)
+        f2 = getinteriorfield_gc(p0, 0, descr=complexrealdescr)
+        f3 = getinteriorfield_gc(p0, 0, descr=compleximagdescr)
+        f4 = float_mul(f2, f3)
+        i0 = escape(f4, p0)
+        finish(i0)
+        """
+        expected = """
+        [f0, f1]
+        f2 = float_mul(f0, f1)
+        p0 = new_array(1, descr=complexarraydescr)
+        setinteriorfield_gc(p0, 0, f0, descr=complexrealdescr)
+        setinteriorfield_gc(p0, 0, f1, descr=compleximagdescr)
+        i0 = escape(f2, p0)
+        finish(i0)
+        """
+        self.optimize_loop(ops, expected)
+
     def test_nonvirtual_1(self):
         ops = """
         [i]
diff --git a/pypy/module/pypyjit/test_pypy_c/test_containers.py b/pypy/module/pypyjit/test_pypy_c/test_containers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_containers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_containers.py
@@ -69,4 +69,51 @@
             i9 = int_add(i5, 1)
             --TICK--
             jump(..., descr=...)
+        """)
+
+    def test_non_virtual_dict(self):
+        def main(n):
+            i = 0
+            while i < n:
+                d = {str(i): i}
+                i += d[str(i)] - i + 1
+            return i
+
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loop, = log.loops_by_filename(self.filepath)
+        assert loop.match("""
+            i8 = int_lt(i5, i7)
+            guard_true(i8, descr=...)
+            guard_not_invalidated(descr=...)
+            p10 = call(ConstClass(ll_int_str), i5, descr=<GcPtrCallDescr>)
+            guard_no_exception(descr=...)
+            i12 = call(ConstClass(ll_strhash), p10, descr=<SignedCallDescr>)
+            p13 = new(descr=...)
+            p15 = new_array(8, descr=<dictentryArrayDescr>)
+            setfield_gc(p13, p15, descr=<GcPtrFieldDescr dicttable.entries .*>)
+            i17 = call(ConstClass(ll_dict_lookup_trampoline), p13, p10, i12, descr=<SignedCallDescr>)
+            setfield_gc(p13, 16, descr=<SignedFieldDescr dicttable.resize_counter 16>)
+            guard_no_exception(descr=...)
+            p20 = new_with_vtable(ConstClass(W_IntObject))
+            call(ConstClass(_ll_dict_setitem_lookup_done_trampoline), p13, p10, p20, i12, i17, descr=<VoidCallDescr>)
+            setfield_gc(p20, i5, descr=<SignedFieldDescr .*W_IntObject.inst_intval .*>)
+            guard_no_exception(descr=...)
+            i23 = call(ConstClass(ll_dict_lookup_trampoline), p13, p10, i12, descr=<SignedCallDescr>)
+            guard_no_exception(descr=...)
+            i26 = int_and(i23, .*)
+            i27 = int_is_true(i26)
+            guard_false(i27, descr=...)
+            p28 = getfield_gc(p13, descr=<GcPtrFieldDescr dicttable.entries .*>)
+            p29 = getinteriorfield_gc(p28, i23, descr=<InteriorFieldDescr <GcPtrFieldDescr dictentry.value .*>>)
+            guard_nonnull_class(p29, ConstClass(W_IntObject), descr=...)
+            i31 = getfield_gc_pure(p29, descr=<SignedFieldDescr .*W_IntObject.inst_intval .*>)
+            i32 = int_sub_ovf(i31, i5)
+            guard_no_overflow(descr=...)
+            i34 = int_add_ovf(i32, 1)
+            guard_no_overflow(descr=...)
+            i35 = int_add_ovf(i5, i34)
+            guard_no_overflow(descr=...)
+            --TICK--
+            jump(p0, p1, p2, p3, p4, i35, p13, i7, descr=<Loop0>)
         """)
\ No newline at end of file

From noreply at buildbot.pypy.org  Fri Oct 28 17:45:53 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Fri, 28 Oct 2011 17:45:53 +0200 (CEST)
Subject: [pypy-commit] pypy default: more tests for array of structs
Message-ID: <20111028154553.4C3AB820C2@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r48570:11d7ea62944e
Date: 2011-10-28 11:44 -0400
http://bitbucket.org/pypy/pypy/changeset/11d7ea62944e/

Log:	more tests for array of structs

diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -976,6 +976,29 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_virtual_array_of_struct_forced(self):
+        ops = """
+        [f0, f1]
+        p0 = new_array(1, descr=complexarraydescr)
+        setinteriorfield_gc(p0, 0, f0, descr=complexrealdescr)
+        setinteriorfield_gc(p0, 0, f1, descr=compleximagdescr)
+        f2 = getinteriorfield_gc(p0, 0, descr=complexrealdescr)
+        f3 = getinteriorfield_gc(p0, 0, descr=compleximagdescr)
+        f4 = float_mul(f2, f3)
+        i0 = escape(f4, p0)
+        finish(i0)
+        """
+        expected = """
+        [f0, f1]
+        f2 = float_mul(f0, f1)
+        p0 = new_array(1, descr=complexarraydescr)
+        setinteriorfield_gc(p0, 0, f0, descr=complexrealdescr)
+        setinteriorfield_gc(p0, 0, f1, descr=compleximagdescr)
+        i0 = escape(f2, p0)
+        finish(i0)
+        """
+        self.optimize_loop(ops, expected)
+
     def test_nonvirtual_1(self):
         ops = """
         [i]
diff --git a/pypy/module/pypyjit/test_pypy_c/test_containers.py b/pypy/module/pypyjit/test_pypy_c/test_containers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_containers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_containers.py
@@ -69,4 +69,51 @@
             i9 = int_add(i5, 1)
             --TICK--
             jump(..., descr=...)
+        """)
+
+    def test_non_virtual_dict(self):
+        def main(n):
+            i = 0
+            while i < n:
+                d = {str(i): i}
+                i += d[str(i)] - i + 1
+            return i
+
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loop, = log.loops_by_filename(self.filepath)
+        assert loop.match("""
+            i8 = int_lt(i5, i7)
+            guard_true(i8, descr=...)
+            guard_not_invalidated(descr=...)
+            p10 = call(ConstClass(ll_int_str), i5, descr=<GcPtrCallDescr>)
+            guard_no_exception(descr=...)
+            i12 = call(ConstClass(ll_strhash), p10, descr=<SignedCallDescr>)
+            p13 = new(descr=...)
+            p15 = new_array(8, descr=<dictentryArrayDescr>)
+            setfield_gc(p13, p15, descr=<GcPtrFieldDescr dicttable.entries .*>)
+            i17 = call(ConstClass(ll_dict_lookup_trampoline), p13, p10, i12, descr=<SignedCallDescr>)
+            setfield_gc(p13, 16, descr=<SignedFieldDescr dicttable.resize_counter 16>)
+            guard_no_exception(descr=...)
+            p20 = new_with_vtable(ConstClass(W_IntObject))
+            call(ConstClass(_ll_dict_setitem_lookup_done_trampoline), p13, p10, p20, i12, i17, descr=<VoidCallDescr>)
+            setfield_gc(p20, i5, descr=<SignedFieldDescr .*W_IntObject.inst_intval .*>)
+            guard_no_exception(descr=...)
+            i23 = call(ConstClass(ll_dict_lookup_trampoline), p13, p10, i12, descr=<SignedCallDescr>)
+            guard_no_exception(descr=...)
+            i26 = int_and(i23, .*)
+            i27 = int_is_true(i26)
+            guard_false(i27, descr=...)
+            p28 = getfield_gc(p13, descr=<GcPtrFieldDescr dicttable.entries .*>)
+            p29 = getinteriorfield_gc(p28, i23, descr=<InteriorFieldDescr <GcPtrFieldDescr dictentry.value .*>>)
+            guard_nonnull_class(p29, ConstClass(W_IntObject), descr=...)
+            i31 = getfield_gc_pure(p29, descr=<SignedFieldDescr .*W_IntObject.inst_intval .*>)
+            i32 = int_sub_ovf(i31, i5)
+            guard_no_overflow(descr=...)
+            i34 = int_add_ovf(i32, 1)
+            guard_no_overflow(descr=...)
+            i35 = int_add_ovf(i5, i34)
+            guard_no_overflow(descr=...)
+            --TICK--
+            jump(p0, p1, p2, p3, p4, i35, p13, i7, descr=<Loop0>)
         """)
\ No newline at end of file

From noreply at buildbot.pypy.org  Fri Oct 28 17:45:54 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Fri, 28 Oct 2011 17:45:54 +0200 (CEST)
Subject: [pypy-commit] pypy default: merged upstream
Message-ID: <20111028154554.8751D820C2@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r48571:d2376dc9d29a
Date: 2011-10-28 11:45 -0400
http://bitbucket.org/pypy/pypy/changeset/d2376dc9d29a/

Log:	merged upstream

diff --git a/pypy/jit/metainterp/history.py b/pypy/jit/metainterp/history.py
--- a/pypy/jit/metainterp/history.py
+++ b/pypy/jit/metainterp/history.py
@@ -929,6 +929,9 @@
     def view(self, **kwds):
         pass
 
+    def clear(self):
+        pass
+
 class Stats(object):
     """For tests."""
 
@@ -943,6 +946,15 @@
         self.aborted_keys = []
         self.invalidated_token_numbers = set()
 
+    def clear(self):
+        del self.loops[:]
+        del self.locations[:]
+        del self.aborted_keys[:]
+        self.invalidated_token_numbers.clear()
+        self.compiled_count = 0
+        self.enter_count = 0
+        self.aborted_count = 0
+
     def set_history(self, history):
         self.operations = history.operations
 
diff --git a/pypy/jit/metainterp/warmspot.py b/pypy/jit/metainterp/warmspot.py
--- a/pypy/jit/metainterp/warmspot.py
+++ b/pypy/jit/metainterp/warmspot.py
@@ -62,7 +62,7 @@
     clear_tcache()
     return jittify_and_run(interp, graph, args, backendopt=backendopt, **kwds)
 
-def jittify_and_run(interp, graph, args, repeat=1,
+def jittify_and_run(interp, graph, args, repeat=1, graph_and_interp_only=False,
                     backendopt=False, trace_limit=sys.maxint,
                     inline=False, loop_longevity=0, retrace_limit=5,
                     function_threshold=4,
@@ -93,6 +93,8 @@
         jd.warmstate.set_param_max_retrace_guards(max_retrace_guards)
         jd.warmstate.set_param_enable_opts(enable_opts)
     warmrunnerdesc.finish()
+    if graph_and_interp_only:
+        return interp, graph
     res = interp.eval_graph(graph, args)
     if not kwds.get('translate_support_code', False):
         warmrunnerdesc.metainterp_sd.profiler.finish()
@@ -157,6 +159,9 @@
 def get_stats():
     return pyjitpl._warmrunnerdesc.stats
 
+def reset_stats():
+    pyjitpl._warmrunnerdesc.stats.clear()
+
 def get_translator():
     return pyjitpl._warmrunnerdesc.translator
 
diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -4,30 +4,52 @@
 """
 
 from pypy.interpreter.baseobjspace import InternalSpaceCache, W_Root
-from pypy.module.micronumpy.interp_dtype import W_Float64Dtype
-from pypy.module.micronumpy.interp_numarray import Scalar, SingleDimArray, BaseArray
+from pypy.module.micronumpy.interp_dtype import W_Float64Dtype, W_BoolDtype
+from pypy.module.micronumpy.interp_numarray import (Scalar, BaseArray,
+     descr_new_array, scalar_w, SingleDimArray)
+from pypy.module.micronumpy import interp_ufuncs
 from pypy.rlib.objectmodel import specialize
 
 
 class BogusBytecode(Exception):
     pass
 
-def create_array(dtype, size):
-    a = SingleDimArray(size, dtype=dtype)
-    for i in range(size):
-        dtype.setitem(a.storage, i, dtype.box(float(i % 10)))
-    return a
+class ArgumentMismatch(Exception):
+    pass
+
+class ArgumentNotAnArray(Exception):
+    pass
+
+class WrongFunctionName(Exception):
+    pass
+
+SINGLE_ARG_FUNCTIONS = ["sum", "prod", "max", "min", "all", "any", "unegative"]
 
 class FakeSpace(object):
     w_ValueError = None
     w_TypeError = None
+    w_None = None
+
+    w_bool = "bool"
+    w_int = "int"
+    w_float = "float"
+    w_list = "list"
+    w_long = "long"
+    w_tuple = 'tuple'
 
     def __init__(self):
         """NOT_RPYTHON"""
         self.fromcache = InternalSpaceCache(self).getorbuild
+        self.w_float64dtype = W_Float64Dtype(self)
 
     def issequence_w(self, w_obj):
-        return True
+        return isinstance(w_obj, ListObject) or isinstance(w_obj, SingleDimArray)
+
+    def isinstance_w(self, w_obj, w_tp):
+        return False
+
+    def decode_index4(self, w_idx, size):
+        return (self.int_w(w_idx), 0, 0, 1)
 
     @specialize.argtype(1)
     def wrap(self, obj):
@@ -39,72 +61,382 @@
             return IntObject(obj)
         raise Exception
 
+    def newlist(self, items):
+        return ListObject(items)
+
+    def listview(self, obj):
+        assert isinstance(obj, ListObject)
+        return obj.items
+
     def float(self, w_obj):
         assert isinstance(w_obj, FloatObject)
         return w_obj
 
     def float_w(self, w_obj):
+        assert isinstance(w_obj, FloatObject)        
         return w_obj.floatval
 
+    def int_w(self, w_obj):
+        if isinstance(w_obj, IntObject):
+            return w_obj.intval
+        elif isinstance(w_obj, FloatObject):
+            return int(w_obj.floatval)
+        raise NotImplementedError
+
+    def int(self, w_obj):
+        return w_obj
+
+    def is_true(self, w_obj):
+        assert isinstance(w_obj, BoolObject)
+        return w_obj.boolval
+
+    def is_w(self, w_obj, w_what):
+        return w_obj is w_what
+
+    def type(self, w_obj):
+        return w_obj.tp
+
+    def gettypefor(self, w_obj):
+        return None
+
+    def call_function(self, tp, w_dtype):
+        return w_dtype
+
+    @specialize.arg(1)
+    def interp_w(self, tp, what):
+        assert isinstance(what, tp)
+        return what
 
 class FloatObject(W_Root):
+    tp = FakeSpace.w_float
     def __init__(self, floatval):
         self.floatval = floatval
 
 class BoolObject(W_Root):
+    tp = FakeSpace.w_bool
     def __init__(self, boolval):
         self.boolval = boolval
 
 class IntObject(W_Root):
+    tp = FakeSpace.w_int
     def __init__(self, intval):
         self.intval = intval
 
+class ListObject(W_Root):
+    tp = FakeSpace.w_list
+    def __init__(self, items):
+        self.items = items
 
-space = FakeSpace()
+class InterpreterState(object):
+    def __init__(self, code):
+        self.code = code
+        self.variables = {}
+        self.results = []
 
-def numpy_compile(bytecode, array_size):
-    stack = []
-    i = 0
-    dtype = space.fromcache(W_Float64Dtype)
-    for b in bytecode:
-        if b == 'a':
-            stack.append(create_array(dtype, array_size))
-            i += 1
-        elif b == 'f':
-            stack.append(Scalar(dtype, dtype.box(1.2)))
-        elif b == '+':
-            right = stack.pop()
-            res = stack.pop().descr_add(space, right)
-            assert isinstance(res, BaseArray)
-            stack.append(res)
-        elif b == '-':
-            right = stack.pop()
-            res = stack.pop().descr_sub(space, right)
-            assert isinstance(res, BaseArray)
-            stack.append(res)
-        elif b == '*':
-            right = stack.pop()
-            res = stack.pop().descr_mul(space, right)
-            assert isinstance(res, BaseArray)
-            stack.append(res)
-        elif b == '/':
-            right = stack.pop()
-            res = stack.pop().descr_div(space, right)
-            assert isinstance(res, BaseArray)
-            stack.append(res)
-        elif b == '%':
-            right = stack.pop()
-            res = stack.pop().descr_mod(space, right)
-            assert isinstance(res, BaseArray)
-            stack.append(res)
-        elif b == '|':
-            res = stack.pop().descr_abs(space)
-            assert isinstance(res, BaseArray)
-            stack.append(res)
+    def run(self, space):
+        self.space = space
+        for stmt in self.code.statements:
+            stmt.execute(self)
+
+class Node(object):
+    def __eq__(self, other):
+        return (self.__class__ == other.__class__ and
+                self.__dict__ == other.__dict__)
+
+    def __ne__(self, other):
+        return not self == other
+
+    def wrap(self, space):
+        raise NotImplementedError
+
+    def execute(self, interp):
+        raise NotImplementedError
+
+class Assignment(Node):
+    def __init__(self, name, expr):
+        self.name = name
+        self.expr = expr
+
+    def execute(self, interp):
+        interp.variables[self.name] = self.expr.execute(interp)
+
+    def __repr__(self):
+        return "%% = %r" % (self.name, self.expr)
+
+class ArrayAssignment(Node):
+    def __init__(self, name, index, expr):
+        self.name = name
+        self.index = index
+        self.expr = expr
+
+    def execute(self, interp):
+        arr = interp.variables[self.name]
+        w_index = self.index.execute(interp).eval(0).wrap(interp.space)
+        w_val = self.expr.execute(interp).eval(0).wrap(interp.space)
+        arr.descr_setitem(interp.space, w_index, w_val)
+
+    def __repr__(self):
+        return "%s[%r] = %r" % (self.name, self.index, self.expr)
+
+class Variable(Node):
+    def __init__(self, name):
+        self.name = name
+
+    def execute(self, interp):
+        return interp.variables[self.name]
+
+    def __repr__(self):
+        return 'v(%s)' % self.name
+
+class Operator(Node):
+    def __init__(self, lhs, name, rhs):
+        self.name = name
+        self.lhs = lhs
+        self.rhs = rhs
+
+    def execute(self, interp):
+        w_lhs = self.lhs.execute(interp)
+        assert isinstance(w_lhs, BaseArray)
+        if isinstance(self.rhs, SliceConstant):
+            # XXX interface has changed on multidim branch
+            raise NotImplementedError
+        w_rhs = self.rhs.execute(interp)
+        if self.name == '+':
+            w_res = w_lhs.descr_add(interp.space, w_rhs)
+        elif self.name == '*':
+            w_res = w_lhs.descr_mul(interp.space, w_rhs)
+        elif self.name == '-':
+            w_res = w_lhs.descr_sub(interp.space, w_rhs)            
+        elif self.name == '->':
+            if isinstance(w_rhs, Scalar):
+                index = int(interp.space.float_w(
+                    w_rhs.value.wrap(interp.space)))
+                dtype = interp.space.fromcache(W_Float64Dtype)
+                return Scalar(dtype, w_lhs.get_concrete().eval(index))
+            else:
+                raise NotImplementedError
         else:
-            print "Unknown opcode: %s" % b
-            raise BogusBytecode()
-    if len(stack) != 1:
-        print "Bogus bytecode, uneven stack length"
-        raise BogusBytecode()
-    return stack[0]
+            raise NotImplementedError
+        if not isinstance(w_res, BaseArray):
+            dtype = interp.space.fromcache(W_Float64Dtype)
+            w_res = scalar_w(interp.space, dtype, w_res)
+        return w_res
+
+    def __repr__(self):
+        return '(%r %s %r)' % (self.lhs, self.name, self.rhs)
+
+class FloatConstant(Node):
+    def __init__(self, v):
+        self.v = float(v)
+
+    def __repr__(self):
+        return "Const(%s)" % self.v
+
+    def wrap(self, space):
+        return space.wrap(self.v)
+
+    def execute(self, interp):
+        dtype = interp.space.fromcache(W_Float64Dtype)
+        assert isinstance(dtype, W_Float64Dtype)
+        return Scalar(dtype, dtype.box(self.v))
+
+class RangeConstant(Node):
+    def __init__(self, v):
+        self.v = int(v)
+
+    def execute(self, interp):
+        w_list = interp.space.newlist(
+            [interp.space.wrap(float(i)) for i in range(self.v)])
+        dtype = interp.space.fromcache(W_Float64Dtype)
+        return descr_new_array(interp.space, None, w_list, w_dtype=dtype)
+
+    def __repr__(self):
+        return 'Range(%s)' % self.v
+
+class Code(Node):
+    def __init__(self, statements):
+        self.statements = statements
+
+    def __repr__(self):
+        return "\n".join([repr(i) for i in self.statements])
+
+class ArrayConstant(Node):
+    def __init__(self, items):
+        self.items = items
+
+    def wrap(self, space):
+        return space.newlist([item.wrap(space) for item in self.items])
+
+    def execute(self, interp):
+        w_list = self.wrap(interp.space)
+        dtype = interp.space.fromcache(W_Float64Dtype)
+        return descr_new_array(interp.space, None, w_list, w_dtype=dtype)
+
+    def __repr__(self):
+        return "[" + ", ".join([repr(item) for item in self.items]) + "]"
+
+class SliceConstant(Node):
+    def __init__(self):
+        pass
+
+    def __repr__(self):
+        return 'slice()'
+
+class Execute(Node):
+    def __init__(self, expr):
+        self.expr = expr
+
+    def __repr__(self):
+        return repr(self.expr)
+
+    def execute(self, interp):
+        interp.results.append(self.expr.execute(interp))
+
+class FunctionCall(Node):
+    def __init__(self, name, args):
+        self.name = name
+        self.args = args
+
+    def __repr__(self):
+        return "%s(%s)" % (self.name, ", ".join([repr(arg)
+                                                 for arg in self.args]))
+
+    def execute(self, interp):
+        if self.name in SINGLE_ARG_FUNCTIONS:
+            if len(self.args) != 1:
+                raise ArgumentMismatch
+            arr = self.args[0].execute(interp)
+            if not isinstance(arr, BaseArray):
+                raise ArgumentNotAnArray
+            if self.name == "sum":
+                w_res = arr.descr_sum(interp.space)
+            elif self.name == "prod":
+                w_res = arr.descr_prod(interp.space)
+            elif self.name == "max":
+                w_res = arr.descr_max(interp.space)
+            elif self.name == "min":
+                w_res = arr.descr_min(interp.space)
+            elif self.name == "any":
+                w_res = arr.descr_any(interp.space)
+            elif self.name == "all":
+                w_res = arr.descr_all(interp.space)
+            elif self.name == "unegative":
+                neg = interp_ufuncs.get(interp.space).negative
+                w_res = neg.call(interp.space, [arr])
+            else:
+                assert False # unreachable code
+            if isinstance(w_res, BaseArray):
+                return w_res
+            if isinstance(w_res, FloatObject):
+                dtype = interp.space.fromcache(W_Float64Dtype)
+            elif isinstance(w_res, BoolObject):
+                dtype = interp.space.fromcache(W_BoolDtype)
+            else:
+                dtype = None
+            return scalar_w(interp.space, dtype, w_res)
+        else:
+            raise WrongFunctionName
+
+class Parser(object):
+    def parse_identifier(self, id):
+        id = id.strip(" ")
+        #assert id.isalpha()
+        return Variable(id)
+
+    def parse_expression(self, expr):
+        tokens = [i for i in expr.split(" ") if i]
+        if len(tokens) == 1:
+            return self.parse_constant_or_identifier(tokens[0])
+        stack = []
+        tokens.reverse()
+        while tokens:
+            token = tokens.pop()
+            if token == ')':
+                raise NotImplementedError
+            elif self.is_identifier_or_const(token):
+                if stack:
+                    name = stack.pop().name
+                    lhs = stack.pop()
+                    rhs = self.parse_constant_or_identifier(token)
+                    stack.append(Operator(lhs, name, rhs))
+                else:
+                    stack.append(self.parse_constant_or_identifier(token))
+            else:
+                stack.append(Variable(token))
+        assert len(stack) == 1
+        return stack[-1]
+
+    def parse_constant(self, v):
+        lgt = len(v)-1
+        assert lgt >= 0
+        if ':' in v:
+            # a slice
+            assert v == ':'
+            return SliceConstant()
+        if v[0] == '[':
+            return ArrayConstant([self.parse_constant(elem)
+                                  for elem in v[1:lgt].split(",")])
+        if v[0] == '|':
+            return RangeConstant(v[1:lgt])
+        return FloatConstant(v)
+
+    def is_identifier_or_const(self, v):
+        c = v[0]
+        if ((c >= 'a' and c <= 'z') or (c >= 'A' and c <= 'Z') or
+            (c >= '0' and c <= '9') or c in '-.[|:'):
+            if v == '-' or v == "->":
+                return False
+            return True
+        return False
+
+    def parse_function_call(self, v):
+        l = v.split('(')
+        assert len(l) == 2
+        name = l[0]
+        cut = len(l[1]) - 1
+        assert cut >= 0
+        args = [self.parse_constant_or_identifier(id)
+                for id in l[1][:cut].split(",")]
+        return FunctionCall(name, args)
+
+    def parse_constant_or_identifier(self, v):
+        c = v[0]
+        if (c >= 'a' and c <= 'z') or (c >= 'A' and c <= 'Z'):
+            if '(' in v:
+                return self.parse_function_call(v)
+            return self.parse_identifier(v)
+        return self.parse_constant(v)
+
+    def parse_array_subscript(self, v):
+        v = v.strip(" ")
+        l = v.split("[")
+        lgt = len(l[1]) - 1
+        assert lgt >= 0
+        rhs = self.parse_constant_or_identifier(l[1][:lgt])
+        return l[0], rhs
+        
+    def parse_statement(self, line):
+        if '=' in line:
+            lhs, rhs = line.split("=")
+            lhs = lhs.strip(" ")
+            if '[' in lhs:
+                name, index = self.parse_array_subscript(lhs)
+                return ArrayAssignment(name, index, self.parse_expression(rhs))
+            else: 
+                return Assignment(lhs, self.parse_expression(rhs))
+        else:
+            return Execute(self.parse_expression(line))
+
+    def parse(self, code):
+        statements = []
+        for line in code.split("\n"):
+            if '#' in line:
+                line = line.split('#', 1)[0]
+            line = line.strip(" ")
+            if line:
+                statements.append(self.parse_statement(line))
+        return Code(statements)
+
+def numpy_compile(code):
+    parser = Parser()
+    return InterpreterState(parser.parse(code))
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -14,6 +14,27 @@
 any_driver = jit.JitDriver(greens=['signature'], reds=['i', 'size', 'self', 'dtype'])
 slice_driver = jit.JitDriver(greens=['signature'], reds=['i', 'j', 'step', 'stop', 'source', 'dest'])
 
+def descr_new_array(space, w_subtype, w_size_or_iterable, w_dtype=None):
+    l = space.listview(w_size_or_iterable)
+    if space.is_w(w_dtype, space.w_None):
+        w_dtype = None
+        for w_item in l:
+            w_dtype = interp_ufuncs.find_dtype_for_scalar(space, w_item, w_dtype)
+            if w_dtype is space.fromcache(interp_dtype.W_Float64Dtype):
+                break
+        if w_dtype is None:
+            w_dtype = space.w_None
+
+    dtype = space.interp_w(interp_dtype.W_Dtype,
+        space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype)
+    )
+    arr = SingleDimArray(len(l), dtype=dtype)
+    i = 0
+    for w_elem in l:
+        dtype.setitem_w(space, arr.storage, i, w_elem)
+        i += 1
+    return arr
+
 class BaseArray(Wrappable):
     _attrs_ = ["invalidates", "signature"]
 
@@ -32,27 +53,6 @@
     def add_invalidates(self, other):
         self.invalidates.append(other)
 
-    def descr__new__(space, w_subtype, w_size_or_iterable, w_dtype=None):
-        l = space.listview(w_size_or_iterable)
-        if space.is_w(w_dtype, space.w_None):
-            w_dtype = None
-            for w_item in l:
-                w_dtype = interp_ufuncs.find_dtype_for_scalar(space, w_item, w_dtype)
-                if w_dtype is space.fromcache(interp_dtype.W_Float64Dtype):
-                    break
-            if w_dtype is None:
-                w_dtype = space.w_None
-
-        dtype = space.interp_w(interp_dtype.W_Dtype,
-            space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype)
-        )
-        arr = SingleDimArray(len(l), dtype=dtype)
-        i = 0
-        for w_elem in l:
-            dtype.setitem_w(space, arr.storage, i, w_elem)
-            i += 1
-        return arr
-
     def _unaryop_impl(ufunc_name):
         def impl(self, space):
             return getattr(interp_ufuncs.get(space), ufunc_name).call(space, [self])
@@ -570,7 +570,7 @@
 
 BaseArray.typedef = TypeDef(
     'numarray',
-    __new__ = interp2app(BaseArray.descr__new__.im_func),
+    __new__ = interp2app(descr_new_array),
 
 
     __len__ = interp2app(BaseArray.descr_len),
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -236,22 +236,20 @@
     return dt
 
 def find_dtype_for_scalar(space, w_obj, current_guess=None):
-    w_type = space.type(w_obj)
-
     bool_dtype = space.fromcache(interp_dtype.W_BoolDtype)
     long_dtype = space.fromcache(interp_dtype.W_LongDtype)
     int64_dtype = space.fromcache(interp_dtype.W_Int64Dtype)
 
-    if space.is_w(w_type, space.w_bool):
+    if space.isinstance_w(w_obj, space.w_bool):
         if current_guess is None or current_guess is bool_dtype:
             return bool_dtype
         return current_guess
-    elif space.is_w(w_type, space.w_int):
+    elif space.isinstance_w(w_obj, space.w_int):
         if (current_guess is None or current_guess is bool_dtype or
             current_guess is long_dtype):
             return long_dtype
         return current_guess
-    elif space.is_w(w_type, space.w_long):
+    elif space.isinstance_w(w_obj, space.w_long):
         if (current_guess is None or current_guess is bool_dtype or
             current_guess is long_dtype or current_guess is int64_dtype):
             return int64_dtype
diff --git a/pypy/module/micronumpy/test/test_compile.py b/pypy/module/micronumpy/test/test_compile.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/test/test_compile.py
@@ -0,0 +1,170 @@
+
+import py
+from pypy.module.micronumpy.compile import *
+
+class TestCompiler(object):
+    def compile(self, code):
+        return numpy_compile(code)
+    
+    def test_vars(self):
+        code = """
+        a = 2
+        b = 3
+        """
+        interp = self.compile(code)
+        assert isinstance(interp.code.statements[0], Assignment)
+        assert interp.code.statements[0].name == 'a'
+        assert interp.code.statements[0].expr.v == 2
+        assert interp.code.statements[1].name == 'b'
+        assert interp.code.statements[1].expr.v == 3
+
+    def test_array_literal(self):
+        code = "a = [1,2,3]"
+        interp = self.compile(code)
+        assert isinstance(interp.code.statements[0].expr, ArrayConstant)
+        st = interp.code.statements[0]
+        assert st.expr.items == [FloatConstant(1), FloatConstant(2),
+                                 FloatConstant(3)]
+    
+    def test_array_literal2(self):
+        code = "a = [[1],[2],[3]]"
+        interp = self.compile(code)
+        assert isinstance(interp.code.statements[0].expr, ArrayConstant)
+        st = interp.code.statements[0]
+        assert st.expr.items == [ArrayConstant([FloatConstant(1)]),
+                                 ArrayConstant([FloatConstant(2)]),
+                                 ArrayConstant([FloatConstant(3)])]
+
+    def test_expr_1(self):
+        code = "b = a + 1"
+        interp = self.compile(code)
+        assert (interp.code.statements[0].expr ==
+                Operator(Variable("a"), "+", FloatConstant(1)))
+
+    def test_expr_2(self):
+        code = "b = a + b - 3"
+        interp = self.compile(code)
+        assert (interp.code.statements[0].expr ==
+                Operator(Operator(Variable("a"), "+", Variable("b")), "-",
+                         FloatConstant(3)))
+
+    def test_expr_3(self):
+        # an equivalent of range
+        code = "a = |20|"
+        interp = self.compile(code)
+        assert interp.code.statements[0].expr == RangeConstant(20)
+
+    def test_expr_only(self):
+        code = "3 + a"
+        interp = self.compile(code)
+        assert interp.code.statements[0] == Execute(
+            Operator(FloatConstant(3), "+", Variable("a")))
+
+    def test_array_access(self):
+        code = "a -> 3"
+        interp = self.compile(code)
+        assert interp.code.statements[0] == Execute(
+            Operator(Variable("a"), "->", FloatConstant(3)))
+
+    def test_function_call(self):
+        code = "sum(a)"
+        interp = self.compile(code)
+        assert interp.code.statements[0] == Execute(
+            FunctionCall("sum", [Variable("a")]))
+
+    def test_comment(self):
+        code = """
+        # some comment
+        a = b + 3  # another comment
+        """
+        interp = self.compile(code)
+        assert interp.code.statements[0] == Assignment(
+            'a', Operator(Variable('b'), "+", FloatConstant(3)))
+
+class TestRunner(object):
+    def run(self, code):
+        interp = numpy_compile(code)
+        space = FakeSpace()
+        interp.run(space)
+        return interp
+
+    def test_one(self):
+        code = """
+        a = 3
+        b = 4
+        a + b
+        """
+        interp = self.run(code)
+        assert sorted(interp.variables.keys()) == ['a', 'b']
+        assert interp.results[0]
+
+    def test_array_add(self):
+        code = """
+        a = [1,2,3,4]
+        b = [4,5,6,5]
+        a + b
+        """
+        interp = self.run(code)
+        assert interp.results[0]._getnums(False) == ["5.0", "7.0", "9.0", "9.0"]
+
+    def test_array_getitem(self):
+        code = """
+        a = [1,2,3,4]
+        b = [4,5,6,5]
+        a + b -> 3
+        """
+        interp = self.run(code)
+        assert interp.results[0].value.val == 3 + 6
+        
+    def test_range_getitem(self):
+        code = """
+        r = |20| + 3
+        r -> 3
+        """
+        interp = self.run(code)
+        assert interp.results[0].value.val == 6
+
+    def test_sum(self):
+        code = """
+        a = [1,2,3,4,5]
+        r = sum(a)
+        r
+        """
+        interp = self.run(code)
+        assert interp.results[0].value.val == 15
+
+    def test_array_write(self):
+        code = """
+        a = [1,2,3,4,5]
+        a[3] = 15
+        a -> 3
+        """
+        interp = self.run(code)
+        assert interp.results[0].value.val == 15
+
+    def test_min(self):
+        interp = self.run("""
+        a = |30|
+        a[15] = -12
+        b = a + a
+        min(b)
+        """)
+        assert interp.results[0].value.val == -24
+
+    def test_max(self):
+        interp = self.run("""
+        a = |30|
+        a[13] = 128
+        b = a + a
+        max(b)
+        """)
+        assert interp.results[0].value.val == 256
+
+    def test_slice(self):
+        py.test.skip("in progress")
+        interp = self.run("""
+        a = [1,2,3,4]
+        b = a -> :
+        b -> 3
+        """)
+        assert interp.results[0].value.val == 3
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -82,6 +82,8 @@
         b = negative(a)
         a[0] = 5.0
         assert b[0] == 5.0
+        a = array(range(30))
+        assert negative(a + a)[3] == -6
 
     def test_abs(self):
         from numpy import array, absolute
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -1,253 +1,195 @@
 from pypy.jit.metainterp.test.support import LLJitMixin
 from pypy.module.micronumpy import interp_ufuncs, signature
-from pypy.module.micronumpy.compile import (numpy_compile, FakeSpace,
-    FloatObject, IntObject)
-from pypy.module.micronumpy.interp_dtype import W_Int32Dtype, W_Float64Dtype, W_Int64Dtype, W_UInt64Dtype
-from pypy.module.micronumpy.interp_numarray import (BaseArray, SingleDimArray,
-    SingleDimSlice, scalar_w)
+from pypy.module.micronumpy.compile import (FakeSpace,
+    FloatObject, IntObject, numpy_compile, BoolObject)
+from pypy.module.micronumpy.interp_numarray import (SingleDimArray,
+    SingleDimSlice)
 from pypy.rlib.nonconst import NonConstant
-from pypy.rpython.annlowlevel import llstr
-from pypy.rpython.test.test_llinterp import interpret
+from pypy.rpython.annlowlevel import llstr, hlstr
+from pypy.jit.metainterp.warmspot import reset_stats
+from pypy.jit.metainterp import pyjitpl
 
 import py
 
 
 class TestNumpyJIt(LLJitMixin):
-    def setup_class(cls):
-        cls.space = FakeSpace()
-        cls.float64_dtype = cls.space.fromcache(W_Float64Dtype)
-        cls.int64_dtype = cls.space.fromcache(W_Int64Dtype)
-        cls.uint64_dtype = cls.space.fromcache(W_UInt64Dtype)
-        cls.int32_dtype = cls.space.fromcache(W_Int32Dtype)
+    graph = None
+    interp = None
+        
+    def run(self, code):
+        space = FakeSpace()
+        
+        def f(code):
+            interp = numpy_compile(hlstr(code))
+            interp.run(space)
+            res = interp.results[-1]
+            w_res = res.eval(0).wrap(interp.space)
+            if isinstance(w_res, BoolObject):
+                return float(w_res.boolval)
+            elif isinstance(w_res, FloatObject):
+                return w_res.floatval
+            elif isinstance(w_res, IntObject):
+                return w_res.intval
+            else:
+                return -42.
+
+        if self.graph is None:
+            interp, graph = self.meta_interp(f, [llstr(code)],
+                                             listops=True,
+                                             backendopt=True,
+                                             graph_and_interp_only=True)
+            self.__class__.interp = interp
+            self.__class__.graph = graph
+
+        reset_stats()
+        pyjitpl._warmrunnerdesc.memory_manager.alive_loops.clear()
+        return self.interp.eval_graph(self.graph, [llstr(code)])
 
     def test_add(self):
-        def f(i):
-            ar = SingleDimArray(i, dtype=self.float64_dtype)
-            v = interp_ufuncs.get(self.space).add.call(self.space, [ar, ar])
-            return v.get_concrete().eval(3).val
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
+        result = self.run("""
+        a = |30|
+        b = a + a
+        b -> 3
+        """)
         self.check_loops({'getarrayitem_raw': 2, 'float_add': 1,
                           'setarrayitem_raw': 1, 'int_add': 1,
                           'int_lt': 1, 'guard_true': 1, 'jump': 1})
-        assert result == f(5)
+        assert result == 3 + 3
 
     def test_floatadd(self):
-        def f(i):
-            ar = SingleDimArray(i, dtype=self.float64_dtype)
-            v = interp_ufuncs.get(self.space).add.call(self.space, [
-                    ar,
-                    scalar_w(self.space, self.float64_dtype, self.space.wrap(4.5))
-                ],
-            )
-            assert isinstance(v, BaseArray)
-            return v.get_concrete().eval(3).val
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
+        result = self.run("""
+        a = |30| + 3
+        a -> 3
+        """)
+        assert result == 3 + 3
         self.check_loops({"getarrayitem_raw": 1, "float_add": 1,
                           "setarrayitem_raw": 1, "int_add": 1,
                           "int_lt": 1, "guard_true": 1, "jump": 1})
-        assert result == f(5)
 
     def test_sum(self):
-        space = self.space
-        float64_dtype = self.float64_dtype
-        int64_dtype = self.int64_dtype
-
-        def f(i):
-            if NonConstant(False):
-                dtype = int64_dtype
-            else:
-                dtype = float64_dtype
-            ar = SingleDimArray(i, dtype=dtype)
-            v = ar.descr_add(space, ar).descr_sum(space)
-            assert isinstance(v, FloatObject)
-            return v.floatval
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
+        result = self.run("""
+        a = |30|
+        b = a + a
+        sum(b)
+        """)
+        assert result == 2 * sum(range(30))
         self.check_loops({"getarrayitem_raw": 2, "float_add": 2,
                           "int_add": 1,
                           "int_lt": 1, "guard_true": 1, "jump": 1})
-        assert result == f(5)
 
     def test_prod(self):
-        space = self.space
-        float64_dtype = self.float64_dtype
-        int64_dtype = self.int64_dtype
-
-        def f(i):
-            if NonConstant(False):
-                dtype = int64_dtype
-            else:
-                dtype = float64_dtype
-            ar = SingleDimArray(i, dtype=dtype)
-            v = ar.descr_add(space, ar).descr_prod(space)
-            assert isinstance(v, FloatObject)
-            return v.floatval
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
+        result = self.run("""
+        a = |30|
+        b = a + a
+        prod(b)
+        """)
+        expected = 1
+        for i in range(30):
+            expected *= i * 2
+        assert result == expected
         self.check_loops({"getarrayitem_raw": 2, "float_add": 1,
                           "float_mul": 1, "int_add": 1,
                           "int_lt": 1, "guard_true": 1, "jump": 1})
-        assert result == f(5)
 
     def test_max(self):
-        space = self.space
-        float64_dtype = self.float64_dtype
-        int64_dtype = self.int64_dtype
-
-        def f(i):
-            if NonConstant(False):
-                dtype = int64_dtype
-            else:
-                dtype = float64_dtype
-            ar = SingleDimArray(i, dtype=dtype)
-            j = 0
-            while j < i:
-                ar.get_concrete().setitem(j, float64_dtype.box(float(j)))
-                j += 1
-            v = ar.descr_add(space, ar).descr_max(space)
-            assert isinstance(v, FloatObject)
-            return v.floatval
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
+        py.test.skip("broken, investigate")
+        result = self.run("""
+        a = |30|
+        a[13] = 128
+        b = a + a
+        max(b)
+        """)
+        assert result == 256
         self.check_loops({"getarrayitem_raw": 2, "float_add": 1,
-                          "float_gt": 1, "int_add": 1,
-                          "int_lt": 1, "guard_true": 1,
-                          "guard_false": 1, "jump": 1})
-        assert result == f(5)
+                          "float_mul": 1, "int_add": 1,
+                          "int_lt": 1, "guard_true": 1, "jump": 1})
 
     def test_min(self):
-        space = self.space
-        float64_dtype = self.float64_dtype
-        int64_dtype = self.int64_dtype
-
-        def f(i):
-            if NonConstant(False):
-                dtype = int64_dtype
-            else:
-                dtype = float64_dtype
-            ar = SingleDimArray(i, dtype=dtype)
-            j = 0
-            while j < i:
-                ar.get_concrete().setitem(j, float64_dtype.box(float(j)))
-                j += 1
-            v = ar.descr_add(space, ar).descr_min(space)
-            assert isinstance(v, FloatObject)
-            return v.floatval
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
+        py.test.skip("broken, investigate")
+        result = self.run("""
+        a = |30|
+        a[15] = -12
+        b = a + a
+        min(b)
+        """)
+        assert result == -24
         self.check_loops({"getarrayitem_raw": 2, "float_add": 1,
-                           "float_lt": 1, "int_add": 1,
-                           "int_lt": 1, "guard_true": 2,
-                           "jump": 1})
-        assert result == f(5)
-
-    def test_argmin(self):
-        space = self.space
-        float64_dtype = self.float64_dtype
-
-        def f(i):
-            ar = SingleDimArray(i, dtype=NonConstant(float64_dtype))
-            j = 0
-            while j < i:
-                ar.get_concrete().setitem(j, float64_dtype.box(float(j)))
-                j += 1
-            return ar.descr_add(space, ar).descr_argmin(space).intval
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
-        self.check_loops({"getarrayitem_raw": 2, "float_add": 1,
-                           "float_lt": 1, "int_add": 1,
-                           "int_lt": 1, "guard_true": 2,
-                           "jump": 1})
-        assert result == f(5)
-
-    def test_all(self):
-        space = self.space
-        float64_dtype = self.float64_dtype
-
-        def f(i):
-            ar = SingleDimArray(i, dtype=NonConstant(float64_dtype))
-            j = 0
-            while j < i:
-                ar.get_concrete().setitem(j, float64_dtype.box(1.0))
-                j += 1
-            return ar.descr_add(space, ar).descr_all(space).boolval
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
-        self.check_loops({"getarrayitem_raw": 2, "float_add": 1,
-                          "int_add": 1, "float_ne": 1,
-                          "int_lt": 1, "guard_true": 2, "jump": 1})
-        assert result == f(5)
+                          "float_mul": 1, "int_add": 1,
+                          "int_lt": 1, "guard_true": 1, "jump": 1})
 
     def test_any(self):
-        space = self.space
-        float64_dtype = self.float64_dtype
-
-        def f(i):
-            ar = SingleDimArray(i, dtype=NonConstant(float64_dtype))
-            return ar.descr_add(space, ar).descr_any(space).boolval
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
+        result = self.run("""
+        a = [0,0,0,0,0,0,0,0,0,0,0]
+        a[8] = -12
+        b = a + a
+        any(b)
+        """)
+        assert result == 1
         self.check_loops({"getarrayitem_raw": 2, "float_add": 1,
-                          "int_add": 1, "float_ne": 1, "guard_false": 1,
-                          "int_lt": 1, "guard_true": 1, "jump": 1})
-        assert result == f(5)
+                          "float_ne": 1, "int_add": 1,
+                          "int_lt": 1, "guard_true": 1, "jump": 1,
+                          "guard_false": 1})
 
     def test_already_forced(self):
-        space = self.space
-
-        def f(i):
-            ar = SingleDimArray(i, dtype=self.float64_dtype)
-            v1 = interp_ufuncs.get(self.space).add.call(space, [ar, scalar_w(space, self.float64_dtype, space.wrap(4.5))])
-            assert isinstance(v1, BaseArray)
-            v2 = interp_ufuncs.get(self.space).multiply.call(space, [v1, scalar_w(space, self.float64_dtype, space.wrap(4.5))])
-            v1.force_if_needed()
-            assert isinstance(v2, BaseArray)
-            return v2.get_concrete().eval(3).val
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
+        result = self.run("""
+        a = |30|
+        b = a + 4.5
+        b -> 5 # forces
+        c = b * 8
+        c -> 5
+        """)
+        assert result == (5 + 4.5) * 8
         # This is the sum of the ops for both loops, however if you remove the
         # optimization then you end up with 2 float_adds, so we can still be
         # sure it was optimized correctly.
         self.check_loops({"getarrayitem_raw": 2, "float_mul": 1, "float_add": 1,
                            "setarrayitem_raw": 2, "int_add": 2,
                            "int_lt": 2, "guard_true": 2, "jump": 2})
-        assert result == f(5)
 
     def test_ufunc(self):
-        space = self.space
-        def f(i):
-            ar = SingleDimArray(i, dtype=self.float64_dtype)
-            v1 = interp_ufuncs.get(self.space).add.call(space, [ar, ar])
-            v2 = interp_ufuncs.get(self.space).negative.call(space, [v1])
-            return v2.get_concrete().eval(3).val
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
+        result = self.run("""
+        a = |30|
+        b = a + a
+        c = unegative(b)
+        c -> 3
+        """)
+        assert result == -6
         self.check_loops({"getarrayitem_raw": 2, "float_add": 1, "float_neg": 1,
                           "setarrayitem_raw": 1, "int_add": 1,
                           "int_lt": 1, "guard_true": 1, "jump": 1,
         })
-        assert result == f(5)
 
-    def test_appropriate_specialization(self):
-        space = self.space
-        def f(i):
-            ar = SingleDimArray(i, dtype=self.float64_dtype)
-
-            v1 = interp_ufuncs.get(self.space).add.call(space, [ar, ar])
-            v2 = interp_ufuncs.get(self.space).negative.call(space, [v1])
-            v2.get_concrete()
-
-            for i in xrange(5):
-                v1 = interp_ufuncs.get(self.space).multiply.call(space, [ar, ar])
-                v2 = interp_ufuncs.get(self.space).negative.call(space, [v1])
-                v2.get_concrete()
-
-        self.meta_interp(f, [5], listops=True, backendopt=True)
+    def test_specialization(self):
+        self.run("""
+        a = |30|
+        b = a + a
+        c = unegative(b)
+        c -> 3
+        d = a * a
+        unegative(d)
+        d -> 3
+        d = a * a
+        unegative(d)
+        d -> 3
+        d = a * a
+        unegative(d)
+        d -> 3
+        d = a * a
+        unegative(d)
+        d -> 3
+        """)
         # This is 3, not 2 because there is a bridge for the exit.
         self.check_loop_count(3)
 
+
+class TestNumpyOld(LLJitMixin):
+    def setup_class(cls):
+        from pypy.module.micronumpy.compile import FakeSpace
+        from pypy.module.micronumpy.interp_dtype import W_Float64Dtype
+        
+        cls.space = FakeSpace()
+        cls.float64_dtype = cls.space.fromcache(W_Float64Dtype)
+    
     def test_slice(self):
         def f(i):
             step = 3
@@ -332,17 +274,3 @@
         result = self.meta_interp(f, [5], listops=True, backendopt=True)
         assert result == f(5)
 
-class TestTranslation(object):
-    def test_compile(self):
-        x = numpy_compile('aa+f*f/a-', 10)
-        x = x.compute()
-        assert isinstance(x, SingleDimArray)
-        assert x.size == 10
-        assert x.eval(0).val == 0
-        assert x.eval(1).val == ((1 + 1) * 1.2) / 1.2 - 1
-
-    def test_translation(self):
-        # we import main to check if the target compiles
-        from pypy.translator.goal.targetnumpystandalone import main
-
-        interpret(main, [llstr('af+'), 100])
diff --git a/pypy/rlib/rsre/rsre_core.py b/pypy/rlib/rsre/rsre_core.py
--- a/pypy/rlib/rsre/rsre_core.py
+++ b/pypy/rlib/rsre/rsre_core.py
@@ -391,6 +391,8 @@
             if self.num_pending >= min:
                 while enum is not None and ptr == ctx.match_end:
                     enum = enum.move_to_next_result(ctx)
+                    # matched marks for zero-width assertions
+                    marks = ctx.match_marks
             #
             if enum is not None:
                 # matched one more 'item'.  record it and continue.
diff --git a/pypy/rlib/rsre/test/test_re.py b/pypy/rlib/rsre/test/test_re.py
--- a/pypy/rlib/rsre/test/test_re.py
+++ b/pypy/rlib/rsre/test/test_re.py
@@ -226,6 +226,13 @@
                          (None, 'b', None))
         assert pat.match('ac').group(1, 'b2', 3) == ('a', None, 'c')
 
+    def test_bug_923(self):
+        # Issue923: grouping inside optional lookahead problem
+        assert re.match(r'a(?=(b))?', "ab").groups() == ("b",)
+        assert re.match(r'(a(?=(b))?)', "ab").groups() == ('a', 'b')
+        assert re.match(r'(a)(?=(b))?', "ab").groups() == ('a', 'b')
+        assert re.match(r'(?P<g1>a)(?=(?P<g2>b))?', "ab").groupdict() == {'g1': 'a', 'g2': 'b'}
+
     def test_re_groupref_exists(self):
         assert re.match('^(\()?([^()]+)(?(1)\))$', '(a)').groups() == (
                          ('(', 'a'))
diff --git a/pypy/rpython/lltypesystem/rstr.py b/pypy/rpython/lltypesystem/rstr.py
--- a/pypy/rpython/lltypesystem/rstr.py
+++ b/pypy/rpython/lltypesystem/rstr.py
@@ -20,6 +20,7 @@
 from pypy.rpython.rmodel import Repr
 from pypy.rpython.lltypesystem import llmemory
 from pypy.tool.sourcetools import func_with_new_name
+from pypy.rpython.lltypesystem.lloperation import llop
 
 # ____________________________________________________________
 #
@@ -364,8 +365,10 @@
             while lpos < rpos and s.chars[lpos] == ch:
                 lpos += 1
         if right:
-            while lpos < rpos and s.chars[rpos] == ch:
+            while lpos < rpos + 1 and s.chars[rpos] == ch:
                 rpos -= 1
+        if rpos < lpos:
+            return s.empty()
         r_len = rpos - lpos + 1
         result = s.malloc(r_len)
         s.copy_contents(s, result, lpos, 0, r_len)
diff --git a/pypy/rpython/test/test_rstr.py b/pypy/rpython/test/test_rstr.py
--- a/pypy/rpython/test/test_rstr.py
+++ b/pypy/rpython/test/test_rstr.py
@@ -372,12 +372,20 @@
             return const('!ab!').lstrip(const('!'))
         def right():
             return const('!ab!').rstrip(const('!'))
+        def empty():
+            return const('    ').strip(' ')
+        def left2():
+            return const('a  ').strip(' ')
         res = self.interpret(both, [])
         assert self.ll_to_string(res) == const('ab')
         res = self.interpret(left, [])
         assert self.ll_to_string(res) == const('ab!')
         res = self.interpret(right, [])
         assert self.ll_to_string(res) == const('!ab')
+        res = self.interpret(empty, [])
+        assert self.ll_to_string(res) == const('')
+        res = self.interpret(left2, [])
+        assert self.ll_to_string(res) == const('a')
 
     def test_upper(self):
         const = self.const

From noreply at buildbot.pypy.org  Fri Oct 28 18:11:44 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Fri, 28 Oct 2011 18:11:44 +0200 (CEST)
Subject: [pypy-commit] pypy default: make more of the micronumpy tests pass
	under -A
Message-ID: <20111028161144.28BDA820C2@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r48572:1847537fd4b5
Date: 2011-10-28 12:11 -0400
http://bitbucket.org/pypy/pypy/changeset/1847537fd4b5/

Log:	make more of the micronumpy tests pass under -A

diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -13,6 +13,9 @@
         'empty': 'interp_numarray.zeros',
         'ones': 'interp_numarray.ones',
         'fromstring': 'interp_support.fromstring',
+
+        'True_': 'space.w_True',
+        'False_': 'space.w_False',
     }
 
     # ufuncs
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -32,11 +32,17 @@
         return self.identity.wrap(space)
 
     def descr_call(self, space, __args__):
-        try:
-            args_w = __args__.fixedunpack(self.argcount)
-        except ValueError, e:
-            raise OperationError(space.w_TypeError, space.wrap(str(e)))
-        return self.call(space, args_w)
+        if __args__.keywords or len(__args__.arguments_w) < self.argcount:
+            raise OperationError(space.w_ValueError,
+                space.wrap("invalid number of arguments")
+            )
+        elif len(__args__.arguments_w) > self.argcount:
+            # The extra arguments should actually be the output array, but we
+            # don't support that yet.
+            raise OperationError(space.w_TypeError,
+                space.wrap("invalid number of arguments")
+            )
+        return self.call(space, __args__.arguments_w)
 
     def descr_reduce(self, space, w_obj):
         from pypy.module.micronumpy.interp_numarray import convert_to_array, Scalar
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -36,37 +36,40 @@
         assert str(d) == "bool"
 
     def test_bool_array(self):
-        from numpy import array
+        import numpy
 
-        a = array([0, 1, 2, 2.5], dtype='?')
-        assert a[0] is False
+        a = numpy.array([0, 1, 2, 2.5], dtype='?')
+        assert a[0] is numpy.False_
         for i in xrange(1, 4):
-            assert a[i] is True
+            assert a[i] is numpy.True_
 
     def test_copy_array_with_dtype(self):
-        from numpy import array
-        a = array([0, 1, 2, 3], dtype=long)
+        import numpy
+
+        a = numpy.array([0, 1, 2, 3], dtype=long)
         # int on 64-bit, long in 32-bit
         assert isinstance(a[0], (int, long))
         b = a.copy()
         assert isinstance(b[0], (int, long))
 
-        a = array([0, 1, 2, 3], dtype=bool)
-        assert isinstance(a[0], bool)
+        a = numpy.array([0, 1, 2, 3], dtype=bool)
+        assert a[0] is numpy.False_
         b = a.copy()
-        assert isinstance(b[0], bool)
+        assert b[0] is numpy.False_
 
     def test_zeros_bool(self):
-        from numpy import zeros
-        a = zeros(10, dtype=bool)
+        import numpy
+
+        a = numpy.zeros(10, dtype=bool)
         for i in range(10):
-            assert a[i] is False
+            assert a[i] is numpy.False_
 
     def test_ones_bool(self):
-        from numpy import ones
-        a = ones(10, dtype=bool)
+        import numpy
+
+        a = numpy.ones(10, dtype=bool)
         for i in range(10):
-            assert a[i] is True
+            assert a[i] is numpy.True_
 
     def test_zeros_long(self):
         from numpy import zeros
@@ -77,7 +80,7 @@
 
     def test_ones_long(self):
         from numpy import ones
-        a = ones(10, dtype=bool)
+        a = ones(10, dtype=long)
         for i in range(10):
             assert isinstance(a[i], (int, long))
             assert a[1] == 1
@@ -96,8 +99,9 @@
 
     def test_bool_binop_types(self):
         from numpy import array, dtype
-        types = ('?','b','B','h','H','i','I','l','L','q','Q','f','d')
-        N = len(types)
+        types = [
+            '?', 'b', 'B', 'h', 'H', 'i', 'I', 'l', 'L', 'q', 'Q', 'f', 'd'
+        ]
         a = array([True], '?')
         for t in types:
             assert (a + array([0], t)).dtype is dtype(t)
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -214,7 +214,7 @@
     def test_add_other(self):
         from numpy import array
         a = array(range(5))
-        b = array(reversed(range(5)))
+        b = array(range(4, -1, -1))
         c = a + b
         for i in range(5):
             assert c[i] == 4
@@ -264,18 +264,19 @@
             assert b[i] == i - 5
 
     def test_mul(self):
-        from numpy import array, dtype
-        a = array(range(5))
+        import numpy
+
+        a = numpy.array(range(5))
         b = a * a
         for i in range(5):
             assert b[i] == i * i
 
-        a = array(range(5), dtype=bool)
+        a = numpy.array(range(5), dtype=bool)
         b = a * a
-        assert b.dtype is dtype(bool)
-        assert b[0] is False
+        assert b.dtype is numpy.dtype(bool)
+        assert b[0] is numpy.False_
         for i in range(1, 5):
-            assert b[i] is True
+            assert b[i] is numpy.True_
 
     def test_mul_constant(self):
         from numpy import array
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -24,10 +24,10 @@
     def test_wrong_arguments(self):
         from numpy import add, sin
 
-        raises(TypeError, add, 1)
+        raises(ValueError, add, 1)
         raises(TypeError, add, 1, 2, 3)
         raises(TypeError, sin, 1, 2)
-        raises(TypeError, sin)
+        raises(ValueError, sin)
 
     def test_single_item(self):
         from numpy import negative, sign, minimum
@@ -357,4 +357,4 @@
                 (3.5, 3),
                 (3, 3.5),
             ]:
-                assert ufunc(a, b) is func(a, b)
+                assert ufunc(a, b) == func(a, b)

From noreply at buildbot.pypy.org  Fri Oct 28 18:30:53 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 28 Oct 2011 18:30:53 +0200 (CEST)
Subject: [pypy-commit] pypy stm: begin_inevitable_transaction.
Message-ID: <20111028163053.4B91C820C2@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm
Changeset: r48573:f0ea7da7446e
Date: 2011-10-28 12:21 +0200
http://bitbucket.org/pypy/pypy/changeset/f0ea7da7446e/

Log:	begin_inevitable_transaction.

diff --git a/pypy/translator/c/funcgen.py b/pypy/translator/c/funcgen.py
--- a/pypy/translator/c/funcgen.py
+++ b/pypy/translator/c/funcgen.py
@@ -598,6 +598,7 @@
     OP_STM_SETFIELD = _OP_STM
     OP_STM_BEGIN_TRANSACTION = _OP_STM
     OP_STM_COMMIT_TRANSACTION = _OP_STM
+    OP_STM_BEGIN_INEVITABLE_TRANSACTION = _OP_STM
 
 
     def OP_PTR_NONZERO(self, op):
diff --git a/pypy/translator/stm/funcgen.py b/pypy/translator/stm/funcgen.py
--- a/pypy/translator/stm/funcgen.py
+++ b/pypy/translator/stm/funcgen.py
@@ -90,6 +90,9 @@
 def stm_commit_transaction(funcgen, op):
     return 'stm_commit_transaction();'
 
+def stm_begin_inevitable_transaction(funcgen, op):
+    return 'stm_begin_inevitable_transaction();'
+
 
 def op_stm(funcgen, op):
     assert funcgen.db.translator.stm_transformation_applied
diff --git a/pypy/translator/stm/rstm.py b/pypy/translator/stm/rstm.py
--- a/pypy/translator/stm/rstm.py
+++ b/pypy/translator/stm/rstm.py
@@ -87,6 +87,10 @@
     "NOT_RPYTHON.  For tests only"
     raise NotImplementedError("hard to really emulate")
 
+def begin_inevitable_transaction():
+    "NOT_RPYTHON.  For tests only, and at start up, but not in normal code."
+    raise NotImplementedError("hard to really emulate")
+
 def transaction_boundary():
     "NOT_RPYTHON.  This is the one normally used"
     raise NotImplementedError("hard to really emulate")
@@ -127,7 +131,8 @@
 
 
 class ExtEntry(ExtRegistryEntry):
-    _about_ = (begin_transaction, commit_transaction, transaction_boundary)
+    _about_ = (begin_transaction, commit_transaction,
+               begin_inevitable_transaction, transaction_boundary)
 
     def compute_result_annotation(self):
         return None
diff --git a/pypy/translator/stm/src_stm/et.c b/pypy/translator/stm/src_stm/et.c
--- a/pypy/translator/stm/src_stm/et.c
+++ b/pypy/translator/stm/src_stm/et.c
@@ -762,6 +762,10 @@
       if (bool_cas(&global_timestamp, curtime, curtime + 1))
         break;
     }
+#ifdef RPY_ASSERT
+  assert(!d->transaction_active);
+  d->transaction_active = 1;
+#endif
   d->setjmp_buf = NULL;
   d->start_time = curtime;
 #ifdef COMMIT_OTHER_INEV
diff --git a/pypy/translator/stm/test/test_transform.py b/pypy/translator/stm/test/test_transform.py
--- a/pypy/translator/stm/test/test_transform.py
+++ b/pypy/translator/stm/test/test_transform.py
@@ -72,3 +72,11 @@
             return 0
         t, cbuilder = self.compile(simplefunc)
         cbuilder.cmdexec('', expect_crash=True)
+
+    def test_begin_inevitable_transaction(self):
+        def simplefunc(argv):
+            rstm.begin_inevitable_transaction()
+            rstm.commit_transaction()
+            return 0
+        t, cbuilder = self.compile(simplefunc)
+        cbuilder.cmdexec('')

From noreply at buildbot.pypy.org  Fri Oct 28 18:30:54 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 28 Oct 2011 18:30:54 +0200 (CEST)
Subject: [pypy-commit] pypy stm: Progress.
Message-ID: <20111028163054.78738820C2@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm
Changeset: r48574:3ec86f341cf6
Date: 2011-10-28 14:22 +0200
http://bitbucket.org/pypy/pypy/changeset/3ec86f341cf6/

Log:	Progress.

diff --git a/pypy/translator/c/funcgen.py b/pypy/translator/c/funcgen.py
--- a/pypy/translator/c/funcgen.py
+++ b/pypy/translator/c/funcgen.py
@@ -226,6 +226,8 @@
                     yield '\tRPyConvertExceptionToCPython();'
                     yield '\treturn NULL;'
                     yield '}'
+                if self.exception_policy == "stm":
+                    yield 'STM_MAKE_INEVITABLE();'
                 retval = self.expr(block.inputargs[0])
                 if self.exception_policy != "exc_helper":
                     yield 'RPY_DEBUG_RETURN();'
@@ -599,6 +601,8 @@
     OP_STM_BEGIN_TRANSACTION = _OP_STM
     OP_STM_COMMIT_TRANSACTION = _OP_STM
     OP_STM_BEGIN_INEVITABLE_TRANSACTION = _OP_STM
+    OP_STM_TRANSACTION_BOUNDARY = _OP_STM
+    OP_STM_DECLARE_VARIABLE = _OP_STM
 
 
     def OP_PTR_NONZERO(self, op):
diff --git a/pypy/translator/stm/_rffi_stm.py b/pypy/translator/stm/_rffi_stm.py
--- a/pypy/translator/stm/_rffi_stm.py
+++ b/pypy/translator/stm/_rffi_stm.py
@@ -26,8 +26,10 @@
 descriptor_init = llexternal('stm_descriptor_init', [], lltype.Void)
 descriptor_done = llexternal('stm_descriptor_done', [], lltype.Void)
 
-#begin_transaction = llexternal('STM_begin_transaction', [], lltype.Void)
-#commit_transaction = llexternal('stm_commit_transaction', [], lltype.Signed)
+begin_transaction = llexternal('STM_begin_transaction', [], lltype.Void)
+begin_inevitable_transaction = llexternal('stm_begin_inevitable_transaction',
+                                          [], lltype.Void)
+commit_transaction = llexternal('stm_commit_transaction', [], lltype.Signed)
 
 stm_read_word = llexternal('stm_read_word', [SignedP], lltype.Signed)
 stm_write_word = llexternal('stm_write_word', [SignedP, lltype.Signed],
diff --git a/pypy/translator/stm/funcgen.py b/pypy/translator/stm/funcgen.py
--- a/pypy/translator/stm/funcgen.py
+++ b/pypy/translator/stm/funcgen.py
@@ -93,6 +93,17 @@
 def stm_begin_inevitable_transaction(funcgen, op):
     return 'stm_begin_inevitable_transaction();'
 
+def stm_declare_variable(funcgen, op):
+    # this operation occurs only once at the start of a function if
+    # it uses stm_transaction_boundary
+    assert funcgen.exception_policy is None
+    funcgen.exception_policy = 'stm'
+    return 'STM_DECLARE_VARIABLE();'
+
+def stm_transaction_boundary(funcgen, op):
+    assert funcgen.exception_policy == 'stm'
+    return 'STM_TRANSACTION_BOUNDARY();'
+
 
 def op_stm(funcgen, op):
     assert funcgen.db.translator.stm_transformation_applied
diff --git a/pypy/translator/stm/src_stm/et.c b/pypy/translator/stm/src_stm/et.c
--- a/pypy/translator/stm/src_stm/et.c
+++ b/pypy/translator/stm/src_stm/et.c
@@ -738,6 +738,13 @@
 #endif
 }
 
+void stm_try_inevitable_if(jmp_buf *buf)
+{
+  struct tx_descriptor *d = thread_descriptor;
+  if (d && d->setjmp_buf == buf)    /* XXX remove the test for 'd != NULL' */
+    stm_try_inevitable();
+}
+
 void stm_begin_inevitable_transaction(void)
 {
   struct tx_descriptor *d = thread_descriptor;
@@ -780,6 +787,13 @@
   tx_abort(7);     /* manual abort */
 }
 
+void stm_transaction_boundary(jmp_buf* buf)
+{
+  stm_commit_transaction();
+  setjmp(*buf);
+  stm_begin_transaction(buf);
+}
+
 // XXX little-endian only!
 void stm_write_partial_word(int fieldsize, char *base, long offset,
                             unsigned long nval)
diff --git a/pypy/translator/stm/src_stm/et.h b/pypy/translator/stm/src_stm/et.h
--- a/pypy/translator/stm/src_stm/et.h
+++ b/pypy/translator/stm/src_stm/et.h
@@ -20,14 +20,21 @@
 long stm_read_word(long* addr);
 void stm_write_word(long* addr, long val);
 void stm_try_inevitable(void);
+void stm_try_inevitable_if(jmp_buf* buf);
 void stm_begin_inevitable_transaction(void);
 void stm_abort_and_retry(void);
+void stm_transaction_boundary(jmp_buf* buf);
 
+/* for testing only: */
 #define STM_begin_transaction()         ; \
        jmp_buf _jmpbuf;                   \
        setjmp(_jmpbuf);                   \
        stm_begin_transaction(&_jmpbuf)
 
+#define STM_DECLARE_VARIABLE()          jmp_buf jmpbuf
+#define STM_MAKE_INEVITABLE()           stm_try_inevitable_if(&jmpbuf)
+#define STM_TRANSACTION_BOUNDARY()      stm_transaction_boundary(&jmpbuf)
+
 // XXX little-endian only!
 #define STM_read_partial_word(T, base, offset)                          \
     (T)(stm_read_word(                                                  \
diff --git a/pypy/translator/stm/test/test_transform.py b/pypy/translator/stm/test/test_transform.py
--- a/pypy/translator/stm/test/test_transform.py
+++ b/pypy/translator/stm/test/test_transform.py
@@ -43,11 +43,9 @@
 
     def test_no_pointer_operations(self):
         def simplefunc(argv):
-            rstm.begin_transaction()
             i = 0
             while i < 100:
                 i += 3
-            rstm.commit_transaction()
             debug_print(i)
             return 0
         t, cbuilder = self.compile(simplefunc)
@@ -56,12 +54,8 @@
         assert '102' in dataerr.splitlines()
 
     def test_fails_when_nonbalanced_begin(self):
-        def g():
-            rstm.begin_transaction()
-        g._dont_inline_ = True
         def simplefunc(argv):
             rstm.begin_transaction()
-            g()
             return 0
         t, cbuilder = self.compile(simplefunc)
         cbuilder.cmdexec('', expect_crash=True)
@@ -69,14 +63,22 @@
     def test_fails_when_nonbalanced_commit(self):
         def simplefunc(argv):
             rstm.commit_transaction()
+            rstm.commit_transaction()
             return 0
         t, cbuilder = self.compile(simplefunc)
         cbuilder.cmdexec('', expect_crash=True)
 
     def test_begin_inevitable_transaction(self):
         def simplefunc(argv):
+            rstm.commit_transaction()
             rstm.begin_inevitable_transaction()
-            rstm.commit_transaction()
             return 0
         t, cbuilder = self.compile(simplefunc)
         cbuilder.cmdexec('')
+
+    def test_transaction_boundary_1(self):
+        def simplefunc(argv):
+            rstm.transaction_boundary()
+            return 0
+        t, cbuilder = self.compile(simplefunc)
+        cbuilder.cmdexec('')
diff --git a/pypy/translator/stm/transform.py b/pypy/translator/stm/transform.py
--- a/pypy/translator/stm/transform.py
+++ b/pypy/translator/stm/transform.py
@@ -1,5 +1,7 @@
 from pypy.objspace.flow.model import SpaceOperation
 from pypy.translator.stm import _rffi_stm
+from pypy.translator.unsimplify import varoftype
+from pypy.rpython.lltypesystem import lltype
 
 
 class STMTransformer(object):
@@ -8,9 +10,12 @@
         self.translator = translator
 
     def transform(self):
+        for graph in self.translator.graphs:
+            self.seen_transaction_boundary = False
+            self.transform_graph(graph)
+            if self.seen_transaction_boundary:
+                self.add_stm_declare_variable(graph)
         self.add_descriptor_init_stuff()
-        for graph in self.translator.graphs:
-            self.transform_graph(graph)
         self.translator.stm_transformation_applied = True
 
     def transform_block(self, block):
@@ -31,11 +36,19 @@
         from pypy.translator.unsimplify import call_final_function
         def descriptor_init():
             _rffi_stm.descriptor_init()
+            _rffi_stm.begin_inevitable_transaction()
         def descriptor_done():
+            _rffi_stm.commit_transaction()
             _rffi_stm.descriptor_done()
         call_initial_function(self.translator, descriptor_init)
         call_final_function(self.translator, descriptor_done)
 
+    def add_stm_declare_variable(self, graph):
+        block = graph.startblock
+        v = varoftype(lltype.Void)
+        op = SpaceOperation('stm_declare_variable', [], v)
+        block.operations.insert(0, op)
+
     # ----------
 
     def stt_getfield(self, newoperations, op):
@@ -50,6 +63,10 @@
         op1 = SpaceOperation('stm_setfield', op.args, op.result)
         newoperations.append(op1)
 
+    def stt_stm_transaction_boundary(self, newoperations, op):
+        self.seen_transaction_boundary = True
+        newoperations.append(op)
+
 
 def transform_graph(graph):
     # for tests: only transforms one graph

From noreply at buildbot.pypy.org  Fri Oct 28 18:30:55 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 28 Oct 2011 18:30:55 +0200 (CEST)
Subject: [pypy-commit] pypy stm: Random progress.
Message-ID: <20111028163055.A12D3820C2@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm
Changeset: r48575:451f6144a150
Date: 2011-10-28 18:01 +0200
http://bitbucket.org/pypy/pypy/changeset/451f6144a150/

Log:	Random progress.

diff --git a/pypy/translator/c/funcgen.py b/pypy/translator/c/funcgen.py
--- a/pypy/translator/c/funcgen.py
+++ b/pypy/translator/c/funcgen.py
@@ -603,6 +603,7 @@
     OP_STM_BEGIN_INEVITABLE_TRANSACTION = _OP_STM
     OP_STM_TRANSACTION_BOUNDARY = _OP_STM
     OP_STM_DECLARE_VARIABLE = _OP_STM
+    OP_STM_TRY_INEVITABLE = _OP_STM
 
 
     def OP_PTR_NONZERO(self, op):
diff --git a/pypy/translator/stm/_rffi_stm.py b/pypy/translator/stm/_rffi_stm.py
--- a/pypy/translator/stm/_rffi_stm.py
+++ b/pypy/translator/stm/_rffi_stm.py
@@ -30,6 +30,7 @@
 begin_inevitable_transaction = llexternal('stm_begin_inevitable_transaction',
                                           [], lltype.Void)
 commit_transaction = llexternal('stm_commit_transaction', [], lltype.Signed)
+try_inevitable = llexternal('stm_try_inevitable', [], lltype.Void)
 
 stm_read_word = llexternal('stm_read_word', [SignedP], lltype.Signed)
 stm_write_word = llexternal('stm_write_word', [SignedP, lltype.Signed],
diff --git a/pypy/translator/stm/funcgen.py b/pypy/translator/stm/funcgen.py
--- a/pypy/translator/stm/funcgen.py
+++ b/pypy/translator/stm/funcgen.py
@@ -104,6 +104,9 @@
     assert funcgen.exception_policy == 'stm'
     return 'STM_TRANSACTION_BOUNDARY();'
 
+def stm_try_inevitable(funcgen, op):
+    return 'stm_try_inevitable();'
+
 
 def op_stm(funcgen, op):
     assert funcgen.db.translator.stm_transformation_applied
diff --git a/pypy/translator/stm/llstminterp.py b/pypy/translator/stm/llstminterp.py
--- a/pypy/translator/stm/llstminterp.py
+++ b/pypy/translator/stm/llstminterp.py
@@ -1,6 +1,7 @@
 from pypy.rpython.lltypesystem import lltype
 from pypy.rpython.llinterp import LLFrame, LLException
 from pypy.translator.stm import rstm
+from pypy.translator.stm.transform import op_in_set, ALWAYS_ALLOW_OPERATIONS
 
 
 class ForbiddenInstructionInSTMMode(Exception):
@@ -30,11 +31,6 @@
 
 class LLSTMFrame(LLFrame):
 
-    ALWAYS_ALLOW_OPERATIONS = set([
-        'int_*', 'same_as', 'cast_*',
-        'direct_call',
-        ])
-
     def eval(self):
         try:
             res = LLFrame.eval(self)
@@ -60,16 +56,8 @@
             setattr(self, 'opstm_' + opname, ophandler)
         return ophandler
 
-    def _op_in_set(self, opname, set):
-        if opname in set:
-            return True
-        for i in range(len(opname)-1, -1, -1):
-            if (opname[:i] + '*') in set:
-                return True
-        return False
-
     def _validate_stmoperation_handler(self, opname):
-        if self._op_in_set(opname, self.ALWAYS_ALLOW_OPERATIONS):
+        if op_in_set(opname, ALWAYS_ALLOW_OPERATIONS):
             return
         raise ForbiddenInstructionInSTMMode(opname, self.graph)
 
diff --git a/pypy/translator/stm/transform.py b/pypy/translator/stm/transform.py
--- a/pypy/translator/stm/transform.py
+++ b/pypy/translator/stm/transform.py
@@ -4,6 +4,23 @@
 from pypy.rpython.lltypesystem import lltype
 
 
+ALWAYS_ALLOW_OPERATIONS = set([
+    'int_*', 'same_as', 'cast_*',
+    'direct_call',
+    'debug_print',
+    ])
+
+def op_in_set(opname, set):
+    if opname in set:
+        return True
+    for i in range(len(opname)-1, -1, -1):
+        if (opname[:i] + '*') in set:
+            return True
+    return False
+
+# ____________________________________________________________
+
+
 class STMTransformer(object):
 
     def __init__(self, translator=None):
@@ -23,7 +40,15 @@
             return
         newoperations = []
         for op in block.operations:
-            meth = getattr(self, 'stt_' + op.opname, list.append)
+            try:
+                meth = getattr(self, 'stt_' + op.opname)
+            except AttributeError:
+                if op_in_set(op.opname, ALWAYS_ALLOW_OPERATIONS):
+                    meth = list.append
+                else:
+                    meth = turn_inevitable_and_proceed
+                setattr(self.__class__, 'stt_' + op.opname,
+                        staticmethod(meth))
             meth(newoperations, op)
         block.operations = newoperations
 
@@ -71,3 +96,9 @@
 def transform_graph(graph):
     # for tests: only transforms one graph
     STMTransformer().transform_graph(graph)
+
+
+def turn_inevitable_and_proceed(newoperations, op):
+    op1 = SpaceOperation('stm_try_inevitable', [], varoftype(lltype.Void))
+    newoperations.append(op1)
+    newoperations.append(op)

From noreply at buildbot.pypy.org  Fri Oct 28 18:30:56 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 28 Oct 2011 18:30:56 +0200 (CEST)
Subject: [pypy-commit] pypy stm: Add the missing operations.
Message-ID: <20111028163056.C8152820C2@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm
Changeset: r48576:f647b24f79f3
Date: 2011-10-28 18:07 +0200
http://bitbucket.org/pypy/pypy/changeset/f647b24f79f3/

Log:	Add the missing operations.

diff --git a/pypy/rpython/lltypesystem/lloperation.py b/pypy/rpython/lltypesystem/lloperation.py
--- a/pypy/rpython/lltypesystem/lloperation.py
+++ b/pypy/rpython/lltypesystem/lloperation.py
@@ -392,6 +392,13 @@
     'stm_getfield':         LLOp(sideeffects=False, canrun=True),
     'stm_setfield':         LLOp(),
 
+    'stm_begin_transaction':            LLOp(),
+    'stm_commit_transaction':           LLOp(),
+    'stm_begin_inevitable_transaction': LLOp(),
+    'stm_transaction_boundary':         LLOp(),
+    'stm_declare_variable':             LLOp(),
+    'stm_try_inevitable':               LLOp(),
+
     # __________ address operations __________
 
     'boehm_malloc':         LLOp(),
diff --git a/pypy/translator/backendopt/canraise.py b/pypy/translator/backendopt/canraise.py
--- a/pypy/translator/backendopt/canraise.py
+++ b/pypy/translator/backendopt/canraise.py
@@ -10,10 +10,14 @@
 py.log.setconsumer("canraise", ansi_log) 
 
 class RaiseAnalyzer(graphanalyze.BoolGraphAnalyzer):
+    fail_on_unknown_operation = False
+
     def analyze_simple_operation(self, op, graphinfo):
         try:
             return bool(LL_OPERATIONS[op.opname].canraise)
         except KeyError:
+            if self.fail_on_unknown_operation:
+                raise
             log.WARNING("Unknown operation: %s" % op.opname)
             return True
 
diff --git a/pypy/translator/stm/test/test_transform.py b/pypy/translator/stm/test/test_transform.py
--- a/pypy/translator/stm/test/test_transform.py
+++ b/pypy/translator/stm/test/test_transform.py
@@ -39,7 +39,16 @@
         from pypy.config.pypyoption import get_pypy_config
         self.config = get_pypy_config(translating=True)
         self.config.translation.stm = True
-        return StandaloneTests.compile(self, entry_point, debug=True)
+        #
+        # Prevent the RaiseAnalyzer from just emitting "WARNING: Unknown
+        # operation".  We want instead it to crash.
+        from pypy.translator.backendopt.canraise import RaiseAnalyzer
+        RaiseAnalyzer.fail_on_unknown_operation = True
+        try:
+            res = StandaloneTests.compile(self, entry_point, debug=True)
+        finally:
+            del RaiseAnalyzer.fail_on_unknown_operation
+        return res
 
     def test_no_pointer_operations(self):
         def simplefunc(argv):

From noreply at buildbot.pypy.org  Fri Oct 28 18:30:57 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 28 Oct 2011 18:30:57 +0200 (CEST)
Subject: [pypy-commit] pypy stm: bug fix
Message-ID: <20111028163057.EEBCF820C2@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm
Changeset: r48577:c30efe18be88
Date: 2011-10-28 18:11 +0200
http://bitbucket.org/pypy/pypy/changeset/c30efe18be88/

Log:	bug fix

diff --git a/pypy/translator/stm/src_stm/et.c b/pypy/translator/stm/src_stm/et.c
--- a/pypy/translator/stm/src_stm/et.c
+++ b/pypy/translator/stm/src_stm/et.c
@@ -698,6 +698,10 @@
      to 1. */
   struct tx_descriptor *d = thread_descriptor;
 
+#ifdef RPY_ASSERT
+  assert(d->transaction_active);
+#endif
+
   if (is_inevitable(d))
     return;  /* I am already inevitable */
 
diff --git a/pypy/translator/stm/transform.py b/pypy/translator/stm/transform.py
--- a/pypy/translator/stm/transform.py
+++ b/pypy/translator/stm/transform.py
@@ -43,7 +43,8 @@
             try:
                 meth = getattr(self, 'stt_' + op.opname)
             except AttributeError:
-                if op_in_set(op.opname, ALWAYS_ALLOW_OPERATIONS):
+                if (op_in_set(op.opname, ALWAYS_ALLOW_OPERATIONS) or
+                        op.opname.startswith('stm_')):
                     meth = list.append
                 else:
                     meth = turn_inevitable_and_proceed

From noreply at buildbot.pypy.org  Fri Oct 28 18:30:59 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 28 Oct 2011 18:30:59 +0200 (CEST)
Subject: [pypy-commit] pypy stm: Test that the stm mode falls back to
 "inevitable_transaction" when
Message-ID: <20111028163059.21678820C2@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm
Changeset: r48578:3126dbb549ce
Date: 2011-10-28 18:30 +0200
http://bitbucket.org/pypy/pypy/changeset/3126dbb549ce/

Log:	Test that the stm mode falls back to "inevitable_transaction" when
	we see an unsupported operation like raw_malloc.

diff --git a/pypy/translator/stm/llstminterp.py b/pypy/translator/stm/llstminterp.py
--- a/pypy/translator/stm/llstminterp.py
+++ b/pypy/translator/stm/llstminterp.py
@@ -51,16 +51,18 @@
     def getoperationhandler(self, opname):
         ophandler = getattr(self, 'opstm_' + opname, None)
         if ophandler is None:
-            self._validate_stmoperation_handler(opname)
             ophandler = LLFrame.getoperationhandler(self, opname)
-            setattr(self, 'opstm_' + opname, ophandler)
+            if op_in_set(opname, ALWAYS_ALLOW_OPERATIONS):
+                # always allow this, so store it back on self.__class__
+                setattr(self.__class__, 'opstm_' + opname,
+                        staticmethod(ophandler))
+            else:
+                # only allow this if we're not in the "regular_transaction"
+                # mode; check every time, so don't store it on self.__class__
+                if self.llinterpreter.stm_mode == "regular_transaction":
+                    raise ForbiddenInstructionInSTMMode(opname, self.graph)
         return ophandler
 
-    def _validate_stmoperation_handler(self, opname):
-        if op_in_set(opname, ALWAYS_ALLOW_OPERATIONS):
-            return
-        raise ForbiddenInstructionInSTMMode(opname, self.graph)
-
     # ---------- operations that are sometimes safe ----------
 
     def opstm_getfield(self, struct, fieldname):
@@ -103,3 +105,7 @@
         self.check_stm_mode(lambda m: m != "not_in_transaction")
         self.llinterpreter.stm_mode = "regular_transaction"
         self.llinterpreter.last_transaction_started_in_frame = self
+
+    def opstm_stm_try_inevitable(self):
+        self.check_stm_mode(lambda m: m != "not_in_transaction")
+        self.llinterpreter.stm_mode = "inevitable_transaction"
diff --git a/pypy/translator/stm/test/test_transform.py b/pypy/translator/stm/test/test_transform.py
--- a/pypy/translator/stm/test/test_transform.py
+++ b/pypy/translator/stm/test/test_transform.py
@@ -1,4 +1,4 @@
-from pypy.rpython.lltypesystem import lltype
+from pypy.rpython.lltypesystem import lltype, llmemory
 from pypy.rpython.test.test_llinterp import get_interpreter
 from pypy.objspace.flow.model import summary
 from pypy.translator.stm.llstminterp import eval_stm_graph
@@ -6,8 +6,21 @@
 from pypy.translator.stm import rstm
 from pypy.translator.c.test.test_standalone import StandaloneTests
 from pypy.rlib.debug import debug_print
+from pypy.conftest import option
 
 
+def eval_stm_func(func, arguments, stm_mode="regular_transaction",
+                  final_stm_mode="regular_transaction"):
+    interp, graph = get_interpreter(func, arguments)
+    transform_graph(graph)
+    #if option.view:
+    #    graph.show()
+    return eval_stm_graph(interp, graph, arguments, stm_mode=stm_mode,
+                          final_stm_mode=final_stm_mode,
+                          automatic_promotion=True)
+
+# ____________________________________________________________
+
 def test_simple():
     S = lltype.GcStruct('S', ('x', lltype.Signed))
     p = lltype.malloc(S, immortal=True)
@@ -32,6 +45,19 @@
     res = eval_stm_graph(interp, graph, [p], stm_mode="regular_transaction")
     assert res == 42
 
+def test_unsupported_operation():
+    def func(n):
+        n += 1
+        if n > 5:
+            p = llmemory.raw_malloc(llmemory.sizeof(lltype.Signed))
+            llmemory.raw_free(p)
+        return n
+    res = eval_stm_func(func, [3], final_stm_mode="regular_transaction")
+    assert res == 4
+    res = eval_stm_func(func, [13], final_stm_mode="inevitable_transaction")
+    assert res == 14
+
+# ____________________________________________________________
 
 class TestTransformSingleThread(StandaloneTests):
 
@@ -91,3 +117,12 @@
             return 0
         t, cbuilder = self.compile(simplefunc)
         cbuilder.cmdexec('')
+
+    def test_transaction_boundary_2(self):
+        def simplefunc(argv):
+            rstm.transaction_boundary()
+            rstm.transaction_boundary()
+            rstm.transaction_boundary()
+            return 0
+        t, cbuilder = self.compile(simplefunc)
+        cbuilder.cmdexec('')
diff --git a/pypy/translator/stm/transform.py b/pypy/translator/stm/transform.py
--- a/pypy/translator/stm/transform.py
+++ b/pypy/translator/stm/transform.py
@@ -5,9 +5,11 @@
 
 
 ALWAYS_ALLOW_OPERATIONS = set([
-    'int_*', 'same_as', 'cast_*',
+    'int_*', 'uint_*', 'llong_*', 'ullong_*',
+    'same_as', 'cast_*',
     'direct_call',
-    'debug_print',
+    'debug_print', 'debug_assert',
+    'malloc', 'malloc_varsize',
     ])
 
 def op_in_set(opname, set):

From noreply at buildbot.pypy.org  Fri Oct 28 18:41:37 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 28 Oct 2011 18:41:37 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-multidim: fix tests broken during the merge
Message-ID: <20111028164137.B5E24820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: numpy-multidim
Changeset: r48579:cfc66011fc77
Date: 2011-10-28 18:40 +0200
http://bitbucket.org/pypy/pypy/changeset/cfc66011fc77/

Log:	fix tests broken during the merge

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -16,22 +16,6 @@
                                                        'dtype'])
 slice_driver = jit.JitDriver(greens=['signature'], reds=['i', 'self', 'source'])
 
-def _find_dtype(space, w_iterable):
-    stack = [w_iterable]
-    w_dtype = None
-    while stack:
-        w_next = stack.pop()
-        if space.issequence_w(w_next):
-            for w_item in space.listview(w_next):
-                stack.append(w_item)
-        else:
-            w_dtype = interp_ufuncs.find_dtype_for_scalar(space, w_next, w_dtype)
-            if w_dtype is space.fromcache(interp_dtype.W_Float64Dtype):
-                return w_dtype
-    if w_dtype is None:
-        return space.w_None
-    return w_dtype
-
 def _find_shape_and_elems(space, w_iterable):
     shape = [space.len_w(w_iterable)]
     batch = space.listview(w_iterable)
@@ -56,15 +40,27 @@
 
 def descr_new_array(space, w_subtype, w_item_or_iterable, w_dtype=None):
     # find scalar
+    if not space.issequence_w(w_item_or_iterable):
+        w_dtype = interp_ufuncs.find_dtype_for_scalar(space,
+                                                      w_item_or_iterable,
+                                                      w_dtype)
+        dtype = space.interp_w(interp_dtype.W_Dtype,
+           space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype))
+        return scalar_w(space, dtype, w_item_or_iterable)
+    shape, elems_w = _find_shape_and_elems(space, w_item_or_iterable)
+    size = len(elems_w)
     if space.is_w(w_dtype, space.w_None):
-        w_dtype = _find_dtype(space, w_item_or_iterable)
+        w_dtype = None
+        for w_elem in elems_w:
+            w_dtype = interp_ufuncs.find_dtype_for_scalar(space, w_elem,
+                                                          w_dtype)
+            if w_dtype is space.fromcache(interp_dtype.W_Float64Dtype):
+                break
+    if w_dtype is None:
+        w_dtype = space.w_None
     dtype = space.interp_w(interp_dtype.W_Dtype,
         space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype)
     )
-    if not space.issequence_w(w_item_or_iterable):
-        return scalar_w(space, dtype, w_item_or_iterable)
-    shape, elems_w = _find_shape_and_elems(space, w_item_or_iterable)
-    size = len(elems_w)
     arr = NDimArray(size, shape, dtype=dtype)
     i = 0
     for i, w_elem in enumerate(elems_w):
@@ -380,10 +376,13 @@
         return space.wrap(space.float_w(self.descr_sum(space))/self.find_size())
 
     def descr_nonzero(self, space):
-        if self.find_size() > 1:
-            raise OperationError(space.w_ValueError, space.wrap(
-                "The truth value of an array with more than one element is ambiguous. Use a.any() or a.all()"))
-        return self.get_concrete().eval(0).wrap(space)
+        try:
+            if self.find_size() > 1:
+                raise OperationError(space.w_ValueError, space.wrap(
+                    "The truth value of an array with more than one element is ambiguous. Use a.any() or a.all()"))
+        except ValueError:
+            pass
+        return space.wrap(space.is_true(self.get_concrete().eval(0).wrap(space)))
 
 def convert_to_array(space, w_obj):
     if isinstance(w_obj, BaseArray):
@@ -582,7 +581,7 @@
         self.chunks = chunks
         self.shape_reduction = 0
         for chunk in chunks:
-            if chunk[-1] == 1:
+            if chunk[-2] == 0:
                 self.shape_reduction += 1
 
     def get_root_storage(self):
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -219,7 +219,7 @@
     def test_add_other(self):
         from numpy import array
         a = array(range(5))
-        b = array(reversed(range(5)))
+        b = array([i for i in reversed(range(5))])
         c = a + b
         for i in range(5):
             assert c[i] == 4

From noreply at buildbot.pypy.org  Fri Oct 28 18:53:05 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 28 Oct 2011 18:53:05 +0200 (CEST)
Subject: [pypy-commit] pypy stm: - mallocs of GC objects is supported by the
 STM system (for now
Message-ID: <20111028165305.80775820C2@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm
Changeset: r48580:1cffa0d605b4
Date: 2011-10-28 18:52 +0200
http://bitbucket.org/pypy/pypy/changeset/1cffa0d605b4/

Log:	- mallocs of GC objects is supported by the STM system (for now
	with Boehm)

	- fix: don't store ophandler on self.__class__, because it is
	typically a method bound to the current self

diff --git a/pypy/translator/stm/llstminterp.py b/pypy/translator/stm/llstminterp.py
--- a/pypy/translator/stm/llstminterp.py
+++ b/pypy/translator/stm/llstminterp.py
@@ -23,7 +23,7 @@
             final_stm_mode = stm_mode
         assert llinterp.stm_mode == final_stm_mode, (
             "llinterp.stm_mode is %r after eval_graph, but should be %r" % (
-            llinterp.stm_mode, stm_mode))
+            llinterp.stm_mode, final_stm_mode))
         return res
     finally:
         llinterp.frame_class = LLFrame
@@ -53,9 +53,8 @@
         if ophandler is None:
             ophandler = LLFrame.getoperationhandler(self, opname)
             if op_in_set(opname, ALWAYS_ALLOW_OPERATIONS):
-                # always allow this, so store it back on self.__class__
-                setattr(self.__class__, 'opstm_' + opname,
-                        staticmethod(ophandler))
+                # always allow this, so store it back on 'self'
+                setattr(self, 'opstm_' + opname, ophandler)
             else:
                 # only allow this if we're not in the "regular_transaction"
                 # mode; check every time, so don't store it on self.__class__
@@ -109,3 +108,8 @@
     def opstm_stm_try_inevitable(self):
         self.check_stm_mode(lambda m: m != "not_in_transaction")
         self.llinterpreter.stm_mode = "inevitable_transaction"
+
+    def opstm_malloc(self, TYPE, flags):
+        if flags['flavor'] != 'gc':
+            self.check_stm_mode(lambda m: m != "regular_transaction")
+        return LLFrame.op_malloc(self, TYPE, flags)
diff --git a/pypy/translator/stm/test/test_transform.py b/pypy/translator/stm/test/test_transform.py
--- a/pypy/translator/stm/test/test_transform.py
+++ b/pypy/translator/stm/test/test_transform.py
@@ -57,6 +57,19 @@
     res = eval_stm_func(func, [13], final_stm_mode="inevitable_transaction")
     assert res == 14
 
+def test_supported_malloc():
+    S = lltype.GcStruct('S', ('x', lltype.Signed))   # GC structure
+    def func():
+        lltype.malloc(S)
+    eval_stm_func(func, [], final_stm_mode="regular_transaction")
+
+def test_unsupported_malloc():
+    S = lltype.Struct('S', ('x', lltype.Signed))   # non-GC structure
+    def func():
+        lltype.malloc(S, flavor='raw')
+    eval_stm_func(func, [], final_stm_mode="inevitable_transaction")
+test_unsupported_malloc.dont_track_allocations = True
+
 # ____________________________________________________________
 
 class TestTransformSingleThread(StandaloneTests):
diff --git a/pypy/translator/stm/transform.py b/pypy/translator/stm/transform.py
--- a/pypy/translator/stm/transform.py
+++ b/pypy/translator/stm/transform.py
@@ -9,7 +9,6 @@
     'same_as', 'cast_*',
     'direct_call',
     'debug_print', 'debug_assert',
-    'malloc', 'malloc_varsize',
     ])
 
 def op_in_set(opname, set):
@@ -52,7 +51,13 @@
                     meth = turn_inevitable_and_proceed
                 setattr(self.__class__, 'stt_' + op.opname,
                         staticmethod(meth))
-            meth(newoperations, op)
+            res = meth(newoperations, op)
+            if res is True:
+                newoperations.append(op)
+            elif res is False:
+                turn_inevitable_and_proceed(newoperations, op)
+            else:
+                assert res is None
         block.operations = newoperations
 
     def transform_graph(self, graph):
@@ -93,7 +98,11 @@
 
     def stt_stm_transaction_boundary(self, newoperations, op):
         self.seen_transaction_boundary = True
-        newoperations.append(op)
+        return True
+
+    def stt_malloc(self, newoperations, op):
+        flags = op.args[1].value
+        return flags['flavor'] == 'gc'
 
 
 def transform_graph(graph):

From noreply at buildbot.pypy.org  Fri Oct 28 19:08:30 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 28 Oct 2011 19:08:30 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-multidim: make test_zjit pass (up to the
	point anyway)
Message-ID: <20111028170830.81888820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: numpy-multidim
Changeset: r48581:488dbd72cd7b
Date: 2011-10-28 19:07 +0200
http://bitbucket.org/pypy/pypy/changeset/488dbd72cd7b/

Log:	make test_zjit pass (up to the point anyway)

diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -28,6 +28,7 @@
 class FakeSpace(object):
     w_ValueError = None
     w_TypeError = None
+    w_IndexError = None
     w_None = None
 
     w_bool = "bool"
@@ -70,6 +71,7 @@
     def listview(self, obj):
         assert isinstance(obj, ListObject)
         return obj.items
+    fixedview = listview
 
     def float(self, w_obj):
         assert isinstance(w_obj, FloatObject)
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -63,12 +63,13 @@
     )
     arr = NDimArray(size, shape, dtype=dtype)
     i = 0
-    for i, w_elem in enumerate(elems_w):
+    for i in range(len(elems_w)):
+        w_elem = elems_w[i]
         dtype.setitem_w(space, arr.storage, i, w_elem)
     return arr
 
 class BaseArray(Wrappable):
-    _attrs_ = ["invalidates", "signature"]
+    _attrs_ = ["invalidates", "signature", "shape"]
 
     def __init__(self, shape):
         self.invalidates = []
@@ -406,7 +407,7 @@
     """
     signature = signature.BaseSignature()
 
-    _attrs_ = ["dtype", "value"]
+    _attrs_ = ["dtype", "value", "shape"]
 
     def __init__(self, dtype, value):
         BaseArray.__init__(self, [])
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -265,7 +265,7 @@
                 dtype = float64_dtype
             else:
                 dtype = int32_dtype
-            ar = SingleDimArray(n, dtype=dtype)
+            ar = NDimArray(n, [n], dtype=dtype)
             i = 0
             while i < n:
                 ar.get_concrete().setitem(i, int32_dtype.box(7))

From noreply at buildbot.pypy.org  Fri Oct 28 19:41:49 2011
From: noreply at buildbot.pypy.org (hager)
Date: Fri, 28 Oct 2011 19:41:49 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-jit-backend: Started implementation of CALL.
Message-ID: <20111028174149.F230D820C2@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r48582:87920fe776f3
Date: 2011-10-28 19:41 +0200
http://bitbucket.org/pypy/pypy/changeset/87920fe776f3/

Log:	Started implementation of CALL.

diff --git a/pypy/jit/backend/ppc/ppcgen/arch.py b/pypy/jit/backend/ppc/ppcgen/arch.py
--- a/pypy/jit/backend/ppc/ppcgen/arch.py
+++ b/pypy/jit/backend/ppc/ppcgen/arch.py
@@ -15,3 +15,4 @@
 MY_COPY_OF_REGS = 0
 
 GPR_SAVE_AREA   = len(NONVOLATILES) * WORD
+MAX_REG_PARAMS = 8
diff --git a/pypy/jit/backend/ppc/ppcgen/codebuilder.py b/pypy/jit/backend/ppc/ppcgen/codebuilder.py
--- a/pypy/jit/backend/ppc/ppcgen/codebuilder.py
+++ b/pypy/jit/backend/ppc/ppcgen/codebuilder.py
@@ -974,6 +974,11 @@
         self.mtctr(0)
         self.bctr()
 
+    def bl_abs(self, address):
+        self.load_imm(r.r0, address)
+        self.mtctr(r.r0.value)
+        self.bctrl()
+
     def prepare_insts_blocks(self, show=False):
         self.assemble(show)
         insts = self.insts
diff --git a/pypy/jit/backend/ppc/ppcgen/helper/assembler.py b/pypy/jit/backend/ppc/ppcgen/helper/assembler.py
--- a/pypy/jit/backend/ppc/ppcgen/helper/assembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/helper/assembler.py
@@ -1,5 +1,7 @@
 import pypy.jit.backend.ppc.ppcgen.condition as c
 from pypy.rlib.rarithmetic import r_uint, r_longlong, intmask
+from pypy.jit.backend.ppc.ppcgen.arch import MAX_REG_PARAMS
+from pypy.jit.metainterp.history import FLOAT
 
 def gen_emit_cmp_op(condition, signed=True):
     def f(self, op, arglocs, regalloc):
@@ -80,6 +82,22 @@
             | ord(mem[index+1]) << 16
             | ord(mem[index]) << 24)
 
+def count_reg_args(args):
+    reg_args = 0
+    words = 0
+    count = 0
+    for x in range(min(len(args), MAX_REG_PARAMS)):
+        if args[x].type == FLOAT:
+            assert 0, "not implemented yet"
+        else:
+            count += 1
+            words += 1
+        reg_args += 1
+        if words > MAX_REG_PARAMS:
+            reg_args = x
+            break
+    return reg_args
+
 class saved_registers(object):
     def __init__(self, assembler, regs_to_save, regalloc=None):
         self.assembler = assembler
diff --git a/pypy/jit/backend/ppc/ppcgen/opassembler.py b/pypy/jit/backend/ppc/ppcgen/opassembler.py
--- a/pypy/jit/backend/ppc/ppcgen/opassembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/opassembler.py
@@ -4,8 +4,10 @@
 import pypy.jit.backend.ppc.ppcgen.register as r
 from pypy.jit.backend.ppc.ppcgen.arch import GPR_SAVE_AREA, IS_PPC_32, WORD
 
-from pypy.jit.metainterp.history import LoopToken, AbstractFailDescr
+from pypy.jit.metainterp.history import LoopToken, AbstractFailDescr, FLOAT
 from pypy.rlib.objectmodel import we_are_translated
+from pypy.jit.backend.ppc.ppcgen.helper.assembler import count_reg_args 
+from pypy.jit.backend.ppc.ppcgen.jump import remap_frame_layout
 
 class GuardToken(object):
     def __init__(self, descr, failargs, faillocs, offset, fcond=c.NE,
@@ -445,6 +447,89 @@
     emit_cast_ptr_to_int = emit_same_as
     emit_cast_int_to_ptr = emit_same_as
 
+    def emit_call(self, op, args, regalloc, force_index=-1):
+        adr = args[0].value
+        arglist = op.getarglist()[1:]
+        if force_index == -1:
+            force_index = self.write_new_force_index()
+        self._emit_call(force_index, adr, arglist, regalloc, op.result)
+        descr = op.getdescr()
+        #XXX Hack, Hack, Hack
+        if op.result and not we_are_translated() and not isinstance(descr,
+                LoopToken):
+            import pdb; pdb.set_trace()
+            #XXX check result type
+            loc = regalloc.rm.call_result_location(op.result)
+            size = descr.get_result_size(False)
+            signed = descr.is_result_signed()
+            self._ensure_result_bit_extension(loc, size, signed)
+
+    def _emit_call(self, force_index, adr, args, regalloc, result=None):
+        n_args = len(args)
+        reg_args = count_reg_args(args)
+
+        n = 0   # used to count the number of words pushed on the stack, so we
+                #can later modify the SP back to its original value
+        if n_args > reg_args:
+            assert 0, "not implemented yet"
+
+        # collect variables that need to go in registers
+        # and the registers they will be stored in 
+        num = 0
+        count = 0
+        non_float_locs = []
+        non_float_regs = []
+        for i in range(reg_args):
+            arg = args[i]
+            if arg.type == FLOAT and count % 2 != 0:
+                assert 0, "not implemented yet"
+            reg = r.PARAM_REGS[num]
+
+            if arg.type == FLOAT:
+                assert 0, "not implemented yet"
+            else:
+                non_float_locs.append(regalloc.loc(arg))
+                non_float_regs.append(reg)
+
+            if arg.type == FLOAT:
+                assert 0, "not implemented yet"
+            else:
+                num += 1
+                count += 1
+
+        # spill variables that need to be saved around calls
+        regalloc.before_call(save_all_regs=2)
+
+        # remap values stored in core registers
+        remap_frame_layout(self, non_float_locs, non_float_regs, r.r0)
+
+        #the actual call
+        self.mc.bl_abs(adr)
+        self.mark_gc_roots(force_index)
+        regalloc.possibly_free_vars(args)
+        # readjust the sp in case we passed some args on the stack
+        if n > 0:
+            assert 0, "not implemented yet"
+
+        # restore the arguments stored on the stack
+        if result is not None:
+            resloc = regalloc.after_call(result)
+        self.mc.trap()
+
+    def write_new_force_index(self):
+        # for shadowstack only: get a new, unused force_index number and
+        # write it to FORCE_INDEX_OFS.  Used to record the call shape
+        # (i.e. where the GC pointers are in the stack) around a CALL
+        # instruction that doesn't already have a force_index.
+        gcrootmap = self.cpu.gc_ll_descr.gcrootmap
+        if gcrootmap and gcrootmap.is_shadow_stack:
+            clt = self.current_clt
+            force_index = clt.reserve_and_record_some_faildescr_index()
+            self._write_fail_index(force_index)
+            return force_index
+        else:
+            return 0
+
     def emit_debug_merge_point(self, op, arglocs, regalloc):
         pass
 
diff --git a/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py b/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py
--- a/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py
@@ -230,14 +230,16 @@
                     i += 4
                     continue
             elif res == self.STACK_LOC:
-                stack_loc = decode32(enc, i+1)
+                stack_location = decode32(enc, i+1)
                 i += 4
                 if group == self.FLOAT_TYPE:
-                    value = decode64(stack, frame_depth - stack_loc*WORD)
+                    value = decode64(stack, frame_depth - stack_location*WORD)
                     self.fail_boxes_float.setitem(fail_index, value)
                     continue
                 else:
-                    value = decode32(spilling_area, spilling_area - stack_loc * WORD)
+                    #value = decode32(spilling_area, spilling_area - stack_location * WORD)
+                    #import pdb; pdb.set_trace()
+                    value = decode32(spilling_area, spilling_depth - stack_location * WORD)
             else: # REG_LOC
                 reg = ord(enc[i])
                 if group == self.FLOAT_TYPE:
@@ -674,6 +676,15 @@
                 self.mc.slw(resloc.value, resloc.value, r.r0.value)
                 self.mc.sraw(resloc.value, resloc.value, r.r0.value)
 
+    def mark_gc_roots(self, force_index, use_copy_area=False):
+        if force_index < 0:
+            return     # not needed
+        gcrootmap = self.cpu.gc_ll_descr.gcrootmap
+        if gcrootmap:
+            mark = self._regalloc.get_mark_gc_roots(gcrootmap, use_copy_area)
+            assert gcrootmap.is_shadow_stack
+            gcrootmap.write_callshape(mark, force_index)
+
 def make_operations():
     def not_implemented(builder, trace_op, cpu, *rest_args):
         raise NotImplementedError, trace_op
diff --git a/pypy/jit/backend/ppc/ppcgen/regalloc.py b/pypy/jit/backend/ppc/ppcgen/regalloc.py
--- a/pypy/jit/backend/ppc/ppcgen/regalloc.py
+++ b/pypy/jit/backend/ppc/ppcgen/regalloc.py
@@ -18,6 +18,7 @@
 from pypy.jit.backend.ppc.ppcgen import locations
 from pypy.rpython.lltypesystem import rffi, lltype, rstr
 from pypy.jit.backend.llsupport import symbolic
+from pypy.jit.codewriter.effectinfo import EffectInfo
 import pypy.jit.backend.ppc.ppcgen.register as r
 
 class TempInt(TempBox):
@@ -181,6 +182,21 @@
     def next_instruction(self):
         self.rm.next_instruction()
 
+    def before_call(self, force_store=[], save_all_regs=False):
+        self.rm.before_call(force_store, save_all_regs)
+
+    def after_call(self, v):
+        if v.type == FLOAT:
+            assert 0, "not implemented yet"
+        else:
+            return self.rm.after_call(v)
+
+    def call_result_location(self, v):
+        if v.type == FLOAT:
+            assert 0, "not implemented yet"
+        else:
+            return self.rm.call_result_location(v)
+
     def _ensure_value_is_boxed(self, thing, forbidden_vars=[]):
         box = None
         loc = None
@@ -548,6 +564,17 @@
     prepare_cast_ptr_to_int = prepare_same_as
     prepare_cast_int_to_ptr = prepare_same_as
 
+    def prepare_call(self, op):
+        effectinfo = op.getdescr().get_extra_info()
+        if effectinfo is not None:
+            oopspecindex = effectinfo.oopspecindex
+            if oopspecindex == EffectInfo.OS_MATH_SQRT:
+                args = self.prepare_op_math_sqrt(op, fcond)
+                self.assembler.emit_op_math_sqrt(op, args, self, fcond)
+                return
+        args = [imm(rffi.cast(lltype.Signed, op.getarg(0).getint()))]
+        return args
+
     def void(self, op):
         return []
 
diff --git a/pypy/jit/backend/ppc/ppcgen/register.py b/pypy/jit/backend/ppc/ppcgen/register.py
--- a/pypy/jit/backend/ppc/ppcgen/register.py
+++ b/pypy/jit/backend/ppc/ppcgen/register.py
@@ -18,3 +18,5 @@
                 r11, r12, r13, r14, r15, r16, r17, r18, 
                 r19, r20, r21, r22, r23, r24, r25, r26,
                 r27, r28, r29, r30]
+
+PARAM_REGS = [r3, r4, r5, r6, r7, r8, r9, r10]
diff --git a/pypy/jit/backend/ppc/runner.py b/pypy/jit/backend/ppc/runner.py
--- a/pypy/jit/backend/ppc/runner.py
+++ b/pypy/jit/backend/ppc/runner.py
@@ -51,40 +51,6 @@
         self.asm.assemble_bridge(faildescr, inputargs, operations,
                                        original_loop_token, log=log)
 
-    #def compile_bridge(self, descr, inputargs, operations, looptoken):
-    #    self.saved_descr = {}
-    #    self.patch_list = []
-    #    self.reg_map = {}
-    #    self.fail_box_count = 0
-
-    #    codebuilder = looptoken.codebuilder
-    #    # jump to the bridge
-    #    current_pos = codebuilder.get_relative_pos()
-    #    offset = current_pos - descr.patch_pos
-    #    codebuilder.b(offset)
-    #    codebuilder.patch_op(descr.patch_op)
-
-    #    # initialize registers from memory
-    #    self.next_free_register = 3
-    #    use_index = 0
-    #    for index, arg in enumerate(inputargs):
-    #        self.reg_map[arg] = self.next_free_register
-    #        addr = self.fail_boxes_int.get_addr_for_num(
-    #                descr.used_mem_indices[use_index])
-    #        codebuilder.load_from(self.next_free_register, addr)
-    #        self.next_free_register += 1
-    #        use_index += 1
-    #        
-    #    self._walk_trace_ops(codebuilder, operations)
-    #    self._make_epilogue(codebuilder)
-
-    #    f = codebuilder.assemble()
-    #    looptoken.ppc_code = f
-    #    looptoken.codebuilder = codebuilder
-
-    #    self.total_compiled_bridges += 1
-    #    self.teardown()
-
     # set value in fail_boxes_int
     def set_future_value_int(self, index, value_int):
         self.asm.fail_boxes_int.setitem(index, value_int)
diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -1257,7 +1257,7 @@
                 else:
                     assert 0
                 assert type(got) == type(val)
-                assert got == val
+                #assert got == val
 
     def test_compile_bridge_float(self):
         if not self.cpu.supports_floats:

From noreply at buildbot.pypy.org  Fri Oct 28 19:46:56 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 28 Oct 2011 19:46:56 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-multidim: basic slice support
Message-ID: <20111028174656.62A7F820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: numpy-multidim
Changeset: r48583:252e03277a09
Date: 2011-10-28 19:46 +0200
http://bitbucket.org/pypy/pypy/changeset/252e03277a09/

Log:	basic slice support

diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -53,7 +53,10 @@
         return False
 
     def decode_index4(self, w_idx, size):
-        return (self.int_w(w_idx), 0, 0, 1)
+        if isinstance(w_idx, IntObject):
+            return (self.int_w(w_idx), 0, 0, 1)
+        else:
+            return (0, size, 1, size)
 
     @specialize.argtype(1)
     def wrap(self, obj):
@@ -63,7 +66,7 @@
             return BoolObject(obj)
         elif isinstance(obj, int):
             return IntObject(obj)
-        raise Exception
+        return obj
 
     def newlist(self, items):
         return ListObject(items)
@@ -138,6 +141,9 @@
     def __init__(self, items):
         self.items = items
 
+class SliceObject(W_Root):
+    tp = FakeSpace.w_slice
+
 class InterpreterState(object):
     def __init__(self, code):
         self.code = code
@@ -210,11 +216,11 @@
 
     def execute(self, interp):
         w_lhs = self.lhs.execute(interp)
+        if isinstance(self.rhs, SliceConstant):
+            w_rhs = self.rhs.wrap(interp.space)
+        else:
+            w_rhs = self.rhs.execute(interp)
         assert isinstance(w_lhs, BaseArray)
-        if isinstance(self.rhs, SliceConstant):
-            # XXX interface has changed on multidim branch
-            raise NotImplementedError
-        w_rhs = self.rhs.execute(interp)
         if self.name == '+':
             w_res = w_lhs.descr_add(interp.space, w_rhs)
         elif self.name == '*':
@@ -223,12 +229,10 @@
             w_res = w_lhs.descr_sub(interp.space, w_rhs)            
         elif self.name == '->':
             if isinstance(w_rhs, Scalar):
-                index = int(interp.space.float_w(
-                    w_rhs.value.wrap(interp.space)))
-                dtype = interp.space.fromcache(W_Float64Dtype)
-                return Scalar(dtype, w_lhs.get_concrete().eval(index))
-            else:
-                raise NotImplementedError
+                w_rhs = w_rhs.eval(0).wrap(interp.space)
+                assert isinstance(w_rhs, FloatObject)
+                w_rhs = IntObject(int(w_rhs.floatval))
+            w_res = w_lhs.descr_getitem(interp.space, w_rhs)
         else:
             raise NotImplementedError
         if not isinstance(w_res, BaseArray):
@@ -293,6 +297,9 @@
     def __init__(self):
         pass
 
+    def wrap(self, space):
+        return SliceObject()
+
     def __repr__(self):
         return 'slice()'
 
diff --git a/pypy/module/micronumpy/test/test_compile.py b/pypy/module/micronumpy/test/test_compile.py
--- a/pypy/module/micronumpy/test/test_compile.py
+++ b/pypy/module/micronumpy/test/test_compile.py
@@ -161,10 +161,9 @@
         assert interp.results[0].value.val == 256
 
     def test_slice(self):
-        py.test.skip("in progress")
         interp = self.run("""
         a = [1,2,3,4]
         b = a -> :
         b -> 3
         """)
-        assert interp.results[0].value.val == 3
+        assert interp.results[0].value.val == 4

From noreply at buildbot.pypy.org  Fri Oct 28 20:34:20 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Fri, 28 Oct 2011 20:34:20 +0200 (CEST)
Subject: [pypy-commit] pypy jit-raw-array-of-struct: progress
Message-ID: <20111028183420.57241820C2@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: jit-raw-array-of-struct
Changeset: r48584:b968a2c1909c
Date: 2011-10-28 14:34 -0400
http://bitbucket.org/pypy/pypy/changeset/b968a2c1909c/

Log:	progress

diff --git a/pypy/jit/codewriter/assembler.py b/pypy/jit/codewriter/assembler.py
--- a/pypy/jit/codewriter/assembler.py
+++ b/pypy/jit/codewriter/assembler.py
@@ -79,6 +79,7 @@
             if TYPE is lltype.SingleFloat:
                 value = longlong.singlefloat2int(value)
             if not isinstance(value, (llmemory.AddressAsInt,
+                                      llmemory.ItemOffset,
                                       ComputedIntSymbolic)):
                 value = lltype.cast_primitive(lltype.Signed, value)
                 if allow_short and -128 <= value <= 127:
diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -788,6 +788,16 @@
             return SpaceOperation('setinteriorfield_gc_%s' % kind, args,
                                   op.result)
 
+    def rewrite_op_getarraysubstruct(self, op):
+        array = op.args[0]
+        assert not self._is_gc(array)
+        tmp = varoftype(lltype.Signed)
+        element_size = llmemory.sizeof(array.concretetype.TO.OF)
+        return [
+            SpaceOperation("int_mul", [op.args[1], Constant(element_size, lltype.Signed)], tmp),
+            SpaceOperation("int_add", [op.args[0], tmp], op.result),
+        ]
+
     def _rewrite_equality(self, op, opname):
         arg0, arg1 = op.args
         if isinstance(arg0, Constant) and not arg0.value:
diff --git a/pypy/jit/codewriter/test/test_jtransform.py b/pypy/jit/codewriter/test/test_jtransform.py
--- a/pypy/jit/codewriter/test/test_jtransform.py
+++ b/pypy/jit/codewriter/test/test_jtransform.py
@@ -1140,4 +1140,24 @@
     assert op1.opname == 'mark_opaque_ptr'
     assert op1.args == [v1]
     assert op1.result is None
-    assert op2 is None
\ No newline at end of file
+    assert op2 is None
+
+def test_getarraysubstruct():
+    S = lltype.Struct("S", ("x", lltype.Float), ("y", lltype.Float))
+    v1 = varoftype(lltype.Ptr(lltype.Array(S)))
+    v2 = varoftype(lltype.Signed)
+    v3 = varoftype(lltype.Ptr(S))
+
+    op = SpaceOperation('getarraysubstruct', [v1, v2], v3)
+    [op1, op2] = Transformer().rewrite_operation(op)
+
+    assert op1.opname == "int_mul"
+    assert len(op1.args) == 2
+    assert op1.args[0] == v2
+    assert op1.args[1].value.TYPE == S
+    assert op1.args[1].value.repeat == 1
+    assert op1.result is not None
+
+    assert op2.opname == "int_add"
+    assert op2.args == [v1, op1.result]
+    assert op2.result == v3
\ No newline at end of file
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -3559,6 +3559,29 @@
         assert res == 0
         self.check_loops({"int_sub": 1, "int_gt": 1, "guard_true": 1, "jump": 1})
 
+    def test_raw_array_of_structs(self):
+        POINT = lltype.Struct("POINT",
+            ("x", lltype.Signed),
+            ("y", lltype.Signed),
+        )
+        ARRAY = lltype.Array(POINT)
+        myjitdriver = JitDriver(greens = [], reds=["vals", "i", "n", "result"])
+        def f(n):
+            vals = lltype.malloc(ARRAY, n, flavor="raw")
+            for i in xrange(n):
+                vals[i].x = n
+                vals[i].y = 2
+            result = 0.0
+            i = 0
+            while i < n:
+                myjitdriver.jit_merge_point(vals=vals, i=i, n=n, result=result)
+                result += vals[i].x * vals[i].y
+                i += 1
+            lltype.free(vals, flavor="raw")
+            return result
+        res = self.meta_interp(f, [10])
+        assert res == f(10)
+        self.check_loops({})
 
 
 class TestLLtype(BaseLLtypeTests, LLJitMixin):
diff --git a/pypy/rpython/lltypesystem/llmemory.py b/pypy/rpython/lltypesystem/llmemory.py
--- a/pypy/rpython/lltypesystem/llmemory.py
+++ b/pypy/rpython/lltypesystem/llmemory.py
@@ -532,6 +532,8 @@
         return self.adr == cast_int_to_adr(other)
     def __ne__(self, other):
         return self.adr != cast_int_to_adr(other)
+    def __add__(self, other):
+        return AddressAsInt(self.adr + other)
     def __nonzero__(self):
         return bool(self.adr)
     def __repr__(self):
@@ -879,7 +881,7 @@
         assert isinstance(s_from, SomeAddress)
         assert isinstance(s_to, SomeAddress)
         assert isinstance(s_size, SomeInteger)
-    
+
     def specialize_call(self, hop):
         hop.exception_cannot_occur()
         v_list = hop.inputargs(Address, Address, lltype.Signed)

From noreply at buildbot.pypy.org  Fri Oct 28 20:39:46 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 28 Oct 2011 20:39:46 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-multidim: better support for slices
Message-ID: <20111028183946.8387D820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: numpy-multidim
Changeset: r48585:a2490f74710a
Date: 2011-10-28 20:38 +0200
http://bitbucket.org/pypy/pypy/changeset/a2490f74710a/

Log:	better support for slices

diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -56,7 +56,15 @@
         if isinstance(w_idx, IntObject):
             return (self.int_w(w_idx), 0, 0, 1)
         else:
-            return (0, size, 1, size)
+            assert isinstance(w_idx, SliceObject)
+            start, stop, step = w_idx.start, w_idx.stop, w_idx.step
+            if step == 0:
+                return (0, size, 1, size)
+            if start < 0:
+                start += size
+            if stop < 0:
+                stop += size
+            return (start, stop, step, size//step)
 
     @specialize.argtype(1)
     def wrap(self, obj):
@@ -143,6 +151,10 @@
 
 class SliceObject(W_Root):
     tp = FakeSpace.w_slice
+    def __init__(self, start, stop, step):
+        self.start = start
+        self.stop = stop
+        self.step = step
 
 class InterpreterState(object):
     def __init__(self, code):
@@ -294,14 +306,17 @@
         return "[" + ", ".join([repr(item) for item in self.items]) + "]"
 
 class SliceConstant(Node):
-    def __init__(self):
-        pass
+    def __init__(self, start, stop, step):
+        # no negative support for now
+        self.start = start
+        self.stop = stop
+        self.step = step
 
     def wrap(self, space):
-        return SliceObject()
+        return SliceObject(self.start, self.stop, self.step)
 
     def __repr__(self):
-        return 'slice()'
+        return 'slice(%s,%s,%s)' % (self.start, self.stop, self.step)
 
 class Execute(Node):
     def __init__(self, expr):
@@ -392,8 +407,31 @@
         assert lgt >= 0
         if ':' in v:
             # a slice
-            assert v == ':'
-            return SliceConstant()
+            if v == ':':
+                return SliceConstant(0, 0, 0)
+            else:
+                l = v.split(':')
+                if len(l) == 2:
+                    one = l[0]
+                    two = l[1]
+                    if not one:
+                        one = 0
+                    else:
+                        one = int(one)
+                    return SliceConstant(int(l[0]), int(l[1]), 1)
+                else:
+                    three = int(l[2])
+                    # all can be empty
+                    if l[0]:
+                        one = int(l[0])
+                    else:
+                        one = 0
+                    if l[1]:
+                        two = int(l[1])
+                    else:
+                        two = -1
+                    return SliceConstant(one, two, three)
+                
         if v[0] == '[':
             return ArrayConstant([self.parse_constant(elem)
                                   for elem in v[1:lgt].split(",")])
diff --git a/pypy/module/micronumpy/test/test_compile.py b/pypy/module/micronumpy/test/test_compile.py
--- a/pypy/module/micronumpy/test/test_compile.py
+++ b/pypy/module/micronumpy/test/test_compile.py
@@ -167,3 +167,12 @@
         b -> 3
         """)
         assert interp.results[0].value.val == 4
+
+    def test_slice_step(self):
+        interp = self.run("""
+        a = |30|
+        b = a -> ::2
+        b -> 3
+        """)
+        assert interp.results[0].value.val == 6
+        

From noreply at buildbot.pypy.org  Fri Oct 28 20:41:18 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Fri, 28 Oct 2011 20:41:18 +0200 (CEST)
Subject: [pypy-commit] pypy jit-raw-array-of-struct: test fix
Message-ID: <20111028184118.6E32F820C2@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: jit-raw-array-of-struct
Changeset: r48586:122a250914a5
Date: 2011-10-28 14:41 -0400
http://bitbucket.org/pypy/pypy/changeset/122a250914a5/

Log:	test fix

diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -3564,7 +3564,7 @@
             ("x", lltype.Signed),
             ("y", lltype.Signed),
         )
-        ARRAY = lltype.Array(POINT)
+        ARRAY = lltype.Array(POINT, hints={"nolength": True})
         myjitdriver = JitDriver(greens = [], reds=["vals", "i", "n", "result"])
         def f(n):
             vals = lltype.malloc(ARRAY, n, flavor="raw")

From noreply at buildbot.pypy.org  Fri Oct 28 21:03:09 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 28 Oct 2011 21:03:09 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-multidim: write another test, I'm a moron
Message-ID: <20111028190309.D3FC8820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: numpy-multidim
Changeset: r48587:d5d0e247d026
Date: 2011-10-28 21:01 +0200
http://bitbucket.org/pypy/pypy/changeset/d5d0e247d026/

Log:	write another test, I'm a moron

diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -74,7 +74,9 @@
             return BoolObject(obj)
         elif isinstance(obj, int):
             return IntObject(obj)
-        return obj
+        elif isinstance(obj, W_Root):
+            return obj
+        raise NotImplementedError
 
     def newlist(self, items):
         return ListObject(items)
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -184,6 +184,17 @@
         # This is 3, not 2 because there is a bridge for the exit.
         self.check_loop_count(3)
 
+    def test_slice(self):
+        result = self.run("""
+        a = |30| * 3
+        b = a -> ::3
+        c = b + b
+        c -> 3
+        """)
+        assert result == 27 * 2
+        self.check_loops({'int_mul': 1, 'getarrayitem_raw': 2, 'float_mul': 2,
+                          'setarrayitem_raw': 1, 'int_add': 1,
+                          'int_lt': 1, 'guard_true': 1, 'jump': 1})
 
 class TestNumpyOld(LLJitMixin):
     def setup_class(cls):

From noreply at buildbot.pypy.org  Fri Oct 28 21:03:11 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 28 Oct 2011 21:03:11 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-multidim: rename a horrible name,
	this one is for alex
Message-ID: <20111028190311.07AE3820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: numpy-multidim
Changeset: r48588:14002f5da120
Date: 2011-10-28 21:02 +0200
http://bitbucket.org/pypy/pypy/changeset/14002f5da120/

Log:	rename a horrible name, this one is for alex

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -257,7 +257,7 @@
         concrete = self.get_concrete()
         return space.wrap("[" + " ".join(concrete._getnums(True)) + "]")
 
-    def _single_item_at_index(self, space, w_idx):
+    def _index_of_single_item(self, space, w_idx):
         # we assume C ordering for now
         if space.isinstance_w(w_idx, space.w_int):
             idx = space.int_w(w_idx)
@@ -350,14 +350,14 @@
 
     def descr_getitem(self, space, w_idx):
         if self._single_item_result(space, w_idx):
-            item = self._single_item_at_index(space, w_idx)
+            item = self._index_of_single_item(space, w_idx)
             return self.get_concrete().eval(item).wrap(space)
         return space.wrap(self._create_slice(space, w_idx))
 
     def descr_setitem(self, space, w_idx, w_value):
         self.invalidated()
         if self._single_item_result(space, w_idx):
-            item = self._single_item_at_index(space, w_idx)
+            item = self._index_of_single_item(space, w_idx)
             self.get_concrete().setitem_w(space, item, w_value)
             return
         concrete = self.get_concrete()

From noreply at buildbot.pypy.org  Fri Oct 28 21:20:18 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 28 Oct 2011 21:20:18 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-multidim: another test. make it work
Message-ID: <20111028192018.3CDFA820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: numpy-multidim
Changeset: r48589:46409ef7c41a
Date: 2011-10-28 21:19 +0200
http://bitbucket.org/pypy/pypy/changeset/46409ef7c41a/

Log:	another test. make it work

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -621,6 +621,7 @@
     # XXX we might want to provide a custom finder of where we look for
     #     a particular item, right now we'll do the calculations again
 
+    @jit.unroll_safe
     def calc_index(self, item):
         index = []
         _item = item
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -186,15 +186,16 @@
 
     def test_slice(self):
         result = self.run("""
-        a = |30| * 3
+        a = |30|
         b = a -> ::3
         c = b + b
         c -> 3
         """)
-        assert result == 27 * 2
-        self.check_loops({'int_mul': 1, 'getarrayitem_raw': 2, 'float_mul': 2,
-                          'setarrayitem_raw': 1, 'int_add': 1,
+        assert result == 18
+        self.check_loops({'int_mul': 1, 'getarrayitem_raw': 2, 'float_add': 1,
+                          'setarrayitem_raw': 1, 'int_add': 3,
                           'int_lt': 1, 'guard_true': 1, 'jump': 1})
+        # XXX int_add should be 1, not 3, think about it
 
 class TestNumpyOld(LLJitMixin):
     def setup_class(cls):
@@ -204,23 +205,6 @@
         cls.space = FakeSpace()
         cls.float64_dtype = cls.space.fromcache(W_Float64Dtype)
     
-    def test_slice(self):
-        def f(i):
-            step = 3
-            ar = NDimArray(step*i, dtype=self.float64_dtype)
-            new_sig = signature.Signature.find_sig([
-                NDimSlice.signature, ar.signature
-            ])
-            s = NDimSlice(0, step*i, step, i, ar, new_sig)
-            v = interp_ufuncs.get(self.space).add.call(self.space, [s, s])
-            return v.get_concrete().eval(3).val
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
-        self.check_loops({'int_mul': 1, 'getarrayitem_raw': 2, 'float_add': 1,
-                          'setarrayitem_raw': 1, 'int_add': 1,
-                          'int_lt': 1, 'guard_true': 1, 'jump': 1})
-        assert result == f(5)
-
     def test_slice2(self):
         def f(i):
             step1 = 2
diff --git a/pypy/rpython/rlist.py b/pypy/rpython/rlist.py
--- a/pypy/rpython/rlist.py
+++ b/pypy/rpython/rlist.py
@@ -668,6 +668,7 @@
     ll_delitem_nonneg(dum_nocheck, l, index)
     return res
 
+ at jit.look_inside_iff(lambda l: jit.isvirtual(l))
 def ll_reverse(l):
     length = l.ll_length()
     i = 0
@@ -678,7 +679,6 @@
         l.ll_setitem_fast(length_1_i, tmp)
         i += 1
         length_1_i -= 1
-ll_reverse.oopspec = 'list.reverse(l)'
 
 def ll_getitem_nonneg(func, l, index):
     ll_assert(index >= 0, "unexpectedly negative list getitem index")

From noreply at buildbot.pypy.org  Fri Oct 28 22:09:44 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 28 Oct 2011 22:09:44 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-multidim: make shape and chunks
	non-resizable lists
Message-ID: <20111028200944.5F67D820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: numpy-multidim
Changeset: r48590:d790f132a006
Date: 2011-10-28 22:08 +0200
http://bitbucket.org/pypy/pypy/changeset/d790f132a006/

Log:	make shape and chunks non-resizable lists

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -61,7 +61,7 @@
     dtype = space.interp_w(interp_dtype.W_Dtype,
         space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype)
     )
-    arr = NDimArray(size, shape, dtype=dtype)
+    arr = NDimArray(size, shape[:], dtype=dtype)
     i = 0
     for i in range(len(elems_w)):
         w_elem = elems_w[i]
@@ -345,8 +345,8 @@
                     shape[i] = -1
                 else:
                     shape[i] = lgt
-            shape = [i for i in shape if i != -1]
-        return NDimSlice(self, new_sig, chunks, shape)
+            shape = [i for i in shape if i != -1][:]
+        return NDimSlice(self, new_sig, chunks[:], shape)
 
     def descr_getitem(self, space, w_idx):
         if self._single_item_result(space, w_idx):
@@ -576,7 +576,7 @@
 
 class NDimSlice(ViewArray):
     signature = signature.BaseSignature()
-
+    
     def __init__(self, parent, signature, chunks, shape):
         ViewArray.__init__(self, parent, signature, shape)
         self.chunks = chunks

From noreply at buildbot.pypy.org  Fri Oct 28 22:09:45 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 28 Oct 2011 22:09:45 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-multidim: make shape and chunk immutable.
 no real benefits unless some array is constant.
Message-ID: <20111028200945.87101820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: numpy-multidim
Changeset: r48591:cd07f23f15fc
Date: 2011-10-28 22:09 +0200
http://bitbucket.org/pypy/pypy/changeset/cd07f23f15fc/

Log:	make shape and chunk immutable. no real benefits unless some array
	is constant.

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -71,6 +71,8 @@
 class BaseArray(Wrappable):
     _attrs_ = ["invalidates", "signature", "shape"]
 
+    _immutable_fields_ = ['shape[*]']
+
     def __init__(self, shape):
         self.invalidates = []
         self.shape = shape
@@ -577,6 +579,8 @@
 class NDimSlice(ViewArray):
     signature = signature.BaseSignature()
     
+    _immutable_fields_ = ['shape[*]', 'chunks[*]']
+
     def __init__(self, parent, signature, chunks, shape):
         ViewArray.__init__(self, parent, signature, shape)
         self.chunks = chunks

From noreply at buildbot.pypy.org  Fri Oct 28 22:11:12 2011
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 28 Oct 2011 22:11:12 +0200 (CEST)
Subject: [pypy-commit] pypy numpy-multidim: fix test, skip the old ones
Message-ID: <20111028201112.6BBDC820C2@wyvern.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski <fijall at gmail.com>
Branch: numpy-multidim
Changeset: r48592:4fde3633c155
Date: 2011-10-28 22:10 +0200
http://bitbucket.org/pypy/pypy/changeset/4fde3633c155/

Log:	fix test, skip the old ones

diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -192,13 +192,13 @@
         c -> 3
         """)
         assert result == 18
-        self.check_loops({'int_mul': 1, 'getarrayitem_raw': 2, 'float_add': 1,
+        self.check_loops({'int_mul': 2, 'getarrayitem_raw': 2, 'float_add': 1,
                           'setarrayitem_raw': 1, 'int_add': 3,
                           'int_lt': 1, 'guard_true': 1, 'jump': 1})
-        # XXX int_add should be 1, not 3, think about it
 
 class TestNumpyOld(LLJitMixin):
     def setup_class(cls):
+        py.test.skip("old")
         from pypy.module.micronumpy.compile import FakeSpace
         from pypy.module.micronumpy.interp_dtype import W_Float64Dtype
         

From noreply at buildbot.pypy.org  Sat Oct 29 01:30:51 2011
From: noreply at buildbot.pypy.org (gutworth)
Date: Sat, 29 Oct 2011 01:30:51 +0200 (CEST)
Subject: [pypy-commit] pypy default: add float.is_integer()
Message-ID: <20111028233051.21377820C2@wyvern.cs.uni-duesseldorf.de>

Author: Benjamin Peterson <benjamin at python.org>
Branch: 
Changeset: r48593:b9587add901b
Date: 2011-10-28 19:30 -0400
http://bitbucket.org/pypy/pypy/changeset/b9587add901b/

Log:	add float.is_integer()

diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py
--- a/pypy/objspace/std/floatobject.py
+++ b/pypy/objspace/std/floatobject.py
@@ -546,6 +546,9 @@
     # Try to return int.
     return space.newtuple([space.int(w_num), space.int(w_den)])
 
+def float_is_integer__Float(space, w_float):
+    return space.wrap(math.floor(w_float.floatval) == w_float.floatval)
+
 from pypy.objspace.std import floattype
 register_all(vars(), floattype)
 
diff --git a/pypy/objspace/std/floattype.py b/pypy/objspace/std/floattype.py
--- a/pypy/objspace/std/floattype.py
+++ b/pypy/objspace/std/floattype.py
@@ -12,6 +12,7 @@
 
 
 float_as_integer_ratio = SMM("as_integer_ratio", 1)
+float_is_integer = SMM("is_integer", 1)
 float_hex = SMM("hex", 1)
 
 def descr_conjugate(space, w_float):
diff --git a/pypy/objspace/std/test/test_floatobject.py b/pypy/objspace/std/test/test_floatobject.py
--- a/pypy/objspace/std/test/test_floatobject.py
+++ b/pypy/objspace/std/test/test_floatobject.py
@@ -63,6 +63,10 @@
     def setup_class(cls):
         cls.w_py26 = cls.space.wrap(sys.version_info >= (2, 6))
 
+    def test_isinteger(self):
+        assert (1.).is_integer()
+        assert not (1.1).is_integer()
+
     def test_conjugate(self):
         assert (1.).conjugate() == 1.
         assert (-1.).conjugate() == -1.
@@ -782,4 +786,4 @@
         # divide by 0
         raises(ZeroDivisionError, lambda: inf % 0)
         raises(ZeroDivisionError, lambda: inf // 0)
-        raises(ZeroDivisionError, divmod, inf, 0)
\ No newline at end of file
+        raises(ZeroDivisionError, divmod, inf, 0)

From noreply at buildbot.pypy.org  Sat Oct 29 01:30:52 2011
From: noreply at buildbot.pypy.org (gutworth)
Date: Sat, 29 Oct 2011 01:30:52 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20111028233052.605FC820C2@wyvern.cs.uni-duesseldorf.de>

Author: Benjamin Peterson <benjamin at python.org>
Branch: 
Changeset: r48594:b9fb5a2b414e
Date: 2011-10-28 19:30 -0400
http://bitbucket.org/pypy/pypy/changeset/b9fb5a2b414e/

Log:	merge heads

diff --git a/pypy/jit/metainterp/history.py b/pypy/jit/metainterp/history.py
--- a/pypy/jit/metainterp/history.py
+++ b/pypy/jit/metainterp/history.py
@@ -929,6 +929,9 @@
     def view(self, **kwds):
         pass
 
+    def clear(self):
+        pass
+
 class Stats(object):
     """For tests."""
 
@@ -943,6 +946,15 @@
         self.aborted_keys = []
         self.invalidated_token_numbers = set()
 
+    def clear(self):
+        del self.loops[:]
+        del self.locations[:]
+        del self.aborted_keys[:]
+        self.invalidated_token_numbers.clear()
+        self.compiled_count = 0
+        self.enter_count = 0
+        self.aborted_count = 0
+
     def set_history(self, history):
         self.operations = history.operations
 
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -976,6 +976,29 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_virtual_array_of_struct_forced(self):
+        ops = """
+        [f0, f1]
+        p0 = new_array(1, descr=complexarraydescr)
+        setinteriorfield_gc(p0, 0, f0, descr=complexrealdescr)
+        setinteriorfield_gc(p0, 0, f1, descr=compleximagdescr)
+        f2 = getinteriorfield_gc(p0, 0, descr=complexrealdescr)
+        f3 = getinteriorfield_gc(p0, 0, descr=compleximagdescr)
+        f4 = float_mul(f2, f3)
+        i0 = escape(f4, p0)
+        finish(i0)
+        """
+        expected = """
+        [f0, f1]
+        f2 = float_mul(f0, f1)
+        p0 = new_array(1, descr=complexarraydescr)
+        setinteriorfield_gc(p0, 0, f0, descr=complexrealdescr)
+        setinteriorfield_gc(p0, 0, f1, descr=compleximagdescr)
+        i0 = escape(f2, p0)
+        finish(i0)
+        """
+        self.optimize_loop(ops, expected)
+
     def test_nonvirtual_1(self):
         ops = """
         [i]
diff --git a/pypy/jit/metainterp/warmspot.py b/pypy/jit/metainterp/warmspot.py
--- a/pypy/jit/metainterp/warmspot.py
+++ b/pypy/jit/metainterp/warmspot.py
@@ -62,7 +62,7 @@
     clear_tcache()
     return jittify_and_run(interp, graph, args, backendopt=backendopt, **kwds)
 
-def jittify_and_run(interp, graph, args, repeat=1,
+def jittify_and_run(interp, graph, args, repeat=1, graph_and_interp_only=False,
                     backendopt=False, trace_limit=sys.maxint,
                     inline=False, loop_longevity=0, retrace_limit=5,
                     function_threshold=4,
@@ -93,6 +93,8 @@
         jd.warmstate.set_param_max_retrace_guards(max_retrace_guards)
         jd.warmstate.set_param_enable_opts(enable_opts)
     warmrunnerdesc.finish()
+    if graph_and_interp_only:
+        return interp, graph
     res = interp.eval_graph(graph, args)
     if not kwds.get('translate_support_code', False):
         warmrunnerdesc.metainterp_sd.profiler.finish()
@@ -157,6 +159,9 @@
 def get_stats():
     return pyjitpl._warmrunnerdesc.stats
 
+def reset_stats():
+    pyjitpl._warmrunnerdesc.stats.clear()
+
 def get_translator():
     return pyjitpl._warmrunnerdesc.translator
 
diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -13,6 +13,9 @@
         'empty': 'interp_numarray.zeros',
         'ones': 'interp_numarray.ones',
         'fromstring': 'interp_support.fromstring',
+
+        'True_': 'space.w_True',
+        'False_': 'space.w_False',
     }
 
     # ufuncs
diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py
--- a/pypy/module/micronumpy/compile.py
+++ b/pypy/module/micronumpy/compile.py
@@ -4,30 +4,52 @@
 """
 
 from pypy.interpreter.baseobjspace import InternalSpaceCache, W_Root
-from pypy.module.micronumpy.interp_dtype import W_Float64Dtype
-from pypy.module.micronumpy.interp_numarray import Scalar, SingleDimArray, BaseArray
+from pypy.module.micronumpy.interp_dtype import W_Float64Dtype, W_BoolDtype
+from pypy.module.micronumpy.interp_numarray import (Scalar, BaseArray,
+     descr_new_array, scalar_w, SingleDimArray)
+from pypy.module.micronumpy import interp_ufuncs
 from pypy.rlib.objectmodel import specialize
 
 
 class BogusBytecode(Exception):
     pass
 
-def create_array(dtype, size):
-    a = SingleDimArray(size, dtype=dtype)
-    for i in range(size):
-        dtype.setitem(a.storage, i, dtype.box(float(i % 10)))
-    return a
+class ArgumentMismatch(Exception):
+    pass
+
+class ArgumentNotAnArray(Exception):
+    pass
+
+class WrongFunctionName(Exception):
+    pass
+
+SINGLE_ARG_FUNCTIONS = ["sum", "prod", "max", "min", "all", "any", "unegative"]
 
 class FakeSpace(object):
     w_ValueError = None
     w_TypeError = None
+    w_None = None
+
+    w_bool = "bool"
+    w_int = "int"
+    w_float = "float"
+    w_list = "list"
+    w_long = "long"
+    w_tuple = 'tuple'
 
     def __init__(self):
         """NOT_RPYTHON"""
         self.fromcache = InternalSpaceCache(self).getorbuild
+        self.w_float64dtype = W_Float64Dtype(self)
 
     def issequence_w(self, w_obj):
-        return True
+        return isinstance(w_obj, ListObject) or isinstance(w_obj, SingleDimArray)
+
+    def isinstance_w(self, w_obj, w_tp):
+        return False
+
+    def decode_index4(self, w_idx, size):
+        return (self.int_w(w_idx), 0, 0, 1)
 
     @specialize.argtype(1)
     def wrap(self, obj):
@@ -39,72 +61,382 @@
             return IntObject(obj)
         raise Exception
 
+    def newlist(self, items):
+        return ListObject(items)
+
+    def listview(self, obj):
+        assert isinstance(obj, ListObject)
+        return obj.items
+
     def float(self, w_obj):
         assert isinstance(w_obj, FloatObject)
         return w_obj
 
     def float_w(self, w_obj):
+        assert isinstance(w_obj, FloatObject)        
         return w_obj.floatval
 
+    def int_w(self, w_obj):
+        if isinstance(w_obj, IntObject):
+            return w_obj.intval
+        elif isinstance(w_obj, FloatObject):
+            return int(w_obj.floatval)
+        raise NotImplementedError
+
+    def int(self, w_obj):
+        return w_obj
+
+    def is_true(self, w_obj):
+        assert isinstance(w_obj, BoolObject)
+        return w_obj.boolval
+
+    def is_w(self, w_obj, w_what):
+        return w_obj is w_what
+
+    def type(self, w_obj):
+        return w_obj.tp
+
+    def gettypefor(self, w_obj):
+        return None
+
+    def call_function(self, tp, w_dtype):
+        return w_dtype
+
+    @specialize.arg(1)
+    def interp_w(self, tp, what):
+        assert isinstance(what, tp)
+        return what
 
 class FloatObject(W_Root):
+    tp = FakeSpace.w_float
     def __init__(self, floatval):
         self.floatval = floatval
 
 class BoolObject(W_Root):
+    tp = FakeSpace.w_bool
     def __init__(self, boolval):
         self.boolval = boolval
 
 class IntObject(W_Root):
+    tp = FakeSpace.w_int
     def __init__(self, intval):
         self.intval = intval
 
+class ListObject(W_Root):
+    tp = FakeSpace.w_list
+    def __init__(self, items):
+        self.items = items
 
-space = FakeSpace()
+class InterpreterState(object):
+    def __init__(self, code):
+        self.code = code
+        self.variables = {}
+        self.results = []
 
-def numpy_compile(bytecode, array_size):
-    stack = []
-    i = 0
-    dtype = space.fromcache(W_Float64Dtype)
-    for b in bytecode:
-        if b == 'a':
-            stack.append(create_array(dtype, array_size))
-            i += 1
-        elif b == 'f':
-            stack.append(Scalar(dtype, dtype.box(1.2)))
-        elif b == '+':
-            right = stack.pop()
-            res = stack.pop().descr_add(space, right)
-            assert isinstance(res, BaseArray)
-            stack.append(res)
-        elif b == '-':
-            right = stack.pop()
-            res = stack.pop().descr_sub(space, right)
-            assert isinstance(res, BaseArray)
-            stack.append(res)
-        elif b == '*':
-            right = stack.pop()
-            res = stack.pop().descr_mul(space, right)
-            assert isinstance(res, BaseArray)
-            stack.append(res)
-        elif b == '/':
-            right = stack.pop()
-            res = stack.pop().descr_div(space, right)
-            assert isinstance(res, BaseArray)
-            stack.append(res)
-        elif b == '%':
-            right = stack.pop()
-            res = stack.pop().descr_mod(space, right)
-            assert isinstance(res, BaseArray)
-            stack.append(res)
-        elif b == '|':
-            res = stack.pop().descr_abs(space)
-            assert isinstance(res, BaseArray)
-            stack.append(res)
+    def run(self, space):
+        self.space = space
+        for stmt in self.code.statements:
+            stmt.execute(self)
+
+class Node(object):
+    def __eq__(self, other):
+        return (self.__class__ == other.__class__ and
+                self.__dict__ == other.__dict__)
+
+    def __ne__(self, other):
+        return not self == other
+
+    def wrap(self, space):
+        raise NotImplementedError
+
+    def execute(self, interp):
+        raise NotImplementedError
+
+class Assignment(Node):
+    def __init__(self, name, expr):
+        self.name = name
+        self.expr = expr
+
+    def execute(self, interp):
+        interp.variables[self.name] = self.expr.execute(interp)
+
+    def __repr__(self):
+        return "%% = %r" % (self.name, self.expr)
+
+class ArrayAssignment(Node):
+    def __init__(self, name, index, expr):
+        self.name = name
+        self.index = index
+        self.expr = expr
+
+    def execute(self, interp):
+        arr = interp.variables[self.name]
+        w_index = self.index.execute(interp).eval(0).wrap(interp.space)
+        w_val = self.expr.execute(interp).eval(0).wrap(interp.space)
+        arr.descr_setitem(interp.space, w_index, w_val)
+
+    def __repr__(self):
+        return "%s[%r] = %r" % (self.name, self.index, self.expr)
+
+class Variable(Node):
+    def __init__(self, name):
+        self.name = name
+
+    def execute(self, interp):
+        return interp.variables[self.name]
+
+    def __repr__(self):
+        return 'v(%s)' % self.name
+
+class Operator(Node):
+    def __init__(self, lhs, name, rhs):
+        self.name = name
+        self.lhs = lhs
+        self.rhs = rhs
+
+    def execute(self, interp):
+        w_lhs = self.lhs.execute(interp)
+        assert isinstance(w_lhs, BaseArray)
+        if isinstance(self.rhs, SliceConstant):
+            # XXX interface has changed on multidim branch
+            raise NotImplementedError
+        w_rhs = self.rhs.execute(interp)
+        if self.name == '+':
+            w_res = w_lhs.descr_add(interp.space, w_rhs)
+        elif self.name == '*':
+            w_res = w_lhs.descr_mul(interp.space, w_rhs)
+        elif self.name == '-':
+            w_res = w_lhs.descr_sub(interp.space, w_rhs)            
+        elif self.name == '->':
+            if isinstance(w_rhs, Scalar):
+                index = int(interp.space.float_w(
+                    w_rhs.value.wrap(interp.space)))
+                dtype = interp.space.fromcache(W_Float64Dtype)
+                return Scalar(dtype, w_lhs.get_concrete().eval(index))
+            else:
+                raise NotImplementedError
         else:
-            print "Unknown opcode: %s" % b
-            raise BogusBytecode()
-    if len(stack) != 1:
-        print "Bogus bytecode, uneven stack length"
-        raise BogusBytecode()
-    return stack[0]
+            raise NotImplementedError
+        if not isinstance(w_res, BaseArray):
+            dtype = interp.space.fromcache(W_Float64Dtype)
+            w_res = scalar_w(interp.space, dtype, w_res)
+        return w_res
+
+    def __repr__(self):
+        return '(%r %s %r)' % (self.lhs, self.name, self.rhs)
+
+class FloatConstant(Node):
+    def __init__(self, v):
+        self.v = float(v)
+
+    def __repr__(self):
+        return "Const(%s)" % self.v
+
+    def wrap(self, space):
+        return space.wrap(self.v)
+
+    def execute(self, interp):
+        dtype = interp.space.fromcache(W_Float64Dtype)
+        assert isinstance(dtype, W_Float64Dtype)
+        return Scalar(dtype, dtype.box(self.v))
+
+class RangeConstant(Node):
+    def __init__(self, v):
+        self.v = int(v)
+
+    def execute(self, interp):
+        w_list = interp.space.newlist(
+            [interp.space.wrap(float(i)) for i in range(self.v)])
+        dtype = interp.space.fromcache(W_Float64Dtype)
+        return descr_new_array(interp.space, None, w_list, w_dtype=dtype)
+
+    def __repr__(self):
+        return 'Range(%s)' % self.v
+
+class Code(Node):
+    def __init__(self, statements):
+        self.statements = statements
+
+    def __repr__(self):
+        return "\n".join([repr(i) for i in self.statements])
+
+class ArrayConstant(Node):
+    def __init__(self, items):
+        self.items = items
+
+    def wrap(self, space):
+        return space.newlist([item.wrap(space) for item in self.items])
+
+    def execute(self, interp):
+        w_list = self.wrap(interp.space)
+        dtype = interp.space.fromcache(W_Float64Dtype)
+        return descr_new_array(interp.space, None, w_list, w_dtype=dtype)
+
+    def __repr__(self):
+        return "[" + ", ".join([repr(item) for item in self.items]) + "]"
+
+class SliceConstant(Node):
+    def __init__(self):
+        pass
+
+    def __repr__(self):
+        return 'slice()'
+
+class Execute(Node):
+    def __init__(self, expr):
+        self.expr = expr
+
+    def __repr__(self):
+        return repr(self.expr)
+
+    def execute(self, interp):
+        interp.results.append(self.expr.execute(interp))
+
+class FunctionCall(Node):
+    def __init__(self, name, args):
+        self.name = name
+        self.args = args
+
+    def __repr__(self):
+        return "%s(%s)" % (self.name, ", ".join([repr(arg)
+                                                 for arg in self.args]))
+
+    def execute(self, interp):
+        if self.name in SINGLE_ARG_FUNCTIONS:
+            if len(self.args) != 1:
+                raise ArgumentMismatch
+            arr = self.args[0].execute(interp)
+            if not isinstance(arr, BaseArray):
+                raise ArgumentNotAnArray
+            if self.name == "sum":
+                w_res = arr.descr_sum(interp.space)
+            elif self.name == "prod":
+                w_res = arr.descr_prod(interp.space)
+            elif self.name == "max":
+                w_res = arr.descr_max(interp.space)
+            elif self.name == "min":
+                w_res = arr.descr_min(interp.space)
+            elif self.name == "any":
+                w_res = arr.descr_any(interp.space)
+            elif self.name == "all":
+                w_res = arr.descr_all(interp.space)
+            elif self.name == "unegative":
+                neg = interp_ufuncs.get(interp.space).negative
+                w_res = neg.call(interp.space, [arr])
+            else:
+                assert False # unreachable code
+            if isinstance(w_res, BaseArray):
+                return w_res
+            if isinstance(w_res, FloatObject):
+                dtype = interp.space.fromcache(W_Float64Dtype)
+            elif isinstance(w_res, BoolObject):
+                dtype = interp.space.fromcache(W_BoolDtype)
+            else:
+                dtype = None
+            return scalar_w(interp.space, dtype, w_res)
+        else:
+            raise WrongFunctionName
+
+class Parser(object):
+    def parse_identifier(self, id):
+        id = id.strip(" ")
+        #assert id.isalpha()
+        return Variable(id)
+
+    def parse_expression(self, expr):
+        tokens = [i for i in expr.split(" ") if i]
+        if len(tokens) == 1:
+            return self.parse_constant_or_identifier(tokens[0])
+        stack = []
+        tokens.reverse()
+        while tokens:
+            token = tokens.pop()
+            if token == ')':
+                raise NotImplementedError
+            elif self.is_identifier_or_const(token):
+                if stack:
+                    name = stack.pop().name
+                    lhs = stack.pop()
+                    rhs = self.parse_constant_or_identifier(token)
+                    stack.append(Operator(lhs, name, rhs))
+                else:
+                    stack.append(self.parse_constant_or_identifier(token))
+            else:
+                stack.append(Variable(token))
+        assert len(stack) == 1
+        return stack[-1]
+
+    def parse_constant(self, v):
+        lgt = len(v)-1
+        assert lgt >= 0
+        if ':' in v:
+            # a slice
+            assert v == ':'
+            return SliceConstant()
+        if v[0] == '[':
+            return ArrayConstant([self.parse_constant(elem)
+                                  for elem in v[1:lgt].split(",")])
+        if v[0] == '|':
+            return RangeConstant(v[1:lgt])
+        return FloatConstant(v)
+
+    def is_identifier_or_const(self, v):
+        c = v[0]
+        if ((c >= 'a' and c <= 'z') or (c >= 'A' and c <= 'Z') or
+            (c >= '0' and c <= '9') or c in '-.[|:'):
+            if v == '-' or v == "->":
+                return False
+            return True
+        return False
+
+    def parse_function_call(self, v):
+        l = v.split('(')
+        assert len(l) == 2
+        name = l[0]
+        cut = len(l[1]) - 1
+        assert cut >= 0
+        args = [self.parse_constant_or_identifier(id)
+                for id in l[1][:cut].split(",")]
+        return FunctionCall(name, args)
+
+    def parse_constant_or_identifier(self, v):
+        c = v[0]
+        if (c >= 'a' and c <= 'z') or (c >= 'A' and c <= 'Z'):
+            if '(' in v:
+                return self.parse_function_call(v)
+            return self.parse_identifier(v)
+        return self.parse_constant(v)
+
+    def parse_array_subscript(self, v):
+        v = v.strip(" ")
+        l = v.split("[")
+        lgt = len(l[1]) - 1
+        assert lgt >= 0
+        rhs = self.parse_constant_or_identifier(l[1][:lgt])
+        return l[0], rhs
+        
+    def parse_statement(self, line):
+        if '=' in line:
+            lhs, rhs = line.split("=")
+            lhs = lhs.strip(" ")
+            if '[' in lhs:
+                name, index = self.parse_array_subscript(lhs)
+                return ArrayAssignment(name, index, self.parse_expression(rhs))
+            else: 
+                return Assignment(lhs, self.parse_expression(rhs))
+        else:
+            return Execute(self.parse_expression(line))
+
+    def parse(self, code):
+        statements = []
+        for line in code.split("\n"):
+            if '#' in line:
+                line = line.split('#', 1)[0]
+            line = line.strip(" ")
+            if line:
+                statements.append(self.parse_statement(line))
+        return Code(statements)
+
+def numpy_compile(code):
+    parser = Parser()
+    return InterpreterState(parser.parse(code))
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -14,6 +14,27 @@
 any_driver = jit.JitDriver(greens=['signature'], reds=['i', 'size', 'self', 'dtype'])
 slice_driver = jit.JitDriver(greens=['signature'], reds=['i', 'j', 'step', 'stop', 'source', 'dest'])
 
+def descr_new_array(space, w_subtype, w_size_or_iterable, w_dtype=None):
+    l = space.listview(w_size_or_iterable)
+    if space.is_w(w_dtype, space.w_None):
+        w_dtype = None
+        for w_item in l:
+            w_dtype = interp_ufuncs.find_dtype_for_scalar(space, w_item, w_dtype)
+            if w_dtype is space.fromcache(interp_dtype.W_Float64Dtype):
+                break
+        if w_dtype is None:
+            w_dtype = space.w_None
+
+    dtype = space.interp_w(interp_dtype.W_Dtype,
+        space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype)
+    )
+    arr = SingleDimArray(len(l), dtype=dtype)
+    i = 0
+    for w_elem in l:
+        dtype.setitem_w(space, arr.storage, i, w_elem)
+        i += 1
+    return arr
+
 class BaseArray(Wrappable):
     _attrs_ = ["invalidates", "signature"]
 
@@ -32,27 +53,6 @@
     def add_invalidates(self, other):
         self.invalidates.append(other)
 
-    def descr__new__(space, w_subtype, w_size_or_iterable, w_dtype=None):
-        l = space.listview(w_size_or_iterable)
-        if space.is_w(w_dtype, space.w_None):
-            w_dtype = None
-            for w_item in l:
-                w_dtype = interp_ufuncs.find_dtype_for_scalar(space, w_item, w_dtype)
-                if w_dtype is space.fromcache(interp_dtype.W_Float64Dtype):
-                    break
-            if w_dtype is None:
-                w_dtype = space.w_None
-
-        dtype = space.interp_w(interp_dtype.W_Dtype,
-            space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype)
-        )
-        arr = SingleDimArray(len(l), dtype=dtype)
-        i = 0
-        for w_elem in l:
-            dtype.setitem_w(space, arr.storage, i, w_elem)
-            i += 1
-        return arr
-
     def _unaryop_impl(ufunc_name):
         def impl(self, space):
             return getattr(interp_ufuncs.get(space), ufunc_name).call(space, [self])
@@ -570,7 +570,7 @@
 
 BaseArray.typedef = TypeDef(
     'numarray',
-    __new__ = interp2app(BaseArray.descr__new__.im_func),
+    __new__ = interp2app(descr_new_array),
 
 
     __len__ = interp2app(BaseArray.descr_len),
diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py
--- a/pypy/module/micronumpy/interp_ufuncs.py
+++ b/pypy/module/micronumpy/interp_ufuncs.py
@@ -32,11 +32,17 @@
         return self.identity.wrap(space)
 
     def descr_call(self, space, __args__):
-        try:
-            args_w = __args__.fixedunpack(self.argcount)
-        except ValueError, e:
-            raise OperationError(space.w_TypeError, space.wrap(str(e)))
-        return self.call(space, args_w)
+        if __args__.keywords or len(__args__.arguments_w) < self.argcount:
+            raise OperationError(space.w_ValueError,
+                space.wrap("invalid number of arguments")
+            )
+        elif len(__args__.arguments_w) > self.argcount:
+            # The extra arguments should actually be the output array, but we
+            # don't support that yet.
+            raise OperationError(space.w_TypeError,
+                space.wrap("invalid number of arguments")
+            )
+        return self.call(space, __args__.arguments_w)
 
     def descr_reduce(self, space, w_obj):
         from pypy.module.micronumpy.interp_numarray import convert_to_array, Scalar
@@ -236,22 +242,20 @@
     return dt
 
 def find_dtype_for_scalar(space, w_obj, current_guess=None):
-    w_type = space.type(w_obj)
-
     bool_dtype = space.fromcache(interp_dtype.W_BoolDtype)
     long_dtype = space.fromcache(interp_dtype.W_LongDtype)
     int64_dtype = space.fromcache(interp_dtype.W_Int64Dtype)
 
-    if space.is_w(w_type, space.w_bool):
+    if space.isinstance_w(w_obj, space.w_bool):
         if current_guess is None or current_guess is bool_dtype:
             return bool_dtype
         return current_guess
-    elif space.is_w(w_type, space.w_int):
+    elif space.isinstance_w(w_obj, space.w_int):
         if (current_guess is None or current_guess is bool_dtype or
             current_guess is long_dtype):
             return long_dtype
         return current_guess
-    elif space.is_w(w_type, space.w_long):
+    elif space.isinstance_w(w_obj, space.w_long):
         if (current_guess is None or current_guess is bool_dtype or
             current_guess is long_dtype or current_guess is int64_dtype):
             return int64_dtype
diff --git a/pypy/module/micronumpy/test/test_compile.py b/pypy/module/micronumpy/test/test_compile.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/micronumpy/test/test_compile.py
@@ -0,0 +1,170 @@
+
+import py
+from pypy.module.micronumpy.compile import *
+
+class TestCompiler(object):
+    def compile(self, code):
+        return numpy_compile(code)
+    
+    def test_vars(self):
+        code = """
+        a = 2
+        b = 3
+        """
+        interp = self.compile(code)
+        assert isinstance(interp.code.statements[0], Assignment)
+        assert interp.code.statements[0].name == 'a'
+        assert interp.code.statements[0].expr.v == 2
+        assert interp.code.statements[1].name == 'b'
+        assert interp.code.statements[1].expr.v == 3
+
+    def test_array_literal(self):
+        code = "a = [1,2,3]"
+        interp = self.compile(code)
+        assert isinstance(interp.code.statements[0].expr, ArrayConstant)
+        st = interp.code.statements[0]
+        assert st.expr.items == [FloatConstant(1), FloatConstant(2),
+                                 FloatConstant(3)]
+    
+    def test_array_literal2(self):
+        code = "a = [[1],[2],[3]]"
+        interp = self.compile(code)
+        assert isinstance(interp.code.statements[0].expr, ArrayConstant)
+        st = interp.code.statements[0]
+        assert st.expr.items == [ArrayConstant([FloatConstant(1)]),
+                                 ArrayConstant([FloatConstant(2)]),
+                                 ArrayConstant([FloatConstant(3)])]
+
+    def test_expr_1(self):
+        code = "b = a + 1"
+        interp = self.compile(code)
+        assert (interp.code.statements[0].expr ==
+                Operator(Variable("a"), "+", FloatConstant(1)))
+
+    def test_expr_2(self):
+        code = "b = a + b - 3"
+        interp = self.compile(code)
+        assert (interp.code.statements[0].expr ==
+                Operator(Operator(Variable("a"), "+", Variable("b")), "-",
+                         FloatConstant(3)))
+
+    def test_expr_3(self):
+        # an equivalent of range
+        code = "a = |20|"
+        interp = self.compile(code)
+        assert interp.code.statements[0].expr == RangeConstant(20)
+
+    def test_expr_only(self):
+        code = "3 + a"
+        interp = self.compile(code)
+        assert interp.code.statements[0] == Execute(
+            Operator(FloatConstant(3), "+", Variable("a")))
+
+    def test_array_access(self):
+        code = "a -> 3"
+        interp = self.compile(code)
+        assert interp.code.statements[0] == Execute(
+            Operator(Variable("a"), "->", FloatConstant(3)))
+
+    def test_function_call(self):
+        code = "sum(a)"
+        interp = self.compile(code)
+        assert interp.code.statements[0] == Execute(
+            FunctionCall("sum", [Variable("a")]))
+
+    def test_comment(self):
+        code = """
+        # some comment
+        a = b + 3  # another comment
+        """
+        interp = self.compile(code)
+        assert interp.code.statements[0] == Assignment(
+            'a', Operator(Variable('b'), "+", FloatConstant(3)))
+
+class TestRunner(object):
+    def run(self, code):
+        interp = numpy_compile(code)
+        space = FakeSpace()
+        interp.run(space)
+        return interp
+
+    def test_one(self):
+        code = """
+        a = 3
+        b = 4
+        a + b
+        """
+        interp = self.run(code)
+        assert sorted(interp.variables.keys()) == ['a', 'b']
+        assert interp.results[0]
+
+    def test_array_add(self):
+        code = """
+        a = [1,2,3,4]
+        b = [4,5,6,5]
+        a + b
+        """
+        interp = self.run(code)
+        assert interp.results[0]._getnums(False) == ["5.0", "7.0", "9.0", "9.0"]
+
+    def test_array_getitem(self):
+        code = """
+        a = [1,2,3,4]
+        b = [4,5,6,5]
+        a + b -> 3
+        """
+        interp = self.run(code)
+        assert interp.results[0].value.val == 3 + 6
+        
+    def test_range_getitem(self):
+        code = """
+        r = |20| + 3
+        r -> 3
+        """
+        interp = self.run(code)
+        assert interp.results[0].value.val == 6
+
+    def test_sum(self):
+        code = """
+        a = [1,2,3,4,5]
+        r = sum(a)
+        r
+        """
+        interp = self.run(code)
+        assert interp.results[0].value.val == 15
+
+    def test_array_write(self):
+        code = """
+        a = [1,2,3,4,5]
+        a[3] = 15
+        a -> 3
+        """
+        interp = self.run(code)
+        assert interp.results[0].value.val == 15
+
+    def test_min(self):
+        interp = self.run("""
+        a = |30|
+        a[15] = -12
+        b = a + a
+        min(b)
+        """)
+        assert interp.results[0].value.val == -24
+
+    def test_max(self):
+        interp = self.run("""
+        a = |30|
+        a[13] = 128
+        b = a + a
+        max(b)
+        """)
+        assert interp.results[0].value.val == 256
+
+    def test_slice(self):
+        py.test.skip("in progress")
+        interp = self.run("""
+        a = [1,2,3,4]
+        b = a -> :
+        b -> 3
+        """)
+        assert interp.results[0].value.val == 3
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -36,37 +36,40 @@
         assert str(d) == "bool"
 
     def test_bool_array(self):
-        from numpy import array
+        import numpy
 
-        a = array([0, 1, 2, 2.5], dtype='?')
-        assert a[0] is False
+        a = numpy.array([0, 1, 2, 2.5], dtype='?')
+        assert a[0] is numpy.False_
         for i in xrange(1, 4):
-            assert a[i] is True
+            assert a[i] is numpy.True_
 
     def test_copy_array_with_dtype(self):
-        from numpy import array
-        a = array([0, 1, 2, 3], dtype=long)
+        import numpy
+
+        a = numpy.array([0, 1, 2, 3], dtype=long)
         # int on 64-bit, long in 32-bit
         assert isinstance(a[0], (int, long))
         b = a.copy()
         assert isinstance(b[0], (int, long))
 
-        a = array([0, 1, 2, 3], dtype=bool)
-        assert isinstance(a[0], bool)
+        a = numpy.array([0, 1, 2, 3], dtype=bool)
+        assert a[0] is numpy.False_
         b = a.copy()
-        assert isinstance(b[0], bool)
+        assert b[0] is numpy.False_
 
     def test_zeros_bool(self):
-        from numpy import zeros
-        a = zeros(10, dtype=bool)
+        import numpy
+
+        a = numpy.zeros(10, dtype=bool)
         for i in range(10):
-            assert a[i] is False
+            assert a[i] is numpy.False_
 
     def test_ones_bool(self):
-        from numpy import ones
-        a = ones(10, dtype=bool)
+        import numpy
+
+        a = numpy.ones(10, dtype=bool)
         for i in range(10):
-            assert a[i] is True
+            assert a[i] is numpy.True_
 
     def test_zeros_long(self):
         from numpy import zeros
@@ -77,7 +80,7 @@
 
     def test_ones_long(self):
         from numpy import ones
-        a = ones(10, dtype=bool)
+        a = ones(10, dtype=long)
         for i in range(10):
             assert isinstance(a[i], (int, long))
             assert a[1] == 1
@@ -96,8 +99,9 @@
 
     def test_bool_binop_types(self):
         from numpy import array, dtype
-        types = ('?','b','B','h','H','i','I','l','L','q','Q','f','d')
-        N = len(types)
+        types = [
+            '?', 'b', 'B', 'h', 'H', 'i', 'I', 'l', 'L', 'q', 'Q', 'f', 'd'
+        ]
         a = array([True], '?')
         for t in types:
             assert (a + array([0], t)).dtype is dtype(t)
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -214,7 +214,7 @@
     def test_add_other(self):
         from numpy import array
         a = array(range(5))
-        b = array(reversed(range(5)))
+        b = array(range(4, -1, -1))
         c = a + b
         for i in range(5):
             assert c[i] == 4
@@ -264,18 +264,19 @@
             assert b[i] == i - 5
 
     def test_mul(self):
-        from numpy import array, dtype
-        a = array(range(5))
+        import numpy
+
+        a = numpy.array(range(5))
         b = a * a
         for i in range(5):
             assert b[i] == i * i
 
-        a = array(range(5), dtype=bool)
+        a = numpy.array(range(5), dtype=bool)
         b = a * a
-        assert b.dtype is dtype(bool)
-        assert b[0] is False
+        assert b.dtype is numpy.dtype(bool)
+        assert b[0] is numpy.False_
         for i in range(1, 5):
-            assert b[i] is True
+            assert b[i] is numpy.True_
 
     def test_mul_constant(self):
         from numpy import array
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -24,10 +24,10 @@
     def test_wrong_arguments(self):
         from numpy import add, sin
 
-        raises(TypeError, add, 1)
+        raises(ValueError, add, 1)
         raises(TypeError, add, 1, 2, 3)
         raises(TypeError, sin, 1, 2)
-        raises(TypeError, sin)
+        raises(ValueError, sin)
 
     def test_single_item(self):
         from numpy import negative, sign, minimum
@@ -82,6 +82,8 @@
         b = negative(a)
         a[0] = 5.0
         assert b[0] == 5.0
+        a = array(range(30))
+        assert negative(a + a)[3] == -6
 
     def test_abs(self):
         from numpy import array, absolute
@@ -355,4 +357,4 @@
                 (3.5, 3),
                 (3, 3.5),
             ]:
-                assert ufunc(a, b) is func(a, b)
+                assert ufunc(a, b) == func(a, b)
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -1,253 +1,195 @@
 from pypy.jit.metainterp.test.support import LLJitMixin
 from pypy.module.micronumpy import interp_ufuncs, signature
-from pypy.module.micronumpy.compile import (numpy_compile, FakeSpace,
-    FloatObject, IntObject)
-from pypy.module.micronumpy.interp_dtype import W_Int32Dtype, W_Float64Dtype, W_Int64Dtype, W_UInt64Dtype
-from pypy.module.micronumpy.interp_numarray import (BaseArray, SingleDimArray,
-    SingleDimSlice, scalar_w)
+from pypy.module.micronumpy.compile import (FakeSpace,
+    FloatObject, IntObject, numpy_compile, BoolObject)
+from pypy.module.micronumpy.interp_numarray import (SingleDimArray,
+    SingleDimSlice)
 from pypy.rlib.nonconst import NonConstant
-from pypy.rpython.annlowlevel import llstr
-from pypy.rpython.test.test_llinterp import interpret
+from pypy.rpython.annlowlevel import llstr, hlstr
+from pypy.jit.metainterp.warmspot import reset_stats
+from pypy.jit.metainterp import pyjitpl
 
 import py
 
 
 class TestNumpyJIt(LLJitMixin):
-    def setup_class(cls):
-        cls.space = FakeSpace()
-        cls.float64_dtype = cls.space.fromcache(W_Float64Dtype)
-        cls.int64_dtype = cls.space.fromcache(W_Int64Dtype)
-        cls.uint64_dtype = cls.space.fromcache(W_UInt64Dtype)
-        cls.int32_dtype = cls.space.fromcache(W_Int32Dtype)
+    graph = None
+    interp = None
+        
+    def run(self, code):
+        space = FakeSpace()
+        
+        def f(code):
+            interp = numpy_compile(hlstr(code))
+            interp.run(space)
+            res = interp.results[-1]
+            w_res = res.eval(0).wrap(interp.space)
+            if isinstance(w_res, BoolObject):
+                return float(w_res.boolval)
+            elif isinstance(w_res, FloatObject):
+                return w_res.floatval
+            elif isinstance(w_res, IntObject):
+                return w_res.intval
+            else:
+                return -42.
+
+        if self.graph is None:
+            interp, graph = self.meta_interp(f, [llstr(code)],
+                                             listops=True,
+                                             backendopt=True,
+                                             graph_and_interp_only=True)
+            self.__class__.interp = interp
+            self.__class__.graph = graph
+
+        reset_stats()
+        pyjitpl._warmrunnerdesc.memory_manager.alive_loops.clear()
+        return self.interp.eval_graph(self.graph, [llstr(code)])
 
     def test_add(self):
-        def f(i):
-            ar = SingleDimArray(i, dtype=self.float64_dtype)
-            v = interp_ufuncs.get(self.space).add.call(self.space, [ar, ar])
-            return v.get_concrete().eval(3).val
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
+        result = self.run("""
+        a = |30|
+        b = a + a
+        b -> 3
+        """)
         self.check_loops({'getarrayitem_raw': 2, 'float_add': 1,
                           'setarrayitem_raw': 1, 'int_add': 1,
                           'int_lt': 1, 'guard_true': 1, 'jump': 1})
-        assert result == f(5)
+        assert result == 3 + 3
 
     def test_floatadd(self):
-        def f(i):
-            ar = SingleDimArray(i, dtype=self.float64_dtype)
-            v = interp_ufuncs.get(self.space).add.call(self.space, [
-                    ar,
-                    scalar_w(self.space, self.float64_dtype, self.space.wrap(4.5))
-                ],
-            )
-            assert isinstance(v, BaseArray)
-            return v.get_concrete().eval(3).val
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
+        result = self.run("""
+        a = |30| + 3
+        a -> 3
+        """)
+        assert result == 3 + 3
         self.check_loops({"getarrayitem_raw": 1, "float_add": 1,
                           "setarrayitem_raw": 1, "int_add": 1,
                           "int_lt": 1, "guard_true": 1, "jump": 1})
-        assert result == f(5)
 
     def test_sum(self):
-        space = self.space
-        float64_dtype = self.float64_dtype
-        int64_dtype = self.int64_dtype
-
-        def f(i):
-            if NonConstant(False):
-                dtype = int64_dtype
-            else:
-                dtype = float64_dtype
-            ar = SingleDimArray(i, dtype=dtype)
-            v = ar.descr_add(space, ar).descr_sum(space)
-            assert isinstance(v, FloatObject)
-            return v.floatval
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
+        result = self.run("""
+        a = |30|
+        b = a + a
+        sum(b)
+        """)
+        assert result == 2 * sum(range(30))
         self.check_loops({"getarrayitem_raw": 2, "float_add": 2,
                           "int_add": 1,
                           "int_lt": 1, "guard_true": 1, "jump": 1})
-        assert result == f(5)
 
     def test_prod(self):
-        space = self.space
-        float64_dtype = self.float64_dtype
-        int64_dtype = self.int64_dtype
-
-        def f(i):
-            if NonConstant(False):
-                dtype = int64_dtype
-            else:
-                dtype = float64_dtype
-            ar = SingleDimArray(i, dtype=dtype)
-            v = ar.descr_add(space, ar).descr_prod(space)
-            assert isinstance(v, FloatObject)
-            return v.floatval
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
+        result = self.run("""
+        a = |30|
+        b = a + a
+        prod(b)
+        """)
+        expected = 1
+        for i in range(30):
+            expected *= i * 2
+        assert result == expected
         self.check_loops({"getarrayitem_raw": 2, "float_add": 1,
                           "float_mul": 1, "int_add": 1,
                           "int_lt": 1, "guard_true": 1, "jump": 1})
-        assert result == f(5)
 
     def test_max(self):
-        space = self.space
-        float64_dtype = self.float64_dtype
-        int64_dtype = self.int64_dtype
-
-        def f(i):
-            if NonConstant(False):
-                dtype = int64_dtype
-            else:
-                dtype = float64_dtype
-            ar = SingleDimArray(i, dtype=dtype)
-            j = 0
-            while j < i:
-                ar.get_concrete().setitem(j, float64_dtype.box(float(j)))
-                j += 1
-            v = ar.descr_add(space, ar).descr_max(space)
-            assert isinstance(v, FloatObject)
-            return v.floatval
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
+        py.test.skip("broken, investigate")
+        result = self.run("""
+        a = |30|
+        a[13] = 128
+        b = a + a
+        max(b)
+        """)
+        assert result == 256
         self.check_loops({"getarrayitem_raw": 2, "float_add": 1,
-                          "float_gt": 1, "int_add": 1,
-                          "int_lt": 1, "guard_true": 1,
-                          "guard_false": 1, "jump": 1})
-        assert result == f(5)
+                          "float_mul": 1, "int_add": 1,
+                          "int_lt": 1, "guard_true": 1, "jump": 1})
 
     def test_min(self):
-        space = self.space
-        float64_dtype = self.float64_dtype
-        int64_dtype = self.int64_dtype
-
-        def f(i):
-            if NonConstant(False):
-                dtype = int64_dtype
-            else:
-                dtype = float64_dtype
-            ar = SingleDimArray(i, dtype=dtype)
-            j = 0
-            while j < i:
-                ar.get_concrete().setitem(j, float64_dtype.box(float(j)))
-                j += 1
-            v = ar.descr_add(space, ar).descr_min(space)
-            assert isinstance(v, FloatObject)
-            return v.floatval
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
+        py.test.skip("broken, investigate")
+        result = self.run("""
+        a = |30|
+        a[15] = -12
+        b = a + a
+        min(b)
+        """)
+        assert result == -24
         self.check_loops({"getarrayitem_raw": 2, "float_add": 1,
-                           "float_lt": 1, "int_add": 1,
-                           "int_lt": 1, "guard_true": 2,
-                           "jump": 1})
-        assert result == f(5)
-
-    def test_argmin(self):
-        space = self.space
-        float64_dtype = self.float64_dtype
-
-        def f(i):
-            ar = SingleDimArray(i, dtype=NonConstant(float64_dtype))
-            j = 0
-            while j < i:
-                ar.get_concrete().setitem(j, float64_dtype.box(float(j)))
-                j += 1
-            return ar.descr_add(space, ar).descr_argmin(space).intval
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
-        self.check_loops({"getarrayitem_raw": 2, "float_add": 1,
-                           "float_lt": 1, "int_add": 1,
-                           "int_lt": 1, "guard_true": 2,
-                           "jump": 1})
-        assert result == f(5)
-
-    def test_all(self):
-        space = self.space
-        float64_dtype = self.float64_dtype
-
-        def f(i):
-            ar = SingleDimArray(i, dtype=NonConstant(float64_dtype))
-            j = 0
-            while j < i:
-                ar.get_concrete().setitem(j, float64_dtype.box(1.0))
-                j += 1
-            return ar.descr_add(space, ar).descr_all(space).boolval
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
-        self.check_loops({"getarrayitem_raw": 2, "float_add": 1,
-                          "int_add": 1, "float_ne": 1,
-                          "int_lt": 1, "guard_true": 2, "jump": 1})
-        assert result == f(5)
+                          "float_mul": 1, "int_add": 1,
+                          "int_lt": 1, "guard_true": 1, "jump": 1})
 
     def test_any(self):
-        space = self.space
-        float64_dtype = self.float64_dtype
-
-        def f(i):
-            ar = SingleDimArray(i, dtype=NonConstant(float64_dtype))
-            return ar.descr_add(space, ar).descr_any(space).boolval
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
+        result = self.run("""
+        a = [0,0,0,0,0,0,0,0,0,0,0]
+        a[8] = -12
+        b = a + a
+        any(b)
+        """)
+        assert result == 1
         self.check_loops({"getarrayitem_raw": 2, "float_add": 1,
-                          "int_add": 1, "float_ne": 1, "guard_false": 1,
-                          "int_lt": 1, "guard_true": 1, "jump": 1})
-        assert result == f(5)
+                          "float_ne": 1, "int_add": 1,
+                          "int_lt": 1, "guard_true": 1, "jump": 1,
+                          "guard_false": 1})
 
     def test_already_forced(self):
-        space = self.space
-
-        def f(i):
-            ar = SingleDimArray(i, dtype=self.float64_dtype)
-            v1 = interp_ufuncs.get(self.space).add.call(space, [ar, scalar_w(space, self.float64_dtype, space.wrap(4.5))])
-            assert isinstance(v1, BaseArray)
-            v2 = interp_ufuncs.get(self.space).multiply.call(space, [v1, scalar_w(space, self.float64_dtype, space.wrap(4.5))])
-            v1.force_if_needed()
-            assert isinstance(v2, BaseArray)
-            return v2.get_concrete().eval(3).val
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
+        result = self.run("""
+        a = |30|
+        b = a + 4.5
+        b -> 5 # forces
+        c = b * 8
+        c -> 5
+        """)
+        assert result == (5 + 4.5) * 8
         # This is the sum of the ops for both loops, however if you remove the
         # optimization then you end up with 2 float_adds, so we can still be
         # sure it was optimized correctly.
         self.check_loops({"getarrayitem_raw": 2, "float_mul": 1, "float_add": 1,
                            "setarrayitem_raw": 2, "int_add": 2,
                            "int_lt": 2, "guard_true": 2, "jump": 2})
-        assert result == f(5)
 
     def test_ufunc(self):
-        space = self.space
-        def f(i):
-            ar = SingleDimArray(i, dtype=self.float64_dtype)
-            v1 = interp_ufuncs.get(self.space).add.call(space, [ar, ar])
-            v2 = interp_ufuncs.get(self.space).negative.call(space, [v1])
-            return v2.get_concrete().eval(3).val
-
-        result = self.meta_interp(f, [5], listops=True, backendopt=True)
+        result = self.run("""
+        a = |30|
+        b = a + a
+        c = unegative(b)
+        c -> 3
+        """)
+        assert result == -6
         self.check_loops({"getarrayitem_raw": 2, "float_add": 1, "float_neg": 1,
                           "setarrayitem_raw": 1, "int_add": 1,
                           "int_lt": 1, "guard_true": 1, "jump": 1,
         })
-        assert result == f(5)
 
-    def test_appropriate_specialization(self):
-        space = self.space
-        def f(i):
-            ar = SingleDimArray(i, dtype=self.float64_dtype)
-
-            v1 = interp_ufuncs.get(self.space).add.call(space, [ar, ar])
-            v2 = interp_ufuncs.get(self.space).negative.call(space, [v1])
-            v2.get_concrete()
-
-            for i in xrange(5):
-                v1 = interp_ufuncs.get(self.space).multiply.call(space, [ar, ar])
-                v2 = interp_ufuncs.get(self.space).negative.call(space, [v1])
-                v2.get_concrete()
-
-        self.meta_interp(f, [5], listops=True, backendopt=True)
+    def test_specialization(self):
+        self.run("""
+        a = |30|
+        b = a + a
+        c = unegative(b)
+        c -> 3
+        d = a * a
+        unegative(d)
+        d -> 3
+        d = a * a
+        unegative(d)
+        d -> 3
+        d = a * a
+        unegative(d)
+        d -> 3
+        d = a * a
+        unegative(d)
+        d -> 3
+        """)
         # This is 3, not 2 because there is a bridge for the exit.
         self.check_loop_count(3)
 
+
+class TestNumpyOld(LLJitMixin):
+    def setup_class(cls):
+        from pypy.module.micronumpy.compile import FakeSpace
+        from pypy.module.micronumpy.interp_dtype import W_Float64Dtype
+        
+        cls.space = FakeSpace()
+        cls.float64_dtype = cls.space.fromcache(W_Float64Dtype)
+    
     def test_slice(self):
         def f(i):
             step = 3
@@ -332,17 +274,3 @@
         result = self.meta_interp(f, [5], listops=True, backendopt=True)
         assert result == f(5)
 
-class TestTranslation(object):
-    def test_compile(self):
-        x = numpy_compile('aa+f*f/a-', 10)
-        x = x.compute()
-        assert isinstance(x, SingleDimArray)
-        assert x.size == 10
-        assert x.eval(0).val == 0
-        assert x.eval(1).val == ((1 + 1) * 1.2) / 1.2 - 1
-
-    def test_translation(self):
-        # we import main to check if the target compiles
-        from pypy.translator.goal.targetnumpystandalone import main
-
-        interpret(main, [llstr('af+'), 100])
diff --git a/pypy/module/pypyjit/test_pypy_c/test_containers.py b/pypy/module/pypyjit/test_pypy_c/test_containers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_containers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_containers.py
@@ -69,4 +69,51 @@
             i9 = int_add(i5, 1)
             --TICK--
             jump(..., descr=...)
+        """)
+
+    def test_non_virtual_dict(self):
+        def main(n):
+            i = 0
+            while i < n:
+                d = {str(i): i}
+                i += d[str(i)] - i + 1
+            return i
+
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loop, = log.loops_by_filename(self.filepath)
+        assert loop.match("""
+            i8 = int_lt(i5, i7)
+            guard_true(i8, descr=...)
+            guard_not_invalidated(descr=...)
+            p10 = call(ConstClass(ll_int_str), i5, descr=<GcPtrCallDescr>)
+            guard_no_exception(descr=...)
+            i12 = call(ConstClass(ll_strhash), p10, descr=<SignedCallDescr>)
+            p13 = new(descr=...)
+            p15 = new_array(8, descr=<dictentryArrayDescr>)
+            setfield_gc(p13, p15, descr=<GcPtrFieldDescr dicttable.entries .*>)
+            i17 = call(ConstClass(ll_dict_lookup_trampoline), p13, p10, i12, descr=<SignedCallDescr>)
+            setfield_gc(p13, 16, descr=<SignedFieldDescr dicttable.resize_counter 16>)
+            guard_no_exception(descr=...)
+            p20 = new_with_vtable(ConstClass(W_IntObject))
+            call(ConstClass(_ll_dict_setitem_lookup_done_trampoline), p13, p10, p20, i12, i17, descr=<VoidCallDescr>)
+            setfield_gc(p20, i5, descr=<SignedFieldDescr .*W_IntObject.inst_intval .*>)
+            guard_no_exception(descr=...)
+            i23 = call(ConstClass(ll_dict_lookup_trampoline), p13, p10, i12, descr=<SignedCallDescr>)
+            guard_no_exception(descr=...)
+            i26 = int_and(i23, .*)
+            i27 = int_is_true(i26)
+            guard_false(i27, descr=...)
+            p28 = getfield_gc(p13, descr=<GcPtrFieldDescr dicttable.entries .*>)
+            p29 = getinteriorfield_gc(p28, i23, descr=<InteriorFieldDescr <GcPtrFieldDescr dictentry.value .*>>)
+            guard_nonnull_class(p29, ConstClass(W_IntObject), descr=...)
+            i31 = getfield_gc_pure(p29, descr=<SignedFieldDescr .*W_IntObject.inst_intval .*>)
+            i32 = int_sub_ovf(i31, i5)
+            guard_no_overflow(descr=...)
+            i34 = int_add_ovf(i32, 1)
+            guard_no_overflow(descr=...)
+            i35 = int_add_ovf(i5, i34)
+            guard_no_overflow(descr=...)
+            --TICK--
+            jump(p0, p1, p2, p3, p4, i35, p13, i7, descr=<Loop0>)
         """)
\ No newline at end of file
diff --git a/pypy/rpython/lltypesystem/rstr.py b/pypy/rpython/lltypesystem/rstr.py
--- a/pypy/rpython/lltypesystem/rstr.py
+++ b/pypy/rpython/lltypesystem/rstr.py
@@ -20,6 +20,7 @@
 from pypy.rpython.rmodel import Repr
 from pypy.rpython.lltypesystem import llmemory
 from pypy.tool.sourcetools import func_with_new_name
+from pypy.rpython.lltypesystem.lloperation import llop
 
 # ____________________________________________________________
 #
@@ -364,8 +365,10 @@
             while lpos < rpos and s.chars[lpos] == ch:
                 lpos += 1
         if right:
-            while lpos < rpos and s.chars[rpos] == ch:
+            while lpos < rpos + 1 and s.chars[rpos] == ch:
                 rpos -= 1
+        if rpos < lpos:
+            return s.empty()
         r_len = rpos - lpos + 1
         result = s.malloc(r_len)
         s.copy_contents(s, result, lpos, 0, r_len)
diff --git a/pypy/rpython/test/test_rstr.py b/pypy/rpython/test/test_rstr.py
--- a/pypy/rpython/test/test_rstr.py
+++ b/pypy/rpython/test/test_rstr.py
@@ -372,12 +372,20 @@
             return const('!ab!').lstrip(const('!'))
         def right():
             return const('!ab!').rstrip(const('!'))
+        def empty():
+            return const('    ').strip(' ')
+        def left2():
+            return const('a  ').strip(' ')
         res = self.interpret(both, [])
         assert self.ll_to_string(res) == const('ab')
         res = self.interpret(left, [])
         assert self.ll_to_string(res) == const('ab!')
         res = self.interpret(right, [])
         assert self.ll_to_string(res) == const('!ab')
+        res = self.interpret(empty, [])
+        assert self.ll_to_string(res) == const('')
+        res = self.interpret(left2, [])
+        assert self.ll_to_string(res) == const('a')
 
     def test_upper(self):
         const = self.const

From noreply at buildbot.pypy.org  Sat Oct 29 01:37:57 2011
From: noreply at buildbot.pypy.org (gutworth)
Date: Sat, 29 Oct 2011 01:37:57 +0200 (CEST)
Subject: [pypy-commit] pypy default: handle special values
Message-ID: <20111028233757.F31AB820C2@wyvern.cs.uni-duesseldorf.de>

Author: Benjamin Peterson <benjamin at python.org>
Branch: 
Changeset: r48595:7cfece2a13d1
Date: 2011-10-28 19:37 -0400
http://bitbucket.org/pypy/pypy/changeset/7cfece2a13d1/

Log:	handle special values

diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py
--- a/pypy/objspace/std/floatobject.py
+++ b/pypy/objspace/std/floatobject.py
@@ -547,7 +547,10 @@
     return space.newtuple([space.int(w_num), space.int(w_den)])
 
 def float_is_integer__Float(space, w_float):
-    return space.wrap(math.floor(w_float.floatval) == w_float.floatval)
+    v = w_float.floatval
+    if not rfloat.isfinite(v):
+        return space.w_False
+    return space.wrap(math.floor(v) == v)
 
 from pypy.objspace.std import floattype
 register_all(vars(), floattype)
diff --git a/pypy/objspace/std/test/test_floatobject.py b/pypy/objspace/std/test/test_floatobject.py
--- a/pypy/objspace/std/test/test_floatobject.py
+++ b/pypy/objspace/std/test/test_floatobject.py
@@ -66,6 +66,8 @@
     def test_isinteger(self):
         assert (1.).is_integer()
         assert not (1.1).is_integer()
+        assert not float("inf").is_integer()
+        assert not float("nan").is_integer()
 
     def test_conjugate(self):
         assert (1.).conjugate() == 1.

From noreply at buildbot.pypy.org  Sat Oct 29 08:14:40 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sat, 29 Oct 2011 08:14:40 +0200 (CEST)
Subject: [pypy-commit] pypy jit-raw-array-of-struct: random changes
Message-ID: <20111029061440.4ABFB820C2@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: jit-raw-array-of-struct
Changeset: r48596:bb85820f08a3
Date: 2011-10-29 02:13 -0400
http://bitbucket.org/pypy/pypy/changeset/bb85820f08a3/

Log:	random changes

diff --git a/pypy/jit/backend/llgraph/llimpl.py b/pypy/jit/backend/llgraph/llimpl.py
--- a/pypy/jit/backend/llgraph/llimpl.py
+++ b/pypy/jit/backend/llgraph/llimpl.py
@@ -1477,7 +1477,7 @@
     return do_setinteriorfield_gc
 do_setinteriorfield_gc_int = new_setinteriorfield_gc(cast_from_int)
 do_setinteriorfield_gc_float = new_setinteriorfield_gc(cast_from_floatstorage)
-do_setinteriorfield_gc_ptr = new_setinteriorfield_gc(cast_from_ptr)        
+do_setinteriorfield_gc_ptr = new_setinteriorfield_gc(cast_from_ptr)
 
 def do_setfield_raw_int(struct, fieldnum, newvalue):
     STRUCT, fieldname = symbolic.TokenToField[fieldnum]
diff --git a/pypy/jit/codewriter/assembler.py b/pypy/jit/codewriter/assembler.py
--- a/pypy/jit/codewriter/assembler.py
+++ b/pypy/jit/codewriter/assembler.py
@@ -7,6 +7,7 @@
 from pypy.rlib.objectmodel import ComputedIntSymbolic
 from pypy.objspace.flow.model import Constant
 from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rffi
+from pypy.rpython.memory.lltypelayout import convert_offset_to_int
 
 
 class AssemblerError(Exception):
@@ -79,7 +80,7 @@
             if TYPE is lltype.SingleFloat:
                 value = longlong.singlefloat2int(value)
             if not isinstance(value, (llmemory.AddressAsInt,
-                                      llmemory.ItemOffset,
+                                      llmemory.AddressOffset,
                                       ComputedIntSymbolic)):
                 value = lltype.cast_primitive(lltype.Signed, value)
                 if allow_short and -128 <= value <= 127:
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -3566,19 +3566,20 @@
         )
         ARRAY = lltype.Array(POINT, hints={"nolength": True})
         myjitdriver = JitDriver(greens = [], reds=["vals", "i", "n", "result"])
-        def f(n):
-            vals = lltype.malloc(ARRAY, n, flavor="raw")
-            for i in xrange(n):
-                vals[i].x = n
-                vals[i].y = 2
+        def g(vals, n):
             result = 0.0
             i = 0
             while i < n:
                 myjitdriver.jit_merge_point(vals=vals, i=i, n=n, result=result)
                 result += vals[i].x * vals[i].y
                 i += 1
-            lltype.free(vals, flavor="raw")
             return result
+        def f(n):
+            with lltype.scoped_alloc(ARRAY, n) as vals:
+                for i in xrange(n):
+                    vals[i].x = n
+                    vals[i].y = 2
+                return g(vals, n)
         res = self.meta_interp(f, [10])
         assert res == f(10)
         self.check_loops({})

From noreply at buildbot.pypy.org  Sat Oct 29 08:14:41 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sat, 29 Oct 2011 08:14:41 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix this test for 32-bit machines
Message-ID: <20111029061441.71669820C2@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r48597:753627dbee28
Date: 2011-10-29 02:14 -0400
http://bitbucket.org/pypy/pypy/changeset/753627dbee28/

Log:	fix this test for 32-bit machines

diff --git a/pypy/module/pypyjit/test_pypy_c/test_containers.py b/pypy/module/pypyjit/test_pypy_c/test_containers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_containers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_containers.py
@@ -93,7 +93,7 @@
             p15 = new_array(8, descr=<dictentryArrayDescr>)
             setfield_gc(p13, p15, descr=<GcPtrFieldDescr dicttable.entries .*>)
             i17 = call(ConstClass(ll_dict_lookup_trampoline), p13, p10, i12, descr=<SignedCallDescr>)
-            setfield_gc(p13, 16, descr=<SignedFieldDescr dicttable.resize_counter 16>)
+            setfield_gc(p13, 16, descr=<SignedFieldDescr dicttable.resize_counter .*>)
             guard_no_exception(descr=...)
             p20 = new_with_vtable(ConstClass(W_IntObject))
             call(ConstClass(_ll_dict_setitem_lookup_done_trampoline), p13, p10, p20, i12, i17, descr=<VoidCallDescr>)

From noreply at buildbot.pypy.org  Sat Oct 29 19:44:59 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sat, 29 Oct 2011 19:44:59 +0200 (CEST)
Subject: [pypy-commit] pypy default: merged default
Message-ID: <20111029174459.7501A82A87@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r48599:a3a5ac0a2daf
Date: 2011-10-29 19:43 +0200
http://bitbucket.org/pypy/pypy/changeset/a3a5ac0a2daf/

Log:	merged default

diff --git a/pypy/module/pypyjit/test_pypy_c/test_containers.py b/pypy/module/pypyjit/test_pypy_c/test_containers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_containers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_containers.py
@@ -93,7 +93,7 @@
             p15 = new_array(8, descr=<dictentryArrayDescr>)
             setfield_gc(p13, p15, descr=<GcPtrFieldDescr dicttable.entries .*>)
             i17 = call(ConstClass(ll_dict_lookup_trampoline), p13, p10, i12, descr=<SignedCallDescr>)
-            setfield_gc(p13, 16, descr=<SignedFieldDescr dicttable.resize_counter 16>)
+            setfield_gc(p13, 16, descr=<SignedFieldDescr dicttable.resize_counter .*>)
             guard_no_exception(descr=...)
             p20 = new_with_vtable(ConstClass(W_IntObject))
             call(ConstClass(_ll_dict_setitem_lookup_done_trampoline), p13, p10, p20, i12, i17, descr=<VoidCallDescr>)

From noreply at buildbot.pypy.org  Sat Oct 29 19:44:58 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sat, 29 Oct 2011 19:44:58 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix so these test don't rely on dict
	ordering
Message-ID: <20111029174458.4DCE1820C2@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r48598:a257711072e5
Date: 2011-10-29 19:31 +0200
http://bitbucket.org/pypy/pypy/changeset/a257711072e5/

Log:	fix so these test don't rely on dict ordering

diff --git a/pypy/jit/metainterp/optimizeopt/virtualize.py b/pypy/jit/metainterp/optimizeopt/virtualize.py
--- a/pypy/jit/metainterp/optimizeopt/virtualize.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualize.py
@@ -294,7 +294,12 @@
         optforce.emit_operation(self.source_op)
         self.box = box = self.source_op.result
         for index in range(len(self._items)):
-            for descr, value in self._items[index].iteritems():
+            iteritems = self._items[index].iteritems()
+            # random order is fine, except for tests
+            if not we_are_translated():
+                iteritems = list(iteritems)
+                iteritems.sort(key = lambda (x, y): x.sort_key())
+            for descr, value in iteritems:
                 subbox = value.force_box(optforce)
                 op = ResOperation(rop.SETINTERIORFIELD_GC,
                     [box, ConstInt(index), subbox], None, descr=descr

From noreply at buildbot.pypy.org  Sun Oct 30 00:46:04 2011
From: noreply at buildbot.pypy.org (matthp)
Date: Sun, 30 Oct 2011 00:46:04 +0200 (CEST)
Subject: [pypy-commit] pypy numpy NDimArray: NDimSlice
Message-ID: <20111029224604.09194820C2@wyvern.cs.uni-duesseldorf.de>

Author: matthp
Branch: numpy NDimArray
Changeset: r48600:a2f79669b123
Date: 2011-10-26 15:12 +0200
http://bitbucket.org/pypy/pypy/changeset/a2f79669b123/

Log:	NDimSlice

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -270,25 +270,31 @@
 
     def descr_getitem(self, space, w_idx):
         # TODO: indexing by arrays and lists
+        start_stop_step_length = []
         if space.isinstance_w(w_idx, space.w_tuple):
             length = space.len_w(w_idx)
             if length == 0:
                 return space.wrap(self)
-            if length > 1: # only one dimension for now.
-                raise OperationError(space.w_IndexError,
-                                     space.wrap("invalid index"))
-            w_idx = space.getitem(w_idx, space.wrap(0))
-        start, stop, step, slice_length = space.decode_index4(w_idx, self.find_size())
-        if step == 0:
-            # Single index
-            return self.get_concrete().eval(start).wrap(space)
+            #if length > 1: # only one dimension for now.
+            #    raise OperationError(space.w_IndexError,
+            #                         space.wrap("invalid index"))
+            for i in range(length):
+                w_idx = space.getitem(w_idx, space.wrap(0))
+                start, stop, step, slice_length = space.decode_index4(w_idx, self.find_size())
+                start_stop_step_length.append((start,stop,step,slice_length)) 
         else:
-            # Slice
-            new_sig = signature.Signature.find_sig([
-                SingleDimSlice.signature, self.signature
-            ])
-            res = SingleDimSlice(start, stop, step, slice_length, self, new_sig)
-            return space.wrap(res)
+            start, stop, step, slice_length = space.decode_index4(w_idx, self.find_size())
+            if step == 0:
+                # Single index
+                return self.get_concrete().eval(start).wrap(space)
+            start_stop_step_length.append((start,stop,step,slice_length)) 
+        # Slice
+        slc = self.slice_type
+        new_sig = signature.Signature.find_sig([
+            slc.signature, self.signature
+        ])
+        res = slc(start_stop_step_length, self, new_sig)
+        return space.wrap(res)
 
     def descr_setitem(self, space, w_idx, w_value):
         # TODO: indexing by arrays and lists
@@ -521,8 +527,9 @@
 class SingleDimSlice(ViewArray):
     signature = signature.BaseSignature()
 
-    def __init__(self, start, stop, step, slice_length, parent, signature):
+    def __init__(self, start_stop_step_length , parent, signature):
         ViewArray.__init__(self, parent, signature)
+        start, stop, step, slice_length = start_stop_step_length[0]
         if isinstance(parent, SingleDimSlice):
             self.start = parent.calc_index(start)
             self.stop = parent.calc_index(stop)
@@ -568,7 +575,59 @@
         res += ")"
         return space.wrap(res)
 
+class NDimSlice(ViewArray):
+    signature = signature.BaseSignature()
+    #start, stop,step,slice_length are lists
+    def __init__(self, start_stop_step_length, parent, signature):
+        ViewArray.__init__(self, parent, signature)
+        
+        if isinstance(parent, NDimSlice):
+            self.start = parent.calc_index(start)
+            self.stop = parent.calc_index(stop)
+            self.step = parent.step * step
+            self.parent = parent.parent
+        else:
+            self.start = start
+            self.stop = stop
+            self.step = step
+            self.parent = parent
+        self.size = slice_length
+
+    def get_root_storage(self):
+        return self.parent.get_concrete().get_root_storage()
+
+    def find_size(self):
+        return self.size
+
+    def find_dtype(self):
+        return self.parent.find_dtype()
+
+    def descr_shape(self,space):
+        return space.newtuple([space.wrap(self.size)])
+
+    def setslice(self, space, start, stop, step, slice_length, arr):
+        start = self.calc_index(start)
+        if stop != -1:
+            stop = self.calc_index(stop)
+        step = self.step * step
+        self._sliceloop(start, stop, step, arr, self.parent)
+
+    def calc_index(self, item):
+        return (self.start + item * self.step)
+
+    def descr_repr(self, space):
+        # Simple implementation so that we can see the array. Needs work.
+        concrete = self.get_concrete()
+        res = "array([" + ", ".join(concrete._getnums(False)) + "]"
+        dtype = concrete.find_dtype()
+        if (dtype is not space.fromcache(interp_dtype.W_Float64Dtype) and
+            dtype is not space.fromcache(interp_dtype.W_Int64Dtype)) or not self.find_size():
+            res += ", dtype=" + dtype.name
+        res += ")"
+        return space.wrap(res)
+
 class SingleDimArray(BaseArray):
+    slice_type = SingleDimSlice
     def __init__(self, size, dtype):
         BaseArray.__init__(self)
         self.size = size
@@ -623,6 +682,7 @@
         lltype.free(self.storage, flavor='raw', track_allocation=False)
 
 class NDimArray(BaseArray):
+    slice_type = NDimSlice
     def __init__(self, shape, dtype):
         BaseArray.__init__(self)
         self.size = 1
@@ -649,7 +709,7 @@
         return self.dtype.getitem(self.storage, i)
 
     def descr_shape(self, space):
-        return self.shape
+        return space.wrap(self.shape)
 
     def descr_len(self, space):
         return space.wrap(self.size)

From noreply at buildbot.pypy.org  Sun Oct 30 00:46:05 2011
From: noreply at buildbot.pypy.org (mattip)
Date: Sun, 30 Oct 2011 00:46:05 +0200 (CEST)
Subject: [pypy-commit] pypy numpy NDimArray: print of ndarray works (kindof)
Message-ID: <20111029224605.40AE8820C2@wyvern.cs.uni-duesseldorf.de>

Author: mattip
Branch: numpy NDimArray
Changeset: r48601:4ef01c9532c9
Date: 2011-10-27 11:31 +0200
http://bitbucket.org/pypy/pypy/changeset/4ef01c9532c9/

Log:	print of ndarray works (kindof)

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -15,6 +15,12 @@
 any_driver = jit.JitDriver(greens=['signature'], reds=['i', 'size', 'self', 'dtype'])
 slice_driver = jit.JitDriver(greens=['signature'], reds=['i', 'j', 'step', 'stop', 'source', 'dest'])
 
+def prod(item):
+    res=1
+    for i in item:
+        res *= i
+    return res
+
 class BaseArray(Wrappable):
     _attrs_ = ["invalidates", "signature"]
 
@@ -279,8 +285,12 @@
             #    raise OperationError(space.w_IndexError,
             #                         space.wrap("invalid index"))
             for i in range(length):
-                w_idx = space.getitem(w_idx, space.wrap(0))
-                start, stop, step, slice_length = space.decode_index4(w_idx, self.find_size())
+                w_idx1 = space.getitem(w_idx, space.wrap(i))
+                start, stop, step, slice_length = space.decode_index4(w_idx1, self.find_size())
+                if step==0:
+                    step = 1
+                    stop = self.shape[i]
+                    #print 'got step==0, start=',start,'stop=',stop,'w_idx',w_idx,'shape',self.shape
                 start_stop_step_length.append((start,stop,step,slice_length)) 
         else:
             start, stop, step, slice_length = space.decode_index4(w_idx, self.find_size())
@@ -580,18 +590,41 @@
     #start, stop,step,slice_length are lists
     def __init__(self, start_stop_step_length, parent, signature):
         ViewArray.__init__(self, parent, signature)
-        
+        self.start = []
+        self.stop  = []
+        self.step  = [] 
+        self.shape = []
+        self.realdims=[]
+        i=0
         if isinstance(parent, NDimSlice):
-            self.start = parent.calc_index(start)
-            self.stop = parent.calc_index(stop)
-            self.step = parent.step * step
             self.parent = parent.parent
+            for sssl in start_stop_step_length:
+                start,stop,step,slice_length = sssl
+                self.start.append(parent.start[i]+start*parent.step[i])
+                self.stop.append(min(self.shape[i],parent.stop[i]+stop*parent.step[i]))
+                self.step.append(step*parent.step[i])
+                self.shape.append(slice_length)
+                if slice_length>1:
+                    self.realdims.append(i)
+                i+=1
         else:
-            self.start = start
-            self.stop = stop
-            self.step = step
             self.parent = parent
-        self.size = slice_length
+            for sssl in start_stop_step_length:
+                start,stop,step,slice_length = sssl
+                self.start.append(start)
+                self.stop.append(stop)
+                self.step.append(step)
+                self.shape.append(slice_length)
+                i+=1
+        for ii in range(i,len(parent.shape)):
+            self.start.append(0)
+            self.stop.append(parent.shape[ii])
+            self.step.append(1)
+            self.shape.append(parent.shape[ii])
+            if parent.shape[ii]>1:
+                self.realdims.append(ii)
+        self.size = prod(self.shape)
+        self.signature = signature
 
     def get_root_storage(self):
         return self.parent.get_concrete().get_root_storage()
@@ -613,12 +646,58 @@
         self._sliceloop(start, stop, step, arr, self.parent)
 
     def calc_index(self, item):
-        return (self.start + item * self.step)
-
+        indx=0
+        assert len(item)==len(self.realdims)
+        for i in range(len(self.shape)):
+            if i in self.realdims:
+                indx = self.start[i] + item[i]*self.step[i]
+            else:
+                indx = self.start[i]
+            if i+1<len(self.shape):
+                indx *= self.shape[i+1]
+        return indx
+    def tostr(self):
+        print 'NDimSlice::tostr, self.shape=',self.shape,'ndims=',self.realdims
+        res=''
+        if len(self.realdims)>2:
+            #Find first real dimension
+            start_stop_step_length=[]
+            for d,n in enumerate(self.shape):
+                start_stop_step_length.append((0,1,1,1))
+                if n>1:
+                    break
+            res += '['
+            for i in range(n):
+                res += NDimSlice(start_stop_step_length+[i,i+1,1,1],self,self.signature).tostr()
+            res += ']'
+        elif len(self.realdims)==2:
+            dtype = self.find_dtype()
+            res += '['
+            for i in range(self.start[self.realdims[0]],
+                           self.stop[self.realdims[0]],
+                           self.step[self.realdims[0]]):
+                res +='['
+                for j in range(self.start[self.realdims[1]],
+                               self.stop[self.realdims[1]],
+                               self.step[self.realdims[1]]):
+                    res += ' ' + dtype.str_format(self.parent.eval(self.calc_index((i,j))))
+                res+= ']\n'
+            res = res[:-1] + ']'
+        elif len(self.realdims)==1:
+            dtype = self.find_dtype()
+            res += '['
+            for i in range(self.start[self.realdims[0]],
+                           self.stop[self.realdims[0]],
+                           self.step[self.realdims[0]]):
+                res += ' ' + dtype.str_format(self.eval(self.calc_index((i))))
+            res += ']'
+        else:
+            res='[tbd]'
+        return res    
     def descr_repr(self, space):
         # Simple implementation so that we can see the array. Needs work.
         concrete = self.get_concrete()
-        res = "array([" + ", ".join(concrete._getnums(False)) + "]"
+        res = "ndarray(" + self.tostr() + ')'
         dtype = concrete.find_dtype()
         if (dtype is not space.fromcache(interp_dtype.W_Float64Dtype) and
             dtype is not space.fromcache(interp_dtype.W_Int64Dtype)) or not self.find_size():
@@ -634,6 +713,7 @@
         self.dtype = dtype
         self.storage = dtype.malloc(size)
         self.signature = dtype.signature
+        self.shape = (size,)
 
     def get_concrete(self):
         return self
@@ -706,7 +786,11 @@
         return self.dtype
 
     def eval(self, i):
-        return self.dtype.getitem(self.storage, i)
+        try:
+            return self.dtype.getitem(self.storage, i)
+        except:
+            print 'NDimArray::eval(',i,')'
+            return self.dtype.getitem(self.storage, i)
 
     def descr_shape(self, space):
         return space.wrap(self.shape)
@@ -745,7 +829,10 @@
     def descr_repr(self, space):
         # Simple implementation so that we can see the array. Needs work.
         concrete = self.get_concrete()
-        res = "ndarray([" + ", ".join(concrete._getnums(False)) + "]"
+        new_sig = signature.Signature.find_sig([ NDimSlice.signature, self.signature ])
+        res = "ndarray(" + \
+        NDimSlice((), self, new_sig).tostr() \
+        + ',' 
         dtype = concrete.find_dtype()
         if (dtype is not space.fromcache(interp_dtype.W_Float64Dtype) and
             dtype is not space.fromcache(interp_dtype.W_Int64Dtype)) or not self.find_size():

From noreply at buildbot.pypy.org  Sun Oct 30 00:46:06 2011
From: noreply at buildbot.pypy.org (mattip)
Date: Sun, 30 Oct 2011 00:46:06 +0200 (CEST)
Subject: [pypy-commit] pypy numpy NDimArray: Slices work for get and set
Message-ID: <20111029224606.769C6820C2@wyvern.cs.uni-duesseldorf.de>

Author: mattip
Branch: numpy NDimArray
Changeset: r48602:827d5d5d5218
Date: 2011-10-30 00:21 +0200
http://bitbucket.org/pypy/pypy/changeset/827d5d5d5218/

Log:	Slices work for get and set

diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py
--- a/pypy/module/micronumpy/__init__.py
+++ b/pypy/module/micronumpy/__init__.py
@@ -10,6 +10,7 @@
         'ufunc': 'interp_ufuncs.W_Ufunc',
 
         'zeros': 'interp_numarray.zeros',
+        'ndzeros': 'interp_numarray.ndzeros',
         'empty': 'interp_numarray.zeros',
         'ones': 'interp_numarray.ones',
         'fromstring': 'interp_support.fromstring',
diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -14,6 +14,7 @@
 all_driver = jit.JitDriver(greens=['signature'], reds=['i', 'size', 'self', 'dtype'])
 any_driver = jit.JitDriver(greens=['signature'], reds=['i', 'size', 'self', 'dtype'])
 slice_driver = jit.JitDriver(greens=['signature'], reds=['i', 'j', 'step', 'stop', 'source', 'dest'])
+nslice_driver = jit.JitDriver(greens=['signature'], reds=['i', 'self', 'count', 'source', 'dest'])
 
 def prod(item):
     res=1
@@ -55,34 +56,9 @@
         else:
             w_dtype = interp_ufuncs.find_dtype_for_scalar(space,w_iterable_or_scalar, w_dtype)
         return w_dtype
-                  
+             
     def descr__new__(space, w_subtype, w_size_or_iterable, w_dtype=None):
         l = space.listview(w_size_or_iterable)
-        w_elem = space.getitem(w_size_or_iterable, space.wrap(0))
-        if space.issequence_w(w_elem):
-            shape = [len(l)]
-            while space.issequence_w(w_elem):
-                shape.append(space.len_w(w_elem))
-                w_elem = space.getitem(w_elem, space.wrap(0))
-            if space.is_w(w_dtype, space.w_None):
-                w_dtype = None
-                w_dtype = BaseArray.find_dtype.im_func(space, w_size_or_iterable, w_dtype)
-                if w_dtype is None:
-                    w_dtype = space.w_None
-
-            dtype = space.interp_w(interp_dtype.W_Dtype,
-                space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype)
-            )
-            arr = NDimArray(shape, dtype=dtype)
-            #Assign all the values
-            try:
-                i = 0
-                for w_elem in l:
-                    arr.setitem_recurse_w(space,i,1,w_elem)
-                    i += 1
-            except:
-                print_exc()
-            return arr
         if space.is_w(w_dtype, space.w_None):
             w_dtype = None
             for w_item in l:
@@ -95,11 +71,29 @@
         dtype = space.interp_w(interp_dtype.W_Dtype,
             space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype)
         )
-        arr = SingleDimArray(len(l), dtype=dtype)
-        i = 0
-        for w_elem in l:
-            dtype.setitem_w(space, arr.storage, i, w_elem)
-            i += 1
+        w_elem = space.getitem(w_size_or_iterable, space.wrap(0))
+        length = len(l)
+        if isinstance(w_size_or_iterable,BaseArray) and len(w_size_or_iterable.find_shape())>1:
+            shape = w_size_or_iterable.find_shape()
+            arr = NDimArray(shape,dtype=dtype)
+            arr.setslice(space,((0,length,1,length),) ,w_size_or_iterable)
+        elif isinstance(w_size_or_iterable,BaseArray):
+            arr = SingleDimArray(length, dtype=dtype)
+            arr.setslice(space,((0,length,1,length),) ,w_size_or_iterable)
+        elif space.issequence_w(w_elem):
+            shape = [len(l)]
+            while space.issequence_w(w_elem):
+                shape.append(space.len_w(w_elem))
+                w_elem = space.getitem(w_elem, space.wrap(0))
+            arr = NDimArray(shape, dtype=dtype)
+            #Assign all the values
+            arr.setslice(space,((0,length,1,length),) ,w_size_or_iterable)
+        else:
+            arr = SingleDimArray(length, dtype=dtype)
+            i = 0
+            for w_elem in l:
+                dtype.setitem_w(space, arr.storage, i, w_elem)
+                i += 1
         return arr
 
     def _unaryop_impl(ufunc_name):
@@ -273,71 +267,64 @@
         # Simple implementation so that we can see the array. Needs work.
         concrete = self.get_concrete()
         return space.wrap("[" + " ".join(concrete._getnums(True)) + "]")
-
-    def descr_getitem(self, space, w_idx):
-        # TODO: indexing by arrays and lists
+    def create_sssl_from_w_idx(self,space, w_idx):
         start_stop_step_length = []
+        myshape = self.find_shape()
+        #print 'descr_getitem(...,',w_idx,')'
         if space.isinstance_w(w_idx, space.w_tuple):
             length = space.len_w(w_idx)
             if length == 0:
                 return space.wrap(self)
-            #if length > 1: # only one dimension for now.
-            #    raise OperationError(space.w_IndexError,
-            #                         space.wrap("invalid index"))
+            if length>len(myshape):
+                raise OperationError(space.w_IndexError,
+                                     space.wrap("invalid index: cannot return %d array from % dim"%(length,len(myshape))))
             for i in range(length):
                 w_idx1 = space.getitem(w_idx, space.wrap(i))
-                start, stop, step, slice_length = space.decode_index4(w_idx1, self.find_size())
+                if space.is_true(space.isinstance(w_idx1,space.w_int)) and space.is_true(space.gt(w_idx1,space.wrap(myshape[i]))):
+                    raise OperationError(space.w_IndexError,
+                             space.wrap("index (%d) out of range (0<=index<%d) in dimension %d"%(space.unwrap(w_idx1),myshape[i],i)))
+                start, stop, step, slice_length = space.decode_index4(w_idx1, myshape[i])
                 if step==0:
                     step = 1
-                    stop = self.shape[i]
+                    stop = min(myshape[i],start+1)
+                    slice_length = 1
                     #print 'got step==0, start=',start,'stop=',stop,'w_idx',w_idx,'shape',self.shape
+                if start>=myshape[i]:
+                    slice_length=0
                 start_stop_step_length.append((start,stop,step,slice_length)) 
         else:
             start, stop, step, slice_length = space.decode_index4(w_idx, self.find_size())
             if step == 0:
+                step = 1
+                stop = min(myshape[0],start+1)
+                slice_length = 1
                 # Single index
-                return self.get_concrete().eval(start).wrap(space)
+            if start>=myshape[0]:
+                slice_length=0
             start_stop_step_length.append((start,stop,step,slice_length)) 
+        return start_stop_step_length
+    def descr_getitem(self, space, w_idx):
+        start_stop_step_length = self.create_sssl_from_w_idx(space,w_idx)
         # Slice
         slc = self.slice_type
-        new_sig = signature.Signature.find_sig([
-            slc.signature, self.signature
-        ])
+        new_sig = signature.Signature.find_sig([ slc.signature, self.signature ])
         res = slc(start_stop_step_length, self, new_sig)
         return space.wrap(res)
 
     def descr_setitem(self, space, w_idx, w_value):
-        # TODO: indexing by arrays and lists
         self.invalidated()
-        if space.isinstance_w(w_idx, space.w_tuple):
-            length = space.len_w(w_idx)
-            if length > 1: # only one dimension for now.
-                raise OperationError(space.w_IndexError,
-                                     space.wrap("invalid index"))
-            if length == 0:
-                w_idx = space.newslice(space.wrap(0),
-                                      space.wrap(self.find_size()),
-                                      space.wrap(1))
-            else:
-                w_idx = space.getitem(w_idx, space.wrap(0))
-        start, stop, step, slice_length = space.decode_index4(w_idx,
-                                                              self.find_size())
-        if step == 0:
-            # Single index
-            self.get_concrete().setitem_w(space, start, w_value)
+        start_stop_step_length = self.create_sssl_from_w_idx(space,w_idx)
+        concrete = self.get_concrete()
+        if isinstance(w_value, BaseArray):
+            # for now we just copy if setting part of an array from
+            # part of itself. can be improved.
+            if (concrete.get_root_storage() ==
+                w_value.get_concrete().get_root_storage()):
+                w_value = space.call_function(space.gettypefor(BaseArray), w_value)
+                assert isinstance(w_value, BaseArray)
         else:
-            concrete = self.get_concrete()
-            if isinstance(w_value, BaseArray):
-                # for now we just copy if setting part of an array from
-                # part of itself. can be improved.
-                if (concrete.get_root_storage() ==
-                    w_value.get_concrete().get_root_storage()):
-                    w_value = space.call_function(space.gettypefor(BaseArray), w_value)
-                    assert isinstance(w_value, BaseArray)
-            else:
-                w_value = convert_to_array(space, w_value)
-            concrete.setslice(space, start, stop, step,
-                                               slice_length, w_value)
+            w_value = convert_to_array(space, w_value)
+        concrete.setslice(space, start_stop_step_length, w_value)
 
     def descr_mean(self, space):
         return space.wrap(space.float_w(self.descr_sum(space))/self.find_size())
@@ -408,14 +395,25 @@
     def compute(self):
         i = 0
         signature = self.signature
-        result_size = self.find_size()
-        result = SingleDimArray(result_size, self.find_dtype())
-        while i < result_size:
-            numpy_driver.jit_merge_point(signature=signature,
+        result_shape = self.find_shape()
+        if len(result_shape)<2:
+            result_size = self.find_size()
+            result = SingleDimArray(result_size, self.find_dtype())
+            while i < result_size:
+                numpy_driver.jit_merge_point(signature=signature,
                                          result_size=result_size, i=i,
                                          self=self, result=result)
-            result.dtype.setitem(result.storage, i, self.eval(i))
-            i += 1
+                result.dtype.setitem(result.storage, i, self.eval(i))
+                i += 1
+        else:
+            result = NDimArray(result_shape, self.find_dtype())
+            result_size = self.find_size()
+            while i < result_size:
+                numpy_driver.jit_merge_point(signature=signature,
+                                         result_size=result_size, i=i,
+                                         self=self, result=result)
+                result.dtype.setitem(result.storage, i, self.eval(i))
+                i += 1
         return result
 
     def force_if_needed(self):
@@ -435,6 +433,12 @@
     def setitem(self, item, value):
         return self.get_concrete().setitem(item, value)
 
+    def find_shape(self):
+        if self.forced_result is not None:
+            # The result has been computed and sources may be unavailable
+            return self.forced_result.find_shape()
+        return self._find_shape()
+
     def find_size(self):
         if self.forced_result is not None:
             # The result has been computed and sources may be unavailable
@@ -453,6 +457,9 @@
     def _del_sources(self):
         self.values = None
 
+    def _find_shape(self):
+        return self.values.find_shape()
+
     def _find_size(self):
         return self.values.find_size()
 
@@ -482,6 +489,13 @@
         self.left = None
         self.right = None
 
+    def _find_shape(self):
+        try:
+            return self.left.find_shape()
+        except ValueError:
+            pass
+        return self.right.find_shape()
+
     def _find_size(self):
         try:
             return self.left.find_size()
@@ -555,6 +569,9 @@
     def get_root_storage(self):
         return self.parent.get_concrete().get_root_storage()
 
+    def find_shape(self):
+        return (self.size,)
+
     def find_size(self):
         return self.size
 
@@ -564,7 +581,8 @@
     def descr_shape(self,space):
         return space.newtuple([space.wrap(self.size)])
 
-    def setslice(self, space, start, stop, step, slice_length, arr):
+    def setslice(self, start_stop_step_length, arr):
+        space, start, stop, step, slice_length = start_stop_step_length[0]
         start = self.calc_index(start)
         if stop != -1:
             stop = self.calc_index(stop)
@@ -584,11 +602,12 @@
             res += ", dtype=" + dtype.name
         res += ")"
         return space.wrap(res)
-
+SingleDimSlice.slice_type = SingleDimSlice
 class NDimSlice(ViewArray):
     signature = signature.BaseSignature()
     #start, stop,step,slice_length are lists
     def __init__(self, start_stop_step_length, parent, signature):
+        #print 'NDimSlice::init(',start_stop_step_length,',...)'
         ViewArray.__init__(self, parent, signature)
         self.start = []
         self.stop  = []
@@ -601,7 +620,7 @@
             for sssl in start_stop_step_length:
                 start,stop,step,slice_length = sssl
                 self.start.append(parent.start[i]+start*parent.step[i])
-                self.stop.append(min(self.shape[i],parent.stop[i]+stop*parent.step[i]))
+                self.stop.append(min(parent.shape[i], parent.stop[i]+stop*parent.step[i]))
                 self.step.append(step*parent.step[i])
                 self.shape.append(slice_length)
                 if slice_length>1:
@@ -615,6 +634,8 @@
                 self.stop.append(stop)
                 self.step.append(step)
                 self.shape.append(slice_length)
+                if slice_length>1:
+                    self.realdims.append(i)
                 i+=1
         for ii in range(i,len(parent.shape)):
             self.start.append(0)
@@ -629,6 +650,9 @@
     def get_root_storage(self):
         return self.parent.get_concrete().get_root_storage()
 
+    def find_shape(self):
+        return self.shape
+
     def find_size(self):
         return self.size
 
@@ -636,28 +660,51 @@
         return self.parent.find_dtype()
 
     def descr_shape(self,space):
-        return space.newtuple([space.wrap(self.size)])
+        return space.wrap(self.shape)
 
-    def setslice(self, space, start, stop, step, slice_length, arr):
-        start = self.calc_index(start)
-        if stop != -1:
-            stop = self.calc_index(stop)
-        step = self.step * step
-        self._sliceloop(start, stop, step, arr, self.parent)
+    def eval(self, i):
+        return self.parent.eval(self.calc_index((i,)))
 
+    def setslice(self, space, start_stop_step_length, arr):
+        sig = signature.Signature.find_sig([ self.signature, self.signature ])
+        NDimSlice(start_stop_step_length, self, sig).setvals(space, arr)
+
+    def setvals(self,space, arr):
+        sig = signature.Signature.find_sig([ self.signature, self.signature ])
+        if len(self.realdims)==0:
+            self.parent.setitem(self.calc_index([]),arr.eval(0).convert_to(self.parent.find_dtype()))
+        elif len(self.realdims)==1:
+            rd = self.realdims[0]
+            i=0
+            count = self.shape[rd]
+            dest = self.parent
+            source = arr
+            while( i < count):
+                nslice_driver.jit_merge_point(signature=self.signature, 
+                                         count=count, i=i, source=source,
+                                         self=self, dest=dest)
+                dest.setitem(self.calc_index((i,)), source.eval(i).convert_to(dest.find_dtype()))
+                i += 1
+        else: 
+            for rd in self.realdims:
+                for i in range(self.shape[rd]):
+                    subarr = NDimSlice(((i,i+1,1,1),) , self, sig)
+                    subarr.setvals(space, arr.descr_getitem(space,space.wrap(i)))
+                    
     def calc_index(self, item):
         indx=0
-        assert len(item)==len(self.realdims)
+        assert len(item)==len(self.realdims),'NDimSlice::calc_index( %s ) does not match %s'%(str(item),str(self.realdims))
         for i in range(len(self.shape)):
             if i in self.realdims:
-                indx = self.start[i] + item[i]*self.step[i]
+                indx += self.start[i] + item[self.realdims.index(i)]*self.step[i]
             else:
-                indx = self.start[i]
+                indx += self.start[i]
             if i+1<len(self.shape):
-                indx *= self.shape[i+1]
+               indx *= self.parent.shape[i+1]
+            #print 'NDimSlice::calc_index(',item,') =>',indx,'after',i,'itterations'
         return indx
     def tostr(self):
-        print 'NDimSlice::tostr, self.shape=',self.shape,'ndims=',self.realdims
+        #print 'NDimSlice::tostr, self.shape=',self.shape,'ndims=',self.realdims
         res=''
         if len(self.realdims)>2:
             #Find first real dimension
@@ -676,34 +723,41 @@
             for i in range(self.start[self.realdims[0]],
                            self.stop[self.realdims[0]],
                            self.step[self.realdims[0]]):
+                if i>0:
+                    res += '         '
                 res +='['
-                for j in range(self.start[self.realdims[1]],
+                res += ', '.join([ dtype.str_format(self.parent.eval(self.calc_index((i,j)))) \
+                          for j in range(self.start[self.realdims[1]],
                                self.stop[self.realdims[1]],
-                               self.step[self.realdims[1]]):
-                    res += ' ' + dtype.str_format(self.parent.eval(self.calc_index((i,j))))
+                               self.step[self.realdims[1]])])
                 res+= ']\n'
             res = res[:-1] + ']'
         elif len(self.realdims)==1:
             dtype = self.find_dtype()
             res += '['
-            for i in range(self.start[self.realdims[0]],
+            res += ', '.join([dtype.str_format(self.parent.eval(self.calc_index((i,)))) \
+                      for i in range(self.start[self.realdims[0]],
                            self.stop[self.realdims[0]],
-                           self.step[self.realdims[0]]):
-                res += ' ' + dtype.str_format(self.eval(self.calc_index((i))))
+                           self.step[self.realdims[0]])])
             res += ']'
+        elif prod(self.shape)==1:
+            dtype = self.find_dtype()
+            res += '[' + dtype.str_format(self.parent.eval(self.calc_index([]))) + ']'
         else:
-            res='[tbd]'
+            print 'NDimSlice::tostr, self.shape=',self.shape,'realdims=',self.realdims,',start=',self.start,',stop=',self.stop
+            res += 'empty'
         return res    
     def descr_repr(self, space):
         # Simple implementation so that we can see the array. Needs work.
         concrete = self.get_concrete()
-        res = "ndarray(" + self.tostr() + ')'
+        res = "ndarray(" + self.tostr() 
         dtype = concrete.find_dtype()
         if (dtype is not space.fromcache(interp_dtype.W_Float64Dtype) and
             dtype is not space.fromcache(interp_dtype.W_Int64Dtype)) or not self.find_size():
             res += ", dtype=" + dtype.name
         res += ")"
         return space.wrap(res)
+NDimSlice.slice_type = NDimSlice
 
 class SingleDimArray(BaseArray):
     slice_type = SingleDimSlice
@@ -713,7 +767,6 @@
         self.dtype = dtype
         self.storage = dtype.malloc(size)
         self.signature = dtype.signature
-        self.shape = (size,)
 
     def get_concrete(self):
         return self
@@ -721,6 +774,9 @@
     def get_root_storage(self):
         return self.storage
 
+    def find_shape(self):
+        return (self.size,)
+
     def find_size(self):
         return self.size
 
@@ -744,7 +800,8 @@
         self.invalidated()
         self.dtype.setitem(self.storage, item, value)
 
-    def setslice(self, space, start, stop, step, slice_length, arr):
+    def setslice(self, space, start_stop_step_length, arr):
+        start, stop, step, slice_length = start_stop_step_length[0]
         self._sliceloop(start, stop, step, arr, self)
 
     def descr_repr(self, space):
@@ -765,12 +822,11 @@
     slice_type = NDimSlice
     def __init__(self, shape, dtype):
         BaseArray.__init__(self)
-        self.size = 1
-        for s in shape:
-            self.size *= s
+        self.size = prod(shape)
         self.shape = shape
         self.dtype = dtype
         self.storage = dtype.malloc(self.size)
+        #print 'Creating NDimArray(',shape,',...)'
         self.signature = dtype.signature
 
     def get_concrete(self):
@@ -779,6 +835,9 @@
     def get_root_storage(self):
         return self.storage
 
+    def find_shape(self):
+        return self.shape
+
     def find_size(self):
         return self.size
 
@@ -802,29 +861,13 @@
         self.invalidated()
         self.dtype.setitem_w(space, self.storage, item, w_value)
 
-    def setitem_recurse_w(self,space,index,depth,w_value):
-        self.invalidated()
-        if space.issequence_w(w_value):
-            i=0
-            w_iterator = space.iter(w_value)
-            while True:
-                try:
-                    w_item = space.next(w_iterator)
-                except OperationError, e:
-                    if not e.match(space, space.w_StopIteration):
-                        raise
-                    return 
-                self.setitem_recurse_w(space,i+index*self.shape[-depth], depth+1, w_item)
-                i+=1
-        else:
-            print 'setting',index,'to',w_value
-            self.setitem_w(space,index,w_value) 
     def setitem(self, item, value):
         self.invalidated()
         self.dtype.setitem(self.storage, item, value)
 
-    def setslice(self, space, start, stop, step, slice_length, arr):
-        self._sliceloop(start, stop, step, arr, self)
+    def setslice(self, space, start_stop_step_length, arr):
+        sig = signature.Signature.find_sig([ self.signature, self.signature ])
+        NDimSlice(start_stop_step_length, self, sig).setvals(space, arr)
 
     def descr_repr(self, space):
         # Simple implementation so that we can see the array. Needs work.
@@ -843,12 +886,16 @@
     def __del__(self):
         lltype.free(self.storage, flavor='raw', track_allocation=False)
 
- at unwrap_spec(size=int)
-def zeros(space, size, w_dtype=None):
+
+def zeros(space, w_size, w_dtype=None):
     dtype = space.interp_w(interp_dtype.W_Dtype,
         space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype)
     )
-    return space.wrap(SingleDimArray(size, dtype=dtype))
+    if space.is_true(space.isinstance(w_size,space.w_int)):
+        return space.wrap(SingleDimArray(space.unwrap(w_size), dtype=dtype))
+    else:
+        size = [space.unwrap(s) for s in space.listview(w_size)]
+        return space.wrap(NDimArray(size, dtype=dtype))
 
 @unwrap_spec(size=int)
 def ones(space, size, w_dtype=None):

From noreply at buildbot.pypy.org  Sun Oct 30 00:46:10 2011
From: noreply at buildbot.pypy.org (mattip)
Date: Sun, 30 Oct 2011 00:46:10 +0200 (CEST)
Subject: [pypy-commit] pypy numpy NDimArray: Merge
Message-ID: <20111029224610.98F2F820C2@wyvern.cs.uni-duesseldorf.de>

Author: mattip
Branch: numpy NDimArray
Changeset: r48603:29e96633349c
Date: 2011-10-30 00:40 +0200
http://bitbucket.org/pypy/pypy/changeset/29e96633349c/

Log:	Merge

diff --git a/lib-python/modified-2.7/json/encoder.py b/lib-python/modified-2.7/json/encoder.py
--- a/lib-python/modified-2.7/json/encoder.py
+++ b/lib-python/modified-2.7/json/encoder.py
@@ -2,14 +2,7 @@
 """
 import re
 
-try:
-    from _json import encode_basestring_ascii as c_encode_basestring_ascii
-except ImportError:
-    c_encode_basestring_ascii = None
-try:
-    from _json import make_encoder as c_make_encoder
-except ImportError:
-    c_make_encoder = None
+from __pypy__.builders import StringBuilder, UnicodeBuilder
 
 ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]')
 ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])')
@@ -24,8 +17,7 @@
     '\t': '\\t',
 }
 for i in range(0x20):
-    ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i))
-    #ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,))
+    ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,))
 
 # Assume this produces an infinity on all machines (probably not guaranteed)
 INFINITY = float('1e66666')
@@ -37,10 +29,9 @@
     """
     def replace(match):
         return ESCAPE_DCT[match.group(0)]
-    return '"' + ESCAPE.sub(replace, s) + '"'
+    return ESCAPE.sub(replace, s)
 
-
-def py_encode_basestring_ascii(s):
+def encode_basestring_ascii(s):
     """Return an ASCII-only JSON representation of a Python string
 
     """
@@ -53,20 +44,18 @@
         except KeyError:
             n = ord(s)
             if n < 0x10000:
-                return '\\u{0:04x}'.format(n)
-                #return '\\u%04x' % (n,)
+                return '\\u%04x' % (n,)
             else:
                 # surrogate pair
                 n -= 0x10000
                 s1 = 0xd800 | ((n >> 10) & 0x3ff)
                 s2 = 0xdc00 | (n & 0x3ff)
-                return '\\u{0:04x}\\u{1:04x}'.format(s1, s2)
-                #return '\\u%04x\\u%04x' % (s1, s2)
-    return '"' + str(ESCAPE_ASCII.sub(replace, s)) + '"'
-
-
-encode_basestring_ascii = (
-    c_encode_basestring_ascii or py_encode_basestring_ascii)
+                return '\\u%04x\\u%04x' % (s1, s2)
+    if ESCAPE_ASCII.search(s):
+        return str(ESCAPE_ASCII.sub(replace, s))
+    return s
+py_encode_basestring_ascii = lambda s: '"' + encode_basestring_ascii(s) + '"'
+c_encode_basestring_ascii = None
 
 class JSONEncoder(object):
     """Extensible JSON <http://json.org> encoder for Python data structures.
@@ -147,6 +136,17 @@
 
         self.skipkeys = skipkeys
         self.ensure_ascii = ensure_ascii
+        if ensure_ascii:
+            self.encoder = encode_basestring_ascii
+        else:
+            self.encoder = encode_basestring
+        if encoding != 'utf-8':
+            orig_encoder = self.encoder
+            def encoder(o):
+                if isinstance(o, str):
+                    o = o.decode(encoding)
+                return orig_encoder(o)
+            self.encoder = encoder
         self.check_circular = check_circular
         self.allow_nan = allow_nan
         self.sort_keys = sort_keys
@@ -184,24 +184,126 @@
         '{"foo": ["bar", "baz"]}'
 
         """
-        # This is for extremely simple cases and benchmarks.
+        if self.check_circular:
+            markers = {}
+        else:
+            markers = None
+        if self.ensure_ascii:
+            builder = StringBuilder()
+        else:
+            builder = UnicodeBuilder()
+        self._encode(o, markers, builder, 0)
+        return builder.build()
+
+    def _emit_indent(self, builder, _current_indent_level):
+        if self.indent is not None:
+            _current_indent_level += 1
+            newline_indent = '\n' + (' ' * (self.indent *
+                                            _current_indent_level))
+            separator = self.item_separator + newline_indent
+            builder.append(newline_indent)
+        else:
+            separator = self.item_separator
+        return separator, _current_indent_level
+
+    def _emit_unindent(self, builder, _current_indent_level):
+        if self.indent is not None:
+            builder.append('\n')
+            builder.append(' ' * (self.indent * (_current_indent_level - 1)))
+
+    def _encode(self, o, markers, builder, _current_indent_level):
         if isinstance(o, basestring):
-            if isinstance(o, str):
-                _encoding = self.encoding
-                if (_encoding is not None
-                        and not (_encoding == 'utf-8')):
-                    o = o.decode(_encoding)
-            if self.ensure_ascii:
-                return encode_basestring_ascii(o)
+            builder.append('"')
+            builder.append(self.encoder(o))
+            builder.append('"')
+        elif o is None:
+            builder.append('null')
+        elif o is True:
+            builder.append('true')
+        elif o is False:
+            builder.append('false')
+        elif isinstance(o, (int, long)):
+            builder.append(str(o))
+        elif isinstance(o, float):
+            builder.append(self._floatstr(o))
+        elif isinstance(o, (list, tuple)):
+            if not o:
+                builder.append('[]')
+                return
+            self._encode_list(o, markers, builder, _current_indent_level)
+        elif isinstance(o, dict):
+            if not o:
+                builder.append('{}')
+                return
+            self._encode_dict(o, markers, builder, _current_indent_level)
+        else:
+            self._mark_markers(markers, o)
+            res = self.default(o)
+            self._encode(res, markers, builder, _current_indent_level)
+            self._remove_markers(markers, o)
+            return res
+
+    def _encode_list(self, l, markers, builder, _current_indent_level):
+        self._mark_markers(markers, l)
+        builder.append('[')
+        first = True
+        separator, _current_indent_level = self._emit_indent(builder,
+                                                      _current_indent_level)
+        for elem in l:
+            if first:
+                first = False
             else:
-                return encode_basestring(o)
-        # This doesn't pass the iterator directly to ''.join() because the
-        # exceptions aren't as detailed.  The list call should be roughly
-        # equivalent to the PySequence_Fast that ''.join() would do.
-        chunks = self.iterencode(o, _one_shot=True)
-        if not isinstance(chunks, (list, tuple)):
-            chunks = list(chunks)
-        return ''.join(chunks)
+                builder.append(separator)
+            self._encode(elem, markers, builder, _current_indent_level)
+            del elem # XXX grumble
+        self._emit_unindent(builder, _current_indent_level)
+        builder.append(']')
+        self._remove_markers(markers, l)
+
+    def _encode_dict(self, d, markers, builder, _current_indent_level):
+        self._mark_markers(markers, d)
+        first = True
+        builder.append('{')
+        separator, _current_indent_level = self._emit_indent(builder,
+                                                         _current_indent_level)
+        if self.sort_keys:
+            items = sorted(d.items(), key=lambda kv: kv[0])
+        else:
+            items = d.iteritems()
+
+        for key, v in items:
+            if first:
+                first = False
+            else:
+                builder.append(separator)
+            if isinstance(key, basestring):
+                pass
+            # JavaScript is weakly typed for these, so it makes sense to
+            # also allow them.  Many encoders seem to do something like this.
+            elif isinstance(key, float):
+                key = self._floatstr(key)
+            elif key is True:
+                key = 'true'
+            elif key is False:
+                key = 'false'
+            elif key is None:
+                key = 'null'
+            elif isinstance(key, (int, long)):
+                key = str(key)
+            elif self.skipkeys:
+                continue
+            else:
+                raise TypeError("key " + repr(key) + " is not a string")
+            builder.append('"')
+            builder.append(self.encoder(key))
+            builder.append('"')
+            builder.append(self.key_separator)
+            self._encode(v, markers, builder, _current_indent_level)
+            del key
+            del v # XXX grumble
+        self._emit_unindent(builder, _current_indent_level)
+        builder.append('}')
+        self._remove_markers(markers, d)
 
     def iterencode(self, o, _one_shot=False):
         """Encode the given object and yield each string
@@ -217,86 +319,54 @@
             markers = {}
         else:
             markers = None
-        if self.ensure_ascii:
-            _encoder = encode_basestring_ascii
+        return self._iterencode(o, markers, 0)
+
+    def _floatstr(self, o):
+        # Check for specials.  Note that this type of test is processor
+        # and/or platform-specific, so do tests which don't depend on the
+        # internals.
+
+        if o != o:
+            text = 'NaN'
+        elif o == INFINITY:
+            text = 'Infinity'
+        elif o == -INFINITY:
+            text = '-Infinity'
         else:
-            _encoder = encode_basestring
-        if self.encoding != 'utf-8':
-            def _encoder(o, _orig_encoder=_encoder, _encoding=self.encoding):
-                if isinstance(o, str):
-                    o = o.decode(_encoding)
-                return _orig_encoder(o)
+            return FLOAT_REPR(o)
 
-        def floatstr(o, allow_nan=self.allow_nan,
-                _repr=FLOAT_REPR, _inf=INFINITY, _neginf=-INFINITY):
-            # Check for specials.  Note that this type of test is processor
-            # and/or platform-specific, so do tests which don't depend on the
-            # internals.
+        if not self.allow_nan:
+            raise ValueError(
+                "Out of range float values are not JSON compliant: " +
+                repr(o))
 
-            if o != o:
-                text = 'NaN'
-            elif o == _inf:
-                text = 'Infinity'
-            elif o == _neginf:
-                text = '-Infinity'
-            else:
-                return _repr(o)
+        return text
 
-            if not allow_nan:
-                raise ValueError(
-                    "Out of range float values are not JSON compliant: " +
-                    repr(o))
+    def _mark_markers(self, markers, o):
+        if markers is not None:
+            if id(o) in markers:
+                raise ValueError("Circular reference detected")
+            markers[id(o)] = None
 
-            return text
+    def _remove_markers(self, markers, o):
+        if markers is not None:
+            del markers[id(o)]
 
-
-        if (_one_shot and c_make_encoder is not None
-                and not self.indent and not self.sort_keys):
-            _iterencode = c_make_encoder(
-                markers, self.default, _encoder, self.indent,
-                self.key_separator, self.item_separator, self.sort_keys,
-                self.skipkeys, self.allow_nan)
-        else:
-            _iterencode = _make_iterencode(
-                markers, self.default, _encoder, self.indent, floatstr,
-                self.key_separator, self.item_separator, self.sort_keys,
-                self.skipkeys, _one_shot)
-        return _iterencode(o, 0)
-
-def _make_iterencode(markers, _default, _encoder, _indent, _floatstr,
-        _key_separator, _item_separator, _sort_keys, _skipkeys, _one_shot,
-        ## HACK: hand-optimized bytecode; turn globals into locals
-        ValueError=ValueError,
-        basestring=basestring,
-        dict=dict,
-        float=float,
-        id=id,
-        int=int,
-        isinstance=isinstance,
-        list=list,
-        long=long,
-        str=str,
-        tuple=tuple,
-    ):
-
-    def _iterencode_list(lst, _current_indent_level):
+    def _iterencode_list(self, lst, markers, _current_indent_level):
         if not lst:
             yield '[]'
             return
-        if markers is not None:
-            markerid = id(lst)
-            if markerid in markers:
-                raise ValueError("Circular reference detected")
-            markers[markerid] = lst
+        self._mark_markers(markers, lst)
         buf = '['
-        if _indent is not None:
+        if self.indent is not None:
             _current_indent_level += 1
-            newline_indent = '\n' + (' ' * (_indent * _current_indent_level))
-            separator = _item_separator + newline_indent
+            newline_indent = '\n' + (' ' * (self.indent *
+                                            _current_indent_level))
+            separator = self.item_separator + newline_indent
             buf += newline_indent
         else:
             newline_indent = None
-            separator = _item_separator
+            separator = self.item_separator
         first = True
         for value in lst:
             if first:
@@ -304,7 +374,7 @@
             else:
                 buf = separator
             if isinstance(value, basestring):
-                yield buf + _encoder(value)
+                yield buf + '"' + self.encoder(value) + '"'
             elif value is None:
                 yield buf + 'null'
             elif value is True:
@@ -314,44 +384,43 @@
             elif isinstance(value, (int, long)):
                 yield buf + str(value)
             elif isinstance(value, float):
-                yield buf + _floatstr(value)
+                yield buf + self._floatstr(value)
             else:
                 yield buf
                 if isinstance(value, (list, tuple)):
-                    chunks = _iterencode_list(value, _current_indent_level)
+                    chunks = self._iterencode_list(value, markers,
+                                                   _current_indent_level)
                 elif isinstance(value, dict):
-                    chunks = _iterencode_dict(value, _current_indent_level)
+                    chunks = self._iterencode_dict(value, markers,
+                                                   _current_indent_level)
                 else:
-                    chunks = _iterencode(value, _current_indent_level)
+                    chunks = self._iterencode(value, markers,
+                                              _current_indent_level)
                 for chunk in chunks:
                     yield chunk
         if newline_indent is not None:
             _current_indent_level -= 1
-            yield '\n' + (' ' * (_indent * _current_indent_level))
+            yield '\n' + (' ' * (self.indent * _current_indent_level))
         yield ']'
-        if markers is not None:
-            del markers[markerid]
+        self._remove_markers(markers, lst)
 
-    def _iterencode_dict(dct, _current_indent_level):
+    def _iterencode_dict(self, dct, markers, _current_indent_level):
         if not dct:
             yield '{}'
             return
-        if markers is not None:
-            markerid = id(dct)
-            if markerid in markers:
-                raise ValueError("Circular reference detected")
-            markers[markerid] = dct
+        self._mark_markers(markers, dct)
         yield '{'
-        if _indent is not None:
+        if self.indent is not None:
             _current_indent_level += 1
-            newline_indent = '\n' + (' ' * (_indent * _current_indent_level))
-            item_separator = _item_separator + newline_indent
+            newline_indent = '\n' + (' ' * (self.indent *
+                                            _current_indent_level))
+            item_separator = self.item_separator + newline_indent
             yield newline_indent
         else:
             newline_indent = None
-            item_separator = _item_separator
+            item_separator = self.item_separator
         first = True
-        if _sort_keys:
+        if self.sort_keys:
             items = sorted(dct.items(), key=lambda kv: kv[0])
         else:
             items = dct.iteritems()
@@ -361,7 +430,7 @@
             # JavaScript is weakly typed for these, so it makes sense to
             # also allow them.  Many encoders seem to do something like this.
             elif isinstance(key, float):
-                key = _floatstr(key)
+                key = self._floatstr(key)
             elif key is True:
                 key = 'true'
             elif key is False:
@@ -370,7 +439,7 @@
                 key = 'null'
             elif isinstance(key, (int, long)):
                 key = str(key)
-            elif _skipkeys:
+            elif self.skipkeys:
                 continue
             else:
                 raise TypeError("key " + repr(key) + " is not a string")
@@ -378,10 +447,10 @@
                 first = False
             else:
                 yield item_separator
-            yield _encoder(key)
-            yield _key_separator
+            yield '"' + self.encoder(key) + '"'
+            yield self.key_separator
             if isinstance(value, basestring):
-                yield _encoder(value)
+                yield '"' + self.encoder(value) + '"'
             elif value is None:
                 yield 'null'
             elif value is True:
@@ -391,26 +460,28 @@
             elif isinstance(value, (int, long)):
                 yield str(value)
             elif isinstance(value, float):
-                yield _floatstr(value)
+                yield self._floatstr(value)
             else:
                 if isinstance(value, (list, tuple)):
-                    chunks = _iterencode_list(value, _current_indent_level)
+                    chunks = self._iterencode_list(value, markers,
+                                                   _current_indent_level)
                 elif isinstance(value, dict):
-                    chunks = _iterencode_dict(value, _current_indent_level)
+                    chunks = self._iterencode_dict(value, markers,
+                                                   _current_indent_level)
                 else:
-                    chunks = _iterencode(value, _current_indent_level)
+                    chunks = self._iterencode(value, markers,
+                                              _current_indent_level)
                 for chunk in chunks:
                     yield chunk
         if newline_indent is not None:
             _current_indent_level -= 1
-            yield '\n' + (' ' * (_indent * _current_indent_level))
+            yield '\n' + (' ' * (self.indent * _current_indent_level))
         yield '}'
-        if markers is not None:
-            del markers[markerid]
+        self._remove_markers(markers, dct)
 
-    def _iterencode(o, _current_indent_level):
+    def _iterencode(self, o, markers, _current_indent_level):
         if isinstance(o, basestring):
-            yield _encoder(o)
+            yield '"' + self.encoder(o) + '"'
         elif o is None:
             yield 'null'
         elif o is True:
@@ -420,23 +491,19 @@
         elif isinstance(o, (int, long)):
             yield str(o)
         elif isinstance(o, float):
-            yield _floatstr(o)
+            yield self._floatstr(o)
         elif isinstance(o, (list, tuple)):
-            for chunk in _iterencode_list(o, _current_indent_level):
+            for chunk in self._iterencode_list(o, markers,
+                                               _current_indent_level):
                 yield chunk
         elif isinstance(o, dict):
-            for chunk in _iterencode_dict(o, _current_indent_level):
+            for chunk in self._iterencode_dict(o, markers,
+                                               _current_indent_level):
                 yield chunk
         else:
-            if markers is not None:
-                markerid = id(o)
-                if markerid in markers:
-                    raise ValueError("Circular reference detected")
-                markers[markerid] = o
-            o = _default(o)
-            for chunk in _iterencode(o, _current_indent_level):
+            self._mark_markers(markers, o)
+            obj = self.default(o)
+            for chunk in self._iterencode(obj, markers,
+                                          _current_indent_level):
                 yield chunk
-            if markers is not None:
-                del markers[markerid]
-
-    return _iterencode
+            self._remove_markers(markers, o)
diff --git a/lib-python/modified-2.7/json/tests/test_unicode.py b/lib-python/modified-2.7/json/tests/test_unicode.py
--- a/lib-python/modified-2.7/json/tests/test_unicode.py
+++ b/lib-python/modified-2.7/json/tests/test_unicode.py
@@ -80,3 +80,9 @@
         self.assertEqual(type(json.loads(u'["a"]')[0]), unicode)
         # Issue 10038.
         self.assertEqual(type(json.loads('"foo"')), unicode)
+
+    def test_encode_not_utf_8(self):
+        self.assertEqual(json.dumps('\xb1\xe6', encoding='iso8859-2'),
+                         '"\\u0105\\u0107"')
+        self.assertEqual(json.dumps(['\xb1\xe6'], encoding='iso8859-2'),
+                         '["\\u0105\\u0107"]')
diff --git a/lib-python/modified-2.7/urllib2.py b/lib-python/modified-2.7/urllib2.py
--- a/lib-python/modified-2.7/urllib2.py
+++ b/lib-python/modified-2.7/urllib2.py
@@ -395,11 +395,7 @@
         meth_name = protocol+"_response"
         for processor in self.process_response.get(protocol, []):
             meth = getattr(processor, meth_name)
-            try:
-                response = meth(req, response)
-            except:
-                response.close()
-                raise
+            response = meth(req, response)
 
         return response
 
diff --git a/lib_pypy/pyrepl/readline.py b/lib_pypy/pyrepl/readline.py
--- a/lib_pypy/pyrepl/readline.py
+++ b/lib_pypy/pyrepl/readline.py
@@ -395,9 +395,21 @@
     _wrapper.f_in = f_in
     _wrapper.f_out = f_out
 
-    if hasattr(sys, '__raw_input__'):    # PyPy
-        _old_raw_input = sys.__raw_input__
+    if '__pypy__' in sys.builtin_module_names:    # PyPy
+
+        def _old_raw_input(prompt=''):
+            # sys.__raw_input__() is only called when stdin and stdout are
+            # as expected and are ttys.  If it is the case, then get_reader()
+            # should not really fail in _wrapper.raw_input().  If it still
+            # does, then we will just cancel the redirection and call again
+            # the built-in raw_input().
+            try:
+                del sys.__raw_input__
+            except AttributeError:
+                pass
+            return raw_input(prompt)
         sys.__raw_input__ = _wrapper.raw_input
+
     else:
         # this is not really what readline.c does.  Better than nothing I guess
         import __builtin__
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -72,6 +72,7 @@
     del working_modules['fcntl']  # LOCK_NB not defined
     del working_modules["_minimal_curses"]
     del working_modules["termios"]
+    del working_modules["_multiprocessing"]   # depends on rctime
 
 
 
diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -17,6 +17,12 @@
 projects, or anything else in PyPy, pop up on IRC or write to us on the
 `mailing list`_.
 
+Make big integers faster
+-------------------------
+
+PyPy's implementation of the Python ``long`` type is slower than CPython's.
+Find out why and optimize them.
+
 Numpy improvements
 ------------------
 
diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -2,7 +2,7 @@
 from pypy.interpreter.baseobjspace import Wrappable
 from pypy.interpreter import typedef
 from pypy.interpreter.gateway import interp2app
-from pypy.interpreter.error import OperationError
+from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.tool.pairtype import extendabletype
 from pypy.tool.sourcetools import func_with_new_name
@@ -2925,14 +2925,13 @@
 def Module_get_body(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -2968,14 +2967,13 @@
 def Interactive_get_body(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3015,8 +3013,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     return space.wrap(w_self.body)
 
 def Expression_set_body(space, w_self, w_new_value):
@@ -3057,14 +3054,13 @@
 def Suite_get_body(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3104,8 +3100,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._lineno_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'lineno'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lineno')
     return space.wrap(w_self.lineno)
 
 def stmt_set_lineno(space, w_self, w_new_value):
@@ -3126,8 +3121,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._col_offset_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'col_offset'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'col_offset')
     return space.wrap(w_self.col_offset)
 
 def stmt_set_col_offset(space, w_self, w_new_value):
@@ -3157,8 +3151,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'name'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name')
     return space.wrap(w_self.name)
 
 def FunctionDef_set_name(space, w_self, w_new_value):
@@ -3179,8 +3172,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'args'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args')
     return space.wrap(w_self.args)
 
 def FunctionDef_set_args(space, w_self, w_new_value):
@@ -3197,14 +3189,13 @@
 def FunctionDef_get_body(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3215,14 +3206,13 @@
 def FunctionDef_get_decorator_list(space, w_self):
     if not w_self.initialization_state & 8:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'decorator_list'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'decorator_list')
     if w_self.w_decorator_list is None:
         if w_self.decorator_list is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.decorator_list]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_decorator_list = w_list
     return w_self.w_decorator_list
 
@@ -3266,8 +3256,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'name'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name')
     return space.wrap(w_self.name)
 
 def ClassDef_set_name(space, w_self, w_new_value):
@@ -3284,14 +3273,13 @@
 def ClassDef_get_bases(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'bases'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'bases')
     if w_self.w_bases is None:
         if w_self.bases is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.bases]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_bases = w_list
     return w_self.w_bases
 
@@ -3302,14 +3290,13 @@
 def ClassDef_get_body(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3320,14 +3307,13 @@
 def ClassDef_get_decorator_list(space, w_self):
     if not w_self.initialization_state & 8:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'decorator_list'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'decorator_list')
     if w_self.w_decorator_list is None:
         if w_self.decorator_list is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.decorator_list]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_decorator_list = w_list
     return w_self.w_decorator_list
 
@@ -3372,8 +3358,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Return_set_value(space, w_self, w_new_value):
@@ -3414,14 +3399,13 @@
 def Delete_get_targets(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'targets'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'targets')
     if w_self.w_targets is None:
         if w_self.targets is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.targets]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_targets = w_list
     return w_self.w_targets
 
@@ -3457,14 +3441,13 @@
 def Assign_get_targets(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'targets'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'targets')
     if w_self.w_targets is None:
         if w_self.targets is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.targets]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_targets = w_list
     return w_self.w_targets
 
@@ -3479,8 +3462,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Assign_set_value(space, w_self, w_new_value):
@@ -3527,8 +3509,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'target'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'target')
     return space.wrap(w_self.target)
 
 def AugAssign_set_target(space, w_self, w_new_value):
@@ -3549,8 +3530,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'op'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op')
     return operator_to_class[w_self.op - 1]()
 
 def AugAssign_set_op(space, w_self, w_new_value):
@@ -3573,8 +3553,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def AugAssign_set_value(space, w_self, w_new_value):
@@ -3621,8 +3600,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'dest'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'dest')
     return space.wrap(w_self.dest)
 
 def Print_set_dest(space, w_self, w_new_value):
@@ -3639,14 +3617,13 @@
 def Print_get_values(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'values'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'values')
     if w_self.w_values is None:
         if w_self.values is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.values]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_values = w_list
     return w_self.w_values
 
@@ -3661,8 +3638,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'nl'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'nl')
     return space.wrap(w_self.nl)
 
 def Print_set_nl(space, w_self, w_new_value):
@@ -3710,8 +3686,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'target'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'target')
     return space.wrap(w_self.target)
 
 def For_set_target(space, w_self, w_new_value):
@@ -3732,8 +3707,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'iter'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'iter')
     return space.wrap(w_self.iter)
 
 def For_set_iter(space, w_self, w_new_value):
@@ -3750,14 +3724,13 @@
 def For_get_body(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3768,14 +3741,13 @@
 def For_get_orelse(space, w_self):
     if not w_self.initialization_state & 8:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'orelse'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
     if w_self.w_orelse is None:
         if w_self.orelse is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.orelse]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_orelse = w_list
     return w_self.w_orelse
 
@@ -3819,8 +3791,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'test'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test')
     return space.wrap(w_self.test)
 
 def While_set_test(space, w_self, w_new_value):
@@ -3837,14 +3808,13 @@
 def While_get_body(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3855,14 +3825,13 @@
 def While_get_orelse(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'orelse'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
     if w_self.w_orelse is None:
         if w_self.orelse is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.orelse]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_orelse = w_list
     return w_self.w_orelse
 
@@ -3905,8 +3874,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'test'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test')
     return space.wrap(w_self.test)
 
 def If_set_test(space, w_self, w_new_value):
@@ -3923,14 +3891,13 @@
 def If_get_body(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -3941,14 +3908,13 @@
 def If_get_orelse(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'orelse'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
     if w_self.w_orelse is None:
         if w_self.orelse is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.orelse]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_orelse = w_list
     return w_self.w_orelse
 
@@ -3991,8 +3957,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'context_expr'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'context_expr')
     return space.wrap(w_self.context_expr)
 
 def With_set_context_expr(space, w_self, w_new_value):
@@ -4013,8 +3978,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'optional_vars'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'optional_vars')
     return space.wrap(w_self.optional_vars)
 
 def With_set_optional_vars(space, w_self, w_new_value):
@@ -4031,14 +3995,13 @@
 def With_get_body(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -4080,8 +4043,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'type'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'type')
     return space.wrap(w_self.type)
 
 def Raise_set_type(space, w_self, w_new_value):
@@ -4102,8 +4064,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'inst'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'inst')
     return space.wrap(w_self.inst)
 
 def Raise_set_inst(space, w_self, w_new_value):
@@ -4124,8 +4085,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'tback'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'tback')
     return space.wrap(w_self.tback)
 
 def Raise_set_tback(space, w_self, w_new_value):
@@ -4168,14 +4128,13 @@
 def TryExcept_get_body(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -4186,14 +4145,13 @@
 def TryExcept_get_handlers(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'handlers'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'handlers')
     if w_self.w_handlers is None:
         if w_self.handlers is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.handlers]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_handlers = w_list
     return w_self.w_handlers
 
@@ -4204,14 +4162,13 @@
 def TryExcept_get_orelse(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'orelse'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
     if w_self.w_orelse is None:
         if w_self.orelse is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.orelse]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_orelse = w_list
     return w_self.w_orelse
 
@@ -4251,14 +4208,13 @@
 def TryFinally_get_body(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -4269,14 +4225,13 @@
 def TryFinally_get_finalbody(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'finalbody'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'finalbody')
     if w_self.w_finalbody is None:
         if w_self.finalbody is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.finalbody]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_finalbody = w_list
     return w_self.w_finalbody
 
@@ -4318,8 +4273,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'test'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test')
     return space.wrap(w_self.test)
 
 def Assert_set_test(space, w_self, w_new_value):
@@ -4340,8 +4294,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'msg'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'msg')
     return space.wrap(w_self.msg)
 
 def Assert_set_msg(space, w_self, w_new_value):
@@ -4383,14 +4336,13 @@
 def Import_get_names(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'names'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'names')
     if w_self.w_names is None:
         if w_self.names is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.names]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_names = w_list
     return w_self.w_names
 
@@ -4430,8 +4382,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'module'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'module')
     return space.wrap(w_self.module)
 
 def ImportFrom_set_module(space, w_self, w_new_value):
@@ -4451,14 +4402,13 @@
 def ImportFrom_get_names(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'names'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'names')
     if w_self.w_names is None:
         if w_self.names is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.names]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_names = w_list
     return w_self.w_names
 
@@ -4473,8 +4423,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'level'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'level')
     return space.wrap(w_self.level)
 
 def ImportFrom_set_level(space, w_self, w_new_value):
@@ -4522,8 +4471,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     return space.wrap(w_self.body)
 
 def Exec_set_body(space, w_self, w_new_value):
@@ -4544,8 +4492,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'globals'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'globals')
     return space.wrap(w_self.globals)
 
 def Exec_set_globals(space, w_self, w_new_value):
@@ -4566,8 +4513,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'locals'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'locals')
     return space.wrap(w_self.locals)
 
 def Exec_set_locals(space, w_self, w_new_value):
@@ -4610,14 +4556,13 @@
 def Global_get_names(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'names'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'names')
     if w_self.w_names is None:
         if w_self.names is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.names]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_names = w_list
     return w_self.w_names
 
@@ -4657,8 +4602,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Expr_set_value(space, w_self, w_new_value):
@@ -4754,8 +4698,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._lineno_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'lineno'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lineno')
     return space.wrap(w_self.lineno)
 
 def expr_set_lineno(space, w_self, w_new_value):
@@ -4776,8 +4719,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._col_offset_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'col_offset'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'col_offset')
     return space.wrap(w_self.col_offset)
 
 def expr_set_col_offset(space, w_self, w_new_value):
@@ -4807,8 +4749,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'op'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op')
     return boolop_to_class[w_self.op - 1]()
 
 def BoolOp_set_op(space, w_self, w_new_value):
@@ -4827,14 +4768,13 @@
 def BoolOp_get_values(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'values'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'values')
     if w_self.w_values is None:
         if w_self.values is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.values]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_values = w_list
     return w_self.w_values
 
@@ -4875,8 +4815,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'left'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'left')
     return space.wrap(w_self.left)
 
 def BinOp_set_left(space, w_self, w_new_value):
@@ -4897,8 +4836,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'op'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op')
     return operator_to_class[w_self.op - 1]()
 
 def BinOp_set_op(space, w_self, w_new_value):
@@ -4921,8 +4859,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'right'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'right')
     return space.wrap(w_self.right)
 
 def BinOp_set_right(space, w_self, w_new_value):
@@ -4969,8 +4906,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'op'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'op')
     return unaryop_to_class[w_self.op - 1]()
 
 def UnaryOp_set_op(space, w_self, w_new_value):
@@ -4993,8 +4929,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'operand'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'operand')
     return space.wrap(w_self.operand)
 
 def UnaryOp_set_operand(space, w_self, w_new_value):
@@ -5040,8 +4975,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'args'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args')
     return space.wrap(w_self.args)
 
 def Lambda_set_args(space, w_self, w_new_value):
@@ -5062,8 +4996,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     return space.wrap(w_self.body)
 
 def Lambda_set_body(space, w_self, w_new_value):
@@ -5109,8 +5042,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'test'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'test')
     return space.wrap(w_self.test)
 
 def IfExp_set_test(space, w_self, w_new_value):
@@ -5131,8 +5063,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     return space.wrap(w_self.body)
 
 def IfExp_set_body(space, w_self, w_new_value):
@@ -5153,8 +5084,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'orelse'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'orelse')
     return space.wrap(w_self.orelse)
 
 def IfExp_set_orelse(space, w_self, w_new_value):
@@ -5197,14 +5127,13 @@
 def Dict_get_keys(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'keys'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'keys')
     if w_self.w_keys is None:
         if w_self.keys is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.keys]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_keys = w_list
     return w_self.w_keys
 
@@ -5215,14 +5144,13 @@
 def Dict_get_values(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'values'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'values')
     if w_self.w_values is None:
         if w_self.values is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.values]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_values = w_list
     return w_self.w_values
 
@@ -5260,14 +5188,13 @@
 def Set_get_elts(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elts'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elts')
     if w_self.w_elts is None:
         if w_self.elts is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.elts]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_elts = w_list
     return w_self.w_elts
 
@@ -5307,8 +5234,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elt'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elt')
     return space.wrap(w_self.elt)
 
 def ListComp_set_elt(space, w_self, w_new_value):
@@ -5325,14 +5251,13 @@
 def ListComp_get_generators(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'generators'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators')
     if w_self.w_generators is None:
         if w_self.generators is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.generators]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_generators = w_list
     return w_self.w_generators
 
@@ -5373,8 +5298,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elt'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elt')
     return space.wrap(w_self.elt)
 
 def SetComp_set_elt(space, w_self, w_new_value):
@@ -5391,14 +5315,13 @@
 def SetComp_get_generators(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'generators'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators')
     if w_self.w_generators is None:
         if w_self.generators is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.generators]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_generators = w_list
     return w_self.w_generators
 
@@ -5439,8 +5362,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'key'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'key')
     return space.wrap(w_self.key)
 
 def DictComp_set_key(space, w_self, w_new_value):
@@ -5461,8 +5383,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def DictComp_set_value(space, w_self, w_new_value):
@@ -5479,14 +5400,13 @@
 def DictComp_get_generators(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'generators'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators')
     if w_self.w_generators is None:
         if w_self.generators is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.generators]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_generators = w_list
     return w_self.w_generators
 
@@ -5528,8 +5448,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elt'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elt')
     return space.wrap(w_self.elt)
 
 def GeneratorExp_set_elt(space, w_self, w_new_value):
@@ -5546,14 +5465,13 @@
 def GeneratorExp_get_generators(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'generators'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'generators')
     if w_self.w_generators is None:
         if w_self.generators is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.generators]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_generators = w_list
     return w_self.w_generators
 
@@ -5594,8 +5512,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Yield_set_value(space, w_self, w_new_value):
@@ -5640,8 +5557,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'left'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'left')
     return space.wrap(w_self.left)
 
 def Compare_set_left(space, w_self, w_new_value):
@@ -5658,14 +5574,13 @@
 def Compare_get_ops(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ops'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ops')
     if w_self.w_ops is None:
         if w_self.ops is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [cmpop_to_class[node - 1]() for node in w_self.ops]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_ops = w_list
     return w_self.w_ops
 
@@ -5676,14 +5591,13 @@
 def Compare_get_comparators(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'comparators'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'comparators')
     if w_self.w_comparators is None:
         if w_self.comparators is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.comparators]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_comparators = w_list
     return w_self.w_comparators
 
@@ -5726,8 +5640,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'func'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'func')
     return space.wrap(w_self.func)
 
 def Call_set_func(space, w_self, w_new_value):
@@ -5744,14 +5657,13 @@
 def Call_get_args(space, w_self):
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'args'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args')
     if w_self.w_args is None:
         if w_self.args is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.args]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_args = w_list
     return w_self.w_args
 
@@ -5762,14 +5674,13 @@
 def Call_get_keywords(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'keywords'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'keywords')
     if w_self.w_keywords is None:
         if w_self.keywords is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.keywords]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_keywords = w_list
     return w_self.w_keywords
 
@@ -5784,8 +5695,7 @@
             return w_obj
     if not w_self.initialization_state & 8:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'starargs'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'starargs')
     return space.wrap(w_self.starargs)
 
 def Call_set_starargs(space, w_self, w_new_value):
@@ -5806,8 +5716,7 @@
             return w_obj
     if not w_self.initialization_state & 16:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'kwargs'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'kwargs')
     return space.wrap(w_self.kwargs)
 
 def Call_set_kwargs(space, w_self, w_new_value):
@@ -5858,8 +5767,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Repr_set_value(space, w_self, w_new_value):
@@ -5904,8 +5812,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'n'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'n')
     return w_self.n
 
 def Num_set_n(space, w_self, w_new_value):
@@ -5950,8 +5857,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 's'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 's')
     return w_self.s
 
 def Str_set_s(space, w_self, w_new_value):
@@ -5996,8 +5902,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Attribute_set_value(space, w_self, w_new_value):
@@ -6018,8 +5923,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'attr'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'attr')
     return space.wrap(w_self.attr)
 
 def Attribute_set_attr(space, w_self, w_new_value):
@@ -6040,8 +5944,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ctx'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def Attribute_set_ctx(space, w_self, w_new_value):
@@ -6090,8 +5993,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Subscript_set_value(space, w_self, w_new_value):
@@ -6112,8 +6014,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'slice'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'slice')
     return space.wrap(w_self.slice)
 
 def Subscript_set_slice(space, w_self, w_new_value):
@@ -6134,8 +6035,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ctx'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def Subscript_set_ctx(space, w_self, w_new_value):
@@ -6184,8 +6084,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'id'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'id')
     return space.wrap(w_self.id)
 
 def Name_set_id(space, w_self, w_new_value):
@@ -6206,8 +6105,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ctx'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def Name_set_ctx(space, w_self, w_new_value):
@@ -6251,14 +6149,13 @@
 def List_get_elts(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elts'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elts')
     if w_self.w_elts is None:
         if w_self.elts is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.elts]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_elts = w_list
     return w_self.w_elts
 
@@ -6273,8 +6170,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ctx'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def List_set_ctx(space, w_self, w_new_value):
@@ -6319,14 +6215,13 @@
 def Tuple_get_elts(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'elts'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'elts')
     if w_self.w_elts is None:
         if w_self.elts is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.elts]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_elts = w_list
     return w_self.w_elts
 
@@ -6341,8 +6236,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ctx'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ctx')
     return expr_context_to_class[w_self.ctx - 1]()
 
 def Tuple_set_ctx(space, w_self, w_new_value):
@@ -6391,8 +6285,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return w_self.value
 
 def Const_set_value(space, w_self, w_new_value):
@@ -6510,8 +6403,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'lower'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lower')
     return space.wrap(w_self.lower)
 
 def Slice_set_lower(space, w_self, w_new_value):
@@ -6532,8 +6424,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'upper'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'upper')
     return space.wrap(w_self.upper)
 
 def Slice_set_upper(space, w_self, w_new_value):
@@ -6554,8 +6445,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'step'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'step')
     return space.wrap(w_self.step)
 
 def Slice_set_step(space, w_self, w_new_value):
@@ -6598,14 +6488,13 @@
 def ExtSlice_get_dims(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'dims'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'dims')
     if w_self.w_dims is None:
         if w_self.dims is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.dims]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_dims = w_list
     return w_self.w_dims
 
@@ -6645,8 +6534,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def Index_set_value(space, w_self, w_new_value):
@@ -6915,8 +6803,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'target'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'target')
     return space.wrap(w_self.target)
 
 def comprehension_set_target(space, w_self, w_new_value):
@@ -6937,8 +6824,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'iter'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'iter')
     return space.wrap(w_self.iter)
 
 def comprehension_set_iter(space, w_self, w_new_value):
@@ -6955,14 +6841,13 @@
 def comprehension_get_ifs(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'ifs'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'ifs')
     if w_self.w_ifs is None:
         if w_self.ifs is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.ifs]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_ifs = w_list
     return w_self.w_ifs
 
@@ -7004,8 +6889,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._lineno_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'lineno'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'lineno')
     return space.wrap(w_self.lineno)
 
 def excepthandler_set_lineno(space, w_self, w_new_value):
@@ -7026,8 +6910,7 @@
             return w_obj
     if not w_self.initialization_state & w_self._col_offset_mask:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'col_offset'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'col_offset')
     return space.wrap(w_self.col_offset)
 
 def excepthandler_set_col_offset(space, w_self, w_new_value):
@@ -7057,8 +6940,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'type'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'type')
     return space.wrap(w_self.type)
 
 def ExceptHandler_set_type(space, w_self, w_new_value):
@@ -7079,8 +6961,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'name'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name')
     return space.wrap(w_self.name)
 
 def ExceptHandler_set_name(space, w_self, w_new_value):
@@ -7097,14 +6978,13 @@
 def ExceptHandler_get_body(space, w_self):
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'body'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'body')
     if w_self.w_body is None:
         if w_self.body is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.body]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_body = w_list
     return w_self.w_body
 
@@ -7142,14 +7022,13 @@
 def arguments_get_args(space, w_self):
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'args'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'args')
     if w_self.w_args is None:
         if w_self.args is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.args]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_args = w_list
     return w_self.w_args
 
@@ -7164,8 +7043,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'vararg'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'vararg')
     return space.wrap(w_self.vararg)
 
 def arguments_set_vararg(space, w_self, w_new_value):
@@ -7189,8 +7067,7 @@
             return w_obj
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'kwarg'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'kwarg')
     return space.wrap(w_self.kwarg)
 
 def arguments_set_kwarg(space, w_self, w_new_value):
@@ -7210,14 +7087,13 @@
 def arguments_get_defaults(space, w_self):
     if not w_self.initialization_state & 8:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'defaults'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'defaults')
     if w_self.w_defaults is None:
         if w_self.defaults is None:
-            w_list = space.newlist([])
+            list_w = []
         else:
             list_w = [space.wrap(node) for node in w_self.defaults]
-            w_list = space.newlist(list_w)
+        w_list = space.newlist(list_w)
         w_self.w_defaults = w_list
     return w_self.w_defaults
 
@@ -7261,8 +7137,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'arg'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'arg')
     return space.wrap(w_self.arg)
 
 def keyword_set_arg(space, w_self, w_new_value):
@@ -7283,8 +7158,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'value'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'value')
     return space.wrap(w_self.value)
 
 def keyword_set_value(space, w_self, w_new_value):
@@ -7330,8 +7204,7 @@
             return w_obj
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'name'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'name')
     return space.wrap(w_self.name)
 
 def alias_set_name(space, w_self, w_new_value):
@@ -7352,8 +7225,7 @@
             return w_obj
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
-        w_err = space.wrap("'%s' object has no attribute 'asname'" % typename)
-        raise OperationError(space.w_AttributeError, w_err)
+        raise operationerrfmt(space.w_AttributeError, "'%s' object has no attribute '%s'", typename, 'asname')
     return space.wrap(w_self.asname)
 
 def alias_set_asname(space, w_self, w_new_value):
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -414,13 +414,12 @@
             self.emit("        return w_obj", 1)
         self.emit("if not w_self.initialization_state & %s:" % (flag,), 1)
         self.emit("typename = space.type(w_self).getname(space)", 2)
-        self.emit("w_err = space.wrap(\"'%%s' object has no attribute '%s'\" %% typename)" %
+        self.emit("raise operationerrfmt(space.w_AttributeError, \"'%%s' object has no attribute '%%s'\", typename, '%s')" %
                   (field.name,), 2)
-        self.emit("raise OperationError(space.w_AttributeError, w_err)", 2)
         if field.seq:
             self.emit("if w_self.w_%s is None:" % (field.name,), 1)
             self.emit("if w_self.%s is None:" % (field.name,), 2)
-            self.emit("w_list = space.newlist([])", 3)
+            self.emit("list_w = []", 3)
             self.emit("else:", 2)
             if field.type.value in self.data.simple_types:
                 wrapper = "%s_to_class[node - 1]()" % (field.type,)
@@ -428,7 +427,7 @@
                 wrapper = "space.wrap(node)"
             self.emit("list_w = [%s for node in w_self.%s]" %
                       (wrapper, field.name), 3)
-            self.emit("w_list = space.newlist(list_w)", 3)
+            self.emit("w_list = space.newlist(list_w)", 2)
             self.emit("w_self.w_%s = w_list" % (field.name,), 2)
             self.emit("return w_self.w_%s" % (field.name,), 1)
         elif field.type.value in self.data.simple_types:
@@ -540,7 +539,7 @@
 from pypy.interpreter.baseobjspace import Wrappable
 from pypy.interpreter import typedef
 from pypy.interpreter.gateway import interp2app
-from pypy.interpreter.error import OperationError
+from pypy.interpreter.error import OperationError, operationerrfmt
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.tool.pairtype import extendabletype
 from pypy.tool.sourcetools import func_with_new_name
@@ -639,9 +638,7 @@
             missing = required[i]
             if missing is not None:
                  err = "required field \\"%s\\" missing from %s"
-                 err = err % (missing, host)
-                 w_err = space.wrap(err)
-                 raise OperationError(space.w_TypeError, w_err)
+                 raise operationerrfmt(space.w_TypeError, err, missing, host)
     raise AssertionError("should not reach here")
 
 
diff --git a/pypy/jit/backend/llgraph/llimpl.py b/pypy/jit/backend/llgraph/llimpl.py
--- a/pypy/jit/backend/llgraph/llimpl.py
+++ b/pypy/jit/backend/llgraph/llimpl.py
@@ -7,9 +7,7 @@
 import weakref
 from pypy.objspace.flow.model import Variable, Constant
 from pypy.annotation import model as annmodel
-from pypy.jit.metainterp.history import (ConstInt, ConstPtr,
-                                         BoxInt, BoxPtr, BoxObj, BoxFloat,
-                                         REF, INT, FLOAT)
+from pypy.jit.metainterp.history import REF, INT, FLOAT
 from pypy.jit.codewriter import heaptracker
 from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr, rffi
 from pypy.rpython.ootypesystem import ootype
@@ -17,7 +15,7 @@
 from pypy.rpython.llinterp import LLException
 from pypy.rpython.extregistry import ExtRegistryEntry
 
-from pypy.jit.metainterp import resoperation, executor
+from pypy.jit.metainterp import resoperation
 from pypy.jit.metainterp.resoperation import rop
 from pypy.jit.backend.llgraph import symbolic
 from pypy.jit.codewriter import longlong
@@ -334,6 +332,13 @@
     assert isinstance(type, str) and len(type) == 1
     op.descr = Descr(ofs, type, arg_types=arg_types)
 
+def compile_add_descr_arg(loop, ofs, type, arg_types):
+    from pypy.jit.backend.llgraph.runner import Descr
+    loop = _from_opaque(loop)
+    op = loop.operations[-1]
+    assert isinstance(type, str) and len(type) == 1
+    op.args.append(Descr(ofs, type, arg_types=arg_types))
+
 def compile_add_loop_token(loop, descr):
     if we_are_translated():
         raise ValueError("CALL_ASSEMBLER not supported")
@@ -438,8 +443,11 @@
         self._may_force = -1
 
     def getenv(self, v):
+        from pypy.jit.backend.llgraph.runner import Descr
         if isinstance(v, Constant):
             return v.value
+        elif isinstance(v, Descr):
+            return v
         else:
             return self.env[v]
 
@@ -807,6 +815,29 @@
         else:
             raise NotImplementedError
 
+    def op_getinteriorfield_gc(self, descr, array, index):
+        if descr.typeinfo == REF:
+            return do_getinteriorfield_gc_ptr(array, index, descr.ofs)
+        elif descr.typeinfo == INT:
+            return do_getinteriorfield_gc_int(array, index, descr.ofs)
+        elif descr.typeinfo == FLOAT:
+            return do_getinteriorfield_gc_float(array, index, descr.ofs)
+        else:
+            raise NotImplementedError
+
+    def op_setinteriorfield_gc(self, descr, array, index, newvalue):
+        if descr.typeinfo == REF:
+            return do_setinteriorfield_gc_ptr(array, index, descr.ofs,
+                                              newvalue)
+        elif descr.typeinfo == INT:
+            return do_setinteriorfield_gc_int(array, index, descr.ofs,
+                                              newvalue)
+        elif descr.typeinfo == FLOAT:
+            return do_setinteriorfield_gc_float(array, index, descr.ofs,
+                                                newvalue)
+        else:
+            raise NotImplementedError
+
     def op_setfield_gc(self, fielddescr, struct, newvalue):
         if fielddescr.typeinfo == REF:
             do_setfield_gc_ptr(struct, fielddescr.ofs, newvalue)
@@ -1354,6 +1385,22 @@
 def do_getfield_gc_ptr(struct, fieldnum):
     return cast_to_ptr(_getfield_gc(struct, fieldnum))
 
+def _getinteriorfield_gc(struct, fieldnum):
+    STRUCT, fieldname = symbolic.TokenToField[fieldnum]
+    return getattr(struct, fieldname)
+
+def do_getinteriorfield_gc_int(array, index, fieldnum):
+    struct = array._obj.container.getitem(index)
+    return cast_to_int(_getinteriorfield_gc(struct, fieldnum))
+
+def do_getinteriorfield_gc_float(array, index, fieldnum):
+    struct = array._obj.container.getitem(index)
+    return cast_to_floatstorage(_getinteriorfield_gc(struct, fieldnum))
+
+def do_getinteriorfield_gc_ptr(array, index, fieldnum):
+    struct = array._obj.container.getitem(index)
+    return cast_to_ptr(_getinteriorfield_gc(struct, fieldnum))
+
 def _getfield_raw(struct, fieldnum):
     STRUCT, fieldname = symbolic.TokenToField[fieldnum]
     ptr = cast_from_int(lltype.Ptr(STRUCT), struct)
@@ -1409,26 +1456,28 @@
     newvalue = cast_from_ptr(ITEMTYPE, newvalue)
     array.setitem(index, newvalue)
 
-def do_setfield_gc_int(struct, fieldnum, newvalue):
-    STRUCT, fieldname = symbolic.TokenToField[fieldnum]
-    ptr = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), struct)
-    FIELDTYPE = getattr(STRUCT, fieldname)
-    newvalue = cast_from_int(FIELDTYPE, newvalue)
-    setattr(ptr, fieldname, newvalue)
+def new_setfield_gc(cast_func):
+    def do_setfield_gc(struct, fieldnum, newvalue):
+        STRUCT, fieldname = symbolic.TokenToField[fieldnum]
+        ptr = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), struct)
+        FIELDTYPE = getattr(STRUCT, fieldname)
+        newvalue = cast_func(FIELDTYPE, newvalue)
+        setattr(ptr, fieldname, newvalue)
+    return do_setfield_gc
+do_setfield_gc_int = new_setfield_gc(cast_from_int)
+do_setfield_gc_float = new_setfield_gc(cast_from_floatstorage)
+do_setfield_gc_ptr = new_setfield_gc(cast_from_ptr)
 
-def do_setfield_gc_float(struct, fieldnum, newvalue):
-    STRUCT, fieldname = symbolic.TokenToField[fieldnum]
-    ptr = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), struct)
-    FIELDTYPE = getattr(STRUCT, fieldname)
-    newvalue = cast_from_floatstorage(FIELDTYPE, newvalue)
-    setattr(ptr, fieldname, newvalue)
-
-def do_setfield_gc_ptr(struct, fieldnum, newvalue):
-    STRUCT, fieldname = symbolic.TokenToField[fieldnum]
-    ptr = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), struct)
-    FIELDTYPE = getattr(STRUCT, fieldname)
-    newvalue = cast_from_ptr(FIELDTYPE, newvalue)
-    setattr(ptr, fieldname, newvalue)
+def new_setinteriorfield_gc(cast_func):
+    def do_setinteriorfield_gc(array, index, fieldnum, newvalue):
+        STRUCT, fieldname = symbolic.TokenToField[fieldnum]
+        struct = array._obj.container.getitem(index)
+        FIELDTYPE = getattr(STRUCT, fieldname)
+        setattr(struct, fieldname, cast_func(FIELDTYPE, newvalue))
+    return do_setinteriorfield_gc
+do_setinteriorfield_gc_int = new_setinteriorfield_gc(cast_from_int)
+do_setinteriorfield_gc_float = new_setinteriorfield_gc(cast_from_floatstorage)
+do_setinteriorfield_gc_ptr = new_setinteriorfield_gc(cast_from_ptr)        
 
 def do_setfield_raw_int(struct, fieldnum, newvalue):
     STRUCT, fieldname = symbolic.TokenToField[fieldnum]
@@ -1694,6 +1743,7 @@
 setannotation(compile_start_float_var, annmodel.SomeInteger())
 setannotation(compile_add, annmodel.s_None)
 setannotation(compile_add_descr, annmodel.s_None)
+setannotation(compile_add_descr_arg, annmodel.s_None)
 setannotation(compile_add_var, annmodel.s_None)
 setannotation(compile_add_int_const, annmodel.s_None)
 setannotation(compile_add_ref_const, annmodel.s_None)
@@ -1741,6 +1791,9 @@
 setannotation(do_getfield_raw_int, annmodel.SomeInteger())
 setannotation(do_getfield_raw_ptr, annmodel.SomePtr(llmemory.GCREF))
 setannotation(do_getfield_raw_float, s_FloatStorage)
+setannotation(do_getinteriorfield_gc_int, annmodel.SomeInteger())
+setannotation(do_getinteriorfield_gc_ptr, annmodel.SomePtr(llmemory.GCREF))
+setannotation(do_getinteriorfield_gc_float, s_FloatStorage)
 setannotation(do_new, annmodel.SomePtr(llmemory.GCREF))
 setannotation(do_new_array, annmodel.SomePtr(llmemory.GCREF))
 setannotation(do_setarrayitem_gc_int, annmodel.s_None)
@@ -1754,6 +1807,9 @@
 setannotation(do_setfield_raw_int, annmodel.s_None)
 setannotation(do_setfield_raw_ptr, annmodel.s_None)
 setannotation(do_setfield_raw_float, annmodel.s_None)
+setannotation(do_setinteriorfield_gc_int, annmodel.s_None)
+setannotation(do_setinteriorfield_gc_ptr, annmodel.s_None)
+setannotation(do_setinteriorfield_gc_float, annmodel.s_None)
 setannotation(do_newstr, annmodel.SomePtr(llmemory.GCREF))
 setannotation(do_strsetitem, annmodel.s_None)
 setannotation(do_newunicode, annmodel.SomePtr(llmemory.GCREF))
diff --git a/pypy/jit/backend/llgraph/runner.py b/pypy/jit/backend/llgraph/runner.py
--- a/pypy/jit/backend/llgraph/runner.py
+++ b/pypy/jit/backend/llgraph/runner.py
@@ -2,21 +2,19 @@
 Minimal-API wrapper around the llinterpreter to run operations.
 """
 
-import sys
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.rpython.lltypesystem import lltype, llmemory, rclass
 from pypy.rpython.ootypesystem import ootype
 from pypy.rpython.llinterp import LLInterpreter
 from pypy.jit.metainterp import history
-from pypy.jit.metainterp.history import REF, INT, FLOAT
+from pypy.jit.metainterp.history import REF, INT, FLOAT, STRUCT
 from pypy.jit.metainterp.warmstate import unwrap
-from pypy.jit.metainterp.resoperation import ResOperation, rop
+from pypy.jit.metainterp.resoperation import rop
 from pypy.jit.backend import model
 from pypy.jit.backend.llgraph import llimpl, symbolic
 from pypy.jit.metainterp.typesystem import llhelper, oohelper
 from pypy.jit.codewriter import heaptracker, longlong
-from pypy.rlib import rgc
 
 class MiniStats:
     pass
@@ -62,6 +60,9 @@
     def is_array_of_floats(self):
         return self.typeinfo == FLOAT
 
+    def is_array_of_structs(self):
+        return self.typeinfo == STRUCT
+
     def as_vtable_size_descr(self):
         return self
 
@@ -177,8 +178,10 @@
             llimpl.compile_add(c, op.getopnum())
             descr = op.getdescr()
             if isinstance(descr, Descr):
-                llimpl.compile_add_descr(c, descr.ofs, descr.typeinfo, descr.arg_types)
-            if isinstance(descr, history.LoopToken) and op.getopnum() != rop.JUMP:
+                llimpl.compile_add_descr(c, descr.ofs, descr.typeinfo,
+                                         descr.arg_types)
+            if (isinstance(descr, history.LoopToken) and
+                op.getopnum() != rop.JUMP):
                 llimpl.compile_add_loop_token(c, descr)
             if self.is_oo and isinstance(descr, (OODescr, MethDescr)):
                 # hack hack, not rpython
@@ -193,6 +196,9 @@
                     llimpl.compile_add_ref_const(c, x.value, self.ts.BASETYPE)
                 elif isinstance(x, history.ConstFloat):
                     llimpl.compile_add_float_const(c, x.value)
+                elif isinstance(x, Descr):
+                    llimpl.compile_add_descr_arg(c, x.ofs, x.typeinfo,
+                                                 x.arg_types)
                 else:
                     raise Exception("'%s' args contain: %r" % (op.getopname(),
                                                                x))
@@ -316,6 +322,13 @@
         token = history.getkind(getattr(S, fieldname))
         return self.getdescr(ofs, token[0], name=fieldname)
 
+    def interiorfielddescrof(self, A, fieldname):
+        S = A.OF
+        ofs2 = symbolic.get_size(A)
+        ofs, size = symbolic.get_field_token(S, fieldname)
+        token = history.getkind(getattr(S, fieldname))
+        return self.getdescr(ofs, token[0], name=fieldname, extrainfo=ofs2)
+
     def calldescrof(self, FUNC, ARGS, RESULT, extrainfo):
         arg_types = []
         for ARG in ARGS:
@@ -353,8 +366,13 @@
     def arraydescrof(self, A):
         assert A.OF != lltype.Void
         size = symbolic.get_size(A)
-        token = history.getkind(A.OF)
-        return self.getdescr(size, token[0])
+        if isinstance(A.OF, lltype.Ptr) or isinstance(A.OF, lltype.Primitive):
+            token = history.getkind(A.OF)[0]
+        elif isinstance(A.OF, lltype.Struct):
+            token = 's'
+        else:
+            token = '?'
+        return self.getdescr(size, token)
 
     # ---------- the backend-dependent operations ----------
 
@@ -406,6 +424,29 @@
         assert isinstance(fielddescr, Descr)
         return llimpl.do_getfield_raw_float(struct, fielddescr.ofs)
 
+    def bh_getinteriorfield_gc_i(self, array, index, descr):
+        assert isinstance(descr, Descr)
+        return llimpl.do_getinteriorfield_gc_int(array, index, descr.ofs)
+    def bh_getinteriorfield_gc_r(self, array, index, descr):
+        assert isinstance(descr, Descr)
+        return llimpl.do_getinteriorfield_gc_ptr(array, index, descr.ofs)
+    def bh_getinteriorfield_gc_f(self, array, index, descr):
+        assert isinstance(descr, Descr)
+        return llimpl.do_getinteriorfield_gc_float(array, index, descr.ofs)
+
+    def bh_setinteriorfield_gc_i(self, array, index, descr, value):
+        assert isinstance(descr, Descr)
+        return llimpl.do_setinteriorfield_gc_int(array, index, descr.ofs,
+                                                 value)
+    def bh_setinteriorfield_gc_r(self, array, index, descr, value):
+        assert isinstance(descr, Descr)
+        return llimpl.do_setinteriorfield_gc_ptr(array, index, descr.ofs,
+                                                 value)
+    def bh_setinteriorfield_gc_f(self, array, index, descr, value):
+        assert isinstance(descr, Descr)
+        return llimpl.do_setinteriorfield_gc_float(array, index, descr.ofs,
+                                                   value)
+
     def bh_new(self, sizedescr):
         assert isinstance(sizedescr, Descr)
         return llimpl.do_new(sizedescr.ofs)
@@ -418,7 +459,6 @@
 
     def bh_classof(self, struct):
         struct = lltype.cast_opaque_ptr(rclass.OBJECTPTR, struct)
-        result = struct.typeptr
         result_adr = llmemory.cast_ptr_to_adr(struct.typeptr)
         return heaptracker.adr2int(result_adr)
 
diff --git a/pypy/jit/backend/llsupport/descr.py b/pypy/jit/backend/llsupport/descr.py
--- a/pypy/jit/backend/llsupport/descr.py
+++ b/pypy/jit/backend/llsupport/descr.py
@@ -1,13 +1,10 @@
 import py
-from pypy.rpython.lltypesystem import lltype, rffi, llmemory, rclass
+from pypy.rpython.lltypesystem import lltype, rffi, llmemory
 from pypy.rpython.lltypesystem.lloperation import llop
 from pypy.jit.backend.llsupport import symbolic, support
-from pypy.jit.metainterp.history import AbstractDescr, getkind, BoxInt, BoxPtr
-from pypy.jit.metainterp.history import BasicFailDescr, LoopToken, BoxFloat
+from pypy.jit.metainterp.history import AbstractDescr, getkind
 from pypy.jit.metainterp import history
-from pypy.jit.metainterp.resoperation import ResOperation, rop
 from pypy.jit.codewriter import heaptracker, longlong
-from pypy.rlib.rarithmetic import r_longlong, r_ulonglong
 
 # The point of the class organization in this file is to make instances
 # as compact as possible.  This is done by not storing the field size or
@@ -23,6 +20,7 @@
         self._cache_field = {}
         self._cache_array = {}
         self._cache_call = {}
+        self._cache_interiorfield = {}
 
     def init_size_descr(self, STRUCT, sizedescr):
         assert isinstance(STRUCT, lltype.GcStruct)
@@ -142,7 +140,6 @@
         cachedict[fieldname] = fielddescr
         return fielddescr
 
-
 # ____________________________________________________________
 # ArrayDescrs
 
@@ -167,6 +164,7 @@
 
     _is_array_of_pointers = False      # unless overridden by GcPtrArrayDescr
     _is_array_of_floats   = False      # unless overridden by FloatArrayDescr
+    _is_array_of_structs  = False      # unless overridden by StructArrayDescr
     _is_item_signed       = False      # unless overridden by XxxArrayDescr
 
     def is_array_of_pointers(self):
@@ -175,6 +173,9 @@
     def is_array_of_floats(self):
         return self._is_array_of_floats
 
+    def is_array_of_structs(self):
+        return self._is_array_of_structs
+
     def is_item_signed(self):
         return self._is_item_signed
 
@@ -199,6 +200,10 @@
     def get_item_size(self, translate_support_code):
         return symbolic.get_size(lltype.Float, translate_support_code)
 
+class StructArrayDescr(BaseArrayDescr):
+    _clsname = 'StructArrayDescr'
+    _is_array_of_structs = True
+
 class BaseArrayNoLengthDescr(BaseArrayDescr):
     def get_base_size(self, translate_support_code):
         return 0
@@ -218,6 +223,13 @@
 def getArrayDescrClass(ARRAY):
     if ARRAY.OF is lltype.Float:
         return FloatArrayDescr
+    elif isinstance(ARRAY.OF, lltype.Struct):
+        class Descr(StructArrayDescr):
+            _clsname = '%sArrayDescr' % ARRAY.OF._name
+            def get_item_size(self, translate_support_code):
+                return symbolic.get_size(ARRAY.OF, translate_support_code)
+        Descr.__name__ = Descr._clsname
+        return Descr
     return getDescrClass(ARRAY.OF, BaseArrayDescr, GcPtrArrayDescr,
                          NonGcPtrArrayDescr, 'Array', 'get_item_size',
                          '_is_array_of_floats', '_is_item_signed')
@@ -252,6 +264,36 @@
         cache[ARRAY] = arraydescr
         return arraydescr
 
+# ____________________________________________________________
+# InteriorFieldDescr
+
+class InteriorFieldDescr(AbstractDescr):
+    arraydescr = BaseArrayDescr()     # workaround for the annotator
+    fielddescr = BaseFieldDescr('', 0)
+
+    def __init__(self, arraydescr, fielddescr):
+        self.arraydescr = arraydescr
+        self.fielddescr = fielddescr
+
+    def is_pointer_field(self):
+        return self.fielddescr.is_pointer_field()
+
+    def is_float_field(self):
+        return self.fielddescr.is_float_field()
+
+    def repr_of_descr(self):
+        return '<InteriorFieldDescr %s>' % self.fielddescr.repr_of_descr()
+
+def get_interiorfield_descr(gc_ll_descr, ARRAY, FIELDTP, name):
+    cache = gc_ll_descr._cache_interiorfield
+    try:
+        return cache[(ARRAY, FIELDTP, name)]
+    except KeyError:
+        arraydescr = get_array_descr(gc_ll_descr, ARRAY)
+        fielddescr = get_field_descr(gc_ll_descr, FIELDTP, name)
+        descr = InteriorFieldDescr(arraydescr, fielddescr)
+        cache[(ARRAY, FIELDTP, name)] = descr
+        return descr
 
 # ____________________________________________________________
 # CallDescrs
@@ -525,7 +567,8 @@
         #
         if TYPE is lltype.Float or is_longlong(TYPE):
             setattr(Descr, floatattrname, True)
-        elif TYPE is not lltype.Bool and rffi.cast(TYPE, -1) == -1:
+        elif (TYPE is not lltype.Bool and isinstance(TYPE, lltype.Number) and
+              rffi.cast(TYPE, -1) == -1):
             setattr(Descr, signedattrname, True)
         #
         _cache[nameprefix, TYPE] = Descr
diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py
--- a/pypy/jit/backend/llsupport/gc.py
+++ b/pypy/jit/backend/llsupport/gc.py
@@ -45,6 +45,14 @@
     def freeing_block(self, start, stop):
         pass
 
+    def get_funcptr_for_newarray(self):
+        return llhelper(self.GC_MALLOC_ARRAY, self.malloc_array)
+    def get_funcptr_for_newstr(self):
+        return llhelper(self.GC_MALLOC_STR_UNICODE, self.malloc_str)
+    def get_funcptr_for_newunicode(self):
+        return llhelper(self.GC_MALLOC_STR_UNICODE, self.malloc_unicode)
+
+
     def record_constptrs(self, op, gcrefs_output_list):
         for i in range(op.numargs()):
             v = op.getarg(i)
@@ -96,6 +104,39 @@
         malloc_fn_ptr = self.configure_boehm_once()
         self.funcptr_for_new = malloc_fn_ptr
 
+        def malloc_array(basesize, itemsize, ofs_length, num_elem):
+            try:
+                size = ovfcheck(basesize + ovfcheck(itemsize * num_elem))
+            except OverflowError:
+                return lltype.nullptr(llmemory.GCREF.TO)
+            res = self.funcptr_for_new(size)
+            if not res:
+                return res
+            rffi.cast(rffi.CArrayPtr(lltype.Signed), res)[ofs_length/WORD] = num_elem
+            return res
+        self.malloc_array = malloc_array
+        self.GC_MALLOC_ARRAY = lltype.Ptr(lltype.FuncType(
+            [lltype.Signed] * 4, llmemory.GCREF))
+
+
+        (str_basesize, str_itemsize, str_ofs_length
+         ) = symbolic.get_array_token(rstr.STR, self.translate_support_code)
+        (unicode_basesize, unicode_itemsize, unicode_ofs_length
+         ) = symbolic.get_array_token(rstr.UNICODE, self.translate_support_code)
+        def malloc_str(length):
+            return self.malloc_array(
+                str_basesize, str_itemsize, str_ofs_length, length
+            )
+        def malloc_unicode(length):
+            return self.malloc_array(
+                unicode_basesize, unicode_itemsize, unicode_ofs_length, length
+            )
+        self.malloc_str = malloc_str
+        self.malloc_unicode = malloc_unicode
+        self.GC_MALLOC_STR_UNICODE = lltype.Ptr(lltype.FuncType(
+            [lltype.Signed], llmemory.GCREF))
+
+
         # on some platform GC_init is required before any other
         # GC_* functions, call it here for the benefit of tests
         # XXX move this to tests
@@ -116,39 +157,27 @@
         ofs_length = arraydescr.get_ofs_length(self.translate_support_code)
         basesize = arraydescr.get_base_size(self.translate_support_code)
         itemsize = arraydescr.get_item_size(self.translate_support_code)
-        size = basesize + itemsize * num_elem
-        res = self.funcptr_for_new(size)
-        rffi.cast(rffi.CArrayPtr(lltype.Signed), res)[ofs_length/WORD] = num_elem
-        return res
+        return self.malloc_array(basesize, itemsize, ofs_length, num_elem)
 
     def gc_malloc_str(self, num_elem):
-        basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR,
-                                                   self.translate_support_code)
-        assert itemsize == 1
-        size = basesize + num_elem
-        res = self.funcptr_for_new(size)
-        rffi.cast(rffi.CArrayPtr(lltype.Signed), res)[ofs_length/WORD] = num_elem
-        return res
+        return self.malloc_str(num_elem)
 
     def gc_malloc_unicode(self, num_elem):
-        basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE,
-                                                   self.translate_support_code)
-        size = basesize + num_elem * itemsize
-        res = self.funcptr_for_new(size)
-        rffi.cast(rffi.CArrayPtr(lltype.Signed), res)[ofs_length/WORD] = num_elem
-        return res
+        return self.malloc_unicode(num_elem)
 
     def args_for_new(self, sizedescr):
         assert isinstance(sizedescr, BaseSizeDescr)
         return [sizedescr.size]
 
+    def args_for_new_array(self, arraydescr):
+        ofs_length = arraydescr.get_ofs_length(self.translate_support_code)
+        basesize = arraydescr.get_base_size(self.translate_support_code)
+        itemsize = arraydescr.get_item_size(self.translate_support_code)
+        return [basesize, itemsize, ofs_length]
+
     def get_funcptr_for_new(self):
         return self.funcptr_for_new
 
-    get_funcptr_for_newarray = None
-    get_funcptr_for_newstr = None
-    get_funcptr_for_newunicode = None
-
     def rewrite_assembler(self, cpu, operations, gcrefs_output_list):
         # record all GCREFs too, because Boehm cannot see them and keep them
         # alive if they end up as constants in the assembler
@@ -620,10 +649,13 @@
         def malloc_basic(size, tid):
             type_id = llop.extract_ushort(llgroup.HALFWORD, tid)
             has_finalizer = bool(tid & (1<<llgroup.HALFSHIFT))
+            has_light_finalizer = bool(tid & (1<<(llgroup.HALFSHIFT + 1)))
             check_typeid(type_id)
             res = llop1.do_malloc_fixedsize_clear(llmemory.GCREF,
                                                   type_id, size,
-                                                  has_finalizer, False)
+                                                  has_finalizer,
+                                                  has_light_finalizer,
+                                                  False)
             # In case the operation above failed, we are returning NULL
             # from this function to assembler.  There is also an RPython
             # exception set, typically MemoryError; but it's easier and
@@ -694,7 +726,7 @@
             # also use it to allocate varsized objects.  The tid
             # and possibly the length are both set afterward.
             gcref = llop1.do_malloc_fixedsize_clear(llmemory.GCREF,
-                                        0, size, False, False)
+                                        0, size, False, False, False)
             return rffi.cast(lltype.Signed, gcref)
         self.malloc_slowpath = malloc_slowpath
         self.MALLOC_SLOWPATH = lltype.FuncType([lltype.Signed], lltype.Signed)
@@ -718,7 +750,9 @@
         type_id = self.layoutbuilder.get_type_id(S)
         assert not self.layoutbuilder.is_weakref_type(S)
         has_finalizer = bool(self.layoutbuilder.has_finalizer(S))
-        flags = int(has_finalizer) << llgroup.HALFSHIFT
+        has_light_finalizer = bool(self.layoutbuilder.has_light_finalizer(S))
+        flags = (int(has_finalizer) << llgroup.HALFSHIFT |
+                 int(has_light_finalizer) << (llgroup.HALFSHIFT + 1))
         descr.tid = llop.combine_ushort(lltype.Signed, type_id, flags)
 
     def init_array_descr(self, A, descr):
@@ -752,15 +786,6 @@
     def get_funcptr_for_new(self):
         return llhelper(self.GC_MALLOC_BASIC, self.malloc_basic)
 
-    def get_funcptr_for_newarray(self):
-        return llhelper(self.GC_MALLOC_ARRAY, self.malloc_array)
-
-    def get_funcptr_for_newstr(self):
-        return llhelper(self.GC_MALLOC_STR_UNICODE, self.malloc_str)
-
-    def get_funcptr_for_newunicode(self):
-        return llhelper(self.GC_MALLOC_STR_UNICODE, self.malloc_unicode)
-
     def do_write_barrier(self, gcref_struct, gcref_newptr):
         hdr_addr = llmemory.cast_ptr_to_adr(gcref_struct)
         hdr_addr -= self.gcheaderbuilder.size_gc_header
diff --git a/pypy/jit/backend/llsupport/llmodel.py b/pypy/jit/backend/llsupport/llmodel.py
--- a/pypy/jit/backend/llsupport/llmodel.py
+++ b/pypy/jit/backend/llsupport/llmodel.py
@@ -1,24 +1,18 @@
-import sys
 from pypy.rpython.lltypesystem import lltype, llmemory, rffi, rclass, rstr
 from pypy.rpython.lltypesystem.lloperation import llop
-from pypy.rpython.llinterp import LLInterpreter, LLException
+from pypy.rpython.llinterp import LLInterpreter
 from pypy.rpython.annlowlevel import llhelper
 from pypy.rlib.objectmodel import we_are_translated, specialize
-from pypy.jit.metainterp.history import BoxInt, BoxPtr, set_future_values,\
-     BoxFloat
 from pypy.jit.metainterp import history
 from pypy.jit.codewriter import heaptracker, longlong
 from pypy.jit.backend.model import AbstractCPU
 from pypy.jit.backend.llsupport import symbolic
 from pypy.jit.backend.llsupport.symbolic import WORD, unroll_basic_sizes
-from pypy.jit.backend.llsupport.descr import get_size_descr,  BaseSizeDescr
-from pypy.jit.backend.llsupport.descr import get_field_descr, BaseFieldDescr
-from pypy.jit.backend.llsupport.descr import get_array_descr, BaseArrayDescr
-from pypy.jit.backend.llsupport.descr import get_call_descr
-from pypy.jit.backend.llsupport.descr import BaseIntCallDescr, GcPtrCallDescr
-from pypy.jit.backend.llsupport.descr import FloatCallDescr, VoidCallDescr
+from pypy.jit.backend.llsupport.descr import (get_size_descr,
+     get_field_descr, BaseFieldDescr, get_array_descr, BaseArrayDescr,
+     get_call_descr, BaseIntCallDescr, GcPtrCallDescr, FloatCallDescr,
+     VoidCallDescr, InteriorFieldDescr, get_interiorfield_descr)
 from pypy.jit.backend.llsupport.asmmemmgr import AsmMemoryManager
-from pypy.rpython.annlowlevel import cast_instance_to_base_ptr
 
 
 class AbstractLLCPU(AbstractCPU):
@@ -241,6 +235,9 @@
     def arraydescrof(self, A):
         return get_array_descr(self.gc_ll_descr, A)
 
+    def interiorfielddescrof(self, A, fieldname):
+        return get_interiorfield_descr(self.gc_ll_descr, A, A.OF, fieldname)
+
     def unpack_arraydescr(self, arraydescr):
         assert isinstance(arraydescr, BaseArrayDescr)
         return arraydescr.get_base_size(self.translate_support_code)
@@ -358,6 +355,100 @@
     bh_getarrayitem_raw_i = bh_getarrayitem_gc_i
     bh_getarrayitem_raw_f = bh_getarrayitem_gc_f
 
+    def bh_getinteriorfield_gc_i(self, gcref, itemindex, descr):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        fieldsize = descr.fielddescr.get_field_size(self.translate_support_code)
+        sign = descr.fielddescr.is_field_signed()
+        fullofs = itemindex * size + ofs
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), fullofs)
+        for STYPE, UTYPE, itemsize in unroll_basic_sizes:
+            if fieldsize == itemsize:
+                if sign:
+                    item = rffi.cast(rffi.CArrayPtr(STYPE), items)
+                    val = item[0]
+                    val = rffi.cast(lltype.Signed, val)
+                else:
+                    item = rffi.cast(rffi.CArrayPtr(UTYPE), items)
+                    val = item[0]
+                    val = rffi.cast(lltype.Signed, val)
+                # --- end of GC unsafe code ---
+                return val
+        else:
+            raise NotImplementedError("size = %d" % fieldsize)
+
+    def bh_getinteriorfield_gc_r(self, gcref, itemindex, descr):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs +
+                            size * itemindex)
+        items = rffi.cast(rffi.CArrayPtr(lltype.Signed), items)
+        pval = self._cast_int_to_gcref(items[0])
+        # --- end of GC unsafe code ---
+        return pval
+
+    def bh_getinteriorfield_gc_f(self, gcref, itemindex, descr):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs +
+                            size * itemindex)
+        items = rffi.cast(rffi.CArrayPtr(longlong.FLOATSTORAGE), items)
+        fval = items[0]
+        # --- end of GC unsafe code ---
+        return fval
+
+    def bh_setinteriorfield_gc_i(self, gcref, itemindex, descr, value):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        fieldsize = descr.fielddescr.get_field_size(self.translate_support_code)
+        ofs = itemindex * size + ofs
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs)
+        for TYPE, _, itemsize in unroll_basic_sizes:
+            if fieldsize == itemsize:
+                items = rffi.cast(rffi.CArrayPtr(TYPE), items)
+                items[0] = rffi.cast(TYPE, value)
+                # --- end of GC unsafe code ---
+                return
+        else:
+            raise NotImplementedError("size = %d" % fieldsize)
+
+    def bh_setinteriorfield_gc_r(self, gcref, itemindex, descr, newvalue):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        self.gc_ll_descr.do_write_barrier(gcref, newvalue)
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref),
+                            ofs + size * itemindex)
+        items = rffi.cast(rffi.CArrayPtr(lltype.Signed), items)
+        items[0] = self.cast_gcref_to_int(newvalue)
+        # --- end of GC unsafe code ---
+
+    def bh_setinteriorfield_gc_f(self, gcref, itemindex, descr, newvalue):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs, size, _ = self.unpack_arraydescr_size(arraydescr)
+        ofs += descr.fielddescr.offset
+        # --- start of GC unsafe code (no GC operation!) ---
+        items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref),
+                            ofs + size * itemindex)
+        items = rffi.cast(rffi.CArrayPtr(longlong.FLOATSTORAGE), items)
+        items[0] = newvalue
+        # --- end of GC unsafe code ---
+
     def bh_strlen(self, string):
         s = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), string)
         return len(s.chars)
@@ -475,7 +566,6 @@
 
     def bh_classof(self, struct):
         struct = lltype.cast_opaque_ptr(rclass.OBJECTPTR, struct)
-        result = struct.typeptr
         result_adr = llmemory.cast_ptr_to_adr(struct.typeptr)
         return heaptracker.adr2int(result_adr)
 
diff --git a/pypy/jit/backend/llsupport/regalloc.py b/pypy/jit/backend/llsupport/regalloc.py
--- a/pypy/jit/backend/llsupport/regalloc.py
+++ b/pypy/jit/backend/llsupport/regalloc.py
@@ -178,6 +178,8 @@
         cur_max_age = -1
         candidate = None
         for next in self.reg_bindings:
+            if isinstance(next, TempBox):
+                continue
             reg = self.reg_bindings[next]
             if next in forbidden_vars:
                 continue
@@ -287,7 +289,6 @@
         self.reg_bindings[to_v] = reg
 
     def _move_variable_away(self, v, prev_loc):
-        reg = None
         if self.free_regs:
             loc = self.free_regs.pop()
             self.reg_bindings[v] = loc
diff --git a/pypy/jit/backend/llsupport/test/test_asmmemmgr.py b/pypy/jit/backend/llsupport/test/test_asmmemmgr.py
--- a/pypy/jit/backend/llsupport/test/test_asmmemmgr.py
+++ b/pypy/jit/backend/llsupport/test/test_asmmemmgr.py
@@ -211,14 +211,14 @@
     debug._log = debug.DebugLog()
     try:
         mc._dump(addr, 'test-logname-section')
-        log = list(debug._log)
+        log = list(debug._log) 
     finally:
         debug._log = None
     encoded = ''.join(writtencode).encode('hex').upper()
     ataddr = '@%x' % addr
     assert log == [('test-logname-section',
                     [('debug_print', 'CODE_DUMP', ataddr, '+0 ', encoded)])]
-    #
+    
     lltype.free(p, flavor='raw')
 
 def test_blockbuildermixin2():
diff --git a/pypy/jit/backend/llsupport/test/test_descr.py b/pypy/jit/backend/llsupport/test/test_descr.py
--- a/pypy/jit/backend/llsupport/test/test_descr.py
+++ b/pypy/jit/backend/llsupport/test/test_descr.py
@@ -3,7 +3,6 @@
 from pypy.jit.backend.llsupport import symbolic
 from pypy.rlib.objectmodel import Symbolic
 from pypy.rpython.annlowlevel import llhelper
-from pypy.jit.metainterp.history import BoxInt, BoxFloat, BoxPtr
 from pypy.jit.metainterp import history
 from pypy.jit.codewriter import longlong
 import sys, struct, py
@@ -149,7 +148,9 @@
     A2 = lltype.GcArray(lltype.Ptr(T))
     A3 = lltype.GcArray(lltype.Ptr(U))
     A4 = lltype.GcArray(lltype.Float)
-    A5 = lltype.GcArray(lltype.SingleFloat)
+    A5 = lltype.GcArray(lltype.Struct('x', ('v', lltype.Signed),
+                                      ('k', lltype.Signed)))
+    A6 = lltype.GcArray(lltype.SingleFloat)
     assert getArrayDescrClass(A2) is GcPtrArrayDescr
     assert getArrayDescrClass(A3) is NonGcPtrArrayDescr
     cls = getArrayDescrClass(A1)
@@ -158,7 +159,7 @@
     clsf = getArrayDescrClass(A4)
     assert clsf != cls
     assert clsf == getArrayDescrClass(lltype.GcArray(lltype.Float))
-    clss = getArrayDescrClass(A5)
+    clss = getArrayDescrClass(A6)
     assert clss not in (clsf, cls)
     assert clss == getArrayDescrClass(lltype.GcArray(rffi.UINT))
     #
@@ -168,11 +169,12 @@
     descr3 = get_array_descr(c0, A3)
     descr4 = get_array_descr(c0, A4)
     descr5 = get_array_descr(c0, A5)
+    descr6 = get_array_descr(c0, A6)
     assert descr1.__class__ is cls
     assert descr2.__class__ is GcPtrArrayDescr
     assert descr3.__class__ is NonGcPtrArrayDescr
     assert descr4.__class__ is clsf
-    assert descr5.__class__ is clss
+    assert descr6.__class__ is clss
     assert descr1 == get_array_descr(c0, lltype.GcArray(lltype.Char))
     assert not descr1.is_array_of_pointers()
     assert     descr2.is_array_of_pointers()
@@ -202,7 +204,8 @@
     assert descr2.get_item_size(False) == rffi.sizeof(lltype.Ptr(T))
     assert descr3.get_item_size(False) == rffi.sizeof(lltype.Ptr(U))
     assert descr4.get_item_size(False) == rffi.sizeof(lltype.Float)
-    assert descr5.get_item_size(False) == rffi.sizeof(lltype.SingleFloat)
+    assert descr5.get_item_size(False) == rffi.sizeof(lltype.Signed) * 2
+    assert descr6.get_item_size(False) == rffi.sizeof(lltype.SingleFloat)
     #
     assert isinstance(descr1.get_base_size(True), Symbolic)
     assert isinstance(descr2.get_base_size(True), Symbolic)
@@ -348,7 +351,6 @@
                             (rffi.SHORT,      True), (rffi.USHORT, False),
                             (rffi.INT,        True), (rffi.UINT,   False),
                             (rffi.LONG,       True), (rffi.ULONG,  False)]:
-        A = lltype.GcArray(RESTYPE)
         for tsc in [False, True]:
             c2 = GcCache(tsc)
             descr1 = get_call_descr(c2, [], RESTYPE)
@@ -379,7 +381,6 @@
     descr3i = get_array_descr(c0, lltype.GcArray(lltype.Char))
     assert descr3i.repr_of_descr() == '<CharArrayDescr>'
     #
-    cache = {}
     descr4 = get_call_descr(c0, [lltype.Char, lltype.Ptr(S)], lltype.Ptr(S))
     assert 'GcPtrCallDescr' in descr4.repr_of_descr()
     #
@@ -412,10 +413,10 @@
     ARGS = [lltype.Float, lltype.Ptr(ARRAY)]
     RES = lltype.Float
 
-    def f(a, b):
+    def f2(a, b):
         return float(b[0]) + a
 
-    fnptr = llhelper(lltype.Ptr(lltype.FuncType(ARGS, RES)), f)
+    fnptr = llhelper(lltype.Ptr(lltype.FuncType(ARGS, RES)), f2)
     descr2 = get_call_descr(c0, ARGS, RES)
     a = lltype.malloc(ARRAY, 3)
     opaquea = lltype.cast_opaque_ptr(llmemory.GCREF, a)
diff --git a/pypy/jit/backend/llsupport/test/test_gc.py b/pypy/jit/backend/llsupport/test/test_gc.py
--- a/pypy/jit/backend/llsupport/test/test_gc.py
+++ b/pypy/jit/backend/llsupport/test/test_gc.py
@@ -247,12 +247,14 @@
         self.record = []
 
     def do_malloc_fixedsize_clear(self, RESTYPE, type_id, size,
-                                  has_finalizer, contains_weakptr):
+                                  has_finalizer, has_light_finalizer,
+                                  contains_weakptr):
         assert not contains_weakptr
+        assert not has_finalizer           # in these tests
+        assert not has_light_finalizer     # in these tests
         p = llmemory.raw_malloc(size)
         p = llmemory.cast_adr_to_ptr(p, RESTYPE)
-        flags = int(has_finalizer) << 16
-        tid = llop.combine_ushort(lltype.Signed, type_id, flags)
+        tid = llop.combine_ushort(lltype.Signed, type_id, 0)
         self.record.append(("fixedsize", repr(size), tid, p))
         return p
 
diff --git a/pypy/jit/backend/model.py b/pypy/jit/backend/model.py
--- a/pypy/jit/backend/model.py
+++ b/pypy/jit/backend/model.py
@@ -1,5 +1,5 @@
 from pypy.rlib.debug import debug_start, debug_print, debug_stop
-from pypy.jit.metainterp import history, compile
+from pypy.jit.metainterp import history
 
 
 class AbstractCPU(object):
@@ -213,6 +213,10 @@
     def typedescrof(TYPE):
         raise NotImplementedError
 
+    @staticmethod
+    def interiorfielddescrof(A, fieldname):
+        raise NotImplementedError
+
     # ---------- the backend-dependent operations ----------
 
     # lltype specific operations
diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py
--- a/pypy/jit/backend/test/runner_test.py
+++ b/pypy/jit/backend/test/runner_test.py
@@ -5,7 +5,7 @@
                                          BoxInt, Box, BoxPtr,
                                          LoopToken,
                                          ConstInt, ConstPtr,
-                                         BoxObj, Const,
+                                         BoxObj,
                                          ConstObj, BoxFloat, ConstFloat)
 from pypy.jit.metainterp.resoperation import ResOperation, rop
 from pypy.jit.metainterp.typesystem import deref
@@ -111,7 +111,7 @@
         self.cpu.set_future_value_int(0, 2)
         fail = self.cpu.execute_token(looptoken)
         res = self.cpu.get_latest_value_int(0)
-        assert res == 3        
+        assert res == 3
         assert fail.identifier == 1
 
     def test_compile_loop(self):
@@ -127,7 +127,7 @@
             ]
         inputargs = [i0]
         operations[2].setfailargs([i1])
-        
+
         self.cpu.compile_loop(inputargs, operations, looptoken)
         self.cpu.set_future_value_int(0, 2)
         fail = self.cpu.execute_token(looptoken)
@@ -148,7 +148,7 @@
             ]
         inputargs = [i0]
         operations[2].setfailargs([None, None, i1, None])
-        
+
         self.cpu.compile_loop(inputargs, operations, looptoken)
         self.cpu.set_future_value_int(0, 2)
         fail = self.cpu.execute_token(looptoken)
@@ -372,7 +372,7 @@
         for opnum, boxargs, retvalue in get_int_tests():
             res = self.execute_operation(opnum, boxargs, 'int')
             assert res.value == retvalue
-        
+
     def test_float_operations(self):
         from pypy.jit.metainterp.test.test_executor import get_float_tests
         for opnum, boxargs, rettype, retvalue in get_float_tests(self.cpu):
@@ -438,7 +438,7 @@
 
     def test_ovf_operations_reversed(self):
         self.test_ovf_operations(reversed=True)
-        
+
     def test_bh_call(self):
         cpu = self.cpu
         #
@@ -503,7 +503,7 @@
                                          [funcbox, BoxInt(num), BoxInt(num)],
                                          'int', descr=dyn_calldescr)
             assert res.value == 2 * num
-            
+
 
         if cpu.supports_floats:
             def func(f0, f1, f2, f3, f4, f5, f6, i0, i1, f7, f8, f9):
@@ -543,7 +543,7 @@
         funcbox = self.get_funcbox(self.cpu, func_ptr)
         res = self.execute_operation(rop.CALL, [funcbox] + map(BoxInt, args), 'int', descr=calldescr)
         assert res.value == func(*args)
-        
+
     def test_call_stack_alignment(self):
         # test stack alignment issues, notably for Mac OS/X.
         # also test the ordering of the arguments.
@@ -615,7 +615,7 @@
         res = self.execute_operation(rop.GETFIELD_GC, [t_box],
                                      'int', descr=shortdescr)
         assert res.value == 1331
-        
+
         #
         u_box, U_box = self.alloc_instance(self.U)
         fielddescr2 = self.cpu.fielddescrof(self.S, 'next')
@@ -695,7 +695,7 @@
 
     def test_failing_guard_class(self):
         t_box, T_box = self.alloc_instance(self.T)
-        u_box, U_box = self.alloc_instance(self.U)        
+        u_box, U_box = self.alloc_instance(self.U)
         null_box = self.null_instance()
         for opname, args in [(rop.GUARD_CLASS, [t_box, U_box]),
                              (rop.GUARD_CLASS, [u_box, T_box]),
@@ -787,7 +787,7 @@
         r = self.execute_operation(rop.GETARRAYITEM_GC, [a_box, BoxInt(3)],
                                    'int', descr=arraydescr)
         assert r.value == 160
-        
+
         #
         if isinstance(A, lltype.GcArray):
             A = lltype.Ptr(A)
@@ -880,6 +880,73 @@
                                    'int', descr=arraydescr)
         assert r.value == 7441
 
+    def test_array_of_structs(self):
+        TP = lltype.GcStruct('x')
+        ITEM = lltype.Struct('x',
+                             ('vs', lltype.Signed),
+                             ('vu', lltype.Unsigned),
+                             ('vsc', rffi.SIGNEDCHAR),
+                             ('vuc', rffi.UCHAR),
+                             ('vss', rffi.SHORT),
+                             ('vus', rffi.USHORT),
+                             ('vsi', rffi.INT),
+                             ('vui', rffi.UINT),
+                             ('k', lltype.Float),
+                             ('p', lltype.Ptr(TP)))
+        a_box, A = self.alloc_array_of(ITEM, 15)
+        s_box, S = self.alloc_instance(TP)
+        kdescr = self.cpu.interiorfielddescrof(A, 'k')
+        pdescr = self.cpu.interiorfielddescrof(A, 'p')
+        self.execute_operation(rop.SETINTERIORFIELD_GC, [a_box, BoxInt(3),
+                                                         boxfloat(1.5)],
+                               'void', descr=kdescr)
+        f = self.cpu.bh_getinteriorfield_gc_f(a_box.getref_base(), 3, kdescr)
+        assert longlong.getrealfloat(f) == 1.5
+        self.cpu.bh_setinteriorfield_gc_f(a_box.getref_base(), 3, kdescr, longlong.getfloatstorage(2.5))
+        r = self.execute_operation(rop.GETINTERIORFIELD_GC, [a_box, BoxInt(3)],
+                                   'float', descr=kdescr)
+        assert r.getfloat() == 2.5
+        #
+        NUMBER_FIELDS = [('vs', lltype.Signed),
+                         ('vu', lltype.Unsigned),
+                         ('vsc', rffi.SIGNEDCHAR),
+                         ('vuc', rffi.UCHAR),
+                         ('vss', rffi.SHORT),
+                         ('vus', rffi.USHORT),
+                         ('vsi', rffi.INT),
+                         ('vui', rffi.UINT)]
+        for name, TYPE in NUMBER_FIELDS[::-1]:
+            vdescr = self.cpu.interiorfielddescrof(A, name)
+            self.execute_operation(rop.SETINTERIORFIELD_GC, [a_box, BoxInt(3),
+                                                             BoxInt(-15)],
+                                   'void', descr=vdescr)
+        for name, TYPE in NUMBER_FIELDS:
+            vdescr = self.cpu.interiorfielddescrof(A, name)
+            i = self.cpu.bh_getinteriorfield_gc_i(a_box.getref_base(), 3,
+                                                  vdescr)
+            assert i == rffi.cast(lltype.Signed, rffi.cast(TYPE, -15))
+        for name, TYPE in NUMBER_FIELDS[::-1]:
+            vdescr = self.cpu.interiorfielddescrof(A, name)
+            self.cpu.bh_setinteriorfield_gc_i(a_box.getref_base(), 3,
+                                              vdescr, -25)
+        for name, TYPE in NUMBER_FIELDS:
+            vdescr = self.cpu.interiorfielddescrof(A, name)
+            r = self.execute_operation(rop.GETINTERIORFIELD_GC,
+                                       [a_box, BoxInt(3)],
+                                       'int', descr=vdescr)
+            assert r.getint() == rffi.cast(lltype.Signed, rffi.cast(TYPE, -25))
+        #
+        self.execute_operation(rop.SETINTERIORFIELD_GC, [a_box, BoxInt(4),
+                                                         s_box],
+                               'void', descr=pdescr)
+        r = self.cpu.bh_getinteriorfield_gc_r(a_box.getref_base(), 4, pdescr)
+        assert r == s_box.getref_base()
+        self.cpu.bh_setinteriorfield_gc_r(a_box.getref_base(), 3, pdescr,
+                                          s_box.getref_base())
+        r = self.execute_operation(rop.GETINTERIORFIELD_GC, [a_box, BoxInt(3)],
+                                   'ref', descr=pdescr)
+        assert r.getref_base() == s_box.getref_base()
+
     def test_string_basic(self):
         s_box = self.alloc_string("hello\xfe")
         r = self.execute_operation(rop.STRLEN, [s_box], 'int')
@@ -1402,7 +1469,7 @@
         addr = llmemory.cast_ptr_to_adr(func_ptr)
         return ConstInt(heaptracker.adr2int(addr))
 
-    
+
     MY_VTABLE = rclass.OBJECT_VTABLE    # for tests only
 
     S = lltype.GcForwardReference()
@@ -1439,7 +1506,6 @@
         return BoxPtr(lltype.nullptr(llmemory.GCREF.TO))
 
     def alloc_array_of(self, ITEM, length):
-        cpu = self.cpu
         A = lltype.GcArray(ITEM)
         a = lltype.malloc(A, length)
         a_box = BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, a))
@@ -2318,7 +2384,7 @@
         for opname, arg, res in ops:
             self.execute_operation(opname, [arg], 'void')
             assert self.guard_failed == res
-        
+
         lltype.free(x, flavor='raw')
 
     def test_assembler_call(self):
@@ -2398,7 +2464,7 @@
         FakeJitDriverSD.portal_calldescr = self.cpu.calldescrof(
             lltype.Ptr(lltype.FuncType(ARGS, RES)), ARGS, RES,
             EffectInfo.MOST_GENERAL)
-        
+
         ops = '''
         [f0, f1]
         f2 = float_add(f0, f1)
@@ -2489,7 +2555,7 @@
         FakeJitDriverSD.portal_calldescr = self.cpu.calldescrof(
             lltype.Ptr(lltype.FuncType(ARGS, RES)), ARGS, RES,
             EffectInfo.MOST_GENERAL)
-        
+
         ops = '''
         [f0, f1]
         f2 = float_add(f0, f1)
@@ -2940,4 +3006,4 @@
 
     def alloc_unicode(self, unicode):
         py.test.skip("implement me")
-    
+
diff --git a/pypy/jit/backend/test/test_ll_random.py b/pypy/jit/backend/test/test_ll_random.py
--- a/pypy/jit/backend/test/test_ll_random.py
+++ b/pypy/jit/backend/test/test_ll_random.py
@@ -28,16 +28,27 @@
         fork.structure_types_and_vtables = self.structure_types_and_vtables
         return fork
 
-    def get_structptr_var(self, r, must_have_vtable=False, type=lltype.Struct):
+    def _choose_ptr_vars(self, from_, type, array_of_structs):
+        ptrvars = []
+        for i in range(len(from_)):
+            v, S = from_[i][:2]
+            if not isinstance(S, type):
+                continue
+            if ((isinstance(S, lltype.Array) and
+                 isinstance(S.OF, lltype.Struct)) == array_of_structs):
+                ptrvars.append((v, S))
+        return ptrvars
+
+    def get_structptr_var(self, r, must_have_vtable=False, type=lltype.Struct,
+                          array_of_structs=False):
         while True:
-            ptrvars = [(v, S) for (v, S) in self.ptrvars
-                              if isinstance(S, type)]
+            ptrvars = self._choose_ptr_vars(self.ptrvars, type,
+                                            array_of_structs)
             if ptrvars and r.random() < 0.8:
                 v, S = r.choice(ptrvars)
             else:
-                prebuilt_ptr_consts = [(v, S)
-                                 for (v, S, _) in self.prebuilt_ptr_consts
-                                 if isinstance(S, type)]
+                prebuilt_ptr_consts = self._choose_ptr_vars(
+                    self.prebuilt_ptr_consts, type, array_of_structs)
                 if prebuilt_ptr_consts and r.random() < 0.7:
                     v, S = r.choice(prebuilt_ptr_consts)
                 else:
@@ -48,7 +59,8 @@
                                                 has_vtable=must_have_vtable)
                     else:
                         # create a new constant array
-                        p = self.get_random_array(r)
+                        p = self.get_random_array(r,
+                                    must_be_array_of_structs=array_of_structs)
                     S = lltype.typeOf(p).TO
                     v = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, p))
                     self.prebuilt_ptr_consts.append((v, S,
@@ -74,7 +86,8 @@
                 TYPE = lltype.Signed
         return TYPE
 
-    def get_random_structure_type(self, r, with_vtable=None, cache=True):
+    def get_random_structure_type(self, r, with_vtable=None, cache=True,
+                                  type=lltype.GcStruct):
         if cache and self.structure_types and r.random() < 0.5:
             return r.choice(self.structure_types)
         fields = []
@@ -85,7 +98,7 @@
         for i in range(r.randrange(1, 5)):
             TYPE = self.get_random_primitive_type(r)
             fields.append(('f%d' % i, TYPE))
-        S = lltype.GcStruct('S%d' % self.counter, *fields, **kwds)
+        S = type('S%d' % self.counter, *fields, **kwds)
         self.counter += 1
         if cache:
             self.structure_types.append(S)
@@ -125,17 +138,29 @@
                 setattr(p, fieldname, rffi.cast(TYPE, r.random_integer()))
         return p
 
-    def get_random_array_type(self, r):
-        TYPE = self.get_random_primitive_type(r)
+    def get_random_array_type(self, r, can_be_array_of_struct=False,
+                              must_be_array_of_structs=False):
+        if ((can_be_array_of_struct and r.random() < 0.1) or
+            must_be_array_of_structs):
+            TYPE = self.get_random_structure_type(r, cache=False,
+                                                  type=lltype.Struct)
+        else:
+            TYPE = self.get_random_primitive_type(r)
         return lltype.GcArray(TYPE)
 
-    def get_random_array(self, r):
-        A = self.get_random_array_type(r)
+    def get_random_array(self, r, must_be_array_of_structs=False):
+        A = self.get_random_array_type(r,
+                           must_be_array_of_structs=must_be_array_of_structs)
         length = (r.random_integer() // 15) % 300  # length: between 0 and 299
                                                    # likely to be small
         p = lltype.malloc(A, length)
-        for i in range(length):
-            p[i] = rffi.cast(A.OF, r.random_integer())
+        if isinstance(A.OF, lltype.Primitive):
+            for i in range(length):
+                p[i] = rffi.cast(A.OF, r.random_integer())
+        else:
+            for i in range(length):
+                for fname, TP in A.OF._flds.iteritems():
+                    setattr(p[i], fname, rffi.cast(TP, r.random_integer()))
         return p
 
     def get_index(self, length, r):
@@ -155,8 +180,16 @@
                     dic[fieldname] = getattr(p, fieldname)
         else:
             assert isinstance(S, lltype.Array)
-            for i in range(len(p)):
-                dic[i] = p[i]
+            if isinstance(S.OF, lltype.Struct):
+                for i in range(len(p)):
+                    item = p[i]
+                    s1 = {}
+                    for fieldname in S.OF._names:
+                        s1[fieldname] = getattr(item, fieldname)
+                    dic[i] = s1
+            else:
+                for i in range(len(p)):
+                    dic[i] = p[i]
         return dic
 
     def print_loop_prebuilt(self, names, writevar, s):
@@ -220,7 +253,7 @@
 
 class GetFieldOperation(test_random.AbstractOperation):
     def field_descr(self, builder, r):
-        v, S = builder.get_structptr_var(r)
+        v, S = builder.get_structptr_var(r, )
         names = S._names
         if names[0] == 'parent':
             names = names[1:]
@@ -239,6 +272,28 @@
                 continue
             break
 
+class GetInteriorFieldOperation(test_random.AbstractOperation):
+    def field_descr(self, builder, r):
+        v, A = builder.get_structptr_var(r, type=lltype.Array,
+                                         array_of_structs=True)
+        array = v.getref(lltype.Ptr(A))
+        v_index = builder.get_index(len(array), r)
+        name = r.choice(A.OF._names)
+        descr = builder.cpu.interiorfielddescrof(A, name)
+        descr._random_info = 'cpu.interiorfielddescrof(%s, %r)' % (A.OF._name,
+                                                                   name)
+        TYPE = getattr(A.OF, name)
+        return v, v_index, descr, TYPE
+
+    def produce_into(self, builder, r):
+        while True:
+            try:
+                v, v_index, descr, _ = self.field_descr(builder, r)
+                self.put(builder, [v, v_index], descr)
+            except lltype.UninitializedMemoryAccess:
+                continue
+            break
+
 class SetFieldOperation(GetFieldOperation):
     def produce_into(self, builder, r):
         v, descr, TYPE = self.field_descr(builder, r)
@@ -251,6 +306,18 @@
                 break
         builder.do(self.opnum, [v, w], descr)
 
+class SetInteriorFieldOperation(GetInteriorFieldOperation):
+    def produce_into(self, builder, r):
+        v, v_index, descr, TYPE = self.field_descr(builder, r)
+        while True:
+            if r.random() < 0.3:
+                w = ConstInt(r.random_integer())
+            else:
+                w = r.choice(builder.intvars)
+            if rffi.cast(lltype.Signed, rffi.cast(TYPE, w.value)) == w.value:
+                break
+        builder.do(self.opnum, [v, v_index, w], descr)
+
 class NewOperation(test_random.AbstractOperation):
     def size_descr(self, builder, S):
         descr = builder.cpu.sizeof(S)
@@ -306,7 +373,7 @@
 
 class NewArrayOperation(ArrayOperation):
     def produce_into(self, builder, r):
-        A = builder.get_random_array_type(r)
+        A = builder.get_random_array_type(r, can_be_array_of_struct=True)
         v_size = builder.get_index(300, r)
         v_ptr = builder.do(self.opnum, [v_size], self.array_descr(builder, A))
         builder.ptrvars.append((v_ptr, A))
@@ -586,7 +653,9 @@
 for i in range(4):      # make more common
     OPERATIONS.append(GetFieldOperation(rop.GETFIELD_GC))
     OPERATIONS.append(GetFieldOperation(rop.GETFIELD_GC))
+    OPERATIONS.append(GetInteriorFieldOperation(rop.GETINTERIORFIELD_GC))
     OPERATIONS.append(SetFieldOperation(rop.SETFIELD_GC))
+    OPERATIONS.append(SetInteriorFieldOperation(rop.SETINTERIORFIELD_GC))
     OPERATIONS.append(NewOperation(rop.NEW))
     OPERATIONS.append(NewOperation(rop.NEW_WITH_VTABLE))
 
diff --git a/pypy/jit/backend/test/test_random.py b/pypy/jit/backend/test/test_random.py
--- a/pypy/jit/backend/test/test_random.py
+++ b/pypy/jit/backend/test/test_random.py
@@ -595,6 +595,10 @@
             for name, value in fields.items():
                 if isinstance(name, str):
                     setattr(container, name, value)
+                elif isinstance(value, dict):
+                    item = container.getitem(name)
+                    for key1, value1 in value.items():
+                        setattr(item, key1, value1)
                 else:
                     container.setitem(name, value)
 
diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1,7 +1,7 @@
 import sys, os
 from pypy.jit.backend.llsupport import symbolic
 from pypy.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper
-from pypy.jit.metainterp.history import Const, Box, BoxInt, BoxPtr, BoxFloat
+from pypy.jit.metainterp.history import Const, Box, BoxInt, ConstInt
 from pypy.jit.metainterp.history import (AbstractFailDescr, INT, REF, FLOAT,
                                          LoopToken)
 from pypy.rpython.lltypesystem import lltype, rffi, rstr, llmemory
@@ -36,7 +36,6 @@
 from pypy.rlib import rgc
 from pypy.rlib.clibffi import FFI_DEFAULT_ABI
 from pypy.jit.backend.x86.jump import remap_frame_layout
-from pypy.jit.metainterp.history import ConstInt, BoxInt
 from pypy.jit.codewriter.effectinfo import EffectInfo
 from pypy.jit.codewriter import longlong
 
@@ -729,8 +728,8 @@
         # Also, make sure this is consistent with FRAME_FIXED_SIZE.
         self.mc.PUSH_r(ebp.value)
         self.mc.MOV_rr(ebp.value, esp.value)
-        for regloc in self.cpu.CALLEE_SAVE_REGISTERS:
-            self.mc.PUSH_r(regloc.value)
+        for loc in self.cpu.CALLEE_SAVE_REGISTERS:
+            self.mc.PUSH_r(loc.value)
 
         gcrootmap = self.cpu.gc_ll_descr.gcrootmap
         if gcrootmap and gcrootmap.is_shadow_stack:
@@ -994,7 +993,7 @@
         effectinfo = op.getdescr().get_extra_info()
         oopspecindex = effectinfo.oopspecindex
         genop_llong_list[oopspecindex](self, op, arglocs, resloc)
-        
+
     def regalloc_perform_math(self, op, arglocs, resloc):
         effectinfo = op.getdescr().get_extra_info()
         oopspecindex = effectinfo.oopspecindex
@@ -1277,8 +1276,8 @@
     genop_int_ne = _cmpop("NE", "NE")
     genop_int_gt = _cmpop("G", "L")
     genop_int_ge = _cmpop("GE", "LE")
-    genop_ptr_eq = genop_int_eq
-    genop_ptr_ne = genop_int_ne
+    genop_ptr_eq = genop_instance_ptr_eq = genop_int_eq
+    genop_ptr_ne = genop_instance_ptr_ne = genop_int_ne
 
     genop_float_lt = _cmpop_float('B', 'A')
     genop_float_le = _cmpop_float('BE', 'AE')
@@ -1298,8 +1297,8 @@
     genop_guard_int_ne = _cmpop_guard("NE", "NE", "E", "E")
     genop_guard_int_gt = _cmpop_guard("G", "L", "LE", "GE")
     genop_guard_int_ge = _cmpop_guard("GE", "LE", "L", "G")
-    genop_guard_ptr_eq = genop_guard_int_eq
-    genop_guard_ptr_ne = genop_guard_int_ne
+    genop_guard_ptr_eq = genop_guard_instance_ptr_eq = genop_guard_int_eq
+    genop_guard_ptr_ne = genop_guard_instance_ptr_ne = genop_guard_int_ne
 
     genop_guard_uint_gt = _cmpop_guard("A", "B", "BE", "AE")
     genop_guard_uint_lt = _cmpop_guard("B", "A", "AE", "BE")
@@ -1311,7 +1310,7 @@
     genop_guard_float_eq = _cmpop_guard_float("E", "E", "NE","NE")
     genop_guard_float_gt = _cmpop_guard_float("A", "B", "BE","AE")
     genop_guard_float_ge = _cmpop_guard_float("AE","BE", "B", "A")
-    
+
     def genop_math_sqrt(self, op, arglocs, resloc):
         self.mc.SQRTSD(arglocs[0], resloc)
 
@@ -1597,12 +1596,30 @@
     genop_getarrayitem_gc_pure = genop_getarrayitem_gc
     genop_getarrayitem_raw = genop_getarrayitem_gc
 
+    def genop_getinteriorfield_gc(self, op, arglocs, resloc):
+        base_loc, ofs_loc, itemsize_loc, fieldsize_loc, index_loc, sign_loc = arglocs
+        # XXX should not use IMUL in most cases
+        self.mc.IMUL(index_loc, itemsize_loc)
+        src_addr = AddressLoc(base_loc, index_loc, 0, ofs_loc.value)
+        self.load_from_mem(resloc, src_addr, fieldsize_loc, sign_loc)
+
+
     def genop_discard_setfield_gc(self, op, arglocs):
         base_loc, ofs_loc, size_loc, value_loc = arglocs
         assert isinstance(size_loc, ImmedLoc)
         dest_addr = AddressLoc(base_loc, ofs_loc)
         self.save_into_mem(dest_addr, value_loc, size_loc)
 
+    def genop_discard_setinteriorfield_gc(self, op, arglocs):
+        base_loc, ofs_loc, itemsize_loc, fieldsize_loc, index_loc, value_loc = arglocs
+        # XXX should not use IMUL in most cases
+        if isinstance(index_loc, ImmedLoc):
+            index_loc = imm(index_loc.value * itemsize_loc.value)
+        else:
+            self.mc.IMUL(index_loc, itemsize_loc)
+        dest_addr = AddressLoc(base_loc, index_loc, 0, ofs_loc.value)
+        self.save_into_mem(dest_addr, value_loc, fieldsize_loc)
+
     def genop_discard_setarrayitem_gc(self, op, arglocs):
         base_loc, ofs_loc, value_loc, size_loc, baseofs = arglocs
         assert isinstance(baseofs, ImmedLoc)
diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py
--- a/pypy/jit/backend/x86/regalloc.py
+++ b/pypy/jit/backend/x86/regalloc.py
@@ -7,7 +7,7 @@
                                          ResOperation, BoxPtr, ConstFloat,
                                          BoxFloat, LoopToken, INT, REF, FLOAT)
 from pypy.jit.backend.x86.regloc import *
-from pypy.rpython.lltypesystem import lltype, ll2ctypes, rffi, rstr
+from pypy.rpython.lltypesystem import lltype, rffi, rstr
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.rlib import rgc
 from pypy.jit.backend.llsupport import symbolic
@@ -17,11 +17,12 @@
 from pypy.jit.metainterp.resoperation import rop
 from pypy.jit.backend.llsupport.descr import BaseFieldDescr, BaseArrayDescr
 from pypy.jit.backend.llsupport.descr import BaseCallDescr, BaseSizeDescr
+from pypy.jit.backend.llsupport.descr import InteriorFieldDescr
 from pypy.jit.backend.llsupport.regalloc import FrameManager, RegisterManager,\
      TempBox
 from pypy.jit.backend.x86.arch import WORD, FRAME_FIXED_SIZE
 from pypy.jit.backend.x86.arch import IS_X86_32, IS_X86_64, MY_COPY_OF_REGS
-from pypy.rlib.rarithmetic import r_longlong, r_uint
+from pypy.rlib.rarithmetic import r_longlong
 
 class X86RegisterManager(RegisterManager):
 
@@ -433,7 +434,7 @@
             if self.can_merge_with_next_guard(op, i, operations):
                 oplist_with_guard[op.getopnum()](self, op, operations[i + 1])
                 i += 1
-            elif not we_are_translated() and op.getopnum() == -124: 
+            elif not we_are_translated() and op.getopnum() == -124:
                 self._consider_force_spill(op)
             else:
                 oplist[op.getopnum()](self, op)
@@ -650,8 +651,8 @@
     consider_uint_lt = _consider_compop
     consider_uint_le = _consider_compop
     consider_uint_ge = _consider_compop
-    consider_ptr_eq = _consider_compop
-    consider_ptr_ne = _consider_compop
+    consider_ptr_eq = consider_instance_ptr_eq = _consider_compop
+    consider_ptr_ne = consider_instance_ptr_ne = _consider_compop
 
     def _consider_float_op(self, op):
         loc1 = self.xrm.loc(op.getarg(1))
@@ -815,7 +816,7 @@
         save_all_regs = guard_not_forced_op is not None
         self.xrm.before_call(force_store, save_all_regs=save_all_regs)
         if not save_all_regs:
-            gcrootmap = gc_ll_descr = self.assembler.cpu.gc_ll_descr.gcrootmap
+            gcrootmap = self.assembler.cpu.gc_ll_descr.gcrootmap
             if gcrootmap and gcrootmap.is_shadow_stack:
                 save_all_regs = 2
         self.rm.before_call(force_store, save_all_regs=save_all_regs)
@@ -972,74 +973,27 @@
             return self._call(op, arglocs)
 
     def consider_newstr(self, op):
-        gc_ll_descr = self.assembler.cpu.gc_ll_descr
-        if gc_ll_descr.get_funcptr_for_newstr is not None:
-            # framework GC
-            loc = self.loc(op.getarg(0))
-            return self._call(op, [loc])
-        # boehm GC (XXX kill the following code at some point)
-        ofs_items, itemsize, ofs = symbolic.get_array_token(rstr.STR, self.translate_support_code)
-        assert itemsize == 1
-        return self._malloc_varsize(ofs_items, ofs, 0, op.getarg(0),
-                                    op.result)
+        loc = self.loc(op.getarg(0))
+        return self._call(op, [loc])
 
     def consider_newunicode(self, op):
-        gc_ll_descr = self.assembler.cpu.gc_ll_descr
-        if gc_ll_descr.get_funcptr_for_newunicode is not None:
-            # framework GC
-            loc = self.loc(op.getarg(0))
-            return self._call(op, [loc])
-        # boehm GC (XXX kill the following code at some point)
-        ofs_items, _, ofs = symbolic.get_array_token(rstr.UNICODE,
-                                                   self.translate_support_code)
-        scale = self._get_unicode_item_scale()
-        return self._malloc_varsize(ofs_items, ofs, scale, op.getarg(0),
-                                    op.result)
-
-    def _malloc_varsize(self, ofs_items, ofs_length, scale, v, res_v):
-        # XXX kill this function at some point
-        if isinstance(v, Box):
-            loc = self.rm.make_sure_var_in_reg(v, [v])
-            tempbox = TempBox()
-            other_loc = self.rm.force_allocate_reg(tempbox, [v])
-            self.assembler.load_effective_addr(loc, ofs_items,scale, other_loc)
-        else:
-            tempbox = None
-            other_loc = imm(ofs_items + (v.getint() << scale))
-        self._call(ResOperation(rop.NEW, [], res_v),
-                   [other_loc], [v])
-        loc = self.rm.make_sure_var_in_reg(v, [res_v])
-        assert self.loc(res_v) == eax
-        # now we have to reload length to some reasonable place
-        self.rm.possibly_free_var(v)
-        if tempbox is not None:
-            self.rm.possibly_free_var(tempbox)
-        self.PerformDiscard(ResOperation(rop.SETFIELD_GC, [None, None], None),
-                            [eax, imm(ofs_length), imm(WORD), loc])
+        loc = self.loc(op.getarg(0))
+        return self._call(op, [loc])
 
     def consider_new_array(self, op):
         gc_ll_descr = self.assembler.cpu.gc_ll_descr
-        if gc_ll_descr.get_funcptr_for_newarray is not None:
-            # framework GC
-            box_num_elem = op.getarg(0)
-            if isinstance(box_num_elem, ConstInt):
-                num_elem = box_num_elem.value
-                if gc_ll_descr.can_inline_malloc_varsize(op.getdescr(),
-                                                         num_elem):
-                    self.fastpath_malloc_varsize(op, op.getdescr(), num_elem)
-                    return
-            args = self.assembler.cpu.gc_ll_descr.args_for_new_array(
-                op.getdescr())
-            arglocs = [imm(x) for x in args]
-            arglocs.append(self.loc(box_num_elem))
-            self._call(op, arglocs)
-            return
-        # boehm GC (XXX kill the following code at some point)
-        itemsize, basesize, ofs_length, _, _ = (
-            self._unpack_arraydescr(op.getdescr()))
-        scale_of_field = _get_scale(itemsize)
-        self._malloc_varsize(basesize, ofs_length, scale_of_field,
-                             op.getarg(0), op.result)
+        box_num_elem = op.getarg(0)
+        if isinstance(box_num_elem, ConstInt):
+            num_elem = box_num_elem.value
+            if gc_ll_descr.can_inline_malloc_varsize(op.getdescr(),
+                                                     num_elem):
+                self.fastpath_malloc_varsize(op, op.getdescr(), num_elem)
+                return
+        args = self.assembler.cpu.gc_ll_descr.args_for_new_array(
+            op.getdescr())
+        arglocs = [imm(x) for x in args]
+        arglocs.append(self.loc(box_num_elem))
+        self._call(op, arglocs)
 
     def _unpack_arraydescr(self, arraydescr):
         assert isinstance(arraydescr, BaseArrayDescr)
@@ -1058,6 +1012,16 @@
         sign = fielddescr.is_field_signed()
         return imm(ofs), imm(size), ptr, sign
 
+    def _unpack_interiorfielddescr(self, descr):
+        assert isinstance(descr, InteriorFieldDescr)
+        arraydescr = descr.arraydescr
+        ofs = arraydescr.get_base_size(self.translate_support_code)
+        itemsize = arraydescr.get_item_size(self.translate_support_code)
+        fieldsize = descr.fielddescr.get_field_size(self.translate_support_code)
+        sign = descr.fielddescr.is_field_signed()
+        ofs += descr.fielddescr.offset
+        return imm(ofs), imm(itemsize), imm(fieldsize), sign
+
     def consider_setfield_gc(self, op):
         ofs_loc, size_loc, _, _ = self._unpack_fielddescr(op.getdescr())
         assert isinstance(size_loc, ImmedLoc)
@@ -1074,6 +1038,25 @@
 
     consider_setfield_raw = consider_setfield_gc
 
+    def consider_setinteriorfield_gc(self, op):
+        t = self._unpack_interiorfielddescr(op.getdescr())
+        ofs, itemsize, fieldsize, _ = t
+        args = op.getarglist()
+        if fieldsize.value == 1:
+            need_lower_byte = True
+        else:
+            need_lower_byte = False
+        base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
+        tempvar = TempBox()
+        index_loc = self.rm.force_result_in_reg(tempvar, op.getarg(1), args)
+        # we're free to modify index now
+        value_loc = self.make_sure_var_in_reg(op.getarg(2), args,
+                                              need_lower_byte=need_lower_byte)
+        self.rm.possibly_free_var(tempvar)
+        self.possibly_free_vars(args)
+        self.PerformDiscard(op, [base_loc, ofs, itemsize, fieldsize,
+                                 index_loc, value_loc])
+
     def consider_strsetitem(self, op):
         args = op.getarglist()
         base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
@@ -1135,6 +1118,24 @@
     consider_getarrayitem_raw = consider_getarrayitem_gc
     consider_getarrayitem_gc_pure = consider_getarrayitem_gc
 
+    def consider_getinteriorfield_gc(self, op):
+        t = self._unpack_interiorfielddescr(op.getdescr())
+        ofs, itemsize, fieldsize, sign = t
+        if sign:
+            sign_loc = imm1
+        else:
+            sign_loc = imm0
+        args = op.getarglist()
+        tmpvar = TempBox()
+        base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
+        index_loc = self.rm.force_result_in_reg(tmpvar, op.getarg(1),
+                                                args)
+        self.rm.possibly_free_vars_for_op(op)
+        self.rm.possibly_free_var(tmpvar)
+        result_loc = self.force_allocate_reg(op.result)
+        self.Perform(op, [base_loc, ofs, itemsize, fieldsize,
+                          index_loc, sign_loc], result_loc)
+
     def consider_int_is_true(self, op, guard_op):
         # doesn't need arg to be in a register
         argloc = self.loc(op.getarg(0))
@@ -1241,7 +1242,6 @@
         self.rm.possibly_free_var(srcaddr_box)
 
     def _gen_address_inside_string(self, baseloc, ofsloc, resloc, is_unicode):
-        cpu = self.assembler.cpu
         if is_unicode:
             ofs_items, _, _ = symbolic.get_array_token(rstr.UNICODE,
                                                   self.translate_support_code)
@@ -1300,7 +1300,7 @@
         tmpreg = X86RegisterManager.all_regs[0]
         tmploc = self.rm.force_allocate_reg(box, selected_reg=tmpreg)
         xmmtmp = X86XMMRegisterManager.all_regs[0]
-        xmmtmploc = self.xrm.force_allocate_reg(box1, selected_reg=xmmtmp)
+        self.xrm.force_allocate_reg(box1, selected_reg=xmmtmp)
         # Part about non-floats
         # XXX we don't need a copy, we only just the original list
         src_locations1 = [self.loc(op.getarg(i)) for i in range(op.numargs())
@@ -1380,7 +1380,7 @@
     return lambda self, op: fn(self, op, None)
 
 def is_comparison_or_ovf_op(opnum):
-    from pypy.jit.metainterp.resoperation import opclasses, AbstractResOp
+    from pypy.jit.metainterp.resoperation import opclasses
     cls = opclasses[opnum]
     # hack hack: in theory they are instance method, but they don't use
     # any instance field, we can use a fake object
diff --git a/pypy/jit/backend/x86/test/test_del.py b/pypy/jit/backend/x86/test/test_del.py
--- a/pypy/jit/backend/x86/test/test_del.py
+++ b/pypy/jit/backend/x86/test/test_del.py
@@ -1,5 +1,4 @@
 
-import py
 from pypy.jit.backend.x86.test.test_basic import Jit386Mixin
 from pypy.jit.metainterp.test.test_del import DelTests
 
diff --git a/pypy/jit/backend/x86/test/test_dict.py b/pypy/jit/backend/x86/test/test_dict.py
new file mode 100644
--- /dev/null
+++ b/pypy/jit/backend/x86/test/test_dict.py
@@ -0,0 +1,9 @@
+
+from pypy.jit.backend.x86.test.test_basic import Jit386Mixin
+from pypy.jit.metainterp.test.test_dict import DictTests
+
+
+class TestDict(Jit386Mixin, DictTests):
+    # for the individual tests see
+    # ====> ../../../metainterp/test/test_dict.py
+    pass
diff --git a/pypy/jit/backend/x86/test/test_runner.py b/pypy/jit/backend/x86/test/test_runner.py
--- a/pypy/jit/backend/x86/test/test_runner.py
+++ b/pypy/jit/backend/x86/test/test_runner.py
@@ -31,7 +31,7 @@
 
     # for the individual tests see
     # ====> ../../test/runner_test.py
-    
+
     def setup_method(self, meth):
         self.cpu = CPU(rtyper=None, stats=FakeStats())
         self.cpu.setup_once()
@@ -69,22 +69,16 @@
 
     def test_allocations(self):
         from pypy.rpython.lltypesystem import rstr
-        
+
         allocs = [None]
         all = []
+        orig_new = self.cpu.gc_ll_descr.funcptr_for_new
         def f(size):
             allocs.insert(0, size)
-            buf = ctypes.create_string_buffer(size)
-            all.append(buf)
-            return ctypes.cast(buf, ctypes.c_void_p).value
-        func = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int)(f)
-        addr = ctypes.cast(func, ctypes.c_void_p).value
-        # ctypes produces an unsigned value. We need it to be signed for, eg,
-        # relative addressing to work properly.
-        addr = rffi.cast(lltype.Signed, addr)
-        
+            return orig_new(size)
+
         self.cpu.assembler.setup_once()
-        self.cpu.assembler.malloc_func_addr = addr
+        self.cpu.gc_ll_descr.funcptr_for_new = f
         ofs = symbolic.get_field_token(rstr.STR, 'chars', False)[0]
 
         res = self.execute_operation(rop.NEWSTR, [ConstInt(7)], 'ref')
@@ -108,7 +102,7 @@
         res = self.execute_operation(rop.NEW_ARRAY, [ConstInt(10)],
                                          'ref', descr)
         assert allocs[0] == 10*WORD + ofs + WORD
-        resbuf = self._resbuf(res)            
+        resbuf = self._resbuf(res)
         assert resbuf[ofs/WORD] == 10
 
         # ------------------------------------------------------------
@@ -116,7 +110,7 @@
         res = self.execute_operation(rop.NEW_ARRAY, [BoxInt(10)],
                                          'ref', descr)
         assert allocs[0] == 10*WORD + ofs + WORD
-        resbuf = self._resbuf(res)                        
+        resbuf = self._resbuf(res)
         assert resbuf[ofs/WORD] == 10
 
     def test_stringitems(self):
@@ -146,7 +140,7 @@
                                                      ConstInt(2), BoxInt(38)],
                                'void', descr)
         assert resbuf[itemsofs/WORD + 2] == 38
-        
+
         self.execute_operation(rop.SETARRAYITEM_GC, [res,
                                                      BoxInt(3), BoxInt(42)],
                                'void', descr)
@@ -167,7 +161,7 @@
                                                          BoxInt(2)],
                                    'int', descr)
         assert r.value == 38
-        
+
         r = self.execute_operation(rop.GETARRAYITEM_GC, [res, BoxInt(3)],
                                    'int', descr)
         assert r.value == 42
@@ -226,7 +220,7 @@
         self.execute_operation(rop.SETFIELD_GC, [res, BoxInt(1234)], 'void', ofs_i)
         i = self.execute_operation(rop.GETFIELD_GC, [res], 'int', ofs_i)
         assert i.value == 1234
-        
+
         #u = self.execute_operation(rop.GETFIELD_GC, [res, ofs_u], 'int')
         #assert u.value == 5
         self.execute_operation(rop.SETFIELD_GC, [res, ConstInt(1)], 'void',
@@ -299,7 +293,7 @@
                     else:
                         assert result != execute(self.cpu, None,
                                                  op, None, b).value
-                    
+
 
     def test_stuff_followed_by_guard(self):
         boxes = [(BoxInt(1), BoxInt(0)),
@@ -523,7 +517,7 @@
     def test_debugger_on(self):
         from pypy.tool.logparser import parse_log_file, extract_category
         from pypy.rlib import debug
-        
+
         loop = """
         [i0]
         debug_merge_point('xyz', 0)
diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py
--- a/pypy/jit/codewriter/jtransform.py
+++ b/pypy/jit/codewriter/jtransform.py
@@ -52,9 +52,11 @@
         newoperations = []
         #
         def do_rename(var, var_or_const):
+            if var.concretetype is lltype.Void:
+                renamings[var] = Constant(None, lltype.Void)
+                return
             renamings[var] = var_or_const
-            if (isinstance(var_or_const, Constant)
-                and var.concretetype != lltype.Void):
+            if isinstance(var_or_const, Constant):
                 value = var_or_const.value
                 value = lltype._cast_whatever(var.concretetype, value)
                 renamings_constants[var] = Constant(value, var.concretetype)
@@ -441,6 +443,8 @@
     rewrite_op_gc_identityhash = _do_builtin_call
     rewrite_op_gc_id           = _do_builtin_call
     rewrite_op_uint_mod        = _do_builtin_call
+    rewrite_op_cast_float_to_uint = _do_builtin_call
+    rewrite_op_cast_uint_to_float = _do_builtin_call
 
     # ----------
     # getfield/setfield/mallocs etc.
@@ -735,29 +739,54 @@
         return SpaceOperation(opname, [op.args[0]], op.result)
 
     def rewrite_op_getinteriorfield(self, op):
-        # only supports strings and unicodes
         assert len(op.args) == 3
-        assert op.args[1].value == 'chars'
         optype = op.args[0].concretetype
         if optype == lltype.Ptr(rstr.STR):
             opname = "strgetitem"
+            return SpaceOperation(opname, [op.args[0], op.args[2]], op.result)
+        elif optype == lltype.Ptr(rstr.UNICODE):
+            opname = "unicodegetitem"
+            return SpaceOperation(opname, [op.args[0], op.args[2]], op.result)
         else:
-            assert optype == lltype.Ptr(rstr.UNICODE)
-            opname = "unicodegetitem"
-        return SpaceOperation(opname, [op.args[0], op.args[2]], op.result)
+            v_inst, v_index, c_field = op.args
+            if op.result.concretetype is lltype.Void:
+                return
+            # only GcArray of Struct supported
+            assert isinstance(v_inst.concretetype.TO, lltype.GcArray)
+            STRUCT = v_inst.concretetype.TO.OF
+            assert isinstance(STRUCT, lltype.Struct)
+            descr = self.cpu.interiorfielddescrof(v_inst.concretetype.TO,
+                                                  c_field.value)
+            args = [v_inst, v_index, descr]
+            kind = getkind(op.result.concretetype)[0]
+            return SpaceOperation('getinteriorfield_gc_%s' % kind, args,
+                                  op.result)
 
     def rewrite_op_setinteriorfield(self, op):
-        # only supports strings and unicodes
         assert len(op.args) == 4
-        assert op.args[1].value == 'chars'
         optype = op.args[0].concretetype
         if optype == lltype.Ptr(rstr.STR):
             opname = "strsetitem"
+            return SpaceOperation(opname, [op.args[0], op.args[2], op.args[3]],
+                                  op.result)
+        elif optype == lltype.Ptr(rstr.UNICODE):
+            opname = "unicodesetitem"
+            return SpaceOperation(opname, [op.args[0], op.args[2], op.args[3]],
+                                  op.result)
         else:
-            assert optype == lltype.Ptr(rstr.UNICODE)
-            opname = "unicodesetitem"
-        return SpaceOperation(opname, [op.args[0], op.args[2], op.args[3]],
-                              op.result)
+            v_inst, v_index, c_field, v_value = op.args
+            if v_value.concretetype is lltype.Void:
+                return
+            # only GcArray of Struct supported
+            assert isinstance(v_inst.concretetype.TO, lltype.GcArray)
+            STRUCT = v_inst.concretetype.TO.OF
+            assert isinstance(STRUCT, lltype.Struct)
+            descr = self.cpu.interiorfielddescrof(v_inst.concretetype.TO,
+                                                  c_field.value)
+            kind = getkind(v_value.concretetype)[0]
+            args = [v_inst, v_index, v_value, descr]
+            return SpaceOperation('setinteriorfield_gc_%s' % kind, args,
+                                  op.result)
 
     def _rewrite_equality(self, op, opname):
         arg0, arg1 = op.args
@@ -771,6 +800,9 @@
     def _is_gc(self, v):
         return getattr(getattr(v.concretetype, "TO", None), "_gckind", "?") == 'gc'
 
+    def _is_rclass_instance(self, v):
+        return lltype._castdepth(v.concretetype.TO, rclass.OBJECT) >= 0
+
     def _rewrite_cmp_ptrs(self, op):
         if self._is_gc(op.args[0]):
             return op
@@ -788,11 +820,21 @@
         return self._rewrite_equality(op, 'int_is_true')
 
     def rewrite_op_ptr_eq(self, op):
-        op1 = self._rewrite_equality(op, 'ptr_iszero')
+        prefix = ''
+        if self._is_rclass_instance(op.args[0]):
+            assert self._is_rclass_instance(op.args[1])
+            op = SpaceOperation('instance_ptr_eq', op.args, op.result)
+            prefix = 'instance_'
+        op1 = self._rewrite_equality(op, prefix + 'ptr_iszero')
         return self._rewrite_cmp_ptrs(op1)
 
     def rewrite_op_ptr_ne(self, op):
-        op1 = self._rewrite_equality(op, 'ptr_nonzero')
+        prefix = ''
+        if self._is_rclass_instance(op.args[0]):
+            assert self._is_rclass_instance(op.args[1])
+            op = SpaceOperation('instance_ptr_ne', op.args, op.result)
+            prefix = 'instance_'
+        op1 = self._rewrite_equality(op, prefix + 'ptr_nonzero')
         return self._rewrite_cmp_ptrs(op1)
 
     rewrite_op_ptr_iszero = _rewrite_cmp_ptrs
@@ -821,26 +863,44 @@
         elif not float_arg and float_res:
             # some int -> some float
             ops = []
-            v1 = varoftype(lltype.Signed)
-            oplist = self.rewrite_operation(
-                SpaceOperation('force_cast', [v_arg], v1)
-            )
-            if oplist:
-                ops.extend(oplist)
+            v2 = varoftype(lltype.Float)
+            sizesign = rffi.size_and_sign(v_arg.concretetype)
+            if sizesign <= rffi.size_and_sign(lltype.Signed):
+                # cast from a type that fits in an int: either the size is
+                # smaller, or it is equal and it is not unsigned
+                v1 = varoftype(lltype.Signed)
+                oplist = self.rewrite_operation(
+                    SpaceOperation('force_cast', [v_arg], v1)
+                )
+                if oplist:
+                    ops.extend(oplist)
+                else:
+                    v1 = v_arg
+                op = self.rewrite_operation(
+                    SpaceOperation('cast_int_to_float', [v1], v2)
+                )
+                ops.append(op)
             else:
-                v1 = v_arg
-            v2 = varoftype(lltype.Float)
-            op = self.rewrite_operation(
-                SpaceOperation('cast_int_to_float', [v1], v2)
-            )
-            ops.append(op)
+                if sizesign == rffi.size_and_sign(lltype.Unsigned):
+                    opname = 'cast_uint_to_float'
+                elif sizesign == rffi.size_and_sign(lltype.SignedLongLong):
+                    opname = 'cast_longlong_to_float'
+                elif sizesign == rffi.size_and_sign(lltype.UnsignedLongLong):
+                    opname = 'cast_ulonglong_to_float'
+                else:
+                    raise AssertionError('cast_x_to_float: %r' % (sizesign,))
+                ops1 = self.rewrite_operation(
+                    SpaceOperation(opname, [v_arg], v2)
+                )
+                if not isinstance(ops1, list): ops1 = [ops1]
+                ops.extend(ops1)
             op2 = self.rewrite_operation(
                 SpaceOperation('force_cast', [v2], v_result)
             )
             if op2:
                 ops.append(op2)
             else:
-                op.result = v_result
+                ops[-1].result = v_result
             return ops
         elif float_arg and not float_res:
             # some float -> some int
@@ -853,18 +913,36 @@
                 ops.append(op1)
             else:
                 v1 = v_arg
-            v2 = varoftype(lltype.Signed)
-            op = self.rewrite_operation(
-                SpaceOperation('cast_float_to_int', [v1], v2)
-            )
-            ops.append(op)
-            oplist = self.rewrite_operation(
-                SpaceOperation('force_cast', [v2], v_result)
-            )
-            if oplist:
-                ops.extend(oplist)
+            sizesign = rffi.size_and_sign(v_result.concretetype)
+            if sizesign <= rffi.size_and_sign(lltype.Signed):
+                # cast to a type that fits in an int: either the size is
+                # smaller, or it is equal and it is not unsigned
+                v2 = varoftype(lltype.Signed)
+                op = self.rewrite_operation(
+                    SpaceOperation('cast_float_to_int', [v1], v2)
+                )
+                ops.append(op)
+                oplist = self.rewrite_operation(
+                    SpaceOperation('force_cast', [v2], v_result)
+                )
+                if oplist:
+                    ops.extend(oplist)
+                else:
+                    op.result = v_result
             else:
-                op.result = v_result
+                if sizesign == rffi.size_and_sign(lltype.Unsigned):
+                    opname = 'cast_float_to_uint'
+                elif sizesign == rffi.size_and_sign(lltype.SignedLongLong):
+                    opname = 'cast_float_to_longlong'
+                elif sizesign == rffi.size_and_sign(lltype.UnsignedLongLong):
+                    opname = 'cast_float_to_ulonglong'
+                else:
+                    raise AssertionError('cast_float_to_x: %r' % (sizesign,))
+                ops1 = self.rewrite_operation(
+                    SpaceOperation(opname, [v1], v_result)
+                )
+                if not isinstance(ops1, list): ops1 = [ops1]
+                ops.extend(ops1)
             return ops
         else:
             assert False
@@ -1070,8 +1148,6 @@
     # The new operation is optionally further processed by rewrite_operation().
     for _old, _new in [('bool_not', 'int_is_zero'),
                        ('cast_bool_to_float', 'cast_int_to_float'),
-                       ('cast_uint_to_float', 'cast_int_to_float'),
-                       ('cast_float_to_uint', 'cast_float_to_int'),
 
                        ('int_add_nonneg_ovf', 'int_add_ovf'),
                        ('keepalive', '-live-'),
diff --git a/pypy/jit/codewriter/support.py b/pypy/jit/codewriter/support.py
--- a/pypy/jit/codewriter/support.py
+++ b/pypy/jit/codewriter/support.py
@@ -13,7 +13,6 @@
 from pypy.translator.simplify import get_funcobj
 from pypy.translator.unsimplify import split_block
 from pypy.objspace.flow.model import Constant
-from pypy import conftest
 from pypy.translator.translator import TranslationContext
 from pypy.annotation.policy import AnnotatorPolicy
 from pypy.annotation import model as annmodel
@@ -48,15 +47,13 @@
     a.build_types(func, argtypes, main_entry_point=True)
     rtyper = t.buildrtyper(type_system = type_system)
     rtyper.specialize()
-    if inline:
-        auto_inlining(t, threshold=inline)
+    #if inline:
+    #    auto_inlining(t, threshold=inline)
     if backendoptimize:
         from pypy.translator.backendopt.all import backend_optimizations
         backend_optimizations(t, inline_threshold=inline or 0,
                 remove_asserts=True, really_remove_asserts=True)
 
-    #if conftest.option.view:
-    #    t.view()
     return rtyper
 
 def getgraph(func, values):
@@ -232,6 +229,17 @@
     else:
         return x
 
+def _ll_1_cast_uint_to_float(x):
+    # XXX on 32-bit platforms, this should be done using cast_longlong_to_float
+    # (which is a residual call right now in the x86 backend)
+    return llop.cast_uint_to_float(lltype.Float, x)
+
+def _ll_1_cast_float_to_uint(x):
+    # XXX on 32-bit platforms, this should be done using cast_float_to_longlong
+    # (which is a residual call right now in the x86 backend)
+    return llop.cast_float_to_uint(lltype.Unsigned, x)
+
+
 # math support
 # ------------
 
@@ -456,6 +464,8 @@
         return LLtypeHelpers._dictnext_items(lltype.Ptr(RES), iter)
     _ll_1_dictiter_nextitems.need_result_type = True
 
+    _ll_1_dict_resize = ll_rdict.ll_dict_resize
+
     # ---------- strings and unicode ----------
 
     _ll_1_str_str2unicode = ll_rstr.LLHelpers.ll_str2unicode
diff --git a/pypy/jit/codewriter/test/test_flatten.py b/pypy/jit/codewriter/test/test_flatten.py
--- a/pypy/jit/codewriter/test/test_flatten.py
+++ b/pypy/jit/codewriter/test/test_flatten.py
@@ -8,7 +8,7 @@
 from pypy.rpython.lltypesystem import lltype, rclass, rstr
 from pypy.objspace.flow.model import SpaceOperation, Variable, Constant
 from pypy.translator.unsimplify import varoftype
-from pypy.rlib.rarithmetic import ovfcheck, r_uint
+from pypy.rlib.rarithmetic import ovfcheck, r_uint, r_longlong, r_ulonglong
 from pypy.rlib.jit import dont_look_inside, _we_are_jitted, JitDriver
 from pypy.rlib.objectmodel import keepalive_until_here
 from pypy.rlib import jit
@@ -70,7 +70,8 @@
         return 'residual'
     def getcalldescr(self, op, oopspecindex=None, extraeffect=None):
         try:
-            if 'cannot_raise' in op.args[0].value._obj.graph.name:
+            name = op.args[0].value._obj._name
+            if 'cannot_raise' in name or name.startswith('cast_'):
                 return self._descr_cannot_raise
         except AttributeError:
             pass
@@ -900,6 +901,67 @@
             int_return %i4
         """, transform=True)
 
+        def f(dbl):
+            return rffi.cast(rffi.UCHAR, dbl)
+        self.encoding_test(f, [12.456], """
+            cast_float_to_int %f0 -> %i0
+            int_and %i0, $255 -> %i1
+            int_return %i1
+        """, transform=True)
+
+        def f(dbl):
+            return rffi.cast(lltype.Unsigned, dbl)
+        self.encoding_test(f, [12.456], """
+            residual_call_irf_i $<* fn cast_float_to_uint>, <Descr>, I[], R[], F[%f0] -> %i0
+            int_return %i0
+        """, transform=True)
+
+        def f(i):
+            return rffi.cast(lltype.Float, chr(i))    # "char -> float"
+        self.encoding_test(f, [12], """
+            cast_int_to_float %i0 -> %f0
+            float_return %f0
+        """, transform=True)
+
+        def f(i):
+            return rffi.cast(lltype.Float, r_uint(i))    # "uint -> float"
+        self.encoding_test(f, [12], """
+            residual_call_irf_f $<* fn cast_uint_to_float>, <Descr>, I[%i0], R[], F[] -> %f0
+            float_return %f0
+        """, transform=True)
+
+        if not longlong.is_64_bit:
+            def f(dbl):
+                return rffi.cast(lltype.SignedLongLong, dbl)
+            self.encoding_test(f, [12.3], """
+                residual_call_irf_f $<* fn llong_from_float>, <Descr>, I[], R[], F[%f0] -> %f1
+                float_return %f1
+            """, transform=True)
+
+            def f(dbl):
+                return rffi.cast(lltype.UnsignedLongLong, dbl)
+            self.encoding_test(f, [12.3], """
+                residual_call_irf_f $<* fn ullong_from_float>, <Descr>, I[], R[], F[%f0] -> %f1
+                float_return %f1
+            """, transform=True)
+
+            def f(x):
+                ll = r_longlong(x)
+                return rffi.cast(lltype.Float, ll)
+            self.encoding_test(f, [12], """
+                residual_call_irf_f $<* fn llong_from_int>, <Descr>, I[%i0], R[], F[] -> %f0
+                residual_call_irf_f $<* fn llong_to_float>, <Descr>, I[], R[], F[%f0] -> %f1
+                float_return %f1
+            """, transform=True)
+
+            def f(x):
+                ll = r_ulonglong(x)
+                return rffi.cast(lltype.Float, ll)
+            self.encoding_test(f, [12], """
+                residual_call_irf_f $<* fn ullong_from_int>, <Descr>, I[%i0], R[], F[] -> %f0
+                residual_call_irf_f $<* fn ullong_u_to_float>, <Descr>, I[], R[], F[%f0] -> %f1
+                float_return %f1
+            """, transform=True)
 
     def test_direct_ptradd(self):
         from pypy.rpython.lltypesystem import rffi
diff --git a/pypy/jit/codewriter/test/test_jtransform.py b/pypy/jit/codewriter/test/test_jtransform.py
--- a/pypy/jit/codewriter/test/test_jtransform.py
+++ b/pypy/jit/codewriter/test/test_jtransform.py
@@ -1,4 +1,3 @@
-import py
 import random
 try:
     from itertools import product
@@ -16,13 +15,13 @@
 
 from pypy.objspace.flow.model import FunctionGraph, Block, Link
 from pypy.objspace.flow.model import SpaceOperation, Variable, Constant
-from pypy.jit.codewriter.jtransform import Transformer
-from pypy.jit.metainterp.history import getkind
-from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr, rlist
+from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr
 from pypy.rpython.lltypesystem.module import ll_math
 from pypy.translator.unsimplify import varoftype
 from pypy.jit.codewriter import heaptracker, effectinfo
 from pypy.jit.codewriter.flatten import ListOfKind
+from pypy.jit.codewriter.jtransform import Transformer
+from pypy.jit.metainterp.history import getkind
 
 def const(x):
     return Constant(x, lltype.typeOf(x))
@@ -37,6 +36,8 @@
         return ('calldescr', FUNC, ARGS, RESULT)
     def fielddescrof(self, STRUCT, name):
         return ('fielddescr', STRUCT, name)
+    def interiorfielddescrof(self, ARRAY, name):
+        return ('interiorfielddescr', ARRAY, name)
     def arraydescrof(self, ARRAY):
         return FakeDescr(('arraydescr', ARRAY))
     def sizeof(self, STRUCT):
@@ -539,7 +540,7 @@
 
 def test_rename_on_links():
     v1 = Variable()
-    v2 = Variable()
+    v2 = Variable(); v2.concretetype = llmemory.Address
     v3 = Variable()
     block = Block([v1])
     block.operations = [SpaceOperation('cast_pointer', [v1], v2)]
@@ -575,10 +576,10 @@
         assert op1.args == [v2]
 
 def test_ptr_eq():
-    v1 = varoftype(rclass.OBJECTPTR)
-    v2 = varoftype(rclass.OBJECTPTR)
+    v1 = varoftype(lltype.Ptr(rstr.STR))
+    v2 = varoftype(lltype.Ptr(rstr.STR))
     v3 = varoftype(lltype.Bool)
-    c0 = const(lltype.nullptr(rclass.OBJECT))
+    c0 = const(lltype.nullptr(rstr.STR))
     #
     for opname, reducedname in [('ptr_eq', 'ptr_iszero'),
                                 ('ptr_ne', 'ptr_nonzero')]:
@@ -597,6 +598,31 @@
         assert op1.opname == reducedname
         assert op1.args == [v2]
 
+def test_instance_ptr_eq():
+    v1 = varoftype(rclass.OBJECTPTR)
+    v2 = varoftype(rclass.OBJECTPTR)
+    v3 = varoftype(lltype.Bool)
+    c0 = const(lltype.nullptr(rclass.OBJECT))
+
+    for opname, newopname, reducedname in [
+        ('ptr_eq', 'instance_ptr_eq', 'instance_ptr_iszero'),
+        ('ptr_ne', 'instance_ptr_ne', 'instance_ptr_nonzero')
+    ]:
+        op = SpaceOperation(opname, [v1, v2], v3)
+        op1 = Transformer().rewrite_operation(op)
+        assert op1.opname == newopname
+        assert op1.args == [v1, v2]
+
+        op = SpaceOperation(opname, [v1, c0], v3)
+        op1 = Transformer().rewrite_operation(op)
+        assert op1.opname == reducedname
+        assert op1.args == [v1]
+
+        op = SpaceOperation(opname, [c0, v1], v3)
+        op1 = Transformer().rewrite_operation(op)
+        assert op1.opname == reducedname
+        assert op1.args == [v1]
+
 def test_nongc_ptr_eq():
     v1 = varoftype(rclass.NONGCOBJECTPTR)
     v2 = varoftype(rclass.NONGCOBJECTPTR)
@@ -676,6 +702,22 @@
     assert op1.args == [v, v_index]
     assert op1.result == v_result
 
+def test_dict_getinteriorfield():
+    DICT = lltype.GcArray(lltype.Struct('ENTRY', ('v', lltype.Signed),
+                                        ('k', lltype.Signed)))
+    v = varoftype(lltype.Ptr(DICT))
+    i = varoftype(lltype.Signed)
+    v_result = varoftype(lltype.Signed)
+    op = SpaceOperation('getinteriorfield', [v, i, Constant('v', lltype.Void)],
+                        v_result)
+    op1 = Transformer(FakeCPU()).rewrite_operation(op)
+    assert op1.opname == 'getinteriorfield_gc_i'
+    assert op1.args == [v, i, ('interiorfielddescr', DICT, 'v')]
+    op = SpaceOperation('getinteriorfield', [v, i, Constant('v', lltype.Void)],
+                        Constant(None, lltype.Void))
+    op1 = Transformer(FakeCPU()).rewrite_operation(op)
+    assert op1 is None
+
 def test_str_setinteriorfield():
     v = varoftype(lltype.Ptr(rstr.STR))
     v_index = varoftype(lltype.Signed)
@@ -702,6 +744,23 @@
     assert op1.args == [v, v_index, v_newchr]
     assert op1.result == v_void
 
+def test_dict_setinteriorfield():
+    DICT = lltype.GcArray(lltype.Struct('ENTRY', ('v', lltype.Signed),
+                                        ('k', lltype.Signed)))
+    v = varoftype(lltype.Ptr(DICT))
+    i = varoftype(lltype.Signed)
+    v_void = varoftype(lltype.Void)
+    op = SpaceOperation('setinteriorfield', [v, i, Constant('v', lltype.Void),
+                                             i],
+                        v_void)
+    op1 = Transformer(FakeCPU()).rewrite_operation(op)
+    assert op1.opname == 'setinteriorfield_gc_i'
+    assert op1.args == [v, i, i, ('interiorfielddescr', DICT, 'v')]
+    op = SpaceOperation('setinteriorfield', [v, i, Constant('v', lltype.Void),
+                                             v_void], v_void)
+    op1 = Transformer(FakeCPU()).rewrite_operation(op)
+    assert not op1
+
 def test_promote_1():
     v1 = varoftype(lltype.Signed)
     v2 = varoftype(lltype.Signed)
diff --git a/pypy/jit/metainterp/blackhole.py b/pypy/jit/metainterp/blackhole.py
--- a/pypy/jit/metainterp/blackhole.py
+++ b/pypy/jit/metainterp/blackhole.py
@@ -6,7 +6,6 @@
 from pypy.rlib.debug import make_sure_not_resized
 from pypy.rpython.lltypesystem import lltype, llmemory, rclass
 from pypy.rpython.lltypesystem.lloperation import llop
-from pypy.rpython.llinterp import LLException
 from pypy.jit.codewriter.jitcode import JitCode, SwitchDictDescr
 from pypy.jit.codewriter import heaptracker, longlong
 from pypy.jit.metainterp.jitexc import JitException, get_llexception, reraise
@@ -500,6 +499,12 @@
     @arguments("r", returns="i")
     def bhimpl_ptr_nonzero(a):
         return bool(a)
+    @arguments("r", "r", returns="i")
+    def bhimpl_instance_ptr_eq(a, b):
+        return a == b
+    @arguments("r", "r", returns="i")
+    def bhimpl_instance_ptr_ne(a, b):
+        return a != b
     @arguments("r", returns="r")
     def bhimpl_cast_opaque_ptr(a):
         return a
@@ -631,6 +636,9 @@
         a = longlong.getrealfloat(a)
         # note: we need to call int() twice to care for the fact that
         # int(-2147483648.0) returns a long :-(
+        # we could also call intmask() instead of the outermost int(), but
+        # it's probably better to explicitly crash (by getting a long) if a
+        # non-translated version tries to cast a too large float to an int.
         return int(int(a))
 
     @arguments("i", returns="f")
@@ -1154,6 +1162,26 @@
         array = cpu.bh_getfield_gc_r(vable, fdescr)
         return cpu.bh_arraylen_gc(adescr, array)
 
+    @arguments("cpu", "r", "i", "d", returns="i")
+    def bhimpl_getinteriorfield_gc_i(cpu, array, index, descr):
+        return cpu.bh_getinteriorfield_gc_i(array, index, descr)
+    @arguments("cpu", "r", "i", "d", returns="r")
+    def bhimpl_getinteriorfield_gc_r(cpu, array, index, descr):
+        return cpu.bh_getinteriorfield_gc_r(array, index, descr)
+    @arguments("cpu", "r", "i", "d", returns="f")
+    def bhimpl_getinteriorfield_gc_f(cpu, array, index, descr):
+        return cpu.bh_getinteriorfield_gc_f(array, index, descr)
+
+    @arguments("cpu", "r", "i", "d", "i")
+    def bhimpl_setinteriorfield_gc_i(cpu, array, index, descr, value):
+        cpu.bh_setinteriorfield_gc_i(array, index, descr, value)
+    @arguments("cpu", "r", "i", "d", "r")
+    def bhimpl_setinteriorfield_gc_r(cpu, array, index, descr, value):
+        cpu.bh_setinteriorfield_gc_r(array, index, descr, value)
+    @arguments("cpu", "r", "i", "d", "f")
+    def bhimpl_setinteriorfield_gc_f(cpu, array, index, descr, value):
+        cpu.bh_setinteriorfield_gc_f(array, index, descr, value)
+
     @arguments("cpu", "r", "d", returns="i")
     def bhimpl_getfield_gc_i(cpu, struct, fielddescr):
         return cpu.bh_getfield_gc_i(struct, fielddescr)
diff --git a/pypy/jit/metainterp/executor.py b/pypy/jit/metainterp/executor.py
--- a/pypy/jit/metainterp/executor.py
+++ b/pypy/jit/metainterp/executor.py
@@ -1,11 +1,8 @@
 """This implements pyjitpl's execution of operations.
 """
 
-import py
-from pypy.rpython.lltypesystem import lltype, llmemory, rstr
-from pypy.rpython.ootypesystem import ootype
-from pypy.rpython.lltypesystem.lloperation import llop
-from pypy.rlib.rarithmetic import ovfcheck, r_uint, intmask, r_longlong
+from pypy.rpython.lltypesystem import lltype, rstr
+from pypy.rlib.rarithmetic import ovfcheck, r_longlong
 from pypy.rlib.rtimer import read_timestamp
 from pypy.rlib.unroll import unrolling_iterable
 from pypy.jit.metainterp.history import BoxInt, BoxPtr, BoxFloat, check_descr
@@ -123,6 +120,29 @@
     else:
         cpu.bh_setarrayitem_raw_i(arraydescr, array, index, itembox.getint())
 
+def do_getinteriorfield_gc(cpu, _, arraybox, indexbox, descr):
+    array = arraybox.getref_base()
+    index = indexbox.getint()
+    if descr.is_pointer_field():
+        return BoxPtr(cpu.bh_getinteriorfield_gc_r(array, index, descr))
+    elif descr.is_float_field():
+        return BoxFloat(cpu.bh_getinteriorfield_gc_f(array, index, descr))
+    else:
+        return BoxInt(cpu.bh_getinteriorfield_gc_i(array, index, descr))
+
+def do_setinteriorfield_gc(cpu, _, arraybox, indexbox, valuebox, descr):
+    array = arraybox.getref_base()
+    index = indexbox.getint()
+    if descr.is_pointer_field():
+        cpu.bh_setinteriorfield_gc_r(array, index, descr,
+                                     valuebox.getref_base())
+    elif descr.is_float_field():
+        cpu.bh_setinteriorfield_gc_f(array, index, descr,
+                                     valuebox.getfloatstorage())
+    else:
+        cpu.bh_setinteriorfield_gc_i(array, index, descr,
+                                     valuebox.getint())
+
 def do_getfield_gc(cpu, _, structbox, fielddescr):
     struct = structbox.getref_base()
     if fielddescr.is_pointer_field():
diff --git a/pypy/jit/metainterp/graphpage.py b/pypy/jit/metainterp/graphpage.py
--- a/pypy/jit/metainterp/graphpage.py
+++ b/pypy/jit/metainterp/graphpage.py
@@ -12,8 +12,8 @@
     def get_display_text(self):
         return None
 
-def display_loops(loops, errmsg=None, highlight_loops=()):
-    graphs = [(loop, loop in highlight_loops) for loop in loops]    
+def display_loops(loops, errmsg=None, highlight_loops={}):
+    graphs = [(loop, highlight_loops.get(loop, 0)) for loop in loops]    
     for graph, highlight in graphs:
         for op in graph.get_operations():
             if is_interesting_guard(op):
@@ -65,8 +65,7 @@
     def add_graph(self, graph, highlight=False):
         graphindex = len(self.graphs)
         self.graphs.append(graph)
-        if highlight:
-            self.highlight_graphs[graph] = True
+        self.highlight_graphs[graph] = highlight
         for i, op in enumerate(graph.get_operations()):
             self.all_operations[op] = graphindex, i
 
@@ -126,10 +125,13 @@
             self.dotgen.emit('subgraph cluster%d {' % graphindex)
         label = graph.get_display_text()
         if label is not None:
-            if self.highlight_graphs.get(graph):
-                fillcolor = '#f084c2'
+            colorindex = self.highlight_graphs.get(graph, 0)
+            if colorindex == 1:
+                fillcolor = '#f084c2'    # highlighted graph
+            elif colorindex == 2:
+                fillcolor = '#808080'    # invalidated graph
             else:
-                fillcolor = '#84f0c2'
+                fillcolor = '#84f0c2'    # normal color
             self.dotgen.emit_node(graphname, shape="octagon",
                                   label=label, fillcolor=fillcolor)
             self.pendingedges.append((graphname,
diff --git a/pypy/jit/metainterp/history.py b/pypy/jit/metainterp/history.py
--- a/pypy/jit/metainterp/history.py
+++ b/pypy/jit/metainterp/history.py
@@ -16,6 +16,7 @@
 INT   = 'i'
 REF   = 'r'
 FLOAT = 'f'
+STRUCT = 's'
 HOLE  = '_'
 VOID  = 'v'
 
@@ -172,6 +173,11 @@
         """
         raise NotImplementedError
 
+    def is_array_of_structs(self):
+        """ Implement for array descr
+        """
+        raise NotImplementedError
+
     def is_pointer_field(self):
         """ Implement for field descr
         """
@@ -732,6 +738,7 @@
     failed_states = None
     retraced_count = 0
     terminating = False # see TerminatingLoopToken in compile.py
+    invalidated = False
     outermost_jitdriver_sd = None
     # and more data specified by the backend when the loop is compiled
     number = -1
@@ -934,6 +941,7 @@
         self.loops = []
         self.locations = []
         self.aborted_keys = []
+        self.invalidated_token_numbers = set()
 
     def set_history(self, history):
         self.operations = history.operations
@@ -1012,7 +1020,12 @@
             if loop in loops:
                 loops.remove(loop)
             loops.append(loop)
-        display_loops(loops, errmsg, extraloops)
+        highlight_loops = dict.fromkeys(extraloops, 1)
+        for loop in loops:
+            if hasattr(loop, '_looptoken_number') and (
+                    loop._looptoken_number in self.invalidated_token_numbers):
+                highlight_loops.setdefault(loop, 2)
+        display_loops(loops, errmsg, highlight_loops)
 
 # ----------------------------------------------------------------
 
diff --git a/pypy/jit/metainterp/memmgr.py b/pypy/jit/metainterp/memmgr.py
--- a/pypy/jit/metainterp/memmgr.py
+++ b/pypy/jit/metainterp/memmgr.py
@@ -68,7 +68,8 @@
         debug_print("Loop tokens before:", oldtotal)
         max_generation = self.current_generation - (self.max_age-1)
         for looptoken in self.alive_loops.keys():
-            if 0 <= looptoken.generation < max_generation:
+            if (0 <= looptoken.generation < max_generation or
+                looptoken.invalidated):
                 del self.alive_loops[looptoken]
         newtotal = len(self.alive_loops)
         debug_print("Loop tokens freed: ", oldtotal - newtotal)
diff --git a/pypy/jit/metainterp/optimizeopt/rewrite.py b/pypy/jit/metainterp/optimizeopt/rewrite.py
--- a/pypy/jit/metainterp/optimizeopt/rewrite.py
+++ b/pypy/jit/metainterp/optimizeopt/rewrite.py
@@ -106,10 +106,9 @@
             self.make_equal_to(op.result, v1)
         else:
             self.emit_operation(op)
-
-        # Synthesize the reverse ops for optimize_default to reuse
-        self.pure(rop.INT_ADD, [op.result, op.getarg(1)], op.getarg(0))
-        self.pure(rop.INT_SUB, [op.getarg(0), op.result], op.getarg(1))
+            # Synthesize the reverse ops for optimize_default to reuse
+            self.pure(rop.INT_ADD, [op.result, op.getarg(1)], op.getarg(0))
+            self.pure(rop.INT_SUB, [op.getarg(0), op.result], op.getarg(1))
 
     def optimize_INT_ADD(self, op):
         v1 = self.getvalue(op.getarg(0))
@@ -122,10 +121,9 @@
             self.make_equal_to(op.result, v1)
         else:
             self.emit_operation(op)
-
-        # Synthesize the reverse op for optimize_default to reuse
-        self.pure(rop.INT_SUB, [op.result, op.getarg(1)], op.getarg(0))
-        self.pure(rop.INT_SUB, [op.result, op.getarg(0)], op.getarg(1))
+            # Synthesize the reverse op for optimize_default to reuse
+            self.pure(rop.INT_SUB, [op.result, op.getarg(1)], op.getarg(0))
+            self.pure(rop.INT_SUB, [op.result, op.getarg(0)], op.getarg(1))
 
     def optimize_INT_MUL(self, op):
         v1 = self.getvalue(op.getarg(0))
@@ -141,13 +139,13 @@
             self.make_constant_int(op.result, 0)
         else:
             for lhs, rhs in [(v1, v2), (v2, v1)]:
-                # x & (x -1) == 0 is a quick test for power of 2
-                if (lhs.is_constant() and
-                    (lhs.box.getint() & (lhs.box.getint() - 1)) == 0):
-                    new_rhs = ConstInt(highest_bit(lhs.box.getint()))
-                    op = op.copy_and_change(rop.INT_LSHIFT, args=[rhs.box, new_rhs])
-                    break
-
+                if lhs.is_constant():
+                    x = lhs.box.getint()
+                    # x & (x - 1) == 0 is a quick test for power of 2
+                    if x & (x - 1) == 0:
+                        new_rhs = ConstInt(highest_bit(lhs.box.getint()))
+                        op = op.copy_and_change(rop.INT_LSHIFT, args=[rhs.box, new_rhs])
+                        break
             self.emit_operation(op)
 
     def optimize_UINT_FLOORDIV(self, op):
@@ -339,7 +337,7 @@
     def optimize_INT_IS_ZERO(self, op):
         self._optimize_nullness(op, op.getarg(0), False)
 
-    def _optimize_oois_ooisnot(self, op, expect_isnot):
+    def _optimize_oois_ooisnot(self, op, expect_isnot, instance):
         value0 = self.getvalue(op.getarg(0))
         value1 = self.getvalue(op.getarg(1))
         if value0.is_virtual():
@@ -357,21 +355,28 @@
         elif value0 is value1:
             self.make_constant_int(op.result, not expect_isnot)
         else:
-            cls0 = value0.get_constant_class(self.optimizer.cpu)
-            if cls0 is not None:
-                cls1 = value1.get_constant_class(self.optimizer.cpu)
-                if cls1 is not None and not cls0.same_constant(cls1):
-                    # cannot be the same object, as we know that their
-                    # class is different
-                    self.make_constant_int(op.result, expect_isnot)
-                    return
+            if instance:
+                cls0 = value0.get_constant_class(self.optimizer.cpu)
+                if cls0 is not None:
+                    cls1 = value1.get_constant_class(self.optimizer.cpu)
+                    if cls1 is not None and not cls0.same_constant(cls1):
+                        # cannot be the same object, as we know that their
+                        # class is different
+                        self.make_constant_int(op.result, expect_isnot)
+                        return
             self.emit_operation(op)
 
+    def optimize_PTR_EQ(self, op):
+        self._optimize_oois_ooisnot(op, False, False)
+
     def optimize_PTR_NE(self, op):
-        self._optimize_oois_ooisnot(op, True)
+        self._optimize_oois_ooisnot(op, True, False)
 
-    def optimize_PTR_EQ(self, op):
-        self._optimize_oois_ooisnot(op, False)
+    def optimize_INSTANCE_PTR_EQ(self, op):
+        self._optimize_oois_ooisnot(op, False, True)
+
+    def optimize_INSTANCE_PTR_NE(self, op):
+        self._optimize_oois_ooisnot(op, True, True)
 
 ##    def optimize_INSTANCEOF(self, op):
 ##        value = self.getvalue(op.args[0])
@@ -450,6 +455,9 @@
         if v2.is_constant() and v2.box.getint() == 1:
             self.make_equal_to(op.result, v1)
             return
+        elif v1.is_constant() and v1.box.getint() == 0:
+            self.make_constant_int(op.result, 0)
+            return
         if v1.intbound.known_ge(IntBound(0, 0)) and v2.is_constant():
             val = v2.box.getint()
             if val & (val - 1) == 0 and val > 0: # val == 2**shift
@@ -462,6 +470,14 @@
         self.optimizer.opaque_pointers[value] = True
         self.make_equal_to(op.result, value)
 
+    def optimize_CAST_PTR_TO_INT(self, op):
+        self.pure(rop.CAST_INT_TO_PTR, [op.result], op.getarg(0))
+        self.emit_operation(op)
+
+    def optimize_CAST_INT_TO_PTR(self, op):
+        self.pure(rop.CAST_PTR_TO_INT, [op.result], op.getarg(0))
+        self.emit_operation(op)
+
 dispatch_opt = make_dispatcher_method(OptRewrite, 'optimize_',
         default=OptRewrite.emit_operation)
 optimize_guards = _findall(OptRewrite, 'optimize_', 'GUARD')
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -508,13 +508,13 @@
         ops = """
         [p0]
         guard_class(p0, ConstClass(node_vtable)) []
-        i0 = ptr_ne(p0, NULL)
+        i0 = instance_ptr_ne(p0, NULL)
         guard_true(i0) []
-        i1 = ptr_eq(p0, NULL)
+        i1 = instance_ptr_eq(p0, NULL)
         guard_false(i1) []
-        i2 = ptr_ne(NULL, p0)
+        i2 = instance_ptr_ne(NULL, p0)
         guard_true(i0) []
-        i3 = ptr_eq(NULL, p0)
+        i3 = instance_ptr_eq(NULL, p0)
         guard_false(i1) []
         jump(p0)
         """
@@ -2026,7 +2026,7 @@
         ops = """
         [p1]
         guard_class(p1, ConstClass(node_vtable2)) []
-        i = ptr_ne(ConstPtr(myptr), p1)
+        i = instance_ptr_ne(ConstPtr(myptr), p1)
         guard_true(i) []
         jump(p1)
         """
@@ -2181,6 +2181,17 @@
         """
         self.optimize_loop(ops, expected)
 
+        ops = """
+        [i0]
+        i1 = int_floordiv(0, i0)
+        jump(i1)
+        """
+        expected = """
+        [i0]
+        jump(0)
+        """
+        self.optimize_loop(ops, expected)
+
     def test_fold_partially_constant_ops_ovf(self):
         ops = """
         [i0]
@@ -4789,6 +4800,18 @@
         """
         self.optimize_strunicode_loop(ops, expected)
 
+    def test_ptr_eq_str_constant(self):
+        ops = """
+        []
+        i0 = ptr_eq(s"abc", s"\x00")
+        finish(i0)
+        """
+        expected = """
+        []
+        finish(0)
+        """
+        self.optimize_loop(ops, expected)
+
 
 class TestLLtype(BaseTestOptimizeBasic, LLtypeMixin):
     pass
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -234,6 +234,30 @@
             """ % expected_value
             self.optimize_loop(ops, expected)
 
+    def test_reverse_of_cast(self):
+        ops = """
+        [i0]
+        p0 = cast_int_to_ptr(i0)
+        i1 = cast_ptr_to_int(p0)
+        jump(i1)
+        """
+        expected = """
+        [i0]
+        jump(i0)
+        """
+        self.optimize_loop(ops, expected)
+        ops = """
+        [p0]
+        i1 = cast_ptr_to_int(p0)
+        p1 = cast_int_to_ptr(i1)
+        jump(p1)
+        """
+        expected = """
+        [p0]
+        jump(p0)
+        """
+        self.optimize_loop(ops, expected)
+
     # ----------
 
     def test_remove_guard_class_1(self):
@@ -2659,7 +2683,7 @@
         ops = """
         [p1]
         guard_class(p1, ConstClass(node_vtable2)) []
-        i = ptr_ne(ConstPtr(myptr), p1)
+        i = instance_ptr_ne(ConstPtr(myptr), p1)
         guard_true(i) []
         jump(p1)
         """
@@ -3307,7 +3331,7 @@
         jump(p1, i1, i2, i6)
         '''
         self.optimize_loop(ops, expected, preamble)
-        
+
 
     # ----------
 
@@ -7256,7 +7280,7 @@
         ops = """
         [p1, p2]
         setarrayitem_gc(p1, 2, 10, descr=arraydescr)
-        setarrayitem_gc(p2, 3, 13, descr=arraydescr)        
+        setarrayitem_gc(p2, 3, 13, descr=arraydescr)
         call(0, p1, p2, 0, 0, 10, descr=arraycopydescr)
         jump(p1, p2)
         """
diff --git a/pypy/jit/metainterp/optimizeopt/virtualize.py b/pypy/jit/metainterp/optimizeopt/virtualize.py
--- a/pypy/jit/metainterp/optimizeopt/virtualize.py
+++ b/pypy/jit/metainterp/optimizeopt/virtualize.py
@@ -59,7 +59,7 @@
 
     def import_from(self, other, optimizer):
         raise NotImplementedError("should not be called at this level")
-    
+
 def get_fielddescrlist_cache(cpu):
     if not hasattr(cpu, '_optimizeopt_fielddescrlist_cache'):
         result = descrlist_dict()
@@ -113,7 +113,7 @@
         #
         if not we_are_translated():
             op.name = 'FORCE ' + self.source_op.name
-            
+
         if self._is_immutable_and_filled_with_constants(optforce):
             box = optforce.optimizer.constant_fold(op)
             self.make_constant(box)
@@ -239,12 +239,12 @@
         for index in range(len(self._items)):
             self._items[index] = self._items[index].force_at_end_of_preamble(already_forced, optforce)
         return self
-    
+
     def _really_force(self, optforce):
         assert self.source_op is not None
         if not we_are_translated():
             self.source_op.name = 'FORCE ' + self.source_op.name
-        optforce.emit_operation(self.source_op)        
+        optforce.emit_operation(self.source_op)
         self.box = box = self.source_op.result
         for index in range(len(self._items)):
             subvalue = self._items[index]
@@ -276,7 +276,7 @@
 
     def new(self):
         return OptVirtualize()
-        
+
     def make_virtual(self, known_class, box, source_op=None):
         vvalue = VirtualValue(self.optimizer.cpu, known_class, box, source_op)
         self.make_equal_to(box, vvalue)
@@ -386,7 +386,8 @@
 
     def optimize_NEW_ARRAY(self, op):
         sizebox = self.get_constant_box(op.getarg(0))
-        if sizebox is not None:
+        # For now we can't make arrays of structs virtual.
+        if sizebox is not None and not op.getdescr().is_array_of_structs():
             # if the original 'op' did not have a ConstInt as argument,
             # build a new one with the ConstInt argument
             if not isinstance(op.getarg(0), ConstInt):
diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py
--- a/pypy/jit/metainterp/pyjitpl.py
+++ b/pypy/jit/metainterp/pyjitpl.py
@@ -36,6 +36,7 @@
 
 
 class MIFrame(object):
+    debug = False
 
     def __init__(self, metainterp):
         self.metainterp = metainterp
@@ -164,7 +165,7 @@
         if not we_are_translated():
             for b in registers[count:]:
                 assert not oldbox.same_box(b)
-                
+
 
     def make_result_of_lastop(self, resultbox):
         got_type = resultbox.type
@@ -198,7 +199,7 @@
                     'float_add', 'float_sub', 'float_mul', 'float_truediv',
                     'float_lt', 'float_le', 'float_eq',
                     'float_ne', 'float_gt', 'float_ge',
-                    'ptr_eq', 'ptr_ne',
+                    'ptr_eq', 'ptr_ne', 'instance_ptr_eq', 'instance_ptr_ne',
                     ]:
         exec py.code.Source('''
             @arguments("box", "box")
@@ -548,6 +549,14 @@
     opimpl_getfield_gc_r_pure = _opimpl_getfield_gc_pure_any
     opimpl_getfield_gc_f_pure = _opimpl_getfield_gc_pure_any
 
+    @arguments("box", "box", "descr")
+    def _opimpl_getinteriorfield_gc_any(self, array, index, descr):
+        return self.execute_with_descr(rop.GETINTERIORFIELD_GC, descr,
+                                       array, index)
+    opimpl_getinteriorfield_gc_i = _opimpl_getinteriorfield_gc_any
+    opimpl_getinteriorfield_gc_f = _opimpl_getinteriorfield_gc_any
+    opimpl_getinteriorfield_gc_r = _opimpl_getinteriorfield_gc_any
+
     @specialize.arg(1)
     def _opimpl_getfield_gc_any_pureornot(self, opnum, box, fielddescr):
         tobox = self.metainterp.heapcache.getfield(box, fielddescr)
@@ -588,6 +597,15 @@
     opimpl_setfield_gc_r = _opimpl_setfield_gc_any
     opimpl_setfield_gc_f = _opimpl_setfield_gc_any
 
+    @arguments("box", "box", "box", "descr")
+    def _opimpl_setinteriorfield_gc_any(self, array, index, value, descr):
+        self.execute_with_descr(rop.SETINTERIORFIELD_GC, descr,
+                                array, index, value)
+    opimpl_setinteriorfield_gc_i = _opimpl_setinteriorfield_gc_any
+    opimpl_setinteriorfield_gc_f = _opimpl_setinteriorfield_gc_any
+    opimpl_setinteriorfield_gc_r = _opimpl_setinteriorfield_gc_any
+
+
     @arguments("box", "descr")
     def _opimpl_getfield_raw_any(self, box, fielddescr):
         return self.execute_with_descr(rop.GETFIELD_RAW, fielddescr, box)
@@ -2588,17 +2606,21 @@
         self.pc = position
         #
         if not we_are_translated():
-            print '\tpyjitpl: %s(%s)' % (name, ', '.join(map(repr, args))),
+            if self.debug:
+                print '\tpyjitpl: %s(%s)' % (name, ', '.join(map(repr, args))),
             try:
                 resultbox = unboundmethod(self, *args)
             except Exception, e:
-                print '-> %s!' % e.__class__.__name__
+                if self.debug:
+                    print '-> %s!' % e.__class__.__name__
                 raise
             if num_return_args == 0:
-                print
+                if self.debug:
+                    print
                 assert resultbox is None
             else:
-                print '-> %r' % (resultbox,)
+                if self.debug:
+                    print '-> %r' % (resultbox,)
                 assert argcodes[next_argcode] == '>'
                 result_argcode = argcodes[next_argcode + 1]
                 assert resultbox.type == {'i': history.INT,
diff --git a/pypy/jit/metainterp/quasiimmut.py b/pypy/jit/metainterp/quasiimmut.py
--- a/pypy/jit/metainterp/quasiimmut.py
+++ b/pypy/jit/metainterp/quasiimmut.py
@@ -2,6 +2,7 @@
 from pypy.rpython.lltypesystem import lltype, rclass
 from pypy.rpython.annlowlevel import cast_base_ptr_to_instance
 from pypy.jit.metainterp.history import AbstractDescr
+from pypy.rlib.objectmodel import we_are_translated
 
 
 def get_mutate_field_name(fieldname):
@@ -50,13 +51,13 @@
 
 class QuasiImmut(object):
     llopaque = True
+    compress_limit = 30
     
     def __init__(self, cpu):
         self.cpu = cpu
         # list of weakrefs to the LoopTokens that must be invalidated if
         # this value ever changes
         self.looptokens_wrefs = []
-        self.compress_limit = 30
 
     def hide(self):
         qmut_ptr = self.cpu.ts.cast_instance_to_base_ref(self)
@@ -75,6 +76,8 @@
     def compress_looptokens_list(self):
         self.looptokens_wrefs = [wref for wref in self.looptokens_wrefs
                                       if wref() is not None]
+        # NB. we must keep around the looptoken_wrefs that are
+        # already invalidated; see below
         self.compress_limit = (len(self.looptokens_wrefs) + 15) * 2
 
     def invalidate(self):
@@ -86,7 +89,16 @@
         for wref in wrefs:
             looptoken = wref()
             if looptoken is not None:
+                looptoken.invalidated = True
                 self.cpu.invalidate_loop(looptoken)
+                # NB. we must call cpu.invalidate_loop() even if
+                # looptoken.invalidated was already set to True.
+                # It's possible to invalidate several times the
+                # same looptoken; see comments in jit.backend.model
+                # in invalidate_loop().
+                if not we_are_translated():
+                    self.cpu.stats.invalidated_token_numbers.add(
+                        looptoken.number)
 
 
 class QuasiImmutDescr(AbstractDescr):
diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py
--- a/pypy/jit/metainterp/resoperation.py
+++ b/pypy/jit/metainterp/resoperation.py
@@ -1,5 +1,4 @@
 from pypy.rlib.objectmodel import we_are_translated
-from pypy.rlib.debug import make_sure_not_resized
 
 def ResOperation(opnum, args, result, descr=None):
     cls = opclasses[opnum]
@@ -405,8 +404,8 @@
     'FLOAT_TRUEDIV/2',
     'FLOAT_NEG/1',
     'FLOAT_ABS/1',
-    'CAST_FLOAT_TO_INT/1',
-    'CAST_INT_TO_FLOAT/1',
+    'CAST_FLOAT_TO_INT/1',          # don't use for unsigned ints; we would
+    'CAST_INT_TO_FLOAT/1',          # need some messy code in the backend
     'CAST_FLOAT_TO_SINGLEFLOAT/1',
     'CAST_SINGLEFLOAT_TO_FLOAT/1',
     #
@@ -431,13 +430,15 @@
     'INT_IS_TRUE/1b',
     'INT_NEG/1',
     'INT_INVERT/1',
+    #
+    'SAME_AS/1',      # gets a Const or a Box, turns it into another Box
     'CAST_PTR_TO_INT/1',
     'CAST_INT_TO_PTR/1',
     #
-    'SAME_AS/1',      # gets a Const or a Box, turns it into another Box
-    #
     'PTR_EQ/2b',
     'PTR_NE/2b',
+    'INSTANCE_PTR_EQ/2b',
+    'INSTANCE_PTR_NE/2b',
     'CAST_OPAQUE_PTR/1b',
     #
     'ARRAYLEN_GC/1d',
@@ -457,6 +458,7 @@
 
     'GETARRAYITEM_GC/2d',
     'GETARRAYITEM_RAW/2d',
+    'GETINTERIORFIELD_GC/2d',
     'GETFIELD_GC/1d',
     'GETFIELD_RAW/1d',
     '_MALLOC_FIRST',
@@ -473,6 +475,7 @@
 
     'SETARRAYITEM_GC/3d',
     'SETARRAYITEM_RAW/3d',
+    'SETINTERIORFIELD_GC/3d',
     'SETFIELD_GC/2d',
     'SETFIELD_RAW/2d',
     'STRSETITEM/3',
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -3435,7 +3435,75 @@
             return sa
         res = self.meta_interp(f, [16])
         assert res == f(16)
-        
+
+    def test_ptr_eq(self):
+        myjitdriver = JitDriver(greens = [], reds = ["n", "x"])
+        class A(object):
+            def __init__(self, v):
+                self.v = v
+        def f(n, x):
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, x=x)
+                z = 0 / x
+                a1 = A("key")
+                a2 = A("\x00")
+                n -= [a1, a2][z].v is not a2.v
+            return n
+        res = self.meta_interp(f, [10, 1])
+        assert res == 0
+
+    def test_instance_ptr_eq(self):
+        myjitdriver = JitDriver(greens = [], reds = ["n", "i", "a1", "a2"])
+        class A(object):
+            pass
+        def f(n):
+            a1 = A()
+            a2 = A()
+            i = 0
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, i=i, a1=a1, a2=a2)
+                if n % 2:
+                    a = a2
+                else:
+                    a = a1
+                i += a is a1
+                n -= 1
+            return i
+        res = self.meta_interp(f, [10])
+        assert res == f(10)
+        def f(n):
+            a1 = A()
+            a2 = A()
+            i = 0
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, i=i, a1=a1, a2=a2)
+                if n % 2:
+                    a = a2
+                else:
+                    a = a1
+                if a is a2:
+                    i += 1
+                n -= 1
+            return i
+        res = self.meta_interp(f, [10])
+        assert res == f(10)
+
+    def test_virtual_array_of_structs(self):
+        myjitdriver = JitDriver(greens = [], reds=["n", "d"])
+        def f(n):
+            d = None
+            while n > 0:
+                myjitdriver.jit_merge_point(n=n, d=d)
+                d = {}
+                if n % 2:
+                    d["k"] = n
+                else:
+                    d["z"] = n
+                n -= len(d)
+            return n
+        res = self.meta_interp(f, [10])
+        assert res == 0
+
 
 
 class TestLLtype(BaseLLtypeTests, LLJitMixin):
diff --git a/pypy/jit/metainterp/test/test_dict.py b/pypy/jit/metainterp/test/test_dict.py
--- a/pypy/jit/metainterp/test/test_dict.py
+++ b/pypy/jit/metainterp/test/test_dict.py
@@ -91,7 +91,7 @@
         res1 = f(100)
         res2 = self.meta_interp(f, [100], listops=True)
         assert res1 == res2
-        self.check_loops(int_mod=1) # the hash was traced
+        self.check_loops(int_mod=1) # the hash was traced and eq, but cached
 
     def test_dict_setdefault(self):
         myjitdriver = JitDriver(greens = [], reds = ['total', 'dct'])
@@ -128,7 +128,7 @@
         assert f(100) == 50
         res = self.meta_interp(f, [100], listops=True)
         assert res == 50
-        self.check_loops(int_mod=1)
+        self.check_loops(int_mod=1) # key + eq, but cached
 
     def test_repeated_lookup(self):
         myjitdriver = JitDriver(greens = [], reds = ['n', 'd'])
@@ -153,10 +153,12 @@
 
         res = self.meta_interp(f, [100], listops=True)
         assert res == f(50)
-        self.check_loops({"call": 7, "guard_false": 1, "guard_no_exception": 6,
+        self.check_loops({"call": 5, "getfield_gc": 1, "getinteriorfield_gc": 1,
+                          "guard_false": 1, "guard_no_exception": 4,
                           "guard_true": 1, "int_and": 1, "int_gt": 1,
                           "int_is_true": 1, "int_sub": 1, "jump": 1,
-                          "new_with_vtable": 1, "setfield_gc": 1})
+                          "new_with_vtable": 1, "new": 1, "new_array": 1,
+                          "setfield_gc": 3, })
 
 
 class TestOOtype(DictTests, OOJitMixin):
diff --git a/pypy/jit/metainterp/test/test_float.py b/pypy/jit/metainterp/test/test_float.py
--- a/pypy/jit/metainterp/test/test_float.py
+++ b/pypy/jit/metainterp/test/test_float.py
@@ -1,5 +1,6 @@
-import math
+import math, sys
 from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin
+from pypy.rlib.rarithmetic import intmask, r_uint
 
 
 class FloatTests:
@@ -45,6 +46,34 @@
         res = self.interp_operations(f, [-2.0])
         assert res == -8.5
 
+    def test_cast_float_to_int(self):
+        def g(f):
+            return int(f)
+        res = self.interp_operations(g, [-12345.9])
+        assert res == -12345
+
+    def test_cast_float_to_uint(self):
+        def g(f):
+            return intmask(r_uint(f))
+        res = self.interp_operations(g, [sys.maxint*2.0])
+        assert res == intmask(long(sys.maxint*2.0))
+        res = self.interp_operations(g, [-12345.9])
+        assert res == -12345
+
+    def test_cast_int_to_float(self):
+        def g(i):
+            return float(i)
+        res = self.interp_operations(g, [-12345])
+        assert type(res) is float and res == -12345.0
+
+    def test_cast_uint_to_float(self):
+        def g(i):
+            return float(r_uint(i))
+        res = self.interp_operations(g, [intmask(sys.maxint*2)])
+        assert type(res) is float and res == float(sys.maxint*2)
+        res = self.interp_operations(g, [-12345])
+        assert type(res) is float and res == float(long(r_uint(-12345)))
+
 
 class TestOOtype(FloatTests, OOJitMixin):
     pass
diff --git a/pypy/jit/metainterp/test/test_memmgr.py b/pypy/jit/metainterp/test/test_memmgr.py
--- a/pypy/jit/metainterp/test/test_memmgr.py
+++ b/pypy/jit/metainterp/test/test_memmgr.py
@@ -18,6 +18,7 @@
 
 class FakeLoopToken:
     generation = 0
+    invalidated = False
 
 
 class _TestMemoryManager:
diff --git a/pypy/jit/metainterp/test/test_quasiimmut.py b/pypy/jit/metainterp/test/test_quasiimmut.py
--- a/pypy/jit/metainterp/test/test_quasiimmut.py
+++ b/pypy/jit/metainterp/test/test_quasiimmut.py
@@ -48,6 +48,13 @@
 
 class QuasiImmutTests(object):
 
+    def setup_method(self, meth):
+        self.prev_compress_limit = QuasiImmut.compress_limit
+        QuasiImmut.compress_limit = 1
+
+    def teardown_method(self, meth):
+        QuasiImmut.compress_limit = self.prev_compress_limit
+
     def test_simple_1(self):
         myjitdriver = JitDriver(greens=['foo'], reds=['x', 'total'])
         class Foo:
@@ -289,7 +296,7 @@
             return total
 
         res = self.meta_interp(main, [])
-        self.check_loop_count(9)
+        self.check_tree_loop_count(6)
         assert res == main()
 
     def test_change_during_running(self):
@@ -317,7 +324,7 @@
         assert f(100, 15) == 3009
         res = self.meta_interp(f, [100, 15])
         assert res == 3009
-        self.check_loops(guard_not_invalidated=2, getfield_gc=0,
+        self.check_loops(guard_not_invalidated=4, getfield_gc=0,
                          call_may_force=0, guard_not_forced=0)
 
     def test_list_simple_1(self):
@@ -453,10 +460,30 @@
         assert f(100, 15) == 3009
         res = self.meta_interp(f, [100, 15])
         assert res == 3009
-        self.check_loops(guard_not_invalidated=2, getfield_gc=0,
+        self.check_loops(guard_not_invalidated=4, getfield_gc=0,
                          getarrayitem_gc=0, getarrayitem_gc_pure=0,
                          call_may_force=0, guard_not_forced=0)
 
+    def test_invalidated_loop_is_not_used_any_more_as_target(self):
+        myjitdriver = JitDriver(greens=['foo'], reds=['x'])
+        class Foo:
+            _immutable_fields_ = ['step?']
+        @dont_look_inside
+        def residual(x, foo):
+            if x == 20:
+                foo.step = 1
+        def f(x):
+            foo = Foo()
+            foo.step = 2
+            while x > 0:
+                myjitdriver.jit_merge_point(foo=foo, x=x)
+                residual(x, foo)
+                x -= foo.step
+            return foo.step
+        res = self.meta_interp(f, [60])
+        assert res == 1
+        self.check_tree_loop_count(4)   # at least not 2 like before
+
 
 class TestLLtypeGreenFieldsTests(QuasiImmutTests, LLJitMixin):
     pass
diff --git a/pypy/jit/metainterp/test/test_resume.py b/pypy/jit/metainterp/test/test_resume.py
--- a/pypy/jit/metainterp/test/test_resume.py
+++ b/pypy/jit/metainterp/test/test_resume.py
@@ -1,3 +1,4 @@
+from __future__ import with_statement
 import py
 from pypy.rpython.lltypesystem import lltype, llmemory, rffi
 from pypy.jit.metainterp.optimizeopt.optimizer import OptValue
diff --git a/pypy/jit/metainterp/warmspot.py b/pypy/jit/metainterp/warmspot.py
--- a/pypy/jit/metainterp/warmspot.py
+++ b/pypy/jit/metainterp/warmspot.py
@@ -206,7 +206,7 @@
         self.make_enter_functions()
         self.rewrite_jit_merge_points(policy)
 
-        verbose = not self.cpu.translate_support_code
+        verbose = False # not self.cpu.translate_support_code
         self.codewriter.make_jitcodes(verbose=verbose)
         self.rewrite_can_enter_jits()
         self.rewrite_set_param()
diff --git a/pypy/jit/metainterp/warmstate.py b/pypy/jit/metainterp/warmstate.py
--- a/pypy/jit/metainterp/warmstate.py
+++ b/pypy/jit/metainterp/warmstate.py
@@ -178,7 +178,7 @@
         if self.compiled_merge_points_wref is not None:
             for wref in self.compiled_merge_points_wref:
                 looptoken = wref()
-                if looptoken is not None:
+                if looptoken is not None and not looptoken.invalidated:
                     result.append(looptoken)
         return result
 
diff --git a/pypy/module/__builtin__/__init__.py b/pypy/module/__builtin__/__init__.py
--- a/pypy/module/__builtin__/__init__.py
+++ b/pypy/module/__builtin__/__init__.py
@@ -23,6 +23,7 @@
         'map'           : 'app_functional.map',
         'reduce'        : 'app_functional.reduce',
         'filter'        : 'app_functional.filter',
+        'zip'           : 'app_functional.zip',
         'vars'          : 'app_inspect.vars',
         'dir'           : 'app_inspect.dir',
 
@@ -89,7 +90,6 @@
         'enumerate'     : 'functional.W_Enumerate',
         'min'           : 'functional.min',
         'max'           : 'functional.max',
-        'zip'           : 'functional.zip',
         'reversed'      : 'functional.reversed',
         'super'         : 'descriptor.W_Super',
         'staticmethod'  : 'descriptor.StaticMethod',
@@ -119,7 +119,7 @@
            builtin = space.interpclass_w(w_builtin)
            if isinstance(builtin, module.Module):
                return builtin
-       # no builtin! make a default one.  Given them None, at least.
+       # no builtin! make a default one.  Give them None, at least.
        builtin = module.Module(space, None)
        space.setitem(builtin.w_dict, space.wrap('None'), space.w_None)
        return builtin
diff --git a/pypy/module/__builtin__/app_functional.py b/pypy/module/__builtin__/app_functional.py
--- a/pypy/module/__builtin__/app_functional.py
+++ b/pypy/module/__builtin__/app_functional.py
@@ -162,4 +162,21 @@
         item = seq[i]
         if func(item):
             result.append(item)
-    return tuple(result)
\ No newline at end of file
+    return tuple(result)
+
+def zip(*sequences):
+    """zip(seq1 [, seq2 [...]]) -> [(seq1[0], seq2[0] ...), (...)]
+
+Return a list of tuples, where each tuple contains the i-th element
+from each of the argument sequences.  The returned list is truncated
+in length to the length of the shortest argument sequence."""
+    if not sequences:
+        return []
+    result = []
+    iterators = [iter(seq) for seq in sequences]
+    while True:
+        try:
+            items = [next(it) for it in iterators]
+        except StopIteration:
+            return result
+        result.append(tuple(items))
diff --git a/pypy/module/__builtin__/app_io.py b/pypy/module/__builtin__/app_io.py
--- a/pypy/module/__builtin__/app_io.py
+++ b/pypy/module/__builtin__/app_io.py
@@ -27,7 +27,20 @@
     co = compile(source.rstrip()+"\n", filename, 'exec')
     exec co in glob, loc
 
-def raw_input(prompt=None):
+def _write_prompt(stdout, prompt):
+    print >> stdout, prompt,
+    try:
+        flush = stdout.flush
+    except AttributeError:
+        pass
+    else:
+        flush()
+    try:
+        stdout.softspace = 0
+    except (AttributeError, TypeError):
+        pass
+
+def raw_input(prompt=''):
     """raw_input([prompt]) -> string
 
 Read a string from standard input.  The trailing newline is stripped.
@@ -47,18 +60,10 @@
     if (hasattr(sys, '__raw_input__') and
         isinstance(stdin, file)  and stdin.fileno() == 0 and stdin.isatty() and
         isinstance(stdout, file) and stdout.fileno() == 1):
-        if prompt is None:
-            prompt = ''
-        return sys.__raw_input__(prompt)
+        _write_prompt(stdout, '')
+        return sys.__raw_input__(str(prompt))
 
-    if prompt is not None:
-        stdout.write(prompt)
-        try:
-            flush = stdout.flush
-        except AttributeError:
-            pass
-        else:
-            flush()
+    _write_prompt(stdout, prompt)
     line = stdin.readline()
     if not line:    # inputting an empty line gives line == '\n'
         raise EOFError
@@ -66,7 +71,7 @@
         return line[:-1]
     return line
 
-def input(prompt=None):
+def input(prompt=''):
     """Equivalent to eval(raw_input(prompt))."""
     return eval(raw_input(prompt))
 
diff --git a/pypy/module/__builtin__/compiling.py b/pypy/module/__builtin__/compiling.py
--- a/pypy/module/__builtin__/compiling.py
+++ b/pypy/module/__builtin__/compiling.py
@@ -97,6 +97,9 @@
     elif space.is_w(w_locals, space.w_None):
         w_locals = w_globals
 
-    space.builtin.pick_builtin(w_globals)
+    # xxx removed: adding '__builtins__' to the w_globals dict, if there
+    # is none.  This logic was removed as costly (it requires to get at
+    # the gettopframe_nohidden()).  I bet no test fails, and it's a really
+    # obscure case.
 
     return codeobj.exec_code(space, w_globals, w_locals)
diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py
--- a/pypy/module/__builtin__/functional.py
+++ b/pypy/module/__builtin__/functional.py
@@ -201,27 +201,6 @@
     """
     return min_max(space, __args__, "min")
 
- at unwrap_spec(sequences_w="args_w")
-def zip(space, sequences_w):
-    """Return a list of tuples, where the nth tuple contains every nth item of
-    each collection.
-
-    If the collections have different lengths, zip returns a list as long as the
-    shortest collection, ignoring the trailing items in the other collections.
-    """
-    if not sequences_w:
-        return space.newlist([])
-    result_w = []
-    iterators_w = [space.iter(w_seq) for w_seq in sequences_w]
-    while True:
-        try:
-            items_w = [space.next(w_it) for w_it in iterators_w]
-        except OperationError, e:
-            if not e.match(space, space.w_StopIteration):
-                raise
-            return space.newlist(result_w)
-        result_w.append(space.newtuple(items_w))
-
 class W_Enumerate(Wrappable):
 
     def __init__(self, w_iter, w_start):
diff --git a/pypy/module/__builtin__/test/test_rawinput.py b/pypy/module/__builtin__/test/test_rawinput.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/__builtin__/test/test_rawinput.py
@@ -0,0 +1,80 @@
+import autopath
+
+
+class AppTestRawInput():
+
+    def test_input_and_raw_input(self):
+        import sys, StringIO
+        for prompt, expected in [("def:", "abc/ def:/ghi\n"),
+                                 ("", "abc/ /ghi\n"),
+                                 (42, "abc/ 42/ghi\n"),
+                                 (None, "abc/ None/ghi\n"),
+                                 (Ellipsis, "abc/ /ghi\n")]:
+            for inputfn, inputtext, gottext in [
+                    (raw_input, "foo\nbar\n", "foo"),
+                    (input, "40+2\n", 42)]:
+                save = sys.stdin, sys.stdout
+                try:
+                    sys.stdin = StringIO.StringIO(inputtext)
+                    out = sys.stdout = StringIO.StringIO()
+                    print "abc",    # softspace = 1
+                    out.write('/')
+                    if prompt is Ellipsis:
+                        got = inputfn()
+                    else:
+                        got = inputfn(prompt)
+                    out.write('/')
+                    print "ghi"
+                finally:
+                    sys.stdin, sys.stdout = save
+                assert out.getvalue() == expected
+                assert got == gottext
+
+    def test_softspace(self):
+        import sys
+        import StringIO
+        fin = StringIO.StringIO()
+        fout = StringIO.StringIO()
+
+        fin.write("Coconuts\n")
+        fin.seek(0)
+
+        sys_stdin_orig = sys.stdin
+        sys_stdout_orig = sys.stdout
+
+        sys.stdin = fin
+        sys.stdout = fout
+
+        print "test",
+        raw_input("test")
+
+        sys.stdin = sys_stdin_orig
+        sys.stdout = sys_stdout_orig
+
+        fout.seek(0)
+        assert fout.read() == "test test"
+
+    def test_softspace_carryover(self):
+        import sys
+        import StringIO
+        fin = StringIO.StringIO()
+        fout = StringIO.StringIO()
+
+        fin.write("Coconuts\n")
+        fin.seek(0)
+
+        sys_stdin_orig = sys.stdin
+        sys_stdout_orig = sys.stdout
+
+        sys.stdin = fin
+        sys.stdout = fout
+
+        print "test",
+        raw_input("test")
+        print "test",
+
+        sys.stdin = sys_stdin_orig
+        sys.stdout = sys_stdout_orig
+
+        fout.seek(0)
+        assert fout.read() == "test testtest"
diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -19,7 +19,7 @@
 class W_RSocket(Wrappable, RSocket):
     def __del__(self):
         self.clear_all_weakrefs()
-        self.close()
+        RSocket.__del__(self)
 
     def accept_w(self, space):
         """accept() -> (socket object, address info)
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -211,7 +211,9 @@
             return result
 
         def __del__(self):
-            self.clear_all_weakrefs()
+            # note that we don't call clear_all_weakrefs here because
+            # an array with freed buffer is ok to see - it's just empty with 0
+            # length
             self.setlen(0)
 
         def setlen(self, size):
diff --git a/pypy/module/array/test/test_array.py b/pypy/module/array/test/test_array.py
--- a/pypy/module/array/test/test_array.py
+++ b/pypy/module/array/test/test_array.py
@@ -824,6 +824,22 @@
         r = weakref.ref(a)
         assert r() is a
 
+    def test_subclass_del(self):
+        import array, gc, weakref
+        l = []
+        
+        class A(array.array):
+            pass
+
+        a = A('d')
+        a.append(3.0)
+        r = weakref.ref(a, lambda a: l.append(a()))
+        del a
+        gc.collect()
+        assert l
+        assert l[0] is None or len(l[0]) == 0
+
+
 class TestCPythonsOwnArray(BaseArrayTests):
 
     def setup_class(cls):
@@ -844,11 +860,7 @@
         cls.w_tempfile = cls.space.wrap(
             str(py.test.ensuretemp('array').join('tmpfile')))
         cls.w_maxint = cls.space.wrap(sys.maxint)
-
-
-
-
-
+    
     def test_buffer_info(self):
         a = self.array('c', 'Hi!')
         bi = a.buffer_info()
diff --git a/pypy/module/pyexpat/__init__.py b/pypy/module/pyexpat/__init__.py
--- a/pypy/module/pyexpat/__init__.py
+++ b/pypy/module/pyexpat/__init__.py
@@ -4,12 +4,8 @@
 
 class ErrorsModule(MixedModule):
     "Definition of pyexpat.errors module."
-
-    appleveldefs = {
-        }
-
-    interpleveldefs = {
-        }
+    appleveldefs = {}
+    interpleveldefs = {}
 
     def setup_after_space_initialization(self):
         from pypy.module.pyexpat import interp_pyexpat
@@ -18,6 +14,18 @@
                     interp_pyexpat.ErrorString(self.space,
                     getattr(interp_pyexpat, name)))
 
+class ModelModule(MixedModule):
+    "Definition of pyexpat.model module."
+    appleveldefs = {}
+    interpleveldefs = {}
+
+    def setup_after_space_initialization(self):
+        from pypy.module.pyexpat import interp_pyexpat
+        space = self.space
+        for name in interp_pyexpat.xml_model_list:
+            value = getattr(interp_pyexpat, name)
+            space.setattr(self, space.wrap(name), space.wrap(value))
+
 class Module(MixedModule):
     "Python wrapper for Expat parser."
 
@@ -39,6 +47,7 @@
 
     submodules = {
         'errors': ErrorsModule,
+        'model':  ModelModule,
     }
 
     for name in ['XML_PARAM_ENTITY_PARSING_NEVER',
diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py
--- a/pypy/module/pyexpat/interp_pyexpat.py
+++ b/pypy/module/pyexpat/interp_pyexpat.py
@@ -76,6 +76,18 @@
     "XML_ERROR_FINISHED",
     "XML_ERROR_SUSPEND_PE",
     ]
+xml_model_list = [
+    "XML_CTYPE_EMPTY",
+    "XML_CTYPE_ANY",
+    "XML_CTYPE_MIXED",
+    "XML_CTYPE_NAME",
+    "XML_CTYPE_CHOICE",
+    "XML_CTYPE_SEQ",
+    "XML_CQUANT_NONE",
+    "XML_CQUANT_OPT",
+    "XML_CQUANT_REP",
+    "XML_CQUANT_PLUS",
+    ]
 
 class CConfigure:
     _compilation_info_ = eci
@@ -104,6 +116,8 @@
 
     for name in xml_error_list:
         locals()[name] = rffi_platform.ConstantInteger(name)
+    for name in xml_model_list:
+        locals()[name] = rffi_platform.ConstantInteger(name)
 
 for k, v in rffi_platform.configure(CConfigure).items():
     globals()[k] = v
diff --git a/pypy/module/pyexpat/test/test_parser.py b/pypy/module/pyexpat/test/test_parser.py
--- a/pypy/module/pyexpat/test/test_parser.py
+++ b/pypy/module/pyexpat/test/test_parser.py
@@ -131,3 +131,7 @@
                'encoding specified in XML declaration is incorrect')
         assert (pyexpat.errors.XML_ERROR_XML_DECL ==
                 'XML declaration not well-formed')
+
+    def test_model(self):
+        import pyexpat
+        assert isinstance(pyexpat.model.XML_CTYPE_EMPTY, int)
diff --git a/pypy/module/pypyjit/policy.py b/pypy/module/pypyjit/policy.py
--- a/pypy/module/pypyjit/policy.py
+++ b/pypy/module/pypyjit/policy.py
@@ -16,7 +16,8 @@
         if modname in ['pypyjit', 'signal', 'micronumpy', 'math', 'exceptions',
                        'imp', 'sys', 'array', '_ffi', 'itertools', 'operator',
                        'posix', '_socket', '_sre', '_lsprof', '_weakref',
-                       '__pypy__', 'cStringIO', '_collections', 'struct']:
+                       '__pypy__', 'cStringIO', '_collections', 'struct',
+                       'mmap']:
             return True
         return False
 
diff --git a/pypy/module/pypyjit/test_pypy_c/model.py b/pypy/module/pypyjit/test_pypy_c/model.py
--- a/pypy/module/pypyjit/test_pypy_c/model.py
+++ b/pypy/module/pypyjit/test_pypy_c/model.py
@@ -387,8 +387,8 @@
                 return ''
             text = str(py.code.Source(src).deindent().indent())
             lines = text.splitlines(True)
-            if opindex is not None and 0 <= opindex < len(lines):
-                lines[opindex] = lines[opindex].rstrip() + '\t<=====\n'
+            if opindex is not None and 0 <= opindex <= len(lines):
+                lines.insert(opindex, '\n\t===== HERE =====\n')
             return ''.join(lines)
         #
         expected_src = self.preprocess_expected_src(expected_src)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_containers.py b/pypy/module/pypyjit/test_pypy_c/test_containers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_containers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_containers.py
@@ -46,7 +46,7 @@
         assert loop.match_by_id("getitem", """
             i28 = call(ConstClass(ll_dict_lookup__dicttablePtr_objectPtr_Signed), p18, p6, i25, descr=...)
             ...
-            p33 = call(ConstClass(ll_get_value__dicttablePtr_Signed), p18, i28, descr=...)
+            p33 = getinteriorfield_gc(p31, i26, descr=<InteriorFieldDescr <GcPtrFieldDescr dictentry.value .*>>)
             ...
         """)
 
diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -41,7 +41,7 @@
             guard_true(i32, descr=...)
             i34 = int_add(i6, 1)
             --TICK--
-            jump(p0, p1, p2, p3, p4, p5, i34, p7, p8, i9, i10, p11, i12, p13, descr=<Loop4>)
+            jump(p0, p1, p2, p3, p4, p5, i34, p7, p8, i9, i10, p11, i12, p13, descr=...)
         """)
 
     def test_long(self):
@@ -93,7 +93,8 @@
             i46 = call(ConstClass(ll_startswith__rpy_stringPtr_rpy_stringPtr), p28, ConstPtr(ptr45), descr=<BoolCallDescr>)
             guard_false(i46, descr=...)
             p51 = new_with_vtable(21136408)
-            setfield_gc(p51, _, descr=...)    # 6 setfields, but the order is dict-order-dependent
+            setfield_gc(p51, _, descr=...)    # 7 setfields, but the order is dict-order-dependent
+            setfield_gc(p51, _, descr=...)
             setfield_gc(p51, _, descr=...)
             setfield_gc(p51, _, descr=...)
             setfield_gc(p51, _, descr=...)
@@ -106,7 +107,7 @@
             i58 = int_add_ovf(i6, i57)
             guard_no_overflow(descr=...)
             --TICK--
-            jump(p0, p1, p2, p3, p4, p5, i58, i7, descr=<Loop4>)
+            jump(p0, p1, p2, p3, p4, p5, i58, i7, descr=...)
         """)
 
     def test_str_mod(self):
diff --git a/pypy/module/rctime/interp_time.py b/pypy/module/rctime/interp_time.py
--- a/pypy/module/rctime/interp_time.py
+++ b/pypy/module/rctime/interp_time.py
@@ -245,6 +245,9 @@
 if sys.platform != 'win32':
     @unwrap_spec(secs=float)
     def sleep(space, secs):
+        if secs < 0:
+            raise OperationError(space.w_IOError,
+                                 space.wrap("Invalid argument: negative time in sleep"))
         pytime.sleep(secs)
 else:
     from pypy.rlib import rwin32
@@ -265,6 +268,9 @@
                                    OSError(EINTR, "sleep() interrupted"))
     @unwrap_spec(secs=float)
     def sleep(space, secs):
+        if secs < 0:
+            raise OperationError(space.w_IOError,
+                                 space.wrap("Invalid argument: negative time in sleep"))
         # as decreed by Guido, only the main thread can be
         # interrupted.
         main_thread = space.fromcache(State).main_thread
diff --git a/pypy/module/rctime/test/test_rctime.py b/pypy/module/rctime/test/test_rctime.py
--- a/pypy/module/rctime/test/test_rctime.py
+++ b/pypy/module/rctime/test/test_rctime.py
@@ -20,8 +20,9 @@
         import sys
         import os
         raises(TypeError, rctime.sleep, "foo")
-        rctime.sleep(1.2345)
-        
+        rctime.sleep(0.12345)
+        raises(IOError, rctime.sleep, -1.0)
+
     def test_clock(self):
         import time as rctime
         rctime.clock()
diff --git a/pypy/objspace/std/newformat.py b/pypy/objspace/std/newformat.py
--- a/pypy/objspace/std/newformat.py
+++ b/pypy/objspace/std/newformat.py
@@ -120,6 +120,8 @@
             out.append_slice(s, last_literal, end)
             return out.build()
 
+        # This is only ever called if we're already unrolling _do_build_string
+        @jit.unroll_safe
         def _parse_field(self, start, end):
             s = self.template
             # Find ":" or "!"
@@ -149,6 +151,7 @@
                 i += 1
             return s[start:end], None, end
 
+        @jit.unroll_safe
         def _get_argument(self, name):
             # First, find the argument.
             space = self.space
@@ -207,6 +210,7 @@
                     raise OperationError(space.w_IndexError, w_msg)
             return self._resolve_lookups(w_arg, name, i, end)
 
+        @jit.unroll_safe
         def _resolve_lookups(self, w_obj, name, start, end):
             # Resolve attribute and item lookups.
             space = self.space
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -83,11 +83,12 @@
         if self.config.objspace.std.withtproxy:
             transparent.setup(self)
 
+        interplevel_classes = {}
         for type, classes in self.model.typeorder.iteritems():
-            if len(classes) >= 3:
+            if len(classes) >= 3: # XXX what does this 3 mean??!
                 # W_Root, AnyXxx and actual object
-                self.gettypefor(type).interplevel_cls = classes[0][0]
-
+                interplevel_classes[self.gettypefor(type)] = classes[0][0]
+        self._interplevel_classes = interplevel_classes
 
     def get_builtin_types(self):
         return self.builtin_types
@@ -579,7 +580,7 @@
             raise OperationError(self.w_TypeError,
                                  self.wrap("need type object"))
         if is_annotation_constant(w_type):
-            cls = w_type.interplevel_cls
+            cls = self._get_interplevel_cls(w_type)
             if cls is not None:
                 assert w_inst is not None
                 if isinstance(w_inst, cls):
@@ -589,3 +590,9 @@
     @specialize.arg_or_var(2)
     def isinstance_w(space, w_inst, w_type):
         return space._type_isinstance(w_inst, w_type)
+
+    @specialize.memo()
+    def _get_interplevel_cls(self, w_type):
+        if not hasattr(self, "_interplevel_classes"):
+            return None # before running initialize
+        return self._interplevel_classes.get(w_type, None)
diff --git a/pypy/objspace/std/smallintobject.py b/pypy/objspace/std/smallintobject.py
--- a/pypy/objspace/std/smallintobject.py
+++ b/pypy/objspace/std/smallintobject.py
@@ -12,6 +12,7 @@
 from pypy.rlib.rbigint import rbigint
 from pypy.rlib.rarithmetic import r_uint
 from pypy.tool.sourcetools import func_with_new_name
+from pypy.objspace.std.inttype import wrapint
 
 class W_SmallIntObject(W_Object, UnboxedValue):
     __slots__ = 'intval'
@@ -48,14 +49,36 @@
 def delegate_SmallInt2Complex(space, w_small):
     return space.newcomplex(float(w_small.intval), 0.0)
 
+def add__SmallInt_SmallInt(space, w_a, w_b):
+    return wrapint(space, w_a.intval + w_b.intval) # cannot overflow
+
+def sub__SmallInt_SmallInt(space, w_a, w_b):
+    return wrapint(space, w_a.intval - w_b.intval) # cannot overflow
+
+def floordiv__SmallInt_SmallInt(space, w_a, w_b):
+    return wrapint(space, w_a.intval // w_b.intval) # cannot overflow
+
+div__SmallInt_SmallInt = floordiv__SmallInt_SmallInt
+
+def mod__SmallInt_SmallInt(space, w_a, w_b):
+    return wrapint(space, w_a.intval % w_b.intval) # cannot overflow
+
+def divmod__SmallInt_SmallInt(space, w_a, w_b):
+    w = wrapint(space, w_a.intval // w_b.intval) # cannot overflow
+    z = wrapint(space, w_a.intval % w_b.intval)
+    return space.newtuple([w, z])
+
 def copy_multimethods(ns):
     """Copy integer multimethods for small int."""
     for name, func in intobject.__dict__.iteritems():
         if "__Int" in name:
             new_name = name.replace("Int", "SmallInt")
-            # Copy the function, so the annotator specializes it for
-            # W_SmallIntObject.
-            ns[new_name] = func_with_new_name(func, new_name)
+            if new_name not in ns:
+                # Copy the function, so the annotator specializes it for
+                # W_SmallIntObject.
+                ns[new_name] = func = func_with_new_name(func, new_name, globals=ns)
+        else:
+            ns[name] = func
     ns["get_integer"] = ns["pos__SmallInt"] = ns["int__SmallInt"]
     ns["get_negint"] = ns["neg__SmallInt"]
 
diff --git a/pypy/objspace/std/strutil.py b/pypy/objspace/std/strutil.py
--- a/pypy/objspace/std/strutil.py
+++ b/pypy/objspace/std/strutil.py
@@ -35,7 +35,7 @@
 
     def error(self):
         raise ParseStringError("invalid literal for %s() with base %d: '%s'" %
-                               (self.fname, self.base, self.literal))
+                               (self.fname, self.original_base, self.literal))
 
     def __init__(self, s, literal, base, fname):
         self.literal = literal
@@ -47,7 +47,8 @@
         elif s.startswith('+'):
             s = strip_spaces(s[1:])
         self.sign = sign
-        
+        self.original_base = base
+
         if base == 0:
             if s.startswith('0x') or s.startswith('0X'):
                 base = 16
diff --git a/pypy/objspace/std/test/test_obj.py b/pypy/objspace/std/test/test_obj.py
--- a/pypy/objspace/std/test/test_obj.py
+++ b/pypy/objspace/std/test/test_obj.py
@@ -102,3 +102,11 @@
             def __repr__(self):
                 return 123456
         assert A().__str__() == 123456
+
+def test_isinstance_shortcut():
+    from pypy.objspace.std import objspace
+    space = objspace.StdObjSpace()
+    w_a = space.wrap("a")
+    space.type = None
+    space.isinstance_w(w_a, space.w_str) # does not crash
+
diff --git a/pypy/objspace/std/test/test_strutil.py b/pypy/objspace/std/test/test_strutil.py
--- a/pypy/objspace/std/test/test_strutil.py
+++ b/pypy/objspace/std/test/test_strutil.py
@@ -89,6 +89,8 @@
 
         exc = raises(ParseStringError, string_to_int, '')
         assert exc.value.msg == "invalid literal for int() with base 10: ''"
+        exc = raises(ParseStringError, string_to_int, '', 0)
+        assert exc.value.msg == "invalid literal for int() with base 0: ''"
 
     def test_string_to_int_overflow(self):
         import sys
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -102,7 +102,6 @@
                           'instancetypedef',
                           'terminator',
                           '_version_tag?',
-                          'interplevel_cls',
                           ]
 
     # for config.objspace.std.getattributeshortcut
@@ -117,9 +116,6 @@
     # of the __new__ is an instance of the type
     w_bltin_new = None
 
-    interplevel_cls = None # not None for prebuilt instances of
-                           # interpreter-level types
-
     @dont_look_inside
     def __init__(w_self, space, name, bases_w, dict_w,
                  overridetypedef=None):
diff --git a/pypy/pytest.ini b/pypy/pytest.ini
--- a/pypy/pytest.ini
+++ b/pypy/pytest.ini
@@ -1,2 +1,2 @@
 [pytest]
-addopts = --assertmode=old
\ No newline at end of file
+addopts = --assertmode=old -rf
diff --git a/pypy/rlib/_rsocket_rffi.py b/pypy/rlib/_rsocket_rffi.py
--- a/pypy/rlib/_rsocket_rffi.py
+++ b/pypy/rlib/_rsocket_rffi.py
@@ -16,6 +16,7 @@
 _MINGW = target_platform.name == "mingw32"
 _SOLARIS = sys.platform == "sunos5"
 _MACOSX = sys.platform == "darwin"
+_HAS_AF_PACKET = sys.platform.startswith('linux')   # only Linux for now
 
 if _POSIX:
     includes = ('sys/types.h',
@@ -34,11 +35,12 @@
                 'stdint.h', 
                 'errno.h',
                 )
+    if _HAS_AF_PACKET:
+        includes += ('netpacket/packet.h',
+                     'sys/ioctl.h',
+                     'net/if.h')
 
-    cond_includes = [('AF_NETLINK', 'linux/netlink.h'),
-                     ('AF_PACKET', 'netpacket/packet.h'),
-                     ('AF_PACKET', 'sys/ioctl.h'),
-                     ('AF_PACKET', 'net/if.h')]
+    cond_includes = [('AF_NETLINK', 'linux/netlink.h')]
     
     libraries = ()
     calling_conv = 'c'
@@ -320,18 +322,18 @@
                                              ('events', rffi.SHORT),
                                              ('revents', rffi.SHORT)])
 
-    CConfig.sockaddr_ll = platform.Struct('struct sockaddr_ll',
+    if _HAS_AF_PACKET:
+        CConfig.sockaddr_ll = platform.Struct('struct sockaddr_ll',
                               [('sll_ifindex', rffi.INT),
                                ('sll_protocol', rffi.INT),
                                ('sll_pkttype', rffi.INT),
                                ('sll_hatype', rffi.INT),
                                ('sll_addr', rffi.CFixedArray(rffi.CHAR, 8)),
-                               ('sll_halen', rffi.INT)],
-                              ifdef='AF_PACKET')
+                               ('sll_halen', rffi.INT)])
 
-    CConfig.ifreq = platform.Struct('struct ifreq', [('ifr_ifindex', rffi.INT),
-                                 ('ifr_name', rffi.CFixedArray(rffi.CHAR, 8))],
-                                    ifdef='AF_PACKET')
+        CConfig.ifreq = platform.Struct('struct ifreq',
+                                [('ifr_ifindex', rffi.INT),
+                                 ('ifr_name', rffi.CFixedArray(rffi.CHAR, 8))])
 
 if _WIN32:
     CConfig.WSAEVENT = platform.SimpleType('WSAEVENT', rffi.VOIDP)
@@ -386,6 +388,8 @@
         constants[name] = value
     else:
         constants[name] = default
+if not _HAS_AF_PACKET and 'AF_PACKET' in constants:
+    del constants['AF_PACKET']
 
 constants['has_ipv6'] = True # This is a configuration option in CPython
 for name, value in constants.items():
@@ -439,21 +443,14 @@
 if _POSIX:
     nfds_t = cConfig.nfds_t
     pollfd = cConfig.pollfd
-    if cConfig.sockaddr_ll is not None:
+    if _HAS_AF_PACKET:
         sockaddr_ll = cConfig.sockaddr_ll
-    ifreq = cConfig.ifreq
+        ifreq = cConfig.ifreq
 if WIN32:
     WSAEVENT = cConfig.WSAEVENT
     WSANETWORKEVENTS = cConfig.WSANETWORKEVENTS
 timeval = cConfig.timeval
 
-#if _POSIX:
-#    includes = list(includes)
-#    for _name, _header in cond_includes:
-#        if getattr(cConfig, _name) is not None:
-#            includes.append(_header)
-#    eci = ExternalCompilationInfo(includes=includes, libraries=libraries,
-#                                  separate_module_sources=sources)
 
 def external(name, args, result, **kwds):
     return rffi.llexternal(name, args, result, compilation_info=eci,
@@ -544,7 +541,7 @@
     socketpair_t = rffi.CArray(socketfd_type)
     socketpair = external('socketpair', [rffi.INT, rffi.INT, rffi.INT,
                           lltype.Ptr(socketpair_t)], rffi.INT)
-    if ifreq is not None:
+    if _HAS_AF_PACKET:
         ioctl = external('ioctl', [socketfd_type, rffi.INT, lltype.Ptr(ifreq)],
                          rffi.INT)
 
diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py
--- a/pypy/rlib/jit.py
+++ b/pypy/rlib/jit.py
@@ -8,6 +8,8 @@
 from pypy.rpython.extregistry import ExtRegistryEntry
 from pypy.tool.sourcetools import func_with_new_name
 
+DEBUG_ELIDABLE_FUNCTIONS = False
+
 
 def elidable(func):
     """ Decorate a function as "trace-elidable". This means precisely that:
@@ -24,6 +26,18 @@
     If a particular call to this function ends up raising an exception, then it
     is handled like a normal function call (this decorator is ignored).
     """
+    if DEBUG_ELIDABLE_FUNCTIONS:
+        cache = {}
+        oldfunc = func
+        def func(*args):
+            result = oldfunc(*args)    # if it raises, no caching
+            try:
+                oldresult = cache.setdefault(args, result)
+            except TypeError:
+                pass           # unhashable args
+            else:
+                assert oldresult == result
+            return result
     func._elidable_function_ = True
     return func
 
diff --git a/pypy/rlib/rerased.py b/pypy/rlib/rerased.py
--- a/pypy/rlib/rerased.py
+++ b/pypy/rlib/rerased.py
@@ -135,6 +135,8 @@
     _about_ = erase_int
 
     def compute_result_annotation(self, s_obj):
+        config = self.bookkeeper.annotator.translator.config
+        assert config.translation.taggedpointers, "need to enable tagged pointers to use erase_int"
         assert annmodel.SomeInteger().contains(s_obj)
         return SomeErased()
 
@@ -228,6 +230,8 @@
 
     def convert_const(self, value):
         if value._identity is _identity_for_ints:
+            config = self.rtyper.annotator.translator.config
+            assert config.translation.taggedpointers, "need to enable tagged pointers to use erase_int"
             return lltype.cast_int_to_ptr(self.lowleveltype, value._x * 2 + 1)
         bk = self.rtyper.annotator.bookkeeper
         s_obj = value._identity.get_input_annotation(bk)
diff --git a/pypy/rlib/rgc.py b/pypy/rlib/rgc.py
--- a/pypy/rlib/rgc.py
+++ b/pypy/rlib/rgc.py
@@ -214,6 +214,10 @@
     func._gc_no_collect_ = True
     return func
 
+def is_light_finalizer(func):
+    func._is_light_finalizer_ = True
+    return func
+
 # ____________________________________________________________
 
 def get_rpy_roots():
diff --git a/pypy/rlib/rmmap.py b/pypy/rlib/rmmap.py
--- a/pypy/rlib/rmmap.py
+++ b/pypy/rlib/rmmap.py
@@ -292,6 +292,9 @@
         elif _POSIX:
             self.closed = True
             if self.fd != -1:
+                # XXX this is buggy - raising in an RPython del is not a good
+                #     idea, we should swallow the exception or ignore the
+                #     underlaying close error code
                 os.close(self.fd)
                 self.fd = -1
             if self.size > 0:
diff --git a/pypy/rlib/rsocket.py b/pypy/rlib/rsocket.py
--- a/pypy/rlib/rsocket.py
+++ b/pypy/rlib/rsocket.py
@@ -8,6 +8,7 @@
 # Known missing features:
 #
 #   - address families other than AF_INET, AF_INET6, AF_UNIX, AF_PACKET
+#   - AF_PACKET is only supported on Linux
 #   - methods makefile(),
 #   - SSL
 #
@@ -55,6 +56,7 @@
 
 
 _FAMILIES = {}
+
 class Address(object):
     """The base class for RPython-level objects representing addresses.
     Fields:  addr    - a _c.sockaddr_ptr (memory owned by the Address instance)
@@ -76,9 +78,8 @@
         self.addrlen = addrlen
 
     def __del__(self):
-        addr = self.addr_p
-        if addr:
-            lltype.free(addr, flavor='raw')
+        if self.addr_p:
+            lltype.free(self.addr_p, flavor='raw')
 
     def setdata(self, addr, addrlen):
         # initialize self.addr and self.addrlen.  'addr' can be a different
@@ -612,7 +613,10 @@
         self.timeout = defaults.timeout
         
     def __del__(self):
-        self.close()
+        fd = self.fd
+        if fd != _c.INVALID_SOCKET:
+            self.fd = _c.INVALID_SOCKET
+            _c.socketclose(fd)
 
     if hasattr(_c, 'fcntl'):
         def _setblocking(self, block):
diff --git a/pypy/rlib/test/test_rerased.py b/pypy/rlib/test/test_rerased.py
--- a/pypy/rlib/test/test_rerased.py
+++ b/pypy/rlib/test/test_rerased.py
@@ -10,6 +10,13 @@
 
 from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin
 
+def make_annotator():
+    a = RPythonAnnotator()
+    a.translator.config.translation.taggedpointers = True
+    return a
+
+
+
 class X(object):
     pass
 
@@ -55,7 +62,7 @@
 def test_annotate_1():
     def f():
         return eraseX(X())
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [])
     assert isinstance(s, SomeErased)
 
@@ -66,7 +73,7 @@
         #assert not is_integer(e)
         x2 = uneraseX(e)
         return x2
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [])
     assert isinstance(s, annmodel.SomeInstance)
     assert s.classdef == a.bookkeeper.getuniqueclassdef(X)
@@ -77,7 +84,7 @@
         #assert is_integer(e)
         x2 = unerase_int(e)
         return x2
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [])
     assert isinstance(s, annmodel.SomeInteger)
 
@@ -105,7 +112,7 @@
         x = make(n)
         return check(x, n)
     #
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [int])
     assert isinstance(s, annmodel.SomeInteger)
 
@@ -135,7 +142,7 @@
             else:
                 return inst
     #
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [])
     assert isinstance(s, annmodel.SomeInstance)
     assert s.classdef == a.bookkeeper.getuniqueclassdef(A)
@@ -155,7 +162,7 @@
             e = e2
         return unerase(e)
     #
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [int])
     assert isinstance(s, annmodel.SomeInstance)
     assert s.classdef == a.bookkeeper.getuniqueclassdef(X)
@@ -165,11 +172,14 @@
     e1 = erase_int(42)
     def f(i):
         return unerase_int(e1)
-    a = RPythonAnnotator()
+    a = make_annotator()
     s = a.build_types(f, [int])
     assert isinstance(s, annmodel.SomeInteger)
 
 class BaseTestRErased(BaseRtypingTest):
+    def interpret(self, *args, **kwargs):
+        kwargs["taggedpointers"] = True
+        return BaseRtypingTest.interpret(self, *args, **kwargs)
     def test_rtype_1(self):
         def f():
             return eraseX(X())
diff --git a/pypy/rpython/llinterp.py b/pypy/rpython/llinterp.py
--- a/pypy/rpython/llinterp.py
+++ b/pypy/rpython/llinterp.py
@@ -1095,13 +1095,6 @@
             assert y >= 0
         return self.op_int_add_ovf(x, y)
 
-    def op_cast_float_to_int(self, f):
-        assert type(f) is float
-        try:
-            return ovfcheck(int(f))
-        except OverflowError:
-            self.make_llexception()
-
     def op_int_is_true(self, x):
         # special case
         if type(x) is CDefinedIntSymbolic:
diff --git a/pypy/rpython/lltypesystem/ll2ctypes.py b/pypy/rpython/lltypesystem/ll2ctypes.py
--- a/pypy/rpython/lltypesystem/ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/ll2ctypes.py
@@ -15,12 +15,11 @@
     load_library_kwargs = {}
 
 import os
-from pypy import conftest
 from pypy.rpython.lltypesystem import lltype, llmemory
 from pypy.rpython.extfunc import ExtRegistryEntry
 from pypy.rlib.objectmodel import Symbolic, ComputedIntSymbolic
 from pypy.tool.uid import fixid
-from pypy.rlib.rarithmetic import r_uint, r_singlefloat, r_longfloat, base_int, intmask
+from pypy.rlib.rarithmetic import r_singlefloat, r_longfloat, base_int, intmask
 from pypy.annotation import model as annmodel
 from pypy.rpython.llinterp import LLInterpreter, LLException
 from pypy.rpython.lltypesystem.rclass import OBJECT, OBJECT_VTABLE
@@ -531,6 +530,10 @@
     def __str__(self):
         return repr(self)
 
+    def _setparentstructure(self, parent, parentindex):
+        super(_parentable_mixin, self)._setparentstructure(parent, parentindex)
+        self._keepparent = parent   # always keep a strong ref
+
 class _struct_mixin(_parentable_mixin):
     """Mixin added to _struct containers when they become ctypes-based."""
     __slots__ = ()
@@ -566,7 +569,10 @@
         return 0, sys.maxint
 
     def getitem(self, index, uninitialized_ok=False):
-        return self._storage.contents._getitem(index, boundscheck=False)
+        res = self._storage.contents._getitem(index, boundscheck=False)
+        if isinstance(self._TYPE.OF, lltype.ContainerType):
+            res._obj._setparentstructure(self, index)
+        return res
 
     def setitem(self, index, value):
         self._storage.contents._setitem(index, value, boundscheck=False)
@@ -1279,6 +1285,8 @@
             self.intval = intmask(void_p.value)
 
     def __eq__(self, other):
+        if not other:
+            return self.intval == 0
         if isinstance(other, _llgcopaque):
             return self.intval == other.intval
         storage = object()
diff --git a/pypy/rpython/lltypesystem/lloperation.py b/pypy/rpython/lltypesystem/lloperation.py
--- a/pypy/rpython/lltypesystem/lloperation.py
+++ b/pypy/rpython/lltypesystem/lloperation.py
@@ -343,8 +343,8 @@
     'cast_uint_to_float':   LLOp(canfold=True),
     'cast_longlong_to_float' :LLOp(canfold=True),
     'cast_ulonglong_to_float':LLOp(canfold=True),
-    'cast_float_to_int':    LLOp(canraise=(OverflowError,), tryfold=True),
-    'cast_float_to_uint':   LLOp(canfold=True),    # XXX need OverflowError?
+    'cast_float_to_int':    LLOp(canfold=True),
+    'cast_float_to_uint':   LLOp(canfold=True),
     'cast_float_to_longlong' :LLOp(canfold=True),
     'cast_float_to_ulonglong':LLOp(canfold=True),
     'truncate_longlong_to_int':LLOp(canfold=True),
diff --git a/pypy/rpython/lltypesystem/lltype.py b/pypy/rpython/lltypesystem/lltype.py
--- a/pypy/rpython/lltypesystem/lltype.py
+++ b/pypy/rpython/lltypesystem/lltype.py
@@ -1522,7 +1522,7 @@
             and parentindex in (self._parent_type._names[0], 0)
             and self._TYPE._gckind == typeOf(parent)._gckind):
             # keep strong reference to parent, we share the same allocation
-            self._keepparent = parent 
+            self._keepparent = parent
 
     def _parentstructure(self, check=True):
         if self._wrparent is not None:
@@ -1713,6 +1713,7 @@
         return v
 
     def setitem(self, index, value):
+        assert typeOf(value) == self._TYPE.OF
         self.items[index] = value
 
 assert not '__dict__' in dir(_array)
@@ -1730,7 +1731,8 @@
         # Keep the parent array alive, we share the same allocation.
         # Don't do it if we are inside a GC object, though -- it's someone
         # else's job to keep the GC object alive
-        if typeOf(top_container(parent))._gckind == 'raw':
+        if (typeOf(top_container(parent))._gckind == 'raw' or
+            hasattr(top_container(parent)._storage, 'contents')):  # ll2ctypes
             self._keepparent = parent
 
     def __str__(self):
diff --git a/pypy/rpython/lltypesystem/opimpl.py b/pypy/rpython/lltypesystem/opimpl.py
--- a/pypy/rpython/lltypesystem/opimpl.py
+++ b/pypy/rpython/lltypesystem/opimpl.py
@@ -355,6 +355,10 @@
     assert type(b) is bool
     return float(b)
 
+def op_cast_float_to_int(f):
+    assert type(f) is float
+    return intmask(int(f))
+
 def op_cast_float_to_uint(f):
     assert type(f) is float
     return r_uint(long(f))
diff --git a/pypy/rpython/lltypesystem/rbuilder.py b/pypy/rpython/lltypesystem/rbuilder.py
--- a/pypy/rpython/lltypesystem/rbuilder.py
+++ b/pypy/rpython/lltypesystem/rbuilder.py
@@ -29,7 +29,7 @@
         except OverflowError:
             raise MemoryError
         newbuf = mallocfn(new_allocated)
-        copycontentsfn(ll_builder.buf, newbuf, 0, 0, ll_builder.allocated)
+        copycontentsfn(ll_builder.buf, newbuf, 0, 0, ll_builder.used)
         ll_builder.buf = newbuf
         ll_builder.allocated = new_allocated
     return func_with_new_name(stringbuilder_grow, name)
@@ -56,7 +56,7 @@
 class BaseStringBuilderRepr(AbstractStringBuilderRepr):
     def empty(self):
         return nullptr(self.lowleveltype.TO)
-    
+
     @classmethod
     def ll_new(cls, init_size):
         if init_size < 0 or init_size > MAX:
diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py
--- a/pypy/rpython/lltypesystem/rdict.py
+++ b/pypy/rpython/lltypesystem/rdict.py
@@ -1,16 +1,14 @@
 from pypy.tool.pairtype import pairtype
-from pypy.annotation import model as annmodel
 from pypy.objspace.flow.model import Constant
-from pypy.rpython.rdict import AbstractDictRepr, AbstractDictIteratorRepr,\
-     rtype_newdict
+from pypy.rpython.rdict import (AbstractDictRepr, AbstractDictIteratorRepr,
+     rtype_newdict)
 from pypy.rpython.lltypesystem import lltype
+from pypy.rlib import objectmodel, jit
 from pypy.rlib.rarithmetic import r_uint, intmask, LONG_BIT
-from pypy.rlib.objectmodel import hlinvoke
-from pypy.rpython import robject
-from pypy.rlib import objectmodel, jit
 from pypy.rpython import rmodel
 from pypy.rpython.error import TyperError
 
+
 HIGHEST_BIT = intmask(1 << (LONG_BIT - 1))
 MASK = intmask(HIGHEST_BIT - 1)
 
@@ -417,17 +415,16 @@
     ENTRIES = lltype.typeOf(entries).TO
     return ENTRIES.fasthashfn(entries[i].key)
 
- at jit.dont_look_inside
 def ll_get_value(d, i):
     return d.entries[i].value
 
 def ll_keyhash_custom(d, key):
     DICT = lltype.typeOf(d).TO
-    return hlinvoke(DICT.r_rdict_hashfn, d.fnkeyhash, key)
+    return objectmodel.hlinvoke(DICT.r_rdict_hashfn, d.fnkeyhash, key)
 
 def ll_keyeq_custom(d, key1, key2):
     DICT = lltype.typeOf(d).TO
-    return hlinvoke(DICT.r_rdict_eqfn, d.fnkeyeq, key1, key2)
+    return objectmodel.hlinvoke(DICT.r_rdict_eqfn, d.fnkeyeq, key1, key2)
 
 def ll_dict_len(d):
     return d.num_items
@@ -448,6 +445,8 @@
     i = ll_dict_lookup(d, key, hash)
     return _ll_dict_setitem_lookup_done(d, key, value, hash, i)
 
+# Leaving as dont_look_inside ATM, it has a few branches which could lead to
+# many bridges if we don't consider their possible frequency.
 @jit.dont_look_inside
 def _ll_dict_setitem_lookup_done(d, key, value, hash, i):
     valid = (i & HIGHEST_BIT) == 0
@@ -492,6 +491,8 @@
         raise KeyError
     _ll_dict_del(d, i)
 
+# XXX: Move the size checking and resize into a single call which is opauqe to
+# the JIT to avoid extra branches.
 @jit.dont_look_inside
 def _ll_dict_del(d, i):
     d.entries.mark_deleted(i)
@@ -532,6 +533,7 @@
 # ------- a port of CPython's dictobject.c's lookdict implementation -------
 PERTURB_SHIFT = 5
 
+ at jit.dont_look_inside
 def ll_dict_lookup(d, key, hash):
     entries = d.entries
     ENTRIES = lltype.typeOf(entries).TO
@@ -621,7 +623,6 @@
     d.num_items = 0
     d.resize_counter = DICT_INITSIZE * 2
     return d
-ll_newdict.oopspec = 'newdict()'
 
 def ll_newdict_size(DICT, length_estimate):
     length_estimate = (length_estimate // 2) * 3
@@ -633,7 +634,6 @@
     d.num_items = 0
     d.resize_counter = n * 2
     return d
-ll_newdict_size.oopspec = 'newdict()'
 
 # pypy.rpython.memory.lldict uses a dict based on Struct and Array
 # instead of GcStruct and GcArray, which is done by using different
@@ -866,7 +866,6 @@
         global_popitem_index.nextindex = base + counter
     return i
 
- at jit.dont_look_inside
 def ll_popitem(ELEM, dic):
     i = _ll_getnextitem(dic)
     entry = dic.entries[i]
diff --git a/pypy/rpython/lltypesystem/test/test_ll2ctypes.py b/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
--- a/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
+++ b/pypy/rpython/lltypesystem/test/test_ll2ctypes.py
@@ -8,6 +8,7 @@
 from pypy.rpython.lltypesystem.ll2ctypes import uninitialized2ctypes
 from pypy.rpython.lltypesystem.ll2ctypes import ALLOCATED, force_cast
 from pypy.rpython.lltypesystem.ll2ctypes import cast_adr_to_int, get_ctypes_type
+from pypy.rpython.lltypesystem.ll2ctypes import _llgcopaque
 from pypy.rpython.annlowlevel import llhelper
 from pypy.rlib import rposix
 from pypy.translator.tool.cbuild import ExternalCompilationInfo
@@ -1349,6 +1350,11 @@
         round = ctypes2lltype(llmemory.GCREF, lltype2ctypes(opaque.hide()))
         assert Opaque.show(round) is opaque
 
+    def test_array_of_structs(self):
+        A = lltype.GcArray(lltype.Struct('x', ('v', lltype.Signed)))
+        a = lltype.malloc(A, 5)
+        a2 = ctypes2lltype(lltype.Ptr(A), lltype2ctypes(a))
+        assert a2._obj.getitem(0)._obj._parentstructure() is a2._obj
 
 class TestPlatform(object):
     def test_lib_on_libpaths(self):
@@ -1390,3 +1396,7 @@
         f = rffi.llexternal('f', [rffi.INT, rffi.INT], rffi.INT,
                             compilation_info=eci)
         assert f(3, 4) == 7
+
+    def test_llgcopaque_eq(self):
+        assert _llgcopaque(1) != None
+        assert _llgcopaque(0) == None
diff --git a/pypy/rpython/memory/gc/base.py b/pypy/rpython/memory/gc/base.py
--- a/pypy/rpython/memory/gc/base.py
+++ b/pypy/rpython/memory/gc/base.py
@@ -1,4 +1,5 @@
-from pypy.rpython.lltypesystem import lltype, llmemory, llarena
+from pypy.rpython.lltypesystem import lltype, llmemory, llarena, rffi
+from pypy.rpython.lltypesystem.lloperation import llop
 from pypy.rlib.debug import ll_assert
 from pypy.rpython.memory.gcheader import GCHeaderBuilder
 from pypy.rpython.memory.support import DEFAULT_CHUNK_SIZE
@@ -62,6 +63,7 @@
     def set_query_functions(self, is_varsize, has_gcptr_in_varsize,
                             is_gcarrayofgcptr,
                             getfinalizer,
+                            getlightfinalizer,
                             offsets_to_gc_pointers,
                             fixed_size, varsize_item_sizes,
                             varsize_offset_to_variable_part,
@@ -74,6 +76,7 @@
                             get_custom_trace,
                             fast_path_tracing):
         self.getfinalizer = getfinalizer
+        self.getlightfinalizer = getlightfinalizer
         self.is_varsize = is_varsize
         self.has_gcptr_in_varsize = has_gcptr_in_varsize
         self.is_gcarrayofgcptr = is_gcarrayofgcptr
@@ -139,6 +142,7 @@
 
         size = self.fixed_size(typeid)
         needs_finalizer = bool(self.getfinalizer(typeid))
+        finalizer_is_light = bool(self.getlightfinalizer(typeid))
         contains_weakptr = self.weakpointer_offset(typeid) >= 0
         assert not (needs_finalizer and contains_weakptr)
         if self.is_varsize(typeid):
@@ -158,6 +162,7 @@
             else:
                 malloc_fixedsize = self.malloc_fixedsize
             ref = malloc_fixedsize(typeid, size, needs_finalizer,
+                                   finalizer_is_light,
                                    contains_weakptr)
         # lots of cast and reverse-cast around...
         return llmemory.cast_ptr_to_adr(ref)
diff --git a/pypy/rpython/memory/gc/generation.py b/pypy/rpython/memory/gc/generation.py
--- a/pypy/rpython/memory/gc/generation.py
+++ b/pypy/rpython/memory/gc/generation.py
@@ -167,7 +167,9 @@
         return self.nursery <= addr < self.nursery_top
 
     def malloc_fixedsize_clear(self, typeid, size,
-                               has_finalizer=False, contains_weakptr=False):
+                               has_finalizer=False,
+                               is_finalizer_light=False,
+                               contains_weakptr=False):
         if (has_finalizer or
             (raw_malloc_usage(size) > self.lb_young_fixedsize and
              raw_malloc_usage(size) > self.largest_young_fixedsize)):
@@ -179,6 +181,7 @@
             # "non-simple" case or object too big: don't use the nursery
             return SemiSpaceGC.malloc_fixedsize_clear(self, typeid, size,
                                                       has_finalizer,
+                                                      is_finalizer_light,
                                                       contains_weakptr)
         size_gc_header = self.gcheaderbuilder.size_gc_header
         totalsize = size_gc_header + size
diff --git a/pypy/rpython/memory/gc/marksweep.py b/pypy/rpython/memory/gc/marksweep.py
--- a/pypy/rpython/memory/gc/marksweep.py
+++ b/pypy/rpython/memory/gc/marksweep.py
@@ -93,7 +93,8 @@
         pass
 
     def malloc_fixedsize(self, typeid16, size,
-                         has_finalizer=False, contains_weakptr=False):
+                         has_finalizer=False, is_finalizer_light=False,
+                         contains_weakptr=False):
         self.maybe_collect()
         size_gc_header = self.gcheaderbuilder.size_gc_header
         try:
@@ -128,7 +129,9 @@
     malloc_fixedsize._dont_inline_ = True
 
     def malloc_fixedsize_clear(self, typeid16, size,
-                               has_finalizer=False, contains_weakptr=False):
+                               has_finalizer=False,
+                               is_finalizer_light=False,
+                               contains_weakptr=False):
         self.maybe_collect()
         size_gc_header = self.gcheaderbuilder.size_gc_header
         try:
diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -290,6 +290,8 @@
         #
         # A list of all objects with finalizers (these are never young).
         self.objects_with_finalizers = self.AddressDeque()
+        self.young_objects_with_light_finalizers = self.AddressStack()
+        self.old_objects_with_light_finalizers = self.AddressStack()
         #
         # Two lists of the objects with weakrefs.  No weakref can be an
         # old object weakly pointing to a young object: indeed, weakrefs
@@ -457,14 +459,16 @@
 
 
     def malloc_fixedsize_clear(self, typeid, size,
-                               needs_finalizer=False, contains_weakptr=False):
+                               needs_finalizer=False,
+                               is_finalizer_light=False,
+                               contains_weakptr=False):
         size_gc_header = self.gcheaderbuilder.size_gc_header
         totalsize = size_gc_header + size
         rawtotalsize = raw_malloc_usage(totalsize)
         #
         # If the object needs a finalizer, ask for a rawmalloc.
         # The following check should be constant-folded.
-        if needs_finalizer:
+        if needs_finalizer and not is_finalizer_light:
             ll_assert(not contains_weakptr,
                      "'needs_finalizer' and 'contains_weakptr' both specified")
             obj = self.external_malloc(typeid, 0, can_make_young=False)
@@ -494,13 +498,14 @@
             #
             # Build the object.
             llarena.arena_reserve(result, totalsize)
+            obj = result + size_gc_header
+            if is_finalizer_light:
+                self.young_objects_with_light_finalizers.append(obj)
             self.init_gc_object(result, typeid, flags=0)
             #
             # If it is a weakref, record it (check constant-folded).
             if contains_weakptr:
-                self.young_objects_with_weakrefs.append(result+size_gc_header)
-            #
-            obj = result + size_gc_header
+                self.young_objects_with_weakrefs.append(obj)
         #
         return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
 
@@ -1264,6 +1269,8 @@
         # weakrefs' targets.
         if self.young_objects_with_weakrefs.non_empty():
             self.invalidate_young_weakrefs()
+        if self.young_objects_with_light_finalizers.non_empty():
+            self.deal_with_young_objects_with_finalizers()
         #
         # Clear this mapping.
         if self.nursery_objects_shadows.length() > 0:
@@ -1292,10 +1299,12 @@
         # if a prebuilt GcStruct contains a pointer to a young object,
         # then the write_barrier must have ensured that the prebuilt
         # GcStruct is in the list self.old_objects_pointing_to_young.
+        debug_start("gc-minor-walkroots")
         self.root_walker.walk_roots(
             MiniMarkGC._trace_drag_out1,  # stack roots
             MiniMarkGC._trace_drag_out1,  # static in prebuilt non-gc
             None)                         # static in prebuilt gc
+        debug_stop("gc-minor-walkroots")
 
     def collect_cardrefs_to_nursery(self):
         size_gc_header = self.gcheaderbuilder.size_gc_header
@@ -1582,6 +1591,9 @@
         # Weakref support: clear the weak pointers to dying objects
         if self.old_objects_with_weakrefs.non_empty():
             self.invalidate_old_weakrefs()
+        if self.old_objects_with_light_finalizers.non_empty():
+            self.deal_with_old_objects_with_finalizers()
+
         #
         # Walk all rawmalloced objects and free the ones that don't
         # have the GCFLAG_VISITED flag.
@@ -1647,8 +1659,7 @@
         if self.header(obj).tid & GCFLAG_VISITED:
             self.header(obj).tid &= ~GCFLAG_VISITED
             return False     # survives
-        else:
-            return True      # dies
+        return True      # dies
 
     def _reset_gcflag_visited(self, obj, ignored):
         self.header(obj).tid &= ~GCFLAG_VISITED
@@ -1827,6 +1838,39 @@
     # ----------
     # Finalizers
 
+    def deal_with_young_objects_with_finalizers(self):
+        """ This is a much simpler version of dealing with finalizers
+        and an optimization - we can reasonably assume that those finalizers
+        don't do anything fancy and *just* call them. Among other things
+        they won't resurrect objects
+        """
+        while self.young_objects_with_light_finalizers.non_empty():
+            obj = self.young_objects_with_light_finalizers.pop()
+            if not self.is_forwarded(obj):
+                finalizer = self.getlightfinalizer(self.get_type_id(obj))
+                ll_assert(bool(finalizer), "no light finalizer found")
+                finalizer(obj, llmemory.NULL)
+
+    def deal_with_old_objects_with_finalizers(self):
+        """ This is a much simpler version of dealing with finalizers
+        and an optimization - we can reasonably assume that those finalizers
+        don't do anything fancy and *just* call them. Among other things
+        they won't resurrect objects
+        """
+        new_objects = self.AddressStack()
+        while self.old_objects_with_light_finalizers.non_empty():
+            obj = self.old_objects_with_light_finalizers.pop()
+            if self.header(obj).tid & GCFLAG_VISITED:
+                # surviving
+                new_objects.append(obj)
+            else:
+                # dying
+                finalizer = self.getlightfinalizer(self.get_type_id(obj))
+                ll_assert(bool(finalizer), "no light finalizer found")
+                finalizer(obj, llmemory.NULL)
+        self.old_objects_with_light_finalizers.delete()
+        self.old_objects_with_light_finalizers = new_objects
+
     def deal_with_objects_with_finalizers(self):
         # Walk over list of objects with finalizers.
         # If it is not surviving, add it to the list of to-be-called
@@ -1957,7 +2001,6 @@
             #
             self.old_objects_with_weakrefs.append(obj)
 
-
     def invalidate_old_weakrefs(self):
         """Called during a major collection."""
         # walk over list of objects that contain weakrefs
diff --git a/pypy/rpython/memory/gc/semispace.py b/pypy/rpython/memory/gc/semispace.py
--- a/pypy/rpython/memory/gc/semispace.py
+++ b/pypy/rpython/memory/gc/semispace.py
@@ -82,6 +82,7 @@
         self.free = self.tospace
         MovingGCBase.setup(self)
         self.objects_with_finalizers = self.AddressDeque()
+        self.objects_with_light_finalizers = self.AddressStack()
         self.objects_with_weakrefs = self.AddressStack()
 
     def _teardown(self):
@@ -93,7 +94,9 @@
     # because the spaces are filled with zeroes in advance.
 
     def malloc_fixedsize_clear(self, typeid16, size,
-                               has_finalizer=False, contains_weakptr=False):
+                               has_finalizer=False,
+                               is_finalizer_light=False,
+                               contains_weakptr=False):
         size_gc_header = self.gcheaderbuilder.size_gc_header
         totalsize = size_gc_header + size
         result = self.free
@@ -102,7 +105,9 @@
         llarena.arena_reserve(result, totalsize)
         self.init_gc_object(result, typeid16)
         self.free = result + totalsize
-        if has_finalizer:
+        if is_finalizer_light:
+            self.objects_with_light_finalizers.append(result + size_gc_header)
+        elif has_finalizer:
             self.objects_with_finalizers.append(result + size_gc_header)
         if contains_weakptr:
             self.objects_with_weakrefs.append(result + size_gc_header)
@@ -263,6 +268,8 @@
         if self.run_finalizers.non_empty():
             self.update_run_finalizers()
         scan = self.scan_copied(scan)
+        if self.objects_with_light_finalizers.non_empty():
+            self.deal_with_objects_with_light_finalizers()
         if self.objects_with_finalizers.non_empty():
             scan = self.deal_with_objects_with_finalizers(scan)
         if self.objects_with_weakrefs.non_empty():
@@ -471,6 +478,23 @@
         # immortal objects always have GCFLAG_FORWARDED set;
         # see get_forwarding_address().
 
+    def deal_with_objects_with_light_finalizers(self):
+        """ This is a much simpler version of dealing with finalizers
+        and an optimization - we can reasonably assume that those finalizers
+        don't do anything fancy and *just* call them. Among other things
+        they won't resurrect objects
+        """
+        new_objects = self.AddressStack()
+        while self.objects_with_light_finalizers.non_empty():
+            obj = self.objects_with_light_finalizers.pop()
+            if self.surviving(obj):
+                new_objects.append(self.get_forwarding_address(obj))
+            else:
+                finalizer = self.getfinalizer(self.get_type_id(obj))
+                finalizer(obj, llmemory.NULL)
+        self.objects_with_light_finalizers.delete()
+        self.objects_with_light_finalizers = new_objects
+
     def deal_with_objects_with_finalizers(self, scan):
         # walk over list of objects with finalizers
         # if it is not copied, add it to the list of to-be-called finalizers
diff --git a/pypy/rpython/memory/gctransform/framework.py b/pypy/rpython/memory/gctransform/framework.py
--- a/pypy/rpython/memory/gctransform/framework.py
+++ b/pypy/rpython/memory/gctransform/framework.py
@@ -12,6 +12,7 @@
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.translator.backendopt import graphanalyze
 from pypy.translator.backendopt.support import var_needsgc
+from pypy.translator.backendopt.finalizer import FinalizerAnalyzer
 from pypy.annotation import model as annmodel
 from pypy.rpython import annlowlevel
 from pypy.rpython.rbuiltin import gen_cast
@@ -258,6 +259,7 @@
             [s_gc, s_typeid16,
              annmodel.SomeInteger(nonneg=True),
              annmodel.SomeBool(),
+             annmodel.SomeBool(),
              annmodel.SomeBool()], s_gcref,
             inline = False)
         if hasattr(GCClass, 'malloc_fixedsize'):
@@ -267,6 +269,7 @@
                 [s_gc, s_typeid16,
                  annmodel.SomeInteger(nonneg=True),
                  annmodel.SomeBool(),
+                 annmodel.SomeBool(),
                  annmodel.SomeBool()], s_gcref,
                 inline = False)
         else:
@@ -319,7 +322,7 @@
             raise NotImplementedError("GC needs write barrier, but does not provide writebarrier_before_copy functionality")
 
         # in some GCs we can inline the common case of
-        # malloc_fixedsize(typeid, size, True, False, False)
+        # malloc_fixedsize(typeid, size, False, False, False)
         if getattr(GCClass, 'inline_simple_malloc', False):
             # make a copy of this function so that it gets annotated
             # independently and the constants are folded inside
@@ -337,7 +340,7 @@
                 malloc_fast,
                 [s_gc, s_typeid16,
                  annmodel.SomeInteger(nonneg=True),
-                 s_False, s_False], s_gcref,
+                 s_False, s_False, s_False], s_gcref,
                 inline = True)
         else:
             self.malloc_fast_ptr = None
@@ -668,7 +671,13 @@
         kind_and_fptr = self.special_funcptr_for_type(TYPE)
         has_finalizer = (kind_and_fptr is not None and
                          kind_and_fptr[0] == "finalizer")
+        has_light_finalizer = (kind_and_fptr is not None and
+                               kind_and_fptr[0] == "light_finalizer")
+        if has_light_finalizer:
+            has_finalizer = True
         c_has_finalizer = rmodel.inputconst(lltype.Bool, has_finalizer)
+        c_has_light_finalizer = rmodel.inputconst(lltype.Bool,
+                                                  has_light_finalizer)
 
         if not op.opname.endswith('_varsize') and not flags.get('varsize'):
             #malloc_ptr = self.malloc_fixedsize_ptr
@@ -682,7 +691,8 @@
             else:
                 malloc_ptr = self.malloc_fixedsize_ptr
             args = [self.c_const_gc, c_type_id, c_size,
-                    c_has_finalizer, rmodel.inputconst(lltype.Bool, False)]
+                    c_has_finalizer, c_has_light_finalizer,
+                    rmodel.inputconst(lltype.Bool, False)]
         else:
             assert not c_has_finalizer.value
             info_varsize = self.layoutbuilder.get_info_varsize(type_id)
@@ -847,12 +857,13 @@
         # used by the JIT (see pypy.jit.backend.llsupport.gc)
         op = hop.spaceop
         [v_typeid, v_size,
-         v_has_finalizer, v_contains_weakptr] = op.args
+         v_has_finalizer, v_has_light_finalizer, v_contains_weakptr] = op.args
         livevars = self.push_roots(hop)
         hop.genop("direct_call",
                   [self.malloc_fixedsize_clear_ptr, self.c_const_gc,
                    v_typeid, v_size,
-                   v_has_finalizer, v_contains_weakptr],
+                   v_has_finalizer, v_has_light_finalizer,
+                   v_contains_weakptr],
                   resultvar=op.result)
         self.pop_roots(hop, livevars)
 
@@ -912,10 +923,10 @@
         info = self.layoutbuilder.get_info(type_id)
         c_size = rmodel.inputconst(lltype.Signed, info.fixedsize)
         malloc_ptr = self.malloc_fixedsize_ptr
-        c_has_finalizer = rmodel.inputconst(lltype.Bool, False)
+        c_false = rmodel.inputconst(lltype.Bool, False)
         c_has_weakptr = rmodel.inputconst(lltype.Bool, True)
         args = [self.c_const_gc, c_type_id, c_size,
-                c_has_finalizer, c_has_weakptr]
+                c_false, c_false, c_has_weakptr]
 
         # push and pop the current live variables *including* the argument
         # to the weakref_create operation, which must be kept alive and
@@ -1250,6 +1261,7 @@
             lltype2vtable = translator.rtyper.lltype2vtable
         else:
             lltype2vtable = None
+        self.translator = translator
         super(TransformerLayoutBuilder, self).__init__(GCClass, lltype2vtable)
 
     def has_finalizer(self, TYPE):
@@ -1257,6 +1269,10 @@
         return rtti is not None and getattr(rtti._obj, 'destructor_funcptr',
                                             None)
 
+    def has_light_finalizer(self, TYPE):
+        special = self.special_funcptr_for_type(TYPE)
+        return special is not None and special[0] == 'light_finalizer'
+
     def has_custom_trace(self, TYPE):
         rtti = get_rtti(TYPE)
         return rtti is not None and getattr(rtti._obj, 'custom_trace_funcptr',
@@ -1264,7 +1280,7 @@
 
     def make_finalizer_funcptr_for_type(self, TYPE):
         if not self.has_finalizer(TYPE):
-            return None
+            return None, False
         rtti = get_rtti(TYPE)
         destrptr = rtti._obj.destructor_funcptr
         DESTR_ARG = lltype.typeOf(destrptr).TO.ARGS[0]
@@ -1276,7 +1292,9 @@
             return llmemory.NULL
         fptr = self.transformer.annotate_finalizer(ll_finalizer,
                 [llmemory.Address, llmemory.Address], llmemory.Address)
-        return fptr
+        g = destrptr._obj.graph
+        light = not FinalizerAnalyzer(self.translator).analyze_light_finalizer(g)
+        return fptr, light
 
     def make_custom_trace_funcptr_for_type(self, TYPE):
         if not self.has_custom_trace(TYPE):
diff --git a/pypy/rpython/memory/gctypelayout.py b/pypy/rpython/memory/gctypelayout.py
--- a/pypy/rpython/memory/gctypelayout.py
+++ b/pypy/rpython/memory/gctypelayout.py
@@ -1,7 +1,6 @@
 from pypy.rpython.lltypesystem import lltype, llmemory, llarena, llgroup
 from pypy.rpython.lltypesystem import rclass
 from pypy.rpython.lltypesystem.lloperation import llop
-from pypy.rlib.objectmodel import we_are_translated
 from pypy.rlib.debug import ll_assert
 from pypy.rlib.rarithmetic import intmask
 from pypy.tool.identity_dict import identity_dict
@@ -85,6 +84,13 @@
         else:
             return lltype.nullptr(GCData.FINALIZER_OR_CT_FUNC)
 
+    def q_light_finalizer(self, typeid):
+        typeinfo = self.get(typeid)
+        if typeinfo.infobits & T_HAS_LIGHTWEIGHT_FINALIZER:
+            return typeinfo.finalizer_or_customtrace
+        else:
+            return lltype.nullptr(GCData.FINALIZER_OR_CT_FUNC)        
+
     def q_offsets_to_gc_pointers(self, typeid):
         return self.get(typeid).ofstoptrs
 
@@ -142,6 +148,7 @@
             self.q_has_gcptr_in_varsize,
             self.q_is_gcarrayofgcptr,
             self.q_finalizer,
+            self.q_light_finalizer,
             self.q_offsets_to_gc_pointers,
             self.q_fixed_size,
             self.q_varsize_item_sizes,
@@ -157,16 +164,17 @@
 
 
 # the lowest 16bits are used to store group member index
-T_MEMBER_INDEX         =   0xffff
-T_IS_VARSIZE           = 0x010000
-T_HAS_GCPTR_IN_VARSIZE = 0x020000
-T_IS_GCARRAY_OF_GCPTR  = 0x040000
-T_IS_WEAKREF           = 0x080000
-T_IS_RPYTHON_INSTANCE  = 0x100000    # the type is a subclass of OBJECT
-T_HAS_FINALIZER        = 0x200000
-T_HAS_CUSTOM_TRACE     = 0x400000
-T_KEY_MASK             = intmask(0xFF000000)
-T_KEY_VALUE            = intmask(0x5A000000)    # bug detection only
+T_MEMBER_INDEX              =   0xffff
+T_IS_VARSIZE                = 0x010000
+T_HAS_GCPTR_IN_VARSIZE      = 0x020000
+T_IS_GCARRAY_OF_GCPTR       = 0x040000
+T_IS_WEAKREF                = 0x080000
+T_IS_RPYTHON_INSTANCE       = 0x100000 # the type is a subclass of OBJECT
+T_HAS_FINALIZER             = 0x200000
+T_HAS_CUSTOM_TRACE          = 0x400000
+T_HAS_LIGHTWEIGHT_FINALIZER = 0x800000
+T_KEY_MASK                  = intmask(0xFF000000)
+T_KEY_VALUE                 = intmask(0x5A000000) # bug detection only
 
 def _check_valid_type_info(p):
     ll_assert(p.infobits & T_KEY_MASK == T_KEY_VALUE, "invalid type_id")
@@ -194,6 +202,8 @@
         info.finalizer_or_customtrace = fptr
         if kind == "finalizer":
             infobits |= T_HAS_FINALIZER
+        elif kind == 'light_finalizer':
+            infobits |= T_HAS_FINALIZER | T_HAS_LIGHTWEIGHT_FINALIZER
         elif kind == "custom_trace":
             infobits |= T_HAS_CUSTOM_TRACE
         else:
@@ -367,12 +377,15 @@
     def special_funcptr_for_type(self, TYPE):
         if TYPE in self._special_funcptrs:
             return self._special_funcptrs[TYPE]
-        fptr1 = self.make_finalizer_funcptr_for_type(TYPE)
+        fptr1, is_lightweight = self.make_finalizer_funcptr_for_type(TYPE)
         fptr2 = self.make_custom_trace_funcptr_for_type(TYPE)
         assert not (fptr1 and fptr2), (
             "type %r needs both a finalizer and a custom tracer" % (TYPE,))
         if fptr1:
-            kind_and_fptr = "finalizer", fptr1
+            if is_lightweight:
+                kind_and_fptr = "light_finalizer", fptr1
+            else:
+                kind_and_fptr = "finalizer", fptr1
         elif fptr2:
             kind_and_fptr = "custom_trace", fptr2
         else:
@@ -382,7 +395,7 @@
 
     def make_finalizer_funcptr_for_type(self, TYPE):
         # must be overridden for proper finalizer support
-        return None
+        return None, False
 
     def make_custom_trace_funcptr_for_type(self, TYPE):
         # must be overridden for proper custom tracer support
diff --git a/pypy/rpython/memory/gcwrapper.py b/pypy/rpython/memory/gcwrapper.py
--- a/pypy/rpython/memory/gcwrapper.py
+++ b/pypy/rpython/memory/gcwrapper.py
@@ -1,3 +1,4 @@
+from pypy.translator.backendopt.finalizer import FinalizerAnalyzer
 from pypy.rpython.lltypesystem import lltype, llmemory, llheap
 from pypy.rpython import llinterp
 from pypy.rpython.annlowlevel import llhelper
@@ -196,9 +197,11 @@
             DESTR_ARG = lltype.typeOf(destrptr).TO.ARGS[0]
             destrgraph = destrptr._obj.graph
         else:
-            return None
+            return None, False
 
         assert not type_contains_pyobjs(TYPE), "not implemented"
+        t = self.llinterp.typer.annotator.translator
+        light = not FinalizerAnalyzer(t).analyze_light_finalizer(destrgraph)
         def ll_finalizer(addr, dummy):
             assert dummy == llmemory.NULL
             try:
@@ -208,7 +211,7 @@
                 raise RuntimeError(
                     "a finalizer raised an exception, shouldn't happen")
             return llmemory.NULL
-        return llhelper(gctypelayout.GCData.FINALIZER_OR_CT, ll_finalizer)
+        return llhelper(gctypelayout.GCData.FINALIZER_OR_CT, ll_finalizer), light
 
     def make_custom_trace_funcptr_for_type(self, TYPE):
         from pypy.rpython.memory.gctransform.support import get_rtti, \
diff --git a/pypy/rpython/memory/test/test_gc.py b/pypy/rpython/memory/test/test_gc.py
--- a/pypy/rpython/memory/test/test_gc.py
+++ b/pypy/rpython/memory/test/test_gc.py
@@ -5,7 +5,6 @@
 from pypy.rpython.memory.test import snippet
 from pypy.rpython.test.test_llinterp import get_interpreter
 from pypy.rpython.lltypesystem import lltype
-from pypy.rpython.lltypesystem.rstr import STR
 from pypy.rpython.lltypesystem.lloperation import llop
 from pypy.rlib.objectmodel import we_are_translated
 from pypy.rlib.objectmodel import compute_unique_id
@@ -57,7 +56,7 @@
                 while j < 20:
                     j += 1
                     a.append(j)
-        res = self.interpret(malloc_a_lot, [])
+        self.interpret(malloc_a_lot, [])
         #assert simulator.current_size - curr < 16000 * INT_SIZE / 4
         #print "size before: %s, size after %s" % (curr, simulator.current_size)
 
@@ -73,7 +72,7 @@
                 while j < 20:
                     j += 1
                     b.append((1, j, i))
-        res = self.interpret(malloc_a_lot, [])
+        self.interpret(malloc_a_lot, [])
         #assert simulator.current_size - curr < 16000 * INT_SIZE / 4
         #print "size before: %s, size after %s" % (curr, simulator.current_size)
 
@@ -129,7 +128,7 @@
         res = self.interpret(concat, [100])
         assert res == concat(100)
         #assert simulator.current_size - curr < 16000 * INT_SIZE / 4
-        
+
     def test_finalizer(self):
         class B(object):
             pass
@@ -278,7 +277,7 @@
                                self.interpret, f, [])
 
     def test_weakref(self):
-        import weakref, gc
+        import weakref
         class A(object):
             pass
         def g():
@@ -299,7 +298,7 @@
         assert res
 
     def test_weakref_to_object_with_finalizer(self):
-        import weakref, gc
+        import weakref
         class A(object):
             count = 0
         a = A()
@@ -338,7 +337,7 @@
         assert res
 
     def test_cycle_with_weakref_and_del(self):
-        import weakref, gc
+        import weakref
         class A(object):
             count = 0
         a = A()
@@ -367,7 +366,7 @@
         assert res == 11
 
     def test_weakref_to_object_with_finalizer_ordering(self):
-        import weakref, gc
+        import weakref
         class A(object):
             count = 0
         a = A()
@@ -616,7 +615,7 @@
                     assert not rgc.can_move(a)
                     return 1
                 return 0
-            except Exception, e:
+            except Exception:
                 return 2
 
         assert self.interpret(func, []) == int(self.GC_CAN_MALLOC_NONMOVABLE)
@@ -647,8 +646,6 @@
         assert self.interpret(f, [bigsize, 0, flag]) == 0x62024241
 
     def test_tagged_simple(self):
-        from pypy.rlib.objectmodel import UnboxedValue
-
         class Unrelated(object):
             pass
 
@@ -689,8 +686,6 @@
         assert res == -897
 
     def test_tagged_id(self):
-        from pypy.rlib.objectmodel import UnboxedValue, compute_unique_id
-
         class Unrelated(object):
             pass
 
diff --git a/pypy/rpython/memory/test/test_transformed_gc.py b/pypy/rpython/memory/test/test_transformed_gc.py
--- a/pypy/rpython/memory/test/test_transformed_gc.py
+++ b/pypy/rpython/memory/test/test_transformed_gc.py
@@ -345,22 +345,22 @@
         b = B()
         b.nextid = 0
         b.num_deleted = 0
-        class A(object):
+        class AAA(object):
             def __init__(self):
                 self.id = b.nextid
                 b.nextid += 1
             def __del__(self):
                 b.num_deleted += 1
                 C()
-        class C(A):
+        class C(AAA):
             def __del__(self):
                 b.num_deleted += 1
         def f(x, y):
-            a = A()
+            a = AAA()
             i = 0
             while i < x:
                 i += 1
-                a = A()
+                a = AAA()
             llop.gc__collect(lltype.Void)
             llop.gc__collect(lltype.Void)
             return b.num_deleted
@@ -807,6 +807,7 @@
                     op.args = [Constant(type_id, llgroup.HALFWORD),
                                Constant(llmemory.sizeof(P), lltype.Signed),
                                Constant(False, lltype.Bool), # has_finalizer
+                               Constant(False, lltype.Bool), # is_finalizer_light
                                Constant(False, lltype.Bool)] # contains_weakptr
                     break
             else:
@@ -843,6 +844,7 @@
                     op.args = [Constant(type_id, llgroup.HALFWORD),
                                Constant(llmemory.sizeof(P), lltype.Signed),
                                Constant(False, lltype.Bool), # has_finalizer
+                               Constant(False, lltype.Bool), # is_finalizer_light
                                Constant(False, lltype.Bool)] # contains_weakptr
                     break
             else:
diff --git a/pypy/rpython/module/ll_os.py b/pypy/rpython/module/ll_os.py
--- a/pypy/rpython/module/ll_os.py
+++ b/pypy/rpython/module/ll_os.py
@@ -959,8 +959,6 @@
                             os_ftruncate(rffi.cast(rffi.INT, fd),
                                          rffi.cast(rffi.LONGLONG, length)))
             if res < 0:
-                # Note: for consistency we raise OSError, but CPython
-                # raises IOError here
                 raise OSError(rposix.get_errno(), "os_ftruncate failed")
 
         return extdef([int, r_longlong], s_None,
diff --git a/pypy/rpython/rtyper.py b/pypy/rpython/rtyper.py
--- a/pypy/rpython/rtyper.py
+++ b/pypy/rpython/rtyper.py
@@ -717,7 +717,7 @@
             raise TyperError("runtime type info function %r returns %r, "
                              "excepted Ptr(RuntimeTypeInfo)" % (func, s))
         funcptr = self.getcallable(graph)
-        attachRuntimeTypeInfo(GCSTRUCT, funcptr, destrptr)
+        attachRuntimeTypeInfo(GCSTRUCT, funcptr, destrptr, None)
 
 # register operations from annotation model
 RPythonTyper._registeroperations(annmodel)
diff --git a/pypy/rpython/test/test_rclass.py b/pypy/rpython/test/test_rclass.py
--- a/pypy/rpython/test/test_rclass.py
+++ b/pypy/rpython/test/test_rclass.py
@@ -3,7 +3,7 @@
 from pypy.translator.translator import TranslationContext, graphof
 from pypy.rpython.lltypesystem.lltype import *
 from pypy.rpython.ootypesystem import ootype
-from pypy.rlib.rarithmetic import intmask, r_longlong
+from pypy.rlib.rarithmetic import r_longlong
 from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin
 from pypy.rpython.rclass import IR_IMMUTABLE, IR_IMMUTABLE_ARRAY
 from pypy.rpython.rclass import IR_QUASIIMMUTABLE, IR_QUASIIMMUTABLE_ARRAY
@@ -972,10 +972,10 @@
         graph = graphof(t, f)
         TYPE = graph.startblock.operations[0].args[0].value
         RTTI = getRuntimeTypeInfo(TYPE)
-        queryptr = RTTI._obj.query_funcptr # should not raise
+        RTTI._obj.query_funcptr # should not raise
         destrptr = RTTI._obj.destructor_funcptr
         assert destrptr is not None
-    
+
     def test_del_inheritance(self):
         from pypy.rlib import rgc
         class State:
diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -56,8 +56,8 @@
     binaries = [(pypy_c, rename_pypy_c)]
     #
     if sys.platform == 'win32':
-        # Can't rename a DLL: it is always called 'libpypy_c.dll'
-        for extra in ['libpypy_c.dll',
+        # Can't rename a DLL: it is always called 'libpypy-c.dll'
+        for extra in ['libpypy-c.dll',
                       'libexpat.dll', 'sqlite3.dll', 'msvcr90.dll']:
             p = pypy_c.dirpath().join(extra)
             if not p.check():
diff --git a/pypy/tool/sourcetools.py b/pypy/tool/sourcetools.py
--- a/pypy/tool/sourcetools.py
+++ b/pypy/tool/sourcetools.py
@@ -216,9 +216,11 @@
 
 # ____________________________________________________________
 
-def func_with_new_name(func, newname):
+def func_with_new_name(func, newname, globals=None):
     """Make a renamed copy of a function."""
-    f = new.function(func.func_code, func.func_globals,
+    if globals is None:
+        globals = func.func_globals
+    f = new.function(func.func_code, globals,
                         newname, func.func_defaults,
                         func.func_closure)
     if func.func_dict:
diff --git a/pypy/translator/backendopt/finalizer.py b/pypy/translator/backendopt/finalizer.py
new file mode 100644
--- /dev/null
+++ b/pypy/translator/backendopt/finalizer.py
@@ -0,0 +1,46 @@
+
+from pypy.translator.backendopt import graphanalyze
+from pypy.rpython.lltypesystem import lltype
+
+class FinalizerError(Exception):
+    """ __del__ marked as lightweight finalizer, but the analyzer did
+    not agreed
+    """
+
+class FinalizerAnalyzer(graphanalyze.BoolGraphAnalyzer):
+    """ Analyzer that determines whether a finalizer is lightweight enough
+    so it can be called without all the complicated logic in the garbage
+    collector. The set of operations here is restrictive for a good reason
+    - it's better to be safe. Specifically disallowed operations:
+
+    * anything that escapes self
+    * anything that can allocate
+    """
+    ok_operations = ['ptr_nonzero', 'ptr_eq', 'ptr_ne', 'free', 'same_as',
+                     'direct_ptradd', 'force_cast', 'track_alloc_stop',
+                     'raw_free']
+    
+    def analyze_light_finalizer(self, graph):
+        result = self.analyze_direct_call(graph)
+        if (result is self.top_result() and
+            getattr(graph.func, '_is_light_finalizer_', False)):
+            raise FinalizerError(FinalizerError.__doc__, graph)
+        return result
+    
+    def analyze_simple_operation(self, op, graphinfo):
+        if op.opname in self.ok_operations:
+            return self.bottom_result()
+        if (op.opname.startswith('int_') or op.opname.startswith('float_')
+            or op.opname.startswith('cast_')):
+            return self.bottom_result()
+        if op.opname == 'setfield' or op.opname == 'bare_setfield':
+            TP = op.args[2].concretetype
+            if not isinstance(TP, lltype.Ptr) or TP.TO._gckind == 'raw':
+                # primitive type
+                return self.bottom_result()
+        if op.opname == 'getfield':
+            TP = op.result.concretetype
+            if not isinstance(TP, lltype.Ptr) or TP.TO._gckind == 'raw':
+                # primitive type
+                return self.bottom_result()            
+        return self.top_result()
diff --git a/pypy/translator/backendopt/test/test_finalizer.py b/pypy/translator/backendopt/test/test_finalizer.py
new file mode 100644
--- /dev/null
+++ b/pypy/translator/backendopt/test/test_finalizer.py
@@ -0,0 +1,142 @@
+
+import py
+from pypy.translator.backendopt.finalizer import FinalizerAnalyzer,\
+     FinalizerError
+from pypy.translator.translator import TranslationContext, graphof
+from pypy.translator.backendopt.all import backend_optimizations
+from pypy.translator.unsimplify import varoftype
+from pypy.rpython.lltypesystem import lltype, rffi
+from pypy.conftest import option
+from pypy.rlib import rgc
+
+
+class BaseFinalizerAnalyzerTests(object):
+    """ Below are typical destructors that we encounter in pypy
+    """
+
+    type_system = None
+    
+    def analyze(self, func, sig, func_to_analyze=None, backendopt=False):
+        if func_to_analyze is None:
+            func_to_analyze = func
+        t = TranslationContext()
+        t.buildannotator().build_types(func, sig)
+        t.buildrtyper(type_system=self.type_system).specialize()
+        if backendopt:
+            backend_optimizations(t)
+        if option.view:
+            t.view()
+        a = FinalizerAnalyzer(t)
+        fgraph = graphof(t, func_to_analyze)
+        result = a.analyze_light_finalizer(fgraph)
+        return result
+
+    def test_nothing(self):
+        def f():
+            pass
+        r = self.analyze(f, [])
+        assert not r
+
+def test_various_ops():
+    from pypy.objspace.flow.model import SpaceOperation, Constant
+
+    X = lltype.Ptr(lltype.GcStruct('X'))
+    Z = lltype.Ptr(lltype.Struct('Z'))
+    S = lltype.GcStruct('S', ('x', lltype.Signed),
+                        ('y', X),
+                        ('z', Z))
+    v1 = varoftype(lltype.Bool)
+    v2 = varoftype(lltype.Signed)
+    f = FinalizerAnalyzer(None)
+    r = f.analyze(SpaceOperation('cast_int_to_bool', [v2],
+                                                       v1))
+    assert not r
+    v1 = varoftype(lltype.Ptr(S))
+    v2 = varoftype(lltype.Signed)
+    v3 = varoftype(X)
+    v4 = varoftype(Z)
+    assert not f.analyze(SpaceOperation('bare_setfield', [v1, Constant('x'),
+                                                          v2], None))
+    assert     f.analyze(SpaceOperation('bare_setfield', [v1, Constant('y'),
+                                                          v3], None))
+    assert not f.analyze(SpaceOperation('bare_setfield', [v1, Constant('z'),
+                                                          v4], None))
+    
+        
+class TestLLType(BaseFinalizerAnalyzerTests):
+    type_system = 'lltype'
+
+    def test_malloc(self):
+        S = lltype.GcStruct('S')
+        
+        def f():
+            return lltype.malloc(S)
+
+        r = self.analyze(f, [])
+        assert r
+
+    def test_raw_free_getfield(self):
+        S = lltype.Struct('S')
+        
+        class A(object):
+            def __init__(self):
+                self.x = lltype.malloc(S, flavor='raw')
+
+            def __del__(self):
+                if self.x:
+                    self.x = lltype.nullptr(S)
+                    lltype.free(self.x, flavor='raw')
+
+        def f():
+            return A()
+
+        r = self.analyze(f, [], A.__del__.im_func)
+        assert not r
+
+    def test_c_call(self):
+        C = rffi.CArray(lltype.Signed)
+        c = rffi.llexternal('x', [lltype.Ptr(C)], lltype.Signed)
+
+        def g():
+            p = lltype.malloc(C, 3, flavor='raw')
+            f(p)
+        
+        def f(p):
+            c(rffi.ptradd(p, 0))
+            lltype.free(p, flavor='raw')
+
+        r = self.analyze(g, [], f, backendopt=True)
+        assert not r
+
+    def test_chain(self):
+        class B(object):
+            def __init__(self):
+                self.counter = 1
+        
+        class A(object):
+            def __init__(self):
+                self.x = B()
+
+            def __del__(self):
+                self.x.counter += 1
+
+        def f():
+            A()
+
+        r = self.analyze(f, [], A.__del__.im_func)
+        assert r
+
+    def test_is_light_finalizer_decorator(self):
+        S = lltype.GcStruct('S')
+
+        @rgc.is_light_finalizer
+        def f():
+            lltype.malloc(S)
+        @rgc.is_light_finalizer
+        def g():
+            pass
+        self.analyze(g, []) # did not explode
+        py.test.raises(FinalizerError, self.analyze, f, [])
+
+class TestOOType(BaseFinalizerAnalyzerTests):
+    type_system = 'ootype'
diff --git a/pypy/translator/c/database.py b/pypy/translator/c/database.py
--- a/pypy/translator/c/database.py
+++ b/pypy/translator/c/database.py
@@ -176,7 +176,7 @@
                      # introduced by the GC transformer, or the type_info_table
         return node
 
-    def get(self, obj):
+    def get(self, obj, funcgen=None):
         if isinstance(obj, CConstant):
             return obj.c_name  # without further checks
         T = typeOf(obj)
@@ -228,6 +228,8 @@
                     return '((%s) %d)' % (cdecl(self.gettype(T), ''),
                                           obj._obj)
                 node = self.getcontainernode(container)
+                if node._funccodegen_owner is None:
+                    node._funccodegen_owner = funcgen
                 return node.getptrname()
             else:
                 return '((%s) NULL)' % (cdecl(self.gettype(T), ''), )
@@ -284,14 +286,16 @@
             finish_callbacks.append(('GC transformer: finished tables',
                                      self.gctransformer.get_finish_tables()))
 
-        def add_dependencies(newdependencies):
+        def add_dependencies(newdependencies, parent=None):
             for value in newdependencies:
                 #if isinstance(value, _uninitialized):
                 #    continue
                 if isinstance(typeOf(value), ContainerType):
-                    self.getcontainernode(value)
+                    node = self.getcontainernode(value)
+                    if parent and node._funccodegen_owner is not None:
+                        node._funccodegen_owner = parent._funccodegen_owner
                 else:
-                    self.get(value)
+                    self.get(value, parent and parent._funccodegen_owner)
 
         while True:
             while True:
@@ -303,7 +307,7 @@
                 if i == len(self.containerlist):
                     break
                 node = self.containerlist[i]
-                add_dependencies(node.enum_dependencies())
+                add_dependencies(node.enum_dependencies(), node)
                 i += 1
                 self.completedcontainers = i
                 if i == show_i:
diff --git a/pypy/translator/c/gc.py b/pypy/translator/c/gc.py
--- a/pypy/translator/c/gc.py
+++ b/pypy/translator/c/gc.py
@@ -170,6 +170,7 @@
     nodekind = 'refcnt rtti'
     globalcontainer = True
     typename = 'void (@)(void *)'
+    _funccodegen_owner = None
 
     def __init__(self, db, T, obj):
         assert T == RuntimeTypeInfo
@@ -266,6 +267,7 @@
     nodekind = 'boehm rtti'
     globalcontainer = True
     typename = 'char @'
+    _funccodegen_owner = None
 
     def __init__(self, db, T, obj):
         assert T == RuntimeTypeInfo
diff --git a/pypy/translator/c/gcc/test/test_asmgcroot.py b/pypy/translator/c/gcc/test/test_asmgcroot.py
--- a/pypy/translator/c/gcc/test/test_asmgcroot.py
+++ b/pypy/translator/c/gcc/test/test_asmgcroot.py
@@ -21,6 +21,7 @@
         config = get_pypy_config(translating=True)
         config.translation.gc = cls.gcpolicy
         config.translation.gcrootfinder = "asmgcc"
+        config.translation.taggedpointers = getattr(cls, "taggedpointers", False)
         return config
 
     @classmethod
diff --git a/pypy/translator/c/genc.py b/pypy/translator/c/genc.py
--- a/pypy/translator/c/genc.py
+++ b/pypy/translator/c/genc.py
@@ -682,8 +682,7 @@
     def getbasecfilefornode(self, node, basecname):
         # For FuncNode instances, use the python source filename (relative to
         # the top directory):
-        if hasattr(node.obj, 'graph'):
-            g = node.obj.graph
+        def invent_nice_name(g):
             # Lookup the filename from the function.
             # However, not all FunctionGraph objs actually have a "func":
             if hasattr(g, 'func'):
@@ -693,6 +692,15 @@
                     if pypkgpath:
                         relpypath =  localpath.relto(pypkgpath)
                         return relpypath.replace('.py', '.c')
+            return None
+        if hasattr(node.obj, 'graph'):
+            name = invent_nice_name(node.obj.graph)
+            if name is not None:
+                return name
+        elif node._funccodegen_owner is not None:
+            name = invent_nice_name(node._funccodegen_owner.graph)
+            if name is not None:
+                return "data_" + name
         return basecname
 
     def splitnodesimpl(self, basecname, nodes, nextra, nbetween,
diff --git a/pypy/translator/c/node.py b/pypy/translator/c/node.py
--- a/pypy/translator/c/node.py
+++ b/pypy/translator/c/node.py
@@ -485,6 +485,7 @@
         __slots__ = """db obj 
                        typename implementationtypename
                         name
+                        _funccodegen_owner
                         globalcontainer""".split()
     eci_name = '_compilation_info'
 
@@ -509,6 +510,7 @@
         if self.typename != self.implementationtypename:
             if db.gettypedefnode(T).extra_union_for_varlength:
                 self.name += '.b'
+        self._funccodegen_owner = None
 
     def getptrname(self):
         return '(&%s)' % self.name
@@ -842,6 +844,9 @@
         if self.funcgens:
             argnames = self.funcgens[0].argnames()  #Assume identical for all funcgens
             self.implementationtypename = self.db.gettype(self.T, argnames=argnames)
+            self._funccodegen_owner = self.funcgens[0]
+        else:
+            self._funccodegen_owner = None
 
     def basename(self):
         return self.obj._name
@@ -1005,6 +1010,7 @@
     globalcontainer = True
     typename = 'PyObject @'
     implementationtypename = 'PyObject *@'
+    _funccodegen_owner = None
 
     def __init__(self, db, T, obj):
         # obj is a _pyobject here; obj.value is the underlying CPython object
diff --git a/pypy/translator/c/primitive.py b/pypy/translator/c/primitive.py
--- a/pypy/translator/c/primitive.py
+++ b/pypy/translator/c/primitive.py
@@ -141,12 +141,22 @@
 
 def name_gcref(value, db):
     if value:
-        realobj = value._obj.container
+        obj = value._obj
+        if isinstance(obj, int):
+            # a tagged pointer
+            return _name_tagged(obj, db)
+        realobj = obj.container
+        if isinstance(realobj, int):
+            return _name_tagged(realobj, db)
         realvalue = cast_opaque_ptr(Ptr(typeOf(realobj)), value)
         return db.get(realvalue)
     else:
         return 'NULL'
 
+def _name_tagged(obj, db):
+    assert obj & 1 == 1
+    return '((%s) %d)' % (cdecl("void*", ''), obj)
+
 def name_small_integer(value, db):
     """Works for integers of size at most INT or UINT."""
     if isinstance(value, Symbolic):
diff --git a/pypy/translator/c/test/test_newgc.py b/pypy/translator/c/test/test_newgc.py
--- a/pypy/translator/c/test/test_newgc.py
+++ b/pypy/translator/c/test/test_newgc.py
@@ -1487,6 +1487,43 @@
         res = self.run("tagged")
         assert res == expected
 
+    def define_erased(cls):
+        from pypy.rlib import rerased
+        erase, unerase = rerased.new_erasing_pair("test")
+        class Unrelated(object):
+            pass
+
+        u = Unrelated()
+        u.tagged = True
+        u.x = rerased.erase_int(41)
+        class A(object):
+            pass
+        def fn():
+            n = 1
+            while n >= 0:
+                if u.tagged:
+                    n = rerased.unerase_int(u.x)
+                    a = A()
+                    a.n = n - 1
+                    u.x = erase(a)
+                    u.tagged = False
+                else:
+                    n = unerase(u.x).n
+                    u.x = rerased.erase_int(n - 1)
+                    u.tagged = True
+        def func():
+            rgc.collect() # check that a prebuilt erased integer doesn't explode
+            u.x = rerased.erase_int(1000)
+            u.tagged = True
+            fn()
+            return 1
+        return func
+
+    def test_erased(self):
+        expected = self.run_orig("erased")
+        res = self.run("erased")
+        assert res == expected
+
 from pypy.rlib.objectmodel import UnboxedValue
 
 class TaggedBase(object):
diff --git a/pypy/translator/c/test/test_rtagged.py b/pypy/translator/c/test/test_rtagged.py
--- a/pypy/translator/c/test/test_rtagged.py
+++ b/pypy/translator/c/test/test_rtagged.py
@@ -77,3 +77,12 @@
     data = g.read()
     g.close()
     assert data.rstrip().endswith('ALL OK')
+
+def test_name_gcref():
+    from pypy.rpython.lltypesystem import lltype, llmemory, rclass
+    from pypy.translator.c import primitive
+    from pypy.translator.c.database import LowLevelDatabase
+    x = lltype.cast_int_to_ptr(rclass.OBJECTPTR, 19)
+    y = lltype.cast_opaque_ptr(llmemory.GCREF, x)
+    db = LowLevelDatabase()
+    assert primitive.name_gcref(y, db) == "((void*) 19)"
diff --git a/pypy/translator/platform/linux.py b/pypy/translator/platform/linux.py
--- a/pypy/translator/platform/linux.py
+++ b/pypy/translator/platform/linux.py
@@ -1,5 +1,6 @@
 """Support for Linux."""
 
+import sys
 from pypy.translator.platform.posix import BasePosix
 
 class BaseLinux(BasePosix):
@@ -26,7 +27,11 @@
 
     def library_dirs_for_libffi_a(self):
         # places where we need to look for libffi.a
-        return self.library_dirs_for_libffi() + ['/usr/lib']
+        # XXX obscuuure!  only look for libffi.a if run with translate.py
+        if 'translate' in sys.modules:
+            return self.library_dirs_for_libffi() + ['/usr/lib']
+        else:
+            return []
 
 
 class Linux(BaseLinux):
diff --git a/pypy/translator/platform/posix.py b/pypy/translator/platform/posix.py
--- a/pypy/translator/platform/posix.py
+++ b/pypy/translator/platform/posix.py
@@ -157,7 +157,7 @@
 
         rules = [
             ('all', '$(DEFAULT_TARGET)', []),
-            ('$(TARGET)', '$(OBJECTS)', '$(CC_LINK) $(LDFLAGS) $(LDFLAGSEXTRA) -o $@ $(OBJECTS) $(LIBDIRS) $(LIBS) $(LINKFILES)'),
+            ('$(TARGET)', '$(OBJECTS)', '$(CC_LINK) $(LDFLAGSEXTRA) -o $@ $(OBJECTS) $(LIBDIRS) $(LIBS) $(LINKFILES) $(LDFLAGS)'),
             ('%.o', '%.c', '$(CC) $(CFLAGS) $(CFLAGSEXTRA) -o $@ -c $< $(INCLUDEDIRS)'),
             ]
 

From noreply at buildbot.pypy.org  Sun Oct 30 14:40:52 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 30 Oct 2011 14:40:52 +0100 (CET)
Subject: [pypy-commit] extradoc extradoc: Updates
Message-ID: <20111030134052.6EEA3820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r3952:340aece5643f
Date: 2011-10-30 14:40 +0100
http://bitbucket.org/pypy/extradoc/changeset/340aece5643f/

Log:	Updates

diff --git a/sprintinfo/gothenburg-2011-2/people.txt b/sprintinfo/gothenburg-2011-2/people.txt
--- a/sprintinfo/gothenburg-2011-2/people.txt
+++ b/sprintinfo/gothenburg-2011-2/people.txt
@@ -10,9 +10,10 @@
 ==================== ============== ===================== ==================
 Jacob Hallen         lives there
 Laura Creighton      lives there
-Armin Rigo           ?              ?
-Maciej Fija&#322;kowski   2-8 Nov        ?                     vegetarian
+Armin Rigo           ?              SGS Veckobostader
+Maciej Fija&#322;kowski   2-8 Nov        SGS Veckobostader     vegetarian
 Antonio Cuni         4-10 november  Laura's and Jacob's
 H&#229;kan Ard&#246;           3-8            ?
 Sam Lade             1-9            Hotel Poseidon
+Mark Pearse          2-10           SGS Veckobostader?
 ==================== ============== ===================== ==================

From noreply at buildbot.pypy.org  Sun Oct 30 15:55:33 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 30 Oct 2011 15:55:33 +0100 (CET)
Subject: [pypy-commit] pypy default: Progress a bit on optimizing int_mod.
Message-ID: <20111030145533.B68CB820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48604:bfea8fbdeed4
Date: 2011-10-30 09:24 +0100
http://bitbucket.org/pypy/pypy/changeset/bfea8fbdeed4/

Log:	Progress a bit on optimizing int_mod.

diff --git a/pypy/jit/metainterp/optimizeopt/intbounds.py b/pypy/jit/metainterp/optimizeopt/intbounds.py
--- a/pypy/jit/metainterp/optimizeopt/intbounds.py
+++ b/pypy/jit/metainterp/optimizeopt/intbounds.py
@@ -5,6 +5,7 @@
     IntUpperBound)
 from pypy.jit.metainterp.optimizeopt.util import make_dispatcher_method
 from pypy.jit.metainterp.resoperation import rop
+from pypy.rlib.rarithmetic import LONG_BIT
 
 
 class OptIntBounds(Optimization):
@@ -126,14 +127,27 @@
         r.intbound.intersect(v1.intbound.div_bound(v2.intbound))
 
     def optimize_INT_MOD(self, op):
+        v1 = self.getvalue(op.getarg(0))
+        v2 = self.getvalue(op.getarg(1))
+        known_nonneg = (v1.intbound.known_ge(IntBound(0, 0)) and 
+                        v2.intbound.known_ge(IntBound(0, 0)))
+        if known_nonneg and v2.is_constant():
+            val = v2.box.getint()
+            if (val & (val-1)) == 0:
+                # nonneg % power-of-two ==> nonneg & (power-of-two - 1)
+                arg1 = op.getarg(0)
+                arg2 = ConstInt(val-1)
+                op = op.copy_and_change(rop.INT_AND, args=[arg1, arg2])
         self.emit_operation(op)
-        v2 = self.getvalue(op.getarg(1))
         if v2.is_constant():
             val = v2.box.getint()
             r = self.getvalue(op.result)
             if val < 0:
                 val = -val
-            r.intbound.make_gt(IntBound(-val, -val))
+            if known_nonneg:
+                r.intbound.make_gt(IntBound(0, 0))
+            else:
+                r.intbound.make_gt(IntBound(-val, -val))
             r.intbound.make_lt(IntBound(val, val))
 
     def optimize_INT_LSHIFT(self, op):
@@ -153,9 +167,14 @@
     def optimize_INT_RSHIFT(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
-        self.emit_operation(op)
-        r = self.getvalue(op.result)
-        r.intbound.intersect(v1.intbound.rshift_bound(v2.intbound))
+        b = v1.intbound.rshift_bound(v2.intbound)
+        if b.has_lower and b.has_upper and b.lower == b.upper:
+            # constant result (likely 0, for rshifts that kill all bits)
+            self.make_constant_int(op.result, b.lower)
+        else:
+            self.emit_operation(op)
+            r = self.getvalue(op.result)
+            r.intbound.intersect(b)
 
     def optimize_INT_ADD_OVF(self, op):
         v1 = self.getvalue(op.getarg(0))
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -9,6 +9,7 @@
 from pypy.jit.metainterp.history import AbstractDescr, ConstInt, BoxInt
 from pypy.jit.metainterp import executor, compile, resume, history
 from pypy.jit.metainterp.resoperation import rop, opname, ResOperation
+from pypy.rlib.rarithmetic import LONG_BIT
 
 
 def test_store_final_boxes_in_guard():
@@ -4691,11 +4692,11 @@
         i5 = int_ge(i0, 0)
         guard_true(i5) []
         i1 = int_mod(i0, 42)
-        i2 = int_rshift(i1, 63)
+        i2 = int_rshift(i1, %d)
         i3 = int_and(42, i2)
         i4 = int_add(i1, i3)
         finish(i4)
-        """
+        """ % (LONG_BIT-1)
         expected = """
         [i0]
         i5 = int_ge(i0, 0)
@@ -4703,21 +4704,41 @@
         i1 = int_mod(i0, 42)
         finish(i1)
         """
-        py.test.skip("in-progress")
         self.optimize_loop(ops, expected)
 
-        # Also, 'n % power-of-two' can be turned into int_and(),
-        # but that's a bit harder to detect here because it turns into
-        # several operations, and of course it is wrong to just turn
+        # 'n % power-of-two' can be turned into int_and(); at least that's
+        # easy to do now if n is known to be non-negative.
+        ops = """
+        [i0]
+        i5 = int_ge(i0, 0)
+        guard_true(i5) []
+        i1 = int_mod(i0, 8)
+        i2 = int_rshift(i1, %d)
+        i3 = int_and(42, i2)
+        i4 = int_add(i1, i3)
+        finish(i4)
+        """ % (LONG_BIT-1)
+        expected = """
+        [i0]
+        i5 = int_ge(i0, 0)
+        guard_true(i5) []
+        i1 = int_and(i0, 7)
+        finish(i1)
+        """
+        self.optimize_loop(ops, expected)
+
+        # Of course any 'maybe-negative % power-of-two' can be turned into
+        # int_and(), but that's a bit harder to detect here because it turns
+        # into several operations, and of course it is wrong to just turn
         # int_mod(i0, 16) into int_and(i0, 15).
         ops = """
         [i0]
         i1 = int_mod(i0, 16)
-        i2 = int_rshift(i1, 63)
+        i2 = int_rshift(i1, %d)
         i3 = int_and(16, i2)
         i4 = int_add(i1, i3)
         finish(i4)
-        """
+        """ % (LONG_BIT-1)
         expected = """
         [i0]
         i4 = int_and(i0, 15)

From noreply at buildbot.pypy.org  Sun Oct 30 15:55:34 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 30 Oct 2011 15:55:34 +0100 (CET)
Subject: [pypy-commit] pypy default: Add skipped tests about division.
Message-ID: <20111030145534.E503C820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48605:4a6eae0d9aa1
Date: 2011-10-30 09:42 +0100
http://bitbucket.org/pypy/pypy/changeset/4a6eae0d9aa1/

Log:	Add skipped tests about division.

diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -4783,6 +4783,52 @@
         """
         self.optimize_loop(ops, expected)
 
+
+    def test_division_nonneg(self):
+        py.test.skip("harder")
+        # this is how an app-level division turns into right now
+        ops = """
+        [i4]
+        i1 = int_ge(i4, 0)
+        guard_true(i1) []
+        i16 = int_floordiv(i4, 3)
+        i18 = int_mul(i16, 3)
+        i19 = int_sub(i4, i18)
+        i21 = int_rshift(i19, %d)
+        i22 = int_add(i16, i21)
+        finish(i22)
+        """ % (LONG_BIT-1)
+        expected = """
+        [i4]
+        i1 = int_ge(i4, 0)
+        guard_true(i1) []
+        i16 = int_floordiv(i4, 3)
+        finish(i16)
+        """
+        self.optimize_loop(ops, expected)
+
+    def test_division_by_2(self):
+        py.test.skip("harder")
+        ops = """
+        [i4]
+        i1 = int_ge(i4, 0)
+        guard_true(i1) []
+        i16 = int_floordiv(i4, 2)
+        i18 = int_mul(i16, 2)
+        i19 = int_sub(i4, i18)
+        i21 = int_rshift(i19, %d)
+        i22 = int_add(i16, i21)
+        finish(i22)
+        """ % (LONG_BIT-1)
+        expected = """
+        [i4]
+        i1 = int_ge(i4, 0)
+        guard_true(i1) []
+        i16 = int_rshift(i4, 1)
+        finish(i16)
+        """
+        self.optimize_loop(ops, expected)
+
     def test_subsub_ovf(self):
         ops = """
         [i0]

From noreply at buildbot.pypy.org  Sun Oct 30 15:55:36 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 30 Oct 2011 15:55:36 +0100 (CET)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20111030145536.232D3820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48606:8e8835c75415
Date: 2011-10-30 15:55 +0100
http://bitbucket.org/pypy/pypy/changeset/8e8835c75415/

Log:	merge heads

diff --git a/pypy/jit/metainterp/optimizeopt/intbounds.py b/pypy/jit/metainterp/optimizeopt/intbounds.py
--- a/pypy/jit/metainterp/optimizeopt/intbounds.py
+++ b/pypy/jit/metainterp/optimizeopt/intbounds.py
@@ -5,6 +5,7 @@
     IntUpperBound)
 from pypy.jit.metainterp.optimizeopt.util import make_dispatcher_method
 from pypy.jit.metainterp.resoperation import rop
+from pypy.rlib.rarithmetic import LONG_BIT
 
 
 class OptIntBounds(Optimization):
@@ -126,14 +127,27 @@
         r.intbound.intersect(v1.intbound.div_bound(v2.intbound))
 
     def optimize_INT_MOD(self, op):
+        v1 = self.getvalue(op.getarg(0))
+        v2 = self.getvalue(op.getarg(1))
+        known_nonneg = (v1.intbound.known_ge(IntBound(0, 0)) and 
+                        v2.intbound.known_ge(IntBound(0, 0)))
+        if known_nonneg and v2.is_constant():
+            val = v2.box.getint()
+            if (val & (val-1)) == 0:
+                # nonneg % power-of-two ==> nonneg & (power-of-two - 1)
+                arg1 = op.getarg(0)
+                arg2 = ConstInt(val-1)
+                op = op.copy_and_change(rop.INT_AND, args=[arg1, arg2])
         self.emit_operation(op)
-        v2 = self.getvalue(op.getarg(1))
         if v2.is_constant():
             val = v2.box.getint()
             r = self.getvalue(op.result)
             if val < 0:
                 val = -val
-            r.intbound.make_gt(IntBound(-val, -val))
+            if known_nonneg:
+                r.intbound.make_gt(IntBound(0, 0))
+            else:
+                r.intbound.make_gt(IntBound(-val, -val))
             r.intbound.make_lt(IntBound(val, val))
 
     def optimize_INT_LSHIFT(self, op):
@@ -153,9 +167,14 @@
     def optimize_INT_RSHIFT(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
-        self.emit_operation(op)
-        r = self.getvalue(op.result)
-        r.intbound.intersect(v1.intbound.rshift_bound(v2.intbound))
+        b = v1.intbound.rshift_bound(v2.intbound)
+        if b.has_lower and b.has_upper and b.lower == b.upper:
+            # constant result (likely 0, for rshifts that kill all bits)
+            self.make_constant_int(op.result, b.lower)
+        else:
+            self.emit_operation(op)
+            r = self.getvalue(op.result)
+            r.intbound.intersect(b)
 
     def optimize_INT_ADD_OVF(self, op):
         v1 = self.getvalue(op.getarg(0))
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -9,6 +9,7 @@
 from pypy.jit.metainterp.history import AbstractDescr, ConstInt, BoxInt
 from pypy.jit.metainterp import executor, compile, resume, history
 from pypy.jit.metainterp.resoperation import rop, opname, ResOperation
+from pypy.rlib.rarithmetic import LONG_BIT
 
 
 def test_store_final_boxes_in_guard():
@@ -4714,11 +4715,11 @@
         i5 = int_ge(i0, 0)
         guard_true(i5) []
         i1 = int_mod(i0, 42)
-        i2 = int_rshift(i1, 63)
+        i2 = int_rshift(i1, %d)
         i3 = int_and(42, i2)
         i4 = int_add(i1, i3)
         finish(i4)
-        """
+        """ % (LONG_BIT-1)
         expected = """
         [i0]
         i5 = int_ge(i0, 0)
@@ -4726,21 +4727,41 @@
         i1 = int_mod(i0, 42)
         finish(i1)
         """
-        py.test.skip("in-progress")
         self.optimize_loop(ops, expected)
 
-        # Also, 'n % power-of-two' can be turned into int_and(),
-        # but that's a bit harder to detect here because it turns into
-        # several operations, and of course it is wrong to just turn
+        # 'n % power-of-two' can be turned into int_and(); at least that's
+        # easy to do now if n is known to be non-negative.
+        ops = """
+        [i0]
+        i5 = int_ge(i0, 0)
+        guard_true(i5) []
+        i1 = int_mod(i0, 8)
+        i2 = int_rshift(i1, %d)
+        i3 = int_and(42, i2)
+        i4 = int_add(i1, i3)
+        finish(i4)
+        """ % (LONG_BIT-1)
+        expected = """
+        [i0]
+        i5 = int_ge(i0, 0)
+        guard_true(i5) []
+        i1 = int_and(i0, 7)
+        finish(i1)
+        """
+        self.optimize_loop(ops, expected)
+
+        # Of course any 'maybe-negative % power-of-two' can be turned into
+        # int_and(), but that's a bit harder to detect here because it turns
+        # into several operations, and of course it is wrong to just turn
         # int_mod(i0, 16) into int_and(i0, 15).
         ops = """
         [i0]
         i1 = int_mod(i0, 16)
-        i2 = int_rshift(i1, 63)
+        i2 = int_rshift(i1, %d)
         i3 = int_and(16, i2)
         i4 = int_add(i1, i3)
         finish(i4)
-        """
+        """ % (LONG_BIT-1)
         expected = """
         [i0]
         i4 = int_and(i0, 15)
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -4783,6 +4783,52 @@
         """
         self.optimize_loop(ops, expected)
 
+
+    def test_division_nonneg(self):
+        py.test.skip("harder")
+        # this is how an app-level division turns into right now
+        ops = """
+        [i4]
+        i1 = int_ge(i4, 0)
+        guard_true(i1) []
+        i16 = int_floordiv(i4, 3)
+        i18 = int_mul(i16, 3)
+        i19 = int_sub(i4, i18)
+        i21 = int_rshift(i19, %d)
+        i22 = int_add(i16, i21)
+        finish(i22)
+        """ % (LONG_BIT-1)
+        expected = """
+        [i4]
+        i1 = int_ge(i4, 0)
+        guard_true(i1) []
+        i16 = int_floordiv(i4, 3)
+        finish(i16)
+        """
+        self.optimize_loop(ops, expected)
+
+    def test_division_by_2(self):
+        py.test.skip("harder")
+        ops = """
+        [i4]
+        i1 = int_ge(i4, 0)
+        guard_true(i1) []
+        i16 = int_floordiv(i4, 2)
+        i18 = int_mul(i16, 2)
+        i19 = int_sub(i4, i18)
+        i21 = int_rshift(i19, %d)
+        i22 = int_add(i16, i21)
+        finish(i22)
+        """ % (LONG_BIT-1)
+        expected = """
+        [i4]
+        i1 = int_ge(i4, 0)
+        guard_true(i1) []
+        i16 = int_rshift(i4, 1)
+        finish(i16)
+        """
+        self.optimize_loop(ops, expected)
+
     def test_subsub_ovf(self):
         ops = """
         [i0]

From noreply at buildbot.pypy.org  Sun Oct 30 15:59:08 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 30 Oct 2011 15:59:08 +0100 (CET)
Subject: [pypy-commit] pypy stm: Move and add comments.
Message-ID: <20111030145908.48979820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm
Changeset: r48607:795b3e474cce
Date: 2011-10-30 15:58 +0100
http://bitbucket.org/pypy/pypy/changeset/795b3e474cce/

Log:	Move and add comments.

diff --git a/pypy/translator/stm/llstminterp.py b/pypy/translator/stm/llstminterp.py
--- a/pypy/translator/stm/llstminterp.py
+++ b/pypy/translator/stm/llstminterp.py
@@ -74,6 +74,13 @@
             self.check_stm_mode(lambda m: False)
             xxx
 
+    def opstm_malloc(self, TYPE, flags):
+        # non-GC must not occur in a regular transaction,
+        # but can occur in inevitable mode or outside a transaction
+        if flags['flavor'] != 'gc':
+            self.check_stm_mode(lambda m: m != "regular_transaction")
+        return LLFrame.op_malloc(self, TYPE, flags)
+
     # ---------- stm-only operations ----------
     # Note that for these tests we assume no real multithreading,
     # so that we just emulate the operations the easy way
@@ -108,8 +115,3 @@
     def opstm_stm_try_inevitable(self):
         self.check_stm_mode(lambda m: m != "not_in_transaction")
         self.llinterpreter.stm_mode = "inevitable_transaction"
-
-    def opstm_malloc(self, TYPE, flags):
-        if flags['flavor'] != 'gc':
-            self.check_stm_mode(lambda m: m != "regular_transaction")
-        return LLFrame.op_malloc(self, TYPE, flags)

From noreply at buildbot.pypy.org  Sun Oct 30 17:05:40 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 30 Oct 2011 17:05:40 +0100 (CET)
Subject: [pypy-commit] pypy stm: Kill the explicit C code manipulating GC
 strings and lists at
Message-ID: <20111030160540.77577820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm
Changeset: r48608:b66d9e6e0eed
Date: 2011-10-09 19:59 +0200
http://bitbucket.org/pypy/pypy/changeset/b66d9e6e0eed/

Log:	Kill the explicit C code manipulating GC strings and lists at start-
	up, and replace it with RPython code. (transplanted from
	c682e7a8ccce6595773aaaafc28eeafab75eee51)

diff --git a/pypy/translator/c/extfunc.py b/pypy/translator/c/extfunc.py
--- a/pypy/translator/c/extfunc.py
+++ b/pypy/translator/c/extfunc.py
@@ -32,47 +32,19 @@
 
 def predeclare_utility_functions(db, rtyper):
     # Common utility functions
+    # (nowadays we are left with only this one function)
     def RPyString_New(length=lltype.Signed):
         return mallocstr(length)
 
-    # !!!
-    # be extremely careful passing a gc tracked object
-    # from such an helper result to another one
-    # as argument, this could result in leaks
-    # Such result should be only from C code
-    # returned directly as results
-
-    LIST_OF_STR = find_list_of_str(rtyper)
-    if LIST_OF_STR is not None:
-        p = lltype.Ptr(LIST_OF_STR)
-
-        def _RPyListOfString_New(length=lltype.Signed):
-            return LIST_OF_STR.ll_newlist(length)
-
-        def _RPyListOfString_New(length=lltype.Signed):
-            return LIST_OF_STR.ll_newlist(length)
-
-        def _RPyListOfString_SetItem(l=p,
-                                    index=lltype.Signed,
-                                    newstring=lltype.Ptr(STR)):
-            rlist.ll_setitem_nonneg(rlist.dum_nocheck, l, index, newstring)
-
-        def _RPyListOfString_GetItem(l=p,
-                                    index=lltype.Signed):
-            return rlist.ll_getitem_fast(l, index)
-
-        def _RPyListOfString_Length(l=p):
-            return rlist.ll_length(l)
-
     for fname, f in locals().items():
         if isinstance(f, types.FunctionType):
             # XXX this is painful :(
-            if (LIST_OF_STR, fname) in db.helper2ptr:
-                yield (fname, db.helper2ptr[LIST_OF_STR, fname])
+            if ("utility", fname) in db.helper2ptr:
+                yield (fname, db.helper2ptr["utility", fname])
             else:
                 # hack: the defaults give the type of the arguments
                 graph = rtyper.annotate_helper(f, f.func_defaults)
-                db.helper2ptr[LIST_OF_STR, fname] = graph
+                db.helper2ptr["utility", fname] = graph
                 yield (fname, graph)
 
 
diff --git a/pypy/translator/c/genc.py b/pypy/translator/c/genc.py
--- a/pypy/translator/c/genc.py
+++ b/pypy/translator/c/genc.py
@@ -427,6 +427,7 @@
     split = True
     executable_name = None
     shared_library_name = None
+    _entrypoint_wrapper = None
 
     def getprofbased(self):
         profbased = None
@@ -449,8 +450,29 @@
     def getentrypointptr(self):
         # XXX check that the entrypoint has the correct
         # signature:  list-of-strings -> int
-        bk = self.translator.annotator.bookkeeper
-        return getfunctionptr(bk.getdesc(self.entrypoint).getuniquegraph())
+        if self._entrypoint_wrapper is not None:
+            return self._entrypoint_wrapper
+        #
+        from pypy.annotation import model as annmodel
+        from pypy.rpython.lltypesystem import rffi
+        from pypy.rpython.annlowlevel import MixLevelHelperAnnotator
+        entrypoint = self.entrypoint
+        #
+        def entrypoint_wrapper(argc, argv):
+            list = [""] * argc
+            i = 0
+            while i < argc:
+                list[i] = rffi.charp2str(argv[i])
+                i += 1
+            return entrypoint(list)
+        #
+        mix = MixLevelHelperAnnotator(self.translator.rtyper)
+        args_s = [annmodel.SomeInteger(),
+                  annmodel.lltype_to_annotation(rffi.CCHARPP)]
+        s_result = annmodel.SomeInteger()
+        graph = mix.getgraph(entrypoint_wrapper, args_s, s_result)
+        mix.finish()
+        return getfunctionptr(graph)
 
     def cmdexec(self, args='', env=None, err=False, expect_crash=False):
         assert self._compiled
diff --git a/pypy/translator/c/src/main.h b/pypy/translator/c/src/main.h
--- a/pypy/translator/c/src/main.h
+++ b/pypy/translator/c/src/main.h
@@ -51,15 +51,7 @@
     errmsg = RPython_StartupCode();
     if (errmsg) goto error;
 
-    list = _RPyListOfString_New(argc);
-    if (RPyExceptionOccurred()) goto memory_out;
-    for (i=0; i<argc; i++) {
-        RPyString *s = RPyString_FromString(argv[i]);
-        if (RPyExceptionOccurred()) goto memory_out;
-        _RPyListOfString_SetItem(list, i, s);
-    }
-
-    exitcode = STANDALONE_ENTRY_POINT(list);
+    exitcode = STANDALONE_ENTRY_POINT(argc, argv);
 
 #ifdef RPY_ASSERT
     pypy_debug_alloc_results();

From noreply at buildbot.pypy.org  Sun Oct 30 17:05:41 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 30 Oct 2011 17:05:41 +0100 (CET)
Subject: [pypy-commit] pypy stm: Progress.
Message-ID: <20111030160541.A51BB820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm
Changeset: r48609:e6522686416a
Date: 2011-10-30 17:03 +0100
http://bitbucket.org/pypy/pypy/changeset/e6522686416a/

Log:	Progress.

diff --git a/pypy/translator/c/genc.py b/pypy/translator/c/genc.py
--- a/pypy/translator/c/genc.py
+++ b/pypy/translator/c/genc.py
@@ -134,7 +134,7 @@
         if self.config.translation.stm:
             from pypy.translator.stm import transform
             transformer = transform.STMTransformer(self.translator)
-            transformer.transform()
+            transformer.transform(self.getentrypointptr())
             log.info("Software Transactional Memory transformation applied")
 
         gcpolicyclass = self.get_gcpolicyclass()
@@ -472,7 +472,9 @@
         s_result = annmodel.SomeInteger()
         graph = mix.getgraph(entrypoint_wrapper, args_s, s_result)
         mix.finish()
-        return getfunctionptr(graph)
+        res = getfunctionptr(graph)
+        self._entrypoint_wrapper = res
+        return res
 
     def cmdexec(self, args='', env=None, err=False, expect_crash=False):
         assert self._compiled
diff --git a/pypy/translator/stm/funcgen.py b/pypy/translator/stm/funcgen.py
--- a/pypy/translator/stm/funcgen.py
+++ b/pypy/translator/stm/funcgen.py
@@ -1,6 +1,6 @@
 from pypy.rpython.lltypesystem import lltype, rffi
 from pypy.objspace.flow.model import Constant
-from pypy.translator.c.support import cdecl
+from pypy.translator.c.support import cdecl, c_string_constant
 from pypy.translator.stm.rstm import size_of_voidp
 
 
@@ -105,7 +105,9 @@
     return 'STM_TRANSACTION_BOUNDARY();'
 
 def stm_try_inevitable(funcgen, op):
-    return 'stm_try_inevitable();'
+    info = op.args[0].value
+    string_literal = c_string_constant(info)
+    return 'stm_try_inevitable(STM_EXPLAIN1(%s));' % (string_literal,)
 
 
 def op_stm(funcgen, op):
diff --git a/pypy/translator/stm/src_stm/et.c b/pypy/translator/stm/src_stm/et.c
--- a/pypy/translator/stm/src_stm/et.c
+++ b/pypy/translator/stm/src_stm/et.c
@@ -21,6 +21,7 @@
 
 #include "src_stm/et.h"
 #include "src_stm/atomic_ops.h"
+#include "src/debug_print.h"
 
 /************************************************************/
 
@@ -537,6 +538,7 @@
 
 void stm_descriptor_init(void)
 {
+  PYPY_DEBUG_START("stm-init");
   if (thread_descriptor != NULL)
     thread_descriptor->init_counter++;
   else
@@ -554,6 +556,7 @@
 
       thread_descriptor = d;
     }
+  PYPY_DEBUG_STOP("stm-init");
 }
 
 void stm_descriptor_done(void)
@@ -563,36 +566,40 @@
   if (d->init_counter > 0)
     return;
 
+  PYPY_DEBUG_START("stm-done");
   thread_descriptor = NULL;
 
-  int num_aborts = 0, num_spinloops = 0;
-  int i, prevchar;
-  for (i=0; i<ABORT_REASONS; i++)
-    num_aborts += d->num_aborts[i];
-  for (i=0; i<SPINLOOP_REASONS; i++)
-    num_spinloops += d->num_spinloops[i];
+  if (PYPY_HAVE_DEBUG_PRINTS) {
+    int num_aborts = 0, num_spinloops = 0;
+    int i, prevchar;
+    for (i=0; i<ABORT_REASONS; i++)
+      num_aborts += d->num_aborts[i];
+    for (i=0; i<SPINLOOP_REASONS; i++)
+      num_spinloops += d->num_spinloops[i];
 
-  fprintf(stderr, "thread %lx: %d commits, %d aborts ",
-          d->my_lock_word,
-          d->num_commits,
-          num_aborts);
+    fprintf(PYPY_DEBUG_FILE, "thread %lx: %d commits, %d aborts ",
+            d->my_lock_word,
+            d->num_commits,
+            num_aborts);
 
-  for (i=0; i<ABORT_REASONS; i++)
-    fprintf(stderr, "%c%d", i == 0 ? '[' : ',',
-            d->num_aborts[i]);
+    for (i=0; i<ABORT_REASONS; i++)
+      fprintf(PYPY_DEBUG_FILE, "%c%d", i == 0 ? '[' : ',',
+              d->num_aborts[i]);
 
-  for (i=1; i<SPINLOOP_REASONS; i++)  /* num_spinloops[0] == num_aborts */
-    fprintf(stderr, "%c%d", i == 1 ? '|' : ',',
-            d->num_spinloops[i]);
+    for (i=1; i<SPINLOOP_REASONS; i++)  /* num_spinloops[0] == num_aborts */
+      fprintf(PYPY_DEBUG_FILE, "%c%d", i == 1 ? '|' : ',',
+              d->num_spinloops[i]);
 
 #ifdef COMMIT_OTHER_INEV
-  for (i=0; i<OTHERINEV_REASONS; i++)
-    fprintf(stderr, "%c%d", i == 0 ? '|' : ',',
-            d->num_otherinev[i]);
+    for (i=0; i<OTHERINEV_REASONS; i++)
+      fprintf(PYPY_DEBUG_FILE, "%c%d", i == 0 ? '|' : ',',
+              d->num_otherinev[i]);
 #endif
 
-  fprintf(stderr, "]\n");
+    fprintf(PYPY_DEBUG_FILE, "]\n");
+  }
   free(d);
+  PYPY_DEBUG_STOP("stm-done");
 }
 
 void* stm_perform_transaction(void*(*callback)(void*), void *arg)
@@ -690,7 +697,7 @@
   return d->end_time;
 }
 
-void stm_try_inevitable(void)
+void stm_try_inevitable(STM_CCHARP1(why))
 {
   /* when a transaction is inevitable, its start_time is equal to
      global_timestamp and global_timestamp cannot be incremented
@@ -699,11 +706,22 @@
   struct tx_descriptor *d = thread_descriptor;
 
 #ifdef RPY_ASSERT
+  PYPY_DEBUG_START("stm-inevitable");
+  if (PYPY_HAVE_DEBUG_PRINTS)
+    {
+      fprintf(PYPY_DEBUG_FILE, "%s%s\n", why,
+              is_inevitable(d) ? " (already inevitable)" : "");
+    }
   assert(d->transaction_active);
 #endif
 
   if (is_inevitable(d))
-    return;  /* I am already inevitable */
+    {
+#ifdef RPY_ASSERT
+      PYPY_DEBUG_STOP("stm-inevitable");
+#endif
+      return;  /* I am already inevitable */
+    }
 
   while (1)
     {
@@ -740,13 +758,16 @@
   CFENCE;
   d_inev_checking = 1;
 #endif
+#ifdef RPY_ASSERT
+  PYPY_DEBUG_STOP("stm-inevitable");
+#endif
 }
 
-void stm_try_inevitable_if(jmp_buf *buf)
+void stm_try_inevitable_if(jmp_buf *buf  STM_CCHARP(why))
 {
   struct tx_descriptor *d = thread_descriptor;
-  if (d && d->setjmp_buf == buf)    /* XXX remove the test for 'd != NULL' */
-    stm_try_inevitable();
+  if (d->setjmp_buf == buf)
+    stm_try_inevitable(STM_EXPLAIN1(why));
 }
 
 void stm_begin_inevitable_transaction(void)
@@ -793,9 +814,15 @@
 
 void stm_transaction_boundary(jmp_buf* buf)
 {
+#ifdef RPY_ASSERT
+  PYPY_DEBUG_START("stm-boundary");
+#endif
   stm_commit_transaction();
   setjmp(*buf);
   stm_begin_transaction(buf);
+#ifdef RPY_ASSERT
+  PYPY_DEBUG_STOP("stm-boundary");
+#endif
 }
 
 // XXX little-endian only!
diff --git a/pypy/translator/stm/src_stm/et.h b/pypy/translator/stm/src_stm/et.h
--- a/pypy/translator/stm/src_stm/et.h
+++ b/pypy/translator/stm/src_stm/et.h
@@ -11,6 +11,18 @@
 #include <setjmp.h>
 #include "src/commondefs.h"
 
+#ifdef RPY_ASSERT
+#  define STM_CCHARP(arg)     , char* arg
+#  define STM_CCHARP1(arg)    char* arg
+#  define STM_EXPLAIN(info)   , info
+#  define STM_EXPLAIN1(info)  info
+#else
+#  define STM_CCHARP(arg)     /* nothing */
+#  define STM_CCHARP1(arg)    void
+#  define STM_EXPLAIN(info)   /* nothing */
+#  define STM_EXPLAIN1(info)  /* nothing */
+#endif
+
 
 void stm_descriptor_init(void);
 void stm_descriptor_done(void);
@@ -19,8 +31,8 @@
 long stm_commit_transaction(void);
 long stm_read_word(long* addr);
 void stm_write_word(long* addr, long val);
-void stm_try_inevitable(void);
-void stm_try_inevitable_if(jmp_buf* buf);
+void stm_try_inevitable(STM_CCHARP1(why));
+void stm_try_inevitable_if(jmp_buf* buf  STM_CCHARP(why));
 void stm_begin_inevitable_transaction(void);
 void stm_abort_and_retry(void);
 void stm_transaction_boundary(jmp_buf* buf);
@@ -31,8 +43,9 @@
        setjmp(_jmpbuf);                   \
        stm_begin_transaction(&_jmpbuf)
 
-#define STM_DECLARE_VARIABLE()          jmp_buf jmpbuf
-#define STM_MAKE_INEVITABLE()           stm_try_inevitable_if(&jmpbuf)
+#define STM_DECLARE_VARIABLE()          ; jmp_buf jmpbuf
+#define STM_MAKE_INEVITABLE()           stm_try_inevitable_if(&jmpbuf  \
+                                                        STM_EXPLAIN("return"))
 #define STM_TRANSACTION_BOUNDARY()      stm_transaction_boundary(&jmpbuf)
 
 // XXX little-endian only!
diff --git a/pypy/translator/stm/test/targetdemo.py b/pypy/translator/stm/test/targetdemo.py
new file mode 100644
--- /dev/null
+++ b/pypy/translator/stm/test/targetdemo.py
@@ -0,0 +1,22 @@
+#from pypy.module.thread import ll_thread
+from pypy.translator.stm import rstm
+
+
+
+
+
+# __________  Entry point  __________
+
+def entry_point(argv):
+    print "hello world"
+    rstm.transaction_boundary()
+    i = 100
+    while i > 1:
+        i *= 0.821
+    rstm.transaction_boundary()
+    return 0
+
+# _____ Define and setup target ___
+
+def target(*args):
+    return entry_point, None
diff --git a/pypy/translator/stm/transform.py b/pypy/translator/stm/transform.py
--- a/pypy/translator/stm/transform.py
+++ b/pypy/translator/stm/transform.py
@@ -1,11 +1,13 @@
-from pypy.objspace.flow.model import SpaceOperation
+from pypy.objspace.flow.model import SpaceOperation, Constant
+from pypy.annotation import model as annmodel
+from pypy.rpython.annlowlevel import MixLevelHelperAnnotator
 from pypy.translator.stm import _rffi_stm
 from pypy.translator.unsimplify import varoftype
 from pypy.rpython.lltypesystem import lltype
 
 
 ALWAYS_ALLOW_OPERATIONS = set([
-    'int_*', 'uint_*', 'llong_*', 'ullong_*',
+    'int_*', 'uint_*', 'llong_*', 'ullong_*', 'float_*',
     'same_as', 'cast_*',
     'direct_call',
     'debug_print', 'debug_assert',
@@ -27,13 +29,17 @@
     def __init__(self, translator=None):
         self.translator = translator
 
-    def transform(self):
+    def transform(self, entrypointptr):
+        assert not hasattr(self.translator, 'stm_transformation_applied')
+        entrypointgraph = entrypointptr._obj.graph
         for graph in self.translator.graphs:
+            if graph is entrypointgraph:
+                continue
             self.seen_transaction_boundary = False
             self.transform_graph(graph)
             if self.seen_transaction_boundary:
                 self.add_stm_declare_variable(graph)
-        self.add_descriptor_init_stuff()
+        self.add_descriptor_init_stuff(entrypointgraph)
         self.translator.stm_transformation_applied = True
 
     def transform_block(self, block):
@@ -64,17 +70,24 @@
         for block in graph.iterblocks():
             self.transform_block(block)
 
-    def add_descriptor_init_stuff(self):
-        from pypy.translator.unsimplify import call_initial_function
-        from pypy.translator.unsimplify import call_final_function
+    def add_descriptor_init_stuff(self, entrypointgraph):
+        #
         def descriptor_init():
             _rffi_stm.descriptor_init()
             _rffi_stm.begin_inevitable_transaction()
-        def descriptor_done():
-            _rffi_stm.commit_transaction()
-            _rffi_stm.descriptor_done()
-        call_initial_function(self.translator, descriptor_init)
-        call_final_function(self.translator, descriptor_done)
+        #def descriptor_done():
+        #    _rffi_stm.commit_transaction()
+        #    _rffi_stm.descriptor_done()
+        #
+        annhelper = MixLevelHelperAnnotator(self.translator.rtyper)
+        c_init = annhelper.constfunc(descriptor_init, [], annmodel.s_None)
+        #c_done = annhelper.constfunc(descriptor_done, [], annmodel.s_None)
+        annhelper.finish()
+        block = entrypointgraph.startblock
+        v = varoftype(lltype.Void)
+        op = SpaceOperation('direct_call', [c_init], v)
+        block.operations.insert(0, op)
+        #...add c_done...
 
     def add_stm_declare_variable(self, graph):
         block = graph.startblock
@@ -111,6 +124,8 @@
 
 
 def turn_inevitable_and_proceed(newoperations, op):
-    op1 = SpaceOperation('stm_try_inevitable', [], varoftype(lltype.Void))
+    c_info = Constant(op.opname, lltype.Void)
+    op1 = SpaceOperation('stm_try_inevitable', [c_info],
+                         varoftype(lltype.Void))
     newoperations.append(op1)
     newoperations.append(op)

From noreply at buildbot.pypy.org  Sun Oct 30 17:16:28 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 30 Oct 2011 17:16:28 +0100 (CET)
Subject: [pypy-commit] pypy stm: Protect debug printing with the
	RPY_STM_ASSERT define.
Message-ID: <20111030161628.D52AE820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm
Changeset: r48610:43fe948fc6ca
Date: 2011-10-30 17:13 +0100
http://bitbucket.org/pypy/pypy/changeset/43fe948fc6ca/

Log:	Protect debug printing with the RPY_STM_ASSERT define. Lets us run
	tests.

diff --git a/pypy/translator/c/genc.py b/pypy/translator/c/genc.py
--- a/pypy/translator/c/genc.py
+++ b/pypy/translator/c/genc.py
@@ -207,7 +207,8 @@
     # use generate_source(defines=DEBUG_DEFINES) to force the #definition
     # of the macros that enable debugging assertions
     DEBUG_DEFINES = {'RPY_ASSERT': 1,
-                     'RPY_LL_ASSERT': 1}
+                     'RPY_LL_ASSERT': 1,
+                     'RPY_STM_ASSERT': 1}
 
     def generate_graphs_for_llinterp(self, db=None):
         # prepare the graphs as when the source is generated, but without
@@ -561,7 +562,7 @@
             ('no_obmalloc', '', '$(MAKE) CFLAGS="-g -O2 -DRPY_ASSERT -DNO_OBMALLOC" $(TARGET)'),
             ('linuxmemchk', '', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -DRPY_ASSERT -DLINUXMEMCHK" $(TARGET)'),
             ('llsafer', '', '$(MAKE) CFLAGS="-O2 -DRPY_LL_ASSERT" $(TARGET)'),
-            ('lldebug', '', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -DRPY_ASSERT -DRPY_LL_ASSERT" $(TARGET)'),
+            ('lldebug', '', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -DRPY_ASSERT -DRPY_LL_ASSERT -DRPY_STM_ASSERT" $(TARGET)'),
             ('profile', '', '$(MAKE) CFLAGS="-g -O1 -pg $(CFLAGS) -fno-omit-frame-pointer" LDFLAGS="-pg $(LDFLAGS)" $(TARGET)'),
             ]
         if self.has_profopt():
diff --git a/pypy/translator/stm/llstminterp.py b/pypy/translator/stm/llstminterp.py
--- a/pypy/translator/stm/llstminterp.py
+++ b/pypy/translator/stm/llstminterp.py
@@ -112,6 +112,7 @@
         self.llinterpreter.stm_mode = "regular_transaction"
         self.llinterpreter.last_transaction_started_in_frame = self
 
-    def opstm_stm_try_inevitable(self):
+    def opstm_stm_try_inevitable(self, why):
         self.check_stm_mode(lambda m: m != "not_in_transaction")
         self.llinterpreter.stm_mode = "inevitable_transaction"
+        print why
diff --git a/pypy/translator/stm/src_stm/et.c b/pypy/translator/stm/src_stm/et.c
--- a/pypy/translator/stm/src_stm/et.c
+++ b/pypy/translator/stm/src_stm/et.c
@@ -21,7 +21,10 @@
 
 #include "src_stm/et.h"
 #include "src_stm/atomic_ops.h"
-#include "src/debug_print.h"
+
+#ifdef RPY_STM_ASSERT
+# include "src/debug_print.h"
+#endif
 
 /************************************************************/
 
@@ -44,7 +47,7 @@
 inline static volatile orec_t* get_orec(void* addr)
 {
   unsigned long index = (unsigned long)addr;
-#ifdef RPY_ASSERT
+#ifdef RPY_STM_ASSERT
   assert(!(index & (sizeof(orec_t)-1)));
 #endif
   char *p = orecs + (index & ((NUM_STRIPES-1) * sizeof(orec_t)));
@@ -78,7 +81,7 @@
   owner_version_t my_lock_word;
   unsigned init_counter;
   struct RedoLog redolog;   /* last item, because it's the biggest one */
-#ifdef RPY_ASSERT
+#ifdef RPY_STM_ASSERT
   int transaction_active;
 #endif
 };
@@ -242,7 +245,7 @@
 {
   d->reads.size = 0;
   redolog_clear(&d->redolog);
-#ifdef RPY_ASSERT
+#ifdef RPY_STM_ASSERT
   assert(d->transaction_active);
   d->transaction_active = 0;
 #endif
@@ -538,7 +541,9 @@
 
 void stm_descriptor_init(void)
 {
+#ifdef RPY_STM_ASSERT
   PYPY_DEBUG_START("stm-init");
+#endif
   if (thread_descriptor != NULL)
     thread_descriptor->init_counter++;
   else
@@ -556,7 +561,9 @@
 
       thread_descriptor = d;
     }
+#ifdef RPY_STM_ASSERT
   PYPY_DEBUG_STOP("stm-init");
+#endif
 }
 
 void stm_descriptor_done(void)
@@ -566,9 +573,10 @@
   if (d->init_counter > 0)
     return;
 
-  PYPY_DEBUG_START("stm-done");
   thread_descriptor = NULL;
 
+#ifdef RPY_STM_ASSERT
+  PYPY_DEBUG_START("stm-done");
   if (PYPY_HAVE_DEBUG_PRINTS) {
     int num_aborts = 0, num_spinloops = 0;
     int i, prevchar;
@@ -598,8 +606,10 @@
 
     fprintf(PYPY_DEBUG_FILE, "]\n");
   }
+  PYPY_DEBUG_STOP("stm-done");
+#endif
+
   free(d);
-  PYPY_DEBUG_STOP("stm-done");
 }
 
 void* stm_perform_transaction(void*(*callback)(void*), void *arg)
@@ -614,7 +624,7 @@
 void stm_begin_transaction(jmp_buf* buf)
 {
   struct tx_descriptor *d = thread_descriptor;
-#ifdef RPY_ASSERT
+#ifdef RPY_STM_ASSERT
   assert(!d->transaction_active);
   d->transaction_active = 1;
 #endif
@@ -705,7 +715,7 @@
      to 1. */
   struct tx_descriptor *d = thread_descriptor;
 
-#ifdef RPY_ASSERT
+#ifdef RPY_STM_ASSERT
   PYPY_DEBUG_START("stm-inevitable");
   if (PYPY_HAVE_DEBUG_PRINTS)
     {
@@ -717,7 +727,7 @@
 
   if (is_inevitable(d))
     {
-#ifdef RPY_ASSERT
+#ifdef RPY_STM_ASSERT
       PYPY_DEBUG_STOP("stm-inevitable");
 #endif
       return;  /* I am already inevitable */
@@ -758,7 +768,7 @@
   CFENCE;
   d_inev_checking = 1;
 #endif
-#ifdef RPY_ASSERT
+#ifdef RPY_STM_ASSERT
   PYPY_DEBUG_STOP("stm-inevitable");
 #endif
 }
@@ -794,7 +804,7 @@
       if (bool_cas(&global_timestamp, curtime, curtime + 1))
         break;
     }
-#ifdef RPY_ASSERT
+#ifdef RPY_STM_ASSERT
   assert(!d->transaction_active);
   d->transaction_active = 1;
 #endif
@@ -814,14 +824,14 @@
 
 void stm_transaction_boundary(jmp_buf* buf)
 {
-#ifdef RPY_ASSERT
-  PYPY_DEBUG_START("stm-boundary");
+#ifdef RPY_STM_ASSERT
+  PYPY_DEBUG_START("stm-transaction-boundary");
 #endif
   stm_commit_transaction();
   setjmp(*buf);
   stm_begin_transaction(buf);
-#ifdef RPY_ASSERT
-  PYPY_DEBUG_STOP("stm-boundary");
+#ifdef RPY_STM_ASSERT
+  PYPY_DEBUG_STOP("stm-transaction-boundary");
 #endif
 }
 
diff --git a/pypy/translator/stm/src_stm/et.h b/pypy/translator/stm/src_stm/et.h
--- a/pypy/translator/stm/src_stm/et.h
+++ b/pypy/translator/stm/src_stm/et.h
@@ -11,7 +11,7 @@
 #include <setjmp.h>
 #include "src/commondefs.h"
 
-#ifdef RPY_ASSERT
+#ifdef RPY_STM_ASSERT
 #  define STM_CCHARP(arg)     , char* arg
 #  define STM_CCHARP1(arg)    char* arg
 #  define STM_EXPLAIN(info)   , info

From noreply at buildbot.pypy.org  Sun Oct 30 18:31:36 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 30 Oct 2011 18:31:36 +0100 (CET)
Subject: [pypy-commit] pypy default: Fix for the JIT: don't use Arrays that
 are not GcArrays but that have a length.
Message-ID: <20111030173136.75EC482A87@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48612:7c12a810b394
Date: 2011-10-30 18:28 +0100
http://bitbucket.org/pypy/pypy/changeset/7c12a810b394/

Log:	Fix for the JIT: don't use Arrays that are not GcArrays but that
	have a length.

diff --git a/pypy/rpython/lltypesystem/rpbc.py b/pypy/rpython/lltypesystem/rpbc.py
--- a/pypy/rpython/lltypesystem/rpbc.py
+++ b/pypy/rpython/lltypesystem/rpbc.py
@@ -149,7 +149,8 @@
         self.descriptions = list(self.s_pbc.descriptions)
         if self.s_pbc.can_be_None:
             self.descriptions.insert(0, None)
-        POINTER_TABLE = Array(self.pointer_repr.lowleveltype)
+        POINTER_TABLE = Array(self.pointer_repr.lowleveltype,
+                              hints={'nolength': True})
         pointer_table = malloc(POINTER_TABLE, len(self.descriptions),
                                immortal=True)
         for i, desc in enumerate(self.descriptions):
@@ -302,7 +303,8 @@
     if r_to in r_from._conversion_tables:
         return r_from._conversion_tables[r_to]
     else:
-        t = malloc(Array(Char), len(r_from.descriptions), immortal=True)
+        t = malloc(Array(Char, hints={'nolength': True}),
+                   len(r_from.descriptions), immortal=True)
         l = []
         for i, d in enumerate(r_from.descriptions):
             if d in r_to.descriptions:

From noreply at buildbot.pypy.org  Sun Oct 30 18:31:35 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 30 Oct 2011 18:31:35 +0100 (CET)
Subject: [pypy-commit] pypy default: A failing test.
Message-ID: <20111030173135.4EDE4820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48611:fd20e5312348
Date: 2011-10-30 18:25 +0100
http://bitbucket.org/pypy/pypy/changeset/fd20e5312348/

Log:	A failing test.

diff --git a/pypy/jit/backend/llgraph/llimpl.py b/pypy/jit/backend/llgraph/llimpl.py
--- a/pypy/jit/backend/llgraph/llimpl.py
+++ b/pypy/jit/backend/llgraph/llimpl.py
@@ -1114,7 +1114,9 @@
     if isinstance(TYPE, lltype.Ptr):
         if isinstance(x, (int, long, llmemory.AddressAsInt)):
             x = llmemory.cast_int_to_adr(x)
-        if TYPE is rffi.VOIDP or TYPE.TO._hints.get("uncast_on_llgraph"):
+        if TYPE is rffi.VOIDP or (
+                hasattr(TYPE.TO, '_hints') and
+                TYPE.TO._hints.get("uncast_on_llgraph")):
             # assume that we want a "C-style" cast, without typechecking the value
             return rffi.cast(TYPE, x)
         return llmemory.cast_adr_to_ptr(x, TYPE)
diff --git a/pypy/jit/backend/llgraph/runner.py b/pypy/jit/backend/llgraph/runner.py
--- a/pypy/jit/backend/llgraph/runner.py
+++ b/pypy/jit/backend/llgraph/runner.py
@@ -365,6 +365,7 @@
 
     def arraydescrof(self, A):
         assert A.OF != lltype.Void
+        assert isinstance(A, lltype.GcArray) or A._hints.get('nolength', False)
         size = symbolic.get_size(A)
         if isinstance(A.OF, lltype.Ptr) or isinstance(A.OF, lltype.Primitive):
             token = history.getkind(A.OF)[0]
diff --git a/pypy/jit/codewriter/support.py b/pypy/jit/codewriter/support.py
--- a/pypy/jit/codewriter/support.py
+++ b/pypy/jit/codewriter/support.py
@@ -40,6 +40,7 @@
              type_system="lltype"):
     # build the normal ll graphs for ll_function
     t = TranslationContext()
+    t.config.translation.withsmallfuncsets = 3
     annpolicy = AnnotatorPolicy()
     annpolicy.allow_someobjects = False
     a = t.buildannotator(policy=annpolicy)
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -3559,6 +3559,33 @@
         assert res == 0
         self.check_loops({"int_sub": 1, "int_gt": 1, "guard_true": 1, "jump": 1})
 
+    def test_convert_from_SmallFunctionSetPBCRepr_to_FunctionsPBCRepr(self):
+        f1 = lambda n: n+1
+        f2 = lambda n: n+2
+        f3 = lambda n: n+3
+        f4 = lambda n: n+4
+        f5 = lambda n: n+5
+        f6 = lambda n: n+6
+        f7 = lambda n: n+7
+        f8 = lambda n: n+8
+        def h(n, x):
+            return x(n)
+        h._dont_inline = True
+        def g(n, x):
+            return h(n, x)
+        g._dont_inline = True
+        def f(n):
+            n = g(n, f1)
+            n = g(n, f2)
+            n = h(n, f3)
+            n = h(n, f4)
+            n = h(n, f5)
+            n = h(n, f6)
+            n = h(n, f7)
+            n = h(n, f8)
+            return n
+        assert f(5) == 41
+        self.interp_operations(f, [5])
 
 
 class TestLLtype(BaseLLtypeTests, LLJitMixin):

From noreply at buildbot.pypy.org  Sun Oct 30 18:36:28 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 30 Oct 2011 18:36:28 +0100 (CET)
Subject: [pypy-commit] pypy default: Revert bfea8fbdeed4: it seems not to
 work correctly. Will investigate.
Message-ID: <20111030173628.24373820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48613:c74b79096726
Date: 2011-10-30 18:36 +0100
http://bitbucket.org/pypy/pypy/changeset/c74b79096726/

Log:	Revert bfea8fbdeed4: it seems not to work correctly. Will
	investigate.

diff --git a/pypy/jit/metainterp/optimizeopt/intbounds.py b/pypy/jit/metainterp/optimizeopt/intbounds.py
--- a/pypy/jit/metainterp/optimizeopt/intbounds.py
+++ b/pypy/jit/metainterp/optimizeopt/intbounds.py
@@ -5,7 +5,6 @@
     IntUpperBound)
 from pypy.jit.metainterp.optimizeopt.util import make_dispatcher_method
 from pypy.jit.metainterp.resoperation import rop
-from pypy.rlib.rarithmetic import LONG_BIT
 
 
 class OptIntBounds(Optimization):
@@ -127,27 +126,14 @@
         r.intbound.intersect(v1.intbound.div_bound(v2.intbound))
 
     def optimize_INT_MOD(self, op):
-        v1 = self.getvalue(op.getarg(0))
+        self.emit_operation(op)
         v2 = self.getvalue(op.getarg(1))
-        known_nonneg = (v1.intbound.known_ge(IntBound(0, 0)) and 
-                        v2.intbound.known_ge(IntBound(0, 0)))
-        if known_nonneg and v2.is_constant():
-            val = v2.box.getint()
-            if (val & (val-1)) == 0:
-                # nonneg % power-of-two ==> nonneg & (power-of-two - 1)
-                arg1 = op.getarg(0)
-                arg2 = ConstInt(val-1)
-                op = op.copy_and_change(rop.INT_AND, args=[arg1, arg2])
-        self.emit_operation(op)
         if v2.is_constant():
             val = v2.box.getint()
             r = self.getvalue(op.result)
             if val < 0:
                 val = -val
-            if known_nonneg:
-                r.intbound.make_gt(IntBound(0, 0))
-            else:
-                r.intbound.make_gt(IntBound(-val, -val))
+            r.intbound.make_gt(IntBound(-val, -val))
             r.intbound.make_lt(IntBound(val, val))
 
     def optimize_INT_LSHIFT(self, op):
@@ -167,14 +153,9 @@
     def optimize_INT_RSHIFT(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
-        b = v1.intbound.rshift_bound(v2.intbound)
-        if b.has_lower and b.has_upper and b.lower == b.upper:
-            # constant result (likely 0, for rshifts that kill all bits)
-            self.make_constant_int(op.result, b.lower)
-        else:
-            self.emit_operation(op)
-            r = self.getvalue(op.result)
-            r.intbound.intersect(b)
+        self.emit_operation(op)
+        r = self.getvalue(op.result)
+        r.intbound.intersect(v1.intbound.rshift_bound(v2.intbound))
 
     def optimize_INT_ADD_OVF(self, op):
         v1 = self.getvalue(op.getarg(0))
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -9,7 +9,6 @@
 from pypy.jit.metainterp.history import AbstractDescr, ConstInt, BoxInt
 from pypy.jit.metainterp import executor, compile, resume, history
 from pypy.jit.metainterp.resoperation import rop, opname, ResOperation
-from pypy.rlib.rarithmetic import LONG_BIT
 
 
 def test_store_final_boxes_in_guard():
@@ -4715,11 +4714,11 @@
         i5 = int_ge(i0, 0)
         guard_true(i5) []
         i1 = int_mod(i0, 42)
-        i2 = int_rshift(i1, %d)
+        i2 = int_rshift(i1, 63)
         i3 = int_and(42, i2)
         i4 = int_add(i1, i3)
         finish(i4)
-        """ % (LONG_BIT-1)
+        """
         expected = """
         [i0]
         i5 = int_ge(i0, 0)
@@ -4727,41 +4726,21 @@
         i1 = int_mod(i0, 42)
         finish(i1)
         """
+        py.test.skip("in-progress")
         self.optimize_loop(ops, expected)
 
-        # 'n % power-of-two' can be turned into int_and(); at least that's
-        # easy to do now if n is known to be non-negative.
-        ops = """
-        [i0]
-        i5 = int_ge(i0, 0)
-        guard_true(i5) []
-        i1 = int_mod(i0, 8)
-        i2 = int_rshift(i1, %d)
-        i3 = int_and(42, i2)
-        i4 = int_add(i1, i3)
-        finish(i4)
-        """ % (LONG_BIT-1)
-        expected = """
-        [i0]
-        i5 = int_ge(i0, 0)
-        guard_true(i5) []
-        i1 = int_and(i0, 7)
-        finish(i1)
-        """
-        self.optimize_loop(ops, expected)
-
-        # Of course any 'maybe-negative % power-of-two' can be turned into
-        # int_and(), but that's a bit harder to detect here because it turns
-        # into several operations, and of course it is wrong to just turn
+        # Also, 'n % power-of-two' can be turned into int_and(),
+        # but that's a bit harder to detect here because it turns into
+        # several operations, and of course it is wrong to just turn
         # int_mod(i0, 16) into int_and(i0, 15).
         ops = """
         [i0]
         i1 = int_mod(i0, 16)
-        i2 = int_rshift(i1, %d)
+        i2 = int_rshift(i1, 63)
         i3 = int_and(16, i2)
         i4 = int_add(i1, i3)
         finish(i4)
-        """ % (LONG_BIT-1)
+        """
         expected = """
         [i0]
         i4 = int_and(i0, 15)

From noreply at buildbot.pypy.org  Sun Oct 30 19:23:30 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sun, 30 Oct 2011 19:23:30 +0100 (CET)
Subject: [pypy-commit] extradoc extradoc: updates to numpy planning
Message-ID: <20111030182330.A5815820B3@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: extradoc
Changeset: r3953:97ca19999a70
Date: 2011-10-30 14:23 -0400
http://bitbucket.org/pypy/extradoc/changeset/97ca19999a70/

Log:	updates to numpy planning

diff --git a/planning/micronumpy.txt b/planning/micronumpy.txt
--- a/planning/micronumpy.txt
+++ b/planning/micronumpy.txt
@@ -1,13 +1,6 @@
 NEW TASKS
 ---------
 
-- add in all base dtypes (bools, unsigned and signed ints, floats)
-
-- get the correct dtype results for binops
-
-- add more ways to select dtypes (aliases or more advanced code to choose the 
-  types)
-
 - add in numpy.generic and the various subclasses, use them in returning
   instances from subscripting (and possibly internally), also make them valid
   for the dtype arguments
@@ -26,4 +19,6 @@
 
   - will need to refactor some functions
 
-  - do we keep single-dim arrays separate?
+- frompyfunc to create ufuncs from python functions
+
+- more ufuncs

From noreply at buildbot.pypy.org  Sun Oct 30 19:23:31 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sun, 30 Oct 2011 19:23:31 +0100 (CET)
Subject: [pypy-commit] extradoc extradoc: merged upstream
Message-ID: <20111030182331.C4E5A82A87@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: extradoc
Changeset: r3954:e1a088b4f3e2
Date: 2011-10-30 14:23 -0400
http://bitbucket.org/pypy/extradoc/changeset/e1a088b4f3e2/

Log:	merged upstream

diff --git a/sprintinfo/gothenburg-2011-2/people.txt b/sprintinfo/gothenburg-2011-2/people.txt
--- a/sprintinfo/gothenburg-2011-2/people.txt
+++ b/sprintinfo/gothenburg-2011-2/people.txt
@@ -10,9 +10,10 @@
 ==================== ============== ===================== ==================
 Jacob Hallen         lives there
 Laura Creighton      lives there
-Armin Rigo           ?              ?
-Maciej Fija&#322;kowski   2-8 Nov        ?                     vegetarian
+Armin Rigo           ?              SGS Veckobostader
+Maciej Fija&#322;kowski   2-8 Nov        SGS Veckobostader     vegetarian
 Antonio Cuni         4-10 november  Laura's and Jacob's
 H&#229;kan Ard&#246;           3-8            ?
 Sam Lade             1-9            Hotel Poseidon
+Mark Pearse          2-10           SGS Veckobostader?
 ==================== ============== ===================== ==================

From notifications-noreply at bitbucket.org  Sun Oct 30 19:26:19 2011
From: notifications-noreply at bitbucket.org (Bitbucket)
Date: Sun, 30 Oct 2011 18:26:19 -0000
Subject: [pypy-commit] Notification: pypy
Message-ID: <20111030182619.6109.46869@bitbucket13.managed.contegix.com>

You have received a notification from fprimex.

Hi, I forked pypy. My fork is at https://bitbucket.org/fprimex/pypy.

--
Disable notifications at https://bitbucket.org/account/notifications/

From noreply at buildbot.pypy.org  Sun Oct 30 22:42:42 2011
From: noreply at buildbot.pypy.org (mattip)
Date: Sun, 30 Oct 2011 22:42:42 +0100 (CET)
Subject: [pypy-commit] pypy numpy-multidim: repr and str rework
Message-ID: <20111030214242.E79A3820B3@wyvern.cs.uni-duesseldorf.de>

Author: mattip
Branch: numpy-multidim
Changeset: r48614:723ed1ac5a3a
Date: 2011-10-30 23:40 +0200
http://bitbucket.org/pypy/pypy/changeset/723ed1ac5a3a/

Log:	repr and str rework

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -246,7 +246,10 @@
     def descr_repr(self, space):
         # Simple implementation so that we can see the array. Needs work.
         concrete = self.get_concrete()
-        res = "array([" + ", ".join(concrete._getnums(False)) + "]"
+        new_sig = signature.Signature.find_sig([
+            NDimSlice.signature, self.signature
+        ])
+        res = "array(" + NDimSlice(concrete, new_sig, [], self.shape[:]).tostr(True)
         dtype = concrete.find_dtype()
         if (dtype is not space.fromcache(interp_dtype.W_Float64Dtype) and
             dtype is not space.fromcache(interp_dtype.W_Int64Dtype)) or not self.find_size():
@@ -257,7 +260,10 @@
     def descr_str(self, space):
         # Simple implementation so that we can see the array. Needs work.
         concrete = self.get_concrete()
-        return space.wrap("[" + " ".join(concrete._getnums(True)) + "]")
+        new_sig = signature.Signature.find_sig([
+            NDimSlice.signature, self.signature
+        ])
+        return space.wrap(NDimSlice(concrete, new_sig, [], self.shape[:]).tostr(False))
 
     def _index_of_single_item(self, space, w_idx):
         # we assume C ordering for now
@@ -582,6 +588,7 @@
     _immutable_fields_ = ['shape[*]', 'chunks[*]']
 
     def __init__(self, parent, signature, chunks, shape):
+        print 'NDimSlice.__init__(...,',chunks,',',shape,')'
         ViewArray.__init__(self, parent, signature, shape)
         self.chunks = chunks
         self.shape_reduction = 0
@@ -628,6 +635,7 @@
     @jit.unroll_safe
     def calc_index(self, item):
         index = []
+        __item = item
         _item = item
         for i in range(len(self.shape) -1, 0, -1):
             s = self.shape[i]
@@ -654,11 +662,49 @@
             if k != 0:
                 item *= shape[k]
             k += 1
-            item += index[i]
+            try:
+                item += index[i]
+            except:
+               import pdb;pdb.set_trace()
             i += 1
         return item
-
-
+    def tostr(self, comma):
+        ret = ''
+        dtype = self.find_dtype()
+        ndims = len(self.shape)#-self.shape_reduction
+        if ndims>2:
+            ret += '['
+            for i in range(self.shape[0]):
+                res = NDimSlice(self.parent, self.signature, [(i,0,0,1)], self.shape[1:]).tostr(comma)
+                ret += '\n '.join(res.split('\n'))
+                if i+1<self.shape[0]:
+                    ret += ',\n\n'
+            ret += ']'
+        elif ndims==2:
+            ret += '['
+            for i in range(self.shape[0]):
+                ret += '['
+                ret += (','*comma + ' ' ).join([dtype.str_format(self.eval(i*self.shape[1]+j)) \
+                                                    for j in range(self.shape[1])])
+                ret += ']'
+                if i+1< self.shape[0]:
+                    ret += ',\n       '
+            ret += ']'
+        elif ndims==1:
+            ret += '['
+            if self.shape[0]>1000:
+                ret += (','*comma + ' ').join([dtype.str_format(self.eval(j)) \
+                                                    for j in range(3)])
+                ret += ','*comma + ' ..., '
+                ret += (','*comma + ' ').join([dtype.str_format(self.eval(j)) \
+                                                    for j in range(self.shape[0]-3,self.shape[0])])
+            else:
+                ret += (','*comma + ' ').join([dtype.str_format(self.eval(j)) \
+                                                    for j in range(self.shape[0])])
+            ret += ']'
+        else:
+            ret += 'shape=%s, reduction=%s'%(str(self.shape), str(self.shape_reduction))
+        return ret
 class NDimArray(BaseArray):
     def __init__(self, size, shape, dtype):
         BaseArray.__init__(self, shape)
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -67,6 +67,10 @@
         assert repr(a) == "array([], dtype=int64)"
         a = array([True, False, True, False], "?")
         assert repr(a) == "array([True, False, True, False], dtype=bool)"
+        a = zeros((3,4))
+        assert repr(a) == '''array([[0.0, 0.0, 0.0, 0.0],
+       [0.0, 0.0, 0.0, 0.0],
+       [0.0, 0.0, 0.0, 0.0]])'''
 
     def test_repr_slice(self):
         from numpy import array, zeros

From noreply at buildbot.pypy.org  Sun Oct 30 23:19:03 2011
From: noreply at buildbot.pypy.org (mattip)
Date: Sun, 30 Oct 2011 23:19:03 +0100 (CET)
Subject: [pypy-commit] pypy numpy-multidim: testing
Message-ID: <20111030221903.D81ED820B3@wyvern.cs.uni-duesseldorf.de>

Author: mattip
Branch: numpy-multidim
Changeset: r48615:0d9e53f59c5d
Date: 2011-10-31 00:17 +0200
http://bitbucket.org/pypy/pypy/changeset/0d9e53f59c5d/

Log:	testing

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -249,7 +249,7 @@
         new_sig = signature.Signature.find_sig([
             NDimSlice.signature, self.signature
         ])
-        res = "array(" + NDimSlice(concrete, new_sig, [], self.shape[:]).tostr(True)
+        res = "array(" + NDimSlice(concrete, new_sig, [], self.shape[:]).tostr(True, indent='       ')
         dtype = concrete.find_dtype()
         if (dtype is not space.fromcache(interp_dtype.W_Float64Dtype) and
             dtype is not space.fromcache(interp_dtype.W_Int64Dtype)) or not self.find_size():
@@ -588,7 +588,6 @@
     _immutable_fields_ = ['shape[*]', 'chunks[*]']
 
     def __init__(self, parent, signature, chunks, shape):
-        print 'NDimSlice.__init__(...,',chunks,',',shape,')'
         ViewArray.__init__(self, parent, signature, shape)
         self.chunks = chunks
         self.shape_reduction = 0
@@ -662,21 +661,17 @@
             if k != 0:
                 item *= shape[k]
             k += 1
-            try:
-                item += index[i]
-            except:
-               import pdb;pdb.set_trace()
+            item += index[i]
             i += 1
         return item
-    def tostr(self, comma):
+    def tostr(self, comma,indent=' '):
         ret = ''
         dtype = self.find_dtype()
         ndims = len(self.shape)#-self.shape_reduction
         if ndims>2:
             ret += '['
             for i in range(self.shape[0]):
-                res = NDimSlice(self.parent, self.signature, [(i,0,0,1)], self.shape[1:]).tostr(comma)
-                ret += '\n '.join(res.split('\n'))
+                ret += NDimSlice(self.parent, self.signature, [(i,0,0,1)], self.shape[1:]).tostr(commai,indent=indent+' ')
                 if i+1<self.shape[0]:
                     ret += ',\n\n'
             ret += ']'
@@ -688,7 +683,7 @@
                                                     for j in range(self.shape[1])])
                 ret += ']'
                 if i+1< self.shape[0]:
-                    ret += ',\n       '
+                    ret += ',\n' + indent
             ret += ']'
         elif ndims==1:
             ret += '['
@@ -703,7 +698,7 @@
                                                     for j in range(self.shape[0])])
             ret += ']'
         else:
-            ret += 'shape=%s, reduction=%s'%(str(self.shape), str(self.shape_reduction))
+            ret += '[]'
         return ret
 class NDimArray(BaseArray):
     def __init__(self, size, shape, dtype):
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -80,6 +80,9 @@
         a = zeros(2002)
         b = a[::2]
         assert repr(b) == "array([0.0, 0.0, 0.0, ..., 0.0, 0.0, 0.0])"
+        a = array((range(5),range(5,10)), dtype="int16")
+        b=a[1,2:]
+        assert repr(b) == "array([7, 8, 9], dtype=int16)"
 
     def test_str(self):
         from numpy import array, zeros
@@ -100,6 +103,9 @@
         a = array(range(5), dtype="int16")
         assert str(a) == "[0 1 2 3 4]"
 
+        a = array((range(5),range(5,10)), dtype="int16")
+        assert str(a) == "[[0 1 2 3 4],\n [5 6 7 8 9]]"
+
     def test_str_slice(self):
         from numpy import array, zeros
         a = array(range(5), float)
@@ -108,6 +114,9 @@
         a = zeros(2002)
         b = a[::2]
         assert str(b) == "[0.0 0.0 0.0 ..., 0.0 0.0 0.0]"
+        a = array((range(5),range(5,10)), dtype="int16")
+        b=a[1,2:]
+        assert str(b) == "[7 8 9]"
 
     def test_getitem(self):
         from numpy import array

From noreply at buildbot.pypy.org  Mon Oct 31 02:02:02 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Mon, 31 Oct 2011 02:02:02 +0100 (CET)
Subject: [pypy-commit] pypy default: translation fix
Message-ID: <20111031010202.7E8C5820B3@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r48616:b994f7ec222e
Date: 2011-10-30 21:01 -0400
http://bitbucket.org/pypy/pypy/changeset/b994f7ec222e/

Log:	translation fix

diff --git a/pypy/rpython/lltypesystem/rpbc.py b/pypy/rpython/lltypesystem/rpbc.py
--- a/pypy/rpython/lltypesystem/rpbc.py
+++ b/pypy/rpython/lltypesystem/rpbc.py
@@ -116,7 +116,7 @@
             fields.append((row.attrname, row.fntype))
         kwds = {'hints': {'immutable': True}}
         return Ptr(Struct('specfunc', *fields, **kwds))
-        
+
     def create_specfunc(self):
         return malloc(self.lowleveltype.TO, immortal=True)
 
@@ -316,7 +316,7 @@
         if l == range(len(r_from.descriptions)):
             r = None
         else:
-            r = inputconst(Ptr(Array(Char)), t)
+            r = inputconst(Ptr(Array(Char, hints={'nolength': True})), t)
         r_from._conversion_tables[r_to] = r
         return r
 
@@ -404,12 +404,12 @@
 
 # ____________________________________________________________
 
-##def rtype_call_memo(hop): 
+##def rtype_call_memo(hop):
 ##    memo_table = hop.args_v[0].value
 ##    if memo_table.s_result.is_constant():
 ##        return hop.inputconst(hop.r_result, memo_table.s_result.const)
-##    fieldname = memo_table.fieldname 
-##    assert hop.nb_args == 2, "XXX"  
+##    fieldname = memo_table.fieldname
+##    assert hop.nb_args == 2, "XXX"
 
 ##    r_pbc = hop.args_r[1]
 ##    assert isinstance(r_pbc, (MultipleFrozenPBCRepr, ClassesPBCRepr))

From noreply at buildbot.pypy.org  Mon Oct 31 09:22:54 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 31 Oct 2011 09:22:54 +0100 (CET)
Subject: [pypy-commit] pypy default: Re-checkin bfea8fbdeed4,
 with the fix: make_ge(0), not make_gt(0)...
Message-ID: <20111031082254.5ABDA820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48617:d58637fb13db
Date: 2011-10-31 08:12 +0100
http://bitbucket.org/pypy/pypy/changeset/d58637fb13db/

Log:	Re-checkin bfea8fbdeed4, with the fix: make_ge(0), not make_gt(0)...

diff --git a/pypy/jit/metainterp/optimizeopt/intbounds.py b/pypy/jit/metainterp/optimizeopt/intbounds.py
--- a/pypy/jit/metainterp/optimizeopt/intbounds.py
+++ b/pypy/jit/metainterp/optimizeopt/intbounds.py
@@ -5,6 +5,7 @@
     IntUpperBound)
 from pypy.jit.metainterp.optimizeopt.util import make_dispatcher_method
 from pypy.jit.metainterp.resoperation import rop
+from pypy.rlib.rarithmetic import LONG_BIT
 
 
 class OptIntBounds(Optimization):
@@ -126,14 +127,27 @@
         r.intbound.intersect(v1.intbound.div_bound(v2.intbound))
 
     def optimize_INT_MOD(self, op):
+        v1 = self.getvalue(op.getarg(0))
+        v2 = self.getvalue(op.getarg(1))
+        known_nonneg = (v1.intbound.known_ge(IntBound(0, 0)) and 
+                        v2.intbound.known_ge(IntBound(0, 0)))
+        if known_nonneg and v2.is_constant():
+            val = v2.box.getint()
+            if (val & (val-1)) == 0:
+                # nonneg % power-of-two ==> nonneg & (power-of-two - 1)
+                arg1 = op.getarg(0)
+                arg2 = ConstInt(val-1)
+                op = op.copy_and_change(rop.INT_AND, args=[arg1, arg2])
         self.emit_operation(op)
-        v2 = self.getvalue(op.getarg(1))
         if v2.is_constant():
             val = v2.box.getint()
             r = self.getvalue(op.result)
-            if val < 0:
+            if val < 0:     # what if val == -sys.maxint-1?
                 val = -val
-            r.intbound.make_gt(IntBound(-val, -val))
+            if known_nonneg:
+                r.intbound.make_ge(IntBound(0, 0))
+            else:
+                r.intbound.make_gt(IntBound(-val, -val))
             r.intbound.make_lt(IntBound(val, val))
 
     def optimize_INT_LSHIFT(self, op):
@@ -153,9 +167,14 @@
     def optimize_INT_RSHIFT(self, op):
         v1 = self.getvalue(op.getarg(0))
         v2 = self.getvalue(op.getarg(1))
-        self.emit_operation(op)
-        r = self.getvalue(op.result)
-        r.intbound.intersect(v1.intbound.rshift_bound(v2.intbound))
+        b = v1.intbound.rshift_bound(v2.intbound)
+        if b.has_lower and b.has_upper and b.lower == b.upper:
+            # constant result (likely 0, for rshifts that kill all bits)
+            self.make_constant_int(op.result, b.lower)
+        else:
+            self.emit_operation(op)
+            r = self.getvalue(op.result)
+            r.intbound.intersect(b)
 
     def optimize_INT_ADD_OVF(self, op):
         v1 = self.getvalue(op.getarg(0))
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -9,6 +9,7 @@
 from pypy.jit.metainterp.history import AbstractDescr, ConstInt, BoxInt
 from pypy.jit.metainterp import executor, compile, resume, history
 from pypy.jit.metainterp.resoperation import rop, opname, ResOperation
+from pypy.rlib.rarithmetic import LONG_BIT
 
 
 def test_store_final_boxes_in_guard():
@@ -4714,11 +4715,11 @@
         i5 = int_ge(i0, 0)
         guard_true(i5) []
         i1 = int_mod(i0, 42)
-        i2 = int_rshift(i1, 63)
+        i2 = int_rshift(i1, %d)
         i3 = int_and(42, i2)
         i4 = int_add(i1, i3)
         finish(i4)
-        """
+        """ % (LONG_BIT-1)
         expected = """
         [i0]
         i5 = int_ge(i0, 0)
@@ -4726,21 +4727,41 @@
         i1 = int_mod(i0, 42)
         finish(i1)
         """
-        py.test.skip("in-progress")
         self.optimize_loop(ops, expected)
 
-        # Also, 'n % power-of-two' can be turned into int_and(),
-        # but that's a bit harder to detect here because it turns into
-        # several operations, and of course it is wrong to just turn
+        # 'n % power-of-two' can be turned into int_and(); at least that's
+        # easy to do now if n is known to be non-negative.
+        ops = """
+        [i0]
+        i5 = int_ge(i0, 0)
+        guard_true(i5) []
+        i1 = int_mod(i0, 8)
+        i2 = int_rshift(i1, %d)
+        i3 = int_and(42, i2)
+        i4 = int_add(i1, i3)
+        finish(i4)
+        """ % (LONG_BIT-1)
+        expected = """
+        [i0]
+        i5 = int_ge(i0, 0)
+        guard_true(i5) []
+        i1 = int_and(i0, 7)
+        finish(i1)
+        """
+        self.optimize_loop(ops, expected)
+
+        # Of course any 'maybe-negative % power-of-two' can be turned into
+        # int_and(), but that's a bit harder to detect here because it turns
+        # into several operations, and of course it is wrong to just turn
         # int_mod(i0, 16) into int_and(i0, 15).
         ops = """
         [i0]
         i1 = int_mod(i0, 16)
-        i2 = int_rshift(i1, 63)
+        i2 = int_rshift(i1, %d)
         i3 = int_and(16, i2)
         i4 = int_add(i1, i3)
         finish(i4)
-        """
+        """ % (LONG_BIT-1)
         expected = """
         [i0]
         i4 = int_and(i0, 15)

From noreply at buildbot.pypy.org  Mon Oct 31 09:22:55 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 31 Oct 2011 09:22:55 +0100 (CET)
Subject: [pypy-commit] pypy default: test and fix
Message-ID: <20111031082255.88DD2820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48618:315c9147dfac
Date: 2011-10-31 09:22 +0100
http://bitbucket.org/pypy/pypy/changeset/315c9147dfac/

Log:	test and fix

diff --git a/pypy/jit/metainterp/optimizeopt/intbounds.py b/pypy/jit/metainterp/optimizeopt/intbounds.py
--- a/pypy/jit/metainterp/optimizeopt/intbounds.py
+++ b/pypy/jit/metainterp/optimizeopt/intbounds.py
@@ -1,3 +1,4 @@
+import sys
 from pypy.jit.metainterp.optimizeopt.optimizer import Optimization, CONST_1, CONST_0, \
                                                   MODE_ARRAY, MODE_STR, MODE_UNICODE
 from pypy.jit.metainterp.history import ConstInt
@@ -142,7 +143,9 @@
         if v2.is_constant():
             val = v2.box.getint()
             r = self.getvalue(op.result)
-            if val < 0:     # what if val == -sys.maxint-1?
+            if val < 0:
+                if val == -sys.maxint-1:
+                    return     # give up
                 val = -val
             if known_nonneg:
                 r.intbound.make_ge(IntBound(0, 0))
diff --git a/pypy/jit/metainterp/optimizeopt/intutils.py b/pypy/jit/metainterp/optimizeopt/intutils.py
--- a/pypy/jit/metainterp/optimizeopt/intutils.py
+++ b/pypy/jit/metainterp/optimizeopt/intutils.py
@@ -1,4 +1,5 @@
 from pypy.rlib.rarithmetic import ovfcheck, ovfcheck_lshift, LONG_BIT
+from pypy.rlib.objectmodel import we_are_translated
 from pypy.jit.metainterp.resoperation import rop, ResOperation
 from pypy.jit.metainterp.history import BoxInt, ConstInt
 import sys
@@ -13,6 +14,10 @@
         self.has_lower = True
         self.upper = upper
         self.lower = lower
+        # check for unexpected overflows:
+        if not we_are_translated():
+            assert type(upper) is not long
+            assert type(lower) is not long
 
     # Returns True if the bound was updated
     def make_le(self, other):
diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -4770,6 +4770,16 @@
         py.test.skip("harder")
         self.optimize_loop(ops, expected)
 
+    def test_intmod_bounds_bug1(self):
+        ops = """
+        [i0]
+        i1 = int_mod(i0, %d)
+        i2 = int_eq(i1, 0)
+        guard_false(i2) []
+        finish()
+        """ % (-(1<<(LONG_BIT-1)),)
+        self.optimize_loop(ops, ops)
+
     def test_bounded_lazy_setfield(self):
         ops = """
         [p0, i0]

From noreply at buildbot.pypy.org  Mon Oct 31 09:22:56 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 31 Oct 2011 09:22:56 +0100 (CET)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20111031082256.BFFBA820B3@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48619:6700af7e083b
Date: 2011-10-31 09:22 +0100
http://bitbucket.org/pypy/pypy/changeset/6700af7e083b/

Log:	merge heads

diff --git a/pypy/rpython/lltypesystem/rpbc.py b/pypy/rpython/lltypesystem/rpbc.py
--- a/pypy/rpython/lltypesystem/rpbc.py
+++ b/pypy/rpython/lltypesystem/rpbc.py
@@ -116,7 +116,7 @@
             fields.append((row.attrname, row.fntype))
         kwds = {'hints': {'immutable': True}}
         return Ptr(Struct('specfunc', *fields, **kwds))
-        
+
     def create_specfunc(self):
         return malloc(self.lowleveltype.TO, immortal=True)
 
@@ -316,7 +316,7 @@
         if l == range(len(r_from.descriptions)):
             r = None
         else:
-            r = inputconst(Ptr(Array(Char)), t)
+            r = inputconst(Ptr(Array(Char, hints={'nolength': True})), t)
         r_from._conversion_tables[r_to] = r
         return r
 
@@ -404,12 +404,12 @@
 
 # ____________________________________________________________
 
-##def rtype_call_memo(hop): 
+##def rtype_call_memo(hop):
 ##    memo_table = hop.args_v[0].value
 ##    if memo_table.s_result.is_constant():
 ##        return hop.inputconst(hop.r_result, memo_table.s_result.const)
-##    fieldname = memo_table.fieldname 
-##    assert hop.nb_args == 2, "XXX"  
+##    fieldname = memo_table.fieldname
+##    assert hop.nb_args == 2, "XXX"
 
 ##    r_pbc = hop.args_r[1]
 ##    assert isinstance(r_pbc, (MultipleFrozenPBCRepr, ClassesPBCRepr))

From noreply at buildbot.pypy.org  Mon Oct 31 17:14:56 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 31 Oct 2011 17:14:56 +0100 (CET)
Subject: [pypy-commit] pypy default: Hack away lightweight-finalizer support
 from the semispace GC.
Message-ID: <20111031161456.88D2811B2E69@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48620:fee2997d53e2
Date: 2011-10-31 16:14 +0000
http://bitbucket.org/pypy/pypy/changeset/fee2997d53e2/

Log:	Hack away lightweight-finalizer support from the semispace GC.
	Translated pypy's using the hybrid GC failed so far, and no longer
	fail now.

diff --git a/pypy/rpython/memory/gc/semispace.py b/pypy/rpython/memory/gc/semispace.py
--- a/pypy/rpython/memory/gc/semispace.py
+++ b/pypy/rpython/memory/gc/semispace.py
@@ -105,9 +105,10 @@
         llarena.arena_reserve(result, totalsize)
         self.init_gc_object(result, typeid16)
         self.free = result + totalsize
-        if is_finalizer_light:
-            self.objects_with_light_finalizers.append(result + size_gc_header)
-        elif has_finalizer:
+        #if is_finalizer_light:
+        #    self.objects_with_light_finalizers.append(result + size_gc_header)
+        #else:
+        if has_finalizer:
             self.objects_with_finalizers.append(result + size_gc_header)
         if contains_weakptr:
             self.objects_with_weakrefs.append(result + size_gc_header)

From noreply at buildbot.pypy.org  Mon Oct 31 17:24:48 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 31 Oct 2011 17:24:48 +0100 (CET)
Subject: [pypy-commit] pypy default: Keep the end-of-line characters,
 otherwise it's all printed on
Message-ID: <20111031162448.760C511B2E69@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48621:7d298f1a3bf6
Date: 2011-10-31 17:24 +0100
http://bitbucket.org/pypy/pypy/changeset/7d298f1a3bf6/

Log:	Keep the end-of-line characters, otherwise it's all printed on one
	line. Fixes c041e29173a9. Restore a comment that was lost from
	that checkin, too.

diff --git a/pypy/jit/backend/x86/tool/viewcode.py b/pypy/jit/backend/x86/tool/viewcode.py
--- a/pypy/jit/backend/x86/tool/viewcode.py
+++ b/pypy/jit/backend/x86/tool/viewcode.py
@@ -58,7 +58,7 @@
     assert not p.returncode, ('Encountered an error running objdump: %s' %
                               stderr)
     # drop some objdump cruft
-    lines = stdout.splitlines()[6:]
+    lines = stdout.splitlines(True)[6:]     # drop some objdump cruft
     return format_code_dump_with_labels(originaddr, lines, label_list)
 
 def format_code_dump_with_labels(originaddr, lines, label_list):
@@ -97,7 +97,7 @@
     stdout, stderr = p.communicate()
     assert not p.returncode, ('Encountered an error running nm: %s' %
                               stderr)
-    for line in stdout.splitlines():
+    for line in stdout.splitlines(True):
         match = re_symbolentry.match(line)
         if match:
             addr = long(match.group(1), 16)

From noreply at buildbot.pypy.org  Mon Oct 31 17:24:49 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 31 Oct 2011 17:24:49 +0100 (CET)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20111031162449.9E45011B2E69@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48622:76c3245ed214
Date: 2011-10-31 17:24 +0100
http://bitbucket.org/pypy/pypy/changeset/76c3245ed214/

Log:	merge heads

diff --git a/pypy/rpython/memory/gc/semispace.py b/pypy/rpython/memory/gc/semispace.py
--- a/pypy/rpython/memory/gc/semispace.py
+++ b/pypy/rpython/memory/gc/semispace.py
@@ -105,9 +105,10 @@
         llarena.arena_reserve(result, totalsize)
         self.init_gc_object(result, typeid16)
         self.free = result + totalsize
-        if is_finalizer_light:
-            self.objects_with_light_finalizers.append(result + size_gc_header)
-        elif has_finalizer:
+        #if is_finalizer_light:
+        #    self.objects_with_light_finalizers.append(result + size_gc_header)
+        #else:
+        if has_finalizer:
             self.objects_with_finalizers.append(result + size_gc_header)
         if contains_weakptr:
             self.objects_with_weakrefs.append(result + size_gc_header)

From noreply at buildbot.pypy.org  Mon Oct 31 17:40:51 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 31 Oct 2011 17:40:51 +0100 (CET)
Subject: [pypy-commit] pypy default: Test and fix.
Message-ID: <20111031164051.4DF2211B2E69@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48623:21b6ffe463ac
Date: 2011-10-31 17:38 +0100
http://bitbucket.org/pypy/pypy/changeset/21b6ffe463ac/

Log:	Test and fix.

diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py
--- a/pypy/jit/backend/x86/assembler.py
+++ b/pypy/jit/backend/x86/assembler.py
@@ -1605,6 +1605,7 @@
             # XXX should not use IMUL in most cases
             assert isinstance(temp_loc, RegLoc)
             assert isinstance(index_loc, RegLoc)
+            assert not temp_loc.is_xmm
             self.mc.IMUL_rri(temp_loc.value, index_loc.value,
                              itemsize_loc.value)
         assert isinstance(ofs_loc, ImmedLoc)
@@ -1612,8 +1613,8 @@
 
     def genop_getinteriorfield_gc(self, op, arglocs, resloc):
         (base_loc, ofs_loc, itemsize_loc, fieldsize_loc,
-            index_loc, sign_loc) = arglocs
-        src_addr = self._get_interiorfield_addr(resloc, index_loc,
+            index_loc, temp_loc, sign_loc) = arglocs
+        src_addr = self._get_interiorfield_addr(temp_loc, index_loc,
                                                 itemsize_loc, base_loc,
                                                 ofs_loc)
         self.load_from_mem(resloc, src_addr, fieldsize_loc, sign_loc)
diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py
--- a/pypy/jit/backend/x86/regalloc.py
+++ b/pypy/jit/backend/x86/regalloc.py
@@ -1143,9 +1143,20 @@
         # 'index' but must be in a different register than 'base'.
         self.rm.possibly_free_var(op.getarg(1))
         result_loc = self.force_allocate_reg(op.result, [op.getarg(0)])
+        assert isinstance(result_loc, RegLoc)
+        # two cases: 1) if result_loc is a normal register, use it as temp_loc
+        if not result_loc.is_xmm:
+            temp_loc = result_loc
+        else:
+            # 2) if result_loc is an xmm register, we (likely) need another
+            # temp_loc that is a normal register.  It can be in the same
+            # register as 'index' but not 'base'.
+            tempvar = TempBox()
+            temp_loc = self.rm.force_allocate_reg(tempvar, [op.getarg(0)])
+            self.rm.possibly_free_var(tempvar)
         self.rm.possibly_free_var(op.getarg(0))
         self.Perform(op, [base_loc, ofs, itemsize, fieldsize,
-                          index_loc, sign_loc], result_loc)
+                          index_loc, temp_loc, sign_loc], result_loc)
 
     def consider_int_is_true(self, op, guard_op):
         # doesn't need arg to be in a register

From noreply at buildbot.pypy.org  Mon Oct 31 17:46:59 2011
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Mon, 31 Oct 2011 17:46:59 +0100 (CET)
Subject: [pypy-commit] pypy default: keep a dict virtual through `del
	d['key']`
Message-ID: <20111031164659.E70FD11B2E69@wyvern.cs.uni-duesseldorf.de>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r48624:43d84b226dc4
Date: 2011-10-31 12:46 -0400
http://bitbucket.org/pypy/pypy/changeset/43d84b226dc4/

Log:	keep a dict virtual through `del d['key']`

diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -3513,7 +3513,9 @@
         def f(n):
             while n > 0:
                 myjitdriver.jit_merge_point(n=n)
-                n = g({"key": n})
+                x = {"key": n}
+                n = g(x)
+                del x["key"]
             return n
 
         res = self.meta_interp(f, [10])
diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py
--- a/pypy/rpython/lltypesystem/rdict.py
+++ b/pypy/rpython/lltypesystem/rdict.py
@@ -492,8 +492,8 @@
     _ll_dict_del(d, i)
 
 # XXX: Move the size checking and resize into a single call which is opauqe to
-# the JIT to avoid extra branches.
- at jit.dont_look_inside
+# the JIT when the dict isn't virtual, to avoid extra branches.
+ at jit.look_inside_iff(lambda d, i: jit.isvirtual(d) and jit.isconstant(i))
 def _ll_dict_del(d, i):
     d.entries.mark_deleted(i)
     d.num_items -= 1

From noreply at buildbot.pypy.org  Mon Oct 31 17:49:28 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 31 Oct 2011 17:49:28 +0100 (CET)
Subject: [pypy-commit] pypy default: Generalize passing of translation
	options.
Message-ID: <20111031164928.3863B11B2E69@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48625:9241337e06ff
Date: 2011-10-31 17:45 +0100
http://bitbucket.org/pypy/pypy/changeset/9241337e06ff/

Log:	Generalize passing of translation options.

diff --git a/pypy/jit/codewriter/support.py b/pypy/jit/codewriter/support.py
--- a/pypy/jit/codewriter/support.py
+++ b/pypy/jit/codewriter/support.py
@@ -37,10 +37,11 @@
     return a.typeannotation(t)
 
 def annotate(func, values, inline=None, backendoptimize=True,
-             type_system="lltype"):
+             type_system="lltype", translationoptions={}):
     # build the normal ll graphs for ll_function
     t = TranslationContext()
-    t.config.translation.withsmallfuncsets = 3
+    for key, value in translationoptions.items():
+        setattr(t.config.translation, key, value)
     annpolicy = AnnotatorPolicy()
     annpolicy.allow_someobjects = False
     a = t.buildannotator(policy=annpolicy)
diff --git a/pypy/jit/metainterp/test/support.py b/pypy/jit/metainterp/test/support.py
--- a/pypy/jit/metainterp/test/support.py
+++ b/pypy/jit/metainterp/test/support.py
@@ -12,7 +12,7 @@
 from pypy.rlib.rfloat import isnan
 
 def _get_jitcodes(testself, CPUClass, func, values, type_system,
-                  supports_longlong=False, **kwds):
+                  supports_longlong=False, translationoptions={}, **kwds):
     from pypy.jit.codewriter import support
 
     class FakeJitCell(object):
@@ -42,7 +42,8 @@
         enable_opts = ALL_OPTS_DICT
 
     func._jit_unroll_safe_ = True
-    rtyper = support.annotate(func, values, type_system=type_system)
+    rtyper = support.annotate(func, values, type_system=type_system,
+                              translationoptions=translationoptions)
     graphs = rtyper.annotator.translator.graphs
     testself.all_graphs = graphs
     result_kind = history.getkind(graphs[0].getreturnvar().concretetype)[0]
diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -3585,7 +3585,8 @@
             n = h(n, f8)
             return n
         assert f(5) == 41
-        self.interp_operations(f, [5])
+        translationoptions = {'withsmallfuncsets': 3}
+        self.interp_operations(f, [5], translationoptions=translationoptions)
 
 
 class TestLLtype(BaseLLtypeTests, LLJitMixin):

From noreply at buildbot.pypy.org  Mon Oct 31 17:49:29 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 31 Oct 2011 17:49:29 +0100 (CET)
Subject: [pypy-commit] pypy default: Fix and refactor passing translation
	options.
Message-ID: <20111031164929.6E24611B2E69@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48626:6b10f883bb6a
Date: 2011-10-31 17:48 +0100
http://bitbucket.org/pypy/pypy/changeset/6b10f883bb6a/

Log:	Fix and refactor passing translation options. Fixes test_rerased.

diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -3641,7 +3641,9 @@
                     o = o.dec()
                 pc += 1
             return pc
-        res = self.meta_interp(main, [False, 100, True], taggedpointers=True)
+        topt = {'taggedpointers': True}
+        res = self.meta_interp(main, [False, 100, True],
+                               translationoptions=topt)
 
     def test_rerased(self):
         eraseX, uneraseX = rerased.new_erasing_pair("X")
@@ -3666,10 +3668,11 @@
             else:
                 return rerased.unerase_int(e)
         #
-        x = self.interp_operations(f, [-128, 0], taggedpointers=True)
+        topt = {'taggedpointers': True}
+        x = self.interp_operations(f, [-128, 0], translationoptions=topt)
         assert x == -128
         bigint = sys.maxint//2 + 1
-        x = self.interp_operations(f, [bigint, 0], taggedpointers=True)
+        x = self.interp_operations(f, [bigint, 0], translationoptions=topt)
         assert x == -42
-        x = self.interp_operations(f, [1000, 1], taggedpointers=True)
+        x = self.interp_operations(f, [1000, 1], translationoptions=topt)
         assert x == 999
diff --git a/pypy/jit/metainterp/warmspot.py b/pypy/jit/metainterp/warmspot.py
--- a/pypy/jit/metainterp/warmspot.py
+++ b/pypy/jit/metainterp/warmspot.py
@@ -48,13 +48,13 @@
     translator.warmrunnerdesc = warmrunnerdesc    # for later debugging
 
 def ll_meta_interp(function, args, backendopt=False, type_system='lltype',
-                   listcomp=False, **kwds):
+                   listcomp=False, translationoptions={}, **kwds):
     if listcomp:
         extraconfigopts = {'translation.list_comprehension_operations': True}
     else:
         extraconfigopts = {}
-    if kwds.pop("taggedpointers", False):
-        extraconfigopts["translation.taggedpointers"] = True
+    for key, value in translationoptions.items():
+        extraconfigopts['translation.' + key] = value
     interp, graph = get_interpreter(function, args,
                                     backendopt=False,  # will be done below
                                     type_system=type_system,

From noreply at buildbot.pypy.org  Mon Oct 31 17:49:30 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 31 Oct 2011 17:49:30 +0100 (CET)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20111031164930.9C03111B2E69@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48627:55e0646208ca
Date: 2011-10-31 17:49 +0100
http://bitbucket.org/pypy/pypy/changeset/55e0646208ca/

Log:	merge heads

diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py
--- a/pypy/jit/metainterp/test/test_ajit.py
+++ b/pypy/jit/metainterp/test/test_ajit.py
@@ -3513,7 +3513,9 @@
         def f(n):
             while n > 0:
                 myjitdriver.jit_merge_point(n=n)
-                n = g({"key": n})
+                x = {"key": n}
+                n = g(x)
+                del x["key"]
             return n
 
         res = self.meta_interp(f, [10])
diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py
--- a/pypy/rpython/lltypesystem/rdict.py
+++ b/pypy/rpython/lltypesystem/rdict.py
@@ -492,8 +492,8 @@
     _ll_dict_del(d, i)
 
 # XXX: Move the size checking and resize into a single call which is opauqe to
-# the JIT to avoid extra branches.
- at jit.dont_look_inside
+# the JIT when the dict isn't virtual, to avoid extra branches.
+ at jit.look_inside_iff(lambda d, i: jit.isvirtual(d) and jit.isconstant(i))
 def _ll_dict_del(d, i):
     d.entries.mark_deleted(i)
     d.num_items -= 1

From noreply at buildbot.pypy.org  Mon Oct 31 17:57:24 2011
From: noreply at buildbot.pypy.org (edelsohn)
Date: Mon, 31 Oct 2011 17:57:24 +0100 (CET)
Subject: [pypy-commit] pypy ppc-jit-backend: Add PPC64 instructions to
	gen_emit_cmp_op
Message-ID: <20111031165724.99A1A11B2E69@wyvern.cs.uni-duesseldorf.de>

Author: edelsohn
Branch: ppc-jit-backend
Changeset: r48628:450839e55d9a
Date: 2011-10-31 12:57 -0400
http://bitbucket.org/pypy/pypy/changeset/450839e55d9a/

Log:	Add PPC64 instructions to gen_emit_cmp_op

diff --git a/pypy/jit/backend/ppc/ppcgen/helper/assembler.py b/pypy/jit/backend/ppc/ppcgen/helper/assembler.py
--- a/pypy/jit/backend/ppc/ppcgen/helper/assembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/helper/assembler.py
@@ -1,6 +1,6 @@
 import pypy.jit.backend.ppc.ppcgen.condition as c
 from pypy.rlib.rarithmetic import r_uint, r_longlong, intmask
-from pypy.jit.backend.ppc.ppcgen.arch import MAX_REG_PARAMS
+from pypy.jit.backend.ppc.ppcgen.arch import MAX_REG_PARAMS, IS_PPC_32
 from pypy.jit.metainterp.history import FLOAT
 
 def gen_emit_cmp_op(condition, signed=True):
@@ -9,9 +9,15 @@
         # do the comparison
         if signed:
             if l1.is_imm():
-                self.mc.cmpwi(0, l0.value, l1.value)
+                if IS_PPC_32:
+                    self.mc.cmpwi(0, l0.value, l1.value)
+                else:
+                    self.mc.cmpdi(0, l0.value, l1.value)
             else:
-                self.mc.cmpw(0, l0.value, l1.value)
+                if IS_PPC_32:
+                    self.mc.cmpw(0, l0.value, l1.value)
+                else:
+                    self.mc.cmpd(0, l0.value, l1.value)
 
             # After the comparison, place the result
             # in the first bit of the CR
@@ -32,9 +38,15 @@
 
         else:
             if l1.is_imm():
-                self.mc.cmpli(0, l0.value, l1.value)
+                if IS_PPC_32:
+                    self.mc.cmplwi(0, l0.value, l1.value)
+                else:
+                    self.mc.cmpldi(0, l0.value, l1.value)
             else:
-                self.mc.cmpl(0, l0.value, l1.value)
+                if IS_PPC_32:
+                    self.mc.cmplw(0, l0.value, l1.value)
+                else:
+                    self.mc.cmpld(0, l0.value, l1.value)
 
             if condition == c.U_LT:
                 self.mc.cror(0, 0, 0)

From noreply at buildbot.pypy.org  Mon Oct 31 18:06:08 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 31 Oct 2011 18:06:08 +0100 (CET)
Subject: [pypy-commit] pypy default: Also disable lightweight-finalizers in
	the minimark GC.
Message-ID: <20111031170608.AA46E11B2E69@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48629:8d0125178e78
Date: 2011-10-31 18:05 +0100
http://bitbucket.org/pypy/pypy/changeset/8d0125178e78/

Log:	Also disable lightweight-finalizers in the minimark GC. One test in
	test_newgc, whose purpose is precisely to check that we can allocate
	tons of stuff temporarily without getting MemoryError, fails.

diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py
--- a/pypy/rpython/memory/gc/minimark.py
+++ b/pypy/rpython/memory/gc/minimark.py
@@ -468,7 +468,7 @@
         #
         # If the object needs a finalizer, ask for a rawmalloc.
         # The following check should be constant-folded.
-        if needs_finalizer and not is_finalizer_light:
+        if needs_finalizer:  ## and not is_finalizer_light:
             ll_assert(not contains_weakptr,
                      "'needs_finalizer' and 'contains_weakptr' both specified")
             obj = self.external_malloc(typeid, 0, can_make_young=False)

From noreply at buildbot.pypy.org  Mon Oct 31 18:13:30 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 31 Oct 2011 18:13:30 +0100 (CET)
Subject: [pypy-commit] pypy default: Fix for test_string and test_unicode,
	failing with the
Message-ID: <20111031171330.0AF0511B2E69@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r48630:fb26ce1b9d1b
Date: 2011-10-31 17:13 +0000
http://bitbucket.org/pypy/pypy/changeset/fb26ce1b9d1b/

Log:	Fix for test_string and test_unicode, failing with the new tests
	added in c91b4f3c204f.

diff --git a/pypy/translator/jvm/src/pypy/PyPy.java b/pypy/translator/jvm/src/pypy/PyPy.java
--- a/pypy/translator/jvm/src/pypy/PyPy.java
+++ b/pypy/translator/jvm/src/pypy/PyPy.java
@@ -755,7 +755,7 @@
         int end = str.length();
 
         if (left) {
-            while (start <= str.length() && str.charAt(start) == ch) start++;
+            while (start < str.length() && str.charAt(start) == ch) start++;
         }
 
         if (right) {

From noreply at buildbot.pypy.org  Mon Oct 31 19:10:25 2011
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 31 Oct 2011 19:10:25 +0100 (CET)
Subject: [pypy-commit] pypy stm: - various small improvements
Message-ID: <20111031181025.804AC11B2E69@wyvern.cs.uni-duesseldorf.de>

Author: Armin Rigo <arigo at tunes.org>
Branch: stm
Changeset: r48631:db149dd0eef2
Date: 2011-10-31 19:10 +0100
http://bitbucket.org/pypy/pypy/changeset/db149dd0eef2/

Log:	- various small improvements
	- distinguish gc- from raw-{get,set}fields
	- start making targetdemo really multithreaded

diff --git a/pypy/translator/stm/_rffi_stm.py b/pypy/translator/stm/_rffi_stm.py
--- a/pypy/translator/stm/_rffi_stm.py
+++ b/pypy/translator/stm/_rffi_stm.py
@@ -32,6 +32,11 @@
 commit_transaction = llexternal('stm_commit_transaction', [], lltype.Signed)
 try_inevitable = llexternal('stm_try_inevitable', [], lltype.Void)
 
+descriptor_init_and_being_inevitable_transaction = llexternal(
+    'stm_descriptor_init_and_being_inevitable_transaction', [], lltype.Void)
+commit_transaction_and_descriptor_done = llexternal(
+    'stm_commit_transaction_and_descriptor_done', [], lltype.Void)
+
 stm_read_word = llexternal('stm_read_word', [SignedP], lltype.Signed)
 stm_write_word = llexternal('stm_write_word', [SignedP, lltype.Signed],
                             lltype.Void)
diff --git a/pypy/translator/stm/llstminterp.py b/pypy/translator/stm/llstminterp.py
--- a/pypy/translator/stm/llstminterp.py
+++ b/pypy/translator/stm/llstminterp.py
@@ -69,10 +69,30 @@
         if STRUCT._immutable_field(fieldname):
             # immutable field reads are always allowed
             return LLFrame.op_getfield(self, struct, fieldname)
+        elif STRUCT._gckind == 'raw':
+            # raw getfields are allowed outside a regular transaction
+            self.check_stm_mode(lambda m: m != "regular_transaction")
+            return LLFrame.op_getfield(self, struct, fieldname)
         else:
             # mutable 'getfields' are always forbidden for now
             self.check_stm_mode(lambda m: False)
-            xxx
+            assert 0
+
+    def opstm_setfield(self, struct, fieldname, newvalue):
+        STRUCT = lltype.typeOf(struct).TO
+        if STRUCT._immutable_field(fieldname):
+            # immutable field writes (i.e. initializing writes) should
+            # always be fine, because they should occur into newly malloced
+            # structures
+            LLFrame.op_setfield(self, struct, fieldname, newvalue)
+        elif STRUCT._gckind == 'raw':
+            # raw setfields are allowed outside a regular transaction
+            self.check_stm_mode(lambda m: m != "regular_transaction")
+            LLFrame.op_setfield(self, struct, fieldname, newvalue)
+        else:
+            # mutable 'setfields' are always forbidden for now
+            self.check_stm_mode(lambda m: False)
+            assert 0
 
     def opstm_malloc(self, TYPE, flags):
         # non-GC must not occur in a regular transaction,
diff --git a/pypy/translator/stm/src_stm/et.c b/pypy/translator/stm/src_stm/et.c
--- a/pypy/translator/stm/src_stm/et.c
+++ b/pypy/translator/stm/src_stm/et.c
@@ -580,31 +580,34 @@
   if (PYPY_HAVE_DEBUG_PRINTS) {
     int num_aborts = 0, num_spinloops = 0;
     int i, prevchar;
+    char line[256], *p = line;
+
     for (i=0; i<ABORT_REASONS; i++)
       num_aborts += d->num_aborts[i];
     for (i=0; i<SPINLOOP_REASONS; i++)
       num_spinloops += d->num_spinloops[i];
 
-    fprintf(PYPY_DEBUG_FILE, "thread %lx: %d commits, %d aborts ",
-            d->my_lock_word,
-            d->num_commits,
-            num_aborts);
+    p += sprintf(p, "thread %lx: %d commits, %d aborts ",
+                 d->my_lock_word,
+                 d->num_commits,
+                 num_aborts);
 
     for (i=0; i<ABORT_REASONS; i++)
-      fprintf(PYPY_DEBUG_FILE, "%c%d", i == 0 ? '[' : ',',
-              d->num_aborts[i]);
+      p += sprintf(p, "%c%d", i == 0 ? '[' : ',',
+                   d->num_aborts[i]);
 
     for (i=1; i<SPINLOOP_REASONS; i++)  /* num_spinloops[0] == num_aborts */
-      fprintf(PYPY_DEBUG_FILE, "%c%d", i == 1 ? '|' : ',',
-              d->num_spinloops[i]);
+      p += sprintf(p, "%c%d", i == 1 ? '|' : ',',
+                   d->num_spinloops[i]);
 
 #ifdef COMMIT_OTHER_INEV
     for (i=0; i<OTHERINEV_REASONS; i++)
-      fprintf(PYPY_DEBUG_FILE, "%c%d", i == 0 ? '|' : ',',
-              d->num_otherinev[i]);
+      p += sprintf(p, "%c%d", i == 0 ? '|' : ',',
+                   d->num_otherinev[i]);
 #endif
 
-    fprintf(PYPY_DEBUG_FILE, "]\n");
+    p += sprintf(p, "]\n");
+    fwrite(line, 1, p - line, PYPY_DEBUG_FILE);
   }
   PYPY_DEBUG_STOP("stm-done");
 #endif
@@ -835,6 +838,21 @@
 #endif
 }
 
+void stm_descriptor_init_and_being_inevitable_transaction(void)
+{
+  int was_not_started = (thread_descriptor == NULL);
+  stm_descriptor_init();
+  if (was_not_started)
+    stm_begin_inevitable_transaction();
+}
+
+void stm_commit_transaction_and_descriptor_done(void)
+{
+  if (thread_descriptor->init_counter == 1)
+    stm_commit_transaction();
+  stm_descriptor_done();
+}
+
 // XXX little-endian only!
 void stm_write_partial_word(int fieldsize, char *base, long offset,
                             unsigned long nval)
diff --git a/pypy/translator/stm/src_stm/et.h b/pypy/translator/stm/src_stm/et.h
--- a/pypy/translator/stm/src_stm/et.h
+++ b/pypy/translator/stm/src_stm/et.h
@@ -36,6 +36,8 @@
 void stm_begin_inevitable_transaction(void);
 void stm_abort_and_retry(void);
 void stm_transaction_boundary(jmp_buf* buf);
+void stm_descriptor_init_and_being_inevitable_transaction(void);
+void stm_commit_transaction_and_descriptor_done(void);
 
 /* for testing only: */
 #define STM_begin_transaction()         ; \
diff --git a/pypy/translator/stm/test/targetdemo.py b/pypy/translator/stm/test/targetdemo.py
--- a/pypy/translator/stm/test/targetdemo.py
+++ b/pypy/translator/stm/test/targetdemo.py
@@ -1,19 +1,44 @@
-#from pypy.module.thread import ll_thread
+import time
+from pypy.module.thread import ll_thread
 from pypy.translator.stm import rstm
 
 
+NUM_THREADS = 4
+LENGTH      = 1000
 
 
+class Node:
+    def __init__(self, value):
+        self.value = value
+        self.next = None
+
+
+def add_at_end_of_chained_list(node, value):
+    while node.next:
+        node = node.next
+    newnode = Node(value)
+    node.next = newnode
+
+
+class Global:
+    anchor = Node(-1)
+glob = Global()
+
+def run_me():
+    print "thread starting..."
+    for i in range(LENGTH):
+        add_at_end_of_chained_list(glob.anchor, i)
+        rstm.transaction_boundary()
+    print "thread done."
+
 
 # __________  Entry point  __________
 
 def entry_point(argv):
     print "hello world"
-    rstm.transaction_boundary()
-    i = 100
-    while i > 1:
-        i *= 0.821
-    rstm.transaction_boundary()
+    for i in range(NUM_THREADS):
+        ll_thread.start_new_thread(run_me, ())
+    time.sleep(10)
     return 0
 
 # _____ Define and setup target ___
diff --git a/pypy/translator/stm/test/test_transform.py b/pypy/translator/stm/test/test_transform.py
--- a/pypy/translator/stm/test/test_transform.py
+++ b/pypy/translator/stm/test/test_transform.py
@@ -33,6 +33,17 @@
     res = eval_stm_graph(interp, graph, [p], stm_mode="regular_transaction")
     assert res == 42
 
+def test_setfield():
+    S = lltype.GcStruct('S', ('x', lltype.Signed))
+    p = lltype.malloc(S, immortal=True)
+    p.x = 42
+    def func(p):
+        p.x = 43
+    interp, graph = get_interpreter(func, [p])
+    transform_graph(graph)
+    assert summary(graph) == {'stm_setfield': 1}
+    eval_stm_graph(interp, graph, [p], stm_mode="regular_transaction")
+
 def test_immutable_field():
     S = lltype.GcStruct('S', ('x', lltype.Signed), hints = {'immutable': True})
     p = lltype.malloc(S, immortal=True)
@@ -70,6 +81,31 @@
     eval_stm_func(func, [], final_stm_mode="inevitable_transaction")
 test_unsupported_malloc.dont_track_allocations = True
 
+def test_unsupported_getfield_raw():
+    S = lltype.Struct('S', ('x', lltype.Signed))
+    p = lltype.malloc(S, immortal=True)
+    p.x = 42
+    def func(p):
+        return p.x
+    interp, graph = get_interpreter(func, [p])
+    transform_graph(graph)
+    assert summary(graph) == {'stm_try_inevitable': 1, 'getfield': 1}
+    res = eval_stm_graph(interp, graph, [p], stm_mode="regular_transaction",
+                         final_stm_mode="inevitable_transaction")
+    assert res == 42
+
+def test_unsupported_setfield_raw():
+    S = lltype.Struct('S', ('x', lltype.Signed))
+    p = lltype.malloc(S, immortal=True)
+    p.x = 42
+    def func(p):
+        p.x = 43
+    interp, graph = get_interpreter(func, [p])
+    transform_graph(graph)
+    assert summary(graph) == {'stm_try_inevitable': 1, 'setfield': 1}
+    eval_stm_graph(interp, graph, [p], stm_mode="regular_transaction",
+                   final_stm_mode="inevitable_transaction")
+
 # ____________________________________________________________
 
 class TestTransformSingleThread(StandaloneTests):
diff --git a/pypy/translator/stm/transform.py b/pypy/translator/stm/transform.py
--- a/pypy/translator/stm/transform.py
+++ b/pypy/translator/stm/transform.py
@@ -1,8 +1,8 @@
 from pypy.objspace.flow.model import SpaceOperation, Constant
+from pypy.objspace.flow.model import Block, Link, checkgraph
 from pypy.annotation import model as annmodel
-from pypy.rpython.annlowlevel import MixLevelHelperAnnotator
 from pypy.translator.stm import _rffi_stm
-from pypy.translator.unsimplify import varoftype
+from pypy.translator.unsimplify import varoftype, copyvar
 from pypy.rpython.lltypesystem import lltype
 
 
@@ -33,12 +33,13 @@
         assert not hasattr(self.translator, 'stm_transformation_applied')
         entrypointgraph = entrypointptr._obj.graph
         for graph in self.translator.graphs:
-            if graph is entrypointgraph:
-                continue
             self.seen_transaction_boundary = False
+            self.seen_gc_stack_bottom = False
             self.transform_graph(graph)
             if self.seen_transaction_boundary:
                 self.add_stm_declare_variable(graph)
+            if self.seen_gc_stack_bottom:
+                self.add_descriptor_init_stuff(graph)
         self.add_descriptor_init_stuff(entrypointgraph)
         self.translator.stm_transformation_applied = True
 
@@ -70,24 +71,29 @@
         for block in graph.iterblocks():
             self.transform_block(block)
 
-    def add_descriptor_init_stuff(self, entrypointgraph):
+    def add_descriptor_init_stuff(self, graph):
+        f_init = _rffi_stm.descriptor_init_and_being_inevitable_transaction
+        f_done = _rffi_stm.commit_transaction_and_descriptor_done
+        c_init = Constant(f_init, lltype.typeOf(f_init))
+        c_done = Constant(f_done, lltype.typeOf(f_done))
         #
-        def descriptor_init():
-            _rffi_stm.descriptor_init()
-            _rffi_stm.begin_inevitable_transaction()
-        #def descriptor_done():
-        #    _rffi_stm.commit_transaction()
-        #    _rffi_stm.descriptor_done()
-        #
-        annhelper = MixLevelHelperAnnotator(self.translator.rtyper)
-        c_init = annhelper.constfunc(descriptor_init, [], annmodel.s_None)
-        #c_done = annhelper.constfunc(descriptor_done, [], annmodel.s_None)
-        annhelper.finish()
-        block = entrypointgraph.startblock
+        block = graph.startblock
         v = varoftype(lltype.Void)
         op = SpaceOperation('direct_call', [c_init], v)
         block.operations.insert(0, op)
-        #...add c_done...
+        #
+        v = copyvar(self.translator.annotator, graph.getreturnvar())
+        extrablock = Block([v])
+        v_none = varoftype(lltype.Void)
+        newop = SpaceOperation('direct_call', [c_done], v_none)
+        extrablock.operations = [newop]
+        extrablock.closeblock(Link([v], graph.returnblock))
+        for block in graph.iterblocks():
+            if block is not extrablock:
+                for link in block.exits:
+                    if link.target is graph.returnblock:
+                        link.target = extrablock
+        checkgraph(graph)
 
     def add_stm_declare_variable(self, graph):
         block = graph.startblock
@@ -101,12 +107,22 @@
         STRUCT = op.args[0].concretetype.TO
         if STRUCT._immutable_field(op.args[1].value):
             op1 = op
+        elif STRUCT._gckind == 'raw':
+            turn_inevitable(newoperations, "getfield_raw")
+            op1 = op
         else:
             op1 = SpaceOperation('stm_getfield', op.args, op.result)
         newoperations.append(op1)
 
     def stt_setfield(self, newoperations, op):
-        op1 = SpaceOperation('stm_setfield', op.args, op.result)
+        STRUCT = op.args[0].concretetype.TO
+        if STRUCT._immutable_field(op.args[1].value):
+            op1 = op
+        elif STRUCT._gckind == 'raw':
+            turn_inevitable(newoperations, "setfield_raw")
+            op1 = op
+        else:
+            op1 = SpaceOperation('stm_setfield', op.args, op.result)
         newoperations.append(op1)
 
     def stt_stm_transaction_boundary(self, newoperations, op):
@@ -117,15 +133,22 @@
         flags = op.args[1].value
         return flags['flavor'] == 'gc'
 
+    def stt_gc_stack_bottom(self, newoperations, op):
+        self.seen_gc_stack_bottom = True
+        newoperations.append(op)
+
 
 def transform_graph(graph):
     # for tests: only transforms one graph
     STMTransformer().transform_graph(graph)
 
 
-def turn_inevitable_and_proceed(newoperations, op):
-    c_info = Constant(op.opname, lltype.Void)
+def turn_inevitable(newoperations, info):
+    c_info = Constant(info, lltype.Void)
     op1 = SpaceOperation('stm_try_inevitable', [c_info],
                          varoftype(lltype.Void))
     newoperations.append(op1)
+
+def turn_inevitable_and_proceed(newoperations, op):
+    turn_inevitable(newoperations, op.opname)
     newoperations.append(op)

From noreply at buildbot.pypy.org  Mon Oct 31 22:18:06 2011
From: noreply at buildbot.pypy.org (hager)
Date: Mon, 31 Oct 2011 22:18:06 +0100 (CET)
Subject: [pypy-commit] pypy ppc-jit-backend: Removed duplicate code.
Message-ID: <20111031211806.3BD9111B2E69@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r48632:9bdd48a23099
Date: 2011-10-31 22:12 +0100
http://bitbucket.org/pypy/pypy/changeset/9bdd48a23099/

Log:	Removed duplicate code.

diff --git a/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py b/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py
--- a/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py
@@ -83,27 +83,6 @@
         self.fail_boxes_count = 0
         self.current_clt = None
 
-    def load_imm(self, rD, word):
-        if word <= 32767 and word >= -32768:
-            self.mc.li(rD, word)
-        elif IS_PPC_32 or (word <= 2147483647 and word >= -2147483648):
-            self.mc.lis(rD, hi(word))
-            if word & 0xFFFF != 0:
-                self.mc.ori(rD, rD, lo(word))
-        else:
-            self.mc.lis(rD, highest(word))
-            self.mc.ori(rD, rD, higher(word))
-            self.mc.sldi(rD, rD, 32)
-            self.mc.oris(rD, rD, high(word))
-            self.mc.ori(rD, rD, lo(word))
-
-    def store_reg(self, source_reg, addr):
-        self.load_imm(r.r0.value, addr)
-        if IS_PPC_32:
-            self.mc.stwx(source_reg.value, 0, 0)
-        else:
-            self.mc.stdx(source_reg.value, 0, 0)
-
     def _save_nonvolatiles(self):
         for i, reg in enumerate(NONVOLATILES):
             # save r31 later on

From noreply at buildbot.pypy.org  Mon Oct 31 22:18:07 2011
From: noreply at buildbot.pypy.org (hager)
Date: Mon, 31 Oct 2011 22:18:07 +0100 (CET)
Subject: [pypy-commit] pypy ppc-jit-backend: remove debug code
Message-ID: <20111031211807.6610B11B2E6A@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r48633:3e069b262a2b
Date: 2011-10-31 22:16 +0100
http://bitbucket.org/pypy/pypy/changeset/3e069b262a2b/

Log:	remove debug code

diff --git a/pypy/jit/backend/ppc/ppcgen/opassembler.py b/pypy/jit/backend/ppc/ppcgen/opassembler.py
--- a/pypy/jit/backend/ppc/ppcgen/opassembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/opassembler.py
@@ -457,7 +457,6 @@
         #XXX Hack, Hack, Hack
         if op.result and not we_are_translated() and not isinstance(descr,
                 LoopToken):
-            import pdb; pdb.set_trace()
             #XXX check result type
             loc = regalloc.rm.call_result_location(op.result)
             size = descr.get_result_size(False)
@@ -514,7 +513,6 @@
         # restore the arguments stored on the stack
         if result is not None:
             resloc = regalloc.after_call(result)
-        self.mc.trap()
 
     def write_new_force_index(self):
         # for shadowstack only: get a new, unused force_index number and

From noreply at buildbot.pypy.org  Mon Oct 31 22:18:08 2011
From: noreply at buildbot.pypy.org (hager)
Date: Mon, 31 Oct 2011 22:18:08 +0100 (CET)
Subject: [pypy-commit] pypy ppc-jit-backend: merge
Message-ID: <20111031211808.8E94311B2E6B@wyvern.cs.uni-duesseldorf.de>

Author: hager <sven.hager at uni-duesseldorf.de>
Branch: ppc-jit-backend
Changeset: r48634:8e8436634afe
Date: 2011-10-31 22:17 +0100
http://bitbucket.org/pypy/pypy/changeset/8e8436634afe/

Log:	merge

diff --git a/pypy/jit/backend/ppc/ppcgen/helper/assembler.py b/pypy/jit/backend/ppc/ppcgen/helper/assembler.py
--- a/pypy/jit/backend/ppc/ppcgen/helper/assembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/helper/assembler.py
@@ -1,6 +1,6 @@
 import pypy.jit.backend.ppc.ppcgen.condition as c
 from pypy.rlib.rarithmetic import r_uint, r_longlong, intmask
-from pypy.jit.backend.ppc.ppcgen.arch import MAX_REG_PARAMS
+from pypy.jit.backend.ppc.ppcgen.arch import MAX_REG_PARAMS, IS_PPC_32
 from pypy.jit.metainterp.history import FLOAT
 
 def gen_emit_cmp_op(condition, signed=True):
@@ -9,9 +9,15 @@
         # do the comparison
         if signed:
             if l1.is_imm():
-                self.mc.cmpwi(0, l0.value, l1.value)
+                if IS_PPC_32:
+                    self.mc.cmpwi(0, l0.value, l1.value)
+                else:
+                    self.mc.cmpdi(0, l0.value, l1.value)
             else:
-                self.mc.cmpw(0, l0.value, l1.value)
+                if IS_PPC_32:
+                    self.mc.cmpw(0, l0.value, l1.value)
+                else:
+                    self.mc.cmpd(0, l0.value, l1.value)
 
             # After the comparison, place the result
             # in the first bit of the CR
@@ -32,9 +38,15 @@
 
         else:
             if l1.is_imm():
-                self.mc.cmpli(0, l0.value, l1.value)
+                if IS_PPC_32:
+                    self.mc.cmplwi(0, l0.value, l1.value)
+                else:
+                    self.mc.cmpldi(0, l0.value, l1.value)
             else:
-                self.mc.cmpl(0, l0.value, l1.value)
+                if IS_PPC_32:
+                    self.mc.cmplw(0, l0.value, l1.value)
+                else:
+                    self.mc.cmpld(0, l0.value, l1.value)
 
             if condition == c.U_LT:
                 self.mc.cror(0, 0, 0)

From noreply at buildbot.pypy.org  Mon Oct 31 23:23:48 2011
From: noreply at buildbot.pypy.org (mattip)
Date: Mon, 31 Oct 2011 23:23:48 +0100 (CET)
Subject: [pypy-commit] pypy numpy-multidim: add test_repr for 3-dim array
Message-ID: <20111031222348.A223411B2E69@wyvern.cs.uni-duesseldorf.de>

Author: mattip
Branch: numpy-multidim
Changeset: r48635:6adc66bed4bc
Date: 2011-11-01 00:19 +0200
http://bitbucket.org/pypy/pypy/changeset/6adc66bed4bc/

Log:	add test_repr for 3-dim array

diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py
--- a/pypy/module/micronumpy/interp_numarray.py
+++ b/pypy/module/micronumpy/interp_numarray.py
@@ -671,9 +671,9 @@
         if ndims>2:
             ret += '['
             for i in range(self.shape[0]):
-                ret += NDimSlice(self.parent, self.signature, [(i,0,0,1)], self.shape[1:]).tostr(commai,indent=indent+' ')
+                ret += NDimSlice(self.parent, self.signature, [(i,0,0,1)], self.shape[1:]).tostr(comma,indent=indent+' ')
                 if i+1<self.shape[0]:
-                    ret += ',\n\n'
+                    ret += ',\n\n'+ indent
             ret += ']'
         elif ndims==2:
             ret += '['
diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py
--- a/pypy/module/micronumpy/test/test_numarray.py
+++ b/pypy/module/micronumpy/test/test_numarray.py
@@ -71,6 +71,14 @@
         assert repr(a) == '''array([[0.0, 0.0, 0.0, 0.0],
        [0.0, 0.0, 0.0, 0.0],
        [0.0, 0.0, 0.0, 0.0]])'''
+        a = zeros((2,3,4))
+        assert repr(a) == '''array([[[0.0, 0.0, 0.0, 0.0],
+        [0.0, 0.0, 0.0, 0.0],
+        [0.0, 0.0, 0.0, 0.0]],
+
+       [[0.0, 0.0, 0.0, 0.0],
+        [0.0, 0.0, 0.0, 0.0],
+        [0.0, 0.0, 0.0, 0.0]]])'''
 
     def test_repr_slice(self):
         from numpy import array, zeros
@@ -200,6 +208,10 @@
         a[1:4:2] = 0.
         assert a[1] == 0.
         assert a[3] == 0.
+    def test_scalar(self):
+        from numpy import array
+        a = array(3)
+        assert a[0] == 3 
 
     def test_len(self):
         from numpy import array

From noreply at buildbot.pypy.org  Mon Oct 31 23:52:41 2011
From: noreply at buildbot.pypy.org (edelsohn)
Date: Mon, 31 Oct 2011 23:52:41 +0100 (CET)
Subject: [pypy-commit] pypy ppc-jit-backend: PPC64 _emit_call
Message-ID: <20111031225242.0106D11B2E69@wyvern.cs.uni-duesseldorf.de>

Author: edelsohn
Branch: ppc-jit-backend
Changeset: r48636:d6110e34f0eb
Date: 2011-10-31 18:52 -0400
http://bitbucket.org/pypy/pypy/changeset/d6110e34f0eb/

Log:	PPC64 _emit_call

diff --git a/pypy/jit/backend/ppc/ppcgen/opassembler.py b/pypy/jit/backend/ppc/ppcgen/opassembler.py
--- a/pypy/jit/backend/ppc/ppcgen/opassembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/opassembler.py
@@ -503,7 +503,17 @@
         remap_frame_layout(self, non_float_locs, non_float_regs, r.r0)
 
         #the actual call
-        self.mc.bl_abs(adr)
+        if IS_PPC_32:
+            self.mc.bl_abs(adr)
+        else:
+            self.mc.std(r.r2.value, r.SP.value, 40)
+            self.mc.load_from_addr(r.r0, adr)
+            self.mc.load_from_addr(r.r2, adr+WORD)
+            self.mc.load_from_addr(r.r11, adr+2*WORD)
+            self.mc.mtctr(r.r0.value)
+            self.mc.bctrl()
+            self.mc.ld(r.r2.value, r.SP.value, 40)
+
         self.mark_gc_roots(force_index)
         regalloc.possibly_free_vars(args)
         # readjust the sp in case we passed some args on the stack

From noreply at buildbot.pypy.org  Mon Oct 31 23:58:07 2011
From: noreply at buildbot.pypy.org (edelsohn)
Date: Mon, 31 Oct 2011 23:58:07 +0100 (CET)
Subject: [pypy-commit] pypy ppc-jit-backend: More PPC64 paths
Message-ID: <20111031225807.4C6FA11B2E69@wyvern.cs.uni-duesseldorf.de>

Author: edelsohn
Branch: ppc-jit-backend
Changeset: r48637:11cd624afd49
Date: 2011-10-31 18:57 -0400
http://bitbucket.org/pypy/pypy/changeset/11cd624afd49/

Log:	More PPC64 paths

diff --git a/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py b/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py
--- a/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py
+++ b/pypy/jit/backend/ppc/ppcgen/ppc_assembler.py
@@ -8,7 +8,8 @@
 from pypy.jit.backend.ppc.ppcgen.opassembler import OpAssembler
 from pypy.jit.backend.ppc.ppcgen.symbol_lookup import lookup
 from pypy.jit.backend.ppc.ppcgen.codebuilder import PPCBuilder
-from pypy.jit.backend.ppc.ppcgen.arch import (IS_PPC_32, WORD, NONVOLATILES,
+from pypy.jit.backend.ppc.ppcgen.arch import (IS_PPC_32, IS_PPC_64,
+                                              WORD, NONVOLATILES,
                                               GPR_SAVE_AREA)
 from pypy.jit.backend.ppc.ppcgen.helper.assembler import (gen_emit_cmp_op, 
                                                           encode32, decode32)
@@ -136,6 +137,7 @@
             self.mc.stdu(r.SP.value, r.SP.value, -frame_depth)
             self.mc.mflr(r.r0.value)
             self.mc.std(r.r0.value, r.SP.value, frame_depth + 2 * WORD)
+            self.mc.std(r.SPP.value, r.SP.value, WORD)
         offset = GPR_SAVE_AREA + WORD
         # compute spilling pointer (SPP)
         self.mc.addi(r.SPP.value, r.SP.value, frame_depth - offset)
@@ -294,6 +296,7 @@
         self._save_managed_regs(mc)
         # adjust SP (r1)
         size = WORD * len(r.MANAGED_REGS)
+        # XXX PPC64 needs to push larger stack frame
         mc.addi(r.SP.value, r.SP.value, -size)
         #
         decode_func_addr = llhelper(self.recovery_func_sign,
@@ -301,17 +304,29 @@
         addr = rffi.cast(lltype.Signed, decode_func_addr)
         #
         # load parameters into parameter registers
-        mc.lwz(r.r3.value, r.SPP.value, 0)
+        if IS_PPC_32:
+            mc.lwz(r.r3.value, r.SPP.value, 0)
+        else:
+            mc.ld(r.r3.value, r.SPP.value, 0)
         #mc.mr(r.r3.value, r.r0.value)          # address of state encoding 
         mc.mr(r.r4.value, r.SP.value)          # load stack pointer
         mc.mr(r.r5.value, r.SPP.value)         # load spilling pointer
         #
         # load address of decoding function into r0
-        mc.load_imm(r.r0, addr)
+        if IS_PPC_32:
+            mc.load_imm(r.r0, addr)
+        else:
+            mc.std(r.r2.value, r.SP.value, 40)
+            mc.load_from_addr(r.r0, addr)
+            mc.load_from_addr(r.r2, addr+WORD)
+            mc.load_from_addr(r.r11, addr+2*WORD)
         # ... and branch there
         mc.mtctr(r.r0.value)
         mc.bctrl()
         #
+        if IS_PPC_64:
+            mc.ld(r.r2.value, r.SP.value, 40)
+        #
         mc.addi(r.SP.value, r.SP.value, size)
         # save SPP in r5
         # (assume that r5 has been written to failboxes)
@@ -338,7 +353,7 @@
             if IS_PPC_32:
                 mc.stw(reg.value, r.SP.value, -(len(r.MANAGED_REGS) - i) * WORD)
             else:
-                assert 0, "not implemented yet"
+                mc.std(reg.value, r.SP.value, -(len(r.MANAGED_REGS) - i) * WORD)
 
     def gen_bootstrap_code(self, nonfloatlocs, inputargs):
         for i in range(len(nonfloatlocs)):
@@ -550,7 +565,10 @@
 
         # store addr in force index field
         self.mc.load_imm(r.r0, memaddr)
-        self.mc.stw(r.r0.value, r.SPP.value, 0)
+        if IS_PPC_32:
+            self.mc.stw(r.r0.value, r.SPP.value, 0)
+        else:
+            self.mc.std(r.r0.value, r.SPP.value, 0)
 
         if save_exc:
             path = self._leave_jitted_hook_save_exc
@@ -602,7 +620,10 @@
             elif loc.is_stack():
                 offset = loc.as_key() * WORD - WORD
                 self.mc.load_imm(r.r0.value, value)
-                self.mc.stw(r.r0.value, r.SPP.value, offset)
+                if IS_PPC_32:
+                    self.mc.stw(r.r0.value, r.SPP.value, offset)
+                else:
+                    self.mc.std(r.r0.value, r.SPP.value, offset)
                 return
             assert 0, "not supported location"
         elif prev_loc.is_stack():
@@ -610,13 +631,20 @@
             # move from memory to register
             if loc.is_reg():
                 reg = loc.as_key()
-                self.mc.lwz(reg, r.SPP.value, offset)
+                if IS_PPC_32:
+                    self.mc.lwz(reg, r.SPP.value, offset)
+                else:
+                    self.mc.ld(reg, r.SPP.value, offset)
                 return
             # move in memory
             elif loc.is_stack():
                 target_offset = loc.as_key() * WORD - WORD
-                self.mc.lwz(r.r0.value, r.SPP.value, offset)
-                self.mc.stw(r.r0.value, r.SPP.value, target_offset)
+                if IS_PPC_32:
+                    self.mc.lwz(r.r0.value, r.SPP.value, offset)
+                    self.mc.stw(r.r0.value, r.SPP.value, target_offset)
+                else:
+                    self.mc.ld(r.r0.value, r.SPP.value, offset)
+                    self.mc.std(r.r0.value, r.SPP.value, target_offset)
                 return
             assert 0, "not supported location"
         elif prev_loc.is_reg():
@@ -629,7 +657,10 @@
             # move to memory
             elif loc.is_stack():
                 offset = loc.as_key() * WORD - WORD
-                self.mc.stw(reg, r.SPP.value, offset)
+                if IS_PPC_32:
+                    self.mc.stw(reg, r.SPP.value, offset)
+                else:
+                    self.mc.std(reg, r.SPP.value, offset)
                 return
             assert 0, "not supported location"
         assert 0, "not supported location"